import { MAX_ATTACHMENT_BYTES, MAX_PDF_BYTES } from '../../src' import { ascii, concat, craftPdf, deflate, pdfOf, streamObject, } from './r4-reader-craft' import type { InputSpec, ReaderFunction } from './r4-reader-inputs' /** * Plan §B.1.2 "JS regex semantics reproduced explicitly", one minimal PDF * per variant, plus the boundaries of the scanner's other rules (dictionary * and keyword windows, candidate and occurrence caps, header position, the * shared budget). Each pair differs only where the rule decides: `\s` (VT, * FF, NBSP yes; NUL, NEL no) against `isWhitespace` (NUL yes; VT, NBSP no), * `.` against CR and LF in the string token, the lazy `/EF` removal, the * `break` on a candidate over the remaining budget. The outcomes are what * the TypeScript does, recorded by the O7 driver, never what a PDF parser * would do. */ const J = '{"version":3}' const FLATE_J = deflate(ascii(J)) const EF2 = `2 0 obj\n<< /Type /EmbeddedFile /Length 13 >>\nstream\n${J}\nendstream\nendobj\n` type Build = InputSpec['build'] const rule = (group: string, exercises: ReaderFunction[]) => (slug: string, title: string, build: Build): InputSpec => ({ slug: `semantics.${slug}`, kind: 'pdf', cases: [{ suite: 'semantics', case: `${group} > ${title}`, exercises }], build, }) // --------------------------------------------------------------------------- // `n 0 obj` header: /(\d{1,10})\s+\d{1,10}\s+obj\s*$/ on a 48-byte window const header = rule('object header regex (\\s, 48-byte window)', [ 'findEmbeddedFiles', ]) const namedStream = (head: string, target = 2) => pdfOf( `3 0 obj\n<< /Type /Filespec /F (osteo-report.json) /EF << /F ${target} 0 R >> >>\nendobj\n`, `${head}<< /Type /EmbeddedFile /Length 13 >>\nstream\n${J}\nendstream\nendobj\n`, ) const HEADER: InputSpec[] = [ header('header-vt', 'VT (0x0B) separators', () => namedStream('2\x0B0\x0Bobj\x0B'), ), header('header-ff', 'FF (0x0C) separators', () => namedStream('2\x0C0\x0Cobj\x0C'), ), header('header-nbsp', 'NBSP (0xA0) separators', () => namedStream('2\xA00\xA0obj\xA0'), ), header('header-nel', 'NEL (0x85) between the numbers', () => namedStream('2\x850\x85obj\n'), ), header('header-nul', 'NUL between the numbers', () => namedStream('2\x000 obj\n'), ), header('header-nul-before-dict', 'NUL between obj and <<', () => namedStream('2 0 obj\x00'), ), header( 'header-window-40-spaces', '12 0 obj then 40 spaces: the number fits the window', () => namedStream(`12 0 obj${' '.repeat(40)}`, 12), ), header( 'header-window-41-spaces', '12 0 obj then 41 spaces: the window cuts the 1', () => namedStream(`12 0 obj${' '.repeat(41)}`, 12), ), ] // --------------------------------------------------------------------------- // /Length: /\/Length\s+(\d{1,10})(?!\d)(?!\s+\d+\s+R)/, then isWhitespace before endstream const length = rule('/Length regex and endstream landing', [ 'findEmbeddedFiles', 'readAttachedDocument', ]) /** Three spaces after the data: a direct /Length gives 13, the measured length 16. */ const lengthStream = (dict: string, tail = ' \n') => pdfOf( `2 0 obj\n<< /Type /EmbeddedFile ${dict} >>\nstream\n${J}${tail}endstream\nendobj\n`, ) const LENGTH: InputSpec[] = [ length('length-space', 'space after /Length (control)', () => lengthStream('/Length 13'), ), length('length-vt', 'VT after /Length', () => lengthStream('/Length\x0B13')), length('length-nbsp', 'NBSP after /Length', () => lengthStream('/Length\xA013'), ), length('length-nel', 'NEL after /Length', () => lengthStream('/Length\x8513'), ), length('length-nul', 'NUL after /Length', () => lengthStream('/Length\x0013'), ), length('length-indirect-space', '13 0 R with spaces (indirect)', () => lengthStream('/Length 13 0 R'), ), length('length-indirect-vt', '13 0 R with VT separators', () => lengthStream('/Length 13\x0B0\x0BR'), ), length('length-indirect-nul', '13 0 R with NUL separators', () => lengthStream('/Length 13\x000\x00R'), ), length('length-10-digits', 'ten digits 0000000013', () => lengthStream('/Length 0000000013'), ), length('length-11-digits', 'eleven digits 00000000013', () => lengthStream('/Length 00000000013'), ), length('length-past-end', 'a direct length past the end of the file', () => lengthStream('/Length 99999'), ), length('length-tail-nul', 'NUL between the data and endstream', () => lengthStream('/Length 13', '\x00\n'), ), length('length-tail-vt', 'VT between the data and endstream', () => lengthStream('/Length 13', '\x0B\n'), ), length('measured-lf', 'no /Length, LF before endstream', () => lengthStream('', '\n'), ), length('measured-crlf', 'no /Length, CRLF before endstream', () => lengthStream('', '\r\n'), ), length('measured-lfcr', 'no /Length, LF CR before endstream', () => lengthStream('', '\n\r'), ), length('measured-cr', 'no /Length, CR before endstream', () => lengthStream('', '\r'), ), ] // --------------------------------------------------------------------------- // /Filter: /\/Filter\s*(\[\s*)?\/FlateDecode\b/ const filter = rule('/Filter regex', [ 'findEmbeddedFiles', 'readAttachedDocument', ]) const filterStream = (dict: string, data: Uint8Array = FLATE_J) => craftPdf([ streamObject({ num: 2, dict: ` /Type /EmbeddedFile ${dict} /Length ${data.byteLength} `, data, }), ]) const HEX_J = ascii( `${Buffer.from(J, 'latin1').toString('hex').toUpperCase()}>`, ) const FILTER: InputSpec[] = [ filter('filter-name', '/Filter /FlateDecode (control)', () => filterStream('/Filter /FlateDecode'), ), filter('filter-unspaced', '/Filter/FlateDecode', () => filterStream('/Filter/FlateDecode'), ), filter('filter-array', '/Filter [/FlateDecode]', () => filterStream('/Filter [/FlateDecode]'), ), filter('filter-array-spaced', '/Filter [ /FlateDecode ]', () => filterStream('/Filter [ /FlateDecode ]'), ), filter( 'filter-array-second', '/Filter [/ASCIIHexDecode /FlateDecode] on Flate data', () => filterStream('/Filter [/ASCIIHexDecode /FlateDecode]'), ), filter('filter-asciihex', '/Filter /ASCIIHexDecode on hex data', () => filterStream('/Filter /ASCIIHexDecode', HEX_J), ), filter( 'filter-asciihex-on-json', '/Filter /ASCIIHexDecode on plain JSON', () => filterStream('/Filter /ASCIIHexDecode', ascii(J)), ), filter('filter-vt', 'VT after /Filter', () => filterStream('/Filter\x0B/FlateDecode'), ), filter('filter-nbsp', 'NBSP after /Filter', () => filterStream('/Filter\xA0/FlateDecode'), ), filter('filter-nul', 'NUL after /Filter', () => filterStream('/Filter\x00/FlateDecode'), ), filter('filter-array-nul', 'NUL inside the array', () => filterStream('/Filter [\x00/FlateDecode]'), ), filter('filter-name-continues', '/Filter /FlateDecodeX', () => filterStream('/Filter /FlateDecodeX'), ), ] // --------------------------------------------------------------------------- // Filespec: /EF removal (lazy), /F then /UF string token, /EF << /F n 0 R >> const filespecRule = rule('Filespec /EF and /F regexes, string token', [ 'findEmbeddedFiles', ]) const filespec = (...dicts: string[]) => pdfOf(...dicts.map((d, i) => `${i + 3} 0 obj\n<<${d}>>\nendobj\n`), EF2) const FILESPEC: InputSpec[] = [ filespecRule('filespec-ef-vt', 'VT around /EF and the reference', () => filespec( ' /Type /Filespec /F (osteo-report.json) /EF\x0B<< /F\x0B2\x0B0\x0BR >> ', ), ), filespecRule('filespec-ef-nbsp', 'NBSP around /EF and the reference', () => filespec( ' /Type /Filespec /F (osteo-report.json) /EF\xA0<< /F\xA02\xA00\xA0R >> ', ), ), filespecRule('filespec-ef-nul', 'NUL between /EF and <<', () => filespec(' /Type /Filespec /F (osteo-report.json) /EF\x00<< /F 2 0 R >> '), ), filespecRule('filespec-ref-nul', 'NUL between /F and the reference', () => filespec(' /Type /Filespec /F (osteo-report.json) /EF << /F\x002 0 R >> '), ), filespecRule('filespec-ref-nel', 'NEL between /F and the reference', () => filespec(' /Type /Filespec /F (osteo-report.json) /EF << /F\x852 0 R >> '), ), filespecRule( 'filespec-f-vt', 'VT between /F and its string, /UF fallback present', () => filespec( ' /Type /Filespec /F\x0B(osteo-report.json) /UF (fallback.json) /EF << /F 2 0 R >> ', ), ), filespecRule( 'filespec-f-nul', 'NUL between /F and its string, /UF fallback present', () => filespec( ' /Type /Filespec /F\x00(osteo-report.json) /UF (fallback.json) /EF << /F 2 0 R >> ', ), ), filespecRule('filespec-uf-only', '/UF only', () => filespec(' /Type /Filespec /UF (osteo-report.json) /EF << /F 2 0 R >> '), ), filespecRule( 'filespec-lazy-ef', '/EF holding a nested dictionary: removal stops at the first >>', () => filespec( ' /Type /Filespec /EF << /F 2 0 R /Sub << >> /F (inner.json) >> /UF (outer.json) ', ), ), filespecRule( 'filespec-tight', 'no spaces, /EF closing right before the Filespec >>', () => filespec('/Type/Filespec/F(osteo-report.json)/EF<>'), ), filespecRule( 'filespec-two-for-one-stream', 'two Filespecs for one stream: the first wins', () => filespec( ' /Type /Filespec /F (decoy.json) /EF << /F 2 0 R >> ', ' /Type /Filespec /F (osteo-report.json) /EF << /F 2 0 R >> ', ), ), filespecRule( 'filespec-backslash-cr', 'backslash then CR inside the /F string', () => filespec( ' /Type /Filespec /F (osteo-report\\\r.json) /UF (fallback.json) /EF << /F 2 0 R >> ', ), ), filespecRule( 'filespec-backslash-lf', 'backslash then LF inside the /F string', () => filespec( ' /Type /Filespec /F (osteo-report\\\n.json) /UF (fallback.json) /EF << /F 2 0 R >> ', ), ), filespecRule('filespec-plain-cr', 'a bare CR inside the /F string', () => filespec(' /Type /Filespec /F (osteo-report\r.json) /EF << /F 2 0 R >> '), ), filespecRule( 'filespec-escaped-paren', 'an escaped parenthesis inside the /F string', () => filespec( ' /Type /Filespec /F (osteo-report\\).json) /EF << /F 2 0 R >> ', ), ), filespecRule('filespec-hex-latin1', 'hex /F name without BOM', () => filespec( ` /Type /Filespec /F <${Buffer.from('osteo-report.json').toString('hex')}> /EF << /F 2 0 R >> `, ), ), ] // --------------------------------------------------------------------------- // /Keywords: isDelimiter after the key, string token, 4,096-byte window, 32 occurrences const keywordsRule = rule('/Keywords stamp', [ 'hasArchiveStamp', 'readAttachedDocument', ]) const STAMP = 'faunosteo-archive/1' const keywords = (body: string) => pdfOf(`3 0 obj\n<< ${body} >>\nendobj\n`) const hexWith = (text: string, insert: string) => { const hex = Buffer.from(text, 'latin1').toString('hex') return `<${hex.slice(0, 8)}${insert}${hex.slice(8)}>` } const utf16Literal = (text: string) => `(\xFE\xFF${Array.from(text, (c) => `\x00${c}`).join('')})` /** A literal token of exactly `total` bytes ending with the stamp. */ const literalOfLength = (total: number) => `(${'x'.repeat(total - STAMP.length - 2)}${STAMP})` const KEYWORDS: InputSpec[] = [ keywordsRule('keywords-literal', 'literal string (control)', () => keywords(`/Keywords (${STAMP})`), ), keywordsRule('keywords-key-vt', 'VT after the key', () => keywords(`/Keywords\x0B(${STAMP})`), ), keywordsRule('keywords-key-nbsp', 'NBSP after the key', () => keywords(`/Keywords\xA0(${STAMP})`), ), keywordsRule('keywords-key-nul', 'NUL after the key', () => keywords(`/Keywords\x00(${STAMP})`), ), keywordsRule('keywords-key-unspaced', 'the string right after the key', () => keywords(`/Keywords(${STAMP})`), ), keywordsRule('keywords-key-continues', '/KeywordsX', () => keywords(`/KeywordsX (${STAMP})`), ), keywordsRule('keywords-hex-vt', 'VT inside a hex string', () => keywords(`/Keywords ${hexWith(STAMP, '\x0B')}`), ), keywordsRule('keywords-hex-nul', 'NUL inside a hex string', () => keywords(`/Keywords ${hexWith(STAMP, '\x00')}`), ), keywordsRule( 'keywords-backslash-cr', 'backslash then CR inside the literal', () => keywords(`/Keywords (${STAMP}\\\r)`), ), keywordsRule('keywords-plain-cr', 'a bare CR inside the literal', () => keywords(`/Keywords (x\r${STAMP})`), ), keywordsRule('keywords-literal-utf16', 'UTF-16BE literal with BOM', () => keywords(`/Keywords ${utf16Literal(STAMP)}`), ), keywordsRule('keywords-token-4096', 'a 4,096-byte token', () => keywords(`/Keywords ${literalOfLength(4096)}`), ), keywordsRule('keywords-token-4097', 'a 4,097-byte token', () => keywords(`/Keywords ${literalOfLength(4097)}`), ), keywordsRule( 'keywords-32nd-occurrence', 'the stamp on the 32nd /Keywords', () => keywords(`${'/Keywords (x) '.repeat(31)}/Keywords (${STAMP})`), ), keywordsRule( 'keywords-33rd-occurrence', 'the stamp on the 33rd /Keywords', () => keywords(`${'/Keywords (x) '.repeat(32)}/Keywords (${STAMP})`), ), ] // --------------------------------------------------------------------------- // isWhitespace / isDelimiter around /Type /EmbeddedFile and `stream`; dictionary bounds; caps const scanRule = rule('isWhitespace and isDelimiter, dictionary bounds, caps', [ 'findEmbeddedFiles', 'readAttachedDocument', ]) const efObject = (dict: string, between = '\n', eol = '\n') => `2 0 obj\n<<${dict}>>${between}stream${eol}${J}\nendstream\nendobj\n` /** An `EmbeddedFile` dictionary of exactly `total` bytes from `<<` to `>>`. */ const efOfLength = (total: number) => { const frame = '<< /Type /EmbeddedFile /Length 13 /Pad () >>' const dict = `/Type /EmbeddedFile /Length 13 /Pad (${'x'.repeat(total - frame.length)}) ` return pdfOf(efObject(` ${dict}`)) } const streamless = (count: number, type = '/Type /EmbeddedFile') => Array.from( { length: count }, (_, i) => `${i + 10} 0 obj\n<< ${type} >>\nendobj\n`, ).join('') const SCAN_RULES: InputSpec[] = [ scanRule('type-nul-gap', 'NUL between /Type and /EmbeddedFile', () => pdfOf(efObject(' /Type\x00/EmbeddedFile /Length 13 ')), ), scanRule('type-vt-gap', 'VT between /Type and /EmbeddedFile', () => pdfOf(efObject(' /Type\x0B/EmbeddedFile /Length 13 ')), ), scanRule('type-nbsp-gap', 'NBSP between /Type and /EmbeddedFile', () => pdfOf(efObject(' /Type\xA0/EmbeddedFile /Length 13 ')), ), scanRule('type-name-end-vt', 'VT right after /EmbeddedFile', () => pdfOf(efObject(' /Type /EmbeddedFile\x0B/Length 13 ')), ), scanRule('type-name-end-nul', 'NUL right after /EmbeddedFile', () => pdfOf(efObject(' /Type /EmbeddedFile\x00/Length 13 ')), ), scanRule('type-name-continues', '/EmbeddedFileX', () => pdfOf(efObject(' /Type /EmbeddedFileX /Length 13 ')), ), scanRule('stream-after-nul', 'NUL between >> and stream', () => pdfOf(efObject(' /Type /EmbeddedFile /Length 13 ', '\x00')), ), scanRule('stream-after-vt', 'VT between >> and stream', () => pdfOf(efObject(' /Type /EmbeddedFile /Length 13 ', '\x0B')), ), scanRule('stream-cr-only', 'stream then CR alone', () => pdfOf(efObject(' /Type /EmbeddedFile /Length 13 ', '\n', '\r')), ), scanRule('stream-space-lf', 'stream then space LF', () => pdfOf(efObject(' /Type /EmbeddedFile /Length 13 ', '\n', ' \n')), ), scanRule( 'dict-length-4096', 'an EmbeddedFile dictionary of 4,096 bytes', () => efOfLength(4096), ), scanRule( 'dict-length-4097', 'an EmbeddedFile dictionary of 4,097 bytes', () => efOfLength(4097), ), scanRule( 'cap-32nd-dict-valid', '31 streamless EmbeddedFile dictionaries, then a valid one', () => pdfOf(streamless(31), EF2), ), scanRule( 'cap-33rd-dict-valid', '32 streamless EmbeddedFile dictionaries, then a valid one', () => pdfOf(streamless(32), EF2), ), scanRule( 'cap-ignores-non-names', '40 /Type /EmbeddedFileX dictionaries, then a valid one', () => pdfOf(streamless(40, '/Type /EmbeddedFileX'), EF2), ), ] // --------------------------------------------------------------------------- // One 16 MiB budget: a stored length over what is left ends the search const budgetRule = rule('shared candidate budget', ['readAttachedDocument']) /** A Flate candidate inflating to `output` bytes of `x`: charged, never JSON. */ const charged = (output: number) => { const data = deflate(new Uint8Array(output).fill(0x78)) return streamObject({ num: 2, dict: `/Type /EmbeddedFile /Filter /FlateDecode /Length ${data.byteLength}`, data, }) } const raw = (num: number, text: string) => streamObject({ num, dict: `/Type /EmbeddedFile /Length ${text.length}`, data: ascii(text), }) const flate = (num: number, data: Uint8Array) => streamObject({ num, dict: `/Type /EmbeddedFile /Filter /FlateDecode /Length ${data.byteLength}`, data, }) const PAD_40 = `{"version":3,"pad":"${'a'.repeat(18)}"}` const PAD_222 = `{"version":3,"pad":"${'a'.repeat(200)}"}` const corrupt = Uint8Array.from(FLATE_J, (b, i) => (i > 2 ? b ^ 0x55 : b)) const BUDGET: InputSpec[] = [ budgetRule( 'budget-break-before-valid', '20 bytes left, a 40-byte candidate, then a 13-byte one', () => craftPdf([charged(MAX_ATTACHMENT_BYTES - 20), raw(3, PAD_40), raw(4, J)]), ), budgetRule( 'budget-skip-control', '20 bytes left, then a 13-byte candidate', () => craftPdf([charged(MAX_ATTACHMENT_BYTES - 20), raw(4, J)]), ), budgetRule( 'budget-exact-fit', '13 bytes left, then a 13-byte candidate', () => craftPdf([charged(MAX_ATTACHMENT_BYTES - 13), raw(4, J)]), ), budgetRule( 'budget-one-short', '12 bytes left, then a 13-byte candidate', () => craftPdf([charged(MAX_ATTACHMENT_BYTES - 12), raw(4, J)]), ), budgetRule( 'budget-inflate-limited-by-remaining', '100 bytes left, a small Flate candidate inflating to 222 bytes, then a 13-byte one', () => craftPdf([ charged(MAX_ATTACHMENT_BYTES - 100), flate(3, deflate(ascii(PAD_222))), raw(4, J), ]), ), budgetRule( 'flate-corrupt-then-valid', 'a corrupt Flate candidate, then a raw one', () => craftPdf([flate(2, corrupt), raw(3, J)]), ), ] // --------------------------------------------------------------------------- // `%PDF-` within the first 1,024 bytes; the 50 MiB file cap const headerRule = rule('file header and size', ['readAttachedDocument']) const headerAt = (offset: number) => { const bytes = new Uint8Array(offset + 8) bytes.set(ascii('%PDF-1.7'), offset) return bytes } const FILE: InputSpec[] = [ headerRule('header-at-1019', '%PDF- at offset 1,019', () => headerAt(1019)), headerRule('header-at-1020', '%PDF- at offset 1,020', () => headerAt(1020)), headerRule('size-exactly-50mib', 'a file of exactly 50 MiB', () => concat([ascii('%PDF-1.7\n'), new Uint8Array(MAX_PDF_BYTES - 9)]), ), ] export const SEMANTIC_INPUTS: InputSpec[] = [ ...HEADER, ...LENGTH, ...FILTER, ...FILESPEC, ...KEYWORDS, ...SCAN_RULES, ...BUDGET, ...FILE, ] // --------------------------------------------------------------------------- // decodePdfString tokens (Latin-1 strings, one character per byte) export interface TokenCase { suite: string case: string token: string } const SCAN_TOKENS = 'crafted stream layouts > decodePdfString: literal with escapes, hex ASCII, hex UTF-16BE with BOM' export const TOKEN_CASES: TokenCase[] = [ ...['(a\\(b\\)c)', '<6F7374656F>', '', '/Name'].map( (token) => ({ suite: 'scan', case: SCAN_TOKENS, token, }), ), ...[ '(a\\nb)', '(a\\\\b)', '(\\157)', '()', '<>', '<6F7>', '<6F 73\x0B74>', '', '', '', '', '(\xFE\xFF\x00o\x00s)', '(\xE9t\xE9)', '(abc', 'abc)', ].map((token) => ({ suite: 'semantics', case: 'decodePdfString > literal unescape, hex digits, UTF-16BE BOM, Latin-1', token, })), ]