Files
faunosteo/packages/pdf/tests/helpers/r4-reader-semantics.ts
T

619 lines
20 KiB
TypeScript

import { MAX_ATTACHMENT_BYTES, MAX_PDF_BYTES } from '../../src'
import {
ascii,
concat,
craftPdf,
deflate,
pdfOf,
streamObject,
} from './r4-reader-craft'
import type { InputSpec, ReaderFunction } from './r4-reader-inputs'
/**
* Plan §B.1.2 "JS regex semantics reproduced explicitly", one minimal PDF
* per variant, plus the boundaries of the scanner's other rules (dictionary
* and keyword windows, candidate and occurrence caps, header position, the
* shared budget). Each pair differs only where the rule decides: `\s` (VT,
* FF, NBSP yes; NUL, NEL no) against `isWhitespace` (NUL yes; VT, NBSP no),
* `.` against CR and LF in the string token, the lazy `/EF` removal, the
* `break` on a candidate over the remaining budget. The outcomes are what
* the TypeScript does, recorded by the O7 driver, never what a PDF parser
* would do.
*/
const J = '{"version":3}'
const FLATE_J = deflate(ascii(J))
const EF2 = `2 0 obj\n<< /Type /EmbeddedFile /Length 13 >>\nstream\n${J}\nendstream\nendobj\n`
type Build = InputSpec['build']
const rule =
(group: string, exercises: ReaderFunction[]) =>
(slug: string, title: string, build: Build): InputSpec => ({
slug: `semantics.${slug}`,
kind: 'pdf',
cases: [{ suite: 'semantics', case: `${group} > ${title}`, exercises }],
build,
})
// ---------------------------------------------------------------------------
// `n 0 obj` header: /(\d{1,10})\s+\d{1,10}\s+obj\s*$/ on a 48-byte window
const header = rule('object header regex (\\s, 48-byte window)', [
'findEmbeddedFiles',
])
const namedStream = (head: string, target = 2) =>
pdfOf(
`3 0 obj\n<< /Type /Filespec /F (osteo-report.json) /EF << /F ${target} 0 R >> >>\nendobj\n`,
`${head}<< /Type /EmbeddedFile /Length 13 >>\nstream\n${J}\nendstream\nendobj\n`,
)
const HEADER: InputSpec[] = [
header('header-vt', 'VT (0x0B) separators', () =>
namedStream('2\x0B0\x0Bobj\x0B'),
),
header('header-ff', 'FF (0x0C) separators', () =>
namedStream('2\x0C0\x0Cobj\x0C'),
),
header('header-nbsp', 'NBSP (0xA0) separators', () =>
namedStream('2\xA00\xA0obj\xA0'),
),
header('header-nel', 'NEL (0x85) between the numbers', () =>
namedStream('2\x850\x85obj\n'),
),
header('header-nul', 'NUL between the numbers', () =>
namedStream('2\x000 obj\n'),
),
header('header-nul-before-dict', 'NUL between obj and <<', () =>
namedStream('2 0 obj\x00'),
),
header(
'header-window-40-spaces',
'12 0 obj then 40 spaces: the number fits the window',
() => namedStream(`12 0 obj${' '.repeat(40)}`, 12),
),
header(
'header-window-41-spaces',
'12 0 obj then 41 spaces: the window cuts the 1',
() => namedStream(`12 0 obj${' '.repeat(41)}`, 12),
),
]
// ---------------------------------------------------------------------------
// /Length: /\/Length\s+(\d{1,10})(?!\d)(?!\s+\d+\s+R)/, then isWhitespace before endstream
const length = rule('/Length regex and endstream landing', [
'findEmbeddedFiles',
'readAttachedDocument',
])
/** Three spaces after the data: a direct /Length gives 13, the measured length 16. */
const lengthStream = (dict: string, tail = ' \n') =>
pdfOf(
`2 0 obj\n<< /Type /EmbeddedFile ${dict} >>\nstream\n${J}${tail}endstream\nendobj\n`,
)
const LENGTH: InputSpec[] = [
length('length-space', 'space after /Length (control)', () =>
lengthStream('/Length 13'),
),
length('length-vt', 'VT after /Length', () => lengthStream('/Length\x0B13')),
length('length-nbsp', 'NBSP after /Length', () =>
lengthStream('/Length\xA013'),
),
length('length-nel', 'NEL after /Length', () =>
lengthStream('/Length\x8513'),
),
length('length-nul', 'NUL after /Length', () =>
lengthStream('/Length\x0013'),
),
length('length-indirect-space', '13 0 R with spaces (indirect)', () =>
lengthStream('/Length 13 0 R'),
),
length('length-indirect-vt', '13 0 R with VT separators', () =>
lengthStream('/Length 13\x0B0\x0BR'),
),
length('length-indirect-nul', '13 0 R with NUL separators', () =>
lengthStream('/Length 13\x000\x00R'),
),
length('length-10-digits', 'ten digits 0000000013', () =>
lengthStream('/Length 0000000013'),
),
length('length-11-digits', 'eleven digits 00000000013', () =>
lengthStream('/Length 00000000013'),
),
length('length-past-end', 'a direct length past the end of the file', () =>
lengthStream('/Length 99999'),
),
length('length-tail-nul', 'NUL between the data and endstream', () =>
lengthStream('/Length 13', '\x00\n'),
),
length('length-tail-vt', 'VT between the data and endstream', () =>
lengthStream('/Length 13', '\x0B\n'),
),
length('measured-lf', 'no /Length, LF before endstream', () =>
lengthStream('', '\n'),
),
length('measured-crlf', 'no /Length, CRLF before endstream', () =>
lengthStream('', '\r\n'),
),
length('measured-lfcr', 'no /Length, LF CR before endstream', () =>
lengthStream('', '\n\r'),
),
length('measured-cr', 'no /Length, CR before endstream', () =>
lengthStream('', '\r'),
),
]
// ---------------------------------------------------------------------------
// /Filter: /\/Filter\s*(\[\s*)?\/FlateDecode\b/
const filter = rule('/Filter regex', [
'findEmbeddedFiles',
'readAttachedDocument',
])
const filterStream = (dict: string, data: Uint8Array = FLATE_J) =>
craftPdf([
streamObject({
num: 2,
dict: ` /Type /EmbeddedFile ${dict} /Length ${data.byteLength} `,
data,
}),
])
const HEX_J = ascii(
`${Buffer.from(J, 'latin1').toString('hex').toUpperCase()}>`,
)
const FILTER: InputSpec[] = [
filter('filter-name', '/Filter /FlateDecode (control)', () =>
filterStream('/Filter /FlateDecode'),
),
filter('filter-unspaced', '/Filter/FlateDecode', () =>
filterStream('/Filter/FlateDecode'),
),
filter('filter-array', '/Filter [/FlateDecode]', () =>
filterStream('/Filter [/FlateDecode]'),
),
filter('filter-array-spaced', '/Filter [ /FlateDecode ]', () =>
filterStream('/Filter [ /FlateDecode ]'),
),
filter(
'filter-array-second',
'/Filter [/ASCIIHexDecode /FlateDecode] on Flate data',
() => filterStream('/Filter [/ASCIIHexDecode /FlateDecode]'),
),
filter('filter-asciihex', '/Filter /ASCIIHexDecode on hex data', () =>
filterStream('/Filter /ASCIIHexDecode', HEX_J),
),
filter(
'filter-asciihex-on-json',
'/Filter /ASCIIHexDecode on plain JSON',
() => filterStream('/Filter /ASCIIHexDecode', ascii(J)),
),
filter('filter-vt', 'VT after /Filter', () =>
filterStream('/Filter\x0B/FlateDecode'),
),
filter('filter-nbsp', 'NBSP after /Filter', () =>
filterStream('/Filter\xA0/FlateDecode'),
),
filter('filter-nul', 'NUL after /Filter', () =>
filterStream('/Filter\x00/FlateDecode'),
),
filter('filter-array-nul', 'NUL inside the array', () =>
filterStream('/Filter [\x00/FlateDecode]'),
),
filter('filter-name-continues', '/Filter /FlateDecodeX', () =>
filterStream('/Filter /FlateDecodeX'),
),
]
// ---------------------------------------------------------------------------
// Filespec: /EF removal (lazy), /F then /UF string token, /EF << /F n 0 R >>
const filespecRule = rule('Filespec /EF and /F regexes, string token', [
'findEmbeddedFiles',
])
const filespec = (...dicts: string[]) =>
pdfOf(...dicts.map((d, i) => `${i + 3} 0 obj\n<<${d}>>\nendobj\n`), EF2)
const FILESPEC: InputSpec[] = [
filespecRule('filespec-ef-vt', 'VT around /EF and the reference', () =>
filespec(
' /Type /Filespec /F (osteo-report.json) /EF\x0B<< /F\x0B2\x0B0\x0BR >> ',
),
),
filespecRule('filespec-ef-nbsp', 'NBSP around /EF and the reference', () =>
filespec(
' /Type /Filespec /F (osteo-report.json) /EF\xA0<< /F\xA02\xA00\xA0R >> ',
),
),
filespecRule('filespec-ef-nul', 'NUL between /EF and <<', () =>
filespec(' /Type /Filespec /F (osteo-report.json) /EF\x00<< /F 2 0 R >> '),
),
filespecRule('filespec-ref-nul', 'NUL between /F and the reference', () =>
filespec(' /Type /Filespec /F (osteo-report.json) /EF << /F\x002 0 R >> '),
),
filespecRule('filespec-ref-nel', 'NEL between /F and the reference', () =>
filespec(' /Type /Filespec /F (osteo-report.json) /EF << /F\x852 0 R >> '),
),
filespecRule(
'filespec-f-vt',
'VT between /F and its string, /UF fallback present',
() =>
filespec(
' /Type /Filespec /F\x0B(osteo-report.json) /UF (fallback.json) /EF << /F 2 0 R >> ',
),
),
filespecRule(
'filespec-f-nul',
'NUL between /F and its string, /UF fallback present',
() =>
filespec(
' /Type /Filespec /F\x00(osteo-report.json) /UF (fallback.json) /EF << /F 2 0 R >> ',
),
),
filespecRule('filespec-uf-only', '/UF only', () =>
filespec(' /Type /Filespec /UF (osteo-report.json) /EF << /F 2 0 R >> '),
),
filespecRule(
'filespec-lazy-ef',
'/EF holding a nested dictionary: removal stops at the first >>',
() =>
filespec(
' /Type /Filespec /EF << /F 2 0 R /Sub << >> /F (inner.json) >> /UF (outer.json) ',
),
),
filespecRule(
'filespec-tight',
'no spaces, /EF closing right before the Filespec >>',
() => filespec('/Type/Filespec/F(osteo-report.json)/EF<</F 2 0 R>>'),
),
filespecRule(
'filespec-two-for-one-stream',
'two Filespecs for one stream: the first wins',
() =>
filespec(
' /Type /Filespec /F (decoy.json) /EF << /F 2 0 R >> ',
' /Type /Filespec /F (osteo-report.json) /EF << /F 2 0 R >> ',
),
),
filespecRule(
'filespec-backslash-cr',
'backslash then CR inside the /F string',
() =>
filespec(
' /Type /Filespec /F (osteo-report\\\r.json) /UF (fallback.json) /EF << /F 2 0 R >> ',
),
),
filespecRule(
'filespec-backslash-lf',
'backslash then LF inside the /F string',
() =>
filespec(
' /Type /Filespec /F (osteo-report\\\n.json) /UF (fallback.json) /EF << /F 2 0 R >> ',
),
),
filespecRule('filespec-plain-cr', 'a bare CR inside the /F string', () =>
filespec(' /Type /Filespec /F (osteo-report\r.json) /EF << /F 2 0 R >> '),
),
filespecRule(
'filespec-escaped-paren',
'an escaped parenthesis inside the /F string',
() =>
filespec(
' /Type /Filespec /F (osteo-report\\).json) /EF << /F 2 0 R >> ',
),
),
filespecRule('filespec-hex-latin1', 'hex /F name without BOM', () =>
filespec(
` /Type /Filespec /F <${Buffer.from('osteo-report.json').toString('hex')}> /EF << /F 2 0 R >> `,
),
),
]
// ---------------------------------------------------------------------------
// /Keywords: isDelimiter after the key, string token, 4,096-byte window, 32 occurrences
const keywordsRule = rule('/Keywords stamp', [
'hasArchiveStamp',
'readAttachedDocument',
])
const STAMP = 'faunosteo-archive/1'
const keywords = (body: string) => pdfOf(`3 0 obj\n<< ${body} >>\nendobj\n`)
const hexWith = (text: string, insert: string) => {
const hex = Buffer.from(text, 'latin1').toString('hex')
return `<${hex.slice(0, 8)}${insert}${hex.slice(8)}>`
}
const utf16Literal = (text: string) =>
`(\xFE\xFF${Array.from(text, (c) => `\x00${c}`).join('')})`
/** A literal token of exactly `total` bytes ending with the stamp. */
const literalOfLength = (total: number) =>
`(${'x'.repeat(total - STAMP.length - 2)}${STAMP})`
const KEYWORDS: InputSpec[] = [
keywordsRule('keywords-literal', 'literal string (control)', () =>
keywords(`/Keywords (${STAMP})`),
),
keywordsRule('keywords-key-vt', 'VT after the key', () =>
keywords(`/Keywords\x0B(${STAMP})`),
),
keywordsRule('keywords-key-nbsp', 'NBSP after the key', () =>
keywords(`/Keywords\xA0(${STAMP})`),
),
keywordsRule('keywords-key-nul', 'NUL after the key', () =>
keywords(`/Keywords\x00(${STAMP})`),
),
keywordsRule('keywords-key-unspaced', 'the string right after the key', () =>
keywords(`/Keywords(${STAMP})`),
),
keywordsRule('keywords-key-continues', '/KeywordsX', () =>
keywords(`/KeywordsX (${STAMP})`),
),
keywordsRule('keywords-hex-vt', 'VT inside a hex string', () =>
keywords(`/Keywords ${hexWith(STAMP, '\x0B')}`),
),
keywordsRule('keywords-hex-nul', 'NUL inside a hex string', () =>
keywords(`/Keywords ${hexWith(STAMP, '\x00')}`),
),
keywordsRule(
'keywords-backslash-cr',
'backslash then CR inside the literal',
() => keywords(`/Keywords (${STAMP}\\\r)`),
),
keywordsRule('keywords-plain-cr', 'a bare CR inside the literal', () =>
keywords(`/Keywords (x\r${STAMP})`),
),
keywordsRule('keywords-literal-utf16', 'UTF-16BE literal with BOM', () =>
keywords(`/Keywords ${utf16Literal(STAMP)}`),
),
keywordsRule('keywords-token-4096', 'a 4,096-byte token', () =>
keywords(`/Keywords ${literalOfLength(4096)}`),
),
keywordsRule('keywords-token-4097', 'a 4,097-byte token', () =>
keywords(`/Keywords ${literalOfLength(4097)}`),
),
keywordsRule(
'keywords-32nd-occurrence',
'the stamp on the 32nd /Keywords',
() => keywords(`${'/Keywords (x) '.repeat(31)}/Keywords (${STAMP})`),
),
keywordsRule(
'keywords-33rd-occurrence',
'the stamp on the 33rd /Keywords',
() => keywords(`${'/Keywords (x) '.repeat(32)}/Keywords (${STAMP})`),
),
]
// ---------------------------------------------------------------------------
// isWhitespace / isDelimiter around /Type /EmbeddedFile and `stream`; dictionary bounds; caps
const scanRule = rule('isWhitespace and isDelimiter, dictionary bounds, caps', [
'findEmbeddedFiles',
'readAttachedDocument',
])
const efObject = (dict: string, between = '\n', eol = '\n') =>
`2 0 obj\n<<${dict}>>${between}stream${eol}${J}\nendstream\nendobj\n`
/** An `EmbeddedFile` dictionary of exactly `total` bytes from `<<` to `>>`. */
const efOfLength = (total: number) => {
const frame = '<< /Type /EmbeddedFile /Length 13 /Pad () >>'
const dict = `/Type /EmbeddedFile /Length 13 /Pad (${'x'.repeat(total - frame.length)}) `
return pdfOf(efObject(` ${dict}`))
}
const streamless = (count: number, type = '/Type /EmbeddedFile') =>
Array.from(
{ length: count },
(_, i) => `${i + 10} 0 obj\n<< ${type} >>\nendobj\n`,
).join('')
const SCAN_RULES: InputSpec[] = [
scanRule('type-nul-gap', 'NUL between /Type and /EmbeddedFile', () =>
pdfOf(efObject(' /Type\x00/EmbeddedFile /Length 13 ')),
),
scanRule('type-vt-gap', 'VT between /Type and /EmbeddedFile', () =>
pdfOf(efObject(' /Type\x0B/EmbeddedFile /Length 13 ')),
),
scanRule('type-nbsp-gap', 'NBSP between /Type and /EmbeddedFile', () =>
pdfOf(efObject(' /Type\xA0/EmbeddedFile /Length 13 ')),
),
scanRule('type-name-end-vt', 'VT right after /EmbeddedFile', () =>
pdfOf(efObject(' /Type /EmbeddedFile\x0B/Length 13 ')),
),
scanRule('type-name-end-nul', 'NUL right after /EmbeddedFile', () =>
pdfOf(efObject(' /Type /EmbeddedFile\x00/Length 13 ')),
),
scanRule('type-name-continues', '/EmbeddedFileX', () =>
pdfOf(efObject(' /Type /EmbeddedFileX /Length 13 ')),
),
scanRule('stream-after-nul', 'NUL between >> and stream', () =>
pdfOf(efObject(' /Type /EmbeddedFile /Length 13 ', '\x00')),
),
scanRule('stream-after-vt', 'VT between >> and stream', () =>
pdfOf(efObject(' /Type /EmbeddedFile /Length 13 ', '\x0B')),
),
scanRule('stream-cr-only', 'stream then CR alone', () =>
pdfOf(efObject(' /Type /EmbeddedFile /Length 13 ', '\n', '\r')),
),
scanRule('stream-space-lf', 'stream then space LF', () =>
pdfOf(efObject(' /Type /EmbeddedFile /Length 13 ', '\n', ' \n')),
),
scanRule(
'dict-length-4096',
'an EmbeddedFile dictionary of 4,096 bytes',
() => efOfLength(4096),
),
scanRule(
'dict-length-4097',
'an EmbeddedFile dictionary of 4,097 bytes',
() => efOfLength(4097),
),
scanRule(
'cap-32nd-dict-valid',
'31 streamless EmbeddedFile dictionaries, then a valid one',
() => pdfOf(streamless(31), EF2),
),
scanRule(
'cap-33rd-dict-valid',
'32 streamless EmbeddedFile dictionaries, then a valid one',
() => pdfOf(streamless(32), EF2),
),
scanRule(
'cap-ignores-non-names',
'40 /Type /EmbeddedFileX dictionaries, then a valid one',
() => pdfOf(streamless(40, '/Type /EmbeddedFileX'), EF2),
),
]
// ---------------------------------------------------------------------------
// One 16 MiB budget: a stored length over what is left ends the search
const budgetRule = rule('shared candidate budget', ['readAttachedDocument'])
/** A Flate candidate inflating to `output` bytes of `x`: charged, never JSON. */
const charged = (output: number) => {
const data = deflate(new Uint8Array(output).fill(0x78))
return streamObject({
num: 2,
dict: `/Type /EmbeddedFile /Filter /FlateDecode /Length ${data.byteLength}`,
data,
})
}
const raw = (num: number, text: string) =>
streamObject({
num,
dict: `/Type /EmbeddedFile /Length ${text.length}`,
data: ascii(text),
})
const flate = (num: number, data: Uint8Array) =>
streamObject({
num,
dict: `/Type /EmbeddedFile /Filter /FlateDecode /Length ${data.byteLength}`,
data,
})
const PAD_40 = `{"version":3,"pad":"${'a'.repeat(18)}"}`
const PAD_222 = `{"version":3,"pad":"${'a'.repeat(200)}"}`
const corrupt = Uint8Array.from(FLATE_J, (b, i) => (i > 2 ? b ^ 0x55 : b))
const BUDGET: InputSpec[] = [
budgetRule(
'budget-break-before-valid',
'20 bytes left, a 40-byte candidate, then a 13-byte one',
() =>
craftPdf([charged(MAX_ATTACHMENT_BYTES - 20), raw(3, PAD_40), raw(4, J)]),
),
budgetRule(
'budget-skip-control',
'20 bytes left, then a 13-byte candidate',
() => craftPdf([charged(MAX_ATTACHMENT_BYTES - 20), raw(4, J)]),
),
budgetRule(
'budget-exact-fit',
'13 bytes left, then a 13-byte candidate',
() => craftPdf([charged(MAX_ATTACHMENT_BYTES - 13), raw(4, J)]),
),
budgetRule(
'budget-one-short',
'12 bytes left, then a 13-byte candidate',
() => craftPdf([charged(MAX_ATTACHMENT_BYTES - 12), raw(4, J)]),
),
budgetRule(
'budget-inflate-limited-by-remaining',
'100 bytes left, a small Flate candidate inflating to 222 bytes, then a 13-byte one',
() =>
craftPdf([
charged(MAX_ATTACHMENT_BYTES - 100),
flate(3, deflate(ascii(PAD_222))),
raw(4, J),
]),
),
budgetRule(
'flate-corrupt-then-valid',
'a corrupt Flate candidate, then a raw one',
() => craftPdf([flate(2, corrupt), raw(3, J)]),
),
]
// ---------------------------------------------------------------------------
// `%PDF-` within the first 1,024 bytes; the 50 MiB file cap
const headerRule = rule('file header and size', ['readAttachedDocument'])
const headerAt = (offset: number) => {
const bytes = new Uint8Array(offset + 8)
bytes.set(ascii('%PDF-1.7'), offset)
return bytes
}
const FILE: InputSpec[] = [
headerRule('header-at-1019', '%PDF- at offset 1,019', () => headerAt(1019)),
headerRule('header-at-1020', '%PDF- at offset 1,020', () => headerAt(1020)),
headerRule('size-exactly-50mib', 'a file of exactly 50 MiB', () =>
concat([ascii('%PDF-1.7\n'), new Uint8Array(MAX_PDF_BYTES - 9)]),
),
]
export const SEMANTIC_INPUTS: InputSpec[] = [
...HEADER,
...LENGTH,
...FILTER,
...FILESPEC,
...KEYWORDS,
...SCAN_RULES,
...BUDGET,
...FILE,
]
// ---------------------------------------------------------------------------
// decodePdfString tokens (Latin-1 strings, one character per byte)
export interface TokenCase {
suite: string
case: string
token: string
}
const SCAN_TOKENS =
'crafted stream layouts > decodePdfString: literal with escapes, hex ASCII, hex UTF-16BE with BOM'
export const TOKEN_CASES: TokenCase[] = [
...['(a\\(b\\)c)', '<6F7374656F>', '<FEFF006F0073>', '/Name'].map(
(token) => ({
suite: 'scan',
case: SCAN_TOKENS,
token,
}),
),
...[
'(a\\nb)',
'(a\\\\b)',
'(\\157)',
'()',
'<>',
'<6F7>',
'<6F 73\x0B74>',
'<zz6F>',
'<FEFF>',
'<FEFF00>',
'<FEFF006F00>',
'(\xFE\xFF\x00o\x00s)',
'(\xE9t\xE9)',
'(abc',
'abc)',
].map((token) => ({
suite: 'semantics',
case: 'decodePdfString > literal unescape, hex digits, UTF-16BE BOM, Latin-1',
token,
})),
]