test(tests): R4.1d O7 reader outcomes (155 inputs, regex-semantics rows) and O9 inflate verdicts (Node)

This commit is contained in:
bchanot committed 2026-10-18 05:35:12 +02:00
1 parent 87b88739f8
commit 96df15edae
187 files changed
+12207

No files matched your search

@@ -0,0 +1,105 @@
import { deflateSync } from 'node:zlib'
/**
* Byte builders of the R4 reader oracles, copied from scan.test.ts and
* inflate.test.ts: a test file cannot be imported without registering its
* cases, and those files stay untouched. Same output for the same arguments.
*/
export const KiB = 1024
export const MiB = 1024 * KiB
export const latin1 = (bytes: Uint8Array) =>
Buffer.from(bytes).toString('latin1')
/** One byte per character: every string given here stays at or below U+00FF. */
export const ascii = (text: string) =>
new Uint8Array(Buffer.from(text, 'latin1'))
export function concat(parts: Uint8Array[]): Uint8Array {
const out = new Uint8Array(parts.reduce((n, p) => n + p.byteLength, 0))
let at = 0
for (const part of parts) {
out.set(part, at)
at += part.byteLength
}
return out
}
export interface StreamSpec {
num: number
dict: string
data: Uint8Array
eol?: string
}
export function streamObject({
num,
dict,
data,
eol = '\n',
}: StreamSpec): Uint8Array {
return concat([
ascii(`${num} 0 obj\n<<${dict}>>\nstream${eol}`),
data,
ascii('\nendstream\nendobj\n'),
])
}
/** The scanner never reads the xref, so none is written. */
export function craftPdf(objects: Uint8Array[]): Uint8Array {
return concat([
ascii('%PDF-1.7\n'),
...objects,
ascii('trailer\n<< /Root 1 0 R >>\n%%EOF\n'),
])
}
/** `craftPdf` over Latin-1 texts and byte parts. */
export function pdfOf(...objects: Array<string | Uint8Array>): Uint8Array {
return craftPdf(objects.map((o) => (typeof o === 'string' ? ascii(o) : o)))
}
export interface Span {
start: number
end: number
}
/** Bounds of the `n 0 obj … endobj` object whose dictionary holds `marker` (scan.test.ts `objectHolding`). */
export function objectSpan(pdf: Uint8Array, marker: string): Span {
const text = latin1(pdf)
const at = text.indexOf(marker)
if (at <= 0) throw new Error(`marker not found: ${marker}`)
const start = text.lastIndexOf(' 0 obj', at)
const headerStart = text.lastIndexOf('\n', start) + 1
const end = text.indexOf('endobj', at) + 'endobj'.length
return { start: headerStart, end }
}
export function objectHolding(pdf: Uint8Array, marker: string): Uint8Array {
const { start, end } = objectSpan(pdf, marker)
return pdf.slice(start, end)
}
export const filled = (byte: string, length: number) =>
ascii(byte.repeat(length))
export const deflate = (data: Uint8Array) => new Uint8Array(deflateSync(data))
/** Incompressible bytes: the compressed stream is about as long as the input. */
export function noise(length: number): Uint8Array {
let seed = 0x9e3779b9
return Uint8Array.from({ length }, () => {
seed = (seed * 1664525 + 1013904223) >>> 0
return seed >>> 24
})
}
/** CompressionStream('deflate') output, as security.test.ts builds its inputs. */
export async function compressionStreamDeflate(
data: Uint8Array,
): Promise<Uint8Array> {
const stream = new Blob([data])
.stream()
.pipeThrough(new CompressionStream('deflate'))
return new Uint8Array(await new Response(stream).arrayBuffer())
}
File diff suppressed because it is too large. Load diff
+215
View File
@@ -0,0 +1,215 @@
import { createHash } from 'node:crypto'
import {
existsSync,
mkdirSync,
readdirSync,
readFileSync,
writeFileSync,
} from 'node:fs'
import { dirname, resolve } from 'node:path'
import { expect } from 'vitest'
import { inflateFlate } from '../../src/attachment/inflate'
import { FIXTURES_DIR, REPO_ROOT } from './draft'
import { ascii, concat, latin1, MiB } from './r4-reader-craft'
/**
* Write-or-check plumbing of the R4.1 reader oracles (O7, O9). With
* `MAKE_R4_ORACLES=1` the generators write their inputs and oracles; without
* it they rebuild everything and require the committed files to be the same
* (bytes for inputs, deep equality for JSON).
*/
export const MAKE = process.env.MAKE_R4_ORACLES === '1'
export const R4_INPUTS_DIR = resolve(FIXTURES_DIR, 'pdf/r4')
export const R4_ORACLES_DIR = resolve(REPO_ROOT, 'crates/core/tests/oracles/r4')
/** Inputs above this size are committed as a generator description, never as bytes. */
export const MAX_STORED_BYTES = 2 * MiB
export const sha256 = (data: Uint8Array | string): string =>
createHash('sha256').update(data).digest('hex')
export const jsonText = (value: unknown): string =>
`${JSON.stringify(value, null, 2)}\n`
export function relativeToRepo(path: string): string {
return path.slice(REPO_ROOT.length + 1)
}
export function storeFile(path: string, bytes: Uint8Array): void {
if (MAKE) {
mkdirSync(dirname(path), { recursive: true })
writeFileSync(path, bytes)
return
}
expect(existsSync(path), `missing ${relativeToRepo(path)}`).toBe(true)
expect(sha256(readFileSync(path)), relativeToRepo(path)).toBe(sha256(bytes))
}
export function readJsonFile<T>(path: string): T {
return JSON.parse(readFileSync(path, 'utf8')) as T
}
/**
* `normalise` maps the committed JSON before the comparison, for the parts
* this generator does not own (the Node version that wrote it, the browser
* columns of O9).
*/
export function storeJson(
path: string,
value: unknown,
normalise: (committed: unknown) => unknown = (committed) => committed,
): void {
if (MAKE) {
storeFile(path, new TextEncoder().encode(jsonText(value)))
return
}
expect(existsSync(path), `missing ${relativeToRepo(path)}`).toBe(true)
expect(normalise(readJsonFile(path)), relativeToRepo(path)).toEqual(
JSON.parse(jsonText(value)),
)
}
/** The Node build that produced an oracle: recorded, never compared (inputs are compared byte for byte). */
export const generatedWith = () => ({
node: process.version,
zlib: process.versions.zlib,
})
/** The input directory holds exactly the listed files: no stale input survives a rename. */
export function expectOnlyFiles(dir: string, files: string[]): void {
expect(readdirSync(dir).sort(), relativeToRepo(dir)).toEqual(
[...new Set(files)].sort(),
)
}
/** inflateFlate's verdict as the oracles record it: null, or the output length and digest. */
export async function inflateVerdict(input: Uint8Array, limit: number) {
const out = await inflateFlate(input, limit)
return out === null ? null : { length: out.byteLength, sha256: sha256(out) }
}
// ---------------------------------------------------------------------------
// Generator descriptions (`<slug>.gen.json`) for the inputs over 2 MiB
/**
* Concatenated in order: `latin1` one byte per character, `fill` the byte
* value repeated `count` times, `slice` the bytes [start, end) of another
* input file of the same directory.
*/
export type Part =
| { latin1: string }
| { fill: number; count: number }
| { slice: string; start: number; end: number }
export interface GeneratorDescription {
generator: 'concat/1'
length: number
sha256: string
parts: Part[]
}
/** Bytes a description may reference instead of spelling them out. */
export interface SliceRef {
file: string
start: number
end: number
bytes: Uint8Array
}
export function expandParts(parts: Part[], dir: string): Uint8Array {
return concat(
parts.map((part) => {
if ('latin1' in part) return ascii(part.latin1)
if ('fill' in part) return new Uint8Array(part.count).fill(part.fill)
const source = new Uint8Array(readFileSync(resolve(dir, part.slice)))
return source.subarray(part.start, part.end)
}),
)
}
/** Runs this long or longer of one byte value become a `fill` part. */
const MIN_RUN = 4096
function runParts(bytes: Uint8Array, start: number, end: number): Part[] {
const parts: Part[] = []
let textFrom = start
let i = start
while (i < end) {
let j = i + 1
while (j < end && bytes[j] === bytes[i]) j += 1
if (j - i >= MIN_RUN) {
if (textFrom < i) {
parts.push({ latin1: latin1(bytes.subarray(textFrom, i)) })
}
parts.push({ fill: bytes[i]!, count: j - i })
textFrom = j
}
i = j
}
if (textFrom < end)
parts.push({ latin1: latin1(bytes.subarray(textFrom, end)) })
return parts
}
interface RefHit {
at: number
ref: SliceRef
}
function refHits(bytes: Uint8Array, refs: SliceRef[]): RefHit[] {
const buffer = Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength)
const hits: RefHit[] = []
for (const ref of refs) {
let at = buffer.indexOf(ref.bytes)
while (at >= 0) {
hits.push({ at, ref })
at = buffer.indexOf(ref.bytes, at + ref.bytes.byteLength)
}
}
return hits.sort((a, b) => a.at - b.at)
}
/** Derives the description from the bytes the original test builds, so both stay the same input. */
export function partsOf(bytes: Uint8Array, refs: SliceRef[]): Part[] {
const parts: Part[] = []
let at = 0
for (const hit of refHits(bytes, refs)) {
parts.push(...runParts(bytes, at, hit.at))
parts.push({ slice: hit.ref.file, start: hit.ref.start, end: hit.ref.end })
at = hit.at + hit.ref.bytes.byteLength
}
parts.push(...runParts(bytes, at, bytes.byteLength))
return parts
}
export interface StoredInput {
file: string
stored: 'bytes' | 'generator'
length: number
sha256: string
}
/** Commits an input as bytes, or above MAX_STORED_BYTES as a description checked to expand to the same bytes. */
export function storeInput(
dir: string,
name: { slug: string; ext: string },
bytes: Uint8Array,
refs: SliceRef[] = [],
): StoredInput {
const digest = { length: bytes.byteLength, sha256: sha256(bytes) }
if (bytes.byteLength <= MAX_STORED_BYTES) {
const file = `${name.slug}${name.ext}`
storeFile(resolve(dir, file), bytes)
return { file, stored: 'bytes', ...digest }
}
const file = `${name.slug}.gen.json`
const description: GeneratorDescription = {
generator: 'concat/1',
...digest,
parts: partsOf(bytes, refs),
}
storeJson(resolve(dir, file), description)
expect(sha256(expandParts(description.parts, dir)), file).toBe(digest.sha256)
return { file, stored: 'generator', ...digest }
}
@@ -0,0 +1,618 @@
import { MAX_ATTACHMENT_BYTES, MAX_PDF_BYTES } from '../../src'
import {
ascii,
concat,
craftPdf,
deflate,
pdfOf,
streamObject,
} from './r4-reader-craft'
import type { InputSpec, ReaderFunction } from './r4-reader-inputs'
/**
* Plan §B.1.2 "JS regex semantics reproduced explicitly", one minimal PDF
* per variant, plus the boundaries of the scanner's other rules (dictionary
* and keyword windows, candidate and occurrence caps, header position, the
* shared budget). Each pair differs only where the rule decides: `\s` (VT,
* FF, NBSP yes; NUL, NEL no) against `isWhitespace` (NUL yes; VT, NBSP no),
* `.` against CR and LF in the string token, the lazy `/EF` removal, the
* `break` on a candidate over the remaining budget. The outcomes are what
* the TypeScript does, recorded by the O7 driver, never what a PDF parser
* would do.
*/
const J = '{"version":3}'
const FLATE_J = deflate(ascii(J))
const EF2 = `2 0 obj\n<< /Type /EmbeddedFile /Length 13 >>\nstream\n${J}\nendstream\nendobj\n`
type Build = InputSpec['build']
const rule =
(group: string, exercises: ReaderFunction[]) =>
(slug: string, title: string, build: Build): InputSpec => ({
slug: `semantics.${slug}`,
kind: 'pdf',
cases: [{ suite: 'semantics', case: `${group} > ${title}`, exercises }],
build,
})
// ---------------------------------------------------------------------------
// `n 0 obj` header: /(\d{1,10})\s+\d{1,10}\s+obj\s*$/ on a 48-byte window
const header = rule('object header regex (\\s, 48-byte window)', [
'findEmbeddedFiles',
])
const namedStream = (head: string, target = 2) =>
pdfOf(
`3 0 obj\n<< /Type /Filespec /F (osteo-report.json) /EF << /F ${target} 0 R >> >>\nendobj\n`,
`${head}<< /Type /EmbeddedFile /Length 13 >>\nstream\n${J}\nendstream\nendobj\n`,
)
const HEADER: InputSpec[] = [
header('header-vt', 'VT (0x0B) separators', () =>
namedStream('2\x0B0\x0Bobj\x0B'),
),
header('header-ff', 'FF (0x0C) separators', () =>
namedStream('2\x0C0\x0Cobj\x0C'),
),
header('header-nbsp', 'NBSP (0xA0) separators', () =>
namedStream('2\xA00\xA0obj\xA0'),
),
header('header-nel', 'NEL (0x85) between the numbers', () =>
namedStream('2\x850\x85obj\n'),
),
header('header-nul', 'NUL between the numbers', () =>
namedStream('2\x000 obj\n'),
),
header('header-nul-before-dict', 'NUL between obj and <<', () =>
namedStream('2 0 obj\x00'),
),
header(
'header-window-40-spaces',
'12 0 obj then 40 spaces: the number fits the window',
() => namedStream(`12 0 obj${' '.repeat(40)}`, 12),
),
header(
'header-window-41-spaces',
'12 0 obj then 41 spaces: the window cuts the 1',
() => namedStream(`12 0 obj${' '.repeat(41)}`, 12),
),
]
// ---------------------------------------------------------------------------
// /Length: /\/Length\s+(\d{1,10})(?!\d)(?!\s+\d+\s+R)/, then isWhitespace before endstream
const length = rule('/Length regex and endstream landing', [
'findEmbeddedFiles',
'readAttachedDocument',
])
/** Three spaces after the data: a direct /Length gives 13, the measured length 16. */
const lengthStream = (dict: string, tail = ' \n') =>
pdfOf(
`2 0 obj\n<< /Type /EmbeddedFile ${dict} >>\nstream\n${J}${tail}endstream\nendobj\n`,
)
const LENGTH: InputSpec[] = [
length('length-space', 'space after /Length (control)', () =>
lengthStream('/Length 13'),
),
length('length-vt', 'VT after /Length', () => lengthStream('/Length\x0B13')),
length('length-nbsp', 'NBSP after /Length', () =>
lengthStream('/Length\xA013'),
),
length('length-nel', 'NEL after /Length', () =>
lengthStream('/Length\x8513'),
),
length('length-nul', 'NUL after /Length', () =>
lengthStream('/Length\x0013'),
),
length('length-indirect-space', '13 0 R with spaces (indirect)', () =>
lengthStream('/Length 13 0 R'),
),
length('length-indirect-vt', '13 0 R with VT separators', () =>
lengthStream('/Length 13\x0B0\x0BR'),
),
length('length-indirect-nul', '13 0 R with NUL separators', () =>
lengthStream('/Length 13\x000\x00R'),
),
length('length-10-digits', 'ten digits 0000000013', () =>
lengthStream('/Length 0000000013'),
),
length('length-11-digits', 'eleven digits 00000000013', () =>
lengthStream('/Length 00000000013'),
),
length('length-past-end', 'a direct length past the end of the file', () =>
lengthStream('/Length 99999'),
),
length('length-tail-nul', 'NUL between the data and endstream', () =>
lengthStream('/Length 13', '\x00\n'),
),
length('length-tail-vt', 'VT between the data and endstream', () =>
lengthStream('/Length 13', '\x0B\n'),
),
length('measured-lf', 'no /Length, LF before endstream', () =>
lengthStream('', '\n'),
),
length('measured-crlf', 'no /Length, CRLF before endstream', () =>
lengthStream('', '\r\n'),
),
length('measured-lfcr', 'no /Length, LF CR before endstream', () =>
lengthStream('', '\n\r'),
),
length('measured-cr', 'no /Length, CR before endstream', () =>
lengthStream('', '\r'),
),
]
// ---------------------------------------------------------------------------
// /Filter: /\/Filter\s*(\[\s*)?\/FlateDecode\b/
const filter = rule('/Filter regex', [
'findEmbeddedFiles',
'readAttachedDocument',
])
const filterStream = (dict: string, data: Uint8Array = FLATE_J) =>
craftPdf([
streamObject({
num: 2,
dict: ` /Type /EmbeddedFile ${dict} /Length ${data.byteLength} `,
data,
}),
])
const HEX_J = ascii(
`${Buffer.from(J, 'latin1').toString('hex').toUpperCase()}>`,
)
const FILTER: InputSpec[] = [
filter('filter-name', '/Filter /FlateDecode (control)', () =>
filterStream('/Filter /FlateDecode'),
),
filter('filter-unspaced', '/Filter/FlateDecode', () =>
filterStream('/Filter/FlateDecode'),
),
filter('filter-array', '/Filter [/FlateDecode]', () =>
filterStream('/Filter [/FlateDecode]'),
),
filter('filter-array-spaced', '/Filter [ /FlateDecode ]', () =>
filterStream('/Filter [ /FlateDecode ]'),
),
filter(
'filter-array-second',
'/Filter [/ASCIIHexDecode /FlateDecode] on Flate data',
() => filterStream('/Filter [/ASCIIHexDecode /FlateDecode]'),
),
filter('filter-asciihex', '/Filter /ASCIIHexDecode on hex data', () =>
filterStream('/Filter /ASCIIHexDecode', HEX_J),
),
filter(
'filter-asciihex-on-json',
'/Filter /ASCIIHexDecode on plain JSON',
() => filterStream('/Filter /ASCIIHexDecode', ascii(J)),
),
filter('filter-vt', 'VT after /Filter', () =>
filterStream('/Filter\x0B/FlateDecode'),
),
filter('filter-nbsp', 'NBSP after /Filter', () =>
filterStream('/Filter\xA0/FlateDecode'),
),
filter('filter-nul', 'NUL after /Filter', () =>
filterStream('/Filter\x00/FlateDecode'),
),
filter('filter-array-nul', 'NUL inside the array', () =>
filterStream('/Filter [\x00/FlateDecode]'),
),
filter('filter-name-continues', '/Filter /FlateDecodeX', () =>
filterStream('/Filter /FlateDecodeX'),
),
]
// ---------------------------------------------------------------------------
// Filespec: /EF removal (lazy), /F then /UF string token, /EF << /F n 0 R >>
const filespecRule = rule('Filespec /EF and /F regexes, string token', [
'findEmbeddedFiles',
])
const filespec = (...dicts: string[]) =>
pdfOf(...dicts.map((d, i) => `${i + 3} 0 obj\n<<${d}>>\nendobj\n`), EF2)
const FILESPEC: InputSpec[] = [
filespecRule('filespec-ef-vt', 'VT around /EF and the reference', () =>
filespec(
' /Type /Filespec /F (osteo-report.json) /EF\x0B<< /F\x0B2\x0B0\x0BR >> ',
),
),
filespecRule('filespec-ef-nbsp', 'NBSP around /EF and the reference', () =>
filespec(
' /Type /Filespec /F (osteo-report.json) /EF\xA0<< /F\xA02\xA00\xA0R >> ',
),
),
filespecRule('filespec-ef-nul', 'NUL between /EF and <<', () =>
filespec(' /Type /Filespec /F (osteo-report.json) /EF\x00<< /F 2 0 R >> '),
),
filespecRule('filespec-ref-nul', 'NUL between /F and the reference', () =>
filespec(' /Type /Filespec /F (osteo-report.json) /EF << /F\x002 0 R >> '),
),
filespecRule('filespec-ref-nel', 'NEL between /F and the reference', () =>
filespec(' /Type /Filespec /F (osteo-report.json) /EF << /F\x852 0 R >> '),
),
filespecRule(
'filespec-f-vt',
'VT between /F and its string, /UF fallback present',
() =>
filespec(
' /Type /Filespec /F\x0B(osteo-report.json) /UF (fallback.json) /EF << /F 2 0 R >> ',
),
),
filespecRule(
'filespec-f-nul',
'NUL between /F and its string, /UF fallback present',
() =>
filespec(
' /Type /Filespec /F\x00(osteo-report.json) /UF (fallback.json) /EF << /F 2 0 R >> ',
),
),
filespecRule('filespec-uf-only', '/UF only', () =>
filespec(' /Type /Filespec /UF (osteo-report.json) /EF << /F 2 0 R >> '),
),
filespecRule(
'filespec-lazy-ef',
'/EF holding a nested dictionary: removal stops at the first >>',
() =>
filespec(
' /Type /Filespec /EF << /F 2 0 R /Sub << >> /F (inner.json) >> /UF (outer.json) ',
),
),
filespecRule(
'filespec-tight',
'no spaces, /EF closing right before the Filespec >>',
() => filespec('/Type/Filespec/F(osteo-report.json)/EF<</F 2 0 R>>'),
),
filespecRule(
'filespec-two-for-one-stream',
'two Filespecs for one stream: the first wins',
() =>
filespec(
' /Type /Filespec /F (decoy.json) /EF << /F 2 0 R >> ',
' /Type /Filespec /F (osteo-report.json) /EF << /F 2 0 R >> ',
),
),
filespecRule(
'filespec-backslash-cr',
'backslash then CR inside the /F string',
() =>
filespec(
' /Type /Filespec /F (osteo-report\\\r.json) /UF (fallback.json) /EF << /F 2 0 R >> ',
),
),
filespecRule(
'filespec-backslash-lf',
'backslash then LF inside the /F string',
() =>
filespec(
' /Type /Filespec /F (osteo-report\\\n.json) /UF (fallback.json) /EF << /F 2 0 R >> ',
),
),
filespecRule('filespec-plain-cr', 'a bare CR inside the /F string', () =>
filespec(' /Type /Filespec /F (osteo-report\r.json) /EF << /F 2 0 R >> '),
),
filespecRule(
'filespec-escaped-paren',
'an escaped parenthesis inside the /F string',
() =>
filespec(
' /Type /Filespec /F (osteo-report\\).json) /EF << /F 2 0 R >> ',
),
),
filespecRule('filespec-hex-latin1', 'hex /F name without BOM', () =>
filespec(
` /Type /Filespec /F <${Buffer.from('osteo-report.json').toString('hex')}> /EF << /F 2 0 R >> `,
),
),
]
// ---------------------------------------------------------------------------
// /Keywords: isDelimiter after the key, string token, 4,096-byte window, 32 occurrences
const keywordsRule = rule('/Keywords stamp', [
'hasArchiveStamp',
'readAttachedDocument',
])
const STAMP = 'faunosteo-archive/1'
const keywords = (body: string) => pdfOf(`3 0 obj\n<< ${body} >>\nendobj\n`)
const hexWith = (text: string, insert: string) => {
const hex = Buffer.from(text, 'latin1').toString('hex')
return `<${hex.slice(0, 8)}${insert}${hex.slice(8)}>`
}
const utf16Literal = (text: string) =>
`(\xFE\xFF${Array.from(text, (c) => `\x00${c}`).join('')})`
/** A literal token of exactly `total` bytes ending with the stamp. */
const literalOfLength = (total: number) =>
`(${'x'.repeat(total - STAMP.length - 2)}${STAMP})`
const KEYWORDS: InputSpec[] = [
keywordsRule('keywords-literal', 'literal string (control)', () =>
keywords(`/Keywords (${STAMP})`),
),
keywordsRule('keywords-key-vt', 'VT after the key', () =>
keywords(`/Keywords\x0B(${STAMP})`),
),
keywordsRule('keywords-key-nbsp', 'NBSP after the key', () =>
keywords(`/Keywords\xA0(${STAMP})`),
),
keywordsRule('keywords-key-nul', 'NUL after the key', () =>
keywords(`/Keywords\x00(${STAMP})`),
),
keywordsRule('keywords-key-unspaced', 'the string right after the key', () =>
keywords(`/Keywords(${STAMP})`),
),
keywordsRule('keywords-key-continues', '/KeywordsX', () =>
keywords(`/KeywordsX (${STAMP})`),
),
keywordsRule('keywords-hex-vt', 'VT inside a hex string', () =>
keywords(`/Keywords ${hexWith(STAMP, '\x0B')}`),
),
keywordsRule('keywords-hex-nul', 'NUL inside a hex string', () =>
keywords(`/Keywords ${hexWith(STAMP, '\x00')}`),
),
keywordsRule(
'keywords-backslash-cr',
'backslash then CR inside the literal',
() => keywords(`/Keywords (${STAMP}\\\r)`),
),
keywordsRule('keywords-plain-cr', 'a bare CR inside the literal', () =>
keywords(`/Keywords (x\r${STAMP})`),
),
keywordsRule('keywords-literal-utf16', 'UTF-16BE literal with BOM', () =>
keywords(`/Keywords ${utf16Literal(STAMP)}`),
),
keywordsRule('keywords-token-4096', 'a 4,096-byte token', () =>
keywords(`/Keywords ${literalOfLength(4096)}`),
),
keywordsRule('keywords-token-4097', 'a 4,097-byte token', () =>
keywords(`/Keywords ${literalOfLength(4097)}`),
),
keywordsRule(
'keywords-32nd-occurrence',
'the stamp on the 32nd /Keywords',
() => keywords(`${'/Keywords (x) '.repeat(31)}/Keywords (${STAMP})`),
),
keywordsRule(
'keywords-33rd-occurrence',
'the stamp on the 33rd /Keywords',
() => keywords(`${'/Keywords (x) '.repeat(32)}/Keywords (${STAMP})`),
),
]
// ---------------------------------------------------------------------------
// isWhitespace / isDelimiter around /Type /EmbeddedFile and `stream`; dictionary bounds; caps
const scanRule = rule('isWhitespace and isDelimiter, dictionary bounds, caps', [
'findEmbeddedFiles',
'readAttachedDocument',
])
const efObject = (dict: string, between = '\n', eol = '\n') =>
`2 0 obj\n<<${dict}>>${between}stream${eol}${J}\nendstream\nendobj\n`
/** An `EmbeddedFile` dictionary of exactly `total` bytes from `<<` to `>>`. */
const efOfLength = (total: number) => {
const frame = '<< /Type /EmbeddedFile /Length 13 /Pad () >>'
const dict = `/Type /EmbeddedFile /Length 13 /Pad (${'x'.repeat(total - frame.length)}) `
return pdfOf(efObject(` ${dict}`))
}
const streamless = (count: number, type = '/Type /EmbeddedFile') =>
Array.from(
{ length: count },
(_, i) => `${i + 10} 0 obj\n<< ${type} >>\nendobj\n`,
).join('')
const SCAN_RULES: InputSpec[] = [
scanRule('type-nul-gap', 'NUL between /Type and /EmbeddedFile', () =>
pdfOf(efObject(' /Type\x00/EmbeddedFile /Length 13 ')),
),
scanRule('type-vt-gap', 'VT between /Type and /EmbeddedFile', () =>
pdfOf(efObject(' /Type\x0B/EmbeddedFile /Length 13 ')),
),
scanRule('type-nbsp-gap', 'NBSP between /Type and /EmbeddedFile', () =>
pdfOf(efObject(' /Type\xA0/EmbeddedFile /Length 13 ')),
),
scanRule('type-name-end-vt', 'VT right after /EmbeddedFile', () =>
pdfOf(efObject(' /Type /EmbeddedFile\x0B/Length 13 ')),
),
scanRule('type-name-end-nul', 'NUL right after /EmbeddedFile', () =>
pdfOf(efObject(' /Type /EmbeddedFile\x00/Length 13 ')),
),
scanRule('type-name-continues', '/EmbeddedFileX', () =>
pdfOf(efObject(' /Type /EmbeddedFileX /Length 13 ')),
),
scanRule('stream-after-nul', 'NUL between >> and stream', () =>
pdfOf(efObject(' /Type /EmbeddedFile /Length 13 ', '\x00')),
),
scanRule('stream-after-vt', 'VT between >> and stream', () =>
pdfOf(efObject(' /Type /EmbeddedFile /Length 13 ', '\x0B')),
),
scanRule('stream-cr-only', 'stream then CR alone', () =>
pdfOf(efObject(' /Type /EmbeddedFile /Length 13 ', '\n', '\r')),
),
scanRule('stream-space-lf', 'stream then space LF', () =>
pdfOf(efObject(' /Type /EmbeddedFile /Length 13 ', '\n', ' \n')),
),
scanRule(
'dict-length-4096',
'an EmbeddedFile dictionary of 4,096 bytes',
() => efOfLength(4096),
),
scanRule(
'dict-length-4097',
'an EmbeddedFile dictionary of 4,097 bytes',
() => efOfLength(4097),
),
scanRule(
'cap-32nd-dict-valid',
'31 streamless EmbeddedFile dictionaries, then a valid one',
() => pdfOf(streamless(31), EF2),
),
scanRule(
'cap-33rd-dict-valid',
'32 streamless EmbeddedFile dictionaries, then a valid one',
() => pdfOf(streamless(32), EF2),
),
scanRule(
'cap-ignores-non-names',
'40 /Type /EmbeddedFileX dictionaries, then a valid one',
() => pdfOf(streamless(40, '/Type /EmbeddedFileX'), EF2),
),
]
// ---------------------------------------------------------------------------
// One 16 MiB budget: a stored length over what is left ends the search
const budgetRule = rule('shared candidate budget', ['readAttachedDocument'])
/** A Flate candidate inflating to `output` bytes of `x`: charged, never JSON. */
const charged = (output: number) => {
const data = deflate(new Uint8Array(output).fill(0x78))
return streamObject({
num: 2,
dict: `/Type /EmbeddedFile /Filter /FlateDecode /Length ${data.byteLength}`,
data,
})
}
const raw = (num: number, text: string) =>
streamObject({
num,
dict: `/Type /EmbeddedFile /Length ${text.length}`,
data: ascii(text),
})
const flate = (num: number, data: Uint8Array) =>
streamObject({
num,
dict: `/Type /EmbeddedFile /Filter /FlateDecode /Length ${data.byteLength}`,
data,
})
const PAD_40 = `{"version":3,"pad":"${'a'.repeat(18)}"}`
const PAD_222 = `{"version":3,"pad":"${'a'.repeat(200)}"}`
const corrupt = Uint8Array.from(FLATE_J, (b, i) => (i > 2 ? b ^ 0x55 : b))
const BUDGET: InputSpec[] = [
budgetRule(
'budget-break-before-valid',
'20 bytes left, a 40-byte candidate, then a 13-byte one',
() =>
craftPdf([charged(MAX_ATTACHMENT_BYTES - 20), raw(3, PAD_40), raw(4, J)]),
),
budgetRule(
'budget-skip-control',
'20 bytes left, then a 13-byte candidate',
() => craftPdf([charged(MAX_ATTACHMENT_BYTES - 20), raw(4, J)]),
),
budgetRule(
'budget-exact-fit',
'13 bytes left, then a 13-byte candidate',
() => craftPdf([charged(MAX_ATTACHMENT_BYTES - 13), raw(4, J)]),
),
budgetRule(
'budget-one-short',
'12 bytes left, then a 13-byte candidate',
() => craftPdf([charged(MAX_ATTACHMENT_BYTES - 12), raw(4, J)]),
),
budgetRule(
'budget-inflate-limited-by-remaining',
'100 bytes left, a small Flate candidate inflating to 222 bytes, then a 13-byte one',
() =>
craftPdf([
charged(MAX_ATTACHMENT_BYTES - 100),
flate(3, deflate(ascii(PAD_222))),
raw(4, J),
]),
),
budgetRule(
'flate-corrupt-then-valid',
'a corrupt Flate candidate, then a raw one',
() => craftPdf([flate(2, corrupt), raw(3, J)]),
),
]
// ---------------------------------------------------------------------------
// `%PDF-` within the first 1,024 bytes; the 50 MiB file cap
const headerRule = rule('file header and size', ['readAttachedDocument'])
const headerAt = (offset: number) => {
const bytes = new Uint8Array(offset + 8)
bytes.set(ascii('%PDF-1.7'), offset)
return bytes
}
const FILE: InputSpec[] = [
headerRule('header-at-1019', '%PDF- at offset 1,019', () => headerAt(1019)),
headerRule('header-at-1020', '%PDF- at offset 1,020', () => headerAt(1020)),
headerRule('size-exactly-50mib', 'a file of exactly 50 MiB', () =>
concat([ascii('%PDF-1.7\n'), new Uint8Array(MAX_PDF_BYTES - 9)]),
),
]
export const SEMANTIC_INPUTS: InputSpec[] = [
...HEADER,
...LENGTH,
...FILTER,
...FILESPEC,
...KEYWORDS,
...SCAN_RULES,
...BUDGET,
...FILE,
]
// ---------------------------------------------------------------------------
// decodePdfString tokens (Latin-1 strings, one character per byte)
export interface TokenCase {
suite: string
case: string
token: string
}
const SCAN_TOKENS =
'crafted stream layouts > decodePdfString: literal with escapes, hex ASCII, hex UTF-16BE with BOM'
export const TOKEN_CASES: TokenCase[] = [
...['(a\\(b\\)c)', '<6F7374656F>', '<FEFF006F0073>', '/Name'].map(
(token) => ({
suite: 'scan',
case: SCAN_TOKENS,
token,
}),
),
...[
'(a\\nb)',
'(a\\\\b)',
'(\\157)',
'()',
'<>',
'<6F7>',
'<6F 73\x0B74>',
'<zz6F>',
'<FEFF>',
'<FEFF00>',
'<FEFF006F00>',
'(\xFE\xFF\x00o\x00s)',
'(\xE9t\xE9)',
'(abc',
'abc)',
].map((token) => ({
suite: 'semantics',
case: 'decodePdfString > literal unescape, hex digits, UTF-16BE BOM, Latin-1',
token,
})),
]