test(tests): R4.1d O7 reader outcomes (155 inputs, regex-semantics rows) and O9 inflate verdicts (Node)
This commit is contained in:
1 parent
87b88739f8
commit
96df15edae
187 files changed
+12207
No files matched your search
@@ -0,0 +1,105 @@
|
||||
import { deflateSync } from 'node:zlib'
|
||||
|
||||
/**
|
||||
* Byte builders of the R4 reader oracles, copied from scan.test.ts and
|
||||
* inflate.test.ts: a test file cannot be imported without registering its
|
||||
* cases, and those files stay untouched. Same output for the same arguments.
|
||||
*/
|
||||
|
||||
export const KiB = 1024
|
||||
export const MiB = 1024 * KiB
|
||||
|
||||
export const latin1 = (bytes: Uint8Array) =>
|
||||
Buffer.from(bytes).toString('latin1')
|
||||
/** One byte per character: every string given here stays at or below U+00FF. */
|
||||
export const ascii = (text: string) =>
|
||||
new Uint8Array(Buffer.from(text, 'latin1'))
|
||||
|
||||
export function concat(parts: Uint8Array[]): Uint8Array {
|
||||
const out = new Uint8Array(parts.reduce((n, p) => n + p.byteLength, 0))
|
||||
let at = 0
|
||||
for (const part of parts) {
|
||||
out.set(part, at)
|
||||
at += part.byteLength
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
export interface StreamSpec {
|
||||
num: number
|
||||
dict: string
|
||||
data: Uint8Array
|
||||
eol?: string
|
||||
}
|
||||
|
||||
export function streamObject({
|
||||
num,
|
||||
dict,
|
||||
data,
|
||||
eol = '\n',
|
||||
}: StreamSpec): Uint8Array {
|
||||
return concat([
|
||||
ascii(`${num} 0 obj\n<<${dict}>>\nstream${eol}`),
|
||||
data,
|
||||
ascii('\nendstream\nendobj\n'),
|
||||
])
|
||||
}
|
||||
|
||||
/** The scanner never reads the xref, so none is written. */
|
||||
export function craftPdf(objects: Uint8Array[]): Uint8Array {
|
||||
return concat([
|
||||
ascii('%PDF-1.7\n'),
|
||||
...objects,
|
||||
ascii('trailer\n<< /Root 1 0 R >>\n%%EOF\n'),
|
||||
])
|
||||
}
|
||||
|
||||
/** `craftPdf` over Latin-1 texts and byte parts. */
|
||||
export function pdfOf(...objects: Array<string | Uint8Array>): Uint8Array {
|
||||
return craftPdf(objects.map((o) => (typeof o === 'string' ? ascii(o) : o)))
|
||||
}
|
||||
|
||||
export interface Span {
|
||||
start: number
|
||||
end: number
|
||||
}
|
||||
|
||||
/** Bounds of the `n 0 obj … endobj` object whose dictionary holds `marker` (scan.test.ts `objectHolding`). */
|
||||
export function objectSpan(pdf: Uint8Array, marker: string): Span {
|
||||
const text = latin1(pdf)
|
||||
const at = text.indexOf(marker)
|
||||
if (at <= 0) throw new Error(`marker not found: ${marker}`)
|
||||
const start = text.lastIndexOf(' 0 obj', at)
|
||||
const headerStart = text.lastIndexOf('\n', start) + 1
|
||||
const end = text.indexOf('endobj', at) + 'endobj'.length
|
||||
return { start: headerStart, end }
|
||||
}
|
||||
|
||||
export function objectHolding(pdf: Uint8Array, marker: string): Uint8Array {
|
||||
const { start, end } = objectSpan(pdf, marker)
|
||||
return pdf.slice(start, end)
|
||||
}
|
||||
|
||||
export const filled = (byte: string, length: number) =>
|
||||
ascii(byte.repeat(length))
|
||||
|
||||
export const deflate = (data: Uint8Array) => new Uint8Array(deflateSync(data))
|
||||
|
||||
/** Incompressible bytes: the compressed stream is about as long as the input. */
|
||||
export function noise(length: number): Uint8Array {
|
||||
let seed = 0x9e3779b9
|
||||
return Uint8Array.from({ length }, () => {
|
||||
seed = (seed * 1664525 + 1013904223) >>> 0
|
||||
return seed >>> 24
|
||||
})
|
||||
}
|
||||
|
||||
/** CompressionStream('deflate') output, as security.test.ts builds its inputs. */
|
||||
export async function compressionStreamDeflate(
|
||||
data: Uint8Array,
|
||||
): Promise<Uint8Array> {
|
||||
const stream = new Blob([data])
|
||||
.stream()
|
||||
.pipeThrough(new CompressionStream('deflate'))
|
||||
return new Uint8Array(await new Response(stream).arrayBuffer())
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,215 @@
|
||||
import { createHash } from 'node:crypto'
|
||||
import {
|
||||
existsSync,
|
||||
mkdirSync,
|
||||
readdirSync,
|
||||
readFileSync,
|
||||
writeFileSync,
|
||||
} from 'node:fs'
|
||||
import { dirname, resolve } from 'node:path'
|
||||
import { expect } from 'vitest'
|
||||
import { inflateFlate } from '../../src/attachment/inflate'
|
||||
import { FIXTURES_DIR, REPO_ROOT } from './draft'
|
||||
import { ascii, concat, latin1, MiB } from './r4-reader-craft'
|
||||
|
||||
/**
|
||||
* Write-or-check plumbing of the R4.1 reader oracles (O7, O9). With
|
||||
* `MAKE_R4_ORACLES=1` the generators write their inputs and oracles; without
|
||||
* it they rebuild everything and require the committed files to be the same
|
||||
* (bytes for inputs, deep equality for JSON).
|
||||
*/
|
||||
|
||||
export const MAKE = process.env.MAKE_R4_ORACLES === '1'
|
||||
export const R4_INPUTS_DIR = resolve(FIXTURES_DIR, 'pdf/r4')
|
||||
export const R4_ORACLES_DIR = resolve(REPO_ROOT, 'crates/core/tests/oracles/r4')
|
||||
|
||||
/** Inputs above this size are committed as a generator description, never as bytes. */
|
||||
export const MAX_STORED_BYTES = 2 * MiB
|
||||
|
||||
export const sha256 = (data: Uint8Array | string): string =>
|
||||
createHash('sha256').update(data).digest('hex')
|
||||
|
||||
export const jsonText = (value: unknown): string =>
|
||||
`${JSON.stringify(value, null, 2)}\n`
|
||||
|
||||
export function relativeToRepo(path: string): string {
|
||||
return path.slice(REPO_ROOT.length + 1)
|
||||
}
|
||||
|
||||
export function storeFile(path: string, bytes: Uint8Array): void {
|
||||
if (MAKE) {
|
||||
mkdirSync(dirname(path), { recursive: true })
|
||||
writeFileSync(path, bytes)
|
||||
return
|
||||
}
|
||||
expect(existsSync(path), `missing ${relativeToRepo(path)}`).toBe(true)
|
||||
expect(sha256(readFileSync(path)), relativeToRepo(path)).toBe(sha256(bytes))
|
||||
}
|
||||
|
||||
export function readJsonFile<T>(path: string): T {
|
||||
return JSON.parse(readFileSync(path, 'utf8')) as T
|
||||
}
|
||||
|
||||
/**
|
||||
* `normalise` maps the committed JSON before the comparison, for the parts
|
||||
* this generator does not own (the Node version that wrote it, the browser
|
||||
* columns of O9).
|
||||
*/
|
||||
export function storeJson(
|
||||
path: string,
|
||||
value: unknown,
|
||||
normalise: (committed: unknown) => unknown = (committed) => committed,
|
||||
): void {
|
||||
if (MAKE) {
|
||||
storeFile(path, new TextEncoder().encode(jsonText(value)))
|
||||
return
|
||||
}
|
||||
expect(existsSync(path), `missing ${relativeToRepo(path)}`).toBe(true)
|
||||
expect(normalise(readJsonFile(path)), relativeToRepo(path)).toEqual(
|
||||
JSON.parse(jsonText(value)),
|
||||
)
|
||||
}
|
||||
|
||||
/** The Node build that produced an oracle: recorded, never compared (inputs are compared byte for byte). */
|
||||
export const generatedWith = () => ({
|
||||
node: process.version,
|
||||
zlib: process.versions.zlib,
|
||||
})
|
||||
|
||||
/** The input directory holds exactly the listed files: no stale input survives a rename. */
|
||||
export function expectOnlyFiles(dir: string, files: string[]): void {
|
||||
expect(readdirSync(dir).sort(), relativeToRepo(dir)).toEqual(
|
||||
[...new Set(files)].sort(),
|
||||
)
|
||||
}
|
||||
|
||||
/** inflateFlate's verdict as the oracles record it: null, or the output length and digest. */
|
||||
export async function inflateVerdict(input: Uint8Array, limit: number) {
|
||||
const out = await inflateFlate(input, limit)
|
||||
return out === null ? null : { length: out.byteLength, sha256: sha256(out) }
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Generator descriptions (`<slug>.gen.json`) for the inputs over 2 MiB
|
||||
|
||||
/**
|
||||
* Concatenated in order: `latin1` one byte per character, `fill` the byte
|
||||
* value repeated `count` times, `slice` the bytes [start, end) of another
|
||||
* input file of the same directory.
|
||||
*/
|
||||
export type Part =
|
||||
| { latin1: string }
|
||||
| { fill: number; count: number }
|
||||
| { slice: string; start: number; end: number }
|
||||
|
||||
export interface GeneratorDescription {
|
||||
generator: 'concat/1'
|
||||
length: number
|
||||
sha256: string
|
||||
parts: Part[]
|
||||
}
|
||||
|
||||
/** Bytes a description may reference instead of spelling them out. */
|
||||
export interface SliceRef {
|
||||
file: string
|
||||
start: number
|
||||
end: number
|
||||
bytes: Uint8Array
|
||||
}
|
||||
|
||||
export function expandParts(parts: Part[], dir: string): Uint8Array {
|
||||
return concat(
|
||||
parts.map((part) => {
|
||||
if ('latin1' in part) return ascii(part.latin1)
|
||||
if ('fill' in part) return new Uint8Array(part.count).fill(part.fill)
|
||||
const source = new Uint8Array(readFileSync(resolve(dir, part.slice)))
|
||||
return source.subarray(part.start, part.end)
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
/** Runs this long or longer of one byte value become a `fill` part. */
|
||||
const MIN_RUN = 4096
|
||||
|
||||
function runParts(bytes: Uint8Array, start: number, end: number): Part[] {
|
||||
const parts: Part[] = []
|
||||
let textFrom = start
|
||||
let i = start
|
||||
while (i < end) {
|
||||
let j = i + 1
|
||||
while (j < end && bytes[j] === bytes[i]) j += 1
|
||||
if (j - i >= MIN_RUN) {
|
||||
if (textFrom < i) {
|
||||
parts.push({ latin1: latin1(bytes.subarray(textFrom, i)) })
|
||||
}
|
||||
parts.push({ fill: bytes[i]!, count: j - i })
|
||||
textFrom = j
|
||||
}
|
||||
i = j
|
||||
}
|
||||
if (textFrom < end)
|
||||
parts.push({ latin1: latin1(bytes.subarray(textFrom, end)) })
|
||||
return parts
|
||||
}
|
||||
|
||||
interface RefHit {
|
||||
at: number
|
||||
ref: SliceRef
|
||||
}
|
||||
|
||||
function refHits(bytes: Uint8Array, refs: SliceRef[]): RefHit[] {
|
||||
const buffer = Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength)
|
||||
const hits: RefHit[] = []
|
||||
for (const ref of refs) {
|
||||
let at = buffer.indexOf(ref.bytes)
|
||||
while (at >= 0) {
|
||||
hits.push({ at, ref })
|
||||
at = buffer.indexOf(ref.bytes, at + ref.bytes.byteLength)
|
||||
}
|
||||
}
|
||||
return hits.sort((a, b) => a.at - b.at)
|
||||
}
|
||||
|
||||
/** Derives the description from the bytes the original test builds, so both stay the same input. */
|
||||
export function partsOf(bytes: Uint8Array, refs: SliceRef[]): Part[] {
|
||||
const parts: Part[] = []
|
||||
let at = 0
|
||||
for (const hit of refHits(bytes, refs)) {
|
||||
parts.push(...runParts(bytes, at, hit.at))
|
||||
parts.push({ slice: hit.ref.file, start: hit.ref.start, end: hit.ref.end })
|
||||
at = hit.at + hit.ref.bytes.byteLength
|
||||
}
|
||||
parts.push(...runParts(bytes, at, bytes.byteLength))
|
||||
return parts
|
||||
}
|
||||
|
||||
export interface StoredInput {
|
||||
file: string
|
||||
stored: 'bytes' | 'generator'
|
||||
length: number
|
||||
sha256: string
|
||||
}
|
||||
|
||||
/** Commits an input as bytes, or above MAX_STORED_BYTES as a description checked to expand to the same bytes. */
|
||||
export function storeInput(
|
||||
dir: string,
|
||||
name: { slug: string; ext: string },
|
||||
bytes: Uint8Array,
|
||||
refs: SliceRef[] = [],
|
||||
): StoredInput {
|
||||
const digest = { length: bytes.byteLength, sha256: sha256(bytes) }
|
||||
if (bytes.byteLength <= MAX_STORED_BYTES) {
|
||||
const file = `${name.slug}${name.ext}`
|
||||
storeFile(resolve(dir, file), bytes)
|
||||
return { file, stored: 'bytes', ...digest }
|
||||
}
|
||||
const file = `${name.slug}.gen.json`
|
||||
const description: GeneratorDescription = {
|
||||
generator: 'concat/1',
|
||||
...digest,
|
||||
parts: partsOf(bytes, refs),
|
||||
}
|
||||
storeJson(resolve(dir, file), description)
|
||||
expect(sha256(expandParts(description.parts, dir)), file).toBe(digest.sha256)
|
||||
return { file, stored: 'generator', ...digest }
|
||||
}
|
||||
@@ -0,0 +1,618 @@
|
||||
import { MAX_ATTACHMENT_BYTES, MAX_PDF_BYTES } from '../../src'
|
||||
import {
|
||||
ascii,
|
||||
concat,
|
||||
craftPdf,
|
||||
deflate,
|
||||
pdfOf,
|
||||
streamObject,
|
||||
} from './r4-reader-craft'
|
||||
import type { InputSpec, ReaderFunction } from './r4-reader-inputs'
|
||||
|
||||
/**
|
||||
* Plan §B.1.2 "JS regex semantics reproduced explicitly", one minimal PDF
|
||||
* per variant, plus the boundaries of the scanner's other rules (dictionary
|
||||
* and keyword windows, candidate and occurrence caps, header position, the
|
||||
* shared budget). Each pair differs only where the rule decides: `\s` (VT,
|
||||
* FF, NBSP yes; NUL, NEL no) against `isWhitespace` (NUL yes; VT, NBSP no),
|
||||
* `.` against CR and LF in the string token, the lazy `/EF` removal, the
|
||||
* `break` on a candidate over the remaining budget. The outcomes are what
|
||||
* the TypeScript does, recorded by the O7 driver, never what a PDF parser
|
||||
* would do.
|
||||
*/
|
||||
|
||||
const J = '{"version":3}'
|
||||
const FLATE_J = deflate(ascii(J))
|
||||
|
||||
const EF2 = `2 0 obj\n<< /Type /EmbeddedFile /Length 13 >>\nstream\n${J}\nendstream\nendobj\n`
|
||||
|
||||
type Build = InputSpec['build']
|
||||
|
||||
const rule =
|
||||
(group: string, exercises: ReaderFunction[]) =>
|
||||
(slug: string, title: string, build: Build): InputSpec => ({
|
||||
slug: `semantics.${slug}`,
|
||||
kind: 'pdf',
|
||||
cases: [{ suite: 'semantics', case: `${group} > ${title}`, exercises }],
|
||||
build,
|
||||
})
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// `n 0 obj` header: /(\d{1,10})\s+\d{1,10}\s+obj\s*$/ on a 48-byte window
|
||||
|
||||
const header = rule('object header regex (\\s, 48-byte window)', [
|
||||
'findEmbeddedFiles',
|
||||
])
|
||||
|
||||
const namedStream = (head: string, target = 2) =>
|
||||
pdfOf(
|
||||
`3 0 obj\n<< /Type /Filespec /F (osteo-report.json) /EF << /F ${target} 0 R >> >>\nendobj\n`,
|
||||
`${head}<< /Type /EmbeddedFile /Length 13 >>\nstream\n${J}\nendstream\nendobj\n`,
|
||||
)
|
||||
|
||||
const HEADER: InputSpec[] = [
|
||||
header('header-vt', 'VT (0x0B) separators', () =>
|
||||
namedStream('2\x0B0\x0Bobj\x0B'),
|
||||
),
|
||||
header('header-ff', 'FF (0x0C) separators', () =>
|
||||
namedStream('2\x0C0\x0Cobj\x0C'),
|
||||
),
|
||||
header('header-nbsp', 'NBSP (0xA0) separators', () =>
|
||||
namedStream('2\xA00\xA0obj\xA0'),
|
||||
),
|
||||
header('header-nel', 'NEL (0x85) between the numbers', () =>
|
||||
namedStream('2\x850\x85obj\n'),
|
||||
),
|
||||
header('header-nul', 'NUL between the numbers', () =>
|
||||
namedStream('2\x000 obj\n'),
|
||||
),
|
||||
header('header-nul-before-dict', 'NUL between obj and <<', () =>
|
||||
namedStream('2 0 obj\x00'),
|
||||
),
|
||||
header(
|
||||
'header-window-40-spaces',
|
||||
'12 0 obj then 40 spaces: the number fits the window',
|
||||
() => namedStream(`12 0 obj${' '.repeat(40)}`, 12),
|
||||
),
|
||||
header(
|
||||
'header-window-41-spaces',
|
||||
'12 0 obj then 41 spaces: the window cuts the 1',
|
||||
() => namedStream(`12 0 obj${' '.repeat(41)}`, 12),
|
||||
),
|
||||
]
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// /Length: /\/Length\s+(\d{1,10})(?!\d)(?!\s+\d+\s+R)/, then isWhitespace before endstream
|
||||
|
||||
const length = rule('/Length regex and endstream landing', [
|
||||
'findEmbeddedFiles',
|
||||
'readAttachedDocument',
|
||||
])
|
||||
|
||||
/** Three spaces after the data: a direct /Length gives 13, the measured length 16. */
|
||||
const lengthStream = (dict: string, tail = ' \n') =>
|
||||
pdfOf(
|
||||
`2 0 obj\n<< /Type /EmbeddedFile ${dict} >>\nstream\n${J}${tail}endstream\nendobj\n`,
|
||||
)
|
||||
|
||||
const LENGTH: InputSpec[] = [
|
||||
length('length-space', 'space after /Length (control)', () =>
|
||||
lengthStream('/Length 13'),
|
||||
),
|
||||
length('length-vt', 'VT after /Length', () => lengthStream('/Length\x0B13')),
|
||||
length('length-nbsp', 'NBSP after /Length', () =>
|
||||
lengthStream('/Length\xA013'),
|
||||
),
|
||||
length('length-nel', 'NEL after /Length', () =>
|
||||
lengthStream('/Length\x8513'),
|
||||
),
|
||||
length('length-nul', 'NUL after /Length', () =>
|
||||
lengthStream('/Length\x0013'),
|
||||
),
|
||||
length('length-indirect-space', '13 0 R with spaces (indirect)', () =>
|
||||
lengthStream('/Length 13 0 R'),
|
||||
),
|
||||
length('length-indirect-vt', '13 0 R with VT separators', () =>
|
||||
lengthStream('/Length 13\x0B0\x0BR'),
|
||||
),
|
||||
length('length-indirect-nul', '13 0 R with NUL separators', () =>
|
||||
lengthStream('/Length 13\x000\x00R'),
|
||||
),
|
||||
length('length-10-digits', 'ten digits 0000000013', () =>
|
||||
lengthStream('/Length 0000000013'),
|
||||
),
|
||||
length('length-11-digits', 'eleven digits 00000000013', () =>
|
||||
lengthStream('/Length 00000000013'),
|
||||
),
|
||||
length('length-past-end', 'a direct length past the end of the file', () =>
|
||||
lengthStream('/Length 99999'),
|
||||
),
|
||||
length('length-tail-nul', 'NUL between the data and endstream', () =>
|
||||
lengthStream('/Length 13', '\x00\n'),
|
||||
),
|
||||
length('length-tail-vt', 'VT between the data and endstream', () =>
|
||||
lengthStream('/Length 13', '\x0B\n'),
|
||||
),
|
||||
length('measured-lf', 'no /Length, LF before endstream', () =>
|
||||
lengthStream('', '\n'),
|
||||
),
|
||||
length('measured-crlf', 'no /Length, CRLF before endstream', () =>
|
||||
lengthStream('', '\r\n'),
|
||||
),
|
||||
length('measured-lfcr', 'no /Length, LF CR before endstream', () =>
|
||||
lengthStream('', '\n\r'),
|
||||
),
|
||||
length('measured-cr', 'no /Length, CR before endstream', () =>
|
||||
lengthStream('', '\r'),
|
||||
),
|
||||
]
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// /Filter: /\/Filter\s*(\[\s*)?\/FlateDecode\b/
|
||||
|
||||
const filter = rule('/Filter regex', [
|
||||
'findEmbeddedFiles',
|
||||
'readAttachedDocument',
|
||||
])
|
||||
|
||||
const filterStream = (dict: string, data: Uint8Array = FLATE_J) =>
|
||||
craftPdf([
|
||||
streamObject({
|
||||
num: 2,
|
||||
dict: ` /Type /EmbeddedFile ${dict} /Length ${data.byteLength} `,
|
||||
data,
|
||||
}),
|
||||
])
|
||||
|
||||
const HEX_J = ascii(
|
||||
`${Buffer.from(J, 'latin1').toString('hex').toUpperCase()}>`,
|
||||
)
|
||||
|
||||
const FILTER: InputSpec[] = [
|
||||
filter('filter-name', '/Filter /FlateDecode (control)', () =>
|
||||
filterStream('/Filter /FlateDecode'),
|
||||
),
|
||||
filter('filter-unspaced', '/Filter/FlateDecode', () =>
|
||||
filterStream('/Filter/FlateDecode'),
|
||||
),
|
||||
filter('filter-array', '/Filter [/FlateDecode]', () =>
|
||||
filterStream('/Filter [/FlateDecode]'),
|
||||
),
|
||||
filter('filter-array-spaced', '/Filter [ /FlateDecode ]', () =>
|
||||
filterStream('/Filter [ /FlateDecode ]'),
|
||||
),
|
||||
filter(
|
||||
'filter-array-second',
|
||||
'/Filter [/ASCIIHexDecode /FlateDecode] on Flate data',
|
||||
() => filterStream('/Filter [/ASCIIHexDecode /FlateDecode]'),
|
||||
),
|
||||
filter('filter-asciihex', '/Filter /ASCIIHexDecode on hex data', () =>
|
||||
filterStream('/Filter /ASCIIHexDecode', HEX_J),
|
||||
),
|
||||
filter(
|
||||
'filter-asciihex-on-json',
|
||||
'/Filter /ASCIIHexDecode on plain JSON',
|
||||
() => filterStream('/Filter /ASCIIHexDecode', ascii(J)),
|
||||
),
|
||||
filter('filter-vt', 'VT after /Filter', () =>
|
||||
filterStream('/Filter\x0B/FlateDecode'),
|
||||
),
|
||||
filter('filter-nbsp', 'NBSP after /Filter', () =>
|
||||
filterStream('/Filter\xA0/FlateDecode'),
|
||||
),
|
||||
filter('filter-nul', 'NUL after /Filter', () =>
|
||||
filterStream('/Filter\x00/FlateDecode'),
|
||||
),
|
||||
filter('filter-array-nul', 'NUL inside the array', () =>
|
||||
filterStream('/Filter [\x00/FlateDecode]'),
|
||||
),
|
||||
filter('filter-name-continues', '/Filter /FlateDecodeX', () =>
|
||||
filterStream('/Filter /FlateDecodeX'),
|
||||
),
|
||||
]
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Filespec: /EF removal (lazy), /F then /UF string token, /EF << /F n 0 R >>
|
||||
|
||||
const filespecRule = rule('Filespec /EF and /F regexes, string token', [
|
||||
'findEmbeddedFiles',
|
||||
])
|
||||
|
||||
const filespec = (...dicts: string[]) =>
|
||||
pdfOf(...dicts.map((d, i) => `${i + 3} 0 obj\n<<${d}>>\nendobj\n`), EF2)
|
||||
|
||||
const FILESPEC: InputSpec[] = [
|
||||
filespecRule('filespec-ef-vt', 'VT around /EF and the reference', () =>
|
||||
filespec(
|
||||
' /Type /Filespec /F (osteo-report.json) /EF\x0B<< /F\x0B2\x0B0\x0BR >> ',
|
||||
),
|
||||
),
|
||||
filespecRule('filespec-ef-nbsp', 'NBSP around /EF and the reference', () =>
|
||||
filespec(
|
||||
' /Type /Filespec /F (osteo-report.json) /EF\xA0<< /F\xA02\xA00\xA0R >> ',
|
||||
),
|
||||
),
|
||||
filespecRule('filespec-ef-nul', 'NUL between /EF and <<', () =>
|
||||
filespec(' /Type /Filespec /F (osteo-report.json) /EF\x00<< /F 2 0 R >> '),
|
||||
),
|
||||
filespecRule('filespec-ref-nul', 'NUL between /F and the reference', () =>
|
||||
filespec(' /Type /Filespec /F (osteo-report.json) /EF << /F\x002 0 R >> '),
|
||||
),
|
||||
filespecRule('filespec-ref-nel', 'NEL between /F and the reference', () =>
|
||||
filespec(' /Type /Filespec /F (osteo-report.json) /EF << /F\x852 0 R >> '),
|
||||
),
|
||||
filespecRule(
|
||||
'filespec-f-vt',
|
||||
'VT between /F and its string, /UF fallback present',
|
||||
() =>
|
||||
filespec(
|
||||
' /Type /Filespec /F\x0B(osteo-report.json) /UF (fallback.json) /EF << /F 2 0 R >> ',
|
||||
),
|
||||
),
|
||||
filespecRule(
|
||||
'filespec-f-nul',
|
||||
'NUL between /F and its string, /UF fallback present',
|
||||
() =>
|
||||
filespec(
|
||||
' /Type /Filespec /F\x00(osteo-report.json) /UF (fallback.json) /EF << /F 2 0 R >> ',
|
||||
),
|
||||
),
|
||||
filespecRule('filespec-uf-only', '/UF only', () =>
|
||||
filespec(' /Type /Filespec /UF (osteo-report.json) /EF << /F 2 0 R >> '),
|
||||
),
|
||||
filespecRule(
|
||||
'filespec-lazy-ef',
|
||||
'/EF holding a nested dictionary: removal stops at the first >>',
|
||||
() =>
|
||||
filespec(
|
||||
' /Type /Filespec /EF << /F 2 0 R /Sub << >> /F (inner.json) >> /UF (outer.json) ',
|
||||
),
|
||||
),
|
||||
filespecRule(
|
||||
'filespec-tight',
|
||||
'no spaces, /EF closing right before the Filespec >>',
|
||||
() => filespec('/Type/Filespec/F(osteo-report.json)/EF<</F 2 0 R>>'),
|
||||
),
|
||||
filespecRule(
|
||||
'filespec-two-for-one-stream',
|
||||
'two Filespecs for one stream: the first wins',
|
||||
() =>
|
||||
filespec(
|
||||
' /Type /Filespec /F (decoy.json) /EF << /F 2 0 R >> ',
|
||||
' /Type /Filespec /F (osteo-report.json) /EF << /F 2 0 R >> ',
|
||||
),
|
||||
),
|
||||
filespecRule(
|
||||
'filespec-backslash-cr',
|
||||
'backslash then CR inside the /F string',
|
||||
() =>
|
||||
filespec(
|
||||
' /Type /Filespec /F (osteo-report\\\r.json) /UF (fallback.json) /EF << /F 2 0 R >> ',
|
||||
),
|
||||
),
|
||||
filespecRule(
|
||||
'filespec-backslash-lf',
|
||||
'backslash then LF inside the /F string',
|
||||
() =>
|
||||
filespec(
|
||||
' /Type /Filespec /F (osteo-report\\\n.json) /UF (fallback.json) /EF << /F 2 0 R >> ',
|
||||
),
|
||||
),
|
||||
filespecRule('filespec-plain-cr', 'a bare CR inside the /F string', () =>
|
||||
filespec(' /Type /Filespec /F (osteo-report\r.json) /EF << /F 2 0 R >> '),
|
||||
),
|
||||
filespecRule(
|
||||
'filespec-escaped-paren',
|
||||
'an escaped parenthesis inside the /F string',
|
||||
() =>
|
||||
filespec(
|
||||
' /Type /Filespec /F (osteo-report\\).json) /EF << /F 2 0 R >> ',
|
||||
),
|
||||
),
|
||||
filespecRule('filespec-hex-latin1', 'hex /F name without BOM', () =>
|
||||
filespec(
|
||||
` /Type /Filespec /F <${Buffer.from('osteo-report.json').toString('hex')}> /EF << /F 2 0 R >> `,
|
||||
),
|
||||
),
|
||||
]
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// /Keywords: isDelimiter after the key, string token, 4,096-byte window, 32 occurrences
|
||||
|
||||
const keywordsRule = rule('/Keywords stamp', [
|
||||
'hasArchiveStamp',
|
||||
'readAttachedDocument',
|
||||
])
|
||||
|
||||
const STAMP = 'faunosteo-archive/1'
|
||||
const keywords = (body: string) => pdfOf(`3 0 obj\n<< ${body} >>\nendobj\n`)
|
||||
const hexWith = (text: string, insert: string) => {
|
||||
const hex = Buffer.from(text, 'latin1').toString('hex')
|
||||
return `<${hex.slice(0, 8)}${insert}${hex.slice(8)}>`
|
||||
}
|
||||
const utf16Literal = (text: string) =>
|
||||
`(\xFE\xFF${Array.from(text, (c) => `\x00${c}`).join('')})`
|
||||
/** A literal token of exactly `total` bytes ending with the stamp. */
|
||||
const literalOfLength = (total: number) =>
|
||||
`(${'x'.repeat(total - STAMP.length - 2)}${STAMP})`
|
||||
|
||||
const KEYWORDS: InputSpec[] = [
|
||||
keywordsRule('keywords-literal', 'literal string (control)', () =>
|
||||
keywords(`/Keywords (${STAMP})`),
|
||||
),
|
||||
keywordsRule('keywords-key-vt', 'VT after the key', () =>
|
||||
keywords(`/Keywords\x0B(${STAMP})`),
|
||||
),
|
||||
keywordsRule('keywords-key-nbsp', 'NBSP after the key', () =>
|
||||
keywords(`/Keywords\xA0(${STAMP})`),
|
||||
),
|
||||
keywordsRule('keywords-key-nul', 'NUL after the key', () =>
|
||||
keywords(`/Keywords\x00(${STAMP})`),
|
||||
),
|
||||
keywordsRule('keywords-key-unspaced', 'the string right after the key', () =>
|
||||
keywords(`/Keywords(${STAMP})`),
|
||||
),
|
||||
keywordsRule('keywords-key-continues', '/KeywordsX', () =>
|
||||
keywords(`/KeywordsX (${STAMP})`),
|
||||
),
|
||||
keywordsRule('keywords-hex-vt', 'VT inside a hex string', () =>
|
||||
keywords(`/Keywords ${hexWith(STAMP, '\x0B')}`),
|
||||
),
|
||||
keywordsRule('keywords-hex-nul', 'NUL inside a hex string', () =>
|
||||
keywords(`/Keywords ${hexWith(STAMP, '\x00')}`),
|
||||
),
|
||||
keywordsRule(
|
||||
'keywords-backslash-cr',
|
||||
'backslash then CR inside the literal',
|
||||
() => keywords(`/Keywords (${STAMP}\\\r)`),
|
||||
),
|
||||
keywordsRule('keywords-plain-cr', 'a bare CR inside the literal', () =>
|
||||
keywords(`/Keywords (x\r${STAMP})`),
|
||||
),
|
||||
keywordsRule('keywords-literal-utf16', 'UTF-16BE literal with BOM', () =>
|
||||
keywords(`/Keywords ${utf16Literal(STAMP)}`),
|
||||
),
|
||||
keywordsRule('keywords-token-4096', 'a 4,096-byte token', () =>
|
||||
keywords(`/Keywords ${literalOfLength(4096)}`),
|
||||
),
|
||||
keywordsRule('keywords-token-4097', 'a 4,097-byte token', () =>
|
||||
keywords(`/Keywords ${literalOfLength(4097)}`),
|
||||
),
|
||||
keywordsRule(
|
||||
'keywords-32nd-occurrence',
|
||||
'the stamp on the 32nd /Keywords',
|
||||
() => keywords(`${'/Keywords (x) '.repeat(31)}/Keywords (${STAMP})`),
|
||||
),
|
||||
keywordsRule(
|
||||
'keywords-33rd-occurrence',
|
||||
'the stamp on the 33rd /Keywords',
|
||||
() => keywords(`${'/Keywords (x) '.repeat(32)}/Keywords (${STAMP})`),
|
||||
),
|
||||
]
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// isWhitespace / isDelimiter around /Type /EmbeddedFile and `stream`; dictionary bounds; caps
|
||||
|
||||
const scanRule = rule('isWhitespace and isDelimiter, dictionary bounds, caps', [
|
||||
'findEmbeddedFiles',
|
||||
'readAttachedDocument',
|
||||
])
|
||||
|
||||
const efObject = (dict: string, between = '\n', eol = '\n') =>
|
||||
`2 0 obj\n<<${dict}>>${between}stream${eol}${J}\nendstream\nendobj\n`
|
||||
|
||||
/** An `EmbeddedFile` dictionary of exactly `total` bytes from `<<` to `>>`. */
|
||||
const efOfLength = (total: number) => {
|
||||
const frame = '<< /Type /EmbeddedFile /Length 13 /Pad () >>'
|
||||
const dict = `/Type /EmbeddedFile /Length 13 /Pad (${'x'.repeat(total - frame.length)}) `
|
||||
return pdfOf(efObject(` ${dict}`))
|
||||
}
|
||||
|
||||
const streamless = (count: number, type = '/Type /EmbeddedFile') =>
|
||||
Array.from(
|
||||
{ length: count },
|
||||
(_, i) => `${i + 10} 0 obj\n<< ${type} >>\nendobj\n`,
|
||||
).join('')
|
||||
|
||||
const SCAN_RULES: InputSpec[] = [
|
||||
scanRule('type-nul-gap', 'NUL between /Type and /EmbeddedFile', () =>
|
||||
pdfOf(efObject(' /Type\x00/EmbeddedFile /Length 13 ')),
|
||||
),
|
||||
scanRule('type-vt-gap', 'VT between /Type and /EmbeddedFile', () =>
|
||||
pdfOf(efObject(' /Type\x0B/EmbeddedFile /Length 13 ')),
|
||||
),
|
||||
scanRule('type-nbsp-gap', 'NBSP between /Type and /EmbeddedFile', () =>
|
||||
pdfOf(efObject(' /Type\xA0/EmbeddedFile /Length 13 ')),
|
||||
),
|
||||
scanRule('type-name-end-vt', 'VT right after /EmbeddedFile', () =>
|
||||
pdfOf(efObject(' /Type /EmbeddedFile\x0B/Length 13 ')),
|
||||
),
|
||||
scanRule('type-name-end-nul', 'NUL right after /EmbeddedFile', () =>
|
||||
pdfOf(efObject(' /Type /EmbeddedFile\x00/Length 13 ')),
|
||||
),
|
||||
scanRule('type-name-continues', '/EmbeddedFileX', () =>
|
||||
pdfOf(efObject(' /Type /EmbeddedFileX /Length 13 ')),
|
||||
),
|
||||
scanRule('stream-after-nul', 'NUL between >> and stream', () =>
|
||||
pdfOf(efObject(' /Type /EmbeddedFile /Length 13 ', '\x00')),
|
||||
),
|
||||
scanRule('stream-after-vt', 'VT between >> and stream', () =>
|
||||
pdfOf(efObject(' /Type /EmbeddedFile /Length 13 ', '\x0B')),
|
||||
),
|
||||
scanRule('stream-cr-only', 'stream then CR alone', () =>
|
||||
pdfOf(efObject(' /Type /EmbeddedFile /Length 13 ', '\n', '\r')),
|
||||
),
|
||||
scanRule('stream-space-lf', 'stream then space LF', () =>
|
||||
pdfOf(efObject(' /Type /EmbeddedFile /Length 13 ', '\n', ' \n')),
|
||||
),
|
||||
scanRule(
|
||||
'dict-length-4096',
|
||||
'an EmbeddedFile dictionary of 4,096 bytes',
|
||||
() => efOfLength(4096),
|
||||
),
|
||||
scanRule(
|
||||
'dict-length-4097',
|
||||
'an EmbeddedFile dictionary of 4,097 bytes',
|
||||
() => efOfLength(4097),
|
||||
),
|
||||
scanRule(
|
||||
'cap-32nd-dict-valid',
|
||||
'31 streamless EmbeddedFile dictionaries, then a valid one',
|
||||
() => pdfOf(streamless(31), EF2),
|
||||
),
|
||||
scanRule(
|
||||
'cap-33rd-dict-valid',
|
||||
'32 streamless EmbeddedFile dictionaries, then a valid one',
|
||||
() => pdfOf(streamless(32), EF2),
|
||||
),
|
||||
scanRule(
|
||||
'cap-ignores-non-names',
|
||||
'40 /Type /EmbeddedFileX dictionaries, then a valid one',
|
||||
() => pdfOf(streamless(40, '/Type /EmbeddedFileX'), EF2),
|
||||
),
|
||||
]
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// One 16 MiB budget: a stored length over what is left ends the search
|
||||
|
||||
const budgetRule = rule('shared candidate budget', ['readAttachedDocument'])
|
||||
|
||||
/** A Flate candidate inflating to `output` bytes of `x`: charged, never JSON. */
|
||||
const charged = (output: number) => {
|
||||
const data = deflate(new Uint8Array(output).fill(0x78))
|
||||
return streamObject({
|
||||
num: 2,
|
||||
dict: `/Type /EmbeddedFile /Filter /FlateDecode /Length ${data.byteLength}`,
|
||||
data,
|
||||
})
|
||||
}
|
||||
|
||||
const raw = (num: number, text: string) =>
|
||||
streamObject({
|
||||
num,
|
||||
dict: `/Type /EmbeddedFile /Length ${text.length}`,
|
||||
data: ascii(text),
|
||||
})
|
||||
|
||||
const flate = (num: number, data: Uint8Array) =>
|
||||
streamObject({
|
||||
num,
|
||||
dict: `/Type /EmbeddedFile /Filter /FlateDecode /Length ${data.byteLength}`,
|
||||
data,
|
||||
})
|
||||
|
||||
const PAD_40 = `{"version":3,"pad":"${'a'.repeat(18)}"}`
|
||||
const PAD_222 = `{"version":3,"pad":"${'a'.repeat(200)}"}`
|
||||
const corrupt = Uint8Array.from(FLATE_J, (b, i) => (i > 2 ? b ^ 0x55 : b))
|
||||
|
||||
const BUDGET: InputSpec[] = [
|
||||
budgetRule(
|
||||
'budget-break-before-valid',
|
||||
'20 bytes left, a 40-byte candidate, then a 13-byte one',
|
||||
() =>
|
||||
craftPdf([charged(MAX_ATTACHMENT_BYTES - 20), raw(3, PAD_40), raw(4, J)]),
|
||||
),
|
||||
budgetRule(
|
||||
'budget-skip-control',
|
||||
'20 bytes left, then a 13-byte candidate',
|
||||
() => craftPdf([charged(MAX_ATTACHMENT_BYTES - 20), raw(4, J)]),
|
||||
),
|
||||
budgetRule(
|
||||
'budget-exact-fit',
|
||||
'13 bytes left, then a 13-byte candidate',
|
||||
() => craftPdf([charged(MAX_ATTACHMENT_BYTES - 13), raw(4, J)]),
|
||||
),
|
||||
budgetRule(
|
||||
'budget-one-short',
|
||||
'12 bytes left, then a 13-byte candidate',
|
||||
() => craftPdf([charged(MAX_ATTACHMENT_BYTES - 12), raw(4, J)]),
|
||||
),
|
||||
budgetRule(
|
||||
'budget-inflate-limited-by-remaining',
|
||||
'100 bytes left, a small Flate candidate inflating to 222 bytes, then a 13-byte one',
|
||||
() =>
|
||||
craftPdf([
|
||||
charged(MAX_ATTACHMENT_BYTES - 100),
|
||||
flate(3, deflate(ascii(PAD_222))),
|
||||
raw(4, J),
|
||||
]),
|
||||
),
|
||||
budgetRule(
|
||||
'flate-corrupt-then-valid',
|
||||
'a corrupt Flate candidate, then a raw one',
|
||||
() => craftPdf([flate(2, corrupt), raw(3, J)]),
|
||||
),
|
||||
]
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// `%PDF-` within the first 1,024 bytes; the 50 MiB file cap
|
||||
|
||||
const headerRule = rule('file header and size', ['readAttachedDocument'])
|
||||
|
||||
const headerAt = (offset: number) => {
|
||||
const bytes = new Uint8Array(offset + 8)
|
||||
bytes.set(ascii('%PDF-1.7'), offset)
|
||||
return bytes
|
||||
}
|
||||
|
||||
const FILE: InputSpec[] = [
|
||||
headerRule('header-at-1019', '%PDF- at offset 1,019', () => headerAt(1019)),
|
||||
headerRule('header-at-1020', '%PDF- at offset 1,020', () => headerAt(1020)),
|
||||
headerRule('size-exactly-50mib', 'a file of exactly 50 MiB', () =>
|
||||
concat([ascii('%PDF-1.7\n'), new Uint8Array(MAX_PDF_BYTES - 9)]),
|
||||
),
|
||||
]
|
||||
|
||||
export const SEMANTIC_INPUTS: InputSpec[] = [
|
||||
...HEADER,
|
||||
...LENGTH,
|
||||
...FILTER,
|
||||
...FILESPEC,
|
||||
...KEYWORDS,
|
||||
...SCAN_RULES,
|
||||
...BUDGET,
|
||||
...FILE,
|
||||
]
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// decodePdfString tokens (Latin-1 strings, one character per byte)
|
||||
|
||||
export interface TokenCase {
|
||||
suite: string
|
||||
case: string
|
||||
token: string
|
||||
}
|
||||
|
||||
const SCAN_TOKENS =
|
||||
'crafted stream layouts > decodePdfString: literal with escapes, hex ASCII, hex UTF-16BE with BOM'
|
||||
|
||||
export const TOKEN_CASES: TokenCase[] = [
|
||||
...['(a\\(b\\)c)', '<6F7374656F>', '<FEFF006F0073>', '/Name'].map(
|
||||
(token) => ({
|
||||
suite: 'scan',
|
||||
case: SCAN_TOKENS,
|
||||
token,
|
||||
}),
|
||||
),
|
||||
...[
|
||||
'(a\\nb)',
|
||||
'(a\\\\b)',
|
||||
'(\\157)',
|
||||
'()',
|
||||
'<>',
|
||||
'<6F7>',
|
||||
'<6F 73\x0B74>',
|
||||
'<zz6F>',
|
||||
'<FEFF>',
|
||||
'<FEFF00>',
|
||||
'<FEFF006F00>',
|
||||
'(\xFE\xFF\x00o\x00s)',
|
||||
'(\xE9t\xE9)',
|
||||
'(abc',
|
||||
'abc)',
|
||||
].map((token) => ({
|
||||
suite: 'semantics',
|
||||
case: 'decodePdfString > literal unescape, hex digits, UTF-16BE BOM, Latin-1',
|
||||
token,
|
||||
})),
|
||||
]
|
||||
Reference in new issue
Block a user