Files
faunosteo/packages/pdf/tests/scan.test.ts
T

609 lines
21 KiB
TypeScript
Raw Blame History

This file contains invisible Unicode characters
This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
/**
* BLK-009: the byte-scan attachment reader. Candidates on a fresh export and
* on the frozen fixtures (object-stream layout), crafted layouts (two files,
* indirect `/Length`, `\r\n`, no space after `/Type`), garbage, the keyword
* stamp, the object-stream bomb the reader never inflates, and the candidate
* cost bounds (monotonic `endstream` search over offset-sorted candidates,
* shared 16 MiB budget, `{` check, bracket-depth pre-scan before JSON.parse).
*/
import { deflateSync } from 'node:zlib'
import { beforeAll, describe, expect, it, vi } from 'vitest'
import { buildSheetSnapshot, type Report } from '@faunosteo/domain'
import {
ArchiveReadError,
defaultPdfLabels,
exportReportArchive,
findEmbeddedFiles,
readAttachedDocument,
readAttachmentBytes,
readReportArchive,
type EmbeddedFileCandidate,
} from '../src'
import { inflateFlate } from '../src/attachment/inflate'
import {
decodePdfString,
hasArchiveStamp,
locateAttachment,
} from '../src/attachment/scan'
import { readFixtureBytes } from './helpers/draft'
import {
FIXED_EXPORTED_AT,
nologosBranding,
rasterizerOf,
} from './helpers/goldens'
import { pdfWithAttachment, pdfWithoutAttachment } from './helpers/pdf'
import { loadGoldenReports, type GoldenReports } from './helpers/reports'
let golden: GoldenReports
let exported: Uint8Array
const map = () => golden.draft.migrationMap
beforeAll(async () => {
golden = loadGoldenReports()
exported = (await exportDemo()).pdf
})
async function exportDemo() {
const report: Report = golden.reports.demo
return exportReportArchive({
report,
sheet: buildSheetSnapshot(golden.draft, report),
branding: nologosBranding(),
labels: defaultPdfLabels,
locale: 'fr',
rasterizer: rasterizerOf('nologos'),
exportedAt: FIXED_EXPORTED_AT,
})
}
// ---------------------------------------------------------------------------
// Hand-built PDFs: the scanner never reads the xref, so none is written
const latin1 = (bytes: Uint8Array) => Buffer.from(bytes).toString('latin1')
const ascii = (text: string) => new Uint8Array(Buffer.from(text, 'latin1'))
function concat(parts: Uint8Array[]): Uint8Array {
const out = new Uint8Array(parts.reduce((n, p) => n + p.byteLength, 0))
let at = 0
for (const part of parts) {
out.set(part, at)
at += part.byteLength
}
return out
}
interface StreamSpec {
num: number
dict: string
data: Uint8Array
eol?: string
}
function streamObject({ num, dict, data, eol = '\n' }: StreamSpec): Uint8Array {
return concat([
ascii(`${num} 0 obj\n<<${dict}>>\nstream${eol}`),
data,
ascii('\nendstream\nendobj\n'),
])
}
function craftPdf(objects: Uint8Array[]): Uint8Array {
return concat([
ascii('%PDF-1.7\n'),
...objects,
ascii('trailer\n<< /Root 1 0 R >>\n%%EOF\n'),
])
}
/** The `n 0 obj … endobj` bytes of the object whose dictionary holds `marker`. */
function objectHolding(pdf: Uint8Array, marker: string): Uint8Array {
const text = latin1(pdf)
const at = text.indexOf(marker)
expect(at).toBeGreaterThan(0)
const start = text.lastIndexOf(' 0 obj', at)
const headerStart = text.lastIndexOf('\n', start) + 1
const end = text.indexOf('endobj', at) + 'endobj'.length
return pdf.slice(headerStart, end)
}
const BOMB_RAW = 64 * 1024 * 1024
const bomb = () => new Uint8Array(deflateSync(new Uint8Array(BOMB_RAW)))
async function rejects(
bytes: Uint8Array,
kind: ArchiveReadError['kind'],
): Promise<void> {
const promise = readReportArchive(bytes, map())
await expect(promise).rejects.toBeInstanceOf(ArchiveReadError)
await expect(promise).rejects.toMatchObject({ kind })
}
// ---------------------------------------------------------------------------
describe('findEmbeddedFiles on a fresh export (no object streams)', () => {
let candidates: EmbeddedFileCandidate[]
beforeAll(() => {
candidates = findEmbeddedFiles(exported)
})
it('finds the one Flate stream, named through its visible Filespec', async () => {
expect(candidates).toHaveLength(1)
const [c] = candidates
expect(c!.filter).toBe('FlateDecode')
expect(c!.name).toBe('osteo-report.json')
const inflated = await inflateFlate(
exported.subarray(c!.offset, c!.offset + c!.length),
)
const json = JSON.parse(new TextDecoder().decode(inflated!)) as {
envelopeVersion: number
}
expect(json.envelopeVersion).toBe(1)
})
it('the length is the declared /Length and the data ends on endstream', () => {
const [c] = candidates
const text = latin1(exported)
expect(text).toContain(`/Length ${c!.length}\n>>\nstream\n`)
expect(text.slice(c!.offset + c!.length)).toMatch(/^\r?\nendstream/)
})
it('the export keeps the Filespec, its name and the stamp in clear, no object stream', () => {
const text = latin1(exported)
expect(text).toContain('/Type /Filespec')
expect(text).toContain('/F (osteo-report.json)')
expect(text).not.toContain('/Type /ObjStm')
expect(hasArchiveStamp(exported)).toBe(true)
})
it('readAttachmentBytes returns the inflated envelope bytes', async () => {
const bytes = await readAttachmentBytes(exported)
const json = JSON.parse(new TextDecoder().decode(bytes)) as {
report: Report
}
expect(json.report).toEqual(golden.reports.demo)
})
})
describe('compatibility: frozen fixtures written with object streams', () => {
it.each([
['report-v2.full.pdf', 'v2'],
['report-v2.minimal.pdf', 'v2'],
['report-v3.bare.pdf', 'bare_v3'],
] as const)('%s → origin %s', async (name, origin) => {
const bytes = readFixtureBytes(name)
expect(latin1(bytes)).toContain('/Type /ObjStm')
const candidates = findEmbeddedFiles(bytes)
expect(candidates).toHaveLength(1)
// The Filespec is packed into the object stream: no name, the JSON decides.
expect(candidates[0]!.name).toBeNull()
expect(candidates[0]!.filter).toBe('FlateDecode')
const outcome = await readReportArchive(bytes, map())
expect(outcome.origin).toBe(origin)
})
it('a stamped PDF with object streams hides its stamp: no_attachment (documented)', async () => {
const bytes = await pdfWithoutAttachment(['faunosteo-archive/1'], true)
expect(hasArchiveStamp(bytes)).toBe(false)
await rejects(bytes, 'no_attachment')
})
})
describe('selection between several embedded files', () => {
it('a decoy EmbeddedFile first, the named Faunosteo one second: the name wins', async () => {
const bytes = await pdfWithAttachment(
'osteo-report.json',
golden.reports.demo,
{
before: ['other.json', { hello: 'world', version: 2 }],
},
)
const names = findEmbeddedFiles(bytes).map((c) => c.name)
expect(names).toEqual(['osteo-report.json', 'other.json'])
const outcome = await readReportArchive(bytes, map())
expect(outcome.origin).toBe('bare_v3')
expect(outcome.report).toEqual(golden.reports.demo)
})
it('an unnamed decoy without a version is skipped for the versioned JSON after it', async () => {
const decoy = streamObject({
num: 2,
dict: '/Type /EmbeddedFile /Length 17',
data: ascii('{"hello":"world"}'),
})
const real = objectHolding(exported, '/Type /EmbeddedFile')
const outcome = await readReportArchive(craftPdf([decoy, real]), map())
expect(outcome.origin).toBe('envelope_v1')
expect(outcome.report).toEqual(golden.reports.demo)
})
it('a foreign PDF, with or without attachments: no_attachment', async () => {
await rejects(await pdfWithoutAttachment(), 'no_attachment')
await rejects(
await pdfWithAttachment('other.json', { hello: 'world' }),
'no_attachment',
)
})
})
describe('crafted stream layouts', () => {
// UTF-8: the demo report holds an en dash that Latin-1 cannot carry.
const demoJson = () =>
new TextEncoder().encode(JSON.stringify(golden.reports.demo))
it('/Length as an indirect reference: read up to endstream', async () => {
const pdf = craftPdf([
streamObject({
num: 2,
dict: '/Type /EmbeddedFile /Length 7 0 R',
data: demoJson(),
}),
ascii('7 0 obj\n123456\nendobj\n'),
])
const [c] = findEmbeddedFiles(pdf)
expect(c!.filter).toBeNull()
expect(c!.length).toBe(demoJson().byteLength)
expect((await readReportArchive(pdf, map())).origin).toBe('bare_v3')
})
it('a wrong direct /Length that does not land on endstream: measured instead', async () => {
const pdf = craftPdf([
streamObject({
num: 2,
dict: '/Type /EmbeddedFile /Length 5',
data: demoJson(),
}),
])
expect(findEmbeddedFiles(pdf)[0]!.length).toBe(demoJson().byteLength)
expect((await readReportArchive(pdf, map())).origin).toBe('bare_v3')
})
it('CRLF after stream, /Type/EmbeddedFile unspaced, a leading /Params dict', async () => {
const data = demoJson()
const pdf = craftPdf([
streamObject({
num: 2,
dict: `/Params << /Size ${data.byteLength} >> /Type/EmbeddedFile/Length ${data.byteLength}`,
data,
eol: '\r\n',
}),
])
const [c] = findEmbeddedFiles(pdf)
expect(c!.length).toBe(data.byteLength)
expect(await readAttachmentBytes(pdf)).toEqual(data)
})
it('a Filespec with a hex /F name and an /EF listing /UF too names its stream', () => {
const name = Buffer.from('osteo-report.json', 'utf16le')
.swap16()
.toString('hex')
const pdf = craftPdf([
ascii(
`3 0 obj\n<< /Type /Filespec /F <${name}> /EF << /F 2 0 R /UF 2 0 R >> >>\nendobj\n`,
),
streamObject({
num: 2,
dict: '/Type /EmbeddedFile /Length 13',
data: ascii('{"version":3}'),
}),
])
expect(findEmbeddedFiles(pdf)[0]!.name).toBe('osteo-report.json')
})
it('decodePdfString: literal with escapes, hex ASCII, hex UTF-16BE with BOM', () => {
expect(decodePdfString('(a\\(b\\)c)')).toBe('a(b)c')
expect(decodePdfString('<6F7374656F>')).toBe('osteo')
expect(decodePdfString('<FEFF006F0073>')).toBe('os')
expect(decodePdfString('/Name')).toBeNull()
})
})
describe('the keyword stamp', () => {
it('hex (pdf-lib) and literal /Keywords strings are both read', async () => {
expect(
hasArchiveStamp(await pdfWithoutAttachment(['faunosteo-archive/1'])),
).toBe(true)
expect(hasArchiveStamp(await pdfWithoutAttachment(['other']))).toBe(false)
const literal = craftPdf([
ascii('3 0 obj\n<< /Keywords (faunosteo-archive/1) >>\nendobj\n'),
])
expect(hasArchiveStamp(literal)).toBe(true)
await rejects(literal, 'attachment_stripped')
})
})
describe('garbage never throws', () => {
const cases: [string, Uint8Array][] = [
['empty', new Uint8Array(0)],
[
'random bytes',
Uint8Array.from({ length: 4096 }, (_, i) => (i * 7919) % 256),
],
['the type alone', ascii('%PDF-1.7\n/Type /EmbeddedFile')],
[
'dict without stream',
ascii('%PDF-1.7\n<< /Type /EmbeddedFile /Length 3 >>\nendobj'),
],
[
'stream without endstream',
ascii('%PDF-1.7\n<< /Type /EmbeddedFile >>\nstream\n{"version":3}'),
],
[
'only the /EmbeddedFiles name tree key',
ascii('%PDF-1.7\n<< /Type /EmbeddedFiles >>\nstream\nx\nendstream'),
],
[
'unterminated dictionary',
ascii(
'%PDF-1.7\n<< /Type /EmbeddedFile /Length 1 << >>\nstream\nx\nendstream',
),
],
[
'hex /Keywords without a closing bracket',
ascii('%PDF-1.7\n<< /Keywords <FEFF0066'),
],
]
it.each(cases)(
'%s: no candidate, no stamp, no_attachment or invalid_pdf',
async (_name, bytes) => {
expect(findEmbeddedFiles(bytes)).toEqual([])
expect(hasArchiveStamp(bytes)).toBe(false)
await expect(readAttachedDocument(bytes)).rejects.toBeInstanceOf(
ArchiveReadError,
)
},
)
it('caps the candidates at 32', () => {
const objects = Array.from({ length: 40 }, (_, i) =>
streamObject({
num: i + 2,
dict: '/Type /EmbeddedFile /Length 1',
data: ascii('x'),
}),
)
expect(findEmbeddedFiles(craftPdf(objects))).toHaveLength(32)
})
})
describe('object-stream bomb (BLK-009)', () => {
const MiB = 1024 * 1024
const objStmBomb = () => {
const data = bomb()
return streamObject({
num: 9,
dict: `/Type /ObjStm /N 1 /First 4 /Filter /FlateDecode /Length ${data.byteLength}`,
data,
})
}
const embeddedBomb = () => {
const data = bomb()
return streamObject({
num: 8,
dict: `/Type /EmbeddedFile /Filter /FlateDecode /Length ${data.byteLength}`,
data,
})
}
it('64 MiB ObjStm bomb beside the attachment: report in ms, bomb never inflated', async () => {
const pdf = craftPdf([
objStmBomb(),
objectHolding(exported, '/Type /Filespec'),
objectHolding(exported, '/Type /EmbeddedFile'),
])
expect(pdf.byteLength).toBeLessThan(512 * 1024)
const rssBefore = process.memoryUsage().rss
const started = performance.now()
const outcome = await readReportArchive(pdf, map())
const elapsed = performance.now() - started
const rssDelta = process.memoryUsage().rss - rssBefore
console.log(
`[bomb] objstm: ${elapsed.toFixed(0)} ms, rss +${(rssDelta / MiB).toFixed(1)} MiB`,
)
expect(outcome.origin).toBe('envelope_v1')
expect(outcome.report).toEqual(golden.reports.demo)
expect(elapsed).toBeLessThan(2000)
expect(rssDelta).toBeLessThan(64 * MiB)
})
it('EmbeddedFile bomb past 16 MiB: inflation cancelled, no_attachment, no hang', async () => {
const pdf = craftPdf([objStmBomb(), embeddedBomb()])
const started = performance.now()
await rejects(pdf, 'no_attachment')
const elapsed = performance.now() - started
console.log(`[bomb] embedded: ${elapsed.toFixed(0)} ms`)
expect(elapsed).toBeLessThan(2000)
})
it('both bombs before an unnamed real attachment: bomb skipped, report read', async () => {
const pdf = craftPdf([
objStmBomb(),
embeddedBomb(),
objectHolding(exported, '/Type /EmbeddedFile'),
])
const started = performance.now()
const outcome = await readReportArchive(pdf, map())
const elapsed = performance.now() - started
console.log(`[bomb] both + unnamed attachment: ${elapsed.toFixed(0)} ms`)
expect(outcome.report).toEqual(golden.reports.demo)
expect(elapsed).toBeLessThan(2000)
})
})
describe('candidate cost bounds (audit M2)', () => {
const MiB = 1024 * 1024
/** `n 0 obj << /Type /EmbeddedFile >> stream` with no `/Length`: the data runs to `endstream`. */
const openStream = (num: number) =>
ascii(`${num} 0 obj\n<< /Type /EmbeddedFile >>\nstream\n`)
const filled = (byte: string, length: number) => ascii(byte.repeat(length))
const timed = async <T>(label: string, run: () => Promise<T> | T) => {
const started = performance.now()
const result = await run()
const elapsed = performance.now() - started
console.log(`[cost] ${label}: ${elapsed.toFixed(0)} ms`)
return { result, elapsed }
}
it('32 candidates without any endstream on a 20 MiB file: no candidate, well under a second', async () => {
const heads = Array.from({ length: 32 }, (_, i) => openStream(i + 2))
const pdf = craftPdf([...heads, filled('x', 20 * MiB)])
const { result, elapsed } = await timed('32 × no endstream', () =>
findEmbeddedFiles(pdf),
)
expect(result).toEqual([])
expect(elapsed).toBeLessThan(1000)
})
it('32 nested EmbeddedFile dicts (descending offsets) without endstream on a 20 MiB file: no candidate, no finder reset', async () => {
// Outermost `/Type` first in the file, its `stream` last: offsets come out
// reversed; the sort by offset keeps the endstream search to one pass.
let nested: Uint8Array = ascii('<< /Type /EmbeddedFile >>\nstream\n')
for (let level = 1; level < 32; level += 1) {
nested = concat([
ascii('<< /Type /EmbeddedFile /A '),
nested,
ascii(' >>\nstream\n'),
])
}
const pdf = craftPdf([ascii('2 0 obj\n'), nested, filled('x', 20 * MiB)])
const { result, elapsed } = await timed('32 nested, no endstream', () =>
findEmbeddedFiles(pdf),
)
expect(result).toEqual([])
expect(elapsed).toBeLessThan(500)
})
it('several indirect /Length streams in a row: each measured to its own endstream', () => {
const payloads = ['{"version":3}', '{"version":4,"x":1}', '{"v":5}']
const pdf = craftPdf(
payloads.map((text, i) =>
streamObject({
num: i + 2,
dict: `/Type /EmbeddedFile /Length ${i + 20} 0 R`,
data: ascii(text),
}),
),
)
const lengths = findEmbeddedFiles(pdf).map((c) => c.length)
expect(lengths).toEqual(payloads.map((p) => p.length))
})
it('a measured stream running past the cap window is not a candidate, the next one is', () => {
const pdf = craftPdf([
openStream(2),
filled('x', 16 * MiB + 64),
ascii('\nendstream\nendobj\n'),
streamObject({
num: 3,
dict: '/Type /EmbeddedFile /Length 13',
data: ascii('{"version":3}'),
}),
])
const candidates = findEmbeddedFiles(pdf)
expect(candidates).toHaveLength(1)
expect(candidates[0]!.length).toBe(13)
})
it('32 raw candidates of 1 MiB of "[": no_attachment under a second, JSON.parse never called', async () => {
const objects = Array.from({ length: 32 }, (_, i) =>
streamObject({
num: i + 2,
dict: `/Type /EmbeddedFile /Length ${MiB}`,
data: filled('[', MiB),
}),
)
const pdf = craftPdf(objects)
const parse = vi.spyOn(JSON, 'parse')
try {
const { elapsed } = await timed('32 × 1 MiB "["', () =>
rejects(pdf, 'no_attachment'),
)
expect(elapsed).toBeLessThan(1000)
expect(parse).not.toHaveBeenCalled()
} finally {
parse.mockRestore()
}
})
describe('bracket-depth pre-scan before JSON.parse (audit finding 3)', () => {
const flateObject = (num: number, raw: Uint8Array) => {
const data = new Uint8Array(deflateSync(raw))
return streamObject({
num,
dict: `/Type /EmbeddedFile /Filter /FlateDecode /Length ${data.byteLength}`,
data,
})
}
const withSpy = async (run: () => Promise<void>) => {
const parse = vi.spyOn(JSON, 'parse')
try {
await run()
return parse.mock.calls.length
} finally {
parse.mockRestore()
}
}
it('a 16 KB Flate candidate opening 8 million "[" deep: skipped unparsed, no_attachment fast', async () => {
const raw = concat([ascii('{"version":3,"x":'), filled('[', 8_000_000)])
const object = flateObject(2, raw)
expect(object.byteLength).toBeLessThan(16 * 1024)
const calls = await withSpy(async () => {
const { elapsed } = await timed('8M-deep candidate', () =>
rejects(craftPdf([object]), 'no_attachment'),
)
expect(elapsed).toBeLessThan(100)
})
expect(calls).toBe(0)
})
it('2000 "[" inside a string with escaped quotes and backslashes: not rejected, parsed once', async () => {
const text = `{"version":3,"note":"\\"${'['.repeat(2000)}\\\\ \\" [[[["}`
const pdf = craftPdf([
streamObject({
num: 2,
dict: `/Type /EmbeddedFile /Length ${text.length}`,
data: ascii(text),
}),
])
const calls = await withSpy(async () => {
const located = await locateAttachment(pdf)
expect(located.data).toMatchObject({ version: 3 })
})
expect(calls).toBe(1)
})
it('the real envelope passes the pre-scan and is parsed once', async () => {
const calls = await withSpy(async () => {
const located = await locateAttachment(exported)
expect(located.data).toMatchObject({ envelopeVersion: 1 })
})
expect(calls).toBe(1)
})
})
describe('one 16 MiB budget shared by every candidate', () => {
const garbageObject = (num: number) =>
streamObject({
num,
dict: `/Type /EmbeddedFile /Length ${10 * MiB}`,
data: concat([ascii('{'), filled('x', 10 * MiB - 1)]),
})
const real = () => objectHolding(exported, '/Type /EmbeddedFile')
it('one 10 MiB garbage object before the attachment: 6 MiB left, the report is read', async () => {
const pdf = craftPdf([garbageObject(2), real()])
const outcome = await readReportArchive(pdf, map())
expect(outcome.report).toEqual(golden.reports.demo)
})
it('two 10 MiB garbage objects before the attachment: budget exhausted, no_attachment', async () => {
const pdf = craftPdf([garbageObject(2), garbageObject(3), real()])
const { elapsed } = await timed('2 × 10 MiB "{" + attachment', () =>
rejects(pdf, 'no_attachment'),
)
expect(elapsed).toBeLessThan(2000)
})
})
})