diff --git a/docs/PDF-FORMAT.md b/docs/PDF-FORMAT.md index fd400536..3f91e404 100644 --- a/docs/PDF-FORMAT.md +++ b/docs/PDF-FORMAT.md @@ -143,10 +143,13 @@ never lost because of its wrapper: with its direct `/Length` or, failing that, the distance to `endstream`; a `/Filespec` dictionary in clear whose `/EF /F` points at the stream gives it a name. Candidates named `osteo-report*` come first, then file - order; Flate data is inflated by the platform inflater under a 16 MiB cap - (cancelled beyond it), and the first candidate whose text is a JSON - object with a numeric `envelopeVersion` or `version` wins. Object - streams, cross-reference streams and page contents are never inflated. + order; Flate data is fed to the platform inflater in 16 KiB slices, so + memory stays bounded in browsers too, under a 16 MiB cap (cancelled + beyond it); at most 16 MiB of candidate content is examined in all, a + candidate not opening on `{` is skipped unparsed, and the first one whose + text is a JSON object with a numeric `envelopeVersion` or `version` + wins. Object streams, cross-reference streams and page contents are never + inflated. No candidate: `attachment_stripped` when a `/Keywords` string in clear contains `faunosteo-archive`, else `no_attachment`. A stamped file saved with object streams (before BLK-009) hides its stamp and reads as diff --git a/docs/development/PDF_IMPORT_HARDENING.md b/docs/development/PDF_IMPORT_HARDENING.md index 2a798121..d8168388 100644 --- a/docs/development/PDF_IMPORT_HARDENING.md +++ b/docs/development/PDF_IMPORT_HARDENING.md @@ -88,6 +88,14 @@ hides its `/Keywords`, so a stripped attachment on such a file reads as - Option A merged and the CLI on the scanner; no `PDFDocument.load` on any untrusted input in the browser. +- A real-browser test of the inflate bound (Chromium, Firefox, Safari): the + vitest suite runs in Node and cannot see a browser inflater's read-ahead. + The audit of 60ee712 measured +2.2 GB RSS in Chromium on a 4 MB / 4 GiB + bomb before the inflater was fed in 16 KiB slices (+215 MB after); that + measurement is manual until a browser runner exists. +- Done: the candidate CPU/byte budget on the scanner, one 16 MiB budget of + examined content shared by every candidate, a monotonic `endstream` search + bounded to the cap, and no JSON parse for a candidate not opening on `{`. - Option C or a server-side import (Step 14) with a hard memory and time budget per file, for any path that still parses PDF objects. - Dependency decision on pdf-lib (unmaintained since 2021): keep it for diff --git a/packages/pdf/src/attachment/inflate.ts b/packages/pdf/src/attachment/inflate.ts index f11af1bc..59521f56 100644 --- a/packages/pdf/src/attachment/inflate.ts +++ b/packages/pdf/src/attachment/inflate.ts @@ -4,6 +4,19 @@ */ export const MAX_ATTACHMENT_BYTES = 16 * 1024 * 1024 +/** + * Compressed bytes handed to the inflater per write. Chromium's + * `DecompressionStream` inflates everything it has been given before the + * consumer reads it, so a stream written whole lets a bomb inflate gigabytes + * before `reader.cancel()` lands (measured: +2.2 GB RSS on a 4 MB / 4 GiB + * bomb). One awaited slice at a time keeps that read-ahead to what 16 KiB + * can inflate (about 16 MiB). + */ +const WRITE_SLICE_BYTES = 16 * 1024 + +type Writer = WritableStreamDefaultWriter> +type Reader = ReadableStreamDefaultReader + function concat(chunks: Uint8Array[], total: number): Uint8Array { const out = new Uint8Array(total) let offset = 0 @@ -14,35 +27,56 @@ function concat(chunks: Uint8Array[], total: number): Uint8Array { return out } +/** Feeds the inflater one slice per awaited write, then closes it. */ +async function feedSlices(writer: Writer, bytes: Uint8Array): Promise { + // The cast compiles under both the engine (node) and the app (DOM) libs. + const source = bytes as Uint8Array + for (let at = 0; at < source.byteLength; at += WRITE_SLICE_BYTES) { + await writer.write(source.subarray(at, at + WRITE_SLICE_BYTES)) + } + await writer.close() +} + +/** Collects the inflated chunks; null as soon as their total exceeds `limit`. */ +async function drainUnder( + reader: Reader, + limit: number, +): Promise { + const chunks: Uint8Array[] = [] + let total = 0 + for (;;) { + const { done, value } = await reader.read() + if (done) return concat(chunks, total) + total += value.byteLength + if (total > limit) return null + chunks.push(value) + } +} + /** * FlateDecode (zlib framing) under `limit`, the attachment cap by default. * Null on any failure (truncated or corrupt stream, trailing junk, bad - * checksum) or as soon as the output exceeds the limit. + * checksum) or as soon as the output exceeds the limit. The feed and the + * drain run concurrently; over the limit both sides are torn down, which + * also rejects the feed's pending write. */ export async function inflateFlate( bytes: Uint8Array, limit: number = MAX_ATTACHMENT_BYTES, ): Promise { if (typeof DecompressionStream === 'undefined') return null + const inflater = new DecompressionStream('deflate') + const writer = inflater.writable.getWriter() + const reader = inflater.readable.getReader() + // A stream error surfaces on the drain; the feed's own rejection is noise. + const fed = feedSlices(writer, bytes) + fed.catch(() => undefined) try { - // The cast compiles under both the engine (node) and the app (DOM) libs. - const stream = new Blob([bytes as Uint8Array]) - .stream() - .pipeThrough(new DecompressionStream('deflate')) - const reader = stream.getReader() - const chunks: Uint8Array[] = [] - let total = 0 - for (;;) { - const { done, value } = await reader.read() - if (done) break - total += value.byteLength - if (total > limit) { - await reader.cancel() - return null - } - chunks.push(value) + const out = await drainUnder(reader, limit) + if (out === null) { + await Promise.allSettled([reader.cancel(), writer.abort()]) } - return concat(chunks, total) + return out } catch { return null } diff --git a/packages/pdf/src/attachment/scan.ts b/packages/pdf/src/attachment/scan.ts index 724d4b2d..b05c9e3e 100644 --- a/packages/pdf/src/attachment/scan.ts +++ b/packages/pdf/src/attachment/scan.ts @@ -253,13 +253,44 @@ function declaredLength( return found === tail ? length : null } +/** Farthest `endstream` worth finding: a longer candidate is refused anyway. */ +const MEASURE_WINDOW = MAX_ATTACHMENT_BYTES + 16 + +/** Index of the first `endstream` at or after `offset`, within MEASURE_WINDOW; -1 when none. */ +type EndstreamFinder = (offset: number) => number + +/** + * Searches `endstream` once over the file instead of once per candidate: + * candidates come in offset order, so the last hit serves every offset + * before it and a miss clears the bytes it covered for the next search. An + * offset out of order (never produced today) just restarts the search. + */ +function endstreamFinder(bytes: Uint8Array, end: number): EndstreamFinder { + let lastHit = -1 + let clearedTo = 0 + let lastOffset = 0 + return (offset) => { + if (offset < lastOffset) { + lastHit = -1 + clearedTo = 0 + } + lastOffset = offset + if (lastHit >= offset) return lastHit + const limit = Math.min(end, offset + MEASURE_WINDOW) + const at = indexOf(bytes, ENDSTREAM, Math.max(offset, clearedTo), limit) + if (at >= 0) lastHit = at + else clearedTo = Math.max(clearedTo, limit - ENDSTREAM.length + 1) + return at + } +} + /** Fallback for an indirect or wrong `/Length`: up to `endstream`, the EOL before it dropped. */ function measuredLength( bytes: Uint8Array, offset: number, - end: number, + findEndstream: EndstreamFinder, ): number | null { - const at = indexOf(bytes, ENDSTREAM, offset, end) + const at = findEndstream(offset) if (at < 0) return null let stop = at if (stop > offset && bytes[stop - 1] === 0x0a) stop -= 1 @@ -275,12 +306,13 @@ function candidateOf( bytes: Uint8Array, dict: DictSpan, end: number, + findEndstream: EndstreamFinder, ): Omit | null { const offset = streamDataOffset(bytes, dict.end, end) if (offset < 0) return null const length = declaredLength(bytes, dict.text, offset, end) ?? - measuredLength(bytes, offset, end) + measuredLength(bytes, offset, findEndstream) if (length === null) return null return { offset, length, filter: filterOf(dict.text) } } @@ -365,11 +397,12 @@ export function hasPdfHeader(bytes: Uint8Array): boolean { export function findEmbeddedFiles(bytes: Uint8Array): EmbeddedFileCandidate[] { const end = Math.min(bytes.byteLength, MAX_PDF_BYTES) const names = filespecNames(bytes, end) + const findEndstream = endstreamFinder(bytes, end) const found: EmbeddedFileCandidate[] = [] for (const pos of findTypedDicts(bytes, EMBEDDED_FILE, end)) { const dict = dictAround(bytes, pos, end) if (!dict) continue - const candidate = candidateOf(bytes, dict, end) + const candidate = candidateOf(bytes, dict, end, findEndstream) if (!candidate) continue const name = dict.objectNumber === null ? null : (names.get(dict.objectNumber) ?? null) @@ -380,18 +413,31 @@ export function findEmbeddedFiles(bytes: Uint8Array): EmbeddedFileCandidate[] { return found.sort((a, b) => named(a) - named(b)) } -/** Raw or inflated stream data under the attachment cap; null when unusable. */ +/** Raw or inflated stream data under `budget`; null when unusable or inflating past it. */ async function candidateBytes( bytes: Uint8Array, candidate: EmbeddedFileCandidate, + budget: number, ): Promise { - if (candidate.length > MAX_ATTACHMENT_BYTES) return null const data = bytes.subarray( candidate.offset, candidate.offset + candidate.length, ) if (candidate.filter === null) return data - return inflateFlate(data) + return inflateFlate(data, budget) +} + +/** `{` first, past an optional UTF-8 BOM and JSON whitespace: the only shape a version lives in. */ +function startsJsonObject(data: Uint8Array): boolean { + let i = 0 + if (data[0] === 0xef && data[1] === 0xbb && data[2] === 0xbf) i = 3 + while ( + i < data.byteLength && + ' \t\n\r'.includes(String.fromCharCode(data[i]!)) + ) { + i += 1 + } + return data[i] === 0x7b } /** JSON with a numeric `envelopeVersion` or `version`; undefined otherwise. */ @@ -418,13 +464,21 @@ export interface LocatedAttachment { /** * The first candidate whose data is a versioned JSON document, decoded once. * None: `attachment_stripped` when the stamp is visible, else `no_attachment`. + * One MAX_ATTACHMENT_BYTES budget of examined content (raw or inflated) is + * shared by every candidate: a candidate that does not fit what is left ends + * the search, so 32 crafted streams cost at most one attachment's worth of + * inflation and parsing. A stream cancelled mid-inflation is not charged. */ export async function locateAttachment( bytes: Uint8Array, ): Promise { + let budget = MAX_ATTACHMENT_BYTES for (const candidate of findEmbeddedFiles(bytes)) { - const data = await candidateBytes(bytes, candidate) + if (candidate.length > budget) break + const data = await candidateBytes(bytes, candidate, budget) if (!data) continue + budget -= data.byteLength + if (!startsJsonObject(data)) continue const parsed = parseVersioned(data) if (parsed !== undefined) return { bytes: data, data: parsed } } diff --git a/packages/pdf/tests/inflate.test.ts b/packages/pdf/tests/inflate.test.ts new file mode 100644 index 00000000..c8a95ee7 --- /dev/null +++ b/packages/pdf/tests/inflate.test.ts @@ -0,0 +1,77 @@ +/** + * The capped inflater: round trips, the exact limit, every failure mode as + * null, the bomb cancelled early, and the compressed bytes fed to the + * platform inflater in slices (the Chromium read-ahead bound; Node cannot + * show the memory effect, only the feeding shape). + */ +import { deflateSync } from 'node:zlib' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { inflateFlate, MAX_ATTACHMENT_BYTES } from '../src/attachment/inflate' + +const KiB = 1024 +const MiB = 1024 * KiB + +/** Incompressible bytes: the compressed stream is about as long as the input. */ +function noise(length: number): Uint8Array { + let seed = 0x9e3779b9 + return Uint8Array.from({ length }, () => { + seed = (seed * 1664525 + 1013904223) >>> 0 + return seed >>> 24 + }) +} + +const deflate = (data: Uint8Array) => new Uint8Array(deflateSync(data)) + +afterEach(() => { + vi.restoreAllMocks() +}) + +describe('inflateFlate', () => { + it('round-trips a payload spanning several slices', async () => { + const payload = noise(200 * KiB) + const compressed = deflate(payload) + expect(compressed.byteLength).toBeGreaterThan(10 * 16 * KiB) + expect(await inflateFlate(compressed)).toEqual(payload) + }) + + it('feeds the inflater in 16 KiB slices, each awaited', async () => { + const compressed = deflate(noise(100 * KiB)) + const write = vi.spyOn(WritableStreamDefaultWriter.prototype, 'write') + await inflateFlate(compressed) + const slices = Math.ceil(compressed.byteLength / (16 * KiB)) + expect(write).toHaveBeenCalledTimes(slices) + const sizes = write.mock.calls.map((c) => (c[0] as Uint8Array).byteLength) + expect(Math.max(...sizes)).toBe(16 * KiB) + }) + + it('an output exactly at the limit passes, one byte more is null', async () => { + const payload = noise(64 * KiB) + const compressed = deflate(payload) + expect(await inflateFlate(compressed, payload.byteLength)).toEqual(payload) + expect(await inflateFlate(compressed, payload.byteLength - 1)).toBeNull() + }) + + it.each([ + ['empty input', new Uint8Array(0)], + ['truncated stream', deflate(noise(4 * KiB)).subarray(0, 2 * KiB)], + ['trailing junk', Uint8Array.from([...deflate(noise(KiB)), 1, 2, 3])], + [ + 'corrupt body', + Uint8Array.from(deflate(noise(KiB)), (b, i) => (i > 8 ? b ^ 0xff : b)), + ], + ['not deflate at all', new TextEncoder().encode('{"version":3}')], + ])('%s: null', async (_name, bytes) => { + expect(await inflateFlate(bytes)).toBeNull() + }) + + it('a 64 MiB bomb past the default cap is cancelled in well under a second', async () => { + const bomb = deflate(new Uint8Array(64 * MiB)) + expect(bomb.byteLength).toBeLessThan(MiB) + const started = performance.now() + expect(await inflateFlate(bomb)).toBeNull() + const elapsed = performance.now() - started + console.log(`[inflate] 64 MiB bomb: ${elapsed.toFixed(0)} ms`) + expect(elapsed).toBeLessThan(1000) + expect(MAX_ATTACHMENT_BYTES).toBe(16 * MiB) + }) +}) diff --git a/packages/pdf/tests/scan.test.ts b/packages/pdf/tests/scan.test.ts index 2efdbb16..65ff79c0 100644 --- a/packages/pdf/tests/scan.test.ts +++ b/packages/pdf/tests/scan.test.ts @@ -2,10 +2,11 @@ * BLK-009: the byte-scan attachment reader. Candidates on a fresh export and * on the frozen fixtures (object-stream layout), crafted layouts (two files, * indirect `/Length`, `\r\n`, no space after `/Type`), garbage, the keyword - * stamp, and the object-stream bomb the reader never inflates. + * stamp, the object-stream bomb the reader never inflates, and the candidate + * cost bounds (monotonic `endstream` search, shared 16 MiB budget, `{` check). */ import { deflateSync } from 'node:zlib' -import { beforeAll, describe, expect, it } from 'vitest' +import { beforeAll, describe, expect, it, vi } from 'vitest' import { buildSheetSnapshot, type Report } from '@faunosteo/domain' import { ArchiveReadError, @@ -423,3 +424,104 @@ describe('object-stream bomb (BLK-009)', () => { expect(elapsed).toBeLessThan(2000) }) }) + +describe('candidate cost bounds (audit M2)', () => { + const MiB = 1024 * 1024 + /** `n 0 obj << /Type /EmbeddedFile >> stream` with no `/Length`: the data runs to `endstream`. */ + const openStream = (num: number) => + ascii(`${num} 0 obj\n<< /Type /EmbeddedFile >>\nstream\n`) + const filled = (byte: string, length: number) => ascii(byte.repeat(length)) + const timed = async (label: string, run: () => Promise | T) => { + const started = performance.now() + const result = await run() + const elapsed = performance.now() - started + console.log(`[cost] ${label}: ${elapsed.toFixed(0)} ms`) + return { result, elapsed } + } + + it('32 candidates without any endstream on a 20 MiB file: no candidate, well under a second', async () => { + const heads = Array.from({ length: 32 }, (_, i) => openStream(i + 2)) + const pdf = craftPdf([...heads, filled('x', 20 * MiB)]) + const { result, elapsed } = await timed('32 × no endstream', () => + findEmbeddedFiles(pdf), + ) + expect(result).toEqual([]) + expect(elapsed).toBeLessThan(1000) + }) + + it('several indirect /Length streams in a row: each measured to its own endstream', () => { + const payloads = ['{"version":3}', '{"version":4,"x":1}', '{"v":5}'] + const pdf = craftPdf( + payloads.map((text, i) => + streamObject({ + num: i + 2, + dict: `/Type /EmbeddedFile /Length ${i + 20} 0 R`, + data: ascii(text), + }), + ), + ) + const lengths = findEmbeddedFiles(pdf).map((c) => c.length) + expect(lengths).toEqual(payloads.map((p) => p.length)) + }) + + it('a measured stream running past the cap window is not a candidate, the next one is', () => { + const pdf = craftPdf([ + openStream(2), + filled('x', 16 * MiB + 64), + ascii('\nendstream\nendobj\n'), + streamObject({ + num: 3, + dict: '/Type /EmbeddedFile /Length 13', + data: ascii('{"version":3}'), + }), + ]) + const candidates = findEmbeddedFiles(pdf) + expect(candidates).toHaveLength(1) + expect(candidates[0]!.length).toBe(13) + }) + + it('32 raw candidates of 1 MiB of "[": no_attachment under a second, JSON.parse never called', async () => { + const objects = Array.from({ length: 32 }, (_, i) => + streamObject({ + num: i + 2, + dict: `/Type /EmbeddedFile /Length ${MiB}`, + data: filled('[', MiB), + }), + ) + const pdf = craftPdf(objects) + const parse = vi.spyOn(JSON, 'parse') + try { + const { elapsed } = await timed('32 × 1 MiB "["', () => + rejects(pdf, 'no_attachment'), + ) + expect(elapsed).toBeLessThan(1000) + expect(parse).not.toHaveBeenCalled() + } finally { + parse.mockRestore() + } + }) + + describe('one 16 MiB budget shared by every candidate', () => { + const garbageObject = (num: number) => + streamObject({ + num, + dict: `/Type /EmbeddedFile /Length ${10 * MiB}`, + data: concat([ascii('{'), filled('x', 10 * MiB - 1)]), + }) + const real = () => objectHolding(exported, '/Type /EmbeddedFile') + + it('one 10 MiB garbage object before the attachment: 6 MiB left, the report is read', async () => { + const pdf = craftPdf([garbageObject(2), real()]) + const outcome = await readReportArchive(pdf, map()) + expect(outcome.report).toEqual(golden.reports.demo) + }) + + it('two 10 MiB garbage objects before the attachment: budget exhausted, no_attachment', async () => { + const pdf = craftPdf([garbageObject(2), garbageObject(3), real()]) + const { elapsed } = await timed('2 × 10 MiB "{" + attachment', () => + rejects(pdf, 'no_attachment'), + ) + expect(elapsed).toBeLessThan(2000) + }) + }) +})