fix(pdf): inflater fed in 16 KiB slices (browser memory bound), shared 16 MiB candidate budget, monotonic bounded endstream search, JSON object pre-check
This commit is contained in:
1 parent
6adc38b027
commit
3a6e92625a
6 files changed
+310
-32
No files matched your search
+7
-4
@@ -143,10 +143,13 @@ never lost because of its wrapper:
|
||||
with its direct `/Length` or, failing that, the distance to `endstream`;
|
||||
a `/Filespec` dictionary in clear whose `/EF /F` points at the stream
|
||||
gives it a name. Candidates named `osteo-report*` come first, then file
|
||||
order; Flate data is inflated by the platform inflater under a 16 MiB cap
|
||||
(cancelled beyond it), and the first candidate whose text is a JSON
|
||||
object with a numeric `envelopeVersion` or `version` wins. Object
|
||||
streams, cross-reference streams and page contents are never inflated.
|
||||
order; Flate data is fed to the platform inflater in 16 KiB slices, so
|
||||
memory stays bounded in browsers too, under a 16 MiB cap (cancelled
|
||||
beyond it); at most 16 MiB of candidate content is examined in all, a
|
||||
candidate not opening on `{` is skipped unparsed, and the first one whose
|
||||
text is a JSON object with a numeric `envelopeVersion` or `version`
|
||||
wins. Object streams, cross-reference streams and page contents are never
|
||||
inflated.
|
||||
No candidate: `attachment_stripped` when a `/Keywords` string in clear
|
||||
contains `faunosteo-archive`, else `no_attachment`. A stamped file saved
|
||||
with object streams (before BLK-009) hides its stamp and reads as
|
||||
|
||||
@@ -88,6 +88,14 @@ hides its `/Keywords`, so a stripped attachment on such a file reads as
|
||||
|
||||
- Option A merged and the CLI on the scanner; no `PDFDocument.load` on any
|
||||
untrusted input in the browser.
|
||||
- A real-browser test of the inflate bound (Chromium, Firefox, Safari): the
|
||||
vitest suite runs in Node and cannot see a browser inflater's read-ahead.
|
||||
The audit of 60ee712 measured +2.2 GB RSS in Chromium on a 4 MB / 4 GiB
|
||||
bomb before the inflater was fed in 16 KiB slices (+215 MB after); that
|
||||
measurement is manual until a browser runner exists.
|
||||
- Done: the candidate CPU/byte budget on the scanner, one 16 MiB budget of
|
||||
examined content shared by every candidate, a monotonic `endstream` search
|
||||
bounded to the cap, and no JSON parse for a candidate not opening on `{`.
|
||||
- Option C or a server-side import (Step 14) with a hard memory and time
|
||||
budget per file, for any path that still parses PDF objects.
|
||||
- Dependency decision on pdf-lib (unmaintained since 2021): keep it for
|
||||
|
||||
@@ -4,6 +4,19 @@
|
||||
*/
|
||||
export const MAX_ATTACHMENT_BYTES = 16 * 1024 * 1024
|
||||
|
||||
/**
|
||||
* Compressed bytes handed to the inflater per write. Chromium's
|
||||
* `DecompressionStream` inflates everything it has been given before the
|
||||
* consumer reads it, so a stream written whole lets a bomb inflate gigabytes
|
||||
* before `reader.cancel()` lands (measured: +2.2 GB RSS on a 4 MB / 4 GiB
|
||||
* bomb). One awaited slice at a time keeps that read-ahead to what 16 KiB
|
||||
* can inflate (about 16 MiB).
|
||||
*/
|
||||
const WRITE_SLICE_BYTES = 16 * 1024
|
||||
|
||||
type Writer = WritableStreamDefaultWriter<Uint8Array<ArrayBuffer>>
|
||||
type Reader = ReadableStreamDefaultReader<Uint8Array>
|
||||
|
||||
function concat(chunks: Uint8Array[], total: number): Uint8Array {
|
||||
const out = new Uint8Array(total)
|
||||
let offset = 0
|
||||
@@ -14,35 +27,56 @@ function concat(chunks: Uint8Array[], total: number): Uint8Array {
|
||||
return out
|
||||
}
|
||||
|
||||
/** Feeds the inflater one slice per awaited write, then closes it. */
|
||||
async function feedSlices(writer: Writer, bytes: Uint8Array): Promise<void> {
|
||||
// The cast compiles under both the engine (node) and the app (DOM) libs.
|
||||
const source = bytes as Uint8Array<ArrayBuffer>
|
||||
for (let at = 0; at < source.byteLength; at += WRITE_SLICE_BYTES) {
|
||||
await writer.write(source.subarray(at, at + WRITE_SLICE_BYTES))
|
||||
}
|
||||
await writer.close()
|
||||
}
|
||||
|
||||
/** Collects the inflated chunks; null as soon as their total exceeds `limit`. */
|
||||
async function drainUnder(
|
||||
reader: Reader,
|
||||
limit: number,
|
||||
): Promise<Uint8Array | null> {
|
||||
const chunks: Uint8Array[] = []
|
||||
let total = 0
|
||||
for (;;) {
|
||||
const { done, value } = await reader.read()
|
||||
if (done) return concat(chunks, total)
|
||||
total += value.byteLength
|
||||
if (total > limit) return null
|
||||
chunks.push(value)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* FlateDecode (zlib framing) under `limit`, the attachment cap by default.
|
||||
* Null on any failure (truncated or corrupt stream, trailing junk, bad
|
||||
* checksum) or as soon as the output exceeds the limit.
|
||||
* checksum) or as soon as the output exceeds the limit. The feed and the
|
||||
* drain run concurrently; over the limit both sides are torn down, which
|
||||
* also rejects the feed's pending write.
|
||||
*/
|
||||
export async function inflateFlate(
|
||||
bytes: Uint8Array,
|
||||
limit: number = MAX_ATTACHMENT_BYTES,
|
||||
): Promise<Uint8Array | null> {
|
||||
if (typeof DecompressionStream === 'undefined') return null
|
||||
const inflater = new DecompressionStream('deflate')
|
||||
const writer = inflater.writable.getWriter()
|
||||
const reader = inflater.readable.getReader()
|
||||
// A stream error surfaces on the drain; the feed's own rejection is noise.
|
||||
const fed = feedSlices(writer, bytes)
|
||||
fed.catch(() => undefined)
|
||||
try {
|
||||
// The cast compiles under both the engine (node) and the app (DOM) libs.
|
||||
const stream = new Blob([bytes as Uint8Array<ArrayBuffer>])
|
||||
.stream()
|
||||
.pipeThrough(new DecompressionStream('deflate'))
|
||||
const reader = stream.getReader()
|
||||
const chunks: Uint8Array[] = []
|
||||
let total = 0
|
||||
for (;;) {
|
||||
const { done, value } = await reader.read()
|
||||
if (done) break
|
||||
total += value.byteLength
|
||||
if (total > limit) {
|
||||
await reader.cancel()
|
||||
return null
|
||||
}
|
||||
chunks.push(value)
|
||||
const out = await drainUnder(reader, limit)
|
||||
if (out === null) {
|
||||
await Promise.allSettled([reader.cancel(), writer.abort()])
|
||||
}
|
||||
return concat(chunks, total)
|
||||
return out
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
|
||||
@@ -253,13 +253,44 @@ function declaredLength(
|
||||
return found === tail ? length : null
|
||||
}
|
||||
|
||||
/** Farthest `endstream` worth finding: a longer candidate is refused anyway. */
|
||||
const MEASURE_WINDOW = MAX_ATTACHMENT_BYTES + 16
|
||||
|
||||
/** Index of the first `endstream` at or after `offset`, within MEASURE_WINDOW; -1 when none. */
|
||||
type EndstreamFinder = (offset: number) => number
|
||||
|
||||
/**
|
||||
* Searches `endstream` once over the file instead of once per candidate:
|
||||
* candidates come in offset order, so the last hit serves every offset
|
||||
* before it and a miss clears the bytes it covered for the next search. An
|
||||
* offset out of order (never produced today) just restarts the search.
|
||||
*/
|
||||
function endstreamFinder(bytes: Uint8Array, end: number): EndstreamFinder {
|
||||
let lastHit = -1
|
||||
let clearedTo = 0
|
||||
let lastOffset = 0
|
||||
return (offset) => {
|
||||
if (offset < lastOffset) {
|
||||
lastHit = -1
|
||||
clearedTo = 0
|
||||
}
|
||||
lastOffset = offset
|
||||
if (lastHit >= offset) return lastHit
|
||||
const limit = Math.min(end, offset + MEASURE_WINDOW)
|
||||
const at = indexOf(bytes, ENDSTREAM, Math.max(offset, clearedTo), limit)
|
||||
if (at >= 0) lastHit = at
|
||||
else clearedTo = Math.max(clearedTo, limit - ENDSTREAM.length + 1)
|
||||
return at
|
||||
}
|
||||
}
|
||||
|
||||
/** Fallback for an indirect or wrong `/Length`: up to `endstream`, the EOL before it dropped. */
|
||||
function measuredLength(
|
||||
bytes: Uint8Array,
|
||||
offset: number,
|
||||
end: number,
|
||||
findEndstream: EndstreamFinder,
|
||||
): number | null {
|
||||
const at = indexOf(bytes, ENDSTREAM, offset, end)
|
||||
const at = findEndstream(offset)
|
||||
if (at < 0) return null
|
||||
let stop = at
|
||||
if (stop > offset && bytes[stop - 1] === 0x0a) stop -= 1
|
||||
@@ -275,12 +306,13 @@ function candidateOf(
|
||||
bytes: Uint8Array,
|
||||
dict: DictSpan,
|
||||
end: number,
|
||||
findEndstream: EndstreamFinder,
|
||||
): Omit<EmbeddedFileCandidate, 'name'> | null {
|
||||
const offset = streamDataOffset(bytes, dict.end, end)
|
||||
if (offset < 0) return null
|
||||
const length =
|
||||
declaredLength(bytes, dict.text, offset, end) ??
|
||||
measuredLength(bytes, offset, end)
|
||||
measuredLength(bytes, offset, findEndstream)
|
||||
if (length === null) return null
|
||||
return { offset, length, filter: filterOf(dict.text) }
|
||||
}
|
||||
@@ -365,11 +397,12 @@ export function hasPdfHeader(bytes: Uint8Array): boolean {
|
||||
export function findEmbeddedFiles(bytes: Uint8Array): EmbeddedFileCandidate[] {
|
||||
const end = Math.min(bytes.byteLength, MAX_PDF_BYTES)
|
||||
const names = filespecNames(bytes, end)
|
||||
const findEndstream = endstreamFinder(bytes, end)
|
||||
const found: EmbeddedFileCandidate[] = []
|
||||
for (const pos of findTypedDicts(bytes, EMBEDDED_FILE, end)) {
|
||||
const dict = dictAround(bytes, pos, end)
|
||||
if (!dict) continue
|
||||
const candidate = candidateOf(bytes, dict, end)
|
||||
const candidate = candidateOf(bytes, dict, end, findEndstream)
|
||||
if (!candidate) continue
|
||||
const name =
|
||||
dict.objectNumber === null ? null : (names.get(dict.objectNumber) ?? null)
|
||||
@@ -380,18 +413,31 @@ export function findEmbeddedFiles(bytes: Uint8Array): EmbeddedFileCandidate[] {
|
||||
return found.sort((a, b) => named(a) - named(b))
|
||||
}
|
||||
|
||||
/** Raw or inflated stream data under the attachment cap; null when unusable. */
|
||||
/** Raw or inflated stream data under `budget`; null when unusable or inflating past it. */
|
||||
async function candidateBytes(
|
||||
bytes: Uint8Array,
|
||||
candidate: EmbeddedFileCandidate,
|
||||
budget: number,
|
||||
): Promise<Uint8Array | null> {
|
||||
if (candidate.length > MAX_ATTACHMENT_BYTES) return null
|
||||
const data = bytes.subarray(
|
||||
candidate.offset,
|
||||
candidate.offset + candidate.length,
|
||||
)
|
||||
if (candidate.filter === null) return data
|
||||
return inflateFlate(data)
|
||||
return inflateFlate(data, budget)
|
||||
}
|
||||
|
||||
/** `{` first, past an optional UTF-8 BOM and JSON whitespace: the only shape a version lives in. */
|
||||
function startsJsonObject(data: Uint8Array): boolean {
|
||||
let i = 0
|
||||
if (data[0] === 0xef && data[1] === 0xbb && data[2] === 0xbf) i = 3
|
||||
while (
|
||||
i < data.byteLength &&
|
||||
' \t\n\r'.includes(String.fromCharCode(data[i]!))
|
||||
) {
|
||||
i += 1
|
||||
}
|
||||
return data[i] === 0x7b
|
||||
}
|
||||
|
||||
/** JSON with a numeric `envelopeVersion` or `version`; undefined otherwise. */
|
||||
@@ -418,13 +464,21 @@ export interface LocatedAttachment {
|
||||
/**
|
||||
* The first candidate whose data is a versioned JSON document, decoded once.
|
||||
* None: `attachment_stripped` when the stamp is visible, else `no_attachment`.
|
||||
* One MAX_ATTACHMENT_BYTES budget of examined content (raw or inflated) is
|
||||
* shared by every candidate: a candidate that does not fit what is left ends
|
||||
* the search, so 32 crafted streams cost at most one attachment's worth of
|
||||
* inflation and parsing. A stream cancelled mid-inflation is not charged.
|
||||
*/
|
||||
export async function locateAttachment(
|
||||
bytes: Uint8Array,
|
||||
): Promise<LocatedAttachment> {
|
||||
let budget = MAX_ATTACHMENT_BYTES
|
||||
for (const candidate of findEmbeddedFiles(bytes)) {
|
||||
const data = await candidateBytes(bytes, candidate)
|
||||
if (candidate.length > budget) break
|
||||
const data = await candidateBytes(bytes, candidate, budget)
|
||||
if (!data) continue
|
||||
budget -= data.byteLength
|
||||
if (!startsJsonObject(data)) continue
|
||||
const parsed = parseVersioned(data)
|
||||
if (parsed !== undefined) return { bytes: data, data: parsed }
|
||||
}
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
/**
|
||||
* The capped inflater: round trips, the exact limit, every failure mode as
|
||||
* null, the bomb cancelled early, and the compressed bytes fed to the
|
||||
* platform inflater in slices (the Chromium read-ahead bound; Node cannot
|
||||
* show the memory effect, only the feeding shape).
|
||||
*/
|
||||
import { deflateSync } from 'node:zlib'
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import { inflateFlate, MAX_ATTACHMENT_BYTES } from '../src/attachment/inflate'
|
||||
|
||||
const KiB = 1024
|
||||
const MiB = 1024 * KiB
|
||||
|
||||
/** Incompressible bytes: the compressed stream is about as long as the input. */
|
||||
function noise(length: number): Uint8Array {
|
||||
let seed = 0x9e3779b9
|
||||
return Uint8Array.from({ length }, () => {
|
||||
seed = (seed * 1664525 + 1013904223) >>> 0
|
||||
return seed >>> 24
|
||||
})
|
||||
}
|
||||
|
||||
const deflate = (data: Uint8Array) => new Uint8Array(deflateSync(data))
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks()
|
||||
})
|
||||
|
||||
describe('inflateFlate', () => {
|
||||
it('round-trips a payload spanning several slices', async () => {
|
||||
const payload = noise(200 * KiB)
|
||||
const compressed = deflate(payload)
|
||||
expect(compressed.byteLength).toBeGreaterThan(10 * 16 * KiB)
|
||||
expect(await inflateFlate(compressed)).toEqual(payload)
|
||||
})
|
||||
|
||||
it('feeds the inflater in 16 KiB slices, each awaited', async () => {
|
||||
const compressed = deflate(noise(100 * KiB))
|
||||
const write = vi.spyOn(WritableStreamDefaultWriter.prototype, 'write')
|
||||
await inflateFlate(compressed)
|
||||
const slices = Math.ceil(compressed.byteLength / (16 * KiB))
|
||||
expect(write).toHaveBeenCalledTimes(slices)
|
||||
const sizes = write.mock.calls.map((c) => (c[0] as Uint8Array).byteLength)
|
||||
expect(Math.max(...sizes)).toBe(16 * KiB)
|
||||
})
|
||||
|
||||
it('an output exactly at the limit passes, one byte more is null', async () => {
|
||||
const payload = noise(64 * KiB)
|
||||
const compressed = deflate(payload)
|
||||
expect(await inflateFlate(compressed, payload.byteLength)).toEqual(payload)
|
||||
expect(await inflateFlate(compressed, payload.byteLength - 1)).toBeNull()
|
||||
})
|
||||
|
||||
it.each([
|
||||
['empty input', new Uint8Array(0)],
|
||||
['truncated stream', deflate(noise(4 * KiB)).subarray(0, 2 * KiB)],
|
||||
['trailing junk', Uint8Array.from([...deflate(noise(KiB)), 1, 2, 3])],
|
||||
[
|
||||
'corrupt body',
|
||||
Uint8Array.from(deflate(noise(KiB)), (b, i) => (i > 8 ? b ^ 0xff : b)),
|
||||
],
|
||||
['not deflate at all', new TextEncoder().encode('{"version":3}')],
|
||||
])('%s: null', async (_name, bytes) => {
|
||||
expect(await inflateFlate(bytes)).toBeNull()
|
||||
})
|
||||
|
||||
it('a 64 MiB bomb past the default cap is cancelled in well under a second', async () => {
|
||||
const bomb = deflate(new Uint8Array(64 * MiB))
|
||||
expect(bomb.byteLength).toBeLessThan(MiB)
|
||||
const started = performance.now()
|
||||
expect(await inflateFlate(bomb)).toBeNull()
|
||||
const elapsed = performance.now() - started
|
||||
console.log(`[inflate] 64 MiB bomb: ${elapsed.toFixed(0)} ms`)
|
||||
expect(elapsed).toBeLessThan(1000)
|
||||
expect(MAX_ATTACHMENT_BYTES).toBe(16 * MiB)
|
||||
})
|
||||
})
|
||||
@@ -2,10 +2,11 @@
|
||||
* BLK-009: the byte-scan attachment reader. Candidates on a fresh export and
|
||||
* on the frozen fixtures (object-stream layout), crafted layouts (two files,
|
||||
* indirect `/Length`, `\r\n`, no space after `/Type`), garbage, the keyword
|
||||
* stamp, and the object-stream bomb the reader never inflates.
|
||||
* stamp, the object-stream bomb the reader never inflates, and the candidate
|
||||
* cost bounds (monotonic `endstream` search, shared 16 MiB budget, `{` check).
|
||||
*/
|
||||
import { deflateSync } from 'node:zlib'
|
||||
import { beforeAll, describe, expect, it } from 'vitest'
|
||||
import { beforeAll, describe, expect, it, vi } from 'vitest'
|
||||
import { buildSheetSnapshot, type Report } from '@faunosteo/domain'
|
||||
import {
|
||||
ArchiveReadError,
|
||||
@@ -423,3 +424,104 @@ describe('object-stream bomb (BLK-009)', () => {
|
||||
expect(elapsed).toBeLessThan(2000)
|
||||
})
|
||||
})
|
||||
|
||||
describe('candidate cost bounds (audit M2)', () => {
|
||||
const MiB = 1024 * 1024
|
||||
/** `n 0 obj << /Type /EmbeddedFile >> stream` with no `/Length`: the data runs to `endstream`. */
|
||||
const openStream = (num: number) =>
|
||||
ascii(`${num} 0 obj\n<< /Type /EmbeddedFile >>\nstream\n`)
|
||||
const filled = (byte: string, length: number) => ascii(byte.repeat(length))
|
||||
const timed = async <T>(label: string, run: () => Promise<T> | T) => {
|
||||
const started = performance.now()
|
||||
const result = await run()
|
||||
const elapsed = performance.now() - started
|
||||
console.log(`[cost] ${label}: ${elapsed.toFixed(0)} ms`)
|
||||
return { result, elapsed }
|
||||
}
|
||||
|
||||
it('32 candidates without any endstream on a 20 MiB file: no candidate, well under a second', async () => {
|
||||
const heads = Array.from({ length: 32 }, (_, i) => openStream(i + 2))
|
||||
const pdf = craftPdf([...heads, filled('x', 20 * MiB)])
|
||||
const { result, elapsed } = await timed('32 × no endstream', () =>
|
||||
findEmbeddedFiles(pdf),
|
||||
)
|
||||
expect(result).toEqual([])
|
||||
expect(elapsed).toBeLessThan(1000)
|
||||
})
|
||||
|
||||
it('several indirect /Length streams in a row: each measured to its own endstream', () => {
|
||||
const payloads = ['{"version":3}', '{"version":4,"x":1}', '{"v":5}']
|
||||
const pdf = craftPdf(
|
||||
payloads.map((text, i) =>
|
||||
streamObject({
|
||||
num: i + 2,
|
||||
dict: `/Type /EmbeddedFile /Length ${i + 20} 0 R`,
|
||||
data: ascii(text),
|
||||
}),
|
||||
),
|
||||
)
|
||||
const lengths = findEmbeddedFiles(pdf).map((c) => c.length)
|
||||
expect(lengths).toEqual(payloads.map((p) => p.length))
|
||||
})
|
||||
|
||||
it('a measured stream running past the cap window is not a candidate, the next one is', () => {
|
||||
const pdf = craftPdf([
|
||||
openStream(2),
|
||||
filled('x', 16 * MiB + 64),
|
||||
ascii('\nendstream\nendobj\n'),
|
||||
streamObject({
|
||||
num: 3,
|
||||
dict: '/Type /EmbeddedFile /Length 13',
|
||||
data: ascii('{"version":3}'),
|
||||
}),
|
||||
])
|
||||
const candidates = findEmbeddedFiles(pdf)
|
||||
expect(candidates).toHaveLength(1)
|
||||
expect(candidates[0]!.length).toBe(13)
|
||||
})
|
||||
|
||||
it('32 raw candidates of 1 MiB of "[": no_attachment under a second, JSON.parse never called', async () => {
|
||||
const objects = Array.from({ length: 32 }, (_, i) =>
|
||||
streamObject({
|
||||
num: i + 2,
|
||||
dict: `/Type /EmbeddedFile /Length ${MiB}`,
|
||||
data: filled('[', MiB),
|
||||
}),
|
||||
)
|
||||
const pdf = craftPdf(objects)
|
||||
const parse = vi.spyOn(JSON, 'parse')
|
||||
try {
|
||||
const { elapsed } = await timed('32 × 1 MiB "["', () =>
|
||||
rejects(pdf, 'no_attachment'),
|
||||
)
|
||||
expect(elapsed).toBeLessThan(1000)
|
||||
expect(parse).not.toHaveBeenCalled()
|
||||
} finally {
|
||||
parse.mockRestore()
|
||||
}
|
||||
})
|
||||
|
||||
describe('one 16 MiB budget shared by every candidate', () => {
|
||||
const garbageObject = (num: number) =>
|
||||
streamObject({
|
||||
num,
|
||||
dict: `/Type /EmbeddedFile /Length ${10 * MiB}`,
|
||||
data: concat([ascii('{'), filled('x', 10 * MiB - 1)]),
|
||||
})
|
||||
const real = () => objectHolding(exported, '/Type /EmbeddedFile')
|
||||
|
||||
it('one 10 MiB garbage object before the attachment: 6 MiB left, the report is read', async () => {
|
||||
const pdf = craftPdf([garbageObject(2), real()])
|
||||
const outcome = await readReportArchive(pdf, map())
|
||||
expect(outcome.report).toEqual(golden.reports.demo)
|
||||
})
|
||||
|
||||
it('two 10 MiB garbage objects before the attachment: budget exhausted, no_attachment', async () => {
|
||||
const pdf = craftPdf([garbageObject(2), garbageObject(3), real()])
|
||||
const { elapsed } = await timed('2 × 10 MiB "{" + attachment', () =>
|
||||
rejects(pdf, 'no_attachment'),
|
||||
)
|
||||
expect(elapsed).toBeLessThan(2000)
|
||||
})
|
||||
})
|
||||
})
|
||||
Reference in new issue
Block a user