fix(pdf): candidates sorted by data offset, JSON depth pre-scan before parse, budget wording; chore(memory): BLK-009 mitigated with gate, EVAL-014, journal, TODO, CLAUDE.md import notes
This commit is contained in:
1 parent
3a6e92625a
commit
62b2b9d86a
9 files changed
+188
-30
No files matched your search
@@ -261,9 +261,10 @@ type EndstreamFinder = (offset: number) => number
|
||||
|
||||
/**
|
||||
* Searches `endstream` once over the file instead of once per candidate:
|
||||
* candidates come in offset order, so the last hit serves every offset
|
||||
* before it and a miss clears the bytes it covered for the next search. An
|
||||
* offset out of order (never produced today) just restarts the search.
|
||||
* candidates come in offset order (`embeddedStreamHeads` sorts them), so the
|
||||
* last hit serves every offset before it and a miss clears the bytes it
|
||||
* covered for the next search. An offset out of order (never produced since
|
||||
* the sort) restarts the search: a defensive fallback, never the normal path.
|
||||
*/
|
||||
function endstreamFinder(bytes: Uint8Array, end: number): EndstreamFinder {
|
||||
let lastHit = -1
|
||||
@@ -302,14 +303,37 @@ function filterOf(dict: string): EmbeddedFileCandidate['filter'] {
|
||||
return /\/Filter\s*(\[\s*)?\/FlateDecode\b/.test(dict) ? 'FlateDecode' : null
|
||||
}
|
||||
|
||||
/** An `EmbeddedFile` dictionary followed by `stream`, before its data is measured. */
|
||||
interface StreamHead {
|
||||
dict: DictSpan
|
||||
offset: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Every `/Type /EmbeddedFile` dictionary followed by `stream`, sorted by data
|
||||
* offset (stable). Nested dictionaries list outermost first, so their offsets
|
||||
* come out descending and each one would reset the `endstream` finder: 32 of
|
||||
* them rescanned 16 MiB each (audit of 3a6e926, finding 1). Sorted, the
|
||||
* finder never restarts.
|
||||
*/
|
||||
function embeddedStreamHeads(bytes: Uint8Array, end: number): StreamHead[] {
|
||||
const heads: StreamHead[] = []
|
||||
for (const pos of findTypedDicts(bytes, EMBEDDED_FILE, end)) {
|
||||
const dict = dictAround(bytes, pos, end)
|
||||
if (!dict) continue
|
||||
const offset = streamDataOffset(bytes, dict.end, end)
|
||||
if (offset < 0) continue
|
||||
heads.push({ dict, offset })
|
||||
}
|
||||
return heads.sort((a, b) => a.offset - b.offset)
|
||||
}
|
||||
|
||||
function candidateOf(
|
||||
bytes: Uint8Array,
|
||||
dict: DictSpan,
|
||||
{ dict, offset }: StreamHead,
|
||||
end: number,
|
||||
findEndstream: EndstreamFinder,
|
||||
): Omit<EmbeddedFileCandidate, 'name'> | null {
|
||||
const offset = streamDataOffset(bytes, dict.end, end)
|
||||
if (offset < 0) return null
|
||||
const length =
|
||||
declaredLength(bytes, dict.text, offset, end) ??
|
||||
measuredLength(bytes, offset, findEndstream)
|
||||
@@ -392,20 +416,19 @@ export function hasPdfHeader(bytes: Uint8Array): boolean {
|
||||
/**
|
||||
* Every `/Type /EmbeddedFile` stream of the file, named when its Filespec
|
||||
* is in clear (export without object streams); `osteo-report*` names first,
|
||||
* then file order. At most MAX_CANDIDATES, within MAX_PDF_BYTES.
|
||||
* then data offset order. At most MAX_CANDIDATES, within MAX_PDF_BYTES.
|
||||
*/
|
||||
export function findEmbeddedFiles(bytes: Uint8Array): EmbeddedFileCandidate[] {
|
||||
const end = Math.min(bytes.byteLength, MAX_PDF_BYTES)
|
||||
const names = filespecNames(bytes, end)
|
||||
const findEndstream = endstreamFinder(bytes, end)
|
||||
const found: EmbeddedFileCandidate[] = []
|
||||
for (const pos of findTypedDicts(bytes, EMBEDDED_FILE, end)) {
|
||||
const dict = dictAround(bytes, pos, end)
|
||||
if (!dict) continue
|
||||
const candidate = candidateOf(bytes, dict, end, findEndstream)
|
||||
for (const head of embeddedStreamHeads(bytes, end)) {
|
||||
const candidate = candidateOf(bytes, head, end, findEndstream)
|
||||
if (!candidate) continue
|
||||
const { objectNumber } = head.dict
|
||||
const name =
|
||||
dict.objectNumber === null ? null : (names.get(dict.objectNumber) ?? null)
|
||||
objectNumber === null ? null : (names.get(objectNumber) ?? null)
|
||||
found.push({ ...candidate, name })
|
||||
}
|
||||
const named = (c: EmbeddedFileCandidate) =>
|
||||
@@ -440,6 +463,34 @@ function startsJsonObject(data: Uint8Array): boolean {
|
||||
return data[i] === 0x7b
|
||||
}
|
||||
|
||||
/** The report and its envelope nest under 16 levels; deeper is not ours. */
|
||||
const MAX_JSON_DEPTH = 64
|
||||
|
||||
/**
|
||||
* Linear bracket-depth count over the raw bytes before `JSON.parse`: a 16 MiB
|
||||
* document nested thousands deep costs the parser a second and a gigabyte
|
||||
* (audit of 3a6e926, finding 3). String literals are skipped, their escapes
|
||||
* honoured; UTF-8 continuation bytes never collide with the ASCII tokens.
|
||||
*/
|
||||
function withinJsonDepth(data: Uint8Array): boolean {
|
||||
let depth = 0
|
||||
let inString = false
|
||||
for (let i = 0; i < data.byteLength; i += 1) {
|
||||
const byte = data[i]!
|
||||
if (inString) {
|
||||
if (byte === 0x5c) i += 1 // backslash: the escaped byte is skipped
|
||||
else if (byte === 0x22) inString = false
|
||||
continue
|
||||
}
|
||||
if (byte === 0x22) inString = true
|
||||
else if (byte === 0x7b || byte === 0x5b) {
|
||||
depth += 1
|
||||
if (depth > MAX_JSON_DEPTH) return false
|
||||
} else if (byte === 0x7d || byte === 0x5d) depth -= 1
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
/** JSON with a numeric `envelopeVersion` or `version`; undefined otherwise. */
|
||||
function parseVersioned(data: Uint8Array): unknown {
|
||||
let parsed: unknown
|
||||
@@ -464,10 +515,13 @@ export interface LocatedAttachment {
|
||||
/**
|
||||
* The first candidate whose data is a versioned JSON document, decoded once.
|
||||
* None: `attachment_stripped` when the stamp is visible, else `no_attachment`.
|
||||
* One MAX_ATTACHMENT_BYTES budget of examined content (raw or inflated) is
|
||||
* shared by every candidate: a candidate that does not fit what is left ends
|
||||
* the search, so 32 crafted streams cost at most one attachment's worth of
|
||||
* inflation and parsing. A stream cancelled mid-inflation is not charged.
|
||||
* One MAX_ATTACHMENT_BYTES budget of content (raw or inflated) is shared by
|
||||
* every candidate: a candidate that does not fit what is left ends the
|
||||
* search, so at most 16 MiB of candidate content is retained and parsed. A
|
||||
* cancelled inflation is not charged (audit of 3a6e926, finding 2), so up to
|
||||
* MAX_CANDIDATES oversized streams may each cost one capped inflation of CPU
|
||||
* (bounded, no memory retained); charging them would let bombs starve a real
|
||||
* unnamed attachment placed after them.
|
||||
*/
|
||||
export async function locateAttachment(
|
||||
bytes: Uint8Array,
|
||||
@@ -478,7 +532,7 @@ export async function locateAttachment(
|
||||
const data = await candidateBytes(bytes, candidate, budget)
|
||||
if (!data) continue
|
||||
budget -= data.byteLength
|
||||
if (!startsJsonObject(data)) continue
|
||||
if (!startsJsonObject(data) || !withinJsonDepth(data)) continue
|
||||
const parsed = parseVersioned(data)
|
||||
if (parsed !== undefined) return { bytes: data, data: parsed }
|
||||
}
|
||||
|
||||
@@ -3,7 +3,8 @@
|
||||
* on the frozen fixtures (object-stream layout), crafted layouts (two files,
|
||||
* indirect `/Length`, `\r\n`, no space after `/Type`), garbage, the keyword
|
||||
* stamp, the object-stream bomb the reader never inflates, and the candidate
|
||||
* cost bounds (monotonic `endstream` search, shared 16 MiB budget, `{` check).
|
||||
* cost bounds (monotonic `endstream` search over offset-sorted candidates,
|
||||
* shared 16 MiB budget, `{` check, bracket-depth pre-scan before JSON.parse).
|
||||
*/
|
||||
import { deflateSync } from 'node:zlib'
|
||||
import { beforeAll, describe, expect, it, vi } from 'vitest'
|
||||
@@ -19,7 +20,11 @@ import {
|
||||
type EmbeddedFileCandidate,
|
||||
} from '../src'
|
||||
import { inflateFlate } from '../src/attachment/inflate'
|
||||
import { decodePdfString, hasArchiveStamp } from '../src/attachment/scan'
|
||||
import {
|
||||
decodePdfString,
|
||||
hasArchiveStamp,
|
||||
locateAttachment,
|
||||
} from '../src/attachment/scan'
|
||||
import { readFixtureBytes } from './helpers/draft'
|
||||
import {
|
||||
FIXED_EXPORTED_AT,
|
||||
@@ -449,6 +454,25 @@ describe('candidate cost bounds (audit M2)', () => {
|
||||
expect(elapsed).toBeLessThan(1000)
|
||||
})
|
||||
|
||||
it('32 nested EmbeddedFile dicts (descending offsets) without endstream on a 20 MiB file: no candidate, no finder reset', async () => {
|
||||
// Outermost `/Type` first in the file, its `stream` last: offsets come out
|
||||
// reversed; the sort by offset keeps the endstream search to one pass.
|
||||
let nested: Uint8Array = ascii('<< /Type /EmbeddedFile >>\nstream\n')
|
||||
for (let level = 1; level < 32; level += 1) {
|
||||
nested = concat([
|
||||
ascii('<< /Type /EmbeddedFile /A '),
|
||||
nested,
|
||||
ascii(' >>\nstream\n'),
|
||||
])
|
||||
}
|
||||
const pdf = craftPdf([ascii('2 0 obj\n'), nested, filled('x', 20 * MiB)])
|
||||
const { result, elapsed } = await timed('32 nested, no endstream', () =>
|
||||
findEmbeddedFiles(pdf),
|
||||
)
|
||||
expect(result).toEqual([])
|
||||
expect(elapsed).toBeLessThan(500)
|
||||
})
|
||||
|
||||
it('several indirect /Length streams in a row: each measured to its own endstream', () => {
|
||||
const payloads = ['{"version":3}', '{"version":4,"x":1}', '{"v":5}']
|
||||
const pdf = craftPdf(
|
||||
@@ -501,6 +525,63 @@ describe('candidate cost bounds (audit M2)', () => {
|
||||
}
|
||||
})
|
||||
|
||||
describe('bracket-depth pre-scan before JSON.parse (audit finding 3)', () => {
|
||||
const flateObject = (num: number, raw: Uint8Array) => {
|
||||
const data = new Uint8Array(deflateSync(raw))
|
||||
return streamObject({
|
||||
num,
|
||||
dict: `/Type /EmbeddedFile /Filter /FlateDecode /Length ${data.byteLength}`,
|
||||
data,
|
||||
})
|
||||
}
|
||||
const withSpy = async (run: () => Promise<void>) => {
|
||||
const parse = vi.spyOn(JSON, 'parse')
|
||||
try {
|
||||
await run()
|
||||
return parse.mock.calls.length
|
||||
} finally {
|
||||
parse.mockRestore()
|
||||
}
|
||||
}
|
||||
|
||||
it('a 16 KB Flate candidate opening 8 million "[" deep: skipped unparsed, no_attachment fast', async () => {
|
||||
const raw = concat([ascii('{"version":3,"x":'), filled('[', 8_000_000)])
|
||||
const object = flateObject(2, raw)
|
||||
expect(object.byteLength).toBeLessThan(16 * 1024)
|
||||
const calls = await withSpy(async () => {
|
||||
const { elapsed } = await timed('8M-deep candidate', () =>
|
||||
rejects(craftPdf([object]), 'no_attachment'),
|
||||
)
|
||||
expect(elapsed).toBeLessThan(100)
|
||||
})
|
||||
expect(calls).toBe(0)
|
||||
})
|
||||
|
||||
it('2000 "[" inside a string with escaped quotes and backslashes: not rejected, parsed once', async () => {
|
||||
const text = `{"version":3,"note":"\\"${'['.repeat(2000)}\\\\ \\" [[[["}`
|
||||
const pdf = craftPdf([
|
||||
streamObject({
|
||||
num: 2,
|
||||
dict: `/Type /EmbeddedFile /Length ${text.length}`,
|
||||
data: ascii(text),
|
||||
}),
|
||||
])
|
||||
const calls = await withSpy(async () => {
|
||||
const located = await locateAttachment(pdf)
|
||||
expect(located.data).toMatchObject({ version: 3 })
|
||||
})
|
||||
expect(calls).toBe(1)
|
||||
})
|
||||
|
||||
it('the real envelope passes the pre-scan and is parsed once', async () => {
|
||||
const calls = await withSpy(async () => {
|
||||
const located = await locateAttachment(exported)
|
||||
expect(located.data).toMatchObject({ envelopeVersion: 1 })
|
||||
})
|
||||
expect(calls).toBe(1)
|
||||
})
|
||||
})
|
||||
|
||||
describe('one 16 MiB budget shared by every candidate', () => {
|
||||
const garbageObject = (num: number) =>
|
||||
streamObject({
|
||||
|
||||
Reference in new issue
Block a user