test(tests): R4.1a frozen TS scanner (N-1 PDF reader) and synthetic L/CAP reports

This commit is contained in:
bchanot committed 2026-10-18 05:19:41 +02:00
1 parent eff54059f4
commit 7ecbee9aae
10 files changed
+32616

No files matched your search

+1
View File
@@ -6,3 +6,4 @@ apps/app/tests/fixtures/goldens/
packages/core-wasm/types/ packages/core-wasm/types/
packages/core-wasm/pkg/ packages/core-wasm/pkg/
packages/core-wasm/pkg-proofs/ packages/core-wasm/pkg-proofs/
packages/core-wasm/tests/n-1/
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,178 @@
/**
* R4.1 freeze O13 (R4 plan B.5.1, B.4): the synthetic large reports L (60 observations) and
* CAP (500, the engine's observation cap), built from the demo report. Every observation
* carries zones on each view of the draft sheet (main and head), so an export rasterises two
* views per observation: 120 and 1,000 views, the memory classes the baseline (O12) and R4.6
* measure. Zones and findings are the demo's, cycled per view; the texts follow the main
* view's source observation. `MAKE_R4_ORACLES=1` writes `tests/fixtures/r4/reports/`;
* without it the reports are rebuilt and compared with the committed files.
*/
import { beforeAll, describe, expect, it } from 'vitest'
import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'
import { resolve } from 'node:path'
import type {
Finding,
Observation,
Report,
ReportZone,
} from '@faunosteo/core-wasm/types'
import { isObservable, type SheetIndex } from '@/services/sheetIndex'
import { FIXTURES_DIR } from '../helpers/goldens'
import { loadTestSheet, sheetSnapshot } from '../helpers/sheet'
const MAKE = process.env.MAKE_R4_ORACLES === '1'
const OUT_DIR = resolve(FIXTURES_DIR, 'r4/reports')
const SIZES = { L: 60, CAP: 500 } as const
type SizeName = keyof typeof SIZES
const NAMES = Object.keys(SIZES) as SizeName[]
const EXPECTED_VIEWS: Record<SizeName, number> = { L: 120, CAP: 1000 }
/** `PAGE_CAPS` of packages/pdf/src/export.ts at the freeze: the engine does not export it. */
const PAGE_CAPS = { observations: 500, viewsPerObservation: 8 } as const
/** The zones a demo observation holds on one view, with the findings of their finding zones. */
interface ViewPart {
source: Observation
zones: ReportZone[]
findings: Finding[]
}
const readJson = (path: string): unknown =>
JSON.parse(readFileSync(path, 'utf8'))
const observationId = (index: number): string =>
`00000000-0000-4000-8000-${String(index + 1).padStart(12, '0')}`
function findingZoneOf(sheet: SheetIndex, annotationId: string): string | null {
const annotation = sheet.annotation(annotationId)
return isObservable(annotation) ? annotation.finding_zone_id : null
}
function partOn(
sheet: SheetIndex,
source: Observation,
viewId: string,
): ViewPart | null {
const zones = source.zones.filter(
(z) => sheet.annotation(z.annotationId)?.view_id === viewId,
)
if (zones.length === 0) return null
const zoneIds = new Set(
zones.map((z) => findingZoneOf(sheet, z.annotationId)),
)
const findings = source.findings.filter((f) => zoneIds.has(f.findingZoneId))
return { source, zones, findings }
}
/** One list of demo parts per view of the sheet, in document order (main first). */
function partsByView(sheet: SheetIndex, demo: Report): ViewPart[][] {
return sheet.sheet.document.views.map((view) =>
demo.observations.flatMap((o) => partOn(sheet, o, view.id) ?? []),
)
}
function observationAt(parts: ViewPart[][], index: number): Observation {
const picks = parts.map((list) => list[index % list.length]!)
return {
...picks[0]!.source,
id: observationId(index),
zones: picks.flatMap((p) => p.zones),
findings: picks.flatMap((p) => p.findings),
}
}
function syntheticReport(
demo: Report,
parts: ViewPart[][],
name: SizeName,
): Report {
return {
...demo,
id: `fixture-r4-${name}`,
observations: Array.from({ length: SIZES[name] }, (_, i) =>
observationAt(parts, i),
),
}
}
/** Per observation, the views the engine rasterises (packages/pdf observationViewIds). */
function rasterisedViews(report: Report, snapshotViews: Set<string>): number[] {
return report.observations.map(
(o) =>
[...new Set(o.zones.map((z) => z.viewId))].filter((id) =>
snapshotViews.has(id),
).length,
)
}
let sheet: SheetIndex
let demo: Report
let reports: Record<SizeName, Report>
beforeAll(async () => {
sheet = await loadTestSheet()
demo = readJson(resolve(FIXTURES_DIR, 'report-v3.demo.json')) as Report
const parts = partsByView(sheet, demo)
reports = {
L: syntheticReport(demo, parts, 'L'),
CAP: syntheticReport(demo, parts, 'CAP'),
}
})
describe('the draft sheet the reports are built on', () => {
it('has two views, main first, and the demo has zones on both', () => {
const views = sheet.sheet.document.views.map((v) => v.id)
expect(views).toEqual(['main', 'head'])
expect(sheet.mainViewId).toBe('main')
for (const list of partsByView(sheet, demo)) {
expect(list.length).toBeGreaterThan(0)
}
})
})
describe.each(NAMES)('synthetic report %s', (name) => {
const file = resolve(OUT_DIR, `${name}.json`)
it('matches the committed file', () => {
if (MAKE) {
mkdirSync(OUT_DIR, { recursive: true })
writeFileSync(file, `${JSON.stringify(reports[name], null, 2)}\n`)
}
expect(readJson(file)).toEqual(reports[name])
})
it('comes back byte for byte through the draft restore (draft.restore)', () => {
const report = reports[name]
// The text the store writes to osteo:draft (persistence.ts) and reads back on reload.
const outcome = sheet.ops.restoreDraft(JSON.stringify(report), null)
if (!outcome.ok) throw new Error(`draft.restore refused: ${outcome.kind}`)
const { rebase, archive, archiveRejected } = outcome.value
expect(rebase.rebased).toBe(false)
expect(rebase.unresolvedZones + rebase.unresolvedFindings).toBe(0)
expect(archive).toBeNull()
expect(archiveRejected).toBe(false)
expect(rebase.report).toStrictEqual(report)
expect(JSON.stringify(rebase.report)).toBe(JSON.stringify(report))
})
it('rasterises two views per observation', () => {
const report = reports[name]
const snapshot = sheetSnapshot(sheet, report)
const views = new Set(snapshot.document.views.map((v) => v.id))
const perObservation = rasterisedViews(report, views)
expect(new Set(perObservation)).toEqual(new Set([2]))
const total = perObservation.reduce((sum, n) => sum + n, 0)
expect(total).toBe(EXPECTED_VIEWS[name])
})
it('stays within the export caps', () => {
const { observations } = reports[name]
expect(observations).toHaveLength(SIZES[name])
expect(observations.length).toBeLessThanOrEqual(PAGE_CAPS.observations)
for (const o of observations) {
const views = new Set(o.zones.map((z) => z.viewId)).size
expect(views).toBeLessThanOrEqual(PAGE_CAPS.viewsPerObservation)
}
})
})
@@ -0,0 +1,90 @@
/**
* O11 (R4 plan B.5.1, B.2.2): the frozen TypeScript scanner, the permanent N-1 PDF reader,
* still reads the archives of every earlier build. The copy under `./n-1/scanner` stands for
* the R3 build and release 0.2.0: an R4 PDF it cannot read would break a rollback.
*/
import { readFileSync } from 'node:fs'
import { fileURLToPath } from 'node:url'
import { readAttachedDocument } from './n-1/scanner/attachment/read'
import { ArchiveReadError } from './n-1/scanner/errors'
const FIXTURES = '../../../apps/app/tests/fixtures/'
const fixtureBytes = (name: string): Uint8Array =>
new Uint8Array(
readFileSync(fileURLToPath(new URL(`${FIXTURES}${name}`, import.meta.url))),
)
const fixtureJson = (name: string): unknown =>
JSON.parse(new TextDecoder().decode(fixtureBytes(name)))
const ascii = (text: string): Uint8Array =>
Uint8Array.from(text, (c) => c.charCodeAt(0))
/** `faunosteo-archive/1` as pdf-lib 1.17's `setKeywords` writes it: UTF-16BE hex, BOM first. */
const HEX_STAMP =
'<FEFF006600610075006E006F007300740065006F002D0061007200630068006900760065002F0031>'
/**
* A one-page PDF with a classic xref and no EmbeddedFile, whose Info dictionary carries
* `keywords` as its `/Keywords` token: what a stamped archive looks like once a tool has
* removed its attachment. Built by hand: pdf-lib is not a dependency of this package.
*/
function pdfWithoutAttachment(keywords: string): Uint8Array {
const objects = [
'<< /Type /Catalog /Pages 2 0 R >>',
'<< /Type /Pages /Kids [3 0 R] /Count 1 >>',
'<< /Type /Page /Parent 2 0 R /MediaBox [0 0 595.28 841.89] >>',
`<< /Keywords ${keywords} >>`,
]
let text = '%PDF-1.7\n'
const offsets = objects.map((body, i) => {
const offset = text.length
text += `${i + 1} 0 obj\n${body}\nendobj\n`
return offset
})
const xrefAt = text.length
text += `xref\n0 ${objects.length + 1}\n0000000000 65535 f \n`
text += offsets
.map((o) => `${String(o).padStart(10, '0')} 00000 n \n`)
.join('')
text += `trailer\n<< /Size ${objects.length + 1} /Root 1 0 R /Info 4 0 R >>\n`
text += `startxref\n${xrefAt}\n%%EOF\n`
return ascii(text)
}
async function readFailure(bytes: Uint8Array): Promise<ArchiveReadError> {
const failure: unknown = await readAttachedDocument(bytes).then(
() => null,
(error: unknown) => error,
)
expect(failure).toBeInstanceOf(ArchiveReadError)
return failure as ArchiveReadError
}
describe('the frozen scanner reads the archives of earlier builds', () => {
it.each([
['report-v2.full.pdf', 'report-v2.full.json'],
['report-v2.minimal.pdf', 'report-v2.minimal.json'],
['report-v3.bare.pdf', 'report-v3.demo.json'],
])('%s carries %s', async (pdf, json) => {
const data = await readAttachedDocument(fixtureBytes(pdf))
expect(data).toStrictEqual(fixtureJson(json))
})
})
describe('a stamped archive whose attachment was removed', () => {
it.each([
['UTF-16BE hex (the export)', HEX_STAMP],
['literal', '(faunosteo-archive/1)'],
])('stamp written as %s: attachment_stripped', async (_form, token) => {
const failure = await readFailure(pdfWithoutAttachment(token))
expect(failure.kind).toBe('attachment_stripped')
})
it('the same file without the stamp: no_attachment', async () => {
const failure = await readFailure(pdfWithoutAttachment('(other)'))
expect(failure.kind).toBe('no_attachment')
})
})
@@ -0,0 +1,26 @@
# Frozen TypeScript scanner (O11)
Never edited; the permanent N-1 PDF reader (R4 plan B.2.2).
The byte-scan attachment reader of the R3 build and release 0.2.0, copied
verbatim from `packages/pdf/src` at commit
`eff54059f45fb24a2255f0c099d4585021ef58eb` (branch `feature/rust-r4-pdf-core`,
R4.1). Every R4 PDF must stay readable by it, and an R4 PDF whose EmbeddedFile
stream was removed must answer `attachment_stripped` through it: a rollback to
the R3 build then reads what R4 wrote. `tests/n-1.scanner.test.ts` proves it
reads the frozen fixture PDFs.
| Copy | Source | sha256 |
| ----------------------- | ---------------------------------------- | ------------------------------------------------------------------ |
| `errors.ts` | `packages/pdf/src/errors.ts` | `06c2f518190f96f969539c4298a222e6f6fbae6a71493294d480d7ea3381c2b2` |
| `attachment/scan.ts` | `packages/pdf/src/attachment/scan.ts` | `82c1c0ef51f29191c8a5238f39ed21222a890658721da7d0c964f2053e7b9894` |
| `attachment/inflate.ts` | `packages/pdf/src/attachment/inflate.ts` | `6a8275a1fe68879d90d49fffcd5203e515f840738ea2083370e831b93beeca10` |
| `attachment/read.ts` | `packages/pdf/src/attachment/read.ts` | `d11e93cc143edaeb3a7f07327b85b669839e43e35817625c14dc6aa03659b91f` |
Changed lines: none. The source tree shape (`errors.ts` beside `attachment/`)
is kept, so every import specifier (`../errors`, `./scan`, `./inflate`)
resolves unchanged. The four modules import nothing else; `attachment/write.ts`
(pdf-lib, export side) is not part of the reader and is not copied.
`attachment/scan.ts` is not Prettier-clean (line 481, as in its source):
never reformat it.
@@ -0,0 +1,83 @@
/**
* Hard ceiling on an attachment, inflated or not: a report archive weighs a
* few hundred KB; anything bigger is a decompression bomb or corruption.
*/
export const MAX_ATTACHMENT_BYTES = 16 * 1024 * 1024
/**
* Compressed bytes handed to the inflater per write. Chromium's
* `DecompressionStream` inflates everything it has been given before the
* consumer reads it, so a stream written whole lets a bomb inflate gigabytes
* before `reader.cancel()` lands (measured: +2.2 GB RSS on a 4 MB / 4 GiB
* bomb). One awaited slice at a time keeps that read-ahead to what 16 KiB
* can inflate (about 16 MiB).
*/
const WRITE_SLICE_BYTES = 16 * 1024
type Writer = WritableStreamDefaultWriter<Uint8Array<ArrayBuffer>>
type Reader = ReadableStreamDefaultReader<Uint8Array>
function concat(chunks: Uint8Array[], total: number): Uint8Array {
const out = new Uint8Array(total)
let offset = 0
for (const chunk of chunks) {
out.set(chunk, offset)
offset += chunk.byteLength
}
return out
}
/** Feeds the inflater one slice per awaited write, then closes it. */
async function feedSlices(writer: Writer, bytes: Uint8Array): Promise<void> {
// The cast compiles under both the engine (node) and the app (DOM) libs.
const source = bytes as Uint8Array<ArrayBuffer>
for (let at = 0; at < source.byteLength; at += WRITE_SLICE_BYTES) {
await writer.write(source.subarray(at, at + WRITE_SLICE_BYTES))
}
await writer.close()
}
/** Collects the inflated chunks; null as soon as their total exceeds `limit`. */
async function drainUnder(
reader: Reader,
limit: number,
): Promise<Uint8Array | null> {
const chunks: Uint8Array[] = []
let total = 0
for (;;) {
const { done, value } = await reader.read()
if (done) return concat(chunks, total)
total += value.byteLength
if (total > limit) return null
chunks.push(value)
}
}
/**
* FlateDecode (zlib framing) under `limit`, the attachment cap by default.
* Null on any failure (truncated or corrupt stream, trailing junk, bad
* checksum) or as soon as the output exceeds the limit. The feed and the
* drain run concurrently; over the limit both sides are torn down, which
* also rejects the feed's pending write.
*/
export async function inflateFlate(
bytes: Uint8Array,
limit: number = MAX_ATTACHMENT_BYTES,
): Promise<Uint8Array | null> {
if (typeof DecompressionStream === 'undefined') return null
const inflater = new DecompressionStream('deflate')
const writer = inflater.writable.getWriter()
const reader = inflater.readable.getReader()
// A stream error surfaces on the drain; the feed's own rejection is noise.
const fed = feedSlices(writer, bytes)
fed.catch(() => undefined)
try {
const out = await drainUnder(reader, limit)
if (out === null) {
await Promise.allSettled([reader.cancel(), writer.abort()])
}
return out
} catch {
return null
}
}
@@ -0,0 +1,36 @@
import { ArchiveReadError } from '../errors'
import {
ARCHIVE_STAMP,
hasPdfHeader,
locateAttachment,
MAX_PDF_BYTES,
} from './scan'
/**
* The raw JSON a Faunosteo PDF carries, read from the bytes alone (BLK-009):
* no PDF object graph is parsed, so no object stream is ever inflated. pdf-lib
* stays on the export side.
*/
export { ATTACHMENT_NAME_PREFIX } from './scan'
export const PDF_KEYWORD = `${ARCHIVE_STAMP}/1`
/** Size cap first, then the `%PDF-` header: anything else is `invalid_pdf`. */
function assertPdfBytes(bytes: Uint8Array): void {
if (bytes.byteLength > MAX_PDF_BYTES || !hasPdfHeader(bytes)) {
throw new ArchiveReadError('invalid_pdf')
}
}
/**
* The raw JSON carried by the PDF, whatever its shape (bare report of any
* version or archive envelope), before any validation: what the archive
* check tool shows next to the read result. Throws ArchiveReadError
* (`invalid_pdf`, `no_attachment`, `attachment_stripped`).
*/
export async function readAttachedDocument(
bytes: Uint8Array,
): Promise<unknown> {
assertPdfBytes(bytes)
return (await locateAttachment(bytes)).data
}
@@ -0,0 +1,549 @@
import { ArchiveReadError } from '../errors'
import { inflateFlate, MAX_ATTACHMENT_BYTES } from './inflate'
/**
* Locates the Faunosteo attachment by scanning the PDF bytes (BLK-009): a
* stream object can never live inside an object stream, so every
* `/Type /EmbeddedFile` dictionary and its data are in clear in the file.
* Nothing here parses the object graph, the xref or an `/ObjStm`, so the
* only inflation on the import path is the capped one of `inflateFlate`.
* The scanner never throws on garbage: it returns fewer candidates.
*/
/** A Faunosteo PDF weighs a few hundred KB; anything larger is not one. */
export const MAX_PDF_BYTES = 50 * 1024 * 1024
export const ATTACHMENT_NAME_PREFIX = 'osteo-report'
/** The keyword stamp every archive carries (`faunosteo-archive/<n>`). */
export const ARCHIVE_STAMP = 'faunosteo-archive'
/** Candidates beyond this count are ignored: a crafted file cannot make the reader loop. */
const MAX_CANDIDATES = 32
/** A stream or Filespec dictionary is small; a longer one is skipped. */
const MAX_DICT_BYTES = 4096
/** Window read before a dictionary for its `n 0 obj` header. */
const HEADER_WINDOW = 48
export interface EmbeddedFileCandidate {
/** First byte of the stream data. */
offset: number
/** Byte count of the stream data (declared, or measured to `endstream`). */
length: number
filter: 'FlateDecode' | null
/** From a visible `/Filespec` whose `/EF /F` points at this stream. */
name: string | null
}
const encode = (text: string): Uint8Array =>
Uint8Array.from(text, (c) => c.charCodeAt(0))
const TYPE = encode('/Type')
const EMBEDDED_FILE = encode('/EmbeddedFile')
const FILESPEC = encode('/Filespec')
const KEYWORDS = encode('/Keywords')
const STREAM = encode('stream')
const ENDSTREAM = encode('endstream')
const PDF_HEADER = encode('%PDF-')
// ---------------------------------------------------------------------------
// Byte helpers: every loop is bounded by `end` (at most MAX_PDF_BYTES)
function isWhitespace(byte: number): boolean {
return (
byte === 0x20 ||
byte === 0x0a ||
byte === 0x0d ||
byte === 0x09 ||
byte === 0x0c ||
byte === 0x00
)
}
/** PDF delimiters and whitespace end a name token. */
function isDelimiter(byte: number): boolean {
return isWhitespace(byte) || '()<>[]{}/%'.includes(String.fromCharCode(byte))
}
/** Naive forward search; the needle's first byte is the fast reject. */
function indexOf(
bytes: Uint8Array,
needle: Uint8Array,
from: number,
end: number,
): number {
const last = end - needle.length
const first = needle[0]!
for (let i = Math.max(from, 0); i <= last; i += 1) {
if (bytes[i] !== first) continue
let j = 1
while (j < needle.length && bytes[i + j] === needle[j]) j += 1
if (j === needle.length) return i
}
return -1
}
function skipWhitespace(bytes: Uint8Array, from: number, end: number): number {
let i = from
while (i < end && isWhitespace(bytes[i]!)) i += 1
return i
}
/** Latin-1 view of a small window (never the whole file). */
function latin1(bytes: Uint8Array, start: number, end: number): string {
let out = ''
for (let i = start; i < end; i += 1) out += String.fromCharCode(bytes[i]!)
return out
}
/** UTF-16BE with BOM (how pdf-lib writes text strings) or Latin-1 bytes. */
function decodeTextBytes(raw: number[]): string {
if (raw.length >= 2 && raw[0] === 0xfe && raw[1] === 0xff) {
let out = ''
for (let i = 2; i + 1 < raw.length; i += 2) {
out += String.fromCharCode((raw[i]! << 8) | raw[i + 1]!)
}
return out
}
return String.fromCharCode(...raw)
}
/** A `(literal)` or `<hex>` string token as text; null for anything else. */
export function decodePdfString(token: string): string | null {
if (token.startsWith('<') && token.endsWith('>')) {
const hex = token.slice(1, -1).replace(/[^0-9a-fA-F]/g, '')
const raw: number[] = []
for (let i = 0; i + 1 < hex.length; i += 2) {
raw.push(parseInt(hex.slice(i, i + 2), 16))
}
return decodeTextBytes(raw)
}
if (token.startsWith('(') && token.endsWith(')')) {
const body = token.slice(1, -1).replace(/\\([()\\])/g, '$1')
return decodeTextBytes(Array.from(body, (c) => c.charCodeAt(0) & 0xff))
}
return null
}
// ---------------------------------------------------------------------------
// Dictionaries: `/Type /Name` matches and their `<< … >>` bounds
/** Every `/Type /<name>` (space optional) whose name ends on a delimiter. */
function findTypedDicts(
bytes: Uint8Array,
name: Uint8Array,
end: number,
): number[] {
const hits: number[] = []
let from = 0
while (hits.length < MAX_CANDIDATES) {
const at = indexOf(bytes, name, from, end)
if (at < 0) break
from = at + name.length
const after = at + name.length
if (after < end && !isDelimiter(bytes[after]!)) continue
let before = at
while (before > 0 && isWhitespace(bytes[before - 1]!)) before -= 1
const typeAt = before - TYPE.length
if (typeAt >= 0 && indexOf(bytes, TYPE, typeAt, before) === typeAt) {
hits.push(typeAt)
}
}
return hits
}
/** Index of the `<<` opening the dictionary holding `pos` (nested dicts skipped); -1 when none. */
function dictStart(bytes: Uint8Array, pos: number): number {
const floor = Math.max(0, pos - MAX_DICT_BYTES)
let depth = 0
let i = pos
while (i - 2 >= floor) {
const pair = (bytes[i - 2]! << 8) | bytes[i - 1]!
if (pair === 0x3e3e)
depth += 1 // >>
else if (pair === 0x3c3c) {
if (depth === 0) return i - 2
depth -= 1
}
i -= pair === 0x3e3e || pair === 0x3c3c ? 2 : 1
}
return -1
}
/** Index just past the `>>` closing the dictionary opened at `start`; -1 when none. */
function dictEnd(bytes: Uint8Array, start: number, end: number): number {
const limit = Math.min(end, start + MAX_DICT_BYTES)
let depth = 0
let i = start
while (i + 1 < limit) {
const pair = (bytes[i]! << 8) | bytes[i + 1]!
if (pair === 0x3c3c) depth += 1
else if (pair === 0x3e3e) {
depth -= 1
if (depth === 0) return i + 2
}
i += pair === 0x3c3c || pair === 0x3e3e ? 2 : 1
}
return -1
}
interface DictSpan {
start: number
end: number
text: string
objectNumber: number | null
}
/** The dictionary around a `/Type` match, its text and the `n 0 obj` header before it. */
function dictAround(
bytes: Uint8Array,
pos: number,
end: number,
): DictSpan | null {
const start = dictStart(bytes, pos)
if (start < 0) return null
const close = dictEnd(bytes, start, end)
if (close < 0) return null
const header = latin1(bytes, Math.max(0, start - HEADER_WINDOW), start)
const match = /(\d{1,10})\s+\d{1,10}\s+obj\s*$/.exec(header)
return {
start,
end: close,
text: latin1(bytes, start, close),
objectNumber: match ? Number(match[1]) : null,
}
}
// ---------------------------------------------------------------------------
// Stream data bounds
/** Data offset after `stream` + `\r\n` or `\n`; -1 when the keyword is missing. */
function streamDataOffset(
bytes: Uint8Array,
dictClose: number,
end: number,
): number {
const at = skipWhitespace(bytes, dictClose, end)
if (indexOf(bytes, STREAM, at, Math.min(end, at + STREAM.length)) !== at) {
return -1
}
let i = at + STREAM.length
if (bytes[i] === 0x0d) i += 1
if (bytes[i] !== 0x0a) return -1
return i + 1
}
/** A direct `/Length N` (never an `N 0 R` reference), when it lands on `endstream`. */
function declaredLength(
bytes: Uint8Array,
dict: string,
offset: number,
end: number,
): number | null {
const match = /\/Length\s+(\d{1,10})(?!\d)(?!\s+\d+\s+R)/.exec(dict)
if (!match) return null
const length = Number(match[1])
if (offset + length > end) return null
const tail = skipWhitespace(bytes, offset + length, end)
const found = indexOf(
bytes,
ENDSTREAM,
tail,
Math.min(end, tail + ENDSTREAM.length),
)
return found === tail ? length : null
}
/** Farthest `endstream` worth finding: a longer candidate is refused anyway. */
const MEASURE_WINDOW = MAX_ATTACHMENT_BYTES + 16
/** Index of the first `endstream` at or after `offset`, within MEASURE_WINDOW; -1 when none. */
type EndstreamFinder = (offset: number) => number
/**
* Searches `endstream` once over the file instead of once per candidate:
* candidates come in offset order (`embeddedStreamHeads` sorts them), so the
* last hit serves every offset before it and a miss clears the bytes it
* covered for the next search. An offset out of order (never produced since
* the sort) restarts the search: a defensive fallback, never the normal path.
*/
function endstreamFinder(bytes: Uint8Array, end: number): EndstreamFinder {
let lastHit = -1
let clearedTo = 0
let lastOffset = 0
return (offset) => {
if (offset < lastOffset) {
lastHit = -1
clearedTo = 0
}
lastOffset = offset
if (lastHit >= offset) return lastHit
const limit = Math.min(end, offset + MEASURE_WINDOW)
const at = indexOf(bytes, ENDSTREAM, Math.max(offset, clearedTo), limit)
if (at >= 0) lastHit = at
else clearedTo = Math.max(clearedTo, limit - ENDSTREAM.length + 1)
return at
}
}
/** Fallback for an indirect or wrong `/Length`: up to `endstream`, the EOL before it dropped. */
function measuredLength(
bytes: Uint8Array,
offset: number,
findEndstream: EndstreamFinder,
): number | null {
const at = findEndstream(offset)
if (at < 0) return null
let stop = at
if (stop > offset && bytes[stop - 1] === 0x0a) stop -= 1
if (stop > offset && bytes[stop - 1] === 0x0d) stop -= 1
return stop - offset
}
function filterOf(dict: string): EmbeddedFileCandidate['filter'] {
return /\/Filter\s*(\[\s*)?\/FlateDecode\b/.test(dict) ? 'FlateDecode' : null
}
/** An `EmbeddedFile` dictionary followed by `stream`, before its data is measured. */
interface StreamHead {
dict: DictSpan
offset: number
}
/**
* Every `/Type /EmbeddedFile` dictionary followed by `stream`, sorted by data
* offset (stable). Nested dictionaries list outermost first, so their offsets
* come out descending and each one would reset the `endstream` finder: 32 of
* them rescanned 16 MiB each (audit of 3a6e926, finding 1). Sorted, the
* finder never restarts.
*/
function embeddedStreamHeads(bytes: Uint8Array, end: number): StreamHead[] {
const heads: StreamHead[] = []
for (const pos of findTypedDicts(bytes, EMBEDDED_FILE, end)) {
const dict = dictAround(bytes, pos, end)
if (!dict) continue
const offset = streamDataOffset(bytes, dict.end, end)
if (offset < 0) continue
heads.push({ dict, offset })
}
return heads.sort((a, b) => a.offset - b.offset)
}
function candidateOf(
bytes: Uint8Array,
{ dict, offset }: StreamHead,
end: number,
findEndstream: EndstreamFinder,
): Omit<EmbeddedFileCandidate, 'name'> | null {
const length =
declaredLength(bytes, dict.text, offset, end) ??
measuredLength(bytes, offset, findEndstream)
if (length === null) return null
return { offset, length, filter: filterOf(dict.text) }
}
// ---------------------------------------------------------------------------
// Filespec names and the keyword stamp
const STRING_TOKEN = String.raw`\((?:\\.|[^\\)])*\)|<[0-9a-fA-F\s]*>`
/** `/F` (else `/UF`) of a Filespec dictionary, the `/EF` sub-dictionary set aside. */
function filespecName(dict: string): string | null {
const withoutEf = dict.replace(/\/EF\s*<<[\s\S]*?>>/, '')
const f = new RegExp(String.raw`/F\s*(${STRING_TOKEN})`).exec(withoutEf)
const uf = new RegExp(String.raw`/UF\s*(${STRING_TOKEN})`).exec(withoutEf)
const token = f?.[1] ?? uf?.[1]
return token ? decodePdfString(token) : null
}
/** Object number of the stream a Filespec's `/EF << /F n 0 R >>` points at. */
function filespecStreamObject(dict: string): number | null {
const ef = /\/EF\s*<<([\s\S]*?)>>/.exec(dict)
const ref = ef ? /\/F\s+(\d{1,10})\s+\d{1,10}\s+R/.exec(ef[1]!) : null
return ref ? Number(ref[1]) : null
}
/** Stream object number → file name, from every Filespec dictionary in clear. */
function filespecNames(bytes: Uint8Array, end: number): Map<number, string> {
const names = new Map<number, string>()
for (const pos of findTypedDicts(bytes, FILESPEC, end)) {
const dict = dictAround(bytes, pos, end)
if (!dict) continue
const target = filespecStreamObject(dict.text)
const name = filespecName(dict.text)
if (target !== null && name !== null && !names.has(target)) {
names.set(target, name)
}
}
return names
}
/** The string token right after a `/Keywords` key, as text. */
function keywordsAt(
bytes: Uint8Array,
keyAt: number,
end: number,
): string | null {
const start = skipWhitespace(bytes, keyAt + KEYWORDS.length, end)
const limit = Math.min(end, start + MAX_DICT_BYTES)
const window = latin1(bytes, start, limit)
const token = new RegExp(`^(?:${STRING_TOKEN})`).exec(window)
return token ? decodePdfString(token[0]) : null
}
/** True when a visible `/Keywords` string carries the archive stamp. */
export function hasArchiveStamp(bytes: Uint8Array): boolean {
const end = Math.min(bytes.byteLength, MAX_PDF_BYTES)
let from = 0
for (let n = 0; n < MAX_CANDIDATES; n += 1) {
const at = indexOf(bytes, KEYWORDS, from, end)
if (at < 0) return false
from = at + KEYWORDS.length
if (isDelimiter(bytes[from] ?? 0x20)) {
if (keywordsAt(bytes, at, end)?.includes(ARCHIVE_STAMP)) return true
}
}
return false
}
/** `%PDF-` within the first KiB, as viewers accept it. */
export function hasPdfHeader(bytes: Uint8Array): boolean {
return indexOf(bytes, PDF_HEADER, 0, Math.min(bytes.byteLength, 1024)) >= 0
}
// ---------------------------------------------------------------------------
// Public scan
/**
* Every `/Type /EmbeddedFile` stream of the file, named when its Filespec
* is in clear (export without object streams); `osteo-report*` names first,
* then data offset order. At most MAX_CANDIDATES, within MAX_PDF_BYTES.
*/
export function findEmbeddedFiles(bytes: Uint8Array): EmbeddedFileCandidate[] {
const end = Math.min(bytes.byteLength, MAX_PDF_BYTES)
const names = filespecNames(bytes, end)
const findEndstream = endstreamFinder(bytes, end)
const found: EmbeddedFileCandidate[] = []
for (const head of embeddedStreamHeads(bytes, end)) {
const candidate = candidateOf(bytes, head, end, findEndstream)
if (!candidate) continue
const { objectNumber } = head.dict
const name =
objectNumber === null ? null : (names.get(objectNumber) ?? null)
found.push({ ...candidate, name })
}
const named = (c: EmbeddedFileCandidate) =>
c.name?.startsWith(ATTACHMENT_NAME_PREFIX) ? 0 : 1
return found.sort((a, b) => named(a) - named(b))
}
/** Raw or inflated stream data under `budget`; null when unusable or inflating past it. */
async function candidateBytes(
bytes: Uint8Array,
candidate: EmbeddedFileCandidate,
budget: number,
): Promise<Uint8Array | null> {
const data = bytes.subarray(
candidate.offset,
candidate.offset + candidate.length,
)
if (candidate.filter === null) return data
return inflateFlate(data, budget)
}
/** `{` first, past an optional UTF-8 BOM and JSON whitespace: the only shape a version lives in. */
function startsJsonObject(data: Uint8Array): boolean {
let i = 0
if (data[0] === 0xef && data[1] === 0xbb && data[2] === 0xbf) i = 3
while (
i < data.byteLength &&
' \t\n\r'.includes(String.fromCharCode(data[i]!))
) {
i += 1
}
return data[i] === 0x7b
}
/** The report and its envelope nest under 16 levels; deeper is not ours. */
const MAX_JSON_DEPTH = 64
/**
* Linear bracket-depth count over the raw bytes before `JSON.parse`: a 16 MiB
* document nested thousands deep costs the parser a second and a gigabyte
* (audit of 3a6e926, finding 3). String literals are skipped, their escapes
* honoured; UTF-8 continuation bytes never collide with the ASCII tokens.
*/
function withinJsonDepth(data: Uint8Array): boolean {
let depth = 0
let inString = false
for (let i = 0; i < data.byteLength; i += 1) {
const byte = data[i]!
if (inString) {
if (byte === 0x5c) i += 1 // backslash: the escaped byte is skipped
else if (byte === 0x22) inString = false
continue
}
if (byte === 0x22) inString = true
else if (byte === 0x7b || byte === 0x5b) {
depth += 1
if (depth > MAX_JSON_DEPTH) return false
} else if (byte === 0x7d || byte === 0x5d) depth -= 1
}
return true
}
/** JSON with a numeric `envelopeVersion` or `version`; undefined otherwise. */
function parseVersioned(data: Uint8Array): unknown {
let parsed: unknown
try {
parsed = JSON.parse(new TextDecoder().decode(data))
} catch {
return undefined
}
if (typeof parsed !== 'object' || parsed === null) return undefined
const record = parsed as Record<string, unknown>
const versioned =
typeof record['envelopeVersion'] === 'number' ||
typeof record['version'] === 'number'
return versioned ? parsed : undefined
}
export interface LocatedAttachment {
bytes: Uint8Array
data: unknown
}
/**
* The first candidate whose data is a versioned JSON document, decoded once.
* None: `attachment_stripped` when the stamp is visible, else `no_attachment`.
* One MAX_ATTACHMENT_BYTES budget of content (raw or inflated) is shared by
* every candidate: a candidate that does not fit what is left ends the
* search, so at most 16 MiB of candidate content is retained and parsed. A
* cancelled inflation is not charged (audit of 3a6e926, finding 2), so up to
* MAX_CANDIDATES oversized streams may each cost one capped inflation of CPU
* (bounded, no memory retained); charging them would let bombs starve a real
* unnamed attachment placed after them.
*/
export async function locateAttachment(
bytes: Uint8Array,
): Promise<LocatedAttachment> {
let budget = MAX_ATTACHMENT_BYTES
for (const candidate of findEmbeddedFiles(bytes)) {
if (candidate.length > budget) break
const data = await candidateBytes(bytes, candidate, budget)
if (!data) continue
budget -= data.byteLength
if (!startsJsonObject(data) || !withinJsonDepth(data)) continue
const parsed = parseVersioned(data)
if (parsed !== undefined) return { bytes: data, data: parsed }
}
throw new ArchiveReadError(
hasArchiveStamp(bytes) ? 'attachment_stripped' : 'no_attachment',
)
}
/** The attachment's decoded bytes (BLK-009 reader). Throws ArchiveReadError. */
export async function readAttachmentBytes(
bytes: Uint8Array,
): Promise<Uint8Array> {
return (await locateAttachment(bytes)).bytes
}
@@ -0,0 +1,55 @@
/**
* Typed failures of the engine. The app maps the kinds to its messages; no
* message here echoes untrusted input.
*/
export type ArchiveReadErrorKind =
| 'invalid_pdf'
| 'no_attachment'
| 'attachment_stripped'
| 'incompatible_version'
export class ArchiveReadError extends Error {
constructor(
public readonly kind: ArchiveReadErrorKind,
/** The report version found, for `incompatible_version`. */
public readonly foundVersion?: number | null,
) {
super(kind)
this.name = 'ArchiveReadError'
}
}
export type ArchiveExportErrorKind =
| 'snapshot_invalid'
| 'snapshot_too_large'
| 'sheet_mismatch'
| 'too_many_pages'
/** The branding or the label set fails its schema (URLs included). */
| 'branding_invalid'
export class ArchiveExportError extends Error {
constructor(
public readonly kind: ArchiveExportErrorKind,
detail = '',
) {
super(detail ? `${kind}: ${detail}` : kind)
this.name = 'ArchiveExportError'
}
}
export type SvgRejection =
| 'doctype'
| 'malformed'
| 'not_svg'
| 'too_many_elements'
| 'too_deep'
| 'too_many_uses'
/** An SVG text the sanitiser refuses outright (STEP6 §15). */
export class SvgRejectedError extends Error {
constructor(public readonly reason: SvgRejection) {
super(`svg rejected: ${reason}`)
this.name = 'SvgRejectedError'
}
}