diff --git a/apps/app/scripts/compare-reports.ts b/apps/app/scripts/compare-reports.ts index 8bd35495..d93ca7e4 100644 --- a/apps/app/scripts/compare-reports.ts +++ b/apps/app/scripts/compare-reports.ts @@ -18,7 +18,6 @@ import { readFileSync, readdirSync, statSync } from 'node:fs' import { basename, join, resolve } from 'node:path' import { fileURLToPath } from 'node:url' -import { PDFDocument } from 'pdf-lib' import { readArchiveDocument, UnsupportedReportVersionError, @@ -87,8 +86,8 @@ async function compareFile( if (!st.isFile()) throw new Error('not a regular file') if (st.size > MAX_PDF_BYTES) throw new Error('file too large') const bytes = new Uint8Array(readFileSync(path)) - const pdf = await PDFDocument.load(bytes, { ignoreEncryption: true }) - const raw = await readAttachedDocument(pdf) + // Byte-scan reader (BLK-009): the PDF object graph is never parsed here. + const raw = await readAttachedDocument(bytes) try { const outcome = readArchiveDocument(raw, sheet.sheet.migrationMap) return compareReport(raw, outcome, sheet, args) diff --git a/apps/app/tests/unit/pdf.import.test.ts b/apps/app/tests/unit/pdf.import.test.ts index cabaec94..fa8c0b37 100644 --- a/apps/app/tests/unit/pdf.import.test.ts +++ b/apps/app/tests/unit/pdf.import.test.ts @@ -16,11 +16,12 @@ beforeAll(async () => { sheet = await loadTestSheet() }) +/** Saved like the export (no object streams, BLK-009) so the keyword stamp is in clear. */ async function pdfWithoutAttachment(keywords?: string[]): Promise { const doc = await PDFDocument.create() doc.addPage() if (keywords) doc.setKeywords(keywords) - return doc.save() + return doc.save({ useObjectStreams: false }) } async function rejectsWith(bytes: Uint8Array, kind: string): Promise { diff --git a/apps/app/tests/unit/scripts.compare.test.ts b/apps/app/tests/unit/scripts.compare.test.ts index dc1f8b21..8026a514 100644 --- a/apps/app/tests/unit/scripts.compare.test.ts +++ b/apps/app/tests/unit/scripts.compare.test.ts @@ -4,7 +4,6 @@ * and on a Step 6 envelope (archive header line). */ import { describe, it, expect, beforeAll } from 'vitest' -import { PDFDocument } from 'pdf-lib' import { readFileSync } from 'node:fs' import { fileURLToPath } from 'node:url' import { dirname, resolve } from 'node:path' @@ -48,8 +47,7 @@ beforeAll(async () => { describe('compareReport on the frozen v2 archive PDF', () => { it('shows every legacy zone and finding next to its canonical target', async () => { const bytes = readFileSync(resolve(HERE, '../fixtures/report-v2.full.pdf')) - const pdf = await PDFDocument.load(new Uint8Array(bytes)) - const raw = await readAttachedDocument(pdf) + const raw = await readAttachedDocument(new Uint8Array(bytes)) const c = compareRaw(raw) const lines = renderComparison('full.pdf', c) expect(c.source).toBe('v2') diff --git a/docs/PDF-FORMAT.md b/docs/PDF-FORMAT.md index 76dc66ca..fd400536 100644 --- a/docs/PDF-FORMAT.md +++ b/docs/PDF-FORMAT.md @@ -35,6 +35,10 @@ branding have changed since the export. Design record: - Document information: title, author, subject, dates = `exportedAt`, and `Keywords: faunosteo-archive/1` so a PDF whose attachment was stripped by a viewer is still recognised. +- Saved without object streams (`useObjectStreams: false`) since BLK-009: the + Filespec, its file name and the Info dictionary stay in clear, where the + byte-scan reader below can see them. Files produced earlier pack them into + an `/ObjStm`; their attachment stream is still in clear and still imports. ## Attachment: the report archive envelope (v1) @@ -53,18 +57,38 @@ schema): "report": { "version": 3, "...": "Report v3 exactly as exported" }, "sheetSnapshot": { "status": "draft", - "ref": { "sheetId": "…", "versionId": null, "documentSha256": "…", "schemaVersion": 2, "speciesId": "equine" }, - "document": { "schema_version": 2, "...": "lean, valid anatomical sheet document" }, - "assets": [ { "svgAssetId": "…", "sha256": "…", "svg": "" } ] + "ref": { + "sheetId": "…", + "versionId": null, + "documentSha256": "…", + "schemaVersion": 2, + "speciesId": "equine" + }, + "document": { + "schema_version": 2, + "...": "lean, valid anatomical sheet document" + }, + "assets": [{ "svgAssetId": "…", "sha256": "…", "svg": "" }] }, "branding": { "siteUrl": "https://…", "headerLogo": { "kind": "png", "base64": "…" }, "footerLogo": { "kind": "png", "base64": "…" }, - "poweredBy": { "logo": { "kind": "png", "base64": "…" }, "url": "https://…", "fallbackText": "…" }, - "partner": { "logo": { "kind": "svg", "svg": "" }, "url": "https://…", "fallbackText": "…" } + "poweredBy": { + "logo": { "kind": "png", "base64": "…" }, + "url": "https://…", + "fallbackText": "…" + }, + "partner": { + "logo": { "kind": "svg", "svg": "" }, + "url": "https://…", + "fallbackText": "…" + } }, - "labels": { "fr": { "pdf.section.zones": "ZONES", "...": "25 keys" }, "en": { "...": "" } } + "labels": { + "fr": { "pdf.section.zones": "ZONES", "...": "25 keys" }, + "en": { "...": "" } + } } ``` @@ -74,14 +98,14 @@ The snapshot is the sheet the bilan was built on, reduced to what the bilan needs, as a **valid** anatomical sheet document (schema v2, same integrity rules as the catalog): -| Part | Content | -|---|---| -| `ref` | sheet id, version id, sha256 of the full catalog document (provenance pointer), schema version, species | -| `document.views`, `annotations`, `categories`, `vertebral_config` | all of them: geometry, colours, names, anatomical relations (`finding_zone_id`, `vertebral_region_code`, `opens_view_id`, laterality) | -| `document.finding_zones` | every zone with its name; for the zones the report's findings use (resolved or `unresolved`), the categories pruned to the referenced paths with the name and vet flag of every node on the path; other zones keep `categories: []` | -| `document.references` | emptied (bibliography) | -| `document.name`, `description`, `scope_disclaimer`, `attribution` | kept | -| `assets` | the SVG text of every asset a view reaches, with the catalog's content hash when known | +| Part | Content | +| ----------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `ref` | sheet id, version id, sha256 of the full catalog document (provenance pointer), schema version, species | +| `document.views`, `annotations`, `categories`, `vertebral_config` | all of them: geometry, colours, names, anatomical relations (`finding_zone_id`, `vertebral_region_code`, `opens_view_id`, laterality) | +| `document.finding_zones` | every zone with its name; for the zones the report's findings use (resolved or `unresolved`), the categories pruned to the referenced paths with the name and vet flag of every node on the path; other zones keep `categories: []` | +| `document.references` | emptied (bibliography) | +| `document.name`, `description`, `scope_disclaimer`, `attribution` | kept | +| `assets` | the SVG text of every asset a view reaches, with the catalog's content hash when known | `branding` is the exact set of images, URLs and credit texts used at export; the application builds it from logos bundled at build time @@ -98,10 +122,10 @@ Invariant: `report.sheet.documentSha256 === sheetSnapshot.ref.documentSha256` Measured on the demo report (6 observations, both views, equine draft): -| Branding | Stored in the PDF (Flate) | Raw JSON | -|---|---|---| -| no logos | 55 KB | 189 KB | -| full branding (3 PNG logos + partner SVG) | 136 KB | 306 KB | +| Branding | Stored in the PDF (Flate) | Raw JSON | +| ----------------------------------------- | ------------------------- | -------- | +| no logos | 55 KB | 189 KB | +| full branding (3 PNG logos + partner SVG) | 136 KB | 306 KB | The drawing is the largest part (47 KB stored); the archived PNG logos add about 80 KB stored. The 250 KB target applies to the stored size. @@ -111,11 +135,22 @@ about 80 KB stored. The 250 KB target applies to the stored size. `readReportArchive(pdfBytes, migrationMap)` reads in layers so the report is never lost because of its wrapper: -1. The PDF must load (`invalid_pdf`, also above 50 MiB). -2. The attachment named `osteo-report*` is taken first, else the first - embedded file; Flate streams are inflated under a 16 MiB cap. No - attachment: `attachment_stripped` when the PDF carries the Faunosteo - keyword, else `no_attachment`. +1. The file must weigh at most 50 MiB and carry `%PDF-` within its first + KiB (`invalid_pdf`). Nothing else of the PDF syntax is required. +2. The attachment is located by a byte scan, never by parsing the PDF object + graph (BLK-009): every `/Type /EmbeddedFile` stream dictionary in clear + (a stream object can never sit inside an object stream) is a candidate, + with its direct `/Length` or, failing that, the distance to `endstream`; + a `/Filespec` dictionary in clear whose `/EF /F` points at the stream + gives it a name. Candidates named `osteo-report*` come first, then file + order; Flate data is inflated by the platform inflater under a 16 MiB cap + (cancelled beyond it), and the first candidate whose text is a JSON + object with a numeric `envelopeVersion` or `version` wins. Object + streams, cross-reference streams and page contents are never inflated. + No candidate: `attachment_stripped` when a `/Keywords` string in clear + contains `faunosteo-archive`, else `no_attachment`. A stamped file saved + with object streams (before BLK-009) hides its stamp and reads as + `no_attachment`. 3. The report is located (`data.report` inside an envelope, `data` for a bare attachment) and read with the report reader: v3 as is, v2 migrated through the sheet's `report-migration-map.json` (items the map cannot @@ -128,12 +163,12 @@ never lost because of its wrapper: closed reason (`schema`, `caps`, `integrity`, `sheet_mismatch`) and keeps the report. -| Archive | Produced by | Read as | -|---|---|---| -| bare Report v2 | application before Step 5 | `origin 'v2'`, migrated, unresolved counts | -| bare Report v3 | Step 5 | `origin 'bare_v3'`, no archive | -| envelope v1 | Step 6 | `origin 'envelope_v1'`, archive kept or `snapshotRejected` | -| envelope v≥2 | future | report kept when its version is known, snapshot dropped | +| Archive | Produced by | Read as | +| -------------- | ------------------------- | ---------------------------------------------------------- | +| bare Report v2 | application before Step 5 | `origin 'v2'`, migrated, unresolved counts | +| bare Report v3 | Step 5 | `origin 'bare_v3'`, no archive | +| envelope v1 | Step 6 | `origin 'envelope_v1'`, archive kept or `snapshotRejected` | +| envelope v≥2 | future | report kept when its version is known, snapshot dropped | Readers never drop a schema version: sheet document schema v2 stays readable as long as envelope v1 archives exist, as Report v2 does today. A rollback to @@ -156,17 +191,17 @@ Two operations are distinct: ## Limits and sanitising -| Item | Cap | -|---|---| -| PDF file | 50 MiB | -| attachment, raw and inflated | 16 MiB | -| snapshot assets | ≤ 4, ≤ 2 MiB each, ≤ 6 MiB total | -| snapshot document | ≤ 64 views, ≤ 1,024 annotations, ≤ 8,192 finding nodes, depth ≤ 8, names ≤ 512 chars | -| branding | PNG base64 ≤ 512 KiB, SVG ≤ 256 KiB, http(s) URLs ≤ 2,048 chars | -| branding URL | `http(s)://` then printable ASCII only, without `"` `'` `(` `)` `<` `>` `\`, no space or control character | -| branding PNG | valid signature, one IHDR (13 bytes) first with a colour type and bit depth the spec allows, compression and filter method 0, no interlace; each side 1 to 8,192 px, ≤ 16,777,216 px (4096²), expected raw scanline size ≤ 16 MiB; then only `PLTE` `tRNS` `IDAT` `gAMA` `cHRM` `sRGB` `sBIT` `pHYs` `bKGD`, no chunk past the buffer, an empty `IEND` as the last bytes; the IDAT stream inflated natively and yielding exactly the raw scanline size | -| report at export | ≤ 500 observations, ≤ 8 views per observation | -| SVG sanitiser | ≤ 20,000 elements, nesting ≤ 64, ≤ 256 ``, blur `stdDeviation` ≤ 100 | +| Item | Cap | +| ---------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| PDF file | 50 MiB | +| attachment, raw and inflated | 16 MiB | +| snapshot assets | ≤ 4, ≤ 2 MiB each, ≤ 6 MiB total | +| snapshot document | ≤ 64 views, ≤ 1,024 annotations, ≤ 8,192 finding nodes, depth ≤ 8, names ≤ 512 chars | +| branding | PNG base64 ≤ 512 KiB, SVG ≤ 256 KiB, http(s) URLs ≤ 2,048 chars | +| branding URL | `http(s)://` then printable ASCII only, without `"` `'` `(` `)` `<` `>` `\`, no space or control character | +| branding PNG | valid signature, one IHDR (13 bytes) first with a colour type and bit depth the spec allows, compression and filter method 0, no interlace; each side 1 to 8,192 px, ≤ 16,777,216 px (4096²), expected raw scanline size ≤ 16 MiB; then only `PLTE` `tRNS` `IDAT` `gAMA` `cHRM` `sRGB` `sBIT` `pHYs` `bKGD`, no chunk past the buffer, an empty `IEND` as the last bytes; the IDAT stream inflated natively and yielding exactly the raw scanline size | +| report at export | ≤ 500 observations, ≤ 8 views per observation | +| SVG sanitiser | ≤ 20,000 elements, nesting ≤ 64, ≤ 256 ``, blur `stdDeviation` ≤ 100 | Every SVG that reaches the engine (sheet asset or branding) is untrusted: a text containing `` expansion is bounded by the rasterizer, not by the sanitiser, which only counts the `` elements (≤ 256). resvg (tests) refuses an oversized tree; the production path is the browser rasterizer diff --git a/docs/development/PDF_IMPORT_HARDENING.md b/docs/development/PDF_IMPORT_HARDENING.md index 37403042..6cca3f3e 100644 --- a/docs/development/PDF_IMPORT_HARDENING.md +++ b/docs/development/PDF_IMPORT_HARDENING.md @@ -1,8 +1,9 @@ # PDF import hardening — BLK-009 analysis -Status 2026-10-11: analysis requested by the user after Step 6. Nothing here is -implemented; §4 says what can land now and §5 what must land before the -import is opened to external users. +Status 2026-10-11: analysis requested by the user after Step 6. §4 landed the +same day on `feature/step-06-pdf-archives` (options A + B, see §4 for the +commit); §5 lists what must land before the import is opened to external +users. ## 1. The problem @@ -36,13 +37,13 @@ PDF object graph at all. ## 3. Options -| Option | Protection | Cost | Verdict | -|---|---|---|---| -| A. Byte-scan reader: find `/Type /EmbeddedFile` streams, read `/Length` (direct integer, as pdf-lib writes it; fall back to `endstream` search), inflate with the existing capped `inflateFlate`, parse JSON; no `PDFDocument.load` on import | removes pdf-lib from the import path entirely: the only inflation is our capped one | one module (~120 lines) + tests on every fixture (v2, bare v3, envelope, object streams on/off, several streams, foreign PDF) | **recommended, can land now** | -| B. Export with `useObjectStreams: false` | the Filespec name, the Info keywords and the Filespec→EF link become visible to a byte scan, so the filename-first rule and the `attachment_stripped` detection keep working without the object graph; files a few KB larger | one save option; older PDFs keep their object streams, which A tolerates | **recommended with A** | -| C. Load in a Web Worker with a timeout and a memory budget | contains the damage (the tab survives, the worker dies) | worker bundling, message protocol, no memory cap API in browsers (only termination on timeout) | defence in depth, later | -| D. Pre-scan declared `/Length` of `/ObjStm` streams and refuse above a sum | partial: declared lengths are the compressed sizes, the expansion is unknown | small | insufficient alone | -| E. Replace pdf-lib's parser (fork `@cantoo/pdf-lib`, or `pdf.js` for reading) | maintained parser; pdf.js has its own limits | dependency change, larger bundle | evaluate for Step 14 (server-side import) | +| Option | Protection | Cost | Verdict | +| --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------- | +| A. Byte-scan reader: find `/Type /EmbeddedFile` streams, read `/Length` (direct integer, as pdf-lib writes it; fall back to `endstream` search), inflate with the existing capped `inflateFlate`, parse JSON; no `PDFDocument.load` on import | removes pdf-lib from the import path entirely: the only inflation is our capped one | one module (~120 lines) + tests on every fixture (v2, bare v3, envelope, object streams on/off, several streams, foreign PDF) | **recommended, can land now** | +| B. Export with `useObjectStreams: false` | the Filespec name, the Info keywords and the Filespec→EF link become visible to a byte scan, so the filename-first rule and the `attachment_stripped` detection keep working without the object graph; files a few KB larger | one save option; older PDFs keep their object streams, which A tolerates | **recommended with A** | +| C. Load in a Web Worker with a timeout and a memory budget | contains the damage (the tab survives, the worker dies) | worker bundling, message protocol, no memory cap API in browsers (only termination on timeout) | defence in depth, later | +| D. Pre-scan declared `/Length` of `/ObjStm` streams and refuse above a sum | partial: declared lengths are the compressed sizes, the expansion is unknown | small | insufficient alone | +| E. Replace pdf-lib's parser (fork `@cantoo/pdf-lib`, or `pdf.js` for reading) | maintained parser; pdf.js has its own limits | dependency change, larger bundle | evaluate for Step 14 (server-side import) | With A, the import path reads bytes, finds at most a few streams, inflates one under 16 MiB, parses JSON under the existing schema caps. pdf-lib stays on @@ -59,22 +60,29 @@ encrypted file is encrypted too and would fail to inflate → `no_attachment`, same outcome as today in practice); the 50 MiB file cap stays as the scan bound. -## 4. What can be done now (before merge of Step 6, or as the next chore) +## 4. Done 2026-10-11 (feature/step-06-pdf-archives, commit: see `git log -- packages/pdf/src/attachment/scan.ts`) -1. Option A in `packages/pdf/src/attachment/scan.ts`, used by - `readReportArchive`; `readAttachedDocument(pdfDoc)` kept for the CLI until - it moves to the scanner too. +1. Option A in `packages/pdf/src/attachment/scan.ts` + (`findEmbeddedFiles`, `readAttachmentBytes`, `hasArchiveStamp`), used by + `readReportArchive`; `readAttachedDocument(bytes)` takes the file bytes + and the `report:compare` CLI runs on the scanner too. No + `PDFDocument.load` remains under `packages/pdf/src` or `apps/app/scripts`. 2. Option B on export (`pdfDoc.save({ useObjectStreams: false })`): no visible - change; goldens unaffected (they record drawn text, images, links); the - envelope roundtrip tests prove the reader still finds the attachment in - both layouts. -3. Tests: every existing import fixture through the scanner; a crafted - object-stream bomb (the audit's recipe: a few hundred KB inflating to - hundreds of MB) imports in milliseconds without inflating it; a PDF with - two embedded files; a PDF whose `/Length` is indirect. + change; goldens unchanged (they record drawn text, images, links); the + envelope roundtrip tests prove the reader finds the attachment in both + layouts. +3. Tests (`packages/pdf/tests/scan.test.ts`): the three frozen fixtures (object + streams) and a fresh export; two embedded files with a decoy first; + indirect and wrong `/Length`; `\r\n` after `stream`; `/Type/EmbeddedFile` + without a space; hex and literal names and keywords; garbage inputs and + the 32-candidate cap; a 64 MiB object-stream bomb next to the real + attachment (report in milliseconds, RSS flat) and an `EmbeddedFile` bomb + (inflation cancelled at 16 MiB, `no_attachment`). 4. Docs: `docs/PDF-FORMAT.md` "Reading an archive" and "Known limits". -Estimated size: one executor phase, about 300 lines including tests. +Known trade-off: a stamped PDF saved before this change (object streams) +hides its `/Keywords`, so a stripped attachment on such a file reads as +`no_attachment` instead of `attachment_stripped`. ## 5. Prerequisites before external users (Step 7+ sync, Step 14 worker) diff --git a/packages/pdf/src/attachment/read.ts b/packages/pdf/src/attachment/read.ts index ff8c1a56..c4c17c0e 100644 --- a/packages/pdf/src/attachment/read.ts +++ b/packages/pdf/src/attachment/read.ts @@ -1,105 +1,36 @@ -import { - PDFDict, - PDFHexString, - PDFName, - PDFRawStream, - PDFStream, - PDFString, - type PDFDocument, -} from 'pdf-lib' import { ArchiveReadError } from '../errors' -import { inflateFlate, MAX_ATTACHMENT_BYTES } from './inflate' +import { + ARCHIVE_STAMP, + hasPdfHeader, + locateAttachment, + MAX_PDF_BYTES, +} from './scan' /** - * Finds the Faunosteo attachment of a loaded PDF: the Filespec whose file - * name starts with `osteo-report` first, else the first embedded file - * (today's behaviour, kept as the fallback). pdf-lib 1.17 exposes no - * attachment API, so this walks the indirect objects. + * The raw JSON a Faunosteo PDF carries, read from the bytes alone (BLK-009): + * no PDF object graph is parsed, so no object stream is ever inflated. pdf-lib + * stays on the export side. */ -export const ATTACHMENT_NAME_PREFIX = 'osteo-report' -export const PDF_KEYWORD = 'faunosteo-archive/1' +export { ATTACHMENT_NAME_PREFIX } from './scan' +export const PDF_KEYWORD = `${ARCHIVE_STAMP}/1` -const TYPE = PDFName.of('Type') -const F = PDFName.of('F') - -function textOf(value: unknown): string | null { - if (value instanceof PDFString || value instanceof PDFHexString) { - return value.decodeText() +/** Size cap first, then the `%PDF-` header: anything else is `invalid_pdf`. */ +function assertPdfBytes(bytes: Uint8Array): void { + if (bytes.byteLength > MAX_PDF_BYTES || !hasPdfHeader(bytes)) { + throw new ArchiveReadError('invalid_pdf') } - return null -} - -/** The embedded stream of a `/Filespec` named for a Faunosteo report. */ -function namedFilespecStream(dict: PDFDict): PDFStream | null { - if (dict.get(TYPE) !== PDFName.of('Filespec')) return null - const name = textOf(dict.lookup(F)) - if (name === null || !name.startsWith(ATTACHMENT_NAME_PREFIX)) return null - const files = dict.lookup(PDFName.of('EF')) - if (!(files instanceof PDFDict)) return null - const stream = files.lookup(F) - return stream instanceof PDFStream ? stream : null -} - -/** Today's heuristic: an `/EmbeddedFile` stream or a JSON-subtyped one. */ -function looksLikeEmbeddedJson(stream: PDFStream): boolean { - const type = String(stream.dict.lookup(TYPE) ?? '') - const subtype = String(stream.dict.lookup(PDFName.of('Subtype')) ?? '') - return ( - type.includes('EmbeddedFile') || - subtype.includes('application/json') || - subtype.includes('application#2Fjson') - ) -} - -function locateAttachment(pdfDoc: PDFDocument): PDFStream | null { - const objects = pdfDoc.context.enumerateIndirectObjects() - for (const [, object] of objects) { - if (!(object instanceof PDFDict)) continue - const stream = namedFilespecStream(object) - if (stream) return stream - } - for (const [, object] of objects) { - if (object instanceof PDFStream && looksLikeEmbeddedJson(object)) { - return object - } - } - return null -} - -/** Stream bytes, inflated when Flate-encoded; null when unreadable or over the cap. */ -async function attachmentBytes(stream: PDFStream): Promise { - const contents = - stream instanceof PDFRawStream ? stream.contents : stream.getContents() - if (contents.byteLength > MAX_ATTACHMENT_BYTES) return null - const filter = String(stream.dict.lookup(PDFName.of('Filter')) ?? '') - if (!filter.includes('FlateDecode')) return contents - return inflateFlate(contents) -} - -/** A PDF stamped as an archive but holding no attachment lost it in a viewer. */ -function missingAttachmentError(pdfDoc: PDFDocument): ArchiveReadError { - const stamped = (pdfDoc.getKeywords() ?? '').includes('faunosteo-archive') - return new ArchiveReadError(stamped ? 'attachment_stripped' : 'no_attachment') } /** * The raw JSON carried by the PDF, whatever its shape (bare report of any * version or archive envelope), before any validation: what the archive - * check tool shows next to the read result. Throws ArchiveReadError. + * check tool shows next to the read result. Throws ArchiveReadError + * (`invalid_pdf`, `no_attachment`, `attachment_stripped`). */ export async function readAttachedDocument( - pdfDoc: PDFDocument, + bytes: Uint8Array, ): Promise { - const stream = locateAttachment(pdfDoc) - if (!stream) throw missingAttachmentError(pdfDoc) - const bytes = await attachmentBytes(stream) - if (!bytes || bytes.byteLength > MAX_ATTACHMENT_BYTES) { - throw new ArchiveReadError('no_attachment') - } - try { - return JSON.parse(new TextDecoder().decode(bytes)) as unknown - } catch { - throw new ArchiveReadError('no_attachment') - } + assertPdfBytes(bytes) + return (await locateAttachment(bytes)).data } diff --git a/packages/pdf/src/attachment/scan.ts b/packages/pdf/src/attachment/scan.ts new file mode 100644 index 00000000..724d4b2d --- /dev/null +++ b/packages/pdf/src/attachment/scan.ts @@ -0,0 +1,441 @@ +import { ArchiveReadError } from '../errors' +import { inflateFlate, MAX_ATTACHMENT_BYTES } from './inflate' + +/** + * Locates the Faunosteo attachment by scanning the PDF bytes (BLK-009): a + * stream object can never live inside an object stream, so every + * `/Type /EmbeddedFile` dictionary and its data are in clear in the file. + * Nothing here parses the object graph, the xref or an `/ObjStm`, so the + * only inflation on the import path is the capped one of `inflateFlate`. + * The scanner never throws on garbage: it returns fewer candidates. + */ + +/** A Faunosteo PDF weighs a few hundred KB; anything larger is not one. */ +export const MAX_PDF_BYTES = 50 * 1024 * 1024 + +export const ATTACHMENT_NAME_PREFIX = 'osteo-report' +/** The keyword stamp every archive carries (`faunosteo-archive/`). */ +export const ARCHIVE_STAMP = 'faunosteo-archive' + +/** Candidates beyond this count are ignored: a crafted file cannot make the reader loop. */ +const MAX_CANDIDATES = 32 +/** A stream or Filespec dictionary is small; a longer one is skipped. */ +const MAX_DICT_BYTES = 4096 +/** Window read before a dictionary for its `n 0 obj` header. */ +const HEADER_WINDOW = 48 + +export interface EmbeddedFileCandidate { + /** First byte of the stream data. */ + offset: number + /** Byte count of the stream data (declared, or measured to `endstream`). */ + length: number + filter: 'FlateDecode' | null + /** From a visible `/Filespec` whose `/EF /F` points at this stream. */ + name: string | null +} + +const encode = (text: string): Uint8Array => + Uint8Array.from(text, (c) => c.charCodeAt(0)) + +const TYPE = encode('/Type') +const EMBEDDED_FILE = encode('/EmbeddedFile') +const FILESPEC = encode('/Filespec') +const KEYWORDS = encode('/Keywords') +const STREAM = encode('stream') +const ENDSTREAM = encode('endstream') +const PDF_HEADER = encode('%PDF-') + +// --------------------------------------------------------------------------- +// Byte helpers: every loop is bounded by `end` (at most MAX_PDF_BYTES) + +function isWhitespace(byte: number): boolean { + return ( + byte === 0x20 || + byte === 0x0a || + byte === 0x0d || + byte === 0x09 || + byte === 0x0c || + byte === 0x00 + ) +} + +/** PDF delimiters and whitespace end a name token. */ +function isDelimiter(byte: number): boolean { + return isWhitespace(byte) || '()<>[]{}/%'.includes(String.fromCharCode(byte)) +} + +/** Naive forward search; the needle's first byte is the fast reject. */ +function indexOf( + bytes: Uint8Array, + needle: Uint8Array, + from: number, + end: number, +): number { + const last = end - needle.length + const first = needle[0]! + for (let i = Math.max(from, 0); i <= last; i += 1) { + if (bytes[i] !== first) continue + let j = 1 + while (j < needle.length && bytes[i + j] === needle[j]) j += 1 + if (j === needle.length) return i + } + return -1 +} + +function skipWhitespace(bytes: Uint8Array, from: number, end: number): number { + let i = from + while (i < end && isWhitespace(bytes[i]!)) i += 1 + return i +} + +/** Latin-1 view of a small window (never the whole file). */ +function latin1(bytes: Uint8Array, start: number, end: number): string { + let out = '' + for (let i = start; i < end; i += 1) out += String.fromCharCode(bytes[i]!) + return out +} + +/** UTF-16BE with BOM (how pdf-lib writes text strings) or Latin-1 bytes. */ +function decodeTextBytes(raw: number[]): string { + if (raw.length >= 2 && raw[0] === 0xfe && raw[1] === 0xff) { + let out = '' + for (let i = 2; i + 1 < raw.length; i += 2) { + out += String.fromCharCode((raw[i]! << 8) | raw[i + 1]!) + } + return out + } + return String.fromCharCode(...raw) +} + +/** A `(literal)` or `` string token as text; null for anything else. */ +export function decodePdfString(token: string): string | null { + if (token.startsWith('<') && token.endsWith('>')) { + const hex = token.slice(1, -1).replace(/[^0-9a-fA-F]/g, '') + const raw: number[] = [] + for (let i = 0; i + 1 < hex.length; i += 2) { + raw.push(parseInt(hex.slice(i, i + 2), 16)) + } + return decodeTextBytes(raw) + } + if (token.startsWith('(') && token.endsWith(')')) { + const body = token.slice(1, -1).replace(/\\([()\\])/g, '$1') + return decodeTextBytes(Array.from(body, (c) => c.charCodeAt(0) & 0xff)) + } + return null +} + +// --------------------------------------------------------------------------- +// Dictionaries: `/Type /Name` matches and their `<< … >>` bounds + +/** Every `/Type /` (space optional) whose name ends on a delimiter. */ +function findTypedDicts( + bytes: Uint8Array, + name: Uint8Array, + end: number, +): number[] { + const hits: number[] = [] + let from = 0 + while (hits.length < MAX_CANDIDATES) { + const at = indexOf(bytes, name, from, end) + if (at < 0) break + from = at + name.length + const after = at + name.length + if (after < end && !isDelimiter(bytes[after]!)) continue + let before = at + while (before > 0 && isWhitespace(bytes[before - 1]!)) before -= 1 + const typeAt = before - TYPE.length + if (typeAt >= 0 && indexOf(bytes, TYPE, typeAt, before) === typeAt) { + hits.push(typeAt) + } + } + return hits +} + +/** Index of the `<<` opening the dictionary holding `pos` (nested dicts skipped); -1 when none. */ +function dictStart(bytes: Uint8Array, pos: number): number { + const floor = Math.max(0, pos - MAX_DICT_BYTES) + let depth = 0 + let i = pos + while (i - 2 >= floor) { + const pair = (bytes[i - 2]! << 8) | bytes[i - 1]! + if (pair === 0x3e3e) + depth += 1 // >> + else if (pair === 0x3c3c) { + if (depth === 0) return i - 2 + depth -= 1 + } + i -= pair === 0x3e3e || pair === 0x3c3c ? 2 : 1 + } + return -1 +} + +/** Index just past the `>>` closing the dictionary opened at `start`; -1 when none. */ +function dictEnd(bytes: Uint8Array, start: number, end: number): number { + const limit = Math.min(end, start + MAX_DICT_BYTES) + let depth = 0 + let i = start + while (i + 1 < limit) { + const pair = (bytes[i]! << 8) | bytes[i + 1]! + if (pair === 0x3c3c) depth += 1 + else if (pair === 0x3e3e) { + depth -= 1 + if (depth === 0) return i + 2 + } + i += pair === 0x3c3c || pair === 0x3e3e ? 2 : 1 + } + return -1 +} + +interface DictSpan { + start: number + end: number + text: string + objectNumber: number | null +} + +/** The dictionary around a `/Type` match, its text and the `n 0 obj` header before it. */ +function dictAround( + bytes: Uint8Array, + pos: number, + end: number, +): DictSpan | null { + const start = dictStart(bytes, pos) + if (start < 0) return null + const close = dictEnd(bytes, start, end) + if (close < 0) return null + const header = latin1(bytes, Math.max(0, start - HEADER_WINDOW), start) + const match = /(\d{1,10})\s+\d{1,10}\s+obj\s*$/.exec(header) + return { + start, + end: close, + text: latin1(bytes, start, close), + objectNumber: match ? Number(match[1]) : null, + } +} + +// --------------------------------------------------------------------------- +// Stream data bounds + +/** Data offset after `stream` + `\r\n` or `\n`; -1 when the keyword is missing. */ +function streamDataOffset( + bytes: Uint8Array, + dictClose: number, + end: number, +): number { + const at = skipWhitespace(bytes, dictClose, end) + if (indexOf(bytes, STREAM, at, Math.min(end, at + STREAM.length)) !== at) { + return -1 + } + let i = at + STREAM.length + if (bytes[i] === 0x0d) i += 1 + if (bytes[i] !== 0x0a) return -1 + return i + 1 +} + +/** A direct `/Length N` (never an `N 0 R` reference), when it lands on `endstream`. */ +function declaredLength( + bytes: Uint8Array, + dict: string, + offset: number, + end: number, +): number | null { + const match = /\/Length\s+(\d{1,10})(?!\d)(?!\s+\d+\s+R)/.exec(dict) + if (!match) return null + const length = Number(match[1]) + if (offset + length > end) return null + const tail = skipWhitespace(bytes, offset + length, end) + const found = indexOf( + bytes, + ENDSTREAM, + tail, + Math.min(end, tail + ENDSTREAM.length), + ) + return found === tail ? length : null +} + +/** Fallback for an indirect or wrong `/Length`: up to `endstream`, the EOL before it dropped. */ +function measuredLength( + bytes: Uint8Array, + offset: number, + end: number, +): number | null { + const at = indexOf(bytes, ENDSTREAM, offset, end) + if (at < 0) return null + let stop = at + if (stop > offset && bytes[stop - 1] === 0x0a) stop -= 1 + if (stop > offset && bytes[stop - 1] === 0x0d) stop -= 1 + return stop - offset +} + +function filterOf(dict: string): EmbeddedFileCandidate['filter'] { + return /\/Filter\s*(\[\s*)?\/FlateDecode\b/.test(dict) ? 'FlateDecode' : null +} + +function candidateOf( + bytes: Uint8Array, + dict: DictSpan, + end: number, +): Omit | null { + const offset = streamDataOffset(bytes, dict.end, end) + if (offset < 0) return null + const length = + declaredLength(bytes, dict.text, offset, end) ?? + measuredLength(bytes, offset, end) + if (length === null) return null + return { offset, length, filter: filterOf(dict.text) } +} + +// --------------------------------------------------------------------------- +// Filespec names and the keyword stamp + +const STRING_TOKEN = String.raw`\((?:\\.|[^\\)])*\)|<[0-9a-fA-F\s]*>` + +/** `/F` (else `/UF`) of a Filespec dictionary, the `/EF` sub-dictionary set aside. */ +function filespecName(dict: string): string | null { + const withoutEf = dict.replace(/\/EF\s*<<[\s\S]*?>>/, '') + const f = new RegExp(String.raw`/F\s*(${STRING_TOKEN})`).exec(withoutEf) + const uf = new RegExp(String.raw`/UF\s*(${STRING_TOKEN})`).exec(withoutEf) + const token = f?.[1] ?? uf?.[1] + return token ? decodePdfString(token) : null +} + +/** Object number of the stream a Filespec's `/EF << /F n 0 R >>` points at. */ +function filespecStreamObject(dict: string): number | null { + const ef = /\/EF\s*<<([\s\S]*?)>>/.exec(dict) + const ref = ef ? /\/F\s+(\d{1,10})\s+\d{1,10}\s+R/.exec(ef[1]!) : null + return ref ? Number(ref[1]) : null +} + +/** Stream object number → file name, from every Filespec dictionary in clear. */ +function filespecNames(bytes: Uint8Array, end: number): Map { + const names = new Map() + for (const pos of findTypedDicts(bytes, FILESPEC, end)) { + const dict = dictAround(bytes, pos, end) + if (!dict) continue + const target = filespecStreamObject(dict.text) + const name = filespecName(dict.text) + if (target !== null && name !== null && !names.has(target)) { + names.set(target, name) + } + } + return names +} + +/** The string token right after a `/Keywords` key, as text. */ +function keywordsAt( + bytes: Uint8Array, + keyAt: number, + end: number, +): string | null { + const start = skipWhitespace(bytes, keyAt + KEYWORDS.length, end) + const limit = Math.min(end, start + MAX_DICT_BYTES) + const window = latin1(bytes, start, limit) + const token = new RegExp(`^(?:${STRING_TOKEN})`).exec(window) + return token ? decodePdfString(token[0]) : null +} + +/** True when a visible `/Keywords` string carries the archive stamp. */ +export function hasArchiveStamp(bytes: Uint8Array): boolean { + const end = Math.min(bytes.byteLength, MAX_PDF_BYTES) + let from = 0 + for (let n = 0; n < MAX_CANDIDATES; n += 1) { + const at = indexOf(bytes, KEYWORDS, from, end) + if (at < 0) return false + from = at + KEYWORDS.length + if (isDelimiter(bytes[from] ?? 0x20)) { + if (keywordsAt(bytes, at, end)?.includes(ARCHIVE_STAMP)) return true + } + } + return false +} + +/** `%PDF-` within the first KiB, as viewers accept it. */ +export function hasPdfHeader(bytes: Uint8Array): boolean { + return indexOf(bytes, PDF_HEADER, 0, Math.min(bytes.byteLength, 1024)) >= 0 +} + +// --------------------------------------------------------------------------- +// Public scan + +/** + * Every `/Type /EmbeddedFile` stream of the file, named when its Filespec + * is in clear (export without object streams); `osteo-report*` names first, + * then file order. At most MAX_CANDIDATES, within MAX_PDF_BYTES. + */ +export function findEmbeddedFiles(bytes: Uint8Array): EmbeddedFileCandidate[] { + const end = Math.min(bytes.byteLength, MAX_PDF_BYTES) + const names = filespecNames(bytes, end) + const found: EmbeddedFileCandidate[] = [] + for (const pos of findTypedDicts(bytes, EMBEDDED_FILE, end)) { + const dict = dictAround(bytes, pos, end) + if (!dict) continue + const candidate = candidateOf(bytes, dict, end) + if (!candidate) continue + const name = + dict.objectNumber === null ? null : (names.get(dict.objectNumber) ?? null) + found.push({ ...candidate, name }) + } + const named = (c: EmbeddedFileCandidate) => + c.name?.startsWith(ATTACHMENT_NAME_PREFIX) ? 0 : 1 + return found.sort((a, b) => named(a) - named(b)) +} + +/** Raw or inflated stream data under the attachment cap; null when unusable. */ +async function candidateBytes( + bytes: Uint8Array, + candidate: EmbeddedFileCandidate, +): Promise { + if (candidate.length > MAX_ATTACHMENT_BYTES) return null + const data = bytes.subarray( + candidate.offset, + candidate.offset + candidate.length, + ) + if (candidate.filter === null) return data + return inflateFlate(data) +} + +/** JSON with a numeric `envelopeVersion` or `version`; undefined otherwise. */ +function parseVersioned(data: Uint8Array): unknown { + let parsed: unknown + try { + parsed = JSON.parse(new TextDecoder().decode(data)) + } catch { + return undefined + } + if (typeof parsed !== 'object' || parsed === null) return undefined + const record = parsed as Record + const versioned = + typeof record['envelopeVersion'] === 'number' || + typeof record['version'] === 'number' + return versioned ? parsed : undefined +} + +export interface LocatedAttachment { + bytes: Uint8Array + data: unknown +} + +/** + * The first candidate whose data is a versioned JSON document, decoded once. + * None: `attachment_stripped` when the stamp is visible, else `no_attachment`. + */ +export async function locateAttachment( + bytes: Uint8Array, +): Promise { + for (const candidate of findEmbeddedFiles(bytes)) { + const data = await candidateBytes(bytes, candidate) + if (!data) continue + const parsed = parseVersioned(data) + if (parsed !== undefined) return { bytes: data, data: parsed } + } + throw new ArchiveReadError( + hasArchiveStamp(bytes) ? 'attachment_stripped' : 'no_attachment', + ) +} + +/** The attachment's decoded bytes (BLK-009 reader). Throws ArchiveReadError. */ +export async function readAttachmentBytes( + bytes: Uint8Array, +): Promise { + return (await locateAttachment(bytes)).bytes +} diff --git a/packages/pdf/src/export.ts b/packages/pdf/src/export.ts index 08a15c67..a74194c0 100644 --- a/packages/pdf/src/export.ts +++ b/packages/pdf/src/export.ts @@ -201,7 +201,10 @@ export async function exportReportArchive( warnings, }) await attachEnvelope(pdfDoc, attachment, input.exportedAt) - return { pdf: await pdfDoc.save(), envelope, warnings } + // No object streams (BLK-009): the Filespec name and the keyword stamp stay + // in clear for the byte-scan reader; the file grows by a few KB. + const pdf = await pdfDoc.save({ useObjectStreams: false }) + return { pdf, envelope, warnings } } export interface ReexportOptions { diff --git a/packages/pdf/src/index.ts b/packages/pdf/src/index.ts index 58018a0c..45b327b4 100644 --- a/packages/pdf/src/index.ts +++ b/packages/pdf/src/index.ts @@ -4,6 +4,11 @@ export { exportReportArchive, exportFromEnvelope } from './export' export { readReportArchive, MAX_PDF_BYTES } from './read' export { readAttachedDocument, PDF_KEYWORD } from './attachment/read' +export { + findEmbeddedFiles, + readAttachmentBytes, + type EmbeddedFileCandidate, +} from './attachment/scan' export { MAX_ATTACHMENT_BYTES } from './attachment/inflate' export { ArchiveReadError, diff --git a/packages/pdf/src/read.ts b/packages/pdf/src/read.ts index bd1ed7fb..56451746 100644 --- a/packages/pdf/src/read.ts +++ b/packages/pdf/src/read.ts @@ -1,4 +1,3 @@ -import { PDFDocument } from 'pdf-lib' import { readArchiveDocument, UnsupportedReportVersionError, @@ -8,23 +7,12 @@ import { import { readAttachedDocument } from './attachment/read' import { ArchiveReadError } from './errors' -/** A Faunosteo PDF weighs a few hundred KB; anything larger is not one. */ -export const MAX_PDF_BYTES = 50 * 1024 * 1024 - -async function loadPdf(pdfBytes: Uint8Array): Promise { - if (pdfBytes.byteLength > MAX_PDF_BYTES) { - throw new ArchiveReadError('invalid_pdf') - } - try { - return await PDFDocument.load(pdfBytes, { ignoreEncryption: true }) - } catch { - throw new ArchiveReadError('invalid_pdf') - } -} +export { MAX_PDF_BYTES } from './attachment/scan' /** - * Reads a Faunosteo PDF back: size cap, PDF parse, attachment (filename - * first), JSON, then the layered reading of the domain (report first, then + * Reads a Faunosteo PDF back: size cap and `%PDF-` header (`invalid_pdf`), + * attachment located by a byte scan (filename first, never a PDF parse: + * BLK-009), JSON, then the layered reading of the domain (report first, then * the envelope accepted whole or dropped with a reason). An unknown report * version is `incompatible_version` with that version; any other content * failure is `no_attachment`, as today. @@ -33,8 +21,7 @@ export async function readReportArchive( pdfBytes: Uint8Array, map: ReportMigrationMap, ): Promise { - const pdfDoc = await loadPdf(pdfBytes) - const data = await readAttachedDocument(pdfDoc) + const data = await readAttachedDocument(pdfBytes) try { return readArchiveDocument(data, map) } catch (error) { diff --git a/packages/pdf/tests/helpers/pdf.ts b/packages/pdf/tests/helpers/pdf.ts index 584a57c4..cfbd3257 100644 --- a/packages/pdf/tests/helpers/pdf.ts +++ b/packages/pdf/tests/helpers/pdf.ts @@ -1,17 +1,29 @@ -import { AFRelationship, PDFDocument, PDFName, PDFRawStream } from 'pdf-lib' +import { AFRelationship, PDFDocument } from 'pdf-lib' +import { findEmbeddedFiles } from '../../src' + +/** + * Crafted PDFs for the read tests, built with pdf-lib's own attach. Saved + * without object streams by default, like the export since BLK-009; pass + * `objectStreams: true` for the pre-BLK-009 layout (Filespec and Info + * packed into an `/ObjStm`, invisible to the byte scan). + */ +export interface CraftOptions { + keywords?: string[] + before?: [string, unknown] + objectStreams?: boolean +} -/** Crafted PDFs for the read tests, built with pdf-lib's own attach. */ export async function pdfWithAttachment( name: string, payload: unknown, - extra: { keywords?: string[]; before?: [string, unknown] } = {}, + extra: CraftOptions = {}, ): Promise { const doc = await PDFDocument.create() doc.addPage() if (extra.keywords) doc.setKeywords(extra.keywords) if (extra.before) await attachJson(doc, extra.before[0], extra.before[1]) await attachJson(doc, name, payload) - return doc.save() + return doc.save({ useObjectStreams: extra.objectStreams ?? false }) } async function attachJson( @@ -27,20 +39,17 @@ async function attachJson( export async function pdfWithoutAttachment( keywords?: string[], + objectStreams = false, ): Promise { const doc = await PDFDocument.create() doc.addPage() if (keywords) doc.setKeywords(keywords) - return doc.save() + return doc.save({ useObjectStreams: objectStreams }) } -/** Stored (Flate) size of the first `/EmbeddedFile` stream of a saved PDF. */ -export async function storedAttachmentSize(bytes: Uint8Array): Promise { - const doc = await PDFDocument.load(bytes) - for (const [, object] of doc.context.enumerateIndirectObjects()) { - if (!(object instanceof PDFRawStream)) continue - const type = object.dict.lookup(PDFName.of('Type')) - if (type === PDFName.of('EmbeddedFile')) return object.contents.length - } - throw new Error('no EmbeddedFile stream') +/** Stored (Flate) size of the first `/EmbeddedFile` stream of a saved PDF, by the byte scan. */ +export function storedAttachmentSize(bytes: Uint8Array): number { + const [first] = findEmbeddedFiles(bytes) + if (!first) throw new Error('no EmbeddedFile stream') + return first.length } diff --git a/packages/pdf/tests/read.test.ts b/packages/pdf/tests/read.test.ts index bfda7ecf..7a41f2e3 100644 --- a/packages/pdf/tests/read.test.ts +++ b/packages/pdf/tests/read.test.ts @@ -149,6 +149,11 @@ describe('attachment presence and selection', () => { await rejects(bytes, { kind: 'attachment_stripped' }) }) + it('a stamped PDF saved with object streams (pre-BLK-009 layout): the stamp is invisible, no_attachment', async () => { + const bytes = await pdfWithoutAttachment(['faunosteo-archive/1'], true) + await rejects(bytes, { kind: 'no_attachment' }) + }) + it('a foreign PDF: no_attachment', async () => { await rejects(await pdfWithoutAttachment(), { kind: 'no_attachment' }) const foreign = await pdfWithAttachment('other.json', { hello: 'world' }) @@ -166,10 +171,13 @@ describe('attachment presence and selection', () => { expect(outcome.report).toEqual(golden.reports.demo) }) - it('refuses an oversized or unparseable PDF as invalid_pdf', async () => { + it('refuses an oversized file or one without a %PDF- header as invalid_pdf', async () => { await rejects(new Uint8Array(MAX_PDF_BYTES + 1), { kind: 'invalid_pdf' }) await rejects(new TextEncoder().encode('not a pdf at all'), { kind: 'invalid_pdf', }) + const late = new Uint8Array(2048) + late.set(new TextEncoder().encode('%PDF-1.7'), 1024) + await rejects(late, { kind: 'invalid_pdf' }) }) }) diff --git a/packages/pdf/tests/scan.test.ts b/packages/pdf/tests/scan.test.ts new file mode 100644 index 00000000..2efdbb16 --- /dev/null +++ b/packages/pdf/tests/scan.test.ts @@ -0,0 +1,425 @@ +/** + * BLK-009: the byte-scan attachment reader. Candidates on a fresh export and + * on the frozen fixtures (object-stream layout), crafted layouts (two files, + * indirect `/Length`, `\r\n`, no space after `/Type`), garbage, the keyword + * stamp, and the object-stream bomb the reader never inflates. + */ +import { deflateSync } from 'node:zlib' +import { beforeAll, describe, expect, it } from 'vitest' +import { buildSheetSnapshot, type Report } from '@faunosteo/domain' +import { + ArchiveReadError, + defaultPdfLabels, + exportReportArchive, + findEmbeddedFiles, + readAttachedDocument, + readAttachmentBytes, + readReportArchive, + type EmbeddedFileCandidate, +} from '../src' +import { inflateFlate } from '../src/attachment/inflate' +import { decodePdfString, hasArchiveStamp } from '../src/attachment/scan' +import { readFixtureBytes } from './helpers/draft' +import { + FIXED_EXPORTED_AT, + nologosBranding, + rasterizerOf, +} from './helpers/goldens' +import { pdfWithAttachment, pdfWithoutAttachment } from './helpers/pdf' +import { loadGoldenReports, type GoldenReports } from './helpers/reports' + +let golden: GoldenReports +let exported: Uint8Array +const map = () => golden.draft.migrationMap + +beforeAll(async () => { + golden = loadGoldenReports() + exported = (await exportDemo()).pdf +}) + +async function exportDemo() { + const report: Report = golden.reports.demo + return exportReportArchive({ + report, + sheet: buildSheetSnapshot(golden.draft, report), + branding: nologosBranding(), + labels: defaultPdfLabels, + locale: 'fr', + rasterizer: rasterizerOf('nologos'), + exportedAt: FIXED_EXPORTED_AT, + }) +} + +// --------------------------------------------------------------------------- +// Hand-built PDFs: the scanner never reads the xref, so none is written + +const latin1 = (bytes: Uint8Array) => Buffer.from(bytes).toString('latin1') +const ascii = (text: string) => new Uint8Array(Buffer.from(text, 'latin1')) + +function concat(parts: Uint8Array[]): Uint8Array { + const out = new Uint8Array(parts.reduce((n, p) => n + p.byteLength, 0)) + let at = 0 + for (const part of parts) { + out.set(part, at) + at += part.byteLength + } + return out +} + +interface StreamSpec { + num: number + dict: string + data: Uint8Array + eol?: string +} + +function streamObject({ num, dict, data, eol = '\n' }: StreamSpec): Uint8Array { + return concat([ + ascii(`${num} 0 obj\n<<${dict}>>\nstream${eol}`), + data, + ascii('\nendstream\nendobj\n'), + ]) +} + +function craftPdf(objects: Uint8Array[]): Uint8Array { + return concat([ + ascii('%PDF-1.7\n'), + ...objects, + ascii('trailer\n<< /Root 1 0 R >>\n%%EOF\n'), + ]) +} + +/** The `n 0 obj … endobj` bytes of the object whose dictionary holds `marker`. */ +function objectHolding(pdf: Uint8Array, marker: string): Uint8Array { + const text = latin1(pdf) + const at = text.indexOf(marker) + expect(at).toBeGreaterThan(0) + const start = text.lastIndexOf(' 0 obj', at) + const headerStart = text.lastIndexOf('\n', start) + 1 + const end = text.indexOf('endobj', at) + 'endobj'.length + return pdf.slice(headerStart, end) +} + +const BOMB_RAW = 64 * 1024 * 1024 +const bomb = () => new Uint8Array(deflateSync(new Uint8Array(BOMB_RAW))) + +async function rejects( + bytes: Uint8Array, + kind: ArchiveReadError['kind'], +): Promise { + const promise = readReportArchive(bytes, map()) + await expect(promise).rejects.toBeInstanceOf(ArchiveReadError) + await expect(promise).rejects.toMatchObject({ kind }) +} + +// --------------------------------------------------------------------------- + +describe('findEmbeddedFiles on a fresh export (no object streams)', () => { + let candidates: EmbeddedFileCandidate[] + beforeAll(() => { + candidates = findEmbeddedFiles(exported) + }) + + it('finds the one Flate stream, named through its visible Filespec', async () => { + expect(candidates).toHaveLength(1) + const [c] = candidates + expect(c!.filter).toBe('FlateDecode') + expect(c!.name).toBe('osteo-report.json') + const inflated = await inflateFlate( + exported.subarray(c!.offset, c!.offset + c!.length), + ) + const json = JSON.parse(new TextDecoder().decode(inflated!)) as { + envelopeVersion: number + } + expect(json.envelopeVersion).toBe(1) + }) + + it('the length is the declared /Length and the data ends on endstream', () => { + const [c] = candidates + const text = latin1(exported) + expect(text).toContain(`/Length ${c!.length}\n>>\nstream\n`) + expect(text.slice(c!.offset + c!.length)).toMatch(/^\r?\nendstream/) + }) + + it('the export keeps the Filespec, its name and the stamp in clear, no object stream', () => { + const text = latin1(exported) + expect(text).toContain('/Type /Filespec') + expect(text).toContain('/F (osteo-report.json)') + expect(text).not.toContain('/Type /ObjStm') + expect(hasArchiveStamp(exported)).toBe(true) + }) + + it('readAttachmentBytes returns the inflated envelope bytes', async () => { + const bytes = await readAttachmentBytes(exported) + const json = JSON.parse(new TextDecoder().decode(bytes)) as { + report: Report + } + expect(json.report).toEqual(golden.reports.demo) + }) +}) + +describe('compatibility: frozen fixtures written with object streams', () => { + it.each([ + ['report-v2.full.pdf', 'v2'], + ['report-v2.minimal.pdf', 'v2'], + ['report-v3.bare.pdf', 'bare_v3'], + ] as const)('%s → origin %s', async (name, origin) => { + const bytes = readFixtureBytes(name) + expect(latin1(bytes)).toContain('/Type /ObjStm') + const candidates = findEmbeddedFiles(bytes) + expect(candidates).toHaveLength(1) + // The Filespec is packed into the object stream: no name, the JSON decides. + expect(candidates[0]!.name).toBeNull() + expect(candidates[0]!.filter).toBe('FlateDecode') + const outcome = await readReportArchive(bytes, map()) + expect(outcome.origin).toBe(origin) + }) + + it('a stamped PDF with object streams hides its stamp: no_attachment (documented)', async () => { + const bytes = await pdfWithoutAttachment(['faunosteo-archive/1'], true) + expect(hasArchiveStamp(bytes)).toBe(false) + await rejects(bytes, 'no_attachment') + }) +}) + +describe('selection between several embedded files', () => { + it('a decoy EmbeddedFile first, the named Faunosteo one second: the name wins', async () => { + const bytes = await pdfWithAttachment( + 'osteo-report.json', + golden.reports.demo, + { + before: ['other.json', { hello: 'world', version: 2 }], + }, + ) + const names = findEmbeddedFiles(bytes).map((c) => c.name) + expect(names).toEqual(['osteo-report.json', 'other.json']) + const outcome = await readReportArchive(bytes, map()) + expect(outcome.origin).toBe('bare_v3') + expect(outcome.report).toEqual(golden.reports.demo) + }) + + it('an unnamed decoy without a version is skipped for the versioned JSON after it', async () => { + const decoy = streamObject({ + num: 2, + dict: '/Type /EmbeddedFile /Length 17', + data: ascii('{"hello":"world"}'), + }) + const real = objectHolding(exported, '/Type /EmbeddedFile') + const outcome = await readReportArchive(craftPdf([decoy, real]), map()) + expect(outcome.origin).toBe('envelope_v1') + expect(outcome.report).toEqual(golden.reports.demo) + }) + + it('a foreign PDF, with or without attachments: no_attachment', async () => { + await rejects(await pdfWithoutAttachment(), 'no_attachment') + await rejects( + await pdfWithAttachment('other.json', { hello: 'world' }), + 'no_attachment', + ) + }) +}) + +describe('crafted stream layouts', () => { + // UTF-8: the demo report holds an en dash that Latin-1 cannot carry. + const demoJson = () => + new TextEncoder().encode(JSON.stringify(golden.reports.demo)) + + it('/Length as an indirect reference: read up to endstream', async () => { + const pdf = craftPdf([ + streamObject({ + num: 2, + dict: '/Type /EmbeddedFile /Length 7 0 R', + data: demoJson(), + }), + ascii('7 0 obj\n123456\nendobj\n'), + ]) + const [c] = findEmbeddedFiles(pdf) + expect(c!.filter).toBeNull() + expect(c!.length).toBe(demoJson().byteLength) + expect((await readReportArchive(pdf, map())).origin).toBe('bare_v3') + }) + + it('a wrong direct /Length that does not land on endstream: measured instead', async () => { + const pdf = craftPdf([ + streamObject({ + num: 2, + dict: '/Type /EmbeddedFile /Length 5', + data: demoJson(), + }), + ]) + expect(findEmbeddedFiles(pdf)[0]!.length).toBe(demoJson().byteLength) + expect((await readReportArchive(pdf, map())).origin).toBe('bare_v3') + }) + + it('CRLF after stream, /Type/EmbeddedFile unspaced, a leading /Params dict', async () => { + const data = demoJson() + const pdf = craftPdf([ + streamObject({ + num: 2, + dict: `/Params << /Size ${data.byteLength} >> /Type/EmbeddedFile/Length ${data.byteLength}`, + data, + eol: '\r\n', + }), + ]) + const [c] = findEmbeddedFiles(pdf) + expect(c!.length).toBe(data.byteLength) + expect(await readAttachmentBytes(pdf)).toEqual(data) + }) + + it('a Filespec with a hex /F name and an /EF listing /UF too names its stream', () => { + const name = Buffer.from('osteo-report.json', 'utf16le') + .swap16() + .toString('hex') + const pdf = craftPdf([ + ascii( + `3 0 obj\n<< /Type /Filespec /F <${name}> /EF << /F 2 0 R /UF 2 0 R >> >>\nendobj\n`, + ), + streamObject({ + num: 2, + dict: '/Type /EmbeddedFile /Length 13', + data: ascii('{"version":3}'), + }), + ]) + expect(findEmbeddedFiles(pdf)[0]!.name).toBe('osteo-report.json') + }) + + it('decodePdfString: literal with escapes, hex ASCII, hex UTF-16BE with BOM', () => { + expect(decodePdfString('(a\\(b\\)c)')).toBe('a(b)c') + expect(decodePdfString('<6F7374656F>')).toBe('osteo') + expect(decodePdfString('')).toBe('os') + expect(decodePdfString('/Name')).toBeNull() + }) +}) + +describe('the keyword stamp', () => { + it('hex (pdf-lib) and literal /Keywords strings are both read', async () => { + expect( + hasArchiveStamp(await pdfWithoutAttachment(['faunosteo-archive/1'])), + ).toBe(true) + expect(hasArchiveStamp(await pdfWithoutAttachment(['other']))).toBe(false) + const literal = craftPdf([ + ascii('3 0 obj\n<< /Keywords (faunosteo-archive/1) >>\nendobj\n'), + ]) + expect(hasArchiveStamp(literal)).toBe(true) + await rejects(literal, 'attachment_stripped') + }) +}) + +describe('garbage never throws', () => { + const cases: [string, Uint8Array][] = [ + ['empty', new Uint8Array(0)], + [ + 'random bytes', + Uint8Array.from({ length: 4096 }, (_, i) => (i * 7919) % 256), + ], + ['the type alone', ascii('%PDF-1.7\n/Type /EmbeddedFile')], + [ + 'dict without stream', + ascii('%PDF-1.7\n<< /Type /EmbeddedFile /Length 3 >>\nendobj'), + ], + [ + 'stream without endstream', + ascii('%PDF-1.7\n<< /Type /EmbeddedFile >>\nstream\n{"version":3}'), + ], + [ + 'only the /EmbeddedFiles name tree key', + ascii('%PDF-1.7\n<< /Type /EmbeddedFiles >>\nstream\nx\nendstream'), + ], + [ + 'unterminated dictionary', + ascii( + '%PDF-1.7\n<< /Type /EmbeddedFile /Length 1 << >>\nstream\nx\nendstream', + ), + ], + [ + 'hex /Keywords without a closing bracket', + ascii('%PDF-1.7\n<< /Keywords { + expect(findEmbeddedFiles(bytes)).toEqual([]) + expect(hasArchiveStamp(bytes)).toBe(false) + await expect(readAttachedDocument(bytes)).rejects.toBeInstanceOf( + ArchiveReadError, + ) + }, + ) + + it('caps the candidates at 32', () => { + const objects = Array.from({ length: 40 }, (_, i) => + streamObject({ + num: i + 2, + dict: '/Type /EmbeddedFile /Length 1', + data: ascii('x'), + }), + ) + expect(findEmbeddedFiles(craftPdf(objects))).toHaveLength(32) + }) +}) + +describe('object-stream bomb (BLK-009)', () => { + const MiB = 1024 * 1024 + const objStmBomb = () => { + const data = bomb() + return streamObject({ + num: 9, + dict: `/Type /ObjStm /N 1 /First 4 /Filter /FlateDecode /Length ${data.byteLength}`, + data, + }) + } + const embeddedBomb = () => { + const data = bomb() + return streamObject({ + num: 8, + dict: `/Type /EmbeddedFile /Filter /FlateDecode /Length ${data.byteLength}`, + data, + }) + } + + it('64 MiB ObjStm bomb beside the attachment: report in ms, bomb never inflated', async () => { + const pdf = craftPdf([ + objStmBomb(), + objectHolding(exported, '/Type /Filespec'), + objectHolding(exported, '/Type /EmbeddedFile'), + ]) + expect(pdf.byteLength).toBeLessThan(512 * 1024) + const rssBefore = process.memoryUsage().rss + const started = performance.now() + const outcome = await readReportArchive(pdf, map()) + const elapsed = performance.now() - started + const rssDelta = process.memoryUsage().rss - rssBefore + console.log( + `[bomb] objstm: ${elapsed.toFixed(0)} ms, rss +${(rssDelta / MiB).toFixed(1)} MiB`, + ) + expect(outcome.origin).toBe('envelope_v1') + expect(outcome.report).toEqual(golden.reports.demo) + expect(elapsed).toBeLessThan(2000) + expect(rssDelta).toBeLessThan(64 * MiB) + }) + + it('EmbeddedFile bomb past 16 MiB: inflation cancelled, no_attachment, no hang', async () => { + const pdf = craftPdf([objStmBomb(), embeddedBomb()]) + const started = performance.now() + await rejects(pdf, 'no_attachment') + const elapsed = performance.now() - started + console.log(`[bomb] embedded: ${elapsed.toFixed(0)} ms`) + expect(elapsed).toBeLessThan(2000) + }) + + it('both bombs before an unnamed real attachment: bomb skipped, report read', async () => { + const pdf = craftPdf([ + objStmBomb(), + embeddedBomb(), + objectHolding(exported, '/Type /EmbeddedFile'), + ]) + const started = performance.now() + const outcome = await readReportArchive(pdf, map()) + const elapsed = performance.now() - started + console.log(`[bomb] both + unnamed attachment: ${elapsed.toFixed(0)} ms`) + expect(outcome.report).toEqual(golden.reports.demo) + expect(elapsed).toBeLessThan(2000) + }) +}) diff --git a/packages/pdf/tests/size.test.ts b/packages/pdf/tests/size.test.ts index 1c10be7f..85482282 100644 --- a/packages/pdf/tests/size.test.ts +++ b/packages/pdf/tests/size.test.ts @@ -36,7 +36,7 @@ describe('demo envelope size', () => { const raw = new TextEncoder().encode( JSON.stringify(envelope, null, 2), ).byteLength - const stored = await storedAttachmentSize(pdf) + const stored = storedAttachmentSize(pdf) console.log( `[size] demo ${mode}: stored ${stored} B, raw ${raw} B, pdf ${pdf.byteLength} B`, )