fix(pdf): bounded attachment extraction by byte scan — no pdf-lib parse on import, export without object streams (BLK-009)

This commit is contained in:
bchanot committed 2026-10-11 16:42:45 +02:00
1 parent 62b8a43e09
commit 60ee71208f
14 files changed
+1054 -198

No files matched your search

+2 -3
View File
@@ -18,7 +18,6 @@
import { readFileSync, readdirSync, statSync } from 'node:fs'
import { basename, join, resolve } from 'node:path'
import { fileURLToPath } from 'node:url'
import { PDFDocument } from 'pdf-lib'
import {
readArchiveDocument,
UnsupportedReportVersionError,
@@ -87,8 +86,8 @@ async function compareFile(
if (!st.isFile()) throw new Error('not a regular file')
if (st.size > MAX_PDF_BYTES) throw new Error('file too large')
const bytes = new Uint8Array(readFileSync(path))
const pdf = await PDFDocument.load(bytes, { ignoreEncryption: true })
const raw = await readAttachedDocument(pdf)
// Byte-scan reader (BLK-009): the PDF object graph is never parsed here.
const raw = await readAttachedDocument(bytes)
try {
const outcome = readArchiveDocument(raw, sheet.sheet.migrationMap)
return compareReport(raw, outcome, sheet, args)
+2 -1
View File
@@ -16,11 +16,12 @@ beforeAll(async () => {
sheet = await loadTestSheet()
})
/** Saved like the export (no object streams, BLK-009) so the keyword stamp is in clear. */
async function pdfWithoutAttachment(keywords?: string[]): Promise<Uint8Array> {
const doc = await PDFDocument.create()
doc.addPage()
if (keywords) doc.setKeywords(keywords)
return doc.save()
return doc.save({ useObjectStreams: false })
}
async function rejectsWith(bytes: Uint8Array, kind: string): Promise<void> {
+1 -3
View File
@@ -4,7 +4,6 @@
* and on a Step 6 envelope (archive header line).
*/
import { describe, it, expect, beforeAll } from 'vitest'
import { PDFDocument } from 'pdf-lib'
import { readFileSync } from 'node:fs'
import { fileURLToPath } from 'node:url'
import { dirname, resolve } from 'node:path'
@@ -48,8 +47,7 @@ beforeAll(async () => {
describe('compareReport on the frozen v2 archive PDF', () => {
it('shows every legacy zone and finding next to its canonical target', async () => {
const bytes = readFileSync(resolve(HERE, '../fixtures/report-v2.full.pdf'))
const pdf = await PDFDocument.load(new Uint8Array(bytes))
const raw = await readAttachedDocument(pdf)
const raw = await readAttachedDocument(new Uint8Array(bytes))
const c = compareRaw(raw)
const lines = renderComparison('full.pdf', c)
expect(c.source).toBe('v2')
+86 -45
View File
@@ -35,6 +35,10 @@ branding have changed since the export. Design record:
- Document information: title, author, subject, dates = `exportedAt`, and
`Keywords: faunosteo-archive/1` so a PDF whose attachment was stripped by a
viewer is still recognised.
- Saved without object streams (`useObjectStreams: false`) since BLK-009: the
Filespec, its file name and the Info dictionary stay in clear, where the
byte-scan reader below can see them. Files produced earlier pack them into
an `/ObjStm`; their attachment stream is still in clear and still imports.
## Attachment: the report archive envelope (v1)
@@ -53,18 +57,38 @@ schema):
"report": { "version": 3, "...": "Report v3 exactly as exported" },
"sheetSnapshot": {
"status": "draft",
"ref": { "sheetId": "…", "versionId": null, "documentSha256": "…", "schemaVersion": 2, "speciesId": "equine" },
"document": { "schema_version": 2, "...": "lean, valid anatomical sheet document" },
"assets": [ { "svgAssetId": "…", "sha256": "…", "svg": "<svg …>" } ]
"ref": {
"sheetId": "…",
"versionId": null,
"documentSha256": "…",
"schemaVersion": 2,
"speciesId": "equine"
},
"document": {
"schema_version": 2,
"...": "lean, valid anatomical sheet document"
},
"assets": [{ "svgAssetId": "…", "sha256": "…", "svg": "<svg …>" }]
},
"branding": {
"siteUrl": "https://…",
"headerLogo": { "kind": "png", "base64": "…" },
"footerLogo": { "kind": "png", "base64": "…" },
"poweredBy": { "logo": { "kind": "png", "base64": "…" }, "url": "https://…", "fallbackText": "…" },
"partner": { "logo": { "kind": "svg", "svg": "<svg …>" }, "url": "https://…", "fallbackText": "…" }
"poweredBy": {
"logo": { "kind": "png", "base64": "…" },
"url": "https://…",
"fallbackText": "…"
},
"partner": {
"logo": { "kind": "svg", "svg": "<svg …>" },
"url": "https://…",
"fallbackText": "…"
}
},
"labels": { "fr": { "pdf.section.zones": "ZONES", "...": "25 keys" }, "en": { "...": "" } }
"labels": {
"fr": { "pdf.section.zones": "ZONES", "...": "25 keys" },
"en": { "...": "" }
}
}
```
@@ -74,14 +98,14 @@ The snapshot is the sheet the bilan was built on, reduced to what the bilan
needs, as a **valid** anatomical sheet document (schema v2, same integrity
rules as the catalog):
| Part | Content |
|---|---|
| `ref` | sheet id, version id, sha256 of the full catalog document (provenance pointer), schema version, species |
| `document.views`, `annotations`, `categories`, `vertebral_config` | all of them: geometry, colours, names, anatomical relations (`finding_zone_id`, `vertebral_region_code`, `opens_view_id`, laterality) |
| `document.finding_zones` | every zone with its name; for the zones the report's findings use (resolved or `unresolved`), the categories pruned to the referenced paths with the name and vet flag of every node on the path; other zones keep `categories: []` |
| `document.references` | emptied (bibliography) |
| `document.name`, `description`, `scope_disclaimer`, `attribution` | kept |
| `assets` | the SVG text of every asset a view reaches, with the catalog's content hash when known |
| Part | Content |
| ----------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `ref` | sheet id, version id, sha256 of the full catalog document (provenance pointer), schema version, species |
| `document.views`, `annotations`, `categories`, `vertebral_config` | all of them: geometry, colours, names, anatomical relations (`finding_zone_id`, `vertebral_region_code`, `opens_view_id`, laterality) |
| `document.finding_zones` | every zone with its name; for the zones the report's findings use (resolved or `unresolved`), the categories pruned to the referenced paths with the name and vet flag of every node on the path; other zones keep `categories: []` |
| `document.references` | emptied (bibliography) |
| `document.name`, `description`, `scope_disclaimer`, `attribution` | kept |
| `assets` | the SVG text of every asset a view reaches, with the catalog's content hash when known |
`branding` is the exact set of images, URLs and credit texts used at export;
the application builds it from logos bundled at build time
@@ -98,10 +122,10 @@ Invariant: `report.sheet.documentSha256 === sheetSnapshot.ref.documentSha256`
Measured on the demo report (6 observations, both views, equine draft):
| Branding | Stored in the PDF (Flate) | Raw JSON |
|---|---|---|
| no logos | 55 KB | 189 KB |
| full branding (3 PNG logos + partner SVG) | 136 KB | 306 KB |
| Branding | Stored in the PDF (Flate) | Raw JSON |
| ----------------------------------------- | ------------------------- | -------- |
| no logos | 55 KB | 189 KB |
| full branding (3 PNG logos + partner SVG) | 136 KB | 306 KB |
The drawing is the largest part (47 KB stored); the archived PNG logos add
about 80 KB stored. The 250 KB target applies to the stored size.
@@ -111,11 +135,22 @@ about 80 KB stored. The 250 KB target applies to the stored size.
`readReportArchive(pdfBytes, migrationMap)` reads in layers so the report is
never lost because of its wrapper:
1. The PDF must load (`invalid_pdf`, also above 50 MiB).
2. The attachment named `osteo-report*` is taken first, else the first
embedded file; Flate streams are inflated under a 16 MiB cap. No
attachment: `attachment_stripped` when the PDF carries the Faunosteo
keyword, else `no_attachment`.
1. The file must weigh at most 50 MiB and carry `%PDF-` within its first
KiB (`invalid_pdf`). Nothing else of the PDF syntax is required.
2. The attachment is located by a byte scan, never by parsing the PDF object
graph (BLK-009): every `/Type /EmbeddedFile` stream dictionary in clear
(a stream object can never sit inside an object stream) is a candidate,
with its direct `/Length` or, failing that, the distance to `endstream`;
a `/Filespec` dictionary in clear whose `/EF /F` points at the stream
gives it a name. Candidates named `osteo-report*` come first, then file
order; Flate data is inflated by the platform inflater under a 16 MiB cap
(cancelled beyond it), and the first candidate whose text is a JSON
object with a numeric `envelopeVersion` or `version` wins. Object
streams, cross-reference streams and page contents are never inflated.
No candidate: `attachment_stripped` when a `/Keywords` string in clear
contains `faunosteo-archive`, else `no_attachment`. A stamped file saved
with object streams (before BLK-009) hides its stamp and reads as
`no_attachment`.
3. The report is located (`data.report` inside an envelope, `data` for a
bare attachment) and read with the report reader: v3 as is, v2 migrated
through the sheet's `report-migration-map.json` (items the map cannot
@@ -128,12 +163,12 @@ never lost because of its wrapper:
closed reason (`schema`, `caps`, `integrity`, `sheet_mismatch`) and keeps
the report.
| Archive | Produced by | Read as |
|---|---|---|
| bare Report v2 | application before Step 5 | `origin 'v2'`, migrated, unresolved counts |
| bare Report v3 | Step 5 | `origin 'bare_v3'`, no archive |
| envelope v1 | Step 6 | `origin 'envelope_v1'`, archive kept or `snapshotRejected` |
| envelope v≥2 | future | report kept when its version is known, snapshot dropped |
| Archive | Produced by | Read as |
| -------------- | ------------------------- | ---------------------------------------------------------- |
| bare Report v2 | application before Step 5 | `origin 'v2'`, migrated, unresolved counts |
| bare Report v3 | Step 5 | `origin 'bare_v3'`, no archive |
| envelope v1 | Step 6 | `origin 'envelope_v1'`, archive kept or `snapshotRejected` |
| envelope v≥2 | future | report kept when its version is known, snapshot dropped |
Readers never drop a schema version: sheet document schema v2 stays readable
as long as envelope v1 archives exist, as Report v2 does today. A rollback to
@@ -156,17 +191,17 @@ Two operations are distinct:
## Limits and sanitising
| Item | Cap |
|---|---|
| PDF file | 50 MiB |
| attachment, raw and inflated | 16 MiB |
| snapshot assets | ≤ 4, ≤ 2 MiB each, ≤ 6 MiB total |
| snapshot document | ≤ 64 views, ≤ 1,024 annotations, ≤ 8,192 finding nodes, depth ≤ 8, names ≤ 512 chars |
| branding | PNG base64 ≤ 512 KiB, SVG ≤ 256 KiB, http(s) URLs ≤ 2,048 chars |
| branding URL | `http(s)://` then printable ASCII only, without `"` `'` `(` `)` `<` `>` `\`, no space or control character |
| branding PNG | valid signature, one IHDR (13 bytes) first with a colour type and bit depth the spec allows, compression and filter method 0, no interlace; each side 1 to 8,192 px, ≤ 16,777,216 px (4096²), expected raw scanline size ≤ 16 MiB; then only `PLTE` `tRNS` `IDAT` `gAMA` `cHRM` `sRGB` `sBIT` `pHYs` `bKGD`, no chunk past the buffer, an empty `IEND` as the last bytes; the IDAT stream inflated natively and yielding exactly the raw scanline size |
| report at export | ≤ 500 observations, ≤ 8 views per observation |
| SVG sanitiser | ≤ 20,000 elements, nesting ≤ 64, ≤ 256 `<use>`, blur `stdDeviation` ≤ 100 |
| Item | Cap |
| ---------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
| PDF file | 50 MiB |
| attachment, raw and inflated | 16 MiB |
| snapshot assets | ≤ 4, ≤ 2 MiB each, ≤ 6 MiB total |
| snapshot document | ≤ 64 views, ≤ 1,024 annotations, ≤ 8,192 finding nodes, depth ≤ 8, names ≤ 512 chars |
| branding | PNG base64 ≤ 512 KiB, SVG ≤ 256 KiB, http(s) URLs ≤ 2,048 chars |
| branding URL | `http(s)://` then printable ASCII only, without `"` `'` `(` `)` `<` `>` `\`, no space or control character |
| branding PNG | valid signature, one IHDR (13 bytes) first with a colour type and bit depth the spec allows, compression and filter method 0, no interlace; each side 1 to 8,192 px, ≤ 16,777,216 px (4096²), expected raw scanline size ≤ 16 MiB; then only `PLTE` `tRNS` `IDAT` `gAMA` `cHRM` `sRGB` `sBIT` `pHYs` `bKGD`, no chunk past the buffer, an empty `IEND` as the last bytes; the IDAT stream inflated natively and yielding exactly the raw scanline size |
| report at export | ≤ 500 observations, ≤ 8 views per observation |
| SVG sanitiser | ≤ 20,000 elements, nesting ≤ 64, ≤ 256 `<use>`, blur `stdDeviation` ≤ 100 |
Every SVG that reaches the engine (sheet asset or branding) is untrusted: a
text containing `<!DOCTYPE` or `<!ENTITY` is refused; the parsed tree is
@@ -223,11 +258,17 @@ fallback text and the export reports `logo_failed`.
### Known limits
- pdf-lib inflates PDF object streams without a cap when loading: a crafted
PDF of a few hundred KB can expand to hundreds of MB in the tab before the
attachment is even looked at. The 50 MiB file cap bounds the input, not
the expansion. Mitigation planned: parse in a Worker, or locate the
attachment without a full load.
- pdf-lib is out of the import path (BLK-009): the browser and the
`report:compare` CLI read the attachment by byte scan, so a crafted object
stream is never inflated (tested with a 64 MiB bomb: milliseconds, no
memory growth). pdf-lib stays on the export side, on trusted inputs, and
in tests that inspect exported files. What remains before external users
(`docs/development/PDF_IMPORT_HARDENING.md` §5): a Worker or server-side
budget for any path that still parses PDF objects, the pdf-lib dependency
decision, and read-side caps on report collections.
- The scanner reads what is in clear: an attachment whose stream dictionary
is malformed beyond `/Length` and `endstream`, or an encrypted file, yields
no candidate (`no_attachment`), as the previous reader did in practice.
- `<use>` expansion is bounded by the rasterizer, not by the sanitiser,
which only counts the `<use>` elements (≤ 256). resvg (tests) refuses an
oversized tree; the production path is the browser rasterizer
+30 -22
View File
@@ -1,8 +1,9 @@
# PDF import hardening — BLK-009 analysis
Status 2026-10-11: analysis requested by the user after Step 6. Nothing here is
implemented; §4 says what can land now and §5 what must land before the
import is opened to external users.
Status 2026-10-11: analysis requested by the user after Step 6. §4 landed the
same day on `feature/step-06-pdf-archives` (options A + B, see §4 for the
commit); §5 lists what must land before the import is opened to external
users.
## 1. The problem
@@ -36,13 +37,13 @@ PDF object graph at all.
## 3. Options
| Option | Protection | Cost | Verdict |
|---|---|---|---|
| A. Byte-scan reader: find `/Type /EmbeddedFile` streams, read `/Length` (direct integer, as pdf-lib writes it; fall back to `endstream` search), inflate with the existing capped `inflateFlate`, parse JSON; no `PDFDocument.load` on import | removes pdf-lib from the import path entirely: the only inflation is our capped one | one module (~120 lines) + tests on every fixture (v2, bare v3, envelope, object streams on/off, several streams, foreign PDF) | **recommended, can land now** |
| B. Export with `useObjectStreams: false` | the Filespec name, the Info keywords and the Filespec→EF link become visible to a byte scan, so the filename-first rule and the `attachment_stripped` detection keep working without the object graph; files a few KB larger | one save option; older PDFs keep their object streams, which A tolerates | **recommended with A** |
| C. Load in a Web Worker with a timeout and a memory budget | contains the damage (the tab survives, the worker dies) | worker bundling, message protocol, no memory cap API in browsers (only termination on timeout) | defence in depth, later |
| D. Pre-scan declared `/Length` of `/ObjStm` streams and refuse above a sum | partial: declared lengths are the compressed sizes, the expansion is unknown | small | insufficient alone |
| E. Replace pdf-lib's parser (fork `@cantoo/pdf-lib`, or `pdf.js` for reading) | maintained parser; pdf.js has its own limits | dependency change, larger bundle | evaluate for Step 14 (server-side import) |
| Option | Protection | Cost | Verdict |
| --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------- |
| A. Byte-scan reader: find `/Type /EmbeddedFile` streams, read `/Length` (direct integer, as pdf-lib writes it; fall back to `endstream` search), inflate with the existing capped `inflateFlate`, parse JSON; no `PDFDocument.load` on import | removes pdf-lib from the import path entirely: the only inflation is our capped one | one module (~120 lines) + tests on every fixture (v2, bare v3, envelope, object streams on/off, several streams, foreign PDF) | **recommended, can land now** |
| B. Export with `useObjectStreams: false` | the Filespec name, the Info keywords and the Filespec→EF link become visible to a byte scan, so the filename-first rule and the `attachment_stripped` detection keep working without the object graph; files a few KB larger | one save option; older PDFs keep their object streams, which A tolerates | **recommended with A** |
| C. Load in a Web Worker with a timeout and a memory budget | contains the damage (the tab survives, the worker dies) | worker bundling, message protocol, no memory cap API in browsers (only termination on timeout) | defence in depth, later |
| D. Pre-scan declared `/Length` of `/ObjStm` streams and refuse above a sum | partial: declared lengths are the compressed sizes, the expansion is unknown | small | insufficient alone |
| E. Replace pdf-lib's parser (fork `@cantoo/pdf-lib`, or `pdf.js` for reading) | maintained parser; pdf.js has its own limits | dependency change, larger bundle | evaluate for Step 14 (server-side import) |
With A, the import path reads bytes, finds at most a few streams, inflates one
under 16 MiB, parses JSON under the existing schema caps. pdf-lib stays on
@@ -59,22 +60,29 @@ encrypted file is encrypted too and would fail to inflate → `no_attachment`,
same outcome as today in practice); the 50 MiB file cap stays as the scan
bound.
## 4. What can be done now (before merge of Step 6, or as the next chore)
## 4. Done 2026-10-11 (feature/step-06-pdf-archives, commit: see `git log -- packages/pdf/src/attachment/scan.ts`)
1. Option A in `packages/pdf/src/attachment/scan.ts`, used by
`readReportArchive`; `readAttachedDocument(pdfDoc)` kept for the CLI until
it moves to the scanner too.
1. Option A in `packages/pdf/src/attachment/scan.ts`
(`findEmbeddedFiles`, `readAttachmentBytes`, `hasArchiveStamp`), used by
`readReportArchive`; `readAttachedDocument(bytes)` takes the file bytes
and the `report:compare` CLI runs on the scanner too. No
`PDFDocument.load` remains under `packages/pdf/src` or `apps/app/scripts`.
2. Option B on export (`pdfDoc.save({ useObjectStreams: false })`): no visible
change; goldens unaffected (they record drawn text, images, links); the
envelope roundtrip tests prove the reader still finds the attachment in
both layouts.
3. Tests: every existing import fixture through the scanner; a crafted
object-stream bomb (the audit's recipe: a few hundred KB inflating to
hundreds of MB) imports in milliseconds without inflating it; a PDF with
two embedded files; a PDF whose `/Length` is indirect.
change; goldens unchanged (they record drawn text, images, links); the
envelope roundtrip tests prove the reader finds the attachment in both
layouts.
3. Tests (`packages/pdf/tests/scan.test.ts`): the three frozen fixtures (object
streams) and a fresh export; two embedded files with a decoy first;
indirect and wrong `/Length`; `\r\n` after `stream`; `/Type/EmbeddedFile`
without a space; hex and literal names and keywords; garbage inputs and
the 32-candidate cap; a 64 MiB object-stream bomb next to the real
attachment (report in milliseconds, RSS flat) and an `EmbeddedFile` bomb
(inflation cancelled at 16 MiB, `no_attachment`).
4. Docs: `docs/PDF-FORMAT.md` "Reading an archive" and "Known limits".
Estimated size: one executor phase, about 300 lines including tests.
Known trade-off: a stamped PDF saved before this change (object streams)
hides its `/Keywords`, so a stripped attachment on such a file reads as
`no_attachment` instead of `attachment_stripped`.
## 5. Prerequisites before external users (Step 7+ sync, Step 14 worker)
+20 -89
View File
@@ -1,105 +1,36 @@
import {
PDFDict,
PDFHexString,
PDFName,
PDFRawStream,
PDFStream,
PDFString,
type PDFDocument,
} from 'pdf-lib'
import { ArchiveReadError } from '../errors'
import { inflateFlate, MAX_ATTACHMENT_BYTES } from './inflate'
import {
ARCHIVE_STAMP,
hasPdfHeader,
locateAttachment,
MAX_PDF_BYTES,
} from './scan'
/**
* Finds the Faunosteo attachment of a loaded PDF: the Filespec whose file
* name starts with `osteo-report` first, else the first embedded file
* (today's behaviour, kept as the fallback). pdf-lib 1.17 exposes no
* attachment API, so this walks the indirect objects.
* The raw JSON a Faunosteo PDF carries, read from the bytes alone (BLK-009):
* no PDF object graph is parsed, so no object stream is ever inflated. pdf-lib
* stays on the export side.
*/
export const ATTACHMENT_NAME_PREFIX = 'osteo-report'
export const PDF_KEYWORD = 'faunosteo-archive/1'
export { ATTACHMENT_NAME_PREFIX } from './scan'
export const PDF_KEYWORD = `${ARCHIVE_STAMP}/1`
const TYPE = PDFName.of('Type')
const F = PDFName.of('F')
function textOf(value: unknown): string | null {
if (value instanceof PDFString || value instanceof PDFHexString) {
return value.decodeText()
/** Size cap first, then the `%PDF-` header: anything else is `invalid_pdf`. */
function assertPdfBytes(bytes: Uint8Array): void {
if (bytes.byteLength > MAX_PDF_BYTES || !hasPdfHeader(bytes)) {
throw new ArchiveReadError('invalid_pdf')
}
return null
}
/** The embedded stream of a `/Filespec` named for a Faunosteo report. */
function namedFilespecStream(dict: PDFDict): PDFStream | null {
if (dict.get(TYPE) !== PDFName.of('Filespec')) return null
const name = textOf(dict.lookup(F))
if (name === null || !name.startsWith(ATTACHMENT_NAME_PREFIX)) return null
const files = dict.lookup(PDFName.of('EF'))
if (!(files instanceof PDFDict)) return null
const stream = files.lookup(F)
return stream instanceof PDFStream ? stream : null
}
/** Today's heuristic: an `/EmbeddedFile` stream or a JSON-subtyped one. */
function looksLikeEmbeddedJson(stream: PDFStream): boolean {
const type = String(stream.dict.lookup(TYPE) ?? '')
const subtype = String(stream.dict.lookup(PDFName.of('Subtype')) ?? '')
return (
type.includes('EmbeddedFile') ||
subtype.includes('application/json') ||
subtype.includes('application#2Fjson')
)
}
function locateAttachment(pdfDoc: PDFDocument): PDFStream | null {
const objects = pdfDoc.context.enumerateIndirectObjects()
for (const [, object] of objects) {
if (!(object instanceof PDFDict)) continue
const stream = namedFilespecStream(object)
if (stream) return stream
}
for (const [, object] of objects) {
if (object instanceof PDFStream && looksLikeEmbeddedJson(object)) {
return object
}
}
return null
}
/** Stream bytes, inflated when Flate-encoded; null when unreadable or over the cap. */
async function attachmentBytes(stream: PDFStream): Promise<Uint8Array | null> {
const contents =
stream instanceof PDFRawStream ? stream.contents : stream.getContents()
if (contents.byteLength > MAX_ATTACHMENT_BYTES) return null
const filter = String(stream.dict.lookup(PDFName.of('Filter')) ?? '')
if (!filter.includes('FlateDecode')) return contents
return inflateFlate(contents)
}
/** A PDF stamped as an archive but holding no attachment lost it in a viewer. */
function missingAttachmentError(pdfDoc: PDFDocument): ArchiveReadError {
const stamped = (pdfDoc.getKeywords() ?? '').includes('faunosteo-archive')
return new ArchiveReadError(stamped ? 'attachment_stripped' : 'no_attachment')
}
/**
* The raw JSON carried by the PDF, whatever its shape (bare report of any
* version or archive envelope), before any validation: what the archive
* check tool shows next to the read result. Throws ArchiveReadError.
* check tool shows next to the read result. Throws ArchiveReadError
* (`invalid_pdf`, `no_attachment`, `attachment_stripped`).
*/
export async function readAttachedDocument(
pdfDoc: PDFDocument,
bytes: Uint8Array,
): Promise<unknown> {
const stream = locateAttachment(pdfDoc)
if (!stream) throw missingAttachmentError(pdfDoc)
const bytes = await attachmentBytes(stream)
if (!bytes || bytes.byteLength > MAX_ATTACHMENT_BYTES) {
throw new ArchiveReadError('no_attachment')
}
try {
return JSON.parse(new TextDecoder().decode(bytes)) as unknown
} catch {
throw new ArchiveReadError('no_attachment')
}
assertPdfBytes(bytes)
return (await locateAttachment(bytes)).data
}
+441
View File
@@ -0,0 +1,441 @@
import { ArchiveReadError } from '../errors'
import { inflateFlate, MAX_ATTACHMENT_BYTES } from './inflate'
/**
* Locates the Faunosteo attachment by scanning the PDF bytes (BLK-009): a
* stream object can never live inside an object stream, so every
* `/Type /EmbeddedFile` dictionary and its data are in clear in the file.
* Nothing here parses the object graph, the xref or an `/ObjStm`, so the
* only inflation on the import path is the capped one of `inflateFlate`.
* The scanner never throws on garbage: it returns fewer candidates.
*/
/** A Faunosteo PDF weighs a few hundred KB; anything larger is not one. */
export const MAX_PDF_BYTES = 50 * 1024 * 1024
export const ATTACHMENT_NAME_PREFIX = 'osteo-report'
/** The keyword stamp every archive carries (`faunosteo-archive/<n>`). */
export const ARCHIVE_STAMP = 'faunosteo-archive'
/** Candidates beyond this count are ignored: a crafted file cannot make the reader loop. */
const MAX_CANDIDATES = 32
/** A stream or Filespec dictionary is small; a longer one is skipped. */
const MAX_DICT_BYTES = 4096
/** Window read before a dictionary for its `n 0 obj` header. */
const HEADER_WINDOW = 48
export interface EmbeddedFileCandidate {
/** First byte of the stream data. */
offset: number
/** Byte count of the stream data (declared, or measured to `endstream`). */
length: number
filter: 'FlateDecode' | null
/** From a visible `/Filespec` whose `/EF /F` points at this stream. */
name: string | null
}
const encode = (text: string): Uint8Array =>
Uint8Array.from(text, (c) => c.charCodeAt(0))
const TYPE = encode('/Type')
const EMBEDDED_FILE = encode('/EmbeddedFile')
const FILESPEC = encode('/Filespec')
const KEYWORDS = encode('/Keywords')
const STREAM = encode('stream')
const ENDSTREAM = encode('endstream')
const PDF_HEADER = encode('%PDF-')
// ---------------------------------------------------------------------------
// Byte helpers: every loop is bounded by `end` (at most MAX_PDF_BYTES)
function isWhitespace(byte: number): boolean {
return (
byte === 0x20 ||
byte === 0x0a ||
byte === 0x0d ||
byte === 0x09 ||
byte === 0x0c ||
byte === 0x00
)
}
/** PDF delimiters and whitespace end a name token. */
function isDelimiter(byte: number): boolean {
return isWhitespace(byte) || '()<>[]{}/%'.includes(String.fromCharCode(byte))
}
/** Naive forward search; the needle's first byte is the fast reject. */
function indexOf(
bytes: Uint8Array,
needle: Uint8Array,
from: number,
end: number,
): number {
const last = end - needle.length
const first = needle[0]!
for (let i = Math.max(from, 0); i <= last; i += 1) {
if (bytes[i] !== first) continue
let j = 1
while (j < needle.length && bytes[i + j] === needle[j]) j += 1
if (j === needle.length) return i
}
return -1
}
function skipWhitespace(bytes: Uint8Array, from: number, end: number): number {
let i = from
while (i < end && isWhitespace(bytes[i]!)) i += 1
return i
}
/** Latin-1 view of a small window (never the whole file). */
function latin1(bytes: Uint8Array, start: number, end: number): string {
let out = ''
for (let i = start; i < end; i += 1) out += String.fromCharCode(bytes[i]!)
return out
}
/** UTF-16BE with BOM (how pdf-lib writes text strings) or Latin-1 bytes. */
function decodeTextBytes(raw: number[]): string {
if (raw.length >= 2 && raw[0] === 0xfe && raw[1] === 0xff) {
let out = ''
for (let i = 2; i + 1 < raw.length; i += 2) {
out += String.fromCharCode((raw[i]! << 8) | raw[i + 1]!)
}
return out
}
return String.fromCharCode(...raw)
}
/** A `(literal)` or `<hex>` string token as text; null for anything else. */
export function decodePdfString(token: string): string | null {
if (token.startsWith('<') && token.endsWith('>')) {
const hex = token.slice(1, -1).replace(/[^0-9a-fA-F]/g, '')
const raw: number[] = []
for (let i = 0; i + 1 < hex.length; i += 2) {
raw.push(parseInt(hex.slice(i, i + 2), 16))
}
return decodeTextBytes(raw)
}
if (token.startsWith('(') && token.endsWith(')')) {
const body = token.slice(1, -1).replace(/\\([()\\])/g, '$1')
return decodeTextBytes(Array.from(body, (c) => c.charCodeAt(0) & 0xff))
}
return null
}
// ---------------------------------------------------------------------------
// Dictionaries: `/Type /Name` matches and their `<< … >>` bounds
/** Every `/Type /<name>` (space optional) whose name ends on a delimiter. */
function findTypedDicts(
bytes: Uint8Array,
name: Uint8Array,
end: number,
): number[] {
const hits: number[] = []
let from = 0
while (hits.length < MAX_CANDIDATES) {
const at = indexOf(bytes, name, from, end)
if (at < 0) break
from = at + name.length
const after = at + name.length
if (after < end && !isDelimiter(bytes[after]!)) continue
let before = at
while (before > 0 && isWhitespace(bytes[before - 1]!)) before -= 1
const typeAt = before - TYPE.length
if (typeAt >= 0 && indexOf(bytes, TYPE, typeAt, before) === typeAt) {
hits.push(typeAt)
}
}
return hits
}
/** Index of the `<<` opening the dictionary holding `pos` (nested dicts skipped); -1 when none. */
function dictStart(bytes: Uint8Array, pos: number): number {
const floor = Math.max(0, pos - MAX_DICT_BYTES)
let depth = 0
let i = pos
while (i - 2 >= floor) {
const pair = (bytes[i - 2]! << 8) | bytes[i - 1]!
if (pair === 0x3e3e)
depth += 1 // >>
else if (pair === 0x3c3c) {
if (depth === 0) return i - 2
depth -= 1
}
i -= pair === 0x3e3e || pair === 0x3c3c ? 2 : 1
}
return -1
}
/** Index just past the `>>` closing the dictionary opened at `start`; -1 when none. */
function dictEnd(bytes: Uint8Array, start: number, end: number): number {
const limit = Math.min(end, start + MAX_DICT_BYTES)
let depth = 0
let i = start
while (i + 1 < limit) {
const pair = (bytes[i]! << 8) | bytes[i + 1]!
if (pair === 0x3c3c) depth += 1
else if (pair === 0x3e3e) {
depth -= 1
if (depth === 0) return i + 2
}
i += pair === 0x3c3c || pair === 0x3e3e ? 2 : 1
}
return -1
}
interface DictSpan {
start: number
end: number
text: string
objectNumber: number | null
}
/** The dictionary around a `/Type` match, its text and the `n 0 obj` header before it. */
function dictAround(
bytes: Uint8Array,
pos: number,
end: number,
): DictSpan | null {
const start = dictStart(bytes, pos)
if (start < 0) return null
const close = dictEnd(bytes, start, end)
if (close < 0) return null
const header = latin1(bytes, Math.max(0, start - HEADER_WINDOW), start)
const match = /(\d{1,10})\s+\d{1,10}\s+obj\s*$/.exec(header)
return {
start,
end: close,
text: latin1(bytes, start, close),
objectNumber: match ? Number(match[1]) : null,
}
}
// ---------------------------------------------------------------------------
// Stream data bounds
/** Data offset after `stream` + `\r\n` or `\n`; -1 when the keyword is missing. */
function streamDataOffset(
bytes: Uint8Array,
dictClose: number,
end: number,
): number {
const at = skipWhitespace(bytes, dictClose, end)
if (indexOf(bytes, STREAM, at, Math.min(end, at + STREAM.length)) !== at) {
return -1
}
let i = at + STREAM.length
if (bytes[i] === 0x0d) i += 1
if (bytes[i] !== 0x0a) return -1
return i + 1
}
/** A direct `/Length N` (never an `N 0 R` reference), when it lands on `endstream`. */
function declaredLength(
bytes: Uint8Array,
dict: string,
offset: number,
end: number,
): number | null {
const match = /\/Length\s+(\d{1,10})(?!\d)(?!\s+\d+\s+R)/.exec(dict)
if (!match) return null
const length = Number(match[1])
if (offset + length > end) return null
const tail = skipWhitespace(bytes, offset + length, end)
const found = indexOf(
bytes,
ENDSTREAM,
tail,
Math.min(end, tail + ENDSTREAM.length),
)
return found === tail ? length : null
}
/** Fallback for an indirect or wrong `/Length`: up to `endstream`, the EOL before it dropped. */
function measuredLength(
bytes: Uint8Array,
offset: number,
end: number,
): number | null {
const at = indexOf(bytes, ENDSTREAM, offset, end)
if (at < 0) return null
let stop = at
if (stop > offset && bytes[stop - 1] === 0x0a) stop -= 1
if (stop > offset && bytes[stop - 1] === 0x0d) stop -= 1
return stop - offset
}
function filterOf(dict: string): EmbeddedFileCandidate['filter'] {
return /\/Filter\s*(\[\s*)?\/FlateDecode\b/.test(dict) ? 'FlateDecode' : null
}
function candidateOf(
bytes: Uint8Array,
dict: DictSpan,
end: number,
): Omit<EmbeddedFileCandidate, 'name'> | null {
const offset = streamDataOffset(bytes, dict.end, end)
if (offset < 0) return null
const length =
declaredLength(bytes, dict.text, offset, end) ??
measuredLength(bytes, offset, end)
if (length === null) return null
return { offset, length, filter: filterOf(dict.text) }
}
// ---------------------------------------------------------------------------
// Filespec names and the keyword stamp
const STRING_TOKEN = String.raw`\((?:\\.|[^\\)])*\)|<[0-9a-fA-F\s]*>`
/** `/F` (else `/UF`) of a Filespec dictionary, the `/EF` sub-dictionary set aside. */
function filespecName(dict: string): string | null {
const withoutEf = dict.replace(/\/EF\s*<<[\s\S]*?>>/, '')
const f = new RegExp(String.raw`/F\s*(${STRING_TOKEN})`).exec(withoutEf)
const uf = new RegExp(String.raw`/UF\s*(${STRING_TOKEN})`).exec(withoutEf)
const token = f?.[1] ?? uf?.[1]
return token ? decodePdfString(token) : null
}
/** Object number of the stream a Filespec's `/EF << /F n 0 R >>` points at. */
function filespecStreamObject(dict: string): number | null {
const ef = /\/EF\s*<<([\s\S]*?)>>/.exec(dict)
const ref = ef ? /\/F\s+(\d{1,10})\s+\d{1,10}\s+R/.exec(ef[1]!) : null
return ref ? Number(ref[1]) : null
}
/** Stream object number → file name, from every Filespec dictionary in clear. */
function filespecNames(bytes: Uint8Array, end: number): Map<number, string> {
const names = new Map<number, string>()
for (const pos of findTypedDicts(bytes, FILESPEC, end)) {
const dict = dictAround(bytes, pos, end)
if (!dict) continue
const target = filespecStreamObject(dict.text)
const name = filespecName(dict.text)
if (target !== null && name !== null && !names.has(target)) {
names.set(target, name)
}
}
return names
}
/** The string token right after a `/Keywords` key, as text. */
function keywordsAt(
bytes: Uint8Array,
keyAt: number,
end: number,
): string | null {
const start = skipWhitespace(bytes, keyAt + KEYWORDS.length, end)
const limit = Math.min(end, start + MAX_DICT_BYTES)
const window = latin1(bytes, start, limit)
const token = new RegExp(`^(?:${STRING_TOKEN})`).exec(window)
return token ? decodePdfString(token[0]) : null
}
/** True when a visible `/Keywords` string carries the archive stamp. */
export function hasArchiveStamp(bytes: Uint8Array): boolean {
const end = Math.min(bytes.byteLength, MAX_PDF_BYTES)
let from = 0
for (let n = 0; n < MAX_CANDIDATES; n += 1) {
const at = indexOf(bytes, KEYWORDS, from, end)
if (at < 0) return false
from = at + KEYWORDS.length
if (isDelimiter(bytes[from] ?? 0x20)) {
if (keywordsAt(bytes, at, end)?.includes(ARCHIVE_STAMP)) return true
}
}
return false
}
/** `%PDF-` within the first KiB, as viewers accept it. */
export function hasPdfHeader(bytes: Uint8Array): boolean {
return indexOf(bytes, PDF_HEADER, 0, Math.min(bytes.byteLength, 1024)) >= 0
}
// ---------------------------------------------------------------------------
// Public scan
/**
* Every `/Type /EmbeddedFile` stream of the file, named when its Filespec
* is in clear (export without object streams); `osteo-report*` names first,
* then file order. At most MAX_CANDIDATES, within MAX_PDF_BYTES.
*/
export function findEmbeddedFiles(bytes: Uint8Array): EmbeddedFileCandidate[] {
const end = Math.min(bytes.byteLength, MAX_PDF_BYTES)
const names = filespecNames(bytes, end)
const found: EmbeddedFileCandidate[] = []
for (const pos of findTypedDicts(bytes, EMBEDDED_FILE, end)) {
const dict = dictAround(bytes, pos, end)
if (!dict) continue
const candidate = candidateOf(bytes, dict, end)
if (!candidate) continue
const name =
dict.objectNumber === null ? null : (names.get(dict.objectNumber) ?? null)
found.push({ ...candidate, name })
}
const named = (c: EmbeddedFileCandidate) =>
c.name?.startsWith(ATTACHMENT_NAME_PREFIX) ? 0 : 1
return found.sort((a, b) => named(a) - named(b))
}
/** Raw or inflated stream data under the attachment cap; null when unusable. */
async function candidateBytes(
bytes: Uint8Array,
candidate: EmbeddedFileCandidate,
): Promise<Uint8Array | null> {
if (candidate.length > MAX_ATTACHMENT_BYTES) return null
const data = bytes.subarray(
candidate.offset,
candidate.offset + candidate.length,
)
if (candidate.filter === null) return data
return inflateFlate(data)
}
/** JSON with a numeric `envelopeVersion` or `version`; undefined otherwise. */
function parseVersioned(data: Uint8Array): unknown {
let parsed: unknown
try {
parsed = JSON.parse(new TextDecoder().decode(data))
} catch {
return undefined
}
if (typeof parsed !== 'object' || parsed === null) return undefined
const record = parsed as Record<string, unknown>
const versioned =
typeof record['envelopeVersion'] === 'number' ||
typeof record['version'] === 'number'
return versioned ? parsed : undefined
}
export interface LocatedAttachment {
bytes: Uint8Array
data: unknown
}
/**
* The first candidate whose data is a versioned JSON document, decoded once.
* None: `attachment_stripped` when the stamp is visible, else `no_attachment`.
*/
export async function locateAttachment(
bytes: Uint8Array,
): Promise<LocatedAttachment> {
for (const candidate of findEmbeddedFiles(bytes)) {
const data = await candidateBytes(bytes, candidate)
if (!data) continue
const parsed = parseVersioned(data)
if (parsed !== undefined) return { bytes: data, data: parsed }
}
throw new ArchiveReadError(
hasArchiveStamp(bytes) ? 'attachment_stripped' : 'no_attachment',
)
}
/** The attachment's decoded bytes (BLK-009 reader). Throws ArchiveReadError. */
export async function readAttachmentBytes(
bytes: Uint8Array,
): Promise<Uint8Array> {
return (await locateAttachment(bytes)).bytes
}
+4 -1
View File
@@ -201,7 +201,10 @@ export async function exportReportArchive(
warnings,
})
await attachEnvelope(pdfDoc, attachment, input.exportedAt)
return { pdf: await pdfDoc.save(), envelope, warnings }
// No object streams (BLK-009): the Filespec name and the keyword stamp stay
// in clear for the byte-scan reader; the file grows by a few KB.
const pdf = await pdfDoc.save({ useObjectStreams: false })
return { pdf, envelope, warnings }
}
export interface ReexportOptions {
+5
View File
@@ -4,6 +4,11 @@
export { exportReportArchive, exportFromEnvelope } from './export'
export { readReportArchive, MAX_PDF_BYTES } from './read'
export { readAttachedDocument, PDF_KEYWORD } from './attachment/read'
export {
findEmbeddedFiles,
readAttachmentBytes,
type EmbeddedFileCandidate,
} from './attachment/scan'
export { MAX_ATTACHMENT_BYTES } from './attachment/inflate'
export {
ArchiveReadError,
+5 -18
View File
@@ -1,4 +1,3 @@
import { PDFDocument } from 'pdf-lib'
import {
readArchiveDocument,
UnsupportedReportVersionError,
@@ -8,23 +7,12 @@ import {
import { readAttachedDocument } from './attachment/read'
import { ArchiveReadError } from './errors'
/** A Faunosteo PDF weighs a few hundred KB; anything larger is not one. */
export const MAX_PDF_BYTES = 50 * 1024 * 1024
async function loadPdf(pdfBytes: Uint8Array): Promise<PDFDocument> {
if (pdfBytes.byteLength > MAX_PDF_BYTES) {
throw new ArchiveReadError('invalid_pdf')
}
try {
return await PDFDocument.load(pdfBytes, { ignoreEncryption: true })
} catch {
throw new ArchiveReadError('invalid_pdf')
}
}
export { MAX_PDF_BYTES } from './attachment/scan'
/**
* Reads a Faunosteo PDF back: size cap, PDF parse, attachment (filename
* first), JSON, then the layered reading of the domain (report first, then
* Reads a Faunosteo PDF back: size cap and `%PDF-` header (`invalid_pdf`),
* attachment located by a byte scan (filename first, never a PDF parse:
* BLK-009), JSON, then the layered reading of the domain (report first, then
* the envelope accepted whole or dropped with a reason). An unknown report
* version is `incompatible_version` with that version; any other content
* failure is `no_attachment`, as today.
@@ -33,8 +21,7 @@ export async function readReportArchive(
pdfBytes: Uint8Array,
map: ReportMigrationMap,
): Promise<ArchiveReadOutcome> {
const pdfDoc = await loadPdf(pdfBytes)
const data = await readAttachedDocument(pdfDoc)
const data = await readAttachedDocument(pdfBytes)
try {
return readArchiveDocument(data, map)
} catch (error) {
+23 -14
View File
@@ -1,17 +1,29 @@
import { AFRelationship, PDFDocument, PDFName, PDFRawStream } from 'pdf-lib'
import { AFRelationship, PDFDocument } from 'pdf-lib'
import { findEmbeddedFiles } from '../../src'
/**
* Crafted PDFs for the read tests, built with pdf-lib's own attach. Saved
* without object streams by default, like the export since BLK-009; pass
* `objectStreams: true` for the pre-BLK-009 layout (Filespec and Info
* packed into an `/ObjStm`, invisible to the byte scan).
*/
export interface CraftOptions {
keywords?: string[]
before?: [string, unknown]
objectStreams?: boolean
}
/** Crafted PDFs for the read tests, built with pdf-lib's own attach. */
export async function pdfWithAttachment(
name: string,
payload: unknown,
extra: { keywords?: string[]; before?: [string, unknown] } = {},
extra: CraftOptions = {},
): Promise<Uint8Array> {
const doc = await PDFDocument.create()
doc.addPage()
if (extra.keywords) doc.setKeywords(extra.keywords)
if (extra.before) await attachJson(doc, extra.before[0], extra.before[1])
await attachJson(doc, name, payload)
return doc.save()
return doc.save({ useObjectStreams: extra.objectStreams ?? false })
}
async function attachJson(
@@ -27,20 +39,17 @@ async function attachJson(
export async function pdfWithoutAttachment(
keywords?: string[],
objectStreams = false,
): Promise<Uint8Array> {
const doc = await PDFDocument.create()
doc.addPage()
if (keywords) doc.setKeywords(keywords)
return doc.save()
return doc.save({ useObjectStreams: objectStreams })
}
/** Stored (Flate) size of the first `/EmbeddedFile` stream of a saved PDF. */
export async function storedAttachmentSize(bytes: Uint8Array): Promise<number> {
const doc = await PDFDocument.load(bytes)
for (const [, object] of doc.context.enumerateIndirectObjects()) {
if (!(object instanceof PDFRawStream)) continue
const type = object.dict.lookup(PDFName.of('Type'))
if (type === PDFName.of('EmbeddedFile')) return object.contents.length
}
throw new Error('no EmbeddedFile stream')
/** Stored (Flate) size of the first `/EmbeddedFile` stream of a saved PDF, by the byte scan. */
export function storedAttachmentSize(bytes: Uint8Array): number {
const [first] = findEmbeddedFiles(bytes)
if (!first) throw new Error('no EmbeddedFile stream')
return first.length
}
+9 -1
View File
@@ -149,6 +149,11 @@ describe('attachment presence and selection', () => {
await rejects(bytes, { kind: 'attachment_stripped' })
})
it('a stamped PDF saved with object streams (pre-BLK-009 layout): the stamp is invisible, no_attachment', async () => {
const bytes = await pdfWithoutAttachment(['faunosteo-archive/1'], true)
await rejects(bytes, { kind: 'no_attachment' })
})
it('a foreign PDF: no_attachment', async () => {
await rejects(await pdfWithoutAttachment(), { kind: 'no_attachment' })
const foreign = await pdfWithAttachment('other.json', { hello: 'world' })
@@ -166,10 +171,13 @@ describe('attachment presence and selection', () => {
expect(outcome.report).toEqual(golden.reports.demo)
})
it('refuses an oversized or unparseable PDF as invalid_pdf', async () => {
it('refuses an oversized file or one without a %PDF- header as invalid_pdf', async () => {
await rejects(new Uint8Array(MAX_PDF_BYTES + 1), { kind: 'invalid_pdf' })
await rejects(new TextEncoder().encode('not a pdf at all'), {
kind: 'invalid_pdf',
})
const late = new Uint8Array(2048)
late.set(new TextEncoder().encode('%PDF-1.7'), 1024)
await rejects(late, { kind: 'invalid_pdf' })
})
})
+425
View File
@@ -0,0 +1,425 @@
/**
* BLK-009: the byte-scan attachment reader. Candidates on a fresh export and
* on the frozen fixtures (object-stream layout), crafted layouts (two files,
* indirect `/Length`, `\r\n`, no space after `/Type`), garbage, the keyword
* stamp, and the object-stream bomb the reader never inflates.
*/
import { deflateSync } from 'node:zlib'
import { beforeAll, describe, expect, it } from 'vitest'
import { buildSheetSnapshot, type Report } from '@faunosteo/domain'
import {
ArchiveReadError,
defaultPdfLabels,
exportReportArchive,
findEmbeddedFiles,
readAttachedDocument,
readAttachmentBytes,
readReportArchive,
type EmbeddedFileCandidate,
} from '../src'
import { inflateFlate } from '../src/attachment/inflate'
import { decodePdfString, hasArchiveStamp } from '../src/attachment/scan'
import { readFixtureBytes } from './helpers/draft'
import {
FIXED_EXPORTED_AT,
nologosBranding,
rasterizerOf,
} from './helpers/goldens'
import { pdfWithAttachment, pdfWithoutAttachment } from './helpers/pdf'
import { loadGoldenReports, type GoldenReports } from './helpers/reports'
let golden: GoldenReports
let exported: Uint8Array
const map = () => golden.draft.migrationMap
beforeAll(async () => {
golden = loadGoldenReports()
exported = (await exportDemo()).pdf
})
async function exportDemo() {
const report: Report = golden.reports.demo
return exportReportArchive({
report,
sheet: buildSheetSnapshot(golden.draft, report),
branding: nologosBranding(),
labels: defaultPdfLabels,
locale: 'fr',
rasterizer: rasterizerOf('nologos'),
exportedAt: FIXED_EXPORTED_AT,
})
}
// ---------------------------------------------------------------------------
// Hand-built PDFs: the scanner never reads the xref, so none is written
const latin1 = (bytes: Uint8Array) => Buffer.from(bytes).toString('latin1')
const ascii = (text: string) => new Uint8Array(Buffer.from(text, 'latin1'))
function concat(parts: Uint8Array[]): Uint8Array {
const out = new Uint8Array(parts.reduce((n, p) => n + p.byteLength, 0))
let at = 0
for (const part of parts) {
out.set(part, at)
at += part.byteLength
}
return out
}
interface StreamSpec {
num: number
dict: string
data: Uint8Array
eol?: string
}
function streamObject({ num, dict, data, eol = '\n' }: StreamSpec): Uint8Array {
return concat([
ascii(`${num} 0 obj\n<<${dict}>>\nstream${eol}`),
data,
ascii('\nendstream\nendobj\n'),
])
}
function craftPdf(objects: Uint8Array[]): Uint8Array {
return concat([
ascii('%PDF-1.7\n'),
...objects,
ascii('trailer\n<< /Root 1 0 R >>\n%%EOF\n'),
])
}
/** The `n 0 obj … endobj` bytes of the object whose dictionary holds `marker`. */
function objectHolding(pdf: Uint8Array, marker: string): Uint8Array {
const text = latin1(pdf)
const at = text.indexOf(marker)
expect(at).toBeGreaterThan(0)
const start = text.lastIndexOf(' 0 obj', at)
const headerStart = text.lastIndexOf('\n', start) + 1
const end = text.indexOf('endobj', at) + 'endobj'.length
return pdf.slice(headerStart, end)
}
const BOMB_RAW = 64 * 1024 * 1024
const bomb = () => new Uint8Array(deflateSync(new Uint8Array(BOMB_RAW)))
async function rejects(
bytes: Uint8Array,
kind: ArchiveReadError['kind'],
): Promise<void> {
const promise = readReportArchive(bytes, map())
await expect(promise).rejects.toBeInstanceOf(ArchiveReadError)
await expect(promise).rejects.toMatchObject({ kind })
}
// ---------------------------------------------------------------------------
describe('findEmbeddedFiles on a fresh export (no object streams)', () => {
let candidates: EmbeddedFileCandidate[]
beforeAll(() => {
candidates = findEmbeddedFiles(exported)
})
it('finds the one Flate stream, named through its visible Filespec', async () => {
expect(candidates).toHaveLength(1)
const [c] = candidates
expect(c!.filter).toBe('FlateDecode')
expect(c!.name).toBe('osteo-report.json')
const inflated = await inflateFlate(
exported.subarray(c!.offset, c!.offset + c!.length),
)
const json = JSON.parse(new TextDecoder().decode(inflated!)) as {
envelopeVersion: number
}
expect(json.envelopeVersion).toBe(1)
})
it('the length is the declared /Length and the data ends on endstream', () => {
const [c] = candidates
const text = latin1(exported)
expect(text).toContain(`/Length ${c!.length}\n>>\nstream\n`)
expect(text.slice(c!.offset + c!.length)).toMatch(/^\r?\nendstream/)
})
it('the export keeps the Filespec, its name and the stamp in clear, no object stream', () => {
const text = latin1(exported)
expect(text).toContain('/Type /Filespec')
expect(text).toContain('/F (osteo-report.json)')
expect(text).not.toContain('/Type /ObjStm')
expect(hasArchiveStamp(exported)).toBe(true)
})
it('readAttachmentBytes returns the inflated envelope bytes', async () => {
const bytes = await readAttachmentBytes(exported)
const json = JSON.parse(new TextDecoder().decode(bytes)) as {
report: Report
}
expect(json.report).toEqual(golden.reports.demo)
})
})
describe('compatibility: frozen fixtures written with object streams', () => {
it.each([
['report-v2.full.pdf', 'v2'],
['report-v2.minimal.pdf', 'v2'],
['report-v3.bare.pdf', 'bare_v3'],
] as const)('%s → origin %s', async (name, origin) => {
const bytes = readFixtureBytes(name)
expect(latin1(bytes)).toContain('/Type /ObjStm')
const candidates = findEmbeddedFiles(bytes)
expect(candidates).toHaveLength(1)
// The Filespec is packed into the object stream: no name, the JSON decides.
expect(candidates[0]!.name).toBeNull()
expect(candidates[0]!.filter).toBe('FlateDecode')
const outcome = await readReportArchive(bytes, map())
expect(outcome.origin).toBe(origin)
})
it('a stamped PDF with object streams hides its stamp: no_attachment (documented)', async () => {
const bytes = await pdfWithoutAttachment(['faunosteo-archive/1'], true)
expect(hasArchiveStamp(bytes)).toBe(false)
await rejects(bytes, 'no_attachment')
})
})
describe('selection between several embedded files', () => {
it('a decoy EmbeddedFile first, the named Faunosteo one second: the name wins', async () => {
const bytes = await pdfWithAttachment(
'osteo-report.json',
golden.reports.demo,
{
before: ['other.json', { hello: 'world', version: 2 }],
},
)
const names = findEmbeddedFiles(bytes).map((c) => c.name)
expect(names).toEqual(['osteo-report.json', 'other.json'])
const outcome = await readReportArchive(bytes, map())
expect(outcome.origin).toBe('bare_v3')
expect(outcome.report).toEqual(golden.reports.demo)
})
it('an unnamed decoy without a version is skipped for the versioned JSON after it', async () => {
const decoy = streamObject({
num: 2,
dict: '/Type /EmbeddedFile /Length 17',
data: ascii('{"hello":"world"}'),
})
const real = objectHolding(exported, '/Type /EmbeddedFile')
const outcome = await readReportArchive(craftPdf([decoy, real]), map())
expect(outcome.origin).toBe('envelope_v1')
expect(outcome.report).toEqual(golden.reports.demo)
})
it('a foreign PDF, with or without attachments: no_attachment', async () => {
await rejects(await pdfWithoutAttachment(), 'no_attachment')
await rejects(
await pdfWithAttachment('other.json', { hello: 'world' }),
'no_attachment',
)
})
})
describe('crafted stream layouts', () => {
// UTF-8: the demo report holds an en dash that Latin-1 cannot carry.
const demoJson = () =>
new TextEncoder().encode(JSON.stringify(golden.reports.demo))
it('/Length as an indirect reference: read up to endstream', async () => {
const pdf = craftPdf([
streamObject({
num: 2,
dict: '/Type /EmbeddedFile /Length 7 0 R',
data: demoJson(),
}),
ascii('7 0 obj\n123456\nendobj\n'),
])
const [c] = findEmbeddedFiles(pdf)
expect(c!.filter).toBeNull()
expect(c!.length).toBe(demoJson().byteLength)
expect((await readReportArchive(pdf, map())).origin).toBe('bare_v3')
})
it('a wrong direct /Length that does not land on endstream: measured instead', async () => {
const pdf = craftPdf([
streamObject({
num: 2,
dict: '/Type /EmbeddedFile /Length 5',
data: demoJson(),
}),
])
expect(findEmbeddedFiles(pdf)[0]!.length).toBe(demoJson().byteLength)
expect((await readReportArchive(pdf, map())).origin).toBe('bare_v3')
})
it('CRLF after stream, /Type/EmbeddedFile unspaced, a leading /Params dict', async () => {
const data = demoJson()
const pdf = craftPdf([
streamObject({
num: 2,
dict: `/Params << /Size ${data.byteLength} >> /Type/EmbeddedFile/Length ${data.byteLength}`,
data,
eol: '\r\n',
}),
])
const [c] = findEmbeddedFiles(pdf)
expect(c!.length).toBe(data.byteLength)
expect(await readAttachmentBytes(pdf)).toEqual(data)
})
it('a Filespec with a hex /F name and an /EF listing /UF too names its stream', () => {
const name = Buffer.from('osteo-report.json', 'utf16le')
.swap16()
.toString('hex')
const pdf = craftPdf([
ascii(
`3 0 obj\n<< /Type /Filespec /F <${name}> /EF << /F 2 0 R /UF 2 0 R >> >>\nendobj\n`,
),
streamObject({
num: 2,
dict: '/Type /EmbeddedFile /Length 13',
data: ascii('{"version":3}'),
}),
])
expect(findEmbeddedFiles(pdf)[0]!.name).toBe('osteo-report.json')
})
it('decodePdfString: literal with escapes, hex ASCII, hex UTF-16BE with BOM', () => {
expect(decodePdfString('(a\\(b\\)c)')).toBe('a(b)c')
expect(decodePdfString('<6F7374656F>')).toBe('osteo')
expect(decodePdfString('<FEFF006F0073>')).toBe('os')
expect(decodePdfString('/Name')).toBeNull()
})
})
describe('the keyword stamp', () => {
it('hex (pdf-lib) and literal /Keywords strings are both read', async () => {
expect(
hasArchiveStamp(await pdfWithoutAttachment(['faunosteo-archive/1'])),
).toBe(true)
expect(hasArchiveStamp(await pdfWithoutAttachment(['other']))).toBe(false)
const literal = craftPdf([
ascii('3 0 obj\n<< /Keywords (faunosteo-archive/1) >>\nendobj\n'),
])
expect(hasArchiveStamp(literal)).toBe(true)
await rejects(literal, 'attachment_stripped')
})
})
describe('garbage never throws', () => {
const cases: [string, Uint8Array][] = [
['empty', new Uint8Array(0)],
[
'random bytes',
Uint8Array.from({ length: 4096 }, (_, i) => (i * 7919) % 256),
],
['the type alone', ascii('%PDF-1.7\n/Type /EmbeddedFile')],
[
'dict without stream',
ascii('%PDF-1.7\n<< /Type /EmbeddedFile /Length 3 >>\nendobj'),
],
[
'stream without endstream',
ascii('%PDF-1.7\n<< /Type /EmbeddedFile >>\nstream\n{"version":3}'),
],
[
'only the /EmbeddedFiles name tree key',
ascii('%PDF-1.7\n<< /Type /EmbeddedFiles >>\nstream\nx\nendstream'),
],
[
'unterminated dictionary',
ascii(
'%PDF-1.7\n<< /Type /EmbeddedFile /Length 1 << >>\nstream\nx\nendstream',
),
],
[
'hex /Keywords without a closing bracket',
ascii('%PDF-1.7\n<< /Keywords <FEFF0066'),
],
]
it.each(cases)(
'%s: no candidate, no stamp, no_attachment or invalid_pdf',
async (_name, bytes) => {
expect(findEmbeddedFiles(bytes)).toEqual([])
expect(hasArchiveStamp(bytes)).toBe(false)
await expect(readAttachedDocument(bytes)).rejects.toBeInstanceOf(
ArchiveReadError,
)
},
)
it('caps the candidates at 32', () => {
const objects = Array.from({ length: 40 }, (_, i) =>
streamObject({
num: i + 2,
dict: '/Type /EmbeddedFile /Length 1',
data: ascii('x'),
}),
)
expect(findEmbeddedFiles(craftPdf(objects))).toHaveLength(32)
})
})
describe('object-stream bomb (BLK-009)', () => {
const MiB = 1024 * 1024
const objStmBomb = () => {
const data = bomb()
return streamObject({
num: 9,
dict: `/Type /ObjStm /N 1 /First 4 /Filter /FlateDecode /Length ${data.byteLength}`,
data,
})
}
const embeddedBomb = () => {
const data = bomb()
return streamObject({
num: 8,
dict: `/Type /EmbeddedFile /Filter /FlateDecode /Length ${data.byteLength}`,
data,
})
}
it('64 MiB ObjStm bomb beside the attachment: report in ms, bomb never inflated', async () => {
const pdf = craftPdf([
objStmBomb(),
objectHolding(exported, '/Type /Filespec'),
objectHolding(exported, '/Type /EmbeddedFile'),
])
expect(pdf.byteLength).toBeLessThan(512 * 1024)
const rssBefore = process.memoryUsage().rss
const started = performance.now()
const outcome = await readReportArchive(pdf, map())
const elapsed = performance.now() - started
const rssDelta = process.memoryUsage().rss - rssBefore
console.log(
`[bomb] objstm: ${elapsed.toFixed(0)} ms, rss +${(rssDelta / MiB).toFixed(1)} MiB`,
)
expect(outcome.origin).toBe('envelope_v1')
expect(outcome.report).toEqual(golden.reports.demo)
expect(elapsed).toBeLessThan(2000)
expect(rssDelta).toBeLessThan(64 * MiB)
})
it('EmbeddedFile bomb past 16 MiB: inflation cancelled, no_attachment, no hang', async () => {
const pdf = craftPdf([objStmBomb(), embeddedBomb()])
const started = performance.now()
await rejects(pdf, 'no_attachment')
const elapsed = performance.now() - started
console.log(`[bomb] embedded: ${elapsed.toFixed(0)} ms`)
expect(elapsed).toBeLessThan(2000)
})
it('both bombs before an unnamed real attachment: bomb skipped, report read', async () => {
const pdf = craftPdf([
objStmBomb(),
embeddedBomb(),
objectHolding(exported, '/Type /EmbeddedFile'),
])
const started = performance.now()
const outcome = await readReportArchive(pdf, map())
const elapsed = performance.now() - started
console.log(`[bomb] both + unnamed attachment: ${elapsed.toFixed(0)} ms`)
expect(outcome.report).toEqual(golden.reports.demo)
expect(elapsed).toBeLessThan(2000)
})
})
+1 -1
View File
@@ -36,7 +36,7 @@ describe('demo envelope size', () => {
const raw = new TextEncoder().encode(
JSON.stringify(envelope, null, 2),
).byteLength
const stored = await storedAttachmentSize(pdf)
const stored = storedAttachmentSize(pdf)
console.log(
`[size] demo ${mode}: stored ${stored} B, raw ${raw} B, pdf ${pdf.byteLength} B`,
)