fix(pdf): bounded attachment extraction by byte scan — no pdf-lib parse on import, export without object streams (BLK-009)
This commit is contained in:
1 parent
62b8a43e09
commit
60ee71208f
14 files changed
+1054
-198
No files matched your search
@@ -18,7 +18,6 @@
|
|||||||
import { readFileSync, readdirSync, statSync } from 'node:fs'
|
import { readFileSync, readdirSync, statSync } from 'node:fs'
|
||||||
import { basename, join, resolve } from 'node:path'
|
import { basename, join, resolve } from 'node:path'
|
||||||
import { fileURLToPath } from 'node:url'
|
import { fileURLToPath } from 'node:url'
|
||||||
import { PDFDocument } from 'pdf-lib'
|
|
||||||
import {
|
import {
|
||||||
readArchiveDocument,
|
readArchiveDocument,
|
||||||
UnsupportedReportVersionError,
|
UnsupportedReportVersionError,
|
||||||
@@ -87,8 +86,8 @@ async function compareFile(
|
|||||||
if (!st.isFile()) throw new Error('not a regular file')
|
if (!st.isFile()) throw new Error('not a regular file')
|
||||||
if (st.size > MAX_PDF_BYTES) throw new Error('file too large')
|
if (st.size > MAX_PDF_BYTES) throw new Error('file too large')
|
||||||
const bytes = new Uint8Array(readFileSync(path))
|
const bytes = new Uint8Array(readFileSync(path))
|
||||||
const pdf = await PDFDocument.load(bytes, { ignoreEncryption: true })
|
// Byte-scan reader (BLK-009): the PDF object graph is never parsed here.
|
||||||
const raw = await readAttachedDocument(pdf)
|
const raw = await readAttachedDocument(bytes)
|
||||||
try {
|
try {
|
||||||
const outcome = readArchiveDocument(raw, sheet.sheet.migrationMap)
|
const outcome = readArchiveDocument(raw, sheet.sheet.migrationMap)
|
||||||
return compareReport(raw, outcome, sheet, args)
|
return compareReport(raw, outcome, sheet, args)
|
||||||
|
|||||||
@@ -16,11 +16,12 @@ beforeAll(async () => {
|
|||||||
sheet = await loadTestSheet()
|
sheet = await loadTestSheet()
|
||||||
})
|
})
|
||||||
|
|
||||||
|
/** Saved like the export (no object streams, BLK-009) so the keyword stamp is in clear. */
|
||||||
async function pdfWithoutAttachment(keywords?: string[]): Promise<Uint8Array> {
|
async function pdfWithoutAttachment(keywords?: string[]): Promise<Uint8Array> {
|
||||||
const doc = await PDFDocument.create()
|
const doc = await PDFDocument.create()
|
||||||
doc.addPage()
|
doc.addPage()
|
||||||
if (keywords) doc.setKeywords(keywords)
|
if (keywords) doc.setKeywords(keywords)
|
||||||
return doc.save()
|
return doc.save({ useObjectStreams: false })
|
||||||
}
|
}
|
||||||
|
|
||||||
async function rejectsWith(bytes: Uint8Array, kind: string): Promise<void> {
|
async function rejectsWith(bytes: Uint8Array, kind: string): Promise<void> {
|
||||||
|
|||||||
@@ -4,7 +4,6 @@
|
|||||||
* and on a Step 6 envelope (archive header line).
|
* and on a Step 6 envelope (archive header line).
|
||||||
*/
|
*/
|
||||||
import { describe, it, expect, beforeAll } from 'vitest'
|
import { describe, it, expect, beforeAll } from 'vitest'
|
||||||
import { PDFDocument } from 'pdf-lib'
|
|
||||||
import { readFileSync } from 'node:fs'
|
import { readFileSync } from 'node:fs'
|
||||||
import { fileURLToPath } from 'node:url'
|
import { fileURLToPath } from 'node:url'
|
||||||
import { dirname, resolve } from 'node:path'
|
import { dirname, resolve } from 'node:path'
|
||||||
@@ -48,8 +47,7 @@ beforeAll(async () => {
|
|||||||
describe('compareReport on the frozen v2 archive PDF', () => {
|
describe('compareReport on the frozen v2 archive PDF', () => {
|
||||||
it('shows every legacy zone and finding next to its canonical target', async () => {
|
it('shows every legacy zone and finding next to its canonical target', async () => {
|
||||||
const bytes = readFileSync(resolve(HERE, '../fixtures/report-v2.full.pdf'))
|
const bytes = readFileSync(resolve(HERE, '../fixtures/report-v2.full.pdf'))
|
||||||
const pdf = await PDFDocument.load(new Uint8Array(bytes))
|
const raw = await readAttachedDocument(new Uint8Array(bytes))
|
||||||
const raw = await readAttachedDocument(pdf)
|
|
||||||
const c = compareRaw(raw)
|
const c = compareRaw(raw)
|
||||||
const lines = renderComparison('full.pdf', c)
|
const lines = renderComparison('full.pdf', c)
|
||||||
expect(c.source).toBe('v2')
|
expect(c.source).toBe('v2')
|
||||||
|
|||||||
+86
-45
@@ -35,6 +35,10 @@ branding have changed since the export. Design record:
|
|||||||
- Document information: title, author, subject, dates = `exportedAt`, and
|
- Document information: title, author, subject, dates = `exportedAt`, and
|
||||||
`Keywords: faunosteo-archive/1` so a PDF whose attachment was stripped by a
|
`Keywords: faunosteo-archive/1` so a PDF whose attachment was stripped by a
|
||||||
viewer is still recognised.
|
viewer is still recognised.
|
||||||
|
- Saved without object streams (`useObjectStreams: false`) since BLK-009: the
|
||||||
|
Filespec, its file name and the Info dictionary stay in clear, where the
|
||||||
|
byte-scan reader below can see them. Files produced earlier pack them into
|
||||||
|
an `/ObjStm`; their attachment stream is still in clear and still imports.
|
||||||
|
|
||||||
## Attachment: the report archive envelope (v1)
|
## Attachment: the report archive envelope (v1)
|
||||||
|
|
||||||
@@ -53,18 +57,38 @@ schema):
|
|||||||
"report": { "version": 3, "...": "Report v3 exactly as exported" },
|
"report": { "version": 3, "...": "Report v3 exactly as exported" },
|
||||||
"sheetSnapshot": {
|
"sheetSnapshot": {
|
||||||
"status": "draft",
|
"status": "draft",
|
||||||
"ref": { "sheetId": "…", "versionId": null, "documentSha256": "…", "schemaVersion": 2, "speciesId": "equine" },
|
"ref": {
|
||||||
"document": { "schema_version": 2, "...": "lean, valid anatomical sheet document" },
|
"sheetId": "…",
|
||||||
"assets": [ { "svgAssetId": "…", "sha256": "…", "svg": "<svg …>" } ]
|
"versionId": null,
|
||||||
|
"documentSha256": "…",
|
||||||
|
"schemaVersion": 2,
|
||||||
|
"speciesId": "equine"
|
||||||
|
},
|
||||||
|
"document": {
|
||||||
|
"schema_version": 2,
|
||||||
|
"...": "lean, valid anatomical sheet document"
|
||||||
|
},
|
||||||
|
"assets": [{ "svgAssetId": "…", "sha256": "…", "svg": "<svg …>" }]
|
||||||
},
|
},
|
||||||
"branding": {
|
"branding": {
|
||||||
"siteUrl": "https://…",
|
"siteUrl": "https://…",
|
||||||
"headerLogo": { "kind": "png", "base64": "…" },
|
"headerLogo": { "kind": "png", "base64": "…" },
|
||||||
"footerLogo": { "kind": "png", "base64": "…" },
|
"footerLogo": { "kind": "png", "base64": "…" },
|
||||||
"poweredBy": { "logo": { "kind": "png", "base64": "…" }, "url": "https://…", "fallbackText": "…" },
|
"poweredBy": {
|
||||||
"partner": { "logo": { "kind": "svg", "svg": "<svg …>" }, "url": "https://…", "fallbackText": "…" }
|
"logo": { "kind": "png", "base64": "…" },
|
||||||
|
"url": "https://…",
|
||||||
|
"fallbackText": "…"
|
||||||
|
},
|
||||||
|
"partner": {
|
||||||
|
"logo": { "kind": "svg", "svg": "<svg …>" },
|
||||||
|
"url": "https://…",
|
||||||
|
"fallbackText": "…"
|
||||||
|
}
|
||||||
},
|
},
|
||||||
"labels": { "fr": { "pdf.section.zones": "ZONES", "...": "25 keys" }, "en": { "...": "" } }
|
"labels": {
|
||||||
|
"fr": { "pdf.section.zones": "ZONES", "...": "25 keys" },
|
||||||
|
"en": { "...": "" }
|
||||||
|
}
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -74,14 +98,14 @@ The snapshot is the sheet the bilan was built on, reduced to what the bilan
|
|||||||
needs, as a **valid** anatomical sheet document (schema v2, same integrity
|
needs, as a **valid** anatomical sheet document (schema v2, same integrity
|
||||||
rules as the catalog):
|
rules as the catalog):
|
||||||
|
|
||||||
| Part | Content |
|
| Part | Content |
|
||||||
|---|---|
|
| ----------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||||
| `ref` | sheet id, version id, sha256 of the full catalog document (provenance pointer), schema version, species |
|
| `ref` | sheet id, version id, sha256 of the full catalog document (provenance pointer), schema version, species |
|
||||||
| `document.views`, `annotations`, `categories`, `vertebral_config` | all of them: geometry, colours, names, anatomical relations (`finding_zone_id`, `vertebral_region_code`, `opens_view_id`, laterality) |
|
| `document.views`, `annotations`, `categories`, `vertebral_config` | all of them: geometry, colours, names, anatomical relations (`finding_zone_id`, `vertebral_region_code`, `opens_view_id`, laterality) |
|
||||||
| `document.finding_zones` | every zone with its name; for the zones the report's findings use (resolved or `unresolved`), the categories pruned to the referenced paths with the name and vet flag of every node on the path; other zones keep `categories: []` |
|
| `document.finding_zones` | every zone with its name; for the zones the report's findings use (resolved or `unresolved`), the categories pruned to the referenced paths with the name and vet flag of every node on the path; other zones keep `categories: []` |
|
||||||
| `document.references` | emptied (bibliography) |
|
| `document.references` | emptied (bibliography) |
|
||||||
| `document.name`, `description`, `scope_disclaimer`, `attribution` | kept |
|
| `document.name`, `description`, `scope_disclaimer`, `attribution` | kept |
|
||||||
| `assets` | the SVG text of every asset a view reaches, with the catalog's content hash when known |
|
| `assets` | the SVG text of every asset a view reaches, with the catalog's content hash when known |
|
||||||
|
|
||||||
`branding` is the exact set of images, URLs and credit texts used at export;
|
`branding` is the exact set of images, URLs and credit texts used at export;
|
||||||
the application builds it from logos bundled at build time
|
the application builds it from logos bundled at build time
|
||||||
@@ -98,10 +122,10 @@ Invariant: `report.sheet.documentSha256 === sheetSnapshot.ref.documentSha256`
|
|||||||
|
|
||||||
Measured on the demo report (6 observations, both views, equine draft):
|
Measured on the demo report (6 observations, both views, equine draft):
|
||||||
|
|
||||||
| Branding | Stored in the PDF (Flate) | Raw JSON |
|
| Branding | Stored in the PDF (Flate) | Raw JSON |
|
||||||
|---|---|---|
|
| ----------------------------------------- | ------------------------- | -------- |
|
||||||
| no logos | 55 KB | 189 KB |
|
| no logos | 55 KB | 189 KB |
|
||||||
| full branding (3 PNG logos + partner SVG) | 136 KB | 306 KB |
|
| full branding (3 PNG logos + partner SVG) | 136 KB | 306 KB |
|
||||||
|
|
||||||
The drawing is the largest part (47 KB stored); the archived PNG logos add
|
The drawing is the largest part (47 KB stored); the archived PNG logos add
|
||||||
about 80 KB stored. The 250 KB target applies to the stored size.
|
about 80 KB stored. The 250 KB target applies to the stored size.
|
||||||
@@ -111,11 +135,22 @@ about 80 KB stored. The 250 KB target applies to the stored size.
|
|||||||
`readReportArchive(pdfBytes, migrationMap)` reads in layers so the report is
|
`readReportArchive(pdfBytes, migrationMap)` reads in layers so the report is
|
||||||
never lost because of its wrapper:
|
never lost because of its wrapper:
|
||||||
|
|
||||||
1. The PDF must load (`invalid_pdf`, also above 50 MiB).
|
1. The file must weigh at most 50 MiB and carry `%PDF-` within its first
|
||||||
2. The attachment named `osteo-report*` is taken first, else the first
|
KiB (`invalid_pdf`). Nothing else of the PDF syntax is required.
|
||||||
embedded file; Flate streams are inflated under a 16 MiB cap. No
|
2. The attachment is located by a byte scan, never by parsing the PDF object
|
||||||
attachment: `attachment_stripped` when the PDF carries the Faunosteo
|
graph (BLK-009): every `/Type /EmbeddedFile` stream dictionary in clear
|
||||||
keyword, else `no_attachment`.
|
(a stream object can never sit inside an object stream) is a candidate,
|
||||||
|
with its direct `/Length` or, failing that, the distance to `endstream`;
|
||||||
|
a `/Filespec` dictionary in clear whose `/EF /F` points at the stream
|
||||||
|
gives it a name. Candidates named `osteo-report*` come first, then file
|
||||||
|
order; Flate data is inflated by the platform inflater under a 16 MiB cap
|
||||||
|
(cancelled beyond it), and the first candidate whose text is a JSON
|
||||||
|
object with a numeric `envelopeVersion` or `version` wins. Object
|
||||||
|
streams, cross-reference streams and page contents are never inflated.
|
||||||
|
No candidate: `attachment_stripped` when a `/Keywords` string in clear
|
||||||
|
contains `faunosteo-archive`, else `no_attachment`. A stamped file saved
|
||||||
|
with object streams (before BLK-009) hides its stamp and reads as
|
||||||
|
`no_attachment`.
|
||||||
3. The report is located (`data.report` inside an envelope, `data` for a
|
3. The report is located (`data.report` inside an envelope, `data` for a
|
||||||
bare attachment) and read with the report reader: v3 as is, v2 migrated
|
bare attachment) and read with the report reader: v3 as is, v2 migrated
|
||||||
through the sheet's `report-migration-map.json` (items the map cannot
|
through the sheet's `report-migration-map.json` (items the map cannot
|
||||||
@@ -128,12 +163,12 @@ never lost because of its wrapper:
|
|||||||
closed reason (`schema`, `caps`, `integrity`, `sheet_mismatch`) and keeps
|
closed reason (`schema`, `caps`, `integrity`, `sheet_mismatch`) and keeps
|
||||||
the report.
|
the report.
|
||||||
|
|
||||||
| Archive | Produced by | Read as |
|
| Archive | Produced by | Read as |
|
||||||
|---|---|---|
|
| -------------- | ------------------------- | ---------------------------------------------------------- |
|
||||||
| bare Report v2 | application before Step 5 | `origin 'v2'`, migrated, unresolved counts |
|
| bare Report v2 | application before Step 5 | `origin 'v2'`, migrated, unresolved counts |
|
||||||
| bare Report v3 | Step 5 | `origin 'bare_v3'`, no archive |
|
| bare Report v3 | Step 5 | `origin 'bare_v3'`, no archive |
|
||||||
| envelope v1 | Step 6 | `origin 'envelope_v1'`, archive kept or `snapshotRejected` |
|
| envelope v1 | Step 6 | `origin 'envelope_v1'`, archive kept or `snapshotRejected` |
|
||||||
| envelope v≥2 | future | report kept when its version is known, snapshot dropped |
|
| envelope v≥2 | future | report kept when its version is known, snapshot dropped |
|
||||||
|
|
||||||
Readers never drop a schema version: sheet document schema v2 stays readable
|
Readers never drop a schema version: sheet document schema v2 stays readable
|
||||||
as long as envelope v1 archives exist, as Report v2 does today. A rollback to
|
as long as envelope v1 archives exist, as Report v2 does today. A rollback to
|
||||||
@@ -156,17 +191,17 @@ Two operations are distinct:
|
|||||||
|
|
||||||
## Limits and sanitising
|
## Limits and sanitising
|
||||||
|
|
||||||
| Item | Cap |
|
| Item | Cap |
|
||||||
|---|---|
|
| ---------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
|
||||||
| PDF file | 50 MiB |
|
| PDF file | 50 MiB |
|
||||||
| attachment, raw and inflated | 16 MiB |
|
| attachment, raw and inflated | 16 MiB |
|
||||||
| snapshot assets | ≤ 4, ≤ 2 MiB each, ≤ 6 MiB total |
|
| snapshot assets | ≤ 4, ≤ 2 MiB each, ≤ 6 MiB total |
|
||||||
| snapshot document | ≤ 64 views, ≤ 1,024 annotations, ≤ 8,192 finding nodes, depth ≤ 8, names ≤ 512 chars |
|
| snapshot document | ≤ 64 views, ≤ 1,024 annotations, ≤ 8,192 finding nodes, depth ≤ 8, names ≤ 512 chars |
|
||||||
| branding | PNG base64 ≤ 512 KiB, SVG ≤ 256 KiB, http(s) URLs ≤ 2,048 chars |
|
| branding | PNG base64 ≤ 512 KiB, SVG ≤ 256 KiB, http(s) URLs ≤ 2,048 chars |
|
||||||
| branding URL | `http(s)://` then printable ASCII only, without `"` `'` `(` `)` `<` `>` `\`, no space or control character |
|
| branding URL | `http(s)://` then printable ASCII only, without `"` `'` `(` `)` `<` `>` `\`, no space or control character |
|
||||||
| branding PNG | valid signature, one IHDR (13 bytes) first with a colour type and bit depth the spec allows, compression and filter method 0, no interlace; each side 1 to 8,192 px, ≤ 16,777,216 px (4096²), expected raw scanline size ≤ 16 MiB; then only `PLTE` `tRNS` `IDAT` `gAMA` `cHRM` `sRGB` `sBIT` `pHYs` `bKGD`, no chunk past the buffer, an empty `IEND` as the last bytes; the IDAT stream inflated natively and yielding exactly the raw scanline size |
|
| branding PNG | valid signature, one IHDR (13 bytes) first with a colour type and bit depth the spec allows, compression and filter method 0, no interlace; each side 1 to 8,192 px, ≤ 16,777,216 px (4096²), expected raw scanline size ≤ 16 MiB; then only `PLTE` `tRNS` `IDAT` `gAMA` `cHRM` `sRGB` `sBIT` `pHYs` `bKGD`, no chunk past the buffer, an empty `IEND` as the last bytes; the IDAT stream inflated natively and yielding exactly the raw scanline size |
|
||||||
| report at export | ≤ 500 observations, ≤ 8 views per observation |
|
| report at export | ≤ 500 observations, ≤ 8 views per observation |
|
||||||
| SVG sanitiser | ≤ 20,000 elements, nesting ≤ 64, ≤ 256 `<use>`, blur `stdDeviation` ≤ 100 |
|
| SVG sanitiser | ≤ 20,000 elements, nesting ≤ 64, ≤ 256 `<use>`, blur `stdDeviation` ≤ 100 |
|
||||||
|
|
||||||
Every SVG that reaches the engine (sheet asset or branding) is untrusted: a
|
Every SVG that reaches the engine (sheet asset or branding) is untrusted: a
|
||||||
text containing `<!DOCTYPE` or `<!ENTITY` is refused; the parsed tree is
|
text containing `<!DOCTYPE` or `<!ENTITY` is refused; the parsed tree is
|
||||||
@@ -223,11 +258,17 @@ fallback text and the export reports `logo_failed`.
|
|||||||
|
|
||||||
### Known limits
|
### Known limits
|
||||||
|
|
||||||
- pdf-lib inflates PDF object streams without a cap when loading: a crafted
|
- pdf-lib is out of the import path (BLK-009): the browser and the
|
||||||
PDF of a few hundred KB can expand to hundreds of MB in the tab before the
|
`report:compare` CLI read the attachment by byte scan, so a crafted object
|
||||||
attachment is even looked at. The 50 MiB file cap bounds the input, not
|
stream is never inflated (tested with a 64 MiB bomb: milliseconds, no
|
||||||
the expansion. Mitigation planned: parse in a Worker, or locate the
|
memory growth). pdf-lib stays on the export side, on trusted inputs, and
|
||||||
attachment without a full load.
|
in tests that inspect exported files. What remains before external users
|
||||||
|
(`docs/development/PDF_IMPORT_HARDENING.md` §5): a Worker or server-side
|
||||||
|
budget for any path that still parses PDF objects, the pdf-lib dependency
|
||||||
|
decision, and read-side caps on report collections.
|
||||||
|
- The scanner reads what is in clear: an attachment whose stream dictionary
|
||||||
|
is malformed beyond `/Length` and `endstream`, or an encrypted file, yields
|
||||||
|
no candidate (`no_attachment`), as the previous reader did in practice.
|
||||||
- `<use>` expansion is bounded by the rasterizer, not by the sanitiser,
|
- `<use>` expansion is bounded by the rasterizer, not by the sanitiser,
|
||||||
which only counts the `<use>` elements (≤ 256). resvg (tests) refuses an
|
which only counts the `<use>` elements (≤ 256). resvg (tests) refuses an
|
||||||
oversized tree; the production path is the browser rasterizer
|
oversized tree; the production path is the browser rasterizer
|
||||||
|
|||||||
@@ -1,8 +1,9 @@
|
|||||||
# PDF import hardening — BLK-009 analysis
|
# PDF import hardening — BLK-009 analysis
|
||||||
|
|
||||||
Status 2026-10-11: analysis requested by the user after Step 6. Nothing here is
|
Status 2026-10-11: analysis requested by the user after Step 6. §4 landed the
|
||||||
implemented; §4 says what can land now and §5 what must land before the
|
same day on `feature/step-06-pdf-archives` (options A + B, see §4 for the
|
||||||
import is opened to external users.
|
commit); §5 lists what must land before the import is opened to external
|
||||||
|
users.
|
||||||
|
|
||||||
## 1. The problem
|
## 1. The problem
|
||||||
|
|
||||||
@@ -36,13 +37,13 @@ PDF object graph at all.
|
|||||||
|
|
||||||
## 3. Options
|
## 3. Options
|
||||||
|
|
||||||
| Option | Protection | Cost | Verdict |
|
| Option | Protection | Cost | Verdict |
|
||||||
|---|---|---|---|
|
| --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------- |
|
||||||
| A. Byte-scan reader: find `/Type /EmbeddedFile` streams, read `/Length` (direct integer, as pdf-lib writes it; fall back to `endstream` search), inflate with the existing capped `inflateFlate`, parse JSON; no `PDFDocument.load` on import | removes pdf-lib from the import path entirely: the only inflation is our capped one | one module (~120 lines) + tests on every fixture (v2, bare v3, envelope, object streams on/off, several streams, foreign PDF) | **recommended, can land now** |
|
| A. Byte-scan reader: find `/Type /EmbeddedFile` streams, read `/Length` (direct integer, as pdf-lib writes it; fall back to `endstream` search), inflate with the existing capped `inflateFlate`, parse JSON; no `PDFDocument.load` on import | removes pdf-lib from the import path entirely: the only inflation is our capped one | one module (~120 lines) + tests on every fixture (v2, bare v3, envelope, object streams on/off, several streams, foreign PDF) | **recommended, can land now** |
|
||||||
| B. Export with `useObjectStreams: false` | the Filespec name, the Info keywords and the Filespec→EF link become visible to a byte scan, so the filename-first rule and the `attachment_stripped` detection keep working without the object graph; files a few KB larger | one save option; older PDFs keep their object streams, which A tolerates | **recommended with A** |
|
| B. Export with `useObjectStreams: false` | the Filespec name, the Info keywords and the Filespec→EF link become visible to a byte scan, so the filename-first rule and the `attachment_stripped` detection keep working without the object graph; files a few KB larger | one save option; older PDFs keep their object streams, which A tolerates | **recommended with A** |
|
||||||
| C. Load in a Web Worker with a timeout and a memory budget | contains the damage (the tab survives, the worker dies) | worker bundling, message protocol, no memory cap API in browsers (only termination on timeout) | defence in depth, later |
|
| C. Load in a Web Worker with a timeout and a memory budget | contains the damage (the tab survives, the worker dies) | worker bundling, message protocol, no memory cap API in browsers (only termination on timeout) | defence in depth, later |
|
||||||
| D. Pre-scan declared `/Length` of `/ObjStm` streams and refuse above a sum | partial: declared lengths are the compressed sizes, the expansion is unknown | small | insufficient alone |
|
| D. Pre-scan declared `/Length` of `/ObjStm` streams and refuse above a sum | partial: declared lengths are the compressed sizes, the expansion is unknown | small | insufficient alone |
|
||||||
| E. Replace pdf-lib's parser (fork `@cantoo/pdf-lib`, or `pdf.js` for reading) | maintained parser; pdf.js has its own limits | dependency change, larger bundle | evaluate for Step 14 (server-side import) |
|
| E. Replace pdf-lib's parser (fork `@cantoo/pdf-lib`, or `pdf.js` for reading) | maintained parser; pdf.js has its own limits | dependency change, larger bundle | evaluate for Step 14 (server-side import) |
|
||||||
|
|
||||||
With A, the import path reads bytes, finds at most a few streams, inflates one
|
With A, the import path reads bytes, finds at most a few streams, inflates one
|
||||||
under 16 MiB, parses JSON under the existing schema caps. pdf-lib stays on
|
under 16 MiB, parses JSON under the existing schema caps. pdf-lib stays on
|
||||||
@@ -59,22 +60,29 @@ encrypted file is encrypted too and would fail to inflate → `no_attachment`,
|
|||||||
same outcome as today in practice); the 50 MiB file cap stays as the scan
|
same outcome as today in practice); the 50 MiB file cap stays as the scan
|
||||||
bound.
|
bound.
|
||||||
|
|
||||||
## 4. What can be done now (before merge of Step 6, or as the next chore)
|
## 4. Done 2026-10-11 (feature/step-06-pdf-archives, commit: see `git log -- packages/pdf/src/attachment/scan.ts`)
|
||||||
|
|
||||||
1. Option A in `packages/pdf/src/attachment/scan.ts`, used by
|
1. Option A in `packages/pdf/src/attachment/scan.ts`
|
||||||
`readReportArchive`; `readAttachedDocument(pdfDoc)` kept for the CLI until
|
(`findEmbeddedFiles`, `readAttachmentBytes`, `hasArchiveStamp`), used by
|
||||||
it moves to the scanner too.
|
`readReportArchive`; `readAttachedDocument(bytes)` takes the file bytes
|
||||||
|
and the `report:compare` CLI runs on the scanner too. No
|
||||||
|
`PDFDocument.load` remains under `packages/pdf/src` or `apps/app/scripts`.
|
||||||
2. Option B on export (`pdfDoc.save({ useObjectStreams: false })`): no visible
|
2. Option B on export (`pdfDoc.save({ useObjectStreams: false })`): no visible
|
||||||
change; goldens unaffected (they record drawn text, images, links); the
|
change; goldens unchanged (they record drawn text, images, links); the
|
||||||
envelope roundtrip tests prove the reader still finds the attachment in
|
envelope roundtrip tests prove the reader finds the attachment in both
|
||||||
both layouts.
|
layouts.
|
||||||
3. Tests: every existing import fixture through the scanner; a crafted
|
3. Tests (`packages/pdf/tests/scan.test.ts`): the three frozen fixtures (object
|
||||||
object-stream bomb (the audit's recipe: a few hundred KB inflating to
|
streams) and a fresh export; two embedded files with a decoy first;
|
||||||
hundreds of MB) imports in milliseconds without inflating it; a PDF with
|
indirect and wrong `/Length`; `\r\n` after `stream`; `/Type/EmbeddedFile`
|
||||||
two embedded files; a PDF whose `/Length` is indirect.
|
without a space; hex and literal names and keywords; garbage inputs and
|
||||||
|
the 32-candidate cap; a 64 MiB object-stream bomb next to the real
|
||||||
|
attachment (report in milliseconds, RSS flat) and an `EmbeddedFile` bomb
|
||||||
|
(inflation cancelled at 16 MiB, `no_attachment`).
|
||||||
4. Docs: `docs/PDF-FORMAT.md` "Reading an archive" and "Known limits".
|
4. Docs: `docs/PDF-FORMAT.md` "Reading an archive" and "Known limits".
|
||||||
|
|
||||||
Estimated size: one executor phase, about 300 lines including tests.
|
Known trade-off: a stamped PDF saved before this change (object streams)
|
||||||
|
hides its `/Keywords`, so a stripped attachment on such a file reads as
|
||||||
|
`no_attachment` instead of `attachment_stripped`.
|
||||||
|
|
||||||
## 5. Prerequisites before external users (Step 7+ sync, Step 14 worker)
|
## 5. Prerequisites before external users (Step 7+ sync, Step 14 worker)
|
||||||
|
|
||||||
|
|||||||
@@ -1,105 +1,36 @@
|
|||||||
import {
|
|
||||||
PDFDict,
|
|
||||||
PDFHexString,
|
|
||||||
PDFName,
|
|
||||||
PDFRawStream,
|
|
||||||
PDFStream,
|
|
||||||
PDFString,
|
|
||||||
type PDFDocument,
|
|
||||||
} from 'pdf-lib'
|
|
||||||
import { ArchiveReadError } from '../errors'
|
import { ArchiveReadError } from '../errors'
|
||||||
import { inflateFlate, MAX_ATTACHMENT_BYTES } from './inflate'
|
import {
|
||||||
|
ARCHIVE_STAMP,
|
||||||
|
hasPdfHeader,
|
||||||
|
locateAttachment,
|
||||||
|
MAX_PDF_BYTES,
|
||||||
|
} from './scan'
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Finds the Faunosteo attachment of a loaded PDF: the Filespec whose file
|
* The raw JSON a Faunosteo PDF carries, read from the bytes alone (BLK-009):
|
||||||
* name starts with `osteo-report` first, else the first embedded file
|
* no PDF object graph is parsed, so no object stream is ever inflated. pdf-lib
|
||||||
* (today's behaviour, kept as the fallback). pdf-lib 1.17 exposes no
|
* stays on the export side.
|
||||||
* attachment API, so this walks the indirect objects.
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
export const ATTACHMENT_NAME_PREFIX = 'osteo-report'
|
export { ATTACHMENT_NAME_PREFIX } from './scan'
|
||||||
export const PDF_KEYWORD = 'faunosteo-archive/1'
|
export const PDF_KEYWORD = `${ARCHIVE_STAMP}/1`
|
||||||
|
|
||||||
const TYPE = PDFName.of('Type')
|
/** Size cap first, then the `%PDF-` header: anything else is `invalid_pdf`. */
|
||||||
const F = PDFName.of('F')
|
function assertPdfBytes(bytes: Uint8Array): void {
|
||||||
|
if (bytes.byteLength > MAX_PDF_BYTES || !hasPdfHeader(bytes)) {
|
||||||
function textOf(value: unknown): string | null {
|
throw new ArchiveReadError('invalid_pdf')
|
||||||
if (value instanceof PDFString || value instanceof PDFHexString) {
|
|
||||||
return value.decodeText()
|
|
||||||
}
|
}
|
||||||
return null
|
|
||||||
}
|
|
||||||
|
|
||||||
/** The embedded stream of a `/Filespec` named for a Faunosteo report. */
|
|
||||||
function namedFilespecStream(dict: PDFDict): PDFStream | null {
|
|
||||||
if (dict.get(TYPE) !== PDFName.of('Filespec')) return null
|
|
||||||
const name = textOf(dict.lookup(F))
|
|
||||||
if (name === null || !name.startsWith(ATTACHMENT_NAME_PREFIX)) return null
|
|
||||||
const files = dict.lookup(PDFName.of('EF'))
|
|
||||||
if (!(files instanceof PDFDict)) return null
|
|
||||||
const stream = files.lookup(F)
|
|
||||||
return stream instanceof PDFStream ? stream : null
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Today's heuristic: an `/EmbeddedFile` stream or a JSON-subtyped one. */
|
|
||||||
function looksLikeEmbeddedJson(stream: PDFStream): boolean {
|
|
||||||
const type = String(stream.dict.lookup(TYPE) ?? '')
|
|
||||||
const subtype = String(stream.dict.lookup(PDFName.of('Subtype')) ?? '')
|
|
||||||
return (
|
|
||||||
type.includes('EmbeddedFile') ||
|
|
||||||
subtype.includes('application/json') ||
|
|
||||||
subtype.includes('application#2Fjson')
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
function locateAttachment(pdfDoc: PDFDocument): PDFStream | null {
|
|
||||||
const objects = pdfDoc.context.enumerateIndirectObjects()
|
|
||||||
for (const [, object] of objects) {
|
|
||||||
if (!(object instanceof PDFDict)) continue
|
|
||||||
const stream = namedFilespecStream(object)
|
|
||||||
if (stream) return stream
|
|
||||||
}
|
|
||||||
for (const [, object] of objects) {
|
|
||||||
if (object instanceof PDFStream && looksLikeEmbeddedJson(object)) {
|
|
||||||
return object
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return null
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Stream bytes, inflated when Flate-encoded; null when unreadable or over the cap. */
|
|
||||||
async function attachmentBytes(stream: PDFStream): Promise<Uint8Array | null> {
|
|
||||||
const contents =
|
|
||||||
stream instanceof PDFRawStream ? stream.contents : stream.getContents()
|
|
||||||
if (contents.byteLength > MAX_ATTACHMENT_BYTES) return null
|
|
||||||
const filter = String(stream.dict.lookup(PDFName.of('Filter')) ?? '')
|
|
||||||
if (!filter.includes('FlateDecode')) return contents
|
|
||||||
return inflateFlate(contents)
|
|
||||||
}
|
|
||||||
|
|
||||||
/** A PDF stamped as an archive but holding no attachment lost it in a viewer. */
|
|
||||||
function missingAttachmentError(pdfDoc: PDFDocument): ArchiveReadError {
|
|
||||||
const stamped = (pdfDoc.getKeywords() ?? '').includes('faunosteo-archive')
|
|
||||||
return new ArchiveReadError(stamped ? 'attachment_stripped' : 'no_attachment')
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* The raw JSON carried by the PDF, whatever its shape (bare report of any
|
* The raw JSON carried by the PDF, whatever its shape (bare report of any
|
||||||
* version or archive envelope), before any validation: what the archive
|
* version or archive envelope), before any validation: what the archive
|
||||||
* check tool shows next to the read result. Throws ArchiveReadError.
|
* check tool shows next to the read result. Throws ArchiveReadError
|
||||||
|
* (`invalid_pdf`, `no_attachment`, `attachment_stripped`).
|
||||||
*/
|
*/
|
||||||
export async function readAttachedDocument(
|
export async function readAttachedDocument(
|
||||||
pdfDoc: PDFDocument,
|
bytes: Uint8Array,
|
||||||
): Promise<unknown> {
|
): Promise<unknown> {
|
||||||
const stream = locateAttachment(pdfDoc)
|
assertPdfBytes(bytes)
|
||||||
if (!stream) throw missingAttachmentError(pdfDoc)
|
return (await locateAttachment(bytes)).data
|
||||||
const bytes = await attachmentBytes(stream)
|
|
||||||
if (!bytes || bytes.byteLength > MAX_ATTACHMENT_BYTES) {
|
|
||||||
throw new ArchiveReadError('no_attachment')
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
return JSON.parse(new TextDecoder().decode(bytes)) as unknown
|
|
||||||
} catch {
|
|
||||||
throw new ArchiveReadError('no_attachment')
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
@@ -0,0 +1,441 @@
|
|||||||
|
import { ArchiveReadError } from '../errors'
|
||||||
|
import { inflateFlate, MAX_ATTACHMENT_BYTES } from './inflate'
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Locates the Faunosteo attachment by scanning the PDF bytes (BLK-009): a
|
||||||
|
* stream object can never live inside an object stream, so every
|
||||||
|
* `/Type /EmbeddedFile` dictionary and its data are in clear in the file.
|
||||||
|
* Nothing here parses the object graph, the xref or an `/ObjStm`, so the
|
||||||
|
* only inflation on the import path is the capped one of `inflateFlate`.
|
||||||
|
* The scanner never throws on garbage: it returns fewer candidates.
|
||||||
|
*/
|
||||||
|
|
||||||
|
/** A Faunosteo PDF weighs a few hundred KB; anything larger is not one. */
|
||||||
|
export const MAX_PDF_BYTES = 50 * 1024 * 1024
|
||||||
|
|
||||||
|
export const ATTACHMENT_NAME_PREFIX = 'osteo-report'
|
||||||
|
/** The keyword stamp every archive carries (`faunosteo-archive/<n>`). */
|
||||||
|
export const ARCHIVE_STAMP = 'faunosteo-archive'
|
||||||
|
|
||||||
|
/** Candidates beyond this count are ignored: a crafted file cannot make the reader loop. */
|
||||||
|
const MAX_CANDIDATES = 32
|
||||||
|
/** A stream or Filespec dictionary is small; a longer one is skipped. */
|
||||||
|
const MAX_DICT_BYTES = 4096
|
||||||
|
/** Window read before a dictionary for its `n 0 obj` header. */
|
||||||
|
const HEADER_WINDOW = 48
|
||||||
|
|
||||||
|
export interface EmbeddedFileCandidate {
|
||||||
|
/** First byte of the stream data. */
|
||||||
|
offset: number
|
||||||
|
/** Byte count of the stream data (declared, or measured to `endstream`). */
|
||||||
|
length: number
|
||||||
|
filter: 'FlateDecode' | null
|
||||||
|
/** From a visible `/Filespec` whose `/EF /F` points at this stream. */
|
||||||
|
name: string | null
|
||||||
|
}
|
||||||
|
|
||||||
|
const encode = (text: string): Uint8Array =>
|
||||||
|
Uint8Array.from(text, (c) => c.charCodeAt(0))
|
||||||
|
|
||||||
|
const TYPE = encode('/Type')
|
||||||
|
const EMBEDDED_FILE = encode('/EmbeddedFile')
|
||||||
|
const FILESPEC = encode('/Filespec')
|
||||||
|
const KEYWORDS = encode('/Keywords')
|
||||||
|
const STREAM = encode('stream')
|
||||||
|
const ENDSTREAM = encode('endstream')
|
||||||
|
const PDF_HEADER = encode('%PDF-')
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Byte helpers: every loop is bounded by `end` (at most MAX_PDF_BYTES)
|
||||||
|
|
||||||
|
function isWhitespace(byte: number): boolean {
|
||||||
|
return (
|
||||||
|
byte === 0x20 ||
|
||||||
|
byte === 0x0a ||
|
||||||
|
byte === 0x0d ||
|
||||||
|
byte === 0x09 ||
|
||||||
|
byte === 0x0c ||
|
||||||
|
byte === 0x00
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
/** PDF delimiters and whitespace end a name token. */
|
||||||
|
function isDelimiter(byte: number): boolean {
|
||||||
|
return isWhitespace(byte) || '()<>[]{}/%'.includes(String.fromCharCode(byte))
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Naive forward search; the needle's first byte is the fast reject. */
|
||||||
|
function indexOf(
|
||||||
|
bytes: Uint8Array,
|
||||||
|
needle: Uint8Array,
|
||||||
|
from: number,
|
||||||
|
end: number,
|
||||||
|
): number {
|
||||||
|
const last = end - needle.length
|
||||||
|
const first = needle[0]!
|
||||||
|
for (let i = Math.max(from, 0); i <= last; i += 1) {
|
||||||
|
if (bytes[i] !== first) continue
|
||||||
|
let j = 1
|
||||||
|
while (j < needle.length && bytes[i + j] === needle[j]) j += 1
|
||||||
|
if (j === needle.length) return i
|
||||||
|
}
|
||||||
|
return -1
|
||||||
|
}
|
||||||
|
|
||||||
|
function skipWhitespace(bytes: Uint8Array, from: number, end: number): number {
|
||||||
|
let i = from
|
||||||
|
while (i < end && isWhitespace(bytes[i]!)) i += 1
|
||||||
|
return i
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Latin-1 view of a small window (never the whole file). */
|
||||||
|
function latin1(bytes: Uint8Array, start: number, end: number): string {
|
||||||
|
let out = ''
|
||||||
|
for (let i = start; i < end; i += 1) out += String.fromCharCode(bytes[i]!)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
/** UTF-16BE with BOM (how pdf-lib writes text strings) or Latin-1 bytes. */
|
||||||
|
function decodeTextBytes(raw: number[]): string {
|
||||||
|
if (raw.length >= 2 && raw[0] === 0xfe && raw[1] === 0xff) {
|
||||||
|
let out = ''
|
||||||
|
for (let i = 2; i + 1 < raw.length; i += 2) {
|
||||||
|
out += String.fromCharCode((raw[i]! << 8) | raw[i + 1]!)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
return String.fromCharCode(...raw)
|
||||||
|
}
|
||||||
|
|
||||||
|
/** A `(literal)` or `<hex>` string token as text; null for anything else. */
|
||||||
|
export function decodePdfString(token: string): string | null {
|
||||||
|
if (token.startsWith('<') && token.endsWith('>')) {
|
||||||
|
const hex = token.slice(1, -1).replace(/[^0-9a-fA-F]/g, '')
|
||||||
|
const raw: number[] = []
|
||||||
|
for (let i = 0; i + 1 < hex.length; i += 2) {
|
||||||
|
raw.push(parseInt(hex.slice(i, i + 2), 16))
|
||||||
|
}
|
||||||
|
return decodeTextBytes(raw)
|
||||||
|
}
|
||||||
|
if (token.startsWith('(') && token.endsWith(')')) {
|
||||||
|
const body = token.slice(1, -1).replace(/\\([()\\])/g, '$1')
|
||||||
|
return decodeTextBytes(Array.from(body, (c) => c.charCodeAt(0) & 0xff))
|
||||||
|
}
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Dictionaries: `/Type /Name` matches and their `<< … >>` bounds
|
||||||
|
|
||||||
|
/** Every `/Type /<name>` (space optional) whose name ends on a delimiter. */
|
||||||
|
function findTypedDicts(
|
||||||
|
bytes: Uint8Array,
|
||||||
|
name: Uint8Array,
|
||||||
|
end: number,
|
||||||
|
): number[] {
|
||||||
|
const hits: number[] = []
|
||||||
|
let from = 0
|
||||||
|
while (hits.length < MAX_CANDIDATES) {
|
||||||
|
const at = indexOf(bytes, name, from, end)
|
||||||
|
if (at < 0) break
|
||||||
|
from = at + name.length
|
||||||
|
const after = at + name.length
|
||||||
|
if (after < end && !isDelimiter(bytes[after]!)) continue
|
||||||
|
let before = at
|
||||||
|
while (before > 0 && isWhitespace(bytes[before - 1]!)) before -= 1
|
||||||
|
const typeAt = before - TYPE.length
|
||||||
|
if (typeAt >= 0 && indexOf(bytes, TYPE, typeAt, before) === typeAt) {
|
||||||
|
hits.push(typeAt)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return hits
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Index of the `<<` opening the dictionary holding `pos` (nested dicts skipped); -1 when none. */
|
||||||
|
function dictStart(bytes: Uint8Array, pos: number): number {
|
||||||
|
const floor = Math.max(0, pos - MAX_DICT_BYTES)
|
||||||
|
let depth = 0
|
||||||
|
let i = pos
|
||||||
|
while (i - 2 >= floor) {
|
||||||
|
const pair = (bytes[i - 2]! << 8) | bytes[i - 1]!
|
||||||
|
if (pair === 0x3e3e)
|
||||||
|
depth += 1 // >>
|
||||||
|
else if (pair === 0x3c3c) {
|
||||||
|
if (depth === 0) return i - 2
|
||||||
|
depth -= 1
|
||||||
|
}
|
||||||
|
i -= pair === 0x3e3e || pair === 0x3c3c ? 2 : 1
|
||||||
|
}
|
||||||
|
return -1
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Index just past the `>>` closing the dictionary opened at `start`; -1 when none. */
|
||||||
|
function dictEnd(bytes: Uint8Array, start: number, end: number): number {
|
||||||
|
const limit = Math.min(end, start + MAX_DICT_BYTES)
|
||||||
|
let depth = 0
|
||||||
|
let i = start
|
||||||
|
while (i + 1 < limit) {
|
||||||
|
const pair = (bytes[i]! << 8) | bytes[i + 1]!
|
||||||
|
if (pair === 0x3c3c) depth += 1
|
||||||
|
else if (pair === 0x3e3e) {
|
||||||
|
depth -= 1
|
||||||
|
if (depth === 0) return i + 2
|
||||||
|
}
|
||||||
|
i += pair === 0x3c3c || pair === 0x3e3e ? 2 : 1
|
||||||
|
}
|
||||||
|
return -1
|
||||||
|
}
|
||||||
|
|
||||||
|
interface DictSpan {
|
||||||
|
start: number
|
||||||
|
end: number
|
||||||
|
text: string
|
||||||
|
objectNumber: number | null
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The dictionary around a `/Type` match, its text and the `n 0 obj` header before it. */
|
||||||
|
function dictAround(
|
||||||
|
bytes: Uint8Array,
|
||||||
|
pos: number,
|
||||||
|
end: number,
|
||||||
|
): DictSpan | null {
|
||||||
|
const start = dictStart(bytes, pos)
|
||||||
|
if (start < 0) return null
|
||||||
|
const close = dictEnd(bytes, start, end)
|
||||||
|
if (close < 0) return null
|
||||||
|
const header = latin1(bytes, Math.max(0, start - HEADER_WINDOW), start)
|
||||||
|
const match = /(\d{1,10})\s+\d{1,10}\s+obj\s*$/.exec(header)
|
||||||
|
return {
|
||||||
|
start,
|
||||||
|
end: close,
|
||||||
|
text: latin1(bytes, start, close),
|
||||||
|
objectNumber: match ? Number(match[1]) : null,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Stream data bounds
|
||||||
|
|
||||||
|
/** Data offset after `stream` + `\r\n` or `\n`; -1 when the keyword is missing. */
|
||||||
|
function streamDataOffset(
|
||||||
|
bytes: Uint8Array,
|
||||||
|
dictClose: number,
|
||||||
|
end: number,
|
||||||
|
): number {
|
||||||
|
const at = skipWhitespace(bytes, dictClose, end)
|
||||||
|
if (indexOf(bytes, STREAM, at, Math.min(end, at + STREAM.length)) !== at) {
|
||||||
|
return -1
|
||||||
|
}
|
||||||
|
let i = at + STREAM.length
|
||||||
|
if (bytes[i] === 0x0d) i += 1
|
||||||
|
if (bytes[i] !== 0x0a) return -1
|
||||||
|
return i + 1
|
||||||
|
}
|
||||||
|
|
||||||
|
/** A direct `/Length N` (never an `N 0 R` reference), when it lands on `endstream`. */
|
||||||
|
function declaredLength(
|
||||||
|
bytes: Uint8Array,
|
||||||
|
dict: string,
|
||||||
|
offset: number,
|
||||||
|
end: number,
|
||||||
|
): number | null {
|
||||||
|
const match = /\/Length\s+(\d{1,10})(?!\d)(?!\s+\d+\s+R)/.exec(dict)
|
||||||
|
if (!match) return null
|
||||||
|
const length = Number(match[1])
|
||||||
|
if (offset + length > end) return null
|
||||||
|
const tail = skipWhitespace(bytes, offset + length, end)
|
||||||
|
const found = indexOf(
|
||||||
|
bytes,
|
||||||
|
ENDSTREAM,
|
||||||
|
tail,
|
||||||
|
Math.min(end, tail + ENDSTREAM.length),
|
||||||
|
)
|
||||||
|
return found === tail ? length : null
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Fallback for an indirect or wrong `/Length`: up to `endstream`, the EOL before it dropped. */
|
||||||
|
function measuredLength(
|
||||||
|
bytes: Uint8Array,
|
||||||
|
offset: number,
|
||||||
|
end: number,
|
||||||
|
): number | null {
|
||||||
|
const at = indexOf(bytes, ENDSTREAM, offset, end)
|
||||||
|
if (at < 0) return null
|
||||||
|
let stop = at
|
||||||
|
if (stop > offset && bytes[stop - 1] === 0x0a) stop -= 1
|
||||||
|
if (stop > offset && bytes[stop - 1] === 0x0d) stop -= 1
|
||||||
|
return stop - offset
|
||||||
|
}
|
||||||
|
|
||||||
|
function filterOf(dict: string): EmbeddedFileCandidate['filter'] {
|
||||||
|
return /\/Filter\s*(\[\s*)?\/FlateDecode\b/.test(dict) ? 'FlateDecode' : null
|
||||||
|
}
|
||||||
|
|
||||||
|
function candidateOf(
|
||||||
|
bytes: Uint8Array,
|
||||||
|
dict: DictSpan,
|
||||||
|
end: number,
|
||||||
|
): Omit<EmbeddedFileCandidate, 'name'> | null {
|
||||||
|
const offset = streamDataOffset(bytes, dict.end, end)
|
||||||
|
if (offset < 0) return null
|
||||||
|
const length =
|
||||||
|
declaredLength(bytes, dict.text, offset, end) ??
|
||||||
|
measuredLength(bytes, offset, end)
|
||||||
|
if (length === null) return null
|
||||||
|
return { offset, length, filter: filterOf(dict.text) }
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Filespec names and the keyword stamp
|
||||||
|
|
||||||
|
const STRING_TOKEN = String.raw`\((?:\\.|[^\\)])*\)|<[0-9a-fA-F\s]*>`
|
||||||
|
|
||||||
|
/** `/F` (else `/UF`) of a Filespec dictionary, the `/EF` sub-dictionary set aside. */
|
||||||
|
function filespecName(dict: string): string | null {
|
||||||
|
const withoutEf = dict.replace(/\/EF\s*<<[\s\S]*?>>/, '')
|
||||||
|
const f = new RegExp(String.raw`/F\s*(${STRING_TOKEN})`).exec(withoutEf)
|
||||||
|
const uf = new RegExp(String.raw`/UF\s*(${STRING_TOKEN})`).exec(withoutEf)
|
||||||
|
const token = f?.[1] ?? uf?.[1]
|
||||||
|
return token ? decodePdfString(token) : null
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Object number of the stream a Filespec's `/EF << /F n 0 R >>` points at. */
|
||||||
|
function filespecStreamObject(dict: string): number | null {
|
||||||
|
const ef = /\/EF\s*<<([\s\S]*?)>>/.exec(dict)
|
||||||
|
const ref = ef ? /\/F\s+(\d{1,10})\s+\d{1,10}\s+R/.exec(ef[1]!) : null
|
||||||
|
return ref ? Number(ref[1]) : null
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Stream object number → file name, from every Filespec dictionary in clear. */
|
||||||
|
function filespecNames(bytes: Uint8Array, end: number): Map<number, string> {
|
||||||
|
const names = new Map<number, string>()
|
||||||
|
for (const pos of findTypedDicts(bytes, FILESPEC, end)) {
|
||||||
|
const dict = dictAround(bytes, pos, end)
|
||||||
|
if (!dict) continue
|
||||||
|
const target = filespecStreamObject(dict.text)
|
||||||
|
const name = filespecName(dict.text)
|
||||||
|
if (target !== null && name !== null && !names.has(target)) {
|
||||||
|
names.set(target, name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return names
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The string token right after a `/Keywords` key, as text. */
|
||||||
|
function keywordsAt(
|
||||||
|
bytes: Uint8Array,
|
||||||
|
keyAt: number,
|
||||||
|
end: number,
|
||||||
|
): string | null {
|
||||||
|
const start = skipWhitespace(bytes, keyAt + KEYWORDS.length, end)
|
||||||
|
const limit = Math.min(end, start + MAX_DICT_BYTES)
|
||||||
|
const window = latin1(bytes, start, limit)
|
||||||
|
const token = new RegExp(`^(?:${STRING_TOKEN})`).exec(window)
|
||||||
|
return token ? decodePdfString(token[0]) : null
|
||||||
|
}
|
||||||
|
|
||||||
|
/** True when a visible `/Keywords` string carries the archive stamp. */
|
||||||
|
export function hasArchiveStamp(bytes: Uint8Array): boolean {
|
||||||
|
const end = Math.min(bytes.byteLength, MAX_PDF_BYTES)
|
||||||
|
let from = 0
|
||||||
|
for (let n = 0; n < MAX_CANDIDATES; n += 1) {
|
||||||
|
const at = indexOf(bytes, KEYWORDS, from, end)
|
||||||
|
if (at < 0) return false
|
||||||
|
from = at + KEYWORDS.length
|
||||||
|
if (isDelimiter(bytes[from] ?? 0x20)) {
|
||||||
|
if (keywordsAt(bytes, at, end)?.includes(ARCHIVE_STAMP)) return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
/** `%PDF-` within the first KiB, as viewers accept it. */
|
||||||
|
export function hasPdfHeader(bytes: Uint8Array): boolean {
|
||||||
|
return indexOf(bytes, PDF_HEADER, 0, Math.min(bytes.byteLength, 1024)) >= 0
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Public scan
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Every `/Type /EmbeddedFile` stream of the file, named when its Filespec
|
||||||
|
* is in clear (export without object streams); `osteo-report*` names first,
|
||||||
|
* then file order. At most MAX_CANDIDATES, within MAX_PDF_BYTES.
|
||||||
|
*/
|
||||||
|
export function findEmbeddedFiles(bytes: Uint8Array): EmbeddedFileCandidate[] {
|
||||||
|
const end = Math.min(bytes.byteLength, MAX_PDF_BYTES)
|
||||||
|
const names = filespecNames(bytes, end)
|
||||||
|
const found: EmbeddedFileCandidate[] = []
|
||||||
|
for (const pos of findTypedDicts(bytes, EMBEDDED_FILE, end)) {
|
||||||
|
const dict = dictAround(bytes, pos, end)
|
||||||
|
if (!dict) continue
|
||||||
|
const candidate = candidateOf(bytes, dict, end)
|
||||||
|
if (!candidate) continue
|
||||||
|
const name =
|
||||||
|
dict.objectNumber === null ? null : (names.get(dict.objectNumber) ?? null)
|
||||||
|
found.push({ ...candidate, name })
|
||||||
|
}
|
||||||
|
const named = (c: EmbeddedFileCandidate) =>
|
||||||
|
c.name?.startsWith(ATTACHMENT_NAME_PREFIX) ? 0 : 1
|
||||||
|
return found.sort((a, b) => named(a) - named(b))
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Raw or inflated stream data under the attachment cap; null when unusable. */
|
||||||
|
async function candidateBytes(
|
||||||
|
bytes: Uint8Array,
|
||||||
|
candidate: EmbeddedFileCandidate,
|
||||||
|
): Promise<Uint8Array | null> {
|
||||||
|
if (candidate.length > MAX_ATTACHMENT_BYTES) return null
|
||||||
|
const data = bytes.subarray(
|
||||||
|
candidate.offset,
|
||||||
|
candidate.offset + candidate.length,
|
||||||
|
)
|
||||||
|
if (candidate.filter === null) return data
|
||||||
|
return inflateFlate(data)
|
||||||
|
}
|
||||||
|
|
||||||
|
/** JSON with a numeric `envelopeVersion` or `version`; undefined otherwise. */
|
||||||
|
function parseVersioned(data: Uint8Array): unknown {
|
||||||
|
let parsed: unknown
|
||||||
|
try {
|
||||||
|
parsed = JSON.parse(new TextDecoder().decode(data))
|
||||||
|
} catch {
|
||||||
|
return undefined
|
||||||
|
}
|
||||||
|
if (typeof parsed !== 'object' || parsed === null) return undefined
|
||||||
|
const record = parsed as Record<string, unknown>
|
||||||
|
const versioned =
|
||||||
|
typeof record['envelopeVersion'] === 'number' ||
|
||||||
|
typeof record['version'] === 'number'
|
||||||
|
return versioned ? parsed : undefined
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface LocatedAttachment {
|
||||||
|
bytes: Uint8Array
|
||||||
|
data: unknown
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The first candidate whose data is a versioned JSON document, decoded once.
|
||||||
|
* None: `attachment_stripped` when the stamp is visible, else `no_attachment`.
|
||||||
|
*/
|
||||||
|
export async function locateAttachment(
|
||||||
|
bytes: Uint8Array,
|
||||||
|
): Promise<LocatedAttachment> {
|
||||||
|
for (const candidate of findEmbeddedFiles(bytes)) {
|
||||||
|
const data = await candidateBytes(bytes, candidate)
|
||||||
|
if (!data) continue
|
||||||
|
const parsed = parseVersioned(data)
|
||||||
|
if (parsed !== undefined) return { bytes: data, data: parsed }
|
||||||
|
}
|
||||||
|
throw new ArchiveReadError(
|
||||||
|
hasArchiveStamp(bytes) ? 'attachment_stripped' : 'no_attachment',
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The attachment's decoded bytes (BLK-009 reader). Throws ArchiveReadError. */
|
||||||
|
export async function readAttachmentBytes(
|
||||||
|
bytes: Uint8Array,
|
||||||
|
): Promise<Uint8Array> {
|
||||||
|
return (await locateAttachment(bytes)).bytes
|
||||||
|
}
|
||||||
@@ -201,7 +201,10 @@ export async function exportReportArchive(
|
|||||||
warnings,
|
warnings,
|
||||||
})
|
})
|
||||||
await attachEnvelope(pdfDoc, attachment, input.exportedAt)
|
await attachEnvelope(pdfDoc, attachment, input.exportedAt)
|
||||||
return { pdf: await pdfDoc.save(), envelope, warnings }
|
// No object streams (BLK-009): the Filespec name and the keyword stamp stay
|
||||||
|
// in clear for the byte-scan reader; the file grows by a few KB.
|
||||||
|
const pdf = await pdfDoc.save({ useObjectStreams: false })
|
||||||
|
return { pdf, envelope, warnings }
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface ReexportOptions {
|
export interface ReexportOptions {
|
||||||
|
|||||||
@@ -4,6 +4,11 @@
|
|||||||
export { exportReportArchive, exportFromEnvelope } from './export'
|
export { exportReportArchive, exportFromEnvelope } from './export'
|
||||||
export { readReportArchive, MAX_PDF_BYTES } from './read'
|
export { readReportArchive, MAX_PDF_BYTES } from './read'
|
||||||
export { readAttachedDocument, PDF_KEYWORD } from './attachment/read'
|
export { readAttachedDocument, PDF_KEYWORD } from './attachment/read'
|
||||||
|
export {
|
||||||
|
findEmbeddedFiles,
|
||||||
|
readAttachmentBytes,
|
||||||
|
type EmbeddedFileCandidate,
|
||||||
|
} from './attachment/scan'
|
||||||
export { MAX_ATTACHMENT_BYTES } from './attachment/inflate'
|
export { MAX_ATTACHMENT_BYTES } from './attachment/inflate'
|
||||||
export {
|
export {
|
||||||
ArchiveReadError,
|
ArchiveReadError,
|
||||||
|
|||||||
@@ -1,4 +1,3 @@
|
|||||||
import { PDFDocument } from 'pdf-lib'
|
|
||||||
import {
|
import {
|
||||||
readArchiveDocument,
|
readArchiveDocument,
|
||||||
UnsupportedReportVersionError,
|
UnsupportedReportVersionError,
|
||||||
@@ -8,23 +7,12 @@ import {
|
|||||||
import { readAttachedDocument } from './attachment/read'
|
import { readAttachedDocument } from './attachment/read'
|
||||||
import { ArchiveReadError } from './errors'
|
import { ArchiveReadError } from './errors'
|
||||||
|
|
||||||
/** A Faunosteo PDF weighs a few hundred KB; anything larger is not one. */
|
export { MAX_PDF_BYTES } from './attachment/scan'
|
||||||
export const MAX_PDF_BYTES = 50 * 1024 * 1024
|
|
||||||
|
|
||||||
async function loadPdf(pdfBytes: Uint8Array): Promise<PDFDocument> {
|
|
||||||
if (pdfBytes.byteLength > MAX_PDF_BYTES) {
|
|
||||||
throw new ArchiveReadError('invalid_pdf')
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
return await PDFDocument.load(pdfBytes, { ignoreEncryption: true })
|
|
||||||
} catch {
|
|
||||||
throw new ArchiveReadError('invalid_pdf')
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Reads a Faunosteo PDF back: size cap, PDF parse, attachment (filename
|
* Reads a Faunosteo PDF back: size cap and `%PDF-` header (`invalid_pdf`),
|
||||||
* first), JSON, then the layered reading of the domain (report first, then
|
* attachment located by a byte scan (filename first, never a PDF parse:
|
||||||
|
* BLK-009), JSON, then the layered reading of the domain (report first, then
|
||||||
* the envelope accepted whole or dropped with a reason). An unknown report
|
* the envelope accepted whole or dropped with a reason). An unknown report
|
||||||
* version is `incompatible_version` with that version; any other content
|
* version is `incompatible_version` with that version; any other content
|
||||||
* failure is `no_attachment`, as today.
|
* failure is `no_attachment`, as today.
|
||||||
@@ -33,8 +21,7 @@ export async function readReportArchive(
|
|||||||
pdfBytes: Uint8Array,
|
pdfBytes: Uint8Array,
|
||||||
map: ReportMigrationMap,
|
map: ReportMigrationMap,
|
||||||
): Promise<ArchiveReadOutcome> {
|
): Promise<ArchiveReadOutcome> {
|
||||||
const pdfDoc = await loadPdf(pdfBytes)
|
const data = await readAttachedDocument(pdfBytes)
|
||||||
const data = await readAttachedDocument(pdfDoc)
|
|
||||||
try {
|
try {
|
||||||
return readArchiveDocument(data, map)
|
return readArchiveDocument(data, map)
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
|
|||||||
@@ -1,17 +1,29 @@
|
|||||||
import { AFRelationship, PDFDocument, PDFName, PDFRawStream } from 'pdf-lib'
|
import { AFRelationship, PDFDocument } from 'pdf-lib'
|
||||||
|
import { findEmbeddedFiles } from '../../src'
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Crafted PDFs for the read tests, built with pdf-lib's own attach. Saved
|
||||||
|
* without object streams by default, like the export since BLK-009; pass
|
||||||
|
* `objectStreams: true` for the pre-BLK-009 layout (Filespec and Info
|
||||||
|
* packed into an `/ObjStm`, invisible to the byte scan).
|
||||||
|
*/
|
||||||
|
export interface CraftOptions {
|
||||||
|
keywords?: string[]
|
||||||
|
before?: [string, unknown]
|
||||||
|
objectStreams?: boolean
|
||||||
|
}
|
||||||
|
|
||||||
/** Crafted PDFs for the read tests, built with pdf-lib's own attach. */
|
|
||||||
export async function pdfWithAttachment(
|
export async function pdfWithAttachment(
|
||||||
name: string,
|
name: string,
|
||||||
payload: unknown,
|
payload: unknown,
|
||||||
extra: { keywords?: string[]; before?: [string, unknown] } = {},
|
extra: CraftOptions = {},
|
||||||
): Promise<Uint8Array> {
|
): Promise<Uint8Array> {
|
||||||
const doc = await PDFDocument.create()
|
const doc = await PDFDocument.create()
|
||||||
doc.addPage()
|
doc.addPage()
|
||||||
if (extra.keywords) doc.setKeywords(extra.keywords)
|
if (extra.keywords) doc.setKeywords(extra.keywords)
|
||||||
if (extra.before) await attachJson(doc, extra.before[0], extra.before[1])
|
if (extra.before) await attachJson(doc, extra.before[0], extra.before[1])
|
||||||
await attachJson(doc, name, payload)
|
await attachJson(doc, name, payload)
|
||||||
return doc.save()
|
return doc.save({ useObjectStreams: extra.objectStreams ?? false })
|
||||||
}
|
}
|
||||||
|
|
||||||
async function attachJson(
|
async function attachJson(
|
||||||
@@ -27,20 +39,17 @@ async function attachJson(
|
|||||||
|
|
||||||
export async function pdfWithoutAttachment(
|
export async function pdfWithoutAttachment(
|
||||||
keywords?: string[],
|
keywords?: string[],
|
||||||
|
objectStreams = false,
|
||||||
): Promise<Uint8Array> {
|
): Promise<Uint8Array> {
|
||||||
const doc = await PDFDocument.create()
|
const doc = await PDFDocument.create()
|
||||||
doc.addPage()
|
doc.addPage()
|
||||||
if (keywords) doc.setKeywords(keywords)
|
if (keywords) doc.setKeywords(keywords)
|
||||||
return doc.save()
|
return doc.save({ useObjectStreams: objectStreams })
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Stored (Flate) size of the first `/EmbeddedFile` stream of a saved PDF. */
|
/** Stored (Flate) size of the first `/EmbeddedFile` stream of a saved PDF, by the byte scan. */
|
||||||
export async function storedAttachmentSize(bytes: Uint8Array): Promise<number> {
|
export function storedAttachmentSize(bytes: Uint8Array): number {
|
||||||
const doc = await PDFDocument.load(bytes)
|
const [first] = findEmbeddedFiles(bytes)
|
||||||
for (const [, object] of doc.context.enumerateIndirectObjects()) {
|
if (!first) throw new Error('no EmbeddedFile stream')
|
||||||
if (!(object instanceof PDFRawStream)) continue
|
return first.length
|
||||||
const type = object.dict.lookup(PDFName.of('Type'))
|
|
||||||
if (type === PDFName.of('EmbeddedFile')) return object.contents.length
|
|
||||||
}
|
|
||||||
throw new Error('no EmbeddedFile stream')
|
|
||||||
}
|
}
|
||||||
@@ -149,6 +149,11 @@ describe('attachment presence and selection', () => {
|
|||||||
await rejects(bytes, { kind: 'attachment_stripped' })
|
await rejects(bytes, { kind: 'attachment_stripped' })
|
||||||
})
|
})
|
||||||
|
|
||||||
|
it('a stamped PDF saved with object streams (pre-BLK-009 layout): the stamp is invisible, no_attachment', async () => {
|
||||||
|
const bytes = await pdfWithoutAttachment(['faunosteo-archive/1'], true)
|
||||||
|
await rejects(bytes, { kind: 'no_attachment' })
|
||||||
|
})
|
||||||
|
|
||||||
it('a foreign PDF: no_attachment', async () => {
|
it('a foreign PDF: no_attachment', async () => {
|
||||||
await rejects(await pdfWithoutAttachment(), { kind: 'no_attachment' })
|
await rejects(await pdfWithoutAttachment(), { kind: 'no_attachment' })
|
||||||
const foreign = await pdfWithAttachment('other.json', { hello: 'world' })
|
const foreign = await pdfWithAttachment('other.json', { hello: 'world' })
|
||||||
@@ -166,10 +171,13 @@ describe('attachment presence and selection', () => {
|
|||||||
expect(outcome.report).toEqual(golden.reports.demo)
|
expect(outcome.report).toEqual(golden.reports.demo)
|
||||||
})
|
})
|
||||||
|
|
||||||
it('refuses an oversized or unparseable PDF as invalid_pdf', async () => {
|
it('refuses an oversized file or one without a %PDF- header as invalid_pdf', async () => {
|
||||||
await rejects(new Uint8Array(MAX_PDF_BYTES + 1), { kind: 'invalid_pdf' })
|
await rejects(new Uint8Array(MAX_PDF_BYTES + 1), { kind: 'invalid_pdf' })
|
||||||
await rejects(new TextEncoder().encode('not a pdf at all'), {
|
await rejects(new TextEncoder().encode('not a pdf at all'), {
|
||||||
kind: 'invalid_pdf',
|
kind: 'invalid_pdf',
|
||||||
})
|
})
|
||||||
|
const late = new Uint8Array(2048)
|
||||||
|
late.set(new TextEncoder().encode('%PDF-1.7'), 1024)
|
||||||
|
await rejects(late, { kind: 'invalid_pdf' })
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
@@ -0,0 +1,425 @@
|
|||||||
|
/**
|
||||||
|
* BLK-009: the byte-scan attachment reader. Candidates on a fresh export and
|
||||||
|
* on the frozen fixtures (object-stream layout), crafted layouts (two files,
|
||||||
|
* indirect `/Length`, `\r\n`, no space after `/Type`), garbage, the keyword
|
||||||
|
* stamp, and the object-stream bomb the reader never inflates.
|
||||||
|
*/
|
||||||
|
import { deflateSync } from 'node:zlib'
|
||||||
|
import { beforeAll, describe, expect, it } from 'vitest'
|
||||||
|
import { buildSheetSnapshot, type Report } from '@faunosteo/domain'
|
||||||
|
import {
|
||||||
|
ArchiveReadError,
|
||||||
|
defaultPdfLabels,
|
||||||
|
exportReportArchive,
|
||||||
|
findEmbeddedFiles,
|
||||||
|
readAttachedDocument,
|
||||||
|
readAttachmentBytes,
|
||||||
|
readReportArchive,
|
||||||
|
type EmbeddedFileCandidate,
|
||||||
|
} from '../src'
|
||||||
|
import { inflateFlate } from '../src/attachment/inflate'
|
||||||
|
import { decodePdfString, hasArchiveStamp } from '../src/attachment/scan'
|
||||||
|
import { readFixtureBytes } from './helpers/draft'
|
||||||
|
import {
|
||||||
|
FIXED_EXPORTED_AT,
|
||||||
|
nologosBranding,
|
||||||
|
rasterizerOf,
|
||||||
|
} from './helpers/goldens'
|
||||||
|
import { pdfWithAttachment, pdfWithoutAttachment } from './helpers/pdf'
|
||||||
|
import { loadGoldenReports, type GoldenReports } from './helpers/reports'
|
||||||
|
|
||||||
|
let golden: GoldenReports
|
||||||
|
let exported: Uint8Array
|
||||||
|
const map = () => golden.draft.migrationMap
|
||||||
|
|
||||||
|
beforeAll(async () => {
|
||||||
|
golden = loadGoldenReports()
|
||||||
|
exported = (await exportDemo()).pdf
|
||||||
|
})
|
||||||
|
|
||||||
|
async function exportDemo() {
|
||||||
|
const report: Report = golden.reports.demo
|
||||||
|
return exportReportArchive({
|
||||||
|
report,
|
||||||
|
sheet: buildSheetSnapshot(golden.draft, report),
|
||||||
|
branding: nologosBranding(),
|
||||||
|
labels: defaultPdfLabels,
|
||||||
|
locale: 'fr',
|
||||||
|
rasterizer: rasterizerOf('nologos'),
|
||||||
|
exportedAt: FIXED_EXPORTED_AT,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Hand-built PDFs: the scanner never reads the xref, so none is written
|
||||||
|
|
||||||
|
const latin1 = (bytes: Uint8Array) => Buffer.from(bytes).toString('latin1')
|
||||||
|
const ascii = (text: string) => new Uint8Array(Buffer.from(text, 'latin1'))
|
||||||
|
|
||||||
|
function concat(parts: Uint8Array[]): Uint8Array {
|
||||||
|
const out = new Uint8Array(parts.reduce((n, p) => n + p.byteLength, 0))
|
||||||
|
let at = 0
|
||||||
|
for (const part of parts) {
|
||||||
|
out.set(part, at)
|
||||||
|
at += part.byteLength
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
interface StreamSpec {
|
||||||
|
num: number
|
||||||
|
dict: string
|
||||||
|
data: Uint8Array
|
||||||
|
eol?: string
|
||||||
|
}
|
||||||
|
|
||||||
|
function streamObject({ num, dict, data, eol = '\n' }: StreamSpec): Uint8Array {
|
||||||
|
return concat([
|
||||||
|
ascii(`${num} 0 obj\n<<${dict}>>\nstream${eol}`),
|
||||||
|
data,
|
||||||
|
ascii('\nendstream\nendobj\n'),
|
||||||
|
])
|
||||||
|
}
|
||||||
|
|
||||||
|
function craftPdf(objects: Uint8Array[]): Uint8Array {
|
||||||
|
return concat([
|
||||||
|
ascii('%PDF-1.7\n'),
|
||||||
|
...objects,
|
||||||
|
ascii('trailer\n<< /Root 1 0 R >>\n%%EOF\n'),
|
||||||
|
])
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The `n 0 obj … endobj` bytes of the object whose dictionary holds `marker`. */
|
||||||
|
function objectHolding(pdf: Uint8Array, marker: string): Uint8Array {
|
||||||
|
const text = latin1(pdf)
|
||||||
|
const at = text.indexOf(marker)
|
||||||
|
expect(at).toBeGreaterThan(0)
|
||||||
|
const start = text.lastIndexOf(' 0 obj', at)
|
||||||
|
const headerStart = text.lastIndexOf('\n', start) + 1
|
||||||
|
const end = text.indexOf('endobj', at) + 'endobj'.length
|
||||||
|
return pdf.slice(headerStart, end)
|
||||||
|
}
|
||||||
|
|
||||||
|
const BOMB_RAW = 64 * 1024 * 1024
|
||||||
|
const bomb = () => new Uint8Array(deflateSync(new Uint8Array(BOMB_RAW)))
|
||||||
|
|
||||||
|
async function rejects(
|
||||||
|
bytes: Uint8Array,
|
||||||
|
kind: ArchiveReadError['kind'],
|
||||||
|
): Promise<void> {
|
||||||
|
const promise = readReportArchive(bytes, map())
|
||||||
|
await expect(promise).rejects.toBeInstanceOf(ArchiveReadError)
|
||||||
|
await expect(promise).rejects.toMatchObject({ kind })
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
describe('findEmbeddedFiles on a fresh export (no object streams)', () => {
|
||||||
|
let candidates: EmbeddedFileCandidate[]
|
||||||
|
beforeAll(() => {
|
||||||
|
candidates = findEmbeddedFiles(exported)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('finds the one Flate stream, named through its visible Filespec', async () => {
|
||||||
|
expect(candidates).toHaveLength(1)
|
||||||
|
const [c] = candidates
|
||||||
|
expect(c!.filter).toBe('FlateDecode')
|
||||||
|
expect(c!.name).toBe('osteo-report.json')
|
||||||
|
const inflated = await inflateFlate(
|
||||||
|
exported.subarray(c!.offset, c!.offset + c!.length),
|
||||||
|
)
|
||||||
|
const json = JSON.parse(new TextDecoder().decode(inflated!)) as {
|
||||||
|
envelopeVersion: number
|
||||||
|
}
|
||||||
|
expect(json.envelopeVersion).toBe(1)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('the length is the declared /Length and the data ends on endstream', () => {
|
||||||
|
const [c] = candidates
|
||||||
|
const text = latin1(exported)
|
||||||
|
expect(text).toContain(`/Length ${c!.length}\n>>\nstream\n`)
|
||||||
|
expect(text.slice(c!.offset + c!.length)).toMatch(/^\r?\nendstream/)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('the export keeps the Filespec, its name and the stamp in clear, no object stream', () => {
|
||||||
|
const text = latin1(exported)
|
||||||
|
expect(text).toContain('/Type /Filespec')
|
||||||
|
expect(text).toContain('/F (osteo-report.json)')
|
||||||
|
expect(text).not.toContain('/Type /ObjStm')
|
||||||
|
expect(hasArchiveStamp(exported)).toBe(true)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('readAttachmentBytes returns the inflated envelope bytes', async () => {
|
||||||
|
const bytes = await readAttachmentBytes(exported)
|
||||||
|
const json = JSON.parse(new TextDecoder().decode(bytes)) as {
|
||||||
|
report: Report
|
||||||
|
}
|
||||||
|
expect(json.report).toEqual(golden.reports.demo)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('compatibility: frozen fixtures written with object streams', () => {
|
||||||
|
it.each([
|
||||||
|
['report-v2.full.pdf', 'v2'],
|
||||||
|
['report-v2.minimal.pdf', 'v2'],
|
||||||
|
['report-v3.bare.pdf', 'bare_v3'],
|
||||||
|
] as const)('%s → origin %s', async (name, origin) => {
|
||||||
|
const bytes = readFixtureBytes(name)
|
||||||
|
expect(latin1(bytes)).toContain('/Type /ObjStm')
|
||||||
|
const candidates = findEmbeddedFiles(bytes)
|
||||||
|
expect(candidates).toHaveLength(1)
|
||||||
|
// The Filespec is packed into the object stream: no name, the JSON decides.
|
||||||
|
expect(candidates[0]!.name).toBeNull()
|
||||||
|
expect(candidates[0]!.filter).toBe('FlateDecode')
|
||||||
|
const outcome = await readReportArchive(bytes, map())
|
||||||
|
expect(outcome.origin).toBe(origin)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('a stamped PDF with object streams hides its stamp: no_attachment (documented)', async () => {
|
||||||
|
const bytes = await pdfWithoutAttachment(['faunosteo-archive/1'], true)
|
||||||
|
expect(hasArchiveStamp(bytes)).toBe(false)
|
||||||
|
await rejects(bytes, 'no_attachment')
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('selection between several embedded files', () => {
|
||||||
|
it('a decoy EmbeddedFile first, the named Faunosteo one second: the name wins', async () => {
|
||||||
|
const bytes = await pdfWithAttachment(
|
||||||
|
'osteo-report.json',
|
||||||
|
golden.reports.demo,
|
||||||
|
{
|
||||||
|
before: ['other.json', { hello: 'world', version: 2 }],
|
||||||
|
},
|
||||||
|
)
|
||||||
|
const names = findEmbeddedFiles(bytes).map((c) => c.name)
|
||||||
|
expect(names).toEqual(['osteo-report.json', 'other.json'])
|
||||||
|
const outcome = await readReportArchive(bytes, map())
|
||||||
|
expect(outcome.origin).toBe('bare_v3')
|
||||||
|
expect(outcome.report).toEqual(golden.reports.demo)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('an unnamed decoy without a version is skipped for the versioned JSON after it', async () => {
|
||||||
|
const decoy = streamObject({
|
||||||
|
num: 2,
|
||||||
|
dict: '/Type /EmbeddedFile /Length 17',
|
||||||
|
data: ascii('{"hello":"world"}'),
|
||||||
|
})
|
||||||
|
const real = objectHolding(exported, '/Type /EmbeddedFile')
|
||||||
|
const outcome = await readReportArchive(craftPdf([decoy, real]), map())
|
||||||
|
expect(outcome.origin).toBe('envelope_v1')
|
||||||
|
expect(outcome.report).toEqual(golden.reports.demo)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('a foreign PDF, with or without attachments: no_attachment', async () => {
|
||||||
|
await rejects(await pdfWithoutAttachment(), 'no_attachment')
|
||||||
|
await rejects(
|
||||||
|
await pdfWithAttachment('other.json', { hello: 'world' }),
|
||||||
|
'no_attachment',
|
||||||
|
)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('crafted stream layouts', () => {
|
||||||
|
// UTF-8: the demo report holds an en dash that Latin-1 cannot carry.
|
||||||
|
const demoJson = () =>
|
||||||
|
new TextEncoder().encode(JSON.stringify(golden.reports.demo))
|
||||||
|
|
||||||
|
it('/Length as an indirect reference: read up to endstream', async () => {
|
||||||
|
const pdf = craftPdf([
|
||||||
|
streamObject({
|
||||||
|
num: 2,
|
||||||
|
dict: '/Type /EmbeddedFile /Length 7 0 R',
|
||||||
|
data: demoJson(),
|
||||||
|
}),
|
||||||
|
ascii('7 0 obj\n123456\nendobj\n'),
|
||||||
|
])
|
||||||
|
const [c] = findEmbeddedFiles(pdf)
|
||||||
|
expect(c!.filter).toBeNull()
|
||||||
|
expect(c!.length).toBe(demoJson().byteLength)
|
||||||
|
expect((await readReportArchive(pdf, map())).origin).toBe('bare_v3')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('a wrong direct /Length that does not land on endstream: measured instead', async () => {
|
||||||
|
const pdf = craftPdf([
|
||||||
|
streamObject({
|
||||||
|
num: 2,
|
||||||
|
dict: '/Type /EmbeddedFile /Length 5',
|
||||||
|
data: demoJson(),
|
||||||
|
}),
|
||||||
|
])
|
||||||
|
expect(findEmbeddedFiles(pdf)[0]!.length).toBe(demoJson().byteLength)
|
||||||
|
expect((await readReportArchive(pdf, map())).origin).toBe('bare_v3')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('CRLF after stream, /Type/EmbeddedFile unspaced, a leading /Params dict', async () => {
|
||||||
|
const data = demoJson()
|
||||||
|
const pdf = craftPdf([
|
||||||
|
streamObject({
|
||||||
|
num: 2,
|
||||||
|
dict: `/Params << /Size ${data.byteLength} >> /Type/EmbeddedFile/Length ${data.byteLength}`,
|
||||||
|
data,
|
||||||
|
eol: '\r\n',
|
||||||
|
}),
|
||||||
|
])
|
||||||
|
const [c] = findEmbeddedFiles(pdf)
|
||||||
|
expect(c!.length).toBe(data.byteLength)
|
||||||
|
expect(await readAttachmentBytes(pdf)).toEqual(data)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('a Filespec with a hex /F name and an /EF listing /UF too names its stream', () => {
|
||||||
|
const name = Buffer.from('osteo-report.json', 'utf16le')
|
||||||
|
.swap16()
|
||||||
|
.toString('hex')
|
||||||
|
const pdf = craftPdf([
|
||||||
|
ascii(
|
||||||
|
`3 0 obj\n<< /Type /Filespec /F <${name}> /EF << /F 2 0 R /UF 2 0 R >> >>\nendobj\n`,
|
||||||
|
),
|
||||||
|
streamObject({
|
||||||
|
num: 2,
|
||||||
|
dict: '/Type /EmbeddedFile /Length 13',
|
||||||
|
data: ascii('{"version":3}'),
|
||||||
|
}),
|
||||||
|
])
|
||||||
|
expect(findEmbeddedFiles(pdf)[0]!.name).toBe('osteo-report.json')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('decodePdfString: literal with escapes, hex ASCII, hex UTF-16BE with BOM', () => {
|
||||||
|
expect(decodePdfString('(a\\(b\\)c)')).toBe('a(b)c')
|
||||||
|
expect(decodePdfString('<6F7374656F>')).toBe('osteo')
|
||||||
|
expect(decodePdfString('<FEFF006F0073>')).toBe('os')
|
||||||
|
expect(decodePdfString('/Name')).toBeNull()
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('the keyword stamp', () => {
|
||||||
|
it('hex (pdf-lib) and literal /Keywords strings are both read', async () => {
|
||||||
|
expect(
|
||||||
|
hasArchiveStamp(await pdfWithoutAttachment(['faunosteo-archive/1'])),
|
||||||
|
).toBe(true)
|
||||||
|
expect(hasArchiveStamp(await pdfWithoutAttachment(['other']))).toBe(false)
|
||||||
|
const literal = craftPdf([
|
||||||
|
ascii('3 0 obj\n<< /Keywords (faunosteo-archive/1) >>\nendobj\n'),
|
||||||
|
])
|
||||||
|
expect(hasArchiveStamp(literal)).toBe(true)
|
||||||
|
await rejects(literal, 'attachment_stripped')
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('garbage never throws', () => {
|
||||||
|
const cases: [string, Uint8Array][] = [
|
||||||
|
['empty', new Uint8Array(0)],
|
||||||
|
[
|
||||||
|
'random bytes',
|
||||||
|
Uint8Array.from({ length: 4096 }, (_, i) => (i * 7919) % 256),
|
||||||
|
],
|
||||||
|
['the type alone', ascii('%PDF-1.7\n/Type /EmbeddedFile')],
|
||||||
|
[
|
||||||
|
'dict without stream',
|
||||||
|
ascii('%PDF-1.7\n<< /Type /EmbeddedFile /Length 3 >>\nendobj'),
|
||||||
|
],
|
||||||
|
[
|
||||||
|
'stream without endstream',
|
||||||
|
ascii('%PDF-1.7\n<< /Type /EmbeddedFile >>\nstream\n{"version":3}'),
|
||||||
|
],
|
||||||
|
[
|
||||||
|
'only the /EmbeddedFiles name tree key',
|
||||||
|
ascii('%PDF-1.7\n<< /Type /EmbeddedFiles >>\nstream\nx\nendstream'),
|
||||||
|
],
|
||||||
|
[
|
||||||
|
'unterminated dictionary',
|
||||||
|
ascii(
|
||||||
|
'%PDF-1.7\n<< /Type /EmbeddedFile /Length 1 << >>\nstream\nx\nendstream',
|
||||||
|
),
|
||||||
|
],
|
||||||
|
[
|
||||||
|
'hex /Keywords without a closing bracket',
|
||||||
|
ascii('%PDF-1.7\n<< /Keywords <FEFF0066'),
|
||||||
|
],
|
||||||
|
]
|
||||||
|
|
||||||
|
it.each(cases)(
|
||||||
|
'%s: no candidate, no stamp, no_attachment or invalid_pdf',
|
||||||
|
async (_name, bytes) => {
|
||||||
|
expect(findEmbeddedFiles(bytes)).toEqual([])
|
||||||
|
expect(hasArchiveStamp(bytes)).toBe(false)
|
||||||
|
await expect(readAttachedDocument(bytes)).rejects.toBeInstanceOf(
|
||||||
|
ArchiveReadError,
|
||||||
|
)
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
it('caps the candidates at 32', () => {
|
||||||
|
const objects = Array.from({ length: 40 }, (_, i) =>
|
||||||
|
streamObject({
|
||||||
|
num: i + 2,
|
||||||
|
dict: '/Type /EmbeddedFile /Length 1',
|
||||||
|
data: ascii('x'),
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
expect(findEmbeddedFiles(craftPdf(objects))).toHaveLength(32)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('object-stream bomb (BLK-009)', () => {
|
||||||
|
const MiB = 1024 * 1024
|
||||||
|
const objStmBomb = () => {
|
||||||
|
const data = bomb()
|
||||||
|
return streamObject({
|
||||||
|
num: 9,
|
||||||
|
dict: `/Type /ObjStm /N 1 /First 4 /Filter /FlateDecode /Length ${data.byteLength}`,
|
||||||
|
data,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
const embeddedBomb = () => {
|
||||||
|
const data = bomb()
|
||||||
|
return streamObject({
|
||||||
|
num: 8,
|
||||||
|
dict: `/Type /EmbeddedFile /Filter /FlateDecode /Length ${data.byteLength}`,
|
||||||
|
data,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
it('64 MiB ObjStm bomb beside the attachment: report in ms, bomb never inflated', async () => {
|
||||||
|
const pdf = craftPdf([
|
||||||
|
objStmBomb(),
|
||||||
|
objectHolding(exported, '/Type /Filespec'),
|
||||||
|
objectHolding(exported, '/Type /EmbeddedFile'),
|
||||||
|
])
|
||||||
|
expect(pdf.byteLength).toBeLessThan(512 * 1024)
|
||||||
|
const rssBefore = process.memoryUsage().rss
|
||||||
|
const started = performance.now()
|
||||||
|
const outcome = await readReportArchive(pdf, map())
|
||||||
|
const elapsed = performance.now() - started
|
||||||
|
const rssDelta = process.memoryUsage().rss - rssBefore
|
||||||
|
console.log(
|
||||||
|
`[bomb] objstm: ${elapsed.toFixed(0)} ms, rss +${(rssDelta / MiB).toFixed(1)} MiB`,
|
||||||
|
)
|
||||||
|
expect(outcome.origin).toBe('envelope_v1')
|
||||||
|
expect(outcome.report).toEqual(golden.reports.demo)
|
||||||
|
expect(elapsed).toBeLessThan(2000)
|
||||||
|
expect(rssDelta).toBeLessThan(64 * MiB)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('EmbeddedFile bomb past 16 MiB: inflation cancelled, no_attachment, no hang', async () => {
|
||||||
|
const pdf = craftPdf([objStmBomb(), embeddedBomb()])
|
||||||
|
const started = performance.now()
|
||||||
|
await rejects(pdf, 'no_attachment')
|
||||||
|
const elapsed = performance.now() - started
|
||||||
|
console.log(`[bomb] embedded: ${elapsed.toFixed(0)} ms`)
|
||||||
|
expect(elapsed).toBeLessThan(2000)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('both bombs before an unnamed real attachment: bomb skipped, report read', async () => {
|
||||||
|
const pdf = craftPdf([
|
||||||
|
objStmBomb(),
|
||||||
|
embeddedBomb(),
|
||||||
|
objectHolding(exported, '/Type /EmbeddedFile'),
|
||||||
|
])
|
||||||
|
const started = performance.now()
|
||||||
|
const outcome = await readReportArchive(pdf, map())
|
||||||
|
const elapsed = performance.now() - started
|
||||||
|
console.log(`[bomb] both + unnamed attachment: ${elapsed.toFixed(0)} ms`)
|
||||||
|
expect(outcome.report).toEqual(golden.reports.demo)
|
||||||
|
expect(elapsed).toBeLessThan(2000)
|
||||||
|
})
|
||||||
|
})
|
||||||
@@ -36,7 +36,7 @@ describe('demo envelope size', () => {
|
|||||||
const raw = new TextEncoder().encode(
|
const raw = new TextEncoder().encode(
|
||||||
JSON.stringify(envelope, null, 2),
|
JSON.stringify(envelope, null, 2),
|
||||||
).byteLength
|
).byteLength
|
||||||
const stored = await storedAttachmentSize(pdf)
|
const stored = storedAttachmentSize(pdf)
|
||||||
console.log(
|
console.log(
|
||||||
`[size] demo ${mode}: stored ${stored} B, raw ${raw} B, pdf ${pdf.byteLength} B`,
|
`[size] demo ${mode}: stored ${stored} B, raw ${raw} B, pdf ${pdf.byteLength} B`,
|
||||||
)
|
)
|
||||||
|
|||||||
Reference in new issue
Block a user