feat(domain): R4.6 ExportSession (incremental export, OD-2): rasters decoded and written on arrival, chunked file handed out in parts, views composed by the core, raster_size_of (O4); wasm ExportSession and CoreExportSession; R4 digests re-frozen (catalog and page tree written last, B2)

This commit is contained in:
bchanot committed 2026-10-19 01:15:43 +02:00
1 parent 8d16be577c
commit 9e688b6801
38 files changed
+2421 -354

No files matched your search

@@ -0,0 +1,392 @@
/**
* R4.6: the export session through the shipped module and the runtime wrapper
* (`CoreExportSession`): driven request by request on the O2 runs it writes the native
* bytes (`goldens/r4/pdf.*.sha256`); every view it asks for is the SVG the TypeScript
* engine composes (`composeHighlightedViewSvg`) and, on the real-raster run, the digests
* and sizes the R4.1 rasteriser recorded (`pdf/r3/real/manifest.json`); the refusals are
* closed kinds; a trap mid-session poisons the instance and the session answers
* `core_unavailable` from then on, `free()` included; the memory held by a fresh instance
* after an export is printed against the file's size (OD-2, RUST.md §10.8).
*/
import { createHash } from 'node:crypto'
import { readFileSync } from 'node:fs'
import { join } from 'node:path'
import { fileURLToPath } from 'node:url'
import { composeHighlightedViewSvg } from '@faunosteo/pdf'
import {
CoreRuntime,
openExportSession,
type CoreExportSession,
type GlueModule,
} from '../src/index'
import { nodeCoreRuntime, nodeCoreSource, type ProofModule } from '../src/node'
import type { ExportStep, ExportWarning } from '../types'
import { readAttachedDocument } from './n-1/scanner/attachment/read'
const ROOT = fileURLToPath(new URL('../../../', import.meta.url))
const FIXTURES = join(ROOT, 'apps/app/tests/fixtures')
const sha256 = (data: Uint8Array | string) =>
createHash('sha256').update(data).digest('hex')
const ONE_PX_PNG = Buffer.from(
'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==',
'base64',
)
const TWO_BY_ONE_PNG = Buffer.from(
'iVBORw0KGgoAAAANSUhEUgAAAAIAAAABCAIAAAB7QOjdAAAAD0lEQVR4nGPYEWX///9/AA0PBE/1T7nxAAAAAElFTkSuQmCC',
'base64',
)
interface Envelope {
report: {
observations: Array<{
id: string
zones: Array<{ viewId: string; annotationId: string }>
}>
}
sheetSnapshot: unknown
branding: unknown
labels: unknown
locale: string
exportedAt: string
}
interface Run {
name: string
r3: string
report?: string
/** The PNG of a request key, or `null` for a raster that fails. */
raster: (key: string) => Uint8Array | null
}
const RUNS: Run[] = []
for (const report of ['demo', 'v2full', 'v2unresolved']) {
for (const locale of ['fr', 'en']) {
for (const mode of ['nologos', 'logos']) {
const png = mode === 'logos' ? TWO_BY_ONE_PNG : ONE_PX_PNG
RUNS.push({
name: `${report}.${locale}.${mode}`,
r3: `pdf/r3/${report}.${locale}.${mode}.pdf`,
raster: () => new Uint8Array(png),
})
}
}
}
RUNS.push({
name: 'real.v2full-obs1.fr.logos',
r3: 'pdf/r3/real/v2full-obs1.fr.logos.pdf',
raster: (key) =>
new Uint8Array(
readFileSync(
join(
FIXTURES,
'pdf/r3/real',
key === 'logo:partner' ? 'raster-0.png' : 'raster-1.png',
),
),
),
})
RUNS.push({
name: 'store-path.fr.nologos',
r3: 'pdf/r3/store-path.fr.nologos.pdf',
report: 'pdf/r3/store-path.report.json',
raster: () => new Uint8Array(ONE_PX_PNG),
})
async function envelopeOf(path: string): Promise<Envelope> {
const pdf = new Uint8Array(readFileSync(join(FIXTURES, path)))
return (await readAttachedDocument(pdf)) as Envelope
}
function inputOf(envelope: Envelope, reportOverride?: string): string {
const report = reportOverride
? JSON.parse(readFileSync(join(FIXTURES, reportOverride), 'utf8'))
: envelope.report
return JSON.stringify({
report,
sheetSnapshot: envelope.sheetSnapshot,
branding: envelope.branding,
labels: envelope.labels,
locale: envelope.locale,
exportedAt: envelope.exportedAt,
})
}
function unwrap<T>(outcome: { ok: true; value: T } | { ok: false; kind: string; detail?: string }, what: string): T {
if (!outcome.ok) throw new Error(`${what}: ${outcome.kind} ${outcome.detail ?? ''}`)
return outcome.value
}
interface Driven {
pdf: Uint8Array
parts: number
warnings: ExportWarning[]
requests: Array<{ key: string; svg: string; widthPx: number; heightPx: number }>
}
/** Drives a session to its file, answering every request from `raster`. */
function drive(session: CoreExportSession, raster: Run['raster']): Driven {
const requests: Driven['requests'] = []
for (;;) {
const step = unwrap(session.next(), 'next')
if (step.step === 'done') break
for (const request of step.requests) {
requests.push(request)
const png = raster(request.key)
unwrap(
png ? session.addRaster(request.key, png) : session.rasterFailed(request.key),
request.key,
)
}
}
const parts = unwrap(session.finish(), 'finish')
const warnings = unwrap(session.warnings(), 'warnings')
return { pdf: Buffer.concat(parts), parts: parts.length, warnings, requests }
}
describe('R4.6 export session through the shipped module', () => {
it('writes the native bytes on every O2 run, asking for the SVGs the engine composes', async () => {
const runtime = await nodeCoreRuntime('pkg')
for (const run of RUNS) {
const envelope = await envelopeOf(run.r3)
const session = unwrap(
openExportSession(runtime, inputOf(envelope, run.report)),
run.name,
)
let driven: Driven
try {
driven = drive(session, run.raster)
} finally {
session.free()
}
const expected = readFileSync(
join(FIXTURES, `goldens/r4/pdf.${run.name}.sha256`),
'utf8',
).trim()
expect(sha256(driven.pdf), run.name).toBe(expected)
expect(driven.warnings, run.name).toEqual([])
// Every view request is the TypeScript composition of that view for that observation.
const report = run.report
? (JSON.parse(readFileSync(join(FIXTURES, run.report), 'utf8')) as Envelope['report'])
: envelope.report
let views = 0
for (const request of driven.requests) {
if (!request.key.startsWith('view:')) continue
const [, observationId, viewId] = request.key.split(':')
const observation = report.observations.find((o) => o.id === observationId)!
expect(
request.svg,
`${run.name} ${request.key}`,
).toBe(
composeHighlightedViewSvg(
envelope.sheetSnapshot as never,
viewId!,
observation as never,
),
)
views += 1
}
// Every (observation, view) the snapshot holds was asked for, once.
const snapshot = envelope.sheetSnapshot as { document: { views: Array<{ id: string }> } }
const held = new Set(snapshot.document.views.map((v) => v.id))
const expectedViews = new Set(
report.observations.flatMap((o) =>
o.zones.filter((z) => held.has(z.viewId)).map((z) => `${o.id}\u0000${z.viewId}`),
),
)
expect(views, run.name).toBe(expectedViews.size)
}
}, 120_000)
it('asks for the SVGs and sizes the R4.1 rasteriser recorded on the real-raster run', async () => {
const runtime = await nodeCoreRuntime('pkg')
const run = RUNS.find((r) => r.name.startsWith('real.'))!
const manifest = JSON.parse(
readFileSync(join(FIXTURES, 'pdf/r3/real/manifest.json'), 'utf8'),
) as {
calls: Array<{
svgSha256: string
widthPx: number
heightPx: number
subject: { kind: string; slot?: string; observationId?: string; viewId?: string }
}>
}
const session = unwrap(openExportSession(runtime, inputOf(await envelopeOf(run.r3))), run.name)
let driven: Driven
try {
driven = drive(session, run.raster)
} finally {
session.free()
}
const byKey = new Map(driven.requests.map((r) => [r.key, r]))
for (const call of manifest.calls) {
const key =
call.subject.kind === 'logo'
? `logo:${call.subject.slot}`
: `view:${call.subject.observationId}:${call.subject.viewId}`
const request = byKey.get(key)
expect(request, key).toBeDefined()
expect(sha256(request!.svg), key).toBe(call.svgSha256)
expect([request!.widthPx, request!.heightPx], key).toEqual([
call.widthPx,
call.heightPx,
])
}
expect(byKey.size).toBe(manifest.calls.length)
})
it('refuses with closed kinds, never a trap: a refused input, a bad key, the order', async () => {
const runtime = await nodeCoreRuntime('pkg')
const envelope = await envelopeOf('pdf/r3/demo.fr.nologos.pdf')
const bad = { ...JSON.parse(inputOf(envelope)), branding: { siteUrl: 'nope' } }
const refused = openExportSession(runtime, JSON.stringify(bad))
expect(refused).toEqual({ ok: false, kind: 'branding_invalid', detail: 'branding' })
expect(openExportSession(runtime, '{ not json')).toMatchObject({
ok: false,
kind: 'invalid_json',
})
expect(runtime.status).toBe('ready')
const session = unwrap(openExportSession(runtime, inputOf(envelope)), 'open')
try {
expect(session.finish()).toEqual({ ok: false, kind: 'out_of_order', detail: '' })
expect(session.addRaster('view:x:y', ONE_PX_PNG)).toEqual({
ok: false,
kind: 'unknown_key',
detail: '',
})
const first = unwrap(session.next(), 'next') as ExportStep
expect(first.step).toBe('rasters')
const key = (first as Extract<ExportStep, { step: 'rasters' }>).requests[0]!.key
expect(session.next()).toEqual({ ok: false, kind: 'out_of_order', detail: '' })
// A PNG the decoder refuses is the view's warning, not a failure.
expect(session.addRaster(key, new Uint8Array([1, 2, 3]))).toEqual({ ok: true, value: null })
expect(session.addRaster(key, ONE_PX_PNG)).toEqual({
ok: false,
kind: 'duplicate_key',
detail: '',
})
const warnings = unwrap(session.warnings(), 'warnings')
expect(warnings).toHaveLength(1)
expect(warnings[0]).toMatchObject({ kind: 'view_raster_failed' })
expect(runtime.status).toBe('ready')
} finally {
session.free()
}
expect(session.next()).toEqual({ ok: false, kind: 'out_of_order', detail: 'session freed' })
session.free()
})
it('answers core_unavailable after a trap mid-session, frees safely, and a new session works after the recovery', async () => {
const runtime = new CoreRuntime(nodeCoreSource('pkg-proofs'))
expect(await runtime.ready()).toBe(true)
const envelope = await envelopeOf('pdf/r3/demo.fr.logos.pdf')
const session = unwrap(openExportSession(runtime, inputOf(envelope)), 'open')
const step = unwrap(session.next(), 'next')
expect(step.step).toBe('rasters')
// The trap: another call of the same instance panics (as any core call could).
const trapped = runtime.invoke((glue: GlueModule) => {
;(glue as ProofModule).panic_for_test()
return { ok: true as const, value: null }
})
expect(trapped).toMatchObject({ ok: false, kind: 'internal' })
expect(session.next()).toEqual({ ok: false, kind: 'core_unavailable' })
expect(session.addRaster('logo:partner', TWO_BY_ONE_PNG)).toEqual({
ok: false,
kind: 'core_unavailable',
})
expect(session.finish()).toEqual({ ok: false, kind: 'core_unavailable' })
session.free()
expect(await runtime.ready()).toBe(true)
expect(runtime.generation).toBe(2)
expect(session.next()).toEqual({ ok: false, kind: 'out_of_order', detail: 'session freed' })
const again = unwrap(openExportSession(runtime, inputOf(envelope)), 'open again')
try {
const driven = drive(again, () => new Uint8Array(TWO_BY_ONE_PNG))
expect(sha256(driven.pdf)).toBe(
readFileSync(join(FIXTURES, 'goldens/r4/pdf.demo.fr.logos.sha256'), 'utf8').trim(),
)
} finally {
again.free()
}
})
})
// ---------------------------------------------------------------------------
// Memory held by the instance (OD-2): a fresh instance per run, linear memory after the
// session is freed (the high-water: linear memory never shrinks) against the file size.
interface Fresh {
runtime: CoreRuntime
memory: () => number
}
async function freshRuntime(): Promise<Fresh> {
let memory: WebAssembly.Memory | undefined
const source = nodeCoreSource('pkg')
const runtime = new CoreRuntime({
compile: source.compile,
importGlue: async (n) => {
const glue = await source.importGlue(n)
return {
...glue,
default: async (input: { module_or_path: WebAssembly.Module }) => {
const exports = (await glue.default(input)) as { memory?: WebAssembly.Memory }
memory = exports.memory
return exports
},
} as GlueModule
},
})
if (!(await runtime.ready())) throw new Error('module failed to load')
return { runtime, memory: () => memory?.buffer.byteLength ?? 0 }
}
describe('R4.6 memory held after an export (printed for RUST.md §10.8)', () => {
it('holds the file plus a bounded transient, never the rasters', async () => {
const demo = await envelopeOf('pdf/r3/demo.fr.logos.pdf')
const real = new Uint8Array(readFileSync(join(FIXTURES, 'pdf/r3/real/raster-1.png')))
const cases: Array<{ name: string; report?: string; png: Uint8Array }> = [
{ name: 'demo stub', png: new Uint8Array(TWO_BY_ONE_PNG) },
{ name: 'L stub', report: 'L', png: new Uint8Array(TWO_BY_ONE_PNG) },
{ name: 'CAP stub', report: 'CAP', png: new Uint8Array(TWO_BY_ONE_PNG) },
{ name: 'demo real', png: real },
{ name: 'L real', report: 'L', png: real },
]
const rows = [
'| Run | views | PDF bytes | parts | linear before | linear after | (after - before) / PDF | ms |',
'|---|---|---|---|---|---|---|---|',
]
for (const c of cases) {
const fresh = await freshRuntime()
const report = c.report
? JSON.parse(readFileSync(join(FIXTURES, `r4/reports/${c.report}.json`), 'utf8'))
: demo.report
if (c.report) report.sheet = (demo.report as unknown as { sheet: unknown }).sheet
const input = JSON.stringify({ ...demo, report, locale: 'fr' })
const before = fresh.memory()
const started = performance.now()
const session = unwrap(openExportSession(fresh.runtime, input), c.name)
let driven: Driven
try {
driven = drive(session, () => c.png)
} finally {
session.free()
}
const ms = performance.now() - started
const after = fresh.memory()
const views = driven.requests.filter((r) => r.key.startsWith('view:')).length
rows.push(
`| ${c.name} | ${views} | ${driven.pdf.byteLength} | ${driven.parts} | ${before} | ${after} | ${((after - before) / driven.pdf.byteLength).toFixed(2)} | ${ms.toFixed(0)} |`,
)
expect(driven.warnings, c.name).toEqual([])
// The rasters never accumulate: what the instance holds beyond the file is the
// parsed input (report, snapshot, trees), one decode transient (RGBA, raw, RGB and
// alpha planes of a 1200 × 943 raster, about 10 MB) and the open chunk's slack (up
// to 8 MiB), whatever the number of views; the PNGs of L real alone weigh 64 MB.
const overhead = after - before - driven.pdf.byteLength
expect(overhead, `${c.name}: held beyond the file`).toBeLessThan(24 * 1024 * 1024)
}
console.info(`[core-wasm] R4.6 session memory (fresh instance per run)\n${rows.join('\n')}`)
}, 600_000)
})