feat(domain): R4.6 ExportSession (incremental export, OD-2): rasters decoded and written on arrival, chunked file handed out in parts, views composed by the core, raster_size_of (O4); wasm ExportSession and CoreExportSession; R4 digests re-frozen (catalog and page tree written last, B2)

This commit is contained in:
bchanot committed 2026-10-19 01:15:43 +02:00
1 parent 8d16be577c
commit 9e688b6801
38 files changed
+2421 -354

No files matched your search

+152
View File
@@ -0,0 +1,152 @@
import type { ExportStep, ExportWarning } from '../types'
import { CORE_UNAVAILABLE, failure, parseOutcome, type CoreOutcome } from './outcome'
import {
closedRefusal,
type CoreRuntime,
type GlueExportSession,
type GlueModule,
} from './runtime'
/**
* The incremental PDF export (R4.6, plan §B.1.4, OD-2) on the page's runtime: every call
* goes through `CoreRuntime.invoke`, so a trap poisons the instance and answers `internal`
* like any core call, and a session opened on an instance the runtime has since dropped
* (a recovery) answers `core_unavailable` without touching the dead handle, as sheet
* handles do. The driver calls `free()` in its `finally`: the module's buffer (the file
* being written) is released whatever the outcome.
*/
export class CoreExportSession {
private handle: GlueExportSession | null
constructor(
private readonly runtime: CoreRuntime,
handle: GlueExportSession,
private readonly generation: number,
) {
this.handle = handle
}
/** The next batch of raster requests, or `done`. */
next(): CoreOutcome<ExportStep> {
return this.call((handle) => parseOutcome<ExportStep>(handle.next_step()))
}
/** The PNG of `key`: `null` on success, else a closed refusal. */
addRaster(key: string, png: Uint8Array): CoreOutcome<null> {
return this.call((handle) =>
parseOutcome<null>(handle.add_raster(key, png)),
)
}
/** `key` could not be rasterised (the session records the warning). */
rasterFailed(key: string): CoreOutcome<null> {
return this.call((handle) => parseOutcome<null>(handle.raster_failed(key)))
}
/**
* The file, as the parts the module hands out one by one (each copied out of linear
* memory and freed there, so the file is never held twice; `new Blob(parts)` takes them
* as they are). The module throws a closed `Error` worded `<kind>: <detail>` for
* `out_of_order` or `self_check_failed`; anything else is a trap.
*/
finish(): CoreOutcome<Uint8Array[]> {
return this.call((handle) => {
let count: number
try {
count = handle.finish()
} catch (error) {
const refusal = closedRefusal(error)
if (refusal === null) throw error
return closedFailure(refusal)
}
const parts: Uint8Array[] = []
for (let i = 0; i < count; i += 1) {
const part = handle.take_part()
if (!part) break
parts.push(part)
}
return { ok: true, value: parts }
})
}
/** The warnings recorded so far (complete after `finish()`). */
warnings(): CoreOutcome<ExportWarning[]> {
return this.call((handle) =>
parseOutcome<ExportWarning[]>(handle.warnings()),
)
}
/** Bytes of the file written so far (DEV diagnostics). */
written(): number {
const outcome = this.call((handle) => ({
ok: true as const,
value: handle.written(),
}))
return outcome.ok ? outcome.value : 0
}
/**
* Releases the module's buffer. Safe to call twice; a handle of a dropped instance is
* never touched (its memory went with the instance).
*/
free(): void {
const handle = this.handle
this.handle = null
if (!handle || !this.live()) return
try {
handle.free()
} catch {
// The glue refuses a double free; nothing else to do.
}
}
private live(): boolean {
return (
this.runtime.generation === this.generation &&
this.runtime.status === 'ready'
)
}
private call<T>(fn: (handle: GlueExportSession) => CoreOutcome<T>): CoreOutcome<T> {
const handle = this.handle
if (!handle) return failure('out_of_order', 'session freed')
if (!this.live()) {
this.runtime.kickRestart()
return CORE_UNAVAILABLE
}
return this.runtime.invoke(() => fn(handle))
}
}
/** `<kind>: <detail>` → the failure. */
function closedFailure(message: string): CoreOutcome<never> {
const colon = message.indexOf(': ')
return colon < 0
? failure(message)
: failure(message.slice(0, colon), message.slice(colon + 2))
}
/**
* Opens a session on the live instance: the module's checks refuse the input with today's
* export kinds (`branding_invalid`, `snapshot_invalid`, `snapshot_too_large`,
* `sheet_mismatch`, `too_many_pages`, `invalid_input`), a trap poisons the instance.
*/
export function openExportSession(
runtime: CoreRuntime,
inputJson: string,
): CoreOutcome<CoreExportSession> {
return runtime.invoke((glue: GlueModule) => {
let handle: GlueExportSession
try {
handle = new glue.ExportSession(inputJson)
} catch (error) {
const refusal = closedRefusal(error)
if (refusal === null) throw error
return closedFailure(refusal)
}
return {
ok: true,
value: new CoreExportSession(runtime, handle, runtime.generation),
}
})
}
+2
View File
@@ -11,10 +11,12 @@ export {
closedRefusal,
type CoreSource,
type CoreStatus,
type GlueExportSession,
type GlueFileBytes,
type GlueModule,
type GlueSheetHandle,
} from './runtime'
export { CoreExportSession, openExportSession } from './export'
export {
CORE_UNAVAILABLE,
parseOutcome,
+16
View File
@@ -24,6 +24,22 @@ export interface GlueModule {
call(op: string, input: string): string
SheetHandle: new (bundle: string) => GlueSheetHandle
FileBytes: new (expected: number) => GlueFileBytes
/** R4.6: the incremental export; the constructor throws `<kind>: <detail>` on a refused input. */
ExportSession: new (inputJson: string) => GlueExportSession
}
/** The export session of the module (crates/wasm `ExportSession`): outcome envelopes out. */
export interface GlueExportSession {
next_step(): string
add_raster(key: string, png: Uint8Array): string
raster_failed(key: string): string
/** Throws `<kind>: <detail>` (`out_of_order`, `self_check_failed`); the parts count otherwise. */
finish(): number
/** The next part of the finished file (copied out, freed in the module), `undefined` at the end. */
take_part(): Uint8Array | undefined
warnings(): string
written(): number
free(): void
}
export interface GlueSheetHandle {
@@ -0,0 +1,392 @@
/**
* R4.6: the export session through the shipped module and the runtime wrapper
* (`CoreExportSession`): driven request by request on the O2 runs it writes the native
* bytes (`goldens/r4/pdf.*.sha256`); every view it asks for is the SVG the TypeScript
* engine composes (`composeHighlightedViewSvg`) and, on the real-raster run, the digests
* and sizes the R4.1 rasteriser recorded (`pdf/r3/real/manifest.json`); the refusals are
* closed kinds; a trap mid-session poisons the instance and the session answers
* `core_unavailable` from then on, `free()` included; the memory held by a fresh instance
* after an export is printed against the file's size (OD-2, RUST.md §10.8).
*/
import { createHash } from 'node:crypto'
import { readFileSync } from 'node:fs'
import { join } from 'node:path'
import { fileURLToPath } from 'node:url'
import { composeHighlightedViewSvg } from '@faunosteo/pdf'
import {
CoreRuntime,
openExportSession,
type CoreExportSession,
type GlueModule,
} from '../src/index'
import { nodeCoreRuntime, nodeCoreSource, type ProofModule } from '../src/node'
import type { ExportStep, ExportWarning } from '../types'
import { readAttachedDocument } from './n-1/scanner/attachment/read'
const ROOT = fileURLToPath(new URL('../../../', import.meta.url))
const FIXTURES = join(ROOT, 'apps/app/tests/fixtures')
const sha256 = (data: Uint8Array | string) =>
createHash('sha256').update(data).digest('hex')
const ONE_PX_PNG = Buffer.from(
'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==',
'base64',
)
const TWO_BY_ONE_PNG = Buffer.from(
'iVBORw0KGgoAAAANSUhEUgAAAAIAAAABCAIAAAB7QOjdAAAAD0lEQVR4nGPYEWX///9/AA0PBE/1T7nxAAAAAElFTkSuQmCC',
'base64',
)
interface Envelope {
report: {
observations: Array<{
id: string
zones: Array<{ viewId: string; annotationId: string }>
}>
}
sheetSnapshot: unknown
branding: unknown
labels: unknown
locale: string
exportedAt: string
}
interface Run {
name: string
r3: string
report?: string
/** The PNG of a request key, or `null` for a raster that fails. */
raster: (key: string) => Uint8Array | null
}
const RUNS: Run[] = []
for (const report of ['demo', 'v2full', 'v2unresolved']) {
for (const locale of ['fr', 'en']) {
for (const mode of ['nologos', 'logos']) {
const png = mode === 'logos' ? TWO_BY_ONE_PNG : ONE_PX_PNG
RUNS.push({
name: `${report}.${locale}.${mode}`,
r3: `pdf/r3/${report}.${locale}.${mode}.pdf`,
raster: () => new Uint8Array(png),
})
}
}
}
RUNS.push({
name: 'real.v2full-obs1.fr.logos',
r3: 'pdf/r3/real/v2full-obs1.fr.logos.pdf',
raster: (key) =>
new Uint8Array(
readFileSync(
join(
FIXTURES,
'pdf/r3/real',
key === 'logo:partner' ? 'raster-0.png' : 'raster-1.png',
),
),
),
})
RUNS.push({
name: 'store-path.fr.nologos',
r3: 'pdf/r3/store-path.fr.nologos.pdf',
report: 'pdf/r3/store-path.report.json',
raster: () => new Uint8Array(ONE_PX_PNG),
})
async function envelopeOf(path: string): Promise<Envelope> {
const pdf = new Uint8Array(readFileSync(join(FIXTURES, path)))
return (await readAttachedDocument(pdf)) as Envelope
}
function inputOf(envelope: Envelope, reportOverride?: string): string {
const report = reportOverride
? JSON.parse(readFileSync(join(FIXTURES, reportOverride), 'utf8'))
: envelope.report
return JSON.stringify({
report,
sheetSnapshot: envelope.sheetSnapshot,
branding: envelope.branding,
labels: envelope.labels,
locale: envelope.locale,
exportedAt: envelope.exportedAt,
})
}
function unwrap<T>(outcome: { ok: true; value: T } | { ok: false; kind: string; detail?: string }, what: string): T {
if (!outcome.ok) throw new Error(`${what}: ${outcome.kind} ${outcome.detail ?? ''}`)
return outcome.value
}
interface Driven {
pdf: Uint8Array
parts: number
warnings: ExportWarning[]
requests: Array<{ key: string; svg: string; widthPx: number; heightPx: number }>
}
/** Drives a session to its file, answering every request from `raster`. */
function drive(session: CoreExportSession, raster: Run['raster']): Driven {
const requests: Driven['requests'] = []
for (;;) {
const step = unwrap(session.next(), 'next')
if (step.step === 'done') break
for (const request of step.requests) {
requests.push(request)
const png = raster(request.key)
unwrap(
png ? session.addRaster(request.key, png) : session.rasterFailed(request.key),
request.key,
)
}
}
const parts = unwrap(session.finish(), 'finish')
const warnings = unwrap(session.warnings(), 'warnings')
return { pdf: Buffer.concat(parts), parts: parts.length, warnings, requests }
}
describe('R4.6 export session through the shipped module', () => {
it('writes the native bytes on every O2 run, asking for the SVGs the engine composes', async () => {
const runtime = await nodeCoreRuntime('pkg')
for (const run of RUNS) {
const envelope = await envelopeOf(run.r3)
const session = unwrap(
openExportSession(runtime, inputOf(envelope, run.report)),
run.name,
)
let driven: Driven
try {
driven = drive(session, run.raster)
} finally {
session.free()
}
const expected = readFileSync(
join(FIXTURES, `goldens/r4/pdf.${run.name}.sha256`),
'utf8',
).trim()
expect(sha256(driven.pdf), run.name).toBe(expected)
expect(driven.warnings, run.name).toEqual([])
// Every view request is the TypeScript composition of that view for that observation.
const report = run.report
? (JSON.parse(readFileSync(join(FIXTURES, run.report), 'utf8')) as Envelope['report'])
: envelope.report
let views = 0
for (const request of driven.requests) {
if (!request.key.startsWith('view:')) continue
const [, observationId, viewId] = request.key.split(':')
const observation = report.observations.find((o) => o.id === observationId)!
expect(
request.svg,
`${run.name} ${request.key}`,
).toBe(
composeHighlightedViewSvg(
envelope.sheetSnapshot as never,
viewId!,
observation as never,
),
)
views += 1
}
// Every (observation, view) the snapshot holds was asked for, once.
const snapshot = envelope.sheetSnapshot as { document: { views: Array<{ id: string }> } }
const held = new Set(snapshot.document.views.map((v) => v.id))
const expectedViews = new Set(
report.observations.flatMap((o) =>
o.zones.filter((z) => held.has(z.viewId)).map((z) => `${o.id}\u0000${z.viewId}`),
),
)
expect(views, run.name).toBe(expectedViews.size)
}
}, 120_000)
it('asks for the SVGs and sizes the R4.1 rasteriser recorded on the real-raster run', async () => {
const runtime = await nodeCoreRuntime('pkg')
const run = RUNS.find((r) => r.name.startsWith('real.'))!
const manifest = JSON.parse(
readFileSync(join(FIXTURES, 'pdf/r3/real/manifest.json'), 'utf8'),
) as {
calls: Array<{
svgSha256: string
widthPx: number
heightPx: number
subject: { kind: string; slot?: string; observationId?: string; viewId?: string }
}>
}
const session = unwrap(openExportSession(runtime, inputOf(await envelopeOf(run.r3))), run.name)
let driven: Driven
try {
driven = drive(session, run.raster)
} finally {
session.free()
}
const byKey = new Map(driven.requests.map((r) => [r.key, r]))
for (const call of manifest.calls) {
const key =
call.subject.kind === 'logo'
? `logo:${call.subject.slot}`
: `view:${call.subject.observationId}:${call.subject.viewId}`
const request = byKey.get(key)
expect(request, key).toBeDefined()
expect(sha256(request!.svg), key).toBe(call.svgSha256)
expect([request!.widthPx, request!.heightPx], key).toEqual([
call.widthPx,
call.heightPx,
])
}
expect(byKey.size).toBe(manifest.calls.length)
})
it('refuses with closed kinds, never a trap: a refused input, a bad key, the order', async () => {
const runtime = await nodeCoreRuntime('pkg')
const envelope = await envelopeOf('pdf/r3/demo.fr.nologos.pdf')
const bad = { ...JSON.parse(inputOf(envelope)), branding: { siteUrl: 'nope' } }
const refused = openExportSession(runtime, JSON.stringify(bad))
expect(refused).toEqual({ ok: false, kind: 'branding_invalid', detail: 'branding' })
expect(openExportSession(runtime, '{ not json')).toMatchObject({
ok: false,
kind: 'invalid_json',
})
expect(runtime.status).toBe('ready')
const session = unwrap(openExportSession(runtime, inputOf(envelope)), 'open')
try {
expect(session.finish()).toEqual({ ok: false, kind: 'out_of_order', detail: '' })
expect(session.addRaster('view:x:y', ONE_PX_PNG)).toEqual({
ok: false,
kind: 'unknown_key',
detail: '',
})
const first = unwrap(session.next(), 'next') as ExportStep
expect(first.step).toBe('rasters')
const key = (first as Extract<ExportStep, { step: 'rasters' }>).requests[0]!.key
expect(session.next()).toEqual({ ok: false, kind: 'out_of_order', detail: '' })
// A PNG the decoder refuses is the view's warning, not a failure.
expect(session.addRaster(key, new Uint8Array([1, 2, 3]))).toEqual({ ok: true, value: null })
expect(session.addRaster(key, ONE_PX_PNG)).toEqual({
ok: false,
kind: 'duplicate_key',
detail: '',
})
const warnings = unwrap(session.warnings(), 'warnings')
expect(warnings).toHaveLength(1)
expect(warnings[0]).toMatchObject({ kind: 'view_raster_failed' })
expect(runtime.status).toBe('ready')
} finally {
session.free()
}
expect(session.next()).toEqual({ ok: false, kind: 'out_of_order', detail: 'session freed' })
session.free()
})
it('answers core_unavailable after a trap mid-session, frees safely, and a new session works after the recovery', async () => {
const runtime = new CoreRuntime(nodeCoreSource('pkg-proofs'))
expect(await runtime.ready()).toBe(true)
const envelope = await envelopeOf('pdf/r3/demo.fr.logos.pdf')
const session = unwrap(openExportSession(runtime, inputOf(envelope)), 'open')
const step = unwrap(session.next(), 'next')
expect(step.step).toBe('rasters')
// The trap: another call of the same instance panics (as any core call could).
const trapped = runtime.invoke((glue: GlueModule) => {
;(glue as ProofModule).panic_for_test()
return { ok: true as const, value: null }
})
expect(trapped).toMatchObject({ ok: false, kind: 'internal' })
expect(session.next()).toEqual({ ok: false, kind: 'core_unavailable' })
expect(session.addRaster('logo:partner', TWO_BY_ONE_PNG)).toEqual({
ok: false,
kind: 'core_unavailable',
})
expect(session.finish()).toEqual({ ok: false, kind: 'core_unavailable' })
session.free()
expect(await runtime.ready()).toBe(true)
expect(runtime.generation).toBe(2)
expect(session.next()).toEqual({ ok: false, kind: 'out_of_order', detail: 'session freed' })
const again = unwrap(openExportSession(runtime, inputOf(envelope)), 'open again')
try {
const driven = drive(again, () => new Uint8Array(TWO_BY_ONE_PNG))
expect(sha256(driven.pdf)).toBe(
readFileSync(join(FIXTURES, 'goldens/r4/pdf.demo.fr.logos.sha256'), 'utf8').trim(),
)
} finally {
again.free()
}
})
})
// ---------------------------------------------------------------------------
// Memory held by the instance (OD-2): a fresh instance per run, linear memory after the
// session is freed (the high-water: linear memory never shrinks) against the file size.
interface Fresh {
runtime: CoreRuntime
memory: () => number
}
async function freshRuntime(): Promise<Fresh> {
let memory: WebAssembly.Memory | undefined
const source = nodeCoreSource('pkg')
const runtime = new CoreRuntime({
compile: source.compile,
importGlue: async (n) => {
const glue = await source.importGlue(n)
return {
...glue,
default: async (input: { module_or_path: WebAssembly.Module }) => {
const exports = (await glue.default(input)) as { memory?: WebAssembly.Memory }
memory = exports.memory
return exports
},
} as GlueModule
},
})
if (!(await runtime.ready())) throw new Error('module failed to load')
return { runtime, memory: () => memory?.buffer.byteLength ?? 0 }
}
describe('R4.6 memory held after an export (printed for RUST.md §10.8)', () => {
it('holds the file plus a bounded transient, never the rasters', async () => {
const demo = await envelopeOf('pdf/r3/demo.fr.logos.pdf')
const real = new Uint8Array(readFileSync(join(FIXTURES, 'pdf/r3/real/raster-1.png')))
const cases: Array<{ name: string; report?: string; png: Uint8Array }> = [
{ name: 'demo stub', png: new Uint8Array(TWO_BY_ONE_PNG) },
{ name: 'L stub', report: 'L', png: new Uint8Array(TWO_BY_ONE_PNG) },
{ name: 'CAP stub', report: 'CAP', png: new Uint8Array(TWO_BY_ONE_PNG) },
{ name: 'demo real', png: real },
{ name: 'L real', report: 'L', png: real },
]
const rows = [
'| Run | views | PDF bytes | parts | linear before | linear after | (after - before) / PDF | ms |',
'|---|---|---|---|---|---|---|---|',
]
for (const c of cases) {
const fresh = await freshRuntime()
const report = c.report
? JSON.parse(readFileSync(join(FIXTURES, `r4/reports/${c.report}.json`), 'utf8'))
: demo.report
if (c.report) report.sheet = (demo.report as unknown as { sheet: unknown }).sheet
const input = JSON.stringify({ ...demo, report, locale: 'fr' })
const before = fresh.memory()
const started = performance.now()
const session = unwrap(openExportSession(fresh.runtime, input), c.name)
let driven: Driven
try {
driven = drive(session, () => c.png)
} finally {
session.free()
}
const ms = performance.now() - started
const after = fresh.memory()
const views = driven.requests.filter((r) => r.key.startsWith('view:')).length
rows.push(
`| ${c.name} | ${views} | ${driven.pdf.byteLength} | ${driven.parts} | ${before} | ${after} | ${((after - before) / driven.pdf.byteLength).toFixed(2)} | ${ms.toFixed(0)} |`,
)
expect(driven.warnings, c.name).toEqual([])
// The rasters never accumulate: what the instance holds beyond the file is the
// parsed input (report, snapshot, trees), one decode transient (RGBA, raw, RGB and
// alpha planes of a 1200 × 943 raster, about 10 MB) and the open chunk's slack (up
// to 8 MiB), whatever the number of views; the PNGs of L real alone weigh 64 MB.
const overhead = after - before - driven.pdf.byteLength
expect(overhead, `${c.name}: held beyond the file`).toBeLessThan(24 * 1024 * 1024)
}
console.info(`[core-wasm] R4.6 session memory (fresh instance per run)\n${rows.join('\n')}`)
}, 600_000)
})
+7
View File
@@ -0,0 +1,7 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { RasterRequest } from "./RasterRequest";
/**
* What `next()` answers.
*/
export type ExportStep = { "step": "rasters", requests: Array<RasterRequest>, } | { "step": "done" };
+10
View File
@@ -0,0 +1,10 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
/**
* One raster the driver owes the session: the SVG text to draw and its pixel size.
*/
export type RasterRequest = {
/**
* `logo:<slot>` or `view:<observationId>:<viewId>`; answered once.
*/
key: string, svg: string, widthPx: number, heightPx: number, };
@@ -0,0 +1,6 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
/**
* A call the state machine refuses (closed kinds).
*/
export type SessionRefusal = "unknown_key" | "duplicate_key" | "out_of_order";
+3
View File
@@ -22,6 +22,7 @@ export type { DraftRestore } from './DraftRestore'
export type { DraftRestoreInput } from './DraftRestoreInput'
export type { ExportErrorKind } from './ExportErrorKind'
export type { ExportInput } from './ExportInput'
export type { ExportStep } from './ExportStep'
export type { ExportWarning } from './ExportWarning'
export type { Finding } from './Finding'
export type { FindingCategory } from './FindingCategory'
@@ -60,6 +61,7 @@ export type { PngAsset } from './PngAsset'
export type { Point } from './Point'
export type { Polygon } from './Polygon'
export type { Practitioner } from './Practitioner'
export type { RasterRequest } from './RasterRequest'
export type { RebaseOutcome } from './RebaseOutcome'
export type { Reference } from './Reference'
export type { ReferenceType } from './ReferenceType'
@@ -69,6 +71,7 @@ export type { ReportIdInput } from './ReportIdInput'
export type { ReportIndexInput } from './ReportIndexInput'
export type { ReportZone } from './ReportZone'
export type { ReviewStatus } from './ReviewStatus'
export type { SessionRefusal } from './SessionRefusal'
export type { Severity } from './Severity'
export type { SheetCheck } from './SheetCheck'
export type { SheetIndexData } from './SheetIndexData'