diff --git a/apps/app/src/core/worker/client.ts b/apps/app/src/core/worker/client.ts index f67a8165..cd09c60c 100644 --- a/apps/app/src/core/worker/client.ts +++ b/apps/app/src/core/worker/client.ts @@ -3,20 +3,23 @@ import { PROTOCOL_VERSION, type ImportDiag, type ImportTaskKind, + type WorkerMessage, } from './protocol' +import { CHUNK_BYTES, chunkSource, type ChunkSource } from './reader' import { PausableTimer, type Visibility, documentVisibility } from './timers' /** * The page side of one import (R4.3, plan §B.1.5 items 1, 3 and 5): creates the worker * in a try/catch (a `SecurityError` is synchronous), installs the listeners before posting, - * arms the start timer until `ready` then the task timer, settles once on the first of - * `result`, `failed`, an `error` event, a timer, or a cancel, and terminates the worker in - * every case. Everything after the first settlement is ignored. + * arms the start timer until `ready` then the task timer, feeds the file chunk by chunk + * (one travelling buffer, each chunk acknowledged before the next), settles once on the + * first of `result`, `failed`, an `error` event, a read failure, a timer, or a cancel, and + * terminates the worker in every case. Everything after the first settlement is ignored. */ /** What `new Worker(…)` gives, narrowed to what the client uses (a test fake fits). */ export interface WorkerLike { - postMessage(message: unknown): void + postMessage(message: unknown, transfer?: Transferable[]): void terminate(): void onmessage: ((event: MessageEvent) => void) | null onerror: ((event: ErrorEvent) => void) | null @@ -55,6 +58,8 @@ export interface ImportClientOptions { /** From `ready` to the result (plan §B.3: 20 s hypothesis, set at J1). */ taskTimeoutMs: number visibility?: Visibility + /** Test seam: the file's chunks. */ + source?: (file: File) => ChunkSource } export interface ImportRun { @@ -71,67 +76,144 @@ function reasonOf(phase: 'start' | 'task', kind: string): ImportFailureReason { return 'engine' } -export function runImportTask( - input: ImportTaskInput, - options: ImportClientOptions, -): ImportRun { - let worker: WorkerLike | null = null - let timer: PausableTimer | null = null - let ready = false - let done = false - let resolve!: (settlement: ImportSettlement) => void - const settled = new Promise((r) => { - resolve = r +/** One import's moving parts, settled once. */ +class ImportClient { + private worker: WorkerLike | null = null + private timer: PausableTimer | null = null + private source: ChunkSource | null = null + private awaitingAck: ((bytes: ArrayBuffer) => void) | null = null + private ready = false + private done = false + private resolve!: (settlement: ImportSettlement) => void + readonly settled = new Promise((resolve) => { + this.resolve = resolve }) - const visibility = options.visibility ?? documentVisibility - const settle = (settlement: ImportSettlement) => { - if (done) return - done = true - timer?.clear() - timer = null - worker?.terminate() - worker = null - resolve(settlement) + constructor( + private readonly input: ImportTaskInput, + private readonly options: ImportClientOptions, + ) {} + + start(): void { + try { + this.worker = this.options.spawn() + } catch (error) { + this.fail( + 'worker_unavailable', + error instanceof Error ? error.name : 'spawn failed', + ) + return + } + this.worker.onmessage = (event) => this.onMessage(event.data) + // An `error` event is never masked: it settles the import (and reaches the console). + this.worker.onerror = (event) => + this.fail(this.ready ? 'crashed' : 'worker_unavailable', event.message) + this.worker.onmessageerror = () => + this.fail(this.ready ? 'engine' : 'worker_unavailable', 'messageerror') + this.arm(this.options.startTimeoutMs, 'worker_unavailable') + const { task, file, sheet, module } = this.input + this.worker.postMessage({ + v: PROTOCOL_VERSION, + type: 'task', + task, + size: file.size, + sheet, + module, + }) } - const fail = (reason: ImportFailureReason, detail: string) => - settle({ kind: 'failed', reason, detail }) - const arm = (ms: number, reason: ImportFailureReason) => { - timer?.clear() - timer = new PausableTimer( + + cancel(): void { + this.fail('cancelled', 'cancelled') + } + + private onMessage(data: unknown): void { + if (this.done || !isWorkerMessage(data)) return + this.dispatch(data) + } + + private dispatch(message: WorkerMessage): void { + switch (message.type) { + case 'ready': + this.ready = true + this.arm(this.options.taskTimeoutMs, 'timeout') + void this.feed() + return + case 'ack': + this.awaitingAck?.(message.bytes) + this.awaitingAck = null + return + case 'result': + this.settle({ kind: 'result', out: message.out, diag: message.diag }) + return + case 'failed': + this.fail(reasonOf(message.phase, message.kind), message.detail) + } + } + + /** Reads the file chunk by chunk into one buffer the worker hands back after each. */ + private async feed(): Promise { + this.source = (this.options.source ?? chunkSource)(this.input.file) + let buffer = new ArrayBuffer(CHUNK_BYTES) + try { + for (;;) { + const chunk = await this.source.next(buffer) + if (this.done) return + if (chunk === null) break + buffer = await this.send(chunk.bytes, chunk.length) + if (this.done) return + } + this.worker?.postMessage({ v: PROTOCOL_VERSION, type: 'end' }) + } catch (error) { + this.fail( + 'read', + error instanceof Error ? `${error.name}: ${error.message}` : 'read', + ) + } + } + + /** Posts a chunk (transferred) and resolves with its buffer once acknowledged. */ + private send(bytes: ArrayBuffer, length: number): Promise { + return new Promise((resolve) => { + this.awaitingAck = resolve + this.worker?.postMessage( + { v: PROTOCOL_VERSION, type: 'chunk', bytes, length }, + [bytes], + ) + }) + } + + private arm(ms: number, reason: ImportFailureReason): void { + this.timer?.clear() + const visibility = this.options.visibility ?? documentVisibility + this.timer = new PausableTimer( ms, - () => fail(reason, `${reason} after ${ms} ms`), + () => this.fail(reason, `${reason} after ${ms} ms`), visibility, ) } - try { - worker = options.spawn() - } catch (error) { - fail( - 'worker_unavailable', - error instanceof Error ? error.name : 'spawn failed', - ) - return { settled, cancel: () => fail('cancelled', 'cancelled') } + private fail(reason: ImportFailureReason, detail: string): void { + this.settle({ kind: 'failed', reason, detail }) } - worker.onmessage = (event) => { - const data: unknown = event.data - if (!isWorkerMessage(data)) return - if (data.type === 'ready') { - ready = true - arm(options.taskTimeoutMs, 'timeout') - } else if (data.type === 'result') { - settle({ kind: 'result', out: data.out, diag: data.diag }) - } else { - fail(reasonOf(data.phase, data.kind), data.detail) - } + + private settle(settlement: ImportSettlement): void { + if (this.done) return + this.done = true + this.timer?.clear() + this.timer = null + this.worker?.terminate() + this.worker = null + void this.source?.cancel() + this.source = null + this.resolve(settlement) } - // An `error` event is never masked: it settles the import (and reaches the console). - worker.onerror = (event) => - fail(ready ? 'crashed' : 'worker_unavailable', event.message) - worker.onmessageerror = () => - fail(ready ? 'engine' : 'worker_unavailable', 'messageerror') - arm(options.startTimeoutMs, 'worker_unavailable') - worker.postMessage({ v: PROTOCOL_VERSION, type: 'task', ...input }) - return { settled, cancel: () => fail('cancelled', 'cancelled') } +} + +export function runImportTask( + input: ImportTaskInput, + options: ImportClientOptions, +): ImportRun { + const client = new ImportClient(input, options) + client.start() + return { settled: client.settled, cancel: () => client.cancel() } } diff --git a/apps/app/src/core/worker/handler.ts b/apps/app/src/core/worker/handler.ts index dda8f4cd..22c5f003 100644 --- a/apps/app/src/core/worker/handler.ts +++ b/apps/app/src/core/worker/handler.ts @@ -3,8 +3,9 @@ import { closedRefusal } from '@faunosteo/core-wasm' import { describeError, failureKindOf, + isPageMessage, PROTOCOL_VERSION, - type ImportTaskKind, + type ChunkMessage, type ImportTaskMessage, type WorkerFailurePhase, type WorkerMessage, @@ -13,9 +14,9 @@ import { /** * The import worker's logic (R4.3, plan §B.1.5 items 2 and 4), apart from the worker * global so that vitest runs it in-process on the Node module: the engine is injected. - * One task per worker: instantiate, answer `ready`, build the sheet handle, stream the - * file into the module's memory (the file lives once, never as an ArrayBuffer here), run - * the core, answer `result`; every throwable becomes one `failed` with its phase and kind. + * One task per worker: instantiate, answer `ready`, build the sheet handle, sink the + * chunks the page reads (one copy, in the module's memory), run the core on `end`, answer + * `result`; every throwable becomes one `failed` with its phase and kind. */ /** What the glue gives once instantiated. */ @@ -30,10 +31,12 @@ export type EngineLoader = ( module: WebAssembly.Module | null, ) => Promise -/** The caps the worker checks before reading a byte (the core enforces the same). */ +/** The caps the worker checks before reserving a byte (the core enforces the same). */ export const MAX_PDF_BYTES = 50 * 1024 * 1024 export const MAX_COPY_BYTES = 16 * 1024 * 1024 +export type Post = (message: WorkerMessage, transfer?: Transferable[]) => void + /** The core's failure envelope, for the caps the worker applies itself. */ function refusal(kind: string, detail: string): string { return JSON.stringify({ ok: false, kind, detail }) @@ -52,99 +55,118 @@ function sheetHandle(engine: WorkerEngine, sheet: string): GlueSheetHandle { } } -interface Gathered { - bytes: GlueFileBytes - chunks: number -} +/** One task's state, from the task message to its settlement. */ +class ImportTask { + private engine: WorkerEngine | null = null + private handle: GlueSheetHandle | null = null + private sink: GlueFileBytes | null = null + private chunks = 0 + private settled = false + private readonly started = performance.now() -/** Streams the file into the module's memory, one chunk live at a time. */ -async function gather( - engine: WorkerEngine, - file: File, -): Promise { - const bytes = new engine.FileBytes(file.size) - const reader = file.stream().getReader() - let chunks = 0 - for (;;) { - const { done, value } = await reader.read() - if (done) return { bytes, chunks } - chunks += 1 - // A push refused means the file is not the size it announced: not a Faunosteo file. - if (!bytes.push(value)) { - await reader.cancel() - bytes.free() - return null + constructor( + private readonly task: ImportTaskMessage, + private readonly load: EngineLoader, + private readonly post: Post, + ) {} + + /** Instantiates, answers `ready`, reserves the sink; a cap refusal settles at once. */ + async start(): Promise { + let phase: WorkerFailurePhase = 'start' + try { + this.engine = await this.load(this.task.module) + this.post({ v: PROTOCOL_VERSION, type: 'ready' }) + phase = 'task' + this.handle = sheetHandle(this.engine, this.task.sheet) + const cap = this.task.task === 'pdf' ? MAX_PDF_BYTES : MAX_COPY_BYTES + if (this.task.size > cap) { + this.result(refusal(this.kind(), 'over the size cap')) + return + } + this.sink = new this.engine.FileBytes(this.task.size) + } catch (error) { + this.fail(phase, error) } } -} -/** The task's outcome envelope text and the chunk count. */ -async function runTask( - engine: WorkerEngine, - handle: GlueSheetHandle, - task: ImportTaskKind, - file: File, -): Promise<{ out: string; chunks: number }> { - const cap = task === 'pdf' ? MAX_PDF_BYTES : MAX_COPY_BYTES - const kind = task === 'pdf' ? 'invalid_pdf' : 'invalid' - if (file.size > cap) - return { out: refusal(kind, 'over the size cap'), chunks: 0 } - const gathered = await gather(engine, file) - if (!gathered) - return { out: refusal(kind, 'size changed while reading'), chunks: 0 } - const out = - task === 'pdf' - ? handle.import_pdf_bytes(gathered.bytes) - : handle.restore_copy_bytes(gathered.bytes) - return { out, chunks: gathered.chunks } -} + /** Sinks one chunk and hands its buffer back; a refused push means the size changed. */ + chunk(message: ChunkMessage): void { + if (this.settled || !this.sink) return + const view = new Uint8Array(message.bytes, 0, message.length) + if (!this.sink.push(view)) { + this.sink.free() + this.sink = null + this.result(refusal(this.kind(), 'size changed while reading')) + return + } + this.chunks += 1 + this.post({ v: PROTOCOL_VERSION, type: 'ack', bytes: message.bytes }, [ + message.bytes, + ]) + } -/** - * The handler of the task message: `load` instantiates the module (the posted one, or - * fetched), `post` sends the answers. Resolves once the task has settled either way. - */ -export function createWorkerHandler( - load: EngineLoader, - post: (message: WorkerMessage) => void, -): (data: unknown) => Promise { - const fail = (phase: WorkerFailurePhase, error: unknown) => - post({ + /** Runs the core on the gathered bytes. */ + end(): void { + if (this.settled || !this.sink || !this.handle) return + try { + const sink = this.sink + this.sink = null + const out = + this.task.task === 'pdf' + ? this.handle.import_pdf_bytes(sink) + : this.handle.restore_copy_bytes(sink) + this.result(out) + } catch (error) { + this.fail('task', error) + } + } + + private kind(): string { + return this.task.task === 'pdf' ? 'invalid_pdf' : 'invalid' + } + + private result(out: string): void { + this.settled = true + const linearBytes = this.engine?.memory?.buffer.byteLength ?? null + const diag = { + ms: performance.now() - this.started, + linearBytes, + chunks: this.chunks, + } + this.post({ v: PROTOCOL_VERSION, type: 'result', out, diag }) + } + + private fail(phase: WorkerFailurePhase, error: unknown): void { + this.settled = true + this.post({ v: PROTOCOL_VERSION, type: 'failed', phase, kind: failureKindOf(error), detail: describeError(error), }) - return async (data) => { - if (!isTaskMessage(data)) return - let phase: WorkerFailurePhase = 'start' - try { - const engine = await load(data.module) - post({ v: PROTOCOL_VERSION, type: 'ready' }) - phase = 'task' - const started = performance.now() - const handle = sheetHandle(engine, data.sheet) - const { out, chunks } = await runTask( - engine, - handle, - data.task, - data.file, - ) - const linearBytes = engine.memory?.buffer.byteLength ?? null - const diag = { ms: performance.now() - started, linearBytes, chunks } - post({ v: PROTOCOL_VERSION, type: 'result', out, diag }) - } catch (error) { - fail(phase, error) - } } } -function isTaskMessage(data: unknown): data is ImportTaskMessage { - if (typeof data !== 'object' || data === null) return false - const record = data as { v?: unknown; type?: unknown; task?: unknown } - return ( - record.v === PROTOCOL_VERSION && - record.type === 'task' && - (record.task === 'pdf' || record.task === 'copy') - ) +/** + * The handler of the page's messages: `load` instantiates the module (the posted one, or + * fetched), `post` sends the answers. Messages outside the protocol are ignored. + */ +export function createWorkerHandler( + load: EngineLoader, + post: Post, +): (data: unknown) => Promise { + let task: ImportTask | null = null + return async (data) => { + if (!isPageMessage(data)) return + if (data.type === 'task') { + if (task) return + task = new ImportTask(data, load, post) + await task.start() + } else if (data.type === 'chunk') { + task?.chunk(data) + } else { + task?.end() + } + } } diff --git a/apps/app/src/core/worker/import.worker.ts b/apps/app/src/core/worker/import.worker.ts index d3e52c25..4db0d6b9 100644 --- a/apps/app/src/core/worker/import.worker.ts +++ b/apps/app/src/core/worker/import.worker.ts @@ -27,8 +27,8 @@ const loadEngine = async ( return { SheetHandle, FileBytes, memory: exports.memory ?? null } } -const handle = createWorkerHandler(loadEngine, (message) => - scope.postMessage(message), +const handle = createWorkerHandler(loadEngine, (message, transfer = []) => + scope.postMessage(message, transfer), ) scope.onmessage = (event: MessageEvent) => { diff --git a/apps/app/src/core/worker/protocol.ts b/apps/app/src/core/worker/protocol.ts index dbeca340..9426a95d 100644 --- a/apps/app/src/core/worker/protocol.ts +++ b/apps/app/src/core/worker/protocol.ts @@ -1,7 +1,8 @@ /** * Messages between the page and the import worker (R4.3, plan §B.1.5), tagged `v: 1`. - * The page posts one task; the worker answers `ready` once its instance runs, then one - * `result` or one `failed`. Anything after the first settlement is ignored by the page. + * The page posts one task, then the file chunk by chunk, then `end`; the worker answers + * `ready` once its instance runs, `ack` per chunk, then one `result` or one `failed`. + * Anything after the first settlement is ignored by the page. */ export const PROTOCOL_VERSION = 1 @@ -12,19 +13,47 @@ export interface ImportTaskMessage { v: typeof PROTOCOL_VERSION type: 'task' task: ImportTaskKind - /** The picked file, never read on the page's thread. */ - file: File + /** The picked file's size: the worker reserves it and refuses more. */ + size: number /** The bundled sheet payload text the page's runtime builds its handle from. */ sheet: string /** The compiled module of the page's runtime when it holds one; else the worker fetches. */ module: WebAssembly.Module | null } +/** + * One chunk of the file, read on the page's thread into a buffer the worker hands back + * with `ack` (both transferred, never copied): the page holds one chunk at a time, the + * worker's sink the only whole copy. Measured at R4.3a: reading a file inside a WebKit + * worker costs about 180 MiB whatever the method; read on the page and streamed, 106. + */ +export interface ChunkMessage { + v: typeof PROTOCOL_VERSION + type: 'chunk' + bytes: ArrayBuffer + length: number +} + +/** The page read the whole file: the worker runs the core. */ +export interface EndMessage { + v: typeof PROTOCOL_VERSION + type: 'end' +} + +export type PageMessage = ImportTaskMessage | ChunkMessage | EndMessage + export interface WorkerReadyMessage { v: typeof PROTOCOL_VERSION type: 'ready' } +/** The chunk is in the sink; its buffer comes back for the next read. */ +export interface WorkerAckMessage { + v: typeof PROTOCOL_VERSION + type: 'ack' + bytes: ArrayBuffer +} + export interface WorkerResultMessage { v: typeof PROTOCOL_VERSION type: 'result' @@ -60,21 +89,33 @@ export interface WorkerFailedMessage { export type WorkerMessage = | WorkerReadyMessage + | WorkerAckMessage | WorkerResultMessage | WorkerFailedMessage -/** True for a message of this protocol, from a worker of the same build. */ -export function isWorkerMessage(data: unknown): data is WorkerMessage { +const WORKER_TYPES = new Set(['ready', 'ack', 'result', 'failed']) +const PAGE_TYPES = new Set(['task', 'chunk', 'end']) + +function ofProtocol(data: unknown, types: Set): boolean { if (typeof data !== 'object' || data === null) return false const record = data as { v?: unknown; type?: unknown } return ( record.v === PROTOCOL_VERSION && - (record.type === 'ready' || - record.type === 'result' || - record.type === 'failed') + typeof record.type === 'string' && + types.has(record.type) ) } +/** True for a message of this protocol, from a worker of the same build. */ +export function isWorkerMessage(data: unknown): data is WorkerMessage { + return ofProtocol(data, WORKER_TYPES) +} + +/** True for a message of this protocol, from the page. */ +export function isPageMessage(data: unknown): data is PageMessage { + return ofProtocol(data, PAGE_TYPES) +} + /** Error name and first message line, bounded, for a `failed` message. */ export function describeError(error: unknown): string { const name = error instanceof Error ? error.name : 'Error' diff --git a/apps/app/src/core/worker/reader.ts b/apps/app/src/core/worker/reader.ts new file mode 100644 index 00000000..cadaac1f --- /dev/null +++ b/apps/app/src/core/worker/reader.ts @@ -0,0 +1,60 @@ +/** + * The page's read of the picked file, one chunk at a time into a buffer that travels to + * the worker and back (R4.3a measurement: a file read inside a WebKit worker costs about + * 180 MiB whatever the method; read here, 106, and 14 on the page alone). A BYOB reader + * fills the buffer the worker handed back; without BYOB support (happy-dom), the default + * reader's chunks are copied into it. Nothing here parses a byte. + */ + +export const CHUNK_BYTES = 1024 * 1024 + +export interface ChunkSource { + /** The next chunk's length in `buffer`, or null at the end; `buffer` is consumed. */ + next( + buffer: ArrayBuffer, + ): Promise<{ bytes: ArrayBuffer; length: number } | null> + cancel(): Promise +} + +type ByobReader = ReadableStreamBYOBReader +type DefaultReader = ReadableStreamDefaultReader + +function byobSource(reader: ByobReader): ChunkSource { + return { + async next(buffer) { + const { done, value } = await reader.read(new Uint8Array(buffer)) + if (done || !value) return null + return { bytes: value.buffer as ArrayBuffer, length: value.byteLength } + }, + cancel: () => reader.cancel().catch(() => undefined), + } +} + +/** The default reader's chunks copied into the travelling buffer, split when longer. */ +function defaultSource(reader: DefaultReader): ChunkSource { + let pending: Uint8Array | null = null + return { + async next(buffer) { + if (!pending || pending.byteLength === 0) { + const { done, value } = await reader.read() + if (done || !value) return null + pending = value + } + const length = Math.min(pending.byteLength, buffer.byteLength) + new Uint8Array(buffer, 0, length).set(pending.subarray(0, length)) + pending = pending.subarray(length) + return { bytes: buffer, length } + }, + cancel: () => reader.cancel().catch(() => undefined), + } +} + +/** A chunk source over `file`, BYOB when the engine offers it. */ +export function chunkSource(file: File): ChunkSource { + const stream = file.stream() + try { + return byobSource(stream.getReader({ mode: 'byob' })) + } catch { + return defaultSource(stream.getReader()) + } +} diff --git a/apps/app/tests/helpers/importWorker.ts b/apps/app/tests/helpers/importWorker.ts index 8f06a75e..8cca5d91 100644 --- a/apps/app/tests/helpers/importWorker.ts +++ b/apps/app/tests/helpers/importWorker.ts @@ -39,6 +39,8 @@ export interface FakeWorker extends WorkerLike { received: unknown[] /** Every message the worker posted, dropped or delivered. */ posted: WorkerMessage[] + /** The posted message types without the per-chunk `ack`s. */ + sequence(): string[] } export function inProcessWorker( @@ -54,6 +56,9 @@ export function inProcessWorker( terminate() { worker.terminated = true }, + sequence() { + return worker.posted.map((m) => m.type).filter((t) => t !== 'ack') + }, postMessage(message) { worker.received.push(message) void handle(message) diff --git a/apps/app/tests/unit/core.importClient.test.ts b/apps/app/tests/unit/core.importClient.test.ts index 0ebe536d..aa445447 100644 --- a/apps/app/tests/unit/core.importClient.test.ts +++ b/apps/app/tests/unit/core.importClient.test.ts @@ -1,19 +1,23 @@ /** * R4.3: the page side of one import (plan §B.1.5 items 1, 3, 5): the worker created in a * try/catch, listeners before the post, the start timer until `ready`, the task timer - * after, settle once on the first of result / failed / error event / timer / cancel, the - * worker terminated in every case, later messages ignored, timers paused while hidden. + * after, the file fed chunk by chunk (one travelling buffer, each chunk acknowledged), + * settle once on the first of result / failed / error event / read failure / timer / + * cancel, the worker terminated in every case, later messages ignored, timers paused + * while hidden. */ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { runImportTask, + type ImportClientOptions, type ImportSettlement, type WorkerLike, } from '@/core/worker/client' import { PROTOCOL_VERSION } from '@/core/worker/protocol' +import type { ChunkSource } from '@/core/worker/reader' import type { Visibility } from '@/core/worker/timers' -/** A scripted worker: the test drives its messages and events. */ +/** A scripted worker: the test drives its messages and events; chunks are acknowledged. */ function scriptedWorker() { const worker: WorkerLike & { terminated: number; sent: unknown[] } = { terminated: 0, @@ -21,7 +25,13 @@ function scriptedWorker() { onmessage: null, onerror: null, onmessageerror: null, - postMessage: (m) => worker.sent.push(m), + postMessage: (m) => { + worker.sent.push(m) + const message = m as { type?: string; bytes?: ArrayBuffer } + if (message.type === 'chunk') { + emit({ v: PROTOCOL_VERSION, type: 'ack', bytes: message.bytes }) + } + }, terminate: () => { worker.terminated += 1 }, @@ -46,6 +56,7 @@ function scriptedWorker() { detail: `${kind}!`, }), error: (message: string) => worker.onerror?.({ message } as ErrorEvent), + types: () => worker.sent.map((m) => (m as { type: string }).type), } } @@ -65,8 +76,37 @@ function visibility(): Visibility & { set: (hidden: boolean) => void } { } } -const file = new File([new Uint8Array([1])], 'a.pdf') +/** A chunk source over `parts`, each filled into the travelling buffer. */ +function partsSource(parts: Uint8Array[]): ChunkSource { + let at = 0 + return { + async next(buffer) { + const part = parts[at] + if (!part) return null + at += 1 + new Uint8Array(buffer, 0, part.byteLength).set(part) + return { bytes: buffer, length: part.byteLength } + }, + cancel: async () => {}, + } +} + +const file = new File([new Uint8Array([1, 2, 3])], 'a.pdf') const input = { task: 'pdf' as const, file, sheet: '{}', module: null } +const empty = () => partsSource([]) + +function options( + spawn: () => WorkerLike, + patch: Partial = {}, +): ImportClientOptions { + return { + spawn, + startTimeoutMs: 100, + taskTimeoutMs: 100, + source: empty, + ...patch, + } +} async function settled(run: { settled: Promise @@ -79,16 +119,21 @@ describe('import client', () => { beforeEach(() => vi.useFakeTimers()) afterEach(() => vi.useRealTimers()) - it('posts the task after installing the listeners, then settles on the result and terminates', async () => { + it('posts the task, feeds the chunks then end, settles on the result and terminates', async () => { const s = scriptedWorker() - const run = runImportTask(input, { - spawn: () => s.worker, - startTimeoutMs: 100, - taskTimeoutMs: 100, - }) + const parts = [new Uint8Array([1, 2]), new Uint8Array([3])] + const run = runImportTask( + input, + options(() => s.worker, { source: () => partsSource(parts) }), + ) expect(s.worker.onmessage).not.toBeNull() - expect(s.worker.sent).toEqual([{ v: 1, type: 'task', ...input }]) + expect(s.worker.sent).toEqual([ + { v: 1, type: 'task', task: 'pdf', size: 3, sheet: '{}', module: null }, + ]) s.ready() + await vi.advanceTimersByTimeAsync(0) + expect(s.types()).toEqual(['task', 'chunk', 'chunk', 'end']) + expect((s.worker.sent[1] as { length: number }).length).toBe(2) s.result('{"ok":true}') expect(await settled(run)).toMatchObject({ kind: 'result', @@ -98,13 +143,12 @@ describe('import client', () => { }) it('is worker_unavailable when the spawn throws, fails before ready, or never answers ready', async () => { - const thrown = runImportTask(input, { - spawn: () => { + const thrown = runImportTask( + input, + options(() => { throw new DOMException('blocked', 'SecurityError') - }, - startTimeoutMs: 100, - taskTimeoutMs: 100, - }) + }), + ) expect(await settled(thrown)).toEqual({ kind: 'failed', reason: 'worker_unavailable', @@ -112,11 +156,10 @@ describe('import client', () => { }) const early = scriptedWorker() - const failedEarly = runImportTask(input, { - spawn: () => early.worker, - startTimeoutMs: 100, - taskTimeoutMs: 100, - }) + const failedEarly = runImportTask( + input, + options(() => early.worker), + ) early.failed('start', 'other') expect(await settled(failedEarly)).toMatchObject({ kind: 'failed', @@ -124,11 +167,10 @@ describe('import client', () => { }) const silent = scriptedWorker() - const timedOut = runImportTask(input, { - spawn: () => silent.worker, - startTimeoutMs: 100, - taskTimeoutMs: 1000, - }) + const timedOut = runImportTask( + input, + options(() => silent.worker, { taskTimeoutMs: 1000 }), + ) await vi.advanceTimersByTimeAsync(99) expect(silent.worker.terminated).toBe(0) await vi.advanceTimersByTimeAsync(2) @@ -140,20 +182,20 @@ describe('import client', () => { }) it('maps task failures: trap → crashed, read → read, sheet/other → engine, timer → timeout', async () => { - for (const [kind, reason] of [ + const cases: Array<[string, string]> = [ ['trap', 'crashed'], ['read', 'read'], ['sheet', 'engine'], ['other', 'engine'], - ]) { + ] + for (const [kind, reason] of cases) { const s = scriptedWorker() - const run = runImportTask(input, { - spawn: () => s.worker, - startTimeoutMs: 100, - taskTimeoutMs: 100, - }) + const run = runImportTask( + input, + options(() => s.worker), + ) s.ready() - s.failed('task', kind!) + s.failed('task', kind) expect(await settled(run)).toEqual({ kind: 'failed', reason, @@ -162,11 +204,10 @@ describe('import client', () => { expect(s.worker.terminated).toBe(1) } const slow = scriptedWorker() - const run = runImportTask(input, { - spawn: () => slow.worker, - startTimeoutMs: 50, - taskTimeoutMs: 200, - }) + const run = runImportTask( + input, + options(() => slow.worker, { startTimeoutMs: 50, taskTimeoutMs: 200 }), + ) await vi.advanceTimersByTimeAsync(40) slow.ready() await vi.advanceTimersByTimeAsync(199) @@ -180,11 +221,10 @@ describe('import client', () => { it('settles once: a result after the timeout, a failure after the result, are ignored', async () => { const s = scriptedWorker() - const run = runImportTask(input, { - spawn: () => s.worker, - startTimeoutMs: 100, - taskTimeoutMs: 100, - }) + const run = runImportTask( + input, + options(() => s.worker), + ) s.ready() await vi.advanceTimersByTimeAsync(101) s.result('{"ok":true}') @@ -195,11 +235,10 @@ describe('import client', () => { expect(s.worker.terminated).toBe(1) const t = scriptedWorker() - const second = runImportTask(input, { - spawn: () => t.worker, - startTimeoutMs: 100, - taskTimeoutMs: 100, - }) + const second = runImportTask( + input, + options(() => t.worker), + ) t.ready() t.result('{"ok":true}') t.failed('task', 'trap') @@ -210,11 +249,10 @@ describe('import client', () => { it('never masks an error event: worker_unavailable before ready, crashed after', async () => { const before = scriptedWorker() - const first = runImportTask(input, { - spawn: () => before.worker, - startTimeoutMs: 100, - taskTimeoutMs: 100, - }) + const first = runImportTask( + input, + options(() => before.worker), + ) before.error('script failed') expect(await settled(first)).toEqual({ kind: 'failed', @@ -222,11 +260,10 @@ describe('import client', () => { detail: 'script failed', }) const after = scriptedWorker() - const second = runImportTask(input, { - spawn: () => after.worker, - startTimeoutMs: 100, - taskTimeoutMs: 100, - }) + const second = runImportTask( + input, + options(() => after.worker), + ) after.ready() after.error('boom') expect(await settled(second)).toEqual({ @@ -236,13 +273,31 @@ describe('import client', () => { }) }) + it('a file the page cannot read settles read and terminates the worker', async () => { + const s = scriptedWorker() + const failing: ChunkSource = { + next: () => Promise.reject(new DOMException('gone', 'NotReadableError')), + cancel: async () => {}, + } + const run = runImportTask( + input, + options(() => s.worker, { source: () => failing }), + ) + s.ready() + expect(await settled(run)).toEqual({ + kind: 'failed', + reason: 'read', + detail: 'NotReadableError: gone', + }) + expect(s.worker.terminated).toBe(1) + }) + it('cancel terminates the worker and settles cancelled, once', async () => { const s = scriptedWorker() - const run = runImportTask(input, { - spawn: () => s.worker, - startTimeoutMs: 100, - taskTimeoutMs: 100, - }) + const run = runImportTask( + input, + options(() => s.worker), + ) s.ready() run.cancel() run.cancel() @@ -257,12 +312,10 @@ describe('import client', () => { it('pauses the task timer while the page is hidden', async () => { const s = scriptedWorker() const vis = visibility() - const run = runImportTask(input, { - spawn: () => s.worker, - startTimeoutMs: 100, - taskTimeoutMs: 100, - visibility: vis, - }) + const run = runImportTask( + input, + options(() => s.worker, { visibility: vis }), + ) s.ready() await vi.advanceTimersByTimeAsync(50) vis.set(true) diff --git a/apps/app/tests/unit/core.importWorker.test.ts b/apps/app/tests/unit/core.importWorker.test.ts index 7d9c4bcd..0fcf0155 100644 --- a/apps/app/tests/unit/core.importWorker.test.ts +++ b/apps/app/tests/unit/core.importWorker.test.ts @@ -27,6 +27,7 @@ async function sheetText(): Promise { interface Run { messages: WorkerMessage[] + acks: number } async function run( @@ -41,11 +42,31 @@ async function run( v: PROTOCOL_VERSION, type: 'task', task, - file, + size: file.size, sheet: sheet ?? (await sheetText()), module: null, }) - return { messages } + // The page's part: the file in 4 KiB chunks, then `end` (no transfer in-process). + if ( + messages.some((m) => m.type === 'ready') && + !messages.some((m) => m.type !== 'ready') + ) { + const bytes = new Uint8Array(await file.arrayBuffer()) + for (let at = 0; at < bytes.byteLength; at += 4096) { + const chunk = bytes.slice(at, at + 4096) + await handle({ + v: PROTOCOL_VERSION, + type: 'chunk', + bytes: chunk.buffer, + length: chunk.byteLength, + }) + } + await handle({ v: PROTOCOL_VERSION, type: 'end' }) + } + return { + messages: messages.filter((m) => m.type !== 'ack'), + acks: messages.filter((m) => m.type === 'ack').length, + } } function envelope(messages: WorkerMessage[]): { @@ -62,8 +83,9 @@ function envelope(messages: WorkerMessage[]): { describe('import worker handler (in-process, shipped module)', () => { it('answers ready then a result for a Faunosteo PDF, streamed in chunks', async () => { const file = new File([fixture('report-v3.bare.pdf')], 'bilan.pdf') - const { messages } = await run('pdf', file) + const { messages, acks } = await run('pdf', file) expect(messages.map((m) => m.type)).toEqual(['ready', 'result']) + expect(acks).toBe(Math.ceil(file.size / 4096)) const out = envelope(messages) expect(out.ok).toBe(true) expect( @@ -87,27 +109,39 @@ describe('import worker handler (in-process, shipped module)', () => { }) }) - it('refuses an oversized file before reading a byte', async () => { - const huge = { - size: MAX_PDF_BYTES + 1, - stream: () => { - throw new Error('read') - }, - } as unknown as File - expect(envelope((await run('pdf', huge)).messages)).toMatchObject({ + it('refuses an oversized file before any chunk, and a file whose size changes', async () => { + const huge = { size: MAX_PDF_BYTES + 1 } as unknown as File + const over = await run('pdf', huge) + expect(over.acks).toBe(0) + expect(envelope(over.messages)).toMatchObject({ ok: false, kind: 'invalid_pdf', }) - const copy = { - size: MAX_COPY_BYTES + 1, - stream: () => { - throw new Error('read') - }, - } as unknown as File + const copy = { size: MAX_COPY_BYTES + 1 } as unknown as File expect(envelope((await run('copy', copy)).messages)).toMatchObject({ ok: false, kind: 'invalid', }) + // Announced 3 bytes, 4 arrive: the sink refuses and the task settles as not a PDF. + const messages: WorkerMessage[] = [] + const handle = createWorkerHandler(nodeEngineLoader, (m) => + messages.push(m), + ) + await handle({ + v: PROTOCOL_VERSION, + type: 'task', + task: 'pdf', + size: 3, + sheet: await sheetText(), + module: null, + }) + await handle({ + v: PROTOCOL_VERSION, + type: 'chunk', + bytes: new ArrayBuffer(4), + length: 4, + }) + expect(envelope(messages)).toMatchObject({ ok: false, kind: 'invalid_pdf' }) }) it('gives the core kinds for a foreign PDF and garbage', async () => { @@ -150,28 +184,6 @@ describe('import worker handler (in-process, shipped module)', () => { }) }) - it('classifies a file that cannot be read as failed/task/read', async () => { - const error = new DOMException( - 'The file could not be read', - 'NotReadableError', - ) - const unreadable = { - size: 10, - stream: () => ({ - getReader: () => ({ - read: () => Promise.reject(error), - cancel: async () => {}, - }), - }), - } as unknown as File - const { messages } = await run('pdf', unreadable) - expect(messages[1]).toMatchObject({ - type: 'failed', - phase: 'task', - kind: 'read', - }) - }) - it('classifies a refused sheet as failed/task/sheet and a load failure as start', async () => { const file = new File([fixture('report-v3.bare.pdf')], 'bilan.pdf') const refused = await run( diff --git a/apps/app/tests/unit/routes.HomeRoute.import.test.tsx b/apps/app/tests/unit/routes.HomeRoute.import.test.tsx index 24f5e9c9..bc14e059 100644 --- a/apps/app/tests/unit/routes.HomeRoute.import.test.tsx +++ b/apps/app/tests/unit/routes.HomeRoute.import.test.tsx @@ -146,12 +146,17 @@ describe('HomeRoute import: success paths', () => { } expect(workers).toHaveLength(1) const task = workers[0]!.received[0] as ImportTaskMessage - expect(task).toMatchObject({ type: 'task', task: 'pdf' }) - expect(task.file).toBe(file) + expect(task).toMatchObject({ type: 'task', task: 'pdf', size: file.size }) expect(task.sheet).toBe(JSON.stringify(await bundledSheet())) expect(task.module).toBeInstanceOf(WebAssembly.Module) + // The page streams the file chunk by chunk, never whole: no arrayBuffer, no text. expect(read).not.toHaveBeenCalled() expect(text).not.toHaveBeenCalled() + const chunks = workers[0]!.received.filter( + (m) => (m as { type: string }).type === 'chunk', + ) + expect(chunks.length).toBeGreaterThanOrEqual(1) + expect(workers[0]!.received.at(-1)).toMatchObject({ type: 'end' }) const imported = resultOf(workers[0]!) expect(hydrate).toHaveBeenCalledTimes(1) const [session, index] = hydrate.mock.calls[0]! @@ -268,7 +273,7 @@ describe('HomeRoute import: a core failure reading the tab draft', () => { await mountAndPickPdf() await screen.findByText(fr['home.import.error_core_unavailable']!) // The worker read the file: the failure is the store's, after the result. - expect(workers[0]!.posted.map((m) => m.type)).toEqual(['ready', 'result']) + expect(workers[0]!.sequence()).toEqual(['ready', 'result']) expect(alertMock()).not.toHaveBeenCalled() expect(screen.queryByTestId('editor-route')).toBeNull() } finally { @@ -284,7 +289,7 @@ describe('HomeRoute import: a core failure reading the tab draft', () => { try { await mountAndPickPdf() await screen.findByTestId('editor-route') - expect(workers[0]!.posted.map((m) => m.type)).toEqual(['ready', 'result']) + expect(workers[0]!.sequence()).toEqual(['ready', 'result']) expect(alertMock()).toHaveBeenCalledWith(fr['session.held_blocked']) } finally { useReportStore.setState({ hydrateImport: real }) diff --git a/apps/app/tests/unit/routes.HomeRoute.rescueCopy.test.tsx b/apps/app/tests/unit/routes.HomeRoute.rescueCopy.test.tsx index 78cab04a..282ae3dd 100644 --- a/apps/app/tests/unit/routes.HomeRoute.rescueCopy.test.tsx +++ b/apps/app/tests/unit/routes.HomeRoute.rescueCopy.test.tsx @@ -194,7 +194,7 @@ describe('refusing a file', () => { await pick(copyFile(JSON.stringify(minimalReport(sheet)))) await screen.findByText(fr['home.import.error_core_unavailable']!) // The worker answered: the line is the core's kind, not a worker failure. - expect(workers[0]!.posted.map((m) => m.type)).toEqual(['ready', 'result']) + expect(workers[0]!.sequence()).toEqual(['ready', 'result']) expect( screen.queryByText(fr['home.import.error_invalid_copy']!), ).toBeNull() diff --git a/apps/app/tests/unit/routes.coreStopped.test.tsx b/apps/app/tests/unit/routes.coreStopped.test.tsx index e365a2b2..b5e408e1 100644 --- a/apps/app/tests/unit/routes.coreStopped.test.tsx +++ b/apps/app/tests/unit/routes.coreStopped.test.tsx @@ -319,7 +319,7 @@ describe('the stopped text on the Home import line and the New report alert', () await pickPdf() await screen.findByText(fr['home.import.error_core_unavailable']!) // The worker read the file: the line is the page's index failing after it. - expect(workers[0]!.posted.map((m) => m.type)).toEqual(['ready', 'result']) + expect(workers[0]!.sequence()).toEqual(['ready', 'result']) }) it('New report: the alert says the stopped text', async () => { diff --git a/apps/app/tests/unit/state.coreFailures.test.tsx b/apps/app/tests/unit/state.coreFailures.test.tsx index 0e68ba11..0a43a99d 100644 --- a/apps/app/tests/unit/state.coreFailures.test.tsx +++ b/apps/app/tests/unit/state.coreFailures.test.tsx @@ -172,7 +172,7 @@ describe('callers stop on a failed core call', () => { }) await screen.findByText(fr['home.import.error_core_unavailable']!) // The worker read the file; the page's core, needed to open it, did not load. - expect(workers[0]!.posted.map((m) => m.type)).toEqual(['ready', 'result']) + expect(workers[0]!.sequence()).toEqual(['ready', 'result']) expect(screen.queryByTestId('editor-route')).toBeNull() expect(useReportStore.getState().report).toBeNull() })