feat(app): R4.3a the page reads the file chunk by chunk and streams it to the worker's sink

Measured at R4.3a (RUST.md §10.5): reading a file inside a WebKit worker costs about
180 MiB of content RSS whatever the method (arrayBuffer, stream, BYOB), 14 MiB on the
page; a page-side BYOB read feeding the worker's FileBytes gives 106 MiB on a 50 MiB file
against 272 in the worker and 76 for the TypeScript. Protocol v1 gains chunk / ack / end:
one 1 MiB buffer travels (transferred) and comes back after each push, so the page holds
one chunk and the worker's sink the only whole copy; no byte is parsed on the page. The
worker refuses an oversize announced before any chunk and a size that changes while
reading. Tests adapted (handler, client with a scripted ack, read failure on the page).
This commit is contained in:
bchanot committed 2026-10-18 18:57:55 +02:00
1 parent 49591c53cd
commit e659924ec0
12 files changed
+552 -272

No files matched your search

+5
View File
@@ -39,6 +39,8 @@ export interface FakeWorker extends WorkerLike {
received: unknown[]
/** Every message the worker posted, dropped or delivered. */
posted: WorkerMessage[]
/** The posted message types without the per-chunk `ack`s. */
sequence(): string[]
}
export function inProcessWorker(
@@ -54,6 +56,9 @@ export function inProcessWorker(
terminate() {
worker.terminated = true
},
sequence() {
return worker.posted.map((m) => m.type).filter((t) => t !== 'ack')
},
postMessage(message) {
worker.received.push(message)
void handle(message)
+125 -72
View File
@@ -1,19 +1,23 @@
/**
* R4.3: the page side of one import (plan §B.1.5 items 1, 3, 5): the worker created in a
* try/catch, listeners before the post, the start timer until `ready`, the task timer
* after, settle once on the first of result / failed / error event / timer / cancel, the
* worker terminated in every case, later messages ignored, timers paused while hidden.
* after, the file fed chunk by chunk (one travelling buffer, each chunk acknowledged),
* settle once on the first of result / failed / error event / read failure / timer /
* cancel, the worker terminated in every case, later messages ignored, timers paused
* while hidden.
*/
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import {
runImportTask,
type ImportClientOptions,
type ImportSettlement,
type WorkerLike,
} from '@/core/worker/client'
import { PROTOCOL_VERSION } from '@/core/worker/protocol'
import type { ChunkSource } from '@/core/worker/reader'
import type { Visibility } from '@/core/worker/timers'
/** A scripted worker: the test drives its messages and events. */
/** A scripted worker: the test drives its messages and events; chunks are acknowledged. */
function scriptedWorker() {
const worker: WorkerLike & { terminated: number; sent: unknown[] } = {
terminated: 0,
@@ -21,7 +25,13 @@ function scriptedWorker() {
onmessage: null,
onerror: null,
onmessageerror: null,
postMessage: (m) => worker.sent.push(m),
postMessage: (m) => {
worker.sent.push(m)
const message = m as { type?: string; bytes?: ArrayBuffer }
if (message.type === 'chunk') {
emit({ v: PROTOCOL_VERSION, type: 'ack', bytes: message.bytes })
}
},
terminate: () => {
worker.terminated += 1
},
@@ -46,6 +56,7 @@ function scriptedWorker() {
detail: `${kind}!`,
}),
error: (message: string) => worker.onerror?.({ message } as ErrorEvent),
types: () => worker.sent.map((m) => (m as { type: string }).type),
}
}
@@ -65,8 +76,37 @@ function visibility(): Visibility & { set: (hidden: boolean) => void } {
}
}
const file = new File([new Uint8Array([1])], 'a.pdf')
/** A chunk source over `parts`, each filled into the travelling buffer. */
function partsSource(parts: Uint8Array[]): ChunkSource {
let at = 0
return {
async next(buffer) {
const part = parts[at]
if (!part) return null
at += 1
new Uint8Array(buffer, 0, part.byteLength).set(part)
return { bytes: buffer, length: part.byteLength }
},
cancel: async () => {},
}
}
const file = new File([new Uint8Array([1, 2, 3])], 'a.pdf')
const input = { task: 'pdf' as const, file, sheet: '{}', module: null }
const empty = () => partsSource([])
function options(
spawn: () => WorkerLike,
patch: Partial<ImportClientOptions> = {},
): ImportClientOptions {
return {
spawn,
startTimeoutMs: 100,
taskTimeoutMs: 100,
source: empty,
...patch,
}
}
async function settled(run: {
settled: Promise<ImportSettlement>
@@ -79,16 +119,21 @@ describe('import client', () => {
beforeEach(() => vi.useFakeTimers())
afterEach(() => vi.useRealTimers())
it('posts the task after installing the listeners, then settles on the result and terminates', async () => {
it('posts the task, feeds the chunks then end, settles on the result and terminates', async () => {
const s = scriptedWorker()
const run = runImportTask(input, {
spawn: () => s.worker,
startTimeoutMs: 100,
taskTimeoutMs: 100,
})
const parts = [new Uint8Array([1, 2]), new Uint8Array([3])]
const run = runImportTask(
input,
options(() => s.worker, { source: () => partsSource(parts) }),
)
expect(s.worker.onmessage).not.toBeNull()
expect(s.worker.sent).toEqual([{ v: 1, type: 'task', ...input }])
expect(s.worker.sent).toEqual([
{ v: 1, type: 'task', task: 'pdf', size: 3, sheet: '{}', module: null },
])
s.ready()
await vi.advanceTimersByTimeAsync(0)
expect(s.types()).toEqual(['task', 'chunk', 'chunk', 'end'])
expect((s.worker.sent[1] as { length: number }).length).toBe(2)
s.result('{"ok":true}')
expect(await settled(run)).toMatchObject({
kind: 'result',
@@ -98,13 +143,12 @@ describe('import client', () => {
})
it('is worker_unavailable when the spawn throws, fails before ready, or never answers ready', async () => {
const thrown = runImportTask(input, {
spawn: () => {
const thrown = runImportTask(
input,
options(() => {
throw new DOMException('blocked', 'SecurityError')
},
startTimeoutMs: 100,
taskTimeoutMs: 100,
})
}),
)
expect(await settled(thrown)).toEqual({
kind: 'failed',
reason: 'worker_unavailable',
@@ -112,11 +156,10 @@ describe('import client', () => {
})
const early = scriptedWorker()
const failedEarly = runImportTask(input, {
spawn: () => early.worker,
startTimeoutMs: 100,
taskTimeoutMs: 100,
})
const failedEarly = runImportTask(
input,
options(() => early.worker),
)
early.failed('start', 'other')
expect(await settled(failedEarly)).toMatchObject({
kind: 'failed',
@@ -124,11 +167,10 @@ describe('import client', () => {
})
const silent = scriptedWorker()
const timedOut = runImportTask(input, {
spawn: () => silent.worker,
startTimeoutMs: 100,
taskTimeoutMs: 1000,
})
const timedOut = runImportTask(
input,
options(() => silent.worker, { taskTimeoutMs: 1000 }),
)
await vi.advanceTimersByTimeAsync(99)
expect(silent.worker.terminated).toBe(0)
await vi.advanceTimersByTimeAsync(2)
@@ -140,20 +182,20 @@ describe('import client', () => {
})
it('maps task failures: trap → crashed, read → read, sheet/other → engine, timer → timeout', async () => {
for (const [kind, reason] of [
const cases: Array<[string, string]> = [
['trap', 'crashed'],
['read', 'read'],
['sheet', 'engine'],
['other', 'engine'],
]) {
]
for (const [kind, reason] of cases) {
const s = scriptedWorker()
const run = runImportTask(input, {
spawn: () => s.worker,
startTimeoutMs: 100,
taskTimeoutMs: 100,
})
const run = runImportTask(
input,
options(() => s.worker),
)
s.ready()
s.failed('task', kind!)
s.failed('task', kind)
expect(await settled(run)).toEqual({
kind: 'failed',
reason,
@@ -162,11 +204,10 @@ describe('import client', () => {
expect(s.worker.terminated).toBe(1)
}
const slow = scriptedWorker()
const run = runImportTask(input, {
spawn: () => slow.worker,
startTimeoutMs: 50,
taskTimeoutMs: 200,
})
const run = runImportTask(
input,
options(() => slow.worker, { startTimeoutMs: 50, taskTimeoutMs: 200 }),
)
await vi.advanceTimersByTimeAsync(40)
slow.ready()
await vi.advanceTimersByTimeAsync(199)
@@ -180,11 +221,10 @@ describe('import client', () => {
it('settles once: a result after the timeout, a failure after the result, are ignored', async () => {
const s = scriptedWorker()
const run = runImportTask(input, {
spawn: () => s.worker,
startTimeoutMs: 100,
taskTimeoutMs: 100,
})
const run = runImportTask(
input,
options(() => s.worker),
)
s.ready()
await vi.advanceTimersByTimeAsync(101)
s.result('{"ok":true}')
@@ -195,11 +235,10 @@ describe('import client', () => {
expect(s.worker.terminated).toBe(1)
const t = scriptedWorker()
const second = runImportTask(input, {
spawn: () => t.worker,
startTimeoutMs: 100,
taskTimeoutMs: 100,
})
const second = runImportTask(
input,
options(() => t.worker),
)
t.ready()
t.result('{"ok":true}')
t.failed('task', 'trap')
@@ -210,11 +249,10 @@ describe('import client', () => {
it('never masks an error event: worker_unavailable before ready, crashed after', async () => {
const before = scriptedWorker()
const first = runImportTask(input, {
spawn: () => before.worker,
startTimeoutMs: 100,
taskTimeoutMs: 100,
})
const first = runImportTask(
input,
options(() => before.worker),
)
before.error('script failed')
expect(await settled(first)).toEqual({
kind: 'failed',
@@ -222,11 +260,10 @@ describe('import client', () => {
detail: 'script failed',
})
const after = scriptedWorker()
const second = runImportTask(input, {
spawn: () => after.worker,
startTimeoutMs: 100,
taskTimeoutMs: 100,
})
const second = runImportTask(
input,
options(() => after.worker),
)
after.ready()
after.error('boom')
expect(await settled(second)).toEqual({
@@ -236,13 +273,31 @@ describe('import client', () => {
})
})
it('a file the page cannot read settles read and terminates the worker', async () => {
const s = scriptedWorker()
const failing: ChunkSource = {
next: () => Promise.reject(new DOMException('gone', 'NotReadableError')),
cancel: async () => {},
}
const run = runImportTask(
input,
options(() => s.worker, { source: () => failing }),
)
s.ready()
expect(await settled(run)).toEqual({
kind: 'failed',
reason: 'read',
detail: 'NotReadableError: gone',
})
expect(s.worker.terminated).toBe(1)
})
it('cancel terminates the worker and settles cancelled, once', async () => {
const s = scriptedWorker()
const run = runImportTask(input, {
spawn: () => s.worker,
startTimeoutMs: 100,
taskTimeoutMs: 100,
})
const run = runImportTask(
input,
options(() => s.worker),
)
s.ready()
run.cancel()
run.cancel()
@@ -257,12 +312,10 @@ describe('import client', () => {
it('pauses the task timer while the page is hidden', async () => {
const s = scriptedWorker()
const vis = visibility()
const run = runImportTask(input, {
spawn: () => s.worker,
startTimeoutMs: 100,
taskTimeoutMs: 100,
visibility: vis,
})
const run = runImportTask(
input,
options(() => s.worker, { visibility: vis }),
)
s.ready()
await vi.advanceTimersByTimeAsync(50)
vis.set(true)
+51 -39
View File
@@ -27,6 +27,7 @@ async function sheetText(): Promise<string> {
interface Run {
messages: WorkerMessage[]
acks: number
}
async function run(
@@ -41,11 +42,31 @@ async function run(
v: PROTOCOL_VERSION,
type: 'task',
task,
file,
size: file.size,
sheet: sheet ?? (await sheetText()),
module: null,
})
return { messages }
// The page's part: the file in 4 KiB chunks, then `end` (no transfer in-process).
if (
messages.some((m) => m.type === 'ready') &&
!messages.some((m) => m.type !== 'ready')
) {
const bytes = new Uint8Array(await file.arrayBuffer())
for (let at = 0; at < bytes.byteLength; at += 4096) {
const chunk = bytes.slice(at, at + 4096)
await handle({
v: PROTOCOL_VERSION,
type: 'chunk',
bytes: chunk.buffer,
length: chunk.byteLength,
})
}
await handle({ v: PROTOCOL_VERSION, type: 'end' })
}
return {
messages: messages.filter((m) => m.type !== 'ack'),
acks: messages.filter((m) => m.type === 'ack').length,
}
}
function envelope(messages: WorkerMessage[]): {
@@ -62,8 +83,9 @@ function envelope(messages: WorkerMessage[]): {
describe('import worker handler (in-process, shipped module)', () => {
it('answers ready then a result for a Faunosteo PDF, streamed in chunks', async () => {
const file = new File([fixture('report-v3.bare.pdf')], 'bilan.pdf')
const { messages } = await run('pdf', file)
const { messages, acks } = await run('pdf', file)
expect(messages.map((m) => m.type)).toEqual(['ready', 'result'])
expect(acks).toBe(Math.ceil(file.size / 4096))
const out = envelope(messages)
expect(out.ok).toBe(true)
expect(
@@ -87,27 +109,39 @@ describe('import worker handler (in-process, shipped module)', () => {
})
})
it('refuses an oversized file before reading a byte', async () => {
const huge = {
size: MAX_PDF_BYTES + 1,
stream: () => {
throw new Error('read')
},
} as unknown as File
expect(envelope((await run('pdf', huge)).messages)).toMatchObject({
it('refuses an oversized file before any chunk, and a file whose size changes', async () => {
const huge = { size: MAX_PDF_BYTES + 1 } as unknown as File
const over = await run('pdf', huge)
expect(over.acks).toBe(0)
expect(envelope(over.messages)).toMatchObject({
ok: false,
kind: 'invalid_pdf',
})
const copy = {
size: MAX_COPY_BYTES + 1,
stream: () => {
throw new Error('read')
},
} as unknown as File
const copy = { size: MAX_COPY_BYTES + 1 } as unknown as File
expect(envelope((await run('copy', copy)).messages)).toMatchObject({
ok: false,
kind: 'invalid',
})
// Announced 3 bytes, 4 arrive: the sink refuses and the task settles as not a PDF.
const messages: WorkerMessage[] = []
const handle = createWorkerHandler(nodeEngineLoader, (m) =>
messages.push(m),
)
await handle({
v: PROTOCOL_VERSION,
type: 'task',
task: 'pdf',
size: 3,
sheet: await sheetText(),
module: null,
})
await handle({
v: PROTOCOL_VERSION,
type: 'chunk',
bytes: new ArrayBuffer(4),
length: 4,
})
expect(envelope(messages)).toMatchObject({ ok: false, kind: 'invalid_pdf' })
})
it('gives the core kinds for a foreign PDF and garbage', async () => {
@@ -150,28 +184,6 @@ describe('import worker handler (in-process, shipped module)', () => {
})
})
it('classifies a file that cannot be read as failed/task/read', async () => {
const error = new DOMException(
'The file could not be read',
'NotReadableError',
)
const unreadable = {
size: 10,
stream: () => ({
getReader: () => ({
read: () => Promise.reject(error),
cancel: async () => {},
}),
}),
} as unknown as File
const { messages } = await run('pdf', unreadable)
expect(messages[1]).toMatchObject({
type: 'failed',
phase: 'task',
kind: 'read',
})
})
it('classifies a refused sheet as failed/task/sheet and a load failure as start', async () => {
const file = new File([fixture('report-v3.bare.pdf')], 'bilan.pdf')
const refused = await run(
@@ -146,12 +146,17 @@ describe('HomeRoute import: success paths', () => {
}
expect(workers).toHaveLength(1)
const task = workers[0]!.received[0] as ImportTaskMessage
expect(task).toMatchObject({ type: 'task', task: 'pdf' })
expect(task.file).toBe(file)
expect(task).toMatchObject({ type: 'task', task: 'pdf', size: file.size })
expect(task.sheet).toBe(JSON.stringify(await bundledSheet()))
expect(task.module).toBeInstanceOf(WebAssembly.Module)
// The page streams the file chunk by chunk, never whole: no arrayBuffer, no text.
expect(read).not.toHaveBeenCalled()
expect(text).not.toHaveBeenCalled()
const chunks = workers[0]!.received.filter(
(m) => (m as { type: string }).type === 'chunk',
)
expect(chunks.length).toBeGreaterThanOrEqual(1)
expect(workers[0]!.received.at(-1)).toMatchObject({ type: 'end' })
const imported = resultOf(workers[0]!)
expect(hydrate).toHaveBeenCalledTimes(1)
const [session, index] = hydrate.mock.calls[0]!
@@ -268,7 +273,7 @@ describe('HomeRoute import: a core failure reading the tab draft', () => {
await mountAndPickPdf()
await screen.findByText(fr['home.import.error_core_unavailable']!)
// The worker read the file: the failure is the store's, after the result.
expect(workers[0]!.posted.map((m) => m.type)).toEqual(['ready', 'result'])
expect(workers[0]!.sequence()).toEqual(['ready', 'result'])
expect(alertMock()).not.toHaveBeenCalled()
expect(screen.queryByTestId('editor-route')).toBeNull()
} finally {
@@ -284,7 +289,7 @@ describe('HomeRoute import: a core failure reading the tab draft', () => {
try {
await mountAndPickPdf()
await screen.findByTestId('editor-route')
expect(workers[0]!.posted.map((m) => m.type)).toEqual(['ready', 'result'])
expect(workers[0]!.sequence()).toEqual(['ready', 'result'])
expect(alertMock()).toHaveBeenCalledWith(fr['session.held_blocked'])
} finally {
useReportStore.setState({ hydrateImport: real })
@@ -194,7 +194,7 @@ describe('refusing a file', () => {
await pick(copyFile(JSON.stringify(minimalReport(sheet))))
await screen.findByText(fr['home.import.error_core_unavailable']!)
// The worker answered: the line is the core's kind, not a worker failure.
expect(workers[0]!.posted.map((m) => m.type)).toEqual(['ready', 'result'])
expect(workers[0]!.sequence()).toEqual(['ready', 'result'])
expect(
screen.queryByText(fr['home.import.error_invalid_copy']!),
).toBeNull()
@@ -319,7 +319,7 @@ describe('the stopped text on the Home import line and the New report alert', ()
await pickPdf()
await screen.findByText(fr['home.import.error_core_unavailable']!)
// The worker read the file: the line is the page's index failing after it.
expect(workers[0]!.posted.map((m) => m.type)).toEqual(['ready', 'result'])
expect(workers[0]!.sequence()).toEqual(['ready', 'result'])
})
it('New report: the alert says the stopped text', async () => {
@@ -172,7 +172,7 @@ describe('callers stop on a failed core call', () => {
})
await screen.findByText(fr['home.import.error_core_unavailable']!)
// The worker read the file; the page's core, needed to open it, did not load.
expect(workers[0]!.posted.map((m) => m.type)).toEqual(['ready', 'result'])
expect(workers[0]!.sequence()).toEqual(['ready', 'result'])
expect(screen.queryByTestId('editor-route')).toBeNull()
expect(useReportStore.getState().report).toBeNull()
})