feat(app): R4.3a the page reads the file chunk by chunk and streams it to the worker's sink
Measured at R4.3a (RUST.md §10.5): reading a file inside a WebKit worker costs about 180 MiB of content RSS whatever the method (arrayBuffer, stream, BYOB), 14 MiB on the page; a page-side BYOB read feeding the worker's FileBytes gives 106 MiB on a 50 MiB file against 272 in the worker and 76 for the TypeScript. Protocol v1 gains chunk / ack / end: one 1 MiB buffer travels (transferred) and comes back after each push, so the page holds one chunk and the worker's sink the only whole copy; no byte is parsed on the page. The worker refuses an oversize announced before any chunk and a size that changes while reading. Tests adapted (handler, client with a scripted ack, read failure on the page).
This commit is contained in:
1 parent
49591c53cd
commit
e659924ec0
12 files changed
+552
-272
No files matched your search
@@ -3,20 +3,23 @@ import {
|
|||||||
PROTOCOL_VERSION,
|
PROTOCOL_VERSION,
|
||||||
type ImportDiag,
|
type ImportDiag,
|
||||||
type ImportTaskKind,
|
type ImportTaskKind,
|
||||||
|
type WorkerMessage,
|
||||||
} from './protocol'
|
} from './protocol'
|
||||||
|
import { CHUNK_BYTES, chunkSource, type ChunkSource } from './reader'
|
||||||
import { PausableTimer, type Visibility, documentVisibility } from './timers'
|
import { PausableTimer, type Visibility, documentVisibility } from './timers'
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* The page side of one import (R4.3, plan §B.1.5 items 1, 3 and 5): creates the worker
|
* The page side of one import (R4.3, plan §B.1.5 items 1, 3 and 5): creates the worker
|
||||||
* in a try/catch (a `SecurityError` is synchronous), installs the listeners before posting,
|
* in a try/catch (a `SecurityError` is synchronous), installs the listeners before posting,
|
||||||
* arms the start timer until `ready` then the task timer, settles once on the first of
|
* arms the start timer until `ready` then the task timer, feeds the file chunk by chunk
|
||||||
* `result`, `failed`, an `error` event, a timer, or a cancel, and terminates the worker in
|
* (one travelling buffer, each chunk acknowledged before the next), settles once on the
|
||||||
* every case. Everything after the first settlement is ignored.
|
* first of `result`, `failed`, an `error` event, a read failure, a timer, or a cancel, and
|
||||||
|
* terminates the worker in every case. Everything after the first settlement is ignored.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
/** What `new Worker(…)` gives, narrowed to what the client uses (a test fake fits). */
|
/** What `new Worker(…)` gives, narrowed to what the client uses (a test fake fits). */
|
||||||
export interface WorkerLike {
|
export interface WorkerLike {
|
||||||
postMessage(message: unknown): void
|
postMessage(message: unknown, transfer?: Transferable[]): void
|
||||||
terminate(): void
|
terminate(): void
|
||||||
onmessage: ((event: MessageEvent) => void) | null
|
onmessage: ((event: MessageEvent) => void) | null
|
||||||
onerror: ((event: ErrorEvent) => void) | null
|
onerror: ((event: ErrorEvent) => void) | null
|
||||||
@@ -55,6 +58,8 @@ export interface ImportClientOptions {
|
|||||||
/** From `ready` to the result (plan §B.3: 20 s hypothesis, set at J1). */
|
/** From `ready` to the result (plan §B.3: 20 s hypothesis, set at J1). */
|
||||||
taskTimeoutMs: number
|
taskTimeoutMs: number
|
||||||
visibility?: Visibility
|
visibility?: Visibility
|
||||||
|
/** Test seam: the file's chunks. */
|
||||||
|
source?: (file: File) => ChunkSource
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface ImportRun {
|
export interface ImportRun {
|
||||||
@@ -71,67 +76,144 @@ function reasonOf(phase: 'start' | 'task', kind: string): ImportFailureReason {
|
|||||||
return 'engine'
|
return 'engine'
|
||||||
}
|
}
|
||||||
|
|
||||||
export function runImportTask(
|
/** One import's moving parts, settled once. */
|
||||||
input: ImportTaskInput,
|
class ImportClient {
|
||||||
options: ImportClientOptions,
|
private worker: WorkerLike | null = null
|
||||||
): ImportRun {
|
private timer: PausableTimer | null = null
|
||||||
let worker: WorkerLike | null = null
|
private source: ChunkSource | null = null
|
||||||
let timer: PausableTimer | null = null
|
private awaitingAck: ((bytes: ArrayBuffer) => void) | null = null
|
||||||
let ready = false
|
private ready = false
|
||||||
let done = false
|
private done = false
|
||||||
let resolve!: (settlement: ImportSettlement) => void
|
private resolve!: (settlement: ImportSettlement) => void
|
||||||
const settled = new Promise<ImportSettlement>((r) => {
|
readonly settled = new Promise<ImportSettlement>((resolve) => {
|
||||||
resolve = r
|
this.resolve = resolve
|
||||||
})
|
})
|
||||||
const visibility = options.visibility ?? documentVisibility
|
|
||||||
|
|
||||||
const settle = (settlement: ImportSettlement) => {
|
constructor(
|
||||||
if (done) return
|
private readonly input: ImportTaskInput,
|
||||||
done = true
|
private readonly options: ImportClientOptions,
|
||||||
timer?.clear()
|
) {}
|
||||||
timer = null
|
|
||||||
worker?.terminate()
|
start(): void {
|
||||||
worker = null
|
try {
|
||||||
resolve(settlement)
|
this.worker = this.options.spawn()
|
||||||
|
} catch (error) {
|
||||||
|
this.fail(
|
||||||
|
'worker_unavailable',
|
||||||
|
error instanceof Error ? error.name : 'spawn failed',
|
||||||
|
)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
this.worker.onmessage = (event) => this.onMessage(event.data)
|
||||||
|
// An `error` event is never masked: it settles the import (and reaches the console).
|
||||||
|
this.worker.onerror = (event) =>
|
||||||
|
this.fail(this.ready ? 'crashed' : 'worker_unavailable', event.message)
|
||||||
|
this.worker.onmessageerror = () =>
|
||||||
|
this.fail(this.ready ? 'engine' : 'worker_unavailable', 'messageerror')
|
||||||
|
this.arm(this.options.startTimeoutMs, 'worker_unavailable')
|
||||||
|
const { task, file, sheet, module } = this.input
|
||||||
|
this.worker.postMessage({
|
||||||
|
v: PROTOCOL_VERSION,
|
||||||
|
type: 'task',
|
||||||
|
task,
|
||||||
|
size: file.size,
|
||||||
|
sheet,
|
||||||
|
module,
|
||||||
|
})
|
||||||
}
|
}
|
||||||
const fail = (reason: ImportFailureReason, detail: string) =>
|
|
||||||
settle({ kind: 'failed', reason, detail })
|
cancel(): void {
|
||||||
const arm = (ms: number, reason: ImportFailureReason) => {
|
this.fail('cancelled', 'cancelled')
|
||||||
timer?.clear()
|
}
|
||||||
timer = new PausableTimer(
|
|
||||||
|
private onMessage(data: unknown): void {
|
||||||
|
if (this.done || !isWorkerMessage(data)) return
|
||||||
|
this.dispatch(data)
|
||||||
|
}
|
||||||
|
|
||||||
|
private dispatch(message: WorkerMessage): void {
|
||||||
|
switch (message.type) {
|
||||||
|
case 'ready':
|
||||||
|
this.ready = true
|
||||||
|
this.arm(this.options.taskTimeoutMs, 'timeout')
|
||||||
|
void this.feed()
|
||||||
|
return
|
||||||
|
case 'ack':
|
||||||
|
this.awaitingAck?.(message.bytes)
|
||||||
|
this.awaitingAck = null
|
||||||
|
return
|
||||||
|
case 'result':
|
||||||
|
this.settle({ kind: 'result', out: message.out, diag: message.diag })
|
||||||
|
return
|
||||||
|
case 'failed':
|
||||||
|
this.fail(reasonOf(message.phase, message.kind), message.detail)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Reads the file chunk by chunk into one buffer the worker hands back after each. */
|
||||||
|
private async feed(): Promise<void> {
|
||||||
|
this.source = (this.options.source ?? chunkSource)(this.input.file)
|
||||||
|
let buffer = new ArrayBuffer(CHUNK_BYTES)
|
||||||
|
try {
|
||||||
|
for (;;) {
|
||||||
|
const chunk = await this.source.next(buffer)
|
||||||
|
if (this.done) return
|
||||||
|
if (chunk === null) break
|
||||||
|
buffer = await this.send(chunk.bytes, chunk.length)
|
||||||
|
if (this.done) return
|
||||||
|
}
|
||||||
|
this.worker?.postMessage({ v: PROTOCOL_VERSION, type: 'end' })
|
||||||
|
} catch (error) {
|
||||||
|
this.fail(
|
||||||
|
'read',
|
||||||
|
error instanceof Error ? `${error.name}: ${error.message}` : 'read',
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Posts a chunk (transferred) and resolves with its buffer once acknowledged. */
|
||||||
|
private send(bytes: ArrayBuffer, length: number): Promise<ArrayBuffer> {
|
||||||
|
return new Promise((resolve) => {
|
||||||
|
this.awaitingAck = resolve
|
||||||
|
this.worker?.postMessage(
|
||||||
|
{ v: PROTOCOL_VERSION, type: 'chunk', bytes, length },
|
||||||
|
[bytes],
|
||||||
|
)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
private arm(ms: number, reason: ImportFailureReason): void {
|
||||||
|
this.timer?.clear()
|
||||||
|
const visibility = this.options.visibility ?? documentVisibility
|
||||||
|
this.timer = new PausableTimer(
|
||||||
ms,
|
ms,
|
||||||
() => fail(reason, `${reason} after ${ms} ms`),
|
() => this.fail(reason, `${reason} after ${ms} ms`),
|
||||||
visibility,
|
visibility,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
private fail(reason: ImportFailureReason, detail: string): void {
|
||||||
worker = options.spawn()
|
this.settle({ kind: 'failed', reason, detail })
|
||||||
} catch (error) {
|
|
||||||
fail(
|
|
||||||
'worker_unavailable',
|
|
||||||
error instanceof Error ? error.name : 'spawn failed',
|
|
||||||
)
|
|
||||||
return { settled, cancel: () => fail('cancelled', 'cancelled') }
|
|
||||||
}
|
}
|
||||||
worker.onmessage = (event) => {
|
|
||||||
const data: unknown = event.data
|
private settle(settlement: ImportSettlement): void {
|
||||||
if (!isWorkerMessage(data)) return
|
if (this.done) return
|
||||||
if (data.type === 'ready') {
|
this.done = true
|
||||||
ready = true
|
this.timer?.clear()
|
||||||
arm(options.taskTimeoutMs, 'timeout')
|
this.timer = null
|
||||||
} else if (data.type === 'result') {
|
this.worker?.terminate()
|
||||||
settle({ kind: 'result', out: data.out, diag: data.diag })
|
this.worker = null
|
||||||
} else {
|
void this.source?.cancel()
|
||||||
fail(reasonOf(data.phase, data.kind), data.detail)
|
this.source = null
|
||||||
}
|
this.resolve(settlement)
|
||||||
}
|
}
|
||||||
// An `error` event is never masked: it settles the import (and reaches the console).
|
}
|
||||||
worker.onerror = (event) =>
|
|
||||||
fail(ready ? 'crashed' : 'worker_unavailable', event.message)
|
export function runImportTask(
|
||||||
worker.onmessageerror = () =>
|
input: ImportTaskInput,
|
||||||
fail(ready ? 'engine' : 'worker_unavailable', 'messageerror')
|
options: ImportClientOptions,
|
||||||
arm(options.startTimeoutMs, 'worker_unavailable')
|
): ImportRun {
|
||||||
worker.postMessage({ v: PROTOCOL_VERSION, type: 'task', ...input })
|
const client = new ImportClient(input, options)
|
||||||
return { settled, cancel: () => fail('cancelled', 'cancelled') }
|
client.start()
|
||||||
|
return { settled: client.settled, cancel: () => client.cancel() }
|
||||||
}
|
}
|
||||||
@@ -3,8 +3,9 @@ import { closedRefusal } from '@faunosteo/core-wasm'
|
|||||||
import {
|
import {
|
||||||
describeError,
|
describeError,
|
||||||
failureKindOf,
|
failureKindOf,
|
||||||
|
isPageMessage,
|
||||||
PROTOCOL_VERSION,
|
PROTOCOL_VERSION,
|
||||||
type ImportTaskKind,
|
type ChunkMessage,
|
||||||
type ImportTaskMessage,
|
type ImportTaskMessage,
|
||||||
type WorkerFailurePhase,
|
type WorkerFailurePhase,
|
||||||
type WorkerMessage,
|
type WorkerMessage,
|
||||||
@@ -13,9 +14,9 @@ import {
|
|||||||
/**
|
/**
|
||||||
* The import worker's logic (R4.3, plan §B.1.5 items 2 and 4), apart from the worker
|
* The import worker's logic (R4.3, plan §B.1.5 items 2 and 4), apart from the worker
|
||||||
* global so that vitest runs it in-process on the Node module: the engine is injected.
|
* global so that vitest runs it in-process on the Node module: the engine is injected.
|
||||||
* One task per worker: instantiate, answer `ready`, build the sheet handle, stream the
|
* One task per worker: instantiate, answer `ready`, build the sheet handle, sink the
|
||||||
* file into the module's memory (the file lives once, never as an ArrayBuffer here), run
|
* chunks the page reads (one copy, in the module's memory), run the core on `end`, answer
|
||||||
* the core, answer `result`; every throwable becomes one `failed` with its phase and kind.
|
* `result`; every throwable becomes one `failed` with its phase and kind.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
/** What the glue gives once instantiated. */
|
/** What the glue gives once instantiated. */
|
||||||
@@ -30,10 +31,12 @@ export type EngineLoader = (
|
|||||||
module: WebAssembly.Module | null,
|
module: WebAssembly.Module | null,
|
||||||
) => Promise<WorkerEngine>
|
) => Promise<WorkerEngine>
|
||||||
|
|
||||||
/** The caps the worker checks before reading a byte (the core enforces the same). */
|
/** The caps the worker checks before reserving a byte (the core enforces the same). */
|
||||||
export const MAX_PDF_BYTES = 50 * 1024 * 1024
|
export const MAX_PDF_BYTES = 50 * 1024 * 1024
|
||||||
export const MAX_COPY_BYTES = 16 * 1024 * 1024
|
export const MAX_COPY_BYTES = 16 * 1024 * 1024
|
||||||
|
|
||||||
|
export type Post = (message: WorkerMessage, transfer?: Transferable[]) => void
|
||||||
|
|
||||||
/** The core's failure envelope, for the caps the worker applies itself. */
|
/** The core's failure envelope, for the caps the worker applies itself. */
|
||||||
function refusal(kind: string, detail: string): string {
|
function refusal(kind: string, detail: string): string {
|
||||||
return JSON.stringify({ ok: false, kind, detail })
|
return JSON.stringify({ ok: false, kind, detail })
|
||||||
@@ -52,99 +55,118 @@ function sheetHandle(engine: WorkerEngine, sheet: string): GlueSheetHandle {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
interface Gathered {
|
/** One task's state, from the task message to its settlement. */
|
||||||
bytes: GlueFileBytes
|
class ImportTask {
|
||||||
chunks: number
|
private engine: WorkerEngine | null = null
|
||||||
}
|
private handle: GlueSheetHandle | null = null
|
||||||
|
private sink: GlueFileBytes | null = null
|
||||||
|
private chunks = 0
|
||||||
|
private settled = false
|
||||||
|
private readonly started = performance.now()
|
||||||
|
|
||||||
/** Streams the file into the module's memory, one chunk live at a time. */
|
constructor(
|
||||||
async function gather(
|
private readonly task: ImportTaskMessage,
|
||||||
engine: WorkerEngine,
|
private readonly load: EngineLoader,
|
||||||
file: File,
|
private readonly post: Post,
|
||||||
): Promise<Gathered | null> {
|
) {}
|
||||||
const bytes = new engine.FileBytes(file.size)
|
|
||||||
const reader = file.stream().getReader()
|
/** Instantiates, answers `ready`, reserves the sink; a cap refusal settles at once. */
|
||||||
let chunks = 0
|
async start(): Promise<void> {
|
||||||
for (;;) {
|
let phase: WorkerFailurePhase = 'start'
|
||||||
const { done, value } = await reader.read()
|
try {
|
||||||
if (done) return { bytes, chunks }
|
this.engine = await this.load(this.task.module)
|
||||||
chunks += 1
|
this.post({ v: PROTOCOL_VERSION, type: 'ready' })
|
||||||
// A push refused means the file is not the size it announced: not a Faunosteo file.
|
phase = 'task'
|
||||||
if (!bytes.push(value)) {
|
this.handle = sheetHandle(this.engine, this.task.sheet)
|
||||||
await reader.cancel()
|
const cap = this.task.task === 'pdf' ? MAX_PDF_BYTES : MAX_COPY_BYTES
|
||||||
bytes.free()
|
if (this.task.size > cap) {
|
||||||
return null
|
this.result(refusal(this.kind(), 'over the size cap'))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
this.sink = new this.engine.FileBytes(this.task.size)
|
||||||
|
} catch (error) {
|
||||||
|
this.fail(phase, error)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
/** The task's outcome envelope text and the chunk count. */
|
/** Sinks one chunk and hands its buffer back; a refused push means the size changed. */
|
||||||
async function runTask(
|
chunk(message: ChunkMessage): void {
|
||||||
engine: WorkerEngine,
|
if (this.settled || !this.sink) return
|
||||||
handle: GlueSheetHandle,
|
const view = new Uint8Array(message.bytes, 0, message.length)
|
||||||
task: ImportTaskKind,
|
if (!this.sink.push(view)) {
|
||||||
file: File,
|
this.sink.free()
|
||||||
): Promise<{ out: string; chunks: number }> {
|
this.sink = null
|
||||||
const cap = task === 'pdf' ? MAX_PDF_BYTES : MAX_COPY_BYTES
|
this.result(refusal(this.kind(), 'size changed while reading'))
|
||||||
const kind = task === 'pdf' ? 'invalid_pdf' : 'invalid'
|
return
|
||||||
if (file.size > cap)
|
}
|
||||||
return { out: refusal(kind, 'over the size cap'), chunks: 0 }
|
this.chunks += 1
|
||||||
const gathered = await gather(engine, file)
|
this.post({ v: PROTOCOL_VERSION, type: 'ack', bytes: message.bytes }, [
|
||||||
if (!gathered)
|
message.bytes,
|
||||||
return { out: refusal(kind, 'size changed while reading'), chunks: 0 }
|
])
|
||||||
const out =
|
}
|
||||||
task === 'pdf'
|
|
||||||
? handle.import_pdf_bytes(gathered.bytes)
|
|
||||||
: handle.restore_copy_bytes(gathered.bytes)
|
|
||||||
return { out, chunks: gathered.chunks }
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/** Runs the core on the gathered bytes. */
|
||||||
* The handler of the task message: `load` instantiates the module (the posted one, or
|
end(): void {
|
||||||
* fetched), `post` sends the answers. Resolves once the task has settled either way.
|
if (this.settled || !this.sink || !this.handle) return
|
||||||
*/
|
try {
|
||||||
export function createWorkerHandler(
|
const sink = this.sink
|
||||||
load: EngineLoader,
|
this.sink = null
|
||||||
post: (message: WorkerMessage) => void,
|
const out =
|
||||||
): (data: unknown) => Promise<void> {
|
this.task.task === 'pdf'
|
||||||
const fail = (phase: WorkerFailurePhase, error: unknown) =>
|
? this.handle.import_pdf_bytes(sink)
|
||||||
post({
|
: this.handle.restore_copy_bytes(sink)
|
||||||
|
this.result(out)
|
||||||
|
} catch (error) {
|
||||||
|
this.fail('task', error)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private kind(): string {
|
||||||
|
return this.task.task === 'pdf' ? 'invalid_pdf' : 'invalid'
|
||||||
|
}
|
||||||
|
|
||||||
|
private result(out: string): void {
|
||||||
|
this.settled = true
|
||||||
|
const linearBytes = this.engine?.memory?.buffer.byteLength ?? null
|
||||||
|
const diag = {
|
||||||
|
ms: performance.now() - this.started,
|
||||||
|
linearBytes,
|
||||||
|
chunks: this.chunks,
|
||||||
|
}
|
||||||
|
this.post({ v: PROTOCOL_VERSION, type: 'result', out, diag })
|
||||||
|
}
|
||||||
|
|
||||||
|
private fail(phase: WorkerFailurePhase, error: unknown): void {
|
||||||
|
this.settled = true
|
||||||
|
this.post({
|
||||||
v: PROTOCOL_VERSION,
|
v: PROTOCOL_VERSION,
|
||||||
type: 'failed',
|
type: 'failed',
|
||||||
phase,
|
phase,
|
||||||
kind: failureKindOf(error),
|
kind: failureKindOf(error),
|
||||||
detail: describeError(error),
|
detail: describeError(error),
|
||||||
})
|
})
|
||||||
return async (data) => {
|
|
||||||
if (!isTaskMessage(data)) return
|
|
||||||
let phase: WorkerFailurePhase = 'start'
|
|
||||||
try {
|
|
||||||
const engine = await load(data.module)
|
|
||||||
post({ v: PROTOCOL_VERSION, type: 'ready' })
|
|
||||||
phase = 'task'
|
|
||||||
const started = performance.now()
|
|
||||||
const handle = sheetHandle(engine, data.sheet)
|
|
||||||
const { out, chunks } = await runTask(
|
|
||||||
engine,
|
|
||||||
handle,
|
|
||||||
data.task,
|
|
||||||
data.file,
|
|
||||||
)
|
|
||||||
const linearBytes = engine.memory?.buffer.byteLength ?? null
|
|
||||||
const diag = { ms: performance.now() - started, linearBytes, chunks }
|
|
||||||
post({ v: PROTOCOL_VERSION, type: 'result', out, diag })
|
|
||||||
} catch (error) {
|
|
||||||
fail(phase, error)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
function isTaskMessage(data: unknown): data is ImportTaskMessage {
|
/**
|
||||||
if (typeof data !== 'object' || data === null) return false
|
* The handler of the page's messages: `load` instantiates the module (the posted one, or
|
||||||
const record = data as { v?: unknown; type?: unknown; task?: unknown }
|
* fetched), `post` sends the answers. Messages outside the protocol are ignored.
|
||||||
return (
|
*/
|
||||||
record.v === PROTOCOL_VERSION &&
|
export function createWorkerHandler(
|
||||||
record.type === 'task' &&
|
load: EngineLoader,
|
||||||
(record.task === 'pdf' || record.task === 'copy')
|
post: Post,
|
||||||
)
|
): (data: unknown) => Promise<void> {
|
||||||
|
let task: ImportTask | null = null
|
||||||
|
return async (data) => {
|
||||||
|
if (!isPageMessage(data)) return
|
||||||
|
if (data.type === 'task') {
|
||||||
|
if (task) return
|
||||||
|
task = new ImportTask(data, load, post)
|
||||||
|
await task.start()
|
||||||
|
} else if (data.type === 'chunk') {
|
||||||
|
task?.chunk(data)
|
||||||
|
} else {
|
||||||
|
task?.end()
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
@@ -27,8 +27,8 @@ const loadEngine = async (
|
|||||||
return { SheetHandle, FileBytes, memory: exports.memory ?? null }
|
return { SheetHandle, FileBytes, memory: exports.memory ?? null }
|
||||||
}
|
}
|
||||||
|
|
||||||
const handle = createWorkerHandler(loadEngine, (message) =>
|
const handle = createWorkerHandler(loadEngine, (message, transfer = []) =>
|
||||||
scope.postMessage(message),
|
scope.postMessage(message, transfer),
|
||||||
)
|
)
|
||||||
|
|
||||||
scope.onmessage = (event: MessageEvent<unknown>) => {
|
scope.onmessage = (event: MessageEvent<unknown>) => {
|
||||||
|
|||||||
@@ -1,7 +1,8 @@
|
|||||||
/**
|
/**
|
||||||
* Messages between the page and the import worker (R4.3, plan §B.1.5), tagged `v: 1`.
|
* Messages between the page and the import worker (R4.3, plan §B.1.5), tagged `v: 1`.
|
||||||
* The page posts one task; the worker answers `ready` once its instance runs, then one
|
* The page posts one task, then the file chunk by chunk, then `end`; the worker answers
|
||||||
* `result` or one `failed`. Anything after the first settlement is ignored by the page.
|
* `ready` once its instance runs, `ack` per chunk, then one `result` or one `failed`.
|
||||||
|
* Anything after the first settlement is ignored by the page.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
export const PROTOCOL_VERSION = 1
|
export const PROTOCOL_VERSION = 1
|
||||||
@@ -12,19 +13,47 @@ export interface ImportTaskMessage {
|
|||||||
v: typeof PROTOCOL_VERSION
|
v: typeof PROTOCOL_VERSION
|
||||||
type: 'task'
|
type: 'task'
|
||||||
task: ImportTaskKind
|
task: ImportTaskKind
|
||||||
/** The picked file, never read on the page's thread. */
|
/** The picked file's size: the worker reserves it and refuses more. */
|
||||||
file: File
|
size: number
|
||||||
/** The bundled sheet payload text the page's runtime builds its handle from. */
|
/** The bundled sheet payload text the page's runtime builds its handle from. */
|
||||||
sheet: string
|
sheet: string
|
||||||
/** The compiled module of the page's runtime when it holds one; else the worker fetches. */
|
/** The compiled module of the page's runtime when it holds one; else the worker fetches. */
|
||||||
module: WebAssembly.Module | null
|
module: WebAssembly.Module | null
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* One chunk of the file, read on the page's thread into a buffer the worker hands back
|
||||||
|
* with `ack` (both transferred, never copied): the page holds one chunk at a time, the
|
||||||
|
* worker's sink the only whole copy. Measured at R4.3a: reading a file inside a WebKit
|
||||||
|
* worker costs about 180 MiB whatever the method; read on the page and streamed, 106.
|
||||||
|
*/
|
||||||
|
export interface ChunkMessage {
|
||||||
|
v: typeof PROTOCOL_VERSION
|
||||||
|
type: 'chunk'
|
||||||
|
bytes: ArrayBuffer
|
||||||
|
length: number
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The page read the whole file: the worker runs the core. */
|
||||||
|
export interface EndMessage {
|
||||||
|
v: typeof PROTOCOL_VERSION
|
||||||
|
type: 'end'
|
||||||
|
}
|
||||||
|
|
||||||
|
export type PageMessage = ImportTaskMessage | ChunkMessage | EndMessage
|
||||||
|
|
||||||
export interface WorkerReadyMessage {
|
export interface WorkerReadyMessage {
|
||||||
v: typeof PROTOCOL_VERSION
|
v: typeof PROTOCOL_VERSION
|
||||||
type: 'ready'
|
type: 'ready'
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** The chunk is in the sink; its buffer comes back for the next read. */
|
||||||
|
export interface WorkerAckMessage {
|
||||||
|
v: typeof PROTOCOL_VERSION
|
||||||
|
type: 'ack'
|
||||||
|
bytes: ArrayBuffer
|
||||||
|
}
|
||||||
|
|
||||||
export interface WorkerResultMessage {
|
export interface WorkerResultMessage {
|
||||||
v: typeof PROTOCOL_VERSION
|
v: typeof PROTOCOL_VERSION
|
||||||
type: 'result'
|
type: 'result'
|
||||||
@@ -60,21 +89,33 @@ export interface WorkerFailedMessage {
|
|||||||
|
|
||||||
export type WorkerMessage =
|
export type WorkerMessage =
|
||||||
| WorkerReadyMessage
|
| WorkerReadyMessage
|
||||||
|
| WorkerAckMessage
|
||||||
| WorkerResultMessage
|
| WorkerResultMessage
|
||||||
| WorkerFailedMessage
|
| WorkerFailedMessage
|
||||||
|
|
||||||
/** True for a message of this protocol, from a worker of the same build. */
|
const WORKER_TYPES = new Set(['ready', 'ack', 'result', 'failed'])
|
||||||
export function isWorkerMessage(data: unknown): data is WorkerMessage {
|
const PAGE_TYPES = new Set(['task', 'chunk', 'end'])
|
||||||
|
|
||||||
|
function ofProtocol(data: unknown, types: Set<string>): boolean {
|
||||||
if (typeof data !== 'object' || data === null) return false
|
if (typeof data !== 'object' || data === null) return false
|
||||||
const record = data as { v?: unknown; type?: unknown }
|
const record = data as { v?: unknown; type?: unknown }
|
||||||
return (
|
return (
|
||||||
record.v === PROTOCOL_VERSION &&
|
record.v === PROTOCOL_VERSION &&
|
||||||
(record.type === 'ready' ||
|
typeof record.type === 'string' &&
|
||||||
record.type === 'result' ||
|
types.has(record.type)
|
||||||
record.type === 'failed')
|
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** True for a message of this protocol, from a worker of the same build. */
|
||||||
|
export function isWorkerMessage(data: unknown): data is WorkerMessage {
|
||||||
|
return ofProtocol(data, WORKER_TYPES)
|
||||||
|
}
|
||||||
|
|
||||||
|
/** True for a message of this protocol, from the page. */
|
||||||
|
export function isPageMessage(data: unknown): data is PageMessage {
|
||||||
|
return ofProtocol(data, PAGE_TYPES)
|
||||||
|
}
|
||||||
|
|
||||||
/** Error name and first message line, bounded, for a `failed` message. */
|
/** Error name and first message line, bounded, for a `failed` message. */
|
||||||
export function describeError(error: unknown): string {
|
export function describeError(error: unknown): string {
|
||||||
const name = error instanceof Error ? error.name : 'Error'
|
const name = error instanceof Error ? error.name : 'Error'
|
||||||
|
|||||||
@@ -0,0 +1,60 @@
|
|||||||
|
/**
|
||||||
|
* The page's read of the picked file, one chunk at a time into a buffer that travels to
|
||||||
|
* the worker and back (R4.3a measurement: a file read inside a WebKit worker costs about
|
||||||
|
* 180 MiB whatever the method; read here, 106, and 14 on the page alone). A BYOB reader
|
||||||
|
* fills the buffer the worker handed back; without BYOB support (happy-dom), the default
|
||||||
|
* reader's chunks are copied into it. Nothing here parses a byte.
|
||||||
|
*/
|
||||||
|
|
||||||
|
export const CHUNK_BYTES = 1024 * 1024
|
||||||
|
|
||||||
|
export interface ChunkSource {
|
||||||
|
/** The next chunk's length in `buffer`, or null at the end; `buffer` is consumed. */
|
||||||
|
next(
|
||||||
|
buffer: ArrayBuffer,
|
||||||
|
): Promise<{ bytes: ArrayBuffer; length: number } | null>
|
||||||
|
cancel(): Promise<void>
|
||||||
|
}
|
||||||
|
|
||||||
|
type ByobReader = ReadableStreamBYOBReader
|
||||||
|
type DefaultReader = ReadableStreamDefaultReader<Uint8Array>
|
||||||
|
|
||||||
|
function byobSource(reader: ByobReader): ChunkSource {
|
||||||
|
return {
|
||||||
|
async next(buffer) {
|
||||||
|
const { done, value } = await reader.read(new Uint8Array(buffer))
|
||||||
|
if (done || !value) return null
|
||||||
|
return { bytes: value.buffer as ArrayBuffer, length: value.byteLength }
|
||||||
|
},
|
||||||
|
cancel: () => reader.cancel().catch(() => undefined),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The default reader's chunks copied into the travelling buffer, split when longer. */
|
||||||
|
function defaultSource(reader: DefaultReader): ChunkSource {
|
||||||
|
let pending: Uint8Array | null = null
|
||||||
|
return {
|
||||||
|
async next(buffer) {
|
||||||
|
if (!pending || pending.byteLength === 0) {
|
||||||
|
const { done, value } = await reader.read()
|
||||||
|
if (done || !value) return null
|
||||||
|
pending = value
|
||||||
|
}
|
||||||
|
const length = Math.min(pending.byteLength, buffer.byteLength)
|
||||||
|
new Uint8Array(buffer, 0, length).set(pending.subarray(0, length))
|
||||||
|
pending = pending.subarray(length)
|
||||||
|
return { bytes: buffer, length }
|
||||||
|
},
|
||||||
|
cancel: () => reader.cancel().catch(() => undefined),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** A chunk source over `file`, BYOB when the engine offers it. */
|
||||||
|
export function chunkSource(file: File): ChunkSource {
|
||||||
|
const stream = file.stream()
|
||||||
|
try {
|
||||||
|
return byobSource(stream.getReader({ mode: 'byob' }))
|
||||||
|
} catch {
|
||||||
|
return defaultSource(stream.getReader())
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -39,6 +39,8 @@ export interface FakeWorker extends WorkerLike {
|
|||||||
received: unknown[]
|
received: unknown[]
|
||||||
/** Every message the worker posted, dropped or delivered. */
|
/** Every message the worker posted, dropped or delivered. */
|
||||||
posted: WorkerMessage[]
|
posted: WorkerMessage[]
|
||||||
|
/** The posted message types without the per-chunk `ack`s. */
|
||||||
|
sequence(): string[]
|
||||||
}
|
}
|
||||||
|
|
||||||
export function inProcessWorker(
|
export function inProcessWorker(
|
||||||
@@ -54,6 +56,9 @@ export function inProcessWorker(
|
|||||||
terminate() {
|
terminate() {
|
||||||
worker.terminated = true
|
worker.terminated = true
|
||||||
},
|
},
|
||||||
|
sequence() {
|
||||||
|
return worker.posted.map((m) => m.type).filter((t) => t !== 'ack')
|
||||||
|
},
|
||||||
postMessage(message) {
|
postMessage(message) {
|
||||||
worker.received.push(message)
|
worker.received.push(message)
|
||||||
void handle(message)
|
void handle(message)
|
||||||
|
|||||||
@@ -1,19 +1,23 @@
|
|||||||
/**
|
/**
|
||||||
* R4.3: the page side of one import (plan §B.1.5 items 1, 3, 5): the worker created in a
|
* R4.3: the page side of one import (plan §B.1.5 items 1, 3, 5): the worker created in a
|
||||||
* try/catch, listeners before the post, the start timer until `ready`, the task timer
|
* try/catch, listeners before the post, the start timer until `ready`, the task timer
|
||||||
* after, settle once on the first of result / failed / error event / timer / cancel, the
|
* after, the file fed chunk by chunk (one travelling buffer, each chunk acknowledged),
|
||||||
* worker terminated in every case, later messages ignored, timers paused while hidden.
|
* settle once on the first of result / failed / error event / read failure / timer /
|
||||||
|
* cancel, the worker terminated in every case, later messages ignored, timers paused
|
||||||
|
* while hidden.
|
||||||
*/
|
*/
|
||||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||||
import {
|
import {
|
||||||
runImportTask,
|
runImportTask,
|
||||||
|
type ImportClientOptions,
|
||||||
type ImportSettlement,
|
type ImportSettlement,
|
||||||
type WorkerLike,
|
type WorkerLike,
|
||||||
} from '@/core/worker/client'
|
} from '@/core/worker/client'
|
||||||
import { PROTOCOL_VERSION } from '@/core/worker/protocol'
|
import { PROTOCOL_VERSION } from '@/core/worker/protocol'
|
||||||
|
import type { ChunkSource } from '@/core/worker/reader'
|
||||||
import type { Visibility } from '@/core/worker/timers'
|
import type { Visibility } from '@/core/worker/timers'
|
||||||
|
|
||||||
/** A scripted worker: the test drives its messages and events. */
|
/** A scripted worker: the test drives its messages and events; chunks are acknowledged. */
|
||||||
function scriptedWorker() {
|
function scriptedWorker() {
|
||||||
const worker: WorkerLike & { terminated: number; sent: unknown[] } = {
|
const worker: WorkerLike & { terminated: number; sent: unknown[] } = {
|
||||||
terminated: 0,
|
terminated: 0,
|
||||||
@@ -21,7 +25,13 @@ function scriptedWorker() {
|
|||||||
onmessage: null,
|
onmessage: null,
|
||||||
onerror: null,
|
onerror: null,
|
||||||
onmessageerror: null,
|
onmessageerror: null,
|
||||||
postMessage: (m) => worker.sent.push(m),
|
postMessage: (m) => {
|
||||||
|
worker.sent.push(m)
|
||||||
|
const message = m as { type?: string; bytes?: ArrayBuffer }
|
||||||
|
if (message.type === 'chunk') {
|
||||||
|
emit({ v: PROTOCOL_VERSION, type: 'ack', bytes: message.bytes })
|
||||||
|
}
|
||||||
|
},
|
||||||
terminate: () => {
|
terminate: () => {
|
||||||
worker.terminated += 1
|
worker.terminated += 1
|
||||||
},
|
},
|
||||||
@@ -46,6 +56,7 @@ function scriptedWorker() {
|
|||||||
detail: `${kind}!`,
|
detail: `${kind}!`,
|
||||||
}),
|
}),
|
||||||
error: (message: string) => worker.onerror?.({ message } as ErrorEvent),
|
error: (message: string) => worker.onerror?.({ message } as ErrorEvent),
|
||||||
|
types: () => worker.sent.map((m) => (m as { type: string }).type),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -65,8 +76,37 @@ function visibility(): Visibility & { set: (hidden: boolean) => void } {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const file = new File([new Uint8Array([1])], 'a.pdf')
|
/** A chunk source over `parts`, each filled into the travelling buffer. */
|
||||||
|
function partsSource(parts: Uint8Array[]): ChunkSource {
|
||||||
|
let at = 0
|
||||||
|
return {
|
||||||
|
async next(buffer) {
|
||||||
|
const part = parts[at]
|
||||||
|
if (!part) return null
|
||||||
|
at += 1
|
||||||
|
new Uint8Array(buffer, 0, part.byteLength).set(part)
|
||||||
|
return { bytes: buffer, length: part.byteLength }
|
||||||
|
},
|
||||||
|
cancel: async () => {},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const file = new File([new Uint8Array([1, 2, 3])], 'a.pdf')
|
||||||
const input = { task: 'pdf' as const, file, sheet: '{}', module: null }
|
const input = { task: 'pdf' as const, file, sheet: '{}', module: null }
|
||||||
|
const empty = () => partsSource([])
|
||||||
|
|
||||||
|
function options(
|
||||||
|
spawn: () => WorkerLike,
|
||||||
|
patch: Partial<ImportClientOptions> = {},
|
||||||
|
): ImportClientOptions {
|
||||||
|
return {
|
||||||
|
spawn,
|
||||||
|
startTimeoutMs: 100,
|
||||||
|
taskTimeoutMs: 100,
|
||||||
|
source: empty,
|
||||||
|
...patch,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async function settled(run: {
|
async function settled(run: {
|
||||||
settled: Promise<ImportSettlement>
|
settled: Promise<ImportSettlement>
|
||||||
@@ -79,16 +119,21 @@ describe('import client', () => {
|
|||||||
beforeEach(() => vi.useFakeTimers())
|
beforeEach(() => vi.useFakeTimers())
|
||||||
afterEach(() => vi.useRealTimers())
|
afterEach(() => vi.useRealTimers())
|
||||||
|
|
||||||
it('posts the task after installing the listeners, then settles on the result and terminates', async () => {
|
it('posts the task, feeds the chunks then end, settles on the result and terminates', async () => {
|
||||||
const s = scriptedWorker()
|
const s = scriptedWorker()
|
||||||
const run = runImportTask(input, {
|
const parts = [new Uint8Array([1, 2]), new Uint8Array([3])]
|
||||||
spawn: () => s.worker,
|
const run = runImportTask(
|
||||||
startTimeoutMs: 100,
|
input,
|
||||||
taskTimeoutMs: 100,
|
options(() => s.worker, { source: () => partsSource(parts) }),
|
||||||
})
|
)
|
||||||
expect(s.worker.onmessage).not.toBeNull()
|
expect(s.worker.onmessage).not.toBeNull()
|
||||||
expect(s.worker.sent).toEqual([{ v: 1, type: 'task', ...input }])
|
expect(s.worker.sent).toEqual([
|
||||||
|
{ v: 1, type: 'task', task: 'pdf', size: 3, sheet: '{}', module: null },
|
||||||
|
])
|
||||||
s.ready()
|
s.ready()
|
||||||
|
await vi.advanceTimersByTimeAsync(0)
|
||||||
|
expect(s.types()).toEqual(['task', 'chunk', 'chunk', 'end'])
|
||||||
|
expect((s.worker.sent[1] as { length: number }).length).toBe(2)
|
||||||
s.result('{"ok":true}')
|
s.result('{"ok":true}')
|
||||||
expect(await settled(run)).toMatchObject({
|
expect(await settled(run)).toMatchObject({
|
||||||
kind: 'result',
|
kind: 'result',
|
||||||
@@ -98,13 +143,12 @@ describe('import client', () => {
|
|||||||
})
|
})
|
||||||
|
|
||||||
it('is worker_unavailable when the spawn throws, fails before ready, or never answers ready', async () => {
|
it('is worker_unavailable when the spawn throws, fails before ready, or never answers ready', async () => {
|
||||||
const thrown = runImportTask(input, {
|
const thrown = runImportTask(
|
||||||
spawn: () => {
|
input,
|
||||||
|
options(() => {
|
||||||
throw new DOMException('blocked', 'SecurityError')
|
throw new DOMException('blocked', 'SecurityError')
|
||||||
},
|
}),
|
||||||
startTimeoutMs: 100,
|
)
|
||||||
taskTimeoutMs: 100,
|
|
||||||
})
|
|
||||||
expect(await settled(thrown)).toEqual({
|
expect(await settled(thrown)).toEqual({
|
||||||
kind: 'failed',
|
kind: 'failed',
|
||||||
reason: 'worker_unavailable',
|
reason: 'worker_unavailable',
|
||||||
@@ -112,11 +156,10 @@ describe('import client', () => {
|
|||||||
})
|
})
|
||||||
|
|
||||||
const early = scriptedWorker()
|
const early = scriptedWorker()
|
||||||
const failedEarly = runImportTask(input, {
|
const failedEarly = runImportTask(
|
||||||
spawn: () => early.worker,
|
input,
|
||||||
startTimeoutMs: 100,
|
options(() => early.worker),
|
||||||
taskTimeoutMs: 100,
|
)
|
||||||
})
|
|
||||||
early.failed('start', 'other')
|
early.failed('start', 'other')
|
||||||
expect(await settled(failedEarly)).toMatchObject({
|
expect(await settled(failedEarly)).toMatchObject({
|
||||||
kind: 'failed',
|
kind: 'failed',
|
||||||
@@ -124,11 +167,10 @@ describe('import client', () => {
|
|||||||
})
|
})
|
||||||
|
|
||||||
const silent = scriptedWorker()
|
const silent = scriptedWorker()
|
||||||
const timedOut = runImportTask(input, {
|
const timedOut = runImportTask(
|
||||||
spawn: () => silent.worker,
|
input,
|
||||||
startTimeoutMs: 100,
|
options(() => silent.worker, { taskTimeoutMs: 1000 }),
|
||||||
taskTimeoutMs: 1000,
|
)
|
||||||
})
|
|
||||||
await vi.advanceTimersByTimeAsync(99)
|
await vi.advanceTimersByTimeAsync(99)
|
||||||
expect(silent.worker.terminated).toBe(0)
|
expect(silent.worker.terminated).toBe(0)
|
||||||
await vi.advanceTimersByTimeAsync(2)
|
await vi.advanceTimersByTimeAsync(2)
|
||||||
@@ -140,20 +182,20 @@ describe('import client', () => {
|
|||||||
})
|
})
|
||||||
|
|
||||||
it('maps task failures: trap → crashed, read → read, sheet/other → engine, timer → timeout', async () => {
|
it('maps task failures: trap → crashed, read → read, sheet/other → engine, timer → timeout', async () => {
|
||||||
for (const [kind, reason] of [
|
const cases: Array<[string, string]> = [
|
||||||
['trap', 'crashed'],
|
['trap', 'crashed'],
|
||||||
['read', 'read'],
|
['read', 'read'],
|
||||||
['sheet', 'engine'],
|
['sheet', 'engine'],
|
||||||
['other', 'engine'],
|
['other', 'engine'],
|
||||||
]) {
|
]
|
||||||
|
for (const [kind, reason] of cases) {
|
||||||
const s = scriptedWorker()
|
const s = scriptedWorker()
|
||||||
const run = runImportTask(input, {
|
const run = runImportTask(
|
||||||
spawn: () => s.worker,
|
input,
|
||||||
startTimeoutMs: 100,
|
options(() => s.worker),
|
||||||
taskTimeoutMs: 100,
|
)
|
||||||
})
|
|
||||||
s.ready()
|
s.ready()
|
||||||
s.failed('task', kind!)
|
s.failed('task', kind)
|
||||||
expect(await settled(run)).toEqual({
|
expect(await settled(run)).toEqual({
|
||||||
kind: 'failed',
|
kind: 'failed',
|
||||||
reason,
|
reason,
|
||||||
@@ -162,11 +204,10 @@ describe('import client', () => {
|
|||||||
expect(s.worker.terminated).toBe(1)
|
expect(s.worker.terminated).toBe(1)
|
||||||
}
|
}
|
||||||
const slow = scriptedWorker()
|
const slow = scriptedWorker()
|
||||||
const run = runImportTask(input, {
|
const run = runImportTask(
|
||||||
spawn: () => slow.worker,
|
input,
|
||||||
startTimeoutMs: 50,
|
options(() => slow.worker, { startTimeoutMs: 50, taskTimeoutMs: 200 }),
|
||||||
taskTimeoutMs: 200,
|
)
|
||||||
})
|
|
||||||
await vi.advanceTimersByTimeAsync(40)
|
await vi.advanceTimersByTimeAsync(40)
|
||||||
slow.ready()
|
slow.ready()
|
||||||
await vi.advanceTimersByTimeAsync(199)
|
await vi.advanceTimersByTimeAsync(199)
|
||||||
@@ -180,11 +221,10 @@ describe('import client', () => {
|
|||||||
|
|
||||||
it('settles once: a result after the timeout, a failure after the result, are ignored', async () => {
|
it('settles once: a result after the timeout, a failure after the result, are ignored', async () => {
|
||||||
const s = scriptedWorker()
|
const s = scriptedWorker()
|
||||||
const run = runImportTask(input, {
|
const run = runImportTask(
|
||||||
spawn: () => s.worker,
|
input,
|
||||||
startTimeoutMs: 100,
|
options(() => s.worker),
|
||||||
taskTimeoutMs: 100,
|
)
|
||||||
})
|
|
||||||
s.ready()
|
s.ready()
|
||||||
await vi.advanceTimersByTimeAsync(101)
|
await vi.advanceTimersByTimeAsync(101)
|
||||||
s.result('{"ok":true}')
|
s.result('{"ok":true}')
|
||||||
@@ -195,11 +235,10 @@ describe('import client', () => {
|
|||||||
expect(s.worker.terminated).toBe(1)
|
expect(s.worker.terminated).toBe(1)
|
||||||
|
|
||||||
const t = scriptedWorker()
|
const t = scriptedWorker()
|
||||||
const second = runImportTask(input, {
|
const second = runImportTask(
|
||||||
spawn: () => t.worker,
|
input,
|
||||||
startTimeoutMs: 100,
|
options(() => t.worker),
|
||||||
taskTimeoutMs: 100,
|
)
|
||||||
})
|
|
||||||
t.ready()
|
t.ready()
|
||||||
t.result('{"ok":true}')
|
t.result('{"ok":true}')
|
||||||
t.failed('task', 'trap')
|
t.failed('task', 'trap')
|
||||||
@@ -210,11 +249,10 @@ describe('import client', () => {
|
|||||||
|
|
||||||
it('never masks an error event: worker_unavailable before ready, crashed after', async () => {
|
it('never masks an error event: worker_unavailable before ready, crashed after', async () => {
|
||||||
const before = scriptedWorker()
|
const before = scriptedWorker()
|
||||||
const first = runImportTask(input, {
|
const first = runImportTask(
|
||||||
spawn: () => before.worker,
|
input,
|
||||||
startTimeoutMs: 100,
|
options(() => before.worker),
|
||||||
taskTimeoutMs: 100,
|
)
|
||||||
})
|
|
||||||
before.error('script failed')
|
before.error('script failed')
|
||||||
expect(await settled(first)).toEqual({
|
expect(await settled(first)).toEqual({
|
||||||
kind: 'failed',
|
kind: 'failed',
|
||||||
@@ -222,11 +260,10 @@ describe('import client', () => {
|
|||||||
detail: 'script failed',
|
detail: 'script failed',
|
||||||
})
|
})
|
||||||
const after = scriptedWorker()
|
const after = scriptedWorker()
|
||||||
const second = runImportTask(input, {
|
const second = runImportTask(
|
||||||
spawn: () => after.worker,
|
input,
|
||||||
startTimeoutMs: 100,
|
options(() => after.worker),
|
||||||
taskTimeoutMs: 100,
|
)
|
||||||
})
|
|
||||||
after.ready()
|
after.ready()
|
||||||
after.error('boom')
|
after.error('boom')
|
||||||
expect(await settled(second)).toEqual({
|
expect(await settled(second)).toEqual({
|
||||||
@@ -236,13 +273,31 @@ describe('import client', () => {
|
|||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
|
it('a file the page cannot read settles read and terminates the worker', async () => {
|
||||||
|
const s = scriptedWorker()
|
||||||
|
const failing: ChunkSource = {
|
||||||
|
next: () => Promise.reject(new DOMException('gone', 'NotReadableError')),
|
||||||
|
cancel: async () => {},
|
||||||
|
}
|
||||||
|
const run = runImportTask(
|
||||||
|
input,
|
||||||
|
options(() => s.worker, { source: () => failing }),
|
||||||
|
)
|
||||||
|
s.ready()
|
||||||
|
expect(await settled(run)).toEqual({
|
||||||
|
kind: 'failed',
|
||||||
|
reason: 'read',
|
||||||
|
detail: 'NotReadableError: gone',
|
||||||
|
})
|
||||||
|
expect(s.worker.terminated).toBe(1)
|
||||||
|
})
|
||||||
|
|
||||||
it('cancel terminates the worker and settles cancelled, once', async () => {
|
it('cancel terminates the worker and settles cancelled, once', async () => {
|
||||||
const s = scriptedWorker()
|
const s = scriptedWorker()
|
||||||
const run = runImportTask(input, {
|
const run = runImportTask(
|
||||||
spawn: () => s.worker,
|
input,
|
||||||
startTimeoutMs: 100,
|
options(() => s.worker),
|
||||||
taskTimeoutMs: 100,
|
)
|
||||||
})
|
|
||||||
s.ready()
|
s.ready()
|
||||||
run.cancel()
|
run.cancel()
|
||||||
run.cancel()
|
run.cancel()
|
||||||
@@ -257,12 +312,10 @@ describe('import client', () => {
|
|||||||
it('pauses the task timer while the page is hidden', async () => {
|
it('pauses the task timer while the page is hidden', async () => {
|
||||||
const s = scriptedWorker()
|
const s = scriptedWorker()
|
||||||
const vis = visibility()
|
const vis = visibility()
|
||||||
const run = runImportTask(input, {
|
const run = runImportTask(
|
||||||
spawn: () => s.worker,
|
input,
|
||||||
startTimeoutMs: 100,
|
options(() => s.worker, { visibility: vis }),
|
||||||
taskTimeoutMs: 100,
|
)
|
||||||
visibility: vis,
|
|
||||||
})
|
|
||||||
s.ready()
|
s.ready()
|
||||||
await vi.advanceTimersByTimeAsync(50)
|
await vi.advanceTimersByTimeAsync(50)
|
||||||
vis.set(true)
|
vis.set(true)
|
||||||
|
|||||||
@@ -27,6 +27,7 @@ async function sheetText(): Promise<string> {
|
|||||||
|
|
||||||
interface Run {
|
interface Run {
|
||||||
messages: WorkerMessage[]
|
messages: WorkerMessage[]
|
||||||
|
acks: number
|
||||||
}
|
}
|
||||||
|
|
||||||
async function run(
|
async function run(
|
||||||
@@ -41,11 +42,31 @@ async function run(
|
|||||||
v: PROTOCOL_VERSION,
|
v: PROTOCOL_VERSION,
|
||||||
type: 'task',
|
type: 'task',
|
||||||
task,
|
task,
|
||||||
file,
|
size: file.size,
|
||||||
sheet: sheet ?? (await sheetText()),
|
sheet: sheet ?? (await sheetText()),
|
||||||
module: null,
|
module: null,
|
||||||
})
|
})
|
||||||
return { messages }
|
// The page's part: the file in 4 KiB chunks, then `end` (no transfer in-process).
|
||||||
|
if (
|
||||||
|
messages.some((m) => m.type === 'ready') &&
|
||||||
|
!messages.some((m) => m.type !== 'ready')
|
||||||
|
) {
|
||||||
|
const bytes = new Uint8Array(await file.arrayBuffer())
|
||||||
|
for (let at = 0; at < bytes.byteLength; at += 4096) {
|
||||||
|
const chunk = bytes.slice(at, at + 4096)
|
||||||
|
await handle({
|
||||||
|
v: PROTOCOL_VERSION,
|
||||||
|
type: 'chunk',
|
||||||
|
bytes: chunk.buffer,
|
||||||
|
length: chunk.byteLength,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
await handle({ v: PROTOCOL_VERSION, type: 'end' })
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
messages: messages.filter((m) => m.type !== 'ack'),
|
||||||
|
acks: messages.filter((m) => m.type === 'ack').length,
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
function envelope(messages: WorkerMessage[]): {
|
function envelope(messages: WorkerMessage[]): {
|
||||||
@@ -62,8 +83,9 @@ function envelope(messages: WorkerMessage[]): {
|
|||||||
describe('import worker handler (in-process, shipped module)', () => {
|
describe('import worker handler (in-process, shipped module)', () => {
|
||||||
it('answers ready then a result for a Faunosteo PDF, streamed in chunks', async () => {
|
it('answers ready then a result for a Faunosteo PDF, streamed in chunks', async () => {
|
||||||
const file = new File([fixture('report-v3.bare.pdf')], 'bilan.pdf')
|
const file = new File([fixture('report-v3.bare.pdf')], 'bilan.pdf')
|
||||||
const { messages } = await run('pdf', file)
|
const { messages, acks } = await run('pdf', file)
|
||||||
expect(messages.map((m) => m.type)).toEqual(['ready', 'result'])
|
expect(messages.map((m) => m.type)).toEqual(['ready', 'result'])
|
||||||
|
expect(acks).toBe(Math.ceil(file.size / 4096))
|
||||||
const out = envelope(messages)
|
const out = envelope(messages)
|
||||||
expect(out.ok).toBe(true)
|
expect(out.ok).toBe(true)
|
||||||
expect(
|
expect(
|
||||||
@@ -87,27 +109,39 @@ describe('import worker handler (in-process, shipped module)', () => {
|
|||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
it('refuses an oversized file before reading a byte', async () => {
|
it('refuses an oversized file before any chunk, and a file whose size changes', async () => {
|
||||||
const huge = {
|
const huge = { size: MAX_PDF_BYTES + 1 } as unknown as File
|
||||||
size: MAX_PDF_BYTES + 1,
|
const over = await run('pdf', huge)
|
||||||
stream: () => {
|
expect(over.acks).toBe(0)
|
||||||
throw new Error('read')
|
expect(envelope(over.messages)).toMatchObject({
|
||||||
},
|
|
||||||
} as unknown as File
|
|
||||||
expect(envelope((await run('pdf', huge)).messages)).toMatchObject({
|
|
||||||
ok: false,
|
ok: false,
|
||||||
kind: 'invalid_pdf',
|
kind: 'invalid_pdf',
|
||||||
})
|
})
|
||||||
const copy = {
|
const copy = { size: MAX_COPY_BYTES + 1 } as unknown as File
|
||||||
size: MAX_COPY_BYTES + 1,
|
|
||||||
stream: () => {
|
|
||||||
throw new Error('read')
|
|
||||||
},
|
|
||||||
} as unknown as File
|
|
||||||
expect(envelope((await run('copy', copy)).messages)).toMatchObject({
|
expect(envelope((await run('copy', copy)).messages)).toMatchObject({
|
||||||
ok: false,
|
ok: false,
|
||||||
kind: 'invalid',
|
kind: 'invalid',
|
||||||
})
|
})
|
||||||
|
// Announced 3 bytes, 4 arrive: the sink refuses and the task settles as not a PDF.
|
||||||
|
const messages: WorkerMessage[] = []
|
||||||
|
const handle = createWorkerHandler(nodeEngineLoader, (m) =>
|
||||||
|
messages.push(m),
|
||||||
|
)
|
||||||
|
await handle({
|
||||||
|
v: PROTOCOL_VERSION,
|
||||||
|
type: 'task',
|
||||||
|
task: 'pdf',
|
||||||
|
size: 3,
|
||||||
|
sheet: await sheetText(),
|
||||||
|
module: null,
|
||||||
|
})
|
||||||
|
await handle({
|
||||||
|
v: PROTOCOL_VERSION,
|
||||||
|
type: 'chunk',
|
||||||
|
bytes: new ArrayBuffer(4),
|
||||||
|
length: 4,
|
||||||
|
})
|
||||||
|
expect(envelope(messages)).toMatchObject({ ok: false, kind: 'invalid_pdf' })
|
||||||
})
|
})
|
||||||
|
|
||||||
it('gives the core kinds for a foreign PDF and garbage', async () => {
|
it('gives the core kinds for a foreign PDF and garbage', async () => {
|
||||||
@@ -150,28 +184,6 @@ describe('import worker handler (in-process, shipped module)', () => {
|
|||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
it('classifies a file that cannot be read as failed/task/read', async () => {
|
|
||||||
const error = new DOMException(
|
|
||||||
'The file could not be read',
|
|
||||||
'NotReadableError',
|
|
||||||
)
|
|
||||||
const unreadable = {
|
|
||||||
size: 10,
|
|
||||||
stream: () => ({
|
|
||||||
getReader: () => ({
|
|
||||||
read: () => Promise.reject(error),
|
|
||||||
cancel: async () => {},
|
|
||||||
}),
|
|
||||||
}),
|
|
||||||
} as unknown as File
|
|
||||||
const { messages } = await run('pdf', unreadable)
|
|
||||||
expect(messages[1]).toMatchObject({
|
|
||||||
type: 'failed',
|
|
||||||
phase: 'task',
|
|
||||||
kind: 'read',
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
it('classifies a refused sheet as failed/task/sheet and a load failure as start', async () => {
|
it('classifies a refused sheet as failed/task/sheet and a load failure as start', async () => {
|
||||||
const file = new File([fixture('report-v3.bare.pdf')], 'bilan.pdf')
|
const file = new File([fixture('report-v3.bare.pdf')], 'bilan.pdf')
|
||||||
const refused = await run(
|
const refused = await run(
|
||||||
|
|||||||
@@ -146,12 +146,17 @@ describe('HomeRoute import: success paths', () => {
|
|||||||
}
|
}
|
||||||
expect(workers).toHaveLength(1)
|
expect(workers).toHaveLength(1)
|
||||||
const task = workers[0]!.received[0] as ImportTaskMessage
|
const task = workers[0]!.received[0] as ImportTaskMessage
|
||||||
expect(task).toMatchObject({ type: 'task', task: 'pdf' })
|
expect(task).toMatchObject({ type: 'task', task: 'pdf', size: file.size })
|
||||||
expect(task.file).toBe(file)
|
|
||||||
expect(task.sheet).toBe(JSON.stringify(await bundledSheet()))
|
expect(task.sheet).toBe(JSON.stringify(await bundledSheet()))
|
||||||
expect(task.module).toBeInstanceOf(WebAssembly.Module)
|
expect(task.module).toBeInstanceOf(WebAssembly.Module)
|
||||||
|
// The page streams the file chunk by chunk, never whole: no arrayBuffer, no text.
|
||||||
expect(read).not.toHaveBeenCalled()
|
expect(read).not.toHaveBeenCalled()
|
||||||
expect(text).not.toHaveBeenCalled()
|
expect(text).not.toHaveBeenCalled()
|
||||||
|
const chunks = workers[0]!.received.filter(
|
||||||
|
(m) => (m as { type: string }).type === 'chunk',
|
||||||
|
)
|
||||||
|
expect(chunks.length).toBeGreaterThanOrEqual(1)
|
||||||
|
expect(workers[0]!.received.at(-1)).toMatchObject({ type: 'end' })
|
||||||
const imported = resultOf(workers[0]!)
|
const imported = resultOf(workers[0]!)
|
||||||
expect(hydrate).toHaveBeenCalledTimes(1)
|
expect(hydrate).toHaveBeenCalledTimes(1)
|
||||||
const [session, index] = hydrate.mock.calls[0]!
|
const [session, index] = hydrate.mock.calls[0]!
|
||||||
@@ -268,7 +273,7 @@ describe('HomeRoute import: a core failure reading the tab draft', () => {
|
|||||||
await mountAndPickPdf()
|
await mountAndPickPdf()
|
||||||
await screen.findByText(fr['home.import.error_core_unavailable']!)
|
await screen.findByText(fr['home.import.error_core_unavailable']!)
|
||||||
// The worker read the file: the failure is the store's, after the result.
|
// The worker read the file: the failure is the store's, after the result.
|
||||||
expect(workers[0]!.posted.map((m) => m.type)).toEqual(['ready', 'result'])
|
expect(workers[0]!.sequence()).toEqual(['ready', 'result'])
|
||||||
expect(alertMock()).not.toHaveBeenCalled()
|
expect(alertMock()).not.toHaveBeenCalled()
|
||||||
expect(screen.queryByTestId('editor-route')).toBeNull()
|
expect(screen.queryByTestId('editor-route')).toBeNull()
|
||||||
} finally {
|
} finally {
|
||||||
@@ -284,7 +289,7 @@ describe('HomeRoute import: a core failure reading the tab draft', () => {
|
|||||||
try {
|
try {
|
||||||
await mountAndPickPdf()
|
await mountAndPickPdf()
|
||||||
await screen.findByTestId('editor-route')
|
await screen.findByTestId('editor-route')
|
||||||
expect(workers[0]!.posted.map((m) => m.type)).toEqual(['ready', 'result'])
|
expect(workers[0]!.sequence()).toEqual(['ready', 'result'])
|
||||||
expect(alertMock()).toHaveBeenCalledWith(fr['session.held_blocked'])
|
expect(alertMock()).toHaveBeenCalledWith(fr['session.held_blocked'])
|
||||||
} finally {
|
} finally {
|
||||||
useReportStore.setState({ hydrateImport: real })
|
useReportStore.setState({ hydrateImport: real })
|
||||||
|
|||||||
@@ -194,7 +194,7 @@ describe('refusing a file', () => {
|
|||||||
await pick(copyFile(JSON.stringify(minimalReport(sheet))))
|
await pick(copyFile(JSON.stringify(minimalReport(sheet))))
|
||||||
await screen.findByText(fr['home.import.error_core_unavailable']!)
|
await screen.findByText(fr['home.import.error_core_unavailable']!)
|
||||||
// The worker answered: the line is the core's kind, not a worker failure.
|
// The worker answered: the line is the core's kind, not a worker failure.
|
||||||
expect(workers[0]!.posted.map((m) => m.type)).toEqual(['ready', 'result'])
|
expect(workers[0]!.sequence()).toEqual(['ready', 'result'])
|
||||||
expect(
|
expect(
|
||||||
screen.queryByText(fr['home.import.error_invalid_copy']!),
|
screen.queryByText(fr['home.import.error_invalid_copy']!),
|
||||||
).toBeNull()
|
).toBeNull()
|
||||||
|
|||||||
@@ -319,7 +319,7 @@ describe('the stopped text on the Home import line and the New report alert', ()
|
|||||||
await pickPdf()
|
await pickPdf()
|
||||||
await screen.findByText(fr['home.import.error_core_unavailable']!)
|
await screen.findByText(fr['home.import.error_core_unavailable']!)
|
||||||
// The worker read the file: the line is the page's index failing after it.
|
// The worker read the file: the line is the page's index failing after it.
|
||||||
expect(workers[0]!.posted.map((m) => m.type)).toEqual(['ready', 'result'])
|
expect(workers[0]!.sequence()).toEqual(['ready', 'result'])
|
||||||
})
|
})
|
||||||
|
|
||||||
it('New report: the alert says the stopped text', async () => {
|
it('New report: the alert says the stopped text', async () => {
|
||||||
|
|||||||
@@ -172,7 +172,7 @@ describe('callers stop on a failed core call', () => {
|
|||||||
})
|
})
|
||||||
await screen.findByText(fr['home.import.error_core_unavailable']!)
|
await screen.findByText(fr['home.import.error_core_unavailable']!)
|
||||||
// The worker read the file; the page's core, needed to open it, did not load.
|
// The worker read the file; the page's core, needed to open it, did not load.
|
||||||
expect(workers[0]!.posted.map((m) => m.type)).toEqual(['ready', 'result'])
|
expect(workers[0]!.sequence()).toEqual(['ready', 'result'])
|
||||||
expect(screen.queryByTestId('editor-route')).toBeNull()
|
expect(screen.queryByTestId('editor-route')).toBeNull()
|
||||||
expect(useReportStore.getState().report).toBeNull()
|
expect(useReportStore.getState().report).toBeNull()
|
||||||
})
|
})
|
||||||
|
|||||||
Reference in new issue
Block a user