feat(app): R4.3a the page reads the file chunk by chunk and streams it to the worker's sink
Measured at R4.3a (RUST.md §10.5): reading a file inside a WebKit worker costs about 180 MiB of content RSS whatever the method (arrayBuffer, stream, BYOB), 14 MiB on the page; a page-side BYOB read feeding the worker's FileBytes gives 106 MiB on a 50 MiB file against 272 in the worker and 76 for the TypeScript. Protocol v1 gains chunk / ack / end: one 1 MiB buffer travels (transferred) and comes back after each push, so the page holds one chunk and the worker's sink the only whole copy; no byte is parsed on the page. The worker refuses an oversize announced before any chunk and a size that changes while reading. Tests adapted (handler, client with a scripted ack, read failure on the page).
This commit is contained in:
1 parent
49591c53cd
commit
e659924ec0
12 files changed
+552
-272
No files matched your search
@@ -1,19 +1,23 @@
|
||||
/**
|
||||
* R4.3: the page side of one import (plan §B.1.5 items 1, 3, 5): the worker created in a
|
||||
* try/catch, listeners before the post, the start timer until `ready`, the task timer
|
||||
* after, settle once on the first of result / failed / error event / timer / cancel, the
|
||||
* worker terminated in every case, later messages ignored, timers paused while hidden.
|
||||
* after, the file fed chunk by chunk (one travelling buffer, each chunk acknowledged),
|
||||
* settle once on the first of result / failed / error event / read failure / timer /
|
||||
* cancel, the worker terminated in every case, later messages ignored, timers paused
|
||||
* while hidden.
|
||||
*/
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import {
|
||||
runImportTask,
|
||||
type ImportClientOptions,
|
||||
type ImportSettlement,
|
||||
type WorkerLike,
|
||||
} from '@/core/worker/client'
|
||||
import { PROTOCOL_VERSION } from '@/core/worker/protocol'
|
||||
import type { ChunkSource } from '@/core/worker/reader'
|
||||
import type { Visibility } from '@/core/worker/timers'
|
||||
|
||||
/** A scripted worker: the test drives its messages and events. */
|
||||
/** A scripted worker: the test drives its messages and events; chunks are acknowledged. */
|
||||
function scriptedWorker() {
|
||||
const worker: WorkerLike & { terminated: number; sent: unknown[] } = {
|
||||
terminated: 0,
|
||||
@@ -21,7 +25,13 @@ function scriptedWorker() {
|
||||
onmessage: null,
|
||||
onerror: null,
|
||||
onmessageerror: null,
|
||||
postMessage: (m) => worker.sent.push(m),
|
||||
postMessage: (m) => {
|
||||
worker.sent.push(m)
|
||||
const message = m as { type?: string; bytes?: ArrayBuffer }
|
||||
if (message.type === 'chunk') {
|
||||
emit({ v: PROTOCOL_VERSION, type: 'ack', bytes: message.bytes })
|
||||
}
|
||||
},
|
||||
terminate: () => {
|
||||
worker.terminated += 1
|
||||
},
|
||||
@@ -46,6 +56,7 @@ function scriptedWorker() {
|
||||
detail: `${kind}!`,
|
||||
}),
|
||||
error: (message: string) => worker.onerror?.({ message } as ErrorEvent),
|
||||
types: () => worker.sent.map((m) => (m as { type: string }).type),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -65,8 +76,37 @@ function visibility(): Visibility & { set: (hidden: boolean) => void } {
|
||||
}
|
||||
}
|
||||
|
||||
const file = new File([new Uint8Array([1])], 'a.pdf')
|
||||
/** A chunk source over `parts`, each filled into the travelling buffer. */
|
||||
function partsSource(parts: Uint8Array[]): ChunkSource {
|
||||
let at = 0
|
||||
return {
|
||||
async next(buffer) {
|
||||
const part = parts[at]
|
||||
if (!part) return null
|
||||
at += 1
|
||||
new Uint8Array(buffer, 0, part.byteLength).set(part)
|
||||
return { bytes: buffer, length: part.byteLength }
|
||||
},
|
||||
cancel: async () => {},
|
||||
}
|
||||
}
|
||||
|
||||
const file = new File([new Uint8Array([1, 2, 3])], 'a.pdf')
|
||||
const input = { task: 'pdf' as const, file, sheet: '{}', module: null }
|
||||
const empty = () => partsSource([])
|
||||
|
||||
function options(
|
||||
spawn: () => WorkerLike,
|
||||
patch: Partial<ImportClientOptions> = {},
|
||||
): ImportClientOptions {
|
||||
return {
|
||||
spawn,
|
||||
startTimeoutMs: 100,
|
||||
taskTimeoutMs: 100,
|
||||
source: empty,
|
||||
...patch,
|
||||
}
|
||||
}
|
||||
|
||||
async function settled(run: {
|
||||
settled: Promise<ImportSettlement>
|
||||
@@ -79,16 +119,21 @@ describe('import client', () => {
|
||||
beforeEach(() => vi.useFakeTimers())
|
||||
afterEach(() => vi.useRealTimers())
|
||||
|
||||
it('posts the task after installing the listeners, then settles on the result and terminates', async () => {
|
||||
it('posts the task, feeds the chunks then end, settles on the result and terminates', async () => {
|
||||
const s = scriptedWorker()
|
||||
const run = runImportTask(input, {
|
||||
spawn: () => s.worker,
|
||||
startTimeoutMs: 100,
|
||||
taskTimeoutMs: 100,
|
||||
})
|
||||
const parts = [new Uint8Array([1, 2]), new Uint8Array([3])]
|
||||
const run = runImportTask(
|
||||
input,
|
||||
options(() => s.worker, { source: () => partsSource(parts) }),
|
||||
)
|
||||
expect(s.worker.onmessage).not.toBeNull()
|
||||
expect(s.worker.sent).toEqual([{ v: 1, type: 'task', ...input }])
|
||||
expect(s.worker.sent).toEqual([
|
||||
{ v: 1, type: 'task', task: 'pdf', size: 3, sheet: '{}', module: null },
|
||||
])
|
||||
s.ready()
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
expect(s.types()).toEqual(['task', 'chunk', 'chunk', 'end'])
|
||||
expect((s.worker.sent[1] as { length: number }).length).toBe(2)
|
||||
s.result('{"ok":true}')
|
||||
expect(await settled(run)).toMatchObject({
|
||||
kind: 'result',
|
||||
@@ -98,13 +143,12 @@ describe('import client', () => {
|
||||
})
|
||||
|
||||
it('is worker_unavailable when the spawn throws, fails before ready, or never answers ready', async () => {
|
||||
const thrown = runImportTask(input, {
|
||||
spawn: () => {
|
||||
const thrown = runImportTask(
|
||||
input,
|
||||
options(() => {
|
||||
throw new DOMException('blocked', 'SecurityError')
|
||||
},
|
||||
startTimeoutMs: 100,
|
||||
taskTimeoutMs: 100,
|
||||
})
|
||||
}),
|
||||
)
|
||||
expect(await settled(thrown)).toEqual({
|
||||
kind: 'failed',
|
||||
reason: 'worker_unavailable',
|
||||
@@ -112,11 +156,10 @@ describe('import client', () => {
|
||||
})
|
||||
|
||||
const early = scriptedWorker()
|
||||
const failedEarly = runImportTask(input, {
|
||||
spawn: () => early.worker,
|
||||
startTimeoutMs: 100,
|
||||
taskTimeoutMs: 100,
|
||||
})
|
||||
const failedEarly = runImportTask(
|
||||
input,
|
||||
options(() => early.worker),
|
||||
)
|
||||
early.failed('start', 'other')
|
||||
expect(await settled(failedEarly)).toMatchObject({
|
||||
kind: 'failed',
|
||||
@@ -124,11 +167,10 @@ describe('import client', () => {
|
||||
})
|
||||
|
||||
const silent = scriptedWorker()
|
||||
const timedOut = runImportTask(input, {
|
||||
spawn: () => silent.worker,
|
||||
startTimeoutMs: 100,
|
||||
taskTimeoutMs: 1000,
|
||||
})
|
||||
const timedOut = runImportTask(
|
||||
input,
|
||||
options(() => silent.worker, { taskTimeoutMs: 1000 }),
|
||||
)
|
||||
await vi.advanceTimersByTimeAsync(99)
|
||||
expect(silent.worker.terminated).toBe(0)
|
||||
await vi.advanceTimersByTimeAsync(2)
|
||||
@@ -140,20 +182,20 @@ describe('import client', () => {
|
||||
})
|
||||
|
||||
it('maps task failures: trap → crashed, read → read, sheet/other → engine, timer → timeout', async () => {
|
||||
for (const [kind, reason] of [
|
||||
const cases: Array<[string, string]> = [
|
||||
['trap', 'crashed'],
|
||||
['read', 'read'],
|
||||
['sheet', 'engine'],
|
||||
['other', 'engine'],
|
||||
]) {
|
||||
]
|
||||
for (const [kind, reason] of cases) {
|
||||
const s = scriptedWorker()
|
||||
const run = runImportTask(input, {
|
||||
spawn: () => s.worker,
|
||||
startTimeoutMs: 100,
|
||||
taskTimeoutMs: 100,
|
||||
})
|
||||
const run = runImportTask(
|
||||
input,
|
||||
options(() => s.worker),
|
||||
)
|
||||
s.ready()
|
||||
s.failed('task', kind!)
|
||||
s.failed('task', kind)
|
||||
expect(await settled(run)).toEqual({
|
||||
kind: 'failed',
|
||||
reason,
|
||||
@@ -162,11 +204,10 @@ describe('import client', () => {
|
||||
expect(s.worker.terminated).toBe(1)
|
||||
}
|
||||
const slow = scriptedWorker()
|
||||
const run = runImportTask(input, {
|
||||
spawn: () => slow.worker,
|
||||
startTimeoutMs: 50,
|
||||
taskTimeoutMs: 200,
|
||||
})
|
||||
const run = runImportTask(
|
||||
input,
|
||||
options(() => slow.worker, { startTimeoutMs: 50, taskTimeoutMs: 200 }),
|
||||
)
|
||||
await vi.advanceTimersByTimeAsync(40)
|
||||
slow.ready()
|
||||
await vi.advanceTimersByTimeAsync(199)
|
||||
@@ -180,11 +221,10 @@ describe('import client', () => {
|
||||
|
||||
it('settles once: a result after the timeout, a failure after the result, are ignored', async () => {
|
||||
const s = scriptedWorker()
|
||||
const run = runImportTask(input, {
|
||||
spawn: () => s.worker,
|
||||
startTimeoutMs: 100,
|
||||
taskTimeoutMs: 100,
|
||||
})
|
||||
const run = runImportTask(
|
||||
input,
|
||||
options(() => s.worker),
|
||||
)
|
||||
s.ready()
|
||||
await vi.advanceTimersByTimeAsync(101)
|
||||
s.result('{"ok":true}')
|
||||
@@ -195,11 +235,10 @@ describe('import client', () => {
|
||||
expect(s.worker.terminated).toBe(1)
|
||||
|
||||
const t = scriptedWorker()
|
||||
const second = runImportTask(input, {
|
||||
spawn: () => t.worker,
|
||||
startTimeoutMs: 100,
|
||||
taskTimeoutMs: 100,
|
||||
})
|
||||
const second = runImportTask(
|
||||
input,
|
||||
options(() => t.worker),
|
||||
)
|
||||
t.ready()
|
||||
t.result('{"ok":true}')
|
||||
t.failed('task', 'trap')
|
||||
@@ -210,11 +249,10 @@ describe('import client', () => {
|
||||
|
||||
it('never masks an error event: worker_unavailable before ready, crashed after', async () => {
|
||||
const before = scriptedWorker()
|
||||
const first = runImportTask(input, {
|
||||
spawn: () => before.worker,
|
||||
startTimeoutMs: 100,
|
||||
taskTimeoutMs: 100,
|
||||
})
|
||||
const first = runImportTask(
|
||||
input,
|
||||
options(() => before.worker),
|
||||
)
|
||||
before.error('script failed')
|
||||
expect(await settled(first)).toEqual({
|
||||
kind: 'failed',
|
||||
@@ -222,11 +260,10 @@ describe('import client', () => {
|
||||
detail: 'script failed',
|
||||
})
|
||||
const after = scriptedWorker()
|
||||
const second = runImportTask(input, {
|
||||
spawn: () => after.worker,
|
||||
startTimeoutMs: 100,
|
||||
taskTimeoutMs: 100,
|
||||
})
|
||||
const second = runImportTask(
|
||||
input,
|
||||
options(() => after.worker),
|
||||
)
|
||||
after.ready()
|
||||
after.error('boom')
|
||||
expect(await settled(second)).toEqual({
|
||||
@@ -236,13 +273,31 @@ describe('import client', () => {
|
||||
})
|
||||
})
|
||||
|
||||
it('a file the page cannot read settles read and terminates the worker', async () => {
|
||||
const s = scriptedWorker()
|
||||
const failing: ChunkSource = {
|
||||
next: () => Promise.reject(new DOMException('gone', 'NotReadableError')),
|
||||
cancel: async () => {},
|
||||
}
|
||||
const run = runImportTask(
|
||||
input,
|
||||
options(() => s.worker, { source: () => failing }),
|
||||
)
|
||||
s.ready()
|
||||
expect(await settled(run)).toEqual({
|
||||
kind: 'failed',
|
||||
reason: 'read',
|
||||
detail: 'NotReadableError: gone',
|
||||
})
|
||||
expect(s.worker.terminated).toBe(1)
|
||||
})
|
||||
|
||||
it('cancel terminates the worker and settles cancelled, once', async () => {
|
||||
const s = scriptedWorker()
|
||||
const run = runImportTask(input, {
|
||||
spawn: () => s.worker,
|
||||
startTimeoutMs: 100,
|
||||
taskTimeoutMs: 100,
|
||||
})
|
||||
const run = runImportTask(
|
||||
input,
|
||||
options(() => s.worker),
|
||||
)
|
||||
s.ready()
|
||||
run.cancel()
|
||||
run.cancel()
|
||||
@@ -257,12 +312,10 @@ describe('import client', () => {
|
||||
it('pauses the task timer while the page is hidden', async () => {
|
||||
const s = scriptedWorker()
|
||||
const vis = visibility()
|
||||
const run = runImportTask(input, {
|
||||
spawn: () => s.worker,
|
||||
startTimeoutMs: 100,
|
||||
taskTimeoutMs: 100,
|
||||
visibility: vis,
|
||||
})
|
||||
const run = runImportTask(
|
||||
input,
|
||||
options(() => s.worker, { visibility: vis }),
|
||||
)
|
||||
s.ready()
|
||||
await vi.advanceTimersByTimeAsync(50)
|
||||
vis.set(true)
|
||||
|
||||
Reference in new issue
Block a user