// Large files through the Rust import worker, both read paths (R4.3a, // BDR-058): the 10 MiB and larger inputs of the import baseline's `read` // group, the 50 MiB file, the CAP envelope, the 16 MiB dense numbers and the // 16 MiB rescue copy, each run once per engine and per read path: // stream `file.stream()` into a `FileBytes` sink (BYOB reader when the // engine offers it), `import_pdf_bytes` / `restore_copy_bytes`; // arrayBuffer the R4.2e path: one ArrayBuffer of the file, `import_pdf` / // the strict decode then `draft.restore`. // Same metrics as r4-import.mjs (page ms, summed and content RSS before, // peak and after, linear memory, chunk count and size), a fresh context per // run, the worker terminated; the settled level before and after each run // is a forced GC + 1 s in Chromium and 5 s idle in Firefox and WebKit (idle // is not a GC). The two paths of one file run back to back, their order // alternated from one file to the next. The markdown table puts the TS // baseline's growth (peak minus before, same files) beside both paths. // // PLAYWRIGHT_NODE_MODULES=... node scripts/browser/r4-large-files.mjs \ // [--engines chromium,firefox,webkit] [--paths stream,arrayBuffer] \ // [--ts ] [--scratch ] [--out ] import { readFileSync, readdirSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' import { parseArgs } from 'node:util' import { memAvailableBytes } from './lib/host.mjs' import { importInputs, inputBytes } from './lib/import-inputs.mjs' import { RESULTS_DIR, relativeToRepo } from './lib/paths.mjs' import { ENGINES, loadPlaywright } from './lib/playwright.mjs' import { largeFilesSummary } from './lib/r4-memory-summary.mjs' import { engineRuns, listOf, oneRun, prepareModuleRun, runConditions, today, writeResult, } from './lib/r4-runs.mjs' import { startServer } from './serve.mjs' const MiB = 1024 * 1024 const LARGE_BYTES = 10 * MiB const READ_PATHS = ['stream', 'arrayBuffer'] const CLASS_IDS = [ 'file-50mib', 'CAP-envelope', 'dense-numbers-16mib', 'dense-numbers-copy-16mib', ] /** The newest `-ts-baseline.json` of the results directory. */ function newestTsBaseline() { const files = readdirSync(RESULTS_DIR).filter((f) => f.endsWith('-ts-baseline.json'), ) const newest = files.sort().pop() return newest ? join(RESULTS_DIR, newest) : null } function options() { const { values } = parseArgs({ options: { engines: { type: 'string', default: ENGINES.join(',') }, paths: { type: 'string', default: READ_PATHS.join(',') }, ts: { type: 'string' }, scratch: { type: 'string', default: tmpdir() }, out: { type: 'string', default: join(RESULTS_DIR, `${today()}-r4-large-files.json`), }, }, }) const engines = listOf(values.engines) const paths = listOf(values.paths) const bad = [...engines.filter((e) => !ENGINES.includes(e))] bad.push(...paths.filter((p) => !READ_PATHS.includes(p))) if (bad.length) throw new Error(`unknown engines or paths: ${bad}`) return { ...values, engines, paths, ts: values.ts ?? newestTsBaseline() } } /** The `read` group's inputs of 10 MiB and more, as the baseline names them. */ function largeInputs() { const { inputs } = importInputs(['read']) return inputs .map((input) => ({ input, bytes: inputBytes(input.path) })) .filter(({ bytes }) => bytes >= LARGE_BYTES) .map(({ input, bytes }) => ({ section: 'large', group: input.group, kind: input.kind ?? null, file: relativeToRepo(input.path), fileBytes: bytes, input, })) } const classItems = (classes) => classes.map((c) => ({ section: 'large', classId: c.id, fileBytes: c.bytes, path: c.path, })) /** Each file once per path, back to back, the order alternated per file. */ function planOf(files, paths) { return files.flatMap((file, i) => { const order = i % 2 ? [...paths].reverse() : paths return order.map((readPath) => ({ ...file, readPath })) }) } const settleOf = (engine) => (engine === 'chromium' ? 'gc' : 'idle') /** The TS baseline's run of each large file: growth and Node-clock time. */ function tsRows(tsPath, files) { if (!tsPath) return { file: null, rows: [] } const ts = JSON.parse(readFileSync(tsPath, 'utf8')) const wanted = new Set(files.map((f) => f.file).filter(Boolean)) const rows = ts.import.runs .filter((r) => wanted.has(r.file)) .map((r) => ({ engine: r.engine, file: r.file, outcome: r.outcome, durationMs: r.durationMs, growthBytes: r.rssPeakBytes - r.rssBeforeBytes, contentGrowthBytes: r.rssPeakContentBytes - r.rssBeforeContentBytes, })) return { file: relativeToRepo(tsPath), rows } } function conditions(server, playwrightCore, module) { const harness = 'scripts/browser/r4-large-files.mjs' return { ...runConditions({ harness, server, playwrightCore, module }), page: { worker: 'module worker /r4/worker.js, one per import, terminated', readPaths: { stream: 'file.stream() (BYOB reader into one 1 MiB buffer when the ' + 'engine offers it, else the default reader) pushed into ' + 'FileBytes(file.size), then import_pdf_bytes / restore_copy_bytes', arrayBuffer: 'file.arrayBuffer(), import_pdf (the glue copies it) or the ' + 'strict TextDecoder then draft.restore (R4.2e)', }, settle: 'Chromium: forced GC + 1 s; Firefox, WebKit: 5 s idle', }, perimeter: 'the TS baseline ran the application (import-baseline.mjs); the Rust ' + 'runs use the measurement page; same files, RSS metrics, watchdog', } } async function main() { const opts = options() const { pw, version } = loadPlaywright() const prepared = await prepareModuleRun({ scratchParent: opts.scratch, classes: CLASS_IDS, }) const server = await startServer({ mounts: prepared.mounts }) try { const files = [...largeInputs(), ...classItems(prepared.classes)] const plan = planOf(files, opts.paths) const result = conditions(server, version, prepared.module) const context = { ...prepared, baseUrl: server.url } const runs = [] for (const engine of opts.engines) { const items = plan.map((item) => ({ ...item, settle: settleOf(engine) })) const runItem = (session, item) => oneRun(session, item, context) runs.push(...(await engineRuns(pw, engine, items, runItem))) } result.host.memAvailableBytesAtEnd = memAvailableBytes() result.endedAt = new Date().toISOString() result.engines = Object.fromEntries( runs.map((r) => [r.engine, r.engineVersion]), ) result.files = files.map((f) => ({ file: f.file ?? f.classId, bytes: f.fileBytes, })) result.tsBaseline = tsRows(opts.ts, files) result.runs = runs writeResult(opts.out, result, largeFilesSummary) } finally { server.close() prepared.cleanup() } } await main()