test(tests): R4.1e browser harness and TS baselines (Chromium, Firefox, WebKit)

Playwright harness under scripts/browser: 127.0.0.1 server with the DEPLOY
CSP, export and import baselines of the TypeScript engine under a 4 GiB RSS
watchdog (not a guaranteed memory limit), O9 browser columns in
inflate.browsers.json (26 rows agree with Node in all three engines).
Results of 2026-10-18 committed; the O9 Node test no longer expects browser
columns in inflate.json.
This commit is contained in:
bchanot committed 2026-10-18 15:05:28 +02:00
1 parent 4bdfda59ba
commit 4439fd2420
23 files changed
+20903 -18

No files matched your search

+266
View File
@@ -0,0 +1,266 @@
import { spread } from './stats.mjs'
/**
* Markdown tables from a results file (the shape lib/harness.mjs writes),
* ready for docs/development/RUST.md §10.
*/
const MB = 1024 * 1024
const orDash = (format) => (value) =>
value === null || value === undefined ? '–' : format(value)
const mb = orDash((bytes) => (bytes / MB).toFixed(1))
const kb = orDash((bytes) => (bytes / 1024).toFixed(0))
const ms = orDash((value) => value.toFixed(0))
const s = orDash((value) => (value / 1000).toFixed(1))
/** "median (min–max)" or the single value. */
function range(values, format) {
const stats = spread(values)
if (!stats) return '–'
if (stats.n === 1) return format(stats.median)
return `${format(stats.median)} (${format(stats.min)}–${format(stats.max)})`
}
function table(header, rows) {
const line = (cells) => `| ${cells.join(' | ')} |`
return [line(header), line(header.map(() => '---')), ...rows.map(line)].join(
'\n',
)
}
function groupBy(items, keyOf) {
const groups = new Map()
for (const item of items) {
const key = keyOf(item)
groups.set(key, [...(groups.get(key) ?? []), item])
}
return groups
}
function outcomes(runs) {
const counts = groupBy(runs, (r) => r.outcome)
return [...counts]
.map(([outcome, list]) => `${outcome} ${list.length}`)
.join(', ')
}
const pick = (runs, path) =>
runs.map((r) => path.split('.').reduce((v, k) => v?.[k], r))
/** Where the watchdog stopped a run: summed RSS and PNG blobs encoded. */
function stopsOf(runs) {
const stopped = runs.filter((r) => r.outcome === 'stopped_by_ceiling')
const at = (r) =>
`${mb(r.stoppedAtBytes)} MB after ${r.rastersBeforeEnd} PNGs`
return stopped.map(at).join('; ') || '–'
}
function exportTimeRow([key, runs]) {
const [engine, report] = key.split(' ')
const done = runs.filter((r) => r.outcome === 'downloaded')
return [
engine,
report,
outcomes(runs),
range(pick(done, 'exportMs'), s),
range(pick(done, 'pdfBytes'), mb),
range(pick(done, 'rasters.count'), String),
range(pick(done, 'rasters.views.totalBytes'), mb),
range(pick(done, 'rasters.views.minBytes'), kb),
range(pick(done, 'rasters.views.maxBytes'), kb),
stopsOf(runs),
]
}
function exportMemoryRow([key, runs]) {
const [engine, report] = key.split(' ')
const done = runs.filter((r) => r.outcome === 'downloaded')
return [
engine,
report,
range(pick(runs, 'before.rssBytes'), mb),
range(pick(runs, 'rssPeakBytes'), mb),
range(pick(done, 'after.rssBytes'), mb),
range(pick(runs, 'before.rssContentBytes'), mb),
range(pick(runs, 'rssPeakContentBytes'), mb),
range(pick(done, 'after.rssContentBytes'), mb),
range(pick(runs, 'before.jsHeapUsedBytes'), mb),
range(pick(done, 'after.jsHeapUsedBytes'), mb),
]
}
export function exportTables(section) {
const groups = [...groupBy(section.runs, (r) => `${r.engine} ${r.report}`)]
const time = [
'Engine',
'Report',
'Outcome',
'Export s',
'PDF MB',
'PNG blobs',
'View PNG sum MB',
'View PNG min KB',
'View PNG max KB',
'Stopped at',
]
const memory = [
'Engine',
'Report',
'RSS before MB',
'RSS peak MB',
'RSS after MB',
'Content before MB',
'Content peak MB',
'Content after MB',
'JS heap before MB',
'JS heap after MB',
]
return [
table(time, groups.map(exportTimeRow)),
table(memory, groups.map(exportMemoryRow)),
].join('\n\n')
}
function importGroupRow([key, runs]) {
const [engine, group] = key.split(' ')
const growth = runs.map(
(r) => (r.rssPeakBytes ?? 0) - (r.rssBeforeBytes ?? 0),
)
const content = runs.map(
(r) => (r.rssPeakContentBytes ?? 0) - (r.rssBeforeContentBytes ?? 0),
)
return [
engine,
group,
String(runs.length),
outcomes(runs),
range(pick(runs, 'durationMs'), ms),
ms(Math.max(...pick(runs, 'durationMs').filter(Number.isFinite))),
mb(Math.max(...growth)),
mb(Math.max(...content)),
range(pick(runs, 'longTasks.maxMs'), ms),
range(pick(runs, 'longTasks.totalMs'), ms),
]
}
export function importGroupTable(section) {
const header = [
'Engine',
'Group',
'Files',
'Outcome',
'Duration ms',
'Max ms',
'Max RSS growth MB',
'Max content growth MB',
'Long task max ms',
'Long tasks total ms',
]
const groups = groupBy(section.runs, (r) => `${r.engine} ${r.group}`)
return table(header, [...groups].map(importGroupRow))
}
const short = (text) => (text ?? '').replace(/\s+/g, ' ').slice(0, 70)
const messageOf = (run) =>
short(run.errorText ?? run.dialogs?.[0] ?? run.error ?? '')
/** Reference files: one row per file, one cell per engine. */
export function importFileTable(section, groups, engines) {
const runs = section.runs.filter((r) => groups.includes(r.group))
const byFile = groupBy(runs, (r) => r.file)
const cell = (list, engine) => {
const run = list.find((r) => r.engine === engine)
return run ? `${run.outcome} ${ms(run.durationMs)} ms` : '–'
}
const rows = [...byFile].map(([file, list]) => [
file.replace('apps/app/tests/fixtures/', ''),
...engines.map((engine) => cell(list, engine)),
messageOf(list[0]),
])
return table(['File', ...engines, 'Message (first engine)'], rows)
}
/** Files whose outcome or message differs between engines. */
export function importDisagreements(section) {
const byFile = groupBy(section.runs, (r) => r.file)
const rows = []
for (const [file, list] of byFile) {
const verdicts = list.map((r) => `${r.outcome}: ${messageOf(r)}`)
if (new Set(verdicts).size < 2) continue
rows.push([
file.replace('apps/app/tests/fixtures/', ''),
...list.map((r, i) => `${r.engine} ${verdicts[i]}`),
])
}
if (rows.length === 0) return 'No file differs between engines.'
return rows.map((r) => `- ${r[0]}: ${r.slice(1).join(' / ')}`).join('\n')
}
/** The slowest and the most memory-hungry imports. */
export function importTop(section, count = 10) {
const growth = (r) => (r.rssPeakBytes ?? 0) - (r.rssBeforeBytes ?? 0)
const row = (r) => [
r.engine,
r.file.replace('apps/app/tests/fixtures/', ''),
r.outcome,
ms(r.durationMs),
mb(growth(r)),
short(messageOf(r)),
]
const header = [
'Engine',
'File',
'Outcome',
'Duration ms',
'RSS growth MB',
'Message',
]
const slow = [...section.runs]
.sort((a, b) => (b.durationMs ?? 0) - (a.durationMs ?? 0))
.slice(0, count)
const heavy = [...section.runs]
.sort((a, b) => growth(b) - growth(a))
.slice(0, count)
return [
'Slowest:',
table(header, slow.map(row)),
'Largest RSS growth:',
table(header, heavy.map(row)),
].join('\n\n')
}
function hostLine(host) {
const ram = `${mb(host.ramBytes)} MB RAM`
return `Host: ${host.cpuModel}, ${host.logicalCpus} threads, ${ram}.`
}
function enginesLine(result, engines) {
const list = engines.map((e) => `${e} ${result.engines[e]}`).join(', ')
return `Engines: ${list} (playwright-core ${result.playwrightCore}).`
}
export function markdownSummary(result) {
const engines = Object.keys(result.engines)
const parts = [
hostLine(result.host),
enginesLine(result, engines),
`Watchdog: ${result.watchdog.caveat}`,
]
if (result.export)
parts.push('Export (TS engine, fr):', exportTables(result.export))
if (result.import) {
parts.push('Import by group:', importGroupTable(result.import))
parts.push(
'Reference files:',
importFileTable(result.import, ['base', 'r3', 'step6'], engines),
)
parts.push('Engines that disagree:', importDisagreements(result.import))
parts.push(importTop(result.import))
if (result.import.hostile.missing.length) {
parts.push(
`Hostile inputs missing: ${result.import.hostile.missing.join(', ')}`,
)
}
}
return parts.join('\n\n')
}