Files
bchanot 4439fd2420 test(tests): R4.1e browser harness and TS baselines (Chromium, Firefox, WebKit)
Playwright harness under scripts/browser: 127.0.0.1 server with the DEPLOY
CSP, export and import baselines of the TypeScript engine under a 4 GiB RSS
watchdog (not a guaranteed memory limit), O9 browser columns in
inflate.browsers.json (26 rows agree with Node in all three engines).
Results of 2026-10-18 committed; the O9 Node test no longer expects browser
columns in inflate.json.
2026-10-18 15:05:28 +02:00

267 lines
7.7 KiB
JavaScript
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import { spread } from './stats.mjs'
/**
* Markdown tables from a results file (the shape lib/harness.mjs writes),
* ready for docs/development/RUST.md §10.
*/
const MB = 1024 * 1024
const orDash = (format) => (value) =>
value === null || value === undefined ? '–' : format(value)
const mb = orDash((bytes) => (bytes / MB).toFixed(1))
const kb = orDash((bytes) => (bytes / 1024).toFixed(0))
const ms = orDash((value) => value.toFixed(0))
const s = orDash((value) => (value / 1000).toFixed(1))
/** "median (min–max)" or the single value. */
function range(values, format) {
const stats = spread(values)
if (!stats) return '–'
if (stats.n === 1) return format(stats.median)
return `${format(stats.median)} (${format(stats.min)}–${format(stats.max)})`
}
function table(header, rows) {
const line = (cells) => `| ${cells.join(' | ')} |`
return [line(header), line(header.map(() => '---')), ...rows.map(line)].join(
'\n',
)
}
function groupBy(items, keyOf) {
const groups = new Map()
for (const item of items) {
const key = keyOf(item)
groups.set(key, [...(groups.get(key) ?? []), item])
}
return groups
}
function outcomes(runs) {
const counts = groupBy(runs, (r) => r.outcome)
return [...counts]
.map(([outcome, list]) => `${outcome} ${list.length}`)
.join(', ')
}
const pick = (runs, path) =>
runs.map((r) => path.split('.').reduce((v, k) => v?.[k], r))
/** Where the watchdog stopped a run: summed RSS and PNG blobs encoded. */
function stopsOf(runs) {
const stopped = runs.filter((r) => r.outcome === 'stopped_by_ceiling')
const at = (r) =>
`${mb(r.stoppedAtBytes)} MB after ${r.rastersBeforeEnd} PNGs`
return stopped.map(at).join('; ') || '–'
}
function exportTimeRow([key, runs]) {
const [engine, report] = key.split(' ')
const done = runs.filter((r) => r.outcome === 'downloaded')
return [
engine,
report,
outcomes(runs),
range(pick(done, 'exportMs'), s),
range(pick(done, 'pdfBytes'), mb),
range(pick(done, 'rasters.count'), String),
range(pick(done, 'rasters.views.totalBytes'), mb),
range(pick(done, 'rasters.views.minBytes'), kb),
range(pick(done, 'rasters.views.maxBytes'), kb),
stopsOf(runs),
]
}
function exportMemoryRow([key, runs]) {
const [engine, report] = key.split(' ')
const done = runs.filter((r) => r.outcome === 'downloaded')
return [
engine,
report,
range(pick(runs, 'before.rssBytes'), mb),
range(pick(runs, 'rssPeakBytes'), mb),
range(pick(done, 'after.rssBytes'), mb),
range(pick(runs, 'before.rssContentBytes'), mb),
range(pick(runs, 'rssPeakContentBytes'), mb),
range(pick(done, 'after.rssContentBytes'), mb),
range(pick(runs, 'before.jsHeapUsedBytes'), mb),
range(pick(done, 'after.jsHeapUsedBytes'), mb),
]
}
export function exportTables(section) {
const groups = [...groupBy(section.runs, (r) => `${r.engine} ${r.report}`)]
const time = [
'Engine',
'Report',
'Outcome',
'Export s',
'PDF MB',
'PNG blobs',
'View PNG sum MB',
'View PNG min KB',
'View PNG max KB',
'Stopped at',
]
const memory = [
'Engine',
'Report',
'RSS before MB',
'RSS peak MB',
'RSS after MB',
'Content before MB',
'Content peak MB',
'Content after MB',
'JS heap before MB',
'JS heap after MB',
]
return [
table(time, groups.map(exportTimeRow)),
table(memory, groups.map(exportMemoryRow)),
].join('\n\n')
}
function importGroupRow([key, runs]) {
const [engine, group] = key.split(' ')
const growth = runs.map(
(r) => (r.rssPeakBytes ?? 0) - (r.rssBeforeBytes ?? 0),
)
const content = runs.map(
(r) => (r.rssPeakContentBytes ?? 0) - (r.rssBeforeContentBytes ?? 0),
)
return [
engine,
group,
String(runs.length),
outcomes(runs),
range(pick(runs, 'durationMs'), ms),
ms(Math.max(...pick(runs, 'durationMs').filter(Number.isFinite))),
mb(Math.max(...growth)),
mb(Math.max(...content)),
range(pick(runs, 'longTasks.maxMs'), ms),
range(pick(runs, 'longTasks.totalMs'), ms),
]
}
export function importGroupTable(section) {
const header = [
'Engine',
'Group',
'Files',
'Outcome',
'Duration ms',
'Max ms',
'Max RSS growth MB',
'Max content growth MB',
'Long task max ms',
'Long tasks total ms',
]
const groups = groupBy(section.runs, (r) => `${r.engine} ${r.group}`)
return table(header, [...groups].map(importGroupRow))
}
const short = (text) => (text ?? '').replace(/\s+/g, ' ').slice(0, 70)
const messageOf = (run) =>
short(run.errorText ?? run.dialogs?.[0] ?? run.error ?? '')
/** Reference files: one row per file, one cell per engine. */
export function importFileTable(section, groups, engines) {
const runs = section.runs.filter((r) => groups.includes(r.group))
const byFile = groupBy(runs, (r) => r.file)
const cell = (list, engine) => {
const run = list.find((r) => r.engine === engine)
return run ? `${run.outcome} ${ms(run.durationMs)} ms` : '–'
}
const rows = [...byFile].map(([file, list]) => [
file.replace('apps/app/tests/fixtures/', ''),
...engines.map((engine) => cell(list, engine)),
messageOf(list[0]),
])
return table(['File', ...engines, 'Message (first engine)'], rows)
}
/** Files whose outcome or message differs between engines. */
export function importDisagreements(section) {
const byFile = groupBy(section.runs, (r) => r.file)
const rows = []
for (const [file, list] of byFile) {
const verdicts = list.map((r) => `${r.outcome}: ${messageOf(r)}`)
if (new Set(verdicts).size < 2) continue
rows.push([
file.replace('apps/app/tests/fixtures/', ''),
...list.map((r, i) => `${r.engine} ${verdicts[i]}`),
])
}
if (rows.length === 0) return 'No file differs between engines.'
return rows.map((r) => `- ${r[0]}: ${r.slice(1).join(' / ')}`).join('\n')
}
/** The slowest and the most memory-hungry imports. */
export function importTop(section, count = 10) {
const growth = (r) => (r.rssPeakBytes ?? 0) - (r.rssBeforeBytes ?? 0)
const row = (r) => [
r.engine,
r.file.replace('apps/app/tests/fixtures/', ''),
r.outcome,
ms(r.durationMs),
mb(growth(r)),
short(messageOf(r)),
]
const header = [
'Engine',
'File',
'Outcome',
'Duration ms',
'RSS growth MB',
'Message',
]
const slow = [...section.runs]
.sort((a, b) => (b.durationMs ?? 0) - (a.durationMs ?? 0))
.slice(0, count)
const heavy = [...section.runs]
.sort((a, b) => growth(b) - growth(a))
.slice(0, count)
return [
'Slowest:',
table(header, slow.map(row)),
'Largest RSS growth:',
table(header, heavy.map(row)),
].join('\n\n')
}
function hostLine(host) {
const ram = `${mb(host.ramBytes)} MB RAM`
return `Host: ${host.cpuModel}, ${host.logicalCpus} threads, ${ram}.`
}
function enginesLine(result, engines) {
const list = engines.map((e) => `${e} ${result.engines[e]}`).join(', ')
return `Engines: ${list} (playwright-core ${result.playwrightCore}).`
}
export function markdownSummary(result) {
const engines = Object.keys(result.engines)
const parts = [
hostLine(result.host),
enginesLine(result, engines),
`Watchdog: ${result.watchdog.caveat}`,
]
if (result.export)
parts.push('Export (TS engine, fr):', exportTables(result.export))
if (result.import) {
parts.push('Import by group:', importGroupTable(result.import))
parts.push(
'Reference files:',
importFileTable(result.import, ['base', 'r3', 'step6'], engines),
)
parts.push('Engines that disagree:', importDisagreements(result.import))
parts.push(importTop(result.import))
if (result.import.hostile.missing.length) {
parts.push(
`Hostile inputs missing: ${result.import.hostile.missing.join(', ')}`,
)
}
}
return parts.join('\n\n')
}