test(tests): R4.1f O10 report:compare outputs and the R4 test ledger (299 cases)

This commit is contained in:
bchanot committed 2026-10-18 06:00:34 +02:00
1 parent 516342beca
commit 4bdfda59ba
19 files changed
+9291

No files matched your search

+418
View File
@@ -0,0 +1,418 @@
// R4 test ledger (plan 2026-10-17 §B.6): one row per TypeScript test case
// of the PDF path at the freeze, with its planned disposition. Runs vitest
// with its JSON reporter (report files in os.tmpdir()) on every packages/pdf
// suite, the application's PDF, route and store suites and its r4.*
// generators, and the core-wasm N-1 scanner test; writes
// crates/core/tests/pdf/LEDGER.tsv (files sorted, cases in declaration
// order) and prints the rows per file and disposition. Fails, writing
// nothing, when a run fails, a requested file is missing from its report, a
// case matches no rule, a rule matches no case, or the rows do not reconcile
// with vitest's totals. "(pending)" targets are filled when the case is
// ported.
// Usage: node scripts/r4-ledger.mjs (core module built: pnpm wasm:build)
import { spawnSync } from 'node:child_process'
import {
existsSync,
mkdirSync,
mkdtempSync,
readdirSync,
readFileSync,
rmdirSync,
unlinkSync,
writeFileSync,
} from 'node:fs'
import { tmpdir } from 'node:os'
import { dirname, join, relative, resolve } from 'node:path'
import { fileURLToPath } from 'node:url'
const ROOT = resolve(fileURLToPath(new URL('..', import.meta.url)))
const OUT = join(ROOT, 'crates/core/tests/pdf/LEDGER.tsv')
const COLUMNS = [
'ts_file',
'case_title',
'disposition',
'target',
'oracle',
'note',
]
const DISPOSITIONS = ['ported', 'replaced', 'kept', 'deleted']
const PDF = 'packages/pdf/tests/'
const APP = 'apps/app/tests/unit/'
const N1_SCANNER = 'packages/core-wasm/tests/n-1.scanner.test.ts'
const APP_PDF_SUITES = [
'pdf.goldens.test.ts',
'pdf.roundtrip.test.ts',
'pdf.import.test.ts',
'archive.envelopes.test.ts',
'n-1.reader.test.ts',
'pdf.export.app.test.ts',
'migration.leaves.test.ts',
'pdf.browserRasterizer.test.ts',
'scripts.compare.test.ts',
]
/** Suites mocking `@/pdf/*`: rewritten when the import (R4.3) and the
* export (R4.6) switch. */
const APP_MOCKING_SUITES = [
'routes.HomeRoute.archive.test.tsx',
'routes.HomeRoute.import.test.tsx',
'routes.HomeRoute.rescueCopy.test.tsx',
'routes.EditorRoute.export.test.tsx',
'routes.coreStopped.test.tsx',
'state.coreFailures.test.tsx',
]
const READER = 'R4.2 cargo (pending)'
const WRITER = 'R4.4/R4.5 cargo (pending)'
const CARGO_GOLDENS = 'R4.4/R4.5 cargo goldens (pending)'
const SAME = 'vitest (same case)'
const NONE = '-'
const GENERATOR = 'generator; oracle files kept (R4.8)'
const N1_SWITCH = 'reader switched to the frozen O11 scanner at R4.8'
const PDF_LIB_LEAVES =
'pdf-lib leaves the app; read-back moves to cargo/lopdf or the wasm reader'
const ported = (target, oracle, note = NONE) => ({
disposition: 'ported',
target,
oracle,
note,
})
const kept = (oracle, note = NONE) => ({
disposition: 'kept',
target: SAME,
oracle,
note,
})
const replaced = (target, oracle, note) => ({
disposition: 'replaced',
target,
oracle,
note,
})
const generator = ([file, oracle]) => ({
file,
disposition: 'deleted',
target: NONE,
oracle,
note: GENERATOR,
})
const SECURITY = `${PDF}security.test.ts`
const GOLDENS = `${APP}pdf.goldens.test.ts`
// First match wins. `title` (a prefix or a RegExp) splits a file's cases.
const RULES = [
// packages/pdf, reader side (R4.2)
{ file: `${PDF}scan.test.ts`, ...ported(READER, 'O7') },
{ file: `${PDF}inflate.test.ts`, ...ported(READER, 'O7/O9') },
{ file: `${PDF}read.test.ts`, ...ported(READER, 'O7') },
{
file: SECURITY,
title: 'import: the report survives a rejected snapshot > ',
...ported(READER, 'O7'),
},
{
file: SECURITY,
title: '__proto__ ids and the inflate bomb > inflates a normal payload',
...ported(READER, 'O7'),
},
// packages/pdf, export side (R4.4 layout, R4.5 serialiser)
{
file: SECURITY,
title: '__proto__ ids and the inflate bomb > report ids named __proto__',
...ported(WRITER, 'O1'),
},
{
file: SECURITY,
title: 'export: a snapshot over the caps fails',
...ported(WRITER, NONE, 'typed export errors, no oracle file'),
},
{
file: SECURITY,
title: 'hostile SVG assets > ',
...ported(
WRITER,
NONE,
'export SVG path; R4 refuses where the TS sanitised (A.3, RUST.md §9.4)',
),
},
{
file: SECURITY,
title: 'branding URL injection (HIGH) > ',
...ported(WRITER, 'O2', 'link sink'),
},
{ file: SECURITY, title: 'PNG budget (MEDIUM) > ', ...ported(WRITER, 'O6') },
{ file: `${PDF}goldens.test.ts`, ...ported(WRITER, 'O1/O2') },
{
file: `${PDF}text.test.ts`,
title: 'formatDate > ',
...ported(WRITER, 'O5'),
},
{ file: `${PDF}text.test.ts`, ...ported(WRITER, 'O4') },
{ file: `${PDF}size.test.ts`, ...ported(WRITER, 'O2', 'E2 budget') },
{ file: `${PDF}raster.test.ts`, ...ported(WRITER, 'O2/O4') },
{ file: `${PDF}reexport.test.ts`, ...ported(WRITER, 'O2') },
// Oracle generators, deleted with the TS they freeze; the oracle files stay
...[
[`${PDF}r3.highlight-oracles.test.ts`, 'R3 highlight'],
[`${PDF}r3.sanitiser-oracles.test.ts`, 'R3 sanitiser'],
[`${PDF}r4.dates-oracles.test.ts`, 'O5'],
[`${PDF}r4.inflate-oracles.test.ts`, 'O9'],
[`${PDF}r4.png-oracles.test.ts`, 'O6'],
[`${PDF}r4.reader-oracles.test.ts`, 'O7'],
[`${PDF}r4.text-oracles.test.ts`, 'O4'],
[`${APP}r4.compare-oracles.test.ts`, 'O10'],
[`${APP}r4.tolerance-oracles.test.ts`, 'O8'],
[`${APP}r4.writer-oracles.test.ts`, 'O1/O2/O3'],
].map(generator),
{
file: `${APP}r4.synthetic-reports.test.ts`,
...kept('O13', 'runs on the core only'),
},
// apps/app PDF suites
{
file: GOLDENS,
title: 'golden inputs > the 2×1 PNG decodes',
...replaced(
CARGO_GOLDENS,
'O6',
'pdf-lib embedPng check of the golden rasters',
),
},
{
file: GOLDENS,
title: 'golden inputs > ',
...kept(NONE, 'inputs of the kept views cases'),
},
{
file: GOLDENS,
title: /^goldens of [^>]+ > [^>]+ > text, images and links per locale/,
...replaced(CARGO_GOLDENS, 'O1', 'Step 6 text, images and links goldens'),
},
{
file: GOLDENS,
title: /^goldens of [^>]+ > highlighted views/,
...kept(NONE, 'app path'),
},
{
file: GOLDENS,
title: 'report-v3.bare.pdf > ',
...kept(NONE, 'bare v3 import'),
},
{
file: `${APP}pdf.import.test.ts`,
...replaced('R4.3/R4.5 (pending)', NONE, PDF_LIB_LEAVES),
},
{
file: `${APP}pdf.roundtrip.test.ts`,
...replaced('R4.3/R4.5 (pending)', NONE, PDF_LIB_LEAVES),
},
{ file: `${APP}archive.envelopes.test.ts`, ...kept(NONE, N1_SWITCH) },
{ file: `${APP}n-1.reader.test.ts`, ...kept(NONE, N1_SWITCH) },
{
file: `${APP}pdf.export.app.test.ts`,
...ported('R4.5 cargo from_envelope (pending)', NONE),
},
{ file: `${APP}migration.leaves.test.ts`, ...kept(NONE) },
{ file: `${APP}pdf.browserRasterizer.test.ts`, ...kept(NONE) },
{
file: `${APP}scripts.compare.test.ts`,
...ported('R4.7 cargo (pending)', 'O10'),
},
...APP_MOCKING_SUITES.map((name) => ({
file: `${APP}${name}`,
...kept(NONE, 'mocks rewritten at R4.3 / R4.6'),
})),
{ file: N1_SCANNER, ...kept('O11') },
]
function testFiles(dir, keep) {
return readdirSync(join(ROOT, dir))
.filter(keep)
.sort()
.map((name) => `${dir}${name}`)
}
function suites() {
const appGenerators = testFiles(APP, (n) => /^r4\..+\.test\.tsx?$/.test(n))
const appNamed = [...APP_PDF_SUITES, ...APP_MOCKING_SUITES].map(
(n) => `${APP}${n}`,
)
return [
{
pkg: 'packages/pdf',
files: testFiles(PDF, (n) => n.endsWith('.test.ts')),
},
{ pkg: 'apps/app', files: [...appNamed, ...appGenerators] },
{ pkg: 'packages/core-wasm', files: [N1_SCANNER] },
]
}
/** vitest's JSON report of `files`, run from the package as test:run. */
function runVitest({ pkg, files }, reportDir, errors) {
const report = join(reportDir, `${pkg.replaceAll('/', '_')}.json`)
const args = [
join(ROOT, pkg, 'node_modules/vitest/vitest.mjs'),
'run',
...files.map((file) => relative(pkg, file)),
'--reporter=json',
`--outputFile=${report}`,
]
const run = spawnSync(process.execPath, args, {
cwd: join(ROOT, pkg),
stdio: ['ignore', 'inherit', 'inherit'],
})
if (run.status !== 0)
errors.push(`${pkg}: vitest exited with ${run.status ?? run.signal}`)
if (!existsSync(report)) {
errors.push(`${pkg}: no JSON report`)
return null
}
const parsed = JSON.parse(readFileSync(report, 'utf8'))
unlinkSync(report)
return parsed
}
function matches(rule, file, title) {
if (rule.file !== file) return false
if (rule.title === undefined) return true
return rule.title instanceof RegExp
? rule.title.test(title)
: title.startsWith(rule.title)
}
function rowOf(file, assertion, used, errors) {
const title = [...assertion.ancestorTitles, assertion.title].join(' > ')
const at = RULES.findIndex((rule) => matches(rule, file, title))
if (at < 0) {
errors.push(`no disposition: ${file} > ${title}`)
return null
}
used.add(at)
const { disposition, target, oracle, note } = RULES[at]
const status =
assertion.status === 'passed'
? ''
: ` (status at freeze: ${assertion.status})`
return {
ts_file: file,
case_title: title,
disposition,
target,
oracle,
note: note + status,
}
}
/** The rows of one package's report, checked to hold the requested files. */
function suiteRows(suite, report, context) {
const { errors, used } = context
const results = report.testResults.map((result) => ({
...result,
file: relative(ROOT, result.name),
}))
const found = results.map((r) => r.file)
for (const file of suite.files)
if (!found.includes(file)) errors.push(`missing: ${file}`)
for (const file of found)
if (!suite.files.includes(file)) errors.push(`unrequested: ${file}`)
for (const r of results)
if (r.status !== 'passed') errors.push(`${r.file}: ${r.status}`)
const rows = results.flatMap((r) =>
r.assertionResults.map((a) => rowOf(r.file, a, used, errors)),
)
if (rows.length !== report.numTotalTests) {
const total = report.numTotalTests
errors.push(`${suite.pkg}: ${rows.length} rows, ${total} vitest cases`)
}
return rows.filter((row) => row !== null)
}
function collect() {
const context = { errors: [], used: new Set(), totals: [] }
const reportDir = mkdtempSync(join(tmpdir(), 'faunosteo-r4-ledger-'))
try {
const rows = suites().flatMap((suite) => {
const report = runVitest(suite, reportDir, context.errors)
if (!report) return []
context.totals.push([suite.pkg, report.numTotalTests])
return suiteRows(suite, report, context)
})
RULES.forEach((rule, at) => {
if (!context.used.has(at))
context.errors.push(
`rule matches no case: ${rule.file} ${rule.title ?? ''}`,
)
})
// Stable sort: files in path order, cases in declaration order.
rows.sort((a, b) =>
a.ts_file === b.ts_file ? 0 : a.ts_file < b.ts_file ? -1 : 1,
)
return { rows, ...context }
} finally {
rmdirSync(reportDir)
}
}
function tsvField(value) {
if (/[\t\r\n]/.test(value))
throw new Error(`tab or newline in ledger field: ${value}`)
return value
}
function writeLedger(rows) {
const lines = rows.map((row) =>
COLUMNS.map((c) => tsvField(row[c])).join('\t'),
)
mkdirSync(dirname(OUT), { recursive: true })
writeFileSync(OUT, [COLUMNS.join('\t'), ...lines].join('\n') + '\n')
}
function countLine(label, rows) {
const counts = DISPOSITIONS.map(
(d) => rows.filter((r) => r.disposition === d).length,
)
return [
label.padEnd(58),
String(rows.length).padStart(5),
...counts.map((n) => String(n).padStart(9)),
].join('')
}
function printSummary(rows, totals) {
console.log(
[
'file'.padEnd(58),
'rows'.padStart(5),
...DISPOSITIONS.map((d) => d.padStart(9)),
].join(''),
)
for (const file of [...new Set(rows.map((r) => r.ts_file))]) {
console.log(
countLine(
file,
rows.filter((r) => r.ts_file === file),
),
)
}
console.log(countLine('total', rows))
const vitest = totals.map(([pkg, n]) => `${pkg} ${n}`).join(', ')
const sum = totals.reduce((n, [, count]) => n + count, 0)
console.log(`vitest cases: ${vitest} = ${sum}; ledger rows: ${rows.length}`)
console.log(`written: ${relative(ROOT, OUT)}`)
}
function main() {
const { rows, errors, totals } = collect()
if (errors.length) {
for (const error of errors) console.error(`r4-ledger: ${error}`)
console.error('r4-ledger: nothing written')
return 1
}
writeLedger(rows)
printSummary(rows, totals)
return 0
}
process.exitCode = main()