Files
faunosteo/scripts/r4-ledger.mjs
T

419 lines
12 KiB
JavaScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
// R4 test ledger (plan 2026-10-17 §B.6): one row per TypeScript test case
// of the PDF path at the freeze, with its planned disposition. Runs vitest
// with its JSON reporter (report files in os.tmpdir()) on every packages/pdf
// suite, the application's PDF, route and store suites and its r4.*
// generators, and the core-wasm N-1 scanner test; writes
// crates/core/tests/pdf/LEDGER.tsv (files sorted, cases in declaration
// order) and prints the rows per file and disposition. Fails, writing
// nothing, when a run fails, a requested file is missing from its report, a
// case matches no rule, a rule matches no case, or the rows do not reconcile
// with vitest's totals. "(pending)" targets are filled when the case is
// ported.
// Usage: node scripts/r4-ledger.mjs (core module built: pnpm wasm:build)
import { spawnSync } from 'node:child_process'
import {
existsSync,
mkdirSync,
mkdtempSync,
readdirSync,
readFileSync,
rmdirSync,
unlinkSync,
writeFileSync,
} from 'node:fs'
import { tmpdir } from 'node:os'
import { dirname, join, relative, resolve } from 'node:path'
import { fileURLToPath } from 'node:url'
const ROOT = resolve(fileURLToPath(new URL('..', import.meta.url)))
const OUT = join(ROOT, 'crates/core/tests/pdf/LEDGER.tsv')
const COLUMNS = [
'ts_file',
'case_title',
'disposition',
'target',
'oracle',
'note',
]
const DISPOSITIONS = ['ported', 'replaced', 'kept', 'deleted']
const PDF = 'packages/pdf/tests/'
const APP = 'apps/app/tests/unit/'
const N1_SCANNER = 'packages/core-wasm/tests/n-1.scanner.test.ts'
const APP_PDF_SUITES = [
'pdf.goldens.test.ts',
'pdf.roundtrip.test.ts',
'pdf.import.test.ts',
'archive.envelopes.test.ts',
'n-1.reader.test.ts',
'pdf.export.app.test.ts',
'migration.leaves.test.ts',
'pdf.browserRasterizer.test.ts',
'scripts.compare.test.ts',
]
/** Suites mocking `@/pdf/*`: rewritten when the import (R4.3) and the
* export (R4.6) switch. */
const APP_MOCKING_SUITES = [
'routes.HomeRoute.archive.test.tsx',
'routes.HomeRoute.import.test.tsx',
'routes.HomeRoute.rescueCopy.test.tsx',
'routes.EditorRoute.export.test.tsx',
'routes.coreStopped.test.tsx',
'state.coreFailures.test.tsx',
]
const READER = 'R4.2 cargo (pending)'
const WRITER = 'R4.4/R4.5 cargo (pending)'
const CARGO_GOLDENS = 'R4.4/R4.5 cargo goldens (pending)'
const SAME = 'vitest (same case)'
const NONE = '-'
const GENERATOR = 'generator; oracle files kept (R4.8)'
const N1_SWITCH = 'reader switched to the frozen O11 scanner at R4.8'
const PDF_LIB_LEAVES =
'pdf-lib leaves the app; read-back moves to cargo/lopdf or the wasm reader'
const ported = (target, oracle, note = NONE) => ({
disposition: 'ported',
target,
oracle,
note,
})
const kept = (oracle, note = NONE) => ({
disposition: 'kept',
target: SAME,
oracle,
note,
})
const replaced = (target, oracle, note) => ({
disposition: 'replaced',
target,
oracle,
note,
})
const generator = ([file, oracle]) => ({
file,
disposition: 'deleted',
target: NONE,
oracle,
note: GENERATOR,
})
const SECURITY = `${PDF}security.test.ts`
const GOLDENS = `${APP}pdf.goldens.test.ts`
// First match wins. `title` (a prefix or a RegExp) splits a file's cases.
const RULES = [
// packages/pdf, reader side (R4.2)
{ file: `${PDF}scan.test.ts`, ...ported(READER, 'O7') },
{ file: `${PDF}inflate.test.ts`, ...ported(READER, 'O7/O9') },
{ file: `${PDF}read.test.ts`, ...ported(READER, 'O7') },
{
file: SECURITY,
title: 'import: the report survives a rejected snapshot > ',
...ported(READER, 'O7'),
},
{
file: SECURITY,
title: '__proto__ ids and the inflate bomb > inflates a normal payload',
...ported(READER, 'O7'),
},
// packages/pdf, export side (R4.4 layout, R4.5 serialiser)
{
file: SECURITY,
title: '__proto__ ids and the inflate bomb > report ids named __proto__',
...ported(WRITER, 'O1'),
},
{
file: SECURITY,
title: 'export: a snapshot over the caps fails',
...ported(WRITER, NONE, 'typed export errors, no oracle file'),
},
{
file: SECURITY,
title: 'hostile SVG assets > ',
...ported(
WRITER,
NONE,
'export SVG path; R4 refuses where the TS sanitised (A.3, RUST.md §9.4)',
),
},
{
file: SECURITY,
title: 'branding URL injection (HIGH) > ',
...ported(WRITER, 'O2', 'link sink'),
},
{ file: SECURITY, title: 'PNG budget (MEDIUM) > ', ...ported(WRITER, 'O6') },
{ file: `${PDF}goldens.test.ts`, ...ported(WRITER, 'O1/O2') },
{
file: `${PDF}text.test.ts`,
title: 'formatDate > ',
...ported(WRITER, 'O5'),
},
{ file: `${PDF}text.test.ts`, ...ported(WRITER, 'O4') },
{ file: `${PDF}size.test.ts`, ...ported(WRITER, 'O2', 'E2 budget') },
{ file: `${PDF}raster.test.ts`, ...ported(WRITER, 'O2/O4') },
{ file: `${PDF}reexport.test.ts`, ...ported(WRITER, 'O2') },
// Oracle generators, deleted with the TS they freeze; the oracle files stay
...[
[`${PDF}r3.highlight-oracles.test.ts`, 'R3 highlight'],
[`${PDF}r3.sanitiser-oracles.test.ts`, 'R3 sanitiser'],
[`${PDF}r4.dates-oracles.test.ts`, 'O5'],
[`${PDF}r4.inflate-oracles.test.ts`, 'O9'],
[`${PDF}r4.png-oracles.test.ts`, 'O6'],
[`${PDF}r4.reader-oracles.test.ts`, 'O7'],
[`${PDF}r4.text-oracles.test.ts`, 'O4'],
[`${APP}r4.compare-oracles.test.ts`, 'O10'],
[`${APP}r4.tolerance-oracles.test.ts`, 'O8'],
[`${APP}r4.writer-oracles.test.ts`, 'O1/O2/O3'],
].map(generator),
{
file: `${APP}r4.synthetic-reports.test.ts`,
...kept('O13', 'runs on the core only'),
},
// apps/app PDF suites
{
file: GOLDENS,
title: 'golden inputs > the 2×1 PNG decodes',
...replaced(
CARGO_GOLDENS,
'O6',
'pdf-lib embedPng check of the golden rasters',
),
},
{
file: GOLDENS,
title: 'golden inputs > ',
...kept(NONE, 'inputs of the kept views cases'),
},
{
file: GOLDENS,
title: /^goldens of [^>]+ > [^>]+ > text, images and links per locale/,
...replaced(CARGO_GOLDENS, 'O1', 'Step 6 text, images and links goldens'),
},
{
file: GOLDENS,
title: /^goldens of [^>]+ > highlighted views/,
...kept(NONE, 'app path'),
},
{
file: GOLDENS,
title: 'report-v3.bare.pdf > ',
...kept(NONE, 'bare v3 import'),
},
{
file: `${APP}pdf.import.test.ts`,
...replaced('R4.3/R4.5 (pending)', NONE, PDF_LIB_LEAVES),
},
{
file: `${APP}pdf.roundtrip.test.ts`,
...replaced('R4.3/R4.5 (pending)', NONE, PDF_LIB_LEAVES),
},
{ file: `${APP}archive.envelopes.test.ts`, ...kept(NONE, N1_SWITCH) },
{ file: `${APP}n-1.reader.test.ts`, ...kept(NONE, N1_SWITCH) },
{
file: `${APP}pdf.export.app.test.ts`,
...ported('R4.5 cargo from_envelope (pending)', NONE),
},
{ file: `${APP}migration.leaves.test.ts`, ...kept(NONE) },
{ file: `${APP}pdf.browserRasterizer.test.ts`, ...kept(NONE) },
{
file: `${APP}scripts.compare.test.ts`,
...ported('R4.7 cargo (pending)', 'O10'),
},
...APP_MOCKING_SUITES.map((name) => ({
file: `${APP}${name}`,
...kept(NONE, 'mocks rewritten at R4.3 / R4.6'),
})),
{ file: N1_SCANNER, ...kept('O11') },
]
function testFiles(dir, keep) {
return readdirSync(join(ROOT, dir))
.filter(keep)
.sort()
.map((name) => `${dir}${name}`)
}
function suites() {
const appGenerators = testFiles(APP, (n) => /^r4\..+\.test\.tsx?$/.test(n))
const appNamed = [...APP_PDF_SUITES, ...APP_MOCKING_SUITES].map(
(n) => `${APP}${n}`,
)
return [
{
pkg: 'packages/pdf',
files: testFiles(PDF, (n) => n.endsWith('.test.ts')),
},
{ pkg: 'apps/app', files: [...appNamed, ...appGenerators] },
{ pkg: 'packages/core-wasm', files: [N1_SCANNER] },
]
}
/** vitest's JSON report of `files`, run from the package as test:run. */
function runVitest({ pkg, files }, reportDir, errors) {
const report = join(reportDir, `${pkg.replaceAll('/', '_')}.json`)
const args = [
join(ROOT, pkg, 'node_modules/vitest/vitest.mjs'),
'run',
...files.map((file) => relative(pkg, file)),
'--reporter=json',
`--outputFile=${report}`,
]
const run = spawnSync(process.execPath, args, {
cwd: join(ROOT, pkg),
stdio: ['ignore', 'inherit', 'inherit'],
})
if (run.status !== 0)
errors.push(`${pkg}: vitest exited with ${run.status ?? run.signal}`)
if (!existsSync(report)) {
errors.push(`${pkg}: no JSON report`)
return null
}
const parsed = JSON.parse(readFileSync(report, 'utf8'))
unlinkSync(report)
return parsed
}
function matches(rule, file, title) {
if (rule.file !== file) return false
if (rule.title === undefined) return true
return rule.title instanceof RegExp
? rule.title.test(title)
: title.startsWith(rule.title)
}
function rowOf(file, assertion, used, errors) {
const title = [...assertion.ancestorTitles, assertion.title].join(' > ')
const at = RULES.findIndex((rule) => matches(rule, file, title))
if (at < 0) {
errors.push(`no disposition: ${file} > ${title}`)
return null
}
used.add(at)
const { disposition, target, oracle, note } = RULES[at]
const status =
assertion.status === 'passed'
? ''
: ` (status at freeze: ${assertion.status})`
return {
ts_file: file,
case_title: title,
disposition,
target,
oracle,
note: note + status,
}
}
/** The rows of one package's report, checked to hold the requested files. */
function suiteRows(suite, report, context) {
const { errors, used } = context
const results = report.testResults.map((result) => ({
...result,
file: relative(ROOT, result.name),
}))
const found = results.map((r) => r.file)
for (const file of suite.files)
if (!found.includes(file)) errors.push(`missing: ${file}`)
for (const file of found)
if (!suite.files.includes(file)) errors.push(`unrequested: ${file}`)
for (const r of results)
if (r.status !== 'passed') errors.push(`${r.file}: ${r.status}`)
const rows = results.flatMap((r) =>
r.assertionResults.map((a) => rowOf(r.file, a, used, errors)),
)
if (rows.length !== report.numTotalTests) {
const total = report.numTotalTests
errors.push(`${suite.pkg}: ${rows.length} rows, ${total} vitest cases`)
}
return rows.filter((row) => row !== null)
}
function collect() {
const context = { errors: [], used: new Set(), totals: [] }
const reportDir = mkdtempSync(join(tmpdir(), 'faunosteo-r4-ledger-'))
try {
const rows = suites().flatMap((suite) => {
const report = runVitest(suite, reportDir, context.errors)
if (!report) return []
context.totals.push([suite.pkg, report.numTotalTests])
return suiteRows(suite, report, context)
})
RULES.forEach((rule, at) => {
if (!context.used.has(at))
context.errors.push(
`rule matches no case: ${rule.file} ${rule.title ?? ''}`,
)
})
// Stable sort: files in path order, cases in declaration order.
rows.sort((a, b) =>
a.ts_file === b.ts_file ? 0 : a.ts_file < b.ts_file ? -1 : 1,
)
return { rows, ...context }
} finally {
rmdirSync(reportDir)
}
}
function tsvField(value) {
if (/[\t\r\n]/.test(value))
throw new Error(`tab or newline in ledger field: ${value}`)
return value
}
function writeLedger(rows) {
const lines = rows.map((row) =>
COLUMNS.map((c) => tsvField(row[c])).join('\t'),
)
mkdirSync(dirname(OUT), { recursive: true })
writeFileSync(OUT, [COLUMNS.join('\t'), ...lines].join('\n') + '\n')
}
function countLine(label, rows) {
const counts = DISPOSITIONS.map(
(d) => rows.filter((r) => r.disposition === d).length,
)
return [
label.padEnd(58),
String(rows.length).padStart(5),
...counts.map((n) => String(n).padStart(9)),
].join('')
}
function printSummary(rows, totals) {
console.log(
[
'file'.padEnd(58),
'rows'.padStart(5),
...DISPOSITIONS.map((d) => d.padStart(9)),
].join(''),
)
for (const file of [...new Set(rows.map((r) => r.ts_file))]) {
console.log(
countLine(
file,
rows.filter((r) => r.ts_file === file),
),
)
}
console.log(countLine('total', rows))
const vitest = totals.map(([pkg, n]) => `${pkg} ${n}`).join(', ')
const sum = totals.reduce((n, [, count]) => n + count, 0)
console.log(`vitest cases: ${vitest} = ${sum}; ledger rows: ${rows.length}`)
console.log(`written: ${relative(ROOT, OUT)}`)
}
function main() {
const { rows, errors, totals } = collect()
if (errors.length) {
for (const error of errors) console.error(`r4-ledger: ${error}`)
console.error('r4-ledger: nothing written')
return 1
}
writeLedger(rows)
printSummary(rows, totals)
return 0
}
process.exitCode = main()