419 lines
12 KiB
JavaScript
419 lines
12 KiB
JavaScript
// R4 test ledger (plan 2026-10-17 §B.6): one row per TypeScript test case
|
||
// of the PDF path at the freeze, with its planned disposition. Runs vitest
|
||
// with its JSON reporter (report files in os.tmpdir()) on every packages/pdf
|
||
// suite, the application's PDF, route and store suites and its r4.*
|
||
// generators, and the core-wasm N-1 scanner test; writes
|
||
// crates/core/tests/pdf/LEDGER.tsv (files sorted, cases in declaration
|
||
// order) and prints the rows per file and disposition. Fails, writing
|
||
// nothing, when a run fails, a requested file is missing from its report, a
|
||
// case matches no rule, a rule matches no case, or the rows do not reconcile
|
||
// with vitest's totals. "(pending)" targets are filled when the case is
|
||
// ported.
|
||
// Usage: node scripts/r4-ledger.mjs (core module built: pnpm wasm:build)
|
||
import { spawnSync } from 'node:child_process'
|
||
import {
|
||
existsSync,
|
||
mkdirSync,
|
||
mkdtempSync,
|
||
readdirSync,
|
||
readFileSync,
|
||
rmdirSync,
|
||
unlinkSync,
|
||
writeFileSync,
|
||
} from 'node:fs'
|
||
import { tmpdir } from 'node:os'
|
||
import { dirname, join, relative, resolve } from 'node:path'
|
||
import { fileURLToPath } from 'node:url'
|
||
|
||
const ROOT = resolve(fileURLToPath(new URL('..', import.meta.url)))
|
||
const OUT = join(ROOT, 'crates/core/tests/pdf/LEDGER.tsv')
|
||
const COLUMNS = [
|
||
'ts_file',
|
||
'case_title',
|
||
'disposition',
|
||
'target',
|
||
'oracle',
|
||
'note',
|
||
]
|
||
const DISPOSITIONS = ['ported', 'replaced', 'kept', 'deleted']
|
||
|
||
const PDF = 'packages/pdf/tests/'
|
||
const APP = 'apps/app/tests/unit/'
|
||
const N1_SCANNER = 'packages/core-wasm/tests/n-1.scanner.test.ts'
|
||
const APP_PDF_SUITES = [
|
||
'pdf.goldens.test.ts',
|
||
'pdf.roundtrip.test.ts',
|
||
'pdf.import.test.ts',
|
||
'archive.envelopes.test.ts',
|
||
'n-1.reader.test.ts',
|
||
'pdf.export.app.test.ts',
|
||
'migration.leaves.test.ts',
|
||
'pdf.browserRasterizer.test.ts',
|
||
'scripts.compare.test.ts',
|
||
]
|
||
/** Suites mocking `@/pdf/*`: rewritten when the import (R4.3) and the
|
||
* export (R4.6) switch. */
|
||
const APP_MOCKING_SUITES = [
|
||
'routes.HomeRoute.archive.test.tsx',
|
||
'routes.HomeRoute.import.test.tsx',
|
||
'routes.HomeRoute.rescueCopy.test.tsx',
|
||
'routes.EditorRoute.export.test.tsx',
|
||
'routes.coreStopped.test.tsx',
|
||
'state.coreFailures.test.tsx',
|
||
]
|
||
|
||
const READER = 'R4.2 cargo (pending)'
|
||
const WRITER = 'R4.4/R4.5 cargo (pending)'
|
||
const CARGO_GOLDENS = 'R4.4/R4.5 cargo goldens (pending)'
|
||
const SAME = 'vitest (same case)'
|
||
const NONE = '-'
|
||
const GENERATOR = 'generator; oracle files kept (R4.8)'
|
||
const N1_SWITCH = 'reader switched to the frozen O11 scanner at R4.8'
|
||
const PDF_LIB_LEAVES =
|
||
'pdf-lib leaves the app; read-back moves to cargo/lopdf or the wasm reader'
|
||
|
||
const ported = (target, oracle, note = NONE) => ({
|
||
disposition: 'ported',
|
||
target,
|
||
oracle,
|
||
note,
|
||
})
|
||
const kept = (oracle, note = NONE) => ({
|
||
disposition: 'kept',
|
||
target: SAME,
|
||
oracle,
|
||
note,
|
||
})
|
||
const replaced = (target, oracle, note) => ({
|
||
disposition: 'replaced',
|
||
target,
|
||
oracle,
|
||
note,
|
||
})
|
||
const generator = ([file, oracle]) => ({
|
||
file,
|
||
disposition: 'deleted',
|
||
target: NONE,
|
||
oracle,
|
||
note: GENERATOR,
|
||
})
|
||
|
||
const SECURITY = `${PDF}security.test.ts`
|
||
const GOLDENS = `${APP}pdf.goldens.test.ts`
|
||
|
||
// First match wins. `title` (a prefix or a RegExp) splits a file's cases.
|
||
const RULES = [
|
||
// packages/pdf, reader side (R4.2)
|
||
{ file: `${PDF}scan.test.ts`, ...ported(READER, 'O7') },
|
||
{ file: `${PDF}inflate.test.ts`, ...ported(READER, 'O7/O9') },
|
||
{ file: `${PDF}read.test.ts`, ...ported(READER, 'O7') },
|
||
{
|
||
file: SECURITY,
|
||
title: 'import: the report survives a rejected snapshot > ',
|
||
...ported(READER, 'O7'),
|
||
},
|
||
{
|
||
file: SECURITY,
|
||
title: '__proto__ ids and the inflate bomb > inflates a normal payload',
|
||
...ported(READER, 'O7'),
|
||
},
|
||
// packages/pdf, export side (R4.4 layout, R4.5 serialiser)
|
||
{
|
||
file: SECURITY,
|
||
title: '__proto__ ids and the inflate bomb > report ids named __proto__',
|
||
...ported(WRITER, 'O1'),
|
||
},
|
||
{
|
||
file: SECURITY,
|
||
title: 'export: a snapshot over the caps fails',
|
||
...ported(WRITER, NONE, 'typed export errors, no oracle file'),
|
||
},
|
||
{
|
||
file: SECURITY,
|
||
title: 'hostile SVG assets > ',
|
||
...ported(
|
||
WRITER,
|
||
NONE,
|
||
'export SVG path; R4 refuses where the TS sanitised (A.3, RUST.md §9.4)',
|
||
),
|
||
},
|
||
{
|
||
file: SECURITY,
|
||
title: 'branding URL injection (HIGH) > ',
|
||
...ported(WRITER, 'O2', 'link sink'),
|
||
},
|
||
{ file: SECURITY, title: 'PNG budget (MEDIUM) > ', ...ported(WRITER, 'O6') },
|
||
{ file: `${PDF}goldens.test.ts`, ...ported(WRITER, 'O1/O2') },
|
||
{
|
||
file: `${PDF}text.test.ts`,
|
||
title: 'formatDate > ',
|
||
...ported(WRITER, 'O5'),
|
||
},
|
||
{ file: `${PDF}text.test.ts`, ...ported(WRITER, 'O4') },
|
||
{ file: `${PDF}size.test.ts`, ...ported(WRITER, 'O2', 'E2 budget') },
|
||
{ file: `${PDF}raster.test.ts`, ...ported(WRITER, 'O2/O4') },
|
||
{ file: `${PDF}reexport.test.ts`, ...ported(WRITER, 'O2') },
|
||
// Oracle generators, deleted with the TS they freeze; the oracle files stay
|
||
...[
|
||
[`${PDF}r3.highlight-oracles.test.ts`, 'R3 highlight'],
|
||
[`${PDF}r3.sanitiser-oracles.test.ts`, 'R3 sanitiser'],
|
||
[`${PDF}r4.dates-oracles.test.ts`, 'O5'],
|
||
[`${PDF}r4.inflate-oracles.test.ts`, 'O9'],
|
||
[`${PDF}r4.png-oracles.test.ts`, 'O6'],
|
||
[`${PDF}r4.reader-oracles.test.ts`, 'O7'],
|
||
[`${PDF}r4.text-oracles.test.ts`, 'O4'],
|
||
[`${APP}r4.compare-oracles.test.ts`, 'O10'],
|
||
[`${APP}r4.tolerance-oracles.test.ts`, 'O8'],
|
||
[`${APP}r4.writer-oracles.test.ts`, 'O1/O2/O3'],
|
||
].map(generator),
|
||
{
|
||
file: `${APP}r4.synthetic-reports.test.ts`,
|
||
...kept('O13', 'runs on the core only'),
|
||
},
|
||
// apps/app PDF suites
|
||
{
|
||
file: GOLDENS,
|
||
title: 'golden inputs > the 2×1 PNG decodes',
|
||
...replaced(
|
||
CARGO_GOLDENS,
|
||
'O6',
|
||
'pdf-lib embedPng check of the golden rasters',
|
||
),
|
||
},
|
||
{
|
||
file: GOLDENS,
|
||
title: 'golden inputs > ',
|
||
...kept(NONE, 'inputs of the kept views cases'),
|
||
},
|
||
{
|
||
file: GOLDENS,
|
||
title: /^goldens of [^>]+ > [^>]+ > text, images and links per locale/,
|
||
...replaced(CARGO_GOLDENS, 'O1', 'Step 6 text, images and links goldens'),
|
||
},
|
||
{
|
||
file: GOLDENS,
|
||
title: /^goldens of [^>]+ > highlighted views/,
|
||
...kept(NONE, 'app path'),
|
||
},
|
||
{
|
||
file: GOLDENS,
|
||
title: 'report-v3.bare.pdf > ',
|
||
...kept(NONE, 'bare v3 import'),
|
||
},
|
||
{
|
||
file: `${APP}pdf.import.test.ts`,
|
||
...replaced('R4.3/R4.5 (pending)', NONE, PDF_LIB_LEAVES),
|
||
},
|
||
{
|
||
file: `${APP}pdf.roundtrip.test.ts`,
|
||
...replaced('R4.3/R4.5 (pending)', NONE, PDF_LIB_LEAVES),
|
||
},
|
||
{ file: `${APP}archive.envelopes.test.ts`, ...kept(NONE, N1_SWITCH) },
|
||
{ file: `${APP}n-1.reader.test.ts`, ...kept(NONE, N1_SWITCH) },
|
||
{
|
||
file: `${APP}pdf.export.app.test.ts`,
|
||
...ported('R4.5 cargo from_envelope (pending)', NONE),
|
||
},
|
||
{ file: `${APP}migration.leaves.test.ts`, ...kept(NONE) },
|
||
{ file: `${APP}pdf.browserRasterizer.test.ts`, ...kept(NONE) },
|
||
{
|
||
file: `${APP}scripts.compare.test.ts`,
|
||
...ported('R4.7 cargo (pending)', 'O10'),
|
||
},
|
||
...APP_MOCKING_SUITES.map((name) => ({
|
||
file: `${APP}${name}`,
|
||
...kept(NONE, 'mocks rewritten at R4.3 / R4.6'),
|
||
})),
|
||
{ file: N1_SCANNER, ...kept('O11') },
|
||
]
|
||
|
||
function testFiles(dir, keep) {
|
||
return readdirSync(join(ROOT, dir))
|
||
.filter(keep)
|
||
.sort()
|
||
.map((name) => `${dir}${name}`)
|
||
}
|
||
|
||
function suites() {
|
||
const appGenerators = testFiles(APP, (n) => /^r4\..+\.test\.tsx?$/.test(n))
|
||
const appNamed = [...APP_PDF_SUITES, ...APP_MOCKING_SUITES].map(
|
||
(n) => `${APP}${n}`,
|
||
)
|
||
return [
|
||
{
|
||
pkg: 'packages/pdf',
|
||
files: testFiles(PDF, (n) => n.endsWith('.test.ts')),
|
||
},
|
||
{ pkg: 'apps/app', files: [...appNamed, ...appGenerators] },
|
||
{ pkg: 'packages/core-wasm', files: [N1_SCANNER] },
|
||
]
|
||
}
|
||
|
||
/** vitest's JSON report of `files`, run from the package as test:run. */
|
||
function runVitest({ pkg, files }, reportDir, errors) {
|
||
const report = join(reportDir, `${pkg.replaceAll('/', '_')}.json`)
|
||
const args = [
|
||
join(ROOT, pkg, 'node_modules/vitest/vitest.mjs'),
|
||
'run',
|
||
...files.map((file) => relative(pkg, file)),
|
||
'--reporter=json',
|
||
`--outputFile=${report}`,
|
||
]
|
||
const run = spawnSync(process.execPath, args, {
|
||
cwd: join(ROOT, pkg),
|
||
stdio: ['ignore', 'inherit', 'inherit'],
|
||
})
|
||
if (run.status !== 0)
|
||
errors.push(`${pkg}: vitest exited with ${run.status ?? run.signal}`)
|
||
if (!existsSync(report)) {
|
||
errors.push(`${pkg}: no JSON report`)
|
||
return null
|
||
}
|
||
const parsed = JSON.parse(readFileSync(report, 'utf8'))
|
||
unlinkSync(report)
|
||
return parsed
|
||
}
|
||
|
||
function matches(rule, file, title) {
|
||
if (rule.file !== file) return false
|
||
if (rule.title === undefined) return true
|
||
return rule.title instanceof RegExp
|
||
? rule.title.test(title)
|
||
: title.startsWith(rule.title)
|
||
}
|
||
|
||
function rowOf(file, assertion, used, errors) {
|
||
const title = [...assertion.ancestorTitles, assertion.title].join(' > ')
|
||
const at = RULES.findIndex((rule) => matches(rule, file, title))
|
||
if (at < 0) {
|
||
errors.push(`no disposition: ${file} > ${title}`)
|
||
return null
|
||
}
|
||
used.add(at)
|
||
const { disposition, target, oracle, note } = RULES[at]
|
||
const status =
|
||
assertion.status === 'passed'
|
||
? ''
|
||
: ` (status at freeze: ${assertion.status})`
|
||
return {
|
||
ts_file: file,
|
||
case_title: title,
|
||
disposition,
|
||
target,
|
||
oracle,
|
||
note: note + status,
|
||
}
|
||
}
|
||
|
||
/** The rows of one package's report, checked to hold the requested files. */
|
||
function suiteRows(suite, report, context) {
|
||
const { errors, used } = context
|
||
const results = report.testResults.map((result) => ({
|
||
...result,
|
||
file: relative(ROOT, result.name),
|
||
}))
|
||
const found = results.map((r) => r.file)
|
||
for (const file of suite.files)
|
||
if (!found.includes(file)) errors.push(`missing: ${file}`)
|
||
for (const file of found)
|
||
if (!suite.files.includes(file)) errors.push(`unrequested: ${file}`)
|
||
for (const r of results)
|
||
if (r.status !== 'passed') errors.push(`${r.file}: ${r.status}`)
|
||
const rows = results.flatMap((r) =>
|
||
r.assertionResults.map((a) => rowOf(r.file, a, used, errors)),
|
||
)
|
||
if (rows.length !== report.numTotalTests) {
|
||
const total = report.numTotalTests
|
||
errors.push(`${suite.pkg}: ${rows.length} rows, ${total} vitest cases`)
|
||
}
|
||
return rows.filter((row) => row !== null)
|
||
}
|
||
|
||
function collect() {
|
||
const context = { errors: [], used: new Set(), totals: [] }
|
||
const reportDir = mkdtempSync(join(tmpdir(), 'faunosteo-r4-ledger-'))
|
||
try {
|
||
const rows = suites().flatMap((suite) => {
|
||
const report = runVitest(suite, reportDir, context.errors)
|
||
if (!report) return []
|
||
context.totals.push([suite.pkg, report.numTotalTests])
|
||
return suiteRows(suite, report, context)
|
||
})
|
||
RULES.forEach((rule, at) => {
|
||
if (!context.used.has(at))
|
||
context.errors.push(
|
||
`rule matches no case: ${rule.file} ${rule.title ?? ''}`,
|
||
)
|
||
})
|
||
// Stable sort: files in path order, cases in declaration order.
|
||
rows.sort((a, b) =>
|
||
a.ts_file === b.ts_file ? 0 : a.ts_file < b.ts_file ? -1 : 1,
|
||
)
|
||
return { rows, ...context }
|
||
} finally {
|
||
rmdirSync(reportDir)
|
||
}
|
||
}
|
||
|
||
function tsvField(value) {
|
||
if (/[\t\r\n]/.test(value))
|
||
throw new Error(`tab or newline in ledger field: ${value}`)
|
||
return value
|
||
}
|
||
|
||
function writeLedger(rows) {
|
||
const lines = rows.map((row) =>
|
||
COLUMNS.map((c) => tsvField(row[c])).join('\t'),
|
||
)
|
||
mkdirSync(dirname(OUT), { recursive: true })
|
||
writeFileSync(OUT, [COLUMNS.join('\t'), ...lines].join('\n') + '\n')
|
||
}
|
||
|
||
function countLine(label, rows) {
|
||
const counts = DISPOSITIONS.map(
|
||
(d) => rows.filter((r) => r.disposition === d).length,
|
||
)
|
||
return [
|
||
label.padEnd(58),
|
||
String(rows.length).padStart(5),
|
||
...counts.map((n) => String(n).padStart(9)),
|
||
].join('')
|
||
}
|
||
|
||
function printSummary(rows, totals) {
|
||
console.log(
|
||
[
|
||
'file'.padEnd(58),
|
||
'rows'.padStart(5),
|
||
...DISPOSITIONS.map((d) => d.padStart(9)),
|
||
].join(''),
|
||
)
|
||
for (const file of [...new Set(rows.map((r) => r.ts_file))]) {
|
||
console.log(
|
||
countLine(
|
||
file,
|
||
rows.filter((r) => r.ts_file === file),
|
||
),
|
||
)
|
||
}
|
||
console.log(countLine('total', rows))
|
||
const vitest = totals.map(([pkg, n]) => `${pkg} ${n}`).join(', ')
|
||
const sum = totals.reduce((n, [, count]) => n + count, 0)
|
||
console.log(`vitest cases: ${vitest} = ${sum}; ledger rows: ${rows.length}`)
|
||
console.log(`written: ${relative(ROOT, OUT)}`)
|
||
}
|
||
|
||
function main() {
|
||
const { rows, errors, totals } = collect()
|
||
if (errors.length) {
|
||
for (const error of errors) console.error(`r4-ledger: ${error}`)
|
||
console.error('r4-ledger: nothing written')
|
||
return 1
|
||
}
|
||
writeLedger(rows)
|
||
printSummary(rows, totals)
|
||
return 0
|
||
}
|
||
|
||
process.exitCode = main()
|