Files
faunosteo/scripts/r4-ledger.mjs
T

531 lines
17 KiB
JavaScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
// R4 test ledger (plan 2026-10-17 §B.6): one row per TypeScript test case
// of the PDF path at the freeze, with its planned disposition. Runs vitest
// with its JSON reporter (report files in os.tmpdir()) on every packages/pdf
// suite, the application's PDF, route and store suites and its r4.*
// generators, and the core-wasm N-1 scanner test; writes
// crates/core/tests/pdf/LEDGER.tsv (files sorted, cases in declaration
// order) and prints the rows per file and disposition. Fails, writing
// nothing, when a run fails, a requested file is missing from its report, a
// case matches no rule, a rule matches no case, or the rows do not reconcile
// with vitest's totals. "(pending)" targets are filled when the case is
// ported.
// Usage: node scripts/r4-ledger.mjs (core module built: pnpm wasm:build)
import { spawnSync } from 'node:child_process'
import {
existsSync,
mkdirSync,
mkdtempSync,
readdirSync,
readFileSync,
rmdirSync,
unlinkSync,
writeFileSync,
} from 'node:fs'
import { tmpdir } from 'node:os'
import { dirname, join, relative, resolve } from 'node:path'
import { fileURLToPath } from 'node:url'
const ROOT = resolve(fileURLToPath(new URL('..', import.meta.url)))
const OUT = join(ROOT, 'crates/core/tests/pdf/LEDGER.tsv')
const COLUMNS = [
'ts_file',
'case_title',
'disposition',
'target',
'oracle',
'note',
]
const DISPOSITIONS = ['ported', 'replaced', 'kept', 'deleted']
const PDF = 'packages/pdf/tests/'
const APP = 'apps/app/tests/unit/'
const N1_SCANNER = 'packages/core-wasm/tests/n-1.scanner.test.ts'
const APP_PDF_SUITES = [
'pdf.goldens.test.ts',
'pdf.roundtrip.test.ts',
'pdf.import.test.ts',
'archive.envelopes.test.ts',
'n-1.reader.test.ts',
'pdf.export.app.test.ts',
'migration.leaves.test.ts',
'pdf.browserRasterizer.test.ts',
'scripts.compare.test.ts',
]
/** Suites mocking `@/pdf/*`: rewritten when the import (R4.3) and the
* export (R4.6) switch. */
const APP_MOCKING_SUITES = [
'routes.HomeRoute.archive.test.tsx',
'routes.HomeRoute.import.test.tsx',
'routes.HomeRoute.rescueCopy.test.tsx',
'routes.EditorRoute.export.test.tsx',
'routes.coreStopped.test.tsx',
'state.coreFailures.test.tsx',
]
// R4.2 targets: one cargo test per oracle checks every frozen row (several TS cases fold
// into one Rust test; the census below checks that each named test exists).
const SCAN = 'cargo pdf_scan::every_pdf_input_scans_as_the_typescript_did'
const DECODE = 'cargo pdf_scan::decode_pdf_string_rows_match'
const LOCATE = 'cargo pdf_import::o7_locate_and_read_match_the_typescript'
const INFLATE =
'cargo pdf_inflate::every_o9_row_matches_node, pdf_inflate::every_o7_flate_input_matches_node'
const WRITER = 'R4.4/R4.5 cargo (pending)'
// R4.5 targets: the serialiser suite against the O2 PDFs and the PNG suite against O6.
const O2 =
'cargo pdf_serialize::o2_streams_images_links_info_and_attachment_match_the_r3_build'
const EXTERNAL =
'cargo pdf_serialize::external_tools_accept_the_files_and_render_them_like_the_r3_build'
const ROUND_TRIP = 'cargo pdf_serialize::restoration_round_trip_loses_nothing'
const SECURITY_EXPORT = 'cargo pdf_serialize::export_side_security_cases_hold'
const PNG_GATE =
'cargo pdf_png::every_o6_gate_verdict_matches_png_issue, pdf_png::every_o6_decode_matches_pdf_lib'
// R4.4 targets: the layout and text suites (one cargo test checks every frozen row).
const LAYOUT =
'cargo pdf_layout::o1_draw_logs_match_the_typescript_at_full_precision'
const CHECKS = 'cargo pdf_layout::the_checks_refuse_what_export_ts_refused'
const SAFE_TEXT =
'cargo pdf_text::safe_text_reproduces_every_oracle_range, pdf_text::safe_text_works_on_whole_strings'
const WRAP = 'cargo pdf_text::wrap_lines_reproduces_every_case'
const DATES =
'cargo pdf_text::format_date_reproduces_the_intl_grid_and_the_others'
/** R4.6: the Step 6 goldens hold on the app path, read back from the file's content
* streams (tests/helpers/pdfRead.ts), and in cargo from the draw log and the O2 runs. */
const CARGO_GOLDENS =
'vitest (same case, read from the file); cargo pdf_layout::step6_text_and_image_goldens_derive_from_the_log, pdf_serialize::o2_streams_images_links_info_and_attachment_match_the_r3_build'
const RASTER_CHECK =
'vitest (same case); cargo pdf_png::every_o6_decode_matches_pdf_lib'
const SAME = 'vitest (same case)'
const NONE = '-'
const GENERATOR = 'generator; oracle files kept (R4.8)'
const N1_SWITCH = 'reader switched to the frozen O11 scanner at R4.8'
const PDF_LIB_LEAVES =
'pdf-lib leaves the app; read-back moves to cargo/lopdf or the wasm reader'
/** R4.6: the export side of these cases runs on the core; the reader is the kept TS one
* until R4.8, the kinds and round trips are cargo's. */
const IMPORT_R46 =
'vitest (same case, export on the core); cargo pdf_import (kinds), pdf_serialize::restoration_round_trip_loses_nothing'
/** R4.6: the roundtrip suite reads the Rust export's content streams and metadata through
* pdf-lib (kept in the tests until R4.8); the read-back is cargo's from R4.8. */
const ROUNDTRIP_R46 =
'vitest (same case, read from the file); cargo pdf_serialize::restoration_round_trip_loses_nothing from R4.8'
const ported = (target, oracle, note = NONE) => ({
disposition: 'ported',
target,
oracle,
note,
})
const kept = (oracle, note = NONE) => ({
disposition: 'kept',
target: SAME,
oracle,
note,
})
const replaced = (target, oracle, note) => ({
disposition: 'replaced',
target,
oracle,
note,
})
const generator = ([file, oracle]) => ({
file,
disposition: 'deleted',
target: NONE,
oracle,
note: GENERATOR,
})
const SECURITY = `${PDF}security.test.ts`
const GOLDENS = `${APP}pdf.goldens.test.ts`
// First match wins. `title` (a prefix or a RegExp) splits a file's cases.
const RULES = [
// packages/pdf, reader side (R4.2)
{
file: `${PDF}scan.test.ts`,
title: /decodePdfString/,
...ported(DECODE, 'O7'),
},
{
file: `${PDF}scan.test.ts`,
title: /locateAttachment|readAttachmentBytes|readAttachedDocument/,
...ported(LOCATE, 'O7'),
},
{ file: `${PDF}scan.test.ts`, ...ported(SCAN, 'O7') },
{ file: `${PDF}inflate.test.ts`, ...ported(INFLATE, 'O7/O9') },
{ file: `${PDF}read.test.ts`, ...ported(LOCATE, 'O7') },
{
file: SECURITY,
title: 'import: the report survives a rejected snapshot > ',
...ported(LOCATE, 'O7'),
},
{
file: SECURITY,
title: '__proto__ ids and the inflate bomb > inflates a normal payload',
...ported(INFLATE, 'O7'),
},
// packages/pdf, export side (R4.4 layout, R4.5 serialiser)
{
file: SECURITY,
title: '__proto__ ids and the inflate bomb > report ids named __proto__',
...ported(LAYOUT, 'O1'),
},
{
file: SECURITY,
title: 'export: a snapshot over the caps fails',
...ported(CHECKS, NONE, 'typed export errors, no oracle file'),
},
{
file: SECURITY,
title: 'hostile SVG assets > ',
...ported(
SECURITY_EXPORT,
NONE,
'export SVG path; R4 refuses where the TS sanitised (A.3, RUST.md §9.4)',
),
},
{
file: SECURITY,
title: 'branding URL injection (HIGH) > ',
...ported(SECURITY_EXPORT, 'O2', 'link sink'),
},
{
file: SECURITY,
title: 'PNG budget (MEDIUM) > ',
...ported(PNG_GATE, 'O6'),
},
{ file: `${PDF}goldens.test.ts`, ...ported(O2, 'O1/O2') },
{
file: `${PDF}text.test.ts`,
title: 'formatDate > ',
...ported(DATES, 'O6'),
},
{
file: `${PDF}text.test.ts`,
title: 'safeText > wrapLines',
...ported(WRAP, 'O4'),
},
{
file: `${PDF}text.test.ts`,
title: 'export with hostile text > ',
...ported(LAYOUT, 'O4', 'the layout draws the safe text'),
},
{ file: `${PDF}text.test.ts`, ...ported(SAFE_TEXT, 'O4') },
{ file: `${PDF}size.test.ts`, ...ported(EXTERNAL, 'O2', 'E2 budget') },
{ file: `${PDF}raster.test.ts`, ...ported(O2, 'O2/O4') },
{ file: `${PDF}reexport.test.ts`, ...ported(ROUND_TRIP, 'O2') },
// Oracle generators, deleted with the TS they freeze; the oracle files stay
...[
[`${PDF}r3.highlight-oracles.test.ts`, 'R3 highlight'],
[`${PDF}r3.sanitiser-oracles.test.ts`, 'R3 sanitiser'],
[`${PDF}r4.dates-oracles.test.ts`, 'O5'],
[`${PDF}r4.inflate-oracles.test.ts`, 'O9'],
[`${PDF}r4.png-oracles.test.ts`, 'O6'],
[`${PDF}r4.reader-oracles.test.ts`, 'O7'],
[`${PDF}r4.text-oracles.test.ts`, 'O4'],
[`${APP}r4.compare-oracles.test.ts`, 'O10'],
[`${APP}r4.tolerance-oracles.test.ts`, 'O8'],
[`${APP}r4.writer-oracles.test.ts`, 'O1/O2/O3'],
].map(generator),
{
file: `${APP}r4.synthetic-reports.test.ts`,
...kept('O13', 'runs on the core only'),
},
// apps/app PDF suites
{
file: GOLDENS,
title: 'golden inputs > the 2×1 PNG decodes',
...replaced(RASTER_CHECK, 'O6', 'pdf-lib embedPng check of the golden rasters'),
},
{
file: GOLDENS,
title: 'golden inputs > ',
...kept(NONE, 'inputs of the kept views cases'),
},
{
file: GOLDENS,
title: /^goldens of [^>]+ > [^>]+ > text, images and links per locale/,
...replaced(CARGO_GOLDENS, 'O1', 'Step 6 text, images and links goldens'),
},
{
file: GOLDENS,
title: /^goldens of [^>]+ > highlighted views/,
...kept(NONE, 'app path'),
},
{
file: GOLDENS,
title: 'report-v3.bare.pdf > ',
...kept(NONE, 'bare v3 import'),
},
{
file: `${APP}pdf.import.test.ts`,
...replaced(IMPORT_R46, NONE, PDF_LIB_LEAVES),
},
{
file: `${APP}pdf.roundtrip.test.ts`,
...replaced(ROUNDTRIP_R46, NONE, PDF_LIB_LEAVES),
},
{ file: `${APP}archive.envelopes.test.ts`, ...kept(NONE, N1_SWITCH) },
{ file: `${APP}n-1.reader.test.ts`, ...kept(NONE, N1_SWITCH) },
{
file: `${APP}pdf.export.app.test.ts`,
...ported(ROUND_TRIP, NONE, 'export_from_envelope'),
},
{ file: `${APP}migration.leaves.test.ts`, ...kept(NONE) },
{ file: `${APP}pdf.browserRasterizer.test.ts`, ...kept(NONE) },
{
file: `${APP}scripts.compare.test.ts`,
...ported('R4.7 cargo (pending)', 'O10'),
},
...APP_MOCKING_SUITES.map((name) => ({
file: `${APP}${name}`,
...kept(NONE, 'mocks rewritten at R4.3 / R4.6'),
})),
{ file: N1_SCANNER, ...kept('O11') },
]
function testFiles(dir, keep) {
return readdirSync(join(ROOT, dir))
.filter(keep)
.sort()
.map((name) => `${dir}${name}`)
}
function suites() {
const appGenerators = testFiles(APP, (n) => /^r4\..+\.test\.tsx?$/.test(n))
const appNamed = [...APP_PDF_SUITES, ...APP_MOCKING_SUITES].map(
(n) => `${APP}${n}`,
)
return [
{
pkg: 'packages/pdf',
files: testFiles(PDF, (n) => n.endsWith('.test.ts')),
},
{ pkg: 'apps/app', files: [...appNamed, ...appGenerators] },
{ pkg: 'packages/core-wasm', files: [N1_SCANNER] },
]
}
/** vitest's JSON report of `files`, run from the package as test:run. */
function runVitest({ pkg, files }, reportDir, errors) {
const report = join(reportDir, `${pkg.replaceAll('/', '_')}.json`)
const args = [
join(ROOT, pkg, 'node_modules/vitest/vitest.mjs'),
'run',
...files.map((file) => relative(pkg, file)),
'--reporter=json',
`--outputFile=${report}`,
]
const run = spawnSync(process.execPath, args, {
cwd: join(ROOT, pkg),
stdio: ['ignore', 'inherit', 'inherit'],
})
if (run.status !== 0)
errors.push(`${pkg}: vitest exited with ${run.status ?? run.signal}`)
if (!existsSync(report)) {
errors.push(`${pkg}: no JSON report`)
return null
}
const parsed = JSON.parse(readFileSync(report, 'utf8'))
unlinkSync(report)
return parsed
}
function matches(rule, file, title) {
if (rule.file !== file) return false
if (rule.title === undefined) return true
return rule.title instanceof RegExp
? rule.title.test(title)
: title.startsWith(rule.title)
}
function rowOf(file, assertion, used, errors) {
const title = [...assertion.ancestorTitles, assertion.title].join(' > ')
const at = RULES.findIndex((rule) => matches(rule, file, title))
if (at < 0) {
errors.push(`no disposition: ${file} > ${title}`)
return null
}
used.add(at)
const { disposition, target, oracle, note } = RULES[at]
const status =
assertion.status === 'passed'
? ''
: ` (status at freeze: ${assertion.status})`
return {
ts_file: file,
case_title: title,
disposition,
target,
oracle,
note: note + status,
}
}
/** The rows of one package's report, checked to hold the requested files. */
function suiteRows(suite, report, context) {
const { errors, used } = context
const results = report.testResults.map((result) => ({
...result,
file: relative(ROOT, result.name),
}))
const found = results.map((r) => r.file)
for (const file of suite.files)
if (!found.includes(file)) errors.push(`missing: ${file}`)
for (const file of found)
if (!suite.files.includes(file)) errors.push(`unrequested: ${file}`)
for (const r of results)
if (r.status !== 'passed') errors.push(`${r.file}: ${r.status}`)
const rows = results.flatMap((r) =>
r.assertionResults.map((a) => rowOf(r.file, a, used, errors)),
)
if (rows.length !== report.numTotalTests) {
const total = report.numTotalTests
errors.push(`${suite.pkg}: ${rows.length} rows, ${total} vitest cases`)
}
return rows.filter((row) => row !== null)
}
function collect() {
const context = { errors: [], used: new Set(), totals: [] }
const reportDir = mkdtempSync(join(tmpdir(), 'faunosteo-r4-ledger-'))
try {
const rows = suites().flatMap((suite) => {
const report = runVitest(suite, reportDir, context.errors)
if (!report) return []
context.totals.push([suite.pkg, report.numTotalTests])
return suiteRows(suite, report, context)
})
RULES.forEach((rule, at) => {
if (!context.used.has(at))
context.errors.push(
`rule matches no case: ${rule.file} ${rule.title ?? ''}`,
)
})
// Stable sort: files in path order, cases in declaration order.
rows.sort((a, b) =>
a.ts_file === b.ts_file ? 0 : a.ts_file < b.ts_file ? -1 : 1,
)
return { rows, ...context }
} finally {
rmdirSync(reportDir)
}
}
function tsvField(value) {
if (/[\t\r\n]/.test(value))
throw new Error(`tab or newline in ledger field: ${value}`)
return value
}
function writeLedger(rows) {
const lines = rows.map((row) =>
COLUMNS.map((c) => tsvField(row[c])).join('\t'),
)
mkdirSync(dirname(OUT), { recursive: true })
writeFileSync(OUT, [COLUMNS.join('\t'), ...lines].join('\n') + '\n')
}
function countLine(label, rows) {
const counts = DISPOSITIONS.map(
(d) => rows.filter((r) => r.disposition === d).length,
)
return [
label.padEnd(58),
String(rows.length).padStart(5),
...counts.map((n) => String(n).padStart(9)),
].join('')
}
function printSummary(rows, totals) {
console.log(
[
'file'.padEnd(58),
'rows'.padStart(5),
...DISPOSITIONS.map((d) => d.padStart(9)),
].join(''),
)
for (const file of [...new Set(rows.map((r) => r.ts_file))]) {
console.log(
countLine(
file,
rows.filter((r) => r.ts_file === file),
),
)
}
console.log(countLine('total', rows))
const vitest = totals.map(([pkg, n]) => `${pkg} ${n}`).join(', ')
const sum = totals.reduce((n, [, count]) => n + count, 0)
console.log(`vitest cases: ${vitest} = ${sum}; ledger rows: ${rows.length}`)
console.log(`written: ${relative(ROOT, OUT)}`)
}
/** The tests of one cargo test file of the core (`cargo test --test <file> -- --list`). */
function cargoTests(file) {
const run = spawnSync(
'cargo',
[
'test',
'-p',
'faunosteo-core',
'--features',
'pdf-read,pdf-write',
'--locked',
'--test',
file,
'--',
'--list',
],
{
cwd: ROOT,
encoding: 'utf8',
env: { ...process.env, CARGO_TERM_COLOR: 'never' },
},
)
if (run.status !== 0) {
throw new Error(`cargo test --test ${file} -- --list failed: ${run.stderr}`)
}
const names = new Set()
for (const line of run.stdout.split('\n')) {
const test = /^(\S+): test$/.exec(line)
if (test) names.add(test[1])
}
return names
}
/** Census (plan §B.6): every `cargo` target of a ported row names an existing test. */
function missingCargoTargets(rows) {
const wanted = new Set()
for (const row of rows) {
if (row.disposition !== 'ported' || !row.target.startsWith('cargo '))
continue
for (const name of row.target.slice('cargo '.length).split(', '))
wanted.add(name)
}
const listed = new Map()
const missing = []
for (const name of wanted) {
const [file, test] = name.split('::')
if (!listed.has(file)) listed.set(file, cargoTests(file))
if (!listed.get(file).has(test)) missing.push(name)
}
return missing
}
function main() {
const { rows, errors, totals } = collect()
for (const name of missingCargoTargets(rows)) {
errors.push(`ported target missing from cargo test --list: ${name}`)
}
if (errors.length) {
for (const error of errors) console.error(`r4-ledger: ${error}`)
console.error('r4-ledger: nothing written')
return 1
}
writeLedger(rows)
printSummary(rows, totals)
return 0
}
process.exitCode = main()