Files
faunosteo/scripts/r4-ledger.mjs
T

584 lines
20 KiB
JavaScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
// R4 test ledger (plan 2026-10-17 §B.6): one row per TypeScript test case
// of the PDF path at the freeze (R4.1), with its disposition. The frozen
// case list is the committed ledger itself (ts_file, case_title: 301 rows;
// the engine suites and the deleted app suites no longer run, R4.8); the
// dispositions, targets and notes are re-derived from the rules below and
// checked: every `cargo` target names a test `cargo test -- --list` knows,
// every `vitest (same case…)` target names a case the live vitest report of
// that file holds (the surviving suites run with the JSON reporter, report
// files in os.tmpdir()). Writes crates/core/tests/pdf/LEDGER.tsv (files
// sorted, cases in frozen order) and prints the rows per file and
// disposition. Fails, writing nothing, when a run fails, a frozen case
// matches no rule, a rule matches no case, a target is missing, or the row
// count is not the freeze's.
// Usage: node scripts/r4-ledger.mjs (core module built: pnpm wasm:build)
import { spawnSync } from 'node:child_process'
import {
existsSync,
mkdirSync,
mkdtempSync,
readFileSync,
rmdirSync,
unlinkSync,
writeFileSync,
} from 'node:fs'
import { tmpdir } from 'node:os'
import { dirname, join, relative, resolve } from 'node:path'
import { fileURLToPath } from 'node:url'
const ROOT = resolve(fileURLToPath(new URL('..', import.meta.url)))
const OUT = join(ROOT, 'crates/core/tests/pdf/LEDGER.tsv')
const COLUMNS = [
'ts_file',
'case_title',
'disposition',
'target',
'oracle',
'note',
]
const DISPOSITIONS = ['ported', 'replaced', 'kept', 'deleted']
/** The rows of the freeze (R4.1: 140 engine cases + 155 app cases + 6 N-1 scanner cases). */
const FROZEN_ROWS = 301
const PDF = 'packages/pdf/tests/'
const APP = 'apps/app/tests/unit/'
const N1_SCANNER = 'packages/core-wasm/tests/n-1.scanner.test.ts'
/** Suites mocking `@/pdf/*`: rewritten when the import (R4.3) and the
* export (R4.6) switch. */
const APP_MOCKING_SUITES = [
'routes.HomeRoute.archive.test.tsx',
'routes.HomeRoute.import.test.tsx',
'routes.HomeRoute.rescueCopy.test.tsx',
'routes.EditorRoute.export.test.tsx',
'routes.coreStopped.test.tsx',
'state.coreFailures.test.tsx',
]
// R4.2 targets: one cargo test per oracle checks every frozen row (several TS cases fold
// into one Rust test; the census below checks that each named test exists).
const SCAN = 'cargo pdf_scan::every_pdf_input_scans_as_the_typescript_did'
const DECODE = 'cargo pdf_scan::decode_pdf_string_rows_match'
const LOCATE = 'cargo pdf_import::o7_locate_and_read_match_the_typescript'
const INFLATE =
'cargo pdf_inflate::every_o9_row_matches_node, pdf_inflate::every_o7_flate_input_matches_node'
const WRITER = 'R4.4/R4.5 cargo (pending)'
// R4.5 targets: the serialiser suite against the O2 PDFs and the PNG suite against O6.
const O2 =
'cargo pdf_serialize::o2_streams_images_links_info_and_attachment_match_the_r3_build'
const EXTERNAL =
'cargo pdf_serialize::external_tools_accept_the_files_and_render_them_like_the_r3_build'
const ROUND_TRIP = 'cargo pdf_serialize::restoration_round_trip_loses_nothing'
const SECURITY_EXPORT = 'cargo pdf_serialize::export_side_security_cases_hold'
const PNG_GATE =
'cargo pdf_png::every_o6_gate_verdict_matches_png_issue, pdf_png::every_o6_decode_matches_pdf_lib'
// R4.4 targets: the layout and text suites (one cargo test checks every frozen row).
const LAYOUT =
'cargo pdf_layout::o1_draw_logs_match_the_typescript_at_full_precision'
const CHECKS = 'cargo pdf_layout::the_checks_refuse_what_export_ts_refused'
const SAFE_TEXT =
'cargo pdf_text::safe_text_reproduces_every_oracle_range, pdf_text::safe_text_works_on_whole_strings'
const WRAP = 'cargo pdf_text::wrap_lines_reproduces_every_case'
const DATES =
'cargo pdf_text::format_date_reproduces_the_intl_grid_and_the_others'
/** R4.6: the Step 6 goldens hold on the app path, read back from the file's content
* streams (tests/helpers/pdfRead.ts), and in cargo from the draw log and the O2 runs. */
const CARGO_GOLDENS =
'vitest (same case, read from the file); cargo pdf_layout::step6_text_and_image_goldens_derive_from_the_log, pdf_serialize::o2_streams_images_links_info_and_attachment_match_the_r3_build'
const RASTER_CHECK =
'vitest (same case); cargo pdf_png::every_o6_decode_matches_pdf_lib'
const SAME = 'vitest (same case)'
const NONE = '-'
const GENERATOR = 'generator deleted at R4.8; oracle files kept'
const ENGINE_GONE = 'engine suite deleted at R4.8 with packages/pdf'
const N1_SWITCH =
'reader = the frozen O11 scanner (@faunosteo/core-wasm/n-1) since R4.8'
const PDF_LIB_LEFT =
'read back by tests/helpers/pdfFile.ts since R4.8 (no pdf-lib); import by the module reader (tests/helpers/pdfImport.ts)'
/** R4.8: the TS import path left the application with its suite; the kinds are cargo's
* and the worker suites'. */
const IMPORT_R48 =
'cargo pdf_import::o8_import_matches_the_r3_path_outside_the_declared_differences, pdf_import::the_facade_envelope_carries_the_closed_kinds; vitest core.importWorker.test.ts (the kinds through the worker)'
const IMPORT_SUITE_GONE =
'suite deleted at R4.8: importReportFromPdf left apps/app/src (the app imports through the worker)'
const REEXPORT_GONE =
'suite deleted at R4.8: the re-export from the envelope alone is cargo export_from_envelope (no sheet, no branding in its signature), byte-equal on 13 runs'
const COMPARE_GONE =
'TS CLI and suite deleted at R4.8 (pnpm report:compare runs the binary)'
/** The roundtrip suite reads the Rust export's content streams and metadata from the file. */
const ROUNDTRIP =
'vitest (same case, read from the file); cargo pdf_serialize::restoration_round_trip_loses_nothing'
/** R4.7: the `scripts.compare.test.ts` case → the cargo test replaying its O10 rows. */
const COMPARE_CASES = {
'compareReport on the frozen v2 archive PDF > shows every legacy zone and finding next to its canonical target':
'frozen_v2_archive_pdf_shows_every_legacy_item_next_to_its_target',
'compareReport on unresolved items > names a removed finding, a finding unknown to the catalog and an unknown zone':
'unresolved_items_name_a_removed_an_unknown_finding_and_an_unknown_zone',
'compareReport on unresolved items > reads a v3 report as is and flags what the sheet cannot place':
'v3_report_read_as_is_flags_what_the_sheet_cannot_place',
'compareReport on unresolved items > prints the archive line of an envelope: snapshot kept, or rejected with its reason':
'envelope_archive_line_says_snapshot_kept_or_rejected_with_its_reason',
'compareReport on unresolved items > warns when a v3 report was written on another sheet document':
'v3_report_written_on_another_sheet_document_warns',
'diff and arguments > diffs the v3 content of a migrated v2 report and its v3 re-reading':
'diff_of_a_migrated_v2_report_and_its_v3_re_reading',
'diff and arguments > compares placeholders by default and real texts with --full':
'diff_compares_placeholders_by_default_and_real_texts_with_full',
'diff and arguments > parses files and options, refuses unknown ones':
'arguments_parsed_and_unknown_options_refused',
'terminal safety and privacy > escapes control characters in every untrusted string':
'control_characters_escaped_in_every_untrusted_string',
'terminal safety and privacy > does not throw on prototype-chain names used as legacy ids':
'prototype_chain_names_as_legacy_ids_do_not_throw',
'terminal safety and privacy > hides names and free texts by default, shows them with --full':
'names_and_free_texts_hidden_by_default_shown_with_full',
}
const ported = (target, oracle, note = NONE) => ({
disposition: 'ported',
target,
oracle,
note,
})
const kept = (oracle, note = NONE) => ({
disposition: 'kept',
target: SAME,
oracle,
note,
})
const replaced = (target, oracle, note) => ({
disposition: 'replaced',
target,
oracle,
note,
})
const generator = ([file, oracle]) => ({
file,
disposition: 'deleted',
target: NONE,
oracle,
note: GENERATOR,
})
const SECURITY = `${PDF}security.test.ts`
const GOLDENS = `${APP}pdf.goldens.test.ts`
// First match wins. `title` (a prefix or a RegExp) splits a file's cases.
const RULES = [
// packages/pdf, reader side (R4.2)
{
file: `${PDF}scan.test.ts`,
title: /decodePdfString/,
...ported(DECODE, 'O7'),
},
{
file: `${PDF}scan.test.ts`,
title: /locateAttachment|readAttachmentBytes|readAttachedDocument/,
...ported(LOCATE, 'O7'),
},
{ file: `${PDF}scan.test.ts`, ...ported(SCAN, 'O7') },
{ file: `${PDF}inflate.test.ts`, ...ported(INFLATE, 'O7/O9') },
{ file: `${PDF}read.test.ts`, ...ported(LOCATE, 'O7') },
{
file: SECURITY,
title: 'import: the report survives a rejected snapshot > ',
...ported(LOCATE, 'O7'),
},
{
file: SECURITY,
title: '__proto__ ids and the inflate bomb > inflates a normal payload',
...ported(INFLATE, 'O7'),
},
// packages/pdf, export side (R4.4 layout, R4.5 serialiser)
{
file: SECURITY,
title: '__proto__ ids and the inflate bomb > report ids named __proto__',
...ported(LAYOUT, 'O1'),
},
{
file: SECURITY,
title: 'export: a snapshot over the caps fails',
...ported(CHECKS, NONE, 'typed export errors, no oracle file'),
},
{
file: SECURITY,
title: 'hostile SVG assets > ',
...ported(
SECURITY_EXPORT,
NONE,
'export SVG path; R4 refuses where the TS sanitised (A.3, RUST.md §9.4)',
),
},
{
file: SECURITY,
title: 'branding URL injection (HIGH) > ',
...ported(SECURITY_EXPORT, 'O2', 'link sink'),
},
{
file: SECURITY,
title: 'PNG budget (MEDIUM) > ',
...ported(PNG_GATE, 'O6'),
},
{ file: `${PDF}goldens.test.ts`, ...ported(O2, 'O1/O2') },
{
file: `${PDF}text.test.ts`,
title: 'formatDate > ',
...ported(DATES, 'O6'),
},
{
file: `${PDF}text.test.ts`,
title: 'safeText > wrapLines',
...ported(WRAP, 'O4'),
},
{
file: `${PDF}text.test.ts`,
title: 'export with hostile text > ',
...ported(LAYOUT, 'O4', 'the layout draws the safe text'),
},
{ file: `${PDF}text.test.ts`, ...ported(SAFE_TEXT, 'O4') },
{ file: `${PDF}size.test.ts`, ...ported(EXTERNAL, 'O2', 'E2 budget') },
{ file: `${PDF}raster.test.ts`, ...ported(O2, 'O2/O4') },
{ file: `${PDF}reexport.test.ts`, ...ported(ROUND_TRIP, 'O2') },
// Oracle generators, deleted with the TS they freeze; the oracle files stay
...[
[`${PDF}r3.highlight-oracles.test.ts`, 'R3 highlight'],
[`${PDF}r3.sanitiser-oracles.test.ts`, 'R3 sanitiser'],
[`${PDF}r4.dates-oracles.test.ts`, 'O5'],
[`${PDF}r4.inflate-oracles.test.ts`, 'O9'],
[`${PDF}r4.png-oracles.test.ts`, 'O6'],
[`${PDF}r4.reader-oracles.test.ts`, 'O7'],
[`${PDF}r4.text-oracles.test.ts`, 'O4'],
[`${APP}r4.compare-oracles.test.ts`, 'O10'],
[`${APP}r4.tolerance-oracles.test.ts`, 'O8'],
[`${APP}r4.writer-oracles.test.ts`, 'O1/O2/O3'],
].map(generator),
{
file: `${APP}r4.synthetic-reports.test.ts`,
...kept('O13', 'runs on the core only'),
},
// apps/app PDF suites
{
file: GOLDENS,
title: 'golden inputs > the 2×1 PNG decodes',
...replaced(
RASTER_CHECK,
'O6',
'the golden rasters as the core writes them (XObject sizes read from the file since R4.8)',
),
},
{
file: GOLDENS,
title: 'golden inputs > ',
...kept(NONE, 'inputs of the kept views cases'),
},
{
file: GOLDENS,
title: /^goldens of [^>]+ > [^>]+ > text, images and links per locale/,
...replaced(
CARGO_GOLDENS,
'O1',
`Step 6 text, images and links goldens; ${PDF_LIB_LEFT}`,
),
},
{
file: GOLDENS,
title: /^goldens of [^>]+ > highlighted views/,
...kept(NONE, 'app path'),
},
{
file: GOLDENS,
title: 'report-v3.bare.pdf > ',
...kept(NONE, 'bare v3 import'),
},
{
file: `${APP}pdf.import.test.ts`,
...replaced(IMPORT_R48, NONE, IMPORT_SUITE_GONE),
},
{
file: `${APP}pdf.roundtrip.test.ts`,
...replaced(ROUNDTRIP, NONE, PDF_LIB_LEFT),
},
{ file: `${APP}archive.envelopes.test.ts`, ...kept(NONE, N1_SWITCH) },
{ file: `${APP}n-1.reader.test.ts`, ...kept(NONE, N1_SWITCH) },
{
file: `${APP}pdf.export.app.test.ts`,
...ported(ROUND_TRIP, NONE, REEXPORT_GONE),
},
{ file: `${APP}migration.leaves.test.ts`, ...kept(NONE) },
{ file: `${APP}pdf.browserRasterizer.test.ts`, ...kept(NONE) },
// R4.7: the archive check tool on the core, one cargo test per case over the O10 rows.
...Object.entries(COMPARE_CASES).map(([title, test]) => ({
file: `${APP}scripts.compare.test.ts`,
title,
...ported(`cargo pdf_compare::${test}`, 'O10', COMPARE_GONE),
})),
...APP_MOCKING_SUITES.map((name) => ({
file: `${APP}${name}`,
...kept(NONE, 'mocks rewritten at R4.3 / R4.6'),
})),
{ file: N1_SCANNER, ...kept('O11') },
]
/** The frozen cases: the committed ledger's (ts_file, case_title), in file order. */
function frozenCases() {
const [header, ...lines] = readFileSync(OUT, 'utf8').trimEnd().split('\n')
const columns = header.split('\t')
const at = (name) => columns.indexOf(name)
const cases = lines.map((line) => {
const fields = line.split('\t')
return { file: fields[at('ts_file')], title: fields[at('case_title')] }
})
if (cases.length !== FROZEN_ROWS) {
throw new Error(
`${relative(ROOT, OUT)}: ${cases.length} rows, ${FROZEN_ROWS} frozen`,
)
}
return cases
}
/** The frozen files that still exist, grouped by the package that runs them. */
function liveSuites(cases) {
const files = [...new Set(cases.map((c) => c.file))].filter((file) =>
existsSync(join(ROOT, file)),
)
const pkgOf = (file) => file.split('/tests/')[0]
const pkgs = [...new Set(files.map(pkgOf))]
return pkgs.map((pkg) => ({
pkg,
files: files.filter((f) => pkgOf(f) === pkg),
}))
}
/** vitest's JSON report of `files`, run from the package as test:run. */
function runVitest({ pkg, files }, reportDir, errors) {
const report = join(reportDir, `${pkg.replaceAll('/', '_')}.json`)
const args = [
join(ROOT, pkg, 'node_modules/vitest/vitest.mjs'),
'run',
...files.map((file) => relative(pkg, file)),
'--reporter=json',
`--outputFile=${report}`,
]
const run = spawnSync(process.execPath, args, {
cwd: join(ROOT, pkg),
stdio: ['ignore', 'inherit', 'inherit'],
})
if (run.status !== 0)
errors.push(`${pkg}: vitest exited with ${run.status ?? run.signal}`)
if (!existsSync(report)) {
errors.push(`${pkg}: no JSON report`)
return null
}
const parsed = JSON.parse(readFileSync(report, 'utf8'))
unlinkSync(report)
return parsed
}
/** file → the titles of its live cases (every one must pass). */
function liveCases(suites, reportDir, errors, totals) {
const live = new Map()
for (const suite of suites) {
const report = runVitest(suite, reportDir, errors)
if (!report) continue
totals.push([suite.pkg, report.numTotalTests])
for (const result of report.testResults) {
const file = relative(ROOT, result.name)
if (result.status !== 'passed') errors.push(`${file}: ${result.status}`)
const titles = result.assertionResults.map((a) =>
[...a.ancestorTitles, a.title].join(' > '),
)
live.set(file, new Set(titles))
}
for (const file of suite.files)
if (!live.has(file)) errors.push(`missing from the report: ${file}`)
}
return live
}
function matches(rule, file, title) {
if (rule.file !== file) return false
if (rule.title === undefined) return true
return rule.title instanceof RegExp
? rule.title.test(title)
: title.startsWith(rule.title)
}
function rowOf({ file, title }, used, errors) {
const at = RULES.findIndex((rule) => matches(rule, file, title))
if (at < 0) {
errors.push(`no disposition: ${file} > ${title}`)
return null
}
used.add(at)
const { disposition, target, oracle, note } = RULES[at]
const gone = file.startsWith(PDF) && note === NONE ? ENGINE_GONE : note
return {
ts_file: file,
case_title: title,
disposition,
target,
oracle,
note: gone,
}
}
/** A `vitest (same case…)` target names a live case of the same file. */
function missingVitestTargets(rows, live) {
return rows
.filter((row) => row.target.startsWith('vitest (same case'))
.filter((row) => !live.get(row.ts_file)?.has(row.case_title))
.map((row) => `${row.ts_file} > ${row.case_title}`)
}
function collect() {
const errors = []
const used = new Set()
const totals = []
const cases = frozenCases()
const reportDir = mkdtempSync(join(tmpdir(), 'faunosteo-r4-ledger-'))
let live
try {
live = liveCases(liveSuites(cases), reportDir, errors, totals)
} finally {
rmdirSync(reportDir)
}
const rows = cases
.map((c) => rowOf(c, used, errors))
.filter((row) => row !== null)
RULES.forEach((rule, at) => {
if (!used.has(at))
errors.push(`rule matches no case: ${rule.file} ${rule.title ?? ''}`)
})
for (const name of missingVitestTargets(rows, live)) {
errors.push(`vitest target missing from the live report: ${name}`)
}
// Stable sort: files in path order, cases in frozen order.
rows.sort((a, b) =>
a.ts_file === b.ts_file ? 0 : a.ts_file < b.ts_file ? -1 : 1,
)
return { rows, errors, totals }
}
function tsvField(value) {
if (/[\t\r\n]/.test(value))
throw new Error(`tab or newline in ledger field: ${value}`)
return value
}
function writeLedger(rows) {
const lines = rows.map((row) =>
COLUMNS.map((c) => tsvField(row[c])).join('\t'),
)
mkdirSync(dirname(OUT), { recursive: true })
writeFileSync(OUT, [COLUMNS.join('\t'), ...lines].join('\n') + '\n')
}
function countLine(label, rows) {
const counts = DISPOSITIONS.map(
(d) => rows.filter((r) => r.disposition === d).length,
)
return [
label.padEnd(58),
String(rows.length).padStart(5),
...counts.map((n) => String(n).padStart(9)),
].join('')
}
function printSummary(rows, totals) {
console.log(
[
'file'.padEnd(58),
'rows'.padStart(5),
...DISPOSITIONS.map((d) => d.padStart(9)),
].join(''),
)
for (const file of [...new Set(rows.map((r) => r.ts_file))]) {
console.log(
countLine(
file,
rows.filter((r) => r.ts_file === file),
),
)
}
console.log(countLine('total', rows))
const vitest = totals.map(([pkg, n]) => `${pkg} ${n}`).join(', ')
const sum = totals.reduce((n, [, count]) => n + count, 0)
console.log(
`live vitest cases of the frozen files: ${vitest} = ${sum}; ledger rows: ${rows.length} (frozen ${FROZEN_ROWS})`,
)
console.log(`written: ${relative(ROOT, OUT)}`)
}
/** The tests of one cargo test file of the core (`cargo test --test <file> -- --list`). */
function cargoTests(file) {
const run = spawnSync(
'cargo',
[
'test',
'-p',
'faunosteo-core',
'--features',
'cli',
'--locked',
'--test',
file,
'--',
'--list',
],
{
cwd: ROOT,
encoding: 'utf8',
env: { ...process.env, CARGO_TERM_COLOR: 'never' },
},
)
if (run.status !== 0) {
throw new Error(`cargo test --test ${file} -- --list failed: ${run.stderr}`)
}
const names = new Set()
for (const line of run.stdout.split('\n')) {
const test = /^(\S+): test$/.exec(line)
if (test) names.add(test[1])
}
return names
}
/** Census (plan §B.6): every `cargo file::test` of every target names an existing test. */
function missingCargoTargets(rows) {
const wanted = new Set()
for (const row of rows) {
for (const part of row.target.split('; ')) {
if (!part.startsWith('cargo ')) continue
for (const name of part.slice('cargo '.length).split(', '))
if (name.includes('::')) wanted.add(name)
}
}
const listed = new Map()
const missing = []
for (const name of wanted) {
const [file, test] = name.split('::')
if (!listed.has(file)) listed.set(file, cargoTests(file))
if (!listed.get(file).has(test)) missing.push(name)
}
return missing
}
function main() {
const { rows, errors, totals } = collect()
for (const name of missingCargoTargets(rows)) {
errors.push(`cargo target missing from cargo test --list: ${name}`)
}
if (rows.length !== FROZEN_ROWS) {
errors.push(`${rows.length} rows, ${FROZEN_ROWS} frozen`)
}
if (errors.length) {
for (const error of errors) console.error(`r4-ledger: ${error}`)
console.error('r4-ledger: nothing written')
return 1
}
writeLedger(rows)
printSummary(rows, totals)
return 0
}
process.exitCode = main()