test(tests): R4.1f O10 report:compare outputs and the R4 test ledger (299 cases)

This commit is contained in:
bchanot committed 2026-10-18 06:00:34 +02:00
1 parent 516342beca
commit 4bdfda59ba
19 files changed
+9291

No files matched your search

@@ -0,0 +1,774 @@
/**
* R4 freeze O10 (plan 2026-10-17 §B.5.1, §B.1.7): what `pnpm report:compare`
* prints and answers today, for the Rust binary of R4.7 to reproduce. The TS
* CLI (scripts/compare-reports.ts) runs as pnpm runs it: tsx from apps/app,
* INIT_CWD at the repository root, arguments typed from there. Invocations:
* every fixture PDF (top level, pdf/r3, pdf/step6, pdf/r4/tolerance and
* pdf/r4/read), one folder, the argument errors, and the 11 scenarios of
* scripts.compare.test.ts, their documents attached to PDFs under
* crates/core/tests/oracles/r4/compare/scenarios/. Absolute paths are made
* repo-relative. Rows whose output the plan §A.3 declares as changing in R4
* carry `r4Declared`. `MAKE_R4_ORACLES=1` writes the scenario PDFs and
* compare.json; never again without the owner.
*/
import { spawn } from 'node:child_process'
import { readdirSync, readFileSync, statSync } from 'node:fs'
import { availableParallelism } from 'node:os'
import { basename, dirname, resolve } from 'node:path'
import { fileURLToPath } from 'node:url'
import { AFRelationship, PDFDocument } from 'pdf-lib'
import { beforeAll, describe, expect, it } from 'vitest'
import { readReportDocument, type ReportV2 } from '@faunosteo/domain'
import {
PDF_KEYWORD,
readAttachedDocument,
readAttachmentBytes,
} from '@faunosteo/pdf'
import type { SheetIndex } from '@/services/sheetIndex'
import {
compareSheetIndex,
type CompareSheetIndex,
} from '../../scripts/compare/sheet'
import {
archiveEnvelope,
bundledSheet,
loadTestSheet,
minimalReport,
} from '../helpers/sheet'
import {
checkBytesOracle,
checkJsonOracle,
MAKE_R4_ORACLES,
sha256,
} from '../helpers/r4-draw'
const HERE = dirname(fileURLToPath(import.meta.url))
const ROOT = resolve(HERE, '../../../..')
const APP_DIR = resolve(ROOT, 'apps/app')
const TSX_CLI = resolve(APP_DIR, 'node_modules/tsx/dist/cli.mjs')
const CLI_SCRIPT = 'scripts/compare-reports.ts'
const FIXTURES = 'apps/app/tests/fixtures'
const ORACLE_DIR = 'crates/core/tests/oracles/r4/compare'
const SCENARIO_DIR = `${ORACLE_DIR}/scenarios`
const OUT = resolve(ROOT, ORACLE_DIR, 'compare.json')
const V2_FULL_PDF = `${FIXTURES}/report-v2.full.pdf`
const STEP6_FOLDER = `${FIXTURES}/pdf/step6/`
const CONCURRENCY = Math.max(1, Math.min(8, availableParallelism()))
const RUN_TIMEOUT = 300_000
/** Every date of the scenario PDFs: they are byte-stable. */
const FIXED_DATE = new Date('2026-10-18T09:00:00.000Z')
const A4: [number, number] = [595.28, 841.89]
const ESC = '\u001b[2K'
const DECLARED = {
zod: 'A.3: the TS prints a ZodError message',
infinity: 'A.3: the TS reads a version as Infinity',
surrogate: 'A.3: the TS keeps a lone surrogate',
overwritten:
'A.3 (OD-4): the TS reads a lone surrogate held by an overwritten ' +
'duplicate',
} as const
type Group = 'fixture' | 'folder' | 'cli' | 'scenario'
interface Invocation {
group: Group
/** The scripts.compare.test.ts case, `describe > it`. */
scenario?: string
args: string[]
}
interface CliResult {
exitCode: number
stdout: string[]
stderr: string[]
}
interface Scenario {
title: string
/** Scenario PDF name → the document attached to it. */
documents: Record<string, unknown>
runs: string[][]
}
// ---------------------------------------------------------------------------
// The 11 scenarios of scripts.compare.test.ts, inputs reproduced as written
const scenarioPdf = (name: string) => `${SCENARIO_DIR}/${name}.pdf`
function minimalV2(): ReportV2 {
const path = resolve(ROOT, FIXTURES, 'report-v2.minimal.json')
return JSON.parse(readFileSync(path, 'utf8')) as ReportV2
}
function frozenV2Scenario(): Scenario {
return {
title:
'compareReport on the frozen v2 archive PDF > shows every legacy zone ' +
'and finding next to its canonical target',
documents: {},
runs: [[V2_FULL_PDF], ['--summary', V2_FULL_PDF], ['--full', V2_FULL_PDF]],
}
}
function unresolvedV2Scenario(): Scenario {
const v2: ReportV2 = {
...minimalV2(),
observations: [
{
id: 'o',
zones: [{ boneId: 'point-gone' }],
findings: [
{
zoneId: 'ssb',
path: ['ssb-dysfunctions', 'sutures', 'sphenozygomatic'],
labelFrSnapshot: 'Suture sphéno-zygomatique',
labelEnSnapshot: 'Sphenozygomatic suture',
requiresVetReferral: true,
},
{
zoneId: 'nowhere',
path: ['x'],
labelFrSnapshot: 'X',
labelEnSnapshot: 'X',
},
],
freeTextFindings: [],
comment: '',
},
],
}
return {
title:
'compareReport on unresolved items > names a removed finding, a ' +
'finding unknown to the catalog and an unknown zone',
documents: { 'v2-unresolved-items': v2 },
runs: [[scenarioPdf('v2-unresolved-items')]],
}
}
function v3AsIsScenario(sheet: SheetIndex): Scenario {
const report = minimalReport(sheet, {
observations: [
{
id: 'o3',
zones: [{ annotationId: 'hip', viewId: 'main', side: 'left' }],
findings: [
{
findingZoneId: 'hip',
path: ['hip-dysfunctions', 'articular', 'restriction-flexion'],
labelFrSnapshot: 'Restriction en flexion',
labelEnSnapshot: 'Flexion restriction',
},
{
findingZoneId: 'atm',
path: ['atm-observations', 'headshaking'],
labelFrSnapshot: 'Headshaking (secouement de tête)',
labelEnSnapshot: 'Headshaking',
unresolved: true,
},
{
findingZoneId: 'hip',
path: ['hip-dysfunctions'],
labelFrSnapshot: 'Catégorie',
labelEnSnapshot: 'Category',
},
],
freeTextFindings: [],
comment: '',
},
],
})
return {
title:
'compareReport on unresolved items > reads a v3 report as is and ' +
'flags what the sheet cannot place',
documents: { 'v3-as-is': report },
runs: [[scenarioPdf('v3-as-is')]],
}
}
function envelopeScenario(sheet: SheetIndex): Scenario {
const envelope = archiveEnvelope(
sheet,
minimalReport(sheet, {
observations: [
{
id: 'o-env',
zones: [{ annotationId: 'hip', viewId: 'main', side: 'left' }],
findings: [],
freeTextFindings: [],
comment: '',
},
],
}),
)
const names = ['envelope-kept', 'envelope-version-2', 'envelope-report-only']
return {
title:
'compareReport on unresolved items > prints the archive line of an ' +
'envelope: snapshot kept, or rejected with its reason',
documents: {
'envelope-kept': envelope,
'envelope-version-2': { ...envelope, envelopeVersion: 2 },
'envelope-report-only': envelope.report,
},
runs: names.map((name) => [scenarioPdf(name)]),
}
}
function otherSheetScenario(sheet: SheetIndex): Scenario {
const report = minimalReport(sheet, {
sheet: { ...sheet.sheet.ref, documentSha256: 'b'.repeat(64) },
})
return {
title:
'compareReport on unresolved items > warns when a v3 report was ' +
'written on another sheet document',
documents: { 'other-sheet-document': report },
runs: [[scenarioPdf('other-sheet-document')]],
}
}
function migratedDiffScenario(cli: CompareSheetIndex): Scenario {
const v2 = minimalV2()
const migrated = readReportDocument(v2, cli.sheet.migrationMap).report
const [a, b, c] = ['diff-v2', 'diff-v2-migrated', 'diff-v2-changed']
return {
title:
'diff and arguments > diffs the v3 content of a migrated v2 report ' +
'and its v3 re-reading',
documents: {
[a]: v2,
[b]: migrated,
[c]: { ...migrated, conclusion: 'changed' },
},
runs: [
['--diff', scenarioPdf(a), scenarioPdf(b)],
['--diff', scenarioPdf(a), scenarioPdf(c)],
],
}
}
function placeholderDiffScenario(cli: CompareSheetIndex): Scenario {
const base = readReportDocument(minimalV2(), cli.sheet.migrationMap)
const [original, edited] = ['diff-original', 'diff-edited']
return {
title:
'diff and arguments > compares placeholders by default and real ' +
'texts with --full',
documents: {
[original]: { ...base.report, conclusion: 'original' },
[edited]: { ...base.report, conclusion: 'changed' },
},
runs: [
['--diff', scenarioPdf(original), scenarioPdf(edited)],
['--full', '--diff', scenarioPdf(original), scenarioPdf(edited)],
],
}
}
function argumentsScenario(): Scenario {
return {
title:
'diff and arguments > parses files and options, refuses unknown ones',
documents: {},
runs: [
['a.pdf', '--summary', 'dir', '--locale', 'en'],
['--full', '--diff', 'a', 'b'],
['--identity'],
['--locale', 'de'],
['--json'],
],
}
}
function controlCharactersScenario(sheet: SheetIndex): Scenario {
const report = minimalReport(sheet, {
observations: [
{
id: `o${ESC}\n1`,
zones: [],
findings: [
{
findingZoneId: 'hip',
path: ['hip-dysfunctions', `x${ESC}\ny`],
labelFrSnapshot: `L${ESC}`,
labelEnSnapshot: 'L',
unresolved: true,
},
],
freeTextFindings: [`t${ESC}\nu`],
comment: `c${ESC}\nd\u009b‮
`,
},
],
})
return {
title:
'terminal safety and privacy > escapes control characters in every ' +
'untrusted string',
documents: { 'control-characters': report },
runs: [['--full', scenarioPdf('control-characters')]],
}
}
function prototypeIdsScenario(): Scenario {
const v2: ReportV2 = {
...minimalV2(),
observations: [
{
id: 'o',
zones: [{ boneId: '__proto__' }, { boneId: 'constructor' }],
findings: [
{
zoneId: '__proto__',
path: ['constructor'],
labelFrSnapshot: 'X',
labelEnSnapshot: 'X',
},
],
freeTextFindings: [],
comment: '',
},
],
}
return {
title:
'terminal safety and privacy > does not throw on prototype-chain ' +
'names used as legacy ids',
documents: { 'prototype-ids': v2 },
runs: [[scenarioPdf('prototype-ids')]],
}
}
function privacyScenario(sheet: SheetIndex): Scenario {
const report = minimalReport(sheet, {
conclusion: 'SECRET-CONCLUSION',
recommendations: 'SECRET-RECO',
nextAppointment: { kind: 'period', text: 'SECRET-WHEN' },
observations: [
{
id: 'o',
zones: [],
findings: [],
freeTextFindings: ['SECRET-TEXT'],
comment: 'SECRET-COMMENT',
},
],
})
const pdf = scenarioPdf('hidden-texts')
return {
title:
'terminal safety and privacy > hides names and free texts by default, ' +
'shows them with --full',
documents: { 'hidden-texts': report },
runs: [[pdf], ['--full', pdf]],
}
}
function buildScenarios(sheet: SheetIndex, cli: CompareSheetIndex) {
return [
frozenV2Scenario(),
unresolvedV2Scenario(),
v3AsIsScenario(sheet),
envelopeScenario(sheet),
otherSheetScenario(sheet),
migratedDiffScenario(cli),
placeholderDiffScenario(cli),
argumentsScenario(),
controlCharactersScenario(sheet),
prototypeIdsScenario(),
privacyScenario(sheet),
]
}
// ---------------------------------------------------------------------------
// Invocation plan
/** PDFs of a folder, repo-relative and sorted; `deep` walks sub-folders. */
function pdfsUnder(folder: string, deep: boolean): string[] {
return readdirSync(resolve(ROOT, folder), { withFileTypes: true })
.flatMap((entry) => {
const path = `${folder}/${entry.name}`
if (entry.isDirectory()) return deep ? pdfsUnder(path, true) : []
return entry.name.toLowerCase().endsWith('.pdf') ? [path] : []
})
.sort()
}
function fixturePdfs(): string[] {
return [
...pdfsUnder(FIXTURES, false),
...pdfsUnder(`${FIXTURES}/pdf/r3`, true),
...pdfsUnder(`${FIXTURES}/pdf/step6`, false),
...pdfsUnder(`${FIXTURES}/pdf/r4/tolerance`, false),
...pdfsUnder(`${FIXTURES}/pdf/r4/read`, false),
]
}
function invocationPlan(scenarios: readonly Scenario[]): Invocation[] {
const of = (group: Group, runs: string[][]) =>
runs.map((args): Invocation => ({ group, args }))
return [
...of(
'fixture',
fixturePdfs().map((pdf) => [pdf]),
),
...of('folder', [
[STEP6_FOLDER],
['--full', '--locale', 'en', STEP6_FOLDER],
]),
...of('cli', [
[],
['--diff', V2_FULL_PDF],
['--locale'],
[V2_FULL_PDF, '--summary', STEP6_FOLDER, '--locale', 'en'],
]),
...scenarios.flatMap(({ title, runs }) =>
runs.map(
(args): Invocation => ({ group: 'scenario', scenario: title, args }),
),
),
]
}
/** The files the CLI reads for `args`: options skipped, folders listed. */
function inputFiles(args: readonly string[]): string[] {
const files: string[] = []
for (let i = 0; i < args.length; i += 1) {
const arg = args[i]!
if (arg === '--locale') i += 1
else if (!arg.startsWith('--')) files.push(...expandArg(arg))
}
return files
}
function expandArg(arg: string): string[] {
try {
if (!statSync(resolve(ROOT, arg)).isDirectory()) return [arg]
} catch {
return [arg]
}
return pdfsUnder(arg.replace(/\/$/, ''), false)
}
// ---------------------------------------------------------------------------
// Running the CLI
function cliEnv(): NodeJS.ProcessEnv {
const env: NodeJS.ProcessEnv = { INIT_CWD: ROOT }
for (const key of ['PATH', 'HOME', 'TMPDIR'] as const) {
if (process.env[key] !== undefined) env[key] = process.env[key]
}
return env
}
/** The exact text split on "\n", the repository root cut from paths. */
function outputLines(chunks: readonly Buffer[]): string[] {
const text = Buffer.concat(chunks).toString('utf8')
return text.split(`${ROOT}/`).join('').split(ROOT).join('.').split('\n')
}
function runCli(args: readonly string[]): Promise<CliResult> {
return new Promise((settle, fail) => {
const child = spawn(process.execPath, [TSX_CLI, CLI_SCRIPT, ...args], {
cwd: APP_DIR,
env: cliEnv(),
stdio: ['ignore', 'pipe', 'pipe'],
})
const stdout: Buffer[] = []
const stderr: Buffer[] = []
child.stdout.on('data', (chunk: Buffer) => stdout.push(chunk))
child.stderr.on('data', (chunk: Buffer) => stderr.push(chunk))
child.on('error', fail)
child.on('close', (code, signal) => {
if (code === null) fail(new Error(`CLI stopped by ${signal}`))
else
settle({
exitCode: code,
stdout: outputLines(stdout),
stderr: outputLines(stderr),
})
})
})
}
/** `run` over `items`, at most `limit` at a time, results in item order. */
async function inPool<T, R>(
items: readonly T[],
limit: number,
run: (item: T) => Promise<R>,
): Promise<R[]> {
const results = new Array<R>(items.length)
let next = 0
const lane = async () => {
while (next < items.length) {
const at = next++
results[at] = await run(items[at]!)
}
}
await Promise.all(Array.from({ length: limit }, lane))
return results
}
// ---------------------------------------------------------------------------
// The A.3 declarations, from the outputs and the attachment each file holds
interface AttachmentFacts {
/** A key or string of the parsed attachment holds a lone surrogate. */
parsedSurrogate: boolean
/** A string literal of the attachment text holds one (overwritten too). */
textSurrogate: boolean
/** `version`, `envelopeVersion` or `report.version` parsed as ±Infinity. */
infiniteVersion: boolean
}
const NO_FACTS: AttachmentFacts = {
parsedSurrogate: false,
textSurrogate: false,
infiniteVersion: false,
}
const LONE_SURROGATE = /\p{Cs}/u
const STRING_LITERAL = /"(?:[^"\\]|\\.)*"/g
function holdsLoneSurrogate(value: unknown): boolean {
if (typeof value === 'string') return LONE_SURROGATE.test(value)
if (typeof value !== 'object' || value === null) return false
return Object.entries(value).some(
([key, item]) => LONE_SURROGATE.test(key) || holdsLoneSurrogate(item),
)
}
function decodesToLoneSurrogate(literal: string): boolean {
try {
return LONE_SURROGATE.test(JSON.parse(literal) as string)
} catch {
return false
}
}
function literalHoldsLoneSurrogate(text: string): boolean {
for (const [literal] of text.matchAll(STRING_LITERAL)) {
if (decodesToLoneSurrogate(literal)) return true
}
return false
}
const versionOf = (value: unknown): unknown =>
typeof value === 'object' && value !== null
? (value as { version?: unknown }).version
: undefined
function hasInfiniteVersion(raw: unknown): boolean {
if (typeof raw !== 'object' || raw === null) return false
const doc = raw as { envelopeVersion?: unknown; report?: unknown }
return [versionOf(raw), doc.envelopeVersion, versionOf(doc.report)].some(
(v) => typeof v === 'number' && !Number.isFinite(v),
)
}
async function settled<T>(promise: Promise<T>): Promise<T | undefined> {
try {
return await promise
} catch {
return undefined
}
}
async function attachmentFacts(file: string): Promise<AttachmentFacts> {
let bytes: Uint8Array
try {
bytes = new Uint8Array(readFileSync(resolve(ROOT, file)))
} catch {
return NO_FACTS
}
const raw = await settled(readAttachedDocument(bytes))
const text = await settled(
readAttachmentBytes(bytes).then((b) => new TextDecoder().decode(b)),
)
return {
parsedSurrogate: raw !== undefined && holdsLoneSurrogate(raw),
textSurrogate: text !== undefined && literalHoldsLoneSurrogate(text),
infiniteVersion: raw !== undefined && hasInfiniteVersion(raw),
}
}
/** A failure line whose reason is a ZodError message (issues as JSON). */
function isZodFailure(line: string): boolean {
const at = line.indexOf(': [')
if (!line.startsWith('!! ') || at < 0) return false
const reason = line
.slice(at + 2)
.replace(/\\u([0-9a-f]{4})/g, (_, hex: string) =>
String.fromCharCode(parseInt(hex, 16)),
)
try {
const issues: unknown = JSON.parse(reason)
return (
Array.isArray(issues) &&
issues.length > 0 &&
issues.every(
(i) => typeof (i as { code?: unknown } | null)?.code === 'string',
)
)
} catch {
return false
}
}
function declaredChange(
files: readonly string[],
stdout: readonly string[],
facts: ReadonlyMap<string, AttachmentFacts>,
): string | undefined {
const reasons = new Set<string>()
if (stdout.some(isZodFailure)) reasons.add(DECLARED.zod)
for (const file of files) {
const fact = facts.get(file) ?? NO_FACTS
if (fact.infiniteVersion) reasons.add(DECLARED.infinity)
if (stdout.some((l) => l.startsWith(`!! ${basename(file)}: `))) continue
if (fact.parsedSurrogate) reasons.add(DECLARED.surrogate)
else if (fact.textSurrogate) reasons.add(DECLARED.overwritten)
}
return reasons.size ? [...reasons].join('; ') : undefined
}
// ---------------------------------------------------------------------------
// Oracle
/** Shaped like an archive export, as r4.tolerance-oracles builds its corpus. */
async function buildPdf(document: unknown): Promise<Uint8Array> {
const doc = await PDFDocument.create()
doc.addPage(A4)
doc.setKeywords([PDF_KEYWORD])
doc.setCreationDate(FIXED_DATE)
doc.setModificationDate(FIXED_DATE)
const bytes = new TextEncoder().encode(JSON.stringify(document))
await doc.attach(bytes, 'osteo-report.json', {
mimeType: 'application/json',
description: 'Faunosteo report archive v1',
creationDate: FIXED_DATE,
modificationDate: FIXED_DATE,
afRelationship: AFRelationship.Data,
})
return doc.save({ useObjectStreams: false })
}
async function factsOf(files: readonly string[]) {
const facts = new Map<string, AttachmentFacts>()
for (const file of files) facts.set(file, await attachmentFacts(file))
return facts
}
/** sha256 of every input file that exists (argument errors name none). */
function inputDigests(files: readonly string[]): Record<string, string> {
const digests: Record<string, string> = {}
for (const file of files) {
const path = resolve(ROOT, file)
if (statSync(path, { throwIfNoEntry: false })?.isFile()) {
digests[file] = sha256(new Uint8Array(readFileSync(path)))
}
}
return digests
}
async function recordInvocations(plan: readonly Invocation[]) {
const results = await inPool(plan, CONCURRENCY, (inv) => runCli(inv.args))
const files = [...new Set(plan.flatMap((inv) => inputFiles(inv.args)))]
const facts = await factsOf(files)
const invocations = plan.map((inv, i) => {
const { exitCode, stdout, stderr } = results[i]!
const declared = declaredChange(inputFiles(inv.args), stdout, facts)
return {
...inv,
...(declared ? { r4Declared: declared } : {}),
exitCode,
stdout,
stderr,
}
})
return { inputs: inputDigests(files), invocations }
}
function refuseFieldCorpus(): void {
if (process.env.FAUNOSTEO_FIELD_CORPUS) {
throw new Error('refused: FAUNOSTEO_FIELD_CORPUS is set (plan §B.5.2)')
}
}
type OracleRow = Invocation & { r4Declared?: string; exitCode: number }
function committedRows(): OracleRow[] {
return (JSON.parse(readFileSync(OUT, 'utf8')) as { invocations: OracleRow[] })
.invocations
}
// ---------------------------------------------------------------------------
describe('R4 O10 report:compare outputs of the TS CLI', () => {
let cli: CompareSheetIndex
let scenarios: Scenario[]
beforeAll(async () => {
if (MAKE_R4_ORACLES) refuseFieldCorpus()
const sheet = await loadTestSheet()
cli = compareSheetIndex(await bundledSheet())
scenarios = buildScenarios(sheet, cli)
})
it('builds byte-stable scenario PDFs (MAKE_R4_ORACLES writes)', async () => {
const names = scenarios.flatMap((s) => Object.keys(s.documents))
expect(new Set(names).size).toBe(names.length)
for (const scenario of scenarios) {
for (const [name, document] of Object.entries(scenario.documents)) {
const path = resolve(ROOT, scenarioPdf(name))
checkBytesOracle(path, await buildPdf(document))
}
}
const listed = readdirSync(resolve(ROOT, SCENARIO_DIR)).sort()
expect(listed).toEqual(names.map((n) => `${n}.pdf`).sort())
})
it(
'records every invocation (written under MAKE_R4_ORACLES)',
async () => {
const { inputs, invocations } = await recordInvocations(
invocationPlan(scenarios),
)
checkJsonOracle(OUT, {
oracle: 'O10',
cli: `apps/app/${CLI_SCRIPT}`,
run:
'node apps/app/node_modules/tsx/dist/cli.mjs ' +
`${CLI_SCRIPT} <args>, cwd apps/app, INIT_CWD at the repository ` +
'root (pnpm report:compare typed from the root); arguments and ' +
'printed paths are repo-relative',
text:
'stdout and stderr are split on "\\n": joined with "\\n" they ' +
'give the exact text',
sheet: cli.sheet.ref,
inputs,
invocations,
})
},
RUN_TIMEOUT,
)
it('covers the 11 scenarios and marks every A.3 class at least once', () => {
const rows = committedRows()
const titles = new Set(rows.flatMap((r) => r.scenario ?? []))
expect(titles.size).toBe(11)
for (const reason of Object.values(DECLARED)) {
expect(
rows.some((r) => r.r4Declared?.includes(reason)),
reason,
).toBe(true)
}
const control = rows.find(
(r) => r.group === 'fixture' && r.args[0] === V2_FULL_PDF,
)
expect(control).toMatchObject({ exitCode: 0 })
expect(control).not.toHaveProperty('r4Declared')
})
})