Files
faunosteo/apps/app/tests/unit/r4.compare-oracles.test.ts
T

775 lines
23 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
/**
* R4 freeze O10 (plan 2026-10-17 §B.5.1, §B.1.7): what `pnpm report:compare`
* prints and answers today, for the Rust binary of R4.7 to reproduce. The TS
* CLI (scripts/compare-reports.ts) runs as pnpm runs it: tsx from apps/app,
* INIT_CWD at the repository root, arguments typed from there. Invocations:
* every fixture PDF (top level, pdf/r3, pdf/step6, pdf/r4/tolerance and
* pdf/r4/read), one folder, the argument errors, and the 11 scenarios of
* scripts.compare.test.ts, their documents attached to PDFs under
* crates/core/tests/oracles/r4/compare/scenarios/. Absolute paths are made
* repo-relative. Rows whose output the plan §A.3 declares as changing in R4
* carry `r4Declared`. `MAKE_R4_ORACLES=1` writes the scenario PDFs and
* compare.json; never again without the owner.
*/
import { spawn } from 'node:child_process'
import { readdirSync, readFileSync, statSync } from 'node:fs'
import { availableParallelism } from 'node:os'
import { basename, dirname, resolve } from 'node:path'
import { fileURLToPath } from 'node:url'
import { AFRelationship, PDFDocument } from 'pdf-lib'
import { beforeAll, describe, expect, it } from 'vitest'
import { readReportDocument, type ReportV2 } from '@faunosteo/domain'
import {
PDF_KEYWORD,
readAttachedDocument,
readAttachmentBytes,
} from '@faunosteo/pdf'
import type { SheetIndex } from '@/services/sheetIndex'
import {
compareSheetIndex,
type CompareSheetIndex,
} from '../../scripts/compare/sheet'
import {
archiveEnvelope,
bundledSheet,
loadTestSheet,
minimalReport,
} from '../helpers/sheet'
import {
checkBytesOracle,
checkJsonOracle,
MAKE_R4_ORACLES,
sha256,
} from '../helpers/r4-draw'
const HERE = dirname(fileURLToPath(import.meta.url))
const ROOT = resolve(HERE, '../../../..')
const APP_DIR = resolve(ROOT, 'apps/app')
const TSX_CLI = resolve(APP_DIR, 'node_modules/tsx/dist/cli.mjs')
const CLI_SCRIPT = 'scripts/compare-reports.ts'
const FIXTURES = 'apps/app/tests/fixtures'
const ORACLE_DIR = 'crates/core/tests/oracles/r4/compare'
const SCENARIO_DIR = `${ORACLE_DIR}/scenarios`
const OUT = resolve(ROOT, ORACLE_DIR, 'compare.json')
const V2_FULL_PDF = `${FIXTURES}/report-v2.full.pdf`
const STEP6_FOLDER = `${FIXTURES}/pdf/step6/`
const CONCURRENCY = Math.max(1, Math.min(8, availableParallelism()))
const RUN_TIMEOUT = 300_000
/** Every date of the scenario PDFs: they are byte-stable. */
const FIXED_DATE = new Date('2026-10-18T09:00:00.000Z')
const A4: [number, number] = [595.28, 841.89]
const ESC = '\u001b[2K'
const DECLARED = {
zod: 'A.3: the TS prints a ZodError message',
infinity: 'A.3: the TS reads a version as Infinity',
surrogate: 'A.3: the TS keeps a lone surrogate',
overwritten:
'A.3 (OD-4): the TS reads a lone surrogate held by an overwritten ' +
'duplicate',
} as const
type Group = 'fixture' | 'folder' | 'cli' | 'scenario'
interface Invocation {
group: Group
/** The scripts.compare.test.ts case, `describe > it`. */
scenario?: string
args: string[]
}
interface CliResult {
exitCode: number
stdout: string[]
stderr: string[]
}
interface Scenario {
title: string
/** Scenario PDF name → the document attached to it. */
documents: Record<string, unknown>
runs: string[][]
}
// ---------------------------------------------------------------------------
// The 11 scenarios of scripts.compare.test.ts, inputs reproduced as written
const scenarioPdf = (name: string) => `${SCENARIO_DIR}/${name}.pdf`
function minimalV2(): ReportV2 {
const path = resolve(ROOT, FIXTURES, 'report-v2.minimal.json')
return JSON.parse(readFileSync(path, 'utf8')) as ReportV2
}
function frozenV2Scenario(): Scenario {
return {
title:
'compareReport on the frozen v2 archive PDF > shows every legacy zone ' +
'and finding next to its canonical target',
documents: {},
runs: [[V2_FULL_PDF], ['--summary', V2_FULL_PDF], ['--full', V2_FULL_PDF]],
}
}
function unresolvedV2Scenario(): Scenario {
const v2: ReportV2 = {
...minimalV2(),
observations: [
{
id: 'o',
zones: [{ boneId: 'point-gone' }],
findings: [
{
zoneId: 'ssb',
path: ['ssb-dysfunctions', 'sutures', 'sphenozygomatic'],
labelFrSnapshot: 'Suture sphéno-zygomatique',
labelEnSnapshot: 'Sphenozygomatic suture',
requiresVetReferral: true,
},
{
zoneId: 'nowhere',
path: ['x'],
labelFrSnapshot: 'X',
labelEnSnapshot: 'X',
},
],
freeTextFindings: [],
comment: '',
},
],
}
return {
title:
'compareReport on unresolved items > names a removed finding, a ' +
'finding unknown to the catalog and an unknown zone',
documents: { 'v2-unresolved-items': v2 },
runs: [[scenarioPdf('v2-unresolved-items')]],
}
}
function v3AsIsScenario(sheet: SheetIndex): Scenario {
const report = minimalReport(sheet, {
observations: [
{
id: 'o3',
zones: [{ annotationId: 'hip', viewId: 'main', side: 'left' }],
findings: [
{
findingZoneId: 'hip',
path: ['hip-dysfunctions', 'articular', 'restriction-flexion'],
labelFrSnapshot: 'Restriction en flexion',
labelEnSnapshot: 'Flexion restriction',
},
{
findingZoneId: 'atm',
path: ['atm-observations', 'headshaking'],
labelFrSnapshot: 'Headshaking (secouement de tête)',
labelEnSnapshot: 'Headshaking',
unresolved: true,
},
{
findingZoneId: 'hip',
path: ['hip-dysfunctions'],
labelFrSnapshot: 'Catégorie',
labelEnSnapshot: 'Category',
},
],
freeTextFindings: [],
comment: '',
},
],
})
return {
title:
'compareReport on unresolved items > reads a v3 report as is and ' +
'flags what the sheet cannot place',
documents: { 'v3-as-is': report },
runs: [[scenarioPdf('v3-as-is')]],
}
}
function envelopeScenario(sheet: SheetIndex): Scenario {
const envelope = archiveEnvelope(
sheet,
minimalReport(sheet, {
observations: [
{
id: 'o-env',
zones: [{ annotationId: 'hip', viewId: 'main', side: 'left' }],
findings: [],
freeTextFindings: [],
comment: '',
},
],
}),
)
const names = ['envelope-kept', 'envelope-version-2', 'envelope-report-only']
return {
title:
'compareReport on unresolved items > prints the archive line of an ' +
'envelope: snapshot kept, or rejected with its reason',
documents: {
'envelope-kept': envelope,
'envelope-version-2': { ...envelope, envelopeVersion: 2 },
'envelope-report-only': envelope.report,
},
runs: names.map((name) => [scenarioPdf(name)]),
}
}
function otherSheetScenario(sheet: SheetIndex): Scenario {
const report = minimalReport(sheet, {
sheet: { ...sheet.sheet.ref, documentSha256: 'b'.repeat(64) },
})
return {
title:
'compareReport on unresolved items > warns when a v3 report was ' +
'written on another sheet document',
documents: { 'other-sheet-document': report },
runs: [[scenarioPdf('other-sheet-document')]],
}
}
function migratedDiffScenario(cli: CompareSheetIndex): Scenario {
const v2 = minimalV2()
const migrated = readReportDocument(v2, cli.sheet.migrationMap).report
const [a, b, c] = ['diff-v2', 'diff-v2-migrated', 'diff-v2-changed']
return {
title:
'diff and arguments > diffs the v3 content of a migrated v2 report ' +
'and its v3 re-reading',
documents: {
[a]: v2,
[b]: migrated,
[c]: { ...migrated, conclusion: 'changed' },
},
runs: [
['--diff', scenarioPdf(a), scenarioPdf(b)],
['--diff', scenarioPdf(a), scenarioPdf(c)],
],
}
}
function placeholderDiffScenario(cli: CompareSheetIndex): Scenario {
const base = readReportDocument(minimalV2(), cli.sheet.migrationMap)
const [original, edited] = ['diff-original', 'diff-edited']
return {
title:
'diff and arguments > compares placeholders by default and real ' +
'texts with --full',
documents: {
[original]: { ...base.report, conclusion: 'original' },
[edited]: { ...base.report, conclusion: 'changed' },
},
runs: [
['--diff', scenarioPdf(original), scenarioPdf(edited)],
['--full', '--diff', scenarioPdf(original), scenarioPdf(edited)],
],
}
}
function argumentsScenario(): Scenario {
return {
title:
'diff and arguments > parses files and options, refuses unknown ones',
documents: {},
runs: [
['a.pdf', '--summary', 'dir', '--locale', 'en'],
['--full', '--diff', 'a', 'b'],
['--identity'],
['--locale', 'de'],
['--json'],
],
}
}
function controlCharactersScenario(sheet: SheetIndex): Scenario {
const report = minimalReport(sheet, {
observations: [
{
id: `o${ESC}\n1`,
zones: [],
findings: [
{
findingZoneId: 'hip',
path: ['hip-dysfunctions', `x${ESC}\ny`],
labelFrSnapshot: `L${ESC}`,
labelEnSnapshot: 'L',
unresolved: true,
},
],
freeTextFindings: [`t${ESC}\nu`],
comment: `c${ESC}\nd\u009b‮
`,
},
],
})
return {
title:
'terminal safety and privacy > escapes control characters in every ' +
'untrusted string',
documents: { 'control-characters': report },
runs: [['--full', scenarioPdf('control-characters')]],
}
}
function prototypeIdsScenario(): Scenario {
const v2: ReportV2 = {
...minimalV2(),
observations: [
{
id: 'o',
zones: [{ boneId: '__proto__' }, { boneId: 'constructor' }],
findings: [
{
zoneId: '__proto__',
path: ['constructor'],
labelFrSnapshot: 'X',
labelEnSnapshot: 'X',
},
],
freeTextFindings: [],
comment: '',
},
],
}
return {
title:
'terminal safety and privacy > does not throw on prototype-chain ' +
'names used as legacy ids',
documents: { 'prototype-ids': v2 },
runs: [[scenarioPdf('prototype-ids')]],
}
}
function privacyScenario(sheet: SheetIndex): Scenario {
const report = minimalReport(sheet, {
conclusion: 'SECRET-CONCLUSION',
recommendations: 'SECRET-RECO',
nextAppointment: { kind: 'period', text: 'SECRET-WHEN' },
observations: [
{
id: 'o',
zones: [],
findings: [],
freeTextFindings: ['SECRET-TEXT'],
comment: 'SECRET-COMMENT',
},
],
})
const pdf = scenarioPdf('hidden-texts')
return {
title:
'terminal safety and privacy > hides names and free texts by default, ' +
'shows them with --full',
documents: { 'hidden-texts': report },
runs: [[pdf], ['--full', pdf]],
}
}
function buildScenarios(sheet: SheetIndex, cli: CompareSheetIndex) {
return [
frozenV2Scenario(),
unresolvedV2Scenario(),
v3AsIsScenario(sheet),
envelopeScenario(sheet),
otherSheetScenario(sheet),
migratedDiffScenario(cli),
placeholderDiffScenario(cli),
argumentsScenario(),
controlCharactersScenario(sheet),
prototypeIdsScenario(),
privacyScenario(sheet),
]
}
// ---------------------------------------------------------------------------
// Invocation plan
/** PDFs of a folder, repo-relative and sorted; `deep` walks sub-folders. */
function pdfsUnder(folder: string, deep: boolean): string[] {
return readdirSync(resolve(ROOT, folder), { withFileTypes: true })
.flatMap((entry) => {
const path = `${folder}/${entry.name}`
if (entry.isDirectory()) return deep ? pdfsUnder(path, true) : []
return entry.name.toLowerCase().endsWith('.pdf') ? [path] : []
})
.sort()
}
function fixturePdfs(): string[] {
return [
...pdfsUnder(FIXTURES, false),
...pdfsUnder(`${FIXTURES}/pdf/r3`, true),
...pdfsUnder(`${FIXTURES}/pdf/step6`, false),
...pdfsUnder(`${FIXTURES}/pdf/r4/tolerance`, false),
...pdfsUnder(`${FIXTURES}/pdf/r4/read`, false),
]
}
function invocationPlan(scenarios: readonly Scenario[]): Invocation[] {
const of = (group: Group, runs: string[][]) =>
runs.map((args): Invocation => ({ group, args }))
return [
...of(
'fixture',
fixturePdfs().map((pdf) => [pdf]),
),
...of('folder', [
[STEP6_FOLDER],
['--full', '--locale', 'en', STEP6_FOLDER],
]),
...of('cli', [
[],
['--diff', V2_FULL_PDF],
['--locale'],
[V2_FULL_PDF, '--summary', STEP6_FOLDER, '--locale', 'en'],
]),
...scenarios.flatMap(({ title, runs }) =>
runs.map(
(args): Invocation => ({ group: 'scenario', scenario: title, args }),
),
),
]
}
/** The files the CLI reads for `args`: options skipped, folders listed. */
function inputFiles(args: readonly string[]): string[] {
const files: string[] = []
for (let i = 0; i < args.length; i += 1) {
const arg = args[i]!
if (arg === '--locale') i += 1
else if (!arg.startsWith('--')) files.push(...expandArg(arg))
}
return files
}
function expandArg(arg: string): string[] {
try {
if (!statSync(resolve(ROOT, arg)).isDirectory()) return [arg]
} catch {
return [arg]
}
return pdfsUnder(arg.replace(/\/$/, ''), false)
}
// ---------------------------------------------------------------------------
// Running the CLI
function cliEnv(): NodeJS.ProcessEnv {
const env: NodeJS.ProcessEnv = { INIT_CWD: ROOT }
for (const key of ['PATH', 'HOME', 'TMPDIR'] as const) {
if (process.env[key] !== undefined) env[key] = process.env[key]
}
return env
}
/** The exact text split on "\n", the repository root cut from paths. */
function outputLines(chunks: readonly Buffer[]): string[] {
const text = Buffer.concat(chunks).toString('utf8')
return text.split(`${ROOT}/`).join('').split(ROOT).join('.').split('\n')
}
function runCli(args: readonly string[]): Promise<CliResult> {
return new Promise((settle, fail) => {
const child = spawn(process.execPath, [TSX_CLI, CLI_SCRIPT, ...args], {
cwd: APP_DIR,
env: cliEnv(),
stdio: ['ignore', 'pipe', 'pipe'],
})
const stdout: Buffer[] = []
const stderr: Buffer[] = []
child.stdout.on('data', (chunk: Buffer) => stdout.push(chunk))
child.stderr.on('data', (chunk: Buffer) => stderr.push(chunk))
child.on('error', fail)
child.on('close', (code, signal) => {
if (code === null) fail(new Error(`CLI stopped by ${signal}`))
else
settle({
exitCode: code,
stdout: outputLines(stdout),
stderr: outputLines(stderr),
})
})
})
}
/** `run` over `items`, at most `limit` at a time, results in item order. */
async function inPool<T, R>(
items: readonly T[],
limit: number,
run: (item: T) => Promise<R>,
): Promise<R[]> {
const results = new Array<R>(items.length)
let next = 0
const lane = async () => {
while (next < items.length) {
const at = next++
results[at] = await run(items[at]!)
}
}
await Promise.all(Array.from({ length: limit }, lane))
return results
}
// ---------------------------------------------------------------------------
// The A.3 declarations, from the outputs and the attachment each file holds
interface AttachmentFacts {
/** A key or string of the parsed attachment holds a lone surrogate. */
parsedSurrogate: boolean
/** A string literal of the attachment text holds one (overwritten too). */
textSurrogate: boolean
/** `version`, `envelopeVersion` or `report.version` parsed as ±Infinity. */
infiniteVersion: boolean
}
const NO_FACTS: AttachmentFacts = {
parsedSurrogate: false,
textSurrogate: false,
infiniteVersion: false,
}
const LONE_SURROGATE = /\p{Cs}/u
const STRING_LITERAL = /"(?:[^"\\]|\\.)*"/g
function holdsLoneSurrogate(value: unknown): boolean {
if (typeof value === 'string') return LONE_SURROGATE.test(value)
if (typeof value !== 'object' || value === null) return false
return Object.entries(value).some(
([key, item]) => LONE_SURROGATE.test(key) || holdsLoneSurrogate(item),
)
}
function decodesToLoneSurrogate(literal: string): boolean {
try {
return LONE_SURROGATE.test(JSON.parse(literal) as string)
} catch {
return false
}
}
function literalHoldsLoneSurrogate(text: string): boolean {
for (const [literal] of text.matchAll(STRING_LITERAL)) {
if (decodesToLoneSurrogate(literal)) return true
}
return false
}
const versionOf = (value: unknown): unknown =>
typeof value === 'object' && value !== null
? (value as { version?: unknown }).version
: undefined
function hasInfiniteVersion(raw: unknown): boolean {
if (typeof raw !== 'object' || raw === null) return false
const doc = raw as { envelopeVersion?: unknown; report?: unknown }
return [versionOf(raw), doc.envelopeVersion, versionOf(doc.report)].some(
(v) => typeof v === 'number' && !Number.isFinite(v),
)
}
async function settled<T>(promise: Promise<T>): Promise<T | undefined> {
try {
return await promise
} catch {
return undefined
}
}
async function attachmentFacts(file: string): Promise<AttachmentFacts> {
let bytes: Uint8Array
try {
bytes = new Uint8Array(readFileSync(resolve(ROOT, file)))
} catch {
return NO_FACTS
}
const raw = await settled(readAttachedDocument(bytes))
const text = await settled(
readAttachmentBytes(bytes).then((b) => new TextDecoder().decode(b)),
)
return {
parsedSurrogate: raw !== undefined && holdsLoneSurrogate(raw),
textSurrogate: text !== undefined && literalHoldsLoneSurrogate(text),
infiniteVersion: raw !== undefined && hasInfiniteVersion(raw),
}
}
/** A failure line whose reason is a ZodError message (issues as JSON). */
function isZodFailure(line: string): boolean {
const at = line.indexOf(': [')
if (!line.startsWith('!! ') || at < 0) return false
const reason = line
.slice(at + 2)
.replace(/\\u([0-9a-f]{4})/g, (_, hex: string) =>
String.fromCharCode(parseInt(hex, 16)),
)
try {
const issues: unknown = JSON.parse(reason)
return (
Array.isArray(issues) &&
issues.length > 0 &&
issues.every(
(i) => typeof (i as { code?: unknown } | null)?.code === 'string',
)
)
} catch {
return false
}
}
function declaredChange(
files: readonly string[],
stdout: readonly string[],
facts: ReadonlyMap<string, AttachmentFacts>,
): string | undefined {
const reasons = new Set<string>()
if (stdout.some(isZodFailure)) reasons.add(DECLARED.zod)
for (const file of files) {
const fact = facts.get(file) ?? NO_FACTS
if (fact.infiniteVersion) reasons.add(DECLARED.infinity)
if (stdout.some((l) => l.startsWith(`!! ${basename(file)}: `))) continue
if (fact.parsedSurrogate) reasons.add(DECLARED.surrogate)
else if (fact.textSurrogate) reasons.add(DECLARED.overwritten)
}
return reasons.size ? [...reasons].join('; ') : undefined
}
// ---------------------------------------------------------------------------
// Oracle
/** Shaped like an archive export, as r4.tolerance-oracles builds its corpus. */
async function buildPdf(document: unknown): Promise<Uint8Array> {
const doc = await PDFDocument.create()
doc.addPage(A4)
doc.setKeywords([PDF_KEYWORD])
doc.setCreationDate(FIXED_DATE)
doc.setModificationDate(FIXED_DATE)
const bytes = new TextEncoder().encode(JSON.stringify(document))
await doc.attach(bytes, 'osteo-report.json', {
mimeType: 'application/json',
description: 'Faunosteo report archive v1',
creationDate: FIXED_DATE,
modificationDate: FIXED_DATE,
afRelationship: AFRelationship.Data,
})
return doc.save({ useObjectStreams: false })
}
async function factsOf(files: readonly string[]) {
const facts = new Map<string, AttachmentFacts>()
for (const file of files) facts.set(file, await attachmentFacts(file))
return facts
}
/** sha256 of every input file that exists (argument errors name none). */
function inputDigests(files: readonly string[]): Record<string, string> {
const digests: Record<string, string> = {}
for (const file of files) {
const path = resolve(ROOT, file)
if (statSync(path, { throwIfNoEntry: false })?.isFile()) {
digests[file] = sha256(new Uint8Array(readFileSync(path)))
}
}
return digests
}
async function recordInvocations(plan: readonly Invocation[]) {
const results = await inPool(plan, CONCURRENCY, (inv) => runCli(inv.args))
const files = [...new Set(plan.flatMap((inv) => inputFiles(inv.args)))]
const facts = await factsOf(files)
const invocations = plan.map((inv, i) => {
const { exitCode, stdout, stderr } = results[i]!
const declared = declaredChange(inputFiles(inv.args), stdout, facts)
return {
...inv,
...(declared ? { r4Declared: declared } : {}),
exitCode,
stdout,
stderr,
}
})
return { inputs: inputDigests(files), invocations }
}
function refuseFieldCorpus(): void {
if (process.env.FAUNOSTEO_FIELD_CORPUS) {
throw new Error('refused: FAUNOSTEO_FIELD_CORPUS is set (plan §B.5.2)')
}
}
type OracleRow = Invocation & { r4Declared?: string; exitCode: number }
function committedRows(): OracleRow[] {
return (JSON.parse(readFileSync(OUT, 'utf8')) as { invocations: OracleRow[] })
.invocations
}
// ---------------------------------------------------------------------------
describe('R4 O10 report:compare outputs of the TS CLI', () => {
let cli: CompareSheetIndex
let scenarios: Scenario[]
beforeAll(async () => {
if (MAKE_R4_ORACLES) refuseFieldCorpus()
const sheet = await loadTestSheet()
cli = compareSheetIndex(await bundledSheet())
scenarios = buildScenarios(sheet, cli)
})
it('builds byte-stable scenario PDFs (MAKE_R4_ORACLES writes)', async () => {
const names = scenarios.flatMap((s) => Object.keys(s.documents))
expect(new Set(names).size).toBe(names.length)
for (const scenario of scenarios) {
for (const [name, document] of Object.entries(scenario.documents)) {
const path = resolve(ROOT, scenarioPdf(name))
checkBytesOracle(path, await buildPdf(document))
}
}
const listed = readdirSync(resolve(ROOT, SCENARIO_DIR)).sort()
expect(listed).toEqual(names.map((n) => `${n}.pdf`).sort())
})
it(
'records every invocation (written under MAKE_R4_ORACLES)',
async () => {
const { inputs, invocations } = await recordInvocations(
invocationPlan(scenarios),
)
checkJsonOracle(OUT, {
oracle: 'O10',
cli: `apps/app/${CLI_SCRIPT}`,
run:
'node apps/app/node_modules/tsx/dist/cli.mjs ' +
`${CLI_SCRIPT} <args>, cwd apps/app, INIT_CWD at the repository ` +
'root (pnpm report:compare typed from the root); arguments and ' +
'printed paths are repo-relative',
text:
'stdout and stderr are split on "\\n": joined with "\\n" they ' +
'give the exact text',
sheet: cli.sheet.ref,
inputs,
invocations,
})
},
RUN_TIMEOUT,
)
it('covers the 11 scenarios and marks every A.3 class at least once', () => {
const rows = committedRows()
const titles = new Set(rows.flatMap((r) => r.scenario ?? []))
expect(titles.size).toBe(11)
for (const reason of Object.values(DECLARED)) {
expect(
rows.some((r) => r.r4Declared?.includes(reason)),
reason,
).toBe(true)
}
const control = rows.find(
(r) => r.group === 'fixture' && r.args[0] === V2_FULL_PDF,
)
expect(control).toMatchObject({ exitCode: 0 })
expect(control).not.toHaveProperty('r4Declared')
})
})