test(anatomy): R5.1 freeze of the TypeScript pipeline (oracles r5: intermediates, ids, fingerprint matrix) and the D-R5 study scripts; RUST.md §11

This commit is contained in:
bchanot committed 2026-10-22 20:18:12 +02:00
1 parent 9c0117a692
commit 60cc0f5fa8
8 files changed
+33630 -1

No files matched your search

@@ -0,0 +1,365 @@
/**
* R5.1 freeze (plan 2026-10-22 §B.2): what the TypeScript pipeline computes
* and emits today, recorded for the Rust pipeline to reproduce. Three
* oracles under crates/core/tests/oracles/r5/:
*
* - pipeline.json: the intermediate values the generated files do not show
* (import stats, anomalies, rows, region inferences, change records,
* stats before and after, integrity warnings, migration stats) and the
* ledger of every generated file (bytes, sha256, the sha256 of the
* markdown before Prettier).
* - ids.json: the deterministic uuid and slug functions over every name the
* pipeline derives an id from, plus edge cases.
* - fingerprints.json: the sheet fingerprint of every JSON artifact under
* each D-R5 candidate rule, the ES6 printing of every non-integer number
* of the two documents, and the RFC 8785 vectors.
*
* `MAKE_R5_ORACLES=1` writes the files; otherwise the pipeline still matches
* its own freeze. The committed generated folders stay the byte oracle
* (seed.golden.test.ts); nothing here rewrites them.
*/
import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'
import { dirname, resolve } from 'node:path'
import { fileURLToPath } from 'node:url'
import { describe, expect, it } from 'vitest'
import { AnatomicalSheetDocumentSchema } from '@faunosteo/domain'
import {
buildReportMigrationMap,
deterministicUuid,
EQUINE_DRAFT_DIR,
EQUINE_LEGACY_IMPORT_DIR,
MAIN_VIEW_ID,
renderArtifacts,
renderChangelog,
renderDraftArtifacts,
renderImportReport,
renderReviewReport,
sha256Hex,
slugify,
type DraftResult,
type ImportResult,
} from '../src'
import { runDraft } from './helpers'
const HERE = dirname(fileURLToPath(import.meta.url))
const REPO = resolve(HERE, '../../..')
const OUT = resolve(REPO, 'crates/core/tests/oracles/r5')
const MAKE = process.env.MAKE_R5_ORACLES === '1'
type Json = Record<string, unknown>
/** The uuid and slug inputs the pipeline derives ids from, plus edges. */
const UUID_NAMES = [
'faunosteo:sheet:equine:v1',
'faunosteo:sheet:equine:v2',
'faunosteo:asset:0ea5202fa3b95bac167b87c12ee969ab0272e085b60c2d34dbca6202c5f08541',
'',
'faunosteo:sheet:équin:v1',
'faunosteo:asset:\u{1f40e}',
]
const SLUG_EDGES = [
'',
' ',
'Cheval',
'Épaule gauche',
'Vertèbres thoraciques T1–T18',
'Œil / orbite',
'Straße',
'Ørsted',
"Rein d'un cheval",
'ÀÉÎÕÜ ç ñ',
'--déjà--',
'a'.repeat(70),
'é'.repeat(70),
'DŽ Dž dž',
'İstanbul',
'12 côtes',
'Ça (combining cedilla)',
]
/** RFC 8785: UTF-16 member order, no whitespace, JSON.stringify scalars. */
function canonicalize(value: unknown): string {
if (value === null || typeof value !== 'object') return JSON.stringify(value)
if (Array.isArray(value)) return `[${value.map(canonicalize).join(',')}]`
const object = value as Json
const keys = Object.keys(object).sort((a, b) => (a < b ? -1 : a > b ? 1 : 0))
const members = keys.map(
(k) => `${JSON.stringify(k)}:${canonicalize(object[k])}`,
)
return `{${members.join(',')}}`
}
/** RFC 8785 §3.2.3 examples: numbers and escapes, then member ordering. */
const RFC_VECTORS: Array<[string, string]> = [
[
'{"numbers": [333333333.33333329, 1E30, 4.50, 2e-3, 0.000000000000000000000000001], ' +
'"string": "\\u20ac$\\u000F\\u000aA\'\\u0042\\u0022\\u005c\\\\\\"\\/", "literals": [null, true, false]}',
'{"literals":[null,true,false],"numbers":[333333333.3333333,1e+30,4.5,0.002,1e-27],' +
'"string":"€$\\u000f\\nA\'B\\"\\\\\\\\\\"/"}',
],
[
'{"\\u20ac": "Euro Sign", "\\r": "Carriage Return", "\\ufb33": "Hebrew Letter Dalet With Dagesh", ' +
'"1": "One", "\\ud83d\\ude00": "Emoji: Grinning Face", "\\u0080": "Control", ' +
'"\\u00f6": "Latin Small Letter O With Diaeresis"}',
'{"\\r":"Carriage Return","1":"One","\u0080":"Control","ö":"Latin Small Letter O With Diaeresis",' +
'"€":"Euro Sign","😀":"Emoji: Grinning Face","דּ":"Hebrew Letter Dalet With Dagesh"}',
],
]
/** Every number of a JSON value that is not an integer, with its JS text. */
function nonIntegers(value: unknown, path: string, out: Json[]): void {
if (typeof value === 'number') {
if (!Number.isInteger(value)) out.push({ path, text: String(value) })
} else if (Array.isArray(value)) {
value.forEach((item, i) => nonIntegers(item, `${path}[${i}]`, out))
} else if (value && typeof value === 'object') {
for (const [k, v] of Object.entries(value as Json))
nonIntegers(v, `${path}.${k}`, out)
}
}
/** Every `name.fr` / `name.en` of a document, unique, in document order. */
function names(value: unknown, out: Set<string>): void {
if (Array.isArray(value)) {
for (const item of value) names(item, out)
} else if (value && typeof value === 'object') {
const object = value as Json
const name = object.name as Json | undefined
if (name && typeof name.fr === 'string') out.add(name.fr)
if (name && typeof name.en === 'string') out.add(name.en)
for (const v of Object.values(object)) names(v, out)
}
}
/**
* The input form is what the file holds (defaults omitted); the validated
* form is what every reader holds after the schema (defaults filled, Zod
* output order). A document.json row carries both.
*/
function fingerprintRow(file: string, text: string): Json {
const value = JSON.parse(text) as unknown
const pretty = JSON.stringify(value, null, 2)
const row: Json = {
file,
bytes: Buffer.byteLength(text),
today_file_bytes: sha256Hex(text),
a_js_pretty_newline: sha256Hex(`${pretty}\n`),
a_js_pretty: sha256Hex(pretty),
a_js_compact: sha256Hex(JSON.stringify(value)),
b_rfc8785: sha256Hex(canonicalize(value)),
prettier_equals_js_pretty: text === `${pretty}\n`,
}
if (file.endsWith('/document.json')) {
const validated = AnatomicalSheetDocumentSchema.parse(value)
row.a_js_pretty_validated = sha256Hex(JSON.stringify(validated, null, 2))
row.b_rfc8785_validated = sha256Hex(canonicalize(validated))
row.validated_equals_input = canonicalize(validated) === canonicalize(value)
}
return row
}
interface Run {
result: ImportResult
draft: DraftResult
legacyFiles: Record<string, string>
draftFiles: Record<string, string>
}
async function run(): Promise<Run> {
const { result, draft } = await runDraft()
return {
result,
draft,
legacyFiles: await renderArtifacts(result),
draftFiles: await renderDraftArtifacts(draft, result.anomalies),
}
}
function ledger(
folder: string,
files: Record<string, string>,
raw: Record<string, string>,
): Json[] {
return Object.entries(files).map(([file, text]) => ({
folder,
file,
bytes: Buffer.byteLength(text),
sha256: sha256Hex(text),
...(file in raw
? {
before_prettier_sha256: sha256Hex(raw[file]!),
prettier_changed: raw[file] !== text,
}
: {}),
}))
}
function pipelineOracle(r: Run): Json {
const { result, draft } = r
const documentJson = r.draftFiles['document.json']!
const migration = buildReportMigrationMap({
baseline: draft.baseline as Json,
final: draft.document as Json,
legacyIdMap: draft.legacyIdMap,
records: draft.records,
sheet: {
sheetId: draft.sheetId,
versionId: null,
documentSha256: sha256Hex(documentJson),
},
mainViewId: MAIN_VIEW_ID,
})
const rawLegacy = { 'IMPORT-REPORT.md': renderImportReport(result) }
const rawDraft = {
'CHANGELOG.md': renderChangelog(
draft,
draft.statsBefore,
draft.statsAfter,
migration.stats,
),
'REVIEW-REPORT.md': renderReviewReport({
...draft,
legacyAnomalies: result.anomalies,
}),
}
return {
frozen_from: 'packages/anatomy (TypeScript pipeline), R5.1',
legacy: {
sheetId: result.sheetId,
viewBoxes: result.viewBoxes,
stats: result.stats,
sourceDigests: result.sourceDigests,
assets: result.assets,
anomalies: result.anomalies,
issues: result.issues,
regionInferences: result.regionInferences,
zoneStats: result.zoneStats,
annotationRows: result.annotationRows,
},
draft: {
sheetId: draft.sheetId,
changes: draft.changes,
assets: draft.assets,
statsBefore: draft.statsBefore,
statsAfter: draft.statsAfter,
issues: draft.issues,
migrationStats: migration.stats,
records: draft.records,
},
artifacts: [
...ledger(EQUINE_LEGACY_IMPORT_DIR, r.legacyFiles, rawLegacy),
...ledger(EQUINE_DRAFT_DIR, r.draftFiles, rawDraft),
{
folder: EQUINE_DRAFT_DIR,
file: `assets/${draft.assets[0]!.file}`,
bytes: Buffer.byteLength(draft.cleanedSvg),
sha256: sha256Hex(draft.cleanedSvg),
},
],
}
}
function idsOracle(r: Run): Json {
const labels = new Set<string>()
names(r.result.document, labels)
names(r.draft.document, labels)
const slugInputs = [...SLUG_EDGES, ...labels]
return {
uuid: UUID_NAMES.map((name) => ({ name, uuid: deterministicUuid(name) })),
slug: slugInputs.map((label) => ({ label, slug: slugify(label) })),
}
}
function fingerprintsOracle(r: Run): Json {
const jsonFiles = [
...Object.entries(r.legacyFiles).map(
([f, t]) => [`${EQUINE_LEGACY_IMPORT_DIR}/${f}`, t] as const,
),
...Object.entries(r.draftFiles).map(
([f, t]) => [`${EQUINE_DRAFT_DIR}/${f}`, t] as const,
),
].filter(([file]) => file.endsWith('.json'))
const numbers: Json[] = []
nonIntegers(
JSON.parse(r.legacyFiles['document.json']!),
'legacy-import/document.json',
numbers,
)
nonIntegers(
JSON.parse(r.draftFiles['document.json']!),
'draft/document.json',
numbers,
)
return {
rules: {
today_file_bytes:
'sha256 of the Prettier-formatted JSON text the pipeline writes (pipeline.ts sheetRef)',
a_js_pretty_newline: 'sha256 of JSON.stringify(v, null, 2) + "\\n"',
a_js_pretty: 'sha256 of JSON.stringify(v, null, 2)',
a_js_compact: 'sha256 of JSON.stringify(v)',
b_rfc8785: 'sha256 of the RFC 8785 canonical text',
a_js_pretty_validated:
'document.json only: JSON.stringify(schema.parse(v), null, 2), the validated form (defaults filled)',
b_rfc8785_validated: 'document.json only: RFC 8785 of the validated form',
},
artifacts: jsonFiles.map(([file, text]) => fingerprintRow(file, text)),
numbers,
rfc8785_vectors: RFC_VECTORS.map(([input, canonical]) => ({
input,
canonical,
})),
}
}
function write(name: string, value: Json): string {
const text = `${JSON.stringify(value, null, 2)}\n`
if (MAKE) {
mkdirSync(OUT, { recursive: true })
writeFileSync(resolve(OUT, name), text)
}
return text
}
function frozen(name: string): string {
return readFileSync(resolve(OUT, name), 'utf8')
}
describe('R5.1 oracles (crates/core/tests/oracles/r5)', () => {
it('the RFC 8785 canonicaliser passes the RFC vectors', () => {
for (const [input, expected] of RFC_VECTORS) {
expect(canonicalize(JSON.parse(input))).toBe(expected)
}
})
it('pipeline.json: intermediate values and the artifact ledger', async () => {
const text = write('pipeline.json', pipelineOracle(await run()))
expect(text).toBe(frozen('pipeline.json'))
})
it('ids.json: deterministic uuids and slugs', async () => {
const text = write('ids.json', idsOracle(await run()))
expect(text).toBe(frozen('ids.json'))
})
it('fingerprints.json: the D-R5 matrix and the number printing', async () => {
const text = write('fingerprints.json', fingerprintsOracle(await run()))
expect(text).toBe(frozen('fingerprints.json'))
})
it('the ledger digests are the committed files (the golden folders)', async () => {
const oracle = JSON.parse(frozen('pipeline.json')) as {
artifacts: Array<{ folder: string; file: string; sha256: string }>
}
for (const row of oracle.artifacts) {
const committed = readFileSync(resolve(REPO, row.folder, row.file))
expect(sha256Hex(committed), `${row.folder}/${row.file}`).toBe(row.sha256)
}
})
it('two runs emit the same bytes for every artifact', async () => {
const [first, second] = [await run(), await run()]
expect(first.legacyFiles).toEqual(second.legacyFiles)
expect(first.draftFiles).toEqual(second.draftFiles)
expect(first.draft.cleanedSvg).toBe(second.draft.cleanedSvg)
})
})
@@ -0,0 +1,70 @@
#!/usr/bin/env node
/**
* D-R5 study (R5.1): does a PostgreSQL `jsonb` round trip keep each candidate
* fingerprint of the sheet document? `jsonb` drops the key order and the
* whitespace and re-prints the numbers, so a fingerprint over the file bytes
* cannot be recomputed from `anatomical_sheet_versions.content`; a canonical
* one (RFC 8785) can if the values survive. PGlite 0.3 = PostgreSQL 17,
* no Docker. Read-only on the repository.
*
* pnpm --filter @faunosteo/supabase exec node scripts/r5-jsonb-fingerprint.mjs
*/
import { PGlite } from '@electric-sql/pglite'
import { createHash } from 'node:crypto'
import { readFileSync } from 'node:fs'
import { dirname, resolve } from 'node:path'
import { fileURLToPath } from 'node:url'
const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '../../..')
const SEED = 'supabase/seed/anatomical-sheets/equine'
const FILES = [
`${SEED}/draft/document.json`,
`${SEED}/legacy-import/document.json`,
]
const sha256 = (text) => createHash('sha256').update(text).digest('hex')
/** RFC 8785 canonical form (the same function as scripts/r5-fingerprints.mjs). */
function canonicalize(value) {
if (value === null || typeof value !== 'object') return JSON.stringify(value)
if (Array.isArray(value)) return `[${value.map(canonicalize).join(',')}]`
const keys = Object.keys(value).sort((a, b) => (a < b ? -1 : a > b ? 1 : 0))
const members = keys.map(
(k) => `${JSON.stringify(k)}:${canonicalize(value[k])}`,
)
return `{${members.join(',')}}`
}
async function roundTrip(db, file) {
const text = readFileSync(resolve(REPO_ROOT, file), 'utf8')
const value = JSON.parse(text)
await db.query('delete from t')
await db.query('insert into t values ($1::jsonb)', [text])
const { rows } = await db.query('select d::text as text from t')
const back = JSON.parse(rows[0].text)
return {
file: file.slice(SEED.length + 1),
today: sha256(text),
today_after_jsonb: sha256(rows[0].text),
a_before: sha256(JSON.stringify(value, null, 2)),
a_after_jsonb: sha256(JSON.stringify(back, null, 2)),
b_before: sha256(canonicalize(value)),
b_after_jsonb: sha256(canonicalize(back)),
}
}
const db = new PGlite()
await db.query('create table t (d jsonb)')
const results = []
for (const file of FILES) results.push(await roundTrip(db, file))
await db.close()
console.log(
'| File | Today = after jsonb | A = after jsonb | B (RFC 8785) = after jsonb |',
)
console.log('|---|---|---|---|')
for (const r of results) {
console.log(
`| ${r.file} | ${r.today === r.today_after_jsonb ? 'yes' : 'no'} | ${r.a_before === r.a_after_jsonb ? 'yes' : 'no'} | ${r.b_before === r.b_after_jsonb ? 'yes' : 'no'} (${r.b_before.slice(0, 16)}…) |`,
)
}