test(anatomy): R5.1 freeze of the TypeScript pipeline (oracles r5: intermediates, ids, fingerprint matrix) and the D-R5 study scripts; RUST.md §11
This commit is contained in:
1 parent
9c0117a692
commit
60cc0f5fa8
8 files changed
+33630
-1
No files matched your search
@@ -0,0 +1,365 @@
|
||||
/**
|
||||
* R5.1 freeze (plan 2026-10-22 §B.2): what the TypeScript pipeline computes
|
||||
* and emits today, recorded for the Rust pipeline to reproduce. Three
|
||||
* oracles under crates/core/tests/oracles/r5/:
|
||||
*
|
||||
* - pipeline.json: the intermediate values the generated files do not show
|
||||
* (import stats, anomalies, rows, region inferences, change records,
|
||||
* stats before and after, integrity warnings, migration stats) and the
|
||||
* ledger of every generated file (bytes, sha256, the sha256 of the
|
||||
* markdown before Prettier).
|
||||
* - ids.json: the deterministic uuid and slug functions over every name the
|
||||
* pipeline derives an id from, plus edge cases.
|
||||
* - fingerprints.json: the sheet fingerprint of every JSON artifact under
|
||||
* each D-R5 candidate rule, the ES6 printing of every non-integer number
|
||||
* of the two documents, and the RFC 8785 vectors.
|
||||
*
|
||||
* `MAKE_R5_ORACLES=1` writes the files; otherwise the pipeline still matches
|
||||
* its own freeze. The committed generated folders stay the byte oracle
|
||||
* (seed.golden.test.ts); nothing here rewrites them.
|
||||
*/
|
||||
import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'
|
||||
import { dirname, resolve } from 'node:path'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { AnatomicalSheetDocumentSchema } from '@faunosteo/domain'
|
||||
import {
|
||||
buildReportMigrationMap,
|
||||
deterministicUuid,
|
||||
EQUINE_DRAFT_DIR,
|
||||
EQUINE_LEGACY_IMPORT_DIR,
|
||||
MAIN_VIEW_ID,
|
||||
renderArtifacts,
|
||||
renderChangelog,
|
||||
renderDraftArtifacts,
|
||||
renderImportReport,
|
||||
renderReviewReport,
|
||||
sha256Hex,
|
||||
slugify,
|
||||
type DraftResult,
|
||||
type ImportResult,
|
||||
} from '../src'
|
||||
import { runDraft } from './helpers'
|
||||
|
||||
const HERE = dirname(fileURLToPath(import.meta.url))
|
||||
const REPO = resolve(HERE, '../../..')
|
||||
const OUT = resolve(REPO, 'crates/core/tests/oracles/r5')
|
||||
const MAKE = process.env.MAKE_R5_ORACLES === '1'
|
||||
|
||||
type Json = Record<string, unknown>
|
||||
|
||||
/** The uuid and slug inputs the pipeline derives ids from, plus edges. */
|
||||
const UUID_NAMES = [
|
||||
'faunosteo:sheet:equine:v1',
|
||||
'faunosteo:sheet:equine:v2',
|
||||
'faunosteo:asset:0ea5202fa3b95bac167b87c12ee969ab0272e085b60c2d34dbca6202c5f08541',
|
||||
'',
|
||||
'faunosteo:sheet:équin:v1',
|
||||
'faunosteo:asset:\u{1f40e}',
|
||||
]
|
||||
const SLUG_EDGES = [
|
||||
'',
|
||||
' ',
|
||||
'Cheval',
|
||||
'Épaule gauche',
|
||||
'Vertèbres thoraciques T1–T18',
|
||||
'Œil / orbite',
|
||||
'Straße',
|
||||
'Ørsted',
|
||||
"Rein d'un cheval",
|
||||
'ÀÉÎÕÜ ç ñ',
|
||||
'--déjà--',
|
||||
'a'.repeat(70),
|
||||
'é'.repeat(70),
|
||||
'DŽ Dž dž',
|
||||
'İstanbul',
|
||||
'12 côtes',
|
||||
'Ça (combining cedilla)',
|
||||
]
|
||||
|
||||
/** RFC 8785: UTF-16 member order, no whitespace, JSON.stringify scalars. */
|
||||
function canonicalize(value: unknown): string {
|
||||
if (value === null || typeof value !== 'object') return JSON.stringify(value)
|
||||
if (Array.isArray(value)) return `[${value.map(canonicalize).join(',')}]`
|
||||
const object = value as Json
|
||||
const keys = Object.keys(object).sort((a, b) => (a < b ? -1 : a > b ? 1 : 0))
|
||||
const members = keys.map(
|
||||
(k) => `${JSON.stringify(k)}:${canonicalize(object[k])}`,
|
||||
)
|
||||
return `{${members.join(',')}}`
|
||||
}
|
||||
|
||||
/** RFC 8785 §3.2.3 examples: numbers and escapes, then member ordering. */
|
||||
const RFC_VECTORS: Array<[string, string]> = [
|
||||
[
|
||||
'{"numbers": [333333333.33333329, 1E30, 4.50, 2e-3, 0.000000000000000000000000001], ' +
|
||||
'"string": "\\u20ac$\\u000F\\u000aA\'\\u0042\\u0022\\u005c\\\\\\"\\/", "literals": [null, true, false]}',
|
||||
'{"literals":[null,true,false],"numbers":[333333333.3333333,1e+30,4.5,0.002,1e-27],' +
|
||||
'"string":"€$\\u000f\\nA\'B\\"\\\\\\\\\\"/"}',
|
||||
],
|
||||
[
|
||||
'{"\\u20ac": "Euro Sign", "\\r": "Carriage Return", "\\ufb33": "Hebrew Letter Dalet With Dagesh", ' +
|
||||
'"1": "One", "\\ud83d\\ude00": "Emoji: Grinning Face", "\\u0080": "Control", ' +
|
||||
'"\\u00f6": "Latin Small Letter O With Diaeresis"}',
|
||||
'{"\\r":"Carriage Return","1":"One","\u0080":"Control","ö":"Latin Small Letter O With Diaeresis",' +
|
||||
'"€":"Euro Sign","😀":"Emoji: Grinning Face","דּ":"Hebrew Letter Dalet With Dagesh"}',
|
||||
],
|
||||
]
|
||||
|
||||
/** Every number of a JSON value that is not an integer, with its JS text. */
|
||||
function nonIntegers(value: unknown, path: string, out: Json[]): void {
|
||||
if (typeof value === 'number') {
|
||||
if (!Number.isInteger(value)) out.push({ path, text: String(value) })
|
||||
} else if (Array.isArray(value)) {
|
||||
value.forEach((item, i) => nonIntegers(item, `${path}[${i}]`, out))
|
||||
} else if (value && typeof value === 'object') {
|
||||
for (const [k, v] of Object.entries(value as Json))
|
||||
nonIntegers(v, `${path}.${k}`, out)
|
||||
}
|
||||
}
|
||||
|
||||
/** Every `name.fr` / `name.en` of a document, unique, in document order. */
|
||||
function names(value: unknown, out: Set<string>): void {
|
||||
if (Array.isArray(value)) {
|
||||
for (const item of value) names(item, out)
|
||||
} else if (value && typeof value === 'object') {
|
||||
const object = value as Json
|
||||
const name = object.name as Json | undefined
|
||||
if (name && typeof name.fr === 'string') out.add(name.fr)
|
||||
if (name && typeof name.en === 'string') out.add(name.en)
|
||||
for (const v of Object.values(object)) names(v, out)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The input form is what the file holds (defaults omitted); the validated
|
||||
* form is what every reader holds after the schema (defaults filled, Zod
|
||||
* output order). A document.json row carries both.
|
||||
*/
|
||||
function fingerprintRow(file: string, text: string): Json {
|
||||
const value = JSON.parse(text) as unknown
|
||||
const pretty = JSON.stringify(value, null, 2)
|
||||
const row: Json = {
|
||||
file,
|
||||
bytes: Buffer.byteLength(text),
|
||||
today_file_bytes: sha256Hex(text),
|
||||
a_js_pretty_newline: sha256Hex(`${pretty}\n`),
|
||||
a_js_pretty: sha256Hex(pretty),
|
||||
a_js_compact: sha256Hex(JSON.stringify(value)),
|
||||
b_rfc8785: sha256Hex(canonicalize(value)),
|
||||
prettier_equals_js_pretty: text === `${pretty}\n`,
|
||||
}
|
||||
if (file.endsWith('/document.json')) {
|
||||
const validated = AnatomicalSheetDocumentSchema.parse(value)
|
||||
row.a_js_pretty_validated = sha256Hex(JSON.stringify(validated, null, 2))
|
||||
row.b_rfc8785_validated = sha256Hex(canonicalize(validated))
|
||||
row.validated_equals_input = canonicalize(validated) === canonicalize(value)
|
||||
}
|
||||
return row
|
||||
}
|
||||
|
||||
interface Run {
|
||||
result: ImportResult
|
||||
draft: DraftResult
|
||||
legacyFiles: Record<string, string>
|
||||
draftFiles: Record<string, string>
|
||||
}
|
||||
|
||||
async function run(): Promise<Run> {
|
||||
const { result, draft } = await runDraft()
|
||||
return {
|
||||
result,
|
||||
draft,
|
||||
legacyFiles: await renderArtifacts(result),
|
||||
draftFiles: await renderDraftArtifacts(draft, result.anomalies),
|
||||
}
|
||||
}
|
||||
|
||||
function ledger(
|
||||
folder: string,
|
||||
files: Record<string, string>,
|
||||
raw: Record<string, string>,
|
||||
): Json[] {
|
||||
return Object.entries(files).map(([file, text]) => ({
|
||||
folder,
|
||||
file,
|
||||
bytes: Buffer.byteLength(text),
|
||||
sha256: sha256Hex(text),
|
||||
...(file in raw
|
||||
? {
|
||||
before_prettier_sha256: sha256Hex(raw[file]!),
|
||||
prettier_changed: raw[file] !== text,
|
||||
}
|
||||
: {}),
|
||||
}))
|
||||
}
|
||||
|
||||
function pipelineOracle(r: Run): Json {
|
||||
const { result, draft } = r
|
||||
const documentJson = r.draftFiles['document.json']!
|
||||
const migration = buildReportMigrationMap({
|
||||
baseline: draft.baseline as Json,
|
||||
final: draft.document as Json,
|
||||
legacyIdMap: draft.legacyIdMap,
|
||||
records: draft.records,
|
||||
sheet: {
|
||||
sheetId: draft.sheetId,
|
||||
versionId: null,
|
||||
documentSha256: sha256Hex(documentJson),
|
||||
},
|
||||
mainViewId: MAIN_VIEW_ID,
|
||||
})
|
||||
const rawLegacy = { 'IMPORT-REPORT.md': renderImportReport(result) }
|
||||
const rawDraft = {
|
||||
'CHANGELOG.md': renderChangelog(
|
||||
draft,
|
||||
draft.statsBefore,
|
||||
draft.statsAfter,
|
||||
migration.stats,
|
||||
),
|
||||
'REVIEW-REPORT.md': renderReviewReport({
|
||||
...draft,
|
||||
legacyAnomalies: result.anomalies,
|
||||
}),
|
||||
}
|
||||
return {
|
||||
frozen_from: 'packages/anatomy (TypeScript pipeline), R5.1',
|
||||
legacy: {
|
||||
sheetId: result.sheetId,
|
||||
viewBoxes: result.viewBoxes,
|
||||
stats: result.stats,
|
||||
sourceDigests: result.sourceDigests,
|
||||
assets: result.assets,
|
||||
anomalies: result.anomalies,
|
||||
issues: result.issues,
|
||||
regionInferences: result.regionInferences,
|
||||
zoneStats: result.zoneStats,
|
||||
annotationRows: result.annotationRows,
|
||||
},
|
||||
draft: {
|
||||
sheetId: draft.sheetId,
|
||||
changes: draft.changes,
|
||||
assets: draft.assets,
|
||||
statsBefore: draft.statsBefore,
|
||||
statsAfter: draft.statsAfter,
|
||||
issues: draft.issues,
|
||||
migrationStats: migration.stats,
|
||||
records: draft.records,
|
||||
},
|
||||
artifacts: [
|
||||
...ledger(EQUINE_LEGACY_IMPORT_DIR, r.legacyFiles, rawLegacy),
|
||||
...ledger(EQUINE_DRAFT_DIR, r.draftFiles, rawDraft),
|
||||
{
|
||||
folder: EQUINE_DRAFT_DIR,
|
||||
file: `assets/${draft.assets[0]!.file}`,
|
||||
bytes: Buffer.byteLength(draft.cleanedSvg),
|
||||
sha256: sha256Hex(draft.cleanedSvg),
|
||||
},
|
||||
],
|
||||
}
|
||||
}
|
||||
|
||||
function idsOracle(r: Run): Json {
|
||||
const labels = new Set<string>()
|
||||
names(r.result.document, labels)
|
||||
names(r.draft.document, labels)
|
||||
const slugInputs = [...SLUG_EDGES, ...labels]
|
||||
return {
|
||||
uuid: UUID_NAMES.map((name) => ({ name, uuid: deterministicUuid(name) })),
|
||||
slug: slugInputs.map((label) => ({ label, slug: slugify(label) })),
|
||||
}
|
||||
}
|
||||
|
||||
function fingerprintsOracle(r: Run): Json {
|
||||
const jsonFiles = [
|
||||
...Object.entries(r.legacyFiles).map(
|
||||
([f, t]) => [`${EQUINE_LEGACY_IMPORT_DIR}/${f}`, t] as const,
|
||||
),
|
||||
...Object.entries(r.draftFiles).map(
|
||||
([f, t]) => [`${EQUINE_DRAFT_DIR}/${f}`, t] as const,
|
||||
),
|
||||
].filter(([file]) => file.endsWith('.json'))
|
||||
const numbers: Json[] = []
|
||||
nonIntegers(
|
||||
JSON.parse(r.legacyFiles['document.json']!),
|
||||
'legacy-import/document.json',
|
||||
numbers,
|
||||
)
|
||||
nonIntegers(
|
||||
JSON.parse(r.draftFiles['document.json']!),
|
||||
'draft/document.json',
|
||||
numbers,
|
||||
)
|
||||
return {
|
||||
rules: {
|
||||
today_file_bytes:
|
||||
'sha256 of the Prettier-formatted JSON text the pipeline writes (pipeline.ts sheetRef)',
|
||||
a_js_pretty_newline: 'sha256 of JSON.stringify(v, null, 2) + "\\n"',
|
||||
a_js_pretty: 'sha256 of JSON.stringify(v, null, 2)',
|
||||
a_js_compact: 'sha256 of JSON.stringify(v)',
|
||||
b_rfc8785: 'sha256 of the RFC 8785 canonical text',
|
||||
a_js_pretty_validated:
|
||||
'document.json only: JSON.stringify(schema.parse(v), null, 2), the validated form (defaults filled)',
|
||||
b_rfc8785_validated: 'document.json only: RFC 8785 of the validated form',
|
||||
},
|
||||
artifacts: jsonFiles.map(([file, text]) => fingerprintRow(file, text)),
|
||||
numbers,
|
||||
rfc8785_vectors: RFC_VECTORS.map(([input, canonical]) => ({
|
||||
input,
|
||||
canonical,
|
||||
})),
|
||||
}
|
||||
}
|
||||
|
||||
function write(name: string, value: Json): string {
|
||||
const text = `${JSON.stringify(value, null, 2)}\n`
|
||||
if (MAKE) {
|
||||
mkdirSync(OUT, { recursive: true })
|
||||
writeFileSync(resolve(OUT, name), text)
|
||||
}
|
||||
return text
|
||||
}
|
||||
|
||||
function frozen(name: string): string {
|
||||
return readFileSync(resolve(OUT, name), 'utf8')
|
||||
}
|
||||
|
||||
describe('R5.1 oracles (crates/core/tests/oracles/r5)', () => {
|
||||
it('the RFC 8785 canonicaliser passes the RFC vectors', () => {
|
||||
for (const [input, expected] of RFC_VECTORS) {
|
||||
expect(canonicalize(JSON.parse(input))).toBe(expected)
|
||||
}
|
||||
})
|
||||
|
||||
it('pipeline.json: intermediate values and the artifact ledger', async () => {
|
||||
const text = write('pipeline.json', pipelineOracle(await run()))
|
||||
expect(text).toBe(frozen('pipeline.json'))
|
||||
})
|
||||
|
||||
it('ids.json: deterministic uuids and slugs', async () => {
|
||||
const text = write('ids.json', idsOracle(await run()))
|
||||
expect(text).toBe(frozen('ids.json'))
|
||||
})
|
||||
|
||||
it('fingerprints.json: the D-R5 matrix and the number printing', async () => {
|
||||
const text = write('fingerprints.json', fingerprintsOracle(await run()))
|
||||
expect(text).toBe(frozen('fingerprints.json'))
|
||||
})
|
||||
|
||||
it('the ledger digests are the committed files (the golden folders)', async () => {
|
||||
const oracle = JSON.parse(frozen('pipeline.json')) as {
|
||||
artifacts: Array<{ folder: string; file: string; sha256: string }>
|
||||
}
|
||||
for (const row of oracle.artifacts) {
|
||||
const committed = readFileSync(resolve(REPO, row.folder, row.file))
|
||||
expect(sha256Hex(committed), `${row.folder}/${row.file}`).toBe(row.sha256)
|
||||
}
|
||||
})
|
||||
|
||||
it('two runs emit the same bytes for every artifact', async () => {
|
||||
const [first, second] = [await run(), await run()]
|
||||
expect(first.legacyFiles).toEqual(second.legacyFiles)
|
||||
expect(first.draftFiles).toEqual(second.draftFiles)
|
||||
expect(first.draft.cleanedSvg).toBe(second.draft.cleanedSvg)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,70 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* D-R5 study (R5.1): does a PostgreSQL `jsonb` round trip keep each candidate
|
||||
* fingerprint of the sheet document? `jsonb` drops the key order and the
|
||||
* whitespace and re-prints the numbers, so a fingerprint over the file bytes
|
||||
* cannot be recomputed from `anatomical_sheet_versions.content`; a canonical
|
||||
* one (RFC 8785) can if the values survive. PGlite 0.3 = PostgreSQL 17,
|
||||
* no Docker. Read-only on the repository.
|
||||
*
|
||||
* pnpm --filter @faunosteo/supabase exec node scripts/r5-jsonb-fingerprint.mjs
|
||||
*/
|
||||
import { PGlite } from '@electric-sql/pglite'
|
||||
import { createHash } from 'node:crypto'
|
||||
import { readFileSync } from 'node:fs'
|
||||
import { dirname, resolve } from 'node:path'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
|
||||
const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '../../..')
|
||||
const SEED = 'supabase/seed/anatomical-sheets/equine'
|
||||
const FILES = [
|
||||
`${SEED}/draft/document.json`,
|
||||
`${SEED}/legacy-import/document.json`,
|
||||
]
|
||||
|
||||
const sha256 = (text) => createHash('sha256').update(text).digest('hex')
|
||||
|
||||
/** RFC 8785 canonical form (the same function as scripts/r5-fingerprints.mjs). */
|
||||
function canonicalize(value) {
|
||||
if (value === null || typeof value !== 'object') return JSON.stringify(value)
|
||||
if (Array.isArray(value)) return `[${value.map(canonicalize).join(',')}]`
|
||||
const keys = Object.keys(value).sort((a, b) => (a < b ? -1 : a > b ? 1 : 0))
|
||||
const members = keys.map(
|
||||
(k) => `${JSON.stringify(k)}:${canonicalize(value[k])}`,
|
||||
)
|
||||
return `{${members.join(',')}}`
|
||||
}
|
||||
|
||||
async function roundTrip(db, file) {
|
||||
const text = readFileSync(resolve(REPO_ROOT, file), 'utf8')
|
||||
const value = JSON.parse(text)
|
||||
await db.query('delete from t')
|
||||
await db.query('insert into t values ($1::jsonb)', [text])
|
||||
const { rows } = await db.query('select d::text as text from t')
|
||||
const back = JSON.parse(rows[0].text)
|
||||
return {
|
||||
file: file.slice(SEED.length + 1),
|
||||
today: sha256(text),
|
||||
today_after_jsonb: sha256(rows[0].text),
|
||||
a_before: sha256(JSON.stringify(value, null, 2)),
|
||||
a_after_jsonb: sha256(JSON.stringify(back, null, 2)),
|
||||
b_before: sha256(canonicalize(value)),
|
||||
b_after_jsonb: sha256(canonicalize(back)),
|
||||
}
|
||||
}
|
||||
|
||||
const db = new PGlite()
|
||||
await db.query('create table t (d jsonb)')
|
||||
const results = []
|
||||
for (const file of FILES) results.push(await roundTrip(db, file))
|
||||
await db.close()
|
||||
|
||||
console.log(
|
||||
'| File | Today = after jsonb | A = after jsonb | B (RFC 8785) = after jsonb |',
|
||||
)
|
||||
console.log('|---|---|---|---|')
|
||||
for (const r of results) {
|
||||
console.log(
|
||||
`| ${r.file} | ${r.today === r.today_after_jsonb ? 'yes' : 'no'} | ${r.a_before === r.a_after_jsonb ? 'yes' : 'no'} | ${r.b_before === r.b_after_jsonb ? 'yes' : 'no'} (${r.b_before.slice(0, 16)}…) |`,
|
||||
)
|
||||
}
|
||||
Reference in new issue
Block a user