fix(supabase): confirmed uploads gate asset reads (hash claim), seeds-listed test, Dockerfile workspace manifests; importer declares every authored label, real in=out control

This commit is contained in:
bchanot committed 2026-10-09 00:50:36 +02:00
1 parent bbfb8d6203
commit dee4ce43d9
28 files changed
+511 -87

No files matched your search

+2 -2
View File
@@ -19,11 +19,11 @@
"import:equine": "tsx src/cli/import-equine.ts"
},
"dependencies": {
"@faunosteo/domain": "workspace:*",
"@xmldom/xmldom": "^0.9.8"
"@faunosteo/domain": "workspace:*"
},
"devDependencies": {
"@types/node": "^20.19.41",
"@xmldom/xmldom": "^0.9.8",
"eslint": "^9.17.0",
"globals": "^15.14.0",
"prettier": "^3.4.2",
+21 -2
View File
@@ -118,8 +118,27 @@ function mapTrigger(
return { annotation, region: null }
}
function observableName(a: LegacyAnnotation) {
/** Missing labels fall back to the other language, never silently. */
function observableName(a: LegacyAnnotation, out: Anomaly[]) {
const fr = a.nameFr ?? a.nameEn ?? a.id
if (a.nameFr === null) {
out.push(
anomaly(
'annotation.name_fr.missing',
a.id,
`data-anno-name-fr absent : nom FR repris de « ${fr} ».`,
),
)
}
if (a.nameEn === null) {
out.push(
anomaly(
'annotation.name_en.missing',
a.id,
`data-anno-name-en absent : nom EN copié du FR « ${fr} », à traduire.`,
),
)
}
return { fr, en: a.nameEn ?? fr }
}
@@ -128,7 +147,7 @@ function mapObservable(
ctx: AnnotationContext,
out: Anomaly[],
) {
const name = observableName(a)
const name = observableName(a, out)
const zoneId = ctx.zoneByNames.get(zoneNameKey(name.fr, name.en)) ?? null
if (zoneId === null) {
out.push(
@@ -0,0 +1,41 @@
import type { AnatomicalSheetDocumentInput } from '@faunosteo/domain'
import type { Anomaly } from './types'
/**
* Lists every FR/EN text the importer writes itself (no source file holds
* it), derived from the document so the list can never drift from what is
* written. Annotation categories are reported by categories.ts.
*/
export const AUTHORED_LABEL_CODE = 'label.authored'
function authored(subject: string, text: { fr?: string; en?: string }) {
const parts = [
text.fr === undefined ? null : `FR « ${text.fr} »`,
text.en === undefined ? null : `EN « ${text.en} »`,
].filter((part): part is string => part !== null)
const anomaly: Anomaly = {
code: AUTHORED_LABEL_CODE,
level: 'a_valider',
subject,
message: `Libellé écrit par l'importateur : ${parts.join(', ')}.`,
}
return anomaly
}
export function regionLabelSubject(code: string): string {
return `vertebral_config.${code}`
}
export function authoredLabelAnomalies(
document: AnatomicalSheetDocumentInput,
): Anomaly[] {
return [
authored('name', document.name),
authored('description', document.description),
...document.views.map((v) => authored(`views.${v.id}`, v.name)),
...document.vertebral_config.regions.map((r) =>
authored(regionLabelSubject(r.code), r.name),
),
]
}
@@ -60,6 +60,12 @@ function collectIds(nodes: readonly LegacyNode[], into: Set<string>): void {
}
}
/** Locale-independent order: artifacts must not vary with the process. */
function byCodePoint(a: string, b: string): number {
if (a === b) return 0
return a < b ? -1 : 1
}
/** Node ids are unique among siblings only; a finding is (zone, path). */
function crossZoneDuplicates(catalog: LegacyCatalog): Anomaly[] {
const zonesById = new Map<string, number>()
@@ -69,9 +75,7 @@ function crossZoneDuplicates(catalog: LegacyCatalog): Anomaly[] {
for (const id of ids) zonesById.set(id, (zonesById.get(id) ?? 0) + 1)
}
const shared = [...zonesById.entries()].filter(([, n]) => n > 1)
const top = [...shared].sort(
(a, b) => b[1] - a[1] || a[0].localeCompare(b[0]),
)
const top = [...shared].sort((a, b) => b[1] - a[1] || byCodePoint(a[0], b[0]))
return [
{
code: 'finding.id.shared_across_zones',
+23 -15
View File
@@ -169,11 +169,7 @@ function mapCategory(
/** Counts categories as nodes, as the audit does (792 = 78 + 714). */
export function countTree(
nodes: ReadonlyArray<LegacyNode | FindingNodeInput>,
): {
nodes: number
leaves: number
vetReferrals: number
} {
): TreeCounts {
let total = { nodes: 0, leaves: 0, vetReferrals: 0 }
for (const node of nodes) {
const children = node.children ?? []
@@ -201,15 +197,23 @@ function zoneStatsOf(zone: LegacyZone): ZoneStats {
}
}
/** Throws when the output tree does not hold exactly the input nodes. */
function assertNothingDropped(zone: LegacyZone, mapped: ZoneInput): void {
const before = countTree(zone.categories)
const after = countTree(mapped.categories)
if (before.nodes !== after.nodes || before.leaves !== after.leaves) {
throw new Error(
`zone ${zone.id}: ${before.nodes} nodes in, ${after.nodes} out`,
)
}
export interface TreeCounts {
nodes: number
leaves: number
vetReferrals: number
}
/** Throws when the output tree does not hold exactly the input tree. */
export function assertNothingDropped(
zoneId: string,
before: TreeCounts,
after: TreeCounts,
): void {
const keys = ['nodes', 'leaves', 'vetReferrals'] as const
const diff = keys.filter((key) => before[key] !== after[key])
if (diff.length === 0) return
const detail = diff.map((k) => `${k} ${before[k]} in, ${after[k]} out`)
throw new Error(`zone ${zoneId}: ${detail.join('; ')}`)
}
function mapZone(key: string, zone: LegacyZone, state: WalkState): ZoneInput {
@@ -219,7 +223,11 @@ function mapZone(key: string, zone: LegacyZone, state: WalkState): ZoneInput {
name: { fr: zone.name_fr, en: zone.name_en },
categories: zone.categories.map((c) => mapCategory(key, c, state)),
}
assertNothingDropped(zone, mapped)
assertNothingDropped(
key,
countTree(zone.categories),
countTree(mapped.categories),
)
return mapped
}
+17 -3
View File
@@ -17,6 +17,7 @@ import {
type LegacyCatalog,
type MappedCatalog,
} from './catalog'
import { authoredLabelAnomalies } from './authored-labels'
import { catalogAnomalies } from './catalog-checks'
import { mapCategories, type MappedCategories } from './categories'
import { assetIdFromSha256, sha256Hex, sheetIdFor } from './ids'
@@ -181,6 +182,7 @@ function countBy(
}
function buildStats(
sources: ParsedSources,
document: AnatomicalSheetDocumentInput,
mapped: Mapped,
): ImportStats {
@@ -189,7 +191,8 @@ function buildStats(
const sum = (key: 'nodes' | 'leaves' | 'vetReferrals') =>
totals.reduce((acc, zone) => acc + zone[key], 0)
return {
legacyAnnotations: mapped.annotations.length,
legacyAnnotations:
sources.main.annotations.length + sources.head.annotations.length,
annotations: annotations.length,
annotationsByView: {
[MAIN_VIEW_ID]: countBy(annotations, (a) => a.view_id === MAIN_VIEW_ID),
@@ -215,6 +218,14 @@ function buildStats(
}
}
/** Every data-anno-id read from the SVGs must be written once. */
function assertAnnotationsKept(stats: ImportStats): void {
if (stats.legacyAnnotations === stats.annotations) return
throw new Error(
`${stats.legacyAnnotations} annotations read, ${stats.annotations} written`,
)
}
function sourceAnomalies(
sources: ParsedSources,
stats: ImportStats,
@@ -272,8 +283,10 @@ function collectAnomalies(
sources: ParsedSources,
mapped: Mapped,
stats: ImportStats,
document: AnatomicalSheetDocumentInput,
): Anomaly[] {
return [
...authoredLabelAnomalies(document),
...sources.main.anomalies,
...sources.head.anomalies,
...mapped.annotations.flatMap((m) => m.anomalies),
@@ -290,7 +303,8 @@ export function importEquineLegacy(input: EquineLegacyInput): ImportResult {
const mapped = mapSources(sources)
const document = buildDocument(sources, mapped)
const parsed = AnatomicalSheetDocumentSchema.parse(document)
const stats = buildStats(document, mapped)
const stats = buildStats(sources, document, mapped)
assertAnnotationsKept(stats)
return {
document,
legacyIdMap: buildIdMap(sources, mapped),
@@ -310,7 +324,7 @@ export function importEquineLegacy(input: EquineLegacyInput): ImportResult {
),
zoneStats: mapped.catalog.zoneStats,
stats,
anomalies: collectAnomalies(sources, mapped, stats),
anomalies: collectAnomalies(sources, mapped, stats, document),
issues: validateSheetDocument(parsed),
}
}
+51 -7
View File
@@ -1,3 +1,4 @@
import { AUTHORED_LABEL_CODE, regionLabelSubject } from './authored-labels'
import { countTree } from './catalog'
import type { Anomaly, ImportResult } from './types'
@@ -96,9 +97,22 @@ function volumesSection(result: ImportResult): string {
'',
table(['Mesure', 'Valeur'], rows),
'',
`Contrôle entrée = sortie : ${s.legacyAnnotations} annotations lues, ` +
`${s.annotations} écrites ; ${s.nodes} nœuds lus, ${out.nodes} écrits. ` +
`L'importateur s'arrête si un nœud manque.`,
'Contrôle entrée = sortie, compté des deux côtés (sources lues, ' +
"document écrit). L'importateur s'arrête au moindre écart :",
'',
table(
['Contrôle', 'Lu (sources)', 'Écrit (document)'],
[
[
'Annotations (data-anno-id distincts)',
s.legacyAnnotations,
s.annotations,
],
['Nœuds (catégories incluses)', s.nodes, out.nodes],
['Feuilles', s.leaves, out.leaves],
['Orientations vétérinaires', s.vetReferrals, out.vetReferrals],
],
),
].join('\n')
}
@@ -167,11 +181,19 @@ function categoriesSection(result: ImportResult): string {
}
function vertebralSection(result: ImportResult): string {
const authored = new Set(
result.anomalies
.filter((a) => a.code === AUTHORED_LABEL_CODE)
.map((a) => a.subject),
)
const regions = result.document.vertebral_config.regions.map((r) => [
code(r.code),
r.name.fr,
r.name.en,
r.count,
authored.has(regionLabelSubject(r.code))
? 'importateur, à valider'
: 'source',
])
const inferences = result.regionInferences.map((i) => [
code(result.legacyIdMap.annotations[i.annotationId] ?? i.annotationId),
@@ -182,7 +204,13 @@ function vertebralSection(result: ImportResult): string {
return [
'## 5. Configuration vertébrale',
'',
table(['Code', 'Nom FR', 'Nom EN', 'Vertèbres'], regions),
table(
['Code', 'Nom FR', 'Nom EN', 'Vertèbres', 'Origine du libellé'],
regions,
),
'',
'Nombres de vertèbres : `vertebrae.ts`. Noms de région : écrits par ' +
"l'importateur (aucune source ne les porte).",
'',
'Codes de région déduits des libellés (racines `cervic`, `thorac`/`dors`, ' +
'`lombair`/`lumbar`, `sacral`/`sacre`, `caudal`/`coccyg`, FR puis ' +
@@ -238,6 +266,22 @@ function anomaliesSection(result: ImportResult): string {
].join('\n')
}
/** Source keys the document does not carry, and where they live now. */
const NOT_CARRIED_OVER = [
'`schema_version` du catalogue (`2.0`) : version du format historique, ' +
'remplacée par `schema_version: 1` du document canonique ; la valeur ' +
"d'origine est citée dans `attribution.notes`.",
'bloc `species` du catalogue (`equine`, Cheval / Horse) : porté par ' +
'`species_id` du document et par la table `public.species` ; le nom ' +
'de la fiche est écrit séparément (voir anomalies `label.authored`).',
'second élément graphique de chaque annotation (marqueur visuel) : ' +
"reste dans l'asset SVG copié à l'identique, le document ne garde que " +
'la zone de clic.',
'`data-anno-category-name` : répète le slug technique (`articulations`, ' +
"`os`), ce n'est pas un libellé ; le libellé de catégorie vient de la " +
"table de l'importateur (section 4).",
]
function notImportedSection(result: ImportResult): string {
const dropped = result.annotationRows.filter((r) => r.geometry === 'aucune')
const lines =
@@ -251,9 +295,9 @@ function notImportedSection(result: ImportResult): string {
'',
...lines,
'',
'Non repris par construction : le second élément graphique de chaque ' +
"annotation (marqueur visuel), qui reste dans l'asset SVG copié à " +
"l'identique.",
'Non repris par construction :',
'',
...NOT_CARRIED_OVER.map((line) => `- ${line}`),
].join('\n')
}
+20 -1
View File
@@ -1,3 +1,4 @@
import type { FindingNodeInput } from '@faunosteo/domain'
import { EQUINE_SOURCE_PATHS, type ImportResult } from '../src'
import { legacyAnnoIds, runImport } from './helpers'
@@ -51,9 +52,27 @@ describe('equine import volumes', () => {
expect(bilateral).toHaveLength(9)
})
it('keeps 26 zones, 792 nodes, 612 leaves, 41 vet referrals', () => {
it('writes 792 nodes, 612 leaves, 41 vet referrals in the document', () => {
const counts = { nodes: 0, leaves: 0, vet: 0 }
const walk = (nodes: readonly FindingNodeInput[]): void => {
for (const node of nodes) {
counts.nodes += 1
if (!node.children?.length) counts.leaves += 1
if (node.requires_vet_referral === true) counts.vet += 1
walk(node.children ?? [])
}
}
for (const zone of result.document.finding_zones) {
counts.nodes += zone.categories.length
for (const category of zone.categories) walk(category.children)
}
expect(counts).toEqual({ nodes: 792, leaves: 612, vet: 41 })
})
it('reports the input side from the sources: 27 ids, 26 zones, 792/612/41', () => {
expect(result.document.finding_zones).toHaveLength(26)
expect(result.stats).toMatchObject({
legacyAnnotations: 27,
findingZones: 26,
nodes: 792,
leaves: 612,
@@ -0,0 +1,46 @@
import { renderSeedSql, DOCUMENT_DOLLAR_TAG } from '../src'
import { mapAnnotation } from '../src/legacy/annotations'
import { assertNothingDropped } from '../src/legacy/catalog'
import { parseAnnotatedSvg } from '../src/legacy/svg'
import { runImport } from './helpers'
/** Guards that never fire on today's data, proven on synthetic inputs. */
const SVG_WITHOUT_EN = `<?xml version="1.0" encoding="UTF-8"?>
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 10 10">
<circle cx="1" cy="2" r="3" data-anno-id="point-x"
data-anno-name-fr="Garrot" data-anno-category="os"/>
</svg>`
describe('importer guards', () => {
it('throws when a zone loses a vet referral', () => {
const before = { nodes: 10, leaves: 7, vetReferrals: 2 }
const after = { nodes: 10, leaves: 7, vetReferrals: 1 }
expect(() => assertNothingDropped('hip', before, after)).toThrow(
'zone hip: vetReferrals 2 in, 1 out',
)
expect(() => assertNothingDropped('hip', before, before)).not.toThrow()
})
it('copies FR into EN when data-anno-name-en is absent, and says so', () => {
const [legacy] = parseAnnotatedSvg(SVG_WITHOUT_EN, 'x.svg').annotations
const mapped = mapAnnotation(legacy!, {
viewId: 'main',
fileName: 'x.svg',
zoneByNames: new Map(),
})
expect(mapped.annotation.name).toEqual({ fr: 'Garrot', en: 'Garrot' })
expect(mapped.anomalies.map((a) => [a.code, a.level])).toContainEqual([
'annotation.name_en.missing',
'a_valider',
])
})
it('refuses a document that contains the dollar-quote tag', async () => {
const result = await runImport()
const documentJson = JSON.stringify({ note: DOCUMENT_DOLLAR_TAG })
expect(() => renderSeedSql({ result, documentJson })).toThrow(
DOCUMENT_DOLLAR_TAG,
)
})
})
@@ -107,3 +107,22 @@ describe('equine import special cases', () => {
expect(Object.keys(leaf).sort()).toEqual(['id', 'name'])
})
})
describe('importer-authored labels', () => {
it('declares every label written without a source', () => {
const subjects = result.anomalies
.filter((a) => a.code === 'label.authored' && a.level === 'a_valider')
.map((a) => a.subject)
expect(subjects).toEqual([
'name',
'description',
'views.main',
'views.head',
'vertebral_config.C',
'vertebral_config.T',
'vertebral_config.L',
'vertebral_config.S',
'vertebral_config.Cd',
])
})
})
@@ -5,13 +5,15 @@
* Operator tool: needs the service key, which never ships to the browser.
* Reads SUPABASE_URL and SUPABASE_SERVICE_ROLE_KEY from the environment
* (local stack: `pnpm supabase status` prints them). Idempotent: an object
* whose sha256 already exists is skipped, a mismatching upload is refused.
* whose sha256 already exists is kept, a local file whose bytes differ from
* the manifest is refused, and each asset row is confirmed server-side
* (confirm_sheet_asset_upload) so the read policy can trust it.
*
* pnpm assets:upload supabase/seed/anatomical-sheets/equine/v1
*/
import { createHash } from 'node:crypto'
import { readFile } from 'node:fs/promises'
import { join, resolve } from 'node:path'
import { basename, join, resolve } from 'node:path'
import { createClient } from '@supabase/supabase-js'
const BUCKET = 'anatomical-sheet-assets'
@@ -20,6 +22,21 @@ interface AssetEntry {
file: string
sha256: string
byte_size: number
svg_asset_id: string
}
const SHA256_PATTERN = /^[0-9a-f]{64}$/
const UUID_PATTERN = /^[0-9a-f-]{36}$/
/** The manifest is committed data, still: no path separators, hex hash, uuid. */
function assertEntry(entry: AssetEntry): void {
if (basename(entry.file) !== entry.file || entry.file.startsWith('.')) {
throw new Error(`${entry.file}: file must be a plain file name`)
}
if (!SHA256_PATTERN.test(entry.sha256))
throw new Error(`${entry.file}: bad sha256`)
if (!UUID_PATTERN.test(entry.svg_asset_id))
throw new Error(`${entry.file}: bad asset id`)
}
function requireEnv(name: string): string {
@@ -30,7 +47,9 @@ function requireEnv(name: string): string {
async function readManifest(dir: string): Promise<AssetEntry[]> {
const raw = await readFile(join(dir, 'assets.json'), 'utf8')
return JSON.parse(raw) as AssetEntry[]
const entries = JSON.parse(raw) as AssetEntry[]
entries.forEach(assertEntry)
return entries
}
/** The bytes on disk must still match what the seed recorded. */
@@ -63,7 +82,14 @@ async function main(): Promise<void> {
if (error && !/already exists/i.test(error.message)) {
throw new Error(`${key}: ${error.message}`)
}
console.log(`${error ? 'kept' : 'uploaded'} ${key} (${entry.file})`)
// The asset row becomes readable only once the server records the upload.
const confirmed = await client.rpc('confirm_sheet_asset_upload', {
asset_id: entry.svg_asset_id,
})
if (confirmed.error) throw new Error(`${key}: ${confirmed.error.message}`)
console.log(
`${error ? 'kept' : 'uploaded'} ${key} (${entry.file}), confirmed`,
)
}
}
+7
View File
@@ -19,6 +19,7 @@ export type Database = {
id: string
sha256: string
sheet_id: string
uploaded_at: string | null
}
ComputedFields: never
Insert: {
@@ -30,6 +31,7 @@ export type Database = {
id?: string
sha256: string
sheet_id: string
uploaded_at?: string | null
}
Update: {
byte_size?: number
@@ -40,6 +42,7 @@ export type Database = {
id?: string
sha256?: string
sheet_id?: string
uploaded_at?: string | null
}
Relationships: [
{
@@ -369,6 +372,10 @@ export type Database = {
}
Functions: {
archive_anatomical_sheet: { Args: { sheet_id: string }; Returns: string }
confirm_sheet_asset_upload: {
Args: { asset_id: string }
Returns: string
}
create_anatomical_sheet: {
Args: {
description?: string
+1 -1
View File
@@ -17,7 +17,7 @@ export const repoRoot = resolve(here, '../../../..')
export const supabaseDir = join(repoRoot, 'supabase')
export const migrationsDir = join(supabaseDir, 'migrations')
export const pgTapTestsDir = join(supabaseDir, 'tests')
/** Versioned seeds (reference data such as the official sheets); seed.sql itself needs the real auth schema and stays out. */
/** Versioned seeds (reference data such as the official sheets); seed.sql itself needs the real auth schema and stays out. The CLI reads literal paths from config.toml: the seeds test checks every file here is listed. */
export const seedDir = join(supabaseDir, 'seed')
async function listSqlFiles(dir: string): Promise<string[]> {
@@ -29,6 +29,7 @@ const expectedColumns = {
'id',
'sha256',
'sheet_id',
'uploaded_at',
],
anatomical_sheet_versions: [
'change_note',
@@ -106,6 +107,7 @@ type FunctionArgs = {
}
const expectedFunctionArgs = {
archive_anatomical_sheet: ['sheet_id'],
confirm_sheet_asset_upload: ['asset_id'],
create_anatomical_sheet: [
'description',
'document',
+20
View File
@@ -0,0 +1,20 @@
import { readFile } from 'node:fs/promises'
import { join, relative } from 'node:path'
import { listSeeds, seedDir, supabaseDir } from './harness'
/**
* The CLI applies only the literal files of config.toml sql_paths, while the
* harness applies every seed on disk: a seed forgotten in the list would pass
* here and never reach `supabase db reset`.
*/
describe('versioned seeds', () => {
it('are all listed in config.toml [db.seed] sql_paths', async () => {
const config = await readFile(join(supabaseDir, 'config.toml'), 'utf8')
const line = config.match(/^sql_paths\s*=\s*\[(.*)\]/m)?.[1] ?? ''
const listed = [...line.matchAll(/"([^"]+)"/g)].map((m) => m[1])
for (const file of await listSeeds()) {
const path = './' + relative(supabaseDir, join(seedDir, file))
expect(listed, `${path} missing from sql_paths`).toContain(path)
}
})
})