/** * V1-2.3 end to end (contract W6, V1-2 contract §6.4, §6.5, §16.4): the REAL * verification worker (`faunosteo_worker:dev`, `run --jobs sheet-verify`) * against the throwaway stack, through a gateway with Kong's routes. Every * drawing goes through the register / Storage / confirm path of an editor at * aal2; every verdict below is the worker's, never written by the test. The * one SQL stand-in is a lease held for another worker (case d), the only way * to keep a round `verifying` while a live worker runs. * * Run through `pnpm db:sandbox verify-e2e`, which starts the gateway and the * worker and sets SANDBOX_VERIFY_WORKER; skipped otherwise, so `pnpm * db:sandbox test` without a worker stays green. */ import { execFile } from 'node:child_process' import { randomUUID } from 'node:crypto' import { readFile } from 'node:fs/promises' import { join } from 'node:path' import { performance } from 'node:perf_hooks' import { repoRoot } from '../db/harness' import type { SteppedAccount } from './mfa' import { holder, message, ok, sha256 } from './sheet-fixtures' import { rpc, sql, stack } from './stack' const WORKER = process.env.SANDBOX_VERIFY_WORKER ?? '' const HISTORICAL = '503dc301-a8b1-50d4-9e44-0919b2fc2916' const BUCKET = 'anatomical-sheet-assets' const SEED = 'supabase/seed/anatomical-sheets/equine/draft' /** The contract's budget for one verification at the caps (§16.4, W6). */ const CAPS_BUDGET_MS = 120_000 const ROUND_TIMEOUT_MS = 60_000 interface Round { id: string status: string attempts: number failure_outcome: string | null failure_code: string | null lease_token: string | null verified_with: string | null verdict: Verdict | null } interface Verdict { outcome: string review_id: string version_id: string sheet_id: string revision: number fingerprint: string | null failure_code: string | null failures: string[] drawings: { asset_id: string; sha256: string; result: string }[] groups: { key: string; digest: string; coverable: boolean }[] } interface Draft { sheet: string draft: string revision: number } const name = (text: string) => ({ fr: text, en: text }) /** A drawing in the stored form (`serialize::stored_form`), distinct per `tag`. */ const storedDrawing = (tag = randomUUID().replace(/-/g, '')) => Buffer.from( '\n' + ``, ) /** The core's `DRAWING` (crates/core/tests/sheet_verdict.rs), the drawing its `at_caps` geometry is sized on. */ const CORE_DRAWING = Buffer.from( '\n', ) /** One view on `asset`, one point, one zone; every element reviewed (publishable). */ function smallDoc(asset: string, geometry: unknown = { type: 'circle', cx: 1, cy: 1, r: 1 }) { return { schema_version: 2, species_id: 'equine', name: name('Cheval vérifié'), description: name('d'), scope_disclaimer: name('Outil'), attribution: { authors: ['F'], license: null, notes: null }, vertebral_config: { regions: [] }, references: [], categories: [{ id: 'c', name: name('C'), color: null }], views: [{ id: 'main', kind: 'main', name: name('Vue'), svg_asset_id: asset, view_box: null }], annotations: [{ id: 'a1', kind: 'point', view_id: 'main', category_id: 'c', laterality: 'unpaired', geometry, review_status: 'reviewed', name: name('P'), finding_zone_id: 'z1', vertebral_region_code: null, }], finding_zones: [{ id: 'z1', name: name('Z'), categories: [{ id: 'cat', kind: 'dysfunction', name: name('D'), review_status: 'reviewed', children: [{ id: 'n1', name: name('N'), review_status: 'reviewed' }], }], }], } } /** The core's `at_caps` document (crates/core/tests/sheet_verdict.rs): 64 views, 1 024 annotations, 8 192 nodes. */ function atCapsDoc(asset: string) { const doc = smallDoc(asset) as Record doc.description = {} const chain = (levels: number) => { let node: Record = { id: `c${levels}`, name: name('N'), review_status: 'reviewed' } for (let level = levels - 1; level >= 1; level--) node = { id: `c${level}`, name: name('N'), review_status: 'reviewed', children: [node] } return node } const views: unknown[] = [{ id: 'main', kind: 'main', name: name('V'), svg_asset_id: asset, view_box: null }] const annotations: unknown[] = [] for (let i = 1; i < 64; i++) { views.push({ id: `d${i}`, kind: 'detail', name: name('D'), svg_asset_id: null, view_box: { min_x: 0, min_y: 0, width: 5, height: 5 }, parent_view_id: 'main' }) annotations.push({ id: `t${i}`, kind: 'detail_trigger', view_id: 'main', category_id: 'c', laterality: 'unpaired', geometry: { type: 'polygon', points: [[0, 0], [1, 0], [1, 1]] }, review_status: 'reviewed', name: null, opens_view_id: `d${i}` }) } for (let i = 0; i < 1024 - 63; i++) annotations.push({ id: `p${i}`, kind: 'point', view_id: `d${1 + (i % 63)}`, category_id: 'c', laterality: 'left', geometry: { type: 'circle', cx: 1.25, cy: 2.5, r: 0.5 }, review_status: 'reviewed', name: name('Point'), finding_zone_id: `z${i % 64}`, vertebral_region_code: null }) doc.views = views doc.annotations = annotations doc.finding_zones = Array.from({ length: 64 }, (_, z) => ({ id: `z${z}`, name: name('Zone'), categories: [{ id: 'cat', kind: 'dysfunction', name: name('Catégorie'), review_status: 'reviewed', children: [chain(7), ...Array.from({ length: 120 }, (_, n) => ({ id: `l${n}`, name: name('Nœud de lésion'), review_status: 'reviewed' }))] }], })) return doc } const objectExists = async (sha: string) => (await sql(`select count(*) from storage.objects where bucket_id = '${BUCKET}' and name = '${sha}.svg'`)) === '1' /** POST of the object under its content key with `token`; the Storage INSERT policy decides. */ async function putObject(token: string, bytes: Buffer): Promise { const res = await fetch(`${stack.storageUrl}/object/${BUCKET}/${sha256(bytes)}.svg`, { method: 'POST', headers: { Authorization: `Bearer ${token}`, 'Content-Type': 'image/svg+xml' }, body: new Uint8Array(bytes), }) await res.arrayBuffer() return res.status } /** Registers `bytes` for `sheet`, uploads them (unless the shared object already exists), confirms. */ async function putDrawing(editor: SteppedAccount, sheet: string, bytes: Buffer, file = 'd.svg'): Promise { const asset = ok(await rpc('register_sheet_asset', { sheet_id: sheet, sha256: sha256(bytes), byte_size: bytes.length, file_name: file }, editor.aal2)) as string if (!(await objectExists(sha256(bytes)))) expect(await putObject(editor.aal2, bytes)).toBe(200) ok(await rpc('confirm_sheet_asset_upload', { asset_id: asset }, editor.aal2)) return asset } /** A new official sheet by `editor`, its draft at revision 0. */ async function newSheet(editor: SteppedAccount): Promise { const placeholder = smallDoc('00000000-0000-4000-8000-000000000000') ;(placeholder.views[0] as { svg_asset_id: string | null }).svg_asset_id = null const sheet = ok(await rpc('create_official_sheet', { slug: `vf-${randomUUID().slice(0, 12)}`, document: placeholder }, editor.aal2)) as string const draft = await sql(`select id from public.anatomical_sheet_versions where sheet_id = '${sheet}' and status = 'draft'`) return { sheet, draft, revision: 0 } } async function save(editor: SteppedAccount, d: Draft, document: unknown): Promise { ok(await rpc('save_sheet_draft', { version_id: d.draft, expected_revision: d.revision, document, change_note: 'e2e' }, editor.aal2)) d.revision += 1 } const submit = async (editor: SteppedAccount, d: Draft): Promise => ok(await rpc('submit_sheet_review', { version_id: d.draft, expected_revision: d.revision, note: 'e2e' }, editor.aal2)) as string async function round(id: string): Promise { return JSON.parse(await sql(`select row_to_json(r)::text from public.anatomical_sheet_reviews r where id = '${id}'`)) } /** The full row as text: a write of any column shows. */ const roundText = (id: string) => sql(`select row_to_json(r)::text from public.anatomical_sheet_reviews r where id = '${id}'`) /** Polls until the worker moved the round out of `verifying`; resolves with it and the elapsed time. */ async function settled(id: string, timeoutMs = ROUND_TIMEOUT_MS): Promise<{ round: Round; ms: number }> { const start = performance.now() while (performance.now() - start < timeoutMs) { const status = await sql(`select status from public.anatomical_sheet_reviews where id = '${id}'`) if (status !== 'verifying') return { round: await round(id), ms: Math.round(performance.now() - start) } await new Promise((resolve) => setTimeout(resolve, 200)) } const last = await round(id) throw new Error(`round ${id} still verifying after ${timeoutMs} ms (attempts ${last.attempts}, code ${last.failure_code})`) } /** The worker's own measure of a round (`duration_ms` of its verify_* line), or null. */ function workerDuration(review: string): Promise { return new Promise((resolve) => { execFile('docker', ['logs', WORKER], { maxBuffer: 16 * 1024 * 1024 }, (error, stdout) => { if (error) return resolve(null) const line = stdout.split('\n').reverse().map((l) => { try { return JSON.parse(l) as Record } catch { return null } }).find((l) => l?.review_id === review && String(l?.event).startsWith('verify_')) resolve(typeof line?.duration_ms === 'number' ? line.duration_ms : null) }) }) } function expectPassedFor(r: Round, d: Draft, assets: { id: string; sha: string }[]): void { expect(r.status, JSON.stringify(r.verdict?.failures)).toBe('open') const v = r.verdict as Verdict expect(v.outcome).toBe('passed') expect([v.sheet_id, v.version_id, v.review_id, v.revision]).toEqual([d.sheet, d.draft, r.id, d.revision]) expect(v.fingerprint).toMatch(/^[0-9a-f]{64}$/) expect(v.drawings.map((x) => [x.asset_id, x.sha256, x.result])).toEqual(assets.map((a) => [a.id, a.sha, 'ok'])) expect(v.groups.map((g) => g.key)).toContain('general') expect(r.verified_with).toMatch(/^faunosteo-core /) expect([r.lease_token, r.failure_code]).toEqual([null, null]) } function expectFailed(r: Round, code: string): void { expect(r.status).toBe('verification_failed') expect([r.failure_outcome, r.failure_code]).toEqual(['failed', code]) expect(r.verdict?.outcome).toBe('failed') expect(r.verdict?.failures[0]).toBe(code) } const avatarsAndPurges = () => sql(`select (select count(*) || ':' || md5(coalesce(string_agg(row_to_json(a)::text, ',' order by a.id), '')) from public.assets a) || ' ' || (select count(*) || ':' || md5(coalesce(string_agg(row_to_json(q)::text, ',' order by q.id), '')) from private.object_purge_queue q)`) const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/ /** An id read back from the stack, checked before it is interpolated into SQL. */ function uuidOf(value: unknown): string { if (typeof value !== 'string' || !UUID.test(value)) throw new Error(`not a uuid: ${String(value)}`) return value } /** `octet_length(value::jsonb::text)` of a value whose strings need no JSON escape (crates/core/src/domain/sheet/size.rs). */ function jsonbBytes(value: unknown): number { if (value === null) return 4 if (typeof value === 'boolean') return value ? 4 : 5 if (typeof value === 'number') return String(value).length if (typeof value === 'string') return Buffer.byteLength(value) + 2 if (Array.isArray(value)) return 2 + value.reduce((n: number, v) => n + jsonbBytes(v), 0) + 2 * Math.max(0, value.length - 1) const entries = Object.entries(value as Record) return 2 + entries.reduce((n, [k, v]) => n + jsonbBytes(k) + 2 + jsonbBytes(v), 0) + 2 * Math.max(0, entries.length - 1) } /** * A drawing of exactly `size` bytes in the stored form (`serialize::stored_form`): the UTF-8 * prolog and one `` holding one `` whose `d` is padded with `L5 5` segments and * spaces. Canonical because it is what the core writes back byte for byte: ASCII only, the * namespace declared on the root, attributes in a fixed order, every value free of `& < > "` * and of tab or line feed, no comment, no text node, the element self-closed; the pad adds * no element, attribute, `<` or `=` (SVG_CAPS: 20 000 elements, 100 000 `<`, 200 000 `=`). */ function sizedDrawing(size: number, tag: string): Buffer { const head = '\n' + `' const pad = size - head.length - tail.length const text = head + ' L5 5'.repeat(Math.floor(pad / 5)) + ' '.repeat(pad % 5) + tail expect(text.length).toBe(size) return Buffer.from(text, 'ascii') } /** `main` on the first drawing, one detail view per other drawing, each opened by a trigger on `main`. */ function multiDoc(assets: string[], descriptionFr = 'd') { const doc = smallDoc(assets[0]) as Record doc.description = { fr: descriptionFr, en: 'd' } const views = doc.views as unknown[] const annotations = doc.annotations as unknown[] assets.slice(1).forEach((asset, i) => { views.push({ id: `d${i}`, kind: 'detail', name: name('D'), svg_asset_id: asset, view_box: null, parent_view_id: 'main' }) annotations.push({ id: `t${i}`, kind: 'detail_trigger', view_id: 'main', category_id: 'c', laterality: 'unpaired', geometry: { type: 'polygon', points: [[0, 0], [1, 0], [1, 1]] }, review_status: 'reviewed', name: null, opens_view_id: `d${i}` }) }) return doc } /** `multiDoc` whose `octet_length(content::text)` is exactly `bytes`; `fill` makes two such documents differ. */ function documentOfBytes(assets: string[], bytes: number, fill = 'x'): unknown { const missing = bytes - jsonbBytes(multiDoc(assets)) expect(missing).toBeGreaterThanOrEqual(0) return multiDoc(assets, 'd' + fill.repeat(missing)) } /** Submits `d` until its round is `verifying` with `attempts` claims already spent and a lease held for another worker (SQL stand-in). */ async function holdLease(editor: SteppedAccount, d: Draft, attempts: number): Promise<{ review: string; token: string }> { const token = randomUUID() for (let turn = 0; turn < 5; turn++) { const id = uuidOf(await submit(editor, d)) const held = await sql(`update public.anatomical_sheet_reviews set lease_token = '${token}', locked_at = now(), attempts = ${attempts} where id = '${id}' and status = 'verifying' and lease_token is null returning id`) if (held === id) return { review: id, token } await settled(id) ok(await rpc('withdraw_sheet_review', { review_id: id, note: 'raced' }, editor.aal2)) } throw new Error('lease not held before the worker claimed') } /** Registers, uploads and confirms one drawing per size on `sheet`; the ids in order. */ async function putDrawings(editor: SteppedAccount, sheet: string, sizes: number[]): Promise<{ id: string; bytes: Buffer }[]> { const out: { id: string; bytes: Buffer }[] = [] for (const [i, size] of sizes.entries()) { const bytes = sizedDrawing(size, `${i}${randomUUID().replace(/-/g, '')}`) out.push({ id: await putDrawing(editor, sheet, bytes), bytes }) } return out } /** Acknowledges every group of the open round, approves it and publishes (the control of case d). */ async function publishRound(editor: SteppedAccount, reviewer: SteppedAccount, review: string, r: Round): Promise { for (const group of r.verdict?.groups ?? []) ok(await rpc('acknowledge_sheet_review_group', { review_id: review, group_key: group.key, group_digest: group.digest }, reviewer.aal2)) ok(await rpc('decide_sheet_review', { review_id: review, approve: true, note: 'ok' }, reviewer.aal2)) ok(await rpc('publish_sheet_version', { review_id: review }, editor.aal2)) } const verdictBytes = async (review: string) => Number(await sql(`select octet_length(verdict::text) from public.anatomical_sheet_reviews where id = '${review}'`)) const timings: Record = {} describe.skipIf(!WORKER)('sheet verification by the real worker (V1-2.3)', () => { let E: SteppedAccount let V: SteppedAccount let horse: Buffer let horseSha: string let before = '' const planted = { asset: randomUUID(), purge: randomUUID() } let setAside: string[] = [] const copies: string[] = [] // Rounds the cases below leave behind, reused by d and e. let cleanRound = '' let failedRound = '' beforeAll(async () => { E = await holder('vf-editor', 'admin', ['sheet_editor', 'sheet_publisher']) V = await holder('vf-reviewer', null, ['sheet_reviewer']) horse = await readFile(join(repoRoot, SEED, 'assets/horse.svg')) horseSha = sha256(horse) // f: a claimable avatar job and a due purge, which the default mode would // take at once; the verification mode must leave them untouched. await sql(`insert into public.assets (id, owner_id, kind, bucket_id, object_key, quarantine_key, status, declared_mime, declared_size) values ('${planted.asset}', '${E.id}', 'profile_avatar', 'avatars', 'profiles/${E.id}/${planted.asset}/avatar.webp', 'quarantine/${E.id}/${planted.asset}/upload', 'uploaded', 'image/png', 100)`) await sql(`insert into private.object_purge_queue (id, bucket_id, object_name, not_before) values ('${planted.purge}', 'avatars', 'profiles/${E.id}/${planted.purge}/avatar.webp', now() - interval '1 hour')`) before = await avatarsAndPurges() // g: a copy left active by an earlier run is set aside and put back after. setAside = (await sql(`update public.anatomical_sheets set status = 'archived', archived_at = now() where status = 'active' and private.is_historical_working_copy(id) returning id`)) .split('\n').filter(Boolean) }, 120_000) afterAll(async () => { console.log(`verification timings: ${JSON.stringify(timings, null, 1)}`) for (const id of copies) await rpc('archive_anatomical_sheet', { sheet_id: id }, E.aal2).catch(() => {}) if (setAside.length > 0) await sql(`update public.anatomical_sheets set status = 'active', archived_at = null where id in (${setAside.map((id) => `'${id}'`).join(', ')})`).catch(() => {}) await sql(`delete from private.object_purge_queue where id = '${planted.purge}'; delete from public.assets where id = '${planted.asset}'`).catch(() => {}) }) it('a: a clean sheet (the equine draft and its drawing) is verified, the round opens', async () => { const d = await newSheet(E) const asset = await putDrawing(E, d.sheet, horse, 'horse.svg') const text = (await readFile(join(repoRoot, SEED, 'document.json'), 'utf8')) .replaceAll('7f767f74-3b36-51a7-8465-52dd7e2ead4a', asset) await save(E, d, JSON.parse(text)) const review = await submit(E, d) const { round: r, ms } = await settled(review) expectPassedFor(r, d, [{ id: asset, sha: horseSha }]) expect(r.verdict?.groups).toHaveLength(30) expect(r.attempts).toBe(1) timings.a_clean_equine = `${ms} ms (worker ${await workerDuration(review)} ms)` cleanRound = review }) it('b: a drawing outside the stored form, then a missing object: integrity failures, and their exits', async () => { const d = await newSheet(E) // Bytes read with no removal but not written in the stored form (no prolog). const loose = Buffer.from(storedDrawing().toString().split('\n')[1]) const looseAsset = await putDrawing(E, d.sheet, loose) await save(E, d, smallDoc(looseAsset)) const first = await submit(E, d) const failed = await settled(first) expectFailed(failed.round, 'verification.integrity.svg_not_canonical') expect(failed.round.verdict?.drawings[0].result).toBe('svg_not_canonical') timings.b_not_canonical = `${failed.ms} ms` failedRound = first // Exit: upload the drawing again in the stored form, submit again: the round opens; withdraw it. const fixed = storedDrawing() const fixedAsset = await putDrawing(E, d.sheet, fixed) await save(E, d, smallDoc(fixedAsset)) const second = await submit(E, d) expectPassedFor((await settled(second)).round, d, [{ id: fixedAsset, sha: sha256(fixed) }]) ok(await rpc('withdraw_sheet_review', { review_id: second, note: 'e2e' }, E.aal2)) expect((await round(second)).status).toBe('withdrawn') // The object of a confirmed row removed from Storage (operator path): drawing missing. const gone = storedDrawing() const goneAsset = await putDrawing(E, d.sheet, gone) await save(E, d, smallDoc(goneAsset)) const del = await fetch(`${stack.storageUrl}/object/${BUCKET}/${sha256(gone)}.svg`, { method: 'DELETE', headers: { Authorization: `Bearer ${stack.serviceKey}` }, }) await del.arrayBuffer() expect(del.status).toBe(200) expect(await objectExists(sha256(gone))).toBe(false) const third = await submit(E, d) const missing = await settled(third) expectFailed(missing.round, 'verification.integrity.drawing_missing') expect(missing.round.verdict?.drawings[0].result).toBe('missing') timings.b_drawing_missing = `${missing.ms} ms` // Exit: the object written again by the operator path, the same revision submitted again. expect(await putObject(stack.serviceKey, gone)).toBe(200) const fourth = await submit(E, d) expectPassedFor((await settled(fourth)).round, d, [{ id: goneAsset, sha: sha256(gone) }]) }) it('c: business failures (no geometry, out of frame) fail as validation; the fixed draft passes', async () => { const d = await newSheet(E) const bytes = storedDrawing() const asset = await putDrawing(E, d.sheet, bytes) await save(E, d, smallDoc(asset, null)) const undrawn = await settled(await submit(E, d)) expectFailed(undrawn.round, 'verification.validation.no_geometry') expect(undrawn.round.verdict?.fingerprint, 'no integrity error').toMatch(/^[0-9a-f]{64}$/) timings.c_no_geometry = `${undrawn.ms} ms` await save(E, d, smallDoc(asset, { type: 'circle', cx: 50, cy: 50, r: 1 })) const outside = await settled(await submit(E, d)) expectFailed(outside.round, 'verification.validation.out_of_frame') timings.c_out_of_frame = `${outside.ms} ms` await save(E, d, smallDoc(asset)) const review = await submit(E, d) expectPassedFor((await settled(review)).round, d, [{ id: asset, sha: sha256(bytes) }]) }) it('d + e: no decision or publication without a current passed verdict; stale and malformed completions write nothing', async () => { const d = await newSheet(E) const bytes = storedDrawing() const asset = await putDrawing(E, d.sheet, bytes) await save(E, d, smallDoc(asset)) // A round kept `verifying`: its lease held for another worker (SQL stand-in), so // this worker skips it. Should this worker claim it first, the round is withdrawn // and submitted again. let review = '' const token = randomUUID() for (let attempt = 0; attempt < 5 && !review; attempt++) { const id = await submit(E, d) const held = await sql(`update public.anatomical_sheet_reviews set lease_token = '${token}', locked_at = now() where id = '${id}' and status = 'verifying' and lease_token is null returning id`) if (held === id) review = id else { await settled(id) ok(await rpc('withdraw_sheet_review', { review_id: id, note: 'raced' }, E.aal2)) } } expect(review, 'lease held before the worker claimed').not.toBe('') // d: verifying → nothing decided, acknowledged or published. const decide = await rpc('decide_sheet_review', { review_id: review, approve: true, note: 'ok' }, V.aal2) expect(message(decide)).toBe('round.not_open') const ack = await rpc('acknowledge_sheet_review_group', { review_id: review, group_key: 'general', group_digest: '1'.repeat(64) }, V.aal2) expect(message(ack)).toBe('round.not_open') const publish = await rpc('publish_sheet_version', { review_id: review }, E.aal2) expect(message(publish)).toBe('publish.review.missing') // e: with the live token, a malformed verdict and one bound to another round write nothing. const held = await roundText(review) const malformed = await rpc('complete_sheet_verification', { review_id: review, lease_token: token, verdict: { format: 1 } }, stack.serviceKey) expect(malformed.body).toBe('malformed') const foreign = { ...(await round(cleanRound)).verdict } const misbound = await rpc('complete_sheet_verification', { review_id: review, lease_token: token, verdict: foreign }, stack.serviceKey) expect(misbound.body).toBe('stale') expect(await roundText(review)).toBe(held) // The lease released: the worker verifies the round, which opens. await sql(`update public.anatomical_sheet_reviews set lease_token = null, locked_at = null where id = '${review}'`) const opened = await settled(review) expectPassedFor(opened.round, d, [{ id: asset, sha: sha256(bytes) }]) timings.d_after_release = `${opened.ms} ms` // e: after the worker opened it, a completion with a random token, with its own // verdict, answers stale and writes nothing. const openText = await roundText(review) const stale = await rpc('complete_sheet_verification', { review_id: review, lease_token: randomUUID(), verdict: opened.round.verdict }, stack.serviceKey) expect(stale.body).toBe('stale') const staleFail = await rpc('fail_sheet_verification', { review_id: review, lease_token: randomUUID(), code: 'verification.worker.timeout' }, stack.serviceKey) expect(staleFail.body).toBe('stale') expect(await roundText(review)).toBe(openText) // d: a failed round is not decided or published either. const failedDecide = await rpc('decide_sheet_review', { review_id: failedRound, approve: true, note: 'ok' }, V.aal2) expect(message(failedDecide)).toBe('round.not_open') expect(message(await rpc('publish_sheet_version', { review_id: failedRound }, E.aal2))).toBe('publish.review.missing') // Control: the open round's real verdict carries a decision and a publication. expect(message(await rpc('publish_sheet_version', { review_id: review }, E.aal2))).toBe('publish.review.missing') for (const group of opened.round.verdict?.groups ?? []) ok(await rpc('acknowledge_sheet_review_group', { review_id: review, group_key: group.key, group_digest: group.digest }, V.aal2)) ok(await rpc('decide_sheet_review', { review_id: review, approve: true, note: 'ok' }, V.aal2)) ok(await rpc('publish_sheet_version', { review_id: review }, E.aal2)) expect(await sql(`select status from public.anatomical_sheet_versions where id = '${d.draft}'`)).toBe('published') }) it('g: the working copy of the historical sheet shares its drawing and is verified in place', async () => { // The historical drawing confirmed by the operator path, as on DEV (§9.5). const [hAsset, hSha, pending] = (await sql(`select id || ' ' || sha256 || ' ' || (uploaded_at is null)::text from public.anatomical_sheet_assets where sheet_id = '${HISTORICAL}'`)).split(' ') expect(hSha).toBe(horseSha) if (pending === 'true') { if (!(await objectExists(hSha))) expect(await putObject(stack.serviceKey, horse)).toBe(200) ok(await rpc('confirm_sheet_asset_upload', { asset_id: hAsset }, stack.serviceKey)) } const source = await sql(`select id from public.anatomical_sheet_versions where sheet_id = '${HISTORICAL}'`) const sheet = ok(await rpc('create_sheet_working_copy', { source_version_id: source, slug: `wc-${randomUUID().slice(0, 12)}` }, E.aal2)) as string copies.push(sheet) const [copyAsset, copySha] = (await sql(`select id || ' ' || sha256 from public.anatomical_sheet_assets where sheet_id = '${sheet}'`)).split(' ') expect(copyAsset).not.toBe(hAsset) expect(copySha).toBe(hSha) // One object for both rows: nothing was uploaded for the copy. expect(await sql(`select count(*) from storage.objects where bucket_id = '${BUCKET}' and name = '${hSha}.svg'`)).toBe('1') const draft = await sql(`select id from public.anatomical_sheet_versions where sheet_id = '${sheet}' and status = 'draft'`) const d: Draft = { sheet, draft, revision: 0 } const review = await submit(E, d) const { round: r, ms } = await settled(review) expectPassedFor(r, d, [{ id: copyAsset, sha: hSha }]) timings.g_working_copy = `${ms} ms (worker ${await workerDuration(review)} ms)` }) it('h: the at-caps document with one real drawing is verified within the budget', async () => { // The fixture verbatim on the drawing it is sized on (its 0.5 markers are too small // on the horse's 2 073-unit frame: marker_too_small, as the core rules). const d = await newSheet(E) const asset = await putDrawing(E, d.sheet, CORE_DRAWING) const doc = atCapsDoc(asset) await save(E, d, doc) const review = await submit(E, d) const { round: r, ms } = await settled(review, CAPS_BUDGET_MS) expectPassedFor(r, d, [{ id: asset, sha: sha256(CORE_DRAWING) }]) expect(r.verdict?.groups).toHaveLength(1 + 64 + 64) const worker = await workerDuration(review) const bytes = Buffer.byteLength(JSON.stringify(doc)) timings.h_at_caps = `${ms} ms from submission (worker ${worker} ms), document ${bytes} bytes, drawing ${CORE_DRAWING.length} bytes` expect(ms).toBeLessThan(CAPS_BUDGET_MS) }, CAPS_BUDGET_MS + 60_000) it('i: a technical failure is retried no earlier than its delay, the worker serving another round meanwhile', async () => { const d = await newSheet(E) const bytes = storedDrawing() const asset = await putDrawing(E, d.sheet, bytes) await save(E, d, smallDoc(asset)) // Three attempts spent and the lease held for another worker: the failure is the third (10 s delay). const { review, token } = await holdLease(E, d, 3) const failed = await rpc('fail_sheet_verification', { review_id: review, lease_token: token, code: 'verification.worker.timeout' }, stack.serviceKey) expect(failed.body).toBe('retry') // not_before = now() of the failing transaction + 10 s: the failure time in SQL's clock. const failAt = Number(await sql(`select extract(epoch from not_before - interval '10 seconds') from public.anatomical_sheet_reviews where id = '${review}'`)) expect(failAt).toBeGreaterThan(0) // Meanwhile the worker serves another round: a clean sheet opens. const other = await newSheet(E) const otherBytes = storedDrawing() const otherAsset = await putDrawing(E, other.sheet, otherBytes) await save(E, other, smallDoc(otherAsset)) const served = await settled(await submit(E, other)) expectPassedFor(served.round, other, [{ id: otherAsset, sha: sha256(otherBytes) }]) // The first round waits: verifying, no lease, three attempts, delay not over. expect(await sql(`select status || '/' || attempts || '/' || (lease_token is null) || '/' || (clock_timestamp() < not_before) from public.anatomical_sheet_reviews where id = '${review}'`)).toBe('verifying/3/true/true') const retried = await settled(review, 40_000) expectPassedFor(retried.round, d, [{ id: asset, sha: sha256(bytes) }]) expect(retried.round.attempts).toBe(4) const gap = Number(await sql(`select extract(epoch from verified_at) - ${failAt} from public.anatomical_sheet_reviews where id = '${review}'`)) expect(gap).toBeGreaterThanOrEqual(10) timings.i_retry = `claimed and verified ${gap.toFixed(1)} s after the failure (delay 10 s, poll 2 s), other round opened in ${served.ms} ms` }, 90_000) describe('input limits (Q3)', () => { let L: SteppedAccount const MIB = 1024 * 1024 beforeAll(async () => { L = await holder('vf-limits', 'admin', ['sheet_editor', 'sheet_publisher']) }, 60_000) it('j: the largest input the policy allows (document 2 MiB, drawings 2+2+1+1 MiB, base 2 MiB) is verified within the budget', async () => { const d = await newSheet(L) const drawings = await putDrawings(L, d.sheet, [2 * MIB, 2 * MIB, MIB, MIB]) const ids = drawings.map((x) => x.id) const first = documentOfBytes(ids, 2 * MIB, 'x') await save(L, d, first) expect(await sql(`select octet_length(content::text) from public.anatomical_sheet_versions where id = '${d.draft}'`)).toBe(String(2 * MIB)) const review = await submit(L, d) const { round: r, ms } = await settled(review, CAPS_BUDGET_MS) // The verdict lists the drawings in document order: both sides are compared sorted by asset id. const expected = drawings.map((x) => ({ id: x.id, sha: sha256(x.bytes) })).sort((a, b) => a.id.localeCompare(b.id)) const byAsset = (x: Round): Round => ({ ...x, verdict: { ...(x.verdict as Verdict), drawings: [...(x.verdict as Verdict).drawings].sort((a, b) => a.asset_id.localeCompare(b.asset_id)) } }) expectPassedFor(byAsset(r), d, expected) const v = r.verdict as Verdict expect(v.groups.map((g) => g.key)).toEqual(expect.arrayContaining(['general', 'view:main', 'view:d0', 'view:d1', 'view:d2'])) const worker = await workerDuration(review) const size = await verdictBytes(review) expect(size).toBeLessThan(3_145_728) timings.j_at_limits = `${ms} ms from submission (worker ${worker} ms), document ${2 * MIB} bytes, drawings ${drawings.map((x) => x.bytes.length).join('+')} bytes, verdict ${size} bytes` expect(ms).toBeLessThan(CAPS_BUDGET_MS) // Second round: its base is the published 2 MiB document (frame: 2 + 2 + 6 MiB). await publishRound(L, V, review, r) const draft = uuidOf(ok(await rpc('create_sheet_draft', { sheet_id: d.sheet, from_version_id: null, change_note: 'next' }, L.aal2))) const next: Draft = { sheet: d.sheet, draft, revision: Number(await sql(`select revision from public.anatomical_sheet_versions where id = '${draft}'`)) } expect(await sql(`select (source_version_id = '${d.draft}')::text from public.anatomical_sheet_versions where id = '${draft}'`)).toBe('true') await save(L, next, documentOfBytes(ids, 2 * MIB, 'y')) const second = await submit(L, next) const again = await settled(second, CAPS_BUDGET_MS) expectPassedFor(byAsset(again.round), next, expected) const size2 = await verdictBytes(second) expect(size2).toBeLessThan(3_145_728) timings.j_with_base = `${again.ms} ms from submission (worker ${await workerDuration(second)} ms), document and base ${2 * MIB} bytes each, drawings 6 MiB, verdict ${size2} bytes` expect(again.ms).toBeLessThan(CAPS_BUDGET_MS) }, 3 * CAPS_BUDGET_MS) it('k: just beyond the limits: refused before the worker, or a business failure; the tracked misclassification', async () => { // A document of 2 097 153 bytes is refused at save. const d = await newSheet(L) const small = await putDrawing(L, d.sheet, storedDrawing()) const save1 = await rpc('save_sheet_draft', { version_id: d.draft, expected_revision: d.revision, document: documentOfBytes([small], 2 * MIB + 1), change_note: 'e2e' }, L.aal2) expect(message(save1)).toBe('document.too_large') // A drawing of 2 097 153 bytes is refused at registration, and by the bucket whatever the row says. const big = sizedDrawing(2 * MIB + 1, randomUUID().replace(/-/g, '')) expect(message(await rpc('register_sheet_asset', { sheet_id: d.sheet, sha256: sha256(big), byte_size: big.length, file_name: 'big.svg' }, L.aal2))).toBe('invalid_input') // Registered with the policy's size, the object itself over it: the bucket's file size limit refuses it. ok(await rpc('register_sheet_asset', { sheet_id: d.sheet, sha256: sha256(big), byte_size: 2 * MIB, file_name: 'big.svg' }, L.aal2)) for (const token of [L.aal2, stack.serviceKey]) { const res = await fetch(`${stack.storageUrl}/object/${BUCKET}/${sha256(big)}.svg`, { method: 'POST', headers: { Authorization: `Bearer ${token}`, 'Content-Type': 'image/svg+xml' }, body: new Uint8Array(big), }) const body = await res.text() console.log(`bucket upload of 2 097 153 bytes: HTTP ${res.status} ${body}`) expect([res.status, JSON.parse(body).statusCode]).toEqual([400, '413']) } expect(await objectExists(sha256(big))).toBe(false) // Four drawings of 2 MiB (8 MiB, over the 6 MiB total): a business failure, no worker code. const four = await putDrawings(L, d.sheet, [2 * MIB, 2 * MIB, 2 * MIB, 2 * MIB]) await save(L, d, multiDoc(four.map((x) => x.id))) const overTotal = await settled(await submit(L, d), CAPS_BUDGET_MS) expect(overTotal.round.status).toBe('verification_failed') expect([overTotal.round.failure_outcome, overTotal.round.attempts]).toEqual(['failed', 1]) expect(overTotal.round.failure_code).not.toMatch(/^verification\.worker\./) expect(overTotal.round.verdict?.failures).toContain('verification.validation.caps') timings.k_over_total = `${overTotal.ms} ms, codes ${overTotal.round.verdict?.failures.join(',')}` // Five drawings (over the count of four): the same business failure. const five = await putDrawings(L, d.sheet, [300, 301, 302, 303, 304]) await save(L, d, multiDoc(five.map((x) => x.id))) const overCount = await settled(await submit(L, d)) expect(overCount.round.status).toBe('verification_failed') expect([overCount.round.failure_outcome, overCount.round.attempts]).toEqual(['failed', 1]) expect(overCount.round.verdict?.failures).toContain('verification.validation.caps') timings.k_over_count = `${overCount.ms} ms, codes ${overCount.round.verdict?.failures.join(',')}` // Tracked misclassification (owner brief 2026-11-01: caps expected; not fixed): a document naming // so many registered drawings (600 rows, about 160 bytes each in the header) that the frame header passes 64 KiB is reported resource_limit. // The cause is the frozen input: definitive at the first attempt, never claimed again. const many = await newSheet(L) const rows = (await sql(`insert into public.anatomical_sheet_assets (sheet_id, sha256, byte_size, file_name, created_by, uploaded_at) select '${uuidOf(many.sheet)}', md5(g::text) || md5(g::text || 'h'), 900, 'h' || g || '.svg', '${uuidOf(L.id)}', now() from generate_series(1, 600) g returning id`)).split('\n').map(uuidOf) expect(rows).toHaveLength(600) await save(L, many, multiDoc(rows)) const header = uuidOf(await submit(L, many)) const closed = await settled(header, 30_000) const state = (r: Round) => [r.status, r.failure_outcome, r.failure_code, r.attempts] const final = ['verification_failed', 'unavailable', 'verification.worker.resource_limit', 1] expect(state(closed.round)).toEqual(final) expect(await sql(`select not_before is null from public.anatomical_sheet_reviews where id = '${header}'`)).toBe('t') // Past the first retry delay (2 s) the round is still closed at attempt 1: no second claim. await new Promise((resolve) => setTimeout(resolve, 3_000)) expect(state(await round(header))).toEqual(final) }, 5 * CAPS_BUDGET_MS) }) it('f: avatar rows and the purge queue are untouched by the verification worker', async () => { expect(await avatarsAndPurges()).toBe(before) expect(await sql(`select status || ' ' || attempts || ' ' || (locked_at is null)::text from public.assets where id = '${planted.asset}'`)).toBe('uploaded 0 true') expect(await sql(`select attempts || ' ' || (locked_at is null)::text from private.object_purge_queue where id = '${planted.purge}'`)).toBe('0 true') }) })