feat(supabase): retry delay of 2, 5, 10, 20 s for technical verification failures

M6 amended in place (applied only to the throwaway sandbox): a hidden
not_before on the round, set by fail_sheet_verification and honoured by the
claim; RPC signatures unchanged. pgTAP 021 (68), sandbox cases i (delay with
the real worker), j (inputs at the caps, 2 MiB base), k (just beyond).
This commit is contained in:
bchanot committed 2026-11-01 21:09:00 +01:00
1 parent e967ce6736
commit 5adad3709f
8 files changed
+398 -29

No files matched your search

@@ -100,6 +100,7 @@ const expectedColumns = {
'id',
'lease_token',
'locked_at',
'not_before',
'revision',
'round',
'status',
@@ -229,6 +229,112 @@ const avatarsAndPurges = () =>
|| ' ' ||
(select count(*) || ':' || md5(coalesce(string_agg(row_to_json(q)::text, ',' order by q.id), '')) from private.object_purge_queue q)`)
const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/
/** An id read back from the stack, checked before it is interpolated into SQL. */
function uuidOf(value: unknown): string {
if (typeof value !== 'string' || !UUID.test(value)) throw new Error(`not a uuid: ${String(value)}`)
return value
}
/** `octet_length(value::jsonb::text)` of a value whose strings need no JSON escape (crates/core/src/domain/sheet/size.rs). */
function jsonbBytes(value: unknown): number {
if (value === null) return 4
if (typeof value === 'boolean') return value ? 4 : 5
if (typeof value === 'number') return String(value).length
if (typeof value === 'string') return Buffer.byteLength(value) + 2
if (Array.isArray(value)) return 2 + value.reduce((n: number, v) => n + jsonbBytes(v), 0) + 2 * Math.max(0, value.length - 1)
const entries = Object.entries(value as Record<string, unknown>)
return 2 + entries.reduce((n, [k, v]) => n + jsonbBytes(k) + 2 + jsonbBytes(v), 0) + 2 * Math.max(0, entries.length - 1)
}
/**
* A drawing of exactly `size` bytes in the stored form (`serialize::stored_form`): the UTF-8
* prolog and one `<svg>` holding one `<path>` whose `d` is padded with `L5 5` segments and
* spaces. Canonical because it is what the core writes back byte for byte: ASCII only, the
* namespace declared on the root, attributes in a fixed order, every value free of `& < > "`
* and of tab or line feed, no comment, no text node, the element self-closed; the pad adds
* no element, attribute, `<` or `=` (SVG_CAPS: 20 000 elements, 100 000 `<`, 200 000 `=`).
*/
function sizedDrawing(size: number, tag: string): Buffer {
const head = '<?xml version="1.0" encoding="UTF-8"?>\n<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 10 10">' +
`<path id="p${tag}" d="M0 0L5 5`
const tail = '"/></svg>'
const pad = size - head.length - tail.length
const text = head + ' L5 5'.repeat(Math.floor(pad / 5)) + ' '.repeat(pad % 5) + tail
expect(text.length).toBe(size)
return Buffer.from(text, 'ascii')
}
/** `main` on the first drawing, one detail view per other drawing, each opened by a trigger on `main`. */
function multiDoc(assets: string[], descriptionFr = 'd') {
const doc = smallDoc(assets[0]) as Record<string, unknown>
doc.description = { fr: descriptionFr, en: 'd' }
const views = doc.views as unknown[]
const annotations = doc.annotations as unknown[]
assets.slice(1).forEach((asset, i) => {
views.push({ id: `d${i}`, kind: 'detail', name: name('D'), svg_asset_id: asset, view_box: null, parent_view_id: 'main' })
annotations.push({ id: `t${i}`, kind: 'detail_trigger', view_id: 'main', category_id: 'c', laterality: 'unpaired',
geometry: { type: 'polygon', points: [[0, 0], [1, 0], [1, 1]] }, review_status: 'reviewed', name: null,
opens_view_id: `d${i}` })
})
return doc
}
/** `multiDoc` whose `octet_length(content::text)` is exactly `bytes`; `fill` makes two such documents differ. */
function documentOfBytes(assets: string[], bytes: number, fill = 'x'): unknown {
const missing = bytes - jsonbBytes(multiDoc(assets))
expect(missing).toBeGreaterThanOrEqual(0)
return multiDoc(assets, 'd' + fill.repeat(missing))
}
/** Submits `d` until its round is `verifying` with `attempts` claims already spent and a lease held for another worker (SQL stand-in). */
async function holdLease(editor: SteppedAccount, d: Draft, attempts: number): Promise<{ review: string; token: string }> {
const token = randomUUID()
for (let turn = 0; turn < 5; turn++) {
const id = uuidOf(await submit(editor, d))
const held = await sql(`update public.anatomical_sheet_reviews set lease_token = '${token}', locked_at = now(), attempts = ${attempts}
where id = '${id}' and status = 'verifying' and lease_token is null returning id`)
if (held === id) return { review: id, token }
await settled(id)
ok(await rpc('withdraw_sheet_review', { review_id: id, note: 'raced' }, editor.aal2))
}
throw new Error('lease not held before the worker claimed')
}
/** Polls a one-value query until it answers `expected`. */
async function waitFor(query: string, expected: string, timeoutMs: number): Promise<void> {
const start = performance.now()
let last = ''
while (performance.now() - start < timeoutMs) {
last = await sql(query)
if (last === expected) return
await new Promise((resolve) => setTimeout(resolve, 200))
}
throw new Error(`expected ${expected}, last ${last}`)
}
/** Registers, uploads and confirms one drawing per size on `sheet`; the ids in order. */
async function putDrawings(editor: SteppedAccount, sheet: string, sizes: number[]): Promise<{ id: string; bytes: Buffer }[]> {
const out: { id: string; bytes: Buffer }[] = []
for (const [i, size] of sizes.entries()) {
const bytes = sizedDrawing(size, `${i}${randomUUID().replace(/-/g, '')}`)
out.push({ id: await putDrawing(editor, sheet, bytes), bytes })
}
return out
}
/** Acknowledges every group of the open round, approves it and publishes (the control of case d). */
async function publishRound(editor: SteppedAccount, reviewer: SteppedAccount, review: string, r: Round): Promise<void> {
for (const group of r.verdict?.groups ?? [])
ok(await rpc('acknowledge_sheet_review_group',
{ review_id: review, group_key: group.key, group_digest: group.digest }, reviewer.aal2))
ok(await rpc('decide_sheet_review', { review_id: review, approve: true, note: 'ok' }, reviewer.aal2))
ok(await rpc('publish_sheet_version', { review_id: review }, editor.aal2))
}
const verdictBytes = async (review: string) =>
Number(await sql(`select octet_length(verdict::text) from public.anatomical_sheet_reviews where id = '${review}'`))
const timings: Record<string, string> = {}
describe.skipIf(!WORKER)('sheet verification by the real worker (V1-2.3)', () => {
@@ -471,6 +577,149 @@ describe.skipIf(!WORKER)('sheet verification by the real worker (V1-2.3)', () =>
expect(ms).toBeLessThan(CAPS_BUDGET_MS)
}, CAPS_BUDGET_MS + 60_000)
it('i: a technical failure is retried no earlier than its delay, the worker serving another round meanwhile', async () => {
const d = await newSheet(E)
const bytes = storedDrawing()
const asset = await putDrawing(E, d.sheet, bytes)
await save(E, d, smallDoc(asset))
// Three attempts spent and the lease held for another worker: the failure is the third (10 s delay).
const { review, token } = await holdLease(E, d, 3)
const failed = await rpc('fail_sheet_verification',
{ review_id: review, lease_token: token, code: 'verification.worker.timeout' }, stack.serviceKey)
expect(failed.body).toBe('retry')
// not_before = now() of the failing transaction + 10 s: the failure time in SQL's clock.
const failAt = Number(await sql(`select extract(epoch from not_before - interval '10 seconds')
from public.anatomical_sheet_reviews where id = '${review}'`))
expect(failAt).toBeGreaterThan(0)
// Meanwhile the worker serves another round: a clean sheet opens.
const other = await newSheet(E)
const otherBytes = storedDrawing()
const otherAsset = await putDrawing(E, other.sheet, otherBytes)
await save(E, other, smallDoc(otherAsset))
const served = await settled(await submit(E, other))
expectPassedFor(served.round, other, [{ id: otherAsset, sha: sha256(otherBytes) }])
// The first round waits: verifying, no lease, three attempts, delay not over.
expect(await sql(`select status || '/' || attempts || '/' || (lease_token is null) || '/' || (clock_timestamp() < not_before)
from public.anatomical_sheet_reviews where id = '${review}'`)).toBe('verifying/3/true/true')
const retried = await settled(review, 40_000)
expectPassedFor(retried.round, d, [{ id: asset, sha: sha256(bytes) }])
expect(retried.round.attempts).toBe(4)
const gap = Number(await sql(`select extract(epoch from verified_at) - ${failAt}
from public.anatomical_sheet_reviews where id = '${review}'`))
expect(gap).toBeGreaterThanOrEqual(10)
timings.i_retry = `claimed and verified ${gap.toFixed(1)} s after the failure (delay 10 s, poll 2 s), other round opened in ${served.ms} ms`
}, 90_000)
describe('input limits (Q3)', () => {
let L: SteppedAccount
const MIB = 1024 * 1024
beforeAll(async () => {
L = await holder('vf-limits', 'admin', ['sheet_editor', 'sheet_publisher'])
}, 60_000)
it('j: the largest input the policy allows (document 2 MiB, drawings 2+2+1+1 MiB, base 2 MiB) is verified within the budget', async () => {
const d = await newSheet(L)
const drawings = await putDrawings(L, d.sheet, [2 * MIB, 2 * MIB, MIB, MIB])
const ids = drawings.map((x) => x.id)
const first = documentOfBytes(ids, 2 * MIB, 'x')
await save(L, d, first)
expect(await sql(`select octet_length(content::text) from public.anatomical_sheet_versions where id = '${d.draft}'`)).toBe(String(2 * MIB))
const review = await submit(L, d)
const { round: r, ms } = await settled(review, CAPS_BUDGET_MS)
// The verdict lists the drawings in document order: both sides are compared sorted by asset id.
const expected = drawings.map((x) => ({ id: x.id, sha: sha256(x.bytes) })).sort((a, b) => a.id.localeCompare(b.id))
const byAsset = (x: Round): Round => ({ ...x, verdict: { ...(x.verdict as Verdict),
drawings: [...(x.verdict as Verdict).drawings].sort((a, b) => a.asset_id.localeCompare(b.asset_id)) } })
expectPassedFor(byAsset(r), d, expected)
const v = r.verdict as Verdict
expect(v.groups.map((g) => g.key)).toEqual(expect.arrayContaining(['general', 'view:main', 'view:d0', 'view:d1', 'view:d2']))
const worker = await workerDuration(review)
const size = await verdictBytes(review)
expect(size).toBeLessThan(3_145_728)
timings.j_at_limits = `${ms} ms from submission (worker ${worker} ms), document ${2 * MIB} bytes, drawings ${drawings.map((x) => x.bytes.length).join('+')} bytes, verdict ${size} bytes`
expect(ms).toBeLessThan(CAPS_BUDGET_MS)
// Second round: its base is the published 2 MiB document (frame: 2 + 2 + 6 MiB).
await publishRound(L, V, review, r)
const draft = uuidOf(ok(await rpc('create_sheet_draft', { sheet_id: d.sheet, from_version_id: null, change_note: 'next' }, L.aal2)))
const next: Draft = { sheet: d.sheet, draft, revision: Number(await sql(`select revision from public.anatomical_sheet_versions where id = '${draft}'`)) }
expect(await sql(`select (source_version_id = '${d.draft}')::text from public.anatomical_sheet_versions where id = '${draft}'`)).toBe('true')
await save(L, next, documentOfBytes(ids, 2 * MIB, 'y'))
const second = await submit(L, next)
const again = await settled(second, CAPS_BUDGET_MS)
expectPassedFor(byAsset(again.round), next, expected)
const size2 = await verdictBytes(second)
expect(size2).toBeLessThan(3_145_728)
timings.j_with_base = `${again.ms} ms from submission (worker ${await workerDuration(second)} ms), document and base ${2 * MIB} bytes each, drawings 6 MiB, verdict ${size2} bytes`
expect(again.ms).toBeLessThan(CAPS_BUDGET_MS)
}, 3 * CAPS_BUDGET_MS)
it('k: just beyond the limits: refused before the worker, or a business failure; the tracked misclassification', async () => {
// A document of 2 097 153 bytes is refused at save.
const d = await newSheet(L)
const small = await putDrawing(L, d.sheet, storedDrawing())
const save1 = await rpc('save_sheet_draft',
{ version_id: d.draft, expected_revision: d.revision, document: documentOfBytes([small], 2 * MIB + 1), change_note: 'e2e' }, L.aal2)
expect(message(save1)).toBe('document.too_large')
// A drawing of 2 097 153 bytes is refused at registration, and by the bucket whatever the row says.
const big = sizedDrawing(2 * MIB + 1, randomUUID().replace(/-/g, ''))
expect(message(await rpc('register_sheet_asset',
{ sheet_id: d.sheet, sha256: sha256(big), byte_size: big.length, file_name: 'big.svg' }, L.aal2))).toBe('invalid_input')
// Registered with the policy's size, the object itself over it: the bucket's file size limit refuses it.
ok(await rpc('register_sheet_asset',
{ sheet_id: d.sheet, sha256: sha256(big), byte_size: 2 * MIB, file_name: 'big.svg' }, L.aal2))
for (const token of [L.aal2, stack.serviceKey]) {
const res = await fetch(`${stack.storageUrl}/object/${BUCKET}/${sha256(big)}.svg`, {
method: 'POST',
headers: { Authorization: `Bearer ${token}`, 'Content-Type': 'image/svg+xml' },
body: new Uint8Array(big),
})
const body = await res.text()
console.log(`bucket upload of 2 097 153 bytes: HTTP ${res.status} ${body}`)
expect([res.status, JSON.parse(body).statusCode]).toEqual([400, '413'])
}
expect(await objectExists(sha256(big))).toBe(false)
// Four drawings of 2 MiB (8 MiB, over the 6 MiB total): a business failure, no worker code.
const four = await putDrawings(L, d.sheet, [2 * MIB, 2 * MIB, 2 * MIB, 2 * MIB])
await save(L, d, multiDoc(four.map((x) => x.id)))
const overTotal = await settled(await submit(L, d), CAPS_BUDGET_MS)
expect(overTotal.round.status).toBe('verification_failed')
expect([overTotal.round.failure_outcome, overTotal.round.attempts]).toEqual(['failed', 1])
expect(overTotal.round.failure_code).not.toMatch(/^verification\.worker\./)
expect(overTotal.round.verdict?.failures).toContain('verification.validation.caps')
timings.k_over_total = `${overTotal.ms} ms, codes ${overTotal.round.verdict?.failures.join(',')}`
// Five drawings (over the count of four): the same business failure.
const five = await putDrawings(L, d.sheet, [300, 301, 302, 303, 304])
await save(L, d, multiDoc(five.map((x) => x.id)))
const overCount = await settled(await submit(L, d))
expect(overCount.round.status).toBe('verification_failed')
expect([overCount.round.failure_outcome, overCount.round.attempts]).toEqual(['failed', 1])
expect(overCount.round.verdict?.failures).toContain('verification.validation.caps')
timings.k_over_count = `${overCount.ms} ms, codes ${overCount.round.verdict?.failures.join(',')}`
// Tracked misclassification (owner brief 2026-11-01: caps expected; not fixed): a document naming
// so many registered drawings (600 rows, about 160 bytes each in the header) that the frame header passes 64 KiB is reported resource_limit.
const many = await newSheet(L)
const rows = (await sql(`insert into public.anatomical_sheet_assets (sheet_id, sha256, byte_size, file_name, created_by, uploaded_at)
select '${uuidOf(many.sheet)}', md5(g::text) || md5(g::text || 'h'), 900, 'h' || g || '.svg', '${uuidOf(L.id)}', now()
from generate_series(1, 600) g returning id`)).split('\n').map(uuidOf)
expect(rows).toHaveLength(600)
await save(L, many, multiDoc(rows))
const header = uuidOf(await submit(L, many))
await waitFor(`select failure_code || '/' || attempts from public.anatomical_sheet_reviews where id = '${header}'`,
'verification.worker.resource_limit/1', 30_000)
// Withdrawn so that the worker stops retrying it (its next claim is 2 s away).
ok(await rpc('withdraw_sheet_review', { review_id: header, note: 'tracked misclassification' }, L.aal2))
expect((await round(header)).status).toBe('withdrawn')
}, 5 * CAPS_BUDGET_MS)
})
it('f: avatar rows and the purge queue are untouched by the verification worker', async () => {
expect(await avatarsAndPurges()).toBe(before)
expect(await sql(`select status || ' ' || attempts || ' ' || (locked_at is null)::text from public.assets