feat(supabase): verification retry by effective cause

resource_limit (intrinsic to the frozen input: frame, input or output cap,
SIGXCPU, SIGABRT, SIGSYS) and verdict_refused close the round at the first
occurrence; timeout, unavailable (now also an external SIGKILL) and
lease_expired keep the 2/5/10/20 s retries up to five attempts. pgTAP 021: 89;
sandbox case k closes at attempt 1.
This commit is contained in:
bchanot committed 2026-11-01 23:01:30 +01:00
1 parent c98a9c8f82
commit a5844b9d56
8 files changed
+156 -46

No files matched your search

@@ -301,18 +301,6 @@ async function holdLease(editor: SteppedAccount, d: Draft, attempts: number): Pr
throw new Error('lease not held before the worker claimed')
}
/** Polls a one-value query until it answers `expected`. */
async function waitFor(query: string, expected: string, timeoutMs: number): Promise<void> {
const start = performance.now()
let last = ''
while (performance.now() - start < timeoutMs) {
last = await sql(query)
if (last === expected) return
await new Promise((resolve) => setTimeout(resolve, 200))
}
throw new Error(`expected ${expected}, last ${last}`)
}
/** Registers, uploads and confirms one drawing per size on `sheet`; the ids in order. */
async function putDrawings(editor: SteppedAccount, sheet: string, sizes: number[]): Promise<{ id: string; bytes: Buffer }[]> {
const out: { id: string; bytes: Buffer }[] = []
@@ -705,6 +693,7 @@ describe.skipIf(!WORKER)('sheet verification by the real worker (V1-2.3)', () =>
// Tracked misclassification (owner brief 2026-11-01: caps expected; not fixed): a document naming
// so many registered drawings (600 rows, about 160 bytes each in the header) that the frame header passes 64 KiB is reported resource_limit.
// The cause is the frozen input: definitive at the first attempt, never claimed again.
const many = await newSheet(L)
const rows = (await sql(`insert into public.anatomical_sheet_assets (sheet_id, sha256, byte_size, file_name, created_by, uploaded_at)
select '${uuidOf(many.sheet)}', md5(g::text) || md5(g::text || 'h'), 900, 'h' || g || '.svg', '${uuidOf(L.id)}', now()
@@ -712,11 +701,14 @@ describe.skipIf(!WORKER)('sheet verification by the real worker (V1-2.3)', () =>
expect(rows).toHaveLength(600)
await save(L, many, multiDoc(rows))
const header = uuidOf(await submit(L, many))
await waitFor(`select failure_code || '/' || attempts from public.anatomical_sheet_reviews where id = '${header}'`,
'verification.worker.resource_limit/1', 30_000)
// Withdrawn so that the worker stops retrying it (its next claim is 2 s away).
ok(await rpc('withdraw_sheet_review', { review_id: header, note: 'tracked misclassification' }, L.aal2))
expect((await round(header)).status).toBe('withdrawn')
const closed = await settled(header, 30_000)
const state = (r: Round) => [r.status, r.failure_outcome, r.failure_code, r.attempts]
const final = ['verification_failed', 'unavailable', 'verification.worker.resource_limit', 1]
expect(state(closed.round)).toEqual(final)
expect(await sql(`select not_before is null from public.anatomical_sheet_reviews where id = '${header}'`)).toBe('t')
// Past the first retry delay (2 s) the round is still closed at attempt 1: no second claim.
await new Promise((resolve) => setTimeout(resolve, 3_000))
expect(state(await round(header))).toEqual(final)
}, 5 * CAPS_BUDGET_MS)
})