feat(supabase): verification retry by effective cause

resource_limit (intrinsic to the frozen input: frame, input or output cap,
SIGXCPU, SIGABRT, SIGSYS) and verdict_refused close the round at the first
occurrence; timeout, unavailable (now also an external SIGKILL) and
lease_expired keep the 2/5/10/20 s retries up to five attempts. pgTAP 021: 89;
sandbox case k closes at attempt 1.
This commit is contained in:
bchanot committed 2026-11-01 23:01:30 +01:00
1 parent c98a9c8f82
commit a5844b9d56
8 files changed
+156 -46

No files matched your search

+12 -1
View File
@@ -246,6 +246,17 @@ enum ChildVerdict {
Stop, Stop,
} }
/// The child's own budgets end it with SIGXCPU (CPU, the hard limit sits two seconds
/// after the soft one), SIGABRT (memory) or SIGSYS (seccomp): the same input ends the
/// same way, so `resource_limit` (definitive). Any other signal, SIGKILL first, came
/// from outside (OOM killer, stop): transient `unavailable`.
fn signal_code(signal: i32) -> &'static str {
match signal {
libc::SIGXCPU | libc::SIGABRT | libc::SIGSYS => codes::RESOURCE_LIMIT,
_ => codes::UNAVAILABLE,
}
}
/// Maps every way the child can end to a verdict on the round (§6.5 rows). /// Maps every way the child can end to a verdict on the round (§6.5 rows).
fn classify_child(run: ChildRun, job: &VerifyJob) -> ChildVerdict { fn classify_child(run: ChildRun, job: &VerifyJob) -> ChildVerdict {
let stderr = run.stderr; let stderr = run.stderr;
@@ -262,7 +273,7 @@ fn classify_child(run: ChildRun, job: &VerifyJob) -> ChildVerdict {
.signal(signal) .signal(signal)
.child_stderr(&stderr) .child_stderr(&stderr)
.emit(); .emit();
ChildVerdict::Report(codes::RESOURCE_LIMIT) ChildVerdict::Report(signal_code(signal))
} }
ChildOutcome::OutputOverflow => { ChildOutcome::OutputOverflow => {
log::warn("child_output_overflow").review(review).emit(); log::warn("child_output_overflow").review(review).emit();
+4 -2
View File
@@ -365,7 +365,7 @@ fn tracked_an_object_over_the_policy_is_reported_drawing_missing() {
// Tracked misclassification (owner brief 2026-11-01), not fixed: a document naming enough // Tracked misclassification (owner brief 2026-11-01), not fixed: a document naming enough
// drawings that the frame header passes 64 KiB is refused by the parent's own framing and // drawings that the frame header passes 64 KiB is refused by the parent's own framing and
// reported `resource_limit` (a worker failure, retried) where `caps` would name it. // reported `resource_limit` (a definitive worker failure) where `caps` would name it.
#[test] #[test]
fn tracked_a_header_over_64_kib_is_reported_resource_limit() { fn tracked_a_header_over_64_kib_is_reported_resource_limit() {
let rows = vec![row(); 600]; let rows = vec![row(); 600];
@@ -378,6 +378,7 @@ fn tracked_a_header_over_64_kib_is_reported_resource_limit() {
fake.calls().last().unwrap(), fake.calls().last().unwrap(),
&fail_call(codes::RESOURCE_LIMIT) &fail_call(codes::RESOURCE_LIMIT)
); );
assert_eq!(fake.names().iter().filter(|n| *n == "fail").count(), 1);
} }
#[test] #[test]
@@ -435,7 +436,8 @@ fn every_child_limit_maps_to_its_closed_code() {
ChildOutcome::Signalled { signal: 24 }, ChildOutcome::Signalled { signal: 24 },
codes::RESOURCE_LIMIT, codes::RESOURCE_LIMIT,
), ),
(ChildOutcome::Signalled { signal: 9 }, codes::RESOURCE_LIMIT), (ChildOutcome::Signalled { signal: 9 }, codes::UNAVAILABLE),
(ChildOutcome::Signalled { signal: 15 }, codes::UNAVAILABLE),
(ChildOutcome::Signalled { signal: 6 }, codes::RESOURCE_LIMIT), (ChildOutcome::Signalled { signal: 6 }, codes::RESOURCE_LIMIT),
( (
ChildOutcome::Signalled { signal: 31 }, ChildOutcome::Signalled { signal: 31 },
+26
View File
@@ -4567,3 +4567,29 @@ through `is distinct from`, service role only) fixed in the three round RPCs, pg
sandbox `test-db` 1 880 PASS, `types --check` equal and `verify-e2e` 10/10 on a fresh stack and on the upgrades from sandbox `test-db` 1 880 PASS, `types --check` equal and `verify-e2e` 10/10 on a fresh stack and on the upgrades from
before M6 and from before M4 (the DEV path); `avatar-e2e` 5/5; `test` 87 and `e2e:shell` 86 (rerun on the final M6 before M6 and from before M4 (the DEV path); `avatar-e2e` 5/5; `test` 87 and `e2e:shell` 86 (rerun on the final M6
after the verifier's ECARTS(2): the earlier run predated the fix); DEV untouched. after the verifier's ECARTS(2): the earlier run predated the fix); DEV untouched.
Retry by effective cause (owner brief 2026-11-01, closure C1). SQL holds the code policy
(`private.sheet_worker_final_codes`: `resource_limit`, `verdict_refused` close the round at the first
occurrence, `closed_by` `definitive` in the audit; the others retry 2, 5, 10, 20 s up to five attempts,
`closed_by` `attempt_limit`); the worker maps the cause to the code (`verify_job::classify_child`,
`signal_code`), as implemented:
| Child end | Code | Policy |
|---|---|---|
| frame refused by the parent (header over 64 KiB) | `resource_limit` | definitive |
| `Rejected` (input over the child cap) | `resource_limit` | definitive |
| `OutputOverflow` | `resource_limit` | definitive |
| SIGXCPU (CPU budget; the hard limit is two seconds after the soft one, so the budget shows as SIGXCPU) | `resource_limit` | definitive |
| SIGABRT (memory budget, abort) | `resource_limit` | definitive |
| SIGSYS (seccomp filter) | `resource_limit` | definitive |
| SIGKILL or any other signal (OOM killer, stop: from outside; the parent's own kills are `Timeout` and `OutputOverflow`) | `unavailable` | transient |
| `Timeout` (wall clock) | `timeout` | transient |
| transport, Storage | `unavailable` | transient |
| `Broken`, `Failed` | `unavailable`, then the worker stops | transient |
| verdict answered `malformed` | `verdict_refused` | definitive |
| dead lease | `lease_expired` (set by the claim) | transient, immediate |
What changes for the codes: a SIGKILL is now reported `unavailable` (it was `resource_limit`); the
outcome (`unavailable`) and the code list are otherwise unchanged. A definitive round closes after one
child run instead of five identical ones. Still tracked and open: the header over 64 KiB is reported
`resource_limit` where `caps` would name it (test `tracked_a_header_over_64_kib_is_reported_resource_limit`).
+14 -8
View File
@@ -261,20 +261,26 @@ unchanged.
- Completion: `complete_sheet_verification` answers `passed`, `failed`, - Completion: `complete_sheet_verification` answers `passed`, `failed`,
`stale` (nothing written, logged) or `malformed` (reported `stale` (nothing written, logged) or `malformed` (reported
`verification.worker.verdict_refused`). Child timeout → `verification.worker.verdict_refused`). Child timeout →
`verification.worker.timeout`; a signal, an output overflow or an input over `verification.worker.timeout` (transient); SIGXCPU, SIGABRT or SIGSYS (the
its cap → `verification.worker.resource_limit`; a transport error → child's own CPU, memory and seccomp budgets), an output overflow or an input
`verification.worker.unavailable`; a broken child → `unavailable`, then the over its cap → `verification.worker.resource_limit` (definitive: the frozen
worker stops (as the avatar loop). A dead lease (10 minutes) is claimed input gives the same end every time); any other signal, SIGKILL first (the
again as `verification.worker.lease_expired`; the fifth attempt closes the OOM killer or a stop, from outside) and a transport error →
round `verification_failed` / `unavailable`. `verification.worker.unavailable` (transient); a broken child →
`unavailable`, then the worker stops (as the avatar loop). A dead lease (10
minutes) is claimed again as `verification.worker.lease_expired`. Retry by
cause: `resource_limit` and `verdict_refused` close the round
`verification_failed` / `unavailable` at the first occurrence; the other
codes retry up to the fifth attempt, which closes it the same way.
- Retry delay (owner decision 2026-11-01): a failure reported by - Retry delay (owner decision 2026-11-01): a failure reported by
`fail_sheet_verification` makes the round claimable again after 2 s, 5 s, `fail_sheet_verification` makes the round claimable again after 2 s, 5 s,
10 s, 20 s (after attempts 1 to 4), kept in SQL (`not_before`, hidden from 10 s, 20 s (after attempts 1 to 4), kept in SQL (`not_before`, hidden from
the API roles), so the worker never sleeps for it and the claim of another the API roles), so the worker never sleeps for it and the claim of another
round is not blocked; the effective delay is that delay plus up to one poll round is not blocked; the effective delay is that delay plus up to one poll
interval (`WORKER_POLL_SECONDS`, 2 s). A verdict (failed or not) is never interval (`WORKER_POLL_SECONDS`, 2 s). A verdict (failed or not) is never
retried; a dead lease is claimed again at once. The worker code is the retried; a dead lease is claimed again at once. Only the transient codes
same as before. wait: the delay applies to `timeout`, `unavailable` and `lease_expired`
(SQL decides, `private.sheet_worker_final_codes`).
- Measured (2026-11-01): the at-caps document (64 views, 1 024 annotations, - Measured (2026-11-01): the at-caps document (64 views, 1 024 annotations,
8 192 nodes) verified in 185 ms by the worker on the sandbox, verdict 8 192 nodes) verified in 185 ms by the worker on the sandbox, verdict
24 689 bytes; the equine draft in about 60 ms. 24 689 bytes; the equine draft in about 60 ms.
+18 -8
View File
@@ -890,12 +890,19 @@ attempts below a limit), plus a progressive delay for technical retries only
else answers `stale` or `malformed` and writes nothing. Moves the round to else answers `stale` or `malformed` and writes nothing. Moves the round to
`open` or to `verification_failed` (§6.5) and audits. `open` or to `verification_failed` (§6.5) and audits.
- `fail_sheet_verification(review_id, lease_token, code)`: a technical - `fail_sheet_verification(review_id, lease_token, code)`: a technical
failure; the round stays `verifying` and becomes claimable after a delay failure, closed by cause (2026-11-01, `private.sheet_worker_final_codes`).
A definitive code (`verification.worker.resource_limit`, cause intrinsic to
the frozen input or the contract caps; `verification.worker.verdict_refused`)
closes the round at its first occurrence: `verification_failed`, outcome
`unavailable`, the code kept, no delay, never claimed again. A transient
code (`timeout`, `unavailable`, `lease_expired`): the round stays `verifying`
and becomes claimable after a delay
that grows with the attempt (2 s after attempt 1, 5 s after 2, 10 s after 3, that grows with the attempt (2 s after attempt 1, 5 s after 2, 10 s after 3,
20 s after 4; `private.sheet_verification_retry_delay`), kept in the round's 20 s after 4; `private.sheet_verification_retry_delay`), kept in the round's
`not_before` (a column the API roles cannot read, set here, cleared by the `not_before` (a column the API roles cannot read, set here, cleared by the
claim); the fifth failure becomes `verification_failed` with outcome claim); the fifth failure of a transient code becomes `verification_failed`
`unavailable` and clears it. Nothing is written for a stale call. The delay with outcome `unavailable` and clears it. The audit payload says which rule
closed the round (`closed_by`: `definitive` or `attempt_limit`). Nothing is written for a stale call. The delay
is persisted, so no worker sleeps and any worker honours it; the effective is persisted, so no worker sleeps and any worker honours it; the effective
wait is the delay plus up to one poll interval of the worker (2 s by wait is the delay plus up to one poll interval of the worker (2 s by
default). Only these technical failures wait: a validation or integrity default). Only these technical failures wait: a validation or integrity
@@ -942,7 +949,8 @@ five, verdict text at most 4 MiB; the completion also refuses an unknown verdict
or code, a duplicated group key and a drawing whose result is not `ok`, and accepts a or code, a duplicated group key and a drawing whose result is not `ok`, and accepts a
failed verdict without groups or drawings; worker codes `verification.worker.unavailable`, failed verdict without groups or drawings; worker codes `verification.worker.unavailable`,
`.timeout`, `.resource_limit`, `.lease_expired` (a dead lease), `.verdict_refused` (a `.timeout`, `.resource_limit`, `.lease_expired` (a dead lease), `.verdict_refused` (a
completion answered `malformed`), a closed list in SQL (`private.sheet_worker_codes`). completion answered `malformed`), a closed list in SQL (`private.sheet_worker_codes`); the
definitive ones are `.resource_limit` and `.verdict_refused` (`private.sheet_worker_final_codes`).
These values and codes are the implementer's, for the owner's confirmation. The These values and codes are the implementer's, for the owner's confirmation. The
service role executes these four RPCs besides the two upload ones (pgTAP 018). service role executes these four RPCs besides the two upload ones (pgTAP 018).
@@ -961,11 +969,13 @@ did not pass; the closed codes keep the categories apart.
|---|---|---|---|---| |---|---|---|---|---|
| Business validation | integrity error, caps exceeded, annotation without geometry, annotation outside its view's frame | `verification_failed`, outcome `failed`, codes `verification.validation.*` and the issues | no | fix the draft, submit again | | Business validation | integrity error, caps exceeded, annotation without geometry, annotation outside its view's frame | `verification_failed`, outcome `failed`, codes `verification.validation.*` and the issues | no | fix the draft, submit again |
| Integrity | strict parse refused, drawing missing, size or sha256 mismatch, SVG refused, needing a removal or outside the stored form (§9.2), `object_conflict` | `verification_failed`, outcome `failed`, codes `verification.integrity.*` | no | fix or upload again; `object_conflict` follows §9.3 | | Integrity | strict parse refused, drawing missing, size or sha256 mismatch, SVG refused, needing a removal or outside the stored form (§9.2), `object_conflict` | `verification_failed`, outcome `failed`, codes `verification.integrity.*` | no | fix or upload again; `object_conflict` follows §9.3 |
| Transient worker error | database or Storage unreachable, worker restart | stays `verifying`, code recorded | yes, after 2 s, 5 s, 10 s, 20 s, up to the limit | automatic | | Transient worker error | database or Storage unreachable, broken child, child killed from outside (SIGKILL or any signal not listed below: OOM killer, stop), worker restart | `verification.worker.unavailable`, stays `verifying`, code recorded | yes, after 2 s, 5 s, 10 s, 20 s, up to the limit | automatic |
| Timeout or resource limit | child over its wall-clock or memory limit | stays `verifying`, code recorded | yes, same delays, up to the limit | reduce the document or the drawing if it repeats | | Timeout | child over its wall-clock deadline | `verification.worker.timeout`, stays `verifying`, code recorded | yes, same delays, up to the limit | reduce the document or the drawing if it repeats |
| Resource limit intrinsic to the content | frame refused, input over the child cap, output over its cap, CPU budget (SIGXCPU), memory budget (SIGABRT), seccomp (SIGSYS) | `verification_failed`, outcome `unavailable`, `verification.worker.resource_limit`, closed at the first occurrence | no (the same input ends the same way) | reduce the document or the drawings, submit again |
| Verdict refused | completion answered `malformed` | `verification_failed`, outcome `unavailable`, `verification.worker.verdict_refused`, closed at the first occurrence | no | submit again later; reported to the operator |
| Interruption | worker crash, lease expired | claimable again at once (the lease was the wait); counts as an attempt | yes, up to the limit | automatic | | Interruption | worker crash, lease expired | claimable again at once (the lease was the wait); counts as an attempt | yes, up to the limit | automatic |
| Stale result | completion with an old token, after a withdrawal, or naming another revision | nothing written, logged | no | none needed | | Stale result | completion with an old token, after a withdrawal, or naming another revision | nothing written, logged | no | none needed |
| Definitive failure | attempt limit reached (five; V1-2.3 implements five, for the owner's confirmation) | `verification_failed`, outcome `unavailable`, last code | no | submit again later | | Attempt limit | a transient code at its fifth attempt (five; V1-2.3 implements five, for the owner's confirmation) | `verification_failed`, outcome `unavailable`, last code | no | submit again later |
| Core change | release that changes rules or reader | rounds withdrawn by the release migration | no | submit again | | Core change | release that changes rules or reader | rounds withdrawn by the release migration | no | submit again |
- Withdrawal is always available: the submitter or any editor withdraws a - Withdrawal is always available: the submitter or any editor withdraws a
@@ -1715,7 +1725,7 @@ of the document's JSON text in one form, the same for every layer:
|---|---|---|---| |---|---|---|---|
| M4 `sheet_access_hardening` | V1-2.1 | `anatomical_catalog_grants` and its RPCs; `has_mfa`, `grant_active`, `caller_has`; `can_read_official_catalog` and the reader row predicate; `can_edit_sheet` redefined (F14); AR-1 B on the platform role RPCs and the private write functions (F19, §5.7), F17, F18; F1, F2, F3 (fork closed), F5, F6, F8, F9, F12 (every catalogue table), F13, F15; asset-row immutability trigger (F16); historical-sheet protection trigger (§14.2); reader column grants narrowed; personal creation and visibility changes refused | no row changes | | M4 `sheet_access_hardening` | V1-2.1 | `anatomical_catalog_grants` and its RPCs; `has_mfa`, `grant_active`, `caller_has`; `can_read_official_catalog` and the reader row predicate; `can_edit_sheet` redefined (F14); AR-1 B on the platform role RPCs and the private write functions (F19, §5.7), F17, F18; F1, F2, F3 (fork closed), F5, F6, F8, F9, F12 (every catalogue table), F13, F15; asset-row immutability trigger (F16); historical-sheet protection trigger (§14.2); reader column grants narrowed; personal creation and visibility changes refused | no row changes |
| M5 `sheet_review_workflow` | V1-2.1 | version columns `revision`, `authors`, `source_version_id`, `published_review_id`, `document_fingerprint`; sheet column `working_copy_source_version_id`; revision, author and freeze triggers, the historical-sheet write rule; extended immutability guard (§6.8); rounds, comments, acknowledgments; `create_official_sheet`, `create_sheet_draft` adapted, `save_sheet_draft`, submit, withdraw, comment, acknowledge, decide, publish; `create_sheet_working_copy`; species RPCs; catalogue and version read RPCs; upload: `register_sheet_asset` extended, `confirm_sheet_asset_upload` with the storage check (F7), INSERT policy, bucket limit 2 MiB; mirror removal and status check (§12.2); superseded signatures dropped; the content-size CHECK replaced by the text measure (`octet_length(content::text)`, §12.2) | new columns take their defaults on the equine v1 row (`revision` 0, empty author set); its document, status, number and assets unchanged; the new size CHECK is validated against the existing rows when M5 is applied (the equine document measures 184 105 bytes as `content::text`, measured on the sandbox); bucket limit lowered; no round, no grant | | M5 `sheet_review_workflow` | V1-2.1 | version columns `revision`, `authors`, `source_version_id`, `published_review_id`, `document_fingerprint`; sheet column `working_copy_source_version_id`; revision, author and freeze triggers, the historical-sheet write rule; extended immutability guard (§6.8); rounds, comments, acknowledgments; `create_official_sheet`, `create_sheet_draft` adapted, `save_sheet_draft`, submit, withdraw, comment, acknowledge, decide, publish; `create_sheet_working_copy`; species RPCs; catalogue and version read RPCs; upload: `register_sheet_asset` extended, `confirm_sheet_asset_upload` with the storage check (F7), INSERT policy, bucket limit 2 MiB; mirror removal and status check (§12.2); superseded signatures dropped; the content-size CHECK replaced by the text measure (`octet_length(content::text)`, §12.2) | new columns take their defaults on the equine v1 row (`revision` 0, empty author set); its document, status, number and assets unchanged; the new size CHECK is validated against the existing rows when M5 is applied (the equine document measures 184 105 bytes as `content::text`, measured on the sandbox); bucket limit lowered; no round, no grant |
| M6 `sheet_verification_jobs` | V1-2.3 | worker RPCs (claim, input, complete with the verdict structure check, fail), service role only, with the retry delay (hidden `not_before` column on the round, `private.sheet_verification_retry_delay`: owner decision 2026-11-01, replaces « without backoff »); written as `20261101100000_sheet_verification_jobs.sql`, pgTAP `021`, rehearsed on the sandbox from the state before M4 (M4 to M6 in one `migration up`) and from the state before M6 | none | | M6 `sheet_verification_jobs` | V1-2.3 | worker RPCs (claim, input, complete with the verdict structure check, fail), service role only, with the retry delay (hidden `not_before` column on the round, `private.sheet_verification_retry_delay`: owner decision 2026-11-01, replaces « without backoff ») and the retry by cause (definitive codes close at the first occurrence, `private.sheet_worker_final_codes`); written as `20261101100000_sheet_verification_jobs.sql`, pgTAP `021`, rehearsed on the sandbox from the state before M4 (M4 to M6 in one `migration up`) and from the state before M6 | none |
Each migration ships its pgTAP file (allowed and denied cases), regenerated Each migration ships its pgTAP file (allowed and denied cases), regenerated
`database.types.ts`, a `db:sandbox up upgrade` run from the DEV state (with a `database.types.ts`, a `db:sandbox up upgrade` run from the DEV state (with a
@@ -301,18 +301,6 @@ async function holdLease(editor: SteppedAccount, d: Draft, attempts: number): Pr
throw new Error('lease not held before the worker claimed') throw new Error('lease not held before the worker claimed')
} }
/** Polls a one-value query until it answers `expected`. */
async function waitFor(query: string, expected: string, timeoutMs: number): Promise<void> {
const start = performance.now()
let last = ''
while (performance.now() - start < timeoutMs) {
last = await sql(query)
if (last === expected) return
await new Promise((resolve) => setTimeout(resolve, 200))
}
throw new Error(`expected ${expected}, last ${last}`)
}
/** Registers, uploads and confirms one drawing per size on `sheet`; the ids in order. */ /** Registers, uploads and confirms one drawing per size on `sheet`; the ids in order. */
async function putDrawings(editor: SteppedAccount, sheet: string, sizes: number[]): Promise<{ id: string; bytes: Buffer }[]> { async function putDrawings(editor: SteppedAccount, sheet: string, sizes: number[]): Promise<{ id: string; bytes: Buffer }[]> {
const out: { id: string; bytes: Buffer }[] = [] const out: { id: string; bytes: Buffer }[] = []
@@ -705,6 +693,7 @@ describe.skipIf(!WORKER)('sheet verification by the real worker (V1-2.3)', () =>
// Tracked misclassification (owner brief 2026-11-01: caps expected; not fixed): a document naming // Tracked misclassification (owner brief 2026-11-01: caps expected; not fixed): a document naming
// so many registered drawings (600 rows, about 160 bytes each in the header) that the frame header passes 64 KiB is reported resource_limit. // so many registered drawings (600 rows, about 160 bytes each in the header) that the frame header passes 64 KiB is reported resource_limit.
// The cause is the frozen input: definitive at the first attempt, never claimed again.
const many = await newSheet(L) const many = await newSheet(L)
const rows = (await sql(`insert into public.anatomical_sheet_assets (sheet_id, sha256, byte_size, file_name, created_by, uploaded_at) const rows = (await sql(`insert into public.anatomical_sheet_assets (sheet_id, sha256, byte_size, file_name, created_by, uploaded_at)
select '${uuidOf(many.sheet)}', md5(g::text) || md5(g::text || 'h'), 900, 'h' || g || '.svg', '${uuidOf(L.id)}', now() select '${uuidOf(many.sheet)}', md5(g::text) || md5(g::text || 'h'), 900, 'h' || g || '.svg', '${uuidOf(L.id)}', now()
@@ -712,11 +701,14 @@ describe.skipIf(!WORKER)('sheet verification by the real worker (V1-2.3)', () =>
expect(rows).toHaveLength(600) expect(rows).toHaveLength(600)
await save(L, many, multiDoc(rows)) await save(L, many, multiDoc(rows))
const header = uuidOf(await submit(L, many)) const header = uuidOf(await submit(L, many))
await waitFor(`select failure_code || '/' || attempts from public.anatomical_sheet_reviews where id = '${header}'`, const closed = await settled(header, 30_000)
'verification.worker.resource_limit/1', 30_000) const state = (r: Round) => [r.status, r.failure_outcome, r.failure_code, r.attempts]
// Withdrawn so that the worker stops retrying it (its next claim is 2 s away). const final = ['verification_failed', 'unavailable', 'verification.worker.resource_limit', 1]
ok(await rpc('withdraw_sheet_review', { review_id: header, note: 'tracked misclassification' }, L.aal2)) expect(state(closed.round)).toEqual(final)
expect((await round(header)).status).toBe('withdrawn') expect(await sql(`select not_before is null from public.anatomical_sheet_reviews where id = '${header}'`)).toBe('t')
// Past the first retry delay (2 s) the round is still closed at attempt 1: no second claim.
await new Promise((resolve) => setTimeout(resolve, 3_000))
expect(state(await round(header))).toEqual(final)
}, 5 * CAPS_BUDGET_MS) }, 5 * CAPS_BUDGET_MS)
}) })
@@ -13,7 +13,9 @@
-- the drawings the document references; -- the drawings the document references;
-- * complete_sheet_verification: the verdict, checked for its binding and -- * complete_sheet_verification: the verdict, checked for its binding and
-- its structure; `stale` or `malformed` write nothing; -- its structure; `stale` or `malformed` write nothing;
-- * fail_sheet_verification: a technical failure, from a closed list. -- * fail_sheet_verification: a technical failure, from a closed list;
-- transient codes retry after a delay up to the attempt limit, the
-- definitive codes (`sheet_worker_final_codes`) close the round at once.
-- The verdict itself is the core's (`compute_verdict`, crates/core); SQL -- The verdict itself is the core's (`compute_verdict`, crates/core); SQL
-- checks only that it answers this round and covers this document. -- checks only that it answers this round and covers this document.
@@ -67,7 +69,8 @@ returns text[] language sql immutable set search_path = '' as $$
$$; $$;
-- The worker's technical failure codes (§6.5 transient, timeout, resource, -- The worker's technical failure codes (§6.5 transient, timeout, resource,
-- interruption rows; crates/worker `verify_job`). -- interruption rows; crates/worker `verify_job`). The worker maps the cause
-- to a code; the retry policy of each code is `sheet_worker_final_codes`.
create or replace function private.sheet_worker_codes() create or replace function private.sheet_worker_codes()
returns text[] language sql immutable set search_path = '' as $$ returns text[] language sql immutable set search_path = '' as $$
select array[ select array[
@@ -77,6 +80,15 @@ returns text[] language sql immutable set search_path = '' as $$
] ]
$$; $$;
-- The codes whose cause is fixed by the frozen input or the contract caps
-- (resource_limit) or by a verdict the SQL refuses (verdict_refused): a retry
-- would give the same answer, so the round closes at the first occurrence.
-- The others (timeout, unavailable, lease_expired) are transient.
create or replace function private.sheet_worker_final_codes()
returns text[] language sql immutable set search_path = '' as $$
select array['verification.worker.resource_limit', 'verification.worker.verdict_refused']
$$;
-- The service role only (the worker's key); anything else is refused. -- The service role only (the worker's key); anything else is refused.
create or replace function private.require_service_role() create or replace function private.require_service_role()
returns void language plpgsql stable set search_path = '' as $$ returns void language plpgsql stable set search_path = '' as $$
@@ -367,9 +379,10 @@ begin
end; end;
$$; $$;
-- A technical failure of a leased round (§6.5): the round stays `verifying` -- A technical failure of a leased round (§6.5): a transient code leaves the
-- and claimable after its retry delay, its code recorded, until the attempt -- round `verifying` and claimable after its retry delay, its code recorded,
-- limit closes it `verification_failed` / `unavailable`. Answers `retry`, `unavailable` or -- until the attempt limit closes it `verification_failed` / `unavailable`; a
-- definitive code closes it at once. Answers `retry`, `unavailable` or
-- `stale` (nothing written). -- `stale` (nothing written).
create or replace function public.fail_sheet_verification(review_id uuid, lease_token uuid, code text) create or replace function public.fail_sheet_verification(review_id uuid, lease_token uuid, code text)
returns text returns text
@@ -382,6 +395,7 @@ as $$
declare declare
v_review public.anatomical_sheet_reviews%rowtype; v_review public.anatomical_sheet_reviews%rowtype;
v_final boolean; v_final boolean;
v_definitive boolean;
begin begin
perform private.require_service_role(); perform private.require_service_role();
if code is null or not (code = any (private.sheet_worker_codes())) then if code is null or not (code = any (private.sheet_worker_codes())) then
@@ -392,7 +406,9 @@ begin
or lease_token is null or v_review.lease_token is distinct from lease_token then or lease_token is null or v_review.lease_token is distinct from lease_token then
return 'stale'; return 'stale';
end if; end if;
v_final := v_review.attempts >= private.sheet_verification_attempt_limit(); v_definitive := code = any (private.sheet_worker_final_codes());
v_final := v_definitive
or v_review.attempts >= private.sheet_verification_attempt_limit();
update public.anatomical_sheet_reviews r update public.anatomical_sheet_reviews r
set failure_code = code, lease_token = null, locked_at = null, set failure_code = code, lease_token = null, locked_at = null,
not_before = case when v_final then null not_before = case when v_final then null
@@ -405,6 +421,9 @@ begin
'anatomical_sheet_review', v_review.id, null, null, 'anatomical_sheet_review', v_review.id, null, null,
jsonb_build_object('code', code, 'attempts', v_review.attempts, jsonb_build_object('code', code, 'attempts', v_review.attempts,
'outcome', case when v_final then 'unavailable' end, 'outcome', case when v_final then 'unavailable' end,
'closed_by', case when not v_final then null
when v_definitive then 'definitive'
else 'attempt_limit' end,
'retry_delay_seconds', case when not v_final then 'retry_delay_seconds', case when not v_final then
extract(epoch from private.sheet_verification_retry_delay(v_review.attempts))::integer end)); extract(epoch from private.sheet_verification_retry_delay(v_review.attempts))::integer end));
return case when v_final then 'unavailable' else 'retry' end; return case when v_final then 'unavailable' else 'retry' end;
@@ -420,6 +439,7 @@ revoke all on function private.sheet_verification_lease() from public;
revoke all on function private.sheet_verification_max_verdict_bytes() from public; revoke all on function private.sheet_verification_max_verdict_bytes() from public;
revoke all on function private.sheet_verdict_codes() from public; revoke all on function private.sheet_verdict_codes() from public;
revoke all on function private.sheet_worker_codes() from public; revoke all on function private.sheet_worker_codes() from public;
revoke all on function private.sheet_worker_final_codes() from public;
revoke all on function private.require_service_role() from public; revoke all on function private.require_service_role() from public;
revoke all on function private.sheet_document_group_keys(jsonb) from public; revoke all on function private.sheet_document_group_keys(jsonb) from public;
revoke all on function private.sheet_review_base(uuid) from public; revoke all on function private.sheet_review_base(uuid) from public;
+47 -4
View File
@@ -4,7 +4,7 @@
-- nothing), technical failures up to the attempt limit with their progressive -- nothing), technical failures up to the attempt limit with their progressive
-- delay (`not_before`), dead leases. -- delay (`not_before`), dead leases.
begin; begin;
select plan(72); select plan(89);
insert into auth.users (id, email, email_confirmed_at) values insert into auth.users (id, email, email_confirmed_at) values
('f0000000-0000-0000-0000-000000000001', 'own@vj.local', now()), ('f0000000-0000-0000-0000-000000000001', 'own@vj.local', now()),
@@ -239,17 +239,19 @@ update public.anatomical_sheet_reviews set locked_at = now() - interval '11 minu
set local role service_role; select pg_temp.worker(); set local role service_role; select pg_temp.worker();
select is((select attempts from public.claim_sheet_verifications(1)), 4, 'a dead lease is claimed again'); select is((select attempts from public.claim_sheet_verifications(1)), 4, 'a dead lease is claimed again');
select is(pg_temp.status('r3'), 'verifying/-/verification.worker.lease_expired', 'the interruption is recorded'); select is(pg_temp.status('r3'), 'verifying/-/verification.worker.lease_expired', 'the interruption is recorded');
select is(public.fail_sheet_verification(pg_temp.id('r3'), pg_temp.token('r3'), 'verification.worker.resource_limit'), 'retry', 'a fourth failure is retried'); select is(public.fail_sheet_verification(pg_temp.id('r3'), pg_temp.token('r3'), 'verification.worker.timeout'), 'retry', 'a fourth failure is retried');
select ok(pg_temp.delayed('r3', interval '20 seconds'), 'after attempt 4 the retry waits 20 s'); select ok(pg_temp.delayed('r3', interval '20 seconds'), 'after attempt 4 the retry waits 20 s');
select pg_temp.set_not_before('r3', interval '-1 second'); select pg_temp.set_not_before('r3', interval '-1 second');
select is((select attempts from public.claim_sheet_verifications(1)), 5, 'claimed after 20 s: fifth attempt'); select is((select attempts from public.claim_sheet_verifications(1)), 5, 'claimed after 20 s: fifth attempt');
-- At the limit, a technical failure closes the round unavailable, delay cleared. -- At the limit, a technical failure closes the round unavailable, delay cleared.
select pg_temp.set_not_before('r3', interval '1 hour'); select pg_temp.set_not_before('r3', interval '1 hour');
select is(public.fail_sheet_verification(pg_temp.id('r3'), pg_temp.token('r3'), 'verification.worker.resource_limit'), 'unavailable', 'the last attempt closes the round'); select is(public.fail_sheet_verification(pg_temp.id('r3'), pg_temp.token('r3'), 'verification.worker.unavailable'), 'unavailable', 'the last attempt closes the round');
select is(pg_temp.status('r3'), 'verification_failed/unavailable/verification.worker.resource_limit', 'round closed unavailable with the last code'); select is(pg_temp.status('r3'), 'verification_failed/unavailable/verification.worker.unavailable', 'round closed unavailable with the last code');
select ok(pg_temp.not_before_is_null('r3'), 'the fifth failure clears the delay'); select ok(pg_temp.not_before_is_null('r3'), 'the fifth failure clears the delay');
reset role; reset role;
select ok(exists (select 1 from audit.audit_logs where action = 'anatomical_sheet_review.verification_attempt_failed' and resource_id = pg_temp.id('r3')), 'attempt failure audited'); select ok(exists (select 1 from audit.audit_logs where action = 'anatomical_sheet_review.verification_attempt_failed' and resource_id = pg_temp.id('r3')), 'attempt failure audited');
select ok(exists (select 1 from audit.audit_logs where action = 'anatomical_sheet_review.verification_failed' and resource_id = pg_temp.id('r3')
and metadata ->> 'closed_by' = 'attempt_limit'), 'a transient code closes at the limit: closed_by attempt_limit');
-- 8. A dead lease at the limit is closed by the next claim. -- 8. A dead lease at the limit is closed by the next claim.
set local role authenticated; set local role authenticated;
@@ -262,5 +264,46 @@ set local role service_role; select pg_temp.worker();
select is((select count(*)::int from public.claim_sheet_verifications(5)), 0, 'a round at the limit is not claimed'); select is((select count(*)::int from public.claim_sheet_verifications(5)), 0, 'a round at the limit is not claimed');
select is(pg_temp.status('r4'), 'verification_failed/unavailable/verification.worker.lease_expired', 'its dead lease closes it unavailable'); select is(pg_temp.status('r4'), 'verification_failed/unavailable/verification.worker.lease_expired', 'its dead lease closes it unavailable');
-- 9. Definitive codes close the round at the first occurrence; transient
-- codes keep their retry. Rounds are sequential: each follows a closed one.
set local role authenticated;
select pg_temp.act('f0000000-0000-0000-0000-00000000000e','f1000000-0000-0000-0000-00000000000e');
select public.save_sheet_draft(pg_temp.id('v1'), 4, pg_temp.doc(pg_temp.id('asset'),'VJ5'), 'five');
insert into pg_temp.k select 'r5', public.submit_sheet_review(pg_temp.id('v1'), 5, 'five');
reset role;
set local role service_role; select pg_temp.worker();
select is((select count(*)::int from public.claim_sheet_verifications(1)), 1, 'fifth round claimed');
select is(public.fail_sheet_verification(pg_temp.id('r5'), pg_temp.token('r5'), 'verification.worker.resource_limit'), 'unavailable', 'resource_limit at attempt 1 closes the round');
select is(pg_temp.status('r5'), 'verification_failed/unavailable/verification.worker.resource_limit', 'closed unavailable, code kept');
select ok(pg_temp.not_before_is_null('r5'), 'a definitive closure sets no delay');
select is((select count(*)::int from public.claim_sheet_verifications(10)), 0, 'a definitively closed round is never claimed again');
reset role;
select ok(exists (select 1 from audit.audit_logs where action = 'anatomical_sheet_review.verification_failed' and resource_id = pg_temp.id('r5')
and metadata ->> 'closed_by' = 'definitive' and metadata ->> 'attempts' = '1'), 'closure audited as definitive at attempt 1');
set local role authenticated;
select pg_temp.act('f0000000-0000-0000-0000-00000000000e','f1000000-0000-0000-0000-00000000000e');
select public.save_sheet_draft(pg_temp.id('v1'), 5, pg_temp.doc(pg_temp.id('asset'),'VJ6'), 'six');
insert into pg_temp.k select 'r6', public.submit_sheet_review(pg_temp.id('v1'), 6, 'six');
reset role;
set local role service_role; select pg_temp.worker();
select is((select count(*)::int from public.claim_sheet_verifications(1)), 1, 'sixth round claimed');
select is(public.fail_sheet_verification(pg_temp.id('r6'), pg_temp.token('r6'), 'verification.worker.verdict_refused'), 'unavailable', 'verdict_refused at attempt 1 closes the round');
select is(pg_temp.status('r6'), 'verification_failed/unavailable/verification.worker.verdict_refused', 'closed unavailable, code kept');
select is((select count(*)::int from public.claim_sheet_verifications(10)), 0, 'a refused verdict is never claimed again');
set local role authenticated;
select pg_temp.act('f0000000-0000-0000-0000-00000000000e','f1000000-0000-0000-0000-00000000000e');
select public.save_sheet_draft(pg_temp.id('v1'), 6, pg_temp.doc(pg_temp.id('asset'),'VJ7'), 'seven');
insert into pg_temp.k select 'r7', public.submit_sheet_review(pg_temp.id('v1'), 7, 'seven');
reset role;
set local role service_role; select pg_temp.worker();
select is((select count(*)::int from public.claim_sheet_verifications(1)), 1, 'seventh round claimed');
select is(public.fail_sheet_verification(pg_temp.id('r7'), gen_random_uuid(), 'verification.worker.resource_limit'), 'stale', 'a definitive code with another token is stale');
select is(public.fail_sheet_verification(pg_temp.id('r7'), null, 'verification.worker.verdict_refused'), 'stale', 'a definitive code with a null token is stale');
select is(pg_temp.status('r7'), 'verifying/-/-', 'a stale definitive failure writes nothing');
select is(public.fail_sheet_verification(pg_temp.id('r7'), pg_temp.token('r7'), 'verification.worker.unavailable'), 'retry', 'unavailable at attempt 1 is retried');
select ok(pg_temp.delayed('r7', interval '2 seconds'), 'after attempt 1 the retry waits 2 s');
select * from finish(); select * from finish();
rollback; rollback;