feat(supabase): verification retry by effective cause
resource_limit (intrinsic to the frozen input: frame, input or output cap, SIGXCPU, SIGABRT, SIGSYS) and verdict_refused close the round at the first occurrence; timeout, unavailable (now also an external SIGKILL) and lease_expired keep the 2/5/10/20 s retries up to five attempts. pgTAP 021: 89; sandbox case k closes at attempt 1.
This commit is contained in:
1 parent
c98a9c8f82
commit
a5844b9d56
8 files changed
+156
-46
No files matched your search
@@ -246,6 +246,17 @@ enum ChildVerdict {
|
||||
Stop,
|
||||
}
|
||||
|
||||
/// The child's own budgets end it with SIGXCPU (CPU, the hard limit sits two seconds
|
||||
/// after the soft one), SIGABRT (memory) or SIGSYS (seccomp): the same input ends the
|
||||
/// same way, so `resource_limit` (definitive). Any other signal, SIGKILL first, came
|
||||
/// from outside (OOM killer, stop): transient `unavailable`.
|
||||
fn signal_code(signal: i32) -> &'static str {
|
||||
match signal {
|
||||
libc::SIGXCPU | libc::SIGABRT | libc::SIGSYS => codes::RESOURCE_LIMIT,
|
||||
_ => codes::UNAVAILABLE,
|
||||
}
|
||||
}
|
||||
|
||||
/// Maps every way the child can end to a verdict on the round (§6.5 rows).
|
||||
fn classify_child(run: ChildRun, job: &VerifyJob) -> ChildVerdict {
|
||||
let stderr = run.stderr;
|
||||
@@ -262,7 +273,7 @@ fn classify_child(run: ChildRun, job: &VerifyJob) -> ChildVerdict {
|
||||
.signal(signal)
|
||||
.child_stderr(&stderr)
|
||||
.emit();
|
||||
ChildVerdict::Report(codes::RESOURCE_LIMIT)
|
||||
ChildVerdict::Report(signal_code(signal))
|
||||
}
|
||||
ChildOutcome::OutputOverflow => {
|
||||
log::warn("child_output_overflow").review(review).emit();
|
||||
|
||||
@@ -365,7 +365,7 @@ fn tracked_an_object_over_the_policy_is_reported_drawing_missing() {
|
||||
|
||||
// Tracked misclassification (owner brief 2026-11-01), not fixed: a document naming enough
|
||||
// drawings that the frame header passes 64 KiB is refused by the parent's own framing and
|
||||
// reported `resource_limit` (a worker failure, retried) where `caps` would name it.
|
||||
// reported `resource_limit` (a definitive worker failure) where `caps` would name it.
|
||||
#[test]
|
||||
fn tracked_a_header_over_64_kib_is_reported_resource_limit() {
|
||||
let rows = vec![row(); 600];
|
||||
@@ -378,6 +378,7 @@ fn tracked_a_header_over_64_kib_is_reported_resource_limit() {
|
||||
fake.calls().last().unwrap(),
|
||||
&fail_call(codes::RESOURCE_LIMIT)
|
||||
);
|
||||
assert_eq!(fake.names().iter().filter(|n| *n == "fail").count(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -435,7 +436,8 @@ fn every_child_limit_maps_to_its_closed_code() {
|
||||
ChildOutcome::Signalled { signal: 24 },
|
||||
codes::RESOURCE_LIMIT,
|
||||
),
|
||||
(ChildOutcome::Signalled { signal: 9 }, codes::RESOURCE_LIMIT),
|
||||
(ChildOutcome::Signalled { signal: 9 }, codes::UNAVAILABLE),
|
||||
(ChildOutcome::Signalled { signal: 15 }, codes::UNAVAILABLE),
|
||||
(ChildOutcome::Signalled { signal: 6 }, codes::RESOURCE_LIMIT),
|
||||
(
|
||||
ChildOutcome::Signalled { signal: 31 },
|
||||
|
||||
@@ -4567,3 +4567,29 @@ through `is distinct from`, service role only) fixed in the three round RPCs, pg
|
||||
sandbox `test-db` 1 880 PASS, `types --check` equal and `verify-e2e` 10/10 on a fresh stack and on the upgrades from
|
||||
before M6 and from before M4 (the DEV path); `avatar-e2e` 5/5; `test` 87 and `e2e:shell` 86 (rerun on the final M6
|
||||
after the verifier's ECARTS(2): the earlier run predated the fix); DEV untouched.
|
||||
|
||||
Retry by effective cause (owner brief 2026-11-01, closure C1). SQL holds the code policy
|
||||
(`private.sheet_worker_final_codes`: `resource_limit`, `verdict_refused` close the round at the first
|
||||
occurrence, `closed_by` `definitive` in the audit; the others retry 2, 5, 10, 20 s up to five attempts,
|
||||
`closed_by` `attempt_limit`); the worker maps the cause to the code (`verify_job::classify_child`,
|
||||
`signal_code`), as implemented:
|
||||
|
||||
| Child end | Code | Policy |
|
||||
|---|---|---|
|
||||
| frame refused by the parent (header over 64 KiB) | `resource_limit` | definitive |
|
||||
| `Rejected` (input over the child cap) | `resource_limit` | definitive |
|
||||
| `OutputOverflow` | `resource_limit` | definitive |
|
||||
| SIGXCPU (CPU budget; the hard limit is two seconds after the soft one, so the budget shows as SIGXCPU) | `resource_limit` | definitive |
|
||||
| SIGABRT (memory budget, abort) | `resource_limit` | definitive |
|
||||
| SIGSYS (seccomp filter) | `resource_limit` | definitive |
|
||||
| SIGKILL or any other signal (OOM killer, stop: from outside; the parent's own kills are `Timeout` and `OutputOverflow`) | `unavailable` | transient |
|
||||
| `Timeout` (wall clock) | `timeout` | transient |
|
||||
| transport, Storage | `unavailable` | transient |
|
||||
| `Broken`, `Failed` | `unavailable`, then the worker stops | transient |
|
||||
| verdict answered `malformed` | `verdict_refused` | definitive |
|
||||
| dead lease | `lease_expired` (set by the claim) | transient, immediate |
|
||||
|
||||
What changes for the codes: a SIGKILL is now reported `unavailable` (it was `resource_limit`); the
|
||||
outcome (`unavailable`) and the code list are otherwise unchanged. A definitive round closes after one
|
||||
child run instead of five identical ones. Still tracked and open: the header over 64 KiB is reported
|
||||
`resource_limit` where `caps` would name it (test `tracked_a_header_over_64_kib_is_reported_resource_limit`).
|
||||
@@ -261,20 +261,26 @@ unchanged.
|
||||
- Completion: `complete_sheet_verification` answers `passed`, `failed`,
|
||||
`stale` (nothing written, logged) or `malformed` (reported
|
||||
`verification.worker.verdict_refused`). Child timeout →
|
||||
`verification.worker.timeout`; a signal, an output overflow or an input over
|
||||
its cap → `verification.worker.resource_limit`; a transport error →
|
||||
`verification.worker.unavailable`; a broken child → `unavailable`, then the
|
||||
worker stops (as the avatar loop). A dead lease (10 minutes) is claimed
|
||||
again as `verification.worker.lease_expired`; the fifth attempt closes the
|
||||
round `verification_failed` / `unavailable`.
|
||||
`verification.worker.timeout` (transient); SIGXCPU, SIGABRT or SIGSYS (the
|
||||
child's own CPU, memory and seccomp budgets), an output overflow or an input
|
||||
over its cap → `verification.worker.resource_limit` (definitive: the frozen
|
||||
input gives the same end every time); any other signal, SIGKILL first (the
|
||||
OOM killer or a stop, from outside) and a transport error →
|
||||
`verification.worker.unavailable` (transient); a broken child →
|
||||
`unavailable`, then the worker stops (as the avatar loop). A dead lease (10
|
||||
minutes) is claimed again as `verification.worker.lease_expired`. Retry by
|
||||
cause: `resource_limit` and `verdict_refused` close the round
|
||||
`verification_failed` / `unavailable` at the first occurrence; the other
|
||||
codes retry up to the fifth attempt, which closes it the same way.
|
||||
- Retry delay (owner decision 2026-11-01): a failure reported by
|
||||
`fail_sheet_verification` makes the round claimable again after 2 s, 5 s,
|
||||
10 s, 20 s (after attempts 1 to 4), kept in SQL (`not_before`, hidden from
|
||||
the API roles), so the worker never sleeps for it and the claim of another
|
||||
round is not blocked; the effective delay is that delay plus up to one poll
|
||||
interval (`WORKER_POLL_SECONDS`, 2 s). A verdict (failed or not) is never
|
||||
retried; a dead lease is claimed again at once. The worker code is the
|
||||
same as before.
|
||||
retried; a dead lease is claimed again at once. Only the transient codes
|
||||
wait: the delay applies to `timeout`, `unavailable` and `lease_expired`
|
||||
(SQL decides, `private.sheet_worker_final_codes`).
|
||||
- Measured (2026-11-01): the at-caps document (64 views, 1 024 annotations,
|
||||
8 192 nodes) verified in 185 ms by the worker on the sandbox, verdict
|
||||
24 689 bytes; the equine draft in about 60 ms.
|
||||
|
||||
@@ -890,12 +890,19 @@ attempts below a limit), plus a progressive delay for technical retries only
|
||||
else answers `stale` or `malformed` and writes nothing. Moves the round to
|
||||
`open` or to `verification_failed` (§6.5) and audits.
|
||||
- `fail_sheet_verification(review_id, lease_token, code)`: a technical
|
||||
failure; the round stays `verifying` and becomes claimable after a delay
|
||||
failure, closed by cause (2026-11-01, `private.sheet_worker_final_codes`).
|
||||
A definitive code (`verification.worker.resource_limit`, cause intrinsic to
|
||||
the frozen input or the contract caps; `verification.worker.verdict_refused`)
|
||||
closes the round at its first occurrence: `verification_failed`, outcome
|
||||
`unavailable`, the code kept, no delay, never claimed again. A transient
|
||||
code (`timeout`, `unavailable`, `lease_expired`): the round stays `verifying`
|
||||
and becomes claimable after a delay
|
||||
that grows with the attempt (2 s after attempt 1, 5 s after 2, 10 s after 3,
|
||||
20 s after 4; `private.sheet_verification_retry_delay`), kept in the round's
|
||||
`not_before` (a column the API roles cannot read, set here, cleared by the
|
||||
claim); the fifth failure becomes `verification_failed` with outcome
|
||||
`unavailable` and clears it. Nothing is written for a stale call. The delay
|
||||
claim); the fifth failure of a transient code becomes `verification_failed`
|
||||
with outcome `unavailable` and clears it. The audit payload says which rule
|
||||
closed the round (`closed_by`: `definitive` or `attempt_limit`). Nothing is written for a stale call. The delay
|
||||
is persisted, so no worker sleeps and any worker honours it; the effective
|
||||
wait is the delay plus up to one poll interval of the worker (2 s by
|
||||
default). Only these technical failures wait: a validation or integrity
|
||||
@@ -942,7 +949,8 @@ five, verdict text at most 4 MiB; the completion also refuses an unknown verdict
|
||||
or code, a duplicated group key and a drawing whose result is not `ok`, and accepts a
|
||||
failed verdict without groups or drawings; worker codes `verification.worker.unavailable`,
|
||||
`.timeout`, `.resource_limit`, `.lease_expired` (a dead lease), `.verdict_refused` (a
|
||||
completion answered `malformed`), a closed list in SQL (`private.sheet_worker_codes`).
|
||||
completion answered `malformed`), a closed list in SQL (`private.sheet_worker_codes`); the
|
||||
definitive ones are `.resource_limit` and `.verdict_refused` (`private.sheet_worker_final_codes`).
|
||||
These values and codes are the implementer's, for the owner's confirmation. The
|
||||
service role executes these four RPCs besides the two upload ones (pgTAP 018).
|
||||
|
||||
@@ -961,11 +969,13 @@ did not pass; the closed codes keep the categories apart.
|
||||
|---|---|---|---|---|
|
||||
| Business validation | integrity error, caps exceeded, annotation without geometry, annotation outside its view's frame | `verification_failed`, outcome `failed`, codes `verification.validation.*` and the issues | no | fix the draft, submit again |
|
||||
| Integrity | strict parse refused, drawing missing, size or sha256 mismatch, SVG refused, needing a removal or outside the stored form (§9.2), `object_conflict` | `verification_failed`, outcome `failed`, codes `verification.integrity.*` | no | fix or upload again; `object_conflict` follows §9.3 |
|
||||
| Transient worker error | database or Storage unreachable, worker restart | stays `verifying`, code recorded | yes, after 2 s, 5 s, 10 s, 20 s, up to the limit | automatic |
|
||||
| Timeout or resource limit | child over its wall-clock or memory limit | stays `verifying`, code recorded | yes, same delays, up to the limit | reduce the document or the drawing if it repeats |
|
||||
| Transient worker error | database or Storage unreachable, broken child, child killed from outside (SIGKILL or any signal not listed below: OOM killer, stop), worker restart | `verification.worker.unavailable`, stays `verifying`, code recorded | yes, after 2 s, 5 s, 10 s, 20 s, up to the limit | automatic |
|
||||
| Timeout | child over its wall-clock deadline | `verification.worker.timeout`, stays `verifying`, code recorded | yes, same delays, up to the limit | reduce the document or the drawing if it repeats |
|
||||
| Resource limit intrinsic to the content | frame refused, input over the child cap, output over its cap, CPU budget (SIGXCPU), memory budget (SIGABRT), seccomp (SIGSYS) | `verification_failed`, outcome `unavailable`, `verification.worker.resource_limit`, closed at the first occurrence | no (the same input ends the same way) | reduce the document or the drawings, submit again |
|
||||
| Verdict refused | completion answered `malformed` | `verification_failed`, outcome `unavailable`, `verification.worker.verdict_refused`, closed at the first occurrence | no | submit again later; reported to the operator |
|
||||
| Interruption | worker crash, lease expired | claimable again at once (the lease was the wait); counts as an attempt | yes, up to the limit | automatic |
|
||||
| Stale result | completion with an old token, after a withdrawal, or naming another revision | nothing written, logged | no | none needed |
|
||||
| Definitive failure | attempt limit reached (five; V1-2.3 implements five, for the owner's confirmation) | `verification_failed`, outcome `unavailable`, last code | no | submit again later |
|
||||
| Attempt limit | a transient code at its fifth attempt (five; V1-2.3 implements five, for the owner's confirmation) | `verification_failed`, outcome `unavailable`, last code | no | submit again later |
|
||||
| Core change | release that changes rules or reader | rounds withdrawn by the release migration | no | submit again |
|
||||
|
||||
- Withdrawal is always available: the submitter or any editor withdraws a
|
||||
@@ -1715,7 +1725,7 @@ of the document's JSON text in one form, the same for every layer:
|
||||
|---|---|---|---|
|
||||
| M4 `sheet_access_hardening` | V1-2.1 | `anatomical_catalog_grants` and its RPCs; `has_mfa`, `grant_active`, `caller_has`; `can_read_official_catalog` and the reader row predicate; `can_edit_sheet` redefined (F14); AR-1 B on the platform role RPCs and the private write functions (F19, §5.7), F17, F18; F1, F2, F3 (fork closed), F5, F6, F8, F9, F12 (every catalogue table), F13, F15; asset-row immutability trigger (F16); historical-sheet protection trigger (§14.2); reader column grants narrowed; personal creation and visibility changes refused | no row changes |
|
||||
| M5 `sheet_review_workflow` | V1-2.1 | version columns `revision`, `authors`, `source_version_id`, `published_review_id`, `document_fingerprint`; sheet column `working_copy_source_version_id`; revision, author and freeze triggers, the historical-sheet write rule; extended immutability guard (§6.8); rounds, comments, acknowledgments; `create_official_sheet`, `create_sheet_draft` adapted, `save_sheet_draft`, submit, withdraw, comment, acknowledge, decide, publish; `create_sheet_working_copy`; species RPCs; catalogue and version read RPCs; upload: `register_sheet_asset` extended, `confirm_sheet_asset_upload` with the storage check (F7), INSERT policy, bucket limit 2 MiB; mirror removal and status check (§12.2); superseded signatures dropped; the content-size CHECK replaced by the text measure (`octet_length(content::text)`, §12.2) | new columns take their defaults on the equine v1 row (`revision` 0, empty author set); its document, status, number and assets unchanged; the new size CHECK is validated against the existing rows when M5 is applied (the equine document measures 184 105 bytes as `content::text`, measured on the sandbox); bucket limit lowered; no round, no grant |
|
||||
| M6 `sheet_verification_jobs` | V1-2.3 | worker RPCs (claim, input, complete with the verdict structure check, fail), service role only, with the retry delay (hidden `not_before` column on the round, `private.sheet_verification_retry_delay`: owner decision 2026-11-01, replaces « without backoff »); written as `20261101100000_sheet_verification_jobs.sql`, pgTAP `021`, rehearsed on the sandbox from the state before M4 (M4 to M6 in one `migration up`) and from the state before M6 | none |
|
||||
| M6 `sheet_verification_jobs` | V1-2.3 | worker RPCs (claim, input, complete with the verdict structure check, fail), service role only, with the retry delay (hidden `not_before` column on the round, `private.sheet_verification_retry_delay`: owner decision 2026-11-01, replaces « without backoff ») and the retry by cause (definitive codes close at the first occurrence, `private.sheet_worker_final_codes`); written as `20261101100000_sheet_verification_jobs.sql`, pgTAP `021`, rehearsed on the sandbox from the state before M4 (M4 to M6 in one `migration up`) and from the state before M6 | none |
|
||||
|
||||
Each migration ships its pgTAP file (allowed and denied cases), regenerated
|
||||
`database.types.ts`, a `db:sandbox up upgrade` run from the DEV state (with a
|
||||
|
||||
@@ -301,18 +301,6 @@ async function holdLease(editor: SteppedAccount, d: Draft, attempts: number): Pr
|
||||
throw new Error('lease not held before the worker claimed')
|
||||
}
|
||||
|
||||
/** Polls a one-value query until it answers `expected`. */
|
||||
async function waitFor(query: string, expected: string, timeoutMs: number): Promise<void> {
|
||||
const start = performance.now()
|
||||
let last = ''
|
||||
while (performance.now() - start < timeoutMs) {
|
||||
last = await sql(query)
|
||||
if (last === expected) return
|
||||
await new Promise((resolve) => setTimeout(resolve, 200))
|
||||
}
|
||||
throw new Error(`expected ${expected}, last ${last}`)
|
||||
}
|
||||
|
||||
/** Registers, uploads and confirms one drawing per size on `sheet`; the ids in order. */
|
||||
async function putDrawings(editor: SteppedAccount, sheet: string, sizes: number[]): Promise<{ id: string; bytes: Buffer }[]> {
|
||||
const out: { id: string; bytes: Buffer }[] = []
|
||||
@@ -705,6 +693,7 @@ describe.skipIf(!WORKER)('sheet verification by the real worker (V1-2.3)', () =>
|
||||
|
||||
// Tracked misclassification (owner brief 2026-11-01: caps expected; not fixed): a document naming
|
||||
// so many registered drawings (600 rows, about 160 bytes each in the header) that the frame header passes 64 KiB is reported resource_limit.
|
||||
// The cause is the frozen input: definitive at the first attempt, never claimed again.
|
||||
const many = await newSheet(L)
|
||||
const rows = (await sql(`insert into public.anatomical_sheet_assets (sheet_id, sha256, byte_size, file_name, created_by, uploaded_at)
|
||||
select '${uuidOf(many.sheet)}', md5(g::text) || md5(g::text || 'h'), 900, 'h' || g || '.svg', '${uuidOf(L.id)}', now()
|
||||
@@ -712,11 +701,14 @@ describe.skipIf(!WORKER)('sheet verification by the real worker (V1-2.3)', () =>
|
||||
expect(rows).toHaveLength(600)
|
||||
await save(L, many, multiDoc(rows))
|
||||
const header = uuidOf(await submit(L, many))
|
||||
await waitFor(`select failure_code || '/' || attempts from public.anatomical_sheet_reviews where id = '${header}'`,
|
||||
'verification.worker.resource_limit/1', 30_000)
|
||||
// Withdrawn so that the worker stops retrying it (its next claim is 2 s away).
|
||||
ok(await rpc('withdraw_sheet_review', { review_id: header, note: 'tracked misclassification' }, L.aal2))
|
||||
expect((await round(header)).status).toBe('withdrawn')
|
||||
const closed = await settled(header, 30_000)
|
||||
const state = (r: Round) => [r.status, r.failure_outcome, r.failure_code, r.attempts]
|
||||
const final = ['verification_failed', 'unavailable', 'verification.worker.resource_limit', 1]
|
||||
expect(state(closed.round)).toEqual(final)
|
||||
expect(await sql(`select not_before is null from public.anatomical_sheet_reviews where id = '${header}'`)).toBe('t')
|
||||
// Past the first retry delay (2 s) the round is still closed at attempt 1: no second claim.
|
||||
await new Promise((resolve) => setTimeout(resolve, 3_000))
|
||||
expect(state(await round(header))).toEqual(final)
|
||||
}, 5 * CAPS_BUDGET_MS)
|
||||
})
|
||||
|
||||
|
||||
@@ -13,7 +13,9 @@
|
||||
-- the drawings the document references;
|
||||
-- * complete_sheet_verification: the verdict, checked for its binding and
|
||||
-- its structure; `stale` or `malformed` write nothing;
|
||||
-- * fail_sheet_verification: a technical failure, from a closed list.
|
||||
-- * fail_sheet_verification: a technical failure, from a closed list;
|
||||
-- transient codes retry after a delay up to the attempt limit, the
|
||||
-- definitive codes (`sheet_worker_final_codes`) close the round at once.
|
||||
-- The verdict itself is the core's (`compute_verdict`, crates/core); SQL
|
||||
-- checks only that it answers this round and covers this document.
|
||||
|
||||
@@ -67,7 +69,8 @@ returns text[] language sql immutable set search_path = '' as $$
|
||||
$$;
|
||||
|
||||
-- The worker's technical failure codes (§6.5 transient, timeout, resource,
|
||||
-- interruption rows; crates/worker `verify_job`).
|
||||
-- interruption rows; crates/worker `verify_job`). The worker maps the cause
|
||||
-- to a code; the retry policy of each code is `sheet_worker_final_codes`.
|
||||
create or replace function private.sheet_worker_codes()
|
||||
returns text[] language sql immutable set search_path = '' as $$
|
||||
select array[
|
||||
@@ -77,6 +80,15 @@ returns text[] language sql immutable set search_path = '' as $$
|
||||
]
|
||||
$$;
|
||||
|
||||
-- The codes whose cause is fixed by the frozen input or the contract caps
|
||||
-- (resource_limit) or by a verdict the SQL refuses (verdict_refused): a retry
|
||||
-- would give the same answer, so the round closes at the first occurrence.
|
||||
-- The others (timeout, unavailable, lease_expired) are transient.
|
||||
create or replace function private.sheet_worker_final_codes()
|
||||
returns text[] language sql immutable set search_path = '' as $$
|
||||
select array['verification.worker.resource_limit', 'verification.worker.verdict_refused']
|
||||
$$;
|
||||
|
||||
-- The service role only (the worker's key); anything else is refused.
|
||||
create or replace function private.require_service_role()
|
||||
returns void language plpgsql stable set search_path = '' as $$
|
||||
@@ -367,9 +379,10 @@ begin
|
||||
end;
|
||||
$$;
|
||||
|
||||
-- A technical failure of a leased round (§6.5): the round stays `verifying`
|
||||
-- and claimable after its retry delay, its code recorded, until the attempt
|
||||
-- limit closes it `verification_failed` / `unavailable`. Answers `retry`, `unavailable` or
|
||||
-- A technical failure of a leased round (§6.5): a transient code leaves the
|
||||
-- round `verifying` and claimable after its retry delay, its code recorded,
|
||||
-- until the attempt limit closes it `verification_failed` / `unavailable`; a
|
||||
-- definitive code closes it at once. Answers `retry`, `unavailable` or
|
||||
-- `stale` (nothing written).
|
||||
create or replace function public.fail_sheet_verification(review_id uuid, lease_token uuid, code text)
|
||||
returns text
|
||||
@@ -382,6 +395,7 @@ as $$
|
||||
declare
|
||||
v_review public.anatomical_sheet_reviews%rowtype;
|
||||
v_final boolean;
|
||||
v_definitive boolean;
|
||||
begin
|
||||
perform private.require_service_role();
|
||||
if code is null or not (code = any (private.sheet_worker_codes())) then
|
||||
@@ -392,7 +406,9 @@ begin
|
||||
or lease_token is null or v_review.lease_token is distinct from lease_token then
|
||||
return 'stale';
|
||||
end if;
|
||||
v_final := v_review.attempts >= private.sheet_verification_attempt_limit();
|
||||
v_definitive := code = any (private.sheet_worker_final_codes());
|
||||
v_final := v_definitive
|
||||
or v_review.attempts >= private.sheet_verification_attempt_limit();
|
||||
update public.anatomical_sheet_reviews r
|
||||
set failure_code = code, lease_token = null, locked_at = null,
|
||||
not_before = case when v_final then null
|
||||
@@ -405,6 +421,9 @@ begin
|
||||
'anatomical_sheet_review', v_review.id, null, null,
|
||||
jsonb_build_object('code', code, 'attempts', v_review.attempts,
|
||||
'outcome', case when v_final then 'unavailable' end,
|
||||
'closed_by', case when not v_final then null
|
||||
when v_definitive then 'definitive'
|
||||
else 'attempt_limit' end,
|
||||
'retry_delay_seconds', case when not v_final then
|
||||
extract(epoch from private.sheet_verification_retry_delay(v_review.attempts))::integer end));
|
||||
return case when v_final then 'unavailable' else 'retry' end;
|
||||
@@ -420,6 +439,7 @@ revoke all on function private.sheet_verification_lease() from public;
|
||||
revoke all on function private.sheet_verification_max_verdict_bytes() from public;
|
||||
revoke all on function private.sheet_verdict_codes() from public;
|
||||
revoke all on function private.sheet_worker_codes() from public;
|
||||
revoke all on function private.sheet_worker_final_codes() from public;
|
||||
revoke all on function private.require_service_role() from public;
|
||||
revoke all on function private.sheet_document_group_keys(jsonb) from public;
|
||||
revoke all on function private.sheet_review_base(uuid) from public;
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
-- nothing), technical failures up to the attempt limit with their progressive
|
||||
-- delay (`not_before`), dead leases.
|
||||
begin;
|
||||
select plan(72);
|
||||
select plan(89);
|
||||
|
||||
insert into auth.users (id, email, email_confirmed_at) values
|
||||
('f0000000-0000-0000-0000-000000000001', 'own@vj.local', now()),
|
||||
@@ -239,17 +239,19 @@ update public.anatomical_sheet_reviews set locked_at = now() - interval '11 minu
|
||||
set local role service_role; select pg_temp.worker();
|
||||
select is((select attempts from public.claim_sheet_verifications(1)), 4, 'a dead lease is claimed again');
|
||||
select is(pg_temp.status('r3'), 'verifying/-/verification.worker.lease_expired', 'the interruption is recorded');
|
||||
select is(public.fail_sheet_verification(pg_temp.id('r3'), pg_temp.token('r3'), 'verification.worker.resource_limit'), 'retry', 'a fourth failure is retried');
|
||||
select is(public.fail_sheet_verification(pg_temp.id('r3'), pg_temp.token('r3'), 'verification.worker.timeout'), 'retry', 'a fourth failure is retried');
|
||||
select ok(pg_temp.delayed('r3', interval '20 seconds'), 'after attempt 4 the retry waits 20 s');
|
||||
select pg_temp.set_not_before('r3', interval '-1 second');
|
||||
select is((select attempts from public.claim_sheet_verifications(1)), 5, 'claimed after 20 s: fifth attempt');
|
||||
-- At the limit, a technical failure closes the round unavailable, delay cleared.
|
||||
select pg_temp.set_not_before('r3', interval '1 hour');
|
||||
select is(public.fail_sheet_verification(pg_temp.id('r3'), pg_temp.token('r3'), 'verification.worker.resource_limit'), 'unavailable', 'the last attempt closes the round');
|
||||
select is(pg_temp.status('r3'), 'verification_failed/unavailable/verification.worker.resource_limit', 'round closed unavailable with the last code');
|
||||
select is(public.fail_sheet_verification(pg_temp.id('r3'), pg_temp.token('r3'), 'verification.worker.unavailable'), 'unavailable', 'the last attempt closes the round');
|
||||
select is(pg_temp.status('r3'), 'verification_failed/unavailable/verification.worker.unavailable', 'round closed unavailable with the last code');
|
||||
select ok(pg_temp.not_before_is_null('r3'), 'the fifth failure clears the delay');
|
||||
reset role;
|
||||
select ok(exists (select 1 from audit.audit_logs where action = 'anatomical_sheet_review.verification_attempt_failed' and resource_id = pg_temp.id('r3')), 'attempt failure audited');
|
||||
select ok(exists (select 1 from audit.audit_logs where action = 'anatomical_sheet_review.verification_failed' and resource_id = pg_temp.id('r3')
|
||||
and metadata ->> 'closed_by' = 'attempt_limit'), 'a transient code closes at the limit: closed_by attempt_limit');
|
||||
|
||||
-- 8. A dead lease at the limit is closed by the next claim.
|
||||
set local role authenticated;
|
||||
@@ -262,5 +264,46 @@ set local role service_role; select pg_temp.worker();
|
||||
select is((select count(*)::int from public.claim_sheet_verifications(5)), 0, 'a round at the limit is not claimed');
|
||||
select is(pg_temp.status('r4'), 'verification_failed/unavailable/verification.worker.lease_expired', 'its dead lease closes it unavailable');
|
||||
|
||||
-- 9. Definitive codes close the round at the first occurrence; transient
|
||||
-- codes keep their retry. Rounds are sequential: each follows a closed one.
|
||||
set local role authenticated;
|
||||
select pg_temp.act('f0000000-0000-0000-0000-00000000000e','f1000000-0000-0000-0000-00000000000e');
|
||||
select public.save_sheet_draft(pg_temp.id('v1'), 4, pg_temp.doc(pg_temp.id('asset'),'VJ5'), 'five');
|
||||
insert into pg_temp.k select 'r5', public.submit_sheet_review(pg_temp.id('v1'), 5, 'five');
|
||||
reset role;
|
||||
set local role service_role; select pg_temp.worker();
|
||||
select is((select count(*)::int from public.claim_sheet_verifications(1)), 1, 'fifth round claimed');
|
||||
select is(public.fail_sheet_verification(pg_temp.id('r5'), pg_temp.token('r5'), 'verification.worker.resource_limit'), 'unavailable', 'resource_limit at attempt 1 closes the round');
|
||||
select is(pg_temp.status('r5'), 'verification_failed/unavailable/verification.worker.resource_limit', 'closed unavailable, code kept');
|
||||
select ok(pg_temp.not_before_is_null('r5'), 'a definitive closure sets no delay');
|
||||
select is((select count(*)::int from public.claim_sheet_verifications(10)), 0, 'a definitively closed round is never claimed again');
|
||||
reset role;
|
||||
select ok(exists (select 1 from audit.audit_logs where action = 'anatomical_sheet_review.verification_failed' and resource_id = pg_temp.id('r5')
|
||||
and metadata ->> 'closed_by' = 'definitive' and metadata ->> 'attempts' = '1'), 'closure audited as definitive at attempt 1');
|
||||
|
||||
set local role authenticated;
|
||||
select pg_temp.act('f0000000-0000-0000-0000-00000000000e','f1000000-0000-0000-0000-00000000000e');
|
||||
select public.save_sheet_draft(pg_temp.id('v1'), 5, pg_temp.doc(pg_temp.id('asset'),'VJ6'), 'six');
|
||||
insert into pg_temp.k select 'r6', public.submit_sheet_review(pg_temp.id('v1'), 6, 'six');
|
||||
reset role;
|
||||
set local role service_role; select pg_temp.worker();
|
||||
select is((select count(*)::int from public.claim_sheet_verifications(1)), 1, 'sixth round claimed');
|
||||
select is(public.fail_sheet_verification(pg_temp.id('r6'), pg_temp.token('r6'), 'verification.worker.verdict_refused'), 'unavailable', 'verdict_refused at attempt 1 closes the round');
|
||||
select is(pg_temp.status('r6'), 'verification_failed/unavailable/verification.worker.verdict_refused', 'closed unavailable, code kept');
|
||||
select is((select count(*)::int from public.claim_sheet_verifications(10)), 0, 'a refused verdict is never claimed again');
|
||||
|
||||
set local role authenticated;
|
||||
select pg_temp.act('f0000000-0000-0000-0000-00000000000e','f1000000-0000-0000-0000-00000000000e');
|
||||
select public.save_sheet_draft(pg_temp.id('v1'), 6, pg_temp.doc(pg_temp.id('asset'),'VJ7'), 'seven');
|
||||
insert into pg_temp.k select 'r7', public.submit_sheet_review(pg_temp.id('v1'), 7, 'seven');
|
||||
reset role;
|
||||
set local role service_role; select pg_temp.worker();
|
||||
select is((select count(*)::int from public.claim_sheet_verifications(1)), 1, 'seventh round claimed');
|
||||
select is(public.fail_sheet_verification(pg_temp.id('r7'), gen_random_uuid(), 'verification.worker.resource_limit'), 'stale', 'a definitive code with another token is stale');
|
||||
select is(public.fail_sheet_verification(pg_temp.id('r7'), null, 'verification.worker.verdict_refused'), 'stale', 'a definitive code with a null token is stale');
|
||||
select is(pg_temp.status('r7'), 'verifying/-/-', 'a stale definitive failure writes nothing');
|
||||
select is(public.fail_sheet_verification(pg_temp.id('r7'), pg_temp.token('r7'), 'verification.worker.unavailable'), 'retry', 'unavailable at attempt 1 is retried');
|
||||
select ok(pg_temp.delayed('r7', interval '2 seconds'), 'after attempt 1 the retry waits 2 s');
|
||||
|
||||
select * from finish();
|
||||
rollback;
|
||||
Reference in new issue
Block a user