feat(supabase): retry delay of 2, 5, 10, 20 s for technical verification failures

M6 amended in place (applied only to the throwaway sandbox): a hidden
not_before on the round, set by fail_sheet_verification and honoured by the
claim; RPC signatures unchanged. pgTAP 021 (68), sandbox cases i (delay with
the real worker), j (inputs at the caps, 2 MiB base), k (just beyond).
This commit is contained in:
bchanot committed 2026-11-01 21:09:00 +01:00
1 parent e967ce6736
commit 5adad3709f
8 files changed
+398 -29

No files matched your search

@@ -4,7 +4,10 @@
--
-- The round row (M5 `anatomical_sheet_reviews`) is the job, with the avatar
-- lease pattern of 20261013100000_files_foundation.sql (lease token,
-- `locked_at`, a fixed lease, attempts below a limit), without backoff:
-- `locked_at`, a fixed lease, attempts below a limit), plus a progressive
-- delay for technical retries only (owner decision 2026-11-01): a failure
-- reported by the worker sets `not_before`, the claim skips the round until
-- then, so no worker sleeps and any worker honours it:
-- * claim_sheet_verifications: rounds `verifying` with no live lease;
-- * get_sheet_verification_input: the frozen document, the base, the rows of
-- the drawings the document references;
@@ -17,11 +20,28 @@
-- ===========================================================================
-- 1. Constants
-- ===========================================================================
-- The earliest time a technical retry may be claimed (null = now). Hidden:
-- the M5 grant to `authenticated` is column-level and does not list it.
alter table public.anatomical_sheet_reviews add column not_before timestamptz;
comment on column public.anatomical_sheet_reviews.not_before is
'Earliest time a technical retry may be claimed; null = now. Set by fail_sheet_verification, cleared by the claim. Service role only.';
-- Attempts before a round closes `verification_failed` / `unavailable`
-- (§6.5 « five, proposal, tuned in V1-2.3 »).
create or replace function private.sheet_verification_attempt_limit()
returns integer language sql immutable set search_path = '' as $$ select 5 $$;
-- Wait before the next claim after a technical failure at attempt n: 2 s, 5 s,
-- 10 s, then 20 s (attempt 1 itself is claimed at once). Validation and
-- integrity refusals are verdicts, never retried.
create or replace function private.sheet_verification_retry_delay(attempts integer)
returns interval language sql immutable set search_path = '' as $$
select case when attempts <= 1 then interval '2 seconds'
when attempts = 2 then interval '5 seconds'
when attempts = 3 then interval '10 seconds'
else interval '20 seconds' end
$$;
-- A lease older than this is dead: the worker crashed or lost the database.
-- The child's wall-clock deadline is far below it (crates/worker limits).
create or replace function private.sheet_verification_lease()
@@ -191,12 +211,14 @@ begin
where r.status = 'verifying'
and (r.locked_at is null or r.locked_at < now() - private.sheet_verification_lease())
and r.attempts < private.sheet_verification_attempt_limit()
and (r.not_before is null or r.not_before <= now())
order by r.submitted_at
limit v_limit
for update skip locked
), leased as (
update public.anatomical_sheet_reviews r
set lease_token = gen_random_uuid(), locked_at = now(), attempts = r.attempts + 1,
not_before = null,
-- A dead lease left no code: record the interruption it was.
failure_code = case when r.locked_at is not null then 'verification.worker.lease_expired' else r.failure_code end
from picked
@@ -344,8 +366,8 @@ end;
$$;
-- A technical failure of a leased round (§6.5): the round stays `verifying`
-- and claimable, its code recorded, until the attempt limit closes it
-- `verification_failed` / `unavailable`. Answers `retry`, `unavailable` or
-- and claimable after its retry delay, its code recorded, until the attempt
-- limit closes it `verification_failed` / `unavailable`. Answers `retry`, `unavailable` or
-- `stale` (nothing written).
create or replace function public.fail_sheet_verification(review_id uuid, lease_token uuid, code text)
returns text
@@ -370,6 +392,8 @@ begin
v_final := v_review.attempts >= private.sheet_verification_attempt_limit();
update public.anatomical_sheet_reviews r
set failure_code = code, lease_token = null, locked_at = null,
not_before = case when v_final then null
else now() + private.sheet_verification_retry_delay(v_review.attempts) end,
status = case when v_final then 'verification_failed'::public.sheet_round_status else r.status end,
failure_outcome = case when v_final then 'unavailable'::public.sheet_verification_outcome else null end
where r.id = v_review.id;
@@ -377,7 +401,9 @@ begin
case when v_final then 'anatomical_sheet_review.verification_failed' else 'anatomical_sheet_review.verification_attempt_failed' end,
'anatomical_sheet_review', v_review.id, null, null,
jsonb_build_object('code', code, 'attempts', v_review.attempts,
'outcome', case when v_final then 'unavailable' end));
'outcome', case when v_final then 'unavailable' end,
'retry_delay_seconds', case when not v_final then
extract(epoch from private.sheet_verification_retry_delay(v_review.attempts))::integer end));
return case when v_final then 'unavailable' else 'retry' end;
end;
$$;
@@ -386,6 +412,7 @@ $$;
-- 4. Privileges: the service role only
-- ===========================================================================
revoke all on function private.sheet_verification_attempt_limit() from public;
revoke all on function private.sheet_verification_retry_delay(integer) from public;
revoke all on function private.sheet_verification_lease() from public;
revoke all on function private.sheet_verification_max_verdict_bytes() from public;
revoke all on function private.sheet_verdict_codes() from public;
+49 -9
View File
@@ -1,9 +1,10 @@
-- V1-2.3 M6: the worker RPCs of the server verdict (contract §6.4, §6.5).
-- Service role only; claim with a lease, input bound to the live token,
-- completion checked for its binding and structure (stale / malformed write
-- nothing), technical failures up to the attempt limit, dead leases.
-- nothing), technical failures up to the attempt limit with their progressive
-- delay (`not_before`), dead leases.
begin;
select plan(51);
select plan(68);
insert into auth.users (id, email, email_confirmed_at) values
('f0000000-0000-0000-0000-000000000001', 'own@vj.local', now()),
@@ -75,6 +76,20 @@ create function pg_temp.token(p text) returns uuid language sql security definer
select lease_token from public.anatomical_sheet_reviews where id = pg_temp.id(p);
$$;
grant execute on function pg_temp.token(text) to service_role;
-- `not_before` is hidden from the API roles: the helpers read and move it as
-- the test owner. now() is the transaction's, so the delay compares exactly.
create function pg_temp.delayed(p text, p_delay interval) returns boolean language sql security definer as $$
select not_before = now() + p_delay from public.anatomical_sheet_reviews where id = pg_temp.id(p);
$$;
grant execute on function pg_temp.delayed(text, interval) to service_role;
create function pg_temp.not_before_is_null(p text) returns boolean language sql security definer as $$
select not_before is null from public.anatomical_sheet_reviews where id = pg_temp.id(p);
$$;
grant execute on function pg_temp.not_before_is_null(text) to service_role;
create function pg_temp.set_not_before(p text, p_offset interval) returns void language sql security definer as $$
update public.anatomical_sheet_reviews set not_before = now() + p_offset where id = pg_temp.id(p);
$$;
grant execute on function pg_temp.set_not_before(text, interval) to service_role;
-- A sheet with one confirmed drawing; three drafts submitted (r1, r2, r3).
set local role authenticated;
@@ -103,6 +118,12 @@ select ok(has_function_privilege('service_role', 'public.claim_sheet_verificatio
and has_function_privilege('service_role', 'public.complete_sheet_verification(uuid, uuid, jsonb)', 'execute')
and has_function_privilege('service_role', 'public.fail_sheet_verification(uuid, uuid, text)', 'execute'),
'the service role executes the four RPCs');
select ok(not has_column_privilege('authenticated', 'public.anatomical_sheet_reviews', 'not_before', 'select')
and not has_column_privilege('anon', 'public.anatomical_sheet_reviews', 'not_before', 'select'),
'the API roles cannot read not_before');
select ok(not has_function_privilege('authenticated', 'private.sheet_verification_retry_delay(integer)', 'execute'), 'authenticated cannot execute the delay constant');
select is((select array_agg(extract(epoch from private.sheet_verification_retry_delay(n))::int order by n) from generate_series(1,5) n),
array[2,5,10,20,20], 'the delay after attempts 1 to 5: 2, 5, 10, 20, 20 seconds');
set local role authenticated;
select pg_temp.act('f0000000-0000-0000-0000-000000000001','f1000000-0000-0000-0000-000000000001');
select throws_ok($$ select public.claim_sheet_verifications(1) $$, '42501', null, 'the owner (authenticated) cannot claim');
@@ -175,6 +196,7 @@ select is((select count(*)::int from public.claim_sheet_verifications(1)), 1, 's
select is(public.complete_sheet_verification(pg_temp.id('r2'), pg_temp.token('r2'),
pg_temp.verdict(pg_temp.id('r2'),'failed','verification.integrity.document')), 'failed', 'a failed verdict without groups is written');
select is(pg_temp.status('r2'), 'verification_failed/failed/verification.integrity.document', 'round closed failed with its code');
select is((select count(*)::int from public.claim_sheet_verifications(10)), 0, 'a round completed with a failed verdict is never claimed again');
-- 7. Technical failures: retry below the limit, unavailable at it.
reset role;
@@ -190,19 +212,37 @@ select throws_ok(format('select public.fail_sheet_verification(%L, %L, %L)', pg_
select is(public.fail_sheet_verification(pg_temp.id('r3'), gen_random_uuid(), 'verification.worker.timeout'), 'stale', 'a failure with another token is stale');
select is(public.fail_sheet_verification(pg_temp.id('r3'), pg_temp.token('r3'), 'verification.worker.timeout'), 'retry', 'a timeout below the limit is retried');
select is(pg_temp.status('r3') || '/' || (pg_temp.token('r3') is null)::text, 'verifying/-/verification.worker.timeout/true', 'round stays verifying, code recorded, lease released');
-- A dead lease is claimed again and recorded as an interruption.
select is((select attempts from public.claim_sheet_verifications(1)), 2, 'reclaimed: second attempt');
select ok(pg_temp.delayed('r3', interval '2 seconds'), 'after attempt 1 the retry waits 2 s');
reset role;
select ok(exists (select 1 from audit.audit_logs where action = 'anatomical_sheet_review.verification_attempt_failed' and resource_id = pg_temp.id('r3')
and metadata ->> 'retry_delay_seconds' = '2'), 'attempt failure audited with its delay');
set local role service_role; select pg_temp.worker();
select is(public.fail_sheet_verification(pg_temp.id('r3'), gen_random_uuid(), 'verification.worker.timeout'), 'stale', 'a stale failure again');
select ok(pg_temp.delayed('r3', interval '2 seconds'), 'a stale failure leaves the delay unchanged');
select is((select count(*)::int from public.claim_sheet_verifications(1)), 0, 'no claim before the delay is over');
-- The delay over exactly now: claimable.
select pg_temp.set_not_before('r3', interval '0');
select is((select attempts from public.claim_sheet_verifications(1)), 2, 'claimed once not_before is reached: second attempt');
select ok(pg_temp.not_before_is_null('r3'), 'the claim clears the delay');
select is(public.fail_sheet_verification(pg_temp.id('r3'), pg_temp.token('r3'), 'verification.worker.unavailable'), 'retry', 'a second failure is retried');
select ok(pg_temp.delayed('r3', interval '5 seconds'), 'after attempt 2 the retry waits 5 s');
select pg_temp.set_not_before('r3', interval '-1 second');
select is((select attempts from public.claim_sheet_verifications(1)), 3, 'claimed after 5 s: third attempt');
-- A dead lease is claimed again at once and recorded as an interruption.
reset role;
update public.anatomical_sheet_reviews set locked_at = now() - interval '11 minutes' where id = pg_temp.id('r3');
set local role service_role; select pg_temp.worker();
select is((select attempts from public.claim_sheet_verifications(1)), 3, 'a dead lease is claimed again');
select is((select attempts from public.claim_sheet_verifications(1)), 4, 'a dead lease is claimed again');
select is(pg_temp.status('r3'), 'verifying/-/verification.worker.lease_expired', 'the interruption is recorded');
-- At the limit, a technical failure closes the round unavailable.
reset role;
update public.anatomical_sheet_reviews set attempts = 5 where id = pg_temp.id('r3');
set local role service_role; select pg_temp.worker();
select is(public.fail_sheet_verification(pg_temp.id('r3'), pg_temp.token('r3'), 'verification.worker.resource_limit'), 'retry', 'a fourth failure is retried');
select ok(pg_temp.delayed('r3', interval '20 seconds'), 'after attempt 4 the retry waits 20 s');
select pg_temp.set_not_before('r3', interval '-1 second');
select is((select attempts from public.claim_sheet_verifications(1)), 5, 'claimed after 20 s: fifth attempt');
-- At the limit, a technical failure closes the round unavailable, delay cleared.
select pg_temp.set_not_before('r3', interval '1 hour');
select is(public.fail_sheet_verification(pg_temp.id('r3'), pg_temp.token('r3'), 'verification.worker.resource_limit'), 'unavailable', 'the last attempt closes the round');
select is(pg_temp.status('r3'), 'verification_failed/unavailable/verification.worker.resource_limit', 'round closed unavailable with the last code');
select ok(pg_temp.not_before_is_null('r3'), 'the fifth failure clears the delay');
reset role;
select ok(exists (select 1 from audit.audit_logs where action = 'anatomical_sheet_review.verification_attempt_failed' and resource_id = pg_temp.id('r3')), 'attempt failure audited');