feat(supabase): retry delay of 2, 5, 10, 20 s for technical verification failures
M6 amended in place (applied only to the throwaway sandbox): a hidden not_before on the round, set by fail_sheet_verification and honoured by the claim; RPC signatures unchanged. pgTAP 021 (68), sandbox cases i (delay with the real worker), j (inputs at the caps, 2 MiB base), k (just beyond).
This commit is contained in:
1 parent
e967ce6736
commit
5adad3709f
8 files changed
+398
-29
No files matched your search
@@ -4,7 +4,10 @@
|
||||
--
|
||||
-- The round row (M5 `anatomical_sheet_reviews`) is the job, with the avatar
|
||||
-- lease pattern of 20261013100000_files_foundation.sql (lease token,
|
||||
-- `locked_at`, a fixed lease, attempts below a limit), without backoff:
|
||||
-- `locked_at`, a fixed lease, attempts below a limit), plus a progressive
|
||||
-- delay for technical retries only (owner decision 2026-11-01): a failure
|
||||
-- reported by the worker sets `not_before`, the claim skips the round until
|
||||
-- then, so no worker sleeps and any worker honours it:
|
||||
-- * claim_sheet_verifications: rounds `verifying` with no live lease;
|
||||
-- * get_sheet_verification_input: the frozen document, the base, the rows of
|
||||
-- the drawings the document references;
|
||||
@@ -17,11 +20,28 @@
|
||||
-- ===========================================================================
|
||||
-- 1. Constants
|
||||
-- ===========================================================================
|
||||
-- The earliest time a technical retry may be claimed (null = now). Hidden:
|
||||
-- the M5 grant to `authenticated` is column-level and does not list it.
|
||||
alter table public.anatomical_sheet_reviews add column not_before timestamptz;
|
||||
comment on column public.anatomical_sheet_reviews.not_before is
|
||||
'Earliest time a technical retry may be claimed; null = now. Set by fail_sheet_verification, cleared by the claim. Service role only.';
|
||||
|
||||
-- Attempts before a round closes `verification_failed` / `unavailable`
|
||||
-- (§6.5 « five, proposal, tuned in V1-2.3 »).
|
||||
create or replace function private.sheet_verification_attempt_limit()
|
||||
returns integer language sql immutable set search_path = '' as $$ select 5 $$;
|
||||
|
||||
-- Wait before the next claim after a technical failure at attempt n: 2 s, 5 s,
|
||||
-- 10 s, then 20 s (attempt 1 itself is claimed at once). Validation and
|
||||
-- integrity refusals are verdicts, never retried.
|
||||
create or replace function private.sheet_verification_retry_delay(attempts integer)
|
||||
returns interval language sql immutable set search_path = '' as $$
|
||||
select case when attempts <= 1 then interval '2 seconds'
|
||||
when attempts = 2 then interval '5 seconds'
|
||||
when attempts = 3 then interval '10 seconds'
|
||||
else interval '20 seconds' end
|
||||
$$;
|
||||
|
||||
-- A lease older than this is dead: the worker crashed or lost the database.
|
||||
-- The child's wall-clock deadline is far below it (crates/worker limits).
|
||||
create or replace function private.sheet_verification_lease()
|
||||
@@ -191,12 +211,14 @@ begin
|
||||
where r.status = 'verifying'
|
||||
and (r.locked_at is null or r.locked_at < now() - private.sheet_verification_lease())
|
||||
and r.attempts < private.sheet_verification_attempt_limit()
|
||||
and (r.not_before is null or r.not_before <= now())
|
||||
order by r.submitted_at
|
||||
limit v_limit
|
||||
for update skip locked
|
||||
), leased as (
|
||||
update public.anatomical_sheet_reviews r
|
||||
set lease_token = gen_random_uuid(), locked_at = now(), attempts = r.attempts + 1,
|
||||
not_before = null,
|
||||
-- A dead lease left no code: record the interruption it was.
|
||||
failure_code = case when r.locked_at is not null then 'verification.worker.lease_expired' else r.failure_code end
|
||||
from picked
|
||||
@@ -344,8 +366,8 @@ end;
|
||||
$$;
|
||||
|
||||
-- A technical failure of a leased round (§6.5): the round stays `verifying`
|
||||
-- and claimable, its code recorded, until the attempt limit closes it
|
||||
-- `verification_failed` / `unavailable`. Answers `retry`, `unavailable` or
|
||||
-- and claimable after its retry delay, its code recorded, until the attempt
|
||||
-- limit closes it `verification_failed` / `unavailable`. Answers `retry`, `unavailable` or
|
||||
-- `stale` (nothing written).
|
||||
create or replace function public.fail_sheet_verification(review_id uuid, lease_token uuid, code text)
|
||||
returns text
|
||||
@@ -370,6 +392,8 @@ begin
|
||||
v_final := v_review.attempts >= private.sheet_verification_attempt_limit();
|
||||
update public.anatomical_sheet_reviews r
|
||||
set failure_code = code, lease_token = null, locked_at = null,
|
||||
not_before = case when v_final then null
|
||||
else now() + private.sheet_verification_retry_delay(v_review.attempts) end,
|
||||
status = case when v_final then 'verification_failed'::public.sheet_round_status else r.status end,
|
||||
failure_outcome = case when v_final then 'unavailable'::public.sheet_verification_outcome else null end
|
||||
where r.id = v_review.id;
|
||||
@@ -377,7 +401,9 @@ begin
|
||||
case when v_final then 'anatomical_sheet_review.verification_failed' else 'anatomical_sheet_review.verification_attempt_failed' end,
|
||||
'anatomical_sheet_review', v_review.id, null, null,
|
||||
jsonb_build_object('code', code, 'attempts', v_review.attempts,
|
||||
'outcome', case when v_final then 'unavailable' end));
|
||||
'outcome', case when v_final then 'unavailable' end,
|
||||
'retry_delay_seconds', case when not v_final then
|
||||
extract(epoch from private.sheet_verification_retry_delay(v_review.attempts))::integer end));
|
||||
return case when v_final then 'unavailable' else 'retry' end;
|
||||
end;
|
||||
$$;
|
||||
@@ -386,6 +412,7 @@ $$;
|
||||
-- 4. Privileges: the service role only
|
||||
-- ===========================================================================
|
||||
revoke all on function private.sheet_verification_attempt_limit() from public;
|
||||
revoke all on function private.sheet_verification_retry_delay(integer) from public;
|
||||
revoke all on function private.sheet_verification_lease() from public;
|
||||
revoke all on function private.sheet_verification_max_verdict_bytes() from public;
|
||||
revoke all on function private.sheet_verdict_codes() from public;
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
-- V1-2.3 M6: the worker RPCs of the server verdict (contract §6.4, §6.5).
|
||||
-- Service role only; claim with a lease, input bound to the live token,
|
||||
-- completion checked for its binding and structure (stale / malformed write
|
||||
-- nothing), technical failures up to the attempt limit, dead leases.
|
||||
-- nothing), technical failures up to the attempt limit with their progressive
|
||||
-- delay (`not_before`), dead leases.
|
||||
begin;
|
||||
select plan(51);
|
||||
select plan(68);
|
||||
|
||||
insert into auth.users (id, email, email_confirmed_at) values
|
||||
('f0000000-0000-0000-0000-000000000001', 'own@vj.local', now()),
|
||||
@@ -75,6 +76,20 @@ create function pg_temp.token(p text) returns uuid language sql security definer
|
||||
select lease_token from public.anatomical_sheet_reviews where id = pg_temp.id(p);
|
||||
$$;
|
||||
grant execute on function pg_temp.token(text) to service_role;
|
||||
-- `not_before` is hidden from the API roles: the helpers read and move it as
|
||||
-- the test owner. now() is the transaction's, so the delay compares exactly.
|
||||
create function pg_temp.delayed(p text, p_delay interval) returns boolean language sql security definer as $$
|
||||
select not_before = now() + p_delay from public.anatomical_sheet_reviews where id = pg_temp.id(p);
|
||||
$$;
|
||||
grant execute on function pg_temp.delayed(text, interval) to service_role;
|
||||
create function pg_temp.not_before_is_null(p text) returns boolean language sql security definer as $$
|
||||
select not_before is null from public.anatomical_sheet_reviews where id = pg_temp.id(p);
|
||||
$$;
|
||||
grant execute on function pg_temp.not_before_is_null(text) to service_role;
|
||||
create function pg_temp.set_not_before(p text, p_offset interval) returns void language sql security definer as $$
|
||||
update public.anatomical_sheet_reviews set not_before = now() + p_offset where id = pg_temp.id(p);
|
||||
$$;
|
||||
grant execute on function pg_temp.set_not_before(text, interval) to service_role;
|
||||
|
||||
-- A sheet with one confirmed drawing; three drafts submitted (r1, r2, r3).
|
||||
set local role authenticated;
|
||||
@@ -103,6 +118,12 @@ select ok(has_function_privilege('service_role', 'public.claim_sheet_verificatio
|
||||
and has_function_privilege('service_role', 'public.complete_sheet_verification(uuid, uuid, jsonb)', 'execute')
|
||||
and has_function_privilege('service_role', 'public.fail_sheet_verification(uuid, uuid, text)', 'execute'),
|
||||
'the service role executes the four RPCs');
|
||||
select ok(not has_column_privilege('authenticated', 'public.anatomical_sheet_reviews', 'not_before', 'select')
|
||||
and not has_column_privilege('anon', 'public.anatomical_sheet_reviews', 'not_before', 'select'),
|
||||
'the API roles cannot read not_before');
|
||||
select ok(not has_function_privilege('authenticated', 'private.sheet_verification_retry_delay(integer)', 'execute'), 'authenticated cannot execute the delay constant');
|
||||
select is((select array_agg(extract(epoch from private.sheet_verification_retry_delay(n))::int order by n) from generate_series(1,5) n),
|
||||
array[2,5,10,20,20], 'the delay after attempts 1 to 5: 2, 5, 10, 20, 20 seconds');
|
||||
set local role authenticated;
|
||||
select pg_temp.act('f0000000-0000-0000-0000-000000000001','f1000000-0000-0000-0000-000000000001');
|
||||
select throws_ok($$ select public.claim_sheet_verifications(1) $$, '42501', null, 'the owner (authenticated) cannot claim');
|
||||
@@ -175,6 +196,7 @@ select is((select count(*)::int from public.claim_sheet_verifications(1)), 1, 's
|
||||
select is(public.complete_sheet_verification(pg_temp.id('r2'), pg_temp.token('r2'),
|
||||
pg_temp.verdict(pg_temp.id('r2'),'failed','verification.integrity.document')), 'failed', 'a failed verdict without groups is written');
|
||||
select is(pg_temp.status('r2'), 'verification_failed/failed/verification.integrity.document', 'round closed failed with its code');
|
||||
select is((select count(*)::int from public.claim_sheet_verifications(10)), 0, 'a round completed with a failed verdict is never claimed again');
|
||||
|
||||
-- 7. Technical failures: retry below the limit, unavailable at it.
|
||||
reset role;
|
||||
@@ -190,19 +212,37 @@ select throws_ok(format('select public.fail_sheet_verification(%L, %L, %L)', pg_
|
||||
select is(public.fail_sheet_verification(pg_temp.id('r3'), gen_random_uuid(), 'verification.worker.timeout'), 'stale', 'a failure with another token is stale');
|
||||
select is(public.fail_sheet_verification(pg_temp.id('r3'), pg_temp.token('r3'), 'verification.worker.timeout'), 'retry', 'a timeout below the limit is retried');
|
||||
select is(pg_temp.status('r3') || '/' || (pg_temp.token('r3') is null)::text, 'verifying/-/verification.worker.timeout/true', 'round stays verifying, code recorded, lease released');
|
||||
-- A dead lease is claimed again and recorded as an interruption.
|
||||
select is((select attempts from public.claim_sheet_verifications(1)), 2, 'reclaimed: second attempt');
|
||||
select ok(pg_temp.delayed('r3', interval '2 seconds'), 'after attempt 1 the retry waits 2 s');
|
||||
reset role;
|
||||
select ok(exists (select 1 from audit.audit_logs where action = 'anatomical_sheet_review.verification_attempt_failed' and resource_id = pg_temp.id('r3')
|
||||
and metadata ->> 'retry_delay_seconds' = '2'), 'attempt failure audited with its delay');
|
||||
set local role service_role; select pg_temp.worker();
|
||||
select is(public.fail_sheet_verification(pg_temp.id('r3'), gen_random_uuid(), 'verification.worker.timeout'), 'stale', 'a stale failure again');
|
||||
select ok(pg_temp.delayed('r3', interval '2 seconds'), 'a stale failure leaves the delay unchanged');
|
||||
select is((select count(*)::int from public.claim_sheet_verifications(1)), 0, 'no claim before the delay is over');
|
||||
-- The delay over exactly now: claimable.
|
||||
select pg_temp.set_not_before('r3', interval '0');
|
||||
select is((select attempts from public.claim_sheet_verifications(1)), 2, 'claimed once not_before is reached: second attempt');
|
||||
select ok(pg_temp.not_before_is_null('r3'), 'the claim clears the delay');
|
||||
select is(public.fail_sheet_verification(pg_temp.id('r3'), pg_temp.token('r3'), 'verification.worker.unavailable'), 'retry', 'a second failure is retried');
|
||||
select ok(pg_temp.delayed('r3', interval '5 seconds'), 'after attempt 2 the retry waits 5 s');
|
||||
select pg_temp.set_not_before('r3', interval '-1 second');
|
||||
select is((select attempts from public.claim_sheet_verifications(1)), 3, 'claimed after 5 s: third attempt');
|
||||
-- A dead lease is claimed again at once and recorded as an interruption.
|
||||
reset role;
|
||||
update public.anatomical_sheet_reviews set locked_at = now() - interval '11 minutes' where id = pg_temp.id('r3');
|
||||
set local role service_role; select pg_temp.worker();
|
||||
select is((select attempts from public.claim_sheet_verifications(1)), 3, 'a dead lease is claimed again');
|
||||
select is((select attempts from public.claim_sheet_verifications(1)), 4, 'a dead lease is claimed again');
|
||||
select is(pg_temp.status('r3'), 'verifying/-/verification.worker.lease_expired', 'the interruption is recorded');
|
||||
-- At the limit, a technical failure closes the round unavailable.
|
||||
reset role;
|
||||
update public.anatomical_sheet_reviews set attempts = 5 where id = pg_temp.id('r3');
|
||||
set local role service_role; select pg_temp.worker();
|
||||
select is(public.fail_sheet_verification(pg_temp.id('r3'), pg_temp.token('r3'), 'verification.worker.resource_limit'), 'retry', 'a fourth failure is retried');
|
||||
select ok(pg_temp.delayed('r3', interval '20 seconds'), 'after attempt 4 the retry waits 20 s');
|
||||
select pg_temp.set_not_before('r3', interval '-1 second');
|
||||
select is((select attempts from public.claim_sheet_verifications(1)), 5, 'claimed after 20 s: fifth attempt');
|
||||
-- At the limit, a technical failure closes the round unavailable, delay cleared.
|
||||
select pg_temp.set_not_before('r3', interval '1 hour');
|
||||
select is(public.fail_sheet_verification(pg_temp.id('r3'), pg_temp.token('r3'), 'verification.worker.resource_limit'), 'unavailable', 'the last attempt closes the round');
|
||||
select is(pg_temp.status('r3'), 'verification_failed/unavailable/verification.worker.resource_limit', 'round closed unavailable with the last code');
|
||||
select ok(pg_temp.not_before_is_null('r3'), 'the fifth failure clears the delay');
|
||||
reset role;
|
||||
select ok(exists (select 1 from audit.audit_logs where action = 'anatomical_sheet_review.verification_attempt_failed' and resource_id = pg_temp.id('r3')), 'attempt failure audited');
|
||||
|
||||
|
||||
Reference in new issue
Block a user