feat(supabase): verification retry by effective cause
resource_limit (intrinsic to the frozen input: frame, input or output cap, SIGXCPU, SIGABRT, SIGSYS) and verdict_refused close the round at the first occurrence; timeout, unavailable (now also an external SIGKILL) and lease_expired keep the 2/5/10/20 s retries up to five attempts. pgTAP 021: 89; sandbox case k closes at attempt 1.
This commit is contained in:
1 parent
c98a9c8f82
commit
a5844b9d56
8 files changed
+156
-46
No files matched your search
@@ -13,7 +13,9 @@
|
||||
-- the drawings the document references;
|
||||
-- * complete_sheet_verification: the verdict, checked for its binding and
|
||||
-- its structure; `stale` or `malformed` write nothing;
|
||||
-- * fail_sheet_verification: a technical failure, from a closed list.
|
||||
-- * fail_sheet_verification: a technical failure, from a closed list;
|
||||
-- transient codes retry after a delay up to the attempt limit, the
|
||||
-- definitive codes (`sheet_worker_final_codes`) close the round at once.
|
||||
-- The verdict itself is the core's (`compute_verdict`, crates/core); SQL
|
||||
-- checks only that it answers this round and covers this document.
|
||||
|
||||
@@ -67,7 +69,8 @@ returns text[] language sql immutable set search_path = '' as $$
|
||||
$$;
|
||||
|
||||
-- The worker's technical failure codes (§6.5 transient, timeout, resource,
|
||||
-- interruption rows; crates/worker `verify_job`).
|
||||
-- interruption rows; crates/worker `verify_job`). The worker maps the cause
|
||||
-- to a code; the retry policy of each code is `sheet_worker_final_codes`.
|
||||
create or replace function private.sheet_worker_codes()
|
||||
returns text[] language sql immutable set search_path = '' as $$
|
||||
select array[
|
||||
@@ -77,6 +80,15 @@ returns text[] language sql immutable set search_path = '' as $$
|
||||
]
|
||||
$$;
|
||||
|
||||
-- The codes whose cause is fixed by the frozen input or the contract caps
|
||||
-- (resource_limit) or by a verdict the SQL refuses (verdict_refused): a retry
|
||||
-- would give the same answer, so the round closes at the first occurrence.
|
||||
-- The others (timeout, unavailable, lease_expired) are transient.
|
||||
create or replace function private.sheet_worker_final_codes()
|
||||
returns text[] language sql immutable set search_path = '' as $$
|
||||
select array['verification.worker.resource_limit', 'verification.worker.verdict_refused']
|
||||
$$;
|
||||
|
||||
-- The service role only (the worker's key); anything else is refused.
|
||||
create or replace function private.require_service_role()
|
||||
returns void language plpgsql stable set search_path = '' as $$
|
||||
@@ -367,9 +379,10 @@ begin
|
||||
end;
|
||||
$$;
|
||||
|
||||
-- A technical failure of a leased round (§6.5): the round stays `verifying`
|
||||
-- and claimable after its retry delay, its code recorded, until the attempt
|
||||
-- limit closes it `verification_failed` / `unavailable`. Answers `retry`, `unavailable` or
|
||||
-- A technical failure of a leased round (§6.5): a transient code leaves the
|
||||
-- round `verifying` and claimable after its retry delay, its code recorded,
|
||||
-- until the attempt limit closes it `verification_failed` / `unavailable`; a
|
||||
-- definitive code closes it at once. Answers `retry`, `unavailable` or
|
||||
-- `stale` (nothing written).
|
||||
create or replace function public.fail_sheet_verification(review_id uuid, lease_token uuid, code text)
|
||||
returns text
|
||||
@@ -382,6 +395,7 @@ as $$
|
||||
declare
|
||||
v_review public.anatomical_sheet_reviews%rowtype;
|
||||
v_final boolean;
|
||||
v_definitive boolean;
|
||||
begin
|
||||
perform private.require_service_role();
|
||||
if code is null or not (code = any (private.sheet_worker_codes())) then
|
||||
@@ -392,7 +406,9 @@ begin
|
||||
or lease_token is null or v_review.lease_token is distinct from lease_token then
|
||||
return 'stale';
|
||||
end if;
|
||||
v_final := v_review.attempts >= private.sheet_verification_attempt_limit();
|
||||
v_definitive := code = any (private.sheet_worker_final_codes());
|
||||
v_final := v_definitive
|
||||
or v_review.attempts >= private.sheet_verification_attempt_limit();
|
||||
update public.anatomical_sheet_reviews r
|
||||
set failure_code = code, lease_token = null, locked_at = null,
|
||||
not_before = case when v_final then null
|
||||
@@ -405,6 +421,9 @@ begin
|
||||
'anatomical_sheet_review', v_review.id, null, null,
|
||||
jsonb_build_object('code', code, 'attempts', v_review.attempts,
|
||||
'outcome', case when v_final then 'unavailable' end,
|
||||
'closed_by', case when not v_final then null
|
||||
when v_definitive then 'definitive'
|
||||
else 'attempt_limit' end,
|
||||
'retry_delay_seconds', case when not v_final then
|
||||
extract(epoch from private.sheet_verification_retry_delay(v_review.attempts))::integer end));
|
||||
return case when v_final then 'unavailable' else 'retry' end;
|
||||
@@ -420,6 +439,7 @@ revoke all on function private.sheet_verification_lease() from public;
|
||||
revoke all on function private.sheet_verification_max_verdict_bytes() from public;
|
||||
revoke all on function private.sheet_verdict_codes() from public;
|
||||
revoke all on function private.sheet_worker_codes() from public;
|
||||
revoke all on function private.sheet_worker_final_codes() from public;
|
||||
revoke all on function private.require_service_role() from public;
|
||||
revoke all on function private.sheet_document_group_keys(jsonb) from public;
|
||||
revoke all on function private.sheet_review_base(uuid) from public;
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
-- nothing), technical failures up to the attempt limit with their progressive
|
||||
-- delay (`not_before`), dead leases.
|
||||
begin;
|
||||
select plan(72);
|
||||
select plan(89);
|
||||
|
||||
insert into auth.users (id, email, email_confirmed_at) values
|
||||
('f0000000-0000-0000-0000-000000000001', 'own@vj.local', now()),
|
||||
@@ -239,17 +239,19 @@ update public.anatomical_sheet_reviews set locked_at = now() - interval '11 minu
|
||||
set local role service_role; select pg_temp.worker();
|
||||
select is((select attempts from public.claim_sheet_verifications(1)), 4, 'a dead lease is claimed again');
|
||||
select is(pg_temp.status('r3'), 'verifying/-/verification.worker.lease_expired', 'the interruption is recorded');
|
||||
select is(public.fail_sheet_verification(pg_temp.id('r3'), pg_temp.token('r3'), 'verification.worker.resource_limit'), 'retry', 'a fourth failure is retried');
|
||||
select is(public.fail_sheet_verification(pg_temp.id('r3'), pg_temp.token('r3'), 'verification.worker.timeout'), 'retry', 'a fourth failure is retried');
|
||||
select ok(pg_temp.delayed('r3', interval '20 seconds'), 'after attempt 4 the retry waits 20 s');
|
||||
select pg_temp.set_not_before('r3', interval '-1 second');
|
||||
select is((select attempts from public.claim_sheet_verifications(1)), 5, 'claimed after 20 s: fifth attempt');
|
||||
-- At the limit, a technical failure closes the round unavailable, delay cleared.
|
||||
select pg_temp.set_not_before('r3', interval '1 hour');
|
||||
select is(public.fail_sheet_verification(pg_temp.id('r3'), pg_temp.token('r3'), 'verification.worker.resource_limit'), 'unavailable', 'the last attempt closes the round');
|
||||
select is(pg_temp.status('r3'), 'verification_failed/unavailable/verification.worker.resource_limit', 'round closed unavailable with the last code');
|
||||
select is(public.fail_sheet_verification(pg_temp.id('r3'), pg_temp.token('r3'), 'verification.worker.unavailable'), 'unavailable', 'the last attempt closes the round');
|
||||
select is(pg_temp.status('r3'), 'verification_failed/unavailable/verification.worker.unavailable', 'round closed unavailable with the last code');
|
||||
select ok(pg_temp.not_before_is_null('r3'), 'the fifth failure clears the delay');
|
||||
reset role;
|
||||
select ok(exists (select 1 from audit.audit_logs where action = 'anatomical_sheet_review.verification_attempt_failed' and resource_id = pg_temp.id('r3')), 'attempt failure audited');
|
||||
select ok(exists (select 1 from audit.audit_logs where action = 'anatomical_sheet_review.verification_failed' and resource_id = pg_temp.id('r3')
|
||||
and metadata ->> 'closed_by' = 'attempt_limit'), 'a transient code closes at the limit: closed_by attempt_limit');
|
||||
|
||||
-- 8. A dead lease at the limit is closed by the next claim.
|
||||
set local role authenticated;
|
||||
@@ -262,5 +264,46 @@ set local role service_role; select pg_temp.worker();
|
||||
select is((select count(*)::int from public.claim_sheet_verifications(5)), 0, 'a round at the limit is not claimed');
|
||||
select is(pg_temp.status('r4'), 'verification_failed/unavailable/verification.worker.lease_expired', 'its dead lease closes it unavailable');
|
||||
|
||||
-- 9. Definitive codes close the round at the first occurrence; transient
|
||||
-- codes keep their retry. Rounds are sequential: each follows a closed one.
|
||||
set local role authenticated;
|
||||
select pg_temp.act('f0000000-0000-0000-0000-00000000000e','f1000000-0000-0000-0000-00000000000e');
|
||||
select public.save_sheet_draft(pg_temp.id('v1'), 4, pg_temp.doc(pg_temp.id('asset'),'VJ5'), 'five');
|
||||
insert into pg_temp.k select 'r5', public.submit_sheet_review(pg_temp.id('v1'), 5, 'five');
|
||||
reset role;
|
||||
set local role service_role; select pg_temp.worker();
|
||||
select is((select count(*)::int from public.claim_sheet_verifications(1)), 1, 'fifth round claimed');
|
||||
select is(public.fail_sheet_verification(pg_temp.id('r5'), pg_temp.token('r5'), 'verification.worker.resource_limit'), 'unavailable', 'resource_limit at attempt 1 closes the round');
|
||||
select is(pg_temp.status('r5'), 'verification_failed/unavailable/verification.worker.resource_limit', 'closed unavailable, code kept');
|
||||
select ok(pg_temp.not_before_is_null('r5'), 'a definitive closure sets no delay');
|
||||
select is((select count(*)::int from public.claim_sheet_verifications(10)), 0, 'a definitively closed round is never claimed again');
|
||||
reset role;
|
||||
select ok(exists (select 1 from audit.audit_logs where action = 'anatomical_sheet_review.verification_failed' and resource_id = pg_temp.id('r5')
|
||||
and metadata ->> 'closed_by' = 'definitive' and metadata ->> 'attempts' = '1'), 'closure audited as definitive at attempt 1');
|
||||
|
||||
set local role authenticated;
|
||||
select pg_temp.act('f0000000-0000-0000-0000-00000000000e','f1000000-0000-0000-0000-00000000000e');
|
||||
select public.save_sheet_draft(pg_temp.id('v1'), 5, pg_temp.doc(pg_temp.id('asset'),'VJ6'), 'six');
|
||||
insert into pg_temp.k select 'r6', public.submit_sheet_review(pg_temp.id('v1'), 6, 'six');
|
||||
reset role;
|
||||
set local role service_role; select pg_temp.worker();
|
||||
select is((select count(*)::int from public.claim_sheet_verifications(1)), 1, 'sixth round claimed');
|
||||
select is(public.fail_sheet_verification(pg_temp.id('r6'), pg_temp.token('r6'), 'verification.worker.verdict_refused'), 'unavailable', 'verdict_refused at attempt 1 closes the round');
|
||||
select is(pg_temp.status('r6'), 'verification_failed/unavailable/verification.worker.verdict_refused', 'closed unavailable, code kept');
|
||||
select is((select count(*)::int from public.claim_sheet_verifications(10)), 0, 'a refused verdict is never claimed again');
|
||||
|
||||
set local role authenticated;
|
||||
select pg_temp.act('f0000000-0000-0000-0000-00000000000e','f1000000-0000-0000-0000-00000000000e');
|
||||
select public.save_sheet_draft(pg_temp.id('v1'), 6, pg_temp.doc(pg_temp.id('asset'),'VJ7'), 'seven');
|
||||
insert into pg_temp.k select 'r7', public.submit_sheet_review(pg_temp.id('v1'), 7, 'seven');
|
||||
reset role;
|
||||
set local role service_role; select pg_temp.worker();
|
||||
select is((select count(*)::int from public.claim_sheet_verifications(1)), 1, 'seventh round claimed');
|
||||
select is(public.fail_sheet_verification(pg_temp.id('r7'), gen_random_uuid(), 'verification.worker.resource_limit'), 'stale', 'a definitive code with another token is stale');
|
||||
select is(public.fail_sheet_verification(pg_temp.id('r7'), null, 'verification.worker.verdict_refused'), 'stale', 'a definitive code with a null token is stale');
|
||||
select is(pg_temp.status('r7'), 'verifying/-/-', 'a stale definitive failure writes nothing');
|
||||
select is(public.fail_sheet_verification(pg_temp.id('r7'), pg_temp.token('r7'), 'verification.worker.unavailable'), 'retry', 'unavailable at attempt 1 is retried');
|
||||
select ok(pg_temp.delayed('r7', interval '2 seconds'), 'after attempt 1 the retry waits 2 s');
|
||||
|
||||
select * from finish();
|
||||
rollback;
|
||||
Reference in new issue
Block a user