Files
faunosteo/crates/worker/tests/verify_loop.rs
T
bchanot a5844b9d56 feat(supabase): verification retry by effective cause
resource_limit (intrinsic to the frozen input: frame, input or output cap,
SIGXCPU, SIGABRT, SIGSYS) and verdict_refused close the round at the first
occurrence; timeout, unavailable (now also an external SIGKILL) and
lease_expired keep the 2/5/10/20 s retries up to five attempts. pgTAP 021: 89;
sandbox case k closes at attempt 1.
2026-11-01 23:01:30 +01:00

566 lines
19 KiB
Rust

//! V1-2.3 proofs of the verification job (contract W4, V1-2 contract §6.5) on an in-memory
//! `VerifyBackend`: the core's verdict computed in process from the very frame the parent
//! builds (or the real child without seccomp under `test-hooks`), every child outcome mapped
//! to its closed worker code, a missing drawing judged by the core, never by the worker.
#![allow(clippy::unwrap_used, clippy::expect_used)]
use std::cell::RefCell;
use std::collections::VecDeque;
use faunosteo_core::domain::sheet::verdict::{codes as verdict_codes, sha256_hex};
use faunosteo_worker::backend::{
BackendError, BackendResult, CompleteAnswer, DrawingRow, FailAnswer, VerifyBackend,
VerifyInput, VerifyJob,
};
use faunosteo_worker::config::Config;
use faunosteo_worker::limits::VERIFY_MAX_DRAWING_BYTES;
use faunosteo_worker::run::{Mode, parse_args, verify_loop};
use faunosteo_worker::spawn::{ChildOutcome, ChildRun};
use faunosteo_worker::verifier::Verifier;
use faunosteo_worker::verify_job::{DRAWING_BUCKET, VerifyEnd, codes, process_verification};
use faunosteo_worker::watchdog::Watchdog;
use faunosteo_worker::{exit_codes, verify};
use serde_json::{Value, json};
const REVIEW: &str = "aaaaaaaa-0000-4000-8000-000000000003";
const VERSION: &str = "aaaaaaaa-0000-4000-8000-000000000002";
const SHEET: &str = "aaaaaaaa-0000-4000-8000-000000000001";
const TOKEN: &str = "30000000-0000-0000-0000-00000000000c";
const ASSET: &str = "11111111-2222-4333-8444-555555555555";
const DRAWING: &str = concat!(
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n",
r#"<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 10 10"><path d="M0 0L5 5"/></svg>"#
);
fn name(text: &str) -> Value {
json!({ "fr": text, "en": text })
}
/// The core's small valid sheet (`crates/core/tests/sheet_verdict.rs::document`).
fn document() -> Value {
json!({
"schema_version": 2, "species_id": "canine", "name": name("Chien"),
"description": {}, "scope_disclaimer": { "fr": "Outil", "en": "Tool" },
"attribution": { "authors": ["A"], "license": null, "notes": null },
"vertebral_config": { "regions": [] }, "references": [],
"categories": [{ "id": "c", "name": name("C"), "color": null }],
"views": [{ "id": "main", "kind": "main", "name": name("Vue"), "svg_asset_id": ASSET, "view_box": null }],
"annotations": [{ "id": "a1", "kind": "point", "view_id": "main", "category_id": "c",
"laterality": "unpaired", "geometry": { "type": "circle", "cx": 1, "cy": 1, "r": 1 },
"review_status": "to_confirm", "name": name("P"), "finding_zone_id": "z1", "vertebral_region_code": null }],
"finding_zones": [{ "id": "z1", "name": name("Z"), "categories": [{ "id": "cat", "kind": "dysfunction",
"name": name("D"), "review_status": "to_confirm",
"children": [{ "id": "n1", "name": name("N"), "review_status": "to_confirm" }] }] }]
})
}
fn job() -> VerifyJob {
VerifyJob {
review_id: REVIEW.to_owned(),
version_id: VERSION.to_owned(),
sheet_id: SHEET.to_owned(),
revision: 3,
lease_token: TOKEN.to_owned(),
attempts: 1,
}
}
fn row() -> DrawingRow {
let sha256 = sha256_hex(DRAWING.as_bytes());
DrawingRow {
asset_id: ASSET.to_owned(),
object_key: format!("{sha256}.svg"),
sha256,
byte_size: DRAWING.len() as u64,
bucket: DRAWING_BUCKET.to_owned(),
uploaded: true,
}
}
fn input(document: String, drawings: Vec<DrawingRow>) -> VerifyInput {
VerifyInput {
review_id: REVIEW.to_owned(),
version_id: VERSION.to_owned(),
sheet_id: SHEET.to_owned(),
revision: 3,
document,
base_document: None,
drawings,
}
}
struct Fake {
calls: RefCell<Vec<String>>,
input: RefCell<Option<BackendResult<Option<VerifyInput>>>>,
downloads: RefCell<VecDeque<BackendResult<Vec<u8>>>>,
complete: RefCell<Option<BackendResult<CompleteAnswer>>>,
fail: RefCell<Option<BackendResult<FailAnswer>>>,
verdict: RefCell<Option<Value>>,
jobs: RefCell<Vec<VerifyJob>>,
}
impl Fake {
fn with_input(input: VerifyInput) -> Self {
Self {
calls: RefCell::default(),
input: RefCell::new(Some(Ok(Some(input)))),
downloads: RefCell::default(),
complete: RefCell::default(),
fail: RefCell::default(),
verdict: RefCell::default(),
jobs: RefCell::default(),
}
}
fn clean() -> Self {
Self::with_input(input(document().to_string(), vec![row()]))
}
fn calls(&self) -> Vec<String> {
self.calls.borrow().clone()
}
fn names(&self) -> Vec<String> {
self.calls()
.iter()
.map(|c| c.split(' ').next().unwrap().to_owned())
.collect()
}
fn verdict(&self) -> Value {
self.verdict.borrow().clone().expect("a completion")
}
}
impl VerifyBackend for Fake {
fn claim_verifications(&self, max_jobs: u32) -> BackendResult<Vec<VerifyJob>> {
self.calls.borrow_mut().push(format!("claim {max_jobs}"));
Ok(self.jobs.borrow_mut().drain(..).collect())
}
fn verification_input(
&self,
review_id: &str,
lease_token: &str,
) -> BackendResult<Option<VerifyInput>> {
self.calls
.borrow_mut()
.push(format!("input {review_id} {lease_token}"));
self.input.borrow_mut().take().unwrap_or(Ok(None))
}
fn download_drawing(
&self,
bucket: &str,
key: &str,
max_bytes: usize,
) -> BackendResult<Vec<u8>> {
self.calls
.borrow_mut()
.push(format!("download {bucket}/{key} {max_bytes}"));
self.downloads
.borrow_mut()
.pop_front()
.unwrap_or_else(|| Ok(DRAWING.as_bytes().to_vec()))
}
fn complete_verification(
&self,
review_id: &str,
lease_token: &str,
verdict: &[u8],
) -> BackendResult<CompleteAnswer> {
self.calls
.borrow_mut()
.push(format!("complete {review_id} {lease_token}"));
let value: Value = serde_json::from_slice(verdict).unwrap();
let outcome = value["outcome"].clone();
*self.verdict.borrow_mut() = Some(value);
self.complete
.borrow_mut()
.take()
.unwrap_or(Ok(if outcome == "passed" {
CompleteAnswer::Passed
} else {
CompleteAnswer::Failed
}))
}
fn fail_verification(
&self,
review_id: &str,
lease_token: &str,
code: &str,
) -> BackendResult<FailAnswer> {
self.calls
.borrow_mut()
.push(format!("fail {review_id} {lease_token} {code}"));
self.fail
.borrow_mut()
.take()
.unwrap_or(Ok(FailAnswer::Retry))
}
}
/// The child's work in process: the same frame, `verify::process`, the core's verdict.
struct InProcess;
impl Verifier for InProcess {
fn verify(&self, frame: &[u8]) -> ChildRun {
let outcome = match verify::process(frame) {
Ok(verdict) => ChildOutcome::Produced(verdict),
Err(failure) => ChildOutcome::Broken {
code: failure.exit_code(),
name: exit_codes::name(failure.exit_code()),
},
};
ChildRun {
outcome,
stderr: String::new(),
}
}
}
/// A child that ends as told, at most once.
struct Fixed(RefCell<Option<ChildOutcome>>);
impl Fixed {
fn with(outcome: ChildOutcome) -> Self {
Self(RefCell::new(Some(outcome)))
}
fn never() -> Self {
Self(RefCell::new(None))
}
}
impl Verifier for Fixed {
fn verify(&self, _frame: &[u8]) -> ChildRun {
ChildRun {
outcome: self.0.borrow_mut().take().expect("no child run expected"),
stderr: "child says\n\u{1b}[0m".to_owned(),
}
}
}
fn process(fake: &Fake, verifier: &dyn Verifier) -> VerifyEnd {
process_verification(fake, verifier, &Watchdog::new(), &job())
}
fn fail_call(code: &str) -> String {
format!("fail {REVIEW} {TOKEN} {code}")
}
#[test]
fn a_clean_sheet_follows_input_download_child_complete() {
let fake = Fake::clean();
assert_eq!(process(&fake, &InProcess), VerifyEnd::Passed);
let key = row().object_key;
assert_eq!(
fake.calls(),
vec![
format!("input {REVIEW} {TOKEN}"),
format!("download {DRAWING_BUCKET}/{key} {VERIFY_MAX_DRAWING_BYTES}"),
format!("complete {REVIEW} {TOKEN}"),
]
);
let verdict = fake.verdict();
assert_eq!(verdict["outcome"], "passed");
assert_eq!(
(
&verdict["review_id"],
&verdict["version_id"],
&verdict["sheet_id"],
&verdict["revision"]
),
(&json!(REVIEW), &json!(VERSION), &json!(SHEET), &json!(3))
);
}
#[test]
fn an_unreadable_document_completes_failed() {
let fake = Fake::with_input(input("not json".to_owned(), vec![row()]));
assert_eq!(process(&fake, &InProcess), VerifyEnd::Failed);
assert_eq!(fake.names(), ["input", "download", "complete"]);
assert_eq!(fake.verdict()["failure_code"], verdict_codes::DOCUMENT);
}
#[test]
fn a_stale_input_ends_without_any_other_call() {
let fake = Fake::clean();
*fake.input.borrow_mut() = Some(Ok(None));
assert_eq!(process(&fake, &Fixed::never()), VerifyEnd::Stale);
assert_eq!(fake.names(), ["input"]);
}
#[test]
fn a_stale_completion_ends_and_a_malformed_one_is_reported() {
let fake = Fake::clean();
*fake.complete.borrow_mut() = Some(Ok(CompleteAnswer::Stale));
assert_eq!(process(&fake, &InProcess), VerifyEnd::Stale);
assert_eq!(fake.names(), ["input", "download", "complete"]);
let fake = Fake::clean();
*fake.complete.borrow_mut() = Some(Ok(CompleteAnswer::Malformed));
assert_eq!(
process(&fake, &InProcess),
VerifyEnd::Reported(codes::VERDICT_REFUSED)
);
assert_eq!(
fake.calls().last().unwrap(),
&fail_call(codes::VERDICT_REFUSED)
);
}
#[test]
fn a_missing_or_oversized_drawing_is_judged_by_the_core() {
for error in [BackendError::Status(404), BackendError::Overflow] {
let fake = Fake::clean();
fake.downloads.borrow_mut().push_back(Err(error));
assert_eq!(process(&fake, &InProcess), VerifyEnd::Failed);
assert_eq!(fake.names(), ["input", "download", "complete"]);
let verdict = fake.verdict();
assert_eq!(verdict["failure_code"], verdict_codes::DRAWING_MISSING);
assert_eq!(verdict["drawings"][0]["result"], "missing");
}
}
#[test]
fn rows_outside_the_policy_are_never_fetched() {
let mut not_uploaded = row();
not_uploaded.uploaded = false;
let mut other_bucket = row();
other_bucket.bucket = "avatars".to_owned();
let mut other_key = row();
other_key.object_key = format!("{}.svg", "0".repeat(64));
let mut hostile_key = row();
hostile_key.object_key = "../avatars/x.svg".to_owned();
let mut too_big = row();
too_big.byte_size = VERIFY_MAX_DRAWING_BYTES as u64 + 1;
for skipped in [not_uploaded, other_bucket, other_key, hostile_key, too_big] {
let fake = Fake::with_input(input(document().to_string(), vec![skipped]));
assert_eq!(process(&fake, &InProcess), VerifyEnd::Failed);
assert_eq!(fake.names(), ["input", "complete"]);
}
}
// Tracked misclassification (owner brief 2026-11-01), not fixed: an object larger than the
// policy is passed without bytes, so the core judges it missing (`drawing_missing`, listed
// after `caps`: the row's declared size already exceeds the cap) where `size_mismatch` would
// name it.
#[test]
fn tracked_an_object_over_the_policy_is_reported_drawing_missing() {
let mut too_big = row();
too_big.byte_size = VERIFY_MAX_DRAWING_BYTES as u64 + 1;
let fake = Fake::with_input(input(document().to_string(), vec![too_big]));
assert_eq!(process(&fake, &InProcess), VerifyEnd::Failed);
assert_eq!(fake.names(), ["input", "complete"]);
let verdict = fake.verdict();
assert_eq!(
verdict["failures"],
json!([verdict_codes::CAPS, verdict_codes::DRAWING_MISSING])
);
assert_eq!(verdict["drawings"][0]["result"], "missing");
}
// Tracked misclassification (owner brief 2026-11-01), not fixed: a document naming enough
// drawings that the frame header passes 64 KiB is refused by the parent's own framing and
// reported `resource_limit` (a definitive worker failure) where `caps` would name it.
#[test]
fn tracked_a_header_over_64_kib_is_reported_resource_limit() {
let rows = vec![row(); 600];
let fake = Fake::with_input(input(document().to_string(), rows));
assert_eq!(
process(&fake, &Fixed::never()),
VerifyEnd::Reported(codes::RESOURCE_LIMIT)
);
assert_eq!(
fake.calls().last().unwrap(),
&fail_call(codes::RESOURCE_LIMIT)
);
assert_eq!(fake.names().iter().filter(|n| *n == "fail").count(), 1);
}
#[test]
fn downloads_stop_at_the_drawing_count() {
let rows = vec![row(); 6];
let fake = Fake::with_input(input(document().to_string(), rows));
process(&fake, &InProcess);
let downloads = fake.names().iter().filter(|n| *n == "download").count();
assert_eq!(downloads, 4);
}
#[test]
fn a_transport_failure_is_reported_unavailable_without_a_child() {
let fake = Fake::clean();
fake.downloads
.borrow_mut()
.push_back(Err(BackendError::Network("refused".to_owned())));
assert_eq!(
process(&fake, &Fixed::never()),
VerifyEnd::Reported(codes::UNAVAILABLE)
);
assert_eq!(fake.names(), ["input", "download", "fail"]);
assert_eq!(fake.calls()[2], fail_call(codes::UNAVAILABLE));
let fake = Fake::clean();
*fake.input.borrow_mut() = Some(Err(BackendError::Status(503)));
assert_eq!(
process(&fake, &Fixed::never()),
VerifyEnd::Reported(codes::UNAVAILABLE)
);
assert_eq!(fake.names(), ["input", "fail"]);
}
#[test]
fn an_unrecordable_failure_leaves_the_lease() {
let fake = Fake::clean();
*fake.input.borrow_mut() = Some(Err(BackendError::Status(503)));
*fake.fail.borrow_mut() = Some(Err(BackendError::Status(503)));
assert_eq!(process(&fake, &Fixed::never()), VerifyEnd::LeftForLease);
let fake = Fake::clean();
*fake.fail.borrow_mut() = Some(Ok(FailAnswer::Stale));
assert_eq!(
process(&fake, &Fixed::with(ChildOutcome::Timeout)),
VerifyEnd::Stale
);
}
#[test]
fn every_child_limit_maps_to_its_closed_code() {
for (outcome, code) in [
(ChildOutcome::Timeout, codes::TIMEOUT),
(ChildOutcome::OutputOverflow, codes::RESOURCE_LIMIT),
(
ChildOutcome::Signalled { signal: 24 },
codes::RESOURCE_LIMIT,
),
(ChildOutcome::Signalled { signal: 9 }, codes::UNAVAILABLE),
(ChildOutcome::Signalled { signal: 15 }, codes::UNAVAILABLE),
(ChildOutcome::Signalled { signal: 6 }, codes::RESOURCE_LIMIT),
(
ChildOutcome::Signalled { signal: 31 },
codes::RESOURCE_LIMIT,
),
(
ChildOutcome::Rejected {
code: exit_codes::INPUT_TOO_LARGE,
name: "input_too_large",
},
codes::RESOURCE_LIMIT,
),
] {
let fake = Fake::clean();
assert_eq!(
process(&fake, &Fixed::with(outcome)),
VerifyEnd::Reported(code)
);
assert_eq!(fake.names(), ["input", "download", "fail"]);
assert_eq!(fake.calls()[2], fail_call(code));
}
}
#[test]
fn a_broken_or_failed_child_is_reported_then_stops_the_worker() {
for outcome in [
ChildOutcome::Broken {
code: exit_codes::SANDBOX_FAILED,
name: "sandbox_failed",
},
ChildOutcome::Failed("output refused: not a JSON object".to_owned()),
] {
let fake = Fake::clean();
assert_eq!(process(&fake, &Fixed::with(outcome)), VerifyEnd::Stop);
assert_eq!(fake.calls().last().unwrap(), &fail_call(codes::UNAVAILABLE));
}
}
#[test]
fn a_round_with_a_hostile_identity_is_left_untouched() {
let fake = Fake::clean();
let mut bad = job();
bad.lease_token = "x".to_owned();
assert_eq!(
process_verification(&fake, &Fixed::never(), &Watchdog::new(), &bad),
VerifyEnd::LeftForLease
);
assert!(fake.calls().is_empty());
}
fn once_config() -> Config {
Config::from_vars(|name| match name {
"SUPABASE_URL" => Some("http://kong:8000".to_owned()),
"SUPABASE_SERVICE_ROLE_KEY" => Some("k".to_owned()),
"WORKER_ONCE" => Some("1".to_owned()),
_ => None,
})
.unwrap()
}
#[test]
fn once_mode_claims_one_round_without_a_sweep_or_stops_on_a_broken_child() {
let fake = Fake::clean();
*fake.jobs.borrow_mut() = vec![job()];
assert_eq!(
verify_loop(&once_config(), &fake, &InProcess, &Watchdog::new()),
exit_codes::OK
);
assert_eq!(fake.names(), ["claim", "input", "download", "complete"]);
let fake = Fake::clean();
*fake.jobs.borrow_mut() = vec![job()];
let broken = Fixed::with(ChildOutcome::Broken {
code: exit_codes::ENV_NOT_EMPTY,
name: "env_not_empty",
});
assert_eq!(
verify_loop(&once_config(), &fake, &broken, &Watchdog::new()),
exit_codes::CHILD_BROKEN
);
}
#[test]
fn run_arguments_select_the_mode() {
let args = |list: &[&str]| -> Vec<String> { list.iter().map(|a| (*a).to_owned()).collect() };
assert_eq!(parse_args(&args(&[])), Ok((Mode::Avatars, false)));
assert_eq!(parse_args(&args(&["--check"])), Ok((Mode::Avatars, true)));
assert_eq!(
parse_args(&args(&["--jobs", "sheet-verify"])),
Ok((Mode::SheetVerify, false))
);
assert_eq!(
parse_args(&args(&["--jobs", "sheet-verify", "--check"])),
Ok((Mode::SheetVerify, true))
);
assert_eq!(
parse_args(&args(&["--check", "--jobs", "sheet-verify"])),
Ok((Mode::SheetVerify, true))
);
assert!(parse_args(&args(&["--jobs", "avatars"])).is_err());
assert!(parse_args(&args(&["--jobs"])).is_err());
assert!(parse_args(&args(&["--check", "--check"])).is_err());
assert!(parse_args(&args(&["--once"])).is_err());
}
/// The real child (this crate's binary), rlimits on, seccomp off: the parent's frame, the
/// child's reading and the spawner's classification end to end on any host (the filter
/// itself is proven in the Alpine stage, tests/sandbox.rs).
#[cfg(feature = "test-hooks")]
#[test]
fn the_real_child_without_seccomp_verifies_the_parents_frame() {
use faunosteo_worker::spawn::Spawner;
struct RealChild(Spawner);
impl Verifier for RealChild {
fn verify(&self, frame: &[u8]) -> ChildRun {
self.0.run(frame)
}
}
let mut spawner = Spawner::for_verify(env!("CARGO_BIN_EXE_faunosteo-worker"));
spawner.extra_args = vec!["--no-seccomp".to_owned()];
let fake = Fake::clean();
assert_eq!(process(&fake, &RealChild(spawner)), VerifyEnd::Passed);
assert_eq!(fake.verdict()["outcome"], "passed");
}