feat(worker): V1-2.3 sandboxed verify child and sheet-verify job mode

This commit is contained in:
bchanot committed 2026-11-01 17:20:07 +01:00
1 parent 345dde59e6
commit 31fbb0317f
24 files changed
+2145 -87

No files matched your search

+100
View File
@@ -210,3 +210,103 @@ fn the_cpu_limit_stops_a_spinning_child_before_the_deadline() {
fn the_linked_libwebp_is_recent_enough() {
assert!(faunosteo_worker::decode::libwebp_version() >= limits::MIN_LIBWEBP_VERSION);
}
// V1-2.3: the `verify` child under the same sandbox (contract W2, W3).
fn verifier(flags: &[&str]) -> Spawner {
let mut spawner = Spawner::for_verify(env!("CARGO_BIN_EXE_faunosteo-worker"));
spawner.extra_args = flags.iter().map(|flag| (*flag).to_owned()).collect();
spawner
}
fn expect_verdict(frame: &[u8]) -> serde_json::Value {
let run = verifier(&[]).run(frame);
match run.outcome {
ChildOutcome::Produced(out) => {
assert!(out.len() < limits::VERIFY_MAX_OUTPUT_BYTES);
serde_json::from_slice(&out).unwrap()
}
other => panic!("{other:?} (stderr: {})", run.stderr),
}
}
#[test]
fn a_clean_sheet_is_verified_passed_through_the_full_sandbox() {
let verdict = expect_verdict(&fixtures::sheet_frame(&fixtures::sheet_document()));
assert_eq!(verdict["outcome"], "passed", "{verdict}");
assert_eq!(verdict["review_id"], fixtures::sheet_binding().review_id);
}
#[test]
fn a_sheet_at_every_cap_is_verified_within_the_limits() {
let frame = fixtures::sheet_frame(&fixtures::sheet_document_at_caps());
let started = std::time::Instant::now();
let verdict = expect_verdict(&frame);
let elapsed = started.elapsed();
assert_eq!(verdict["outcome"], "passed", "{}", verdict["failures"]);
assert!(elapsed < limits::VERIFY_DEADLINE, "{elapsed:?}");
eprintln!(
"verify at caps: input {} bytes, verdict {} bytes, {elapsed:?}",
frame.len(),
verdict.to_string().len()
);
}
#[test]
fn the_verify_child_refuses_a_non_empty_environment() {
let status = Command::new(env!("CARGO_BIN_EXE_faunosteo-worker"))
.arg("verify")
.env_clear()
.env("LEAKED", "1")
.stdin(Stdio::null())
.stdout(Stdio::null())
.stderr(Stdio::null())
.status()
.unwrap();
assert_eq!(status.code(), Some(exit_codes::ENV_NOT_EMPTY));
}
#[test]
fn an_input_over_the_verify_cap_is_refused_unread() {
let run = verifier(&[]).run(&vec![0u8; limits::VERIFY_MAX_INPUT_BYTES + 1]);
assert!(
matches!(run.outcome, ChildOutcome::Rejected { code, .. } if code == exit_codes::INPUT_TOO_LARGE),
"{:?} {}",
run.outcome,
run.stderr
);
}
#[test]
fn a_malformed_frame_is_a_broken_parent_not_content() {
let run = verifier(&[]).run(b"\x03\0\0\0{x}");
assert!(
matches!(run.outcome, ChildOutcome::Broken { code, .. } if code == exit_codes::FRAME_INVALID),
"{:?} {}",
run.outcome,
run.stderr
);
}
#[test]
fn a_denied_syscall_kills_the_verify_child_with_sigsys() {
let run = verifier(&["--violate"]).run(b"");
assert!(
matches!(run.outcome, ChildOutcome::Signalled { signal: SIGSYS }),
"{:?} {}",
run.outcome,
run.stderr
);
}
#[test]
fn a_verify_output_over_its_cap_is_cut() {
let mut spawner = verifier(&["--flood"]);
spawner.max_output = 64 * 1024;
let run = spawner.run(b"");
assert!(
matches!(run.outcome, ChildOutcome::OutputOverflow),
"{:?}",
run.outcome
);
}
+527
View File
@@ -0,0 +1,527 @@
//! V1-2.3 proofs of the verification job (contract W4, V1-2 contract §6.5) on an in-memory
//! `VerifyBackend`: the core's verdict computed in process from the very frame the parent
//! builds (or the real child without seccomp under `test-hooks`), every child outcome mapped
//! to its closed worker code, a missing drawing judged by the core, never by the worker.
#![allow(clippy::unwrap_used, clippy::expect_used)]
use std::cell::RefCell;
use std::collections::VecDeque;
use faunosteo_core::domain::sheet::verdict::{codes as verdict_codes, sha256_hex};
use faunosteo_worker::backend::{
BackendError, BackendResult, CompleteAnswer, DrawingRow, FailAnswer, VerifyBackend,
VerifyInput, VerifyJob,
};
use faunosteo_worker::config::Config;
use faunosteo_worker::limits::VERIFY_MAX_DRAWING_BYTES;
use faunosteo_worker::run::{Mode, parse_args, verify_loop};
use faunosteo_worker::spawn::{ChildOutcome, ChildRun};
use faunosteo_worker::verifier::Verifier;
use faunosteo_worker::verify_job::{DRAWING_BUCKET, VerifyEnd, codes, process_verification};
use faunosteo_worker::watchdog::Watchdog;
use faunosteo_worker::{exit_codes, verify};
use serde_json::{Value, json};
const REVIEW: &str = "aaaaaaaa-0000-4000-8000-000000000003";
const VERSION: &str = "aaaaaaaa-0000-4000-8000-000000000002";
const SHEET: &str = "aaaaaaaa-0000-4000-8000-000000000001";
const TOKEN: &str = "30000000-0000-0000-0000-00000000000c";
const ASSET: &str = "11111111-2222-4333-8444-555555555555";
const DRAWING: &str = concat!(
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n",
r#"<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 10 10"><path d="M0 0L5 5"/></svg>"#
);
fn name(text: &str) -> Value {
json!({ "fr": text, "en": text })
}
/// The core's small valid sheet (`crates/core/tests/sheet_verdict.rs::document`).
fn document() -> Value {
json!({
"schema_version": 2, "species_id": "canine", "name": name("Chien"),
"description": {}, "scope_disclaimer": { "fr": "Outil", "en": "Tool" },
"attribution": { "authors": ["A"], "license": null, "notes": null },
"vertebral_config": { "regions": [] }, "references": [],
"categories": [{ "id": "c", "name": name("C"), "color": null }],
"views": [{ "id": "main", "kind": "main", "name": name("Vue"), "svg_asset_id": ASSET, "view_box": null }],
"annotations": [{ "id": "a1", "kind": "point", "view_id": "main", "category_id": "c",
"laterality": "unpaired", "geometry": { "type": "circle", "cx": 1, "cy": 1, "r": 1 },
"review_status": "to_confirm", "name": name("P"), "finding_zone_id": "z1", "vertebral_region_code": null }],
"finding_zones": [{ "id": "z1", "name": name("Z"), "categories": [{ "id": "cat", "kind": "dysfunction",
"name": name("D"), "review_status": "to_confirm",
"children": [{ "id": "n1", "name": name("N"), "review_status": "to_confirm" }] }] }]
})
}
fn job() -> VerifyJob {
VerifyJob {
review_id: REVIEW.to_owned(),
version_id: VERSION.to_owned(),
sheet_id: SHEET.to_owned(),
revision: 3,
lease_token: TOKEN.to_owned(),
attempts: 1,
}
}
fn row() -> DrawingRow {
let sha256 = sha256_hex(DRAWING.as_bytes());
DrawingRow {
asset_id: ASSET.to_owned(),
object_key: format!("{sha256}.svg"),
sha256,
byte_size: DRAWING.len() as u64,
bucket: DRAWING_BUCKET.to_owned(),
uploaded: true,
}
}
fn input(document: String, drawings: Vec<DrawingRow>) -> VerifyInput {
VerifyInput {
review_id: REVIEW.to_owned(),
version_id: VERSION.to_owned(),
sheet_id: SHEET.to_owned(),
revision: 3,
document,
base_document: None,
drawings,
}
}
struct Fake {
calls: RefCell<Vec<String>>,
input: RefCell<Option<BackendResult<Option<VerifyInput>>>>,
downloads: RefCell<VecDeque<BackendResult<Vec<u8>>>>,
complete: RefCell<Option<BackendResult<CompleteAnswer>>>,
fail: RefCell<Option<BackendResult<FailAnswer>>>,
verdict: RefCell<Option<Value>>,
jobs: RefCell<Vec<VerifyJob>>,
}
impl Fake {
fn with_input(input: VerifyInput) -> Self {
Self {
calls: RefCell::default(),
input: RefCell::new(Some(Ok(Some(input)))),
downloads: RefCell::default(),
complete: RefCell::default(),
fail: RefCell::default(),
verdict: RefCell::default(),
jobs: RefCell::default(),
}
}
fn clean() -> Self {
Self::with_input(input(document().to_string(), vec![row()]))
}
fn calls(&self) -> Vec<String> {
self.calls.borrow().clone()
}
fn names(&self) -> Vec<String> {
self.calls()
.iter()
.map(|c| c.split(' ').next().unwrap().to_owned())
.collect()
}
fn verdict(&self) -> Value {
self.verdict.borrow().clone().expect("a completion")
}
}
impl VerifyBackend for Fake {
fn claim_verifications(&self, max_jobs: u32) -> BackendResult<Vec<VerifyJob>> {
self.calls.borrow_mut().push(format!("claim {max_jobs}"));
Ok(self.jobs.borrow_mut().drain(..).collect())
}
fn verification_input(
&self,
review_id: &str,
lease_token: &str,
) -> BackendResult<Option<VerifyInput>> {
self.calls
.borrow_mut()
.push(format!("input {review_id} {lease_token}"));
self.input.borrow_mut().take().unwrap_or(Ok(None))
}
fn download_drawing(
&self,
bucket: &str,
key: &str,
max_bytes: usize,
) -> BackendResult<Vec<u8>> {
self.calls
.borrow_mut()
.push(format!("download {bucket}/{key} {max_bytes}"));
self.downloads
.borrow_mut()
.pop_front()
.unwrap_or_else(|| Ok(DRAWING.as_bytes().to_vec()))
}
fn complete_verification(
&self,
review_id: &str,
lease_token: &str,
verdict: &[u8],
) -> BackendResult<CompleteAnswer> {
self.calls
.borrow_mut()
.push(format!("complete {review_id} {lease_token}"));
let value: Value = serde_json::from_slice(verdict).unwrap();
let outcome = value["outcome"].clone();
*self.verdict.borrow_mut() = Some(value);
self.complete
.borrow_mut()
.take()
.unwrap_or(Ok(if outcome == "passed" {
CompleteAnswer::Passed
} else {
CompleteAnswer::Failed
}))
}
fn fail_verification(
&self,
review_id: &str,
lease_token: &str,
code: &str,
) -> BackendResult<FailAnswer> {
self.calls
.borrow_mut()
.push(format!("fail {review_id} {lease_token} {code}"));
self.fail
.borrow_mut()
.take()
.unwrap_or(Ok(FailAnswer::Retry))
}
}
/// The child's work in process: the same frame, `verify::process`, the core's verdict.
struct InProcess;
impl Verifier for InProcess {
fn verify(&self, frame: &[u8]) -> ChildRun {
let outcome = match verify::process(frame) {
Ok(verdict) => ChildOutcome::Produced(verdict),
Err(failure) => ChildOutcome::Broken {
code: failure.exit_code(),
name: exit_codes::name(failure.exit_code()),
},
};
ChildRun {
outcome,
stderr: String::new(),
}
}
}
/// A child that ends as told, at most once.
struct Fixed(RefCell<Option<ChildOutcome>>);
impl Fixed {
fn with(outcome: ChildOutcome) -> Self {
Self(RefCell::new(Some(outcome)))
}
fn never() -> Self {
Self(RefCell::new(None))
}
}
impl Verifier for Fixed {
fn verify(&self, _frame: &[u8]) -> ChildRun {
ChildRun {
outcome: self.0.borrow_mut().take().expect("no child run expected"),
stderr: "child says\n\u{1b}[0m".to_owned(),
}
}
}
fn process(fake: &Fake, verifier: &dyn Verifier) -> VerifyEnd {
process_verification(fake, verifier, &Watchdog::new(), &job())
}
fn fail_call(code: &str) -> String {
format!("fail {REVIEW} {TOKEN} {code}")
}
#[test]
fn a_clean_sheet_follows_input_download_child_complete() {
let fake = Fake::clean();
assert_eq!(process(&fake, &InProcess), VerifyEnd::Passed);
let key = row().object_key;
assert_eq!(
fake.calls(),
vec![
format!("input {REVIEW} {TOKEN}"),
format!("download {DRAWING_BUCKET}/{key} {VERIFY_MAX_DRAWING_BYTES}"),
format!("complete {REVIEW} {TOKEN}"),
]
);
let verdict = fake.verdict();
assert_eq!(verdict["outcome"], "passed");
assert_eq!(
(
&verdict["review_id"],
&verdict["version_id"],
&verdict["sheet_id"],
&verdict["revision"]
),
(&json!(REVIEW), &json!(VERSION), &json!(SHEET), &json!(3))
);
}
#[test]
fn an_unreadable_document_completes_failed() {
let fake = Fake::with_input(input("not json".to_owned(), vec![row()]));
assert_eq!(process(&fake, &InProcess), VerifyEnd::Failed);
assert_eq!(fake.names(), ["input", "download", "complete"]);
assert_eq!(fake.verdict()["failure_code"], verdict_codes::DOCUMENT);
}
#[test]
fn a_stale_input_ends_without_any_other_call() {
let fake = Fake::clean();
*fake.input.borrow_mut() = Some(Ok(None));
assert_eq!(process(&fake, &Fixed::never()), VerifyEnd::Stale);
assert_eq!(fake.names(), ["input"]);
}
#[test]
fn a_stale_completion_ends_and_a_malformed_one_is_reported() {
let fake = Fake::clean();
*fake.complete.borrow_mut() = Some(Ok(CompleteAnswer::Stale));
assert_eq!(process(&fake, &InProcess), VerifyEnd::Stale);
assert_eq!(fake.names(), ["input", "download", "complete"]);
let fake = Fake::clean();
*fake.complete.borrow_mut() = Some(Ok(CompleteAnswer::Malformed));
assert_eq!(
process(&fake, &InProcess),
VerifyEnd::Reported(codes::VERDICT_REFUSED)
);
assert_eq!(
fake.calls().last().unwrap(),
&fail_call(codes::VERDICT_REFUSED)
);
}
#[test]
fn a_missing_or_oversized_drawing_is_judged_by_the_core() {
for error in [BackendError::Status(404), BackendError::Overflow] {
let fake = Fake::clean();
fake.downloads.borrow_mut().push_back(Err(error));
assert_eq!(process(&fake, &InProcess), VerifyEnd::Failed);
assert_eq!(fake.names(), ["input", "download", "complete"]);
let verdict = fake.verdict();
assert_eq!(verdict["failure_code"], verdict_codes::DRAWING_MISSING);
assert_eq!(verdict["drawings"][0]["result"], "missing");
}
}
#[test]
fn rows_outside_the_policy_are_never_fetched() {
let mut not_uploaded = row();
not_uploaded.uploaded = false;
let mut other_bucket = row();
other_bucket.bucket = "avatars".to_owned();
let mut other_key = row();
other_key.object_key = format!("{}.svg", "0".repeat(64));
let mut hostile_key = row();
hostile_key.object_key = "../avatars/x.svg".to_owned();
let mut too_big = row();
too_big.byte_size = VERIFY_MAX_DRAWING_BYTES as u64 + 1;
for skipped in [not_uploaded, other_bucket, other_key, hostile_key, too_big] {
let fake = Fake::with_input(input(document().to_string(), vec![skipped]));
assert_eq!(process(&fake, &InProcess), VerifyEnd::Failed);
assert_eq!(fake.names(), ["input", "complete"]);
}
}
#[test]
fn downloads_stop_at_the_drawing_count() {
let rows = vec![row(); 6];
let fake = Fake::with_input(input(document().to_string(), rows));
process(&fake, &InProcess);
let downloads = fake.names().iter().filter(|n| *n == "download").count();
assert_eq!(downloads, 4);
}
#[test]
fn a_transport_failure_is_reported_unavailable_without_a_child() {
let fake = Fake::clean();
fake.downloads
.borrow_mut()
.push_back(Err(BackendError::Network("refused".to_owned())));
assert_eq!(
process(&fake, &Fixed::never()),
VerifyEnd::Reported(codes::UNAVAILABLE)
);
assert_eq!(fake.names(), ["input", "download", "fail"]);
assert_eq!(fake.calls()[2], fail_call(codes::UNAVAILABLE));
let fake = Fake::clean();
*fake.input.borrow_mut() = Some(Err(BackendError::Status(503)));
assert_eq!(
process(&fake, &Fixed::never()),
VerifyEnd::Reported(codes::UNAVAILABLE)
);
assert_eq!(fake.names(), ["input", "fail"]);
}
#[test]
fn an_unrecordable_failure_leaves_the_lease() {
let fake = Fake::clean();
*fake.input.borrow_mut() = Some(Err(BackendError::Status(503)));
*fake.fail.borrow_mut() = Some(Err(BackendError::Status(503)));
assert_eq!(process(&fake, &Fixed::never()), VerifyEnd::LeftForLease);
let fake = Fake::clean();
*fake.fail.borrow_mut() = Some(Ok(FailAnswer::Stale));
assert_eq!(
process(&fake, &Fixed::with(ChildOutcome::Timeout)),
VerifyEnd::Stale
);
}
#[test]
fn every_child_limit_maps_to_its_closed_code() {
for (outcome, code) in [
(ChildOutcome::Timeout, codes::TIMEOUT),
(ChildOutcome::OutputOverflow, codes::RESOURCE_LIMIT),
(
ChildOutcome::Signalled { signal: 24 },
codes::RESOURCE_LIMIT,
),
(ChildOutcome::Signalled { signal: 9 }, codes::RESOURCE_LIMIT),
(ChildOutcome::Signalled { signal: 6 }, codes::RESOURCE_LIMIT),
(
ChildOutcome::Signalled { signal: 31 },
codes::RESOURCE_LIMIT,
),
(
ChildOutcome::Rejected {
code: exit_codes::INPUT_TOO_LARGE,
name: "input_too_large",
},
codes::RESOURCE_LIMIT,
),
] {
let fake = Fake::clean();
assert_eq!(
process(&fake, &Fixed::with(outcome)),
VerifyEnd::Reported(code)
);
assert_eq!(fake.names(), ["input", "download", "fail"]);
assert_eq!(fake.calls()[2], fail_call(code));
}
}
#[test]
fn a_broken_or_failed_child_is_reported_then_stops_the_worker() {
for outcome in [
ChildOutcome::Broken {
code: exit_codes::SANDBOX_FAILED,
name: "sandbox_failed",
},
ChildOutcome::Failed("output refused: not a JSON object".to_owned()),
] {
let fake = Fake::clean();
assert_eq!(process(&fake, &Fixed::with(outcome)), VerifyEnd::Stop);
assert_eq!(fake.calls().last().unwrap(), &fail_call(codes::UNAVAILABLE));
}
}
#[test]
fn a_round_with_a_hostile_identity_is_left_untouched() {
let fake = Fake::clean();
let mut bad = job();
bad.lease_token = "x".to_owned();
assert_eq!(
process_verification(&fake, &Fixed::never(), &Watchdog::new(), &bad),
VerifyEnd::LeftForLease
);
assert!(fake.calls().is_empty());
}
fn once_config() -> Config {
Config::from_vars(|name| match name {
"SUPABASE_URL" => Some("http://kong:8000".to_owned()),
"SUPABASE_SERVICE_ROLE_KEY" => Some("k".to_owned()),
"WORKER_ONCE" => Some("1".to_owned()),
_ => None,
})
.unwrap()
}
#[test]
fn once_mode_claims_one_round_without_a_sweep_or_stops_on_a_broken_child() {
let fake = Fake::clean();
*fake.jobs.borrow_mut() = vec![job()];
assert_eq!(
verify_loop(&once_config(), &fake, &InProcess, &Watchdog::new()),
exit_codes::OK
);
assert_eq!(fake.names(), ["claim", "input", "download", "complete"]);
let fake = Fake::clean();
*fake.jobs.borrow_mut() = vec![job()];
let broken = Fixed::with(ChildOutcome::Broken {
code: exit_codes::ENV_NOT_EMPTY,
name: "env_not_empty",
});
assert_eq!(
verify_loop(&once_config(), &fake, &broken, &Watchdog::new()),
exit_codes::CHILD_BROKEN
);
}
#[test]
fn run_arguments_select_the_mode() {
let args = |list: &[&str]| -> Vec<String> { list.iter().map(|a| (*a).to_owned()).collect() };
assert_eq!(parse_args(&args(&[])), Ok((Mode::Avatars, false)));
assert_eq!(parse_args(&args(&["--check"])), Ok((Mode::Avatars, true)));
assert_eq!(
parse_args(&args(&["--jobs", "sheet-verify"])),
Ok((Mode::SheetVerify, false))
);
assert_eq!(
parse_args(&args(&["--jobs", "sheet-verify", "--check"])),
Ok((Mode::SheetVerify, true))
);
assert_eq!(
parse_args(&args(&["--check", "--jobs", "sheet-verify"])),
Ok((Mode::SheetVerify, true))
);
assert!(parse_args(&args(&["--jobs", "avatars"])).is_err());
assert!(parse_args(&args(&["--jobs"])).is_err());
assert!(parse_args(&args(&["--check", "--check"])).is_err());
assert!(parse_args(&args(&["--once"])).is_err());
}
/// The real child (this crate's binary), rlimits on, seccomp off: the parent's frame, the
/// child's reading and the spawner's classification end to end on any host (the filter
/// itself is proven in the Alpine stage, tests/sandbox.rs).
#[cfg(feature = "test-hooks")]
#[test]
fn the_real_child_without_seccomp_verifies_the_parents_frame() {
use faunosteo_worker::spawn::Spawner;
struct RealChild(Spawner);
impl Verifier for RealChild {
fn verify(&self, frame: &[u8]) -> ChildRun {
self.0.run(frame)
}
}
let mut spawner = Spawner::for_verify(env!("CARGO_BIN_EXE_faunosteo-worker"));
spawner.extra_args = vec!["--no-seccomp".to_owned()];
let fake = Fake::clean();
assert_eq!(process(&fake, &RealChild(spawner)), VerifyEnd::Passed);
assert_eq!(fake.verdict()["outcome"], "passed");
}