From 31fbb0317f8958557a2cf35586ac187502d6418f Mon Sep 17 00:00:00 2001 From: bchanot Date: Sun, 1 Nov 2026 17:20:07 +0100 Subject: [PATCH] feat(worker): V1-2.3 sandboxed verify child and sheet-verify job mode --- Cargo.lock | 1 + crates/worker/Cargo.toml | 5 +- crates/worker/scripts/derive-seccomp.sh | 34 +- crates/worker/src/backend.rs | 89 +++- crates/worker/src/decode.rs | 15 +- crates/worker/src/exit_codes.rs | 5 +- crates/worker/src/fixtures.rs | 116 +++++- crates/worker/src/http.rs | 159 ++++++- crates/worker/src/keys.rs | 22 + crates/worker/src/lib.rs | 7 + crates/worker/src/limits.rs | 26 +- crates/worker/src/log.rs | 5 + crates/worker/src/main.rs | 33 +- crates/worker/src/run.rs | 150 ++++++- crates/worker/src/sandbox.rs | 19 +- crates/worker/src/spawn.rs | 87 +++- crates/worker/src/verifier.rs | 29 ++ crates/worker/src/verify.rs | 97 +++++ crates/worker/src/verify_frame.rs | 321 +++++++++++++++ crates/worker/src/verify_job.rs | 361 ++++++++++++++++ crates/worker/src/watchdog.rs | 2 + crates/worker/tests/sandbox.rs | 100 +++++ crates/worker/tests/verify_loop.rs | 527 ++++++++++++++++++++++++ docker-compose.yml | 22 + 24 files changed, 2145 insertions(+), 87 deletions(-) create mode 100644 crates/worker/src/verifier.rs create mode 100644 crates/worker/src/verify.rs create mode 100644 crates/worker/src/verify_frame.rs create mode 100644 crates/worker/src/verify_job.rs create mode 100644 crates/worker/tests/verify_loop.rs diff --git a/Cargo.lock b/Cargo.lock index 436117b3..acccc3af 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -391,6 +391,7 @@ name = "faunosteo-worker" version = "0.1.0" dependencies = [ "crc32fast", + "faunosteo-core", "flate2", "image", "libc", diff --git a/crates/worker/Cargo.toml b/crates/worker/Cargo.toml index a3e0b2d3..b21071cb 100644 --- a/crates/worker/Cargo.toml +++ b/crates/worker/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "faunosteo-worker" -description = "Faunosteo file-processing worker: keyed job loop and sweeper (R1), sandboxed decode child (R0)" +description = "Faunosteo file-processing worker: keyed job loop and sweeper (R1), sandboxed decode child (R0), sheet verification (V1-2.3)" version.workspace = true edition.workspace = true license.workspace = true @@ -21,6 +21,9 @@ default = [] test-hooks = ["dep:crc32fast", "dep:flate2"] [dependencies] +# The sheet verdict (V1-2.3), run in the keyless `verify` child; default features only (no PDF, +# no curation). +faunosteo-core = { path = "../core" } image.workspace = true libwebp-sys.workspace = true seccompiler.workspace = true diff --git a/crates/worker/scripts/derive-seccomp.sh b/crates/worker/scripts/derive-seccomp.sh index 60d946d6..87f13526 100755 --- a/crates/worker/scripts/derive-seccomp.sh +++ b/crates/worker/scripts/derive-seccomp.sh @@ -1,30 +1,38 @@ #!/bin/sh -# Derives the seccomp allowlist of the decode child from a real trace and checks it against -# `sandbox::ALLOWED_SYSCALLS` (review Part IV.1: "derived by strace over the fixture corpus -# at the caps on the production image, not hand-written"). Runs in the Alpine `test` stage -# of crates/worker/Dockerfile, where the musl binary that ships is built. +# Derives the seccomp allowlist of the decode and verify children from a real trace and +# checks it against `sandbox::ALLOWED_SYSCALLS` (review Part IV.1: "derived by strace over +# the fixture corpus at the caps on the production image, not hand-written"). Runs in the +# Alpine `test` stage of crates/worker/Dockerfile, where the musl binary that ships is built. +# The verify child (V1-2.3) shares the filter: its framed fixtures (a small sheet and the +# sheet at every cap) are traced the same way. # # Exit 1 when a traced syscall is missing from the allowlist (the sandbox would kill a -# legitimate decode). Syscalls the list allows but the trace never shows are printed for -# review (the abort path is expected there). +# legitimate decode or verification). Syscalls the list allows but the trace never shows are +# printed for review (the abort path is expected there). set -eu bin=target/release/faunosteo-worker cargo build -p faunosteo-worker --release --features test-hooks --locked work=$(mktemp -d) -fixtures="png jpeg webp big-jpeg big-png" : > "$work/traced.raw" -for kind in $fixtures; do - "$bin" fixture "$kind" > "$work/$kind.bin" +# trace : one child run, the syscalls after the marker appended. +trace() { + "$bin" fixture "$2" > "$work/$2.bin" # `env -i`: the child refuses a non-empty environment; `--no-seccomp` keeps the rlimits # and makes one sched_yield where the filter would be installed (strace cannot read the # child's memory once PR_SET_DUMPABLE is 0, so the marker is a syscall, not a string). - env -i strace -f -qq -o "$work/trace.$kind" "$bin" decode --no-seccomp \ - < "$work/$kind.bin" > "$work/$kind.webp" - awk 'found { print } /sched_yield\(/ { found = 1 }' "$work/trace.$kind" \ + env -i strace -f -qq -o "$work/trace.$2" "$bin" "$1" --no-seccomp \ + < "$work/$2.bin" > "$work/$2.out" + awk 'found { print } /sched_yield\(/ { found = 1 }' "$work/trace.$2" \ | sed -E 's/^[0-9]+ +//; /^(\+\+\+|---)/d; s/^<\.\.\. ([a-z0-9_]+) resumed>.*/\1/; s/\(.*//' \ | grep -E '^[a-z0-9_]+$' >> "$work/traced.raw" || true +} +for kind in png jpeg webp big-jpeg big-png; do + trace decode "$kind" +done +for kind in verify-clean verify-caps; do + trace verify "$kind" done sort -u "$work/traced.raw" > "$work/traced.txt" "$bin" print-allowlist | sort -u > "$work/allowed.txt" @@ -41,7 +49,7 @@ missing=$(comm -23 "$work/traced.txt" "$work/allowed.txt") extra=$(comm -13 "$work/traced.txt" "$work/allowed.txt") echo "== allowed but not traced (abort path expected): $(echo "$extra" | tr '\n' ' ')" if [ -n "$missing" ]; then - echo "== TRACED BUT NOT ALLOWED (the filter would kill a legitimate decode):" + echo "== TRACED BUT NOT ALLOWED (the filter would kill a legitimate decode or verification):" echo "$missing" exit 1 fi diff --git a/crates/worker/src/backend.rs b/crates/worker/src/backend.rs index d171e41e..ed5d19ce 100644 --- a/crates/worker/src/backend.rs +++ b/crates/worker/src/backend.rs @@ -1,6 +1,6 @@ -//! What the parent needs from Supabase (STORAGE.md §2.2 RPCs and §3.3 Storage calls), -//! behind one trait so the job and sweep logic is tested on an in-memory fake and the HTTP -//! implementation stays thin. +//! What the parent needs from Supabase (STORAGE.md §2.2 RPCs and §3.3 Storage calls), behind +//! one trait so the job and sweep logic is tested on an in-memory fake and the HTTP +//! implementation stays thin. The M6 verification RPCs (V1-2.3) have a trait of their own. use std::fmt; @@ -105,3 +105,86 @@ pub trait Backend { ) -> BackendResult>; fn expire_pending_uploads(&self, max_age_seconds: u64) -> BackendResult; } + +/// One claimed round (`claim_sheet_verifications`, M6). +#[derive(Debug, Clone, Deserialize, PartialEq, Eq)] +pub struct VerifyJob { + pub review_id: String, + pub version_id: String, + pub sheet_id: String, + pub revision: u32, + pub lease_token: String, + pub attempts: i64, +} + +/// One drawing row of a round's input: registered for the sheet, referenced by the document. +#[derive(Debug, Clone, Deserialize, PartialEq, Eq)] +pub struct DrawingRow { + pub asset_id: String, + pub sha256: String, + pub byte_size: u64, + pub bucket: String, + pub object_key: String, + pub uploaded: bool, +} + +/// The input of a leased round (`get_sheet_verification_input`): the texts are passed to +/// the child as they are, never parsed by the parent. +#[derive(Debug, Clone, Deserialize, PartialEq, Eq)] +pub struct VerifyInput { + pub review_id: String, + pub version_id: String, + pub sheet_id: String, + pub revision: u32, + pub document: String, + pub base_document: Option, + pub drawings: Vec, +} + +/// `complete_sheet_verification`: `stale` and `malformed` wrote nothing. +#[derive(Debug, Clone, Copy, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +pub enum CompleteAnswer { + Passed, + Failed, + Stale, + Malformed, +} + +/// `fail_sheet_verification`: claimable again, closed `unavailable` at the attempt limit, +/// or `stale` (nothing written). +#[derive(Debug, Clone, Copy, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +pub enum FailAnswer { + Retry, + Unavailable, + Stale, +} + +/// The four RPCs of M6 and the drawing download (V1-2.3, `run --jobs sheet-verify`). +pub trait VerifyBackend { + fn claim_verifications(&self, max_jobs: u32) -> BackendResult>; + /// `None` when the lease or the round is stale. + fn verification_input( + &self, + review_id: &str, + lease_token: &str, + ) -> BackendResult>; + /// `Status(404)` when the object is missing; `Overflow` past `max_bytes`. + fn download_drawing(&self, bucket: &str, key: &str, max_bytes: usize) + -> BackendResult>; + /// `verdict` is the child's output, a JSON object. + fn complete_verification( + &self, + review_id: &str, + lease_token: &str, + verdict: &[u8], + ) -> BackendResult; + /// `code` is one of the closed worker codes (`verify_job::codes`). + fn fail_verification( + &self, + review_id: &str, + lease_token: &str, + code: &str, + ) -> BackendResult; +} diff --git a/crates/worker/src/decode.rs b/crates/worker/src/decode.rs index aee12efd..5b04caf5 100644 --- a/crates/worker/src/decode.rs +++ b/crates/worker/src/decode.rs @@ -219,7 +219,8 @@ enum HookAction { } impl TestHooks { - fn parse(args: &[String]) -> Result { + /// Shared with the `verify` child, which takes the same flags. + pub(crate) fn parse(args: &[String]) -> Result { #[cfg(not(feature = "test-hooks"))] let hooks = Self { seccomp: true }; #[cfg(feature = "test-hooks")] @@ -257,7 +258,11 @@ impl TestHooks { } #[cfg(feature = "test-hooks")] - fn run_after_sandbox(&self, stdout: &mut dyn Write, stderr: &mut dyn Write) -> Option { + pub(crate) fn run_after_sandbox( + &self, + stdout: &mut dyn Write, + stderr: &mut dyn Write, + ) -> Option { if !self.seccomp { // Marker for crates/worker/scripts/derive-seccomp.sh: one `sched_yield`, a syscall // the decode path never makes, right where the filter would be installed. The @@ -314,7 +319,11 @@ impl TestHooks { } #[cfg(not(feature = "test-hooks"))] - fn run_after_sandbox(&self, _stdout: &mut dyn Write, _stderr: &mut dyn Write) -> Option { + pub(crate) fn run_after_sandbox( + &self, + _stdout: &mut dyn Write, + _stderr: &mut dyn Write, + ) -> Option { None } } diff --git a/crates/worker/src/exit_codes.rs b/crates/worker/src/exit_codes.rs index 5d31575a..c1a53db2 100644 --- a/crates/worker/src/exit_codes.rs +++ b/crates/worker/src/exit_codes.rs @@ -1,4 +1,4 @@ -//! Exit codes of the `decode` child, reserved away from 1 (generic failure), 101 (Rust +//! Exit codes of the `decode` and `verify` children, reserved away from 1 (generic failure), 101 (Rust //! panic) and signal deaths (reported separately by the parent). The parent maps them to //! the closed rejection reasons of STEP8_FILES_ANALYSIS II.7. @@ -19,6 +19,8 @@ pub const RESOURCE_LIMIT: i32 = 19; pub const CHILD_BROKEN: i32 = 20; /// The parent's watchdog found a stage past its deadline and exited (R1). pub const WATCHDOG: i32 = 21; +/// The `verify` child could not read its framed input: a parent bug, never the content's. +pub const FRAME_INVALID: i32 = 22; /// Codes that judge the content (the upload is refused). Every other non-zero code means /// the child's environment is broken (`Broken` outcome: the parent alerts rather than @@ -48,6 +50,7 @@ pub fn name(code: i32) -> &'static str { RESOURCE_LIMIT => "resource_limit", CHILD_BROKEN => "child_broken", WATCHDOG => "watchdog", + FRAME_INVALID => "frame_invalid", _ => "unknown", } } diff --git a/crates/worker/src/fixtures.rs b/crates/worker/src/fixtures.rs index 387bc096..00b2a062 100644 --- a/crates/worker/src/fixtures.rs +++ b/crates/worker/src/fixtures.rs @@ -1,9 +1,15 @@ -//! Synthetic images for the sandbox tests and the seccomp derivation (`test-hooks` only): -//! nothing binary lives in the repository. +//! Synthetic images and framed sheet inputs for the sandbox tests and the seccomp +//! derivation (`test-hooks` only): nothing binary lives in the repository. use std::io::Cursor; +use faunosteo_core::domain::sheet::verdict::{ + VerdictBinding, VerdictDrawing, VerdictInput, sha256_hex, +}; use image::{DynamicImage, ImageBuffer, ImageFormat, Rgba}; +use serde_json::{Value, json}; + +use crate::verify_frame; fn gradient(width: u32, height: u32) -> DynamicImage { let buffer = ImageBuffer::from_fn(width, height, |x, y| { @@ -107,6 +113,110 @@ pub fn truncated_png() -> Vec { bytes } +/// The asset id the verification fixtures draw on. +pub const SHEET_ASSET: &str = "11111111-2222-4333-8444-555555555555"; +/// A drawing in its stored form, as the page's cleaning uploads it +/// (`crates/core/tests/sheet_verdict.rs`). +pub const SHEET_DRAWING: &str = concat!( + "\n", + r#""# +); + +/// The round of the verification fixtures. +pub fn sheet_binding() -> VerdictBinding { + VerdictBinding { + sheet_id: "aaaaaaaa-0000-4000-8000-000000000001".to_owned(), + version_id: "aaaaaaaa-0000-4000-8000-000000000002".to_owned(), + revision: 3, + review_id: "aaaaaaaa-0000-4000-8000-000000000003".to_owned(), + } +} + +fn name(text: &str) -> Value { + json!({ "fr": text, "en": text }) +} + +/// A small valid document drawn on `SHEET_ASSET` (the core's `sheet_verdict::document`). +pub fn sheet_document() -> Value { + json!({ + "schema_version": 2, "species_id": "canine", "name": name("Chien"), + "description": {}, "scope_disclaimer": { "fr": "Outil", "en": "Tool" }, + "attribution": { "authors": ["A"], "license": null, "notes": null }, + "vertebral_config": { "regions": [] }, "references": [], + "categories": [{ "id": "c", "name": name("C"), "color": null }], + "views": [{ "id": "main", "kind": "main", "name": name("Vue"), "svg_asset_id": SHEET_ASSET, "view_box": null }], + "annotations": [{ "id": "a1", "kind": "point", "view_id": "main", "category_id": "c", + "laterality": "unpaired", "geometry": { "type": "circle", "cx": 1, "cy": 1, "r": 1 }, + "review_status": "to_confirm", "name": name("P"), "finding_zone_id": "z1", "vertebral_region_code": null }], + "finding_zones": [{ "id": "z1", "name": name("Z"), "categories": [{ "id": "cat", "kind": "dysfunction", + "name": name("D"), "review_status": "to_confirm", + "children": [{ "id": "n1", "name": name("N"), "review_status": "to_confirm" }] }] }] + }) +} + +/// A document at every cap (the core's `sheet_verdict::at_caps`): 64 views, 1024 +/// annotations, 8192 finding nodes, depth 8. +pub fn sheet_document_at_caps() -> Value { + let mut doc = sheet_document(); + let mut views = vec![ + json!({ "id": "main", "kind": "main", "name": name("V"), "svg_asset_id": SHEET_ASSET, "view_box": null }), + ]; + let mut annotations = Vec::new(); + for i in 1..64 { + views.push(json!({ "id": format!("d{i}"), "kind": "detail", "name": name("D"), "svg_asset_id": null, + "view_box": { "min_x": 0, "min_y": 0, "width": 5, "height": 5 }, "parent_view_id": "main" })); + annotations.push(json!({ "id": format!("t{i}"), "kind": "detail_trigger", "view_id": "main", "category_id": "c", + "laterality": "unpaired", "geometry": { "type": "polygon", "points": [[0, 0], [1, 0], [1, 1]] }, + "review_status": "reviewed", "name": null, "opens_view_id": format!("d{i}") })); + } + let zones: Vec = (0..64) + .map(|z| { + let mut children = vec![chain_reviewed(7)]; + children.extend((0..120).map(|n| json!({ "id": format!("l{n}"), "name": name("Nœud de lésion"), "review_status": "reviewed" }))); + json!({ "id": format!("z{z}"), "name": name("Zone"), "categories": [{ "id": "cat", "kind": "dysfunction", + "name": name("Catégorie"), "review_status": "reviewed", "children": children }] }) + }) + .collect(); + for i in 0..(1024 - 63) { + annotations.push(json!({ "id": format!("p{i}"), "kind": "point", "view_id": format!("d{}", 1 + i % 63), + "category_id": "c", "laterality": "left", "geometry": { "type": "circle", "cx": 1.25, "cy": 2.5, "r": 0.5 }, + "review_status": "reviewed", "name": name("Point"), "finding_zone_id": format!("z{}", i % 64), + "vertebral_region_code": null })); + } + doc["views"] = Value::Array(views); + doc["annotations"] = Value::Array(annotations); + doc["finding_zones"] = Value::Array(zones); + doc +} + +fn chain_reviewed(levels: usize) -> Value { + let mut node = + json!({ "id": format!("c{levels}"), "name": name("N"), "review_status": "reviewed" }); + for level in (1..levels).rev() { + node = json!({ "id": format!("c{level}"), "name": name("N"), "review_status": "reviewed", "children": [node] }); + } + node +} + +/// The framed input of `document` drawn on `SHEET_DRAWING`, the document also as its base +/// (the base path is traced too); empty on a framing error. +pub fn sheet_frame(document: &Value) -> Vec { + let text = document.to_string(); + let sha256 = sha256_hex(SHEET_DRAWING.as_bytes()); + let input = VerdictInput { + binding: sheet_binding(), + document: &text, + base_document: Some(&text), + drawings: vec![VerdictDrawing { + asset_id: SHEET_ASSET, + sha256: &sha256, + byte_size: SHEET_DRAWING.len() as u64, + bytes: Some(SHEET_DRAWING.as_bytes()), + }], + }; + verify_frame::encode(&input).unwrap_or_default() +} + /// Writes one named fixture on stdout (`faunosteo-worker fixture `). pub fn by_name(name: &str) -> Option> { Some(match name { @@ -122,6 +232,8 @@ pub fn by_name(name: &str) -> Option> { "iccp-bomb" => png_iccp_bomb(300 << 20), "gif" => gif(), "truncated" => truncated_png(), + "verify-clean" => sheet_frame(&sheet_document()), + "verify-caps" => sheet_frame(&sheet_document_at_caps()), _ => return None, }) } diff --git a/crates/worker/src/http.rs b/crates/worker/src/http.rs index e42ee6a7..49fbf4fd 100644 --- a/crates/worker/src/http.rs +++ b/crates/worker/src/http.rs @@ -1,6 +1,8 @@ //! `Backend` over PostgREST and the Storage API with the service key (STORAGE.md §3.3). //! Thin by design: key shapes are checked before any URL is built, every status is //! classified here, and the closed `P0001` refusals of the RPCs come back as `Refused`. +//! The same client serves `VerifyBackend` (V1-2.3): larger caps for the round's input and +//! the verdict, drawing keys checked before any URL is built. use std::time::Duration; @@ -9,13 +11,19 @@ use serde_json::{Value, json}; use ureq::Agent; use ureq::http::Response; -use crate::backend::{Backend, BackendError, BackendResult, Job, Leftover, Purge}; +use crate::backend::{ + Backend, BackendError, BackendResult, CompleteAnswer, FailAnswer, Job, Leftover, Purge, + VerifyBackend, VerifyInput, VerifyJob, +}; use crate::config::Config; use crate::keys; use crate::limits::MAX_INPUT_BYTES; /// Largest RPC response the worker reads (a claim of 100 rows is a few KiB). const MAX_RPC_BODY: u64 = 1024 * 1024; +/// Largest `get_sheet_verification_input` response: two 2 MiB documents, whose JSON string +/// escaping can double them, and the drawing rows. +const MAX_INPUT_RPC_BODY: u64 = 16 * 1024 * 1024; pub struct HttpBackend { agent: Agent, @@ -52,8 +60,22 @@ impl HttpBackend { Ok(format!("{}/storage/v1/object/{bucket}/{key}", self.base)) } + /// A sheet drawing: `.svg` in a well-formed bucket, nothing else. + fn drawing_url(&self, bucket: &str, key: &str) -> BackendResult { + validate_bucket(bucket)?; + if !keys::is_drawing_key(key) { + return Err(BackendError::Internal("drawing key refused".to_owned())); + } + Ok(format!("{}/storage/v1/object/{bucket}/{key}", self.base)) + } + /// Calls an RPC and returns the response body text of a 2xx. fn rpc(&self, name: &str, body: &Value) -> BackendResult { + self.rpc_capped(name, body, MAX_RPC_BODY) + } + + /// `rpc` with its own response cap. + fn rpc_capped(&self, name: &str, body: &Value, cap: u64) -> BackendResult { let mut response = self .agent .post(&self.rpc_url(name)) @@ -63,7 +85,7 @@ impl HttpBackend { .send_json(body) .map_err(network)?; let status = response.status().as_u16(); - let text = read_text(&mut response)?; + let text = read_text_capped(&mut response, cap)?; if (200..300).contains(&status) { Ok(text) } else { @@ -79,18 +101,12 @@ impl HttpBackend { fn rpc_void(&self, name: &str, body: &Value) -> BackendResult<()> { self.rpc(name, body).map(|_| ()) } -} -impl Backend for HttpBackend { - fn claim_jobs(&self, max_jobs: u32) -> BackendResult> { - self.rpc_json("claim_avatar_jobs", &json!({ "max_jobs": max_jobs })) - } - - fn download_quarantine(&self, bucket: &str, key: &str) -> BackendResult> { - let url = self.object_url(bucket, key)?; + /// GET of an object, read up to `cap` bytes (`Overflow` past it). + fn get_object(&self, url: &str, cap: usize) -> BackendResult> { let mut response = self .agent - .get(&url) + .get(url) .header("apikey", &self.key) .header("Authorization", &format!("Bearer {}", self.key)) .call() @@ -103,14 +119,25 @@ impl Backend for HttpBackend { let bytes = response .body_mut() .with_config() - .limit(MAX_INPUT_BYTES as u64 + 1) + .limit(cap as u64 + 1) .read_to_vec() .map_err(|e| BackendError::Network(e.to_string()))?; - if bytes.len() > MAX_INPUT_BYTES { + if bytes.len() > cap { return Err(BackendError::Overflow); } Ok(bytes) } +} + +impl Backend for HttpBackend { + fn claim_jobs(&self, max_jobs: u32) -> BackendResult> { + self.rpc_json("claim_avatar_jobs", &json!({ "max_jobs": max_jobs })) + } + + fn download_quarantine(&self, bucket: &str, key: &str) -> BackendResult> { + let url = self.object_url(bucket, key)?; + self.get_object(&url, MAX_INPUT_BYTES) + } fn put_clean(&self, bucket: &str, key: &str, bytes: &[u8]) -> BackendResult<()> { let url = self.object_url(bucket, key)?; @@ -194,6 +221,65 @@ impl Backend for HttpBackend { } } +impl VerifyBackend for HttpBackend { + fn claim_verifications(&self, max_jobs: u32) -> BackendResult> { + self.rpc_json( + "claim_sheet_verifications", + &json!({ "max_jobs": max_jobs }), + ) + } + + fn verification_input( + &self, + review_id: &str, + lease_token: &str, + ) -> BackendResult> { + let text = self.rpc_capped( + "get_sheet_verification_input", + &json!({ "review_id": review_id, "lease_token": lease_token }), + MAX_INPUT_RPC_BODY, + )?; + serde_json::from_str(&text).map_err(|e| BackendError::Decode(e.to_string())) + } + + fn download_drawing( + &self, + bucket: &str, + key: &str, + max_bytes: usize, + ) -> BackendResult> { + let url = self.drawing_url(bucket, key)?; + self.get_object(&url, max_bytes) + } + + fn complete_verification( + &self, + review_id: &str, + lease_token: &str, + verdict: &[u8], + ) -> BackendResult { + // The spawner already checked the bytes are a JSON object (under the output cap). + let verdict: Value = + serde_json::from_slice(verdict).map_err(|e| BackendError::Internal(e.to_string()))?; + self.rpc_json( + "complete_sheet_verification", + &json!({ "review_id": review_id, "lease_token": lease_token, "verdict": verdict }), + ) + } + + fn fail_verification( + &self, + review_id: &str, + lease_token: &str, + code: &str, + ) -> BackendResult { + self.rpc_json( + "fail_sheet_verification", + &json!({ "review_id": review_id, "lease_token": lease_token, "code": code }), + ) + } +} + fn validate_bucket(bucket: &str) -> BackendResult<()> { let ok = !bucket.is_empty() && bucket @@ -211,10 +297,14 @@ fn network(error: ureq::Error) -> BackendError { } fn read_text(response: &mut Response) -> BackendResult { + read_text_capped(response, MAX_RPC_BODY) +} + +fn read_text_capped(response: &mut Response, cap: u64) -> BackendResult { response .body_mut() .with_config() - .limit(MAX_RPC_BODY) + .limit(cap) .read_to_string() .map_err(|e| BackendError::Network(e.to_string())) } @@ -295,6 +385,36 @@ mod tests { ); } + #[test] + fn the_m6_answers_and_rows_deserialize() { + assert_eq!( + serde_json::from_str::("\"malformed\"").ok(), + Some(CompleteAnswer::Malformed) + ); + assert_eq!( + serde_json::from_str::("\"retry\"").ok(), + Some(FailAnswer::Retry) + ); + assert!(serde_json::from_str::("\"other\"").is_err()); + assert_eq!( + serde_json::from_str::>("null").ok(), + Some(None) + ); + let input = serde_json::from_value::>(json!({ + "review_id": "r", "version_id": "v", "sheet_id": "s", "revision": 2, + "document": "{}", "base_document": null, + "drawings": [{ "asset_id": "a", "sha256": "h", "byte_size": 10, + "bucket": "anatomical-sheet-assets", "object_key": "h.svg", "uploaded": true }] + })); + assert_eq!( + input.ok().flatten().map(|i| i.drawings[0].byte_size), + Some(10) + ); + let jobs = serde_json::from_value::>(json!([{ "review_id": "r", + "version_id": "v", "sheet_id": "s", "revision": 1, "lease_token": "t", "attempts": 1 }])); + assert_eq!(jobs.ok().map(|j| j[0].revision), Some(1)); + } + #[test] fn object_urls_need_valid_buckets_and_keys() { let config = Config::from_vars(|name| match name { @@ -320,6 +440,17 @@ mod tests { .object_url("avatars", "profiles/../etc/passwd") .is_err() ); + let drawing = format!("{}.svg", "ab".repeat(32)); + assert_eq!( + backend + .drawing_url("anatomical-sheet-assets", &drawing) + .ok(), + Some(format!( + "http://kong:8000/storage/v1/object/anatomical-sheet-assets/{drawing}" + )) + ); + assert!(backend.drawing_url("anatomical-sheet-assets", key).is_err()); + assert!(backend.drawing_url("a/b", &drawing).is_err()); assert_eq!( backend.rpc_url("claim_avatar_jobs"), "http://kong:8000/rest/v1/rpc/claim_avatar_jobs" diff --git a/crates/worker/src/keys.rs b/crates/worker/src/keys.rs index 41b1e2f5..bf7676a0 100644 --- a/crates/worker/src/keys.rs +++ b/crates/worker/src/keys.rs @@ -79,6 +79,17 @@ pub fn validate_any_key(key: &str) -> Result<(), KeyError> { } } +/// Object key of a sheet drawing (bucket `anatomical-sheet-assets`): `.svg`, +/// lowercase (V1-2.3, M6 `get_sheet_verification_input`). +pub fn is_drawing_key(key: &str) -> bool { + key.strip_suffix(".svg").is_some_and(|hash| { + hash.len() == 64 + && hash + .bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) + }) +} + /// True when a (valid) key's owner and asset segments are exactly these ids. pub fn belongs_to(key: &str, owner_id: &str, asset_id: &str) -> bool { let mut segments = key.split('/'); @@ -153,4 +164,15 @@ mod tests { assert!(!belongs_to(&key, U1, U1)); assert!(!is_uuid("10000000-0000-0000-0000-00000000000")); } + + #[test] + fn drawing_keys_are_a_lowercase_sha256_and_svg() { + let hash = "0123456789abcdef".repeat(4); + assert!(is_drawing_key(&format!("{hash}.svg"))); + assert!(!is_drawing_key(&format!("{}.svg", hash.to_uppercase()))); + assert!(!is_drawing_key(&format!("{hash}.png"))); + assert!(!is_drawing_key(&format!("{}.svg", &hash[1..]))); + assert!(!is_drawing_key(&format!("../{}.svg", &hash[3..]))); + assert!(!is_drawing_key(&format!("x/{hash}.svg"))); + } } diff --git a/crates/worker/src/lib.rs b/crates/worker/src/lib.rs index cb2bdbd6..c777a77e 100644 --- a/crates/worker/src/lib.rs +++ b/crates/worker/src/lib.rs @@ -4,6 +4,9 @@ //! child with an empty environment that confines itself (rlimits + seccomp) before reading //! one image on stdin and writing one 512 × 512 WebP on stdout. The parent never decodes: //! it checks the container header of what comes back (`webp_check`). +//! +//! V1-2.3 adds the `verify` child on the same model: one framed sheet input on stdin +//! (`verify_frame`), the core's verdict JSON on stdout; `run --jobs sheet-verify` drives it. pub mod backend; pub mod config; @@ -21,5 +24,9 @@ pub mod run; pub mod sandbox; pub mod spawn; pub mod sweep; +pub mod verifier; +pub mod verify; +pub mod verify_frame; +pub mod verify_job; pub mod watchdog; pub mod webp_check; diff --git a/crates/worker/src/limits.rs b/crates/worker/src/limits.rs index f418bd04..c204ec79 100644 --- a/crates/worker/src/limits.rs +++ b/crates/worker/src/limits.rs @@ -1,7 +1,10 @@ -//! Caps of the decode child (review Part IV.1 and STEP8_FILES_ANALYSIS II.5 step 6). +//! Caps of the decode child (review Part IV.1 and STEP8_FILES_ANALYSIS II.5 step 6) and of the +//! verify child (V1-2.3). use std::time::Duration; +use faunosteo_core::domain::archive::snapshot::SnapshotCaps; + /// Largest accepted upload (STEP8 D4: 5 MiB at upload). pub const MAX_INPUT_BYTES: usize = 5 * 1024 * 1024; /// Largest output the parent reads before killing the child (a 512 × 512 WebP is ~30 KB). @@ -24,3 +27,24 @@ pub const CHILD_CPU_SECONDS: u64 = 10; pub const CHILD_DEADLINE: Duration = Duration::from_secs(20); /// Oldest libwebp the worker accepts to link (D-R4: the linked version is controlled). pub const MIN_LIBWEBP_VERSION: (u8, u8, u8) = (1, 6, 0); + +/// Caps of the `verify` child (V1-2.3, contract W3), sized on the at-caps fixture: the +/// document and the base (2 MiB each, SQL CHECK), four drawings (6 MiB in all, +/// `SnapshotCaps::ASSETS*`), the header and a margin. +pub const VERIFY_MAX_INPUT_BYTES: usize = 12 * 1024 * 1024; +/// Largest frame header (ids and drawing rows, a few KiB in practice). +pub const VERIFY_MAX_HEADER_BYTES: usize = 64 * 1024; +/// Largest verdict the parent reads (= SQL `sheet_verification_max_verdict_bytes`). +pub const VERIFY_MAX_OUTPUT_BYTES: usize = 4 * 1024 * 1024; +/// Hard address-space cap of the `verify` child (`RLIMIT_AS`). +pub const VERIFY_ADDRESS_SPACE_BYTES: u64 = 1024 * 1024 * 1024; +/// CPU-time cap of the `verify` child (`RLIMIT_CPU`, seconds). +pub const VERIFY_CPU_SECONDS: u64 = 60; +/// Wall-clock deadline of the `verify` child (contract budget: under 120 s at the caps). +pub const VERIFY_DEADLINE: Duration = Duration::from_secs(90); +/// Drawings the parent downloads for one verification, at most. +pub const VERIFY_MAX_DRAWINGS: usize = SnapshotCaps::ASSETS; +/// Largest drawing the parent downloads. +pub const VERIFY_MAX_DRAWING_BYTES: usize = SnapshotCaps::ASSET_BYTES; +/// Largest total of the drawings downloaded for one verification. +pub const VERIFY_MAX_DRAWINGS_TOTAL_BYTES: usize = SnapshotCaps::ASSETS_TOTAL_BYTES; diff --git a/crates/worker/src/log.rs b/crates/worker/src/log.rs index 1d6e73f2..0d33e4fb 100644 --- a/crates/worker/src/log.rs +++ b/crates/worker/src/log.rs @@ -57,6 +57,11 @@ impl LogLine { self.with("asset_id", id) } + /// A sheet verification round (V1-2.3). + pub fn review(self, id: &str) -> Self { + self.with("review_id", id) + } + pub fn stage(self, stage: &str) -> Self { self.with("stage", stage) } diff --git a/crates/worker/src/main.rs b/crates/worker/src/main.rs index f2639a41..e3544c01 100644 --- a/crates/worker/src/main.rs +++ b/crates/worker/src/main.rs @@ -1,14 +1,14 @@ //! `faunosteo-worker` entry: a plain `main`, no async runtime, no thread, so the `decode` //! child installs its sandbox in a single-threaded process (review Part IV.1). //! -//! Subcommands: `decode` (the sandboxed child, R0), `run` (the keyed job loop, R1), -//! `run --check` (prints the resolved configuration without secrets), `version`. +//! Subcommands: `decode` (the sandboxed child, R0), `verify` (the sandboxed sheet verdict +//! child, V1-2.3), `run` (the keyed job loop, R1), `run --jobs sheet-verify` (the verification +//! loop), `run --check` (prints the resolved configuration without secrets), `version`. #[cfg(feature = "test-hooks")] use std::io::Write; -use faunosteo_worker::decode; -use faunosteo_worker::exit_codes; +use faunosteo_worker::{decode, exit_codes, verify}; fn main() { let args: Vec = std::env::args().collect(); @@ -24,6 +24,17 @@ fn main() { &mut stderr.lock(), ) } + Some("verify") => { + let stdin = std::io::stdin(); + let stdout = std::io::stdout(); + let stderr = std::io::stderr(); + verify::main( + &args[2..], + &mut stdin.lock(), + &mut stdout.lock(), + &mut stderr.lock(), + ) + } Some("run") => faunosteo_worker::run::main(&args[2..]), Some("version") => version(), #[cfg(feature = "test-hooks")] @@ -36,7 +47,9 @@ fn main() { exit_codes::OK } _ => { - eprintln!("usage: faunosteo-worker "); + eprintln!( + "usage: faunosteo-worker " + ); exit_codes::USAGE } }; @@ -48,17 +61,11 @@ fn version() -> i32 { println!( "faunosteo-worker {} (core {}, libwebp {major}.{minor}.{revision})", env!("CARGO_PKG_VERSION"), - faunosteo_core_version() + faunosteo_core::CORE_VERSION ); exit_codes::OK } -fn faunosteo_core_version() -> &'static str { - // The worker links no core module before R1 (nothing to share yet); the version printed - // is the workspace version the two crates share. - env!("CARGO_PKG_VERSION") -} - #[cfg(feature = "test-hooks")] fn fixture(args: &[String]) -> i32 { let Some(bytes) = args @@ -66,7 +73,7 @@ fn fixture(args: &[String]) -> i32 { .and_then(|name| faunosteo_worker::fixtures::by_name(name)) else { eprintln!( - "usage: faunosteo-worker fixture " + "usage: faunosteo-worker fixture " ); return exit_codes::USAGE; }; diff --git a/crates/worker/src/run.rs b/crates/worker/src/run.rs index 4bed4af2..b1699d17 100644 --- a/crates/worker/src/run.rs +++ b/crates/worker/src/run.rs @@ -1,10 +1,11 @@ -//! `faunosteo-worker run`: the keyed parent's loop (STORAGE.md §3.4, §3.5) and -//! `run --check`, the start-up self-description without any secret value. +//! `faunosteo-worker run`: the keyed parent's loop (STORAGE.md §3.4, §3.5), `run --jobs +//! sheet-verify` (the verification loop of V1-2.3, no sweep) and `run --check`, the start-up +//! self-description without any secret value. use std::io::Write; use std::time::{Instant, SystemTime, UNIX_EPOCH}; -use crate::backend::Backend; +use crate::backend::{Backend, VerifyBackend}; use crate::config::{self, Config, ConfigError}; use crate::decode::libwebp_version; use crate::decoder::{ChildDecoder, Decoder}; @@ -12,23 +13,56 @@ use crate::http::HttpBackend; use crate::job::{JobEnd, process_job}; use crate::limits::MIN_LIBWEBP_VERSION; use crate::sweep::{SweepConfig, run_sweep}; +use crate::verifier::{ChildVerifier, Verifier}; +use crate::verify_job::{VerifyEnd, process_verification}; use crate::watchdog::{Watchdog, deadlines}; use crate::{exit_codes, log}; const THIRTY_DAYS: i64 = 30 * 24 * 3600; +/// What a `run` process works on. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Mode { + /// Avatar jobs and the sweep (the default). + Avatars, + /// Sheet verification rounds only (`--jobs sheet-verify`). + SheetVerify, +} + /// Entry of the `run` subcommand; returns the process exit code. pub fn main(args: &[String]) -> i32 { - match args.first().map(String::as_str) { - Some("--check") => check(), - Some(other) => { - eprintln!("run: unknown argument {other} (use --check to print the configuration)"); + match parse_args(args) { + Ok((mode, true)) => check(mode), + Ok((Mode::Avatars, false)) => start(), + Ok((Mode::SheetVerify, false)) => start_verify(), + Err(other) => { + eprintln!( + "run: unknown argument {other} (use --check to print the configuration, --jobs sheet-verify for the verification mode)" + ); exit_codes::USAGE } - None => start(), } } +/// `[--jobs sheet-verify] [--check]` in any order; `Err` names the first unknown argument. +pub fn parse_args(args: &[String]) -> Result<(Mode, bool), String> { + let mut mode = Mode::Avatars; + let mut check = false; + let mut iter = args.iter(); + while let Some(arg) = iter.next() { + match arg.as_str() { + "--check" if !check => check = true, + "--jobs" if mode == Mode::Avatars => match iter.next().map(String::as_str) { + Some("sheet-verify") => mode = Mode::SheetVerify, + Some(value) => return Err(format!("--jobs {value}")), + None => return Err("--jobs".to_owned()), + }, + other => return Err(other.to_owned()), + } + } + Ok((mode, check)) +} + fn start() -> i32 { let config = match Config::from_env() { Ok(config) => config, @@ -43,6 +77,7 @@ fn start() -> i32 { } log::set_worker_id(&config.worker_id); log_start(&config); + log_token_expiry(&config); let decoder = match ChildDecoder::from_current_exe() { Ok(decoder) => decoder, Err(error) => { @@ -58,6 +93,83 @@ fn start() -> i32 { run_loop(&config, &backend, &decoder, &watchdog) } +fn start_verify() -> i32 { + let config = match Config::from_env() { + Ok(config) => config, + Err(error) => { + eprintln!("run: {error}"); + return exit_codes::USAGE; + } + }; + log::set_worker_id(&config.worker_id); + log::info("worker_start") + .detail(&format!( + "mode sheet-verify; poll {}s once {}", + config.poll.as_secs(), + config.once + )) + .emit(); + log_token_expiry(&config); + let verifier = match ChildVerifier::from_current_exe() { + Ok(verifier) => verifier, + Err(error) => { + log::error("start_failed") + .detail(&format!("current_exe: {error}")) + .emit(); + return exit_codes::INTERNAL; + } + }; + let backend = HttpBackend::new(&config); + let watchdog = Watchdog::new(); + watchdog.spawn_monitor(); + verify_loop(&config, &backend, &verifier, &watchdog) +} + +/// Claims one round per pass, no sweep (the avatar worker owns it); idles on `poll`. +pub fn verify_loop( + config: &Config, + backend: &dyn VerifyBackend, + verifier: &dyn Verifier, + watchdog: &Watchdog, +) -> i32 { + loop { + let worked = match verify_pass(backend, verifier, watchdog) { + Ok(count) => count > 0, + Err(code) => return code, + }; + if config.once { + return exit_codes::OK; + } + if !worked { + std::thread::sleep(config.poll); + } + } +} + +/// One claim and the rounds it returned; `Err` carries the exit code of a stop. +fn verify_pass( + backend: &dyn VerifyBackend, + verifier: &dyn Verifier, + watchdog: &Watchdog, +) -> Result { + let jobs = { + let _stage = watchdog.enter("claim", deadlines::CLAIM); + match backend.claim_verifications(1) { + Ok(jobs) => jobs, + Err(error) => { + log_claim_failure(&error); + return Ok(0); + } + } + }; + for job in &jobs { + if process_verification(backend, verifier, watchdog, job) == VerifyEnd::Stop { + return Err(exit_codes::CHILD_BROKEN); + } + } + Ok(jobs.len()) +} + /// Sweep at start and every `sweep` interval; claim one job per pass; idle on `poll`. pub fn run_loop( config: &Config, @@ -99,11 +211,7 @@ fn claim_pass( match backend.claim_jobs(1) { Ok(jobs) => jobs, Err(error) => { - let line = log::warn("claim_failed").code(error.kind()); - match error { - crate::backend::BackendError::Status(status) => line.http_status(status).emit(), - _ => line.emit(), - } + log_claim_failure(&error); return Ok(0); } } @@ -116,6 +224,14 @@ fn claim_pass( Ok(jobs.len()) } +fn log_claim_failure(error: &crate::backend::BackendError) { + let line = log::warn("claim_failed").code(error.kind()); + match error { + crate::backend::BackendError::Status(status) => line.http_status(*status).emit(), + _ => line.emit(), + } +} + fn log_start(config: &Config) { let (major, minor, rev) = libwebp_version(); log::info("worker_start") @@ -128,6 +244,9 @@ fn log_start(config: &Config) { config.once )) .emit(); +} + +fn log_token_expiry(config: &Config) { match token_expiry(config) { TokenExpiry::NotJwt => log::info("token_no_expiry").emit(), TokenExpiry::Valid(days) => log::info("token_expiry").count(days).emit(), @@ -170,7 +289,10 @@ fn libwebp_ok() -> bool { /// worker could start with this environment, or when no worker variable is set at all /// (an unconfigured image is a valid artifact check); a partial or invalid configuration /// and a too-old libwebp exit non-zero and name the variable. -fn check() -> i32 { +fn check(mode: Mode) -> i32 { + if mode == Mode::SheetVerify { + println!("mode: sheet-verify (verification rounds only, no sweep)"); + } let (major, minor, rev) = libwebp_version(); let libwebp_ok = libwebp_ok(); println!( diff --git a/crates/worker/src/sandbox.rs b/crates/worker/src/sandbox.rs index 015cf5c3..cbf96ecb 100644 --- a/crates/worker/src/sandbox.rs +++ b/crates/worker/src/sandbox.rs @@ -1,7 +1,8 @@ -//! Confinement of the decode child (review Part IV.1), applied by the child itself from a -//! single-threaded process before it reads any input. Three layers: an empty environment -//! (cleared by the parent, checked in `decode`), rlimits on address space and CPU time, and -//! a seccomp allowlist installed with TSYNC that kills the process on any other syscall. +//! Confinement of the decode and verify children (review Part IV.1), applied by the child +//! itself from a single-threaded process before it reads any input. Three layers: an empty +//! environment (cleared by the parent, checked in `decode` and `verify`), rlimits on address +//! space and CPU time, and a seccomp allowlist installed with TSYNC that kills the process on +//! any other syscall. use std::collections::BTreeMap; use std::fmt; @@ -13,16 +14,20 @@ use seccompiler::{ use crate::limits; -/// Syscalls the decode child makes once the filter is installed (static musl binary, x86_64). +/// Syscalls the decode and verify children make once the filter is installed (static musl +/// binary, x86_64). /// /// Derived with `strace -f` over the fixture corpus at the caps (`crates/worker/scripts/ /// derive-seccomp.sh`, run by the Dockerfile stage `test`, 2026-10-12: exactly these seven /// after the marker) and proven by running every fixture through the full sandbox. musl's -/// mallocng used neither `brk` nor `mprotect`, and no `futex` appears in a single-threaded -/// decode. +/// mallocng used neither `brk` nor `mprotect` there, and no `futex` appears in a +/// single-threaded child. The verify child at the caps (2026-11-01) adds `brk`: mallocng +/// grows its heap for the core's many small allocations; it only moves the program break, +/// bounded by `RLIMIT_AS`. pub const ALLOWED_SYSCALLS: &[(&str, i64)] = &[ ("read", libc::SYS_read), ("write", libc::SYS_write), + ("brk", libc::SYS_brk), ("mmap", libc::SYS_mmap), ("munmap", libc::SYS_munmap), ("mremap", libc::SYS_mremap), diff --git a/crates/worker/src/spawn.rs b/crates/worker/src/spawn.rs index bf9d068f..97ba92ac 100644 --- a/crates/worker/src/spawn.rs +++ b/crates/worker/src/spawn.rs @@ -1,6 +1,7 @@ -//! Runs the decode child from the keyed parent (review Part IV.1): empty environment, -//! bounded output, wall-clock deadline, and the mapping of every way the child can end to -//! a closed outcome. The parent reads the container header of the output, never the pixels. +//! Runs a child from the keyed parent (review Part IV.1): empty environment, bounded +//! output, wall-clock deadline, and the mapping of every way the child can end to a closed +//! outcome. The parent reads the container header of a `decode` output, never the pixels, +//! and checks only that a `verify` output is a JSON object (V1-2.3). use std::io::{Read, Write}; use std::os::unix::process::ExitStatusExt; @@ -9,13 +10,34 @@ use std::process::{Command, ExitStatus, Stdio}; use std::sync::mpsc; use std::time::{Duration, Instant}; -use crate::limits::{CHILD_DEADLINE, MAX_OUTPUT_BYTES, OUTPUT_SIDE}; +use serde_json::{Map, Value}; + +use crate::limits::{ + CHILD_DEADLINE, MAX_OUTPUT_BYTES, OUTPUT_SIDE, VERIFY_DEADLINE, VERIFY_MAX_OUTPUT_BYTES, +}; use crate::{exit_codes, webp_check}; -/// How a decode child ended. +/// The subcommand a spawner runs; the output check depends on it. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ChildCommand { + Decode, + Verify, +} + +impl ChildCommand { + fn arg(self) -> &'static str { + match self { + Self::Decode => "decode", + Self::Verify => "verify", + } + } +} + +/// How a child ended. #[derive(Debug)] pub enum ChildOutcome { - /// A 512 × 512 WebP whose container passed the parent's check. + /// A 512 × 512 WebP whose container passed the parent's check (`decode`), or a JSON + /// object (`verify`). Produced(Vec), /// The child refused the input with one of its content exit codes /// (`exit_codes::is_content_rejection`). @@ -44,29 +66,51 @@ pub struct ChildRun { pub stderr: String, } -/// Spawns `decode` children of one worker binary. +/// Spawns children of one worker binary, one command with its caps. #[derive(Debug, Clone)] pub struct Spawner { pub exe: PathBuf, + pub command: ChildCommand, pub deadline: Duration, pub max_output: usize, pub extra_args: Vec, } impl Spawner { + /// A `decode` spawner (the R0 name, kept for its callers). pub fn new(exe: impl Into) -> Self { + Self::for_decode(exe) + } + + pub fn for_decode(exe: impl Into) -> Self { Self { exe: exe.into(), + command: ChildCommand::Decode, deadline: CHILD_DEADLINE, max_output: MAX_OUTPUT_BYTES, extra_args: Vec::new(), } } - /// Feeds `input` to a fresh child and classifies what comes back. + pub fn for_verify(exe: impl Into) -> Self { + Self { + exe: exe.into(), + command: ChildCommand::Verify, + deadline: VERIFY_DEADLINE, + max_output: VERIFY_MAX_OUTPUT_BYTES, + extra_args: Vec::new(), + } + } + + /// Runs the spawner's command (`decode` for a spawner built by `new`); the R0 name. pub fn decode(&self, input: &[u8]) -> ChildRun { + self.run(input) + } + + /// Feeds `input` to a fresh child and classifies what comes back. + pub fn run(&self, input: &[u8]) -> ChildRun { let spawned = Command::new(&self.exe) - .arg("decode") + .arg(self.command.arg()) .args(&self.extra_args) .env_clear() .stdin(Stdio::piped()) @@ -149,18 +193,15 @@ impl Spawner { } let outcome = match status { None => ChildOutcome::Timeout, - Some(status) => classify(status, output), + Some(status) => classify(self.command, status, output), }; ChildRun { outcome, stderr } } } -fn classify(status: ExitStatus, output: Vec) -> ChildOutcome { +fn classify(command: ChildCommand, status: ExitStatus, output: Vec) -> ChildOutcome { match status.code() { - Some(0) => match webp_check::expect_square(&output, OUTPUT_SIDE) { - Ok(_) => ChildOutcome::Produced(output), - Err(issue) => ChildOutcome::Failed(format!("output refused: {issue}")), - }, + Some(0) => check_output(command, output), Some(code) if exit_codes::is_content_rejection(code) => ChildOutcome::Rejected { code, name: exit_codes::name(code), @@ -175,6 +216,22 @@ fn classify(status: ExitStatus, output: Vec) -> ChildOutcome { } } +/// The output of a child that exited 0: a square WebP container, or a JSON object. +fn check_output(command: ChildCommand, output: Vec) -> ChildOutcome { + let checked = match command { + ChildCommand::Decode => webp_check::expect_square(&output, OUTPUT_SIDE) + .map(|_| ()) + .map_err(|issue| issue.to_string()), + ChildCommand::Verify => serde_json::from_slice::>(&output) + .map(|_| ()) + .map_err(|_| "not a JSON object".to_owned()), + }; + match checked { + Ok(()) => ChildOutcome::Produced(output), + Err(issue) => ChildOutcome::Failed(format!("output refused: {issue}")), + } +} + fn failed(detail: String) -> ChildRun { ChildRun { outcome: ChildOutcome::Failed(detail), diff --git a/crates/worker/src/verifier.rs b/crates/worker/src/verifier.rs new file mode 100644 index 00000000..d8e349c7 --- /dev/null +++ b/crates/worker/src/verifier.rs @@ -0,0 +1,29 @@ +//! The verify step as the job sees it: a framed input in, a classified child run out. The +//! real implementation spawns this very binary as `verify` (V1-2.3); tests inject outcomes. + +use crate::spawn::{ChildRun, Spawner}; + +pub trait Verifier { + fn verify(&self, frame: &[u8]) -> ChildRun; +} + +/// Spawns `faunosteo-worker verify` from the running executable. +#[derive(Debug, Clone)] +pub struct ChildVerifier { + spawner: Spawner, +} + +impl ChildVerifier { + /// Fails only when the executable path cannot be resolved (then nothing can be verified). + pub fn from_current_exe() -> std::io::Result { + Ok(Self { + spawner: Spawner::for_verify(std::env::current_exe()?), + }) + } +} + +impl Verifier for ChildVerifier { + fn verify(&self, frame: &[u8]) -> ChildRun { + self.spawner.run(frame) + } +} diff --git a/crates/worker/src/verify.rs b/crates/worker/src/verify.rs new file mode 100644 index 00000000..9b7cca5b --- /dev/null +++ b/crates/worker/src/verify.rs @@ -0,0 +1,97 @@ +//! The verify child (V1-2.3, contract W2): one framed input in (the round, the drawing rows, +//! the document, the base, the drawing bytes), the core's verdict JSON out. Same binary, +//! same confinement as `decode`; the verdict itself is `faunosteo_core`'s `compute_verdict`. + +use std::io::{Read, Write}; + +use faunosteo_core::domain::sheet::verdict::compute_verdict; + +use crate::decode::TestHooks; +use crate::limits::{VERIFY_ADDRESS_SPACE_BYTES, VERIFY_CPU_SECONDS, VERIFY_MAX_INPUT_BYTES}; +use crate::verify_frame::{self, FrameError}; +use crate::{exit_codes, sandbox}; + +/// Why the child produced no verdict (one exit code each). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum VerifyFailure { + /// The parent framed the input wrongly: a parent bug, never the content's. + Frame(FrameError), + /// The verdict could not be serialised: a fault of the child. + Encode, +} + +impl VerifyFailure { + pub fn exit_code(self) -> i32 { + match self { + Self::Frame(_) => exit_codes::FRAME_INVALID, + Self::Encode => exit_codes::INTERNAL, + } + } +} + +/// Entry of `faunosteo-worker verify`: returns the process exit code. +/// +/// Same order as `decode::main`: the environment check and the sandbox come before the +/// first byte of input is read. A failed verdict is exit 0: it is content, judged by the core. +pub fn main( + args: &[String], + stdin: &mut dyn Read, + stdout: &mut dyn Write, + stderr: &mut dyn Write, +) -> i32 { + if std::env::vars_os().next().is_some() { + let _ = writeln!(stderr, "verify: the environment must be empty"); + return exit_codes::ENV_NOT_EMPTY; + } + let hooks = match TestHooks::parse(args) { + Ok(hooks) => hooks, + Err(message) => { + let _ = writeln!(stderr, "verify: {message}"); + return exit_codes::USAGE; + } + }; + let limits = sandbox::Limits { + address_space_bytes: VERIFY_ADDRESS_SPACE_BYTES, + cpu_seconds: VERIFY_CPU_SECONDS, + }; + if let Err(error) = sandbox::apply(&limits, hooks.seccomp) { + let _ = writeln!(stderr, "verify: sandbox: {error}"); + return exit_codes::SANDBOX_FAILED; + } + if let Some(code) = hooks.run_after_sandbox(stdout, stderr) { + return code; + } + + let mut input = Vec::new(); + let read = stdin + .take(VERIFY_MAX_INPUT_BYTES as u64 + 1) + .read_to_end(&mut input); + if read.is_err() { + return exit_codes::INTERNAL; + } + if input.len() > VERIFY_MAX_INPUT_BYTES { + let _ = writeln!(stderr, "verify: input over {VERIFY_MAX_INPUT_BYTES} bytes"); + return exit_codes::INPUT_TOO_LARGE; + } + + match process(&input) { + Ok(verdict) => match stdout.write_all(&verdict).and_then(|()| stdout.flush()) { + Ok(()) => exit_codes::OK, + Err(_) => exit_codes::OUTPUT_FAILED, + }, + Err(failure) => { + let _ = match failure { + VerifyFailure::Frame(error) => writeln!(stderr, "verify: frame: {error}"), + VerifyFailure::Encode => writeln!(stderr, "verify: verdict not serialised"), + }; + failure.exit_code() + } + } +} + +/// Reads one frame and returns the verdict JSON; pure function of the bytes. +pub fn process(frame: &[u8]) -> Result, VerifyFailure> { + let frame = verify_frame::decode(frame).map_err(VerifyFailure::Frame)?; + let verdict = compute_verdict(&frame.input()); + serde_json::to_vec(&verdict).map_err(|_| VerifyFailure::Encode) +} diff --git a/crates/worker/src/verify_frame.rs b/crates/worker/src/verify_frame.rs new file mode 100644 index 00000000..0dcde32d --- /dev/null +++ b/crates/worker/src/verify_frame.rs @@ -0,0 +1,321 @@ +//! The framed stdin input of the `verify` child (V1-2.3), written by the parent and read by +//! the child: `[u32 LE header length][header JSON][document][base?][drawing bytes in header +//! order]`. The parent frames bytes it never parses (document, base, drawings); the child +//! reads them as borrowed slices. + +use std::fmt; + +use faunosteo_core::domain::sheet::verdict::{VerdictBinding, VerdictDrawing, VerdictInput}; +use serde::{Deserialize, Serialize}; + +use crate::limits::VERIFY_MAX_HEADER_BYTES; + +/// The header: the round, the lengths of the parts that follow, the drawing rows. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct FrameHeader { + pub sheet_id: String, + pub version_id: String, + pub revision: u32, + pub review_id: String, + pub document_len: usize, + pub base_len: Option, + pub drawings: Vec, +} + +/// One drawing row; `bytes_len` is absent when the parent passes the row without bytes. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct FrameDrawing { + pub asset_id: String, + pub sha256: String, + pub byte_size: u64, + pub bytes_len: Option, +} + +/// A decoded frame, borrowing the input buffer. +#[derive(Debug)] +pub struct Frame<'a> { + pub header: FrameHeader, + pub document: &'a str, + pub base: Option<&'a str>, + /// One entry per header drawing, in header order. + pub drawing_bytes: Vec>, +} + +/// Why a frame could not be built or read (closed: the child's stderr names it). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum FrameError { + Truncated, + HeaderTooLarge, + HeaderInvalid, + LengthMismatch, + NotUtf8, +} + +impl fmt::Display for FrameError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let text = match self { + Self::Truncated => "frame shorter than its header", + Self::HeaderTooLarge => "header over its cap", + Self::HeaderInvalid => "header is not the expected JSON", + Self::LengthMismatch => "part lengths do not sum to the frame", + Self::NotUtf8 => "document or base is not UTF-8", + }; + f.write_str(text) + } +} + +impl std::error::Error for FrameError {} + +/// Frames a verification input (the parent side). +pub fn encode(input: &VerdictInput<'_>) -> Result, FrameError> { + let header = header_of(input); + let header_json = serde_json::to_vec(&header).map_err(|_| FrameError::HeaderInvalid)?; + if header_json.len() > VERIFY_MAX_HEADER_BYTES { + return Err(FrameError::HeaderTooLarge); + } + let header_len = u32::try_from(header_json.len()).map_err(|_| FrameError::HeaderTooLarge)?; + let body = checked_body_len(&header).unwrap_or(0); + let mut out = Vec::with_capacity(4 + header_json.len() + body); + out.extend_from_slice(&header_len.to_le_bytes()); + out.extend_from_slice(&header_json); + out.extend_from_slice(input.document.as_bytes()); + if let Some(base) = input.base_document { + out.extend_from_slice(base.as_bytes()); + } + for bytes in input.drawings.iter().filter_map(|d| d.bytes) { + out.extend_from_slice(bytes); + } + Ok(out) +} + +fn header_of(input: &VerdictInput<'_>) -> FrameHeader { + FrameHeader { + sheet_id: input.binding.sheet_id.clone(), + version_id: input.binding.version_id.clone(), + revision: input.binding.revision, + review_id: input.binding.review_id.clone(), + document_len: input.document.len(), + base_len: input.base_document.map(str::len), + drawings: input + .drawings + .iter() + .map(|d| FrameDrawing { + asset_id: d.asset_id.to_owned(), + sha256: d.sha256.to_owned(), + byte_size: d.byte_size, + bytes_len: d.bytes.map(<[u8]>::len), + }) + .collect(), + } +} + +/// Total length of the parts after the header; `None` on overflow. +fn checked_body_len(header: &FrameHeader) -> Option { + header + .drawings + .iter() + .filter_map(|d| d.bytes_len) + .try_fold(header.document_len, usize::checked_add)? + .checked_add(header.base_len.unwrap_or(0)) +} + +/// Reads a frame (the child side): every length must be exact, nothing may trail. +pub fn decode(bytes: &[u8]) -> Result, FrameError> { + let (prefix, rest) = bytes.split_at_checked(4).ok_or(FrameError::Truncated)?; + let mut length = [0u8; 4]; + length.copy_from_slice(prefix); + let header_len = + usize::try_from(u32::from_le_bytes(length)).map_err(|_| FrameError::HeaderTooLarge)?; + if header_len > VERIFY_MAX_HEADER_BYTES { + return Err(FrameError::HeaderTooLarge); + } + let (header_json, mut body) = rest + .split_at_checked(header_len) + .ok_or(FrameError::Truncated)?; + let header: FrameHeader = + serde_json::from_slice(header_json).map_err(|_| FrameError::HeaderInvalid)?; + if checked_body_len(&header) != Some(body.len()) { + return Err(FrameError::LengthMismatch); + } + let document = take_text(&mut body, header.document_len)?; + let base = header + .base_len + .map(|len| take_text(&mut body, len)) + .transpose()?; + let drawing_bytes = header + .drawings + .iter() + .map(|d| d.bytes_len.map(|len| take(&mut body, len))) + .collect(); + Ok(Frame { + header, + document, + base, + drawing_bytes, + }) +} + +/// Splits `len` bytes off the front; the lengths were checked against the body beforehand. +fn take<'a>(body: &mut &'a [u8], len: usize) -> &'a [u8] { + let (head, tail) = body.split_at(len.min(body.len())); + *body = tail; + head +} + +fn take_text<'a>(body: &mut &'a [u8], len: usize) -> Result<&'a str, FrameError> { + std::str::from_utf8(take(body, len)).map_err(|_| FrameError::NotUtf8) +} + +impl Frame<'_> { + /// The core's input, borrowing the frame. + pub fn input(&self) -> VerdictInput<'_> { + VerdictInput { + binding: VerdictBinding { + sheet_id: self.header.sheet_id.clone(), + version_id: self.header.version_id.clone(), + revision: self.header.revision, + review_id: self.header.review_id.clone(), + }, + document: self.document, + base_document: self.base, + drawings: self + .header + .drawings + .iter() + .zip(&self.drawing_bytes) + .map(|(row, bytes)| VerdictDrawing { + asset_id: &row.asset_id, + sha256: &row.sha256, + byte_size: row.byte_size, + bytes: *bytes, + }) + .collect(), + } + } +} + +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used)] + use super::*; + + fn binding() -> VerdictBinding { + VerdictBinding { + sheet_id: "s".to_owned(), + version_id: "v".to_owned(), + revision: 3, + review_id: "r".to_owned(), + } + } + + fn input<'a>(base: Option<&'a str>, drawings: Vec>) -> VerdictInput<'a> { + VerdictInput { + binding: binding(), + document: "{\"d\":\"é\"}", + base_document: base, + drawings, + } + } + + fn drawing(bytes: Option<&[u8]>) -> VerdictDrawing<'_> { + VerdictDrawing { + asset_id: "a", + sha256: "h", + byte_size: 3, + bytes, + } + } + + /// Rewrites the header of a valid frame with `edit`, keeping its body. + fn with_header(frame: &[u8], edit: impl Fn(&mut serde_json::Value)) -> Vec { + let len = u32::from_le_bytes(frame[..4].try_into().unwrap()) as usize; + let mut header: serde_json::Value = serde_json::from_slice(&frame[4..4 + len]).unwrap(); + edit(&mut header); + let json = serde_json::to_vec(&header).unwrap(); + let mut out = (json.len() as u32).to_le_bytes().to_vec(); + out.extend_from_slice(&json); + out.extend_from_slice(&frame[4 + len..]); + out + } + + #[test] + fn a_frame_round_trips() { + let original = input( + Some("{}"), + vec![drawing(Some(b"abc")), drawing(None), drawing(Some(b"xyz"))], + ); + let bytes = encode(&original).unwrap(); + let frame = decode(&bytes).unwrap(); + assert_eq!(frame.document, original.document); + assert_eq!(frame.base, Some("{}")); + assert_eq!( + frame.drawing_bytes, + vec![Some(&b"abc"[..]), None, Some(&b"xyz"[..])] + ); + let read = frame.input(); + assert_eq!(read.binding, original.binding); + assert_eq!(read.drawings.len(), 3); + assert_eq!(read.drawings[1].bytes, None); + assert_eq!(read.drawings[2].byte_size, 3); + } + + #[test] + fn an_absent_and_an_empty_base_differ() { + let none = encode(&input(None, vec![])).unwrap(); + assert_eq!(decode(&none).unwrap().base, None); + let empty = encode(&input(Some(""), vec![])).unwrap(); + assert_eq!(decode(&empty).unwrap().base, Some("")); + } + + #[test] + fn malformed_frames_are_refused() { + let good = encode(&input(Some("{}"), vec![drawing(Some(b"abc"))])).unwrap(); + assert_eq!(decode(&good[..3]).err(), Some(FrameError::Truncated)); + assert_eq!(decode(&good[..10]).err(), Some(FrameError::Truncated)); + assert_eq!( + decode(&good[..good.len() - 1]).err(), + Some(FrameError::LengthMismatch) + ); + let mut trailing = good.clone(); + trailing.push(0); + assert_eq!(decode(&trailing).err(), Some(FrameError::LengthMismatch)); + let mut huge = ((VERIFY_MAX_HEADER_BYTES + 1) as u32) + .to_le_bytes() + .to_vec(); + huge.extend(vec![b' '; VERIFY_MAX_HEADER_BYTES + 1]); + assert_eq!(decode(&huge).err(), Some(FrameError::HeaderTooLarge)); + let mut not_json = 3u32.to_le_bytes().to_vec(); + not_json.extend_from_slice(b"{x}"); + assert_eq!(decode(¬_json).err(), Some(FrameError::HeaderInvalid)); + let unknown = with_header(&good, |h| h["extra"] = 1.into()); + assert_eq!(decode(&unknown).err(), Some(FrameError::HeaderInvalid)); + let overflow = with_header(&good, |h| h["document_len"] = usize::MAX.into()); + assert_eq!(decode(&overflow).err(), Some(FrameError::LengthMismatch)); + } + + #[test] + fn a_document_or_base_that_is_not_utf8_is_refused() { + let good = encode(&input(Some("ab"), vec![])).unwrap(); + let mut bad_base = good.clone(); + let last = bad_base.len() - 1; + bad_base[last] = 0xff; + assert_eq!(decode(&bad_base).err(), Some(FrameError::NotUtf8)); + let mut bad_document = good; + let len = u32::from_le_bytes(bad_document[..4].try_into().unwrap()) as usize; + bad_document[4 + len] = 0xc3; + assert_eq!(decode(&bad_document).err(), Some(FrameError::NotUtf8)); + } + + #[test] + fn missing_drawing_bytes_are_a_length_mismatch() { + let good = encode(&input(None, vec![drawing(Some(b"abc"))])).unwrap(); + let missing = &good[..good.len() - 3]; + assert_eq!(decode(missing).err(), Some(FrameError::LengthMismatch)); + let without = with_header(&good[..good.len() - 3], |h| { + h["drawings"][0]["bytes_len"] = serde_json::Value::Null; + }); + assert_eq!(decode(&without).unwrap().drawing_bytes, vec![None]); + } +} diff --git a/crates/worker/src/verify_job.rs b/crates/worker/src/verify_job.rs new file mode 100644 index 00000000..2cb3fc5c --- /dev/null +++ b/crates/worker/src/verify_job.rs @@ -0,0 +1,361 @@ +//! One sheet verification (V1-2.3, contract W4; V1-2 contract §6.4, §6.5): read the leased +//! round's input, download the drawings it references, hand everything framed to the +//! sandboxed `verify` child, record its verdict. A missing or refused drawing is passed to +//! the core without bytes (a business failure the verdict names); only the transport, the +//! child's limits and a broken child are worker failures, each reported with a closed code. + +use std::time::Instant; + +use faunosteo_core::domain::sheet::verdict::{VerdictBinding, VerdictDrawing, VerdictInput}; + +use crate::backend::{ + BackendError, CompleteAnswer, DrawingRow, FailAnswer, VerifyBackend, VerifyInput, VerifyJob, +}; +use crate::limits::{ + VERIFY_MAX_DRAWING_BYTES, VERIFY_MAX_DRAWINGS, VERIFY_MAX_DRAWINGS_TOTAL_BYTES, +}; +use crate::spawn::{ChildOutcome, ChildRun}; +use crate::verifier::Verifier; +use crate::watchdog::{Watchdog, deadlines}; +use crate::{keys, log, verify_frame}; + +/// The only bucket a sheet drawing is read from (M6 `get_sheet_verification_input`). +pub const DRAWING_BUCKET: &str = "anatomical-sheet-assets"; + +/// Closed worker codes `fail_sheet_verification` accepts (M6 `private.sheet_worker_codes`). +pub mod codes { + pub const UNAVAILABLE: &str = "verification.worker.unavailable"; + pub const TIMEOUT: &str = "verification.worker.timeout"; + pub const RESOURCE_LIMIT: &str = "verification.worker.resource_limit"; + pub const VERDICT_REFUSED: &str = "verification.worker.verdict_refused"; +} + +/// How a verification ended, as the loop sees it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum VerifyEnd { + /// The verdict was recorded `passed`: the round is `open`. + Passed, + /// The verdict was recorded `failed`: the round is `verification_failed`. + Failed, + /// The round moved on without us (input, completion or failure answered `stale`). + Stale, + /// A worker failure was recorded with this closed code. + Reported(&'static str), + /// The failure could not be recorded either: the lease expires and the round is + /// claimed again. + LeftForLease, + /// The child reported a broken environment: the loop must stop. + Stop, +} + +/// Runs one claimed round to its end. +pub fn process_verification( + backend: &dyn VerifyBackend, + verifier: &dyn Verifier, + watchdog: &Watchdog, + job: &VerifyJob, +) -> VerifyEnd { + let started = Instant::now(); + let end = run(backend, verifier, watchdog, job); + let line = match &end { + VerifyEnd::Passed => log::info("verify_passed"), + VerifyEnd::Failed => log::info("verify_failed"), + VerifyEnd::Stale => log::info("verify_stale"), + VerifyEnd::Reported(code) => log::warn("verify_reported").code(code), + VerifyEnd::LeftForLease => log::warn("verify_left_for_lease"), + VerifyEnd::Stop => log::error("verify_stop"), + }; + line.review(&job.review_id) + .duration_ms(started.elapsed().as_millis()) + .emit(); + end +} + +fn run( + backend: &dyn VerifyBackend, + verifier: &dyn Verifier, + watchdog: &Watchdog, + job: &VerifyJob, +) -> VerifyEnd { + if !keys::is_uuid(&job.review_id) || !keys::is_uuid(&job.lease_token) { + log::error("verify_invalid_identity").emit(); + return VerifyEnd::LeftForLease; + } + let input = match read_input(backend, watchdog, job) { + Ok(Some(input)) => input, + Ok(None) => { + log::info("verify_stale_input") + .review(&job.review_id) + .emit(); + return VerifyEnd::Stale; + } + Err(error) => return transport_failure(backend, job, "input", &error), + }; + let drawings = match fetch_drawings(backend, watchdog, job, &input.drawings) { + Ok(drawings) => drawings, + Err(error) => return transport_failure(backend, job, "drawing", &error), + }; + let Ok(frame) = verify_frame::encode(&verdict_input(&input, &drawings)) else { + log::error("verify_frame_refused") + .review(&job.review_id) + .emit(); + return report(backend, job, codes::RESOURCE_LIMIT); + }; + let child = { + let _stage = watchdog.enter("verify", deadlines::VERIFY); + classify_child(verifier.verify(&frame), job) + }; + match child { + ChildVerdict::Verdict(verdict) => complete(backend, watchdog, job, &verdict), + ChildVerdict::Report(code) => report(backend, job, code), + ChildVerdict::Stop => { + report(backend, job, codes::UNAVAILABLE); + VerifyEnd::Stop + } + } +} + +fn read_input( + backend: &dyn VerifyBackend, + watchdog: &Watchdog, + job: &VerifyJob, +) -> Result, BackendError> { + let _stage = watchdog.enter("input", deadlines::DOWNLOAD); + backend.verification_input(&job.review_id, &job.lease_token) +} + +/// The bytes of each row, or `None` for a row passed without bytes. `Err` only on a +/// transport failure (the round is reported, not judged). +fn fetch_drawings( + backend: &dyn VerifyBackend, + watchdog: &Watchdog, + job: &VerifyJob, + rows: &[DrawingRow], +) -> Result>>, BackendError> { + let mut budget = DownloadBudget::default(); + let mut drawings = Vec::with_capacity(rows.len()); + for row in rows { + let bytes = match budget.allowance(row) { + Some(max_bytes) => download(backend, watchdog, job, row, max_bytes)?, + None => { + log::info("verify_drawing_skipped") + .review(&job.review_id) + .asset(&row.asset_id) + .emit(); + None + } + }; + if let Some(bytes) = &bytes { + budget.spend(bytes.len()); + } + drawings.push(bytes); + } + Ok(drawings) +} + +/// The download policy (`limits::VERIFY_MAX_DRAWING*`): at most so many drawings, each and +/// all under their caps. What it leaves out the core reports (missing or caps). +#[derive(Debug, Default)] +struct DownloadBudget { + count: usize, + total: usize, +} + +impl DownloadBudget { + /// The byte cap of this row's download, or `None` when the row is passed without bytes. + fn allowance(&self, row: &DrawingRow) -> Option { + let size = usize::try_from(row.byte_size).ok()?; + let left = VERIFY_MAX_DRAWINGS_TOTAL_BYTES.saturating_sub(self.total); + let fits = size <= VERIFY_MAX_DRAWING_BYTES && size <= left; + (is_fetchable(row) && fits && self.count < VERIFY_MAX_DRAWINGS) + .then(|| VERIFY_MAX_DRAWING_BYTES.min(left)) + } + + fn spend(&mut self, bytes: usize) { + self.count += 1; + self.total += bytes; + } +} + +/// An uploaded row in the drawings bucket whose key is its own sha256: anything else is +/// never fetched (a corrupted row cannot point the worker at another object). +fn is_fetchable(row: &DrawingRow) -> bool { + row.uploaded + && row.bucket == DRAWING_BUCKET + && keys::is_drawing_key(&row.object_key) + && row.object_key.strip_suffix(".svg") == Some(row.sha256.as_str()) +} + +fn download( + backend: &dyn VerifyBackend, + watchdog: &Watchdog, + job: &VerifyJob, + row: &DrawingRow, + max_bytes: usize, +) -> Result>, BackendError> { + let _stage = watchdog.enter("download", deadlines::DOWNLOAD); + match backend.download_drawing(&row.bucket, &row.object_key, max_bytes) { + Ok(bytes) => Ok(Some(bytes)), + Err(BackendError::Status(404)) => { + log::warn("verify_drawing_missing") + .review(&job.review_id) + .asset(&row.asset_id) + .emit(); + Ok(None) + } + Err(BackendError::Overflow) => { + log::warn("verify_drawing_overflow") + .review(&job.review_id) + .asset(&row.asset_id) + .emit(); + Ok(None) + } + Err(error) => Err(error), + } +} + +/// The core's input: the round as the input RPC names it, the texts as they are. +fn verdict_input<'a>(input: &'a VerifyInput, drawings: &'a [Option>]) -> VerdictInput<'a> { + VerdictInput { + binding: VerdictBinding { + sheet_id: input.sheet_id.clone(), + version_id: input.version_id.clone(), + revision: input.revision, + review_id: input.review_id.clone(), + }, + document: &input.document, + base_document: input.base_document.as_deref(), + drawings: input + .drawings + .iter() + .zip(drawings) + .map(|(row, bytes)| VerdictDrawing { + asset_id: &row.asset_id, + sha256: &row.sha256, + byte_size: row.byte_size, + bytes: bytes.as_deref(), + }) + .collect(), + } +} + +/// What the child's end means for the round. +enum ChildVerdict { + Verdict(Vec), + Report(&'static str), + Stop, +} + +/// Maps every way the child can end to a verdict on the round (§6.5 rows). +fn classify_child(run: ChildRun, job: &VerifyJob) -> ChildVerdict { + let stderr = run.stderr; + let review = job.review_id.as_str(); + match run.outcome { + ChildOutcome::Produced(verdict) => ChildVerdict::Verdict(verdict), + ChildOutcome::Timeout => { + log::warn("child_timeout").review(review).emit(); + ChildVerdict::Report(codes::TIMEOUT) + } + ChildOutcome::Signalled { signal } => { + log::warn("child_signalled") + .review(review) + .signal(signal) + .child_stderr(&stderr) + .emit(); + ChildVerdict::Report(codes::RESOURCE_LIMIT) + } + ChildOutcome::OutputOverflow => { + log::warn("child_output_overflow").review(review).emit(); + ChildVerdict::Report(codes::RESOURCE_LIMIT) + } + ChildOutcome::Rejected { name, .. } => { + log::warn("child_rejected") + .review(review) + .code(name) + .child_stderr(&stderr) + .emit(); + ChildVerdict::Report(codes::RESOURCE_LIMIT) + } + ChildOutcome::Broken { name, .. } => { + log::error("child_broken") + .review(review) + .code(name) + .child_stderr(&stderr) + .emit(); + ChildVerdict::Stop + } + ChildOutcome::Failed(detail) => { + log::error("child_failed") + .review(review) + .detail(&detail) + .emit(); + ChildVerdict::Stop + } + } +} + +fn complete( + backend: &dyn VerifyBackend, + watchdog: &Watchdog, + job: &VerifyJob, + verdict: &[u8], +) -> VerifyEnd { + let answer = { + let _stage = watchdog.enter("commit", deadlines::COMMIT); + backend.complete_verification(&job.review_id, &job.lease_token, verdict) + }; + match answer { + Ok(CompleteAnswer::Passed) => VerifyEnd::Passed, + Ok(CompleteAnswer::Failed) => VerifyEnd::Failed, + Ok(CompleteAnswer::Stale) => { + log::info("complete_stale").review(&job.review_id).emit(); + VerifyEnd::Stale + } + Ok(CompleteAnswer::Malformed) => { + log::error("complete_malformed") + .review(&job.review_id) + .emit(); + report(backend, job, codes::VERDICT_REFUSED) + } + Err(error) => transport_failure(backend, job, "commit", &error), + } +} + +/// Records a worker failure with a closed code; `stale` means the round moved on. +fn report(backend: &dyn VerifyBackend, job: &VerifyJob, code: &'static str) -> VerifyEnd { + match backend.fail_verification(&job.review_id, &job.lease_token, code) { + Ok(FailAnswer::Retry | FailAnswer::Unavailable) => VerifyEnd::Reported(code), + Ok(FailAnswer::Stale) => { + log::info("fail_stale").review(&job.review_id).emit(); + VerifyEnd::Stale + } + Err(error) => { + log_backend_failure("fail", job, &error); + VerifyEnd::LeftForLease + } + } +} + +/// A transport failure (§6.5 transient row): reported `unavailable`. +fn transport_failure( + backend: &dyn VerifyBackend, + job: &VerifyJob, + stage: &str, + error: &BackendError, +) -> VerifyEnd { + log_backend_failure(stage, job, error); + report(backend, job, codes::UNAVAILABLE) +} + +fn log_backend_failure(stage: &str, job: &VerifyJob, error: &BackendError) { + let line = log::warn("backend_failure") + .review(&job.review_id) + .stage(stage) + .code(error.kind()); + // The library's error text stays out of the logs: the closed kind and the status are + // the record (as in `job`). + match error { + BackendError::Status(status) => line.http_status(*status).emit(), + _ => line.emit(), + } +} diff --git a/crates/worker/src/watchdog.rs b/crates/worker/src/watchdog.rs index 009a6e26..2c964f8f 100644 --- a/crates/worker/src/watchdog.rs +++ b/crates/worker/src/watchdog.rs @@ -13,6 +13,8 @@ pub mod deadlines { pub const CLAIM: Duration = Duration::from_secs(35); pub const DOWNLOAD: Duration = Duration::from_secs(35); pub const DECODE: Duration = Duration::from_secs(25); + /// Above the verify child's own deadline (`limits::VERIFY_DEADLINE`, 90 s). + pub const VERIFY: Duration = Duration::from_secs(95); pub const UPLOAD: Duration = Duration::from_secs(35); pub const COMMIT: Duration = Duration::from_secs(35); pub const SWEEP: Duration = Duration::from_secs(120); diff --git a/crates/worker/tests/sandbox.rs b/crates/worker/tests/sandbox.rs index 6fa19fc0..00353822 100644 --- a/crates/worker/tests/sandbox.rs +++ b/crates/worker/tests/sandbox.rs @@ -210,3 +210,103 @@ fn the_cpu_limit_stops_a_spinning_child_before_the_deadline() { fn the_linked_libwebp_is_recent_enough() { assert!(faunosteo_worker::decode::libwebp_version() >= limits::MIN_LIBWEBP_VERSION); } + +// V1-2.3: the `verify` child under the same sandbox (contract W2, W3). + +fn verifier(flags: &[&str]) -> Spawner { + let mut spawner = Spawner::for_verify(env!("CARGO_BIN_EXE_faunosteo-worker")); + spawner.extra_args = flags.iter().map(|flag| (*flag).to_owned()).collect(); + spawner +} + +fn expect_verdict(frame: &[u8]) -> serde_json::Value { + let run = verifier(&[]).run(frame); + match run.outcome { + ChildOutcome::Produced(out) => { + assert!(out.len() < limits::VERIFY_MAX_OUTPUT_BYTES); + serde_json::from_slice(&out).unwrap() + } + other => panic!("{other:?} (stderr: {})", run.stderr), + } +} + +#[test] +fn a_clean_sheet_is_verified_passed_through_the_full_sandbox() { + let verdict = expect_verdict(&fixtures::sheet_frame(&fixtures::sheet_document())); + assert_eq!(verdict["outcome"], "passed", "{verdict}"); + assert_eq!(verdict["review_id"], fixtures::sheet_binding().review_id); +} + +#[test] +fn a_sheet_at_every_cap_is_verified_within_the_limits() { + let frame = fixtures::sheet_frame(&fixtures::sheet_document_at_caps()); + let started = std::time::Instant::now(); + let verdict = expect_verdict(&frame); + let elapsed = started.elapsed(); + assert_eq!(verdict["outcome"], "passed", "{}", verdict["failures"]); + assert!(elapsed < limits::VERIFY_DEADLINE, "{elapsed:?}"); + eprintln!( + "verify at caps: input {} bytes, verdict {} bytes, {elapsed:?}", + frame.len(), + verdict.to_string().len() + ); +} + +#[test] +fn the_verify_child_refuses_a_non_empty_environment() { + let status = Command::new(env!("CARGO_BIN_EXE_faunosteo-worker")) + .arg("verify") + .env_clear() + .env("LEAKED", "1") + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + .unwrap(); + assert_eq!(status.code(), Some(exit_codes::ENV_NOT_EMPTY)); +} + +#[test] +fn an_input_over_the_verify_cap_is_refused_unread() { + let run = verifier(&[]).run(&vec![0u8; limits::VERIFY_MAX_INPUT_BYTES + 1]); + assert!( + matches!(run.outcome, ChildOutcome::Rejected { code, .. } if code == exit_codes::INPUT_TOO_LARGE), + "{:?} {}", + run.outcome, + run.stderr + ); +} + +#[test] +fn a_malformed_frame_is_a_broken_parent_not_content() { + let run = verifier(&[]).run(b"\x03\0\0\0{x}"); + assert!( + matches!(run.outcome, ChildOutcome::Broken { code, .. } if code == exit_codes::FRAME_INVALID), + "{:?} {}", + run.outcome, + run.stderr + ); +} + +#[test] +fn a_denied_syscall_kills_the_verify_child_with_sigsys() { + let run = verifier(&["--violate"]).run(b""); + assert!( + matches!(run.outcome, ChildOutcome::Signalled { signal: SIGSYS }), + "{:?} {}", + run.outcome, + run.stderr + ); +} + +#[test] +fn a_verify_output_over_its_cap_is_cut() { + let mut spawner = verifier(&["--flood"]); + spawner.max_output = 64 * 1024; + let run = spawner.run(b""); + assert!( + matches!(run.outcome, ChildOutcome::OutputOverflow), + "{:?}", + run.outcome + ); +} diff --git a/crates/worker/tests/verify_loop.rs b/crates/worker/tests/verify_loop.rs new file mode 100644 index 00000000..06c5f792 --- /dev/null +++ b/crates/worker/tests/verify_loop.rs @@ -0,0 +1,527 @@ +//! V1-2.3 proofs of the verification job (contract W4, V1-2 contract §6.5) on an in-memory +//! `VerifyBackend`: the core's verdict computed in process from the very frame the parent +//! builds (or the real child without seccomp under `test-hooks`), every child outcome mapped +//! to its closed worker code, a missing drawing judged by the core, never by the worker. +#![allow(clippy::unwrap_used, clippy::expect_used)] + +use std::cell::RefCell; +use std::collections::VecDeque; + +use faunosteo_core::domain::sheet::verdict::{codes as verdict_codes, sha256_hex}; +use faunosteo_worker::backend::{ + BackendError, BackendResult, CompleteAnswer, DrawingRow, FailAnswer, VerifyBackend, + VerifyInput, VerifyJob, +}; +use faunosteo_worker::config::Config; +use faunosteo_worker::limits::VERIFY_MAX_DRAWING_BYTES; +use faunosteo_worker::run::{Mode, parse_args, verify_loop}; +use faunosteo_worker::spawn::{ChildOutcome, ChildRun}; +use faunosteo_worker::verifier::Verifier; +use faunosteo_worker::verify_job::{DRAWING_BUCKET, VerifyEnd, codes, process_verification}; +use faunosteo_worker::watchdog::Watchdog; +use faunosteo_worker::{exit_codes, verify}; +use serde_json::{Value, json}; + +const REVIEW: &str = "aaaaaaaa-0000-4000-8000-000000000003"; +const VERSION: &str = "aaaaaaaa-0000-4000-8000-000000000002"; +const SHEET: &str = "aaaaaaaa-0000-4000-8000-000000000001"; +const TOKEN: &str = "30000000-0000-0000-0000-00000000000c"; +const ASSET: &str = "11111111-2222-4333-8444-555555555555"; +const DRAWING: &str = concat!( + "\n", + r#""# +); + +fn name(text: &str) -> Value { + json!({ "fr": text, "en": text }) +} + +/// The core's small valid sheet (`crates/core/tests/sheet_verdict.rs::document`). +fn document() -> Value { + json!({ + "schema_version": 2, "species_id": "canine", "name": name("Chien"), + "description": {}, "scope_disclaimer": { "fr": "Outil", "en": "Tool" }, + "attribution": { "authors": ["A"], "license": null, "notes": null }, + "vertebral_config": { "regions": [] }, "references": [], + "categories": [{ "id": "c", "name": name("C"), "color": null }], + "views": [{ "id": "main", "kind": "main", "name": name("Vue"), "svg_asset_id": ASSET, "view_box": null }], + "annotations": [{ "id": "a1", "kind": "point", "view_id": "main", "category_id": "c", + "laterality": "unpaired", "geometry": { "type": "circle", "cx": 1, "cy": 1, "r": 1 }, + "review_status": "to_confirm", "name": name("P"), "finding_zone_id": "z1", "vertebral_region_code": null }], + "finding_zones": [{ "id": "z1", "name": name("Z"), "categories": [{ "id": "cat", "kind": "dysfunction", + "name": name("D"), "review_status": "to_confirm", + "children": [{ "id": "n1", "name": name("N"), "review_status": "to_confirm" }] }] }] + }) +} + +fn job() -> VerifyJob { + VerifyJob { + review_id: REVIEW.to_owned(), + version_id: VERSION.to_owned(), + sheet_id: SHEET.to_owned(), + revision: 3, + lease_token: TOKEN.to_owned(), + attempts: 1, + } +} + +fn row() -> DrawingRow { + let sha256 = sha256_hex(DRAWING.as_bytes()); + DrawingRow { + asset_id: ASSET.to_owned(), + object_key: format!("{sha256}.svg"), + sha256, + byte_size: DRAWING.len() as u64, + bucket: DRAWING_BUCKET.to_owned(), + uploaded: true, + } +} + +fn input(document: String, drawings: Vec) -> VerifyInput { + VerifyInput { + review_id: REVIEW.to_owned(), + version_id: VERSION.to_owned(), + sheet_id: SHEET.to_owned(), + revision: 3, + document, + base_document: None, + drawings, + } +} + +struct Fake { + calls: RefCell>, + input: RefCell>>>, + downloads: RefCell>>>, + complete: RefCell>>, + fail: RefCell>>, + verdict: RefCell>, + jobs: RefCell>, +} + +impl Fake { + fn with_input(input: VerifyInput) -> Self { + Self { + calls: RefCell::default(), + input: RefCell::new(Some(Ok(Some(input)))), + downloads: RefCell::default(), + complete: RefCell::default(), + fail: RefCell::default(), + verdict: RefCell::default(), + jobs: RefCell::default(), + } + } + + fn clean() -> Self { + Self::with_input(input(document().to_string(), vec![row()])) + } + + fn calls(&self) -> Vec { + self.calls.borrow().clone() + } + + fn names(&self) -> Vec { + self.calls() + .iter() + .map(|c| c.split(' ').next().unwrap().to_owned()) + .collect() + } + + fn verdict(&self) -> Value { + self.verdict.borrow().clone().expect("a completion") + } +} + +impl VerifyBackend for Fake { + fn claim_verifications(&self, max_jobs: u32) -> BackendResult> { + self.calls.borrow_mut().push(format!("claim {max_jobs}")); + Ok(self.jobs.borrow_mut().drain(..).collect()) + } + + fn verification_input( + &self, + review_id: &str, + lease_token: &str, + ) -> BackendResult> { + self.calls + .borrow_mut() + .push(format!("input {review_id} {lease_token}")); + self.input.borrow_mut().take().unwrap_or(Ok(None)) + } + + fn download_drawing( + &self, + bucket: &str, + key: &str, + max_bytes: usize, + ) -> BackendResult> { + self.calls + .borrow_mut() + .push(format!("download {bucket}/{key} {max_bytes}")); + self.downloads + .borrow_mut() + .pop_front() + .unwrap_or_else(|| Ok(DRAWING.as_bytes().to_vec())) + } + + fn complete_verification( + &self, + review_id: &str, + lease_token: &str, + verdict: &[u8], + ) -> BackendResult { + self.calls + .borrow_mut() + .push(format!("complete {review_id} {lease_token}")); + let value: Value = serde_json::from_slice(verdict).unwrap(); + let outcome = value["outcome"].clone(); + *self.verdict.borrow_mut() = Some(value); + self.complete + .borrow_mut() + .take() + .unwrap_or(Ok(if outcome == "passed" { + CompleteAnswer::Passed + } else { + CompleteAnswer::Failed + })) + } + + fn fail_verification( + &self, + review_id: &str, + lease_token: &str, + code: &str, + ) -> BackendResult { + self.calls + .borrow_mut() + .push(format!("fail {review_id} {lease_token} {code}")); + self.fail + .borrow_mut() + .take() + .unwrap_or(Ok(FailAnswer::Retry)) + } +} + +/// The child's work in process: the same frame, `verify::process`, the core's verdict. +struct InProcess; + +impl Verifier for InProcess { + fn verify(&self, frame: &[u8]) -> ChildRun { + let outcome = match verify::process(frame) { + Ok(verdict) => ChildOutcome::Produced(verdict), + Err(failure) => ChildOutcome::Broken { + code: failure.exit_code(), + name: exit_codes::name(failure.exit_code()), + }, + }; + ChildRun { + outcome, + stderr: String::new(), + } + } +} + +/// A child that ends as told, at most once. +struct Fixed(RefCell>); + +impl Fixed { + fn with(outcome: ChildOutcome) -> Self { + Self(RefCell::new(Some(outcome))) + } + + fn never() -> Self { + Self(RefCell::new(None)) + } +} + +impl Verifier for Fixed { + fn verify(&self, _frame: &[u8]) -> ChildRun { + ChildRun { + outcome: self.0.borrow_mut().take().expect("no child run expected"), + stderr: "child says\n\u{1b}[0m".to_owned(), + } + } +} + +fn process(fake: &Fake, verifier: &dyn Verifier) -> VerifyEnd { + process_verification(fake, verifier, &Watchdog::new(), &job()) +} + +fn fail_call(code: &str) -> String { + format!("fail {REVIEW} {TOKEN} {code}") +} + +#[test] +fn a_clean_sheet_follows_input_download_child_complete() { + let fake = Fake::clean(); + assert_eq!(process(&fake, &InProcess), VerifyEnd::Passed); + let key = row().object_key; + assert_eq!( + fake.calls(), + vec![ + format!("input {REVIEW} {TOKEN}"), + format!("download {DRAWING_BUCKET}/{key} {VERIFY_MAX_DRAWING_BYTES}"), + format!("complete {REVIEW} {TOKEN}"), + ] + ); + let verdict = fake.verdict(); + assert_eq!(verdict["outcome"], "passed"); + assert_eq!( + ( + &verdict["review_id"], + &verdict["version_id"], + &verdict["sheet_id"], + &verdict["revision"] + ), + (&json!(REVIEW), &json!(VERSION), &json!(SHEET), &json!(3)) + ); +} + +#[test] +fn an_unreadable_document_completes_failed() { + let fake = Fake::with_input(input("not json".to_owned(), vec![row()])); + assert_eq!(process(&fake, &InProcess), VerifyEnd::Failed); + assert_eq!(fake.names(), ["input", "download", "complete"]); + assert_eq!(fake.verdict()["failure_code"], verdict_codes::DOCUMENT); +} + +#[test] +fn a_stale_input_ends_without_any_other_call() { + let fake = Fake::clean(); + *fake.input.borrow_mut() = Some(Ok(None)); + assert_eq!(process(&fake, &Fixed::never()), VerifyEnd::Stale); + assert_eq!(fake.names(), ["input"]); +} + +#[test] +fn a_stale_completion_ends_and_a_malformed_one_is_reported() { + let fake = Fake::clean(); + *fake.complete.borrow_mut() = Some(Ok(CompleteAnswer::Stale)); + assert_eq!(process(&fake, &InProcess), VerifyEnd::Stale); + assert_eq!(fake.names(), ["input", "download", "complete"]); + + let fake = Fake::clean(); + *fake.complete.borrow_mut() = Some(Ok(CompleteAnswer::Malformed)); + assert_eq!( + process(&fake, &InProcess), + VerifyEnd::Reported(codes::VERDICT_REFUSED) + ); + assert_eq!( + fake.calls().last().unwrap(), + &fail_call(codes::VERDICT_REFUSED) + ); +} + +#[test] +fn a_missing_or_oversized_drawing_is_judged_by_the_core() { + for error in [BackendError::Status(404), BackendError::Overflow] { + let fake = Fake::clean(); + fake.downloads.borrow_mut().push_back(Err(error)); + assert_eq!(process(&fake, &InProcess), VerifyEnd::Failed); + assert_eq!(fake.names(), ["input", "download", "complete"]); + let verdict = fake.verdict(); + assert_eq!(verdict["failure_code"], verdict_codes::DRAWING_MISSING); + assert_eq!(verdict["drawings"][0]["result"], "missing"); + } +} + +#[test] +fn rows_outside_the_policy_are_never_fetched() { + let mut not_uploaded = row(); + not_uploaded.uploaded = false; + let mut other_bucket = row(); + other_bucket.bucket = "avatars".to_owned(); + let mut other_key = row(); + other_key.object_key = format!("{}.svg", "0".repeat(64)); + let mut hostile_key = row(); + hostile_key.object_key = "../avatars/x.svg".to_owned(); + let mut too_big = row(); + too_big.byte_size = VERIFY_MAX_DRAWING_BYTES as u64 + 1; + for skipped in [not_uploaded, other_bucket, other_key, hostile_key, too_big] { + let fake = Fake::with_input(input(document().to_string(), vec![skipped])); + assert_eq!(process(&fake, &InProcess), VerifyEnd::Failed); + assert_eq!(fake.names(), ["input", "complete"]); + } +} + +#[test] +fn downloads_stop_at_the_drawing_count() { + let rows = vec![row(); 6]; + let fake = Fake::with_input(input(document().to_string(), rows)); + process(&fake, &InProcess); + let downloads = fake.names().iter().filter(|n| *n == "download").count(); + assert_eq!(downloads, 4); +} + +#[test] +fn a_transport_failure_is_reported_unavailable_without_a_child() { + let fake = Fake::clean(); + fake.downloads + .borrow_mut() + .push_back(Err(BackendError::Network("refused".to_owned()))); + assert_eq!( + process(&fake, &Fixed::never()), + VerifyEnd::Reported(codes::UNAVAILABLE) + ); + assert_eq!(fake.names(), ["input", "download", "fail"]); + assert_eq!(fake.calls()[2], fail_call(codes::UNAVAILABLE)); + + let fake = Fake::clean(); + *fake.input.borrow_mut() = Some(Err(BackendError::Status(503))); + assert_eq!( + process(&fake, &Fixed::never()), + VerifyEnd::Reported(codes::UNAVAILABLE) + ); + assert_eq!(fake.names(), ["input", "fail"]); +} + +#[test] +fn an_unrecordable_failure_leaves_the_lease() { + let fake = Fake::clean(); + *fake.input.borrow_mut() = Some(Err(BackendError::Status(503))); + *fake.fail.borrow_mut() = Some(Err(BackendError::Status(503))); + assert_eq!(process(&fake, &Fixed::never()), VerifyEnd::LeftForLease); + + let fake = Fake::clean(); + *fake.fail.borrow_mut() = Some(Ok(FailAnswer::Stale)); + assert_eq!( + process(&fake, &Fixed::with(ChildOutcome::Timeout)), + VerifyEnd::Stale + ); +} + +#[test] +fn every_child_limit_maps_to_its_closed_code() { + for (outcome, code) in [ + (ChildOutcome::Timeout, codes::TIMEOUT), + (ChildOutcome::OutputOverflow, codes::RESOURCE_LIMIT), + ( + ChildOutcome::Signalled { signal: 24 }, + codes::RESOURCE_LIMIT, + ), + (ChildOutcome::Signalled { signal: 9 }, codes::RESOURCE_LIMIT), + (ChildOutcome::Signalled { signal: 6 }, codes::RESOURCE_LIMIT), + ( + ChildOutcome::Signalled { signal: 31 }, + codes::RESOURCE_LIMIT, + ), + ( + ChildOutcome::Rejected { + code: exit_codes::INPUT_TOO_LARGE, + name: "input_too_large", + }, + codes::RESOURCE_LIMIT, + ), + ] { + let fake = Fake::clean(); + assert_eq!( + process(&fake, &Fixed::with(outcome)), + VerifyEnd::Reported(code) + ); + assert_eq!(fake.names(), ["input", "download", "fail"]); + assert_eq!(fake.calls()[2], fail_call(code)); + } +} + +#[test] +fn a_broken_or_failed_child_is_reported_then_stops_the_worker() { + for outcome in [ + ChildOutcome::Broken { + code: exit_codes::SANDBOX_FAILED, + name: "sandbox_failed", + }, + ChildOutcome::Failed("output refused: not a JSON object".to_owned()), + ] { + let fake = Fake::clean(); + assert_eq!(process(&fake, &Fixed::with(outcome)), VerifyEnd::Stop); + assert_eq!(fake.calls().last().unwrap(), &fail_call(codes::UNAVAILABLE)); + } +} + +#[test] +fn a_round_with_a_hostile_identity_is_left_untouched() { + let fake = Fake::clean(); + let mut bad = job(); + bad.lease_token = "x".to_owned(); + assert_eq!( + process_verification(&fake, &Fixed::never(), &Watchdog::new(), &bad), + VerifyEnd::LeftForLease + ); + assert!(fake.calls().is_empty()); +} + +fn once_config() -> Config { + Config::from_vars(|name| match name { + "SUPABASE_URL" => Some("http://kong:8000".to_owned()), + "SUPABASE_SERVICE_ROLE_KEY" => Some("k".to_owned()), + "WORKER_ONCE" => Some("1".to_owned()), + _ => None, + }) + .unwrap() +} + +#[test] +fn once_mode_claims_one_round_without_a_sweep_or_stops_on_a_broken_child() { + let fake = Fake::clean(); + *fake.jobs.borrow_mut() = vec![job()]; + assert_eq!( + verify_loop(&once_config(), &fake, &InProcess, &Watchdog::new()), + exit_codes::OK + ); + assert_eq!(fake.names(), ["claim", "input", "download", "complete"]); + + let fake = Fake::clean(); + *fake.jobs.borrow_mut() = vec![job()]; + let broken = Fixed::with(ChildOutcome::Broken { + code: exit_codes::ENV_NOT_EMPTY, + name: "env_not_empty", + }); + assert_eq!( + verify_loop(&once_config(), &fake, &broken, &Watchdog::new()), + exit_codes::CHILD_BROKEN + ); +} + +#[test] +fn run_arguments_select_the_mode() { + let args = |list: &[&str]| -> Vec { list.iter().map(|a| (*a).to_owned()).collect() }; + assert_eq!(parse_args(&args(&[])), Ok((Mode::Avatars, false))); + assert_eq!(parse_args(&args(&["--check"])), Ok((Mode::Avatars, true))); + assert_eq!( + parse_args(&args(&["--jobs", "sheet-verify"])), + Ok((Mode::SheetVerify, false)) + ); + assert_eq!( + parse_args(&args(&["--jobs", "sheet-verify", "--check"])), + Ok((Mode::SheetVerify, true)) + ); + assert_eq!( + parse_args(&args(&["--check", "--jobs", "sheet-verify"])), + Ok((Mode::SheetVerify, true)) + ); + assert!(parse_args(&args(&["--jobs", "avatars"])).is_err()); + assert!(parse_args(&args(&["--jobs"])).is_err()); + assert!(parse_args(&args(&["--check", "--check"])).is_err()); + assert!(parse_args(&args(&["--once"])).is_err()); +} + +/// The real child (this crate's binary), rlimits on, seccomp off: the parent's frame, the +/// child's reading and the spawner's classification end to end on any host (the filter +/// itself is proven in the Alpine stage, tests/sandbox.rs). +#[cfg(feature = "test-hooks")] +#[test] +fn the_real_child_without_seccomp_verifies_the_parents_frame() { + use faunosteo_worker::spawn::Spawner; + + struct RealChild(Spawner); + impl Verifier for RealChild { + fn verify(&self, frame: &[u8]) -> ChildRun { + self.0.run(frame) + } + } + let mut spawner = Spawner::for_verify(env!("CARGO_BIN_EXE_faunosteo-worker")); + spawner.extra_args = vec!["--no-seccomp".to_owned()]; + let fake = Fake::clean(); + assert_eq!(process(&fake, &RealChild(spawner)), VerifyEnd::Passed); + assert_eq!(fake.verdict()["outcome"], "passed"); +} diff --git a/docker-compose.yml b/docker-compose.yml index 981cc197..251846a6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -40,6 +40,28 @@ services: - supabase restart: unless-stopped + # Sheet verification worker (V1-2.3, V1-2 contract §6.4): the same image and hardening as + # `worker`, the verification loop only (no avatar, no sweep). Its `verify` child is capped + # at 1 GiB of address space (crates/worker/src/limits.rs). Started only with the `worker` + # profile: `docker compose --profile worker up -d worker-verify`; never on DEV. + worker-verify: + container_name: faunosteo_worker_verify + profiles: ["worker"] + image: faunosteo_worker:local + env_file: + - .env.worker + command: ["run", "--jobs", "sheet-verify"] + read_only: true + cap_drop: [ALL] + security_opt: + - no-new-privileges:true + mem_limit: 512m + cpus: 1 + pids_limit: 64 + networks: + - supabase + restart: unless-stopped + # DEV gateway (docs/development/ENVIRONMENTS.md §3): Mailpit behind basic auth on # :54325, the only LAN entry to the DEV mails (the CLI's raw Mailpit port # 54324 is closed to the LAN by the host firewall rule). Started by `pnpm dev:gateway`