feat(worker): V1-2.3 sandboxed verify child and sheet-verify job mode
This commit is contained in:
1 parent
345dde59e6
commit
31fbb0317f
24 files changed
+2145
-87
No files matched your search
Generated
+1
@@ -391,6 +391,7 @@ name = "faunosteo-worker"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"crc32fast",
|
||||
"faunosteo-core",
|
||||
"flate2",
|
||||
"image",
|
||||
"libc",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "faunosteo-worker"
|
||||
description = "Faunosteo file-processing worker: keyed job loop and sweeper (R1), sandboxed decode child (R0)"
|
||||
description = "Faunosteo file-processing worker: keyed job loop and sweeper (R1), sandboxed decode child (R0), sheet verification (V1-2.3)"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
@@ -21,6 +21,9 @@ default = []
|
||||
test-hooks = ["dep:crc32fast", "dep:flate2"]
|
||||
|
||||
[dependencies]
|
||||
# The sheet verdict (V1-2.3), run in the keyless `verify` child; default features only (no PDF,
|
||||
# no curation).
|
||||
faunosteo-core = { path = "../core" }
|
||||
image.workspace = true
|
||||
libwebp-sys.workspace = true
|
||||
seccompiler.workspace = true
|
||||
|
||||
@@ -1,30 +1,38 @@
|
||||
#!/bin/sh
|
||||
# Derives the seccomp allowlist of the decode child from a real trace and checks it against
|
||||
# `sandbox::ALLOWED_SYSCALLS` (review Part IV.1: "derived by strace over the fixture corpus
|
||||
# at the caps on the production image, not hand-written"). Runs in the Alpine `test` stage
|
||||
# of crates/worker/Dockerfile, where the musl binary that ships is built.
|
||||
# Derives the seccomp allowlist of the decode and verify children from a real trace and
|
||||
# checks it against `sandbox::ALLOWED_SYSCALLS` (review Part IV.1: "derived by strace over
|
||||
# the fixture corpus at the caps on the production image, not hand-written"). Runs in the
|
||||
# Alpine `test` stage of crates/worker/Dockerfile, where the musl binary that ships is built.
|
||||
# The verify child (V1-2.3) shares the filter: its framed fixtures (a small sheet and the
|
||||
# sheet at every cap) are traced the same way.
|
||||
#
|
||||
# Exit 1 when a traced syscall is missing from the allowlist (the sandbox would kill a
|
||||
# legitimate decode). Syscalls the list allows but the trace never shows are printed for
|
||||
# review (the abort path is expected there).
|
||||
# legitimate decode or verification). Syscalls the list allows but the trace never shows are
|
||||
# printed for review (the abort path is expected there).
|
||||
set -eu
|
||||
|
||||
bin=target/release/faunosteo-worker
|
||||
cargo build -p faunosteo-worker --release --features test-hooks --locked
|
||||
work=$(mktemp -d)
|
||||
|
||||
fixtures="png jpeg webp big-jpeg big-png"
|
||||
: > "$work/traced.raw"
|
||||
for kind in $fixtures; do
|
||||
"$bin" fixture "$kind" > "$work/$kind.bin"
|
||||
# trace <command> <fixture>: one child run, the syscalls after the marker appended.
|
||||
trace() {
|
||||
"$bin" fixture "$2" > "$work/$2.bin"
|
||||
# `env -i`: the child refuses a non-empty environment; `--no-seccomp` keeps the rlimits
|
||||
# and makes one sched_yield where the filter would be installed (strace cannot read the
|
||||
# child's memory once PR_SET_DUMPABLE is 0, so the marker is a syscall, not a string).
|
||||
env -i strace -f -qq -o "$work/trace.$kind" "$bin" decode --no-seccomp \
|
||||
< "$work/$kind.bin" > "$work/$kind.webp"
|
||||
awk 'found { print } /sched_yield\(/ { found = 1 }' "$work/trace.$kind" \
|
||||
env -i strace -f -qq -o "$work/trace.$2" "$bin" "$1" --no-seccomp \
|
||||
< "$work/$2.bin" > "$work/$2.out"
|
||||
awk 'found { print } /sched_yield\(/ { found = 1 }' "$work/trace.$2" \
|
||||
| sed -E 's/^[0-9]+ +//; /^(\+\+\+|---)/d; s/^<\.\.\. ([a-z0-9_]+) resumed>.*/\1/; s/\(.*//' \
|
||||
| grep -E '^[a-z0-9_]+$' >> "$work/traced.raw" || true
|
||||
}
|
||||
for kind in png jpeg webp big-jpeg big-png; do
|
||||
trace decode "$kind"
|
||||
done
|
||||
for kind in verify-clean verify-caps; do
|
||||
trace verify "$kind"
|
||||
done
|
||||
sort -u "$work/traced.raw" > "$work/traced.txt"
|
||||
"$bin" print-allowlist | sort -u > "$work/allowed.txt"
|
||||
@@ -41,7 +49,7 @@ missing=$(comm -23 "$work/traced.txt" "$work/allowed.txt")
|
||||
extra=$(comm -13 "$work/traced.txt" "$work/allowed.txt")
|
||||
echo "== allowed but not traced (abort path expected): $(echo "$extra" | tr '\n' ' ')"
|
||||
if [ -n "$missing" ]; then
|
||||
echo "== TRACED BUT NOT ALLOWED (the filter would kill a legitimate decode):"
|
||||
echo "== TRACED BUT NOT ALLOWED (the filter would kill a legitimate decode or verification):"
|
||||
echo "$missing"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
//! What the parent needs from Supabase (STORAGE.md §2.2 RPCs and §3.3 Storage calls),
|
||||
//! behind one trait so the job and sweep logic is tested on an in-memory fake and the HTTP
|
||||
//! implementation stays thin.
|
||||
//! What the parent needs from Supabase (STORAGE.md §2.2 RPCs and §3.3 Storage calls), behind
|
||||
//! one trait so the job and sweep logic is tested on an in-memory fake and the HTTP
|
||||
//! implementation stays thin. The M6 verification RPCs (V1-2.3) have a trait of their own.
|
||||
|
||||
use std::fmt;
|
||||
|
||||
@@ -105,3 +105,86 @@ pub trait Backend {
|
||||
) -> BackendResult<Vec<Leftover>>;
|
||||
fn expire_pending_uploads(&self, max_age_seconds: u64) -> BackendResult<i64>;
|
||||
}
|
||||
|
||||
/// One claimed round (`claim_sheet_verifications`, M6).
|
||||
#[derive(Debug, Clone, Deserialize, PartialEq, Eq)]
|
||||
pub struct VerifyJob {
|
||||
pub review_id: String,
|
||||
pub version_id: String,
|
||||
pub sheet_id: String,
|
||||
pub revision: u32,
|
||||
pub lease_token: String,
|
||||
pub attempts: i64,
|
||||
}
|
||||
|
||||
/// One drawing row of a round's input: registered for the sheet, referenced by the document.
|
||||
#[derive(Debug, Clone, Deserialize, PartialEq, Eq)]
|
||||
pub struct DrawingRow {
|
||||
pub asset_id: String,
|
||||
pub sha256: String,
|
||||
pub byte_size: u64,
|
||||
pub bucket: String,
|
||||
pub object_key: String,
|
||||
pub uploaded: bool,
|
||||
}
|
||||
|
||||
/// The input of a leased round (`get_sheet_verification_input`): the texts are passed to
|
||||
/// the child as they are, never parsed by the parent.
|
||||
#[derive(Debug, Clone, Deserialize, PartialEq, Eq)]
|
||||
pub struct VerifyInput {
|
||||
pub review_id: String,
|
||||
pub version_id: String,
|
||||
pub sheet_id: String,
|
||||
pub revision: u32,
|
||||
pub document: String,
|
||||
pub base_document: Option<String>,
|
||||
pub drawings: Vec<DrawingRow>,
|
||||
}
|
||||
|
||||
/// `complete_sheet_verification`: `stale` and `malformed` wrote nothing.
|
||||
#[derive(Debug, Clone, Copy, Deserialize, PartialEq, Eq)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum CompleteAnswer {
|
||||
Passed,
|
||||
Failed,
|
||||
Stale,
|
||||
Malformed,
|
||||
}
|
||||
|
||||
/// `fail_sheet_verification`: claimable again, closed `unavailable` at the attempt limit,
|
||||
/// or `stale` (nothing written).
|
||||
#[derive(Debug, Clone, Copy, Deserialize, PartialEq, Eq)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum FailAnswer {
|
||||
Retry,
|
||||
Unavailable,
|
||||
Stale,
|
||||
}
|
||||
|
||||
/// The four RPCs of M6 and the drawing download (V1-2.3, `run --jobs sheet-verify`).
|
||||
pub trait VerifyBackend {
|
||||
fn claim_verifications(&self, max_jobs: u32) -> BackendResult<Vec<VerifyJob>>;
|
||||
/// `None` when the lease or the round is stale.
|
||||
fn verification_input(
|
||||
&self,
|
||||
review_id: &str,
|
||||
lease_token: &str,
|
||||
) -> BackendResult<Option<VerifyInput>>;
|
||||
/// `Status(404)` when the object is missing; `Overflow` past `max_bytes`.
|
||||
fn download_drawing(&self, bucket: &str, key: &str, max_bytes: usize)
|
||||
-> BackendResult<Vec<u8>>;
|
||||
/// `verdict` is the child's output, a JSON object.
|
||||
fn complete_verification(
|
||||
&self,
|
||||
review_id: &str,
|
||||
lease_token: &str,
|
||||
verdict: &[u8],
|
||||
) -> BackendResult<CompleteAnswer>;
|
||||
/// `code` is one of the closed worker codes (`verify_job::codes`).
|
||||
fn fail_verification(
|
||||
&self,
|
||||
review_id: &str,
|
||||
lease_token: &str,
|
||||
code: &str,
|
||||
) -> BackendResult<FailAnswer>;
|
||||
}
|
||||
@@ -219,7 +219,8 @@ enum HookAction {
|
||||
}
|
||||
|
||||
impl TestHooks {
|
||||
fn parse(args: &[String]) -> Result<Self, String> {
|
||||
/// Shared with the `verify` child, which takes the same flags.
|
||||
pub(crate) fn parse(args: &[String]) -> Result<Self, String> {
|
||||
#[cfg(not(feature = "test-hooks"))]
|
||||
let hooks = Self { seccomp: true };
|
||||
#[cfg(feature = "test-hooks")]
|
||||
@@ -257,7 +258,11 @@ impl TestHooks {
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-hooks")]
|
||||
fn run_after_sandbox(&self, stdout: &mut dyn Write, stderr: &mut dyn Write) -> Option<i32> {
|
||||
pub(crate) fn run_after_sandbox(
|
||||
&self,
|
||||
stdout: &mut dyn Write,
|
||||
stderr: &mut dyn Write,
|
||||
) -> Option<i32> {
|
||||
if !self.seccomp {
|
||||
// Marker for crates/worker/scripts/derive-seccomp.sh: one `sched_yield`, a syscall
|
||||
// the decode path never makes, right where the filter would be installed. The
|
||||
@@ -314,7 +319,11 @@ impl TestHooks {
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "test-hooks"))]
|
||||
fn run_after_sandbox(&self, _stdout: &mut dyn Write, _stderr: &mut dyn Write) -> Option<i32> {
|
||||
pub(crate) fn run_after_sandbox(
|
||||
&self,
|
||||
_stdout: &mut dyn Write,
|
||||
_stderr: &mut dyn Write,
|
||||
) -> Option<i32> {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
//! Exit codes of the `decode` child, reserved away from 1 (generic failure), 101 (Rust
|
||||
//! Exit codes of the `decode` and `verify` children, reserved away from 1 (generic failure), 101 (Rust
|
||||
//! panic) and signal deaths (reported separately by the parent). The parent maps them to
|
||||
//! the closed rejection reasons of STEP8_FILES_ANALYSIS II.7.
|
||||
|
||||
@@ -19,6 +19,8 @@ pub const RESOURCE_LIMIT: i32 = 19;
|
||||
pub const CHILD_BROKEN: i32 = 20;
|
||||
/// The parent's watchdog found a stage past its deadline and exited (R1).
|
||||
pub const WATCHDOG: i32 = 21;
|
||||
/// The `verify` child could not read its framed input: a parent bug, never the content's.
|
||||
pub const FRAME_INVALID: i32 = 22;
|
||||
|
||||
/// Codes that judge the content (the upload is refused). Every other non-zero code means
|
||||
/// the child's environment is broken (`Broken` outcome: the parent alerts rather than
|
||||
@@ -48,6 +50,7 @@ pub fn name(code: i32) -> &'static str {
|
||||
RESOURCE_LIMIT => "resource_limit",
|
||||
CHILD_BROKEN => "child_broken",
|
||||
WATCHDOG => "watchdog",
|
||||
FRAME_INVALID => "frame_invalid",
|
||||
_ => "unknown",
|
||||
}
|
||||
}
|
||||
@@ -1,9 +1,15 @@
|
||||
//! Synthetic images for the sandbox tests and the seccomp derivation (`test-hooks` only):
|
||||
//! nothing binary lives in the repository.
|
||||
//! Synthetic images and framed sheet inputs for the sandbox tests and the seccomp
|
||||
//! derivation (`test-hooks` only): nothing binary lives in the repository.
|
||||
|
||||
use std::io::Cursor;
|
||||
|
||||
use faunosteo_core::domain::sheet::verdict::{
|
||||
VerdictBinding, VerdictDrawing, VerdictInput, sha256_hex,
|
||||
};
|
||||
use image::{DynamicImage, ImageBuffer, ImageFormat, Rgba};
|
||||
use serde_json::{Value, json};
|
||||
|
||||
use crate::verify_frame;
|
||||
|
||||
fn gradient(width: u32, height: u32) -> DynamicImage {
|
||||
let buffer = ImageBuffer::from_fn(width, height, |x, y| {
|
||||
@@ -107,6 +113,110 @@ pub fn truncated_png() -> Vec<u8> {
|
||||
bytes
|
||||
}
|
||||
|
||||
/// The asset id the verification fixtures draw on.
|
||||
pub const SHEET_ASSET: &str = "11111111-2222-4333-8444-555555555555";
|
||||
/// A drawing in its stored form, as the page's cleaning uploads it
|
||||
/// (`crates/core/tests/sheet_verdict.rs`).
|
||||
pub const SHEET_DRAWING: &str = concat!(
|
||||
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n",
|
||||
r#"<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 10 10"><path d="M0 0L5 5"/></svg>"#
|
||||
);
|
||||
|
||||
/// The round of the verification fixtures.
|
||||
pub fn sheet_binding() -> VerdictBinding {
|
||||
VerdictBinding {
|
||||
sheet_id: "aaaaaaaa-0000-4000-8000-000000000001".to_owned(),
|
||||
version_id: "aaaaaaaa-0000-4000-8000-000000000002".to_owned(),
|
||||
revision: 3,
|
||||
review_id: "aaaaaaaa-0000-4000-8000-000000000003".to_owned(),
|
||||
}
|
||||
}
|
||||
|
||||
fn name(text: &str) -> Value {
|
||||
json!({ "fr": text, "en": text })
|
||||
}
|
||||
|
||||
/// A small valid document drawn on `SHEET_ASSET` (the core's `sheet_verdict::document`).
|
||||
pub fn sheet_document() -> Value {
|
||||
json!({
|
||||
"schema_version": 2, "species_id": "canine", "name": name("Chien"),
|
||||
"description": {}, "scope_disclaimer": { "fr": "Outil", "en": "Tool" },
|
||||
"attribution": { "authors": ["A"], "license": null, "notes": null },
|
||||
"vertebral_config": { "regions": [] }, "references": [],
|
||||
"categories": [{ "id": "c", "name": name("C"), "color": null }],
|
||||
"views": [{ "id": "main", "kind": "main", "name": name("Vue"), "svg_asset_id": SHEET_ASSET, "view_box": null }],
|
||||
"annotations": [{ "id": "a1", "kind": "point", "view_id": "main", "category_id": "c",
|
||||
"laterality": "unpaired", "geometry": { "type": "circle", "cx": 1, "cy": 1, "r": 1 },
|
||||
"review_status": "to_confirm", "name": name("P"), "finding_zone_id": "z1", "vertebral_region_code": null }],
|
||||
"finding_zones": [{ "id": "z1", "name": name("Z"), "categories": [{ "id": "cat", "kind": "dysfunction",
|
||||
"name": name("D"), "review_status": "to_confirm",
|
||||
"children": [{ "id": "n1", "name": name("N"), "review_status": "to_confirm" }] }] }]
|
||||
})
|
||||
}
|
||||
|
||||
/// A document at every cap (the core's `sheet_verdict::at_caps`): 64 views, 1024
|
||||
/// annotations, 8192 finding nodes, depth 8.
|
||||
pub fn sheet_document_at_caps() -> Value {
|
||||
let mut doc = sheet_document();
|
||||
let mut views = vec![
|
||||
json!({ "id": "main", "kind": "main", "name": name("V"), "svg_asset_id": SHEET_ASSET, "view_box": null }),
|
||||
];
|
||||
let mut annotations = Vec::new();
|
||||
for i in 1..64 {
|
||||
views.push(json!({ "id": format!("d{i}"), "kind": "detail", "name": name("D"), "svg_asset_id": null,
|
||||
"view_box": { "min_x": 0, "min_y": 0, "width": 5, "height": 5 }, "parent_view_id": "main" }));
|
||||
annotations.push(json!({ "id": format!("t{i}"), "kind": "detail_trigger", "view_id": "main", "category_id": "c",
|
||||
"laterality": "unpaired", "geometry": { "type": "polygon", "points": [[0, 0], [1, 0], [1, 1]] },
|
||||
"review_status": "reviewed", "name": null, "opens_view_id": format!("d{i}") }));
|
||||
}
|
||||
let zones: Vec<Value> = (0..64)
|
||||
.map(|z| {
|
||||
let mut children = vec![chain_reviewed(7)];
|
||||
children.extend((0..120).map(|n| json!({ "id": format!("l{n}"), "name": name("Nœud de lésion"), "review_status": "reviewed" })));
|
||||
json!({ "id": format!("z{z}"), "name": name("Zone"), "categories": [{ "id": "cat", "kind": "dysfunction",
|
||||
"name": name("Catégorie"), "review_status": "reviewed", "children": children }] })
|
||||
})
|
||||
.collect();
|
||||
for i in 0..(1024 - 63) {
|
||||
annotations.push(json!({ "id": format!("p{i}"), "kind": "point", "view_id": format!("d{}", 1 + i % 63),
|
||||
"category_id": "c", "laterality": "left", "geometry": { "type": "circle", "cx": 1.25, "cy": 2.5, "r": 0.5 },
|
||||
"review_status": "reviewed", "name": name("Point"), "finding_zone_id": format!("z{}", i % 64),
|
||||
"vertebral_region_code": null }));
|
||||
}
|
||||
doc["views"] = Value::Array(views);
|
||||
doc["annotations"] = Value::Array(annotations);
|
||||
doc["finding_zones"] = Value::Array(zones);
|
||||
doc
|
||||
}
|
||||
|
||||
fn chain_reviewed(levels: usize) -> Value {
|
||||
let mut node =
|
||||
json!({ "id": format!("c{levels}"), "name": name("N"), "review_status": "reviewed" });
|
||||
for level in (1..levels).rev() {
|
||||
node = json!({ "id": format!("c{level}"), "name": name("N"), "review_status": "reviewed", "children": [node] });
|
||||
}
|
||||
node
|
||||
}
|
||||
|
||||
/// The framed input of `document` drawn on `SHEET_DRAWING`, the document also as its base
|
||||
/// (the base path is traced too); empty on a framing error.
|
||||
pub fn sheet_frame(document: &Value) -> Vec<u8> {
|
||||
let text = document.to_string();
|
||||
let sha256 = sha256_hex(SHEET_DRAWING.as_bytes());
|
||||
let input = VerdictInput {
|
||||
binding: sheet_binding(),
|
||||
document: &text,
|
||||
base_document: Some(&text),
|
||||
drawings: vec![VerdictDrawing {
|
||||
asset_id: SHEET_ASSET,
|
||||
sha256: &sha256,
|
||||
byte_size: SHEET_DRAWING.len() as u64,
|
||||
bytes: Some(SHEET_DRAWING.as_bytes()),
|
||||
}],
|
||||
};
|
||||
verify_frame::encode(&input).unwrap_or_default()
|
||||
}
|
||||
|
||||
/// Writes one named fixture on stdout (`faunosteo-worker fixture <name>`).
|
||||
pub fn by_name(name: &str) -> Option<Vec<u8>> {
|
||||
Some(match name {
|
||||
@@ -122,6 +232,8 @@ pub fn by_name(name: &str) -> Option<Vec<u8>> {
|
||||
"iccp-bomb" => png_iccp_bomb(300 << 20),
|
||||
"gif" => gif(),
|
||||
"truncated" => truncated_png(),
|
||||
"verify-clean" => sheet_frame(&sheet_document()),
|
||||
"verify-caps" => sheet_frame(&sheet_document_at_caps()),
|
||||
_ => return None,
|
||||
})
|
||||
}
|
||||
+145
-14
@@ -1,6 +1,8 @@
|
||||
//! `Backend` over PostgREST and the Storage API with the service key (STORAGE.md §3.3).
|
||||
//! Thin by design: key shapes are checked before any URL is built, every status is
|
||||
//! classified here, and the closed `P0001` refusals of the RPCs come back as `Refused`.
|
||||
//! The same client serves `VerifyBackend` (V1-2.3): larger caps for the round's input and
|
||||
//! the verdict, drawing keys checked before any URL is built.
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
@@ -9,13 +11,19 @@ use serde_json::{Value, json};
|
||||
use ureq::Agent;
|
||||
use ureq::http::Response;
|
||||
|
||||
use crate::backend::{Backend, BackendError, BackendResult, Job, Leftover, Purge};
|
||||
use crate::backend::{
|
||||
Backend, BackendError, BackendResult, CompleteAnswer, FailAnswer, Job, Leftover, Purge,
|
||||
VerifyBackend, VerifyInput, VerifyJob,
|
||||
};
|
||||
use crate::config::Config;
|
||||
use crate::keys;
|
||||
use crate::limits::MAX_INPUT_BYTES;
|
||||
|
||||
/// Largest RPC response the worker reads (a claim of 100 rows is a few KiB).
|
||||
const MAX_RPC_BODY: u64 = 1024 * 1024;
|
||||
/// Largest `get_sheet_verification_input` response: two 2 MiB documents, whose JSON string
|
||||
/// escaping can double them, and the drawing rows.
|
||||
const MAX_INPUT_RPC_BODY: u64 = 16 * 1024 * 1024;
|
||||
|
||||
pub struct HttpBackend {
|
||||
agent: Agent,
|
||||
@@ -52,8 +60,22 @@ impl HttpBackend {
|
||||
Ok(format!("{}/storage/v1/object/{bucket}/{key}", self.base))
|
||||
}
|
||||
|
||||
/// A sheet drawing: `<sha256>.svg` in a well-formed bucket, nothing else.
|
||||
fn drawing_url(&self, bucket: &str, key: &str) -> BackendResult<String> {
|
||||
validate_bucket(bucket)?;
|
||||
if !keys::is_drawing_key(key) {
|
||||
return Err(BackendError::Internal("drawing key refused".to_owned()));
|
||||
}
|
||||
Ok(format!("{}/storage/v1/object/{bucket}/{key}", self.base))
|
||||
}
|
||||
|
||||
/// Calls an RPC and returns the response body text of a 2xx.
|
||||
fn rpc(&self, name: &str, body: &Value) -> BackendResult<String> {
|
||||
self.rpc_capped(name, body, MAX_RPC_BODY)
|
||||
}
|
||||
|
||||
/// `rpc` with its own response cap.
|
||||
fn rpc_capped(&self, name: &str, body: &Value, cap: u64) -> BackendResult<String> {
|
||||
let mut response = self
|
||||
.agent
|
||||
.post(&self.rpc_url(name))
|
||||
@@ -63,7 +85,7 @@ impl HttpBackend {
|
||||
.send_json(body)
|
||||
.map_err(network)?;
|
||||
let status = response.status().as_u16();
|
||||
let text = read_text(&mut response)?;
|
||||
let text = read_text_capped(&mut response, cap)?;
|
||||
if (200..300).contains(&status) {
|
||||
Ok(text)
|
||||
} else {
|
||||
@@ -79,18 +101,12 @@ impl HttpBackend {
|
||||
fn rpc_void(&self, name: &str, body: &Value) -> BackendResult<()> {
|
||||
self.rpc(name, body).map(|_| ())
|
||||
}
|
||||
}
|
||||
|
||||
impl Backend for HttpBackend {
|
||||
fn claim_jobs(&self, max_jobs: u32) -> BackendResult<Vec<Job>> {
|
||||
self.rpc_json("claim_avatar_jobs", &json!({ "max_jobs": max_jobs }))
|
||||
}
|
||||
|
||||
fn download_quarantine(&self, bucket: &str, key: &str) -> BackendResult<Vec<u8>> {
|
||||
let url = self.object_url(bucket, key)?;
|
||||
/// GET of an object, read up to `cap` bytes (`Overflow` past it).
|
||||
fn get_object(&self, url: &str, cap: usize) -> BackendResult<Vec<u8>> {
|
||||
let mut response = self
|
||||
.agent
|
||||
.get(&url)
|
||||
.get(url)
|
||||
.header("apikey", &self.key)
|
||||
.header("Authorization", &format!("Bearer {}", self.key))
|
||||
.call()
|
||||
@@ -103,14 +119,25 @@ impl Backend for HttpBackend {
|
||||
let bytes = response
|
||||
.body_mut()
|
||||
.with_config()
|
||||
.limit(MAX_INPUT_BYTES as u64 + 1)
|
||||
.limit(cap as u64 + 1)
|
||||
.read_to_vec()
|
||||
.map_err(|e| BackendError::Network(e.to_string()))?;
|
||||
if bytes.len() > MAX_INPUT_BYTES {
|
||||
if bytes.len() > cap {
|
||||
return Err(BackendError::Overflow);
|
||||
}
|
||||
Ok(bytes)
|
||||
}
|
||||
}
|
||||
|
||||
impl Backend for HttpBackend {
|
||||
fn claim_jobs(&self, max_jobs: u32) -> BackendResult<Vec<Job>> {
|
||||
self.rpc_json("claim_avatar_jobs", &json!({ "max_jobs": max_jobs }))
|
||||
}
|
||||
|
||||
fn download_quarantine(&self, bucket: &str, key: &str) -> BackendResult<Vec<u8>> {
|
||||
let url = self.object_url(bucket, key)?;
|
||||
self.get_object(&url, MAX_INPUT_BYTES)
|
||||
}
|
||||
|
||||
fn put_clean(&self, bucket: &str, key: &str, bytes: &[u8]) -> BackendResult<()> {
|
||||
let url = self.object_url(bucket, key)?;
|
||||
@@ -194,6 +221,65 @@ impl Backend for HttpBackend {
|
||||
}
|
||||
}
|
||||
|
||||
impl VerifyBackend for HttpBackend {
|
||||
fn claim_verifications(&self, max_jobs: u32) -> BackendResult<Vec<VerifyJob>> {
|
||||
self.rpc_json(
|
||||
"claim_sheet_verifications",
|
||||
&json!({ "max_jobs": max_jobs }),
|
||||
)
|
||||
}
|
||||
|
||||
fn verification_input(
|
||||
&self,
|
||||
review_id: &str,
|
||||
lease_token: &str,
|
||||
) -> BackendResult<Option<VerifyInput>> {
|
||||
let text = self.rpc_capped(
|
||||
"get_sheet_verification_input",
|
||||
&json!({ "review_id": review_id, "lease_token": lease_token }),
|
||||
MAX_INPUT_RPC_BODY,
|
||||
)?;
|
||||
serde_json::from_str(&text).map_err(|e| BackendError::Decode(e.to_string()))
|
||||
}
|
||||
|
||||
fn download_drawing(
|
||||
&self,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
max_bytes: usize,
|
||||
) -> BackendResult<Vec<u8>> {
|
||||
let url = self.drawing_url(bucket, key)?;
|
||||
self.get_object(&url, max_bytes)
|
||||
}
|
||||
|
||||
fn complete_verification(
|
||||
&self,
|
||||
review_id: &str,
|
||||
lease_token: &str,
|
||||
verdict: &[u8],
|
||||
) -> BackendResult<CompleteAnswer> {
|
||||
// The spawner already checked the bytes are a JSON object (under the output cap).
|
||||
let verdict: Value =
|
||||
serde_json::from_slice(verdict).map_err(|e| BackendError::Internal(e.to_string()))?;
|
||||
self.rpc_json(
|
||||
"complete_sheet_verification",
|
||||
&json!({ "review_id": review_id, "lease_token": lease_token, "verdict": verdict }),
|
||||
)
|
||||
}
|
||||
|
||||
fn fail_verification(
|
||||
&self,
|
||||
review_id: &str,
|
||||
lease_token: &str,
|
||||
code: &str,
|
||||
) -> BackendResult<FailAnswer> {
|
||||
self.rpc_json(
|
||||
"fail_sheet_verification",
|
||||
&json!({ "review_id": review_id, "lease_token": lease_token, "code": code }),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_bucket(bucket: &str) -> BackendResult<()> {
|
||||
let ok = !bucket.is_empty()
|
||||
&& bucket
|
||||
@@ -211,10 +297,14 @@ fn network(error: ureq::Error) -> BackendError {
|
||||
}
|
||||
|
||||
fn read_text(response: &mut Response<ureq::Body>) -> BackendResult<String> {
|
||||
read_text_capped(response, MAX_RPC_BODY)
|
||||
}
|
||||
|
||||
fn read_text_capped(response: &mut Response<ureq::Body>, cap: u64) -> BackendResult<String> {
|
||||
response
|
||||
.body_mut()
|
||||
.with_config()
|
||||
.limit(MAX_RPC_BODY)
|
||||
.limit(cap)
|
||||
.read_to_string()
|
||||
.map_err(|e| BackendError::Network(e.to_string()))
|
||||
}
|
||||
@@ -295,6 +385,36 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_m6_answers_and_rows_deserialize() {
|
||||
assert_eq!(
|
||||
serde_json::from_str::<CompleteAnswer>("\"malformed\"").ok(),
|
||||
Some(CompleteAnswer::Malformed)
|
||||
);
|
||||
assert_eq!(
|
||||
serde_json::from_str::<FailAnswer>("\"retry\"").ok(),
|
||||
Some(FailAnswer::Retry)
|
||||
);
|
||||
assert!(serde_json::from_str::<CompleteAnswer>("\"other\"").is_err());
|
||||
assert_eq!(
|
||||
serde_json::from_str::<Option<VerifyInput>>("null").ok(),
|
||||
Some(None)
|
||||
);
|
||||
let input = serde_json::from_value::<Option<VerifyInput>>(json!({
|
||||
"review_id": "r", "version_id": "v", "sheet_id": "s", "revision": 2,
|
||||
"document": "{}", "base_document": null,
|
||||
"drawings": [{ "asset_id": "a", "sha256": "h", "byte_size": 10,
|
||||
"bucket": "anatomical-sheet-assets", "object_key": "h.svg", "uploaded": true }]
|
||||
}));
|
||||
assert_eq!(
|
||||
input.ok().flatten().map(|i| i.drawings[0].byte_size),
|
||||
Some(10)
|
||||
);
|
||||
let jobs = serde_json::from_value::<Vec<VerifyJob>>(json!([{ "review_id": "r",
|
||||
"version_id": "v", "sheet_id": "s", "revision": 1, "lease_token": "t", "attempts": 1 }]));
|
||||
assert_eq!(jobs.ok().map(|j| j[0].revision), Some(1));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn object_urls_need_valid_buckets_and_keys() {
|
||||
let config = Config::from_vars(|name| match name {
|
||||
@@ -320,6 +440,17 @@ mod tests {
|
||||
.object_url("avatars", "profiles/../etc/passwd")
|
||||
.is_err()
|
||||
);
|
||||
let drawing = format!("{}.svg", "ab".repeat(32));
|
||||
assert_eq!(
|
||||
backend
|
||||
.drawing_url("anatomical-sheet-assets", &drawing)
|
||||
.ok(),
|
||||
Some(format!(
|
||||
"http://kong:8000/storage/v1/object/anatomical-sheet-assets/{drawing}"
|
||||
))
|
||||
);
|
||||
assert!(backend.drawing_url("anatomical-sheet-assets", key).is_err());
|
||||
assert!(backend.drawing_url("a/b", &drawing).is_err());
|
||||
assert_eq!(
|
||||
backend.rpc_url("claim_avatar_jobs"),
|
||||
"http://kong:8000/rest/v1/rpc/claim_avatar_jobs"
|
||||
|
||||
@@ -79,6 +79,17 @@ pub fn validate_any_key(key: &str) -> Result<(), KeyError> {
|
||||
}
|
||||
}
|
||||
|
||||
/// Object key of a sheet drawing (bucket `anatomical-sheet-assets`): `<sha256 hex>.svg`,
|
||||
/// lowercase (V1-2.3, M6 `get_sheet_verification_input`).
|
||||
pub fn is_drawing_key(key: &str) -> bool {
|
||||
key.strip_suffix(".svg").is_some_and(|hash| {
|
||||
hash.len() == 64
|
||||
&& hash
|
||||
.bytes()
|
||||
.all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b))
|
||||
})
|
||||
}
|
||||
|
||||
/// True when a (valid) key's owner and asset segments are exactly these ids.
|
||||
pub fn belongs_to(key: &str, owner_id: &str, asset_id: &str) -> bool {
|
||||
let mut segments = key.split('/');
|
||||
@@ -153,4 +164,15 @@ mod tests {
|
||||
assert!(!belongs_to(&key, U1, U1));
|
||||
assert!(!is_uuid("10000000-0000-0000-0000-00000000000"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn drawing_keys_are_a_lowercase_sha256_and_svg() {
|
||||
let hash = "0123456789abcdef".repeat(4);
|
||||
assert!(is_drawing_key(&format!("{hash}.svg")));
|
||||
assert!(!is_drawing_key(&format!("{}.svg", hash.to_uppercase())));
|
||||
assert!(!is_drawing_key(&format!("{hash}.png")));
|
||||
assert!(!is_drawing_key(&format!("{}.svg", &hash[1..])));
|
||||
assert!(!is_drawing_key(&format!("../{}.svg", &hash[3..])));
|
||||
assert!(!is_drawing_key(&format!("x/{hash}.svg")));
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,9 @@
|
||||
//! child with an empty environment that confines itself (rlimits + seccomp) before reading
|
||||
//! one image on stdin and writing one 512 × 512 WebP on stdout. The parent never decodes:
|
||||
//! it checks the container header of what comes back (`webp_check`).
|
||||
//!
|
||||
//! V1-2.3 adds the `verify` child on the same model: one framed sheet input on stdin
|
||||
//! (`verify_frame`), the core's verdict JSON on stdout; `run --jobs sheet-verify` drives it.
|
||||
|
||||
pub mod backend;
|
||||
pub mod config;
|
||||
@@ -21,5 +24,9 @@ pub mod run;
|
||||
pub mod sandbox;
|
||||
pub mod spawn;
|
||||
pub mod sweep;
|
||||
pub mod verifier;
|
||||
pub mod verify;
|
||||
pub mod verify_frame;
|
||||
pub mod verify_job;
|
||||
pub mod watchdog;
|
||||
pub mod webp_check;
|
||||
@@ -1,7 +1,10 @@
|
||||
//! Caps of the decode child (review Part IV.1 and STEP8_FILES_ANALYSIS II.5 step 6).
|
||||
//! Caps of the decode child (review Part IV.1 and STEP8_FILES_ANALYSIS II.5 step 6) and of the
|
||||
//! verify child (V1-2.3).
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
use faunosteo_core::domain::archive::snapshot::SnapshotCaps;
|
||||
|
||||
/// Largest accepted upload (STEP8 D4: 5 MiB at upload).
|
||||
pub const MAX_INPUT_BYTES: usize = 5 * 1024 * 1024;
|
||||
/// Largest output the parent reads before killing the child (a 512 × 512 WebP is ~30 KB).
|
||||
@@ -24,3 +27,24 @@ pub const CHILD_CPU_SECONDS: u64 = 10;
|
||||
pub const CHILD_DEADLINE: Duration = Duration::from_secs(20);
|
||||
/// Oldest libwebp the worker accepts to link (D-R4: the linked version is controlled).
|
||||
pub const MIN_LIBWEBP_VERSION: (u8, u8, u8) = (1, 6, 0);
|
||||
|
||||
/// Caps of the `verify` child (V1-2.3, contract W3), sized on the at-caps fixture: the
|
||||
/// document and the base (2 MiB each, SQL CHECK), four drawings (6 MiB in all,
|
||||
/// `SnapshotCaps::ASSETS*`), the header and a margin.
|
||||
pub const VERIFY_MAX_INPUT_BYTES: usize = 12 * 1024 * 1024;
|
||||
/// Largest frame header (ids and drawing rows, a few KiB in practice).
|
||||
pub const VERIFY_MAX_HEADER_BYTES: usize = 64 * 1024;
|
||||
/// Largest verdict the parent reads (= SQL `sheet_verification_max_verdict_bytes`).
|
||||
pub const VERIFY_MAX_OUTPUT_BYTES: usize = 4 * 1024 * 1024;
|
||||
/// Hard address-space cap of the `verify` child (`RLIMIT_AS`).
|
||||
pub const VERIFY_ADDRESS_SPACE_BYTES: u64 = 1024 * 1024 * 1024;
|
||||
/// CPU-time cap of the `verify` child (`RLIMIT_CPU`, seconds).
|
||||
pub const VERIFY_CPU_SECONDS: u64 = 60;
|
||||
/// Wall-clock deadline of the `verify` child (contract budget: under 120 s at the caps).
|
||||
pub const VERIFY_DEADLINE: Duration = Duration::from_secs(90);
|
||||
/// Drawings the parent downloads for one verification, at most.
|
||||
pub const VERIFY_MAX_DRAWINGS: usize = SnapshotCaps::ASSETS;
|
||||
/// Largest drawing the parent downloads.
|
||||
pub const VERIFY_MAX_DRAWING_BYTES: usize = SnapshotCaps::ASSET_BYTES;
|
||||
/// Largest total of the drawings downloaded for one verification.
|
||||
pub const VERIFY_MAX_DRAWINGS_TOTAL_BYTES: usize = SnapshotCaps::ASSETS_TOTAL_BYTES;
|
||||
@@ -57,6 +57,11 @@ impl LogLine {
|
||||
self.with("asset_id", id)
|
||||
}
|
||||
|
||||
/// A sheet verification round (V1-2.3).
|
||||
pub fn review(self, id: &str) -> Self {
|
||||
self.with("review_id", id)
|
||||
}
|
||||
|
||||
pub fn stage(self, stage: &str) -> Self {
|
||||
self.with("stage", stage)
|
||||
}
|
||||
|
||||
+20
-13
@@ -1,14 +1,14 @@
|
||||
//! `faunosteo-worker` entry: a plain `main`, no async runtime, no thread, so the `decode`
|
||||
//! child installs its sandbox in a single-threaded process (review Part IV.1).
|
||||
//!
|
||||
//! Subcommands: `decode` (the sandboxed child, R0), `run` (the keyed job loop, R1),
|
||||
//! `run --check` (prints the resolved configuration without secrets), `version`.
|
||||
//! Subcommands: `decode` (the sandboxed child, R0), `verify` (the sandboxed sheet verdict
|
||||
//! child, V1-2.3), `run` (the keyed job loop, R1), `run --jobs sheet-verify` (the verification
|
||||
//! loop), `run --check` (prints the resolved configuration without secrets), `version`.
|
||||
|
||||
#[cfg(feature = "test-hooks")]
|
||||
use std::io::Write;
|
||||
|
||||
use faunosteo_worker::decode;
|
||||
use faunosteo_worker::exit_codes;
|
||||
use faunosteo_worker::{decode, exit_codes, verify};
|
||||
|
||||
fn main() {
|
||||
let args: Vec<String> = std::env::args().collect();
|
||||
@@ -24,6 +24,17 @@ fn main() {
|
||||
&mut stderr.lock(),
|
||||
)
|
||||
}
|
||||
Some("verify") => {
|
||||
let stdin = std::io::stdin();
|
||||
let stdout = std::io::stdout();
|
||||
let stderr = std::io::stderr();
|
||||
verify::main(
|
||||
&args[2..],
|
||||
&mut stdin.lock(),
|
||||
&mut stdout.lock(),
|
||||
&mut stderr.lock(),
|
||||
)
|
||||
}
|
||||
Some("run") => faunosteo_worker::run::main(&args[2..]),
|
||||
Some("version") => version(),
|
||||
#[cfg(feature = "test-hooks")]
|
||||
@@ -36,7 +47,9 @@ fn main() {
|
||||
exit_codes::OK
|
||||
}
|
||||
_ => {
|
||||
eprintln!("usage: faunosteo-worker <decode | run --check | version>");
|
||||
eprintln!(
|
||||
"usage: faunosteo-worker <decode | verify | run [--jobs sheet-verify] [--check] | version>"
|
||||
);
|
||||
exit_codes::USAGE
|
||||
}
|
||||
};
|
||||
@@ -48,17 +61,11 @@ fn version() -> i32 {
|
||||
println!(
|
||||
"faunosteo-worker {} (core {}, libwebp {major}.{minor}.{revision})",
|
||||
env!("CARGO_PKG_VERSION"),
|
||||
faunosteo_core_version()
|
||||
faunosteo_core::CORE_VERSION
|
||||
);
|
||||
exit_codes::OK
|
||||
}
|
||||
|
||||
fn faunosteo_core_version() -> &'static str {
|
||||
// The worker links no core module before R1 (nothing to share yet); the version printed
|
||||
// is the workspace version the two crates share.
|
||||
env!("CARGO_PKG_VERSION")
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-hooks")]
|
||||
fn fixture(args: &[String]) -> i32 {
|
||||
let Some(bytes) = args
|
||||
@@ -66,7 +73,7 @@ fn fixture(args: &[String]) -> i32 {
|
||||
.and_then(|name| faunosteo_worker::fixtures::by_name(name))
|
||||
else {
|
||||
eprintln!(
|
||||
"usage: faunosteo-worker fixture <png|jpeg|webp|big-jpeg|big-png|tiny|huge-header|gif|truncated>"
|
||||
"usage: faunosteo-worker fixture <png|jpeg|webp|big-jpeg|big-png|tiny|huge-header|iccp-bomb|gif|truncated|verify-clean|verify-caps>"
|
||||
);
|
||||
return exit_codes::USAGE;
|
||||
};
|
||||
|
||||
+136
-14
@@ -1,10 +1,11 @@
|
||||
//! `faunosteo-worker run`: the keyed parent's loop (STORAGE.md §3.4, §3.5) and
|
||||
//! `run --check`, the start-up self-description without any secret value.
|
||||
//! `faunosteo-worker run`: the keyed parent's loop (STORAGE.md §3.4, §3.5), `run --jobs
|
||||
//! sheet-verify` (the verification loop of V1-2.3, no sweep) and `run --check`, the start-up
|
||||
//! self-description without any secret value.
|
||||
|
||||
use std::io::Write;
|
||||
use std::time::{Instant, SystemTime, UNIX_EPOCH};
|
||||
|
||||
use crate::backend::Backend;
|
||||
use crate::backend::{Backend, VerifyBackend};
|
||||
use crate::config::{self, Config, ConfigError};
|
||||
use crate::decode::libwebp_version;
|
||||
use crate::decoder::{ChildDecoder, Decoder};
|
||||
@@ -12,23 +13,56 @@ use crate::http::HttpBackend;
|
||||
use crate::job::{JobEnd, process_job};
|
||||
use crate::limits::MIN_LIBWEBP_VERSION;
|
||||
use crate::sweep::{SweepConfig, run_sweep};
|
||||
use crate::verifier::{ChildVerifier, Verifier};
|
||||
use crate::verify_job::{VerifyEnd, process_verification};
|
||||
use crate::watchdog::{Watchdog, deadlines};
|
||||
use crate::{exit_codes, log};
|
||||
|
||||
const THIRTY_DAYS: i64 = 30 * 24 * 3600;
|
||||
|
||||
/// What a `run` process works on.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum Mode {
|
||||
/// Avatar jobs and the sweep (the default).
|
||||
Avatars,
|
||||
/// Sheet verification rounds only (`--jobs sheet-verify`).
|
||||
SheetVerify,
|
||||
}
|
||||
|
||||
/// Entry of the `run` subcommand; returns the process exit code.
|
||||
pub fn main(args: &[String]) -> i32 {
|
||||
match args.first().map(String::as_str) {
|
||||
Some("--check") => check(),
|
||||
Some(other) => {
|
||||
eprintln!("run: unknown argument {other} (use --check to print the configuration)");
|
||||
match parse_args(args) {
|
||||
Ok((mode, true)) => check(mode),
|
||||
Ok((Mode::Avatars, false)) => start(),
|
||||
Ok((Mode::SheetVerify, false)) => start_verify(),
|
||||
Err(other) => {
|
||||
eprintln!(
|
||||
"run: unknown argument {other} (use --check to print the configuration, --jobs sheet-verify for the verification mode)"
|
||||
);
|
||||
exit_codes::USAGE
|
||||
}
|
||||
None => start(),
|
||||
}
|
||||
}
|
||||
|
||||
/// `[--jobs sheet-verify] [--check]` in any order; `Err` names the first unknown argument.
|
||||
pub fn parse_args(args: &[String]) -> Result<(Mode, bool), String> {
|
||||
let mut mode = Mode::Avatars;
|
||||
let mut check = false;
|
||||
let mut iter = args.iter();
|
||||
while let Some(arg) = iter.next() {
|
||||
match arg.as_str() {
|
||||
"--check" if !check => check = true,
|
||||
"--jobs" if mode == Mode::Avatars => match iter.next().map(String::as_str) {
|
||||
Some("sheet-verify") => mode = Mode::SheetVerify,
|
||||
Some(value) => return Err(format!("--jobs {value}")),
|
||||
None => return Err("--jobs".to_owned()),
|
||||
},
|
||||
other => return Err(other.to_owned()),
|
||||
}
|
||||
}
|
||||
Ok((mode, check))
|
||||
}
|
||||
|
||||
fn start() -> i32 {
|
||||
let config = match Config::from_env() {
|
||||
Ok(config) => config,
|
||||
@@ -43,6 +77,7 @@ fn start() -> i32 {
|
||||
}
|
||||
log::set_worker_id(&config.worker_id);
|
||||
log_start(&config);
|
||||
log_token_expiry(&config);
|
||||
let decoder = match ChildDecoder::from_current_exe() {
|
||||
Ok(decoder) => decoder,
|
||||
Err(error) => {
|
||||
@@ -58,6 +93,83 @@ fn start() -> i32 {
|
||||
run_loop(&config, &backend, &decoder, &watchdog)
|
||||
}
|
||||
|
||||
fn start_verify() -> i32 {
|
||||
let config = match Config::from_env() {
|
||||
Ok(config) => config,
|
||||
Err(error) => {
|
||||
eprintln!("run: {error}");
|
||||
return exit_codes::USAGE;
|
||||
}
|
||||
};
|
||||
log::set_worker_id(&config.worker_id);
|
||||
log::info("worker_start")
|
||||
.detail(&format!(
|
||||
"mode sheet-verify; poll {}s once {}",
|
||||
config.poll.as_secs(),
|
||||
config.once
|
||||
))
|
||||
.emit();
|
||||
log_token_expiry(&config);
|
||||
let verifier = match ChildVerifier::from_current_exe() {
|
||||
Ok(verifier) => verifier,
|
||||
Err(error) => {
|
||||
log::error("start_failed")
|
||||
.detail(&format!("current_exe: {error}"))
|
||||
.emit();
|
||||
return exit_codes::INTERNAL;
|
||||
}
|
||||
};
|
||||
let backend = HttpBackend::new(&config);
|
||||
let watchdog = Watchdog::new();
|
||||
watchdog.spawn_monitor();
|
||||
verify_loop(&config, &backend, &verifier, &watchdog)
|
||||
}
|
||||
|
||||
/// Claims one round per pass, no sweep (the avatar worker owns it); idles on `poll`.
|
||||
pub fn verify_loop(
|
||||
config: &Config,
|
||||
backend: &dyn VerifyBackend,
|
||||
verifier: &dyn Verifier,
|
||||
watchdog: &Watchdog,
|
||||
) -> i32 {
|
||||
loop {
|
||||
let worked = match verify_pass(backend, verifier, watchdog) {
|
||||
Ok(count) => count > 0,
|
||||
Err(code) => return code,
|
||||
};
|
||||
if config.once {
|
||||
return exit_codes::OK;
|
||||
}
|
||||
if !worked {
|
||||
std::thread::sleep(config.poll);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// One claim and the rounds it returned; `Err` carries the exit code of a stop.
|
||||
fn verify_pass(
|
||||
backend: &dyn VerifyBackend,
|
||||
verifier: &dyn Verifier,
|
||||
watchdog: &Watchdog,
|
||||
) -> Result<usize, i32> {
|
||||
let jobs = {
|
||||
let _stage = watchdog.enter("claim", deadlines::CLAIM);
|
||||
match backend.claim_verifications(1) {
|
||||
Ok(jobs) => jobs,
|
||||
Err(error) => {
|
||||
log_claim_failure(&error);
|
||||
return Ok(0);
|
||||
}
|
||||
}
|
||||
};
|
||||
for job in &jobs {
|
||||
if process_verification(backend, verifier, watchdog, job) == VerifyEnd::Stop {
|
||||
return Err(exit_codes::CHILD_BROKEN);
|
||||
}
|
||||
}
|
||||
Ok(jobs.len())
|
||||
}
|
||||
|
||||
/// Sweep at start and every `sweep` interval; claim one job per pass; idle on `poll`.
|
||||
pub fn run_loop(
|
||||
config: &Config,
|
||||
@@ -99,11 +211,7 @@ fn claim_pass(
|
||||
match backend.claim_jobs(1) {
|
||||
Ok(jobs) => jobs,
|
||||
Err(error) => {
|
||||
let line = log::warn("claim_failed").code(error.kind());
|
||||
match error {
|
||||
crate::backend::BackendError::Status(status) => line.http_status(status).emit(),
|
||||
_ => line.emit(),
|
||||
}
|
||||
log_claim_failure(&error);
|
||||
return Ok(0);
|
||||
}
|
||||
}
|
||||
@@ -116,6 +224,14 @@ fn claim_pass(
|
||||
Ok(jobs.len())
|
||||
}
|
||||
|
||||
fn log_claim_failure(error: &crate::backend::BackendError) {
|
||||
let line = log::warn("claim_failed").code(error.kind());
|
||||
match error {
|
||||
crate::backend::BackendError::Status(status) => line.http_status(*status).emit(),
|
||||
_ => line.emit(),
|
||||
}
|
||||
}
|
||||
|
||||
fn log_start(config: &Config) {
|
||||
let (major, minor, rev) = libwebp_version();
|
||||
log::info("worker_start")
|
||||
@@ -128,6 +244,9 @@ fn log_start(config: &Config) {
|
||||
config.once
|
||||
))
|
||||
.emit();
|
||||
}
|
||||
|
||||
fn log_token_expiry(config: &Config) {
|
||||
match token_expiry(config) {
|
||||
TokenExpiry::NotJwt => log::info("token_no_expiry").emit(),
|
||||
TokenExpiry::Valid(days) => log::info("token_expiry").count(days).emit(),
|
||||
@@ -170,7 +289,10 @@ fn libwebp_ok() -> bool {
|
||||
/// worker could start with this environment, or when no worker variable is set at all
|
||||
/// (an unconfigured image is a valid artifact check); a partial or invalid configuration
|
||||
/// and a too-old libwebp exit non-zero and name the variable.
|
||||
fn check() -> i32 {
|
||||
fn check(mode: Mode) -> i32 {
|
||||
if mode == Mode::SheetVerify {
|
||||
println!("mode: sheet-verify (verification rounds only, no sweep)");
|
||||
}
|
||||
let (major, minor, rev) = libwebp_version();
|
||||
let libwebp_ok = libwebp_ok();
|
||||
println!(
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
//! Confinement of the decode child (review Part IV.1), applied by the child itself from a
|
||||
//! single-threaded process before it reads any input. Three layers: an empty environment
|
||||
//! (cleared by the parent, checked in `decode`), rlimits on address space and CPU time, and
|
||||
//! a seccomp allowlist installed with TSYNC that kills the process on any other syscall.
|
||||
//! Confinement of the decode and verify children (review Part IV.1), applied by the child
|
||||
//! itself from a single-threaded process before it reads any input. Three layers: an empty
|
||||
//! environment (cleared by the parent, checked in `decode` and `verify`), rlimits on address
|
||||
//! space and CPU time, and a seccomp allowlist installed with TSYNC that kills the process on
|
||||
//! any other syscall.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::fmt;
|
||||
@@ -13,16 +14,20 @@ use seccompiler::{
|
||||
|
||||
use crate::limits;
|
||||
|
||||
/// Syscalls the decode child makes once the filter is installed (static musl binary, x86_64).
|
||||
/// Syscalls the decode and verify children make once the filter is installed (static musl
|
||||
/// binary, x86_64).
|
||||
///
|
||||
/// Derived with `strace -f` over the fixture corpus at the caps (`crates/worker/scripts/
|
||||
/// derive-seccomp.sh`, run by the Dockerfile stage `test`, 2026-10-12: exactly these seven
|
||||
/// after the marker) and proven by running every fixture through the full sandbox. musl's
|
||||
/// mallocng used neither `brk` nor `mprotect`, and no `futex` appears in a single-threaded
|
||||
/// decode.
|
||||
/// mallocng used neither `brk` nor `mprotect` there, and no `futex` appears in a
|
||||
/// single-threaded child. The verify child at the caps (2026-11-01) adds `brk`: mallocng
|
||||
/// grows its heap for the core's many small allocations; it only moves the program break,
|
||||
/// bounded by `RLIMIT_AS`.
|
||||
pub const ALLOWED_SYSCALLS: &[(&str, i64)] = &[
|
||||
("read", libc::SYS_read),
|
||||
("write", libc::SYS_write),
|
||||
("brk", libc::SYS_brk),
|
||||
("mmap", libc::SYS_mmap),
|
||||
("munmap", libc::SYS_munmap),
|
||||
("mremap", libc::SYS_mremap),
|
||||
|
||||
+72
-15
@@ -1,6 +1,7 @@
|
||||
//! Runs the decode child from the keyed parent (review Part IV.1): empty environment,
|
||||
//! bounded output, wall-clock deadline, and the mapping of every way the child can end to
|
||||
//! a closed outcome. The parent reads the container header of the output, never the pixels.
|
||||
//! Runs a child from the keyed parent (review Part IV.1): empty environment, bounded
|
||||
//! output, wall-clock deadline, and the mapping of every way the child can end to a closed
|
||||
//! outcome. The parent reads the container header of a `decode` output, never the pixels,
|
||||
//! and checks only that a `verify` output is a JSON object (V1-2.3).
|
||||
|
||||
use std::io::{Read, Write};
|
||||
use std::os::unix::process::ExitStatusExt;
|
||||
@@ -9,13 +10,34 @@ use std::process::{Command, ExitStatus, Stdio};
|
||||
use std::sync::mpsc;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use crate::limits::{CHILD_DEADLINE, MAX_OUTPUT_BYTES, OUTPUT_SIDE};
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
use crate::limits::{
|
||||
CHILD_DEADLINE, MAX_OUTPUT_BYTES, OUTPUT_SIDE, VERIFY_DEADLINE, VERIFY_MAX_OUTPUT_BYTES,
|
||||
};
|
||||
use crate::{exit_codes, webp_check};
|
||||
|
||||
/// How a decode child ended.
|
||||
/// The subcommand a spawner runs; the output check depends on it.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum ChildCommand {
|
||||
Decode,
|
||||
Verify,
|
||||
}
|
||||
|
||||
impl ChildCommand {
|
||||
fn arg(self) -> &'static str {
|
||||
match self {
|
||||
Self::Decode => "decode",
|
||||
Self::Verify => "verify",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// How a child ended.
|
||||
#[derive(Debug)]
|
||||
pub enum ChildOutcome {
|
||||
/// A 512 × 512 WebP whose container passed the parent's check.
|
||||
/// A 512 × 512 WebP whose container passed the parent's check (`decode`), or a JSON
|
||||
/// object (`verify`).
|
||||
Produced(Vec<u8>),
|
||||
/// The child refused the input with one of its content exit codes
|
||||
/// (`exit_codes::is_content_rejection`).
|
||||
@@ -44,29 +66,51 @@ pub struct ChildRun {
|
||||
pub stderr: String,
|
||||
}
|
||||
|
||||
/// Spawns `decode` children of one worker binary.
|
||||
/// Spawns children of one worker binary, one command with its caps.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Spawner {
|
||||
pub exe: PathBuf,
|
||||
pub command: ChildCommand,
|
||||
pub deadline: Duration,
|
||||
pub max_output: usize,
|
||||
pub extra_args: Vec<String>,
|
||||
}
|
||||
|
||||
impl Spawner {
|
||||
/// A `decode` spawner (the R0 name, kept for its callers).
|
||||
pub fn new(exe: impl Into<PathBuf>) -> Self {
|
||||
Self::for_decode(exe)
|
||||
}
|
||||
|
||||
pub fn for_decode(exe: impl Into<PathBuf>) -> Self {
|
||||
Self {
|
||||
exe: exe.into(),
|
||||
command: ChildCommand::Decode,
|
||||
deadline: CHILD_DEADLINE,
|
||||
max_output: MAX_OUTPUT_BYTES,
|
||||
extra_args: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Feeds `input` to a fresh child and classifies what comes back.
|
||||
pub fn for_verify(exe: impl Into<PathBuf>) -> Self {
|
||||
Self {
|
||||
exe: exe.into(),
|
||||
command: ChildCommand::Verify,
|
||||
deadline: VERIFY_DEADLINE,
|
||||
max_output: VERIFY_MAX_OUTPUT_BYTES,
|
||||
extra_args: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Runs the spawner's command (`decode` for a spawner built by `new`); the R0 name.
|
||||
pub fn decode(&self, input: &[u8]) -> ChildRun {
|
||||
self.run(input)
|
||||
}
|
||||
|
||||
/// Feeds `input` to a fresh child and classifies what comes back.
|
||||
pub fn run(&self, input: &[u8]) -> ChildRun {
|
||||
let spawned = Command::new(&self.exe)
|
||||
.arg("decode")
|
||||
.arg(self.command.arg())
|
||||
.args(&self.extra_args)
|
||||
.env_clear()
|
||||
.stdin(Stdio::piped())
|
||||
@@ -149,18 +193,15 @@ impl Spawner {
|
||||
}
|
||||
let outcome = match status {
|
||||
None => ChildOutcome::Timeout,
|
||||
Some(status) => classify(status, output),
|
||||
Some(status) => classify(self.command, status, output),
|
||||
};
|
||||
ChildRun { outcome, stderr }
|
||||
}
|
||||
}
|
||||
|
||||
fn classify(status: ExitStatus, output: Vec<u8>) -> ChildOutcome {
|
||||
fn classify(command: ChildCommand, status: ExitStatus, output: Vec<u8>) -> ChildOutcome {
|
||||
match status.code() {
|
||||
Some(0) => match webp_check::expect_square(&output, OUTPUT_SIDE) {
|
||||
Ok(_) => ChildOutcome::Produced(output),
|
||||
Err(issue) => ChildOutcome::Failed(format!("output refused: {issue}")),
|
||||
},
|
||||
Some(0) => check_output(command, output),
|
||||
Some(code) if exit_codes::is_content_rejection(code) => ChildOutcome::Rejected {
|
||||
code,
|
||||
name: exit_codes::name(code),
|
||||
@@ -175,6 +216,22 @@ fn classify(status: ExitStatus, output: Vec<u8>) -> ChildOutcome {
|
||||
}
|
||||
}
|
||||
|
||||
/// The output of a child that exited 0: a square WebP container, or a JSON object.
|
||||
fn check_output(command: ChildCommand, output: Vec<u8>) -> ChildOutcome {
|
||||
let checked = match command {
|
||||
ChildCommand::Decode => webp_check::expect_square(&output, OUTPUT_SIDE)
|
||||
.map(|_| ())
|
||||
.map_err(|issue| issue.to_string()),
|
||||
ChildCommand::Verify => serde_json::from_slice::<Map<String, Value>>(&output)
|
||||
.map(|_| ())
|
||||
.map_err(|_| "not a JSON object".to_owned()),
|
||||
};
|
||||
match checked {
|
||||
Ok(()) => ChildOutcome::Produced(output),
|
||||
Err(issue) => ChildOutcome::Failed(format!("output refused: {issue}")),
|
||||
}
|
||||
}
|
||||
|
||||
fn failed(detail: String) -> ChildRun {
|
||||
ChildRun {
|
||||
outcome: ChildOutcome::Failed(detail),
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
//! The verify step as the job sees it: a framed input in, a classified child run out. The
|
||||
//! real implementation spawns this very binary as `verify` (V1-2.3); tests inject outcomes.
|
||||
|
||||
use crate::spawn::{ChildRun, Spawner};
|
||||
|
||||
pub trait Verifier {
|
||||
fn verify(&self, frame: &[u8]) -> ChildRun;
|
||||
}
|
||||
|
||||
/// Spawns `faunosteo-worker verify` from the running executable.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ChildVerifier {
|
||||
spawner: Spawner,
|
||||
}
|
||||
|
||||
impl ChildVerifier {
|
||||
/// Fails only when the executable path cannot be resolved (then nothing can be verified).
|
||||
pub fn from_current_exe() -> std::io::Result<Self> {
|
||||
Ok(Self {
|
||||
spawner: Spawner::for_verify(std::env::current_exe()?),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl Verifier for ChildVerifier {
|
||||
fn verify(&self, frame: &[u8]) -> ChildRun {
|
||||
self.spawner.run(frame)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
//! The verify child (V1-2.3, contract W2): one framed input in (the round, the drawing rows,
|
||||
//! the document, the base, the drawing bytes), the core's verdict JSON out. Same binary,
|
||||
//! same confinement as `decode`; the verdict itself is `faunosteo_core`'s `compute_verdict`.
|
||||
|
||||
use std::io::{Read, Write};
|
||||
|
||||
use faunosteo_core::domain::sheet::verdict::compute_verdict;
|
||||
|
||||
use crate::decode::TestHooks;
|
||||
use crate::limits::{VERIFY_ADDRESS_SPACE_BYTES, VERIFY_CPU_SECONDS, VERIFY_MAX_INPUT_BYTES};
|
||||
use crate::verify_frame::{self, FrameError};
|
||||
use crate::{exit_codes, sandbox};
|
||||
|
||||
/// Why the child produced no verdict (one exit code each).
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum VerifyFailure {
|
||||
/// The parent framed the input wrongly: a parent bug, never the content's.
|
||||
Frame(FrameError),
|
||||
/// The verdict could not be serialised: a fault of the child.
|
||||
Encode,
|
||||
}
|
||||
|
||||
impl VerifyFailure {
|
||||
pub fn exit_code(self) -> i32 {
|
||||
match self {
|
||||
Self::Frame(_) => exit_codes::FRAME_INVALID,
|
||||
Self::Encode => exit_codes::INTERNAL,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Entry of `faunosteo-worker verify`: returns the process exit code.
|
||||
///
|
||||
/// Same order as `decode::main`: the environment check and the sandbox come before the
|
||||
/// first byte of input is read. A failed verdict is exit 0: it is content, judged by the core.
|
||||
pub fn main(
|
||||
args: &[String],
|
||||
stdin: &mut dyn Read,
|
||||
stdout: &mut dyn Write,
|
||||
stderr: &mut dyn Write,
|
||||
) -> i32 {
|
||||
if std::env::vars_os().next().is_some() {
|
||||
let _ = writeln!(stderr, "verify: the environment must be empty");
|
||||
return exit_codes::ENV_NOT_EMPTY;
|
||||
}
|
||||
let hooks = match TestHooks::parse(args) {
|
||||
Ok(hooks) => hooks,
|
||||
Err(message) => {
|
||||
let _ = writeln!(stderr, "verify: {message}");
|
||||
return exit_codes::USAGE;
|
||||
}
|
||||
};
|
||||
let limits = sandbox::Limits {
|
||||
address_space_bytes: VERIFY_ADDRESS_SPACE_BYTES,
|
||||
cpu_seconds: VERIFY_CPU_SECONDS,
|
||||
};
|
||||
if let Err(error) = sandbox::apply(&limits, hooks.seccomp) {
|
||||
let _ = writeln!(stderr, "verify: sandbox: {error}");
|
||||
return exit_codes::SANDBOX_FAILED;
|
||||
}
|
||||
if let Some(code) = hooks.run_after_sandbox(stdout, stderr) {
|
||||
return code;
|
||||
}
|
||||
|
||||
let mut input = Vec::new();
|
||||
let read = stdin
|
||||
.take(VERIFY_MAX_INPUT_BYTES as u64 + 1)
|
||||
.read_to_end(&mut input);
|
||||
if read.is_err() {
|
||||
return exit_codes::INTERNAL;
|
||||
}
|
||||
if input.len() > VERIFY_MAX_INPUT_BYTES {
|
||||
let _ = writeln!(stderr, "verify: input over {VERIFY_MAX_INPUT_BYTES} bytes");
|
||||
return exit_codes::INPUT_TOO_LARGE;
|
||||
}
|
||||
|
||||
match process(&input) {
|
||||
Ok(verdict) => match stdout.write_all(&verdict).and_then(|()| stdout.flush()) {
|
||||
Ok(()) => exit_codes::OK,
|
||||
Err(_) => exit_codes::OUTPUT_FAILED,
|
||||
},
|
||||
Err(failure) => {
|
||||
let _ = match failure {
|
||||
VerifyFailure::Frame(error) => writeln!(stderr, "verify: frame: {error}"),
|
||||
VerifyFailure::Encode => writeln!(stderr, "verify: verdict not serialised"),
|
||||
};
|
||||
failure.exit_code()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Reads one frame and returns the verdict JSON; pure function of the bytes.
|
||||
pub fn process(frame: &[u8]) -> Result<Vec<u8>, VerifyFailure> {
|
||||
let frame = verify_frame::decode(frame).map_err(VerifyFailure::Frame)?;
|
||||
let verdict = compute_verdict(&frame.input());
|
||||
serde_json::to_vec(&verdict).map_err(|_| VerifyFailure::Encode)
|
||||
}
|
||||
@@ -0,0 +1,321 @@
|
||||
//! The framed stdin input of the `verify` child (V1-2.3), written by the parent and read by
|
||||
//! the child: `[u32 LE header length][header JSON][document][base?][drawing bytes in header
|
||||
//! order]`. The parent frames bytes it never parses (document, base, drawings); the child
|
||||
//! reads them as borrowed slices.
|
||||
|
||||
use std::fmt;
|
||||
|
||||
use faunosteo_core::domain::sheet::verdict::{VerdictBinding, VerdictDrawing, VerdictInput};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::limits::VERIFY_MAX_HEADER_BYTES;
|
||||
|
||||
/// The header: the round, the lengths of the parts that follow, the drawing rows.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct FrameHeader {
|
||||
pub sheet_id: String,
|
||||
pub version_id: String,
|
||||
pub revision: u32,
|
||||
pub review_id: String,
|
||||
pub document_len: usize,
|
||||
pub base_len: Option<usize>,
|
||||
pub drawings: Vec<FrameDrawing>,
|
||||
}
|
||||
|
||||
/// One drawing row; `bytes_len` is absent when the parent passes the row without bytes.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct FrameDrawing {
|
||||
pub asset_id: String,
|
||||
pub sha256: String,
|
||||
pub byte_size: u64,
|
||||
pub bytes_len: Option<usize>,
|
||||
}
|
||||
|
||||
/// A decoded frame, borrowing the input buffer.
|
||||
#[derive(Debug)]
|
||||
pub struct Frame<'a> {
|
||||
pub header: FrameHeader,
|
||||
pub document: &'a str,
|
||||
pub base: Option<&'a str>,
|
||||
/// One entry per header drawing, in header order.
|
||||
pub drawing_bytes: Vec<Option<&'a [u8]>>,
|
||||
}
|
||||
|
||||
/// Why a frame could not be built or read (closed: the child's stderr names it).
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum FrameError {
|
||||
Truncated,
|
||||
HeaderTooLarge,
|
||||
HeaderInvalid,
|
||||
LengthMismatch,
|
||||
NotUtf8,
|
||||
}
|
||||
|
||||
impl fmt::Display for FrameError {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
let text = match self {
|
||||
Self::Truncated => "frame shorter than its header",
|
||||
Self::HeaderTooLarge => "header over its cap",
|
||||
Self::HeaderInvalid => "header is not the expected JSON",
|
||||
Self::LengthMismatch => "part lengths do not sum to the frame",
|
||||
Self::NotUtf8 => "document or base is not UTF-8",
|
||||
};
|
||||
f.write_str(text)
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for FrameError {}
|
||||
|
||||
/// Frames a verification input (the parent side).
|
||||
pub fn encode(input: &VerdictInput<'_>) -> Result<Vec<u8>, FrameError> {
|
||||
let header = header_of(input);
|
||||
let header_json = serde_json::to_vec(&header).map_err(|_| FrameError::HeaderInvalid)?;
|
||||
if header_json.len() > VERIFY_MAX_HEADER_BYTES {
|
||||
return Err(FrameError::HeaderTooLarge);
|
||||
}
|
||||
let header_len = u32::try_from(header_json.len()).map_err(|_| FrameError::HeaderTooLarge)?;
|
||||
let body = checked_body_len(&header).unwrap_or(0);
|
||||
let mut out = Vec::with_capacity(4 + header_json.len() + body);
|
||||
out.extend_from_slice(&header_len.to_le_bytes());
|
||||
out.extend_from_slice(&header_json);
|
||||
out.extend_from_slice(input.document.as_bytes());
|
||||
if let Some(base) = input.base_document {
|
||||
out.extend_from_slice(base.as_bytes());
|
||||
}
|
||||
for bytes in input.drawings.iter().filter_map(|d| d.bytes) {
|
||||
out.extend_from_slice(bytes);
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
fn header_of(input: &VerdictInput<'_>) -> FrameHeader {
|
||||
FrameHeader {
|
||||
sheet_id: input.binding.sheet_id.clone(),
|
||||
version_id: input.binding.version_id.clone(),
|
||||
revision: input.binding.revision,
|
||||
review_id: input.binding.review_id.clone(),
|
||||
document_len: input.document.len(),
|
||||
base_len: input.base_document.map(str::len),
|
||||
drawings: input
|
||||
.drawings
|
||||
.iter()
|
||||
.map(|d| FrameDrawing {
|
||||
asset_id: d.asset_id.to_owned(),
|
||||
sha256: d.sha256.to_owned(),
|
||||
byte_size: d.byte_size,
|
||||
bytes_len: d.bytes.map(<[u8]>::len),
|
||||
})
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Total length of the parts after the header; `None` on overflow.
|
||||
fn checked_body_len(header: &FrameHeader) -> Option<usize> {
|
||||
header
|
||||
.drawings
|
||||
.iter()
|
||||
.filter_map(|d| d.bytes_len)
|
||||
.try_fold(header.document_len, usize::checked_add)?
|
||||
.checked_add(header.base_len.unwrap_or(0))
|
||||
}
|
||||
|
||||
/// Reads a frame (the child side): every length must be exact, nothing may trail.
|
||||
pub fn decode(bytes: &[u8]) -> Result<Frame<'_>, FrameError> {
|
||||
let (prefix, rest) = bytes.split_at_checked(4).ok_or(FrameError::Truncated)?;
|
||||
let mut length = [0u8; 4];
|
||||
length.copy_from_slice(prefix);
|
||||
let header_len =
|
||||
usize::try_from(u32::from_le_bytes(length)).map_err(|_| FrameError::HeaderTooLarge)?;
|
||||
if header_len > VERIFY_MAX_HEADER_BYTES {
|
||||
return Err(FrameError::HeaderTooLarge);
|
||||
}
|
||||
let (header_json, mut body) = rest
|
||||
.split_at_checked(header_len)
|
||||
.ok_or(FrameError::Truncated)?;
|
||||
let header: FrameHeader =
|
||||
serde_json::from_slice(header_json).map_err(|_| FrameError::HeaderInvalid)?;
|
||||
if checked_body_len(&header) != Some(body.len()) {
|
||||
return Err(FrameError::LengthMismatch);
|
||||
}
|
||||
let document = take_text(&mut body, header.document_len)?;
|
||||
let base = header
|
||||
.base_len
|
||||
.map(|len| take_text(&mut body, len))
|
||||
.transpose()?;
|
||||
let drawing_bytes = header
|
||||
.drawings
|
||||
.iter()
|
||||
.map(|d| d.bytes_len.map(|len| take(&mut body, len)))
|
||||
.collect();
|
||||
Ok(Frame {
|
||||
header,
|
||||
document,
|
||||
base,
|
||||
drawing_bytes,
|
||||
})
|
||||
}
|
||||
|
||||
/// Splits `len` bytes off the front; the lengths were checked against the body beforehand.
|
||||
fn take<'a>(body: &mut &'a [u8], len: usize) -> &'a [u8] {
|
||||
let (head, tail) = body.split_at(len.min(body.len()));
|
||||
*body = tail;
|
||||
head
|
||||
}
|
||||
|
||||
fn take_text<'a>(body: &mut &'a [u8], len: usize) -> Result<&'a str, FrameError> {
|
||||
std::str::from_utf8(take(body, len)).map_err(|_| FrameError::NotUtf8)
|
||||
}
|
||||
|
||||
impl Frame<'_> {
|
||||
/// The core's input, borrowing the frame.
|
||||
pub fn input(&self) -> VerdictInput<'_> {
|
||||
VerdictInput {
|
||||
binding: VerdictBinding {
|
||||
sheet_id: self.header.sheet_id.clone(),
|
||||
version_id: self.header.version_id.clone(),
|
||||
revision: self.header.revision,
|
||||
review_id: self.header.review_id.clone(),
|
||||
},
|
||||
document: self.document,
|
||||
base_document: self.base,
|
||||
drawings: self
|
||||
.header
|
||||
.drawings
|
||||
.iter()
|
||||
.zip(&self.drawing_bytes)
|
||||
.map(|(row, bytes)| VerdictDrawing {
|
||||
asset_id: &row.asset_id,
|
||||
sha256: &row.sha256,
|
||||
byte_size: row.byte_size,
|
||||
bytes: *bytes,
|
||||
})
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
#![allow(clippy::unwrap_used)]
|
||||
use super::*;
|
||||
|
||||
fn binding() -> VerdictBinding {
|
||||
VerdictBinding {
|
||||
sheet_id: "s".to_owned(),
|
||||
version_id: "v".to_owned(),
|
||||
revision: 3,
|
||||
review_id: "r".to_owned(),
|
||||
}
|
||||
}
|
||||
|
||||
fn input<'a>(base: Option<&'a str>, drawings: Vec<VerdictDrawing<'a>>) -> VerdictInput<'a> {
|
||||
VerdictInput {
|
||||
binding: binding(),
|
||||
document: "{\"d\":\"é\"}",
|
||||
base_document: base,
|
||||
drawings,
|
||||
}
|
||||
}
|
||||
|
||||
fn drawing(bytes: Option<&[u8]>) -> VerdictDrawing<'_> {
|
||||
VerdictDrawing {
|
||||
asset_id: "a",
|
||||
sha256: "h",
|
||||
byte_size: 3,
|
||||
bytes,
|
||||
}
|
||||
}
|
||||
|
||||
/// Rewrites the header of a valid frame with `edit`, keeping its body.
|
||||
fn with_header(frame: &[u8], edit: impl Fn(&mut serde_json::Value)) -> Vec<u8> {
|
||||
let len = u32::from_le_bytes(frame[..4].try_into().unwrap()) as usize;
|
||||
let mut header: serde_json::Value = serde_json::from_slice(&frame[4..4 + len]).unwrap();
|
||||
edit(&mut header);
|
||||
let json = serde_json::to_vec(&header).unwrap();
|
||||
let mut out = (json.len() as u32).to_le_bytes().to_vec();
|
||||
out.extend_from_slice(&json);
|
||||
out.extend_from_slice(&frame[4 + len..]);
|
||||
out
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_frame_round_trips() {
|
||||
let original = input(
|
||||
Some("{}"),
|
||||
vec![drawing(Some(b"abc")), drawing(None), drawing(Some(b"xyz"))],
|
||||
);
|
||||
let bytes = encode(&original).unwrap();
|
||||
let frame = decode(&bytes).unwrap();
|
||||
assert_eq!(frame.document, original.document);
|
||||
assert_eq!(frame.base, Some("{}"));
|
||||
assert_eq!(
|
||||
frame.drawing_bytes,
|
||||
vec![Some(&b"abc"[..]), None, Some(&b"xyz"[..])]
|
||||
);
|
||||
let read = frame.input();
|
||||
assert_eq!(read.binding, original.binding);
|
||||
assert_eq!(read.drawings.len(), 3);
|
||||
assert_eq!(read.drawings[1].bytes, None);
|
||||
assert_eq!(read.drawings[2].byte_size, 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_absent_and_an_empty_base_differ() {
|
||||
let none = encode(&input(None, vec![])).unwrap();
|
||||
assert_eq!(decode(&none).unwrap().base, None);
|
||||
let empty = encode(&input(Some(""), vec![])).unwrap();
|
||||
assert_eq!(decode(&empty).unwrap().base, Some(""));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn malformed_frames_are_refused() {
|
||||
let good = encode(&input(Some("{}"), vec![drawing(Some(b"abc"))])).unwrap();
|
||||
assert_eq!(decode(&good[..3]).err(), Some(FrameError::Truncated));
|
||||
assert_eq!(decode(&good[..10]).err(), Some(FrameError::Truncated));
|
||||
assert_eq!(
|
||||
decode(&good[..good.len() - 1]).err(),
|
||||
Some(FrameError::LengthMismatch)
|
||||
);
|
||||
let mut trailing = good.clone();
|
||||
trailing.push(0);
|
||||
assert_eq!(decode(&trailing).err(), Some(FrameError::LengthMismatch));
|
||||
let mut huge = ((VERIFY_MAX_HEADER_BYTES + 1) as u32)
|
||||
.to_le_bytes()
|
||||
.to_vec();
|
||||
huge.extend(vec![b' '; VERIFY_MAX_HEADER_BYTES + 1]);
|
||||
assert_eq!(decode(&huge).err(), Some(FrameError::HeaderTooLarge));
|
||||
let mut not_json = 3u32.to_le_bytes().to_vec();
|
||||
not_json.extend_from_slice(b"{x}");
|
||||
assert_eq!(decode(¬_json).err(), Some(FrameError::HeaderInvalid));
|
||||
let unknown = with_header(&good, |h| h["extra"] = 1.into());
|
||||
assert_eq!(decode(&unknown).err(), Some(FrameError::HeaderInvalid));
|
||||
let overflow = with_header(&good, |h| h["document_len"] = usize::MAX.into());
|
||||
assert_eq!(decode(&overflow).err(), Some(FrameError::LengthMismatch));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_document_or_base_that_is_not_utf8_is_refused() {
|
||||
let good = encode(&input(Some("ab"), vec![])).unwrap();
|
||||
let mut bad_base = good.clone();
|
||||
let last = bad_base.len() - 1;
|
||||
bad_base[last] = 0xff;
|
||||
assert_eq!(decode(&bad_base).err(), Some(FrameError::NotUtf8));
|
||||
let mut bad_document = good;
|
||||
let len = u32::from_le_bytes(bad_document[..4].try_into().unwrap()) as usize;
|
||||
bad_document[4 + len] = 0xc3;
|
||||
assert_eq!(decode(&bad_document).err(), Some(FrameError::NotUtf8));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_drawing_bytes_are_a_length_mismatch() {
|
||||
let good = encode(&input(None, vec![drawing(Some(b"abc"))])).unwrap();
|
||||
let missing = &good[..good.len() - 3];
|
||||
assert_eq!(decode(missing).err(), Some(FrameError::LengthMismatch));
|
||||
let without = with_header(&good[..good.len() - 3], |h| {
|
||||
h["drawings"][0]["bytes_len"] = serde_json::Value::Null;
|
||||
});
|
||||
assert_eq!(decode(&without).unwrap().drawing_bytes, vec![None]);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,361 @@
|
||||
//! One sheet verification (V1-2.3, contract W4; V1-2 contract §6.4, §6.5): read the leased
|
||||
//! round's input, download the drawings it references, hand everything framed to the
|
||||
//! sandboxed `verify` child, record its verdict. A missing or refused drawing is passed to
|
||||
//! the core without bytes (a business failure the verdict names); only the transport, the
|
||||
//! child's limits and a broken child are worker failures, each reported with a closed code.
|
||||
|
||||
use std::time::Instant;
|
||||
|
||||
use faunosteo_core::domain::sheet::verdict::{VerdictBinding, VerdictDrawing, VerdictInput};
|
||||
|
||||
use crate::backend::{
|
||||
BackendError, CompleteAnswer, DrawingRow, FailAnswer, VerifyBackend, VerifyInput, VerifyJob,
|
||||
};
|
||||
use crate::limits::{
|
||||
VERIFY_MAX_DRAWING_BYTES, VERIFY_MAX_DRAWINGS, VERIFY_MAX_DRAWINGS_TOTAL_BYTES,
|
||||
};
|
||||
use crate::spawn::{ChildOutcome, ChildRun};
|
||||
use crate::verifier::Verifier;
|
||||
use crate::watchdog::{Watchdog, deadlines};
|
||||
use crate::{keys, log, verify_frame};
|
||||
|
||||
/// The only bucket a sheet drawing is read from (M6 `get_sheet_verification_input`).
|
||||
pub const DRAWING_BUCKET: &str = "anatomical-sheet-assets";
|
||||
|
||||
/// Closed worker codes `fail_sheet_verification` accepts (M6 `private.sheet_worker_codes`).
|
||||
pub mod codes {
|
||||
pub const UNAVAILABLE: &str = "verification.worker.unavailable";
|
||||
pub const TIMEOUT: &str = "verification.worker.timeout";
|
||||
pub const RESOURCE_LIMIT: &str = "verification.worker.resource_limit";
|
||||
pub const VERDICT_REFUSED: &str = "verification.worker.verdict_refused";
|
||||
}
|
||||
|
||||
/// How a verification ended, as the loop sees it.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum VerifyEnd {
|
||||
/// The verdict was recorded `passed`: the round is `open`.
|
||||
Passed,
|
||||
/// The verdict was recorded `failed`: the round is `verification_failed`.
|
||||
Failed,
|
||||
/// The round moved on without us (input, completion or failure answered `stale`).
|
||||
Stale,
|
||||
/// A worker failure was recorded with this closed code.
|
||||
Reported(&'static str),
|
||||
/// The failure could not be recorded either: the lease expires and the round is
|
||||
/// claimed again.
|
||||
LeftForLease,
|
||||
/// The child reported a broken environment: the loop must stop.
|
||||
Stop,
|
||||
}
|
||||
|
||||
/// Runs one claimed round to its end.
|
||||
pub fn process_verification(
|
||||
backend: &dyn VerifyBackend,
|
||||
verifier: &dyn Verifier,
|
||||
watchdog: &Watchdog,
|
||||
job: &VerifyJob,
|
||||
) -> VerifyEnd {
|
||||
let started = Instant::now();
|
||||
let end = run(backend, verifier, watchdog, job);
|
||||
let line = match &end {
|
||||
VerifyEnd::Passed => log::info("verify_passed"),
|
||||
VerifyEnd::Failed => log::info("verify_failed"),
|
||||
VerifyEnd::Stale => log::info("verify_stale"),
|
||||
VerifyEnd::Reported(code) => log::warn("verify_reported").code(code),
|
||||
VerifyEnd::LeftForLease => log::warn("verify_left_for_lease"),
|
||||
VerifyEnd::Stop => log::error("verify_stop"),
|
||||
};
|
||||
line.review(&job.review_id)
|
||||
.duration_ms(started.elapsed().as_millis())
|
||||
.emit();
|
||||
end
|
||||
}
|
||||
|
||||
fn run(
|
||||
backend: &dyn VerifyBackend,
|
||||
verifier: &dyn Verifier,
|
||||
watchdog: &Watchdog,
|
||||
job: &VerifyJob,
|
||||
) -> VerifyEnd {
|
||||
if !keys::is_uuid(&job.review_id) || !keys::is_uuid(&job.lease_token) {
|
||||
log::error("verify_invalid_identity").emit();
|
||||
return VerifyEnd::LeftForLease;
|
||||
}
|
||||
let input = match read_input(backend, watchdog, job) {
|
||||
Ok(Some(input)) => input,
|
||||
Ok(None) => {
|
||||
log::info("verify_stale_input")
|
||||
.review(&job.review_id)
|
||||
.emit();
|
||||
return VerifyEnd::Stale;
|
||||
}
|
||||
Err(error) => return transport_failure(backend, job, "input", &error),
|
||||
};
|
||||
let drawings = match fetch_drawings(backend, watchdog, job, &input.drawings) {
|
||||
Ok(drawings) => drawings,
|
||||
Err(error) => return transport_failure(backend, job, "drawing", &error),
|
||||
};
|
||||
let Ok(frame) = verify_frame::encode(&verdict_input(&input, &drawings)) else {
|
||||
log::error("verify_frame_refused")
|
||||
.review(&job.review_id)
|
||||
.emit();
|
||||
return report(backend, job, codes::RESOURCE_LIMIT);
|
||||
};
|
||||
let child = {
|
||||
let _stage = watchdog.enter("verify", deadlines::VERIFY);
|
||||
classify_child(verifier.verify(&frame), job)
|
||||
};
|
||||
match child {
|
||||
ChildVerdict::Verdict(verdict) => complete(backend, watchdog, job, &verdict),
|
||||
ChildVerdict::Report(code) => report(backend, job, code),
|
||||
ChildVerdict::Stop => {
|
||||
report(backend, job, codes::UNAVAILABLE);
|
||||
VerifyEnd::Stop
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn read_input(
|
||||
backend: &dyn VerifyBackend,
|
||||
watchdog: &Watchdog,
|
||||
job: &VerifyJob,
|
||||
) -> Result<Option<VerifyInput>, BackendError> {
|
||||
let _stage = watchdog.enter("input", deadlines::DOWNLOAD);
|
||||
backend.verification_input(&job.review_id, &job.lease_token)
|
||||
}
|
||||
|
||||
/// The bytes of each row, or `None` for a row passed without bytes. `Err` only on a
|
||||
/// transport failure (the round is reported, not judged).
|
||||
fn fetch_drawings(
|
||||
backend: &dyn VerifyBackend,
|
||||
watchdog: &Watchdog,
|
||||
job: &VerifyJob,
|
||||
rows: &[DrawingRow],
|
||||
) -> Result<Vec<Option<Vec<u8>>>, BackendError> {
|
||||
let mut budget = DownloadBudget::default();
|
||||
let mut drawings = Vec::with_capacity(rows.len());
|
||||
for row in rows {
|
||||
let bytes = match budget.allowance(row) {
|
||||
Some(max_bytes) => download(backend, watchdog, job, row, max_bytes)?,
|
||||
None => {
|
||||
log::info("verify_drawing_skipped")
|
||||
.review(&job.review_id)
|
||||
.asset(&row.asset_id)
|
||||
.emit();
|
||||
None
|
||||
}
|
||||
};
|
||||
if let Some(bytes) = &bytes {
|
||||
budget.spend(bytes.len());
|
||||
}
|
||||
drawings.push(bytes);
|
||||
}
|
||||
Ok(drawings)
|
||||
}
|
||||
|
||||
/// The download policy (`limits::VERIFY_MAX_DRAWING*`): at most so many drawings, each and
|
||||
/// all under their caps. What it leaves out the core reports (missing or caps).
|
||||
#[derive(Debug, Default)]
|
||||
struct DownloadBudget {
|
||||
count: usize,
|
||||
total: usize,
|
||||
}
|
||||
|
||||
impl DownloadBudget {
|
||||
/// The byte cap of this row's download, or `None` when the row is passed without bytes.
|
||||
fn allowance(&self, row: &DrawingRow) -> Option<usize> {
|
||||
let size = usize::try_from(row.byte_size).ok()?;
|
||||
let left = VERIFY_MAX_DRAWINGS_TOTAL_BYTES.saturating_sub(self.total);
|
||||
let fits = size <= VERIFY_MAX_DRAWING_BYTES && size <= left;
|
||||
(is_fetchable(row) && fits && self.count < VERIFY_MAX_DRAWINGS)
|
||||
.then(|| VERIFY_MAX_DRAWING_BYTES.min(left))
|
||||
}
|
||||
|
||||
fn spend(&mut self, bytes: usize) {
|
||||
self.count += 1;
|
||||
self.total += bytes;
|
||||
}
|
||||
}
|
||||
|
||||
/// An uploaded row in the drawings bucket whose key is its own sha256: anything else is
|
||||
/// never fetched (a corrupted row cannot point the worker at another object).
|
||||
fn is_fetchable(row: &DrawingRow) -> bool {
|
||||
row.uploaded
|
||||
&& row.bucket == DRAWING_BUCKET
|
||||
&& keys::is_drawing_key(&row.object_key)
|
||||
&& row.object_key.strip_suffix(".svg") == Some(row.sha256.as_str())
|
||||
}
|
||||
|
||||
fn download(
|
||||
backend: &dyn VerifyBackend,
|
||||
watchdog: &Watchdog,
|
||||
job: &VerifyJob,
|
||||
row: &DrawingRow,
|
||||
max_bytes: usize,
|
||||
) -> Result<Option<Vec<u8>>, BackendError> {
|
||||
let _stage = watchdog.enter("download", deadlines::DOWNLOAD);
|
||||
match backend.download_drawing(&row.bucket, &row.object_key, max_bytes) {
|
||||
Ok(bytes) => Ok(Some(bytes)),
|
||||
Err(BackendError::Status(404)) => {
|
||||
log::warn("verify_drawing_missing")
|
||||
.review(&job.review_id)
|
||||
.asset(&row.asset_id)
|
||||
.emit();
|
||||
Ok(None)
|
||||
}
|
||||
Err(BackendError::Overflow) => {
|
||||
log::warn("verify_drawing_overflow")
|
||||
.review(&job.review_id)
|
||||
.asset(&row.asset_id)
|
||||
.emit();
|
||||
Ok(None)
|
||||
}
|
||||
Err(error) => Err(error),
|
||||
}
|
||||
}
|
||||
|
||||
/// The core's input: the round as the input RPC names it, the texts as they are.
|
||||
fn verdict_input<'a>(input: &'a VerifyInput, drawings: &'a [Option<Vec<u8>>]) -> VerdictInput<'a> {
|
||||
VerdictInput {
|
||||
binding: VerdictBinding {
|
||||
sheet_id: input.sheet_id.clone(),
|
||||
version_id: input.version_id.clone(),
|
||||
revision: input.revision,
|
||||
review_id: input.review_id.clone(),
|
||||
},
|
||||
document: &input.document,
|
||||
base_document: input.base_document.as_deref(),
|
||||
drawings: input
|
||||
.drawings
|
||||
.iter()
|
||||
.zip(drawings)
|
||||
.map(|(row, bytes)| VerdictDrawing {
|
||||
asset_id: &row.asset_id,
|
||||
sha256: &row.sha256,
|
||||
byte_size: row.byte_size,
|
||||
bytes: bytes.as_deref(),
|
||||
})
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
|
||||
/// What the child's end means for the round.
|
||||
enum ChildVerdict {
|
||||
Verdict(Vec<u8>),
|
||||
Report(&'static str),
|
||||
Stop,
|
||||
}
|
||||
|
||||
/// Maps every way the child can end to a verdict on the round (§6.5 rows).
|
||||
fn classify_child(run: ChildRun, job: &VerifyJob) -> ChildVerdict {
|
||||
let stderr = run.stderr;
|
||||
let review = job.review_id.as_str();
|
||||
match run.outcome {
|
||||
ChildOutcome::Produced(verdict) => ChildVerdict::Verdict(verdict),
|
||||
ChildOutcome::Timeout => {
|
||||
log::warn("child_timeout").review(review).emit();
|
||||
ChildVerdict::Report(codes::TIMEOUT)
|
||||
}
|
||||
ChildOutcome::Signalled { signal } => {
|
||||
log::warn("child_signalled")
|
||||
.review(review)
|
||||
.signal(signal)
|
||||
.child_stderr(&stderr)
|
||||
.emit();
|
||||
ChildVerdict::Report(codes::RESOURCE_LIMIT)
|
||||
}
|
||||
ChildOutcome::OutputOverflow => {
|
||||
log::warn("child_output_overflow").review(review).emit();
|
||||
ChildVerdict::Report(codes::RESOURCE_LIMIT)
|
||||
}
|
||||
ChildOutcome::Rejected { name, .. } => {
|
||||
log::warn("child_rejected")
|
||||
.review(review)
|
||||
.code(name)
|
||||
.child_stderr(&stderr)
|
||||
.emit();
|
||||
ChildVerdict::Report(codes::RESOURCE_LIMIT)
|
||||
}
|
||||
ChildOutcome::Broken { name, .. } => {
|
||||
log::error("child_broken")
|
||||
.review(review)
|
||||
.code(name)
|
||||
.child_stderr(&stderr)
|
||||
.emit();
|
||||
ChildVerdict::Stop
|
||||
}
|
||||
ChildOutcome::Failed(detail) => {
|
||||
log::error("child_failed")
|
||||
.review(review)
|
||||
.detail(&detail)
|
||||
.emit();
|
||||
ChildVerdict::Stop
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn complete(
|
||||
backend: &dyn VerifyBackend,
|
||||
watchdog: &Watchdog,
|
||||
job: &VerifyJob,
|
||||
verdict: &[u8],
|
||||
) -> VerifyEnd {
|
||||
let answer = {
|
||||
let _stage = watchdog.enter("commit", deadlines::COMMIT);
|
||||
backend.complete_verification(&job.review_id, &job.lease_token, verdict)
|
||||
};
|
||||
match answer {
|
||||
Ok(CompleteAnswer::Passed) => VerifyEnd::Passed,
|
||||
Ok(CompleteAnswer::Failed) => VerifyEnd::Failed,
|
||||
Ok(CompleteAnswer::Stale) => {
|
||||
log::info("complete_stale").review(&job.review_id).emit();
|
||||
VerifyEnd::Stale
|
||||
}
|
||||
Ok(CompleteAnswer::Malformed) => {
|
||||
log::error("complete_malformed")
|
||||
.review(&job.review_id)
|
||||
.emit();
|
||||
report(backend, job, codes::VERDICT_REFUSED)
|
||||
}
|
||||
Err(error) => transport_failure(backend, job, "commit", &error),
|
||||
}
|
||||
}
|
||||
|
||||
/// Records a worker failure with a closed code; `stale` means the round moved on.
|
||||
fn report(backend: &dyn VerifyBackend, job: &VerifyJob, code: &'static str) -> VerifyEnd {
|
||||
match backend.fail_verification(&job.review_id, &job.lease_token, code) {
|
||||
Ok(FailAnswer::Retry | FailAnswer::Unavailable) => VerifyEnd::Reported(code),
|
||||
Ok(FailAnswer::Stale) => {
|
||||
log::info("fail_stale").review(&job.review_id).emit();
|
||||
VerifyEnd::Stale
|
||||
}
|
||||
Err(error) => {
|
||||
log_backend_failure("fail", job, &error);
|
||||
VerifyEnd::LeftForLease
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A transport failure (§6.5 transient row): reported `unavailable`.
|
||||
fn transport_failure(
|
||||
backend: &dyn VerifyBackend,
|
||||
job: &VerifyJob,
|
||||
stage: &str,
|
||||
error: &BackendError,
|
||||
) -> VerifyEnd {
|
||||
log_backend_failure(stage, job, error);
|
||||
report(backend, job, codes::UNAVAILABLE)
|
||||
}
|
||||
|
||||
fn log_backend_failure(stage: &str, job: &VerifyJob, error: &BackendError) {
|
||||
let line = log::warn("backend_failure")
|
||||
.review(&job.review_id)
|
||||
.stage(stage)
|
||||
.code(error.kind());
|
||||
// The library's error text stays out of the logs: the closed kind and the status are
|
||||
// the record (as in `job`).
|
||||
match error {
|
||||
BackendError::Status(status) => line.http_status(*status).emit(),
|
||||
_ => line.emit(),
|
||||
}
|
||||
}
|
||||
@@ -13,6 +13,8 @@ pub mod deadlines {
|
||||
pub const CLAIM: Duration = Duration::from_secs(35);
|
||||
pub const DOWNLOAD: Duration = Duration::from_secs(35);
|
||||
pub const DECODE: Duration = Duration::from_secs(25);
|
||||
/// Above the verify child's own deadline (`limits::VERIFY_DEADLINE`, 90 s).
|
||||
pub const VERIFY: Duration = Duration::from_secs(95);
|
||||
pub const UPLOAD: Duration = Duration::from_secs(35);
|
||||
pub const COMMIT: Duration = Duration::from_secs(35);
|
||||
pub const SWEEP: Duration = Duration::from_secs(120);
|
||||
|
||||
@@ -210,3 +210,103 @@ fn the_cpu_limit_stops_a_spinning_child_before_the_deadline() {
|
||||
fn the_linked_libwebp_is_recent_enough() {
|
||||
assert!(faunosteo_worker::decode::libwebp_version() >= limits::MIN_LIBWEBP_VERSION);
|
||||
}
|
||||
|
||||
// V1-2.3: the `verify` child under the same sandbox (contract W2, W3).
|
||||
|
||||
fn verifier(flags: &[&str]) -> Spawner {
|
||||
let mut spawner = Spawner::for_verify(env!("CARGO_BIN_EXE_faunosteo-worker"));
|
||||
spawner.extra_args = flags.iter().map(|flag| (*flag).to_owned()).collect();
|
||||
spawner
|
||||
}
|
||||
|
||||
fn expect_verdict(frame: &[u8]) -> serde_json::Value {
|
||||
let run = verifier(&[]).run(frame);
|
||||
match run.outcome {
|
||||
ChildOutcome::Produced(out) => {
|
||||
assert!(out.len() < limits::VERIFY_MAX_OUTPUT_BYTES);
|
||||
serde_json::from_slice(&out).unwrap()
|
||||
}
|
||||
other => panic!("{other:?} (stderr: {})", run.stderr),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_clean_sheet_is_verified_passed_through_the_full_sandbox() {
|
||||
let verdict = expect_verdict(&fixtures::sheet_frame(&fixtures::sheet_document()));
|
||||
assert_eq!(verdict["outcome"], "passed", "{verdict}");
|
||||
assert_eq!(verdict["review_id"], fixtures::sheet_binding().review_id);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_sheet_at_every_cap_is_verified_within_the_limits() {
|
||||
let frame = fixtures::sheet_frame(&fixtures::sheet_document_at_caps());
|
||||
let started = std::time::Instant::now();
|
||||
let verdict = expect_verdict(&frame);
|
||||
let elapsed = started.elapsed();
|
||||
assert_eq!(verdict["outcome"], "passed", "{}", verdict["failures"]);
|
||||
assert!(elapsed < limits::VERIFY_DEADLINE, "{elapsed:?}");
|
||||
eprintln!(
|
||||
"verify at caps: input {} bytes, verdict {} bytes, {elapsed:?}",
|
||||
frame.len(),
|
||||
verdict.to_string().len()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_verify_child_refuses_a_non_empty_environment() {
|
||||
let status = Command::new(env!("CARGO_BIN_EXE_faunosteo-worker"))
|
||||
.arg("verify")
|
||||
.env_clear()
|
||||
.env("LEAKED", "1")
|
||||
.stdin(Stdio::null())
|
||||
.stdout(Stdio::null())
|
||||
.stderr(Stdio::null())
|
||||
.status()
|
||||
.unwrap();
|
||||
assert_eq!(status.code(), Some(exit_codes::ENV_NOT_EMPTY));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_input_over_the_verify_cap_is_refused_unread() {
|
||||
let run = verifier(&[]).run(&vec![0u8; limits::VERIFY_MAX_INPUT_BYTES + 1]);
|
||||
assert!(
|
||||
matches!(run.outcome, ChildOutcome::Rejected { code, .. } if code == exit_codes::INPUT_TOO_LARGE),
|
||||
"{:?} {}",
|
||||
run.outcome,
|
||||
run.stderr
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_malformed_frame_is_a_broken_parent_not_content() {
|
||||
let run = verifier(&[]).run(b"\x03\0\0\0{x}");
|
||||
assert!(
|
||||
matches!(run.outcome, ChildOutcome::Broken { code, .. } if code == exit_codes::FRAME_INVALID),
|
||||
"{:?} {}",
|
||||
run.outcome,
|
||||
run.stderr
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_denied_syscall_kills_the_verify_child_with_sigsys() {
|
||||
let run = verifier(&["--violate"]).run(b"");
|
||||
assert!(
|
||||
matches!(run.outcome, ChildOutcome::Signalled { signal: SIGSYS }),
|
||||
"{:?} {}",
|
||||
run.outcome,
|
||||
run.stderr
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_verify_output_over_its_cap_is_cut() {
|
||||
let mut spawner = verifier(&["--flood"]);
|
||||
spawner.max_output = 64 * 1024;
|
||||
let run = spawner.run(b"");
|
||||
assert!(
|
||||
matches!(run.outcome, ChildOutcome::OutputOverflow),
|
||||
"{:?}",
|
||||
run.outcome
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,527 @@
|
||||
//! V1-2.3 proofs of the verification job (contract W4, V1-2 contract §6.5) on an in-memory
|
||||
//! `VerifyBackend`: the core's verdict computed in process from the very frame the parent
|
||||
//! builds (or the real child without seccomp under `test-hooks`), every child outcome mapped
|
||||
//! to its closed worker code, a missing drawing judged by the core, never by the worker.
|
||||
#![allow(clippy::unwrap_used, clippy::expect_used)]
|
||||
|
||||
use std::cell::RefCell;
|
||||
use std::collections::VecDeque;
|
||||
|
||||
use faunosteo_core::domain::sheet::verdict::{codes as verdict_codes, sha256_hex};
|
||||
use faunosteo_worker::backend::{
|
||||
BackendError, BackendResult, CompleteAnswer, DrawingRow, FailAnswer, VerifyBackend,
|
||||
VerifyInput, VerifyJob,
|
||||
};
|
||||
use faunosteo_worker::config::Config;
|
||||
use faunosteo_worker::limits::VERIFY_MAX_DRAWING_BYTES;
|
||||
use faunosteo_worker::run::{Mode, parse_args, verify_loop};
|
||||
use faunosteo_worker::spawn::{ChildOutcome, ChildRun};
|
||||
use faunosteo_worker::verifier::Verifier;
|
||||
use faunosteo_worker::verify_job::{DRAWING_BUCKET, VerifyEnd, codes, process_verification};
|
||||
use faunosteo_worker::watchdog::Watchdog;
|
||||
use faunosteo_worker::{exit_codes, verify};
|
||||
use serde_json::{Value, json};
|
||||
|
||||
const REVIEW: &str = "aaaaaaaa-0000-4000-8000-000000000003";
|
||||
const VERSION: &str = "aaaaaaaa-0000-4000-8000-000000000002";
|
||||
const SHEET: &str = "aaaaaaaa-0000-4000-8000-000000000001";
|
||||
const TOKEN: &str = "30000000-0000-0000-0000-00000000000c";
|
||||
const ASSET: &str = "11111111-2222-4333-8444-555555555555";
|
||||
const DRAWING: &str = concat!(
|
||||
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n",
|
||||
r#"<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 10 10"><path d="M0 0L5 5"/></svg>"#
|
||||
);
|
||||
|
||||
fn name(text: &str) -> Value {
|
||||
json!({ "fr": text, "en": text })
|
||||
}
|
||||
|
||||
/// The core's small valid sheet (`crates/core/tests/sheet_verdict.rs::document`).
|
||||
fn document() -> Value {
|
||||
json!({
|
||||
"schema_version": 2, "species_id": "canine", "name": name("Chien"),
|
||||
"description": {}, "scope_disclaimer": { "fr": "Outil", "en": "Tool" },
|
||||
"attribution": { "authors": ["A"], "license": null, "notes": null },
|
||||
"vertebral_config": { "regions": [] }, "references": [],
|
||||
"categories": [{ "id": "c", "name": name("C"), "color": null }],
|
||||
"views": [{ "id": "main", "kind": "main", "name": name("Vue"), "svg_asset_id": ASSET, "view_box": null }],
|
||||
"annotations": [{ "id": "a1", "kind": "point", "view_id": "main", "category_id": "c",
|
||||
"laterality": "unpaired", "geometry": { "type": "circle", "cx": 1, "cy": 1, "r": 1 },
|
||||
"review_status": "to_confirm", "name": name("P"), "finding_zone_id": "z1", "vertebral_region_code": null }],
|
||||
"finding_zones": [{ "id": "z1", "name": name("Z"), "categories": [{ "id": "cat", "kind": "dysfunction",
|
||||
"name": name("D"), "review_status": "to_confirm",
|
||||
"children": [{ "id": "n1", "name": name("N"), "review_status": "to_confirm" }] }] }]
|
||||
})
|
||||
}
|
||||
|
||||
fn job() -> VerifyJob {
|
||||
VerifyJob {
|
||||
review_id: REVIEW.to_owned(),
|
||||
version_id: VERSION.to_owned(),
|
||||
sheet_id: SHEET.to_owned(),
|
||||
revision: 3,
|
||||
lease_token: TOKEN.to_owned(),
|
||||
attempts: 1,
|
||||
}
|
||||
}
|
||||
|
||||
fn row() -> DrawingRow {
|
||||
let sha256 = sha256_hex(DRAWING.as_bytes());
|
||||
DrawingRow {
|
||||
asset_id: ASSET.to_owned(),
|
||||
object_key: format!("{sha256}.svg"),
|
||||
sha256,
|
||||
byte_size: DRAWING.len() as u64,
|
||||
bucket: DRAWING_BUCKET.to_owned(),
|
||||
uploaded: true,
|
||||
}
|
||||
}
|
||||
|
||||
fn input(document: String, drawings: Vec<DrawingRow>) -> VerifyInput {
|
||||
VerifyInput {
|
||||
review_id: REVIEW.to_owned(),
|
||||
version_id: VERSION.to_owned(),
|
||||
sheet_id: SHEET.to_owned(),
|
||||
revision: 3,
|
||||
document,
|
||||
base_document: None,
|
||||
drawings,
|
||||
}
|
||||
}
|
||||
|
||||
struct Fake {
|
||||
calls: RefCell<Vec<String>>,
|
||||
input: RefCell<Option<BackendResult<Option<VerifyInput>>>>,
|
||||
downloads: RefCell<VecDeque<BackendResult<Vec<u8>>>>,
|
||||
complete: RefCell<Option<BackendResult<CompleteAnswer>>>,
|
||||
fail: RefCell<Option<BackendResult<FailAnswer>>>,
|
||||
verdict: RefCell<Option<Value>>,
|
||||
jobs: RefCell<Vec<VerifyJob>>,
|
||||
}
|
||||
|
||||
impl Fake {
|
||||
fn with_input(input: VerifyInput) -> Self {
|
||||
Self {
|
||||
calls: RefCell::default(),
|
||||
input: RefCell::new(Some(Ok(Some(input)))),
|
||||
downloads: RefCell::default(),
|
||||
complete: RefCell::default(),
|
||||
fail: RefCell::default(),
|
||||
verdict: RefCell::default(),
|
||||
jobs: RefCell::default(),
|
||||
}
|
||||
}
|
||||
|
||||
fn clean() -> Self {
|
||||
Self::with_input(input(document().to_string(), vec![row()]))
|
||||
}
|
||||
|
||||
fn calls(&self) -> Vec<String> {
|
||||
self.calls.borrow().clone()
|
||||
}
|
||||
|
||||
fn names(&self) -> Vec<String> {
|
||||
self.calls()
|
||||
.iter()
|
||||
.map(|c| c.split(' ').next().unwrap().to_owned())
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn verdict(&self) -> Value {
|
||||
self.verdict.borrow().clone().expect("a completion")
|
||||
}
|
||||
}
|
||||
|
||||
impl VerifyBackend for Fake {
|
||||
fn claim_verifications(&self, max_jobs: u32) -> BackendResult<Vec<VerifyJob>> {
|
||||
self.calls.borrow_mut().push(format!("claim {max_jobs}"));
|
||||
Ok(self.jobs.borrow_mut().drain(..).collect())
|
||||
}
|
||||
|
||||
fn verification_input(
|
||||
&self,
|
||||
review_id: &str,
|
||||
lease_token: &str,
|
||||
) -> BackendResult<Option<VerifyInput>> {
|
||||
self.calls
|
||||
.borrow_mut()
|
||||
.push(format!("input {review_id} {lease_token}"));
|
||||
self.input.borrow_mut().take().unwrap_or(Ok(None))
|
||||
}
|
||||
|
||||
fn download_drawing(
|
||||
&self,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
max_bytes: usize,
|
||||
) -> BackendResult<Vec<u8>> {
|
||||
self.calls
|
||||
.borrow_mut()
|
||||
.push(format!("download {bucket}/{key} {max_bytes}"));
|
||||
self.downloads
|
||||
.borrow_mut()
|
||||
.pop_front()
|
||||
.unwrap_or_else(|| Ok(DRAWING.as_bytes().to_vec()))
|
||||
}
|
||||
|
||||
fn complete_verification(
|
||||
&self,
|
||||
review_id: &str,
|
||||
lease_token: &str,
|
||||
verdict: &[u8],
|
||||
) -> BackendResult<CompleteAnswer> {
|
||||
self.calls
|
||||
.borrow_mut()
|
||||
.push(format!("complete {review_id} {lease_token}"));
|
||||
let value: Value = serde_json::from_slice(verdict).unwrap();
|
||||
let outcome = value["outcome"].clone();
|
||||
*self.verdict.borrow_mut() = Some(value);
|
||||
self.complete
|
||||
.borrow_mut()
|
||||
.take()
|
||||
.unwrap_or(Ok(if outcome == "passed" {
|
||||
CompleteAnswer::Passed
|
||||
} else {
|
||||
CompleteAnswer::Failed
|
||||
}))
|
||||
}
|
||||
|
||||
fn fail_verification(
|
||||
&self,
|
||||
review_id: &str,
|
||||
lease_token: &str,
|
||||
code: &str,
|
||||
) -> BackendResult<FailAnswer> {
|
||||
self.calls
|
||||
.borrow_mut()
|
||||
.push(format!("fail {review_id} {lease_token} {code}"));
|
||||
self.fail
|
||||
.borrow_mut()
|
||||
.take()
|
||||
.unwrap_or(Ok(FailAnswer::Retry))
|
||||
}
|
||||
}
|
||||
|
||||
/// The child's work in process: the same frame, `verify::process`, the core's verdict.
|
||||
struct InProcess;
|
||||
|
||||
impl Verifier for InProcess {
|
||||
fn verify(&self, frame: &[u8]) -> ChildRun {
|
||||
let outcome = match verify::process(frame) {
|
||||
Ok(verdict) => ChildOutcome::Produced(verdict),
|
||||
Err(failure) => ChildOutcome::Broken {
|
||||
code: failure.exit_code(),
|
||||
name: exit_codes::name(failure.exit_code()),
|
||||
},
|
||||
};
|
||||
ChildRun {
|
||||
outcome,
|
||||
stderr: String::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A child that ends as told, at most once.
|
||||
struct Fixed(RefCell<Option<ChildOutcome>>);
|
||||
|
||||
impl Fixed {
|
||||
fn with(outcome: ChildOutcome) -> Self {
|
||||
Self(RefCell::new(Some(outcome)))
|
||||
}
|
||||
|
||||
fn never() -> Self {
|
||||
Self(RefCell::new(None))
|
||||
}
|
||||
}
|
||||
|
||||
impl Verifier for Fixed {
|
||||
fn verify(&self, _frame: &[u8]) -> ChildRun {
|
||||
ChildRun {
|
||||
outcome: self.0.borrow_mut().take().expect("no child run expected"),
|
||||
stderr: "child says\n\u{1b}[0m".to_owned(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn process(fake: &Fake, verifier: &dyn Verifier) -> VerifyEnd {
|
||||
process_verification(fake, verifier, &Watchdog::new(), &job())
|
||||
}
|
||||
|
||||
fn fail_call(code: &str) -> String {
|
||||
format!("fail {REVIEW} {TOKEN} {code}")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_clean_sheet_follows_input_download_child_complete() {
|
||||
let fake = Fake::clean();
|
||||
assert_eq!(process(&fake, &InProcess), VerifyEnd::Passed);
|
||||
let key = row().object_key;
|
||||
assert_eq!(
|
||||
fake.calls(),
|
||||
vec![
|
||||
format!("input {REVIEW} {TOKEN}"),
|
||||
format!("download {DRAWING_BUCKET}/{key} {VERIFY_MAX_DRAWING_BYTES}"),
|
||||
format!("complete {REVIEW} {TOKEN}"),
|
||||
]
|
||||
);
|
||||
let verdict = fake.verdict();
|
||||
assert_eq!(verdict["outcome"], "passed");
|
||||
assert_eq!(
|
||||
(
|
||||
&verdict["review_id"],
|
||||
&verdict["version_id"],
|
||||
&verdict["sheet_id"],
|
||||
&verdict["revision"]
|
||||
),
|
||||
(&json!(REVIEW), &json!(VERSION), &json!(SHEET), &json!(3))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_unreadable_document_completes_failed() {
|
||||
let fake = Fake::with_input(input("not json".to_owned(), vec![row()]));
|
||||
assert_eq!(process(&fake, &InProcess), VerifyEnd::Failed);
|
||||
assert_eq!(fake.names(), ["input", "download", "complete"]);
|
||||
assert_eq!(fake.verdict()["failure_code"], verdict_codes::DOCUMENT);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_stale_input_ends_without_any_other_call() {
|
||||
let fake = Fake::clean();
|
||||
*fake.input.borrow_mut() = Some(Ok(None));
|
||||
assert_eq!(process(&fake, &Fixed::never()), VerifyEnd::Stale);
|
||||
assert_eq!(fake.names(), ["input"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_stale_completion_ends_and_a_malformed_one_is_reported() {
|
||||
let fake = Fake::clean();
|
||||
*fake.complete.borrow_mut() = Some(Ok(CompleteAnswer::Stale));
|
||||
assert_eq!(process(&fake, &InProcess), VerifyEnd::Stale);
|
||||
assert_eq!(fake.names(), ["input", "download", "complete"]);
|
||||
|
||||
let fake = Fake::clean();
|
||||
*fake.complete.borrow_mut() = Some(Ok(CompleteAnswer::Malformed));
|
||||
assert_eq!(
|
||||
process(&fake, &InProcess),
|
||||
VerifyEnd::Reported(codes::VERDICT_REFUSED)
|
||||
);
|
||||
assert_eq!(
|
||||
fake.calls().last().unwrap(),
|
||||
&fail_call(codes::VERDICT_REFUSED)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_missing_or_oversized_drawing_is_judged_by_the_core() {
|
||||
for error in [BackendError::Status(404), BackendError::Overflow] {
|
||||
let fake = Fake::clean();
|
||||
fake.downloads.borrow_mut().push_back(Err(error));
|
||||
assert_eq!(process(&fake, &InProcess), VerifyEnd::Failed);
|
||||
assert_eq!(fake.names(), ["input", "download", "complete"]);
|
||||
let verdict = fake.verdict();
|
||||
assert_eq!(verdict["failure_code"], verdict_codes::DRAWING_MISSING);
|
||||
assert_eq!(verdict["drawings"][0]["result"], "missing");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rows_outside_the_policy_are_never_fetched() {
|
||||
let mut not_uploaded = row();
|
||||
not_uploaded.uploaded = false;
|
||||
let mut other_bucket = row();
|
||||
other_bucket.bucket = "avatars".to_owned();
|
||||
let mut other_key = row();
|
||||
other_key.object_key = format!("{}.svg", "0".repeat(64));
|
||||
let mut hostile_key = row();
|
||||
hostile_key.object_key = "../avatars/x.svg".to_owned();
|
||||
let mut too_big = row();
|
||||
too_big.byte_size = VERIFY_MAX_DRAWING_BYTES as u64 + 1;
|
||||
for skipped in [not_uploaded, other_bucket, other_key, hostile_key, too_big] {
|
||||
let fake = Fake::with_input(input(document().to_string(), vec![skipped]));
|
||||
assert_eq!(process(&fake, &InProcess), VerifyEnd::Failed);
|
||||
assert_eq!(fake.names(), ["input", "complete"]);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn downloads_stop_at_the_drawing_count() {
|
||||
let rows = vec![row(); 6];
|
||||
let fake = Fake::with_input(input(document().to_string(), rows));
|
||||
process(&fake, &InProcess);
|
||||
let downloads = fake.names().iter().filter(|n| *n == "download").count();
|
||||
assert_eq!(downloads, 4);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_transport_failure_is_reported_unavailable_without_a_child() {
|
||||
let fake = Fake::clean();
|
||||
fake.downloads
|
||||
.borrow_mut()
|
||||
.push_back(Err(BackendError::Network("refused".to_owned())));
|
||||
assert_eq!(
|
||||
process(&fake, &Fixed::never()),
|
||||
VerifyEnd::Reported(codes::UNAVAILABLE)
|
||||
);
|
||||
assert_eq!(fake.names(), ["input", "download", "fail"]);
|
||||
assert_eq!(fake.calls()[2], fail_call(codes::UNAVAILABLE));
|
||||
|
||||
let fake = Fake::clean();
|
||||
*fake.input.borrow_mut() = Some(Err(BackendError::Status(503)));
|
||||
assert_eq!(
|
||||
process(&fake, &Fixed::never()),
|
||||
VerifyEnd::Reported(codes::UNAVAILABLE)
|
||||
);
|
||||
assert_eq!(fake.names(), ["input", "fail"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_unrecordable_failure_leaves_the_lease() {
|
||||
let fake = Fake::clean();
|
||||
*fake.input.borrow_mut() = Some(Err(BackendError::Status(503)));
|
||||
*fake.fail.borrow_mut() = Some(Err(BackendError::Status(503)));
|
||||
assert_eq!(process(&fake, &Fixed::never()), VerifyEnd::LeftForLease);
|
||||
|
||||
let fake = Fake::clean();
|
||||
*fake.fail.borrow_mut() = Some(Ok(FailAnswer::Stale));
|
||||
assert_eq!(
|
||||
process(&fake, &Fixed::with(ChildOutcome::Timeout)),
|
||||
VerifyEnd::Stale
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_child_limit_maps_to_its_closed_code() {
|
||||
for (outcome, code) in [
|
||||
(ChildOutcome::Timeout, codes::TIMEOUT),
|
||||
(ChildOutcome::OutputOverflow, codes::RESOURCE_LIMIT),
|
||||
(
|
||||
ChildOutcome::Signalled { signal: 24 },
|
||||
codes::RESOURCE_LIMIT,
|
||||
),
|
||||
(ChildOutcome::Signalled { signal: 9 }, codes::RESOURCE_LIMIT),
|
||||
(ChildOutcome::Signalled { signal: 6 }, codes::RESOURCE_LIMIT),
|
||||
(
|
||||
ChildOutcome::Signalled { signal: 31 },
|
||||
codes::RESOURCE_LIMIT,
|
||||
),
|
||||
(
|
||||
ChildOutcome::Rejected {
|
||||
code: exit_codes::INPUT_TOO_LARGE,
|
||||
name: "input_too_large",
|
||||
},
|
||||
codes::RESOURCE_LIMIT,
|
||||
),
|
||||
] {
|
||||
let fake = Fake::clean();
|
||||
assert_eq!(
|
||||
process(&fake, &Fixed::with(outcome)),
|
||||
VerifyEnd::Reported(code)
|
||||
);
|
||||
assert_eq!(fake.names(), ["input", "download", "fail"]);
|
||||
assert_eq!(fake.calls()[2], fail_call(code));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_broken_or_failed_child_is_reported_then_stops_the_worker() {
|
||||
for outcome in [
|
||||
ChildOutcome::Broken {
|
||||
code: exit_codes::SANDBOX_FAILED,
|
||||
name: "sandbox_failed",
|
||||
},
|
||||
ChildOutcome::Failed("output refused: not a JSON object".to_owned()),
|
||||
] {
|
||||
let fake = Fake::clean();
|
||||
assert_eq!(process(&fake, &Fixed::with(outcome)), VerifyEnd::Stop);
|
||||
assert_eq!(fake.calls().last().unwrap(), &fail_call(codes::UNAVAILABLE));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_round_with_a_hostile_identity_is_left_untouched() {
|
||||
let fake = Fake::clean();
|
||||
let mut bad = job();
|
||||
bad.lease_token = "x".to_owned();
|
||||
assert_eq!(
|
||||
process_verification(&fake, &Fixed::never(), &Watchdog::new(), &bad),
|
||||
VerifyEnd::LeftForLease
|
||||
);
|
||||
assert!(fake.calls().is_empty());
|
||||
}
|
||||
|
||||
fn once_config() -> Config {
|
||||
Config::from_vars(|name| match name {
|
||||
"SUPABASE_URL" => Some("http://kong:8000".to_owned()),
|
||||
"SUPABASE_SERVICE_ROLE_KEY" => Some("k".to_owned()),
|
||||
"WORKER_ONCE" => Some("1".to_owned()),
|
||||
_ => None,
|
||||
})
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn once_mode_claims_one_round_without_a_sweep_or_stops_on_a_broken_child() {
|
||||
let fake = Fake::clean();
|
||||
*fake.jobs.borrow_mut() = vec![job()];
|
||||
assert_eq!(
|
||||
verify_loop(&once_config(), &fake, &InProcess, &Watchdog::new()),
|
||||
exit_codes::OK
|
||||
);
|
||||
assert_eq!(fake.names(), ["claim", "input", "download", "complete"]);
|
||||
|
||||
let fake = Fake::clean();
|
||||
*fake.jobs.borrow_mut() = vec![job()];
|
||||
let broken = Fixed::with(ChildOutcome::Broken {
|
||||
code: exit_codes::ENV_NOT_EMPTY,
|
||||
name: "env_not_empty",
|
||||
});
|
||||
assert_eq!(
|
||||
verify_loop(&once_config(), &fake, &broken, &Watchdog::new()),
|
||||
exit_codes::CHILD_BROKEN
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn run_arguments_select_the_mode() {
|
||||
let args = |list: &[&str]| -> Vec<String> { list.iter().map(|a| (*a).to_owned()).collect() };
|
||||
assert_eq!(parse_args(&args(&[])), Ok((Mode::Avatars, false)));
|
||||
assert_eq!(parse_args(&args(&["--check"])), Ok((Mode::Avatars, true)));
|
||||
assert_eq!(
|
||||
parse_args(&args(&["--jobs", "sheet-verify"])),
|
||||
Ok((Mode::SheetVerify, false))
|
||||
);
|
||||
assert_eq!(
|
||||
parse_args(&args(&["--jobs", "sheet-verify", "--check"])),
|
||||
Ok((Mode::SheetVerify, true))
|
||||
);
|
||||
assert_eq!(
|
||||
parse_args(&args(&["--check", "--jobs", "sheet-verify"])),
|
||||
Ok((Mode::SheetVerify, true))
|
||||
);
|
||||
assert!(parse_args(&args(&["--jobs", "avatars"])).is_err());
|
||||
assert!(parse_args(&args(&["--jobs"])).is_err());
|
||||
assert!(parse_args(&args(&["--check", "--check"])).is_err());
|
||||
assert!(parse_args(&args(&["--once"])).is_err());
|
||||
}
|
||||
|
||||
/// The real child (this crate's binary), rlimits on, seccomp off: the parent's frame, the
|
||||
/// child's reading and the spawner's classification end to end on any host (the filter
|
||||
/// itself is proven in the Alpine stage, tests/sandbox.rs).
|
||||
#[cfg(feature = "test-hooks")]
|
||||
#[test]
|
||||
fn the_real_child_without_seccomp_verifies_the_parents_frame() {
|
||||
use faunosteo_worker::spawn::Spawner;
|
||||
|
||||
struct RealChild(Spawner);
|
||||
impl Verifier for RealChild {
|
||||
fn verify(&self, frame: &[u8]) -> ChildRun {
|
||||
self.0.run(frame)
|
||||
}
|
||||
}
|
||||
let mut spawner = Spawner::for_verify(env!("CARGO_BIN_EXE_faunosteo-worker"));
|
||||
spawner.extra_args = vec!["--no-seccomp".to_owned()];
|
||||
let fake = Fake::clean();
|
||||
assert_eq!(process(&fake, &RealChild(spawner)), VerifyEnd::Passed);
|
||||
assert_eq!(fake.verdict()["outcome"], "passed");
|
||||
}
|
||||
@@ -40,6 +40,28 @@ services:
|
||||
- supabase
|
||||
restart: unless-stopped
|
||||
|
||||
# Sheet verification worker (V1-2.3, V1-2 contract §6.4): the same image and hardening as
|
||||
# `worker`, the verification loop only (no avatar, no sweep). Its `verify` child is capped
|
||||
# at 1 GiB of address space (crates/worker/src/limits.rs). Started only with the `worker`
|
||||
# profile: `docker compose --profile worker up -d worker-verify`; never on DEV.
|
||||
worker-verify:
|
||||
container_name: faunosteo_worker_verify
|
||||
profiles: ["worker"]
|
||||
image: faunosteo_worker:local
|
||||
env_file:
|
||||
- .env.worker
|
||||
command: ["run", "--jobs", "sheet-verify"]
|
||||
read_only: true
|
||||
cap_drop: [ALL]
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
mem_limit: 512m
|
||||
cpus: 1
|
||||
pids_limit: 64
|
||||
networks:
|
||||
- supabase
|
||||
restart: unless-stopped
|
||||
|
||||
# DEV gateway (docs/development/ENVIRONMENTS.md §3): Mailpit behind basic auth on
|
||||
# <DEV host>:54325, the only LAN entry to the DEV mails (the CLI's raw Mailpit port
|
||||
# 54324 is closed to the LAN by the host firewall rule). Started by `pnpm dev:gateway`
|
||||
|
||||
Reference in new issue
Block a user