Files
faunosteo/crates/worker/tests/job_loop.rs
T
bchanot 0410af0c35 feat(worker): R1 job loop — claim, download, sandboxed decode, clean upload, commit, purge sweeps
Keyed parent on ureq 3 (rustls + ring, bundled roots), no async runtime:
config from the environment (secret never printed, JWT expiry logged), JSON
line logs with a field allowlist (child stderr escaped and truncated), strict
key validation before any URL, Backend trait over PostgREST RPCs and Storage
objects with closed error kinds, one job per claim with the STORAGE.md §3.4
mapping (content codes, processing_error on signals / timeout / overflow,
Broken stops the loop with exit 20, a refused commit deletes nothing), sweeper
(expire pending, purge queue, quarantine leftovers), per-stage watchdog
(exit 21), WORKER_ONCE for tests, run --check extended. 11 loop tests on a
fake backend and decoder; sandbox tests and seccomp derivation unchanged
(the child still makes the same 7 syscalls); image 11.6 MB.
2026-10-13 19:32:38 +02:00

462 lines
14 KiB
Rust
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
//! R1 proofs of the job and sweep logic on an in-memory backend and an injected decoder
//! (no network, no child): the call order, the mapping of every child outcome, the
//! lease-left rule on transport failures, the no-delete rule on a refused commit, and the
//! sweeper's deletion targets.
#![allow(clippy::unwrap_used, clippy::expect_used)]
use std::cell::RefCell;
use std::collections::HashSet;
use std::time::Duration;
use faunosteo_worker::backend::{Backend, BackendError, BackendResult, Job, Leftover, Purge};
use faunosteo_worker::config::Config;
use faunosteo_worker::decoder::Decoder;
use faunosteo_worker::job::{JobEnd, process_job, reasons, sha256_hex};
use faunosteo_worker::run::run_loop;
use faunosteo_worker::spawn::{ChildOutcome, ChildRun};
use faunosteo_worker::sweep::{SweepConfig, SweepReport, run_sweep};
use faunosteo_worker::watchdog::Watchdog;
use faunosteo_worker::{exit_codes, webp_check};
const OWNER: &str = "10000000-0000-0000-0000-00000000000a";
const ASSET: &str = "20000000-0000-0000-0000-00000000000b";
const TOKEN: &str = "30000000-0000-0000-0000-00000000000c";
fn job() -> Job {
Job {
asset_id: ASSET.to_owned(),
owner_id: OWNER.to_owned(),
bucket_id: "avatars".to_owned(),
object_key: format!("profiles/{OWNER}/{ASSET}/avatar.webp"),
quarantine_key: Some(format!("quarantine/{OWNER}/{ASSET}/upload")),
lock_token: TOKEN.to_owned(),
attempts: 1,
}
}
/// A 512 × 512 lossless WebP container that passes the parent's check.
fn clean_webp() -> Vec<u8> {
let bits: u32 = (512 - 1) | ((512 - 1) << 14);
let mut vp8l = vec![0x2f];
vp8l.extend_from_slice(&bits.to_le_bytes());
let mut body = b"WEBP".to_vec();
body.extend_from_slice(b"VP8L");
body.extend_from_slice(&(vp8l.len() as u32).to_le_bytes());
body.extend_from_slice(&vp8l);
body.push(0);
let mut out = b"RIFF".to_vec();
out.extend_from_slice(&(body.len() as u32).to_le_bytes());
out.extend_from_slice(&body);
assert!(webp_check::expect_square(&out, 512).is_ok());
out
}
#[derive(Default)]
struct Fake {
calls: RefCell<Vec<String>>,
download: RefCell<Option<BackendResult<Vec<u8>>>>,
put: RefCell<Option<BackendResult<()>>>,
complete: RefCell<Option<BackendResult<()>>>,
reject: RefCell<Option<BackendResult<()>>>,
delete: RefCell<Option<BackendResult<()>>>,
purges: RefCell<Vec<Purge>>,
leftovers: RefCell<Vec<Leftover>>,
jobs: RefCell<Vec<Job>>,
}
impl Fake {
fn record(&self, call: String) {
self.calls.borrow_mut().push(call);
}
fn calls(&self) -> Vec<String> {
self.calls.borrow().clone()
}
fn names(&self) -> Vec<String> {
self.calls()
.iter()
.map(|c| c.split(' ').next().unwrap().to_owned())
.collect()
}
}
impl Backend for Fake {
fn claim_jobs(&self, max_jobs: u32) -> BackendResult<Vec<Job>> {
self.record(format!("claim {max_jobs}"));
Ok(self.jobs.borrow_mut().drain(..).collect())
}
fn download_quarantine(&self, bucket: &str, key: &str) -> BackendResult<Vec<u8>> {
self.record(format!("download {bucket}/{key}"));
self.download
.borrow_mut()
.take()
.unwrap_or_else(|| Ok(b"input".to_vec()))
}
fn put_clean(&self, bucket: &str, key: &str, bytes: &[u8]) -> BackendResult<()> {
self.record(format!("put {bucket}/{key} {}", bytes.len()));
self.put.borrow_mut().take().unwrap_or(Ok(()))
}
fn complete(
&self,
asset_id: &str,
lock_token: &str,
byte_size: usize,
sha256: &str,
) -> BackendResult<()> {
self.record(format!(
"complete {asset_id} {lock_token} {byte_size} {sha256}"
));
self.complete.borrow_mut().take().unwrap_or(Ok(()))
}
fn reject(&self, asset_id: &str, lock_token: &str, reason: &str) -> BackendResult<()> {
self.record(format!("reject {asset_id} {lock_token} {reason}"));
self.reject.borrow_mut().take().unwrap_or(Ok(()))
}
fn delete_object(&self, bucket: &str, key: &str) -> BackendResult<()> {
self.record(format!("delete {bucket}/{key}"));
self.delete.borrow_mut().take().unwrap_or(Ok(()))
}
fn mark_quarantine_purged(&self, asset_id: &str) -> BackendResult<()> {
self.record(format!("mark_purged {asset_id}"));
Ok(())
}
fn claim_purges(&self, max_items: u32) -> BackendResult<Vec<Purge>> {
self.record(format!("claim_purges {max_items}"));
Ok(self.purges.borrow_mut().drain(..).collect())
}
fn ack_purge(&self, purge_id: &str) -> BackendResult<()> {
self.record(format!("ack {purge_id}"));
Ok(())
}
fn claim_quarantine_leftovers(
&self,
older: u64,
max_items: u32,
) -> BackendResult<Vec<Leftover>> {
self.record(format!("claim_leftovers {older} {max_items}"));
Ok(self.leftovers.borrow_mut().drain(..).collect())
}
fn expire_pending_uploads(&self, max_age: u64) -> BackendResult<i64> {
self.record(format!("expire {max_age}"));
Ok(2)
}
}
struct FakeDecoder(RefCell<Option<ChildOutcome>>);
impl FakeDecoder {
fn with(outcome: ChildOutcome) -> Self {
Self(RefCell::new(Some(outcome)))
}
}
impl Decoder for FakeDecoder {
fn decode(&self, _bytes: &[u8]) -> ChildRun {
ChildRun {
outcome: self.0.borrow_mut().take().expect("one decode per test"),
stderr: "child says\n\u{1b}[0m".to_owned(),
}
}
}
fn run_job(fake: &Fake, outcome: ChildOutcome) -> JobEnd {
process_job(fake, &FakeDecoder::with(outcome), &Watchdog::new(), &job())
}
#[test]
fn a_produced_avatar_follows_download_decode_put_complete_delete_mark() {
let fake = Fake::default();
let webp = clean_webp();
let end = run_job(&fake, ChildOutcome::Produced(webp.clone()));
assert_eq!(end, JobEnd::Completed);
assert_eq!(
fake.calls(),
vec![
format!("download avatars/quarantine/{OWNER}/{ASSET}/upload"),
format!(
"put avatars/profiles/{OWNER}/{ASSET}/avatar.webp {}",
webp.len()
),
format!(
"complete {ASSET} {TOKEN} {} {}",
webp.len(),
sha256_hex(&webp)
),
format!("delete avatars/quarantine/{OWNER}/{ASSET}/upload"),
format!("mark_purged {ASSET}"),
]
);
}
#[test]
fn a_child_rejection_rejects_the_row_with_the_mapped_code() {
for (code, expected) in [
(exit_codes::UNSUPPORTED_TYPE, reasons::UNSUPPORTED_TYPE),
(exit_codes::DIMENSIONS, reasons::DIMENSIONS),
(exit_codes::DECODE_FAILED, reasons::DECODE_FAILED),
(exit_codes::RESOURCE_LIMIT, reasons::DECODE_FAILED),
(exit_codes::INPUT_TOO_LARGE, reasons::DECODE_FAILED),
] {
let fake = Fake::default();
let end = run_job(
&fake,
ChildOutcome::Rejected {
code,
name: exit_codes::name(code),
},
);
assert_eq!(end, JobEnd::Rejected(expected));
assert_eq!(fake.names(), vec!["download", "reject"]);
assert!(fake.calls()[1].ends_with(expected), "{:?}", fake.calls());
}
}
#[test]
fn every_abnormal_child_end_is_a_processing_error() {
for outcome in [
ChildOutcome::Signalled { signal: 31 },
ChildOutcome::Timeout,
ChildOutcome::OutputOverflow,
ChildOutcome::Failed("output refused".to_owned()),
] {
let fake = Fake::default();
assert_eq!(
run_job(&fake, outcome),
JobEnd::Rejected(reasons::PROCESSING_ERROR)
);
assert_eq!(fake.names(), vec!["download", "reject"]);
}
}
#[test]
fn a_broken_child_stops_the_loop_without_touching_the_row() {
let fake = Fake::default();
let end = run_job(
&fake,
ChildOutcome::Broken {
code: exit_codes::SANDBOX_FAILED,
name: "sandbox_failed",
},
);
assert_eq!(end, JobEnd::Stop);
assert_eq!(fake.names(), vec!["download"]);
}
#[test]
fn a_refused_commit_deletes_nothing() {
let fake = Fake::default();
*fake.complete.borrow_mut() = Some(Err(BackendError::Refused("invalid_state".to_owned())));
let end = run_job(&fake, ChildOutcome::Produced(clean_webp()));
assert_eq!(end, JobEnd::Superseded);
assert_eq!(fake.names(), vec!["download", "put", "complete"]);
}
#[test]
fn a_transport_failure_leaves_the_lease_and_rejects_nothing() {
let fake = Fake::default();
*fake.download.borrow_mut() = Some(Err(BackendError::Network("refused".to_owned())));
let end = process_job(
&fake,
&FakeDecoder(RefCell::new(None)),
&Watchdog::new(),
&job(),
);
assert_eq!(end, JobEnd::LeftForLease);
assert_eq!(fake.names(), vec!["download"]);
let fake = Fake::default();
*fake.put.borrow_mut() = Some(Err(BackendError::Status(503)));
let end = run_job(&fake, ChildOutcome::Produced(clean_webp()));
assert_eq!(end, JobEnd::LeftForLease);
assert_eq!(fake.names(), vec!["download", "put"]);
}
#[test]
fn missing_or_oversized_quarantine_bytes_are_decode_failures() {
for error in [BackendError::Status(404), BackendError::Overflow] {
let fake = Fake::default();
*fake.download.borrow_mut() = Some(Err(error));
let end = process_job(
&fake,
&FakeDecoder(RefCell::new(None)),
&Watchdog::new(),
&job(),
);
assert_eq!(end, JobEnd::Rejected(reasons::DECODE_FAILED));
assert_eq!(fake.names(), vec!["download", "reject"]);
}
}
#[test]
fn a_row_with_hostile_keys_is_rejected_before_any_storage_call() {
let fake = Fake::default();
let mut bad = job();
bad.quarantine_key = Some("quarantine/../../etc/passwd".to_owned());
let end = process_job(
&fake,
&FakeDecoder(RefCell::new(None)),
&Watchdog::new(),
&bad,
);
assert_eq!(end, JobEnd::Rejected(reasons::PROCESSING_ERROR));
assert_eq!(fake.names(), vec!["reject"]);
let fake = Fake::default();
let mut bad = job();
bad.asset_id = "x".to_owned();
let end = process_job(
&fake,
&FakeDecoder(RefCell::new(None)),
&Watchdog::new(),
&bad,
);
assert_eq!(end, JobEnd::LeftForLease);
assert!(fake.calls().is_empty());
}
#[test]
fn the_sweeper_deletes_only_queued_and_leftover_keys_and_acks() {
let fake = Fake::default();
let final_key = format!("profiles/{OWNER}/{ASSET}/avatar.webp");
let quarantine = format!("quarantine/{OWNER}/{ASSET}/upload");
*fake.purges.borrow_mut() = vec![
Purge {
purge_id: "p1".into(),
bucket_id: "avatars".into(),
object_name: final_key.clone(),
},
Purge {
purge_id: "p2".into(),
bucket_id: "avatars".into(),
object_name: "profiles/x/y/avatar.webp".into(),
},
Purge {
purge_id: "p3".into(),
bucket_id: "avatars".into(),
object_name: quarantine.clone(),
},
];
*fake.leftovers.borrow_mut() = vec![
Leftover {
bucket_id: "avatars".into(),
object_name: quarantine.clone(),
asset_id: Some(ASSET.into()),
},
Leftover {
bucket_id: "avatars".into(),
object_name: final_key.clone(),
asset_id: None,
},
];
let report = run_sweep(
&fake,
&Watchdog::new(),
SweepConfig {
pending_max_age: Duration::from_secs(86_400),
quarantine_grace: Duration::from_secs(10_800),
},
);
assert_eq!(
report,
SweepReport {
expired: 2,
purged: 2,
leftovers: 1,
errors: 2
}
);
assert_eq!(
fake.calls(),
vec![
"expire 86400".to_owned(),
"claim_purges 100".to_owned(),
format!("delete avatars/{final_key}"),
"ack p1".to_owned(),
format!("delete avatars/{quarantine}"),
"ack p3".to_owned(),
"claim_leftovers 10800 100".to_owned(),
format!("delete avatars/{quarantine}"),
format!("mark_purged {ASSET}"),
]
);
let deleted: HashSet<String> = fake
.calls()
.into_iter()
.filter(|c| c.starts_with("delete"))
.collect();
assert_eq!(deleted.len(), 2);
}
#[test]
fn a_missing_object_counts_as_purged() {
let fake = Fake::default();
*fake.purges.borrow_mut() = vec![Purge {
purge_id: "p1".into(),
bucket_id: "avatars".into(),
object_name: format!("profiles/{OWNER}/{ASSET}/avatar.webp"),
}];
// The HTTP backend maps a 404 to Ok before the sweeper sees it; a real error is not acked.
*fake.delete.borrow_mut() = Some(Err(BackendError::Status(500)));
let report = run_sweep(
&fake,
&Watchdog::new(),
SweepConfig {
pending_max_age: Duration::from_secs(1),
quarantine_grace: Duration::from_secs(1),
},
);
assert_eq!(report.purged, 0);
assert_eq!(report.errors, 1);
assert!(!fake.names().contains(&"ack".to_owned()));
}
#[test]
fn once_mode_sweeps_then_claims_once_and_exits_zero_or_stops_on_a_broken_child() {
let config = Config::from_vars(|name| match name {
"SUPABASE_URL" => Some("http://kong:8000".to_owned()),
"SUPABASE_SERVICE_ROLE_KEY" => Some("k".to_owned()),
"WORKER_ONCE" => Some("1".to_owned()),
_ => None,
})
.unwrap();
let fake = Fake::default();
*fake.jobs.borrow_mut() = vec![job()];
let decoder = FakeDecoder::with(ChildOutcome::Produced(clean_webp()));
assert_eq!(
run_loop(&config, &fake, &decoder, &Watchdog::new()),
exit_codes::OK
);
assert_eq!(
&fake.names()[..5],
[
"expire",
"claim_purges",
"claim_leftovers",
"claim",
"download"
]
);
let fake = Fake::default();
*fake.jobs.borrow_mut() = vec![job()];
let decoder = FakeDecoder::with(ChildOutcome::Broken {
code: exit_codes::ENV_NOT_EMPTY,
name: "env_not_empty",
});
assert_eq!(
run_loop(&config, &fake, &decoder, &Watchdog::new()),
exit_codes::CHILD_BROKEN
);
}