//! R1 proofs of the job and sweep logic on an in-memory backend and an injected decoder //! (no network, no child): the call order, the mapping of every child outcome, the //! lease-left rule on transport failures, the no-delete rule on a refused commit, and the //! sweeper's deletion targets. #![allow(clippy::unwrap_used, clippy::expect_used)] use std::cell::RefCell; use std::collections::HashSet; use std::time::Duration; use faunosteo_worker::backend::{Backend, BackendError, BackendResult, Job, Leftover, Purge}; use faunosteo_worker::config::Config; use faunosteo_worker::decoder::Decoder; use faunosteo_worker::job::{JobEnd, process_job, reasons, sha256_hex}; use faunosteo_worker::run::run_loop; use faunosteo_worker::spawn::{ChildOutcome, ChildRun}; use faunosteo_worker::sweep::{SweepConfig, SweepReport, run_sweep}; use faunosteo_worker::watchdog::Watchdog; use faunosteo_worker::{exit_codes, webp_check}; const OWNER: &str = "10000000-0000-0000-0000-00000000000a"; const ASSET: &str = "20000000-0000-0000-0000-00000000000b"; const TOKEN: &str = "30000000-0000-0000-0000-00000000000c"; fn job() -> Job { Job { asset_id: ASSET.to_owned(), owner_id: OWNER.to_owned(), bucket_id: "avatars".to_owned(), object_key: format!("profiles/{OWNER}/{ASSET}/avatar.webp"), quarantine_key: Some(format!("quarantine/{OWNER}/{ASSET}/upload")), lock_token: TOKEN.to_owned(), attempts: 1, } } /// A 512 × 512 lossless WebP container that passes the parent's check. fn clean_webp() -> Vec { let bits: u32 = (512 - 1) | ((512 - 1) << 14); let mut vp8l = vec![0x2f]; vp8l.extend_from_slice(&bits.to_le_bytes()); let mut body = b"WEBP".to_vec(); body.extend_from_slice(b"VP8L"); body.extend_from_slice(&(vp8l.len() as u32).to_le_bytes()); body.extend_from_slice(&vp8l); body.push(0); let mut out = b"RIFF".to_vec(); out.extend_from_slice(&(body.len() as u32).to_le_bytes()); out.extend_from_slice(&body); assert!(webp_check::expect_square(&out, 512).is_ok()); out } #[derive(Default)] struct Fake { calls: RefCell>, download: RefCell>>>, put: RefCell>>, complete: RefCell>>, reject: RefCell>>, delete: RefCell>>, purges: RefCell>, leftovers: RefCell>, jobs: RefCell>, } impl Fake { fn record(&self, call: String) { self.calls.borrow_mut().push(call); } fn calls(&self) -> Vec { self.calls.borrow().clone() } fn names(&self) -> Vec { self.calls() .iter() .map(|c| c.split(' ').next().unwrap().to_owned()) .collect() } } impl Backend for Fake { fn claim_jobs(&self, max_jobs: u32) -> BackendResult> { self.record(format!("claim {max_jobs}")); Ok(self.jobs.borrow_mut().drain(..).collect()) } fn download_quarantine(&self, bucket: &str, key: &str) -> BackendResult> { self.record(format!("download {bucket}/{key}")); self.download .borrow_mut() .take() .unwrap_or_else(|| Ok(b"input".to_vec())) } fn put_clean(&self, bucket: &str, key: &str, bytes: &[u8]) -> BackendResult<()> { self.record(format!("put {bucket}/{key} {}", bytes.len())); self.put.borrow_mut().take().unwrap_or(Ok(())) } fn complete( &self, asset_id: &str, lock_token: &str, byte_size: usize, sha256: &str, ) -> BackendResult<()> { self.record(format!( "complete {asset_id} {lock_token} {byte_size} {sha256}" )); self.complete.borrow_mut().take().unwrap_or(Ok(())) } fn reject(&self, asset_id: &str, lock_token: &str, reason: &str) -> BackendResult<()> { self.record(format!("reject {asset_id} {lock_token} {reason}")); self.reject.borrow_mut().take().unwrap_or(Ok(())) } fn delete_object(&self, bucket: &str, key: &str) -> BackendResult<()> { self.record(format!("delete {bucket}/{key}")); self.delete.borrow_mut().take().unwrap_or(Ok(())) } fn mark_quarantine_purged(&self, asset_id: &str) -> BackendResult<()> { self.record(format!("mark_purged {asset_id}")); Ok(()) } fn claim_purges(&self, max_items: u32) -> BackendResult> { self.record(format!("claim_purges {max_items}")); Ok(self.purges.borrow_mut().drain(..).collect()) } fn ack_purge(&self, purge_id: &str) -> BackendResult<()> { self.record(format!("ack {purge_id}")); Ok(()) } fn claim_quarantine_leftovers( &self, older: u64, max_items: u32, ) -> BackendResult> { self.record(format!("claim_leftovers {older} {max_items}")); Ok(self.leftovers.borrow_mut().drain(..).collect()) } fn expire_pending_uploads(&self, max_age: u64) -> BackendResult { self.record(format!("expire {max_age}")); Ok(2) } } struct FakeDecoder(RefCell>); impl FakeDecoder { fn with(outcome: ChildOutcome) -> Self { Self(RefCell::new(Some(outcome))) } } impl Decoder for FakeDecoder { fn decode(&self, _bytes: &[u8]) -> ChildRun { ChildRun { outcome: self.0.borrow_mut().take().expect("one decode per test"), stderr: "child says\n\u{1b}[0m".to_owned(), } } } fn run_job(fake: &Fake, outcome: ChildOutcome) -> JobEnd { process_job(fake, &FakeDecoder::with(outcome), &Watchdog::new(), &job()) } #[test] fn a_produced_avatar_follows_download_decode_put_complete_delete_mark() { let fake = Fake::default(); let webp = clean_webp(); let end = run_job(&fake, ChildOutcome::Produced(webp.clone())); assert_eq!(end, JobEnd::Completed); assert_eq!( fake.calls(), vec![ format!("download avatars/quarantine/{OWNER}/{ASSET}/upload"), format!( "put avatars/profiles/{OWNER}/{ASSET}/avatar.webp {}", webp.len() ), format!( "complete {ASSET} {TOKEN} {} {}", webp.len(), sha256_hex(&webp) ), format!("delete avatars/quarantine/{OWNER}/{ASSET}/upload"), format!("mark_purged {ASSET}"), ] ); } #[test] fn a_child_rejection_rejects_the_row_with_the_mapped_code() { for (code, expected) in [ (exit_codes::UNSUPPORTED_TYPE, reasons::UNSUPPORTED_TYPE), (exit_codes::DIMENSIONS, reasons::DIMENSIONS), (exit_codes::DECODE_FAILED, reasons::DECODE_FAILED), (exit_codes::RESOURCE_LIMIT, reasons::DECODE_FAILED), (exit_codes::INPUT_TOO_LARGE, reasons::DECODE_FAILED), ] { let fake = Fake::default(); let end = run_job( &fake, ChildOutcome::Rejected { code, name: exit_codes::name(code), }, ); assert_eq!(end, JobEnd::Rejected(expected)); assert_eq!(fake.names(), vec!["download", "reject"]); assert!(fake.calls()[1].ends_with(expected), "{:?}", fake.calls()); } } #[test] fn every_abnormal_child_end_is_a_processing_error() { for outcome in [ ChildOutcome::Signalled { signal: 31 }, ChildOutcome::Timeout, ChildOutcome::OutputOverflow, ChildOutcome::Failed("output refused".to_owned()), ] { let fake = Fake::default(); assert_eq!( run_job(&fake, outcome), JobEnd::Rejected(reasons::PROCESSING_ERROR) ); assert_eq!(fake.names(), vec!["download", "reject"]); } } #[test] fn a_broken_child_stops_the_loop_without_touching_the_row() { let fake = Fake::default(); let end = run_job( &fake, ChildOutcome::Broken { code: exit_codes::SANDBOX_FAILED, name: "sandbox_failed", }, ); assert_eq!(end, JobEnd::Stop); assert_eq!(fake.names(), vec!["download"]); } #[test] fn a_refused_commit_deletes_nothing() { let fake = Fake::default(); *fake.complete.borrow_mut() = Some(Err(BackendError::Refused("invalid_state".to_owned()))); let end = run_job(&fake, ChildOutcome::Produced(clean_webp())); assert_eq!(end, JobEnd::Superseded); assert_eq!(fake.names(), vec!["download", "put", "complete"]); } #[test] fn a_transport_failure_leaves_the_lease_and_rejects_nothing() { let fake = Fake::default(); *fake.download.borrow_mut() = Some(Err(BackendError::Network("refused".to_owned()))); let end = process_job( &fake, &FakeDecoder(RefCell::new(None)), &Watchdog::new(), &job(), ); assert_eq!(end, JobEnd::LeftForLease); assert_eq!(fake.names(), vec!["download"]); let fake = Fake::default(); *fake.put.borrow_mut() = Some(Err(BackendError::Status(503))); let end = run_job(&fake, ChildOutcome::Produced(clean_webp())); assert_eq!(end, JobEnd::LeftForLease); assert_eq!(fake.names(), vec!["download", "put"]); } #[test] fn missing_or_oversized_quarantine_bytes_are_decode_failures() { for error in [BackendError::Status(404), BackendError::Overflow] { let fake = Fake::default(); *fake.download.borrow_mut() = Some(Err(error)); let end = process_job( &fake, &FakeDecoder(RefCell::new(None)), &Watchdog::new(), &job(), ); assert_eq!(end, JobEnd::Rejected(reasons::DECODE_FAILED)); assert_eq!(fake.names(), vec!["download", "reject"]); } } #[test] fn a_row_with_hostile_keys_is_rejected_before_any_storage_call() { let fake = Fake::default(); let mut bad = job(); bad.quarantine_key = Some("quarantine/../../etc/passwd".to_owned()); let end = process_job( &fake, &FakeDecoder(RefCell::new(None)), &Watchdog::new(), &bad, ); assert_eq!(end, JobEnd::Rejected(reasons::PROCESSING_ERROR)); assert_eq!(fake.names(), vec!["reject"]); let fake = Fake::default(); let mut bad = job(); bad.asset_id = "x".to_owned(); let end = process_job( &fake, &FakeDecoder(RefCell::new(None)), &Watchdog::new(), &bad, ); assert_eq!(end, JobEnd::LeftForLease); assert!(fake.calls().is_empty()); } #[test] fn the_sweeper_deletes_only_queued_and_leftover_keys_and_acks() { let fake = Fake::default(); let final_key = format!("profiles/{OWNER}/{ASSET}/avatar.webp"); let quarantine = format!("quarantine/{OWNER}/{ASSET}/upload"); *fake.purges.borrow_mut() = vec![ Purge { purge_id: "p1".into(), bucket_id: "avatars".into(), object_name: final_key.clone(), }, Purge { purge_id: "p2".into(), bucket_id: "avatars".into(), object_name: "profiles/x/y/avatar.webp".into(), }, Purge { purge_id: "p3".into(), bucket_id: "avatars".into(), object_name: quarantine.clone(), }, ]; *fake.leftovers.borrow_mut() = vec![ Leftover { bucket_id: "avatars".into(), object_name: quarantine.clone(), asset_id: Some(ASSET.into()), }, Leftover { bucket_id: "avatars".into(), object_name: final_key.clone(), asset_id: None, }, ]; let report = run_sweep( &fake, &Watchdog::new(), SweepConfig { pending_max_age: Duration::from_secs(86_400), quarantine_grace: Duration::from_secs(10_800), }, ); assert_eq!( report, SweepReport { expired: 2, purged: 2, leftovers: 1, errors: 2 } ); assert_eq!( fake.calls(), vec![ "expire 86400".to_owned(), "claim_purges 100".to_owned(), format!("delete avatars/{final_key}"), "ack p1".to_owned(), format!("delete avatars/{quarantine}"), "ack p3".to_owned(), "claim_leftovers 10800 100".to_owned(), format!("delete avatars/{quarantine}"), format!("mark_purged {ASSET}"), ] ); let deleted: HashSet = fake .calls() .into_iter() .filter(|c| c.starts_with("delete")) .collect(); assert_eq!(deleted.len(), 2); } #[test] fn a_missing_object_counts_as_purged() { let fake = Fake::default(); *fake.purges.borrow_mut() = vec![Purge { purge_id: "p1".into(), bucket_id: "avatars".into(), object_name: format!("profiles/{OWNER}/{ASSET}/avatar.webp"), }]; // The HTTP backend maps a 404 to Ok before the sweeper sees it; a real error is not acked. *fake.delete.borrow_mut() = Some(Err(BackendError::Status(500))); let report = run_sweep( &fake, &Watchdog::new(), SweepConfig { pending_max_age: Duration::from_secs(1), quarantine_grace: Duration::from_secs(1), }, ); assert_eq!(report.purged, 0); assert_eq!(report.errors, 1); assert!(!fake.names().contains(&"ack".to_owned())); } #[test] fn once_mode_sweeps_then_claims_once_and_exits_zero_or_stops_on_a_broken_child() { let config = Config::from_vars(|name| match name { "SUPABASE_URL" => Some("http://kong:8000".to_owned()), "SUPABASE_SERVICE_ROLE_KEY" => Some("k".to_owned()), "WORKER_ONCE" => Some("1".to_owned()), _ => None, }) .unwrap(); let fake = Fake::default(); *fake.jobs.borrow_mut() = vec![job()]; let decoder = FakeDecoder::with(ChildOutcome::Produced(clean_webp())); assert_eq!( run_loop(&config, &fake, &decoder, &Watchdog::new()), exit_codes::OK ); assert_eq!( &fake.names()[..5], [ "expire", "claim_purges", "claim_leftovers", "claim", "download" ] ); let fake = Fake::default(); *fake.jobs.borrow_mut() = vec![job()]; let decoder = FakeDecoder::with(ChildOutcome::Broken { code: exit_codes::ENV_NOT_EMPTY, name: "env_not_empty", }); assert_eq!( run_loop(&config, &fake, &decoder, &Watchdog::new()), exit_codes::CHILD_BROKEN ); }