feat(worker): R1 job loop — claim, download, sandboxed decode, clean upload, commit, purge sweeps

Keyed parent on ureq 3 (rustls + ring, bundled roots), no async runtime:
config from the environment (secret never printed, JWT expiry logged), JSON
line logs with a field allowlist (child stderr escaped and truncated), strict
key validation before any URL, Backend trait over PostgREST RPCs and Storage
objects with closed error kinds, one job per claim with the STORAGE.md §3.4
mapping (content codes, processing_error on signals / timeout / overflow,
Broken stops the loop with exit 20, a refused commit deletes nothing), sweeper
(expire pending, purge queue, quarantine leftovers), per-stage watchdog
(exit 21), WORKER_ONCE for tests, run --check extended. 11 loop tests on a
fake backend and decoder; sandbox tests and seccomp derivation unchanged
(the child still makes the same 7 syscalls); image 11.6 MB.
This commit is contained in:
bchanot committed 2026-10-13 19:32:38 +02:00
1 parent 926dd7df8c
commit 0410af0c35
18 files changed
+2921 -42

No files matched your search

+111
View File
@@ -0,0 +1,111 @@
//! The sweeper (STORAGE.md §3.5): expires never-confirmed uploads, serves the purge queue
//! and deletes quarantine leftovers. These are the only deletions the worker makes, and a
//! key that fails the shape check is never sent to the Storage API.
use std::time::Duration;
use crate::backend::{Backend, BackendError, Leftover, Purge};
use crate::watchdog::{Watchdog, deadlines};
use crate::{keys, log};
/// Ages the sweeper applies.
#[derive(Debug, Clone, Copy)]
pub struct SweepConfig {
pub pending_max_age: Duration,
pub quarantine_grace: Duration,
}
/// What one sweep did.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct SweepReport {
pub expired: i64,
pub purged: usize,
pub leftovers: usize,
pub errors: usize,
}
const BATCH: u32 = 100;
/// One full sweep; every failure is logged and counted, never fatal.
pub fn run_sweep(backend: &dyn Backend, watchdog: &Watchdog, config: SweepConfig) -> SweepReport {
let _stage = watchdog.enter("sweep", deadlines::SWEEP);
let mut report = SweepReport::default();
match backend.expire_pending_uploads(config.pending_max_age.as_secs()) {
Ok(count) => report.expired = count,
Err(error) => failure("expire_pending_uploads", &error, &mut report),
}
match backend.claim_purges(BATCH) {
Ok(purges) => purges
.iter()
.for_each(|p| serve_purge(backend, p, &mut report)),
Err(error) => failure("claim_purges", &error, &mut report),
}
match backend.claim_quarantine_leftovers(config.quarantine_grace.as_secs(), BATCH) {
Ok(items) => items
.iter()
.for_each(|l| serve_leftover(backend, l, &mut report)),
Err(error) => failure("claim_quarantine_leftovers", &error, &mut report),
}
log::info("sweep_done")
.count(report.purged + report.leftovers)
.detail(&format!(
"expired={} errors={}",
report.expired, report.errors
))
.emit();
report
}
fn serve_purge(backend: &dyn Backend, purge: &Purge, report: &mut SweepReport) {
if !deletable(&purge.bucket_id, &purge.object_name) {
log::error("purge_invalid_key")
.detail(&purge.purge_id)
.emit();
report.errors += 1;
return;
}
if let Err(error) = backend.delete_object(&purge.bucket_id, &purge.object_name) {
return failure("delete_object", &error, report);
}
match backend.ack_purge(&purge.purge_id) {
Ok(()) => report.purged += 1,
Err(error) => failure("ack_purge", &error, report),
}
}
fn serve_leftover(backend: &dyn Backend, leftover: &Leftover, report: &mut SweepReport) {
if !deletable(&leftover.bucket_id, &leftover.object_name)
|| keys::validate_quarantine_key(&leftover.object_name).is_err()
{
log::error("leftover_invalid_key").emit();
report.errors += 1;
return;
}
if let Err(error) = backend.delete_object(&leftover.bucket_id, &leftover.object_name) {
return failure("delete_object", &error, report);
}
report.leftovers += 1;
if let Some(asset_id) = leftover.asset_id.as_deref().filter(|id| keys::is_uuid(id))
&& let Err(error) = backend.mark_quarantine_purged(asset_id)
{
failure("mark_quarantine_purged", &error, report);
}
}
/// Bucket literal and one of the two key shapes: the only targets the worker deletes.
fn deletable(bucket: &str, key: &str) -> bool {
let bucket_ok = !bucket.is_empty()
&& bucket
.bytes()
.all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-');
bucket_ok && keys::validate_any_key(key).is_ok()
}
fn failure(call: &str, error: &BackendError, report: &mut SweepReport) {
report.errors += 1;
let line = log::warn("sweep_failure").stage(call).code(error.kind());
match error {
BackendError::Status(status) => line.http_status(*status).emit(),
other => line.detail(&other.to_string()).emit(),
}
}