Security audit (PASS, 2 MEDIUM + LOW): the global ceiling summed every non-deleted row, so rejected uploads ratcheted it toward storage_full with no disk used → in-flight rows by declared size, clean rows by byte_size, nothing else (pgTAP: rejected rows do not count, in-flight rows do); confirm refuses a deactivated account like begin; complete_avatar_job refuses a null sha256; begin / confirm / remove granted to authenticated only. Worker: SUPABASE_URL over http:// accepted only for loopback or a single-label Docker service name; ureq proxy(None); the job's keys must embed its owner and asset ids; backend_failure / claim_failed / sweep_failure lines carry the closed kind and the HTTP status, never the library's error text. e2e: uuid shape checked before the psql query. Contract revision 2 (oracle fixes).
112 lines
3.9 KiB
Rust
112 lines
3.9 KiB
Rust
//! The sweeper (STORAGE.md §3.5): expires never-confirmed uploads, serves the purge queue
|
|
//! and deletes quarantine leftovers. These are the only deletions the worker makes, and a
|
|
//! key that fails the shape check is never sent to the Storage API.
|
|
|
|
use std::time::Duration;
|
|
|
|
use crate::backend::{Backend, BackendError, Leftover, Purge};
|
|
use crate::watchdog::{Watchdog, deadlines};
|
|
use crate::{keys, log};
|
|
|
|
/// Ages the sweeper applies.
|
|
#[derive(Debug, Clone, Copy)]
|
|
pub struct SweepConfig {
|
|
pub pending_max_age: Duration,
|
|
pub quarantine_grace: Duration,
|
|
}
|
|
|
|
/// What one sweep did.
|
|
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
|
pub struct SweepReport {
|
|
pub expired: i64,
|
|
pub purged: usize,
|
|
pub leftovers: usize,
|
|
pub errors: usize,
|
|
}
|
|
|
|
const BATCH: u32 = 100;
|
|
|
|
/// One full sweep; every failure is logged and counted, never fatal.
|
|
pub fn run_sweep(backend: &dyn Backend, watchdog: &Watchdog, config: SweepConfig) -> SweepReport {
|
|
let _stage = watchdog.enter("sweep", deadlines::SWEEP);
|
|
let mut report = SweepReport::default();
|
|
match backend.expire_pending_uploads(config.pending_max_age.as_secs()) {
|
|
Ok(count) => report.expired = count,
|
|
Err(error) => failure("expire_pending_uploads", &error, &mut report),
|
|
}
|
|
match backend.claim_purges(BATCH) {
|
|
Ok(purges) => purges
|
|
.iter()
|
|
.for_each(|p| serve_purge(backend, p, &mut report)),
|
|
Err(error) => failure("claim_purges", &error, &mut report),
|
|
}
|
|
match backend.claim_quarantine_leftovers(config.quarantine_grace.as_secs(), BATCH) {
|
|
Ok(items) => items
|
|
.iter()
|
|
.for_each(|l| serve_leftover(backend, l, &mut report)),
|
|
Err(error) => failure("claim_quarantine_leftovers", &error, &mut report),
|
|
}
|
|
log::info("sweep_done")
|
|
.count(report.purged + report.leftovers)
|
|
.detail(&format!(
|
|
"expired={} errors={}",
|
|
report.expired, report.errors
|
|
))
|
|
.emit();
|
|
report
|
|
}
|
|
|
|
fn serve_purge(backend: &dyn Backend, purge: &Purge, report: &mut SweepReport) {
|
|
if !deletable(&purge.bucket_id, &purge.object_name) {
|
|
log::error("purge_invalid_key")
|
|
.detail(&purge.purge_id)
|
|
.emit();
|
|
report.errors += 1;
|
|
return;
|
|
}
|
|
if let Err(error) = backend.delete_object(&purge.bucket_id, &purge.object_name) {
|
|
return failure("delete_object", &error, report);
|
|
}
|
|
match backend.ack_purge(&purge.purge_id) {
|
|
Ok(()) => report.purged += 1,
|
|
Err(error) => failure("ack_purge", &error, report),
|
|
}
|
|
}
|
|
|
|
fn serve_leftover(backend: &dyn Backend, leftover: &Leftover, report: &mut SweepReport) {
|
|
if !deletable(&leftover.bucket_id, &leftover.object_name)
|
|
|| keys::validate_quarantine_key(&leftover.object_name).is_err()
|
|
{
|
|
log::error("leftover_invalid_key").emit();
|
|
report.errors += 1;
|
|
return;
|
|
}
|
|
if let Err(error) = backend.delete_object(&leftover.bucket_id, &leftover.object_name) {
|
|
return failure("delete_object", &error, report);
|
|
}
|
|
report.leftovers += 1;
|
|
if let Some(asset_id) = leftover.asset_id.as_deref().filter(|id| keys::is_uuid(id))
|
|
&& let Err(error) = backend.mark_quarantine_purged(asset_id)
|
|
{
|
|
failure("mark_quarantine_purged", &error, report);
|
|
}
|
|
}
|
|
|
|
/// Bucket literal and one of the two key shapes: the only targets the worker deletes.
|
|
fn deletable(bucket: &str, key: &str) -> bool {
|
|
let bucket_ok = !bucket.is_empty()
|
|
&& bucket
|
|
.bytes()
|
|
.all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-');
|
|
bucket_ok && keys::validate_any_key(key).is_ok()
|
|
}
|
|
|
|
fn failure(call: &str, error: &BackendError, report: &mut SweepReport) {
|
|
report.errors += 1;
|
|
let line = log::warn("sweep_failure").stage(call).code(error.kind());
|
|
match error {
|
|
BackendError::Status(status) => line.http_status(*status).emit(),
|
|
_ => line.emit(),
|
|
}
|
|
}
|