Security audit (PASS, 2 MEDIUM + LOW): the global ceiling summed every non-deleted row, so rejected uploads ratcheted it toward storage_full with no disk used → in-flight rows by declared size, clean rows by byte_size, nothing else (pgTAP: rejected rows do not count, in-flight rows do); confirm refuses a deactivated account like begin; complete_avatar_job refuses a null sha256; begin / confirm / remove granted to authenticated only. Worker: SUPABASE_URL over http:// accepted only for loopback or a single-label Docker service name; ureq proxy(None); the job's keys must embed its owner and asset ids; backend_failure / claim_failed / sweep_failure lines carry the closed kind and the HTTP status, never the library's error text. e2e: uuid shape checked before the psql query. Contract revision 2 (oracle fixes).
271 lines
9.2 KiB
Rust
271 lines
9.2 KiB
Rust
//! One avatar job (STORAGE.md §3.4): download the quarantine bytes, hand them to the
|
|
//! sandboxed child, upload the clean output, commit the row, clean the quarantine. Every
|
|
//! outcome of the child and of the backend maps to one closed end; nothing is deleted on a
|
|
//! refused commit (the purge queue owns every object of a terminal row).
|
|
|
|
use std::time::Instant;
|
|
|
|
use sha2::{Digest, Sha256};
|
|
|
|
use crate::backend::{Backend, BackendError, Job};
|
|
use crate::decoder::Decoder;
|
|
use crate::spawn::ChildOutcome;
|
|
use crate::watchdog::{Watchdog, deadlines};
|
|
use crate::{exit_codes, keys, log};
|
|
|
|
/// How a job ended, as the loop sees it.
|
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
pub enum JobEnd {
|
|
/// The row is `clean`.
|
|
Completed,
|
|
/// The row was refused with this closed code.
|
|
Rejected(&'static str),
|
|
/// The commit or the rejection was refused: the row moved on without us.
|
|
Superseded,
|
|
/// A transport failure: the lease expires and the row is claimed again.
|
|
LeftForLease,
|
|
/// The child reported a broken environment: the loop must stop.
|
|
Stop,
|
|
}
|
|
|
|
/// Closed rejection reasons the row accepts (STORAGE.md §1.2).
|
|
pub mod reasons {
|
|
pub const UNSUPPORTED_TYPE: &str = "unsupported_type";
|
|
pub const DECODE_FAILED: &str = "decode_failed";
|
|
pub const DIMENSIONS: &str = "dimensions";
|
|
pub const PROCESSING_ERROR: &str = "processing_error";
|
|
}
|
|
|
|
/// Runs one claimed job to its end.
|
|
pub fn process_job(
|
|
backend: &dyn Backend,
|
|
decoder: &dyn Decoder,
|
|
watchdog: &Watchdog,
|
|
job: &Job,
|
|
) -> JobEnd {
|
|
let started = Instant::now();
|
|
let end = run(backend, decoder, watchdog, job);
|
|
let line = match &end {
|
|
JobEnd::Completed => log::info("job_completed"),
|
|
JobEnd::Rejected(code) => log::info("job_rejected").code(code),
|
|
JobEnd::Superseded => log::info("job_superseded"),
|
|
JobEnd::LeftForLease => log::warn("job_left_for_lease"),
|
|
JobEnd::Stop => log::error("job_stop"),
|
|
};
|
|
line.asset(&job.asset_id)
|
|
.duration_ms(started.elapsed().as_millis())
|
|
.emit();
|
|
end
|
|
}
|
|
|
|
fn run(backend: &dyn Backend, decoder: &dyn Decoder, watchdog: &Watchdog, job: &Job) -> JobEnd {
|
|
if !keys::is_uuid(&job.asset_id) || !keys::is_uuid(&job.lock_token) {
|
|
log::error("job_invalid_identity").emit();
|
|
return JobEnd::LeftForLease;
|
|
}
|
|
let Some(quarantine_key) = validated_keys(job) else {
|
|
return reject(backend, job, reasons::PROCESSING_ERROR);
|
|
};
|
|
let bytes = match download(backend, watchdog, job, quarantine_key) {
|
|
Ok(bytes) => bytes,
|
|
Err(end) => return end,
|
|
};
|
|
let verdict = {
|
|
let _stage = watchdog.enter("decode", deadlines::DECODE);
|
|
classify_child(decoder.decode(&bytes), job)
|
|
};
|
|
match verdict {
|
|
ChildVerdict::Clean(webp) => commit(backend, watchdog, job, quarantine_key, &webp),
|
|
ChildVerdict::Reject(reason) => reject(backend, job, reason),
|
|
ChildVerdict::Stop => JobEnd::Stop,
|
|
}
|
|
}
|
|
|
|
/// Both keys and the bucket must have the shapes the RPC builds; `None` = corrupted row.
|
|
fn validated_keys(job: &Job) -> Option<&str> {
|
|
let quarantine_key = job.quarantine_key.as_deref()?;
|
|
let bucket_ok = !job.bucket_id.is_empty()
|
|
&& job
|
|
.bucket_id
|
|
.bytes()
|
|
.all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-');
|
|
// Shape, then ownership: both keys must embed this job's owner and asset ids, so a
|
|
// mismatched row can never make the worker write or read another account's key.
|
|
if !bucket_ok
|
|
|| keys::validate_object_key(&job.object_key).is_err()
|
|
|| keys::validate_quarantine_key(quarantine_key).is_err()
|
|
|| !keys::belongs_to(&job.object_key, &job.owner_id, &job.asset_id)
|
|
|| !keys::belongs_to(quarantine_key, &job.owner_id, &job.asset_id)
|
|
{
|
|
log::error("job_invalid_keys").asset(&job.asset_id).emit();
|
|
return None;
|
|
}
|
|
Some(quarantine_key)
|
|
}
|
|
|
|
fn download(
|
|
backend: &dyn Backend,
|
|
watchdog: &Watchdog,
|
|
job: &Job,
|
|
quarantine_key: &str,
|
|
) -> Result<Vec<u8>, JobEnd> {
|
|
let _stage = watchdog.enter("download", deadlines::DOWNLOAD);
|
|
match backend.download_quarantine(&job.bucket_id, quarantine_key) {
|
|
Ok(bytes) => Ok(bytes),
|
|
Err(BackendError::Status(404)) => {
|
|
log::warn("download_missing").asset(&job.asset_id).emit();
|
|
Err(reject(backend, job, reasons::DECODE_FAILED))
|
|
}
|
|
Err(BackendError::Overflow) => {
|
|
log::warn("download_overflow").asset(&job.asset_id).emit();
|
|
Err(reject(backend, job, reasons::DECODE_FAILED))
|
|
}
|
|
Err(error) => Err(transport_failure("download", job, &error)),
|
|
}
|
|
}
|
|
|
|
/// What the child's end means for the row.
|
|
enum ChildVerdict {
|
|
Clean(Vec<u8>),
|
|
Reject(&'static str),
|
|
Stop,
|
|
}
|
|
|
|
/// Maps every way the child can end (R0 `ChildOutcome`) to a verdict on the row.
|
|
fn classify_child(run: crate::spawn::ChildRun, job: &Job) -> ChildVerdict {
|
|
let stderr = run.stderr;
|
|
match run.outcome {
|
|
ChildOutcome::Produced(webp) => ChildVerdict::Clean(webp),
|
|
ChildOutcome::Rejected { code, name } => {
|
|
log::info("child_rejected")
|
|
.asset(&job.asset_id)
|
|
.code(name)
|
|
.child_stderr(&stderr)
|
|
.emit();
|
|
ChildVerdict::Reject(content_reason(code))
|
|
}
|
|
ChildOutcome::Broken { name, .. } => {
|
|
log::error("child_broken")
|
|
.asset(&job.asset_id)
|
|
.code(name)
|
|
.child_stderr(&stderr)
|
|
.emit();
|
|
ChildVerdict::Stop
|
|
}
|
|
ChildOutcome::Signalled { signal } => {
|
|
log::warn("child_signalled")
|
|
.asset(&job.asset_id)
|
|
.signal(signal)
|
|
.child_stderr(&stderr)
|
|
.emit();
|
|
ChildVerdict::Reject(reasons::PROCESSING_ERROR)
|
|
}
|
|
ChildOutcome::OutputOverflow => {
|
|
log::warn("child_output_overflow")
|
|
.asset(&job.asset_id)
|
|
.emit();
|
|
ChildVerdict::Reject(reasons::PROCESSING_ERROR)
|
|
}
|
|
ChildOutcome::Timeout => {
|
|
log::warn("child_timeout").asset(&job.asset_id).emit();
|
|
ChildVerdict::Reject(reasons::PROCESSING_ERROR)
|
|
}
|
|
ChildOutcome::Failed(detail) => {
|
|
log::warn("child_failed")
|
|
.asset(&job.asset_id)
|
|
.detail(&detail)
|
|
.emit();
|
|
ChildVerdict::Reject(reasons::PROCESSING_ERROR)
|
|
}
|
|
}
|
|
}
|
|
|
|
/// The row's closed reason for a content exit code of the child.
|
|
pub fn content_reason(code: i32) -> &'static str {
|
|
match code {
|
|
exit_codes::UNSUPPORTED_TYPE => reasons::UNSUPPORTED_TYPE,
|
|
exit_codes::DIMENSIONS => reasons::DIMENSIONS,
|
|
_ => reasons::DECODE_FAILED,
|
|
}
|
|
}
|
|
|
|
fn commit(
|
|
backend: &dyn Backend,
|
|
watchdog: &Watchdog,
|
|
job: &Job,
|
|
quarantine_key: &str,
|
|
webp: &[u8],
|
|
) -> JobEnd {
|
|
{
|
|
let _stage = watchdog.enter("upload", deadlines::UPLOAD);
|
|
if let Err(error) = backend.put_clean(&job.bucket_id, &job.object_key, webp) {
|
|
return transport_failure("upload", job, &error);
|
|
}
|
|
}
|
|
let _stage = watchdog.enter("commit", deadlines::COMMIT);
|
|
match backend.complete(
|
|
&job.asset_id,
|
|
&job.lock_token,
|
|
webp.len(),
|
|
&sha256_hex(webp),
|
|
) {
|
|
Ok(()) => {}
|
|
Err(BackendError::Refused(code)) => {
|
|
log::info("complete_refused")
|
|
.asset(&job.asset_id)
|
|
.code(&code)
|
|
.emit();
|
|
return JobEnd::Superseded;
|
|
}
|
|
Err(error) => return transport_failure("commit", job, &error),
|
|
}
|
|
// Best effort: the sweeper finishes a quarantine the worker could not delete.
|
|
if backend
|
|
.delete_object(&job.bucket_id, quarantine_key)
|
|
.is_ok()
|
|
&& backend.mark_quarantine_purged(&job.asset_id).is_err()
|
|
{
|
|
log::warn("mark_quarantine_purged_failed")
|
|
.asset(&job.asset_id)
|
|
.emit();
|
|
}
|
|
JobEnd::Completed
|
|
}
|
|
|
|
/// Rejection of the row with a closed reason; a refusal means the row moved on.
|
|
fn reject(backend: &dyn Backend, job: &Job, reason: &'static str) -> JobEnd {
|
|
match backend.reject(&job.asset_id, &job.lock_token, reason) {
|
|
Ok(()) => JobEnd::Rejected(reason),
|
|
Err(BackendError::Refused(code)) => {
|
|
log::info("reject_refused")
|
|
.asset(&job.asset_id)
|
|
.code(&code)
|
|
.emit();
|
|
JobEnd::Superseded
|
|
}
|
|
Err(error) => transport_failure("reject", job, &error),
|
|
}
|
|
}
|
|
|
|
fn transport_failure(stage: &str, job: &Job, error: &BackendError) -> JobEnd {
|
|
let line = log::warn("backend_failure")
|
|
.asset(&job.asset_id)
|
|
.stage(stage)
|
|
.code(error.kind());
|
|
// The library's error text (URI fragments, server bodies) stays out of the logs:
|
|
// the closed kind and the HTTP status are the record.
|
|
match error {
|
|
BackendError::Status(status) => line.http_status(*status).emit(),
|
|
_ => line.emit(),
|
|
}
|
|
JobEnd::LeftForLease
|
|
}
|
|
|
|
/// Lowercase hex sha256 of the clean output.
|
|
pub fn sha256_hex(bytes: &[u8]) -> String {
|
|
Sha256::digest(bytes)
|
|
.iter()
|
|
.map(|b| format!("{b:02x}"))
|
|
.collect()
|
|
}
|