feat(worker): R1 job loop — claim, download, sandboxed decode, clean upload, commit, purge sweeps

Keyed parent on ureq 3 (rustls + ring, bundled roots), no async runtime:
config from the environment (secret never printed, JWT expiry logged), JSON
line logs with a field allowlist (child stderr escaped and truncated), strict
key validation before any URL, Backend trait over PostgREST RPCs and Storage
objects with closed error kinds, one job per claim with the STORAGE.md §3.4
mapping (content codes, processing_error on signals / timeout / overflow,
Broken stops the loop with exit 20, a refused commit deletes nothing), sweeper
(expire pending, purge queue, quarantine leftovers), per-stage watchdog
(exit 21), WORKER_ONCE for tests, run --check extended. 11 loop tests on a
fake backend and decoder; sandbox tests and seccomp derivation unchanged
(the child still makes the same 7 syscalls); image 11.6 MB.
This commit is contained in:
bchanot committed 2026-10-13 19:32:38 +02:00
1 parent 926dd7df8c
commit 0410af0c35
18 files changed
+2921 -42

No files matched your search

+300
View File
@@ -0,0 +1,300 @@
//! `Backend` over PostgREST and the Storage API with the service key (STORAGE.md §3.3).
//! Thin by design: key shapes are checked before any URL is built, every status is
//! classified here, and the closed `P0001` refusals of the RPCs come back as `Refused`.
use std::time::Duration;
use serde::de::DeserializeOwned;
use serde_json::{Value, json};
use ureq::Agent;
use ureq::http::Response;
use crate::backend::{Backend, BackendError, BackendResult, Job, Leftover, Purge};
use crate::config::Config;
use crate::keys;
use crate::limits::MAX_INPUT_BYTES;
/// Largest RPC response the worker reads (a claim of 100 rows is a few KiB).
const MAX_RPC_BODY: u64 = 1024 * 1024;
pub struct HttpBackend {
agent: Agent,
base: String,
key: String,
}
impl HttpBackend {
pub fn new(config: &Config) -> Self {
let agent: Agent = Agent::config_builder()
.timeout_global(Some(Duration::from_secs(30)))
.timeout_connect(Some(Duration::from_secs(5)))
.max_redirects(0)
.http_status_as_error(false)
.user_agent("faunosteo-worker")
.build()
.into();
Self {
agent,
base: config.url.clone(),
key: config.service_key.expose().to_owned(),
}
}
fn rpc_url(&self, name: &str) -> String {
format!("{}/rest/v1/rpc/{name}", self.base)
}
fn object_url(&self, bucket: &str, key: &str) -> BackendResult<String> {
validate_bucket(bucket)?;
keys::validate_any_key(key).map_err(|e| BackendError::Internal(e.to_string()))?;
Ok(format!("{}/storage/v1/object/{bucket}/{key}", self.base))
}
/// Calls an RPC and returns the response body text of a 2xx.
fn rpc(&self, name: &str, body: &Value) -> BackendResult<String> {
let mut response = self
.agent
.post(&self.rpc_url(name))
.header("apikey", &self.key)
.header("Authorization", &format!("Bearer {}", self.key))
.header("Prefer", "return=representation")
.send_json(body)
.map_err(network)?;
let status = response.status().as_u16();
let text = read_text(&mut response)?;
if (200..300).contains(&status) {
Ok(text)
} else {
Err(classify(status, &text))
}
}
fn rpc_json<T: DeserializeOwned>(&self, name: &str, body: &Value) -> BackendResult<T> {
let text = self.rpc(name, body)?;
serde_json::from_str(&text).map_err(|e| BackendError::Decode(e.to_string()))
}
fn rpc_void(&self, name: &str, body: &Value) -> BackendResult<()> {
self.rpc(name, body).map(|_| ())
}
}
impl Backend for HttpBackend {
fn claim_jobs(&self, max_jobs: u32) -> BackendResult<Vec<Job>> {
self.rpc_json("claim_avatar_jobs", &json!({ "max_jobs": max_jobs }))
}
fn download_quarantine(&self, bucket: &str, key: &str) -> BackendResult<Vec<u8>> {
let url = self.object_url(bucket, key)?;
let mut response = self
.agent
.get(&url)
.header("apikey", &self.key)
.header("Authorization", &format!("Bearer {}", self.key))
.call()
.map_err(network)?;
let status = response.status().as_u16();
if status != 200 {
let text = read_text(&mut response).unwrap_or_default();
return Err(classify(status, &text));
}
let bytes = response
.body_mut()
.with_config()
.limit(MAX_INPUT_BYTES as u64 + 1)
.read_to_vec()
.map_err(|e| BackendError::Network(e.to_string()))?;
if bytes.len() > MAX_INPUT_BYTES {
return Err(BackendError::Overflow);
}
Ok(bytes)
}
fn put_clean(&self, bucket: &str, key: &str, bytes: &[u8]) -> BackendResult<()> {
let url = self.object_url(bucket, key)?;
let mut response = self
.agent
.post(&url)
.header("apikey", &self.key)
.header("Authorization", &format!("Bearer {}", self.key))
.header("Content-Type", "image/webp")
.header("x-upsert", "true")
.send(bytes)
.map_err(network)?;
expect_success(&mut response)
}
fn complete(
&self,
asset_id: &str,
lock_token: &str,
byte_size: usize,
sha256: &str,
) -> BackendResult<()> {
self.rpc_void(
"complete_avatar_job",
&json!({ "asset_id": asset_id, "lock_token": lock_token, "byte_size": byte_size, "sha256": sha256 }),
)
}
fn reject(&self, asset_id: &str, lock_token: &str, reason: &str) -> BackendResult<()> {
self.rpc_void(
"reject_avatar_job",
&json!({ "asset_id": asset_id, "lock_token": lock_token, "reason": reason }),
)
}
fn delete_object(&self, bucket: &str, key: &str) -> BackendResult<()> {
let url = self.object_url(bucket, key)?;
let mut response = self
.agent
.delete(&url)
.header("apikey", &self.key)
.header("Authorization", &format!("Bearer {}", self.key))
.call()
.map_err(network)?;
if response.status().as_u16() == 404 {
return Ok(());
}
expect_success(&mut response)
}
fn mark_quarantine_purged(&self, asset_id: &str) -> BackendResult<()> {
self.rpc_void("mark_quarantine_purged", &json!({ "asset_id": asset_id }))
}
fn claim_purges(&self, max_items: u32) -> BackendResult<Vec<Purge>> {
self.rpc_json("claim_purges", &json!({ "max_items": max_items }))
}
fn ack_purge(&self, purge_id: &str) -> BackendResult<()> {
self.rpc_void("ack_purge", &json!({ "purge_id": purge_id }))
}
fn claim_quarantine_leftovers(
&self,
older_than_seconds: u64,
max_items: u32,
) -> BackendResult<Vec<Leftover>> {
self.rpc_json(
"claim_quarantine_leftovers",
&json!({ "older_than_seconds": older_than_seconds, "max_items": max_items }),
)
}
fn expire_pending_uploads(&self, max_age_seconds: u64) -> BackendResult<i64> {
self.rpc_json(
"expire_pending_uploads",
&json!({ "max_age_seconds": max_age_seconds }),
)
}
}
fn validate_bucket(bucket: &str) -> BackendResult<()> {
let ok = !bucket.is_empty()
&& bucket
.bytes()
.all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-');
if ok {
Ok(())
} else {
Err(BackendError::Internal("bucket name refused".to_owned()))
}
}
fn network(error: ureq::Error) -> BackendError {
BackendError::Network(error.to_string())
}
fn read_text(response: &mut Response<ureq::Body>) -> BackendResult<String> {
response
.body_mut()
.with_config()
.limit(MAX_RPC_BODY)
.read_to_string()
.map_err(|e| BackendError::Network(e.to_string()))
}
fn expect_success(response: &mut Response<ureq::Body>) -> BackendResult<()> {
let status = response.status().as_u16();
if (200..300).contains(&status) {
return Ok(());
}
let text = read_text(response).unwrap_or_default();
Err(classify(status, &text))
}
/// A PostgREST `P0001` body names a closed refusal; any other status is reported as such.
pub fn classify(status: u16, body: &str) -> BackendError {
if let Ok(value) = serde_json::from_str::<Value>(body)
&& value.get("code").and_then(Value::as_str) == Some("P0001")
&& let Some(message) = value.get("message").and_then(Value::as_str)
&& is_closed_code(message)
{
return BackendError::Refused(message.to_owned());
}
BackendError::Status(status)
}
/// Shape of a closed code (`invalid_state`): the server's free text never becomes a code.
fn is_closed_code(message: &str) -> bool {
!message.is_empty()
&& message.len() <= 40
&& message.bytes().all(|b| b.is_ascii_lowercase() || b == b'_')
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn p0001_bodies_become_closed_refusals_and_others_statuses() {
assert_eq!(
classify(
400,
r#"{"code":"P0001","message":"invalid_state","details":null}"#
),
BackendError::Refused("invalid_state".to_owned())
);
assert_eq!(
classify(403, r#"{"code":"42501","message":"denied"}"#),
BackendError::Status(403)
);
assert_eq!(
classify(400, r#"{"code":"P0001","message":"Free text!"}"#),
BackendError::Status(400)
);
assert_eq!(classify(500, "not json"), BackendError::Status(500));
}
#[test]
fn object_urls_need_valid_buckets_and_keys() {
let config = Config::from_vars(|name| match name {
"SUPABASE_URL" => Some("http://kong:8000".to_owned()),
"SUPABASE_SERVICE_ROLE_KEY" => Some("k".to_owned()),
_ => None,
})
.ok();
let Some(config) = config else {
panic!("config");
};
let backend = HttpBackend::new(&config);
let key = "profiles/10000000-0000-0000-0000-00000000000a/10000000-0000-0000-0000-00000000000b/avatar.webp";
assert_eq!(
backend.object_url("avatars", key).ok().as_deref(),
Some(
"http://kong:8000/storage/v1/object/avatars/profiles/10000000-0000-0000-0000-00000000000a/10000000-0000-0000-0000-00000000000b/avatar.webp"
)
);
assert!(backend.object_url("avatars/../x", key).is_err());
assert!(
backend
.object_url("avatars", "profiles/../etc/passwd")
.is_err()
);
assert_eq!(
backend.rpc_url("claim_avatar_jobs"),
"http://kong:8000/rest/v1/rpc/claim_avatar_jobs"
);
}
}