//! `Backend` over PostgREST and the Storage API with the service key (STORAGE.md ยง3.3). //! Thin by design: key shapes are checked before any URL is built, every status is //! classified here, and the closed `P0001` refusals of the RPCs come back as `Refused`. //! The same client serves `VerifyBackend` (V1-2.3): larger caps for the round's input and //! the verdict, drawing keys checked before any URL is built. use std::time::Duration; use serde::de::DeserializeOwned; use serde_json::{Value, json}; use ureq::Agent; use ureq::http::Response; use crate::backend::{ Backend, BackendError, BackendResult, CompleteAnswer, FailAnswer, Job, Leftover, Purge, VerifyBackend, VerifyInput, VerifyJob, }; use crate::config::Config; use crate::keys; use crate::limits::MAX_INPUT_BYTES; /// Largest RPC response the worker reads (a claim of 100 rows is a few KiB). const MAX_RPC_BODY: u64 = 1024 * 1024; /// Largest `get_sheet_verification_input` response: two 2 MiB documents, whose JSON string /// escaping can double them, and the drawing rows. const MAX_INPUT_RPC_BODY: u64 = 16 * 1024 * 1024; pub struct HttpBackend { agent: Agent, base: String, key: String, } impl HttpBackend { pub fn new(config: &Config) -> Self { let agent: Agent = Agent::config_builder() .timeout_global(Some(Duration::from_secs(30))) .timeout_connect(Some(Duration::from_secs(5))) .max_redirects(0) // Never route the service key through a proxy picked up from the environment. .proxy(None) .http_status_as_error(false) .user_agent("faunosteo-worker") .build() .into(); Self { agent, base: config.url.clone(), key: config.service_key.expose().to_owned(), } } fn rpc_url(&self, name: &str) -> String { format!("{}/rest/v1/rpc/{name}", self.base) } fn object_url(&self, bucket: &str, key: &str) -> BackendResult { validate_bucket(bucket)?; keys::validate_any_key(key).map_err(|e| BackendError::Internal(e.to_string()))?; Ok(format!("{}/storage/v1/object/{bucket}/{key}", self.base)) } /// A sheet drawing: `.svg` in a well-formed bucket, nothing else. fn drawing_url(&self, bucket: &str, key: &str) -> BackendResult { validate_bucket(bucket)?; if !keys::is_drawing_key(key) { return Err(BackendError::Internal("drawing key refused".to_owned())); } Ok(format!("{}/storage/v1/object/{bucket}/{key}", self.base)) } /// Calls an RPC and returns the response body text of a 2xx. fn rpc(&self, name: &str, body: &Value) -> BackendResult { self.rpc_capped(name, body, MAX_RPC_BODY) } /// `rpc` with its own response cap. fn rpc_capped(&self, name: &str, body: &Value, cap: u64) -> BackendResult { let mut response = self .agent .post(&self.rpc_url(name)) .header("apikey", &self.key) .header("Authorization", &format!("Bearer {}", self.key)) .header("Prefer", "return=representation") .send_json(body) .map_err(network)?; let status = response.status().as_u16(); let text = read_text_capped(&mut response, cap)?; if (200..300).contains(&status) { Ok(text) } else { Err(classify(status, &text)) } } fn rpc_json(&self, name: &str, body: &Value) -> BackendResult { let text = self.rpc(name, body)?; serde_json::from_str(&text).map_err(|e| BackendError::Decode(e.to_string())) } fn rpc_void(&self, name: &str, body: &Value) -> BackendResult<()> { self.rpc(name, body).map(|_| ()) } /// GET of an object, read up to `cap` bytes (`Overflow` past it). fn get_object(&self, url: &str, cap: usize) -> BackendResult> { let mut response = self .agent .get(url) .header("apikey", &self.key) .header("Authorization", &format!("Bearer {}", self.key)) .call() .map_err(network)?; let status = response.status().as_u16(); if status != 200 { let text = read_text(&mut response).unwrap_or_default(); return Err(classify(status, &text)); } let bytes = response .body_mut() .with_config() .limit(cap as u64 + 1) .read_to_vec() .map_err(|e| BackendError::Network(e.to_string()))?; if bytes.len() > cap { return Err(BackendError::Overflow); } Ok(bytes) } } impl Backend for HttpBackend { fn claim_jobs(&self, max_jobs: u32) -> BackendResult> { self.rpc_json("claim_avatar_jobs", &json!({ "max_jobs": max_jobs })) } fn download_quarantine(&self, bucket: &str, key: &str) -> BackendResult> { let url = self.object_url(bucket, key)?; self.get_object(&url, MAX_INPUT_BYTES) } fn put_clean(&self, bucket: &str, key: &str, bytes: &[u8]) -> BackendResult<()> { let url = self.object_url(bucket, key)?; let mut response = self .agent .post(&url) .header("apikey", &self.key) .header("Authorization", &format!("Bearer {}", self.key)) .header("Content-Type", "image/webp") .header("x-upsert", "true") .send(bytes) .map_err(network)?; expect_success(&mut response) } fn complete( &self, asset_id: &str, lock_token: &str, byte_size: usize, sha256: &str, ) -> BackendResult<()> { self.rpc_void( "complete_avatar_job", &json!({ "asset_id": asset_id, "lock_token": lock_token, "byte_size": byte_size, "sha256": sha256 }), ) } fn reject(&self, asset_id: &str, lock_token: &str, reason: &str) -> BackendResult<()> { self.rpc_void( "reject_avatar_job", &json!({ "asset_id": asset_id, "lock_token": lock_token, "reason": reason }), ) } fn delete_object(&self, bucket: &str, key: &str) -> BackendResult<()> { let url = self.object_url(bucket, key)?; let mut response = self .agent .delete(&url) .header("apikey", &self.key) .header("Authorization", &format!("Bearer {}", self.key)) .call() .map_err(network)?; // A missing object is a purge already done (HTTP 404, or this Storage API's // HTTP 400 body with statusCode 404, normalised by `classify`). match expect_success(&mut response) { Err(BackendError::Status(404)) => Ok(()), other => other, } } fn mark_quarantine_purged(&self, asset_id: &str) -> BackendResult<()> { self.rpc_void("mark_quarantine_purged", &json!({ "asset_id": asset_id })) } fn claim_purges(&self, max_items: u32) -> BackendResult> { self.rpc_json("claim_purges", &json!({ "max_items": max_items })) } fn ack_purge(&self, purge_id: &str) -> BackendResult<()> { self.rpc_void("ack_purge", &json!({ "purge_id": purge_id })) } fn claim_quarantine_leftovers( &self, older_than_seconds: u64, max_items: u32, ) -> BackendResult> { self.rpc_json( "claim_quarantine_leftovers", &json!({ "older_than_seconds": older_than_seconds, "max_items": max_items }), ) } fn expire_pending_uploads(&self, max_age_seconds: u64) -> BackendResult { self.rpc_json( "expire_pending_uploads", &json!({ "max_age_seconds": max_age_seconds }), ) } } impl VerifyBackend for HttpBackend { fn claim_verifications(&self, max_jobs: u32) -> BackendResult> { self.rpc_json( "claim_sheet_verifications", &json!({ "max_jobs": max_jobs }), ) } fn verification_input( &self, review_id: &str, lease_token: &str, ) -> BackendResult> { let text = self.rpc_capped( "get_sheet_verification_input", &json!({ "review_id": review_id, "lease_token": lease_token }), MAX_INPUT_RPC_BODY, )?; serde_json::from_str(&text).map_err(|e| BackendError::Decode(e.to_string())) } fn download_drawing( &self, bucket: &str, key: &str, max_bytes: usize, ) -> BackendResult> { let url = self.drawing_url(bucket, key)?; self.get_object(&url, max_bytes) } fn complete_verification( &self, review_id: &str, lease_token: &str, verdict: &[u8], ) -> BackendResult { // The spawner already checked the bytes are a JSON object (under the output cap). let verdict: Value = serde_json::from_slice(verdict).map_err(|e| BackendError::Internal(e.to_string()))?; self.rpc_json( "complete_sheet_verification", &json!({ "review_id": review_id, "lease_token": lease_token, "verdict": verdict }), ) } fn fail_verification( &self, review_id: &str, lease_token: &str, code: &str, ) -> BackendResult { self.rpc_json( "fail_sheet_verification", &json!({ "review_id": review_id, "lease_token": lease_token, "code": code }), ) } } fn validate_bucket(bucket: &str) -> BackendResult<()> { let ok = !bucket.is_empty() && bucket .bytes() .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-'); if ok { Ok(()) } else { Err(BackendError::Internal("bucket name refused".to_owned())) } } fn network(error: ureq::Error) -> BackendError { BackendError::Network(error.to_string()) } fn read_text(response: &mut Response) -> BackendResult { read_text_capped(response, MAX_RPC_BODY) } fn read_text_capped(response: &mut Response, cap: u64) -> BackendResult { response .body_mut() .with_config() .limit(cap) .read_to_string() .map_err(|e| BackendError::Network(e.to_string())) } fn expect_success(response: &mut Response) -> BackendResult<()> { let status = response.status().as_u16(); if (200..300).contains(&status) { return Ok(()); } let text = read_text(response).unwrap_or_default(); Err(classify(status, &text)) } /// A PostgREST `P0001` body names a closed refusal; a Storage API "not found" (HTTP 400 /// with `statusCode` 404 on this image family) is reported as 404; any other status as such. pub fn classify(status: u16, body: &str) -> BackendError { let Ok(value) = serde_json::from_str::(body) else { return BackendError::Status(status); }; if value.get("code").and_then(Value::as_str) == Some("P0001") && let Some(message) = value.get("message").and_then(Value::as_str) && is_closed_code(message) { return BackendError::Refused(message.to_owned()); } if status == 400 && (value.get("statusCode").and_then(Value::as_str) == Some("404") || value.get("error").and_then(Value::as_str) == Some("not_found")) { return BackendError::Status(404); } BackendError::Status(status) } /// Shape of a closed code (`invalid_state`): the server's free text never becomes a code. fn is_closed_code(message: &str) -> bool { !message.is_empty() && message.len() <= 40 && message.bytes().all(|b| b.is_ascii_lowercase() || b == b'_') } #[cfg(test)] mod tests { use super::*; #[test] fn p0001_bodies_become_closed_refusals_and_others_statuses() { assert_eq!( classify( 400, r#"{"code":"P0001","message":"invalid_state","details":null}"# ), BackendError::Refused("invalid_state".to_owned()) ); assert_eq!( classify(403, r#"{"code":"42501","message":"denied"}"#), BackendError::Status(403) ); assert_eq!( classify(400, r#"{"code":"P0001","message":"Free text!"}"#), BackendError::Status(400) ); assert_eq!(classify(500, "not json"), BackendError::Status(500)); // storage-api v1.79 answers a missing object with HTTP 400 and statusCode "404". assert_eq!( classify( 400, r#"{"statusCode":"404","error":"not_found","message":"Object not found"}"# ), BackendError::Status(404) ); assert_eq!( classify( 400, r#"{"statusCode":"400","error":"InvalidRequest","message":"x"}"# ), BackendError::Status(400) ); } #[test] fn the_m6_answers_and_rows_deserialize() { assert_eq!( serde_json::from_str::("\"malformed\"").ok(), Some(CompleteAnswer::Malformed) ); assert_eq!( serde_json::from_str::("\"retry\"").ok(), Some(FailAnswer::Retry) ); assert!(serde_json::from_str::("\"other\"").is_err()); assert_eq!( serde_json::from_str::>("null").ok(), Some(None) ); let input = serde_json::from_value::>(json!({ "review_id": "r", "version_id": "v", "sheet_id": "s", "revision": 2, "document": "{}", "base_document": null, "drawings": [{ "asset_id": "a", "sha256": "h", "byte_size": 10, "bucket": "anatomical-sheet-assets", "object_key": "h.svg", "uploaded": true }] })); assert_eq!( input.ok().flatten().map(|i| i.drawings[0].byte_size), Some(10) ); let jobs = serde_json::from_value::>(json!([{ "review_id": "r", "version_id": "v", "sheet_id": "s", "revision": 1, "lease_token": "t", "attempts": 1 }])); assert_eq!(jobs.ok().map(|j| j[0].revision), Some(1)); } #[test] fn object_urls_need_valid_buckets_and_keys() { let config = Config::from_vars(|name| match name { "SUPABASE_URL" => Some("http://kong:8000".to_owned()), "SUPABASE_SERVICE_ROLE_KEY" => Some("k".to_owned()), _ => None, }) .ok(); let Some(config) = config else { panic!("config"); }; let backend = HttpBackend::new(&config); let key = "profiles/10000000-0000-0000-0000-00000000000a/10000000-0000-0000-0000-00000000000b/avatar.webp"; assert_eq!( backend.object_url("avatars", key).ok().as_deref(), Some( "http://kong:8000/storage/v1/object/avatars/profiles/10000000-0000-0000-0000-00000000000a/10000000-0000-0000-0000-00000000000b/avatar.webp" ) ); assert!(backend.object_url("avatars/../x", key).is_err()); assert!( backend .object_url("avatars", "profiles/../etc/passwd") .is_err() ); let drawing = format!("{}.svg", "ab".repeat(32)); assert_eq!( backend .drawing_url("anatomical-sheet-assets", &drawing) .ok(), Some(format!( "http://kong:8000/storage/v1/object/anatomical-sheet-assets/{drawing}" )) ); assert!(backend.drawing_url("anatomical-sheet-assets", key).is_err()); assert!(backend.drawing_url("a/b", &drawing).is_err()); assert_eq!( backend.rpc_url("claim_avatar_jobs"), "http://kong:8000/rest/v1/rpc/claim_avatar_jobs" ); } }