Files
faunosteo/crates/worker/src/http.rs
T

460 lines
16 KiB
Rust

//! `Backend` over PostgREST and the Storage API with the service key (STORAGE.md §3.3).
//! Thin by design: key shapes are checked before any URL is built, every status is
//! classified here, and the closed `P0001` refusals of the RPCs come back as `Refused`.
//! The same client serves `VerifyBackend` (V1-2.3): larger caps for the round's input and
//! the verdict, drawing keys checked before any URL is built.
use std::time::Duration;
use serde::de::DeserializeOwned;
use serde_json::{Value, json};
use ureq::Agent;
use ureq::http::Response;
use crate::backend::{
Backend, BackendError, BackendResult, CompleteAnswer, FailAnswer, Job, Leftover, Purge,
VerifyBackend, VerifyInput, VerifyJob,
};
use crate::config::Config;
use crate::keys;
use crate::limits::MAX_INPUT_BYTES;
/// Largest RPC response the worker reads (a claim of 100 rows is a few KiB).
const MAX_RPC_BODY: u64 = 1024 * 1024;
/// Largest `get_sheet_verification_input` response: two 2 MiB documents, whose JSON string
/// escaping can double them, and the drawing rows.
const MAX_INPUT_RPC_BODY: u64 = 16 * 1024 * 1024;
pub struct HttpBackend {
agent: Agent,
base: String,
key: String,
}
impl HttpBackend {
pub fn new(config: &Config) -> Self {
let agent: Agent = Agent::config_builder()
.timeout_global(Some(Duration::from_secs(30)))
.timeout_connect(Some(Duration::from_secs(5)))
.max_redirects(0)
// Never route the service key through a proxy picked up from the environment.
.proxy(None)
.http_status_as_error(false)
.user_agent("faunosteo-worker")
.build()
.into();
Self {
agent,
base: config.url.clone(),
key: config.service_key.expose().to_owned(),
}
}
fn rpc_url(&self, name: &str) -> String {
format!("{}/rest/v1/rpc/{name}", self.base)
}
fn object_url(&self, bucket: &str, key: &str) -> BackendResult<String> {
validate_bucket(bucket)?;
keys::validate_any_key(key).map_err(|e| BackendError::Internal(e.to_string()))?;
Ok(format!("{}/storage/v1/object/{bucket}/{key}", self.base))
}
/// A sheet drawing: `<sha256>.svg` in a well-formed bucket, nothing else.
fn drawing_url(&self, bucket: &str, key: &str) -> BackendResult<String> {
validate_bucket(bucket)?;
if !keys::is_drawing_key(key) {
return Err(BackendError::Internal("drawing key refused".to_owned()));
}
Ok(format!("{}/storage/v1/object/{bucket}/{key}", self.base))
}
/// Calls an RPC and returns the response body text of a 2xx.
fn rpc(&self, name: &str, body: &Value) -> BackendResult<String> {
self.rpc_capped(name, body, MAX_RPC_BODY)
}
/// `rpc` with its own response cap.
fn rpc_capped(&self, name: &str, body: &Value, cap: u64) -> BackendResult<String> {
let mut response = self
.agent
.post(&self.rpc_url(name))
.header("apikey", &self.key)
.header("Authorization", &format!("Bearer {}", self.key))
.header("Prefer", "return=representation")
.send_json(body)
.map_err(network)?;
let status = response.status().as_u16();
let text = read_text_capped(&mut response, cap)?;
if (200..300).contains(&status) {
Ok(text)
} else {
Err(classify(status, &text))
}
}
fn rpc_json<T: DeserializeOwned>(&self, name: &str, body: &Value) -> BackendResult<T> {
let text = self.rpc(name, body)?;
serde_json::from_str(&text).map_err(|e| BackendError::Decode(e.to_string()))
}
fn rpc_void(&self, name: &str, body: &Value) -> BackendResult<()> {
self.rpc(name, body).map(|_| ())
}
/// GET of an object, read up to `cap` bytes (`Overflow` past it).
fn get_object(&self, url: &str, cap: usize) -> BackendResult<Vec<u8>> {
let mut response = self
.agent
.get(url)
.header("apikey", &self.key)
.header("Authorization", &format!("Bearer {}", self.key))
.call()
.map_err(network)?;
let status = response.status().as_u16();
if status != 200 {
let text = read_text(&mut response).unwrap_or_default();
return Err(classify(status, &text));
}
let bytes = response
.body_mut()
.with_config()
.limit(cap as u64 + 1)
.read_to_vec()
.map_err(|e| BackendError::Network(e.to_string()))?;
if bytes.len() > cap {
return Err(BackendError::Overflow);
}
Ok(bytes)
}
}
impl Backend for HttpBackend {
fn claim_jobs(&self, max_jobs: u32) -> BackendResult<Vec<Job>> {
self.rpc_json("claim_avatar_jobs", &json!({ "max_jobs": max_jobs }))
}
fn download_quarantine(&self, bucket: &str, key: &str) -> BackendResult<Vec<u8>> {
let url = self.object_url(bucket, key)?;
self.get_object(&url, MAX_INPUT_BYTES)
}
fn put_clean(&self, bucket: &str, key: &str, bytes: &[u8]) -> BackendResult<()> {
let url = self.object_url(bucket, key)?;
let mut response = self
.agent
.post(&url)
.header("apikey", &self.key)
.header("Authorization", &format!("Bearer {}", self.key))
.header("Content-Type", "image/webp")
.header("x-upsert", "true")
.send(bytes)
.map_err(network)?;
expect_success(&mut response)
}
fn complete(
&self,
asset_id: &str,
lock_token: &str,
byte_size: usize,
sha256: &str,
) -> BackendResult<()> {
self.rpc_void(
"complete_avatar_job",
&json!({ "asset_id": asset_id, "lock_token": lock_token, "byte_size": byte_size, "sha256": sha256 }),
)
}
fn reject(&self, asset_id: &str, lock_token: &str, reason: &str) -> BackendResult<()> {
self.rpc_void(
"reject_avatar_job",
&json!({ "asset_id": asset_id, "lock_token": lock_token, "reason": reason }),
)
}
fn delete_object(&self, bucket: &str, key: &str) -> BackendResult<()> {
let url = self.object_url(bucket, key)?;
let mut response = self
.agent
.delete(&url)
.header("apikey", &self.key)
.header("Authorization", &format!("Bearer {}", self.key))
.call()
.map_err(network)?;
// A missing object is a purge already done (HTTP 404, or this Storage API's
// HTTP 400 body with statusCode 404, normalised by `classify`).
match expect_success(&mut response) {
Err(BackendError::Status(404)) => Ok(()),
other => other,
}
}
fn mark_quarantine_purged(&self, asset_id: &str) -> BackendResult<()> {
self.rpc_void("mark_quarantine_purged", &json!({ "asset_id": asset_id }))
}
fn claim_purges(&self, max_items: u32) -> BackendResult<Vec<Purge>> {
self.rpc_json("claim_purges", &json!({ "max_items": max_items }))
}
fn ack_purge(&self, purge_id: &str) -> BackendResult<()> {
self.rpc_void("ack_purge", &json!({ "purge_id": purge_id }))
}
fn claim_quarantine_leftovers(
&self,
older_than_seconds: u64,
max_items: u32,
) -> BackendResult<Vec<Leftover>> {
self.rpc_json(
"claim_quarantine_leftovers",
&json!({ "older_than_seconds": older_than_seconds, "max_items": max_items }),
)
}
fn expire_pending_uploads(&self, max_age_seconds: u64) -> BackendResult<i64> {
self.rpc_json(
"expire_pending_uploads",
&json!({ "max_age_seconds": max_age_seconds }),
)
}
}
impl VerifyBackend for HttpBackend {
fn claim_verifications(&self, max_jobs: u32) -> BackendResult<Vec<VerifyJob>> {
self.rpc_json(
"claim_sheet_verifications",
&json!({ "max_jobs": max_jobs }),
)
}
fn verification_input(
&self,
review_id: &str,
lease_token: &str,
) -> BackendResult<Option<VerifyInput>> {
let text = self.rpc_capped(
"get_sheet_verification_input",
&json!({ "review_id": review_id, "lease_token": lease_token }),
MAX_INPUT_RPC_BODY,
)?;
serde_json::from_str(&text).map_err(|e| BackendError::Decode(e.to_string()))
}
fn download_drawing(
&self,
bucket: &str,
key: &str,
max_bytes: usize,
) -> BackendResult<Vec<u8>> {
let url = self.drawing_url(bucket, key)?;
self.get_object(&url, max_bytes)
}
fn complete_verification(
&self,
review_id: &str,
lease_token: &str,
verdict: &[u8],
) -> BackendResult<CompleteAnswer> {
// The spawner already checked the bytes are a JSON object (under the output cap).
let verdict: Value =
serde_json::from_slice(verdict).map_err(|e| BackendError::Internal(e.to_string()))?;
self.rpc_json(
"complete_sheet_verification",
&json!({ "review_id": review_id, "lease_token": lease_token, "verdict": verdict }),
)
}
fn fail_verification(
&self,
review_id: &str,
lease_token: &str,
code: &str,
) -> BackendResult<FailAnswer> {
self.rpc_json(
"fail_sheet_verification",
&json!({ "review_id": review_id, "lease_token": lease_token, "code": code }),
)
}
}
fn validate_bucket(bucket: &str) -> BackendResult<()> {
let ok = !bucket.is_empty()
&& bucket
.bytes()
.all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-');
if ok {
Ok(())
} else {
Err(BackendError::Internal("bucket name refused".to_owned()))
}
}
fn network(error: ureq::Error) -> BackendError {
BackendError::Network(error.to_string())
}
fn read_text(response: &mut Response<ureq::Body>) -> BackendResult<String> {
read_text_capped(response, MAX_RPC_BODY)
}
fn read_text_capped(response: &mut Response<ureq::Body>, cap: u64) -> BackendResult<String> {
response
.body_mut()
.with_config()
.limit(cap)
.read_to_string()
.map_err(|e| BackendError::Network(e.to_string()))
}
fn expect_success(response: &mut Response<ureq::Body>) -> BackendResult<()> {
let status = response.status().as_u16();
if (200..300).contains(&status) {
return Ok(());
}
let text = read_text(response).unwrap_or_default();
Err(classify(status, &text))
}
/// A PostgREST `P0001` body names a closed refusal; a Storage API "not found" (HTTP 400
/// with `statusCode` 404 on this image family) is reported as 404; any other status as such.
pub fn classify(status: u16, body: &str) -> BackendError {
let Ok(value) = serde_json::from_str::<Value>(body) else {
return BackendError::Status(status);
};
if value.get("code").and_then(Value::as_str) == Some("P0001")
&& let Some(message) = value.get("message").and_then(Value::as_str)
&& is_closed_code(message)
{
return BackendError::Refused(message.to_owned());
}
if status == 400
&& (value.get("statusCode").and_then(Value::as_str) == Some("404")
|| value.get("error").and_then(Value::as_str) == Some("not_found"))
{
return BackendError::Status(404);
}
BackendError::Status(status)
}
/// Shape of a closed code (`invalid_state`): the server's free text never becomes a code.
fn is_closed_code(message: &str) -> bool {
!message.is_empty()
&& message.len() <= 40
&& message.bytes().all(|b| b.is_ascii_lowercase() || b == b'_')
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn p0001_bodies_become_closed_refusals_and_others_statuses() {
assert_eq!(
classify(
400,
r#"{"code":"P0001","message":"invalid_state","details":null}"#
),
BackendError::Refused("invalid_state".to_owned())
);
assert_eq!(
classify(403, r#"{"code":"42501","message":"denied"}"#),
BackendError::Status(403)
);
assert_eq!(
classify(400, r#"{"code":"P0001","message":"Free text!"}"#),
BackendError::Status(400)
);
assert_eq!(classify(500, "not json"), BackendError::Status(500));
// storage-api v1.79 answers a missing object with HTTP 400 and statusCode "404".
assert_eq!(
classify(
400,
r#"{"statusCode":"404","error":"not_found","message":"Object not found"}"#
),
BackendError::Status(404)
);
assert_eq!(
classify(
400,
r#"{"statusCode":"400","error":"InvalidRequest","message":"x"}"#
),
BackendError::Status(400)
);
}
#[test]
fn the_m6_answers_and_rows_deserialize() {
assert_eq!(
serde_json::from_str::<CompleteAnswer>("\"malformed\"").ok(),
Some(CompleteAnswer::Malformed)
);
assert_eq!(
serde_json::from_str::<FailAnswer>("\"retry\"").ok(),
Some(FailAnswer::Retry)
);
assert!(serde_json::from_str::<CompleteAnswer>("\"other\"").is_err());
assert_eq!(
serde_json::from_str::<Option<VerifyInput>>("null").ok(),
Some(None)
);
let input = serde_json::from_value::<Option<VerifyInput>>(json!({
"review_id": "r", "version_id": "v", "sheet_id": "s", "revision": 2,
"document": "{}", "base_document": null,
"drawings": [{ "asset_id": "a", "sha256": "h", "byte_size": 10,
"bucket": "anatomical-sheet-assets", "object_key": "h.svg", "uploaded": true }]
}));
assert_eq!(
input.ok().flatten().map(|i| i.drawings[0].byte_size),
Some(10)
);
let jobs = serde_json::from_value::<Vec<VerifyJob>>(json!([{ "review_id": "r",
"version_id": "v", "sheet_id": "s", "revision": 1, "lease_token": "t", "attempts": 1 }]));
assert_eq!(jobs.ok().map(|j| j[0].revision), Some(1));
}
#[test]
fn object_urls_need_valid_buckets_and_keys() {
let config = Config::from_vars(|name| match name {
"SUPABASE_URL" => Some("http://kong:8000".to_owned()),
"SUPABASE_SERVICE_ROLE_KEY" => Some("k".to_owned()),
_ => None,
})
.ok();
let Some(config) = config else {
panic!("config");
};
let backend = HttpBackend::new(&config);
let key = "profiles/10000000-0000-0000-0000-00000000000a/10000000-0000-0000-0000-00000000000b/avatar.webp";
assert_eq!(
backend.object_url("avatars", key).ok().as_deref(),
Some(
"http://kong:8000/storage/v1/object/avatars/profiles/10000000-0000-0000-0000-00000000000a/10000000-0000-0000-0000-00000000000b/avatar.webp"
)
);
assert!(backend.object_url("avatars/../x", key).is_err());
assert!(
backend
.object_url("avatars", "profiles/../etc/passwd")
.is_err()
);
let drawing = format!("{}.svg", "ab".repeat(32));
assert_eq!(
backend
.drawing_url("anatomical-sheet-assets", &drawing)
.ok(),
Some(format!(
"http://kong:8000/storage/v1/object/anatomical-sheet-assets/{drawing}"
))
);
assert!(backend.drawing_url("anatomical-sheet-assets", key).is_err());
assert!(backend.drawing_url("a/b", &drawing).is_err());
assert_eq!(
backend.rpc_url("claim_avatar_jobs"),
"http://kong:8000/rest/v1/rpc/claim_avatar_jobs"
);
}
}