460 lines
16 KiB
Rust
460 lines
16 KiB
Rust
//! `Backend` over PostgREST and the Storage API with the service key (STORAGE.md §3.3).
|
|
//! Thin by design: key shapes are checked before any URL is built, every status is
|
|
//! classified here, and the closed `P0001` refusals of the RPCs come back as `Refused`.
|
|
//! The same client serves `VerifyBackend` (V1-2.3): larger caps for the round's input and
|
|
//! the verdict, drawing keys checked before any URL is built.
|
|
|
|
use std::time::Duration;
|
|
|
|
use serde::de::DeserializeOwned;
|
|
use serde_json::{Value, json};
|
|
use ureq::Agent;
|
|
use ureq::http::Response;
|
|
|
|
use crate::backend::{
|
|
Backend, BackendError, BackendResult, CompleteAnswer, FailAnswer, Job, Leftover, Purge,
|
|
VerifyBackend, VerifyInput, VerifyJob,
|
|
};
|
|
use crate::config::Config;
|
|
use crate::keys;
|
|
use crate::limits::MAX_INPUT_BYTES;
|
|
|
|
/// Largest RPC response the worker reads (a claim of 100 rows is a few KiB).
|
|
const MAX_RPC_BODY: u64 = 1024 * 1024;
|
|
/// Largest `get_sheet_verification_input` response: two 2 MiB documents, whose JSON string
|
|
/// escaping can double them, and the drawing rows.
|
|
const MAX_INPUT_RPC_BODY: u64 = 16 * 1024 * 1024;
|
|
|
|
pub struct HttpBackend {
|
|
agent: Agent,
|
|
base: String,
|
|
key: String,
|
|
}
|
|
|
|
impl HttpBackend {
|
|
pub fn new(config: &Config) -> Self {
|
|
let agent: Agent = Agent::config_builder()
|
|
.timeout_global(Some(Duration::from_secs(30)))
|
|
.timeout_connect(Some(Duration::from_secs(5)))
|
|
.max_redirects(0)
|
|
// Never route the service key through a proxy picked up from the environment.
|
|
.proxy(None)
|
|
.http_status_as_error(false)
|
|
.user_agent("faunosteo-worker")
|
|
.build()
|
|
.into();
|
|
Self {
|
|
agent,
|
|
base: config.url.clone(),
|
|
key: config.service_key.expose().to_owned(),
|
|
}
|
|
}
|
|
|
|
fn rpc_url(&self, name: &str) -> String {
|
|
format!("{}/rest/v1/rpc/{name}", self.base)
|
|
}
|
|
|
|
fn object_url(&self, bucket: &str, key: &str) -> BackendResult<String> {
|
|
validate_bucket(bucket)?;
|
|
keys::validate_any_key(key).map_err(|e| BackendError::Internal(e.to_string()))?;
|
|
Ok(format!("{}/storage/v1/object/{bucket}/{key}", self.base))
|
|
}
|
|
|
|
/// A sheet drawing: `<sha256>.svg` in a well-formed bucket, nothing else.
|
|
fn drawing_url(&self, bucket: &str, key: &str) -> BackendResult<String> {
|
|
validate_bucket(bucket)?;
|
|
if !keys::is_drawing_key(key) {
|
|
return Err(BackendError::Internal("drawing key refused".to_owned()));
|
|
}
|
|
Ok(format!("{}/storage/v1/object/{bucket}/{key}", self.base))
|
|
}
|
|
|
|
/// Calls an RPC and returns the response body text of a 2xx.
|
|
fn rpc(&self, name: &str, body: &Value) -> BackendResult<String> {
|
|
self.rpc_capped(name, body, MAX_RPC_BODY)
|
|
}
|
|
|
|
/// `rpc` with its own response cap.
|
|
fn rpc_capped(&self, name: &str, body: &Value, cap: u64) -> BackendResult<String> {
|
|
let mut response = self
|
|
.agent
|
|
.post(&self.rpc_url(name))
|
|
.header("apikey", &self.key)
|
|
.header("Authorization", &format!("Bearer {}", self.key))
|
|
.header("Prefer", "return=representation")
|
|
.send_json(body)
|
|
.map_err(network)?;
|
|
let status = response.status().as_u16();
|
|
let text = read_text_capped(&mut response, cap)?;
|
|
if (200..300).contains(&status) {
|
|
Ok(text)
|
|
} else {
|
|
Err(classify(status, &text))
|
|
}
|
|
}
|
|
|
|
fn rpc_json<T: DeserializeOwned>(&self, name: &str, body: &Value) -> BackendResult<T> {
|
|
let text = self.rpc(name, body)?;
|
|
serde_json::from_str(&text).map_err(|e| BackendError::Decode(e.to_string()))
|
|
}
|
|
|
|
fn rpc_void(&self, name: &str, body: &Value) -> BackendResult<()> {
|
|
self.rpc(name, body).map(|_| ())
|
|
}
|
|
|
|
/// GET of an object, read up to `cap` bytes (`Overflow` past it).
|
|
fn get_object(&self, url: &str, cap: usize) -> BackendResult<Vec<u8>> {
|
|
let mut response = self
|
|
.agent
|
|
.get(url)
|
|
.header("apikey", &self.key)
|
|
.header("Authorization", &format!("Bearer {}", self.key))
|
|
.call()
|
|
.map_err(network)?;
|
|
let status = response.status().as_u16();
|
|
if status != 200 {
|
|
let text = read_text(&mut response).unwrap_or_default();
|
|
return Err(classify(status, &text));
|
|
}
|
|
let bytes = response
|
|
.body_mut()
|
|
.with_config()
|
|
.limit(cap as u64 + 1)
|
|
.read_to_vec()
|
|
.map_err(|e| BackendError::Network(e.to_string()))?;
|
|
if bytes.len() > cap {
|
|
return Err(BackendError::Overflow);
|
|
}
|
|
Ok(bytes)
|
|
}
|
|
}
|
|
|
|
impl Backend for HttpBackend {
|
|
fn claim_jobs(&self, max_jobs: u32) -> BackendResult<Vec<Job>> {
|
|
self.rpc_json("claim_avatar_jobs", &json!({ "max_jobs": max_jobs }))
|
|
}
|
|
|
|
fn download_quarantine(&self, bucket: &str, key: &str) -> BackendResult<Vec<u8>> {
|
|
let url = self.object_url(bucket, key)?;
|
|
self.get_object(&url, MAX_INPUT_BYTES)
|
|
}
|
|
|
|
fn put_clean(&self, bucket: &str, key: &str, bytes: &[u8]) -> BackendResult<()> {
|
|
let url = self.object_url(bucket, key)?;
|
|
let mut response = self
|
|
.agent
|
|
.post(&url)
|
|
.header("apikey", &self.key)
|
|
.header("Authorization", &format!("Bearer {}", self.key))
|
|
.header("Content-Type", "image/webp")
|
|
.header("x-upsert", "true")
|
|
.send(bytes)
|
|
.map_err(network)?;
|
|
expect_success(&mut response)
|
|
}
|
|
|
|
fn complete(
|
|
&self,
|
|
asset_id: &str,
|
|
lock_token: &str,
|
|
byte_size: usize,
|
|
sha256: &str,
|
|
) -> BackendResult<()> {
|
|
self.rpc_void(
|
|
"complete_avatar_job",
|
|
&json!({ "asset_id": asset_id, "lock_token": lock_token, "byte_size": byte_size, "sha256": sha256 }),
|
|
)
|
|
}
|
|
|
|
fn reject(&self, asset_id: &str, lock_token: &str, reason: &str) -> BackendResult<()> {
|
|
self.rpc_void(
|
|
"reject_avatar_job",
|
|
&json!({ "asset_id": asset_id, "lock_token": lock_token, "reason": reason }),
|
|
)
|
|
}
|
|
|
|
fn delete_object(&self, bucket: &str, key: &str) -> BackendResult<()> {
|
|
let url = self.object_url(bucket, key)?;
|
|
let mut response = self
|
|
.agent
|
|
.delete(&url)
|
|
.header("apikey", &self.key)
|
|
.header("Authorization", &format!("Bearer {}", self.key))
|
|
.call()
|
|
.map_err(network)?;
|
|
// A missing object is a purge already done (HTTP 404, or this Storage API's
|
|
// HTTP 400 body with statusCode 404, normalised by `classify`).
|
|
match expect_success(&mut response) {
|
|
Err(BackendError::Status(404)) => Ok(()),
|
|
other => other,
|
|
}
|
|
}
|
|
|
|
fn mark_quarantine_purged(&self, asset_id: &str) -> BackendResult<()> {
|
|
self.rpc_void("mark_quarantine_purged", &json!({ "asset_id": asset_id }))
|
|
}
|
|
|
|
fn claim_purges(&self, max_items: u32) -> BackendResult<Vec<Purge>> {
|
|
self.rpc_json("claim_purges", &json!({ "max_items": max_items }))
|
|
}
|
|
|
|
fn ack_purge(&self, purge_id: &str) -> BackendResult<()> {
|
|
self.rpc_void("ack_purge", &json!({ "purge_id": purge_id }))
|
|
}
|
|
|
|
fn claim_quarantine_leftovers(
|
|
&self,
|
|
older_than_seconds: u64,
|
|
max_items: u32,
|
|
) -> BackendResult<Vec<Leftover>> {
|
|
self.rpc_json(
|
|
"claim_quarantine_leftovers",
|
|
&json!({ "older_than_seconds": older_than_seconds, "max_items": max_items }),
|
|
)
|
|
}
|
|
|
|
fn expire_pending_uploads(&self, max_age_seconds: u64) -> BackendResult<i64> {
|
|
self.rpc_json(
|
|
"expire_pending_uploads",
|
|
&json!({ "max_age_seconds": max_age_seconds }),
|
|
)
|
|
}
|
|
}
|
|
|
|
impl VerifyBackend for HttpBackend {
|
|
fn claim_verifications(&self, max_jobs: u32) -> BackendResult<Vec<VerifyJob>> {
|
|
self.rpc_json(
|
|
"claim_sheet_verifications",
|
|
&json!({ "max_jobs": max_jobs }),
|
|
)
|
|
}
|
|
|
|
fn verification_input(
|
|
&self,
|
|
review_id: &str,
|
|
lease_token: &str,
|
|
) -> BackendResult<Option<VerifyInput>> {
|
|
let text = self.rpc_capped(
|
|
"get_sheet_verification_input",
|
|
&json!({ "review_id": review_id, "lease_token": lease_token }),
|
|
MAX_INPUT_RPC_BODY,
|
|
)?;
|
|
serde_json::from_str(&text).map_err(|e| BackendError::Decode(e.to_string()))
|
|
}
|
|
|
|
fn download_drawing(
|
|
&self,
|
|
bucket: &str,
|
|
key: &str,
|
|
max_bytes: usize,
|
|
) -> BackendResult<Vec<u8>> {
|
|
let url = self.drawing_url(bucket, key)?;
|
|
self.get_object(&url, max_bytes)
|
|
}
|
|
|
|
fn complete_verification(
|
|
&self,
|
|
review_id: &str,
|
|
lease_token: &str,
|
|
verdict: &[u8],
|
|
) -> BackendResult<CompleteAnswer> {
|
|
// The spawner already checked the bytes are a JSON object (under the output cap).
|
|
let verdict: Value =
|
|
serde_json::from_slice(verdict).map_err(|e| BackendError::Internal(e.to_string()))?;
|
|
self.rpc_json(
|
|
"complete_sheet_verification",
|
|
&json!({ "review_id": review_id, "lease_token": lease_token, "verdict": verdict }),
|
|
)
|
|
}
|
|
|
|
fn fail_verification(
|
|
&self,
|
|
review_id: &str,
|
|
lease_token: &str,
|
|
code: &str,
|
|
) -> BackendResult<FailAnswer> {
|
|
self.rpc_json(
|
|
"fail_sheet_verification",
|
|
&json!({ "review_id": review_id, "lease_token": lease_token, "code": code }),
|
|
)
|
|
}
|
|
}
|
|
|
|
fn validate_bucket(bucket: &str) -> BackendResult<()> {
|
|
let ok = !bucket.is_empty()
|
|
&& bucket
|
|
.bytes()
|
|
.all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-');
|
|
if ok {
|
|
Ok(())
|
|
} else {
|
|
Err(BackendError::Internal("bucket name refused".to_owned()))
|
|
}
|
|
}
|
|
|
|
fn network(error: ureq::Error) -> BackendError {
|
|
BackendError::Network(error.to_string())
|
|
}
|
|
|
|
fn read_text(response: &mut Response<ureq::Body>) -> BackendResult<String> {
|
|
read_text_capped(response, MAX_RPC_BODY)
|
|
}
|
|
|
|
fn read_text_capped(response: &mut Response<ureq::Body>, cap: u64) -> BackendResult<String> {
|
|
response
|
|
.body_mut()
|
|
.with_config()
|
|
.limit(cap)
|
|
.read_to_string()
|
|
.map_err(|e| BackendError::Network(e.to_string()))
|
|
}
|
|
|
|
fn expect_success(response: &mut Response<ureq::Body>) -> BackendResult<()> {
|
|
let status = response.status().as_u16();
|
|
if (200..300).contains(&status) {
|
|
return Ok(());
|
|
}
|
|
let text = read_text(response).unwrap_or_default();
|
|
Err(classify(status, &text))
|
|
}
|
|
|
|
/// A PostgREST `P0001` body names a closed refusal; a Storage API "not found" (HTTP 400
|
|
/// with `statusCode` 404 on this image family) is reported as 404; any other status as such.
|
|
pub fn classify(status: u16, body: &str) -> BackendError {
|
|
let Ok(value) = serde_json::from_str::<Value>(body) else {
|
|
return BackendError::Status(status);
|
|
};
|
|
if value.get("code").and_then(Value::as_str) == Some("P0001")
|
|
&& let Some(message) = value.get("message").and_then(Value::as_str)
|
|
&& is_closed_code(message)
|
|
{
|
|
return BackendError::Refused(message.to_owned());
|
|
}
|
|
if status == 400
|
|
&& (value.get("statusCode").and_then(Value::as_str) == Some("404")
|
|
|| value.get("error").and_then(Value::as_str) == Some("not_found"))
|
|
{
|
|
return BackendError::Status(404);
|
|
}
|
|
BackendError::Status(status)
|
|
}
|
|
|
|
/// Shape of a closed code (`invalid_state`): the server's free text never becomes a code.
|
|
fn is_closed_code(message: &str) -> bool {
|
|
!message.is_empty()
|
|
&& message.len() <= 40
|
|
&& message.bytes().all(|b| b.is_ascii_lowercase() || b == b'_')
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn p0001_bodies_become_closed_refusals_and_others_statuses() {
|
|
assert_eq!(
|
|
classify(
|
|
400,
|
|
r#"{"code":"P0001","message":"invalid_state","details":null}"#
|
|
),
|
|
BackendError::Refused("invalid_state".to_owned())
|
|
);
|
|
assert_eq!(
|
|
classify(403, r#"{"code":"42501","message":"denied"}"#),
|
|
BackendError::Status(403)
|
|
);
|
|
assert_eq!(
|
|
classify(400, r#"{"code":"P0001","message":"Free text!"}"#),
|
|
BackendError::Status(400)
|
|
);
|
|
assert_eq!(classify(500, "not json"), BackendError::Status(500));
|
|
// storage-api v1.79 answers a missing object with HTTP 400 and statusCode "404".
|
|
assert_eq!(
|
|
classify(
|
|
400,
|
|
r#"{"statusCode":"404","error":"not_found","message":"Object not found"}"#
|
|
),
|
|
BackendError::Status(404)
|
|
);
|
|
assert_eq!(
|
|
classify(
|
|
400,
|
|
r#"{"statusCode":"400","error":"InvalidRequest","message":"x"}"#
|
|
),
|
|
BackendError::Status(400)
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn the_m6_answers_and_rows_deserialize() {
|
|
assert_eq!(
|
|
serde_json::from_str::<CompleteAnswer>("\"malformed\"").ok(),
|
|
Some(CompleteAnswer::Malformed)
|
|
);
|
|
assert_eq!(
|
|
serde_json::from_str::<FailAnswer>("\"retry\"").ok(),
|
|
Some(FailAnswer::Retry)
|
|
);
|
|
assert!(serde_json::from_str::<CompleteAnswer>("\"other\"").is_err());
|
|
assert_eq!(
|
|
serde_json::from_str::<Option<VerifyInput>>("null").ok(),
|
|
Some(None)
|
|
);
|
|
let input = serde_json::from_value::<Option<VerifyInput>>(json!({
|
|
"review_id": "r", "version_id": "v", "sheet_id": "s", "revision": 2,
|
|
"document": "{}", "base_document": null,
|
|
"drawings": [{ "asset_id": "a", "sha256": "h", "byte_size": 10,
|
|
"bucket": "anatomical-sheet-assets", "object_key": "h.svg", "uploaded": true }]
|
|
}));
|
|
assert_eq!(
|
|
input.ok().flatten().map(|i| i.drawings[0].byte_size),
|
|
Some(10)
|
|
);
|
|
let jobs = serde_json::from_value::<Vec<VerifyJob>>(json!([{ "review_id": "r",
|
|
"version_id": "v", "sheet_id": "s", "revision": 1, "lease_token": "t", "attempts": 1 }]));
|
|
assert_eq!(jobs.ok().map(|j| j[0].revision), Some(1));
|
|
}
|
|
|
|
#[test]
|
|
fn object_urls_need_valid_buckets_and_keys() {
|
|
let config = Config::from_vars(|name| match name {
|
|
"SUPABASE_URL" => Some("http://kong:8000".to_owned()),
|
|
"SUPABASE_SERVICE_ROLE_KEY" => Some("k".to_owned()),
|
|
_ => None,
|
|
})
|
|
.ok();
|
|
let Some(config) = config else {
|
|
panic!("config");
|
|
};
|
|
let backend = HttpBackend::new(&config);
|
|
let key = "profiles/10000000-0000-0000-0000-00000000000a/10000000-0000-0000-0000-00000000000b/avatar.webp";
|
|
assert_eq!(
|
|
backend.object_url("avatars", key).ok().as_deref(),
|
|
Some(
|
|
"http://kong:8000/storage/v1/object/avatars/profiles/10000000-0000-0000-0000-00000000000a/10000000-0000-0000-0000-00000000000b/avatar.webp"
|
|
)
|
|
);
|
|
assert!(backend.object_url("avatars/../x", key).is_err());
|
|
assert!(
|
|
backend
|
|
.object_url("avatars", "profiles/../etc/passwd")
|
|
.is_err()
|
|
);
|
|
let drawing = format!("{}.svg", "ab".repeat(32));
|
|
assert_eq!(
|
|
backend
|
|
.drawing_url("anatomical-sheet-assets", &drawing)
|
|
.ok(),
|
|
Some(format!(
|
|
"http://kong:8000/storage/v1/object/anatomical-sheet-assets/{drawing}"
|
|
))
|
|
);
|
|
assert!(backend.drawing_url("anatomical-sheet-assets", key).is_err());
|
|
assert!(backend.drawing_url("a/b", &drawing).is_err());
|
|
assert_eq!(
|
|
backend.rpc_url("claim_avatar_jobs"),
|
|
"http://kong:8000/rest/v1/rpc/claim_avatar_jobs"
|
|
);
|
|
}
|
|
}
|