feat(worker): R1 job loop — claim, download, sandboxed decode, clean upload, commit, purge sweeps
Keyed parent on ureq 3 (rustls + ring, bundled roots), no async runtime: config from the environment (secret never printed, JWT expiry logged), JSON line logs with a field allowlist (child stderr escaped and truncated), strict key validation before any URL, Backend trait over PostgREST RPCs and Storage objects with closed error kinds, one job per claim with the STORAGE.md §3.4 mapping (content codes, processing_error on signals / timeout / overflow, Broken stops the loop with exit 20, a refused commit deletes nothing), sweeper (expire pending, purge queue, quarantine leftovers), per-stage watchdog (exit 21), WORKER_ONCE for tests, run --check extended. 11 loop tests on a fake backend and decoder; sandbox tests and seccomp derivation unchanged (the child still makes the same 7 syscalls); image 11.6 MB.
This commit is contained in:
1 parent
926dd7df8c
commit
0410af0c35
18 files changed
+2921
-42
No files matched your search
@@ -0,0 +1,284 @@
|
||||
//! Configuration of the keyed parent (STORAGE.md §3.2): read from the environment once at
|
||||
//! start, never from the child. The service key is a `Secret` that no `Debug` or `Display`
|
||||
//! output can print.
|
||||
|
||||
use std::fmt;
|
||||
use std::time::Duration;
|
||||
|
||||
/// A value that must never appear in a log line or an error message.
|
||||
#[derive(Clone)]
|
||||
pub struct Secret(String);
|
||||
|
||||
impl Secret {
|
||||
pub fn new(value: String) -> Self {
|
||||
Self(value)
|
||||
}
|
||||
|
||||
/// The only accessor; call sites are the HTTP headers and the JWT expiry parser.
|
||||
pub fn expose(&self) -> &str {
|
||||
&self.0
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Debug for Secret {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str("<redacted>")
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolved worker configuration.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Config {
|
||||
/// Origin of the Supabase stack, without a trailing slash.
|
||||
pub url: String,
|
||||
pub service_key: Secret,
|
||||
pub worker_id: String,
|
||||
pub poll: Duration,
|
||||
pub sweep: Duration,
|
||||
pub quarantine_grace: Duration,
|
||||
pub pending_max_age: Duration,
|
||||
/// One claim pass and one sweep, then exit (tests).
|
||||
pub once: bool,
|
||||
}
|
||||
|
||||
/// Why the configuration could not be built (the variable name, never its value).
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum ConfigError {
|
||||
Missing(&'static str),
|
||||
Invalid(&'static str),
|
||||
}
|
||||
|
||||
impl fmt::Display for ConfigError {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
match self {
|
||||
Self::Missing(name) => write!(f, "{name} is required"),
|
||||
Self::Invalid(name) => write!(f, "{name} is invalid"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for ConfigError {}
|
||||
|
||||
const URL: &str = "SUPABASE_URL";
|
||||
const KEY: &str = "SUPABASE_SERVICE_ROLE_KEY";
|
||||
const WORKER_ID: &str = "WORKER_ID";
|
||||
const POLL: &str = "WORKER_POLL_SECONDS";
|
||||
const SWEEP: &str = "WORKER_SWEEP_SECONDS";
|
||||
const QUARANTINE_GRACE: &str = "WORKER_QUARANTINE_GRACE_SECONDS";
|
||||
const PENDING_MAX_AGE: &str = "WORKER_PENDING_MAX_AGE_SECONDS";
|
||||
const ONCE: &str = "WORKER_ONCE";
|
||||
|
||||
/// Names of the variables the worker reads, for `run --check`.
|
||||
pub const VARIABLES: &[&str] = &[
|
||||
URL,
|
||||
KEY,
|
||||
WORKER_ID,
|
||||
POLL,
|
||||
SWEEP,
|
||||
QUARANTINE_GRACE,
|
||||
PENDING_MAX_AGE,
|
||||
ONCE,
|
||||
];
|
||||
|
||||
impl Config {
|
||||
/// Builds the configuration from the process environment.
|
||||
pub fn from_env() -> Result<Self, ConfigError> {
|
||||
Self::from_vars(|name| std::env::var(name).ok())
|
||||
}
|
||||
|
||||
/// Builds the configuration from any variable source (tests).
|
||||
pub fn from_vars(var: impl Fn(&str) -> Option<String>) -> Result<Self, ConfigError> {
|
||||
let url = parse_url(var(URL).ok_or(ConfigError::Missing(URL))?)?;
|
||||
let key = var(KEY).filter(|k| !k.trim().is_empty());
|
||||
let service_key = Secret::new(key.ok_or(ConfigError::Missing(KEY))?);
|
||||
let worker_id = match var(WORKER_ID).filter(|id| !id.trim().is_empty()) {
|
||||
Some(id)
|
||||
if id.len() <= 64
|
||||
&& id
|
||||
.chars()
|
||||
.all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_') =>
|
||||
{
|
||||
id
|
||||
}
|
||||
Some(_) => return Err(ConfigError::Invalid(WORKER_ID)),
|
||||
None => format!("worker-{}", std::process::id()),
|
||||
};
|
||||
Ok(Self {
|
||||
url,
|
||||
service_key,
|
||||
worker_id,
|
||||
poll: seconds(&var, POLL, 2)?,
|
||||
sweep: seconds(&var, SWEEP, 600)?,
|
||||
quarantine_grace: seconds(&var, QUARANTINE_GRACE, 10_800)?,
|
||||
pending_max_age: seconds(&var, PENDING_MAX_AGE, 86_400)?,
|
||||
once: var(ONCE).is_some_and(|v| v == "1"),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// `http(s)://host[:port]`, no path, no whitespace; a trailing slash is dropped.
|
||||
fn parse_url(raw: String) -> Result<String, ConfigError> {
|
||||
let url = raw.trim().trim_end_matches('/').to_owned();
|
||||
let rest = url
|
||||
.strip_prefix("http://")
|
||||
.or_else(|| url.strip_prefix("https://"))
|
||||
.ok_or(ConfigError::Invalid(URL))?;
|
||||
let host_ok = !rest.is_empty()
|
||||
&& rest
|
||||
.chars()
|
||||
.all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '-' | ':' | '_'));
|
||||
if host_ok {
|
||||
Ok(url)
|
||||
} else {
|
||||
Err(ConfigError::Invalid(URL))
|
||||
}
|
||||
}
|
||||
|
||||
fn seconds(
|
||||
var: &impl Fn(&str) -> Option<String>,
|
||||
name: &'static str,
|
||||
default: u64,
|
||||
) -> Result<Duration, ConfigError> {
|
||||
match var(name) {
|
||||
None => Ok(Duration::from_secs(default)),
|
||||
Some(raw) => raw
|
||||
.trim()
|
||||
.parse::<u64>()
|
||||
.map(Duration::from_secs)
|
||||
.map_err(|_| ConfigError::Invalid(name)),
|
||||
}
|
||||
}
|
||||
|
||||
/// `exp` claim of a JWT (three base64url parts), when the key is one. A `sb_secret_…`
|
||||
/// key is not a JWT and yields `None`.
|
||||
pub fn jwt_exp(token: &str) -> Option<i64> {
|
||||
let mut parts = token.split('.');
|
||||
let (_, payload, _) = (parts.next()?, parts.next()?, parts.next()?);
|
||||
if parts.next().is_some() {
|
||||
return None;
|
||||
}
|
||||
let json = base64url_decode(payload)?;
|
||||
let value: serde_json::Value = serde_json::from_slice(&json).ok()?;
|
||||
value.get("exp")?.as_i64()
|
||||
}
|
||||
|
||||
/// Base64url without padding (RFC 7515); padding characters are tolerated.
|
||||
pub fn base64url_decode(text: &str) -> Option<Vec<u8>> {
|
||||
let mut out = Vec::with_capacity(text.len() * 3 / 4);
|
||||
let mut buffer: u32 = 0;
|
||||
let mut bits = 0;
|
||||
for byte in text.bytes() {
|
||||
let value = match byte {
|
||||
b'A'..=b'Z' => byte - b'A',
|
||||
b'a'..=b'z' => byte - b'a' + 26,
|
||||
b'0'..=b'9' => byte - b'0' + 52,
|
||||
b'-' => 62,
|
||||
b'_' => 63,
|
||||
b'=' => break,
|
||||
_ => return None,
|
||||
};
|
||||
buffer = (buffer << 6) | u32::from(value);
|
||||
bits += 6;
|
||||
if bits >= 8 {
|
||||
bits -= 8;
|
||||
out.push(u8::try_from((buffer >> bits) & 0xff).ok()?);
|
||||
}
|
||||
}
|
||||
Some(out)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
#![allow(clippy::unwrap_used)]
|
||||
use super::*;
|
||||
use std::collections::HashMap;
|
||||
|
||||
fn vars(pairs: &[(&str, &str)]) -> impl Fn(&str) -> Option<String> {
|
||||
let map: HashMap<String, String> = pairs
|
||||
.iter()
|
||||
.map(|(k, v)| ((*k).to_owned(), (*v).to_owned()))
|
||||
.collect();
|
||||
move |name| map.get(name).cloned()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn defaults_apply_when_only_the_required_variables_are_set() {
|
||||
let cfg = Config::from_vars(vars(&[
|
||||
("SUPABASE_URL", "http://127.0.0.1:54321/"),
|
||||
("SUPABASE_SERVICE_ROLE_KEY", "k"),
|
||||
]))
|
||||
.unwrap();
|
||||
assert_eq!(cfg.url, "http://127.0.0.1:54321");
|
||||
assert_eq!(cfg.poll, Duration::from_secs(2));
|
||||
assert_eq!(cfg.sweep, Duration::from_secs(600));
|
||||
assert_eq!(cfg.quarantine_grace, Duration::from_secs(10_800));
|
||||
assert_eq!(cfg.pending_max_age, Duration::from_secs(86_400));
|
||||
assert!(!cfg.once);
|
||||
assert!(cfg.worker_id.starts_with("worker-"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_and_invalid_variables_are_named() {
|
||||
assert_eq!(
|
||||
Config::from_vars(vars(&[("SUPABASE_SERVICE_ROLE_KEY", "k")])).err(),
|
||||
Some(ConfigError::Missing("SUPABASE_URL"))
|
||||
);
|
||||
assert_eq!(
|
||||
Config::from_vars(vars(&[("SUPABASE_URL", "http://x")])).err(),
|
||||
Some(ConfigError::Missing("SUPABASE_SERVICE_ROLE_KEY"))
|
||||
);
|
||||
assert_eq!(
|
||||
Config::from_vars(vars(&[
|
||||
("SUPABASE_URL", "ftp://x"),
|
||||
("SUPABASE_SERVICE_ROLE_KEY", "k")
|
||||
]))
|
||||
.err(),
|
||||
Some(ConfigError::Invalid("SUPABASE_URL"))
|
||||
);
|
||||
assert_eq!(
|
||||
Config::from_vars(vars(&[
|
||||
("SUPABASE_URL", "http://x"),
|
||||
("SUPABASE_SERVICE_ROLE_KEY", "k"),
|
||||
("WORKER_POLL_SECONDS", "two"),
|
||||
]))
|
||||
.err(),
|
||||
Some(ConfigError::Invalid("WORKER_POLL_SECONDS"))
|
||||
);
|
||||
assert_eq!(
|
||||
Config::from_vars(vars(&[
|
||||
("SUPABASE_URL", "http://x"),
|
||||
("SUPABASE_SERVICE_ROLE_KEY", "k"),
|
||||
("WORKER_ID", "bad id"),
|
||||
]))
|
||||
.err(),
|
||||
Some(ConfigError::Invalid("WORKER_ID"))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_key_never_appears_in_debug_output() {
|
||||
let cfg = Config::from_vars(vars(&[
|
||||
("SUPABASE_URL", "http://x"),
|
||||
("SUPABASE_SERVICE_ROLE_KEY", "super-secret-value-9f8e7d"),
|
||||
("WORKER_ONCE", "1"),
|
||||
]))
|
||||
.unwrap();
|
||||
let printed = format!("{cfg:?}");
|
||||
assert!(!printed.contains("super-secret-value-9f8e7d"));
|
||||
assert!(printed.contains("<redacted>"));
|
||||
assert!(cfg.once);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn jwt_exp_reads_the_payload_claim() {
|
||||
// header {"alg":"HS256"} . payload {"role":"service_role","exp":1983812996} . sig
|
||||
let token =
|
||||
"eyJhbGciOiJIUzI1NiJ9.eyJyb2xlIjoic2VydmljZV9yb2xlIiwiZXhwIjoxOTgzODEyOTk2fQ.sig";
|
||||
assert_eq!(jwt_exp(token), Some(1_983_812_996));
|
||||
assert_eq!(jwt_exp("sb_secret_abc"), None);
|
||||
assert_eq!(jwt_exp("a.b"), None);
|
||||
assert_eq!(base64url_decode("aGk"), Some(b"hi".to_vec()));
|
||||
assert_eq!(base64url_decode("aGk="), Some(b"hi".to_vec()));
|
||||
assert_eq!(base64url_decode("a*"), None);
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user