Keyed parent on ureq 3 (rustls + ring, bundled roots), no async runtime: config from the environment (secret never printed, JWT expiry logged), JSON line logs with a field allowlist (child stderr escaped and truncated), strict key validation before any URL, Backend trait over PostgREST RPCs and Storage objects with closed error kinds, one job per claim with the STORAGE.md §3.4 mapping (content codes, processing_error on signals / timeout / overflow, Broken stops the loop with exit 20, a refused commit deletes nothing), sweeper (expire pending, purge queue, quarantine leftovers), per-stage watchdog (exit 21), WORKER_ONCE for tests, run --check extended. 11 loop tests on a fake backend and decoder; sandbox tests and seccomp derivation unchanged (the child still makes the same 7 syscalls); image 11.6 MB.
114 lines
3.3 KiB
Rust
114 lines
3.3 KiB
Rust
//! Per-stage deadlines of the parent (STORAGE.md §3.4): the loop announces the stage it
|
|
//! enters; a monitor thread exits the process when a stage outlives its deadline, so a hung
|
|
//! network call or a wedged child never burns leases silently. Docker restarts the container.
|
|
|
|
use std::sync::{Arc, Mutex};
|
|
use std::time::{Duration, Instant};
|
|
|
|
use crate::{exit_codes, log};
|
|
|
|
/// Deadlines per stage (review Part IV.1).
|
|
pub mod deadlines {
|
|
use std::time::Duration;
|
|
pub const CLAIM: Duration = Duration::from_secs(35);
|
|
pub const DOWNLOAD: Duration = Duration::from_secs(35);
|
|
pub const DECODE: Duration = Duration::from_secs(25);
|
|
pub const UPLOAD: Duration = Duration::from_secs(35);
|
|
pub const COMMIT: Duration = Duration::from_secs(35);
|
|
pub const SWEEP: Duration = Duration::from_secs(120);
|
|
}
|
|
|
|
#[derive(Debug, Clone)]
|
|
struct Stage {
|
|
name: &'static str,
|
|
started: Instant,
|
|
deadline: Duration,
|
|
}
|
|
|
|
/// Shared stage state; cloning shares it.
|
|
#[derive(Debug, Clone, Default)]
|
|
pub struct Watchdog {
|
|
current: Arc<Mutex<Option<Stage>>>,
|
|
}
|
|
|
|
/// Clears the stage when dropped.
|
|
pub struct StageGuard {
|
|
watchdog: Watchdog,
|
|
}
|
|
|
|
impl Drop for StageGuard {
|
|
fn drop(&mut self) {
|
|
self.watchdog.set(None);
|
|
}
|
|
}
|
|
|
|
impl Watchdog {
|
|
pub fn new() -> Self {
|
|
Self::default()
|
|
}
|
|
|
|
fn set(&self, stage: Option<Stage>) {
|
|
match self.current.lock() {
|
|
Ok(mut guard) => *guard = stage,
|
|
Err(poisoned) => *poisoned.into_inner() = stage,
|
|
}
|
|
}
|
|
|
|
/// Announces a stage; the guard ends it.
|
|
pub fn enter(&self, name: &'static str, deadline: Duration) -> StageGuard {
|
|
self.set(Some(Stage {
|
|
name,
|
|
started: Instant::now(),
|
|
deadline,
|
|
}));
|
|
StageGuard {
|
|
watchdog: self.clone(),
|
|
}
|
|
}
|
|
|
|
/// Name of the stage past its deadline, if any.
|
|
pub fn overrun(&self) -> Option<&'static str> {
|
|
let guard = match self.current.lock() {
|
|
Ok(guard) => guard,
|
|
Err(poisoned) => poisoned.into_inner(),
|
|
};
|
|
guard
|
|
.as_ref()
|
|
.filter(|stage| stage.started.elapsed() > stage.deadline)
|
|
.map(|stage| stage.name)
|
|
}
|
|
|
|
/// Starts the monitor thread: checks every second, exits the process on overrun.
|
|
pub fn spawn_monitor(&self) {
|
|
let watchdog = self.clone();
|
|
std::thread::spawn(move || {
|
|
loop {
|
|
std::thread::sleep(Duration::from_secs(1));
|
|
if let Some(stage) = watchdog.overrun() {
|
|
log::error("watchdog_overrun").stage(stage).emit();
|
|
std::process::exit(exit_codes::WATCHDOG);
|
|
}
|
|
}
|
|
});
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn a_stage_overruns_only_past_its_deadline_and_clears_on_drop() {
|
|
let watchdog = Watchdog::new();
|
|
assert_eq!(watchdog.overrun(), None);
|
|
{
|
|
let _guard = watchdog.enter("claim", Duration::from_millis(0));
|
|
std::thread::sleep(Duration::from_millis(5));
|
|
assert_eq!(watchdog.overrun(), Some("claim"));
|
|
}
|
|
assert_eq!(watchdog.overrun(), None);
|
|
let _guard = watchdog.enter("download", Duration::from_secs(60));
|
|
assert_eq!(watchdog.overrun(), None);
|
|
}
|
|
}
|