Files
faunosteo/crates/worker/src/watchdog.rs
T

116 lines
3.4 KiB
Rust

//! Per-stage deadlines of the parent (STORAGE.md §3.4): the loop announces the stage it
//! enters; a monitor thread exits the process when a stage outlives its deadline, so a hung
//! network call or a wedged child never burns leases silently. Docker restarts the container.
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};
use crate::{exit_codes, log};
/// Deadlines per stage (review Part IV.1).
pub mod deadlines {
use std::time::Duration;
pub const CLAIM: Duration = Duration::from_secs(35);
pub const DOWNLOAD: Duration = Duration::from_secs(35);
pub const DECODE: Duration = Duration::from_secs(25);
/// Above the verify child's own deadline (`limits::VERIFY_DEADLINE`, 90 s).
pub const VERIFY: Duration = Duration::from_secs(95);
pub const UPLOAD: Duration = Duration::from_secs(35);
pub const COMMIT: Duration = Duration::from_secs(35);
pub const SWEEP: Duration = Duration::from_secs(120);
}
#[derive(Debug, Clone)]
struct Stage {
name: &'static str,
started: Instant,
deadline: Duration,
}
/// Shared stage state; cloning shares it.
#[derive(Debug, Clone, Default)]
pub struct Watchdog {
current: Arc<Mutex<Option<Stage>>>,
}
/// Clears the stage when dropped.
pub struct StageGuard {
watchdog: Watchdog,
}
impl Drop for StageGuard {
fn drop(&mut self) {
self.watchdog.set(None);
}
}
impl Watchdog {
pub fn new() -> Self {
Self::default()
}
fn set(&self, stage: Option<Stage>) {
match self.current.lock() {
Ok(mut guard) => *guard = stage,
Err(poisoned) => *poisoned.into_inner() = stage,
}
}
/// Announces a stage; the guard ends it.
pub fn enter(&self, name: &'static str, deadline: Duration) -> StageGuard {
self.set(Some(Stage {
name,
started: Instant::now(),
deadline,
}));
StageGuard {
watchdog: self.clone(),
}
}
/// Name of the stage past its deadline, if any.
pub fn overrun(&self) -> Option<&'static str> {
let guard = match self.current.lock() {
Ok(guard) => guard,
Err(poisoned) => poisoned.into_inner(),
};
guard
.as_ref()
.filter(|stage| stage.started.elapsed() > stage.deadline)
.map(|stage| stage.name)
}
/// Starts the monitor thread: checks every second, exits the process on overrun.
pub fn spawn_monitor(&self) {
let watchdog = self.clone();
std::thread::spawn(move || {
loop {
std::thread::sleep(Duration::from_secs(1));
if let Some(stage) = watchdog.overrun() {
log::error("watchdog_overrun").stage(stage).emit();
std::process::exit(exit_codes::WATCHDOG);
}
}
});
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn a_stage_overruns_only_past_its_deadline_and_clears_on_drop() {
let watchdog = Watchdog::new();
assert_eq!(watchdog.overrun(), None);
{
let _guard = watchdog.enter("claim", Duration::from_millis(0));
std::thread::sleep(Duration::from_millis(5));
assert_eq!(watchdog.overrun(), Some("claim"));
}
assert_eq!(watchdog.overrun(), None);
let _guard = watchdog.enter("download", Duration::from_secs(60));
assert_eq!(watchdog.overrun(), None);
}
}