116 lines
3.4 KiB
Rust
116 lines
3.4 KiB
Rust
//! Per-stage deadlines of the parent (STORAGE.md §3.4): the loop announces the stage it
|
|
//! enters; a monitor thread exits the process when a stage outlives its deadline, so a hung
|
|
//! network call or a wedged child never burns leases silently. Docker restarts the container.
|
|
|
|
use std::sync::{Arc, Mutex};
|
|
use std::time::{Duration, Instant};
|
|
|
|
use crate::{exit_codes, log};
|
|
|
|
/// Deadlines per stage (review Part IV.1).
|
|
pub mod deadlines {
|
|
use std::time::Duration;
|
|
pub const CLAIM: Duration = Duration::from_secs(35);
|
|
pub const DOWNLOAD: Duration = Duration::from_secs(35);
|
|
pub const DECODE: Duration = Duration::from_secs(25);
|
|
/// Above the verify child's own deadline (`limits::VERIFY_DEADLINE`, 90 s).
|
|
pub const VERIFY: Duration = Duration::from_secs(95);
|
|
pub const UPLOAD: Duration = Duration::from_secs(35);
|
|
pub const COMMIT: Duration = Duration::from_secs(35);
|
|
pub const SWEEP: Duration = Duration::from_secs(120);
|
|
}
|
|
|
|
#[derive(Debug, Clone)]
|
|
struct Stage {
|
|
name: &'static str,
|
|
started: Instant,
|
|
deadline: Duration,
|
|
}
|
|
|
|
/// Shared stage state; cloning shares it.
|
|
#[derive(Debug, Clone, Default)]
|
|
pub struct Watchdog {
|
|
current: Arc<Mutex<Option<Stage>>>,
|
|
}
|
|
|
|
/// Clears the stage when dropped.
|
|
pub struct StageGuard {
|
|
watchdog: Watchdog,
|
|
}
|
|
|
|
impl Drop for StageGuard {
|
|
fn drop(&mut self) {
|
|
self.watchdog.set(None);
|
|
}
|
|
}
|
|
|
|
impl Watchdog {
|
|
pub fn new() -> Self {
|
|
Self::default()
|
|
}
|
|
|
|
fn set(&self, stage: Option<Stage>) {
|
|
match self.current.lock() {
|
|
Ok(mut guard) => *guard = stage,
|
|
Err(poisoned) => *poisoned.into_inner() = stage,
|
|
}
|
|
}
|
|
|
|
/// Announces a stage; the guard ends it.
|
|
pub fn enter(&self, name: &'static str, deadline: Duration) -> StageGuard {
|
|
self.set(Some(Stage {
|
|
name,
|
|
started: Instant::now(),
|
|
deadline,
|
|
}));
|
|
StageGuard {
|
|
watchdog: self.clone(),
|
|
}
|
|
}
|
|
|
|
/// Name of the stage past its deadline, if any.
|
|
pub fn overrun(&self) -> Option<&'static str> {
|
|
let guard = match self.current.lock() {
|
|
Ok(guard) => guard,
|
|
Err(poisoned) => poisoned.into_inner(),
|
|
};
|
|
guard
|
|
.as_ref()
|
|
.filter(|stage| stage.started.elapsed() > stage.deadline)
|
|
.map(|stage| stage.name)
|
|
}
|
|
|
|
/// Starts the monitor thread: checks every second, exits the process on overrun.
|
|
pub fn spawn_monitor(&self) {
|
|
let watchdog = self.clone();
|
|
std::thread::spawn(move || {
|
|
loop {
|
|
std::thread::sleep(Duration::from_secs(1));
|
|
if let Some(stage) = watchdog.overrun() {
|
|
log::error("watchdog_overrun").stage(stage).emit();
|
|
std::process::exit(exit_codes::WATCHDOG);
|
|
}
|
|
}
|
|
});
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn a_stage_overruns_only_past_its_deadline_and_clears_on_drop() {
|
|
let watchdog = Watchdog::new();
|
|
assert_eq!(watchdog.overrun(), None);
|
|
{
|
|
let _guard = watchdog.enter("claim", Duration::from_millis(0));
|
|
std::thread::sleep(Duration::from_millis(5));
|
|
assert_eq!(watchdog.overrun(), Some("claim"));
|
|
}
|
|
assert_eq!(watchdog.overrun(), None);
|
|
let _guard = watchdog.enter("download", Duration::from_secs(60));
|
|
assert_eq!(watchdog.overrun(), None);
|
|
}
|
|
}
|