//! `faunosteo-worker run`: the keyed parent's loop (STORAGE.md §3.4, §3.5), `run --jobs //! sheet-verify` (the verification loop of V1-2.3, no sweep) and `run --check`, the start-up //! self-description without any secret value. use std::io::Write; use std::time::{Instant, SystemTime, UNIX_EPOCH}; use crate::backend::{Backend, VerifyBackend}; use crate::config::{self, Config, ConfigError}; use crate::decode::libwebp_version; use crate::decoder::{ChildDecoder, Decoder}; use crate::http::HttpBackend; use crate::job::{JobEnd, process_job}; use crate::limits::MIN_LIBWEBP_VERSION; use crate::sweep::{SweepConfig, run_sweep}; use crate::verifier::{ChildVerifier, Verifier}; use crate::verify_job::{VerifyEnd, process_verification}; use crate::watchdog::{Watchdog, deadlines}; use crate::{exit_codes, log}; const THIRTY_DAYS: i64 = 30 * 24 * 3600; /// What a `run` process works on. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum Mode { /// Avatar jobs and the sweep (the default). Avatars, /// Sheet verification rounds only (`--jobs sheet-verify`). SheetVerify, } /// Entry of the `run` subcommand; returns the process exit code. pub fn main(args: &[String]) -> i32 { match parse_args(args) { Ok((mode, true)) => check(mode), Ok((Mode::Avatars, false)) => start(), Ok((Mode::SheetVerify, false)) => start_verify(), Err(other) => { eprintln!( "run: unknown argument {other} (use --check to print the configuration, --jobs sheet-verify for the verification mode)" ); exit_codes::USAGE } } } /// `[--jobs sheet-verify] [--check]` in any order; `Err` names the first unknown argument. pub fn parse_args(args: &[String]) -> Result<(Mode, bool), String> { let mut mode = Mode::Avatars; let mut check = false; let mut iter = args.iter(); while let Some(arg) = iter.next() { match arg.as_str() { "--check" if !check => check = true, "--jobs" if mode == Mode::Avatars => match iter.next().map(String::as_str) { Some("sheet-verify") => mode = Mode::SheetVerify, Some(value) => return Err(format!("--jobs {value}")), None => return Err("--jobs".to_owned()), }, other => return Err(other.to_owned()), } } Ok((mode, check)) } fn start() -> i32 { let config = match Config::from_env() { Ok(config) => config, Err(error) => { eprintln!("run: {error}"); return exit_codes::USAGE; } }; if !libwebp_ok() { eprintln!("run: linked libwebp below the minimum, refusing to start"); return exit_codes::INTERNAL; } log::set_worker_id(&config.worker_id); log_start(&config); log_token_expiry(&config); let decoder = match ChildDecoder::from_current_exe() { Ok(decoder) => decoder, Err(error) => { log::error("start_failed") .detail(&format!("current_exe: {error}")) .emit(); return exit_codes::INTERNAL; } }; let backend = HttpBackend::new(&config); let watchdog = Watchdog::new(); watchdog.spawn_monitor(); run_loop(&config, &backend, &decoder, &watchdog) } fn start_verify() -> i32 { let config = match Config::from_env() { Ok(config) => config, Err(error) => { eprintln!("run: {error}"); return exit_codes::USAGE; } }; log::set_worker_id(&config.worker_id); log::info("worker_start") .detail(&format!( "mode sheet-verify; poll {}s once {}", config.poll.as_secs(), config.once )) .emit(); log_token_expiry(&config); let verifier = match ChildVerifier::from_current_exe() { Ok(verifier) => verifier, Err(error) => { log::error("start_failed") .detail(&format!("current_exe: {error}")) .emit(); return exit_codes::INTERNAL; } }; let backend = HttpBackend::new(&config); let watchdog = Watchdog::new(); watchdog.spawn_monitor(); verify_loop(&config, &backend, &verifier, &watchdog) } /// Claims one round per pass, no sweep (the avatar worker owns it); idles on `poll`. pub fn verify_loop( config: &Config, backend: &dyn VerifyBackend, verifier: &dyn Verifier, watchdog: &Watchdog, ) -> i32 { loop { let worked = match verify_pass(backend, verifier, watchdog) { Ok(count) => count > 0, Err(code) => return code, }; if config.once { return exit_codes::OK; } if !worked { std::thread::sleep(config.poll); } } } /// One claim and the rounds it returned; `Err` carries the exit code of a stop. fn verify_pass( backend: &dyn VerifyBackend, verifier: &dyn Verifier, watchdog: &Watchdog, ) -> Result { let jobs = { let _stage = watchdog.enter("claim", deadlines::CLAIM); match backend.claim_verifications(1) { Ok(jobs) => jobs, Err(error) => { log_claim_failure(&error); return Ok(0); } } }; for job in &jobs { if process_verification(backend, verifier, watchdog, job) == VerifyEnd::Stop { return Err(exit_codes::CHILD_BROKEN); } } Ok(jobs.len()) } /// Sweep at start and every `sweep` interval; claim one job per pass; idle on `poll`. pub fn run_loop( config: &Config, backend: &dyn Backend, decoder: &dyn Decoder, watchdog: &Watchdog, ) -> i32 { let sweep_config = SweepConfig { pending_max_age: config.pending_max_age, quarantine_grace: config.quarantine_grace, }; let mut next_sweep = Instant::now(); loop { if Instant::now() >= next_sweep { run_sweep(backend, watchdog, sweep_config); next_sweep = Instant::now() + config.sweep; } let worked = match claim_pass(backend, decoder, watchdog) { Ok(count) => count > 0, Err(code) => return code, }; if config.once { return exit_codes::OK; } if !worked { std::thread::sleep(config.poll); } } } /// One claim and the jobs it returned; `Err` carries the exit code of a stop. fn claim_pass( backend: &dyn Backend, decoder: &dyn Decoder, watchdog: &Watchdog, ) -> Result { let jobs = { let _stage = watchdog.enter("claim", deadlines::CLAIM); match backend.claim_jobs(1) { Ok(jobs) => jobs, Err(error) => { log_claim_failure(&error); return Ok(0); } } }; for job in &jobs { if process_job(backend, decoder, watchdog, job) == JobEnd::Stop { return Err(exit_codes::CHILD_BROKEN); } } Ok(jobs.len()) } fn log_claim_failure(error: &crate::backend::BackendError) { let line = log::warn("claim_failed").code(error.kind()); match error { crate::backend::BackendError::Status(status) => line.http_status(*status).emit(), _ => line.emit(), } } fn log_start(config: &Config) { let (major, minor, rev) = libwebp_version(); log::info("worker_start") .detail(&format!( "libwebp {major}.{minor}.{rev}; poll {}s sweep {}s grace {}s pending {}s once {}", config.poll.as_secs(), config.sweep.as_secs(), config.quarantine_grace.as_secs(), config.pending_max_age.as_secs(), config.once )) .emit(); } fn log_token_expiry(config: &Config) { match token_expiry(config) { TokenExpiry::NotJwt => log::info("token_no_expiry").emit(), TokenExpiry::Valid(days) => log::info("token_expiry").count(days).emit(), TokenExpiry::Soon(days) => log::warn("token_expires_soon").count(days).emit(), TokenExpiry::Expired => log::error("token_expired").emit(), } } enum TokenExpiry { NotJwt, Valid(usize), Soon(usize), Expired, } fn token_expiry(config: &Config) -> TokenExpiry { let Some(exp) = config::jwt_exp(config.service_key.expose()) else { return TokenExpiry::NotJwt; }; let now = SystemTime::now() .duration_since(UNIX_EPOCH) .map(|d| i64::try_from(d.as_secs()).unwrap_or(i64::MAX)) .unwrap_or(0); let left = exp - now; let days = usize::try_from(left / 86_400).unwrap_or(0); if left <= 0 { TokenExpiry::Expired } else if left < THIRTY_DAYS { TokenExpiry::Soon(days) } else { TokenExpiry::Valid(days) } } fn libwebp_ok() -> bool { libwebp_version() >= MIN_LIBWEBP_VERSION } /// Prints the resolved configuration without any value of a secret. Exit 0 when the /// worker could start with this environment, or when no worker variable is set at all /// (an unconfigured image is a valid artifact check); a partial or invalid configuration /// and a too-old libwebp exit non-zero and name the variable. fn check(mode: Mode) -> i32 { if mode == Mode::SheetVerify { println!("mode: sheet-verify (verification rounds only, no sweep)"); } let (major, minor, rev) = libwebp_version(); let libwebp_ok = libwebp_ok(); println!( "libwebp: {major}.{minor}.{rev} ({})", if libwebp_ok { "accepted" } else { "below the minimum, refusing to start" } ); let mut any_set = false; for name in config::VARIABLES { let set = std::env::var_os(name).is_some(); any_set |= set; println!("{name}: {}", if set { "set" } else { "unset" }); } let configuration = if any_set { Some(Config::from_env()) } else { None }; print_configuration(configuration.as_ref()); let _ = std::io::stdout().flush(); match (libwebp_ok, configuration) { (false, _) => exit_codes::INTERNAL, (true, None | Some(Ok(_))) => exit_codes::OK, (true, Some(Err(_))) => exit_codes::USAGE, } } fn print_configuration(configuration: Option<&Result>) { match configuration { None => println!("configuration: none (no worker variable set; the loop would not start)"), Some(Err(error)) => println!("configuration: invalid ({error})"), Some(Ok(config)) => { println!( "configuration: ok (url {}, worker_id {}, poll {}s, sweep {}s, quarantine grace {}s, pending max age {}s, once {})", config.url, config.worker_id, config.poll.as_secs(), config.sweep.as_secs(), config.quarantine_grace.as_secs(), config.pending_max_age.as_secs(), config.once ); match token_expiry(config) { TokenExpiry::NotJwt => println!("token: not a JWT (no expiry to track)"), TokenExpiry::Valid(days) => println!("token: JWT expiring in {days} days"), TokenExpiry::Soon(days) => { println!("token: JWT expiring in {days} days (WARNING: under 30)") } TokenExpiry::Expired => println!("token: JWT EXPIRED"), } } } }