Files
faunosteo/crates/worker/src/run.rs
T

352 lines
11 KiB
Rust

//! `faunosteo-worker run`: the keyed parent's loop (STORAGE.md §3.4, §3.5), `run --jobs
//! sheet-verify` (the verification loop of V1-2.3, no sweep) and `run --check`, the start-up
//! self-description without any secret value.
use std::io::Write;
use std::time::{Instant, SystemTime, UNIX_EPOCH};
use crate::backend::{Backend, VerifyBackend};
use crate::config::{self, Config, ConfigError};
use crate::decode::libwebp_version;
use crate::decoder::{ChildDecoder, Decoder};
use crate::http::HttpBackend;
use crate::job::{JobEnd, process_job};
use crate::limits::MIN_LIBWEBP_VERSION;
use crate::sweep::{SweepConfig, run_sweep};
use crate::verifier::{ChildVerifier, Verifier};
use crate::verify_job::{VerifyEnd, process_verification};
use crate::watchdog::{Watchdog, deadlines};
use crate::{exit_codes, log};
const THIRTY_DAYS: i64 = 30 * 24 * 3600;
/// What a `run` process works on.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Mode {
/// Avatar jobs and the sweep (the default).
Avatars,
/// Sheet verification rounds only (`--jobs sheet-verify`).
SheetVerify,
}
/// Entry of the `run` subcommand; returns the process exit code.
pub fn main(args: &[String]) -> i32 {
match parse_args(args) {
Ok((mode, true)) => check(mode),
Ok((Mode::Avatars, false)) => start(),
Ok((Mode::SheetVerify, false)) => start_verify(),
Err(other) => {
eprintln!(
"run: unknown argument {other} (use --check to print the configuration, --jobs sheet-verify for the verification mode)"
);
exit_codes::USAGE
}
}
}
/// `[--jobs sheet-verify] [--check]` in any order; `Err` names the first unknown argument.
pub fn parse_args(args: &[String]) -> Result<(Mode, bool), String> {
let mut mode = Mode::Avatars;
let mut check = false;
let mut iter = args.iter();
while let Some(arg) = iter.next() {
match arg.as_str() {
"--check" if !check => check = true,
"--jobs" if mode == Mode::Avatars => match iter.next().map(String::as_str) {
Some("sheet-verify") => mode = Mode::SheetVerify,
Some(value) => return Err(format!("--jobs {value}")),
None => return Err("--jobs".to_owned()),
},
other => return Err(other.to_owned()),
}
}
Ok((mode, check))
}
fn start() -> i32 {
let config = match Config::from_env() {
Ok(config) => config,
Err(error) => {
eprintln!("run: {error}");
return exit_codes::USAGE;
}
};
if !libwebp_ok() {
eprintln!("run: linked libwebp below the minimum, refusing to start");
return exit_codes::INTERNAL;
}
log::set_worker_id(&config.worker_id);
log_start(&config);
log_token_expiry(&config);
let decoder = match ChildDecoder::from_current_exe() {
Ok(decoder) => decoder,
Err(error) => {
log::error("start_failed")
.detail(&format!("current_exe: {error}"))
.emit();
return exit_codes::INTERNAL;
}
};
let backend = HttpBackend::new(&config);
let watchdog = Watchdog::new();
watchdog.spawn_monitor();
run_loop(&config, &backend, &decoder, &watchdog)
}
fn start_verify() -> i32 {
let config = match Config::from_env() {
Ok(config) => config,
Err(error) => {
eprintln!("run: {error}");
return exit_codes::USAGE;
}
};
log::set_worker_id(&config.worker_id);
log::info("worker_start")
.detail(&format!(
"mode sheet-verify; poll {}s once {}",
config.poll.as_secs(),
config.once
))
.emit();
log_token_expiry(&config);
let verifier = match ChildVerifier::from_current_exe() {
Ok(verifier) => verifier,
Err(error) => {
log::error("start_failed")
.detail(&format!("current_exe: {error}"))
.emit();
return exit_codes::INTERNAL;
}
};
let backend = HttpBackend::new(&config);
let watchdog = Watchdog::new();
watchdog.spawn_monitor();
verify_loop(&config, &backend, &verifier, &watchdog)
}
/// Claims one round per pass, no sweep (the avatar worker owns it); idles on `poll`.
pub fn verify_loop(
config: &Config,
backend: &dyn VerifyBackend,
verifier: &dyn Verifier,
watchdog: &Watchdog,
) -> i32 {
loop {
let worked = match verify_pass(backend, verifier, watchdog) {
Ok(count) => count > 0,
Err(code) => return code,
};
if config.once {
return exit_codes::OK;
}
if !worked {
std::thread::sleep(config.poll);
}
}
}
/// One claim and the rounds it returned; `Err` carries the exit code of a stop.
fn verify_pass(
backend: &dyn VerifyBackend,
verifier: &dyn Verifier,
watchdog: &Watchdog,
) -> Result<usize, i32> {
let jobs = {
let _stage = watchdog.enter("claim", deadlines::CLAIM);
match backend.claim_verifications(1) {
Ok(jobs) => jobs,
Err(error) => {
log_claim_failure(&error);
return Ok(0);
}
}
};
for job in &jobs {
if process_verification(backend, verifier, watchdog, job) == VerifyEnd::Stop {
return Err(exit_codes::CHILD_BROKEN);
}
}
Ok(jobs.len())
}
/// Sweep at start and every `sweep` interval; claim one job per pass; idle on `poll`.
pub fn run_loop(
config: &Config,
backend: &dyn Backend,
decoder: &dyn Decoder,
watchdog: &Watchdog,
) -> i32 {
let sweep_config = SweepConfig {
pending_max_age: config.pending_max_age,
quarantine_grace: config.quarantine_grace,
};
let mut next_sweep = Instant::now();
loop {
if Instant::now() >= next_sweep {
run_sweep(backend, watchdog, sweep_config);
next_sweep = Instant::now() + config.sweep;
}
let worked = match claim_pass(backend, decoder, watchdog) {
Ok(count) => count > 0,
Err(code) => return code,
};
if config.once {
return exit_codes::OK;
}
if !worked {
std::thread::sleep(config.poll);
}
}
}
/// One claim and the jobs it returned; `Err` carries the exit code of a stop.
fn claim_pass(
backend: &dyn Backend,
decoder: &dyn Decoder,
watchdog: &Watchdog,
) -> Result<usize, i32> {
let jobs = {
let _stage = watchdog.enter("claim", deadlines::CLAIM);
match backend.claim_jobs(1) {
Ok(jobs) => jobs,
Err(error) => {
log_claim_failure(&error);
return Ok(0);
}
}
};
for job in &jobs {
if process_job(backend, decoder, watchdog, job) == JobEnd::Stop {
return Err(exit_codes::CHILD_BROKEN);
}
}
Ok(jobs.len())
}
fn log_claim_failure(error: &crate::backend::BackendError) {
let line = log::warn("claim_failed").code(error.kind());
match error {
crate::backend::BackendError::Status(status) => line.http_status(*status).emit(),
_ => line.emit(),
}
}
fn log_start(config: &Config) {
let (major, minor, rev) = libwebp_version();
log::info("worker_start")
.detail(&format!(
"libwebp {major}.{minor}.{rev}; poll {}s sweep {}s grace {}s pending {}s once {}",
config.poll.as_secs(),
config.sweep.as_secs(),
config.quarantine_grace.as_secs(),
config.pending_max_age.as_secs(),
config.once
))
.emit();
}
fn log_token_expiry(config: &Config) {
match token_expiry(config) {
TokenExpiry::NotJwt => log::info("token_no_expiry").emit(),
TokenExpiry::Valid(days) => log::info("token_expiry").count(days).emit(),
TokenExpiry::Soon(days) => log::warn("token_expires_soon").count(days).emit(),
TokenExpiry::Expired => log::error("token_expired").emit(),
}
}
enum TokenExpiry {
NotJwt,
Valid(usize),
Soon(usize),
Expired,
}
fn token_expiry(config: &Config) -> TokenExpiry {
let Some(exp) = config::jwt_exp(config.service_key.expose()) else {
return TokenExpiry::NotJwt;
};
let now = SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|d| i64::try_from(d.as_secs()).unwrap_or(i64::MAX))
.unwrap_or(0);
let left = exp - now;
let days = usize::try_from(left / 86_400).unwrap_or(0);
if left <= 0 {
TokenExpiry::Expired
} else if left < THIRTY_DAYS {
TokenExpiry::Soon(days)
} else {
TokenExpiry::Valid(days)
}
}
fn libwebp_ok() -> bool {
libwebp_version() >= MIN_LIBWEBP_VERSION
}
/// Prints the resolved configuration without any value of a secret. Exit 0 when the
/// worker could start with this environment, or when no worker variable is set at all
/// (an unconfigured image is a valid artifact check); a partial or invalid configuration
/// and a too-old libwebp exit non-zero and name the variable.
fn check(mode: Mode) -> i32 {
if mode == Mode::SheetVerify {
println!("mode: sheet-verify (verification rounds only, no sweep)");
}
let (major, minor, rev) = libwebp_version();
let libwebp_ok = libwebp_ok();
println!(
"libwebp: {major}.{minor}.{rev} ({})",
if libwebp_ok {
"accepted"
} else {
"below the minimum, refusing to start"
}
);
let mut any_set = false;
for name in config::VARIABLES {
let set = std::env::var_os(name).is_some();
any_set |= set;
println!("{name}: {}", if set { "set" } else { "unset" });
}
let configuration = if any_set {
Some(Config::from_env())
} else {
None
};
print_configuration(configuration.as_ref());
let _ = std::io::stdout().flush();
match (libwebp_ok, configuration) {
(false, _) => exit_codes::INTERNAL,
(true, None | Some(Ok(_))) => exit_codes::OK,
(true, Some(Err(_))) => exit_codes::USAGE,
}
}
fn print_configuration(configuration: Option<&Result<Config, ConfigError>>) {
match configuration {
None => println!("configuration: none (no worker variable set; the loop would not start)"),
Some(Err(error)) => println!("configuration: invalid ({error})"),
Some(Ok(config)) => {
println!(
"configuration: ok (url {}, worker_id {}, poll {}s, sweep {}s, quarantine grace {}s, pending max age {}s, once {})",
config.url,
config.worker_id,
config.poll.as_secs(),
config.sweep.as_secs(),
config.quarantine_grace.as_secs(),
config.pending_max_age.as_secs(),
config.once
);
match token_expiry(config) {
TokenExpiry::NotJwt => println!("token: not a JWT (no expiry to track)"),
TokenExpiry::Valid(days) => println!("token: JWT expiring in {days} days"),
TokenExpiry::Soon(days) => {
println!("token: JWT expiring in {days} days (WARNING: under 30)")
}
TokenExpiry::Expired => println!("token: JWT EXPIRED"),
}
}
}
}