352 lines
11 KiB
Rust
352 lines
11 KiB
Rust
//! `faunosteo-worker run`: the keyed parent's loop (STORAGE.md §3.4, §3.5), `run --jobs
|
|
//! sheet-verify` (the verification loop of V1-2.3, no sweep) and `run --check`, the start-up
|
|
//! self-description without any secret value.
|
|
|
|
use std::io::Write;
|
|
use std::time::{Instant, SystemTime, UNIX_EPOCH};
|
|
|
|
use crate::backend::{Backend, VerifyBackend};
|
|
use crate::config::{self, Config, ConfigError};
|
|
use crate::decode::libwebp_version;
|
|
use crate::decoder::{ChildDecoder, Decoder};
|
|
use crate::http::HttpBackend;
|
|
use crate::job::{JobEnd, process_job};
|
|
use crate::limits::MIN_LIBWEBP_VERSION;
|
|
use crate::sweep::{SweepConfig, run_sweep};
|
|
use crate::verifier::{ChildVerifier, Verifier};
|
|
use crate::verify_job::{VerifyEnd, process_verification};
|
|
use crate::watchdog::{Watchdog, deadlines};
|
|
use crate::{exit_codes, log};
|
|
|
|
const THIRTY_DAYS: i64 = 30 * 24 * 3600;
|
|
|
|
/// What a `run` process works on.
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
pub enum Mode {
|
|
/// Avatar jobs and the sweep (the default).
|
|
Avatars,
|
|
/// Sheet verification rounds only (`--jobs sheet-verify`).
|
|
SheetVerify,
|
|
}
|
|
|
|
/// Entry of the `run` subcommand; returns the process exit code.
|
|
pub fn main(args: &[String]) -> i32 {
|
|
match parse_args(args) {
|
|
Ok((mode, true)) => check(mode),
|
|
Ok((Mode::Avatars, false)) => start(),
|
|
Ok((Mode::SheetVerify, false)) => start_verify(),
|
|
Err(other) => {
|
|
eprintln!(
|
|
"run: unknown argument {other} (use --check to print the configuration, --jobs sheet-verify for the verification mode)"
|
|
);
|
|
exit_codes::USAGE
|
|
}
|
|
}
|
|
}
|
|
|
|
/// `[--jobs sheet-verify] [--check]` in any order; `Err` names the first unknown argument.
|
|
pub fn parse_args(args: &[String]) -> Result<(Mode, bool), String> {
|
|
let mut mode = Mode::Avatars;
|
|
let mut check = false;
|
|
let mut iter = args.iter();
|
|
while let Some(arg) = iter.next() {
|
|
match arg.as_str() {
|
|
"--check" if !check => check = true,
|
|
"--jobs" if mode == Mode::Avatars => match iter.next().map(String::as_str) {
|
|
Some("sheet-verify") => mode = Mode::SheetVerify,
|
|
Some(value) => return Err(format!("--jobs {value}")),
|
|
None => return Err("--jobs".to_owned()),
|
|
},
|
|
other => return Err(other.to_owned()),
|
|
}
|
|
}
|
|
Ok((mode, check))
|
|
}
|
|
|
|
fn start() -> i32 {
|
|
let config = match Config::from_env() {
|
|
Ok(config) => config,
|
|
Err(error) => {
|
|
eprintln!("run: {error}");
|
|
return exit_codes::USAGE;
|
|
}
|
|
};
|
|
if !libwebp_ok() {
|
|
eprintln!("run: linked libwebp below the minimum, refusing to start");
|
|
return exit_codes::INTERNAL;
|
|
}
|
|
log::set_worker_id(&config.worker_id);
|
|
log_start(&config);
|
|
log_token_expiry(&config);
|
|
let decoder = match ChildDecoder::from_current_exe() {
|
|
Ok(decoder) => decoder,
|
|
Err(error) => {
|
|
log::error("start_failed")
|
|
.detail(&format!("current_exe: {error}"))
|
|
.emit();
|
|
return exit_codes::INTERNAL;
|
|
}
|
|
};
|
|
let backend = HttpBackend::new(&config);
|
|
let watchdog = Watchdog::new();
|
|
watchdog.spawn_monitor();
|
|
run_loop(&config, &backend, &decoder, &watchdog)
|
|
}
|
|
|
|
fn start_verify() -> i32 {
|
|
let config = match Config::from_env() {
|
|
Ok(config) => config,
|
|
Err(error) => {
|
|
eprintln!("run: {error}");
|
|
return exit_codes::USAGE;
|
|
}
|
|
};
|
|
log::set_worker_id(&config.worker_id);
|
|
log::info("worker_start")
|
|
.detail(&format!(
|
|
"mode sheet-verify; poll {}s once {}",
|
|
config.poll.as_secs(),
|
|
config.once
|
|
))
|
|
.emit();
|
|
log_token_expiry(&config);
|
|
let verifier = match ChildVerifier::from_current_exe() {
|
|
Ok(verifier) => verifier,
|
|
Err(error) => {
|
|
log::error("start_failed")
|
|
.detail(&format!("current_exe: {error}"))
|
|
.emit();
|
|
return exit_codes::INTERNAL;
|
|
}
|
|
};
|
|
let backend = HttpBackend::new(&config);
|
|
let watchdog = Watchdog::new();
|
|
watchdog.spawn_monitor();
|
|
verify_loop(&config, &backend, &verifier, &watchdog)
|
|
}
|
|
|
|
/// Claims one round per pass, no sweep (the avatar worker owns it); idles on `poll`.
|
|
pub fn verify_loop(
|
|
config: &Config,
|
|
backend: &dyn VerifyBackend,
|
|
verifier: &dyn Verifier,
|
|
watchdog: &Watchdog,
|
|
) -> i32 {
|
|
loop {
|
|
let worked = match verify_pass(backend, verifier, watchdog) {
|
|
Ok(count) => count > 0,
|
|
Err(code) => return code,
|
|
};
|
|
if config.once {
|
|
return exit_codes::OK;
|
|
}
|
|
if !worked {
|
|
std::thread::sleep(config.poll);
|
|
}
|
|
}
|
|
}
|
|
|
|
/// One claim and the rounds it returned; `Err` carries the exit code of a stop.
|
|
fn verify_pass(
|
|
backend: &dyn VerifyBackend,
|
|
verifier: &dyn Verifier,
|
|
watchdog: &Watchdog,
|
|
) -> Result<usize, i32> {
|
|
let jobs = {
|
|
let _stage = watchdog.enter("claim", deadlines::CLAIM);
|
|
match backend.claim_verifications(1) {
|
|
Ok(jobs) => jobs,
|
|
Err(error) => {
|
|
log_claim_failure(&error);
|
|
return Ok(0);
|
|
}
|
|
}
|
|
};
|
|
for job in &jobs {
|
|
if process_verification(backend, verifier, watchdog, job) == VerifyEnd::Stop {
|
|
return Err(exit_codes::CHILD_BROKEN);
|
|
}
|
|
}
|
|
Ok(jobs.len())
|
|
}
|
|
|
|
/// Sweep at start and every `sweep` interval; claim one job per pass; idle on `poll`.
|
|
pub fn run_loop(
|
|
config: &Config,
|
|
backend: &dyn Backend,
|
|
decoder: &dyn Decoder,
|
|
watchdog: &Watchdog,
|
|
) -> i32 {
|
|
let sweep_config = SweepConfig {
|
|
pending_max_age: config.pending_max_age,
|
|
quarantine_grace: config.quarantine_grace,
|
|
};
|
|
let mut next_sweep = Instant::now();
|
|
loop {
|
|
if Instant::now() >= next_sweep {
|
|
run_sweep(backend, watchdog, sweep_config);
|
|
next_sweep = Instant::now() + config.sweep;
|
|
}
|
|
let worked = match claim_pass(backend, decoder, watchdog) {
|
|
Ok(count) => count > 0,
|
|
Err(code) => return code,
|
|
};
|
|
if config.once {
|
|
return exit_codes::OK;
|
|
}
|
|
if !worked {
|
|
std::thread::sleep(config.poll);
|
|
}
|
|
}
|
|
}
|
|
|
|
/// One claim and the jobs it returned; `Err` carries the exit code of a stop.
|
|
fn claim_pass(
|
|
backend: &dyn Backend,
|
|
decoder: &dyn Decoder,
|
|
watchdog: &Watchdog,
|
|
) -> Result<usize, i32> {
|
|
let jobs = {
|
|
let _stage = watchdog.enter("claim", deadlines::CLAIM);
|
|
match backend.claim_jobs(1) {
|
|
Ok(jobs) => jobs,
|
|
Err(error) => {
|
|
log_claim_failure(&error);
|
|
return Ok(0);
|
|
}
|
|
}
|
|
};
|
|
for job in &jobs {
|
|
if process_job(backend, decoder, watchdog, job) == JobEnd::Stop {
|
|
return Err(exit_codes::CHILD_BROKEN);
|
|
}
|
|
}
|
|
Ok(jobs.len())
|
|
}
|
|
|
|
fn log_claim_failure(error: &crate::backend::BackendError) {
|
|
let line = log::warn("claim_failed").code(error.kind());
|
|
match error {
|
|
crate::backend::BackendError::Status(status) => line.http_status(*status).emit(),
|
|
_ => line.emit(),
|
|
}
|
|
}
|
|
|
|
fn log_start(config: &Config) {
|
|
let (major, minor, rev) = libwebp_version();
|
|
log::info("worker_start")
|
|
.detail(&format!(
|
|
"libwebp {major}.{minor}.{rev}; poll {}s sweep {}s grace {}s pending {}s once {}",
|
|
config.poll.as_secs(),
|
|
config.sweep.as_secs(),
|
|
config.quarantine_grace.as_secs(),
|
|
config.pending_max_age.as_secs(),
|
|
config.once
|
|
))
|
|
.emit();
|
|
}
|
|
|
|
fn log_token_expiry(config: &Config) {
|
|
match token_expiry(config) {
|
|
TokenExpiry::NotJwt => log::info("token_no_expiry").emit(),
|
|
TokenExpiry::Valid(days) => log::info("token_expiry").count(days).emit(),
|
|
TokenExpiry::Soon(days) => log::warn("token_expires_soon").count(days).emit(),
|
|
TokenExpiry::Expired => log::error("token_expired").emit(),
|
|
}
|
|
}
|
|
|
|
enum TokenExpiry {
|
|
NotJwt,
|
|
Valid(usize),
|
|
Soon(usize),
|
|
Expired,
|
|
}
|
|
|
|
fn token_expiry(config: &Config) -> TokenExpiry {
|
|
let Some(exp) = config::jwt_exp(config.service_key.expose()) else {
|
|
return TokenExpiry::NotJwt;
|
|
};
|
|
let now = SystemTime::now()
|
|
.duration_since(UNIX_EPOCH)
|
|
.map(|d| i64::try_from(d.as_secs()).unwrap_or(i64::MAX))
|
|
.unwrap_or(0);
|
|
let left = exp - now;
|
|
let days = usize::try_from(left / 86_400).unwrap_or(0);
|
|
if left <= 0 {
|
|
TokenExpiry::Expired
|
|
} else if left < THIRTY_DAYS {
|
|
TokenExpiry::Soon(days)
|
|
} else {
|
|
TokenExpiry::Valid(days)
|
|
}
|
|
}
|
|
|
|
fn libwebp_ok() -> bool {
|
|
libwebp_version() >= MIN_LIBWEBP_VERSION
|
|
}
|
|
|
|
/// Prints the resolved configuration without any value of a secret. Exit 0 when the
|
|
/// worker could start with this environment, or when no worker variable is set at all
|
|
/// (an unconfigured image is a valid artifact check); a partial or invalid configuration
|
|
/// and a too-old libwebp exit non-zero and name the variable.
|
|
fn check(mode: Mode) -> i32 {
|
|
if mode == Mode::SheetVerify {
|
|
println!("mode: sheet-verify (verification rounds only, no sweep)");
|
|
}
|
|
let (major, minor, rev) = libwebp_version();
|
|
let libwebp_ok = libwebp_ok();
|
|
println!(
|
|
"libwebp: {major}.{minor}.{rev} ({})",
|
|
if libwebp_ok {
|
|
"accepted"
|
|
} else {
|
|
"below the minimum, refusing to start"
|
|
}
|
|
);
|
|
let mut any_set = false;
|
|
for name in config::VARIABLES {
|
|
let set = std::env::var_os(name).is_some();
|
|
any_set |= set;
|
|
println!("{name}: {}", if set { "set" } else { "unset" });
|
|
}
|
|
let configuration = if any_set {
|
|
Some(Config::from_env())
|
|
} else {
|
|
None
|
|
};
|
|
print_configuration(configuration.as_ref());
|
|
let _ = std::io::stdout().flush();
|
|
match (libwebp_ok, configuration) {
|
|
(false, _) => exit_codes::INTERNAL,
|
|
(true, None | Some(Ok(_))) => exit_codes::OK,
|
|
(true, Some(Err(_))) => exit_codes::USAGE,
|
|
}
|
|
}
|
|
|
|
fn print_configuration(configuration: Option<&Result<Config, ConfigError>>) {
|
|
match configuration {
|
|
None => println!("configuration: none (no worker variable set; the loop would not start)"),
|
|
Some(Err(error)) => println!("configuration: invalid ({error})"),
|
|
Some(Ok(config)) => {
|
|
println!(
|
|
"configuration: ok (url {}, worker_id {}, poll {}s, sweep {}s, quarantine grace {}s, pending max age {}s, once {})",
|
|
config.url,
|
|
config.worker_id,
|
|
config.poll.as_secs(),
|
|
config.sweep.as_secs(),
|
|
config.quarantine_grace.as_secs(),
|
|
config.pending_max_age.as_secs(),
|
|
config.once
|
|
);
|
|
match token_expiry(config) {
|
|
TokenExpiry::NotJwt => println!("token: not a JWT (no expiry to track)"),
|
|
TokenExpiry::Valid(days) => println!("token: JWT expiring in {days} days"),
|
|
TokenExpiry::Soon(days) => {
|
|
println!("token: JWT expiring in {days} days (WARNING: under 30)")
|
|
}
|
|
TokenExpiry::Expired => println!("token: JWT EXPIRED"),
|
|
}
|
|
}
|
|
}
|
|
}
|