feat(worker): V1-2.3 sandboxed verify child and sheet-verify job mode
This commit is contained in:
1 parent
345dde59e6
commit
31fbb0317f
24 files changed
+2145
-87
No files matched your search
+136
-14
@@ -1,10 +1,11 @@
|
||||
//! `faunosteo-worker run`: the keyed parent's loop (STORAGE.md §3.4, §3.5) and
|
||||
//! `run --check`, the start-up self-description without any secret value.
|
||||
//! `faunosteo-worker run`: the keyed parent's loop (STORAGE.md §3.4, §3.5), `run --jobs
|
||||
//! sheet-verify` (the verification loop of V1-2.3, no sweep) and `run --check`, the start-up
|
||||
//! self-description without any secret value.
|
||||
|
||||
use std::io::Write;
|
||||
use std::time::{Instant, SystemTime, UNIX_EPOCH};
|
||||
|
||||
use crate::backend::Backend;
|
||||
use crate::backend::{Backend, VerifyBackend};
|
||||
use crate::config::{self, Config, ConfigError};
|
||||
use crate::decode::libwebp_version;
|
||||
use crate::decoder::{ChildDecoder, Decoder};
|
||||
@@ -12,23 +13,56 @@ use crate::http::HttpBackend;
|
||||
use crate::job::{JobEnd, process_job};
|
||||
use crate::limits::MIN_LIBWEBP_VERSION;
|
||||
use crate::sweep::{SweepConfig, run_sweep};
|
||||
use crate::verifier::{ChildVerifier, Verifier};
|
||||
use crate::verify_job::{VerifyEnd, process_verification};
|
||||
use crate::watchdog::{Watchdog, deadlines};
|
||||
use crate::{exit_codes, log};
|
||||
|
||||
const THIRTY_DAYS: i64 = 30 * 24 * 3600;
|
||||
|
||||
/// What a `run` process works on.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum Mode {
|
||||
/// Avatar jobs and the sweep (the default).
|
||||
Avatars,
|
||||
/// Sheet verification rounds only (`--jobs sheet-verify`).
|
||||
SheetVerify,
|
||||
}
|
||||
|
||||
/// Entry of the `run` subcommand; returns the process exit code.
|
||||
pub fn main(args: &[String]) -> i32 {
|
||||
match args.first().map(String::as_str) {
|
||||
Some("--check") => check(),
|
||||
Some(other) => {
|
||||
eprintln!("run: unknown argument {other} (use --check to print the configuration)");
|
||||
match parse_args(args) {
|
||||
Ok((mode, true)) => check(mode),
|
||||
Ok((Mode::Avatars, false)) => start(),
|
||||
Ok((Mode::SheetVerify, false)) => start_verify(),
|
||||
Err(other) => {
|
||||
eprintln!(
|
||||
"run: unknown argument {other} (use --check to print the configuration, --jobs sheet-verify for the verification mode)"
|
||||
);
|
||||
exit_codes::USAGE
|
||||
}
|
||||
None => start(),
|
||||
}
|
||||
}
|
||||
|
||||
/// `[--jobs sheet-verify] [--check]` in any order; `Err` names the first unknown argument.
|
||||
pub fn parse_args(args: &[String]) -> Result<(Mode, bool), String> {
|
||||
let mut mode = Mode::Avatars;
|
||||
let mut check = false;
|
||||
let mut iter = args.iter();
|
||||
while let Some(arg) = iter.next() {
|
||||
match arg.as_str() {
|
||||
"--check" if !check => check = true,
|
||||
"--jobs" if mode == Mode::Avatars => match iter.next().map(String::as_str) {
|
||||
Some("sheet-verify") => mode = Mode::SheetVerify,
|
||||
Some(value) => return Err(format!("--jobs {value}")),
|
||||
None => return Err("--jobs".to_owned()),
|
||||
},
|
||||
other => return Err(other.to_owned()),
|
||||
}
|
||||
}
|
||||
Ok((mode, check))
|
||||
}
|
||||
|
||||
fn start() -> i32 {
|
||||
let config = match Config::from_env() {
|
||||
Ok(config) => config,
|
||||
@@ -43,6 +77,7 @@ fn start() -> i32 {
|
||||
}
|
||||
log::set_worker_id(&config.worker_id);
|
||||
log_start(&config);
|
||||
log_token_expiry(&config);
|
||||
let decoder = match ChildDecoder::from_current_exe() {
|
||||
Ok(decoder) => decoder,
|
||||
Err(error) => {
|
||||
@@ -58,6 +93,83 @@ fn start() -> i32 {
|
||||
run_loop(&config, &backend, &decoder, &watchdog)
|
||||
}
|
||||
|
||||
fn start_verify() -> i32 {
|
||||
let config = match Config::from_env() {
|
||||
Ok(config) => config,
|
||||
Err(error) => {
|
||||
eprintln!("run: {error}");
|
||||
return exit_codes::USAGE;
|
||||
}
|
||||
};
|
||||
log::set_worker_id(&config.worker_id);
|
||||
log::info("worker_start")
|
||||
.detail(&format!(
|
||||
"mode sheet-verify; poll {}s once {}",
|
||||
config.poll.as_secs(),
|
||||
config.once
|
||||
))
|
||||
.emit();
|
||||
log_token_expiry(&config);
|
||||
let verifier = match ChildVerifier::from_current_exe() {
|
||||
Ok(verifier) => verifier,
|
||||
Err(error) => {
|
||||
log::error("start_failed")
|
||||
.detail(&format!("current_exe: {error}"))
|
||||
.emit();
|
||||
return exit_codes::INTERNAL;
|
||||
}
|
||||
};
|
||||
let backend = HttpBackend::new(&config);
|
||||
let watchdog = Watchdog::new();
|
||||
watchdog.spawn_monitor();
|
||||
verify_loop(&config, &backend, &verifier, &watchdog)
|
||||
}
|
||||
|
||||
/// Claims one round per pass, no sweep (the avatar worker owns it); idles on `poll`.
|
||||
pub fn verify_loop(
|
||||
config: &Config,
|
||||
backend: &dyn VerifyBackend,
|
||||
verifier: &dyn Verifier,
|
||||
watchdog: &Watchdog,
|
||||
) -> i32 {
|
||||
loop {
|
||||
let worked = match verify_pass(backend, verifier, watchdog) {
|
||||
Ok(count) => count > 0,
|
||||
Err(code) => return code,
|
||||
};
|
||||
if config.once {
|
||||
return exit_codes::OK;
|
||||
}
|
||||
if !worked {
|
||||
std::thread::sleep(config.poll);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// One claim and the rounds it returned; `Err` carries the exit code of a stop.
|
||||
fn verify_pass(
|
||||
backend: &dyn VerifyBackend,
|
||||
verifier: &dyn Verifier,
|
||||
watchdog: &Watchdog,
|
||||
) -> Result<usize, i32> {
|
||||
let jobs = {
|
||||
let _stage = watchdog.enter("claim", deadlines::CLAIM);
|
||||
match backend.claim_verifications(1) {
|
||||
Ok(jobs) => jobs,
|
||||
Err(error) => {
|
||||
log_claim_failure(&error);
|
||||
return Ok(0);
|
||||
}
|
||||
}
|
||||
};
|
||||
for job in &jobs {
|
||||
if process_verification(backend, verifier, watchdog, job) == VerifyEnd::Stop {
|
||||
return Err(exit_codes::CHILD_BROKEN);
|
||||
}
|
||||
}
|
||||
Ok(jobs.len())
|
||||
}
|
||||
|
||||
/// Sweep at start and every `sweep` interval; claim one job per pass; idle on `poll`.
|
||||
pub fn run_loop(
|
||||
config: &Config,
|
||||
@@ -99,11 +211,7 @@ fn claim_pass(
|
||||
match backend.claim_jobs(1) {
|
||||
Ok(jobs) => jobs,
|
||||
Err(error) => {
|
||||
let line = log::warn("claim_failed").code(error.kind());
|
||||
match error {
|
||||
crate::backend::BackendError::Status(status) => line.http_status(status).emit(),
|
||||
_ => line.emit(),
|
||||
}
|
||||
log_claim_failure(&error);
|
||||
return Ok(0);
|
||||
}
|
||||
}
|
||||
@@ -116,6 +224,14 @@ fn claim_pass(
|
||||
Ok(jobs.len())
|
||||
}
|
||||
|
||||
fn log_claim_failure(error: &crate::backend::BackendError) {
|
||||
let line = log::warn("claim_failed").code(error.kind());
|
||||
match error {
|
||||
crate::backend::BackendError::Status(status) => line.http_status(*status).emit(),
|
||||
_ => line.emit(),
|
||||
}
|
||||
}
|
||||
|
||||
fn log_start(config: &Config) {
|
||||
let (major, minor, rev) = libwebp_version();
|
||||
log::info("worker_start")
|
||||
@@ -128,6 +244,9 @@ fn log_start(config: &Config) {
|
||||
config.once
|
||||
))
|
||||
.emit();
|
||||
}
|
||||
|
||||
fn log_token_expiry(config: &Config) {
|
||||
match token_expiry(config) {
|
||||
TokenExpiry::NotJwt => log::info("token_no_expiry").emit(),
|
||||
TokenExpiry::Valid(days) => log::info("token_expiry").count(days).emit(),
|
||||
@@ -170,7 +289,10 @@ fn libwebp_ok() -> bool {
|
||||
/// worker could start with this environment, or when no worker variable is set at all
|
||||
/// (an unconfigured image is a valid artifact check); a partial or invalid configuration
|
||||
/// and a too-old libwebp exit non-zero and name the variable.
|
||||
fn check() -> i32 {
|
||||
fn check(mode: Mode) -> i32 {
|
||||
if mode == Mode::SheetVerify {
|
||||
println!("mode: sheet-verify (verification rounds only, no sweep)");
|
||||
}
|
||||
let (major, minor, rev) = libwebp_version();
|
||||
let libwebp_ok = libwebp_ok();
|
||||
println!(
|
||||
|
||||
Reference in new issue
Block a user