Files
faunosteo/crates/worker/src/verify_job.rs
T
bchanot a5844b9d56 feat(supabase): verification retry by effective cause
resource_limit (intrinsic to the frozen input: frame, input or output cap,
SIGXCPU, SIGABRT, SIGSYS) and verdict_refused close the round at the first
occurrence; timeout, unavailable (now also an external SIGKILL) and
lease_expired keep the 2/5/10/20 s retries up to five attempts. pgTAP 021: 89;
sandbox case k closes at attempt 1.
2026-11-01 23:01:30 +01:00

373 lines
13 KiB
Rust

//! One sheet verification (V1-2.3, contract W4; V1-2 contract §6.4, §6.5): read the leased
//! round's input, download the drawings it references, hand everything framed to the
//! sandboxed `verify` child, record its verdict. A missing or refused drawing is passed to
//! the core without bytes (a business failure the verdict names); only the transport, the
//! child's limits and a broken child are worker failures, each reported with a closed code.
use std::time::Instant;
use faunosteo_core::domain::sheet::verdict::{VerdictBinding, VerdictDrawing, VerdictInput};
use crate::backend::{
BackendError, CompleteAnswer, DrawingRow, FailAnswer, VerifyBackend, VerifyInput, VerifyJob,
};
use crate::limits::{
VERIFY_MAX_DRAWING_BYTES, VERIFY_MAX_DRAWINGS, VERIFY_MAX_DRAWINGS_TOTAL_BYTES,
};
use crate::spawn::{ChildOutcome, ChildRun};
use crate::verifier::Verifier;
use crate::watchdog::{Watchdog, deadlines};
use crate::{keys, log, verify_frame};
/// The only bucket a sheet drawing is read from (M6 `get_sheet_verification_input`).
pub const DRAWING_BUCKET: &str = "anatomical-sheet-assets";
/// Closed worker codes `fail_sheet_verification` accepts (M6 `private.sheet_worker_codes`).
pub mod codes {
pub const UNAVAILABLE: &str = "verification.worker.unavailable";
pub const TIMEOUT: &str = "verification.worker.timeout";
pub const RESOURCE_LIMIT: &str = "verification.worker.resource_limit";
pub const VERDICT_REFUSED: &str = "verification.worker.verdict_refused";
}
/// How a verification ended, as the loop sees it.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum VerifyEnd {
/// The verdict was recorded `passed`: the round is `open`.
Passed,
/// The verdict was recorded `failed`: the round is `verification_failed`.
Failed,
/// The round moved on without us (input, completion or failure answered `stale`).
Stale,
/// A worker failure was recorded with this closed code.
Reported(&'static str),
/// The failure could not be recorded either: the lease expires and the round is
/// claimed again.
LeftForLease,
/// The child reported a broken environment: the loop must stop.
Stop,
}
/// Runs one claimed round to its end.
pub fn process_verification(
backend: &dyn VerifyBackend,
verifier: &dyn Verifier,
watchdog: &Watchdog,
job: &VerifyJob,
) -> VerifyEnd {
let started = Instant::now();
let end = run(backend, verifier, watchdog, job);
let line = match &end {
VerifyEnd::Passed => log::info("verify_passed"),
VerifyEnd::Failed => log::info("verify_failed"),
VerifyEnd::Stale => log::info("verify_stale"),
VerifyEnd::Reported(code) => log::warn("verify_reported").code(code),
VerifyEnd::LeftForLease => log::warn("verify_left_for_lease"),
VerifyEnd::Stop => log::error("verify_stop"),
};
line.review(&job.review_id)
.duration_ms(started.elapsed().as_millis())
.emit();
end
}
fn run(
backend: &dyn VerifyBackend,
verifier: &dyn Verifier,
watchdog: &Watchdog,
job: &VerifyJob,
) -> VerifyEnd {
if !keys::is_uuid(&job.review_id) || !keys::is_uuid(&job.lease_token) {
log::error("verify_invalid_identity").emit();
return VerifyEnd::LeftForLease;
}
let input = match read_input(backend, watchdog, job) {
Ok(Some(input)) => input,
Ok(None) => {
log::info("verify_stale_input")
.review(&job.review_id)
.emit();
return VerifyEnd::Stale;
}
Err(error) => return transport_failure(backend, job, "input", &error),
};
let drawings = match fetch_drawings(backend, watchdog, job, &input.drawings) {
Ok(drawings) => drawings,
Err(error) => return transport_failure(backend, job, "drawing", &error),
};
let Ok(frame) = verify_frame::encode(&verdict_input(&input, &drawings)) else {
log::error("verify_frame_refused")
.review(&job.review_id)
.emit();
return report(backend, job, codes::RESOURCE_LIMIT);
};
let child = {
let _stage = watchdog.enter("verify", deadlines::VERIFY);
classify_child(verifier.verify(&frame), job)
};
match child {
ChildVerdict::Verdict(verdict) => complete(backend, watchdog, job, &verdict),
ChildVerdict::Report(code) => report(backend, job, code),
ChildVerdict::Stop => {
report(backend, job, codes::UNAVAILABLE);
VerifyEnd::Stop
}
}
}
fn read_input(
backend: &dyn VerifyBackend,
watchdog: &Watchdog,
job: &VerifyJob,
) -> Result<Option<VerifyInput>, BackendError> {
let _stage = watchdog.enter("input", deadlines::DOWNLOAD);
backend.verification_input(&job.review_id, &job.lease_token)
}
/// The bytes of each row, or `None` for a row passed without bytes. `Err` only on a
/// transport failure (the round is reported, not judged).
fn fetch_drawings(
backend: &dyn VerifyBackend,
watchdog: &Watchdog,
job: &VerifyJob,
rows: &[DrawingRow],
) -> Result<Vec<Option<Vec<u8>>>, BackendError> {
let mut budget = DownloadBudget::default();
let mut drawings = Vec::with_capacity(rows.len());
for row in rows {
let bytes = match budget.allowance(row) {
Some(max_bytes) => download(backend, watchdog, job, row, max_bytes)?,
None => {
log::info("verify_drawing_skipped")
.review(&job.review_id)
.asset(&row.asset_id)
.emit();
None
}
};
if let Some(bytes) = &bytes {
budget.spend(bytes.len());
}
drawings.push(bytes);
}
Ok(drawings)
}
/// The download policy (`limits::VERIFY_MAX_DRAWING*`): at most so many drawings, each and
/// all under their caps. What it leaves out the core reports (missing or caps).
#[derive(Debug, Default)]
struct DownloadBudget {
count: usize,
total: usize,
}
impl DownloadBudget {
/// The byte cap of this row's download, or `None` when the row is passed without bytes.
fn allowance(&self, row: &DrawingRow) -> Option<usize> {
let size = usize::try_from(row.byte_size).ok()?;
let left = VERIFY_MAX_DRAWINGS_TOTAL_BYTES.saturating_sub(self.total);
let fits = size <= VERIFY_MAX_DRAWING_BYTES && size <= left;
(is_fetchable(row) && fits && self.count < VERIFY_MAX_DRAWINGS)
.then(|| VERIFY_MAX_DRAWING_BYTES.min(left))
}
fn spend(&mut self, bytes: usize) {
self.count += 1;
self.total += bytes;
}
}
/// An uploaded row in the drawings bucket whose key is its own sha256: anything else is
/// never fetched (a corrupted row cannot point the worker at another object).
fn is_fetchable(row: &DrawingRow) -> bool {
row.uploaded
&& row.bucket == DRAWING_BUCKET
&& keys::is_drawing_key(&row.object_key)
&& row.object_key.strip_suffix(".svg") == Some(row.sha256.as_str())
}
fn download(
backend: &dyn VerifyBackend,
watchdog: &Watchdog,
job: &VerifyJob,
row: &DrawingRow,
max_bytes: usize,
) -> Result<Option<Vec<u8>>, BackendError> {
let _stage = watchdog.enter("download", deadlines::DOWNLOAD);
match backend.download_drawing(&row.bucket, &row.object_key, max_bytes) {
Ok(bytes) => Ok(Some(bytes)),
Err(BackendError::Status(404)) => {
log::warn("verify_drawing_missing")
.review(&job.review_id)
.asset(&row.asset_id)
.emit();
Ok(None)
}
Err(BackendError::Overflow) => {
log::warn("verify_drawing_overflow")
.review(&job.review_id)
.asset(&row.asset_id)
.emit();
Ok(None)
}
Err(error) => Err(error),
}
}
/// The core's input: the round as the input RPC names it, the texts as they are.
fn verdict_input<'a>(input: &'a VerifyInput, drawings: &'a [Option<Vec<u8>>]) -> VerdictInput<'a> {
VerdictInput {
binding: VerdictBinding {
sheet_id: input.sheet_id.clone(),
version_id: input.version_id.clone(),
revision: input.revision,
review_id: input.review_id.clone(),
},
document: &input.document,
base_document: input.base_document.as_deref(),
drawings: input
.drawings
.iter()
.zip(drawings)
.map(|(row, bytes)| VerdictDrawing {
asset_id: &row.asset_id,
sha256: &row.sha256,
byte_size: row.byte_size,
bytes: bytes.as_deref(),
})
.collect(),
}
}
/// What the child's end means for the round.
enum ChildVerdict {
Verdict(Vec<u8>),
Report(&'static str),
Stop,
}
/// The child's own budgets end it with SIGXCPU (CPU, the hard limit sits two seconds
/// after the soft one), SIGABRT (memory) or SIGSYS (seccomp): the same input ends the
/// same way, so `resource_limit` (definitive). Any other signal, SIGKILL first, came
/// from outside (OOM killer, stop): transient `unavailable`.
fn signal_code(signal: i32) -> &'static str {
match signal {
libc::SIGXCPU | libc::SIGABRT | libc::SIGSYS => codes::RESOURCE_LIMIT,
_ => codes::UNAVAILABLE,
}
}
/// Maps every way the child can end to a verdict on the round (§6.5 rows).
fn classify_child(run: ChildRun, job: &VerifyJob) -> ChildVerdict {
let stderr = run.stderr;
let review = job.review_id.as_str();
match run.outcome {
ChildOutcome::Produced(verdict) => ChildVerdict::Verdict(verdict),
ChildOutcome::Timeout => {
log::warn("child_timeout").review(review).emit();
ChildVerdict::Report(codes::TIMEOUT)
}
ChildOutcome::Signalled { signal } => {
log::warn("child_signalled")
.review(review)
.signal(signal)
.child_stderr(&stderr)
.emit();
ChildVerdict::Report(signal_code(signal))
}
ChildOutcome::OutputOverflow => {
log::warn("child_output_overflow").review(review).emit();
ChildVerdict::Report(codes::RESOURCE_LIMIT)
}
ChildOutcome::Rejected { name, .. } => {
log::warn("child_rejected")
.review(review)
.code(name)
.child_stderr(&stderr)
.emit();
ChildVerdict::Report(codes::RESOURCE_LIMIT)
}
ChildOutcome::Broken { name, .. } => {
log::error("child_broken")
.review(review)
.code(name)
.child_stderr(&stderr)
.emit();
ChildVerdict::Stop
}
ChildOutcome::Failed(detail) => {
log::error("child_failed")
.review(review)
.detail(&detail)
.emit();
ChildVerdict::Stop
}
}
}
fn complete(
backend: &dyn VerifyBackend,
watchdog: &Watchdog,
job: &VerifyJob,
verdict: &[u8],
) -> VerifyEnd {
let answer = {
let _stage = watchdog.enter("commit", deadlines::COMMIT);
backend.complete_verification(&job.review_id, &job.lease_token, verdict)
};
match answer {
Ok(CompleteAnswer::Passed) => VerifyEnd::Passed,
Ok(CompleteAnswer::Failed) => VerifyEnd::Failed,
Ok(CompleteAnswer::Stale) => {
log::info("complete_stale").review(&job.review_id).emit();
VerifyEnd::Stale
}
Ok(CompleteAnswer::Malformed) => {
log::error("complete_malformed")
.review(&job.review_id)
.emit();
report(backend, job, codes::VERDICT_REFUSED)
}
Err(error) => transport_failure(backend, job, "commit", &error),
}
}
/// Records a worker failure with a closed code; `stale` means the round moved on.
fn report(backend: &dyn VerifyBackend, job: &VerifyJob, code: &'static str) -> VerifyEnd {
match backend.fail_verification(&job.review_id, &job.lease_token, code) {
Ok(FailAnswer::Retry | FailAnswer::Unavailable) => VerifyEnd::Reported(code),
Ok(FailAnswer::Stale) => {
log::info("fail_stale").review(&job.review_id).emit();
VerifyEnd::Stale
}
Err(error) => {
log_backend_failure("fail", job, &error);
VerifyEnd::LeftForLease
}
}
}
/// A transport failure (§6.5 transient row): reported `unavailable`.
fn transport_failure(
backend: &dyn VerifyBackend,
job: &VerifyJob,
stage: &str,
error: &BackendError,
) -> VerifyEnd {
log_backend_failure(stage, job, error);
report(backend, job, codes::UNAVAILABLE)
}
fn log_backend_failure(stage: &str, job: &VerifyJob, error: &BackendError) {
let line = log::warn("backend_failure")
.review(&job.review_id)
.stage(stage)
.code(error.kind());
// The library's error text stays out of the logs: the closed kind and the status are
// the record (as in `job`).
match error {
BackendError::Status(status) => line.http_status(*status).emit(),
_ => line.emit(),
}
}