resource_limit (intrinsic to the frozen input: frame, input or output cap, SIGXCPU, SIGABRT, SIGSYS) and verdict_refused close the round at the first occurrence; timeout, unavailable (now also an external SIGKILL) and lease_expired keep the 2/5/10/20 s retries up to five attempts. pgTAP 021: 89; sandbox case k closes at attempt 1.
373 lines
13 KiB
Rust
373 lines
13 KiB
Rust
//! One sheet verification (V1-2.3, contract W4; V1-2 contract §6.4, §6.5): read the leased
|
|
//! round's input, download the drawings it references, hand everything framed to the
|
|
//! sandboxed `verify` child, record its verdict. A missing or refused drawing is passed to
|
|
//! the core without bytes (a business failure the verdict names); only the transport, the
|
|
//! child's limits and a broken child are worker failures, each reported with a closed code.
|
|
|
|
use std::time::Instant;
|
|
|
|
use faunosteo_core::domain::sheet::verdict::{VerdictBinding, VerdictDrawing, VerdictInput};
|
|
|
|
use crate::backend::{
|
|
BackendError, CompleteAnswer, DrawingRow, FailAnswer, VerifyBackend, VerifyInput, VerifyJob,
|
|
};
|
|
use crate::limits::{
|
|
VERIFY_MAX_DRAWING_BYTES, VERIFY_MAX_DRAWINGS, VERIFY_MAX_DRAWINGS_TOTAL_BYTES,
|
|
};
|
|
use crate::spawn::{ChildOutcome, ChildRun};
|
|
use crate::verifier::Verifier;
|
|
use crate::watchdog::{Watchdog, deadlines};
|
|
use crate::{keys, log, verify_frame};
|
|
|
|
/// The only bucket a sheet drawing is read from (M6 `get_sheet_verification_input`).
|
|
pub const DRAWING_BUCKET: &str = "anatomical-sheet-assets";
|
|
|
|
/// Closed worker codes `fail_sheet_verification` accepts (M6 `private.sheet_worker_codes`).
|
|
pub mod codes {
|
|
pub const UNAVAILABLE: &str = "verification.worker.unavailable";
|
|
pub const TIMEOUT: &str = "verification.worker.timeout";
|
|
pub const RESOURCE_LIMIT: &str = "verification.worker.resource_limit";
|
|
pub const VERDICT_REFUSED: &str = "verification.worker.verdict_refused";
|
|
}
|
|
|
|
/// How a verification ended, as the loop sees it.
|
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
pub enum VerifyEnd {
|
|
/// The verdict was recorded `passed`: the round is `open`.
|
|
Passed,
|
|
/// The verdict was recorded `failed`: the round is `verification_failed`.
|
|
Failed,
|
|
/// The round moved on without us (input, completion or failure answered `stale`).
|
|
Stale,
|
|
/// A worker failure was recorded with this closed code.
|
|
Reported(&'static str),
|
|
/// The failure could not be recorded either: the lease expires and the round is
|
|
/// claimed again.
|
|
LeftForLease,
|
|
/// The child reported a broken environment: the loop must stop.
|
|
Stop,
|
|
}
|
|
|
|
/// Runs one claimed round to its end.
|
|
pub fn process_verification(
|
|
backend: &dyn VerifyBackend,
|
|
verifier: &dyn Verifier,
|
|
watchdog: &Watchdog,
|
|
job: &VerifyJob,
|
|
) -> VerifyEnd {
|
|
let started = Instant::now();
|
|
let end = run(backend, verifier, watchdog, job);
|
|
let line = match &end {
|
|
VerifyEnd::Passed => log::info("verify_passed"),
|
|
VerifyEnd::Failed => log::info("verify_failed"),
|
|
VerifyEnd::Stale => log::info("verify_stale"),
|
|
VerifyEnd::Reported(code) => log::warn("verify_reported").code(code),
|
|
VerifyEnd::LeftForLease => log::warn("verify_left_for_lease"),
|
|
VerifyEnd::Stop => log::error("verify_stop"),
|
|
};
|
|
line.review(&job.review_id)
|
|
.duration_ms(started.elapsed().as_millis())
|
|
.emit();
|
|
end
|
|
}
|
|
|
|
fn run(
|
|
backend: &dyn VerifyBackend,
|
|
verifier: &dyn Verifier,
|
|
watchdog: &Watchdog,
|
|
job: &VerifyJob,
|
|
) -> VerifyEnd {
|
|
if !keys::is_uuid(&job.review_id) || !keys::is_uuid(&job.lease_token) {
|
|
log::error("verify_invalid_identity").emit();
|
|
return VerifyEnd::LeftForLease;
|
|
}
|
|
let input = match read_input(backend, watchdog, job) {
|
|
Ok(Some(input)) => input,
|
|
Ok(None) => {
|
|
log::info("verify_stale_input")
|
|
.review(&job.review_id)
|
|
.emit();
|
|
return VerifyEnd::Stale;
|
|
}
|
|
Err(error) => return transport_failure(backend, job, "input", &error),
|
|
};
|
|
let drawings = match fetch_drawings(backend, watchdog, job, &input.drawings) {
|
|
Ok(drawings) => drawings,
|
|
Err(error) => return transport_failure(backend, job, "drawing", &error),
|
|
};
|
|
let Ok(frame) = verify_frame::encode(&verdict_input(&input, &drawings)) else {
|
|
log::error("verify_frame_refused")
|
|
.review(&job.review_id)
|
|
.emit();
|
|
return report(backend, job, codes::RESOURCE_LIMIT);
|
|
};
|
|
let child = {
|
|
let _stage = watchdog.enter("verify", deadlines::VERIFY);
|
|
classify_child(verifier.verify(&frame), job)
|
|
};
|
|
match child {
|
|
ChildVerdict::Verdict(verdict) => complete(backend, watchdog, job, &verdict),
|
|
ChildVerdict::Report(code) => report(backend, job, code),
|
|
ChildVerdict::Stop => {
|
|
report(backend, job, codes::UNAVAILABLE);
|
|
VerifyEnd::Stop
|
|
}
|
|
}
|
|
}
|
|
|
|
fn read_input(
|
|
backend: &dyn VerifyBackend,
|
|
watchdog: &Watchdog,
|
|
job: &VerifyJob,
|
|
) -> Result<Option<VerifyInput>, BackendError> {
|
|
let _stage = watchdog.enter("input", deadlines::DOWNLOAD);
|
|
backend.verification_input(&job.review_id, &job.lease_token)
|
|
}
|
|
|
|
/// The bytes of each row, or `None` for a row passed without bytes. `Err` only on a
|
|
/// transport failure (the round is reported, not judged).
|
|
fn fetch_drawings(
|
|
backend: &dyn VerifyBackend,
|
|
watchdog: &Watchdog,
|
|
job: &VerifyJob,
|
|
rows: &[DrawingRow],
|
|
) -> Result<Vec<Option<Vec<u8>>>, BackendError> {
|
|
let mut budget = DownloadBudget::default();
|
|
let mut drawings = Vec::with_capacity(rows.len());
|
|
for row in rows {
|
|
let bytes = match budget.allowance(row) {
|
|
Some(max_bytes) => download(backend, watchdog, job, row, max_bytes)?,
|
|
None => {
|
|
log::info("verify_drawing_skipped")
|
|
.review(&job.review_id)
|
|
.asset(&row.asset_id)
|
|
.emit();
|
|
None
|
|
}
|
|
};
|
|
if let Some(bytes) = &bytes {
|
|
budget.spend(bytes.len());
|
|
}
|
|
drawings.push(bytes);
|
|
}
|
|
Ok(drawings)
|
|
}
|
|
|
|
/// The download policy (`limits::VERIFY_MAX_DRAWING*`): at most so many drawings, each and
|
|
/// all under their caps. What it leaves out the core reports (missing or caps).
|
|
#[derive(Debug, Default)]
|
|
struct DownloadBudget {
|
|
count: usize,
|
|
total: usize,
|
|
}
|
|
|
|
impl DownloadBudget {
|
|
/// The byte cap of this row's download, or `None` when the row is passed without bytes.
|
|
fn allowance(&self, row: &DrawingRow) -> Option<usize> {
|
|
let size = usize::try_from(row.byte_size).ok()?;
|
|
let left = VERIFY_MAX_DRAWINGS_TOTAL_BYTES.saturating_sub(self.total);
|
|
let fits = size <= VERIFY_MAX_DRAWING_BYTES && size <= left;
|
|
(is_fetchable(row) && fits && self.count < VERIFY_MAX_DRAWINGS)
|
|
.then(|| VERIFY_MAX_DRAWING_BYTES.min(left))
|
|
}
|
|
|
|
fn spend(&mut self, bytes: usize) {
|
|
self.count += 1;
|
|
self.total += bytes;
|
|
}
|
|
}
|
|
|
|
/// An uploaded row in the drawings bucket whose key is its own sha256: anything else is
|
|
/// never fetched (a corrupted row cannot point the worker at another object).
|
|
fn is_fetchable(row: &DrawingRow) -> bool {
|
|
row.uploaded
|
|
&& row.bucket == DRAWING_BUCKET
|
|
&& keys::is_drawing_key(&row.object_key)
|
|
&& row.object_key.strip_suffix(".svg") == Some(row.sha256.as_str())
|
|
}
|
|
|
|
fn download(
|
|
backend: &dyn VerifyBackend,
|
|
watchdog: &Watchdog,
|
|
job: &VerifyJob,
|
|
row: &DrawingRow,
|
|
max_bytes: usize,
|
|
) -> Result<Option<Vec<u8>>, BackendError> {
|
|
let _stage = watchdog.enter("download", deadlines::DOWNLOAD);
|
|
match backend.download_drawing(&row.bucket, &row.object_key, max_bytes) {
|
|
Ok(bytes) => Ok(Some(bytes)),
|
|
Err(BackendError::Status(404)) => {
|
|
log::warn("verify_drawing_missing")
|
|
.review(&job.review_id)
|
|
.asset(&row.asset_id)
|
|
.emit();
|
|
Ok(None)
|
|
}
|
|
Err(BackendError::Overflow) => {
|
|
log::warn("verify_drawing_overflow")
|
|
.review(&job.review_id)
|
|
.asset(&row.asset_id)
|
|
.emit();
|
|
Ok(None)
|
|
}
|
|
Err(error) => Err(error),
|
|
}
|
|
}
|
|
|
|
/// The core's input: the round as the input RPC names it, the texts as they are.
|
|
fn verdict_input<'a>(input: &'a VerifyInput, drawings: &'a [Option<Vec<u8>>]) -> VerdictInput<'a> {
|
|
VerdictInput {
|
|
binding: VerdictBinding {
|
|
sheet_id: input.sheet_id.clone(),
|
|
version_id: input.version_id.clone(),
|
|
revision: input.revision,
|
|
review_id: input.review_id.clone(),
|
|
},
|
|
document: &input.document,
|
|
base_document: input.base_document.as_deref(),
|
|
drawings: input
|
|
.drawings
|
|
.iter()
|
|
.zip(drawings)
|
|
.map(|(row, bytes)| VerdictDrawing {
|
|
asset_id: &row.asset_id,
|
|
sha256: &row.sha256,
|
|
byte_size: row.byte_size,
|
|
bytes: bytes.as_deref(),
|
|
})
|
|
.collect(),
|
|
}
|
|
}
|
|
|
|
/// What the child's end means for the round.
|
|
enum ChildVerdict {
|
|
Verdict(Vec<u8>),
|
|
Report(&'static str),
|
|
Stop,
|
|
}
|
|
|
|
/// The child's own budgets end it with SIGXCPU (CPU, the hard limit sits two seconds
|
|
/// after the soft one), SIGABRT (memory) or SIGSYS (seccomp): the same input ends the
|
|
/// same way, so `resource_limit` (definitive). Any other signal, SIGKILL first, came
|
|
/// from outside (OOM killer, stop): transient `unavailable`.
|
|
fn signal_code(signal: i32) -> &'static str {
|
|
match signal {
|
|
libc::SIGXCPU | libc::SIGABRT | libc::SIGSYS => codes::RESOURCE_LIMIT,
|
|
_ => codes::UNAVAILABLE,
|
|
}
|
|
}
|
|
|
|
/// Maps every way the child can end to a verdict on the round (§6.5 rows).
|
|
fn classify_child(run: ChildRun, job: &VerifyJob) -> ChildVerdict {
|
|
let stderr = run.stderr;
|
|
let review = job.review_id.as_str();
|
|
match run.outcome {
|
|
ChildOutcome::Produced(verdict) => ChildVerdict::Verdict(verdict),
|
|
ChildOutcome::Timeout => {
|
|
log::warn("child_timeout").review(review).emit();
|
|
ChildVerdict::Report(codes::TIMEOUT)
|
|
}
|
|
ChildOutcome::Signalled { signal } => {
|
|
log::warn("child_signalled")
|
|
.review(review)
|
|
.signal(signal)
|
|
.child_stderr(&stderr)
|
|
.emit();
|
|
ChildVerdict::Report(signal_code(signal))
|
|
}
|
|
ChildOutcome::OutputOverflow => {
|
|
log::warn("child_output_overflow").review(review).emit();
|
|
ChildVerdict::Report(codes::RESOURCE_LIMIT)
|
|
}
|
|
ChildOutcome::Rejected { name, .. } => {
|
|
log::warn("child_rejected")
|
|
.review(review)
|
|
.code(name)
|
|
.child_stderr(&stderr)
|
|
.emit();
|
|
ChildVerdict::Report(codes::RESOURCE_LIMIT)
|
|
}
|
|
ChildOutcome::Broken { name, .. } => {
|
|
log::error("child_broken")
|
|
.review(review)
|
|
.code(name)
|
|
.child_stderr(&stderr)
|
|
.emit();
|
|
ChildVerdict::Stop
|
|
}
|
|
ChildOutcome::Failed(detail) => {
|
|
log::error("child_failed")
|
|
.review(review)
|
|
.detail(&detail)
|
|
.emit();
|
|
ChildVerdict::Stop
|
|
}
|
|
}
|
|
}
|
|
|
|
fn complete(
|
|
backend: &dyn VerifyBackend,
|
|
watchdog: &Watchdog,
|
|
job: &VerifyJob,
|
|
verdict: &[u8],
|
|
) -> VerifyEnd {
|
|
let answer = {
|
|
let _stage = watchdog.enter("commit", deadlines::COMMIT);
|
|
backend.complete_verification(&job.review_id, &job.lease_token, verdict)
|
|
};
|
|
match answer {
|
|
Ok(CompleteAnswer::Passed) => VerifyEnd::Passed,
|
|
Ok(CompleteAnswer::Failed) => VerifyEnd::Failed,
|
|
Ok(CompleteAnswer::Stale) => {
|
|
log::info("complete_stale").review(&job.review_id).emit();
|
|
VerifyEnd::Stale
|
|
}
|
|
Ok(CompleteAnswer::Malformed) => {
|
|
log::error("complete_malformed")
|
|
.review(&job.review_id)
|
|
.emit();
|
|
report(backend, job, codes::VERDICT_REFUSED)
|
|
}
|
|
Err(error) => transport_failure(backend, job, "commit", &error),
|
|
}
|
|
}
|
|
|
|
/// Records a worker failure with a closed code; `stale` means the round moved on.
|
|
fn report(backend: &dyn VerifyBackend, job: &VerifyJob, code: &'static str) -> VerifyEnd {
|
|
match backend.fail_verification(&job.review_id, &job.lease_token, code) {
|
|
Ok(FailAnswer::Retry | FailAnswer::Unavailable) => VerifyEnd::Reported(code),
|
|
Ok(FailAnswer::Stale) => {
|
|
log::info("fail_stale").review(&job.review_id).emit();
|
|
VerifyEnd::Stale
|
|
}
|
|
Err(error) => {
|
|
log_backend_failure("fail", job, &error);
|
|
VerifyEnd::LeftForLease
|
|
}
|
|
}
|
|
}
|
|
|
|
/// A transport failure (§6.5 transient row): reported `unavailable`.
|
|
fn transport_failure(
|
|
backend: &dyn VerifyBackend,
|
|
job: &VerifyJob,
|
|
stage: &str,
|
|
error: &BackendError,
|
|
) -> VerifyEnd {
|
|
log_backend_failure(stage, job, error);
|
|
report(backend, job, codes::UNAVAILABLE)
|
|
}
|
|
|
|
fn log_backend_failure(stage: &str, job: &VerifyJob, error: &BackendError) {
|
|
let line = log::warn("backend_failure")
|
|
.review(&job.review_id)
|
|
.stage(stage)
|
|
.code(error.kind());
|
|
// The library's error text stays out of the logs: the closed kind and the status are
|
|
// the record (as in `job`).
|
|
match error {
|
|
BackendError::Status(status) => line.http_status(*status).emit(),
|
|
_ => line.emit(),
|
|
}
|
|
}
|