diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index aa3a86d7..747191c1 100644 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -19,6 +19,9 @@ ts = ["dep:ts-rs"] pdf-read = ["dep:miniz_oxide"] # The PDF writer (R4, plan §B.1.3): layout to a draw log (R4.4), the serialiser (R4.5). pdf-write = ["pdf-read"] +# The archive check tool (R4.7, plan §B.1.7): `pdf::compare` and the `report-compare` binary, +# the one file of the crate that reads the file system. Never enabled by the wasm facade. +cli = ["pdf-read", "pdf-write"] [dependencies] serde.workspace = true @@ -28,6 +31,11 @@ roxmltree.workspace = true ts-rs = { workspace = true, optional = true } miniz_oxide = { workspace = true, optional = true } +[[bin]] +name = "report-compare" +path = "src/bin/report_compare.rs" +required-features = ["cli"] + [dev-dependencies] # Reads the PDFs the tests compare (the R3 fixtures and the R4 output): objects, streams, # annotations; never in the library. @@ -84,5 +92,9 @@ required-features = ["pdf-read", "pdf-write"] name = "pdf_session" required-features = ["pdf-read", "pdf-write"] +[[test]] +name = "pdf_compare" +required-features = ["cli"] + [lints] workspace = true diff --git a/crates/core/src/bin/report_compare.rs b/crates/core/src/bin/report_compare.rs new file mode 100644 index 00000000..ca90efcf --- /dev/null +++ b/crates/core/src/bin/report_compare.rs @@ -0,0 +1,160 @@ +//! `pnpm report:compare` (R4.7, plan §B.1.7): the archive check tool on the core. Reads the +//! report attached to one or more Faunosteo PDFs and prints it next to its v3 reading on the +//! reference sheet (the seed draft of the repository). +//! +//! report-compare ... [--summary] [--full] [--locale fr|en] +//! [--diff a.pdf b.pdf] +//! +//! Paths are typed from the folder pnpm was run in (`INIT_CWD`), else the current one. The +//! lines are [`faunosteo_core::pdf::compare`]'s; this file only reads the seed and the +//! entries, and is the one file of the crate allowed the file system and the environment. +#![allow(clippy::disallowed_methods)] + +use std::collections::HashMap; +use std::io::{self, ErrorKind, Write}; +use std::path::{Path, PathBuf}; +use std::{env, fs, process}; + +use serde::Deserialize; + +use faunosteo_core::domain::archive::SheetStatus; +use faunosteo_core::domain::sheet::loaded::BundledRef; +use faunosteo_core::domain::sheet::{BundledSheetData, LoadedSheet}; +use faunosteo_core::pdf::compare::{self, FileFailure, Files}; +use faunosteo_core::pdf::read::MAX_PDF_BYTES; + +/// The seed of the reference sheet, from this crate's manifest folder. +const SEED_DIR: &str = concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../supabase/seed/anatomical-sheets/equine/draft" +); + +/// `sheet-ref.json` of the seed. +#[derive(Deserialize)] +struct SeedRef { + sheet_id: String, + version_id: Option, + document_sha256: String, + schema_version: f64, + species_id: String, +} + +fn seed_json Deserialize<'de>>(name: &str) -> Result { + let path = Path::new(SEED_DIR).join(name); + let text = fs::read_to_string(&path).map_err(|e| format!("{}: {e}", path.display()))?; + serde_json::from_str(&text).map_err(|e| format!("{}: {e}", path.display())) +} + +/// The seed document and migration map, indexed; no drawing (the comparison needs none). +fn load_sheet() -> Result { + let reference: SeedRef = seed_json("sheet-ref.json")?; + let schema_version = faunosteo_core::domain::js::JsInt::new(reference.schema_version) + .ok_or_else(|| "sheet-ref.json: schema_version is not an integer".to_owned())?; + let data = BundledSheetData { + status: SheetStatus::Draft, + reference: BundledRef { + sheet_id: reference.sheet_id, + version_id: reference.version_id, + document_sha256: reference.document_sha256, + schema_version, + species_id: reference.species_id, + }, + document: seed_json("document.json")?, + migration_map: seed_json("report-migration-map.json")?, + assets: HashMap::new(), + asset_hashes: HashMap::new(), + }; + LoadedSheet::from_bundle(data).map_err(|e| format!("reference sheet: {e:?}")) +} + +/// The file system behind the tool: entries resolved against `base`. +struct Disk { + base: PathBuf, +} + +fn is_pdf_name(name: &str) -> bool { + name.to_lowercase().ends_with(".pdf") +} + +fn failure_of(error: &io::Error) -> FileFailure { + match error.kind() { + ErrorKind::NotFound => FileFailure::NotFound, + ErrorKind::PermissionDenied => FileFailure::NotReadable, + _ => FileFailure::Other(error.to_string()), + } +} + +impl Disk { + fn resolve(&self, arg: &str) -> PathBuf { + self.base.join(arg) + } + + /// The `.pdf` entries of a folder, by name. + fn pdfs_of(folder: &Path) -> Vec { + let Ok(entries) = fs::read_dir(folder) else { + return vec![folder.to_string_lossy().into_owned()]; + }; + let mut names: Vec = entries + .filter_map(Result::ok) + .map(|e| e.file_name().to_string_lossy().into_owned()) + .filter(|n| is_pdf_name(n)) + .collect(); + names.sort(); + names + .into_iter() + .map(|n| folder.join(n).to_string_lossy().into_owned()) + .collect() + } +} + +impl Files for Disk { + fn expand(&self, arg: &str) -> Vec { + let path = self.resolve(arg); + match fs::metadata(&path) { + Ok(meta) if meta.is_dir() => Self::pdfs_of(&path), + // A missing entry is reported when read, as a failed entry. + _ => vec![path.to_string_lossy().into_owned()], + } + } + + fn read(&self, path: &str) -> Result, FileFailure> { + let meta = fs::metadata(path).map_err(|e| failure_of(&e))?; + if !meta.is_file() { + return Err(FileFailure::NotRegular); + } + if meta.len() > MAX_PDF_BYTES as u64 { + return Err(FileFailure::TooLarge); + } + fs::read(path).map_err(|e| failure_of(&e)) + } +} + +fn base_dir() -> PathBuf { + env::var_os("INIT_CWD") + .map(PathBuf::from) + .or_else(|| env::current_dir().ok()) + .unwrap_or_default() +} + +fn main() { + let argv: Vec = env::args_os() + .skip(1) + .map(|a| a.to_string_lossy().into_owned()) + .collect(); + let sheet = match load_sheet() { + Ok(sheet) => sheet, + Err(message) => { + eprintln!("{message}"); + process::exit(1); + } + }; + let disk = Disk { base: base_dir() }; + let run = compare::run(&sheet, &argv, &disk); + let written = io::stdout() + .write_all(run.stdout.as_bytes()) + .and_then(|()| io::stderr().write_all(run.stderr.as_bytes())); + if written.is_err() { + process::exit(1); + } + process::exit(run.exit_code); +} diff --git a/crates/core/src/domain/archive/snapshot_rules.rs b/crates/core/src/domain/archive/snapshot_rules.rs index 3e068b4c..6e00fb4a 100644 --- a/crates/core/src/domain/archive/snapshot_rules.rs +++ b/crates/core/src/domain/archive/snapshot_rules.rs @@ -33,6 +33,19 @@ pub enum SnapshotRejection { SheetMismatch, } +impl SnapshotRejection { + /// The serde spelling, as the import outcome names it. + pub fn as_str(self) -> &'static str { + match self { + Self::UnsupportedEnvelope => "unsupported_envelope", + Self::Schema => "schema", + Self::Caps => "caps", + Self::Integrity => "integrity", + Self::SheetMismatch => "sheet_mismatch", + } + } +} + // --------------------------------------------------------------------------- // Build diff --git a/crates/core/src/domain/report/identity.rs b/crates/core/src/domain/report/identity.rs index 5deff80c..c17c25e0 100644 --- a/crates/core/src/domain/report/identity.rs +++ b/crates/core/src/domain/report/identity.rs @@ -102,3 +102,13 @@ pub enum LateralSide { Left, Right, } + +impl LateralSide { + /// The serde spelling, as the report carries it. + pub fn as_str(self) -> &'static str { + match self { + Self::Left => "left", + Self::Right => "right", + } + } +} diff --git a/crates/core/src/domain/report/v2.rs b/crates/core/src/domain/report/v2.rs index ce671327..0568eb00 100644 --- a/crates/core/src/domain/report/v2.rs +++ b/crates/core/src/domain/report/v2.rs @@ -21,6 +21,17 @@ pub enum SeverityV2 { Severe, } +impl SeverityV2 { + /// The legacy spelling, as the v2 report carries it. + pub fn as_str(self) -> &'static str { + match self { + Self::Legere => "légère", + Self::Moderee => "modérée", + Self::Severe => "sévère", + } + } +} + /// A vertebra of the legacy model: regions C, T, L, S, Cd only. #[derive(Debug, Clone, PartialEq, Deserialize)] pub struct VertebraV2 { diff --git a/crates/core/src/domain/report/v3.rs b/crates/core/src/domain/report/v3.rs index 771ffc31..19b906f5 100644 --- a/crates/core/src/domain/report/v3.rs +++ b/crates/core/src/domain/report/v3.rs @@ -25,6 +25,17 @@ pub enum Severity { Severe, } +impl Severity { + /// The serde spelling, as the report carries it. + pub fn as_str(self) -> &'static str { + match self { + Self::Mild => "mild", + Self::Moderate => "moderate", + Self::Severe => "severe", + } + } +} + /// One vertebra: the sheet's region code and the 1-based index in that region. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[cfg_attr(feature = "ts", derive(TS), ts(export))] diff --git a/crates/core/src/pdf/compare/lines.rs b/crates/core/src/pdf/compare/lines.rs new file mode 100644 index 00000000..35e8dd27 --- /dev/null +++ b/crates/core/src/pdf/compare/lines.rs @@ -0,0 +1,423 @@ +//! The lines of a comparison (`compare/report.ts`): one zone or finding per line, the sheet's +//! current label next to the archived one, `!! UNRESOLVED` with the reason where the sheet +//! cannot place an item. Every untrusted string goes through [`esc`]. + +use serde::Deserialize; +use serde_json::Value; + +use super::{CompareOptions, Counts, Locale, Source, UNRESOLVED}; +use crate::domain::archive::{ArchiveReadOutcome, SheetStatus}; +use crate::domain::report::v2::{FindingV2, ObservationV2, ObservationZoneV2, ReportV2}; +use crate::domain::report::{ + Finding, LateralSide, NextAppointment, Observation, Report, ReportZone, finding_path_key, +}; +use crate::domain::sheet::{LoadedSheet, LocalizedText}; +use crate::json::{js_number, to_js_compact}; +use crate::pdf::write::text::is_control_or_format; + +/// Escapes Unicode control (C0, DEL, C1) and format characters (bidi overrides, zero-width) +/// plus the line and paragraph separators as `\uXXXX` (newline and carriage return included): +/// a PDF is untrusted, and a raw escape sequence printed to a terminal can rewrite the screen +/// or fake a line. The class is V8's `[\p{Cc}\p{Cf}

]` (`esc` of `report.ts`). +pub fn esc(text: &str) -> String { + let mut out = String::with_capacity(text.len()); + for c in text.chars() { + if is_control_or_format(c) { + out.push_str(&format!("\\u{:04x}", u32::from(c))); + } else { + out.push(c); + } + } + out +} + +fn q(text: &str) -> String { + format!("\"{}\"", esc(text)) +} + +/// `pickLabel`: the locale's label, the other one when it is empty. +fn pick_label(text: &LocalizedText, locale: Locale) -> &str { + let (own, other) = match locale { + Locale::En => (&text.en, &text.fr), + Locale::Fr => (&text.fr, &text.en), + }; + if own.is_empty() { other } else { own } +} + +fn side_suffix(side: Option) -> String { + side.map_or_else(String::new, |s| format!(" {}", s.as_str())) +} + +fn zone_ref(zone: &ReportZone) -> String { + let text = format!("{}@{}", zone.annotation_id, zone.view_id); + esc(&format!("{text}{}", side_suffix(zone.side))) +} + +fn finding_ref(finding: &Finding) -> String { + esc(&finding_path_key(&finding.finding_zone_id, &finding.path)) +} + +/// ` [C3, C4]` for a zone with vertebrae, nothing otherwise (an empty list included). +fn vertebrae_of<'a>(vertebrae: impl Iterator) -> String { + let listed: Vec = vertebrae + .map(|(region, index)| esc(&format!("{region}{}", js_number(index)))) + .collect(); + if listed.is_empty() { + String::new() + } else { + format!(" [{}]", listed.join(", ")) + } +} + +fn v3_vertebrae(zone: &ReportZone) -> String { + vertebrae_of( + zone.vertebrae + .iter() + .flatten() + .map(|v| (v.region.as_str(), v.index.get())), + ) +} + +fn v2_vertebrae(zone: &ObservationZoneV2) -> String { + vertebrae_of( + zone.vertebrae + .iter() + .flatten() + .map(|v| (v.region.as_str(), v.index.get())), + ) +} + +/// Free-text lines: the texts with `full`, else only how many there are. +fn text_lines(texts: &[String], full: bool) -> Vec { + if texts.is_empty() { + return Vec::new(); + } + if !full { + return vec![format!("text ({} entries)", texts.len())]; + } + texts.iter().map(|t| format!("text {}", q(t))).collect() +} + +/// A single free-text field: its content with `full`, else a placeholder. +fn field_line(label: &str, value: &str, full: bool) -> Vec { + if value.is_empty() { + return Vec::new(); + } + let line = if full { + format!("{label} {}", q(value)) + } else { + format!("{label} (present)") + }; + vec![line] +} + +/// A v3 zone against the sheet: current label, or why it cannot be shown. +fn describe_zone(zone: &ReportZone, sheet: &LoadedSheet, locale: Locale) -> String { + let reference = format!("{}{}", zone_ref(zone), v3_vertebrae(zone)); + let snapshot = q(zone.label_fr_snapshot.as_deref().unwrap_or("")); + if zone.unresolved.is_some() { + return format!("{reference} {snapshot} {UNRESOLVED} legacy zone kept from the archive"); + } + let indexed = sheet.indexed(); + match indexed.annotation(&zone.annotation_id) { + Some(anno) if anno.view_id() == zone.view_id => { + let label = q(pick_label(&indexed.name(anno), locale)); + format!("{reference} {label}") + } + _ => format!("{reference} {snapshot} !! not in the sheet (unflagged)"), + } +} + +fn snapshot_label(finding: &Finding, locale: Locale) -> String { + q(match locale { + Locale::En => &finding.label_en_snapshot, + Locale::Fr => &finding.label_fr_snapshot, + }) +} + +/// A v3 finding against the sheet: current label, or why it cannot be shown. +fn describe_finding(finding: &Finding, sheet: &LoadedSheet, locale: Locale) -> String { + let snapshot = snapshot_label(finding, locale); + let vet = if finding.requires_vet_referral == Some(true) { + " (!)" + } else { + "" + }; + let reference = format!("{}{}{vet}", finding_ref(finding), side_suffix(finding.side)); + if finding.unresolved.is_some() { + return format!("{reference} {snapshot} {UNRESOLVED} kept with its snapshot"); + } + let Some(node) = sheet + .indexed() + .node_at(&finding.finding_zone_id, &finding.path) + else { + return format!("{reference} {snapshot} !! not in the sheet (unflagged)"); + }; + if !node.children().is_empty() { + return format!("{reference} {snapshot} !! points at a category, not a leaf"); + } + format!("{reference} {}", q(pick_label(node.name(), locale))) +} + +/// Legacy zone → migrated zone. +fn describe_zone_migration( + legacy: &ObservationZoneV2, + out: &ReportZone, + sheet: &LoadedSheet, + locale: Locale, +) -> String { + // JavaScript truthiness: an empty detail key is no detail. + let detail = legacy + .detail_source_id + .as_deref() + .filter(|key| !key.is_empty()) + .map_or_else(String::new, |key| format!("@{key}")); + let from = format!( + "{}{}", + esc(&format!( + "{}{detail}{}", + legacy.bone_id, + side_suffix(legacy.side) + )), + v2_vertebrae(legacy) + ); + if out.unresolved.is_some() { + return format!( + "{from} {UNRESOLVED} unknown legacy id, kept as {}", + zone_ref(out) + ); + } + let indexed = sheet.indexed(); + let label = indexed.annotation(&out.annotation_id).map_or_else( + || "!! missing".to_owned(), + |anno| q(pick_label(&indexed.name(anno), locale)), + ); + let snapshot = q(out.label_fr_snapshot.as_deref().unwrap_or("")); + format!("{from} {snapshot} → {} {label}", zone_ref(out)) +} + +/// Legacy finding → migrated finding, with the fate the map decided. +fn describe_finding_migration( + legacy: &FindingV2, + out: &Finding, + sheet: &LoadedSheet, + locale: Locale, +) -> String { + let vet = if legacy.requires_vet_referral == Some(true) { + " (!)" + } else { + "" + }; + let legacy_key = finding_path_key(&legacy.zone_id, &legacy.path); + let key = format!("{}{vet}", esc(&legacy_key)); + let snapshot = q(match locale { + Locale::En => &legacy.label_en_snapshot, + Locale::Fr => &legacy.label_fr_snapshot, + }); + if out.unresolved.is_some() { + let why = if sheet.migration_map.finding_paths.contains_key(&legacy_key) { + "removed from the sheet, kept with its label" + } else { + "never in the legacy catalog, kept with its label" + }; + return format!("{key} {snapshot} {UNRESOLVED} {why}"); + } + let label = sheet + .indexed() + .node_at(&out.finding_zone_id, &out.path) + .map_or_else( + || "!! missing".to_owned(), + |node| q(pick_label(node.name(), locale)), + ); + let target = if finding_ref(out) == esc(&legacy_key) { + "same".to_owned() + } else { + finding_ref(out) + }; + format!("{key} {snapshot} → {target} {label}") +} + +pub(super) fn observation_lines( + obs: &Observation, + sheet: &LoadedSheet, + opts: CompareOptions, +) -> Vec { + let severity = obs.severity.map_or("none", |s| s.as_str()); + let mut lines = vec![format!("[{}] severity {}", esc(&obs.id), esc(severity))]; + for zone in &obs.zones { + lines.push(format!( + "zone {}", + describe_zone(zone, sheet, opts.locale) + )); + } + for finding in &obs.findings { + lines.push(format!( + "finding {}", + describe_finding(finding, sheet, opts.locale) + )); + } + lines.extend(text_lines(&obs.free_text_findings, opts.full)); + lines.extend(field_line("comment ", &obs.comment, opts.full)); + lines +} + +/// The report document inside the attachment: the envelope's, or the bare one. +fn report_data_of(data: &Value) -> &Value { + let is_envelope = data.get("envelopeVersion").is_some_and(Value::is_number); + if is_envelope { + data.get("report").unwrap_or(&Value::Null) + } else { + data + } +} + +fn migrated_observation( + legacy: &ObservationV2, + out: &Observation, + sheet: &LoadedSheet, + opts: CompareOptions, +) -> Vec { + let severity = match (legacy.severity, out.severity) { + (Some(from), Some(to)) => esc(&format!("{} → {}", from.as_str(), to.as_str())), + _ => "none".to_owned(), + }; + let mut lines = vec![format!("[{}] severity {severity}", esc(&legacy.id))]; + for (zone, migrated) in legacy.zones.iter().zip(&out.zones) { + let line = describe_zone_migration(zone, migrated, sheet, opts.locale); + lines.push(format!("zone {line}")); + } + for (finding, migrated) in legacy.findings.iter().zip(&out.findings) { + let line = describe_finding_migration(finding, migrated, sheet, opts.locale); + lines.push(format!("finding {line}")); + } + lines.extend(text_lines(&legacy.free_text_findings, opts.full)); + lines.extend(field_line("comment ", &legacy.comment, opts.full)); + lines +} + +/// The legacy observations next to their migration. The v2 document parsed again from the +/// attachment: the layered reader already accepted it, so the parse cannot fail; were it to, +/// the v3 lines stand in. +pub(super) fn migrated_observation_lines( + data: &Value, + report: &Report, + sheet: &LoadedSheet, + opts: CompareOptions, +) -> Vec> { + let Ok(v2) = ReportV2::deserialize(report_data_of(data)) else { + return report + .observations + .iter() + .map(|o| observation_lines(o, sheet, opts)) + .collect(); + }; + v2.observations + .iter() + .zip(&report.observations) + .map(|(legacy, out)| migrated_observation(legacy, out, sheet, opts)) + .collect() +} + +pub(super) fn counts_of(report: &Report) -> Counts { + let zones = || report.observations.iter().flat_map(|o| &o.zones); + let findings = || report.observations.iter().flat_map(|o| &o.findings); + Counts { + observations: report.observations.len(), + zones: zones().count(), + unresolved_zones: zones().filter(|z| z.unresolved.is_some()).count(), + findings: findings().count(), + unresolved_findings: findings().filter(|f| f.unresolved.is_some()).count(), + } +} + +/// The envelope line: present only when the attachment was an archive. +fn archive_line(outcome: &ArchiveReadOutcome) -> Vec { + if outcome.archive.is_some() { + return vec!["archive v1 (snapshot kept)".to_owned()]; + } + match outcome.snapshot_rejected { + Some(reason) => vec![format!( + "archive v1 (snapshot rejected: {})", + esc(reason.as_str()) + )], + None => Vec::new(), + } +} + +fn sheet_line(report: &Report, sheet: &LoadedSheet) -> String { + let reference = &sheet.reference; + let same_sheet = report.sheet.sheet_id == reference.sheet_id + && report.sheet.document_sha256 == reference.document_sha256; + if same_sheet { + let status = match sheet.status { + SheetStatus::Draft => "draft", + SheetStatus::Published => "published", + }; + return format!( + "sheet {} ({status}, matches the bundled sheet)", + esc(&reference.sheet_id) + ); + } + let sha: String = report.sheet.document_sha256.chars().take(12).collect(); + format!( + "sheet {} !! written on another sheet document (sha {}…)", + esc(&report.sheet.sheet_id), + esc(&sha) + ) +} + +pub(super) fn header_lines( + source: Source, + outcome: &ArchiveReadOutcome, + sheet: &LoadedSheet, + opts: CompareOptions, +) -> Vec { + let report = &outcome.report; + let c = counts_of(report); + let mut lines = vec![match source { + Source::V2 => "v2 → v3 (migrated)".to_owned(), + Source::V3 => "v3 (read as is)".to_owned(), + }]; + lines.extend(archive_line(outcome)); + lines.push(sheet_line(report, sheet)); + lines.push(format!( + "id {} · date {} · {} observations · {} zones ({} unresolved) · {} findings ({} unresolved)", + esc(&report.id), + esc(&report.date), + c.observations, + c.zones, + c.unresolved_zones, + c.findings, + c.unresolved_findings + )); + if opts.full { + lines.push(format!( + "practitioner {} · animal {} · owner {}", + q(&report.practitioner.name), + q(&report.animal.name), + q(&report.owner.name) + )); + } + lines +} + +fn appointment_line(appointment: &NextAppointment, full: bool) -> String { + if full { + let json = to_js_compact(appointment).unwrap_or_default(); + return format!("next appointment {}", esc(&json)); + } + let kind = match appointment { + NextAppointment::Date(_) => "date", + NextAppointment::Period(_) => "period", + }; + format!("next appointment ({kind})") +} + +pub(super) fn footer_lines(report: &Report, full: bool) -> Vec { + let mut lines = field_line("conclusion", &report.conclusion, full); + lines.extend(field_line("recommendations", &report.recommendations, full)); + if let Some(appointment) = &report.next_appointment { + lines.push(appointment_line(appointment, full)); + } + lines +} diff --git a/crates/core/src/pdf/compare/mod.rs b/crates/core/src/pdf/compare/mod.rs new file mode 100644 index 00000000..abd91318 --- /dev/null +++ b/crates/core/src/pdf/compare/mod.rs @@ -0,0 +1,177 @@ +//! R4.7: the archive check tool (`apps/app/scripts/compare-reports.ts` and `compare/report.ts`, +//! plan §B.1.7), pure. A PDF's attachment is read by the R4 reader, parsed as the session +//! parses it, read through the layered archive reader, then printed next to its v3 reading on +//! the reference sheet: a v2 archive line by line legacy id → canonical id, a v3 report +//! checked against the sheet, every item the sheet cannot place marked `!! UNRESOLVED` with +//! its reason. Nothing is summarised away. The lines, the summary, the `--diff` and the +//! argument parsing are the TypeScript's, pinned by the frozen O10 oracle +//! (`tests/oracles/r4/compare/compare.json`): 230 invocations, of which 72 declared in plan +//! §A.3 (a ZodError dump, a lone surrogate kept, a version read as `Infinity`) print the +//! closed wording of [`CompareFailure::reason`] instead. +//! +//! No I/O here: the binary (`src/bin/report_compare.rs`) owns the file system through the +//! [`run::Files`] trait, and so do the tests. + +pub mod lines; +pub mod run; + +use serde_json::Value; + +use crate::domain::archive::ArchiveReadOutcome; +use crate::domain::archive::read::read_archive_document; +use crate::domain::js::{JsonReadError, parse_json}; +use crate::domain::report::ReadFailure; +use crate::domain::sheet::LoadedSheet; +use crate::json::js_number; +use crate::pdf::read::ReadKind; +use crate::pdf::read::probe::{ProbeLimits, read_attached_document}; + +pub use run::{CliArgs, FileFailure, Files, Run, parse_args, run}; + +/// The marker of an item the sheet cannot place. +pub const UNRESOLVED: &str = "!! UNRESOLVED"; + +/// The two locales of the sheet labels (`--locale fr|en`). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Locale { + Fr, + En, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct CompareOptions { + pub locale: Locale, + /// Print names and the real free texts (off by default: placeholders). + pub full: bool, +} + +/// `v2` when the attachment was migrated, `v3` when read as is. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Source { + V2, + V3, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct Counts { + pub observations: usize, + pub zones: usize, + pub unresolved_zones: usize, + pub findings: usize, + pub unresolved_findings: usize, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Comparison { + pub source: Source, + pub header: Vec, + pub observations: Vec>, + pub footer: Vec, + /// Lines in the v3 vocabulary only, for `--diff` between two PDFs. + pub normalized: Vec, + pub counts: Counts, +} + +/// Why one PDF gives no comparison, worded without echoing the file (`READ_FAILURES` and +/// `failureReason` of the TypeScript CLI; the closed lines of plan §A.3 for what the +/// TypeScript printed as a ZodError, read as `Infinity` or kept as a lone surrogate). +#[derive(Debug)] +pub enum CompareFailure { + Read(ReadKind), + /// The attachment text the probe accepted does not read as JSON here: a lone surrogate + /// escape, refused unchanged (A.3), or a syntax the probe tolerated. + Json(JsonReadError), + /// The `version` is not 2 or 3: the number found, `None` for none (an overflowing number + /// reads as `null`: A.3). + Version(Option), + /// A document of a known version that fails its schema (the TypeScript printed the + /// ZodError issues: A.3). + Invalid(String), +} + +impl CompareFailure { + /// The reason printed after `!! : `. + pub fn reason(&self) -> String { + match self { + Self::Read(ReadKind::InvalidPdf) => "not a readable PDF".to_owned(), + Self::Read(ReadKind::NoAttachment) => "no Faunosteo attachment found".to_owned(), + Self::Read(ReadKind::AttachmentStripped) => { + "attachment stripped (the PDF is stamped as a Faunosteo archive)".to_owned() + } + Self::Json(error @ JsonReadError::LoneSurrogate { .. }) => { + format!("attachment refused: {error}") + } + Self::Json(JsonReadError::Syntax(_)) => "attachment is not valid JSON".to_owned(), + Self::Version(found) => { + let found = found.map_or_else(|| "null".to_owned(), js_number); + format!("report version {found} cannot be read") + } + Self::Invalid(detail) => format!("report document invalid: {detail}"), + } + } +} + +impl From for CompareFailure { + fn from(failure: ReadFailure) -> Self { + match failure { + ReadFailure::UnsupportedVersion(found) => Self::Version(found), + ReadFailure::Invalid(detail) => Self::Invalid(detail), + } + } +} + +/// The attachment of `bytes` read and compared: the reader's steps of the import, then the +/// layered reading, then the lines. No cap on the JSON values (the TypeScript had none). +pub fn compare_bytes( + sheet: &LoadedSheet, + bytes: &[u8], + opts: CompareOptions, +) -> Result { + let located = + read_attached_document(bytes, ProbeLimits::UNBOUNDED).map_err(CompareFailure::Read)?; + let data: Value = parse_json(&located.text).map_err(CompareFailure::Json)?; + let outcome = read_archive_document(&data, &sheet.migration_map)?; + Ok(compare_report(&data, &outcome, sheet, opts)) +} + +/// `data` is the attachment as found in the PDF (bare report or envelope), `outcome` its +/// layered reading; `outcome.migrated` tells which vocabulary the lines use. +pub fn compare_report( + data: &Value, + outcome: &ArchiveReadOutcome, + sheet: &LoadedSheet, + opts: CompareOptions, +) -> Comparison { + let report = &outcome.report; + let source = if outcome.migrated { + Source::V2 + } else { + Source::V3 + }; + let v3_lines: Vec> = report + .observations + .iter() + .map(|o| lines::observation_lines(o, sheet, opts)) + .collect(); + let footer = lines::footer_lines(report, opts.full); + // Same privacy default as the listing: texts are compared with `--full`. + let normalized = v3_lines + .iter() + .flatten() + .chain(footer.iter()) + .cloned() + .collect(); + let observations = if outcome.migrated { + lines::migrated_observation_lines(data, report, sheet, opts) + } else { + v3_lines + }; + Comparison { + source, + header: lines::header_lines(source, outcome, sheet, opts), + observations, + footer, + normalized, + counts: lines::counts_of(report), + } +} diff --git a/crates/core/src/pdf/compare/run.rs b/crates/core/src/pdf/compare/run.rs new file mode 100644 index 00000000..1cd4c631 --- /dev/null +++ b/crates/core/src/pdf/compare/run.rs @@ -0,0 +1,254 @@ +//! The command line of the tool (`compare-reports.ts` `main`), without its file system: the +//! arguments, the loop over the entries, what goes to stdout and stderr, the exit code. The +//! caller lists folders and reads files through [`Files`]; a missing or unreadable file is a +//! failed entry, not a crash. + +use std::collections::HashSet; + +use super::lines::esc; +use super::{CompareOptions, Comparison, Locale, Source, compare_bytes}; +use crate::domain::sheet::LoadedSheet; + +pub const USAGE: &str = "usage: pnpm report:compare ... [--summary] [--full] [--locale fr|en] [--diff a.pdf b.pdf]"; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CliArgs { + pub files: Vec, + pub locale: Locale, + pub full: bool, + pub summary: bool, + pub diff: bool, +} + +impl CliArgs { + fn options(&self) -> CompareOptions { + CompareOptions { + locale: self.locale, + full: self.full, + } + } +} + +/// `parseArgs`: options anywhere, every other argument an entry; the error is the message +/// the TypeScript threw (printed on stderr, exit 1). +pub fn parse_args(argv: &[String]) -> Result { + let mut args = CliArgs { + files: Vec::new(), + locale: Locale::Fr, + full: false, + summary: false, + diff: false, + }; + let mut rest = argv.iter(); + while let Some(arg) = rest.next() { + match arg.as_str() { + "--full" => args.full = true, + "--summary" => args.summary = true, + "--diff" => args.diff = true, + "--locale" => { + args.locale = match rest.next().map(String::as_str) { + Some("fr") => Locale::Fr, + Some("en") => Locale::En, + _ => return Err("--locale takes fr or en".to_owned()), + } + } + other if other.starts_with("--") => { + return Err(format!("unknown option {}", esc(other))); + } + _ => args.files.push(arg.clone()), + } + } + Ok(args) +} + +/// Why an entry cannot be read, worded as the TypeScript CLI did. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum FileFailure { + NotFound, + NotReadable, + NotRegular, + TooLarge, + /// Any other error of the file system, its message. + Other(String), +} + +impl FileFailure { + fn reason(&self) -> String { + match self { + Self::NotFound => "file not found".to_owned(), + Self::NotReadable => "file not readable".to_owned(), + Self::NotRegular => "not a regular file".to_owned(), + Self::TooLarge => "file too large".to_owned(), + Self::Other(message) => message.clone(), + } + } +} + +/// The file system as the tool uses it. Paths are the caller's: resolved against the folder +/// the command was typed from, printed by their last segment only. +pub trait Files { + /// A folder stands for its PDFs (`.pdf` in any case, sorted); any other argument stays + /// one entry, a missing one included (reported when read). + fn expand(&self, arg: &str) -> Vec; + /// The bytes of one entry: a regular file under the reader's size cap. + fn read(&self, path: &str) -> Result, FileFailure>; +} + +/// What a run printed and answered. +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub struct Run { + pub stdout: String, + pub stderr: String, + pub exit_code: i32, +} + +impl Run { + /// `console.log`: the text and a newline. + fn log(&mut self, text: &str) { + self.stdout.push_str(text); + self.stdout.push('\n'); + } + + fn fail(message: &str) -> Self { + Self { + stdout: String::new(), + stderr: format!("{message}\n"), + exit_code: 1, + } + } +} + +/// The last path segment, as `basename` printed it. +fn name_of(path: &str) -> String { + let name = std::path::Path::new(path) + .file_name() + .map(|n| n.to_string_lossy().into_owned()) + .unwrap_or_else(|| path.to_owned()); + esc(&name) +} + +pub fn render_comparison(name: &str, c: &Comparison) -> Vec { + let mut lines = vec![format!("== {name}")]; + lines.extend(c.header.iter().map(|l| format!(" {l}"))); + for observation in &c.observations { + for (i, line) in observation.iter().enumerate() { + let indent = if i == 0 { " " } else { " " }; + lines.push(format!("{indent}{line}")); + } + } + lines.extend(c.footer.iter().map(|l| format!(" {l}"))); + lines +} + +pub fn render_summary(name: &str, c: &Comparison) -> String { + let n = c.counts; + let source = match c.source { + Source::V2 => "v2→v3", + Source::V3 => "v3", + }; + format!( + "{name} · {source} · {} obs · zones {}/{} unresolved · findings {}/{} unresolved", + n.observations, n.zones, n.unresolved_zones, n.findings, n.unresolved_findings + ) +} + +/// Lines of A absent from B and lines of B absent from A, in order. +pub fn diff_lines(a: &[String], b: &[String]) -> Vec { + let in_a: HashSet<&String> = a.iter().collect(); + let in_b: HashSet<&String> = b.iter().collect(); + let mut out: Vec = a + .iter() + .filter(|l| !in_b.contains(l)) + .map(|l| format!("- {l}")) + .collect(); + out.extend( + b.iter() + .filter(|l| !in_a.contains(l)) + .map(|l| format!("+ {l}")), + ); + out +} + +fn compare_entry( + sheet: &LoadedSheet, + files: &dyn Files, + path: &str, + opts: CompareOptions, +) -> Result { + let bytes = files.read(path).map_err(|f| f.reason())?; + compare_bytes(sheet, &bytes, opts).map_err(|f| f.reason()) +} + +fn print_entry(run: &mut Run, name: &str, c: &Comparison, args: &CliArgs) { + if args.diff { + return; + } + if args.summary { + run.log(&render_summary(name, c)); + } else { + let mut lines = render_comparison(name, c); + lines.push(String::new()); + run.log(&lines.join("\n")); + } +} + +/// Every entry compared and printed, a failure printed as `!! : `: the +/// comparisons, and how many entries failed. +fn compare_entries( + sheet: &LoadedSheet, + files: &dyn Files, + entries: &[String], + args: &CliArgs, + run: &mut Run, +) -> (Vec, usize) { + let mut results = Vec::new(); + let mut failures = 0; + for path in entries { + let name = name_of(path); + match compare_entry(sheet, files, path, args.options()) { + Ok(c) => { + print_entry(run, &name, &c, args); + results.push(c); + } + Err(reason) => { + failures += 1; + run.log(&format!("!! {name}: {}", esc(&reason))); + } + } + } + (results, failures) +} + +/// `--diff`: the lines one of the two comparisons has and the other lacks, or that they are +/// identical; nothing when one of the two entries failed. +fn print_diff(run: &mut Run, results: &[Comparison]) { + let [a, b] = results else { + return; + }; + let diff = diff_lines(&a.normalized, &b.normalized); + if diff.is_empty() { + run.log("identical v3 content"); + } else { + run.log(&diff.join("\n")); + } +} + +/// `main`: usage on stderr (exit 1) without an entry or with `--diff` and not two; the +/// entries compared and printed (exit 2 when any failed); the diff of the two printed last. +pub fn run(sheet: &LoadedSheet, argv: &[String], files: &dyn Files) -> Run { + let args = match parse_args(argv) { + Ok(args) => args, + Err(message) => return Run::fail(&message), + }; + let entries: Vec = args.files.iter().flat_map(|a| files.expand(a)).collect(); + if entries.is_empty() || (args.diff && entries.len() != 2) { + return Run::fail(USAGE); + } + let mut run = Run::default(); + let (results, failures) = compare_entries(sheet, files, &entries, &args, &mut run); + if args.diff { + print_diff(&mut run, &results); + } + run.exit_code = if failures > 0 { 2 } else { 0 }; + run +} diff --git a/crates/core/src/pdf/mod.rs b/crates/core/src/pdf/mod.rs index 9a757c0f..33d9e10c 100644 --- a/crates/core/src/pdf/mod.rs +++ b/crates/core/src/pdf/mod.rs @@ -4,8 +4,11 @@ //! of the object graph, BLK-009), inflated under a cap, probed as JSON, then handed to the //! R2 session import. Every rule is the one `packages/pdf/src/attachment/scan.ts` applied, //! pinned by the frozen oracles O7 (`tests/oracles/r4/read.json`), O8 (`tolerance.json`) and -//! O9 (`inflate.json`). The writer (`write`) and the import entry (`import`) follow. +//! O9 (`inflate.json`). The writer (`write`) and the import entry (`import`) follow; the +//! archive check tool (`compare`, R4.7) sits on both, behind the `cli` feature of its binary. +#[cfg(feature = "cli")] +pub mod compare; #[cfg(feature = "pdf-read")] pub mod import; pub mod json_js; diff --git a/crates/core/src/pdf/write/text.rs b/crates/core/src/pdf/write/text.rs index 768521c7..94a8e5a7 100644 --- a/crates/core/src/pdf/write/text.rs +++ b/crates/core/src/pdf/write/text.rs @@ -56,8 +56,9 @@ fn winansi_byte(c: char) -> Option { table.get(index).map(|&(_, byte)| byte) } -/// `[\p{Cc}\p{Cf}\p{Zl}\p{Zp}]` as V8 matched it (`text.ts` `DROPPED`). -fn is_dropped(c: char) -> bool { +/// `[\p{Cc}\p{Cf}\p{Zl}\p{Zp}]` as V8 matched it (`text.ts` `DROPPED`; the same class the +/// compare tool escapes, `[\p{Cc}\p{Cf}\u2028\u2029]`: U+2028 and U+2029 are Zl and Zp whole). +pub fn is_control_or_format(c: char) -> bool { in_ranges(afm_data::SAFE_TEXT_DROPPED, u32::from(c)) } @@ -65,7 +66,7 @@ fn is_dropped(c: char) -> bool { /// WinAnsi cannot encode replaced by `?`. U+00AD (Cf) is dropped although WinAnsi has it. pub fn safe_text(text: &str) -> String { text.chars() - .filter(|&c| !is_dropped(c)) + .filter(|&c| !is_control_or_format(c)) .map(|c| if winansi_byte(c).is_some() { c } else { '?' }) .collect() } diff --git a/crates/core/tests/pdf/LEDGER.tsv b/crates/core/tests/pdf/LEDGER.tsv index 8515f345..13636d11 100644 --- a/crates/core/tests/pdf/LEDGER.tsv +++ b/crates/core/tests/pdf/LEDGER.tsv @@ -132,17 +132,17 @@ apps/app/tests/unit/routes.coreStopped.test.tsx the stopped text on the Home imp apps/app/tests/unit/routes.coreStopped.test.tsx the stopped text on the Home import line and the New report alert > New report: the alert says the stopped text kept vitest (same case) - mocks rewritten at R4.3 / R4.6 apps/app/tests/unit/routes.coreStopped.test.tsx the stopped text on the Home import line and the New report alert > New report: the unsaved stopped text when storage is full kept vitest (same case) - mocks rewritten at R4.3 / R4.6 apps/app/tests/unit/routes.coreStopped.test.tsx the stopped text on the Home import line and the New report alert > New report: keeps the sheet alert while the engine is not stopped kept vitest (same case) - mocks rewritten at R4.3 / R4.6 -apps/app/tests/unit/scripts.compare.test.ts compareReport on the frozen v2 archive PDF > shows every legacy zone and finding next to its canonical target ported R4.7 cargo (pending) O10 - -apps/app/tests/unit/scripts.compare.test.ts compareReport on unresolved items > names a removed finding, a finding unknown to the catalog and an unknown zone ported R4.7 cargo (pending) O10 - -apps/app/tests/unit/scripts.compare.test.ts compareReport on unresolved items > reads a v3 report as is and flags what the sheet cannot place ported R4.7 cargo (pending) O10 - -apps/app/tests/unit/scripts.compare.test.ts compareReport on unresolved items > prints the archive line of an envelope: snapshot kept, or rejected with its reason ported R4.7 cargo (pending) O10 - -apps/app/tests/unit/scripts.compare.test.ts compareReport on unresolved items > warns when a v3 report was written on another sheet document ported R4.7 cargo (pending) O10 - -apps/app/tests/unit/scripts.compare.test.ts diff and arguments > diffs the v3 content of a migrated v2 report and its v3 re-reading ported R4.7 cargo (pending) O10 - -apps/app/tests/unit/scripts.compare.test.ts diff and arguments > compares placeholders by default and real texts with --full ported R4.7 cargo (pending) O10 - -apps/app/tests/unit/scripts.compare.test.ts diff and arguments > parses files and options, refuses unknown ones ported R4.7 cargo (pending) O10 - -apps/app/tests/unit/scripts.compare.test.ts terminal safety and privacy > escapes control characters in every untrusted string ported R4.7 cargo (pending) O10 - -apps/app/tests/unit/scripts.compare.test.ts terminal safety and privacy > does not throw on prototype-chain names used as legacy ids ported R4.7 cargo (pending) O10 - -apps/app/tests/unit/scripts.compare.test.ts terminal safety and privacy > hides names and free texts by default, shows them with --full ported R4.7 cargo (pending) O10 - +apps/app/tests/unit/scripts.compare.test.ts compareReport on the frozen v2 archive PDF > shows every legacy zone and finding next to its canonical target ported cargo pdf_compare::frozen_v2_archive_pdf_shows_every_legacy_item_next_to_its_target O10 - +apps/app/tests/unit/scripts.compare.test.ts compareReport on unresolved items > names a removed finding, a finding unknown to the catalog and an unknown zone ported cargo pdf_compare::unresolved_items_name_a_removed_an_unknown_finding_and_an_unknown_zone O10 - +apps/app/tests/unit/scripts.compare.test.ts compareReport on unresolved items > reads a v3 report as is and flags what the sheet cannot place ported cargo pdf_compare::v3_report_read_as_is_flags_what_the_sheet_cannot_place O10 - +apps/app/tests/unit/scripts.compare.test.ts compareReport on unresolved items > prints the archive line of an envelope: snapshot kept, or rejected with its reason ported cargo pdf_compare::envelope_archive_line_says_snapshot_kept_or_rejected_with_its_reason O10 - +apps/app/tests/unit/scripts.compare.test.ts compareReport on unresolved items > warns when a v3 report was written on another sheet document ported cargo pdf_compare::v3_report_written_on_another_sheet_document_warns O10 - +apps/app/tests/unit/scripts.compare.test.ts diff and arguments > diffs the v3 content of a migrated v2 report and its v3 re-reading ported cargo pdf_compare::diff_of_a_migrated_v2_report_and_its_v3_re_reading O10 - +apps/app/tests/unit/scripts.compare.test.ts diff and arguments > compares placeholders by default and real texts with --full ported cargo pdf_compare::diff_compares_placeholders_by_default_and_real_texts_with_full O10 - +apps/app/tests/unit/scripts.compare.test.ts diff and arguments > parses files and options, refuses unknown ones ported cargo pdf_compare::arguments_parsed_and_unknown_options_refused O10 - +apps/app/tests/unit/scripts.compare.test.ts terminal safety and privacy > escapes control characters in every untrusted string ported cargo pdf_compare::control_characters_escaped_in_every_untrusted_string O10 - +apps/app/tests/unit/scripts.compare.test.ts terminal safety and privacy > does not throw on prototype-chain names used as legacy ids ported cargo pdf_compare::prototype_chain_names_as_legacy_ids_do_not_throw O10 - +apps/app/tests/unit/scripts.compare.test.ts terminal safety and privacy > hides names and free texts by default, shows them with --full ported cargo pdf_compare::names_and_free_texts_hidden_by_default_shown_with_full O10 - apps/app/tests/unit/state.coreFailures.test.tsx store actions on a failing core > change nothing and answer { ok: false } kept vitest (same case) - mocks rewritten at R4.3 / R4.6 apps/app/tests/unit/state.coreFailures.test.tsx store actions on a failing core > answer { ok: true } on the live core kept vitest (same case) - mocks rewritten at R4.3 / R4.6 apps/app/tests/unit/state.coreFailures.test.tsx callers stop on a failed core call > the new-report page stays put when the core cannot load kept vitest (same case) - mocks rewritten at R4.3 / R4.6 diff --git a/crates/core/tests/pdf_compare.rs b/crates/core/tests/pdf_compare.rs new file mode 100644 index 00000000..c864071e --- /dev/null +++ b/crates/core/tests/pdf_compare.rs @@ -0,0 +1,458 @@ +//! R4.7c: the archive check tool against the frozen O10 oracle +//! (`oracles/r4/compare/compare.json`, `apps/app/tests/unit/r4.compare-oracles.test.ts`): the +//! 230 invocations of the TypeScript CLI (every fixture PDF, a folder, the argument errors and +//! the 11 cases of `apps/app/tests/unit/scripts.compare.test.ts`, their documents attached to +//! the scenario PDFs) replayed in-process through `pdf::compare::run` on the seed draft sheet. +//! A row without `r4Declared` must match exactly (stdout, stderr, exit code); a declared row +//! (plan §A.3: a ZodError dump, a version read as `Infinity`, a lone surrogate kept) must +//! print the closed wording of `CompareFailure::reason`. The real binary is spawned once per +//! argument shape to prove it prints what the pure run prints. +#![allow(clippy::disallowed_methods, clippy::unwrap_used, clippy::expect_used)] + +mod common; + +use std::collections::HashMap; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::sync::OnceLock; + +use common::r4::{r4_oracle, repo_root, sha256_hex}; +use faunosteo_core::domain::sheet::{BundledSheetData, LoadedSheet}; +use faunosteo_core::pdf::compare::lines::esc; +use faunosteo_core::pdf::compare::run::diff_lines; +use faunosteo_core::pdf::compare::{CliArgs, FileFailure, Files, Locale, Run, parse_args, run}; +use faunosteo_core::pdf::read::MAX_PDF_BYTES; +use serde_json::{Value, json}; + +const SEED: &str = "supabase/seed/anatomical-sheets/equine/draft"; + +/// The seed sheet as the binary loads it: document and map, no drawing. +fn draft_sheet() -> &'static LoadedSheet { + static SHEET: OnceLock = OnceLock::new(); + SHEET.get_or_init(|| { + let seed = repo_root().join(SEED); + let read = |name: &str| -> Value { + serde_json::from_str(&std::fs::read_to_string(seed.join(name)).unwrap()).unwrap() + }; + let payload = json!({ + "status": "draft", + "ref": read("sheet-ref.json"), + "document": read("document.json"), + "migrationMap": read("report-migration-map.json"), + "assets": {}, + "assetHashes": {}, + }); + let data: BundledSheetData = serde_json::from_value(payload).unwrap(); + LoadedSheet::from_bundle(data).unwrap() + }) +} + +/// The repository as the TypeScript CLI saw it: arguments typed from the root. +struct RepoFiles; + +impl Files for RepoFiles { + fn expand(&self, arg: &str) -> Vec { + let path = repo_root().join(arg); + let Ok(entries) = std::fs::read_dir(&path) else { + return vec![path.to_string_lossy().into_owned()]; + }; + let mut names: Vec = entries + .map(|e| e.unwrap().file_name().to_string_lossy().into_owned()) + .filter(|n| n.to_lowercase().ends_with(".pdf")) + .collect(); + names.sort(); + names + .iter() + .map(|n| path.join(n).to_string_lossy().into_owned()) + .collect() + } + + fn read(&self, path: &str) -> Result, FileFailure> { + let meta = std::fs::metadata(path).map_err(|_| FileFailure::NotFound)?; + if !meta.is_file() { + return Err(FileFailure::NotRegular); + } + if meta.len() > MAX_PDF_BYTES as u64 { + return Err(FileFailure::TooLarge); + } + std::fs::read(path).map_err(|_| FileFailure::NotReadable) + } +} + +/// One recorded invocation of the TypeScript CLI. +struct Row { + group: String, + scenario: Option, + args: Vec, + declared: Option, + exit_code: i32, + stdout: String, + stderr: String, +} + +fn lines_text(value: &Value) -> String { + let lines: Vec<&str> = value + .as_array() + .unwrap() + .iter() + .map(|l| l.as_str().unwrap()) + .collect(); + lines.join("\n") +} + +fn oracle() -> &'static Value { + static ORACLE: OnceLock = OnceLock::new(); + ORACLE.get_or_init(|| r4_oracle("compare/compare.json")) +} + +fn rows() -> Vec { + oracle()["invocations"] + .as_array() + .unwrap() + .iter() + .map(|row| Row { + group: row["group"].as_str().unwrap().to_owned(), + scenario: row["scenario"].as_str().map(str::to_owned), + args: row["args"] + .as_array() + .unwrap() + .iter() + .map(|a| a.as_str().unwrap().to_owned()) + .collect(), + declared: row["r4Declared"].as_str().map(str::to_owned), + exit_code: i32::try_from(row["exitCode"].as_i64().unwrap()).unwrap(), + stdout: lines_text(&row["stdout"]), + stderr: lines_text(&row["stderr"]), + }) + .collect() +} + +fn replay(row: &Row) -> Run { + run(draft_sheet(), &row.args, &RepoFiles) +} + +fn assert_exact(row: &Row) { + let got = replay(row); + let args = row.args.join(" "); + assert_eq!(got.stdout, row.stdout, "stdout of `{args}`"); + assert_eq!(got.stderr, row.stderr, "stderr of `{args}`"); + assert_eq!(got.exit_code, row.exit_code, "exit code of `{args}`"); +} + +/// The file named by a failure line (`!! : …`), or the one argument's last segment. +fn entry_name(row: &Row) -> String { + let path = row.args.iter().find(|a| !a.starts_with("--")).unwrap(); + Path::new(path) + .file_name() + .unwrap() + .to_string_lossy() + .into_owned() +} + +/// The Rust wording a declared row printed: which §A.3 line it is. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +enum Printed { + /// `report document invalid: ` where the TypeScript dumped the ZodError issues. + Invalid, + /// `attachment refused: lone UTF-16 surrogate escape at byte N` where the TypeScript + /// kept the surrogate (or, for one decoy-first file, dumped a ZodError on the decoy). + Surrogate, + /// `report version null cannot be read` where the TypeScript read `Infinity`. + Null, +} + +/// A declared row: the Rust closed wording of its §A.3 class, one line, exit 2. +fn assert_declared(row: &Row, declared: &str) -> Printed { + let got = replay(row); + let name = entry_name(row); + let args = row.args.join(" "); + assert_eq!(got.stderr, "", "stderr of `{args}`"); + assert_eq!(got.exit_code, 2, "`{args}`"); + assert_eq!(got.stdout.matches('\n').count(), 1, "`{args}`: one line"); + let invalid = format!("!! {name}: report document invalid: "); + let surrogate = format!("!! {name}: attachment refused: lone UTF-16 surrogate escape at byte "); + let printed = if got.stdout.starts_with(&invalid) { + Printed::Invalid + } else if got.stdout.starts_with(&surrogate) { + Printed::Surrogate + } else { + assert_infinity_row(row, &got, &name); + Printed::Null + }; + let allowed: &[Printed] = if declared.contains("ZodError") { + &[Printed::Invalid, Printed::Surrogate] + } else if declared.contains("lone surrogate") { + &[Printed::Surrogate] + } else if declared.contains("Infinity") { + &[Printed::Null] + } else { + panic!("`{args}`: unknown declaration {declared}"); + }; + assert!(allowed.contains(&printed), "`{args}`: {}", got.stdout); + printed +} + +/// `Infinity` read as `null` (the core's reading, the one the application's import gave on +/// the R3 path too: O8 `no_attachment`): a report version overflow is `report version null +/// cannot be read`; an envelope version overflow leaves a bare object without a version, +/// the same line, where the TypeScript CLI printed the report under a rejected envelope. +fn assert_infinity_row(row: &Row, got: &Run, name: &str) { + let args = row.args.join(" "); + let infinity = format!("!! {name}: report version Infinity cannot be read\n"); + let envelope_line = " archive v1 (snapshot rejected: unsupported_envelope)\n"; + assert!( + row.stdout == infinity || row.stdout.contains(envelope_line), + "`{args}`: the TypeScript line" + ); + let expected = format!("!! {name}: report version null cannot be read\n"); + assert_eq!(got.stdout, expected, "`{args}`"); + assert_eq!(got.exit_code, 2, "`{args}`"); +} + +/// Every invocation of one `scripts.compare.test.ts` case, exact. +fn assert_scenario(title: &str) { + let rows: Vec = rows() + .into_iter() + .filter(|r| r.scenario.as_deref() == Some(title)) + .collect(); + assert!(!rows.is_empty(), "no invocation recorded for `{title}`"); + for row in &rows { + assert!( + row.declared.is_none(), + "`{title}`: declared rows are not scenarios" + ); + assert_exact(row); + } +} + +// --------------------------------------------------------------------------- +// O10: every invocation + +#[test] +fn o10_every_invocation_matches_or_prints_the_declared_wording() { + let rows = rows(); + assert_eq!(rows.len(), 230); + let mut printed: HashMap = HashMap::new(); + for row in &rows { + match &row.declared { + None => assert_exact(row), + Some(reason) => *printed.entry(assert_declared(row, reason)).or_default() += 1, + } + } + // 63 ZodError dumps: 62 invalid documents, one decoy-first file whose decoy holds a lone + // surrogate (refused before any schema); 6 surrogates kept or overwritten; 3 overflows. + assert_eq!( + printed[&Printed::Invalid], + 62, + "the rows plan §A.3 declares" + ); + assert_eq!(printed[&Printed::Surrogate], 7); + assert_eq!(printed[&Printed::Null], 3); + let groups: Vec<&str> = rows.iter().map(|r| r.group.as_str()).collect(); + assert_eq!(groups.iter().filter(|g| **g == "fixture").count(), 202); + assert_eq!(groups.iter().filter(|g| **g == "folder").count(), 2); + assert_eq!(groups.iter().filter(|g| **g == "cli").count(), 4); + assert_eq!(groups.iter().filter(|g| **g == "scenario").count(), 22); +} + +#[test] +fn o10_inputs_are_the_recorded_bytes() { + let inputs = oracle()["inputs"].as_object().unwrap(); + assert!(inputs.len() >= 202); + for (file, digest) in inputs { + let bytes = std::fs::read(repo_root().join(file)).unwrap(); + assert_eq!(sha256_hex(&bytes), digest.as_str().unwrap(), "{file}"); + } + let sheet = draft_sheet(); + assert_eq!(oracle()["sheet"]["sheetId"], sheet.reference.sheet_id); + assert_eq!( + oracle()["sheet"]["documentSha256"], + sheet.reference.document_sha256 + ); +} + +#[test] +fn o10_declared_rows_name_each_a3_class_at_least_once() { + let mut classes: HashMap<&str, usize> = HashMap::new(); + for row in rows() { + if let Some(reason) = &row.declared { + let class = [ + "ZodError", + "Infinity", + "overwritten duplicate", + "keeps a lone surrogate", + ] + .into_iter() + .find(|c| reason.contains(c)) + .unwrap_or_else(|| panic!("unknown declaration {reason}")); + *classes.entry(class).or_default() += 1; + } + } + assert_eq!(classes["ZodError"], 63); + assert_eq!(classes["Infinity"], 3); + assert_eq!(classes["keeps a lone surrogate"], 3); + assert_eq!(classes["overwritten duplicate"], 3); +} + +// --------------------------------------------------------------------------- +// The 11 cases of scripts.compare.test.ts (ledger targets) + +#[test] +fn frozen_v2_archive_pdf_shows_every_legacy_item_next_to_its_target() { + assert_scenario( + "compareReport on the frozen v2 archive PDF > shows every legacy zone and finding next to its canonical target", + ); +} + +#[test] +fn unresolved_items_name_a_removed_an_unknown_finding_and_an_unknown_zone() { + assert_scenario( + "compareReport on unresolved items > names a removed finding, a finding unknown to the catalog and an unknown zone", + ); +} + +#[test] +fn v3_report_read_as_is_flags_what_the_sheet_cannot_place() { + assert_scenario( + "compareReport on unresolved items > reads a v3 report as is and flags what the sheet cannot place", + ); +} + +#[test] +fn envelope_archive_line_says_snapshot_kept_or_rejected_with_its_reason() { + assert_scenario( + "compareReport on unresolved items > prints the archive line of an envelope: snapshot kept, or rejected with its reason", + ); +} + +#[test] +fn v3_report_written_on_another_sheet_document_warns() { + assert_scenario( + "compareReport on unresolved items > warns when a v3 report was written on another sheet document", + ); +} + +#[test] +fn diff_of_a_migrated_v2_report_and_its_v3_re_reading() { + assert_scenario( + "diff and arguments > diffs the v3 content of a migrated v2 report and its v3 re-reading", + ); +} + +#[test] +fn diff_compares_placeholders_by_default_and_real_texts_with_full() { + assert_scenario( + "diff and arguments > compares placeholders by default and real texts with --full", + ); +} + +#[test] +fn arguments_parsed_and_unknown_options_refused() { + assert_scenario("diff and arguments > parses files and options, refuses unknown ones"); + let argv = |list: &[&str]| -> Vec { list.iter().map(|s| (*s).to_owned()).collect() }; + assert_eq!( + parse_args(&argv(&["a.pdf", "--summary", "dir", "--locale", "en"])), + Ok(CliArgs { + files: vec!["a.pdf".to_owned(), "dir".to_owned()], + locale: Locale::En, + full: false, + summary: true, + diff: false, + }) + ); + assert!( + parse_args(&argv(&["--full", "--diff", "a", "b"])) + .unwrap() + .full + ); + assert_eq!( + parse_args(&argv(&["--identity"])), + Err("unknown option --identity".to_owned()) + ); + assert_eq!( + parse_args(&argv(&["--locale", "de"])), + Err("--locale takes fr or en".to_owned()) + ); + assert_eq!( + parse_args(&argv(&["--json"])), + Err("unknown option --json".to_owned()) + ); +} + +#[test] +fn control_characters_escaped_in_every_untrusted_string() { + assert_scenario( + "terminal safety and privacy > escapes control characters in every untrusted string", + ); + assert_eq!(esc("a\u{1b}[2K\nb"), "a\\u001b[2K\\u000ab"); + assert_eq!( + esc("\u{9b}\u{202e}\u{2028}\u{2029}"), + "\\u009b\\u202e\\u2028\\u2029" + ); + assert_eq!(esc("\u{e0001}\u{feff}"), "\\ue0001\\ufeff"); + assert_eq!(esc("plain é 漢 😀"), "plain é 漢 😀"); +} + +#[test] +fn prototype_chain_names_as_legacy_ids_do_not_throw() { + assert_scenario( + "terminal safety and privacy > does not throw on prototype-chain names used as legacy ids", + ); +} + +#[test] +fn names_and_free_texts_hidden_by_default_shown_with_full() { + assert_scenario( + "terminal safety and privacy > hides names and free texts by default, shows them with --full", + ); + let lines = |list: &[&str]| -> Vec { list.iter().map(|s| (*s).to_owned()).collect() }; + assert_eq!( + diff_lines(&lines(&["a", "b", "b"]), &lines(&["b", "c"])), + lines(&["- a", "+ c"]) + ); + assert!(diff_lines(&lines(&["a"]), &lines(&["a"])).is_empty()); +} + +// --------------------------------------------------------------------------- +// The binary + +fn spawn(args: &[String]) -> Run { + let output = Command::new(env!("CARGO_BIN_EXE_report-compare")) + .args(args) + .env("INIT_CWD", repo_root()) + .current_dir(PathBuf::from(env!("CARGO_MANIFEST_DIR"))) + .output() + .unwrap(); + Run { + stdout: String::from_utf8(output.stdout).unwrap(), + stderr: String::from_utf8(output.stderr).unwrap(), + exit_code: output.status.code().unwrap(), + } +} + +#[test] +fn binary_prints_what_the_pure_run_prints_from_init_cwd() { + let rows = rows(); + let shapes = ["cli", "folder"]; + let mut spawned = 0; + for row in rows.iter().filter(|r| shapes.contains(&r.group.as_str())) { + let got = spawn(&row.args); + let args = row.args.join(" "); + assert_eq!(got.stdout, row.stdout, "binary stdout of `{args}`"); + assert_eq!(got.stderr, row.stderr, "binary stderr of `{args}`"); + assert_eq!(got.exit_code, row.exit_code, "binary exit code of `{args}`"); + spawned += 1; + } + assert_eq!(spawned, 6); + // A fixture typed from the root, a folder with mixed options, and a missing entry. + let scenario = rows + .iter() + .find(|r| r.args.first().map(String::as_str) == Some("a.pdf")) + .unwrap(); + assert_eq!(spawn(&scenario.args), replay(scenario)); + let full = rows + .iter() + .find(|r| r.args.first().map(String::as_str) == Some("--full") && r.args.len() == 2) + .unwrap(); + assert_eq!(spawn(&full.args), replay(full)); +} diff --git a/package.json b/package.json index be37348a..3b336114 100644 --- a/package.json +++ b/package.json @@ -24,7 +24,7 @@ "format": "prettier --write .", "check": "pnpm typecheck && pnpm test:run && pnpm lint && pnpm build", "check:all": "bash scripts/check-all.sh", - "report:compare": "pnpm --filter @faunosteo/app report:compare", + "report:compare": "bash scripts/report-compare.sh", "import:equine": "pnpm --filter @faunosteo/anatomy import:equine", "supabase": "bash scripts/supabase.sh", "db:start": "bash scripts/supabase.sh start", diff --git a/scripts/r4-ledger.mjs b/scripts/r4-ledger.mjs index d3167c18..7fb50812 100644 --- a/scripts/r4-ledger.mjs +++ b/scripts/r4-ledger.mjs @@ -109,6 +109,32 @@ const IMPORT_R46 = const ROUNDTRIP_R46 = 'vitest (same case, read from the file); cargo pdf_serialize::restoration_round_trip_loses_nothing from R4.8' +/** R4.7: the `scripts.compare.test.ts` case → the cargo test replaying its O10 rows. */ +const COMPARE_CASES = { + 'compareReport on the frozen v2 archive PDF > shows every legacy zone and finding next to its canonical target': + 'frozen_v2_archive_pdf_shows_every_legacy_item_next_to_its_target', + 'compareReport on unresolved items > names a removed finding, a finding unknown to the catalog and an unknown zone': + 'unresolved_items_name_a_removed_an_unknown_finding_and_an_unknown_zone', + 'compareReport on unresolved items > reads a v3 report as is and flags what the sheet cannot place': + 'v3_report_read_as_is_flags_what_the_sheet_cannot_place', + 'compareReport on unresolved items > prints the archive line of an envelope: snapshot kept, or rejected with its reason': + 'envelope_archive_line_says_snapshot_kept_or_rejected_with_its_reason', + 'compareReport on unresolved items > warns when a v3 report was written on another sheet document': + 'v3_report_written_on_another_sheet_document_warns', + 'diff and arguments > diffs the v3 content of a migrated v2 report and its v3 re-reading': + 'diff_of_a_migrated_v2_report_and_its_v3_re_reading', + 'diff and arguments > compares placeholders by default and real texts with --full': + 'diff_compares_placeholders_by_default_and_real_texts_with_full', + 'diff and arguments > parses files and options, refuses unknown ones': + 'arguments_parsed_and_unknown_options_refused', + 'terminal safety and privacy > escapes control characters in every untrusted string': + 'control_characters_escaped_in_every_untrusted_string', + 'terminal safety and privacy > does not throw on prototype-chain names used as legacy ids': + 'prototype_chain_names_as_legacy_ids_do_not_throw', + 'terminal safety and privacy > hides names and free texts by default, shows them with --full': + 'names_and_free_texts_hidden_by_default_shown_with_full', +} + const ported = (target, oracle, note = NONE) => ({ disposition: 'ported', target, @@ -273,10 +299,12 @@ const RULES = [ }, { file: `${APP}migration.leaves.test.ts`, ...kept(NONE) }, { file: `${APP}pdf.browserRasterizer.test.ts`, ...kept(NONE) }, - { + // R4.7: the archive check tool on the core, one cargo test per case over the O10 rows. + ...Object.entries(COMPARE_CASES).map(([title, test]) => ({ file: `${APP}scripts.compare.test.ts`, - ...ported('R4.7 cargo (pending)', 'O10'), - }, + title, + ...ported(`cargo pdf_compare::${test}`, 'O10'), + })), ...APP_MOCKING_SUITES.map((name) => ({ file: `${APP}${name}`, ...kept(NONE, 'mocks rewritten at R4.3 / R4.6'), @@ -469,7 +497,7 @@ function cargoTests(file) { '-p', 'faunosteo-core', '--features', - 'pdf-read,pdf-write', + 'cli', '--locked', '--test', file, diff --git a/scripts/report-compare.sh b/scripts/report-compare.sh new file mode 100755 index 00000000..8458d3b5 --- /dev/null +++ b/scripts/report-compare.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# `pnpm report:compare ... [--summary] [--full] [--locale fr|en] +# [--diff a.pdf b.pdf]`: the archive check tool on the core (R4.7, plan §B.1.7), the +# `report-compare` binary of crates/core behind its `cli` feature, on the native toolchain of +# the development machine (scripts/rust-env.sh). The first run compiles it (release profile); +# the next ones start at once. Paths are typed from the folder pnpm was run in (INIT_CWD). +set -euo pipefail + +cd "$(dirname "$0")/.." +. scripts/rust-env.sh + +exec cargo run --release --locked -q -p faunosteo-core --features cli --bin report-compare -- "$@" diff --git a/scripts/rust-check.sh b/scripts/rust-check.sh index 06be5fc6..316b7d9f 100755 --- a/scripts/rust-check.sh +++ b/scripts/rust-check.sh @@ -15,10 +15,11 @@ cd "$(dirname "$0")/.." cargo fmt --all --check cargo clippy --workspace --all-targets --locked -- -D warnings cargo test --workspace --locked -# The PDF reader (R4) is feature-gated in the core: its code and suites compile only here -# and in the wasm module, never in the worker. -cargo clippy -p faunosteo-core --all-targets --features pdf-read,pdf-write --locked -- -D warnings -cargo test -p faunosteo-core --features pdf-read,pdf-write --locked +# The PDF reader and writer (R4) are feature-gated in the core: their code and suites compile +# only here and in the wasm module, never in the worker; `cli` adds the archive check tool +# (`report-compare`, R4.7) and its O10 suite, never built into the module. +cargo clippy -p faunosteo-core --all-targets --features cli --locked -- -D warnings +cargo test -p faunosteo-core --features cli --locked # Exports the TypeScript types of the contracts (ts-rs) into packages/core-wasm/types. # `git status --porcelain --untracked-files=all -- packages/core-wasm/types` must stay # empty afterwards (scripts/check-all.sh, .gitea/workflows/ci.yml): the committed types