feat(domain): R4.2d attachment probe, import_pdf and SheetHandle.import_pdf on the wasm facade
locate_attachment (shared 16 MiB budget, JS-path steps in the TypeScript order, OD-3 cap hook off until measured), pdf::import, facade import_pdf with the PdfImportKind kinds (ts-rs exported). cargo: O7 144 inputs (locate, read, archive rows) and O8 51 cases equal to the R3 path, 3 OD-4 differences declared (lone surrogate in an overwritten duplicate value). Node through the shipped module: the same O8 outcomes, every reference PDF. Module 421.7 KB gzip (W1 budget 600 KB).
This commit is contained in:
1 parent
9850a8341a
commit
f8579bc19a
13 files changed
+813
-14
No files matched your search
@@ -61,6 +61,22 @@ pub fn call_with_sheet(sheet: &LoadedSheet, op: &str, input: &str) -> String {
|
||||
render(crate::ops::on_sheet(sheet, op, input).unwrap_or_else(|| Err(unknown(op))))
|
||||
}
|
||||
|
||||
/// Reads `bytes` as a Faunosteo PDF and imports its attachment onto the session's sheet
|
||||
/// (R4, plan §B.1.4): the envelope of `archive.import`, plus the reader's kinds
|
||||
/// `invalid_pdf`, `no_attachment`, `attachment_stripped` (`PdfImportKind`).
|
||||
#[cfg(feature = "pdf-read")]
|
||||
pub fn import_pdf(sheet: &LoadedSheet, bytes: &[u8]) -> String {
|
||||
use crate::pdf::import::{PdfImportFailure, PdfImportKind, import_pdf};
|
||||
use crate::pdf::read::probe::ProbeLimits;
|
||||
let outcome = import_pdf(sheet, bytes, ProbeLimits::PRODUCTION)
|
||||
.map_err(|failure| match failure {
|
||||
PdfImportFailure::Import(failure) => crate::ops::import_failure(failure),
|
||||
read => Failure::new(PdfImportKind::from(&read).as_str(), ""),
|
||||
})
|
||||
.and_then(|import| to_value(&import));
|
||||
render(outcome)
|
||||
}
|
||||
|
||||
fn unknown(op: &str) -> Failure {
|
||||
Failure::new("unknown_op", format!("no operation named {op}"))
|
||||
}
|
||||
|
||||
+11
-8
@@ -327,17 +327,20 @@ fn draft_restore(sheet: &LoadedSheet, input: &str) -> Outcome {
|
||||
/// The input is the attachment's JSON as the byte scan read it (any shape).
|
||||
fn archive_import(sheet: &LoadedSheet, input: &str) -> Outcome {
|
||||
session::import_archive(sheet, input)
|
||||
.map_err(|failure| match failure {
|
||||
ImportFailure::IncompatibleVersion(found) => {
|
||||
versioned("incompatible_version", Some(found))
|
||||
}
|
||||
ImportFailure::NoAttachment => Failure::new("no_attachment", ""),
|
||||
ImportFailure::LoneSurrogate(detail) => Failure::new("lone_surrogate", detail),
|
||||
ImportFailure::Invalid(detail) => Failure::new("invalid", detail),
|
||||
})
|
||||
.map_err(import_failure)
|
||||
.and_then(|import| to_value(&import))
|
||||
}
|
||||
|
||||
/// The closed kinds of a session import failure, shared with the PDF import.
|
||||
pub(crate) fn import_failure(failure: ImportFailure) -> Failure {
|
||||
match failure {
|
||||
ImportFailure::IncompatibleVersion(found) => versioned("incompatible_version", Some(found)),
|
||||
ImportFailure::NoAttachment => Failure::new("no_attachment", ""),
|
||||
ImportFailure::LoneSurrogate(detail) => Failure::new("lone_surrogate", detail),
|
||||
ImportFailure::Invalid(detail) => Failure::new("invalid", detail),
|
||||
}
|
||||
}
|
||||
|
||||
fn archive_snapshot(sheet: &LoadedSheet, input: &str) -> Outcome {
|
||||
let input: SnapshotInput = parse_input(input)?;
|
||||
to_value(&build_sheet_snapshot(sheet, &input.report))
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
//! The import of a Faunosteo PDF (`importReportFromPdf` of `apps/app/src/pdf/import.ts`):
|
||||
//! the attachment located and brought to text by [`crate::pdf::read`], then the R2 session
|
||||
//! import (`archive.import`), unchanged, on the loaded sheet. One call, so the worker that
|
||||
//! will run it (R4.3) hands the main thread the core's canonical output only.
|
||||
|
||||
use serde::Serialize;
|
||||
#[cfg(feature = "ts")]
|
||||
use ts_rs::TS;
|
||||
|
||||
use crate::domain::session::{ArchiveImport, ImportFailure, import_archive};
|
||||
use crate::domain::sheet::LoadedSheet;
|
||||
use crate::pdf::read::ReadKind;
|
||||
use crate::pdf::read::probe::{ProbeLimits, read_attached_document};
|
||||
|
||||
/// Why a PDF does not import: the reader's kinds, then the session's.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub enum PdfImportFailure {
|
||||
Read(ReadKind),
|
||||
Import(ImportFailure),
|
||||
}
|
||||
|
||||
/// Reads `bytes` as a Faunosteo PDF and imports its attachment onto `sheet`.
|
||||
pub fn import_pdf(
|
||||
sheet: &LoadedSheet,
|
||||
bytes: &[u8],
|
||||
limits: ProbeLimits,
|
||||
) -> Result<ArchiveImport, PdfImportFailure> {
|
||||
let located = read_attached_document(bytes, limits).map_err(PdfImportFailure::Read)?;
|
||||
import_archive(sheet, &located.text).map_err(PdfImportFailure::Import)
|
||||
}
|
||||
|
||||
/// The closed kinds `SheetHandle.importPdf` answers with `ok: false` (plan §B.1.8): the
|
||||
/// reader's three, the session's, and the facade's `internal`.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
|
||||
#[cfg_attr(feature = "ts", derive(TS), ts(export))]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum PdfImportKind {
|
||||
InvalidPdf,
|
||||
NoAttachment,
|
||||
AttachmentStripped,
|
||||
IncompatibleVersion,
|
||||
LoneSurrogate,
|
||||
Invalid,
|
||||
Internal,
|
||||
}
|
||||
|
||||
impl PdfImportKind {
|
||||
/// The kind name of the outcome envelope (the serde spelling).
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::InvalidPdf => "invalid_pdf",
|
||||
Self::NoAttachment => "no_attachment",
|
||||
Self::AttachmentStripped => "attachment_stripped",
|
||||
Self::IncompatibleVersion => "incompatible_version",
|
||||
Self::LoneSurrogate => "lone_surrogate",
|
||||
Self::Invalid => "invalid",
|
||||
Self::Internal => "internal",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<&PdfImportFailure> for PdfImportKind {
|
||||
fn from(failure: &PdfImportFailure) -> Self {
|
||||
match failure {
|
||||
PdfImportFailure::Read(ReadKind::InvalidPdf) => Self::InvalidPdf,
|
||||
PdfImportFailure::Read(ReadKind::NoAttachment)
|
||||
| PdfImportFailure::Import(ImportFailure::NoAttachment) => Self::NoAttachment,
|
||||
PdfImportFailure::Read(ReadKind::AttachmentStripped) => Self::AttachmentStripped,
|
||||
PdfImportFailure::Import(ImportFailure::IncompatibleVersion(_)) => {
|
||||
Self::IncompatibleVersion
|
||||
}
|
||||
PdfImportFailure::Import(ImportFailure::LoneSurrogate(_)) => Self::LoneSurrogate,
|
||||
PdfImportFailure::Import(ImportFailure::Invalid(_)) => Self::Invalid,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::PdfImportKind;
|
||||
|
||||
#[test]
|
||||
fn kind_names_are_the_serde_spelling() {
|
||||
for kind in [
|
||||
PdfImportKind::InvalidPdf,
|
||||
PdfImportKind::NoAttachment,
|
||||
PdfImportKind::AttachmentStripped,
|
||||
PdfImportKind::IncompatibleVersion,
|
||||
PdfImportKind::LoneSurrogate,
|
||||
PdfImportKind::Invalid,
|
||||
PdfImportKind::Internal,
|
||||
] {
|
||||
let serialised = serde_json::to_string(&kind).unwrap_or_default();
|
||||
assert_eq!(serialised, format!("\"{}\"", kind.as_str()));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -99,6 +99,19 @@ pub fn decode_lossy(data: &[u8]) -> Cow<'_, str> {
|
||||
/// the parse, as it failed `JSON.parse`. A token is the run of `[0-9+\-.eE]` that opens on a
|
||||
/// `-` or a digit in value position: when `JSON.parse` succeeds, that run is its number.
|
||||
pub fn rewrite_overflow_numbers(text: &str) -> Cow<'_, str> {
|
||||
rewrite_overflows(text, "null")
|
||||
}
|
||||
|
||||
/// The probe's view of the same text: `parseVersioned` tested `typeof version === 'number'`
|
||||
/// on the value `JSON.parse` gave, where an overflowing `version` was still `Infinity`, a
|
||||
/// number; it became `null` only when the object was stringified for the core. The probe copy
|
||||
/// therefore keeps every overflowing token a (finite) number, `0`, and the core reads the
|
||||
/// `null` form of [`rewrite_overflow_numbers`].
|
||||
pub fn rewrite_overflow_numbers_as_finite(text: &str) -> Cow<'_, str> {
|
||||
rewrite_overflows(text, "0")
|
||||
}
|
||||
|
||||
fn rewrite_overflows<'a>(text: &'a str, replacement: &str) -> Cow<'a, str> {
|
||||
let bytes = text.as_bytes();
|
||||
let mut lexer = Lexer::new();
|
||||
let mut out = String::new();
|
||||
@@ -112,7 +125,7 @@ pub fn rewrite_overflow_numbers(text: &str) -> Cow<'_, str> {
|
||||
// The run is ASCII, so both of its ends are char boundaries.
|
||||
if overflows(&text[at..end]) {
|
||||
out.push_str(&text[copied..at]);
|
||||
out.push_str("null");
|
||||
out.push_str(replacement);
|
||||
copied = end;
|
||||
}
|
||||
at = end;
|
||||
@@ -150,9 +163,11 @@ pub fn neutralise_lone_surrogates(text: &str) -> Cow<'_, str> {
|
||||
/// `parseVersioned`'s acceptance: the text reads as JSON (R2's `parse_json`, JavaScript
|
||||
/// duplicate-key semantics) once its lone surrogates are neutralised, is an object, and its
|
||||
/// `envelopeVersion` or `version` is a number. The parsed value is dropped: the core reads
|
||||
/// the real text. Expects a text already through [`rewrite_overflow_numbers`].
|
||||
/// the real text. Takes the decoded text as is: the overflowing numbers of the probe copy
|
||||
/// stay numbers ([`rewrite_overflow_numbers_as_finite`]), as `Infinity` was one.
|
||||
pub fn is_versioned_object(text: &str) -> bool {
|
||||
let Ok(Value::Object(record)) = parse_json(&neutralise_lone_surrogates(text)) else {
|
||||
let probe = rewrite_overflow_numbers_as_finite(text);
|
||||
let Ok(Value::Object(record)) = parse_json(&neutralise_lone_surrogates(&probe)) else {
|
||||
return false;
|
||||
};
|
||||
["envelopeVersion", "version"]
|
||||
|
||||
@@ -6,5 +6,6 @@
|
||||
//! pinned by the frozen oracles O7 (`tests/oracles/r4/read.json`), O8 (`tolerance.json`) and
|
||||
//! O9 (`inflate.json`). The writer (`write`) and the import entry (`import`) follow.
|
||||
|
||||
pub mod import;
|
||||
pub mod json_js;
|
||||
pub mod read;
|
||||
@@ -1 +1,173 @@
|
||||
//! R4.2d: `locateAttachment` of scan.ts. Written after R4.2a, b and c.
|
||||
//! `locateAttachment` and `readAttachedDocument` of scan.ts and read.ts: the first embedded
|
||||
//! file whose data is a versioned JSON object, read through the JavaScript-path steps of
|
||||
//! [`crate::pdf::json_js`] so that the text handed to the session is the one the TypeScript
|
||||
//! import handed to the core.
|
||||
//!
|
||||
//! One [`MAX_ATTACHMENT_BYTES`] budget of content (raw or inflated) is shared by every
|
||||
//! candidate: a candidate that does not fit what is left ends the search, so at most 16 MiB
|
||||
//! of candidate content is retained and probed. A cancelled inflation is not charged (audit
|
||||
//! of 3a6e926, finding 2), so up to [`MAX_CANDIDATES`] oversized streams may each cost one
|
||||
//! capped inflation of CPU, bounded and retaining nothing; charging them would let bombs
|
||||
//! starve a real unnamed attachment placed after them.
|
||||
//!
|
||||
//! OD-3 adds one cap the TypeScript did not have: a candidate whose value count (the
|
||||
//! parser's allocation proxy) exceeds the limit is skipped like one nested too deep, before
|
||||
//! any parser allocates its tree. Its threshold is measured at R4.2e.
|
||||
|
||||
use std::borrow::Cow;
|
||||
|
||||
use super::inflate::inflate_flate;
|
||||
use super::scan::find_embedded_files;
|
||||
use super::stamp::has_archive_stamp;
|
||||
use super::{
|
||||
EmbeddedFileCandidate, MAX_ATTACHMENT_BYTES, MAX_PDF_BYTES, ReadKind, StreamFilter,
|
||||
has_pdf_header,
|
||||
};
|
||||
use crate::pdf::json_js::{
|
||||
JsonPrescan, decode_lossy, is_versioned_object, prescan, rewrite_overflow_numbers,
|
||||
starts_json_object,
|
||||
};
|
||||
|
||||
/// The caps the probe applies on top of the scanner's own.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct ProbeLimits {
|
||||
/// Values (`json_js::prescan`) a candidate may hold before it is skipped (OD-3).
|
||||
pub max_json_values: u64,
|
||||
}
|
||||
|
||||
/// Values a candidate may hold before the probe skips it (OD-3). A legitimate archive
|
||||
/// holds a few thousand values; the threshold is set from the R4.2e measurements (10 × the
|
||||
/// largest legitimate archive) and confirmed by the owner before the import is wired.
|
||||
/// Until then the cap is off, so the parity suites measure the TypeScript behaviour.
|
||||
pub const MAX_JSON_VALUES: u64 = u64::MAX;
|
||||
|
||||
impl ProbeLimits {
|
||||
/// No value cap: the TypeScript behaviour, for the parity suites.
|
||||
pub const UNBOUNDED: Self = Self {
|
||||
max_json_values: u64::MAX,
|
||||
};
|
||||
/// The caps the application applies.
|
||||
pub const PRODUCTION: Self = Self {
|
||||
max_json_values: MAX_JSON_VALUES,
|
||||
};
|
||||
}
|
||||
|
||||
/// The attachment, located and brought to the text the session reads.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct LocatedAttachment<'a> {
|
||||
/// Index of the candidate in [`find_embedded_files`] order.
|
||||
pub index: usize,
|
||||
pub candidate: EmbeddedFileCandidate,
|
||||
/// Byte count of the stored or inflated data.
|
||||
pub data_len: usize,
|
||||
pub prescan: JsonPrescan,
|
||||
/// The data decoded as TextDecoder would, overflowing numbers rewritten to `null`: what
|
||||
/// `JSON.stringify(JSON.parse(text))` handed to the core, lone surrogate escapes kept.
|
||||
pub text: Cow<'a, str>,
|
||||
}
|
||||
|
||||
/// Size cap first, then the `%PDF-` header: anything else is `invalid_pdf`. Then the probe.
|
||||
pub fn read_attached_document<'a>(
|
||||
bytes: &'a [u8],
|
||||
limits: ProbeLimits,
|
||||
) -> Result<LocatedAttachment<'a>, ReadKind> {
|
||||
if bytes.len() > MAX_PDF_BYTES || !has_pdf_header(bytes) {
|
||||
return Err(ReadKind::InvalidPdf);
|
||||
}
|
||||
locate_attachment(bytes, limits)
|
||||
}
|
||||
|
||||
/// The first candidate whose data is a versioned JSON object. None: `attachment_stripped`
|
||||
/// when the stamp is visible, else `no_attachment`.
|
||||
pub fn locate_attachment<'a>(
|
||||
bytes: &'a [u8],
|
||||
limits: ProbeLimits,
|
||||
) -> Result<LocatedAttachment<'a>, ReadKind> {
|
||||
let mut budget = MAX_ATTACHMENT_BYTES;
|
||||
for (index, candidate) in find_embedded_files(bytes).into_iter().enumerate() {
|
||||
if candidate.length > budget {
|
||||
break;
|
||||
}
|
||||
let Some(data) = candidate_bytes(bytes, &candidate, budget) else {
|
||||
continue;
|
||||
};
|
||||
budget -= data.len();
|
||||
if let Some(located) = probe(index, candidate, data, limits) {
|
||||
return Ok(located);
|
||||
}
|
||||
}
|
||||
Err(if has_archive_stamp(bytes) {
|
||||
ReadKind::AttachmentStripped
|
||||
} else {
|
||||
ReadKind::NoAttachment
|
||||
})
|
||||
}
|
||||
|
||||
/// Raw stream data borrowed from the file, or its inflation under `budget`; `None` when the
|
||||
/// stream lies outside the file or inflates past the budget.
|
||||
pub fn candidate_bytes<'a>(
|
||||
bytes: &'a [u8],
|
||||
candidate: &EmbeddedFileCandidate,
|
||||
budget: usize,
|
||||
) -> Option<Cow<'a, [u8]>> {
|
||||
let end = candidate.offset.checked_add(candidate.length)?;
|
||||
let data = bytes.get(candidate.offset..end)?;
|
||||
match candidate.filter {
|
||||
StreamFilter::None => Some(Cow::Borrowed(data)),
|
||||
StreamFilter::FlateDecode => inflate_flate(data, budget).map(Cow::Owned),
|
||||
}
|
||||
}
|
||||
|
||||
/// The JavaScript-path steps on one candidate, in the TypeScript order: `{` first, the
|
||||
/// linear depth and value scan, the lossy decode, the overflow rewrite, the version probe.
|
||||
fn probe<'a>(
|
||||
index: usize,
|
||||
candidate: EmbeddedFileCandidate,
|
||||
data: Cow<'a, [u8]>,
|
||||
limits: ProbeLimits,
|
||||
) -> Option<LocatedAttachment<'a>> {
|
||||
if !starts_json_object(&data) {
|
||||
return None;
|
||||
}
|
||||
let scan = prescan(&data);
|
||||
if !scan.within_depth || scan.values > limits.max_json_values {
|
||||
return None;
|
||||
}
|
||||
let data_len = data.len();
|
||||
let text = into_text(data);
|
||||
if !is_versioned_object(&text) {
|
||||
return None;
|
||||
}
|
||||
let text = rewritten(text);
|
||||
Some(LocatedAttachment {
|
||||
index,
|
||||
candidate,
|
||||
data_len,
|
||||
prescan: scan,
|
||||
text,
|
||||
})
|
||||
}
|
||||
|
||||
/// The lossy decode without a second copy: borrowed data stays borrowed, owned data that is
|
||||
/// valid UTF-8 moves into its `String`.
|
||||
fn into_text(data: Cow<'_, [u8]>) -> Cow<'_, str> {
|
||||
match data {
|
||||
Cow::Borrowed(bytes) => decode_lossy(bytes),
|
||||
Cow::Owned(mut bytes) => {
|
||||
if bytes.starts_with(b"\xEF\xBB\xBF") {
|
||||
bytes.drain(..3);
|
||||
}
|
||||
match String::from_utf8(bytes) {
|
||||
Ok(text) => Cow::Owned(text),
|
||||
Err(error) => Cow::Owned(String::from_utf8_lossy(error.as_bytes()).into_owned()),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn rewritten(text: Cow<'_, str>) -> Cow<'_, str> {
|
||||
match rewrite_overflow_numbers(&text) {
|
||||
Cow::Borrowed(_) => text,
|
||||
Cow::Owned(rewritten) => Cow::Owned(rewritten),
|
||||
}
|
||||
}
|
||||
@@ -1 +1,297 @@
|
||||
//! R4.2: placeholder, written by its track.
|
||||
//! R4.2d: the attachment probe and the PDF import against the frozen TypeScript outcomes.
|
||||
//!
|
||||
//! O7 (`oracles/r4/read.json`, `packages/pdf/tests/r4.reader-oracles.test.ts`): per input,
|
||||
//! `locateAttachment` / `readAttachedDocument` (candidate index, name, data offset, bytes
|
||||
//! digest, digest of `JSON.stringify` of the parsed value) and `readReportArchive` on the
|
||||
//! draft migration map (origin, counters, snapshot rejection, report and archive digests).
|
||||
//! O8 (`oracles/r4/tolerance.json`, `apps/app/tests/unit/r4.tolerance-oracles.test.ts`): the
|
||||
//! full R3 import path (scan → core `archive.import` on the bundled draft sheet) on 51 valid
|
||||
//! and hostile PDFs, compared with `import_pdf` and with the facade envelope.
|
||||
//!
|
||||
//! Differences the plan declares (§A.3, OD-4) are listed in `declared()` with the Rust
|
||||
//! outcome; any other difference fails. The cap is off (`ProbeLimits::UNBOUNDED`) so that the
|
||||
//! comparison measures the TypeScript behaviour; the OD-3 cap is measured in R4.2e.
|
||||
#![allow(clippy::disallowed_methods, clippy::unwrap_used, clippy::expect_used)]
|
||||
|
||||
mod common;
|
||||
|
||||
use common::r4::{r4_input, r4_oracle, repo_root, sha256_hex};
|
||||
use faunosteo_core::domain::archive::read_archive_document;
|
||||
use faunosteo_core::domain::js::parse_json;
|
||||
use faunosteo_core::domain::sheet::{BundledSheetData, LoadedSheet};
|
||||
use faunosteo_core::facade;
|
||||
use faunosteo_core::json::to_js_compact;
|
||||
use faunosteo_core::pdf::import::{PdfImportFailure, PdfImportKind, import_pdf};
|
||||
use faunosteo_core::pdf::read::probe::{
|
||||
ProbeLimits, candidate_bytes, locate_attachment, read_attached_document,
|
||||
};
|
||||
use faunosteo_core::pdf::read::scan::find_embedded_files;
|
||||
use serde_json::{Value, json};
|
||||
|
||||
const SEED: &str = "supabase/seed/anatomical-sheets/equine/draft";
|
||||
|
||||
/// The bundled draft sheet exactly as `apps/app/vite/sheet-bundle.ts` embeds it.
|
||||
fn draft_sheet() -> LoadedSheet {
|
||||
let seed = repo_root().join(SEED);
|
||||
let read = |name: &str| std::fs::read_to_string(seed.join(name)).unwrap();
|
||||
let manifest: Value = serde_json::from_str(&read("assets.json")).unwrap();
|
||||
let mut assets = serde_json::Map::new();
|
||||
let mut hashes = serde_json::Map::new();
|
||||
for asset in manifest.as_array().unwrap() {
|
||||
let id = asset["svg_asset_id"].as_str().unwrap().to_owned();
|
||||
let file = format!("assets/{}", asset["file"].as_str().unwrap());
|
||||
assets.insert(id.clone(), Value::String(read(&file)));
|
||||
hashes.insert(id, asset["sha256"].clone());
|
||||
}
|
||||
let payload = json!({
|
||||
"status": "draft",
|
||||
"ref": parse_json(&read("sheet-ref.json")).unwrap(),
|
||||
"document": parse_json(&read("document.json")).unwrap(),
|
||||
"migrationMap": parse_json(&read("report-migration-map.json")).unwrap(),
|
||||
"assets": assets,
|
||||
"assetHashes": hashes,
|
||||
});
|
||||
let data: BundledSheetData = serde_json::from_value(payload).unwrap();
|
||||
LoadedSheet::from_bundle(data).unwrap()
|
||||
}
|
||||
|
||||
fn digest_js(value: &impl serde::Serialize) -> String {
|
||||
sha256_hex(to_js_compact(value).unwrap().as_bytes())
|
||||
}
|
||||
|
||||
/// The `file` an O7 row loads (a generator row names its description).
|
||||
fn input_bytes(row: &Value) -> Vec<u8> {
|
||||
r4_input("read", row["file"].as_str().unwrap())
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// O7: locate and read
|
||||
|
||||
/// `{ ok: true, candidate, name, dataOffset, bytesLength, bytesSha256, objectSha256 }` or
|
||||
/// `{ ok: false, kind }`, as the TypeScript recorded `locateAttachment`.
|
||||
fn located_row(
|
||||
bytes: &[u8],
|
||||
located: Result<
|
||||
&faunosteo_core::pdf::read::probe::LocatedAttachment<'_>,
|
||||
&faunosteo_core::pdf::read::ReadKind,
|
||||
>,
|
||||
) -> Value {
|
||||
match located {
|
||||
Err(kind) => json!({ "ok": false, "kind": kind.as_str() }),
|
||||
Ok(found) => {
|
||||
let candidates = find_embedded_files(bytes);
|
||||
let data = candidate_bytes(bytes, &candidates[found.index], usize::MAX).unwrap();
|
||||
// The TypeScript digested `JSON.stringify(JSON.parse(text))`; the core reads the
|
||||
// same text (lone surrogate inputs are the declared OD-4 rows, listed below).
|
||||
let object = parse_json(&found.text).map(|value| digest_js(&value)).ok();
|
||||
json!({
|
||||
"ok": true,
|
||||
"candidate": found.index,
|
||||
"name": found.candidate.name,
|
||||
"dataOffset": found.candidate.offset,
|
||||
"bytesLength": found.data_len,
|
||||
"bytesSha256": sha256_hex(&data),
|
||||
"objectSha256": object,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// `readReportArchive` on the draft map: the layered reading before any rebase.
|
||||
fn archive_row(text: &str, sheet: &LoadedSheet) -> Value {
|
||||
let Ok(data) = parse_json(text) else {
|
||||
return json!({ "ok": false, "kind": "lone_surrogate_or_syntax" });
|
||||
};
|
||||
match read_archive_document(&data, &sheet.migration_map) {
|
||||
Ok(read) => json!({
|
||||
"ok": true,
|
||||
"origin": read.origin,
|
||||
"migrated": read.migrated,
|
||||
"unresolvedZones": read.unresolved_zones,
|
||||
"unresolvedFindings": read.unresolved_findings,
|
||||
"snapshotRejected": read.snapshot_rejected,
|
||||
"reportSha256": digest_js(&read.report),
|
||||
// The frozen TS domain stringified its Zod envelope output, keys in schema
|
||||
// order; the core keeps the document's key order (R2, what the R3 application
|
||||
// stored and what O8 pins). Presence is compared, not the digest.
|
||||
"archiveSha256": read.archive.as_ref().map(|_| "present"),
|
||||
}),
|
||||
Err(faunosteo_core::domain::report::ReadFailure::UnsupportedVersion(Some(found))) => {
|
||||
json!({ "ok": false, "kind": "incompatible_version", "foundVersion": found })
|
||||
}
|
||||
Err(_) => json!({ "ok": false, "kind": "no_attachment" }),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn o7_locate_and_read_match_the_typescript() {
|
||||
let oracle = r4_oracle("read.json");
|
||||
let sheet = draft_sheet();
|
||||
let (mut inputs, mut located, mut archives, mut mismatches) = (0, 0, 0, Vec::new());
|
||||
for row in oracle["inputs"].as_array().unwrap() {
|
||||
if row["kind"] != "pdf" {
|
||||
continue;
|
||||
}
|
||||
inputs += 1;
|
||||
let file = row["file"].as_str().unwrap();
|
||||
let bytes = input_bytes(row);
|
||||
let outcomes = &row["outcomes"];
|
||||
if let Some(expected) = outcomes.get("locateAttachment") {
|
||||
located += 1;
|
||||
let found = locate_attachment(&bytes, ProbeLimits::UNBOUNDED);
|
||||
let actual = located_row(&bytes, found.as_ref());
|
||||
compare(&mut mismatches, file, "locateAttachment", expected, &actual);
|
||||
}
|
||||
let read = read_attached_document(&bytes, ProbeLimits::UNBOUNDED);
|
||||
if let Some(expected) = outcomes.get("readAttachedDocument") {
|
||||
let actual = located_row(&bytes, read.as_ref());
|
||||
compare(
|
||||
&mut mismatches,
|
||||
file,
|
||||
"readAttachedDocument",
|
||||
expected,
|
||||
&actual,
|
||||
);
|
||||
}
|
||||
if let Some(expected) = outcomes.get("readReportArchive") {
|
||||
archives += 1;
|
||||
let actual = match &read {
|
||||
Err(kind) => json!({ "ok": false, "kind": kind.as_str() }),
|
||||
Ok(found) => archive_row(&found.text, &sheet),
|
||||
};
|
||||
let mut expected = expected.clone();
|
||||
if let Some(digest) = expected.get_mut("archiveSha256")
|
||||
&& !digest.is_null()
|
||||
{
|
||||
*digest = Value::from("present");
|
||||
}
|
||||
compare(
|
||||
&mut mismatches,
|
||||
file,
|
||||
"readReportArchive",
|
||||
&expected,
|
||||
&actual,
|
||||
);
|
||||
}
|
||||
}
|
||||
assert!(mismatches.is_empty(), "{}", mismatches.join("\n"));
|
||||
assert_eq!(inputs, 144);
|
||||
println!("O7: {inputs} inputs, {located} locate rows, {archives} archive rows");
|
||||
}
|
||||
|
||||
fn compare(
|
||||
mismatches: &mut Vec<String>,
|
||||
file: &str,
|
||||
field: &str,
|
||||
expected: &Value,
|
||||
actual: &Value,
|
||||
) {
|
||||
// Compared in the JavaScript spelling: `4` and `4.0` are one number there.
|
||||
if to_js_compact(expected).unwrap() != to_js_compact(actual).unwrap() {
|
||||
mismatches.push(format!(
|
||||
"{file} {field}\n expected {expected}\n actual {actual}"
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// O8: the full import path
|
||||
|
||||
/// The O8 outcome of one PDF through `import_pdf`, in the oracle's shape.
|
||||
fn o8_outcome(sheet: &LoadedSheet, pdf: &[u8]) -> Value {
|
||||
match import_pdf(sheet, pdf, ProbeLimits::UNBOUNDED) {
|
||||
Ok(import) => json!({
|
||||
"ok": true,
|
||||
"reportSha256": digest_js(&import.rebase.report),
|
||||
"archiveSha256": import.read.archive.as_ref().map(digest_js),
|
||||
"conclusion": import.rebase.report.conclusion,
|
||||
"read": {
|
||||
"origin": import.read.origin,
|
||||
"migrated": import.read.migrated,
|
||||
"unresolvedZones": import.read.unresolved_zones,
|
||||
"unresolvedFindings": import.read.unresolved_findings,
|
||||
"snapshotRejected": import.read.snapshot_rejected,
|
||||
},
|
||||
"rebase": {
|
||||
"rebased": import.rebase.rebased,
|
||||
"unresolvedZones": import.rebase.unresolved_zones,
|
||||
"unresolvedFindings": import.rebase.unresolved_findings,
|
||||
},
|
||||
}),
|
||||
Err(PdfImportFailure::Import(
|
||||
faunosteo_core::domain::session::ImportFailure::IncompatibleVersion(found),
|
||||
)) => json!({ "ok": false, "kind": "incompatible_version", "foundVersion": found }),
|
||||
Err(failure) => json!({ "ok": false, "kind": PdfImportKind::from(&failure).as_str() }),
|
||||
}
|
||||
}
|
||||
|
||||
/// O8 cases whose Rust outcome differs on purpose (plan §A.3, OD-4: a lone surrogate escape
|
||||
/// in a value a duplicate key overwrote imports in JavaScript, which dropped it when it
|
||||
/// stringified the parsed object; the core refuses the text whole, nothing altered).
|
||||
fn declared_o8(id: &str) -> Option<Value> {
|
||||
match id {
|
||||
"surrogate-content-overwritten"
|
||||
| "surrogate-version-overwritten"
|
||||
| "two-files-decoy-first" => Some(json!({ "ok": false, "kind": "lone_surrogate" })),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn o8_import_matches_the_r3_path_outside_the_declared_differences() {
|
||||
let oracle = r4_oracle("tolerance.json");
|
||||
let sheet = draft_sheet();
|
||||
let mut mismatches = Vec::new();
|
||||
let (mut cases, mut declared) = (0, 0);
|
||||
for case in oracle["cases"].as_array().unwrap() {
|
||||
cases += 1;
|
||||
let id = case["id"].as_str().unwrap();
|
||||
let path = repo_root().join(case["pdf"].as_str().unwrap());
|
||||
let pdf = std::fs::read(&path).unwrap();
|
||||
assert_eq!(sha256_hex(&pdf), case["pdfSha256"], "{id}: fixture bytes");
|
||||
let actual = o8_outcome(&sheet, &pdf);
|
||||
let expected = match declared_o8(id) {
|
||||
Some(rust) => {
|
||||
declared += 1;
|
||||
assert_ne!(
|
||||
rust, case["outcome"],
|
||||
"{id}: declared difference is not one"
|
||||
);
|
||||
rust
|
||||
}
|
||||
None => case["outcome"].clone(),
|
||||
};
|
||||
compare(&mut mismatches, id, "outcome", &expected, &actual);
|
||||
}
|
||||
assert!(mismatches.is_empty(), "{}", mismatches.join("\n"));
|
||||
assert_eq!(cases, 51);
|
||||
assert_eq!(declared, 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_facade_envelope_carries_the_closed_kinds() {
|
||||
let sheet = draft_sheet();
|
||||
let oracle = r4_oracle("tolerance.json");
|
||||
for case in oracle["cases"].as_array().unwrap() {
|
||||
let id = case["id"].as_str().unwrap();
|
||||
let pdf = std::fs::read(repo_root().join(case["pdf"].as_str().unwrap())).unwrap();
|
||||
let envelope: Value = serde_json::from_str(&facade::import_pdf(&sheet, &pdf)).unwrap();
|
||||
let expected = declared_o8(id).unwrap_or_else(|| case["outcome"].clone());
|
||||
assert_eq!(envelope["ok"], expected["ok"], "{id}");
|
||||
if expected["ok"] == false {
|
||||
assert_eq!(envelope["kind"], expected["kind"], "{id}");
|
||||
assert_eq!(
|
||||
envelope.get("found").cloned(),
|
||||
expected.get("foundVersion").cloned(),
|
||||
"{id}"
|
||||
);
|
||||
} else {
|
||||
assert_eq!(
|
||||
envelope["value"]["rebase"]["report"]["conclusion"], expected["conclusion"],
|
||||
"{id}"
|
||||
);
|
||||
}
|
||||
}
|
||||
let not_a_pdf = facade::import_pdf(&sheet, b"hello");
|
||||
assert!(not_a_pdf.contains(r#""kind":"invalid_pdf""#), "{not_a_pdf}");
|
||||
}
|
||||
@@ -41,6 +41,13 @@ impl SheetHandle {
|
||||
pub fn call(&self, op: &str, input: &str) -> String {
|
||||
faunosteo_core::facade::call_with_sheet(&self.sheet, op, input)
|
||||
}
|
||||
|
||||
/// Reads `bytes` as a Faunosteo PDF and imports its attachment onto this sheet (R4, plan
|
||||
/// §B.1.4): the envelope of `archive.import`, plus the reader's kinds (`PdfImportKind`).
|
||||
/// The bytes are copied once into the module's memory by the glue.
|
||||
pub fn import_pdf(&self, bytes: &[u8]) -> String {
|
||||
faunosteo_core::facade::import_pdf(&self.sheet, bytes)
|
||||
}
|
||||
}
|
||||
|
||||
/// R0 proof: on wasm32 a panic is a trap (panic = abort), reaching JavaScript as a
|
||||
|
||||
@@ -27,6 +27,8 @@ export interface GlueModule {
|
||||
|
||||
export interface GlueSheetHandle {
|
||||
call(op: string, input: string): string
|
||||
/** R4: the PDF import, bytes in, the same envelope out (`PdfImportKind` on failure). */
|
||||
import_pdf(bytes: Uint8Array): string
|
||||
}
|
||||
|
||||
/** Where the module comes from: the browser (Vite asset URLs) or Node (files). */
|
||||
|
||||
@@ -0,0 +1,182 @@
|
||||
/**
|
||||
* R4.2d: the PDF import through the shipped module (`SheetHandle.import_pdf`), in Node,
|
||||
* against the frozen R3 outcomes: O8 (`crates/core/tests/oracles/r4/tolerance.json`, the
|
||||
* full R3 import path on the bundled draft sheet, 51 valid and hostile PDFs), the O2 PDFs
|
||||
* of the R3 build, the O3 envelopes of Step 6 and the Step 5 fixtures. The cargo suite
|
||||
* `crates/core/tests/pdf_import.rs` pins the same outcomes natively; this one proves the
|
||||
* wasm build and the glue (bytes copied once, the envelope string) give the same answers,
|
||||
* and that none of the hostile files traps the instance.
|
||||
*/
|
||||
import { createHash } from 'node:crypto'
|
||||
import { readdirSync, readFileSync } from 'node:fs'
|
||||
import { join } from 'node:path'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
|
||||
import {
|
||||
CoreRuntime,
|
||||
type GlueModule,
|
||||
type GlueSheetHandle,
|
||||
} from '../src/index'
|
||||
import { nodeCoreSource } from '../src/node'
|
||||
|
||||
const ROOT = fileURLToPath(new URL('../../../', import.meta.url))
|
||||
const SEED = join(ROOT, 'supabase/seed/anatomical-sheets/equine/draft')
|
||||
const FIXTURES = join(ROOT, 'apps/app/tests/fixtures')
|
||||
const read = (path: string) => readFileSync(path, 'utf8')
|
||||
const sha256 = (data: Uint8Array | string) =>
|
||||
createHash('sha256').update(data).digest('hex')
|
||||
|
||||
/** The bundled draft exactly as `apps/app/vite/sheet-bundle.ts` embeds it. */
|
||||
function draftBundle(): string {
|
||||
const manifest = JSON.parse(read(join(SEED, 'assets.json'))) as Array<{
|
||||
file: string
|
||||
sha256: string
|
||||
svg_asset_id: string
|
||||
}>
|
||||
const assets: Record<string, string> = {}
|
||||
const assetHashes: Record<string, string> = {}
|
||||
for (const asset of manifest) {
|
||||
assets[asset.svg_asset_id] = read(join(SEED, 'assets', asset.file))
|
||||
assetHashes[asset.svg_asset_id] = asset.sha256
|
||||
}
|
||||
return JSON.stringify({
|
||||
status: 'draft',
|
||||
ref: JSON.parse(read(join(SEED, 'sheet-ref.json'))),
|
||||
document: JSON.parse(read(join(SEED, 'document.json'))),
|
||||
migrationMap: JSON.parse(read(join(SEED, 'report-migration-map.json'))),
|
||||
assets,
|
||||
assetHashes,
|
||||
})
|
||||
}
|
||||
|
||||
interface Envelope {
|
||||
ok: boolean
|
||||
kind?: string
|
||||
found?: number
|
||||
value?: {
|
||||
read: {
|
||||
origin: string
|
||||
migrated: boolean
|
||||
unresolvedZones: number
|
||||
unresolvedFindings: number
|
||||
snapshotRejected?: string
|
||||
archive?: unknown
|
||||
}
|
||||
rebase: {
|
||||
report: { conclusion: string }
|
||||
rebased: boolean
|
||||
unresolvedZones: number
|
||||
unresolvedFindings: number
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** The O8 row of one envelope, in the oracle's shape. */
|
||||
function o8Row(envelope: Envelope): unknown {
|
||||
if (!envelope.ok) {
|
||||
return envelope.found === undefined
|
||||
? { ok: false, kind: envelope.kind }
|
||||
: { ok: false, kind: envelope.kind, foundVersion: envelope.found }
|
||||
}
|
||||
const { read, rebase } = envelope.value!
|
||||
return {
|
||||
ok: true,
|
||||
reportSha256: sha256(JSON.stringify(rebase.report)),
|
||||
archiveSha256: read.archive ? sha256(JSON.stringify(read.archive)) : null,
|
||||
conclusion: rebase.report.conclusion,
|
||||
read: {
|
||||
origin: read.origin,
|
||||
migrated: read.migrated,
|
||||
unresolvedZones: read.unresolvedZones,
|
||||
unresolvedFindings: read.unresolvedFindings,
|
||||
snapshotRejected: read.snapshotRejected ?? null,
|
||||
},
|
||||
rebase: {
|
||||
rebased: rebase.rebased,
|
||||
unresolvedZones: rebase.unresolvedZones,
|
||||
unresolvedFindings: rebase.unresolvedFindings,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/** O8 cases whose Rust outcome differs on purpose (plan §A.3, OD-4). */
|
||||
const DECLARED: Record<string, unknown> = {
|
||||
'surrogate-content-overwritten': { ok: false, kind: 'lone_surrogate' },
|
||||
'surrogate-version-overwritten': { ok: false, kind: 'lone_surrogate' },
|
||||
'two-files-decoy-first': { ok: false, kind: 'lone_surrogate' },
|
||||
}
|
||||
|
||||
function pdfsUnder(dir: string): string[] {
|
||||
return readdirSync(dir, { recursive: true, encoding: 'utf8' })
|
||||
.filter((name) => name.endsWith('.pdf'))
|
||||
.map((name) => join(dir, name))
|
||||
}
|
||||
|
||||
describe('R4 PDF import through the shipped module (Node)', () => {
|
||||
let runtime: CoreRuntime
|
||||
let sheet: GlueSheetHandle
|
||||
let glue: GlueModule
|
||||
beforeAll(async () => {
|
||||
const source = nodeCoreSource('pkg')
|
||||
runtime = new CoreRuntime({
|
||||
compile: source.compile,
|
||||
importGlue: async (n) => {
|
||||
glue = await source.importGlue(n)
|
||||
return glue
|
||||
},
|
||||
})
|
||||
expect(await runtime.ready()).toBe(true)
|
||||
sheet = new glue.SheetHandle(draftBundle())
|
||||
})
|
||||
|
||||
it('gives the R3 outcome on every O8 case, outside the declared differences', () => {
|
||||
const oracle = JSON.parse(
|
||||
read(join(ROOT, 'crates/core/tests/oracles/r4/tolerance.json')),
|
||||
) as {
|
||||
cases: Array<{
|
||||
id: string
|
||||
pdf: string
|
||||
pdfSha256: string
|
||||
outcome: unknown
|
||||
}>
|
||||
}
|
||||
expect(oracle.cases).toHaveLength(51)
|
||||
let declared = 0
|
||||
for (const c of oracle.cases) {
|
||||
const pdf = new Uint8Array(readFileSync(join(ROOT, c.pdf)))
|
||||
expect(sha256(pdf), c.id).toBe(c.pdfSha256)
|
||||
const envelope = JSON.parse(sheet.import_pdf(pdf)) as Envelope
|
||||
const expected = DECLARED[c.id] ?? c.outcome
|
||||
if (c.id in DECLARED) declared += 1
|
||||
expect(o8Row(envelope), c.id).toEqual(expected)
|
||||
}
|
||||
expect(declared).toBe(3)
|
||||
expect(runtime.status).toBe('ready')
|
||||
})
|
||||
|
||||
it('imports every reference PDF of the R3 build, Step 6 and Step 5', () => {
|
||||
const files = [
|
||||
join(FIXTURES, 'report-v2.full.pdf'),
|
||||
join(FIXTURES, 'report-v2.minimal.pdf'),
|
||||
join(FIXTURES, 'report-v3.bare.pdf'),
|
||||
...pdfsUnder(join(FIXTURES, 'pdf/r3')),
|
||||
...pdfsUnder(join(FIXTURES, 'pdf/step6')),
|
||||
]
|
||||
expect(files.length).toBeGreaterThanOrEqual(20)
|
||||
for (const file of files) {
|
||||
const envelope = JSON.parse(
|
||||
sheet.import_pdf(new Uint8Array(readFileSync(file))),
|
||||
) as Envelope
|
||||
expect(envelope.ok, file).toBe(true)
|
||||
expect(typeof envelope.value?.rebase.report.conclusion, file).toBe(
|
||||
'string',
|
||||
)
|
||||
}
|
||||
})
|
||||
|
||||
it('answers invalid_pdf on bytes that are not a PDF, without a trap', () => {
|
||||
const envelope = JSON.parse(sheet.import_pdf(new Uint8Array([1, 2, 3])))
|
||||
expect(envelope).toEqual({ ok: false, kind: 'invalid_pdf', detail: '' })
|
||||
expect(runtime.status).toBe('ready')
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,7 @@
|
||||
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
|
||||
|
||||
/**
|
||||
* The closed kinds `SheetHandle.importPdf` answers with `ok: false` (plan §B.1.8): the
|
||||
* reader's three, the session's, and the facade's `internal`.
|
||||
*/
|
||||
export type PdfImportKind = "invalid_pdf" | "no_attachment" | "attachment_stripped" | "incompatible_version" | "lone_surrogate" | "invalid" | "internal";
|
||||
@@ -49,6 +49,7 @@ export type { Observation } from './Observation'
|
||||
export type { ObservationIndexInput } from './ObservationIndexInput'
|
||||
export type { Owner } from './Owner'
|
||||
export type { PathGeometry } from './PathGeometry'
|
||||
export type { PdfImportKind } from './PdfImportKind'
|
||||
export type { PdfLabelSet } from './PdfLabelSet'
|
||||
export type { PdfLabels } from './PdfLabels'
|
||||
export type { PngAsset } from './PngAsset'
|
||||
|
||||
@@ -23,5 +23,5 @@ cargo test -p faunosteo-core --features pdf-read --locked
|
||||
# `git status --porcelain --untracked-files=all -- packages/core-wasm/types` must stay
|
||||
# empty afterwards (scripts/check-all.sh, .gitea/workflows/ci.yml): the committed types
|
||||
# never drift from the Rust structs, and a new type file counts as drift.
|
||||
cargo test -p faunosteo-core --features ts --locked
|
||||
cargo test -p faunosteo-core --features ts,pdf-read --locked
|
||||
cargo deny check licenses bans sources
|
||||
Reference in new issue
Block a user