feat(domain): R4.2d attachment probe, import_pdf and SheetHandle.import_pdf on the wasm facade
locate_attachment (shared 16 MiB budget, JS-path steps in the TypeScript order, OD-3 cap hook off until measured), pdf::import, facade import_pdf with the PdfImportKind kinds (ts-rs exported). cargo: O7 144 inputs (locate, read, archive rows) and O8 51 cases equal to the R3 path, 3 OD-4 differences declared (lone surrogate in an overwritten duplicate value). Node through the shipped module: the same O8 outcomes, every reference PDF. Module 421.7 KB gzip (W1 budget 600 KB).
This commit is contained in:
1 parent
9850a8341a
commit
f8579bc19a
13 files changed
+813
-14
No files matched your search
@@ -61,6 +61,22 @@ pub fn call_with_sheet(sheet: &LoadedSheet, op: &str, input: &str) -> String {
|
||||
render(crate::ops::on_sheet(sheet, op, input).unwrap_or_else(|| Err(unknown(op))))
|
||||
}
|
||||
|
||||
/// Reads `bytes` as a Faunosteo PDF and imports its attachment onto the session's sheet
|
||||
/// (R4, plan §B.1.4): the envelope of `archive.import`, plus the reader's kinds
|
||||
/// `invalid_pdf`, `no_attachment`, `attachment_stripped` (`PdfImportKind`).
|
||||
#[cfg(feature = "pdf-read")]
|
||||
pub fn import_pdf(sheet: &LoadedSheet, bytes: &[u8]) -> String {
|
||||
use crate::pdf::import::{PdfImportFailure, PdfImportKind, import_pdf};
|
||||
use crate::pdf::read::probe::ProbeLimits;
|
||||
let outcome = import_pdf(sheet, bytes, ProbeLimits::PRODUCTION)
|
||||
.map_err(|failure| match failure {
|
||||
PdfImportFailure::Import(failure) => crate::ops::import_failure(failure),
|
||||
read => Failure::new(PdfImportKind::from(&read).as_str(), ""),
|
||||
})
|
||||
.and_then(|import| to_value(&import));
|
||||
render(outcome)
|
||||
}
|
||||
|
||||
fn unknown(op: &str) -> Failure {
|
||||
Failure::new("unknown_op", format!("no operation named {op}"))
|
||||
}
|
||||
|
||||
+11
-8
@@ -327,17 +327,20 @@ fn draft_restore(sheet: &LoadedSheet, input: &str) -> Outcome {
|
||||
/// The input is the attachment's JSON as the byte scan read it (any shape).
|
||||
fn archive_import(sheet: &LoadedSheet, input: &str) -> Outcome {
|
||||
session::import_archive(sheet, input)
|
||||
.map_err(|failure| match failure {
|
||||
ImportFailure::IncompatibleVersion(found) => {
|
||||
versioned("incompatible_version", Some(found))
|
||||
}
|
||||
ImportFailure::NoAttachment => Failure::new("no_attachment", ""),
|
||||
ImportFailure::LoneSurrogate(detail) => Failure::new("lone_surrogate", detail),
|
||||
ImportFailure::Invalid(detail) => Failure::new("invalid", detail),
|
||||
})
|
||||
.map_err(import_failure)
|
||||
.and_then(|import| to_value(&import))
|
||||
}
|
||||
|
||||
/// The closed kinds of a session import failure, shared with the PDF import.
|
||||
pub(crate) fn import_failure(failure: ImportFailure) -> Failure {
|
||||
match failure {
|
||||
ImportFailure::IncompatibleVersion(found) => versioned("incompatible_version", Some(found)),
|
||||
ImportFailure::NoAttachment => Failure::new("no_attachment", ""),
|
||||
ImportFailure::LoneSurrogate(detail) => Failure::new("lone_surrogate", detail),
|
||||
ImportFailure::Invalid(detail) => Failure::new("invalid", detail),
|
||||
}
|
||||
}
|
||||
|
||||
fn archive_snapshot(sheet: &LoadedSheet, input: &str) -> Outcome {
|
||||
let input: SnapshotInput = parse_input(input)?;
|
||||
to_value(&build_sheet_snapshot(sheet, &input.report))
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
//! The import of a Faunosteo PDF (`importReportFromPdf` of `apps/app/src/pdf/import.ts`):
|
||||
//! the attachment located and brought to text by [`crate::pdf::read`], then the R2 session
|
||||
//! import (`archive.import`), unchanged, on the loaded sheet. One call, so the worker that
|
||||
//! will run it (R4.3) hands the main thread the core's canonical output only.
|
||||
|
||||
use serde::Serialize;
|
||||
#[cfg(feature = "ts")]
|
||||
use ts_rs::TS;
|
||||
|
||||
use crate::domain::session::{ArchiveImport, ImportFailure, import_archive};
|
||||
use crate::domain::sheet::LoadedSheet;
|
||||
use crate::pdf::read::ReadKind;
|
||||
use crate::pdf::read::probe::{ProbeLimits, read_attached_document};
|
||||
|
||||
/// Why a PDF does not import: the reader's kinds, then the session's.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub enum PdfImportFailure {
|
||||
Read(ReadKind),
|
||||
Import(ImportFailure),
|
||||
}
|
||||
|
||||
/// Reads `bytes` as a Faunosteo PDF and imports its attachment onto `sheet`.
|
||||
pub fn import_pdf(
|
||||
sheet: &LoadedSheet,
|
||||
bytes: &[u8],
|
||||
limits: ProbeLimits,
|
||||
) -> Result<ArchiveImport, PdfImportFailure> {
|
||||
let located = read_attached_document(bytes, limits).map_err(PdfImportFailure::Read)?;
|
||||
import_archive(sheet, &located.text).map_err(PdfImportFailure::Import)
|
||||
}
|
||||
|
||||
/// The closed kinds `SheetHandle.importPdf` answers with `ok: false` (plan §B.1.8): the
|
||||
/// reader's three, the session's, and the facade's `internal`.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
|
||||
#[cfg_attr(feature = "ts", derive(TS), ts(export))]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum PdfImportKind {
|
||||
InvalidPdf,
|
||||
NoAttachment,
|
||||
AttachmentStripped,
|
||||
IncompatibleVersion,
|
||||
LoneSurrogate,
|
||||
Invalid,
|
||||
Internal,
|
||||
}
|
||||
|
||||
impl PdfImportKind {
|
||||
/// The kind name of the outcome envelope (the serde spelling).
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::InvalidPdf => "invalid_pdf",
|
||||
Self::NoAttachment => "no_attachment",
|
||||
Self::AttachmentStripped => "attachment_stripped",
|
||||
Self::IncompatibleVersion => "incompatible_version",
|
||||
Self::LoneSurrogate => "lone_surrogate",
|
||||
Self::Invalid => "invalid",
|
||||
Self::Internal => "internal",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<&PdfImportFailure> for PdfImportKind {
|
||||
fn from(failure: &PdfImportFailure) -> Self {
|
||||
match failure {
|
||||
PdfImportFailure::Read(ReadKind::InvalidPdf) => Self::InvalidPdf,
|
||||
PdfImportFailure::Read(ReadKind::NoAttachment)
|
||||
| PdfImportFailure::Import(ImportFailure::NoAttachment) => Self::NoAttachment,
|
||||
PdfImportFailure::Read(ReadKind::AttachmentStripped) => Self::AttachmentStripped,
|
||||
PdfImportFailure::Import(ImportFailure::IncompatibleVersion(_)) => {
|
||||
Self::IncompatibleVersion
|
||||
}
|
||||
PdfImportFailure::Import(ImportFailure::LoneSurrogate(_)) => Self::LoneSurrogate,
|
||||
PdfImportFailure::Import(ImportFailure::Invalid(_)) => Self::Invalid,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::PdfImportKind;
|
||||
|
||||
#[test]
|
||||
fn kind_names_are_the_serde_spelling() {
|
||||
for kind in [
|
||||
PdfImportKind::InvalidPdf,
|
||||
PdfImportKind::NoAttachment,
|
||||
PdfImportKind::AttachmentStripped,
|
||||
PdfImportKind::IncompatibleVersion,
|
||||
PdfImportKind::LoneSurrogate,
|
||||
PdfImportKind::Invalid,
|
||||
PdfImportKind::Internal,
|
||||
] {
|
||||
let serialised = serde_json::to_string(&kind).unwrap_or_default();
|
||||
assert_eq!(serialised, format!("\"{}\"", kind.as_str()));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -99,6 +99,19 @@ pub fn decode_lossy(data: &[u8]) -> Cow<'_, str> {
|
||||
/// the parse, as it failed `JSON.parse`. A token is the run of `[0-9+\-.eE]` that opens on a
|
||||
/// `-` or a digit in value position: when `JSON.parse` succeeds, that run is its number.
|
||||
pub fn rewrite_overflow_numbers(text: &str) -> Cow<'_, str> {
|
||||
rewrite_overflows(text, "null")
|
||||
}
|
||||
|
||||
/// The probe's view of the same text: `parseVersioned` tested `typeof version === 'number'`
|
||||
/// on the value `JSON.parse` gave, where an overflowing `version` was still `Infinity`, a
|
||||
/// number; it became `null` only when the object was stringified for the core. The probe copy
|
||||
/// therefore keeps every overflowing token a (finite) number, `0`, and the core reads the
|
||||
/// `null` form of [`rewrite_overflow_numbers`].
|
||||
pub fn rewrite_overflow_numbers_as_finite(text: &str) -> Cow<'_, str> {
|
||||
rewrite_overflows(text, "0")
|
||||
}
|
||||
|
||||
fn rewrite_overflows<'a>(text: &'a str, replacement: &str) -> Cow<'a, str> {
|
||||
let bytes = text.as_bytes();
|
||||
let mut lexer = Lexer::new();
|
||||
let mut out = String::new();
|
||||
@@ -112,7 +125,7 @@ pub fn rewrite_overflow_numbers(text: &str) -> Cow<'_, str> {
|
||||
// The run is ASCII, so both of its ends are char boundaries.
|
||||
if overflows(&text[at..end]) {
|
||||
out.push_str(&text[copied..at]);
|
||||
out.push_str("null");
|
||||
out.push_str(replacement);
|
||||
copied = end;
|
||||
}
|
||||
at = end;
|
||||
@@ -150,9 +163,11 @@ pub fn neutralise_lone_surrogates(text: &str) -> Cow<'_, str> {
|
||||
/// `parseVersioned`'s acceptance: the text reads as JSON (R2's `parse_json`, JavaScript
|
||||
/// duplicate-key semantics) once its lone surrogates are neutralised, is an object, and its
|
||||
/// `envelopeVersion` or `version` is a number. The parsed value is dropped: the core reads
|
||||
/// the real text. Expects a text already through [`rewrite_overflow_numbers`].
|
||||
/// the real text. Takes the decoded text as is: the overflowing numbers of the probe copy
|
||||
/// stay numbers ([`rewrite_overflow_numbers_as_finite`]), as `Infinity` was one.
|
||||
pub fn is_versioned_object(text: &str) -> bool {
|
||||
let Ok(Value::Object(record)) = parse_json(&neutralise_lone_surrogates(text)) else {
|
||||
let probe = rewrite_overflow_numbers_as_finite(text);
|
||||
let Ok(Value::Object(record)) = parse_json(&neutralise_lone_surrogates(&probe)) else {
|
||||
return false;
|
||||
};
|
||||
["envelopeVersion", "version"]
|
||||
|
||||
@@ -6,5 +6,6 @@
|
||||
//! pinned by the frozen oracles O7 (`tests/oracles/r4/read.json`), O8 (`tolerance.json`) and
|
||||
//! O9 (`inflate.json`). The writer (`write`) and the import entry (`import`) follow.
|
||||
|
||||
pub mod import;
|
||||
pub mod json_js;
|
||||
pub mod read;
|
||||
@@ -1 +1,173 @@
|
||||
//! R4.2d: `locateAttachment` of scan.ts. Written after R4.2a, b and c.
|
||||
//! `locateAttachment` and `readAttachedDocument` of scan.ts and read.ts: the first embedded
|
||||
//! file whose data is a versioned JSON object, read through the JavaScript-path steps of
|
||||
//! [`crate::pdf::json_js`] so that the text handed to the session is the one the TypeScript
|
||||
//! import handed to the core.
|
||||
//!
|
||||
//! One [`MAX_ATTACHMENT_BYTES`] budget of content (raw or inflated) is shared by every
|
||||
//! candidate: a candidate that does not fit what is left ends the search, so at most 16 MiB
|
||||
//! of candidate content is retained and probed. A cancelled inflation is not charged (audit
|
||||
//! of 3a6e926, finding 2), so up to [`MAX_CANDIDATES`] oversized streams may each cost one
|
||||
//! capped inflation of CPU, bounded and retaining nothing; charging them would let bombs
|
||||
//! starve a real unnamed attachment placed after them.
|
||||
//!
|
||||
//! OD-3 adds one cap the TypeScript did not have: a candidate whose value count (the
|
||||
//! parser's allocation proxy) exceeds the limit is skipped like one nested too deep, before
|
||||
//! any parser allocates its tree. Its threshold is measured at R4.2e.
|
||||
|
||||
use std::borrow::Cow;
|
||||
|
||||
use super::inflate::inflate_flate;
|
||||
use super::scan::find_embedded_files;
|
||||
use super::stamp::has_archive_stamp;
|
||||
use super::{
|
||||
EmbeddedFileCandidate, MAX_ATTACHMENT_BYTES, MAX_PDF_BYTES, ReadKind, StreamFilter,
|
||||
has_pdf_header,
|
||||
};
|
||||
use crate::pdf::json_js::{
|
||||
JsonPrescan, decode_lossy, is_versioned_object, prescan, rewrite_overflow_numbers,
|
||||
starts_json_object,
|
||||
};
|
||||
|
||||
/// The caps the probe applies on top of the scanner's own.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct ProbeLimits {
|
||||
/// Values (`json_js::prescan`) a candidate may hold before it is skipped (OD-3).
|
||||
pub max_json_values: u64,
|
||||
}
|
||||
|
||||
/// Values a candidate may hold before the probe skips it (OD-3). A legitimate archive
|
||||
/// holds a few thousand values; the threshold is set from the R4.2e measurements (10 × the
|
||||
/// largest legitimate archive) and confirmed by the owner before the import is wired.
|
||||
/// Until then the cap is off, so the parity suites measure the TypeScript behaviour.
|
||||
pub const MAX_JSON_VALUES: u64 = u64::MAX;
|
||||
|
||||
impl ProbeLimits {
|
||||
/// No value cap: the TypeScript behaviour, for the parity suites.
|
||||
pub const UNBOUNDED: Self = Self {
|
||||
max_json_values: u64::MAX,
|
||||
};
|
||||
/// The caps the application applies.
|
||||
pub const PRODUCTION: Self = Self {
|
||||
max_json_values: MAX_JSON_VALUES,
|
||||
};
|
||||
}
|
||||
|
||||
/// The attachment, located and brought to the text the session reads.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct LocatedAttachment<'a> {
|
||||
/// Index of the candidate in [`find_embedded_files`] order.
|
||||
pub index: usize,
|
||||
pub candidate: EmbeddedFileCandidate,
|
||||
/// Byte count of the stored or inflated data.
|
||||
pub data_len: usize,
|
||||
pub prescan: JsonPrescan,
|
||||
/// The data decoded as TextDecoder would, overflowing numbers rewritten to `null`: what
|
||||
/// `JSON.stringify(JSON.parse(text))` handed to the core, lone surrogate escapes kept.
|
||||
pub text: Cow<'a, str>,
|
||||
}
|
||||
|
||||
/// Size cap first, then the `%PDF-` header: anything else is `invalid_pdf`. Then the probe.
|
||||
pub fn read_attached_document<'a>(
|
||||
bytes: &'a [u8],
|
||||
limits: ProbeLimits,
|
||||
) -> Result<LocatedAttachment<'a>, ReadKind> {
|
||||
if bytes.len() > MAX_PDF_BYTES || !has_pdf_header(bytes) {
|
||||
return Err(ReadKind::InvalidPdf);
|
||||
}
|
||||
locate_attachment(bytes, limits)
|
||||
}
|
||||
|
||||
/// The first candidate whose data is a versioned JSON object. None: `attachment_stripped`
|
||||
/// when the stamp is visible, else `no_attachment`.
|
||||
pub fn locate_attachment<'a>(
|
||||
bytes: &'a [u8],
|
||||
limits: ProbeLimits,
|
||||
) -> Result<LocatedAttachment<'a>, ReadKind> {
|
||||
let mut budget = MAX_ATTACHMENT_BYTES;
|
||||
for (index, candidate) in find_embedded_files(bytes).into_iter().enumerate() {
|
||||
if candidate.length > budget {
|
||||
break;
|
||||
}
|
||||
let Some(data) = candidate_bytes(bytes, &candidate, budget) else {
|
||||
continue;
|
||||
};
|
||||
budget -= data.len();
|
||||
if let Some(located) = probe(index, candidate, data, limits) {
|
||||
return Ok(located);
|
||||
}
|
||||
}
|
||||
Err(if has_archive_stamp(bytes) {
|
||||
ReadKind::AttachmentStripped
|
||||
} else {
|
||||
ReadKind::NoAttachment
|
||||
})
|
||||
}
|
||||
|
||||
/// Raw stream data borrowed from the file, or its inflation under `budget`; `None` when the
|
||||
/// stream lies outside the file or inflates past the budget.
|
||||
pub fn candidate_bytes<'a>(
|
||||
bytes: &'a [u8],
|
||||
candidate: &EmbeddedFileCandidate,
|
||||
budget: usize,
|
||||
) -> Option<Cow<'a, [u8]>> {
|
||||
let end = candidate.offset.checked_add(candidate.length)?;
|
||||
let data = bytes.get(candidate.offset..end)?;
|
||||
match candidate.filter {
|
||||
StreamFilter::None => Some(Cow::Borrowed(data)),
|
||||
StreamFilter::FlateDecode => inflate_flate(data, budget).map(Cow::Owned),
|
||||
}
|
||||
}
|
||||
|
||||
/// The JavaScript-path steps on one candidate, in the TypeScript order: `{` first, the
|
||||
/// linear depth and value scan, the lossy decode, the overflow rewrite, the version probe.
|
||||
fn probe<'a>(
|
||||
index: usize,
|
||||
candidate: EmbeddedFileCandidate,
|
||||
data: Cow<'a, [u8]>,
|
||||
limits: ProbeLimits,
|
||||
) -> Option<LocatedAttachment<'a>> {
|
||||
if !starts_json_object(&data) {
|
||||
return None;
|
||||
}
|
||||
let scan = prescan(&data);
|
||||
if !scan.within_depth || scan.values > limits.max_json_values {
|
||||
return None;
|
||||
}
|
||||
let data_len = data.len();
|
||||
let text = into_text(data);
|
||||
if !is_versioned_object(&text) {
|
||||
return None;
|
||||
}
|
||||
let text = rewritten(text);
|
||||
Some(LocatedAttachment {
|
||||
index,
|
||||
candidate,
|
||||
data_len,
|
||||
prescan: scan,
|
||||
text,
|
||||
})
|
||||
}
|
||||
|
||||
/// The lossy decode without a second copy: borrowed data stays borrowed, owned data that is
|
||||
/// valid UTF-8 moves into its `String`.
|
||||
fn into_text(data: Cow<'_, [u8]>) -> Cow<'_, str> {
|
||||
match data {
|
||||
Cow::Borrowed(bytes) => decode_lossy(bytes),
|
||||
Cow::Owned(mut bytes) => {
|
||||
if bytes.starts_with(b"\xEF\xBB\xBF") {
|
||||
bytes.drain(..3);
|
||||
}
|
||||
match String::from_utf8(bytes) {
|
||||
Ok(text) => Cow::Owned(text),
|
||||
Err(error) => Cow::Owned(String::from_utf8_lossy(error.as_bytes()).into_owned()),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn rewritten(text: Cow<'_, str>) -> Cow<'_, str> {
|
||||
match rewrite_overflow_numbers(&text) {
|
||||
Cow::Borrowed(_) => text,
|
||||
Cow::Owned(rewritten) => Cow::Owned(rewritten),
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user