feat(domain): R4.2e OD-3 value cap (400,000), rescue-copy guard, 1 GiB linear ceiling, budget measurements
json_js::MAX_JSON_VALUES from the R4.2e measurements (CAP envelope 39,652 values, 612-leaf export 23,803; ~100 B of tree per value, a 16 MiB dense document 784 MiB with the cap off, 16 MiB with it); draft.restore refuses an over-budget copy as invalid; --max-memory 1 GiB proven to trap on the proofs build; cargo pdf_budget prints the value table, the Node budget suite prints the per-class cost (fresh instance, W4 asserted).
This commit is contained in:
1 parent
f8579bc19a
commit
6ee04c96c5
13 files changed
+599
-14
No files matched your search
+6
-1
@@ -3,7 +3,12 @@
|
|||||||
[target.wasm32-unknown-unknown]
|
[target.wasm32-unknown-unknown]
|
||||||
# The shadow stack of the browser module, set explicitly (the toolchain default is also
|
# The shadow stack of the browser module, set explicitly (the toolchain default is also
|
||||||
# 1 MiB). packages/core-wasm/tests/panic.test.ts proves that exhausting it traps.
|
# 1 MiB). packages/core-wasm/tests/panic.test.ts proves that exhausting it traps.
|
||||||
rustflags = ["-C", "link-arg=-zstack-size=1048576"]
|
# Linear memory ceiling (R4, OD-3 (a)): a runaway allocation traps at 1 GiB (the instance is
|
||||||
|
# poisoned and re-created by CoreRuntime) instead of growing until the OS kills the tab. Far
|
||||||
|
# above the needs measured in RUST.md §10 (import worst case under the value cap, the export
|
||||||
|
# of a report at the collection caps); packages/core-wasm/tests/pdf-import.budget.test.ts
|
||||||
|
# proves the trap on the proofs build.
|
||||||
|
rustflags = ["-C", "link-arg=-zstack-size=1048576", "-C", "link-arg=--max-memory=1073741824"]
|
||||||
|
|
||||||
[env]
|
[env]
|
||||||
# ts-rs writes the TypeScript types of the contracts here on `cargo test -p faunosteo-core --features ts`.
|
# ts-rs writes the TypeScript types of the contracts here on `cargo test -p faunosteo-core --features ts`.
|
||||||
|
|||||||
@@ -47,5 +47,9 @@ required-features = ["pdf-read"]
|
|||||||
name = "pdf_import"
|
name = "pdf_import"
|
||||||
required-features = ["pdf-read"]
|
required-features = ["pdf-read"]
|
||||||
|
|
||||||
|
[[test]]
|
||||||
|
name = "pdf_budget"
|
||||||
|
required-features = ["pdf-read"]
|
||||||
|
|
||||||
[lints]
|
[lints]
|
||||||
workspace = true
|
workspace = true
|
||||||
@@ -10,7 +10,6 @@ pub mod domain;
|
|||||||
pub mod facade;
|
pub mod facade;
|
||||||
pub mod json;
|
pub mod json;
|
||||||
pub mod ops;
|
pub mod ops;
|
||||||
#[cfg(feature = "pdf-read")]
|
|
||||||
pub mod pdf;
|
pub mod pdf;
|
||||||
|
|
||||||
/// Version of the core, reported by the wasm facade and the worker.
|
/// Version of the core, reported by the wasm facade and the worker.
|
||||||
|
|||||||
@@ -18,6 +18,7 @@ use crate::domain::report::{LateralSide, Observation, Report, SheetRef, Vertebra
|
|||||||
use crate::domain::session::{self, ImportFailure, RestoreFailure};
|
use crate::domain::session::{self, ImportFailure, RestoreFailure};
|
||||||
use crate::domain::sheet::{LoadedSheet, SheetIndexData, check_sheet_document};
|
use crate::domain::sheet::{LoadedSheet, SheetIndexData, check_sheet_document};
|
||||||
use crate::facade::{Failure, parse_input, to_value};
|
use crate::facade::{Failure, parse_input, to_value};
|
||||||
|
use crate::pdf::json_js::{MAX_JSON_VALUES, prescan};
|
||||||
|
|
||||||
type Outcome = Result<Value, Failure>;
|
type Outcome = Result<Value, Failure>;
|
||||||
|
|
||||||
@@ -314,6 +315,11 @@ fn versioned(kind: &'static str, found: Option<f64>) -> Failure {
|
|||||||
/// draft, and leaves it where it is.
|
/// draft, and leaves it where it is.
|
||||||
fn draft_restore(sheet: &LoadedSheet, input: &str) -> Outcome {
|
fn draft_restore(sheet: &LoadedSheet, input: &str) -> Outcome {
|
||||||
let input: DraftRestoreInput = parse_input(input)?;
|
let input: DraftRestoreInput = parse_input(input)?;
|
||||||
|
// OD-3: a token-dense text is refused before the parser allocates its tree; the caller
|
||||||
|
// sets the text aside verbatim, as for any unreadable draft (`invalid`).
|
||||||
|
if prescan(input.draft.as_bytes()).values > MAX_JSON_VALUES {
|
||||||
|
return Err(Failure::new("invalid", "json value budget exceeded"));
|
||||||
|
}
|
||||||
session::restore_draft(sheet, &input.draft, input.archive.as_deref())
|
session::restore_draft(sheet, &input.draft, input.archive.as_deref())
|
||||||
.map_err(|failure| match failure {
|
.map_err(|failure| match failure {
|
||||||
RestoreFailure::UnsupportedVersion(found) => versioned("unsupported_version", found),
|
RestoreFailure::UnsupportedVersion(found) => versioned("unsupported_version", found),
|
||||||
|
|||||||
@@ -26,6 +26,15 @@ use crate::domain::js::{find_lone_surrogate, parse_json};
|
|||||||
/// The report and its envelope nest under 16 levels; deeper is not ours (`MAX_JSON_DEPTH`).
|
/// The report and its envelope nest under 16 levels; deeper is not ours (`MAX_JSON_DEPTH`).
|
||||||
pub const MAX_JSON_DEPTH: usize = 64;
|
pub const MAX_JSON_DEPTH: usize = 64;
|
||||||
|
|
||||||
|
/// Values ([`prescan`]) a document may hold before the probe and the rescue copy refuse it
|
||||||
|
/// (OD-3). Measured at R4.2e in the shipped module: the parser's tree costs about 100 bytes
|
||||||
|
/// of linear memory per value, a 16 MiB token-dense document holds 8.4 million (784 MiB,
|
||||||
|
/// what BLK-012 observed); the largest legitimate archive, an envelope at the collection
|
||||||
|
/// caps (500 observations, 1,000 views, logos) holds 39,652, the 612-leaf export 23,803.
|
||||||
|
/// Ten times the largest legitimate archive, rounded: the worst case left is about 40 MiB
|
||||||
|
/// of tree (W4 ≤ 256 MiB). Set here, confirmed by the owner before R4.3 wires the import.
|
||||||
|
pub const MAX_JSON_VALUES: u64 = 400_000;
|
||||||
|
|
||||||
const BOM: &[u8] = b"\xEF\xBB\xBF";
|
const BOM: &[u8] = b"\xEF\xBB\xBF";
|
||||||
|
|
||||||
/// The signed form of the cap: the depth may go negative, as in the TypeScript.
|
/// The signed form of the cap: the depth may go negative, as in the TypeScript.
|
||||||
|
|||||||
@@ -6,6 +6,8 @@
|
|||||||
//! pinned by the frozen oracles O7 (`tests/oracles/r4/read.json`), O8 (`tolerance.json`) and
|
//! pinned by the frozen oracles O7 (`tests/oracles/r4/read.json`), O8 (`tolerance.json`) and
|
||||||
//! O9 (`inflate.json`). The writer (`write`) and the import entry (`import`) follow.
|
//! O9 (`inflate.json`). The writer (`write`) and the import entry (`import`) follow.
|
||||||
|
|
||||||
|
#[cfg(feature = "pdf-read")]
|
||||||
pub mod import;
|
pub mod import;
|
||||||
pub mod json_js;
|
pub mod json_js;
|
||||||
|
#[cfg(feature = "pdf-read")]
|
||||||
pub mod read;
|
pub mod read;
|
||||||
@@ -12,7 +12,7 @@
|
|||||||
//!
|
//!
|
||||||
//! OD-3 adds one cap the TypeScript did not have: a candidate whose value count (the
|
//! OD-3 adds one cap the TypeScript did not have: a candidate whose value count (the
|
||||||
//! parser's allocation proxy) exceeds the limit is skipped like one nested too deep, before
|
//! parser's allocation proxy) exceeds the limit is skipped like one nested too deep, before
|
||||||
//! any parser allocates its tree. Its threshold is measured at R4.2e.
|
//! any parser allocates its tree (`json_js::MAX_JSON_VALUES`, measured at R4.2e).
|
||||||
|
|
||||||
use std::borrow::Cow;
|
use std::borrow::Cow;
|
||||||
|
|
||||||
@@ -24,8 +24,8 @@ use super::{
|
|||||||
has_pdf_header,
|
has_pdf_header,
|
||||||
};
|
};
|
||||||
use crate::pdf::json_js::{
|
use crate::pdf::json_js::{
|
||||||
JsonPrescan, decode_lossy, is_versioned_object, prescan, rewrite_overflow_numbers,
|
JsonPrescan, MAX_JSON_VALUES, decode_lossy, is_versioned_object, prescan,
|
||||||
starts_json_object,
|
rewrite_overflow_numbers, starts_json_object,
|
||||||
};
|
};
|
||||||
|
|
||||||
/// The caps the probe applies on top of the scanner's own.
|
/// The caps the probe applies on top of the scanner's own.
|
||||||
@@ -35,12 +35,6 @@ pub struct ProbeLimits {
|
|||||||
pub max_json_values: u64,
|
pub max_json_values: u64,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Values a candidate may hold before the probe skips it (OD-3). A legitimate archive
|
|
||||||
/// holds a few thousand values; the threshold is set from the R4.2e measurements (10 × the
|
|
||||||
/// largest legitimate archive) and confirmed by the owner before the import is wired.
|
|
||||||
/// Until then the cap is off, so the parity suites measure the TypeScript behaviour.
|
|
||||||
pub const MAX_JSON_VALUES: u64 = u64::MAX;
|
|
||||||
|
|
||||||
impl ProbeLimits {
|
impl ProbeLimits {
|
||||||
/// No value cap: the TypeScript behaviour, for the parity suites.
|
/// No value cap: the TypeScript behaviour, for the parity suites.
|
||||||
pub const UNBOUNDED: Self = Self {
|
pub const UNBOUNDED: Self = Self {
|
||||||
|
|||||||
@@ -0,0 +1,127 @@
|
|||||||
|
//! R4.2e, OD-3: the value count (`json_js::prescan`, the parser's allocation proxy) of the
|
||||||
|
//! legitimate archives against the hostile references, printed for RUST.md §10.3 and pinned
|
||||||
|
//! loosely so a drift of an order of magnitude is seen.
|
||||||
|
//!
|
||||||
|
//! Legitimate: the R3 export of the demo with logos (its attachment located by the reader),
|
||||||
|
//! the v2 full and 612-leaves exports, and envelopes built like the export does for the
|
||||||
|
//! synthetic L (60 observations) and CAP (500, the collection cap) reports: the demo's
|
||||||
|
//! branding with logos and labels, a snapshot of the draft sheet for that report, the
|
||||||
|
//! `JSON.stringify(_, null, 2)` form. Hostile: the token-dense `values-1m.pdf` of O8.
|
||||||
|
#![allow(clippy::disallowed_methods, clippy::unwrap_used, clippy::expect_used)]
|
||||||
|
|
||||||
|
mod common;
|
||||||
|
|
||||||
|
use common::r4::{r4_input, repo_root};
|
||||||
|
use faunosteo_core::domain::archive::{ReportArchiveEnvelope, build_sheet_snapshot};
|
||||||
|
use faunosteo_core::domain::js::parse_json;
|
||||||
|
use faunosteo_core::domain::report::Report;
|
||||||
|
use faunosteo_core::domain::sheet::{BundledSheetData, LoadedSheet};
|
||||||
|
use faunosteo_core::json::to_js_pretty;
|
||||||
|
use faunosteo_core::pdf::import::import_pdf;
|
||||||
|
use faunosteo_core::pdf::json_js::{JsonPrescan, prescan};
|
||||||
|
use faunosteo_core::pdf::read::probe::{ProbeLimits, read_attached_document};
|
||||||
|
use serde_json::{Value, json};
|
||||||
|
|
||||||
|
const SEED: &str = "supabase/seed/anatomical-sheets/equine/draft";
|
||||||
|
const FIXTURES: &str = "apps/app/tests/fixtures";
|
||||||
|
|
||||||
|
fn draft_sheet() -> LoadedSheet {
|
||||||
|
let seed = repo_root().join(SEED);
|
||||||
|
let read = |name: &str| std::fs::read_to_string(seed.join(name)).unwrap();
|
||||||
|
let manifest: Value = serde_json::from_str(&read("assets.json")).unwrap();
|
||||||
|
let mut assets = serde_json::Map::new();
|
||||||
|
let mut hashes = serde_json::Map::new();
|
||||||
|
for asset in manifest.as_array().unwrap() {
|
||||||
|
let id = asset["svg_asset_id"].as_str().unwrap().to_owned();
|
||||||
|
let file = format!("assets/{}", asset["file"].as_str().unwrap());
|
||||||
|
assets.insert(id.clone(), Value::String(read(&file)));
|
||||||
|
hashes.insert(id, asset["sha256"].clone());
|
||||||
|
}
|
||||||
|
let payload = json!({
|
||||||
|
"status": "draft",
|
||||||
|
"ref": parse_json(&read("sheet-ref.json")).unwrap(),
|
||||||
|
"document": parse_json(&read("document.json")).unwrap(),
|
||||||
|
"migrationMap": parse_json(&read("report-migration-map.json")).unwrap(),
|
||||||
|
"assets": assets,
|
||||||
|
"assetHashes": hashes,
|
||||||
|
});
|
||||||
|
let data: BundledSheetData = serde_json::from_value(payload).unwrap();
|
||||||
|
LoadedSheet::from_bundle(data).unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn fixture(path: &str) -> Vec<u8> {
|
||||||
|
std::fs::read(repo_root().join(FIXTURES).join(path)).unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The attachment text of a PDF, as the reader hands it to the session.
|
||||||
|
fn attachment_text(pdf: &[u8]) -> String {
|
||||||
|
read_attached_document(pdf, ProbeLimits::UNBOUNDED)
|
||||||
|
.unwrap()
|
||||||
|
.text
|
||||||
|
.into_owned()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// An envelope like the export writes for `report`: the demo's branding (logos) and labels,
|
||||||
|
/// a snapshot of the draft for that report.
|
||||||
|
fn envelope_for(sheet: &LoadedSheet, demo: &ReportArchiveEnvelope, report: Report) -> String {
|
||||||
|
let envelope = ReportArchiveEnvelope {
|
||||||
|
sheet_snapshot: build_sheet_snapshot(sheet, &report),
|
||||||
|
report,
|
||||||
|
..demo.clone()
|
||||||
|
};
|
||||||
|
to_js_pretty(&envelope).unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn report(path: &str) -> Report {
|
||||||
|
let text = String::from_utf8(fixture(path)).unwrap();
|
||||||
|
serde_json::from_value(parse_json(&text).unwrap()).unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn row(name: &str, text: &str) -> (String, usize, JsonPrescan) {
|
||||||
|
(name.to_owned(), text.len(), prescan(text.as_bytes()))
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn legitimate_archives_stay_far_below_the_hostile_references() {
|
||||||
|
let sheet = draft_sheet();
|
||||||
|
let demo_pdf = fixture("pdf/r3/demo.fr.logos.pdf");
|
||||||
|
let demo = import_pdf(&sheet, &demo_pdf, ProbeLimits::UNBOUNDED)
|
||||||
|
.unwrap()
|
||||||
|
.read
|
||||||
|
.archive
|
||||||
|
.unwrap();
|
||||||
|
let rows = [
|
||||||
|
row("demo fr logos (R3 export)", &attachment_text(&demo_pdf)),
|
||||||
|
row(
|
||||||
|
"v2full fr logos (R3 export)",
|
||||||
|
&attachment_text(&fixture("pdf/r3/v2full.fr.logos.pdf")),
|
||||||
|
),
|
||||||
|
row(
|
||||||
|
"612 leaves (O7 read.612-leaves-exported)",
|
||||||
|
&attachment_text(&r4_input("read", "read.612-leaves-exported.pdf")),
|
||||||
|
),
|
||||||
|
row(
|
||||||
|
"L envelope (60 observations, 120 views, logos)",
|
||||||
|
&envelope_for(&sheet, &demo, report("r4/reports/L.json")),
|
||||||
|
),
|
||||||
|
row(
|
||||||
|
"CAP envelope (500 observations, 1,000 views, logos)",
|
||||||
|
&envelope_for(&sheet, &demo, report("r4/reports/CAP.json")),
|
||||||
|
),
|
||||||
|
row(
|
||||||
|
"hostile: tolerance/values-1m.pdf",
|
||||||
|
&attachment_text(&fixture("pdf/r4/tolerance/values-1m.pdf")),
|
||||||
|
),
|
||||||
|
];
|
||||||
|
println!("| Document | Bytes | Values | Within depth |");
|
||||||
|
println!("|---|---|---|---|");
|
||||||
|
for (name, bytes, scan) in &rows {
|
||||||
|
println!(
|
||||||
|
"| {name} | {bytes} | {} | {} |",
|
||||||
|
scan.values, scan.within_depth
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let cap = rows[4].2.values;
|
||||||
|
assert!(cap < 200_000, "CAP envelope values: {cap}");
|
||||||
|
assert!(rows[5].2.values > 5 * cap, "values-1m is not far above CAP");
|
||||||
|
}
|
||||||
@@ -268,6 +268,16 @@ fn o8_import_matches_the_r3_path_outside_the_declared_differences() {
|
|||||||
assert_eq!(declared, 3);
|
assert_eq!(declared, 3);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// O8 cases the application's caps refuse (OD-3, `MAX_JSON_VALUES`): a token-dense
|
||||||
|
/// document is skipped like one nested too deep, so the kind follows the stamp, as the
|
||||||
|
/// depth cap's does today (`depth-65` is `attachment_stripped` in O8).
|
||||||
|
fn declared_od3(id: &str) -> Option<Value> {
|
||||||
|
match id {
|
||||||
|
"values-1m" => Some(json!({ "ok": false, "kind": "attachment_stripped" })),
|
||||||
|
_ => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn the_facade_envelope_carries_the_closed_kinds() {
|
fn the_facade_envelope_carries_the_closed_kinds() {
|
||||||
let sheet = draft_sheet();
|
let sheet = draft_sheet();
|
||||||
@@ -276,7 +286,9 @@ fn the_facade_envelope_carries_the_closed_kinds() {
|
|||||||
let id = case["id"].as_str().unwrap();
|
let id = case["id"].as_str().unwrap();
|
||||||
let pdf = std::fs::read(repo_root().join(case["pdf"].as_str().unwrap())).unwrap();
|
let pdf = std::fs::read(repo_root().join(case["pdf"].as_str().unwrap())).unwrap();
|
||||||
let envelope: Value = serde_json::from_str(&facade::import_pdf(&sheet, &pdf)).unwrap();
|
let envelope: Value = serde_json::from_str(&facade::import_pdf(&sheet, &pdf)).unwrap();
|
||||||
let expected = declared_o8(id).unwrap_or_else(|| case["outcome"].clone());
|
let expected = declared_od3(id)
|
||||||
|
.or_else(|| declared_o8(id))
|
||||||
|
.unwrap_or_else(|| case["outcome"].clone());
|
||||||
assert_eq!(envelope["ok"], expected["ok"], "{id}");
|
assert_eq!(envelope["ok"], expected["ok"], "{id}");
|
||||||
if expected["ok"] == false {
|
if expected["ok"] == false {
|
||||||
assert_eq!(envelope["kind"], expected["kind"], "{id}");
|
assert_eq!(envelope["kind"], expected["kind"], "{id}");
|
||||||
|
|||||||
@@ -50,6 +50,35 @@ impl SheetHandle {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// R4.2e proof: the import with the OD-3 value cap off, so that a token-dense document runs
|
||||||
|
/// into the linear memory ceiling and traps (never in the module the application ships).
|
||||||
|
#[cfg(feature = "proofs")]
|
||||||
|
#[wasm_bindgen]
|
||||||
|
pub fn import_pdf_unbounded_for_test(bundle: &str, bytes: &[u8]) -> Result<String, JsError> {
|
||||||
|
use faunosteo_core::pdf::import::import_pdf;
|
||||||
|
use faunosteo_core::pdf::read::probe::ProbeLimits;
|
||||||
|
let sheet = faunosteo_core::facade::load_sheet(bundle)
|
||||||
|
.map_err(|failure| JsError::new(&format!("{}: {}", failure.kind, failure.detail)))?;
|
||||||
|
Ok(match import_pdf(&sheet, bytes, ProbeLimits::UNBOUNDED) {
|
||||||
|
Ok(_) => "ok".to_owned(),
|
||||||
|
Err(failure) => format!("{failure:?}"),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// R4.2e proof: the linear memory ceiling (`--max-memory`, `.cargo/config.toml`). Allocates
|
||||||
|
/// and touches `mib` MiB; past the ceiling the allocation fails and Rust aborts, a trap.
|
||||||
|
#[cfg(feature = "proofs")]
|
||||||
|
#[wasm_bindgen]
|
||||||
|
pub fn allocate_for_test(mib: u32) -> u32 {
|
||||||
|
let bytes = vec![1u8; (mib as usize) * 1024 * 1024];
|
||||||
|
std::hint::black_box(&bytes);
|
||||||
|
bytes
|
||||||
|
.iter()
|
||||||
|
.step_by(1024 * 1024)
|
||||||
|
.map(|b| u32::from(*b))
|
||||||
|
.sum()
|
||||||
|
}
|
||||||
|
|
||||||
/// R0 proof: on wasm32 a panic is a trap (panic = abort), reaching JavaScript as a
|
/// R0 proof: on wasm32 a panic is a trap (panic = abort), reaching JavaScript as a
|
||||||
/// `WebAssembly.RuntimeError`; the caller re-instantiates the module.
|
/// `WebAssembly.RuntimeError`; the caller re-instantiates the module.
|
||||||
#[cfg(feature = "proofs")]
|
#[cfg(feature = "proofs")]
|
||||||
|
|||||||
@@ -52,6 +52,10 @@ export async function nodeCoreRuntime(
|
|||||||
export interface ProofModule extends GlueModule {
|
export interface ProofModule extends GlueModule {
|
||||||
panic_for_test(): void
|
panic_for_test(): void
|
||||||
recurse_for_test(depth: number): number
|
recurse_for_test(depth: number): number
|
||||||
|
/** R4.2e: the PDF import with the OD-3 value cap off (the linear ceiling proof). */
|
||||||
|
import_pdf_unbounded_for_test(bundle: string, bytes: Uint8Array): string
|
||||||
|
/** R4.2e: allocates `mib` MiB, the linear ceiling proof. */
|
||||||
|
allocate_for_test(mib: number): number
|
||||||
}
|
}
|
||||||
|
|
||||||
/** A fresh, initialised glue of the proofs build, outside any runtime (R0 proofs). */
|
/** A fresh, initialised glue of the proofs build, outside any runtime (R0 proofs). */
|
||||||
|
|||||||
@@ -0,0 +1,389 @@
|
|||||||
|
/**
|
||||||
|
* R4.2e, OD-3 (plan §B.3, §B.8 W4): what the PDF import costs in the shipped module, one
|
||||||
|
* fresh instance per case (the worker of R4.3 is disposable too), on the legitimate archives
|
||||||
|
* and on the hostile classes of §B.3: a 16 MiB token-dense attachment, the same through
|
||||||
|
* the rescue copy (`draft.restore`), the 50 MiB file, the 16 MiB inflation, 32 oversized
|
||||||
|
* candidates, a 16 MiB document at the depth cap. Printed for RUST.md §10.3: outcome, wall
|
||||||
|
* time, linear memory growth of the instance (never shrinks, so "after" is the high-water
|
||||||
|
* mark) and the Node process RSS growth (the glue's copies and the result string).
|
||||||
|
*
|
||||||
|
* Loose assertions only: no case traps or exceeds the W4 linear budget once the OD-3 cap
|
||||||
|
* applies; the printed numbers are the record. `R4_BUDGET_ALL=1` also runs the slowest
|
||||||
|
* classes (the 50 MiB scan, the 32 inflations).
|
||||||
|
*/
|
||||||
|
import { readFileSync } from 'node:fs'
|
||||||
|
import { join } from 'node:path'
|
||||||
|
import { fileURLToPath } from 'node:url'
|
||||||
|
import { deflateSync } from 'node:zlib'
|
||||||
|
|
||||||
|
import {
|
||||||
|
CoreRuntime,
|
||||||
|
type GlueModule,
|
||||||
|
type GlueSheetHandle,
|
||||||
|
} from '../src/index'
|
||||||
|
import { loadProofModule, nodeCoreSource } from '../src/node'
|
||||||
|
|
||||||
|
const ROOT = fileURLToPath(new URL('../../../', import.meta.url))
|
||||||
|
const SEED = join(ROOT, 'supabase/seed/anatomical-sheets/equine/draft')
|
||||||
|
const FIXTURES = join(ROOT, 'apps/app/tests/fixtures')
|
||||||
|
const MiB = 1024 * 1024
|
||||||
|
const ALL = process.env.R4_BUDGET_ALL === '1'
|
||||||
|
const read = (path: string) => readFileSync(path, 'utf8')
|
||||||
|
|
||||||
|
function draftBundle(): string {
|
||||||
|
const manifest = JSON.parse(read(join(SEED, 'assets.json'))) as Array<{
|
||||||
|
file: string
|
||||||
|
sha256: string
|
||||||
|
svg_asset_id: string
|
||||||
|
}>
|
||||||
|
const assets: Record<string, string> = {}
|
||||||
|
const assetHashes: Record<string, string> = {}
|
||||||
|
for (const asset of manifest) {
|
||||||
|
assets[asset.svg_asset_id] = read(join(SEED, 'assets', asset.file))
|
||||||
|
assetHashes[asset.svg_asset_id] = asset.sha256
|
||||||
|
}
|
||||||
|
return JSON.stringify({
|
||||||
|
status: 'draft',
|
||||||
|
ref: JSON.parse(read(join(SEED, 'sheet-ref.json'))),
|
||||||
|
document: JSON.parse(read(join(SEED, 'document.json'))),
|
||||||
|
migrationMap: JSON.parse(read(join(SEED, 'report-migration-map.json'))),
|
||||||
|
assets,
|
||||||
|
assetHashes,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
const BUNDLE = draftBundle()
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Minimal PDFs the scanner reads (plan §B.1.2): every object in clear.
|
||||||
|
|
||||||
|
interface Stream {
|
||||||
|
bytes: Uint8Array
|
||||||
|
flate?: boolean
|
||||||
|
name?: string
|
||||||
|
}
|
||||||
|
|
||||||
|
const ascii = (text: string) => new TextEncoder().encode(text)
|
||||||
|
|
||||||
|
function concat(parts: Uint8Array[]): Uint8Array {
|
||||||
|
const out = new Uint8Array(parts.reduce((n, p) => n + p.byteLength, 0))
|
||||||
|
let at = 0
|
||||||
|
for (const part of parts) {
|
||||||
|
out.set(part, at)
|
||||||
|
at += part.byteLength
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
/** A PDF holding `streams` as EmbeddedFile objects, Filespecs for the named ones, the stamp. */
|
||||||
|
function pdfWith(streams: Stream[], padding = 0): Uint8Array {
|
||||||
|
const parts: Uint8Array[] = [ascii('%PDF-1.7\n%\xE2\xE3\xCF\xD3\n')]
|
||||||
|
if (padding > 0) parts.push(ascii(`%${'x'.repeat(padding - 2)}\n`))
|
||||||
|
let n = 1
|
||||||
|
for (const stream of streams) {
|
||||||
|
const filter = stream.flate ? ' /Filter /FlateDecode' : ''
|
||||||
|
parts.push(
|
||||||
|
ascii(
|
||||||
|
`${n} 0 obj\n<< /Type /EmbeddedFile /Length ${stream.bytes.byteLength}${filter} >>\nstream\n`,
|
||||||
|
),
|
||||||
|
stream.bytes,
|
||||||
|
ascii('\nendstream\nendobj\n'),
|
||||||
|
)
|
||||||
|
if (stream.name) {
|
||||||
|
parts.push(
|
||||||
|
ascii(
|
||||||
|
`${n + 1} 0 obj\n<< /Type /Filespec /F (${stream.name}) /EF << /F ${n} 0 R >> >>\nendobj\n`,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
n += 2
|
||||||
|
}
|
||||||
|
parts.push(
|
||||||
|
ascii(`${n} 0 obj\n<< /Keywords (faunosteo-archive/1) >>\nendobj\n%%EOF\n`),
|
||||||
|
)
|
||||||
|
return concat(parts)
|
||||||
|
}
|
||||||
|
|
||||||
|
const zeros = (bytes: number) => new Uint8Array(bytes)
|
||||||
|
const deflate = (bytes: Uint8Array) => new Uint8Array(deflateSync(bytes))
|
||||||
|
|
||||||
|
/** `{"version":3,"a":[1,1,…]}` of about `bytes` bytes: one value per two bytes. */
|
||||||
|
function denseNumbers(bytes: number): Uint8Array {
|
||||||
|
const head = '{"version":3,"a":['
|
||||||
|
const count = Math.floor((bytes - head.length - 2) / 2)
|
||||||
|
return ascii(`${head}${'1,'.repeat(count - 1)}1]}`)
|
||||||
|
}
|
||||||
|
|
||||||
|
/** `{"version":3,"a":["","",…]}`: one allocated string per three bytes. */
|
||||||
|
function denseStrings(bytes: number): Uint8Array {
|
||||||
|
const head = '{"version":3,"a":['
|
||||||
|
const count = Math.floor((bytes - head.length - 2) / 3)
|
||||||
|
return ascii(`${head}${'"",'.repeat(count - 1)}""]}`)
|
||||||
|
}
|
||||||
|
|
||||||
|
/** `{"version":3,"a":[[[…[1,1,…]…]]]}` nested to the depth cap, about `bytes` bytes. */
|
||||||
|
function deepArrays(bytes: number, depth: number): Uint8Array {
|
||||||
|
const head = `{"version":3,"a":${'['.repeat(depth - 1)}`
|
||||||
|
const tail = `${']'.repeat(depth - 1)}}`
|
||||||
|
const count = Math.floor((bytes - head.length - tail.length) / 2)
|
||||||
|
return ascii(`${head}${'1,'.repeat(count - 1)}1${tail}`)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// One fresh instance per case
|
||||||
|
|
||||||
|
interface Instance {
|
||||||
|
runtime: CoreRuntime
|
||||||
|
sheet: GlueSheetHandle
|
||||||
|
memory: () => number
|
||||||
|
}
|
||||||
|
|
||||||
|
async function freshInstance(): Promise<Instance> {
|
||||||
|
let memory: WebAssembly.Memory | undefined
|
||||||
|
let glue: GlueModule | undefined
|
||||||
|
const source = nodeCoreSource('pkg')
|
||||||
|
const runtime = new CoreRuntime({
|
||||||
|
compile: source.compile,
|
||||||
|
importGlue: async (n) => {
|
||||||
|
glue = await source.importGlue(n)
|
||||||
|
return {
|
||||||
|
...glue,
|
||||||
|
default: async (input: { module_or_path: WebAssembly.Module }) => {
|
||||||
|
const exports = (await glue!.default(input)) as {
|
||||||
|
memory?: WebAssembly.Memory
|
||||||
|
}
|
||||||
|
memory = exports.memory
|
||||||
|
return exports
|
||||||
|
},
|
||||||
|
} as GlueModule
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if (!(await runtime.ready())) throw new Error('module failed to load')
|
||||||
|
const sheet = new glue!.SheetHandle(BUNDLE)
|
||||||
|
return { runtime, sheet, memory: () => memory?.buffer.byteLength ?? 0 }
|
||||||
|
}
|
||||||
|
|
||||||
|
interface Measure {
|
||||||
|
outcome: string
|
||||||
|
ms: number
|
||||||
|
linearMiB: number
|
||||||
|
rssMiB: number
|
||||||
|
}
|
||||||
|
|
||||||
|
function measure(instance: Instance, run: () => string): Measure {
|
||||||
|
const rss0 = process.memoryUsage().rss
|
||||||
|
const linear0 = instance.memory()
|
||||||
|
const started = performance.now()
|
||||||
|
let outcome: string
|
||||||
|
try {
|
||||||
|
const envelope = JSON.parse(run()) as { ok: boolean; kind?: string }
|
||||||
|
outcome = envelope.ok ? 'ok' : envelope.kind!
|
||||||
|
} catch (error) {
|
||||||
|
outcome = `trap: ${(error as Error).constructor.name}`
|
||||||
|
}
|
||||||
|
const ms = performance.now() - started
|
||||||
|
return {
|
||||||
|
outcome,
|
||||||
|
ms,
|
||||||
|
linearMiB: (instance.memory() - linear0) / MiB,
|
||||||
|
rssMiB: (process.memoryUsage().rss - rss0) / MiB,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
interface Case {
|
||||||
|
name: string
|
||||||
|
slow?: boolean
|
||||||
|
build: () => Uint8Array
|
||||||
|
/** `pdf` through import_pdf (default), `copy` through draft.restore as a rescue copy. */
|
||||||
|
path?: 'pdf' | 'copy'
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The demo's envelope with `report` and a fresh snapshot: what the export writes for it. */
|
||||||
|
function envelopePdf(instance: Instance, reportFile: string): Uint8Array {
|
||||||
|
const demo = JSON.parse(
|
||||||
|
instance.sheet.import_pdf(
|
||||||
|
new Uint8Array(readFileSync(join(FIXTURES, 'pdf/r3/demo.fr.logos.pdf'))),
|
||||||
|
),
|
||||||
|
) as { value: { read: { archive: Record<string, unknown> } } }
|
||||||
|
const report = JSON.parse(read(join(FIXTURES, reportFile)))
|
||||||
|
const snapshot = JSON.parse(
|
||||||
|
instance.sheet.call('archive.snapshot', JSON.stringify({ report })),
|
||||||
|
) as { value: unknown }
|
||||||
|
const envelope = {
|
||||||
|
...demo.value.read.archive,
|
||||||
|
report,
|
||||||
|
sheetSnapshot: snapshot.value,
|
||||||
|
}
|
||||||
|
return pdfWith([
|
||||||
|
{
|
||||||
|
bytes: ascii(JSON.stringify(envelope, null, 2)),
|
||||||
|
name: 'osteo-report.json',
|
||||||
|
},
|
||||||
|
])
|
||||||
|
}
|
||||||
|
|
||||||
|
const CASES: Case[] = [
|
||||||
|
{
|
||||||
|
name: 'demo fr logos (R3 export, 237 KB)',
|
||||||
|
build: () =>
|
||||||
|
new Uint8Array(readFileSync(join(FIXTURES, 'pdf/r3/demo.fr.logos.pdf'))),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: '612 leaves (O7 export)',
|
||||||
|
build: () =>
|
||||||
|
new Uint8Array(
|
||||||
|
readFileSync(
|
||||||
|
join(FIXTURES, 'pdf/r4/read/read.612-leaves-exported.pdf'),
|
||||||
|
),
|
||||||
|
),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: 'values-1m (O8 hostile, 2 MB attachment)',
|
||||||
|
build: () =>
|
||||||
|
new Uint8Array(
|
||||||
|
readFileSync(join(FIXTURES, 'pdf/r4/tolerance/values-1m.pdf')),
|
||||||
|
),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: '16 MiB dense numbers (8.4 M values)',
|
||||||
|
build: () =>
|
||||||
|
pdfWith([
|
||||||
|
{ bytes: denseNumbers(16 * MiB - 64), name: 'osteo-report.json' },
|
||||||
|
]),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: '16 MiB dense strings (5.6 M values)',
|
||||||
|
build: () =>
|
||||||
|
pdfWith([
|
||||||
|
{ bytes: denseStrings(16 * MiB - 64), name: 'osteo-report.json' },
|
||||||
|
]),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: '16 MiB at depth 64',
|
||||||
|
build: () =>
|
||||||
|
pdfWith([
|
||||||
|
{ bytes: deepArrays(16 * MiB - 64, 64), name: 'osteo-report.json' },
|
||||||
|
]),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: '16 MiB dense numbers as a rescue copy',
|
||||||
|
path: 'copy',
|
||||||
|
build: () => denseNumbers(16 * MiB - 64),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: '16 MiB inflation (zeros, not JSON)',
|
||||||
|
build: () =>
|
||||||
|
pdfWith([
|
||||||
|
{
|
||||||
|
bytes: deflate(zeros(16 * MiB)),
|
||||||
|
flate: true,
|
||||||
|
name: 'osteo-report.json',
|
||||||
|
},
|
||||||
|
]),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: '16 MiB + 1 inflation (bomb, refused)',
|
||||||
|
build: () =>
|
||||||
|
pdfWith([
|
||||||
|
{
|
||||||
|
bytes: deflate(zeros(16 * MiB + 1)),
|
||||||
|
flate: true,
|
||||||
|
name: 'osteo-report.json',
|
||||||
|
},
|
||||||
|
]),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: '32 bombs (17 MiB each) before the attachment',
|
||||||
|
slow: true,
|
||||||
|
build: () =>
|
||||||
|
pdfWith([
|
||||||
|
...Array.from({ length: 32 }, () => ({
|
||||||
|
bytes: deflate(zeros(17 * MiB)),
|
||||||
|
flate: true,
|
||||||
|
})),
|
||||||
|
{ bytes: ascii('{"version":3}'), name: 'osteo-report.json' },
|
||||||
|
]),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: '50 MiB file, attachment at the end',
|
||||||
|
slow: true,
|
||||||
|
build: () =>
|
||||||
|
pdfWith(
|
||||||
|
[{ bytes: ascii('{"version":3}'), name: 'osteo-report.json' }],
|
||||||
|
50 * MiB - 400,
|
||||||
|
),
|
||||||
|
},
|
||||||
|
]
|
||||||
|
|
||||||
|
describe('R4 import budget (shipped module, fresh instance per case)', () => {
|
||||||
|
it('prints the cost of each class and stays within W4 where the cap applies', async () => {
|
||||||
|
const rows: string[] = [
|
||||||
|
'| Case | Outcome | ms | Linear MiB | Node RSS MiB |',
|
||||||
|
'|---|---|---|---|---|',
|
||||||
|
]
|
||||||
|
for (const file of ['r4/reports/L.json', 'r4/reports/CAP.json']) {
|
||||||
|
const instance = await freshInstance()
|
||||||
|
const pdf = envelopePdf(instance, file)
|
||||||
|
const fresh = await freshInstance()
|
||||||
|
const m = measure(fresh, () => fresh.sheet.import_pdf(pdf))
|
||||||
|
rows.push(
|
||||||
|
`| ${file} envelope (${(pdf.byteLength / 1024).toFixed(0)} KB) | ${m.outcome} | ${m.ms.toFixed(0)} | ${m.linearMiB.toFixed(1)} | ${m.rssMiB.toFixed(1)} |`,
|
||||||
|
)
|
||||||
|
expect(m.outcome, file).toBe('ok')
|
||||||
|
}
|
||||||
|
for (const c of CASES) {
|
||||||
|
if (c.slow && !ALL) continue
|
||||||
|
const bytes = c.build()
|
||||||
|
const instance = await freshInstance()
|
||||||
|
const m = measure(instance, () =>
|
||||||
|
c.path === 'copy'
|
||||||
|
? instance.sheet.call(
|
||||||
|
'draft.restore',
|
||||||
|
JSON.stringify({
|
||||||
|
draft: new TextDecoder().decode(bytes),
|
||||||
|
archive: null,
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
: instance.sheet.import_pdf(bytes),
|
||||||
|
)
|
||||||
|
rows.push(
|
||||||
|
`| ${c.name} | ${m.outcome} | ${m.ms.toFixed(0)} | ${m.linearMiB.toFixed(1)} | ${m.rssMiB.toFixed(1)} |`,
|
||||||
|
)
|
||||||
|
// W4: no trap, linear growth under 256 MiB on every hostile class.
|
||||||
|
expect(m.outcome, c.name).not.toMatch(/^trap/)
|
||||||
|
expect(m.linearMiB, c.name).toBeLessThan(256)
|
||||||
|
expect(instance.runtime.status, c.name).toBe('ready')
|
||||||
|
}
|
||||||
|
console.info(`[core-wasm] R4 import budget\n${rows.join('\n')}`)
|
||||||
|
}, 600_000)
|
||||||
|
|
||||||
|
it('with the cap off, a 16 MiB token-dense document costs most of the ceiling (proofs build)', async () => {
|
||||||
|
const proofs = await loadProofModule()
|
||||||
|
const bytes = pdfWith([
|
||||||
|
{ bytes: denseNumbers(16 * MiB - 64), name: 'osteo-report.json' },
|
||||||
|
])
|
||||||
|
const started = performance.now()
|
||||||
|
let outcome = 'returned'
|
||||||
|
try {
|
||||||
|
outcome = proofs.import_pdf_unbounded_for_test(BUNDLE, bytes)
|
||||||
|
} catch (error) {
|
||||||
|
outcome = `trap: ${(error as Error).constructor.name}`
|
||||||
|
}
|
||||||
|
console.info(
|
||||||
|
`[core-wasm] cap off, 16 MiB dense numbers: ${outcome} after ${(performance.now() - started).toFixed(0)} ms`,
|
||||||
|
)
|
||||||
|
// The tree fits under 1 GiB (784 MiB measured): the cap, not the ceiling, is the guard.
|
||||||
|
expect(outcome).toBe('Import(NoAttachment)')
|
||||||
|
}, 120_000)
|
||||||
|
|
||||||
|
it('traps at the 1 GiB linear ceiling instead of growing (proofs build)', async () => {
|
||||||
|
const proofs = await loadProofModule()
|
||||||
|
expect(proofs.allocate_for_test(256)).toBe(256)
|
||||||
|
let trapped = 'returned'
|
||||||
|
try {
|
||||||
|
proofs.allocate_for_test(1100)
|
||||||
|
} catch (error) {
|
||||||
|
trapped = (error as Error).constructor.name
|
||||||
|
}
|
||||||
|
expect(trapped).toBe('RuntimeError')
|
||||||
|
}, 120_000)
|
||||||
|
})
|
||||||
@@ -99,11 +99,16 @@ function o8Row(envelope: Envelope): unknown {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/** O8 cases whose Rust outcome differs on purpose (plan §A.3, OD-4). */
|
/**
|
||||||
|
* O8 cases whose Rust outcome differs on purpose (plan §A.3): OD-4, a lone surrogate escape
|
||||||
|
* in a value a duplicate key overwrote; OD-3, a token-dense document over `MAX_JSON_VALUES`
|
||||||
|
* (skipped like one nested too deep, so the kind follows the stamp).
|
||||||
|
*/
|
||||||
const DECLARED: Record<string, unknown> = {
|
const DECLARED: Record<string, unknown> = {
|
||||||
'surrogate-content-overwritten': { ok: false, kind: 'lone_surrogate' },
|
'surrogate-content-overwritten': { ok: false, kind: 'lone_surrogate' },
|
||||||
'surrogate-version-overwritten': { ok: false, kind: 'lone_surrogate' },
|
'surrogate-version-overwritten': { ok: false, kind: 'lone_surrogate' },
|
||||||
'two-files-decoy-first': { ok: false, kind: 'lone_surrogate' },
|
'two-files-decoy-first': { ok: false, kind: 'lone_surrogate' },
|
||||||
|
'values-1m': { ok: false, kind: 'attachment_stripped' },
|
||||||
}
|
}
|
||||||
|
|
||||||
function pdfsUnder(dir: string): string[] {
|
function pdfsUnder(dir: string): string[] {
|
||||||
@@ -150,7 +155,7 @@ describe('R4 PDF import through the shipped module (Node)', () => {
|
|||||||
if (c.id in DECLARED) declared += 1
|
if (c.id in DECLARED) declared += 1
|
||||||
expect(o8Row(envelope), c.id).toEqual(expected)
|
expect(o8Row(envelope), c.id).toEqual(expected)
|
||||||
}
|
}
|
||||||
expect(declared).toBe(3)
|
expect(declared).toBe(4)
|
||||||
expect(runtime.status).toBe('ready')
|
expect(runtime.status).toBe('ready')
|
||||||
})
|
})
|
||||||
|
|
||||||
|
|||||||
Reference in new issue
Block a user