Files
faunosteo/crates/core/src/pdf/json_js.rs
T
bchanot 9850a8341a feat(domain): R4.2c json_js: JS-path side effects of the attachment probe
starts_json_object, depth and value prescan (OD-3 proxy), TextDecoder-parity lossy decode
(400-case oracle generated by scripts/r4-utf8-oracle.mjs, 0 difference), overflow → null
rewrite with the JSON number grammar, lone-surrogate neutralised probe (OD-4).
2026-10-18 15:44:54 +02:00

285 lines
11 KiB
Rust

//! The JavaScript side effects of the attachment probe (R4.2c, plan §B.1.2, OD-3, OD-4).
//!
//! `packages/pdf/src/attachment/scan.ts` checked a candidate with `startsJsonObject` and
//! `withinJsonDepth`, then `parseVersioned` read it with `JSON.parse(new TextDecoder()
//! .decode(data))`, and the parsed value reached the core through `JSON.stringify`. That path
//! changed the text in ways serde_json does not (LRN-059), so each one is a step here, kept
//! apart so that the caller composes them and each can be measured:
//!
//! 1. [`starts_json_object`] and [`prescan`] on the raw bytes: the opening `{`, the depth cap
//! of `withinJsonDepth`, and a value count for the OD-3 budget, in the same linear pass.
//! 2. [`decode_lossy`]: invalid UTF-8 becomes U+FFFD and one leading BOM is dropped, as
//! TextDecoder does (measured identical, LRN-059, and pinned by the UTF-8 oracle).
//! 3. [`rewrite_overflow_numbers`]: a number that overflows f64 was ±Infinity for
//! `JSON.parse` and `null` once stringified, where serde_json refuses it; the rewritten
//! text is the one the core reads.
//! 4. [`is_versioned_object`]: `parseVersioned`'s acceptance, on a copy whose lone surrogate
//! escapes are neutralised ([`neutralise_lone_surrogates`]), because `JSON.parse` kept
//! them; the core then refuses the real text with `lone_surrogate` (OD-4, declared).
use std::borrow::Cow;
use serde_json::Value;
use crate::domain::js::{find_lone_surrogate, parse_json};
/// The report and its envelope nest under 16 levels; deeper is not ours (`MAX_JSON_DEPTH`).
pub const MAX_JSON_DEPTH: usize = 64;
const BOM: &[u8] = b"\xEF\xBB\xBF";
/// The signed form of the cap: the depth may go negative, as in the TypeScript.
const MAX_DEPTH: i64 = MAX_JSON_DEPTH as i64;
/// `{` first, past an optional UTF-8 BOM and JSON whitespace: the only shape a version
/// lives in (`startsJsonObject`).
pub fn starts_json_object(data: &[u8]) -> bool {
let body = data.strip_prefix(BOM).unwrap_or(data);
body.iter().find(|byte| !is_json_whitespace(**byte)) == Some(&b'{')
}
/// What [`prescan`] measured.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct JsonPrescan {
/// `withinJsonDepth`: the bracket depth never went over [`MAX_JSON_DEPTH`].
pub within_depth: bool,
/// Values counted before the scan ended (at the refusal when `within_depth` is false).
pub values: u64,
}
/// One linear pass over the raw bytes before any parser sees them: `withinJsonDepth` exactly
/// (string literals skipped, a backslash skipping the next byte; `{` and `[` deepen, `}` and
/// `]` rise, below zero too; the scan stops as soon as the depth exceeds the cap) and a
/// count of the values the parser would allocate, for the OD-3 budget.
///
/// The count is an allocation proxy, not a strict JSON value count: every `{`, every `[`,
/// every string literal (object keys included, the parser allocates them too), and every
/// number or `true` / `false` / `null` start, which is a `-`, a digit, `t`, `f` or `n`
/// outside a string at the start of the text or after a structural byte or whitespace.
/// Malformed text is counted the same way. The count depends on the bytes only, so a
/// threshold measured against it is stable. One leading BOM is skipped (it never matches a
/// JSON byte, so the depth outcome is unchanged).
pub fn prescan(data: &[u8]) -> JsonPrescan {
let body = data.strip_prefix(BOM).unwrap_or(data);
let mut lexer = Lexer::new();
let (mut depth, mut values) = (0_i64, 0_u64);
for &byte in body {
match lexer.next(byte) {
Lexeme::Open => {
depth += 1;
values += 1;
}
Lexeme::Close => depth -= 1,
Lexeme::StringStart | Lexeme::NumberStart | Lexeme::LiteralStart => values += 1,
Lexeme::InString | Lexeme::Other => {}
}
if depth > MAX_DEPTH {
return JsonPrescan {
within_depth: false,
values,
};
}
}
JsonPrescan {
within_depth: true,
values,
}
}
/// `new TextDecoder().decode(data)`: one leading BOM dropped, invalid UTF-8 replaced by
/// U+FFFD (maximal subparts, as the WHATWG decoder does).
pub fn decode_lossy(data: &[u8]) -> Cow<'_, str> {
String::from_utf8_lossy(data.strip_prefix(BOM).unwrap_or(data))
}
/// The text `JSON.stringify(JSON.parse(text))` would have handed on, as far as numbers go:
/// outside string literals, every number token whose value overflows f64 (`1e999`, `-1e999`,
/// a 400-digit integer) becomes `null`, everything else stays byte-identical. Underflow
/// reads as 0 on both sides and is kept; a token that is not a JSON number is kept and fails
/// the parse, as it failed `JSON.parse`. A token is the run of `[0-9+\-.eE]` that opens on a
/// `-` or a digit in value position: when `JSON.parse` succeeds, that run is its number.
pub fn rewrite_overflow_numbers(text: &str) -> Cow<'_, str> {
let bytes = text.as_bytes();
let mut lexer = Lexer::new();
let mut out = String::new();
let (mut copied, mut at) = (0, 0);
while at < bytes.len() {
if lexer.next(bytes[at]) != Lexeme::NumberStart {
at += 1;
continue;
}
let end = at + number_run_len(&bytes[at..]);
// The run is ASCII, so both of its ends are char boundaries.
if overflows(&text[at..end]) {
out.push_str(&text[copied..at]);
out.push_str("null");
copied = end;
}
at = end;
}
if copied == 0 {
return Cow::Borrowed(text);
}
out.push_str(&text[copied..]);
Cow::Owned(out)
}
/// The text with every lone UTF-16 surrogate escape (`\uD800` with no low half, `\uDC00`
/// alone) replaced by `\uFFFD`: same length, so offsets still point into the real text.
/// Only the probe reads this copy; the real text keeps its escapes for the core.
pub fn neutralise_lone_surrogates(text: &str) -> Cow<'_, str> {
let Some(mut at) = find_lone_surrogate(text) else {
return Cow::Borrowed(text);
};
let mut out = String::with_capacity(text.len());
let mut rest = text;
loop {
// An escape is six ASCII bytes: the offsets around it are char boundaries.
out.push_str(&rest[..at]);
out.push_str("\\uFFFD");
rest = &rest[at + 6..];
match find_lone_surrogate(rest) {
Some(next) => at = next,
None => break,
}
}
out.push_str(rest);
Cow::Owned(out)
}
/// `parseVersioned`'s acceptance: the text reads as JSON (R2's `parse_json`, JavaScript
/// duplicate-key semantics) once its lone surrogates are neutralised, is an object, and its
/// `envelopeVersion` or `version` is a number. The parsed value is dropped: the core reads
/// the real text. Expects a text already through [`rewrite_overflow_numbers`].
pub fn is_versioned_object(text: &str) -> bool {
let Ok(Value::Object(record)) = parse_json(&neutralise_lone_surrogates(text)) else {
return false;
};
["envelopeVersion", "version"]
.iter()
.any(|key| record.get(*key).is_some_and(Value::is_number))
}
/// Space, tab, line feed, carriage return: JSON's whitespace (form feed is not).
fn is_json_whitespace(byte: u8) -> bool {
matches!(byte, b' ' | b'\t' | b'\n' | b'\r')
}
/// What one byte is to the lexer.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum Lexeme {
/// Inside a string literal, its closing quote included.
InString,
StringStart,
/// `{` or `[`.
Open,
/// `}` or `]`.
Close,
/// `-` or a digit in value position.
NumberStart,
/// `t`, `f` or `n` in value position.
LiteralStart,
Other,
}
/// `withinJsonDepth`'s string skipping (a backslash skips the next byte, `"` closes), plus
/// the value position: the start of the text, or after a structural byte or whitespace.
struct Lexer {
in_string: bool,
escaped: bool,
value_position: bool,
}
impl Lexer {
fn new() -> Self {
Self {
in_string: false,
escaped: false,
value_position: true,
}
}
fn next(&mut self, byte: u8) -> Lexeme {
if self.in_string {
self.string_byte(byte);
return Lexeme::InString;
}
let structural = matches!(byte, b'{' | b'[' | b'}' | b']' | b':' | b',');
let at_value = std::mem::replace(
&mut self.value_position,
structural || is_json_whitespace(byte),
);
match byte {
b'"' => {
self.in_string = true;
Lexeme::StringStart
}
b'{' | b'[' => Lexeme::Open,
b'}' | b']' => Lexeme::Close,
b'-' | b'0'..=b'9' if at_value => Lexeme::NumberStart,
b't' | b'f' | b'n' if at_value => Lexeme::LiteralStart,
_ => Lexeme::Other,
}
}
fn string_byte(&mut self, byte: u8) {
if self.escaped {
self.escaped = false;
} else if byte == b'\\' {
self.escaped = true;
} else if byte == b'"' {
self.in_string = false;
}
}
}
/// Length of the run of number bytes at the start of `bytes`.
fn number_run_len(bytes: &[u8]) -> usize {
bytes
.iter()
.take_while(|byte| matches!(byte, b'0'..=b'9' | b'+' | b'-' | b'.' | b'e' | b'E'))
.count()
}
/// A JSON number whose value is ±Infinity for `JSON.parse` (same IEEE rounding as Rust's).
fn overflows(token: &str) -> bool {
is_json_number(token.as_bytes()) && token.parse::<f64>().is_ok_and(f64::is_infinite)
}
/// `-? (0 | [1-9][0-9]*) (\.[0-9]+)? ([eE][+-]?[0-9]+)?`, the whole token. Rust's float
/// parser also accepts `01`, `1.` or `.5`, which `JSON.parse` refuses.
fn is_json_number(token: &[u8]) -> bool {
let rest = token.strip_prefix(b"-").unwrap_or(token);
let rest = match rest {
[b'0', tail @ ..] => tail,
[b'1'..=b'9', tail @ ..] => skip_digits(tail),
_ => return false,
};
let rest = match rest.strip_prefix(b".") {
Some(fraction) => some_digits(fraction),
None => Some(rest),
};
let rest = match rest {
Some([b'e' | b'E', tail @ ..]) => {
let sign = tail.strip_prefix(b"+").or_else(|| tail.strip_prefix(b"-"));
some_digits(sign.unwrap_or(tail))
}
other => other,
};
rest.is_some_and(<[u8]>::is_empty)
}
fn skip_digits(bytes: &[u8]) -> &[u8] {
let count = bytes
.iter()
.take_while(|byte| byte.is_ascii_digit())
.count();
&bytes[count..]
}
/// What follows one or more digits; `None` when there is no digit.
fn some_digits(bytes: &[u8]) -> Option<&[u8]> {
let rest = skip_digits(bytes);
(rest.len() < bytes.len()).then_some(rest)
}