From 9850a8341a118636051f9a5941911a7ea67ff70e Mon Sep 17 00:00:00 2001 From: bchanot Date: Sun, 18 Oct 2026 15:44:54 +0200 Subject: [PATCH] feat(domain): R4.2c json_js: JS-path side effects of the attachment probe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit starts_json_object, depth and value prescan (OD-3 proxy), TextDecoder-parity lossy decode (400-case oracle generated by scripts/r4-utf8-oracle.mjs, 0 difference), overflow → null rewrite with the JSON number grammar, lone-surrogate neutralised probe (OD-4). --- crates/core/src/pdf/json_js.rs | 285 +++++++++++- .../tests/fixtures/pdf/r4/utf8-lossy.json | 405 ++++++++++++++++++ crates/core/tests/pdf_json_js.rs | 270 +++++++++++- scripts/r4-utf8-oracle.mjs | 125 ++++++ 4 files changed, 1083 insertions(+), 2 deletions(-) create mode 100644 crates/core/tests/fixtures/pdf/r4/utf8-lossy.json create mode 100644 scripts/r4-utf8-oracle.mjs diff --git a/crates/core/src/pdf/json_js.rs b/crates/core/src/pdf/json_js.rs index c5599c4d..f2f22644 100644 --- a/crates/core/src/pdf/json_js.rs +++ b/crates/core/src/pdf/json_js.rs @@ -1 +1,284 @@ -//! R4.2c: the JavaScript-path JSON side effects of the import. Written by the R4.2c track. +//! The JavaScript side effects of the attachment probe (R4.2c, plan §B.1.2, OD-3, OD-4). +//! +//! `packages/pdf/src/attachment/scan.ts` checked a candidate with `startsJsonObject` and +//! `withinJsonDepth`, then `parseVersioned` read it with `JSON.parse(new TextDecoder() +//! .decode(data))`, and the parsed value reached the core through `JSON.stringify`. That path +//! changed the text in ways serde_json does not (LRN-059), so each one is a step here, kept +//! apart so that the caller composes them and each can be measured: +//! +//! 1. [`starts_json_object`] and [`prescan`] on the raw bytes: the opening `{`, the depth cap +//! of `withinJsonDepth`, and a value count for the OD-3 budget, in the same linear pass. +//! 2. [`decode_lossy`]: invalid UTF-8 becomes U+FFFD and one leading BOM is dropped, as +//! TextDecoder does (measured identical, LRN-059, and pinned by the UTF-8 oracle). +//! 3. [`rewrite_overflow_numbers`]: a number that overflows f64 was ±Infinity for +//! `JSON.parse` and `null` once stringified, where serde_json refuses it; the rewritten +//! text is the one the core reads. +//! 4. [`is_versioned_object`]: `parseVersioned`'s acceptance, on a copy whose lone surrogate +//! escapes are neutralised ([`neutralise_lone_surrogates`]), because `JSON.parse` kept +//! them; the core then refuses the real text with `lone_surrogate` (OD-4, declared). + +use std::borrow::Cow; + +use serde_json::Value; + +use crate::domain::js::{find_lone_surrogate, parse_json}; + +/// The report and its envelope nest under 16 levels; deeper is not ours (`MAX_JSON_DEPTH`). +pub const MAX_JSON_DEPTH: usize = 64; + +const BOM: &[u8] = b"\xEF\xBB\xBF"; + +/// The signed form of the cap: the depth may go negative, as in the TypeScript. +const MAX_DEPTH: i64 = MAX_JSON_DEPTH as i64; + +/// `{` first, past an optional UTF-8 BOM and JSON whitespace: the only shape a version +/// lives in (`startsJsonObject`). +pub fn starts_json_object(data: &[u8]) -> bool { + let body = data.strip_prefix(BOM).unwrap_or(data); + body.iter().find(|byte| !is_json_whitespace(**byte)) == Some(&b'{') +} + +/// What [`prescan`] measured. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct JsonPrescan { + /// `withinJsonDepth`: the bracket depth never went over [`MAX_JSON_DEPTH`]. + pub within_depth: bool, + /// Values counted before the scan ended (at the refusal when `within_depth` is false). + pub values: u64, +} + +/// One linear pass over the raw bytes before any parser sees them: `withinJsonDepth` exactly +/// (string literals skipped, a backslash skipping the next byte; `{` and `[` deepen, `}` and +/// `]` rise, below zero too; the scan stops as soon as the depth exceeds the cap) and a +/// count of the values the parser would allocate, for the OD-3 budget. +/// +/// The count is an allocation proxy, not a strict JSON value count: every `{`, every `[`, +/// every string literal (object keys included, the parser allocates them too), and every +/// number or `true` / `false` / `null` start, which is a `-`, a digit, `t`, `f` or `n` +/// outside a string at the start of the text or after a structural byte or whitespace. +/// Malformed text is counted the same way. The count depends on the bytes only, so a +/// threshold measured against it is stable. One leading BOM is skipped (it never matches a +/// JSON byte, so the depth outcome is unchanged). +pub fn prescan(data: &[u8]) -> JsonPrescan { + let body = data.strip_prefix(BOM).unwrap_or(data); + let mut lexer = Lexer::new(); + let (mut depth, mut values) = (0_i64, 0_u64); + for &byte in body { + match lexer.next(byte) { + Lexeme::Open => { + depth += 1; + values += 1; + } + Lexeme::Close => depth -= 1, + Lexeme::StringStart | Lexeme::NumberStart | Lexeme::LiteralStart => values += 1, + Lexeme::InString | Lexeme::Other => {} + } + if depth > MAX_DEPTH { + return JsonPrescan { + within_depth: false, + values, + }; + } + } + JsonPrescan { + within_depth: true, + values, + } +} + +/// `new TextDecoder().decode(data)`: one leading BOM dropped, invalid UTF-8 replaced by +/// U+FFFD (maximal subparts, as the WHATWG decoder does). +pub fn decode_lossy(data: &[u8]) -> Cow<'_, str> { + String::from_utf8_lossy(data.strip_prefix(BOM).unwrap_or(data)) +} + +/// The text `JSON.stringify(JSON.parse(text))` would have handed on, as far as numbers go: +/// outside string literals, every number token whose value overflows f64 (`1e999`, `-1e999`, +/// a 400-digit integer) becomes `null`, everything else stays byte-identical. Underflow +/// reads as 0 on both sides and is kept; a token that is not a JSON number is kept and fails +/// the parse, as it failed `JSON.parse`. A token is the run of `[0-9+\-.eE]` that opens on a +/// `-` or a digit in value position: when `JSON.parse` succeeds, that run is its number. +pub fn rewrite_overflow_numbers(text: &str) -> Cow<'_, str> { + let bytes = text.as_bytes(); + let mut lexer = Lexer::new(); + let mut out = String::new(); + let (mut copied, mut at) = (0, 0); + while at < bytes.len() { + if lexer.next(bytes[at]) != Lexeme::NumberStart { + at += 1; + continue; + } + let end = at + number_run_len(&bytes[at..]); + // The run is ASCII, so both of its ends are char boundaries. + if overflows(&text[at..end]) { + out.push_str(&text[copied..at]); + out.push_str("null"); + copied = end; + } + at = end; + } + if copied == 0 { + return Cow::Borrowed(text); + } + out.push_str(&text[copied..]); + Cow::Owned(out) +} + +/// The text with every lone UTF-16 surrogate escape (`\uD800` with no low half, `\uDC00` +/// alone) replaced by `\uFFFD`: same length, so offsets still point into the real text. +/// Only the probe reads this copy; the real text keeps its escapes for the core. +pub fn neutralise_lone_surrogates(text: &str) -> Cow<'_, str> { + let Some(mut at) = find_lone_surrogate(text) else { + return Cow::Borrowed(text); + }; + let mut out = String::with_capacity(text.len()); + let mut rest = text; + loop { + // An escape is six ASCII bytes: the offsets around it are char boundaries. + out.push_str(&rest[..at]); + out.push_str("\\uFFFD"); + rest = &rest[at + 6..]; + match find_lone_surrogate(rest) { + Some(next) => at = next, + None => break, + } + } + out.push_str(rest); + Cow::Owned(out) +} + +/// `parseVersioned`'s acceptance: the text reads as JSON (R2's `parse_json`, JavaScript +/// duplicate-key semantics) once its lone surrogates are neutralised, is an object, and its +/// `envelopeVersion` or `version` is a number. The parsed value is dropped: the core reads +/// the real text. Expects a text already through [`rewrite_overflow_numbers`]. +pub fn is_versioned_object(text: &str) -> bool { + let Ok(Value::Object(record)) = parse_json(&neutralise_lone_surrogates(text)) else { + return false; + }; + ["envelopeVersion", "version"] + .iter() + .any(|key| record.get(*key).is_some_and(Value::is_number)) +} + +/// Space, tab, line feed, carriage return: JSON's whitespace (form feed is not). +fn is_json_whitespace(byte: u8) -> bool { + matches!(byte, b' ' | b'\t' | b'\n' | b'\r') +} + +/// What one byte is to the lexer. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Lexeme { + /// Inside a string literal, its closing quote included. + InString, + StringStart, + /// `{` or `[`. + Open, + /// `}` or `]`. + Close, + /// `-` or a digit in value position. + NumberStart, + /// `t`, `f` or `n` in value position. + LiteralStart, + Other, +} + +/// `withinJsonDepth`'s string skipping (a backslash skips the next byte, `"` closes), plus +/// the value position: the start of the text, or after a structural byte or whitespace. +struct Lexer { + in_string: bool, + escaped: bool, + value_position: bool, +} + +impl Lexer { + fn new() -> Self { + Self { + in_string: false, + escaped: false, + value_position: true, + } + } + + fn next(&mut self, byte: u8) -> Lexeme { + if self.in_string { + self.string_byte(byte); + return Lexeme::InString; + } + let structural = matches!(byte, b'{' | b'[' | b'}' | b']' | b':' | b','); + let at_value = std::mem::replace( + &mut self.value_position, + structural || is_json_whitespace(byte), + ); + match byte { + b'"' => { + self.in_string = true; + Lexeme::StringStart + } + b'{' | b'[' => Lexeme::Open, + b'}' | b']' => Lexeme::Close, + b'-' | b'0'..=b'9' if at_value => Lexeme::NumberStart, + b't' | b'f' | b'n' if at_value => Lexeme::LiteralStart, + _ => Lexeme::Other, + } + } + + fn string_byte(&mut self, byte: u8) { + if self.escaped { + self.escaped = false; + } else if byte == b'\\' { + self.escaped = true; + } else if byte == b'"' { + self.in_string = false; + } + } +} + +/// Length of the run of number bytes at the start of `bytes`. +fn number_run_len(bytes: &[u8]) -> usize { + bytes + .iter() + .take_while(|byte| matches!(byte, b'0'..=b'9' | b'+' | b'-' | b'.' | b'e' | b'E')) + .count() +} + +/// A JSON number whose value is ±Infinity for `JSON.parse` (same IEEE rounding as Rust's). +fn overflows(token: &str) -> bool { + is_json_number(token.as_bytes()) && token.parse::().is_ok_and(f64::is_infinite) +} + +/// `-? (0 | [1-9][0-9]*) (\.[0-9]+)? ([eE][+-]?[0-9]+)?`, the whole token. Rust's float +/// parser also accepts `01`, `1.` or `.5`, which `JSON.parse` refuses. +fn is_json_number(token: &[u8]) -> bool { + let rest = token.strip_prefix(b"-").unwrap_or(token); + let rest = match rest { + [b'0', tail @ ..] => tail, + [b'1'..=b'9', tail @ ..] => skip_digits(tail), + _ => return false, + }; + let rest = match rest.strip_prefix(b".") { + Some(fraction) => some_digits(fraction), + None => Some(rest), + }; + let rest = match rest { + Some([b'e' | b'E', tail @ ..]) => { + let sign = tail.strip_prefix(b"+").or_else(|| tail.strip_prefix(b"-")); + some_digits(sign.unwrap_or(tail)) + } + other => other, + }; + rest.is_some_and(<[u8]>::is_empty) +} + +fn skip_digits(bytes: &[u8]) -> &[u8] { + let count = bytes + .iter() + .take_while(|byte| byte.is_ascii_digit()) + .count(); + &bytes[count..] +} + +/// What follows one or more digits; `None` when there is no digit. +fn some_digits(bytes: &[u8]) -> Option<&[u8]> { + let rest = skip_digits(bytes); + (rest.len() < bytes.len()).then_some(rest) +} diff --git a/crates/core/tests/fixtures/pdf/r4/utf8-lossy.json b/crates/core/tests/fixtures/pdf/r4/utf8-lossy.json new file mode 100644 index 00000000..d33b77f5 --- /dev/null +++ b/crates/core/tests/fixtures/pdf/r4/utf8-lossy.json @@ -0,0 +1,405 @@ +{ + "generator": "scripts/r4-utf8-oracle.mjs, Node v24.21.0", + "cases": [ + {"bytes":"efbbbf","text":""}, + {"bytes":"c080","text":"��"}, + {"bytes":"c1bf","text":"��"}, + {"bytes":"e08080","text":"���"}, + {"bytes":"e09fbf","text":"���"}, + {"bytes":"f0808080","text":"����"}, + {"bytes":"f08fbfbf","text":"����"}, + {"bytes":"eda080","text":"���"}, + {"bytes":"edafbf","text":"���"}, + {"bytes":"edb080","text":"���"}, + {"bytes":"edbfbf","text":"���"}, + {"bytes":"eda080edb080","text":"������"}, + {"bytes":"f4908080","text":"����"}, + {"bytes":"f7bfbfbf","text":"����"}, + {"bytes":"f5","text":"�"}, + {"bytes":"f888808080","text":"�����"}, + {"bytes":"fc8480808080","text":"������"}, + {"bytes":"fe","text":"�"}, + {"bytes":"ff","text":"�"}, + {"bytes":"feff","text":"��"}, + {"bytes":"80","text":"�"}, + {"bytes":"bf","text":"�"}, + {"bytes":"808080","text":"���"}, + {"bytes":"c3","text":"�"}, + {"bytes":"e2","text":"�"}, + {"bytes":"e282","text":"�"}, + {"bytes":"f0","text":"�"}, + {"bytes":"f09f","text":"�"}, + {"bytes":"f09f98","text":"�"}, + {"bytes":"efbb","text":"�"}, + {"bytes":"c280","text":"€"}, + {"bytes":"c3a9","text":"é"}, + {"bytes":"e282ac","text":"€"}, + {"bytes":"ed9fbf","text":"퟿"}, + {"bytes":"ee8080","text":""}, + {"bytes":"efbfbd","text":"�"}, + {"bytes":"f09f9880","text":"😀"}, + {"bytes":"f48fbfbf","text":"􏿿"}, + {"bytes":"00","text":"\u0000"}, + {"bytes":"7f","text":""}, + {"bytes":"efbbbfefbbbf","text":""}, + {"bytes":"efbbbf7b2261223a317d","text":"{\"a\":1}"}, + {"bytes":"efbbbfefbbbf7b7d","text":"{}"}, + {"bytes":"61efbbbf62","text":"ab"}, + {"bytes":"efbb7b","text":"�{"}, + {"bytes":"efbbbfefbb","text":"�"}, + {"bytes":"efbbbf80","text":"�"}, + {"bytes":"3eccf4908080e09fbfedafbffefff2c1b4f09f9880eda08034f09f98f888808080f09f9880f09f78dd04","text":">����������������😀���4������😀�x�\u0004"}, + {"bytes":"384765efbfbd2afeff4679c3a9483ae08080efbbbf","text":"8Ge�*��FyéH:���"}, + {"bytes":"61b8c3a95c64eda08044b670c2e244e282f48fbfbfc28025c368f09f98676af8888080806def","text":"a�é\\d���D�p��D�􏿿€%�h�gj�����m�"}, + {"bytes":"efbbbfc1bf57f09f9880537fe2efbbbf7f663d572d49","text":"��W😀S�f=W-I"}, + {"bytes":"eda080edb080c1bfe09fbfc3edbfbf","text":"���������������"}, + {"bytes":"00eda080edb0804b731ac080feef","text":"\u0000������Ks\u001a����"}, + {"bytes":"aee4f7bfbfbf27a5c08026f038e87434eda080edb0","text":"������'���&�8�t4�����"}, + {"bytes":"d4484a5a56","text":"�HJZV"}, + {"bytes":"5b7fefbfbd802bf09f9880f0c3f49080","text":"[��+😀�����"}, + {"bytes":"794d256bedafbfcce09fbf2cdbe08080fff0808080f580e09fbffe5e5e9e69feffc341efbfbdc280d2f09f98ee8080fe","text":"yM%k�������,���������������^^�i���A�€���"}, + {"bytes":"19efbfbd6450666ef09f988034f08fbfbfe0","text":"\u0019�dPfn😀4�����"}, + {"bytes":"c280eda080e282ef","text":"€�����"}, + {"bytes":"c080302bbfc080e09fbf6c78e3edbfbf3d35f5003cbaeda080edbfbffc78fefff7bfbfbf2bedb0","text":"��0+������lx����=5�\u0000<��������x������+��"}, + {"bytes":"6e5149c280eda080fe7dedb080e2f061f078edb080c2804aefbb46e2","text":"nQI€����}�����a�x���€J�F�"}, + {"bytes":"fc8480808080f09f98feeb674f498bc080e09fbfc0804e2000f48fbfbf9345c33b50f4908080f09f98f08fbfbfed9f","text":"���������gOI��������N \u0000􏿿�E�;P����������"}, + {"bytes":"f09f98c9ff494eeda080edb08022daff80c3feff","text":"���IN������\"������"}, + {"bytes":"59d2feeda080edb0807fdc196b35f5f7bfbf","text":"Y���������\u0019k5����"}, + {"bytes":"ee8080c3c08029f08fbfbf","text":"���)����"}, + {"bytes":"c1bf2032f7bfbfbfe282acee8080e282ac784039eda080e2825214307f723699fc84808080807b28ff317fee8080","text":"�� 2����€€x@9����R\u00140r6�������{(�1"}, + {"bytes":"efbbbf00ee8080ca486a3d2880005f","text":"\u0000�Hj=(�\u0000_"}, + {"bytes":"f09f9880e0bff080","text":"😀���"}, + {"bytes":"efbbbff5b8f550f09f9880efbbbf","text":"���P😀"}, + {"bytes":"f7bfbfbf51a5f48fbfbf53f4908080555a34ee8080f490","text":"����Q�􏿿S����UZ4��"}, + {"bytes":"3de270f08fbf","text":"=�p���"}, + {"bytes":"e2f0","text":"��"}, + {"bytes":"c37fa093263af4908080eda080c080fe60fe4c5c379ee28278febf7fed9fbf","text":"���&:����������`�L\\7��x��퟿"}, + {"bytes":"30c2e282ac904b8ef4908080f7bfbfbfb3e2edbfbff34b7a5d","text":"0�€�K���������������Kz]"}, + {"bytes":"1ffe63ed9fbf","text":"\u001f�c퟿"}, + {"bytes":"00645dc280867ff09f9853e2625a9bed9fbfe282ac","text":"\u0000d]€��S�bZ�퟿€"}, + {"bytes":"bfc0803a2e45efbbc3a9ff63ff2f57f08fbfbfee79007c7fedafbfedb080edbf","text":"���:.E�é�c�/W�����y\u0000|��������"}, + {"bytes":"9a3cefbfbd46c3a9808080efbbbfd3","text":"�<�Fé����"}, + {"bytes":"7c7ffc84808080804e7f47e282596d3af7bfbfbfedbfbf","text":"|������NG�Ym:�������"}, + {"bytes":"e2fc848080808080f09fee8080c280c3a92228","text":"���������€é\"("}, + {"bytes":"ee8080f080808041","text":"����A"}, + {"bytes":"efbfbde080807fefbbeda0800e57bf49fefff09f98c1bff52bf09f","text":"��������\u000eW�I������+�"}, + {"bytes":"efbbbfeda0807f9339f0eda0","text":"����9���"}, + {"bytes":"c280f09ff09f9880ed9fbf804a5a3eca2a377ae28213506b00d62ef48fbfbf4393efbbeda080edb08050","text":"€�😀퟿�JZ>�*7z�\u0013Pk\u0000�.􏿿C��������P"}, + {"bytes":"62636df4908080fc848080808017cc03f043fc84808080800abfefbbcfefbfbdffc3a9f7bfbfbf","text":"bcm����������\u0017�\u0003�C������\n�����é����"}, + {"bytes":"efbbbf59800fe282ac5dee80802d68e282ace282acedb08044f07daaf09f9880fe","text":"Y�\u000f€]-h€€���D�}�😀�"}, + {"bytes":"3e77353440ed9fbfa8097a217d2db6f0f4908080c2805cf09f98803fb300c080f7bfbfbff5f09f","text":">w54@퟿�\tz!}-������€\\😀?�\u0000��������"}, + {"bytes":"c6f08fbfbfed9fbf5c2579f0","text":"�����퟿\\%y�"}, + {"bytes":"c0803d5ce282f4908080f08fbfbfefbbbfedb08071","text":"��=\\������������q"}, + {"bytes":"f09f5266f09f7732f5eda0","text":"�Rf�w2���"}, + {"bytes":"efbbbf49eda080876d672859ed9fbf44","text":"I����mg(Y퟿D"}, + {"bytes":"717aeda080edb080b00f","text":"qz�������\u000f"}, + {"bytes":"f09f988055c080f09f98803fe2b336efbbbfa9f7bfbf","text":"😀U��😀?�6����"}, + {"bytes":"ff3be282acee8080e28223997a274c44c3a952f0","text":"�;€�#�z'LDéR�"}, + {"bytes":"edafbfc1bf60f0edafbfa535fffef7bfbfbfee8080f09f98802246c380f5aefeff80e282ac5cf0ed9f","text":"�����`�����5������😀\"FÀ�����€\\��"}, + {"bytes":"efbbbfefbbbfedb080e2f09f4239c6e282ac84615c873af48fbfbfaa42","text":"�����B9�€�a\\�:􏿿�B"}, + {"bytes":"42ee8080ff29e08080eaed9fbff08080","text":"B�)����퟿���"}, + {"bytes":"f0808080eda0804bf0808080c6f09f9880feff51c280f09f9880","text":"�������K�����😀��Q€😀"}, + {"bytes":"efbbbfefbbbf68f48fbfbfefbbbf358c35e2fc8480808080413e67c2eda0","text":"h􏿿5�5�������A>g���"}, + {"bytes":"22a3e09fbf34efbbbfdb3158dac93d27e282f08fbfbff09f7407f08fbfbf7fb05d8f63c3f08080","text":"\"����4�1X��='������t\u0007�����]�c����"}, + {"bytes":"c2807c2916","text":"€|)\u0016"}, + {"bytes":"bfe282edb080edafbff4908080f48fbfbf2915f48fbfbf4103d47c23eda080feffc0","text":"������������􏿿)\u0015􏿿A\u0003�|#������"}, + {"bytes":"f6bf1929fc8480808080f05778eda080edb080f48fbfbfed9fbf57","text":"��\u0019)�������Wx������􏿿퟿W"}, + {"bytes":"c080f5bee160b1dae27943edbfbfa1c3565befbfbd753b8080805c0e705c","text":"�����`���yC�����V[�u;���\\\u000ep\\"}, + {"bytes":"efbbbfdd33ee8080e282ac","text":"�3€"}, + {"bytes":"54f4908080edb080f525c3a9f7bfbfbff09f5ffe","text":"T��������%é�����_�"}, + {"bytes":"efbbbf22edb080f5c2807fd547a2f02cedafbffefeb1feedb080f5586865f7bfbfbf65bf","text":"\"����€�G��,�����������Xhe����e�"}, + {"bytes":"efbb29e9feffc3c1bf1df0808080","text":"�)������\u001d����"}, + {"bytes":"dc4b0cf6f09f98678035f0","text":"�K\f��g�5�"}, + {"bytes":"832280ad4916edb080fc8480808080e09fbfe09fbf99e2eda080f88880808055f08fbfbf3dc080f7bfbfbf","text":"�\"��I\u0016�������������������������U����=������"}, + {"bytes":"efbbbfefbbbf7fce","text":"�"}, + {"bytes":"edb0807fc1bfefbb2f7d4551c3","text":"������/}EQ�"}, + {"bytes":"efbbf08fbfbf007bf09f71edb080c1bfa72cf080808044","text":"�����\u0000{�q������,����D"}, + {"bytes":"00d916daf0808080f0808080f48fbfbff09fc1bff48fbfbf6cfeff6c45340befbfbde0808004","text":"\u0000�\u0016���������􏿿���􏿿l��lE4\u000b����\u0004"}, + {"bytes":"6b0d86257015c3a9f09f982ef7bfbfbff4908080e282aceda080edb080f09ffeff3f4d4df036ed","text":"k\r�%p\u0015é�.��������€���������?MM�6�"}, + {"bytes":"52c08000efbfbdedafbfe2e08080dc6df08fbfbf21f48fbfbf57efbbe282acfee282ac422d003d","text":"R��\u0000���������m����!􏿿W�€�€B-\u0000="}, + {"bytes":"e282c3a9febfb1f09f625c514ef08fbfbfedbfbf34efbbbff48fbfbf7d27f0c080efbfbdf5f0feffa3","text":"�é����b\\QN�������4􏿿}'���������"}, + {"bytes":"b72a1e41bfefbbf2628080803057e080","text":"�*\u001eA���b���0W��"}, + {"bytes":"e28230edafbf21ee808048e282ace0","text":"�0���!H€�"}, + {"bytes":"3dffc0804239feff3dc3a9ae8080804b0010ee8080f4908080767dbfe2efbbf09f9880f0fc8480808080e09fbf70","text":"=���B9��=é����K\u0000\u0010����v}���😀����������p"}, + {"bytes":"60e09fbf6ae0808096e2ed9fbfed","text":"`���j�����퟿�"}, + {"bytes":"f09f98804eef","text":"😀N�"}, + {"bytes":"80808063f49080803eb4","text":"���c����>�"}, + {"bytes":"efbbbff53ceda080edb080","text":"�<������"}, + {"bytes":"efbbbf51ee80805c21feedbfbf6ac280","text":"Q\\!����j€"}, + {"bytes":"31e282ac645f30bf007ffee08080ed9fbfc280edb080005fe09fbf9be7efbbf09fefbbbf64","text":"1€d_0�\u0000����퟿€���\u0000_�������d"}, + {"bytes":"efbbbf4e45e3804980ee80802571f7bfbfbfe2c2807fe09fbf56","text":"NE�I�%q�����€���V"}, + {"bytes":"efbbbf44f09f98ee80805c4d7c5ac373ee8080c1bfe5efbbbff08fbf","text":"D�\\M|Z�s������"}, + {"bytes":"efbbbfefbbbff7bfbfbfe09fbfffedb0807de08080f08080802d79f1","text":"�����������}�������-y�"}, + {"bytes":"53ef","text":"S�"}, + {"bytes":"f09f9880eda080edb080ed9fbfeda08073c080f45638efbbbff7bfbfbf24ef0885","text":"😀������퟿���s���V8����$�\b�"}, + {"bytes":"523f7111c1bfe08080edb080bfee808066c280f5f8","text":"R?q\u0011���������f€��"}, + {"bytes":"ee808075","text":"u"}, + {"bytes":"efbbbfc280efbfbde09fbf2068fc5835ebf05d45efbbbf56b8f09f806be282c376f4908080e6","text":"€���� h�X5��]EV��k��v�����"}, + {"bytes":"efbbbfefbbbfad44f4908080a2b6f7c1bf890303f7bfbfbfc2804dedb080efbbefbb3dc9f7bfbfbf","text":"�D����������\u0003\u0003����€M�����=�����"}, + {"bytes":"c3f0808080ed","text":"������"}, + {"bytes":"efbbbfefbbbfe28267f09f98edbfbf5c7ef0f09f985b31df28f09f","text":"�g����\\~��[1�(�"}, + {"bytes":"efbbbfefbbbfd1bff08fbfbf80ee8080c343ed9fbf34c080f0edbfbfc1bff7bfbfbfeda0","text":"ѿ������C퟿4��������������"}, + {"bytes":"427d3efe48e09fbffffeff43c280f09f98feedafbfe09fbfc1bfefbbefbfbdc56e2bb2027f23c0736725d9d92f8080","text":"B}>�H������C€�������������n+�\u0002#�sg%��/��"}, + {"bytes":"c280ff4ff56d7f2672","text":"€�O�m&r"}, + {"bytes":"efbbbfefbbbfeda080edb0800b","text":"������\u000b"}, + {"bytes":"efbbbfefbbbfe09fbf27746154f888","text":"���'taT��"}, + {"bytes":"a1f08dc080f08fbfbf","text":"���������"}, + {"bytes":"efbbbf19fc8480808080e282f888808080587ff09f988050efbbedafbfc100feff6ff08080802a","text":"\u0019������������X😀P�����\u0000��o����*"}, + {"bytes":"e282f56df4908080bce282","text":"��m������"}, + {"bytes":"eda0806d297d35c3a9f56731ff2c7f6ceda080edb080e09fbff08080806f5fdbedafbff8","text":"���m)}5é�g1�,l�������������o_�����"}, + {"bytes":"47005be282f7bfbfbf203517f1329ffaefbfbd43f09f98","text":"G\u0000[����� 5\u0017�2���C�"}, + {"bytes":"7f20244d5c","text":" $M\\"}, + {"bytes":"efbbbfefbbbfe9c1bf317ef09fc3a96a0019f0808080ee8080","text":"���1~�éj\u0000\u0019����"}, + {"bytes":"efbbbf3ceda08029002e","text":"<���)\u0000."}, + {"bytes":"f09f2b","text":"�+"}, + {"bytes":"3b77edafbfcce09fbf2e9cefbbbfff6336","text":";w�������.��c6"}, + {"bytes":"efbbbfefbbbfc3e3e777f08fbfbfbfb7efbbbf582b3bfffeff70","text":"���w������X+;���p"}, + {"bytes":"edb0808b77da6095ff40e0a85ef0808080eda080eda080edb08044750736ffdc306332f5ff3d623ef09fe09f","text":"����w�`��@�^�������������Du\u00076��0c2��=b>���"}, + {"bytes":"efbbbfef","text":"�"}, + {"bytes":"f48fbfbfeede277f8ee282acbf6df000feff4a6defbbbf3575c6ee80805d8670ed9fbf658080801029b1f08fbfbff08f","text":"􏿿��'�€�m�\u0000��Jm5u�]�p퟿e���\u0010)�������"}, + {"bytes":"efbbbfc3a9fefff080808024477328e02048f09f980abff09f9880efbfbd7932feff77798479f09f98","text":"é������$Gs(� H�\n�😀�y2��wy�y�"}, + {"bytes":"ed9fbff691efbb","text":"퟿���"}, + {"bytes":"5f44f5efbbfe37efbbee80806713f09f98804c53eda080fac08080808080f09f98f56fe26123ef","text":"_D���7�g\u0013😀LS������������o�a#�"}, + {"bytes":"2d7a","text":"-z"}, + {"bytes":"6afeff5059eda080edb080feedaf","text":"j��PY���������"}, + {"bytes":"6b32f0c3a9b8eda0807b4dc280edb0806c58926d5dd7c3a967beceb836807f0250eda080edb0802620eda080edb080","text":"k2�é����{M€���lX�m]�ég�θ6�\u0002P������& ������"}, + {"bytes":"81edbfbf2b2e","text":"����+."}, + {"bytes":"e282ee8080e08080","text":"����"}, + {"bytes":"d88037f4908080","text":"؀7����"}, + {"bytes":"efbbbfefbbbf81ee808003edb080792200f4908080f507edb080c36ff08fbfbf","text":"�\u0003���y\"\u0000�����\u0007����o����"}, + {"bytes":"6ae282acaa4a466defbfbde0808063e27623efbbbfdb6b48f09f5da9efbb2ae2","text":"j€�JFm����c�v#�kH�]��*�"}, + {"bytes":"efbbbfefbfbd00a45eefbfbd72efbbfc84","text":"�\u0000�^�r���"}, + {"bytes":"38fc848080808048ff6dff73efbbbfbf54c080edbfbf808080efbfbdfe6467602df888808080f09f","text":"8������H�m�s�T����������dg`-������"}, + {"bytes":"b5eda0803934737cfec3808080c08038964ff09fc1bf2c565fedbfbfaa5804fc8480808080ee8080","text":"����94s|�À����8�O���,V_����X\u0004������"}, + {"bytes":"efbbbf255a29c97cf7bfbfbf2beabfe282ac22c280680080808040f0","text":"%Z)�|����+�€\"€h\u0000���@�"}, + {"bytes":"bf6200efbbbf71edafbf","text":"�b\u0000q���"}, + {"bytes":"f09f9880f08fbfbf0b267be08080eda0806ef7bfbfbf94f0","text":"😀����\u000b&{������n������"}, + {"bytes":"efbbbf6a5a0ff09f9880f08fbfbfc08073feff5c4557efbbbff09f98643defbbbfc3feff","text":"jZ\u000f😀������s��\\EW�d=���"}, + {"bytes":"60ed9fbfefbbbff8f07befbbe24269","text":"`퟿��{��Bi"}, + {"bytes":"4936e080801073f0","text":"I6���\u0010s�"}, + {"bytes":"4fffee80","text":"O��"}, + {"bytes":"efbbbff5f09f9880eda080a7bff09f62c3fe01f09f98806b","text":"�😀������b��\u0001😀k"}, + {"bytes":"5a771eed9fbf6aeda080","text":"Zw\u001e퟿j���"}, + {"bytes":"4eedbfbfedb0806ff48fbf","text":"N������o�"}, + {"bytes":"f7bfbfbff49080802900eda080edb0805f72efbbbf716880e282","text":"��������)\u0000������_rqh��"}, + {"bytes":"cc7f4ff09feda080edb080f09f0053bf30e28271c08036ed7930808080e282acbfd2f0808080430cf88880808087","text":"�O��������\u0000S�0�q��6�y0���€������C\f������"}, + {"bytes":"4ec3d029afe08080f5fe260123edbfbf7df09f9844563e7cff50feff59808092","text":"N��)������&\u0001#���}�DV>|�P��Y���"}, + {"bytes":"efbbbfc3a98ff09f988072f09f9880edafbfc3edb080ef094732d4e2a3f490808000fc848080808031f09f6ec379","text":"é�😀r😀��������\tG2������\u0000������1�n�y"}, + {"bytes":"5df08fbfbfc07aeda080c080f09f98c32cedafbf7f","text":"]�����z�������,���"}, + {"bytes":"efbbbfefbbbfee8080ff95283a7be0808035e09fbfbff08fbfbf8093f08fbfbfed9fbfc080efbfbdcaf8888080","text":"��(:{���5��������������퟿��������"}, + {"bytes":"e282","text":"�"}, + {"bytes":"568033378c306dedafbff09f98805746f8888080802ff09fe080806ee09fbfbf35e8700b","text":"V�37�0m���😀WF�����/����n����5�p\u000b"}, + {"bytes":"c3a905f48fbfbf80c23a63f48fbfbf795abe37f09f56cde0808032edbfbfefbfbde282acedb0802bf09f98ef","text":"é\u0005􏿿��:c􏿿yZ�7�V����2����€���+��"}, + {"bytes":"e282","text":"�"}, + {"bytes":"09e282f080808078ed9fbf2ac433fff08fbfbf3079808080c339f0808080fefff48fbf","text":"\t�����x퟿*�3�����0y����9�������"}, + {"bytes":"2570efbfbd0d886b3efeff79d0f5","text":"%p�\r�k>��y��"}, + {"bytes":"bfd675ed9fbff1edafbfbf6aaee28226c376f09f984f43efbbd323845847f08fbfbfed9fbf35ee8080e09fbf","text":"��u퟿�����j��&�v�OC��#�XG����퟿5���"}, + {"bytes":"4f33bf44eda080f169fb69a571ee8080fef080808058434afc84808080","text":"O3�D����i�i�q�����XCJ�����"}, + {"bytes":"f48fbfbff09fefbfbdfb7ceda0802bc3f4908080edbfbfed9fbffeffe0e8efbbf08080805beda080f09f","text":"􏿿���|���+��������퟿���������[����"}, + {"bytes":"6bf48fbf","text":"k�"}, + {"bytes":"efbbbfefbbbf53e09fbf","text":"S���"}, + {"bytes":"8080807fee8080c3a932d415f08fbfbf2cf7bfbfbfed","text":"���é2�\u0015����,�����"}, + {"bytes":"ed9fbf42f08fbfbf5e43efbfbd4f50c3a9ed9fbff0efbbee80804c00939fd47efeff7e8f46","text":"퟿B����^C�OPé퟿��L\u0000���~��~�F"}, + {"bytes":"f09f6f76c3a9f08fbfbffdbf3a3c70f08fbfbf22","text":"�ové������:���������€�Y�"}, + {"bytes":"edafbf4d97c3a9f50cf5efbbf7bfbfbfe180c365d3f077e2c3","text":"���M�é�\f��������e��w��"}, + {"bytes":"730648ee8080efbbbfeed469eda0803428efbfbd293cc80533","text":"s\u0006H��i���4(�)<�\u00053"}, + {"bytes":"4ec0801f43ef","text":"N��\u001fC�"}, + {"bytes":"efbbbfe23bfc84808080806df09ffefffc848080808052ed9fbf3e5ef48fbfbf3d","text":"�;������m���������R퟿>^􏿿="}, + {"bytes":"7d47c080eda080edb080f08fbfbf2be5e282a1c2803f26275809c1bf616de28200efbfbdd8efbbbfefbbbfefbfbde0","text":"}G������������+�₡€?&'X\t��am�\u0000����"}, + {"bytes":"f09f98805a64efbb618080804ce08080feffc28035d4edafbfdcf09f98f08080803d3530f09fedafbf","text":"😀Zd�a���L�����€5����������=50����"}, + {"bytes":"efbbbffc8480808080c080e2edafbfefbbee80802ff09f98f0f08fbfbf40c3a9f08fbfbf7fe282e282","text":"�������������/������@é������"}, + {"bytes":"6a3b4beda080edb080dded9fbfe2822556f4908080edafbf9a69c3f888808080bf","text":"j;K�������퟿�%V��������i�������"}, + {"bytes":"6510f8bf19f09f95edafbf58f09f93f133009df09f9880f09f9880ed9fbff888808080f09f988019e080806fc280e0","text":"e\u0010��\u0019����X��3\u0000�😀😀퟿�����😀\u0019���o€�"}, + {"bytes":"5d7ce27a42e09fbf313fedbfbfe09fbf28c72f5cf4908080c0f48fbfbfedbfbfc1bfe282bfc3a95dc080fe797f79","text":"]|�zB���1?������(�/\\�����􏿿�����₿é]���yy"}, + {"bytes":"edafbff5faf09f98edbfbf91fec3f4908080edb080feedbfbf2f60fee08080","text":"�����������������������/`����"}, + {"bytes":"db702969e20d13f0808080","text":"�p)i�\r\u0013����"}, + {"bytes":"4440eda080e282ac6b26","text":"D@���€k&"}, + {"bytes":"efbbbf4e78654280b9","text":"NxeB��"}, + {"bytes":"25e282f4908080007af08fbfbf00235ff0eda080edb080f08080806648e8feff31efbfbd","text":"%�����\u0000z����\u0000#_�����������fH���1�"}, + {"bytes":"7efe","text":"~�"}, + {"bytes":"edafbf73c3e508a90b85cb524f8388eda08076f09f9880","text":"���s��\b�\u000b��RO�����v😀"}, + {"bytes":"f09f78c2808ded9fbfc280c1bf2172f09f985ef09f98e0","text":"�x€�퟿€��!r�^��"}, + {"bytes":"efbbbfefbbbf00e09fbffc8480808080f080808073edb0805407789ef0","text":"\u0000�������������s���T\u0007x��"}, + {"bytes":"073621f09f987f75e08080ed9fbff08080805b6529fc8480808080c3a969e08080c344e09fbffe","text":"\u00076!�u���퟿����[e)������éi����D����"}, + {"bytes":"eda080edb080f09f988037edb080dc7f634feda080f068517fedbfbfc280f0","text":"������😀7����cO����hQ���€�"}, + {"bytes":"f08fbfbf552cee8080edbfbfc3a93200f8888080","text":"����U,���é2\u0000����"}, + {"bytes":"efbbbfefbbbfc967efbbbf6a2b35","text":"�gj+5"}, + {"bytes":"6ec1","text":"n�"}, + {"bytes":"efbbbf55926e72c32039edafbf0726327ff0808080f0808080a9e6c1e282f7bfbfbf80308080806b7112edb0","text":"U�nr� 9���\u0007&2�����������������0���kq\u0012��"}, + {"bytes":"e282acf0582eff49215f3f325dc080feffefbfbd6480382fff7cc1","text":"€�X.�I!_?2]�����d�8/�|�"}, + {"bytes":"fb50","text":"�P"}, + {"bytes":"efbbbfefbbbf7c647fc3a9342253a13677e282f09f0332c3c573caf5c280efbbefbbbfee80","text":"|dé4\"S�6w��\u00032��s��€��"}, + {"bytes":"efbbbf21eda080eda080edb08000465d5c6aefbb663ef08080805ceda080","text":"!���������\u0000F]\\j�f>����\\���"}, + {"bytes":"efbbbf7fefbfbdb6eda080ef","text":"������"}, + {"bytes":"62fc8480808080bcf00033ffbf4b7fc080f0fc4e2cf09f98f09f98803cf09f984fefbbbff8fc8480808080edafbf","text":"b��������\u00003��K����N,�😀<�O����������"}, + {"bytes":"efbbbf4a80ee80804223b4c280e532e282ace2366dba426aefbbedbfbf235ef5ed9fbfe5efbfbdc1","text":"J�B#�€�2€�6m�Bj����#^�퟿���"}, + {"bytes":"edb080802d2ef0efbb6f2cf0e08080b6c1bf157f7b5a","text":"����-.��o,�������\u0015{Z"}, + {"bytes":"6aefbb3731eda08024808080f580f7bfbfbf807ff0808080343460f49080806befbbbfedbfbf4c5bf888808080","text":"j�71���$��������������44`����k���L[�����"}, + {"bytes":"7cf5edbfbfb653bf6ea57cf7bfbfbf32f49080","text":"|�����S�n�|����2���"}, + {"bytes":"efbbbfefbbbf35c280f4908080bdedbfbfefbfbde9efbfbd808080654b60efbfbd0e552e5bedbf","text":"5€��������������eK`�\u000eU.[��"}, + {"bytes":"f08fbfbfbfefbbbf4bf09f98e2","text":"�����K��"}, + {"bytes":"efed9fbf10396459","text":"�퟿\u00109dY"}, + {"bytes":"ed9fbfc280aab43a8080807fed9fbfedbfbfe2f50d594d5de09fbfaff5f080808051b863edbfbf66e2","text":"퟿€��:���퟿�����\rYM]���������Q�c���f�"}, + {"bytes":"0c808022b158c08010f09f98a6e270e282eda080c0802172f0808080c1bf6b75f08fbfbf2af08080807f86","text":"\f��\"�X��\u0010😦�p������!r������ku����*�����"}, + {"bytes":"c280f0808080f4908080f09f987bedbfbfb9460553e0808055e2e282acc39bf08f","text":"€���������{����F\u0005S���U�€Û��"}, + {"bytes":"7fe282ac","text":"€"}, + {"bytes":"efbbbf41427441","text":"ABtA"}, + {"bytes":"6a7031e09fbf272e6f26e08080c080e08080f88880808000f08fbfbf50a6c339f7bfbfbffc8480808080f490","text":"jp1���'.o&�������������\u0000����P��9������������"}, + {"bytes":"60866ac3a9c280ed9fbf72efbbed9fbfedb08039c2807ff4908080edbfbfc080e2ed","text":"`�jé€퟿r�퟿���9€�����������"}, + {"bytes":"66a4fe6defbb3ceb","text":"f��m�<�"}, + {"bytes":"eda080edb080eafc8480808080fc8480808080bffa8001e282feefbbbf5f5a","text":"����������������������\u0001��_Z"}, + {"bytes":"64eda0808d808080bf72e282acf48fbfbff7bfbfbffe47efbbbfeda0","text":"d��������r€􏿿�����G��"}, + {"bytes":"aceda080f08fbfbfb51e6659c080efbfbd26f7bfbfbf76f09f98e09fbfe282ace46034e0808024f08fbfbff5f07c8080","text":"���������\u001efY���&����v����€�`4���$������|��"}, + {"bytes":"efbbbff09f9880e08080595ceda080edb0805741eda080f09f988063f5b7f490","text":"😀���Y\\������WA���😀c����"}, + {"bytes":"3dc3a9e080","text":"=é��"}, + {"bytes":"efbbbfefbbbff57244586a5857edbfbf7fc0","text":"�rDXjXW����"}, + {"bytes":"efbbbfefbbbf3afe5afc84808080808f2f6c59e9ee8080edb0802fc1bf","text":":�Z�������/lY����/��"}, + {"bytes":"877cbfc3e09fbff490808000e2f08080","text":"�|���������\u0000����"}, + {"bytes":"92ed9fbf1b24c35ebf8652771c48f55530b42eff73efbb55f08fbfbf4afc84808080","text":"�퟿\u001b$�^��Rw\u001cH�U0�.�s�U����J�����"}, + {"bytes":"797a71","text":"yzq"}, + {"bytes":"efbbbffc848080808049e09fbfefbbbfc280e282acede09fbfc1bf00c080f05ce28220975f00fff0808080ed","text":"������I���€€������\u0000���\\� �_\u0000������"}, + {"bytes":"e28250c3feffc280efbfbd6f3bbb3b7f8072c3a957eda080e2f48fbfbf53c3326529b42823fc8480","text":"�P���€�o;�;�réW����􏿿S�2e)�(#���"}, + {"bytes":"7a753880eda080c280f09f50f0808080eda08055bf3c25e282efbfbd51efbfbd8c","text":"zu8����€�P�������U�<%��Q��"}, + {"bytes":"a8c280f48fbfbffc848080808042e08080e74dedafbf2befbbbfedaf","text":"�€􏿿������B����M���+��"}, + {"bytes":"aae282ac5809e282acf48fbfbf4f7ff5c1bff09f35efbbbf77f062fc8480808080c3a9e282edbfbf612a4571feffe9","text":"�€X\t€􏿿O����5w�b������é����a*Eq���"}, + {"bytes":"efbbbf1af0b2a266f48fbfbf8956c3a9f48fbfbfcbe20365cff48fbfbff888808080f7bfbfbfc2802c51","text":"\u001a�f􏿿�Vé􏿿��\u0003e�􏿿���������€,Q"}, + {"bytes":"10f09f98f7bfbfbff09ffe7e5bc3a986f48fbfbfc080f48fbfbff88880808051edbfbfe0808046c1bf1afefe777feb","text":"\u0010�������~[é�􏿿��􏿿�����Q������F��\u001a��w�"}, + {"bytes":"efbbbfefbbbfb7efbb3d00748d","text":"��=\u0000t�"}, + {"bytes":"efbbbf7dedafbf6cbfaa156b38eda08022681ecbedafbfe09fbf457ff09f98c378f49080807fc9c35651cd","text":"}���l��\u0015k8���\"h\u001e�������E��x������VQ�"}, + {"bytes":"efbbbfefbb","text":"�"}, + {"bytes":"efbbbfefbbbf3aa027e09fbfefbbbf63f080808061f09f9880feff63f09f9880bf7b182370efbbbf66373121f4908080","text":":�'���c����a😀��c😀�{\u0018#pf71!����"}, + {"bytes":"efbfbd815c7f5cf4908080e2e282acc080f5693eefbb7f6c808080f48fbfbfefbbf02ef09f4f3e1052","text":"��\\\\�����€���i>�l���􏿿��.�O>\u0010R"}, + {"bytes":"bfc3fefffeffc0f09f98ed9fbfc35cd0f09f64eda080ed","text":"��������퟿�\\��d����"}, + {"bytes":"e2c1bf2611ab4feebff88880808075","text":"���&\u0011�O������u"}, + {"bytes":"c1bffc84808080809b5cf09f9828eff080808060","text":"���������\\�(�����`"}, + {"bytes":"efbb5e4ac080e2bf282e25c303f08fbfbfe282ac1af7bfbfbff7bfbfbfedbfbfe282fe","text":"�^J���(.%�\u0003����€\u001a�������������"}, + {"bytes":"c0804c76ee8080","text":"��Lv"}, + {"bytes":"312e4c3ce2f4908080fd42f09f72cd80","text":"1.L<������B�r̀"}, + {"bytes":"c1bf4f32f48fbfbffc8480808080efbbbfc3a9f8","text":"��O2􏿿������é�"}, + {"bytes":"27f09f9880e282dbf09ff08fbfbfed9fbfefbfbd68f0808080edbfbfefbbbfb9e282f5f09f98ed9f","text":"'😀�������퟿�h������������"}, + {"bytes":"c1bff0f025f09f8070ff2af0","text":"����%�p�*�"}, + {"bytes":"5bedb080e28281c0805b459c55379d1635feff22ed","text":"[���₁��[E�U7�\u00165��\"�"}, + {"bytes":"e0808080808051f09f98efbb00c3a9edafbfed9fbfe282edafbf283df7bfbfbff0808080f48fbfbff09ff09f98fc84","text":"������Q��\u0000é���퟿����(=��������􏿿����"}, + {"bytes":"5d2e5444efbbededafbf9e50ed9fbf4cd9c0","text":"].TD������P퟿L��"}, + {"bytes":"efbbbff48fbfbfeda080c1bf67b7f0f51e712f6eedafbfffe282ac6f3d4e87d22deda080edb0","text":"􏿿�����g���\u001eq/n����€o=N��-�����"}, + {"bytes":"e2f48fbfbfbf2c473fede282acfeefbfbd1a2780807fcaee6df09f","text":"�􏿿�,G?�€��\u001a'����m�"}, + {"bytes":"efbbbfefbbbf5be2f0808080e982fc848080808050eda08034","text":"[������������P���4"}, + {"bytes":"e28266ed9fbffc8480808080b73d24edbfbffc8480808080f7bfbfbfefbbbf91eda08080808061e2f48fbfbf3e","text":"�f퟿�������=$��������������������a�􏿿>"}, + {"bytes":"fc8480808080e282dc45f09fc01eefbbbf7ff08fbfbf73f8888080806b4dfeff0840c28052402ae25fa7732f04422e","text":"��������E��\u001e����s�����kM��\b@€R@*�_�s/\u0004B."}, + {"bytes":"efbbbfefbb4af0edb08087edafbf1176e237f076fe","text":"�J��������\u0011v�7�v�"}, + {"bytes":"da00deedbfbff09f98807903c3e24e5a23efbbbf808080f0808080f490","text":"�\u0000����😀y\u0003��NZ#���������"}, + {"bytes":"7afe","text":"z�"}, + {"bytes":"e282ac3862808080efbbbfefbbbf7470","text":"€8b���tp"}, + {"bytes":"efbbbfad19e282ac54427aff","text":"�\u0019€TBz�"}, + {"bytes":"efbbbfc3e09fbf5c8b57f09f9880feee8080c3c080f080808064e2820635ed9fbf42bf7527004e","text":"����\\�W😀��������d�\u00065퟿B�u'\u0000N"}, + {"bytes":"29efbbbffc8480808080","text":")������"}, + {"bytes":"f09fa863e08080c08032f49080","text":"�c�����2���"}, + {"bytes":"c280f09f98f7e235c6ee808072feff7ffec080ed","text":"€���5�r������"}, + {"bytes":"3155f0f7bfbfbfc523ec3aee8080fc8480808080","text":"1U������#�:������"}, + {"bytes":"7befbbbffe3946c3a9eda080edb08004465af7bfbfbffeff","text":"{�9Fé������\u0004FZ������"}, + {"bytes":"5d3a754a2aedb080eda080ed","text":"]:uJ*�������"}, + {"bytes":"fe22edafbff08fbfbf8032ed9fbfa4e2e5f0808080f4908080450b32f48fbfbfedb08077b534e2f08fbfbf","text":"�\"��������2퟿�����������E\u000b2􏿿���w�4�����"}, + {"bytes":"efbbbfefbbbfefbfbd29f0f08fbfbf64e282d0a7bbed9fbf00f051efbbbffefffc8480808080","text":"�)�����d�Ч�퟿\u0000�Q��������"}, + {"bytes":"80265422ae6e7054","text":"�&T\"�npT"}, + {"bytes":"f09f9880c8efbbbff4908080e2ee8080ed","text":"😀�������"}, + {"bytes":"6beda080ba41c92ffeff9643226312f08080805af888808080e08080f4908080c3a926808080ef","text":"k����A�/���C\"c\u0012����Z������������é&����"}, + {"bytes":"19ee8080","text":"\u0019"}, + {"bytes":"efbbbfefbbbf8080807e76f09ff09f98807b06f4908080c2c93bfe6f35e282ac60f48fbfbf29345ef0808080","text":"���~v�😀{\u0006������;�o5€`􏿿)4^����"}, + {"bytes":"22f88880808024748080804b7a","text":"\"�����$t���Kz"}, + {"bytes":"b43df48fbfbfbf25c3a97cc3e08080feedafbf4bedb080eef08fbfbf792dc3335638ed9fbf","text":"�=􏿿�%é|��������K��������y-�3V8퟿"}, + {"bytes":"efbbbfefbbbfedafbf337fc34dc421feffd05ff0","text":"���3�M�!���_�"}, + {"bytes":"efbbbfefbb","text":"�"}, + {"bytes":"bfc700eda080efbfbdd8bf9b89b8efbbbf27f08fbfbf6428f09f9880ff36efbbbe3cf48fbfbf8f6780808077","text":"��\u0000����ؿ���'����d(😀�6﻾<􏿿�g���w"}, + {"bytes":"bf3575e282ace2efbbc3a9e09fbffc8480808080e080802cf0808080f09f74","text":"�5u€��é������������,�����t"}, + {"bytes":"4259f08fbfbf41c080eda080edb080ffe08080f8888080800080","text":"BY����A�����������������\u0000�"}, + {"bytes":"c080eda0","text":"����"}, + {"bytes":"efbb2ee09fbfe09fbf72eda080edb080fe","text":"�.������r�������"}, + {"bytes":"c319b9f09f6ceaf888808080f080808064f7bfbfbfe08080774eed9fbfbbf5eda0802de09fbfc3f48fbfbfa4c3a9f48f","text":"�\u0019��l����������d�������wN퟿�����-����􏿿�é�"}, + {"bytes":"727e774250c328f0","text":"r~wBP�(�"}, + {"bytes":"e282acff80808080efbfbdc280","text":"€������€"}, + {"bytes":"abf09f9880e08080c1bf61f7bfbfbfefbfbdeda080ee8080d5","text":"�😀�����a���������"}, + {"bytes":"704478eda080f49080802f4cb9aec6e09fbf","text":"pDx�������/L������"}, + {"bytes":"e2","text":"�"}, + {"bytes":"e282acc3a946feed9fbfed9fbff8888080805a6ae2edafbff5e09fbf67feffeda080edb0804ff09f30feffedbfbf","text":"€éF�퟿퟿�����Zj��������g��������O�0�����"}, + {"bytes":"efbbbfefbbbf580a44487ea67ffeff90f0808080c3c1bf57a7f888808080f7bfbfbf3af7bfbfbf25f5","text":"X\nDH~�����������W����������:����%�"}, + {"bytes":"fc8480808080eda080edb080efbbbfffff53f4908080f09f","text":"��������������S�����"}, + {"bytes":"c2807665e09fbfedafbfe2f09fd979c280c2","text":"€ve���������y€�"}, + {"bytes":"f08080800c7fffc3a95cf0808080bf3ce53eee8080f8888080802df09f98f88880808080","text":"����\f�é\\�����<�>�����-�������"}, + {"bytes":"efbbbff0808080c82cc1bf315a0168c3a9c28066f09f9880ff244beda080","text":"�����,��1Z\u0001hé€f😀�$K���"}, + {"bytes":"41eda080eda080edb08040edafbff5723f79437995c325e327bfefbbf09f9880ee8080ecff9546","text":"A���������@����r?yCy��%�'��😀���F"}, + {"bytes":"ed9fbf1feda080f49080","text":"퟿\u001f������"}, + {"bytes":"efbbbfefbbbfc080efbbbf53f09ff09f988032eda080edb08041288080804ea0ee8080f48fbfbfe282f09f9875","text":"��S�😀2������A(���N�􏿿��u"}, + {"bytes":"809ce2edafbf5fa685e282ace282ac505a55fef09f98e09fbfefbb","text":"������_��€€PZU������"}, + {"bytes":"edbfbfe2822d8080805257ee8080feffff5f48f08fbfbf50ff2cb1bf2ef888808080edbfbfe080","text":"����-���RW���_H����P�,��.����������"}, + {"bytes":"efbbbfa7534ac3a9ed","text":"�SJé�"}, + {"bytes":"efbbbfefbbbf","text":""}, + {"bytes":"efbbbfefbb","text":"�"}, + {"bytes":"3bfc8480808080c1bf5f7fee80805cfefef888808080f08fbfbf69ce5c00eda080f0f56758ed9fbf6c","text":";��������_\\�����������i�\\\u0000�����gX퟿l"}, + {"bytes":"f48fbfbf2aeda080edb08032efbbbfedb080e08080efbb7c6ac28020f57fc3a9ed9fbfb6f48fbfbffff09f98","text":"􏿿*������2�������|j€ �é퟿�􏿿��"}, + {"bytes":"fbee8080075c2a266b00fe73efbbbff08fbfbfb81d4f29a0e2e7c3baeda080edb0","text":"�\u0007\\*&k\u0000�s�����\u001dO)���ú�����"}, + {"bytes":"efbbbfe0808088e08080f0808080d060edbfbf6ef5fc84808080804b","text":"������������`���n�������K"}, + {"bytes":"fe267158c280f7bfbfbfeda080edb080808080e2feff7ff7bfbfbf23005579f09f988076edb080e22f75693fefbbbf67","text":"�&qX€��������������������#\u0000Uy😀v����/ui?g"}, + {"bytes":"c080e2efbb1445ee808024ee808034eda080edb080c080f5415365eda080","text":"����\u0014E$4���������ASe���"}, + {"bytes":"efbbbfb839c08038f48fbfbfe282f09ff080","text":"�9��8􏿿����"}, + {"bytes":"42468fc080e282ac7d00e0808051ed9fbf8e","text":"BF���€}\u0000���Q퟿�"}, + {"bytes":"2e2ef08fbfbf73f08abff09f9880efbfbdbd35c280","text":"..����s���😀��5€"}, + {"bytes":"f0efbfbdedbfbfedbfbf9872ee808073e080807defbbbfc3a92280784e0761f4908080edafbff09f98","text":"���������rs���}é\"�xN\u0007a��������"}, + {"bytes":"ff29c3eda080edb080c08000eda08017e23831efbfbdf490808041f4908080e282efbbbf7fc778f48fbfbf6d","text":"�)���������\u0000���\u0017�81�����A������x􏿿m"}, + {"bytes":"bfaf64f08fbfbff08080806af5afff2ac1bf7ff58521f7bfbf","text":"��d��������j���*����!���"}, + {"bytes":"edb080fe3324260362f08fbfbfe282ffedbfbfc280e282ac215de28266a3e282ac8027f48fbfbfe0f5f09f98","text":"����3$&\u0003b���������€€!]�f�€�'􏿿���"}, + {"bytes":"c373e282ac2dfe2970c3c1bf7af48fbfbf20c0804cf48fbfbf0080808068bf4f489a36ed","text":"�s€-�)p���z􏿿 ��L􏿿\u0000���h�OH�6�"}, + {"bytes":"21f09f98806befbbbf386d804fe24ae04da9ffefbbbfedb0803aefbbbfefbfbdbfefbbbfc280f0808080efbfbd2ac0","text":"!😀k8m�O�J�M�����:��€�����*�"}, + {"bytes":"efbbbf3d","text":"="}, + {"bytes":"fb23c93bf0808080efbbf0f7bf","text":"�#�;��������"}, + {"bytes":"f888808080c280f48fbfbfb1fd7c12c2","text":"�����€􏿿��|\u0012�"}, + {"bytes":"efbbbfefbbbf2df08fbfbf54f0808080f7bfbfbf2e3df49080","text":"-����T��������.=���"}, + {"bytes":"4c804ab7936d9311ee80803c7ff09f98725e4ce2c3edbfbf003e57f5f4","text":"L�J��m�\u0011<�r^L�����\u0000>W��"}, + {"bytes":"fc8480808080f0","text":"�������"}, + {"bytes":"f7bfbfbfedbfbff888808080f09f2e6cc080fc848080808076efbbbff09f9824c28061a13e444c697f709cffbfc0","text":"�������������.l��������v�$€a�>DLip����"}, + {"bytes":"c1bfbfeda080dd22806af7bfbfbf6e52eda0","text":"�������\"�j����nR��"}, + {"bytes":"d85ffe5674f08fbfbffe23","text":"�_�Vt�����#"}, + {"bytes":"edafbfeda080ed9fbff7bfbfbfe2ee8080efbbbf808080edaf","text":"������퟿����������"}, + {"bytes":"e4e0808043a65ae780f888808080eda080fdf7bfbfbff7bfbfbf","text":"����C�Z������������������"}, + {"bytes":"20ee7f3a124a7f917530c3e2edafbfc1bf6ede1feb4fefbfbdc3a9ed9fbf","text":" �:\u0012J�u0�������n�\u001f�O�é퟿"}, + {"bytes":"efbbbfefbbbf57f48fbfbfc08019eda080edb080c3bb6fc335feabefbb42e1808080f7bf","text":"W􏿿��\u0019������ûo�5���Bက���"}, + {"bytes":"ed","text":"�"}, + {"bytes":"efbbbfefbbbf80808048","text":"���H"}, + {"bytes":"36e09fbf0f807fedb0807f47f000805e55f4908080bce7c1bfffc280f7","text":"6���\u000f����G�\u0000�^U���������€�"}, + {"bytes":"e25bf08fbfbf2af8888080807fe282acee8080","text":"�[����*�����€"}, + {"bytes":"f0fefeff938080808a57bff08fbfbf672ac280ed9fbff08080","text":"���������W�����g*€퟿���"}, + {"bytes":"e240c3c1bffeff668048eda0806ceda080edb0802afc8480808080f09f98f0808080e01de09fbf","text":"�@�����f�H���l������*������������\u001d���"}, + {"bytes":"efbbbfefbbbfefbbbff08fbfbff08fbfbffc848080","text":"������������"}, + {"bytes":"8a393bf007520aedafbfbff7bfbfbfbf42fe34e282ac6af888808080e242544f06515f32f080","text":"�9;�\u0007R\n���������B�4€j������BTO\u0006Q_2��"}, + {"bytes":"f08080807fefbb7c78f48fbfbfc280e282edb080592e57c280f0f464c3a93ae08080fff48fbfbfedbf","text":"�����|x􏿿€����Y.W€��dé:����􏿿��"}, + {"bytes":"e2268080806f2e5d6be282acf0","text":"�&���o.]k€�"}, + {"bytes":"fef08fbfbfeda080edb0806bf48fbfbf2ef0808080edafbfe282e0808056f18f34c317f5e2edafbf808080f09f30","text":"�����������k􏿿.�����������V�4�\u0017���������0"}, + {"bytes":"e24465d5c280e942edafbf00b9efbfbdcf69eda080f0da169dc3a9edb080f4c080dfedafbfefbfbd808080808080c1bf","text":"�De�€�B���\u0000���i�����\u0016�é�������������������"}, + {"bytes":"efbbbffeffedb080d7f0808080efbbbf645bc3a97d68c280f888808080edb080f08fbfbf8080805639c1","text":"����������d[é}h€���������������V9�"}, + {"bytes":"5e0ef09f988054edb080e89aeda080edb080bf1cc3a9efbfbd6aefbfbdf09fde","text":"^\u000e😀T�����������\u001cé�j���"}, + {"bytes":"ee65feffedb08005377e13e280f09f98f0","text":"�e�����\u00057~\u0013���"}, + {"bytes":"f48fbfbf48ed9fbf50f5c3ff6fa2bc602408edafbf43f08080808df09f98ed9fbffc","text":"􏿿H퟿P���o��`$\b���C������퟿�"} + ] +} diff --git a/crates/core/tests/pdf_json_js.rs b/crates/core/tests/pdf_json_js.rs index 194024a4..1f2273b2 100644 --- a/crates/core/tests/pdf_json_js.rs +++ b/crates/core/tests/pdf_json_js.rs @@ -1 +1,269 @@ -//! R4.2: placeholder, written by its track. +//! R4.2c: the JavaScript-path JSON steps of the attachment probe (`pdf::json_js`), case by +//! case against `scan.ts` and `JSON.parse`, plus the UTF-8 lossy parity with TextDecoder on +//! the oracle `fixtures/pdf/r4/utf8-lossy.json` (`scripts/r4-utf8-oracle.mjs`). +#![allow(clippy::disallowed_methods, clippy::unwrap_used, clippy::expect_used)] + +mod common; + +use std::borrow::Cow; + +use faunosteo_core::domain::js::{JsonReadError, parse_json}; +use faunosteo_core::pdf::json_js::{ + JsonPrescan, MAX_JSON_DEPTH, decode_lossy, is_versioned_object, neutralise_lone_surrogates, + prescan, rewrite_overflow_numbers, starts_json_object, +}; +use serde_json::Value; + +fn nested(depth: usize) -> String { + format!("{}{}", "[".repeat(depth), "]".repeat(depth)) +} + +#[test] +fn starts_json_object_takes_one_bom_and_json_whitespace() { + for accepted in [ + &b"{"[..], + b"{}", + b"\xEF\xBB\xBF{", + b" \t\r\n{", + b"\xEF\xBB\xBF \n{\"a\":1}", + ] { + assert!(starts_json_object(accepted), "{accepted:?}"); + } + let refused: [&[u8]; 9] = [ + b"", + b" ", + b"\xEF\xBB\xBF", + b"\xEF\xBB\xBF\xEF\xBB\xBF{", + b" \xEF\xBB\xBF{", + b"\xEF\xBB{", + b"\x0c{", + b"[{}]", + b"\"{\"", + ]; + for data in refused { + assert!(!starts_json_object(data), "{data:?}"); + } +} + +#[test] +fn prescan_caps_the_depth_at_64() { + assert_eq!(MAX_JSON_DEPTH, 64); + assert!(prescan(nested(64).as_bytes()).within_depth); + let deep = prescan(nested(65).as_bytes()); + assert_eq!( + deep, + JsonPrescan { + within_depth: false, + values: 65 + } + ); + let objects = format!("{}{}", "{\"a\":".repeat(65), "}".repeat(65)); + assert!(!prescan(objects.as_bytes()).within_depth); + let objects = format!("{}{}", "{\"a\":".repeat(64), "}".repeat(64)); + assert!(prescan(objects.as_bytes()).within_depth); +} + +#[test] +fn prescan_skips_strings_and_their_escapes() { + let braces = format!("{{\"a\":\"{}\"}}", "{[".repeat(100)); + assert!(prescan(braces.as_bytes()).within_depth); + let quoted = format!("{{\"a\":\"\\\"{}\"}}", "[".repeat(100)); + assert!(prescan(quoted.as_bytes()).within_depth); + // `\\` is an escaped backslash: the next quote closes the string. + let closed = format!("[\"\\\\\"{}", "[".repeat(64)); + assert!(!prescan(closed.as_bytes()).within_depth); +} + +#[test] +fn prescan_lets_the_depth_go_negative_as_the_typescript() { + let below = format!("]{}", "[".repeat(65)); + assert!(prescan(below.as_bytes()).within_depth); + let below = format!("]{}", "[".repeat(66)); + assert!(!prescan(below.as_bytes()).within_depth); +} + +#[test] +fn prescan_counts_values_keys_included() { + // { "a" [ 1 -2.5e3 true false null "x\"y{" "b" { : 11 allocations. + let text = br#"{"a":[1,-2.5e3,true,false,null,"x\"y{"],"b":{}}"#; + assert_eq!( + prescan(text), + JsonPrescan { + within_depth: true, + values: 11 + } + ); + assert_eq!(prescan(b"\xEF\xBB\xBF{ \"a\" : [ 1 , -2 ] }").values, 5); + assert_eq!(prescan(b"42").values, 1); + assert_eq!(prescan(b"").values, 0); + // Digits, signs and letters inside a token or a string are not starts. + assert_eq!(prescan(br#"[123,1e-5,"nft-1"]"#).values, 4); + // Stable: the same bytes give the same count. + assert_eq!(prescan(text), prescan(text)); +} + +#[test] +fn decode_lossy_drops_one_bom_and_replaces_invalid_utf8() { + assert!(matches!( + decode_lossy(b"{\"a\":1}"), + Cow::Borrowed("{\"a\":1}") + )); + assert_eq!(decode_lossy(b"\xEF\xBB\xBF{}"), "{}"); + assert_eq!(decode_lossy(b"\xEF\xBB\xBF\xEF\xBB\xBF{}"), "\u{feff}{}"); + assert_eq!(decode_lossy(b"a\xEF\xBB\xBFb"), "a\u{feff}b"); + assert_eq!(decode_lossy(b"\xEF\xBB{"), "\u{fffd}{"); + assert_eq!(decode_lossy(b"\xC0\x80"), "\u{fffd}\u{fffd}"); + assert_eq!(decode_lossy(b"\xED\xA0\x80"), "\u{fffd}\u{fffd}\u{fffd}"); + assert_eq!(decode_lossy(b"\xE2\x82"), "\u{fffd}"); +} + +#[test] +fn decode_lossy_matches_text_decoder_on_the_oracle() { + let path = common::r4::repo_root().join("crates/core/tests/fixtures/pdf/r4/utf8-lossy.json"); + let oracle: Value = serde_json::from_str(&std::fs::read_to_string(path).unwrap()).unwrap(); + let cases = oracle["cases"].as_array().unwrap(); + assert_eq!(cases.len(), 400); + let differences: Vec = cases + .iter() + .filter_map(|case| { + let hex = case["bytes"].as_str().unwrap(); + let expected = case["text"].as_str().unwrap(); + let bytes = hex_bytes(hex); + let actual = decode_lossy(&bytes); + (actual != expected).then(|| format!("{hex}: rust {actual:?}, js {expected:?}")) + }) + .collect(); + assert!(differences.is_empty(), "{}", differences.join("\n")); +} + +fn hex_bytes(hex: &str) -> Vec { + (0..hex.len()) + .step_by(2) + .map(|at| u8::from_str_radix(&hex[at..at + 2], 16).unwrap()) + .collect() +} + +#[test] +fn rewrite_turns_overflowing_numbers_into_null() { + let long_integer = format!("1{}", "0".repeat(399)); + let cases = [ + ("1e999", "null".to_owned()), + ("-1e999", "null".to_owned()), + ("1E400", "null".to_owned()), + ("1e+400", "null".to_owned()), + ("-0.5e310", "null".to_owned()), + ("1.7976931348623159e308", "null".to_owned()), + (long_integer.as_str(), "null".to_owned()), + ( + r#"{"a":1e999,"b":[2,-2e999]}"#, + r#"{"a":null,"b":[2,null]}"#.to_owned(), + ), + (r#"["\\",1e999]"#, r#"["\\",null]"#.to_owned()), + ("[\"é\", 1e999 ]", "[\"é\", null ]".to_owned()), + ]; + for (text, expected) in cases { + let rewritten = rewrite_overflow_numbers(text); + assert_eq!(rewritten, expected, "{text}"); + // serde_json refused the token; the rewritten text reads as JSON.stringify wrote it. + assert!( + matches!(parse_json(text), Err(JsonReadError::Syntax(_))), + "{text}" + ); + assert!(parse_json(&rewritten).is_ok(), "{text}"); + } +} + +#[test] +fn rewrite_keeps_strings_finite_numbers_and_malformed_tokens() { + let kept = [ + "1e308", + "1.7976931348623158e308", + "1e-999", + "-0", + r#"{"a":"1e999"}"#, + r#"{"a":"\"1e999"}"#, + r#"{"1e999":true}"#, + "01e999", + "1e999e", + "[1e999-]", + "-", + "", + ]; + for text in kept { + assert!( + matches!(rewrite_overflow_numbers(text), Cow::Borrowed(_)), + "{text}" + ); + } +} + +#[test] +fn neutralise_replaces_lone_surrogate_escapes_only() { + let cases = [ + (r#""\uD800""#, r#""\uFFFD""#), + (r#""\udfff x""#, r#""\uFFFD x""#), + (r#""\uDC00\uD800""#, r#""\uFFFD\uFFFD""#), + (r#""\uD800\uD800\uDC00""#, r#""\uFFFD\uD800\uDC00""#), + (r#"["\uD800","a","\uDBFF"]"#, r#"["\uFFFD","a","\uFFFD"]"#), + ]; + for (text, expected) in cases { + let neutral = neutralise_lone_surrogates(text); + assert_eq!(neutral, expected); + assert_eq!(neutral.len(), text.len()); + assert!(parse_json(&neutral).is_ok(), "{text}"); + } + for kept in [r#""\uD800\uDC00""#, r#""\\uD800""#, r#""\u0041""#, "{}"] { + assert!( + matches!(neutralise_lone_surrogates(kept), Cow::Borrowed(_)), + "{kept}" + ); + } +} + +#[test] +fn is_versioned_object_is_parse_versioned() { + let accepted = [ + r#"{"version":3}"#, + r#"{"envelopeVersion":1}"#, + r#"{"version":"3","envelopeVersion":1.5}"#, + r#"{"version":"3","version":3}"#, + r#"{"version":3,"text":"\uD800"}"#, + ]; + for text in accepted { + assert!(is_versioned_object(text), "{text}"); + } + let refused = [ + r#"{"version":"3"}"#, + r#"{"version":null}"#, + r#"{"version":3,"version":"3"}"#, + r#"{"Version":3}"#, + "[1]", + "3", + r#""version""#, + r#"{"version":3"#, + "", + ]; + for text in refused { + assert!(!is_versioned_object(text), "{text}"); + } +} + +#[test] +fn an_overflowing_version_is_null_after_the_rewrite() { + // JS: JSON.parse gave Infinity, a number; the old path then stringified it as null, + // which is what the core reads, so the probe sees null (typeof null !== 'number'). + let rewritten = rewrite_overflow_numbers(r#"{"version":1e999}"#); + assert_eq!(rewritten, r#"{"version":null}"#); + assert!(!is_versioned_object(&rewritten)); +} + +#[test] +fn the_probe_accepts_a_lone_surrogate_that_the_core_then_refuses() { + // OD-4: the probe reads the neutralised copy, the real text keeps its escape. + let text = r#"{"version":3,"animal":{"name":"\uD800"}}"#; + assert!(is_versioned_object(text)); + assert!(matches!( + parse_json(text), + Err(JsonReadError::LoneSurrogate { .. }) + )); +} diff --git a/scripts/r4-utf8-oracle.mjs b/scripts/r4-utf8-oracle.mjs new file mode 100644 index 00000000..fcb05e0b --- /dev/null +++ b/scripts/r4-utf8-oracle.mjs @@ -0,0 +1,125 @@ +// R4.2c UTF-8 lossy oracle (plan 2026-10-17 §B.1.2, LRN-059): what the +// TypeScript import path's `new TextDecoder().decode(bytes)` (non-fatal, one +// leading BOM stripped) gives for 400 byte strings of 1 to 48 bytes. The +// strings come from a seeded xorshift32 (seed 20261018) mixing random bytes +// with the edge sequences of UTF-8 decoding: truncated multi-byte sequences, +// overlongs, UTF-8-encoded surrogates, code points above U+10FFFF, F5..FF, +// lone continuation bytes, one and two leading BOMs, a BOM in the middle. +// Writes crates/core/tests/fixtures/pdf/r4/utf8-lossy.json, read by +// crates/core/tests/pdf_json_js.rs (`decode_lossy` must give the same text). +// Deterministic: rerunning rewrites the same cases (the generator field +// records the Node version that decoded them). +// Usage: node scripts/r4-utf8-oracle.mjs +import { mkdirSync, writeFileSync } from 'node:fs' +import { dirname, join, resolve } from 'node:path' +import { fileURLToPath } from 'node:url' + +const ROOT = resolve(fileURLToPath(new URL('..', import.meta.url))) +const OUT = join(ROOT, 'crates/core/tests/fixtures/pdf/r4/utf8-lossy.json') +const SEED = 20261018 +const CASES = 400 +const MAX_LENGTH = 48 + +const BOM = [0xef, 0xbb, 0xbf] +const EDGES = [ + BOM, + // overlongs + [0xc0, 0x80], + [0xc1, 0xbf], + [0xe0, 0x80, 0x80], + [0xe0, 0x9f, 0xbf], + [0xf0, 0x80, 0x80, 0x80], + [0xf0, 0x8f, 0xbf, 0xbf], + // UTF-8-encoded surrogates, alone and as a CESU-8 pair + [0xed, 0xa0, 0x80], + [0xed, 0xaf, 0xbf], + [0xed, 0xb0, 0x80], + [0xed, 0xbf, 0xbf], + [0xed, 0xa0, 0x80, 0xed, 0xb0, 0x80], + // above U+10FFFF, bytes never valid + [0xf4, 0x90, 0x80, 0x80], + [0xf7, 0xbf, 0xbf, 0xbf], + [0xf5], + [0xf8, 0x88, 0x80, 0x80, 0x80], + [0xfc, 0x84, 0x80, 0x80, 0x80, 0x80], + [0xfe], + [0xff], + [0xfe, 0xff], + // lone continuation bytes + [0x80], + [0xbf], + [0x80, 0x80, 0x80], + // truncated sequences + [0xc3], + [0xe2], + [0xe2, 0x82], + [0xf0], + [0xf0, 0x9f], + [0xf0, 0x9f, 0x98], + [0xef, 0xbb], + // valid, range boundaries included + [0xc2, 0x80], + [0xc3, 0xa9], + [0xe2, 0x82, 0xac], + [0xed, 0x9f, 0xbf], + [0xee, 0x80, 0x80], + [0xef, 0xbf, 0xbd], + [0xf0, 0x9f, 0x98, 0x80], + [0xf4, 0x8f, 0xbf, 0xbf], + [0x00], + [0x7f], +] +const ASCII = (text) => Array.from(text, (c) => c.charCodeAt(0)) +const FIXED = [ + ...EDGES, + [...BOM, ...BOM], + [...BOM, ...ASCII('{"a":1}')], + [...BOM, ...BOM, ...ASCII('{}')], + [...ASCII('a'), ...BOM, ...ASCII('b')], + [0xef, 0xbb, ...ASCII('{')], + [...BOM, 0xef, 0xbb], + [...BOM, 0x80], +] + +function xorshift32(seed) { + let state = seed >>> 0 + return (bound) => { + state ^= state << 13 + state >>>= 0 + state ^= state >>> 17 + state ^= state << 5 + state >>>= 0 + return state % bound + } +} + +function randomCase(next) { + const length = 1 + next(MAX_LENGTH) + const prefix = next(8) + const bytes = prefix === 0 ? [...BOM] : prefix === 1 ? [...BOM, ...BOM] : [] + while (bytes.length < length) { + const pick = next(4) + if (pick === 0) bytes.push(next(256)) + else if (pick === 1) bytes.push(0x20 + next(0x5f)) + else bytes.push(...EDGES[next(EDGES.length)]) + } + return bytes.slice(0, length) +} + +const next = xorshift32(SEED) +const inputs = [...FIXED] +while (inputs.length < CASES) inputs.push(randomCase(next)) +const decoder = new TextDecoder() +const cases = inputs.map((bytes) => ({ + bytes: Buffer.from(bytes).toString('hex'), + text: decoder.decode(Uint8Array.from(bytes)), +})) +const generator = `scripts/r4-utf8-oracle.mjs, Node ${process.version}` +// One case per line: the file stays readable and diffable. +const rows = cases.map((c) => ` ${JSON.stringify(c)}`).join(',\n') +const head = `{\n "generator": ${JSON.stringify(generator)},\n "cases": [\n` +const text = `${head}${rows}\n ]\n}\n` +mkdirSync(dirname(OUT), { recursive: true }) +writeFileSync(OUT, text) +const size = Buffer.byteLength(text) +console.log(`${cases.length} cases, ${size} bytes -> ${OUT}`)