281 lines
9.2 KiB
Rust
281 lines
9.2 KiB
Rust
//! R4.2c: the JavaScript-path JSON steps of the attachment probe (`pdf::json_js`), case by
|
|
//! case against `scan.ts` and `JSON.parse`, plus the UTF-8 lossy parity with TextDecoder on
|
|
//! the oracle `fixtures/pdf/r4/utf8-lossy.json` (`scripts/r4-utf8-oracle.mjs`).
|
|
#![allow(clippy::disallowed_methods, clippy::unwrap_used, clippy::expect_used)]
|
|
|
|
mod common;
|
|
|
|
use std::borrow::Cow;
|
|
|
|
use faunosteo_core::domain::js::{JsonReadError, parse_json};
|
|
use faunosteo_core::pdf::json_js::{
|
|
JsonPrescan, MAX_JSON_DEPTH, decode_lossy, is_versioned_object, neutralise_lone_surrogates,
|
|
prescan, rewrite_overflow_numbers, starts_json_object,
|
|
};
|
|
use serde_json::Value;
|
|
|
|
fn nested(depth: usize) -> String {
|
|
format!("{}{}", "[".repeat(depth), "]".repeat(depth))
|
|
}
|
|
|
|
#[test]
|
|
fn starts_json_object_takes_one_bom_and_json_whitespace() {
|
|
for accepted in [
|
|
&b"{"[..],
|
|
b"{}",
|
|
b"\xEF\xBB\xBF{",
|
|
b" \t\r\n{",
|
|
b"\xEF\xBB\xBF \n{\"a\":1}",
|
|
] {
|
|
assert!(starts_json_object(accepted), "{accepted:?}");
|
|
}
|
|
let refused: [&[u8]; 9] = [
|
|
b"",
|
|
b" ",
|
|
b"\xEF\xBB\xBF",
|
|
b"\xEF\xBB\xBF\xEF\xBB\xBF{",
|
|
b" \xEF\xBB\xBF{",
|
|
b"\xEF\xBB{",
|
|
b"\x0c{",
|
|
b"[{}]",
|
|
b"\"{\"",
|
|
];
|
|
for data in refused {
|
|
assert!(!starts_json_object(data), "{data:?}");
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn prescan_caps_the_depth_at_64() {
|
|
assert_eq!(MAX_JSON_DEPTH, 64);
|
|
assert!(prescan(nested(64).as_bytes()).within_depth);
|
|
let deep = prescan(nested(65).as_bytes());
|
|
assert_eq!(
|
|
deep,
|
|
JsonPrescan {
|
|
within_depth: false,
|
|
values: 65
|
|
}
|
|
);
|
|
let objects = format!("{}{}", "{\"a\":".repeat(65), "}".repeat(65));
|
|
assert!(!prescan(objects.as_bytes()).within_depth);
|
|
let objects = format!("{}{}", "{\"a\":".repeat(64), "}".repeat(64));
|
|
assert!(prescan(objects.as_bytes()).within_depth);
|
|
}
|
|
|
|
#[test]
|
|
fn prescan_keeps_counting_values_past_the_depth_cap() {
|
|
// A text nested past the cap must still report its size: the rescue copy applies the
|
|
// value budget alone (R4.2 security gate).
|
|
let deep = format!("{}{}{}", "[".repeat(65), "1,".repeat(1000), "1]".repeat(65));
|
|
let scan = prescan(deep.as_bytes());
|
|
assert!(!scan.within_depth);
|
|
// 65 opens, 1,000 numbers, then one number after each `]`.
|
|
assert_eq!(scan.values, 65 + 1000 + 65);
|
|
}
|
|
|
|
#[test]
|
|
fn prescan_skips_strings_and_their_escapes() {
|
|
let braces = format!("{{\"a\":\"{}\"}}", "{[".repeat(100));
|
|
assert!(prescan(braces.as_bytes()).within_depth);
|
|
let quoted = format!("{{\"a\":\"\\\"{}\"}}", "[".repeat(100));
|
|
assert!(prescan(quoted.as_bytes()).within_depth);
|
|
// `\\` is an escaped backslash: the next quote closes the string.
|
|
let closed = format!("[\"\\\\\"{}", "[".repeat(64));
|
|
assert!(!prescan(closed.as_bytes()).within_depth);
|
|
}
|
|
|
|
#[test]
|
|
fn prescan_lets_the_depth_go_negative_as_the_typescript() {
|
|
let below = format!("]{}", "[".repeat(65));
|
|
assert!(prescan(below.as_bytes()).within_depth);
|
|
let below = format!("]{}", "[".repeat(66));
|
|
assert!(!prescan(below.as_bytes()).within_depth);
|
|
}
|
|
|
|
#[test]
|
|
fn prescan_counts_values_keys_included() {
|
|
// { "a" [ 1 -2.5e3 true false null "x\"y{" "b" { : 11 allocations.
|
|
let text = br#"{"a":[1,-2.5e3,true,false,null,"x\"y{"],"b":{}}"#;
|
|
assert_eq!(
|
|
prescan(text),
|
|
JsonPrescan {
|
|
within_depth: true,
|
|
values: 11
|
|
}
|
|
);
|
|
assert_eq!(prescan(b"\xEF\xBB\xBF{ \"a\" : [ 1 , -2 ] }").values, 5);
|
|
assert_eq!(prescan(b"42").values, 1);
|
|
assert_eq!(prescan(b"").values, 0);
|
|
// Digits, signs and letters inside a token or a string are not starts.
|
|
assert_eq!(prescan(br#"[123,1e-5,"nft-1"]"#).values, 4);
|
|
// Stable: the same bytes give the same count.
|
|
assert_eq!(prescan(text), prescan(text));
|
|
}
|
|
|
|
#[test]
|
|
fn decode_lossy_drops_one_bom_and_replaces_invalid_utf8() {
|
|
assert!(matches!(
|
|
decode_lossy(b"{\"a\":1}"),
|
|
Cow::Borrowed("{\"a\":1}")
|
|
));
|
|
assert_eq!(decode_lossy(b"\xEF\xBB\xBF{}"), "{}");
|
|
assert_eq!(decode_lossy(b"\xEF\xBB\xBF\xEF\xBB\xBF{}"), "\u{feff}{}");
|
|
assert_eq!(decode_lossy(b"a\xEF\xBB\xBFb"), "a\u{feff}b");
|
|
assert_eq!(decode_lossy(b"\xEF\xBB{"), "\u{fffd}{");
|
|
assert_eq!(decode_lossy(b"\xC0\x80"), "\u{fffd}\u{fffd}");
|
|
assert_eq!(decode_lossy(b"\xED\xA0\x80"), "\u{fffd}\u{fffd}\u{fffd}");
|
|
assert_eq!(decode_lossy(b"\xE2\x82"), "\u{fffd}");
|
|
}
|
|
|
|
#[test]
|
|
fn decode_lossy_matches_text_decoder_on_the_oracle() {
|
|
let path = common::r4::repo_root().join("crates/core/tests/fixtures/pdf/r4/utf8-lossy.json");
|
|
let oracle: Value = serde_json::from_str(&std::fs::read_to_string(path).unwrap()).unwrap();
|
|
let cases = oracle["cases"].as_array().unwrap();
|
|
assert_eq!(cases.len(), 400);
|
|
let differences: Vec<String> = cases
|
|
.iter()
|
|
.filter_map(|case| {
|
|
let hex = case["bytes"].as_str().unwrap();
|
|
let expected = case["text"].as_str().unwrap();
|
|
let bytes = hex_bytes(hex);
|
|
let actual = decode_lossy(&bytes);
|
|
(actual != expected).then(|| format!("{hex}: rust {actual:?}, js {expected:?}"))
|
|
})
|
|
.collect();
|
|
assert!(differences.is_empty(), "{}", differences.join("\n"));
|
|
}
|
|
|
|
fn hex_bytes(hex: &str) -> Vec<u8> {
|
|
(0..hex.len())
|
|
.step_by(2)
|
|
.map(|at| u8::from_str_radix(&hex[at..at + 2], 16).unwrap())
|
|
.collect()
|
|
}
|
|
|
|
#[test]
|
|
fn rewrite_turns_overflowing_numbers_into_null() {
|
|
let long_integer = format!("1{}", "0".repeat(399));
|
|
let cases = [
|
|
("1e999", "null".to_owned()),
|
|
("-1e999", "null".to_owned()),
|
|
("1E400", "null".to_owned()),
|
|
("1e+400", "null".to_owned()),
|
|
("-0.5e310", "null".to_owned()),
|
|
("1.7976931348623159e308", "null".to_owned()),
|
|
(long_integer.as_str(), "null".to_owned()),
|
|
(
|
|
r#"{"a":1e999,"b":[2,-2e999]}"#,
|
|
r#"{"a":null,"b":[2,null]}"#.to_owned(),
|
|
),
|
|
(r#"["\\",1e999]"#, r#"["\\",null]"#.to_owned()),
|
|
("[\"é\", 1e999 ]", "[\"é\", null ]".to_owned()),
|
|
];
|
|
for (text, expected) in cases {
|
|
let rewritten = rewrite_overflow_numbers(text);
|
|
assert_eq!(rewritten, expected, "{text}");
|
|
// serde_json refused the token; the rewritten text reads as JSON.stringify wrote it.
|
|
assert!(
|
|
matches!(parse_json(text), Err(JsonReadError::Syntax(_))),
|
|
"{text}"
|
|
);
|
|
assert!(parse_json(&rewritten).is_ok(), "{text}");
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn rewrite_keeps_strings_finite_numbers_and_malformed_tokens() {
|
|
let kept = [
|
|
"1e308",
|
|
"1.7976931348623158e308",
|
|
"1e-999",
|
|
"-0",
|
|
r#"{"a":"1e999"}"#,
|
|
r#"{"a":"\"1e999"}"#,
|
|
r#"{"1e999":true}"#,
|
|
"01e999",
|
|
"1e999e",
|
|
"[1e999-]",
|
|
"-",
|
|
"",
|
|
];
|
|
for text in kept {
|
|
assert!(
|
|
matches!(rewrite_overflow_numbers(text), Cow::Borrowed(_)),
|
|
"{text}"
|
|
);
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn neutralise_replaces_lone_surrogate_escapes_only() {
|
|
let cases = [
|
|
(r#""\uD800""#, r#""\uFFFD""#),
|
|
(r#""\udfff x""#, r#""\uFFFD x""#),
|
|
(r#""\uDC00\uD800""#, r#""\uFFFD\uFFFD""#),
|
|
(r#""\uD800\uD800\uDC00""#, r#""\uFFFD\uD800\uDC00""#),
|
|
(r#"["\uD800","a","\uDBFF"]"#, r#"["\uFFFD","a","\uFFFD"]"#),
|
|
];
|
|
for (text, expected) in cases {
|
|
let neutral = neutralise_lone_surrogates(text);
|
|
assert_eq!(neutral, expected);
|
|
assert_eq!(neutral.len(), text.len());
|
|
assert!(parse_json(&neutral).is_ok(), "{text}");
|
|
}
|
|
for kept in [r#""\uD800\uDC00""#, r#""\\uD800""#, r#""\u0041""#, "{}"] {
|
|
assert!(
|
|
matches!(neutralise_lone_surrogates(kept), Cow::Borrowed(_)),
|
|
"{kept}"
|
|
);
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn is_versioned_object_is_parse_versioned() {
|
|
let accepted = [
|
|
r#"{"version":3}"#,
|
|
r#"{"envelopeVersion":1}"#,
|
|
r#"{"version":"3","envelopeVersion":1.5}"#,
|
|
r#"{"version":"3","version":3}"#,
|
|
r#"{"version":3,"text":"\uD800"}"#,
|
|
];
|
|
for text in accepted {
|
|
assert!(is_versioned_object(text), "{text}");
|
|
}
|
|
let refused = [
|
|
r#"{"version":"3"}"#,
|
|
r#"{"version":null}"#,
|
|
r#"{"version":3,"version":"3"}"#,
|
|
r#"{"Version":3}"#,
|
|
"[1]",
|
|
"3",
|
|
r#""version""#,
|
|
r#"{"version":3"#,
|
|
"",
|
|
];
|
|
for text in refused {
|
|
assert!(!is_versioned_object(text), "{text}");
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn an_overflowing_version_is_null_after_the_rewrite() {
|
|
// JS: JSON.parse gave Infinity, a number; the old path then stringified it as null,
|
|
// which is what the core reads, so the probe sees null (typeof null !== 'number').
|
|
let rewritten = rewrite_overflow_numbers(r#"{"version":1e999}"#);
|
|
assert_eq!(rewritten, r#"{"version":null}"#);
|
|
assert!(!is_versioned_object(&rewritten));
|
|
}
|
|
|
|
#[test]
|
|
fn the_probe_accepts_a_lone_surrogate_that_the_core_then_refuses() {
|
|
// OD-4: the probe reads the neutralised copy, the real text keeps its escape.
|
|
let text = r#"{"version":3,"animal":{"name":"\uD800"}}"#;
|
|
assert!(is_versioned_object(text));
|
|
assert!(matches!(
|
|
parse_json(text),
|
|
Err(JsonReadError::LoneSurrogate { .. })
|
|
));
|
|
}
|