//! R4.2c: the JavaScript-path JSON steps of the attachment probe (`pdf::json_js`), case by //! case against `scan.ts` and `JSON.parse`, plus the UTF-8 lossy parity with TextDecoder on //! the oracle `fixtures/pdf/r4/utf8-lossy.json` (`scripts/r4-utf8-oracle.mjs`). #![allow(clippy::disallowed_methods, clippy::unwrap_used, clippy::expect_used)] mod common; use std::borrow::Cow; use faunosteo_core::domain::js::{JsonReadError, parse_json}; use faunosteo_core::pdf::json_js::{ JsonPrescan, MAX_JSON_DEPTH, decode_lossy, is_versioned_object, neutralise_lone_surrogates, prescan, rewrite_overflow_numbers, starts_json_object, }; use serde_json::Value; fn nested(depth: usize) -> String { format!("{}{}", "[".repeat(depth), "]".repeat(depth)) } #[test] fn starts_json_object_takes_one_bom_and_json_whitespace() { for accepted in [ &b"{"[..], b"{}", b"\xEF\xBB\xBF{", b" \t\r\n{", b"\xEF\xBB\xBF \n{\"a\":1}", ] { assert!(starts_json_object(accepted), "{accepted:?}"); } let refused: [&[u8]; 9] = [ b"", b" ", b"\xEF\xBB\xBF", b"\xEF\xBB\xBF\xEF\xBB\xBF{", b" \xEF\xBB\xBF{", b"\xEF\xBB{", b"\x0c{", b"[{}]", b"\"{\"", ]; for data in refused { assert!(!starts_json_object(data), "{data:?}"); } } #[test] fn prescan_caps_the_depth_at_64() { assert_eq!(MAX_JSON_DEPTH, 64); assert!(prescan(nested(64).as_bytes()).within_depth); let deep = prescan(nested(65).as_bytes()); assert_eq!( deep, JsonPrescan { within_depth: false, values: 65 } ); let objects = format!("{}{}", "{\"a\":".repeat(65), "}".repeat(65)); assert!(!prescan(objects.as_bytes()).within_depth); let objects = format!("{}{}", "{\"a\":".repeat(64), "}".repeat(64)); assert!(prescan(objects.as_bytes()).within_depth); } #[test] fn prescan_keeps_counting_values_past_the_depth_cap() { // A text nested past the cap must still report its size: the rescue copy applies the // value budget alone (R4.2 security gate). let deep = format!("{}{}{}", "[".repeat(65), "1,".repeat(1000), "1]".repeat(65)); let scan = prescan(deep.as_bytes()); assert!(!scan.within_depth); // 65 opens, 1,000 numbers, then one number after each `]`. assert_eq!(scan.values, 65 + 1000 + 65); } #[test] fn prescan_skips_strings_and_their_escapes() { let braces = format!("{{\"a\":\"{}\"}}", "{[".repeat(100)); assert!(prescan(braces.as_bytes()).within_depth); let quoted = format!("{{\"a\":\"\\\"{}\"}}", "[".repeat(100)); assert!(prescan(quoted.as_bytes()).within_depth); // `\\` is an escaped backslash: the next quote closes the string. let closed = format!("[\"\\\\\"{}", "[".repeat(64)); assert!(!prescan(closed.as_bytes()).within_depth); } #[test] fn prescan_lets_the_depth_go_negative_as_the_typescript() { let below = format!("]{}", "[".repeat(65)); assert!(prescan(below.as_bytes()).within_depth); let below = format!("]{}", "[".repeat(66)); assert!(!prescan(below.as_bytes()).within_depth); } #[test] fn prescan_counts_values_keys_included() { // { "a" [ 1 -2.5e3 true false null "x\"y{" "b" { : 11 allocations. let text = br#"{"a":[1,-2.5e3,true,false,null,"x\"y{"],"b":{}}"#; assert_eq!( prescan(text), JsonPrescan { within_depth: true, values: 11 } ); assert_eq!(prescan(b"\xEF\xBB\xBF{ \"a\" : [ 1 , -2 ] }").values, 5); assert_eq!(prescan(b"42").values, 1); assert_eq!(prescan(b"").values, 0); // Digits, signs and letters inside a token or a string are not starts. assert_eq!(prescan(br#"[123,1e-5,"nft-1"]"#).values, 4); // Stable: the same bytes give the same count. assert_eq!(prescan(text), prescan(text)); } #[test] fn decode_lossy_drops_one_bom_and_replaces_invalid_utf8() { assert!(matches!( decode_lossy(b"{\"a\":1}"), Cow::Borrowed("{\"a\":1}") )); assert_eq!(decode_lossy(b"\xEF\xBB\xBF{}"), "{}"); assert_eq!(decode_lossy(b"\xEF\xBB\xBF\xEF\xBB\xBF{}"), "\u{feff}{}"); assert_eq!(decode_lossy(b"a\xEF\xBB\xBFb"), "a\u{feff}b"); assert_eq!(decode_lossy(b"\xEF\xBB{"), "\u{fffd}{"); assert_eq!(decode_lossy(b"\xC0\x80"), "\u{fffd}\u{fffd}"); assert_eq!(decode_lossy(b"\xED\xA0\x80"), "\u{fffd}\u{fffd}\u{fffd}"); assert_eq!(decode_lossy(b"\xE2\x82"), "\u{fffd}"); } #[test] fn decode_lossy_matches_text_decoder_on_the_oracle() { let path = common::r4::repo_root().join("crates/core/tests/fixtures/pdf/r4/utf8-lossy.json"); let oracle: Value = serde_json::from_str(&std::fs::read_to_string(path).unwrap()).unwrap(); let cases = oracle["cases"].as_array().unwrap(); assert_eq!(cases.len(), 400); let differences: Vec = cases .iter() .filter_map(|case| { let hex = case["bytes"].as_str().unwrap(); let expected = case["text"].as_str().unwrap(); let bytes = hex_bytes(hex); let actual = decode_lossy(&bytes); (actual != expected).then(|| format!("{hex}: rust {actual:?}, js {expected:?}")) }) .collect(); assert!(differences.is_empty(), "{}", differences.join("\n")); } fn hex_bytes(hex: &str) -> Vec { (0..hex.len()) .step_by(2) .map(|at| u8::from_str_radix(&hex[at..at + 2], 16).unwrap()) .collect() } #[test] fn rewrite_turns_overflowing_numbers_into_null() { let long_integer = format!("1{}", "0".repeat(399)); let cases = [ ("1e999", "null".to_owned()), ("-1e999", "null".to_owned()), ("1E400", "null".to_owned()), ("1e+400", "null".to_owned()), ("-0.5e310", "null".to_owned()), ("1.7976931348623159e308", "null".to_owned()), (long_integer.as_str(), "null".to_owned()), ( r#"{"a":1e999,"b":[2,-2e999]}"#, r#"{"a":null,"b":[2,null]}"#.to_owned(), ), (r#"["\\",1e999]"#, r#"["\\",null]"#.to_owned()), ("[\"é\", 1e999 ]", "[\"é\", null ]".to_owned()), ]; for (text, expected) in cases { let rewritten = rewrite_overflow_numbers(text); assert_eq!(rewritten, expected, "{text}"); // serde_json refused the token; the rewritten text reads as JSON.stringify wrote it. assert!( matches!(parse_json(text), Err(JsonReadError::Syntax(_))), "{text}" ); assert!(parse_json(&rewritten).is_ok(), "{text}"); } } #[test] fn rewrite_keeps_strings_finite_numbers_and_malformed_tokens() { let kept = [ "1e308", "1.7976931348623158e308", "1e-999", "-0", r#"{"a":"1e999"}"#, r#"{"a":"\"1e999"}"#, r#"{"1e999":true}"#, "01e999", "1e999e", "[1e999-]", "-", "", ]; for text in kept { assert!( matches!(rewrite_overflow_numbers(text), Cow::Borrowed(_)), "{text}" ); } } #[test] fn neutralise_replaces_lone_surrogate_escapes_only() { let cases = [ (r#""\uD800""#, r#""\uFFFD""#), (r#""\udfff x""#, r#""\uFFFD x""#), (r#""\uDC00\uD800""#, r#""\uFFFD\uFFFD""#), (r#""\uD800\uD800\uDC00""#, r#""\uFFFD\uD800\uDC00""#), (r#"["\uD800","a","\uDBFF"]"#, r#"["\uFFFD","a","\uFFFD"]"#), ]; for (text, expected) in cases { let neutral = neutralise_lone_surrogates(text); assert_eq!(neutral, expected); assert_eq!(neutral.len(), text.len()); assert!(parse_json(&neutral).is_ok(), "{text}"); } for kept in [r#""\uD800\uDC00""#, r#""\\uD800""#, r#""\u0041""#, "{}"] { assert!( matches!(neutralise_lone_surrogates(kept), Cow::Borrowed(_)), "{kept}" ); } } #[test] fn is_versioned_object_is_parse_versioned() { let accepted = [ r#"{"version":3}"#, r#"{"envelopeVersion":1}"#, r#"{"version":"3","envelopeVersion":1.5}"#, r#"{"version":"3","version":3}"#, r#"{"version":3,"text":"\uD800"}"#, ]; for text in accepted { assert!(is_versioned_object(text), "{text}"); } let refused = [ r#"{"version":"3"}"#, r#"{"version":null}"#, r#"{"version":3,"version":"3"}"#, r#"{"Version":3}"#, "[1]", "3", r#""version""#, r#"{"version":3"#, "", ]; for text in refused { assert!(!is_versioned_object(text), "{text}"); } } #[test] fn an_overflowing_version_is_null_after_the_rewrite() { // JS: JSON.parse gave Infinity, a number; the old path then stringified it as null, // which is what the core reads, so the probe sees null (typeof null !== 'number'). let rewritten = rewrite_overflow_numbers(r#"{"version":1e999}"#); assert_eq!(rewritten, r#"{"version":null}"#); assert!(!is_versioned_object(&rewritten)); } #[test] fn the_probe_accepts_a_lone_surrogate_that_the_core_then_refuses() { // OD-4: the probe reads the neutralised copy, the real text keeps its escape. let text = r#"{"version":3,"animal":{"name":"\uD800"}}"#; assert!(is_versioned_object(text)); assert!(matches!( parse_json(text), Err(JsonReadError::LoneSurrogate { .. }) )); }