fix(domain): refuse lone UTF-16 surrogates instead of replacing them (L1)

serde_json already refuses a lone surrogate escape; the U+FFFD pre-pass
is gone and parse_json classifies the refusal (strict escape scan, on
failure only) so callers name it: closed kind lone_surrogate from
operation inputs, sheet.check, draft.restore and archive.import.
Oracles untouched: the two L1 cases and the Zod verdict now assert that
TypeScript kept the surrogate and Rust refuses the text.
This commit is contained in:
bchanot committed 2026-10-14 21:49:28 +02:00
1 parent 36316cf67f
commit 20d03cf7fd
9 files changed
+275 -75

No files matched your search

+94
View File
@@ -882,3 +882,97 @@ fn import_reads_and_rebases_or_names_the_failure_as_the_engine_does() {
);
}
}
// ---------------------------------------------------------------------------
// L1: a lone UTF-16 surrogate escape refuses the text whole, never replaced
/// `value` as JSON text with `MARK` swapped for the raw escape `escape` (serde_json cannot
/// write a lone surrogate itself; `JSON.stringify` writes exactly this).
fn text_with_escape(value: &Value, escape: &str) -> String {
serde_json::to_string(value)
.unwrap()
.replace("MARK", escape)
}
#[test]
fn a_draft_with_a_lone_surrogate_is_refused_as_content_never_rewritten() {
let f = EnvelopeFixture::new();
let report = with(report_value(&f.report), "conclusion", json!("aMARKb"));
for escape in ["\\ud800", "\\udfff", "\\ud83d\\ud83d"] {
let draft = text_with_escape(&report, escape);
assert!(
matches!(
restore_draft(&f.draft, &draft, None),
Err(RestoreFailure::LoneSurrogate(_))
),
"{escape}"
);
}
let v2 = common::repo_file("packages/domain/tests/fixtures/report-v2.full.json").replace(
"\"comment\": \"Réagit à la palpation du garrot.\"",
"\"comment\": \"Réagit \\udc00\"",
);
assert!(v2.contains("\\udc00"));
assert!(matches!(
restore_draft(&f.draft, &v2, None),
Err(RestoreFailure::LoneSurrogate(_))
));
}
#[test]
fn an_archive_with_a_lone_surrogate_is_set_aside_while_the_draft_restores() {
let f = EnvelopeFixture::new();
let draft = serde_json::to_string(&report_value(&f.report)).unwrap();
let mut envelope = f.envelope();
envelope["branding"]["poweredBy"]["fallbackText"] = json!("MARK");
let readable = serde_json::to_string(&envelope)
.unwrap()
.replace("MARK", "zen");
assert!(
!restore_draft(&f.draft, &draft, Some(&readable))
.unwrap()
.archive_rejected
);
let archive = text_with_escape(&envelope, "\\ud800");
let restored = restore_draft(&f.draft, &draft, Some(&archive)).unwrap();
assert!(restored.archive.is_none() && restored.archive_rejected);
assert_eq!(restored.rebase.report, f.report);
}
#[test]
fn an_attachment_with_a_lone_surrogate_is_refused_with_its_own_kind() {
let f = EnvelopeFixture::new();
let mut envelope = f.envelope();
envelope["report"]["conclusion"] = json!("MARK");
let text = text_with_escape(&envelope, "\\ud800");
assert!(matches!(
import_archive(&f.draft, &text),
Err(ImportFailure::LoneSurrogate(_))
));
let bare = text_with_escape(
&with(report_value(&f.report), "conclusion", json!("MARK")),
"\\udbff",
);
assert!(matches!(
import_archive(&f.draft, &bare),
Err(ImportFailure::LoneSurrogate(_))
));
}
#[test]
fn normal_text_with_pairs_and_escaped_backslashes_reads_unchanged() {
let f = EnvelopeFixture::new();
let kept = "😀 \\ud800 é \u{FFFD}";
let report = with(report_value(&f.report), "conclusion", json!(kept));
let text = serde_json::to_string(&report).unwrap();
assert!(text.contains("\\\\ud800"));
let restored = restore_draft(&f.draft, &text, None).unwrap();
assert_eq!(restored.rebase.report.conclusion, kept);
let mut envelope = f.envelope();
envelope["report"] = report.clone();
let paired = serde_json::to_string(&envelope)
.unwrap()
.replace('😀', "\\ud83d\\ude00");
let imported = import_archive(&f.draft, &paired).unwrap();
assert_eq!(imported.rebase.report.conclusion, kept);
}
+1 -2
View File
@@ -81,8 +81,7 @@ pub fn draft_map() -> ReportMigrationMap {
/// The horse slice of `packages/domain/tests/fixtures/horse-slice.ts`, as Zod parsed it (the
/// characterisation oracle carries it, written by the TypeScript generator).
pub fn horse_slice() -> Value {
json_file("crates/core/tests/oracles/characterisation.json")["sheets"]["slice"]["document"]
.clone()
super::common::oracle_file("characterisation.json").1["sheets"]["slice"]["document"].clone()
}
pub fn snake_ref(sheet_id: &str, version_id: Option<&str>, sha: &str) -> Value {
+30 -13
View File
@@ -10,7 +10,7 @@ use faunosteo_core::domain::archive::{
Branding, PdfLabelSet, ReportArchiveEnvelope, SheetSnapshot, assert_snapshot,
build_sheet_snapshot, read_archive_document,
};
use faunosteo_core::domain::js::parse_json;
use faunosteo_core::domain::js::{JsonReadError, parse_json};
use faunosteo_core::domain::report::v2::ReportV2;
use faunosteo_core::domain::report::{
ReadFailure, Report, ReportMigrationMap, read_report_document, rebase_report_on_sheet,
@@ -21,9 +21,15 @@ use serde::Serialize;
use serde_json::{Map, Value, json};
fn oracle(name: &str) -> (String, Value) {
let text = common::repo_file(&format!("crates/core/tests/oracles/{name}"));
let value = parse_json(&text).unwrap();
(text, value)
common::oracle_file(name)
}
/// A case input as the core reads it; a lone surrogate refuses it whole (L1).
fn read_input(text: &str) -> Result<Value, Value> {
parse_json(text).map_err(|error| match error {
JsonReadError::LoneSurrogate { .. } => json!({ "kind": "lone_surrogate" }),
JsonReadError::Syntax(error) => panic!("unreadable oracle input: {error}"),
})
}
/// Object keys sorted at every level: equality whatever the declared field order.
@@ -77,7 +83,7 @@ fn read_error(failure: &ReadFailure) -> Value {
/// What Rust returns for a case: `Ok(output)` or `Err(error)` in the oracle's shapes.
fn run(context: &Context, case: &Value) -> Result<Value, Value> {
let input = parse_json(case["input"].as_str().unwrap()).unwrap();
let input = read_input(case["input"].as_str().unwrap())?;
match case["fn"].as_str().unwrap() {
"readReport" => read_report_document(&input, &context.map(case))
.map(|o| serde_json::to_value(o).unwrap())
@@ -143,7 +149,7 @@ fn cases(with_rules: bool) -> (Context, Vec<Value>) {
.as_array()
.unwrap()
.iter()
.filter(|c| c.get("needsRules").is_some() == with_rules)
.filter(|c| c.get("needsRules").is_some() == with_rules && c.get("l1").is_none())
.cloned()
.collect();
(context, list)
@@ -167,13 +173,15 @@ fn rust_does_what_the_typescript_did_with_the_integrity_rules() {
assert!(failures.is_empty(), "{}", failures.join("\n"));
}
/// Decision L1: where `JSON.parse` keeps a lone surrogate (the TypeScript output still
/// carries the `\ud800` escape), Rust reads U+FFFD; nothing else differs.
/// Decision L1 (owner, 2026-10-14): where `JSON.parse` kept a lone surrogate (the frozen
/// TypeScript output still carries the `\\ud800` escape), Rust refuses the input whole with
/// `lone_surrogate` instead of altering it; no output, no U+FFFD.
#[test]
fn lone_surrogates_are_the_one_declared_difference() {
fn lone_surrogates_are_refused_never_replaced() {
let (raw, value) = oracle("characterisation.json");
let backslash = char::from(92);
let lone_escape = format!("{backslash}ud800");
let context = cases(false).0;
let flagged: Vec<&Value> = value["cases"]
.as_array()
.unwrap()
@@ -189,9 +197,11 @@ fn lone_surrogates_are_the_one_declared_difference() {
output.contains(&lone_escape),
"{name}: TS output keeps the lone surrogate"
);
let context = cases(false).0;
let rust = pretty(&run(&context, case).unwrap());
assert!(rust.contains('\u{FFFD}'), "{name}: Rust reads U+FFFD");
assert_eq!(
run(&context, case),
Err(json!({ "kind": "lone_surrogate" })),
"{name}: Rust refuses the text"
);
}
}
@@ -215,7 +225,14 @@ fn the_contracts_accept_and_refuse_what_zod_does() {
let (_, list) = oracle("verdicts.json");
let list = list.as_array().unwrap();
assert!(list.len() > 500, "{} verdicts", list.len());
let failures: Vec<String> = list
// L1: the one input Zod accepted with a lone surrogate is refused by the reader instead.
let (lone, readable): (Vec<&Value>, Vec<&Value>) = list
.iter()
.partition(|v| read_input(v["input"].as_str().unwrap()).is_err());
assert_eq!(lone.len(), 1);
assert_eq!(lone[0]["name"], "lone surrogate in a comment");
assert_eq!(lone[0]["accepted"], true);
let failures: Vec<String> = readable
.iter()
.filter_map(|v| {
let schema = v["schema"].as_str().unwrap();
+20
View File
@@ -6,6 +6,7 @@
clippy::expect_used
)]
use faunosteo_core::domain::js::{find_lone_surrogate, parse_json};
use serde_json::Value;
/// A repository file, by path from the repository root.
@@ -14,6 +15,25 @@ pub fn repo_file(relative: &str) -> String {
std::fs::read_to_string(&path).unwrap_or_else(|error| panic!("{path}: {error}"))
}
/// A frozen oracle file (`crates/core/tests/oracles/<name>`): its text and its value. The
/// TypeScript outputs it freezes keep the lone surrogate escapes `JSON.parse` kept (the L1
/// cases), which the core refuses to read; each one is read here as literal text (`\\uD800`
/// spelt out), never replaced, so the raw text stays the evidence of what TypeScript did.
pub fn oracle_file(name: &str) -> (String, Value) {
let text = repo_file(&format!("crates/core/tests/oracles/{name}"));
let mut spelt = String::with_capacity(text.len());
let mut rest = text.as_str();
while let Some(at) = find_lone_surrogate(rest) {
spelt.push_str(&rest[..at]);
spelt.push('\\');
spelt.push_str(&rest[at..at + 6]);
rest = &rest[at + 6..];
}
spelt.push_str(rest);
let value = parse_json(&spelt).unwrap();
(text, value)
}
/// A Zod oracle (`crates/core/tests/oracles/<name>`): its source file and the Zod output in
/// the JavaScript pretty form.
pub fn zod_oracle(name: &str) -> (String, String) {