fix(domain): refuse lone UTF-16 surrogates instead of replacing them (L1)

serde_json already refuses a lone surrogate escape; the U+FFFD pre-pass
is gone and parse_json classifies the refusal (strict escape scan, on
failure only) so callers name it: closed kind lone_surrogate from
operation inputs, sheet.check, draft.restore and archive.import.
Oracles untouched: the two L1 cases and the Zod verdict now assert that
TypeScript kept the surrogate and Rust refuses the text.
This commit is contained in:
bchanot committed 2026-10-14 21:49:28 +02:00
1 parent 36316cf67f
commit 20d03cf7fd
9 files changed
+275 -75

No files matched your search

+88 -50
View File
@@ -1,31 +1,55 @@
//! JavaScript and Zod semantics the ported contracts depend on (R2 plan P1, P2).
//!
//! The TypeScript domain reads JSON with `JSON.parse` and validates it with Zod 3: lone
//! surrogate escapes are kept, a duplicate key keeps its last value at its first position,
//! `.int()` accepts any number with an integer value (`2.0`, `1e20`), string lengths count
//! UTF-16 code units, and the patterns are JavaScript regular expressions whose `$` only
//! matches at the end of the input. The helpers below reproduce exactly that, so a document
//! the old code accepted is accepted here and prints the same.
//! The TypeScript domain reads JSON with `JSON.parse` and validates it with Zod 3: a duplicate
//! key keeps its last value at its first position, `.int()` accepts any number with an
//! integer value (`2.0`, `1e20`), string lengths count UTF-16 code units, and the patterns are
//! JavaScript regular expressions whose `$` only matches at the end of the input. The helpers
//! below reproduce exactly that, so a document the old code accepted is accepted here and
//! prints the same. One declared exception: a lone surrogate escape, which `JSON.parse` keeps
//! and a Rust string cannot hold, refuses the text instead of being altered (L1).
use std::borrow::Cow;
use std::fmt;
use serde::de::Error as _;
use serde::{Deserialize, Deserializer, Serialize, Serializer};
use serde_json::Value;
/// Reads JSON text as `JSON.parse` does, as far as a Rust string allows: an escape of a
/// lone UTF-16 surrogate (which `JSON.parse` keeps and Rust strings cannot hold) becomes
/// U+FFFD instead of refusing the whole document (leniency decision L1).
pub fn parse_json(text: &str) -> Result<Value, serde_json::Error> {
serde_json::from_str(&replace_lone_surrogates(text))
/// Why a JSON text cannot be read.
#[derive(Debug)]
pub enum JsonReadError {
/// The text escapes a lone UTF-16 surrogate (`\uD800`–`\uDFFF` outside a high + low
/// pair). `JSON.parse` keeps it; reading it here would replace it, so the whole text is
/// refused and the caller keeps it verbatim (L1: user data is never altered silently).
LoneSurrogate { offset: usize },
/// Not JSON.
Syntax(serde_json::Error),
}
/// Rewrites `\uD800`–`\uDFFF` escapes that are not a high + low pair to `�`.
/// Escapes are consumed left to right, so `\\u…` (an escaped backslash) is left alone.
pub fn replace_lone_surrogates(text: &str) -> Cow<'_, str> {
impl fmt::Display for JsonReadError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::LoneSurrogate { offset } => {
write!(f, "lone UTF-16 surrogate escape at byte {offset}")
}
Self::Syntax(error) => error.fmt(f),
}
}
}
/// Reads JSON text as `JSON.parse` does, as far as a Rust string allows. serde_json refuses
/// a lone surrogate escape like any other error; the refusal is then classified so callers
/// can name it (the scan runs on failures only).
pub fn parse_json(text: &str) -> Result<Value, JsonReadError> {
serde_json::from_str(text).map_err(|error| match find_lone_surrogate(text) {
Some(offset) => JsonReadError::LoneSurrogate { offset },
None => JsonReadError::Syntax(error),
})
}
/// Byte offset of the first `\uD800`–`\uDFFF` escape that is not a high + low pair.
/// Escapes are consumed left to right, so `\\u…` (an escaped backslash) is skipped.
pub fn find_lone_surrogate(text: &str) -> Option<usize> {
let bytes = text.as_bytes();
let mut out: Option<String> = None;
let mut copied = 0;
let mut i = 0;
while i < bytes.len() {
if bytes[i] != b'\\' {
@@ -36,36 +60,27 @@ pub fn replace_lone_surrogates(text: &str) -> Cow<'_, str> {
i += 2;
continue;
};
if (0xD800..=0xDBFF).contains(&unit)
&& escape_unit(bytes, i + 6).is_some_and(|low| (0xDC00..=0xDFFF).contains(&low))
{
let is_pair = (0xD800..=0xDBFF).contains(&unit)
&& escape_unit(bytes, i + 6).is_some_and(|low| (0xDC00..=0xDFFF).contains(&low));
if is_pair {
i += 12;
continue;
}
if (0xD800..=0xDFFF).contains(&unit) {
let buffer = out.get_or_insert_with(|| String::with_capacity(text.len()));
buffer.push_str(&text[copied..i]);
buffer.push_str("\\uFFFD");
copied = i + 6;
return Some(i);
}
i += 6;
}
match out {
None => Cow::Borrowed(text),
Some(mut buffer) => {
buffer.push_str(&text[copied..]);
Cow::Owned(buffer)
}
}
None
}
/// The code unit of a `\uXXXX` escape starting at `at`, if there is one.
fn escape_unit(bytes: &[u8], at: usize) -> Option<u32> {
let hex = bytes.get(at..at + 6)?;
if hex[0] != b'\\' || hex[1] != b'u' {
let escape = bytes.get(at..at + 6)?;
if escape[0] != b'\\' || escape[1] != b'u' || !escape[2..].iter().all(u8::is_ascii_hexdigit) {
return None;
}
let digits = std::str::from_utf8(&hex[2..]).ok()?;
let digits = std::str::from_utf8(&escape[2..]).ok()?;
u32::from_str_radix(digits, 16).ok()
}
@@ -279,23 +294,46 @@ mod tests {
use super::*;
#[test]
fn lone_surrogates_become_replacement_characters() {
assert_eq!(replace_lone_surrogates("\"a\\ud800b\""), "\"a\\uFFFDb\"");
assert_eq!(replace_lone_surrogates("\"\\udc00\""), "\"\\uFFFD\"");
assert_eq!(
replace_lone_surrogates("\"\\ud83d\\ude00\""),
"\"\\ud83d\\ude00\""
);
assert_eq!(replace_lone_surrogates("\"\\\\ud800\""), "\"\\\\ud800\"");
assert_eq!(
replace_lone_surrogates("\"\\ud800\\ud800\""),
"\"\\uFFFD\\uFFFD\""
);
assert!(matches!(replace_lone_surrogates("plain"), Cow::Borrowed(_)));
let Ok(value) = parse_json("{\"t\":\"x\\ud800\"}") else {
panic!("lenient parse failed");
fn lone_surrogate_escapes_are_found_and_pairs_are_not() {
assert_eq!(find_lone_surrogate("\"a\\ud800b\""), Some(2));
assert_eq!(find_lone_surrogate("\"\\udc00\""), Some(1));
assert_eq!(find_lone_surrogate("\"\\uDBFF\""), Some(1));
assert_eq!(find_lone_surrogate("\"\\ud800\\ud800\""), Some(1));
assert_eq!(find_lone_surrogate("\"\\ud800\\n\""), Some(1));
assert_eq!(find_lone_surrogate("\"\\ud83d\\ude00\\udc00\""), Some(13));
assert_eq!(find_lone_surrogate("\"x\\ud800"), Some(2));
assert_eq!(find_lone_surrogate("\"\\ud83d\\ude00\""), None);
assert_eq!(find_lone_surrogate("\"\\\\ud800\""), None);
assert_eq!(find_lone_surrogate("\"\\u+800 \\u00e9 é\""), None);
assert_eq!(find_lone_surrogate("plain"), None);
}
#[test]
fn a_lone_surrogate_refuses_the_text_and_is_never_replaced() {
for text in [
"{\"t\":\"x\\ud800\"}",
"{\"t\":\"\\udfff\"}",
"{\"t\":[\"ok\",\"\\ud83d\"]}",
"{\"\\ud800\":1}",
] {
assert!(
matches!(parse_json(text), Err(JsonReadError::LoneSurrogate { .. })),
"{text}"
);
}
assert!(matches!(
parse_json("{ not json"),
Err(JsonReadError::Syntax(_))
));
}
#[test]
fn normal_text_reads_unchanged() {
let Ok(value) = parse_json("{\"t\":\"é \\u00e9 \\ud83d\\ude00 😀 \\\\ud800 \\uFFFD\"}")
else {
panic!("parse failed");
};
assert_eq!(value["t"], "x\u{FFFD}");
assert_eq!(value["t"], "é é 😀 😀 \\ud800 \u{FFFD}");
}
#[test]
+15 -4
View File
@@ -9,7 +9,7 @@ use ts_rs::TS;
use super::archive::ReportArchiveEnvelope;
use super::archive::read::{ArchiveReadOutcome, parse_envelope, read_archive_document};
use super::js::parse_json;
use super::js::{JsonReadError, parse_json};
use super::report::{ReadFailure, RebaseOutcome, read_report_document, rebase_report_on_sheet};
use super::sheet::LoadedSheet;
@@ -30,6 +30,8 @@ pub struct DraftRestore {
pub enum RestoreFailure {
/// The draft carries a report version this build does not read (`None`: no version).
UnsupportedVersion(Option<f64>),
/// The draft escapes a lone UTF-16 surrogate this build cannot read without altering it.
LoneSurrogate(String),
/// Not JSON, or a known version that does not parse.
Invalid(String),
/// The draft reads but cannot be moved onto the loaded sheet (its reference fails the
@@ -52,6 +54,9 @@ pub enum ImportFailure {
IncompatibleVersion(f64),
/// No version at all, or a corrupt report: not a Faunosteo attachment.
NoAttachment,
/// The attachment escapes a lone UTF-16 surrogate this build cannot read without
/// altering it: refused, never rewritten.
LoneSurrogate(String),
/// The read report could not be moved onto the loaded sheet (the rebased report fails
/// its schema: a defect of the loaded sheet's reference, never of the file).
Invalid(String),
@@ -73,7 +78,10 @@ pub fn restore_draft(
draft_text: &str,
archive_text: Option<&str>,
) -> Result<DraftRestore, RestoreFailure> {
let draft = parse_json(draft_text).map_err(|e| RestoreFailure::Invalid(e.to_string()))?;
let draft = parse_json(draft_text).map_err(|error| match error {
JsonReadError::LoneSurrogate { .. } => RestoreFailure::LoneSurrogate(error.to_string()),
JsonReadError::Syntax(_) => RestoreFailure::Invalid(error.to_string()),
})?;
let report = read_report_document(&draft, &sheet.migration_map)?.report;
let archive = archive_text.and_then(read_envelope_text);
let archive_rejected = archive_text.is_some() && archive.is_none();
@@ -99,12 +107,15 @@ fn read_envelope_text(text: &str) -> Option<ReportArchiveEnvelope> {
}
/// Reads an attachment's JSON text and rebases the report onto `sheet` with the archived
/// snapshot when the envelope was accepted.
/// snapshot when the envelope was accepted. A text the reader would alter is refused whole.
pub fn import_archive(
sheet: &LoadedSheet,
data_text: &str,
) -> Result<ArchiveImport, ImportFailure> {
let data: Value = parse_json(data_text).map_err(|_| ImportFailure::NoAttachment)?;
let data: Value = parse_json(data_text).map_err(|error| match error {
JsonReadError::LoneSurrogate { .. } => ImportFailure::LoneSurrogate(error.to_string()),
JsonReadError::Syntax(_) => ImportFailure::NoAttachment,
})?;
let read =
read_archive_document(&data, &sheet.migration_map).map_err(|failure| match failure {
ReadFailure::UnsupportedVersion(Some(found)) => {
+21 -2
View File
@@ -7,7 +7,7 @@ use serde::Serialize;
use serde::de::DeserializeOwned;
use serde_json::{Map, Value};
use crate::domain::js::parse_json;
use crate::domain::js::{JsonReadError, parse_json};
use crate::domain::sheet::{BundledSheetData, LoadError, LoadedSheet};
use crate::domain::{ReportZone, SheetRef};
use crate::json::{pretty_js_round_trip, to_js_compact};
@@ -65,9 +65,21 @@ fn unknown(op: &str) -> Failure {
Failure::new("unknown_op", format!("no operation named {op}"))
}
/// A JSON text the core cannot read: `lone_surrogate` when reading it would alter a string
/// (L1, refused whole), `invalid_json` otherwise.
impl From<JsonReadError> for Failure {
fn from(error: JsonReadError) -> Self {
let kind = match error {
JsonReadError::LoneSurrogate { .. } => "lone_surrogate",
JsonReadError::Syntax(_) => "invalid_json",
};
Self::new(kind, error)
}
}
/// Parses the JSON text of an operation input into its contract type.
pub(crate) fn parse_input<T: DeserializeOwned>(input: &str) -> Result<T, Failure> {
let value = parse_json(input).map_err(|error| Failure::new("invalid_json", error))?;
let value = parse_json(input)?;
T::deserialize(&value).map_err(|error| Failure::new("invalid_shape", error))
}
@@ -118,6 +130,13 @@ mod tests {
assert!(outcome.contains(r#""kind":"invalid_json""#));
}
#[test]
fn a_lone_surrogate_input_is_a_closed_kind_never_a_replacement() {
let outcome = call("report_zone", r#"{"annotationId":"a\ud800","viewId":"v"}"#);
assert!(outcome.contains(r#""kind":"lone_surrogate""#), "{outcome}");
assert!(!outcome.contains('\u{FFFD}'));
}
#[test]
fn report_zone_round_trip_keeps_the_declared_order() {
let outcome = call(
+5 -3
View File
@@ -105,7 +105,7 @@ pub fn free(op: &str, input: &str) -> Option<Outcome> {
}
fn sheet_check(input: &str) -> Outcome {
let document = parse_json(input).map_err(|error| Failure::new("invalid_json", error))?;
let document = parse_json(input)?;
let check =
check_sheet_document(document).map_err(|error| Failure::new("invalid_shape", error))?;
to_value(&check)
@@ -306,14 +306,15 @@ fn versioned(kind: &'static str, found: Option<f64>) -> Failure {
}
}
/// Content kinds (`invalid`, `unsupported_version`) mean the draft itself is unreadable:
/// the caller sets it aside. `rebase_failed` is a defect of the loaded sheet, never of the
/// Content kinds (`invalid`, `unsupported_version`, `lone_surrogate`) mean the draft itself
/// is unreadable: the caller sets it aside, verbatim. `rebase_failed` is a defect of the loaded sheet, never of the
/// draft, and leaves it where it is.
fn draft_restore(sheet: &LoadedSheet, input: &str) -> Outcome {
let input: DraftRestoreInput = parse_input(input)?;
session::restore_draft(sheet, &input.draft, input.archive.as_deref())
.map_err(|failure| match failure {
RestoreFailure::UnsupportedVersion(found) => versioned("unsupported_version", found),
RestoreFailure::LoneSurrogate(detail) => Failure::new("lone_surrogate", detail),
RestoreFailure::Invalid(detail) => Failure::new("invalid", detail),
RestoreFailure::Rebase(detail) => Failure::new("rebase_failed", detail),
})
@@ -328,6 +329,7 @@ fn archive_import(sheet: &LoadedSheet, input: &str) -> Outcome {
versioned("incompatible_version", Some(found))
}
ImportFailure::NoAttachment => Failure::new("no_attachment", ""),
ImportFailure::LoneSurrogate(detail) => Failure::new("lone_surrogate", detail),
ImportFailure::Invalid(detail) => Failure::new("invalid", detail),
})
.and_then(|import| to_value(&import))