fix(domain): refuse lone UTF-16 surrogates instead of replacing them (L1)
serde_json already refuses a lone surrogate escape; the U+FFFD pre-pass is gone and parse_json classifies the refusal (strict escape scan, on failure only) so callers name it: closed kind lone_surrogate from operation inputs, sheet.check, draft.restore and archive.import. Oracles untouched: the two L1 cases and the Zod verdict now assert that TypeScript kept the surrogate and Rust refuses the text.
This commit is contained in:
1 parent
36316cf67f
commit
20d03cf7fd
9 files changed
+275
-75
No files matched your search
@@ -1,31 +1,55 @@
|
||||
//! JavaScript and Zod semantics the ported contracts depend on (R2 plan P1, P2).
|
||||
//!
|
||||
//! The TypeScript domain reads JSON with `JSON.parse` and validates it with Zod 3: lone
|
||||
//! surrogate escapes are kept, a duplicate key keeps its last value at its first position,
|
||||
//! `.int()` accepts any number with an integer value (`2.0`, `1e20`), string lengths count
|
||||
//! UTF-16 code units, and the patterns are JavaScript regular expressions whose `$` only
|
||||
//! matches at the end of the input. The helpers below reproduce exactly that, so a document
|
||||
//! the old code accepted is accepted here and prints the same.
|
||||
//! The TypeScript domain reads JSON with `JSON.parse` and validates it with Zod 3: a duplicate
|
||||
//! key keeps its last value at its first position, `.int()` accepts any number with an
|
||||
//! integer value (`2.0`, `1e20`), string lengths count UTF-16 code units, and the patterns are
|
||||
//! JavaScript regular expressions whose `$` only matches at the end of the input. The helpers
|
||||
//! below reproduce exactly that, so a document the old code accepted is accepted here and
|
||||
//! prints the same. One declared exception: a lone surrogate escape, which `JSON.parse` keeps
|
||||
//! and a Rust string cannot hold, refuses the text instead of being altered (L1).
|
||||
|
||||
use std::borrow::Cow;
|
||||
use std::fmt;
|
||||
|
||||
use serde::de::Error as _;
|
||||
use serde::{Deserialize, Deserializer, Serialize, Serializer};
|
||||
use serde_json::Value;
|
||||
|
||||
/// Reads JSON text as `JSON.parse` does, as far as a Rust string allows: an escape of a
|
||||
/// lone UTF-16 surrogate (which `JSON.parse` keeps and Rust strings cannot hold) becomes
|
||||
/// U+FFFD instead of refusing the whole document (leniency decision L1).
|
||||
pub fn parse_json(text: &str) -> Result<Value, serde_json::Error> {
|
||||
serde_json::from_str(&replace_lone_surrogates(text))
|
||||
/// Why a JSON text cannot be read.
|
||||
#[derive(Debug)]
|
||||
pub enum JsonReadError {
|
||||
/// The text escapes a lone UTF-16 surrogate (`\uD800`–`\uDFFF` outside a high + low
|
||||
/// pair). `JSON.parse` keeps it; reading it here would replace it, so the whole text is
|
||||
/// refused and the caller keeps it verbatim (L1: user data is never altered silently).
|
||||
LoneSurrogate { offset: usize },
|
||||
/// Not JSON.
|
||||
Syntax(serde_json::Error),
|
||||
}
|
||||
|
||||
/// Rewrites `\uD800`–`\uDFFF` escapes that are not a high + low pair to `�`.
|
||||
/// Escapes are consumed left to right, so `\\u…` (an escaped backslash) is left alone.
|
||||
pub fn replace_lone_surrogates(text: &str) -> Cow<'_, str> {
|
||||
impl fmt::Display for JsonReadError {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
match self {
|
||||
Self::LoneSurrogate { offset } => {
|
||||
write!(f, "lone UTF-16 surrogate escape at byte {offset}")
|
||||
}
|
||||
Self::Syntax(error) => error.fmt(f),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Reads JSON text as `JSON.parse` does, as far as a Rust string allows. serde_json refuses
|
||||
/// a lone surrogate escape like any other error; the refusal is then classified so callers
|
||||
/// can name it (the scan runs on failures only).
|
||||
pub fn parse_json(text: &str) -> Result<Value, JsonReadError> {
|
||||
serde_json::from_str(text).map_err(|error| match find_lone_surrogate(text) {
|
||||
Some(offset) => JsonReadError::LoneSurrogate { offset },
|
||||
None => JsonReadError::Syntax(error),
|
||||
})
|
||||
}
|
||||
|
||||
/// Byte offset of the first `\uD800`–`\uDFFF` escape that is not a high + low pair.
|
||||
/// Escapes are consumed left to right, so `\\u…` (an escaped backslash) is skipped.
|
||||
pub fn find_lone_surrogate(text: &str) -> Option<usize> {
|
||||
let bytes = text.as_bytes();
|
||||
let mut out: Option<String> = None;
|
||||
let mut copied = 0;
|
||||
let mut i = 0;
|
||||
while i < bytes.len() {
|
||||
if bytes[i] != b'\\' {
|
||||
@@ -36,36 +60,27 @@ pub fn replace_lone_surrogates(text: &str) -> Cow<'_, str> {
|
||||
i += 2;
|
||||
continue;
|
||||
};
|
||||
if (0xD800..=0xDBFF).contains(&unit)
|
||||
&& escape_unit(bytes, i + 6).is_some_and(|low| (0xDC00..=0xDFFF).contains(&low))
|
||||
{
|
||||
let is_pair = (0xD800..=0xDBFF).contains(&unit)
|
||||
&& escape_unit(bytes, i + 6).is_some_and(|low| (0xDC00..=0xDFFF).contains(&low));
|
||||
if is_pair {
|
||||
i += 12;
|
||||
continue;
|
||||
}
|
||||
if (0xD800..=0xDFFF).contains(&unit) {
|
||||
let buffer = out.get_or_insert_with(|| String::with_capacity(text.len()));
|
||||
buffer.push_str(&text[copied..i]);
|
||||
buffer.push_str("\\uFFFD");
|
||||
copied = i + 6;
|
||||
return Some(i);
|
||||
}
|
||||
i += 6;
|
||||
}
|
||||
match out {
|
||||
None => Cow::Borrowed(text),
|
||||
Some(mut buffer) => {
|
||||
buffer.push_str(&text[copied..]);
|
||||
Cow::Owned(buffer)
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// The code unit of a `\uXXXX` escape starting at `at`, if there is one.
|
||||
fn escape_unit(bytes: &[u8], at: usize) -> Option<u32> {
|
||||
let hex = bytes.get(at..at + 6)?;
|
||||
if hex[0] != b'\\' || hex[1] != b'u' {
|
||||
let escape = bytes.get(at..at + 6)?;
|
||||
if escape[0] != b'\\' || escape[1] != b'u' || !escape[2..].iter().all(u8::is_ascii_hexdigit) {
|
||||
return None;
|
||||
}
|
||||
let digits = std::str::from_utf8(&hex[2..]).ok()?;
|
||||
let digits = std::str::from_utf8(&escape[2..]).ok()?;
|
||||
u32::from_str_radix(digits, 16).ok()
|
||||
}
|
||||
|
||||
@@ -279,23 +294,46 @@ mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn lone_surrogates_become_replacement_characters() {
|
||||
assert_eq!(replace_lone_surrogates("\"a\\ud800b\""), "\"a\\uFFFDb\"");
|
||||
assert_eq!(replace_lone_surrogates("\"\\udc00\""), "\"\\uFFFD\"");
|
||||
assert_eq!(
|
||||
replace_lone_surrogates("\"\\ud83d\\ude00\""),
|
||||
"\"\\ud83d\\ude00\""
|
||||
);
|
||||
assert_eq!(replace_lone_surrogates("\"\\\\ud800\""), "\"\\\\ud800\"");
|
||||
assert_eq!(
|
||||
replace_lone_surrogates("\"\\ud800\\ud800\""),
|
||||
"\"\\uFFFD\\uFFFD\""
|
||||
);
|
||||
assert!(matches!(replace_lone_surrogates("plain"), Cow::Borrowed(_)));
|
||||
let Ok(value) = parse_json("{\"t\":\"x\\ud800\"}") else {
|
||||
panic!("lenient parse failed");
|
||||
fn lone_surrogate_escapes_are_found_and_pairs_are_not() {
|
||||
assert_eq!(find_lone_surrogate("\"a\\ud800b\""), Some(2));
|
||||
assert_eq!(find_lone_surrogate("\"\\udc00\""), Some(1));
|
||||
assert_eq!(find_lone_surrogate("\"\\uDBFF\""), Some(1));
|
||||
assert_eq!(find_lone_surrogate("\"\\ud800\\ud800\""), Some(1));
|
||||
assert_eq!(find_lone_surrogate("\"\\ud800\\n\""), Some(1));
|
||||
assert_eq!(find_lone_surrogate("\"\\ud83d\\ude00\\udc00\""), Some(13));
|
||||
assert_eq!(find_lone_surrogate("\"x\\ud800"), Some(2));
|
||||
assert_eq!(find_lone_surrogate("\"\\ud83d\\ude00\""), None);
|
||||
assert_eq!(find_lone_surrogate("\"\\\\ud800\""), None);
|
||||
assert_eq!(find_lone_surrogate("\"\\u+800 \\u00e9 é\""), None);
|
||||
assert_eq!(find_lone_surrogate("plain"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_lone_surrogate_refuses_the_text_and_is_never_replaced() {
|
||||
for text in [
|
||||
"{\"t\":\"x\\ud800\"}",
|
||||
"{\"t\":\"\\udfff\"}",
|
||||
"{\"t\":[\"ok\",\"\\ud83d\"]}",
|
||||
"{\"\\ud800\":1}",
|
||||
] {
|
||||
assert!(
|
||||
matches!(parse_json(text), Err(JsonReadError::LoneSurrogate { .. })),
|
||||
"{text}"
|
||||
);
|
||||
}
|
||||
assert!(matches!(
|
||||
parse_json("{ not json"),
|
||||
Err(JsonReadError::Syntax(_))
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normal_text_reads_unchanged() {
|
||||
let Ok(value) = parse_json("{\"t\":\"é \\u00e9 \\ud83d\\ude00 😀 \\\\ud800 \\uFFFD\"}")
|
||||
else {
|
||||
panic!("parse failed");
|
||||
};
|
||||
assert_eq!(value["t"], "x\u{FFFD}");
|
||||
assert_eq!(value["t"], "é é 😀 😀 \\ud800 \u{FFFD}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -9,7 +9,7 @@ use ts_rs::TS;
|
||||
|
||||
use super::archive::ReportArchiveEnvelope;
|
||||
use super::archive::read::{ArchiveReadOutcome, parse_envelope, read_archive_document};
|
||||
use super::js::parse_json;
|
||||
use super::js::{JsonReadError, parse_json};
|
||||
use super::report::{ReadFailure, RebaseOutcome, read_report_document, rebase_report_on_sheet};
|
||||
use super::sheet::LoadedSheet;
|
||||
|
||||
@@ -30,6 +30,8 @@ pub struct DraftRestore {
|
||||
pub enum RestoreFailure {
|
||||
/// The draft carries a report version this build does not read (`None`: no version).
|
||||
UnsupportedVersion(Option<f64>),
|
||||
/// The draft escapes a lone UTF-16 surrogate this build cannot read without altering it.
|
||||
LoneSurrogate(String),
|
||||
/// Not JSON, or a known version that does not parse.
|
||||
Invalid(String),
|
||||
/// The draft reads but cannot be moved onto the loaded sheet (its reference fails the
|
||||
@@ -52,6 +54,9 @@ pub enum ImportFailure {
|
||||
IncompatibleVersion(f64),
|
||||
/// No version at all, or a corrupt report: not a Faunosteo attachment.
|
||||
NoAttachment,
|
||||
/// The attachment escapes a lone UTF-16 surrogate this build cannot read without
|
||||
/// altering it: refused, never rewritten.
|
||||
LoneSurrogate(String),
|
||||
/// The read report could not be moved onto the loaded sheet (the rebased report fails
|
||||
/// its schema: a defect of the loaded sheet's reference, never of the file).
|
||||
Invalid(String),
|
||||
@@ -73,7 +78,10 @@ pub fn restore_draft(
|
||||
draft_text: &str,
|
||||
archive_text: Option<&str>,
|
||||
) -> Result<DraftRestore, RestoreFailure> {
|
||||
let draft = parse_json(draft_text).map_err(|e| RestoreFailure::Invalid(e.to_string()))?;
|
||||
let draft = parse_json(draft_text).map_err(|error| match error {
|
||||
JsonReadError::LoneSurrogate { .. } => RestoreFailure::LoneSurrogate(error.to_string()),
|
||||
JsonReadError::Syntax(_) => RestoreFailure::Invalid(error.to_string()),
|
||||
})?;
|
||||
let report = read_report_document(&draft, &sheet.migration_map)?.report;
|
||||
let archive = archive_text.and_then(read_envelope_text);
|
||||
let archive_rejected = archive_text.is_some() && archive.is_none();
|
||||
@@ -99,12 +107,15 @@ fn read_envelope_text(text: &str) -> Option<ReportArchiveEnvelope> {
|
||||
}
|
||||
|
||||
/// Reads an attachment's JSON text and rebases the report onto `sheet` with the archived
|
||||
/// snapshot when the envelope was accepted.
|
||||
/// snapshot when the envelope was accepted. A text the reader would alter is refused whole.
|
||||
pub fn import_archive(
|
||||
sheet: &LoadedSheet,
|
||||
data_text: &str,
|
||||
) -> Result<ArchiveImport, ImportFailure> {
|
||||
let data: Value = parse_json(data_text).map_err(|_| ImportFailure::NoAttachment)?;
|
||||
let data: Value = parse_json(data_text).map_err(|error| match error {
|
||||
JsonReadError::LoneSurrogate { .. } => ImportFailure::LoneSurrogate(error.to_string()),
|
||||
JsonReadError::Syntax(_) => ImportFailure::NoAttachment,
|
||||
})?;
|
||||
let read =
|
||||
read_archive_document(&data, &sheet.migration_map).map_err(|failure| match failure {
|
||||
ReadFailure::UnsupportedVersion(Some(found)) => {
|
||||
|
||||
@@ -7,7 +7,7 @@ use serde::Serialize;
|
||||
use serde::de::DeserializeOwned;
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
use crate::domain::js::parse_json;
|
||||
use crate::domain::js::{JsonReadError, parse_json};
|
||||
use crate::domain::sheet::{BundledSheetData, LoadError, LoadedSheet};
|
||||
use crate::domain::{ReportZone, SheetRef};
|
||||
use crate::json::{pretty_js_round_trip, to_js_compact};
|
||||
@@ -65,9 +65,21 @@ fn unknown(op: &str) -> Failure {
|
||||
Failure::new("unknown_op", format!("no operation named {op}"))
|
||||
}
|
||||
|
||||
/// A JSON text the core cannot read: `lone_surrogate` when reading it would alter a string
|
||||
/// (L1, refused whole), `invalid_json` otherwise.
|
||||
impl From<JsonReadError> for Failure {
|
||||
fn from(error: JsonReadError) -> Self {
|
||||
let kind = match error {
|
||||
JsonReadError::LoneSurrogate { .. } => "lone_surrogate",
|
||||
JsonReadError::Syntax(_) => "invalid_json",
|
||||
};
|
||||
Self::new(kind, error)
|
||||
}
|
||||
}
|
||||
|
||||
/// Parses the JSON text of an operation input into its contract type.
|
||||
pub(crate) fn parse_input<T: DeserializeOwned>(input: &str) -> Result<T, Failure> {
|
||||
let value = parse_json(input).map_err(|error| Failure::new("invalid_json", error))?;
|
||||
let value = parse_json(input)?;
|
||||
T::deserialize(&value).map_err(|error| Failure::new("invalid_shape", error))
|
||||
}
|
||||
|
||||
@@ -118,6 +130,13 @@ mod tests {
|
||||
assert!(outcome.contains(r#""kind":"invalid_json""#));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_lone_surrogate_input_is_a_closed_kind_never_a_replacement() {
|
||||
let outcome = call("report_zone", r#"{"annotationId":"a\ud800","viewId":"v"}"#);
|
||||
assert!(outcome.contains(r#""kind":"lone_surrogate""#), "{outcome}");
|
||||
assert!(!outcome.contains('\u{FFFD}'));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn report_zone_round_trip_keeps_the_declared_order() {
|
||||
let outcome = call(
|
||||
|
||||
@@ -105,7 +105,7 @@ pub fn free(op: &str, input: &str) -> Option<Outcome> {
|
||||
}
|
||||
|
||||
fn sheet_check(input: &str) -> Outcome {
|
||||
let document = parse_json(input).map_err(|error| Failure::new("invalid_json", error))?;
|
||||
let document = parse_json(input)?;
|
||||
let check =
|
||||
check_sheet_document(document).map_err(|error| Failure::new("invalid_shape", error))?;
|
||||
to_value(&check)
|
||||
@@ -306,14 +306,15 @@ fn versioned(kind: &'static str, found: Option<f64>) -> Failure {
|
||||
}
|
||||
}
|
||||
|
||||
/// Content kinds (`invalid`, `unsupported_version`) mean the draft itself is unreadable:
|
||||
/// the caller sets it aside. `rebase_failed` is a defect of the loaded sheet, never of the
|
||||
/// Content kinds (`invalid`, `unsupported_version`, `lone_surrogate`) mean the draft itself
|
||||
/// is unreadable: the caller sets it aside, verbatim. `rebase_failed` is a defect of the loaded sheet, never of the
|
||||
/// draft, and leaves it where it is.
|
||||
fn draft_restore(sheet: &LoadedSheet, input: &str) -> Outcome {
|
||||
let input: DraftRestoreInput = parse_input(input)?;
|
||||
session::restore_draft(sheet, &input.draft, input.archive.as_deref())
|
||||
.map_err(|failure| match failure {
|
||||
RestoreFailure::UnsupportedVersion(found) => versioned("unsupported_version", found),
|
||||
RestoreFailure::LoneSurrogate(detail) => Failure::new("lone_surrogate", detail),
|
||||
RestoreFailure::Invalid(detail) => Failure::new("invalid", detail),
|
||||
RestoreFailure::Rebase(detail) => Failure::new("rebase_failed", detail),
|
||||
})
|
||||
@@ -328,6 +329,7 @@ fn archive_import(sheet: &LoadedSheet, input: &str) -> Outcome {
|
||||
versioned("incompatible_version", Some(found))
|
||||
}
|
||||
ImportFailure::NoAttachment => Failure::new("no_attachment", ""),
|
||||
ImportFailure::LoneSurrogate(detail) => Failure::new("lone_surrogate", detail),
|
||||
ImportFailure::Invalid(detail) => Failure::new("invalid", detail),
|
||||
})
|
||||
.and_then(|import| to_value(&import))
|
||||
|
||||
@@ -882,3 +882,97 @@ fn import_reads_and_rebases_or_names_the_failure_as_the_engine_does() {
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// L1: a lone UTF-16 surrogate escape refuses the text whole, never replaced
|
||||
|
||||
/// `value` as JSON text with `MARK` swapped for the raw escape `escape` (serde_json cannot
|
||||
/// write a lone surrogate itself; `JSON.stringify` writes exactly this).
|
||||
fn text_with_escape(value: &Value, escape: &str) -> String {
|
||||
serde_json::to_string(value)
|
||||
.unwrap()
|
||||
.replace("MARK", escape)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_draft_with_a_lone_surrogate_is_refused_as_content_never_rewritten() {
|
||||
let f = EnvelopeFixture::new();
|
||||
let report = with(report_value(&f.report), "conclusion", json!("aMARKb"));
|
||||
for escape in ["\\ud800", "\\udfff", "\\ud83d\\ud83d"] {
|
||||
let draft = text_with_escape(&report, escape);
|
||||
assert!(
|
||||
matches!(
|
||||
restore_draft(&f.draft, &draft, None),
|
||||
Err(RestoreFailure::LoneSurrogate(_))
|
||||
),
|
||||
"{escape}"
|
||||
);
|
||||
}
|
||||
let v2 = common::repo_file("packages/domain/tests/fixtures/report-v2.full.json").replace(
|
||||
"\"comment\": \"Réagit à la palpation du garrot.\"",
|
||||
"\"comment\": \"Réagit \\udc00\"",
|
||||
);
|
||||
assert!(v2.contains("\\udc00"));
|
||||
assert!(matches!(
|
||||
restore_draft(&f.draft, &v2, None),
|
||||
Err(RestoreFailure::LoneSurrogate(_))
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_archive_with_a_lone_surrogate_is_set_aside_while_the_draft_restores() {
|
||||
let f = EnvelopeFixture::new();
|
||||
let draft = serde_json::to_string(&report_value(&f.report)).unwrap();
|
||||
let mut envelope = f.envelope();
|
||||
envelope["branding"]["poweredBy"]["fallbackText"] = json!("MARK");
|
||||
let readable = serde_json::to_string(&envelope)
|
||||
.unwrap()
|
||||
.replace("MARK", "zen");
|
||||
assert!(
|
||||
!restore_draft(&f.draft, &draft, Some(&readable))
|
||||
.unwrap()
|
||||
.archive_rejected
|
||||
);
|
||||
let archive = text_with_escape(&envelope, "\\ud800");
|
||||
let restored = restore_draft(&f.draft, &draft, Some(&archive)).unwrap();
|
||||
assert!(restored.archive.is_none() && restored.archive_rejected);
|
||||
assert_eq!(restored.rebase.report, f.report);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_attachment_with_a_lone_surrogate_is_refused_with_its_own_kind() {
|
||||
let f = EnvelopeFixture::new();
|
||||
let mut envelope = f.envelope();
|
||||
envelope["report"]["conclusion"] = json!("MARK");
|
||||
let text = text_with_escape(&envelope, "\\ud800");
|
||||
assert!(matches!(
|
||||
import_archive(&f.draft, &text),
|
||||
Err(ImportFailure::LoneSurrogate(_))
|
||||
));
|
||||
let bare = text_with_escape(
|
||||
&with(report_value(&f.report), "conclusion", json!("MARK")),
|
||||
"\\udbff",
|
||||
);
|
||||
assert!(matches!(
|
||||
import_archive(&f.draft, &bare),
|
||||
Err(ImportFailure::LoneSurrogate(_))
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normal_text_with_pairs_and_escaped_backslashes_reads_unchanged() {
|
||||
let f = EnvelopeFixture::new();
|
||||
let kept = "😀 \\ud800 é \u{FFFD}";
|
||||
let report = with(report_value(&f.report), "conclusion", json!(kept));
|
||||
let text = serde_json::to_string(&report).unwrap();
|
||||
assert!(text.contains("\\\\ud800"));
|
||||
let restored = restore_draft(&f.draft, &text, None).unwrap();
|
||||
assert_eq!(restored.rebase.report.conclusion, kept);
|
||||
let mut envelope = f.envelope();
|
||||
envelope["report"] = report.clone();
|
||||
let paired = serde_json::to_string(&envelope)
|
||||
.unwrap()
|
||||
.replace('😀', "\\ud83d\\ude00");
|
||||
let imported = import_archive(&f.draft, &paired).unwrap();
|
||||
assert_eq!(imported.rebase.report.conclusion, kept);
|
||||
}
|
||||
@@ -81,8 +81,7 @@ pub fn draft_map() -> ReportMigrationMap {
|
||||
/// The horse slice of `packages/domain/tests/fixtures/horse-slice.ts`, as Zod parsed it (the
|
||||
/// characterisation oracle carries it, written by the TypeScript generator).
|
||||
pub fn horse_slice() -> Value {
|
||||
json_file("crates/core/tests/oracles/characterisation.json")["sheets"]["slice"]["document"]
|
||||
.clone()
|
||||
super::common::oracle_file("characterisation.json").1["sheets"]["slice"]["document"].clone()
|
||||
}
|
||||
|
||||
pub fn snake_ref(sheet_id: &str, version_id: Option<&str>, sha: &str) -> Value {
|
||||
|
||||
@@ -10,7 +10,7 @@ use faunosteo_core::domain::archive::{
|
||||
Branding, PdfLabelSet, ReportArchiveEnvelope, SheetSnapshot, assert_snapshot,
|
||||
build_sheet_snapshot, read_archive_document,
|
||||
};
|
||||
use faunosteo_core::domain::js::parse_json;
|
||||
use faunosteo_core::domain::js::{JsonReadError, parse_json};
|
||||
use faunosteo_core::domain::report::v2::ReportV2;
|
||||
use faunosteo_core::domain::report::{
|
||||
ReadFailure, Report, ReportMigrationMap, read_report_document, rebase_report_on_sheet,
|
||||
@@ -21,9 +21,15 @@ use serde::Serialize;
|
||||
use serde_json::{Map, Value, json};
|
||||
|
||||
fn oracle(name: &str) -> (String, Value) {
|
||||
let text = common::repo_file(&format!("crates/core/tests/oracles/{name}"));
|
||||
let value = parse_json(&text).unwrap();
|
||||
(text, value)
|
||||
common::oracle_file(name)
|
||||
}
|
||||
|
||||
/// A case input as the core reads it; a lone surrogate refuses it whole (L1).
|
||||
fn read_input(text: &str) -> Result<Value, Value> {
|
||||
parse_json(text).map_err(|error| match error {
|
||||
JsonReadError::LoneSurrogate { .. } => json!({ "kind": "lone_surrogate" }),
|
||||
JsonReadError::Syntax(error) => panic!("unreadable oracle input: {error}"),
|
||||
})
|
||||
}
|
||||
|
||||
/// Object keys sorted at every level: equality whatever the declared field order.
|
||||
@@ -77,7 +83,7 @@ fn read_error(failure: &ReadFailure) -> Value {
|
||||
|
||||
/// What Rust returns for a case: `Ok(output)` or `Err(error)` in the oracle's shapes.
|
||||
fn run(context: &Context, case: &Value) -> Result<Value, Value> {
|
||||
let input = parse_json(case["input"].as_str().unwrap()).unwrap();
|
||||
let input = read_input(case["input"].as_str().unwrap())?;
|
||||
match case["fn"].as_str().unwrap() {
|
||||
"readReport" => read_report_document(&input, &context.map(case))
|
||||
.map(|o| serde_json::to_value(o).unwrap())
|
||||
@@ -143,7 +149,7 @@ fn cases(with_rules: bool) -> (Context, Vec<Value>) {
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.filter(|c| c.get("needsRules").is_some() == with_rules)
|
||||
.filter(|c| c.get("needsRules").is_some() == with_rules && c.get("l1").is_none())
|
||||
.cloned()
|
||||
.collect();
|
||||
(context, list)
|
||||
@@ -167,13 +173,15 @@ fn rust_does_what_the_typescript_did_with_the_integrity_rules() {
|
||||
assert!(failures.is_empty(), "{}", failures.join("\n"));
|
||||
}
|
||||
|
||||
/// Decision L1: where `JSON.parse` keeps a lone surrogate (the TypeScript output still
|
||||
/// carries the `\ud800` escape), Rust reads U+FFFD; nothing else differs.
|
||||
/// Decision L1 (owner, 2026-10-14): where `JSON.parse` kept a lone surrogate (the frozen
|
||||
/// TypeScript output still carries the `\\ud800` escape), Rust refuses the input whole with
|
||||
/// `lone_surrogate` instead of altering it; no output, no U+FFFD.
|
||||
#[test]
|
||||
fn lone_surrogates_are_the_one_declared_difference() {
|
||||
fn lone_surrogates_are_refused_never_replaced() {
|
||||
let (raw, value) = oracle("characterisation.json");
|
||||
let backslash = char::from(92);
|
||||
let lone_escape = format!("{backslash}ud800");
|
||||
let context = cases(false).0;
|
||||
let flagged: Vec<&Value> = value["cases"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
@@ -189,9 +197,11 @@ fn lone_surrogates_are_the_one_declared_difference() {
|
||||
output.contains(&lone_escape),
|
||||
"{name}: TS output keeps the lone surrogate"
|
||||
);
|
||||
let context = cases(false).0;
|
||||
let rust = pretty(&run(&context, case).unwrap());
|
||||
assert!(rust.contains('\u{FFFD}'), "{name}: Rust reads U+FFFD");
|
||||
assert_eq!(
|
||||
run(&context, case),
|
||||
Err(json!({ "kind": "lone_surrogate" })),
|
||||
"{name}: Rust refuses the text"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -215,7 +225,14 @@ fn the_contracts_accept_and_refuse_what_zod_does() {
|
||||
let (_, list) = oracle("verdicts.json");
|
||||
let list = list.as_array().unwrap();
|
||||
assert!(list.len() > 500, "{} verdicts", list.len());
|
||||
let failures: Vec<String> = list
|
||||
// L1: the one input Zod accepted with a lone surrogate is refused by the reader instead.
|
||||
let (lone, readable): (Vec<&Value>, Vec<&Value>) = list
|
||||
.iter()
|
||||
.partition(|v| read_input(v["input"].as_str().unwrap()).is_err());
|
||||
assert_eq!(lone.len(), 1);
|
||||
assert_eq!(lone[0]["name"], "lone surrogate in a comment");
|
||||
assert_eq!(lone[0]["accepted"], true);
|
||||
let failures: Vec<String> = readable
|
||||
.iter()
|
||||
.filter_map(|v| {
|
||||
let schema = v["schema"].as_str().unwrap();
|
||||
|
||||
@@ -6,6 +6,7 @@
|
||||
clippy::expect_used
|
||||
)]
|
||||
|
||||
use faunosteo_core::domain::js::{find_lone_surrogate, parse_json};
|
||||
use serde_json::Value;
|
||||
|
||||
/// A repository file, by path from the repository root.
|
||||
@@ -14,6 +15,25 @@ pub fn repo_file(relative: &str) -> String {
|
||||
std::fs::read_to_string(&path).unwrap_or_else(|error| panic!("{path}: {error}"))
|
||||
}
|
||||
|
||||
/// A frozen oracle file (`crates/core/tests/oracles/<name>`): its text and its value. The
|
||||
/// TypeScript outputs it freezes keep the lone surrogate escapes `JSON.parse` kept (the L1
|
||||
/// cases), which the core refuses to read; each one is read here as literal text (`\\uD800`
|
||||
/// spelt out), never replaced, so the raw text stays the evidence of what TypeScript did.
|
||||
pub fn oracle_file(name: &str) -> (String, Value) {
|
||||
let text = repo_file(&format!("crates/core/tests/oracles/{name}"));
|
||||
let mut spelt = String::with_capacity(text.len());
|
||||
let mut rest = text.as_str();
|
||||
while let Some(at) = find_lone_surrogate(rest) {
|
||||
spelt.push_str(&rest[..at]);
|
||||
spelt.push('\\');
|
||||
spelt.push_str(&rest[at..at + 6]);
|
||||
rest = &rest[at + 6..];
|
||||
}
|
||||
spelt.push_str(rest);
|
||||
let value = parse_json(&spelt).unwrap();
|
||||
(text, value)
|
||||
}
|
||||
|
||||
/// A Zod oracle (`crates/core/tests/oracles/<name>`): its source file and the Zod output in
|
||||
/// the JavaScript pretty form.
|
||||
pub fn zod_oracle(name: &str) -> (String, String) {
|
||||
|
||||
@@ -320,7 +320,7 @@ interface Case {
|
||||
archived?: string | null
|
||||
output?: Json
|
||||
error?: { kind: string; found?: number | null }
|
||||
/** The TS result keeps a lone surrogate; the Rust one reads U+FFFD (decision L1). */
|
||||
/** The TS result keeps a lone surrogate; Rust refuses the text (`lone_surrogate`, L1). */
|
||||
l1?: true
|
||||
/** The outcome depends on the sheet integrity rules (fork A's validate.rs). */
|
||||
needsRules?: true
|
||||
|
||||
Reference in new issue
Block a user