fix(anatomy): R5.4 records charged at each record, review drawings bounded before rendering
This commit is contained in:
1 parent
5880dfa959
commit
8bdb079f70
10 files changed
+357
-75
No files matched your search
@@ -15,7 +15,7 @@ use serde_json::Value;
|
||||
|
||||
use super::ops::{apply_op, assert_declared};
|
||||
use super::records::{
|
||||
ChangeRecord, CurationResult, Recorder, record_weight, strip_marks, touch_order,
|
||||
ChangeRecord, CurationResult, RecordParts, Recorder, charge, strip_marks, touch_order,
|
||||
};
|
||||
use super::schema::{Change, ChangeSummary, Op, SetOp};
|
||||
use super::shape::assert_same_shape;
|
||||
@@ -67,6 +67,7 @@ fn mark_to_confirm(
|
||||
doc: &mut Value,
|
||||
change: &ChangeSummary,
|
||||
records: &mut Vec<ChangeRecord>,
|
||||
spent: &mut usize,
|
||||
) -> Result<(), PipelineError> {
|
||||
for touched in touched_reviewable(doc)? {
|
||||
let item = item_at_mut(doc, &touched.path)?;
|
||||
@@ -85,64 +86,47 @@ fn mark_to_confirm(
|
||||
value: Value::from("to_confirm"),
|
||||
scope: None,
|
||||
});
|
||||
let after = Value::from("to_confirm");
|
||||
let parts = RecordParts {
|
||||
change,
|
||||
op: &op,
|
||||
target: &touched.target,
|
||||
field: Some("review_status"),
|
||||
before: &before,
|
||||
after: &after,
|
||||
};
|
||||
charge(spent, parts)?;
|
||||
records.push(ChangeRecord {
|
||||
change: change.clone(),
|
||||
op,
|
||||
target: touched.target,
|
||||
field: Some("review_status".to_owned()),
|
||||
before,
|
||||
after: Value::from("to_confirm"),
|
||||
after,
|
||||
});
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// The total weight of the records (`records::record_weight`): about 16 MiB of text, or a
|
||||
/// quarter of a million JSON values, far past the equine changeset (test
|
||||
/// `the_records_cannot_outgrow_their_bound`) and small enough that the reports built from them
|
||||
/// stay bounded too. Checked after every op, before the next one records anything.
|
||||
pub const MAX_RECORDED_WEIGHT: usize = 16 * 1024 * 1024;
|
||||
|
||||
/// The records counted so far and their weight.
|
||||
#[derive(Default)]
|
||||
struct Tally {
|
||||
counted: usize,
|
||||
weight: usize,
|
||||
}
|
||||
|
||||
fn assert_recorded(records: &[ChangeRecord], tally: &mut Tally) -> Result<(), PipelineError> {
|
||||
for record in &records[tally.counted..] {
|
||||
tally.weight = tally.weight.saturating_add(record_weight(record));
|
||||
}
|
||||
tally.counted = records.len();
|
||||
if tally.weight > MAX_RECORDED_WEIGHT {
|
||||
return Err(fail(format!(
|
||||
"the records weigh more than {MAX_RECORDED_WEIGHT} units"
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn apply_change(
|
||||
doc: &mut Value,
|
||||
change: &Change,
|
||||
records: &mut Vec<ChangeRecord>,
|
||||
tally: &mut Tally,
|
||||
spent: &mut usize,
|
||||
) -> Result<(), PipelineError> {
|
||||
let summary = change.summary();
|
||||
assert_declared(doc, &change.reference_ids)
|
||||
.map_err(|error| fail(format!("{}: {error}", change.id)))?;
|
||||
let mut next_touch = 0;
|
||||
for (index, op) in change.ops.iter().enumerate() {
|
||||
let mut recorder = Recorder::new(&summary, op, records, &mut next_touch);
|
||||
let mut recorder = Recorder::new(&summary, op, records, (&mut next_touch, &mut *spent));
|
||||
apply_op(doc, &mut recorder)
|
||||
.and_then(|()| assert_depth(doc))
|
||||
.and_then(|()| assert_recorded(records, tally))
|
||||
.map_err(|error| fail(format!("{} op {index}: {error}", change.id)))?;
|
||||
}
|
||||
if change.level.needs_confirmation() {
|
||||
mark_to_confirm(doc, &summary, records)?;
|
||||
assert_recorded(records, tally).map_err(|error| fail(format!("{}: {error}", change.id)))?;
|
||||
mark_to_confirm(doc, &summary, records, spent)
|
||||
.map_err(|error| fail(format!("{}: {error}", change.id)))?;
|
||||
}
|
||||
strip_marks(doc);
|
||||
Ok(())
|
||||
@@ -235,9 +219,9 @@ pub fn apply_changeset(
|
||||
.ok_or_else(|| fail("the baseline is not an object"))?
|
||||
.insert("references".to_owned(), registry);
|
||||
let mut records = Vec::new();
|
||||
let mut tally = Tally::default();
|
||||
let mut spent = 0;
|
||||
for change in changes {
|
||||
apply_change(&mut doc, change, &mut records, &mut tally)?;
|
||||
apply_change(&mut doc, change, &mut records, &mut spent)?;
|
||||
}
|
||||
curated_issues(&doc)?;
|
||||
Ok(CurationResult {
|
||||
|
||||
@@ -118,7 +118,7 @@ fn set_self(
|
||||
recorder.touch(item);
|
||||
let before = read_field(item, &op.field)?;
|
||||
write_field(item, &op.field, &op.value)?;
|
||||
recorder.record(&op.target, Some(&op.field), before, Some(op.value.clone()));
|
||||
recorder.record(&op.target, Some(&op.field), before, Some(op.value.clone()))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -163,7 +163,7 @@ fn set_subtree(
|
||||
Some(&op.field),
|
||||
before,
|
||||
Some(op.value.clone()),
|
||||
);
|
||||
)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -199,7 +199,7 @@ fn apply_rename(
|
||||
item.insert(key.to_owned(), Value::from(op.id.clone()));
|
||||
recorder.touch(item);
|
||||
let after = Value::Object(item.clone());
|
||||
recorder.record(&op.target, None, Some(before), Some(after));
|
||||
recorder.record(&op.target, None, Some(before), Some(after))?;
|
||||
let rename = Rename {
|
||||
kind: located.target.kind,
|
||||
from,
|
||||
@@ -331,7 +331,7 @@ fn apply_add(
|
||||
added.push(Step::Index(index));
|
||||
recorder.touch_tree(doc, &added)?;
|
||||
let target = format!("{}{id}", dest.prefix);
|
||||
recorder.record(&target, None, None, Some(op.value.clone()));
|
||||
recorder.record(&target, None, None, Some(op.value.clone()))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -345,7 +345,7 @@ fn apply_remove(
|
||||
return Err(fail("the document cannot be removed"));
|
||||
};
|
||||
let item = list_at_mut(doc, list)?.remove(located.index);
|
||||
recorder.record(&op.target, None, Some(item), None);
|
||||
recorder.record(&op.target, None, Some(item), None)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -383,7 +383,7 @@ fn apply_move(
|
||||
recorder.touch(item);
|
||||
let before = Value::Object(item.clone());
|
||||
let after = format!("{}{id}", child_prefix(&parent));
|
||||
recorder.record(&op.target, None, Some(before), Some(Value::from(after)));
|
||||
recorder.record(&op.target, None, Some(before), Some(Value::from(after)))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -449,7 +449,7 @@ fn apply_cite(
|
||||
Some("reference_ids"),
|
||||
Some(Value::Array(before)),
|
||||
Some(Value::Array(after)),
|
||||
);
|
||||
)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -199,13 +199,26 @@ fn view_ids(document: &Value) -> Vec<String> {
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// The review drawings' bytes, projected before any is rendered: every drawing repeats the
|
||||
/// whole cleaned drawing, two per view (the equine draft: 2 views, about 0.55 MB). Views added
|
||||
/// by a changeset multiply it (2,000 views asked for 547 MB, security gate R5.4 round 8).
|
||||
pub const MAX_REVIEW_DRAWINGS_BYTES: usize = 64 * 1024 * 1024;
|
||||
|
||||
/// One review overlay per view (`review/annotations-<view>.svg`), then the application's
|
||||
/// drawing of each view (`review/app-<view>.svg`).
|
||||
fn review_drawings(draft: &DraftResult) -> Result<Vec<(String, String)>, PipelineError> {
|
||||
let views = view_ids(&draft.document);
|
||||
let projected = draft.cleaned_svg.len().saturating_mul(2 * views.len());
|
||||
if projected > MAX_REVIEW_DRAWINGS_BYTES {
|
||||
return Err(fail(format!(
|
||||
"{} views would write {projected} bytes of review drawings, past the \
|
||||
{MAX_REVIEW_DRAWINGS_BYTES} bound",
|
||||
views.len()
|
||||
)));
|
||||
}
|
||||
let drawing = read_cleaned_drawing(&draft.cleaned_svg)?;
|
||||
let typed: AnatomicalSheetDocument = serde_json::from_value(draft.document.clone())
|
||||
.map_err(|error| fail(format!("curated document: {error}")))?;
|
||||
let views = view_ids(&draft.document);
|
||||
let mut files = Vec::new();
|
||||
for view_id in &views {
|
||||
let overlay = render_review_overlay(&OverlayInput {
|
||||
|
||||
@@ -33,6 +33,34 @@ pub struct CurationResult {
|
||||
pub records: Vec<ChangeRecord>,
|
||||
}
|
||||
|
||||
/// The total weight of the records (`record_weight`): about 16 MiB of text, or a quarter of a
|
||||
/// million JSON values, far past the equine changeset (test `the_records_cannot_outgrow_their_bound`)
|
||||
/// and small enough that the reports built from them stay bounded too. Charged as each record
|
||||
/// is made, before the change and the op are cloned into it: one op touching many items is
|
||||
/// refused at the record that passes the bound.
|
||||
pub const MAX_RECORDED_WEIGHT: usize = 16 * 1024 * 1024;
|
||||
|
||||
/// The records' running weight; refuses the record that would pass `MAX_RECORDED_WEIGHT`.
|
||||
pub fn charge(spent: &mut usize, record: RecordParts<'_>) -> Result<(), PipelineError> {
|
||||
*spent = spent.saturating_add(parts_weight(&record));
|
||||
if *spent > MAX_RECORDED_WEIGHT {
|
||||
return Err(PipelineError::Guard(format!(
|
||||
"the records weigh more than {MAX_RECORDED_WEIGHT} units"
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// What a record is made of, borrowed: weighed before anything is cloned.
|
||||
pub struct RecordParts<'a> {
|
||||
pub change: &'a ChangeSummary,
|
||||
pub op: &'a Op,
|
||||
pub target: &'a str,
|
||||
pub field: Option<&'a str>,
|
||||
pub before: &'a Value,
|
||||
pub after: &'a Value,
|
||||
}
|
||||
|
||||
/// What the op functions of one change share besides the document; the touch counter spans
|
||||
/// the ops of the change (the first touch of an item fixes its place in the review marking).
|
||||
pub struct Recorder<'a> {
|
||||
@@ -40,6 +68,7 @@ pub struct Recorder<'a> {
|
||||
pub op: &'a Op,
|
||||
pub records: &'a mut Vec<ChangeRecord>,
|
||||
next_touch: &'a mut u64,
|
||||
spent: &'a mut usize,
|
||||
}
|
||||
|
||||
impl<'a> Recorder<'a> {
|
||||
@@ -47,32 +76,45 @@ impl<'a> Recorder<'a> {
|
||||
change: &'a ChangeSummary,
|
||||
op: &'a Op,
|
||||
records: &'a mut Vec<ChangeRecord>,
|
||||
next_touch: &'a mut u64,
|
||||
counters: (&'a mut u64, &'a mut usize),
|
||||
) -> Self {
|
||||
Self {
|
||||
change,
|
||||
op,
|
||||
records,
|
||||
next_touch,
|
||||
next_touch: counters.0,
|
||||
spent: counters.1,
|
||||
}
|
||||
}
|
||||
|
||||
/// Absent values are recorded as null so records stay plain JSON.
|
||||
/// Absent values are recorded as null so records stay plain JSON. Refused when the
|
||||
/// records would pass their bound (`charge`), before the change and the op are cloned.
|
||||
pub fn record(
|
||||
&mut self,
|
||||
target: &str,
|
||||
field: Option<&str>,
|
||||
before: Option<Value>,
|
||||
after: Option<Value>,
|
||||
) {
|
||||
) -> Result<(), PipelineError> {
|
||||
let (before, after) = (unmarked(before), unmarked(after));
|
||||
let parts = RecordParts {
|
||||
change: self.change,
|
||||
op: self.op,
|
||||
target,
|
||||
field,
|
||||
before: &before,
|
||||
after: &after,
|
||||
};
|
||||
charge(self.spent, parts)?;
|
||||
self.records.push(ChangeRecord {
|
||||
change: self.change.clone(),
|
||||
op: self.op.clone(),
|
||||
target: target.to_owned(),
|
||||
field: field.map(str::to_owned),
|
||||
before: unmarked(before),
|
||||
after: unmarked(after),
|
||||
before,
|
||||
after,
|
||||
});
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Marks an item as touched by the change, once (the first touch keeps its order).
|
||||
@@ -105,29 +147,48 @@ impl<'a> Recorder<'a> {
|
||||
}
|
||||
}
|
||||
|
||||
/// What the records hold, in weight units: 64 per JSON value (a `Value` node costs tens of
|
||||
/// bytes) plus one per byte of every string (the values' and the change's texts, which each
|
||||
/// record clones). A rename or a remove records the whole item, so the records can grow far
|
||||
/// past the document; `apply` bounds their total (`MAX_RECORDED_WEIGHT`).
|
||||
/// What a record holds, in weight units: 64 per JSON value (a `Value` node costs tens of
|
||||
/// bytes) plus one per byte of every string: the values, and the change and the op, which each
|
||||
/// record clones. A rename or a remove records the whole item and one op can record once per
|
||||
/// item it reaches, so the records can grow far past the document: `charge` bounds their total.
|
||||
pub fn record_weight(record: &ChangeRecord) -> usize {
|
||||
let change = &record.change;
|
||||
parts_weight(&RecordParts {
|
||||
change: &record.change,
|
||||
op: &record.op,
|
||||
target: &record.target,
|
||||
field: record.field.as_deref(),
|
||||
before: &record.before,
|
||||
after: &record.after,
|
||||
})
|
||||
}
|
||||
|
||||
fn parts_weight(parts: &RecordParts<'_>) -> usize {
|
||||
let change = parts.change;
|
||||
let texts = [
|
||||
change.id.as_str(),
|
||||
&change.title.fr,
|
||||
&change.title.en,
|
||||
&change.rationale,
|
||||
change.question.as_deref().unwrap_or_default(),
|
||||
&record.target,
|
||||
record.field.as_deref().unwrap_or_default(),
|
||||
parts.target,
|
||||
parts.field.unwrap_or_default(),
|
||||
];
|
||||
let text: usize = texts.iter().map(|text| text.len()).sum::<usize>()
|
||||
+ change.reference_ids.iter().map(String::len).sum::<usize>();
|
||||
let op_value = match &record.op {
|
||||
Op::Set(op) => value_weight(&op.value),
|
||||
text + op_weight(parts.op) + value_weight(parts.before) + value_weight(parts.after)
|
||||
}
|
||||
|
||||
/// The op a record clones: its target, its value or its ids.
|
||||
fn op_weight(op: &Op) -> usize {
|
||||
let carried = match op {
|
||||
Op::Set(op) => op.field.len() + value_weight(&op.value),
|
||||
Op::Add(op) => value_weight(&op.value),
|
||||
_ => 0,
|
||||
Op::Rename(op) => op.id.len(),
|
||||
Op::Move(op) => op.to.len(),
|
||||
Op::Cite(op) => op.reference_ids.iter().map(|id| 64 + id.len()).sum(),
|
||||
Op::Remove(_) => 0,
|
||||
};
|
||||
text + op_value + value_weight(&record.before) + value_weight(&record.after)
|
||||
64 + op.target().len() + carried
|
||||
}
|
||||
|
||||
/// The weight of a value, walked with an explicit stack (no recursion).
|
||||
|
||||
@@ -74,7 +74,7 @@ fn follow_link(
|
||||
Some(link.field),
|
||||
Some(from.clone()),
|
||||
Some(Value::from(rename.to.as_str())),
|
||||
);
|
||||
)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -109,7 +109,7 @@ fn follow_citations(
|
||||
Some("reference_ids"),
|
||||
Some(before),
|
||||
Some(Value::Array(after.clone())),
|
||||
);
|
||||
)?;
|
||||
item.insert("reference_ids".to_owned(), Value::Array(after));
|
||||
}
|
||||
Ok(())
|
||||
|
||||
@@ -764,14 +764,26 @@ fn an_integer_beyond_2_53_is_read_as_the_double_json_parse_gives() {
|
||||
);
|
||||
}
|
||||
|
||||
/// The records bound now stops a growing changeset first (every added value is recorded); the
|
||||
/// document bound still holds a baseline near it: one small add past 200,000 values is refused.
|
||||
#[test]
|
||||
fn the_document_cannot_hold_more_values_than_the_bound() {
|
||||
let children: Vec<Value> = (0..70_000)
|
||||
.map(|i| json!({ "id": format!("n{i}"), "name": { "fr": "a", "en": "a" } }))
|
||||
.collect();
|
||||
let value = json!({ "id": "many", "name": { "fr": "a", "en": "a" }, "children": children });
|
||||
let op = json!({ "op": "add", "target": MOBILITY, "value": value });
|
||||
let error = mini_apply(json!([op])).unwrap_err();
|
||||
let frozen = oracle("curation-synthetic.json");
|
||||
let references = parse_references(&to_js_compact(&frozen["references_file"]).unwrap()).unwrap();
|
||||
let mut baseline = frozen["baseline"].clone();
|
||||
baseline["padding"] = Value::from(vec![Value::Null; 199_950]);
|
||||
let value = json!({ "id": "small", "name": { "fr": "a", "en": "a" },
|
||||
"children": (0..20).map(|i| json!({ "id": format!("n{i}"), "name": { "fr": "a", "en": "a" } }))
|
||||
.collect::<Vec<_>>() });
|
||||
let file = json!({ "changes": [{
|
||||
"id": "CHG-001", "level": "A", "kind": "technical",
|
||||
"title": { "fr": "Essai", "en": "Test" }, "rationale": "Essai",
|
||||
"reference_ids": [], "ops": [{ "op": "add", "target": MOBILITY, "value": value }],
|
||||
}]});
|
||||
let changeset = parse_changeset(&to_js_compact(&file).unwrap()).unwrap();
|
||||
let error = apply_changeset(&baseline, &references, &changeset.changes)
|
||||
.unwrap_err()
|
||||
.to_string();
|
||||
assert!(error.contains("more than 200000 values"), "{error}");
|
||||
}
|
||||
|
||||
@@ -780,7 +792,7 @@ fn the_document_cannot_hold_more_values_than_the_bound() {
|
||||
/// 5,000-child item asked for 4.3 GB). The equine changeset stays far below the bound.
|
||||
#[test]
|
||||
fn the_records_cannot_outgrow_their_bound() {
|
||||
use faunosteo_core::curation::draft::apply::MAX_RECORDED_WEIGHT;
|
||||
use faunosteo_core::curation::draft::records::MAX_RECORDED_WEIGHT;
|
||||
use faunosteo_core::curation::draft::records::record_weight;
|
||||
let equine: usize = equine_draft().records.iter().map(record_weight).sum();
|
||||
println!("equine records weigh {equine} units");
|
||||
@@ -802,6 +814,42 @@ fn the_records_cannot_outgrow_their_bound() {
|
||||
assert!(error.contains("the records weigh more than"), "{error}");
|
||||
}
|
||||
|
||||
/// One op, or the B/C marking, records once per item it reaches: the bound is charged at each
|
||||
/// record, before the change and the op are cloned into it (security gate R5.4 round 8: a level
|
||||
/// B add of 10,000 children with a 128 KiB rationale reached 1.37 GB inside one marking).
|
||||
#[test]
|
||||
fn the_records_bound_holds_inside_one_op_and_the_marking() {
|
||||
let children: Vec<Value> = (0..5_000)
|
||||
.map(|i| json!({ "id": format!("n{i}"), "name": { "fr": "a", "en": "a" } }))
|
||||
.collect();
|
||||
let value = json!({ "id": "many", "name": { "fr": "a", "en": "a" }, "children": children });
|
||||
let add = json!({ "op": "add", "target": MOBILITY, "value": value });
|
||||
let frozen = oracle("curation-synthetic.json");
|
||||
let references = parse_references(&to_js_compact(&frozen["references_file"]).unwrap()).unwrap();
|
||||
let description = json!({ "fr": "d".repeat(4_096), "en": "d" });
|
||||
let subtree = json!({ "op": "set", "target": format!("{MOBILITY}/many"), "field": "description",
|
||||
"value": description, "scope": "subtree" });
|
||||
let cases = [
|
||||
("B", "r".repeat(8_192), json!([add])),
|
||||
("A", "Essai".to_owned(), json!([add, subtree])),
|
||||
];
|
||||
for (level, rationale, ops) in cases {
|
||||
let file = json!({ "changes": [{
|
||||
"id": "CHG-001", "level": level, "kind": "technical",
|
||||
"title": { "fr": "Essai", "en": "Test" }, "rationale": rationale,
|
||||
"reference_ids": [], "ops": ops,
|
||||
}]});
|
||||
let changeset = parse_changeset(&to_js_compact(&file).unwrap()).unwrap();
|
||||
let error = apply_changeset(&frozen["baseline"], &references, &changeset.changes)
|
||||
.unwrap_err()
|
||||
.to_string();
|
||||
assert!(
|
||||
error.contains("the records weigh more than"),
|
||||
"{level}: {error}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// A `__proto__` segment had no faithful reading in the TypeScript (a prototype set, a
|
||||
/// polluted `Object.prototype`, or nothing written): the port refuses it at the op.
|
||||
#[test]
|
||||
|
||||
@@ -496,3 +496,24 @@ fn ids_and_field_segments_that_could_forge_report_structure_are_refused() {
|
||||
let rule = TriggerNames::of(&typed(&draft.document));
|
||||
assert!(compare_drawing("<svg/>", &deep, Some(&rule)).is_err());
|
||||
}
|
||||
|
||||
/// Every review drawing repeats the whole cleaned drawing, two per view: the bytes are
|
||||
/// projected and bounded before any drawing is rendered (security gate R5.4 round 8).
|
||||
#[test]
|
||||
fn review_drawings_past_their_bound_are_refused_before_rendering() {
|
||||
let (result, mut draft) = equine();
|
||||
let views = draft.document["views"].as_array_mut().unwrap();
|
||||
let main = views[0].clone();
|
||||
for index in 0..300 {
|
||||
let mut view = main.clone();
|
||||
view["id"] = Value::from(format!("extra-{index}"));
|
||||
views.push(view);
|
||||
}
|
||||
let error = render_draft_artifacts(&draft, &result.anomalies).unwrap_err();
|
||||
assert!(
|
||||
error
|
||||
.to_string()
|
||||
.contains("bytes of review drawings, past the"),
|
||||
"{error}"
|
||||
);
|
||||
}
|
||||
Reference in new issue
Block a user