fix(anatomy): R5.4 records charged at each record, review drawings bounded before rendering
This commit is contained in:
1 parent
5880dfa959
commit
8bdb079f70
10 files changed
+357
-75
No files matched your search
@@ -15,7 +15,7 @@ use serde_json::Value;
|
||||
|
||||
use super::ops::{apply_op, assert_declared};
|
||||
use super::records::{
|
||||
ChangeRecord, CurationResult, Recorder, record_weight, strip_marks, touch_order,
|
||||
ChangeRecord, CurationResult, RecordParts, Recorder, charge, strip_marks, touch_order,
|
||||
};
|
||||
use super::schema::{Change, ChangeSummary, Op, SetOp};
|
||||
use super::shape::assert_same_shape;
|
||||
@@ -67,6 +67,7 @@ fn mark_to_confirm(
|
||||
doc: &mut Value,
|
||||
change: &ChangeSummary,
|
||||
records: &mut Vec<ChangeRecord>,
|
||||
spent: &mut usize,
|
||||
) -> Result<(), PipelineError> {
|
||||
for touched in touched_reviewable(doc)? {
|
||||
let item = item_at_mut(doc, &touched.path)?;
|
||||
@@ -85,64 +86,47 @@ fn mark_to_confirm(
|
||||
value: Value::from("to_confirm"),
|
||||
scope: None,
|
||||
});
|
||||
let after = Value::from("to_confirm");
|
||||
let parts = RecordParts {
|
||||
change,
|
||||
op: &op,
|
||||
target: &touched.target,
|
||||
field: Some("review_status"),
|
||||
before: &before,
|
||||
after: &after,
|
||||
};
|
||||
charge(spent, parts)?;
|
||||
records.push(ChangeRecord {
|
||||
change: change.clone(),
|
||||
op,
|
||||
target: touched.target,
|
||||
field: Some("review_status".to_owned()),
|
||||
before,
|
||||
after: Value::from("to_confirm"),
|
||||
after,
|
||||
});
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// The total weight of the records (`records::record_weight`): about 16 MiB of text, or a
|
||||
/// quarter of a million JSON values, far past the equine changeset (test
|
||||
/// `the_records_cannot_outgrow_their_bound`) and small enough that the reports built from them
|
||||
/// stay bounded too. Checked after every op, before the next one records anything.
|
||||
pub const MAX_RECORDED_WEIGHT: usize = 16 * 1024 * 1024;
|
||||
|
||||
/// The records counted so far and their weight.
|
||||
#[derive(Default)]
|
||||
struct Tally {
|
||||
counted: usize,
|
||||
weight: usize,
|
||||
}
|
||||
|
||||
fn assert_recorded(records: &[ChangeRecord], tally: &mut Tally) -> Result<(), PipelineError> {
|
||||
for record in &records[tally.counted..] {
|
||||
tally.weight = tally.weight.saturating_add(record_weight(record));
|
||||
}
|
||||
tally.counted = records.len();
|
||||
if tally.weight > MAX_RECORDED_WEIGHT {
|
||||
return Err(fail(format!(
|
||||
"the records weigh more than {MAX_RECORDED_WEIGHT} units"
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn apply_change(
|
||||
doc: &mut Value,
|
||||
change: &Change,
|
||||
records: &mut Vec<ChangeRecord>,
|
||||
tally: &mut Tally,
|
||||
spent: &mut usize,
|
||||
) -> Result<(), PipelineError> {
|
||||
let summary = change.summary();
|
||||
assert_declared(doc, &change.reference_ids)
|
||||
.map_err(|error| fail(format!("{}: {error}", change.id)))?;
|
||||
let mut next_touch = 0;
|
||||
for (index, op) in change.ops.iter().enumerate() {
|
||||
let mut recorder = Recorder::new(&summary, op, records, &mut next_touch);
|
||||
let mut recorder = Recorder::new(&summary, op, records, (&mut next_touch, &mut *spent));
|
||||
apply_op(doc, &mut recorder)
|
||||
.and_then(|()| assert_depth(doc))
|
||||
.and_then(|()| assert_recorded(records, tally))
|
||||
.map_err(|error| fail(format!("{} op {index}: {error}", change.id)))?;
|
||||
}
|
||||
if change.level.needs_confirmation() {
|
||||
mark_to_confirm(doc, &summary, records)?;
|
||||
assert_recorded(records, tally).map_err(|error| fail(format!("{}: {error}", change.id)))?;
|
||||
mark_to_confirm(doc, &summary, records, spent)
|
||||
.map_err(|error| fail(format!("{}: {error}", change.id)))?;
|
||||
}
|
||||
strip_marks(doc);
|
||||
Ok(())
|
||||
@@ -235,9 +219,9 @@ pub fn apply_changeset(
|
||||
.ok_or_else(|| fail("the baseline is not an object"))?
|
||||
.insert("references".to_owned(), registry);
|
||||
let mut records = Vec::new();
|
||||
let mut tally = Tally::default();
|
||||
let mut spent = 0;
|
||||
for change in changes {
|
||||
apply_change(&mut doc, change, &mut records, &mut tally)?;
|
||||
apply_change(&mut doc, change, &mut records, &mut spent)?;
|
||||
}
|
||||
curated_issues(&doc)?;
|
||||
Ok(CurationResult {
|
||||
|
||||
@@ -118,7 +118,7 @@ fn set_self(
|
||||
recorder.touch(item);
|
||||
let before = read_field(item, &op.field)?;
|
||||
write_field(item, &op.field, &op.value)?;
|
||||
recorder.record(&op.target, Some(&op.field), before, Some(op.value.clone()));
|
||||
recorder.record(&op.target, Some(&op.field), before, Some(op.value.clone()))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -163,7 +163,7 @@ fn set_subtree(
|
||||
Some(&op.field),
|
||||
before,
|
||||
Some(op.value.clone()),
|
||||
);
|
||||
)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -199,7 +199,7 @@ fn apply_rename(
|
||||
item.insert(key.to_owned(), Value::from(op.id.clone()));
|
||||
recorder.touch(item);
|
||||
let after = Value::Object(item.clone());
|
||||
recorder.record(&op.target, None, Some(before), Some(after));
|
||||
recorder.record(&op.target, None, Some(before), Some(after))?;
|
||||
let rename = Rename {
|
||||
kind: located.target.kind,
|
||||
from,
|
||||
@@ -331,7 +331,7 @@ fn apply_add(
|
||||
added.push(Step::Index(index));
|
||||
recorder.touch_tree(doc, &added)?;
|
||||
let target = format!("{}{id}", dest.prefix);
|
||||
recorder.record(&target, None, None, Some(op.value.clone()));
|
||||
recorder.record(&target, None, None, Some(op.value.clone()))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -345,7 +345,7 @@ fn apply_remove(
|
||||
return Err(fail("the document cannot be removed"));
|
||||
};
|
||||
let item = list_at_mut(doc, list)?.remove(located.index);
|
||||
recorder.record(&op.target, None, Some(item), None);
|
||||
recorder.record(&op.target, None, Some(item), None)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -383,7 +383,7 @@ fn apply_move(
|
||||
recorder.touch(item);
|
||||
let before = Value::Object(item.clone());
|
||||
let after = format!("{}{id}", child_prefix(&parent));
|
||||
recorder.record(&op.target, None, Some(before), Some(Value::from(after)));
|
||||
recorder.record(&op.target, None, Some(before), Some(Value::from(after)))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -449,7 +449,7 @@ fn apply_cite(
|
||||
Some("reference_ids"),
|
||||
Some(Value::Array(before)),
|
||||
Some(Value::Array(after)),
|
||||
);
|
||||
)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -199,13 +199,26 @@ fn view_ids(document: &Value) -> Vec<String> {
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// The review drawings' bytes, projected before any is rendered: every drawing repeats the
|
||||
/// whole cleaned drawing, two per view (the equine draft: 2 views, about 0.55 MB). Views added
|
||||
/// by a changeset multiply it (2,000 views asked for 547 MB, security gate R5.4 round 8).
|
||||
pub const MAX_REVIEW_DRAWINGS_BYTES: usize = 64 * 1024 * 1024;
|
||||
|
||||
/// One review overlay per view (`review/annotations-<view>.svg`), then the application's
|
||||
/// drawing of each view (`review/app-<view>.svg`).
|
||||
fn review_drawings(draft: &DraftResult) -> Result<Vec<(String, String)>, PipelineError> {
|
||||
let views = view_ids(&draft.document);
|
||||
let projected = draft.cleaned_svg.len().saturating_mul(2 * views.len());
|
||||
if projected > MAX_REVIEW_DRAWINGS_BYTES {
|
||||
return Err(fail(format!(
|
||||
"{} views would write {projected} bytes of review drawings, past the \
|
||||
{MAX_REVIEW_DRAWINGS_BYTES} bound",
|
||||
views.len()
|
||||
)));
|
||||
}
|
||||
let drawing = read_cleaned_drawing(&draft.cleaned_svg)?;
|
||||
let typed: AnatomicalSheetDocument = serde_json::from_value(draft.document.clone())
|
||||
.map_err(|error| fail(format!("curated document: {error}")))?;
|
||||
let views = view_ids(&draft.document);
|
||||
let mut files = Vec::new();
|
||||
for view_id in &views {
|
||||
let overlay = render_review_overlay(&OverlayInput {
|
||||
|
||||
@@ -33,6 +33,34 @@ pub struct CurationResult {
|
||||
pub records: Vec<ChangeRecord>,
|
||||
}
|
||||
|
||||
/// The total weight of the records (`record_weight`): about 16 MiB of text, or a quarter of a
|
||||
/// million JSON values, far past the equine changeset (test `the_records_cannot_outgrow_their_bound`)
|
||||
/// and small enough that the reports built from them stay bounded too. Charged as each record
|
||||
/// is made, before the change and the op are cloned into it: one op touching many items is
|
||||
/// refused at the record that passes the bound.
|
||||
pub const MAX_RECORDED_WEIGHT: usize = 16 * 1024 * 1024;
|
||||
|
||||
/// The records' running weight; refuses the record that would pass `MAX_RECORDED_WEIGHT`.
|
||||
pub fn charge(spent: &mut usize, record: RecordParts<'_>) -> Result<(), PipelineError> {
|
||||
*spent = spent.saturating_add(parts_weight(&record));
|
||||
if *spent > MAX_RECORDED_WEIGHT {
|
||||
return Err(PipelineError::Guard(format!(
|
||||
"the records weigh more than {MAX_RECORDED_WEIGHT} units"
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// What a record is made of, borrowed: weighed before anything is cloned.
|
||||
pub struct RecordParts<'a> {
|
||||
pub change: &'a ChangeSummary,
|
||||
pub op: &'a Op,
|
||||
pub target: &'a str,
|
||||
pub field: Option<&'a str>,
|
||||
pub before: &'a Value,
|
||||
pub after: &'a Value,
|
||||
}
|
||||
|
||||
/// What the op functions of one change share besides the document; the touch counter spans
|
||||
/// the ops of the change (the first touch of an item fixes its place in the review marking).
|
||||
pub struct Recorder<'a> {
|
||||
@@ -40,6 +68,7 @@ pub struct Recorder<'a> {
|
||||
pub op: &'a Op,
|
||||
pub records: &'a mut Vec<ChangeRecord>,
|
||||
next_touch: &'a mut u64,
|
||||
spent: &'a mut usize,
|
||||
}
|
||||
|
||||
impl<'a> Recorder<'a> {
|
||||
@@ -47,32 +76,45 @@ impl<'a> Recorder<'a> {
|
||||
change: &'a ChangeSummary,
|
||||
op: &'a Op,
|
||||
records: &'a mut Vec<ChangeRecord>,
|
||||
next_touch: &'a mut u64,
|
||||
counters: (&'a mut u64, &'a mut usize),
|
||||
) -> Self {
|
||||
Self {
|
||||
change,
|
||||
op,
|
||||
records,
|
||||
next_touch,
|
||||
next_touch: counters.0,
|
||||
spent: counters.1,
|
||||
}
|
||||
}
|
||||
|
||||
/// Absent values are recorded as null so records stay plain JSON.
|
||||
/// Absent values are recorded as null so records stay plain JSON. Refused when the
|
||||
/// records would pass their bound (`charge`), before the change and the op are cloned.
|
||||
pub fn record(
|
||||
&mut self,
|
||||
target: &str,
|
||||
field: Option<&str>,
|
||||
before: Option<Value>,
|
||||
after: Option<Value>,
|
||||
) {
|
||||
) -> Result<(), PipelineError> {
|
||||
let (before, after) = (unmarked(before), unmarked(after));
|
||||
let parts = RecordParts {
|
||||
change: self.change,
|
||||
op: self.op,
|
||||
target,
|
||||
field,
|
||||
before: &before,
|
||||
after: &after,
|
||||
};
|
||||
charge(self.spent, parts)?;
|
||||
self.records.push(ChangeRecord {
|
||||
change: self.change.clone(),
|
||||
op: self.op.clone(),
|
||||
target: target.to_owned(),
|
||||
field: field.map(str::to_owned),
|
||||
before: unmarked(before),
|
||||
after: unmarked(after),
|
||||
before,
|
||||
after,
|
||||
});
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Marks an item as touched by the change, once (the first touch keeps its order).
|
||||
@@ -105,29 +147,48 @@ impl<'a> Recorder<'a> {
|
||||
}
|
||||
}
|
||||
|
||||
/// What the records hold, in weight units: 64 per JSON value (a `Value` node costs tens of
|
||||
/// bytes) plus one per byte of every string (the values' and the change's texts, which each
|
||||
/// record clones). A rename or a remove records the whole item, so the records can grow far
|
||||
/// past the document; `apply` bounds their total (`MAX_RECORDED_WEIGHT`).
|
||||
/// What a record holds, in weight units: 64 per JSON value (a `Value` node costs tens of
|
||||
/// bytes) plus one per byte of every string: the values, and the change and the op, which each
|
||||
/// record clones. A rename or a remove records the whole item and one op can record once per
|
||||
/// item it reaches, so the records can grow far past the document: `charge` bounds their total.
|
||||
pub fn record_weight(record: &ChangeRecord) -> usize {
|
||||
let change = &record.change;
|
||||
parts_weight(&RecordParts {
|
||||
change: &record.change,
|
||||
op: &record.op,
|
||||
target: &record.target,
|
||||
field: record.field.as_deref(),
|
||||
before: &record.before,
|
||||
after: &record.after,
|
||||
})
|
||||
}
|
||||
|
||||
fn parts_weight(parts: &RecordParts<'_>) -> usize {
|
||||
let change = parts.change;
|
||||
let texts = [
|
||||
change.id.as_str(),
|
||||
&change.title.fr,
|
||||
&change.title.en,
|
||||
&change.rationale,
|
||||
change.question.as_deref().unwrap_or_default(),
|
||||
&record.target,
|
||||
record.field.as_deref().unwrap_or_default(),
|
||||
parts.target,
|
||||
parts.field.unwrap_or_default(),
|
||||
];
|
||||
let text: usize = texts.iter().map(|text| text.len()).sum::<usize>()
|
||||
+ change.reference_ids.iter().map(String::len).sum::<usize>();
|
||||
let op_value = match &record.op {
|
||||
Op::Set(op) => value_weight(&op.value),
|
||||
text + op_weight(parts.op) + value_weight(parts.before) + value_weight(parts.after)
|
||||
}
|
||||
|
||||
/// The op a record clones: its target, its value or its ids.
|
||||
fn op_weight(op: &Op) -> usize {
|
||||
let carried = match op {
|
||||
Op::Set(op) => op.field.len() + value_weight(&op.value),
|
||||
Op::Add(op) => value_weight(&op.value),
|
||||
_ => 0,
|
||||
Op::Rename(op) => op.id.len(),
|
||||
Op::Move(op) => op.to.len(),
|
||||
Op::Cite(op) => op.reference_ids.iter().map(|id| 64 + id.len()).sum(),
|
||||
Op::Remove(_) => 0,
|
||||
};
|
||||
text + op_value + value_weight(&record.before) + value_weight(&record.after)
|
||||
64 + op.target().len() + carried
|
||||
}
|
||||
|
||||
/// The weight of a value, walked with an explicit stack (no recursion).
|
||||
|
||||
@@ -74,7 +74,7 @@ fn follow_link(
|
||||
Some(link.field),
|
||||
Some(from.clone()),
|
||||
Some(Value::from(rename.to.as_str())),
|
||||
);
|
||||
)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -109,7 +109,7 @@ fn follow_citations(
|
||||
Some("reference_ids"),
|
||||
Some(before),
|
||||
Some(Value::Array(after.clone())),
|
||||
);
|
||||
)?;
|
||||
item.insert("reference_ids".to_owned(), Value::Array(after));
|
||||
}
|
||||
Ok(())
|
||||
|
||||
Reference in new issue
Block a user