fix(anatomy): R5.3 shapes refused as Zod did (struct as array, enum as map), reads bounded to regular files, --check never follows links, plain names in --out
This commit is contained in:
1 parent
314cf04e6a
commit
776c7095b3
9 files changed
+258
-25
No files matched your search
@@ -57,8 +57,21 @@ struct Folder {
|
||||
files: Vec<FolderFile>,
|
||||
}
|
||||
|
||||
/// A bound no source or generated file approaches (the largest, the draft CHANGELOG, is about
|
||||
/// 1.1 MB): a resource cap on what the tool reads.
|
||||
const MAX_FILE_BYTES: u64 = 16 * 1024 * 1024;
|
||||
|
||||
/// A regular file of the seed (never a symbolic link, a device or a pipe), read under the cap.
|
||||
fn read(path: &Path) -> Result<Vec<u8>, String> {
|
||||
fs::read(path).map_err(|error| format!("{}: {error}", path.display()))
|
||||
let refuse = |why: &str| format!("{}: {why}", path.display());
|
||||
let metadata = fs::symlink_metadata(path).map_err(|error| refuse(&error.to_string()))?;
|
||||
if !metadata.file_type().is_file() {
|
||||
return Err(refuse("not a regular file"));
|
||||
}
|
||||
if metadata.len() > MAX_FILE_BYTES {
|
||||
return Err(refuse(&format!("larger than {MAX_FILE_BYTES} bytes")));
|
||||
}
|
||||
fs::read(path).map_err(|error| refuse(&error.to_string()))
|
||||
}
|
||||
|
||||
/// `readFile(path, 'utf8')`: invalid UTF-8 becomes U+FFFD, as the TypeScript read the
|
||||
@@ -145,7 +158,9 @@ fn listed(dir: &Path, prefix: &str, out: &mut Vec<String>) -> Result<(), String>
|
||||
for entry in entries {
|
||||
let entry = entry.map_err(|error| error.to_string())?;
|
||||
let name = format!("{prefix}{}", entry.file_name().to_string_lossy());
|
||||
if entry.path().is_dir() {
|
||||
// `file_type` does not follow a symbolic link: a link is listed, never walked into.
|
||||
let is_dir = entry.file_type().is_ok_and(|kind| kind.is_dir());
|
||||
if is_dir {
|
||||
listed(&entry.path(), &format!("{name}/"), out)?;
|
||||
} else {
|
||||
out.push(name);
|
||||
@@ -160,7 +175,7 @@ fn check(folder: &Folder) -> Result<Vec<String>, String> {
|
||||
let target = Path::new(SEED).join(folder.name);
|
||||
let mut problems = Vec::new();
|
||||
for (name, bytes) in &folder.files {
|
||||
match fs::read(target.join(name)) {
|
||||
match read(&target.join(name)) {
|
||||
Ok(committed) if committed == *bytes => {}
|
||||
Ok(_) => problems.push(format!("{}: differs: {name}", folder.name)),
|
||||
Err(_) => problems.push(format!("{}: missing: {name}", folder.name)),
|
||||
@@ -205,7 +220,7 @@ fn claim_destination(typed: &Path) -> Result<PathBuf, String> {
|
||||
None => return Err(refuse("not a destination folder")),
|
||||
};
|
||||
let seed = fs::canonicalize(SEED).map_err(|error| format!("{SEED}: {error}"))?;
|
||||
if parent.starts_with(&seed) {
|
||||
if inside(&parent, &seed) {
|
||||
return Err(refuse("inside the seed folder"));
|
||||
}
|
||||
let name = out
|
||||
@@ -227,6 +242,11 @@ fn claim_destination(typed: &Path) -> Result<PathBuf, String> {
|
||||
Ok(root)
|
||||
}
|
||||
|
||||
/// Whether a canonical folder is the seed folder or lies under it.
|
||||
fn inside(folder: &Path, seed: &Path) -> bool {
|
||||
folder.starts_with(seed)
|
||||
}
|
||||
|
||||
/// Where a relative destination is typed from: `INIT_CWD` when pnpm set it to an absolute
|
||||
/// path, else the current folder (the repository root when the script is run directly: it
|
||||
/// moves there first).
|
||||
@@ -242,8 +262,18 @@ fn base_dir() -> PathBuf {
|
||||
/// appeared since the claim, file, folder or symbolic link, is refused, never followed or
|
||||
/// truncated (`create_dir`, `create_new`).
|
||||
fn write_exclusive(root: &Path, relative: &Path, bytes: &[u8]) -> Result<(), String> {
|
||||
// The claimed folder is checked again: a swap for a symbolic link since the claim is refused.
|
||||
if !fs::symlink_metadata(root).is_ok_and(|m| m.file_type().is_dir()) {
|
||||
return Err(format!("{}: no longer the claimed folder", root.display()));
|
||||
}
|
||||
let mut folder = root.to_path_buf();
|
||||
let components: Vec<_> = relative.components().collect();
|
||||
if !components.iter().all(|c| matches!(c, Component::Normal(_))) {
|
||||
return Err(format!(
|
||||
"{}: not a plain relative file name",
|
||||
relative.display()
|
||||
));
|
||||
}
|
||||
let (file, folders) = components.split_last().ok_or("an empty file name")?;
|
||||
for component in folders {
|
||||
folder.push(component);
|
||||
@@ -327,3 +357,25 @@ fn main() {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn a_folder_under_the_seed_or_the_seed_itself_is_inside_it() {
|
||||
let seed = Path::new("/repo/supabase/seed/anatomical-sheets/equine");
|
||||
assert!(inside(seed, seed));
|
||||
assert!(inside(&seed.join("draft"), seed));
|
||||
assert!(inside(&seed.join("x/y"), seed));
|
||||
assert!(!inside(
|
||||
Path::new("/repo/supabase/seed/anatomical-sheets"),
|
||||
seed
|
||||
));
|
||||
assert!(!inside(
|
||||
Path::new("/repo/supabase/seed/anatomical-sheets/equine-copy"),
|
||||
seed
|
||||
));
|
||||
assert!(!inside(Path::new("/tmp/out"), seed));
|
||||
}
|
||||
}
|
||||
@@ -16,6 +16,7 @@ use serde_json::Value;
|
||||
use super::ops::{apply_op, assert_declared};
|
||||
use super::records::{ChangeRecord, CurationResult, Recorder, strip_marks, touch_order};
|
||||
use super::schema::{Change, ChangeSummary, Op, SetOp};
|
||||
use super::shape::assert_same_shape;
|
||||
use super::targets::{
|
||||
Path, Step, TargetKind, all_findings, at, fail, item_at_mut, list_at, top_collection_path,
|
||||
};
|
||||
@@ -159,6 +160,7 @@ fn describe(issues: &[SheetIssue]) -> String {
|
||||
pub fn curated_issues(doc: &Value) -> Result<Vec<SheetIssue>, PipelineError> {
|
||||
let parsed: AnatomicalSheetDocument = serde_json::from_value(doc.clone())
|
||||
.map_err(|error| fail(format!("curated document is not shape-valid: {error}")))?;
|
||||
assert_same_shape("curated document is not shape-valid", doc, &parsed)?;
|
||||
let issues = validate_sheet_document(&parsed);
|
||||
let errors: Vec<SheetIssue> = issues
|
||||
.iter()
|
||||
|
||||
@@ -13,6 +13,7 @@ pub mod records;
|
||||
pub mod rename;
|
||||
pub mod schema;
|
||||
pub mod seed;
|
||||
pub mod shape;
|
||||
pub mod stats;
|
||||
pub mod targets;
|
||||
|
||||
|
||||
@@ -12,6 +12,7 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
use super::shape::assert_same_shape;
|
||||
use crate::curation::PipelineError;
|
||||
use crate::domain::js;
|
||||
use crate::domain::sheet::Reference;
|
||||
@@ -246,7 +247,7 @@ pub struct ChangeSummary {
|
||||
}
|
||||
|
||||
/// One authored change: `ChangeSchema` (strict, `priority` defaulted, `ops` non-empty).
|
||||
#[derive(Debug, Clone, PartialEq, Deserialize)]
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Change {
|
||||
#[serde(deserialize_with = "change_id")]
|
||||
@@ -258,7 +259,11 @@ pub struct Change {
|
||||
pub rationale: String,
|
||||
#[serde(deserialize_with = "non_empty_strings")]
|
||||
pub reference_ids: Vec<String>,
|
||||
#[serde(default, deserialize_with = "optional_non_empty")]
|
||||
#[serde(
|
||||
default,
|
||||
deserialize_with = "optional_non_empty",
|
||||
skip_serializing_if = "Option::is_none"
|
||||
)]
|
||||
pub question: Option<String>,
|
||||
#[serde(default = "default_priority")]
|
||||
pub priority: ChangePriority,
|
||||
@@ -286,13 +291,13 @@ impl Change {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Deserialize)]
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Changeset {
|
||||
pub changes: Vec<Change>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Deserialize)]
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct ReferencesFile {
|
||||
references: Vec<Reference>,
|
||||
@@ -407,7 +412,8 @@ pub fn parse_changeset(text: &str) -> Result<Changeset, PipelineError> {
|
||||
return Err(refuse("changes.json", format!("reserved key {key:?}")));
|
||||
}
|
||||
let changeset: Changeset =
|
||||
serde_json::from_value(value).map_err(|e| refuse("changes.json", e))?;
|
||||
serde_json::from_value(value.clone()).map_err(|e| refuse("changes.json", e))?;
|
||||
assert_same_shape("changes.json", &value, &changeset)?;
|
||||
for change in &changeset.changes {
|
||||
if change.level == ChangeLevel::C && change.question.is_none() {
|
||||
return Err(refuse(
|
||||
@@ -427,7 +433,8 @@ pub fn parse_changeset(text: &str) -> Result<Changeset, PipelineError> {
|
||||
pub fn parse_references(text: &str) -> Result<Vec<Reference>, PipelineError> {
|
||||
let value = json_parse("references.json", text)?;
|
||||
let file: ReferencesFile =
|
||||
serde_json::from_value(value).map_err(|e| refuse("references.json", e))?;
|
||||
serde_json::from_value(value.clone()).map_err(|e| refuse("references.json", e))?;
|
||||
assert_same_shape("references.json", &value, &file)?;
|
||||
let ids = file
|
||||
.references
|
||||
.iter()
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
//! The JSON shapes Zod refused and serde's derives accept. serde reads a struct from a
|
||||
//! positional array (`"title": ["Titre", "Title"]`) and a unit enum variant from a one-key map
|
||||
//! (`"level": {"A": null}`); Zod refused both (`invalid_type`). The authored files and the
|
||||
//! curated document are therefore compared, by shape, with the typed value serialised back:
|
||||
//! an object must stay an object, an array an array of the same length, a scalar a scalar of
|
||||
//! the same kind. Keys the typed value adds (defaults filled in) are allowed.
|
||||
//!
|
||||
//! The R2 contracts carry the same derives; their readers are outside R5.3 (owner's go).
|
||||
|
||||
use serde::Serialize;
|
||||
use serde_json::Value;
|
||||
|
||||
use super::targets::fail;
|
||||
use crate::curation::PipelineError;
|
||||
|
||||
fn kind(value: &Value) -> &'static str {
|
||||
match value {
|
||||
Value::Null => "null",
|
||||
Value::Bool(_) => "a boolean",
|
||||
Value::Number(_) => "a number",
|
||||
Value::String(_) => "a string",
|
||||
Value::Array(_) => "an array",
|
||||
Value::Object(_) => "an object",
|
||||
}
|
||||
}
|
||||
|
||||
fn mismatch(path: &str, typed: &Value, written: &Value) -> String {
|
||||
let at = if path.is_empty() { "the root" } else { path };
|
||||
format!(
|
||||
"expected {} at {at}, received {}",
|
||||
kind(typed),
|
||||
kind(written)
|
||||
)
|
||||
}
|
||||
|
||||
/// The first place where `written` (as parsed) and `typed` (read and serialised back) differ
|
||||
/// in shape.
|
||||
fn first_mismatch(written: &Value, typed: &Value, path: &str) -> Option<String> {
|
||||
match (written, typed) {
|
||||
(Value::Object(w), Value::Object(t)) => w.iter().find_map(|(key, value)| {
|
||||
let here = format!("{path}.{key}");
|
||||
match t.get(key) {
|
||||
Some(typed) => first_mismatch(value, typed, &here),
|
||||
None => Some(format!("unexpected key at {here}")),
|
||||
}
|
||||
}),
|
||||
(Value::Array(w), Value::Array(t)) if w.len() == t.len() => w
|
||||
.iter()
|
||||
.zip(t)
|
||||
.enumerate()
|
||||
.find_map(|(index, (w, t))| first_mismatch(w, t, &format!("{path}.{index}"))),
|
||||
_ if std::mem::discriminant(written) == std::mem::discriminant(typed) => None,
|
||||
_ => Some(mismatch(path, typed, written)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Refuses `written` when its typed reading `typed` does not keep its shape.
|
||||
pub fn assert_same_shape<T: Serialize>(
|
||||
what: &str,
|
||||
written: &Value,
|
||||
typed: &T,
|
||||
) -> Result<(), PipelineError> {
|
||||
let typed = serde_json::to_value(typed).map_err(|error| fail(format!("{what}: {error}")))?;
|
||||
match first_mismatch(written, &typed, "") {
|
||||
None => Ok(()),
|
||||
Some(why) => Err(fail(format!("{what}: {why}"))),
|
||||
}
|
||||
}
|
||||
@@ -190,9 +190,11 @@ pub fn at_mut<'a>(doc: &'a mut Value, path: &[Step]) -> Result<&'a mut Value, Pi
|
||||
/// `listAt`: the array at a path, or "document has no list".
|
||||
pub fn list_at<'a>(doc: &'a Value, path: &[Step]) -> Result<&'a Vec<Value>, PipelineError> {
|
||||
let last = path.last().map(describe).unwrap_or_default();
|
||||
at(doc, path)?
|
||||
let no_list = || fail(format!("document has no list \"{last}\""));
|
||||
at(doc, path)
|
||||
.map_err(|_| no_list())?
|
||||
.as_array()
|
||||
.ok_or_else(|| fail(format!("document has no list \"{last}\"")))
|
||||
.ok_or_else(no_list)
|
||||
}
|
||||
|
||||
pub fn list_at_mut<'a>(
|
||||
@@ -200,9 +202,11 @@ pub fn list_at_mut<'a>(
|
||||
path: &[Step],
|
||||
) -> Result<&'a mut Vec<Value>, PipelineError> {
|
||||
let last = path.last().map(describe).unwrap_or_default();
|
||||
at_mut(doc, path)?
|
||||
let no_list = || fail(format!("document has no list \"{last}\""));
|
||||
at_mut(doc, path)
|
||||
.map_err(|_| no_list())?
|
||||
.as_array_mut()
|
||||
.ok_or_else(|| fail(format!("document has no list \"{last}\"")))
|
||||
.ok_or_else(no_list)
|
||||
}
|
||||
|
||||
pub fn item_at<'a>(doc: &'a Value, path: &[Step]) -> Result<&'a Item, PipelineError> {
|
||||
|
||||
Reference in new issue
Block a user