feat(supabase): R5.5 upload-sheet-assets in Rust, R5.4 caps and FLOOR clarification
Core rules curation::upload (manifest, local destination, bytes through the SVG reader, plan from observed state); binary upload-sheet-assets over ureq (handshake, no overwrite, confirm after verify, bounded retries, key never printed); pnpm assets:upload repointed, e2e:sheet-assets for the local stack. 4 MiB cap on changes.json and references.json. FLOOR clarification for audit prose with its negative proof.
This commit is contained in:
1 parent
637ea6334c
commit
2c445063a4
20 files changed
+1595
-21
No files matched your search
@@ -25,6 +25,9 @@ cli = ["pdf-read", "pdf-write"]
|
||||
# The anatomical pipeline (R5): legacy importer, curation, generated files and their
|
||||
# formatting, the `import-equine` tool. Never enabled by the wasm facade or the worker.
|
||||
curation = ["dep:sha2", "dep:icu_normalizer"]
|
||||
# The sheet-asset upload tool (R5.5): the `upload-sheet-assets` binary, a local-stack operator
|
||||
# tool. The rules are in `curation::upload`; only the binary reaches the network.
|
||||
upload = ["curation", "dep:ureq"]
|
||||
|
||||
[dependencies]
|
||||
serde.workspace = true
|
||||
@@ -36,6 +39,7 @@ ts-rs = { workspace = true, optional = true }
|
||||
miniz_oxide = { workspace = true, optional = true }
|
||||
sha2 = { workspace = true, optional = true }
|
||||
icu_normalizer = { workspace = true, optional = true }
|
||||
ureq = { workspace = true, optional = true }
|
||||
|
||||
[[bin]]
|
||||
name = "report-compare"
|
||||
@@ -47,6 +51,11 @@ name = "import-equine"
|
||||
path = "src/bin/import_equine.rs"
|
||||
required-features = ["curation"]
|
||||
|
||||
[[bin]]
|
||||
name = "upload-sheet-assets"
|
||||
path = "src/bin/upload_sheet_assets/main.rs"
|
||||
required-features = ["upload"]
|
||||
|
||||
[dev-dependencies]
|
||||
# Reads the PDFs the tests compare (the R3 fixtures and the R4 output): objects, streams,
|
||||
# annotations; never in the library.
|
||||
@@ -130,3 +139,15 @@ required-features = ["curation"]
|
||||
[[test]]
|
||||
name = "curation_markdown"
|
||||
required-features = ["curation"]
|
||||
|
||||
[[test]]
|
||||
name = "curation_caps"
|
||||
required-features = ["curation"]
|
||||
|
||||
[[test]]
|
||||
name = "curation_upload"
|
||||
required-features = ["curation"]
|
||||
|
||||
[[test]]
|
||||
name = "upload_tool"
|
||||
required-features = ["upload"]
|
||||
@@ -17,10 +17,11 @@
|
||||
#![allow(clippy::disallowed_methods)]
|
||||
|
||||
use std::fs::OpenOptions;
|
||||
use std::io::Write;
|
||||
use std::io::{Read, Write};
|
||||
use std::path::{Component, Path, PathBuf};
|
||||
use std::{env, fs, process};
|
||||
|
||||
use faunosteo_core::curation::draft::schema::MAX_CURATION_FILE_BYTES;
|
||||
use faunosteo_core::curation::draft::svg_compare::{TriggerNames, compare_drawing};
|
||||
use faunosteo_core::curation::draft::{
|
||||
DraftInput, build_equine_draft, parse_changeset, parse_references, parse_view_box,
|
||||
@@ -60,11 +61,21 @@ struct Folder {
|
||||
/// 1.1 MB): a resource cap on what the tool reads.
|
||||
const MAX_FILE_BYTES: u64 = 16 * 1024 * 1024;
|
||||
|
||||
/// The cap of an authored curation file, the core's own (`MAX_CURATION_FILE_BYTES`): the file
|
||||
/// is never read beyond it, so a larger one is refused before the allocation.
|
||||
const MAX_CURATION_BYTES: u64 = MAX_CURATION_FILE_BYTES as u64;
|
||||
|
||||
/// A regular file of the seed folder, read under the cap: its canonical path (every symbolic
|
||||
/// link resolved, folders included) must lie in the canonical seed folder, so a link never
|
||||
/// leads a read outside it; devices and pipes are refused. The repository is a single user's
|
||||
/// working copy: a swap between the check and the read is not defended (RUST.md §11.13).
|
||||
fn read(path: &Path) -> Result<Vec<u8>, String> {
|
||||
read_capped(path, MAX_FILE_BYTES)
|
||||
}
|
||||
|
||||
/// `read` under a given cap, enforced on the bytes actually read as well as on the size seen
|
||||
/// first, so a file that grows between the two is still refused before it is held.
|
||||
fn read_capped(path: &Path, cap: u64) -> Result<Vec<u8>, String> {
|
||||
let refuse = |why: &str| format!("{}: {why}", path.display());
|
||||
let seed = fs::canonicalize(SEED).map_err(|error| format!("{SEED}: {error}"))?;
|
||||
let canonical = fs::canonicalize(path).map_err(|error| refuse(&error.to_string()))?;
|
||||
@@ -75,16 +86,24 @@ fn read(path: &Path) -> Result<Vec<u8>, String> {
|
||||
if !metadata.file_type().is_file() {
|
||||
return Err(refuse("not a regular file"));
|
||||
}
|
||||
if metadata.len() > MAX_FILE_BYTES {
|
||||
return Err(refuse(&format!("larger than {MAX_FILE_BYTES} bytes")));
|
||||
if metadata.len() > cap {
|
||||
return Err(refuse(&format!("larger than {cap} bytes")));
|
||||
}
|
||||
fs::read(&canonical).map_err(|error| refuse(&error.to_string()))
|
||||
let mut bytes = Vec::new();
|
||||
let file = fs::File::open(&canonical).map_err(|error| refuse(&error.to_string()))?;
|
||||
file.take(cap + 1)
|
||||
.read_to_end(&mut bytes)
|
||||
.map_err(|error| refuse(&error.to_string()))?;
|
||||
if bytes.len() as u64 > cap {
|
||||
return Err(refuse(&format!("larger than {cap} bytes")));
|
||||
}
|
||||
Ok(bytes)
|
||||
}
|
||||
|
||||
/// `readFile(path, 'utf8')`: invalid UTF-8 becomes U+FFFD, as the TypeScript read the
|
||||
/// curation files.
|
||||
fn read_text(path: &Path) -> Result<String, String> {
|
||||
Ok(String::from_utf8_lossy(&read(path)?).into_owned())
|
||||
/// `readFile(path, 'utf8')` of an authored curation file, under the 4 MiB curation cap: invalid
|
||||
/// UTF-8 becomes U+FFFD, as the TypeScript read them.
|
||||
fn read_curation_text(path: &Path) -> Result<String, String> {
|
||||
Ok(String::from_utf8_lossy(&read_capped(path, MAX_CURATION_BYTES)?).into_owned())
|
||||
}
|
||||
|
||||
fn svg(name: &str) -> Result<SvgSource, String> {
|
||||
@@ -128,9 +147,9 @@ fn legacy_folder(input: &EquineLegacyInput, result: &ImportResult) -> Result<Fol
|
||||
/// `draft/`: the R5.3 texts, then the frozen cleaned drawing (read, never re-cleaned).
|
||||
fn draft_folder(result: &ImportResult) -> Result<Folder, String> {
|
||||
let curation = Path::new(SEED).join("curation");
|
||||
let references = parse_references(&read_text(&curation.join("references.json"))?)
|
||||
let references = parse_references(&read_curation_text(&curation.join("references.json"))?)
|
||||
.map_err(|error| error.to_string())?;
|
||||
let changeset = parse_changeset(&read_text(&curation.join("changes.json"))?)
|
||||
let changeset = parse_changeset(&read_curation_text(&curation.join("changes.json"))?)
|
||||
.map_err(|error| error.to_string())?;
|
||||
let cleaned_svg = read(&Path::new(SEED).join(DRAFT_FOLDER).join(CLEANED_ASSET))?;
|
||||
let head = result.view_boxes.get(HEAD_VIEW_ID).and_then(|v| v.as_str());
|
||||
|
||||
@@ -0,0 +1,214 @@
|
||||
//! `upload-sheet-assets` (R5.5): uploads the SVG drawings of a generated sheet seed folder to
|
||||
//! the private Storage bucket `anatomical-sheet-assets` of a LOCAL Supabase stack, content
|
||||
//! addressed (`<sha256>.svg`), and confirms each asset row once its object is verified.
|
||||
//!
|
||||
//! upload-sheet-assets --check <seed folder> read-only: verify the files, the destination
|
||||
//! and what is already there; write nothing
|
||||
//! upload-sheet-assets <seed folder> the same, then upload and confirm what is missing
|
||||
//!
|
||||
//! Environment: `SUPABASE_URL` (a local stack: loopback, or the DEV address
|
||||
//! `FAUNOSTEO_DEV_HOST`; `http` with an explicit port; no default) and
|
||||
//! `SUPABASE_SERVICE_ROLE_KEY` (never printed). The folder lies under
|
||||
//! `supabase/seed/anatomical-sheets`; a relative path is typed from the folder pnpm was run in
|
||||
//! (`INIT_CWD`), else the current one.
|
||||
//!
|
||||
//! Order of a run: files and manifest checked, destination checked, handshake, then every
|
||||
//! entry planned from the state found; any refusal stops the run before the first write. An
|
||||
//! object that exists is read back and compared with its key, never replaced; an asset row is
|
||||
//! confirmed only after its object is verified. A rerun after a failure resumes safely.
|
||||
//! Like `import-equine`, this tool owns the file system, the environment and the network; the
|
||||
//! rules are the core's (`curation::upload`).
|
||||
#![allow(clippy::disallowed_methods)]
|
||||
|
||||
mod storage;
|
||||
|
||||
use std::io::Read;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::{env, fs, process};
|
||||
|
||||
use faunosteo_core::curation::upload::{
|
||||
AssetEntry, MAX_ASSET_BYTES, MAX_MANIFEST_BYTES, ObjectState, Plan, UploadError, UploadKind,
|
||||
check_destination, parse_manifest, plan_entry, refuse, verify_asset,
|
||||
};
|
||||
use storage::{Secret, Storage};
|
||||
|
||||
const SEED_ROOT: &str = concat!(
|
||||
env!("CARGO_MANIFEST_DIR"),
|
||||
"/../../supabase/seed/anatomical-sheets"
|
||||
);
|
||||
|
||||
/// A local entry whose bytes passed `verify_asset`.
|
||||
struct Local {
|
||||
entry: AssetEntry,
|
||||
bytes: Vec<u8>,
|
||||
}
|
||||
|
||||
/// A regular file under `root` (every link resolved), read under a cap: a file that grows
|
||||
/// between the size check and the read is still refused. The seed is a single user's working
|
||||
/// copy: a swap between the check and the read is not defended.
|
||||
fn read_confined(path: &Path, root: &Path, cap: usize) -> Result<Vec<u8>, UploadError> {
|
||||
let bad = |why: &str| refuse(UploadKind::Content, format!("{}: {why}", path.display()));
|
||||
let canonical = fs::canonicalize(path).map_err(|e| bad(&e.to_string()))?;
|
||||
if !canonical.starts_with(root) {
|
||||
return Err(bad("leads outside the seed folder"));
|
||||
}
|
||||
let metadata = fs::metadata(&canonical).map_err(|e| bad(&e.to_string()))?;
|
||||
if !metadata.is_file() || metadata.len() > cap as u64 {
|
||||
return Err(bad("not a regular file, or too large"));
|
||||
}
|
||||
let mut bytes = Vec::new();
|
||||
let file = fs::File::open(&canonical).map_err(|e| bad(&e.to_string()))?;
|
||||
file.take(cap as u64 + 1)
|
||||
.read_to_end(&mut bytes)
|
||||
.map_err(|e| bad(&e.to_string()))?;
|
||||
if bytes.len() > cap {
|
||||
return Err(bad("too large"));
|
||||
}
|
||||
Ok(bytes)
|
||||
}
|
||||
|
||||
/// The folder typed on the command line: from `INIT_CWD` when pnpm set it, canonical, a
|
||||
/// directory, inside the seed root.
|
||||
fn resolve_folder(typed: &str) -> Result<PathBuf, UploadError> {
|
||||
let bad = |why: &str| refuse(UploadKind::Content, format!("{typed}: {why}"));
|
||||
let base = env::var_os("INIT_CWD")
|
||||
.map(PathBuf::from)
|
||||
.filter(|p| p.is_absolute())
|
||||
.map_or_else(env::current_dir, Ok)
|
||||
.map_err(|e| bad(&e.to_string()))?;
|
||||
let folder = fs::canonicalize(base.join(typed)).map_err(|e| bad(&e.to_string()))?;
|
||||
let root = fs::canonicalize(SEED_ROOT).map_err(|e| bad(&e.to_string()))?;
|
||||
if !folder.starts_with(&root) || !folder.is_dir() {
|
||||
return Err(bad("not a folder under supabase/seed/anatomical-sheets"));
|
||||
}
|
||||
Ok(folder)
|
||||
}
|
||||
|
||||
/// Manifest, then every drawing, all verified before any request is made.
|
||||
fn load_local(folder: &Path) -> Result<Vec<Local>, UploadError> {
|
||||
let manifest = read_confined(&folder.join("assets.json"), folder, MAX_MANIFEST_BYTES)?;
|
||||
let text = String::from_utf8(manifest)
|
||||
.map_err(|_| refuse(UploadKind::Manifest, "assets.json is not UTF-8"))?;
|
||||
parse_manifest(&text)?
|
||||
.into_iter()
|
||||
.map(|entry| {
|
||||
let path = folder.join("assets").join(&entry.file);
|
||||
let bytes = read_confined(&path, folder, MAX_ASSET_BYTES)?;
|
||||
verify_asset(&entry, &bytes)?;
|
||||
Ok(Local { entry, bytes })
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn required_env(name: &str) -> Result<String, UploadError> {
|
||||
env::var(name)
|
||||
.ok()
|
||||
.filter(|value| !value.is_empty())
|
||||
.ok_or_else(|| refuse(UploadKind::Destination, format!("{name} is required")))
|
||||
}
|
||||
|
||||
fn connect() -> Result<Storage, UploadError> {
|
||||
let dev_host = env::var("FAUNOSTEO_DEV_HOST").ok();
|
||||
let destination = check_destination(&required_env("SUPABASE_URL")?, dev_host.as_deref())?;
|
||||
let key = Secret::new(required_env("SUPABASE_SERVICE_ROLE_KEY")?);
|
||||
Ok(Storage::new(&destination, key))
|
||||
}
|
||||
|
||||
/// What the destination holds for each entry and the steps that follow; refuses on the first
|
||||
/// entry that cannot be planned, so nothing is written for a run that cannot complete.
|
||||
fn plan_all(storage: &Storage, locals: &[Local]) -> Result<Vec<(Plan, ObjectState)>, UploadError> {
|
||||
locals
|
||||
.iter()
|
||||
.map(|local| {
|
||||
let object = storage.object_state(&local.entry)?;
|
||||
let row = storage.row_state(&local.entry)?;
|
||||
Ok((plan_entry(&local.entry, &object, &row)?, object))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn describe(local: &Local, plan: Plan, check: bool) -> String {
|
||||
let steps: Vec<&str> = match (plan.upload, plan.confirm) {
|
||||
(false, false) => vec!["object verified, row confirmed, nothing to do"],
|
||||
(true, true) => vec!["upload", "confirm"],
|
||||
(true, false) => vec!["upload"],
|
||||
(false, true) => vec!["object verified", "confirm"],
|
||||
};
|
||||
let verb = if check { "would" } else { "will" };
|
||||
format!(
|
||||
"{} {}: {verb} {}",
|
||||
local.entry.file,
|
||||
local.entry.object_key(),
|
||||
steps.join(", ")
|
||||
)
|
||||
}
|
||||
|
||||
/// Uploads (if absent), verifies the object by reading it back, then confirms (if pending).
|
||||
fn apply(storage: &Storage, local: &Local, plan: Plan) -> Result<(), UploadError> {
|
||||
if plan.upload {
|
||||
storage.upload(&local.entry, &local.bytes)?;
|
||||
}
|
||||
if storage.object_state(&local.entry)? != ObjectState::Verified {
|
||||
return Err(refuse(
|
||||
UploadKind::State,
|
||||
format!(
|
||||
"{}: the object is not the expected bytes after upload",
|
||||
local.entry.file
|
||||
),
|
||||
));
|
||||
}
|
||||
if plan.confirm {
|
||||
storage.confirm(&local.entry)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn run(args: &[String]) -> Result<(), UploadError> {
|
||||
let (check, folder) = match args {
|
||||
[flag, folder] if flag == "--check" => (true, folder),
|
||||
[folder] if folder != "--check" => (false, folder),
|
||||
_ => {
|
||||
return Err(refuse(
|
||||
UploadKind::Content,
|
||||
"usage: upload-sheet-assets [--check] <seed folder>",
|
||||
));
|
||||
}
|
||||
};
|
||||
let locals = load_local(&resolve_folder(folder)?)?;
|
||||
let storage = connect()?;
|
||||
storage.handshake()?;
|
||||
let plans = plan_all(&storage, &locals)?;
|
||||
for (local, (plan, _)) in locals.iter().zip(&plans) {
|
||||
println!("{}", describe(local, *plan, check));
|
||||
}
|
||||
if check {
|
||||
return Ok(());
|
||||
}
|
||||
for (local, (plan, _)) in locals.iter().zip(&plans) {
|
||||
apply(&storage, local, *plan)?;
|
||||
println!("{} done", local.entry.file);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn main() {
|
||||
let args: Option<Vec<String>> = env::args_os()
|
||||
.skip(1)
|
||||
.map(|a| a.into_string().ok())
|
||||
.collect();
|
||||
let Some(args) = args else {
|
||||
eprintln!("an argument is not valid UTF-8");
|
||||
process::exit(2);
|
||||
};
|
||||
match run(&args) {
|
||||
Ok(()) => println!("ok"),
|
||||
Err(error) => {
|
||||
eprintln!("refused: {error}");
|
||||
process::exit(if error.kind == UploadKind::Network {
|
||||
3
|
||||
} else {
|
||||
1
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,248 @@
|
||||
//! The destination, over HTTP: Storage objects and the asset rows of a local Supabase stack,
|
||||
//! with the service key. Every status is classified into the closed `UploadKind`s; the key is
|
||||
//! held in a `Secret` and goes in two request headers, nowhere else.
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
use faunosteo_core::curation::ids::sha256_hex;
|
||||
use faunosteo_core::curation::upload::{
|
||||
AssetEntry, BUCKET, Destination, MAX_ASSET_BYTES, ObjectState, RowState, UploadError,
|
||||
UploadKind, refuse,
|
||||
};
|
||||
use serde_json::{Value, json};
|
||||
use ureq::Agent;
|
||||
|
||||
const MAX_BODY: u64 = 1024 * 1024;
|
||||
const ATTEMPTS: u32 = 3;
|
||||
|
||||
/// A key that never prints: `Debug` hides it and there is no `Display`.
|
||||
pub struct Secret(String);
|
||||
|
||||
impl Secret {
|
||||
pub fn new(text: String) -> Self {
|
||||
Self(text)
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for Secret {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.write_str("Secret(<redacted>)")
|
||||
}
|
||||
}
|
||||
|
||||
/// What an HTTP exchange gave back: the status and the (capped) body.
|
||||
struct Reply {
|
||||
status: u16,
|
||||
body: Vec<u8>,
|
||||
}
|
||||
|
||||
pub struct Storage {
|
||||
agent: Agent,
|
||||
origin: String,
|
||||
key: Secret,
|
||||
}
|
||||
|
||||
impl Storage {
|
||||
pub fn new(destination: &Destination, key: Secret) -> Self {
|
||||
let agent: Agent = Agent::config_builder()
|
||||
.timeout_global(Some(Duration::from_secs(30)))
|
||||
.timeout_connect(Some(Duration::from_secs(5)))
|
||||
.max_redirects(0)
|
||||
// The service key never goes through a proxy picked up from the environment.
|
||||
.proxy(None)
|
||||
.http_status_as_error(false)
|
||||
.user_agent("faunosteo-upload-sheet-assets")
|
||||
.build()
|
||||
.into();
|
||||
Self {
|
||||
agent,
|
||||
origin: destination.origin().to_owned(),
|
||||
key,
|
||||
}
|
||||
}
|
||||
|
||||
/// One exchange, retried on a transport error or a 5xx only (a refusal is final). The
|
||||
/// request is built again on each attempt.
|
||||
fn exchange(
|
||||
&self,
|
||||
limit: u64,
|
||||
send: impl Fn(&Agent, &Self) -> Result<ureq::http::Response<ureq::Body>, ureq::Error>,
|
||||
) -> Result<Reply, UploadError> {
|
||||
let mut last = refuse(UploadKind::Network, "no attempt made");
|
||||
for attempt in 1..=ATTEMPTS {
|
||||
if attempt > 1 {
|
||||
std::thread::sleep(Duration::from_millis(500 * u64::from(attempt)));
|
||||
}
|
||||
match send(&self.agent, self).and_then(|mut response| read_reply(&mut response, limit))
|
||||
{
|
||||
Ok(reply) if reply.status < 500 => return Ok(reply),
|
||||
Ok(reply) => last = refuse(UploadKind::Network, format!("HTTP {}", reply.status)),
|
||||
Err(error) => last = refuse(UploadKind::Network, error.to_string()),
|
||||
}
|
||||
}
|
||||
Err(last)
|
||||
}
|
||||
|
||||
fn authorised<B>(&self, request: ureq::RequestBuilder<B>) -> ureq::RequestBuilder<B> {
|
||||
request
|
||||
.header("apikey", &self.key.0)
|
||||
.header("Authorization", &format!("Bearer {}", self.key.0))
|
||||
}
|
||||
|
||||
fn object_url(&self, entry: &AssetEntry) -> String {
|
||||
format!(
|
||||
"{}/storage/v1/object/{BUCKET}/{}",
|
||||
self.origin,
|
||||
entry.object_key()
|
||||
)
|
||||
}
|
||||
|
||||
/// The destination is the local Faunosteo stack: the sheet bucket exists and is private,
|
||||
/// and the key is accepted. Nothing is written.
|
||||
pub fn handshake(&self) -> Result<(), UploadError> {
|
||||
let url = format!("{}/storage/v1/bucket/{BUCKET}", self.origin);
|
||||
let reply = self.exchange(MAX_BODY, |agent, me| me.authorised(agent.get(&url)).call())?;
|
||||
check_access(&reply)?;
|
||||
let bucket: Value = serde_json::from_slice(&reply.body)
|
||||
.map_err(|_| refuse(UploadKind::Handshake, "the bucket answer is not JSON"))?;
|
||||
let name = bucket.get("name").and_then(Value::as_str);
|
||||
let public = bucket.get("public").and_then(Value::as_bool);
|
||||
if reply.status != 200 || name != Some(BUCKET) || public != Some(false) {
|
||||
return Err(refuse(
|
||||
UploadKind::Handshake,
|
||||
format!("{BUCKET} is missing, or not a private bucket"),
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Absent, or present with bytes that hash to the key, or present with other bytes.
|
||||
pub fn object_state(&self, entry: &AssetEntry) -> Result<ObjectState, UploadError> {
|
||||
let url = self.object_url(entry);
|
||||
let limit = MAX_ASSET_BYTES as u64 + 1;
|
||||
let reply = self.exchange(limit, |agent, me| me.authorised(agent.get(&url)).call())?;
|
||||
if is_not_found(&reply) {
|
||||
return Ok(ObjectState::Absent);
|
||||
}
|
||||
check_access(&reply)?;
|
||||
expect_status(&reply, 200, "object read")?;
|
||||
let found = sha256_hex(&reply.body);
|
||||
Ok(if found == entry.sha256 {
|
||||
ObjectState::Verified
|
||||
} else {
|
||||
ObjectState::Different(found)
|
||||
})
|
||||
}
|
||||
|
||||
/// Creates the object; an existing key is never replaced (`x-upsert: false`). Returns
|
||||
/// whether this call created it.
|
||||
pub fn upload(&self, entry: &AssetEntry, bytes: &[u8]) -> Result<bool, UploadError> {
|
||||
let url = self.object_url(entry);
|
||||
let reply = self.exchange(MAX_BODY, |agent, me| {
|
||||
me.authorised(agent.post(&url))
|
||||
.header("Content-Type", "image/svg+xml")
|
||||
.header("x-upsert", "false")
|
||||
.send(bytes)
|
||||
})?;
|
||||
check_access(&reply)?;
|
||||
if is_duplicate(&reply) {
|
||||
return Ok(false);
|
||||
}
|
||||
expect_status(&reply, 200, "upload")?;
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
/// The row of the asset: absent, another hash, pending or confirmed.
|
||||
pub fn row_state(&self, entry: &AssetEntry) -> Result<RowState, UploadError> {
|
||||
let url = format!(
|
||||
"{}/rest/v1/anatomical_sheet_assets?id=eq.{}&select=sha256,uploaded_at",
|
||||
self.origin, entry.svg_asset_id
|
||||
);
|
||||
let reply = self.exchange(MAX_BODY, |agent, me| me.authorised(agent.get(&url)).call())?;
|
||||
check_access(&reply)?;
|
||||
expect_status(&reply, 200, "row read")?;
|
||||
let rows: Vec<Value> = serde_json::from_slice(&reply.body)
|
||||
.map_err(|_| refuse(UploadKind::Handshake, "the row answer is not a JSON list"))?;
|
||||
Ok(match rows.first() {
|
||||
None => RowState::Unknown,
|
||||
Some(row) => row_state_of(row, &entry.sha256),
|
||||
})
|
||||
}
|
||||
|
||||
/// Marks the asset bytes as present (idempotent on the server, service role only).
|
||||
pub fn confirm(&self, entry: &AssetEntry) -> Result<(), UploadError> {
|
||||
let url = format!("{}/rest/v1/rpc/confirm_sheet_asset_upload", self.origin);
|
||||
let body = json!({ "asset_id": entry.svg_asset_id });
|
||||
let reply = self.exchange(MAX_BODY, |agent, me| {
|
||||
me.authorised(agent.post(&url)).send_json(&body)
|
||||
})?;
|
||||
check_access(&reply)?;
|
||||
expect_status(&reply, 200, "confirm")
|
||||
}
|
||||
}
|
||||
|
||||
fn row_state_of(row: &Value, expected_sha256: &str) -> RowState {
|
||||
let found = row
|
||||
.get("sha256")
|
||||
.and_then(Value::as_str)
|
||||
.unwrap_or_default();
|
||||
if found != expected_sha256 {
|
||||
return RowState::OtherHash(found.to_owned());
|
||||
}
|
||||
match row.get("uploaded_at") {
|
||||
Some(Value::Null) | None => RowState::Pending,
|
||||
Some(_) => RowState::Confirmed,
|
||||
}
|
||||
}
|
||||
|
||||
fn read_reply(
|
||||
response: &mut ureq::http::Response<ureq::Body>,
|
||||
limit: u64,
|
||||
) -> Result<Reply, ureq::Error> {
|
||||
let body = response
|
||||
.body_mut()
|
||||
.with_config()
|
||||
.limit(limit)
|
||||
.read_to_vec()?;
|
||||
Ok(Reply {
|
||||
status: response.status().as_u16(),
|
||||
body,
|
||||
})
|
||||
}
|
||||
|
||||
/// The `statusCode` of a Storage error body (a string, "404" or "409" here).
|
||||
fn storage_code(reply: &Reply) -> Option<String> {
|
||||
let body: Value = serde_json::from_slice(&reply.body).ok()?;
|
||||
body.get("statusCode")?.as_str().map(str::to_owned)
|
||||
}
|
||||
|
||||
/// Storage answers a missing object with HTTP 400 and `statusCode` "404" in this image family.
|
||||
fn is_not_found(reply: &Reply) -> bool {
|
||||
reply.status == 404 || (reply.status == 400 && storage_code(reply).as_deref() == Some("404"))
|
||||
}
|
||||
|
||||
/// A key that already exists answers 409, or 400 with `statusCode` "409" ("Duplicate").
|
||||
fn is_duplicate(reply: &Reply) -> bool {
|
||||
reply.status == 409 || (reply.status == 400 && storage_code(reply).as_deref() == Some("409"))
|
||||
}
|
||||
|
||||
fn check_access(reply: &Reply) -> Result<(), UploadError> {
|
||||
if matches!(reply.status, 401 | 403) {
|
||||
return Err(refuse(
|
||||
UploadKind::Access,
|
||||
format!("HTTP {}: the destination refused the key", reply.status),
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn expect_status(reply: &Reply, wanted: u16, what: &str) -> Result<(), UploadError> {
|
||||
if reply.status == wanted {
|
||||
return Ok(());
|
||||
}
|
||||
// The server's free text is not echoed: only the step and the status.
|
||||
Err(refuse(
|
||||
UploadKind::Handshake,
|
||||
format!("{what}: unexpected HTTP {}", reply.status),
|
||||
))
|
||||
}
|
||||
@@ -375,10 +375,21 @@ fn refuse(file: &str, error: impl std::fmt::Display) -> PipelineError {
|
||||
PipelineError::Source(format!("{file}: {error}"))
|
||||
}
|
||||
|
||||
/// The size cap of an authored curation file (`changes.json`, `references.json`), checked
|
||||
/// before any parse allocation. The committed files are about 222 KB and 17 KB; the cap leaves
|
||||
/// a wide margin to grow and bounds the parse (RUST.md §11.16).
|
||||
pub const MAX_CURATION_FILE_BYTES: usize = 4 * 1024 * 1024;
|
||||
|
||||
/// `JSON.parse`: a repeated key keeps its first position and its last value (the parsed
|
||||
/// value is read into the typed shape afterwards, so duplicates never reach serde's derive),
|
||||
/// array index keys come first, integers beyond 2^53 are rounded (`js_normalize`).
|
||||
fn json_parse(file: &str, text: &str) -> Result<Value, PipelineError> {
|
||||
if text.len() > MAX_CURATION_FILE_BYTES {
|
||||
return Err(refuse(
|
||||
file,
|
||||
format!("larger than {MAX_CURATION_FILE_BYTES} bytes"),
|
||||
));
|
||||
}
|
||||
let mut value = js::parse_json(text).map_err(|error| refuse(file, error))?;
|
||||
js_normalize(&mut value);
|
||||
Ok(value)
|
||||
|
||||
@@ -16,6 +16,7 @@ pub mod markdown;
|
||||
pub mod prettier_json;
|
||||
pub mod report_guard;
|
||||
pub mod text;
|
||||
pub mod upload;
|
||||
|
||||
/// Why the pipeline refuses to produce a file. The TypeScript accepted some of these inputs
|
||||
/// and wrote whatever JavaScript made of them; the port refuses instead (declared in
|
||||
|
||||
@@ -0,0 +1,287 @@
|
||||
//! The rules of the sheet-asset upload (R5.5): the manifest, the destination, the bytes, and
|
||||
//! what to do given what the destination already holds. Values in, values out; the
|
||||
//! `upload-sheet-assets` binary owns the file system, the environment and the network.
|
||||
|
||||
use std::net::{IpAddr, Ipv4Addr};
|
||||
|
||||
use serde::Deserialize;
|
||||
use url::{Host, Url};
|
||||
|
||||
use crate::anatomy::svg::read_svg;
|
||||
use crate::curation::ids::sha256_hex;
|
||||
|
||||
/// The private Storage bucket of the sheet drawings (`20261009100000_sheet_asset_storage`).
|
||||
pub const BUCKET: &str = "anatomical-sheet-assets";
|
||||
/// The manifest of a generated seed folder, a few hundred bytes today.
|
||||
pub const MAX_MANIFEST_BYTES: usize = 1024 * 1024;
|
||||
/// Entries one run handles (a seed holds a handful of drawings).
|
||||
pub const MAX_ENTRIES: usize = 64;
|
||||
/// One drawing; the bucket itself refuses beyond 20 MiB.
|
||||
pub const MAX_ASSET_BYTES: usize = 16 * 1024 * 1024;
|
||||
|
||||
/// The closed kinds a refusal falls into; the message names the file or the value.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum UploadKind {
|
||||
/// `assets.json` is malformed, too large, or lists a bad or repeated entry.
|
||||
Manifest,
|
||||
/// The destination is not a local stack the tool is allowed to write to.
|
||||
Destination,
|
||||
/// A local file does not match its entry, or is not a drawing the product accepts.
|
||||
Content,
|
||||
/// The destination answered something other than the expected local stack.
|
||||
Handshake,
|
||||
/// The destination holds different bytes under a content-addressed key, or no row for the
|
||||
/// asset: nothing is overwritten, nothing is invented.
|
||||
State,
|
||||
/// The destination refused the key (401, 403).
|
||||
Access,
|
||||
/// The destination could not be reached, or answered with a server error.
|
||||
Network,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct UploadError {
|
||||
pub kind: UploadKind,
|
||||
pub message: String,
|
||||
}
|
||||
|
||||
impl std::fmt::Display for UploadError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
write!(f, "{:?}: {}", self.kind, self.message)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn refuse(kind: UploadKind, message: impl Into<String>) -> UploadError {
|
||||
UploadError {
|
||||
kind,
|
||||
message: message.into(),
|
||||
}
|
||||
}
|
||||
|
||||
/// One line of `assets.json`, as the generated seed writes it.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct AssetEntry {
|
||||
pub file: String,
|
||||
pub sha256: String,
|
||||
pub byte_size: usize,
|
||||
pub svg_asset_id: String,
|
||||
}
|
||||
|
||||
impl AssetEntry {
|
||||
/// The Storage object key: content-addressed, so the same bytes are the same object.
|
||||
pub fn object_key(&self) -> String {
|
||||
format!("{}.svg", self.sha256)
|
||||
}
|
||||
}
|
||||
|
||||
fn is_lower_hex(text: &str, length: usize) -> bool {
|
||||
text.len() == length && text.bytes().all(|b| matches!(b, b'0'..=b'9' | b'a'..=b'f'))
|
||||
}
|
||||
|
||||
fn is_uuid(text: &str) -> bool {
|
||||
let parts: Vec<&str> = text.split('-').collect();
|
||||
let lengths = [8, 4, 4, 4, 12];
|
||||
parts.len() == 5
|
||||
&& parts
|
||||
.iter()
|
||||
.zip(lengths)
|
||||
.all(|(part, length)| is_lower_hex(part, length))
|
||||
}
|
||||
|
||||
/// A plain file name of a drawing: no separator, no leading dot, `.svg`.
|
||||
fn is_plain_svg_name(name: &str) -> bool {
|
||||
name.len() <= 100
|
||||
&& name.ends_with(".svg")
|
||||
&& !name.starts_with('.')
|
||||
&& name
|
||||
.bytes()
|
||||
.all(|b| b.is_ascii_alphanumeric() || matches!(b, b'.' | b'_' | b'-'))
|
||||
}
|
||||
|
||||
fn check_entry(entry: &AssetEntry) -> Result<(), UploadError> {
|
||||
let bad = |what: &str| refuse(UploadKind::Manifest, format!("{}: {what}", entry.file));
|
||||
if !is_plain_svg_name(&entry.file) {
|
||||
return Err(refuse(
|
||||
UploadKind::Manifest,
|
||||
"file must be a plain .svg name",
|
||||
));
|
||||
}
|
||||
if !is_lower_hex(&entry.sha256, 64) {
|
||||
return Err(bad("sha256 must be 64 lowercase hex digits"));
|
||||
}
|
||||
if !is_uuid(&entry.svg_asset_id) {
|
||||
return Err(bad("svg_asset_id must be a lowercase uuid"));
|
||||
}
|
||||
if entry.byte_size == 0 || entry.byte_size > MAX_ASSET_BYTES {
|
||||
return Err(bad("byte_size out of range"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn repeated(entries: &[AssetEntry]) -> Option<String> {
|
||||
for (index, entry) in entries.iter().enumerate() {
|
||||
let twin = entries[..index].iter().any(|other| {
|
||||
other.file == entry.file
|
||||
|| other.sha256 == entry.sha256
|
||||
|| other.svg_asset_id == entry.svg_asset_id
|
||||
});
|
||||
if twin {
|
||||
return Some(entry.file.clone());
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Reads `assets.json`: size-bounded before the parse, strict on keys, every entry checked,
|
||||
/// no repeated file, hash or asset id.
|
||||
pub fn parse_manifest(text: &str) -> Result<Vec<AssetEntry>, UploadError> {
|
||||
if text.len() > MAX_MANIFEST_BYTES {
|
||||
return Err(refuse(UploadKind::Manifest, "assets.json is too large"));
|
||||
}
|
||||
let entries: Vec<AssetEntry> = serde_json::from_str(text)
|
||||
.map_err(|error| refuse(UploadKind::Manifest, format!("assets.json: {error}")))?;
|
||||
if entries.is_empty() || entries.len() > MAX_ENTRIES {
|
||||
return Err(refuse(UploadKind::Manifest, "assets.json: 1 to 64 entries"));
|
||||
}
|
||||
entries.iter().try_for_each(check_entry)?;
|
||||
if let Some(file) = repeated(&entries) {
|
||||
return Err(refuse(
|
||||
UploadKind::Manifest,
|
||||
format!("{file}: repeated entry"),
|
||||
));
|
||||
}
|
||||
Ok(entries)
|
||||
}
|
||||
|
||||
/// The bytes on disk are what the manifest recorded, and a drawing the product accepts: the
|
||||
/// core's one SVG reader keeps every element (nothing removed), as a load of the sheet would.
|
||||
pub fn verify_asset(entry: &AssetEntry, bytes: &[u8]) -> Result<(), UploadError> {
|
||||
let bad = |what: &str| refuse(UploadKind::Content, format!("{}: {what}", entry.file));
|
||||
if bytes.len() != entry.byte_size || sha256_hex(bytes) != entry.sha256 {
|
||||
return Err(bad("bytes differ from assets.json"));
|
||||
}
|
||||
let text = std::str::from_utf8(bytes).map_err(|_| bad("not UTF-8 text"))?;
|
||||
match read_svg(text) {
|
||||
Ok(read) if read.removed.is_empty() => Ok(()),
|
||||
Ok(_) => Err(bad("the SVG reader would remove content")),
|
||||
Err(error) => Err(bad(&format!("refused by the SVG reader: {error:?}"))),
|
||||
}
|
||||
}
|
||||
|
||||
/// A local destination the tool may write to: `http`, an explicit port, no credentials, query
|
||||
/// or path, and a host that is this machine (loopback) or the configured DEV address.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct Destination {
|
||||
origin: String,
|
||||
}
|
||||
|
||||
impl Destination {
|
||||
/// `http://host:port`, no trailing slash.
|
||||
pub fn origin(&self) -> &str {
|
||||
&self.origin
|
||||
}
|
||||
}
|
||||
|
||||
fn is_private_v4(ip: Ipv4Addr) -> bool {
|
||||
ip.is_loopback() || ip.is_private()
|
||||
}
|
||||
|
||||
fn host_is_allowed(host: &Host<&str>, dev_host: Option<&str>) -> bool {
|
||||
match host {
|
||||
Host::Domain(name) => name.eq_ignore_ascii_case("localhost"),
|
||||
Host::Ipv6(ip) => ip.is_loopback(),
|
||||
Host::Ipv4(ip) => {
|
||||
ip.is_loopback()
|
||||
|| (is_private_v4(*ip)
|
||||
&& dev_host.and_then(|d| d.parse::<IpAddr>().ok()) == Some(IpAddr::V4(*ip)))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Refuses anything but a local stack; there is no default to fall back to.
|
||||
pub fn check_destination(raw: &str, dev_host: Option<&str>) -> Result<Destination, UploadError> {
|
||||
let bad = |what: &str| refuse(UploadKind::Destination, what);
|
||||
let url = Url::parse(raw).map_err(|_| bad("not a URL"))?;
|
||||
if url.scheme() != "http" {
|
||||
return Err(bad("only http to a local stack"));
|
||||
}
|
||||
if !url.username().is_empty() || url.password().is_some() {
|
||||
return Err(bad("no credentials in the URL"));
|
||||
}
|
||||
if url.query().is_some() || url.fragment().is_some() || url.path() != "/" {
|
||||
return Err(bad("an origin only: no path, query or fragment"));
|
||||
}
|
||||
let port = url
|
||||
.port()
|
||||
.ok_or_else(|| bad("an explicit port is required"))?;
|
||||
let host = url.host().ok_or_else(|| bad("no host"))?;
|
||||
if !host_is_allowed(&host, dev_host) {
|
||||
return Err(bad(
|
||||
"host is not this machine or the configured DEV address",
|
||||
));
|
||||
}
|
||||
Ok(Destination {
|
||||
origin: format!("http://{}:{port}", host_text(&host)),
|
||||
})
|
||||
}
|
||||
|
||||
fn host_text(host: &Host<&str>) -> String {
|
||||
match host {
|
||||
Host::Ipv6(ip) => format!("[{ip}]"),
|
||||
other => other.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
/// What the Storage object `<sha256>.svg` holds at the destination.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum ObjectState {
|
||||
Absent,
|
||||
/// Present, and its bytes hash to the key.
|
||||
Verified,
|
||||
/// Present under this key but with other bytes (the hash found).
|
||||
Different(String),
|
||||
}
|
||||
|
||||
/// The `anatomical_sheet_assets` row of the entry, as the destination reports it.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum RowState {
|
||||
Unknown,
|
||||
/// The row records another hash than the manifest.
|
||||
OtherHash(String),
|
||||
Pending,
|
||||
Confirmed,
|
||||
}
|
||||
|
||||
/// What a run does for one entry; both false means nothing to do.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct Plan {
|
||||
pub upload: bool,
|
||||
pub confirm: bool,
|
||||
}
|
||||
|
||||
/// Decides the steps of one entry from what the destination holds, or refuses: nothing is
|
||||
/// ever overwritten, and a row the seed did not create is not guessed at.
|
||||
pub fn plan_entry(
|
||||
entry: &AssetEntry,
|
||||
object: &ObjectState,
|
||||
row: &RowState,
|
||||
) -> Result<Plan, UploadError> {
|
||||
let bad = |what: String| refuse(UploadKind::State, format!("{}: {what}", entry.file));
|
||||
match row {
|
||||
RowState::Unknown => return Err(bad("no asset row (apply the seed first)".into())),
|
||||
RowState::OtherHash(found) => {
|
||||
return Err(bad(format!("the asset row records sha256 {found}")));
|
||||
}
|
||||
RowState::Pending | RowState::Confirmed => {}
|
||||
}
|
||||
if let ObjectState::Different(found) = object {
|
||||
return Err(bad(format!(
|
||||
"the object holds other bytes (sha256 {found})"
|
||||
)));
|
||||
}
|
||||
Ok(Plan {
|
||||
upload: *object == ObjectState::Absent,
|
||||
confirm: *row == RowState::Pending,
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
//! The resource caps and the stable-id acceptance of the curation inputs (R5.5 adjustments of
|
||||
//! R5.4, owner 2026-10-23 §3 and §6): an authored file over 4 MiB is refused by name before any
|
||||
//! parse allocation, the committed files keep a wide margin, and the stable-id grammar the
|
||||
//! applier enforces is the sheet contract's, not an equine list.
|
||||
#![allow(clippy::disallowed_methods, clippy::unwrap_used, clippy::expect_used)]
|
||||
|
||||
use faunosteo_core::curation::draft::schema::MAX_CURATION_FILE_BYTES;
|
||||
use faunosteo_core::curation::draft::{parse_changeset, parse_references};
|
||||
use faunosteo_core::domain::js::is_stable_id;
|
||||
|
||||
const CURATION: &str = "../../supabase/seed/anatomical-sheets/equine/curation";
|
||||
|
||||
fn committed(name: &str) -> String {
|
||||
std::fs::read_to_string(format!("{CURATION}/{name}")).unwrap()
|
||||
}
|
||||
|
||||
/// A valid-looking JSON text of exactly `len` bytes (spaces pad the tail).
|
||||
fn padded(len: usize) -> String {
|
||||
let mut text = String::from("{\"changes\":[]}");
|
||||
text.push_str(&" ".repeat(len - text.len()));
|
||||
text
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn committed_files_keep_a_wide_margin_under_the_cap() {
|
||||
for name in ["changes.json", "references.json"] {
|
||||
let size = committed(name).len();
|
||||
assert!(size * 10 < MAX_CURATION_FILE_BYTES, "{name}: {size} bytes");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_file_over_the_cap_is_refused_by_name_before_the_parse() {
|
||||
let over = padded(MAX_CURATION_FILE_BYTES + 1);
|
||||
let changes = parse_changeset(&over).unwrap_err().to_string();
|
||||
assert!(
|
||||
changes.starts_with("changes.json: larger than "),
|
||||
"{changes}"
|
||||
);
|
||||
let references = parse_references(&over).unwrap_err().to_string();
|
||||
assert!(
|
||||
references.starts_with("references.json: larger than "),
|
||||
"{references}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_file_at_the_cap_reaches_the_parser() {
|
||||
let at = padded(MAX_CURATION_FILE_BYTES);
|
||||
assert!(parse_changeset(&at).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stable_ids_of_any_species_are_accepted_and_the_rest_refused() {
|
||||
for id in [
|
||||
"canine-skull",
|
||||
"feline_thorax-2",
|
||||
"0-bovine",
|
||||
"a",
|
||||
&"x".repeat(64),
|
||||
] {
|
||||
assert!(is_stable_id(id), "{id}");
|
||||
}
|
||||
for id in [
|
||||
"",
|
||||
"Equine",
|
||||
"-x",
|
||||
"a b",
|
||||
"a/b",
|
||||
"a.b",
|
||||
"é",
|
||||
&"x".repeat(65),
|
||||
] {
|
||||
assert!(!is_stable_id(id), "{id:?}");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,208 @@
|
||||
//! The rules of the sheet-asset upload (R5.5): manifest, destination, bytes, plan. The tool
|
||||
//! around them is exercised by `upload_tool.rs` against a stub of the local stack.
|
||||
#![allow(clippy::disallowed_methods, clippy::unwrap_used, clippy::expect_used)]
|
||||
|
||||
use faunosteo_core::curation::ids::sha256_hex;
|
||||
use faunosteo_core::curation::upload::{
|
||||
AssetEntry, ObjectState, Plan, RowState, UploadKind, check_destination, parse_manifest,
|
||||
plan_entry, verify_asset,
|
||||
};
|
||||
|
||||
const SEED: &str = "../../supabase/seed/anatomical-sheets/equine/draft";
|
||||
const UUID: &str = "7f767f74-3b36-51a7-8465-52dd7e2ead4a";
|
||||
const SVG: &[u8] = b"<svg xmlns=\"http://www.w3.org/2000/svg\" viewBox=\"0 0 10 10\"><g/></svg>";
|
||||
|
||||
fn entry_for(bytes: &[u8]) -> AssetEntry {
|
||||
AssetEntry {
|
||||
file: "a.svg".into(),
|
||||
sha256: sha256_hex(bytes),
|
||||
byte_size: bytes.len(),
|
||||
svg_asset_id: UUID.into(),
|
||||
}
|
||||
}
|
||||
|
||||
fn manifest(entries: &[(&str, &str, usize, &str)]) -> String {
|
||||
let items: Vec<String> = entries
|
||||
.iter()
|
||||
.map(|(file, sha, size, id)| {
|
||||
format!("{{\"file\":\"{file}\",\"sha256\":\"{sha}\",\"byte_size\":{size},\"svg_asset_id\":\"{id}\"}}")
|
||||
})
|
||||
.collect();
|
||||
format!("[{}]", items.join(","))
|
||||
}
|
||||
|
||||
fn kind(
|
||||
result: Result<impl std::fmt::Debug, faunosteo_core::curation::upload::UploadError>,
|
||||
) -> UploadKind {
|
||||
result.unwrap_err().kind
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_committed_manifest_and_drawing_pass() {
|
||||
let text = std::fs::read_to_string(format!("{SEED}/assets.json")).unwrap();
|
||||
let entries = parse_manifest(&text).unwrap();
|
||||
assert_eq!(entries.len(), 1);
|
||||
let bytes = std::fs::read(format!("{SEED}/assets/{}", entries[0].file)).unwrap();
|
||||
verify_asset(&entries[0], &bytes).unwrap();
|
||||
assert_eq!(
|
||||
entries[0].object_key(),
|
||||
format!("{}.svg", entries[0].sha256)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_bad_manifest_is_refused_whole() {
|
||||
let sha = "a".repeat(64);
|
||||
let good = (&"a.svg", sha.as_str(), 5usize, UUID);
|
||||
let cases = [
|
||||
manifest(&[]),
|
||||
"{}".into(),
|
||||
"[{\"file\":\"a.svg\"}]".into(),
|
||||
manifest(&[("../a.svg", &sha, 5, UUID)]),
|
||||
manifest(&[(".a.svg", &sha, 5, UUID)]),
|
||||
manifest(&[("a.png", &sha, 5, UUID)]),
|
||||
manifest(&[("a.svg", &sha.to_uppercase(), 5, UUID)]),
|
||||
manifest(&[("a.svg", "abc", 5, UUID)]),
|
||||
manifest(&[("a.svg", &sha, 5, "not-a-uuid")]),
|
||||
manifest(&[("a.svg", &sha, 0, UUID)]),
|
||||
manifest(&[("a.svg", &sha, 17 * 1024 * 1024, UUID)]),
|
||||
manifest(&[
|
||||
(good.0, good.1, good.2, good.3),
|
||||
(
|
||||
"a.svg",
|
||||
&"b".repeat(64),
|
||||
5,
|
||||
"11111111-1111-1111-1111-111111111111",
|
||||
),
|
||||
]),
|
||||
manifest(&[
|
||||
(good.0, good.1, good.2, good.3),
|
||||
("b.svg", &sha, 5, "11111111-1111-1111-1111-111111111111"),
|
||||
]),
|
||||
manifest(&[
|
||||
(good.0, good.1, good.2, good.3),
|
||||
("b.svg", &"b".repeat(64), 5, UUID),
|
||||
]),
|
||||
"x".repeat(2 * 1024 * 1024),
|
||||
];
|
||||
for text in cases {
|
||||
assert_eq!(
|
||||
kind(parse_manifest(&text)),
|
||||
UploadKind::Manifest,
|
||||
"{}",
|
||||
&text[..text.len().min(80)]
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_manifest_with_an_unknown_key_is_refused() {
|
||||
let text = manifest(&[("a.svg", &"a".repeat(64), 5, UUID)]).replace("}", ",\"x\":1}");
|
||||
assert_eq!(kind(parse_manifest(&text)), UploadKind::Manifest);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn only_bytes_that_match_and_that_the_svg_reader_keeps_whole_are_uploaded() {
|
||||
let entry = entry_for(SVG);
|
||||
verify_asset(&entry, SVG).unwrap();
|
||||
let mut other = SVG.to_vec();
|
||||
other[5] = b'X';
|
||||
assert_eq!(kind(verify_asset(&entry, &other)), UploadKind::Content);
|
||||
let scripted: &[u8] = b"<svg xmlns=\"http://www.w3.org/2000/svg\"><script>x</script></svg>";
|
||||
assert_eq!(
|
||||
kind(verify_asset(&entry_for(scripted), scripted)),
|
||||
UploadKind::Content
|
||||
);
|
||||
let not_svg: &[u8] = b"<html/>";
|
||||
assert_eq!(
|
||||
kind(verify_asset(&entry_for(not_svg), not_svg)),
|
||||
UploadKind::Content
|
||||
);
|
||||
let binary: &[u8] = &[0xff, 0xfe, 0x00];
|
||||
assert_eq!(
|
||||
kind(verify_asset(&entry_for(binary), binary)),
|
||||
UploadKind::Content
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn only_a_local_stack_is_a_destination() {
|
||||
for ok in [
|
||||
"http://127.0.0.1:54321",
|
||||
"http://localhost:54321",
|
||||
"http://[::1]:54321",
|
||||
"http://127.0.0.2:9",
|
||||
] {
|
||||
let destination = check_destination(ok, None).unwrap();
|
||||
assert!(destination.origin().starts_with("http://"), "{ok}");
|
||||
assert!(!destination.origin().ends_with('/'));
|
||||
}
|
||||
assert_eq!(
|
||||
check_destination("http://LOCALHOST:1", None)
|
||||
.unwrap()
|
||||
.origin(),
|
||||
"http://localhost:1"
|
||||
);
|
||||
for refused in [
|
||||
"https://127.0.0.1:54321",
|
||||
"http://127.0.0.1",
|
||||
"http://127.0.0.1:80",
|
||||
"http://user@127.0.0.1:54321",
|
||||
"http://127.0.0.1:54321/x",
|
||||
"http://127.0.0.1:54321/?a=1",
|
||||
"http://127.0.0.1:54321/#f",
|
||||
"http://10.1.2.3:54321",
|
||||
"http://example.test:54321",
|
||||
"http://abcd.supabase.co:54321",
|
||||
"http://faunosteo.com:54321",
|
||||
"http://0.0.0.0:54321",
|
||||
"http://[2001:db8::1]:54321",
|
||||
"127.0.0.1:54321",
|
||||
"",
|
||||
] {
|
||||
assert_eq!(
|
||||
kind(check_destination(refused, None)),
|
||||
UploadKind::Destination,
|
||||
"{refused}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_dev_address_is_allowed_only_when_configured_and_private() {
|
||||
let url = "http://192.168.1.100:54321";
|
||||
assert!(check_destination(url, None).is_err());
|
||||
assert!(check_destination(url, Some("192.168.1.101")).is_err());
|
||||
assert!(check_destination(url, Some("192.168.1.100")).is_ok());
|
||||
// A public address is never a DEV address, even when named as one.
|
||||
assert!(check_destination("http://8.8.8.8:54321", Some("8.8.8.8")).is_err());
|
||||
// A name is not resolved: it cannot be proven local.
|
||||
assert!(check_destination("http://mybox:54321", Some("mybox")).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_plan_follows_what_the_destination_holds_and_never_overwrites() {
|
||||
let entry = entry_for(SVG);
|
||||
let plan = |object, row| plan_entry(&entry, &object, &row);
|
||||
let go = |upload, confirm| Ok(Plan { upload, confirm });
|
||||
assert_eq!(plan(ObjectState::Absent, RowState::Pending), go(true, true));
|
||||
assert_eq!(
|
||||
plan(ObjectState::Verified, RowState::Pending),
|
||||
go(false, true)
|
||||
);
|
||||
assert_eq!(
|
||||
plan(ObjectState::Verified, RowState::Confirmed),
|
||||
go(false, false)
|
||||
);
|
||||
assert_eq!(
|
||||
plan(ObjectState::Absent, RowState::Confirmed),
|
||||
go(true, false)
|
||||
);
|
||||
for (object, row) in [
|
||||
(ObjectState::Different("f".repeat(64)), RowState::Pending),
|
||||
(ObjectState::Absent, RowState::Unknown),
|
||||
(ObjectState::Verified, RowState::OtherHash("f".repeat(64))),
|
||||
] {
|
||||
assert_eq!(kind(plan(object, row)), UploadKind::State);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,316 @@
|
||||
//! The `upload-sheet-assets` binary against an in-process stub of the local Supabase API
|
||||
//! (R5.5): the plan of a run, what is written and what never is, resume after a partial
|
||||
//! failure, refusals by category, network errors, and the key never printed. The stub listens
|
||||
//! on loopback with a made-up key; the real local stack is exercised by
|
||||
//! `scripts/sheet-assets-e2e.sh`, run by the owner.
|
||||
#![allow(clippy::disallowed_methods, clippy::unwrap_used, clippy::expect_used)]
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::io::{BufRead, BufReader, Read, Write};
|
||||
use std::net::{TcpListener, TcpStream};
|
||||
use std::process::{Command, Output};
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
const KEY: &str = "stub-service-key-0123456789";
|
||||
const DRAFT: &str = concat!(
|
||||
env!("CARGO_MANIFEST_DIR"),
|
||||
"/../../supabase/seed/anatomical-sheets/equine/draft"
|
||||
);
|
||||
const SHA: &str = "0ea5202fa3b95bac167b87c12ee969ab0272e085b60c2d34dbca6202c5f08541";
|
||||
|
||||
#[derive(Default)]
|
||||
struct State {
|
||||
objects: HashMap<String, Vec<u8>>,
|
||||
/// `None`: no row. `Some((sha256, confirmed))`.
|
||||
row: Option<(String, bool)>,
|
||||
bucket_status: u16,
|
||||
fail_confirm: bool,
|
||||
log: Vec<String>,
|
||||
}
|
||||
|
||||
type Shared = Arc<Mutex<State>>;
|
||||
|
||||
fn horse() -> Vec<u8> {
|
||||
std::fs::read(format!("{DRAFT}/assets/horse.svg")).unwrap()
|
||||
}
|
||||
|
||||
fn read_request(stream: &mut TcpStream) -> (String, String, Vec<u8>) {
|
||||
let mut reader = BufReader::new(stream.try_clone().unwrap());
|
||||
let mut line = String::new();
|
||||
reader.read_line(&mut line).unwrap();
|
||||
let mut parts = line.split_whitespace();
|
||||
let (method, path) = (
|
||||
parts.next().unwrap().to_owned(),
|
||||
parts.next().unwrap().to_owned(),
|
||||
);
|
||||
let mut length = 0;
|
||||
loop {
|
||||
let mut header = String::new();
|
||||
reader.read_line(&mut header).unwrap();
|
||||
if header.trim().is_empty() {
|
||||
break;
|
||||
}
|
||||
if let Some(value) = header.to_ascii_lowercase().strip_prefix("content-length:") {
|
||||
length = value.trim().parse().unwrap();
|
||||
}
|
||||
}
|
||||
let mut body = vec![0; length];
|
||||
reader.read_exact(&mut body).unwrap();
|
||||
(method, path, body)
|
||||
}
|
||||
|
||||
fn respond(stream: &mut TcpStream, status: u16, body: &[u8]) {
|
||||
let head = format!(
|
||||
"HTTP/1.1 {status} X\r\nContent-Length: {}\r\nConnection: close\r\nContent-Type: application/json\r\n\r\n",
|
||||
body.len()
|
||||
);
|
||||
stream.write_all(head.as_bytes()).unwrap();
|
||||
stream.write_all(body).unwrap();
|
||||
}
|
||||
|
||||
fn route(state: &mut State, method: &str, path: &str, body: Vec<u8>) -> (u16, Vec<u8>) {
|
||||
state.log.push(format!("{method} {path}"));
|
||||
let object = path.strip_prefix("/storage/v1/object/anatomical-sheet-assets/");
|
||||
match (method, object) {
|
||||
("GET", None) if path == "/storage/v1/bucket/anatomical-sheet-assets" => (
|
||||
state.bucket_status,
|
||||
br#"{"name":"anatomical-sheet-assets","public":false}"#.to_vec(),
|
||||
),
|
||||
("GET", Some(key)) => match state.objects.get(key) {
|
||||
Some(bytes) => (200, bytes.clone()),
|
||||
None => (400, br#"{"statusCode":"404","error":"not_found"}"#.to_vec()),
|
||||
},
|
||||
("POST", Some(key)) if state.objects.contains_key(key) => {
|
||||
(400, br#"{"statusCode":"409","error":"Duplicate"}"#.to_vec())
|
||||
}
|
||||
("POST", Some(key)) => {
|
||||
state.objects.insert(key.to_owned(), body);
|
||||
(200, b"{}".to_vec())
|
||||
}
|
||||
("GET", None) if path.starts_with("/rest/v1/anatomical_sheet_assets") => {
|
||||
let rows = match &state.row {
|
||||
None => "[]".to_owned(),
|
||||
Some((sha, confirmed)) => {
|
||||
let at = if *confirmed {
|
||||
"\"2026-10-23T00:00:00Z\""
|
||||
} else {
|
||||
"null"
|
||||
};
|
||||
format!("[{{\"sha256\":\"{sha}\",\"uploaded_at\":{at}}}]")
|
||||
}
|
||||
};
|
||||
(200, rows.into_bytes())
|
||||
}
|
||||
("POST", None) if path.ends_with("/rpc/confirm_sheet_asset_upload") => {
|
||||
if state.fail_confirm {
|
||||
return (500, b"{}".to_vec());
|
||||
}
|
||||
if let Some((_, confirmed)) = state.row.as_mut() {
|
||||
*confirmed = true;
|
||||
}
|
||||
(200, b"\"id\"".to_vec())
|
||||
}
|
||||
_ => (404, b"{}".to_vec()),
|
||||
}
|
||||
}
|
||||
|
||||
fn serve(state: Shared) -> u16 {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
let port = listener.local_addr().unwrap().port();
|
||||
std::thread::spawn(move || {
|
||||
for stream in listener.incoming() {
|
||||
let mut stream = stream.unwrap();
|
||||
let (method, path, body) = read_request(&mut stream);
|
||||
let (status, reply) = route(&mut state.lock().unwrap(), &method, &path, body);
|
||||
respond(&mut stream, status, &reply);
|
||||
}
|
||||
});
|
||||
port
|
||||
}
|
||||
|
||||
/// A stub with the seed applied: the row exists, pending, the bucket answers 200.
|
||||
fn stub() -> (Shared, String) {
|
||||
let state: Shared = Arc::new(Mutex::new(State {
|
||||
row: Some((SHA.to_owned(), false)),
|
||||
bucket_status: 200,
|
||||
..State::default()
|
||||
}));
|
||||
let port = serve(Arc::clone(&state));
|
||||
(state, format!("http://127.0.0.1:{port}"))
|
||||
}
|
||||
|
||||
fn run(url: &str, args: &[&str]) -> Output {
|
||||
Command::new(env!("CARGO_BIN_EXE_upload-sheet-assets"))
|
||||
.args(args)
|
||||
.env_clear()
|
||||
.env("SUPABASE_URL", url)
|
||||
.env("SUPABASE_SERVICE_ROLE_KEY", KEY)
|
||||
.output()
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn text(output: &Output) -> String {
|
||||
format!(
|
||||
"{}{}",
|
||||
String::from_utf8_lossy(&output.stdout),
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
)
|
||||
}
|
||||
|
||||
fn writes(state: &Shared) -> usize {
|
||||
let log = &state.lock().unwrap().log;
|
||||
log.iter().filter(|line| line.starts_with("POST")).count()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn check_reads_and_writes_nothing() {
|
||||
let (state, url) = stub();
|
||||
let output = run(&url, &["--check", DRAFT]);
|
||||
let said = text(&output);
|
||||
assert!(output.status.success(), "{said}");
|
||||
assert!(said.contains("would upload, confirm"), "{said}");
|
||||
assert_eq!(writes(&state), 0);
|
||||
assert!(!said.contains(KEY));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_run_uploads_verifies_then_confirms_and_a_rerun_does_nothing() {
|
||||
let (state, url) = stub();
|
||||
let first = run(&url, &[DRAFT]);
|
||||
assert!(first.status.success(), "{}", text(&first));
|
||||
{
|
||||
let held = state.lock().unwrap();
|
||||
assert_eq!(held.objects.get(&format!("{SHA}.svg")), Some(&horse()));
|
||||
assert_eq!(held.row, Some((SHA.to_owned(), true)));
|
||||
}
|
||||
let before = writes(&state);
|
||||
let again = run(&url, &[DRAFT]);
|
||||
let said = text(&again);
|
||||
assert!(again.status.success(), "{said}");
|
||||
assert!(said.contains("nothing to do"), "{said}");
|
||||
assert_eq!(writes(&state), before);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn different_bytes_under_the_key_are_never_overwritten() {
|
||||
let (state, url) = stub();
|
||||
state
|
||||
.lock()
|
||||
.unwrap()
|
||||
.objects
|
||||
.insert(format!("{SHA}.svg"), b"other".to_vec());
|
||||
let output = run(&url, &[DRAFT]);
|
||||
let said = text(&output);
|
||||
assert_eq!(output.status.code(), Some(1), "{said}");
|
||||
assert!(
|
||||
said.contains("State") && said.contains("other bytes"),
|
||||
"{said}"
|
||||
);
|
||||
assert_eq!(writes(&state), 0);
|
||||
assert_eq!(
|
||||
state.lock().unwrap().objects[&format!("{SHA}.svg")],
|
||||
b"other"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_missing_or_foreign_row_stops_the_run_before_any_write() {
|
||||
for row in [None, Some(("f".repeat(64), false))] {
|
||||
let (state, url) = stub();
|
||||
state.lock().unwrap().row = row;
|
||||
let output = run(&url, &[DRAFT]);
|
||||
assert_eq!(output.status.code(), Some(1), "{}", text(&output));
|
||||
assert_eq!(writes(&state), 0);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_refused_key_is_an_access_refusal_and_never_printed() {
|
||||
for status in [401, 403] {
|
||||
let (state, url) = stub();
|
||||
state.lock().unwrap().bucket_status = status;
|
||||
let output = run(&url, &[DRAFT]);
|
||||
let said = text(&output);
|
||||
assert_eq!(output.status.code(), Some(1), "{said}");
|
||||
assert!(said.contains("Access") && !said.contains(KEY), "{said}");
|
||||
assert_eq!(writes(&state), 0);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_failure_after_the_upload_resumes_with_the_confirmation_only() {
|
||||
let (state, url) = stub();
|
||||
state.lock().unwrap().fail_confirm = true;
|
||||
let broken = run(&url, &[DRAFT]);
|
||||
assert_eq!(broken.status.code(), Some(3), "{}", text(&broken));
|
||||
assert!(
|
||||
state
|
||||
.lock()
|
||||
.unwrap()
|
||||
.objects
|
||||
.contains_key(&format!("{SHA}.svg"))
|
||||
);
|
||||
assert_eq!(state.lock().unwrap().row, Some((SHA.to_owned(), false)));
|
||||
{
|
||||
let mut held = state.lock().unwrap();
|
||||
held.fail_confirm = false;
|
||||
held.log.clear();
|
||||
}
|
||||
let resumed = run(&url, &[DRAFT]);
|
||||
assert!(resumed.status.success(), "{}", text(&resumed));
|
||||
let log = state.lock().unwrap().log.clone();
|
||||
assert!(
|
||||
log.iter()
|
||||
.all(|l| !l.contains("/storage/v1/object") || l.starts_with("GET"))
|
||||
);
|
||||
assert_eq!(state.lock().unwrap().row, Some((SHA.to_owned(), true)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_unreachable_destination_is_a_network_error_after_bounded_retries() {
|
||||
let free = TcpListener::bind("127.0.0.1:0")
|
||||
.unwrap()
|
||||
.local_addr()
|
||||
.unwrap()
|
||||
.port();
|
||||
let output = run(&format!("http://127.0.0.1:{free}"), &[DRAFT]);
|
||||
let said = text(&output);
|
||||
assert_eq!(output.status.code(), Some(3), "{said}");
|
||||
assert!(said.contains("Network") && !said.contains(KEY), "{said}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn destinations_that_are_not_a_local_stack_are_refused_before_any_request() {
|
||||
for url in [
|
||||
"https://127.0.0.1:54321",
|
||||
"http://127.0.0.1",
|
||||
"http://10.1.2.3:54321",
|
||||
"http://example.test:54321",
|
||||
"http://user:pw@127.0.0.1:54321",
|
||||
"http://127.0.0.1:54321/rest/v1",
|
||||
"",
|
||||
] {
|
||||
let output = run(url, &[DRAFT]);
|
||||
let said = text(&output);
|
||||
assert_eq!(output.status.code(), Some(1), "{url}: {said}");
|
||||
assert!(said.contains("Destination"), "{url}: {said}");
|
||||
assert!(!said.contains("pw"), "{url}: {said}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_folder_outside_the_seed_is_refused() {
|
||||
let (state, url) = stub();
|
||||
let outside = concat!(env!("CARGO_MANIFEST_DIR"), "/src");
|
||||
let output = run(&url, &["--check", outside]);
|
||||
assert_eq!(output.status.code(), Some(1), "{}", text(&output));
|
||||
assert!(state.lock().unwrap().log.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_usage_error_names_the_usage() {
|
||||
let (_, url) = stub();
|
||||
let output = run(&url, &["--check"]);
|
||||
assert_eq!(output.status.code(), Some(1));
|
||||
assert!(text(&output).contains("usage"));
|
||||
}
|
||||
Reference in new issue
Block a user