feat(supabase): R5.5 upload-sheet-assets in Rust, R5.4 caps and FLOOR clarification
Core rules curation::upload (manifest, local destination, bytes through the SVG reader, plan from observed state); binary upload-sheet-assets over ureq (handshake, no overwrite, confirm after verify, bounded retries, key never printed); pnpm assets:upload repointed, e2e:sheet-assets for the local stack. 4 MiB cap on changes.json and references.json. FLOOR clarification for audit prose with its negative proof.
This commit is contained in:
1 parent
637ea6334c
commit
2c445063a4
20 files changed
+1595
-21
No files matched your search
@@ -375,10 +375,21 @@ fn refuse(file: &str, error: impl std::fmt::Display) -> PipelineError {
|
||||
PipelineError::Source(format!("{file}: {error}"))
|
||||
}
|
||||
|
||||
/// The size cap of an authored curation file (`changes.json`, `references.json`), checked
|
||||
/// before any parse allocation. The committed files are about 222 KB and 17 KB; the cap leaves
|
||||
/// a wide margin to grow and bounds the parse (RUST.md §11.16).
|
||||
pub const MAX_CURATION_FILE_BYTES: usize = 4 * 1024 * 1024;
|
||||
|
||||
/// `JSON.parse`: a repeated key keeps its first position and its last value (the parsed
|
||||
/// value is read into the typed shape afterwards, so duplicates never reach serde's derive),
|
||||
/// array index keys come first, integers beyond 2^53 are rounded (`js_normalize`).
|
||||
fn json_parse(file: &str, text: &str) -> Result<Value, PipelineError> {
|
||||
if text.len() > MAX_CURATION_FILE_BYTES {
|
||||
return Err(refuse(
|
||||
file,
|
||||
format!("larger than {MAX_CURATION_FILE_BYTES} bytes"),
|
||||
));
|
||||
}
|
||||
let mut value = js::parse_json(text).map_err(|error| refuse(file, error))?;
|
||||
js_normalize(&mut value);
|
||||
Ok(value)
|
||||
|
||||
@@ -16,6 +16,7 @@ pub mod markdown;
|
||||
pub mod prettier_json;
|
||||
pub mod report_guard;
|
||||
pub mod text;
|
||||
pub mod upload;
|
||||
|
||||
/// Why the pipeline refuses to produce a file. The TypeScript accepted some of these inputs
|
||||
/// and wrote whatever JavaScript made of them; the port refuses instead (declared in
|
||||
|
||||
@@ -0,0 +1,287 @@
|
||||
//! The rules of the sheet-asset upload (R5.5): the manifest, the destination, the bytes, and
|
||||
//! what to do given what the destination already holds. Values in, values out; the
|
||||
//! `upload-sheet-assets` binary owns the file system, the environment and the network.
|
||||
|
||||
use std::net::{IpAddr, Ipv4Addr};
|
||||
|
||||
use serde::Deserialize;
|
||||
use url::{Host, Url};
|
||||
|
||||
use crate::anatomy::svg::read_svg;
|
||||
use crate::curation::ids::sha256_hex;
|
||||
|
||||
/// The private Storage bucket of the sheet drawings (`20261009100000_sheet_asset_storage`).
|
||||
pub const BUCKET: &str = "anatomical-sheet-assets";
|
||||
/// The manifest of a generated seed folder, a few hundred bytes today.
|
||||
pub const MAX_MANIFEST_BYTES: usize = 1024 * 1024;
|
||||
/// Entries one run handles (a seed holds a handful of drawings).
|
||||
pub const MAX_ENTRIES: usize = 64;
|
||||
/// One drawing; the bucket itself refuses beyond 20 MiB.
|
||||
pub const MAX_ASSET_BYTES: usize = 16 * 1024 * 1024;
|
||||
|
||||
/// The closed kinds a refusal falls into; the message names the file or the value.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum UploadKind {
|
||||
/// `assets.json` is malformed, too large, or lists a bad or repeated entry.
|
||||
Manifest,
|
||||
/// The destination is not a local stack the tool is allowed to write to.
|
||||
Destination,
|
||||
/// A local file does not match its entry, or is not a drawing the product accepts.
|
||||
Content,
|
||||
/// The destination answered something other than the expected local stack.
|
||||
Handshake,
|
||||
/// The destination holds different bytes under a content-addressed key, or no row for the
|
||||
/// asset: nothing is overwritten, nothing is invented.
|
||||
State,
|
||||
/// The destination refused the key (401, 403).
|
||||
Access,
|
||||
/// The destination could not be reached, or answered with a server error.
|
||||
Network,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct UploadError {
|
||||
pub kind: UploadKind,
|
||||
pub message: String,
|
||||
}
|
||||
|
||||
impl std::fmt::Display for UploadError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
write!(f, "{:?}: {}", self.kind, self.message)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn refuse(kind: UploadKind, message: impl Into<String>) -> UploadError {
|
||||
UploadError {
|
||||
kind,
|
||||
message: message.into(),
|
||||
}
|
||||
}
|
||||
|
||||
/// One line of `assets.json`, as the generated seed writes it.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct AssetEntry {
|
||||
pub file: String,
|
||||
pub sha256: String,
|
||||
pub byte_size: usize,
|
||||
pub svg_asset_id: String,
|
||||
}
|
||||
|
||||
impl AssetEntry {
|
||||
/// The Storage object key: content-addressed, so the same bytes are the same object.
|
||||
pub fn object_key(&self) -> String {
|
||||
format!("{}.svg", self.sha256)
|
||||
}
|
||||
}
|
||||
|
||||
fn is_lower_hex(text: &str, length: usize) -> bool {
|
||||
text.len() == length && text.bytes().all(|b| matches!(b, b'0'..=b'9' | b'a'..=b'f'))
|
||||
}
|
||||
|
||||
fn is_uuid(text: &str) -> bool {
|
||||
let parts: Vec<&str> = text.split('-').collect();
|
||||
let lengths = [8, 4, 4, 4, 12];
|
||||
parts.len() == 5
|
||||
&& parts
|
||||
.iter()
|
||||
.zip(lengths)
|
||||
.all(|(part, length)| is_lower_hex(part, length))
|
||||
}
|
||||
|
||||
/// A plain file name of a drawing: no separator, no leading dot, `.svg`.
|
||||
fn is_plain_svg_name(name: &str) -> bool {
|
||||
name.len() <= 100
|
||||
&& name.ends_with(".svg")
|
||||
&& !name.starts_with('.')
|
||||
&& name
|
||||
.bytes()
|
||||
.all(|b| b.is_ascii_alphanumeric() || matches!(b, b'.' | b'_' | b'-'))
|
||||
}
|
||||
|
||||
fn check_entry(entry: &AssetEntry) -> Result<(), UploadError> {
|
||||
let bad = |what: &str| refuse(UploadKind::Manifest, format!("{}: {what}", entry.file));
|
||||
if !is_plain_svg_name(&entry.file) {
|
||||
return Err(refuse(
|
||||
UploadKind::Manifest,
|
||||
"file must be a plain .svg name",
|
||||
));
|
||||
}
|
||||
if !is_lower_hex(&entry.sha256, 64) {
|
||||
return Err(bad("sha256 must be 64 lowercase hex digits"));
|
||||
}
|
||||
if !is_uuid(&entry.svg_asset_id) {
|
||||
return Err(bad("svg_asset_id must be a lowercase uuid"));
|
||||
}
|
||||
if entry.byte_size == 0 || entry.byte_size > MAX_ASSET_BYTES {
|
||||
return Err(bad("byte_size out of range"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn repeated(entries: &[AssetEntry]) -> Option<String> {
|
||||
for (index, entry) in entries.iter().enumerate() {
|
||||
let twin = entries[..index].iter().any(|other| {
|
||||
other.file == entry.file
|
||||
|| other.sha256 == entry.sha256
|
||||
|| other.svg_asset_id == entry.svg_asset_id
|
||||
});
|
||||
if twin {
|
||||
return Some(entry.file.clone());
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Reads `assets.json`: size-bounded before the parse, strict on keys, every entry checked,
|
||||
/// no repeated file, hash or asset id.
|
||||
pub fn parse_manifest(text: &str) -> Result<Vec<AssetEntry>, UploadError> {
|
||||
if text.len() > MAX_MANIFEST_BYTES {
|
||||
return Err(refuse(UploadKind::Manifest, "assets.json is too large"));
|
||||
}
|
||||
let entries: Vec<AssetEntry> = serde_json::from_str(text)
|
||||
.map_err(|error| refuse(UploadKind::Manifest, format!("assets.json: {error}")))?;
|
||||
if entries.is_empty() || entries.len() > MAX_ENTRIES {
|
||||
return Err(refuse(UploadKind::Manifest, "assets.json: 1 to 64 entries"));
|
||||
}
|
||||
entries.iter().try_for_each(check_entry)?;
|
||||
if let Some(file) = repeated(&entries) {
|
||||
return Err(refuse(
|
||||
UploadKind::Manifest,
|
||||
format!("{file}: repeated entry"),
|
||||
));
|
||||
}
|
||||
Ok(entries)
|
||||
}
|
||||
|
||||
/// The bytes on disk are what the manifest recorded, and a drawing the product accepts: the
|
||||
/// core's one SVG reader keeps every element (nothing removed), as a load of the sheet would.
|
||||
pub fn verify_asset(entry: &AssetEntry, bytes: &[u8]) -> Result<(), UploadError> {
|
||||
let bad = |what: &str| refuse(UploadKind::Content, format!("{}: {what}", entry.file));
|
||||
if bytes.len() != entry.byte_size || sha256_hex(bytes) != entry.sha256 {
|
||||
return Err(bad("bytes differ from assets.json"));
|
||||
}
|
||||
let text = std::str::from_utf8(bytes).map_err(|_| bad("not UTF-8 text"))?;
|
||||
match read_svg(text) {
|
||||
Ok(read) if read.removed.is_empty() => Ok(()),
|
||||
Ok(_) => Err(bad("the SVG reader would remove content")),
|
||||
Err(error) => Err(bad(&format!("refused by the SVG reader: {error:?}"))),
|
||||
}
|
||||
}
|
||||
|
||||
/// A local destination the tool may write to: `http`, an explicit port, no credentials, query
|
||||
/// or path, and a host that is this machine (loopback) or the configured DEV address.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct Destination {
|
||||
origin: String,
|
||||
}
|
||||
|
||||
impl Destination {
|
||||
/// `http://host:port`, no trailing slash.
|
||||
pub fn origin(&self) -> &str {
|
||||
&self.origin
|
||||
}
|
||||
}
|
||||
|
||||
fn is_private_v4(ip: Ipv4Addr) -> bool {
|
||||
ip.is_loopback() || ip.is_private()
|
||||
}
|
||||
|
||||
fn host_is_allowed(host: &Host<&str>, dev_host: Option<&str>) -> bool {
|
||||
match host {
|
||||
Host::Domain(name) => name.eq_ignore_ascii_case("localhost"),
|
||||
Host::Ipv6(ip) => ip.is_loopback(),
|
||||
Host::Ipv4(ip) => {
|
||||
ip.is_loopback()
|
||||
|| (is_private_v4(*ip)
|
||||
&& dev_host.and_then(|d| d.parse::<IpAddr>().ok()) == Some(IpAddr::V4(*ip)))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Refuses anything but a local stack; there is no default to fall back to.
|
||||
pub fn check_destination(raw: &str, dev_host: Option<&str>) -> Result<Destination, UploadError> {
|
||||
let bad = |what: &str| refuse(UploadKind::Destination, what);
|
||||
let url = Url::parse(raw).map_err(|_| bad("not a URL"))?;
|
||||
if url.scheme() != "http" {
|
||||
return Err(bad("only http to a local stack"));
|
||||
}
|
||||
if !url.username().is_empty() || url.password().is_some() {
|
||||
return Err(bad("no credentials in the URL"));
|
||||
}
|
||||
if url.query().is_some() || url.fragment().is_some() || url.path() != "/" {
|
||||
return Err(bad("an origin only: no path, query or fragment"));
|
||||
}
|
||||
let port = url
|
||||
.port()
|
||||
.ok_or_else(|| bad("an explicit port is required"))?;
|
||||
let host = url.host().ok_or_else(|| bad("no host"))?;
|
||||
if !host_is_allowed(&host, dev_host) {
|
||||
return Err(bad(
|
||||
"host is not this machine or the configured DEV address",
|
||||
));
|
||||
}
|
||||
Ok(Destination {
|
||||
origin: format!("http://{}:{port}", host_text(&host)),
|
||||
})
|
||||
}
|
||||
|
||||
fn host_text(host: &Host<&str>) -> String {
|
||||
match host {
|
||||
Host::Ipv6(ip) => format!("[{ip}]"),
|
||||
other => other.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
/// What the Storage object `<sha256>.svg` holds at the destination.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum ObjectState {
|
||||
Absent,
|
||||
/// Present, and its bytes hash to the key.
|
||||
Verified,
|
||||
/// Present under this key but with other bytes (the hash found).
|
||||
Different(String),
|
||||
}
|
||||
|
||||
/// The `anatomical_sheet_assets` row of the entry, as the destination reports it.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum RowState {
|
||||
Unknown,
|
||||
/// The row records another hash than the manifest.
|
||||
OtherHash(String),
|
||||
Pending,
|
||||
Confirmed,
|
||||
}
|
||||
|
||||
/// What a run does for one entry; both false means nothing to do.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct Plan {
|
||||
pub upload: bool,
|
||||
pub confirm: bool,
|
||||
}
|
||||
|
||||
/// Decides the steps of one entry from what the destination holds, or refuses: nothing is
|
||||
/// ever overwritten, and a row the seed did not create is not guessed at.
|
||||
pub fn plan_entry(
|
||||
entry: &AssetEntry,
|
||||
object: &ObjectState,
|
||||
row: &RowState,
|
||||
) -> Result<Plan, UploadError> {
|
||||
let bad = |what: String| refuse(UploadKind::State, format!("{}: {what}", entry.file));
|
||||
match row {
|
||||
RowState::Unknown => return Err(bad("no asset row (apply the seed first)".into())),
|
||||
RowState::OtherHash(found) => {
|
||||
return Err(bad(format!("the asset row records sha256 {found}")));
|
||||
}
|
||||
RowState::Pending | RowState::Confirmed => {}
|
||||
}
|
||||
if let ObjectState::Different(found) = object {
|
||||
return Err(bad(format!(
|
||||
"the object holds other bytes (sha256 {found})"
|
||||
)));
|
||||
}
|
||||
Ok(Plan {
|
||||
upload: *object == ObjectState::Absent,
|
||||
confirm: *row == RowState::Pending,
|
||||
})
|
||||
}
|
||||
Reference in new issue
Block a user