diff --git a/Cargo.lock b/Cargo.lock index 71145926..8f470cab 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -14,12 +14,27 @@ version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + [[package]] name = "bitflags" version = "2.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + [[package]] name = "bumpalo" version = "3.20.3" @@ -38,6 +53,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495" +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + [[package]] name = "cc" version = "1.8.0" @@ -56,6 +77,50 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + +[[package]] +name = "cookie" +version = "0.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a373e3602691c3cdea496d2f0ee5935151e6168fe87739483c463db1b2f2f87" +dependencies = [ + "percent-encoding", + "time", + "version_check", +] + +[[package]] +name = "cookie_store" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15b2c103cf610ec6cae3da84a766285b42fd16aad564758459e6ecf128c75206" +dependencies = [ + "cookie", + "document-features", + "idna", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "time", + "url", +] + +[[package]] +name = "cpufeatures" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" +dependencies = [ + "libc", +] + [[package]] name = "crc32fast" version = "1.5.2" @@ -65,6 +130,52 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "deranged" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e9de72ce2ad1f90dc62fa25f0f430ef85eb4b0d8fa0be4f30373bc40a21d28e" + +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer", + "const-oid", + "crypto-common", +] + +[[package]] +name = "displaydoc" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "document-features" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4b8a88685455ed29a21542a33abd9cb6510b6b129abadabdcef0f4c55bc8f61" +dependencies = [ + "litrs", +] + [[package]] name = "equivalent" version = "1.0.2" @@ -99,6 +210,10 @@ dependencies = [ "libwebp-sys", "rlimit", "seccompiler", + "serde", + "serde_json", + "sha2", + "ureq", ] [[package]] @@ -133,6 +248,26 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + [[package]] name = "getrandom" version = "0.4.3" @@ -156,6 +291,135 @@ version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +[[package]] +name = "http" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "hybrid-array" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17" +dependencies = [ + "typenum", +] + +[[package]] +name = "icu_collections" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f9cf5f235641ed274641dd81c3f28d870e276763d0797aeeab72317b1c646f" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0" + +[[package]] +name = "icu_properties" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e7ca276ad3145661a65914e6daf131ca5120cd3dcee8f8f3214b8875184a148" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa" + +[[package]] +name = "icu_provider" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + [[package]] name = "image" version = "0.25.10" @@ -204,7 +468,7 @@ version = "0.1.35" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" dependencies = [ - "getrandom", + "getrandom 0.4.3", "libc", ] @@ -225,6 +489,24 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "litemap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" + +[[package]] +name = "litrs" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4744e383959f0db86ede514b809b1c53251889093803c05267acc7d4e7030d70" + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + [[package]] name = "memchr" version = "2.8.3" @@ -261,6 +543,12 @@ dependencies = [ "pxfm", ] +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + [[package]] name = "num-traits" version = "0.2.19" @@ -276,6 +564,12 @@ version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + [[package]] name = "pkg-config" version = "0.3.34" @@ -295,6 +589,21 @@ dependencies = [ "miniz_oxide 0.8.9", ] +[[package]] +name = "potential_utf" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" +dependencies = [ + "zerovec", +] + +[[package]] +name = "powerfmt" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a6394b9e965e73d0a289ee54f589087e2c676aedf60885baf52c76b771e4958" + [[package]] name = "proc-macro2" version = "1.0.107" @@ -331,6 +640,20 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + [[package]] name = "rlimit" version = "0.11.0" @@ -340,6 +663,41 @@ dependencies = [ "libc", ] +[[package]] +name = "rustls" +version = "0.23.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" +dependencies = [ + "log", + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pki-types" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" +dependencies = [ + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + [[package]] name = "rustversion" version = "1.0.23" @@ -400,6 +758,17 @@ dependencies = [ "zmij", ] +[[package]] +name = "sha2" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d7069beb7d6ac7b9acd1039986e73443f24234f41074da099d6f994ac9ad19" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "shlex" version = "2.0.1" @@ -412,6 +781,24 @@ version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" +[[package]] +name = "smallvec" +version = "1.16.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b3dc8af474f516a851ff4bd12db780f948b9250ad37211e4eec0bccea54e01b" + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + [[package]] name = "syn" version = "2.0.119" @@ -434,6 +821,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "synstructure" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "901704edd0dfe137f1987838ee4f259e4e063c31371bdb423f7ae38ec6f77f02" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + [[package]] name = "termcolor" version = "1.4.1" @@ -463,6 +861,46 @@ dependencies = [ "syn 3.0.9", ] +[[package]] +name = "time" +version = "0.3.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb87b95ec50ddfa440816d227a17b2ccbdda963a316a727fda0fc4334f7d134" +dependencies = [ + "deranged", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" + +[[package]] +name = "time-macros" +version = "0.2.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e689342a48d2ea927c87ea50cabf8594854bf940e9310208848d680d668ed85" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tinystr" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" +dependencies = [ + "displaydoc", + "zerovec", +] + [[package]] name = "ts-rs" version = "12.0.1" @@ -485,12 +923,91 @@ dependencies = [ "termcolor", ] +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + [[package]] name = "unicode-ident" version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "ureq" +version = "3.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a7ac20be9b7726e0bbdbf974c059676d9acb1cd414961f570a4e8231cacd7fc" +dependencies = [ + "base64", + "cookie_store", + "log", + "percent-encoding", + "rustls", + "rustls-pki-types", + "serde", + "serde_json", + "ureq-proto", + "utf8-zero", + "webpki-roots", +] + +[[package]] +name = "ureq-proto" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f86fd172ccca569e458f61b6bdd6220965a9ef36e672a6852953b51a0e1583be" +dependencies = [ + "base64", + "http", + "httparse", + "log", +] + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8-zero" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8c0a043c9540bae7c578c88f91dda8bd82e59ae27c21baca69c8b191aaf5a6e" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + [[package]] name = "wasm-bindgen" version = "0.2.129" @@ -536,13 +1053,22 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "webpki-roots" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "winapi-util" version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -551,6 +1077,15 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + [[package]] name = "windows-sys" version = "0.61.2" @@ -560,6 +1095,159 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "writeable" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec8ebde2db3681e8c9980cc27822030e68752690ddfa9473e739aeb4dbde6d71" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", + "synstructure", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f75b4683f6c7f45248d4d64056a24298c6281e0993356d7d1b4a1a962ef10d4a" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13084392c5e4bc371903e2935a5eaeed24905a7511356b883835e18a78f6879" + +[[package]] +name = "zerotrie" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + [[package]] name = "zlib-rs" version = "0.6.8" diff --git a/Cargo.toml b/Cargo.toml index b8d3faf9..5efce14c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,6 +29,10 @@ libwebp-sys = "0.14.4" seccompiler = "0.5.0" rlimit = "0.11" libc = "0.2" +# Worker job loop (R1): blocking HTTP client (rustls + ring, bundled webpki roots for a managed +# Supabase over HTTPS) and the hash of the clean output. No async runtime: the loop is sequential. +ureq = { version = "3.4.2", default-features = false, features = ["rustls", "json"] } +sha2 = "0.11" # The library crates deny unwrap/expect so that no panic path depends on content; # the worker binary relaxes disallowed_methods (it owns I/O) in its own [lints]. diff --git a/crates/worker/Cargo.toml b/crates/worker/Cargo.toml index d74f8a80..28430201 100644 --- a/crates/worker/Cargo.toml +++ b/crates/worker/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "faunosteo-worker" -description = "Faunosteo file-processing worker: job loop (R1) and the sandboxed decode child (R0 proof)" +description = "Faunosteo file-processing worker: keyed job loop and sweeper (R1), sandboxed decode child (R0)" version.workspace = true edition.workspace = true license.workspace = true @@ -26,6 +26,10 @@ libwebp-sys.workspace = true seccompiler.workspace = true rlimit.workspace = true libc.workspace = true +serde.workspace = true +serde_json.workspace = true +ureq.workspace = true +sha2.workspace = true crc32fast = { version = "1.4", optional = true } # Already in the tree through png: used by the iCCP-bomb fixture only. flate2 = { version = "1.1", optional = true } diff --git a/crates/worker/src/backend.rs b/crates/worker/src/backend.rs new file mode 100644 index 00000000..d171e41e --- /dev/null +++ b/crates/worker/src/backend.rs @@ -0,0 +1,107 @@ +//! What the parent needs from Supabase (STORAGE.md §2.2 RPCs and §3.3 Storage calls), +//! behind one trait so the job and sweep logic is tested on an in-memory fake and the HTTP +//! implementation stays thin. + +use std::fmt; + +use serde::Deserialize; + +/// One claimed row of `assets` (`claim_avatar_jobs`). +#[derive(Debug, Clone, Deserialize, PartialEq, Eq)] +pub struct Job { + pub asset_id: String, + pub owner_id: String, + pub bucket_id: String, + pub object_key: String, + pub quarantine_key: Option, + pub lock_token: String, + pub attempts: i64, +} + +/// One claimed purge (`claim_purges`). +#[derive(Debug, Clone, Deserialize, PartialEq, Eq)] +pub struct Purge { + pub purge_id: String, + pub bucket_id: String, + pub object_name: String, +} + +/// One quarantine object without a live row (`claim_quarantine_leftovers`). +#[derive(Debug, Clone, Deserialize, PartialEq, Eq)] +pub struct Leftover { + pub bucket_id: String, + pub object_name: String, + pub asset_id: Option, +} + +/// Closed failure kinds of a backend call. `Refused` carries the closed code a `P0001` +/// refusal names (`invalid_state`, `invalid_reason`…); every other kind is transport-side. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum BackendError { + Network(String), + Status(u16), + Refused(String), + Decode(String), + /// The downloaded body exceeded the input cap. + Overflow, + Internal(String), +} + +impl fmt::Display for BackendError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Network(detail) => write!(f, "network: {detail}"), + Self::Status(status) => write!(f, "http status {status}"), + Self::Refused(code) => write!(f, "refused: {code}"), + Self::Decode(detail) => write!(f, "decode: {detail}"), + Self::Overflow => f.write_str("body over the cap"), + Self::Internal(detail) => write!(f, "internal: {detail}"), + } + } +} + +impl std::error::Error for BackendError {} + +impl BackendError { + /// Stable name for the `code` log field. + pub fn kind(&self) -> &'static str { + match self { + Self::Network(_) => "network", + Self::Status(_) => "status", + Self::Refused(_) => "refused", + Self::Decode(_) => "decode", + Self::Overflow => "overflow", + Self::Internal(_) => "internal", + } + } +} + +pub type BackendResult = Result; + +/// The eleven operations of STORAGE.md §2.2 and §3.3. +pub trait Backend { + fn claim_jobs(&self, max_jobs: u32) -> BackendResult>; + /// `Status(404)` when the object never landed; `Overflow` past the input cap. + fn download_quarantine(&self, bucket: &str, key: &str) -> BackendResult>; + /// Upsert of the clean bytes at the final key (idempotent on retry). + fn put_clean(&self, bucket: &str, key: &str, bytes: &[u8]) -> BackendResult<()>; + fn complete( + &self, + asset_id: &str, + lock_token: &str, + byte_size: usize, + sha256: &str, + ) -> BackendResult<()>; + fn reject(&self, asset_id: &str, lock_token: &str, reason: &str) -> BackendResult<()>; + /// A missing object counts as deleted. + fn delete_object(&self, bucket: &str, key: &str) -> BackendResult<()>; + fn mark_quarantine_purged(&self, asset_id: &str) -> BackendResult<()>; + fn claim_purges(&self, max_items: u32) -> BackendResult>; + fn ack_purge(&self, purge_id: &str) -> BackendResult<()>; + fn claim_quarantine_leftovers( + &self, + older_than_seconds: u64, + max_items: u32, + ) -> BackendResult>; + fn expire_pending_uploads(&self, max_age_seconds: u64) -> BackendResult; +} diff --git a/crates/worker/src/config.rs b/crates/worker/src/config.rs new file mode 100644 index 00000000..3453f0b6 --- /dev/null +++ b/crates/worker/src/config.rs @@ -0,0 +1,284 @@ +//! Configuration of the keyed parent (STORAGE.md §3.2): read from the environment once at +//! start, never from the child. The service key is a `Secret` that no `Debug` or `Display` +//! output can print. + +use std::fmt; +use std::time::Duration; + +/// A value that must never appear in a log line or an error message. +#[derive(Clone)] +pub struct Secret(String); + +impl Secret { + pub fn new(value: String) -> Self { + Self(value) + } + + /// The only accessor; call sites are the HTTP headers and the JWT expiry parser. + pub fn expose(&self) -> &str { + &self.0 + } +} + +impl fmt::Debug for Secret { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("") + } +} + +/// Resolved worker configuration. +#[derive(Debug, Clone)] +pub struct Config { + /// Origin of the Supabase stack, without a trailing slash. + pub url: String, + pub service_key: Secret, + pub worker_id: String, + pub poll: Duration, + pub sweep: Duration, + pub quarantine_grace: Duration, + pub pending_max_age: Duration, + /// One claim pass and one sweep, then exit (tests). + pub once: bool, +} + +/// Why the configuration could not be built (the variable name, never its value). +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ConfigError { + Missing(&'static str), + Invalid(&'static str), +} + +impl fmt::Display for ConfigError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Missing(name) => write!(f, "{name} is required"), + Self::Invalid(name) => write!(f, "{name} is invalid"), + } + } +} + +impl std::error::Error for ConfigError {} + +const URL: &str = "SUPABASE_URL"; +const KEY: &str = "SUPABASE_SERVICE_ROLE_KEY"; +const WORKER_ID: &str = "WORKER_ID"; +const POLL: &str = "WORKER_POLL_SECONDS"; +const SWEEP: &str = "WORKER_SWEEP_SECONDS"; +const QUARANTINE_GRACE: &str = "WORKER_QUARANTINE_GRACE_SECONDS"; +const PENDING_MAX_AGE: &str = "WORKER_PENDING_MAX_AGE_SECONDS"; +const ONCE: &str = "WORKER_ONCE"; + +/// Names of the variables the worker reads, for `run --check`. +pub const VARIABLES: &[&str] = &[ + URL, + KEY, + WORKER_ID, + POLL, + SWEEP, + QUARANTINE_GRACE, + PENDING_MAX_AGE, + ONCE, +]; + +impl Config { + /// Builds the configuration from the process environment. + pub fn from_env() -> Result { + Self::from_vars(|name| std::env::var(name).ok()) + } + + /// Builds the configuration from any variable source (tests). + pub fn from_vars(var: impl Fn(&str) -> Option) -> Result { + let url = parse_url(var(URL).ok_or(ConfigError::Missing(URL))?)?; + let key = var(KEY).filter(|k| !k.trim().is_empty()); + let service_key = Secret::new(key.ok_or(ConfigError::Missing(KEY))?); + let worker_id = match var(WORKER_ID).filter(|id| !id.trim().is_empty()) { + Some(id) + if id.len() <= 64 + && id + .chars() + .all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_') => + { + id + } + Some(_) => return Err(ConfigError::Invalid(WORKER_ID)), + None => format!("worker-{}", std::process::id()), + }; + Ok(Self { + url, + service_key, + worker_id, + poll: seconds(&var, POLL, 2)?, + sweep: seconds(&var, SWEEP, 600)?, + quarantine_grace: seconds(&var, QUARANTINE_GRACE, 10_800)?, + pending_max_age: seconds(&var, PENDING_MAX_AGE, 86_400)?, + once: var(ONCE).is_some_and(|v| v == "1"), + }) + } +} + +/// `http(s)://host[:port]`, no path, no whitespace; a trailing slash is dropped. +fn parse_url(raw: String) -> Result { + let url = raw.trim().trim_end_matches('/').to_owned(); + let rest = url + .strip_prefix("http://") + .or_else(|| url.strip_prefix("https://")) + .ok_or(ConfigError::Invalid(URL))?; + let host_ok = !rest.is_empty() + && rest + .chars() + .all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '-' | ':' | '_')); + if host_ok { + Ok(url) + } else { + Err(ConfigError::Invalid(URL)) + } +} + +fn seconds( + var: &impl Fn(&str) -> Option, + name: &'static str, + default: u64, +) -> Result { + match var(name) { + None => Ok(Duration::from_secs(default)), + Some(raw) => raw + .trim() + .parse::() + .map(Duration::from_secs) + .map_err(|_| ConfigError::Invalid(name)), + } +} + +/// `exp` claim of a JWT (three base64url parts), when the key is one. A `sb_secret_…` +/// key is not a JWT and yields `None`. +pub fn jwt_exp(token: &str) -> Option { + let mut parts = token.split('.'); + let (_, payload, _) = (parts.next()?, parts.next()?, parts.next()?); + if parts.next().is_some() { + return None; + } + let json = base64url_decode(payload)?; + let value: serde_json::Value = serde_json::from_slice(&json).ok()?; + value.get("exp")?.as_i64() +} + +/// Base64url without padding (RFC 7515); padding characters are tolerated. +pub fn base64url_decode(text: &str) -> Option> { + let mut out = Vec::with_capacity(text.len() * 3 / 4); + let mut buffer: u32 = 0; + let mut bits = 0; + for byte in text.bytes() { + let value = match byte { + b'A'..=b'Z' => byte - b'A', + b'a'..=b'z' => byte - b'a' + 26, + b'0'..=b'9' => byte - b'0' + 52, + b'-' => 62, + b'_' => 63, + b'=' => break, + _ => return None, + }; + buffer = (buffer << 6) | u32::from(value); + bits += 6; + if bits >= 8 { + bits -= 8; + out.push(u8::try_from((buffer >> bits) & 0xff).ok()?); + } + } + Some(out) +} + +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used)] + use super::*; + use std::collections::HashMap; + + fn vars(pairs: &[(&str, &str)]) -> impl Fn(&str) -> Option { + let map: HashMap = pairs + .iter() + .map(|(k, v)| ((*k).to_owned(), (*v).to_owned())) + .collect(); + move |name| map.get(name).cloned() + } + + #[test] + fn defaults_apply_when_only_the_required_variables_are_set() { + let cfg = Config::from_vars(vars(&[ + ("SUPABASE_URL", "http://127.0.0.1:54321/"), + ("SUPABASE_SERVICE_ROLE_KEY", "k"), + ])) + .unwrap(); + assert_eq!(cfg.url, "http://127.0.0.1:54321"); + assert_eq!(cfg.poll, Duration::from_secs(2)); + assert_eq!(cfg.sweep, Duration::from_secs(600)); + assert_eq!(cfg.quarantine_grace, Duration::from_secs(10_800)); + assert_eq!(cfg.pending_max_age, Duration::from_secs(86_400)); + assert!(!cfg.once); + assert!(cfg.worker_id.starts_with("worker-")); + } + + #[test] + fn missing_and_invalid_variables_are_named() { + assert_eq!( + Config::from_vars(vars(&[("SUPABASE_SERVICE_ROLE_KEY", "k")])).err(), + Some(ConfigError::Missing("SUPABASE_URL")) + ); + assert_eq!( + Config::from_vars(vars(&[("SUPABASE_URL", "http://x")])).err(), + Some(ConfigError::Missing("SUPABASE_SERVICE_ROLE_KEY")) + ); + assert_eq!( + Config::from_vars(vars(&[ + ("SUPABASE_URL", "ftp://x"), + ("SUPABASE_SERVICE_ROLE_KEY", "k") + ])) + .err(), + Some(ConfigError::Invalid("SUPABASE_URL")) + ); + assert_eq!( + Config::from_vars(vars(&[ + ("SUPABASE_URL", "http://x"), + ("SUPABASE_SERVICE_ROLE_KEY", "k"), + ("WORKER_POLL_SECONDS", "two"), + ])) + .err(), + Some(ConfigError::Invalid("WORKER_POLL_SECONDS")) + ); + assert_eq!( + Config::from_vars(vars(&[ + ("SUPABASE_URL", "http://x"), + ("SUPABASE_SERVICE_ROLE_KEY", "k"), + ("WORKER_ID", "bad id"), + ])) + .err(), + Some(ConfigError::Invalid("WORKER_ID")) + ); + } + + #[test] + fn the_key_never_appears_in_debug_output() { + let cfg = Config::from_vars(vars(&[ + ("SUPABASE_URL", "http://x"), + ("SUPABASE_SERVICE_ROLE_KEY", "super-secret-value-9f8e7d"), + ("WORKER_ONCE", "1"), + ])) + .unwrap(); + let printed = format!("{cfg:?}"); + assert!(!printed.contains("super-secret-value-9f8e7d")); + assert!(printed.contains("")); + assert!(cfg.once); + } + + #[test] + fn jwt_exp_reads_the_payload_claim() { + // header {"alg":"HS256"} . payload {"role":"service_role","exp":1983812996} . sig + let token = + "eyJhbGciOiJIUzI1NiJ9.eyJyb2xlIjoic2VydmljZV9yb2xlIiwiZXhwIjoxOTgzODEyOTk2fQ.sig"; + assert_eq!(jwt_exp(token), Some(1_983_812_996)); + assert_eq!(jwt_exp("sb_secret_abc"), None); + assert_eq!(jwt_exp("a.b"), None); + assert_eq!(base64url_decode("aGk"), Some(b"hi".to_vec())); + assert_eq!(base64url_decode("aGk="), Some(b"hi".to_vec())); + assert_eq!(base64url_decode("a*"), None); + } +} diff --git a/crates/worker/src/decoder.rs b/crates/worker/src/decoder.rs new file mode 100644 index 00000000..60a73fde --- /dev/null +++ b/crates/worker/src/decoder.rs @@ -0,0 +1,29 @@ +//! The decode step as the job sees it: bytes in, a classified child run out. The real +//! implementation spawns this very binary as `decode` (R0 `Spawner`); tests inject outcomes. + +use crate::spawn::{ChildRun, Spawner}; + +pub trait Decoder { + fn decode(&self, bytes: &[u8]) -> ChildRun; +} + +/// Spawns `faunosteo-worker decode` from the running executable. +#[derive(Debug, Clone)] +pub struct ChildDecoder { + spawner: Spawner, +} + +impl ChildDecoder { + /// Fails only when the executable path cannot be resolved (then nothing can be decoded). + pub fn from_current_exe() -> std::io::Result { + Ok(Self { + spawner: Spawner::new(std::env::current_exe()?), + }) + } +} + +impl Decoder for ChildDecoder { + fn decode(&self, bytes: &[u8]) -> ChildRun { + self.spawner.decode(bytes) + } +} diff --git a/crates/worker/src/exit_codes.rs b/crates/worker/src/exit_codes.rs index f787e494..5d31575a 100644 --- a/crates/worker/src/exit_codes.rs +++ b/crates/worker/src/exit_codes.rs @@ -15,6 +15,10 @@ pub const OUTPUT_FAILED: i32 = 17; pub const INTERNAL: i32 = 18; /// The decoders' allocation or complexity budget was exceeded by the content. pub const RESOURCE_LIMIT: i32 = 19; +/// The parent stopped its loop because a child reported a broken environment (R1). +pub const CHILD_BROKEN: i32 = 20; +/// The parent's watchdog found a stage past its deadline and exited (R1). +pub const WATCHDOG: i32 = 21; /// Codes that judge the content (the upload is refused). Every other non-zero code means /// the child's environment is broken (`Broken` outcome: the parent alerts rather than @@ -42,6 +46,8 @@ pub fn name(code: i32) -> &'static str { OUTPUT_FAILED => "output_failed", INTERNAL => "internal", RESOURCE_LIMIT => "resource_limit", + CHILD_BROKEN => "child_broken", + WATCHDOG => "watchdog", _ => "unknown", } } diff --git a/crates/worker/src/http.rs b/crates/worker/src/http.rs new file mode 100644 index 00000000..9b302b45 --- /dev/null +++ b/crates/worker/src/http.rs @@ -0,0 +1,300 @@ +//! `Backend` over PostgREST and the Storage API with the service key (STORAGE.md §3.3). +//! Thin by design: key shapes are checked before any URL is built, every status is +//! classified here, and the closed `P0001` refusals of the RPCs come back as `Refused`. + +use std::time::Duration; + +use serde::de::DeserializeOwned; +use serde_json::{Value, json}; +use ureq::Agent; +use ureq::http::Response; + +use crate::backend::{Backend, BackendError, BackendResult, Job, Leftover, Purge}; +use crate::config::Config; +use crate::keys; +use crate::limits::MAX_INPUT_BYTES; + +/// Largest RPC response the worker reads (a claim of 100 rows is a few KiB). +const MAX_RPC_BODY: u64 = 1024 * 1024; + +pub struct HttpBackend { + agent: Agent, + base: String, + key: String, +} + +impl HttpBackend { + pub fn new(config: &Config) -> Self { + let agent: Agent = Agent::config_builder() + .timeout_global(Some(Duration::from_secs(30))) + .timeout_connect(Some(Duration::from_secs(5))) + .max_redirects(0) + .http_status_as_error(false) + .user_agent("faunosteo-worker") + .build() + .into(); + Self { + agent, + base: config.url.clone(), + key: config.service_key.expose().to_owned(), + } + } + + fn rpc_url(&self, name: &str) -> String { + format!("{}/rest/v1/rpc/{name}", self.base) + } + + fn object_url(&self, bucket: &str, key: &str) -> BackendResult { + validate_bucket(bucket)?; + keys::validate_any_key(key).map_err(|e| BackendError::Internal(e.to_string()))?; + Ok(format!("{}/storage/v1/object/{bucket}/{key}", self.base)) + } + + /// Calls an RPC and returns the response body text of a 2xx. + fn rpc(&self, name: &str, body: &Value) -> BackendResult { + let mut response = self + .agent + .post(&self.rpc_url(name)) + .header("apikey", &self.key) + .header("Authorization", &format!("Bearer {}", self.key)) + .header("Prefer", "return=representation") + .send_json(body) + .map_err(network)?; + let status = response.status().as_u16(); + let text = read_text(&mut response)?; + if (200..300).contains(&status) { + Ok(text) + } else { + Err(classify(status, &text)) + } + } + + fn rpc_json(&self, name: &str, body: &Value) -> BackendResult { + let text = self.rpc(name, body)?; + serde_json::from_str(&text).map_err(|e| BackendError::Decode(e.to_string())) + } + + fn rpc_void(&self, name: &str, body: &Value) -> BackendResult<()> { + self.rpc(name, body).map(|_| ()) + } +} + +impl Backend for HttpBackend { + fn claim_jobs(&self, max_jobs: u32) -> BackendResult> { + self.rpc_json("claim_avatar_jobs", &json!({ "max_jobs": max_jobs })) + } + + fn download_quarantine(&self, bucket: &str, key: &str) -> BackendResult> { + let url = self.object_url(bucket, key)?; + let mut response = self + .agent + .get(&url) + .header("apikey", &self.key) + .header("Authorization", &format!("Bearer {}", self.key)) + .call() + .map_err(network)?; + let status = response.status().as_u16(); + if status != 200 { + let text = read_text(&mut response).unwrap_or_default(); + return Err(classify(status, &text)); + } + let bytes = response + .body_mut() + .with_config() + .limit(MAX_INPUT_BYTES as u64 + 1) + .read_to_vec() + .map_err(|e| BackendError::Network(e.to_string()))?; + if bytes.len() > MAX_INPUT_BYTES { + return Err(BackendError::Overflow); + } + Ok(bytes) + } + + fn put_clean(&self, bucket: &str, key: &str, bytes: &[u8]) -> BackendResult<()> { + let url = self.object_url(bucket, key)?; + let mut response = self + .agent + .post(&url) + .header("apikey", &self.key) + .header("Authorization", &format!("Bearer {}", self.key)) + .header("Content-Type", "image/webp") + .header("x-upsert", "true") + .send(bytes) + .map_err(network)?; + expect_success(&mut response) + } + + fn complete( + &self, + asset_id: &str, + lock_token: &str, + byte_size: usize, + sha256: &str, + ) -> BackendResult<()> { + self.rpc_void( + "complete_avatar_job", + &json!({ "asset_id": asset_id, "lock_token": lock_token, "byte_size": byte_size, "sha256": sha256 }), + ) + } + + fn reject(&self, asset_id: &str, lock_token: &str, reason: &str) -> BackendResult<()> { + self.rpc_void( + "reject_avatar_job", + &json!({ "asset_id": asset_id, "lock_token": lock_token, "reason": reason }), + ) + } + + fn delete_object(&self, bucket: &str, key: &str) -> BackendResult<()> { + let url = self.object_url(bucket, key)?; + let mut response = self + .agent + .delete(&url) + .header("apikey", &self.key) + .header("Authorization", &format!("Bearer {}", self.key)) + .call() + .map_err(network)?; + if response.status().as_u16() == 404 { + return Ok(()); + } + expect_success(&mut response) + } + + fn mark_quarantine_purged(&self, asset_id: &str) -> BackendResult<()> { + self.rpc_void("mark_quarantine_purged", &json!({ "asset_id": asset_id })) + } + + fn claim_purges(&self, max_items: u32) -> BackendResult> { + self.rpc_json("claim_purges", &json!({ "max_items": max_items })) + } + + fn ack_purge(&self, purge_id: &str) -> BackendResult<()> { + self.rpc_void("ack_purge", &json!({ "purge_id": purge_id })) + } + + fn claim_quarantine_leftovers( + &self, + older_than_seconds: u64, + max_items: u32, + ) -> BackendResult> { + self.rpc_json( + "claim_quarantine_leftovers", + &json!({ "older_than_seconds": older_than_seconds, "max_items": max_items }), + ) + } + + fn expire_pending_uploads(&self, max_age_seconds: u64) -> BackendResult { + self.rpc_json( + "expire_pending_uploads", + &json!({ "max_age_seconds": max_age_seconds }), + ) + } +} + +fn validate_bucket(bucket: &str) -> BackendResult<()> { + let ok = !bucket.is_empty() + && bucket + .bytes() + .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-'); + if ok { + Ok(()) + } else { + Err(BackendError::Internal("bucket name refused".to_owned())) + } +} + +fn network(error: ureq::Error) -> BackendError { + BackendError::Network(error.to_string()) +} + +fn read_text(response: &mut Response) -> BackendResult { + response + .body_mut() + .with_config() + .limit(MAX_RPC_BODY) + .read_to_string() + .map_err(|e| BackendError::Network(e.to_string())) +} + +fn expect_success(response: &mut Response) -> BackendResult<()> { + let status = response.status().as_u16(); + if (200..300).contains(&status) { + return Ok(()); + } + let text = read_text(response).unwrap_or_default(); + Err(classify(status, &text)) +} + +/// A PostgREST `P0001` body names a closed refusal; any other status is reported as such. +pub fn classify(status: u16, body: &str) -> BackendError { + if let Ok(value) = serde_json::from_str::(body) + && value.get("code").and_then(Value::as_str) == Some("P0001") + && let Some(message) = value.get("message").and_then(Value::as_str) + && is_closed_code(message) + { + return BackendError::Refused(message.to_owned()); + } + BackendError::Status(status) +} + +/// Shape of a closed code (`invalid_state`): the server's free text never becomes a code. +fn is_closed_code(message: &str) -> bool { + !message.is_empty() + && message.len() <= 40 + && message.bytes().all(|b| b.is_ascii_lowercase() || b == b'_') +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn p0001_bodies_become_closed_refusals_and_others_statuses() { + assert_eq!( + classify( + 400, + r#"{"code":"P0001","message":"invalid_state","details":null}"# + ), + BackendError::Refused("invalid_state".to_owned()) + ); + assert_eq!( + classify(403, r#"{"code":"42501","message":"denied"}"#), + BackendError::Status(403) + ); + assert_eq!( + classify(400, r#"{"code":"P0001","message":"Free text!"}"#), + BackendError::Status(400) + ); + assert_eq!(classify(500, "not json"), BackendError::Status(500)); + } + + #[test] + fn object_urls_need_valid_buckets_and_keys() { + let config = Config::from_vars(|name| match name { + "SUPABASE_URL" => Some("http://kong:8000".to_owned()), + "SUPABASE_SERVICE_ROLE_KEY" => Some("k".to_owned()), + _ => None, + }) + .ok(); + let Some(config) = config else { + panic!("config"); + }; + let backend = HttpBackend::new(&config); + let key = "profiles/10000000-0000-0000-0000-00000000000a/10000000-0000-0000-0000-00000000000b/avatar.webp"; + assert_eq!( + backend.object_url("avatars", key).ok().as_deref(), + Some( + "http://kong:8000/storage/v1/object/avatars/profiles/10000000-0000-0000-0000-00000000000a/10000000-0000-0000-0000-00000000000b/avatar.webp" + ) + ); + assert!(backend.object_url("avatars/../x", key).is_err()); + assert!( + backend + .object_url("avatars", "profiles/../etc/passwd") + .is_err() + ); + assert_eq!( + backend.rpc_url("claim_avatar_jobs"), + "http://kong:8000/rest/v1/rpc/claim_avatar_jobs" + ); + } +} diff --git a/crates/worker/src/job.rs b/crates/worker/src/job.rs new file mode 100644 index 00000000..a1d8231f --- /dev/null +++ b/crates/worker/src/job.rs @@ -0,0 +1,264 @@ +//! One avatar job (STORAGE.md §3.4): download the quarantine bytes, hand them to the +//! sandboxed child, upload the clean output, commit the row, clean the quarantine. Every +//! outcome of the child and of the backend maps to one closed end; nothing is deleted on a +//! refused commit (the purge queue owns every object of a terminal row). + +use std::time::Instant; + +use sha2::{Digest, Sha256}; + +use crate::backend::{Backend, BackendError, Job}; +use crate::decoder::Decoder; +use crate::spawn::ChildOutcome; +use crate::watchdog::{Watchdog, deadlines}; +use crate::{exit_codes, keys, log}; + +/// How a job ended, as the loop sees it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum JobEnd { + /// The row is `clean`. + Completed, + /// The row was refused with this closed code. + Rejected(&'static str), + /// The commit or the rejection was refused: the row moved on without us. + Superseded, + /// A transport failure: the lease expires and the row is claimed again. + LeftForLease, + /// The child reported a broken environment: the loop must stop. + Stop, +} + +/// Closed rejection reasons the row accepts (STORAGE.md §1.2). +pub mod reasons { + pub const UNSUPPORTED_TYPE: &str = "unsupported_type"; + pub const DECODE_FAILED: &str = "decode_failed"; + pub const DIMENSIONS: &str = "dimensions"; + pub const PROCESSING_ERROR: &str = "processing_error"; +} + +/// Runs one claimed job to its end. +pub fn process_job( + backend: &dyn Backend, + decoder: &dyn Decoder, + watchdog: &Watchdog, + job: &Job, +) -> JobEnd { + let started = Instant::now(); + let end = run(backend, decoder, watchdog, job); + let line = match &end { + JobEnd::Completed => log::info("job_completed"), + JobEnd::Rejected(code) => log::info("job_rejected").code(code), + JobEnd::Superseded => log::info("job_superseded"), + JobEnd::LeftForLease => log::warn("job_left_for_lease"), + JobEnd::Stop => log::error("job_stop"), + }; + line.asset(&job.asset_id) + .duration_ms(started.elapsed().as_millis()) + .emit(); + end +} + +fn run(backend: &dyn Backend, decoder: &dyn Decoder, watchdog: &Watchdog, job: &Job) -> JobEnd { + if !keys::is_uuid(&job.asset_id) || !keys::is_uuid(&job.lock_token) { + log::error("job_invalid_identity").emit(); + return JobEnd::LeftForLease; + } + let Some(quarantine_key) = validated_keys(job) else { + return reject(backend, job, reasons::PROCESSING_ERROR); + }; + let bytes = match download(backend, watchdog, job, quarantine_key) { + Ok(bytes) => bytes, + Err(end) => return end, + }; + let verdict = { + let _stage = watchdog.enter("decode", deadlines::DECODE); + classify_child(decoder.decode(&bytes), job) + }; + match verdict { + ChildVerdict::Clean(webp) => commit(backend, watchdog, job, quarantine_key, &webp), + ChildVerdict::Reject(reason) => reject(backend, job, reason), + ChildVerdict::Stop => JobEnd::Stop, + } +} + +/// Both keys and the bucket must have the shapes the RPC builds; `None` = corrupted row. +fn validated_keys(job: &Job) -> Option<&str> { + let quarantine_key = job.quarantine_key.as_deref()?; + let bucket_ok = !job.bucket_id.is_empty() + && job + .bucket_id + .bytes() + .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-'); + if !bucket_ok + || keys::validate_object_key(&job.object_key).is_err() + || keys::validate_quarantine_key(quarantine_key).is_err() + { + log::error("job_invalid_keys").asset(&job.asset_id).emit(); + return None; + } + Some(quarantine_key) +} + +fn download( + backend: &dyn Backend, + watchdog: &Watchdog, + job: &Job, + quarantine_key: &str, +) -> Result, JobEnd> { + let _stage = watchdog.enter("download", deadlines::DOWNLOAD); + match backend.download_quarantine(&job.bucket_id, quarantine_key) { + Ok(bytes) => Ok(bytes), + Err(BackendError::Status(404)) => { + log::warn("download_missing").asset(&job.asset_id).emit(); + Err(reject(backend, job, reasons::DECODE_FAILED)) + } + Err(BackendError::Overflow) => { + log::warn("download_overflow").asset(&job.asset_id).emit(); + Err(reject(backend, job, reasons::DECODE_FAILED)) + } + Err(error) => Err(transport_failure("download", job, &error)), + } +} + +/// What the child's end means for the row. +enum ChildVerdict { + Clean(Vec), + Reject(&'static str), + Stop, +} + +/// Maps every way the child can end (R0 `ChildOutcome`) to a verdict on the row. +fn classify_child(run: crate::spawn::ChildRun, job: &Job) -> ChildVerdict { + let stderr = run.stderr; + match run.outcome { + ChildOutcome::Produced(webp) => ChildVerdict::Clean(webp), + ChildOutcome::Rejected { code, name } => { + log::info("child_rejected") + .asset(&job.asset_id) + .code(name) + .child_stderr(&stderr) + .emit(); + ChildVerdict::Reject(content_reason(code)) + } + ChildOutcome::Broken { name, .. } => { + log::error("child_broken") + .asset(&job.asset_id) + .code(name) + .child_stderr(&stderr) + .emit(); + ChildVerdict::Stop + } + ChildOutcome::Signalled { signal } => { + log::warn("child_signalled") + .asset(&job.asset_id) + .signal(signal) + .child_stderr(&stderr) + .emit(); + ChildVerdict::Reject(reasons::PROCESSING_ERROR) + } + ChildOutcome::OutputOverflow => { + log::warn("child_output_overflow") + .asset(&job.asset_id) + .emit(); + ChildVerdict::Reject(reasons::PROCESSING_ERROR) + } + ChildOutcome::Timeout => { + log::warn("child_timeout").asset(&job.asset_id).emit(); + ChildVerdict::Reject(reasons::PROCESSING_ERROR) + } + ChildOutcome::Failed(detail) => { + log::warn("child_failed") + .asset(&job.asset_id) + .detail(&detail) + .emit(); + ChildVerdict::Reject(reasons::PROCESSING_ERROR) + } + } +} + +/// The row's closed reason for a content exit code of the child. +pub fn content_reason(code: i32) -> &'static str { + match code { + exit_codes::UNSUPPORTED_TYPE => reasons::UNSUPPORTED_TYPE, + exit_codes::DIMENSIONS => reasons::DIMENSIONS, + _ => reasons::DECODE_FAILED, + } +} + +fn commit( + backend: &dyn Backend, + watchdog: &Watchdog, + job: &Job, + quarantine_key: &str, + webp: &[u8], +) -> JobEnd { + { + let _stage = watchdog.enter("upload", deadlines::UPLOAD); + if let Err(error) = backend.put_clean(&job.bucket_id, &job.object_key, webp) { + return transport_failure("upload", job, &error); + } + } + let _stage = watchdog.enter("commit", deadlines::COMMIT); + match backend.complete( + &job.asset_id, + &job.lock_token, + webp.len(), + &sha256_hex(webp), + ) { + Ok(()) => {} + Err(BackendError::Refused(code)) => { + log::info("complete_refused") + .asset(&job.asset_id) + .code(&code) + .emit(); + return JobEnd::Superseded; + } + Err(error) => return transport_failure("commit", job, &error), + } + // Best effort: the sweeper finishes a quarantine the worker could not delete. + if backend + .delete_object(&job.bucket_id, quarantine_key) + .is_ok() + && backend.mark_quarantine_purged(&job.asset_id).is_err() + { + log::warn("mark_quarantine_purged_failed") + .asset(&job.asset_id) + .emit(); + } + JobEnd::Completed +} + +/// Rejection of the row with a closed reason; a refusal means the row moved on. +fn reject(backend: &dyn Backend, job: &Job, reason: &'static str) -> JobEnd { + match backend.reject(&job.asset_id, &job.lock_token, reason) { + Ok(()) => JobEnd::Rejected(reason), + Err(BackendError::Refused(code)) => { + log::info("reject_refused") + .asset(&job.asset_id) + .code(&code) + .emit(); + JobEnd::Superseded + } + Err(error) => transport_failure("reject", job, &error), + } +} + +fn transport_failure(stage: &str, job: &Job, error: &BackendError) -> JobEnd { + let line = log::warn("backend_failure") + .asset(&job.asset_id) + .stage(stage) + .code(error.kind()); + match error { + BackendError::Status(status) => line.http_status(*status).emit(), + other => line.detail(&other.to_string()).emit(), + } + JobEnd::LeftForLease +} + +/// Lowercase hex sha256 of the clean output. +pub fn sha256_hex(bytes: &[u8]) -> String { + Sha256::digest(bytes) + .iter() + .map(|b| format!("{b:02x}")) + .collect() +} diff --git a/crates/worker/src/keys.rs b/crates/worker/src/keys.rs new file mode 100644 index 00000000..fa2f4a7b --- /dev/null +++ b/crates/worker/src/keys.rs @@ -0,0 +1,145 @@ +//! Shape of the object keys the worker accepts from the database (STORAGE.md §1.4). A key +//! is validated before it reaches a URL, so a corrupted or hostile row can never make the +//! worker read or delete outside the two prefixes. + +use std::fmt; + +/// Final avatar object: `profiles///avatar.webp`. +pub const FINAL_PREFIX: &str = "profiles"; +/// Upload landing key: `quarantine///upload`. +pub const QUARANTINE_PREFIX: &str = "quarantine"; + +/// Why a key or an id was refused. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum KeyError { + Segments, + Prefix, + Uuid, + FileName, +} + +impl fmt::Display for KeyError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let text = match self { + Self::Segments => "key must have four segments", + Self::Prefix => "unknown key prefix", + Self::Uuid => "segment is not a lowercase uuid", + Self::FileName => "unexpected file name", + }; + f.write_str(text) + } +} + +/// Lowercase hyphenated uuid (36 characters). +pub fn is_uuid(text: &str) -> bool { + let bytes = text.as_bytes(); + bytes.len() == 36 + && bytes.iter().enumerate().all(|(i, b)| { + if matches!(i, 8 | 13 | 18 | 23) { + *b == b'-' + } else { + b.is_ascii_digit() || (b'a'..=b'f').contains(b) + } + }) +} + +fn validate(key: &str, prefix: &str, file_name: &str) -> Result<(), KeyError> { + let segments: Vec<&str> = key.split('/').collect(); + let [p, owner, asset, name] = segments.as_slice() else { + return Err(KeyError::Segments); + }; + if *p != prefix { + return Err(KeyError::Prefix); + } + if !is_uuid(owner) || !is_uuid(asset) { + return Err(KeyError::Uuid); + } + if *name != file_name { + return Err(KeyError::FileName); + } + Ok(()) +} + +/// `profiles///avatar.webp`. +pub fn validate_object_key(key: &str) -> Result<(), KeyError> { + validate(key, FINAL_PREFIX, "avatar.webp") +} + +/// `quarantine///upload`. +pub fn validate_quarantine_key(key: &str) -> Result<(), KeyError> { + validate(key, QUARANTINE_PREFIX, "upload") +} + +/// Either shape: the only keys the worker may delete. +pub fn validate_any_key(key: &str) -> Result<(), KeyError> { + if key.starts_with("profiles/") { + validate_object_key(key) + } else { + validate_quarantine_key(key) + } +} + +/// The asset id embedded in a valid quarantine key. +pub fn asset_id_of_quarantine_key(key: &str) -> Option<&str> { + validate_quarantine_key(key).ok()?; + key.split('/').nth(2) +} + +#[cfg(test)] +mod tests { + use super::*; + + const U1: &str = "10000000-0000-0000-0000-00000000000a"; + const U2: &str = "7f2f0c3e-1c7a-4b2b-9c8d-0123456789ab"; + + #[test] + fn valid_keys_are_accepted() { + assert_eq!( + validate_object_key(&format!("{FINAL_PREFIX}/{U1}/{U2}/avatar.webp")), + Ok(()) + ); + assert_eq!( + validate_quarantine_key(&format!("{QUARANTINE_PREFIX}/{U1}/{U2}/upload")), + Ok(()) + ); + assert_eq!( + asset_id_of_quarantine_key(&format!("{QUARANTINE_PREFIX}/{U1}/{U2}/upload")), + Some(U2) + ); + assert_eq!( + validate_any_key(&format!("{FINAL_PREFIX}/{U1}/{U2}/avatar.webp")), + Ok(()) + ); + } + + #[test] + fn deviations_are_refused() { + let cases: &[(&str, KeyError)] = &[ + ("profiles/x/avatar.webp", KeyError::Segments), + ( + &format!("profiles/{U1}/{U2}/avatar.webp/"), + KeyError::Segments, + ), + (&format!("avatars/{U1}/{U2}/avatar.webp"), KeyError::Prefix), + (&format!("profiles/{U1}/../avatar.webp"), KeyError::Uuid), + ( + &format!("profiles/{}/{U2}/avatar.webp", U1.to_uppercase()), + KeyError::Uuid, + ), + ( + &format!("profiles/{U1}/{U2}/avatar.png"), + KeyError::FileName, + ), + (&format!("profiles/{U1}/{U2}/upload"), KeyError::FileName), + ( + &format!("quarantine/{U1}/{U2}/avatar.webp"), + KeyError::FileName, + ), + ]; + for (key, expected) in cases { + assert_eq!(validate_any_key(key).err(), Some(*expected), "{key}"); + } + assert_eq!(asset_id_of_quarantine_key("quarantine/a/b/upload"), None); + assert!(!is_uuid("10000000-0000-0000-0000-00000000000")); + } +} diff --git a/crates/worker/src/lib.rs b/crates/worker/src/lib.rs index 8981b33b..cb2bdbd6 100644 --- a/crates/worker/src/lib.rs +++ b/crates/worker/src/lib.rs @@ -1,15 +1,25 @@ -//! `faunosteo-worker` library: the pieces the binary, the tests and the R1 job loop share. +//! `faunosteo-worker` library: the sandboxed decode child (R0) and the keyed job loop (R1). //! //! Process model (review Part IV.1): a keyed parent spawns the same binary as `decode`, a //! child with an empty environment that confines itself (rlimits + seccomp) before reading //! one image on stdin and writing one 512 × 512 WebP on stdout. The parent never decodes: //! it checks the container header of what comes back (`webp_check`). +pub mod backend; +pub mod config; pub mod decode; +pub mod decoder; pub mod exit_codes; #[cfg(feature = "test-hooks")] pub mod fixtures; +pub mod http; +pub mod job; +pub mod keys; pub mod limits; +pub mod log; +pub mod run; pub mod sandbox; pub mod spawn; +pub mod sweep; +pub mod watchdog; pub mod webp_check; diff --git a/crates/worker/src/log.rs b/crates/worker/src/log.rs new file mode 100644 index 00000000..1d6e73f2 --- /dev/null +++ b/crates/worker/src/log.rs @@ -0,0 +1,156 @@ +//! JSON-lines logging of the parent (STORAGE.md §3.6): one object per line on stdout, an +//! allowlist of fields, values authored by the worker only. Text a child wrote is escaped +//! by the JSON encoder and truncated; no key, no token, no user content ever reaches a line. + +use std::io::Write; +use std::sync::OnceLock; +use std::time::{SystemTime, UNIX_EPOCH}; + +use serde_json::{Map, Value}; + +static WORKER_ID: OnceLock = OnceLock::new(); + +/// Tags every later line with the worker id (first call wins). +pub fn set_worker_id(id: &str) { + let _ = WORKER_ID.set(id.to_owned()); +} + +/// Longest child stderr excerpt kept in a line. +pub const CHILD_STDERR_MAX: usize = 200; + +/// One line under construction. +#[must_use = "a log line is written by emit()"] +pub struct LogLine { + fields: Map, +} + +pub fn info(event: &str) -> LogLine { + LogLine::new("info", event) +} + +pub fn warn(event: &str) -> LogLine { + LogLine::new("warn", event) +} + +pub fn error(event: &str) -> LogLine { + LogLine::new("error", event) +} + +impl LogLine { + fn new(level: &str, event: &str) -> Self { + let mut fields = Map::new(); + fields.insert("ts".into(), Value::from(unix_seconds())); + fields.insert("level".into(), Value::from(level)); + fields.insert("event".into(), Value::from(event)); + if let Some(id) = WORKER_ID.get() { + fields.insert("worker_id".into(), Value::from(id.as_str())); + } + Self { fields } + } + + fn with(mut self, key: &str, value: impl Into) -> Self { + self.fields.insert(key.to_owned(), value.into()); + self + } + + pub fn asset(self, id: &str) -> Self { + self.with("asset_id", id) + } + + pub fn stage(self, stage: &str) -> Self { + self.with("stage", stage) + } + + pub fn code(self, code: &str) -> Self { + self.with("code", code) + } + + pub fn signal(self, signal: i32) -> Self { + self.with("signal", signal) + } + + pub fn http_status(self, status: u16) -> Self { + self.with("http_status", status) + } + + pub fn duration_ms(self, ms: u128) -> Self { + self.with("duration_ms", u64::try_from(ms).unwrap_or(u64::MAX)) + } + + pub fn count(self, count: usize) -> Self { + self.with("count", u64::try_from(count).unwrap_or(u64::MAX)) + } + + /// Worker-authored text only (never a child's or a server's free text). + pub fn detail(self, detail: &str) -> Self { + self.with("detail", detail) + } + + /// Child-controlled text: truncated on a character boundary, escaped by the encoder. + pub fn child_stderr(self, text: &str) -> Self { + self.with("child_stderr", truncate(text, CHILD_STDERR_MAX)) + } + + /// Renders the line (tests read it, `emit` writes it). + pub fn render(&self) -> String { + Value::Object(self.fields.clone()).to_string() + } + + pub fn emit(self) { + let stdout = std::io::stdout(); + let mut lock = stdout.lock(); + let _ = writeln!(lock, "{}", self.render()); + let _ = lock.flush(); + } +} + +fn truncate(text: &str, max: usize) -> &str { + if text.len() <= max { + return text; + } + let mut end = max; + while !text.is_char_boundary(end) { + end -= 1; + } + &text[..end] +} + +fn unix_seconds() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0) +} + +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used)] + use super::*; + + #[test] + fn lines_are_json_objects_with_the_allowlisted_fields() { + let line = info("job_done") + .asset("a1") + .stage("commit") + .duration_ms(12) + .render(); + let value: Value = serde_json::from_str(&line).unwrap(); + assert_eq!(value["event"], "job_done"); + assert_eq!(value["asset_id"], "a1"); + assert_eq!(value["stage"], "commit"); + assert_eq!(value["duration_ms"], 12); + assert!(value["ts"].as_u64().is_some()); + } + + #[test] + fn child_text_is_escaped_and_truncated() { + let hostile = format!("line\n\u{1b}[31mred\"quote{}", "é".repeat(300)); + let line = warn("child").child_stderr(&hostile).render(); + assert!(!line.contains('\n')); + assert!(!line.contains('\u{1b}')); + let value: Value = serde_json::from_str(&line).unwrap(); + let kept = value["child_stderr"].as_str().unwrap(); + assert!(kept.len() <= CHILD_STDERR_MAX); + assert!(kept.starts_with("line\n")); + } +} diff --git a/crates/worker/src/main.rs b/crates/worker/src/main.rs index 7eea8959..f2639a41 100644 --- a/crates/worker/src/main.rs +++ b/crates/worker/src/main.rs @@ -1,14 +1,14 @@ //! `faunosteo-worker` entry: a plain `main`, no async runtime, no thread, so the `decode` //! child installs its sandbox in a single-threaded process (review Part IV.1). //! -//! R0 subcommands: `decode` (the sandboxed child), `run --check` (prints the resolved -//! configuration, no job loop before R1), `version`. +//! Subcommands: `decode` (the sandboxed child, R0), `run` (the keyed job loop, R1), +//! `run --check` (prints the resolved configuration without secrets), `version`. +#[cfg(feature = "test-hooks")] use std::io::Write; use faunosteo_worker::decode; use faunosteo_worker::exit_codes; -use faunosteo_worker::limits::MIN_LIBWEBP_VERSION; fn main() { let args: Vec = std::env::args().collect(); @@ -24,7 +24,7 @@ fn main() { &mut stderr.lock(), ) } - Some("run") => run(&args[2..]), + Some("run") => faunosteo_worker::run::main(&args[2..]), Some("version") => version(), #[cfg(feature = "test-hooks")] Some("fixture") => fixture(&args[2..]), @@ -59,40 +59,6 @@ fn faunosteo_core_version() -> &'static str { env!("CARGO_PKG_VERSION") } -/// `run --check`: prints what the worker would start with, without secrets. The job loop -/// arrives at R1; a plain `run` therefore refuses to pretend it works. -fn run(args: &[String]) -> i32 { - let (major, minor, revision) = decode::libwebp_version(); - let libwebp_ok = (major, minor, revision) >= MIN_LIBWEBP_VERSION; - println!("faunosteo-worker run: R0 skeleton, the job loop arrives at R1"); - println!( - "libwebp {major}.{minor}.{revision} ({})", - if libwebp_ok { - "accepted" - } else { - "below the minimum, refusing to start" - } - ); - for name in ["SUPABASE_URL", "SUPABASE_SERVICE_ROLE_KEY", "WORKER_ID"] { - let state = if std::env::var_os(name).is_some() { - "set" - } else { - "unset" - }; - println!("{name}: {state}"); - } - let _ = std::io::stdout().flush(); - if !libwebp_ok { - return exit_codes::INTERNAL; - } - if args.iter().any(|arg| arg == "--check") { - exit_codes::OK - } else { - eprintln!("run: nothing to do before R1 (use --check to print the configuration)"); - exit_codes::USAGE - } -} - #[cfg(feature = "test-hooks")] fn fixture(args: &[String]) -> i32 { let Some(bytes) = args diff --git a/crates/worker/src/run.rs b/crates/worker/src/run.rs new file mode 100644 index 00000000..37459edb --- /dev/null +++ b/crates/worker/src/run.rs @@ -0,0 +1,228 @@ +//! `faunosteo-worker run`: the keyed parent's loop (STORAGE.md §3.4, §3.5) and +//! `run --check`, the start-up self-description without any secret value. + +use std::io::Write; +use std::time::{Instant, SystemTime, UNIX_EPOCH}; + +use crate::backend::Backend; +use crate::config::{self, Config, ConfigError}; +use crate::decode::libwebp_version; +use crate::decoder::{ChildDecoder, Decoder}; +use crate::http::HttpBackend; +use crate::job::{JobEnd, process_job}; +use crate::limits::MIN_LIBWEBP_VERSION; +use crate::sweep::{SweepConfig, run_sweep}; +use crate::watchdog::{Watchdog, deadlines}; +use crate::{exit_codes, log}; + +const THIRTY_DAYS: i64 = 30 * 24 * 3600; + +/// Entry of the `run` subcommand; returns the process exit code. +pub fn main(args: &[String]) -> i32 { + match args.first().map(String::as_str) { + Some("--check") => check(), + Some(other) => { + eprintln!("run: unknown argument {other} (use --check to print the configuration)"); + exit_codes::USAGE + } + None => start(), + } +} + +fn start() -> i32 { + let config = match Config::from_env() { + Ok(config) => config, + Err(error) => { + eprintln!("run: {error}"); + return exit_codes::USAGE; + } + }; + if !libwebp_ok() { + eprintln!("run: linked libwebp below the minimum, refusing to start"); + return exit_codes::INTERNAL; + } + log::set_worker_id(&config.worker_id); + log_start(&config); + let decoder = match ChildDecoder::from_current_exe() { + Ok(decoder) => decoder, + Err(error) => { + log::error("start_failed") + .detail(&format!("current_exe: {error}")) + .emit(); + return exit_codes::INTERNAL; + } + }; + let backend = HttpBackend::new(&config); + let watchdog = Watchdog::new(); + watchdog.spawn_monitor(); + run_loop(&config, &backend, &decoder, &watchdog) +} + +/// Sweep at start and every `sweep` interval; claim one job per pass; idle on `poll`. +pub fn run_loop( + config: &Config, + backend: &dyn Backend, + decoder: &dyn Decoder, + watchdog: &Watchdog, +) -> i32 { + let sweep_config = SweepConfig { + pending_max_age: config.pending_max_age, + quarantine_grace: config.quarantine_grace, + }; + let mut next_sweep = Instant::now(); + loop { + if Instant::now() >= next_sweep { + run_sweep(backend, watchdog, sweep_config); + next_sweep = Instant::now() + config.sweep; + } + let worked = match claim_pass(backend, decoder, watchdog) { + Ok(count) => count > 0, + Err(code) => return code, + }; + if config.once { + return exit_codes::OK; + } + if !worked { + std::thread::sleep(config.poll); + } + } +} + +/// One claim and the jobs it returned; `Err` carries the exit code of a stop. +fn claim_pass( + backend: &dyn Backend, + decoder: &dyn Decoder, + watchdog: &Watchdog, +) -> Result { + let jobs = { + let _stage = watchdog.enter("claim", deadlines::CLAIM); + match backend.claim_jobs(1) { + Ok(jobs) => jobs, + Err(error) => { + log::warn("claim_failed") + .code(error.kind()) + .detail(&error.to_string()) + .emit(); + return Ok(0); + } + } + }; + for job in &jobs { + if process_job(backend, decoder, watchdog, job) == JobEnd::Stop { + return Err(exit_codes::CHILD_BROKEN); + } + } + Ok(jobs.len()) +} + +fn log_start(config: &Config) { + let (major, minor, rev) = libwebp_version(); + log::info("worker_start") + .detail(&format!( + "libwebp {major}.{minor}.{rev}; poll {}s sweep {}s grace {}s pending {}s once {}", + config.poll.as_secs(), + config.sweep.as_secs(), + config.quarantine_grace.as_secs(), + config.pending_max_age.as_secs(), + config.once + )) + .emit(); + match token_expiry(config) { + TokenExpiry::NotJwt => log::info("token_no_expiry").emit(), + TokenExpiry::Valid(days) => log::info("token_expiry").count(days).emit(), + TokenExpiry::Soon(days) => log::warn("token_expires_soon").count(days).emit(), + TokenExpiry::Expired => log::error("token_expired").emit(), + } +} + +enum TokenExpiry { + NotJwt, + Valid(usize), + Soon(usize), + Expired, +} + +fn token_expiry(config: &Config) -> TokenExpiry { + let Some(exp) = config::jwt_exp(config.service_key.expose()) else { + return TokenExpiry::NotJwt; + }; + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| i64::try_from(d.as_secs()).unwrap_or(i64::MAX)) + .unwrap_or(0); + let left = exp - now; + let days = usize::try_from(left / 86_400).unwrap_or(0); + if left <= 0 { + TokenExpiry::Expired + } else if left < THIRTY_DAYS { + TokenExpiry::Soon(days) + } else { + TokenExpiry::Valid(days) + } +} + +fn libwebp_ok() -> bool { + libwebp_version() >= MIN_LIBWEBP_VERSION +} + +/// Prints the resolved configuration without any value of a secret. Exit 0 when the +/// worker could start with this environment, or when no worker variable is set at all +/// (an unconfigured image is a valid artifact check); a partial or invalid configuration +/// and a too-old libwebp exit non-zero and name the variable. +fn check() -> i32 { + let (major, minor, rev) = libwebp_version(); + let libwebp_ok = libwebp_ok(); + println!( + "libwebp: {major}.{minor}.{rev} ({})", + if libwebp_ok { + "accepted" + } else { + "below the minimum, refusing to start" + } + ); + let mut any_set = false; + for name in config::VARIABLES { + let set = std::env::var_os(name).is_some(); + any_set |= set; + println!("{name}: {}", if set { "set" } else { "unset" }); + } + let configuration = if any_set { + Some(Config::from_env()) + } else { + None + }; + print_configuration(configuration.as_ref()); + let _ = std::io::stdout().flush(); + match (libwebp_ok, configuration) { + (false, _) => exit_codes::INTERNAL, + (true, None | Some(Ok(_))) => exit_codes::OK, + (true, Some(Err(_))) => exit_codes::USAGE, + } +} + +fn print_configuration(configuration: Option<&Result>) { + match configuration { + None => println!("configuration: none (no worker variable set; the loop would not start)"), + Some(Err(error)) => println!("configuration: invalid ({error})"), + Some(Ok(config)) => { + println!( + "configuration: ok (url {}, worker_id {}, poll {}s, sweep {}s, quarantine grace {}s, pending max age {}s, once {})", + config.url, + config.worker_id, + config.poll.as_secs(), + config.sweep.as_secs(), + config.quarantine_grace.as_secs(), + config.pending_max_age.as_secs(), + config.once + ); + match token_expiry(config) { + TokenExpiry::NotJwt => println!("token: not a JWT (no expiry to track)"), + TokenExpiry::Valid(days) => println!("token: JWT expiring in {days} days"), + TokenExpiry::Soon(days) => { + println!("token: JWT expiring in {days} days (WARNING: under 30)") + } + TokenExpiry::Expired => println!("token: JWT EXPIRED"), + } + } + } +} diff --git a/crates/worker/src/sweep.rs b/crates/worker/src/sweep.rs new file mode 100644 index 00000000..4b1d4e65 --- /dev/null +++ b/crates/worker/src/sweep.rs @@ -0,0 +1,111 @@ +//! The sweeper (STORAGE.md §3.5): expires never-confirmed uploads, serves the purge queue +//! and deletes quarantine leftovers. These are the only deletions the worker makes, and a +//! key that fails the shape check is never sent to the Storage API. + +use std::time::Duration; + +use crate::backend::{Backend, BackendError, Leftover, Purge}; +use crate::watchdog::{Watchdog, deadlines}; +use crate::{keys, log}; + +/// Ages the sweeper applies. +#[derive(Debug, Clone, Copy)] +pub struct SweepConfig { + pub pending_max_age: Duration, + pub quarantine_grace: Duration, +} + +/// What one sweep did. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct SweepReport { + pub expired: i64, + pub purged: usize, + pub leftovers: usize, + pub errors: usize, +} + +const BATCH: u32 = 100; + +/// One full sweep; every failure is logged and counted, never fatal. +pub fn run_sweep(backend: &dyn Backend, watchdog: &Watchdog, config: SweepConfig) -> SweepReport { + let _stage = watchdog.enter("sweep", deadlines::SWEEP); + let mut report = SweepReport::default(); + match backend.expire_pending_uploads(config.pending_max_age.as_secs()) { + Ok(count) => report.expired = count, + Err(error) => failure("expire_pending_uploads", &error, &mut report), + } + match backend.claim_purges(BATCH) { + Ok(purges) => purges + .iter() + .for_each(|p| serve_purge(backend, p, &mut report)), + Err(error) => failure("claim_purges", &error, &mut report), + } + match backend.claim_quarantine_leftovers(config.quarantine_grace.as_secs(), BATCH) { + Ok(items) => items + .iter() + .for_each(|l| serve_leftover(backend, l, &mut report)), + Err(error) => failure("claim_quarantine_leftovers", &error, &mut report), + } + log::info("sweep_done") + .count(report.purged + report.leftovers) + .detail(&format!( + "expired={} errors={}", + report.expired, report.errors + )) + .emit(); + report +} + +fn serve_purge(backend: &dyn Backend, purge: &Purge, report: &mut SweepReport) { + if !deletable(&purge.bucket_id, &purge.object_name) { + log::error("purge_invalid_key") + .detail(&purge.purge_id) + .emit(); + report.errors += 1; + return; + } + if let Err(error) = backend.delete_object(&purge.bucket_id, &purge.object_name) { + return failure("delete_object", &error, report); + } + match backend.ack_purge(&purge.purge_id) { + Ok(()) => report.purged += 1, + Err(error) => failure("ack_purge", &error, report), + } +} + +fn serve_leftover(backend: &dyn Backend, leftover: &Leftover, report: &mut SweepReport) { + if !deletable(&leftover.bucket_id, &leftover.object_name) + || keys::validate_quarantine_key(&leftover.object_name).is_err() + { + log::error("leftover_invalid_key").emit(); + report.errors += 1; + return; + } + if let Err(error) = backend.delete_object(&leftover.bucket_id, &leftover.object_name) { + return failure("delete_object", &error, report); + } + report.leftovers += 1; + if let Some(asset_id) = leftover.asset_id.as_deref().filter(|id| keys::is_uuid(id)) + && let Err(error) = backend.mark_quarantine_purged(asset_id) + { + failure("mark_quarantine_purged", &error, report); + } +} + +/// Bucket literal and one of the two key shapes: the only targets the worker deletes. +fn deletable(bucket: &str, key: &str) -> bool { + let bucket_ok = !bucket.is_empty() + && bucket + .bytes() + .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-'); + bucket_ok && keys::validate_any_key(key).is_ok() +} + +fn failure(call: &str, error: &BackendError, report: &mut SweepReport) { + report.errors += 1; + let line = log::warn("sweep_failure").stage(call).code(error.kind()); + match error { + BackendError::Status(status) => line.http_status(*status).emit(), + other => line.detail(&other.to_string()).emit(), + } +} diff --git a/crates/worker/src/watchdog.rs b/crates/worker/src/watchdog.rs new file mode 100644 index 00000000..009a6e26 --- /dev/null +++ b/crates/worker/src/watchdog.rs @@ -0,0 +1,113 @@ +//! Per-stage deadlines of the parent (STORAGE.md §3.4): the loop announces the stage it +//! enters; a monitor thread exits the process when a stage outlives its deadline, so a hung +//! network call or a wedged child never burns leases silently. Docker restarts the container. + +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +use crate::{exit_codes, log}; + +/// Deadlines per stage (review Part IV.1). +pub mod deadlines { + use std::time::Duration; + pub const CLAIM: Duration = Duration::from_secs(35); + pub const DOWNLOAD: Duration = Duration::from_secs(35); + pub const DECODE: Duration = Duration::from_secs(25); + pub const UPLOAD: Duration = Duration::from_secs(35); + pub const COMMIT: Duration = Duration::from_secs(35); + pub const SWEEP: Duration = Duration::from_secs(120); +} + +#[derive(Debug, Clone)] +struct Stage { + name: &'static str, + started: Instant, + deadline: Duration, +} + +/// Shared stage state; cloning shares it. +#[derive(Debug, Clone, Default)] +pub struct Watchdog { + current: Arc>>, +} + +/// Clears the stage when dropped. +pub struct StageGuard { + watchdog: Watchdog, +} + +impl Drop for StageGuard { + fn drop(&mut self) { + self.watchdog.set(None); + } +} + +impl Watchdog { + pub fn new() -> Self { + Self::default() + } + + fn set(&self, stage: Option) { + match self.current.lock() { + Ok(mut guard) => *guard = stage, + Err(poisoned) => *poisoned.into_inner() = stage, + } + } + + /// Announces a stage; the guard ends it. + pub fn enter(&self, name: &'static str, deadline: Duration) -> StageGuard { + self.set(Some(Stage { + name, + started: Instant::now(), + deadline, + })); + StageGuard { + watchdog: self.clone(), + } + } + + /// Name of the stage past its deadline, if any. + pub fn overrun(&self) -> Option<&'static str> { + let guard = match self.current.lock() { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), + }; + guard + .as_ref() + .filter(|stage| stage.started.elapsed() > stage.deadline) + .map(|stage| stage.name) + } + + /// Starts the monitor thread: checks every second, exits the process on overrun. + pub fn spawn_monitor(&self) { + let watchdog = self.clone(); + std::thread::spawn(move || { + loop { + std::thread::sleep(Duration::from_secs(1)); + if let Some(stage) = watchdog.overrun() { + log::error("watchdog_overrun").stage(stage).emit(); + std::process::exit(exit_codes::WATCHDOG); + } + } + }); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_stage_overruns_only_past_its_deadline_and_clears_on_drop() { + let watchdog = Watchdog::new(); + assert_eq!(watchdog.overrun(), None); + { + let _guard = watchdog.enter("claim", Duration::from_millis(0)); + std::thread::sleep(Duration::from_millis(5)); + assert_eq!(watchdog.overrun(), Some("claim")); + } + assert_eq!(watchdog.overrun(), None); + let _guard = watchdog.enter("download", Duration::from_secs(60)); + assert_eq!(watchdog.overrun(), None); + } +} diff --git a/crates/worker/tests/job_loop.rs b/crates/worker/tests/job_loop.rs new file mode 100644 index 00000000..7fb8fcc6 --- /dev/null +++ b/crates/worker/tests/job_loop.rs @@ -0,0 +1,461 @@ +//! R1 proofs of the job and sweep logic on an in-memory backend and an injected decoder +//! (no network, no child): the call order, the mapping of every child outcome, the +//! lease-left rule on transport failures, the no-delete rule on a refused commit, and the +//! sweeper's deletion targets. +#![allow(clippy::unwrap_used, clippy::expect_used)] + +use std::cell::RefCell; +use std::collections::HashSet; +use std::time::Duration; + +use faunosteo_worker::backend::{Backend, BackendError, BackendResult, Job, Leftover, Purge}; +use faunosteo_worker::config::Config; +use faunosteo_worker::decoder::Decoder; +use faunosteo_worker::job::{JobEnd, process_job, reasons, sha256_hex}; +use faunosteo_worker::run::run_loop; +use faunosteo_worker::spawn::{ChildOutcome, ChildRun}; +use faunosteo_worker::sweep::{SweepConfig, SweepReport, run_sweep}; +use faunosteo_worker::watchdog::Watchdog; +use faunosteo_worker::{exit_codes, webp_check}; + +const OWNER: &str = "10000000-0000-0000-0000-00000000000a"; +const ASSET: &str = "20000000-0000-0000-0000-00000000000b"; +const TOKEN: &str = "30000000-0000-0000-0000-00000000000c"; + +fn job() -> Job { + Job { + asset_id: ASSET.to_owned(), + owner_id: OWNER.to_owned(), + bucket_id: "avatars".to_owned(), + object_key: format!("profiles/{OWNER}/{ASSET}/avatar.webp"), + quarantine_key: Some(format!("quarantine/{OWNER}/{ASSET}/upload")), + lock_token: TOKEN.to_owned(), + attempts: 1, + } +} + +/// A 512 × 512 lossless WebP container that passes the parent's check. +fn clean_webp() -> Vec { + let bits: u32 = (512 - 1) | ((512 - 1) << 14); + let mut vp8l = vec![0x2f]; + vp8l.extend_from_slice(&bits.to_le_bytes()); + let mut body = b"WEBP".to_vec(); + body.extend_from_slice(b"VP8L"); + body.extend_from_slice(&(vp8l.len() as u32).to_le_bytes()); + body.extend_from_slice(&vp8l); + body.push(0); + let mut out = b"RIFF".to_vec(); + out.extend_from_slice(&(body.len() as u32).to_le_bytes()); + out.extend_from_slice(&body); + assert!(webp_check::expect_square(&out, 512).is_ok()); + out +} + +#[derive(Default)] +struct Fake { + calls: RefCell>, + download: RefCell>>>, + put: RefCell>>, + complete: RefCell>>, + reject: RefCell>>, + delete: RefCell>>, + purges: RefCell>, + leftovers: RefCell>, + jobs: RefCell>, +} + +impl Fake { + fn record(&self, call: String) { + self.calls.borrow_mut().push(call); + } + + fn calls(&self) -> Vec { + self.calls.borrow().clone() + } + + fn names(&self) -> Vec { + self.calls() + .iter() + .map(|c| c.split(' ').next().unwrap().to_owned()) + .collect() + } +} + +impl Backend for Fake { + fn claim_jobs(&self, max_jobs: u32) -> BackendResult> { + self.record(format!("claim {max_jobs}")); + Ok(self.jobs.borrow_mut().drain(..).collect()) + } + + fn download_quarantine(&self, bucket: &str, key: &str) -> BackendResult> { + self.record(format!("download {bucket}/{key}")); + self.download + .borrow_mut() + .take() + .unwrap_or_else(|| Ok(b"input".to_vec())) + } + + fn put_clean(&self, bucket: &str, key: &str, bytes: &[u8]) -> BackendResult<()> { + self.record(format!("put {bucket}/{key} {}", bytes.len())); + self.put.borrow_mut().take().unwrap_or(Ok(())) + } + + fn complete( + &self, + asset_id: &str, + lock_token: &str, + byte_size: usize, + sha256: &str, + ) -> BackendResult<()> { + self.record(format!( + "complete {asset_id} {lock_token} {byte_size} {sha256}" + )); + self.complete.borrow_mut().take().unwrap_or(Ok(())) + } + + fn reject(&self, asset_id: &str, lock_token: &str, reason: &str) -> BackendResult<()> { + self.record(format!("reject {asset_id} {lock_token} {reason}")); + self.reject.borrow_mut().take().unwrap_or(Ok(())) + } + + fn delete_object(&self, bucket: &str, key: &str) -> BackendResult<()> { + self.record(format!("delete {bucket}/{key}")); + self.delete.borrow_mut().take().unwrap_or(Ok(())) + } + + fn mark_quarantine_purged(&self, asset_id: &str) -> BackendResult<()> { + self.record(format!("mark_purged {asset_id}")); + Ok(()) + } + + fn claim_purges(&self, max_items: u32) -> BackendResult> { + self.record(format!("claim_purges {max_items}")); + Ok(self.purges.borrow_mut().drain(..).collect()) + } + + fn ack_purge(&self, purge_id: &str) -> BackendResult<()> { + self.record(format!("ack {purge_id}")); + Ok(()) + } + + fn claim_quarantine_leftovers( + &self, + older: u64, + max_items: u32, + ) -> BackendResult> { + self.record(format!("claim_leftovers {older} {max_items}")); + Ok(self.leftovers.borrow_mut().drain(..).collect()) + } + + fn expire_pending_uploads(&self, max_age: u64) -> BackendResult { + self.record(format!("expire {max_age}")); + Ok(2) + } +} + +struct FakeDecoder(RefCell>); + +impl FakeDecoder { + fn with(outcome: ChildOutcome) -> Self { + Self(RefCell::new(Some(outcome))) + } +} + +impl Decoder for FakeDecoder { + fn decode(&self, _bytes: &[u8]) -> ChildRun { + ChildRun { + outcome: self.0.borrow_mut().take().expect("one decode per test"), + stderr: "child says\n\u{1b}[0m".to_owned(), + } + } +} + +fn run_job(fake: &Fake, outcome: ChildOutcome) -> JobEnd { + process_job(fake, &FakeDecoder::with(outcome), &Watchdog::new(), &job()) +} + +#[test] +fn a_produced_avatar_follows_download_decode_put_complete_delete_mark() { + let fake = Fake::default(); + let webp = clean_webp(); + let end = run_job(&fake, ChildOutcome::Produced(webp.clone())); + assert_eq!(end, JobEnd::Completed); + assert_eq!( + fake.calls(), + vec![ + format!("download avatars/quarantine/{OWNER}/{ASSET}/upload"), + format!( + "put avatars/profiles/{OWNER}/{ASSET}/avatar.webp {}", + webp.len() + ), + format!( + "complete {ASSET} {TOKEN} {} {}", + webp.len(), + sha256_hex(&webp) + ), + format!("delete avatars/quarantine/{OWNER}/{ASSET}/upload"), + format!("mark_purged {ASSET}"), + ] + ); +} + +#[test] +fn a_child_rejection_rejects_the_row_with_the_mapped_code() { + for (code, expected) in [ + (exit_codes::UNSUPPORTED_TYPE, reasons::UNSUPPORTED_TYPE), + (exit_codes::DIMENSIONS, reasons::DIMENSIONS), + (exit_codes::DECODE_FAILED, reasons::DECODE_FAILED), + (exit_codes::RESOURCE_LIMIT, reasons::DECODE_FAILED), + (exit_codes::INPUT_TOO_LARGE, reasons::DECODE_FAILED), + ] { + let fake = Fake::default(); + let end = run_job( + &fake, + ChildOutcome::Rejected { + code, + name: exit_codes::name(code), + }, + ); + assert_eq!(end, JobEnd::Rejected(expected)); + assert_eq!(fake.names(), vec!["download", "reject"]); + assert!(fake.calls()[1].ends_with(expected), "{:?}", fake.calls()); + } +} + +#[test] +fn every_abnormal_child_end_is_a_processing_error() { + for outcome in [ + ChildOutcome::Signalled { signal: 31 }, + ChildOutcome::Timeout, + ChildOutcome::OutputOverflow, + ChildOutcome::Failed("output refused".to_owned()), + ] { + let fake = Fake::default(); + assert_eq!( + run_job(&fake, outcome), + JobEnd::Rejected(reasons::PROCESSING_ERROR) + ); + assert_eq!(fake.names(), vec!["download", "reject"]); + } +} + +#[test] +fn a_broken_child_stops_the_loop_without_touching_the_row() { + let fake = Fake::default(); + let end = run_job( + &fake, + ChildOutcome::Broken { + code: exit_codes::SANDBOX_FAILED, + name: "sandbox_failed", + }, + ); + assert_eq!(end, JobEnd::Stop); + assert_eq!(fake.names(), vec!["download"]); +} + +#[test] +fn a_refused_commit_deletes_nothing() { + let fake = Fake::default(); + *fake.complete.borrow_mut() = Some(Err(BackendError::Refused("invalid_state".to_owned()))); + let end = run_job(&fake, ChildOutcome::Produced(clean_webp())); + assert_eq!(end, JobEnd::Superseded); + assert_eq!(fake.names(), vec!["download", "put", "complete"]); +} + +#[test] +fn a_transport_failure_leaves_the_lease_and_rejects_nothing() { + let fake = Fake::default(); + *fake.download.borrow_mut() = Some(Err(BackendError::Network("refused".to_owned()))); + let end = process_job( + &fake, + &FakeDecoder(RefCell::new(None)), + &Watchdog::new(), + &job(), + ); + assert_eq!(end, JobEnd::LeftForLease); + assert_eq!(fake.names(), vec!["download"]); + + let fake = Fake::default(); + *fake.put.borrow_mut() = Some(Err(BackendError::Status(503))); + let end = run_job(&fake, ChildOutcome::Produced(clean_webp())); + assert_eq!(end, JobEnd::LeftForLease); + assert_eq!(fake.names(), vec!["download", "put"]); +} + +#[test] +fn missing_or_oversized_quarantine_bytes_are_decode_failures() { + for error in [BackendError::Status(404), BackendError::Overflow] { + let fake = Fake::default(); + *fake.download.borrow_mut() = Some(Err(error)); + let end = process_job( + &fake, + &FakeDecoder(RefCell::new(None)), + &Watchdog::new(), + &job(), + ); + assert_eq!(end, JobEnd::Rejected(reasons::DECODE_FAILED)); + assert_eq!(fake.names(), vec!["download", "reject"]); + } +} + +#[test] +fn a_row_with_hostile_keys_is_rejected_before_any_storage_call() { + let fake = Fake::default(); + let mut bad = job(); + bad.quarantine_key = Some("quarantine/../../etc/passwd".to_owned()); + let end = process_job( + &fake, + &FakeDecoder(RefCell::new(None)), + &Watchdog::new(), + &bad, + ); + assert_eq!(end, JobEnd::Rejected(reasons::PROCESSING_ERROR)); + assert_eq!(fake.names(), vec!["reject"]); + + let fake = Fake::default(); + let mut bad = job(); + bad.asset_id = "x".to_owned(); + let end = process_job( + &fake, + &FakeDecoder(RefCell::new(None)), + &Watchdog::new(), + &bad, + ); + assert_eq!(end, JobEnd::LeftForLease); + assert!(fake.calls().is_empty()); +} + +#[test] +fn the_sweeper_deletes_only_queued_and_leftover_keys_and_acks() { + let fake = Fake::default(); + let final_key = format!("profiles/{OWNER}/{ASSET}/avatar.webp"); + let quarantine = format!("quarantine/{OWNER}/{ASSET}/upload"); + *fake.purges.borrow_mut() = vec![ + Purge { + purge_id: "p1".into(), + bucket_id: "avatars".into(), + object_name: final_key.clone(), + }, + Purge { + purge_id: "p2".into(), + bucket_id: "avatars".into(), + object_name: "profiles/x/y/avatar.webp".into(), + }, + Purge { + purge_id: "p3".into(), + bucket_id: "avatars".into(), + object_name: quarantine.clone(), + }, + ]; + *fake.leftovers.borrow_mut() = vec![ + Leftover { + bucket_id: "avatars".into(), + object_name: quarantine.clone(), + asset_id: Some(ASSET.into()), + }, + Leftover { + bucket_id: "avatars".into(), + object_name: final_key.clone(), + asset_id: None, + }, + ]; + let report = run_sweep( + &fake, + &Watchdog::new(), + SweepConfig { + pending_max_age: Duration::from_secs(86_400), + quarantine_grace: Duration::from_secs(10_800), + }, + ); + assert_eq!( + report, + SweepReport { + expired: 2, + purged: 2, + leftovers: 1, + errors: 2 + } + ); + assert_eq!( + fake.calls(), + vec![ + "expire 86400".to_owned(), + "claim_purges 100".to_owned(), + format!("delete avatars/{final_key}"), + "ack p1".to_owned(), + format!("delete avatars/{quarantine}"), + "ack p3".to_owned(), + "claim_leftovers 10800 100".to_owned(), + format!("delete avatars/{quarantine}"), + format!("mark_purged {ASSET}"), + ] + ); + let deleted: HashSet = fake + .calls() + .into_iter() + .filter(|c| c.starts_with("delete")) + .collect(); + assert_eq!(deleted.len(), 2); +} + +#[test] +fn a_missing_object_counts_as_purged() { + let fake = Fake::default(); + *fake.purges.borrow_mut() = vec![Purge { + purge_id: "p1".into(), + bucket_id: "avatars".into(), + object_name: format!("profiles/{OWNER}/{ASSET}/avatar.webp"), + }]; + // The HTTP backend maps a 404 to Ok before the sweeper sees it; a real error is not acked. + *fake.delete.borrow_mut() = Some(Err(BackendError::Status(500))); + let report = run_sweep( + &fake, + &Watchdog::new(), + SweepConfig { + pending_max_age: Duration::from_secs(1), + quarantine_grace: Duration::from_secs(1), + }, + ); + assert_eq!(report.purged, 0); + assert_eq!(report.errors, 1); + assert!(!fake.names().contains(&"ack".to_owned())); +} + +#[test] +fn once_mode_sweeps_then_claims_once_and_exits_zero_or_stops_on_a_broken_child() { + let config = Config::from_vars(|name| match name { + "SUPABASE_URL" => Some("http://kong:8000".to_owned()), + "SUPABASE_SERVICE_ROLE_KEY" => Some("k".to_owned()), + "WORKER_ONCE" => Some("1".to_owned()), + _ => None, + }) + .unwrap(); + let fake = Fake::default(); + *fake.jobs.borrow_mut() = vec![job()]; + let decoder = FakeDecoder::with(ChildOutcome::Produced(clean_webp())); + assert_eq!( + run_loop(&config, &fake, &decoder, &Watchdog::new()), + exit_codes::OK + ); + assert_eq!( + &fake.names()[..5], + [ + "expire", + "claim_purges", + "claim_leftovers", + "claim", + "download" + ] + ); + + let fake = Fake::default(); + *fake.jobs.borrow_mut() = vec![job()]; + let decoder = FakeDecoder::with(ChildOutcome::Broken { + code: exit_codes::ENV_NOT_EMPTY, + name: "env_not_empty", + }); + assert_eq!( + run_loop(&config, &fake, &decoder, &Watchdog::new()), + exit_codes::CHILD_BROKEN + ); +} diff --git a/deny.toml b/deny.toml index f079a33b..fbf83176 100644 --- a/deny.toml +++ b/deny.toml @@ -27,6 +27,9 @@ allow = [ "CC0-1.0", "Unlicense", "BSL-1.0", + # Mozilla root certificate store bundled by webpki-roots (the worker's TLS roots on a + # managed Supabase); data licence, permissive. + "CDLA-Permissive-2.0", ] confidence-threshold = 0.9