diff --git a/crates/worker/src/config.rs b/crates/worker/src/config.rs index 492461a2..8e48392b 100644 --- a/crates/worker/src/config.rs +++ b/crates/worker/src/config.rs @@ -147,10 +147,9 @@ fn cleartext_allowed(host_port: &str) -> bool { if let Ok(address) = host.parse::() { return address.is_loopback(); } - !host.is_empty() - && !host.contains('.') - && host.chars().any(|c| c.is_ascii_alphabetic()) - && !host.starts_with("0x") + // A service name starts with a letter: no inet_aton or WHATWG numeric form does + // (`167772165`, `0x0a000005`, `0X0A000005` all resolve to an address). + !host.contains('.') && host.chars().next().is_some_and(|c| c.is_ascii_alphabetic()) } fn seconds( @@ -237,6 +236,7 @@ mod tests { "http://127.evil.example", "http://167772165:8000", "http://0x0a000005", + "http://0X0A000005:8000", "http://127.0.0.1.example", "ftp://x", "http://", diff --git a/docs/development/STEP8_FILES_ANALYSIS.md b/docs/development/STEP8_FILES_ANALYSIS.md index f20845fe..0594bd1f 100644 --- a/docs/development/STEP8_FILES_ANALYSIS.md +++ b/docs/development/STEP8_FILES_ANALYSIS.md @@ -174,7 +174,7 @@ new predicates. | RPC | Role | Effect | |---|---|---| -| `begin_avatar_upload(declared_mime, declared_size)` | authenticated | locks the caller's `profiles` row (`for update`, serialises a double click); rate: ≤ 10 rows of the caller created in the last hour, all statuses counted → `rate_limited`; global ceiling: bytes that can exist (live rows, plus terminal rows still queued for purge; as built, STORAGE.md §2.1) < `private.storage_ceiling()` → `storage_full`; caps (type in the list, size ≤ 5 MiB); supersedes any `pending_upload` / `uploaded` row of the caller (→ `deleted`, enqueued for purge); inserts the row with `object_key = profiles///avatar.webp`, `quarantine_key = quarantine///upload`; returns `{asset_id, bucket_id, quarantine_key}` | +| `begin_avatar_upload(declared_mime, declared_size)` | authenticated | locks the caller's `profiles` row (`for update`, serialises a double click); rate: ≤ 10 rows of the caller created in the last hour, all statuses counted → `rate_limited`; global ceiling: bytes present in the bucket plus declared sizes of pending rows without bytes (as built, STORAGE.md §2.1) < `private.storage_ceiling()` → `storage_full`; caps (type in the list, size ≤ 5 MiB); supersedes any `pending_upload` / `uploaded` row of the caller (→ `deleted`, enqueued for purge); inserts the row with `object_key = profiles///avatar.webp`, `quarantine_key = quarantine///upload`; returns `{asset_id, bucket_id, quarantine_key}` | | `confirm_asset_upload(asset_id)` | authenticated | own row, `pending_upload`, a `storage.objects` row exists at `(bucket_id, quarantine_key)` with `metadata->>'size'` ≤ cap → `uploaded`. Kept as an explicit client signal: the worker never has to guess from Storage API internals whether a PUT finished | | `remove_avatar()` | authenticated | every `profile_avatar` row of the caller not already `deleted` / `rejected` (clean and in flight) → `deleted`; logs `asset.avatar_removed` | | `claim_avatar_jobs(max_jobs)` | service_role | `uploaded` rows with `locked_at is null or locked_at < now() - 10 min` and `attempts < 3`, `for update skip locked`, oldest first → sets `locked_at`, `attempts + 1`, returns the rows; a row at its 3rd expired lease becomes `rejected (processing_error)` instead. VOLATILE (a STABLE function runs read-only through PostgREST) | diff --git a/docs/development/STORAGE.md b/docs/development/STORAGE.md index cfc24ecf..22ec5412 100644 --- a/docs/development/STORAGE.md +++ b/docs/development/STORAGE.md @@ -94,8 +94,8 @@ granted to the one role that calls it. Business refusals raise SQLSTATE | RPC | Effect | Refusals (message) | |---|---|---| -| `begin_avatar_upload(declared_mime text, declared_size integer) returns jsonb` | locks the caller's `profiles` row (`for update`); checks the type and the size; rate: at most 10 rows created by the caller in the last hour, every status counted; global ceiling: the bytes that can exist (`declared_size` of in-flight rows, `byte_size` of clean rows, and terminal rows while their keys still wait in the purge queue) below `private.storage_ceiling()` (10 GiB); residual: a burst of accounts within the ceiling fills the filesystem quota, the hard bound; supersedes the caller's `pending_upload` / `uploaded` rows (→ `deleted`); inserts the row; returns `{"asset_id", "bucket_id", "quarantine_key"}` | `unsupported_type`, `too_large`, `rate_limited`, `storage_full` | -| `confirm_asset_upload(asset_id uuid) returns void` | active account; own `pending_upload` row with a `storage.objects` row at `(bucket_id, quarantine_key)` whose `metadata->>'size'` is within the cap → `uploaded` | `not_found` (unknown or not own), `invalid_state` (not pending), `upload_missing` (no object yet), `too_large` (object over the cap or over the declared size: the row stays pending and expires after 24 h) | +| `begin_avatar_upload(declared_mime text, declared_size integer) returns jsonb` | locks the caller's `profiles` row (`for update`); checks the type and the size; rate: at most 10 rows created by the caller in the last hour, every status counted; global ceiling: the bytes present in the bucket (sum of `storage.objects` sizes, whatever each row's state) plus the `declared_size` of `pending_upload` rows whose bytes have not landed, below `private.storage_ceiling()` (10 GiB): a claim dies with its row, so begin spam pins nothing; residual: a burst of accounts within the ceiling fills the filesystem quota, the hard bound; supersedes the caller's `pending_upload` / `uploaded` rows (→ `deleted`); inserts the row; returns `{"asset_id", "bucket_id", "quarantine_key"}` | `unsupported_type`, `too_large`, `rate_limited`, `storage_full` | +| `confirm_asset_upload(asset_id uuid) returns void` | active account; own `pending_upload` row with a `storage.objects` row at `(bucket_id, quarantine_key)` whose `metadata->>'size'` is set and within the cap → `uploaded` | `not_found` (unknown or not own), `invalid_state` (not pending), `upload_missing` (no object yet, or an object without a size: fail closed), `too_large` (object over the cap or over the declared size: the row stays pending and expires after 24 h) | | `remove_avatar() returns integer` | every `profile_avatar` row of the caller in `pending_upload` / `uploaded` / `clean` → `deleted`; returns the count; logs `asset.avatar_removed` | none (0 rows is fine) | `auth.uid() is null` → `42501` on all three. @@ -170,7 +170,7 @@ metadata chunk), hashes it and uploads it. | Variable | Required | Default | Meaning | |---|---|---|---| -| `SUPABASE_URL` | yes | | origin of the stack (`http://supabase_kong_faunosteo:8000` from the Supabase compose network, `http://127.0.0.1:54321` natively). Cleartext `http://` is accepted only for `localhost`, a dotted-quad loopback address, or a single-label host that is not a number (a Docker service name on the compose network, where a `search` domain cannot apply); any other host must be `https://`, the service key travels in the headers | +| `SUPABASE_URL` | yes | | origin of the stack (`http://supabase_kong_faunosteo:8000` from the Supabase compose network, `http://127.0.0.1:54321` natively). Cleartext `http://` is accepted only for `localhost`, a dotted-quad loopback address, or a single-label host starting with a letter (a Docker service name; inside the compose container Docker sets `ndots:0`, so no search domain applies; on a native glibc run a mistyped name could be searched, so prefer loopback there); any other host must be `https://`, the service key travels in the headers | | `SUPABASE_SERVICE_ROLE_KEY` | yes | | the service key; never printed; the JWT `exp` claim, when the key is a JWT, is logged at start and warned about 30 days ahead | | `WORKER_ID` | no | `worker-` | tag of every log line | | `WORKER_POLL_SECONDS` | no | `2` | idle interval between claims | diff --git a/supabase/migrations/20261013100000_files_foundation.sql b/supabase/migrations/20261013100000_files_foundation.sql index 433aabbf..0190d107 100644 --- a/supabase/migrations/20261013100000_files_foundation.sql +++ b/supabase/migrations/20261013100000_files_foundation.sql @@ -201,8 +201,8 @@ create trigger assets_enqueue_purge_on_delete after delete on public.assets for each row execute function private.enqueue_asset_objects(); --- Global ceiling on the bytes that can exist in the bucket (10 GiB to start): --- live rows, plus terminal rows whose objects still wait in the purge queue. +-- Global ceiling on the bytes present in the avatars bucket (10 GiB to start), +-- measured on storage.objects, plus the declared size of uploads still expected. create or replace function private.storage_ceiling() returns bigint language sql @@ -297,6 +297,7 @@ declare v_id uuid := gen_random_uuid(); v_recent integer; v_used bigint; + v_expected bigint; v_object_key text; v_quarantine_key text; begin @@ -319,17 +320,20 @@ begin if v_recent >= 10 then raise exception using message = 'rate_limited', errcode = 'P0001'; end if; - -- Only rows whose bytes can exist count: in-flight rows by the declared - -- transport size, clean rows by their output; rejected and deleted rows are - -- purged and never ratchet the counter. - select coalesce(sum(case when a.status = 'clean' then coalesce(a.byte_size, a.declared_size) else a.declared_size end), 0) - into v_used from public.assets a - where a.status in ('pending_upload', 'uploaded', 'clean') - or exists ( - select 1 from private.object_purge_queue q - where q.bucket_id = a.bucket_id and q.object_name in (a.object_key, a.quarantine_key) - ); - if v_used + declared_size > private.storage_ceiling() then + -- Bytes that exist (every object of the bucket, whatever its row's state) + -- plus the declared size of pending uploads whose bytes have not landed yet. + -- A claim never outlives the row it belongs to, and a row with no bytes + -- weighs nothing once it moved on, so no account can pin the ceiling. + select coalesce(sum((o.metadata ->> 'size')::bigint), 0) into v_used + from storage.objects o where o.bucket_id = 'avatars'; + select coalesce(sum(a.declared_size), 0) into v_expected + from public.assets a + where a.status = 'pending_upload' + and not exists ( + select 1 from storage.objects o + where o.bucket_id = a.bucket_id and o.name = a.quarantine_key + ); + if v_used + v_expected + declared_size > private.storage_ceiling() then raise exception using message = 'storage_full', errcode = 'P0001'; end if; @@ -379,10 +383,11 @@ begin if v_row.status <> 'pending_upload' then raise exception using message = 'invalid_state', errcode = 'P0001'; end if; - select true, coalesce((o.metadata ->> 'size')::bigint, 0) into v_found, v_size + select true, (o.metadata ->> 'size')::bigint into v_found, v_size from storage.objects o where o.bucket_id = v_row.bucket_id and o.name = v_row.quarantine_key; - if v_found is distinct from true then + -- No object, or an object the Storage API has not sized: nothing to confirm (fail closed). + if v_found is distinct from true or v_size is null then raise exception using message = 'upload_missing', errcode = 'P0001'; end if; -- The object may not exceed the cap nor what the caller declared (the diff --git a/supabase/tests/010_files_foundation.sql b/supabase/tests/010_files_foundation.sql index 5bb3cf15..e8f12bb6 100644 --- a/supabase/tests/010_files_foundation.sql +++ b/supabase/tests/010_files_foundation.sql @@ -2,7 +2,7 @@ -- browser and worker RPCs, the purge queue. Every allowed path has its denial -- twin; the eight scenarios of the Step 8 brief are named in the descriptions. begin; -select plan(137); +select plan(138); insert into auth.users (id, email) values ('40000000-0000-0000-0000-00000000000a', 'alice-files@test.local'), @@ -475,8 +475,8 @@ select is( (select count(*) from private.object_purge_queue where object_name like 'profiles/40000000-0000-0000-0000-00000000000b/%' or object_name like 'quarantine/40000000-0000-0000-0000-00000000000b/%'), 2::bigint, 'the cascade enqueues both keys of the deleted row' ); --- Ceiling: 2048 rejected rows of 5 MiB do NOT count (purged bytes never ratchet --- the counter); 2048 in-flight rows of 5 MiB fill the 10 GiB. +-- Ceiling = bytes present in the bucket + declared sizes of pending rows without bytes. +-- 2048 rejected rows of 5 MiB (no object) weigh nothing. insert into public.assets (owner_id, kind, bucket_id, object_key, quarantine_key, status, declared_mime, declared_size, rejection_reason, created_at) select '40000000-0000-0000-0000-00000000000a', 'profile_avatar', 'avatars', 'profiles/40000000-0000-0000-0000-00000000000a/' || g || '/avatar.webp', @@ -485,34 +485,42 @@ select '40000000-0000-0000-0000-00000000000a', 'profile_avatar', 'avatars', from (select gen_random_uuid() as g from generate_series(1, 2048)) s; set local role authenticated; select set_config('request.jwt.claims', '{"sub":"40000000-0000-0000-0000-00000000000c","role":"authenticated"}', true); -select lives_ok($$ select public.begin_avatar_upload('image/jpeg', 1000) $$, 'rejected rows do not count toward the ceiling'); +select lives_ok($$ select public.begin_avatar_upload('image/jpeg', 1000) $$, 'rejected rows without bytes do not count toward the ceiling'); reset role; +-- 2048 pending rows of 5 MiB whose bytes have not landed are expected bytes. insert into public.assets (owner_id, kind, bucket_id, object_key, quarantine_key, status, declared_mime, declared_size, created_at) select '40000000-0000-0000-0000-00000000000c', 'profile_avatar', 'avatars', 'profiles/40000000-0000-0000-0000-00000000000c/' || g || '/avatar.webp', 'quarantine/40000000-0000-0000-0000-00000000000c/' || g || '/upload', - 'uploaded', 'image/jpeg', 5242880, now() - interval '2 hours' + 'pending_upload', 'image/jpeg', 5242880, now() - interval '2 hours' from (select gen_random_uuid() as g from generate_series(1, 2048)) s; set local role authenticated; select set_config('request.jwt.claims', '{"sub":"40000000-0000-0000-0000-00000000000c","role":"authenticated"}', true); -select throws_ok($$ select public.begin_avatar_upload('image/jpeg', 1000) $$, 'P0001', 'storage_full', 'in-flight bytes fill the global ceiling'); +select throws_ok($$ select public.begin_avatar_upload('image/jpeg', 1000) $$, 'P0001', 'storage_full', 'declared sizes of pending uploads fill the ceiling'); reset role; --- Deleted rows still count while their objects wait in the purge queue. -update public.assets set status = 'deleted' where owner_id = '40000000-0000-0000-0000-00000000000c' and status = 'uploaded'; +-- Superseded or expired without bytes: the claim dies with the row (no pinning by begin spam). +update public.assets set status = 'deleted' where owner_id = '40000000-0000-0000-0000-00000000000c' and status = 'pending_upload' and created_at < now() - interval '1 hour'; set local role authenticated; select set_config('request.jwt.claims', '{"sub":"40000000-0000-0000-0000-00000000000c","role":"authenticated"}', true); -select throws_ok($$ select public.begin_avatar_upload('image/jpeg', 1000) $$, 'P0001', 'storage_full', 'objects still queued for purge count toward the ceiling'); +select lives_ok($$ select public.begin_avatar_upload('image/jpeg', 1000) $$, 'scenario: rows that moved on without bytes weigh nothing, even while queued for purge'); reset role; -delete from private.object_purge_queue where object_name like '%/40000000-0000-0000-0000-00000000000c/%'; +-- Bytes present count whatever the row says: one 10 GiB object fills the ceiling. +insert into storage.objects (bucket_id, name, metadata) values ('avatars', 'quarantine/40000000-0000-0000-0000-00000000000c/00000000-0000-0000-0000-00000000ffff/upload', '{"size":10737418240}'); set local role authenticated; select set_config('request.jwt.claims', '{"sub":"40000000-0000-0000-0000-00000000000c","role":"authenticated"}', true); -select lives_ok($$ select public.begin_avatar_upload('image/jpeg', 1000) $$, 'served purge entries free the ceiling'); +select throws_ok($$ select public.begin_avatar_upload('image/jpeg', 1000) $$, 'P0001', 'storage_full', 'bytes present in the bucket fill the ceiling whatever their row'); reset role; +update storage.objects set metadata = '{"size":1}' where name = 'quarantine/40000000-0000-0000-0000-00000000000c/00000000-0000-0000-0000-00000000ffff/upload'; -- Carol's pending row (begun above): the object may not exceed the declared size. set local role authenticated; select set_config('request.jwt.claims', '{"sub":"40000000-0000-0000-0000-00000000000c","role":"authenticated"}', true); insert into pg_temp.ids select 'c1', id from public.assets where owner_id = '40000000-0000-0000-0000-00000000000c' and status = 'pending_upload'; -insert into storage.objects (bucket_id, name, metadata) values ('avatars', (pg_temp.row('c1')).quarantine_key, '{"size":4000}'); +insert into storage.objects (bucket_id, name, metadata) values ('avatars', (pg_temp.row('c1')).quarantine_key, '{}'); +select throws_ok($$ select public.confirm_asset_upload(pg_temp.id('c1')) $$, 'P0001', 'upload_missing', 'an object without a size is not confirmable (fail closed)'); +reset role; +update storage.objects set metadata = '{"size":4000}' where name = (pg_temp.row('c1')).quarantine_key; +set local role authenticated; +select set_config('request.jwt.claims', '{"sub":"40000000-0000-0000-0000-00000000000c","role":"authenticated"}', true); select throws_ok($$ select public.confirm_asset_upload(pg_temp.id('c1')) $$, 'P0001', 'too_large', 'an object larger than the declared size is refused at confirm'); reset role; update storage.objects set metadata = '{"size":1000}' where name = (pg_temp.row('c1')).quarantine_key;