diff --git a/docs/development/STORAGE.md b/docs/development/STORAGE.md index 22ec5412..28b2214a 100644 --- a/docs/development/STORAGE.md +++ b/docs/development/STORAGE.md @@ -94,7 +94,7 @@ granted to the one role that calls it. Business refusals raise SQLSTATE | RPC | Effect | Refusals (message) | |---|---|---| -| `begin_avatar_upload(declared_mime text, declared_size integer) returns jsonb` | locks the caller's `profiles` row (`for update`); checks the type and the size; rate: at most 10 rows created by the caller in the last hour, every status counted; global ceiling: the bytes present in the bucket (sum of `storage.objects` sizes, whatever each row's state) plus the `declared_size` of `pending_upload` rows whose bytes have not landed, below `private.storage_ceiling()` (10 GiB): a claim dies with its row, so begin spam pins nothing; residual: a burst of accounts within the ceiling fills the filesystem quota, the hard bound; supersedes the caller's `pending_upload` / `uploaded` rows (→ `deleted`); inserts the row; returns `{"asset_id", "bucket_id", "quarantine_key"}` | `unsupported_type`, `too_large`, `rate_limited`, `storage_full` | +| `begin_avatar_upload(declared_mime text, declared_size integer) returns jsonb` | locks the caller's `profiles` row (`for update`); checks the type and the size; rate: at most 10 rows created by the caller in the last hour, every status counted; per-account budget: the caller's bytes present under `profiles//` and `quarantine//` plus one cap below `private.account_byte_budget()` (64 MiB) → `rate_limited`; global ceiling: the bytes present in the bucket (sum of sized `storage.objects`, whatever each row's state) plus one 5 MiB cap per `pending_upload` row whose bytes have not landed, below `private.storage_ceiling()` (10 GiB): a claim dies with its row, so begin spam pins nothing; residuals: about 160 colluding confirmed accounts at their budget fill the ceiling, and a burst of accounts fills the filesystem quota, the hard bound; supersedes the caller's `pending_upload` / `uploaded` rows (→ `deleted`); inserts the row; returns `{"asset_id", "bucket_id", "quarantine_key"}` | `unsupported_type`, `too_large`, `rate_limited`, `storage_full` | | `confirm_asset_upload(asset_id uuid) returns void` | active account; own `pending_upload` row with a `storage.objects` row at `(bucket_id, quarantine_key)` whose `metadata->>'size'` is set and within the cap → `uploaded` | `not_found` (unknown or not own), `invalid_state` (not pending), `upload_missing` (no object yet, or an object without a size: fail closed), `too_large` (object over the cap or over the declared size: the row stays pending and expires after 24 h) | | `remove_avatar() returns integer` | every `profile_avatar` row of the caller in `pending_upload` / `uploaded` / `clean` → `deleted`; returns the count; logs `asset.avatar_removed` | none (0 rows is fine) | @@ -170,7 +170,7 @@ metadata chunk), hashes it and uploads it. | Variable | Required | Default | Meaning | |---|---|---|---| -| `SUPABASE_URL` | yes | | origin of the stack (`http://supabase_kong_faunosteo:8000` from the Supabase compose network, `http://127.0.0.1:54321` natively). Cleartext `http://` is accepted only for `localhost`, a dotted-quad loopback address, or a single-label host starting with a letter (a Docker service name; inside the compose container Docker sets `ndots:0`, so no search domain applies; on a native glibc run a mistyped name could be searched, so prefer loopback there); any other host must be `https://`, the service key travels in the headers | +| `SUPABASE_URL` | yes | | origin of the stack (`http://supabase_kong_faunosteo:8000` from the Supabase compose network, `http://127.0.0.1:54321` natively). Cleartext `http://` is accepted only for `localhost`, a dotted-quad loopback address, or a single-label host starting with a letter (a Docker service name; inside the musl container Docker's `ndots:0` makes musl skip the search list; glibc, natively, may still try search domains on a mistyped name, so prefer loopback there); any other host must be `https://`, the service key travels in the headers | | `SUPABASE_SERVICE_ROLE_KEY` | yes | | the service key; never printed; the JWT `exp` claim, when the key is a JWT, is logged at start and warned about 30 days ahead | | `WORKER_ID` | no | `worker-` | tag of every log line | | `WORKER_POLL_SECONDS` | no | `2` | idle interval between claims | diff --git a/supabase/migrations/20261013100000_files_foundation.sql b/supabase/migrations/20261013100000_files_foundation.sql index 0190d107..76cb1435 100644 --- a/supabase/migrations/20261013100000_files_foundation.sql +++ b/supabase/migrations/20261013100000_files_foundation.sql @@ -101,6 +101,8 @@ comment on column public.assets.lock_token is create unique index assets_one_clean_avatar on public.assets (owner_id) where kind = 'profile_avatar' and status = 'clean'; create index assets_claimable on public.assets (created_at) where status = 'uploaded'; +-- The ceiling's pending term reads only the rows still waiting for bytes. +create index assets_pending_claims on public.assets (quarantine_key) where status = 'pending_upload'; create index assets_owner_idx on public.assets (owner_id, created_at desc); create trigger assets_set_updated_at @@ -212,6 +214,17 @@ as $$ select 10737418240::bigint; $$; revoke all on function private.storage_ceiling() from public; +-- Bytes one account may hold in the bucket at once (its clean photo plus the +-- quarantine objects of its last uploads during their grace): 64 MiB. +create or replace function private.account_byte_budget() +returns bigint +language sql +immutable +set search_path = '' +as $$ select 67108864::bigint; $$; + +revoke all on function private.account_byte_budget() from public; + -- --------------------------------------------------------------------------- -- Storage policies for bucket avatars, evaluated through assets. -- --------------------------------------------------------------------------- @@ -298,6 +311,7 @@ declare v_recent integer; v_used bigint; v_expected bigint; + v_mine bigint; v_object_key text; v_quarantine_key text; begin @@ -320,20 +334,32 @@ begin if v_recent >= 10 then raise exception using message = 'rate_limited', errcode = 'P0001'; end if; - -- Bytes that exist (every object of the bucket, whatever its row's state) - -- plus the declared size of pending uploads whose bytes have not landed yet. - -- A claim never outlives the row it belongs to, and a row with no bytes - -- weighs nothing once it moved on, so no account can pin the ceiling. + -- Per-account budget on bytes present under the caller's two prefixes + -- (clean photo, quarantine objects within their grace): bounds what one + -- account can hold against the global ceiling. + select coalesce(sum((o.metadata ->> 'size')::bigint), 0) into v_mine + from storage.objects o + where o.bucket_id = 'avatars' + and (o.name like 'profiles/' || v_actor || '/%' or o.name like 'quarantine/' || v_actor || '/%'); + if v_mine + 5242880 > private.account_byte_budget() then + raise exception using message = 'rate_limited', errcode = 'P0001'; + end if; + -- Global ceiling: bytes that exist (every sized object of the bucket, + -- whatever its row's state) plus one cap per pending upload whose bytes + -- have not landed yet (the declared size is a claim the upload does not + -- enforce). A claim never outlives its row and a row without bytes weighs + -- nothing once it moved on, so no account can pin the ceiling. select coalesce(sum((o.metadata ->> 'size')::bigint), 0) into v_used from storage.objects o where o.bucket_id = 'avatars'; - select coalesce(sum(a.declared_size), 0) into v_expected + select count(*) * 5242880 into v_expected from public.assets a where a.status = 'pending_upload' and not exists ( select 1 from storage.objects o where o.bucket_id = a.bucket_id and o.name = a.quarantine_key + and o.metadata ->> 'size' is not null ); - if v_used + v_expected + declared_size > private.storage_ceiling() then + if v_used + v_expected + 5242880 > private.storage_ceiling() then raise exception using message = 'storage_full', errcode = 'P0001'; end if; diff --git a/supabase/tests/010_files_foundation.sql b/supabase/tests/010_files_foundation.sql index e8f12bb6..7389470c 100644 --- a/supabase/tests/010_files_foundation.sql +++ b/supabase/tests/010_files_foundation.sql @@ -2,7 +2,7 @@ -- browser and worker RPCs, the purge queue. Every allowed path has its denial -- twin; the eight scenarios of the Step 8 brief are named in the descriptions. begin; -select plan(138); +select plan(140); insert into auth.users (id, email) values ('40000000-0000-0000-0000-00000000000a', 'alice-files@test.local'), @@ -504,13 +504,30 @@ set local role authenticated; select set_config('request.jwt.claims', '{"sub":"40000000-0000-0000-0000-00000000000c","role":"authenticated"}', true); select lives_ok($$ select public.begin_avatar_upload('image/jpeg', 1000) $$, 'scenario: rows that moved on without bytes weigh nothing, even while queued for purge'); reset role; --- Bytes present count whatever the row says: one 10 GiB object fills the ceiling. -insert into storage.objects (bucket_id, name, metadata) values ('avatars', 'quarantine/40000000-0000-0000-0000-00000000000c/00000000-0000-0000-0000-00000000ffff/upload', '{"size":10737418240}'); +-- Bytes present count whatever the row says: one 10 GiB object (under a foreign +-- prefix, so the per-account budget stays out of the way) fills the ceiling. +insert into storage.objects (bucket_id, name, metadata) values ('avatars', 'quarantine/40000000-0000-0000-0000-00000000000d/00000000-0000-0000-0000-00000000ffff/upload', '{"size":10737418240}'); set local role authenticated; select set_config('request.jwt.claims', '{"sub":"40000000-0000-0000-0000-00000000000c","role":"authenticated"}', true); select throws_ok($$ select public.begin_avatar_upload('image/jpeg', 1000) $$, 'P0001', 'storage_full', 'bytes present in the bucket fill the ceiling whatever their row'); reset role; -update storage.objects set metadata = '{"size":1}' where name = 'quarantine/40000000-0000-0000-0000-00000000000c/00000000-0000-0000-0000-00000000ffff/upload'; +update storage.objects set metadata = '{"size":1}' where name = 'quarantine/40000000-0000-0000-0000-00000000000d/00000000-0000-0000-0000-00000000ffff/upload'; +-- A pending row whose bytes landed is charged once, by its object (no double count). +update public.assets set declared_size = 5242880 where owner_id = '40000000-0000-0000-0000-00000000000c' and status = 'pending_upload'; +insert into storage.objects (bucket_id, name, metadata) + select 'avatars', a.quarantine_key, '{"size":1024}' from public.assets a + where a.owner_id = '40000000-0000-0000-0000-00000000000c' and a.status = 'pending_upload'; +set local role authenticated; +select set_config('request.jwt.claims', '{"sub":"40000000-0000-0000-0000-00000000000c","role":"authenticated"}', true); +select lives_ok($$ select public.begin_avatar_upload('image/jpeg', 1000) $$, 'a landed upload is counted by its bytes, not by its claim'); +reset role; +-- Per-account budget: 64 MiB of the caller's own bytes present → rate_limited. +insert into storage.objects (bucket_id, name, metadata) values ('avatars', 'quarantine/40000000-0000-0000-0000-00000000000c/00000000-0000-0000-0000-00000000fffe/upload', '{"size":67108864}'); +set local role authenticated; +select set_config('request.jwt.claims', '{"sub":"40000000-0000-0000-0000-00000000000c","role":"authenticated"}', true); +select throws_ok($$ select public.begin_avatar_upload('image/jpeg', 1000) $$, 'P0001', 'rate_limited', 'scenario: one account cannot hold more than its byte budget in the bucket'); +reset role; +update storage.objects set metadata = '{"size":1}' where name = 'quarantine/40000000-0000-0000-0000-00000000000c/00000000-0000-0000-0000-00000000fffe/upload'; -- Carol's pending row (begun above): the object may not exceed the declared size. set local role authenticated; select set_config('request.jwt.claims', '{"sub":"40000000-0000-0000-0000-00000000000c","role":"authenticated"}', true);