feat(workspace): real-stack avatar e2e, compose worker service, STORAGE.md contract
packages/supabase/tests/e2e/avatar-flow.e2e.ts (pnpm e2e:avatar through scripts/avatar-e2e.sh: worker container on the Supabase network, 5 s sweep, no quarantine grace; a fresh account per run): clean upload and owner-only download (Bob, anon, quarantine key, direct final-key write refused), replacement purged after the grace (queue clock shifted through psql), late bytes through a still-valid signed upload URL swept, hostile and corrupted uploads rejected with closed codes, removal with the purge queue drained. Worker: the Storage API answers a missing object with HTTP 400 and statusCode 404; normalised to 404 so a purge of a never-written key is acked. docker-compose.yml: service worker under the worker profile (read-only, cap_drop ALL, no-new-privileges, 512m, 1 cpu, 64 pids, .env.worker, external Supabase network). docs/development/STORAGE.md: the implementation contract.
This commit is contained in:
1 parent
0410af0c35
commit
0c6053c174
7 files changed
+798
-7
No files matched your search
@@ -0,0 +1,331 @@
|
||||
/**
|
||||
* Avatar flow against the real local stack and the worker container
|
||||
* (docs/development/STORAGE.md §4, §3; STEP8_FILES_ANALYSIS IV.2 "integration"
|
||||
* and "real-stack e2e"). Run through `pnpm e2e:avatar` (scripts/avatar-e2e.sh),
|
||||
* which starts the worker with a 5 s sweep and a 0 s quarantine grace.
|
||||
*
|
||||
* Environment: SUPABASE_URL, SUPABASE_ANON_KEY, SUPABASE_SERVICE_ROLE_KEY
|
||||
* (the service key is used for inspection only: listing objects and shifting
|
||||
* the purge clock through psql in the database container).
|
||||
*/
|
||||
import { createHash } from 'node:crypto'
|
||||
import { execFileSync } from 'node:child_process'
|
||||
import { deflateSync } from 'node:zlib'
|
||||
import { createClient, type SupabaseClient } from '@supabase/supabase-js'
|
||||
import {
|
||||
beginAvatarUpload,
|
||||
confirmAssetUpload,
|
||||
downloadAvatar,
|
||||
fetchOwnAvatarState,
|
||||
removeAvatar,
|
||||
uploadToQuarantine,
|
||||
type AssetRow,
|
||||
type Database,
|
||||
} from '../../src'
|
||||
|
||||
const URL_ = process.env.SUPABASE_URL
|
||||
const ANON = process.env.SUPABASE_ANON_KEY
|
||||
const SERVICE = process.env.SUPABASE_SERVICE_ROLE_KEY
|
||||
const DB_CONTAINER = process.env.AVATAR_E2E_DB_CONTAINER ?? 'supabase_db_faunosteo'
|
||||
const PASSWORD = 'LocalDev-only-password-1'
|
||||
const BOB = 'bob@faunosteo.local'
|
||||
const WORKER_WAIT_MS = 60_000
|
||||
|
||||
type Client = SupabaseClient<Database>
|
||||
|
||||
function anonClient(): Client {
|
||||
return createClient<Database>(URL_ ?? '', ANON ?? '', {
|
||||
auth: { persistSession: false, autoRefreshToken: false },
|
||||
})
|
||||
}
|
||||
|
||||
async function signedIn(email: string): Promise<Client> {
|
||||
const client = anonClient()
|
||||
const { error } = await client.auth.signInWithPassword({ email, password: PASSWORD })
|
||||
if (error) throw new Error(`sign-in ${email}: ${error.message}`)
|
||||
return client
|
||||
}
|
||||
|
||||
function serviceClient(): Client {
|
||||
return createClient<Database>(URL_ ?? '', SERVICE ?? '', {
|
||||
auth: { persistSession: false, autoRefreshToken: false },
|
||||
})
|
||||
}
|
||||
|
||||
// --- fixtures ---------------------------------------------------------------
|
||||
|
||||
const CRC_TABLE = new Uint32Array(256).map((_, n) => {
|
||||
let c = n
|
||||
for (let k = 0; k < 8; k++) c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1
|
||||
return c >>> 0
|
||||
})
|
||||
|
||||
function crc32(bytes: Uint8Array): number {
|
||||
let c = 0xffffffff
|
||||
for (const b of bytes) c = CRC_TABLE[(c ^ b) & 0xff] ^ (c >>> 8)
|
||||
return (c ^ 0xffffffff) >>> 0
|
||||
}
|
||||
|
||||
function chunk(type: string, payload: Uint8Array): Uint8Array {
|
||||
const out = new Uint8Array(12 + payload.length)
|
||||
const view = new DataView(out.buffer)
|
||||
view.setUint32(0, payload.length)
|
||||
out.set(Buffer.from(type, 'ascii'), 4)
|
||||
out.set(payload, 8)
|
||||
const crc = crc32(out.subarray(4, 8 + payload.length))
|
||||
view.setUint32(8 + payload.length, crc)
|
||||
return out
|
||||
}
|
||||
|
||||
/** A valid RGBA PNG gradient of the given size (no dependency, like the worker's fixtures). */
|
||||
function png(width: number, height: number): Uint8Array {
|
||||
const raw = new Uint8Array(height * (1 + width * 4))
|
||||
for (let y = 0; y < height; y++) {
|
||||
const row = y * (1 + width * 4)
|
||||
raw[row] = 0
|
||||
for (let x = 0; x < width; x++) {
|
||||
const i = row + 1 + x * 4
|
||||
raw[i] = (x * 255) / width
|
||||
raw[i + 1] = (y * 255) / height
|
||||
raw[i + 2] = 128
|
||||
raw[i + 3] = 255
|
||||
}
|
||||
}
|
||||
const ihdr = new Uint8Array(13)
|
||||
const view = new DataView(ihdr.buffer)
|
||||
view.setUint32(0, width)
|
||||
view.setUint32(4, height)
|
||||
ihdr.set([8, 6, 0, 0, 0], 8)
|
||||
return Buffer.concat([
|
||||
Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]),
|
||||
chunk('IHDR', ihdr),
|
||||
chunk('IDAT', deflateSync(raw)),
|
||||
chunk('IEND', new Uint8Array(0)),
|
||||
])
|
||||
}
|
||||
|
||||
const asBlob = (bytes: Uint8Array, type: string) => new Blob([Uint8Array.from(bytes)], { type })
|
||||
|
||||
// --- helpers ----------------------------------------------------------------
|
||||
|
||||
async function waitFor<T>(
|
||||
read: () => Promise<T | null>,
|
||||
timeoutMs: number,
|
||||
label: string,
|
||||
): Promise<T> {
|
||||
const started = Date.now()
|
||||
for (;;) {
|
||||
const value = await read()
|
||||
if (value !== null) return value
|
||||
if (Date.now() - started > timeoutMs) throw new Error(`timeout: ${label}`)
|
||||
await new Promise((r) => setTimeout(r, 1000))
|
||||
}
|
||||
}
|
||||
|
||||
async function ownRow(client: Client, userId: string, assetId: string): Promise<AssetRow | null> {
|
||||
const { data, error } = await client
|
||||
.from('assets')
|
||||
.select('id, owner_id, kind, bucket_id, object_key, status, declared_mime, declared_size, byte_size, sha256, rejection_reason, created_at, updated_at')
|
||||
.eq('owner_id', userId)
|
||||
.eq('id', assetId)
|
||||
.maybeSingle()
|
||||
if (error) throw new Error(error.message)
|
||||
return data
|
||||
}
|
||||
|
||||
/** Row once the worker has settled it (clean or rejected). */
|
||||
function settled(client: Client, userId: string, assetId: string) {
|
||||
return waitFor(
|
||||
async () => {
|
||||
const row = await ownRow(client, userId, assetId)
|
||||
return row && (row.status === 'clean' || row.status === 'rejected') ? row : null
|
||||
},
|
||||
WORKER_WAIT_MS,
|
||||
`worker verdict on ${assetId}`,
|
||||
)
|
||||
}
|
||||
|
||||
/** Upload one blob end to end and return the settled row. */
|
||||
async function uploadAndSettle(
|
||||
client: Client,
|
||||
userId: string,
|
||||
blob: Blob,
|
||||
declaredMime: 'image/jpeg' | 'image/png' | 'image/webp',
|
||||
): Promise<AssetRow> {
|
||||
const ticket = await beginAvatarUpload(client, { declaredMime, declaredSize: blob.size })
|
||||
await uploadToQuarantine(client, { ...ticket, blob })
|
||||
await confirmAssetUpload(client, ticket.assetId)
|
||||
return settled(client, userId, ticket.assetId)
|
||||
}
|
||||
|
||||
/** True when the service role can still download the object (it bypasses the policies). */
|
||||
async function objectExists(service: Client, bucket: string, key: string): Promise<boolean> {
|
||||
const { data, error } = await service.storage.from(bucket).download(key)
|
||||
return !error && data !== null && data.size > 0
|
||||
}
|
||||
|
||||
function psql(sql: string): string {
|
||||
return execFileSync('docker', [
|
||||
'exec', DB_CONTAINER, 'psql', '-U', 'postgres', '-d', 'postgres', '-Atc', sql,
|
||||
]).toString().trim()
|
||||
}
|
||||
|
||||
/** Moves every due date of the purge queue to now (the grace is 48 h / 1 h in production). */
|
||||
function expeditePurges(): void {
|
||||
psql('update private.object_purge_queue set not_before = now() where not_before > now()')
|
||||
}
|
||||
|
||||
/** Purge entries still queued for an account (served entries are acked, hence deleted). */
|
||||
function queuedPurges(userId: string): number {
|
||||
return Number(psql(`select count(*) from private.object_purge_queue where object_name like '%/${userId}/%'`))
|
||||
}
|
||||
|
||||
async function downloadFails(client: Client, key: string): Promise<boolean> {
|
||||
const { data, error } = await client.storage.from('avatars').download(key)
|
||||
return Boolean(error) || data === null
|
||||
}
|
||||
|
||||
function sha256(bytes: Uint8Array): string {
|
||||
return createHash('sha256').update(bytes).digest('hex')
|
||||
}
|
||||
|
||||
// --- suite ------------------------------------------------------------------
|
||||
|
||||
const configured = Boolean(URL_ && ANON && SERVICE)
|
||||
|
||||
describe.skipIf(!configured)('avatar flow on the real stack', () => {
|
||||
const service = serviceClient()
|
||||
let alice: Client
|
||||
let aliceId: string
|
||||
let bob: Client
|
||||
|
||||
beforeAll(async () => {
|
||||
// A fresh account per run keeps the seeded accounts clean and the per-hour rate out of reach.
|
||||
const email = `e2e-avatar-${Date.now()}@faunosteo.local`
|
||||
const created = await service.auth.admin.createUser({
|
||||
email,
|
||||
password: PASSWORD,
|
||||
email_confirm: true,
|
||||
user_metadata: { display_name: 'E2E Avatar', locale: 'fr' },
|
||||
})
|
||||
if (created.error || !created.data.user) throw new Error(`createUser: ${created.error?.message}`)
|
||||
aliceId = created.data.user.id
|
||||
alice = await signedIn(email)
|
||||
bob = await signedIn(BOB)
|
||||
})
|
||||
|
||||
afterAll(async () => {
|
||||
if (aliceId) await service.auth.admin.deleteUser(aliceId)
|
||||
})
|
||||
|
||||
it('cleans a valid upload, and only the owner can download the result', async () => {
|
||||
const source = png(640, 480)
|
||||
const row = await uploadAndSettle(alice, aliceId, asBlob(source, 'image/png'), 'image/png')
|
||||
expect(row.status).toBe('clean')
|
||||
const blob = await downloadAvatar(alice, row.object_key)
|
||||
const bytes = new Uint8Array(await blob.arrayBuffer())
|
||||
expect(Buffer.from(bytes.subarray(0, 4)).toString('ascii')).toBe('RIFF')
|
||||
expect(Buffer.from(bytes.subarray(8, 12)).toString('ascii')).toBe('WEBP')
|
||||
expect(row.byte_size).toBe(bytes.length)
|
||||
expect(row.sha256).toBe(sha256(bytes))
|
||||
expect(bytes.length).toBeLessThan(512 * 1024)
|
||||
|
||||
const quarantineKey = `quarantine/${aliceId}/${row.id}/upload`
|
||||
expect(await downloadFails(alice, quarantineKey)).toBe(true)
|
||||
expect(await downloadFails(bob, row.object_key)).toBe(true)
|
||||
expect(await downloadFails(anonClient(), row.object_key)).toBe(true)
|
||||
const bobRows = await bob.from('assets').select('id').eq('owner_id', aliceId)
|
||||
expect(bobRows.data).toEqual([])
|
||||
const direct = await alice.storage.from('avatars').upload(row.object_key, asBlob(source, 'image/png'), {
|
||||
upsert: false,
|
||||
contentType: 'image/png',
|
||||
})
|
||||
expect(direct.error).not.toBeNull()
|
||||
// The worker deleted the quarantine object once the row was clean.
|
||||
await waitFor(
|
||||
async () => ((await objectExists(service, 'avatars', quarantineKey)) ? null : true),
|
||||
30_000,
|
||||
'quarantine object deleted',
|
||||
)
|
||||
})
|
||||
|
||||
it('replaces the photo and purges the previous object after its grace', async () => {
|
||||
const before = await fetchOwnAvatarState(alice, aliceId)
|
||||
const previous = before.clean
|
||||
expect(previous).not.toBeNull()
|
||||
const row = await uploadAndSettle(alice, aliceId, asBlob(png(300, 700), 'image/png'), 'image/png')
|
||||
expect(row.status).toBe('clean')
|
||||
const state = await fetchOwnAvatarState(alice, aliceId)
|
||||
expect(state.clean?.id).toBe(row.id)
|
||||
expect(await downloadFails(alice, previous!.object_key)).toBe(true)
|
||||
expect(await objectExists(service, 'avatars', previous!.object_key)).toBe(true)
|
||||
expeditePurges()
|
||||
await waitFor(
|
||||
async () => ((await objectExists(service, 'avatars', previous!.object_key)) ? null : true),
|
||||
30_000,
|
||||
'replaced object purged after its grace',
|
||||
)
|
||||
})
|
||||
|
||||
it('sweeps bytes landed through a still-valid signed upload URL after the row moved on', async () => {
|
||||
const stale = await beginAvatarUpload(alice, { declaredMime: 'image/png', declaredSize: 1000 })
|
||||
const signed = await alice.storage.from('avatars').createSignedUploadUrl(stale.quarantineKey)
|
||||
expect(signed.error).toBeNull()
|
||||
// A new pick supersedes the row; the signed URL is still valid for two hours.
|
||||
const fresh = await beginAvatarUpload(alice, { declaredMime: 'image/png', declaredSize: 1000 })
|
||||
expect(fresh.assetId).not.toBe(stale.assetId)
|
||||
const late = await alice.storage
|
||||
.from('avatars')
|
||||
.uploadToSignedUrl(stale.quarantineKey, signed.data!.token, asBlob(png(100, 100), 'image/png'), {
|
||||
contentType: 'image/png',
|
||||
})
|
||||
// Either the API refused the late upload, or the sweeper removes the leftover.
|
||||
if (!late.error) {
|
||||
await waitFor(
|
||||
async () => ((await objectExists(service, 'avatars', stale.quarantineKey)) ? null : true),
|
||||
30_000,
|
||||
'late quarantine bytes swept',
|
||||
)
|
||||
}
|
||||
const staleRow = await ownRow(alice, aliceId, stale.assetId)
|
||||
expect(staleRow?.status).toBe('deleted')
|
||||
expect(await downloadFails(alice, stale.quarantineKey)).toBe(true)
|
||||
})
|
||||
|
||||
it('rejects hostile and corrupted uploads with closed codes', async () => {
|
||||
const html = Buffer.from('<!doctype html><script>alert(1)</script>')
|
||||
const htmlRow = await uploadAndSettle(alice, aliceId, asBlob(html, 'image/jpeg'), 'image/jpeg')
|
||||
expect(htmlRow.status).toBe('rejected')
|
||||
expect(htmlRow.rejection_reason).toBe('unsupported_type')
|
||||
|
||||
const truncated = png(200, 200).subarray(0, 80)
|
||||
const truncatedRow = await uploadAndSettle(alice, aliceId, asBlob(truncated, 'image/png'), 'image/png')
|
||||
expect(truncatedRow.status).toBe('rejected')
|
||||
expect(truncatedRow.rejection_reason).toBe('decode_failed')
|
||||
|
||||
const tinyRow = await uploadAndSettle(alice, aliceId, asBlob(png(32, 32), 'image/png'), 'image/png')
|
||||
expect(tinyRow.status).toBe('rejected')
|
||||
expect(tinyRow.rejection_reason).toBe('dimensions')
|
||||
})
|
||||
|
||||
it('removes the photo, cuts the download at once and empties the objects after the grace', async () => {
|
||||
const row = await uploadAndSettle(alice, aliceId, asBlob(png(512, 512), 'image/png'), 'image/png')
|
||||
expect(row.status).toBe('clean')
|
||||
expect(await removeAvatar(alice)).toBeGreaterThanOrEqual(1)
|
||||
expect(await downloadFails(alice, row.object_key)).toBe(true)
|
||||
const state = await fetchOwnAvatarState(alice, aliceId)
|
||||
expect(state.clean).toBeNull()
|
||||
expeditePurges()
|
||||
await waitFor(
|
||||
async () => {
|
||||
const listed = await service.storage.from('avatars').list(`profiles/${aliceId}`, { limit: 100 })
|
||||
const remaining = (listed.data ?? []).length
|
||||
return remaining === 0 ? true : null
|
||||
},
|
||||
30_000,
|
||||
'every object of the account purged',
|
||||
)
|
||||
// Every served entry was acked, a missing object counting as purged.
|
||||
await waitFor(async () => (queuedPurges(aliceId) === 0 ? true : null), 30_000, 'purge queue drained')
|
||||
})
|
||||
})
|
||||
Reference in new issue
Block a user