feat(supabase): db:trial backup, restore, destroy, migrate, rotations

Cold read-only backups proven before any change (checksums covering
the folder, entry counts, keys, images); typed confirmations at a
terminal only; rotate-secrets in one explicit transaction with a commit
marker, no secrets.prev; destroy refuses without a restorable backup and
names the manual way out. Harness: 210 stub-only cases.
This commit is contained in:
bchanot committed 2026-11-10 19:08:33 +01:00
1 parent 1a7b952f2f
commit b85614aa09
3 files changed
+994 -11

No files matched your search

+519 -4
View File
@@ -875,9 +875,8 @@ check_helper_image() {
# so a create that died after making something is never mistaken for one that made nothing: past
# it, the files go only when no faunosteo_trial_ resource exists; else nothing is removed and the
# way out is named.
create_stopped() {
local status=$? present
[ "$status" -ne 0 ] || return 0
create_stopped() { # cleared on success: a run of the trap is always a failure
local present
present=''
if [ "$FIRST_RESOURCE" = yes ]; then present="$(existing_resources 2>/dev/null)" || present=unknown; fi
if [ -z "$present" ]; then
@@ -887,7 +886,7 @@ create_stopped() {
fi
echo "db-trial: create stopped before the end; nothing was removed. Present now:" >&2
existing_resources | sed 's/^/ /' >&2
echo "pnpm db:trial status shows the state; pnpm db:trial destroy (after the owner's go) clears it." >&2
echo "pnpm db:trial status shows the state; pnpm db:trial destroy (after the owner's go) backs the data up and clears it, or names the manual removal when it cannot back up." >&2
}
FIRST_RESOURCE=no
# Everything that can refuse comes first: no file, no Docker write yet.
@@ -1031,6 +1030,12 @@ status_without_stack() {
existing_resources | sed 's/^/ present anyway: /'
list_backups
}
# Only that it exists, never what it holds.
status_leftover_rotation() {
if [ -e "$STATE/secrets.next.env" ]; then
say "secrets.next.env exists: a rotate-secrets stopped part-way and rotate-secrets refuses until it is dealt with (pnpm db:trial restore <pre-rotate backup>)"
fi
}
cmd_status() {
[ $# -eq 0 ] || die "status takes no argument"
if [ ! -e "$MARKER" ] || [ ! -f "$SECRETS" ]; then status_without_stack; return 0; fi
@@ -1040,10 +1045,509 @@ cmd_status() {
status_containers
say "Docker endpoint: $(daemon_endpoint)"
status_keys_and_host
status_leftover_rotation
if container_running "$DB"; then status_database; else say "database stopped: pnpm db:trial start"; fi
list_backups
}
# The exact phrase, typed at a terminal: piped stdin (an agent, a script) is refused before the
# prompt; anything else typed, or nothing, stops too.
confirm() { # phrase
local typed=''
[ -t 0 ] || die "a typed confirmation needs a terminal (stdin is not one): nothing was changed"
printf 'Type "%s" to go on (anything else stops): ' "$1" >&2
read -r typed || true
[ "$typed" = "$1" ] || die "not confirmed: nothing was changed"
}
# Every name a backup folder may hold: a failed backup removes exactly these, then the folder.
BACKUP_FILES=(db.tar storage.tar secrets.env secrets.next.env stack.conf migrations.sha256 SHA256SUMS SHA256SUMS.new)
new_backup_dir() { # label → a fresh folder (700)
local base dir n=1
# A link would send the archives wherever it points.
[ ! -L "$BACKUPS" ] || die "$BACKUPS is a symbolic link: refused, nothing was changed"
base="$BACKUPS/$(date -u +%Y%m%dT%H%M%SZ)-$1"
dir="$base"
while [ -e "$dir" ]; do n=$((n + 1)); dir="$base-$n"; done
( umask 077; mkdir -p "$BACKUPS"; mkdir "$dir" ) || die "cannot create $dir"
printf '%s' "$dir"
}
# Size in KiB and entry count of a volume, by the helper (read-only, no network); the entries are
# counted from the root tar archives, so the archive must list as many.
volume_measure() { # volume → "<KiB> <entries>"
docker run --rm --name "$HELPER" --network none -v "$1:/volume:ro" --entrypoint sh "$IMAGE_DB" \
-c 'du -sk /volume | cut -f1 && cd /volume && find . ! -type s | wc -l' | tr '\n' ' '
}
# The listing of an archive cut at a member boundary still exits 0: its entry count is compared.
backup_volume() { # dir volume entries
local file="$1/${2#faunosteo_trial_}.tar" listed
( umask 077
docker run --rm --name "$HELPER" --network none -v "$2:/volume:ro" --entrypoint tar "$IMAGE_DB" \
--numeric-owner -C /volume -cf - . > "$file" ) || return 1
listed="$(tar -tf "$file" | wc -l)" || return 1
[ "$listed" -eq "$3" ]
}
# A caller that cannot go on without its backup (destroy) names a function that prints the way
# out; backup_failed appends it, so the one failure message stays one message.
BACKUP_WAY_OUT=''
backup_failed() { # dir why: this run's files go by name, then the (now empty) folder
local f
trap - EXIT
for f in "${BACKUP_FILES[@]}"; do rm -f -- "$1/$f"; done
rmdir -- "$1" 2>/dev/null || true
die "backup failed ($2): nothing kept, the stack stays stopped (pnpm db:trial start)${BACKUP_WAY_OUT:+
$($BACKUP_WAY_OUT '')}"
}
# An exit that is not backup_failed's (Ctrl+C, a failing command): the folder stays, unfinished.
BACKUP_RUNNING=''
# Cleared on success and by backup_failed, so a run of the trap is always a failure (its $? is not
# reliable: a SIGTERM leaves 0 in it).
backup_interrupted() {
echo "db-trial: backup $(basename "$BACKUP_RUNNING") is incomplete (no SHA256SUMS: status lists it as incomplete, restore refuses it); the stack may be stopped (pnpm db:trial start)" >&2
}
# Cold copy of every trial volume that exists, with the state files present (the volumes are the
# data, the files only follow): the stack stops, the volumes are measured and free space checked,
# each volume is archived read-only by a --rm helper with no network and its listing must hold
# as many entries as the volume, then the checksums are written last
# (a folder without SHA256SUMS never finished). The stack restarts if it ran and the caller asks.
backup_now() { # label restart(yes|no) → LAST_BACKUP
local dir name vol kib entries i free need=0 vols=() counts=() was_running=no
for name in "${CONTAINERS[@]}"; do if container_running "$name"; then was_running=yes; fi; done
for vol in "$VOL_DB" "$VOL_STORAGE"; do if volume_exists "$vol"; then vols+=("$vol"); fi; done
[ "${#vols[@]}" -gt 0 ] || die "no trial volume to back up"
# A helper left over would collide with every archive run below, after the stack stopped.
if container_exists "$HELPER"; then
die "a $HELPER container is left over (docker rm $HELPER): refused, nothing was changed"
fi
dir="$(new_backup_dir "$1")"
BACKUP_RUNNING="$dir"
trap backup_interrupted EXIT
stop_containers
for vol in "${vols[@]}"; do
read -r kib entries <<<"$(volume_measure "$vol")"
if ! [[ "$kib" =~ ^[0-9]+$ && "$entries" =~ ^[0-9]+$ ]]; then backup_failed "$dir" "measure of $vol"; fi
need=$((need + kib))
counts+=("$entries")
done
# The backups share the disk of Docker's data root (DEV, the sandbox): never fill it.
free="$(df -Pk "$dir" | awk 'NR == 2 { print $4 }')"
if ! [[ "$free" =~ ^[0-9]+$ ]] || [ "$free" -lt $((need * 11 / 10)) ]; then
backup_failed "$dir" "less free space under $BACKUPS than 110 % of the volumes"
fi
for i in "${!vols[@]}"; do
backup_volume "$dir" "${vols[$i]}" "${counts[$i]}" || backup_failed "$dir" "archive of ${vols[$i]} unreadable or incomplete"
done
for name in secrets.env secrets.next.env stack.conf migrations.sha256; do
if [ -f "$STATE/$name" ]; then cp -p "$STATE/$name" "$dir/" || backup_failed "$dir" "copy of $name"; fi
done
( cd "$dir" && umask 077 && sha256sum -- * > SHA256SUMS.new && mv SHA256SUMS.new SHA256SUMS ) \
|| backup_failed "$dir" "checksums"
LAST_BACKUP="$dir"
trap - EXIT
if [ "$2" = yes ] && [ "$was_running" = yes ]; then start_containers; fi
say "backup $(basename "$dir") ($(du -sh "$dir" | cut -f1)) in $BACKUPS"
}
cmd_backup() {
[ $# -eq 0 ] || die "backup takes no argument"
require_state
take_lock
require_created
backup_now manual yes
}
secret_digest() { grep -E '^TR_(PW|JWT_SECRET)=' "$1" | sha256sum | cut -d' ' -f1; }
# The caller says what became of the stack ("nothing was changed", "the stack stays stopped"):
# these checks also run after a backup was taken, when that sentence differs.
refuse_backup() { # dir reason [outcome]
die "backup $(basename "$1") $2${3:+: $3}"
}
# What a restore may trust of a folder: exactly the files its checksums cover, all regular, all
# matching, every archive readable. Run again right before the extraction (the folder is a
# user-writable place: time of check against time of use).
verify_backup_integrity() { # dir [outcome]
local dir="$1" outcome="${2:-}" present listed archive
if [ -L "$BACKUPS" ] || [ -L "$dir" ]; then refuse_backup "$dir" "is reached through a symbolic link" "$outcome"; fi
[ -f "$dir/SHA256SUMS" ] || refuse_backup "$dir" "is incomplete (no SHA256SUMS)" "$outcome"
[ -z "$(find "$dir" -mindepth 1 ! -type f)" ] \
|| refuse_backup "$dir" "holds a symbolic link or an entry that is not a regular file" "$outcome"
present="$(find "$dir" -mindepth 1 ! -name SHA256SUMS -printf '%f\n' | LC_ALL=C sort)"
listed="$(sed 's/^[0-9a-f]\{64\} //' "$dir/SHA256SUMS" | LC_ALL=C sort)"
[ "$present" = "$listed" ] \
|| refuse_backup "$dir" "has a SHA256SUMS that does not list exactly the files of its folder" "$outcome"
( cd "$dir" && sha256sum --quiet --strict -c SHA256SUMS ) || refuse_backup "$dir" "fails its checksums" "$outcome"
for archive in "$dir"/*.tar; do
tar -tf "$archive" >/dev/null 2>&1 || refuse_backup "$dir" "holds an unreadable archive ($(basename "$archive"))" "$outcome"
done
}
first_absent_image() { # the recorded IMAGE_* (parse_conf ran), the first one not here; nothing when all are
local image
for image in "$IMAGE_DB" "$IMAGE_AUTH" "$IMAGE_REST" "$IMAGE_STORAGE" "$IMAGE_MAIL" "$IMAGE_GATEWAY"; do
if ! docker image inspect -f '{{.Id}}' "$image" >/dev/null 2>&1; then printf '%s' "$image"; return 0; fi
done
}
# Restorable: the folder is intact, both archives, the secrets and stack.conf are there and
# parse, the keys are signed with the backup's own secret and the six images it recorded are here.
check_backup() { # dir [outcome]
local dir="$1" outcome="${2:-}" file image
verify_backup_integrity "$dir" "$outcome"
for file in db.tar storage.tar secrets.env stack.conf; do
[ -f "$dir/$file" ] || refuse_backup "$dir" "lacks $file" "$outcome"
done
( parse_secrets "$dir/secrets.env" && parse_conf "$dir/stack.conf" ) 2>/dev/null \
|| refuse_backup "$dir" "holds incomplete secrets or stack.conf" "$outcome"
# Its keys must be signed with its own secret (expiry aside: rotate-keys renews them after).
( parse_secrets "$dir/secrets.env" && key_check "$TR_ANON_KEY" anon any "$TR_JWT_SECRET" \
&& key_check "$TR_SERVICE_KEY" service_role any "$TR_JWT_SECRET" ) >/dev/null 2>&1 \
|| refuse_backup "$dir" "holds keys that are not signed with its own secret" "$outcome"
image="$(parse_conf "$dir/stack.conf"; first_absent_image)"
[ -z "$image" ] || refuse_backup "$dir" "records an image that is not present locally ($image: docker pull it first)" "$outcome"
}
# The pre-restore backup, when one was taken: the way back if the restore stops part-way.
PRE_RESTORE=''
RESTORE_TARGET=''
restore_stopped() { # cleared on success: a run of the trap is always a failure
echo "db-trial: restore stopped part-way: containers and volumes may be missing or half-filled." >&2
if [ -n "$PRE_RESTORE" ]; then
echo "The state before it is backup $(basename "$PRE_RESTORE"): pnpm db:trial restore $(basename "$PRE_RESTORE")" >&2
else
echo "No backup of the state before it was taken (there was no trial volume)." >&2
fi
echo "The same restore can be rerun: pnpm db:trial restore $RESTORE_TARGET" >&2
}
# A restore must stay possible over a damaged stack: the pre-restore backup only has to be
# intact (it is the way back); when a restore would refuse it as it is, the owner is told.
check_pre_restore_backup() {
local note
verify_backup_integrity "$LAST_BACKUP" "nothing was removed, the stack stays stopped (pnpm db:trial start)"
if ! note="$(check_backup "$LAST_BACKUP" 2>&1)"; then
echo "note: the backup before this restore keeps both volumes, but restore will refuse it as it is (${note#db-trial: })"
fi
if [ -f "$LAST_BACKUP/secrets.next.env" ]; then
echo "note: that backup also holds a leftover secrets.next.env (a rotate-secrets stopped part-way): restoring it brings the file back"
fi
}
# Rebuilds the stack from <backup>: volumes and secrets replaced, containers recreated from the
# images the backup recorded. Works on a destroyed stack too (backups survive destroy).
cmd_restore() {
local name="${1:-}" dir vol f
[ $# -eq 1 ] && [[ "$name" =~ ^[0-9]{8}T[0-9]{6}Z-[a-z0-9-]+$ ]] \
|| die "restore <backup>: the name of a folder in $BACKUPS (pnpm db:trial status lists them)"
require_state
take_lock
dir="$BACKUPS/$name"
[ ! -L "$BACKUPS" ] || die "$BACKUPS is a symbolic link: refused, nothing was changed"
[ ! -L "$dir" ] || die "backup $name is a symbolic link: refused, nothing was changed"
[ -d "$dir" ] || die "no backup $name in $BACKUPS"
check_backup "$dir" "nothing was changed"
refuse_while_app_runs
# Without a trial volume backup_now's own helper check does not run: refused here, before the confirmation.
if container_exists "$HELPER"; then
die "a $HELPER container is left over (docker rm $HELPER): refused, nothing was changed"
fi
if [ -f "$CONF" ]; then parse_conf "$CONF" || true; fi
if [ -f "$SECRETS" ] && [ "$(secret_digest "$SECRETS")" != "$(secret_digest "$dir/secrets.env")" ]; then
echo "note: backup $name holds other secrets than the current ones (taken before a rotate-secrets?): the restore brings them back"
fi
echo "restore replaces the trial stack's data and secrets by backup $name. Present now:"
existing_resources | sed 's/^/ /'
confirm "restore $name"
if volume_exists "$VOL_DB" || volume_exists "$VOL_STORAGE"; then
backup_now pre-restore no
PRE_RESTORE="$LAST_BACKUP"
check_pre_restore_backup
else
echo "no trial volume: nothing to back up before the restore"
fi
RESTORE_TARGET="$name"
stop_containers
# Armed once nothing can refuse any more: a refusal above touched no container or volume.
ports_free "${CONTAINERS[@]}"
trap restore_stopped EXIT
remove_containers
parse_conf "$dir/stack.conf"
verify_backup_integrity "$dir"
for vol in "$VOL_DB" "$VOL_STORAGE"; do
if volume_exists "$vol"; then docker volume rm "$vol" >/dev/null; fi
docker volume create --label faunosteo.stack=trial "$vol" >/dev/null
docker run --rm -i --name "$HELPER" --network none -v "$vol:/volume" --entrypoint tar "$IMAGE_DB" \
--numeric-owner -xpf - -C /volume < "$dir/${vol#faunosteo_trial_}.tar"
done
remove_live_files
for f in secrets.env secrets.next.env stack.conf migrations.sha256; do
if [ -f "$dir/$f" ]; then write_private "$STATE/$f" < "$dir/$f"; fi
done
load_conf
load_secrets
start_containers
trap - EXIT
say "restored $(basename "$dir")${PRE_RESTORE:+; the state before it: $(basename "$PRE_RESTORE")}"
}
# What destroy tells the owner when it refuses for want of a restorable backup: the backup kept
# (or none), the removals the owner may run by hand for what exists now, then the state files.
# The script never runs these itself.
destroy_way_out_line() { # kind command
local names
names="$(existing_resources | sed -n "s/^$1 //p" | tr '\n' ' ')"
if [ -n "$names" ]; then echo " $2 ${names% }"; fi
}
destroy_way_out() { # kept backup folder, or empty
if [ -n "$1" ]; then echo "backup $(basename "$1") is kept."; else echo "no backup could be made."; fi
echo "Nothing was removed. To remove the stack yourself, run by hand:"
destroy_way_out_line container 'docker rm'
destroy_way_out_line network 'docker network rm'
destroy_way_out_line volume 'docker volume rm'
echo "then pnpm db:trial destroy clears the state files (backups kept)."
}
DESTROY_BACKUP=''
destroy_stopped() { # cleared on success: a run of the trap is always a failure
echo "db-trial: destroy stopped part-way: containers, the network or volumes may remain (pnpm db:trial status)." >&2
echo "Its backup $(basename "$DESTROY_BACKUP") is kept: pnpm db:trial restore $(basename "$DESTROY_BACKUP") rebuilds the stack; pnpm db:trial destroy can be rerun." >&2
}
cmd_destroy() {
[ $# -eq 0 ] || die "destroy takes no argument"
local found vol
require_state
take_lock
refuse_while_app_runs
if [ -f "$CONF" ]; then parse_conf "$CONF" || true; fi
found="$(existing_resources)"
if [ -z "$found" ]; then
echo "no faunosteo_trial_ resource exists: destroy removes the live state files of $STATE (backups kept)"
confirm faunosteo_trial
remove_live_files
say "live state files removed; backups kept in $BACKUPS"
return 0
fi
echo "destroy removes:"
printf '%s\n' "$found" | sed 's/^/ /'
if container_running "$DB" && ( parse_secrets "$SECRETS" ) 2>/dev/null; then
load_secrets
echo "content:"
# A damaged database must not stop the destroy that gets rid of it.
{ inventory_counts || echo "(content unreadable)"; } | sed 's/^/ /'
fi
echo "kept: every backup in $BACKUPS, including the one taken now"
confirm faunosteo_trial
if volume_exists "$VOL_DB" || volume_exists "$VOL_STORAGE"; then
BACKUP_WAY_OUT=destroy_way_out
backup_now pre-destroy no
BACKUP_WAY_OUT=''
check_backup "$LAST_BACKUP" "the stack stays stopped (pnpm db:trial start)
$(destroy_way_out "$LAST_BACKUP")"
DESTROY_BACKUP="$LAST_BACKUP"
trap destroy_stopped EXIT
fi
stop_containers
remove_containers
if network_exists "$NET"; then docker network rm "$NET" >/dev/null; fi
for vol in "$VOL_DB" "$VOL_STORAGE"; do if volume_exists "$vol"; then docker volume rm "$vol" >/dev/null; fi; done
remove_live_files
trap - EXIT
say "trial stack destroyed; backups kept in $BACKUPS"
}
migrate_stopped() { # cleared on success: a run of the trap is always a failure
echo "db-trial: migrate stopped part-way: pnpm db:trial restore $(basename "$LAST_BACKUP") returns to the state before it" >&2
}
cmd_migrate() {
[ $# -eq 0 ] || die "migrate takes no argument"
local pending
require_state
take_lock
require_running
published_exactly "$DB"
check_manifest
pending="$(pending_migrations)"
[ -n "$pending" ] || { say "no new migration"; return 0; }
echo "new migrations:"
printf '%s\n' "$pending" | sed 's/^/ /'
backup_now pre-migrate yes
check_backup "$LAST_BACKUP" "no migration was applied; the backup is kept and the stack runs"
trap migrate_stopped EXIT
migrate_up
write_manifest
sb_psql -c "-- db-trial:reload
notify pgrst, 'reload schema'" >/dev/null
trap - EXIT
say "migrated; the backup before it: $(basename "$LAST_BACKUP")"
}
keys_stopped() { # cleared on success: a run of the trap is always a failure
echo "db-trial: rotate-keys stopped part-way; secrets.env holds the new keys; pnpm db:trial start recreates a missing container and restarts the gateway" >&2
}
# A backup first (secrets.env included), then new keys signed with the same secret: the expiry moves one year on. The previous keys stay valid
# until their own expiry (rotate-secrets revokes them). Storage holds the keys in its env; the
# gateway restarts after it (nginx resolved the old container's address).
cmd_rotate_keys() {
[ $# -eq 0 ] || die "rotate-keys takes no argument"
local was_running=no
require_state
take_lock
# A created stack is enough, running or stopped: the way out of expired keys must never need
# the keys to be valid.
require_created
refuse_while_app_runs
# Keys that are not this stack's: refused before the backup stops anything.
require_trial_keys
if container_running "$DB"; then was_running=yes; fi
# The same cold, read-back backup as rotate-secrets, before any change, and proven restorable
# (checksums, archives, keys signed with its secret, image) before any key changes.
backup_now pre-keys yes
check_backup "$LAST_BACKUP" "no key was changed; the backup is kept"
if ! ( umask 077
printf 'TR_PW=%s\nTR_JWT_SECRET=%s\n' "$TR_PW" "$TR_JWT_SECRET" > "$SECRETS.new"
TR_JWT_SECRET="$TR_JWT_SECRET" node -e "$KEYGEN" keys >> "$SECRETS.new" ) \
|| ! ( parse_secrets "$SECRETS.new" ); then
rm -f "$SECRETS.new"
die "key generation failed: nothing was changed"
fi
mv -f "$SECRETS.new" "$SECRETS"
# Armed once secrets.env holds the new keys: before it, only the backup and a .new file exist.
trap keys_stopped EXIT
load_secrets
write_env_files
if container_running "$STORAGE"; then docker stop -t 30 "$STORAGE" >/dev/null; fi
if container_exists "$STORAGE"; then docker rm "$STORAGE" >/dev/null; fi
if [ "$was_running" = yes ]; then
run_service "$STORAGE"
restart_gateway
check_loopback
fi
trap - EXIT
say "anon and service keys renewed, valid until $(key_expiry); the app takes the new anon key at its next start"
if [ "$was_running" = no ]; then echo "the stack is stopped: pnpm db:trial start recreates Storage with the new keys"; fi
}
# The branch comes from the state: secrets.next.env is gone once the rename happened, even when the
# signal fell before ROTATION_COMMITTED was set. The trap is armed only once that file exists.
rotation_stopped() { # cleared on success: a run of the trap is always a failure
if [ "$ROTATION_COMMITTED" = yes ] || [ ! -e "$STATE/secrets.next.env" ]; then
echo "db-trial: rotate-secrets stopped after the database took the new secrets; secrets.env holds them." >&2
echo "pnpm db:trial start recreates the missing containers from it." >&2
else
echo "db-trial: rotate-secrets stopped before secrets.env took the new secrets (the database may or may not have them); secrets.env (previous) and secrets.next.env (new) are both kept." >&2
echo "pnpm db:trial restore $(basename "$LAST_BACKUP") returns to the state before it." >&2
fi
}
ROTATION_COMMITTED=no
# The rotation's own database session name, and how long settle_rotation waits for it to end.
ROTATION_APP=db-trial-rotate
ROTATION_WAIT_POLLS=10
# One transaction run as supabase_admin; values through psql variables read from the exec
# environment (never argv, never the SQL text); statement logging off, errors included. The
# transaction is explicit (begin ... commit), not psql --single-transaction: that flag would
# COMMIT a script cut short at end of input, whereas an input cut before `commit;` leaves the
# server to roll the open transaction back. The same transaction records the digest of the new secrets file in a database setting: the only
# trustworthy proof of the commit (password probes over 127.0.0.1 are `trust` in this image's
# pg_hba.conf and succeed whatever the password).
rotate_in_database() { # new password, new JWT secret, digest of the new secrets file
PGPASSWORD="$TR_PW" PGAPPNAME="$ROTATION_APP" NEW_PW="$1" NEW_JWT="$2" ROT_DIGEST="$3" \
docker exec -i -e PGPASSWORD -e PGAPPNAME -e NEW_PW -e NEW_JWT -e ROT_DIGEST "$DB" \
psql -X -q -h 127.0.0.1 -U supabase_admin -d postgres -v ON_ERROR_STOP=1 >/dev/null <<'SQL'
-- db-trial:rotate
begin;
set local log_statement = 'none';
set local log_min_duration_statement = -1;
set local log_min_error_statement = panic;
set local pgaudit.log = 'none';
\set newpw `printf '%s' "$NEW_PW"`
\set newjwt `printf '%s' "$NEW_JWT"`
\set digest `printf '%s' "$ROT_DIGEST"`
alter role supabase_admin with password :'newpw';
alter role postgres with password :'newpw';
alter role authenticator with password :'newpw';
alter role supabase_auth_admin with password :'newpw';
alter role supabase_storage_admin with password :'newpw';
alter database postgres set "app.settings.jwt_secret" to :'newjwt';
alter database postgres set "app.trial_rotation" to :'digest';
commit;
SQL
}
# The setting the rotation transaction wrote, read in a fresh session (a database setting applies
# to new sessions only). Empty when no rotation committed.
rotation_marker() {
sb_psql -t -A -c "-- db-trial:rotation-marker
select coalesce(current_setting('app.trial_rotation', true), '')"
}
# Sessions still named as the rotation's: a transaction whose client is gone stays open until the
# server notices, and may still commit or roll back.
rotation_sessions() {
sb_psql -t -A -c "-- db-trial:rotation-active
select count(*) from pg_stat_activity where application_name = '$ROTATION_APP'"
}
# Waits (a poll a second, at most ROTATION_WAIT_POLLS) for the rotation session to end: 0 once it
# is gone, 1 when it is still there or the database cannot say.
wait_rotation_session_gone() {
local poll count
for ((poll = 1; poll <= ROTATION_WAIT_POLLS; poll++)); do
count="$(rotation_sessions 2>/dev/null)" || return 1
[[ "$count" =~ ^[0-9]+$ ]] || return 1
[ "$count" -eq 0 ] && return 0
if [ "$poll" -lt "$ROTATION_WAIT_POLLS" ]; then sleep 1; fi
done
return 1
}
rotation_unknown() { # why
trap - EXIT
die "$1: whether the rotation committed is unknown; secrets.env (previous) and secrets.next.env (new) are both kept; pnpm db:trial restore $(basename "$LAST_BACKUP") returns to the state before it"
}
# psql failed: did the transaction commit anyway (a dropped exec stream, a daemon hiccup)? Once
# no session of the rotation is left, the marker decides; the new values are dropped only when
# the database answers without it.
settle_rotation() { # next file, digest of it
local marker
wait_rotation_session_gone \
|| rotation_unknown "a rotation session is still active after $ROTATION_WAIT_POLLS checks, or the database does not answer"
marker="$(rotation_marker 2>/dev/null)" || rotation_unknown "the database does not answer"
if [ "$marker" = "$2" ]; then
say "the database took the new secrets although psql failed: the rotation goes on"
return 0
fi
trap - EXIT
rm -f "$1"
die "the database did not take the new secrets: nothing was changed (backup $(basename "$LAST_BACKUP"))"
}
# New JWT secret, keys and role passwords; account passwords and TOTP factors untouched. The
# new values sit in secrets.next.env (600) before the database changes, so they exist somewhere
# whatever happens next (Ctrl+C included: it exits through the EXIT trap, which keeps that file).
cmd_rotate_secrets() {
[ $# -eq 0 ] || die "rotate-secrets takes no argument"
local next="$STATE/secrets.next.env" new_pw new_jwt digest
require_state
take_lock
require_running
# A previous rotation stopped part-way: that file may be the only copy of the secrets the
# database holds, so it is never overwritten.
[ ! -e "$next" ] || die "$next exists: a previous rotate-secrets left it, and it may hold the only copy of the secrets the database took. Never overwritten: pnpm db:trial restore <pre-rotate backup> (pnpm db:trial status lists the backups) is the way back; nothing was changed"
refuse_while_app_runs
# Keys that are not this stack's: refused before the backup stops anything.
require_trial_keys
backup_now pre-rotate yes
# The backup is the only copy of the previous secrets (no secrets.prev): proven restorable first.
check_backup "$LAST_BACKUP" "no secret was changed; the backup is kept"
new_secrets_file "$next" || true
if ! ( parse_secrets "$next" ); then rm -f "$next"; die "secret generation failed: nothing was changed"; fi
trap rotation_stopped EXIT
new_pw="$(sed -n 's/^TR_PW=//p' "$next")"
new_jwt="$(sed -n 's/^TR_JWT_SECRET=//p' "$next")"
digest="$(sha256sum < "$next" | cut -d' ' -f1)"
if ! rotate_in_database "$new_pw" "$new_jwt" "$digest"; then settle_rotation "$next" "$digest"; fi
# Committed means secrets.env holds the new values: set once the rename is done.
mv -f "$next" "$SECRETS"
ROTATION_COMMITTED=yes
load_secrets
# The containers carry the previous values in their env: removed at once, recreated by
# start_containers (which checks every port before any start).
stop_containers
remove_containers
start_containers
trap - EXIT
say "secrets renewed: sessions signed with the old secret are refused (sign in again); account passwords and TOTP factors unchanged; backup before it: $(basename "$LAST_BACKUP")"
}
# Every command's arguments are checked before any Docker call (a refusal here logs nothing).
# The commands keep their own checks too, so each stays safe if called on its own.
check_arguments() { # action args...
@@ -1052,6 +1556,11 @@ check_arguments() { # action args...
case "$action" in
plan | create | start | stop | status)
[ $# -eq 0 ] || die "$action takes no argument" ;;
migrate | backup | rotate-keys | rotate-secrets | destroy)
[ $# -eq 0 ] || die "$action takes no argument" ;;
restore)
[ $# -eq 1 ] && [[ "$1" =~ ^[0-9]{8}T[0-9]{6}Z-[a-z0-9-]+$ ]] \
|| die "restore <backup>: the name of a folder in $BACKUPS (pnpm db:trial status lists them)" ;;
*) usage; exit 1 ;;
esac
}
@@ -1076,5 +1585,11 @@ case "$action" in
start) cmd_start "$@" ;;
stop) cmd_stop "$@" ;;
status) cmd_status "$@" ;;
migrate) cmd_migrate "$@" ;;
backup) cmd_backup "$@" ;;
restore) cmd_restore "$@" ;;
rotate-keys) cmd_rotate_keys "$@" ;;
rotate-secrets) cmd_rotate_secrets "$@" ;;
destroy) cmd_destroy "$@" ;;
*) usage; exit 1 ;;
esac