feat(supabase): db:trial backup, restore, destroy, migrate, rotations
Cold read-only backups proven before any change (checksums covering the folder, entry counts, keys, images); typed confirmations at a terminal only; rotate-secrets in one explicit transaction with a commit marker, no secrets.prev; destroy refuses without a restorable backup and names the manual way out. Harness: 210 stub-only cases.
This commit is contained in:
1 parent
1a7b952f2f
commit
b85614aa09
3 files changed
+994
-11
No files matched your search
+519
-4
@@ -875,9 +875,8 @@ check_helper_image() {
|
||||
# so a create that died after making something is never mistaken for one that made nothing: past
|
||||
# it, the files go only when no faunosteo_trial_ resource exists; else nothing is removed and the
|
||||
# way out is named.
|
||||
create_stopped() {
|
||||
local status=$? present
|
||||
[ "$status" -ne 0 ] || return 0
|
||||
create_stopped() { # cleared on success: a run of the trap is always a failure
|
||||
local present
|
||||
present=''
|
||||
if [ "$FIRST_RESOURCE" = yes ]; then present="$(existing_resources 2>/dev/null)" || present=unknown; fi
|
||||
if [ -z "$present" ]; then
|
||||
@@ -887,7 +886,7 @@ create_stopped() {
|
||||
fi
|
||||
echo "db-trial: create stopped before the end; nothing was removed. Present now:" >&2
|
||||
existing_resources | sed 's/^/ /' >&2
|
||||
echo "pnpm db:trial status shows the state; pnpm db:trial destroy (after the owner's go) clears it." >&2
|
||||
echo "pnpm db:trial status shows the state; pnpm db:trial destroy (after the owner's go) backs the data up and clears it, or names the manual removal when it cannot back up." >&2
|
||||
}
|
||||
FIRST_RESOURCE=no
|
||||
# Everything that can refuse comes first: no file, no Docker write yet.
|
||||
@@ -1031,6 +1030,12 @@ status_without_stack() {
|
||||
existing_resources | sed 's/^/ present anyway: /'
|
||||
list_backups
|
||||
}
|
||||
# Only that it exists, never what it holds.
|
||||
status_leftover_rotation() {
|
||||
if [ -e "$STATE/secrets.next.env" ]; then
|
||||
say "secrets.next.env exists: a rotate-secrets stopped part-way and rotate-secrets refuses until it is dealt with (pnpm db:trial restore <pre-rotate backup>)"
|
||||
fi
|
||||
}
|
||||
cmd_status() {
|
||||
[ $# -eq 0 ] || die "status takes no argument"
|
||||
if [ ! -e "$MARKER" ] || [ ! -f "$SECRETS" ]; then status_without_stack; return 0; fi
|
||||
@@ -1040,10 +1045,509 @@ cmd_status() {
|
||||
status_containers
|
||||
say "Docker endpoint: $(daemon_endpoint)"
|
||||
status_keys_and_host
|
||||
status_leftover_rotation
|
||||
if container_running "$DB"; then status_database; else say "database stopped: pnpm db:trial start"; fi
|
||||
list_backups
|
||||
}
|
||||
|
||||
# The exact phrase, typed at a terminal: piped stdin (an agent, a script) is refused before the
|
||||
# prompt; anything else typed, or nothing, stops too.
|
||||
confirm() { # phrase
|
||||
local typed=''
|
||||
[ -t 0 ] || die "a typed confirmation needs a terminal (stdin is not one): nothing was changed"
|
||||
printf 'Type "%s" to go on (anything else stops): ' "$1" >&2
|
||||
read -r typed || true
|
||||
[ "$typed" = "$1" ] || die "not confirmed: nothing was changed"
|
||||
}
|
||||
|
||||
# Every name a backup folder may hold: a failed backup removes exactly these, then the folder.
|
||||
BACKUP_FILES=(db.tar storage.tar secrets.env secrets.next.env stack.conf migrations.sha256 SHA256SUMS SHA256SUMS.new)
|
||||
new_backup_dir() { # label → a fresh folder (700)
|
||||
local base dir n=1
|
||||
# A link would send the archives wherever it points.
|
||||
[ ! -L "$BACKUPS" ] || die "$BACKUPS is a symbolic link: refused, nothing was changed"
|
||||
base="$BACKUPS/$(date -u +%Y%m%dT%H%M%SZ)-$1"
|
||||
dir="$base"
|
||||
while [ -e "$dir" ]; do n=$((n + 1)); dir="$base-$n"; done
|
||||
( umask 077; mkdir -p "$BACKUPS"; mkdir "$dir" ) || die "cannot create $dir"
|
||||
printf '%s' "$dir"
|
||||
}
|
||||
# Size in KiB and entry count of a volume, by the helper (read-only, no network); the entries are
|
||||
# counted from the root tar archives, so the archive must list as many.
|
||||
volume_measure() { # volume → "<KiB> <entries>"
|
||||
docker run --rm --name "$HELPER" --network none -v "$1:/volume:ro" --entrypoint sh "$IMAGE_DB" \
|
||||
-c 'du -sk /volume | cut -f1 && cd /volume && find . ! -type s | wc -l' | tr '\n' ' '
|
||||
}
|
||||
# The listing of an archive cut at a member boundary still exits 0: its entry count is compared.
|
||||
backup_volume() { # dir volume entries
|
||||
local file="$1/${2#faunosteo_trial_}.tar" listed
|
||||
( umask 077
|
||||
docker run --rm --name "$HELPER" --network none -v "$2:/volume:ro" --entrypoint tar "$IMAGE_DB" \
|
||||
--numeric-owner -C /volume -cf - . > "$file" ) || return 1
|
||||
listed="$(tar -tf "$file" | wc -l)" || return 1
|
||||
[ "$listed" -eq "$3" ]
|
||||
}
|
||||
# A caller that cannot go on without its backup (destroy) names a function that prints the way
|
||||
# out; backup_failed appends it, so the one failure message stays one message.
|
||||
BACKUP_WAY_OUT=''
|
||||
backup_failed() { # dir why: this run's files go by name, then the (now empty) folder
|
||||
local f
|
||||
trap - EXIT
|
||||
for f in "${BACKUP_FILES[@]}"; do rm -f -- "$1/$f"; done
|
||||
rmdir -- "$1" 2>/dev/null || true
|
||||
die "backup failed ($2): nothing kept, the stack stays stopped (pnpm db:trial start)${BACKUP_WAY_OUT:+
|
||||
$($BACKUP_WAY_OUT '')}"
|
||||
}
|
||||
# An exit that is not backup_failed's (Ctrl+C, a failing command): the folder stays, unfinished.
|
||||
BACKUP_RUNNING=''
|
||||
# Cleared on success and by backup_failed, so a run of the trap is always a failure (its $? is not
|
||||
# reliable: a SIGTERM leaves 0 in it).
|
||||
backup_interrupted() {
|
||||
echo "db-trial: backup $(basename "$BACKUP_RUNNING") is incomplete (no SHA256SUMS: status lists it as incomplete, restore refuses it); the stack may be stopped (pnpm db:trial start)" >&2
|
||||
}
|
||||
# Cold copy of every trial volume that exists, with the state files present (the volumes are the
|
||||
# data, the files only follow): the stack stops, the volumes are measured and free space checked,
|
||||
# each volume is archived read-only by a --rm helper with no network and its listing must hold
|
||||
# as many entries as the volume, then the checksums are written last
|
||||
# (a folder without SHA256SUMS never finished). The stack restarts if it ran and the caller asks.
|
||||
backup_now() { # label restart(yes|no) → LAST_BACKUP
|
||||
local dir name vol kib entries i free need=0 vols=() counts=() was_running=no
|
||||
for name in "${CONTAINERS[@]}"; do if container_running "$name"; then was_running=yes; fi; done
|
||||
for vol in "$VOL_DB" "$VOL_STORAGE"; do if volume_exists "$vol"; then vols+=("$vol"); fi; done
|
||||
[ "${#vols[@]}" -gt 0 ] || die "no trial volume to back up"
|
||||
# A helper left over would collide with every archive run below, after the stack stopped.
|
||||
if container_exists "$HELPER"; then
|
||||
die "a $HELPER container is left over (docker rm $HELPER): refused, nothing was changed"
|
||||
fi
|
||||
dir="$(new_backup_dir "$1")"
|
||||
BACKUP_RUNNING="$dir"
|
||||
trap backup_interrupted EXIT
|
||||
stop_containers
|
||||
for vol in "${vols[@]}"; do
|
||||
read -r kib entries <<<"$(volume_measure "$vol")"
|
||||
if ! [[ "$kib" =~ ^[0-9]+$ && "$entries" =~ ^[0-9]+$ ]]; then backup_failed "$dir" "measure of $vol"; fi
|
||||
need=$((need + kib))
|
||||
counts+=("$entries")
|
||||
done
|
||||
# The backups share the disk of Docker's data root (DEV, the sandbox): never fill it.
|
||||
free="$(df -Pk "$dir" | awk 'NR == 2 { print $4 }')"
|
||||
if ! [[ "$free" =~ ^[0-9]+$ ]] || [ "$free" -lt $((need * 11 / 10)) ]; then
|
||||
backup_failed "$dir" "less free space under $BACKUPS than 110 % of the volumes"
|
||||
fi
|
||||
for i in "${!vols[@]}"; do
|
||||
backup_volume "$dir" "${vols[$i]}" "${counts[$i]}" || backup_failed "$dir" "archive of ${vols[$i]} unreadable or incomplete"
|
||||
done
|
||||
for name in secrets.env secrets.next.env stack.conf migrations.sha256; do
|
||||
if [ -f "$STATE/$name" ]; then cp -p "$STATE/$name" "$dir/" || backup_failed "$dir" "copy of $name"; fi
|
||||
done
|
||||
( cd "$dir" && umask 077 && sha256sum -- * > SHA256SUMS.new && mv SHA256SUMS.new SHA256SUMS ) \
|
||||
|| backup_failed "$dir" "checksums"
|
||||
LAST_BACKUP="$dir"
|
||||
trap - EXIT
|
||||
if [ "$2" = yes ] && [ "$was_running" = yes ]; then start_containers; fi
|
||||
say "backup $(basename "$dir") ($(du -sh "$dir" | cut -f1)) in $BACKUPS"
|
||||
}
|
||||
cmd_backup() {
|
||||
[ $# -eq 0 ] || die "backup takes no argument"
|
||||
require_state
|
||||
take_lock
|
||||
require_created
|
||||
backup_now manual yes
|
||||
}
|
||||
|
||||
secret_digest() { grep -E '^TR_(PW|JWT_SECRET)=' "$1" | sha256sum | cut -d' ' -f1; }
|
||||
# The caller says what became of the stack ("nothing was changed", "the stack stays stopped"):
|
||||
# these checks also run after a backup was taken, when that sentence differs.
|
||||
refuse_backup() { # dir reason [outcome]
|
||||
die "backup $(basename "$1") $2${3:+: $3}"
|
||||
}
|
||||
# What a restore may trust of a folder: exactly the files its checksums cover, all regular, all
|
||||
# matching, every archive readable. Run again right before the extraction (the folder is a
|
||||
# user-writable place: time of check against time of use).
|
||||
verify_backup_integrity() { # dir [outcome]
|
||||
local dir="$1" outcome="${2:-}" present listed archive
|
||||
if [ -L "$BACKUPS" ] || [ -L "$dir" ]; then refuse_backup "$dir" "is reached through a symbolic link" "$outcome"; fi
|
||||
[ -f "$dir/SHA256SUMS" ] || refuse_backup "$dir" "is incomplete (no SHA256SUMS)" "$outcome"
|
||||
[ -z "$(find "$dir" -mindepth 1 ! -type f)" ] \
|
||||
|| refuse_backup "$dir" "holds a symbolic link or an entry that is not a regular file" "$outcome"
|
||||
present="$(find "$dir" -mindepth 1 ! -name SHA256SUMS -printf '%f\n' | LC_ALL=C sort)"
|
||||
listed="$(sed 's/^[0-9a-f]\{64\} //' "$dir/SHA256SUMS" | LC_ALL=C sort)"
|
||||
[ "$present" = "$listed" ] \
|
||||
|| refuse_backup "$dir" "has a SHA256SUMS that does not list exactly the files of its folder" "$outcome"
|
||||
( cd "$dir" && sha256sum --quiet --strict -c SHA256SUMS ) || refuse_backup "$dir" "fails its checksums" "$outcome"
|
||||
for archive in "$dir"/*.tar; do
|
||||
tar -tf "$archive" >/dev/null 2>&1 || refuse_backup "$dir" "holds an unreadable archive ($(basename "$archive"))" "$outcome"
|
||||
done
|
||||
}
|
||||
first_absent_image() { # the recorded IMAGE_* (parse_conf ran), the first one not here; nothing when all are
|
||||
local image
|
||||
for image in "$IMAGE_DB" "$IMAGE_AUTH" "$IMAGE_REST" "$IMAGE_STORAGE" "$IMAGE_MAIL" "$IMAGE_GATEWAY"; do
|
||||
if ! docker image inspect -f '{{.Id}}' "$image" >/dev/null 2>&1; then printf '%s' "$image"; return 0; fi
|
||||
done
|
||||
}
|
||||
# Restorable: the folder is intact, both archives, the secrets and stack.conf are there and
|
||||
# parse, the keys are signed with the backup's own secret and the six images it recorded are here.
|
||||
check_backup() { # dir [outcome]
|
||||
local dir="$1" outcome="${2:-}" file image
|
||||
verify_backup_integrity "$dir" "$outcome"
|
||||
for file in db.tar storage.tar secrets.env stack.conf; do
|
||||
[ -f "$dir/$file" ] || refuse_backup "$dir" "lacks $file" "$outcome"
|
||||
done
|
||||
( parse_secrets "$dir/secrets.env" && parse_conf "$dir/stack.conf" ) 2>/dev/null \
|
||||
|| refuse_backup "$dir" "holds incomplete secrets or stack.conf" "$outcome"
|
||||
# Its keys must be signed with its own secret (expiry aside: rotate-keys renews them after).
|
||||
( parse_secrets "$dir/secrets.env" && key_check "$TR_ANON_KEY" anon any "$TR_JWT_SECRET" \
|
||||
&& key_check "$TR_SERVICE_KEY" service_role any "$TR_JWT_SECRET" ) >/dev/null 2>&1 \
|
||||
|| refuse_backup "$dir" "holds keys that are not signed with its own secret" "$outcome"
|
||||
image="$(parse_conf "$dir/stack.conf"; first_absent_image)"
|
||||
[ -z "$image" ] || refuse_backup "$dir" "records an image that is not present locally ($image: docker pull it first)" "$outcome"
|
||||
}
|
||||
|
||||
# The pre-restore backup, when one was taken: the way back if the restore stops part-way.
|
||||
PRE_RESTORE=''
|
||||
RESTORE_TARGET=''
|
||||
restore_stopped() { # cleared on success: a run of the trap is always a failure
|
||||
echo "db-trial: restore stopped part-way: containers and volumes may be missing or half-filled." >&2
|
||||
if [ -n "$PRE_RESTORE" ]; then
|
||||
echo "The state before it is backup $(basename "$PRE_RESTORE"): pnpm db:trial restore $(basename "$PRE_RESTORE")" >&2
|
||||
else
|
||||
echo "No backup of the state before it was taken (there was no trial volume)." >&2
|
||||
fi
|
||||
echo "The same restore can be rerun: pnpm db:trial restore $RESTORE_TARGET" >&2
|
||||
}
|
||||
# A restore must stay possible over a damaged stack: the pre-restore backup only has to be
|
||||
# intact (it is the way back); when a restore would refuse it as it is, the owner is told.
|
||||
check_pre_restore_backup() {
|
||||
local note
|
||||
verify_backup_integrity "$LAST_BACKUP" "nothing was removed, the stack stays stopped (pnpm db:trial start)"
|
||||
if ! note="$(check_backup "$LAST_BACKUP" 2>&1)"; then
|
||||
echo "note: the backup before this restore keeps both volumes, but restore will refuse it as it is (${note#db-trial: })"
|
||||
fi
|
||||
if [ -f "$LAST_BACKUP/secrets.next.env" ]; then
|
||||
echo "note: that backup also holds a leftover secrets.next.env (a rotate-secrets stopped part-way): restoring it brings the file back"
|
||||
fi
|
||||
}
|
||||
|
||||
# Rebuilds the stack from <backup>: volumes and secrets replaced, containers recreated from the
|
||||
# images the backup recorded. Works on a destroyed stack too (backups survive destroy).
|
||||
cmd_restore() {
|
||||
local name="${1:-}" dir vol f
|
||||
[ $# -eq 1 ] && [[ "$name" =~ ^[0-9]{8}T[0-9]{6}Z-[a-z0-9-]+$ ]] \
|
||||
|| die "restore <backup>: the name of a folder in $BACKUPS (pnpm db:trial status lists them)"
|
||||
require_state
|
||||
take_lock
|
||||
dir="$BACKUPS/$name"
|
||||
[ ! -L "$BACKUPS" ] || die "$BACKUPS is a symbolic link: refused, nothing was changed"
|
||||
[ ! -L "$dir" ] || die "backup $name is a symbolic link: refused, nothing was changed"
|
||||
[ -d "$dir" ] || die "no backup $name in $BACKUPS"
|
||||
check_backup "$dir" "nothing was changed"
|
||||
refuse_while_app_runs
|
||||
# Without a trial volume backup_now's own helper check does not run: refused here, before the confirmation.
|
||||
if container_exists "$HELPER"; then
|
||||
die "a $HELPER container is left over (docker rm $HELPER): refused, nothing was changed"
|
||||
fi
|
||||
if [ -f "$CONF" ]; then parse_conf "$CONF" || true; fi
|
||||
if [ -f "$SECRETS" ] && [ "$(secret_digest "$SECRETS")" != "$(secret_digest "$dir/secrets.env")" ]; then
|
||||
echo "note: backup $name holds other secrets than the current ones (taken before a rotate-secrets?): the restore brings them back"
|
||||
fi
|
||||
echo "restore replaces the trial stack's data and secrets by backup $name. Present now:"
|
||||
existing_resources | sed 's/^/ /'
|
||||
confirm "restore $name"
|
||||
if volume_exists "$VOL_DB" || volume_exists "$VOL_STORAGE"; then
|
||||
backup_now pre-restore no
|
||||
PRE_RESTORE="$LAST_BACKUP"
|
||||
check_pre_restore_backup
|
||||
else
|
||||
echo "no trial volume: nothing to back up before the restore"
|
||||
fi
|
||||
RESTORE_TARGET="$name"
|
||||
stop_containers
|
||||
# Armed once nothing can refuse any more: a refusal above touched no container or volume.
|
||||
ports_free "${CONTAINERS[@]}"
|
||||
trap restore_stopped EXIT
|
||||
remove_containers
|
||||
parse_conf "$dir/stack.conf"
|
||||
verify_backup_integrity "$dir"
|
||||
for vol in "$VOL_DB" "$VOL_STORAGE"; do
|
||||
if volume_exists "$vol"; then docker volume rm "$vol" >/dev/null; fi
|
||||
docker volume create --label faunosteo.stack=trial "$vol" >/dev/null
|
||||
docker run --rm -i --name "$HELPER" --network none -v "$vol:/volume" --entrypoint tar "$IMAGE_DB" \
|
||||
--numeric-owner -xpf - -C /volume < "$dir/${vol#faunosteo_trial_}.tar"
|
||||
done
|
||||
remove_live_files
|
||||
for f in secrets.env secrets.next.env stack.conf migrations.sha256; do
|
||||
if [ -f "$dir/$f" ]; then write_private "$STATE/$f" < "$dir/$f"; fi
|
||||
done
|
||||
load_conf
|
||||
load_secrets
|
||||
start_containers
|
||||
trap - EXIT
|
||||
say "restored $(basename "$dir")${PRE_RESTORE:+; the state before it: $(basename "$PRE_RESTORE")}"
|
||||
}
|
||||
|
||||
# What destroy tells the owner when it refuses for want of a restorable backup: the backup kept
|
||||
# (or none), the removals the owner may run by hand for what exists now, then the state files.
|
||||
# The script never runs these itself.
|
||||
destroy_way_out_line() { # kind command
|
||||
local names
|
||||
names="$(existing_resources | sed -n "s/^$1 //p" | tr '\n' ' ')"
|
||||
if [ -n "$names" ]; then echo " $2 ${names% }"; fi
|
||||
}
|
||||
destroy_way_out() { # kept backup folder, or empty
|
||||
if [ -n "$1" ]; then echo "backup $(basename "$1") is kept."; else echo "no backup could be made."; fi
|
||||
echo "Nothing was removed. To remove the stack yourself, run by hand:"
|
||||
destroy_way_out_line container 'docker rm'
|
||||
destroy_way_out_line network 'docker network rm'
|
||||
destroy_way_out_line volume 'docker volume rm'
|
||||
echo "then pnpm db:trial destroy clears the state files (backups kept)."
|
||||
}
|
||||
DESTROY_BACKUP=''
|
||||
destroy_stopped() { # cleared on success: a run of the trap is always a failure
|
||||
echo "db-trial: destroy stopped part-way: containers, the network or volumes may remain (pnpm db:trial status)." >&2
|
||||
echo "Its backup $(basename "$DESTROY_BACKUP") is kept: pnpm db:trial restore $(basename "$DESTROY_BACKUP") rebuilds the stack; pnpm db:trial destroy can be rerun." >&2
|
||||
}
|
||||
cmd_destroy() {
|
||||
[ $# -eq 0 ] || die "destroy takes no argument"
|
||||
local found vol
|
||||
require_state
|
||||
take_lock
|
||||
refuse_while_app_runs
|
||||
if [ -f "$CONF" ]; then parse_conf "$CONF" || true; fi
|
||||
found="$(existing_resources)"
|
||||
if [ -z "$found" ]; then
|
||||
echo "no faunosteo_trial_ resource exists: destroy removes the live state files of $STATE (backups kept)"
|
||||
confirm faunosteo_trial
|
||||
remove_live_files
|
||||
say "live state files removed; backups kept in $BACKUPS"
|
||||
return 0
|
||||
fi
|
||||
echo "destroy removes:"
|
||||
printf '%s\n' "$found" | sed 's/^/ /'
|
||||
if container_running "$DB" && ( parse_secrets "$SECRETS" ) 2>/dev/null; then
|
||||
load_secrets
|
||||
echo "content:"
|
||||
# A damaged database must not stop the destroy that gets rid of it.
|
||||
{ inventory_counts || echo "(content unreadable)"; } | sed 's/^/ /'
|
||||
fi
|
||||
echo "kept: every backup in $BACKUPS, including the one taken now"
|
||||
confirm faunosteo_trial
|
||||
if volume_exists "$VOL_DB" || volume_exists "$VOL_STORAGE"; then
|
||||
BACKUP_WAY_OUT=destroy_way_out
|
||||
backup_now pre-destroy no
|
||||
BACKUP_WAY_OUT=''
|
||||
check_backup "$LAST_BACKUP" "the stack stays stopped (pnpm db:trial start)
|
||||
$(destroy_way_out "$LAST_BACKUP")"
|
||||
DESTROY_BACKUP="$LAST_BACKUP"
|
||||
trap destroy_stopped EXIT
|
||||
fi
|
||||
stop_containers
|
||||
remove_containers
|
||||
if network_exists "$NET"; then docker network rm "$NET" >/dev/null; fi
|
||||
for vol in "$VOL_DB" "$VOL_STORAGE"; do if volume_exists "$vol"; then docker volume rm "$vol" >/dev/null; fi; done
|
||||
remove_live_files
|
||||
trap - EXIT
|
||||
say "trial stack destroyed; backups kept in $BACKUPS"
|
||||
}
|
||||
|
||||
migrate_stopped() { # cleared on success: a run of the trap is always a failure
|
||||
echo "db-trial: migrate stopped part-way: pnpm db:trial restore $(basename "$LAST_BACKUP") returns to the state before it" >&2
|
||||
}
|
||||
cmd_migrate() {
|
||||
[ $# -eq 0 ] || die "migrate takes no argument"
|
||||
local pending
|
||||
require_state
|
||||
take_lock
|
||||
require_running
|
||||
published_exactly "$DB"
|
||||
check_manifest
|
||||
pending="$(pending_migrations)"
|
||||
[ -n "$pending" ] || { say "no new migration"; return 0; }
|
||||
echo "new migrations:"
|
||||
printf '%s\n' "$pending" | sed 's/^/ /'
|
||||
backup_now pre-migrate yes
|
||||
check_backup "$LAST_BACKUP" "no migration was applied; the backup is kept and the stack runs"
|
||||
trap migrate_stopped EXIT
|
||||
migrate_up
|
||||
write_manifest
|
||||
sb_psql -c "-- db-trial:reload
|
||||
notify pgrst, 'reload schema'" >/dev/null
|
||||
trap - EXIT
|
||||
say "migrated; the backup before it: $(basename "$LAST_BACKUP")"
|
||||
}
|
||||
|
||||
keys_stopped() { # cleared on success: a run of the trap is always a failure
|
||||
echo "db-trial: rotate-keys stopped part-way; secrets.env holds the new keys; pnpm db:trial start recreates a missing container and restarts the gateway" >&2
|
||||
}
|
||||
# A backup first (secrets.env included), then new keys signed with the same secret: the expiry moves one year on. The previous keys stay valid
|
||||
# until their own expiry (rotate-secrets revokes them). Storage holds the keys in its env; the
|
||||
# gateway restarts after it (nginx resolved the old container's address).
|
||||
cmd_rotate_keys() {
|
||||
[ $# -eq 0 ] || die "rotate-keys takes no argument"
|
||||
local was_running=no
|
||||
require_state
|
||||
take_lock
|
||||
# A created stack is enough, running or stopped: the way out of expired keys must never need
|
||||
# the keys to be valid.
|
||||
require_created
|
||||
refuse_while_app_runs
|
||||
# Keys that are not this stack's: refused before the backup stops anything.
|
||||
require_trial_keys
|
||||
if container_running "$DB"; then was_running=yes; fi
|
||||
# The same cold, read-back backup as rotate-secrets, before any change, and proven restorable
|
||||
# (checksums, archives, keys signed with its secret, image) before any key changes.
|
||||
backup_now pre-keys yes
|
||||
check_backup "$LAST_BACKUP" "no key was changed; the backup is kept"
|
||||
if ! ( umask 077
|
||||
printf 'TR_PW=%s\nTR_JWT_SECRET=%s\n' "$TR_PW" "$TR_JWT_SECRET" > "$SECRETS.new"
|
||||
TR_JWT_SECRET="$TR_JWT_SECRET" node -e "$KEYGEN" keys >> "$SECRETS.new" ) \
|
||||
|| ! ( parse_secrets "$SECRETS.new" ); then
|
||||
rm -f "$SECRETS.new"
|
||||
die "key generation failed: nothing was changed"
|
||||
fi
|
||||
mv -f "$SECRETS.new" "$SECRETS"
|
||||
# Armed once secrets.env holds the new keys: before it, only the backup and a .new file exist.
|
||||
trap keys_stopped EXIT
|
||||
load_secrets
|
||||
write_env_files
|
||||
if container_running "$STORAGE"; then docker stop -t 30 "$STORAGE" >/dev/null; fi
|
||||
if container_exists "$STORAGE"; then docker rm "$STORAGE" >/dev/null; fi
|
||||
if [ "$was_running" = yes ]; then
|
||||
run_service "$STORAGE"
|
||||
restart_gateway
|
||||
check_loopback
|
||||
fi
|
||||
trap - EXIT
|
||||
say "anon and service keys renewed, valid until $(key_expiry); the app takes the new anon key at its next start"
|
||||
if [ "$was_running" = no ]; then echo "the stack is stopped: pnpm db:trial start recreates Storage with the new keys"; fi
|
||||
}
|
||||
|
||||
# The branch comes from the state: secrets.next.env is gone once the rename happened, even when the
|
||||
# signal fell before ROTATION_COMMITTED was set. The trap is armed only once that file exists.
|
||||
rotation_stopped() { # cleared on success: a run of the trap is always a failure
|
||||
if [ "$ROTATION_COMMITTED" = yes ] || [ ! -e "$STATE/secrets.next.env" ]; then
|
||||
echo "db-trial: rotate-secrets stopped after the database took the new secrets; secrets.env holds them." >&2
|
||||
echo "pnpm db:trial start recreates the missing containers from it." >&2
|
||||
else
|
||||
echo "db-trial: rotate-secrets stopped before secrets.env took the new secrets (the database may or may not have them); secrets.env (previous) and secrets.next.env (new) are both kept." >&2
|
||||
echo "pnpm db:trial restore $(basename "$LAST_BACKUP") returns to the state before it." >&2
|
||||
fi
|
||||
}
|
||||
ROTATION_COMMITTED=no
|
||||
# The rotation's own database session name, and how long settle_rotation waits for it to end.
|
||||
ROTATION_APP=db-trial-rotate
|
||||
ROTATION_WAIT_POLLS=10
|
||||
# One transaction run as supabase_admin; values through psql variables read from the exec
|
||||
# environment (never argv, never the SQL text); statement logging off, errors included. The
|
||||
# transaction is explicit (begin ... commit), not psql --single-transaction: that flag would
|
||||
# COMMIT a script cut short at end of input, whereas an input cut before `commit;` leaves the
|
||||
# server to roll the open transaction back. The same transaction records the digest of the new secrets file in a database setting: the only
|
||||
# trustworthy proof of the commit (password probes over 127.0.0.1 are `trust` in this image's
|
||||
# pg_hba.conf and succeed whatever the password).
|
||||
rotate_in_database() { # new password, new JWT secret, digest of the new secrets file
|
||||
PGPASSWORD="$TR_PW" PGAPPNAME="$ROTATION_APP" NEW_PW="$1" NEW_JWT="$2" ROT_DIGEST="$3" \
|
||||
docker exec -i -e PGPASSWORD -e PGAPPNAME -e NEW_PW -e NEW_JWT -e ROT_DIGEST "$DB" \
|
||||
psql -X -q -h 127.0.0.1 -U supabase_admin -d postgres -v ON_ERROR_STOP=1 >/dev/null <<'SQL'
|
||||
-- db-trial:rotate
|
||||
begin;
|
||||
set local log_statement = 'none';
|
||||
set local log_min_duration_statement = -1;
|
||||
set local log_min_error_statement = panic;
|
||||
set local pgaudit.log = 'none';
|
||||
\set newpw `printf '%s' "$NEW_PW"`
|
||||
\set newjwt `printf '%s' "$NEW_JWT"`
|
||||
\set digest `printf '%s' "$ROT_DIGEST"`
|
||||
alter role supabase_admin with password :'newpw';
|
||||
alter role postgres with password :'newpw';
|
||||
alter role authenticator with password :'newpw';
|
||||
alter role supabase_auth_admin with password :'newpw';
|
||||
alter role supabase_storage_admin with password :'newpw';
|
||||
alter database postgres set "app.settings.jwt_secret" to :'newjwt';
|
||||
alter database postgres set "app.trial_rotation" to :'digest';
|
||||
commit;
|
||||
SQL
|
||||
}
|
||||
# The setting the rotation transaction wrote, read in a fresh session (a database setting applies
|
||||
# to new sessions only). Empty when no rotation committed.
|
||||
rotation_marker() {
|
||||
sb_psql -t -A -c "-- db-trial:rotation-marker
|
||||
select coalesce(current_setting('app.trial_rotation', true), '')"
|
||||
}
|
||||
# Sessions still named as the rotation's: a transaction whose client is gone stays open until the
|
||||
# server notices, and may still commit or roll back.
|
||||
rotation_sessions() {
|
||||
sb_psql -t -A -c "-- db-trial:rotation-active
|
||||
select count(*) from pg_stat_activity where application_name = '$ROTATION_APP'"
|
||||
}
|
||||
# Waits (a poll a second, at most ROTATION_WAIT_POLLS) for the rotation session to end: 0 once it
|
||||
# is gone, 1 when it is still there or the database cannot say.
|
||||
wait_rotation_session_gone() {
|
||||
local poll count
|
||||
for ((poll = 1; poll <= ROTATION_WAIT_POLLS; poll++)); do
|
||||
count="$(rotation_sessions 2>/dev/null)" || return 1
|
||||
[[ "$count" =~ ^[0-9]+$ ]] || return 1
|
||||
[ "$count" -eq 0 ] && return 0
|
||||
if [ "$poll" -lt "$ROTATION_WAIT_POLLS" ]; then sleep 1; fi
|
||||
done
|
||||
return 1
|
||||
}
|
||||
rotation_unknown() { # why
|
||||
trap - EXIT
|
||||
die "$1: whether the rotation committed is unknown; secrets.env (previous) and secrets.next.env (new) are both kept; pnpm db:trial restore $(basename "$LAST_BACKUP") returns to the state before it"
|
||||
}
|
||||
# psql failed: did the transaction commit anyway (a dropped exec stream, a daemon hiccup)? Once
|
||||
# no session of the rotation is left, the marker decides; the new values are dropped only when
|
||||
# the database answers without it.
|
||||
settle_rotation() { # next file, digest of it
|
||||
local marker
|
||||
wait_rotation_session_gone \
|
||||
|| rotation_unknown "a rotation session is still active after $ROTATION_WAIT_POLLS checks, or the database does not answer"
|
||||
marker="$(rotation_marker 2>/dev/null)" || rotation_unknown "the database does not answer"
|
||||
if [ "$marker" = "$2" ]; then
|
||||
say "the database took the new secrets although psql failed: the rotation goes on"
|
||||
return 0
|
||||
fi
|
||||
trap - EXIT
|
||||
rm -f "$1"
|
||||
die "the database did not take the new secrets: nothing was changed (backup $(basename "$LAST_BACKUP"))"
|
||||
}
|
||||
# New JWT secret, keys and role passwords; account passwords and TOTP factors untouched. The
|
||||
# new values sit in secrets.next.env (600) before the database changes, so they exist somewhere
|
||||
# whatever happens next (Ctrl+C included: it exits through the EXIT trap, which keeps that file).
|
||||
cmd_rotate_secrets() {
|
||||
[ $# -eq 0 ] || die "rotate-secrets takes no argument"
|
||||
local next="$STATE/secrets.next.env" new_pw new_jwt digest
|
||||
require_state
|
||||
take_lock
|
||||
require_running
|
||||
# A previous rotation stopped part-way: that file may be the only copy of the secrets the
|
||||
# database holds, so it is never overwritten.
|
||||
[ ! -e "$next" ] || die "$next exists: a previous rotate-secrets left it, and it may hold the only copy of the secrets the database took. Never overwritten: pnpm db:trial restore <pre-rotate backup> (pnpm db:trial status lists the backups) is the way back; nothing was changed"
|
||||
refuse_while_app_runs
|
||||
# Keys that are not this stack's: refused before the backup stops anything.
|
||||
require_trial_keys
|
||||
backup_now pre-rotate yes
|
||||
# The backup is the only copy of the previous secrets (no secrets.prev): proven restorable first.
|
||||
check_backup "$LAST_BACKUP" "no secret was changed; the backup is kept"
|
||||
new_secrets_file "$next" || true
|
||||
if ! ( parse_secrets "$next" ); then rm -f "$next"; die "secret generation failed: nothing was changed"; fi
|
||||
trap rotation_stopped EXIT
|
||||
new_pw="$(sed -n 's/^TR_PW=//p' "$next")"
|
||||
new_jwt="$(sed -n 's/^TR_JWT_SECRET=//p' "$next")"
|
||||
digest="$(sha256sum < "$next" | cut -d' ' -f1)"
|
||||
if ! rotate_in_database "$new_pw" "$new_jwt" "$digest"; then settle_rotation "$next" "$digest"; fi
|
||||
# Committed means secrets.env holds the new values: set once the rename is done.
|
||||
mv -f "$next" "$SECRETS"
|
||||
ROTATION_COMMITTED=yes
|
||||
load_secrets
|
||||
# The containers carry the previous values in their env: removed at once, recreated by
|
||||
# start_containers (which checks every port before any start).
|
||||
stop_containers
|
||||
remove_containers
|
||||
start_containers
|
||||
trap - EXIT
|
||||
say "secrets renewed: sessions signed with the old secret are refused (sign in again); account passwords and TOTP factors unchanged; backup before it: $(basename "$LAST_BACKUP")"
|
||||
}
|
||||
|
||||
# Every command's arguments are checked before any Docker call (a refusal here logs nothing).
|
||||
# The commands keep their own checks too, so each stays safe if called on its own.
|
||||
check_arguments() { # action args...
|
||||
@@ -1052,6 +1556,11 @@ check_arguments() { # action args...
|
||||
case "$action" in
|
||||
plan | create | start | stop | status)
|
||||
[ $# -eq 0 ] || die "$action takes no argument" ;;
|
||||
migrate | backup | rotate-keys | rotate-secrets | destroy)
|
||||
[ $# -eq 0 ] || die "$action takes no argument" ;;
|
||||
restore)
|
||||
[ $# -eq 1 ] && [[ "$1" =~ ^[0-9]{8}T[0-9]{6}Z-[a-z0-9-]+$ ]] \
|
||||
|| die "restore <backup>: the name of a folder in $BACKUPS (pnpm db:trial status lists them)" ;;
|
||||
*) usage; exit 1 ;;
|
||||
esac
|
||||
}
|
||||
@@ -1076,5 +1585,11 @@ case "$action" in
|
||||
start) cmd_start "$@" ;;
|
||||
stop) cmd_stop "$@" ;;
|
||||
status) cmd_status "$@" ;;
|
||||
migrate) cmd_migrate "$@" ;;
|
||||
backup) cmd_backup "$@" ;;
|
||||
restore) cmd_restore "$@" ;;
|
||||
rotate-keys) cmd_rotate_keys "$@" ;;
|
||||
rotate-secrets) cmd_rotate_secrets "$@" ;;
|
||||
destroy) cmd_destroy "$@" ;;
|
||||
*) usage; exit 1 ;;
|
||||
esac
|
||||
Reference in new issue
Block a user