From b85614aa09d5ccc157da4c3c34550b5b52a647be Mon Sep 17 00:00:00 2001 From: bchanot Date: Tue, 10 Nov 2026 19:08:33 +0100 Subject: [PATCH] feat(supabase): db:trial backup, restore, destroy, migrate, rotations Cold read-only backups proven before any change (checksums covering the folder, entry counts, keys, images); typed confirmations at a terminal only; rotate-secrets in one explicit transaction with a commit marker, no secrets.prev; destroy refuses without a restorable backup and names the manual way out. Harness: 210 stub-only cases. --- .../tasks/plans/2026-11-09-db-trial-stack.md | 19 + scripts/db-trial-check.sh | 463 +++++++++++++++- scripts/db-trial.sh | 523 +++++++++++++++++- 3 files changed, 994 insertions(+), 11 deletions(-) diff --git a/.claude/tasks/plans/2026-11-09-db-trial-stack.md b/.claude/tasks/plans/2026-11-09-db-trial-stack.md index 1b5646d9..9ba17f74 100644 --- a/.claude/tasks/plans/2026-11-09-db-trial-stack.md +++ b/.claude/tasks/plans/2026-11-09-db-trial-stack.md @@ -3016,6 +3016,25 @@ fresh_log; trial_tty faunosteo_trial destroy check "destroy with no resource: after the phrase, the live files go, backups and marker kept, no Docker removal" bash -c '[ "$1" -eq 0 ] && [ ! -e "$2/secrets.env" ] && [ ! -e "$2/db.env" ] && [ -d "$2/backups" ] && [ -e "$2/.faunosteo-trial-state" ] && ! grep -qE "^docker (rm|volume rm|network rm) " "$3"' _ "$RC" "$STATE_DIR" "$CASE/log" ``` +#### Lot B2 review fixes (2026-11-10) + +- Backup proof: `verify_backup_integrity` (no symlinked `backups/` or folder, no non-regular entry, SHA256SUMS lists exactly the folder's files, checksums, every tar lists) and `check_backup` (integrity, the four required files, parse, keys signed by its own secret, all six recorded images present). `refuse_backup` takes the caller's outcome sentence, so a check after a backup never says "nothing was changed". +- Uses: restore (`check_backup` before the confirm, `verify_backup_integrity` again right before the extraction loop); migrate, rotate-keys, rotate-secrets and destroy (`check_backup` right after their backup, before any change or removal; destroy stays stopped and names the kept backup); restore's pre-restore backup (`check_pre_restore_backup`: integrity mandatory, a note, no stop, when `check_backup` would refuse it). +- Restore: `restore_stopped` EXIT trap (set after the pre-restore backup, cleared at the end) names the pre-restore backup and the rerun, or says none was taken; state files come back through `write_private` (600). +- Restore refuses a symlinked `backups/` or backup folder; `new_backup_dir` refuses a symlinked `backups/`. +- rotate-secrets: `ROTATION_COMMITTED=yes` only after `mv -f` of secrets.next.env; the "not committed" message now says secrets.env did not take the new secrets (true in the commit-to-rename window). A leftover secrets.next.env is refused before the key check and any backup; `status_leftover_rotation` prints one line, never the content. +- Rotation SQL: explicit `begin;` ... `commit;` (no `--single-transaction`, which would commit a truncated script), session named `PGAPPNAME=db-trial-rotate`; `settle_rotation` first waits (`wait_rotation_session_gone`, `ROTATION_WAIT_POLLS` = 10, one poll a second) for that session to end, else `rotation_unknown`. +- Lock first: backup, migrate, rotate-keys, rotate-secrets call `require_state; take_lock` before `require_created` / `require_running`. +- Destroy: the content inventory cannot stop it (`(content unreadable)`). backup_now: refuses a leftover helper container before any stop; `backup_interrupted` EXIT trap (cleared on success and by `backup_failed`) names the incomplete folder; its guard on `$?` is dropped because a SIGTERM leaves 0 in it. +- Harness H5 "B2 review fixes": one case per fix plus the spec gaps; stub hooks `image-nondb`, `extract`, `term-in-archive`, `append-on-rm`, `answers/measure-kib`, `answers/.seq`; `bin-mvfail` PATH for the commit-to-rename window. Two earlier assertions changed: the psql argv now carries `-e PGAPPNAME`; the stop-then-helper-then-start order matches the helper `docker run` line (the leftover-helper inspect now logs the name earlier). +- Destroy dead end: when its pre-destroy backup cannot be made (`backup_failed` appends `BACKUP_WAY_OUT`, set by destroy only) or fails `check_backup`, destroy still refuses and removes nothing, and `destroy_way_out` ends the message with the backup kept (or none), the exact `docker rm` / `docker network rm` / `docker volume rm` lines for what exists, then `pnpm db:trial destroy`; the script never runs them. `create_stopped`'s hint says so. +- `rotation_stopped` picks its branch from the state (secrets.next.env gone means the rename happened), and its trap is armed only after secrets.next.env exists and parses; `settle_rotation` clears the trap before removing the file. `keys_stopped` is armed after the `mv -f` of secrets.env. +- Restore: `restore_stopped` armed after `ports_free` (a refusal there prints no half-filled message); a leftover helper container is refused before the confirmation. +- `destroy_stopped` EXIT trap, armed once the pre-destroy backup passed `check_backup`, cleared on success: names that backup and what may remain. +- A leftover secrets.next.env is in `BACKUP_FILES`, copied by `backup_now`, brought back by restore (`write_private`, 600; status flags it), noted by `check_pre_restore_backup`; the exact-listing rule of `verify_backup_integrity` already accepts it (it compares the folder to its own SHA256SUMS). +- `backup_interrupted` says "the stack may be stopped". `check_backup` keeps `2>/dev/null` on the two parses: `parse_secrets` and `parse_conf` print nothing, so there is no reason to show. +- Harness: one case per fix (stub PATH `bin-mvterm` for a SIGTERM on a rename). Changed assertions: the trial-port restore refusal asserts no "half-filled" message instead of the pre-restore backup name; the backup-interruption case asserts the new wording. + --- diff --git a/scripts/db-trial-check.sh b/scripts/db-trial-check.sh index e9c5b806..5ec923b3 100755 --- a/scripts/db-trial-check.sh +++ b/scripts/db-trial-check.sh @@ -41,7 +41,7 @@ case "$WORK" in "$TMP_ROOT"/db-trial-check.?*) [ -d "$WORK" ] || exit 1 ;; *) echo "db-trial-check: unexpected temporary folder '$WORK': nothing was run" >&2; exit 1 ;; esac -cleanup() { [ -z "${LISTENER:-}" ] || kill "$LISTENER" 2>/dev/null; ( cd "$WORK" && rm -rf -- ./repo ./bin ./bin-no-ss ./path-no-ss ./cases ./docker-config ); rmdir -- "$WORK" 2>/dev/null || true; } +cleanup() { [ -z "${LISTENER:-}" ] || kill "$LISTENER" 2>/dev/null; ( cd "$WORK" && rm -rf -- ./repo ./bin ./bin-no-ss ./bin-mvfail ./bin-mvterm ./path-no-ss ./cases ./docker-config ); rmdir -- "$WORK" 2>/dev/null || true; } trap cleanup EXIT PASS=0 FAIL=0 pass() { echo "PASS $1"; PASS=$((PASS + 1)); } @@ -57,16 +57,39 @@ cp -R "$REPO/supabase/migrations" "$WORK/repo/supabase/migrations" for f in equine_draft.sql document.json sheet-ref.json assets.json; do cp "$REPO/$SEED_REL/$f" "$WORK/repo/$SEED_REL/"; done cp "$REPO/$SEED_REL/assets/"*.svg "$WORK/repo/$SEED_REL/assets/" +# SIGTERM for the outermost process running db-trial.sh: the stub that calls this walks its own parent +# chain, so nothing outside its ancestry is ever signalled. +cat > "$WORK/term-trial.sh" <<'STUB' +#!/usr/bin/env bash +target='' p=$PPID +while [ "$p" -gt 1 ]; do + if tr '\0' ' ' < "/proc/$p/cmdline" | grep -q '^bash scripts/db-trial\.sh '; then target=$p; fi + p="$(sed 's/^.*) [A-Za-z] \([0-9]*\) .*/\1/' "/proc/$p/stat")" +done +[ -z "$target" ] || kill -TERM "$target" +sleep 0.5 +STUB + cat > "$WORK/bin/docker" <<'STUB' #!/usr/bin/env bash # docker stand-in: one log line per call (newlines shown as \n), one file per pretend resource. # Files under $STUB_STATE/fail/ make one answer fail: daemon, networks-create, networks-create-after # (the network is made, then the call fails), inspect-error (every inspect fails with an answer other -# than "no such" or "not found"), helper, archive, . An absent resource answers in Docker's own wording per kind. +# than "no such" or "not found"), helper, archive, extract (the restore's tar -x), image-nondb (every +# image but the database's is absent), . Other hooks: term-in-archive (the script gets a +# SIGTERM while it archives), append-on-rm (the file named inside gets a byte when a container is +# removed), term-on (an ERE: the first call whose argv line, or tag: for an exec, matches it +# sends the script a SIGTERM), answers/measure-kib (the volumes' measured size), answers/.seq (one line per call). +# An absent resource answers in Docker's own wording per kind. set -uo pipefail st="$STUB_STATE" { printf 'docker'; for a in "$@"; do printf ' %s' "${a//$'\n'/\\n}"; done; printf '\n'; } >> "$STUB_LOG" save_stdin() { local f; f="$st/stdin/$(date +%s%N)"; cat > "$f"; printf '%s' "$f"; } +term_if() { # text: once, when it matches the term-on pattern + [ -f "$st/term-on" ] && grep -qEf "$st/term-on" <<<"$1" || return 0 + rm -f "$st/term-on"; bash "$STUB_TERM" +} +term_if "docker $*" sub="${1:-}" [ $# -eq 0 ] || shift case "$sub" in @@ -76,7 +99,10 @@ case "$sub" in [ ! -f "$st/fail/context" ] || exit 1 if [ -f "$st/answers/context" ]; then cat "$st/answers/context"; else echo unix:///stub/docker.sock; fi ;; ps) exit 0 ;; - image) [ "${1:-}" = inspect ] || exit 97; [ ! -f "$st/fail/image" ] || exit 1 ;; + image) + [ "${1:-}" = inspect ] || exit 97 + [ ! -f "$st/fail/image" ] || exit 1 + if [ -f "$st/fail/image-nondb" ]; then case "${*: -1}" in *supabase/postgres*) ;; *) exit 1 ;; esac; fi ;; network | volume) kind="${sub}s" op="${1:-}" shift @@ -129,11 +155,14 @@ case "$sub" in done if [ "$remove" -eq 1 ]; then if [ "$name" = faunosteo_trial_helper ] && [ -f "$st/fail/helper" ]; then exit 1; fi - if [ "$input" -eq 1 ]; then save_stdin >/dev/null; fi + if [ "$input" -eq 1 ]; then [ ! -f "$st/fail/extract" ] || exit 1; save_stdin >/dev/null; fi # Measure: 100 KiB and 2 entries ("." and "./content"), what the archive below lists. - if printf '%s\n' "$@" | grep -q -- 'du -sk /volume'; then printf '100\n2\n'; fi + if printf '%s\n' "$@" | grep -q -- 'du -sk /volume'; then + printf '%s\n2\n' "$(cat "$st/answers/measure-kib" 2>/dev/null || echo 100)" + fi if printf '%s\n' "$@" | grep -qx -- -cf; then [ ! -f "$st/fail/archive" ] || exit 3 + if [ -f "$st/term-in-archive" ]; then bash "$STUB_TERM"; fi # A real (tiny) tar, so that the script's read-back has something to list. short-archive: # the root entry only, exit 0 (an archive cut at a member boundary). mkdir -p "$st/archive"; printf '%s\n' "$*" > "$st/archive/content" @@ -160,6 +189,7 @@ case "$sub" in for n in "$@"; do [ -f "$st/containers/$n" ] || exit 1; echo exited > "$st/containers/$n"; done ;; rm) if [ "${1:-}" = -f ]; then shift; fi + if [ -f "$st/append-on-rm" ]; then printf x >> "$(cat "$st/append-on-rm")"; fi for n in "$@"; do [ -f "$st/containers/$n" ] || exit 1; rm -f "$st/containers/$n" "$st/ports/$n"; done ;; port) [ -f "$st/ports/$1" ] || exit 1 @@ -176,13 +206,16 @@ case "$sub" in for a in "$@"; do if [ "$prev" = -c ]; then query="$a"; has_c=1; fi; prev="$a"; done if [ "$input" -eq 1 ] && [ "$has_c" -eq 0 ]; then query="$(head -c 65536 "$(save_stdin)")"; fi tag="$(printf '%s\n' "$query" | sed -n 's/^-- db-trial:\([a-z-]*\)$/\1/p' | head -n 1)" + term_if "tag:$tag" # commit-then-fail: the rotation commits (its marker becomes the digest), then the exec stream # is lost. Without it, fail/rotate leaves the marker empty: not committed. if [ "$tag" = rotate ] && [ -f "$st/commit-then-fail" ]; then printf '%s\n' "${ROT_DIGEST:-}" > "$st/answers/rotation-marker"; echo "stub: stream lost" >&2; exit 3 fi if [ -n "$tag" ] && [ -f "$st/fail/$tag" ]; then echo "stub: $tag refused" >&2; exit 3; fi - if [ -n "$tag" ] && [ -f "$st/answers/$tag" ]; then cat "$st/answers/$tag"; fi + if [ -n "$tag" ] && [ -s "$st/answers/$tag.seq" ]; then + head -n 1 "$st/answers/$tag.seq"; sed -i 1d "$st/answers/$tag.seq" + elif [ -n "$tag" ] && [ -f "$st/answers/$tag" ]; then cat "$st/answers/$tag"; fi exit 0 ;; *) echo "stub docker: unsupported $sub" >&2; exit 97 ;; esac @@ -228,6 +261,7 @@ cat > "$WORK/repo/node_modules/.bin/supabase" <<'STUB' #!/usr/bin/env bash { printf 'supabase'; for a in "$@"; do printf ' %s' "$a"; done; printf ' PGPASSWORD=%s\n' "${PGPASSWORD:+set}"; } >> "$STUB_LOG" if [ "$*" = "--workdir . migration up --db-url postgresql://postgres@127.0.0.1:58432/postgres?sslmode=disable" ]; then + if [ -f "$STUB_STATE/term-in-migrate" ]; then bash "$STUB_TERM"; fi for f in supabase/migrations/*.sql; do basename "$f" | cut -d_ -f1; done > "$STUB_STATE/answers/applied" if [ -f "$STUB_STATE/fail/migrate" ]; then echo "stub: a migration failed after the others applied" >&2; exit 3; fi exit 0 @@ -289,6 +323,30 @@ for dir in "${PATH_DIRS[@]}"; do done done +# A PATH whose first folder holds a copy of the stubs and an `mv` that refuses the one rename of +# rotate-secrets (secrets.next.env over secrets.env): the window between the commit and the rename. +mkdir -p "$WORK/bin-mvfail" +cp -p "$WORK/bin/"* "$WORK/bin-mvfail/" +cat > "$WORK/bin-mvfail/mv" <&2; exit 1 ;; esac +exec $(command -v mv) "\$@" +STUB +chmod +x "$WORK/bin-mvfail/mv" +# The same, for a SIGTERM that falls on a rename: rotate-keys' (secrets.env.new, never moved) and +# rotate-secrets' (secrets.next.env, moved, then the signal: before ROTATION_COMMITTED is set). +mkdir -p "$WORK/bin-mvterm" +cp -p "$WORK/bin/"* "$WORK/bin-mvterm/" +cat > "$WORK/bin-mvterm/mv" < "$CASE/stub/answers/equine" printf '%s\n' "$ASSET_LINE" > "$CASE/stub/answers/asset-rows" echo 1 > "$CASE/stub/answers/assets-uploaded" + echo 0 > "$CASE/stub/answers/rotation-active" echo 'granted false' > "$CASE/stub/answers/grant-catalog" EXTRA=() CASE_PATH="$WORK/bin:$BASE_PATH" @@ -340,7 +399,7 @@ run_script() { # script args... → RC, output in $CASE/out; stdin as the caller verify_stub_shadow ( cd "$WORK/repo" && env -i HOME="$CASE/home" LANG=C.UTF-8 \ DOCKER_HOST="$CASE_DOCKER_HOST" DOCKER_CONTEXT=faunosteo-check-no-such-context DOCKER_CONFIG="$WORK/docker-config" \ - STUB_LOG="$CASE/log" STUB_STATE="$CASE/stub" FAUNOSTEO_TRIAL_HOST=127.0.0.1 "${EXTRA[@]}" PATH="$CASE_PATH" \ + STUB_LOG="$CASE/log" STUB_STATE="$CASE/stub" STUB_TERM="$WORK/term-trial.sh" FAUNOSTEO_TRIAL_HOST=127.0.0.1 "${EXTRA[@]}" PATH="$CASE_PATH" \ "${LAUNCH[@]}" bash "scripts/$script" "$@" ) > "$CASE/out" 2>&1 RC=$? } @@ -364,6 +423,7 @@ changes() { out_has() { grep -qF -- "$1" "$CASE/out"; } log_has() { grep -qF -- "$1" "$CASE/log"; } refused() { [ "$RC" -ne 0 ] && [ -z "$(changes)" ] && out_has "$1"; } +refused_no_docker() { refused "$1" && [ -z "$(docker_calls)" ]; } refused_early() { [ "$RC" -ne 0 ] && [ ! -s "$CASE/log" ] && out_has "$1"; } mode_is() { [ "$(stat -c %a "$1")" = "$2" ]; } values_absent_from() { # secrets file, files... @@ -591,6 +651,395 @@ chmod 644 "$STATE_DIR/secrets.env"; fresh_log; trial status check "secrets readable by others: refused" refused "mode 600" chmod 600 "$STATE_DIR/secrets.env" +echo "-- backup" +new_case backups; trial create +check "backups: create" [ "$RC" -eq 0 ] +fresh_log; trial backup +BK="$(find "$STATE_DIR/backups" -mindepth 1 -maxdepth 1 -type d -name '*-manual' | head -n 1)" +check "backup: folder 700, files 600, checksums valid, both archives listed" bash -c '[ -n "$1" ] && [ "$(stat -c %a "$1")" = 700 ] && for f in db.tar storage.tar secrets.env stack.conf migrations.sha256 SHA256SUMS; do [ "$(stat -c %a "$1/$f")" = 600 ] || exit 1; done && cd "$1" && sha256sum --quiet -c SHA256SUMS && grep -q " db.tar$" SHA256SUMS && grep -q " storage.tar$" SHA256SUMS && [ ! -e SHA256SUMS.new ]' _ "$BK" +check "backup: both volumes measured by the helper, read-only, no network" bash -c 'grep -qF "docker run --rm --name faunosteo_trial_helper --network none -v faunosteo_trial_db:/volume:ro --entrypoint sh public.ecr.aws/supabase/postgres:17.11.0.004 -c du -sk /volume" "$1" && grep -qF "docker run --rm --name faunosteo_trial_helper --network none -v faunosteo_trial_storage:/volume:ro --entrypoint sh public.ecr.aws/supabase/postgres:17.11.0.004 -c du -sk /volume" "$1"' _ "$CASE/log" +check "backup: both volumes archived read-only by the helper, no network" bash -c 'grep -qF "docker run --rm --name faunosteo_trial_helper --network none -v faunosteo_trial_db:/volume:ro --entrypoint tar" "$1" && grep -qF "docker run --rm --name faunosteo_trial_helper --network none -v faunosteo_trial_storage:/volume:ro --entrypoint tar" "$1"' _ "$CASE/log" +check "backup: the stack stops before the copy and restarts after" bash -c 'stop=$(grep -n "^docker stop" "$1" | tail -n 1 | cut -d: -f1); helper=$(grep -n "^docker run --rm --name faunosteo_trial_helper" "$1" | head -n 1 | cut -d: -f1); start=$(grep -n "^docker start" "$1" | head -n 1 | cut -d: -f1); [ -n "$stop" ] && [ -n "$helper" ] && [ -n "$start" ] && [ "$stop" -lt "$helper" ] && [ "$helper" -lt "$start" ]' _ "$CASE/log" +check "backup: no volume removal" no_volume_removal +check "backup: no secret in calls or output" secrets_absent "$STATE_DIR/secrets.env" +COUNT="$(backup_count)"; : > "$CASE/stub/fail/archive"; fresh_log; trial backup +check "a failed archive: no folder kept, the stack left stopped, the way out named" bash -c '[ "$1" -ne 0 ] && [ "$2" -eq "$3" ] && grep -qF "backup failed" "$4" && grep -qF "pnpm db:trial start" "$4"' _ "$RC" "$COUNT" "$(backup_count)" "$CASE/out" +rm -f "$CASE/stub/fail/archive"; fresh_log; trial start +check "after a failed backup, start brings the stack back" [ "$RC" -eq 0 ] +COUNT="$(backup_count)"; : > "$CASE/stub/short-archive"; fresh_log; trial backup +check "an archive that lists fewer entries than its volume (exit 0): refused, no folder kept" bash -c '[ "$1" -ne 0 ] && [ "$2" -eq "$3" ] && grep -qF "unreadable or incomplete" "$4"' _ "$RC" "$COUNT" "$(backup_count)" "$CASE/out" +rm -f "$CASE/stub/short-archive"; fresh_log; trial start +mkdir -m 700 "$STATE_DIR/backups/20000101T000000Z-partial" +fresh_log; trial status +check "status marks a folder without SHA256SUMS incomplete" bash -c 'grep -F "20000101T000000Z-partial" "$1" | grep -qF incomplete' _ "$CASE/out" +fresh_log; trial restore 20000101T000000Z-partial +check "restore refuses an incomplete backup before any change" refused "is incomplete" +rmdir "$STATE_DIR/backups/20000101T000000Z-partial" + +echo "-- migrate" +cp -p "$CASE/stub/ports/faunosteo_trial_db" "$CASE/db.port" +echo '5432/tcp -> 0.0.0.0:58432' > "$CASE/stub/ports/faunosteo_trial_db" +fresh_log; trial migrate +check "migrate: Postgres published beyond 127.0.0.1 is refused before any backup" bash -c '[ "$1" -ne 0 ] && grep -qF "faunosteo_trial_db is not published on 127.0.0.1:58432 only" "$2" && ! grep -qF faunosteo_trial_helper "$3"' _ "$RC" "$CASE/out" "$CASE/log" +cp -p "$CASE/db.port" "$CASE/stub/ports/faunosteo_trial_db" +fresh_log; trial migrate +check "migrate with nothing new: no backup, no change" bash -c '[ "$1" -eq 0 ] && grep -qF "no new migration" "$2" && ! grep -qF faunosteo_trial_helper "$3"' _ "$RC" "$CASE/out" "$CASE/log" +FIRST="$(find "$WORK/repo/supabase/migrations" -name '*.sql' | sort | head -n 1)" +cp -p "$FIRST" "$CASE/first.sql"; printf -- '-- edited\n' >> "$FIRST" +fresh_log; trial migrate +check "an applied migration edited in place: refused before any backup" refused "was edited after it was applied" +cp -p "$CASE/first.sql" "$FIRST" +printf -- '-- harness\n' > "$WORK/repo/supabase/migrations/20000101000000_harness_old.sql" +fresh_log; trial migrate +check "a new migration older than the last applied: refused before any backup" refused "older than the last applied migration" +rm -f "$WORK/repo/supabase/migrations/20000101000000_harness_old.sql" +printf -- '-- harness\n' > "$WORK/repo/supabase/migrations/29991231000000_harness_new.sql" +fresh_log; trial migrate +check "migrate: succeeds" [ "$RC" -eq 0 ] +check "migrate: the backup precedes the migration" before 'faunosteo_trial_helper' 'supabase --workdir . migration up' +check "migrate: the new file joins migrations.sha256" grep -q '^29991231000000 ' "$STATE_DIR/migrations.sha256" +check "migrate: PostgREST schema reloaded" log_has "notify pgrst, 'reload schema'" +printf -- '-- harness\n' > "$WORK/repo/supabase/migrations/29991231000001_harness_newer.sql" +: > "$CASE/stub/fail/migrate"; fresh_log; trial migrate +check "a migrate that stops part-way names its pre-migrate backup" bash -c '[ "$1" -ne 0 ] && grep -qE "restore [0-9]{8}T[0-9]{6}Z-pre-migrate" "$2"' _ "$RC" "$CASE/out" +fresh_log; trial status +check "after it, status names the remedy" out_has "pre-migrate backup" +rm -f "$CASE/stub/fail/migrate" "$WORK/repo/supabase/migrations/29991231000001_harness_newer.sql" +( cd "$WORK/repo" && for f in supabase/migrations/*.sql; do basename "$f" | cut -d_ -f1; done ) > "$CASE/stub/answers/applied" +rm -f "$WORK/repo/supabase/migrations/29991231000000_harness_new.sql" + +echo "-- rotate-keys" +if hold_port 5194; then fresh_log; trial rotate-keys; check "rotate-keys while port 5194 is in use (the app): refused, nothing changed" refused "5194 is in use"; release_port +else fail "could not hold 127.0.0.1:5194 for the app case"; fi +cp -p "$STATE_DIR/secrets.env" "$CASE/secrets.before-keys" +fresh_log; trial rotate-keys +check "rotate-keys: password and JWT secret kept, keys renewed" bash -c '[ "$1" -eq 0 ] && [ "$(grep -E "^TR_(PW|JWT_SECRET)=" "$2")" = "$(grep -E "^TR_(PW|JWT_SECRET)=" "$3")" ] && [ "$(grep "^TR_ANON_KEY=" "$2")" != "$(grep "^TR_ANON_KEY=" "$3")" ] && [ "$(stat -c %a "$2")" = 600 ]' _ "$RC" "$STATE_DIR/secrets.env" "$CASE/secrets.before-keys" +check "rotate-keys: a backup first, then storage recreated and the gateway restarted, nothing else" bash -c 'last=$(printf "%s\n" "$1" | tail -n 4); printf "%s\n" "$last" | head -n 1 | grep -qx "docker stop -t 30 faunosteo_trial_storage" && printf "%s\n" "$last" | grep -qx "docker rm faunosteo_trial_storage" && printf "%s\n" "$last" | grep -q "^docker run -d --name faunosteo_trial_storage " && printf "%s\n" "$last" | tail -n 1 | grep -qx "docker restart -t 10 faunosteo_trial_gateway" && printf "%s\n" "$1" | head -n 1 | grep -q "^docker stop -t 60 "' _ "$(changes)" +check "rotate-keys: its backup is restorable and protected (700, 600, checksums)" bash -c 'd=$(ls -d "$1"/*-pre-keys* | head -n 1); [ -n "$d" ] && [ "$(stat -c %a "$d")" = 700 ] && for f in db.tar storage.tar secrets.env stack.conf migrations.sha256 SHA256SUMS; do [ "$(stat -c %a "$d/$f")" = 600 ] || exit 1; done && cd "$d" && sha256sum --quiet -c SHA256SUMS && cmp -s secrets.env "$2"' _ "$STATE_DIR/backups" "$CASE/secrets.before-keys" +check "rotate-keys: no secret in calls or output" secrets_absent "$STATE_DIR/secrets.env" + +echo "-- rotate-secrets" +cp -p "$STATE_DIR/secrets.env" "$CASE/secrets.before-refusal" +: > "$CASE/stub/fail/rotate"; fresh_log; trial rotate-secrets +check "rotation refused by the database: secrets unchanged, no container removed" bash -c '[ "$1" -ne 0 ] && cmp -s "$2" "$3" && ! grep -q "^docker rm " "$4" && [ ! -e "$5" ] && grep -qF "nothing was changed" "$6"' _ "$RC" "$STATE_DIR/secrets.env" "$CASE/secrets.before-refusal" "$CASE/log" "$STATE_DIR/secrets.next.env" "$CASE/out" +: > "$CASE/stub/fail/rotation-marker"; fresh_log; trial rotate-secrets +check "rotation failed and the database does not answer: both files kept, the backup named" bash -c '[ "$1" -ne 0 ] && cmp -s "$2" "$3" && [ -f "$4" ] && grep -qF "secrets.next.env (new) are both kept" "$5" && grep -qE "restore [0-9]{8}T[0-9]{6}Z-pre-rotate" "$5"' _ "$RC" "$STATE_DIR/secrets.env" "$CASE/secrets.before-refusal" "$STATE_DIR/secrets.next.env" "$CASE/out" +rm -f "$CASE/stub/fail/rotate" "$CASE/stub/fail/rotation-marker" "$STATE_DIR/secrets.next.env" +cp -p "$STATE_DIR/secrets.env" "$CASE/secrets.before-lost" +: > "$CASE/stub/commit-then-fail"; fresh_log; trial rotate-secrets +check "psql lost after the commit: the database's answer decides, the rotation completes" bash -c '[ "$1" -eq 0 ] && ! cmp -s "$2" "$3" && grep -qF "took the new secrets although psql failed" "$4" && grep -qx "docker rm faunosteo_trial_db" "$5" && [ ! -e "$6" ]' _ "$RC" "$STATE_DIR/secrets.env" "$CASE/secrets.before-lost" "$CASE/out" "$CASE/log" "$STATE_DIR/secrets.next.env" +rm -f "$CASE/stub/commit-then-fail" "$CASE/stub/answers/rotation-marker" +cp -p "$STATE_DIR/secrets.env" "$CASE/secrets.before-rotation" +fresh_log; trial rotate-secrets +check "rotate-secrets: succeeds" [ "$RC" -eq 0 ] +check "rotate-secrets: every value renewed" bash -c 'for k in TR_PW TR_JWT_SECRET TR_ANON_KEY TR_SERVICE_KEY; do [ "$(grep "^$k=" "$1")" != "$(grep "^$k=" "$2")" ] || exit 1; done' _ "$STATE_DIR/secrets.env" "$CASE/secrets.before-rotation" +check "rotate-secrets: a restorable, protected backup holding the previous secrets (no secrets.prev needed)" bash -c 'd=$(ls -d "$1"/*-pre-rotate* | tail -n 1); [ -n "$d" ] && [ "$(stat -c %a "$d")" = 700 ] && for f in db.tar storage.tar secrets.env stack.conf migrations.sha256 SHA256SUMS; do [ "$(stat -c %a "$d/$f")" = 600 ] || exit 1; done && cd "$d" && sha256sum --quiet -c SHA256SUMS && cmp -s secrets.env "$2"' _ "$STATE_DIR/backups" "$CASE/secrets.before-rotation" +check "rotate-secrets: values reach psql by environment names only" log_has 'docker exec -i -e PGPASSWORD -e PGAPPNAME -e NEW_PW -e NEW_JWT -e ROT_DIGEST faunosteo_trial_db psql -X -q -h 127.0.0.1 -U supabase_admin' +check "rotate-secrets: no rotation SQL text holds a secret, old or new" bash -c 'files=$(grep -lF -- "-- db-trial:rotate" $(find "$1" -type f)); [ -n "$files" ] && for f in $files; do grep -qF "set local log_min_error_statement = panic;" "$f" && grep -qF "app.trial_rotation" "$f" || exit 1; for s in "${@:2}"; do while IFS="=" read -r _ v; do [ -z "$v" ] || ! grep -qF -- "$v" "$f" || exit 1; done < "$s"; done; done' _ "$CASE/stub/stdin" "$STATE_DIR/secrets.env" "$CASE/secrets.before-rotation" "$CASE/secrets.before-lost" "$CASE/secrets.before-refusal" +check "rotate-secrets: six containers recreated, volumes kept" bash -c 'for n in db mail auth storage rest gateway; do grep -qx "docker rm faunosteo_trial_$n" "$1" && grep -q "^docker run -d --name faunosteo_trial_$n " "$1" || exit 1; done; ! grep -qF "docker volume rm" "$1"' _ "$CASE/log" +check "rotate-secrets: no leftover secrets.next.env, no secrets.prev.env" bash -c '[ ! -e "$1/secrets.prev.env" ] && [ ! -e "$1/secrets.next.env" ]' _ "$STATE_DIR" +check "rotate-secrets: new secrets absent from calls and output" secrets_absent "$STATE_DIR/secrets.env" +check "rotate-secrets: old secrets absent from calls and output" secrets_absent "$CASE/secrets.before-rotation" + +echo "-- rotations refuse keys that are not this stack's" +cp -p "$STATE_DIR/secrets.env" "$CASE/secrets.unforged" +forge TR_SERVICE_KEY faunosteo-trial other service_role +cp -p "$STATE_DIR/secrets.env" "$CASE/secrets.forged" +for action in rotate-keys rotate-secrets; do + COUNT="$(backup_count)"; fresh_log; trial "$action" + check "$action: keys that are not this stack's are refused before any stop, no backup taken" bash -c '[ "$1" -ne 0 ] && grep -qF "service_role key in secrets.env is not this stack" "$2" && grep -qF "pnpm db:trial restore " "$2" && cmp -s "$3" "$4" && [ -z "$5" ] && [ "$6" -eq "$7" ] && [ ! -e "$8" ]' _ "$RC" "$CASE/out" "$STATE_DIR/secrets.env" "$CASE/secrets.forged" "$(changes)" "$COUNT" "$(backup_count)" "$STATE_DIR/secrets.next.env" +done +cp -p "$CASE/secrets.unforged" "$STATE_DIR/secrets.env" + +echo "-- restore" +NAME="$(basename "$BK")" +fresh_log; trial restore ../etc +check "restore: a name that is not a backup folder is refused, before any Docker call" refused_no_docker "restore " +cp -rp "$BK" "$STATE_DIR/backups/20000101T000000Z-tampered"; printf 'x' >> "$STATE_DIR/backups/20000101T000000Z-tampered/db.tar" +fresh_log; trial_tty 'restore 20000101T000000Z-tampered' restore 20000101T000000Z-tampered +check "restore: a tampered backup is refused before any change" refused "fails its checksums" +FORGED=20000101T000000Z-forged-key +cp -rp "$BK" "$STATE_DIR/backups/$FORGED" +forge TR_SERVICE_KEY faunosteo-trial other service_role 3600 "$STATE_DIR/backups/$FORGED/secrets.env" +( cd "$STATE_DIR/backups/$FORGED" && sha256sum -- db.tar storage.tar secrets.env stack.conf migrations.sha256 > SHA256SUMS ) +fresh_log; trial_tty "restore $FORGED" restore "$FORGED" +check "restore: a backup whose key is not signed with its own secret is refused before any change" refused "holds keys that are not signed with its own secret" +: > "$CASE/stub/fail/image"; fresh_log; trial_tty "restore $NAME" restore "$NAME" +check "restore: the recorded database image is absent: refused before any change" refused "is not present locally" +rm -f "$CASE/stub/fail/image" +fresh_log; trial_tty nope restore "$NAME" +check "restore: a wrong phrase changes nothing" refused "not confirmed" +fresh_log; trial_input restore "$NAME" <<<"restore $NAME" +check "restore: the phrase piped without a terminal is refused" refused "needs a terminal" +fresh_log; trial_tty "restore $NAME" restore "$NAME" +check "restore: succeeds" [ "$RC" -eq 0 ] +check "restore: says the backup brings back older secrets" out_has "holds other secrets than the current ones" +check "restore: a backup of the current state first" before 'faunosteo_trial_db:/volume:ro --entrypoint tar' 'docker volume rm faunosteo_trial_db' +check "restore: volumes replaced then filled by the helper" before 'docker volume create --label faunosteo.stack=trial faunosteo_trial_db' 'docker run --rm -i --name faunosteo_trial_helper --network none -v faunosteo_trial_db:/volume --entrypoint tar' +check "restore: the backup's secrets are back" cmp -s "$STATE_DIR/secrets.env" "$BK/secrets.env" +check "restore: six containers recreated" bash -c 'for n in db mail auth storage rest gateway; do grep -q "^docker run -d --name faunosteo_trial_$n " "$1" || exit 1; done' _ "$CASE/log" +check "restore: only trial names" only_trial_names + +echo "-- destroy" +fresh_log; trial_tty '' destroy +check "destroy with an empty answer: nothing changed" refused "not confirmed" +fresh_log; trial_input destroy <<<"faunosteo_trial" +check "destroy with the phrase piped without a terminal: refused, nothing changed" refused "needs a terminal" +fresh_log; trial_tty faunosteo_trial destroy +check "destroy: succeeds" [ "$RC" -eq 0 ] +check "destroy: a backup before any removal" before 'faunosteo_trial_db:/volume:ro --entrypoint tar' 'docker rm faunosteo_trial_db' +check "destroy: every removal names a trial resource" bash -c 'grep -E "^docker (rm|volume rm|network rm) " "$1" | grep -qvE " faunosteo_trial_(db|mail|auth|storage|rest|gateway|net)$" && exit 1; grep -qx "docker volume rm faunosteo_trial_db" "$1" && grep -qx "docker volume rm faunosteo_trial_storage" "$1" && grep -qx "docker network rm faunosteo_trial_net" "$1"' _ "$CASE/log" +check "destroy: live state removed, marker and backups kept" bash -c 'for f in secrets.env secrets.next.env db.env auth.env storage.env rest.env stack.conf migrations.sha256; do [ ! -e "$1/$f" ] || exit 1; done; [ -e "$1/.faunosteo-trial-state" ] && ls -d "$1"/backups/*-pre-destroy* >/dev/null 2>&1 && [ -d "$2" ]' _ "$STATE_DIR" "$BK" +fresh_log; trial_tty "restore $NAME" restore "$NAME" +check "restore after destroy rebuilds the stack from the backup" bash -c '[ "$1" -eq 0 ] && grep -qF "no trial volume" "$2" && grep -qx "docker network create --label faunosteo.stack=trial faunosteo_trial_net" "$3"' _ "$RC" "$CASE/out" "$CASE/log" + +echo "-- destroy with nothing left in Docker" +new_case leftover; trial create +rm -f "$CASE/stub/containers/"* "$CASE/stub/ports/"* "$CASE/stub/volumes/"* "$CASE/stub/networks/"* +fresh_log; trial_tty faunosteo_trial destroy +check "destroy with no resource: after the phrase, the live files go, backups and marker kept, no Docker removal" bash -c '[ "$1" -eq 0 ] && [ ! -e "$2/secrets.env" ] && [ ! -e "$2/db.env" ] && [ -d "$2/backups" ] && [ -e "$2/.faunosteo-trial-state" ] && ! grep -qE "^docker (rm|volume rm|network rm) " "$3"' _ "$RC" "$STATE_DIR" "$CASE/log" + +echo "-- B2 review fixes" +# A created stack with one manual backup: RB is its folder, RN its name. +b2_stack() { # case name + new_case "$1"; trial create; trial backup + RB="$(find "$STATE_DIR/backups" -mindepth 1 -maxdepth 1 -type d -name '*-manual' | head -n 1)" + RN="$(basename "$RB")" +} +restore_refused() { # backup name, message: refused at the check, nothing removed + fresh_log; trial_tty "restore $1" restore "$1" + refused "$2" && no_volume_removal +} +stopped_part_way() { [ "$RC" -ne 0 ] && out_has "restore stopped part-way" && out_has "$1"; } +polls() { grep -c 'db-trial:rotation-active' "$CASE/log" || true; } +rotation_sql() { stdin_files | xargs -r grep -lx -- '-- db-trial:rotate' | head -n 1; } +sql_is_one_explicit_transaction() { # file: begin before every alter, commit last + local first_alter begin + begin="$(grep -n '^begin;$' "$1" | head -n 1 | cut -d: -f1)" + first_alter="$(grep -n '^alter ' "$1" | head -n 1 | cut -d: -f1)" + [ -n "$begin" ] && [ -n "$first_alter" ] && [ "$begin" -lt "$first_alter" ] && [ "$(grep -c '^begin;$' "$1")" -eq 1 ] \ + && [ "$(tail -n 1 "$1")" = 'commit;' ] +} +no_single_transaction_rotation() { ! docker_calls | grep -F 'supabase_admin' | grep -qF -- --single-transaction; } + +# -- the backup is proven before anything is removed +b2_stack b2-proof +OLD=20000101T000000Z +cp -rp "$RB" "$STATE_DIR/backups/$OLD-nolist" +( cd "$STATE_DIR/backups/$OLD-nolist" && grep -v ' secrets.env$' SHA256SUMS > S.new && cat S.new > SHA256SUMS && rm S.new ) +check "restore: a SHA256SUMS missing a required entry is refused, nothing changed" restore_refused "$OLD-nolist" "does not list exactly" +cp -rp "$RB" "$STATE_DIR/backups/$OLD-extra"; printf 'x' > "$STATE_DIR/backups/$OLD-extra/extra.txt" +check "restore: a file the SHA256SUMS does not list is refused, nothing changed" restore_refused "$OLD-extra" "does not list exactly" +ln -s "$RB" "$STATE_DIR/backups/$OLD-linked" +check "restore: a backup folder that is a symbolic link is refused, nothing changed" restore_refused "$OLD-linked" "symbolic link" +cp -rp "$RB" "$STATE_DIR/backups/$OLD-filelink" +mv "$STATE_DIR/backups/$OLD-filelink/storage.tar" "$CASE/storage.real"; ln -s "$CASE/storage.real" "$STATE_DIR/backups/$OLD-filelink/storage.tar" +check "restore: a symbolic link inside a backup (checksums still valid) is refused, nothing changed" restore_refused "$OLD-filelink" "symbolic link" +mv "$STATE_DIR/backups" "$CASE/backups.real"; ln -s "$CASE/backups.real" "$STATE_DIR/backups" +check "restore: a backups/ that is a symbolic link is refused, nothing changed" restore_refused "$RN" "symbolic link" +fresh_log; trial backup +check "backup: a backups/ that is a symbolic link is refused before any stop" refused "symbolic link" +rm -- "$STATE_DIR/backups"; mv "$CASE/backups.real" "$STATE_DIR/backups" + +# -- the folder is checked again right before the extraction +b2_stack b2-toctou +printf '%s' "$RB/db.tar" > "$CASE/stub/append-on-rm" +fresh_log; trial_tty "restore $RN" restore "$RN" +check "restore: a backup changed after the first check is refused before any volume removal" bash -c '[ "$1" -ne 0 ] && grep -qF "fails its checksums" "$2" && ! grep -qF "docker volume rm" "$3"' _ "$RC" "$CASE/out" "$CASE/log" +check "restore: that refusal names the pre-restore backup and the rerun" bash -c 'grep -qE "restore [0-9]{8}T[0-9]{6}Z-pre-restore" "$1" && grep -qF "pnpm db:trial restore $2" "$1"' _ "$CASE/out" "$RN" + +# -- check_backup after each backup of a command that goes on to change something +b2_stack b2-after +cp -p "$STATE_DIR/secrets.env" "$CASE/secrets.kept" +printf -- '-- harness\n' > "$WORK/repo/supabase/migrations/29991231000002_harness_b2.sql" +: > "$CASE/stub/fail/image-nondb" +stops_after_backup() { + [ "$RC" -ne 0 ] && out_has "is not present locally" && ! out_has "nothing was changed" && ! log_has 'docker rm ' \ + && ! log_has 'migration up' && cmp -s "$STATE_DIR/secrets.env" "$CASE/secrets.kept" +} +for action in rotate-keys rotate-secrets migrate; do + fresh_log; trial "$action" + check "$action: a backup whose recorded image is absent stops the command before any change" stops_after_backup +done +rm -f "$WORK/repo/supabase/migrations/29991231000002_harness_b2.sql" +fresh_log; trial_tty faunosteo_trial destroy +check "destroy: a backup that fails its check stops before any removal, the stack stays stopped, the backup is named" bash -c '[ "$1" -ne 0 ] && grep -qF "is not present locally" "$2" && grep -qF "stack stays stopped (pnpm db:trial start)" "$2" && grep -qE "pre-destroy" "$2" && ! grep -qE "^docker (rm|volume rm|network rm) " "$3" && [ -f "$4/secrets.env" ]' _ "$RC" "$CASE/out" "$CASE/log" "$STATE_DIR" + +# -- a restore that stops part-way says where the way back is +b2_stack b2-extract +: > "$CASE/stub/fail/extract" +fresh_log; trial_tty "restore $RN" restore "$RN" +check "restore whose extraction fails: names the pre-restore backup and the rerun" bash -c '[ "$1" -ne 0 ] && grep -qF "restore stopped part-way" "$2" && grep -qE "pnpm db:trial restore [0-9]{8}T[0-9]{6}Z-pre-restore" "$2" && grep -qF "pnpm db:trial restore $3" "$2"' _ "$RC" "$CASE/out" "$RN" +rm -f "$CASE/stub/volumes/"* "$CASE/stub/containers/"* "$CASE/stub/ports/"* +fresh_log; trial_tty "restore $RN" restore "$RN" +check "restore with no trial volume that stops part-way says no backup was taken" stopped_part_way "No backup of the state before it" + +# -- restore: a restored state file is written 600, whatever the backup's mode; the note on a pre-restore backup +b2_stack b2-mode +cp -rp "$RB" "$STATE_DIR/backups/$OLD-mode644" +chmod 644 "$STATE_DIR/backups/$OLD-mode644/secrets.env" "$STATE_DIR/backups/$OLD-mode644/stack.conf" "$STATE_DIR/backups/$OLD-mode644/migrations.sha256" +fresh_log; trial_tty "restore $OLD-mode644" restore "$OLD-mode644" +check "restore from a backup holding 644 files: the restored state files are 600" bash -c '[ "$1" -eq 0 ] && for f in secrets.env stack.conf migrations.sha256; do [ "$(stat -c %a "$2/$f")" = 600 ] || exit 1; done' _ "$RC" "$STATE_DIR" +forge TR_SERVICE_KEY faunosteo-trial other service_role +fresh_log; trial_tty "restore $RN" restore "$RN" +check "restore: a pre-restore backup that restore itself would refuse is kept and noted, the restore goes on" bash -c '[ "$1" -eq 0 ] && grep -qF "restore will refuse it as it is" "$2" && grep -qF "not signed with its own secret" "$2"' _ "$RC" "$CASE/out" + +# -- the app is running: nothing changes +b2_stack b2-app +if hold_port 5194; then + fresh_log; trial_tty "restore $RN" restore "$RN"; check "restore while port 5194 is in use: refused, nothing changed" refused "5194 is in use" + fresh_log; trial_tty faunosteo_trial destroy; check "destroy while port 5194 is in use: refused, nothing changed" refused "5194 is in use" + fresh_log; trial rotate-secrets; check "rotate-secrets while port 5194 is in use: refused, nothing changed" refused "5194 is in use" + release_port +else fail "could not hold 127.0.0.1:5194 for the app cases"; fi + +# -- restore: a trial port taken by another process, found before any volume removal +b2_stack b2-ports +if hold_port 58432; then + fresh_log; trial_tty "restore $RN" restore "$RN" + check "restore with a trial port taken: refused before any removal, no half-filled message" bash -c '[ "$1" -ne 0 ] && grep -qF "127.0.0.1:58432 is taken" "$2" && ! grep -qE "^docker (rm|volume rm) " "$3" && ! grep -qF "half-filled" "$2"' _ "$RC" "$CASE/out" "$CASE/log" + release_port +else fail "could not hold 127.0.0.1:58432 for the port case"; fi + +# -- backup: free space, a helper left over, an interruption +new_case b2-backup; trial create +echo 999999999999 > "$CASE/stub/answers/measure-kib"; COUNT="$(backup_count)"; fresh_log; trial backup +check "backup: not enough free space is refused, no folder kept, the stack stays stopped" bash -c '[ "$1" -ne 0 ] && grep -qF "less free space" "$2" && grep -qF "the stack stays stopped" "$2" && grep -q "^docker stop" "$3" && [ "$4" -eq "$5" ]' _ "$RC" "$CASE/out" "$CASE/log" "$COUNT" "$(backup_count)" +rm -f "$CASE/stub/answers/measure-kib"; trial start +: > "$CASE/stub/containers/faunosteo_trial_helper"; COUNT="$(backup_count)"; fresh_log; trial backup +check "backup: a leftover helper container is refused before any stop, the hint named, no folder" bash -c '[ "$1" -ne 0 ] && grep -qF "docker rm faunosteo_trial_helper" "$2" && ! grep -q "^docker stop" "$3" && [ "$4" -eq "$5" ]' _ "$RC" "$CASE/out" "$CASE/log" "$COUNT" "$(backup_count)" +rm -f "$CASE/stub/containers/faunosteo_trial_helper" +: > "$CASE/stub/fail/archive"; fresh_log; trial backup +check "backup: a failed archive prints one message (backup failed), not the interruption one" bash -c 'grep -qF "backup failed" "$1" && ! grep -qF "is incomplete" "$1"' _ "$CASE/out" +rm -f "$CASE/stub/fail/archive"; trial start +: > "$CASE/stub/term-in-archive"; COUNT="$(backup_count)"; fresh_log; trial backup +check "backup interrupted by SIGTERM: one message, the folder named incomplete, the stack stays stopped" bash -c '[ "$1" -ne 0 ] && [ "$(grep -c "is incomplete" "$2")" -eq 1 ] && grep -qF "the stack may be stopped (pnpm db:trial start)" "$2" && [ "$3" -eq $(( $4 + 1 )) ]' _ "$RC" "$CASE/out" "$(backup_count)" "$COUNT" +rm -f "$CASE/stub/term-in-archive"; fresh_log; trial status +check "status lists the interrupted backup as incomplete" bash -c 'grep -F "backups in" -A9 "$1" | grep -qF incomplete' _ "$CASE/out" + +# -- the lock comes before the state checks +new_case b2-lock; trial create +rm -f "$CASE/stub/volumes/faunosteo_trial_db" +flock "$STATE_DIR/.lock" sleep 6 & LOCK_HOLDER=$! +sleep 0.5 +for action in backup migrate rotate-keys rotate-secrets; do + fresh_log; trial "$action" + check "$action: the lock is taken before the stack is checked" bash -c '[ "$1" -ne 0 ] && grep -qF "another pnpm db:trial command is running" "$2"' _ "$RC" "$CASE/out" +done +wait "$LOCK_HOLDER" + +# -- a rotation that stopped part-way leaves secrets.next.env: never overwritten +b2_stack b2-next +printf 'SENTINEL-NEXT\n' > "$STATE_DIR/secrets.next.env"; chmod 600 "$STATE_DIR/secrets.next.env" +fresh_log; trial status +check "status mentions a leftover secrets.next.env, never its content" bash -c 'grep -qF "secrets.next.env exists" "$1" && ! grep -qF SENTINEL-NEXT "$1"' _ "$CASE/out" +cp -p "$STATE_DIR/secrets.env" "$CASE/secrets.unforged" +forge TR_SERVICE_KEY faunosteo-trial other service_role +COUNT="$(backup_count)"; fresh_log; trial rotate-secrets +check "rotate-secrets with a leftover secrets.next.env: refused before the key check and any backup, the file untouched" bash -c '[ "$1" -ne 0 ] && grep -qF "secrets.next.env exists" "$2" && ! grep -qF "is not this stack" "$2" && grep -qF "pnpm db:trial restore " "$2" && [ "$3" -eq "$4" ] && ! grep -q "^docker stop" "$5" && [ "$(cat "$6")" = SENTINEL-NEXT ]' _ "$RC" "$CASE/out" "$COUNT" "$(backup_count)" "$CASE/log" "$STATE_DIR/secrets.next.env" +cp -p "$CASE/secrets.unforged" "$STATE_DIR/secrets.env" +fresh_log; trial_tty "restore $RN" restore "$RN" +check "restore removes a leftover secrets.next.env once it succeeded" bash -c '[ "$1" -eq 0 ] && [ ! -e "$2/secrets.next.env" ]' _ "$RC" "$STATE_DIR" +printf 'SENTINEL-NEXT\n' > "$STATE_DIR/secrets.next.env" +fresh_log; trial_tty faunosteo_trial destroy +check "destroy removes a leftover secrets.next.env once it succeeded" bash -c '[ "$1" -eq 0 ] && [ ! -e "$2/secrets.next.env" ]' _ "$RC" "$STATE_DIR" + +# -- destroy on a damaged database +new_case b2-inventory; trial create +: > "$CASE/stub/fail/inventory"; fresh_log; trial_tty faunosteo_trial destroy +check "destroy whose inventory query fails still reaches the confirmation and works" bash -c '[ "$1" -eq 0 ] && grep -qF "(content unreadable)" "$2" && grep -qx "docker volume rm faunosteo_trial_db" "$3"' _ "$RC" "$CASE/out" "$CASE/log" + +# -- rotate-secrets: the transaction, the commit window, the rename +new_case b2-rotation; trial create +cp -p "$STATE_DIR/secrets.env" "$CASE/secrets.before" +CASE_PATH="$WORK/bin-mvfail:$BASE_PATH"; fresh_log; trial rotate-secrets +check "rotate-secrets: a rename that fails after the commit still says both files are kept" bash -c '[ "$1" -ne 0 ] && grep -qF "secrets.env (previous) and secrets.next.env (new) are both kept" "$2" && ! grep -qF "secrets.env holds them" "$2" && [ -f "$3/secrets.next.env" ] && cmp -s "$3/secrets.env" "$4"' _ "$RC" "$CASE/out" "$STATE_DIR" "$CASE/secrets.before" +CASE_PATH="$WORK/bin:$BASE_PATH"; rm -f "$STATE_DIR/secrets.next.env" +: > "$CASE/stub/fail/rotate"; echo 1 > "$CASE/stub/answers/rotation-active"; fresh_log; trial rotate-secrets +check "a rotation session still active after the limit: unknown, both files kept, nothing removed, the marker unread" bash -c '[ "$1" -ne 0 ] && grep -qF "whether the rotation committed is unknown" "$2" && grep -qF "secrets.env (previous) and secrets.next.env (new) are both kept" "$2" && grep -qE "restore [0-9]{8}T[0-9]{6}Z-pre-rotate" "$2" && [ -f "$3/secrets.next.env" ] && cmp -s "$3/secrets.env" "$4" && ! grep -q "^docker rm " "$5" && ! grep -qF "db-trial:rotation-marker" "$5" && [ "$6" -eq 10 ]' _ "$RC" "$CASE/out" "$STATE_DIR" "$CASE/secrets.before" "$CASE/log" "$(polls)" +rm -f "$STATE_DIR/secrets.next.env"; echo 0 > "$CASE/stub/answers/rotation-active"; : > "$CASE/stub/fail/rotation-active"; fresh_log; trial rotate-secrets +check "a failing session query: unknown, both files kept, the marker unread" bash -c '[ "$1" -ne 0 ] && grep -qF "whether the rotation committed is unknown" "$2" && [ -f "$3/secrets.next.env" ] && ! grep -qF "db-trial:rotation-marker" "$4"' _ "$RC" "$CASE/out" "$STATE_DIR" "$CASE/log" +rm -f "$STATE_DIR/secrets.next.env" "$CASE/stub/fail/rotation-active"; printf '1\n1\n0\n' > "$CASE/stub/answers/rotation-active.seq"; fresh_log; trial rotate-secrets +check "a rotation session active for two polls then gone, marker absent: not committed, the new file removed" bash -c '[ "$1" -ne 0 ] && grep -qF "did not take the new secrets" "$2" && [ ! -e "$3/secrets.next.env" ] && cmp -s "$3/secrets.env" "$4" && [ "$5" -eq 3 ]' _ "$RC" "$CASE/out" "$STATE_DIR" "$CASE/secrets.before" "$(polls)" +rm -f "$CASE/stub/fail/rotate"; : > "$CASE/stub/commit-then-fail"; printf '1\n0\n' > "$CASE/stub/answers/rotation-active.seq"; fresh_log; trial rotate-secrets +check "a rotation session active once then gone, marker equal: the rotation goes on" bash -c '[ "$1" -eq 0 ] && grep -qF "took the new secrets although psql failed" "$2" && [ "$3" -eq 2 ]' _ "$RC" "$CASE/out" "$(polls)" +rm -f "$CASE/stub/commit-then-fail" "$CASE/stub/answers/rotation-marker"; fresh_log; trial rotate-secrets +check "rotate-secrets: the SQL is one explicit transaction (begin before every alter, commit last)" sql_is_one_explicit_transaction "$(rotation_sql)" +check "rotate-secrets: the rotation session is named (PGAPPNAME passed by name)" bash -c '[ "$1" -eq 0 ] && grep -qF -- "-e PGAPPNAME" "$2"' _ "$RC" "$CASE/log" +check "rotate-secrets: no --single-transaction on the rotation session" no_single_transaction_rotation + +# -- destroy refuses without a restorable backup, and names the way out +no_removal() { ! grep -qE '^docker (rm|volume rm|network rm) ' "$CASE/log"; } +new_case h5-way-one-volume; trial create +rm -f "$CASE/stub/volumes/faunosteo_trial_storage" +fresh_log; trial_tty faunosteo_trial destroy +check "destroy with one volume missing: refused, nothing removed, the way out lists what exists" bash -c '[ "$1" -ne 0 ] && grep -qF "lacks storage.tar" "$2" && grep -qF "docker rm faunosteo_trial_db faunosteo_trial_mail" "$2" && grep -qF " docker network rm faunosteo_trial_net" "$2" && grep -qF " docker volume rm faunosteo_trial_db" "$2" && grep -qF "pnpm db:trial destroy" "$2" && grep -qF "pre-destroy" "$2"' _ "$RC" "$CASE/out" +check "destroy with one volume missing: no removal was run" no_removal +for n in db mail auth storage rest gateway; do rm -f "$CASE/stub/containers/faunosteo_trial_$n" "$CASE/stub/ports/faunosteo_trial_$n"; done +rm -f "$CASE/stub/networks/faunosteo_trial_net" "$CASE/stub/volumes/faunosteo_trial_db" +fresh_log; trial_tty faunosteo_trial destroy +check "destroy after the manual removal: the state files go, backups kept" bash -c '[ "$1" -eq 0 ] && [ ! -e "$2/secrets.env" ] && [ -d "$2/backups" ] && ls -d "$2"/backups/*-pre-destroy* >/dev/null 2>&1' _ "$RC" "$STATE_DIR" +new_case h5-way-space; trial create +echo 999999999999 > "$CASE/stub/answers/measure-kib" +fresh_log; trial_tty faunosteo_trial destroy +check "destroy whose backup cannot be made: one failure message, the way out, nothing removed" bash -c '[ "$1" -ne 0 ] && [ "$(grep -c "backup failed" "$2")" -eq 1 ] && grep -qF "no backup could be made" "$2" && grep -qF " docker volume rm faunosteo_trial_db faunosteo_trial_storage" "$2" && grep -qF " docker network rm faunosteo_trial_net" "$2" && grep -qF "pnpm db:trial destroy" "$2"' _ "$RC" "$CASE/out" +check "destroy whose backup cannot be made: no removal was run" no_removal +new_case h5-way-forged; trial create +forge TR_SERVICE_KEY faunosteo-trial other service_role +fresh_log; trial_tty faunosteo_trial destroy +check "destroy with a forged key: refused with the way out, the backup named" bash -c '[ "$1" -ne 0 ] && grep -qF "not signed with its own secret" "$2" && grep -qE "backup [0-9]{8}T[0-9]{6}Z-pre-destroy[a-z0-9-]* is kept" "$2" && grep -qF " docker network rm faunosteo_trial_net" "$2" && grep -qF "pnpm db:trial destroy" "$2"' _ "$RC" "$CASE/out" +check "destroy with a forged key: no removal was run" no_removal + +# -- destroy interrupted mid-removal names its backup +new_case h5-destroy-term; trial create +echo '^docker volume rm ' > "$CASE/stub/term-on"; fresh_log; trial_tty faunosteo_trial destroy +check "destroy terminated by SIGTERM mid-removal: one message naming the pre-destroy backup" bash -c '[ "$(grep -c "destroy stopped part-way" "$1")" -eq 1 ] && grep -qE "restore [0-9]{8}T[0-9]{6}Z-pre-destroy" "$1"' _ "$CASE/out" + +# -- the rotations' traps follow the state +new_case h5-rotation-window; trial create +CASE_PATH="$WORK/bin-mvterm:$BASE_PATH"; fresh_log; trial rotate-secrets +check "rotate-secrets terminated between the rename and the flag: the committed branch, from the state" bash -c '[ "$1" -ne 0 ] && grep -qF "stopped after the database took the new secrets" "$2" && ! grep -qF "stopped before" "$2" && [ ! -e "$3/secrets.next.env" ]' _ "$RC" "$CASE/out" "$STATE_DIR" +new_case h5-keys-before-rename; trial create +cp -p "$STATE_DIR/secrets.env" "$CASE/secrets.before" +CASE_PATH="$WORK/bin-mvterm:$BASE_PATH"; fresh_log; trial rotate-keys +check "rotate-keys terminated before the rename: no 'holds the new keys' message, secrets.env unchanged" bash -c '[ "$1" -ne 0 ] && ! grep -qF "holds the new keys" "$2" && cmp -s "$3" "$4"' _ "$RC" "$CASE/out" "$STATE_DIR/secrets.env" "$CASE/secrets.before" + +# -- restore: a leftover helper without a trial volume is refused before the confirmation +b2_stack h5-restore-helper +trial_tty faunosteo_trial destroy +: > "$CASE/stub/containers/faunosteo_trial_helper" +fresh_log; trial_tty "restore $RN" restore "$RN" +check "restore with a leftover helper and no trial volume: refused before the confirmation, the hint named" bash -c '[ "$1" -ne 0 ] && grep -qF "docker rm faunosteo_trial_helper" "$2" && ! grep -qF "to go on" "$2" && ! grep -qE "^docker (run|volume|rm|network)" "$3"' _ "$RC" "$CASE/out" "$CASE/log" + +# -- a leftover secrets.next.env is kept by backups and brought back by restore +new_case h5-next-backup; trial create +printf 'SENTINEL-NEXT\n' > "$STATE_DIR/secrets.next.env"; chmod 600 "$STATE_DIR/secrets.next.env" +trial backup +RB="$(find "$STATE_DIR/backups" -mindepth 1 -maxdepth 1 -type d -name '*-manual' | head -n 1)"; RN="$(basename "$RB")" +check "backup with a leftover secrets.next.env holds it, listed in SHA256SUMS" bash -c '[ "$(cat "$1/secrets.next.env")" = SENTINEL-NEXT ] && grep -qF " secrets.next.env" "$1/SHA256SUMS"' _ "$RB" +rm -f "$STATE_DIR/secrets.next.env" +fresh_log; trial_tty "restore $RN" restore "$RN" +check "restore of that backup brings secrets.next.env back (600) and status flags it" bash -c '[ "$1" -eq 0 ] && [ "$(cat "$2/secrets.next.env")" = SENTINEL-NEXT ] && [ "$(stat -c %a "$2/secrets.next.env")" = 600 ]' _ "$RC" "$STATE_DIR" +fresh_log; trial status +check "status flags the restored secrets.next.env" out_has "secrets.next.env exists" +fresh_log; trial_tty "restore $RN" restore "$RN" +check "restore with a leftover secrets.next.env: the pre-restore backup note mentions it" bash -c '[ "$1" -eq 0 ] && grep -qF "also holds a leftover secrets.next.env" "$2"' _ "$RC" "$CASE/out" + +echo "-- SIGTERM after the trap is set: one message" +new_case term-create; echo '^docker volume create .* faunosteo_trial_db$' > "$CASE/stub/term-on"; fresh_log; trial create +check "create terminated by SIGTERM: one message, nothing removed" bash -c '[ "$(grep -c "create stopped before the end" "$1")" -eq 1 ]' _ "$CASE/out" +new_case term-migrate; trial create +printf -- '-- harness\n' > "$WORK/repo/supabase/migrations/29991231000000_harness_term.sql" +: > "$CASE/stub/term-in-migrate"; fresh_log; trial migrate +rm -f "$WORK/repo/supabase/migrations/29991231000000_harness_term.sql" +check "migrate terminated by SIGTERM: one message naming the pre-migrate backup" bash -c '[ "$(grep -c "migrate stopped part-way" "$1")" -eq 1 ] && grep -qE "restore [0-9]{8}T[0-9]{6}Z-pre-migrate" "$1"' _ "$CASE/out" +new_case term-keys; trial create +echo '^docker run -d --name faunosteo_trial_storage ' > "$CASE/stub/term-on"; fresh_log; trial rotate-keys +check "rotate-keys terminated by SIGTERM: one message" bash -c '[ "$(grep -c "rotate-keys stopped part-way" "$1")" -eq 1 ]' _ "$CASE/out" +new_case term-rotation; trial create +echo '^tag:rotate$' > "$CASE/stub/term-on"; fresh_log; trial rotate-secrets +check "rotate-secrets terminated by SIGTERM: one message, the uncommitted branch" bash -c '[ "$(grep -c "rotate-secrets stopped" "$1")" -eq 1 ] && grep -qF "stopped before secrets.env took the new secrets" "$1"' _ "$CASE/out" + echo "-- db:sandbox state guard" new_case sandbox-guard; mkdir -p "$CASE/trialstate"; : > "$CASE/trialstate/.faunosteo-trial-state" EXTRA=(FAUNOSTEO_SANDBOX_DIR="$CASE/trialstate"); sandbox down diff --git a/scripts/db-trial.sh b/scripts/db-trial.sh index ca57fac1..26c14b24 100755 --- a/scripts/db-trial.sh +++ b/scripts/db-trial.sh @@ -875,9 +875,8 @@ check_helper_image() { # so a create that died after making something is never mistaken for one that made nothing: past # it, the files go only when no faunosteo_trial_ resource exists; else nothing is removed and the # way out is named. -create_stopped() { - local status=$? present - [ "$status" -ne 0 ] || return 0 +create_stopped() { # cleared on success: a run of the trap is always a failure + local present present='' if [ "$FIRST_RESOURCE" = yes ]; then present="$(existing_resources 2>/dev/null)" || present=unknown; fi if [ -z "$present" ]; then @@ -887,7 +886,7 @@ create_stopped() { fi echo "db-trial: create stopped before the end; nothing was removed. Present now:" >&2 existing_resources | sed 's/^/ /' >&2 - echo "pnpm db:trial status shows the state; pnpm db:trial destroy (after the owner's go) clears it." >&2 + echo "pnpm db:trial status shows the state; pnpm db:trial destroy (after the owner's go) backs the data up and clears it, or names the manual removal when it cannot back up." >&2 } FIRST_RESOURCE=no # Everything that can refuse comes first: no file, no Docker write yet. @@ -1031,6 +1030,12 @@ status_without_stack() { existing_resources | sed 's/^/ present anyway: /' list_backups } +# Only that it exists, never what it holds. +status_leftover_rotation() { + if [ -e "$STATE/secrets.next.env" ]; then + say "secrets.next.env exists: a rotate-secrets stopped part-way and rotate-secrets refuses until it is dealt with (pnpm db:trial restore )" + fi +} cmd_status() { [ $# -eq 0 ] || die "status takes no argument" if [ ! -e "$MARKER" ] || [ ! -f "$SECRETS" ]; then status_without_stack; return 0; fi @@ -1040,10 +1045,509 @@ cmd_status() { status_containers say "Docker endpoint: $(daemon_endpoint)" status_keys_and_host + status_leftover_rotation if container_running "$DB"; then status_database; else say "database stopped: pnpm db:trial start"; fi list_backups } +# The exact phrase, typed at a terminal: piped stdin (an agent, a script) is refused before the +# prompt; anything else typed, or nothing, stops too. +confirm() { # phrase + local typed='' + [ -t 0 ] || die "a typed confirmation needs a terminal (stdin is not one): nothing was changed" + printf 'Type "%s" to go on (anything else stops): ' "$1" >&2 + read -r typed || true + [ "$typed" = "$1" ] || die "not confirmed: nothing was changed" +} + +# Every name a backup folder may hold: a failed backup removes exactly these, then the folder. +BACKUP_FILES=(db.tar storage.tar secrets.env secrets.next.env stack.conf migrations.sha256 SHA256SUMS SHA256SUMS.new) +new_backup_dir() { # label → a fresh folder (700) + local base dir n=1 + # A link would send the archives wherever it points. + [ ! -L "$BACKUPS" ] || die "$BACKUPS is a symbolic link: refused, nothing was changed" + base="$BACKUPS/$(date -u +%Y%m%dT%H%M%SZ)-$1" + dir="$base" + while [ -e "$dir" ]; do n=$((n + 1)); dir="$base-$n"; done + ( umask 077; mkdir -p "$BACKUPS"; mkdir "$dir" ) || die "cannot create $dir" + printf '%s' "$dir" +} +# Size in KiB and entry count of a volume, by the helper (read-only, no network); the entries are +# counted from the root tar archives, so the archive must list as many. +volume_measure() { # volume → " " + docker run --rm --name "$HELPER" --network none -v "$1:/volume:ro" --entrypoint sh "$IMAGE_DB" \ + -c 'du -sk /volume | cut -f1 && cd /volume && find . ! -type s | wc -l' | tr '\n' ' ' +} +# The listing of an archive cut at a member boundary still exits 0: its entry count is compared. +backup_volume() { # dir volume entries + local file="$1/${2#faunosteo_trial_}.tar" listed + ( umask 077 + docker run --rm --name "$HELPER" --network none -v "$2:/volume:ro" --entrypoint tar "$IMAGE_DB" \ + --numeric-owner -C /volume -cf - . > "$file" ) || return 1 + listed="$(tar -tf "$file" | wc -l)" || return 1 + [ "$listed" -eq "$3" ] +} +# A caller that cannot go on without its backup (destroy) names a function that prints the way +# out; backup_failed appends it, so the one failure message stays one message. +BACKUP_WAY_OUT='' +backup_failed() { # dir why: this run's files go by name, then the (now empty) folder + local f + trap - EXIT + for f in "${BACKUP_FILES[@]}"; do rm -f -- "$1/$f"; done + rmdir -- "$1" 2>/dev/null || true + die "backup failed ($2): nothing kept, the stack stays stopped (pnpm db:trial start)${BACKUP_WAY_OUT:+ +$($BACKUP_WAY_OUT '')}" +} +# An exit that is not backup_failed's (Ctrl+C, a failing command): the folder stays, unfinished. +BACKUP_RUNNING='' +# Cleared on success and by backup_failed, so a run of the trap is always a failure (its $? is not +# reliable: a SIGTERM leaves 0 in it). +backup_interrupted() { + echo "db-trial: backup $(basename "$BACKUP_RUNNING") is incomplete (no SHA256SUMS: status lists it as incomplete, restore refuses it); the stack may be stopped (pnpm db:trial start)" >&2 +} +# Cold copy of every trial volume that exists, with the state files present (the volumes are the +# data, the files only follow): the stack stops, the volumes are measured and free space checked, +# each volume is archived read-only by a --rm helper with no network and its listing must hold +# as many entries as the volume, then the checksums are written last +# (a folder without SHA256SUMS never finished). The stack restarts if it ran and the caller asks. +backup_now() { # label restart(yes|no) → LAST_BACKUP + local dir name vol kib entries i free need=0 vols=() counts=() was_running=no + for name in "${CONTAINERS[@]}"; do if container_running "$name"; then was_running=yes; fi; done + for vol in "$VOL_DB" "$VOL_STORAGE"; do if volume_exists "$vol"; then vols+=("$vol"); fi; done + [ "${#vols[@]}" -gt 0 ] || die "no trial volume to back up" + # A helper left over would collide with every archive run below, after the stack stopped. + if container_exists "$HELPER"; then + die "a $HELPER container is left over (docker rm $HELPER): refused, nothing was changed" + fi + dir="$(new_backup_dir "$1")" + BACKUP_RUNNING="$dir" + trap backup_interrupted EXIT + stop_containers + for vol in "${vols[@]}"; do + read -r kib entries <<<"$(volume_measure "$vol")" + if ! [[ "$kib" =~ ^[0-9]+$ && "$entries" =~ ^[0-9]+$ ]]; then backup_failed "$dir" "measure of $vol"; fi + need=$((need + kib)) + counts+=("$entries") + done + # The backups share the disk of Docker's data root (DEV, the sandbox): never fill it. + free="$(df -Pk "$dir" | awk 'NR == 2 { print $4 }')" + if ! [[ "$free" =~ ^[0-9]+$ ]] || [ "$free" -lt $((need * 11 / 10)) ]; then + backup_failed "$dir" "less free space under $BACKUPS than 110 % of the volumes" + fi + for i in "${!vols[@]}"; do + backup_volume "$dir" "${vols[$i]}" "${counts[$i]}" || backup_failed "$dir" "archive of ${vols[$i]} unreadable or incomplete" + done + for name in secrets.env secrets.next.env stack.conf migrations.sha256; do + if [ -f "$STATE/$name" ]; then cp -p "$STATE/$name" "$dir/" || backup_failed "$dir" "copy of $name"; fi + done + ( cd "$dir" && umask 077 && sha256sum -- * > SHA256SUMS.new && mv SHA256SUMS.new SHA256SUMS ) \ + || backup_failed "$dir" "checksums" + LAST_BACKUP="$dir" + trap - EXIT + if [ "$2" = yes ] && [ "$was_running" = yes ]; then start_containers; fi + say "backup $(basename "$dir") ($(du -sh "$dir" | cut -f1)) in $BACKUPS" +} +cmd_backup() { + [ $# -eq 0 ] || die "backup takes no argument" + require_state + take_lock + require_created + backup_now manual yes +} + +secret_digest() { grep -E '^TR_(PW|JWT_SECRET)=' "$1" | sha256sum | cut -d' ' -f1; } +# The caller says what became of the stack ("nothing was changed", "the stack stays stopped"): +# these checks also run after a backup was taken, when that sentence differs. +refuse_backup() { # dir reason [outcome] + die "backup $(basename "$1") $2${3:+: $3}" +} +# What a restore may trust of a folder: exactly the files its checksums cover, all regular, all +# matching, every archive readable. Run again right before the extraction (the folder is a +# user-writable place: time of check against time of use). +verify_backup_integrity() { # dir [outcome] + local dir="$1" outcome="${2:-}" present listed archive + if [ -L "$BACKUPS" ] || [ -L "$dir" ]; then refuse_backup "$dir" "is reached through a symbolic link" "$outcome"; fi + [ -f "$dir/SHA256SUMS" ] || refuse_backup "$dir" "is incomplete (no SHA256SUMS)" "$outcome" + [ -z "$(find "$dir" -mindepth 1 ! -type f)" ] \ + || refuse_backup "$dir" "holds a symbolic link or an entry that is not a regular file" "$outcome" + present="$(find "$dir" -mindepth 1 ! -name SHA256SUMS -printf '%f\n' | LC_ALL=C sort)" + listed="$(sed 's/^[0-9a-f]\{64\} //' "$dir/SHA256SUMS" | LC_ALL=C sort)" + [ "$present" = "$listed" ] \ + || refuse_backup "$dir" "has a SHA256SUMS that does not list exactly the files of its folder" "$outcome" + ( cd "$dir" && sha256sum --quiet --strict -c SHA256SUMS ) || refuse_backup "$dir" "fails its checksums" "$outcome" + for archive in "$dir"/*.tar; do + tar -tf "$archive" >/dev/null 2>&1 || refuse_backup "$dir" "holds an unreadable archive ($(basename "$archive"))" "$outcome" + done +} +first_absent_image() { # the recorded IMAGE_* (parse_conf ran), the first one not here; nothing when all are + local image + for image in "$IMAGE_DB" "$IMAGE_AUTH" "$IMAGE_REST" "$IMAGE_STORAGE" "$IMAGE_MAIL" "$IMAGE_GATEWAY"; do + if ! docker image inspect -f '{{.Id}}' "$image" >/dev/null 2>&1; then printf '%s' "$image"; return 0; fi + done +} +# Restorable: the folder is intact, both archives, the secrets and stack.conf are there and +# parse, the keys are signed with the backup's own secret and the six images it recorded are here. +check_backup() { # dir [outcome] + local dir="$1" outcome="${2:-}" file image + verify_backup_integrity "$dir" "$outcome" + for file in db.tar storage.tar secrets.env stack.conf; do + [ -f "$dir/$file" ] || refuse_backup "$dir" "lacks $file" "$outcome" + done + ( parse_secrets "$dir/secrets.env" && parse_conf "$dir/stack.conf" ) 2>/dev/null \ + || refuse_backup "$dir" "holds incomplete secrets or stack.conf" "$outcome" + # Its keys must be signed with its own secret (expiry aside: rotate-keys renews them after). + ( parse_secrets "$dir/secrets.env" && key_check "$TR_ANON_KEY" anon any "$TR_JWT_SECRET" \ + && key_check "$TR_SERVICE_KEY" service_role any "$TR_JWT_SECRET" ) >/dev/null 2>&1 \ + || refuse_backup "$dir" "holds keys that are not signed with its own secret" "$outcome" + image="$(parse_conf "$dir/stack.conf"; first_absent_image)" + [ -z "$image" ] || refuse_backup "$dir" "records an image that is not present locally ($image: docker pull it first)" "$outcome" +} + +# The pre-restore backup, when one was taken: the way back if the restore stops part-way. +PRE_RESTORE='' +RESTORE_TARGET='' +restore_stopped() { # cleared on success: a run of the trap is always a failure + echo "db-trial: restore stopped part-way: containers and volumes may be missing or half-filled." >&2 + if [ -n "$PRE_RESTORE" ]; then + echo "The state before it is backup $(basename "$PRE_RESTORE"): pnpm db:trial restore $(basename "$PRE_RESTORE")" >&2 + else + echo "No backup of the state before it was taken (there was no trial volume)." >&2 + fi + echo "The same restore can be rerun: pnpm db:trial restore $RESTORE_TARGET" >&2 +} +# A restore must stay possible over a damaged stack: the pre-restore backup only has to be +# intact (it is the way back); when a restore would refuse it as it is, the owner is told. +check_pre_restore_backup() { + local note + verify_backup_integrity "$LAST_BACKUP" "nothing was removed, the stack stays stopped (pnpm db:trial start)" + if ! note="$(check_backup "$LAST_BACKUP" 2>&1)"; then + echo "note: the backup before this restore keeps both volumes, but restore will refuse it as it is (${note#db-trial: })" + fi + if [ -f "$LAST_BACKUP/secrets.next.env" ]; then + echo "note: that backup also holds a leftover secrets.next.env (a rotate-secrets stopped part-way): restoring it brings the file back" + fi +} + +# Rebuilds the stack from : volumes and secrets replaced, containers recreated from the +# images the backup recorded. Works on a destroyed stack too (backups survive destroy). +cmd_restore() { + local name="${1:-}" dir vol f + [ $# -eq 1 ] && [[ "$name" =~ ^[0-9]{8}T[0-9]{6}Z-[a-z0-9-]+$ ]] \ + || die "restore : the name of a folder in $BACKUPS (pnpm db:trial status lists them)" + require_state + take_lock + dir="$BACKUPS/$name" + [ ! -L "$BACKUPS" ] || die "$BACKUPS is a symbolic link: refused, nothing was changed" + [ ! -L "$dir" ] || die "backup $name is a symbolic link: refused, nothing was changed" + [ -d "$dir" ] || die "no backup $name in $BACKUPS" + check_backup "$dir" "nothing was changed" + refuse_while_app_runs + # Without a trial volume backup_now's own helper check does not run: refused here, before the confirmation. + if container_exists "$HELPER"; then + die "a $HELPER container is left over (docker rm $HELPER): refused, nothing was changed" + fi + if [ -f "$CONF" ]; then parse_conf "$CONF" || true; fi + if [ -f "$SECRETS" ] && [ "$(secret_digest "$SECRETS")" != "$(secret_digest "$dir/secrets.env")" ]; then + echo "note: backup $name holds other secrets than the current ones (taken before a rotate-secrets?): the restore brings them back" + fi + echo "restore replaces the trial stack's data and secrets by backup $name. Present now:" + existing_resources | sed 's/^/ /' + confirm "restore $name" + if volume_exists "$VOL_DB" || volume_exists "$VOL_STORAGE"; then + backup_now pre-restore no + PRE_RESTORE="$LAST_BACKUP" + check_pre_restore_backup + else + echo "no trial volume: nothing to back up before the restore" + fi + RESTORE_TARGET="$name" + stop_containers + # Armed once nothing can refuse any more: a refusal above touched no container or volume. + ports_free "${CONTAINERS[@]}" + trap restore_stopped EXIT + remove_containers + parse_conf "$dir/stack.conf" + verify_backup_integrity "$dir" + for vol in "$VOL_DB" "$VOL_STORAGE"; do + if volume_exists "$vol"; then docker volume rm "$vol" >/dev/null; fi + docker volume create --label faunosteo.stack=trial "$vol" >/dev/null + docker run --rm -i --name "$HELPER" --network none -v "$vol:/volume" --entrypoint tar "$IMAGE_DB" \ + --numeric-owner -xpf - -C /volume < "$dir/${vol#faunosteo_trial_}.tar" + done + remove_live_files + for f in secrets.env secrets.next.env stack.conf migrations.sha256; do + if [ -f "$dir/$f" ]; then write_private "$STATE/$f" < "$dir/$f"; fi + done + load_conf + load_secrets + start_containers + trap - EXIT + say "restored $(basename "$dir")${PRE_RESTORE:+; the state before it: $(basename "$PRE_RESTORE")}" +} + +# What destroy tells the owner when it refuses for want of a restorable backup: the backup kept +# (or none), the removals the owner may run by hand for what exists now, then the state files. +# The script never runs these itself. +destroy_way_out_line() { # kind command + local names + names="$(existing_resources | sed -n "s/^$1 //p" | tr '\n' ' ')" + if [ -n "$names" ]; then echo " $2 ${names% }"; fi +} +destroy_way_out() { # kept backup folder, or empty + if [ -n "$1" ]; then echo "backup $(basename "$1") is kept."; else echo "no backup could be made."; fi + echo "Nothing was removed. To remove the stack yourself, run by hand:" + destroy_way_out_line container 'docker rm' + destroy_way_out_line network 'docker network rm' + destroy_way_out_line volume 'docker volume rm' + echo "then pnpm db:trial destroy clears the state files (backups kept)." +} +DESTROY_BACKUP='' +destroy_stopped() { # cleared on success: a run of the trap is always a failure + echo "db-trial: destroy stopped part-way: containers, the network or volumes may remain (pnpm db:trial status)." >&2 + echo "Its backup $(basename "$DESTROY_BACKUP") is kept: pnpm db:trial restore $(basename "$DESTROY_BACKUP") rebuilds the stack; pnpm db:trial destroy can be rerun." >&2 +} +cmd_destroy() { + [ $# -eq 0 ] || die "destroy takes no argument" + local found vol + require_state + take_lock + refuse_while_app_runs + if [ -f "$CONF" ]; then parse_conf "$CONF" || true; fi + found="$(existing_resources)" + if [ -z "$found" ]; then + echo "no faunosteo_trial_ resource exists: destroy removes the live state files of $STATE (backups kept)" + confirm faunosteo_trial + remove_live_files + say "live state files removed; backups kept in $BACKUPS" + return 0 + fi + echo "destroy removes:" + printf '%s\n' "$found" | sed 's/^/ /' + if container_running "$DB" && ( parse_secrets "$SECRETS" ) 2>/dev/null; then + load_secrets + echo "content:" + # A damaged database must not stop the destroy that gets rid of it. + { inventory_counts || echo "(content unreadable)"; } | sed 's/^/ /' + fi + echo "kept: every backup in $BACKUPS, including the one taken now" + confirm faunosteo_trial + if volume_exists "$VOL_DB" || volume_exists "$VOL_STORAGE"; then + BACKUP_WAY_OUT=destroy_way_out + backup_now pre-destroy no + BACKUP_WAY_OUT='' + check_backup "$LAST_BACKUP" "the stack stays stopped (pnpm db:trial start) +$(destroy_way_out "$LAST_BACKUP")" + DESTROY_BACKUP="$LAST_BACKUP" + trap destroy_stopped EXIT + fi + stop_containers + remove_containers + if network_exists "$NET"; then docker network rm "$NET" >/dev/null; fi + for vol in "$VOL_DB" "$VOL_STORAGE"; do if volume_exists "$vol"; then docker volume rm "$vol" >/dev/null; fi; done + remove_live_files + trap - EXIT + say "trial stack destroyed; backups kept in $BACKUPS" +} + +migrate_stopped() { # cleared on success: a run of the trap is always a failure + echo "db-trial: migrate stopped part-way: pnpm db:trial restore $(basename "$LAST_BACKUP") returns to the state before it" >&2 +} +cmd_migrate() { + [ $# -eq 0 ] || die "migrate takes no argument" + local pending + require_state + take_lock + require_running + published_exactly "$DB" + check_manifest + pending="$(pending_migrations)" + [ -n "$pending" ] || { say "no new migration"; return 0; } + echo "new migrations:" + printf '%s\n' "$pending" | sed 's/^/ /' + backup_now pre-migrate yes + check_backup "$LAST_BACKUP" "no migration was applied; the backup is kept and the stack runs" + trap migrate_stopped EXIT + migrate_up + write_manifest + sb_psql -c "-- db-trial:reload +notify pgrst, 'reload schema'" >/dev/null + trap - EXIT + say "migrated; the backup before it: $(basename "$LAST_BACKUP")" +} + +keys_stopped() { # cleared on success: a run of the trap is always a failure + echo "db-trial: rotate-keys stopped part-way; secrets.env holds the new keys; pnpm db:trial start recreates a missing container and restarts the gateway" >&2 +} +# A backup first (secrets.env included), then new keys signed with the same secret: the expiry moves one year on. The previous keys stay valid +# until their own expiry (rotate-secrets revokes them). Storage holds the keys in its env; the +# gateway restarts after it (nginx resolved the old container's address). +cmd_rotate_keys() { + [ $# -eq 0 ] || die "rotate-keys takes no argument" + local was_running=no + require_state + take_lock + # A created stack is enough, running or stopped: the way out of expired keys must never need + # the keys to be valid. + require_created + refuse_while_app_runs + # Keys that are not this stack's: refused before the backup stops anything. + require_trial_keys + if container_running "$DB"; then was_running=yes; fi + # The same cold, read-back backup as rotate-secrets, before any change, and proven restorable + # (checksums, archives, keys signed with its secret, image) before any key changes. + backup_now pre-keys yes + check_backup "$LAST_BACKUP" "no key was changed; the backup is kept" + if ! ( umask 077 + printf 'TR_PW=%s\nTR_JWT_SECRET=%s\n' "$TR_PW" "$TR_JWT_SECRET" > "$SECRETS.new" + TR_JWT_SECRET="$TR_JWT_SECRET" node -e "$KEYGEN" keys >> "$SECRETS.new" ) \ + || ! ( parse_secrets "$SECRETS.new" ); then + rm -f "$SECRETS.new" + die "key generation failed: nothing was changed" + fi + mv -f "$SECRETS.new" "$SECRETS" + # Armed once secrets.env holds the new keys: before it, only the backup and a .new file exist. + trap keys_stopped EXIT + load_secrets + write_env_files + if container_running "$STORAGE"; then docker stop -t 30 "$STORAGE" >/dev/null; fi + if container_exists "$STORAGE"; then docker rm "$STORAGE" >/dev/null; fi + if [ "$was_running" = yes ]; then + run_service "$STORAGE" + restart_gateway + check_loopback + fi + trap - EXIT + say "anon and service keys renewed, valid until $(key_expiry); the app takes the new anon key at its next start" + if [ "$was_running" = no ]; then echo "the stack is stopped: pnpm db:trial start recreates Storage with the new keys"; fi +} + +# The branch comes from the state: secrets.next.env is gone once the rename happened, even when the +# signal fell before ROTATION_COMMITTED was set. The trap is armed only once that file exists. +rotation_stopped() { # cleared on success: a run of the trap is always a failure + if [ "$ROTATION_COMMITTED" = yes ] || [ ! -e "$STATE/secrets.next.env" ]; then + echo "db-trial: rotate-secrets stopped after the database took the new secrets; secrets.env holds them." >&2 + echo "pnpm db:trial start recreates the missing containers from it." >&2 + else + echo "db-trial: rotate-secrets stopped before secrets.env took the new secrets (the database may or may not have them); secrets.env (previous) and secrets.next.env (new) are both kept." >&2 + echo "pnpm db:trial restore $(basename "$LAST_BACKUP") returns to the state before it." >&2 + fi +} +ROTATION_COMMITTED=no +# The rotation's own database session name, and how long settle_rotation waits for it to end. +ROTATION_APP=db-trial-rotate +ROTATION_WAIT_POLLS=10 +# One transaction run as supabase_admin; values through psql variables read from the exec +# environment (never argv, never the SQL text); statement logging off, errors included. The +# transaction is explicit (begin ... commit), not psql --single-transaction: that flag would +# COMMIT a script cut short at end of input, whereas an input cut before `commit;` leaves the +# server to roll the open transaction back. The same transaction records the digest of the new secrets file in a database setting: the only +# trustworthy proof of the commit (password probes over 127.0.0.1 are `trust` in this image's +# pg_hba.conf and succeed whatever the password). +rotate_in_database() { # new password, new JWT secret, digest of the new secrets file + PGPASSWORD="$TR_PW" PGAPPNAME="$ROTATION_APP" NEW_PW="$1" NEW_JWT="$2" ROT_DIGEST="$3" \ + docker exec -i -e PGPASSWORD -e PGAPPNAME -e NEW_PW -e NEW_JWT -e ROT_DIGEST "$DB" \ + psql -X -q -h 127.0.0.1 -U supabase_admin -d postgres -v ON_ERROR_STOP=1 >/dev/null <<'SQL' +-- db-trial:rotate +begin; +set local log_statement = 'none'; +set local log_min_duration_statement = -1; +set local log_min_error_statement = panic; +set local pgaudit.log = 'none'; +\set newpw `printf '%s' "$NEW_PW"` +\set newjwt `printf '%s' "$NEW_JWT"` +\set digest `printf '%s' "$ROT_DIGEST"` +alter role supabase_admin with password :'newpw'; +alter role postgres with password :'newpw'; +alter role authenticator with password :'newpw'; +alter role supabase_auth_admin with password :'newpw'; +alter role supabase_storage_admin with password :'newpw'; +alter database postgres set "app.settings.jwt_secret" to :'newjwt'; +alter database postgres set "app.trial_rotation" to :'digest'; +commit; +SQL +} +# The setting the rotation transaction wrote, read in a fresh session (a database setting applies +# to new sessions only). Empty when no rotation committed. +rotation_marker() { + sb_psql -t -A -c "-- db-trial:rotation-marker +select coalesce(current_setting('app.trial_rotation', true), '')" +} +# Sessions still named as the rotation's: a transaction whose client is gone stays open until the +# server notices, and may still commit or roll back. +rotation_sessions() { + sb_psql -t -A -c "-- db-trial:rotation-active +select count(*) from pg_stat_activity where application_name = '$ROTATION_APP'" +} +# Waits (a poll a second, at most ROTATION_WAIT_POLLS) for the rotation session to end: 0 once it +# is gone, 1 when it is still there or the database cannot say. +wait_rotation_session_gone() { + local poll count + for ((poll = 1; poll <= ROTATION_WAIT_POLLS; poll++)); do + count="$(rotation_sessions 2>/dev/null)" || return 1 + [[ "$count" =~ ^[0-9]+$ ]] || return 1 + [ "$count" -eq 0 ] && return 0 + if [ "$poll" -lt "$ROTATION_WAIT_POLLS" ]; then sleep 1; fi + done + return 1 +} +rotation_unknown() { # why + trap - EXIT + die "$1: whether the rotation committed is unknown; secrets.env (previous) and secrets.next.env (new) are both kept; pnpm db:trial restore $(basename "$LAST_BACKUP") returns to the state before it" +} +# psql failed: did the transaction commit anyway (a dropped exec stream, a daemon hiccup)? Once +# no session of the rotation is left, the marker decides; the new values are dropped only when +# the database answers without it. +settle_rotation() { # next file, digest of it + local marker + wait_rotation_session_gone \ + || rotation_unknown "a rotation session is still active after $ROTATION_WAIT_POLLS checks, or the database does not answer" + marker="$(rotation_marker 2>/dev/null)" || rotation_unknown "the database does not answer" + if [ "$marker" = "$2" ]; then + say "the database took the new secrets although psql failed: the rotation goes on" + return 0 + fi + trap - EXIT + rm -f "$1" + die "the database did not take the new secrets: nothing was changed (backup $(basename "$LAST_BACKUP"))" +} +# New JWT secret, keys and role passwords; account passwords and TOTP factors untouched. The +# new values sit in secrets.next.env (600) before the database changes, so they exist somewhere +# whatever happens next (Ctrl+C included: it exits through the EXIT trap, which keeps that file). +cmd_rotate_secrets() { + [ $# -eq 0 ] || die "rotate-secrets takes no argument" + local next="$STATE/secrets.next.env" new_pw new_jwt digest + require_state + take_lock + require_running + # A previous rotation stopped part-way: that file may be the only copy of the secrets the + # database holds, so it is never overwritten. + [ ! -e "$next" ] || die "$next exists: a previous rotate-secrets left it, and it may hold the only copy of the secrets the database took. Never overwritten: pnpm db:trial restore (pnpm db:trial status lists the backups) is the way back; nothing was changed" + refuse_while_app_runs + # Keys that are not this stack's: refused before the backup stops anything. + require_trial_keys + backup_now pre-rotate yes + # The backup is the only copy of the previous secrets (no secrets.prev): proven restorable first. + check_backup "$LAST_BACKUP" "no secret was changed; the backup is kept" + new_secrets_file "$next" || true + if ! ( parse_secrets "$next" ); then rm -f "$next"; die "secret generation failed: nothing was changed"; fi + trap rotation_stopped EXIT + new_pw="$(sed -n 's/^TR_PW=//p' "$next")" + new_jwt="$(sed -n 's/^TR_JWT_SECRET=//p' "$next")" + digest="$(sha256sum < "$next" | cut -d' ' -f1)" + if ! rotate_in_database "$new_pw" "$new_jwt" "$digest"; then settle_rotation "$next" "$digest"; fi + # Committed means secrets.env holds the new values: set once the rename is done. + mv -f "$next" "$SECRETS" + ROTATION_COMMITTED=yes + load_secrets + # The containers carry the previous values in their env: removed at once, recreated by + # start_containers (which checks every port before any start). + stop_containers + remove_containers + start_containers + trap - EXIT + say "secrets renewed: sessions signed with the old secret are refused (sign in again); account passwords and TOTP factors unchanged; backup before it: $(basename "$LAST_BACKUP")" +} + # Every command's arguments are checked before any Docker call (a refusal here logs nothing). # The commands keep their own checks too, so each stays safe if called on its own. check_arguments() { # action args... @@ -1052,6 +1556,11 @@ check_arguments() { # action args... case "$action" in plan | create | start | stop | status) [ $# -eq 0 ] || die "$action takes no argument" ;; + migrate | backup | rotate-keys | rotate-secrets | destroy) + [ $# -eq 0 ] || die "$action takes no argument" ;; + restore) + [ $# -eq 1 ] && [[ "$1" =~ ^[0-9]{8}T[0-9]{6}Z-[a-z0-9-]+$ ]] \ + || die "restore : the name of a folder in $BACKUPS (pnpm db:trial status lists them)" ;; *) usage; exit 1 ;; esac } @@ -1076,5 +1585,11 @@ case "$action" in start) cmd_start "$@" ;; stop) cmd_stop "$@" ;; status) cmd_status "$@" ;; + migrate) cmd_migrate "$@" ;; + backup) cmd_backup "$@" ;; + restore) cmd_restore "$@" ;; + rotate-keys) cmd_rotate_keys "$@" ;; + rotate-secrets) cmd_rotate_secrets "$@" ;; + destroy) cmd_destroy "$@" ;; *) usage; exit 1 ;; esac