feat(supabase): db:trial backup, restore, destroy, migrate, rotations
Cold read-only backups proven before any change (checksums covering the folder, entry counts, keys, images); typed confirmations at a terminal only; rotate-secrets in one explicit transaction with a commit marker, no secrets.prev; destroy refuses without a restorable backup and names the manual way out. Harness: 210 stub-only cases.
This commit is contained in:
1 parent
1a7b952f2f
commit
b85614aa09
3 files changed
+994
-11
No files matched your search
+456
-7
@@ -41,7 +41,7 @@ case "$WORK" in
|
||||
"$TMP_ROOT"/db-trial-check.?*) [ -d "$WORK" ] || exit 1 ;;
|
||||
*) echo "db-trial-check: unexpected temporary folder '$WORK': nothing was run" >&2; exit 1 ;;
|
||||
esac
|
||||
cleanup() { [ -z "${LISTENER:-}" ] || kill "$LISTENER" 2>/dev/null; ( cd "$WORK" && rm -rf -- ./repo ./bin ./bin-no-ss ./path-no-ss ./cases ./docker-config ); rmdir -- "$WORK" 2>/dev/null || true; }
|
||||
cleanup() { [ -z "${LISTENER:-}" ] || kill "$LISTENER" 2>/dev/null; ( cd "$WORK" && rm -rf -- ./repo ./bin ./bin-no-ss ./bin-mvfail ./bin-mvterm ./path-no-ss ./cases ./docker-config ); rmdir -- "$WORK" 2>/dev/null || true; }
|
||||
trap cleanup EXIT
|
||||
PASS=0 FAIL=0
|
||||
pass() { echo "PASS $1"; PASS=$((PASS + 1)); }
|
||||
@@ -57,16 +57,39 @@ cp -R "$REPO/supabase/migrations" "$WORK/repo/supabase/migrations"
|
||||
for f in equine_draft.sql document.json sheet-ref.json assets.json; do cp "$REPO/$SEED_REL/$f" "$WORK/repo/$SEED_REL/"; done
|
||||
cp "$REPO/$SEED_REL/assets/"*.svg "$WORK/repo/$SEED_REL/assets/"
|
||||
|
||||
# SIGTERM for the outermost process running db-trial.sh: the stub that calls this walks its own parent
|
||||
# chain, so nothing outside its ancestry is ever signalled.
|
||||
cat > "$WORK/term-trial.sh" <<'STUB'
|
||||
#!/usr/bin/env bash
|
||||
target='' p=$PPID
|
||||
while [ "$p" -gt 1 ]; do
|
||||
if tr '\0' ' ' < "/proc/$p/cmdline" | grep -q '^bash scripts/db-trial\.sh '; then target=$p; fi
|
||||
p="$(sed 's/^.*) [A-Za-z] \([0-9]*\) .*/\1/' "/proc/$p/stat")"
|
||||
done
|
||||
[ -z "$target" ] || kill -TERM "$target"
|
||||
sleep 0.5
|
||||
STUB
|
||||
|
||||
cat > "$WORK/bin/docker" <<'STUB'
|
||||
#!/usr/bin/env bash
|
||||
# docker stand-in: one log line per call (newlines shown as \n), one file per pretend resource.
|
||||
# Files under $STUB_STATE/fail/ make one answer fail: daemon, networks-create, networks-create-after
|
||||
# (the network is made, then the call fails), inspect-error (every inspect fails with an answer other
|
||||
# than "no such" or "not found"), helper, archive, <sql tag>. An absent resource answers in Docker's own wording per kind.
|
||||
# than "no such" or "not found"), helper, archive, extract (the restore's tar -x), image-nondb (every
|
||||
# image but the database's is absent), <sql tag>. Other hooks: term-in-archive (the script gets a
|
||||
# SIGTERM while it archives), append-on-rm (the file named inside gets a byte when a container is
|
||||
# removed), term-on (an ERE: the first call whose argv line, or tag:<sql tag> for an exec, matches it
|
||||
# sends the script a SIGTERM), answers/measure-kib (the volumes' measured size), answers/<tag>.seq (one line per call).
|
||||
# An absent resource answers in Docker's own wording per kind.
|
||||
set -uo pipefail
|
||||
st="$STUB_STATE"
|
||||
{ printf 'docker'; for a in "$@"; do printf ' %s' "${a//$'\n'/\\n}"; done; printf '\n'; } >> "$STUB_LOG"
|
||||
save_stdin() { local f; f="$st/stdin/$(date +%s%N)"; cat > "$f"; printf '%s' "$f"; }
|
||||
term_if() { # text: once, when it matches the term-on pattern
|
||||
[ -f "$st/term-on" ] && grep -qEf "$st/term-on" <<<"$1" || return 0
|
||||
rm -f "$st/term-on"; bash "$STUB_TERM"
|
||||
}
|
||||
term_if "docker $*"
|
||||
sub="${1:-}"
|
||||
[ $# -eq 0 ] || shift
|
||||
case "$sub" in
|
||||
@@ -76,7 +99,10 @@ case "$sub" in
|
||||
[ ! -f "$st/fail/context" ] || exit 1
|
||||
if [ -f "$st/answers/context" ]; then cat "$st/answers/context"; else echo unix:///stub/docker.sock; fi ;;
|
||||
ps) exit 0 ;;
|
||||
image) [ "${1:-}" = inspect ] || exit 97; [ ! -f "$st/fail/image" ] || exit 1 ;;
|
||||
image)
|
||||
[ "${1:-}" = inspect ] || exit 97
|
||||
[ ! -f "$st/fail/image" ] || exit 1
|
||||
if [ -f "$st/fail/image-nondb" ]; then case "${*: -1}" in *supabase/postgres*) ;; *) exit 1 ;; esac; fi ;;
|
||||
network | volume)
|
||||
kind="${sub}s" op="${1:-}"
|
||||
shift
|
||||
@@ -129,11 +155,14 @@ case "$sub" in
|
||||
done
|
||||
if [ "$remove" -eq 1 ]; then
|
||||
if [ "$name" = faunosteo_trial_helper ] && [ -f "$st/fail/helper" ]; then exit 1; fi
|
||||
if [ "$input" -eq 1 ]; then save_stdin >/dev/null; fi
|
||||
if [ "$input" -eq 1 ]; then [ ! -f "$st/fail/extract" ] || exit 1; save_stdin >/dev/null; fi
|
||||
# Measure: 100 KiB and 2 entries ("." and "./content"), what the archive below lists.
|
||||
if printf '%s\n' "$@" | grep -q -- 'du -sk /volume'; then printf '100\n2\n'; fi
|
||||
if printf '%s\n' "$@" | grep -q -- 'du -sk /volume'; then
|
||||
printf '%s\n2\n' "$(cat "$st/answers/measure-kib" 2>/dev/null || echo 100)"
|
||||
fi
|
||||
if printf '%s\n' "$@" | grep -qx -- -cf; then
|
||||
[ ! -f "$st/fail/archive" ] || exit 3
|
||||
if [ -f "$st/term-in-archive" ]; then bash "$STUB_TERM"; fi
|
||||
# A real (tiny) tar, so that the script's read-back has something to list. short-archive:
|
||||
# the root entry only, exit 0 (an archive cut at a member boundary).
|
||||
mkdir -p "$st/archive"; printf '%s\n' "$*" > "$st/archive/content"
|
||||
@@ -160,6 +189,7 @@ case "$sub" in
|
||||
for n in "$@"; do [ -f "$st/containers/$n" ] || exit 1; echo exited > "$st/containers/$n"; done ;;
|
||||
rm)
|
||||
if [ "${1:-}" = -f ]; then shift; fi
|
||||
if [ -f "$st/append-on-rm" ]; then printf x >> "$(cat "$st/append-on-rm")"; fi
|
||||
for n in "$@"; do [ -f "$st/containers/$n" ] || exit 1; rm -f "$st/containers/$n" "$st/ports/$n"; done ;;
|
||||
port)
|
||||
[ -f "$st/ports/$1" ] || exit 1
|
||||
@@ -176,13 +206,16 @@ case "$sub" in
|
||||
for a in "$@"; do if [ "$prev" = -c ]; then query="$a"; has_c=1; fi; prev="$a"; done
|
||||
if [ "$input" -eq 1 ] && [ "$has_c" -eq 0 ]; then query="$(head -c 65536 "$(save_stdin)")"; fi
|
||||
tag="$(printf '%s\n' "$query" | sed -n 's/^-- db-trial:\([a-z-]*\)$/\1/p' | head -n 1)"
|
||||
term_if "tag:$tag"
|
||||
# commit-then-fail: the rotation commits (its marker becomes the digest), then the exec stream
|
||||
# is lost. Without it, fail/rotate leaves the marker empty: not committed.
|
||||
if [ "$tag" = rotate ] && [ -f "$st/commit-then-fail" ]; then
|
||||
printf '%s\n' "${ROT_DIGEST:-}" > "$st/answers/rotation-marker"; echo "stub: stream lost" >&2; exit 3
|
||||
fi
|
||||
if [ -n "$tag" ] && [ -f "$st/fail/$tag" ]; then echo "stub: $tag refused" >&2; exit 3; fi
|
||||
if [ -n "$tag" ] && [ -f "$st/answers/$tag" ]; then cat "$st/answers/$tag"; fi
|
||||
if [ -n "$tag" ] && [ -s "$st/answers/$tag.seq" ]; then
|
||||
head -n 1 "$st/answers/$tag.seq"; sed -i 1d "$st/answers/$tag.seq"
|
||||
elif [ -n "$tag" ] && [ -f "$st/answers/$tag" ]; then cat "$st/answers/$tag"; fi
|
||||
exit 0 ;;
|
||||
*) echo "stub docker: unsupported $sub" >&2; exit 97 ;;
|
||||
esac
|
||||
@@ -228,6 +261,7 @@ cat > "$WORK/repo/node_modules/.bin/supabase" <<'STUB'
|
||||
#!/usr/bin/env bash
|
||||
{ printf 'supabase'; for a in "$@"; do printf ' %s' "$a"; done; printf ' PGPASSWORD=%s\n' "${PGPASSWORD:+set}"; } >> "$STUB_LOG"
|
||||
if [ "$*" = "--workdir . migration up --db-url postgresql://postgres@127.0.0.1:58432/postgres?sslmode=disable" ]; then
|
||||
if [ -f "$STUB_STATE/term-in-migrate" ]; then bash "$STUB_TERM"; fi
|
||||
for f in supabase/migrations/*.sql; do basename "$f" | cut -d_ -f1; done > "$STUB_STATE/answers/applied"
|
||||
if [ -f "$STUB_STATE/fail/migrate" ]; then echo "stub: a migration failed after the others applied" >&2; exit 3; fi
|
||||
exit 0
|
||||
@@ -289,6 +323,30 @@ for dir in "${PATH_DIRS[@]}"; do
|
||||
done
|
||||
done
|
||||
|
||||
# A PATH whose first folder holds a copy of the stubs and an `mv` that refuses the one rename of
|
||||
# rotate-secrets (secrets.next.env over secrets.env): the window between the commit and the rename.
|
||||
mkdir -p "$WORK/bin-mvfail"
|
||||
cp -p "$WORK/bin/"* "$WORK/bin-mvfail/"
|
||||
cat > "$WORK/bin-mvfail/mv" <<STUB
|
||||
#!/usr/bin/env bash
|
||||
case "\$*" in *secrets.next.env*secrets.env) echo "stub mv: refused" >&2; exit 1 ;; esac
|
||||
exec $(command -v mv) "\$@"
|
||||
STUB
|
||||
chmod +x "$WORK/bin-mvfail/mv"
|
||||
# The same, for a SIGTERM that falls on a rename: rotate-keys' (secrets.env.new, never moved) and
|
||||
# rotate-secrets' (secrets.next.env, moved, then the signal: before ROTATION_COMMITTED is set).
|
||||
mkdir -p "$WORK/bin-mvterm"
|
||||
cp -p "$WORK/bin/"* "$WORK/bin-mvterm/"
|
||||
cat > "$WORK/bin-mvterm/mv" <<STUB
|
||||
#!/usr/bin/env bash
|
||||
case "\$*" in
|
||||
*secrets.env.new*) bash "\$STUB_TERM"; exit 0 ;;
|
||||
*secrets.next.env*secrets.env) $(command -v mv) "\$@"; bash "\$STUB_TERM"; exit 0 ;;
|
||||
esac
|
||||
exec $(command -v mv) "\$@"
|
||||
STUB
|
||||
chmod +x "$WORK/bin-mvterm/mv"
|
||||
|
||||
ASSET_LINE="$(node -e 'for (const a of require(process.argv[1])) console.log(a.sha256 + " " + a.byte_size)' "$WORK/repo/$SEED_REL/assets.json")"
|
||||
new_case() { # name
|
||||
CASE="$WORK/cases/$1"
|
||||
@@ -299,6 +357,7 @@ new_case() { # name
|
||||
echo 'official=1 historical=1 versions=1 draft_equal=true accounts=0' > "$CASE/stub/answers/equine"
|
||||
printf '%s\n' "$ASSET_LINE" > "$CASE/stub/answers/asset-rows"
|
||||
echo 1 > "$CASE/stub/answers/assets-uploaded"
|
||||
echo 0 > "$CASE/stub/answers/rotation-active"
|
||||
echo 'granted false' > "$CASE/stub/answers/grant-catalog"
|
||||
EXTRA=()
|
||||
CASE_PATH="$WORK/bin:$BASE_PATH"
|
||||
@@ -340,7 +399,7 @@ run_script() { # script args... → RC, output in $CASE/out; stdin as the caller
|
||||
verify_stub_shadow
|
||||
( cd "$WORK/repo" && env -i HOME="$CASE/home" LANG=C.UTF-8 \
|
||||
DOCKER_HOST="$CASE_DOCKER_HOST" DOCKER_CONTEXT=faunosteo-check-no-such-context DOCKER_CONFIG="$WORK/docker-config" \
|
||||
STUB_LOG="$CASE/log" STUB_STATE="$CASE/stub" FAUNOSTEO_TRIAL_HOST=127.0.0.1 "${EXTRA[@]}" PATH="$CASE_PATH" \
|
||||
STUB_LOG="$CASE/log" STUB_STATE="$CASE/stub" STUB_TERM="$WORK/term-trial.sh" FAUNOSTEO_TRIAL_HOST=127.0.0.1 "${EXTRA[@]}" PATH="$CASE_PATH" \
|
||||
"${LAUNCH[@]}" bash "scripts/$script" "$@" ) > "$CASE/out" 2>&1
|
||||
RC=$?
|
||||
}
|
||||
@@ -364,6 +423,7 @@ changes() {
|
||||
out_has() { grep -qF -- "$1" "$CASE/out"; }
|
||||
log_has() { grep -qF -- "$1" "$CASE/log"; }
|
||||
refused() { [ "$RC" -ne 0 ] && [ -z "$(changes)" ] && out_has "$1"; }
|
||||
refused_no_docker() { refused "$1" && [ -z "$(docker_calls)" ]; }
|
||||
refused_early() { [ "$RC" -ne 0 ] && [ ! -s "$CASE/log" ] && out_has "$1"; }
|
||||
mode_is() { [ "$(stat -c %a "$1")" = "$2" ]; }
|
||||
values_absent_from() { # secrets file, files...
|
||||
@@ -591,6 +651,395 @@ chmod 644 "$STATE_DIR/secrets.env"; fresh_log; trial status
|
||||
check "secrets readable by others: refused" refused "mode 600"
|
||||
chmod 600 "$STATE_DIR/secrets.env"
|
||||
|
||||
echo "-- backup"
|
||||
new_case backups; trial create
|
||||
check "backups: create" [ "$RC" -eq 0 ]
|
||||
fresh_log; trial backup
|
||||
BK="$(find "$STATE_DIR/backups" -mindepth 1 -maxdepth 1 -type d -name '*-manual' | head -n 1)"
|
||||
check "backup: folder 700, files 600, checksums valid, both archives listed" bash -c '[ -n "$1" ] && [ "$(stat -c %a "$1")" = 700 ] && for f in db.tar storage.tar secrets.env stack.conf migrations.sha256 SHA256SUMS; do [ "$(stat -c %a "$1/$f")" = 600 ] || exit 1; done && cd "$1" && sha256sum --quiet -c SHA256SUMS && grep -q " db.tar$" SHA256SUMS && grep -q " storage.tar$" SHA256SUMS && [ ! -e SHA256SUMS.new ]' _ "$BK"
|
||||
check "backup: both volumes measured by the helper, read-only, no network" bash -c 'grep -qF "docker run --rm --name faunosteo_trial_helper --network none -v faunosteo_trial_db:/volume:ro --entrypoint sh public.ecr.aws/supabase/postgres:17.11.0.004 -c du -sk /volume" "$1" && grep -qF "docker run --rm --name faunosteo_trial_helper --network none -v faunosteo_trial_storage:/volume:ro --entrypoint sh public.ecr.aws/supabase/postgres:17.11.0.004 -c du -sk /volume" "$1"' _ "$CASE/log"
|
||||
check "backup: both volumes archived read-only by the helper, no network" bash -c 'grep -qF "docker run --rm --name faunosteo_trial_helper --network none -v faunosteo_trial_db:/volume:ro --entrypoint tar" "$1" && grep -qF "docker run --rm --name faunosteo_trial_helper --network none -v faunosteo_trial_storage:/volume:ro --entrypoint tar" "$1"' _ "$CASE/log"
|
||||
check "backup: the stack stops before the copy and restarts after" bash -c 'stop=$(grep -n "^docker stop" "$1" | tail -n 1 | cut -d: -f1); helper=$(grep -n "^docker run --rm --name faunosteo_trial_helper" "$1" | head -n 1 | cut -d: -f1); start=$(grep -n "^docker start" "$1" | head -n 1 | cut -d: -f1); [ -n "$stop" ] && [ -n "$helper" ] && [ -n "$start" ] && [ "$stop" -lt "$helper" ] && [ "$helper" -lt "$start" ]' _ "$CASE/log"
|
||||
check "backup: no volume removal" no_volume_removal
|
||||
check "backup: no secret in calls or output" secrets_absent "$STATE_DIR/secrets.env"
|
||||
COUNT="$(backup_count)"; : > "$CASE/stub/fail/archive"; fresh_log; trial backup
|
||||
check "a failed archive: no folder kept, the stack left stopped, the way out named" bash -c '[ "$1" -ne 0 ] && [ "$2" -eq "$3" ] && grep -qF "backup failed" "$4" && grep -qF "pnpm db:trial start" "$4"' _ "$RC" "$COUNT" "$(backup_count)" "$CASE/out"
|
||||
rm -f "$CASE/stub/fail/archive"; fresh_log; trial start
|
||||
check "after a failed backup, start brings the stack back" [ "$RC" -eq 0 ]
|
||||
COUNT="$(backup_count)"; : > "$CASE/stub/short-archive"; fresh_log; trial backup
|
||||
check "an archive that lists fewer entries than its volume (exit 0): refused, no folder kept" bash -c '[ "$1" -ne 0 ] && [ "$2" -eq "$3" ] && grep -qF "unreadable or incomplete" "$4"' _ "$RC" "$COUNT" "$(backup_count)" "$CASE/out"
|
||||
rm -f "$CASE/stub/short-archive"; fresh_log; trial start
|
||||
mkdir -m 700 "$STATE_DIR/backups/20000101T000000Z-partial"
|
||||
fresh_log; trial status
|
||||
check "status marks a folder without SHA256SUMS incomplete" bash -c 'grep -F "20000101T000000Z-partial" "$1" | grep -qF incomplete' _ "$CASE/out"
|
||||
fresh_log; trial restore 20000101T000000Z-partial
|
||||
check "restore refuses an incomplete backup before any change" refused "is incomplete"
|
||||
rmdir "$STATE_DIR/backups/20000101T000000Z-partial"
|
||||
|
||||
echo "-- migrate"
|
||||
cp -p "$CASE/stub/ports/faunosteo_trial_db" "$CASE/db.port"
|
||||
echo '5432/tcp -> 0.0.0.0:58432' > "$CASE/stub/ports/faunosteo_trial_db"
|
||||
fresh_log; trial migrate
|
||||
check "migrate: Postgres published beyond 127.0.0.1 is refused before any backup" bash -c '[ "$1" -ne 0 ] && grep -qF "faunosteo_trial_db is not published on 127.0.0.1:58432 only" "$2" && ! grep -qF faunosteo_trial_helper "$3"' _ "$RC" "$CASE/out" "$CASE/log"
|
||||
cp -p "$CASE/db.port" "$CASE/stub/ports/faunosteo_trial_db"
|
||||
fresh_log; trial migrate
|
||||
check "migrate with nothing new: no backup, no change" bash -c '[ "$1" -eq 0 ] && grep -qF "no new migration" "$2" && ! grep -qF faunosteo_trial_helper "$3"' _ "$RC" "$CASE/out" "$CASE/log"
|
||||
FIRST="$(find "$WORK/repo/supabase/migrations" -name '*.sql' | sort | head -n 1)"
|
||||
cp -p "$FIRST" "$CASE/first.sql"; printf -- '-- edited\n' >> "$FIRST"
|
||||
fresh_log; trial migrate
|
||||
check "an applied migration edited in place: refused before any backup" refused "was edited after it was applied"
|
||||
cp -p "$CASE/first.sql" "$FIRST"
|
||||
printf -- '-- harness\n' > "$WORK/repo/supabase/migrations/20000101000000_harness_old.sql"
|
||||
fresh_log; trial migrate
|
||||
check "a new migration older than the last applied: refused before any backup" refused "older than the last applied migration"
|
||||
rm -f "$WORK/repo/supabase/migrations/20000101000000_harness_old.sql"
|
||||
printf -- '-- harness\n' > "$WORK/repo/supabase/migrations/29991231000000_harness_new.sql"
|
||||
fresh_log; trial migrate
|
||||
check "migrate: succeeds" [ "$RC" -eq 0 ]
|
||||
check "migrate: the backup precedes the migration" before 'faunosteo_trial_helper' 'supabase --workdir . migration up'
|
||||
check "migrate: the new file joins migrations.sha256" grep -q '^29991231000000 ' "$STATE_DIR/migrations.sha256"
|
||||
check "migrate: PostgREST schema reloaded" log_has "notify pgrst, 'reload schema'"
|
||||
printf -- '-- harness\n' > "$WORK/repo/supabase/migrations/29991231000001_harness_newer.sql"
|
||||
: > "$CASE/stub/fail/migrate"; fresh_log; trial migrate
|
||||
check "a migrate that stops part-way names its pre-migrate backup" bash -c '[ "$1" -ne 0 ] && grep -qE "restore [0-9]{8}T[0-9]{6}Z-pre-migrate" "$2"' _ "$RC" "$CASE/out"
|
||||
fresh_log; trial status
|
||||
check "after it, status names the remedy" out_has "pre-migrate backup"
|
||||
rm -f "$CASE/stub/fail/migrate" "$WORK/repo/supabase/migrations/29991231000001_harness_newer.sql"
|
||||
( cd "$WORK/repo" && for f in supabase/migrations/*.sql; do basename "$f" | cut -d_ -f1; done ) > "$CASE/stub/answers/applied"
|
||||
rm -f "$WORK/repo/supabase/migrations/29991231000000_harness_new.sql"
|
||||
|
||||
echo "-- rotate-keys"
|
||||
if hold_port 5194; then fresh_log; trial rotate-keys; check "rotate-keys while port 5194 is in use (the app): refused, nothing changed" refused "5194 is in use"; release_port
|
||||
else fail "could not hold 127.0.0.1:5194 for the app case"; fi
|
||||
cp -p "$STATE_DIR/secrets.env" "$CASE/secrets.before-keys"
|
||||
fresh_log; trial rotate-keys
|
||||
check "rotate-keys: password and JWT secret kept, keys renewed" bash -c '[ "$1" -eq 0 ] && [ "$(grep -E "^TR_(PW|JWT_SECRET)=" "$2")" = "$(grep -E "^TR_(PW|JWT_SECRET)=" "$3")" ] && [ "$(grep "^TR_ANON_KEY=" "$2")" != "$(grep "^TR_ANON_KEY=" "$3")" ] && [ "$(stat -c %a "$2")" = 600 ]' _ "$RC" "$STATE_DIR/secrets.env" "$CASE/secrets.before-keys"
|
||||
check "rotate-keys: a backup first, then storage recreated and the gateway restarted, nothing else" bash -c 'last=$(printf "%s\n" "$1" | tail -n 4); printf "%s\n" "$last" | head -n 1 | grep -qx "docker stop -t 30 faunosteo_trial_storage" && printf "%s\n" "$last" | grep -qx "docker rm faunosteo_trial_storage" && printf "%s\n" "$last" | grep -q "^docker run -d --name faunosteo_trial_storage " && printf "%s\n" "$last" | tail -n 1 | grep -qx "docker restart -t 10 faunosteo_trial_gateway" && printf "%s\n" "$1" | head -n 1 | grep -q "^docker stop -t 60 "' _ "$(changes)"
|
||||
check "rotate-keys: its backup is restorable and protected (700, 600, checksums)" bash -c 'd=$(ls -d "$1"/*-pre-keys* | head -n 1); [ -n "$d" ] && [ "$(stat -c %a "$d")" = 700 ] && for f in db.tar storage.tar secrets.env stack.conf migrations.sha256 SHA256SUMS; do [ "$(stat -c %a "$d/$f")" = 600 ] || exit 1; done && cd "$d" && sha256sum --quiet -c SHA256SUMS && cmp -s secrets.env "$2"' _ "$STATE_DIR/backups" "$CASE/secrets.before-keys"
|
||||
check "rotate-keys: no secret in calls or output" secrets_absent "$STATE_DIR/secrets.env"
|
||||
|
||||
echo "-- rotate-secrets"
|
||||
cp -p "$STATE_DIR/secrets.env" "$CASE/secrets.before-refusal"
|
||||
: > "$CASE/stub/fail/rotate"; fresh_log; trial rotate-secrets
|
||||
check "rotation refused by the database: secrets unchanged, no container removed" bash -c '[ "$1" -ne 0 ] && cmp -s "$2" "$3" && ! grep -q "^docker rm " "$4" && [ ! -e "$5" ] && grep -qF "nothing was changed" "$6"' _ "$RC" "$STATE_DIR/secrets.env" "$CASE/secrets.before-refusal" "$CASE/log" "$STATE_DIR/secrets.next.env" "$CASE/out"
|
||||
: > "$CASE/stub/fail/rotation-marker"; fresh_log; trial rotate-secrets
|
||||
check "rotation failed and the database does not answer: both files kept, the backup named" bash -c '[ "$1" -ne 0 ] && cmp -s "$2" "$3" && [ -f "$4" ] && grep -qF "secrets.next.env (new) are both kept" "$5" && grep -qE "restore [0-9]{8}T[0-9]{6}Z-pre-rotate" "$5"' _ "$RC" "$STATE_DIR/secrets.env" "$CASE/secrets.before-refusal" "$STATE_DIR/secrets.next.env" "$CASE/out"
|
||||
rm -f "$CASE/stub/fail/rotate" "$CASE/stub/fail/rotation-marker" "$STATE_DIR/secrets.next.env"
|
||||
cp -p "$STATE_DIR/secrets.env" "$CASE/secrets.before-lost"
|
||||
: > "$CASE/stub/commit-then-fail"; fresh_log; trial rotate-secrets
|
||||
check "psql lost after the commit: the database's answer decides, the rotation completes" bash -c '[ "$1" -eq 0 ] && ! cmp -s "$2" "$3" && grep -qF "took the new secrets although psql failed" "$4" && grep -qx "docker rm faunosteo_trial_db" "$5" && [ ! -e "$6" ]' _ "$RC" "$STATE_DIR/secrets.env" "$CASE/secrets.before-lost" "$CASE/out" "$CASE/log" "$STATE_DIR/secrets.next.env"
|
||||
rm -f "$CASE/stub/commit-then-fail" "$CASE/stub/answers/rotation-marker"
|
||||
cp -p "$STATE_DIR/secrets.env" "$CASE/secrets.before-rotation"
|
||||
fresh_log; trial rotate-secrets
|
||||
check "rotate-secrets: succeeds" [ "$RC" -eq 0 ]
|
||||
check "rotate-secrets: every value renewed" bash -c 'for k in TR_PW TR_JWT_SECRET TR_ANON_KEY TR_SERVICE_KEY; do [ "$(grep "^$k=" "$1")" != "$(grep "^$k=" "$2")" ] || exit 1; done' _ "$STATE_DIR/secrets.env" "$CASE/secrets.before-rotation"
|
||||
check "rotate-secrets: a restorable, protected backup holding the previous secrets (no secrets.prev needed)" bash -c 'd=$(ls -d "$1"/*-pre-rotate* | tail -n 1); [ -n "$d" ] && [ "$(stat -c %a "$d")" = 700 ] && for f in db.tar storage.tar secrets.env stack.conf migrations.sha256 SHA256SUMS; do [ "$(stat -c %a "$d/$f")" = 600 ] || exit 1; done && cd "$d" && sha256sum --quiet -c SHA256SUMS && cmp -s secrets.env "$2"' _ "$STATE_DIR/backups" "$CASE/secrets.before-rotation"
|
||||
check "rotate-secrets: values reach psql by environment names only" log_has 'docker exec -i -e PGPASSWORD -e PGAPPNAME -e NEW_PW -e NEW_JWT -e ROT_DIGEST faunosteo_trial_db psql -X -q -h 127.0.0.1 -U supabase_admin'
|
||||
check "rotate-secrets: no rotation SQL text holds a secret, old or new" bash -c 'files=$(grep -lF -- "-- db-trial:rotate" $(find "$1" -type f)); [ -n "$files" ] && for f in $files; do grep -qF "set local log_min_error_statement = panic;" "$f" && grep -qF "app.trial_rotation" "$f" || exit 1; for s in "${@:2}"; do while IFS="=" read -r _ v; do [ -z "$v" ] || ! grep -qF -- "$v" "$f" || exit 1; done < "$s"; done; done' _ "$CASE/stub/stdin" "$STATE_DIR/secrets.env" "$CASE/secrets.before-rotation" "$CASE/secrets.before-lost" "$CASE/secrets.before-refusal"
|
||||
check "rotate-secrets: six containers recreated, volumes kept" bash -c 'for n in db mail auth storage rest gateway; do grep -qx "docker rm faunosteo_trial_$n" "$1" && grep -q "^docker run -d --name faunosteo_trial_$n " "$1" || exit 1; done; ! grep -qF "docker volume rm" "$1"' _ "$CASE/log"
|
||||
check "rotate-secrets: no leftover secrets.next.env, no secrets.prev.env" bash -c '[ ! -e "$1/secrets.prev.env" ] && [ ! -e "$1/secrets.next.env" ]' _ "$STATE_DIR"
|
||||
check "rotate-secrets: new secrets absent from calls and output" secrets_absent "$STATE_DIR/secrets.env"
|
||||
check "rotate-secrets: old secrets absent from calls and output" secrets_absent "$CASE/secrets.before-rotation"
|
||||
|
||||
echo "-- rotations refuse keys that are not this stack's"
|
||||
cp -p "$STATE_DIR/secrets.env" "$CASE/secrets.unforged"
|
||||
forge TR_SERVICE_KEY faunosteo-trial other service_role
|
||||
cp -p "$STATE_DIR/secrets.env" "$CASE/secrets.forged"
|
||||
for action in rotate-keys rotate-secrets; do
|
||||
COUNT="$(backup_count)"; fresh_log; trial "$action"
|
||||
check "$action: keys that are not this stack's are refused before any stop, no backup taken" bash -c '[ "$1" -ne 0 ] && grep -qF "service_role key in secrets.env is not this stack" "$2" && grep -qF "pnpm db:trial restore <backup>" "$2" && cmp -s "$3" "$4" && [ -z "$5" ] && [ "$6" -eq "$7" ] && [ ! -e "$8" ]' _ "$RC" "$CASE/out" "$STATE_DIR/secrets.env" "$CASE/secrets.forged" "$(changes)" "$COUNT" "$(backup_count)" "$STATE_DIR/secrets.next.env"
|
||||
done
|
||||
cp -p "$CASE/secrets.unforged" "$STATE_DIR/secrets.env"
|
||||
|
||||
echo "-- restore"
|
||||
NAME="$(basename "$BK")"
|
||||
fresh_log; trial restore ../etc
|
||||
check "restore: a name that is not a backup folder is refused, before any Docker call" refused_no_docker "restore <backup>"
|
||||
cp -rp "$BK" "$STATE_DIR/backups/20000101T000000Z-tampered"; printf 'x' >> "$STATE_DIR/backups/20000101T000000Z-tampered/db.tar"
|
||||
fresh_log; trial_tty 'restore 20000101T000000Z-tampered' restore 20000101T000000Z-tampered
|
||||
check "restore: a tampered backup is refused before any change" refused "fails its checksums"
|
||||
FORGED=20000101T000000Z-forged-key
|
||||
cp -rp "$BK" "$STATE_DIR/backups/$FORGED"
|
||||
forge TR_SERVICE_KEY faunosteo-trial other service_role 3600 "$STATE_DIR/backups/$FORGED/secrets.env"
|
||||
( cd "$STATE_DIR/backups/$FORGED" && sha256sum -- db.tar storage.tar secrets.env stack.conf migrations.sha256 > SHA256SUMS )
|
||||
fresh_log; trial_tty "restore $FORGED" restore "$FORGED"
|
||||
check "restore: a backup whose key is not signed with its own secret is refused before any change" refused "holds keys that are not signed with its own secret"
|
||||
: > "$CASE/stub/fail/image"; fresh_log; trial_tty "restore $NAME" restore "$NAME"
|
||||
check "restore: the recorded database image is absent: refused before any change" refused "is not present locally"
|
||||
rm -f "$CASE/stub/fail/image"
|
||||
fresh_log; trial_tty nope restore "$NAME"
|
||||
check "restore: a wrong phrase changes nothing" refused "not confirmed"
|
||||
fresh_log; trial_input restore "$NAME" <<<"restore $NAME"
|
||||
check "restore: the phrase piped without a terminal is refused" refused "needs a terminal"
|
||||
fresh_log; trial_tty "restore $NAME" restore "$NAME"
|
||||
check "restore: succeeds" [ "$RC" -eq 0 ]
|
||||
check "restore: says the backup brings back older secrets" out_has "holds other secrets than the current ones"
|
||||
check "restore: a backup of the current state first" before 'faunosteo_trial_db:/volume:ro --entrypoint tar' 'docker volume rm faunosteo_trial_db'
|
||||
check "restore: volumes replaced then filled by the helper" before 'docker volume create --label faunosteo.stack=trial faunosteo_trial_db' 'docker run --rm -i --name faunosteo_trial_helper --network none -v faunosteo_trial_db:/volume --entrypoint tar'
|
||||
check "restore: the backup's secrets are back" cmp -s "$STATE_DIR/secrets.env" "$BK/secrets.env"
|
||||
check "restore: six containers recreated" bash -c 'for n in db mail auth storage rest gateway; do grep -q "^docker run -d --name faunosteo_trial_$n " "$1" || exit 1; done' _ "$CASE/log"
|
||||
check "restore: only trial names" only_trial_names
|
||||
|
||||
echo "-- destroy"
|
||||
fresh_log; trial_tty '' destroy
|
||||
check "destroy with an empty answer: nothing changed" refused "not confirmed"
|
||||
fresh_log; trial_input destroy <<<"faunosteo_trial"
|
||||
check "destroy with the phrase piped without a terminal: refused, nothing changed" refused "needs a terminal"
|
||||
fresh_log; trial_tty faunosteo_trial destroy
|
||||
check "destroy: succeeds" [ "$RC" -eq 0 ]
|
||||
check "destroy: a backup before any removal" before 'faunosteo_trial_db:/volume:ro --entrypoint tar' 'docker rm faunosteo_trial_db'
|
||||
check "destroy: every removal names a trial resource" bash -c 'grep -E "^docker (rm|volume rm|network rm) " "$1" | grep -qvE " faunosteo_trial_(db|mail|auth|storage|rest|gateway|net)$" && exit 1; grep -qx "docker volume rm faunosteo_trial_db" "$1" && grep -qx "docker volume rm faunosteo_trial_storage" "$1" && grep -qx "docker network rm faunosteo_trial_net" "$1"' _ "$CASE/log"
|
||||
check "destroy: live state removed, marker and backups kept" bash -c 'for f in secrets.env secrets.next.env db.env auth.env storage.env rest.env stack.conf migrations.sha256; do [ ! -e "$1/$f" ] || exit 1; done; [ -e "$1/.faunosteo-trial-state" ] && ls -d "$1"/backups/*-pre-destroy* >/dev/null 2>&1 && [ -d "$2" ]' _ "$STATE_DIR" "$BK"
|
||||
fresh_log; trial_tty "restore $NAME" restore "$NAME"
|
||||
check "restore after destroy rebuilds the stack from the backup" bash -c '[ "$1" -eq 0 ] && grep -qF "no trial volume" "$2" && grep -qx "docker network create --label faunosteo.stack=trial faunosteo_trial_net" "$3"' _ "$RC" "$CASE/out" "$CASE/log"
|
||||
|
||||
echo "-- destroy with nothing left in Docker"
|
||||
new_case leftover; trial create
|
||||
rm -f "$CASE/stub/containers/"* "$CASE/stub/ports/"* "$CASE/stub/volumes/"* "$CASE/stub/networks/"*
|
||||
fresh_log; trial_tty faunosteo_trial destroy
|
||||
check "destroy with no resource: after the phrase, the live files go, backups and marker kept, no Docker removal" bash -c '[ "$1" -eq 0 ] && [ ! -e "$2/secrets.env" ] && [ ! -e "$2/db.env" ] && [ -d "$2/backups" ] && [ -e "$2/.faunosteo-trial-state" ] && ! grep -qE "^docker (rm|volume rm|network rm) " "$3"' _ "$RC" "$STATE_DIR" "$CASE/log"
|
||||
|
||||
echo "-- B2 review fixes"
|
||||
# A created stack with one manual backup: RB is its folder, RN its name.
|
||||
b2_stack() { # case name
|
||||
new_case "$1"; trial create; trial backup
|
||||
RB="$(find "$STATE_DIR/backups" -mindepth 1 -maxdepth 1 -type d -name '*-manual' | head -n 1)"
|
||||
RN="$(basename "$RB")"
|
||||
}
|
||||
restore_refused() { # backup name, message: refused at the check, nothing removed
|
||||
fresh_log; trial_tty "restore $1" restore "$1"
|
||||
refused "$2" && no_volume_removal
|
||||
}
|
||||
stopped_part_way() { [ "$RC" -ne 0 ] && out_has "restore stopped part-way" && out_has "$1"; }
|
||||
polls() { grep -c 'db-trial:rotation-active' "$CASE/log" || true; }
|
||||
rotation_sql() { stdin_files | xargs -r grep -lx -- '-- db-trial:rotate' | head -n 1; }
|
||||
sql_is_one_explicit_transaction() { # file: begin before every alter, commit last
|
||||
local first_alter begin
|
||||
begin="$(grep -n '^begin;$' "$1" | head -n 1 | cut -d: -f1)"
|
||||
first_alter="$(grep -n '^alter ' "$1" | head -n 1 | cut -d: -f1)"
|
||||
[ -n "$begin" ] && [ -n "$first_alter" ] && [ "$begin" -lt "$first_alter" ] && [ "$(grep -c '^begin;$' "$1")" -eq 1 ] \
|
||||
&& [ "$(tail -n 1 "$1")" = 'commit;' ]
|
||||
}
|
||||
no_single_transaction_rotation() { ! docker_calls | grep -F 'supabase_admin' | grep -qF -- --single-transaction; }
|
||||
|
||||
# -- the backup is proven before anything is removed
|
||||
b2_stack b2-proof
|
||||
OLD=20000101T000000Z
|
||||
cp -rp "$RB" "$STATE_DIR/backups/$OLD-nolist"
|
||||
( cd "$STATE_DIR/backups/$OLD-nolist" && grep -v ' secrets.env$' SHA256SUMS > S.new && cat S.new > SHA256SUMS && rm S.new )
|
||||
check "restore: a SHA256SUMS missing a required entry is refused, nothing changed" restore_refused "$OLD-nolist" "does not list exactly"
|
||||
cp -rp "$RB" "$STATE_DIR/backups/$OLD-extra"; printf 'x' > "$STATE_DIR/backups/$OLD-extra/extra.txt"
|
||||
check "restore: a file the SHA256SUMS does not list is refused, nothing changed" restore_refused "$OLD-extra" "does not list exactly"
|
||||
ln -s "$RB" "$STATE_DIR/backups/$OLD-linked"
|
||||
check "restore: a backup folder that is a symbolic link is refused, nothing changed" restore_refused "$OLD-linked" "symbolic link"
|
||||
cp -rp "$RB" "$STATE_DIR/backups/$OLD-filelink"
|
||||
mv "$STATE_DIR/backups/$OLD-filelink/storage.tar" "$CASE/storage.real"; ln -s "$CASE/storage.real" "$STATE_DIR/backups/$OLD-filelink/storage.tar"
|
||||
check "restore: a symbolic link inside a backup (checksums still valid) is refused, nothing changed" restore_refused "$OLD-filelink" "symbolic link"
|
||||
mv "$STATE_DIR/backups" "$CASE/backups.real"; ln -s "$CASE/backups.real" "$STATE_DIR/backups"
|
||||
check "restore: a backups/ that is a symbolic link is refused, nothing changed" restore_refused "$RN" "symbolic link"
|
||||
fresh_log; trial backup
|
||||
check "backup: a backups/ that is a symbolic link is refused before any stop" refused "symbolic link"
|
||||
rm -- "$STATE_DIR/backups"; mv "$CASE/backups.real" "$STATE_DIR/backups"
|
||||
|
||||
# -- the folder is checked again right before the extraction
|
||||
b2_stack b2-toctou
|
||||
printf '%s' "$RB/db.tar" > "$CASE/stub/append-on-rm"
|
||||
fresh_log; trial_tty "restore $RN" restore "$RN"
|
||||
check "restore: a backup changed after the first check is refused before any volume removal" bash -c '[ "$1" -ne 0 ] && grep -qF "fails its checksums" "$2" && ! grep -qF "docker volume rm" "$3"' _ "$RC" "$CASE/out" "$CASE/log"
|
||||
check "restore: that refusal names the pre-restore backup and the rerun" bash -c 'grep -qE "restore [0-9]{8}T[0-9]{6}Z-pre-restore" "$1" && grep -qF "pnpm db:trial restore $2" "$1"' _ "$CASE/out" "$RN"
|
||||
|
||||
# -- check_backup after each backup of a command that goes on to change something
|
||||
b2_stack b2-after
|
||||
cp -p "$STATE_DIR/secrets.env" "$CASE/secrets.kept"
|
||||
printf -- '-- harness\n' > "$WORK/repo/supabase/migrations/29991231000002_harness_b2.sql"
|
||||
: > "$CASE/stub/fail/image-nondb"
|
||||
stops_after_backup() {
|
||||
[ "$RC" -ne 0 ] && out_has "is not present locally" && ! out_has "nothing was changed" && ! log_has 'docker rm ' \
|
||||
&& ! log_has 'migration up' && cmp -s "$STATE_DIR/secrets.env" "$CASE/secrets.kept"
|
||||
}
|
||||
for action in rotate-keys rotate-secrets migrate; do
|
||||
fresh_log; trial "$action"
|
||||
check "$action: a backup whose recorded image is absent stops the command before any change" stops_after_backup
|
||||
done
|
||||
rm -f "$WORK/repo/supabase/migrations/29991231000002_harness_b2.sql"
|
||||
fresh_log; trial_tty faunosteo_trial destroy
|
||||
check "destroy: a backup that fails its check stops before any removal, the stack stays stopped, the backup is named" bash -c '[ "$1" -ne 0 ] && grep -qF "is not present locally" "$2" && grep -qF "stack stays stopped (pnpm db:trial start)" "$2" && grep -qE "pre-destroy" "$2" && ! grep -qE "^docker (rm|volume rm|network rm) " "$3" && [ -f "$4/secrets.env" ]' _ "$RC" "$CASE/out" "$CASE/log" "$STATE_DIR"
|
||||
|
||||
# -- a restore that stops part-way says where the way back is
|
||||
b2_stack b2-extract
|
||||
: > "$CASE/stub/fail/extract"
|
||||
fresh_log; trial_tty "restore $RN" restore "$RN"
|
||||
check "restore whose extraction fails: names the pre-restore backup and the rerun" bash -c '[ "$1" -ne 0 ] && grep -qF "restore stopped part-way" "$2" && grep -qE "pnpm db:trial restore [0-9]{8}T[0-9]{6}Z-pre-restore" "$2" && grep -qF "pnpm db:trial restore $3" "$2"' _ "$RC" "$CASE/out" "$RN"
|
||||
rm -f "$CASE/stub/volumes/"* "$CASE/stub/containers/"* "$CASE/stub/ports/"*
|
||||
fresh_log; trial_tty "restore $RN" restore "$RN"
|
||||
check "restore with no trial volume that stops part-way says no backup was taken" stopped_part_way "No backup of the state before it"
|
||||
|
||||
# -- restore: a restored state file is written 600, whatever the backup's mode; the note on a pre-restore backup
|
||||
b2_stack b2-mode
|
||||
cp -rp "$RB" "$STATE_DIR/backups/$OLD-mode644"
|
||||
chmod 644 "$STATE_DIR/backups/$OLD-mode644/secrets.env" "$STATE_DIR/backups/$OLD-mode644/stack.conf" "$STATE_DIR/backups/$OLD-mode644/migrations.sha256"
|
||||
fresh_log; trial_tty "restore $OLD-mode644" restore "$OLD-mode644"
|
||||
check "restore from a backup holding 644 files: the restored state files are 600" bash -c '[ "$1" -eq 0 ] && for f in secrets.env stack.conf migrations.sha256; do [ "$(stat -c %a "$2/$f")" = 600 ] || exit 1; done' _ "$RC" "$STATE_DIR"
|
||||
forge TR_SERVICE_KEY faunosteo-trial other service_role
|
||||
fresh_log; trial_tty "restore $RN" restore "$RN"
|
||||
check "restore: a pre-restore backup that restore itself would refuse is kept and noted, the restore goes on" bash -c '[ "$1" -eq 0 ] && grep -qF "restore will refuse it as it is" "$2" && grep -qF "not signed with its own secret" "$2"' _ "$RC" "$CASE/out"
|
||||
|
||||
# -- the app is running: nothing changes
|
||||
b2_stack b2-app
|
||||
if hold_port 5194; then
|
||||
fresh_log; trial_tty "restore $RN" restore "$RN"; check "restore while port 5194 is in use: refused, nothing changed" refused "5194 is in use"
|
||||
fresh_log; trial_tty faunosteo_trial destroy; check "destroy while port 5194 is in use: refused, nothing changed" refused "5194 is in use"
|
||||
fresh_log; trial rotate-secrets; check "rotate-secrets while port 5194 is in use: refused, nothing changed" refused "5194 is in use"
|
||||
release_port
|
||||
else fail "could not hold 127.0.0.1:5194 for the app cases"; fi
|
||||
|
||||
# -- restore: a trial port taken by another process, found before any volume removal
|
||||
b2_stack b2-ports
|
||||
if hold_port 58432; then
|
||||
fresh_log; trial_tty "restore $RN" restore "$RN"
|
||||
check "restore with a trial port taken: refused before any removal, no half-filled message" bash -c '[ "$1" -ne 0 ] && grep -qF "127.0.0.1:58432 is taken" "$2" && ! grep -qE "^docker (rm|volume rm) " "$3" && ! grep -qF "half-filled" "$2"' _ "$RC" "$CASE/out" "$CASE/log"
|
||||
release_port
|
||||
else fail "could not hold 127.0.0.1:58432 for the port case"; fi
|
||||
|
||||
# -- backup: free space, a helper left over, an interruption
|
||||
new_case b2-backup; trial create
|
||||
echo 999999999999 > "$CASE/stub/answers/measure-kib"; COUNT="$(backup_count)"; fresh_log; trial backup
|
||||
check "backup: not enough free space is refused, no folder kept, the stack stays stopped" bash -c '[ "$1" -ne 0 ] && grep -qF "less free space" "$2" && grep -qF "the stack stays stopped" "$2" && grep -q "^docker stop" "$3" && [ "$4" -eq "$5" ]' _ "$RC" "$CASE/out" "$CASE/log" "$COUNT" "$(backup_count)"
|
||||
rm -f "$CASE/stub/answers/measure-kib"; trial start
|
||||
: > "$CASE/stub/containers/faunosteo_trial_helper"; COUNT="$(backup_count)"; fresh_log; trial backup
|
||||
check "backup: a leftover helper container is refused before any stop, the hint named, no folder" bash -c '[ "$1" -ne 0 ] && grep -qF "docker rm faunosteo_trial_helper" "$2" && ! grep -q "^docker stop" "$3" && [ "$4" -eq "$5" ]' _ "$RC" "$CASE/out" "$CASE/log" "$COUNT" "$(backup_count)"
|
||||
rm -f "$CASE/stub/containers/faunosteo_trial_helper"
|
||||
: > "$CASE/stub/fail/archive"; fresh_log; trial backup
|
||||
check "backup: a failed archive prints one message (backup failed), not the interruption one" bash -c 'grep -qF "backup failed" "$1" && ! grep -qF "is incomplete" "$1"' _ "$CASE/out"
|
||||
rm -f "$CASE/stub/fail/archive"; trial start
|
||||
: > "$CASE/stub/term-in-archive"; COUNT="$(backup_count)"; fresh_log; trial backup
|
||||
check "backup interrupted by SIGTERM: one message, the folder named incomplete, the stack stays stopped" bash -c '[ "$1" -ne 0 ] && [ "$(grep -c "is incomplete" "$2")" -eq 1 ] && grep -qF "the stack may be stopped (pnpm db:trial start)" "$2" && [ "$3" -eq $(( $4 + 1 )) ]' _ "$RC" "$CASE/out" "$(backup_count)" "$COUNT"
|
||||
rm -f "$CASE/stub/term-in-archive"; fresh_log; trial status
|
||||
check "status lists the interrupted backup as incomplete" bash -c 'grep -F "backups in" -A9 "$1" | grep -qF incomplete' _ "$CASE/out"
|
||||
|
||||
# -- the lock comes before the state checks
|
||||
new_case b2-lock; trial create
|
||||
rm -f "$CASE/stub/volumes/faunosteo_trial_db"
|
||||
flock "$STATE_DIR/.lock" sleep 6 & LOCK_HOLDER=$!
|
||||
sleep 0.5
|
||||
for action in backup migrate rotate-keys rotate-secrets; do
|
||||
fresh_log; trial "$action"
|
||||
check "$action: the lock is taken before the stack is checked" bash -c '[ "$1" -ne 0 ] && grep -qF "another pnpm db:trial command is running" "$2"' _ "$RC" "$CASE/out"
|
||||
done
|
||||
wait "$LOCK_HOLDER"
|
||||
|
||||
# -- a rotation that stopped part-way leaves secrets.next.env: never overwritten
|
||||
b2_stack b2-next
|
||||
printf 'SENTINEL-NEXT\n' > "$STATE_DIR/secrets.next.env"; chmod 600 "$STATE_DIR/secrets.next.env"
|
||||
fresh_log; trial status
|
||||
check "status mentions a leftover secrets.next.env, never its content" bash -c 'grep -qF "secrets.next.env exists" "$1" && ! grep -qF SENTINEL-NEXT "$1"' _ "$CASE/out"
|
||||
cp -p "$STATE_DIR/secrets.env" "$CASE/secrets.unforged"
|
||||
forge TR_SERVICE_KEY faunosteo-trial other service_role
|
||||
COUNT="$(backup_count)"; fresh_log; trial rotate-secrets
|
||||
check "rotate-secrets with a leftover secrets.next.env: refused before the key check and any backup, the file untouched" bash -c '[ "$1" -ne 0 ] && grep -qF "secrets.next.env exists" "$2" && ! grep -qF "is not this stack" "$2" && grep -qF "pnpm db:trial restore <pre-rotate backup>" "$2" && [ "$3" -eq "$4" ] && ! grep -q "^docker stop" "$5" && [ "$(cat "$6")" = SENTINEL-NEXT ]' _ "$RC" "$CASE/out" "$COUNT" "$(backup_count)" "$CASE/log" "$STATE_DIR/secrets.next.env"
|
||||
cp -p "$CASE/secrets.unforged" "$STATE_DIR/secrets.env"
|
||||
fresh_log; trial_tty "restore $RN" restore "$RN"
|
||||
check "restore removes a leftover secrets.next.env once it succeeded" bash -c '[ "$1" -eq 0 ] && [ ! -e "$2/secrets.next.env" ]' _ "$RC" "$STATE_DIR"
|
||||
printf 'SENTINEL-NEXT\n' > "$STATE_DIR/secrets.next.env"
|
||||
fresh_log; trial_tty faunosteo_trial destroy
|
||||
check "destroy removes a leftover secrets.next.env once it succeeded" bash -c '[ "$1" -eq 0 ] && [ ! -e "$2/secrets.next.env" ]' _ "$RC" "$STATE_DIR"
|
||||
|
||||
# -- destroy on a damaged database
|
||||
new_case b2-inventory; trial create
|
||||
: > "$CASE/stub/fail/inventory"; fresh_log; trial_tty faunosteo_trial destroy
|
||||
check "destroy whose inventory query fails still reaches the confirmation and works" bash -c '[ "$1" -eq 0 ] && grep -qF "(content unreadable)" "$2" && grep -qx "docker volume rm faunosteo_trial_db" "$3"' _ "$RC" "$CASE/out" "$CASE/log"
|
||||
|
||||
# -- rotate-secrets: the transaction, the commit window, the rename
|
||||
new_case b2-rotation; trial create
|
||||
cp -p "$STATE_DIR/secrets.env" "$CASE/secrets.before"
|
||||
CASE_PATH="$WORK/bin-mvfail:$BASE_PATH"; fresh_log; trial rotate-secrets
|
||||
check "rotate-secrets: a rename that fails after the commit still says both files are kept" bash -c '[ "$1" -ne 0 ] && grep -qF "secrets.env (previous) and secrets.next.env (new) are both kept" "$2" && ! grep -qF "secrets.env holds them" "$2" && [ -f "$3/secrets.next.env" ] && cmp -s "$3/secrets.env" "$4"' _ "$RC" "$CASE/out" "$STATE_DIR" "$CASE/secrets.before"
|
||||
CASE_PATH="$WORK/bin:$BASE_PATH"; rm -f "$STATE_DIR/secrets.next.env"
|
||||
: > "$CASE/stub/fail/rotate"; echo 1 > "$CASE/stub/answers/rotation-active"; fresh_log; trial rotate-secrets
|
||||
check "a rotation session still active after the limit: unknown, both files kept, nothing removed, the marker unread" bash -c '[ "$1" -ne 0 ] && grep -qF "whether the rotation committed is unknown" "$2" && grep -qF "secrets.env (previous) and secrets.next.env (new) are both kept" "$2" && grep -qE "restore [0-9]{8}T[0-9]{6}Z-pre-rotate" "$2" && [ -f "$3/secrets.next.env" ] && cmp -s "$3/secrets.env" "$4" && ! grep -q "^docker rm " "$5" && ! grep -qF "db-trial:rotation-marker" "$5" && [ "$6" -eq 10 ]' _ "$RC" "$CASE/out" "$STATE_DIR" "$CASE/secrets.before" "$CASE/log" "$(polls)"
|
||||
rm -f "$STATE_DIR/secrets.next.env"; echo 0 > "$CASE/stub/answers/rotation-active"; : > "$CASE/stub/fail/rotation-active"; fresh_log; trial rotate-secrets
|
||||
check "a failing session query: unknown, both files kept, the marker unread" bash -c '[ "$1" -ne 0 ] && grep -qF "whether the rotation committed is unknown" "$2" && [ -f "$3/secrets.next.env" ] && ! grep -qF "db-trial:rotation-marker" "$4"' _ "$RC" "$CASE/out" "$STATE_DIR" "$CASE/log"
|
||||
rm -f "$STATE_DIR/secrets.next.env" "$CASE/stub/fail/rotation-active"; printf '1\n1\n0\n' > "$CASE/stub/answers/rotation-active.seq"; fresh_log; trial rotate-secrets
|
||||
check "a rotation session active for two polls then gone, marker absent: not committed, the new file removed" bash -c '[ "$1" -ne 0 ] && grep -qF "did not take the new secrets" "$2" && [ ! -e "$3/secrets.next.env" ] && cmp -s "$3/secrets.env" "$4" && [ "$5" -eq 3 ]' _ "$RC" "$CASE/out" "$STATE_DIR" "$CASE/secrets.before" "$(polls)"
|
||||
rm -f "$CASE/stub/fail/rotate"; : > "$CASE/stub/commit-then-fail"; printf '1\n0\n' > "$CASE/stub/answers/rotation-active.seq"; fresh_log; trial rotate-secrets
|
||||
check "a rotation session active once then gone, marker equal: the rotation goes on" bash -c '[ "$1" -eq 0 ] && grep -qF "took the new secrets although psql failed" "$2" && [ "$3" -eq 2 ]' _ "$RC" "$CASE/out" "$(polls)"
|
||||
rm -f "$CASE/stub/commit-then-fail" "$CASE/stub/answers/rotation-marker"; fresh_log; trial rotate-secrets
|
||||
check "rotate-secrets: the SQL is one explicit transaction (begin before every alter, commit last)" sql_is_one_explicit_transaction "$(rotation_sql)"
|
||||
check "rotate-secrets: the rotation session is named (PGAPPNAME passed by name)" bash -c '[ "$1" -eq 0 ] && grep -qF -- "-e PGAPPNAME" "$2"' _ "$RC" "$CASE/log"
|
||||
check "rotate-secrets: no --single-transaction on the rotation session" no_single_transaction_rotation
|
||||
|
||||
# -- destroy refuses without a restorable backup, and names the way out
|
||||
no_removal() { ! grep -qE '^docker (rm|volume rm|network rm) ' "$CASE/log"; }
|
||||
new_case h5-way-one-volume; trial create
|
||||
rm -f "$CASE/stub/volumes/faunosteo_trial_storage"
|
||||
fresh_log; trial_tty faunosteo_trial destroy
|
||||
check "destroy with one volume missing: refused, nothing removed, the way out lists what exists" bash -c '[ "$1" -ne 0 ] && grep -qF "lacks storage.tar" "$2" && grep -qF "docker rm faunosteo_trial_db faunosteo_trial_mail" "$2" && grep -qF " docker network rm faunosteo_trial_net" "$2" && grep -qF " docker volume rm faunosteo_trial_db" "$2" && grep -qF "pnpm db:trial destroy" "$2" && grep -qF "pre-destroy" "$2"' _ "$RC" "$CASE/out"
|
||||
check "destroy with one volume missing: no removal was run" no_removal
|
||||
for n in db mail auth storage rest gateway; do rm -f "$CASE/stub/containers/faunosteo_trial_$n" "$CASE/stub/ports/faunosteo_trial_$n"; done
|
||||
rm -f "$CASE/stub/networks/faunosteo_trial_net" "$CASE/stub/volumes/faunosteo_trial_db"
|
||||
fresh_log; trial_tty faunosteo_trial destroy
|
||||
check "destroy after the manual removal: the state files go, backups kept" bash -c '[ "$1" -eq 0 ] && [ ! -e "$2/secrets.env" ] && [ -d "$2/backups" ] && ls -d "$2"/backups/*-pre-destroy* >/dev/null 2>&1' _ "$RC" "$STATE_DIR"
|
||||
new_case h5-way-space; trial create
|
||||
echo 999999999999 > "$CASE/stub/answers/measure-kib"
|
||||
fresh_log; trial_tty faunosteo_trial destroy
|
||||
check "destroy whose backup cannot be made: one failure message, the way out, nothing removed" bash -c '[ "$1" -ne 0 ] && [ "$(grep -c "backup failed" "$2")" -eq 1 ] && grep -qF "no backup could be made" "$2" && grep -qF " docker volume rm faunosteo_trial_db faunosteo_trial_storage" "$2" && grep -qF " docker network rm faunosteo_trial_net" "$2" && grep -qF "pnpm db:trial destroy" "$2"' _ "$RC" "$CASE/out"
|
||||
check "destroy whose backup cannot be made: no removal was run" no_removal
|
||||
new_case h5-way-forged; trial create
|
||||
forge TR_SERVICE_KEY faunosteo-trial other service_role
|
||||
fresh_log; trial_tty faunosteo_trial destroy
|
||||
check "destroy with a forged key: refused with the way out, the backup named" bash -c '[ "$1" -ne 0 ] && grep -qF "not signed with its own secret" "$2" && grep -qE "backup [0-9]{8}T[0-9]{6}Z-pre-destroy[a-z0-9-]* is kept" "$2" && grep -qF " docker network rm faunosteo_trial_net" "$2" && grep -qF "pnpm db:trial destroy" "$2"' _ "$RC" "$CASE/out"
|
||||
check "destroy with a forged key: no removal was run" no_removal
|
||||
|
||||
# -- destroy interrupted mid-removal names its backup
|
||||
new_case h5-destroy-term; trial create
|
||||
echo '^docker volume rm ' > "$CASE/stub/term-on"; fresh_log; trial_tty faunosteo_trial destroy
|
||||
check "destroy terminated by SIGTERM mid-removal: one message naming the pre-destroy backup" bash -c '[ "$(grep -c "destroy stopped part-way" "$1")" -eq 1 ] && grep -qE "restore [0-9]{8}T[0-9]{6}Z-pre-destroy" "$1"' _ "$CASE/out"
|
||||
|
||||
# -- the rotations' traps follow the state
|
||||
new_case h5-rotation-window; trial create
|
||||
CASE_PATH="$WORK/bin-mvterm:$BASE_PATH"; fresh_log; trial rotate-secrets
|
||||
check "rotate-secrets terminated between the rename and the flag: the committed branch, from the state" bash -c '[ "$1" -ne 0 ] && grep -qF "stopped after the database took the new secrets" "$2" && ! grep -qF "stopped before" "$2" && [ ! -e "$3/secrets.next.env" ]' _ "$RC" "$CASE/out" "$STATE_DIR"
|
||||
new_case h5-keys-before-rename; trial create
|
||||
cp -p "$STATE_DIR/secrets.env" "$CASE/secrets.before"
|
||||
CASE_PATH="$WORK/bin-mvterm:$BASE_PATH"; fresh_log; trial rotate-keys
|
||||
check "rotate-keys terminated before the rename: no 'holds the new keys' message, secrets.env unchanged" bash -c '[ "$1" -ne 0 ] && ! grep -qF "holds the new keys" "$2" && cmp -s "$3" "$4"' _ "$RC" "$CASE/out" "$STATE_DIR/secrets.env" "$CASE/secrets.before"
|
||||
|
||||
# -- restore: a leftover helper without a trial volume is refused before the confirmation
|
||||
b2_stack h5-restore-helper
|
||||
trial_tty faunosteo_trial destroy
|
||||
: > "$CASE/stub/containers/faunosteo_trial_helper"
|
||||
fresh_log; trial_tty "restore $RN" restore "$RN"
|
||||
check "restore with a leftover helper and no trial volume: refused before the confirmation, the hint named" bash -c '[ "$1" -ne 0 ] && grep -qF "docker rm faunosteo_trial_helper" "$2" && ! grep -qF "to go on" "$2" && ! grep -qE "^docker (run|volume|rm|network)" "$3"' _ "$RC" "$CASE/out" "$CASE/log"
|
||||
|
||||
# -- a leftover secrets.next.env is kept by backups and brought back by restore
|
||||
new_case h5-next-backup; trial create
|
||||
printf 'SENTINEL-NEXT\n' > "$STATE_DIR/secrets.next.env"; chmod 600 "$STATE_DIR/secrets.next.env"
|
||||
trial backup
|
||||
RB="$(find "$STATE_DIR/backups" -mindepth 1 -maxdepth 1 -type d -name '*-manual' | head -n 1)"; RN="$(basename "$RB")"
|
||||
check "backup with a leftover secrets.next.env holds it, listed in SHA256SUMS" bash -c '[ "$(cat "$1/secrets.next.env")" = SENTINEL-NEXT ] && grep -qF " secrets.next.env" "$1/SHA256SUMS"' _ "$RB"
|
||||
rm -f "$STATE_DIR/secrets.next.env"
|
||||
fresh_log; trial_tty "restore $RN" restore "$RN"
|
||||
check "restore of that backup brings secrets.next.env back (600) and status flags it" bash -c '[ "$1" -eq 0 ] && [ "$(cat "$2/secrets.next.env")" = SENTINEL-NEXT ] && [ "$(stat -c %a "$2/secrets.next.env")" = 600 ]' _ "$RC" "$STATE_DIR"
|
||||
fresh_log; trial status
|
||||
check "status flags the restored secrets.next.env" out_has "secrets.next.env exists"
|
||||
fresh_log; trial_tty "restore $RN" restore "$RN"
|
||||
check "restore with a leftover secrets.next.env: the pre-restore backup note mentions it" bash -c '[ "$1" -eq 0 ] && grep -qF "also holds a leftover secrets.next.env" "$2"' _ "$RC" "$CASE/out"
|
||||
|
||||
echo "-- SIGTERM after the trap is set: one message"
|
||||
new_case term-create; echo '^docker volume create .* faunosteo_trial_db$' > "$CASE/stub/term-on"; fresh_log; trial create
|
||||
check "create terminated by SIGTERM: one message, nothing removed" bash -c '[ "$(grep -c "create stopped before the end" "$1")" -eq 1 ]' _ "$CASE/out"
|
||||
new_case term-migrate; trial create
|
||||
printf -- '-- harness\n' > "$WORK/repo/supabase/migrations/29991231000000_harness_term.sql"
|
||||
: > "$CASE/stub/term-in-migrate"; fresh_log; trial migrate
|
||||
rm -f "$WORK/repo/supabase/migrations/29991231000000_harness_term.sql"
|
||||
check "migrate terminated by SIGTERM: one message naming the pre-migrate backup" bash -c '[ "$(grep -c "migrate stopped part-way" "$1")" -eq 1 ] && grep -qE "restore [0-9]{8}T[0-9]{6}Z-pre-migrate" "$1"' _ "$CASE/out"
|
||||
new_case term-keys; trial create
|
||||
echo '^docker run -d --name faunosteo_trial_storage ' > "$CASE/stub/term-on"; fresh_log; trial rotate-keys
|
||||
check "rotate-keys terminated by SIGTERM: one message" bash -c '[ "$(grep -c "rotate-keys stopped part-way" "$1")" -eq 1 ]' _ "$CASE/out"
|
||||
new_case term-rotation; trial create
|
||||
echo '^tag:rotate$' > "$CASE/stub/term-on"; fresh_log; trial rotate-secrets
|
||||
check "rotate-secrets terminated by SIGTERM: one message, the uncommitted branch" bash -c '[ "$(grep -c "rotate-secrets stopped" "$1")" -eq 1 ] && grep -qF "stopped before secrets.env took the new secrets" "$1"' _ "$CASE/out"
|
||||
|
||||
echo "-- db:sandbox state guard"
|
||||
new_case sandbox-guard; mkdir -p "$CASE/trialstate"; : > "$CASE/trialstate/.faunosteo-trial-state"
|
||||
EXTRA=(FAUNOSTEO_SANDBOX_DIR="$CASE/trialstate"); sandbox down
|
||||
|
||||
Reference in new issue
Block a user