Files
claude/lib/tests/guardrails-probe.test.sh
T
bchanot f0bcbfef46 feat(guardrails): read-only live probe to confirm a hole from outside
probe.py observes a running target the user owns and names, driven by a
plan built from the static findings. Five probes: anon-read, cross-user,
no-auth route, CORS reflection, capped rate-limit burst. GET/HEAD/OPTIONS
only, --authorized gate, secrets from the environment, same-host redirect
only, bounded request budget. SKILL STEP 5, checks.md reference, docs.
Suite guardrails-probe.test.sh (16 checks) against a localhost fixture.
2026-10-18 18:49:51 +02:00

157 lines
5.6 KiB
Bash

#!/usr/bin/env bash
# lib/tests/guardrails-probe.test.sh — floor of the /guardrails live probe.
# A throwaway http.server on 127.0.0.1 serves scripted responses; the probe
# runs against it and each verdict is asserted. Nothing leaves the machine.
# The probe sends read-only methods; this suite proves the OPEN/CLOSED logic,
# the --authorized gate, the same-host redirect guard and the request budget.
set -u
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
PROBE="$ROOT/skills/guardrails/scripts/probe.py"
WORK="$(mktemp -d)"
PASS=0; FAIL=0; SRV=""
cleanup() { [ -n "$SRV" ] && kill "$SRV" 2>/dev/null; rm -rf "$WORK"; }
trap cleanup EXIT
check() { # check <label> <got> <want>
if [ "$2" = "$3" ]; then PASS=$((PASS+1)); echo " PASS $1"
else FAIL=$((FAIL+1)); echo " FAIL $1 (got '$2', want '$3')"; fi
}
# Fixture server: a fixed map of path -> (status, body, headers). Read-only,
# bound to the loopback, on an OS-chosen port it prints on the first line.
cat > "$WORK/server.py" <<'PY'
import json
from http.server import BaseHTTPRequestHandler, HTTPServer
HITS = {"rl": 0}
class H(BaseHTTPRequestHandler):
def log_message(self, *a):
pass
def _send(self, code, body=b"", headers=None):
self.send_response(code)
for k, v in (headers or {}).items():
self.send_header(k, v)
self.end_headers()
self.wfile.write(body)
def do_OPTIONS(self):
origin = self.headers.get("Origin", "")
self._send(204, b"", {
"Access-Control-Allow-Origin": origin,
"Access-Control-Allow-Credentials": "true"})
def do_GET(self):
path = self.path.split("?")[0]
if path == "/rest/v1/open_table":
return self._send(200, b'[{"id": 1}]')
if path == "/rest/v1/closed_table":
return self._send(200, b'[]')
if path == "/api/private":
return self._send(200, b"secret")
if path == "/api/guarded":
return self._send(401, b"no")
if path == "/rl":
HITS["rl"] += 1
return self._send(429 if HITS["rl"] > 5 else 200, b"")
if path == "/nolimit":
return self._send(200, b"")
return self._send(404, b"")
srv = HTTPServer(("127.0.0.1", 0), H)
print(srv.server_port, flush=True)
srv.serve_forever()
PY
python3 -B "$WORK/server.py" > "$WORK/port" &
SRV=$!
for _ in $(seq 1 50); do [ -s "$WORK/port" ] && break; sleep 0.1; done
PORT="$(head -1 "$WORK/port")"
if [ -z "$PORT" ]; then echo " FAIL fixture server did not start"; exit 1; fi
BASE="http://127.0.0.1:$PORT"
probe() { python3 -B "$PROBE" "$@"; }
verdict() { printf '%s\n' "$1" | grep -A1 "$2" | grep -oE 'OPEN|CLOSED|SKIPPED|ERROR' | head -1; }
echo "== authorization gate"
cat > "$WORK/plan.json" <<EOF
{"base_url": "$BASE",
"supabase": {"url": "$BASE", "tables": ["open_table", "closed_table"],
"cross_user_paths": ["/rest/v1/open_table"]},
"routes": [{"path": "/api/private"}, {"path": "/api/guarded"},
{"path": "/api/public", "public": true}],
"cors": true}
EOF
probe "$WORK/plan.json" >/dev/null 2>&1
check "refuses without --authorized (exit 2)" "$?" 2
echo "== read-only probes against the fixture"
export GUARDRAILS_ANON_KEY="anon-test-key"
export GUARDRAILS_JWT2="second-user-test-token"
OUT="$(probe --authorized "$WORK/plan.json")"
RC=$?
check "anon read of an open table is OPEN" \
"$(verdict "$OUT" 'anon-read:open_table')" OPEN
check "anon read of a closed table is CLOSED" \
"$(verdict "$OUT" 'anon-read:closed_table')" CLOSED
check "cross-user read returning rows is OPEN" \
"$(verdict "$OUT" 'cross-user:')" OPEN
check "unauth route returning 200 is OPEN" \
"$(verdict "$OUT" 'no-auth:/api/private')" OPEN
check "unauth route returning 401 is CLOSED" \
"$(verdict "$OUT" 'no-auth:/api/guarded')" CLOSED
check "public route is not probed" \
"$(printf '%s\n' "$OUT" | grep -c 'no-auth:/api/public')" 0
check "reflected origin with credentials is OPEN" \
"$(verdict "$OUT" 'cors-reflect')" OPEN
check "a found OPEN verdict exits 1" "$RC" 1
echo "== skips when a secret is absent"
unset GUARDRAILS_ANON_KEY
OUT="$(probe --authorized "$WORK/plan.json")"
check "anon-read skips without the key" \
"$(verdict "$OUT" 'anon-read')" SKIPPED
export GUARDRAILS_ANON_KEY="anon-test-key"
echo "== rate limit"
cat > "$WORK/rl.json" <<EOF
{"base_url": "$BASE", "rate_limit": {"path": "/rl", "count": 10}}
EOF
OUT="$(probe --authorized "$WORK/rl.json")"
check "a route that starts sending 429 is CLOSED" \
"$(verdict "$OUT" 'rate-limit')" CLOSED
cat > "$WORK/nolimit.json" <<EOF
{"base_url": "$BASE", "rate_limit": {"path": "/nolimit", "count": 10}}
EOF
OUT="$(probe --authorized "$WORK/nolimit.json")"
check "a route with no 429 is OPEN" \
"$(verdict "$OUT" 'rate-limit')" OPEN
echo "== budget caps the burst"
cat > "$WORK/burst.json" <<EOF
{"base_url": "$BASE", "rate_limit": {"path": "/nolimit", "count": 99}}
EOF
OUT="$(probe --authorized "$WORK/burst.json")"
check "the burst is capped, never 99 requests" \
"$(printf '%s\n' "$OUT" | grep -oE 'sent [0-9]+' | grep -oE '[0-9]+')" 25
echo "== usage errors"
probe --authorized "$WORK/missing.json" >/dev/null 2>&1
check "a missing plan exits 2" "$?" 2
printf '{"base_url": "%s", "rate_limit": {"path": "/x", "method": "DELETE"}}\n' \
"$BASE" > "$WORK/bad.json"
probe --authorized "$WORK/bad.json" >/dev/null 2>&1
check "a write method in the plan exits 2" "$?" 2
echo "== JSON output"
check "json reports the open count" \
"$(probe --authorized --json "$WORK/plan.json" \
| python3 -B -c 'import json,sys; print(json.load(sys.stdin)["open"])')" 4
echo "---"
echo "passed: $PASS failed: $FAIL"
[ "$FAIL" -eq 0 ]