fix(guardrails,a11y): CLOSED fields probe assertion, drop floor suppressions

This commit is contained in:
bchanot committed 2026-10-19 10:52:27 +02:00
1 parent 91b988b3a5
commit 6c0006bce3
4 files changed
+16 -11

No files matched your search

@@ -1129,6 +1129,7 @@ Runs the guardrails scan engine (skills/guardrails/scripts/scanner.py) on
the a11y catalogue. Same options and exit codes; markers A11Y-CLEAR and
A11Y-NO-NEW.
"""
import importlib
import sys
from pathlib import Path
@@ -1139,8 +1140,8 @@ if not ENGINE.is_dir():
sys.exit(2)
sys.path.insert(0, str(ENGINE))
import scanner # noqa: E402
import a11y_checks # noqa: E402
scanner = importlib.import_module("scanner")
a11y_checks = importlib.import_module("a11y_checks")
CATALOGUE = scanner.Catalogue(
name="a11y",
@@ -1172,13 +1173,12 @@ The suite proves the a11y catalogue: the clean fixture, one mutant per rule, the
# (positive control). The A11Y-CLEAR / A11Y-NO-NEW markers, the gates.sh
# contract wiring and the rejection of guardrails rules are checked too; the
# shared engine itself is proven in guardrails-scan.test.sh.
# shellcheck disable=SC2016
set -u
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
SCAN="$ROOT/skills/a11y/scripts/scan.py"
GOOD="$ROOT/lib/tests/fixtures/a11y-good"
WORK="$(mktemp -d)"; trap 'rm -rf "$WORK"' EXIT
PASS=0; FAIL=0; LAST_OUT=""; LAST_RC=0
PASS=0; FAIL=0; LAST_OUT=""
check() { # check <label> <got> <want>
if [ "$2" = "$3" ]; then PASS=$((PASS+1)); echo " PASS $1"
@@ -1201,7 +1201,7 @@ mutant() { # mutant <label> <SEVERITY> <RULE> <snippet run inside a copy>
if ! ( cd "$dir" && eval "$snippet" ) >/dev/null 2>&1; then
check "$label setup" failed ok; return
fi
LAST_OUT=$(scan "$dir"); LAST_RC=$?
LAST_OUT=$(scan "$dir")
check "$label raises $sev $rule" "$(has "$sev" "$rule")" yes
}
+2 -3
View File
@@ -5,13 +5,12 @@
# (positive control). The A11Y-CLEAR / A11Y-NO-NEW markers, the gates.sh
# contract wiring and the rejection of guardrails rules are checked too; the
# shared engine itself is proven in guardrails-scan.test.sh.
# shellcheck disable=SC2016
set -u
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
SCAN="$ROOT/skills/a11y/scripts/scan.py"
GOOD="$ROOT/lib/tests/fixtures/a11y-good"
WORK="$(mktemp -d)"; trap 'rm -rf "$WORK"' EXIT
PASS=0; FAIL=0; LAST_OUT=""; LAST_RC=0
PASS=0; FAIL=0; LAST_OUT=""
check() { # check <label> <got> <want>
if [ "$2" = "$3" ]; then PASS=$((PASS+1)); echo " PASS $1"
@@ -34,7 +33,7 @@ mutant() { # mutant <label> <SEVERITY> <RULE> <snippet run inside a copy>
if ! ( cd "$dir" && eval "$snippet" ) >/dev/null 2>&1; then
check "$label setup" failed ok; return
fi
LAST_OUT=$(scan "$dir"); LAST_RC=$?
LAST_OUT=$(scan "$dir")
check "$label raises $sev $rule" "$(has "$sev" "$rule")" yes
}
+6 -1
View File
@@ -85,6 +85,9 @@ class H(BaseHTTPRequestHandler):
if path == "/api/session":
return self._send(200, b'{"id": 2, "token": "t", "profile": {"private": false}}',
{"Content-Type": "application/json"})
if path == "/api/clean":
return self._send(200, b'{"id": 7, "name": "Ada", "role": "member"}',
{"Content-Type": "application/json"})
if path == "/boom":
return self._send(500, b"Traceback (most recent call last):\n"
b' File "app.py", line 12, in handler\n')
@@ -185,7 +188,7 @@ echo "== exposure probes"
cat > "$WORK/exposure.json" <<EOF
{"base_url": "$BASE",
"routes": [{"path": "/api/me"}, {"path": "/api/guarded"}, {"path": "/api/private"},
{"path": "/api/session"}],
{"path": "/api/session"}, {"path": "/api/clean"}],
"exposure": {"error_routes": ["/boom"]}}
EOF
OUT="$(probe --authorized "$WORK/exposure.json")"
@@ -211,6 +214,8 @@ check "a bare token key is OPEN, a bare private key is not" \
"$(verdict "$OUT" 'fields:/api/session')" OPEN
check "the session key list is token only" \
"$(printf '%s\n' "$OUT" | grep -A1 'fields:/api/session' | grep -c 'token$')" 1
check "a clean JSON response is CLOSED" \
"$(verdict "$OUT" 'fields:/api/clean')" CLOSED
check "a 503 map answer is SKIPPED" \
"$(verdict "$OUT" 'sourcemap:/err.js')" SKIPPED
check "fields on a 401 is SKIPPED, never CLOSED" \
+3 -2
View File
@@ -5,6 +5,7 @@ Runs the guardrails scan engine (skills/guardrails/scripts/scanner.py) on
the a11y catalogue. Same options and exit codes; markers A11Y-CLEAR and
A11Y-NO-NEW.
"""
import importlib
import sys
from pathlib import Path
@@ -15,8 +16,8 @@ if not ENGINE.is_dir():
sys.exit(2)
sys.path.insert(0, str(ENGINE))
import scanner # noqa: E402
import a11y_checks # noqa: E402
scanner = importlib.import_module("scanner")
a11y_checks = importlib.import_module("a11y_checks")
CATALOGUE = scanner.Catalogue(
name="a11y",