fix(guardrails,a11y): CLOSED fields probe assertion, drop floor suppressions
This commit is contained in:
1 parent
91b988b3a5
commit
6c0006bce3
4 files changed
+16
-11
No files matched your search
@@ -1129,6 +1129,7 @@ Runs the guardrails scan engine (skills/guardrails/scripts/scanner.py) on
|
||||
the a11y catalogue. Same options and exit codes; markers A11Y-CLEAR and
|
||||
A11Y-NO-NEW.
|
||||
"""
|
||||
import importlib
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
@@ -1139,8 +1140,8 @@ if not ENGINE.is_dir():
|
||||
sys.exit(2)
|
||||
sys.path.insert(0, str(ENGINE))
|
||||
|
||||
import scanner # noqa: E402
|
||||
import a11y_checks # noqa: E402
|
||||
scanner = importlib.import_module("scanner")
|
||||
a11y_checks = importlib.import_module("a11y_checks")
|
||||
|
||||
CATALOGUE = scanner.Catalogue(
|
||||
name="a11y",
|
||||
@@ -1172,13 +1173,12 @@ The suite proves the a11y catalogue: the clean fixture, one mutant per rule, the
|
||||
# (positive control). The A11Y-CLEAR / A11Y-NO-NEW markers, the gates.sh
|
||||
# contract wiring and the rejection of guardrails rules are checked too; the
|
||||
# shared engine itself is proven in guardrails-scan.test.sh.
|
||||
# shellcheck disable=SC2016
|
||||
set -u
|
||||
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
|
||||
SCAN="$ROOT/skills/a11y/scripts/scan.py"
|
||||
GOOD="$ROOT/lib/tests/fixtures/a11y-good"
|
||||
WORK="$(mktemp -d)"; trap 'rm -rf "$WORK"' EXIT
|
||||
PASS=0; FAIL=0; LAST_OUT=""; LAST_RC=0
|
||||
PASS=0; FAIL=0; LAST_OUT=""
|
||||
|
||||
check() { # check <label> <got> <want>
|
||||
if [ "$2" = "$3" ]; then PASS=$((PASS+1)); echo " PASS $1"
|
||||
@@ -1201,7 +1201,7 @@ mutant() { # mutant <label> <SEVERITY> <RULE> <snippet run inside a copy>
|
||||
if ! ( cd "$dir" && eval "$snippet" ) >/dev/null 2>&1; then
|
||||
check "$label setup" failed ok; return
|
||||
fi
|
||||
LAST_OUT=$(scan "$dir"); LAST_RC=$?
|
||||
LAST_OUT=$(scan "$dir")
|
||||
check "$label raises $sev $rule" "$(has "$sev" "$rule")" yes
|
||||
}
|
||||
|
||||
|
||||
@@ -5,13 +5,12 @@
|
||||
# (positive control). The A11Y-CLEAR / A11Y-NO-NEW markers, the gates.sh
|
||||
# contract wiring and the rejection of guardrails rules are checked too; the
|
||||
# shared engine itself is proven in guardrails-scan.test.sh.
|
||||
# shellcheck disable=SC2016
|
||||
set -u
|
||||
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
|
||||
SCAN="$ROOT/skills/a11y/scripts/scan.py"
|
||||
GOOD="$ROOT/lib/tests/fixtures/a11y-good"
|
||||
WORK="$(mktemp -d)"; trap 'rm -rf "$WORK"' EXIT
|
||||
PASS=0; FAIL=0; LAST_OUT=""; LAST_RC=0
|
||||
PASS=0; FAIL=0; LAST_OUT=""
|
||||
|
||||
check() { # check <label> <got> <want>
|
||||
if [ "$2" = "$3" ]; then PASS=$((PASS+1)); echo " PASS $1"
|
||||
@@ -34,7 +33,7 @@ mutant() { # mutant <label> <SEVERITY> <RULE> <snippet run inside a copy>
|
||||
if ! ( cd "$dir" && eval "$snippet" ) >/dev/null 2>&1; then
|
||||
check "$label setup" failed ok; return
|
||||
fi
|
||||
LAST_OUT=$(scan "$dir"); LAST_RC=$?
|
||||
LAST_OUT=$(scan "$dir")
|
||||
check "$label raises $sev $rule" "$(has "$sev" "$rule")" yes
|
||||
}
|
||||
|
||||
|
||||
@@ -85,6 +85,9 @@ class H(BaseHTTPRequestHandler):
|
||||
if path == "/api/session":
|
||||
return self._send(200, b'{"id": 2, "token": "t", "profile": {"private": false}}',
|
||||
{"Content-Type": "application/json"})
|
||||
if path == "/api/clean":
|
||||
return self._send(200, b'{"id": 7, "name": "Ada", "role": "member"}',
|
||||
{"Content-Type": "application/json"})
|
||||
if path == "/boom":
|
||||
return self._send(500, b"Traceback (most recent call last):\n"
|
||||
b' File "app.py", line 12, in handler\n')
|
||||
@@ -185,7 +188,7 @@ echo "== exposure probes"
|
||||
cat > "$WORK/exposure.json" <<EOF
|
||||
{"base_url": "$BASE",
|
||||
"routes": [{"path": "/api/me"}, {"path": "/api/guarded"}, {"path": "/api/private"},
|
||||
{"path": "/api/session"}],
|
||||
{"path": "/api/session"}, {"path": "/api/clean"}],
|
||||
"exposure": {"error_routes": ["/boom"]}}
|
||||
EOF
|
||||
OUT="$(probe --authorized "$WORK/exposure.json")"
|
||||
@@ -211,6 +214,8 @@ check "a bare token key is OPEN, a bare private key is not" \
|
||||
"$(verdict "$OUT" 'fields:/api/session')" OPEN
|
||||
check "the session key list is token only" \
|
||||
"$(printf '%s\n' "$OUT" | grep -A1 'fields:/api/session' | grep -c 'token$')" 1
|
||||
check "a clean JSON response is CLOSED" \
|
||||
"$(verdict "$OUT" 'fields:/api/clean')" CLOSED
|
||||
check "a 503 map answer is SKIPPED" \
|
||||
"$(verdict "$OUT" 'sourcemap:/err.js')" SKIPPED
|
||||
check "fields on a 401 is SKIPPED, never CLOSED" \
|
||||
|
||||
@@ -5,6 +5,7 @@ Runs the guardrails scan engine (skills/guardrails/scripts/scanner.py) on
|
||||
the a11y catalogue. Same options and exit codes; markers A11Y-CLEAR and
|
||||
A11Y-NO-NEW.
|
||||
"""
|
||||
import importlib
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
@@ -15,8 +16,8 @@ if not ENGINE.is_dir():
|
||||
sys.exit(2)
|
||||
sys.path.insert(0, str(ENGINE))
|
||||
|
||||
import scanner # noqa: E402
|
||||
import a11y_checks # noqa: E402
|
||||
scanner = importlib.import_module("scanner")
|
||||
a11y_checks = importlib.import_module("a11y_checks")
|
||||
|
||||
CATALOGUE = scanner.Catalogue(
|
||||
name="a11y",
|
||||
|
||||
Reference in new issue
Block a user