forked from bchanot/claude
fix(guardrails,a11y): CLOSED fields probe assertion, drop floor suppressions
This commit is contained in:
1 parent
91b988b3a5
commit
6c0006bce3
4 files changed
+16
-11
No files matched your search
@@ -1129,6 +1129,7 @@ Runs the guardrails scan engine (skills/guardrails/scripts/scanner.py) on
|
|||||||
the a11y catalogue. Same options and exit codes; markers A11Y-CLEAR and
|
the a11y catalogue. Same options and exit codes; markers A11Y-CLEAR and
|
||||||
A11Y-NO-NEW.
|
A11Y-NO-NEW.
|
||||||
"""
|
"""
|
||||||
|
import importlib
|
||||||
import sys
|
import sys
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
@@ -1139,8 +1140,8 @@ if not ENGINE.is_dir():
|
|||||||
sys.exit(2)
|
sys.exit(2)
|
||||||
sys.path.insert(0, str(ENGINE))
|
sys.path.insert(0, str(ENGINE))
|
||||||
|
|
||||||
import scanner # noqa: E402
|
scanner = importlib.import_module("scanner")
|
||||||
import a11y_checks # noqa: E402
|
a11y_checks = importlib.import_module("a11y_checks")
|
||||||
|
|
||||||
CATALOGUE = scanner.Catalogue(
|
CATALOGUE = scanner.Catalogue(
|
||||||
name="a11y",
|
name="a11y",
|
||||||
@@ -1172,13 +1173,12 @@ The suite proves the a11y catalogue: the clean fixture, one mutant per rule, the
|
|||||||
# (positive control). The A11Y-CLEAR / A11Y-NO-NEW markers, the gates.sh
|
# (positive control). The A11Y-CLEAR / A11Y-NO-NEW markers, the gates.sh
|
||||||
# contract wiring and the rejection of guardrails rules are checked too; the
|
# contract wiring and the rejection of guardrails rules are checked too; the
|
||||||
# shared engine itself is proven in guardrails-scan.test.sh.
|
# shared engine itself is proven in guardrails-scan.test.sh.
|
||||||
# shellcheck disable=SC2016
|
|
||||||
set -u
|
set -u
|
||||||
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
|
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
|
||||||
SCAN="$ROOT/skills/a11y/scripts/scan.py"
|
SCAN="$ROOT/skills/a11y/scripts/scan.py"
|
||||||
GOOD="$ROOT/lib/tests/fixtures/a11y-good"
|
GOOD="$ROOT/lib/tests/fixtures/a11y-good"
|
||||||
WORK="$(mktemp -d)"; trap 'rm -rf "$WORK"' EXIT
|
WORK="$(mktemp -d)"; trap 'rm -rf "$WORK"' EXIT
|
||||||
PASS=0; FAIL=0; LAST_OUT=""; LAST_RC=0
|
PASS=0; FAIL=0; LAST_OUT=""
|
||||||
|
|
||||||
check() { # check <label> <got> <want>
|
check() { # check <label> <got> <want>
|
||||||
if [ "$2" = "$3" ]; then PASS=$((PASS+1)); echo " PASS $1"
|
if [ "$2" = "$3" ]; then PASS=$((PASS+1)); echo " PASS $1"
|
||||||
@@ -1201,7 +1201,7 @@ mutant() { # mutant <label> <SEVERITY> <RULE> <snippet run inside a copy>
|
|||||||
if ! ( cd "$dir" && eval "$snippet" ) >/dev/null 2>&1; then
|
if ! ( cd "$dir" && eval "$snippet" ) >/dev/null 2>&1; then
|
||||||
check "$label setup" failed ok; return
|
check "$label setup" failed ok; return
|
||||||
fi
|
fi
|
||||||
LAST_OUT=$(scan "$dir"); LAST_RC=$?
|
LAST_OUT=$(scan "$dir")
|
||||||
check "$label raises $sev $rule" "$(has "$sev" "$rule")" yes
|
check "$label raises $sev $rule" "$(has "$sev" "$rule")" yes
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -5,13 +5,12 @@
|
|||||||
# (positive control). The A11Y-CLEAR / A11Y-NO-NEW markers, the gates.sh
|
# (positive control). The A11Y-CLEAR / A11Y-NO-NEW markers, the gates.sh
|
||||||
# contract wiring and the rejection of guardrails rules are checked too; the
|
# contract wiring and the rejection of guardrails rules are checked too; the
|
||||||
# shared engine itself is proven in guardrails-scan.test.sh.
|
# shared engine itself is proven in guardrails-scan.test.sh.
|
||||||
# shellcheck disable=SC2016
|
|
||||||
set -u
|
set -u
|
||||||
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
|
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
|
||||||
SCAN="$ROOT/skills/a11y/scripts/scan.py"
|
SCAN="$ROOT/skills/a11y/scripts/scan.py"
|
||||||
GOOD="$ROOT/lib/tests/fixtures/a11y-good"
|
GOOD="$ROOT/lib/tests/fixtures/a11y-good"
|
||||||
WORK="$(mktemp -d)"; trap 'rm -rf "$WORK"' EXIT
|
WORK="$(mktemp -d)"; trap 'rm -rf "$WORK"' EXIT
|
||||||
PASS=0; FAIL=0; LAST_OUT=""; LAST_RC=0
|
PASS=0; FAIL=0; LAST_OUT=""
|
||||||
|
|
||||||
check() { # check <label> <got> <want>
|
check() { # check <label> <got> <want>
|
||||||
if [ "$2" = "$3" ]; then PASS=$((PASS+1)); echo " PASS $1"
|
if [ "$2" = "$3" ]; then PASS=$((PASS+1)); echo " PASS $1"
|
||||||
@@ -34,7 +33,7 @@ mutant() { # mutant <label> <SEVERITY> <RULE> <snippet run inside a copy>
|
|||||||
if ! ( cd "$dir" && eval "$snippet" ) >/dev/null 2>&1; then
|
if ! ( cd "$dir" && eval "$snippet" ) >/dev/null 2>&1; then
|
||||||
check "$label setup" failed ok; return
|
check "$label setup" failed ok; return
|
||||||
fi
|
fi
|
||||||
LAST_OUT=$(scan "$dir"); LAST_RC=$?
|
LAST_OUT=$(scan "$dir")
|
||||||
check "$label raises $sev $rule" "$(has "$sev" "$rule")" yes
|
check "$label raises $sev $rule" "$(has "$sev" "$rule")" yes
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -85,6 +85,9 @@ class H(BaseHTTPRequestHandler):
|
|||||||
if path == "/api/session":
|
if path == "/api/session":
|
||||||
return self._send(200, b'{"id": 2, "token": "t", "profile": {"private": false}}',
|
return self._send(200, b'{"id": 2, "token": "t", "profile": {"private": false}}',
|
||||||
{"Content-Type": "application/json"})
|
{"Content-Type": "application/json"})
|
||||||
|
if path == "/api/clean":
|
||||||
|
return self._send(200, b'{"id": 7, "name": "Ada", "role": "member"}',
|
||||||
|
{"Content-Type": "application/json"})
|
||||||
if path == "/boom":
|
if path == "/boom":
|
||||||
return self._send(500, b"Traceback (most recent call last):\n"
|
return self._send(500, b"Traceback (most recent call last):\n"
|
||||||
b' File "app.py", line 12, in handler\n')
|
b' File "app.py", line 12, in handler\n')
|
||||||
@@ -185,7 +188,7 @@ echo "== exposure probes"
|
|||||||
cat > "$WORK/exposure.json" <<EOF
|
cat > "$WORK/exposure.json" <<EOF
|
||||||
{"base_url": "$BASE",
|
{"base_url": "$BASE",
|
||||||
"routes": [{"path": "/api/me"}, {"path": "/api/guarded"}, {"path": "/api/private"},
|
"routes": [{"path": "/api/me"}, {"path": "/api/guarded"}, {"path": "/api/private"},
|
||||||
{"path": "/api/session"}],
|
{"path": "/api/session"}, {"path": "/api/clean"}],
|
||||||
"exposure": {"error_routes": ["/boom"]}}
|
"exposure": {"error_routes": ["/boom"]}}
|
||||||
EOF
|
EOF
|
||||||
OUT="$(probe --authorized "$WORK/exposure.json")"
|
OUT="$(probe --authorized "$WORK/exposure.json")"
|
||||||
@@ -211,6 +214,8 @@ check "a bare token key is OPEN, a bare private key is not" \
|
|||||||
"$(verdict "$OUT" 'fields:/api/session')" OPEN
|
"$(verdict "$OUT" 'fields:/api/session')" OPEN
|
||||||
check "the session key list is token only" \
|
check "the session key list is token only" \
|
||||||
"$(printf '%s\n' "$OUT" | grep -A1 'fields:/api/session' | grep -c 'token$')" 1
|
"$(printf '%s\n' "$OUT" | grep -A1 'fields:/api/session' | grep -c 'token$')" 1
|
||||||
|
check "a clean JSON response is CLOSED" \
|
||||||
|
"$(verdict "$OUT" 'fields:/api/clean')" CLOSED
|
||||||
check "a 503 map answer is SKIPPED" \
|
check "a 503 map answer is SKIPPED" \
|
||||||
"$(verdict "$OUT" 'sourcemap:/err.js')" SKIPPED
|
"$(verdict "$OUT" 'sourcemap:/err.js')" SKIPPED
|
||||||
check "fields on a 401 is SKIPPED, never CLOSED" \
|
check "fields on a 401 is SKIPPED, never CLOSED" \
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ Runs the guardrails scan engine (skills/guardrails/scripts/scanner.py) on
|
|||||||
the a11y catalogue. Same options and exit codes; markers A11Y-CLEAR and
|
the a11y catalogue. Same options and exit codes; markers A11Y-CLEAR and
|
||||||
A11Y-NO-NEW.
|
A11Y-NO-NEW.
|
||||||
"""
|
"""
|
||||||
|
import importlib
|
||||||
import sys
|
import sys
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
@@ -15,8 +16,8 @@ if not ENGINE.is_dir():
|
|||||||
sys.exit(2)
|
sys.exit(2)
|
||||||
sys.path.insert(0, str(ENGINE))
|
sys.path.insert(0, str(ENGINE))
|
||||||
|
|
||||||
import scanner # noqa: E402
|
scanner = importlib.import_module("scanner")
|
||||||
import a11y_checks # noqa: E402
|
a11y_checks = importlib.import_module("a11y_checks")
|
||||||
|
|
||||||
CATALOGUE = scanner.Catalogue(
|
CATALOGUE = scanner.Catalogue(
|
||||||
name="a11y",
|
name="a11y",
|
||||||
|
|||||||
Reference in new issue
Block a user