feat(guardrails): propose, apply accepted fixes, re-pass to prove them

After the report the skill proposes fixes (plan phase), stops for the
user's acceptance, applies accepted ones via bugfixer and the
verify-secure gates, then re-scans. scan.py gains --assert-absent
(contract oracle, positive control) and --baseline (no new hit).
This commit is contained in:
bchanot committed 2026-10-18 19:04:13 +02:00
1 parent b17de15c22
commit 73c25d759e
14 files changed
+382 -29

No files matched your search

+1 -1
View File
@@ -114,7 +114,7 @@ Template:
```markdown
# CONTRACT — <slug>
- date: <YYYY-MM-DD> | flow: <ship-feature|feat|bugfix|hotfix|init-project|onboard> | branch: <branch>
- date: <YYYY-MM-DD> | flow: <ship-feature|feat|bugfix|hotfix|init-project|onboard|guardrails> | branch: <branch>
- status: active
## REQUEST (verbatim — IMMUTABLE)
+60
View File
@@ -252,6 +252,66 @@ LAST_OUT=$(scan "$WORK/tx-bulk")
check "a subscribers table makes the same senders fail" \
"$(has FAIL MSG-NO-CONSENT-RECORD)" yes
echo "== check modes (contract oracle and re-pass baseline)"
IMG="A11Y-IMG-ALT:$TSX"
cp -R "$GOOD" "$WORK/cm"; edit "$WORK/cm/$TSX" 's/ alt="Company logo"//'
out=$(scan "$WORK/cm" --assert-absent "$IMG"); rc=$?
check "positive control: the oracle fails while the hit is there" "$rc" 1
check "a failing oracle never prints the marker" \
"$(printf '%s\n' "$out" | grep -c GUARDRAILS-CLEAR)" 0
out=$(scan "$GOOD" --assert-absent "$IMG"); rc=$?
check "the oracle passes once the hit is gone" "$rc" 0
check "a passing oracle prints the marker" \
"$(printf '%s\n' "$out" | grep -c '^GUARDRAILS-CLEAR 1$')" 1
out=$(scan "$GOOD" --domain api --assert-absent "$IMG"); rc=$?
check "an unscanned domain proves nothing" "$rc" 1
check "the control failure is named" \
"$(printf '%s\n' "$out" | grep -c '^ CONTROL ')" 1
mkdir "$WORK/cm-empty"
scan "$WORK/cm-empty" --assert-absent "$IMG" >/dev/null
check "an empty tree proves nothing" "$?" 1
scan "$GOOD" --assert-absent HOST-SPEND-CAP:. >/dev/null 2>&1
check "a MANUAL rule cannot be asserted (exit 2)" "$?" 2
scan "$GOOD" --assert-absent A11Y-IMG-ALT >/dev/null 2>&1
check "an item without a path is a usage error" "$?" 2
scan "$WORK/cm" --json > "$WORK/cm-before.json"
out=$(scan "$GOOD" --baseline "$WORK/cm-before.json"); rc=$?
check "baseline lists the fixed hit" \
"$(printf '%s\n' "$out" | grep -c '^ FIXED A11Y-IMG-ALT ')" 1
check "baseline with no new hit exits 0 with the marker" \
"$rc:$(printf '%s\n' "$out" | grep -c '^GUARDRAILS-NO-NEW$')" "0:1"
scan "$GOOD" --json > "$WORK/clean-before.json"
out=$(scan "$WORK/cm" --baseline "$WORK/clean-before.json"); rc=$?
check "baseline lists a new hit" \
"$(printf '%s\n' "$out" | grep -c '^ NEW A11Y-IMG-ALT ')" 1
check "baseline with a new hit exits 1 without the marker" \
"$rc:$(printf '%s\n' "$out" | grep -c GUARDRAILS-NO-NEW)" "1:0"
printf 'not json' > "$WORK/bad.json"
scan "$GOOD" --baseline "$WORK/bad.json" >/dev/null 2>&1
check "an unreadable baseline exits 2" "$?" 2
# contract <dir> <file>: a fix contract whose oracle is the scanner itself
contract() {
cat > "$2" <<EOF
# CONTRACT — guardrails-oracle-test
## ACCEPTANCE CRITERIA
1. every image in the contact form has an alt
CHECK: python3 -B '$SCAN' '$1' --assert-absent '$IMG'
EXPECT: GUARDRAILS-CLEAR
EVIDENCE: pending
## FILE SCOPE
$TSX
EOF
}
contract "$WORK/cm" "$WORK/c-unfixed.md"
bash "$ROOT/lib/gates.sh" run "$WORK/c-unfixed.md" >/dev/null 2>&1
check "gates.sh: the oracle is UNMET before the fix" "$?" 2
contract "$GOOD" "$WORK/c-fixed.md"
bash "$ROOT/lib/gates.sh" run "$WORK/c-fixed.md" >/dev/null 2>&1
check "gates.sh: the oracle is MET after the fix" "$?" 0
check "gates.sh writes the evidence back" \
"$(grep -c 'EVIDENCE: pending' "$WORK/c-fixed.md")" 0
echo "== cli"
scan "$WORK/does-not-exist" >/dev/null 2>&1; check "bad path exits 2" "$?" 2
scan --domain nope "$GOOD" >/dev/null 2>&1; check "bad domain exits 2" "$?" 2
+10
View File
@@ -14,6 +14,7 @@ BUG="$REPO/agents/bugfixer.md"
BSK="$REPO/skills/bugfix/SKILL.md"
HOT="$REPO/agents/hotfixer.md"
HSK="$REPO/skills/hotfix/SKILL.md"
GSK="$REPO/skills/guardrails/SKILL.md"
HSKL="$REPO/skills/hotfix/SKILL.md"
PASS=0; FAIL=0
@@ -69,6 +70,15 @@ tf "bug uses shared include" "$BSK" "lib/verify-secure-loop.md"
tf "bug gate0 floor" "$BSK" "GATE 0 — deterministic floor"
tf "bug dispatches bugfixer" "$BSK" 'subagent_type="bugfixer"'
echo "── skills/guardrails/SKILL.md (fix loop reuses the bugfix circuit) ──"
tf "guardrails contract include" "$GSK" "lib/contract-interview.md"
tf "guardrails uses shared include" "$GSK" "lib/verify-secure-loop.md"
tf "guardrails dispatches bugfixer" "$GSK" 'subagent_type="bugfixer"'
tf "guardrails scanner is the oracle" "$GSK" "EXPECT: GUARDRAILS-CLEAR"
tf "guardrails positive control" "$GSK" "Positive control"
tf "guardrails approval gate" "$GSK" "Nothing is applied without the user's explicit acceptance"
tf "guardrails re-pass baseline" "$GSK" "GUARDRAILS-NO-NEW"
echo "── agents/bugfixer.md (bugfix executor — sonnet, no Agent) ──"
tn "bugfixer lacks Agent tool" "$BUG" "Agent"
tf "bugfixer model sonnet" "$BUG" "model: sonnet"
+2 -2
View File
@@ -9,8 +9,8 @@ has() { if grep -qF "$2" "$R/$1"; then ok; else ko "$1 missing: $2"; fi; }
lacks() { if grep -qF "$2" "$R/$1"; then ko "$1 must NOT contain: $2"; else ok; fi; }
fm_lacks() { if awk 'NR<=10' "$R/$1" | grep -qF "$2"; then ko "$1 frontmatter must NOT contain: $2"; else ok; fi; }
# 1) gate wired in the 15 reflection skills (orchestrators + /analyze)
for s in ship-feature init-project feat bugfix onboard seo geo web-validate harden audit-delta tour code-clean hotfix client-handover analyze; do
# 1) gate wired in the 16 reflection skills (orchestrators, /analyze, /guardrails)
for s in ship-feature init-project feat bugfix onboard seo geo web-validate harden audit-delta tour code-clean hotfix client-handover analyze guardrails; do
has "skills/$s/SKILL.md" 'lib/model-gate.md'
done
# 2) gate NOT wired in the pure-execution/read-only skills (exclusion list)
+1 -1
View File
@@ -1,4 +1,4 @@
# Verify + secure loop — shared orchestrator include (feat, bugfix)
# Verify + secure loop — shared orchestrator include (feat, bugfix, guardrails)
Runs in the ORCHESTRATOR MAIN LOOP after the dev step completes. Turns a
finished diff into a verified, security-cleared change through two fresh