forked from bchanot/claude
fix(gates): reject an oracle under a missed criterion label; default timeout 600 s
This commit is contained in:
1 parent
2a5e3a41d8
commit
c25a7ea017
2 files changed
+91
-5
No files matched your search
+27
-5
@@ -29,7 +29,10 @@
|
|||||||
# lib/url-guard.sh first.
|
# lib/url-guard.sh first.
|
||||||
set -uo pipefail
|
set -uo pipefail
|
||||||
|
|
||||||
TIMEOUT="${GATES_TIMEOUT:-120}"
|
# TIMEOUT is per CHECK and a run lasts their sum: the caller's own limit (a
|
||||||
|
# foreground tool call stops at 600 s) must cover it, else run it in the
|
||||||
|
# background. A killed run writes nothing back; the floor stays PENDING.
|
||||||
|
TIMEOUT="${GATES_TIMEOUT:-600}"
|
||||||
EVIDENCE_CAP=140
|
EVIDENCE_CAP=140
|
||||||
|
|
||||||
# Module-level parse tables, index-aligned. Bash has no record type; threading
|
# Module-level parse tables, index-aligned. Bash has no record type; threading
|
||||||
@@ -68,11 +71,23 @@ _new_crit() { # _new_crit <id> <text>
|
|||||||
_CUR=$((${#_ID[@]} - 1))
|
_CUR=$((${#_ID[@]} - 1))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# A second attribute on one criterion is an error, never an overwrite: it
|
||||||
|
# is what an oracle under a label the parser missed looks like (LRN-284).
|
||||||
_set_attr() { # _set_attr <CHECK|EXPECT|EVIDENCE> <value> <lineno>
|
_set_attr() { # _set_attr <CHECK|EXPECT|EVIDENCE> <value> <lineno>
|
||||||
if [ "$_CUR" -lt 0 ]; then
|
if [ "$_CUR" -lt 0 ]; then
|
||||||
_err "$1 at line $3 belongs to no criterion"
|
_err "$1 at line $3 belongs to no criterion"
|
||||||
return 0
|
return 0
|
||||||
fi
|
fi
|
||||||
|
local seen=""
|
||||||
|
case "$1" in
|
||||||
|
CHECK) seen="${_CHECK[_CUR]}" ;;
|
||||||
|
EXPECT) seen="${_EXPECT[_CUR]}" ;;
|
||||||
|
EVIDENCE) [ "${_EVLINE[_CUR]}" = 0 ] || seen=yes ;;
|
||||||
|
esac
|
||||||
|
if [ -n "$seen" ]; then
|
||||||
|
_err "second $1: at line $3 in criterion ${_ID[_CUR]}"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
case "$1" in
|
case "$1" in
|
||||||
CHECK) _CHECK[_CUR]="$2" ;;
|
CHECK) _CHECK[_CUR]="$2" ;;
|
||||||
EXPECT) _EXPECT[_CUR]="$2" ;;
|
EXPECT) _EXPECT[_CUR]="$2" ;;
|
||||||
@@ -82,11 +97,16 @@ _set_attr() { # _set_attr <CHECK|EXPECT|EVIDENCE> <value> <lineno>
|
|||||||
|
|
||||||
# An UNINDENTED attribute is diagnosed, never absorbed: silently ignoring it
|
# An UNINDENTED attribute is diagnosed, never absorbed: silently ignoring it
|
||||||
# would demote a runnable criterion to a manual one, which is the one parse
|
# would demote a runnable criterion to a manual one, which is the one parse
|
||||||
# bug that turns this checker into a rubber stamp.
|
# bug that turns this checker into a rubber stamp. A digit-led line that is
|
||||||
|
# not `N. text` (`1b.`, `1.1`, `2)`) is rejected and orphans what follows, like
|
||||||
|
# a duplicate id.
|
||||||
_absorb() { # _absorb <raw-line> <lineno>
|
_absorb() { # _absorb <raw-line> <lineno>
|
||||||
local body
|
local body
|
||||||
if [[ "$1" =~ ^([0-9]+)\.[[:space:]]+(.*)$ ]]; then
|
if [[ "$1" =~ ^([0-9]+)\.[[:space:]]+(.*)$ ]]; then
|
||||||
_new_crit "${BASH_REMATCH[1]}" "${BASH_REMATCH[2]}"
|
_new_crit "${BASH_REMATCH[1]}" "${BASH_REMATCH[2]}"
|
||||||
|
elif [[ "$1" =~ ^[0-9] ]]; then
|
||||||
|
_err "malformed criterion label at line $2 (want \`N. text\`)"
|
||||||
|
_CUR=-1
|
||||||
elif [[ "$1" =~ ^ABANDON:[[:space:]]*([0-9]+)?[[:space:]]*(.*)$ ]]; then
|
elif [[ "$1" =~ ^ABANDON:[[:space:]]*([0-9]+)?[[:space:]]*(.*)$ ]]; then
|
||||||
_ABANDON_ID+=("${BASH_REMATCH[1]}"); _ABANDON_WHY+=("${BASH_REMATCH[2]}")
|
_ABANDON_ID+=("${BASH_REMATCH[1]}"); _ABANDON_WHY+=("${BASH_REMATCH[2]}")
|
||||||
elif [[ "$1" =~ ^(CHECK|EXPECT|EVIDENCE): ]]; then
|
elif [[ "$1" =~ ^(CHECK|EXPECT|EVIDENCE): ]]; then
|
||||||
@@ -300,11 +320,13 @@ _runnable_count() {
|
|||||||
# ── entry point ─────────────────────────────────────────────────────────────
|
# ── entry point ─────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
main() { # main <status|run> <contract>
|
main() { # main <status|run> <contract>
|
||||||
local mode="$1" file="$2"
|
local mode="$1" file="$2" first
|
||||||
[ -r "$file" ] || _die "contract unreadable: $file"
|
[ -r "$file" ] || _die "contract unreadable: $file"
|
||||||
_parse "$file"
|
_parse "$file"
|
||||||
[ "${#_ID[@]}" -gt 0 ] ||
|
if [ "${#_ID[@]}" -eq 0 ]; then
|
||||||
_die "no numbered criteria under ## ACCEPTANCE CRITERIA"
|
first="${_ERRORS[0]:+; ${_ERRORS[0]}}"
|
||||||
|
_die "no numbered criteria under ## ACCEPTANCE CRITERIA$first"
|
||||||
|
fi
|
||||||
_validate_oracles
|
_validate_oracles
|
||||||
_validate_abandons
|
_validate_abandons
|
||||||
if [ "$mode" = "run" ] && [ "${#_ERRORS[@]}" -eq 0 ]; then
|
if [ "$mode" = "run" ] && [ "${#_ERRORS[@]}" -eq 0 ]; then
|
||||||
|
|||||||
@@ -89,6 +89,17 @@ GATES_TIMEOUT=1 gate "timeout = UNMET" run "GATES — VERDICT: UNMET(1)" 2 <<'EO
|
|||||||
EOF
|
EOF
|
||||||
has "timeout recorded" "NOT-MET timeout=1s"
|
has "timeout recorded" "NOT-MET timeout=1s"
|
||||||
|
|
||||||
|
# The default must outlast the full suite (~135 s on 2026-11-16): at 120 s
|
||||||
|
# a "full make test green" criterion went NOT-MET on the timeout alone.
|
||||||
|
LAST="$WORK/default-timeout.md"
|
||||||
|
printf '## ACCEPTANCE CRITERIA\n1. manual\n' > "$LAST"
|
||||||
|
out="$(env -u GATES_TIMEOUT bash "$GATES" status "$LAST" 2>&1)"
|
||||||
|
if [[ "$out" == *"timeout 600s"* ]]; then
|
||||||
|
ok "default timeout 600s"
|
||||||
|
else
|
||||||
|
bad "default timeout 600s" "status did not print timeout 600s"
|
||||||
|
fi
|
||||||
|
|
||||||
gate "manual-only contract passes through" run "RUNNABLE: 0 of 2" 0 <<'EOF'
|
gate "manual-only contract passes through" run "RUNNABLE: 0 of 2" 0 <<'EOF'
|
||||||
## ACCEPTANCE CRITERIA
|
## ACCEPTANCE CRITERIA
|
||||||
1. a human reads the copy
|
1. a human reads the copy
|
||||||
@@ -203,6 +214,59 @@ gate "duplicate orphans what follows" run "belongs to no criterion" 2 <<'EOF'
|
|||||||
EVIDENCE: pending
|
EVIDENCE: pending
|
||||||
EOF
|
EOF
|
||||||
|
|
||||||
|
# An oracle under a label the parser does not read as `N. text` must never
|
||||||
|
# overwrite the previous criterion's: criterion 1 fails, the stray oracle
|
||||||
|
# passes, a false MET (LRN-284). A digit-led label is rejected by name;
|
||||||
|
# any other shape trips the second-attribute guard when the previous
|
||||||
|
# criterion already has that attribute (after a manual one it does not).
|
||||||
|
gate "1b. label rejected" run "malformed criterion label at line 6" 2 <<'EOF'
|
||||||
|
## ACCEPTANCE CRITERIA
|
||||||
|
1. real
|
||||||
|
CHECK: exit 1
|
||||||
|
EXPECT: x
|
||||||
|
EVIDENCE: pending
|
||||||
|
1b. sub
|
||||||
|
CHECK: echo y
|
||||||
|
EXPECT: y
|
||||||
|
EVIDENCE: pending
|
||||||
|
EOF
|
||||||
|
|
||||||
|
gate "1.1 label rejected" run "malformed criterion label at line 3" 2 <<'EOF'
|
||||||
|
## ACCEPTANCE CRITERIA
|
||||||
|
1. first
|
||||||
|
1.1 dotted
|
||||||
|
EVIDENCE: pending
|
||||||
|
EOF
|
||||||
|
|
||||||
|
gate "no valid label" run "malformed criterion label at line 2" 2 <<'EOF'
|
||||||
|
## ACCEPTANCE CRITERIA
|
||||||
|
2) paren
|
||||||
|
EOF
|
||||||
|
|
||||||
|
gate "indented label: second CHECK" run "second CHECK: at line 7" 2 <<'EOF'
|
||||||
|
## ACCEPTANCE CRITERIA
|
||||||
|
1. real
|
||||||
|
CHECK: exit 1
|
||||||
|
EXPECT: x
|
||||||
|
EVIDENCE: pending
|
||||||
|
2. sub
|
||||||
|
CHECK: echo y
|
||||||
|
EXPECT: y
|
||||||
|
EVIDENCE: pending
|
||||||
|
EOF
|
||||||
|
|
||||||
|
gate "bullet label: second EVIDENCE" run "second EVIDENCE: at line 9" 2 <<'EOF'
|
||||||
|
## ACCEPTANCE CRITERIA
|
||||||
|
1. real
|
||||||
|
CHECK: exit 1
|
||||||
|
EXPECT: x
|
||||||
|
EVIDENCE: pending
|
||||||
|
- sub
|
||||||
|
CHECK: echo y
|
||||||
|
EXPECT: y
|
||||||
|
EVIDENCE: pending
|
||||||
|
EOF
|
||||||
|
|
||||||
gate "no numbered criteria" run "no numbered criteria" 2 <<'EOF'
|
gate "no numbered criteria" run "no numbered criteria" 2 <<'EOF'
|
||||||
## ACCEPTANCE CRITERIA
|
## ACCEPTANCE CRITERIA
|
||||||
nothing numbered here
|
nothing numbered here
|
||||||
|
|||||||
Reference in new issue
Block a user