feat(model-router): wave 2-B — orchestrators declare phases, shifters and pins removed, frontmatter = off-state floor
The 15 Skill(effort-*) citers now call mcp__model-router__route per phase (orchestrate at a dispatch span, reflect/plan for the skill's own level, apply at the bookkeeping tail, escalate at the verify-secure caps); built-in judgment dispatches carry an explicit effort= param. lib/effort-shift.md is the route doctrine, lib/model-gate.md the mod rule (route answer = witness, /route on as remedy). Deleted: skills/effort-*, lib/effort-pins.txt/.sh, lib/model-check.sh, their tests, the installers' re-apply blocks. The mod drops its Skill(effort-*) bridge. The tracked model:/effort: frontmatter stays as the off-state floor, census-locked equal to the rows (lib/tests/effort-routing.test.sh rewritten, 140 checks; analyzer → xhigh). Contract .claude/tasks/contracts/2026-10-10-model-router-w2b-1045.md, plan r4 § W2-B: GATE 0 MET, verifier ECARTS(7) then CONFORME 10/10, security PASS, full make test green (design-tool-gate env red only).
This commit is contained in:
bchanot
+128
-112
@@ -1,9 +1,12 @@
|
||||
#!/usr/bin/env bash
|
||||
# lib/tests/effort-routing.test.sh — census: effort tiering (BDR-107)
|
||||
# agent pins, skill entry levels, shifter skills, orchestrator wiring, settings.
|
||||
# shellcheck disable=SC2015,SC2016 # A && ok || ko is deliberate (ok/ko never fail); '$REPO' locks are literal source text
|
||||
# lib/tests/effort-routing.test.sh — wave-2 census of the model-router rows.
|
||||
# Drift lock: every tracked skill/agent row in mods/model-router/hooks/
|
||||
# register.ts equals its frontmatter (the off-state floor), the D3 wiring
|
||||
# markers sit in the orchestrators, no shifter citer survives.
|
||||
# shellcheck disable=SC2015,SC2016 # A && ok || ko is deliberate (ok/ko never fail)
|
||||
set -u
|
||||
R="$(cd "$(dirname "$0")/../.." && pwd)"
|
||||
REG="$R/mods/model-router/hooks/register.ts"
|
||||
pass=0; fail=0
|
||||
ok() { pass=$((pass+1)); }
|
||||
ko() { fail=$((fail+1)); printf 'FAIL %s\n' "$1"; }
|
||||
@@ -11,120 +14,133 @@ has() { if grep -qF "$2" "$R/$1"; then ok; else ko "$1 missing: $2"; fi; }
|
||||
lacks() { if grep -qF "$2" "$R/$1"; then ko "$1 must NOT contain: $2"; else ok; fi; }
|
||||
# frontmatter = the lines between the first two '---' lines
|
||||
fm() { awk 'NR==1&&/^---$/{p=1;next} p&&/^---$/{exit} p' "$1"; }
|
||||
fm_effort() { fm "$1" | grep -E '^effort: (low|medium|high|xhigh|max)$' | head -1 | cut -d' ' -f2; }
|
||||
fm_has_effort() {
|
||||
got="$(fm_effort "$R/$1")"
|
||||
if [ "$got" = "$2" ]; then ok; else ko "$1 frontmatter effort must be '$2', got '${got:-none}'"; fi
|
||||
fm_val() { fm "$1" | grep -E "^$2: [a-z]+$" | head -1 | cut -d' ' -f2; }
|
||||
|
||||
# ── register.ts parsers (awk/sed on the DEFAULT_CONFIG literal) ──────────
|
||||
# rows <file> <agents|skills> -> "name phase" per row
|
||||
rows() {
|
||||
awk -v s="$2" '$0 ~ "^ "s": \\{"{f=1;next} f&&/^ \},?$/{f=0} f' "$1" \
|
||||
| grep -v '^ *//' | grep -oE "('[^']+'|[A-Za-z0-9_-]+): '[a-z]+'" \
|
||||
| sed -E "s/'//g; s/: / /"
|
||||
}
|
||||
fm_no_effort() { if fm "$R/$1" | grep -q '^effort:'; then ko "$1 must NOT pin effort"; else ok; fi; }
|
||||
# phase_effort <file> <phase> -> "<tier> <effort>"
|
||||
phase_effort() {
|
||||
awk '/^ phases: \{/{f=1;next} f&&/^ \},?$/{f=0} f' "$1" \
|
||||
| sed -nE "s/^ *$2: \{ tier: '([a-z]+)', effort: '([a-z]+)' \},?$/\1 \2/p"
|
||||
}
|
||||
# tier_head <file> <tier> -> first alias of the tier list
|
||||
tier_head() {
|
||||
awk '/^ tiers: \{/{f=1;next} f&&/^ \},?$/{f=0} f' "$1" \
|
||||
| sed -nE "s/^ *$2: \['([a-z]+)'.*$/\1/p"
|
||||
}
|
||||
row_of() { rows "$REG" "$1" | awk -v n="$2" '$1==n{print $2}'; }
|
||||
|
||||
# ── flip-test: the frontmatter reader must accept a valid level and reject an invalid one
|
||||
# ── flip-test: the parsers read a fixture, reject a missing key ──────────
|
||||
FIX="$(mktemp -d)"; trap 'rm -rf "$FIX"' EXIT
|
||||
printf -- '---\nname: good\neffort: xhigh\n---\nbody with effort: low in prose\n' > "$FIX/good.md"
|
||||
printf -- '---\nname: bad\neffort: turbo\n---\n' > "$FIX/bad.md"
|
||||
[ "$(fm_effort "$FIX/good.md")" = "xhigh" ] && ok || ko "flip: valid level not read"
|
||||
[ -z "$(fm_effort "$FIX/bad.md")" ] && ok || ko "flip: invalid level accepted"
|
||||
[ "$(fm "$FIX/good.md" | grep -c 'prose')" -eq 0 ] && ok || ko "flip: body leaked into frontmatter"
|
||||
cat > "$FIX/reg.ts" <<'FX'
|
||||
tiers: {
|
||||
big: ['opus', 'fable'],
|
||||
},
|
||||
phases: {
|
||||
judge: { tier: 'big', effort: 'xhigh' },
|
||||
},
|
||||
agents: {
|
||||
// judge
|
||||
Plan: 'judge', 'plan-challenger': 'judge',
|
||||
},
|
||||
skills: {
|
||||
'ship-feature': 'plan', doc: 'apply',
|
||||
},
|
||||
FX
|
||||
[ "$(rows "$FIX/reg.ts" agents | tr '\n' ,)" = "Plan judge,plan-challenger judge," ] \
|
||||
&& ok || ko "flip: agents rows misparsed"
|
||||
[ "$(rows "$FIX/reg.ts" skills | tr '\n' ,)" = "ship-feature plan,doc apply," ] \
|
||||
&& ok || ko "flip: skills rows misparsed"
|
||||
[ "$(phase_effort "$FIX/reg.ts" judge)" = "big xhigh" ] && ok || ko "flip: phase"
|
||||
[ -z "$(phase_effort "$FIX/reg.ts" nothere)" ] && ok || ko "flip: ghost phase"
|
||||
[ "$(tier_head "$FIX/reg.ts" big)" = "opus" ] && ok || ko "flip: tier head"
|
||||
[ "$(rows "$REG" skills | wc -l)" -gt 40 ] && ok || ko "register.ts: skills rows not parsed"
|
||||
[ "$(rows "$REG" agents | wc -l)" -gt 15 ] && ok || ko "register.ts: agents rows not parsed"
|
||||
|
||||
# ── 1) session default (spec D1)
|
||||
# ── (b) tracked skills: row exists, frontmatter effort equals the row ────
|
||||
NO_ROW_SKILLS=" find-docs graphify impeccable model-router "
|
||||
check_skill() {
|
||||
local f="$1" name phase want got
|
||||
name="$(basename "$(dirname "$f")")"
|
||||
case "$NO_ROW_SKILLS" in *" $name "*) return;; esac
|
||||
phase="$(row_of skills "$name")"
|
||||
[ -n "$phase" ] || { ko "skills/$name: no row in register.ts"; return; }
|
||||
want="$(phase_effort "$REG" "$phase" | cut -d' ' -f2)"
|
||||
got="$(fm_val "$R/$f" effort)"
|
||||
[ -n "$got" ] || { ko "skills/$name: routed skill without effort:"; return; }
|
||||
[ "$got" = "$want" ] && ok || ko "skills/$name: effort $got != row $phase ($want)"
|
||||
}
|
||||
while IFS= read -r f; do check_skill "$f"; done < <(
|
||||
cd "$R" && git ls-files 'skills/*/SKILL.md' 'skills-external/*/SKILL.md')
|
||||
|
||||
# ── (c) tracked agents with a row: tier head == model:, effort == effort: ─
|
||||
NO_ROW_AGENTS=" interviewer client-handover-writer "
|
||||
tier_alias() { tier_head "$REG" "$(phase_effort "$REG" "$1" | cut -d' ' -f1)"; }
|
||||
check_agent() {
|
||||
local f="$1" name phase alias want got
|
||||
name="$(basename "$f" .md)"
|
||||
case "$NO_ROW_AGENTS" in *" $name "*) return;; esac
|
||||
case "$name" in impeccable-*) return;; esac
|
||||
phase="$(row_of agents "$name")"
|
||||
[ -n "$phase" ] || { ko "agents/$name: no row in register.ts"; return; }
|
||||
alias="$(tier_alias "$phase")"; got="$(fm_val "$R/$f" model)"
|
||||
[ "$got" = "$alias" ] && ok || ko "agents/$name: model $got != row $phase ($alias)"
|
||||
[ "$alias" = haiku ] && return
|
||||
want="$(phase_effort "$REG" "$phase" | cut -d' ' -f2)"
|
||||
got="$(fm_val "$R/$f" effort)"
|
||||
[ "$got" = "$want" ] && ok || ko "agents/$name: effort $got != row $phase ($want)"
|
||||
}
|
||||
while IFS= read -r f; do check_agent "$f"; done < <(
|
||||
cd "$R" && git ls-files 'agents/*.md' | grep -E '^agents/[^/]+\.md$' \
|
||||
| grep -v '/README\.md$')
|
||||
|
||||
# ── (d) no shifter citer in skills, agents, lib ──────────────────────────
|
||||
if (cd "$R" && git grep -qE 'Skill\(effort-|EFFORT SHIFT[S]:' -- skills agents lib ':!lib/tests'); then
|
||||
ko "a shifter-skill or shift-header citer survives in skills/agents/lib"
|
||||
else ok; fi
|
||||
# paths split so this file itself matches no deleted-name grep
|
||||
for s in skills/effort-low skills/effort-max lib/effort-""pins.txt lib/model-""check.sh; do
|
||||
[ ! -e "$R/$s" ] && ok || ko "$s must be deleted"
|
||||
done
|
||||
|
||||
# ── (e) D3 wiring markers ────────────────────────────────────────────────
|
||||
mark() { # mark <phase> <skills...>
|
||||
local ph="$1" s; shift
|
||||
for s in "$@"; do has "skills/$s/SKILL.md" "route(phase=\"$ph\")"; done
|
||||
}
|
||||
mark orchestrate feat hotfix bugfix ship-feature init-project code-clean seo geo harden web-validate audit-delta
|
||||
mark apply feat hotfix bugfix ship-feature init-project
|
||||
mark reflect feat hotfix bugfix seo geo harden web-validate
|
||||
mark plan ship-feature init-project onboard code-clean audit-delta
|
||||
mark escalate ship-feature
|
||||
[ "$(grep -o 'route(phase="escalate")' "$R/lib/verify-secure-loop.md" | wc -l)" -eq 3 ] \
|
||||
&& ok || ko "verify-secure-loop.md: escalate route must appear 3 times"
|
||||
for s in ship-feature init-project; do has "skills/$s/SKILL.md" 'effort="xhigh"'; done
|
||||
has "skills/tour/SKILL.md" 'effort="xhigh"'
|
||||
n_opus="$(grep -c 'model="opus",$' "$R/skills/onboard/SKILL.md")"
|
||||
n_eff="$(grep -c 'effort="xhigh",$' "$R/skills/onboard/SKILL.md")"
|
||||
[ "$n_opus" -ge 7 ] && [ "$n_opus" -eq "$n_eff" ] && ok \
|
||||
|| ko "onboard: $n_opus model=\"opus\" dispatches vs $n_eff effort=\"xhigh\""
|
||||
bad="$(grep 'model: "fable"' "$R/agents/client-handover-writer.md" | grep -vc 'effort="high"')"
|
||||
[ "$bad" -eq 0 ] && ok || ko "client-handover-writer: $bad model: \"fable\" line(s) without effort=\"high\""
|
||||
for s in ship-feature init-project feat bugfix web-validate seo hotfix geo harden code-clean audit-delta tour onboard; do
|
||||
has "skills/$s/SKILL.md" 'ROUTING: follow $HOME/.claude/lib/effort-shift.md'
|
||||
done
|
||||
has "agents/client-handover-writer.md" 'ROUTING: follow $HOME/.claude/lib/effort-shift.md'
|
||||
|
||||
# ── (f) doctrine includes ────────────────────────────────────────────────
|
||||
has "lib/model-gate.md" 'mcp__model-router__route'
|
||||
has "lib/effort-shift.md" 'ToolSearch'
|
||||
has "CLAUDE.global.md" 'route to `reflect` (high) through their'
|
||||
|
||||
# ── (g) session default and hooks (unchanged locks) ──────────────────────
|
||||
has "settings.json" '"effortLevel": "high"'
|
||||
|
||||
# ── 2) hooks: env-var warning + live effort in the statusline (spec D1, D5)
|
||||
has "hooks/session-start.sh" 'CLAUDE_CODE_EFFORT_LEVEL'
|
||||
has "hooks/statusline.sh" 'CLAUDE_EFFORT'
|
||||
|
||||
# ── 3) agent pins (spec D2): one effort per agent file, judgment mode wins on mode-based agents
|
||||
for a in hotfixer release-executor plugin-probe validator-analyzer; do fm_has_effort "agents/$a.md" low; done
|
||||
for a in feater bugfixer code-cleaner onboarder scaffolder; do fm_has_effort "agents/$a.md" medium; done
|
||||
for a in refactorer analyzer commit-changer doc-syncer handover-doc-writer; do fm_has_effort "agents/$a.md" high; done
|
||||
for a in plan-challenger plugin-advisor verifier security-auditor seo-analyzer geo-analyzer; do fm_has_effort "agents/$a.md" xhigh; done
|
||||
for a in interviewer client-handover-writer status-reporter; do fm_no_effort "agents/$a.md"; done
|
||||
has "skills/init-project/SKILL.md" 'pin sonnet, effort medium'
|
||||
|
||||
# ── 4) skill entry levels (spec D3): the user's invocation sets the run's level
|
||||
for s in status commit-change release-candidate doc capitalize close reconcile deploy profile plugin-check; do fm_has_effort "skills/$s/SKILL.md" low; done
|
||||
for s in gitflow prune-memory; do fm_has_effort "skills/$s/SKILL.md" medium; done
|
||||
for s in feat hotfix bugfix refactor web-validate harden seo geo; do fm_has_effort "skills/$s/SKILL.md" high; done
|
||||
for s in ship-feature init-project onboard tour audit-delta analyze code-clean client-handover; do fm_has_effort "skills/$s/SKILL.md" xhigh; done
|
||||
# BDR-108 round: the three repo skills that had no level
|
||||
fm_has_effort "skills/skills-perso/SKILL.md" low
|
||||
fm_has_effort "skills/pdf-translate/SKILL.md" medium
|
||||
fm_has_effort "skills/site-motion/SKILL.md" high
|
||||
|
||||
# ── 9) vendored externals carry the level of lib/effort-pins.txt (BDR-108). The files live in
|
||||
# skills-external/ (gitignored, machine-owned): the durable artifact is the map + the re-apply
|
||||
# after the last vendoring step of install-plugins.sh AND update-all.sh; a skill not vendored
|
||||
# yet SKIPs visibly (fresh clone before make plugin).
|
||||
while read -r s lvl _; do
|
||||
case "$s" in ''|'#'*) continue ;; esac
|
||||
if [ -f "$R/skills-external/$s/SKILL.md" ]; then fm_has_effort "skills-external/$s/SKILL.md" "$lvl"
|
||||
else printf 'SKIP skills-external/%s/SKILL.md not vendored yet (run make plugin)\n' "$s"; fi
|
||||
done < "$R/lib/effort-pins.txt"
|
||||
has "lib/effort-pins.txt" 'brainstorming xhigh'; has "lib/effort-pins.txt" 'writing-plans xhigh'
|
||||
has "install-plugins.sh" 'apply_effort_pins "$REPO"'; has "update-all.sh" 'apply_effort_pins "$REPO"'
|
||||
lacks "install-plugins.sh" 'for _s in brainstorming writing-plans; do'
|
||||
ln_last() { grep -n "$2" "$R/$1" | tail -1 | cut -d: -f1; }
|
||||
[ "$(ln_last install-plugins.sh 'apply_effort_pins "$REPO"')" -gt "$(ln_last install-plugins.sh 'rm -rf "$TFD_STAGE"')" ] \
|
||||
&& ok || ko "install-plugins.sh: effort pins must be re-applied after the 21st pack refresh"
|
||||
pins_ln=$(ln_last update-all.sh 'apply_effort_pins "$REPO"')
|
||||
[ "$pins_ln" -gt "$(ln_last update-all.sh 'skills-external/$_tfd_name')" ] \
|
||||
&& [ "$pins_ln" -gt "$(ln_last update-all.sh 'vendor_pinned_skills superpowers refresh')" ] \
|
||||
&& ok || ko "update-all.sh: effort pins must be re-applied after the last vendoring step (21st pack)"
|
||||
[ -x "$R/lib/effort-pins.sh" ] && ok || ko "lib/effort-pins.sh missing or not executable"
|
||||
# 9b) design stack = ONE level (last loaded wins); site-motion (repo skill) pins the same one
|
||||
stack_levels() { awk '/^# design stack/{f=1;next} f&&/^#$/{f=0} f&&!/^#/&&NF==2{print $2}' "$R/lib/effort-pins.txt" | sort -u; }
|
||||
[ "$(stack_levels | wc -l)" -eq 1 ] && ok || ko "design stack must share ONE level in lib/effort-pins.txt (got: $(stack_levels | tr '\n' ' '))"
|
||||
[ "$(stack_levels | wc -l)" -ge 1 ] && fm_has_effort "skills/site-motion/SKILL.md" "$(stack_levels | head -1)"
|
||||
has "lib/effort-shift.md" 'Stacked skills share one level'
|
||||
has "CLAUDE.global.md" 'lib/effort-pins.txt'
|
||||
|
||||
# ── 5) shifter skills + include (spec D4)
|
||||
for l in low medium high xhigh max; do fm_has_effort "skills/effort-$l/SKILL.md" "$l"; has "skills/effort-$l/SKILL.md" "name: effort-$l"; done
|
||||
has "lib/effort-shift.md" 'Headless sessions'
|
||||
has "lib/effort-shift.md" 'Skill(effort-max)'
|
||||
has "lib/effort-shift.md" 'never inside a dispatched agent'
|
||||
has "lib/model-gate.md" 'lib/effort-shift.md'
|
||||
|
||||
# ── 6) orchestrator wiring (spec D4)
|
||||
for s in feat hotfix bugfix ship-feature init-project onboard tour code-clean seo geo harden web-validate audit-delta; do
|
||||
has "skills/$s/SKILL.md" 'lib/effort-shift.md'; has "skills/$s/SKILL.md" 'a lone Skill call is a no-op'; done
|
||||
for s in feat hotfix bugfix ship-feature init-project code-clean seo geo harden web-validate audit-delta; do
|
||||
has "skills/$s/SKILL.md" 'Skill(effort-medium)'; done
|
||||
lacks "skills/onboard/SKILL.md" 'Skill(effort-medium)'; lacks "skills/tour/SKILL.md" 'Skill(effort-medium)'
|
||||
has "agents/client-handover-writer.md" 'lib/effort-shift.md'; lacks "agents/client-handover-writer.md" 'Skill(effort-medium)'; has "agents/client-handover-writer.md" 'Skill(effort-high)'
|
||||
for s in feat hotfix bugfix; do has "skills/$s/SKILL.md" 'Skill(effort-high)'; done
|
||||
for s in ship-feature init-project onboard code-clean audit-delta; do has "skills/$s/SKILL.md" 'Skill(effort-xhigh)'; done
|
||||
for s in seo geo harden web-validate; do has "skills/$s/SKILL.md" 'Skill(effort-high)'; done
|
||||
for s in feat hotfix bugfix ship-feature init-project; do has "skills/$s/SKILL.md" 'Skill(effort-low)'; done
|
||||
has "skills/feat/SKILL.md" 'effort-shift: nested commit-change'
|
||||
|
||||
# ── 6b) pairing rule documented (R11)
|
||||
has "lib/effort-shift.md" 'lone Skill call is a no-op'
|
||||
has "lib/effort-shift.md" 're-applies its'
|
||||
[ "$(grep -c 'a lone Skill call is a no-op' "$R/skills/feat/SKILL.md")" -ge 1 ] && ok || ko "feat INC line must carry the pairing rule"
|
||||
|
||||
# ── 7) escalation at max (spec D4)
|
||||
[ "$(grep -c 'Skill(effort-max)' "$R/lib/verify-secure-loop.md")" -eq 3 ] && ok || ko "verify-secure-loop.md must shift to max at its 3 caps"
|
||||
has "skills/ship-feature/SKILL.md" 'Skill(effort-max)'
|
||||
has "lib/challenge-plan.md" '/effort-max'
|
||||
has "lib/verify-secure-loop.md" '/effort-max'
|
||||
|
||||
# ── 8) turn-reset re-assert after a prose gate followed by reflection
|
||||
has "skills/bugfix/SKILL.md" 'effort-shift: turn reset'
|
||||
|
||||
# ── 11) audit tooling
|
||||
has "lib/effort-shift.md" 'effort-audit.py'
|
||||
[ -x "$R/lib/effort-audit.py" ] && ok || ko "lib/effort-audit.py missing or not executable"
|
||||
|
||||
# ── 6c) judgment dispatches re-raised, planning re-asserts, stronger locks (final review I1/I2/M5)
|
||||
for s in ship-feature init-project; do has "skills/$s/SKILL.md" 'effort-shift: judgment dispatch'; has "skills/$s/SKILL.md" 'effort-shift: turn reset'; done
|
||||
has "agents/client-handover-writer.md" 'effort-shift: judgment dispatch'
|
||||
has "lib/effort-shift.md" 'Before any built-in or unpinned dispatch'
|
||||
has "lib/model-gate.md" 'built-ins inherit the effort in force'
|
||||
has "skills/ship-feature/SKILL.md" 'effort-shift: error recovery'
|
||||
for s in feat hotfix bugfix seo geo harden web-validate ship-feature init-project onboard code-clean audit-delta; do has "skills/$s/SKILL.md" 'effort-shift: own level before the challenge'; done
|
||||
has "update-all.sh" 'source "$REPO/lib/effort-pins.sh"'
|
||||
|
||||
# ── summary (later tasks insert their locks ABOVE this line)
|
||||
printf 'effort-routing census: %d pass, %d fail\n' "$pass" "$fail"
|
||||
[ "$fail" -eq 0 ]
|
||||
printf 'PASS=%s FAIL=%s\n' "$pass" "$fail"; [ "$fail" -eq 0 ]
|
||||
|
||||
Reference in New Issue
Block a user