feat(model-router): wave 3-A — first-use route confirmation, decision memory, project exceptions
routing.json (tracked, reached through the plugin directory) is now the single source of the phase table and of every skill/agent row, plus the decisions: confirmed rows/phases, changed rows (from/to) and projects exceptions keyed by a normalized git remote (credentials never stored, no machine paths). First use of a rowed typed skill, a rowed agent spawn or a main-loop phase opens the engine's dialog (Later / Keep / two alternative phases; Other = a phase name); a change asks Everywhere or This project only. One dialog at a time, never in headless, never inside an agent, never written by the model: only a dialog answer or /route ask writes, serialized, size-capped, never creating the file. Layers: routing.json < ~/.claude/model-router.json; the project tree is never read. /route pending, /route ask on|off. Census reads rows and phases from the file and tolerates a user-changed row (WARN). Kit suite 86 → 190 tests. Contract .claude/tasks/contracts/2026-10-10-model-router-w3a-confirm-1201.md, plan r4: 3 lenses + 1 confirmation, feater + 4 rounds, GATE 0 MET, verifier CONFORME then re-verify after security, security BLOCK(1) fixed then PASS. Live: T2 dialogs answered by the user from the hot-loaded mod.
This commit is contained in:
bchanot
@@ -48,6 +48,10 @@ Builtin `/effort` is NOT a lever inside a run: rows and routes outrank it.
|
||||
typed marker (unverified live 2026-10-10).
|
||||
- Headless (`-p`, SDK) runs the hooks, so routing works there too.
|
||||
- Mod off: typed agents fall back to their `model:`/`effort:` frontmatter.
|
||||
- First use of a row asks once (Keep, Later or another phase, then
|
||||
Everywhere or this project only); the answer is kept in
|
||||
`mods/model-router/routing.json`. `/route pending` lists what is still
|
||||
unconfirmed, `/route ask off|on` toggles the dialog.
|
||||
|
||||
Measure the split any time: `python3 ~/.claude/lib/effort-audit.py`
|
||||
(thinking/output/cache tokens per scope, model and effort).
|
||||
|
||||
@@ -1,12 +1,16 @@
|
||||
#!/usr/bin/env bash
|
||||
# lib/tests/effort-routing.test.sh — wave-2 census of the model-router rows.
|
||||
# Drift lock: every tracked skill/agent row in mods/model-router/hooks/
|
||||
# register.ts equals its frontmatter (the off-state floor), the D3 wiring
|
||||
# markers sit in the orchestrators, no shifter citer survives.
|
||||
# Drift lock: every tracked skill/agent row of mods/model-router/routing.json
|
||||
# equals its frontmatter (the off-state floor), except a row the user changed
|
||||
# through the first-use dialog (WARN floor drift: the floor may lag a
|
||||
# confirmed decision); the D3 wiring markers sit in the orchestrators, no
|
||||
# shifter citer survives.
|
||||
# shellcheck disable=SC2015,SC2016 # A && ok || ko is deliberate (ok/ko never fail)
|
||||
set -u
|
||||
R="$(cd "$(dirname "$0")/../.." && pwd)"
|
||||
REG="$R/mods/model-router/hooks/register.ts"
|
||||
MOD="$R/mods/model-router"
|
||||
RJ="$MOD/routing.json"
|
||||
REG="$MOD/hooks/register.ts"
|
||||
pass=0; fail=0
|
||||
ok() { pass=$((pass+1)); }
|
||||
ko() { fail=$((fail+1)); printf 'FAIL %s\n' "$1"; }
|
||||
@@ -16,84 +20,145 @@ lacks() { if grep -qF "$2" "$R/$1"; then ko "$1 must NOT contain: $2"; else ok;
|
||||
fm() { awk 'NR==1&&/^---$/{p=1;next} p&&/^---$/{exit} p' "$1"; }
|
||||
fm_val() { fm "$1" | grep -E "^$2: [a-z]+$" | head -1 | cut -d' ' -f2; }
|
||||
|
||||
# ── register.ts parsers (awk/sed on the DEFAULT_CONFIG literal) ──────────
|
||||
# rows <file> <agents|skills> -> "name phase" per row
|
||||
rows() {
|
||||
awk -v s="$2" '$0 ~ "^ "s": \\{"{f=1;next} f&&/^ \},?$/{f=0} f' "$1" \
|
||||
| grep -v '^ *//' | grep -oE "('[^']+'|[A-Za-z0-9_-]+): '[a-z]+'" \
|
||||
| sed -E "s/'//g; s/: / /"
|
||||
}
|
||||
# phase_effort <file> <phase> -> "<tier> <effort>"
|
||||
phase_effort() {
|
||||
awk '/^ phases: \{/{f=1;next} f&&/^ \},?$/{f=0} f' "$1" \
|
||||
| sed -nE "s/^ *$2: \{ tier: '([a-z]+)', effort: '([a-z]+)' \},?$/\1 \2/p"
|
||||
# ── routing.json (rows, phases, decisions) and the tier heads ────────────
|
||||
# dump <routing.json> -> "skills|agents <name> <phase>", "phases <p> <tier>
|
||||
# <effort>", "changed <kind> <name> <from> <to>" lines
|
||||
dump() {
|
||||
python3 -I - "$1" <<'PY'
|
||||
import json, sys
|
||||
d = json.load(open(sys.argv[1]))
|
||||
for kind in ("skills", "agents"):
|
||||
for k, v in d.get(kind, {}).items():
|
||||
print(kind, k, v)
|
||||
for k, v in d.get("phases", {}).items():
|
||||
print("phases", k, v.get("tier", ""), v.get("effort", ""))
|
||||
for kind in ("skills", "agents"):
|
||||
for k, v in d.get("changed", {}).get(kind, {}).items():
|
||||
print("changed", kind, k, v["from"], v["to"])
|
||||
PY
|
||||
}
|
||||
# lookups on a dump text: row_of D kind name · phase_effort D phase ·
|
||||
# changed_of D kind name -> "from to"
|
||||
row_of() { printf '%s\n' "$1" | awk -v k="$2" -v n="$3" '$1==k&&$2==n{print $3}'; }
|
||||
phase_effort() { printf '%s\n' "$1" | awk -v p="$2" '$1=="phases"&&$2==p{print $3, $4}'; }
|
||||
changed_of() { printf '%s\n' "$1" | awk -v k="$2" -v n="$3" '$1=="changed"&&$2==k&&$3==n{print $4, $5}'; }
|
||||
# tier_head <file> <tier> -> first alias of the tier list
|
||||
tier_head() {
|
||||
awk '/^ tiers: \{/{f=1;next} f&&/^ \},?$/{f=0} f' "$1" \
|
||||
| sed -nE "s/^ *$2: \['([a-z]+)'.*$/\1/p"
|
||||
}
|
||||
row_of() { rows "$REG" "$1" | awk -v n="$2" '$1==n{print $2}'; }
|
||||
# the fallback copy of the phases inside the mod's code: "<tier> <effort>"
|
||||
code_phase() {
|
||||
awk '/^ phases: \{/{f=1;next} f&&/^ \},?$/{f=0} f' "$REG" \
|
||||
| sed -nE "s/^ *$1: \{ tier: '([a-z]+)', effort: '([a-z]+)' \},?$/\1 \2/p"
|
||||
}
|
||||
# field_of <dump> <phase> <effort|model> -> that phase's frontmatter value
|
||||
field_of() {
|
||||
local pe tier
|
||||
pe="$(phase_effort "$1" "$2")"
|
||||
if [ "$3" = effort ]; then printf '%s\n' "${pe#* }"; return; fi
|
||||
tier="${pe% *}"; tier_head "$REG" "$tier"
|
||||
}
|
||||
# verdict <dump> <kind> <name> <effort|model> <got> <phase>: ok when the
|
||||
# frontmatter equals the row's phase; warn when the row was changed through
|
||||
# the dialog (changed.from is the shipped phase, changed.to the row) and the
|
||||
# frontmatter still holds the shipped value; fail otherwise (a Keep alone,
|
||||
# an empty or unknown value, a hand edit after a change)
|
||||
verdict() {
|
||||
local D="$1" kind="$2" name="$3" field="$4" got="$5" phase="$6" ch from fromval
|
||||
[ -n "$got" ] || { echo fail; return; }
|
||||
[ "$got" = "$(field_of "$D" "$phase" "$field")" ] && { echo ok; return; }
|
||||
ch="$(changed_of "$D" "$kind" "$name")"; from="${ch% *}"
|
||||
[ -n "$ch" ] && [ "${ch#* }" = "$phase" ] || { echo fail; return; }
|
||||
fromval="$(field_of "$D" "$from" "$field")"
|
||||
[ -n "$fromval" ] && [ "$got" = "$fromval" ] && echo warn || echo fail
|
||||
}
|
||||
# floor_check <dump> <kind> <name> <effort|model> <got> <phase> <label>
|
||||
floor_check() {
|
||||
case "$(verdict "$1" "$2" "$3" "$4" "$5" "$6")" in
|
||||
ok) ok;;
|
||||
warn) printf 'WARN floor drift: %s %s vs row %s (%s)\n' "$7" "$5" "$6" "$4"; ok;;
|
||||
*) ko "$7: $4 '$5' != row $6 ($(field_of "$1" "$6" "$4"))";;
|
||||
esac
|
||||
}
|
||||
RD="$(dump "$RJ")"
|
||||
|
||||
# ── flip-test: the parsers read a fixture, reject a missing key ──────────
|
||||
# ── flip-test: the readers parse a fixture and reject the wrong verdicts ──
|
||||
FIX="$(mktemp -d)"; trap 'rm -rf "$FIX"' EXIT
|
||||
cat > "$FIX/reg.ts" <<'FX'
|
||||
tiers: {
|
||||
big: ['opus', 'fable'],
|
||||
},
|
||||
phases: {
|
||||
judge: { tier: 'big', effort: 'xhigh' },
|
||||
},
|
||||
agents: {
|
||||
// judge
|
||||
Plan: 'judge', 'plan-challenger': 'judge',
|
||||
},
|
||||
skills: {
|
||||
'ship-feature': 'plan', doc: 'apply',
|
||||
},
|
||||
FX
|
||||
[ "$(rows "$FIX/reg.ts" agents | tr '\n' ,)" = "Plan judge,plan-challenger judge," ] \
|
||||
&& ok || ko "flip: agents rows misparsed"
|
||||
[ "$(rows "$FIX/reg.ts" skills | tr '\n' ,)" = "ship-feature plan,doc apply," ] \
|
||||
&& ok || ko "flip: skills rows misparsed"
|
||||
[ "$(phase_effort "$FIX/reg.ts" judge)" = "big xhigh" ] && ok || ko "flip: phase"
|
||||
[ -z "$(phase_effort "$FIX/reg.ts" nothere)" ] && ok || ko "flip: ghost phase"
|
||||
cat > "$FIX/routing.json" <<'FX'
|
||||
{
|
||||
"phases": {
|
||||
"judge": { "tier": "big", "effort": "xhigh" },
|
||||
"reflect": { "tier": "best", "effort": "high" },
|
||||
"implement": { "tier": "work", "effort": "medium" }
|
||||
},
|
||||
"skills": { "feat": "implement", "doc": "reflect" },
|
||||
"agents": { "Plan": "judge" },
|
||||
"changed": { "skills": { "feat": { "from": "reflect", "to": "implement" } } },
|
||||
"ask": true
|
||||
}
|
||||
FX
|
||||
FD="$(dump "$FIX/routing.json")"
|
||||
[ "$(row_of "$FD" agents Plan)" = judge ] && ok || ko "flip: agents row misread"
|
||||
[ "$(row_of "$FD" skills feat)" = implement ] && ok || ko "flip: skills row misread"
|
||||
[ -z "$(row_of "$FD" skills ghost)" ] && ok || ko "flip: ghost row"
|
||||
[ "$(phase_effort "$FD" judge)" = "big xhigh" ] && ok || ko "flip: phase"
|
||||
[ -z "$(phase_effort "$FD" nothere)" ] && ok || ko "flip: ghost phase"
|
||||
[ "$(tier_head "$FIX/reg.ts" big)" = "opus" ] && ok || ko "flip: tier head"
|
||||
[ "$(rows "$REG" skills | wc -l)" -gt 40 ] && ok || ko "register.ts: skills rows not parsed"
|
||||
[ "$(rows "$REG" agents | wc -l)" -gt 15 ] && ok || ko "register.ts: agents rows not parsed"
|
||||
[ "$(changed_of "$FD" skills feat)" = "reflect implement" ] && ok || ko "flip: changed entry"
|
||||
[ -z "$(changed_of "$FD" skills doc)" ] && ok || ko "flip: ghost changed entry"
|
||||
# the drift verdicts (high = reflect's effort, medium = implement's)
|
||||
[ "$(verdict "$FD" skills doc effort high reflect)" = ok ] && ok || ko "flip: equal must pass"
|
||||
[ "$(verdict "$FD" skills doc effort medium reflect)" = fail ] && ok || ko "flip: a Keep-only drift must FAIL"
|
||||
[ "$(verdict "$FD" skills feat effort high implement)" = warn ] && ok || ko "flip: changed.from drift must WARN"
|
||||
[ "$(verdict "$FD" skills feat effort '' implement)" = fail ] && ok || ko "flip: an empty value must FAIL"
|
||||
[ "$(verdict "$FD" skills feat effort banana implement)" = fail ] && ok || ko "flip: an unknown value must FAIL"
|
||||
[ "$(verdict "$FD" skills feat effort low implement)" = fail ] && ok || ko "flip: a drift off the shipped value must FAIL"
|
||||
[ "$(verdict "$FD" skills feat effort high reflect)" = ok ] && ok || ko "flip: row back to its shipped phase passes"
|
||||
[ "$(verdict "$FD" skills feat effort high judge)" = fail ] && ok || ko "flip: a hand edit after a change must FAIL"
|
||||
[ "$(echo "$RD" | grep -c '^skills ')" -gt 40 ] && ok || ko "routing.json: skills rows not read"
|
||||
[ "$(echo "$RD" | grep -c '^agents ')" -gt 15 ] && ok || ko "routing.json: agents rows not read"
|
||||
|
||||
# ── the code's fallback phases equal routing.json's (never a stale route) ─
|
||||
n_code="$(awk '/^ phases: \{/{f=1;next} f&&/^ \},?$/{f=0} f' "$REG" | grep -c "tier:")"
|
||||
n_json="$(echo "$RD" | grep -c '^phases ')"
|
||||
[ "$n_code" -eq "$n_json" ] && ok || ko "phases: $n_code in code vs $n_json in routing.json"
|
||||
while read -r _ p tier effort; do
|
||||
[ "$(code_phase "$p")" = "$tier $effort" ] && ok \
|
||||
|| ko "phase $p: code '$(code_phase "$p")' != routing.json '$tier $effort'"
|
||||
done < <(echo "$RD" | grep '^phases ')
|
||||
|
||||
# ── (b) tracked skills: row exists, frontmatter effort equals the row ────
|
||||
NO_ROW_SKILLS=" find-docs graphify impeccable model-router "
|
||||
check_skill() {
|
||||
local f="$1" name phase want got
|
||||
local f="$1" name phase
|
||||
name="$(basename "$(dirname "$f")")"
|
||||
case "$NO_ROW_SKILLS" in *" $name "*) return;; esac
|
||||
phase="$(row_of skills "$name")"
|
||||
[ -n "$phase" ] || { ko "skills/$name: no row in register.ts"; return; }
|
||||
want="$(phase_effort "$REG" "$phase" | cut -d' ' -f2)"
|
||||
got="$(fm_val "$R/$f" effort)"
|
||||
[ -n "$got" ] || { ko "skills/$name: routed skill without effort:"; return; }
|
||||
[ "$got" = "$want" ] && ok || ko "skills/$name: effort $got != row $phase ($want)"
|
||||
phase="$(row_of "$RD" skills "$name")"
|
||||
[ -n "$phase" ] || { ko "skills/$name: no row in routing.json"; return; }
|
||||
[ -n "$(fm_val "$R/$f" effort)" ] || { ko "skills/$name: routed skill without effort:"; return; }
|
||||
floor_check "$RD" skills "$name" effort "$(fm_val "$R/$f" effort)" "$phase" "skills/$name"
|
||||
}
|
||||
while IFS= read -r f; do check_skill "$f"; done < <(
|
||||
cd "$R" && git ls-files 'skills/*/SKILL.md' 'skills-external/*/SKILL.md')
|
||||
|
||||
# ── (c) tracked agents with a row: tier head == model:, effort == effort: ─
|
||||
NO_ROW_AGENTS=" interviewer client-handover-writer "
|
||||
tier_alias() { tier_head "$REG" "$(phase_effort "$REG" "$1" | cut -d' ' -f1)"; }
|
||||
check_agent() {
|
||||
local f="$1" name phase alias want got
|
||||
local f="$1" name phase
|
||||
name="$(basename "$f" .md)"
|
||||
case "$NO_ROW_AGENTS" in *" $name "*) return;; esac
|
||||
case "$name" in impeccable-*) return;; esac
|
||||
phase="$(row_of agents "$name")"
|
||||
[ -n "$phase" ] || { ko "agents/$name: no row in register.ts"; return; }
|
||||
alias="$(tier_alias "$phase")"; got="$(fm_val "$R/$f" model)"
|
||||
[ "$got" = "$alias" ] && ok || ko "agents/$name: model $got != row $phase ($alias)"
|
||||
[ "$alias" = haiku ] && return
|
||||
want="$(phase_effort "$REG" "$phase" | cut -d' ' -f2)"
|
||||
got="$(fm_val "$R/$f" effort)"
|
||||
[ "$got" = "$want" ] && ok || ko "agents/$name: effort $got != row $phase ($want)"
|
||||
phase="$(row_of "$RD" agents "$name")"
|
||||
[ -n "$phase" ] || { ko "agents/$name: no row in routing.json"; return; }
|
||||
floor_check "$RD" agents "$name" model "$(fm_val "$R/$f" model)" "$phase" "agents/$name"
|
||||
[ "$(field_of "$RD" "$phase" model)" = haiku ] && return
|
||||
floor_check "$RD" agents "$name" effort "$(fm_val "$R/$f" effort)" "$phase" "agents/$name"
|
||||
}
|
||||
while IFS= read -r f; do check_agent "$f"; done < <(
|
||||
cd "$R" && git ls-files 'agents/*.md' | grep -E '^agents/[^/]+\.md$' \
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
+703
-104
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,144 @@
|
||||
{
|
||||
"phases": {
|
||||
"plan": {
|
||||
"tier": "best",
|
||||
"effort": "xhigh"
|
||||
},
|
||||
"reflect": {
|
||||
"tier": "best",
|
||||
"effort": "high"
|
||||
},
|
||||
"orchestrate": {
|
||||
"tier": "best",
|
||||
"effort": "medium"
|
||||
},
|
||||
"escalate": {
|
||||
"tier": "best",
|
||||
"effort": "max"
|
||||
},
|
||||
"judge": {
|
||||
"tier": "big",
|
||||
"effort": "xhigh"
|
||||
},
|
||||
"implement": {
|
||||
"tier": "work",
|
||||
"effort": "medium"
|
||||
},
|
||||
"write": {
|
||||
"tier": "work",
|
||||
"effort": "high"
|
||||
},
|
||||
"verify": {
|
||||
"tier": "work",
|
||||
"effort": "xhigh"
|
||||
},
|
||||
"explore": {
|
||||
"tier": "work",
|
||||
"effort": "medium"
|
||||
},
|
||||
"apply": {
|
||||
"tier": "work",
|
||||
"effort": "low"
|
||||
},
|
||||
"mechanical": {
|
||||
"tier": "cheap",
|
||||
"effort": "low"
|
||||
}
|
||||
},
|
||||
"skills": {
|
||||
"ship-feature": "plan",
|
||||
"init-project": "plan",
|
||||
"onboard": "plan",
|
||||
"tour": "plan",
|
||||
"audit-delta": "plan",
|
||||
"analyze": "plan",
|
||||
"code-clean": "plan",
|
||||
"client-handover": "plan",
|
||||
"brainstorming": "plan",
|
||||
"writing-plans": "plan",
|
||||
"requesting-code-review": "plan",
|
||||
"21st-ui-review": "plan",
|
||||
"feat": "reflect",
|
||||
"hotfix": "reflect",
|
||||
"bugfix": "reflect",
|
||||
"refactor": "reflect",
|
||||
"web-validate": "reflect",
|
||||
"harden": "reflect",
|
||||
"seo": "reflect",
|
||||
"geo": "reflect",
|
||||
"site-motion": "reflect",
|
||||
"frontend-design": "reflect",
|
||||
"emil-design-eng": "reflect",
|
||||
"design-motion-principles": "reflect",
|
||||
"21st-ui-build": "reflect",
|
||||
"scroll-world-storytelling": "reflect",
|
||||
"build-threejs-scroll-worlds": "reflect",
|
||||
"scroll-scrubbed-visual-sequence": "reflect",
|
||||
"scroll-scrubbed-word-reveal": "reflect",
|
||||
"scroll-progress-timeline": "reflect",
|
||||
"subagent-driven-development": "reflect",
|
||||
"writing-skills": "reflect",
|
||||
"deprecation-and-migration": "reflect",
|
||||
"21st-ai": "reflect",
|
||||
"21st-ui-explore": "reflect",
|
||||
"gitflow": "implement",
|
||||
"prune-memory": "implement",
|
||||
"pdf-translate": "implement",
|
||||
"ci-cd-and-automation": "implement",
|
||||
"observability-and-instrumentation": "implement",
|
||||
"test-driven-development": "implement",
|
||||
"commit-change": "apply",
|
||||
"release-candidate": "apply",
|
||||
"doc": "apply",
|
||||
"capitalize": "apply",
|
||||
"close": "apply",
|
||||
"reconcile": "apply",
|
||||
"deploy": "apply",
|
||||
"status": "mechanical",
|
||||
"profile": "mechanical",
|
||||
"plugin-check": "mechanical",
|
||||
"skills-perso": "mechanical",
|
||||
"using-git-worktrees": "mechanical",
|
||||
"21st-cli-use": "mechanical",
|
||||
"21st-registry": "mechanical",
|
||||
"21st-design-sync": "mechanical"
|
||||
},
|
||||
"agents": {
|
||||
"Explore": "explore",
|
||||
"Plan": "judge",
|
||||
"plan-challenger": "judge",
|
||||
"plugin-advisor": "judge",
|
||||
"seo-analyzer": "judge",
|
||||
"geo-analyzer": "judge",
|
||||
"analyzer": "judge",
|
||||
"feater": "implement",
|
||||
"bugfixer": "implement",
|
||||
"code-cleaner": "implement",
|
||||
"scaffolder": "implement",
|
||||
"onboarder": "implement",
|
||||
"commit-changer": "write",
|
||||
"doc-syncer": "write",
|
||||
"handover-doc-writer": "write",
|
||||
"refactorer": "write",
|
||||
"hotfixer": "apply",
|
||||
"release-executor": "apply",
|
||||
"plugin-probe": "apply",
|
||||
"validator-analyzer": "apply",
|
||||
"verifier": "verify",
|
||||
"security-auditor": "verify",
|
||||
"status-reporter": "mechanical"
|
||||
},
|
||||
"projects": {},
|
||||
"confirmed": {
|
||||
"agents": {
|
||||
"verifier": "verify",
|
||||
"feater": "implement"
|
||||
},
|
||||
"phases": {
|
||||
"verify": "verify",
|
||||
"implement": "implement"
|
||||
}
|
||||
},
|
||||
"changed": {},
|
||||
"ask": true
|
||||
}
|
||||
Reference in New Issue
Block a user