From a7d747c65ffad7a4d635b70491a84b2e8bf71c61 Mon Sep 17 00:00:00 2001 From: bchanot Date: Wed, 21 Oct 2026 16:50:47 +0200 Subject: [PATCH] feat(model-router): audit phase puts the verifier and security-auditor on Fable New best-tier phase audit (xhigh) carries both merge gates; their opus frontmatter stays the outage floor, since inTier never writes below a tier head. The census floors best-tier rows on the second alias, and the citers now name the audit row. --- README.md | 13 +++++----- agents/plan-challenger.md | 10 +++++--- agents/verifier.md | 2 +- lib/challenge-plan.md | 3 ++- lib/tests/effort-routing.test.sh | 14 +++++++++-- lib/tests/model-routing.test.sh | 4 +-- mods/model-router/hooks/register.test.ts | 31 ++++++++++++++++++++++-- mods/model-router/hooks/register.ts | 1 + mods/model-router/routing.json | 17 ++++++++----- skills/tour/SKILL.md | 4 +-- 10 files changed, 73 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 7560329..bc1626e 100644 --- a/README.md +++ b/README.md @@ -83,8 +83,8 @@ inherits silently: typed agents run on their model-router row, their | Agent | Model | Tier | |---|---|---| | feater, hotfixer, bugfixer | sonnet (pinned) | executors — code from a closed plan (feat), fix from a closed diagnosis (bugfix), fix-bundle appliers | -| verifier | sonnet (pinned) | fresh gate (≤3×/loop) | -| security-auditor | opus (judge row: opus/xhigh, the user's first-use decision; frontmatter floor aligned) | fresh gate (≤3×/loop) | +| verifier | Fable (`audit` row: best/xhigh, opus frontmatter floor) | fresh gate (≤3×/loop) | +| security-auditor | Fable (`audit` row: best/xhigh, opus frontmatter floor) | fresh gate (≤3×/loop) | | commit-changer, release-executor, code-cleaner | sonnet (pinned) | dispatched execution — grouping+commit / release spans / approved cleanup (audit + approval gates stay in the dispatcher) | | onboarder, scaffolder, refactorer, validator-analyzer, plugin-probe | sonnet (pinned) | workers — config generation, scaffold, refactor, deterministic W3C/WCAG runner, mechanical plugin probe | | status-reporter | haiku (pinned) | mechanical collector | @@ -109,12 +109,13 @@ Second axis of the same table: how hard each phase thinks. Session default model-router mod (below): the phase table and one phase row per repo skill and agent, each phase naming a tier and an effort level (`plan` best/xhigh, `reflect` best/high, `orchestrate` best/medium, -`escalate` best/max, `judge` big/xhigh, `implement` work/medium, `write` +`escalate` best/max, `judge` big/xhigh, `audit` best/xhigh, `implement` work/medium, `write` work/high, `verify` work/xhigh, `explore` work/medium, `apply` work/low, `mechanical` cheap/low). The `model:` and `effort:` frontmatter of every typed agent and user-invoked skill stays as the off-state floor, kept equal -to the rows by the census `lib/tests/effort-routing.test.sh` (a row -changed through the first-use dialog passes with a `WARN floor drift` line +to the rows by the census `lib/tests/effort-routing.test.sh` (a best-tier +row floors on the tier's second alias, opus, since Fable is credit-gated; a +row changed through the first-use dialog passes with a `WARN floor drift` line until its frontmatter follows): low appliers, medium executors, high writers, xhigh judgment and gates (none on haiku, which rejects the parameter); `/status` low … `/ship-feature` @@ -132,7 +133,7 @@ effort, never version. Transcript audit `python3 lib/effort-audit.py`. `mods/model-router/` is a Claude Code mod (a function-hooks plugin) that applies this table per request. It loads in every session through the tracked symlink `skills/model-router`, as `model-router@skills-dir`. -- Main loop: every request gets the route in force. A typed skill with a row routes the main loop to it; a best-tier row (`plan`, `reflect`, `orchestrate`, `escalate`) holds across turns in a run slot until `/route clear`, `/route off`, a user `/model` or a typed skill on a non-best row. A skill without a row leaves the route as it is. A rowed skill the model loads inside a best-tier run moves effort only when its row ranks below the run's model. Main routed onto a model whose context window the conversation outgrew returns to the session model. Before main runs on a model, the one it moves to or the one it keeps (the session's own model included), the mod probes it with one small completion bounded to 8 seconds; the step (and `/route show`, the route tool's answer) waits for it. A timeout or an error marks the model down (reason `probe`, held for `cooldownMinutes` like any breaker hold) and the decision is taken again once. A model already marked down is never probed, and a model that answered is not probed again, `/clear` included, until it is marked down. When a lower model stands in for its tier's head held down (Opus for Fable on a best-tier phase), the phase's own effort goes one notch up: `reflect` xhigh, `plan` max, `orchestrate` high, `escalate` stays max. An effort asked explicitly (`/route ... effort=`, the `route` tool's `effort`) or set by a floor is never raised. `/route show` and the route tool's answer print the level with `(opus stands in for fable)`; the verbose transition line names `stand-in` as a cause. +- Main loop: every request gets the route in force. A typed skill with a row routes the main loop to it; a best-tier row (`plan`, `reflect`, `orchestrate`, `escalate`; `audit` is an agent row, never typed) holds across turns in a run slot until `/route clear`, `/route off`, a user `/model` or a typed skill on a non-best row. A skill without a row leaves the route as it is. A rowed skill the model loads inside a best-tier run moves effort only when its row ranks below the run's model. Main routed onto a model whose context window the conversation outgrew returns to the session model. Before main runs on a model, the one it moves to or the one it keeps (the session's own model included), the mod probes it with one small completion bounded to 8 seconds; the step (and `/route show`, the route tool's answer) waits for it. A timeout or an error marks the model down (reason `probe`, held for `cooldownMinutes` like any breaker hold) and the decision is taken again once. A model already marked down is never probed, and a model that answered is not probed again, `/clear` included, until it is marked down. When a lower model stands in for its tier's head held down (Opus for Fable on a best-tier phase), the phase's own effort goes one notch up: `reflect` xhigh, `plan` max, `orchestrate` high, `escalate` stays max. An effort asked explicitly (`/route ... effort=`, the `route` tool's `effort`) or set by a floor is never raised. `/route show` and the route tool's answer print the level with `(opus stands in for fable)`; the verbose transition line names `stand-in` as a cause. - Sub-agents: a routed agent gets its row's model at spawn, within its tier and only upward from its frontmatter model, and the row's effort on every step. Explicit `model` / `effort` params on the Agent call win; a project-defined agent of the same name keeps its own definition. Built-ins: Explore runs on sonnet/medium, Plan on opus/xhigh. A built-in `general-purpose` agent that main spawns with no `model` param, while main runs below the session model (a downgrade or a breaker fallback), starts on the session model instead of inheriting the lower one; it is not pinned when the session model is marked down, for a fork, a workflow or a nested spawn, or when a project or user definition shadows the built-in. Other agent types are never pinned this way: the hook cannot see their frontmatter model. - Levers inside a run: `ultrathink` in a prompt sets the turn's minimum effort; `/route effort=max` holds until `/route clear`. The built-in `/effort` is not a lever inside a run, rows and routes outrank it. - Prompt classifier: a plain prompt typed in the composer (12 characters or more, not a slash command) that no keyword rule, skill, run slot, `/route` or floor covers is sent to the engine's small fast model, one call, bounded to 3 seconds (the prompt waits for it, typically under a second). The label (`design`, `analysis`, `coding`, `bookkeeping`, `chat`) maps to plan, reflect, implement or apply (`chat` routes nothing), and the turn takes that phase's route: its effort becomes the turn's default, and its model may move the main loop, never above the session model. A `coding` or `bookkeeping` guess moves main down only when `mainModelSwitch` is on and the cold read fits `downgradeMaxTokens`; an `ultrathink` floor in the same turn keeps its own model. No dialog opens and no file is written. The route is gone at the end of the turn, a later route call or typed skill replaces it, and a `/effort` you type between turns drops the guess for that turn only. A typed best-tier skill holds its run slot across turns, so nothing is classified there until `/route clear`. Long prompts are sampled (first 500 and last 1500 code points), so pasted text at the end can steer one turn's route. Three failures in a row pause it for `cooldownMinutes` (one toast, `classifier: paused` in the status line). Never in a sub-agent, mid-turn or under `-p`. Off per session with `/route classify off`, per machine with `"classify": false`. The status line reads `route: classifier `; `/route verbose on` adds the transition line. diff --git a/agents/plan-challenger.md b/agents/plan-challenger.md index f09e1e0..8381ea3 100644 --- a/agents/plan-challenger.md +++ b/agents/plan-challenger.md @@ -107,10 +107,12 @@ the MAIN loop, never here): (correctness / robustness / simplicity), each blind to the others. - MODEL (BDR-076, supersedes the BDR-066 inherit): plan critique is AUDIT JUDGMENT, not a procedural gate — the challenger is routed to the `judge` - row (opus) by the model-router, the `model: opus` frontmatter being the off-state floor (big - tier; the session model stays on the inline loop). Never `model: "sonnet"` — - a silent judgment downgrade. (Contrast the verifier on its `verify` row and - the executors on their `implement` row, both sonnet, oracle-anchored to a contract.) + row (opus) by the model-router, the `model: opus` frontmatter being the + off-state floor (big tier; the session model stays on the inline loop). + Never `model: "sonnet"` — a silent judgment downgrade. (Contrast the + executors on their `implement` row, sonnet, oracle-anchored to a contract; + the verifier and security-auditor sit on the `audit` row, Fable with an + opus floor.) - FAIL-SAFE — never fail open: a malformed/empty verdict, a missing `PROOF`, or a dead challenger → retry ONCE fresh; a 2nd failure → escalate to the human and NAME the lens. Never report "plan challenged" on a silently dropped lens (same diff --git a/agents/verifier.md b/agents/verifier.md index 3fb7116..6a1205f 100644 --- a/agents/verifier.md +++ b/agents/verifier.md @@ -2,7 +2,7 @@ name: verifier description: Fresh independent verifier — reads a CONTRACT file from disk and renders a structured verdict (CONFORME / ECARTS / ERROR) on the implemented diff. Report-only, never fixes. Dispatched fresh at every iteration; receives no iteration history. tools: Read, Grep, Glob, Bash -model: sonnet +model: opus effort: xhigh --- diff --git a/lib/challenge-plan.md b/lib/challenge-plan.md index 56d454c..b769acd 100644 --- a/lib/challenge-plan.md +++ b/lib/challenge-plan.md @@ -47,7 +47,8 @@ JUDGMENT — the challengers are routed to the `judge` row (opus) by the model-r the `model: opus` frontmatter being the off-state floor: a big tier, off the session model. The session model (Fable) keeps only this loop — synthesis, RE-THINK, gate. Never sonnet: that would -silently downgrade the judgment. (The executor gates stay sonnet.) +silently downgrade the judgment. (The executors stay sonnet; the two merge +gates sit on the `audit` row, Fable with an opus floor.) **Lens framing by `KIND`** (the agent's three lenses, read against the artifact): - `build-plan` — will it WORK / will it BREAK / is it needlessly COMPLEX. diff --git a/lib/tests/effort-routing.test.sh b/lib/tests/effort-routing.test.sh index 3d7f855..fab824b 100755 --- a/lib/tests/effort-routing.test.sh +++ b/lib/tests/effort-routing.test.sh @@ -52,12 +52,19 @@ code_phase() { awk '/^ phases: \{/{f=1;next} f&&/^ \},?$/{f=0} f' "$REG" \ | sed -nE "s/^ *$1: \{ tier: '([a-z]+)', effort: '([a-z]+)' \},?$/\1 \2/p" } +# tier_floor -> the model: floor of a tier's agents +# a best-tier row floors on its 2nd alias: inTier keeps the frontmatter when the head is down (Fable credits, BLK-030); a fable floor would spawn the gates on a dead model +tier_floor() { + if [ "$2" != best ]; then tier_head "$1" "$2"; return; fi + awk '/^ tiers: \{/{f=1;next} f&&/^ \},?$/{f=0} f' "$1" \ + | sed -nE "s/^ *best: \['[a-z]+', '([a-z]+)'.*$/\1/p" +} # field_of -> that phase's frontmatter value field_of() { local pe tier pe="$(phase_effort "$1" "$2")" if [ "$3" = effort ]; then printf '%s\n' "${pe#* }"; return; fi - tier="${pe% *}"; tier_head "$REG" "$tier" + tier="${pe% *}"; tier_floor "$REG" "$tier" } # verdict : ok when the # frontmatter equals the row's phase; warn when the row was changed through @@ -87,6 +94,7 @@ RD="$(dump "$RJ")" FIX="$(mktemp -d)"; trap 'rm -rf "$FIX"' EXIT cat > "$FIX/reg.ts" <<'FX' tiers: { + best: ['fable', 'opus', 'sonnet'], big: ['opus', 'fable'], }, FX @@ -110,6 +118,8 @@ FD="$(dump "$FIX/routing.json")" [ "$(phase_effort "$FD" judge)" = "big xhigh" ] && ok || ko "flip: phase" [ -z "$(phase_effort "$FD" nothere)" ] && ok || ko "flip: ghost phase" [ "$(tier_head "$FIX/reg.ts" big)" = "opus" ] && ok || ko "flip: tier head" +[ "$(tier_floor "$FIX/reg.ts" best)" = opus ] && ok || ko "flip: best floors on its 2nd alias" +[ "$(tier_floor "$FIX/reg.ts" big)" = opus ] && ok || ko "flip: big floors on its head" [ "$(changed_of "$FD" skills feat)" = "reflect implement" ] && ok || ko "flip: changed entry" [ -z "$(changed_of "$FD" skills doc)" ] && ok || ko "flip: ghost changed entry" # the drift verdicts (high = reflect's effort, medium = implement's) @@ -147,7 +157,7 @@ check_skill() { while IFS= read -r f; do check_skill "$f"; done < <( cd "$R" && git ls-files 'skills/*/SKILL.md' 'skills-external/*/SKILL.md') -# ── (c) tracked agents with a row: tier head == model:, effort == effort: ─ +# ── (c) tracked agents with a row: tier floor == model:, effort == effort: ─ NO_ROW_AGENTS=" interviewer client-handover-writer " check_agent() { local f="$1" name phase diff --git a/lib/tests/model-routing.test.sh b/lib/tests/model-routing.test.sh index 5352078..ed47650 100755 --- a/lib/tests/model-routing.test.sh +++ b/lib/tests/model-routing.test.sh @@ -20,8 +20,8 @@ done # 3) executor + gate pins has "agents/feater.md" 'model: sonnet' has "agents/hotfixer.md" 'model: sonnet' -has "agents/verifier.md" 'model: sonnet' -has "agents/security-auditor.md" 'model: opus' # judge row since 2026-10-11 (user first-use decision) +has "agents/verifier.md" 'model: opus' # audit row (best tier, opus = outage floor; BLK-032 policy 2026-10-21) +has "agents/security-auditor.md" 'model: opus' # audit row since 2026-10-21 (was judge since 2026-10-11) has "agents/analyzer.md" 'model: opus' # 4) /feat executor shape has "skills/feat/SKILL.md" 'subagent_type="feater"' diff --git a/mods/model-router/hooks/register.test.ts b/mods/model-router/hooks/register.test.ts index aef5dae..6ee9966 100644 --- a/mods/model-router/hooks/register.test.ts +++ b/mods/model-router/hooks/register.test.ts @@ -69,6 +69,7 @@ const baseRouting = (): Rec => ({ orchestrate: { tier: 'best', effort: 'medium' }, escalate: { tier: 'best', effort: 'max' }, judge: { tier: 'big', effort: 'xhigh' }, + audit: { tier: 'best', effort: 'xhigh' }, implement: { tier: 'work', effort: 'medium' }, write: { tier: 'work', effort: 'high' }, verify: { tier: 'work', effort: 'xhigh' }, @@ -1290,6 +1291,32 @@ test('agent row: opus and fable down, no write and one log line', async ( ]) }) +const auditWorld = () => newWorld({ + edit: f => { (f.agents as Rec)['security-auditor'] = 'audit' }, +}) + +test('audit row: fable down, no write, frontmatter runs', async ( + $, on) => { + const logs = recordLogs(on) + const rig = await bootRun($, on, FABLE, auditWorld()) + await switchTo($, FABLE, SONNET, 'auto') + await $.agent.spawn(spawnOf('security-auditor')) + expect(rig.specs).toEqual([undefined]) + const down = logs.filter(l => l.includes('frontmatter model kept')) + expect(down).toEqual([ + 'model-router: security-auditor tier best down, frontmatter model kept', + ]) +}) + +test('audit row: first spawn dies on fable, the retry lands on the floor', + async ($, on) => { + const rig = await bootRun($, on, FABLE, auditWorld()) + await $.agent.spawn(spawnOf('security-auditor')) + await failWith($, 'rate_limit', 'a1') + await $.agent.spawn(spawnOf('security-auditor')) + expect(rig.specs).toEqual([FABLE, undefined]) + }) + test('agent row: explicit model, explicit effort and fork win', async ( $, on) => { const rig = await bootRun($, on) @@ -2552,7 +2579,7 @@ test('/route pending lists what is not confirmed, asked first', async ( await bootRun($, on, FABLE, w) await typeFeat($) const lines = (await route($, 'pending')).split('\n') - expect(lines[0]).toBe('pending confirmation: 19') + expect(lines[0]).toBe('pending confirmation: 20') expect(lines[1]).toBe('asked this session: skills:feat') expect(lines[2]).toContain('agents:feater') expect(lines[2]).toContain('phases:plan') @@ -2564,7 +2591,7 @@ test('/route pending drops a confirmed row', async ($, on) => { await bootRun($, on, FABLE, w) await typeFeat($) const text = await route($, 'pending') - expect(text).toContain('pending confirmation: 17') + expect(text).toContain('pending confirmation: 18') expect(text).not.toContain('phases:reflect') }) diff --git a/mods/model-router/hooks/register.ts b/mods/model-router/hooks/register.ts index 3d2f321..eb586bd 100644 --- a/mods/model-router/hooks/register.ts +++ b/mods/model-router/hooks/register.ts @@ -278,6 +278,7 @@ const DEFAULT_CONFIG: Config = { orchestrate: { tier: 'best', effort: 'medium' }, escalate: { tier: 'best', effort: 'max' }, judge: { tier: 'big', effort: 'xhigh' }, + audit: { tier: 'best', effort: 'xhigh' }, implement: { tier: 'work', effort: 'medium' }, write: { tier: 'work', effort: 'high' }, verify: { tier: 'work', effort: 'xhigh' }, diff --git a/mods/model-router/routing.json b/mods/model-router/routing.json index 38d53b7..c288f55 100644 --- a/mods/model-router/routing.json +++ b/mods/model-router/routing.json @@ -23,7 +23,12 @@ "judge": { "tier": "big", "effort": "xhigh", - "about": "independent judgment of work: audits, challenges and verdicts" + "about": "independent judgment of work: challenges, analyses and reviews" + }, + "audit": { + "tier": "best", + "effort": "xhigh", + "about": "the fresh gates: contract verdict and security scan; dispatched agents, not a main-loop phase" }, "implement": { "tier": "work", @@ -38,7 +43,7 @@ "verify": { "tier": "work", "effort": "xhigh", - "about": "checking a result against its spec: tests, security review, gates" + "about": "checking a result against its spec: tests and suites" }, "explore": { "tier": "work", @@ -138,17 +143,17 @@ "release-executor": "apply", "plugin-probe": "apply", "validator-analyzer": "apply", - "verifier": "verify", - "security-auditor": "judge", + "verifier": "audit", + "security-auditor": "audit", "status-reporter": "mechanical" }, "projects": {}, "confirmed": { "agents": { - "verifier": "verify", + "verifier": "audit", "feater": "implement", "doc-syncer": "write", - "security-auditor": "judge", + "security-auditor": "audit", "plan-challenger": "judge", "analyzer": "judge", "hotfixer": "apply", diff --git a/skills/tour/SKILL.md b/skills/tour/SKILL.md index f62fa78..44ad537 100644 --- a/skills/tour/SKILL.md +++ b/skills/tour/SKILL.md @@ -80,8 +80,8 @@ Model discipline (the user-fixed invariant behind this mode): carries this skill's reflection (fix decisions, convergence calls); it must never be pinned down to an executor tier. - Inside a runner, every dispatched agent keeps the tier this skill - already defines: security-auditor (sonnet frontmatter), the Phase B - audit (analyzer opus pin or `model="opus"`), doc-syncer (audit on + already defines: security-auditor (`audit` row: Fable, opus floor), the + Phase B audit (analyzer opus pin or `model="opus"`), doc-syncer (audit on `model="opus"`, patch on its sonnet frontmatter — BDR-077). Runner dispatch, one per project: