diff --git a/lib/tests/effort-routing.test.sh b/lib/tests/effort-routing.test.sh index 34545d1..c398842 100755 --- a/lib/tests/effort-routing.test.sh +++ b/lib/tests/effort-routing.test.sh @@ -53,19 +53,13 @@ code_phase() { awk '/^ phases: \{/{f=1;next} f&&/^ \},?$/{f=0} f' "$REG" \ | sed -nE "s/^ *$1: \{ tier: '([a-z]+)', effort: '([a-z]+)' \},?$/\1 \2/p" } -# tier_floor -> the model: floor of a tier's agents -# a best-tier row floors on its 2nd alias: inTier keeps the frontmatter when the head is down (Fable credits, BLK-030); a fable floor would spawn the gates on a dead model -tier_floor() { - if [ "$2" != best ]; then tier_head "$1" "$2"; return; fi - awk '/^ tiers: \{/{f=1;next} f&&/^ \},?$/{f=0} f' "$1" \ - | sed -nE "s/^ *best: \['[a-z]+', '([a-z]+)'.*$/\1/p" -} # field_of -> that phase's frontmatter value +# best is headed by opus since 2026-11-16: the floor is the tier head field_of() { local pe tier pe="$(phase_effort "$1" "$2")" if [ "$3" = effort ]; then printf '%s\n' "${pe#* }"; return; fi - tier="${pe% *}"; tier_floor "$REG" "$tier" + tier="${pe% *}"; tier_head "$REG" "$tier" } # verdict : ok when the # frontmatter equals the row's phase; warn when the row was changed through @@ -119,8 +113,7 @@ FD="$(dump "$FIX/routing.json")" [ "$(phase_effort "$FD" judge)" = "big xhigh" ] && ok || ko "flip: phase" [ -z "$(phase_effort "$FD" nothere)" ] && ok || ko "flip: ghost phase" [ "$(tier_head "$FIX/reg.ts" big)" = "opus" ] && ok || ko "flip: tier head" -[ "$(tier_floor "$FIX/reg.ts" best)" = opus ] && ok || ko "flip: best floors on its 2nd alias" -[ "$(tier_floor "$FIX/reg.ts" big)" = opus ] && ok || ko "flip: big floors on its head" +[ "$(tier_head "$FIX/reg.ts" best)" = fable ] && ok || ko "flip: best head" [ "$(changed_of "$FD" skills feat)" = "reflect implement" ] && ok || ko "flip: changed entry" [ -z "$(changed_of "$FD" skills doc)" ] && ok || ko "flip: ghost changed entry" # the drift verdicts (high = reflect's effort, medium = implement's) diff --git a/mods/model-router/hooks/register.test.ts b/mods/model-router/hooks/register.test.ts index 2fd2174..1b2e66a 100644 --- a/mods/model-router/hooks/register.test.ts +++ b/mods/model-router/hooks/register.test.ts @@ -75,6 +75,7 @@ type Fixture = { answers?: string[] repo?: World['repo'] noHome?: boolean // HOME unset: env.get answers undefined + legacyTiers?: boolean // machine override puts fable back in best and big surfaces?: readonly string[] // default ['terminal'] clockFailsAfterAsk?: boolean // see World; the rig gets no MockClock sleepFails?: boolean // see World; the rig gets no MockClock @@ -118,6 +119,21 @@ const baseRouting = (): Rec => ({ askTier: false, }) +// The tiers shipped before 2026-11-16: fable heads best, opus heads big. +const LEGACY_TIERS = { + best: ['fable', 'opus', 'sonnet'], + big: ['opus', 'fable', 'sonnet'], +} + +/** The fixture's home file, with the legacy tiers merged under it. */ +function homeOf(fx: Fixture): Fixture['home'] { + if (!fx.legacyTiers) return fx.home + if (fx.noHome || fx.home === 'broken') { + throw new Error('legacyTiers needs a readable machine override') + } + return { tiers: LEGACY_TIERS, ...fx.home } +} + function newWorld(fx: Fixture = {}): World { const file = baseRouting() file.ask = fx.ask ?? false @@ -128,9 +144,9 @@ function newWorld(fx: Fixture = {}): World { else if (fx.routing !== 'absent') { files.set('routing', JSON.stringify(file, null, 2)) } - if (fx.home !== undefined) { - files.set(HOME_FILE, - fx.home === 'broken' ? '{nope' : JSON.stringify(fx.home)) + const home = homeOf(fx) + if (home !== undefined) { + files.set(HOME_FILE, home === 'broken' ? '{nope' : JSON.stringify(home)) } return { files, @@ -489,7 +505,7 @@ test('the main step carries the routed effort, model untouched', async ( await boot($, on) await $.tool.call({ tool: 'mcp__model-router__route', phase: 'plan' }) await runStep($, stepInput()) - expect(seen).toEqual([{ model: 'claude-fable-5-2', effort: 'xhigh' }]) + expect(seen).toEqual([{ model: 'claude-opus-5-5', effort: 'xhigh' }]) }) test('a tabled agent steps at its table effort', async ($, on) => { @@ -809,12 +825,12 @@ async function bootRigVar( return { rig: await bootRig($, on, model, null, world), usage } } -test('tier: a plan route upgrades a haiku session to fable xhigh', async ( +test('tier: a plan route upgrades a haiku session to opus xhigh', async ( $, on) => { const rig = await bootRig($, on, HAIKU) await planRoute($) expect(await stepOn($, rig, HAIKU)).toEqual({ - model: FABLE, + model: OPUS, effort: 'xhigh', }) }) @@ -823,7 +839,7 @@ test('tier: show prints the resolved id of each phase and a down line', async ( $, on) => { await bootRig($, on) const text = await route($, 'show') - expect(text).toContain('plan=best→claude-fable-5-2/xhigh') + expect(text).toContain('plan=best→claude-opus-5-5/xhigh') expect(text).toContain('judge=big→claude-opus-5-5/xhigh') expect(text).toContain('down: none') }) @@ -925,7 +941,7 @@ test('haiku 5: a plan keyword on a haiku session is xhigh', async ( test('fallback: a rate limit on fable moves plan to opus max', async ( $, on) => { - const rig = await bootRig($, on) + const rig = await bootRig($, on, FABLE, 1000, legacyWorld()) await planRoute($) expect((await stepOn($, rig, FABLE)).model).toBe(FABLE) await failWith($, 'rate_limit') @@ -955,7 +971,7 @@ test('breaker: an invalid_request or a turn error never marks down', async ( test('breaker: the hold lapses with the clock, a second one doubles', async ( $, on) => { - const rig = await bootRig($, on) + const rig = await bootRig($, on, FABLE, 1000, legacyWorld()) await planRoute($) await stepOn($, rig, FABLE) await failWith($, 'overloaded') @@ -970,7 +986,7 @@ test('breaker: the hold lapses with the clock, a second one doubles', async ( test('breaker: a /model command clears the mark of its target', async ( $, on) => { - const rig = await bootRig($, on) + const rig = await bootRig($, on, FABLE, 1000, legacyWorld()) await planRoute($) await stepOn($, rig, FABLE) await failWith($, 'rate_limit') @@ -1014,7 +1030,7 @@ test('unknown: the haiku alias id is in the table, a route moves it', async ( const logs = recordLogs(on) const rig = await bootRig($, on, HAIKU) await planRoute($) - expect((await stepOn($, rig, HAIKU)).model).toBe(FABLE) + expect((await stepOn($, rig, HAIKU)).model).toBe(OPUS) expect(logsWith(logs, 'unknown to the models table')).toBe(0) }) @@ -1090,7 +1106,7 @@ test('per axis: a model-less sticky never hides a turn route tier', async ( const rig = await bootRig($, on, HAIKU) await route($, 'effort=low') await planRoute($) - expect(await stepOn($, rig, HAIKU)).toEqual({ model: FABLE, effort: 'low' }) + expect(await stepOn($, rig, HAIKU)).toEqual({ model: OPUS, effort: 'low' }) }) // ---- texts follow the decision, from the turn's model ------------------ @@ -1117,12 +1133,12 @@ test('text: show names the session model after the turn ended', async ( $, on) => { const rig = await bootRig($, on, HAIKU) await planRoute($) - expect((await stepOn($, rig, HAIKU)).model).toBe(FABLE) + expect((await stepOn($, rig, HAIKU)).model).toBe(OPUS) await endTurn($) await route($, 'mechanical') const line = mainLine(await route($, 'show')) expect(line).toContain(`model ${HAIKU}`) - expect(line).not.toContain(FABLE) + expect(line).not.toContain(OPUS) }) test('text: show reflects a /model command to opus', async ($, on) => { @@ -1180,6 +1196,10 @@ async function bootRun( return bootRig($, on, model, tokens, world) } +/** A world on the pre-2026-11-16 tiers: fable heads best (LEGACY tests). */ +const legacyWorld = (fx: Fixture = {}): World => + newWorld({ legacyTiers: true, ...fx }) + const spawnOf = (agent: string, extra: object = {}) => ({ ...spawnInput(), subagentType: agent, ...extra }) @@ -1308,7 +1328,7 @@ test('run slot: a model-loaded helper skill moves effort only', async ( await loadSkill($, 'feat') await loadSkill($, 'using-git-worktrees') expect(mainLine(await route($, 'show'))).toContain('skill mechanical') - expect(await stepOn($, rig, FABLE)).toEqual({ model: FABLE, effort: 'low' }) + expect(await stepOn($, rig, FABLE)).toEqual({ model: OPUS, effort: 'low' }) await endTurn($) expect(mainLine(await route($, 'show'))).toContain('run reflect') }) @@ -1369,7 +1389,7 @@ test('route answer names the id even with a floor in force', async ( await ultrathink($) const out = await $.tool.call({ tool: ROUTE_TOOL, phase: 'plan' }) const text = JSON.stringify(out) - expect(text).toContain(`model ${FABLE}`) + expect(text).toContain(`model ${OPUS}`) expect(text).toContain('user floor max') }) @@ -1403,12 +1423,12 @@ test('agent skill: a rowed one writes the loop effort', async ($, on) => { expect(await effortAfterSkill($, on, 'status')).toBe('low') }) -test('agent row: judge moves up to fable when opus is down', async ( +test('agent row: judge keeps its frontmatter when opus is down', async ( $, on) => { const rig = await bootRun($, on) await switchTo($, OPUS, SONNET, 'auto') await $.agent.spawn(spawnOf('plan-challenger')) - expect(rig.specs).toEqual([FABLE]) + expect(rig.specs).toEqual([undefined]) }) test('agent row: opus and fable down, no write and one log line', async ( @@ -1426,14 +1446,15 @@ test('agent row: opus and fable down, no write and one log line', async ( ]) }) -const auditWorld = () => newWorld({ +const auditWorld = (fx: Fixture = {}) => newWorld({ + ...fx, edit: f => { (f.agents as Rec)['security-auditor'] = 'audit' }, }) test('audit row: fable down, no write, frontmatter runs', async ( $, on) => { const logs = recordLogs(on) - const rig = await bootRun($, on, FABLE, auditWorld()) + const rig = await bootRun($, on, FABLE, auditWorld({ legacyTiers: true })) await switchTo($, FABLE, SONNET, 'auto') await $.agent.spawn(spawnOf('security-auditor')) expect(rig.specs).toEqual([undefined]) @@ -1445,7 +1466,7 @@ test('audit row: fable down, no write, frontmatter runs', async ( test('audit row: first spawn dies on fable, the retry lands on the floor', async ($, on) => { - const rig = await bootRun($, on, FABLE, auditWorld()) + const rig = await bootRun($, on, FABLE, auditWorld({ legacyTiers: true })) await $.agent.spawn(spawnOf('security-auditor')) await failWith($, 'rate_limit', 'a1') await $.agent.spawn(spawnOf('security-auditor')) @@ -1738,7 +1759,7 @@ test('layers: an invalid routing.json phase falls back by name', async ( }) await bootRun($, on, FABLE, w) const text = await route($, 'show') - expect(text).toContain('reflect=best→claude-fable-5-2/high') + expect(text).toContain('reflect=best→claude-opus-5-5/high') expect(logs.join('\n')).toContain('config phases.reflect ignored') }) @@ -1757,7 +1778,7 @@ test('layers: a routing.json phase gone missing falls back, logged', async ( const logs = recordLogs(on) const w = newWorld({ edit: f => { delete (f.phases as Rec).reflect } }) await bootRun($, on, FABLE, w) - expect(await route($, 'show')).toContain('reflect=best→claude-fable-5-2/high') + expect(await route($, 'show')).toContain('reflect=best→claude-opus-5-5/high') expect(logs.filter(l => l.includes('lacks phases'))).toEqual([ 'model-router: routing.json lacks phases reflect; code defaults kept', ]) @@ -1852,7 +1873,7 @@ test('layers: routing.json unreadable at start means defaults, no rows', async ( const w = newWorld({ routing: 'broken', ask: true }) const rig = await bootRun($, on, FABLE, w) expect(await featerModel($, rig)).toBeUndefined() - expect(await route($, 'show')).toContain('plan=best→claude-fable-5-2/xhigh') + expect(await route($, 'show')).toContain('plan=best→claude-opus-5-5/xhigh') const said = logs.filter(l => l.includes('routing.json unreadable')) expect(said).toHaveLength(1) }) @@ -2026,7 +2047,7 @@ test('T1: a typed skill asks once, with the real id and effort', async ( expect(w.asked).toHaveLength(1) const first = w.asked[0] as Asked expect(first.text).toContain('/feat') - expect(first.text).toContain('claude-fable-5-2 at high') + expect(first.text).toContain('claude-opus-5-5 at high') expect(first.options).toEqual(['Later', 'Keep', 'Change']) expect(first.header).toBe('model-router') }) @@ -2036,7 +2057,7 @@ test('T1: the question follows the model main will really run on', async ( const w = asks(['Later']) await bootRun($, on, SONNET, w) await typeFeat($) - expect(w.asked[0]?.text).toContain('claude-fable-5-2 at high') + expect(w.asked[0]?.text).toContain('claude-opus-5-5 at high') }) test('T1: Keep writes the row and its phase, then the skill never asks', async ( @@ -2561,7 +2582,7 @@ test('T2: Keep writes the agent row and the spawn goes on', async ($, on) => { test('T2: a change routes this very spawn with the new row', async ( $, on) => { - const w = asks([...TO_JUDGE, 'Everywhere']) + const w = asks([...TO_JUDGE, 'Everywhere'], { legacyTiers: true }) const rig = await bootRun($, on, FABLE, w) await $.agent.spawn(spawnOf('feater')) expect(rig.specs).toEqual([OPUS]) @@ -2657,7 +2678,7 @@ test('T3: a phase declared through the route tool asks Keep or Later', async ( await $.tool.call({ tool: ROUTE_TOOL, phase: 'orchestrate' }) const q = w.asked[0] as Asked expect(q.text).toContain('First use of orchestrate on the main loop') - expect(q.text).toContain('claude-fable-5-2 at medium') + expect(q.text).toContain('claude-opus-5-5 at medium') expect(q.options).toEqual(['Later', 'Keep']) }) @@ -2811,7 +2832,7 @@ test('layers: enabled:false in the override survives /clear', async ( const skillPath = (name: string) => `${HOME}/.claude/skills/${name}/SKILL.md` const skillMd = (front: string, body = '# body') => `---\nname: feat\n${front}\nallowed-tools: Read\n---\n${body}\n` -const ROUTED = 'Routed to reflect: claude-fable-5-2 at high. OK?' +const ROUTED = 'Routed to reflect: claude-opus-5-5 at high. OK?' /** A typed /feat with `md` served as its SKILL.md; the world after. */ async function featWith( @@ -2947,7 +2968,7 @@ test('text: the phase about shows in the skill question', async ($, on) => { const w = await featWith($, on, undefined, { edit: setPhase('reflect', { about: 'deep thinking' }) }) expect(w.asked[0]?.text).toBe('First route for /feat. Routed to reflect ' + - '(deep thinking): claude-fable-5-2 at high. OK?') + '(deep thinking): claude-opus-5-5 at high. OK?') }) test('text: the about is cleaned before it enters a question', async ( @@ -3014,7 +3035,7 @@ test('text: a phase question names its about and its consumers', async ( await $.tool.call({ tool: ROUTE_TOOL, phase: 'orchestrate' }) expect(w.asked[0]?.text).toBe('First use of orchestrate on the main loop — ' + 'dispatch work; used by 0 rows, the derived route of dispatches: ' + - 'claude-fable-5-2 at medium. OK?') + 'claude-opus-5-5 at medium. OK?') }) test('text: a phase question counts rows and prompt rules', async ($, on) => { @@ -3022,7 +3043,7 @@ test('text: a phase question counts rows and prompt rules', async ($, on) => { await bootRun($, on, FABLE, w) await $.tool.call({ tool: ROUTE_TOOL, phase: 'reflect' }) expect(w.asked[0]?.text).toBe('First use of reflect on the main loop — ' + - 'used by 1 row, 1 prompt rule: claude-fable-5-2 at high. OK?') + 'used by 1 row, 1 prompt rule: claude-opus-5-5 at high. OK?') }) test('text: a phase question names its floor rule', async ($, on) => { @@ -3030,7 +3051,7 @@ test('text: a phase question names its floor rule', async ($, on) => { await bootRun($, on, FABLE, w) await $.tool.call({ tool: ROUTE_TOOL, phase: 'escalate' }) expect(w.asked[0]?.text).toBe('First use of escalate on the main loop — ' + - 'used by 0 rows, 1 floor rule: claude-fable-5-2 at max. OK?') + 'used by 0 rows, 1 floor rule: claude-opus-5-5 at max. OK?') }) test('T3: a Change answer changes nothing and is said', async ($, on) => { @@ -3043,19 +3064,19 @@ test('T3: a Change answer changes nothing and is said', async ($, on) => { expect(toasts).toContain('answer not recognised, default kept') }) -test('change: the model question offers the four tiered aliases', async ( +test('change: the model question offers the three tiered aliases', async ( $, on) => { const w = asks(['Change', 'Later']) await bootRun($, on, FABLE, w) await typeFeat($) expect(w.asked[1]?.text).toBe('Which model for feat?') expect(w.asked[1]?.options).toEqual([ - 'fable (best)', 'opus (big)', 'sonnet (work)', 'haiku (cheap)', + 'opus (best)', 'sonnet (work)', 'haiku (cheap)', ]) }) for (const [label, alias, efforts] of [ - ['fable (best)', 'fable', ['medium', 'high', 'xhigh', 'max']], + ['opus (best)', 'opus', ['medium', 'high', 'xhigh', 'max']], ['sonnet (work)', 'sonnet', ['low', 'medium', 'high', 'xhigh']], ] as const) { test(`change: ${alias} offers the efforts of its phases, ascending`, async ( @@ -3074,7 +3095,8 @@ for (const [label, phase] of [ ] as const) { test(`change: ${label} has one effort, the question is skipped`, async ( $, on) => { - const w = asks(['Change', label, 'Later']) + const w = asks(['Change', label, 'Later'], + { legacyTiers: label === 'opus (big)' }) await bootRun($, on, FABLE, w) await typeFeat($) expect(w.asked).toHaveLength(3) @@ -3161,16 +3183,16 @@ test('change: with the switch on the toast carries no hint', async ( test('change: an upgrade toast carries no hint', async ($, on) => { const toasts = recordToasts(on) - const w = asks(['Change', 'fable (best)', 'xhigh', 'Everywhere']) + const w = asks(['Change', 'opus (best)', 'xhigh', 'Everywhere']) await bootRun($, on, SONNET, w) await typeFeat($) - expect(toasts).toContain('/feat now runs claude-fable-5-2 at xhigh') + expect(toasts).toContain('/feat now runs claude-opus-5-5 at xhigh') }) test('change: an upgrade the config holds back carries no hint', async ( $, on) => { const toasts = recordToasts(on) - const w = asks(['Change', 'fable (best)', 'xhigh', 'Everywhere'], + const w = asks(['Change', 'opus (best)', 'xhigh', 'Everywhere'], { home: { mainUpgrade: false } }) await bootRun($, on, SONNET, w) await typeFeat($) @@ -3190,6 +3212,7 @@ test('change: a project scope stores the derived phase, keeps the row', async ( test('change: a security auditor moves to opus at xhigh, at once', async ( $, on) => { const w = asks([...TO_JUDGE, 'Everywhere'], { + legacyTiers: true, edit: f => { (f.agents as Rec)['security-auditor'] = 'verify' }, }) const rig = await bootRun($, on, FABLE, w) @@ -3876,7 +3899,7 @@ test('logs: a route-tool plan logs one line, not one per step', async ( await runStep($, highStep()) await runStep($, highStep()) expect(logs).toEqual([ - mainWord('fable/high', 'fable/xhigh', 'plan: route tool'), + mainWord('fable/high', 'opus/xhigh', 'plan: route tool, switch'), ]) }) @@ -3885,7 +3908,8 @@ test('logs: a typed skill names itself as the cause', async ($, on) => { await typeFeat($) await runStep($, stepInput()) expect(logs).toEqual([ - mainWord('fable/low', 'fable/high', 'reflect: skill /feat'), + mainWord('fable/low', 'opus/high', + 'reflect: skill /feat, switch'), ]) }) @@ -3894,7 +3918,7 @@ test('logs: an upgrade rides on the same line', async ($, on) => { await typeFeat($) await runStep($, { ...highStep(), model: SONNET }) expect(logs).toEqual([ - mainWord('sonnet/high', 'fable/high', 'reflect: skill /feat, upgrade'), + mainWord('sonnet/high', 'opus/high', 'reflect: skill /feat, upgrade'), ]) }) @@ -4007,7 +4031,7 @@ test('logs: the transition line precedes the step below it', async ( await planRoute($) await runStep($, highStep()) expect(logs).toEqual([ - mainWord('fable/high', 'fable/xhigh', 'plan: route tool'), + mainWord('fable/high', 'opus/xhigh', 'plan: route tool, switch'), 'STEP', ]) }) @@ -4053,7 +4077,9 @@ for (const [name, text, act] of LEGACY) { const { logs } = await bootLogged($, on) await act($) await runStep($, highStep()) - expect(logs.filter(l => l.includes(text))).toEqual([]) + // the session on fable moves to opus: its transition line is the new one + expect(logs.filter(l => !l.startsWith('main:') && l.includes(text))) + .toEqual([]) }) } @@ -4169,7 +4195,7 @@ test('logs: one agent type to two models is two hand-offs', async ( test('logs: a tier moved up in-tier names the alias that was down', async ( $, on) => { - const { logs } = await bootLogged($, on) + const { logs } = await bootLogged($, on, { world: legacyWorld() }) await switchTo($, OPUS, SONNET, 'auto') await $.agent.spawn(spawnOf('plan-challenger')) expect(handoffs(logs)).toEqual([ @@ -4522,26 +4548,26 @@ test('residual e: a changed.phases entry is never restored', async ($, on) => { expect(w.written).toEqual([]) }) -/** The efforts the Change chain offers for fable, from the second question. */ -async function fableEfforts($: Engine, on: On, fx: Fixture) { - const w = asks(['Change', 'fable (best)', 'Later'], fx) +/** The efforts the Change chain offers for opus, from the second question. */ +async function opusEfforts($: Engine, on: On, fx: Fixture) { + const w = asks(['Change', 'opus (best)', 'Later'], fx) await bootRun($, on, FABLE, w) await typeFeat($) return w.asked[2]?.options } -const FABLE_EFFORTS = ['medium', 'high', 'xhigh', 'max'] +const OPUS_EFFORTS = ['medium', 'high', 'xhigh', 'max'] test('residual f: a null phase in the file is ignored, no crash', async ( $, on) => { const edit = (f: Rec): void => { (f.phases as Rec).plan = null } - expect(await fableEfforts($, on, { edit })).toEqual(FABLE_EFFORTS) + expect(await opusEfforts($, on, { edit })).toEqual(OPUS_EFFORTS) }) test('residual f: a file without phases offers the code defaults', async ( $, on) => { const edit = (f: Rec): void => { delete f.phases } - expect(await fableEfforts($, on, { edit })).toEqual(FABLE_EFFORTS) + expect(await opusEfforts($, on, { edit })).toEqual(OPUS_EFFORTS) }) const CUSTOM_CHEAP = { phases: { custom: { tier: 'cheap', effort: 'high' } } } @@ -4618,7 +4644,7 @@ test('logs: a hostile skill name is cleaned in the main line', async ( await loadSkill($, HOSTILE_SKILL) await runStep($, { ...highStep(), effort: 'low' as const }) const line = logs.find(l => l.includes('skill /')) ?? '' - const name = nameAfter(line, 'skill /', ')') + const name = nameAfter(line, 'skill /', ')').replace(', switch', '') expect(name).not.toBe('') expect(hasCtrl(line)).toBe(false) expect([...name].length).toBeLessThanOrEqual(40) @@ -5249,7 +5275,7 @@ test('classify visibility: verbose says (reflect: classifier)', async ( await typed($, PLAIN) await runStep($, { ...highStep(), effort: 'low' as const }) expect(c.logs).toContain( - mainWord('fable/low', 'fable/high', 'reflect: classifier')) + mainWord('fable/low', 'opus/high', 'reflect: classifier, switch')) }) test('classify: a repeated undefined label never pauses', async ($, on) => { @@ -5550,8 +5576,8 @@ test('spawn: an upgraded main never pins general-purpose', async ($, on) => { const { rig } = await bootRigVar($, on, HAIKU, 1000) await offerOf($, GP, 'built-in') await planRoute($) - expect((await stepOn($, rig, HAIKU)).model).toBe(FABLE) - await $.agent.spawn(spawnOf(GP, { parentModel: FABLE })) + expect((await stepOn($, rig, HAIKU)).model).toBe(OPUS) + await $.agent.spawn(spawnOf(GP, { parentModel: OPUS })) expect(rig.specs).toEqual([undefined]) }) @@ -5731,7 +5757,7 @@ const downLine = (text: string): string => test('probe: a fable that never answers lands the plan step on opus', async ( $, on) => { - const world = probeWorld(hangsOn(FABLE)) + const world = probeWorld(hangsOn(FABLE), { legacyTiers: true }) const rig = await bootRig($, on, OPUS, 1000, world) // a keyword no longer lifts: the route tool carries the plan tier await pastProbe(rig, planRoute($)) @@ -5756,7 +5782,7 @@ test('probe: the route tool answers an opus id, never fable', async ( }) test('probe: an answered probe is cached across turns', async ($, on) => { - const world = newWorld() + const world = legacyWorld() const rig = await bootRig($, on, OPUS, 1000, world) await planRoute($) expect((await stepOn($, rig, OPUS)).model).toBe(FABLE) @@ -5767,7 +5793,7 @@ test('probe: an answered probe is cached across turns', async ($, on) => { }) test('probe: a lapsed hold probes again and holds longer', async ($, on) => { - const world = probeWorld(hangsOn(FABLE)) + const world = probeWorld(hangsOn(FABLE), { legacyTiers: true }) const rig = await bootRig($, on, OPUS, 1000, world) for (const wait of [15, 30]) { // the route tool lifts, a keyword no longer does @@ -5784,7 +5810,7 @@ test('probe: a lapsed hold probes again and holds longer', async ($, on) => { test('probe: a rejection marks the target down like a timeout', async ( $, on) => { - const world = probeWorld(rejectsOn(FABLE)) + const world = probeWorld(rejectsOn(FABLE), { legacyTiers: true }) const rig = await bootRig($, on, OPUS, 1000, world) await planRoute($) expect((await stepOn($, rig, OPUS)).model).toBe(OPUS) @@ -5796,7 +5822,7 @@ for (const [shape, answer] of [ ['a string', 'ok'], ['an object', PROBE_OK], ['nothing', undefined], ] as const) { test(`probe: ${shape} as the answer lets the move go`, async ($, on) => { - const world = probeWorld(() => answer) + const world = probeWorld(() => answer, { legacyTiers: true }) const rig = await bootRig($, on, OPUS, 1000, world) await planRoute($) expect((await stepOn($, rig, OPUS)).model).toBe(FABLE) @@ -5808,7 +5834,8 @@ test('probe: a failure during the flight keeps the target unverified', async ( $, on) => { let open: () => void = () => undefined const gate = new Promise(resolve => { open = resolve }) - const world = probeWorld(async () => { await gate; return PROBE_OK }) + const world = probeWorld(async () => { await gate; return PROBE_OK }, + { legacyTiers: true }) const rig = await bootRig($, on, OPUS, 1000, world) // the route tool's probe is the one in flight (a keyword no longer lifts) const call = planRoute($) @@ -5876,7 +5903,7 @@ test('probe: a session model that never answers leaves for opus on the first ste test('probe: a healthy session model is probed once and kept', async ( $, on) => { - const world = newWorld() + const world = legacyWorld() const rig = await bootRig($, on, FABLE, 1000, world) await planRoute($) expect((await stepOn($, rig, FABLE)).model).toBe(FABLE) @@ -5904,7 +5931,8 @@ test('probe: a lapsed hold on the session model probes it again', async ( test('probe: false in the override makes no completion at all', async ( $, on) => { - const world = probeWorld(hangsOn(FABLE), { home: { probe: false } }) + const world = probeWorld(hangsOn(FABLE), + { legacyTiers: true, home: { probe: false } }) const rig = await bootRig($, on, OPUS, 1000, world) await planRoute($) expect((await stepOn($, rig, OPUS)).model).toBe(FABLE) @@ -5915,7 +5943,7 @@ test('probe: false in the override makes no completion at all', async ( test('probe: a non-boolean setting is ignored, logged, probe stays on', async ( $, on) => { const logs = recordLogs(on) - const world = newWorld({ home: { probe: 'no' } }) + const world = legacyWorld({ home: { probe: 'no' } }) const rig = await bootRig($, on, OPUS, 1000, world) await planRoute($) await stepOn($, rig, OPUS) @@ -5926,7 +5954,7 @@ test('probe: a non-boolean setting is ignored, logged, probe stays on', async ( test('probe: /clear keeps the verified models', async ($, on) => { on('session.end', ($, e) => ({ sessionId: e.sessionId })) - const world = newWorld() + const world = legacyWorld() const rig = await bootRig($, on, OPUS, 1000, world) await planRoute($) await stepOn($, rig, OPUS) @@ -5946,7 +5974,7 @@ async function fableHeld($: Engine, rig: Rig): Promise { test('stand-in: reflect on opus runs xhigh, and the answer says so', async ( $, on) => { - const rig = await bootRig($, on) + const rig = await bootRig($, on, FABLE, 1000, legacyWorld()) await fableHeld($, rig) const out = await $.tool.call({ tool: ROUTE_TOOL, phase: 'reflect' }) expect(JSON.stringify(out)).toContain( @@ -5956,7 +5984,7 @@ test('stand-in: reflect on opus runs xhigh, and the answer says so', async ( test('stand-in: orchestrate goes medium to high, escalate stays max', async ( $, on) => { - const rig = await bootRig($, on) + const rig = await bootRig($, on, FABLE, 1000, legacyWorld()) await fableHeld($, rig) await $.tool.call({ tool: ROUTE_TOOL, phase: 'orchestrate' }) expect((await stepOn($, rig, FABLE)).effort).toBe('high') @@ -5972,7 +6000,7 @@ test('stand-in: an explicit effort is never bumped', async ($, on) => { }) test('stand-in: fable available, no bump', async ($, on) => { - const rig = await bootRig($, on, OPUS) + const rig = await bootRig($, on, OPUS, 1000, legacyWorld()) await $.tool.call({ tool: ROUTE_TOOL, phase: 'reflect' }) expect(await stepOn($, rig, OPUS)).toEqual({ model: FABLE, effort: 'high' }) }) @@ -6014,7 +6042,7 @@ test('stand-in: an ultrathink floor ends at max, not past it', async ( test('stand-in: the verbose line names it as a cause', async ($, on) => { const logs = recordLogs(on) - const rig = await bootRig($, on) + const rig = await bootRig($, on, FABLE, 1000, legacyWorld()) await route($, 'verbose on') await fableHeld($, rig) await $.tool.call({ tool: ROUTE_TOOL, phase: 'reflect' }) @@ -6048,7 +6076,7 @@ test('keyword cap: ultrathink on an opus session stays on opus at max', async ( test('keyword cap: a route tool call after a keyword still lifts to fable', async ( $, on) => { - const rig = await bootRig($, on, OPUS) + const rig = await bootRig($, on, OPUS, 1000, legacyWorld()) await typed($, 'plan the billing migration') await planRoute($) expect((await stepOn($, rig, OPUS)).model).toBe(FABLE) @@ -6066,7 +6094,7 @@ test('keyword cap: a held fable gives no stand-in notch to a capped keyword', as test('keyword cap: a keyword inside a held best-tier run keeps the run on fable', async ( $, on) => { - const rig = await bootRun($, on, OPUS) + const rig = await bootRun($, on, OPUS, legacyWorld()) await loadSkill($, 'feat') await endTurn($) await typed($, 'plan the next step') @@ -6437,7 +6465,7 @@ for (const scope of ['This conversation', 'This project']) { // ---- bare /model: the router's own dialog --------------------------------- -const STEP1 = ['fable (best)', 'opus (big)', 'sonnet (work)', +const STEP1 = ['fable (custom)', 'opus (best)', 'sonnet (work)', 'Other model (native picker)'] const VARIANTS = ['Standard', '[1m]'] const NO_MODEL = 'model: no model chosen, nothing changed' @@ -6458,7 +6486,7 @@ test('bare model: step 1 lists the table and the native picker', async ( test('bare model: opus, [1m], This project writes the full id', async ( $, on) => { const m = await bootModel($, on, - ['opus (big)', '[1m]', 'This project']) + ['opus (best)', '[1m]', 'This project']) await modelCmd($, '') expect(m.w.asked.map(q => q.options)).toEqual([STEP1, VARIANTS, SCOPES]) const variant = m.w.asked[1] as Asked @@ -6469,7 +6497,7 @@ test('bare model: opus, [1m], This project writes the full id', async ( test('bare model: the current model asks its scope too', async ($, on) => { const m = await bootModel($, on, - ['opus (big)', 'Standard', 'This conversation']) + ['opus (best)', 'Standard', 'This conversation']) expect(await modelCmd($, '')).toContain(`sticky route ${OPUS}`) expect(m.w.asked).toHaveLength(3) expect(m.ran).toEqual([]) @@ -6525,7 +6553,7 @@ test('bare model: Everywhere runs native with the token', async ($, on) => { test('bare model: [1m] with This conversation carries the note', async ( $, on) => { - await bootModel($, on, ['fable (best)', '[1m]', 'This conversation']) + await bootModel($, on, ['fable (custom)', '[1m]', 'This conversation']) expect(await modelCmd($, '')).toContain("[1m] follows the session's variant") }) @@ -6951,9 +6979,12 @@ const INSTEAD = 'Opus instead (big tier)' const STOP = 'Stop asking this session' const TIER_OPTIONS = ['Fable', INSTEAD, STOP] -/** A world with the tier ask on; the first-use `ask` stays off. */ +/** + * A world with the tier ask on; the first-use `ask` stays off. The ask only + * runs when fable heads best: an override puts it back (LEGACY tiers). + */ const asksTier = (answers: string[], fx: Fixture = {}) => - newWorld({ answers, askTier: 'best', ...fx }) + newWorld({ answers, askTier: 'best', legacyTiers: true, ...fx }) /** An extra skill row, for the tests that need a second typed skill. */ const withRows = (rows: Table) => (file: Rec): void => { @@ -7178,6 +7209,18 @@ test('TIER: a rowed spawn asks once per type; Opus instead sticks', async ( expect(rig.specs).toEqual([OPUS, OPUS]) }) +test('TIER: ask-tier off keeps an Opus instead answer on opus', async ( + $, on) => { + const w = asksTier([INSTEAD, INSTEAD], { edit: auditVerifier }) + const rig = await bootRun($, on, OPUS, w) + await typeFeat($) + await $.agent.spawn(spawnOf('verifier')) + expect(await route($, 'ask-tier off')).toBe('ask-tier off') + expect(await lastStep($, rig)).toEqual({ model: OPUS, effort: 'high' }) + await $.agent.spawn(spawnOf('verifier')) + expect(rig.specs).toEqual([OPUS, OPUS]) +}) + test('TIER: Fable on a spawn runs the head', async ($, on) => { const rig = await bootRun($, on, OPUS, asksTier(['Fable'], { edit: auditVerifier, @@ -7199,7 +7242,7 @@ test('TIER: an explicit model, a nested spawn and a work row never ask', async ( test('TIER: the first-use dialog runs first, then the tier ask', async ( $, on) => { - const w = asks(['Keep', INSTEAD], { askTier: 'best' }) + const w = asks(['Keep', INSTEAD], { legacyTiers: true, askTier: 'best' }) const rig = await bootRun($, on, OPUS, w) await typeFeat($) expect(w.asked).toHaveLength(2) @@ -7210,7 +7253,8 @@ test('TIER: the first-use dialog runs first, then the tier ask', async ( test('TIER: a stored spawn answer holds while another dialog is open', async ( $, on) => { - const w = asks(['Keep', INSTEAD], { askTier: 'best', edit: auditVerifier }) + const w = asks(['Keep', INSTEAD], + { legacyTiers: true, askTier: 'best', edit: auditVerifier }) const rig = await bootRun($, on, OPUS, w) await $.agent.spawn(spawnOf('verifier')) const hold = deferred() @@ -7228,7 +7272,7 @@ test('TIER: a stored spawn answer holds while another dialog is open', async ( }) test('TIER: a dialog in flight routes a typed skill unasked', async ($, on) => { - const w = asks(['Keep'], { askTier: 'best' }) + const w = asks(['Keep'], { legacyTiers: true, askTier: 'best' }) const hold = deferred() const open = deferred() w.gate = hold.gate @@ -7290,6 +7334,130 @@ test('TIER: no routing.json means no ask; the route is set', async ($, on) => { expect(mainLine(await route($, 'show'))).toContain('model plan') }) +// ---- SHIPPED tiers: fable is ranked, never an auto-route target ---------- +// No override here: best and big are [opus, sonnet]. A machine override can +// put fable back (the TIER section above runs on those LEGACY tiers). + +const routeTo = (phase: string) => (file: Rec): void => { + (file.skills as Table).feat = phase + ;(file.agents as Table).feater = phase +} + +for (const phase of ['plan', 'judge', 'audit']) { + test(`shipped: a typed ${phase} row routes main to opus`, async ( + $, on) => { + const rig = await bootRun($, on, OPUS, newWorld({ edit: routeTo(phase) })) + await typeFeat($) + expect((await lastStep($, rig)).model).toBe(OPUS) + }) + + test(`shipped: a ${phase} agent row spawns on opus`, async ($, on) => { + const rig = await bootRun($, on, OPUS, newWorld({ edit: routeTo(phase) })) + await $.agent.spawn(spawnOf('feater')) + expect(rig.specs).toEqual([OPUS]) + }) +} + +test('shipped: with opus down best and big never choose fable', async ( + $, on) => { + const logs = recordLogs(on) + const w = newWorld({ edit: auditVerifier }) + const rig = await bootRun($, on, OPUS, w) + await switchTo($, OPUS, SONNET, 'auto') + await planRoute($) + expect((await stepOn($, rig, SONNET)).model).toBe(SONNET) + await $.agent.spawn(spawnOf('plan-challenger')) + await $.agent.spawn(spawnOf('verifier')) + // the frontmatter floor runs: no model is chosen, fable least of all + expect(rig.specs).toEqual([undefined, undefined]) + expect(logs).toContain( + 'model-router: plan-challenger tier big down, frontmatter model kept') + expect(logs).toContain( + 'model-router: verifier tier best down, frontmatter model kept') +}) + +test('shipped: a hand-picked fable ranks above opus, a limit leaves it', async ( + $, on) => { + const h = await bootHand($, on, ['This conversation']) + await modelCmd($, 'fable') + expect((await stepOn($, h.rig, OPUS)).model).toBe(FABLE) + await failWith($, 'rate_limit') + expect((await stepOn($, h.rig, FABLE)).model).toBe(OPUS) +}) + +test('shipped: boot and reload log no "does not lead" line', async ( + $, on) => { + const logs = recordLogs(on) + await bootRun($, on, OPUS) + await route($, 'reload') + expect(logs.filter(l => l.includes('does not lead'))).toEqual([]) +}) + +/** The machine override turns the tier ask on: best is still not fable's. */ +const askOn = (answers: string[], edit?: (file: Rec) => void): World => + newWorld({ answers, home: { askTier: 'best' }, ...(edit ? { edit } : {}) }) + +test('shipped: an audit spawn asks no tier question, the answer stays', async ( + $, on) => { + const w = askOn([INSTEAD], auditVerifier) + const rig = await bootRun($, on, OPUS, w) + await $.agent.spawn(spawnOf('verifier')) + expect(w.asked).toEqual([]) + expect(w.answers).toEqual([INSTEAD]) + expect(rig.specs).toEqual([OPUS]) +}) + +test('shipped: a best row typed from sonnet asks nothing, main on opus', async ( + $, on) => { + const w = askOn([INSTEAD]) + const rig = await bootRun($, on, SONNET, w) + await typeFeat($) + expect(w.asked).toEqual([]) + expect((await lastStep($, rig, SONNET)).model).toBe(OPUS) +}) + +test('shipped: /route ask-tier on says inert, show says inert', async ( + $, on) => { + await bootRun($, on, OPUS, newWorld()) + expect(await route($, 'show')).toContain('ask-tier: off') + expect(await route($, 'ask-tier on')) + .toBe('ask-tier on (inert: best is not headed by fable)') + expect(await route($, 'show')).toContain('ask-tier: inert') + expect(await route($, 'ask-tier off')).toBe('ask-tier off') +}) + +test('legacy: an override heading best and big with fable asks', async ( + $, on) => { + const both = ['fable', 'opus', 'sonnet'] + const w = asksTier([INSTEAD], { + home: { tiers: { best: both, big: both } }, + }) + await bootRun($, on, OPUS, w) + await typeFeat($) + expect(w.asked).toHaveLength(1) + expect(w.asked[0]?.text).toContain('Run it on fable?') +}) + +test('shipped: the Change dialog offers no fable, judge stays judge', async ( + $, on) => { + const w = asks(['Change', 'opus (best)', 'xhigh', 'Everywhere']) + await bootRun($, on, OPUS, w) + await $.agent.spawn(spawnOf('plan-challenger')) + expect(w.asked[1]?.text).toBe('Which model for plan-challenger?') + expect(w.asked[1]?.options).toEqual([ + 'opus (best)', 'sonnet (work)', 'haiku (cheap)', + ]) + expect((routingOf(w).agents as Rec)['plan-challenger']).toBe('judge') +}) + +test('shipped: the bare model dialog still offers fable, as custom', async ( + $, on) => { + const m = await bootModel($, on, []) + await modelCmd($, '') + expect(m.w.asked[0]?.options).toContain('fable (custom)') + expect(m.w.asked[0]?.options).toEqual(STEP1) +}) + // ---- GUARD: main-loop code writes go through a skill -------------------- const GUARD = 'model-router: code is written inside a skill run.' diff --git a/mods/model-router/hooks/register.ts b/mods/model-router/hooks/register.ts index 112c605..721d3ec 100644 --- a/mods/model-router/hooks/register.ts +++ b/mods/model-router/hooks/register.ts @@ -204,7 +204,11 @@ const KEEP = 'Keep' const CHANGE = 'Change' const EVERYWHERE = 'Everywhere' // Tier ask: the answers; INSTEAD_TIER is the tier "Opus instead" resolves by. +// shortcut: the ask-before-Fable dialog only runs when an override heads +// `best` with fable (BLK-030 credit guard), hence the hardcoded label; remove +// the feature if Fable never returns to a tier. const TIER_HEAD = 'Fable' +const TIER_HEAD_ALIAS = 'fable' const TIER_INSTEAD = 'Opus instead (big tier)' const TIER_STOP = 'Stop asking this session' const INSTEAD_TIER = 'big' @@ -223,7 +227,7 @@ const BARE_ALIASES: readonly string[] = ['fable', 'opus', 'sonnet'] const STANDARD = 'Standard' const LONG = '[1m]' const LOCAL_SETTINGS = '.claude/settings.local.json' -const CHOICE_ALIASES: readonly string[] = ['fable', 'opus', 'sonnet', 'haiku'] +const CHOICE_ALIASES: readonly string[] = ['opus', 'sonnet', 'haiku'] const MAX_DESC = 140 // chars of a skill or agent description in a question const MAX_ABOUT = 120 // chars of a phase `about` const MAX_DESCS = 256 // agent descriptions kept (oldest dropped) @@ -294,10 +298,12 @@ const DEFAULT_CONFIG: Config = { fable: 'claude-fable-5-2', }, windows: {}, - // A tier is an ordered alias list: the first one not down is used. + // A tier is an ordered alias list: the first one not down is used. Fable is + // ranked (hand picks, the advisor model) but never an auto-route target + // (user 2026-11-16); a machine override may put it back in a tier. tiers: { - best: ['fable', 'opus', 'sonnet'], - big: ['opus', 'fable', 'sonnet'], + best: ['opus', 'sonnet'], + big: ['opus', 'sonnet'], work: ['sonnet', 'opus'], cheap: ['haiku', 'sonnet'], }, @@ -615,10 +621,6 @@ function mergeRouting( const tiers = mergeTable( base.tiers, user.tiers, 'tiers', acceptTier(models, log), log) const fallback = pickFallback(user.fallback, base.fallback, models, log) - if (tiers.best?.[0] !== fallback[0]) { - log('model-router: tiers.best does not lead the fallback chain; rank ' + - 'comes from the fallback list alone') - } return { tiers, fallback, @@ -1148,7 +1150,7 @@ function availableIn(st: State, aliases: readonly string[]) { function nextAvailable(st: State, after: string | undefined) { const { fallback, models } = st.cfg const at = fallback.findIndex(alias => models[alias] === after) - return availableIn(st, fallback.slice(at + 1)) + return at < 0 ? undefined : availableIn(st, fallback.slice(at + 1)) } /** @@ -1171,14 +1173,27 @@ function headId(st: State): string | undefined { return alias === undefined ? undefined : st.cfg.models[alias] } +/** The ask-before-Fable dialog is fable-specific: best must be headed by it. */ +function fableHeadsBest(st: State): boolean { + const list = hasKey(st.cfg.tiers, BEST) ? st.cfg.tiers[BEST] : undefined + return list?.[0] === TIER_HEAD_ALIAS +} + /** * The head a route is asked about before it applies; undefined = off. The * machine override wins; an absent key reads best, so does any other value - * (asking is the safe side, `warnAskTier` says so once). + * (asking is the safe side, `warnAskTier` says so once). Inert (undefined) + * unless the head of `best` is fable. */ function tierHead(st: State): string | undefined { const value = st.mem.askTierLocal ?? st.mem.askTier - return value === false ? undefined : headId(st) + return value === false || !fableHeadsBest(st) ? undefined : headId(st) +} + +/** The ask is on by config but best is not headed by fable: nothing asks. */ +function tierInert(st: State): boolean { + const value = st.mem.askTierLocal ?? st.mem.askTier + return value !== false && !fableHeadsBest(st) } /** The `big` tier's models, the head left out, in order. */ @@ -1605,7 +1620,8 @@ type Verdict = { */ function wantedModel(st: State, cur: string): string | undefined { const slot = mainModelSlot(st) - const instead = slot?.tierChoice === 'big' && slot.route.tier === BEST + const instead = slot?.tierChoice === 'big' && slot.route.tier === BEST && + fableHeadsBest(st) const name = instead ? insteadName(st) ?? routeName(slot.route) : routeName(slot?.route) if (name === undefined) return undefined @@ -1992,6 +2008,7 @@ async function snapshot($: Api, st: State) { /** `/route show`: the tier ask's state, a Stop answer included. */ function askTierWord(st: State): string { + if (tierInert(st)) return 'inert' if (tierHead(st) === undefined) return 'off' return st.tierPaused ? 'best (paused this session)' : BEST } @@ -2799,7 +2816,7 @@ function headAlias(cfg: Config, route: Route): string | undefined { : aliasOf(cfg, canonical(cfg, route.model)) } -/** "fable (best)": the alias and the tier it heads ("custom" if none). */ +/** "opus (best)": the alias and the tier it heads ("custom" if none). */ function modelLabel(cfg: Config, alias: string): string { const tier = Object.keys(cfg.tiers).find(t => cfg.tiers[t]?.[0] === alias) return `${alias} (${tier ?? 'custom'})` @@ -3244,7 +3261,9 @@ async function spawnTier( ) { if (route === undefined || route.tier !== BEST) return route if (!st.tierSpawn.has(e.subagentType)) await spawnTierAsk($, st, e, route) - return st.tierSpawn.get(e.subagentType) === 'big' + const instead = st.tierSpawn.get(e.subagentType) === 'big' && + fableHeadsBest(st) + return instead ? insteadRoute(st, route) : route } @@ -3309,7 +3328,9 @@ async function flagCommand( const local = flag.local(st.mem) const note = local === undefined ? '' : `; ~/${OVERRIDE} sets ${key} ${ local === false ? 'off' : 'on'} and wins here` - return `${word} ${arg}${note}` + const inert = flag === ASK_TIER_FLAG && arg === 'on' && tierInert(st) + return `${word} ${arg}${note}${ + inert ? ' (inert: best is not headed by fable)' : ''}` } // ---- /route forget: the user takes decisions back -------------------------