From 2cccc43d374faf0db1159316979f2ee579f25fa5 Mon Sep 17 00:00:00 2001 From: bchanot Date: Thu, 15 Oct 2026 00:07:10 +0200 Subject: [PATCH 1/3] fix(model-router): probe a move target before routing main to it The breaker learned availability only from a failed request: with Fable out of usage credits, every new session and every hold lapse routed main to Fable and lost the turn to a 429 (model_requires_usage_credits, typed rate_limit). decideFor now probes the model a decision moves main to, once per session, with a 1-call $.model.complete raced against 8 s: a timeout or rejection marks it down (reason "probe") and re-decides once. The current model and held ids are never probed; `probe: false` in ~/.claude/model-router.json turns it off. When a model ranked below its tier head stands in where the head would have run, the phase's own effort goes one notch up (reflect xhigh, plan max on Opus). --- mods/model-router/hooks/register.test.ts | 302 ++++++++++++++++++++++- mods/model-router/hooks/register.ts | 276 +++++++++++++++++---- 2 files changed, 532 insertions(+), 46 deletions(-) diff --git a/mods/model-router/hooks/register.test.ts b/mods/model-router/hooks/register.test.ts index 288b664..e6447ad 100644 --- a/mods/model-router/hooks/register.test.ts +++ b/mods/model-router/hooks/register.test.ts @@ -25,6 +25,10 @@ const TO_APPLY = ['Change', 'sonnet (work)', 'low'] const TO_JUDGE = ['Change', 'opus (big)'] const SCOPE_ASK = 3 // index of the scope question after a Change chain +// What the engine answers a completion with: an object, not the documented +// string (observed live); the router accepts any resolved shape. +const PROBE_OK = { isAnswered: true, text: 'ok' } as unknown as string + /** What the bottom `fs`, `env`, `session.repo` and dialog hooks serve. */ type World = { files: Map // 'routing' (routing.json) or a full path @@ -40,6 +44,8 @@ type World = { repo: { root: string; remote: string | null } | null | 'fail' env: string | undefined // HOME as env.get answers it (undefined = unset) kinds: Map // per-path stat kind override + complete: (model: string) => unknown // what a completion resolves to + completions: string[] // every model a completion was asked of } type Fixture = { @@ -115,6 +121,8 @@ function newWorld(fx: Fixture = {}): World { repo: fx.repo === undefined ? { root: ROOT, remote: REMOTE } : fx.repo, env: fx.noHome ? undefined : HOME, kinds: new Map(), + complete: () => PROBE_OK, + completions: [], } } @@ -228,6 +236,11 @@ async function boot( on('classic.PostModelSwitch', () => ({})) on('skill.prompt', ($, e) => ({ text: e.text })) on('agent.offer', () => ({ isOffered: true })) + // the probe of a move target: every model answers unless a test says not + on('model.complete', async (_$, e) => { + world.completions.push(e.model) + return { value: (await world.complete(e.model)) as string } + }) installWorld(on, world) if (tokens !== null) usageOf(on, tokens) on('session.start', ($, e) => ({ cwd: e.cwd })) @@ -784,13 +797,14 @@ test('cap: a downgrade under the cold-read cap moves main', async ( expect(sent.effort).toBeUndefined() }) -test('fallback: a rate limit on fable moves plan to opus xhigh', async ( +test('fallback: a rate limit on fable moves plan to opus max', async ( $, on) => { const rig = await bootRig($, on) await planRoute($) expect((await stepOn($, rig, FABLE)).model).toBe(FABLE) await failWith($, 'rate_limit') - expect(await stepOn($, rig, FABLE)).toEqual({ model: OPUS, effort: 'xhigh' }) + // opus stands in for fable: plan's xhigh goes one notch up + expect(await stepOn($, rig, FABLE)).toEqual({ model: OPUS, effort: 'max' }) expect(await route($, 'show')).toContain('claude-fable-5-1 for 15 min') await route($, 'reload') expect((await stepOn($, rig, FABLE)).model).toBe(FABLE) @@ -5489,3 +5503,287 @@ test('ttl: show says 60 min by default and 5 min after a 5m event', async ( await switchTo($, FABLE, FABLE, 'command', '5m') expect(await route($, 'show')).toContain('ttl 5 min)') }) + +// ---- probe: a move target answers before main is sent to it -------------- + +const PROBE_MS = 8000 +const FABLE_PLAN = 'plan the billing migration' + +/** A world whose completions resolve as `complete(model)` says. */ +function probeWorld( + complete: (model: string) => unknown, + fx: Fixture = {}, +): World { + return { ...newWorld(fx), complete } +} + +const hangsOn = (id: string) => (model: string): unknown => + model === id ? never() : PROBE_OK +const rejectsOn = (id: string) => (model: string): unknown => + model === id ? Promise.reject(new Error('credits')) : PROBE_OK + +/** Lets a probe that never answers hit its bound while `work` waits on it. */ +async function pastProbe(rig: Rig, work: Promise): Promise { + await rig.clock.settle() + await rig.clock.advance(PROBE_MS) + return work +} + +/** The `down:` line of `show`. */ +const downLine = (text: string): string => + text.split('\n').find(l => l.startsWith('down:')) ?? '' + +test('probe: a fable that never answers lands the plan step on opus', async ( + $, on) => { + const world = probeWorld(hangsOn(FABLE)) + const rig = await bootRig($, on, OPUS, 1000, world) + await typed($, FABLE_PLAN) + await pastProbe(rig, runStep($, { ...highStep(), model: OPUS })) + expect(rig.seen).toEqual([{ model: OPUS, effort: 'max' }]) + expect(world.completions).toEqual([FABLE]) + expect(downLine(await route($, 'show'))).toContain( + `${FABLE} for 15 min (probe)`) +}) + +test('probe: the route tool answers an opus id, never fable', async ( + $, on) => { + const rig = await bootRig($, on, OPUS, 1000, probeWorld(hangsOn(FABLE))) + const out = await pastProbe(rig, $.tool.call({ + tool: ROUTE_TOOL, + phase: 'reflect', + })) + const text = JSON.stringify(out) + expect(text).toContain(OPUS) + expect(text).not.toContain(FABLE) +}) + +test('probe: an answered probe is cached across turns', async ($, on) => { + const world = newWorld() + const rig = await bootRig($, on, OPUS, 1000, world) + await planRoute($) + expect((await stepOn($, rig, OPUS)).model).toBe(FABLE) + await endTurn($) + await planRoute($) + expect((await stepOn($, rig, OPUS)).model).toBe(FABLE) + expect(world.completions).toEqual([FABLE]) +}) + +test('probe: a lapsed hold probes again and holds longer', async ($, on) => { + const world = probeWorld(hangsOn(FABLE)) + const rig = await bootRig($, on, OPUS, 1000, world) + for (const wait of [15, 30]) { + await typed($, FABLE_PLAN) + await pastProbe(rig, runStep($, { ...highStep(), model: OPUS })) + expect(downLine(await route($, 'show'))).toContain(`for ${wait} min`) + await endTurn($) + await rig.clock.advance(wait * 60000) + } + expect(world.completions).toEqual([FABLE, FABLE]) + expect(rig.seen.map(s => s.model)).toEqual([OPUS, OPUS]) +}) + +test('probe: a rejection marks the target down like a timeout', async ( + $, on) => { + const world = probeWorld(rejectsOn(FABLE)) + const rig = await bootRig($, on, OPUS, 1000, world) + await planRoute($) + expect((await stepOn($, rig, OPUS)).model).toBe(OPUS) + expect(world.completions).toEqual([FABLE]) + expect(downLine(await route($, 'show'))).toContain('(probe)') +}) + +for (const [shape, answer] of [ + ['a string', 'ok'], ['an object', PROBE_OK], ['nothing', undefined], +] as const) { + test(`probe: ${shape} as the answer lets the move go`, async ($, on) => { + const world = probeWorld(() => answer) + const rig = await bootRig($, on, OPUS, 1000, world) + await planRoute($) + expect((await stepOn($, rig, OPUS)).model).toBe(FABLE) + expect(world.completions).toEqual([FABLE]) + }) +} + +test('probe: a failure during the flight keeps the target unverified', async ( + $, on) => { + let open: () => void = () => undefined + const gate = new Promise(resolve => { open = resolve }) + const world = probeWorld(async () => { await gate; return PROBE_OK }) + const rig = await bootRig($, on, OPUS, 1000, world) + await typed($, FABLE_PLAN) + const step = runStep($, { ...highStep(), model: OPUS }) + await rig.clock.settle() + await switchTo($, FABLE, OPUS, 'auto') + open() + await step + expect(rig.seen).toEqual([{ model: OPUS, effort: 'max' }]) + await endTurn($) + await rig.clock.advance(15 * 60000) + await typed($, FABLE_PLAN) + await runStep($, { ...highStep(), model: OPUS }) + expect(world.completions).toEqual([FABLE, FABLE]) +}) + +test('probe: an explicit route onto a held model probes once in all', async ( + $, on) => { + const world = probeWorld(hangsOn(FABLE)) + const rig = await bootRig($, on, OPUS, 1000, world) + await pastProbe(rig, route($, 'fable')) + await runStep($, { ...highStep(), model: OPUS }) + await runStep($, { ...highStep(), model: OPUS }) + await route($, 'show') + // fable is asked once; the second step leaves the held fable for opus, + // a move like any other, so opus is tried the same way + expect(world.completions).toEqual([FABLE, OPUS]) + expect(rig.seen.map(s => s.model)).toEqual([FABLE, OPUS]) +}) + +test('probe: no probe when the classifier guess is above the session', async ( + $, on) => { + const world = probeWorld(hangsOn(FABLE)) + const c = await bootCls($, on, { + model: SONNET, reply: () => 'design', world, + }) + await typed($, PLAIN) + await runStep($, { ...highStep(), model: SONNET }) + expect(world.completions).toEqual([]) + expect(c.seen[c.seen.length - 1]?.model).toBe(SONNET) +}) + +test('probe: no probe when the upgrade switch refuses the move', async ( + $, on) => { + const world = probeWorld(hangsOn(FABLE), { home: { mainUpgrade: false } }) + const rig = await bootRig($, on, OPUS, 1000, world) + await planRoute($) + expect((await stepOn($, rig, OPUS)).model).toBe(OPUS) + expect(world.completions).toEqual([]) +}) + +test('probe: the current model is never probed', async ($, on) => { + const world = probeWorld(hangsOn(FABLE)) + const rig = await bootRig($, on, FABLE, 1000, world) + await planRoute($) + expect((await stepOn($, rig, FABLE)).model).toBe(FABLE) + expect(world.completions).toEqual([]) +}) + +test('probe: false in the override makes no completion at all', async ( + $, on) => { + const world = probeWorld(hangsOn(FABLE), { home: { probe: false } }) + const rig = await bootRig($, on, OPUS, 1000, world) + await planRoute($) + expect((await stepOn($, rig, OPUS)).model).toBe(FABLE) + expect(world.completions).toEqual([]) + expect(await route($, 'show')).toContain('probe: off') +}) + +test('probe: a non-boolean setting is ignored, logged, probe stays on', async ( + $, on) => { + const logs = recordLogs(on) + const world = newWorld({ home: { probe: 'no' } }) + const rig = await bootRig($, on, OPUS, 1000, world) + await planRoute($) + await stepOn($, rig, OPUS) + expect(world.completions).toEqual([FABLE]) + expect(logs.some(l => l.includes('"probe" is not a boolean'))).toBe(true) + expect(await route($, 'show')).toContain('probe: on') +}) + +test('probe: /clear keeps the verified models', async ($, on) => { + on('session.end', ($, e) => ({ sessionId: e.sessionId })) + const world = newWorld() + const rig = await bootRig($, on, OPUS, 1000, world) + await planRoute($) + await stepOn($, rig, OPUS) + await clearSession($) + await planRoute($) + await stepOn($, rig, OPUS) + expect(world.completions).toEqual([FABLE]) +}) + +// ---- stand-in: a lower model runs the tier one notch up ------------------ + +/** Main on fable fails: the hold stands, the next move is a stand-in. */ +async function fableHeld($: Engine, rig: Rig): Promise { + await stepOn($, rig, FABLE) + await failWith($, 'rate_limit') +} + +test('stand-in: reflect on opus runs xhigh, and the answer says so', async ( + $, on) => { + const rig = await bootRig($, on) + await fableHeld($, rig) + const out = await $.tool.call({ tool: ROUTE_TOOL, phase: 'reflect' }) + expect(JSON.stringify(out)).toContain( + 'effort xhigh (opus stands in for fable)') + expect(await stepOn($, rig, FABLE)).toEqual({ model: OPUS, effort: 'xhigh' }) +}) + +test('stand-in: orchestrate goes medium to high, escalate stays max', async ( + $, on) => { + const rig = await bootRig($, on) + await fableHeld($, rig) + await $.tool.call({ tool: ROUTE_TOOL, phase: 'orchestrate' }) + expect((await stepOn($, rig, FABLE)).effort).toBe('high') + await $.tool.call({ tool: ROUTE_TOOL, phase: 'escalate' }) + expect((await stepOn($, rig, FABLE)).effort).toBe('max') +}) + +test('stand-in: an explicit effort is never bumped', async ($, on) => { + const rig = await bootRig($, on) + await fableHeld($, rig) + await $.tool.call({ tool: ROUTE_TOOL, phase: 'reflect', effort: 'high' }) + expect(await stepOn($, rig, FABLE)).toEqual({ model: OPUS, effort: 'high' }) +}) + +test('stand-in: fable available, no bump', async ($, on) => { + const rig = await bootRig($, on, OPUS) + await $.tool.call({ tool: ROUTE_TOOL, phase: 'reflect' }) + expect(await stepOn($, rig, OPUS)).toEqual({ model: FABLE, effort: 'high' }) +}) + +test('stand-in: a classifier guess above the session is no stand-in', async ( + $, on) => { + const c = await bootCls($, on, { model: OPUS, reply: () => 'analysis' }) + await switchTo($, FABLE, OPUS, 'auto') + await typed($, PLAIN) + await runStep($, { ...highStep(), model: OPUS }) + expect(c.seen[c.seen.length - 1]).toEqual({ model: OPUS, effort: 'high' }) +}) + +test('stand-in: with the upgrade switch off, no bump', async ($, on) => { + const world = newWorld({ home: { mainUpgrade: false } }) + const rig = await bootRig($, on, OPUS, 1000, world) + await switchTo($, FABLE, OPUS, 'auto') + await $.tool.call({ tool: ROUTE_TOOL, phase: 'reflect' }) + expect(await stepOn($, rig, OPUS)).toEqual({ model: OPUS, effort: 'high' }) +}) + +test('stand-in: a sticky effort from another slot is not bumped', async ( + $, on) => { + const rig = await bootRig($, on) + await fableHeld($, rig) + await route($, 'effort=low') + await planRoute($) + expect(await stepOn($, rig, FABLE)).toEqual({ model: OPUS, effort: 'low' }) +}) + +test('stand-in: an ultrathink floor ends at max, not past it', async ( + $, on) => { + const rig = await bootRig($, on) + await fableHeld($, rig) + await ultrathink($) + await $.tool.call({ tool: ROUTE_TOOL, phase: 'reflect' }) + expect(await stepOn($, rig, FABLE)).toEqual({ model: OPUS, effort: 'max' }) +}) + +test('stand-in: the verbose line names it as a cause', async ($, on) => { + const logs = recordLogs(on) + const rig = await bootRig($, on) + await route($, 'verbose on') + await fableHeld($, rig) + await $.tool.call({ tool: ROUTE_TOOL, phase: 'reflect' }) + await stepOn($, rig, FABLE) + expect(logs.some(l => l.startsWith('main:') && l.includes('stand-in'))) + .toBe(true) +}) diff --git a/mods/model-router/hooks/register.ts b/mods/model-router/hooks/register.ts index b2a72f8..68fa9f9 100644 --- a/mods/model-router/hooks/register.ts +++ b/mods/model-router/hooks/register.ts @@ -34,6 +34,7 @@ type Config = { mainModelSwitch: boolean // gates DOWNGRADES only verbose: boolean classify: boolean // a small-model guess of the typed prompt's phase + probe: boolean // a move target is tried once before main is sent to it spinner: boolean enabled: boolean // false: every hook passes through (per machine) } @@ -45,6 +46,7 @@ type Routed = { route: Route source: Source label?: string // the skill a skill route came from, for the logs + explicitEffort?: boolean // the effort was asked for, not the phase's own } type Served = { at: number; tokens: number } // ms, context tokens type Hold = { until: number; reason: string } // until: ms, Infinity = reload @@ -92,6 +94,7 @@ type State = { sessionModel: string // the main loop's model as /model shows it, '' unknown down: Map // canonical id -> unavailable until (breaker) strikes: Map // canonical id -> episodes, drives the backoff + verified: Set // canonical ids a probe saw answer (account-wide) agentModels: Map // agentId -> exact id sent at spawn pushed: Pushed | null // derived orchestrate in force (background dispatch) logged: Set // session-once log lines @@ -132,7 +135,7 @@ type RouteInput = { clear?: unknown agentId?: string } -type Picked = { phase: string; route: Route } +type Picked = { phase: string; route: Route; explicitEffort?: boolean } type Effort = StepIn['effort'] type Toggle = 'switch' | 'verbose' | 'classify' type EffortBy = 'floor' | 'sticky' | 'turn' | 'engine' @@ -231,6 +234,8 @@ const LABEL_PHASE: Readonly> = { } const CLASSIFY_MIN = 12 // code points of a typed prompt worth a call const CLASSIFY_MS = 3000 // the call is abandoned after this +const PROBE_MS = 8000 // a move target that has not answered by then is down +const PROBE_TOKENS = 16 // the probe's reply cap const CLASSIFY_MAX = 2000 // code points sent whole; longer: head + tail const CLASSIFY_HEAD = 500 const CLASSIFY_TAIL = 1500 @@ -245,7 +250,7 @@ const CACHE_TTLS: Readonly> = { '1h': 60 * 60000, } const ESTIMATE_DIVISOR = 3 // code points per token, conservative -const TIMEOUT: unique symbol = Symbol('classify timeout') +const TIMEOUT: unique symbol = Symbol('timeout') const DEFAULT_CONFIG: Config = { models: { @@ -304,6 +309,7 @@ const DEFAULT_CONFIG: Config = { mainModelSwitch: true, verbose: false, classify: true, + probe: true, spinner: true, enabled: true, } @@ -635,6 +641,9 @@ function mergeConfig( classify: pickSwitch( 'model-router: config "classify" is not a boolean; ignored', user.classify, base.classify, log), + probe: pickSwitch( + 'model-router: config "probe" is not a boolean; ignored', + user.probe, base.probe, log), spinner: pickBool(user.spinner, base.spinner), enabled: pickEnabled(user.enabled, base.enabled, log), } @@ -968,6 +977,7 @@ function newState(cfg: Config, source: string): State { sessionModel: '', down: new Map(), strikes: new Map(), + verified: new Set(), agentModels: new Map(), pushed: null, logged: new Set(), @@ -995,16 +1005,18 @@ function newState(cfg: Config, source: string): State { /** * /clear rebuilds the state but keeps what is account- or process-wide: - * the breaker, the classifier's pause and effort baseline (a model's quota - * and the user's own /effort outlive the conversation), the model, the - * agent definitions' sources (the listing is not offered again), the config - * with its session toggles and the dialog in flight. The layers are read - * again at the next prompt; the config is swapped only if that read works. + * the breaker and the models a probe verified, the classifier's pause and + * effort baseline (a model's quota and the user's own /effort outlive the + * conversation), the model, the agent definitions' sources (the listing is + * not offered again), the config with its session toggles and the dialog in + * flight. The layers are read again at the next prompt; the config is + * swapped only if that read works. */ function resetSession(st: State): void { const kept = { down: st.down, strikes: st.strikes, + verified: st.verified, agentModels: st.agentModels, sessionModel: st.sessionModel, offers: st.offers, @@ -1146,7 +1158,8 @@ function markDown( st.strikes.set(id, strikes) const until = now + holdMinutes(st.cfg, strikes, reason) * 60000 st.down.set(id, { until, reason }) - $.ui.log(`model-router: ${id} unavailable (${reason}) ${ + st.verified.delete(id) + tryLog($, `model-router: ${id} unavailable (${reason}) ${ holdWord(until, now)}; routing falls back`) } @@ -1447,7 +1460,11 @@ async function addEstimate( const pointsOf = (texts: readonly string[]): number => texts.reduce((n, t) => n + Array.from(t).length, 0) -type Verdict = { call: Call; wanted: string | undefined } +type Verdict = { + call: Call + wanted: string | undefined + standIn: boolean // a lower model runs where the tier's held head would +} /** * The model main is asked to run on from `cur`. A classifier's guess never @@ -1477,9 +1494,123 @@ async function readingFor( return { tokens: withEstimate(st, raw), now: await $.clock.now() } } +/** + * The race of a call against the clock: its value, or TIMEOUT. With a + * `signal` the wait rejects when the dispatch aborts. + */ +function raceTimeout( + $: Api, + call: Promise, + ms: number, + signal?: AbortSignal, +): Promise { + return Promise.race([ + call, + $.clock.sleep(ms, signal === undefined ? {} : { signal }) + .then((): typeof TIMEOUT => TIMEOUT), + ]) +} + +/** True when `id` answered a tiny completion in time; no hook signal. */ +async function probeAnswers($: Api, id: string): Promise { + try { + const ask = { model: id, prompt: 'ok', maxTokens: PROBE_TOKENS } + return (await raceTimeout($, $.model.complete(ask), PROBE_MS)) !== TIMEOUT + } catch { + return false + } +} + +/** + * Tries a move target before main is sent to it. An answer verifies it, + * unless a failure landed meanwhile (the hold wins); a timeout or a + * rejection marks it down. True when the move may go ahead. + */ +async function probeModel($: Api, st: State, id: string): Promise { + if (await probeAnswers($, id)) { + if (!st.down.has(id)) st.verified.add(id) + } else { + markDown($, st, id, 'probe', await $.clock.now()) + } + return !st.down.has(id) +} + +/** The table id a move lands on that still wants a probe, else undefined. */ +function unprobed(st: State, call: Call): string | undefined { + if (!call.moved || !st.cfg.probe || st.off) return undefined + const id = canonical(st.cfg, call.model) + const fresh = !st.verified.has(id) && !st.down.has(id) + return aliasOf(st.cfg, id) !== undefined && fresh ? id : undefined +} + +type Decided = { call: Call; wanted: string | undefined } + +/** The decision, probing a move target once; a failed probe: decide again. */ +async function decideProbed( + $: Api, + st: State, + cur: string, + read: Reading, +): Promise { + const wanted = wantedModel(st, cur) + const call = decideMain(st, cur, wanted, read) + const id = unprobed(st, call) + if (id === undefined || await probeModel($, st, id)) return { call, wanted } + const after = wantedModel(st, cur) + return { call: decideMain(st, cur, after, read), wanted: after } +} + +/** The held head of the tier the main model slot names, or undefined. */ +function heldHead(st: State): string | undefined { + const tier = mainModelSlot(st)?.route.tier + const list = tier !== undefined && hasKey(st.cfg.tiers, tier) + ? st.cfg.tiers[tier] + : undefined + const alias = list?.[0] + const id = alias === undefined ? undefined : st.cfg.models[alias] + return id !== undefined && st.down.has(id) ? id : undefined +} + +/** + * With `head` available, the router would run it: the classifier ceiling + * does not refuse it and the decision lands on it (switch, caps, window). + */ +function wouldRunHead( + st: State, + cur: string, + head: string, + read: Reading, +): boolean { + const capped = mainModelSlot(st)?.source === 'classifier' && + ranksAbove(st, head, canonical(st.cfg, st.sessionModel)) + if (capped) return false + const held = new Map([...st.down].filter(([id]) => id !== head)) + const open: State = { ...st, down: held } + return canonical(st.cfg, decideMain(open, cur, head, read).model) === head +} + +/** + * A lower model runs where the tier's head, held down, would have: the + * decision is that tier's resolution, ranked below the head, and the head + * would have been run. Pure; the step and every text read this one value. + */ +function isStandIn( + st: State, + cur: string, + d: Decided, + read: Reading, +): boolean { + const head = heldHead(st) + if (st.off || head === undefined || d.wanted === undefined) return false + const id = canonical(st.cfg, d.call.model) + return id === d.wanted && ranksAbove(st, head, id) && + wouldRunHead(st, cur, head, read) +} + /** * What the main loop would run on from `cur`: wanted model + decision. The - * step passes the `known` reading it took; a text reads for itself. + * step passes the `known` reading it took; a text reads for itself. A move + * target is probed here, after the decision (see `decideProbed`). */ async function decideFor( $: Api, @@ -1487,9 +1618,9 @@ async function decideFor( cur: string, known?: Reading, ): Promise { - const wanted = wantedModel(st, cur) - const read = known ?? await readingFor($, st, wanted) - return { call: decideMain(st, cur, wanted, read), wanted } + const read = known ?? await readingFor($, st, wantedModel(st, cur)) + const decided = await decideProbed($, st, cur, read) + return { ...decided, standIn: isStandIn(st, cur, decided, read) } } function logCall($: Api, st: State, call: Call): void { @@ -1541,6 +1672,34 @@ function mainEffort(st: State, engine: Effort): Decision { return { effort, by: sticky === undefined ? 'turn' : 'sticky' } } +/** The route slot that supplied a sticky or turn effort, else null. */ +function effortSlot(st: State, by: EffortBy): Routed | null { + if (by === 'sticky') return st.userMain + if (by !== 'turn') return null + return st.turnMain?.route.effort === undefined ? st.runMain : st.turnMain +} + +/** One notch up on LEVELS; `max` stays. */ +const notchUp = (level: Level): Level => + LEVELS[Math.min(rank(level) + 1, LEVELS.length - 1)] ?? level + +type Lifted = Decision & { standIn: boolean } + +/** + * A stand-in model runs the tier one notch up: only the effort the slot + * that names the tier supplied, never an explicit one, a floor's or an + * engine number. `standIn` is true when the level really moved. + */ +function standInEffort(st: State, d: Decision, v: Verdict): Lifted { + const level = d.effort + const slot = effortSlot(st, d.by) + const own = slot !== null && slot === mainModelSlot(st) && + slot.explicitEffort !== true + if (!v.standIn || !own || !isLevel(level)) return { ...d, standIn: false } + const lifted = notchUp(level) + return { effort: lifted, by: d.by, standIn: lifted !== level } +} + const floorWord = (f: Routed): string => `user floor ${f.route.effort ?? '-'} (prompt rule ${f.phase})` @@ -1636,10 +1795,19 @@ function tierWord(st: State, word: string): string { return tier ? `${word} [tier ${name}]` : word } +/** "opus stands in for fable": the model that runs and the held head. */ +function standInWord(st: State, v: Verdict): string { + const head = aliasOf(st.cfg, heldHead(st) ?? '') ?? '?' + const alias = aliasOf(st.cfg, canonical(st.cfg, v.call.model)) ?? '?' + return `${alias} stands in for ${head}` +} + /** Haiku takes no effort: the main line says so when it will run there. */ function effortWord(st: State, v: Verdict): string { if (v.call.model.startsWith(HAIKU)) return '- (haiku takes none)' - return String(mainEffort(st, undefined).effort ?? '-') + const d = standInEffort(st, mainEffort(st, undefined), v) + const word = String(d.effort ?? '-') + return d.standIn ? `${word} (${standInWord(st, v)})` : word } function mainText(st: State, v: Verdict): string { @@ -1697,8 +1865,8 @@ async function show($: Api, st: State): Promise { mainText(st, s), `router: ${routerWord(st)} · switch: ${switchWord(st)} · upgrade: ${ flag(c.mainUpgrade)} · verbose: ${ - flag(c.verbose)} · classify: ${flag(c.classify)} · spinner: ${ - flag(c.spinner)}`, + flag(c.verbose)} · classify: ${flag(c.classify)} · probe: ${ + flag(c.probe)} · spinner: ${flag(c.spinner)}`, downText(st, s.now), `live loops: ${st.loops.size}`, phasesText(st), @@ -1756,7 +1924,9 @@ function parseRoute(cfg: Config, args: string): Routed | string { for (const word of words) { if (!applyToken(cfg, route, word)) return unknownText(cfg, word) } - return { phase: 'custom', route, source: 'user' } + const explicitEffort = route.effort !== undefined + return { phase: 'custom', route, source: 'user', + ...(explicitEffort ? { explicitEffort } : {}) } } async function toggle( @@ -2785,12 +2955,14 @@ function pickRoute(cfg: Config, phase: unknown, effort: unknown) { } const route: Route = { ...named, ...(isLevel(effort) ? { effort } : {}) } const name = typeof phase === 'string' ? phase : `effort-${String(effort)}` - return { phase: name, route } + const explicit = isLevel(effort) ? { explicitEffort: true } : {} + return { phase: name, route, ...explicit } } function applyRoute(st: State, agentId: string | undefined, p: Picked): void { if (agentId === undefined) { - st.turnMain = { phase: p.phase, route: p.route, source: 'model' } + st.turnMain = { phase: p.phase, route: p.route, source: 'model', + ...(p.explicitEffort ? { explicitEffort: true } : {}) } } else { writeLoop(loopOf(st, agentId), p.route, 'route tool') } @@ -2813,16 +2985,15 @@ function clearLoop(st: State, agentId: string | undefined): string { return 'route cleared for this agent' } -/** The id the next main step runs on, with the decision's reason. */ -async function mainAnswer($: Api, st: State): Promise { - const { call } = await snapshot($, st) - return `${idWord(st, call.model)} (${call.why})` -} - /** Main's answer: effort and model asked, the sticky/floor note appended. */ async function mainRoutedText($: Api, st: State, p: Picked) { - const base = `routed main to ${p.phase}: effort ${ - p.route.effort ?? 'unchanged'}, model ${await mainAnswer($, st)}` + const v = await snapshot($, st) + const d = standInEffort(st, mainEffort(st, undefined), v) + const effort = d.standIn + ? `${String(d.effort)} (${standInWord(st, v)})` + : p.route.effort ?? 'unchanged' + const base = `routed main to ${p.phase}: effort ${effort}, model ${ + idWord(st, v.call.model)} (${v.call.why})` const note = mainNote(st, p.route.effort) return note ? `${base}; but ${note}` : base } @@ -3199,7 +3370,9 @@ function mainMove( const [was, now] = [planWord(st.cfg, from), planWord(st.cfg, step.plan)] if (step.why === undefined || was === now) return undefined const { phase, word } = effortCause(st, step.why.by) - const causes = [word, modelCause(st, step.why)].filter(c => c !== undefined) + const standIn = step.why.standIn ? 'stand-in' : undefined + const causes = [word, standIn, modelCause(st, step.why)] + .filter(c => c !== undefined) const lead = phase === undefined ? '' : `${clean(phase, NAME_CAP)}: ` return `main: ${was} → ${now} (${lead}${causes.join(', ')})` } @@ -3309,8 +3482,8 @@ function agentPlan(st: State, e: StepIn): Plan { } /** What a main plan was decided from: the log words are built from it. */ -type Why = { by: EffortBy; call: Call; cur: string } -type Step = { plan: Plan; why?: Why } +type Why = { by: EffortBy; call: Call; cur: string; standIn: boolean } +type Step = { plan: Plan; why?: Why; aborted?: boolean } /** The engine effort of this turn's first main step: the user's default. */ function noteEngineBase(st: State, e: StepIn): void { @@ -3355,27 +3528,43 @@ function capLogged(st: State, call: Call): Call { * the engine's own (verbatim: an engine fallback is respected). Every step * reads the context and the clock and records the model it ran on. */ -async function mainPlan($: Api, st: State, e: StepIn): Promise { +async function mainPlan( + $: Api, + st: State, + e: StepIn, + signal: AbortSignal, +): Promise { noteEngineBase(st, e) if (userMovedEffort(st)) st.turnMain = null - const { effort, by } = mainEffort(st, e.effort) + const base = mainEffort(st, e.effort) await prune($, st) const cur = st.turnModel ?? e.model const raw = await readTokens($) const now = await $.clock.now() - const { call } = await decideFor( - $, st, cur, { tokens: withEstimate(st, raw), now }) + const v = await decideFor($, st, cur, { tokens: withEstimate(st, raw), now }) + const { call } = v + const { effort, by, standIn } = standInEffort(st, base, v) logCall($, st, capLogged(st, call)) + const step = { plan: { model: call.model, effort }, + why: { by, call, cur, standIn } } + // Esc during a probe: the mark stands, but the abandoned step records + // nothing a later step would read as what main ran + if (signal.aborted) return { ...step, aborted: true } if (call.moved) st.turnModel = call.model // the record stays raw: an estimate in it would trip the compaction path noteServed(st, { model: call.model, effort }, { tokens: raw, now }) st.promptEstimate = null // consumed by the turn's first main step - return { plan: { model: call.model, effort }, why: { by, call, cur } } + return step } -async function planStep($: Api, st: State, e: StepIn): Promise { +async function planStep( + $: Api, + st: State, + e: StepIn, + signal: AbortSignal, +): Promise { const step: Step = e.agentId === undefined - ? await mainPlan($, st, e) + ? await mainPlan($, st, e, signal) : { plan: agentPlan(st, e) } // Haiku takes no effort: omit it rather than send a hook-set value. return step.plan.model.startsWith(HAIKU) @@ -3689,11 +3878,13 @@ function registerSpawn(on: On, st: State): void { function registerTurns(on: On, st: State): void { on('turn.step', async function* ($, e, next) { if (st.off) return yield* next(e) - const step = await planStep($, st, e) + const step = await planStep($, st, e, next.signal) const plan = step.plan const changed = plan.model !== e.model || plan.effort !== e.effort const prev = st.lastPlan // main's last plan, before noteMain replaces it - if (e.agentId === undefined) noteMain($, st, plan) + if (e.agentId === undefined && step.aborted !== true) { + noteMain($, st, plan) + } safely(st, $, 'turn.step log', () => logMove($, st, e, prev, step)) const result = yield* next(changed ? withPlan(e, plan) : e) safely(st, $, 'turn.step', () => @@ -3850,11 +4041,8 @@ const isAbort = (err: unknown): boolean => /** The label, or TIMEOUT; rejects like classify or when the dispatch aborts. */ function raceClassify($: Api, text: string, signal: AbortSignal) { - return Promise.race([ - $.model.classify(sampleOf(text), CLASSIFY_LABELS), - $.clock.sleep(CLASSIFY_MS, { signal }) - .then((): typeof TIMEOUT => TIMEOUT), - ]) + const call = $.model.classify(sampleOf(text), CLASSIFY_LABELS) + return raceTimeout($, call, CLASSIFY_MS, signal) } /** The label, or undefined: no answer, an interrupt or a counted failure. */ From da0b4a65bae124af0a17e9b5c9e7e74a3049e09a Mon Sep 17 00:00:00 2001 From: bchanot Date: Thu, 15 Oct 2026 00:12:10 +0200 Subject: [PATCH 2/3] =?UTF-8?q?docs:=20model-router=20move-target=20probe,?= =?UTF-8?q?=20probe=20key,=20stand-in=20effort=20=E2=80=94=20bugfix=20rout?= =?UTF-8?q?er-probe-before-route?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 1 + README.md | 6 +++--- USAGE.md | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 79c38a3..3f4a20d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,7 @@ Format follows [Keep a Changelog](https://keepachangelog.com/) and this project - The five `effort-low` … `effort-max` skills, `lib/effort-pins.txt`, `lib/effort-pins.sh` and `lib/model-check.sh`, with their tests and the effort-pin re-apply steps of `install-plugins.sh` and `update-all.sh`. The model-router rows replace them. A typed `/effort-` no longer exists: use `ultrathink` or `/route effort=max`. Breaking: the next release is 3.0.0. ### Fixed +- model-router probes a move target before routing main to it: with a model out of usage credits, every new session and hold lapse used to route main there and lose the turn to a 429. The model main moves to is tried once with a small completion bounded to 8 s; a timeout or error marks it down (reason `probe`) and the decision is taken again once. A model that answered is not probed again until marked down (kept across `/clear`). `"probe": false` in `~/.claude/model-router.json` turns it off; `/route show` prints `probe:`. When a lower model stands in for its tier's held head, the phase's own effort goes one notch up (reflect xhigh, plan max, orchestrate high on Opus); explicit and floor efforts are never raised. Tests: `mods/model-router/hooks/register.test.ts`. - model-router residuals of waves 1-4 (2026-10-11): the kill switch fails closed (a `~/.claude/model-router.json` present but unreadable at first load, or an `enabled` that is not a boolean, turns the router off and says so; `/route reload` answers `config unreadable; previous config kept` instead of `config reloaded`); a prompt rule whose pattern nests quantifiers (`(a+)+$`) is dropped; a config file that is not a regular file reads as failed; every log of a caught error prints its kind and code only, never the message (a JSON parse error quotes the file); rejected table keys are masked in logs and model alias keys must be short printable names; `__proto__`, `constructor` and `prototype` are never read or written as keys; the agent offers map is capped like the descriptions; a `models` alias added without a `fallback` list is ranked at the end of the default chain (the upgrade gates now apply to it); a dated or variant id maps to its table id only at a `-` boundary (`claude-sonnet-5-50` no longer reads as sonnet); the status refresh and the main-decision log on the step path can no longer unroute a step; the route tool's failure answer says the route may be partly applied. `lib/tests/mods.test.sh` proves `claude plugin test` exists from its usage line (an unknown subcommand exits 0), `doctor.sh` bounds `claude plugin list --json` to 60 s, `update-all.sh` skips `@skills-dir` mods. Kit 428 → 431. - model-router residuals of the confirmation waves: a failed `/route forget` or `/route ask` write logs its cause and answers a generic text; forget answers clean every file-sourced name, phase and key; a restore names a frontmatter file only for a valid skill name and refuses a `__proto__` row; `/route forget` never restores `changed.phases`; the dialog's Change offers and writes shipped phases only (a machine-only phase never reaches routing.json, and an alias no shipped phase uses says so); a failure after the user answered keeps the key asked instead of asking again at every use. - `gitflow delete` (and `finish`) land on the base that contains the branch and drop the branch's upstream before `git branch -d`, so a branch whose upstream lags (manual-push mode) is deleted instead of refused by git (T18k). diff --git a/README.md b/README.md index a7944b4..63f3df3 100644 --- a/README.md +++ b/README.md @@ -132,7 +132,7 @@ effort, never version. Transcript audit `python3 lib/effort-audit.py`. `mods/model-router/` is a Claude Code mod (a function-hooks plugin) that applies this table per request. It loads in every session through the tracked symlink `skills/model-router`, as `model-router@skills-dir`. -- Main loop: every request gets the route in force. A typed skill with a row routes the main loop to it; a best-tier row (`plan`, `reflect`, `orchestrate`, `escalate`) holds across turns in a run slot until `/route clear`, `/route off`, a user `/model` or a typed skill on a non-best row. A skill without a row leaves the route as it is. A rowed skill the model loads inside a best-tier run moves effort only when its row ranks below the run's model. Main routed onto a model whose context window the conversation outgrew returns to the session model. +- Main loop: every request gets the route in force. A typed skill with a row routes the main loop to it; a best-tier row (`plan`, `reflect`, `orchestrate`, `escalate`) holds across turns in a run slot until `/route clear`, `/route off`, a user `/model` or a typed skill on a non-best row. A skill without a row leaves the route as it is. A rowed skill the model loads inside a best-tier run moves effort only when its row ranks below the run's model. Main routed onto a model whose context window the conversation outgrew returns to the session model. Before main moves to another model, the mod probes it with one small completion bounded to 8 seconds; the step (and `/route show`, the route tool's answer) waits for it. A timeout or an error marks the model down (reason `probe`, held for `cooldownMinutes` like any breaker hold) and the decision is taken again once. The current model and a model already marked down are never probed, and a model that answered is not probed again, `/clear` included, until it is marked down. When a lower model stands in for its tier's head held down (Opus for Fable on a best-tier phase), the phase's own effort goes one notch up: `reflect` xhigh, `plan` max, `orchestrate` high, `escalate` stays max. An effort asked explicitly (`/route ... effort=`, the `route` tool's `effort`) or set by a floor is never raised. `/route show` and the route tool's answer print the level with `(opus stands in for fable)`; the verbose transition line names `stand-in` as a cause. - Sub-agents: a routed agent gets its row's model at spawn, within its tier and only upward from its frontmatter model, and the row's effort on every step. Explicit `model` / `effort` params on the Agent call win; a project-defined agent of the same name keeps its own definition. Built-ins: Explore runs on sonnet/medium, Plan on opus/xhigh. A built-in `general-purpose` agent that main spawns with no `model` param, while main runs below the session model (a downgrade or a breaker fallback), starts on the session model instead of inheriting the lower one; it is not pinned when the session model is marked down, for a fork, a workflow or a nested spawn, or when a project or user definition shadows the built-in. Other agent types are never pinned this way: the hook cannot see their frontmatter model. - Levers inside a run: `ultrathink` in a prompt sets the turn's minimum effort; `/route effort=max` holds until `/route clear`. The built-in `/effort` is not a lever inside a run, rows and routes outrank it. - Prompt classifier: a plain prompt typed in the composer (12 characters or more, not a slash command) that no keyword rule, skill, run slot, `/route` or floor covers is sent to the engine's small fast model, one call, bounded to 3 seconds (the prompt waits for it, typically under a second). The label (`design`, `analysis`, `coding`, `bookkeeping`, `chat`) maps to plan, reflect, implement or apply (`chat` routes nothing), and the turn takes that phase's route: its effort becomes the turn's default, and its model may move the main loop, never above the session model. A `coding` or `bookkeeping` guess moves main down only when `mainModelSwitch` is on and the cold read fits `downgradeMaxTokens`; an `ultrathink` floor in the same turn keeps its own model. No dialog opens and no file is written. The route is gone at the end of the turn, a later route call or typed skill replaces it, and a `/effort` you type between turns drops the guess for that turn only. A typed best-tier skill holds its run slot across turns, so nothing is classified there until `/route clear`. Long prompts are sampled (first 500 and last 1500 code points), so pasted text at the end can steer one turn's route. Three failures in a row pause it for `cooldownMinutes` (one toast, `classifier: paused` in the status line). Never in a sub-agent, mid-turn or under `-p`. Off per session with `/route classify off`, per machine with `"classify": false`. The status line reads `route: classifier `; `/route verbose on` adds the transition line. @@ -141,9 +141,9 @@ effort, never version. Transcript audit `python3 lib/effort-audit.py`. - Only a dialog answer, `/route ask on|off` or `/route forget` writes `routing.json`, never the model. Writes are serialized, capped at 64 KiB, and refused when the file is missing (it is never created). Each write leaves the config repo dirty; a toast reminds you to commit it from there. - The spinner suffix and the status line under the prompt show the route in force; the status line adds ` · switch off` when the downgrade switch is off. -Config layers: `mods/model-router/routing.json` (tracked: phases with their tier, effort and `about` line, rows, decisions, `ask`), then the optional per-machine `~/.claude/model-router.json`, which wins. A `.claude/model-router.json` inside a project is never read. Machine keys: `models` (alias → full id), `windows` (context window per full id), `tiers` (ordered alias lists per tier: `best` fable>opus>sonnet, `big` opus>fable>sonnet, `work` sonnet>opus, `cheap` haiku>sonnet; the first alias not down is used), `fallback` (the rank order of the aliases, best first, used by the breaker), `cooldownMinutes` (how long a model stays marked down after an availability error, 15 by default, doubling per episode up to 300), `mainUpgrade` (default `true`: the main loop may move up to a phase's tier), `upgradeMaxTokens` (default 200000: the largest cold read a main-loop upgrade may cost), `downgradeMaxTokens` (default 120000: the same cap for a downgrade; the cold read is the context minus what the target model's prompt cache already holds, taken from main's last step on that model within the prompt-cache TTL, forgotten by `/clear` and `/route reload`. The TTL follows the engine's `cache_ttl` (5 or 60 minutes) carried by main's model-switch events, is 60 minutes until the first one and survives `/clear`. The context also counts the prompt just submitted, which the engine's usage may not include yet: code points / 3 of its text, its context entries and a typed skill's body, summed and added at the turn's first main step unless usage already grew by half of it. A `document` attachment makes the size unknown, and an unknown context size blocks the switch; the step, `/route show`, the first-use dialog and the toast read the same adjusted size), `phases`, `agents`, `skills`, `prompt` (rules), `mainModelSwitch` (default `true`: the main loop may move down to a cheaper model; a value that is not a boolean turns it off and says so), `classify` (default `true`; `false` stops the prompt classifier on that machine), `verbose` (default `false`; `true` logs transitions only, a non-boolean value falls back to `false`), `spinner` (default `true`), `enabled` (default `true`; `false` turns the mod off on that machine), `ask` (overrides routing.json's `ask` on that machine). A `null` value in `agents` or `skills` drops a row. Edit phases and rows by hand in routing.json (a phase's `about`, 120 characters at most, is read from there only and shown in the dialog; a model and effort pair no phase offers needs a new phase there); without it the mod runs the code's default phases, with no rows and no dialog. `/route reload` re-reads both files. +Config layers: `mods/model-router/routing.json` (tracked: phases with their tier, effort and `about` line, rows, decisions, `ask`), then the optional per-machine `~/.claude/model-router.json`, which wins. A `.claude/model-router.json` inside a project is never read. Machine keys: `models` (alias → full id), `windows` (context window per full id), `tiers` (ordered alias lists per tier: `best` fable>opus>sonnet, `big` opus>fable>sonnet, `work` sonnet>opus, `cheap` haiku>sonnet; the first alias not down is used), `fallback` (the rank order of the aliases, best first, used by the breaker), `cooldownMinutes` (how long a model stays marked down after an availability error, 15 by default, doubling per episode up to 300), `mainUpgrade` (default `true`: the main loop may move up to a phase's tier), `upgradeMaxTokens` (default 200000: the largest cold read a main-loop upgrade may cost), `downgradeMaxTokens` (default 120000: the same cap for a downgrade; the cold read is the context minus what the target model's prompt cache already holds, taken from main's last step on that model within the prompt-cache TTL, forgotten by `/clear` and `/route reload`. The TTL follows the engine's `cache_ttl` (5 or 60 minutes) carried by main's model-switch events, is 60 minutes until the first one and survives `/clear`. The context also counts the prompt just submitted, which the engine's usage may not include yet: code points / 3 of its text, its context entries and a typed skill's body, summed and added at the turn's first main step unless usage already grew by half of it. A `document` attachment makes the size unknown, and an unknown context size blocks the switch; the step, `/route show`, the first-use dialog and the toast read the same adjusted size), `phases`, `agents`, `skills`, `prompt` (rules), `mainModelSwitch` (default `true`: the main loop may move down to a cheaper model; a value that is not a boolean turns it off and says so), `classify` (default `true`; `false` stops the prompt classifier on that machine), `probe` (default `true`: a model main moves to is tried once, 8 seconds at most, before the move; `false` skips the probe on that machine; a non-boolean value is ignored and said; `/route show` prints `probe:`), `verbose` (default `false`; `true` logs transitions only, a non-boolean value falls back to `false`), `spinner` (default `true`), `enabled` (default `true`; `false` turns the mod off on that machine), `ask` (overrides routing.json's `ask` on that machine). A `null` value in `agents` or `skills` drops a row. Edit phases and rows by hand in routing.json (a phase's `about`, 120 characters at most, is read from there only and shown in the dialog; a model and effort pair no phase offers needs a new phase there); without it the mod runs the code's default phases, with no rows and no dialog. `/route reload` re-reads both files. -Limits: a main-loop switch re-reads cold the part of the context the target model's cache does not hold; a switch whose cold read is over its cap (`upgradeMaxTokens` up, `downgradeMaxTokens` down) is skipped and main keeps its model, and a downgrade also needs `mainModelSwitch`. The prompt estimate is conservative (3 code points per token), so a long pasted prompt can hold back a switch that would have fit. The hooks send full model ids, so the `models` table has to follow new model versions. +Limits: a main-loop switch re-reads cold the part of the context the target model's cache does not hold; a switch whose cold read is over its cap (`upgradeMaxTokens` up, `downgradeMaxTokens` down) is skipped and main keeps its model, and a downgrade also needs `mainModelSwitch`. The prompt estimate is conservative (3 code points per token), so a long pasted prompt can hold back a switch that would have fit. The hooks send full model ids, so the `models` table has to follow new model versions. A probe costs one small completion per model main moves to, and holds that step up to 8 seconds when the model does not answer. --- diff --git a/USAGE.md b/USAGE.md index 553fb0c..655d75d 100644 --- a/USAGE.md +++ b/USAGE.md @@ -187,7 +187,7 @@ l'outil `mcp__model-router__route` (`lib/effort-shift.md`). Les skills externes vendorés (pile design, superpowers, agent-skills, skills scroll MengTo, 21st) ont aussi leur ligne. -Taper un skill qui a une ligne route la boucle principale dessus. Une ligne du tier best (plan, reflect, orchestrate, escalate) tient d'un tour à l'autre pendant tout le run, jusqu'à `/route clear`, `/route off`, un `/model` tapé ou un skill d'un autre tier tapé. Pour relancer un tour bloqué : `ultrathink` dans le prompt (plancher du tour) ou `/route effort=max` (tient jusqu'à `/route clear`). Le `/effort` intégré n'a pas d'effet dans un run : les lignes et les routes passent devant. Un prompt tapé sans règle, skill ni route en cours passe par un classifieur (un appel au petit modèle, 3 secondes au plus) : le tour prend la route de la phase devinée : son niveau d'effort, et son modèle, jamais au-dessus de celui de la session. Une devinette `coding` ou `bookkeeping` fait descendre la boucle principale vers un modèle moins cher quand `mainModelSwitch` est actif (par défaut) et que la relecture à froid tient sous `downgradeMaxTokens` (120000 tokens par défaut). Cette relecture compte aussi le prompt qui vient d'être envoyé (un token pour 3 caractères), et un document joint bloque la descente pour cette étape. Le cache compte comme chaud pendant le TTL du moteur (`cache_ttl`, 5 ou 60 minutes), 60 minutes par défaut. `/route classify off` ou `"classify": false` dans la config machine le coupe, et un `/effort` tapé entre deux tours annule la devinette pour ce tour seulement. Les sous-agents reçoivent le modèle de leur ligne au lancement (dans leur tier, jamais en dessous de leur frontmatter) et son niveau à chaque étape ; un `model` ou `effort` explicite sur l'appel gagne. Explore tourne en sonnet/medium, Plan en opus/xhigh. Un agent intégré `general-purpose` lancé par la boucle principale sans `model`, pendant que celle-ci tourne sous le modèle de la session (descente, ou repli quand un modèle est en panne), part sur le modèle de la session au lieu d'hériter du modèle inférieur ; les autres types d'agents ne sont jamais épinglés ainsi. `/route show` affiche la route en cours, slot de run compris (`main: run `), et le switch avec son plafond et ce TTL (`switch: on (downgrade, cap 120000 tokens cold, ttl 60 min)`). Première utilisation : la première fois qu'un skill avec ligne est tapé, qu'un agent avec ligne est lancé ou qu'une phase est déclarée sur la boucle principale via l'outil `route`, le mod demande une fois si la route convient. La question donne le contexte : la description du skill (première phrase du frontmatter de son `SKILL.md`) ou celle de l'agent, la phase avec sa ligne `about` (champ de la phase dans `routing.json`), puis le modèle et le niveau réels de l'étape suivante ; une phase déclarée dit aussi qui l'utilise. Pour une ligne : Later, Keep ou Change ; pour une phase déclarée : Later ou Keep. Change demande le modèle (fable, opus, sonnet, haiku, chacun avec son tier), puis le niveau parmi ceux des phases de ce modèle (fable : medium, high, xhigh ou max ; sonnet : low, medium, high ou xhigh ; opus et haiku n'en ont qu'un, la question est sautée), puis Everywhere ou This project only (exception rangée sous le remote origin du dépôt réduit à `host/path`, jamais d'identifiants). Une ligne reste un nom de phase : le couple modèle et niveau désigne une phase existante, et la même phase qu'avant vaut Keep. Un couple qu'aucune phase n'offre s'ajoute à la main comme nouvelle phase dans `routing.json`. Après un changement sur un skill, un toast donne le modèle et le niveau réels, et rappelle `/route switch on` quand le switch est coupé, ou nomme le plafond quand la relecture à froid bloque la descente. La réponse est écrite dans `routing.json` ; Later, un texte libre (Other) ou un dialogue fermé redemande à une session suivante. Un seul dialogue à la fois, jamais en headless (`-p`) ni dans un sous-agent, et le modèle n'écrit jamais ce fichier. Chaque réponse laisse le repo de config modifié : commite-le depuis ce repo. `/route pending` liste ce qui reste à confirmer, `/route ask off|on` coupe ou rallume le dialogue, `/route forget ` efface une décision (une ligne changée revient à la phase livrée ; `all` et `projects` demandent d'abord). +Taper un skill qui a une ligne route la boucle principale dessus. Une ligne du tier best (plan, reflect, orchestrate, escalate) tient d'un tour à l'autre pendant tout le run, jusqu'à `/route clear`, `/route off`, un `/model` tapé ou un skill d'un autre tier tapé. Pour relancer un tour bloqué : `ultrathink` dans le prompt (plancher du tour) ou `/route effort=max` (tient jusqu'à `/route clear`). Le `/effort` intégré n'a pas d'effet dans un run : les lignes et les routes passent devant. Un prompt tapé sans règle, skill ni route en cours passe par un classifieur (un appel au petit modèle, 3 secondes au plus) : le tour prend la route de la phase devinée : son niveau d'effort, et son modèle, jamais au-dessus de celui de la session. Une devinette `coding` ou `bookkeeping` fait descendre la boucle principale vers un modèle moins cher quand `mainModelSwitch` est actif (par défaut) et que la relecture à froid tient sous `downgradeMaxTokens` (120000 tokens par défaut). Cette relecture compte aussi le prompt qui vient d'être envoyé (un token pour 3 caractères), et un document joint bloque la descente pour cette étape. Le cache compte comme chaud pendant le TTL du moteur (`cache_ttl`, 5 ou 60 minutes), 60 minutes par défaut. `/route classify off` ou `"classify": false` dans la config machine le coupe, et un `/effort` tapé entre deux tours annule la devinette pour ce tour seulement. Avant de déplacer la boucle principale vers un autre modèle, le mod l'essaie une fois (un appel minimal, 8 secondes au plus) : sans réponse, ce modèle est marqué en panne (raison `probe`) et la décision est reprise une fois ; `"probe": false` dans la config machine coupe cet essai, et `/route show` affiche `probe:`. Quand un modèle inférieur remplace la tête de son tier en panne (Opus pour Fable), le niveau propre de la phase monte d'un cran (reflect xhigh, plan max, orchestrate high) ; un niveau demandé explicitement ou un plancher ne bouge jamais. Les sous-agents reçoivent le modèle de leur ligne au lancement (dans leur tier, jamais en dessous de leur frontmatter) et son niveau à chaque étape ; un `model` ou `effort` explicite sur l'appel gagne. Explore tourne en sonnet/medium, Plan en opus/xhigh. Un agent intégré `general-purpose` lancé par la boucle principale sans `model`, pendant que celle-ci tourne sous le modèle de la session (descente, ou repli quand un modèle est en panne), part sur le modèle de la session au lieu d'hériter du modèle inférieur ; les autres types d'agents ne sont jamais épinglés ainsi. `/route show` affiche la route en cours, slot de run compris (`main: run `), et le switch avec son plafond et ce TTL (`switch: on (downgrade, cap 120000 tokens cold, ttl 60 min)`). Première utilisation : la première fois qu'un skill avec ligne est tapé, qu'un agent avec ligne est lancé ou qu'une phase est déclarée sur la boucle principale via l'outil `route`, le mod demande une fois si la route convient. La question donne le contexte : la description du skill (première phrase du frontmatter de son `SKILL.md`) ou celle de l'agent, la phase avec sa ligne `about` (champ de la phase dans `routing.json`), puis le modèle et le niveau réels de l'étape suivante ; une phase déclarée dit aussi qui l'utilise. Pour une ligne : Later, Keep ou Change ; pour une phase déclarée : Later ou Keep. Change demande le modèle (fable, opus, sonnet, haiku, chacun avec son tier), puis le niveau parmi ceux des phases de ce modèle (fable : medium, high, xhigh ou max ; sonnet : low, medium, high ou xhigh ; opus et haiku n'en ont qu'un, la question est sautée), puis Everywhere ou This project only (exception rangée sous le remote origin du dépôt réduit à `host/path`, jamais d'identifiants). Une ligne reste un nom de phase : le couple modèle et niveau désigne une phase existante, et la même phase qu'avant vaut Keep. Un couple qu'aucune phase n'offre s'ajoute à la main comme nouvelle phase dans `routing.json`. Après un changement sur un skill, un toast donne le modèle et le niveau réels, et rappelle `/route switch on` quand le switch est coupé, ou nomme le plafond quand la relecture à froid bloque la descente. La réponse est écrite dans `routing.json` ; Later, un texte libre (Other) ou un dialogue fermé redemande à une session suivante. Un seul dialogue à la fois, jamais en headless (`-p`) ni dans un sous-agent, et le modèle n'écrit jamais ce fichier. Chaque réponse laisse le repo de config modifié : commite-le depuis ce repo. `/route pending` liste ce qui reste à confirmer, `/route ask off|on` coupe ou rallume le dialogue, `/route forget ` efface une décision (une ligne changée revient à la phase livrée ; `all` et `projects` demandent d'abord). La config par machine, optionnelle, vit dans `~/.claude/model-router.json` et passe devant `routing.json` ; `"enabled": false` y coupe le mod sur cette machine, `"ask": false` y coupe le dialogue, une ligne à `null` y retire une ligne. Un `.claude/model-router.json` dans un projet n'est jamais lu. Les phases et les lignes se modifient à la main dans `routing.json` ; `/route reload` relit les deux fichiers. From 304270fe5bbc3f12d0f0e43220b5eb3d2f959774 Mon Sep 17 00:00:00 2001 From: bchanot Date: Thu, 15 Oct 2026 00:12:18 +0200 Subject: [PATCH 3/3] chore(memory): BLK-030, LRN-218, TODO + journal, router-probe-before-route contract + plan --- .claude/memory/blockers.md | 7 + .claude/memory/journal.md | 4 + .claude/memory/learnings.md | 5 + .claude/tasks/TODO.md | 6 + ...26-10-14-router-probe-before-route-0005.md | 44 +++++ ...26-10-14-router-probe-before-route-0005.md | 170 ++++++++++++++++++ 6 files changed, 236 insertions(+) create mode 100644 .claude/tasks/contracts/2026-10-14-router-probe-before-route-0005.md create mode 100644 .claude/tasks/plans/2026-10-14-router-probe-before-route-0005.md diff --git a/.claude/memory/blockers.md b/.claude/memory/blockers.md index cbf3a64..e6ccbc1 100644 --- a/.claude/memory/blockers.md +++ b/.claude/memory/blockers.md @@ -49,6 +49,7 @@ rules: | BLK-027 | 2026-10-06 | this machine never ran `make link`/`make plugin`: no global `core.hooksPath` → post-commit push never fired, branches landed ahead of upstream; 11 vendored skills + `~/.claude/.env` missing | resolved (link) / open (plugin) | | BLK-028 | 2026-10-06 | notify-attention on a VS Code client: bell + toast silent-degradation faults (merge of BLK-019 + BLK-020): terminalBell sound default off, ext hooks only terminals born after activation, Code muted in Windows mixer | resolved | | BLK-029 | 2026-10-08 | Claude Code mods (2.1.294): alias → id resolution for a model set by a hook lags the Agent tool's (`sonnet` → `claude-sonnet-5`, 404); Agent tool schema refuses full ids | upstream | +| BLK-030 | 2026-10-15 | model-router kept routing best-tier phases to Fable out of usage credits: each new session + each hold lapse lost a turn to 429 `model_requires_usage_credits`; breaker reactive only | resolved | --- @@ -314,3 +315,9 @@ rules: - **Status**: upstream (report to anthropics/claude-code with the request id `req_011Cfppp7VFt8z2Pd3zJrpUi`); workaround in model-router. - **Reference**: [[LRN-203]], plan `.claude/tasks/plans/2026-10-08-model-router-mod.md`. +## BLK-030 — model-router routes main to Fable out of usage credits; turn lost every session + hold lapse — 2026-10-15 +- **Friction**: Fable 5.1 out of credits. `route(reflect)` answered `claude-fable-5-1 (upgrade)` after hold lapse; next request died `` "You're out of usage credits", `apiError: model_requires_usage_credits`, `error: rate_limit`, HTTP 429. Holds 15→30→60 min, user typed "reprends" each time. +- **Real cause**: availability learned only reactively. Breaker (`markDown`) fed by `classic.StopFailure` + `PostModelSwitch auto`, i.e. AFTER real request died. Hold in-memory w/ backoff; nothing checked move target before main sent there. 429 body generic (no credits marker) → no cheap typed signal. +- **Solution**: 2cccc43 (bugfix/router-probe-before-route). `decideFor` decides first, then probes decided move target once: `$.model.complete` raced vs `PROBE_MS` 8000 via shared `raceTimeout`; timeout/reject → `markDown(..., 'probe')` + ONE re-decision, no further probe. `verified` set kept across `/clear`, cleared by `markDown`. Never probes cur or held id. Config `probe` (default true). Stand-in effort one notch up (reflect xhigh, plan max, orchestrate high) when lower model replaces held tier head where head would have run; explicit/floor effort never bumped. 497 tests. +- **Status**: resolved. Residuals in TODO (explicit `Agent(model:"fable")` pins, spawn path, session model itself out of credits, ≤8 s stall per new session). +- **Reference**: [[LRN-218]], [[BDR-116]], [[BDR-115]], [[LRN-207]], contract `.claude/tasks/contracts/2026-10-14-router-probe-before-route-0005.md`, plan r3 `.claude/tasks/plans/2026-10-14-router-probe-before-route-0005.md`. diff --git a/.claude/memory/journal.md b/.claude/memory/journal.md index 06abff1..1473581 100644 --- a/.claude/memory/journal.md +++ b/.claude/memory/journal.md @@ -611,3 +611,7 @@ rules: ## 2026-10-13 - model-router /feat `downgradeMaxTokens` (user: switch on by default, classifier may switch model early, back up to fable on reflection). Plan r1 context-size cap → challenge: cap must measure COLD READ (LRN-204), bypass dropped, classifier down-only vs session model; confirmation pass FATAL(7), BLOCKER served-record-on-routed-steps-only → r3. feater DONE 450 tests, verifier CONFORME 10/10, security PASS, commit 2ce203a on feature/router-downgrade-cap. Doc-sync + BDR/LRN/EVAL pending. - Residuals /feat (user "fix les residus"): plan r1 → 3 lenses FATAL×2 (BLOCKER: `e.model===undefined && route===undefined` ≠ "inherits", frontmatter model invisible to the hook) → allowlist `INHERITING_AGENTS` + built-in source + below-session guard; estimate via shared `withEstimate`, raw served record; TTL enum lookup. Confirmation CONCERNS(3) → r3. feater DONE 472 tests, verifier CONFORME 7/7, security PASS, commit 1be57a3. Doc-sync + capitalize pending. + +## 2026-10-15 +- /bugfix model-router: Fable out of credits still routed (BLK-030). Live check: `complete()` on credit-less model never settles → probe w/ 8 s timeout (LRN-218). Plan r1 BLOCKERs (loop, cur probed) → r3 decide-then-probe. User: stand-in effort one notch up, `probe` key, explicit fable pins out of scope. bugfixer 497 tests, GATE 0 MET, verifier CONFORME 11/11, security PASS. Commit 2cccc43, docs P1-P5, merged to develop on user signal. + diff --git a/.claude/memory/learnings.md b/.claude/memory/learnings.md index fe19c2f..146f4d0 100644 --- a/.claude/memory/learnings.md +++ b/.claude/memory/learnings.md @@ -1809,3 +1809,8 @@ Rule: when editing a doctrine file under structure locks, grep the test's lock s ## LRN-217 — In an `agent.spawn` hook, "no explicit `model` and no routing row" does NOT mean "inherits the parent": the agent's frontmatter `model:` is invisible to the hook; only an allowlist of types known to inherit is safe, and `agent.offer.source` tells a built-in from a shadowing definition - **Context**: model-router residuals 2026-10-13, plan r1 pinned every unrowed, model-less spawn to the session model during a demoted turn. Two lenses BLOCKER: engine 2.1.296 builds the spawn input as `model: e.model` (Agent tool param only); frontmatter model never reaches `agent.spawn` nor `agent.offer`; `route === undefined` also covers frozen forks/workflows (hook model ignored) and project-source skips; nested spawns carry the SUB-AGENT's `parentModel`; `turnModel` is set on upgrades too. A repo/user `general-purpose.md` can shadow the built-in with its own `model:` → `offers.get(type) === 'built-in'` is the discriminator. - **Apply**: before rewriting a spawn's model, prove the agent would inherit: allowlist of built-in types with no frontmatter (`general-purpose`), source `built-in`, not fork/workflow, `parentAgentId` undefined, and the parent really sits below the baseline. Record what the hook cannot see as a `shortcut:` at the allowlist. Links [[BDR-116]], [[BDR-115]], [[LRN-203]]. + +## LRN-218 — `$.model.complete` as availability probe: credits-exhausted model NEVER settles (timeout = only failure signal); answer is an object; probe the decided target only, after the decision +- **Context**: model-router bugfix 2026-10-14/15, live scratch mod (`claude -p --plugin-dir`). `complete({model:'claude-opus-5-5', prompt:'ok', maxTokens:1})` resolved 1308 ms with OBJECT `{ isAnswered, text, usage }` (d.ts says string; maxTokens not honored, 43 output tokens). Same call on `claude-fable-5-1` without credits: no settle, no rejection after 900 s (client retries the 429). Plan r1 probed in a loop before deciding → BLOCKERs: unbounded spin on explicit alias, cur probed. +- **Apply**: availability check via `complete()` = race vs own timeout, treat timeout + rejection as down, any resolved shape as up; never read `isAnswered:false` as failure (unobserved). Decide first, probe only the move target, once, skip cur and held ids, one re-decision without probe. No hook signal on the probe (Esc must not mark model down); probe sleep counts against 10 s `turn.step` budget. Links [[BLK-030]], [[LRN-207]]. + diff --git a/.claude/tasks/TODO.md b/.claude/tasks/TODO.md index 01dd11f..2d21e73 100644 --- a/.claude/tasks/TODO.md +++ b/.claude/tasks/TODO.md @@ -1,5 +1,11 @@ # TODO +## 2026-10-15 — model-router: probe a move target before routing main to it (bugfix/router-probe-before-route) +Contract `.claude/tasks/contracts/2026-10-14-router-probe-before-route-0005.md`, plan r3 `.claude/tasks/plans/2026-10-14-router-probe-before-route-0005.md`. +- [x] /bugfix run: Fable out of credits routed (BLK-030, LRN-218); r1 → r3; 497 tests, verifier CONFORME 11/11, security PASS; commit 2cccc43 + docs. +- [ ] Residuals (accepted): explicit `Agent(model:"fable")` pins (client-handover) still fail w/o credits (BDR-115 kept); spawn path not probed; session model itself out of credits → first turn fails reactively (cur never probed); no cross-session persistence → ≤8 s stall per new session; timed-out probe keeps retrying in engine (no cancel); `[1m]` gating unverified (bare id probed); no Esc regression test (kit cannot abort signal); probe sleep eats ~8 of 10 s `turn.step` budget. +- [ ] LIVE CHECK after `/reload-plugins`: best-tier phase w/ Fable out of credits → `/route show` lists fable down (probe), main on opus, effort bumped. + ## 2026-10-13 — model-router: cold-read cap, switch on by default, classifier routes down (feature/router-downgrade-cap) Contract `.claude/tasks/contracts/2026-10-13-router-downgrade-cap-0013.md`, plan r3 `.claude/tasks/plans/2026-10-13-router-downgrade-cap-0013.md`. - [x] /feat run: 3 lenses (10 MAJOR) → r2; confirmation pass FATAL(7) (BLOCKER: served record only on routed steps) → r3; feater DONE (19 tests + 3 extra, 450 pass), GATE 0 MET 4/4, verifier CONFORME 10/10 first pass, security PASS. Commit 2ce203a. diff --git a/.claude/tasks/contracts/2026-10-14-router-probe-before-route-0005.md b/.claude/tasks/contracts/2026-10-14-router-probe-before-route-0005.md new file mode 100644 index 0000000..f8ede3b --- /dev/null +++ b/.claude/tasks/contracts/2026-10-14-router-probe-before-route-0005.md @@ -0,0 +1,44 @@ +# CONTRACT — router-probe-before-route +- date: 2026-10-14 | flow: bugfix | branch: bugfix/router-probe-before-route +- status: active +- plan: .claude/tasks/plans/2026-10-14-router-probe-before-route-0005.md (r3) + +## REQUEST (verbatim — IMMUTABLE) +dans le model-router, quand on a plus d'usage fable 5, comme maintenant, il essaie quand meme de router vers fable 5, il faudrait detecter quand il n'y a plus de credit fable, alors il faut utiliser opus 5.5 xhigh ou max a la place. et donc verifier si fable est dispo avant de router vers fable + +## CLARIFICATIONS +Q: Opus effort when it stands in for Fable? / A: one notch up (orchestrate medium→high, reflect high→xhigh, plan xhigh→max, escalate max) [gated 2026-10-14] +Q: Explicit Agent(model:"fable") pins (client-handover)? / A: leave them (BDR-115 kept), log a TODO residual [gated 2026-10-14] +Q: Approve plan r3? / A: yes [gated 2026-10-14] +Q: Config key `probe` (bool, default true) in ~/.claude/model-router.json to turn the probe off alone? / A: yes [gated 2026-10-14] +Q: plan test 8 records FABLE then OPUS probes (leaveDown leaves the held explicit fable for opus, an unverified move target)? / A: accepted (orchestrator, internal) — fable probed once, no spin; criterion = no re-probe of a held id [gated 2026-10-14] +Q: criterion 9 (Esc) has no regression test: claude-code/testing cannot abort the dispatch signal? / A: accepted (orchestrator, internal) — verifier judges the code path; residual noted [gated 2026-10-14] + +## ACCEPTANCE CRITERIA +1. The model-router mod test suite passes with zero failures. + CHECK: cd /home/bchanot/Documents/claude && out=$(claude plugin test mods/model-router 2>&1); rc=$?; printf '%s\n' "$out" | tail -4; [ "$rc" -eq 0 ] && printf '%s\n' "$out" | grep -Eq '^ *0 fail$' && ! printf '%s\n' "$out" | grep -q '^(fail)' && echo SUITE_GREEN + EXPECT: SUITE_GREEN + EVIDENCE: MET exit=0 marker-found :: 497 pass 0 fail Ran 497 tests across 1 file. [23.89s] SUITE_GREEN +2. Regression tests for the probe exist and pass: at least 9 passing tests whose names start with `probe:` (plan §3 tests 1-4, 6-11), among them one where the fable probe times out and the first step of a plan route lands on opus with exactly one model.complete call and no StopFailure. + CHECK: cd /home/bchanot/Documents/claude && out=$(claude plugin test mods/model-router 2>&1); n=$(printf '%s\n' "$out" | grep -c '^(pass) probe:'); f=$(printf '%s\n' "$out" | grep -c '^(fail) probe:'); echo "probe pass=$n fail=$f"; [ "$n" -ge 9 ] && [ "$f" -eq 0 ] && echo PROBE_TESTS_PASS + EXPECT: PROBE_TESTS_PASS + EVIDENCE: MET exit=0 marker-found :: probe pass=16 fail=0 PROBE_TESTS_PASS +3. Regression tests for the stand-in effort exist and pass: at least 5 passing tests whose names start with `stand-in:` (plan §3 test 5 cases). + CHECK: cd /home/bchanot/Documents/claude && out=$(claude plugin test mods/model-router 2>&1); n=$(printf '%s\n' "$out" | grep -c '^(pass) stand-in:'); f=$(printf '%s\n' "$out" | grep -c '^(fail) stand-in:'); echo "stand-in pass=$n fail=$f"; [ "$n" -ge 5 ] && [ "$f" -eq 0 ] && echo STANDIN_TESTS_PASS + EXPECT: STANDIN_TESTS_PASS + EVIDENCE: MET exit=0 marker-found :: stand-in pass=9 fail=0 STANDIN_TESTS_PASS +4. The mods census (manifest, link, `claude plugin validate`, `claude plugin test`) passes. + CHECK: cd /home/bchanot/Documents/claude && out=$(make test suite=lib/tests/mods.test.sh 2>&1); printf '%s\n' "$out" | tail -3; printf '%s\n' "$out" | grep -Eq '^mods: [0-9]+ pass, 0 fail$' && ! printf '%s\n' "$out" | grep -q '^FAIL ' && echo MODS_GREEN + EXPECT: MODS_GREEN + EVIDENCE: MET exit=0 marker-found :: PASS model-router: plugin test passed mods: 4 pass, 0 fail all suites green MODS_GREEN +5. Probe semantics (plan §1): the probe fires only in `decideFor`, AFTER `decideMain`, only when the call moves main to a table model that is neither verified nor held down; it probes the bare canonical id via `$.model.complete` raced against PROBE_MS (8000) through one `raceTimeout` helper shared with the classifier; no hook abort signal on the probe; timeout or rejection → `markDown(id, 'probe')` then ONE re-decision without a further probe; success (any resolved shape) → verified only if not held at resolution; `markDown` drops the id from `verified` and logs through `tryLog`; `verified` survives `/clear`; the current model is never probed; the spawn path is unchanged. +6. Stand-in effort (plan §2): main only; one notch up on LEVELS (max stays max) exactly when the counterfactual rule holds (tier route, head held down, decided model = tier resolution ranked below head, and with the head available the router would have run it: classifier ceiling, upgrade switch, cold-read cap, window); never when the effort came from a different slot than the tier route, an explicit effort (`explicitEffort` flag via parseRoute and Picked→applyRoute), a prompt floor, or a non-Level engine effort; computed once and carried on the Verdict; `effortWord`, `mainRoutedText` and the verbose transition line report it ("stands in", "stand-in"). +7. Truthful route answer: with the fable probe failing, `route(phase=reflect)` answers an opus id, never `claude-fable-5-1`. +8. Config key `probe` (boolean, default true) parsed and validated like `classify`, settable in `~/.claude/model-router.json`, shown in `/route show`; `probe: false` → no `model.complete` call at all. +9. Esc during a probe: the mark stands but `mainPlan` writes no turn state (`turnModel`, served record, prompt estimate, `lastPlan`) when the dispatch signal has aborted. +10. Existing tests whose expectations change under the bump are updated, not deleted (e.g. 'fallback: a rate limit on fable moves plan to opus xhigh' now expects max); `boot` installs a default successful `model.complete` answer. +11. No edit outside FILE SCOPE; the user's uncommitted `mods/model-router/routing.json` and `settings.json` changes are left untouched. + +## FILE SCOPE +mods/model-router/hooks/register.ts +mods/model-router/hooks/register.test.ts diff --git a/.claude/tasks/plans/2026-10-14-router-probe-before-route-0005.md b/.claude/tasks/plans/2026-10-14-router-probe-before-route-0005.md new file mode 100644 index 0000000..636e74b --- /dev/null +++ b/.claude/tasks/plans/2026-10-14-router-probe-before-route-0005.md @@ -0,0 +1,170 @@ +# Bugfix plan — model-router: probe a move target before routing to it (Fable out of credits) + +Branch: bugfix/router-probe-before-route (off develop). Skill: /bugfix. +Revision r3 (r1: correctness FATAL(4), simplicity FATAL(3), robustness +CONCERNS(3) → r2 rebuild; r2 confirmation correctness CONCERNS(3), 0 BLOCKER +→ r3 folds its findings; live engine checks). + +## Bug +Fable 5.1 out of usage credits. Router still routes best-tier phases (plan, +reflect, orchestrate, escalate) to `claude-fable-5-1`. Next request dies: +`` message "You're out of usage credits. Run /usage-credits to keep +using Fable 5.1 or /model to switch models.", `apiError: +model_requires_usage_credits`, `error: rate_limit`, HTTP 429. Turn lost. + +## Root cause +Availability learned only reactively. Breaker (`markDown`, register.ts +~l.1134) fed solely by `classic.StopFailure` (UNAVAILABLE kinds) and +`PostModelSwitch auto`, i.e. AFTER a real user request died. Nothing checks a +move target before main is sent to it. Hold in-memory, backoff 15→30→60…: +each new session and each hold lapse costs one user turn on a 429. + +## Evidence +- Session e2ecb2c1 transcript: 21:31:29 `model_requires_usage_credits` → + "fable unavailable (rate_limit) for 15 min"; hold lapsed; `route(reflect)` + answered "model claude-fable-5-1 (upgrade)"; 22:04:34 same error → "for 30 + min". `errorDetails` = generic 429 body ("would exceed your account's rate + limit"), no credits marker → no cheap typed signal to hold longer. +- LIVE CHECK (scratch mod, `claude -p --plugin-dir`, 2026-10-14): + `$.model.complete({model:'claude-opus-5-5', prompt:'ok', maxTokens:1})` + RESOLVED in 1308 ms with an OBJECT `{ isAnswered: true, text, usage }` (not + the documented string; maxTokens 1 not honored: 43 output tokens). + Same call on `claude-fable-5-1` (no credits): NO settle after >190 s (still + pending; the client apparently retries the 429). → for the credits case the + probe's failure signal IS the timeout. Long run: fable never settled + within 900 s (killed by timeout, rc 124) → never settles in useful time. `isAnswered:false` never + observed → NOT used as a failure signal. + +## Constraints (in force) +- LRN-207: `rateLimits` = account windows, not per model → no pre-check + there. Breaker per episode, idempotent while held; `/clear` keeps it. +- BDR-115: explicit Agent params never rewritten (user kept it 2026-10-14); + explicit `Agent(model:"fable")` pins (client-handover) → TODO residual. +- BDR-116 amendments: `decideMain` = THE single main decision, step and every + text agree; caps; agents move UP within tier only (`inTier`). +- User 2026-10-14: stand-in effort ONE NOTCH UP; no cross-session persistence. + +## Fix plan + +### 1. Probe the decided move target, once — `hooks/register.ts` +- Decide FIRST, probe after (closes r1 BLOCKERs: no loop, no explicit-alias + spin, cur never probed, no probe for a refused move). In `decideFor`: + `call = decideMain(...)`; if `call.moved` AND `id = canonical(call.model)` + is a table id AND not in `st.verified` AND NOT `st.down.has(id)` (a hold + already answers it: an explicit route onto a held id is sent as today, + never re-probed each step) → `ok = await probeModel($, st, id)`. The BARE + canonical id is probed (`[1m]` acceptance by complete() unverified; a + separately gated `[1m]` stays the reactive breaker's job). + - ok → `verified.add(canonical id)` only if `!st.down.has(id)` at + resolution (a StopFailure during flight wins). + - not ok → `markDown($, st, id, 'probe', now)`, then re-run `wantedModel` + + `decideMain` ONCE, no further probe (a double outage stays the reactive + breaker's job; an explicit alias/id route re-decides to the same id, as + "explicit, never skipped" already means). + - Covers every move kind decideMain makes (upgrade, downgrade, leaveDown + fallback, windowReturn) and every caller of decideFor: the step + (`mainPlan`), the route tool's answer (`snapshot`/`mainAnswer`, read by + the model gate), `/route show`, first-use dialog text, `toastChange`. +- `probeModel`: `$.model.complete({ model, prompt: 'ok', maxTokens: 16 })` + raced against `PROBE_MS` (8000, next to CLASSIFY_MS) through ONE extracted + helper `raceTimeout($, promise, ms)` shared with `raceClassify` (classify + keeps its abort signal; the probe passes NO hook signal: a user Esc must not + mark a model down). Result: + - resolved, whatever the shape (documented string or observed object) → ok; + - TIMEOUT → not ok (the live credits case); + - rejection → not ok (no typed kinds documented for `complete`). + - Esc during the probe: the probe completes (no signal), the mark stands, + but `mainPlan` skips its turn-state writes (`turnModel`, `noteServed`, + `promptEstimate`, `lastPlan`) when the dispatch signal has aborted. +- State: `verified: Set` in State, kept by `resetSession` (account- + wide like `down`). `markDown` deletes the id from `verified`. No `probing` + map, `clearBreaker` leaves `verified` alone (a user `/model` target becomes + cur, never probed). +- `markDown` logs through `tryLog` (it now runs on the step path; a throwing + `ui.log` must not unroute the step). +- A lapsed hold leaves the id unverified → the next move to it probes again + instead of spending a user turn. +- Config key `probe` (bool, default true) in DEFAULT_CONFIG + override + validation like `classify`, and `/route show` lists it: lets a machine turn + the probe off without `/route off`. [gated: user approval at the gate] + +### 2. Stand-in effort, one notch up — `hooks/register.ts` +- One pure helper computes `standIn` and decideFor returns it on the Verdict + (step and texts read the same value). COUNTERFACTUAL rule: the bump tracks + a real stand-in, not the head's availability. True when ALL hold: + (a) the main model slot's route has a `tier` and the head alias's id is + held down; (b) `canonical(call.model)` equals the tier's `resolveName` id + and ranks BELOW the head (fallback rank); (c) with the head treated as + available, the router WOULD have run it: the classifier ceiling of + `wantedModel` does not refuse the head, and `decideMain(st, cur, headId, + read)` lands on the head (upgrade on, cap not blocking, window fits). + Today: best with fable down → opus, only where fable would have run. +- `mainPlan`: effort = `floored(standIn && !explicit ? notchUp(base) : + base, floor)`, notchUp on LEVELS, max stays max. Precedence of + `mainEffort` unchanged. +- Only the phase's own effort is bumped: bump iff the slot that SUPPLIED the + effort (per `mainEffort`'s `by`: sticky→userMain, turn→turnMain or + runMain) is the same slot whose route names the tier, that slot carries no + explicit effort, and the base is a Level (never an engine number). A + floor-supplied base (`by: 'floor'`) is user-explicit → never bumped. Flag + `explicitEffort` on `Routed`, set when an effort was given: `parseRoute` + (`/route`), and the route tool through `Picked` → `applyRoute`. Case to + keep: `/route effort=low` sticky + plan turn route with fable down → low. +- Main only: agents move up within tier, never get a below-head stand-in. +- Texts: `effortWord` and `mainRoutedText` report the bumped level and why, + e.g. "effort xhigh (opus stands in for fable)"; the bump is recorded on + `Step.why` so the verbose transition line names "stand-in" as the effort + cause. `phasesText` unchanged. + +### 3. Tests — `hooks/register.test.ts` +- `boot`: default bottom `on('model.complete', …)` answering success so the + 472 existing tests keep every model available; helpers: `probeHangs(on, id)` + (never settles; advance the mock clock past PROBE_MS), `probeRejects(on, + id)`, a call counter. +- New tests (boot on OPUS, so a best-tier route is a move): + 1. plan route, fable probe times out → the step lands on opus/max, exactly + ONE model.complete call (on fable), no StopFailure, `show` lists fable + down (probe). + 2. route tool answer with the fable probe failing names opus, never fable. + 3. probe success cached: move to fable, end turn, re-route next turn from + opus → no second call. + 4. hold lapse (clock +15 min) → re-probe; still failing → fable held 30 min, + main on opus; no step sent on fable. + 5. bump: reflect stand-in → xhigh; orchestrate → high; `route(phase=reflect, + effort=high)` → high (explicit); fable available → no bump; classifier + reflect on an opus session with fable down → no bump (ceiling); upgrade + off → no bump; `/route effort=low` sticky + plan route, fable down → low; + an ultrathink floor → max, not bumped past. + 6. a probe rejection → marked down like a timeout; a resolved object or + string answer → ok. + 7. a StopFailure on fable while its probe is in flight → probe success does + NOT re-verify it. + 8. `/route fable` (explicit) with the probe failing → ONE probe in total + across `/route` itself, two consecutive steps and a `/route show`. + 11. Esc during a probe: fable marked down, no stale `turnModel` next turn. + 9. no probe when the move is refused (classifier guess above the session + model; upgrade off) and none for cur. + 10. `probe: false` in the override → no model.complete call. [if gated in] +- Update the existing tests whose expectations change ('fallback: a rate + limit on fable moves plan to opus xhigh' → opus max under the bump). +- Suite: `claude plugin test mods/model-router` (472 pass baseline) and + `make test suite=lib/tests/mods.test.sh`. + +## Out of scope / residuals (TODO) +- Explicit `Agent(model:"fable")` pins (client-handover) still fail without + credits (BDR-115 kept). +- Spawn path not probed: no agent row maps to best; `inTier` only reaches + fable when opus is also down (double outage → reactive breaker). +- Session model itself out of credits (settings `model: fable`): cur never + probed, first turn fails reactively. +- No cross-session persistence: with fable out of credits each new session + pays one PROBE_MS stall (≤8 s) at its first best-tier move. +- A timed-out probe's request keeps retrying in the engine (no cancel param). +- `[1m]` gating: the bare id is probed; a separately gated `[1m]` would pass + the probe and fail reactively, as today. + +## Risk +Medium. ≤8 s stall once per model per session on a failing target; ~1.3 s on +a healthy one. A network blip marks the move target down 15 min (main stays +on its current model: errs toward a working one). Probe spends a tiny +completion when the target IS available.