From 2cccc43d374faf0db1159316979f2ee579f25fa5 Mon Sep 17 00:00:00 2001 From: bchanot Date: Thu, 15 Oct 2026 00:07:10 +0200 Subject: [PATCH] fix(model-router): probe a move target before routing main to it The breaker learned availability only from a failed request: with Fable out of usage credits, every new session and every hold lapse routed main to Fable and lost the turn to a 429 (model_requires_usage_credits, typed rate_limit). decideFor now probes the model a decision moves main to, once per session, with a 1-call $.model.complete raced against 8 s: a timeout or rejection marks it down (reason "probe") and re-decides once. The current model and held ids are never probed; `probe: false` in ~/.claude/model-router.json turns it off. When a model ranked below its tier head stands in where the head would have run, the phase's own effort goes one notch up (reflect xhigh, plan max on Opus). --- mods/model-router/hooks/register.test.ts | 302 ++++++++++++++++++++++- mods/model-router/hooks/register.ts | 276 +++++++++++++++++---- 2 files changed, 532 insertions(+), 46 deletions(-) diff --git a/mods/model-router/hooks/register.test.ts b/mods/model-router/hooks/register.test.ts index 288b664..e6447ad 100644 --- a/mods/model-router/hooks/register.test.ts +++ b/mods/model-router/hooks/register.test.ts @@ -25,6 +25,10 @@ const TO_APPLY = ['Change', 'sonnet (work)', 'low'] const TO_JUDGE = ['Change', 'opus (big)'] const SCOPE_ASK = 3 // index of the scope question after a Change chain +// What the engine answers a completion with: an object, not the documented +// string (observed live); the router accepts any resolved shape. +const PROBE_OK = { isAnswered: true, text: 'ok' } as unknown as string + /** What the bottom `fs`, `env`, `session.repo` and dialog hooks serve. */ type World = { files: Map // 'routing' (routing.json) or a full path @@ -40,6 +44,8 @@ type World = { repo: { root: string; remote: string | null } | null | 'fail' env: string | undefined // HOME as env.get answers it (undefined = unset) kinds: Map // per-path stat kind override + complete: (model: string) => unknown // what a completion resolves to + completions: string[] // every model a completion was asked of } type Fixture = { @@ -115,6 +121,8 @@ function newWorld(fx: Fixture = {}): World { repo: fx.repo === undefined ? { root: ROOT, remote: REMOTE } : fx.repo, env: fx.noHome ? undefined : HOME, kinds: new Map(), + complete: () => PROBE_OK, + completions: [], } } @@ -228,6 +236,11 @@ async function boot( on('classic.PostModelSwitch', () => ({})) on('skill.prompt', ($, e) => ({ text: e.text })) on('agent.offer', () => ({ isOffered: true })) + // the probe of a move target: every model answers unless a test says not + on('model.complete', async (_$, e) => { + world.completions.push(e.model) + return { value: (await world.complete(e.model)) as string } + }) installWorld(on, world) if (tokens !== null) usageOf(on, tokens) on('session.start', ($, e) => ({ cwd: e.cwd })) @@ -784,13 +797,14 @@ test('cap: a downgrade under the cold-read cap moves main', async ( expect(sent.effort).toBeUndefined() }) -test('fallback: a rate limit on fable moves plan to opus xhigh', async ( +test('fallback: a rate limit on fable moves plan to opus max', async ( $, on) => { const rig = await bootRig($, on) await planRoute($) expect((await stepOn($, rig, FABLE)).model).toBe(FABLE) await failWith($, 'rate_limit') - expect(await stepOn($, rig, FABLE)).toEqual({ model: OPUS, effort: 'xhigh' }) + // opus stands in for fable: plan's xhigh goes one notch up + expect(await stepOn($, rig, FABLE)).toEqual({ model: OPUS, effort: 'max' }) expect(await route($, 'show')).toContain('claude-fable-5-1 for 15 min') await route($, 'reload') expect((await stepOn($, rig, FABLE)).model).toBe(FABLE) @@ -5489,3 +5503,287 @@ test('ttl: show says 60 min by default and 5 min after a 5m event', async ( await switchTo($, FABLE, FABLE, 'command', '5m') expect(await route($, 'show')).toContain('ttl 5 min)') }) + +// ---- probe: a move target answers before main is sent to it -------------- + +const PROBE_MS = 8000 +const FABLE_PLAN = 'plan the billing migration' + +/** A world whose completions resolve as `complete(model)` says. */ +function probeWorld( + complete: (model: string) => unknown, + fx: Fixture = {}, +): World { + return { ...newWorld(fx), complete } +} + +const hangsOn = (id: string) => (model: string): unknown => + model === id ? never() : PROBE_OK +const rejectsOn = (id: string) => (model: string): unknown => + model === id ? Promise.reject(new Error('credits')) : PROBE_OK + +/** Lets a probe that never answers hit its bound while `work` waits on it. */ +async function pastProbe(rig: Rig, work: Promise): Promise { + await rig.clock.settle() + await rig.clock.advance(PROBE_MS) + return work +} + +/** The `down:` line of `show`. */ +const downLine = (text: string): string => + text.split('\n').find(l => l.startsWith('down:')) ?? '' + +test('probe: a fable that never answers lands the plan step on opus', async ( + $, on) => { + const world = probeWorld(hangsOn(FABLE)) + const rig = await bootRig($, on, OPUS, 1000, world) + await typed($, FABLE_PLAN) + await pastProbe(rig, runStep($, { ...highStep(), model: OPUS })) + expect(rig.seen).toEqual([{ model: OPUS, effort: 'max' }]) + expect(world.completions).toEqual([FABLE]) + expect(downLine(await route($, 'show'))).toContain( + `${FABLE} for 15 min (probe)`) +}) + +test('probe: the route tool answers an opus id, never fable', async ( + $, on) => { + const rig = await bootRig($, on, OPUS, 1000, probeWorld(hangsOn(FABLE))) + const out = await pastProbe(rig, $.tool.call({ + tool: ROUTE_TOOL, + phase: 'reflect', + })) + const text = JSON.stringify(out) + expect(text).toContain(OPUS) + expect(text).not.toContain(FABLE) +}) + +test('probe: an answered probe is cached across turns', async ($, on) => { + const world = newWorld() + const rig = await bootRig($, on, OPUS, 1000, world) + await planRoute($) + expect((await stepOn($, rig, OPUS)).model).toBe(FABLE) + await endTurn($) + await planRoute($) + expect((await stepOn($, rig, OPUS)).model).toBe(FABLE) + expect(world.completions).toEqual([FABLE]) +}) + +test('probe: a lapsed hold probes again and holds longer', async ($, on) => { + const world = probeWorld(hangsOn(FABLE)) + const rig = await bootRig($, on, OPUS, 1000, world) + for (const wait of [15, 30]) { + await typed($, FABLE_PLAN) + await pastProbe(rig, runStep($, { ...highStep(), model: OPUS })) + expect(downLine(await route($, 'show'))).toContain(`for ${wait} min`) + await endTurn($) + await rig.clock.advance(wait * 60000) + } + expect(world.completions).toEqual([FABLE, FABLE]) + expect(rig.seen.map(s => s.model)).toEqual([OPUS, OPUS]) +}) + +test('probe: a rejection marks the target down like a timeout', async ( + $, on) => { + const world = probeWorld(rejectsOn(FABLE)) + const rig = await bootRig($, on, OPUS, 1000, world) + await planRoute($) + expect((await stepOn($, rig, OPUS)).model).toBe(OPUS) + expect(world.completions).toEqual([FABLE]) + expect(downLine(await route($, 'show'))).toContain('(probe)') +}) + +for (const [shape, answer] of [ + ['a string', 'ok'], ['an object', PROBE_OK], ['nothing', undefined], +] as const) { + test(`probe: ${shape} as the answer lets the move go`, async ($, on) => { + const world = probeWorld(() => answer) + const rig = await bootRig($, on, OPUS, 1000, world) + await planRoute($) + expect((await stepOn($, rig, OPUS)).model).toBe(FABLE) + expect(world.completions).toEqual([FABLE]) + }) +} + +test('probe: a failure during the flight keeps the target unverified', async ( + $, on) => { + let open: () => void = () => undefined + const gate = new Promise(resolve => { open = resolve }) + const world = probeWorld(async () => { await gate; return PROBE_OK }) + const rig = await bootRig($, on, OPUS, 1000, world) + await typed($, FABLE_PLAN) + const step = runStep($, { ...highStep(), model: OPUS }) + await rig.clock.settle() + await switchTo($, FABLE, OPUS, 'auto') + open() + await step + expect(rig.seen).toEqual([{ model: OPUS, effort: 'max' }]) + await endTurn($) + await rig.clock.advance(15 * 60000) + await typed($, FABLE_PLAN) + await runStep($, { ...highStep(), model: OPUS }) + expect(world.completions).toEqual([FABLE, FABLE]) +}) + +test('probe: an explicit route onto a held model probes once in all', async ( + $, on) => { + const world = probeWorld(hangsOn(FABLE)) + const rig = await bootRig($, on, OPUS, 1000, world) + await pastProbe(rig, route($, 'fable')) + await runStep($, { ...highStep(), model: OPUS }) + await runStep($, { ...highStep(), model: OPUS }) + await route($, 'show') + // fable is asked once; the second step leaves the held fable for opus, + // a move like any other, so opus is tried the same way + expect(world.completions).toEqual([FABLE, OPUS]) + expect(rig.seen.map(s => s.model)).toEqual([FABLE, OPUS]) +}) + +test('probe: no probe when the classifier guess is above the session', async ( + $, on) => { + const world = probeWorld(hangsOn(FABLE)) + const c = await bootCls($, on, { + model: SONNET, reply: () => 'design', world, + }) + await typed($, PLAIN) + await runStep($, { ...highStep(), model: SONNET }) + expect(world.completions).toEqual([]) + expect(c.seen[c.seen.length - 1]?.model).toBe(SONNET) +}) + +test('probe: no probe when the upgrade switch refuses the move', async ( + $, on) => { + const world = probeWorld(hangsOn(FABLE), { home: { mainUpgrade: false } }) + const rig = await bootRig($, on, OPUS, 1000, world) + await planRoute($) + expect((await stepOn($, rig, OPUS)).model).toBe(OPUS) + expect(world.completions).toEqual([]) +}) + +test('probe: the current model is never probed', async ($, on) => { + const world = probeWorld(hangsOn(FABLE)) + const rig = await bootRig($, on, FABLE, 1000, world) + await planRoute($) + expect((await stepOn($, rig, FABLE)).model).toBe(FABLE) + expect(world.completions).toEqual([]) +}) + +test('probe: false in the override makes no completion at all', async ( + $, on) => { + const world = probeWorld(hangsOn(FABLE), { home: { probe: false } }) + const rig = await bootRig($, on, OPUS, 1000, world) + await planRoute($) + expect((await stepOn($, rig, OPUS)).model).toBe(FABLE) + expect(world.completions).toEqual([]) + expect(await route($, 'show')).toContain('probe: off') +}) + +test('probe: a non-boolean setting is ignored, logged, probe stays on', async ( + $, on) => { + const logs = recordLogs(on) + const world = newWorld({ home: { probe: 'no' } }) + const rig = await bootRig($, on, OPUS, 1000, world) + await planRoute($) + await stepOn($, rig, OPUS) + expect(world.completions).toEqual([FABLE]) + expect(logs.some(l => l.includes('"probe" is not a boolean'))).toBe(true) + expect(await route($, 'show')).toContain('probe: on') +}) + +test('probe: /clear keeps the verified models', async ($, on) => { + on('session.end', ($, e) => ({ sessionId: e.sessionId })) + const world = newWorld() + const rig = await bootRig($, on, OPUS, 1000, world) + await planRoute($) + await stepOn($, rig, OPUS) + await clearSession($) + await planRoute($) + await stepOn($, rig, OPUS) + expect(world.completions).toEqual([FABLE]) +}) + +// ---- stand-in: a lower model runs the tier one notch up ------------------ + +/** Main on fable fails: the hold stands, the next move is a stand-in. */ +async function fableHeld($: Engine, rig: Rig): Promise { + await stepOn($, rig, FABLE) + await failWith($, 'rate_limit') +} + +test('stand-in: reflect on opus runs xhigh, and the answer says so', async ( + $, on) => { + const rig = await bootRig($, on) + await fableHeld($, rig) + const out = await $.tool.call({ tool: ROUTE_TOOL, phase: 'reflect' }) + expect(JSON.stringify(out)).toContain( + 'effort xhigh (opus stands in for fable)') + expect(await stepOn($, rig, FABLE)).toEqual({ model: OPUS, effort: 'xhigh' }) +}) + +test('stand-in: orchestrate goes medium to high, escalate stays max', async ( + $, on) => { + const rig = await bootRig($, on) + await fableHeld($, rig) + await $.tool.call({ tool: ROUTE_TOOL, phase: 'orchestrate' }) + expect((await stepOn($, rig, FABLE)).effort).toBe('high') + await $.tool.call({ tool: ROUTE_TOOL, phase: 'escalate' }) + expect((await stepOn($, rig, FABLE)).effort).toBe('max') +}) + +test('stand-in: an explicit effort is never bumped', async ($, on) => { + const rig = await bootRig($, on) + await fableHeld($, rig) + await $.tool.call({ tool: ROUTE_TOOL, phase: 'reflect', effort: 'high' }) + expect(await stepOn($, rig, FABLE)).toEqual({ model: OPUS, effort: 'high' }) +}) + +test('stand-in: fable available, no bump', async ($, on) => { + const rig = await bootRig($, on, OPUS) + await $.tool.call({ tool: ROUTE_TOOL, phase: 'reflect' }) + expect(await stepOn($, rig, OPUS)).toEqual({ model: FABLE, effort: 'high' }) +}) + +test('stand-in: a classifier guess above the session is no stand-in', async ( + $, on) => { + const c = await bootCls($, on, { model: OPUS, reply: () => 'analysis' }) + await switchTo($, FABLE, OPUS, 'auto') + await typed($, PLAIN) + await runStep($, { ...highStep(), model: OPUS }) + expect(c.seen[c.seen.length - 1]).toEqual({ model: OPUS, effort: 'high' }) +}) + +test('stand-in: with the upgrade switch off, no bump', async ($, on) => { + const world = newWorld({ home: { mainUpgrade: false } }) + const rig = await bootRig($, on, OPUS, 1000, world) + await switchTo($, FABLE, OPUS, 'auto') + await $.tool.call({ tool: ROUTE_TOOL, phase: 'reflect' }) + expect(await stepOn($, rig, OPUS)).toEqual({ model: OPUS, effort: 'high' }) +}) + +test('stand-in: a sticky effort from another slot is not bumped', async ( + $, on) => { + const rig = await bootRig($, on) + await fableHeld($, rig) + await route($, 'effort=low') + await planRoute($) + expect(await stepOn($, rig, FABLE)).toEqual({ model: OPUS, effort: 'low' }) +}) + +test('stand-in: an ultrathink floor ends at max, not past it', async ( + $, on) => { + const rig = await bootRig($, on) + await fableHeld($, rig) + await ultrathink($) + await $.tool.call({ tool: ROUTE_TOOL, phase: 'reflect' }) + expect(await stepOn($, rig, FABLE)).toEqual({ model: OPUS, effort: 'max' }) +}) + +test('stand-in: the verbose line names it as a cause', async ($, on) => { + const logs = recordLogs(on) + const rig = await bootRig($, on) + await route($, 'verbose on') + await fableHeld($, rig) + await $.tool.call({ tool: ROUTE_TOOL, phase: 'reflect' }) + await stepOn($, rig, FABLE) + expect(logs.some(l => l.startsWith('main:') && l.includes('stand-in'))) + .toBe(true) +}) diff --git a/mods/model-router/hooks/register.ts b/mods/model-router/hooks/register.ts index b2a72f8..68fa9f9 100644 --- a/mods/model-router/hooks/register.ts +++ b/mods/model-router/hooks/register.ts @@ -34,6 +34,7 @@ type Config = { mainModelSwitch: boolean // gates DOWNGRADES only verbose: boolean classify: boolean // a small-model guess of the typed prompt's phase + probe: boolean // a move target is tried once before main is sent to it spinner: boolean enabled: boolean // false: every hook passes through (per machine) } @@ -45,6 +46,7 @@ type Routed = { route: Route source: Source label?: string // the skill a skill route came from, for the logs + explicitEffort?: boolean // the effort was asked for, not the phase's own } type Served = { at: number; tokens: number } // ms, context tokens type Hold = { until: number; reason: string } // until: ms, Infinity = reload @@ -92,6 +94,7 @@ type State = { sessionModel: string // the main loop's model as /model shows it, '' unknown down: Map // canonical id -> unavailable until (breaker) strikes: Map // canonical id -> episodes, drives the backoff + verified: Set // canonical ids a probe saw answer (account-wide) agentModels: Map // agentId -> exact id sent at spawn pushed: Pushed | null // derived orchestrate in force (background dispatch) logged: Set // session-once log lines @@ -132,7 +135,7 @@ type RouteInput = { clear?: unknown agentId?: string } -type Picked = { phase: string; route: Route } +type Picked = { phase: string; route: Route; explicitEffort?: boolean } type Effort = StepIn['effort'] type Toggle = 'switch' | 'verbose' | 'classify' type EffortBy = 'floor' | 'sticky' | 'turn' | 'engine' @@ -231,6 +234,8 @@ const LABEL_PHASE: Readonly> = { } const CLASSIFY_MIN = 12 // code points of a typed prompt worth a call const CLASSIFY_MS = 3000 // the call is abandoned after this +const PROBE_MS = 8000 // a move target that has not answered by then is down +const PROBE_TOKENS = 16 // the probe's reply cap const CLASSIFY_MAX = 2000 // code points sent whole; longer: head + tail const CLASSIFY_HEAD = 500 const CLASSIFY_TAIL = 1500 @@ -245,7 +250,7 @@ const CACHE_TTLS: Readonly> = { '1h': 60 * 60000, } const ESTIMATE_DIVISOR = 3 // code points per token, conservative -const TIMEOUT: unique symbol = Symbol('classify timeout') +const TIMEOUT: unique symbol = Symbol('timeout') const DEFAULT_CONFIG: Config = { models: { @@ -304,6 +309,7 @@ const DEFAULT_CONFIG: Config = { mainModelSwitch: true, verbose: false, classify: true, + probe: true, spinner: true, enabled: true, } @@ -635,6 +641,9 @@ function mergeConfig( classify: pickSwitch( 'model-router: config "classify" is not a boolean; ignored', user.classify, base.classify, log), + probe: pickSwitch( + 'model-router: config "probe" is not a boolean; ignored', + user.probe, base.probe, log), spinner: pickBool(user.spinner, base.spinner), enabled: pickEnabled(user.enabled, base.enabled, log), } @@ -968,6 +977,7 @@ function newState(cfg: Config, source: string): State { sessionModel: '', down: new Map(), strikes: new Map(), + verified: new Set(), agentModels: new Map(), pushed: null, logged: new Set(), @@ -995,16 +1005,18 @@ function newState(cfg: Config, source: string): State { /** * /clear rebuilds the state but keeps what is account- or process-wide: - * the breaker, the classifier's pause and effort baseline (a model's quota - * and the user's own /effort outlive the conversation), the model, the - * agent definitions' sources (the listing is not offered again), the config - * with its session toggles and the dialog in flight. The layers are read - * again at the next prompt; the config is swapped only if that read works. + * the breaker and the models a probe verified, the classifier's pause and + * effort baseline (a model's quota and the user's own /effort outlive the + * conversation), the model, the agent definitions' sources (the listing is + * not offered again), the config with its session toggles and the dialog in + * flight. The layers are read again at the next prompt; the config is + * swapped only if that read works. */ function resetSession(st: State): void { const kept = { down: st.down, strikes: st.strikes, + verified: st.verified, agentModels: st.agentModels, sessionModel: st.sessionModel, offers: st.offers, @@ -1146,7 +1158,8 @@ function markDown( st.strikes.set(id, strikes) const until = now + holdMinutes(st.cfg, strikes, reason) * 60000 st.down.set(id, { until, reason }) - $.ui.log(`model-router: ${id} unavailable (${reason}) ${ + st.verified.delete(id) + tryLog($, `model-router: ${id} unavailable (${reason}) ${ holdWord(until, now)}; routing falls back`) } @@ -1447,7 +1460,11 @@ async function addEstimate( const pointsOf = (texts: readonly string[]): number => texts.reduce((n, t) => n + Array.from(t).length, 0) -type Verdict = { call: Call; wanted: string | undefined } +type Verdict = { + call: Call + wanted: string | undefined + standIn: boolean // a lower model runs where the tier's held head would +} /** * The model main is asked to run on from `cur`. A classifier's guess never @@ -1477,9 +1494,123 @@ async function readingFor( return { tokens: withEstimate(st, raw), now: await $.clock.now() } } +/** + * The race of a call against the clock: its value, or TIMEOUT. With a + * `signal` the wait rejects when the dispatch aborts. + */ +function raceTimeout( + $: Api, + call: Promise, + ms: number, + signal?: AbortSignal, +): Promise { + return Promise.race([ + call, + $.clock.sleep(ms, signal === undefined ? {} : { signal }) + .then((): typeof TIMEOUT => TIMEOUT), + ]) +} + +/** True when `id` answered a tiny completion in time; no hook signal. */ +async function probeAnswers($: Api, id: string): Promise { + try { + const ask = { model: id, prompt: 'ok', maxTokens: PROBE_TOKENS } + return (await raceTimeout($, $.model.complete(ask), PROBE_MS)) !== TIMEOUT + } catch { + return false + } +} + +/** + * Tries a move target before main is sent to it. An answer verifies it, + * unless a failure landed meanwhile (the hold wins); a timeout or a + * rejection marks it down. True when the move may go ahead. + */ +async function probeModel($: Api, st: State, id: string): Promise { + if (await probeAnswers($, id)) { + if (!st.down.has(id)) st.verified.add(id) + } else { + markDown($, st, id, 'probe', await $.clock.now()) + } + return !st.down.has(id) +} + +/** The table id a move lands on that still wants a probe, else undefined. */ +function unprobed(st: State, call: Call): string | undefined { + if (!call.moved || !st.cfg.probe || st.off) return undefined + const id = canonical(st.cfg, call.model) + const fresh = !st.verified.has(id) && !st.down.has(id) + return aliasOf(st.cfg, id) !== undefined && fresh ? id : undefined +} + +type Decided = { call: Call; wanted: string | undefined } + +/** The decision, probing a move target once; a failed probe: decide again. */ +async function decideProbed( + $: Api, + st: State, + cur: string, + read: Reading, +): Promise { + const wanted = wantedModel(st, cur) + const call = decideMain(st, cur, wanted, read) + const id = unprobed(st, call) + if (id === undefined || await probeModel($, st, id)) return { call, wanted } + const after = wantedModel(st, cur) + return { call: decideMain(st, cur, after, read), wanted: after } +} + +/** The held head of the tier the main model slot names, or undefined. */ +function heldHead(st: State): string | undefined { + const tier = mainModelSlot(st)?.route.tier + const list = tier !== undefined && hasKey(st.cfg.tiers, tier) + ? st.cfg.tiers[tier] + : undefined + const alias = list?.[0] + const id = alias === undefined ? undefined : st.cfg.models[alias] + return id !== undefined && st.down.has(id) ? id : undefined +} + +/** + * With `head` available, the router would run it: the classifier ceiling + * does not refuse it and the decision lands on it (switch, caps, window). + */ +function wouldRunHead( + st: State, + cur: string, + head: string, + read: Reading, +): boolean { + const capped = mainModelSlot(st)?.source === 'classifier' && + ranksAbove(st, head, canonical(st.cfg, st.sessionModel)) + if (capped) return false + const held = new Map([...st.down].filter(([id]) => id !== head)) + const open: State = { ...st, down: held } + return canonical(st.cfg, decideMain(open, cur, head, read).model) === head +} + +/** + * A lower model runs where the tier's head, held down, would have: the + * decision is that tier's resolution, ranked below the head, and the head + * would have been run. Pure; the step and every text read this one value. + */ +function isStandIn( + st: State, + cur: string, + d: Decided, + read: Reading, +): boolean { + const head = heldHead(st) + if (st.off || head === undefined || d.wanted === undefined) return false + const id = canonical(st.cfg, d.call.model) + return id === d.wanted && ranksAbove(st, head, id) && + wouldRunHead(st, cur, head, read) +} + /** * What the main loop would run on from `cur`: wanted model + decision. The - * step passes the `known` reading it took; a text reads for itself. + * step passes the `known` reading it took; a text reads for itself. A move + * target is probed here, after the decision (see `decideProbed`). */ async function decideFor( $: Api, @@ -1487,9 +1618,9 @@ async function decideFor( cur: string, known?: Reading, ): Promise { - const wanted = wantedModel(st, cur) - const read = known ?? await readingFor($, st, wanted) - return { call: decideMain(st, cur, wanted, read), wanted } + const read = known ?? await readingFor($, st, wantedModel(st, cur)) + const decided = await decideProbed($, st, cur, read) + return { ...decided, standIn: isStandIn(st, cur, decided, read) } } function logCall($: Api, st: State, call: Call): void { @@ -1541,6 +1672,34 @@ function mainEffort(st: State, engine: Effort): Decision { return { effort, by: sticky === undefined ? 'turn' : 'sticky' } } +/** The route slot that supplied a sticky or turn effort, else null. */ +function effortSlot(st: State, by: EffortBy): Routed | null { + if (by === 'sticky') return st.userMain + if (by !== 'turn') return null + return st.turnMain?.route.effort === undefined ? st.runMain : st.turnMain +} + +/** One notch up on LEVELS; `max` stays. */ +const notchUp = (level: Level): Level => + LEVELS[Math.min(rank(level) + 1, LEVELS.length - 1)] ?? level + +type Lifted = Decision & { standIn: boolean } + +/** + * A stand-in model runs the tier one notch up: only the effort the slot + * that names the tier supplied, never an explicit one, a floor's or an + * engine number. `standIn` is true when the level really moved. + */ +function standInEffort(st: State, d: Decision, v: Verdict): Lifted { + const level = d.effort + const slot = effortSlot(st, d.by) + const own = slot !== null && slot === mainModelSlot(st) && + slot.explicitEffort !== true + if (!v.standIn || !own || !isLevel(level)) return { ...d, standIn: false } + const lifted = notchUp(level) + return { effort: lifted, by: d.by, standIn: lifted !== level } +} + const floorWord = (f: Routed): string => `user floor ${f.route.effort ?? '-'} (prompt rule ${f.phase})` @@ -1636,10 +1795,19 @@ function tierWord(st: State, word: string): string { return tier ? `${word} [tier ${name}]` : word } +/** "opus stands in for fable": the model that runs and the held head. */ +function standInWord(st: State, v: Verdict): string { + const head = aliasOf(st.cfg, heldHead(st) ?? '') ?? '?' + const alias = aliasOf(st.cfg, canonical(st.cfg, v.call.model)) ?? '?' + return `${alias} stands in for ${head}` +} + /** Haiku takes no effort: the main line says so when it will run there. */ function effortWord(st: State, v: Verdict): string { if (v.call.model.startsWith(HAIKU)) return '- (haiku takes none)' - return String(mainEffort(st, undefined).effort ?? '-') + const d = standInEffort(st, mainEffort(st, undefined), v) + const word = String(d.effort ?? '-') + return d.standIn ? `${word} (${standInWord(st, v)})` : word } function mainText(st: State, v: Verdict): string { @@ -1697,8 +1865,8 @@ async function show($: Api, st: State): Promise { mainText(st, s), `router: ${routerWord(st)} · switch: ${switchWord(st)} · upgrade: ${ flag(c.mainUpgrade)} · verbose: ${ - flag(c.verbose)} · classify: ${flag(c.classify)} · spinner: ${ - flag(c.spinner)}`, + flag(c.verbose)} · classify: ${flag(c.classify)} · probe: ${ + flag(c.probe)} · spinner: ${flag(c.spinner)}`, downText(st, s.now), `live loops: ${st.loops.size}`, phasesText(st), @@ -1756,7 +1924,9 @@ function parseRoute(cfg: Config, args: string): Routed | string { for (const word of words) { if (!applyToken(cfg, route, word)) return unknownText(cfg, word) } - return { phase: 'custom', route, source: 'user' } + const explicitEffort = route.effort !== undefined + return { phase: 'custom', route, source: 'user', + ...(explicitEffort ? { explicitEffort } : {}) } } async function toggle( @@ -2785,12 +2955,14 @@ function pickRoute(cfg: Config, phase: unknown, effort: unknown) { } const route: Route = { ...named, ...(isLevel(effort) ? { effort } : {}) } const name = typeof phase === 'string' ? phase : `effort-${String(effort)}` - return { phase: name, route } + const explicit = isLevel(effort) ? { explicitEffort: true } : {} + return { phase: name, route, ...explicit } } function applyRoute(st: State, agentId: string | undefined, p: Picked): void { if (agentId === undefined) { - st.turnMain = { phase: p.phase, route: p.route, source: 'model' } + st.turnMain = { phase: p.phase, route: p.route, source: 'model', + ...(p.explicitEffort ? { explicitEffort: true } : {}) } } else { writeLoop(loopOf(st, agentId), p.route, 'route tool') } @@ -2813,16 +2985,15 @@ function clearLoop(st: State, agentId: string | undefined): string { return 'route cleared for this agent' } -/** The id the next main step runs on, with the decision's reason. */ -async function mainAnswer($: Api, st: State): Promise { - const { call } = await snapshot($, st) - return `${idWord(st, call.model)} (${call.why})` -} - /** Main's answer: effort and model asked, the sticky/floor note appended. */ async function mainRoutedText($: Api, st: State, p: Picked) { - const base = `routed main to ${p.phase}: effort ${ - p.route.effort ?? 'unchanged'}, model ${await mainAnswer($, st)}` + const v = await snapshot($, st) + const d = standInEffort(st, mainEffort(st, undefined), v) + const effort = d.standIn + ? `${String(d.effort)} (${standInWord(st, v)})` + : p.route.effort ?? 'unchanged' + const base = `routed main to ${p.phase}: effort ${effort}, model ${ + idWord(st, v.call.model)} (${v.call.why})` const note = mainNote(st, p.route.effort) return note ? `${base}; but ${note}` : base } @@ -3199,7 +3370,9 @@ function mainMove( const [was, now] = [planWord(st.cfg, from), planWord(st.cfg, step.plan)] if (step.why === undefined || was === now) return undefined const { phase, word } = effortCause(st, step.why.by) - const causes = [word, modelCause(st, step.why)].filter(c => c !== undefined) + const standIn = step.why.standIn ? 'stand-in' : undefined + const causes = [word, standIn, modelCause(st, step.why)] + .filter(c => c !== undefined) const lead = phase === undefined ? '' : `${clean(phase, NAME_CAP)}: ` return `main: ${was} → ${now} (${lead}${causes.join(', ')})` } @@ -3309,8 +3482,8 @@ function agentPlan(st: State, e: StepIn): Plan { } /** What a main plan was decided from: the log words are built from it. */ -type Why = { by: EffortBy; call: Call; cur: string } -type Step = { plan: Plan; why?: Why } +type Why = { by: EffortBy; call: Call; cur: string; standIn: boolean } +type Step = { plan: Plan; why?: Why; aborted?: boolean } /** The engine effort of this turn's first main step: the user's default. */ function noteEngineBase(st: State, e: StepIn): void { @@ -3355,27 +3528,43 @@ function capLogged(st: State, call: Call): Call { * the engine's own (verbatim: an engine fallback is respected). Every step * reads the context and the clock and records the model it ran on. */ -async function mainPlan($: Api, st: State, e: StepIn): Promise { +async function mainPlan( + $: Api, + st: State, + e: StepIn, + signal: AbortSignal, +): Promise { noteEngineBase(st, e) if (userMovedEffort(st)) st.turnMain = null - const { effort, by } = mainEffort(st, e.effort) + const base = mainEffort(st, e.effort) await prune($, st) const cur = st.turnModel ?? e.model const raw = await readTokens($) const now = await $.clock.now() - const { call } = await decideFor( - $, st, cur, { tokens: withEstimate(st, raw), now }) + const v = await decideFor($, st, cur, { tokens: withEstimate(st, raw), now }) + const { call } = v + const { effort, by, standIn } = standInEffort(st, base, v) logCall($, st, capLogged(st, call)) + const step = { plan: { model: call.model, effort }, + why: { by, call, cur, standIn } } + // Esc during a probe: the mark stands, but the abandoned step records + // nothing a later step would read as what main ran + if (signal.aborted) return { ...step, aborted: true } if (call.moved) st.turnModel = call.model // the record stays raw: an estimate in it would trip the compaction path noteServed(st, { model: call.model, effort }, { tokens: raw, now }) st.promptEstimate = null // consumed by the turn's first main step - return { plan: { model: call.model, effort }, why: { by, call, cur } } + return step } -async function planStep($: Api, st: State, e: StepIn): Promise { +async function planStep( + $: Api, + st: State, + e: StepIn, + signal: AbortSignal, +): Promise { const step: Step = e.agentId === undefined - ? await mainPlan($, st, e) + ? await mainPlan($, st, e, signal) : { plan: agentPlan(st, e) } // Haiku takes no effort: omit it rather than send a hook-set value. return step.plan.model.startsWith(HAIKU) @@ -3689,11 +3878,13 @@ function registerSpawn(on: On, st: State): void { function registerTurns(on: On, st: State): void { on('turn.step', async function* ($, e, next) { if (st.off) return yield* next(e) - const step = await planStep($, st, e) + const step = await planStep($, st, e, next.signal) const plan = step.plan const changed = plan.model !== e.model || plan.effort !== e.effort const prev = st.lastPlan // main's last plan, before noteMain replaces it - if (e.agentId === undefined) noteMain($, st, plan) + if (e.agentId === undefined && step.aborted !== true) { + noteMain($, st, plan) + } safely(st, $, 'turn.step log', () => logMove($, st, e, prev, step)) const result = yield* next(changed ? withPlan(e, plan) : e) safely(st, $, 'turn.step', () => @@ -3850,11 +4041,8 @@ const isAbort = (err: unknown): boolean => /** The label, or TIMEOUT; rejects like classify or when the dispatch aborts. */ function raceClassify($: Api, text: string, signal: AbortSignal) { - return Promise.race([ - $.model.classify(sampleOf(text), CLASSIFY_LABELS), - $.clock.sleep(CLASSIFY_MS, { signal }) - .then((): typeof TIMEOUT => TIMEOUT), - ]) + const call = $.model.classify(sampleOf(text), CLASSIFY_LABELS) + return raceTimeout($, call, CLASSIFY_MS, signal) } /** The label, or undefined: no answer, an interrupt or a counted failure. */