From c11cd3b53b4697da55e205ae173e1ca6b38351e7 Mon Sep 17 00:00:00 2001 From: bchanot Date: Fri, 16 Oct 2026 19:06:43 +0200 Subject: [PATCH] fix(model-router): probe the kept session model, not only a move target A session on Fable without credits kept Fable unprobed and lost its first turn, and each hold lapse, to the 429. The probe now covers the model the decision lands on, moved or kept. --- mods/model-router/hooks/register.test.ts | 49 ++++++++++++++++++++---- mods/model-router/hooks/register.ts | 21 +++++----- 2 files changed, 52 insertions(+), 18 deletions(-) diff --git a/mods/model-router/hooks/register.test.ts b/mods/model-router/hooks/register.test.ts index e6447ad..aef5dae 100644 --- a/mods/model-router/hooks/register.test.ts +++ b/mods/model-router/hooks/register.test.ts @@ -236,7 +236,7 @@ async function boot( on('classic.PostModelSwitch', () => ({})) on('skill.prompt', ($, e) => ({ text: e.text })) on('agent.offer', () => ({ isOffered: true })) - // the probe of a move target: every model answers unless a test says not + // the probe of the model main runs on: every model answers unless a test says not on('model.complete', async (_$, e) => { world.completions.push(e.model) return { value: (await world.complete(e.model)) as string } @@ -5504,7 +5504,7 @@ test('ttl: show says 60 min by default and 5 min after a 5m event', async ( expect(await route($, 'show')).toContain('ttl 5 min)') }) -// ---- probe: a move target answers before main is sent to it -------------- +// ---- probe: the model main runs on answers before main is sent ----------- const PROBE_MS = 8000 const FABLE_PLAN = 'plan the billing migration' @@ -5578,7 +5578,8 @@ test('probe: a lapsed hold probes again and holds longer', async ($, on) => { await endTurn($) await rig.clock.advance(wait * 60000) } - expect(world.completions).toEqual([FABLE, FABLE]) + // the fallback opus is verified by its own probe the first time only + expect(world.completions).toEqual([FABLE, OPUS, FABLE]) expect(rig.seen.map(s => s.model)).toEqual([OPUS, OPUS]) }) @@ -5588,7 +5589,7 @@ test('probe: a rejection marks the target down like a timeout', async ( const rig = await bootRig($, on, OPUS, 1000, world) await planRoute($) expect((await stepOn($, rig, OPUS)).model).toBe(OPUS) - expect(world.completions).toEqual([FABLE]) + expect(world.completions).toEqual([FABLE, OPUS]) expect(downLine(await route($, 'show'))).toContain('(probe)') }) @@ -5646,7 +5647,7 @@ test('probe: no probe when the classifier guess is above the session', async ( }) await typed($, PLAIN) await runStep($, { ...highStep(), model: SONNET }) - expect(world.completions).toEqual([]) + expect(world.completions).toEqual([SONNET]) expect(c.seen[c.seen.length - 1]?.model).toBe(SONNET) }) @@ -5656,15 +5657,47 @@ test('probe: no probe when the upgrade switch refuses the move', async ( const rig = await bootRig($, on, OPUS, 1000, world) await planRoute($) expect((await stepOn($, rig, OPUS)).model).toBe(OPUS) - expect(world.completions).toEqual([]) + expect(world.completions).toEqual([OPUS]) }) -test('probe: the current model is never probed', async ($, on) => { +test('probe: a session model that never answers leaves for opus on the first step', async ( + $, on) => { const world = probeWorld(hangsOn(FABLE)) const rig = await bootRig($, on, FABLE, 1000, world) + await typed($, FABLE_PLAN) + await pastProbe(rig, runStep($, { ...highStep(), model: FABLE })) + expect(rig.seen).toEqual([{ model: OPUS, effort: 'max' }]) + expect(world.completions).toEqual([FABLE]) + expect(downLine(await route($, 'show'))).toContain( + `${FABLE} for 15 min (probe)`) +}) + +test('probe: a healthy session model is probed once and kept', async ( + $, on) => { + const world = newWorld() + const rig = await bootRig($, on, FABLE, 1000, world) await planRoute($) expect((await stepOn($, rig, FABLE)).model).toBe(FABLE) - expect(world.completions).toEqual([]) + await endTurn($) + await planRoute($) + expect((await stepOn($, rig, FABLE)).model).toBe(FABLE) + expect(world.completions).toEqual([FABLE]) +}) + +test('probe: a lapsed hold on the session model probes it again', async ( + $, on) => { + const world = probeWorld(hangsOn(FABLE)) + const rig = await bootRig($, on, FABLE, 1000, world) + for (const wait of [15, 30]) { + await typed($, FABLE_PLAN) + await pastProbe(rig, runStep($, { ...highStep(), model: FABLE })) + expect(downLine(await route($, 'show'))).toContain(`for ${wait} min`) + await endTurn($) + await rig.clock.advance(wait * 60000) + } + // opus, the fallback, answers its own probe once + expect(world.completions).toEqual([FABLE, OPUS, FABLE]) + expect(rig.seen.map(s => s.model)).toEqual([OPUS, OPUS]) }) test('probe: false in the override makes no completion at all', async ( diff --git a/mods/model-router/hooks/register.ts b/mods/model-router/hooks/register.ts index 68fa9f9..3d2f321 100644 --- a/mods/model-router/hooks/register.ts +++ b/mods/model-router/hooks/register.ts @@ -34,7 +34,7 @@ type Config = { mainModelSwitch: boolean // gates DOWNGRADES only verbose: boolean classify: boolean // a small-model guess of the typed prompt's phase - probe: boolean // a move target is tried once before main is sent to it + probe: boolean // the model main would run on is tried once first spinner: boolean enabled: boolean // false: every hook passes through (per machine) } @@ -234,7 +234,7 @@ const LABEL_PHASE: Readonly> = { } const CLASSIFY_MIN = 12 // code points of a typed prompt worth a call const CLASSIFY_MS = 3000 // the call is abandoned after this -const PROBE_MS = 8000 // a move target that has not answered by then is down +const PROBE_MS = 8000 // a probed model that has not answered by then is down const PROBE_TOKENS = 16 // the probe's reply cap const CLASSIFY_MAX = 2000 // code points sent whole; longer: head + tail const CLASSIFY_HEAD = 500 @@ -1522,9 +1522,9 @@ async function probeAnswers($: Api, id: string): Promise { } /** - * Tries a move target before main is sent to it. An answer verifies it, - * unless a failure landed meanwhile (the hold wins); a timeout or a - * rejection marks it down. True when the move may go ahead. + * Tries the model main would run on before main is sent to it. An answer + * verifies it, unless a failure landed meanwhile (the hold wins); a timeout + * or a rejection marks it down. True when main may go ahead on it. */ async function probeModel($: Api, st: State, id: string): Promise { if (await probeAnswers($, id)) { @@ -1535,9 +1535,9 @@ async function probeModel($: Api, st: State, id: string): Promise { return !st.down.has(id) } -/** The table id a move lands on that still wants a probe, else undefined. */ +/** The table id the decision lands on (moved or kept) that wants a probe. */ function unprobed(st: State, call: Call): string | undefined { - if (!call.moved || !st.cfg.probe || st.off) return undefined + if (!st.cfg.probe || st.off) return undefined const id = canonical(st.cfg, call.model) const fresh = !st.verified.has(id) && !st.down.has(id) return aliasOf(st.cfg, id) !== undefined && fresh ? id : undefined @@ -1545,7 +1545,7 @@ function unprobed(st: State, call: Call): string | undefined { type Decided = { call: Call; wanted: string | undefined } -/** The decision, probing a move target once; a failed probe: decide again. */ +/** The decision, probing the model it lands on once; failure: decide again. */ async function decideProbed( $: Api, st: State, @@ -1609,8 +1609,9 @@ function isStandIn( /** * What the main loop would run on from `cur`: wanted model + decision. The - * step passes the `known` reading it took; a text reads for itself. A move - * target is probed here, after the decision (see `decideProbed`). + * step passes the `known` reading it took; a text reads for itself. The + * model main would run on is probed here, after the decision (see + * `decideProbed`). */ async function decideFor( $: Api,