fix(model-router): probe the kept session model, not only a move target

A session on Fable without credits kept Fable unprobed and lost its first turn, and each hold lapse, to the 429. The probe now covers the model the decision lands on, moved or kept.
This commit is contained in:
bchanot committed 2026-10-16 19:06:43 +02:00
1 parent a8eb3f2e90
commit c11cd3b53b
2 files changed
+52 -18

No files matched your search

+41 -8
View File
@@ -236,7 +236,7 @@ async function boot(
on('classic.PostModelSwitch', () => ({})) on('classic.PostModelSwitch', () => ({}))
on('skill.prompt', ($, e) => ({ text: e.text })) on('skill.prompt', ($, e) => ({ text: e.text }))
on('agent.offer', () => ({ isOffered: true })) on('agent.offer', () => ({ isOffered: true }))
// the probe of a move target: every model answers unless a test says not // the probe of the model main runs on: every model answers unless a test says not
on('model.complete', async (_$, e) => { on('model.complete', async (_$, e) => {
world.completions.push(e.model) world.completions.push(e.model)
return { value: (await world.complete(e.model)) as string } return { value: (await world.complete(e.model)) as string }
@@ -5504,7 +5504,7 @@ test('ttl: show says 60 min by default and 5 min after a 5m event', async (
expect(await route($, 'show')).toContain('ttl 5 min)') expect(await route($, 'show')).toContain('ttl 5 min)')
}) })
// ---- probe: a move target answers before main is sent to it -------------- // ---- probe: the model main runs on answers before main is sent -----------
const PROBE_MS = 8000 const PROBE_MS = 8000
const FABLE_PLAN = 'plan the billing migration' const FABLE_PLAN = 'plan the billing migration'
@@ -5578,7 +5578,8 @@ test('probe: a lapsed hold probes again and holds longer', async ($, on) => {
await endTurn($) await endTurn($)
await rig.clock.advance(wait * 60000) await rig.clock.advance(wait * 60000)
} }
expect(world.completions).toEqual([FABLE, FABLE]) // the fallback opus is verified by its own probe the first time only
expect(world.completions).toEqual([FABLE, OPUS, FABLE])
expect(rig.seen.map(s => s.model)).toEqual([OPUS, OPUS]) expect(rig.seen.map(s => s.model)).toEqual([OPUS, OPUS])
}) })
@@ -5588,7 +5589,7 @@ test('probe: a rejection marks the target down like a timeout', async (
const rig = await bootRig($, on, OPUS, 1000, world) const rig = await bootRig($, on, OPUS, 1000, world)
await planRoute($) await planRoute($)
expect((await stepOn($, rig, OPUS)).model).toBe(OPUS) expect((await stepOn($, rig, OPUS)).model).toBe(OPUS)
expect(world.completions).toEqual([FABLE]) expect(world.completions).toEqual([FABLE, OPUS])
expect(downLine(await route($, 'show'))).toContain('(probe)') expect(downLine(await route($, 'show'))).toContain('(probe)')
}) })
@@ -5646,7 +5647,7 @@ test('probe: no probe when the classifier guess is above the session', async (
}) })
await typed($, PLAIN) await typed($, PLAIN)
await runStep($, { ...highStep(), model: SONNET }) await runStep($, { ...highStep(), model: SONNET })
expect(world.completions).toEqual([]) expect(world.completions).toEqual([SONNET])
expect(c.seen[c.seen.length - 1]?.model).toBe(SONNET) expect(c.seen[c.seen.length - 1]?.model).toBe(SONNET)
}) })
@@ -5656,15 +5657,47 @@ test('probe: no probe when the upgrade switch refuses the move', async (
const rig = await bootRig($, on, OPUS, 1000, world) const rig = await bootRig($, on, OPUS, 1000, world)
await planRoute($) await planRoute($)
expect((await stepOn($, rig, OPUS)).model).toBe(OPUS) expect((await stepOn($, rig, OPUS)).model).toBe(OPUS)
expect(world.completions).toEqual([]) expect(world.completions).toEqual([OPUS])
}) })
test('probe: the current model is never probed', async ($, on) => { test('probe: a session model that never answers leaves for opus on the first step', async (
$, on) => {
const world = probeWorld(hangsOn(FABLE)) const world = probeWorld(hangsOn(FABLE))
const rig = await bootRig($, on, FABLE, 1000, world) const rig = await bootRig($, on, FABLE, 1000, world)
await typed($, FABLE_PLAN)
await pastProbe(rig, runStep($, { ...highStep(), model: FABLE }))
expect(rig.seen).toEqual([{ model: OPUS, effort: 'max' }])
expect(world.completions).toEqual([FABLE])
expect(downLine(await route($, 'show'))).toContain(
`${FABLE} for 15 min (probe)`)
})
test('probe: a healthy session model is probed once and kept', async (
$, on) => {
const world = newWorld()
const rig = await bootRig($, on, FABLE, 1000, world)
await planRoute($) await planRoute($)
expect((await stepOn($, rig, FABLE)).model).toBe(FABLE) expect((await stepOn($, rig, FABLE)).model).toBe(FABLE)
expect(world.completions).toEqual([]) await endTurn($)
await planRoute($)
expect((await stepOn($, rig, FABLE)).model).toBe(FABLE)
expect(world.completions).toEqual([FABLE])
})
test('probe: a lapsed hold on the session model probes it again', async (
$, on) => {
const world = probeWorld(hangsOn(FABLE))
const rig = await bootRig($, on, FABLE, 1000, world)
for (const wait of [15, 30]) {
await typed($, FABLE_PLAN)
await pastProbe(rig, runStep($, { ...highStep(), model: FABLE }))
expect(downLine(await route($, 'show'))).toContain(`for ${wait} min`)
await endTurn($)
await rig.clock.advance(wait * 60000)
}
// opus, the fallback, answers its own probe once
expect(world.completions).toEqual([FABLE, OPUS, FABLE])
expect(rig.seen.map(s => s.model)).toEqual([OPUS, OPUS])
}) })
test('probe: false in the override makes no completion at all', async ( test('probe: false in the override makes no completion at all', async (
+11 -10
View File
@@ -34,7 +34,7 @@ type Config = {
mainModelSwitch: boolean // gates DOWNGRADES only mainModelSwitch: boolean // gates DOWNGRADES only
verbose: boolean verbose: boolean
classify: boolean // a small-model guess of the typed prompt's phase classify: boolean // a small-model guess of the typed prompt's phase
probe: boolean // a move target is tried once before main is sent to it probe: boolean // the model main would run on is tried once first
spinner: boolean spinner: boolean
enabled: boolean // false: every hook passes through (per machine) enabled: boolean // false: every hook passes through (per machine)
} }
@@ -234,7 +234,7 @@ const LABEL_PHASE: Readonly<Record<string, string>> = {
} }
const CLASSIFY_MIN = 12 // code points of a typed prompt worth a call const CLASSIFY_MIN = 12 // code points of a typed prompt worth a call
const CLASSIFY_MS = 3000 // the call is abandoned after this const CLASSIFY_MS = 3000 // the call is abandoned after this
const PROBE_MS = 8000 // a move target that has not answered by then is down const PROBE_MS = 8000 // a probed model that has not answered by then is down
const PROBE_TOKENS = 16 // the probe's reply cap const PROBE_TOKENS = 16 // the probe's reply cap
const CLASSIFY_MAX = 2000 // code points sent whole; longer: head + tail const CLASSIFY_MAX = 2000 // code points sent whole; longer: head + tail
const CLASSIFY_HEAD = 500 const CLASSIFY_HEAD = 500
@@ -1522,9 +1522,9 @@ async function probeAnswers($: Api, id: string): Promise<boolean> {
} }
/** /**
* Tries a move target before main is sent to it. An answer verifies it, * Tries the model main would run on before main is sent to it. An answer
* unless a failure landed meanwhile (the hold wins); a timeout or a * verifies it, unless a failure landed meanwhile (the hold wins); a timeout
* rejection marks it down. True when the move may go ahead. * or a rejection marks it down. True when main may go ahead on it.
*/ */
async function probeModel($: Api, st: State, id: string): Promise<boolean> { async function probeModel($: Api, st: State, id: string): Promise<boolean> {
if (await probeAnswers($, id)) { if (await probeAnswers($, id)) {
@@ -1535,9 +1535,9 @@ async function probeModel($: Api, st: State, id: string): Promise<boolean> {
return !st.down.has(id) return !st.down.has(id)
} }
/** The table id a move lands on that still wants a probe, else undefined. */ /** The table id the decision lands on (moved or kept) that wants a probe. */
function unprobed(st: State, call: Call): string | undefined { function unprobed(st: State, call: Call): string | undefined {
if (!call.moved || !st.cfg.probe || st.off) return undefined if (!st.cfg.probe || st.off) return undefined
const id = canonical(st.cfg, call.model) const id = canonical(st.cfg, call.model)
const fresh = !st.verified.has(id) && !st.down.has(id) const fresh = !st.verified.has(id) && !st.down.has(id)
return aliasOf(st.cfg, id) !== undefined && fresh ? id : undefined return aliasOf(st.cfg, id) !== undefined && fresh ? id : undefined
@@ -1545,7 +1545,7 @@ function unprobed(st: State, call: Call): string | undefined {
type Decided = { call: Call; wanted: string | undefined } type Decided = { call: Call; wanted: string | undefined }
/** The decision, probing a move target once; a failed probe: decide again. */ /** The decision, probing the model it lands on once; failure: decide again. */
async function decideProbed( async function decideProbed(
$: Api, $: Api,
st: State, st: State,
@@ -1609,8 +1609,9 @@ function isStandIn(
/** /**
* What the main loop would run on from `cur`: wanted model + decision. The * What the main loop would run on from `cur`: wanted model + decision. The
* step passes the `known` reading it took; a text reads for itself. A move * step passes the `known` reading it took; a text reads for itself. The
* target is probed here, after the decision (see `decideProbed`). * model main would run on is probed here, after the decision (see
* `decideProbed`).
*/ */
async function decideFor( async function decideFor(
$: Api, $: Api,