fix(model-router): probe the kept session model, not only a move target

A session on Fable without credits kept Fable unprobed and lost its first turn, and each hold lapse, to the 429. The probe now covers the model the decision lands on, moved or kept.
This commit is contained in:
bchanot committed 2026-10-16 19:06:43 +02:00
1 parent a8eb3f2e90
commit c11cd3b53b
2 files changed
+52 -18

No files matched your search

+41 -8
View File
@@ -236,7 +236,7 @@ async function boot(
on('classic.PostModelSwitch', () => ({}))
on('skill.prompt', ($, e) => ({ text: e.text }))
on('agent.offer', () => ({ isOffered: true }))
// the probe of a move target: every model answers unless a test says not
// the probe of the model main runs on: every model answers unless a test says not
on('model.complete', async (_$, e) => {
world.completions.push(e.model)
return { value: (await world.complete(e.model)) as string }
@@ -5504,7 +5504,7 @@ test('ttl: show says 60 min by default and 5 min after a 5m event', async (
expect(await route($, 'show')).toContain('ttl 5 min)')
})
// ---- probe: a move target answers before main is sent to it --------------
// ---- probe: the model main runs on answers before main is sent -----------
const PROBE_MS = 8000
const FABLE_PLAN = 'plan the billing migration'
@@ -5578,7 +5578,8 @@ test('probe: a lapsed hold probes again and holds longer', async ($, on) => {
await endTurn($)
await rig.clock.advance(wait * 60000)
}
expect(world.completions).toEqual([FABLE, FABLE])
// the fallback opus is verified by its own probe the first time only
expect(world.completions).toEqual([FABLE, OPUS, FABLE])
expect(rig.seen.map(s => s.model)).toEqual([OPUS, OPUS])
})
@@ -5588,7 +5589,7 @@ test('probe: a rejection marks the target down like a timeout', async (
const rig = await bootRig($, on, OPUS, 1000, world)
await planRoute($)
expect((await stepOn($, rig, OPUS)).model).toBe(OPUS)
expect(world.completions).toEqual([FABLE])
expect(world.completions).toEqual([FABLE, OPUS])
expect(downLine(await route($, 'show'))).toContain('(probe)')
})
@@ -5646,7 +5647,7 @@ test('probe: no probe when the classifier guess is above the session', async (
})
await typed($, PLAIN)
await runStep($, { ...highStep(), model: SONNET })
expect(world.completions).toEqual([])
expect(world.completions).toEqual([SONNET])
expect(c.seen[c.seen.length - 1]?.model).toBe(SONNET)
})
@@ -5656,15 +5657,47 @@ test('probe: no probe when the upgrade switch refuses the move', async (
const rig = await bootRig($, on, OPUS, 1000, world)
await planRoute($)
expect((await stepOn($, rig, OPUS)).model).toBe(OPUS)
expect(world.completions).toEqual([])
expect(world.completions).toEqual([OPUS])
})
test('probe: the current model is never probed', async ($, on) => {
test('probe: a session model that never answers leaves for opus on the first step', async (
$, on) => {
const world = probeWorld(hangsOn(FABLE))
const rig = await bootRig($, on, FABLE, 1000, world)
await typed($, FABLE_PLAN)
await pastProbe(rig, runStep($, { ...highStep(), model: FABLE }))
expect(rig.seen).toEqual([{ model: OPUS, effort: 'max' }])
expect(world.completions).toEqual([FABLE])
expect(downLine(await route($, 'show'))).toContain(
`${FABLE} for 15 min (probe)`)
})
test('probe: a healthy session model is probed once and kept', async (
$, on) => {
const world = newWorld()
const rig = await bootRig($, on, FABLE, 1000, world)
await planRoute($)
expect((await stepOn($, rig, FABLE)).model).toBe(FABLE)
expect(world.completions).toEqual([])
await endTurn($)
await planRoute($)
expect((await stepOn($, rig, FABLE)).model).toBe(FABLE)
expect(world.completions).toEqual([FABLE])
})
test('probe: a lapsed hold on the session model probes it again', async (
$, on) => {
const world = probeWorld(hangsOn(FABLE))
const rig = await bootRig($, on, FABLE, 1000, world)
for (const wait of [15, 30]) {
await typed($, FABLE_PLAN)
await pastProbe(rig, runStep($, { ...highStep(), model: FABLE }))
expect(downLine(await route($, 'show'))).toContain(`for ${wait} min`)
await endTurn($)
await rig.clock.advance(wait * 60000)
}
// opus, the fallback, answers its own probe once
expect(world.completions).toEqual([FABLE, OPUS, FABLE])
expect(rig.seen.map(s => s.model)).toEqual([OPUS, OPUS])
})
test('probe: false in the override makes no completion at all', async (
+11 -10
View File
@@ -34,7 +34,7 @@ type Config = {
mainModelSwitch: boolean // gates DOWNGRADES only
verbose: boolean
classify: boolean // a small-model guess of the typed prompt's phase
probe: boolean // a move target is tried once before main is sent to it
probe: boolean // the model main would run on is tried once first
spinner: boolean
enabled: boolean // false: every hook passes through (per machine)
}
@@ -234,7 +234,7 @@ const LABEL_PHASE: Readonly<Record<string, string>> = {
}
const CLASSIFY_MIN = 12 // code points of a typed prompt worth a call
const CLASSIFY_MS = 3000 // the call is abandoned after this
const PROBE_MS = 8000 // a move target that has not answered by then is down
const PROBE_MS = 8000 // a probed model that has not answered by then is down
const PROBE_TOKENS = 16 // the probe's reply cap
const CLASSIFY_MAX = 2000 // code points sent whole; longer: head + tail
const CLASSIFY_HEAD = 500
@@ -1522,9 +1522,9 @@ async function probeAnswers($: Api, id: string): Promise<boolean> {
}
/**
* Tries a move target before main is sent to it. An answer verifies it,
* unless a failure landed meanwhile (the hold wins); a timeout or a
* rejection marks it down. True when the move may go ahead.
* Tries the model main would run on before main is sent to it. An answer
* verifies it, unless a failure landed meanwhile (the hold wins); a timeout
* or a rejection marks it down. True when main may go ahead on it.
*/
async function probeModel($: Api, st: State, id: string): Promise<boolean> {
if (await probeAnswers($, id)) {
@@ -1535,9 +1535,9 @@ async function probeModel($: Api, st: State, id: string): Promise<boolean> {
return !st.down.has(id)
}
/** The table id a move lands on that still wants a probe, else undefined. */
/** The table id the decision lands on (moved or kept) that wants a probe. */
function unprobed(st: State, call: Call): string | undefined {
if (!call.moved || !st.cfg.probe || st.off) return undefined
if (!st.cfg.probe || st.off) return undefined
const id = canonical(st.cfg, call.model)
const fresh = !st.verified.has(id) && !st.down.has(id)
return aliasOf(st.cfg, id) !== undefined && fresh ? id : undefined
@@ -1545,7 +1545,7 @@ function unprobed(st: State, call: Call): string | undefined {
type Decided = { call: Call; wanted: string | undefined }
/** The decision, probing a move target once; a failed probe: decide again. */
/** The decision, probing the model it lands on once; failure: decide again. */
async function decideProbed(
$: Api,
st: State,
@@ -1609,8 +1609,9 @@ function isStandIn(
/**
* What the main loop would run on from `cur`: wanted model + decision. The
* step passes the `known` reading it took; a text reads for itself. A move
* target is probed here, after the decision (see `decideProbed`).
* step passes the `known` reading it took; a text reads for itself. The
* model main would run on is probed here, after the decision (see
* `decideProbed`).
*/
async function decideFor(
$: Api,