diff --git a/mods/model-router/hooks/register.test.ts b/mods/model-router/hooks/register.test.ts index e6447ad..aef5dae 100644 --- a/mods/model-router/hooks/register.test.ts +++ b/mods/model-router/hooks/register.test.ts @@ -236,7 +236,7 @@ async function boot( on('classic.PostModelSwitch', () => ({})) on('skill.prompt', ($, e) => ({ text: e.text })) on('agent.offer', () => ({ isOffered: true })) - // the probe of a move target: every model answers unless a test says not + // the probe of the model main runs on: every model answers unless a test says not on('model.complete', async (_$, e) => { world.completions.push(e.model) return { value: (await world.complete(e.model)) as string } @@ -5504,7 +5504,7 @@ test('ttl: show says 60 min by default and 5 min after a 5m event', async ( expect(await route($, 'show')).toContain('ttl 5 min)') }) -// ---- probe: a move target answers before main is sent to it -------------- +// ---- probe: the model main runs on answers before main is sent ----------- const PROBE_MS = 8000 const FABLE_PLAN = 'plan the billing migration' @@ -5578,7 +5578,8 @@ test('probe: a lapsed hold probes again and holds longer', async ($, on) => { await endTurn($) await rig.clock.advance(wait * 60000) } - expect(world.completions).toEqual([FABLE, FABLE]) + // the fallback opus is verified by its own probe the first time only + expect(world.completions).toEqual([FABLE, OPUS, FABLE]) expect(rig.seen.map(s => s.model)).toEqual([OPUS, OPUS]) }) @@ -5588,7 +5589,7 @@ test('probe: a rejection marks the target down like a timeout', async ( const rig = await bootRig($, on, OPUS, 1000, world) await planRoute($) expect((await stepOn($, rig, OPUS)).model).toBe(OPUS) - expect(world.completions).toEqual([FABLE]) + expect(world.completions).toEqual([FABLE, OPUS]) expect(downLine(await route($, 'show'))).toContain('(probe)') }) @@ -5646,7 +5647,7 @@ test('probe: no probe when the classifier guess is above the session', async ( }) await typed($, PLAIN) await runStep($, { ...highStep(), model: SONNET }) - expect(world.completions).toEqual([]) + expect(world.completions).toEqual([SONNET]) expect(c.seen[c.seen.length - 1]?.model).toBe(SONNET) }) @@ -5656,15 +5657,47 @@ test('probe: no probe when the upgrade switch refuses the move', async ( const rig = await bootRig($, on, OPUS, 1000, world) await planRoute($) expect((await stepOn($, rig, OPUS)).model).toBe(OPUS) - expect(world.completions).toEqual([]) + expect(world.completions).toEqual([OPUS]) }) -test('probe: the current model is never probed', async ($, on) => { +test('probe: a session model that never answers leaves for opus on the first step', async ( + $, on) => { const world = probeWorld(hangsOn(FABLE)) const rig = await bootRig($, on, FABLE, 1000, world) + await typed($, FABLE_PLAN) + await pastProbe(rig, runStep($, { ...highStep(), model: FABLE })) + expect(rig.seen).toEqual([{ model: OPUS, effort: 'max' }]) + expect(world.completions).toEqual([FABLE]) + expect(downLine(await route($, 'show'))).toContain( + `${FABLE} for 15 min (probe)`) +}) + +test('probe: a healthy session model is probed once and kept', async ( + $, on) => { + const world = newWorld() + const rig = await bootRig($, on, FABLE, 1000, world) await planRoute($) expect((await stepOn($, rig, FABLE)).model).toBe(FABLE) - expect(world.completions).toEqual([]) + await endTurn($) + await planRoute($) + expect((await stepOn($, rig, FABLE)).model).toBe(FABLE) + expect(world.completions).toEqual([FABLE]) +}) + +test('probe: a lapsed hold on the session model probes it again', async ( + $, on) => { + const world = probeWorld(hangsOn(FABLE)) + const rig = await bootRig($, on, FABLE, 1000, world) + for (const wait of [15, 30]) { + await typed($, FABLE_PLAN) + await pastProbe(rig, runStep($, { ...highStep(), model: FABLE })) + expect(downLine(await route($, 'show'))).toContain(`for ${wait} min`) + await endTurn($) + await rig.clock.advance(wait * 60000) + } + // opus, the fallback, answers its own probe once + expect(world.completions).toEqual([FABLE, OPUS, FABLE]) + expect(rig.seen.map(s => s.model)).toEqual([OPUS, OPUS]) }) test('probe: false in the override makes no completion at all', async ( diff --git a/mods/model-router/hooks/register.ts b/mods/model-router/hooks/register.ts index 68fa9f9..3d2f321 100644 --- a/mods/model-router/hooks/register.ts +++ b/mods/model-router/hooks/register.ts @@ -34,7 +34,7 @@ type Config = { mainModelSwitch: boolean // gates DOWNGRADES only verbose: boolean classify: boolean // a small-model guess of the typed prompt's phase - probe: boolean // a move target is tried once before main is sent to it + probe: boolean // the model main would run on is tried once first spinner: boolean enabled: boolean // false: every hook passes through (per machine) } @@ -234,7 +234,7 @@ const LABEL_PHASE: Readonly> = { } const CLASSIFY_MIN = 12 // code points of a typed prompt worth a call const CLASSIFY_MS = 3000 // the call is abandoned after this -const PROBE_MS = 8000 // a move target that has not answered by then is down +const PROBE_MS = 8000 // a probed model that has not answered by then is down const PROBE_TOKENS = 16 // the probe's reply cap const CLASSIFY_MAX = 2000 // code points sent whole; longer: head + tail const CLASSIFY_HEAD = 500 @@ -1522,9 +1522,9 @@ async function probeAnswers($: Api, id: string): Promise { } /** - * Tries a move target before main is sent to it. An answer verifies it, - * unless a failure landed meanwhile (the hold wins); a timeout or a - * rejection marks it down. True when the move may go ahead. + * Tries the model main would run on before main is sent to it. An answer + * verifies it, unless a failure landed meanwhile (the hold wins); a timeout + * or a rejection marks it down. True when main may go ahead on it. */ async function probeModel($: Api, st: State, id: string): Promise { if (await probeAnswers($, id)) { @@ -1535,9 +1535,9 @@ async function probeModel($: Api, st: State, id: string): Promise { return !st.down.has(id) } -/** The table id a move lands on that still wants a probe, else undefined. */ +/** The table id the decision lands on (moved or kept) that wants a probe. */ function unprobed(st: State, call: Call): string | undefined { - if (!call.moved || !st.cfg.probe || st.off) return undefined + if (!st.cfg.probe || st.off) return undefined const id = canonical(st.cfg, call.model) const fresh = !st.verified.has(id) && !st.down.has(id) return aliasOf(st.cfg, id) !== undefined && fresh ? id : undefined @@ -1545,7 +1545,7 @@ function unprobed(st: State, call: Call): string | undefined { type Decided = { call: Call; wanted: string | undefined } -/** The decision, probing a move target once; a failed probe: decide again. */ +/** The decision, probing the model it lands on once; failure: decide again. */ async function decideProbed( $: Api, st: State, @@ -1609,8 +1609,9 @@ function isStandIn( /** * What the main loop would run on from `cur`: wanted model + decision. The - * step passes the `known` reading it took; a text reads for itself. A move - * target is probed here, after the decision (see `decideProbed`). + * step passes the `known` reading it took; a text reads for itself. The + * model main would run on is probed here, after the decision (see + * `decideProbed`). */ async function decideFor( $: Api,