feat(router): the /model picker asks its scope; a hand pick always switches

A pick in the picker is held (PreModelSwitch deny), then the same project / conversation / everywhere question runs; only Everywhere reaches the native write to user settings.json. A model chosen in that dialog bypasses the cold-read caps but waits while held down.
This commit is contained in:
bchanot committed 2026-10-25 20:36:49 +01:00
1 parent 48778af154
commit 2202964d02
5 files changed
+709 -31

No files matched your search

+396 -4
View File
@@ -40,6 +40,8 @@ type World = {
writeGate: Promise<void> | undefined // holds an fs.write open
failWrite: boolean // fs.write rejects
clockFailsAfterAsk: boolean // clock.now rejects once a dialog opened
sleepFails: boolean // clock.sleep rejects (the picker's delay)
sessionId: string // what session.id answers
onAsk: (() => void) | undefined // called when a dialog opens
onWrite: (() => void) | undefined // called when an fs.write starts
repo: { root: string; remote: string | null } | null | 'fail'
@@ -67,6 +69,7 @@ type Fixture = {
noHome?: boolean // HOME unset: env.get answers undefined
surfaces?: readonly string[] // default ['terminal']
clockFailsAfterAsk?: boolean // see World; the rig gets no MockClock
sleepFails?: boolean // see World; the rig gets no MockClock
}
/**
@@ -129,6 +132,8 @@ function newWorld(fx: Fixture = {}): World {
writeGate: undefined,
failWrite: false,
clockFailsAfterAsk: fx.clockFailsAfterAsk ?? false,
sleepFails: fx.sleepFails ?? false,
sessionId: 's1',
onAsk: undefined,
onWrite: undefined,
repo: fx.repo === undefined ? { root: ROOT, remote: REMOTE } : fx.repo,
@@ -253,9 +258,25 @@ function failingClock(on: On, w: World): MockClock {
if (w.asked.length > 0) throw new Error('boom')
return { value: 0 }
})
on('clock.sleep', () => ({ value: undefined }))
return undefined as unknown as MockClock
}
/** A clock whose sleep rejects (LRN-229: a whole-clock replacement). */
function sleepFailsClock(on: On): MockClock {
on('clock.now', () => ({ value: 0 }))
on('clock.sleep', () => {
throw new Error('boom')
})
return undefined as unknown as MockClock
}
/** The clock the world asks for: a mock one unless a fixture breaks it. */
function clockOf(on: On, w: World): MockClock {
if (w.clockFailsAfterAsk) return failingClock(on, w)
return w.sleepFails ? sleepFailsClock(on) : mock.clock(on)
}
/**
* Fires session.start so the mod loads its config and registers /route.
* The bottom hooks the router reads are in place before the first `$` call:
@@ -270,12 +291,12 @@ async function boot(
tokens: number | null = 1000,
world: World = newWorld(),
): Promise<MockClock> {
const clock = world.clockFailsAfterAsk
? failingClock(on, world)
: mock.clock(on)
const clock = clockOf(on, world)
on('session.model', () => ({ value: model }))
on('classic.StopFailure', () => ({}))
on('classic.PostModelSwitch', () => ({}))
on('classic.PreModelSwitch', () => ({}))
on('session.id', () => ({ value: world.sessionId }))
on('skill.prompt', ($, e) => ({ text: e.text }))
on('agent.offer', () => ({ isOffered: true }))
// the probe of the model main runs on: every model answers unless a test says not
@@ -721,7 +742,7 @@ const switchTo = (
$: Engine,
from: string,
to: string,
source: 'command' | 'auto',
source: 'command' | 'auto' | 'picker',
cache_ttl: '5m' | '1h' = '5m',
agent_id?: string,
) => $.classic.PostModelSwitch({
@@ -6328,3 +6349,374 @@ test('writes: kit: only the project settings.local.json is carved out', {
}
expect(worldServes(m.w, LOCAL_FILE)).toBe(true)
})
// ---- /model picker: deny now, ask after ----------------------------------
const BUSY = 'answer the open dialog first'
const DENY = 'model-router asks where this switch applies'
type SwitchSource = 'picker' | 'command' | 'sdk'
/** The PreModelSwitch input of a switch to `requested` from `source`. */
const preSwitch = (requested: string | null, source: SwitchSource) => ({
from_model: OPUS,
to_model: requested ?? OPUS,
requested_model: requested,
source,
context_tokens: 0,
prompt_cache_warm: false,
cache_ttl: '5m' as const,
estimated_cache_write_usd: 0,
pricing: 'catalog' as const,
})
const pick = (
$: Engine,
requested: string | null,
source: SwitchSource = 'picker',
) => $.classic.PreModelSwitch(preSwitch(requested, source))
/** Lets the timers and the hooks of the deferred dialog run. */
async function settle(done: () => boolean = () => false): Promise<void> {
for (let i = 0; i < 100 && !done(); i++) {
await new Promise(r => setTimeout(r, 0))
}
for (let i = 0; i < 5; i++) await new Promise(r => setTimeout(r, 0))
}
type PickerRig = ModelRig & { logs: string[] }
async function bootPicker(
$: Engine,
on: On,
answers: string[],
fx: Fixture = {},
): Promise<PickerRig> {
const logs = recordLogs(on)
const m = await bootModel($, on, answers, OPUS, fx)
return { ...m, logs }
}
/** The picker's switch, then the deferred question answered. */
async function pickAndAnswer(
$: Engine,
p: PickerRig,
requested: string,
): Promise<unknown> {
const out = await pick($, requested)
await p.rig.clock.advance(300)
await settle(() => p.w.asked.length > 0 && p.logs.length > 0)
return out
}
test('picker: a known model is denied at once, the question comes after', async (
$, on) => {
const p = await bootPicker($, on, ['This conversation'])
const held = deferred()
p.w.gate = held.gate
const out = await pick($, 'fable')
expect(JSON.stringify(out)).toContain(DENY)
expect(JSON.stringify(out)).toContain('deny')
expect(p.w.asked).toEqual([])
await p.rig.clock.advance(300)
await settle(() => p.w.asked.length > 0)
const [q] = p.w.asked as [Asked]
expect(q.options).toEqual(SCOPES)
expect(q.header).toBe('model-router')
expect(q.text.endsWith(`Switch main to ${FABLE} where?`)).toBe(true)
held.release()
await settle()
expect(p.ran).toEqual([])
})
const PASSTHROUGH: [string, string | null, SwitchSource][] = [
['source command', 'fable', 'command'],
['source sdk', 'fable', 'sdk'],
['Default (null)', null, 'picker'],
['opusplan', 'opusplan', 'picker'],
['an unknown id', 'claude-opus-5-6', 'picker'],
]
for (const [name, requested, source] of PASSTHROUGH) {
test(`picker: ${name} passes through, no question`, async ($, on) => {
const p = await bootPicker($, on, ['This project'])
const out = await pick($, requested, source)
await p.rig.clock.advance(300)
await settle()
expect(JSON.stringify(out)).not.toContain('deny')
expect(p.w.asked).toEqual([])
expect(p.toasts).toEqual([])
})
}
test('picker: router off passes through', async ($, on) => {
const p = await bootPicker($, on, [])
await route($, 'off')
expect(JSON.stringify(await pick($, 'fable'))).not.toContain('deny')
await settle()
expect(p.w.asked).toEqual([])
})
test('picker: no surface passes through', async ($, on) => {
const p = await bootPicker($, on, [], { surfaces: [] })
expect(JSON.stringify(await pick($, 'fable'))).not.toContain('deny')
await settle()
expect(p.w.asked).toEqual([])
})
test('picker: an open dialog denies with a toast, never native', async (
$, on) => {
const p = await bootPicker($, on, ['This conversation'])
const held = deferred()
const open = deferred()
p.w.gate = held.gate
p.w.onAsk = open.release
const first = modelCmd($, 'fable')
await open.gate
const out = await pick($, 'sonnet')
expect(JSON.stringify(out)).toContain('deny')
expect(p.toasts).toEqual([BUSY])
held.release()
await first
expect(p.w.asked).toHaveLength(1)
expect(p.ran).toEqual([])
})
test('picker: This project writes the full id, native never runs', async (
$, on) => {
const p = await bootPicker($, on, ['This project'])
await pickAndAnswer($, p, 'fable')
expect(localOf(p.w)).toEqual({ model: FABLE })
expect(p.logs.join('\n')).toContain(`${FABLE} written to`)
expect(p.ran).toEqual([])
expect((await stepOn($, p.rig, OPUS)).model).toBe(FABLE)
})
test('picker: This conversation routes only, writes nothing', async (
$, on) => {
const p = await bootPicker($, on, ['This conversation'])
await pickAndAnswer($, p, 'fable')
expect(p.w.written).toEqual([])
expect(p.logs.join('\n')).toContain('for this conversation')
expect(p.ran).toEqual([])
expect((await stepOn($, p.rig, OPUS)).model).toBe(FABLE)
})
test('picker: Everywhere drops a hand pick, runs native once', async (
$, on) => {
const p = await bootPicker($, on, ['This conversation', 'Everywhere'])
await modelCmd($, 'fable')
await pickAndAnswer($, p, 'sonnet')
expect(p.ran).toEqual(['sonnet'])
expect(p.w.asked).toHaveLength(2)
expect(p.w.written).toEqual([])
expect(mainLine(await route($, 'show'))).toContain('session defaults')
expect(p.logs.filter(l => l.includes('native'))).toEqual([])
})
test('picker: dismissed logs the no-scope text and frees the slot', async (
$, on) => {
const p = await bootPicker($, on, [])
await pickAndAnswer($, p, 'fable')
expect(p.logs).toContain(NO_SCOPE)
expect(p.ran).toEqual([])
expect(p.w.written).toEqual([])
await pickAndAnswer($, p, 'fable')
expect(p.w.asked).toHaveLength(2)
})
test('picker: a throwing sleep is logged under its key, slot freed', async (
$, on) => {
const logs = recordLogs(on)
const m = await bootModel($, on, ['This project'], OPUS,
{ sleepFails: true })
expect(JSON.stringify(await pick($, 'fable'))).toContain('deny')
await settle(() => logs.some(l => l.includes('PreModelSwitch picker')))
expect(logs.join('\n')).toContain('PreModelSwitch picker failed')
expect(m.w.asked).toEqual([])
expect(m.w.written).toEqual([])
const again = await pick($, 'fable')
expect(JSON.stringify(again)).toContain('deny')
expect(m.toasts).toEqual([])
})
test('picker: a throw after the answer is logged, native never runs', async (
$, on) => {
const logs = recordLogs(on)
const m = await bootModel($, on, ['This conversation'], OPUS,
{ clockFailsAfterAsk: true })
await pick($, 'fable')
await settle(() => logs.some(l => l.includes('model failed (')))
expect(logs.join('\n')).toContain('model failed (')
expect(m.ran).toEqual([])
await pick($, 'fable')
await settle()
expect(m.w.asked).toHaveLength(2)
})
test('picker: a changed conversation applies nothing', async ($, on) => {
const p = await bootPicker($, on, ['This project'])
const held = deferred()
const open = deferred()
p.w.gate = held.gate
p.w.onAsk = open.release
await pick($, 'fable')
await p.rig.clock.advance(300)
await open.gate
p.w.sessionId = 's2'
held.release()
await settle(() => p.logs.length > 0)
expect(p.logs.join('\n')).toContain('conversation changed')
expect(p.w.written).toEqual([])
expect(mainLine(await route($, 'show'))).toContain('session defaults')
})
test('picker: a native switch passes without a second dialog', async (
$, on) => {
const p = await bootPicker($, on, ['Everywhere'])
await pickAndAnswer($, p, 'opus')
expect(JSON.stringify(await pick($, 'opus', 'command'))).not.toContain('deny')
expect(p.w.asked).toHaveLength(1)
})
test('picker: a pick clears the mark of its target', async ($, on) => {
const p = await bootPicker($, on, ['This conversation'])
await route($, 'model=fable')
await stepOn($, p.rig, FABLE)
await failWith($, 'overloaded')
expect(await route($, 'show')).toContain('claude-fable-5-1 for')
await pickAndAnswer($, p, 'fable')
expect(await route($, 'show')).toContain('down: none')
})
test('picker: a PostModelSwitch of a denied pick keeps the hand pick', async (
$, on) => {
const p = await bootPicker($, on, ['This conversation', 'This conversation'])
await modelCmd($, 'fable')
const held = deferred()
p.w.gate = held.gate
await pick($, 'sonnet')
await switchTo($, OPUS, SONNET, 'picker')
held.release()
await settle()
expect(mainLine(await route($, 'show'))).toContain('user custom')
})
// ---- hand pick: a model chosen by hand always switches --------------------
type HandRig = {
w: World
rig: Rig
usage: { set: (n: number) => void }
logs: string[]
}
async function bootHand(
$: Engine,
on: On,
answers: string[],
session: string = OPUS,
tokens = 1000,
fx: Fixture = {},
): Promise<HandRig> {
on('command.run', { command: 'model' }, () => ({ text: 'native' }))
const logs = recordLogs(on)
const w = newWorld({ answers, ...fx })
const { rig, usage } = await bootRigVar($, on, session, tokens, w)
return { w, rig, usage, logs }
}
test('hand pick: a held-down target waits across the turn boundary', async (
$, on) => {
const h = await bootHand($, on, ['This conversation'])
await modelCmd($, 'fable')
expect((await stepOn($, h.rig, OPUS)).model).toBe(FABLE)
await failWith($, 'overloaded')
await endTurn($)
expect((await stepOn($, h.rig, OPUS)).model).toBe(OPUS)
await stepOn($, h.rig, OPUS)
expect(h.logs.filter(l => l.includes('hand pick waits'))).toHaveLength(1)
await endTurn($)
await h.rig.clock.advance(15 * 60000)
expect((await stepOn($, h.rig, OPUS)).model).toBe(FABLE)
})
test('hand pick: the current model down leaves it for the hand pick', async (
$, on) => {
const h = await bootHand($, on, ['This conversation'], OPUS, 300000,
{ home: { mainUpgrade: false } })
await stepOn($, h.rig, OPUS)
await failWith($, 'overloaded')
await endTurn($)
await modelCmd($, 'fable')
expect((await stepOn($, h.rig, OPUS)).model).toBe(FABLE)
})
test('hand pick: an unranked session model does not trap it', async (
$, on) => {
const h = await bootHand($, on, ['This conversation'], 'claude-zz-9')
await modelCmd($, 'opus')
expect((await stepOn($, h.rig, 'claude-zz-9')).model).toBe(OPUS)
})
test('hand pick: over the upgrade cap and with the switch off still moves', async (
$, on) => {
const h = await bootHand($, on, ['This conversation'], OPUS, 300000,
{ home: { mainUpgrade: false } })
await modelCmd($, 'fable')
expect((await stepOn($, h.rig, OPUS)).model).toBe(FABLE)
await endTurn($)
expect((await stepOn($, h.rig, OPUS)).model).toBe(FABLE)
})
test('hand pick: over the downgrade cap with switch off still moves', async (
$, on) => {
const h = await bootHand($, on, ['This conversation'], FABLE, 300000)
await route($, 'switch off')
await modelCmd($, 'sonnet')
expect((await stepOn($, h.rig, FABLE)).model).toBe(SONNET)
})
test('hand pick: a typed /route model= keeps the caps', async ($, on) => {
const h = await bootHand($, on, [], OPUS, 300000)
await route($, 'model=fable')
expect((await stepOn($, h.rig, OPUS)).model).toBe(OPUS)
})
test('hand pick: the current model pinned and down falls to the next', async (
$, on) => {
const h = await bootHand($, on, ['This conversation'])
await modelCmd($, 'opus')
expect((await stepOn($, h.rig, OPUS)).model).toBe(OPUS)
await failWith($, 'overloaded')
const same = (await stepOn($, h.rig, OPUS)).model
expect(same).not.toBe(OPUS)
await endTurn($)
expect((await stepOn($, h.rig, OPUS)).model).toBe(same)
})
test('hand pick: a target that fails mid-turn is left', async ($, on) => {
const h = await bootHand($, on, ['This conversation'])
await modelCmd($, 'fable')
expect((await stepOn($, h.rig, OPUS)).model).toBe(FABLE)
await failWith($, 'overloaded')
expect((await stepOn($, h.rig, OPUS)).model).not.toBe(FABLE)
})
test('hand pick: outgrowing a known window still returns home', async (
$, on) => {
const h = await bootHand($, on, ['This conversation'], FABLE, 1000,
{ home: { windows: { [HAIKU]: 200000 } } })
await modelCmd($, 'haiku')
expect((await stepOn($, h.rig, FABLE)).model).toBe(HAIKU)
h.usage.set(250000)
expect((await stepOn($, h.rig, FABLE)).model).toBe(FABLE)
})
test('hand pick: a native user switch drops it', async ($, on) => {
const h = await bootHand($, on, ['This conversation'])
await modelCmd($, 'fable')
await switchTo($, OPUS, SONNET, 'picker')
expect((await stepOn($, h.rig, SONNET)).model).toBe(SONNET)
expect(mainLine(await route($, 'show'))).toContain('session defaults')
})