forked from bchanot/claude
feat(router): the /model picker asks its scope; a hand pick always switches
A pick in the picker is held (PreModelSwitch deny), then the same project / conversation / everywhere question runs; only Everywhere reaches the native write to user settings.json. A model chosen in that dialog bypasses the cold-read caps but waits while held down.
This commit is contained in:
1 parent
48778af154
commit
2202964d02
5 files changed
+709
-31
No files matched your search
@@ -40,6 +40,8 @@ type World = {
|
||||
writeGate: Promise<void> | undefined // holds an fs.write open
|
||||
failWrite: boolean // fs.write rejects
|
||||
clockFailsAfterAsk: boolean // clock.now rejects once a dialog opened
|
||||
sleepFails: boolean // clock.sleep rejects (the picker's delay)
|
||||
sessionId: string // what session.id answers
|
||||
onAsk: (() => void) | undefined // called when a dialog opens
|
||||
onWrite: (() => void) | undefined // called when an fs.write starts
|
||||
repo: { root: string; remote: string | null } | null | 'fail'
|
||||
@@ -67,6 +69,7 @@ type Fixture = {
|
||||
noHome?: boolean // HOME unset: env.get answers undefined
|
||||
surfaces?: readonly string[] // default ['terminal']
|
||||
clockFailsAfterAsk?: boolean // see World; the rig gets no MockClock
|
||||
sleepFails?: boolean // see World; the rig gets no MockClock
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -129,6 +132,8 @@ function newWorld(fx: Fixture = {}): World {
|
||||
writeGate: undefined,
|
||||
failWrite: false,
|
||||
clockFailsAfterAsk: fx.clockFailsAfterAsk ?? false,
|
||||
sleepFails: fx.sleepFails ?? false,
|
||||
sessionId: 's1',
|
||||
onAsk: undefined,
|
||||
onWrite: undefined,
|
||||
repo: fx.repo === undefined ? { root: ROOT, remote: REMOTE } : fx.repo,
|
||||
@@ -253,9 +258,25 @@ function failingClock(on: On, w: World): MockClock {
|
||||
if (w.asked.length > 0) throw new Error('boom')
|
||||
return { value: 0 }
|
||||
})
|
||||
on('clock.sleep', () => ({ value: undefined }))
|
||||
return undefined as unknown as MockClock
|
||||
}
|
||||
|
||||
/** A clock whose sleep rejects (LRN-229: a whole-clock replacement). */
|
||||
function sleepFailsClock(on: On): MockClock {
|
||||
on('clock.now', () => ({ value: 0 }))
|
||||
on('clock.sleep', () => {
|
||||
throw new Error('boom')
|
||||
})
|
||||
return undefined as unknown as MockClock
|
||||
}
|
||||
|
||||
/** The clock the world asks for: a mock one unless a fixture breaks it. */
|
||||
function clockOf(on: On, w: World): MockClock {
|
||||
if (w.clockFailsAfterAsk) return failingClock(on, w)
|
||||
return w.sleepFails ? sleepFailsClock(on) : mock.clock(on)
|
||||
}
|
||||
|
||||
/**
|
||||
* Fires session.start so the mod loads its config and registers /route.
|
||||
* The bottom hooks the router reads are in place before the first `$` call:
|
||||
@@ -270,12 +291,12 @@ async function boot(
|
||||
tokens: number | null = 1000,
|
||||
world: World = newWorld(),
|
||||
): Promise<MockClock> {
|
||||
const clock = world.clockFailsAfterAsk
|
||||
? failingClock(on, world)
|
||||
: mock.clock(on)
|
||||
const clock = clockOf(on, world)
|
||||
on('session.model', () => ({ value: model }))
|
||||
on('classic.StopFailure', () => ({}))
|
||||
on('classic.PostModelSwitch', () => ({}))
|
||||
on('classic.PreModelSwitch', () => ({}))
|
||||
on('session.id', () => ({ value: world.sessionId }))
|
||||
on('skill.prompt', ($, e) => ({ text: e.text }))
|
||||
on('agent.offer', () => ({ isOffered: true }))
|
||||
// the probe of the model main runs on: every model answers unless a test says not
|
||||
@@ -721,7 +742,7 @@ const switchTo = (
|
||||
$: Engine,
|
||||
from: string,
|
||||
to: string,
|
||||
source: 'command' | 'auto',
|
||||
source: 'command' | 'auto' | 'picker',
|
||||
cache_ttl: '5m' | '1h' = '5m',
|
||||
agent_id?: string,
|
||||
) => $.classic.PostModelSwitch({
|
||||
@@ -6328,3 +6349,374 @@ test('writes: kit: only the project settings.local.json is carved out', {
|
||||
}
|
||||
expect(worldServes(m.w, LOCAL_FILE)).toBe(true)
|
||||
})
|
||||
|
||||
// ---- /model picker: deny now, ask after ----------------------------------
|
||||
|
||||
const BUSY = 'answer the open dialog first'
|
||||
const DENY = 'model-router asks where this switch applies'
|
||||
|
||||
type SwitchSource = 'picker' | 'command' | 'sdk'
|
||||
|
||||
/** The PreModelSwitch input of a switch to `requested` from `source`. */
|
||||
const preSwitch = (requested: string | null, source: SwitchSource) => ({
|
||||
from_model: OPUS,
|
||||
to_model: requested ?? OPUS,
|
||||
requested_model: requested,
|
||||
source,
|
||||
context_tokens: 0,
|
||||
prompt_cache_warm: false,
|
||||
cache_ttl: '5m' as const,
|
||||
estimated_cache_write_usd: 0,
|
||||
pricing: 'catalog' as const,
|
||||
})
|
||||
|
||||
const pick = (
|
||||
$: Engine,
|
||||
requested: string | null,
|
||||
source: SwitchSource = 'picker',
|
||||
) => $.classic.PreModelSwitch(preSwitch(requested, source))
|
||||
|
||||
/** Lets the timers and the hooks of the deferred dialog run. */
|
||||
async function settle(done: () => boolean = () => false): Promise<void> {
|
||||
for (let i = 0; i < 100 && !done(); i++) {
|
||||
await new Promise(r => setTimeout(r, 0))
|
||||
}
|
||||
for (let i = 0; i < 5; i++) await new Promise(r => setTimeout(r, 0))
|
||||
}
|
||||
|
||||
type PickerRig = ModelRig & { logs: string[] }
|
||||
|
||||
async function bootPicker(
|
||||
$: Engine,
|
||||
on: On,
|
||||
answers: string[],
|
||||
fx: Fixture = {},
|
||||
): Promise<PickerRig> {
|
||||
const logs = recordLogs(on)
|
||||
const m = await bootModel($, on, answers, OPUS, fx)
|
||||
return { ...m, logs }
|
||||
}
|
||||
|
||||
/** The picker's switch, then the deferred question answered. */
|
||||
async function pickAndAnswer(
|
||||
$: Engine,
|
||||
p: PickerRig,
|
||||
requested: string,
|
||||
): Promise<unknown> {
|
||||
const out = await pick($, requested)
|
||||
await p.rig.clock.advance(300)
|
||||
await settle(() => p.w.asked.length > 0 && p.logs.length > 0)
|
||||
return out
|
||||
}
|
||||
|
||||
test('picker: a known model is denied at once, the question comes after', async (
|
||||
$, on) => {
|
||||
const p = await bootPicker($, on, ['This conversation'])
|
||||
const held = deferred()
|
||||
p.w.gate = held.gate
|
||||
const out = await pick($, 'fable')
|
||||
expect(JSON.stringify(out)).toContain(DENY)
|
||||
expect(JSON.stringify(out)).toContain('deny')
|
||||
expect(p.w.asked).toEqual([])
|
||||
await p.rig.clock.advance(300)
|
||||
await settle(() => p.w.asked.length > 0)
|
||||
const [q] = p.w.asked as [Asked]
|
||||
expect(q.options).toEqual(SCOPES)
|
||||
expect(q.header).toBe('model-router')
|
||||
expect(q.text.endsWith(`Switch main to ${FABLE} where?`)).toBe(true)
|
||||
held.release()
|
||||
await settle()
|
||||
expect(p.ran).toEqual([])
|
||||
})
|
||||
|
||||
const PASSTHROUGH: [string, string | null, SwitchSource][] = [
|
||||
['source command', 'fable', 'command'],
|
||||
['source sdk', 'fable', 'sdk'],
|
||||
['Default (null)', null, 'picker'],
|
||||
['opusplan', 'opusplan', 'picker'],
|
||||
['an unknown id', 'claude-opus-5-6', 'picker'],
|
||||
]
|
||||
|
||||
for (const [name, requested, source] of PASSTHROUGH) {
|
||||
test(`picker: ${name} passes through, no question`, async ($, on) => {
|
||||
const p = await bootPicker($, on, ['This project'])
|
||||
const out = await pick($, requested, source)
|
||||
await p.rig.clock.advance(300)
|
||||
await settle()
|
||||
expect(JSON.stringify(out)).not.toContain('deny')
|
||||
expect(p.w.asked).toEqual([])
|
||||
expect(p.toasts).toEqual([])
|
||||
})
|
||||
}
|
||||
|
||||
test('picker: router off passes through', async ($, on) => {
|
||||
const p = await bootPicker($, on, [])
|
||||
await route($, 'off')
|
||||
expect(JSON.stringify(await pick($, 'fable'))).not.toContain('deny')
|
||||
await settle()
|
||||
expect(p.w.asked).toEqual([])
|
||||
})
|
||||
|
||||
test('picker: no surface passes through', async ($, on) => {
|
||||
const p = await bootPicker($, on, [], { surfaces: [] })
|
||||
expect(JSON.stringify(await pick($, 'fable'))).not.toContain('deny')
|
||||
await settle()
|
||||
expect(p.w.asked).toEqual([])
|
||||
})
|
||||
|
||||
test('picker: an open dialog denies with a toast, never native', async (
|
||||
$, on) => {
|
||||
const p = await bootPicker($, on, ['This conversation'])
|
||||
const held = deferred()
|
||||
const open = deferred()
|
||||
p.w.gate = held.gate
|
||||
p.w.onAsk = open.release
|
||||
const first = modelCmd($, 'fable')
|
||||
await open.gate
|
||||
const out = await pick($, 'sonnet')
|
||||
expect(JSON.stringify(out)).toContain('deny')
|
||||
expect(p.toasts).toEqual([BUSY])
|
||||
held.release()
|
||||
await first
|
||||
expect(p.w.asked).toHaveLength(1)
|
||||
expect(p.ran).toEqual([])
|
||||
})
|
||||
|
||||
test('picker: This project writes the full id, native never runs', async (
|
||||
$, on) => {
|
||||
const p = await bootPicker($, on, ['This project'])
|
||||
await pickAndAnswer($, p, 'fable')
|
||||
expect(localOf(p.w)).toEqual({ model: FABLE })
|
||||
expect(p.logs.join('\n')).toContain(`${FABLE} written to`)
|
||||
expect(p.ran).toEqual([])
|
||||
expect((await stepOn($, p.rig, OPUS)).model).toBe(FABLE)
|
||||
})
|
||||
|
||||
test('picker: This conversation routes only, writes nothing', async (
|
||||
$, on) => {
|
||||
const p = await bootPicker($, on, ['This conversation'])
|
||||
await pickAndAnswer($, p, 'fable')
|
||||
expect(p.w.written).toEqual([])
|
||||
expect(p.logs.join('\n')).toContain('for this conversation')
|
||||
expect(p.ran).toEqual([])
|
||||
expect((await stepOn($, p.rig, OPUS)).model).toBe(FABLE)
|
||||
})
|
||||
|
||||
test('picker: Everywhere drops a hand pick, runs native once', async (
|
||||
$, on) => {
|
||||
const p = await bootPicker($, on, ['This conversation', 'Everywhere'])
|
||||
await modelCmd($, 'fable')
|
||||
await pickAndAnswer($, p, 'sonnet')
|
||||
expect(p.ran).toEqual(['sonnet'])
|
||||
expect(p.w.asked).toHaveLength(2)
|
||||
expect(p.w.written).toEqual([])
|
||||
expect(mainLine(await route($, 'show'))).toContain('session defaults')
|
||||
expect(p.logs.filter(l => l.includes('native'))).toEqual([])
|
||||
})
|
||||
|
||||
test('picker: dismissed logs the no-scope text and frees the slot', async (
|
||||
$, on) => {
|
||||
const p = await bootPicker($, on, [])
|
||||
await pickAndAnswer($, p, 'fable')
|
||||
expect(p.logs).toContain(NO_SCOPE)
|
||||
expect(p.ran).toEqual([])
|
||||
expect(p.w.written).toEqual([])
|
||||
await pickAndAnswer($, p, 'fable')
|
||||
expect(p.w.asked).toHaveLength(2)
|
||||
})
|
||||
|
||||
test('picker: a throwing sleep is logged under its key, slot freed', async (
|
||||
$, on) => {
|
||||
const logs = recordLogs(on)
|
||||
const m = await bootModel($, on, ['This project'], OPUS,
|
||||
{ sleepFails: true })
|
||||
expect(JSON.stringify(await pick($, 'fable'))).toContain('deny')
|
||||
await settle(() => logs.some(l => l.includes('PreModelSwitch picker')))
|
||||
expect(logs.join('\n')).toContain('PreModelSwitch picker failed')
|
||||
expect(m.w.asked).toEqual([])
|
||||
expect(m.w.written).toEqual([])
|
||||
const again = await pick($, 'fable')
|
||||
expect(JSON.stringify(again)).toContain('deny')
|
||||
expect(m.toasts).toEqual([])
|
||||
})
|
||||
|
||||
test('picker: a throw after the answer is logged, native never runs', async (
|
||||
$, on) => {
|
||||
const logs = recordLogs(on)
|
||||
const m = await bootModel($, on, ['This conversation'], OPUS,
|
||||
{ clockFailsAfterAsk: true })
|
||||
await pick($, 'fable')
|
||||
await settle(() => logs.some(l => l.includes('model failed (')))
|
||||
expect(logs.join('\n')).toContain('model failed (')
|
||||
expect(m.ran).toEqual([])
|
||||
await pick($, 'fable')
|
||||
await settle()
|
||||
expect(m.w.asked).toHaveLength(2)
|
||||
})
|
||||
|
||||
test('picker: a changed conversation applies nothing', async ($, on) => {
|
||||
const p = await bootPicker($, on, ['This project'])
|
||||
const held = deferred()
|
||||
const open = deferred()
|
||||
p.w.gate = held.gate
|
||||
p.w.onAsk = open.release
|
||||
await pick($, 'fable')
|
||||
await p.rig.clock.advance(300)
|
||||
await open.gate
|
||||
p.w.sessionId = 's2'
|
||||
held.release()
|
||||
await settle(() => p.logs.length > 0)
|
||||
expect(p.logs.join('\n')).toContain('conversation changed')
|
||||
expect(p.w.written).toEqual([])
|
||||
expect(mainLine(await route($, 'show'))).toContain('session defaults')
|
||||
})
|
||||
|
||||
test('picker: a native switch passes without a second dialog', async (
|
||||
$, on) => {
|
||||
const p = await bootPicker($, on, ['Everywhere'])
|
||||
await pickAndAnswer($, p, 'opus')
|
||||
expect(JSON.stringify(await pick($, 'opus', 'command'))).not.toContain('deny')
|
||||
expect(p.w.asked).toHaveLength(1)
|
||||
})
|
||||
|
||||
test('picker: a pick clears the mark of its target', async ($, on) => {
|
||||
const p = await bootPicker($, on, ['This conversation'])
|
||||
await route($, 'model=fable')
|
||||
await stepOn($, p.rig, FABLE)
|
||||
await failWith($, 'overloaded')
|
||||
expect(await route($, 'show')).toContain('claude-fable-5-1 for')
|
||||
await pickAndAnswer($, p, 'fable')
|
||||
expect(await route($, 'show')).toContain('down: none')
|
||||
})
|
||||
|
||||
test('picker: a PostModelSwitch of a denied pick keeps the hand pick', async (
|
||||
$, on) => {
|
||||
const p = await bootPicker($, on, ['This conversation', 'This conversation'])
|
||||
await modelCmd($, 'fable')
|
||||
const held = deferred()
|
||||
p.w.gate = held.gate
|
||||
await pick($, 'sonnet')
|
||||
await switchTo($, OPUS, SONNET, 'picker')
|
||||
held.release()
|
||||
await settle()
|
||||
expect(mainLine(await route($, 'show'))).toContain('user custom')
|
||||
})
|
||||
|
||||
// ---- hand pick: a model chosen by hand always switches --------------------
|
||||
|
||||
type HandRig = {
|
||||
w: World
|
||||
rig: Rig
|
||||
usage: { set: (n: number) => void }
|
||||
logs: string[]
|
||||
}
|
||||
|
||||
async function bootHand(
|
||||
$: Engine,
|
||||
on: On,
|
||||
answers: string[],
|
||||
session: string = OPUS,
|
||||
tokens = 1000,
|
||||
fx: Fixture = {},
|
||||
): Promise<HandRig> {
|
||||
on('command.run', { command: 'model' }, () => ({ text: 'native' }))
|
||||
const logs = recordLogs(on)
|
||||
const w = newWorld({ answers, ...fx })
|
||||
const { rig, usage } = await bootRigVar($, on, session, tokens, w)
|
||||
return { w, rig, usage, logs }
|
||||
}
|
||||
|
||||
test('hand pick: a held-down target waits across the turn boundary', async (
|
||||
$, on) => {
|
||||
const h = await bootHand($, on, ['This conversation'])
|
||||
await modelCmd($, 'fable')
|
||||
expect((await stepOn($, h.rig, OPUS)).model).toBe(FABLE)
|
||||
await failWith($, 'overloaded')
|
||||
await endTurn($)
|
||||
expect((await stepOn($, h.rig, OPUS)).model).toBe(OPUS)
|
||||
await stepOn($, h.rig, OPUS)
|
||||
expect(h.logs.filter(l => l.includes('hand pick waits'))).toHaveLength(1)
|
||||
await endTurn($)
|
||||
await h.rig.clock.advance(15 * 60000)
|
||||
expect((await stepOn($, h.rig, OPUS)).model).toBe(FABLE)
|
||||
})
|
||||
|
||||
test('hand pick: the current model down leaves it for the hand pick', async (
|
||||
$, on) => {
|
||||
const h = await bootHand($, on, ['This conversation'], OPUS, 300000,
|
||||
{ home: { mainUpgrade: false } })
|
||||
await stepOn($, h.rig, OPUS)
|
||||
await failWith($, 'overloaded')
|
||||
await endTurn($)
|
||||
await modelCmd($, 'fable')
|
||||
expect((await stepOn($, h.rig, OPUS)).model).toBe(FABLE)
|
||||
})
|
||||
|
||||
test('hand pick: an unranked session model does not trap it', async (
|
||||
$, on) => {
|
||||
const h = await bootHand($, on, ['This conversation'], 'claude-zz-9')
|
||||
await modelCmd($, 'opus')
|
||||
expect((await stepOn($, h.rig, 'claude-zz-9')).model).toBe(OPUS)
|
||||
})
|
||||
|
||||
test('hand pick: over the upgrade cap and with the switch off still moves', async (
|
||||
$, on) => {
|
||||
const h = await bootHand($, on, ['This conversation'], OPUS, 300000,
|
||||
{ home: { mainUpgrade: false } })
|
||||
await modelCmd($, 'fable')
|
||||
expect((await stepOn($, h.rig, OPUS)).model).toBe(FABLE)
|
||||
await endTurn($)
|
||||
expect((await stepOn($, h.rig, OPUS)).model).toBe(FABLE)
|
||||
})
|
||||
|
||||
test('hand pick: over the downgrade cap with switch off still moves', async (
|
||||
$, on) => {
|
||||
const h = await bootHand($, on, ['This conversation'], FABLE, 300000)
|
||||
await route($, 'switch off')
|
||||
await modelCmd($, 'sonnet')
|
||||
expect((await stepOn($, h.rig, FABLE)).model).toBe(SONNET)
|
||||
})
|
||||
|
||||
test('hand pick: a typed /route model= keeps the caps', async ($, on) => {
|
||||
const h = await bootHand($, on, [], OPUS, 300000)
|
||||
await route($, 'model=fable')
|
||||
expect((await stepOn($, h.rig, OPUS)).model).toBe(OPUS)
|
||||
})
|
||||
|
||||
test('hand pick: the current model pinned and down falls to the next', async (
|
||||
$, on) => {
|
||||
const h = await bootHand($, on, ['This conversation'])
|
||||
await modelCmd($, 'opus')
|
||||
expect((await stepOn($, h.rig, OPUS)).model).toBe(OPUS)
|
||||
await failWith($, 'overloaded')
|
||||
const same = (await stepOn($, h.rig, OPUS)).model
|
||||
expect(same).not.toBe(OPUS)
|
||||
await endTurn($)
|
||||
expect((await stepOn($, h.rig, OPUS)).model).toBe(same)
|
||||
})
|
||||
|
||||
test('hand pick: a target that fails mid-turn is left', async ($, on) => {
|
||||
const h = await bootHand($, on, ['This conversation'])
|
||||
await modelCmd($, 'fable')
|
||||
expect((await stepOn($, h.rig, OPUS)).model).toBe(FABLE)
|
||||
await failWith($, 'overloaded')
|
||||
expect((await stepOn($, h.rig, OPUS)).model).not.toBe(FABLE)
|
||||
})
|
||||
|
||||
test('hand pick: outgrowing a known window still returns home', async (
|
||||
$, on) => {
|
||||
const h = await bootHand($, on, ['This conversation'], FABLE, 1000,
|
||||
{ home: { windows: { [HAIKU]: 200000 } } })
|
||||
await modelCmd($, 'haiku')
|
||||
expect((await stepOn($, h.rig, FABLE)).model).toBe(HAIKU)
|
||||
h.usage.set(250000)
|
||||
expect((await stepOn($, h.rig, FABLE)).model).toBe(FABLE)
|
||||
})
|
||||
|
||||
test('hand pick: a native user switch drops it', async ($, on) => {
|
||||
const h = await bootHand($, on, ['This conversation'])
|
||||
await modelCmd($, 'fable')
|
||||
await switchTo($, OPUS, SONNET, 'picker')
|
||||
expect((await stepOn($, h.rig, SONNET)).model).toBe(SONNET)
|
||||
expect(mainLine(await route($, 'show'))).toContain('session defaults')
|
||||
})
|
||||
Reference in new issue
Block a user