feat(model-router): wave 4-B — prompt classifier
Plain composer prompts with no route in force get a turn-scoped effort from $.model.classify (design/analysis/coding/bookkeeping/chat → phase effort, effort only, never the model). Bounded by a 3 s clock race, a user interrupt neither counts nor writes, three failures in a row pause it for cooldownMinutes (one toast, status marker, cleared by /route reload). No dialog, no routing.json write on a classified route. New Config key `classify` (default on, pickSwitch), `/route classify on|off` session toggle, `/route show` line and argumentHint. An explicit user /effort beats the guess (engineBase check). Docs: README classifier paragraph, USAGE row + sentence, CHANGELOG, effort-shift.md. Kit suite 428 tests.
This commit is contained in:
bchanot
@@ -4377,3 +4377,622 @@ test('logs: an ended agent forgets its answer marks (reused id)', async (
|
||||
await runStep($, { ...highStep('a1'), model: SONNET })
|
||||
expect(answers(logs)).toHaveLength(2)
|
||||
})
|
||||
|
||||
// ---- wave 4-B: prompt classifier -----------------------------------------
|
||||
|
||||
type Ask = { text: string; labels: readonly string[]; options: unknown }
|
||||
type Reply = (n: number) => Promise<string | undefined> | string | undefined
|
||||
type Cls = {
|
||||
asks: Ask[]
|
||||
seen: Seen[]
|
||||
clock: MockClock
|
||||
logs: string[]
|
||||
toasts: string[]
|
||||
status: string[]
|
||||
world: World
|
||||
}
|
||||
type ClsOpts = {
|
||||
reply?: Reply
|
||||
model?: string
|
||||
world?: World
|
||||
verbose?: boolean
|
||||
}
|
||||
|
||||
const PLAIN = 'refactor the billing module please'
|
||||
const PAUSE_MS = 15 * 60000 // cooldownMinutes default, in ms
|
||||
|
||||
/** A ui.status recorder, same shape as recordLogs. */
|
||||
function recordStatus(on: On): string[] {
|
||||
const lines: string[] = []
|
||||
on('ui.status', async ($, e, next) => {
|
||||
lines.push(e.text ?? '')
|
||||
try {
|
||||
await next(e)
|
||||
} catch {
|
||||
// nothing implements ui.status below the recorder in the kit
|
||||
}
|
||||
})
|
||||
return lines
|
||||
}
|
||||
|
||||
/** A bottom `model.classify` answering `reply`; records every ask. */
|
||||
async function bootCls($: Engine, on: On, o: ClsOpts = {}): Promise<Cls> {
|
||||
const logs = recordLogs(on)
|
||||
const toasts = recordToasts(on)
|
||||
const status = recordStatus(on)
|
||||
const seen: Seen[] = []
|
||||
const asks: Ask[] = []
|
||||
recordSteps(on, seen)
|
||||
on('prompt.submit', ($, e) => ({ text: e.text }))
|
||||
on('turn.complete', () => ({ text: '' }))
|
||||
on('session.end', ($, e) => ({ sessionId: e.sessionId }))
|
||||
on('tool.call', { tool: 'Skill' }, () => ({
|
||||
result: { success: true, commandName: 'bottom' },
|
||||
}))
|
||||
on('agent.spawn', ($, e) => ({
|
||||
model: e.model ?? e.parentModel,
|
||||
agentId: 'a1',
|
||||
}))
|
||||
on('model.classify', async (_$, e) => {
|
||||
asks.push({ text: e.text, labels: e.labels, options: e.options })
|
||||
return { value: await (o.reply ?? (() => 'coding'))(asks.length) }
|
||||
})
|
||||
const world = o.world ?? newWorld()
|
||||
const clock = await boot($, on, o.model ?? FABLE, 1000, world)
|
||||
if (o.verbose) await route($, 'verbose on')
|
||||
logs.splice(0)
|
||||
return { asks, seen, clock, logs, toasts, status, world }
|
||||
}
|
||||
|
||||
const clearSession = ($: Engine) => $.session.end({
|
||||
reason: 'clear',
|
||||
sessionId: 's1',
|
||||
resume: { id: 's1' },
|
||||
})
|
||||
|
||||
/** The effort a main step reaches the bottom with (engine asks high). */
|
||||
async function clsEffort($: Engine, c: Cls): Promise<unknown> {
|
||||
await runStep($, highStep())
|
||||
return c.seen[c.seen.length - 1]?.effort
|
||||
}
|
||||
|
||||
const never = (): Promise<never> => new Promise(() => undefined)
|
||||
const boom = (): never => {
|
||||
throw new Error('overloaded')
|
||||
}
|
||||
|
||||
/** `n` typed prompts that each fail, a turn end between them. */
|
||||
async function failTimes($: Engine, n: number): Promise<void> {
|
||||
for (let i = 0; i < n; i++) {
|
||||
await typed($, PLAIN)
|
||||
await endTurn($)
|
||||
}
|
||||
}
|
||||
|
||||
const LABELS_EFFORT: [string, string][] = [
|
||||
['design', 'xhigh'],
|
||||
['analysis', 'high'],
|
||||
['coding', 'medium'],
|
||||
['bookkeeping', 'low'],
|
||||
]
|
||||
|
||||
test('classify: one call with the five labels and no model option', async (
|
||||
$, on) => {
|
||||
const c = await bootCls($, on)
|
||||
await typed($, PLAIN)
|
||||
expect(c.asks).toEqual([{
|
||||
text: PLAIN,
|
||||
labels: ['design', 'analysis', 'coding', 'bookkeeping', 'chat'],
|
||||
options: undefined,
|
||||
}])
|
||||
})
|
||||
|
||||
for (const [label, effort] of LABELS_EFFORT) {
|
||||
test(`classify: ${label} becomes the turn effort ${effort}`, async (
|
||||
$, on) => {
|
||||
const c = await bootCls($, on, { reply: () => label })
|
||||
await runStep($, highStep())
|
||||
await typed($, PLAIN)
|
||||
expect(await clsEffort($, c)).toBe(effort)
|
||||
})
|
||||
}
|
||||
|
||||
test('classify: coding lowers the engine effort to medium', async ($, on) => {
|
||||
const c = await bootCls($, on)
|
||||
await typed($, PLAIN)
|
||||
expect(await clsEffort($, c)).toBe('medium')
|
||||
})
|
||||
|
||||
test('classify: a chat answer routes nothing', async ($, on) => {
|
||||
const c = await bootCls($, on, { reply: () => 'chat' })
|
||||
await typed($, PLAIN)
|
||||
expect(await clsEffort($, c)).toBe('high')
|
||||
})
|
||||
|
||||
test('classify: no label at all routes nothing', async ($, on) => {
|
||||
const c = await bootCls($, on, { reply: () => undefined })
|
||||
await typed($, PLAIN)
|
||||
expect(await clsEffort($, c)).toBe('high')
|
||||
})
|
||||
|
||||
test('classify: a label naming an object key routes nothing', async (
|
||||
$, on) => {
|
||||
const c = await bootCls($, on, { reply: () => 'constructor' })
|
||||
await typed($, PLAIN)
|
||||
expect(await clsEffort($, c)).toBe('high')
|
||||
})
|
||||
|
||||
test('classify: an unknown label neither warns nor counts', async ($, on) => {
|
||||
const c = await bootCls($, on, { reply: () => 'nonsense' })
|
||||
await failTimes($, 4)
|
||||
expect(c.asks).toHaveLength(4)
|
||||
expect(c.toasts).toEqual([])
|
||||
expect(c.logs).toEqual([])
|
||||
})
|
||||
|
||||
test('classify: the main model never moves for a design guess', async (
|
||||
$, on) => {
|
||||
const c = await bootCls($, on, { model: SONNET, reply: () => 'design' })
|
||||
await typed($, PLAIN)
|
||||
await runStep($, { ...highStep(), model: SONNET })
|
||||
expect(c.seen[c.seen.length - 1]).toEqual({ model: SONNET, effort: 'xhigh' })
|
||||
})
|
||||
|
||||
test('classify: the route carries no tier', async ($, on) => {
|
||||
const c = await bootCls($, on, { reply: () => 'design' })
|
||||
await typed($, PLAIN)
|
||||
const line = mainLine(await route($, 'show'))
|
||||
expect(line).toContain('classifier plan')
|
||||
expect(line).not.toContain('[tier')
|
||||
expect(c.asks).toHaveLength(1)
|
||||
})
|
||||
|
||||
test('classify: the route is gone at turn.complete', async ($, on) => {
|
||||
const c = await bootCls($, on, { reply: () => 'design' })
|
||||
await typed($, PLAIN)
|
||||
await endTurn($)
|
||||
expect(await clsEffort($, c)).toBe('high')
|
||||
expect(mainLine(await route($, 'show'))).toContain('session defaults')
|
||||
})
|
||||
|
||||
test('classify: a route tool call in the turn replaces it', async ($, on) => {
|
||||
const c = await bootCls($, on, { reply: () => 'bookkeeping' })
|
||||
await typed($, PLAIN)
|
||||
await $.tool.call({ tool: ROUTE_TOOL, phase: 'plan' })
|
||||
expect(await clsEffort($, c)).toBe('xhigh')
|
||||
})
|
||||
|
||||
test('classify: a typed skill in the turn replaces it', async ($, on) => {
|
||||
const c = await bootCls($, on, { reply: () => 'design' })
|
||||
await typed($, PLAIN)
|
||||
await typed($, '/feat add a thing')
|
||||
await skillPrompt($, 'feat')
|
||||
expect(await clsEffort($, c)).toBe('high')
|
||||
})
|
||||
|
||||
test('classify: a user /effort change beats the guessed effort', async (
|
||||
$, on) => {
|
||||
const c = await bootCls($, on, { reply: () => 'design' })
|
||||
await runStep($, highStep())
|
||||
await typed($, PLAIN)
|
||||
await runStep($, { ...highStep(), effort: 'low' as const })
|
||||
expect(c.seen[c.seen.length - 1]?.effort).toBe('low')
|
||||
})
|
||||
|
||||
test('classify: the baseline survives /clear', async ($, on) => {
|
||||
const c = await bootCls($, on, { reply: () => 'design' })
|
||||
await runStep($, highStep())
|
||||
await clearSession($)
|
||||
await typed($, PLAIN)
|
||||
await runStep($, { ...highStep(), effort: 'low' as const })
|
||||
expect(c.seen[c.seen.length - 1]?.effort).toBe('low')
|
||||
})
|
||||
|
||||
test('classify: a baseline-effort step takes the guessed effort', async (
|
||||
$, on) => {
|
||||
const c = await bootCls($, on, { reply: () => 'design' })
|
||||
await runStep($, highStep())
|
||||
await typed($, PLAIN)
|
||||
await runStep($, highStep())
|
||||
expect(c.seen[c.seen.length - 1]?.effort).toBe('xhigh')
|
||||
})
|
||||
|
||||
// ---- the guards: one clause each ---
|
||||
|
||||
/** The prompt is typed, then the asks are counted. */
|
||||
async function asksFor($: Engine, c: Cls, text: string): Promise<number> {
|
||||
await typed($, text)
|
||||
return c.asks.length
|
||||
}
|
||||
|
||||
test('classify guard: text under 12 characters is not sent', async (
|
||||
$, on) => {
|
||||
const c = await bootCls($, on)
|
||||
expect(await asksFor($, c, 'hello world')).toBe(0)
|
||||
})
|
||||
|
||||
test('classify guard: 12 characters are sent', async ($, on) => {
|
||||
const c = await bootCls($, on)
|
||||
expect(await asksFor($, c, 'hello worlds')).toBe(1)
|
||||
})
|
||||
|
||||
test('classify guard: padded short text is not sent', async ($, on) => {
|
||||
const c = await bootCls($, on)
|
||||
expect(await asksFor($, c, ' short ')).toBe(0)
|
||||
})
|
||||
|
||||
test('classify guard: a padded slash prompt is not sent', async ($, on) => {
|
||||
const c = await bootCls($, on)
|
||||
expect(await asksFor($, c, ' /feat add a long thing')).toBe(0)
|
||||
})
|
||||
|
||||
test('classify guard: the sample is sent trimmed', async ($, on) => {
|
||||
const c = await bootCls($, on)
|
||||
await typed($, ` ${PLAIN} `)
|
||||
expect(c.asks.map(a => a.text)).toEqual([PLAIN])
|
||||
})
|
||||
|
||||
test('classify guard: a slash prompt is not sent', async ($, on) => {
|
||||
const c = await bootCls($, on)
|
||||
expect(await asksFor($, c, '/unknown with a long argument')).toBe(0)
|
||||
})
|
||||
|
||||
test('classify guard: a non-composer origin is not sent', async ($, on) => {
|
||||
const c = await bootCls($, on)
|
||||
await $.prompt.submit({ text: PLAIN, wait: false, origin: { kind: 'sdk' } })
|
||||
expect(c.asks).toHaveLength(0)
|
||||
})
|
||||
|
||||
test('classify guard: a mid-turn prompt is not sent', async ($, on) => {
|
||||
const c = await bootCls($, on)
|
||||
await queued($, 'composer', PLAIN)
|
||||
expect(c.asks).toHaveLength(0)
|
||||
})
|
||||
|
||||
test('classify guard: a keyword rule match is not sent', async ($, on) => {
|
||||
const c = await bootCls($, on)
|
||||
expect(await asksFor($, c, 'please review the billing module')).toBe(0)
|
||||
})
|
||||
|
||||
test('classify guard: ultrathink is not sent', async ($, on) => {
|
||||
const c = await bootCls($, on)
|
||||
expect(await asksFor($, c, 'ultrathink about the billing module')).toBe(0)
|
||||
})
|
||||
|
||||
test('classify guard: a sticky /route is not overridden', async ($, on) => {
|
||||
const c = await bootCls($, on)
|
||||
await route($, 'mechanical')
|
||||
expect(await asksFor($, c, PLAIN)).toBe(0)
|
||||
})
|
||||
|
||||
test('classify guard: a held run slot is not sent', async ($, on) => {
|
||||
const c = await bootCls($, on)
|
||||
await typed($, '/feat add a thing')
|
||||
await skillPrompt($, 'feat')
|
||||
await endTurn($)
|
||||
expect(await asksFor($, c, PLAIN)).toBe(0)
|
||||
})
|
||||
|
||||
test('classify guard: a carried floor is not sent', async ($, on) => {
|
||||
const c = await bootCls($, on)
|
||||
await queued($, 'composer', 'ultrathink about the billing module')
|
||||
await endTurn($)
|
||||
expect(await asksFor($, c, PLAIN)).toBe(0)
|
||||
})
|
||||
|
||||
test('classify guard: a sub-agent never reaches the classifier', async (
|
||||
$, on) => {
|
||||
const c = await bootCls($, on)
|
||||
await $.agent.spawn(spawnInput())
|
||||
await runStep($, highStep('a1'))
|
||||
expect(c.asks).toHaveLength(0)
|
||||
})
|
||||
|
||||
test('classify guard: a config classify=false is not sent', async ($, on) => {
|
||||
const c = await bootCls($, on, {
|
||||
world: newWorld({ home: { classify: false } }),
|
||||
})
|
||||
expect(await asksFor($, c, PLAIN)).toBe(0)
|
||||
})
|
||||
|
||||
test('classify guard: /route classify off is not sent', async ($, on) => {
|
||||
const c = await bootCls($, on)
|
||||
await route($, 'classify off')
|
||||
expect(await asksFor($, c, PLAIN)).toBe(0)
|
||||
})
|
||||
|
||||
test('classify guard: the mod off is not sent', async ($, on) => {
|
||||
const c = await bootCls($, on)
|
||||
await route($, 'off')
|
||||
expect(await asksFor($, c, PLAIN)).toBe(0)
|
||||
})
|
||||
|
||||
test('classify guard: a haiku id marked down is not sent', async ($, on) => {
|
||||
const c = await bootCls($, on)
|
||||
await switchTo($, HAIKU, SONNET, 'auto')
|
||||
expect(await asksFor($, c, PLAIN)).toBe(0)
|
||||
})
|
||||
|
||||
test('classify guard: a main loop on haiku is not sent', async ($, on) => {
|
||||
const c = await bootCls($, on, { model: HAIKU })
|
||||
expect(await asksFor($, c, PLAIN)).toBe(0)
|
||||
})
|
||||
|
||||
test('classify guard: a phase without effort routes nothing', async (
|
||||
$, on) => {
|
||||
const world = newWorld({ edit: f => {
|
||||
(f.phases as Rec).plan = { tier: 'best' }
|
||||
} })
|
||||
const c = await bootCls($, on, { reply: () => 'design', world })
|
||||
await typed($, PLAIN)
|
||||
expect(await clsEffort($, c)).toBe('high')
|
||||
})
|
||||
|
||||
test('classify guard: a non-best skill turn leaves the next open', async (
|
||||
$, on) => {
|
||||
const c = await bootCls($, on)
|
||||
await typed($, '/status now')
|
||||
await skillPrompt($, 'status')
|
||||
await endTurn($)
|
||||
expect(await asksFor($, c, PLAIN)).toBe(1)
|
||||
})
|
||||
|
||||
test('classify guard: /route clear reopens it after a best-tier run', async (
|
||||
$, on) => {
|
||||
const c = await bootCls($, on)
|
||||
await typed($, '/feat add a thing')
|
||||
await skillPrompt($, 'feat')
|
||||
await endTurn($)
|
||||
await route($, 'clear')
|
||||
expect(await asksFor($, c, PLAIN)).toBe(1)
|
||||
})
|
||||
|
||||
test('classify guard: a route landing during the call wins', async (
|
||||
$, on) => {
|
||||
let release: () => void = () => undefined
|
||||
const gate = new Promise<void>(resolve => { release = resolve })
|
||||
const c = await bootCls($, on, {
|
||||
reply: async () => { await gate; return 'design' },
|
||||
})
|
||||
const pending = typed($, PLAIN)
|
||||
await c.clock.settle()
|
||||
await $.tool.call({ tool: ROUTE_TOOL, phase: 'mechanical' })
|
||||
release()
|
||||
await pending
|
||||
expect(mainLine(await route($, 'show'))).toContain('model mechanical')
|
||||
expect(await clsEffort($, c)).toBe('low')
|
||||
})
|
||||
|
||||
// ---- bounded and fail-safe ---
|
||||
|
||||
test('classify bound: a call never answering gives no route at 3 s', async (
|
||||
$, on) => {
|
||||
const c = await bootCls($, on, { reply: never })
|
||||
const pending = typed($, PLAIN)
|
||||
await c.clock.settle()
|
||||
await c.clock.advance(3000)
|
||||
await pending
|
||||
expect(await clsEffort($, c)).toBe('high')
|
||||
})
|
||||
|
||||
test('classify bound: the prompt proceeds after a timeout', async ($, on) => {
|
||||
const c = await bootCls($, on, { reply: never })
|
||||
const pending = typed($, PLAIN)
|
||||
await c.clock.settle()
|
||||
await c.clock.advance(3000)
|
||||
expect((await pending).text).toBe(PLAIN)
|
||||
})
|
||||
|
||||
test('classify bound: three timeouts pause the classifier', async ($, on) => {
|
||||
const c = await bootCls($, on, { reply: never })
|
||||
for (let i = 0; i < 3; i++) {
|
||||
const pending = typed($, PLAIN)
|
||||
await c.clock.settle()
|
||||
await c.clock.advance(3000)
|
||||
await pending
|
||||
}
|
||||
expect(c.toasts).toEqual(['classifier paused 15 min (3 failures)'])
|
||||
})
|
||||
|
||||
test('classify bound: a rejection is not an error for the prompt', async (
|
||||
$, on) => {
|
||||
const c = await bootCls($, on, { reply: boom })
|
||||
await typed($, PLAIN)
|
||||
expect(c.logs.filter(l => l.includes('failed'))).toEqual([])
|
||||
})
|
||||
|
||||
test('classify fail: three rejections pause with one toast', async ($, on) => {
|
||||
const c = await bootCls($, on, { reply: boom })
|
||||
await failTimes($, 3)
|
||||
expect(c.toasts).toEqual(['classifier paused 15 min (3 failures)'])
|
||||
})
|
||||
|
||||
test('classify fail: two rejections do not pause', async ($, on) => {
|
||||
const c = await bootCls($, on, { reply: boom })
|
||||
await failTimes($, 2)
|
||||
expect(c.toasts).toEqual([])
|
||||
})
|
||||
|
||||
test('classify fail: the status line shows the pause marker', async (
|
||||
$, on) => {
|
||||
const c = await bootCls($, on, { reply: boom })
|
||||
await failTimes($, 3)
|
||||
expect(c.status[c.status.length - 1]).toContain('classifier: paused')
|
||||
})
|
||||
|
||||
test('classify fail: no call during the cooldown', async ($, on) => {
|
||||
const c = await bootCls($, on, { reply: boom })
|
||||
await failTimes($, 3)
|
||||
expect(await asksFor($, c, PLAIN)).toBe(3)
|
||||
})
|
||||
|
||||
test('classify fail: a settled answer resets the counter', async ($, on) => {
|
||||
const c = await bootCls($, on, {
|
||||
reply: n => (n === 3 ? 'chat' : boom()),
|
||||
})
|
||||
await failTimes($, 5)
|
||||
expect(c.toasts).toEqual([])
|
||||
expect(c.asks).toHaveLength(5)
|
||||
})
|
||||
|
||||
test('classify pause: still paused one ms before the cooldown ends', async (
|
||||
$, on) => {
|
||||
const c = await bootCls($, on, { reply: boom })
|
||||
await failTimes($, 3)
|
||||
await c.clock.advance(PAUSE_MS - 1)
|
||||
expect(await asksFor($, c, PLAIN)).toBe(3)
|
||||
})
|
||||
|
||||
test('classify pause: resumes when the cooldown ends', async ($, on) => {
|
||||
const c = await bootCls($, on, { reply: boom })
|
||||
await failTimes($, 3)
|
||||
await c.clock.advance(PAUSE_MS)
|
||||
expect(await asksFor($, c, PLAIN)).toBe(4)
|
||||
})
|
||||
|
||||
test('classify pause: the marker clears when it ends', async ($, on) => {
|
||||
const c = await bootCls($, on, { reply: boom })
|
||||
await failTimes($, 3)
|
||||
await c.clock.advance(PAUSE_MS)
|
||||
await typed($, '/status')
|
||||
expect(c.status[c.status.length - 1]).not.toContain('paused')
|
||||
})
|
||||
|
||||
test('classify pause: /clear keeps it', async ($, on) => {
|
||||
const c = await bootCls($, on, { reply: boom })
|
||||
await failTimes($, 3)
|
||||
await clearSession($)
|
||||
expect(await asksFor($, c, PLAIN)).toBe(3)
|
||||
})
|
||||
|
||||
test('classify pause: /route reload clears it', async ($, on) => {
|
||||
const c = await bootCls($, on, { reply: boom })
|
||||
await failTimes($, 3)
|
||||
await route($, 'reload')
|
||||
expect(await asksFor($, c, PLAIN)).toBe(4)
|
||||
})
|
||||
|
||||
// ---- sample ---
|
||||
|
||||
test('classify sample: 2000 code points are sent whole', async ($, on) => {
|
||||
const c = await bootCls($, on)
|
||||
const text = 'é'.repeat(2000)
|
||||
await typed($, text)
|
||||
expect(c.asks[0]?.text).toBe(text)
|
||||
})
|
||||
|
||||
test('classify sample: a longer text is cut to head and tail', async (
|
||||
$, on) => {
|
||||
const c = await bootCls($, on)
|
||||
await typed($, 'H'.repeat(600) + 'M'.repeat(1000) + 'T'.repeat(1500))
|
||||
expect(c.asks[0]?.text).toBe('H'.repeat(500) + 'T'.repeat(1500))
|
||||
})
|
||||
|
||||
test('classify sample: a cut never leaves a lone surrogate', async (
|
||||
$, on) => {
|
||||
const c = await bootCls($, on)
|
||||
await typed($, '😀'.repeat(2400))
|
||||
const sent = c.asks[0]?.text ?? ''
|
||||
expect(sent.isWellFormed()).toBe(true)
|
||||
expect(Array.from(sent)).toHaveLength(2000)
|
||||
})
|
||||
|
||||
// ---- no dialog, config, visibility ---
|
||||
|
||||
test('classify: no dialog and no file write on a classified prompt', async (
|
||||
$, on) => {
|
||||
const c = await bootCls($, on, { world: newWorld({ ask: true }) })
|
||||
await typed($, PLAIN)
|
||||
expect(c.asks).toHaveLength(1)
|
||||
expect(c.world.asked).toEqual([])
|
||||
expect(c.world.written).toEqual([])
|
||||
})
|
||||
|
||||
test('classify config: an invalid value keeps the default on', async (
|
||||
$, on) => {
|
||||
const c = await bootCls($, on, {
|
||||
world: newWorld({ home: { classify: 'yes' } }),
|
||||
})
|
||||
expect(await asksFor($, c, PLAIN)).toBe(1)
|
||||
})
|
||||
|
||||
test('classify config: an invalid value is logged once', async ($, on) => {
|
||||
const c = await bootCls($, on, {
|
||||
world: newWorld({ home: { classify: 'yes' } }),
|
||||
})
|
||||
await route($, 'reload')
|
||||
const said = c.logs.filter(l => l.includes('"classify" is not a boolean'))
|
||||
expect(said).toHaveLength(1)
|
||||
})
|
||||
|
||||
test('classify config: /route show prints classify: on', async ($, on) => {
|
||||
await bootCls($, on)
|
||||
expect(await route($, 'show')).toContain('classify: on')
|
||||
})
|
||||
|
||||
test('classify config: /route classify off prints classify: off', async (
|
||||
$, on) => {
|
||||
await bootCls($, on)
|
||||
expect(await route($, 'classify off')).toContain('classify: off')
|
||||
})
|
||||
|
||||
test('classify config: a bad toggle argument prints the usage', async (
|
||||
$, on) => {
|
||||
await bootCls($, on)
|
||||
expect(await route($, 'classify maybe')).toBe('usage: /route classify on|off')
|
||||
})
|
||||
|
||||
test('classify config: the toggle survives a rebuild', async ($, on) => {
|
||||
const c = await bootCls($, on)
|
||||
await route($, 'classify off')
|
||||
expect(await route($, 'reload')).toContain('classify: off')
|
||||
expect(await asksFor($, c, PLAIN)).toBe(0)
|
||||
})
|
||||
|
||||
test('classify config: /route is refused from a plugin origin', async (
|
||||
$, on) => {
|
||||
await bootCls($, on)
|
||||
const out = await $.command.run({
|
||||
command: 'route',
|
||||
args: 'classify off',
|
||||
origin: { kind: 'plugin', name: 'x' },
|
||||
presentation: { isFullscreen: false, columns: 80 },
|
||||
})
|
||||
expect(out.text).toContain('user-only')
|
||||
})
|
||||
|
||||
test('classify visibility: the status line reads classifier <phase>', async (
|
||||
$, on) => {
|
||||
const c = await bootCls($, on, { reply: () => 'analysis' })
|
||||
await typed($, PLAIN)
|
||||
expect(c.status[c.status.length - 1]).toContain('route: classifier reflect')
|
||||
})
|
||||
|
||||
test('classify visibility: verbose says (reflect: classifier)', async (
|
||||
$, on) => {
|
||||
const c = await bootCls($, on, { reply: () => 'analysis', verbose: true })
|
||||
await runStep($, { ...highStep(), effort: 'low' as const })
|
||||
await typed($, PLAIN)
|
||||
await runStep($, { ...highStep(), effort: 'low' as const })
|
||||
expect(c.logs).toContain(
|
||||
mainWord('fable/low', 'fable/high', 'reflect: classifier'))
|
||||
})
|
||||
|
||||
test('classify: a repeated undefined label never pauses', async ($, on) => {
|
||||
const c = await bootCls($, on, { reply: () => undefined })
|
||||
await failTimes($, 3)
|
||||
expect(c.asks).toHaveLength(3)
|
||||
expect(c.toasts).toEqual([])
|
||||
expect(c.status.some(l => l.includes('paused'))).toBe(false)
|
||||
})
|
||||
|
||||
test('classify: /route advertises classify on|off in its hint', async (
|
||||
$, on) => {
|
||||
const hints: unknown[] = []
|
||||
on('command.register', (_$, e) => {
|
||||
hints.push(e.argumentHint)
|
||||
return { value: { command: e.name } }
|
||||
})
|
||||
await bootCls($, on)
|
||||
expect(hints.join('\n')).toContain('classify on|off')
|
||||
})
|
||||
|
||||
@@ -32,11 +32,13 @@ type Config = {
|
||||
upgradeMaxTokens: number // above this context an upgrade is skipped
|
||||
mainModelSwitch: boolean // gates DOWNGRADES only
|
||||
verbose: boolean
|
||||
classify: boolean // a small-model guess of the typed prompt's phase
|
||||
spinner: boolean
|
||||
enabled: boolean // false: every hook passes through (per machine)
|
||||
}
|
||||
type Rule = { re: RegExp; phase: string; mode: PromptMode }
|
||||
type Source = 'user' | 'model' | 'skill' | 'prompt' | 'run' | 'derived'
|
||||
type Source =
|
||||
| 'user' | 'model' | 'skill' | 'prompt' | 'run' | 'derived' | 'classifier'
|
||||
type Routed = {
|
||||
phase: string
|
||||
route: Route
|
||||
@@ -91,13 +93,21 @@ type State = {
|
||||
override: Rec | undefined // the machine override's data, if it loaded
|
||||
loaded: boolean // the layers were read once (else: lazily, on first use)
|
||||
rebuildDue: boolean // /clear: re-read the layers at the next prompt
|
||||
toggles: { switch?: boolean; verbose?: boolean } // session toggles
|
||||
toggles: Partial<Record<Toggle, boolean>> // session toggles
|
||||
engineBase: NonNullable<Effort> | null // engine effort of main's first step
|
||||
classifyFails: number // classifier failures in a row
|
||||
classifyUntil: number // classifier paused until (ms); 0 = running
|
||||
asking: string | null // key of the one first-use dialog in flight
|
||||
asked: Set<string> // keys answered (or failed) this session
|
||||
writes: Promise<unknown> // the serialized routing.json write chain
|
||||
}
|
||||
type Log = (text: string) => void
|
||||
type StepIn = Readonly<TurnStepInput>
|
||||
type PromptIn = {
|
||||
text: string
|
||||
turnId?: string
|
||||
origin: { kind: string }
|
||||
}
|
||||
type Plan = { model: string; effort: StepIn['effort'] }
|
||||
type RouteInput = {
|
||||
phase?: unknown
|
||||
@@ -107,6 +117,7 @@ type RouteInput = {
|
||||
}
|
||||
type Picked = { phase: string; route: Route }
|
||||
type Effort = StepIn['effort']
|
||||
type Toggle = 'switch' | 'verbose' | 'classify'
|
||||
type EffortBy = 'floor' | 'sticky' | 'turn' | 'engine'
|
||||
type Decision = { effort: Effort; by: EffortBy }
|
||||
type Rec = Record<string, unknown>
|
||||
@@ -176,6 +187,23 @@ const UNAVAILABLE: ReadonlySet<string> = new Set([
|
||||
'rate_limit', 'overloaded', 'billing_error', 'model_not_found',
|
||||
])
|
||||
const PHASE_KEY = /^[a-z][a-z0-9_-]{0,31}$/
|
||||
// Prompt classifier: label -> the phase whose effort the turn takes. `chat`
|
||||
// (and any label not here) routes nothing.
|
||||
const CLASSIFY_LABELS: readonly string[] =
|
||||
['design', 'analysis', 'coding', 'bookkeeping', 'chat']
|
||||
const LABEL_PHASE: Readonly<Record<string, string>> = {
|
||||
design: 'plan',
|
||||
analysis: 'reflect',
|
||||
coding: 'implement',
|
||||
bookkeeping: 'apply',
|
||||
}
|
||||
const CLASSIFY_MIN = 12 // code points of a typed prompt worth a call
|
||||
const CLASSIFY_MS = 3000 // the call is abandoned after this
|
||||
const CLASSIFY_MAX = 2000 // code points sent whole; longer: head + tail
|
||||
const CLASSIFY_HEAD = 500
|
||||
const CLASSIFY_TAIL = 1500
|
||||
const CLASSIFY_FAILS = 3 // failures in a row that pause the classifier
|
||||
const TIMEOUT: unique symbol = Symbol('classify timeout')
|
||||
|
||||
const DEFAULT_CONFIG: Config = {
|
||||
models: {
|
||||
@@ -232,6 +260,7 @@ const DEFAULT_CONFIG: Config = {
|
||||
],
|
||||
mainModelSwitch: false,
|
||||
verbose: false,
|
||||
classify: true,
|
||||
spinner: true,
|
||||
enabled: true,
|
||||
}
|
||||
@@ -512,6 +541,9 @@ function mergeConfig(
|
||||
mainModelSwitch: pickBool(user.mainModelSwitch, base.mainModelSwitch),
|
||||
verbose: pickSwitch(
|
||||
'config "verbose" is not a boolean; ignored', user.verbose, base.verbose, log),
|
||||
classify: pickSwitch(
|
||||
'model-router: config "classify" is not a boolean; ignored',
|
||||
user.classify, base.classify, log),
|
||||
spinner: pickBool(user.spinner, base.spinner),
|
||||
enabled: pickSwitch(
|
||||
'model-router: config "enabled" is not a boolean; ignored',
|
||||
@@ -845,6 +877,9 @@ function newState(cfg: Config, source: string): State {
|
||||
loaded: false,
|
||||
rebuildDue: false,
|
||||
toggles: {},
|
||||
engineBase: null,
|
||||
classifyFails: 0,
|
||||
classifyUntil: 0,
|
||||
asking: null,
|
||||
asked: new Set(),
|
||||
writes: Promise.resolve(),
|
||||
@@ -853,7 +888,8 @@ function newState(cfg: Config, source: string): State {
|
||||
|
||||
/**
|
||||
* /clear rebuilds the state but keeps what is account- or process-wide:
|
||||
* the breaker (a model's quota outlives the conversation), the model, the
|
||||
* the breaker, the classifier's pause and effort baseline (a model's quota
|
||||
* and the user's own /effort outlive the conversation), the model, the
|
||||
* agent definitions' sources (the listing is not offered again), the config
|
||||
* with its session toggles and the dialog in flight. The layers are read
|
||||
* again at the next prompt; the config is swapped only if that read works.
|
||||
@@ -870,6 +906,9 @@ function resetSession(st: State): void {
|
||||
override: st.override,
|
||||
loaded: st.loaded,
|
||||
toggles: st.toggles,
|
||||
engineBase: st.engineBase,
|
||||
classifyFails: st.classifyFails,
|
||||
classifyUntil: st.classifyUntil,
|
||||
asking: st.asking,
|
||||
writes: st.writes,
|
||||
}
|
||||
@@ -1204,11 +1243,12 @@ function higherFloor(cur: Routed | null, next: Routed): Routed {
|
||||
* The one decision of the main loop's effort. The user's floor is the turn's
|
||||
* default (no sticky or turn route names an effort) and its minimum.
|
||||
* `by` names who set the value: the floor when it raised or supplied it.
|
||||
* `skipTurn`: the turn route yields (a classified effort under a user /effort).
|
||||
*/
|
||||
function mainEffort(st: State, engine: Effort): Decision {
|
||||
function mainEffort(st: State, engine: Effort, skipTurn = false): Decision {
|
||||
const sticky = st.userMain?.route.effort
|
||||
const named = sticky ?? st.turnMain?.route.effort ??
|
||||
st.runMain?.route.effort
|
||||
const turn = skipTurn ? undefined : st.turnMain?.route.effort
|
||||
const named = sticky ?? turn ?? st.runMain?.route.effort
|
||||
const floor = st.turnFloor?.route.effort
|
||||
const base = named ?? floor ?? engine
|
||||
const effort = floored(base, floor)
|
||||
@@ -1369,7 +1409,8 @@ async function show($: Api, st: State): Promise<string> {
|
||||
mainText(st, s),
|
||||
`router: ${routerWord(st)} · switch: ${flag(c.mainModelSwitch)} ` +
|
||||
`(downgrade) · upgrade: ${flag(c.mainUpgrade)} · verbose: ${
|
||||
flag(c.verbose)} · spinner: ${flag(c.spinner)}`,
|
||||
flag(c.verbose)} · classify: ${flag(c.classify)} · spinner: ${
|
||||
flag(c.spinner)}`,
|
||||
downText(st, s.now),
|
||||
`live loops: ${st.loops.size}`,
|
||||
phasesText(st),
|
||||
@@ -1383,8 +1424,10 @@ function statusLine(st: State): string {
|
||||
? routerWord(st)
|
||||
: r ? `${r.source} ${r.phase}` : 'session defaults'
|
||||
const floor = liveFloor(st)
|
||||
const paused = st.cfg.classify && st.classifyUntil !== 0
|
||||
return `route: ${now}${floor ? ` · floor ${floor.level}` : ''}${
|
||||
st.cfg.mainModelSwitch ? ' · switch on' : ''}`
|
||||
st.cfg.mainModelSwitch ? ' · switch on' : ''}${
|
||||
paused ? ' · classifier: paused' : ''}`
|
||||
}
|
||||
|
||||
const refresh = ($: Api, st: State): void => $.ui.status(statusLine(st))
|
||||
@@ -1431,12 +1474,11 @@ function parseRoute(cfg: Config, args: string): Routed | string {
|
||||
async function toggle(
|
||||
$: Api,
|
||||
st: State,
|
||||
what: string,
|
||||
what: Toggle,
|
||||
arg: string | undefined,
|
||||
): Promise<string> {
|
||||
if (arg !== 'on' && arg !== 'off') return `usage: /route ${what} on|off`
|
||||
if (what === 'switch') st.toggles.switch = arg === 'on'
|
||||
else st.toggles.verbose = arg === 'on'
|
||||
st.toggles[what] = arg === 'on'
|
||||
applyToggles(st)
|
||||
return show($, st)
|
||||
}
|
||||
@@ -1451,9 +1493,10 @@ async function setUserRoute($: Api, st: State, args: string) {
|
||||
|
||||
/** Session toggles outlive any rebuild of the config. */
|
||||
function applyToggles(st: State): void {
|
||||
const { switch: downgrade, verbose } = st.toggles
|
||||
const { switch: downgrade, verbose, classify } = st.toggles
|
||||
if (downgrade !== undefined) st.cfg.mainModelSwitch = downgrade
|
||||
if (verbose !== undefined) st.cfg.verbose = verbose
|
||||
if (classify !== undefined) st.cfg.classify = classify
|
||||
}
|
||||
|
||||
function applyLoaded(st: State, loaded: Loaded, log: Log): void {
|
||||
@@ -1497,6 +1540,8 @@ async function ensureConfig($: Api, st: State, atPrompt = false) {
|
||||
/** The breaker is cleared first, whatever the config read then does. */
|
||||
async function reloadCommand($: Api, st: State): Promise<string> {
|
||||
clearBreaker(st)
|
||||
st.classifyFails = 0
|
||||
st.classifyUntil = 0
|
||||
await reloadConfig($, st)
|
||||
refresh($, st)
|
||||
return 'config reloaded\n' + (await show($, st))
|
||||
@@ -1533,6 +1578,7 @@ async function handleCommand($: Api, st: State, args: string): Promise<string> {
|
||||
return forgetCommand($, st, rest)
|
||||
case 'switch':
|
||||
case 'verbose':
|
||||
case 'classify':
|
||||
return toggle($, st, head, rest[0])
|
||||
default:
|
||||
return setUserRoute($, st, args)
|
||||
@@ -2414,7 +2460,8 @@ async function registerCommand($: Api): Promise<void> {
|
||||
argumentHint:
|
||||
'[show|clear|off|on|reload|pending|ask on|off|' +
|
||||
'forget <name|all|projects>|<phase>|' +
|
||||
'model=<alias|id> effort=<level>|switch on|off|verbose on|off]',
|
||||
'model=<alias|id> effort=<level>|switch on|off|verbose on|off|' +
|
||||
'classify on|off]',
|
||||
immediate: true,
|
||||
})
|
||||
} catch (err) {
|
||||
@@ -2770,6 +2817,7 @@ function slotWord(slot: Routed | null): string {
|
||||
case 'model': return 'route tool'
|
||||
case 'prompt': return 'prompt rule'
|
||||
case 'derived': return 'dispatch'
|
||||
case 'classifier': return 'classifier'
|
||||
case 'skill':
|
||||
case 'run': return 'skill' + label
|
||||
default: return 'session'
|
||||
@@ -2921,13 +2969,27 @@ function agentPlan(st: State, e: StepIn): Plan {
|
||||
type Why = { by: EffortBy; call: Call; cur: string }
|
||||
type Step = { plan: Plan; why?: Why }
|
||||
|
||||
/** The engine effort of main's first step: what the user left as default. */
|
||||
function noteEngineBase(st: State, e: StepIn): void {
|
||||
if (st.engineBase === null && e.effort !== undefined) {
|
||||
st.engineBase = e.effort
|
||||
}
|
||||
}
|
||||
|
||||
/** A user /effort beats a guessed phase: the engine effort left its base. */
|
||||
function userMovedEffort(st: State, e: StepIn): boolean {
|
||||
return st.turnMain?.source === 'classifier' && e.effort !== undefined &&
|
||||
st.engineBase !== null && e.effort !== st.engineBase
|
||||
}
|
||||
|
||||
/**
|
||||
* The main loop's plan: effort from the floor/sticky/turn decision, model
|
||||
* from `decideMain`. `cur` is the model the router moved this turn, else
|
||||
* the engine's own (verbatim: an engine fallback is respected).
|
||||
*/
|
||||
async function mainPlan($: Api, st: State, e: StepIn): Promise<Step> {
|
||||
const { effort, by } = mainEffort(st, e.effort)
|
||||
noteEngineBase(st, e)
|
||||
const { effort, by } = mainEffort(st, e.effort, userMovedEffort(st, e))
|
||||
await prune($, st)
|
||||
const cur = st.turnModel ?? e.model
|
||||
const { call } = await decideFor($, st, cur)
|
||||
@@ -3309,6 +3371,126 @@ function noteTyped(st: State, kind: string, text: string, midTurn: boolean) {
|
||||
}
|
||||
}
|
||||
|
||||
// ---- prompt classifier -----------------------------------------------
|
||||
// A typed prompt no rule, skill, route or floor covers is guessed by one
|
||||
// small-model call: the phase's EFFORT becomes the turn's default (never a
|
||||
// model move, never a dialog, never a file write). The call is bounded and
|
||||
// every failure leaves the prompt as it was.
|
||||
|
||||
/** The whole text up to CLASSIFY_MAX code points, else its head and tail. */
|
||||
function sampleOf(text: string): string {
|
||||
const points = Array.from(text.trim())
|
||||
if (points.length <= CLASSIFY_MAX) return points.join('')
|
||||
return points.slice(0, CLASSIFY_HEAD).join('') +
|
||||
points.slice(-CLASSIFY_TAIL).join('')
|
||||
}
|
||||
|
||||
/** No route in force on main and the classifier is on and not paused. */
|
||||
const classifierIdle = (st: State): boolean =>
|
||||
!st.off && st.cfg.classify && st.classifyUntil === 0 &&
|
||||
mainRoute(st) === null && st.turnFloor === null
|
||||
|
||||
/** A lapsed pause ends here, before any other guard (no stale marker). */
|
||||
async function expirePause($: Api, st: State): Promise<void> {
|
||||
if (st.classifyUntil === 0 || (await $.clock.now()) < st.classifyUntil) {
|
||||
return
|
||||
}
|
||||
st.classifyUntil = 0
|
||||
refresh($, st)
|
||||
}
|
||||
|
||||
/** The small model is the haiku id: marked down, or main already runs it. */
|
||||
async function smallModelBusy($: Api, st: State): Promise<boolean> {
|
||||
await prune($, st)
|
||||
const small = st.cfg.models.haiku
|
||||
const main = st.turnModel ?? st.sessionModel
|
||||
return (small !== undefined && st.down.has(small)) || isHaiku(st.cfg, main)
|
||||
}
|
||||
|
||||
/** The guards of D8, in order; the text itself is checked last. */
|
||||
async function classifierWanted($: Api, st: State, e: PromptIn) {
|
||||
if (!st.cfg.classify || e.origin.kind !== 'composer') return false
|
||||
if (e.turnId !== undefined) return false
|
||||
await expirePause($, st)
|
||||
const text = e.text.trim()
|
||||
if (text.startsWith('/') || Array.from(text).length < CLASSIFY_MIN) {
|
||||
return false
|
||||
}
|
||||
return classifierIdle(st) && !(await smallModelBusy($, st))
|
||||
}
|
||||
|
||||
/** One failure; the third in a row pauses the classifier, said once. */
|
||||
async function countFailure($: Api, st: State): Promise<void> {
|
||||
st.classifyFails += 1
|
||||
if (st.classifyFails < CLASSIFY_FAILS) return
|
||||
st.classifyFails = 0
|
||||
st.classifyUntil = (await $.clock.now()) + st.cfg.cooldownMinutes * 60000
|
||||
$.ui.toast(`classifier paused ${st.cfg.cooldownMinutes} min (${
|
||||
CLASSIFY_FAILS} failures)`)
|
||||
refresh($, st)
|
||||
}
|
||||
|
||||
/** classify's own abort cause: a user interrupt, not a failure. */
|
||||
const isAbort = (err: unknown): boolean =>
|
||||
err instanceof Error && (err.name === 'AbortError' ||
|
||||
/\baborted\b/i.test(err.message))
|
||||
|
||||
/** The label, or TIMEOUT; rejects like classify or when the dispatch aborts. */
|
||||
function raceClassify($: Api, text: string, signal: AbortSignal) {
|
||||
return Promise.race([
|
||||
$.model.classify(sampleOf(text), CLASSIFY_LABELS),
|
||||
$.clock.sleep(CLASSIFY_MS, { signal })
|
||||
.then((): typeof TIMEOUT => TIMEOUT),
|
||||
])
|
||||
}
|
||||
|
||||
/** The label, or undefined: no answer, an interrupt or a counted failure. */
|
||||
async function askClassifier(
|
||||
$: Api,
|
||||
st: State,
|
||||
text: string,
|
||||
signal: AbortSignal,
|
||||
): Promise<string | undefined> {
|
||||
let label: string | undefined | typeof TIMEOUT
|
||||
try {
|
||||
label = await raceClassify($, text, signal)
|
||||
} catch (err) {
|
||||
if (!signal.aborted && !isAbort(err)) await countFailure($, st)
|
||||
return undefined
|
||||
}
|
||||
if (label === TIMEOUT) {
|
||||
await countFailure($, st)
|
||||
return undefined
|
||||
}
|
||||
st.classifyFails = 0
|
||||
return label
|
||||
}
|
||||
|
||||
/** The turn route of a label: its phase's effort only; false = nothing. */
|
||||
function routeFromLabel(st: State, label: string): boolean {
|
||||
const phase = hasKey(LABEL_PHASE, label) ? LABEL_PHASE[label] : undefined
|
||||
const effort = phase === undefined
|
||||
? undefined
|
||||
: phaseRoute(st.cfg, phase)?.effort
|
||||
if (phase === undefined || effort === undefined) return false
|
||||
st.turnMain = { phase, route: { effort }, source: 'classifier' }
|
||||
return true
|
||||
}
|
||||
|
||||
/** Fills an empty route with the guessed phase's effort; never throws on it. */
|
||||
async function classifyPrompt(
|
||||
$: Api,
|
||||
st: State,
|
||||
e: PromptIn,
|
||||
signal: AbortSignal,
|
||||
): Promise<void> {
|
||||
if (!(await classifierWanted($, st, e))) return
|
||||
const label = await askClassifier($, st, e.text, signal)
|
||||
// a route that landed meanwhile wins; a late answer never crosses turns
|
||||
if (label === undefined || signal.aborted || !classifierIdle(st)) return
|
||||
if (routeFromLabel(st, label)) refresh($, st)
|
||||
}
|
||||
|
||||
function registerPrompt(on: On, st: State): void {
|
||||
on('prompt.submit', async ($, e, next) => {
|
||||
await ensureConfig($, st, true)
|
||||
@@ -3317,6 +3499,7 @@ function registerPrompt(on: On, st: State): void {
|
||||
noteTyped(st, e.origin.kind, e.text, midTurn)
|
||||
if (e.origin.kind === 'composer') routeFromPrompt(st, e.text, midTurn)
|
||||
refresh($, st)
|
||||
await classifyPrompt($, st, e, next.signal)
|
||||
return next(e)
|
||||
}).catch(($, e, next) => {
|
||||
warnOnce(st, $, 'prompt.submit', next.error.kind)
|
||||
|
||||
Reference in New Issue
Block a user