forked from bchanot/claude
feat(model-router): close the cap residuals: session pin for inheriting spawns, prompt-size estimate, engine cache TTL
A built-in general-purpose agent with no explicit model, spawned by main
while main sits below the session model, is sent the session model
instead of inheriting the demoted one (narrow: built-in source, no fork,
no workflow, main's own child; other types carry their own frontmatter
model the hook cannot see). The cold-read cap now counts the prompt the
engine's usage may not include yet: code points / 3 of the submitted
text, its context entries and main's typed skill body, summed, added at
the turn's first main step unless usage already grew by half of it;
a document attachment makes the size unknown (blocks). Texts and the
step read the same adjusted size; the served record stays raw. The
cache TTL follows the engine's cache_ttl ('5m' | '1h') from main's
PostModelSwitch events, survives /clear, and /route show prints it.
This commit is contained in:
1 parent
ab963e97dc
commit
1be57a382c
2 files changed
+396
-25
No files matched your search
@@ -56,6 +56,12 @@ type Loop = {
|
||||
last?: Plan // the plan this loop last ran, once it was told
|
||||
cause?: string // who set the effort in force, for the log lines
|
||||
}
|
||||
/** A submitted prompt's size where `session.usage` may not count it yet. */
|
||||
type Estimate = {
|
||||
tokens: number
|
||||
usage: number | undefined // context size read when the first entry came
|
||||
unknown: boolean // carries a non-text item: the size cannot be guessed
|
||||
}
|
||||
type State = {
|
||||
cfg: Config
|
||||
rules: Rule[]
|
||||
@@ -101,6 +107,9 @@ type State = {
|
||||
turnEngine: NonNullable<Effort> | null // same, for this turn
|
||||
served: Map<string, Served> // canonical id -> main's last step on it
|
||||
capSaid: boolean // the downgrade skip was logged for this cap crossing
|
||||
cacheTtlMs: number // the prompt-cache TTL a cold read is measured against
|
||||
promptEstimate: Estimate | null // the submitted prompt, not in usage yet
|
||||
pendingEstimate: Estimate | null // same, typed mid-turn: next turn's
|
||||
classifyFails: number // classifier failures in a row
|
||||
classifyUntil: number // classifier paused until (ms); 0 = running
|
||||
asking: string | null // key of the one first-use dialog in flight
|
||||
@@ -111,6 +120,8 @@ type Log = (text: string) => void
|
||||
type StepIn = Readonly<TurnStepInput>
|
||||
type PromptIn = {
|
||||
text: string
|
||||
attachments?: readonly { type: string }[]
|
||||
context?: readonly string[]
|
||||
turnId?: string
|
||||
origin: { kind: string }
|
||||
}
|
||||
@@ -155,6 +166,11 @@ const TYPED_ORIGINS: ReadonlySet<string> = new Set([
|
||||
const PROJECT_SOURCES: ReadonlySet<string> = new Set([
|
||||
'projectSettings', 'localSettings',
|
||||
])
|
||||
// Built-in agent types with no frontmatter model: an undefined `model` makes
|
||||
// them inherit the spawning loop's model.
|
||||
// shortcut: the frontmatter model of other types is invisible to the hook;
|
||||
// widen the set when the engine exposes it on agent.offer.
|
||||
const INHERITING_AGENTS: ReadonlySet<string> = new Set(['general-purpose'])
|
||||
// PostModelSwitch sources a person chose (auto and resume are not).
|
||||
const USER_SWITCH: ReadonlySet<string> = new Set([
|
||||
'command', 'picker', 'sdk',
|
||||
@@ -219,9 +235,16 @@ const CLASSIFY_MAX = 2000 // code points sent whole; longer: head + tail
|
||||
const CLASSIFY_HEAD = 500
|
||||
const CLASSIFY_TAIL = 1500
|
||||
const CLASSIFY_FAILS = 3 // failures in a row that pause the classifier
|
||||
// The account's prompt-cache TTL (1 h here; an engine-reported `cache_ttl`
|
||||
// would supersede it, TODO): a model that served main within it is warm.
|
||||
// The account's prompt-cache TTL: a model that served main within it is
|
||||
// warm. 1 h default; replaced by the engine's `cache_ttl` at the first
|
||||
// model-switch event.
|
||||
const CACHE_TTL_MS = 60 * 60000
|
||||
// The engine's two `cache_ttl` values (a closed enum), in ms.
|
||||
const CACHE_TTLS: Readonly<Record<string, number>> = {
|
||||
'5m': 5 * 60000,
|
||||
'1h': 60 * 60000,
|
||||
}
|
||||
const ESTIMATE_DIVISOR = 3 // code points per token, conservative
|
||||
const TIMEOUT: unique symbol = Symbol('classify timeout')
|
||||
|
||||
const DEFAULT_CONFIG: Config = {
|
||||
@@ -959,6 +982,9 @@ function newState(cfg: Config, source: string): State {
|
||||
turnEngine: null,
|
||||
served: new Map(),
|
||||
capSaid: false,
|
||||
cacheTtlMs: CACHE_TTL_MS,
|
||||
promptEstimate: null,
|
||||
pendingEstimate: null,
|
||||
classifyFails: 0,
|
||||
classifyUntil: 0,
|
||||
asking: null,
|
||||
@@ -988,6 +1014,7 @@ function resetSession(st: State): void {
|
||||
loaded: st.loaded,
|
||||
toggles: st.toggles,
|
||||
engineBase: st.engineBase,
|
||||
cacheTtlMs: st.cacheTtlMs,
|
||||
classifyFails: st.classifyFails,
|
||||
classifyUntil: st.classifyUntil,
|
||||
asking: st.asking,
|
||||
@@ -1185,7 +1212,7 @@ function coldRead(
|
||||
): number {
|
||||
const key = canonical(st.cfg, id)
|
||||
const hit = st.served.get(key)
|
||||
if (hit === undefined || now - hit.at > CACHE_TTL_MS) return tokens
|
||||
if (hit === undefined || now - hit.at > st.cacheTtlMs) return tokens
|
||||
if (tokens < hit.tokens) {
|
||||
st.served.delete(key)
|
||||
return tokens
|
||||
@@ -1366,6 +1393,60 @@ async function readTokens($: Api): Promise<number | undefined> {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The context size a cap reads: `raw` plus the pending prompt's estimate,
|
||||
* unless usage already grew by half of it (the prompt was counted). Pure:
|
||||
* the step and every text read the same figure. Unknown = blocked.
|
||||
*/
|
||||
function withEstimate(st: State, raw: number | undefined) {
|
||||
const est = st.promptEstimate
|
||||
if (raw === undefined || est === null) return raw
|
||||
if (est.unknown) return undefined
|
||||
const counted = est.usage !== undefined &&
|
||||
raw - est.usage >= est.tokens / 2
|
||||
return counted ? raw : raw + est.tokens
|
||||
}
|
||||
|
||||
/** Sums an entry into a pending estimate; the first baseline is kept. */
|
||||
function sumEstimate(
|
||||
cur: Estimate | null,
|
||||
tokens: number,
|
||||
usage: number | undefined,
|
||||
unknown: boolean,
|
||||
): Estimate {
|
||||
return {
|
||||
tokens: (cur?.tokens ?? 0) + tokens,
|
||||
usage: cur === null ? usage : cur.usage,
|
||||
unknown: (cur?.unknown ?? false) || unknown,
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds a submitted text (and a flag for a non-text item) to the pending
|
||||
* estimate. A mid-turn entry waits for the next turn, where its baseline is
|
||||
* reset; the first entry of a turn reads the usage as its baseline.
|
||||
*/
|
||||
async function addEstimate(
|
||||
$: Api,
|
||||
st: State,
|
||||
size: { points: number; unknown: boolean },
|
||||
midTurn: boolean,
|
||||
): Promise<void> {
|
||||
const tokens = Math.ceil(size.points / ESTIMATE_DIVISOR)
|
||||
if (midTurn) {
|
||||
st.pendingEstimate = sumEstimate(
|
||||
st.pendingEstimate, tokens, undefined, size.unknown)
|
||||
return
|
||||
}
|
||||
const usage = st.promptEstimate === null ? await readTokens($) : undefined
|
||||
st.promptEstimate = sumEstimate(
|
||||
st.promptEstimate, tokens, usage, size.unknown)
|
||||
}
|
||||
|
||||
/** Code points of a text, as the estimate counts them. */
|
||||
const pointsOf = (texts: readonly string[]): number =>
|
||||
texts.reduce((n, t) => n + Array.from(t).length, 0)
|
||||
|
||||
type Verdict = { call: Call; wanted: string | undefined }
|
||||
|
||||
/**
|
||||
@@ -1392,7 +1473,8 @@ async function readingFor(
|
||||
const needs = wanted !== undefined || st.down.size > 0 ||
|
||||
st.turnModel !== undefined
|
||||
if (!needs) return { tokens: undefined, now: 0 }
|
||||
return { tokens: await readTokens($), now: await $.clock.now() }
|
||||
const raw = await readTokens($)
|
||||
return { tokens: withEstimate(st, raw), now: await $.clock.now() }
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -1602,8 +1684,9 @@ async function snapshot($: Api, st: State) {
|
||||
}
|
||||
|
||||
/** The downgrade switch and, when on, the cold read it is capped at. */
|
||||
const switchWord = (c: Config): string => c.mainModelSwitch
|
||||
? `on (downgrade, cap ${c.downgradeMaxTokens} tokens cold)`
|
||||
const switchWord = (st: State): string => st.cfg.mainModelSwitch
|
||||
? `on (downgrade, cap ${st.cfg.downgradeMaxTokens} tokens cold, ttl ${
|
||||
Math.round(st.cacheTtlMs / 60000)} min)`
|
||||
: 'off (downgrade)'
|
||||
|
||||
async function show($: Api, st: State): Promise<string> {
|
||||
@@ -1612,7 +1695,7 @@ async function show($: Api, st: State): Promise<string> {
|
||||
const s = await snapshot($, st)
|
||||
return [
|
||||
mainText(st, s),
|
||||
`router: ${routerWord(st)} · switch: ${switchWord(c)} · upgrade: ${
|
||||
`router: ${routerWord(st)} · switch: ${switchWord(st)} · upgrade: ${
|
||||
flag(c.mainUpgrade)} · verbose: ${
|
||||
flag(c.verbose)} · classify: ${flag(c.classify)} · spinner: ${
|
||||
flag(c.spinner)}`,
|
||||
@@ -2837,23 +2920,28 @@ function typedPath(st: State, skill: string): TypedPath | undefined {
|
||||
return st.promptAllowed && idle ? 'typed-fallback' : undefined
|
||||
}
|
||||
|
||||
/** Routes main by the typed skill's row; the row, for the first-use ask. */
|
||||
function armTypedSkill($: Api, st: State, skill: string) {
|
||||
type Armed = { row: Picked | undefined; typed: boolean }
|
||||
|
||||
/**
|
||||
* Routes main by the typed skill's row; the row, for the first-use ask, and
|
||||
* whether the user typed the skill (rowed or not: its body is theirs).
|
||||
*/
|
||||
function armTypedSkill($: Api, st: State, skill: string): Armed {
|
||||
const row = skillRow(st, skill)
|
||||
const path = typedPath(st, skill)
|
||||
if (row === undefined || path === undefined) return undefined
|
||||
const typed = typedPath(st, skill) !== undefined
|
||||
if (row === undefined || !typed) return { row: undefined, typed }
|
||||
routeMainBySkill(st, skill, row, true)
|
||||
refresh($, st)
|
||||
return row
|
||||
return { row, typed }
|
||||
}
|
||||
|
||||
/** Bookkeeping that may fail without breaking the hook: fail-open. */
|
||||
function armed($: Api, st: State, skill: string): Picked | undefined {
|
||||
function armed($: Api, st: State, skill: string): Armed {
|
||||
try {
|
||||
return armTypedSkill($, st, skill)
|
||||
} catch {
|
||||
warnOnce(st, $, 'skill.prompt', 'bookkeeping')
|
||||
return undefined
|
||||
return { row: undefined, typed: false }
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2913,9 +3001,31 @@ function inTier($: Api, st: State, agent: string, tier: string) {
|
||||
return undefined
|
||||
}
|
||||
|
||||
/**
|
||||
* A built-in inheriting type spawned by main while main sits below the
|
||||
* session model (a demotion or a breaker fallback) would inherit that lower
|
||||
* model: it starts on the session model instead. Forks and workflows are
|
||||
* frozen, a nested spawn inherits an agent's model, never main's, and a
|
||||
* shadowing project or user definition may carry its own frontmatter model.
|
||||
*/
|
||||
function sessionPin(st: State, e: SpawnIn): string | undefined {
|
||||
const turn = st.turnModel
|
||||
const inherits = INHERITING_AGENTS.has(e.subagentType) &&
|
||||
st.offers.get(e.subagentType) === 'built-in'
|
||||
const main = !e.fork && e.workflow === undefined &&
|
||||
e.parentAgentId === undefined
|
||||
if (turn === undefined || st.sessionModel === '' || !inherits || !main) {
|
||||
return undefined
|
||||
}
|
||||
const session = canonical(st.cfg, st.sessionModel)
|
||||
const demoted = ranksAbove(st, session, canonical(st.cfg, turn))
|
||||
return demoted && !st.down.has(session) ? st.sessionModel : undefined
|
||||
}
|
||||
|
||||
/** The model a spawn is rewritten to; an explicit `model` param wins. */
|
||||
function spawnTarget($: Api, st: State, e: SpawnIn, route: Route | undefined) {
|
||||
if (e.model !== undefined || route === undefined) return undefined
|
||||
if (e.model !== undefined) return undefined
|
||||
if (route === undefined) return sessionPin(st, e)
|
||||
if (route.tier === undefined) return resolveRoute(st, route)
|
||||
return inTier($, st, e.subagentType, route.tier)
|
||||
}
|
||||
@@ -3145,6 +3255,7 @@ type Handoff = { to: Plan; modelCause: string; effortCause: string }
|
||||
function spawnModelCause(st: State, sp: Spawned): string {
|
||||
const { e, route, wanted } = sp
|
||||
if (e.model !== undefined) return 'explicit'
|
||||
if (route === undefined && wanted !== undefined) return 'session'
|
||||
if (route === undefined || wanted === undefined) return 'frontmatter'
|
||||
const head = headAlias(st.cfg, route)
|
||||
const lead = head === undefined ? undefined : st.cfg.models[head]
|
||||
@@ -3161,7 +3272,9 @@ function handoffOf(
|
||||
given: Level | undefined,
|
||||
): Handoff | undefined {
|
||||
const { e, route, started } = sp
|
||||
if (route === undefined && e.model === undefined && given === undefined) {
|
||||
const pinned = sp.wanted !== undefined
|
||||
if (route === undefined && e.model === undefined && given === undefined &&
|
||||
!pinned) {
|
||||
return undefined
|
||||
}
|
||||
const effort = isHaiku(st.cfg, started.model)
|
||||
@@ -3248,11 +3361,15 @@ async function mainPlan($: Api, st: State, e: StepIn): Promise<Step> {
|
||||
const { effort, by } = mainEffort(st, e.effort)
|
||||
await prune($, st)
|
||||
const cur = st.turnModel ?? e.model
|
||||
const read = { tokens: await readTokens($), now: await $.clock.now() }
|
||||
const { call } = await decideFor($, st, cur, read)
|
||||
const raw = await readTokens($)
|
||||
const now = await $.clock.now()
|
||||
const { call } = await decideFor(
|
||||
$, st, cur, { tokens: withEstimate(st, raw), now })
|
||||
logCall($, st, capLogged(st, call))
|
||||
if (call.moved) st.turnModel = call.model
|
||||
noteServed(st, { model: call.model, effort }, read)
|
||||
// the record stays raw: an estimate in it would trip the compaction path
|
||||
noteServed(st, { model: call.model, effort }, { tokens: raw, now })
|
||||
st.promptEstimate = null // consumed by the turn's first main step
|
||||
return { plan: { model: call.model, effort }, why: { by, call, cur } }
|
||||
}
|
||||
|
||||
@@ -3278,7 +3395,21 @@ function noteMain($: Api, st: State, plan: Plan): void {
|
||||
safely(st, $, 'turn.step status', () => refresh($, st))
|
||||
}
|
||||
|
||||
/**
|
||||
* An estimate no step consumed, and one typed mid-turn, carry to the next
|
||||
* turn with their baseline reset: added at its first step, never dropped.
|
||||
*/
|
||||
function carryEstimate(st: State): void {
|
||||
const left = [st.promptEstimate, st.pendingEstimate]
|
||||
.filter((x): x is Estimate => x !== null)
|
||||
const sum = left.reduce<Estimate | null>(
|
||||
(acc, x) => sumEstimate(acc, x.tokens, undefined, x.unknown), null)
|
||||
st.promptEstimate = sum
|
||||
st.pendingEstimate = null
|
||||
}
|
||||
|
||||
function endMainTurn($: Api, st: State): void {
|
||||
carryEstimate(st)
|
||||
st.turnFloor = st.pendingPrompt
|
||||
st.pendingPrompt = null
|
||||
st.turnMain = null
|
||||
@@ -3327,8 +3458,14 @@ type SwitchIn = {
|
||||
to_model: string
|
||||
requested_model: string | null
|
||||
source: string
|
||||
cache_ttl?: unknown
|
||||
agent_id?: unknown // set when a sub-agent's model switched
|
||||
}
|
||||
|
||||
/** The engine's `cache_ttl` in ms, or undefined for any other value. */
|
||||
const ttlOf = (v: unknown): number | undefined =>
|
||||
typeof v === 'string' && hasKey(CACHE_TTLS, v) ? CACHE_TTLS[v] : undefined
|
||||
|
||||
/**
|
||||
* The engine switched the model by itself. The model it left is marked, one
|
||||
* strike, unless it landed where the router already was; the mark targets
|
||||
@@ -3347,9 +3484,15 @@ async function onAutoSwitch($: Api, st: State, e: SwitchIn): Promise<void> {
|
||||
markDown($, st, target, 'engine fallback', await $.clock.now())
|
||||
}
|
||||
|
||||
/** Any model change: keeps `sessionModel`; a user choice clears its mark. */
|
||||
/**
|
||||
* Any model change: keeps `sessionModel` and the cache TTL (main's own
|
||||
* switches only); a user choice clears its mark.
|
||||
*/
|
||||
async function onModelSwitch($: Api, st: State, e: SwitchIn): Promise<void> {
|
||||
st.sessionModel = e.to_model
|
||||
if (e.agent_id === undefined) {
|
||||
st.sessionModel = e.to_model
|
||||
st.cacheTtlMs = ttlOf(e.cache_ttl) ?? st.cacheTtlMs
|
||||
}
|
||||
if (USER_SWITCH.has(e.source)) st.runMain = null
|
||||
if (st.off || e.source === 'resume') return
|
||||
if (e.source === 'auto') await onAutoSwitch($, st, e)
|
||||
@@ -3462,7 +3605,11 @@ function registerSkills(on: On, st: State): void {
|
||||
on('skill.prompt', async ($, e, next) => {
|
||||
await ensureConfig($, st)
|
||||
if (st.off || st.skillCalls > 0) return next(e)
|
||||
const row = armed($, st, e.skill)
|
||||
const { row, typed } = armed($, st, e.skill)
|
||||
if (typed) {
|
||||
const size = { points: pointsOf([e.text]), unknown: false }
|
||||
await addEstimate($, st, size, false)
|
||||
}
|
||||
if (row) await confirmSkill($, st, e.skill, row)
|
||||
return next(e)
|
||||
}).catch(($, e, next) => {
|
||||
@@ -3755,12 +3902,21 @@ async function classifyPrompt(
|
||||
if (routeFromLabel(st, label)) refresh($, st)
|
||||
}
|
||||
|
||||
/** What a prompt adds to the context: its text, context blocks, items. */
|
||||
function promptSize(e: PromptIn): { points: number; unknown: boolean } {
|
||||
return {
|
||||
points: pointsOf([e.text, ...(e.context ?? [])]),
|
||||
unknown: (e.attachments ?? []).some(a => a.type === 'document'),
|
||||
}
|
||||
}
|
||||
|
||||
function registerPrompt(on: On, st: State): void {
|
||||
on('prompt.submit', async ($, e, next) => {
|
||||
await ensureConfig($, st, true)
|
||||
if (st.off) return next(e)
|
||||
const midTurn = e.turnId !== undefined
|
||||
noteTyped(st, e.origin.kind, e.text, midTurn)
|
||||
await addEstimate($, st, promptSize(e), midTurn)
|
||||
if (e.origin.kind === 'composer') routeFromPrompt(st, e.text, midTurn)
|
||||
refresh($, st)
|
||||
await classifyPrompt($, st, e, next.signal)
|
||||
|
||||
Reference in new issue
Block a user