feat(memory): log cands and rerank ms on prompt recall
This commit is contained in:
1 parent
21483dc48c
commit
efe7f78d49
2 files changed
+91
-12
No files matched your search
@@ -9,7 +9,8 @@ import {
|
|||||||
parseIds, recallLine, recallBlock, failureKey,
|
parseIds, recallLine, recallBlock, failureKey,
|
||||||
segments, suiteKeyOf, commitLabel, finishLabel,
|
segments, suiteKeyOf, commitLabel, finishLabel,
|
||||||
newState, switchOf, claim, onContextStart, resetContext, endContext,
|
newState, switchOf, claim, onContextStart, resetContext, endContext,
|
||||||
appendLog, loadLog, RECALL_CAP, isTrigger, MIN_CONTENT_WORDS, promptContext,
|
appendLog, loadLog, isLogRecord, RECALL_CAP, isTrigger,
|
||||||
|
MIN_CONTENT_WORDS, promptContext,
|
||||||
DESIGN_RULE, ANALYSIS_RULE, HAIKU, RERANK_MS, IDLE_MS,
|
DESIGN_RULE, ANALYSIS_RULE, HAIKU, RERANK_MS, IDLE_MS,
|
||||||
} from './register'
|
} from './register'
|
||||||
import type { LogRecord } from './register'
|
import type { LogRecord } from './register'
|
||||||
@@ -2412,6 +2413,72 @@ test('LO7: a pick the ledger refuses logs refused', async ($, on) => {
|
|||||||
.toEqual(['injected', 'refused'])
|
.toEqual(['injected', 'refused'])
|
||||||
})
|
})
|
||||||
|
|
||||||
|
/** [outcome, cands, ms] of every logged ask, oldest first. */
|
||||||
|
async function askMeta(w: World) {
|
||||||
|
return (await loadLog(storeApi(w), ROOT))
|
||||||
|
.map(r => [r.outcome, r.cands, r.ms])
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The single ask of a fresh world answered by `model`. */
|
||||||
|
async function oneAsk($: Engine, on: On, model?: Model) {
|
||||||
|
const { w } = await bootRecall($, on)
|
||||||
|
if (model !== undefined) w.model = model
|
||||||
|
await start($)
|
||||||
|
await say($, GO)
|
||||||
|
return (await askMeta(w))[0] ?? []
|
||||||
|
}
|
||||||
|
|
||||||
|
test('LF1: an injected ask logs cands and ms', async ($, on) => {
|
||||||
|
const [outcome, cands, ms] = await oneAsk($, on)
|
||||||
|
expect(outcome).toBe('injected')
|
||||||
|
expect(typeof cands === 'number' && cands >= 1).toBe(true)
|
||||||
|
expect(ms).toBe(0) // the mock clock does not move during an answer
|
||||||
|
})
|
||||||
|
|
||||||
|
test('LF1b: a declined ask logs cands and ms', async ($, on) => {
|
||||||
|
const [outcome, cands, ms] = await oneAsk($, on, { text: 'none' })
|
||||||
|
expect(outcome).toBe('none')
|
||||||
|
expect(typeof cands === 'number' && cands >= 1).toBe(true)
|
||||||
|
expect(ms).toBe(0)
|
||||||
|
})
|
||||||
|
|
||||||
|
test('LF1c: an unanswered ask logs cands and ms', async ($, on) => {
|
||||||
|
const [outcome, cands, ms] = await oneAsk($, on, 'unanswered')
|
||||||
|
expect(outcome).toBe('timeout')
|
||||||
|
expect(typeof cands === 'number' && cands >= 1).toBe(true)
|
||||||
|
expect(ms).toBe(0)
|
||||||
|
})
|
||||||
|
|
||||||
|
test('LF2: no candidate logs cands 0 and no ms', async ($, on) => {
|
||||||
|
const { w } = await bootRecall($, on)
|
||||||
|
w.proc = { stdout: rowsOut([['BLK-001', 'x'], ['LRN-002', 'y']])
|
||||||
|
.replace(/open/g, 'superseded') }
|
||||||
|
await start($)
|
||||||
|
await say($, GO)
|
||||||
|
expect(await askMeta(w)).toEqual([['none', 0, undefined]])
|
||||||
|
})
|
||||||
|
|
||||||
|
test('LF3: a paused ask logs cands without ms', async ($, on) => {
|
||||||
|
const { w } = await bootRecall($, on)
|
||||||
|
w.model = 'unanswered'
|
||||||
|
await start($)
|
||||||
|
for (let i = 0; i < 4; i++) await say($, DESIGN)
|
||||||
|
const last = (await askMeta(w)).at(-1)
|
||||||
|
expect(last?.[0]).toBe('paused')
|
||||||
|
expect((last?.[1] as number) >= 1).toBe(true)
|
||||||
|
expect(last?.[2]).toBeUndefined()
|
||||||
|
})
|
||||||
|
|
||||||
|
test('LF4: cands and ms are optional counts', async ($, on) => {
|
||||||
|
const good = { ...REC(2), cands: 4, ms: 812 }
|
||||||
|
const neg = { ...REC(3), cands: -1 }
|
||||||
|
const frac = { ...REC(4), ms: 1.5 }
|
||||||
|
const seed = { [`l:${ROOT}`]: [REC(1), good, neg, frac] }
|
||||||
|
const w = await boot($, on, { seed })
|
||||||
|
expect(await loadLog(storeApi(w), ROOT)).toEqual([REC(1), good])
|
||||||
|
expect(isLogRecord(REC(1))).toBe(true)
|
||||||
|
})
|
||||||
|
|
||||||
test('IR1: the first prompt of a context asks and injects the block',
|
test('IR1: the first prompt of a context asks and injects the block',
|
||||||
async ($, on) => {
|
async ($, on) => {
|
||||||
const { w, seen } = await bootRecall($, on)
|
const { w, seen } = await bootRecall($, on)
|
||||||
@@ -2569,7 +2636,7 @@ test('IR11: an injection leaves one log record; 205 keep 200', async (
|
|||||||
const { w } = await bootRecall($, on)
|
const { w } = await bootRecall($, on)
|
||||||
await start($)
|
await start($)
|
||||||
await say($, GO)
|
await say($, GO)
|
||||||
expect(await loadLog(storeApi(w), ROOT)).toEqual([
|
expect(await loadLog(storeApi(w), ROOT)).toMatchObject([
|
||||||
{ at: 1000, session: 's1', agent: 'main', channel: 'prompt',
|
{ at: 1000, session: 's1', agent: 'main', channel: 'prompt',
|
||||||
ids: ['BLK-001'], trigger: 'fresh', outcome: 'injected' },
|
ids: ['BLK-001'], trigger: 'fresh', outcome: 'injected' },
|
||||||
])
|
])
|
||||||
|
|||||||
@@ -973,7 +973,8 @@ export type Outcome = 'injected' | 'none' | 'timeout' | 'refused' | 'paused'
|
|||||||
const TRIGGERS: string[] = ['fresh', 'idle', 'keyword', 'tool']
|
const TRIGGERS: string[] = ['fresh', 'idle', 'keyword', 'tool']
|
||||||
const OUTCOMES: string[] = ['injected', 'none', 'timeout', 'refused', 'paused']
|
const OUTCOMES: string[] = ['injected', 'none', 'timeout', 'refused', 'paused']
|
||||||
|
|
||||||
// `trigger` and `outcome` are absent on records written before 4c's review.
|
// `trigger` and `outcome` are absent on records written before 4c's
|
||||||
|
// review; `cands` and `ms` on records written before 4d.1.
|
||||||
export type LogRecord = {
|
export type LogRecord = {
|
||||||
at: number
|
at: number
|
||||||
session: string
|
session: string
|
||||||
@@ -982,16 +983,23 @@ export type LogRecord = {
|
|||||||
ids: string[]
|
ids: string[]
|
||||||
trigger?: Trigger
|
trigger?: Trigger
|
||||||
outcome?: Outcome
|
outcome?: Outcome
|
||||||
|
cands?: number // prompt: BM25 candidates after the seen filter
|
||||||
|
ms?: number // prompt: rerank wall time, when a rerank ran
|
||||||
}
|
}
|
||||||
|
|
||||||
// shortcut: one `l:<root>` key per project (about 28 KiB at 200 records),
|
// shortcut: one `l:<root>` key per project (about 28 KiB at 200 records),
|
||||||
// swept by `sweepLogs` against the engine's 4 MiB store cap, oldest key
|
// swept by `sweepLogs` against the engine's 4 MiB store cap, oldest key
|
||||||
// first; upgrade path: one global log key, or a smaller per-root cap.
|
// first; upgrade path: one global log key, or a smaller per-root cap.
|
||||||
const LOG_PREFIX = 'l:'
|
export const LOG_PREFIX = 'l:'
|
||||||
const MAX_LOG = 200
|
export const MAX_LOG = 200
|
||||||
const MAX_LOG_BYTES = 65536
|
const MAX_LOG_BYTES = 65536
|
||||||
|
|
||||||
function isLogRecord(v: unknown): v is LogRecord {
|
/** Absent, or a non-negative integer. */
|
||||||
|
const isCount = (v: unknown) =>
|
||||||
|
v === undefined || (Number.isInteger(v) && (v as number) >= 0)
|
||||||
|
|
||||||
|
/** A well-formed log record; the 4d reader imports it (LRN-261). */
|
||||||
|
export function isLogRecord(v: unknown): v is LogRecord {
|
||||||
const r = v as Partial<LogRecord> | null
|
const r = v as Partial<LogRecord> | null
|
||||||
return typeof r === 'object' && r !== null &&
|
return typeof r === 'object' && r !== null &&
|
||||||
typeof r.at === 'number' && typeof r.session === 'string' &&
|
typeof r.at === 'number' && typeof r.session === 'string' &&
|
||||||
@@ -999,7 +1007,8 @@ function isLogRecord(v: unknown): v is LogRecord {
|
|||||||
(r.channel === 'prompt' || r.channel === 'tool') &&
|
(r.channel === 'prompt' || r.channel === 'tool') &&
|
||||||
Array.isArray(r.ids) && r.ids.every(i => typeof i === 'string') &&
|
Array.isArray(r.ids) && r.ids.every(i => typeof i === 'string') &&
|
||||||
(r.trigger === undefined || TRIGGERS.includes(r.trigger)) &&
|
(r.trigger === undefined || TRIGGERS.includes(r.trigger)) &&
|
||||||
(r.outcome === undefined || OUTCOMES.includes(r.outcome))
|
(r.outcome === undefined || OUTCOMES.includes(r.outcome)) &&
|
||||||
|
isCount(r.cands) && isCount(r.ms)
|
||||||
}
|
}
|
||||||
|
|
||||||
const logKey = (root: string) => LOG_PREFIX + root.slice(0, MAX_ROOT)
|
const logKey = (root: string) => LOG_PREFIX + root.slice(0, MAX_ROOT)
|
||||||
@@ -1705,7 +1714,7 @@ function candidatesFor(
|
|||||||
}
|
}
|
||||||
|
|
||||||
type Ask = { at: Where; text: string; trigger: Exclude<Trigger, 'tool'> }
|
type Ask = { at: Where; text: string; trigger: Exclude<Trigger, 'tool'> }
|
||||||
type Picked = { block: string; ids: string[]; outcome: Outcome }
|
type Picked = { block: string; ids: string[]; outcome: Outcome; ms?: number }
|
||||||
|
|
||||||
const nothing = (outcome: Outcome): Picked => ({ block: '', ids: [], outcome })
|
const nothing = (outcome: Outcome): Picked => ({ block: '', ids: [], outcome })
|
||||||
|
|
||||||
@@ -1736,14 +1745,16 @@ async function pickRecall(
|
|||||||
const now = await $.clock.now()
|
const now = await $.clock.now()
|
||||||
if (now < st.rerank.until) return nothing('paused')
|
if (now < st.rerank.until) return nothing('paused')
|
||||||
const answer = await askRerank($, st, text, cands, now)
|
const answer = await askRerank($, st, text, cands, now)
|
||||||
if (answer === undefined) return nothing('timeout')
|
// a clock step back must not make the record invalid (isCount)
|
||||||
|
const ms = Math.max(0, Math.round((await $.clock.now()) - now))
|
||||||
|
if (answer === undefined) return { ...nothing('timeout'), ms }
|
||||||
const picked = picksOf(answer, cands)
|
const picked = picksOf(answer, cands)
|
||||||
if (picked.length === 0) return nothing('none')
|
if (picked.length === 0) return { ...nothing('none'), ms }
|
||||||
const taken = claimPicks(st, key, picked)
|
const taken = claimPicks(st, key, picked)
|
||||||
const block = recallBlock(
|
const block = recallBlock(
|
||||||
picked.filter(e => taken.includes(e.id)).map(recallLine), RECALL_CAP)
|
picked.filter(e => taken.includes(e.id)).map(recallLine), RECALL_CAP)
|
||||||
if (block === '') return nothing('refused')
|
if (block === '') return { ...nothing('refused'), ms }
|
||||||
return { block, ids: taken, outcome: 'injected' }
|
return { block, ids: taken, outcome: 'injected', ms }
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -1764,6 +1775,7 @@ async function recallFor(
|
|||||||
const record = {
|
const record = {
|
||||||
at: await $.clock.now(), session: ask.at.id, agent: 'main',
|
at: await $.clock.now(), session: ask.at.id, agent: 'main',
|
||||||
ids: got.ids, trigger: ask.trigger, outcome: got.outcome,
|
ids: got.ids, trigger: ask.trigger, outcome: got.outcome,
|
||||||
|
cands: cands.length, ...(got.ms === undefined ? {} : { ms: got.ms }),
|
||||||
}
|
}
|
||||||
await appendLog($, st, ask.at.root, { ...record, channel: 'prompt' })
|
await appendLog($, st, ask.at.root, { ...record, channel: 'prompt' })
|
||||||
.catch(() => undefined)
|
.catch(() => undefined)
|
||||||
|
|||||||
Reference in new issue
Block a user