diff --git a/mods/memory/hooks/register.test.ts b/mods/memory/hooks/register.test.ts index 9c369b6..bf9d79f 100644 --- a/mods/memory/hooks/register.test.ts +++ b/mods/memory/hooks/register.test.ts @@ -2149,6 +2149,8 @@ const storeApi = (w: World) => ({ store: { w.sets.push(k) w.kv.set(k, JSON.parse(JSON.stringify(v))) }, + keys: async () => [...w.kv.keys()], + delete: async (k: string) => void w.kv.delete(k), } }) as never const REC = (at: number, ids = ['BLK-001']): LogRecord => @@ -2198,6 +2200,56 @@ test('LG3: over 64 KiB serialized, the oldest records are dropped', expect(JSON.stringify(log).length).toBeLessThanOrEqual(65536) }) +test('LG5: a record without trigger and outcome stays valid', async ( + $, on, +) => { + const bad = { ...REC(3), trigger: 'x' } + const odd = { ...REC(4), outcome: 'x' } + const ok = { ...REC(5), trigger: 'idle', outcome: 'none' } + const seed = { [`l:${ROOT}`]: [REC(1), bad, odd, ok] } + const w = await boot($, on, { seed }) + expect(await loadLog(storeApi(w), ROOT)).toEqual([REC(1), ok]) +}) + +/** A raw log of `n` records of about `kb` KiB, the newest at `at`. */ +const bigLog = (at: number, kb: number) => + [REC(at, ['X'.repeat(kb * 1024)])] + +test('LG6: over the engine cap, the oldest l: key goes first', async ( + $, on, +) => { + const seed: Record = {} + for (let i = 1; i <= 68; i++) seed[`l:/r${i}`] = bigLog(i * 10, 60) + const w = await boot($, on, { seed }) + await failing($, w, EXIT1 + 'fatal: bad thing') + expect(w.kv.has('l:/r1')).toBe(false) + expect(w.kv.has('l:/r68')).toBe(true) + expect(failuresOf(w)).toHaveLength(1) + const bytes = [...w.kv.values()].reduce( + (n: number, v) => n + JSON.stringify(v).length, 0) + expect(bytes).toBeLessThanOrEqual(4194304) +}) + +test('LG7: the current root log is never evicted while others exist', + async ($, on) => { + const seed: Record = { [`l:${ROOT}`]: bigLog(1, 60) } + for (let i = 1; i <= 68; i++) seed[`l:/r${i}`] = bigLog(i * 10, 60) + const w = await boot($, on, { seed }) + await failing($, w, EXIT1 + 'fatal: bad thing') + expect(w.kv.has(`l:${ROOT}`)).toBe(true) + expect(w.kv.has('l:/r1')).toBe(false) + }) + +test('LG8: a new log key sweeps too, before its own write', async ( + $, on, +) => { + const w = await boot($, on) + for (let i = 1; i <= 68; i++) w.kv.set(`l:/r${i}`, bigLog(i * 10, 60)) + await appendLog(storeApi(w), newState(), ROOT, REC(900)) + expect(w.kv.has('l:/r1')).toBe(false) + expect(w.kv.get(`l:${ROOT}`)).toEqual([REC(900)]) +}) + test('LG4: loadLog drops malformed records; project key untouched', async ($, on) => { const bad = { at: 'x', session: 's1', agent: 'main', ids: [] } @@ -2229,7 +2281,7 @@ const REC_FILES = { [BLK]: BLK1, [LRN]: LRN2 } const REC_INDEX = rowsOut([['BLK-001', 'mod design'], ['LRN-002', 'design notes']]) -const GO = 'on va concevoir la suite du mod' +const GO = 'on va concevoir la suite du mod memory recall' const DESIGN = 'design the mod suite and tests please' const NOTES = 'write notes about the mod now' @@ -2253,6 +2305,81 @@ const say = ($: Engine, text: string, turnId?: string) => $.prompt.submit({ const BLOCK1 = `memory recall:\n${BLK1_LINE}` +/** The `trigger` and `outcome` of every logged ask, oldest first. */ +async function askLog(w: World) { + return (await loadLog(storeApi(w), ROOT)) + .map(r => [r.trigger, r.outcome, r.ids]) +} + +test('LO1: a fresh ask that injects logs fresh and injected', async ( + $, on, +) => { + const { w } = await bootRecall($, on) + await start($) + await say($, GO) + expect(await askLog(w)).toEqual([['fresh', 'injected', ['BLK-001']]]) +}) + +test('LO2: idle and keyword triggers are told apart', async ($, on) => { + const { w } = await bootRecall($, on) + await start($) + await say($, GO) + await w.clock.advance(1860000) + await say($, NOTES) + await say($, DESIGN) + expect((await askLog(w)).map(r => r[0])) + .toEqual(['fresh', 'idle', 'keyword']) +}) + +test('LO3: an answer of none logs an empty ask', async ($, on) => { + const { w, seen } = await bootRecall($, on) + w.model = { text: 'none' } + await start($) + await say($, GO) + expect(seen).toEqual([undefined]) + expect(await askLog(w)).toEqual([['fresh', 'none', []]]) +}) + +test('LO4: no candidate left logs none without asking', async ($, on) => { + const { w } = await bootRecall($, on) + w.proc = { stdout: rowsOut([['BLK-001', 'x'], ['LRN-002', 'y']]) + .replace(/open/g, 'superseded') } + await start($) + await say($, GO) + expect(w.asks).toEqual([]) + expect(await askLog(w)).toEqual([['fresh', 'none', []]]) +}) + +test('LO5: an unanswered call and a throw log timeout', async ($, on) => { + const { w } = await bootRecall($, on) + await start($) + for (const m of ['unanswered', 'throw'] as Model[]) { + w.model = m + await say($, DESIGN) + } + expect(await askLog(w)).toEqual([ + ['fresh', 'timeout', []], ['keyword', 'timeout', []], + ]) +}) + +test('LO6: an ask during the pause logs paused', async ($, on) => { + const { w } = await bootRecall($, on) + w.model = 'unanswered' + await start($) + for (let i = 0; i < 4; i++) await say($, DESIGN) + expect((await askLog(w)).map(r => r[1])) + .toEqual(['timeout', 'timeout', 'timeout', 'paused']) + expect(w.asks).toHaveLength(3) +}) + +test('LO7: a pick the ledger refuses logs refused', async ($, on) => { + const { w } = await bootRecall($, on) + await start($) + await Promise.all([say($, GO), say($, DESIGN)]) + expect((await askLog(w)).map(r => r[1]).sort()) + .toEqual(['injected', 'refused']) +}) + test('IR1: the first prompt of a context asks and injects the block', async ($, on) => { const { w, seen } = await bootRecall($, on) @@ -2412,7 +2539,7 @@ test('IR11: an injection leaves one log record; 205 keep 200', async ( await say($, GO) expect(await loadLog(storeApi(w), ROOT)).toEqual([ { at: 1000, session: 's1', agent: 'main', channel: 'prompt', - ids: ['BLK-001'] }, + ids: ['BLK-001'], trigger: 'fresh', outcome: 'injected' }, ]) for (let i = 0; i < 204; i++) { await start($, 'clear') @@ -2495,11 +2622,12 @@ test('TIMErules: both rules on 64 KiB of letters, under 50 ms', () => { }) test('isTrigger: rules, fresh, idle, slash and length', () => { + const SUITE = 'on va concevoir la suite du mod memory' const long = 'please review this module and the new tests' expect(isTrigger(long, false, 0)).toBe(true) - expect(isTrigger('on va concevoir la suite', false, 0)).toBe(false) - expect(isTrigger('on va concevoir la suite', true, 0)).toBe(true) - expect(isTrigger('on va concevoir la suite', false, IDLE_MS)).toBe(true) + expect(isTrigger(SUITE, false, 0)).toBe(false) + expect(isTrigger(SUITE, true, 0)).toBe(true) + expect(isTrigger(SUITE, false, IDLE_MS)).toBe(true) expect(isTrigger('/feat design the thing', true, IDLE_MS)).toBe(false) expect(isTrigger('design note', true, 0)).toBe(false) expect(isTrigger('un-design-ed thing here', false, 0)).toBe(false) @@ -2508,6 +2636,14 @@ test('isTrigger: rules, fresh, idle, slash and length', () => { test('isTrigger: fewer than five content words never triggers', () => { expect(MIN_CONTENT_WORDS).toBe(5) + // 2-letter function words (finding I1) are no content + for (const vague of ['ok tu peux continuer la suite stp', + 'ok je valide, on passe à la suite', 'is it ok to do it now']) { + expect(isTrigger(vague, true, 0)).toBe(false) + expect(isTrigger(vague, false, IDLE_MS)).toBe(false) + } + const topical = 'why does the recall rerank pick stale memory entries' + expect(isTrigger(topical, false, 0)).toBe(true) expect(isTrigger('oui ça me convient', true, 0)).toBe(false) expect(isTrigger('non pas de changement go', false, IDLE_MS)).toBe(false) expect(isTrigger('i approve all for all', false, IDLE_MS)).toBe(false) @@ -2733,7 +2869,7 @@ test('TE6b: an unavailable status reader injects nothing', async ( expect(out).toEqual(w.answer) }) -test('TE7: at 1448 chars spent, a 200-char line is refused', async ( +test('TE7c: at 1448 chars spent, a 200-char line is refused', async ( $, on, ) => { // four lines of 362 chars (57 title + 280 lesson) = 1448 spent diff --git a/mods/memory/hooks/register.ts b/mods/memory/hooks/register.ts index 1ac74c8..5bcc2be 100644 --- a/mods/memory/hooks/register.ts +++ b/mods/memory/hooks/register.ts @@ -401,6 +401,40 @@ async function sweepStore($: Api, curKey: string, id: string): Promise { } } +// The engine rejects every `$.store.set` once the store passes 4 MiB in all. +// The `p:` keys are swept at STORE_CAP; the `l:` keys are swept here, oldest +// first, so the two together stay under the engine cap and capture lives. +const ENGINE_CAP = 4194304 +const ENGINE_MARGIN = 131072 // room for the next session and log write + +type Weighed = { key: string; size: number; at: number } + +/** A store key's bytes and, for a log, the `at` of its newest record. */ +async function weigh($: Api, key: string): Promise { + const raw = await $.store.get(key) + const size = JSON.stringify(raw ?? null).length + const times = Array.isArray(raw) ? raw.filter(isLogRecord).map(r => r.at) : [] + return { key, size, at: Math.max(0, ...times) } +} + +/** + * Deletes the oldest `l:` keys, by their newest record, while the `p:` + * and `l:` keys together pass the engine cap. Never the `keep` key. + */ +async function sweepLogs($: Api, keep: string): Promise { + const ours = (await $.store.keys()) + .filter(k => k.startsWith(STORE_PREFIX) || k.startsWith(LOG_PREFIX)) + const all = await Promise.all(ours.map(k => weigh($, k))) + let total = all.reduce((n, w) => n + w.size, 0) + const logs = all.filter(w => w.key.startsWith(LOG_PREFIX) && w.key !== keep) + .sort((a, b) => a.at - b.at) + for (const w of logs) { + if (total <= ENGINE_CAP - ENGINE_MARGIN) break + await $.store.delete(w.key) + total -= w.size + } +} + // The registry error index: which captured failures a registry knows. const MAX_REGISTRY = 4194304 const MIN_SPAN = 12 @@ -639,11 +673,16 @@ async function statusMap($: Api, root: string): Promise { } } +// Function words, 3+ letters first, then the 2-letter FR and EN ones: TOKEN +// takes 2+ letters, so without them "ok tu peux continuer la suite stp" +// counts 7 content words and passes the floor. At most 120 words. const STOP_WORDS = new Set( ('the and for with that this from are was were not but you your have has ' + 'had will can all any its into than then them they their there what ' + 'when which who how why les des une pour dans avec est pas que qui sur ' + - 'par aux ces ses son mais ont sont plus comme').split(' '), + 'par aux ces ses son mais ont sont plus comme ' + + 'ok tu je il on la le de du et en un au ce se sa ne ni ou si ' + + 'is it to do of in at an be by or as if so we me my no go up').split(' '), ) const TOKEN = /[\p{L}\p{N}]{2,40}/gu const MAX_QUERY = 2000 @@ -913,16 +952,26 @@ function queueWrite(st: State, job: () => Promise): Promise { return run } +export type Trigger = 'fresh' | 'idle' | 'keyword' | 'tool' +export type Outcome = 'injected' | 'none' | 'timeout' | 'refused' | 'paused' + +const TRIGGERS: string[] = ['fresh', 'idle', 'keyword', 'tool'] +const OUTCOMES: string[] = ['injected', 'none', 'timeout', 'refused', 'paused'] + +// `trigger` and `outcome` are absent on records written before 4c's review. export type LogRecord = { at: number session: string agent: string channel: 'prompt' | 'tool' ids: string[] + trigger?: Trigger + outcome?: Outcome } -// shortcut: one `l:` key per project, so no cap across roots; upgrade -// path: one global log key, or a sweep of the `l:` keys by age. +// shortcut: one `l:` key per project (about 28 KiB at 200 records), +// swept by `sweepLogs` against the engine's 4 MiB store cap, oldest key +// first; upgrade path: one global log key, or a smaller per-root cap. const LOG_PREFIX = 'l:' const MAX_LOG = 200 const MAX_LOG_BYTES = 65536 @@ -933,7 +982,9 @@ function isLogRecord(v: unknown): v is LogRecord { typeof r.at === 'number' && typeof r.session === 'string' && typeof r.agent === 'string' && (r.channel === 'prompt' || r.channel === 'tool') && - Array.isArray(r.ids) && r.ids.every(i => typeof i === 'string') + Array.isArray(r.ids) && r.ids.every(i => typeof i === 'string') && + (r.trigger === undefined || TRIGGERS.includes(r.trigger)) && + (r.outcome === undefined || OUTCOMES.includes(r.outcome)) } const logKey = (root: string) => LOG_PREFIX + root.slice(0, MAX_ROOT) @@ -962,6 +1013,7 @@ export function appendLog( ): Promise { return queueWrite(st, async () => { const log = await loadLog($, root) + if (log.length === 0) await sweepLogs($, logKey(root)) await $.store.set(logKey(root), fitLog([...log, record])) }) } @@ -979,7 +1031,10 @@ async function mutateSession( const created = !p.sessions.some(s => s.id === at.id) change(sessionOf(p, at.id, now)) await saveProject($, at.root, p) - if (created) await sweepStore($, storeKey(at.root), at.id) + if (created) { + await sweepStore($, storeKey(at.root), at.id) + await sweepLogs($, logKey(at.root)) + } } const recordFailure = ($: Api, at: Where, f: Failure) => @@ -1447,6 +1502,7 @@ async function knownLine( const record = { at: now, session: at.id, ids: [entry.id] } await appendLog($, st, at.root, { ...record, agent: agent ?? 'main', channel: 'tool', + trigger: 'tool', outcome: 'injected', }).catch(() => undefined) return line } @@ -1537,21 +1593,28 @@ const MAX_CANDS = 30 const RERANK_FAILS = 3 // failed reranks in a row that pause it const RERANK_PAUSE_MS = 900000 -/** True when a prompt may ask for recall: fresh context, idle, or a rule. */ -export function isTrigger( +/** What made a prompt ask for recall: a fresh context, idle, or a rule. */ +export function triggerKind( text: string, fresh: boolean, idleMs: number, -): boolean { +): Exclude | undefined { if (text.startsWith('/') || Array.from(text).length < MIN_PROMPT) { - return false + return undefined } - if (tokensOf(text).length < MIN_CONTENT_WORDS) return false + if (tokensOf(text).length < MIN_CONTENT_WORDS) return undefined + if (fresh) return 'fresh' + if (idleMs >= IDLE_MS) return 'idle' const asked = cap(text, MAX_REGEX_INPUT) - return fresh || idleMs >= IDLE_MS || - DESIGN_RULE.test(asked) || ANALYSIS_RULE.test(asked) + return DESIGN_RULE.test(asked) || ANALYSIS_RULE.test(asked) + ? 'keyword' + : undefined } +/** True when a prompt may ask for recall. */ +export const isTrigger = (text: string, fresh: boolean, idleMs: number) => + triggerKind(text, fresh, idleMs) !== undefined + /** Counts a rerank: the third failure in a row pauses it for 15 min. */ function countRerank(st: State, ok: boolean, now: number): void { if (ok) { @@ -1563,15 +1626,14 @@ function countRerank(st: State, ok: boolean, now: number): void { st.rerank = { fails: 0, until: now + RERANK_PAUSE_MS } } -/** The haiku answer, or undefined: failed, unanswered, late or paused. */ +/** The haiku answer, or undefined: failed, unanswered or late. */ async function askRerank( $: Api, st: State, text: string, cands: Entry[], + now: number, ): Promise { - const now = await $.clock.now() - if (now < st.rerank.until) return undefined const ask = { model: HAIKU, prompt: rerankPrompt(text, cands), @@ -1601,38 +1663,70 @@ function candidatesFor( return bm25Rank(open, text, MAX_CANDS) } +type Ask = { at: Where; text: string; trigger: Exclude } +type Picked = { block: string; ids: string[]; outcome: Outcome } + +const nothing = (outcome: Outcome): Picked => ({ block: '', ids: [], outcome }) + +/** The candidates a model answer names, in the answer's order. */ +function picksOf(answer: string, cands: Entry[]): Entry[] { + const byId = new Map(cands.map(c => [c.id, c])) + return parseIds(answer, cands, MAX_PICKS).flatMap(id => byId.get(id) ?? []) +} + +/** Claims the picks one by one; the header is charged to the first taken. */ +function claimPicks(st: State, key: string, picked: Entry[]): string[] { + const taken: string[] = [] + for (const e of picked) { + const header = taken.length === 0 ? RECALL_HEADER.length : 0 + taken.push(...claim(st, key, [e.id], [recallLine(e).length + 1 + header])) + } + return taken +} + +/** Asks haiku among the candidates and claims its picks. */ +async function pickRecall( + $: Api, + st: State, + key: string, + text: string, + cands: Entry[], +): Promise { + const now = await $.clock.now() + if (now < st.rerank.until) return nothing('paused') + const answer = await askRerank($, st, text, cands, now) + if (answer === undefined) return nothing('timeout') + const picked = picksOf(answer, cands) + if (picked.length === 0) return nothing('none') + const taken = claimPicks(st, key, picked) + const block = recallBlock( + picked.filter(e => taken.includes(e.id)).map(recallLine), RECALL_CAP) + if (block === '') return nothing('refused') + return { block, ids: taken, outcome: 'injected' } +} + /** * The recall block for a prompt: BM25 candidates, haiku picks, the ledger - * claim, the log. '' when nothing is to be injected. + * claim. Every ask is logged with its outcome; '' when nothing is injected. */ async function recallFor( $: Api, st: State, - at: Where, - text: string, + ask: Ask, rx: Registries, ): Promise { - const key = contextKey(at.id, undefined) - const cands = candidatesFor(st, key, rx, text) - if (cands.length === 0) return '' - const answer = await askRerank($, st, text, cands) - const byId = new Map(cands.map(c => [c.id, c])) - const picked = parseIds(answer ?? '', cands, MAX_PICKS) - .flatMap(id => byId.get(id) ?? []) - const when = await $.clock.now() - const taken: string[] = [] - for (const e of picked) { - // the header is charged to the first line actually taken - const header = taken.length === 0 ? RECALL_HEADER.length : 0 - taken.push(...claim(st, key, [e.id], [recallLine(e).length + 1 + header])) + const key = contextKey(ask.at.id, undefined) + const cands = candidatesFor(st, key, rx, ask.text) + const got = cands.length === 0 + ? nothing('none') + : await pickRecall($, st, key, ask.text, cands) + const record = { + at: await $.clock.now(), session: ask.at.id, agent: 'main', + ids: got.ids, trigger: ask.trigger, outcome: got.outcome, } - const block = recallBlock( - picked.filter(e => taken.includes(e.id)).map(recallLine), RECALL_CAP) - if (block === '') return '' - const record = { at: when, session: at.id, agent: 'main', ids: taken } - await appendLog($, st, at.root, { ...record, channel: 'prompt' }) + await appendLog($, st, ask.at.root, { ...record, channel: 'prompt' }) .catch(() => undefined) - return block + return got.block } const ALL_FILES = ['decisions.md', ...FAILURE_FILES] @@ -1643,7 +1737,7 @@ async function additions( st: State, at: Where, text: string, - wantRecall: boolean, + trigger: Ask['trigger'] | undefined, ): Promise { const rx = await loadRegistries($, at.root, ALL_FILES) // separate failures: a throw in one never drops the other @@ -1653,8 +1747,8 @@ async function additions( return '' }) : '' - const block = wantRecall - ? await recallFor($, st, at, text, rx).catch(() => '') + const block = trigger !== undefined + ? await recallFor($, st, { at, text, trigger }, rx).catch(() => '') : '' return [note, block].filter(part => part !== '') } @@ -1683,11 +1777,12 @@ export async function promptContext( const idle = last === undefined ? 0 : now - last // unknown: not idle st.lastPrompt.set(at.id, now) st.spent.delete(contextKey(at.id, undefined)) - const wantRecall = (await ensureRecall($, st)) && - isTrigger(e.text, st.fresh.has(at.id), idle) - if (wantRecall) st.fresh.delete(at.id) - if (!wantRecall && st.pending.length === 0) return [] - return additions($, st, at, e.text, wantRecall) + const trigger = (await ensureRecall($, st)) + ? triggerKind(e.text, st.fresh.has(at.id), idle) + : undefined + if (trigger !== undefined) st.fresh.delete(at.id) + if (trigger === undefined && st.pending.length === 0) return [] + return additions($, st, at, e.text, trigger) } function registerPrompt(on: On, st: State): void {