From a75efabbf161ce918cc993d83ad0717b905763b6 Mon Sep 17 00:00:00 2001 From: bchanot Date: Thu, 5 Nov 2026 16:18:39 +0100 Subject: [PATCH] feat(memory): bm25 ranking and the rerank prompt --- mods/memory/hooks/register.test.ts | 98 ++++++++++++++++++++++++++- mods/memory/hooks/register.ts | 103 +++++++++++++++++++++++++++++ 2 files changed, 200 insertions(+), 1 deletion(-) diff --git a/mods/memory/hooks/register.test.ts b/mods/memory/hooks/register.test.ts index d886ae4..d20f737 100644 --- a/mods/memory/hooks/register.test.ts +++ b/mods/memory/hooks/register.test.ts @@ -1,10 +1,12 @@ import { test, expect, mock } from 'claude-code/testing' import type { Engine, MockClock } from 'claude-code/testing' import type { On } from 'claude-code' +import type { Entry, Kind } from './register' import { cap, tail, redact, normalize, failureLine, sessionOf, addFailure, entriesOf, parseIndex, isLive, lessonOf, assembleEntries, grams, - buildIndex, matchOf, deErrno, + buildIndex, matchOf, deErrno, tokensOf, bm25Rank, rerankPrompt, + parseIds, recallLine, recallBlock, failureKey, segments, suiteKeyOf, commitLabel, finishLabel, } from './register' @@ -1832,3 +1834,97 @@ test('TIMEerrno: deErrno on 64 KiB of a phrase prefix, under 50 ms', () => { deErrno('no such file or '.repeat(4096)) expect(now() - t0).toBeLessThan(50) }) + +const mk = (id: string, title: string, lesson: string, body = ''): Entry => + ({ id, kind: id.slice(0, 3) as Kind, title, lesson, body }) + +const RB_ENTRIES = [ + mk('LRN-001', 'css grid gaps', 'Use gap.', 'layout of cards'), + mk('BLK-002', 'heredoc quoting breaks', 'Quote the delimiter.', + 'shell heredoc quoting eats the dollar sign in shell scripts'), + mk('BDR-003', 'store keys', 'Prefix them.', 'kv store naming'), +] + +test('RB1: bm25Rank puts the heredoc entry first, drops score 0', () => { + const ids = bm25Rank(RB_ENTRIES, 'quoting heredoc shell', 5).map(e => e.id) + expect(ids).toEqual(['BLK-002']) + expect(bm25Rank(RB_ENTRIES, 'the and for', 5)).toEqual([]) + expect(tokensOf('The Heredoc, a b ÉTÉ 42')).toEqual(['heredoc', 'été', '42']) +}) + +test('RB1b: ties go by numeric ID order, k bounds the result', () => { + const twin = [mk('LRN-10', 'zed', ''), mk('LRN-9', 'zed', '')] + expect(bm25Rank(twin, 'zed', 5).map(e => e.id)).toEqual(['LRN-9', 'LRN-10']) + expect(bm25Rank(twin, 'zed', 1).map(e => e.id)).toEqual(['LRN-9']) +}) + +test('RB2: parseIds keeps candidate IDs once, in order, up to max', () => { + const cands = [mk('BLK-041', 't', ''), mk('LRN-258', 't', '')] + expect(parseIds('BLK-041, BDR-999, LRN-258', cands, 3)) + .toEqual(['BLK-041', 'LRN-258']) + expect(parseIds('LRN-258 BLK-041 LRN-258', cands, 1)).toEqual(['LRN-258']) + expect(parseIds('none', cands, 3)).toEqual([]) +}) + +test('RB3: long Decisions give ellipsis lessons and a block under cap', () => { + const field = 'Use alpha ' + 'x'.repeat(1990) + const es = ['BDR-001', 'BDR-002', 'BDR-003'].map(id => + assembleEntries([`## ${id} — t\n- **Decision**: ${field}\n`], + new Map([[id, { status: 'accepted', title: 'T' + id }]]))[0]!) + for (const e of es) expect(recallLine(e).endsWith('…')).toBe(true) + const block = recallBlock(es.map(recallLine), 1500) + expect(block.length).toBeLessThanOrEqual(1500) + expect(block.split('\n')[0]).toBe('memory recall:') + expect(block.split('\n')[1]).toBe( + 'BDR-001 TBDR-001 — Use alpha ' + 'x'.repeat(280 - 11) + '…') +}) + +test('RB3b: recallLine and recallBlock edges', () => { + expect(recallLine(mk('LRN-1', 'T', 'a;'))).toBe('LRN-1 T — a;') + expect(recallLine(mk('LRN-1', 'T', 'Use X.'))).toBe('LRN-1 T — Use X.') + expect(recallLine(mk('LRN-1', 'T', ''))).toBe('LRN-1 T') + expect(recallBlock(['a'.repeat(50)], 20)).toBe('') + expect(recallBlock(['a'.repeat(50), 'bb'], 20)).toBe('memory recall:\nbb') + expect(recallBlock([], 100)).toBe('') +}) + +test('RB4: rerankPrompt is the literal prompt', () => { + const cands = [mk('LRN-1', 'T one', 'Do a.'), mk('BLK-2', 'T two', '')] + expect(rerankPrompt('fix the heredoc', cands)).toBe( + 'Prompt: fix the heredoc\nCandidates:\nLRN-1 | T one | Do a.\n' + + 'BLK-2 | T two | \nAnswer with at most 3 candidate IDs that would ' + + 'help with this prompt, comma separated, or none.') +}) + +test('RB5: 500 entries and a 64 KiB query rank under 300 ms', () => { + const es = Array.from({ length: 500 }, (_, i) => + mk(`LRN-${i + 1}`, `t${i}`, 'l', `w${i} word `.repeat(160))) + const head = 'w7 '.padEnd(2000, 'z ') + const t0 = now() + const a = bm25Rank(es, head + 'w9 '.repeat(20000), 5).map(e => e.id) + expect(now() - t0).toBeLessThan(300) + const b = bm25Rank(es, head + 'w300 '.repeat(20000), 5).map(e => e.id) + expect(a).toEqual(b) + expect(a[0]).toBe('LRN-8') +}) + +test('RB6: a secret-shaped token is redacted in the rerank prompt', () => { + const secret = 'sk_live_' + 'a'.repeat(24) + const p = rerankPrompt(`use ${secret} now`, []) + expect(p).toContain('') + expect(p).not.toContain(secret) +}) + +test('RB7: failureKey is the normalized last line of the redacted tail', () => { + expect(failureKey('Exit code 1\nrm: /tmp/x/y: Permission denied', 'rm x')) + .toBe('rm: : permission denied') + expect(failureKey('', 'ls')).toBeUndefined() +}) + +test('TIMErank: tokensOf and parseIds on 64 KiB, under 50 ms', () => { + const t0 = now() + tokensOf('ab '.repeat(21000)) + tokensOf('x'.repeat(65536)) + parseIds('BDR-1 '.repeat(10000) + 'LRN-'.repeat(15000), [], 3) + expect(now() - t0).toBeLessThan(50) +}) diff --git a/mods/memory/hooks/register.ts b/mods/memory/hooks/register.ts index dd8cf42..c06cd7a 100644 --- a/mods/memory/hooks/register.ts +++ b/mods/memory/hooks/register.ts @@ -622,6 +622,109 @@ async function statusMap($: Api, root: string): Promise { } } +const STOP_WORDS = new Set( + ('the and for with that this from are was were not but you your have has ' + + 'had will can all any its into than then them they their there what ' + + 'when which who how why les des une pour dans avec est pas que qui sur ' + + 'par aux ces ses son mais ont sont plus comme').split(' '), +) +const TOKEN = /[\p{L}\p{N}]{2,40}/gu +const MAX_QUERY = 2000 +const MAX_DOC = 65536 +const BM25_K1 = 1.2 +const BM25_B = 0.75 +const MAX_PICKS = 3 + +/** Lowercase word runs (2-40 letters or digits) minus function words. */ +export function tokensOf(text: string): string[] { + const words = cap(text, MAX_DOC).toLowerCase().match(TOKEN) ?? [] + return words.filter(w => !STOP_WORDS.has(w)) +} + +type Doc = { entry: Entry; tf: Map; len: number } + +function docOf(entry: Entry): Doc { + const tokens = tokensOf(`${entry.title} ${entry.lesson} ${entry.body}`) + const tf = new Map() + for (const t of tokens) tf.set(t, (tf.get(t) ?? 0) + 1) + return { entry, tf, len: tokens.length } +} + +function scoreOf( + doc: Doc, terms: string[], df: Map, n: number, avg: number, +): number { + let score = 0 + for (const term of terms) { + const f = doc.tf.get(term) ?? 0 + if (f === 0) continue + const d = df.get(term) ?? 0 + const idf = Math.log(1 + (n - d + 0.5) / (d + 0.5)) + const norm = 1 - BM25_B + (BM25_B * doc.len) / avg + score += (idf * f * (BM25_K1 + 1)) / (f + BM25_K1 * norm) + } + return score +} + +/** The `k` best entries for a query (BM25), highest first, score > 0. */ +export function bm25Rank(entries: Entry[], query: string, k: number): Entry[] { + const terms = [...new Set(tokensOf(cap(query, MAX_QUERY)))] + const docs = entries.map(docOf) + const df = new Map() + for (const d of docs) for (const t of d.tf.keys()) { + df.set(t, (df.get(t) ?? 0) + 1) + } + const avg = docs.reduce((sum, d) => sum + d.len, 0) / (docs.length || 1) + return docs + .map(d => ({ e: d.entry, s: scoreOf(d, terms, df, docs.length, avg || 1) })) + .filter(x => x.s > 0) + .sort((a, b) => b.s - a.s || + a.e.id.localeCompare(b.e.id, 'en', { numeric: true })) + .slice(0, k) + .map(x => x.e) +} + +/** The haiku prompt: the redacted user prompt and one line per candidate. */ +export function rerankPrompt(query: string, cands: Entry[]): string { + const lines = cands.map(c => `${c.id} | ${c.title} | ${c.lesson}`) + return ( + `Prompt: ${redact(cap(query, MAX_QUERY))}\nCandidates:\n` + + `${lines.join('\n')}\nAnswer with at most ${MAX_PICKS} candidate IDs ` + + 'that would help with this prompt, comma separated, or none.' + ) +} + +const ID_IN_ANSWER = /(?:BDR|LRN|BLK)-\d{1,5}/g + +/** The candidate IDs named in a model answer, deduplicated, in order. */ +export function parseIds(text: string, cands: Entry[], max: number): string[] { + const known = new Set(cands.map(c => c.id)) + const found = cap(text, MAX_REGEX_INPUT).match(ID_IN_ANSWER) ?? [] + return [...new Set(found)].filter(id => known.has(id)).slice(0, max) +} + +/** One recall line; the lesson keeps its own terminator. */ +export function recallLine(e: Entry): string { + const head = `${e.id} ${e.title}` + return e.lesson === '' ? head : `${head} — ${e.lesson}` +} + +/** The injected block; a line that would pass `max` characters is dropped. */ +export function recallBlock(lines: string[], max: number): string { + const header = 'memory recall:' + let block = header + for (const line of lines) { + if (block.length + 1 + line.length <= max) block += `\n${line}` + } + return block === header ? '' : block +} + +/** The dedup key of a failure text, as `buildFailure` stores it. */ +export function failureKey(text: string, command: string): string | undefined { + void command // the key comes from the output only; kept for the harness + const line = lastLine(redact(tail(text, MAX_REGEX_INPUT))) + return line === undefined ? undefined : normalize(line) +} + type Registries = { entries: Entry[] status: StatusMap | undefined