From adb3db97cb2c4a9a7d3681128ec0522c769085c7 Mon Sep 17 00:00:00 2001 From: bchanot Date: Thu, 5 Nov 2026 17:38:30 +0100 Subject: [PATCH] chore(bench): phase 4c recall measured on 14 days of transcripts, content floor gated --- .../2026-11-04-memory-recall-1859.md | 2 + .../2026-11-04-memory-recall-spike/RECALL.md | 27 ++ .../recall-bench.ts | 268 ++++++++++++++++++ 3 files changed, 297 insertions(+) create mode 100644 .claude/tasks/plans/2026-11-04-memory-recall-spike/RECALL.md create mode 100644 .claude/tasks/plans/2026-11-04-memory-recall-spike/recall-bench.ts diff --git a/.claude/tasks/contracts/2026-11-04-memory-recall-1859.md b/.claude/tasks/contracts/2026-11-04-memory-recall-1859.md index d4ff2a2..26e85f1 100644 --- a/.claude/tasks/contracts/2026-11-04-memory-recall-1859.md +++ b/.claude/tasks/contracts/2026-11-04-memory-recall-1859.md @@ -46,6 +46,8 @@ STEP 3 gate `[gated 2026-11-05]` (user: "oui"): plan r3 (`docs/superpowers/plans Digest cap order `[gated 2026-11-05]` (user: "Retirer d'abord les TODO"): over 800 chars the digest drops TODO lines first (last to first), then open-BLK lines; the header, the `+N` count line and the journal line stay. +Measurement gate `[gated 2026-11-05]` (RECALL.md: prompt channel 19/38 = 50 % relevant < 64 % bar, noise on content-free idle prompts; timeout share 4/20 = 20 %; user: "Plancher de contenu"): no intent recall on a prompt with fewer than 5 content words (`tokensOf` minus stop words); recall ships on and switchable; the 3 s bound stays; 4d reads the injection log. + 9. `[gated 2026-11-05]` Mod tests: `mods/memory/hooks/register.test.ts` holds the plan's named tests and the mods suite is green; no line of register.ts over 80 columns. CHECK: cd /home/bchanot/Documents/claude && T=mods/memory/hooks/register.test.ts && for n in SR1 SR3 SR5 LS1 LS4 ER1 ER3 RB1 RB3 RB5 IR1 IR5 IR6 IR7 IR8 IR9 IR11 TE1 TE2 TE4 PM1 PM2; do /usr/bin/grep -q "$n" "$T" || exit 1; done && [ "$(awk 'length($0) > 80' mods/memory/hooks/register.ts | wc -l)" = 0 ] && a=$(make test suite=lib/tests/mods.test.sh 2>&1) && printf '%s\n' "$a" | /usr/bin/grep -q 'all suites green' && echo MOD4C-OK EXPECT: MOD4C-OK diff --git a/.claude/tasks/plans/2026-11-04-memory-recall-spike/RECALL.md b/.claude/tasks/plans/2026-11-04-memory-recall-spike/RECALL.md new file mode 100644 index 0000000..fa8e6dc --- /dev/null +++ b/.claude/tasks/plans/2026-11-04-memory-recall-spike/RECALL.md @@ -0,0 +1,27 @@ +# Phase 4c recall measured on 14 days of transcripts (2026-11-05) + +Harness: `recall-bench.ts` (bun) imports the mod's own functions (`isTrigger`, `bm25Rank`, `rerankPrompt`, `parseIds`, `assembleEntries`, `failureKey`, `buildIndex`, `matchOf`…); statuses from `lib/memory.sh index`; haiku `claude-haiku-5-5` through `claude -p` from a `mktemp -d` cwd with `--setting-sources local`, no tools. Counts and shares only; no prompt or error text here (LRN-124). + +## Prompt channel + +Population: main-loop transcripts with `entrypoint == "cli"`, user entries with `origin.kind == "human"` (typed prompts), not meta, not compaction summaries: 65 prompts, 11 older entries skipped (no `origin` field), 24 meet the trigger, 20 sampled (seed 42). A first run sampled `` and summary entries (origin `task-notification` / `isCompactSummary`, never seen by the mod's `composer` gate): discarded as a measurement. + +| measure | value | +|---|---| +| asks | n = 20 | +| injections (prompts with ≥ 1 entry) | 15 / 20 | +| entries injected | n = 38 (2.5 per injection) | +| relevant entries (controller labels) | 19 / 38 = 50 % | +| relevant on prompts with ≥ 5 content words (estimate) | 16 / 28 = 57 % | +| bar (BDR-125 amendment 2, haiku rerank baseline) | 14 / 22 = 64 % | +| haiku `duration_api_ms` > 3000 (upper bound) | 4 / 20 = 20 % (first run: 7 / 20) | + +The noise sits on content-free prompts triggered by idle time (short approvals): 0 relevant of 10 injected. Prompts with a topic recall the right entries. Caveat: the harness reads today's registries, so entries written after a prompt can be recalled for it (temporal leakage): the true shares are somewhat lower. + +## Error channel + +36 Bash failures keyed over 14 days (main + sub-agents), 3 known-line injections (2 sessions): one exact lesson, two plausible. n = 3: insufficient sample. + +## Decision (2026-11-05) + +User: "Plancher de contenu" — no intent recall on a prompt with fewer than 5 content words (stop words excluded); recall ships on and switchable (`"recall": false`); the injection log (4c) feeds the 4d measurement on real use. The 3 s rerank bound stays (timeout share 20 % < 25 %). diff --git a/.claude/tasks/plans/2026-11-04-memory-recall-spike/recall-bench.ts b/.claude/tasks/plans/2026-11-04-memory-recall-spike/recall-bench.ts new file mode 100644 index 0000000..7916607 --- /dev/null +++ b/.claude/tasks/plans/2026-11-04-memory-recall-spike/recall-bench.ts @@ -0,0 +1,268 @@ +// Measurement harness of the phase 4c memory recall (read-only, stdout). +// Transcript walk follows noise.py (2026-11-03-memory-capture-spike): +// its find_files / pairs_of logic, ported to TS. +// Run: bun recall-bench.ts prompts [--seed N] [--n 20] +// bun recall-bench.ts errors +import * as fs from 'node:fs' +import * as os from 'node:os' +import * as path from 'node:path' +import { + IDLE_MS, RECALL_CAP, HAIKU, RERANK_MS, assembleEntries, bm25Rank, + buildIndex, entriesOf, failureKey, isLive, isTrigger, matchOf, parseIds, + parseIndex, recallBlock, recallLine, redact, rerankPrompt, + type Entry, type StatusMap, +} from '../../../../mods/memory/hooks/register.ts' + +const REPO = path.resolve(import.meta.dir, '../../../..') +const ROOT = path.join( + os.homedir(), '.claude/projects/-home-bchanot-Documents-claude') +const DAYS = 14 +const MAX_CANDS = 30 +const MAX_PICKS = 3 +const MIN_SAMPLE = 10 +const MEMORY_DIR = path.join(REPO, '.claude/memory') + +type Rec = Record +type Found = { sid: string; agent: boolean; file: string } +type Composer = { sid: string; text: string; fresh: boolean; idle: number } + +function findFiles(): Found[] { + const cutoff = Date.now() - DAYS * 86400000 + const recent = (p: string) => + (fs.statSync(p, { throwIfNoEntry: false })?.mtimeMs ?? 0) >= cutoff + const found: Found[] = [] + for (const name of fs.readdirSync(ROOT).sort()) { + const full = path.join(ROOT, name) + if (name.endsWith('.jsonl') && recent(full)) { + found.push({ sid: name.slice(0, -6), agent: false, file: full }) + } + if (name.endsWith('.jsonl')) continue + const sub = path.join(full, 'subagents') + if (!fs.statSync(sub, { throwIfNoEntry: false })?.isDirectory()) continue + for (const f of fs.readdirSync(sub).sort()) { + const p = path.join(sub, f) + if (f.endsWith('.jsonl') && recent(p)) { + found.push({ sid: name, agent: true, file: p }) + } + } + } + return found +} + +function records(file: string): Rec[] { + const out: Rec[] = [] + for (const raw of fs.readFileSync(file, 'utf8').split('\n')) { + try { out.push(JSON.parse(raw)) } catch { /* skip bad line */ } + } + return out +} + +function blocksOf(r: Rec): Rec[] { + const c = r.message?.content + return Array.isArray(c) ? c.filter(b => b && typeof b === 'object') : [] +} + +function resultText(content: unknown): string { + if (typeof content === 'string') return content + if (!Array.isArray(content)) return '' + return content.filter(b => b?.type === 'text').map(b => b.text).join('') +} + +/** Bash failures: tool_use paired with its tool_result (is_error). */ +function failuresOf(file: string): { text: string; command: string }[] { + const calls = new Map() + const out: { text: string; command: string }[] = [] + for (const r of records(file)) { + for (const b of blocksOf(r)) { + if (b.type === 'tool_use' && b.name === 'Bash') { + calls.set(b.id, String(b.input?.command ?? '')) + } else if (b.type === 'tool_result' && calls.has(b.tool_use_id) && + b.is_error === true) { + out.push({ text: resultText(b.content), + command: calls.get(b.tool_use_id) ?? '' }) + } + } + } + return out +} + +const WRAPPERS = [' t.startsWith(w)) ? undefined : t +} + +/** Trigger-meeting typed prompts; fresh = first after a summary or start. */ +function triggerPrompts(files: Found[]): { all: number; hits: Composer[] } { + const hits: Composer[] = [] + let all = 0 + for (const f of files.filter(x => !x.agent)) { + let last: number | undefined + let fresh = true + for (const r of records(f.file)) { + if (r.isCompactSummary === true) { fresh = true; continue } + const text = composerText(r) + if (text === undefined) continue + all += 1 + const at = Date.parse(r.timestamp) + const idle = last === undefined || Number.isNaN(at) ? 0 : at - last + if (!Number.isNaN(at)) last = at + if (isTrigger(text, fresh, idle)) { + hits.push({ sid: f.sid, text, fresh, idle }) + } + fresh = false + } + } + return { all, hits } +} + +function rng(seed: number): () => number { + let a = seed >>> 0 + return () => { + a = (a + 0x6d2b79f5) >>> 0 + let t = Math.imul(a ^ (a >>> 15), 1 | a) + t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t + return ((t ^ (t >>> 14)) >>> 0) / 4294967296 + } +} + +function sample(items: T[], n: number, seed: number): T[] { + const pool = [...items] + const next = rng(seed) + for (let i = pool.length - 1; i > 0; i--) { + const j = Math.floor(next() * (i + 1)); + [pool[i], pool[j]] = [pool[j]!, pool[i]!] + } + return pool.slice(0, n) +} + +function statuses(): StatusMap { + const r = Bun.spawnSync(['bash', 'lib/memory.sh', '-C', '.', 'index'], + { cwd: REPO }) + if (r.exitCode !== 0) throw new Error('memory.sh index failed') + return parseIndex(r.stdout.toString()) +} + +function registryTexts(names: string[]): string[] { + return names.map(n => fs.readFileSync(path.join(MEMORY_DIR, n), 'utf8')) +} + +function haiku(prompt: string): { text: string; ms: number } | undefined { + const dir = fs.mkdtempSync(os.tmpdir() + '/recall-') + try { + const r = Bun.spawnSync(['claude', '-p', '--model', HAIKU, + '--output-format', 'json', '--setting-sources', 'local', + '--tools', ''], { cwd: dir, stdin: new TextEncoder().encode(prompt) }) + const j = JSON.parse(r.stdout.toString()) + return { text: String(j.result ?? ''), ms: Number(j.duration_api_ms) } + } catch { + return undefined + } finally { + fs.rmSync(dir, { recursive: true, force: true }) + } +} + +function argOf(name: string, fallback: number): number { + const i = process.argv.indexOf(name) + const v = i < 0 ? NaN : Number(process.argv[i + 1]) + return Number.isFinite(v) ? v : fallback +} + +function share(k: number, n: number): string { + return n < MIN_SAMPLE ? `${k}/${n} (insufficient sample, < ${MIN_SAMPLE})` + : `${k}/${n}` +} + +function oneLine(text: string, n: number): string { + return redact(text).replace(/\s+/g, ' ').slice(0, n) +} + +function runPrompts(): void { + const status = statuses() + const entries = assembleEntries( + registryTexts(['decisions.md', 'learnings.md', 'blockers.md']), status) + const live = entries.filter(e => isLive(status, e.id)) + const { all, hits } = triggerPrompts(findFiles()) + const picked = sample(hits, argOf('--n', 20), argOf('--seed', 1)) + console.log(`human prompts (cli): ${all}, skipped no origin: ${skipped.noOrigin}, meeting isTrigger: ` + + `${hits.length}, sampled: ${picked.length}, live entries: ${live.length}`) + let asks = 0, answered = 0, inj = 0, ids = 0, late = 0 + picked.forEach((p, i) => { + asks += 1 + const cands = bm25Rank(live, p.text, MAX_CANDS) + const a = haiku(rerankPrompt(p.text, cands)) + const got = parseIds(a?.text ?? '', cands, MAX_PICKS) + const byId = new Map(cands.map(c => [c.id, c])) + const block = recallBlock( + got.flatMap(id => byId.get(id) ?? []).map(recallLine), RECALL_CAP) + const slow = a !== undefined && a.ms > RERANK_MS + if (a !== undefined) answered += 1 + if (slow) late += 1 + if (block !== '') { inj += 1; ids += got.length } + console.log(`#${i} [${p.fresh ? 'fresh' : 'idle/rule'}] ` + + `"${oneLine(p.text, 120)}"`) + for (const id of got) console.log(` ${id} ${byId.get(id)?.title}`) + console.log(` duration_api_ms=${a?.ms ?? 'n/a'} (upper bound) ` + + `timeout(>${RERANK_MS})=${a === undefined ? 'failed' : slow ? 'yes' : 'no'}`) + }) + console.log(`TOTAL asks=${asks} answered=${answered} injections=${inj} ` + + `mean IDs per injection=${inj ? (ids / inj).toFixed(2) : 'n/a'} ` + + `timeout share=${share(late, answered)} injection share=` + + share(inj, asks)) +} + +function errorIndex(status: StatusMap): Map { + const texts = registryTexts(['blockers.md', 'learnings.md']) + return buildIndex(assembleEntries(texts, status), status) +} + +function runErrors(): void { + const status = statuses() + const index = errorIndex(status) + const titles = new Map( + registryTexts(['blockers.md', 'learnings.md']).flatMap(entriesOf) + .map(p => [p.id, status.get(p.id)?.title ?? ''] as const)) + const seen = new Map>() + const perSession = new Map() + let failures = 0 + for (const f of findFiles()) { + const ledger = `${f.sid}|${f.agent}` + for (const x of failuresOf(f.file)) { + const key = failureKey(x.text, x.command) + if (key === undefined) continue + failures += 1 + const id = matchOf(index, key) + const done = seen.get(ledger) ?? new Set() + if (id === undefined || done.has(id)) continue + seen.set(ledger, done.add(id)) + perSession.set(f.sid, (perSession.get(f.sid) ?? 0) + 1) + console.log(`${f.sid.slice(0, 8)}${f.agent ? ' sub' : ''}: ` + + `"${oneLine(key, 120)}" -> ${id} ${titles.get(id) ?? ''}`) + } + } + const total = [...perSession.values()].reduce((a, b) => a + b, 0) + console.log('per session: ' + [...perSession] + .map(([s, n]) => `${s.slice(0, 8)}=${n}`).join(' ')) + console.log(`TOTAL failures keyed=${failures} known-line injections=` + + `${share(total, failures)} sessions with injection=` + + `${perSession.size}`) +} + +const mode = process.argv[2] +if (mode === 'prompts') runPrompts() +else if (mode === 'errors') runErrors() +else console.error('usage: recall-bench.ts prompts|errors')