chore(bench): phase 4c recall measured on 14 days of transcripts, content floor gated

This commit is contained in:
bchanot committed 2026-11-05 17:38:30 +01:00
1 parent f9787044ff
commit adb3db97cb
3 files changed
+297

No files matched your search

@@ -46,6 +46,8 @@ STEP 3 gate `[gated 2026-11-05]` (user: "oui"): plan r3 (`docs/superpowers/plans
Digest cap order `[gated 2026-11-05]` (user: "Retirer d'abord les TODO"): over 800 chars the digest drops TODO lines first (last to first), then open-BLK lines; the header, the `+N` count line and the journal line stay.
Measurement gate `[gated 2026-11-05]` (RECALL.md: prompt channel 19/38 = 50 % relevant < 64 % bar, noise on content-free idle prompts; timeout share 4/20 = 20 %; user: "Plancher de contenu"): no intent recall on a prompt with fewer than 5 content words (`tokensOf` minus stop words); recall ships on and switchable; the 3 s bound stays; 4d reads the injection log.
9. `[gated 2026-11-05]` Mod tests: `mods/memory/hooks/register.test.ts` holds the plan's named tests and the mods suite is green; no line of register.ts over 80 columns.
CHECK: cd /home/bchanot/Documents/claude && T=mods/memory/hooks/register.test.ts && for n in SR1 SR3 SR5 LS1 LS4 ER1 ER3 RB1 RB3 RB5 IR1 IR5 IR6 IR7 IR8 IR9 IR11 TE1 TE2 TE4 PM1 PM2; do /usr/bin/grep -q "$n" "$T" || exit 1; done && [ "$(awk 'length($0) > 80' mods/memory/hooks/register.ts | wc -l)" = 0 ] && a=$(make test suite=lib/tests/mods.test.sh 2>&1) && printf '%s\n' "$a" | /usr/bin/grep -q 'all suites green' && echo MOD4C-OK
EXPECT: MOD4C-OK
@@ -0,0 +1,27 @@
# Phase 4c recall measured on 14 days of transcripts (2026-11-05)
Harness: `recall-bench.ts` (bun) imports the mod's own functions (`isTrigger`, `bm25Rank`, `rerankPrompt`, `parseIds`, `assembleEntries`, `failureKey`, `buildIndex`, `matchOf`…); statuses from `lib/memory.sh index`; haiku `claude-haiku-5-5` through `claude -p` from a `mktemp -d` cwd with `--setting-sources local`, no tools. Counts and shares only; no prompt or error text here (LRN-124).
## Prompt channel
Population: main-loop transcripts with `entrypoint == "cli"`, user entries with `origin.kind == "human"` (typed prompts), not meta, not compaction summaries: 65 prompts, 11 older entries skipped (no `origin` field), 24 meet the trigger, 20 sampled (seed 42). A first run sampled `<task-notification>` and summary entries (origin `task-notification` / `isCompactSummary`, never seen by the mod's `composer` gate): discarded as a measurement.
| measure | value |
|---|---|
| asks | n = 20 |
| injections (prompts with ≥ 1 entry) | 15 / 20 |
| entries injected | n = 38 (2.5 per injection) |
| relevant entries (controller labels) | 19 / 38 = 50 % |
| relevant on prompts with ≥ 5 content words (estimate) | 16 / 28 = 57 % |
| bar (BDR-125 amendment 2, haiku rerank baseline) | 14 / 22 = 64 % |
| haiku `duration_api_ms` > 3000 (upper bound) | 4 / 20 = 20 % (first run: 7 / 20) |
The noise sits on content-free prompts triggered by idle time (short approvals): 0 relevant of 10 injected. Prompts with a topic recall the right entries. Caveat: the harness reads today's registries, so entries written after a prompt can be recalled for it (temporal leakage): the true shares are somewhat lower.
## Error channel
36 Bash failures keyed over 14 days (main + sub-agents), 3 known-line injections (2 sessions): one exact lesson, two plausible. n = 3: insufficient sample.
## Decision (2026-11-05)
User: "Plancher de contenu" — no intent recall on a prompt with fewer than 5 content words (stop words excluded); recall ships on and switchable (`"recall": false`); the injection log (4c) feeds the 4d measurement on real use. The 3 s rerank bound stays (timeout share 20 % < 25 %).
@@ -0,0 +1,268 @@
// Measurement harness of the phase 4c memory recall (read-only, stdout).
// Transcript walk follows noise.py (2026-11-03-memory-capture-spike):
// its find_files / pairs_of logic, ported to TS.
// Run: bun recall-bench.ts prompts [--seed N] [--n 20]
// bun recall-bench.ts errors
import * as fs from 'node:fs'
import * as os from 'node:os'
import * as path from 'node:path'
import {
IDLE_MS, RECALL_CAP, HAIKU, RERANK_MS, assembleEntries, bm25Rank,
buildIndex, entriesOf, failureKey, isLive, isTrigger, matchOf, parseIds,
parseIndex, recallBlock, recallLine, redact, rerankPrompt,
type Entry, type StatusMap,
} from '../../../../mods/memory/hooks/register.ts'
const REPO = path.resolve(import.meta.dir, '../../../..')
const ROOT = path.join(
os.homedir(), '.claude/projects/-home-bchanot-Documents-claude')
const DAYS = 14
const MAX_CANDS = 30
const MAX_PICKS = 3
const MIN_SAMPLE = 10
const MEMORY_DIR = path.join(REPO, '.claude/memory')
type Rec = Record<string, any>
type Found = { sid: string; agent: boolean; file: string }
type Composer = { sid: string; text: string; fresh: boolean; idle: number }
function findFiles(): Found[] {
const cutoff = Date.now() - DAYS * 86400000
const recent = (p: string) =>
(fs.statSync(p, { throwIfNoEntry: false })?.mtimeMs ?? 0) >= cutoff
const found: Found[] = []
for (const name of fs.readdirSync(ROOT).sort()) {
const full = path.join(ROOT, name)
if (name.endsWith('.jsonl') && recent(full)) {
found.push({ sid: name.slice(0, -6), agent: false, file: full })
}
if (name.endsWith('.jsonl')) continue
const sub = path.join(full, 'subagents')
if (!fs.statSync(sub, { throwIfNoEntry: false })?.isDirectory()) continue
for (const f of fs.readdirSync(sub).sort()) {
const p = path.join(sub, f)
if (f.endsWith('.jsonl') && recent(p)) {
found.push({ sid: name, agent: true, file: p })
}
}
}
return found
}
function records(file: string): Rec[] {
const out: Rec[] = []
for (const raw of fs.readFileSync(file, 'utf8').split('\n')) {
try { out.push(JSON.parse(raw)) } catch { /* skip bad line */ }
}
return out
}
function blocksOf(r: Rec): Rec[] {
const c = r.message?.content
return Array.isArray(c) ? c.filter(b => b && typeof b === 'object') : []
}
function resultText(content: unknown): string {
if (typeof content === 'string') return content
if (!Array.isArray(content)) return ''
return content.filter(b => b?.type === 'text').map(b => b.text).join('')
}
/** Bash failures: tool_use paired with its tool_result (is_error). */
function failuresOf(file: string): { text: string; command: string }[] {
const calls = new Map<string, string>()
const out: { text: string; command: string }[] = []
for (const r of records(file)) {
for (const b of blocksOf(r)) {
if (b.type === 'tool_use' && b.name === 'Bash') {
calls.set(b.id, String(b.input?.command ?? ''))
} else if (b.type === 'tool_result' && calls.has(b.tool_use_id) &&
b.is_error === true) {
out.push({ text: resultText(b.content),
command: calls.get(b.tool_use_id) ?? '' })
}
}
}
return out
}
const WRAPPERS = ['<command-', '<local-command', '<system-reminder']
const skipped = { noOrigin: 0 }
/** A typed prompt: origin.kind human, string content, not a command. */
function composerText(r: Rec): string | undefined {
if (r.type !== 'user' || r.isSidechain === true) return undefined
if (r.isMeta === true || r.isCompactSummary === true) return undefined
if (r.entrypoint !== 'cli') return undefined
const c = r.message?.content
if (typeof c !== 'string') return undefined
if (r.origin?.kind === undefined) {
skipped.noOrigin += 1
return undefined
}
const t = c.trim()
if (r.origin.kind !== 'human' || t.startsWith('/')) return undefined
return WRAPPERS.some(w => t.startsWith(w)) ? undefined : t
}
/** Trigger-meeting typed prompts; fresh = first after a summary or start. */
function triggerPrompts(files: Found[]): { all: number; hits: Composer[] } {
const hits: Composer[] = []
let all = 0
for (const f of files.filter(x => !x.agent)) {
let last: number | undefined
let fresh = true
for (const r of records(f.file)) {
if (r.isCompactSummary === true) { fresh = true; continue }
const text = composerText(r)
if (text === undefined) continue
all += 1
const at = Date.parse(r.timestamp)
const idle = last === undefined || Number.isNaN(at) ? 0 : at - last
if (!Number.isNaN(at)) last = at
if (isTrigger(text, fresh, idle)) {
hits.push({ sid: f.sid, text, fresh, idle })
}
fresh = false
}
}
return { all, hits }
}
function rng(seed: number): () => number {
let a = seed >>> 0
return () => {
a = (a + 0x6d2b79f5) >>> 0
let t = Math.imul(a ^ (a >>> 15), 1 | a)
t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t
return ((t ^ (t >>> 14)) >>> 0) / 4294967296
}
}
function sample<T>(items: T[], n: number, seed: number): T[] {
const pool = [...items]
const next = rng(seed)
for (let i = pool.length - 1; i > 0; i--) {
const j = Math.floor(next() * (i + 1));
[pool[i], pool[j]] = [pool[j]!, pool[i]!]
}
return pool.slice(0, n)
}
function statuses(): StatusMap {
const r = Bun.spawnSync(['bash', 'lib/memory.sh', '-C', '.', 'index'],
{ cwd: REPO })
if (r.exitCode !== 0) throw new Error('memory.sh index failed')
return parseIndex(r.stdout.toString())
}
function registryTexts(names: string[]): string[] {
return names.map(n => fs.readFileSync(path.join(MEMORY_DIR, n), 'utf8'))
}
function haiku(prompt: string): { text: string; ms: number } | undefined {
const dir = fs.mkdtempSync(os.tmpdir() + '/recall-')
try {
const r = Bun.spawnSync(['claude', '-p', '--model', HAIKU,
'--output-format', 'json', '--setting-sources', 'local',
'--tools', ''], { cwd: dir, stdin: new TextEncoder().encode(prompt) })
const j = JSON.parse(r.stdout.toString())
return { text: String(j.result ?? ''), ms: Number(j.duration_api_ms) }
} catch {
return undefined
} finally {
fs.rmSync(dir, { recursive: true, force: true })
}
}
function argOf(name: string, fallback: number): number {
const i = process.argv.indexOf(name)
const v = i < 0 ? NaN : Number(process.argv[i + 1])
return Number.isFinite(v) ? v : fallback
}
function share(k: number, n: number): string {
return n < MIN_SAMPLE ? `${k}/${n} (insufficient sample, < ${MIN_SAMPLE})`
: `${k}/${n}`
}
function oneLine(text: string, n: number): string {
return redact(text).replace(/\s+/g, ' ').slice(0, n)
}
function runPrompts(): void {
const status = statuses()
const entries = assembleEntries(
registryTexts(['decisions.md', 'learnings.md', 'blockers.md']), status)
const live = entries.filter(e => isLive(status, e.id))
const { all, hits } = triggerPrompts(findFiles())
const picked = sample(hits, argOf('--n', 20), argOf('--seed', 1))
console.log(`human prompts (cli): ${all}, skipped no origin: ${skipped.noOrigin}, meeting isTrigger: ` +
`${hits.length}, sampled: ${picked.length}, live entries: ${live.length}`)
let asks = 0, answered = 0, inj = 0, ids = 0, late = 0
picked.forEach((p, i) => {
asks += 1
const cands = bm25Rank(live, p.text, MAX_CANDS)
const a = haiku(rerankPrompt(p.text, cands))
const got = parseIds(a?.text ?? '', cands, MAX_PICKS)
const byId = new Map(cands.map(c => [c.id, c]))
const block = recallBlock(
got.flatMap(id => byId.get(id) ?? []).map(recallLine), RECALL_CAP)
const slow = a !== undefined && a.ms > RERANK_MS
if (a !== undefined) answered += 1
if (slow) late += 1
if (block !== '') { inj += 1; ids += got.length }
console.log(`#${i} [${p.fresh ? 'fresh' : 'idle/rule'}] ` +
`"${oneLine(p.text, 120)}"`)
for (const id of got) console.log(` ${id} ${byId.get(id)?.title}`)
console.log(` duration_api_ms=${a?.ms ?? 'n/a'} (upper bound) ` +
`timeout(>${RERANK_MS})=${a === undefined ? 'failed' : slow ? 'yes' : 'no'}`)
})
console.log(`TOTAL asks=${asks} answered=${answered} injections=${inj} ` +
`mean IDs per injection=${inj ? (ids / inj).toFixed(2) : 'n/a'} ` +
`timeout share=${share(late, answered)} injection share=` +
share(inj, asks))
}
function errorIndex(status: StatusMap): Map<string, string> {
const texts = registryTexts(['blockers.md', 'learnings.md'])
return buildIndex(assembleEntries(texts, status), status)
}
function runErrors(): void {
const status = statuses()
const index = errorIndex(status)
const titles = new Map(
registryTexts(['blockers.md', 'learnings.md']).flatMap(entriesOf)
.map(p => [p.id, status.get(p.id)?.title ?? ''] as const))
const seen = new Map<string, Set<string>>()
const perSession = new Map<string, number>()
let failures = 0
for (const f of findFiles()) {
const ledger = `${f.sid}|${f.agent}`
for (const x of failuresOf(f.file)) {
const key = failureKey(x.text, x.command)
if (key === undefined) continue
failures += 1
const id = matchOf(index, key)
const done = seen.get(ledger) ?? new Set<string>()
if (id === undefined || done.has(id)) continue
seen.set(ledger, done.add(id))
perSession.set(f.sid, (perSession.get(f.sid) ?? 0) + 1)
console.log(`${f.sid.slice(0, 8)}${f.agent ? ' sub' : ''}: ` +
`"${oneLine(key, 120)}" -> ${id} ${titles.get(id) ?? ''}`)
}
}
const total = [...perSession.values()].reduce((a, b) => a + b, 0)
console.log('per session: ' + [...perSession]
.map(([s, n]) => `${s.slice(0, 8)}=${n}`).join(' '))
console.log(`TOTAL failures keyed=${failures} known-line injections=` +
`${share(total, failures)} sessions with injection=` +
`${perSession.size}`)
}
const mode = process.argv[2]
if (mode === 'prompts') runPrompts()
else if (mode === 'errors') runErrors()
else console.error('usage: recall-bench.ts prompts|errors')