forked from bchanot/claude
chore(bench): phase 4c recall measured on 14 days of transcripts, content floor gated
This commit is contained in:
1 parent
f9787044ff
commit
adb3db97cb
3 files changed
+297
No files matched your search
@@ -46,6 +46,8 @@ STEP 3 gate `[gated 2026-11-05]` (user: "oui"): plan r3 (`docs/superpowers/plans
|
||||
|
||||
Digest cap order `[gated 2026-11-05]` (user: "Retirer d'abord les TODO"): over 800 chars the digest drops TODO lines first (last to first), then open-BLK lines; the header, the `+N` count line and the journal line stay.
|
||||
|
||||
Measurement gate `[gated 2026-11-05]` (RECALL.md: prompt channel 19/38 = 50 % relevant < 64 % bar, noise on content-free idle prompts; timeout share 4/20 = 20 %; user: "Plancher de contenu"): no intent recall on a prompt with fewer than 5 content words (`tokensOf` minus stop words); recall ships on and switchable; the 3 s bound stays; 4d reads the injection log.
|
||||
|
||||
9. `[gated 2026-11-05]` Mod tests: `mods/memory/hooks/register.test.ts` holds the plan's named tests and the mods suite is green; no line of register.ts over 80 columns.
|
||||
CHECK: cd /home/bchanot/Documents/claude && T=mods/memory/hooks/register.test.ts && for n in SR1 SR3 SR5 LS1 LS4 ER1 ER3 RB1 RB3 RB5 IR1 IR5 IR6 IR7 IR8 IR9 IR11 TE1 TE2 TE4 PM1 PM2; do /usr/bin/grep -q "$n" "$T" || exit 1; done && [ "$(awk 'length($0) > 80' mods/memory/hooks/register.ts | wc -l)" = 0 ] && a=$(make test suite=lib/tests/mods.test.sh 2>&1) && printf '%s\n' "$a" | /usr/bin/grep -q 'all suites green' && echo MOD4C-OK
|
||||
EXPECT: MOD4C-OK
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
# Phase 4c recall measured on 14 days of transcripts (2026-11-05)
|
||||
|
||||
Harness: `recall-bench.ts` (bun) imports the mod's own functions (`isTrigger`, `bm25Rank`, `rerankPrompt`, `parseIds`, `assembleEntries`, `failureKey`, `buildIndex`, `matchOf`…); statuses from `lib/memory.sh index`; haiku `claude-haiku-5-5` through `claude -p` from a `mktemp -d` cwd with `--setting-sources local`, no tools. Counts and shares only; no prompt or error text here (LRN-124).
|
||||
|
||||
## Prompt channel
|
||||
|
||||
Population: main-loop transcripts with `entrypoint == "cli"`, user entries with `origin.kind == "human"` (typed prompts), not meta, not compaction summaries: 65 prompts, 11 older entries skipped (no `origin` field), 24 meet the trigger, 20 sampled (seed 42). A first run sampled `<task-notification>` and summary entries (origin `task-notification` / `isCompactSummary`, never seen by the mod's `composer` gate): discarded as a measurement.
|
||||
|
||||
| measure | value |
|
||||
|---|---|
|
||||
| asks | n = 20 |
|
||||
| injections (prompts with ≥ 1 entry) | 15 / 20 |
|
||||
| entries injected | n = 38 (2.5 per injection) |
|
||||
| relevant entries (controller labels) | 19 / 38 = 50 % |
|
||||
| relevant on prompts with ≥ 5 content words (estimate) | 16 / 28 = 57 % |
|
||||
| bar (BDR-125 amendment 2, haiku rerank baseline) | 14 / 22 = 64 % |
|
||||
| haiku `duration_api_ms` > 3000 (upper bound) | 4 / 20 = 20 % (first run: 7 / 20) |
|
||||
|
||||
The noise sits on content-free prompts triggered by idle time (short approvals): 0 relevant of 10 injected. Prompts with a topic recall the right entries. Caveat: the harness reads today's registries, so entries written after a prompt can be recalled for it (temporal leakage): the true shares are somewhat lower.
|
||||
|
||||
## Error channel
|
||||
|
||||
36 Bash failures keyed over 14 days (main + sub-agents), 3 known-line injections (2 sessions): one exact lesson, two plausible. n = 3: insufficient sample.
|
||||
|
||||
## Decision (2026-11-05)
|
||||
|
||||
User: "Plancher de contenu" — no intent recall on a prompt with fewer than 5 content words (stop words excluded); recall ships on and switchable (`"recall": false`); the injection log (4c) feeds the 4d measurement on real use. The 3 s rerank bound stays (timeout share 20 % < 25 %).
|
||||
@@ -0,0 +1,268 @@
|
||||
// Measurement harness of the phase 4c memory recall (read-only, stdout).
|
||||
// Transcript walk follows noise.py (2026-11-03-memory-capture-spike):
|
||||
// its find_files / pairs_of logic, ported to TS.
|
||||
// Run: bun recall-bench.ts prompts [--seed N] [--n 20]
|
||||
// bun recall-bench.ts errors
|
||||
import * as fs from 'node:fs'
|
||||
import * as os from 'node:os'
|
||||
import * as path from 'node:path'
|
||||
import {
|
||||
IDLE_MS, RECALL_CAP, HAIKU, RERANK_MS, assembleEntries, bm25Rank,
|
||||
buildIndex, entriesOf, failureKey, isLive, isTrigger, matchOf, parseIds,
|
||||
parseIndex, recallBlock, recallLine, redact, rerankPrompt,
|
||||
type Entry, type StatusMap,
|
||||
} from '../../../../mods/memory/hooks/register.ts'
|
||||
|
||||
const REPO = path.resolve(import.meta.dir, '../../../..')
|
||||
const ROOT = path.join(
|
||||
os.homedir(), '.claude/projects/-home-bchanot-Documents-claude')
|
||||
const DAYS = 14
|
||||
const MAX_CANDS = 30
|
||||
const MAX_PICKS = 3
|
||||
const MIN_SAMPLE = 10
|
||||
const MEMORY_DIR = path.join(REPO, '.claude/memory')
|
||||
|
||||
type Rec = Record<string, any>
|
||||
type Found = { sid: string; agent: boolean; file: string }
|
||||
type Composer = { sid: string; text: string; fresh: boolean; idle: number }
|
||||
|
||||
function findFiles(): Found[] {
|
||||
const cutoff = Date.now() - DAYS * 86400000
|
||||
const recent = (p: string) =>
|
||||
(fs.statSync(p, { throwIfNoEntry: false })?.mtimeMs ?? 0) >= cutoff
|
||||
const found: Found[] = []
|
||||
for (const name of fs.readdirSync(ROOT).sort()) {
|
||||
const full = path.join(ROOT, name)
|
||||
if (name.endsWith('.jsonl') && recent(full)) {
|
||||
found.push({ sid: name.slice(0, -6), agent: false, file: full })
|
||||
}
|
||||
if (name.endsWith('.jsonl')) continue
|
||||
const sub = path.join(full, 'subagents')
|
||||
if (!fs.statSync(sub, { throwIfNoEntry: false })?.isDirectory()) continue
|
||||
for (const f of fs.readdirSync(sub).sort()) {
|
||||
const p = path.join(sub, f)
|
||||
if (f.endsWith('.jsonl') && recent(p)) {
|
||||
found.push({ sid: name, agent: true, file: p })
|
||||
}
|
||||
}
|
||||
}
|
||||
return found
|
||||
}
|
||||
|
||||
function records(file: string): Rec[] {
|
||||
const out: Rec[] = []
|
||||
for (const raw of fs.readFileSync(file, 'utf8').split('\n')) {
|
||||
try { out.push(JSON.parse(raw)) } catch { /* skip bad line */ }
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
function blocksOf(r: Rec): Rec[] {
|
||||
const c = r.message?.content
|
||||
return Array.isArray(c) ? c.filter(b => b && typeof b === 'object') : []
|
||||
}
|
||||
|
||||
function resultText(content: unknown): string {
|
||||
if (typeof content === 'string') return content
|
||||
if (!Array.isArray(content)) return ''
|
||||
return content.filter(b => b?.type === 'text').map(b => b.text).join('')
|
||||
}
|
||||
|
||||
/** Bash failures: tool_use paired with its tool_result (is_error). */
|
||||
function failuresOf(file: string): { text: string; command: string }[] {
|
||||
const calls = new Map<string, string>()
|
||||
const out: { text: string; command: string }[] = []
|
||||
for (const r of records(file)) {
|
||||
for (const b of blocksOf(r)) {
|
||||
if (b.type === 'tool_use' && b.name === 'Bash') {
|
||||
calls.set(b.id, String(b.input?.command ?? ''))
|
||||
} else if (b.type === 'tool_result' && calls.has(b.tool_use_id) &&
|
||||
b.is_error === true) {
|
||||
out.push({ text: resultText(b.content),
|
||||
command: calls.get(b.tool_use_id) ?? '' })
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
const WRAPPERS = ['<command-', '<local-command', '<system-reminder']
|
||||
|
||||
const skipped = { noOrigin: 0 }
|
||||
|
||||
/** A typed prompt: origin.kind human, string content, not a command. */
|
||||
function composerText(r: Rec): string | undefined {
|
||||
if (r.type !== 'user' || r.isSidechain === true) return undefined
|
||||
if (r.isMeta === true || r.isCompactSummary === true) return undefined
|
||||
if (r.entrypoint !== 'cli') return undefined
|
||||
const c = r.message?.content
|
||||
if (typeof c !== 'string') return undefined
|
||||
if (r.origin?.kind === undefined) {
|
||||
skipped.noOrigin += 1
|
||||
return undefined
|
||||
}
|
||||
const t = c.trim()
|
||||
if (r.origin.kind !== 'human' || t.startsWith('/')) return undefined
|
||||
return WRAPPERS.some(w => t.startsWith(w)) ? undefined : t
|
||||
}
|
||||
|
||||
/** Trigger-meeting typed prompts; fresh = first after a summary or start. */
|
||||
function triggerPrompts(files: Found[]): { all: number; hits: Composer[] } {
|
||||
const hits: Composer[] = []
|
||||
let all = 0
|
||||
for (const f of files.filter(x => !x.agent)) {
|
||||
let last: number | undefined
|
||||
let fresh = true
|
||||
for (const r of records(f.file)) {
|
||||
if (r.isCompactSummary === true) { fresh = true; continue }
|
||||
const text = composerText(r)
|
||||
if (text === undefined) continue
|
||||
all += 1
|
||||
const at = Date.parse(r.timestamp)
|
||||
const idle = last === undefined || Number.isNaN(at) ? 0 : at - last
|
||||
if (!Number.isNaN(at)) last = at
|
||||
if (isTrigger(text, fresh, idle)) {
|
||||
hits.push({ sid: f.sid, text, fresh, idle })
|
||||
}
|
||||
fresh = false
|
||||
}
|
||||
}
|
||||
return { all, hits }
|
||||
}
|
||||
|
||||
function rng(seed: number): () => number {
|
||||
let a = seed >>> 0
|
||||
return () => {
|
||||
a = (a + 0x6d2b79f5) >>> 0
|
||||
let t = Math.imul(a ^ (a >>> 15), 1 | a)
|
||||
t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t
|
||||
return ((t ^ (t >>> 14)) >>> 0) / 4294967296
|
||||
}
|
||||
}
|
||||
|
||||
function sample<T>(items: T[], n: number, seed: number): T[] {
|
||||
const pool = [...items]
|
||||
const next = rng(seed)
|
||||
for (let i = pool.length - 1; i > 0; i--) {
|
||||
const j = Math.floor(next() * (i + 1));
|
||||
[pool[i], pool[j]] = [pool[j]!, pool[i]!]
|
||||
}
|
||||
return pool.slice(0, n)
|
||||
}
|
||||
|
||||
function statuses(): StatusMap {
|
||||
const r = Bun.spawnSync(['bash', 'lib/memory.sh', '-C', '.', 'index'],
|
||||
{ cwd: REPO })
|
||||
if (r.exitCode !== 0) throw new Error('memory.sh index failed')
|
||||
return parseIndex(r.stdout.toString())
|
||||
}
|
||||
|
||||
function registryTexts(names: string[]): string[] {
|
||||
return names.map(n => fs.readFileSync(path.join(MEMORY_DIR, n), 'utf8'))
|
||||
}
|
||||
|
||||
function haiku(prompt: string): { text: string; ms: number } | undefined {
|
||||
const dir = fs.mkdtempSync(os.tmpdir() + '/recall-')
|
||||
try {
|
||||
const r = Bun.spawnSync(['claude', '-p', '--model', HAIKU,
|
||||
'--output-format', 'json', '--setting-sources', 'local',
|
||||
'--tools', ''], { cwd: dir, stdin: new TextEncoder().encode(prompt) })
|
||||
const j = JSON.parse(r.stdout.toString())
|
||||
return { text: String(j.result ?? ''), ms: Number(j.duration_api_ms) }
|
||||
} catch {
|
||||
return undefined
|
||||
} finally {
|
||||
fs.rmSync(dir, { recursive: true, force: true })
|
||||
}
|
||||
}
|
||||
|
||||
function argOf(name: string, fallback: number): number {
|
||||
const i = process.argv.indexOf(name)
|
||||
const v = i < 0 ? NaN : Number(process.argv[i + 1])
|
||||
return Number.isFinite(v) ? v : fallback
|
||||
}
|
||||
|
||||
function share(k: number, n: number): string {
|
||||
return n < MIN_SAMPLE ? `${k}/${n} (insufficient sample, < ${MIN_SAMPLE})`
|
||||
: `${k}/${n}`
|
||||
}
|
||||
|
||||
function oneLine(text: string, n: number): string {
|
||||
return redact(text).replace(/\s+/g, ' ').slice(0, n)
|
||||
}
|
||||
|
||||
function runPrompts(): void {
|
||||
const status = statuses()
|
||||
const entries = assembleEntries(
|
||||
registryTexts(['decisions.md', 'learnings.md', 'blockers.md']), status)
|
||||
const live = entries.filter(e => isLive(status, e.id))
|
||||
const { all, hits } = triggerPrompts(findFiles())
|
||||
const picked = sample(hits, argOf('--n', 20), argOf('--seed', 1))
|
||||
console.log(`human prompts (cli): ${all}, skipped no origin: ${skipped.noOrigin}, meeting isTrigger: ` +
|
||||
`${hits.length}, sampled: ${picked.length}, live entries: ${live.length}`)
|
||||
let asks = 0, answered = 0, inj = 0, ids = 0, late = 0
|
||||
picked.forEach((p, i) => {
|
||||
asks += 1
|
||||
const cands = bm25Rank(live, p.text, MAX_CANDS)
|
||||
const a = haiku(rerankPrompt(p.text, cands))
|
||||
const got = parseIds(a?.text ?? '', cands, MAX_PICKS)
|
||||
const byId = new Map(cands.map(c => [c.id, c]))
|
||||
const block = recallBlock(
|
||||
got.flatMap(id => byId.get(id) ?? []).map(recallLine), RECALL_CAP)
|
||||
const slow = a !== undefined && a.ms > RERANK_MS
|
||||
if (a !== undefined) answered += 1
|
||||
if (slow) late += 1
|
||||
if (block !== '') { inj += 1; ids += got.length }
|
||||
console.log(`#${i} [${p.fresh ? 'fresh' : 'idle/rule'}] ` +
|
||||
`"${oneLine(p.text, 120)}"`)
|
||||
for (const id of got) console.log(` ${id} ${byId.get(id)?.title}`)
|
||||
console.log(` duration_api_ms=${a?.ms ?? 'n/a'} (upper bound) ` +
|
||||
`timeout(>${RERANK_MS})=${a === undefined ? 'failed' : slow ? 'yes' : 'no'}`)
|
||||
})
|
||||
console.log(`TOTAL asks=${asks} answered=${answered} injections=${inj} ` +
|
||||
`mean IDs per injection=${inj ? (ids / inj).toFixed(2) : 'n/a'} ` +
|
||||
`timeout share=${share(late, answered)} injection share=` +
|
||||
share(inj, asks))
|
||||
}
|
||||
|
||||
function errorIndex(status: StatusMap): Map<string, string> {
|
||||
const texts = registryTexts(['blockers.md', 'learnings.md'])
|
||||
return buildIndex(assembleEntries(texts, status), status)
|
||||
}
|
||||
|
||||
function runErrors(): void {
|
||||
const status = statuses()
|
||||
const index = errorIndex(status)
|
||||
const titles = new Map(
|
||||
registryTexts(['blockers.md', 'learnings.md']).flatMap(entriesOf)
|
||||
.map(p => [p.id, status.get(p.id)?.title ?? ''] as const))
|
||||
const seen = new Map<string, Set<string>>()
|
||||
const perSession = new Map<string, number>()
|
||||
let failures = 0
|
||||
for (const f of findFiles()) {
|
||||
const ledger = `${f.sid}|${f.agent}`
|
||||
for (const x of failuresOf(f.file)) {
|
||||
const key = failureKey(x.text, x.command)
|
||||
if (key === undefined) continue
|
||||
failures += 1
|
||||
const id = matchOf(index, key)
|
||||
const done = seen.get(ledger) ?? new Set<string>()
|
||||
if (id === undefined || done.has(id)) continue
|
||||
seen.set(ledger, done.add(id))
|
||||
perSession.set(f.sid, (perSession.get(f.sid) ?? 0) + 1)
|
||||
console.log(`${f.sid.slice(0, 8)}${f.agent ? ' sub' : ''}: ` +
|
||||
`"${oneLine(key, 120)}" -> ${id} ${titles.get(id) ?? ''}`)
|
||||
}
|
||||
}
|
||||
const total = [...perSession.values()].reduce((a, b) => a + b, 0)
|
||||
console.log('per session: ' + [...perSession]
|
||||
.map(([s, n]) => `${s.slice(0, 8)}=${n}`).join(' '))
|
||||
console.log(`TOTAL failures keyed=${failures} known-line injections=` +
|
||||
`${share(total, failures)} sessions with injection=` +
|
||||
`${perSession.size}`)
|
||||
}
|
||||
|
||||
const mode = process.argv[2]
|
||||
if (mode === 'prompts') runPrompts()
|
||||
else if (mode === 'errors') runErrors()
|
||||
else console.error('usage: recall-bench.ts prompts|errors')
|
||||
Reference in new issue
Block a user