diff --git a/lib/memory-feedback/metrics.test.ts b/lib/memory-feedback/metrics.test.ts index 01514bb..c64aedd 100644 --- a/lib/memory-feedback/metrics.test.ts +++ b/lib/memory-feedback/metrics.test.ts @@ -118,3 +118,14 @@ test('M9: FEEDBACK.md counts, no repo tag without a noisy entry', () => { expect(text).not.toContain('client#abcdef') expect(text).toContain('Prompt gate: open (1/20)') }) + +test('M10: FEEDBACK.md shows n per channel and trigger', () => { + const rows = [...ask(['relevant', 'irrelevant']), + ...ask(['irrelevant'], { trigger: 'idle' })] + const mm = metricsOf(rows) + const text = renderFeedback(mm, gateVerdict(mm, 1), 0) + expect(text).toContain('| precision, idle | 0 % (n=1) |') + expect(text).toContain('| precision, fresh | 50 % (n=2) |') + expect(text).toContain('| prompt precision | 33 % (n=3) |') + expect(text).toContain('| tool precision | n/a (n=0) |') +}) diff --git a/lib/memory-feedback/metrics.ts b/lib/memory-feedback/metrics.ts index 3552df4..ff025af 100644 --- a/lib/memory-feedback/metrics.ts +++ b/lib/memory-feedback/metrics.ts @@ -10,11 +10,13 @@ export type Gate = 'open' | 'reached' export type ToolGate = Gate | 'extended' | 'insufficient' export type Verdict = { prompt: Gate; tool: ToolGate } type Rate = number | undefined +type Sized = { rate: Rate; n: number } export type Metrics = { start?: number; promptN: number; toolN: number; lost: number - precision: Rate; byTrigger: Record; thirdPick: Rate + precision: Rate; byTrigger: Record; thirdPick: Rate late: Rate; msP90: Rate; tool: Rate; missShare: Rate; sampled: number noisy: string[]; promote: string[] + n: { prompt: number; third: number; tool: number } // entries } const share = (k: number, n: number): Rate => n === 0 ? undefined : k / n @@ -52,14 +54,22 @@ export function p90(values: number[]): Rate { return s[Math.ceil(0.9 * s.length) - 1] } -function byTrigger(entries: Row[]): Record { - const out: Record = {} +function byTrigger(entries: Row[]): Record { + const out: Record = {} for (const k of ['fresh', 'idle', 'keyword']) { - out[k] = precision(entries.filter(r => r.trigger === k)) + const mine = entries.filter(r => r.trigger === k) + out[k] = { rate: precision(mine), n: mine.length } } return out } +/** Entry counts behind the three channel precisions. */ +const countsOf = (entries: Row[]) => ({ + prompt: entries.filter(r => r.channel === 'prompt').length, + third: entries.filter(r => r.channel === 'prompt' && r.pick === 3).length, + tool: entries.filter(r => r.channel === 'tool').length, +}) + /** timeout + paused among asks (lost ones too) that had candidates. */ function lateShare(asks: Row[]): Rate { const had = asks.filter(r => (r.cands ?? 0) > 0) @@ -117,6 +127,7 @@ export function metricsOf(all: Row[]): Metrics { tool: precision(entries.filter(r => r.channel === 'tool')), missShare: miss.rate, sampled: miss.sampled, noisy: noisy(entries), promote: promote(entries), + n: countsOf(entries), } } @@ -137,10 +148,10 @@ export function gateVerdict(m: Metrics, now: number): Verdict { } /** The worst trigger under the floor; ties go idle, keyword, fresh. */ -function worstTrigger(by: Record): string | undefined { +function worstTrigger(by: Record): string | undefined { const low = ['idle', 'keyword', 'fresh'] - .filter(k => under(by[k], T.triggerFloor)) - low.sort((a, b) => (by[a] ?? 0) - (by[b] ?? 0)) + .filter(k => under(by[k]?.rate, T.triggerFloor)) + low.sort((a, b) => (by[a]?.rate ?? 0) - (by[b]?.rate ?? 0)) return low[0] } @@ -195,13 +206,13 @@ export function renderFeedback(m: Metrics, v: Verdict, now: number) { `Prompt gate: ${v.prompt} (${m.promptN}/${PROMPT_GATE_N})`, toolLine(m, v), `Lost records: ${m.lost}`, '', '| Signal | Value |', '|---|---|', - `| prompt precision | ${pct(m.precision)} |`, + `| prompt precision | ${pct(m.precision)} (n=${m.n.prompt}) |`, ...Object.entries(m.byTrigger).map(([k, x]) => - `| precision, ${k} | ${pct(x)} |`), - `| 3rd pick precision | ${pct(m.thirdPick)} |`, + `| precision, ${k} | ${pct(x.rate)} (n=${x.n}) |`), + `| 3rd pick precision | ${pct(m.thirdPick)} (n=${m.n.third}) |`, `| timeout + paused | ${pct(m.late)} |`, `| rerank ms p90 | ${m.msP90 ?? 'n/a'} |`, - `| tool precision | ${pct(m.tool)} |`, + `| tool precision | ${pct(m.tool)} (n=${m.n.tool}) |`, `| asks with a miss | ${pct(m.missShare)} of ${m.sampled} |`, '', `Noisy: ${m.noisy.join(', ') || 'none'}`, `Promotion candidates: ${m.promote.join(', ') || 'none'}`, '', diff --git a/lib/memory-feedback/reader.ts b/lib/memory-feedback/reader.ts index 10bde18..6114d26 100644 --- a/lib/memory-feedback/reader.ts +++ b/lib/memory-feedback/reader.ts @@ -13,7 +13,8 @@ import { } from '../../mods/memory/hooks/register.ts' import { appendRows, keyOf, parseDraft, readRows, readState, rowsOf, skeleton, - tagOf, writeSafe, writeState, type Item, type State, + tagOf, writeSafe, writeState, type Draft, type Item, type Row, + type State, } from './labels.ts' import { clockStart, gateVerdict, metricsOf, renderFeedback, type Verdict, @@ -56,8 +57,7 @@ export function storeFile(dir: string): string { return files.sort((a, b) => mtime(b) - mtime(a))[0] as string } -/** Every valid log record of every root, oldest first. */ -export function readLogs(file: string): Logged[] { +function readStore(file: string): object { let data: unknown try { data = JSON.parse(fs.readFileSync(file, 'utf8')) @@ -67,13 +67,16 @@ export function readLogs(file: string): Logged[] { if (typeof data !== 'object' || data === null) { throw new Error(`store file is not an object: ${file}`) } - const out: Logged[] = [] - for (const [k, v] of Object.entries(data)) { - if (!k.startsWith(LOG_PREFIX) || !Array.isArray(v)) continue - const root = k.slice(LOG_PREFIX.length) - for (const rec of v.filter(isLogRecord)) out.push({ root, rec }) - } - return out.sort((a, b) => a.rec.at - b.rec.at) + return data +} + +/** Every valid log record of every root, oldest first. */ +export function readLogs(file: string): Logged[] { + return Object.entries(readStore(file)) + .filter(([k, v]) => k.startsWith(LOG_PREFIX) && Array.isArray(v)) + .flatMap(([k, v]) => (v as unknown[]).filter(isLogRecord) + .map(rec => ({ root: k.slice(LOG_PREFIX.length), rec }))) + .sort((a, b) => a.rec.at - b.rec.at) } /** Unlabelled records from the clock start, AGE_FLOOR_MS old or more. */ @@ -109,6 +112,8 @@ function header(b: Base, repo: string, lost: boolean): string { const ids = b.ids.join(' ') || '-' return `## Item ${b.n}: ${b.channel}/${b.trigger}/${b.outcome}` + ` · cands ${b.cands ?? '-'} · ms ${b.ms ?? '-'}` + + ` · s:${b.session.slice(0, 8)}` + + ` · ${new Date(b.at).toISOString().slice(11, 16)} UTC` + ` · ${b.root} (${repo})` + ` · ids ${ids}${lost ? ' · LOST (no transcript turn)' : ''}` } @@ -188,24 +193,34 @@ function sheetHead(n: number, torn: number, evicted: string[]): string { return lines.join('\n') } -/** Writes worksheet.md, items.json, labels-draft.tsv (0600); the count. */ -export function worksheet(dir: string, p: Paths, now: number): number { +const logAtsOf = (logs: Logged[]) => + logs.filter(l => l.rec.cands !== undefined).map(l => l.rec.at) + +/** Joins the unlabelled population to its transcripts. */ +function collect(p: Paths, now: number) { const rows = readRows(labelsOf(p)) const logs = readLogs(storeFile(p.store)) - const logAts = logs.filter(l => l.rec.cands !== undefined) - .map(l => l.rec.at) const pop = population(logs, new Set(rows.map(keyOf)), - clockStart(rows, logAts), now) + clockStart(rows, logAtsOf(logs)), now) const loader = loaderFor(p.projects) const joined = pop.map((l, i) => joinOne(l, i + 1, loader.load)) - const head = sheetHead(pop.length, loader.torn(), - evictions(logs, readState(stateOf(p)).lastRun)) + return { joined, head: sheetHead(pop.length, loader.torn(), + evictions(logs, readState(stateOf(p)).lastRun)) } +} + +function writeSheet(dir: string, joined: Joined[], head: string): void { writeSafe(path.join(dir, 'worksheet.md'), ['# Memory feedback worksheet', head, DRAFT_HELP, ...joined.map(j => j.text)].join('\n\n') + '\n') const items = joined.map(j => j.item) writeSafe(path.join(dir, 'items.json'), JSON.stringify(items)) writeSafe(path.join(dir, 'labels-draft.tsv'), skeleton(items)) - return pop.length +} + +/** Writes worksheet.md, items.json, labels-draft.tsv (0600); the count. */ +export function worksheet(dir: string, p: Paths, now: number): number { + const { joined, head } = collect(p, now) + writeSheet(dir, joined, head) + return joined.length } const draftOf = (dir: string) => { @@ -230,26 +245,45 @@ function nextState(st: State, v: Verdict, now: number) { return { state: { lastRun: now, promptAt, toolAt, closed }, news } } -/** Validates the draft; writes state, then labels, then FEEDBACK.md. */ +const assertFresh = (items: Item[], old: Row[]) => { + const done = new Set(old.map(keyOf)) + if (items.some(i => done.has(keyOf(i)))) { + throw new Error('items already applied: run worksheet again') + } +} + +/** The new rows, and the metrics and verdict once they are added. */ +function projected( + items: Item[], drafts: Draft[], old: Row[], now: number, +) { + const fresh = items.flatMap(i => rowsOf(i, drafts)) + const m = metricsOf([...old, ...fresh]) + return { fresh, m, v: gateVerdict(m, now) } +} + +/** Appends labels, then writes state, then FEEDBACK.md. */ +function commit( + p: Paths, items: Item[], drafts: Draft[], now: number, +): string { + const old = readRows(labelsOf(p)) + assertFresh(items, old) + const { fresh, m, v } = projected(items, drafts, old, now) + const next = nextState(readState(stateOf(p)), v, now) + const feedback = renderFeedback(m, v, now) + if (fresh.length > 0) appendRows(labelsOf(p), fresh) + writeState(stateOf(p), next.state) + writeSafe(path.join(p.out, 'FEEDBACK.md'), feedback) + return feedback + next.news.map(n => n + '\n').join('') +} + +/** Validates the draft; appends labels, writes state, then FEEDBACK.md + * (a failed state write leaves the NEW line to print on the next run). */ export function apply(dir: string, p: Paths, now: number): string { const items: Item[] = JSON.parse( fs.readFileSync(path.join(dir, 'items.json'), 'utf8')) const drafts = parseDraft(draftOf(dir), items) if (typeof drafts === 'string') throw new Error(`draft refused: ${drafts}`) - const old = readRows(labelsOf(p)) - const done = new Set(old.map(keyOf)) - if (items.some(i => done.has(keyOf(i)))) { - throw new Error('items already applied: run worksheet again') - } - const fresh = items.flatMap(i => rowsOf(i, drafts)) - const m = metricsOf([...old, ...fresh]) - const v = gateVerdict(m, now) - const { state, news } = nextState(readState(stateOf(p)), v, now) - const feedback = renderFeedback(m, v, now) - writeState(stateOf(p), state) - if (fresh.length > 0) appendRows(labelsOf(p), fresh) - writeSafe(path.join(p.out, 'FEEDBACK.md'), feedback) - return feedback + news.map(n => n + '\n').join('') + return commit(p, items, drafts, now) } /** Up to `n` entry lines of the draft, picked at random. */ @@ -269,6 +303,9 @@ export function checkDir(dir: string | undefined): string { if (!fs.existsSync(dir)) return 'does not exist' const git = spawnSync('git', ['-C', dir, 'rev-parse', '--is-inside-work-tree'], { encoding: 'utf8' }) + if (git.error !== undefined || typeof git.stdout !== 'string') { + return 'git unavailable' + } return git.stdout.trim() === 'true' ? 'inside a git work tree' : '' } diff --git a/lib/memory-feedback/transcripts.test.ts b/lib/memory-feedback/transcripts.test.ts index c014137..d919879 100644 --- a/lib/memory-feedback/transcripts.test.ts +++ b/lib/memory-feedback/transcripts.test.ts @@ -41,6 +41,9 @@ test('X1: slug, the main or one sub-agent file, torn lines counted', () => { expect(sessionFiles(p, '/r', 's', 'a1').map(f => path.basename(f))) .toEqual(['agent-a1.jsonl']) expect(records(main[0]!)).toEqual({ recs: [{ a: 1 }], torn: 1 }) + fs.writeFileSync(path.join(dir, 'n.jsonl'), '{"a":1}\nnull\n[1]\n7\n') + expect(records(path.join(dir, 'n.jsonl'))) + .toEqual({ recs: [{ a: 1 }], torn: 3 }) expect(sessionFiles(p, '/r', 'gone', 'main')).toEqual([]) }) diff --git a/lib/memory-feedback/transcripts.ts b/lib/memory-feedback/transcripts.ts index 127587d..57999f6 100644 --- a/lib/memory-feedback/transcripts.ts +++ b/lib/memory-feedback/transcripts.ts @@ -28,13 +28,19 @@ export function sessionFiles( return fs.existsSync(file) ? [file] : [] } -/** Parsed lines of a transcript; torn or partial lines are counted. */ +/** Parsed lines of a transcript; torn, partial or non-object lines + * (`null`, a number, an array) are counted as torn. */ export function records(file: string): Loaded { const recs: Rec[] = [] let torn = 0 for (const raw of fs.readFileSync(file, 'utf8').split('\n')) { if (raw.trim() === '') continue - try { recs.push(JSON.parse(raw)) } catch { torn += 1 } + try { + const v = JSON.parse(raw) + if (v === null || typeof v !== 'object' || Array.isArray(v)) { + torn += 1 + } else recs.push(v) + } catch { torn += 1 } } return { recs, torn } } @@ -90,7 +96,8 @@ function promptNear(recs: Rec[], ts: number): Prompt | undefined { for (const r of recs) { const text = composerText(r) const t = tsOf(r) - if (text === undefined || Math.abs(t - ts) > NEAR_MS) continue + if (text === undefined || !Number.isFinite(t)) continue + if (Math.abs(t - ts) > NEAR_MS) continue if (best === undefined || Math.abs(t - ts) < Math.abs(best.t - ts)) { best = { t, text } } @@ -108,22 +115,22 @@ function nextPromptAt(recs: Rec[], ts: number): number { return end } +/** The [dedupe key, text] of each text block of an assistant record. */ +const textsOf = (r: Rec): [string, string][] => blocksOf(r) + .filter(b => b.type === 'text') + .map(b => [`${r.message?.id}:${b.text}`, String(b.text)]) // EVAL-037 + /** Main-thread assistant text after `ts` up to the next prompt. */ function replyAfter(recs: Rec[], ts: number): string { const end = nextPromptAt(recs, ts) const seen = new Set() - const parts: string[] = [] - for (const r of recs) { - const t = tsOf(r) - if (r.type !== 'assistant' || r.isSidechain === true) continue - if (!(t > ts && t < end)) continue - for (const b of blocksOf(r).filter(b => b.type === 'text')) { - const key = `${r.message?.id}:${b.text}` // EVAL-037 - if (!seen.has(key)) parts.push(String(b.text)) - seen.add(key) - } - } - return parts.join('\n') + return recs + .filter(r => r.type === 'assistant' && r.isSidechain !== true) + .filter(r => tsOf(r) > ts && tsOf(r) < end) + .flatMap(textsOf) + .filter(([key]) => !seen.has(key) && !!seen.add(key)) + .map(([, text]) => text) + .join('\n') } export type PromptJoin = { prompt: string; injected: string; reply: string } @@ -178,7 +185,7 @@ export type ToolJoin = { command: string; known: string; next: boolean[] } /** The failure a known line answered and the next 3 Bash results. */ export function joinTool(recs: Rec[], rec: LogRecord): ToolJoin | undefined { - const needle = KNOWN_HEAD + (rec.ids[0] ?? '') + const needle = `${KNOWN_HEAD}${rec.ids[0] ?? ''} ` // BLK-003 != BLK-0031 const att = attachmentAfter(recs, rec.at, 'tool.call', needle) const raw = att?.attachment?.toolUseID if (att === undefined || typeof raw !== 'string') return undefined diff --git a/lib/tests/memory-feedback-remind.test.sh b/lib/tests/memory-feedback-remind.test.sh index 37f18b2..37dc40e 100755 --- a/lib/tests/memory-feedback-remind.test.sh +++ b/lib/tests/memory-feedback-remind.test.sh @@ -13,8 +13,10 @@ check() { if [ "$2" = "$3" ]; then pass=$((pass+1)); else fail=$((fail+1)); printf 'FAIL %s: got[%s] want[%s]\n' "$1" "$2" "$3"; fi; } # home : a fixture HOME with a store file (unless "nostore") home() { - mkdir -p "$WORK/$1/.claude/plugins/store" "$WORK/$1/.claude/memory-feedback" - [ "$1" = nostore ] || : > "$WORK/$1/.claude/plugins/store/memory_skills-dir-a.json" + mkdir -p "$WORK/$1/.claude/plugins/store" \ + "$WORK/$1/.claude/memory-feedback" + [ "$1" = nostore ] || + : > "$WORK/$1/.claude/plugins/store/memory_skills-dir-a.json" echo "$WORK/$1" } OPEN='{"lastRun":1,"closed":false}' @@ -24,24 +26,29 @@ old() { touch -t 202601010000 "$1/.claude/memory-feedback/state.json" } run() { HOME="$1" bash "$LIB" 2>&1; echo "rc=$?"; } -DUE='memory feedback: last run 5+ d ago, transcripts expire at 7 → /memory-feedback' +DUE='memory feedback: last run 5+ d ago, ' +DUE+='transcripts expire at 7 → /memory-feedback' check MR1-no-store "$(run "$(home nostore)")" "rc=0" -check MR2-never "$(run "$(home never)")" "memory feedback: never run → /memory-feedback +NEVER='memory feedback: never run → /memory-feedback' +check MR2-never "$(run "$(home never)")" "$NEVER rc=0" h=$(home due); old "$h" "$OPEN" check MR3-due "$(run "$h")" "$DUE rc=0" h=$(home quiet); old "$h" "$OPEN" -touch -t 202512010000 "$h/.claude/plugins/store/memory_skills-dir-a.json" +touch -t 202512010000 \ + "$h/.claude/plugins/store/memory_skills-dir-a.json" check MR4-store-unchanged "$(run "$h")" "rc=0" h=$(home fresh) printf '%s' "$OPEN" > "$h/.claude/memory-feedback/state.json" -touch -t 209901010000 "$h/.claude/plugins/store/memory_skills-dir-a.json" +touch -t 209901010000 \ + "$h/.claude/plugins/store/memory_skills-dir-a.json" check MR5-ran-today "$(run "$h")" "rc=0" h=$(home closed); old "$h" '{"lastRun":1,"closed":true}' check MR6-closed "$(run "$h")" "rc=0" -h=$(home off); old "$h" "$OPEN"; printf '{"recall": false}' > "$h/.claude/memory.json" +h=$(home off); old "$h" "$OPEN" +printf '{"recall": false}' > "$h/.claude/memory.json" check MR7-recall-off "$(run "$h")" "rc=0" h=$(home locked); old "$h" "$OPEN" chmod 000 "$h/.claude/memory-feedback/state.json" diff --git a/mods/memory/hooks/register.ts b/mods/memory/hooks/register.ts index 8e31f53..1f4a826 100644 --- a/mods/memory/hooks/register.ts +++ b/mods/memory/hooks/register.ts @@ -1734,6 +1734,22 @@ function claimPicks(st: State, key: string, picked: Entry[]): string[] { return taken } +/** Claims the picks of a rerank answer; the block to inject. */ +function injectPicks( + st: State, + key: string, + answer: string, + cands: Entry[], +): Picked { + const picked = picksOf(answer, cands) + if (picked.length === 0) return nothing('none') + const taken = claimPicks(st, key, picked) + const block = recallBlock( + picked.filter(e => taken.includes(e.id)).map(recallLine), RECALL_CAP) + if (block === '') return nothing('refused') + return { block, ids: taken, outcome: 'injected' } +} + /** Asks haiku among the candidates and claims its picks. */ async function pickRecall( $: Api, @@ -1748,13 +1764,7 @@ async function pickRecall( // a clock step back must not make the record invalid (isCount) const ms = Math.max(0, Math.round((await $.clock.now()) - now)) if (answer === undefined) return { ...nothing('timeout'), ms } - const picked = picksOf(answer, cands) - if (picked.length === 0) return { ...nothing('none'), ms } - const taken = claimPicks(st, key, picked) - const block = recallBlock( - picked.filter(e => taken.includes(e.id)).map(recallLine), RECALL_CAP) - if (block === '') return { ...nothing('refused'), ms } - return { block, ids: taken, outcome: 'injected', ms } + return { ...injectPicks(st, key, answer, cands), ms } } /** diff --git a/skills/memory-feedback/SKILL.md b/skills/memory-feedback/SKILL.md index 816e861..f765b17 100644 --- a/skills/memory-feedback/SKILL.md +++ b/skills/memory-feedback/SKILL.md @@ -33,8 +33,10 @@ Read `/worksheet.md`. Fill the third column of the skeleton - `relevant`: the entry bears on what the prompt asked or what the reply did; `irrelevant` otherwise. Judge the entry against the ask, not the reply's quality. -- A proof, bench or recall-testing session: replace that item's lines by - one ` - test` line. +- A proof, bench or recall-testing session: every item of that session, + with lines or not (none, timeout and paused items included), gets one + ` - test` line in place of its lines. The header's `s:` shows + the session, the time the order. Lost items have no line. ## STEP 3 — SPOT CHECK @@ -54,8 +56,9 @@ user confirms or flips each. Apply the flips to the draft (LRN-168). On up to 5 prompt items whose outcome is injected or none, search the registries of that item's repo, whose absolute path is in the item header -(`bash ~/.claude/lib/memory.sh -C index` + entry bodies) for live entries that bore on the ask and were not -injected. Each one: ` missed `, cause `no-overlap` (no +(`bash ~/.claude/lib/memory.sh -C index` + entry bodies) for +live entries that bore on the ask and were not injected. Each one: +` missed `, cause `no-overlap` (no content word shared with the prompt) or `overlap`. A searched item with no miss: ` - checked`. Example line: `3 LRN-901 missed no-overlap`.