fix(memory-feedback): final review: locals, per-trigger n, write order

This commit is contained in:
bchanot committed 2026-11-08 17:55:08 +01:00
1 parent c00649f516
commit 271172564c
8 files changed
+167 -78

No files matched your search

+11
View File
@@ -118,3 +118,14 @@ test('M9: FEEDBACK.md counts, no repo tag without a noisy entry', () => {
expect(text).not.toContain('client#abcdef')
expect(text).toContain('Prompt gate: open (1/20)')
})
test('M10: FEEDBACK.md shows n per channel and trigger', () => {
const rows = [...ask(['relevant', 'irrelevant']),
...ask(['irrelevant'], { trigger: 'idle' })]
const mm = metricsOf(rows)
const text = renderFeedback(mm, gateVerdict(mm, 1), 0)
expect(text).toContain('| precision, idle | 0 % (n=1) |')
expect(text).toContain('| precision, fresh | 50 % (n=2) |')
expect(text).toContain('| prompt precision | 33 % (n=3) |')
expect(text).toContain('| tool precision | n/a (n=0) |')
})
+22 -11
View File
@@ -10,11 +10,13 @@ export type Gate = 'open' | 'reached'
export type ToolGate = Gate | 'extended' | 'insufficient'
export type Verdict = { prompt: Gate; tool: ToolGate }
type Rate = number | undefined
type Sized = { rate: Rate; n: number }
export type Metrics = {
start?: number; promptN: number; toolN: number; lost: number
precision: Rate; byTrigger: Record<string, Rate>; thirdPick: Rate
precision: Rate; byTrigger: Record<string, Sized>; thirdPick: Rate
late: Rate; msP90: Rate; tool: Rate; missShare: Rate; sampled: number
noisy: string[]; promote: string[]
n: { prompt: number; third: number; tool: number } // entries
}
const share = (k: number, n: number): Rate => n === 0 ? undefined : k / n
@@ -52,14 +54,22 @@ export function p90(values: number[]): Rate {
return s[Math.ceil(0.9 * s.length) - 1]
}
function byTrigger(entries: Row[]): Record<string, Rate> {
const out: Record<string, Rate> = {}
function byTrigger(entries: Row[]): Record<string, Sized> {
const out: Record<string, Sized> = {}
for (const k of ['fresh', 'idle', 'keyword']) {
out[k] = precision(entries.filter(r => r.trigger === k))
const mine = entries.filter(r => r.trigger === k)
out[k] = { rate: precision(mine), n: mine.length }
}
return out
}
/** Entry counts behind the three channel precisions. */
const countsOf = (entries: Row[]) => ({
prompt: entries.filter(r => r.channel === 'prompt').length,
third: entries.filter(r => r.channel === 'prompt' && r.pick === 3).length,
tool: entries.filter(r => r.channel === 'tool').length,
})
/** timeout + paused among asks (lost ones too) that had candidates. */
function lateShare(asks: Row[]): Rate {
const had = asks.filter(r => (r.cands ?? 0) > 0)
@@ -117,6 +127,7 @@ export function metricsOf(all: Row[]): Metrics {
tool: precision(entries.filter(r => r.channel === 'tool')),
missShare: miss.rate, sampled: miss.sampled,
noisy: noisy(entries), promote: promote(entries),
n: countsOf(entries),
}
}
@@ -137,10 +148,10 @@ export function gateVerdict(m: Metrics, now: number): Verdict {
}
/** The worst trigger under the floor; ties go idle, keyword, fresh. */
function worstTrigger(by: Record<string, Rate>): string | undefined {
function worstTrigger(by: Record<string, Sized>): string | undefined {
const low = ['idle', 'keyword', 'fresh']
.filter(k => under(by[k], T.triggerFloor))
low.sort((a, b) => (by[a] ?? 0) - (by[b] ?? 0))
.filter(k => under(by[k]?.rate, T.triggerFloor))
low.sort((a, b) => (by[a]?.rate ?? 0) - (by[b]?.rate ?? 0))
return low[0]
}
@@ -195,13 +206,13 @@ export function renderFeedback(m: Metrics, v: Verdict, now: number) {
`Prompt gate: ${v.prompt} (${m.promptN}/${PROMPT_GATE_N})`,
toolLine(m, v), `Lost records: ${m.lost}`, '',
'| Signal | Value |', '|---|---|',
`| prompt precision | ${pct(m.precision)} |`,
`| prompt precision | ${pct(m.precision)} (n=${m.n.prompt}) |`,
...Object.entries(m.byTrigger).map(([k, x]) =>
`| precision, ${k} | ${pct(x)} |`),
`| 3rd pick precision | ${pct(m.thirdPick)} |`,
`| precision, ${k} | ${pct(x.rate)} (n=${x.n}) |`),
`| 3rd pick precision | ${pct(m.thirdPick)} (n=${m.n.third}) |`,
`| timeout + paused | ${pct(m.late)} |`,
`| rerank ms p90 | ${m.msP90 ?? 'n/a'} |`,
`| tool precision | ${pct(m.tool)} |`,
`| tool precision | ${pct(m.tool)} (n=${m.n.tool}) |`,
`| asks with a miss | ${pct(m.missShare)} of ${m.sampled} |`, '',
`Noisy: ${m.noisy.join(', ') || 'none'}`,
`Promotion candidates: ${m.promote.join(', ') || 'none'}`, '',
+70 -33
View File
@@ -13,7 +13,8 @@ import {
} from '../../mods/memory/hooks/register.ts'
import {
appendRows, keyOf, parseDraft, readRows, readState, rowsOf, skeleton,
tagOf, writeSafe, writeState, type Item, type State,
tagOf, writeSafe, writeState, type Draft, type Item, type Row,
type State,
} from './labels.ts'
import {
clockStart, gateVerdict, metricsOf, renderFeedback, type Verdict,
@@ -56,8 +57,7 @@ export function storeFile(dir: string): string {
return files.sort((a, b) => mtime(b) - mtime(a))[0] as string
}
/** Every valid log record of every root, oldest first. */
export function readLogs(file: string): Logged[] {
function readStore(file: string): object {
let data: unknown
try {
data = JSON.parse(fs.readFileSync(file, 'utf8'))
@@ -67,13 +67,16 @@ export function readLogs(file: string): Logged[] {
if (typeof data !== 'object' || data === null) {
throw new Error(`store file is not an object: ${file}`)
}
const out: Logged[] = []
for (const [k, v] of Object.entries(data)) {
if (!k.startsWith(LOG_PREFIX) || !Array.isArray(v)) continue
const root = k.slice(LOG_PREFIX.length)
for (const rec of v.filter(isLogRecord)) out.push({ root, rec })
}
return out.sort((a, b) => a.rec.at - b.rec.at)
return data
}
/** Every valid log record of every root, oldest first. */
export function readLogs(file: string): Logged[] {
return Object.entries(readStore(file))
.filter(([k, v]) => k.startsWith(LOG_PREFIX) && Array.isArray(v))
.flatMap(([k, v]) => (v as unknown[]).filter(isLogRecord)
.map(rec => ({ root: k.slice(LOG_PREFIX.length), rec })))
.sort((a, b) => a.rec.at - b.rec.at)
}
/** Unlabelled records from the clock start, AGE_FLOOR_MS old or more. */
@@ -109,6 +112,8 @@ function header(b: Base, repo: string, lost: boolean): string {
const ids = b.ids.join(' ') || '-'
return `## Item ${b.n}: ${b.channel}/${b.trigger}/${b.outcome}` +
` · cands ${b.cands ?? '-'} · ms ${b.ms ?? '-'}` +
` · s:${b.session.slice(0, 8)}` +
` · ${new Date(b.at).toISOString().slice(11, 16)} UTC` +
` · ${b.root} (${repo})` +
` · ids ${ids}${lost ? ' · LOST (no transcript turn)' : ''}`
}
@@ -188,24 +193,34 @@ function sheetHead(n: number, torn: number, evicted: string[]): string {
return lines.join('\n')
}
/** Writes worksheet.md, items.json, labels-draft.tsv (0600); the count. */
export function worksheet(dir: string, p: Paths, now: number): number {
const logAtsOf = (logs: Logged[]) =>
logs.filter(l => l.rec.cands !== undefined).map(l => l.rec.at)
/** Joins the unlabelled population to its transcripts. */
function collect(p: Paths, now: number) {
const rows = readRows(labelsOf(p))
const logs = readLogs(storeFile(p.store))
const logAts = logs.filter(l => l.rec.cands !== undefined)
.map(l => l.rec.at)
const pop = population(logs, new Set(rows.map(keyOf)),
clockStart(rows, logAts), now)
clockStart(rows, logAtsOf(logs)), now)
const loader = loaderFor(p.projects)
const joined = pop.map((l, i) => joinOne(l, i + 1, loader.load))
const head = sheetHead(pop.length, loader.torn(),
evictions(logs, readState(stateOf(p)).lastRun))
return { joined, head: sheetHead(pop.length, loader.torn(),
evictions(logs, readState(stateOf(p)).lastRun)) }
}
function writeSheet(dir: string, joined: Joined[], head: string): void {
writeSafe(path.join(dir, 'worksheet.md'), ['# Memory feedback worksheet',
head, DRAFT_HELP, ...joined.map(j => j.text)].join('\n\n') + '\n')
const items = joined.map(j => j.item)
writeSafe(path.join(dir, 'items.json'), JSON.stringify(items))
writeSafe(path.join(dir, 'labels-draft.tsv'), skeleton(items))
return pop.length
}
/** Writes worksheet.md, items.json, labels-draft.tsv (0600); the count. */
export function worksheet(dir: string, p: Paths, now: number): number {
const { joined, head } = collect(p, now)
writeSheet(dir, joined, head)
return joined.length
}
const draftOf = (dir: string) => {
@@ -230,26 +245,45 @@ function nextState(st: State, v: Verdict, now: number) {
return { state: { lastRun: now, promptAt, toolAt, closed }, news }
}
/** Validates the draft; writes state, then labels, then FEEDBACK.md. */
const assertFresh = (items: Item[], old: Row[]) => {
const done = new Set(old.map(keyOf))
if (items.some(i => done.has(keyOf(i)))) {
throw new Error('items already applied: run worksheet again')
}
}
/** The new rows, and the metrics and verdict once they are added. */
function projected(
items: Item[], drafts: Draft[], old: Row[], now: number,
) {
const fresh = items.flatMap(i => rowsOf(i, drafts))
const m = metricsOf([...old, ...fresh])
return { fresh, m, v: gateVerdict(m, now) }
}
/** Appends labels, then writes state, then FEEDBACK.md. */
function commit(
p: Paths, items: Item[], drafts: Draft[], now: number,
): string {
const old = readRows(labelsOf(p))
assertFresh(items, old)
const { fresh, m, v } = projected(items, drafts, old, now)
const next = nextState(readState(stateOf(p)), v, now)
const feedback = renderFeedback(m, v, now)
if (fresh.length > 0) appendRows(labelsOf(p), fresh)
writeState(stateOf(p), next.state)
writeSafe(path.join(p.out, 'FEEDBACK.md'), feedback)
return feedback + next.news.map(n => n + '\n').join('')
}
/** Validates the draft; appends labels, writes state, then FEEDBACK.md
* (a failed state write leaves the NEW line to print on the next run). */
export function apply(dir: string, p: Paths, now: number): string {
const items: Item[] = JSON.parse(
fs.readFileSync(path.join(dir, 'items.json'), 'utf8'))
const drafts = parseDraft(draftOf(dir), items)
if (typeof drafts === 'string') throw new Error(`draft refused: ${drafts}`)
const old = readRows(labelsOf(p))
const done = new Set(old.map(keyOf))
if (items.some(i => done.has(keyOf(i)))) {
throw new Error('items already applied: run worksheet again')
}
const fresh = items.flatMap(i => rowsOf(i, drafts))
const m = metricsOf([...old, ...fresh])
const v = gateVerdict(m, now)
const { state, news } = nextState(readState(stateOf(p)), v, now)
const feedback = renderFeedback(m, v, now)
writeState(stateOf(p), state)
if (fresh.length > 0) appendRows(labelsOf(p), fresh)
writeSafe(path.join(p.out, 'FEEDBACK.md'), feedback)
return feedback + news.map(n => n + '\n').join('')
return commit(p, items, drafts, now)
}
/** Up to `n` entry lines of the draft, picked at random. */
@@ -269,6 +303,9 @@ export function checkDir(dir: string | undefined): string {
if (!fs.existsSync(dir)) return 'does not exist'
const git = spawnSync('git', ['-C', dir, 'rev-parse',
'--is-inside-work-tree'], { encoding: 'utf8' })
if (git.error !== undefined || typeof git.stdout !== 'string') {
return 'git unavailable'
}
return git.stdout.trim() === 'true' ? 'inside a git work tree' : ''
}
+3
View File
@@ -41,6 +41,9 @@ test('X1: slug, the main or one sub-agent file, torn lines counted', () => {
expect(sessionFiles(p, '/r', 's', 'a1').map(f => path.basename(f)))
.toEqual(['agent-a1.jsonl'])
expect(records(main[0]!)).toEqual({ recs: [{ a: 1 }], torn: 1 })
fs.writeFileSync(path.join(dir, 'n.jsonl'), '{"a":1}\nnull\n[1]\n7\n')
expect(records(path.join(dir, 'n.jsonl')))
.toEqual({ recs: [{ a: 1 }], torn: 3 })
expect(sessionFiles(p, '/r', 'gone', 'main')).toEqual([])
})
+23 -16
View File
@@ -28,13 +28,19 @@ export function sessionFiles(
return fs.existsSync(file) ? [file] : []
}
/** Parsed lines of a transcript; torn or partial lines are counted. */
/** Parsed lines of a transcript; torn, partial or non-object lines
* (`null`, a number, an array) are counted as torn. */
export function records(file: string): Loaded {
const recs: Rec[] = []
let torn = 0
for (const raw of fs.readFileSync(file, 'utf8').split('\n')) {
if (raw.trim() === '') continue
try { recs.push(JSON.parse(raw)) } catch { torn += 1 }
try {
const v = JSON.parse(raw)
if (v === null || typeof v !== 'object' || Array.isArray(v)) {
torn += 1
} else recs.push(v)
} catch { torn += 1 }
}
return { recs, torn }
}
@@ -90,7 +96,8 @@ function promptNear(recs: Rec[], ts: number): Prompt | undefined {
for (const r of recs) {
const text = composerText(r)
const t = tsOf(r)
if (text === undefined || Math.abs(t - ts) > NEAR_MS) continue
if (text === undefined || !Number.isFinite(t)) continue
if (Math.abs(t - ts) > NEAR_MS) continue
if (best === undefined || Math.abs(t - ts) < Math.abs(best.t - ts)) {
best = { t, text }
}
@@ -108,22 +115,22 @@ function nextPromptAt(recs: Rec[], ts: number): number {
return end
}
/** The [dedupe key, text] of each text block of an assistant record. */
const textsOf = (r: Rec): [string, string][] => blocksOf(r)
.filter(b => b.type === 'text')
.map(b => [`${r.message?.id}:${b.text}`, String(b.text)]) // EVAL-037
/** Main-thread assistant text after `ts` up to the next prompt. */
function replyAfter(recs: Rec[], ts: number): string {
const end = nextPromptAt(recs, ts)
const seen = new Set<string>()
const parts: string[] = []
for (const r of recs) {
const t = tsOf(r)
if (r.type !== 'assistant' || r.isSidechain === true) continue
if (!(t > ts && t < end)) continue
for (const b of blocksOf(r).filter(b => b.type === 'text')) {
const key = `${r.message?.id}:${b.text}` // EVAL-037
if (!seen.has(key)) parts.push(String(b.text))
seen.add(key)
}
}
return parts.join('\n')
return recs
.filter(r => r.type === 'assistant' && r.isSidechain !== true)
.filter(r => tsOf(r) > ts && tsOf(r) < end)
.flatMap(textsOf)
.filter(([key]) => !seen.has(key) && !!seen.add(key))
.map(([, text]) => text)
.join('\n')
}
export type PromptJoin = { prompt: string; injected: string; reply: string }
@@ -178,7 +185,7 @@ export type ToolJoin = { command: string; known: string; next: boolean[] }
/** The failure a known line answered and the next 3 Bash results. */
export function joinTool(recs: Rec[], rec: LogRecord): ToolJoin | undefined {
const needle = KNOWN_HEAD + (rec.ids[0] ?? '')
const needle = `${KNOWN_HEAD}${rec.ids[0] ?? ''} ` // BLK-003 != BLK-0031
const att = attachmentAfter(recs, rec.at, 'tool.call', needle)
const raw = att?.attachment?.toolUseID
if (att === undefined || typeof raw !== 'string') return undefined
+14 -7
View File
@@ -13,8 +13,10 @@ check() { if [ "$2" = "$3" ]; then pass=$((pass+1)); else fail=$((fail+1));
printf 'FAIL %s: got[%s] want[%s]\n' "$1" "$2" "$3"; fi; }
# home <name>: a fixture HOME with a store file (unless "nostore")
home() {
mkdir -p "$WORK/$1/.claude/plugins/store" "$WORK/$1/.claude/memory-feedback"
[ "$1" = nostore ] || : > "$WORK/$1/.claude/plugins/store/memory_skills-dir-a.json"
mkdir -p "$WORK/$1/.claude/plugins/store" \
"$WORK/$1/.claude/memory-feedback"
[ "$1" = nostore ] ||
: > "$WORK/$1/.claude/plugins/store/memory_skills-dir-a.json"
echo "$WORK/$1"
}
OPEN='{"lastRun":1,"closed":false}'
@@ -24,24 +26,29 @@ old() {
touch -t 202601010000 "$1/.claude/memory-feedback/state.json"
}
run() { HOME="$1" bash "$LIB" 2>&1; echo "rc=$?"; }
DUE='memory feedback: last run 5+ d ago, transcripts expire at 7 → /memory-feedback'
DUE='memory feedback: last run 5+ d ago, '
DUE+='transcripts expire at 7 → /memory-feedback'
check MR1-no-store "$(run "$(home nostore)")" "rc=0"
check MR2-never "$(run "$(home never)")" "memory feedback: never run → /memory-feedback
NEVER='memory feedback: never run → /memory-feedback'
check MR2-never "$(run "$(home never)")" "$NEVER
rc=0"
h=$(home due); old "$h" "$OPEN"
check MR3-due "$(run "$h")" "$DUE
rc=0"
h=$(home quiet); old "$h" "$OPEN"
touch -t 202512010000 "$h/.claude/plugins/store/memory_skills-dir-a.json"
touch -t 202512010000 \
"$h/.claude/plugins/store/memory_skills-dir-a.json"
check MR4-store-unchanged "$(run "$h")" "rc=0"
h=$(home fresh)
printf '%s' "$OPEN" > "$h/.claude/memory-feedback/state.json"
touch -t 209901010000 "$h/.claude/plugins/store/memory_skills-dir-a.json"
touch -t 209901010000 \
"$h/.claude/plugins/store/memory_skills-dir-a.json"
check MR5-ran-today "$(run "$h")" "rc=0"
h=$(home closed); old "$h" '{"lastRun":1,"closed":true}'
check MR6-closed "$(run "$h")" "rc=0"
h=$(home off); old "$h" "$OPEN"; printf '{"recall": false}' > "$h/.claude/memory.json"
h=$(home off); old "$h" "$OPEN"
printf '{"recall": false}' > "$h/.claude/memory.json"
check MR7-recall-off "$(run "$h")" "rc=0"
h=$(home locked); old "$h" "$OPEN"
chmod 000 "$h/.claude/memory-feedback/state.json"
+17 -7
View File
@@ -1734,6 +1734,22 @@ function claimPicks(st: State, key: string, picked: Entry[]): string[] {
return taken
}
/** Claims the picks of a rerank answer; the block to inject. */
function injectPicks(
st: State,
key: string,
answer: string,
cands: Entry[],
): Picked {
const picked = picksOf(answer, cands)
if (picked.length === 0) return nothing('none')
const taken = claimPicks(st, key, picked)
const block = recallBlock(
picked.filter(e => taken.includes(e.id)).map(recallLine), RECALL_CAP)
if (block === '') return nothing('refused')
return { block, ids: taken, outcome: 'injected' }
}
/** Asks haiku among the candidates and claims its picks. */
async function pickRecall(
$: Api,
@@ -1748,13 +1764,7 @@ async function pickRecall(
// a clock step back must not make the record invalid (isCount)
const ms = Math.max(0, Math.round((await $.clock.now()) - now))
if (answer === undefined) return { ...nothing('timeout'), ms }
const picked = picksOf(answer, cands)
if (picked.length === 0) return { ...nothing('none'), ms }
const taken = claimPicks(st, key, picked)
const block = recallBlock(
picked.filter(e => taken.includes(e.id)).map(recallLine), RECALL_CAP)
if (block === '') return { ...nothing('refused'), ms }
return { block, ids: taken, outcome: 'injected', ms }
return { ...injectPicks(st, key, answer, cands), ms }
}
/**
+7 -4
View File
@@ -33,8 +33,10 @@ Read `<dir>/worksheet.md`. Fill the third column of the skeleton
- `relevant`: the entry bears on what the prompt asked or what the reply
did; `irrelevant` otherwise. Judge the entry against the ask, not the
reply's quality.
- A proof, bench or recall-testing session: replace that item's lines by
one `<item> - test` line.
- A proof, bench or recall-testing session: every item of that session,
with lines or not (none, timeout and paused items included), gets one
`<item> - test` line in place of its lines. The header's `s:` shows
the session, the time the order.
Lost items have no line.
## STEP 3 — SPOT CHECK
@@ -54,8 +56,9 @@ user confirms or flips each. Apply the flips to the draft (LRN-168).
On up to 5 prompt items whose outcome is injected or none, search the
registries of that item's repo, whose absolute path is in the item header
(`bash ~/.claude/lib/memory.sh -C <path> index` + entry bodies) for live entries that bore on the ask and were not
injected. Each one: `<item> <id> missed <cause>`, cause `no-overlap` (no
(`bash ~/.claude/lib/memory.sh -C <path> index` + entry bodies) for
live entries that bore on the ask and were not injected. Each one:
`<item> <id> missed <cause>`, cause `no-overlap` (no
content word shared with the prompt) or `overlap`. A searched item with no
miss: `<item> - checked`. Example line: `3 LRN-901 missed no-overlap`.