fix(memory-feedback): final review: locals, per-trigger n, write order

This commit is contained in:
bchanot committed 2026-11-08 17:55:08 +01:00
1 parent c00649f516
commit 271172564c
8 files changed
+166 -77

No files matched your search

+11
View File
@@ -118,3 +118,14 @@ test('M9: FEEDBACK.md counts, no repo tag without a noisy entry', () => {
expect(text).not.toContain('client#abcdef') expect(text).not.toContain('client#abcdef')
expect(text).toContain('Prompt gate: open (1/20)') expect(text).toContain('Prompt gate: open (1/20)')
}) })
test('M10: FEEDBACK.md shows n per channel and trigger', () => {
const rows = [...ask(['relevant', 'irrelevant']),
...ask(['irrelevant'], { trigger: 'idle' })]
const mm = metricsOf(rows)
const text = renderFeedback(mm, gateVerdict(mm, 1), 0)
expect(text).toContain('| precision, idle | 0 % (n=1) |')
expect(text).toContain('| precision, fresh | 50 % (n=2) |')
expect(text).toContain('| prompt precision | 33 % (n=3) |')
expect(text).toContain('| tool precision | n/a (n=0) |')
})
+22 -11
View File
@@ -10,11 +10,13 @@ export type Gate = 'open' | 'reached'
export type ToolGate = Gate | 'extended' | 'insufficient' export type ToolGate = Gate | 'extended' | 'insufficient'
export type Verdict = { prompt: Gate; tool: ToolGate } export type Verdict = { prompt: Gate; tool: ToolGate }
type Rate = number | undefined type Rate = number | undefined
type Sized = { rate: Rate; n: number }
export type Metrics = { export type Metrics = {
start?: number; promptN: number; toolN: number; lost: number start?: number; promptN: number; toolN: number; lost: number
precision: Rate; byTrigger: Record<string, Rate>; thirdPick: Rate precision: Rate; byTrigger: Record<string, Sized>; thirdPick: Rate
late: Rate; msP90: Rate; tool: Rate; missShare: Rate; sampled: number late: Rate; msP90: Rate; tool: Rate; missShare: Rate; sampled: number
noisy: string[]; promote: string[] noisy: string[]; promote: string[]
n: { prompt: number; third: number; tool: number } // entries
} }
const share = (k: number, n: number): Rate => n === 0 ? undefined : k / n const share = (k: number, n: number): Rate => n === 0 ? undefined : k / n
@@ -52,14 +54,22 @@ export function p90(values: number[]): Rate {
return s[Math.ceil(0.9 * s.length) - 1] return s[Math.ceil(0.9 * s.length) - 1]
} }
function byTrigger(entries: Row[]): Record<string, Rate> { function byTrigger(entries: Row[]): Record<string, Sized> {
const out: Record<string, Rate> = {} const out: Record<string, Sized> = {}
for (const k of ['fresh', 'idle', 'keyword']) { for (const k of ['fresh', 'idle', 'keyword']) {
out[k] = precision(entries.filter(r => r.trigger === k)) const mine = entries.filter(r => r.trigger === k)
out[k] = { rate: precision(mine), n: mine.length }
} }
return out return out
} }
/** Entry counts behind the three channel precisions. */
const countsOf = (entries: Row[]) => ({
prompt: entries.filter(r => r.channel === 'prompt').length,
third: entries.filter(r => r.channel === 'prompt' && r.pick === 3).length,
tool: entries.filter(r => r.channel === 'tool').length,
})
/** timeout + paused among asks (lost ones too) that had candidates. */ /** timeout + paused among asks (lost ones too) that had candidates. */
function lateShare(asks: Row[]): Rate { function lateShare(asks: Row[]): Rate {
const had = asks.filter(r => (r.cands ?? 0) > 0) const had = asks.filter(r => (r.cands ?? 0) > 0)
@@ -117,6 +127,7 @@ export function metricsOf(all: Row[]): Metrics {
tool: precision(entries.filter(r => r.channel === 'tool')), tool: precision(entries.filter(r => r.channel === 'tool')),
missShare: miss.rate, sampled: miss.sampled, missShare: miss.rate, sampled: miss.sampled,
noisy: noisy(entries), promote: promote(entries), noisy: noisy(entries), promote: promote(entries),
n: countsOf(entries),
} }
} }
@@ -137,10 +148,10 @@ export function gateVerdict(m: Metrics, now: number): Verdict {
} }
/** The worst trigger under the floor; ties go idle, keyword, fresh. */ /** The worst trigger under the floor; ties go idle, keyword, fresh. */
function worstTrigger(by: Record<string, Rate>): string | undefined { function worstTrigger(by: Record<string, Sized>): string | undefined {
const low = ['idle', 'keyword', 'fresh'] const low = ['idle', 'keyword', 'fresh']
.filter(k => under(by[k], T.triggerFloor)) .filter(k => under(by[k]?.rate, T.triggerFloor))
low.sort((a, b) => (by[a] ?? 0) - (by[b] ?? 0)) low.sort((a, b) => (by[a]?.rate ?? 0) - (by[b]?.rate ?? 0))
return low[0] return low[0]
} }
@@ -195,13 +206,13 @@ export function renderFeedback(m: Metrics, v: Verdict, now: number) {
`Prompt gate: ${v.prompt} (${m.promptN}/${PROMPT_GATE_N})`, `Prompt gate: ${v.prompt} (${m.promptN}/${PROMPT_GATE_N})`,
toolLine(m, v), `Lost records: ${m.lost}`, '', toolLine(m, v), `Lost records: ${m.lost}`, '',
'| Signal | Value |', '|---|---|', '| Signal | Value |', '|---|---|',
`| prompt precision | ${pct(m.precision)} |`, `| prompt precision | ${pct(m.precision)} (n=${m.n.prompt}) |`,
...Object.entries(m.byTrigger).map(([k, x]) => ...Object.entries(m.byTrigger).map(([k, x]) =>
`| precision, ${k} | ${pct(x)} |`), `| precision, ${k} | ${pct(x.rate)} (n=${x.n}) |`),
`| 3rd pick precision | ${pct(m.thirdPick)} |`, `| 3rd pick precision | ${pct(m.thirdPick)} (n=${m.n.third}) |`,
`| timeout + paused | ${pct(m.late)} |`, `| timeout + paused | ${pct(m.late)} |`,
`| rerank ms p90 | ${m.msP90 ?? 'n/a'} |`, `| rerank ms p90 | ${m.msP90 ?? 'n/a'} |`,
`| tool precision | ${pct(m.tool)} |`, `| tool precision | ${pct(m.tool)} (n=${m.n.tool}) |`,
`| asks with a miss | ${pct(m.missShare)} of ${m.sampled} |`, '', `| asks with a miss | ${pct(m.missShare)} of ${m.sampled} |`, '',
`Noisy: ${m.noisy.join(', ') || 'none'}`, `Noisy: ${m.noisy.join(', ') || 'none'}`,
`Promotion candidates: ${m.promote.join(', ') || 'none'}`, '', `Promotion candidates: ${m.promote.join(', ') || 'none'}`, '',
+69 -32
View File
@@ -13,7 +13,8 @@ import {
} from '../../mods/memory/hooks/register.ts' } from '../../mods/memory/hooks/register.ts'
import { import {
appendRows, keyOf, parseDraft, readRows, readState, rowsOf, skeleton, appendRows, keyOf, parseDraft, readRows, readState, rowsOf, skeleton,
tagOf, writeSafe, writeState, type Item, type State, tagOf, writeSafe, writeState, type Draft, type Item, type Row,
type State,
} from './labels.ts' } from './labels.ts'
import { import {
clockStart, gateVerdict, metricsOf, renderFeedback, type Verdict, clockStart, gateVerdict, metricsOf, renderFeedback, type Verdict,
@@ -56,8 +57,7 @@ export function storeFile(dir: string): string {
return files.sort((a, b) => mtime(b) - mtime(a))[0] as string return files.sort((a, b) => mtime(b) - mtime(a))[0] as string
} }
/** Every valid log record of every root, oldest first. */ function readStore(file: string): object {
export function readLogs(file: string): Logged[] {
let data: unknown let data: unknown
try { try {
data = JSON.parse(fs.readFileSync(file, 'utf8')) data = JSON.parse(fs.readFileSync(file, 'utf8'))
@@ -67,13 +67,16 @@ export function readLogs(file: string): Logged[] {
if (typeof data !== 'object' || data === null) { if (typeof data !== 'object' || data === null) {
throw new Error(`store file is not an object: ${file}`) throw new Error(`store file is not an object: ${file}`)
} }
const out: Logged[] = [] return data
for (const [k, v] of Object.entries(data)) {
if (!k.startsWith(LOG_PREFIX) || !Array.isArray(v)) continue
const root = k.slice(LOG_PREFIX.length)
for (const rec of v.filter(isLogRecord)) out.push({ root, rec })
} }
return out.sort((a, b) => a.rec.at - b.rec.at)
/** Every valid log record of every root, oldest first. */
export function readLogs(file: string): Logged[] {
return Object.entries(readStore(file))
.filter(([k, v]) => k.startsWith(LOG_PREFIX) && Array.isArray(v))
.flatMap(([k, v]) => (v as unknown[]).filter(isLogRecord)
.map(rec => ({ root: k.slice(LOG_PREFIX.length), rec })))
.sort((a, b) => a.rec.at - b.rec.at)
} }
/** Unlabelled records from the clock start, AGE_FLOOR_MS old or more. */ /** Unlabelled records from the clock start, AGE_FLOOR_MS old or more. */
@@ -109,6 +112,8 @@ function header(b: Base, repo: string, lost: boolean): string {
const ids = b.ids.join(' ') || '-' const ids = b.ids.join(' ') || '-'
return `## Item ${b.n}: ${b.channel}/${b.trigger}/${b.outcome}` + return `## Item ${b.n}: ${b.channel}/${b.trigger}/${b.outcome}` +
` · cands ${b.cands ?? '-'} · ms ${b.ms ?? '-'}` + ` · cands ${b.cands ?? '-'} · ms ${b.ms ?? '-'}` +
` · s:${b.session.slice(0, 8)}` +
` · ${new Date(b.at).toISOString().slice(11, 16)} UTC` +
` · ${b.root} (${repo})` + ` · ${b.root} (${repo})` +
` · ids ${ids}${lost ? ' · LOST (no transcript turn)' : ''}` ` · ids ${ids}${lost ? ' · LOST (no transcript turn)' : ''}`
} }
@@ -188,24 +193,34 @@ function sheetHead(n: number, torn: number, evicted: string[]): string {
return lines.join('\n') return lines.join('\n')
} }
/** Writes worksheet.md, items.json, labels-draft.tsv (0600); the count. */ const logAtsOf = (logs: Logged[]) =>
export function worksheet(dir: string, p: Paths, now: number): number { logs.filter(l => l.rec.cands !== undefined).map(l => l.rec.at)
/** Joins the unlabelled population to its transcripts. */
function collect(p: Paths, now: number) {
const rows = readRows(labelsOf(p)) const rows = readRows(labelsOf(p))
const logs = readLogs(storeFile(p.store)) const logs = readLogs(storeFile(p.store))
const logAts = logs.filter(l => l.rec.cands !== undefined)
.map(l => l.rec.at)
const pop = population(logs, new Set(rows.map(keyOf)), const pop = population(logs, new Set(rows.map(keyOf)),
clockStart(rows, logAts), now) clockStart(rows, logAtsOf(logs)), now)
const loader = loaderFor(p.projects) const loader = loaderFor(p.projects)
const joined = pop.map((l, i) => joinOne(l, i + 1, loader.load)) const joined = pop.map((l, i) => joinOne(l, i + 1, loader.load))
const head = sheetHead(pop.length, loader.torn(), return { joined, head: sheetHead(pop.length, loader.torn(),
evictions(logs, readState(stateOf(p)).lastRun)) evictions(logs, readState(stateOf(p)).lastRun)) }
}
function writeSheet(dir: string, joined: Joined[], head: string): void {
writeSafe(path.join(dir, 'worksheet.md'), ['# Memory feedback worksheet', writeSafe(path.join(dir, 'worksheet.md'), ['# Memory feedback worksheet',
head, DRAFT_HELP, ...joined.map(j => j.text)].join('\n\n') + '\n') head, DRAFT_HELP, ...joined.map(j => j.text)].join('\n\n') + '\n')
const items = joined.map(j => j.item) const items = joined.map(j => j.item)
writeSafe(path.join(dir, 'items.json'), JSON.stringify(items)) writeSafe(path.join(dir, 'items.json'), JSON.stringify(items))
writeSafe(path.join(dir, 'labels-draft.tsv'), skeleton(items)) writeSafe(path.join(dir, 'labels-draft.tsv'), skeleton(items))
return pop.length }
/** Writes worksheet.md, items.json, labels-draft.tsv (0600); the count. */
export function worksheet(dir: string, p: Paths, now: number): number {
const { joined, head } = collect(p, now)
writeSheet(dir, joined, head)
return joined.length
} }
const draftOf = (dir: string) => { const draftOf = (dir: string) => {
@@ -230,26 +245,45 @@ function nextState(st: State, v: Verdict, now: number) {
return { state: { lastRun: now, promptAt, toolAt, closed }, news } return { state: { lastRun: now, promptAt, toolAt, closed }, news }
} }
/** Validates the draft; writes state, then labels, then FEEDBACK.md. */ const assertFresh = (items: Item[], old: Row[]) => {
const done = new Set(old.map(keyOf))
if (items.some(i => done.has(keyOf(i)))) {
throw new Error('items already applied: run worksheet again')
}
}
/** The new rows, and the metrics and verdict once they are added. */
function projected(
items: Item[], drafts: Draft[], old: Row[], now: number,
) {
const fresh = items.flatMap(i => rowsOf(i, drafts))
const m = metricsOf([...old, ...fresh])
return { fresh, m, v: gateVerdict(m, now) }
}
/** Appends labels, then writes state, then FEEDBACK.md. */
function commit(
p: Paths, items: Item[], drafts: Draft[], now: number,
): string {
const old = readRows(labelsOf(p))
assertFresh(items, old)
const { fresh, m, v } = projected(items, drafts, old, now)
const next = nextState(readState(stateOf(p)), v, now)
const feedback = renderFeedback(m, v, now)
if (fresh.length > 0) appendRows(labelsOf(p), fresh)
writeState(stateOf(p), next.state)
writeSafe(path.join(p.out, 'FEEDBACK.md'), feedback)
return feedback + next.news.map(n => n + '\n').join('')
}
/** Validates the draft; appends labels, writes state, then FEEDBACK.md
* (a failed state write leaves the NEW line to print on the next run). */
export function apply(dir: string, p: Paths, now: number): string { export function apply(dir: string, p: Paths, now: number): string {
const items: Item[] = JSON.parse( const items: Item[] = JSON.parse(
fs.readFileSync(path.join(dir, 'items.json'), 'utf8')) fs.readFileSync(path.join(dir, 'items.json'), 'utf8'))
const drafts = parseDraft(draftOf(dir), items) const drafts = parseDraft(draftOf(dir), items)
if (typeof drafts === 'string') throw new Error(`draft refused: ${drafts}`) if (typeof drafts === 'string') throw new Error(`draft refused: ${drafts}`)
const old = readRows(labelsOf(p)) return commit(p, items, drafts, now)
const done = new Set(old.map(keyOf))
if (items.some(i => done.has(keyOf(i)))) {
throw new Error('items already applied: run worksheet again')
}
const fresh = items.flatMap(i => rowsOf(i, drafts))
const m = metricsOf([...old, ...fresh])
const v = gateVerdict(m, now)
const { state, news } = nextState(readState(stateOf(p)), v, now)
const feedback = renderFeedback(m, v, now)
writeState(stateOf(p), state)
if (fresh.length > 0) appendRows(labelsOf(p), fresh)
writeSafe(path.join(p.out, 'FEEDBACK.md'), feedback)
return feedback + news.map(n => n + '\n').join('')
} }
/** Up to `n` entry lines of the draft, picked at random. */ /** Up to `n` entry lines of the draft, picked at random. */
@@ -269,6 +303,9 @@ export function checkDir(dir: string | undefined): string {
if (!fs.existsSync(dir)) return 'does not exist' if (!fs.existsSync(dir)) return 'does not exist'
const git = spawnSync('git', ['-C', dir, 'rev-parse', const git = spawnSync('git', ['-C', dir, 'rev-parse',
'--is-inside-work-tree'], { encoding: 'utf8' }) '--is-inside-work-tree'], { encoding: 'utf8' })
if (git.error !== undefined || typeof git.stdout !== 'string') {
return 'git unavailable'
}
return git.stdout.trim() === 'true' ? 'inside a git work tree' : '' return git.stdout.trim() === 'true' ? 'inside a git work tree' : ''
} }
+3
View File
@@ -41,6 +41,9 @@ test('X1: slug, the main or one sub-agent file, torn lines counted', () => {
expect(sessionFiles(p, '/r', 's', 'a1').map(f => path.basename(f))) expect(sessionFiles(p, '/r', 's', 'a1').map(f => path.basename(f)))
.toEqual(['agent-a1.jsonl']) .toEqual(['agent-a1.jsonl'])
expect(records(main[0]!)).toEqual({ recs: [{ a: 1 }], torn: 1 }) expect(records(main[0]!)).toEqual({ recs: [{ a: 1 }], torn: 1 })
fs.writeFileSync(path.join(dir, 'n.jsonl'), '{"a":1}\nnull\n[1]\n7\n')
expect(records(path.join(dir, 'n.jsonl')))
.toEqual({ recs: [{ a: 1 }], torn: 3 })
expect(sessionFiles(p, '/r', 'gone', 'main')).toEqual([]) expect(sessionFiles(p, '/r', 'gone', 'main')).toEqual([])
}) })
+23 -16
View File
@@ -28,13 +28,19 @@ export function sessionFiles(
return fs.existsSync(file) ? [file] : [] return fs.existsSync(file) ? [file] : []
} }
/** Parsed lines of a transcript; torn or partial lines are counted. */ /** Parsed lines of a transcript; torn, partial or non-object lines
* (`null`, a number, an array) are counted as torn. */
export function records(file: string): Loaded { export function records(file: string): Loaded {
const recs: Rec[] = [] const recs: Rec[] = []
let torn = 0 let torn = 0
for (const raw of fs.readFileSync(file, 'utf8').split('\n')) { for (const raw of fs.readFileSync(file, 'utf8').split('\n')) {
if (raw.trim() === '') continue if (raw.trim() === '') continue
try { recs.push(JSON.parse(raw)) } catch { torn += 1 } try {
const v = JSON.parse(raw)
if (v === null || typeof v !== 'object' || Array.isArray(v)) {
torn += 1
} else recs.push(v)
} catch { torn += 1 }
} }
return { recs, torn } return { recs, torn }
} }
@@ -90,7 +96,8 @@ function promptNear(recs: Rec[], ts: number): Prompt | undefined {
for (const r of recs) { for (const r of recs) {
const text = composerText(r) const text = composerText(r)
const t = tsOf(r) const t = tsOf(r)
if (text === undefined || Math.abs(t - ts) > NEAR_MS) continue if (text === undefined || !Number.isFinite(t)) continue
if (Math.abs(t - ts) > NEAR_MS) continue
if (best === undefined || Math.abs(t - ts) < Math.abs(best.t - ts)) { if (best === undefined || Math.abs(t - ts) < Math.abs(best.t - ts)) {
best = { t, text } best = { t, text }
} }
@@ -108,22 +115,22 @@ function nextPromptAt(recs: Rec[], ts: number): number {
return end return end
} }
/** The [dedupe key, text] of each text block of an assistant record. */
const textsOf = (r: Rec): [string, string][] => blocksOf(r)
.filter(b => b.type === 'text')
.map(b => [`${r.message?.id}:${b.text}`, String(b.text)]) // EVAL-037
/** Main-thread assistant text after `ts` up to the next prompt. */ /** Main-thread assistant text after `ts` up to the next prompt. */
function replyAfter(recs: Rec[], ts: number): string { function replyAfter(recs: Rec[], ts: number): string {
const end = nextPromptAt(recs, ts) const end = nextPromptAt(recs, ts)
const seen = new Set<string>() const seen = new Set<string>()
const parts: string[] = [] return recs
for (const r of recs) { .filter(r => r.type === 'assistant' && r.isSidechain !== true)
const t = tsOf(r) .filter(r => tsOf(r) > ts && tsOf(r) < end)
if (r.type !== 'assistant' || r.isSidechain === true) continue .flatMap(textsOf)
if (!(t > ts && t < end)) continue .filter(([key]) => !seen.has(key) && !!seen.add(key))
for (const b of blocksOf(r).filter(b => b.type === 'text')) { .map(([, text]) => text)
const key = `${r.message?.id}:${b.text}` // EVAL-037 .join('\n')
if (!seen.has(key)) parts.push(String(b.text))
seen.add(key)
}
}
return parts.join('\n')
} }
export type PromptJoin = { prompt: string; injected: string; reply: string } export type PromptJoin = { prompt: string; injected: string; reply: string }
@@ -178,7 +185,7 @@ export type ToolJoin = { command: string; known: string; next: boolean[] }
/** The failure a known line answered and the next 3 Bash results. */ /** The failure a known line answered and the next 3 Bash results. */
export function joinTool(recs: Rec[], rec: LogRecord): ToolJoin | undefined { export function joinTool(recs: Rec[], rec: LogRecord): ToolJoin | undefined {
const needle = KNOWN_HEAD + (rec.ids[0] ?? '') const needle = `${KNOWN_HEAD}${rec.ids[0] ?? ''} ` // BLK-003 != BLK-0031
const att = attachmentAfter(recs, rec.at, 'tool.call', needle) const att = attachmentAfter(recs, rec.at, 'tool.call', needle)
const raw = att?.attachment?.toolUseID const raw = att?.attachment?.toolUseID
if (att === undefined || typeof raw !== 'string') return undefined if (att === undefined || typeof raw !== 'string') return undefined
+14 -7
View File
@@ -13,8 +13,10 @@ check() { if [ "$2" = "$3" ]; then pass=$((pass+1)); else fail=$((fail+1));
printf 'FAIL %s: got[%s] want[%s]\n' "$1" "$2" "$3"; fi; } printf 'FAIL %s: got[%s] want[%s]\n' "$1" "$2" "$3"; fi; }
# home <name>: a fixture HOME with a store file (unless "nostore") # home <name>: a fixture HOME with a store file (unless "nostore")
home() { home() {
mkdir -p "$WORK/$1/.claude/plugins/store" "$WORK/$1/.claude/memory-feedback" mkdir -p "$WORK/$1/.claude/plugins/store" \
[ "$1" = nostore ] || : > "$WORK/$1/.claude/plugins/store/memory_skills-dir-a.json" "$WORK/$1/.claude/memory-feedback"
[ "$1" = nostore ] ||
: > "$WORK/$1/.claude/plugins/store/memory_skills-dir-a.json"
echo "$WORK/$1" echo "$WORK/$1"
} }
OPEN='{"lastRun":1,"closed":false}' OPEN='{"lastRun":1,"closed":false}'
@@ -24,24 +26,29 @@ old() {
touch -t 202601010000 "$1/.claude/memory-feedback/state.json" touch -t 202601010000 "$1/.claude/memory-feedback/state.json"
} }
run() { HOME="$1" bash "$LIB" 2>&1; echo "rc=$?"; } run() { HOME="$1" bash "$LIB" 2>&1; echo "rc=$?"; }
DUE='memory feedback: last run 5+ d ago, transcripts expire at 7 → /memory-feedback' DUE='memory feedback: last run 5+ d ago, '
DUE+='transcripts expire at 7 → /memory-feedback'
check MR1-no-store "$(run "$(home nostore)")" "rc=0" check MR1-no-store "$(run "$(home nostore)")" "rc=0"
check MR2-never "$(run "$(home never)")" "memory feedback: never run → /memory-feedback NEVER='memory feedback: never run → /memory-feedback'
check MR2-never "$(run "$(home never)")" "$NEVER
rc=0" rc=0"
h=$(home due); old "$h" "$OPEN" h=$(home due); old "$h" "$OPEN"
check MR3-due "$(run "$h")" "$DUE check MR3-due "$(run "$h")" "$DUE
rc=0" rc=0"
h=$(home quiet); old "$h" "$OPEN" h=$(home quiet); old "$h" "$OPEN"
touch -t 202512010000 "$h/.claude/plugins/store/memory_skills-dir-a.json" touch -t 202512010000 \
"$h/.claude/plugins/store/memory_skills-dir-a.json"
check MR4-store-unchanged "$(run "$h")" "rc=0" check MR4-store-unchanged "$(run "$h")" "rc=0"
h=$(home fresh) h=$(home fresh)
printf '%s' "$OPEN" > "$h/.claude/memory-feedback/state.json" printf '%s' "$OPEN" > "$h/.claude/memory-feedback/state.json"
touch -t 209901010000 "$h/.claude/plugins/store/memory_skills-dir-a.json" touch -t 209901010000 \
"$h/.claude/plugins/store/memory_skills-dir-a.json"
check MR5-ran-today "$(run "$h")" "rc=0" check MR5-ran-today "$(run "$h")" "rc=0"
h=$(home closed); old "$h" '{"lastRun":1,"closed":true}' h=$(home closed); old "$h" '{"lastRun":1,"closed":true}'
check MR6-closed "$(run "$h")" "rc=0" check MR6-closed "$(run "$h")" "rc=0"
h=$(home off); old "$h" "$OPEN"; printf '{"recall": false}' > "$h/.claude/memory.json" h=$(home off); old "$h" "$OPEN"
printf '{"recall": false}' > "$h/.claude/memory.json"
check MR7-recall-off "$(run "$h")" "rc=0" check MR7-recall-off "$(run "$h")" "rc=0"
h=$(home locked); old "$h" "$OPEN" h=$(home locked); old "$h" "$OPEN"
chmod 000 "$h/.claude/memory-feedback/state.json" chmod 000 "$h/.claude/memory-feedback/state.json"
+17 -7
View File
@@ -1734,6 +1734,22 @@ function claimPicks(st: State, key: string, picked: Entry[]): string[] {
return taken return taken
} }
/** Claims the picks of a rerank answer; the block to inject. */
function injectPicks(
st: State,
key: string,
answer: string,
cands: Entry[],
): Picked {
const picked = picksOf(answer, cands)
if (picked.length === 0) return nothing('none')
const taken = claimPicks(st, key, picked)
const block = recallBlock(
picked.filter(e => taken.includes(e.id)).map(recallLine), RECALL_CAP)
if (block === '') return nothing('refused')
return { block, ids: taken, outcome: 'injected' }
}
/** Asks haiku among the candidates and claims its picks. */ /** Asks haiku among the candidates and claims its picks. */
async function pickRecall( async function pickRecall(
$: Api, $: Api,
@@ -1748,13 +1764,7 @@ async function pickRecall(
// a clock step back must not make the record invalid (isCount) // a clock step back must not make the record invalid (isCount)
const ms = Math.max(0, Math.round((await $.clock.now()) - now)) const ms = Math.max(0, Math.round((await $.clock.now()) - now))
if (answer === undefined) return { ...nothing('timeout'), ms } if (answer === undefined) return { ...nothing('timeout'), ms }
const picked = picksOf(answer, cands) return { ...injectPicks(st, key, answer, cands), ms }
if (picked.length === 0) return { ...nothing('none'), ms }
const taken = claimPicks(st, key, picked)
const block = recallBlock(
picked.filter(e => taken.includes(e.id)).map(recallLine), RECALL_CAP)
if (block === '') return { ...nothing('refused'), ms }
return { block, ids: taken, outcome: 'injected', ms }
} }
/** /**
+7 -4
View File
@@ -33,8 +33,10 @@ Read `<dir>/worksheet.md`. Fill the third column of the skeleton
- `relevant`: the entry bears on what the prompt asked or what the reply - `relevant`: the entry bears on what the prompt asked or what the reply
did; `irrelevant` otherwise. Judge the entry against the ask, not the did; `irrelevant` otherwise. Judge the entry against the ask, not the
reply's quality. reply's quality.
- A proof, bench or recall-testing session: replace that item's lines by - A proof, bench or recall-testing session: every item of that session,
one `<item> - test` line. with lines or not (none, timeout and paused items included), gets one
`<item> - test` line in place of its lines. The header's `s:` shows
the session, the time the order.
Lost items have no line. Lost items have no line.
## STEP 3 — SPOT CHECK ## STEP 3 — SPOT CHECK
@@ -54,8 +56,9 @@ user confirms or flips each. Apply the flips to the draft (LRN-168).
On up to 5 prompt items whose outcome is injected or none, search the On up to 5 prompt items whose outcome is injected or none, search the
registries of that item's repo, whose absolute path is in the item header registries of that item's repo, whose absolute path is in the item header
(`bash ~/.claude/lib/memory.sh -C <path> index` + entry bodies) for live entries that bore on the ask and were not (`bash ~/.claude/lib/memory.sh -C <path> index` + entry bodies) for
injected. Each one: `<item> <id> missed <cause>`, cause `no-overlap` (no live entries that bore on the ask and were not injected. Each one:
`<item> <id> missed <cause>`, cause `no-overlap` (no
content word shared with the prompt) or `overlap`. A searched item with no content word shared with the prompt) or `overlap`. A searched item with no
miss: `<item> - checked`. Example line: `3 LRN-901 missed no-overlap`. miss: `<item> - checked`. Example line: `3 LRN-901 missed no-overlap`.