fix(memory-feedback): post-fix precision split, 3rd pick retired, bounded transcript cache, strict labels

Precision, triggers and the miss share count only rows after the gate-1
fix merge (d3d5e6c, 2026-11-12T19:53:59Z); their actions wait for 20
post-fix asks, a one-off NEW line marks the sample and the reminder
keeps prompting until then. The 3rd-pick signal is gone. The reader
holds one parsed transcript at a time. labels.tsv rows and state.json
are validated on read. Exact key sets of log records pinned.
This commit is contained in:
bchanot committed 2026-11-13 19:47:37 +01:00
1 parent e132488aef
commit 9f9412aaba
11 files changed
+478 -67

No files matched your search

+4 -3
View File
@@ -2,9 +2,10 @@
# lib/memory-feedback-remind.sh — the phase 4d reminder line of the
# session-start banner (BDR-057: transcripts live 7 days). One line when
# /memory-feedback never ran, or ran REMIND_DAYS+ days ago and the memory
# store changed since. Silent once both gates are recorded ("closed":true),
# with "recall": false, or on any error; always rc 0. bash 3.2 safe,
# portable (find -mtime, no stat -c), no JSON parse.
# store changed since. Silent once all three gate marks (prompt, tool,
# post-fix sample) are recorded ("closed":true), with "recall": false, or
# on any error; always rc 0. bash 3.2 safe, portable (find -mtime, no
# stat -c), no JSON parse.
REMIND_DAYS=5
state="$HOME/.claude/memory-feedback/state.json"
+60
View File
@@ -105,6 +105,15 @@ test('L8: state defaults, round trip, a corrupt file throws', () => {
expect(() => readState(f)).toThrow('state.json')
})
test('L8b: closed is derived from the three marks, not the stored flag', () => {
const f = path.join(tmp(), 'state.json')
fs.writeFileSync(f, '{"lastRun":1,"promptAt":1,"toolAt":1,"closed":true}')
expect(readState(f).closed).toBe(false)
fs.writeFileSync(f,
'{"lastRun":1,"promptAt":1,"toolAt":1,"postFixAt":1,"closed":false}')
expect(readState(f).closed).toBe(true)
})
test('L9: repo tags keep same-named repos apart', () => {
expect(tagOf('/a/x')).toMatch(/^x#[0-9a-f]{6}$/)
expect(tagOf('/a/x')).toBe(tagOf('/a/x'))
@@ -116,3 +125,54 @@ test('L10: the skeleton draft lists each injected id once', () => {
item(3, { lost: true })]).split('\n').filter(l => !l.startsWith('#'))
expect(lines.filter(l => l !== '')).toEqual(['1\tBDR-001\t', '1\tLRN-002\t'])
})
test('L11: readRows round trips every label kind', () => {
const file = path.join(tmp(), 'mf', 'labels.tsv')
const full = item(1)
const drafts = parseDraft([OK, '2\tBDR-009\tmissed\tno-overlap',
'1\t-\tchecked', '2\t-\ttest'].join('\n'), [full,
item(2, { ids: [] })])
if (typeof drafts === 'string') throw new Error(drafts)
const rows = [...rowsOf(full, drafts), ...rowsOf(item(2, { ids: [] }),
drafts), ...rowsOf(item(3, { lost: true }), [])]
appendRows(file, rows)
expect(new Set(rows.map(r => r.label))).toEqual(new Set(['ask', 'lost',
'relevant', 'irrelevant', 'missed', 'checked', 'test']))
expect(readRows(file)).toEqual(rows)
})
test('L12: a line with a wrong id, label, channel or at is refused', () => {
const good: Row = { at: 5, root: 'r#000000', session: 's',
channel: 'prompt', trigger: 'fresh', outcome: 'injected', cands: 1,
ms: 2, pick: 1, id: 'BDR-001', label: 'relevant', cause: '' }
expect(fromLine(toLine(good))).toEqual(good)
const bad: Partial<Row>[] = [{ label: 'maybe' as never },
{ id: 'BDR-1' }, { id: '' }, { label: 'ask' }, // ask carries no id
{ channel: 'x' as never }, { at: Number.NaN }, { at: -5 },
{ at: 1.5 }]
for (const b of bad) expect(fromLine(toLine({ ...good, ...b })))
.toBeUndefined()
expect(fromLine(toLine({ ...good, id: '', label: 'ask' }))).toBeDefined()
})
test('L13: a damaged line names itself and what the .bak holds', () => {
const file = path.join(tmp(), 'labels.tsv')
appendRows(file, rowsOf(item(1), []))
fs.appendFileSync(file, '1\tr\ts\tprompt\t\t\t\t\t\tNOPE\trelevant\t\n')
expect(() => readRows(file)).toThrow(/line 3 unreadable.*labels\.tsv\.bak/)
})
test('L14: state.json that is not an object is refused', () => {
const f = path.join(tmp(), 'state.json')
for (const raw of ['[1]', 'null', '7', '"x"']) {
fs.writeFileSync(f, raw)
expect(() => readState(f)).toThrow('state.json')
}
writeState(f, { lastRun: 3, postFixAt: 3, closed: false })
expect(readState(f).postFixAt).toBe(3)
})
test('L15: an entry label with a cause says it takes none', () => {
const d = parseDraft(`${OK}\n1\tBDR-001\trelevant\toverlap`, ITEMS)
expect(d).toContain('takes no cause')
})
+32 -8
View File
@@ -12,6 +12,8 @@ const DRAFT_LABELS = ['relevant', 'irrelevant', 'test', 'missed', 'checked']
const ENTRY_LABELS = ['relevant', 'irrelevant']
const CAUSES = ['no-overlap', 'overlap']
const ID_RE = /^(BDR|LRN|BLK)-\d{3,}$/
const ROW_LABELS = [...DRAFT_LABELS, 'ask', 'lost']
const BARE_LABELS = ['ask', 'lost', 'test', 'checked'] // carry no entry id
/** One worksheet item: a log record and what the join found. */
export type Item = {
@@ -25,9 +27,11 @@ export type Row = {
pick?: number; id: string; label: Label; cause: string
}
export type Draft = { item: number; id: string; label: Label; cause: string }
/** promptAt/toolAt: when each gate verdict was first reached. */
/** promptAt/toolAt/postFixAt: when each gate verdict or floor was first
* reached. */
export type State = {
lastRun: number; promptAt?: number; toolAt?: number; closed: boolean
postFixAt?: number // the post-fix prompt floor, first crossed (optional)
}
const COLS = ['at', 'root', 'session', 'channel', 'trigger', 'outcome',
@@ -52,10 +56,20 @@ export function toLine(r: Row): string {
.join('\t')
}
/** A labels.tsv line as a row; the header or a short line: undefined. */
/** Whether the fields hold a known label, its id shape, a channel and
* an integer `at`. */
function sound(f: string[]): boolean {
const label = f[10] ?? ''
const id = f[9] ?? ''
const idOk = BARE_LABELS.includes(label) ? id === '' : ID_RE.test(id)
return ROW_LABELS.includes(label) && idOk &&
(f[3] === 'prompt' || f[3] === 'tool') && /^\d+$/.test(f[0] ?? '')
}
/** A labels.tsv line as a row; the header or a damaged line: undefined. */
export function fromLine(line: string): Row | undefined {
const f = line.split('\t')
if (f.length !== COLS.length || f[0] === 'at') return undefined
if (f.length !== COLS.length || !sound(f)) return undefined
return {
at: Number(f[0]), root: f[1] ?? '', session: f[2] ?? '',
channel: f[3] as Channel, trigger: f[4] ?? '', outcome: f[5] ?? '',
@@ -87,11 +101,12 @@ export function readRows(file: string): Row[] {
const r = fromLine(line)
if (r !== undefined) return [r]
throw new Error(`labels.tsv line ${i + 1} unreadable: fix it by hand ` +
'or restore labels.tsv.bak')
'(labels.tsv.bak holds the state before the last append)')
})
}
/** Appends rows; the previous file is copied to labels.tsv.bak first. */
/** Appends rows; the previous file is copied to labels.tsv.bak first (one
* state: the one before the last append). */
export function appendRows(file: string, rows: Row[]): void {
mkdir(file)
if (fs.existsSync(file)) fs.copyFileSync(file, `${file}.bak`)
@@ -104,9 +119,17 @@ export function readState(file: string): State {
if (!fs.existsSync(file)) return { lastRun: 0, closed: false }
try {
const s = JSON.parse(fs.readFileSync(file, 'utf8'))
if (typeof s !== 'object' || s === null || Array.isArray(s)) {
throw new Error('state.json is not an object')
}
const at = (v: unknown) => typeof v === 'number' ? v : undefined
return { lastRun: Number(s.lastRun) || 0, promptAt: at(s.promptAt),
toolAt: at(s.toolAt), closed: s.closed === true }
const [promptAt, toolAt, postFixAt] =
[at(s.promptAt), at(s.toolAt), at(s.postFixAt)]
// closed is derived: a stored flag without all three marks reads open
const closed = promptAt !== undefined && toolAt !== undefined &&
postFixAt !== undefined
return { lastRun: Number(s.lastRun) || 0, promptAt, toolAt, postFixAt,
closed }
} catch {
throw new Error(`state.json unreadable (${file}): fix or delete it`)
}
@@ -135,7 +158,8 @@ function fitError(d: Draft, it: Item | undefined): string {
? '' : `item ${d.item}: only checked or test take id -`
}
if (ENTRY_LABELS.includes(d.label)) {
return plain && it.ids.includes(d.id)
if (!plain) return `item ${d.item}: ${d.label} takes no cause`
return it.ids.includes(d.id)
? '' : `item ${d.item}: ${d.id} was not injected there`
}
const ok = d.label === 'missed' && it.channel === 'prompt' &&
+126 -2
View File
@@ -1,12 +1,12 @@
import { expect, test } from 'bun:test'
import { DAY_MS, GATE_DAYS } from './thresholds.ts'
import { DAY_MS, GATE1_FIX_AT, GATE_DAYS } from './thresholds.ts'
import {
actionsOf, clockStart, gateVerdict, metricsOf, p90, precision,
renderFeedback, type Metrics,
} from './metrics.ts'
import type { Label, Row } from './labels.ts'
let clock = 1000
let clock = GATE1_FIX_AT + 1000 // every default row is post-fix
/** A labels.tsv row; each call gets its own `at` unless given. */
function row(over: Partial<Row> & { label: Label }): Row {
clock += 1
@@ -129,3 +129,127 @@ test('M10: FEEDBACK.md shows n per channel and trigger', () => {
expect(text).toContain('| prompt precision | 33 % (n=3) |')
expect(text).toContain('| tool precision | n/a (n=0) |')
})
const shift = (rs: Row[], at: number) => rs.map(r => ({ ...r, at }))
const PRE = Date.parse('2026-11-12T15:00:00Z')
test('M11: the fix instant splits prompt precision, not noisy', () => {
const old = shift(ask(['irrelevant', 'irrelevant', 'irrelevant']), PRE)
const mm = metricsOf([...old, ...ask(['relevant'])])
expect(mm.precision).toBe(1)
expect(mm.n.prompt).toBe(1)
expect(mm.preFix).toBe(3)
expect(mm.promptN).toBe(2) // the gate count reads every row
const thrice = [1, 2, 3].flatMap(() => shift(ask(['irrelevant']), PRE))
expect(metricsOf(thrice).noisy).toEqual(['r#000000 LRN-001'])
const edge = [...shift(ask(['irrelevant']), GATE1_FIX_AT - 1),
...shift(ask(['relevant']), GATE1_FIX_AT)]
expect(metricsOf(edge).precision).toBe(1)
expect(metricsOf(edge).n.prompt).toBe(1)
})
test('M12: the miss share reads post-fix asks only', () => {
const miss = (at: number) => [row({ at, id: '', label: 'ask' }),
row({ at, id: 'BDR-009', label: 'missed', cause: 'overlap' })]
const mm = metricsOf([...miss(GATE1_FIX_AT - 1), ...miss(GATE1_FIX_AT)])
expect(mm.sampled).toBe(1)
expect(mm.missShare).toBe(1)
})
/** Metrics whose every gate is reached, with `over` on top. */
const reached = (over: Partial<Metrics>): Metrics => ({
...metricsOf([]), start: 0, promptN: 20, toolN: 10, postAsks: 20, ...over,
})
const acts = (over: Partial<Metrics>) => {
const mm = reached(over)
return actionsOf(mm, gateVerdict(mm, 1))
}
const sized = (i: number, k: number, f: number) => ({
idle: { rate: i, n: 5 }, keyword: { rate: k, n: 5 },
fresh: { rate: f, n: 5 },
})
test('M13: each comparator at its threshold and one step past', () => {
const has = (over: Partial<Metrics>, part: string) =>
acts(over).some(a => a.includes(part))
expect(has({ late: 0.25 }, 'timeout + paused')).toBe(false)
expect(has({ late: 0.26 }, 'timeout + paused')).toBe(true)
expect(has({ msP90: 2500 }, 'rerank p90')).toBe(false)
expect(has({ msP90: 2499 }, 'rerank p90')).toBe(true)
expect(has({ tool: 0.5 }, 'tool precision')).toBe(false)
expect(has({ tool: 0.49 }, 'tool precision')).toBe(true)
expect(has({ missShare: 1 / 3 }, 'misses in 1 ask')).toBe(true)
expect(has({ missShare: 0.33 }, 'misses in 1 ask')).toBe(false)
const by = sized(0.4, 0.9, 0.9)
expect(has({ precision: 0.64, byTrigger: by }, 'prompt precision'))
.toBe(false)
expect(has({ precision: 0.63, byTrigger: by }, 'prompt precision'))
.toBe(true)
const low = (r: number) => sized(r, 0.9, 0.9)
expect(has({ precision: 0.5, byTrigger: low(0.5) }, 'prompt precision'))
.toBe(false)
expect(has({ precision: 0.5, byTrigger: low(0.49) }, 'prompt precision'))
.toBe(true)
})
test('M14: the worst trigger ties go idle, keyword, fresh', () => {
const pick = (by: ReturnType<typeof sized>) => acts({ precision: 0.3,
byTrigger: by }).find(a => a.startsWith('prompt precision'))
expect(pick(sized(0.25, 0.25, 0.25))).toContain('the idle trigger')
expect(pick(sized(0.9, 0.25, 0.25))).toContain('the keyword trigger')
expect(pick(sized(0.25, 0.2, 0.9))).toContain('the keyword trigger')
expect(pick(sized(0.9, 0.9, 0.1))).toContain('the fresh trigger')
})
test('M15: promotion needs 3 hits in 2 sessions, noisy 3 never hit', () => {
const rel = (s: string) => ask(['relevant'], { session: s })
expect(metricsOf([...rel('a'), ...rel('a'), ...rel('b')]).promote)
.toEqual(['r#000000 LRN-001'])
expect(metricsOf([...rel('a'), ...rel('b')]).promote).toEqual([])
const irr = () => ask(['irrelevant'])
expect(metricsOf([...irr(), ...irr(), ...irr()]).noisy).toHaveLength(1)
expect(metricsOf([...irr(), ...irr()]).noisy).toEqual([])
})
test('M16: the post-fix floor gates the prompt actions, 19 vs 20', () => {
const idle = (n: number) => Array.from({ length: n }, () =>
ask(['irrelevant'], { trigger: 'idle' })).flat()
const run = (n: number) => {
const mm = { ...metricsOf(idle(n)), start: 0, promptN: 20 }
return actionsOf(mm, gateVerdict(mm, 1))
}
expect(run(19).some(a => a.startsWith('prompt precision'))).toBe(false)
expect(run(20).some(a => a.startsWith('prompt precision'))).toBe(true)
const text = renderFeedback(metricsOf(idle(19)), gateVerdict(
metricsOf(idle(19)), 1), 0)
expect(text).toContain('Post-fix prompt asks: 19/20 (since ' +
'2026-11-12T19:53:59.000Z), pre-fix entries excluded: 0')
})
test('M17: below the floor the miss action waits, late and p90 fire', () => {
const asks = Array.from({ length: 19 }, () =>
ask(['irrelevant'], { trigger: 'idle', ms: 2000 })).flat()
const miss = row({ id: 'BDR-009', label: 'missed', cause: 'overlap',
at: asks[0]!.at })
const late = Array.from({ length: 8 }, () =>
row({ id: '', label: 'ask', outcome: 'timeout' }))
const mm = { ...metricsOf([...asks, miss, ...late]), start: 0, promptN: 20 }
expect(mm.missShare).toBeGreaterThanOrEqual(1 / 3)
const out = actionsOf(mm, gateVerdict(mm, 1))
expect(out.some(a => a.startsWith('misses in 1 ask'))).toBe(false)
expect(out.some(a => a.startsWith('timeout + paused'))).toBe(true)
expect(out.some(a => a.startsWith('rerank p90'))).toBe(true)
})
test('M18: an ask with 2 entries counts once toward the floor', () => {
const two = Array.from({ length: 19 }, () =>
ask(['relevant', 'relevant'])).flat()
expect(metricsOf(two).postAsks).toBe(19)
expect(metricsOf([...two, ...ask(['relevant'])]).postAsks).toBe(20)
})
test('M19: a pre-fix row still counts toward promote', () => {
const old = (s: string) => shift(ask(['relevant'], { session: s }), PRE)
expect(metricsOf([...old('a'), ...old('a'), ...old('b')]).promote)
.toEqual(['r#000000 LRN-001'])
})
+24 -17
View File
@@ -2,7 +2,7 @@
// FEEDBACK.md text, computed from labels.tsv rows only (transcripts expire
// at 7 days, the rows stay). Spec §4.
import {
DAY_MS, GATE_DAYS, PROMPT_GATE_N, T, TOOL_GATE_N,
DAY_MS, GATE1_FIX_AT, GATE_DAYS, PROMPT_GATE_N, T, TOOL_GATE_N,
} from './thresholds.ts'
import { keyOf, type Row } from './labels.ts'
@@ -13,10 +13,12 @@ type Rate = number | undefined
type Sized = { rate: Rate; n: number }
export type Metrics = {
start?: number; promptN: number; toolN: number; lost: number
precision: Rate; byTrigger: Record<string, Sized>; thirdPick: Rate
precision: Rate; byTrigger: Record<string, Sized>
late: Rate; msP90: Rate; tool: Rate; missShare: Rate; sampled: number
noisy: string[]; promote: string[]
n: { prompt: number; third: number; tool: number } // entries
n: { prompt: number; tool: number } // entries
postAsks: number // post-fix prompt asks labelled: the floor of the actions
preFix: number // pre-fix prompt entries left out of the prompt signals
}
const share = (k: number, n: number): Rate => n === 0 ? undefined : k / n
@@ -66,7 +68,6 @@ function byTrigger(entries: Row[]): Record<string, Sized> {
/** Entry counts behind the three channel precisions. */
const countsOf = (entries: Row[]) => ({
prompt: entries.filter(r => r.channel === 'prompt').length,
third: entries.filter(r => r.channel === 'prompt' && r.pick === 3).length,
tool: entries.filter(r => r.channel === 'tool').length,
})
@@ -109,25 +110,29 @@ function promote(entries: Row[]): string[] {
const itemsOf = (entries: Row[], ch: string) =>
new Set(entries.filter(r => r.channel === ch).map(keyOf)).size
/** Every 4d signal, from labels.tsv rows. */
const postFix = (r: Row) => r.at >= GATE1_FIX_AT
/** Every 4d signal, from labels.tsv rows. Prompt precision, triggers and
* the miss share read post-fix rows (BDR-146); the rest read all rows. */
export function metricsOf(all: Row[]): Metrics {
const rows = liveRows(all)
const entries = rows.filter(isEntry)
const prompt = entries.filter(r => r.channel === 'prompt')
const recent = entries.filter(r => r.channel === 'prompt' && postFix(r))
const asks = rows.filter(r => r.channel === 'prompt' &&
(r.label === 'ask' || r.label === 'lost'))
const miss = misses(rows)
const miss = misses(rows.filter(postFix))
return {
start: clockStart(all, []),
promptN: itemsOf(entries, 'prompt'), toolN: itemsOf(entries, 'tool'),
lost: rows.filter(r => r.label === 'lost').length,
precision: precision(prompt), byTrigger: byTrigger(prompt),
thirdPick: precision(prompt.filter(r => r.pick === 3)),
precision: precision(recent), byTrigger: byTrigger(recent),
late: lateShare(asks), msP90: p90(asks.flatMap(r => r.ms ?? [])),
tool: precision(entries.filter(r => r.channel === 'tool')),
missShare: miss.rate, sampled: miss.sampled,
noisy: noisy(entries), promote: promote(entries),
n: countsOf(entries),
n: { ...countsOf(entries), prompt: recent.length },
postAsks: itemsOf(recent, 'prompt'),
preFix: countsOf(entries).prompt - recent.length,
}
}
@@ -158,13 +163,11 @@ function worstTrigger(by: Record<string, Sized>): string | undefined {
function promptActions(m: Metrics): string[] {
const out: string[] = []
const worst = worstTrigger(m.byTrigger)
if (under(m.precision, T.promptKeep) && worst !== undefined) {
const floor = m.postAsks >= PROMPT_GATE_N
if (floor && under(m.precision, T.promptKeep) && worst !== undefined) {
out.push(`prompt precision under ${pct(T.promptKeep)}: ` +
`tighten or drop the ${worst} trigger`)
}
if (under(m.thirdPick, T.thirdPick)) {
out.push(`3rd pick under ${pct(T.thirdPick)}: cap recall at 2 entries`)
}
if ((m.late ?? 0) > T.lateShare) {
out.push(`timeout + paused over ${pct(T.lateShare)}: ` +
'revisit RERANK_MS or skip the rerank on short lists')
@@ -172,7 +175,7 @@ function promptActions(m: Metrics): string[] {
if (under(m.msP90, T.msP90)) {
out.push(`rerank p90 under ${T.msP90} ms: lower RERANK_MS`)
}
if ((m.missShare ?? 0) >= T.missShare) {
if (floor && (m.missShare ?? 0) >= T.missShare) {
out.push('misses in 1 ask in 3 or more: diagnose by cause')
}
return out
@@ -197,6 +200,11 @@ function toolLine(m: Metrics, v: Verdict): string {
: `Tool gate: ${v.tool} (${m.toolN}/${TOOL_GATE_N})`
}
const postFixLine = (m: Metrics) =>
`Post-fix prompt asks: ${m.postAsks}/${PROMPT_GATE_N} (since ` +
`${new Date(GATE1_FIX_AT).toISOString()}), pre-fix entries excluded: ` +
`${m.preFix}`
/** FEEDBACK.md: counts and shares; repo tags only in the last lists. */
export function renderFeedback(m: Metrics, v: Verdict, now: number) {
const acts = actionsOf(m, v)
@@ -204,12 +212,11 @@ export function renderFeedback(m: Metrics, v: Verdict, now: number) {
`# Memory feedback (${day(now)})`, '',
`Gate clock start: ${m.start === undefined ? 'not started' : day(m.start)}`,
`Prompt gate: ${v.prompt} (${m.promptN}/${PROMPT_GATE_N})`,
toolLine(m, v), `Lost records: ${m.lost}`, '',
toolLine(m, v), `Lost records: ${m.lost}`, postFixLine(m), '',
'| Signal | Value |', '|---|---|',
`| prompt precision | ${pct(m.precision)} (n=${m.n.prompt}) |`,
...Object.entries(m.byTrigger).map(([k, x]) =>
`| precision, ${k} | ${pct(x.rate)} (n=${x.n}) |`),
`| 3rd pick precision | ${pct(m.thirdPick)} (n=${m.n.third}) |`,
`| timeout + paused | ${pct(m.late)} |`,
`| rerank ms p90 | ${m.msP90 ?? 'n/a'} |`,
`| tool precision | ${pct(m.tool)} (n=${m.n.tool}) |`,
+136 -2
View File
@@ -3,6 +3,8 @@ import { spawnSync } from 'node:child_process'
import * as fs from 'node:fs'
import * as os from 'node:os'
import * as path from 'node:path'
import { GATE1_FIX_AT } from './thresholds.ts'
import { tagOf } from './labels.ts'
import { slugOf } from './transcripts.ts'
import {
apply, checkDir, population, readLogs, safePath, sample, storeFile,
@@ -17,7 +19,7 @@ const mk = (pfx: string) => {
dirs.push(d)
return d
}
const T0 = Date.parse('2026-11-10T10:00:00.000Z')
const T0 = GATE1_FIX_AT + 20 * 3600000 // post-fix: the floor counts it
const NOW = T0 + 2 * 3600000 // every fixture record is 2 h old
const iso = (ms: number) => new Date(T0 + ms).toISOString()
// built at run time: no secret-shaped literal in the repo (BLK-041)
@@ -156,7 +158,7 @@ test('R8: the CLI refuses a dir inside a git work tree', () => {
test('R10: a tampered items.json is refused, nothing written', () => {
for (const bad of [{ root: '/repo/x' }, { root: 'a\tb#abcdef' },
{ session: '../s' }, { channel: 'x' }, { ids: ['../x'] },
{ cited: ['BDR-999'] }, { at: -1 }, { lost: 'no' }]) {
{ cited: ['BDR-999'] }, { at: -1 }, { at: 1e21 }, { lost: 'no' }]) {
const { p, work } = fixture(STORE)
worksheet(work, p, NOW)
const f = path.join(work, 'items.json')
@@ -181,3 +183,135 @@ test('safePath keeps plain paths and withholds shell-active ones', () => {
for (const bad of ['/tmp/$(id)', '/tmp/`id`', "/tmp/a'b", '/a/../b'])
expect(safePath(bad)).toBe(w)
})
/** The 3 transcript records of the prompt asked at `T0 + ms`. */
const turn = (ms: number, tag: string) => [
{ type: 'user', timestamp: iso(ms - 50), origin: { kind: 'human' },
message: { content: `prompt-${tag}` } },
{ type: 'attachment', timestamp: iso(ms + 400), attachment: {
type: 'hook_additional_context', hookName: 'prompt.submit',
content: ['memory recall:\nBDR-001 x'] } },
{ type: 'assistant', timestamp: iso(ms + 900),
message: { id: `m${ms}`, content: [{ type: 'text', text: 'ok' }] } },
]
const jsonl = (recs: object[], torn = 0) => recs.map(r => JSON.stringify(r))
.concat(Array(torn).fill('{"torn')).join('\n') + '\n'
const put = (p: Paths, session: string, text: string) => fs.writeFileSync(
path.join(p.projects, slugOf('/repo/x'), `${session}.jsonl`), text)
const sheetOf = (work: string) =>
fs.readFileSync(path.join(work, 'worksheet.md'), 'utf8')
const promptAt = (sheet: string, t: string) =>
sheet.indexOf(`Prompt: prompt-${t}`)
const itemsOf = (work: string): { n: number }[] => JSON.parse(
fs.readFileSync(path.join(work, 'items.json'), 'utf8'))
test('R11: A,B,A sessions: each file read once, order kept', () => {
const { p, work } = fixture({ 'l:/repo/x': [
REC(T0, { session: 'sA' }), REC(T0 + 60000, { session: 'sB' }),
REC(T0 + 120000, { session: 'sA' })] })
put(p, 'sA', jsonl([...turn(0, 'A1'), ...turn(120000, 'A2')], 1))
put(p, 'sB', jsonl(turn(60000, 'B1')))
expect(worksheet(work, p, NOW)).toBe(3)
const sheet = sheetOf(work)
expect(sheet).toContain('Skipped 1 torn transcript lines.')
const heads = sheet.split('\n').filter(l => l.startsWith('## Item'))
expect(heads.map(h => h.slice(0, 9))).toEqual(['## Item 1',
'## Item 2', '## Item 3'])
expect(heads.map(h => /s:(s.)/.exec(h)?.[1])).toEqual(['sA', 'sB', 'sA'])
expect(promptAt(sheet, 'A1')).toBeLessThan(promptAt(sheet, 'B1'))
expect(promptAt(sheet, 'B1')).toBeLessThan(promptAt(sheet, 'A2'))
expect(itemsOf(work).map(i => i.n)).toEqual([1, 2, 3])
})
test('R12: torn lines are summed across files', () => {
const { p, work } = fixture({ 'l:/repo/x': [
REC(T0, { session: 'sA' }), REC(T0 + 60000, { session: 'sB' })] })
put(p, 'sA', jsonl(turn(0, 'A'), 1))
put(p, 'sB', jsonl(turn(60000, 'B'), 2))
worksheet(work, p, NOW)
expect(sheetOf(work)).toContain('Skipped 3 torn transcript lines.')
})
test('R13: a root at the log cap, newer than the last run, is noted', () => {
const full = Array.from({ length: 200 }, (_, i) =>
REC(T0 + i * 1000, { ids: [], outcome: 'none' }))
const { p, work } = fixture({ 'l:/repo/x': full })
fs.mkdirSync(p.out)
fs.writeFileSync(path.join(p.out, 'state.json'),
JSON.stringify({ lastRun: T0 - 1, closed: false }))
worksheet(work, p, NOW)
expect(sheetOf(work)).toContain(
`Possible eviction before labelling: ${tagOf('/repo/x')}`)
})
const use = (ms: number, id: string, command: string) => ({
type: 'assistant', timestamp: iso(ms), message: { id: `a${id}`,
content: [{ type: 'tool_use', id, name: 'Bash',
input: { command } }] } })
const result = (ms: number, id: string, err: boolean) => ({
type: 'user', timestamp: iso(ms), message: { content: [
{ type: 'tool_result', tool_use_id: id, is_error: err }] } })
const toolRec = (at: number, session: string) => REC(at, { session,
channel: 'tool', ids: ['BLK-003'], trigger: undefined,
outcome: undefined, cands: 1, ms: undefined })
test('R14: a tool item shows command, known line, next results', () => {
const { p, work } = fixture({ 'l:/repo/x': [
toolRec(T0, 'sA'), toolRec(T0 + 60000, 'sB')] })
put(p, 'sA', jsonl([use(-100, 'u1', 'make test'), result(-50, 'u1', true),
{ type: 'attachment', timestamp: iso(400), attachment: {
type: 'hook_additional_context', hookName: 'tool.call',
toolUseID: 'u1-context', content: ['memory: known BLK-003 x'] } },
use(500, 'u2', 'ls'), result(600, 'u2', false)]))
expect(worksheet(work, p, NOW)).toBe(2)
expect(sheetOf(work)).toContain('Command: make test')
expect(sheetOf(work)).toContain('Known: memory: known BLK-003 x')
expect(sheetOf(work)).toContain('Next Bash results: ok')
expect(sheetOf(work)).toContain('LOST') // sB has no transcript
})
test('R15: the post-fix floor is NEW once, state keeps postFixAt', () => {
const recs = Array.from({ length: 20 }, (_, i) =>
REC(T0 + i * 1000, { session: `s${i}`, ids: ['BDR-001'] }))
const { p, work } = fixture({ 'l:/repo/x': recs })
recs.forEach((r, i) => put(p, r.session, jsonl(turn(i * 1000, 'f'))))
worksheet(work, p, NOW)
draft(work, Array.from({ length: 20 }, (_, i) =>
`${i + 1}\tBDR-001\trelevant`).join('\n'))
const first = apply(work, p, NOW)
expect(first).toContain('NEW: post-fix prompt sample reached (20/20 asks)')
expect(first).toContain('Post-fix prompt asks: 20/20 (since ' +
'2026-11-12T19:53:59.000Z)')
const st = JSON.parse(fs.readFileSync(path.join(p.out, 'state.json'),
'utf8'))
expect(st.postFixAt).toBe(NOW)
worksheet(work, p, NOW + 1)
draft(work, '')
expect(apply(work, p, NOW + 1)).not.toContain('post-fix prompt sample')
})
/** Applies `n` relevant asks over an old state file: both gate-1 gates
* recorded, no postFixAt. Returns the state written. */
function closedAfter(n: number, closed = false): { closed: boolean } {
const recs = Array.from({ length: n }, (_, i) =>
REC(T0 + i * 1000, { session: `s${i}`, ids: ['BDR-001'] }))
const { p, work } = fixture({ 'l:/repo/x': recs })
fs.mkdirSync(p.out)
fs.writeFileSync(path.join(p.out, 'state.json'), JSON.stringify(
{ lastRun: T0 - 1, promptAt: T0 - 2, toolAt: T0 - 2, closed }))
recs.forEach((r, i) => put(p, r.session, jsonl(turn(i * 1000, 'f'))))
worksheet(work, p, NOW)
draft(work, recs.map((_, i) => `${i + 1}\tBDR-001\trelevant`).join('\n'))
apply(work, p, NOW)
return JSON.parse(fs.readFileSync(path.join(p.out, 'state.json'), 'utf8'))
}
test('R16: closed waits for the post-fix floor, old state included', () => {
expect(closedAfter(19).closed).toBe(false)
expect(closedAfter(20).closed).toBe(true)
})
test('R17: a legacy closed:true without postFixAt is rewritten open', () => {
expect(closedAfter(1, true).closed).toBe(false)
})
+53 -27
View File
@@ -16,8 +16,10 @@ import {
tagOf, writeSafe, writeState, type Draft, type Item, type Row,
type State,
} from './labels.ts'
import { PROMPT_GATE_N } from './thresholds.ts'
import {
clockStart, gateVerdict, metricsOf, renderFeedback, type Verdict,
clockStart, gateVerdict, metricsOf, renderFeedback, type Metrics,
type Verdict,
} from './metrics.ts'
import {
joinPrompt, joinTool, records, sessionFiles, type Loaded,
@@ -158,23 +160,39 @@ function toolItem(
return { item: { ...b, cited: [], lost: false }, text }
}
/** A per-run transcript loader: each file read once (R8); a record
* reads its main thread or its own sub-agent file only; torn lines
* summed once per file. */
const fileOf = (projects: string, l: Logged): string =>
sessionFiles(projects, l.root, l.rec.session, l.rec.agent)[0] ?? ''
/** A per-run transcript loader with one slot: the work is sorted by file,
* so each file is parsed once and dropped when the next one loads (memory
* stays at one transcript); a record reads its main thread or its own
* sub-agent file only; torn lines are summed at each load. */
function loaderFor(projects: string): Loader {
const files = new Map<string, Loaded>()
const read = (f: string) => {
const got = files.get(f) ?? records(f)
files.set(f, got)
return got
}
let slot: { file: string; got: Loaded } | undefined
let torn = 0
const load = (l: Logged): Loaded => {
const r = l.rec
const fs_ = sessionFiles(projects, l.root, r.session, r.agent)
return { torn: 0, recs: fs_.flatMap(f => read(f).recs) }
const file = fileOf(projects, l)
if (file === '') return { torn: 0, recs: [] }
if (slot?.file !== file) {
slot = { file, got: records(file) }
torn += slot.got.torn
}
return slot.got
}
const torn = () => [...files.values()].reduce((t, x) => t + x.torn, 0)
return { load, torn }
return { load, torn: () => torn }
}
const byFile = (a: { file: string; n: number }, b: typeof a) =>
a.file === b.file ? a.n - b.n : a.file < b.file ? -1 : 1
/** Every record joined, file by file; the items come back in `n` order. */
function joinAll(pop: Logged[], projects: string) {
const loader = loaderFor(projects)
const work = pop.map((l, i) =>
({ l, n: i + 1, file: fileOf(projects, l) })).sort(byFile)
const joined = work.map(w => joinOne(w.l, w.n, loader.load))
.sort((a, b) => a.item.n - b.item.n)
return { joined, torn: loader.torn() }
}
/** One worksheet item and its redacted text. */
@@ -212,9 +230,8 @@ function collect(p: Paths, now: number) {
const logs = readLogs(storeFile(p.store))
const pop = population(logs, new Set(rows.map(keyOf)),
clockStart(rows, logAtsOf(logs)), now)
const loader = loaderFor(p.projects)
const joined = pop.map((l, i) => joinOne(l, i + 1, loader.load))
return { joined, head: sheetHead(pop.length, loader.torn(),
const { joined, torn } = joinAll(pop, p.projects)
return { joined, head: sheetHead(pop.length, torn,
evictions(logs, readState(stateOf(p)).lastRun)) }
}
@@ -238,21 +255,29 @@ const draftOf = (dir: string) => {
return fs.existsSync(f) ? fs.readFileSync(f, 'utf8') : ''
}
/** The state after this run, and a NEW line per gate first reached. */
function nextState(st: State, v: Verdict, now: number) {
/** The state after this run, and a NEW line per gate or floor first
* reached. */
function nextState(st: State, m: Metrics, v: Verdict, now: number) {
const news: string[] = []
let { promptAt, toolAt } = st
let { promptAt, toolAt, postFixAt } = st
if (promptAt === undefined && v.prompt === 'reached') {
promptAt = now
news.push('NEW: prompt gate reached')
}
const toolDone = v.tool === 'reached' || v.tool === 'insufficient'
if (toolAt === undefined && toolDone) {
if (toolAt === undefined &&
(v.tool === 'reached' || v.tool === 'insufficient')) {
toolAt = now
news.push(`NEW: tool gate ${v.tool}`)
}
const closed = promptAt !== undefined && toolAt !== undefined
return { state: { lastRun: now, promptAt, toolAt, closed }, news }
if (postFixAt === undefined && m.postAsks >= PROMPT_GATE_N) {
postFixAt = now
news.push('NEW: post-fix prompt sample reached ' +
`(${m.postAsks}/${PROMPT_GATE_N} asks)`)
}
const closed = promptAt !== undefined && toolAt !== undefined &&
postFixAt !== undefined
return { state: { lastRun: now, promptAt, toolAt, postFixAt, closed },
news }
}
const assertFresh = (items: Item[], file: string) => {
@@ -275,7 +300,7 @@ function commit(
): string {
assertFresh(items, labelsOf(p))
const { fresh, m, v } = projected(items, drafts, p, now)
const next = nextState(readState(stateOf(p)), v, now)
const next = nextState(readState(stateOf(p)), m, v, now)
const feedback = renderFeedback(m, v, now)
if (fresh.length > 0) appendRows(labelsOf(p), fresh)
writeState(stateOf(p), next.state)
@@ -285,6 +310,7 @@ function commit(
const isCount = (v: unknown) =>
typeof v === 'number' && Number.isInteger(v) && v >= 0
const isSafeCount = (v: unknown) => isCount(v) && Number.isSafeInteger(v)
const optCount = (v: unknown) => v === undefined || isCount(v)
const idsOk = (v: unknown): v is string[] => Array.isArray(v) &&
v.every(x => typeof x === 'string' && ENTRY_ID.test(x))
@@ -305,7 +331,7 @@ function itemError(raw: unknown): string {
['ids', idsOk(i.ids)],
['cited', idsOk(i.cited) && i.cited.every(c =>
(i.ids as string[]).includes(c))],
['numbers', isCount(i.at) && isCount(i.n) && optCount(i.cands) &&
['numbers', isSafeCount(i.at) && isCount(i.n) && optCount(i.cands) &&
optCount(i.ms)],
['lost', typeof i.lost === 'boolean'],
]
+3 -1
View File
@@ -5,11 +5,13 @@ export const DAY_MS = 86400000
export const GATE_DAYS = 35 // the 5-week cap from the gate clock start
export const PROMPT_GATE_N = 20 // labelled injected prompt asks
export const TOOL_GATE_N = 10 // labelled tool injections
// Merge d3d5e6c of 7e87d77 (idle trigger >= 8 words, cap 2 entries): prompt
// rows before it still reflect the old triggers (BDR-146 residual).
export const GATE1_FIX_AT = Date.parse('2026-11-12T19:53:59Z')
export const T = {
promptKeep: 0.64, // relevant / labelled entries (4c baseline 14/22)
triggerFloor: 0.5, // worst trigger kind under this gets tightened
thirdPick: 0.4, // 3rd pick precision under this caps picks at 2
lateShare: 0.25, // timeout + paused among asks with candidates
msP90: 2500, // rerank p90 under this lowers RERANK_MS
toolKeep: 0.5, // tool precision under this narrows the match
+2 -5
View File
@@ -126,14 +126,11 @@ const textsOf = (r: Rec): [string, string][] => blocksOf(r)
/** Main-thread assistant text after `ts` up to the next prompt. */
function replyAfter(recs: Rec[], ts: number): string {
const end = nextPromptAt(recs, ts)
const seen = new Set<string>()
return recs
const texts = recs
.filter(r => r.type === 'assistant' && r.isSidechain !== true)
.filter(r => tsOf(r) > ts && tsOf(r) < end)
.flatMap(textsOf)
.filter(([key]) => !seen.has(key) && !!seen.add(key))
.map(([, text]) => text)
.join('\n')
return [...new Map(texts).values()].join('\n') // first position kept; the value is derived from the key
}
export type PromptJoin = { prompt: string; injected: string; reply: string }
+2 -2
View File
@@ -1,8 +1,8 @@
#!/usr/bin/env bash
# lib/tests/memory-feedback-remind.test.sh — the phase 4d banner line:
# shown when /memory-feedback never ran, or ran 5+ days ago and the store
# changed since; silent once both gates are recorded, with recall off, or
# on any error.
# changed since; silent once all three gate marks are recorded, with recall
# off, or on any error.
set -u
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
LIB="$ROOT/lib/memory-feedback-remind.sh"
+36
View File
@@ -2658,6 +2658,42 @@ test('IR11: an injection leaves one log record; 205 keep 200', async (
expect(await loadLog(storeApi(w), ROOT)).toHaveLength(200)
})
/** The key set of a log record is pinned: a text field must fail. */
const KEYS = ['agent', 'at', 'cands', 'channel', 'ids', 'ms', 'outcome',
'session', 'trigger']
test('IR11b: an injected record holds exactly these keys', async (
$, on,
) => {
const { w } = await bootRecall($, on)
await start($)
await say($, GO)
const log = await loadLog(storeApi(w), ROOT)
expect(log.map(r => Object.keys(r).sort())).toEqual([KEYS])
})
test('IR11c: a none record holds exactly these keys', async ($, on) => {
const { w } = await bootRecall($, on)
w.model = { text: 'none' }
await start($)
await say($, GO)
const log = await loadLog(storeApi(w), ROOT)
expect(log.map(r => Object.keys(r).sort())).toEqual([KEYS])
})
test('IR11d: a no-candidate record holds these keys, no ms', async (
$, on,
) => {
const { w } = await bootRecall($, on)
w.proc = { stdout: rowsOut([['BLK-001', 'x'], ['LRN-002', 'y']])
.replace(/open/g, 'superseded') }
await start($)
await say($, GO)
const log = await loadLog(storeApi(w), ROOT)
expect(log.map(r => Object.keys(r).sort()))
.toEqual([KEYS.filter(k => k !== 'ms')])
})
test('IR12: a pending milestone and a trigger give [note, block]', async (
$, on,
) => {