diff --git a/lib/memory-feedback-remind.sh b/lib/memory-feedback-remind.sh index 20b9a69..63d3d69 100755 --- a/lib/memory-feedback-remind.sh +++ b/lib/memory-feedback-remind.sh @@ -2,9 +2,10 @@ # lib/memory-feedback-remind.sh — the phase 4d reminder line of the # session-start banner (BDR-057: transcripts live 7 days). One line when # /memory-feedback never ran, or ran REMIND_DAYS+ days ago and the memory -# store changed since. Silent once both gates are recorded ("closed":true), -# with "recall": false, or on any error; always rc 0. bash 3.2 safe, -# portable (find -mtime, no stat -c), no JSON parse. +# store changed since. Silent once all three gate marks (prompt, tool, +# post-fix sample) are recorded ("closed":true), with "recall": false, or +# on any error; always rc 0. bash 3.2 safe, portable (find -mtime, no +# stat -c), no JSON parse. REMIND_DAYS=5 state="$HOME/.claude/memory-feedback/state.json" diff --git a/lib/memory-feedback/labels.test.ts b/lib/memory-feedback/labels.test.ts index a483f7f..38dafd7 100644 --- a/lib/memory-feedback/labels.test.ts +++ b/lib/memory-feedback/labels.test.ts @@ -105,6 +105,15 @@ test('L8: state defaults, round trip, a corrupt file throws', () => { expect(() => readState(f)).toThrow('state.json') }) +test('L8b: closed is derived from the three marks, not the stored flag', () => { + const f = path.join(tmp(), 'state.json') + fs.writeFileSync(f, '{"lastRun":1,"promptAt":1,"toolAt":1,"closed":true}') + expect(readState(f).closed).toBe(false) + fs.writeFileSync(f, + '{"lastRun":1,"promptAt":1,"toolAt":1,"postFixAt":1,"closed":false}') + expect(readState(f).closed).toBe(true) +}) + test('L9: repo tags keep same-named repos apart', () => { expect(tagOf('/a/x')).toMatch(/^x#[0-9a-f]{6}$/) expect(tagOf('/a/x')).toBe(tagOf('/a/x')) @@ -116,3 +125,54 @@ test('L10: the skeleton draft lists each injected id once', () => { item(3, { lost: true })]).split('\n').filter(l => !l.startsWith('#')) expect(lines.filter(l => l !== '')).toEqual(['1\tBDR-001\t', '1\tLRN-002\t']) }) + +test('L11: readRows round trips every label kind', () => { + const file = path.join(tmp(), 'mf', 'labels.tsv') + const full = item(1) + const drafts = parseDraft([OK, '2\tBDR-009\tmissed\tno-overlap', + '1\t-\tchecked', '2\t-\ttest'].join('\n'), [full, + item(2, { ids: [] })]) + if (typeof drafts === 'string') throw new Error(drafts) + const rows = [...rowsOf(full, drafts), ...rowsOf(item(2, { ids: [] }), + drafts), ...rowsOf(item(3, { lost: true }), [])] + appendRows(file, rows) + expect(new Set(rows.map(r => r.label))).toEqual(new Set(['ask', 'lost', + 'relevant', 'irrelevant', 'missed', 'checked', 'test'])) + expect(readRows(file)).toEqual(rows) +}) + +test('L12: a line with a wrong id, label, channel or at is refused', () => { + const good: Row = { at: 5, root: 'r#000000', session: 's', + channel: 'prompt', trigger: 'fresh', outcome: 'injected', cands: 1, + ms: 2, pick: 1, id: 'BDR-001', label: 'relevant', cause: '' } + expect(fromLine(toLine(good))).toEqual(good) + const bad: Partial[] = [{ label: 'maybe' as never }, + { id: 'BDR-1' }, { id: '' }, { label: 'ask' }, // ask carries no id + { channel: 'x' as never }, { at: Number.NaN }, { at: -5 }, + { at: 1.5 }] + for (const b of bad) expect(fromLine(toLine({ ...good, ...b }))) + .toBeUndefined() + expect(fromLine(toLine({ ...good, id: '', label: 'ask' }))).toBeDefined() +}) + +test('L13: a damaged line names itself and what the .bak holds', () => { + const file = path.join(tmp(), 'labels.tsv') + appendRows(file, rowsOf(item(1), [])) + fs.appendFileSync(file, '1\tr\ts\tprompt\t\t\t\t\t\tNOPE\trelevant\t\n') + expect(() => readRows(file)).toThrow(/line 3 unreadable.*labels\.tsv\.bak/) +}) + +test('L14: state.json that is not an object is refused', () => { + const f = path.join(tmp(), 'state.json') + for (const raw of ['[1]', 'null', '7', '"x"']) { + fs.writeFileSync(f, raw) + expect(() => readState(f)).toThrow('state.json') + } + writeState(f, { lastRun: 3, postFixAt: 3, closed: false }) + expect(readState(f).postFixAt).toBe(3) +}) + +test('L15: an entry label with a cause says it takes none', () => { + const d = parseDraft(`${OK}\n1\tBDR-001\trelevant\toverlap`, ITEMS) + expect(d).toContain('takes no cause') +}) diff --git a/lib/memory-feedback/labels.ts b/lib/memory-feedback/labels.ts index a90a137..daaa146 100644 --- a/lib/memory-feedback/labels.ts +++ b/lib/memory-feedback/labels.ts @@ -12,6 +12,8 @@ const DRAFT_LABELS = ['relevant', 'irrelevant', 'test', 'missed', 'checked'] const ENTRY_LABELS = ['relevant', 'irrelevant'] const CAUSES = ['no-overlap', 'overlap'] const ID_RE = /^(BDR|LRN|BLK)-\d{3,}$/ +const ROW_LABELS = [...DRAFT_LABELS, 'ask', 'lost'] +const BARE_LABELS = ['ask', 'lost', 'test', 'checked'] // carry no entry id /** One worksheet item: a log record and what the join found. */ export type Item = { @@ -25,9 +27,11 @@ export type Row = { pick?: number; id: string; label: Label; cause: string } export type Draft = { item: number; id: string; label: Label; cause: string } -/** promptAt/toolAt: when each gate verdict was first reached. */ +/** promptAt/toolAt/postFixAt: when each gate verdict or floor was first + * reached. */ export type State = { lastRun: number; promptAt?: number; toolAt?: number; closed: boolean + postFixAt?: number // the post-fix prompt floor, first crossed (optional) } const COLS = ['at', 'root', 'session', 'channel', 'trigger', 'outcome', @@ -52,10 +56,20 @@ export function toLine(r: Row): string { .join('\t') } -/** A labels.tsv line as a row; the header or a short line: undefined. */ +/** Whether the fields hold a known label, its id shape, a channel and + * an integer `at`. */ +function sound(f: string[]): boolean { + const label = f[10] ?? '' + const id = f[9] ?? '' + const idOk = BARE_LABELS.includes(label) ? id === '' : ID_RE.test(id) + return ROW_LABELS.includes(label) && idOk && + (f[3] === 'prompt' || f[3] === 'tool') && /^\d+$/.test(f[0] ?? '') +} + +/** A labels.tsv line as a row; the header or a damaged line: undefined. */ export function fromLine(line: string): Row | undefined { const f = line.split('\t') - if (f.length !== COLS.length || f[0] === 'at') return undefined + if (f.length !== COLS.length || !sound(f)) return undefined return { at: Number(f[0]), root: f[1] ?? '', session: f[2] ?? '', channel: f[3] as Channel, trigger: f[4] ?? '', outcome: f[5] ?? '', @@ -87,11 +101,12 @@ export function readRows(file: string): Row[] { const r = fromLine(line) if (r !== undefined) return [r] throw new Error(`labels.tsv line ${i + 1} unreadable: fix it by hand ` + - 'or restore labels.tsv.bak') + '(labels.tsv.bak holds the state before the last append)') }) } -/** Appends rows; the previous file is copied to labels.tsv.bak first. */ +/** Appends rows; the previous file is copied to labels.tsv.bak first (one + * state: the one before the last append). */ export function appendRows(file: string, rows: Row[]): void { mkdir(file) if (fs.existsSync(file)) fs.copyFileSync(file, `${file}.bak`) @@ -104,9 +119,17 @@ export function readState(file: string): State { if (!fs.existsSync(file)) return { lastRun: 0, closed: false } try { const s = JSON.parse(fs.readFileSync(file, 'utf8')) + if (typeof s !== 'object' || s === null || Array.isArray(s)) { + throw new Error('state.json is not an object') + } const at = (v: unknown) => typeof v === 'number' ? v : undefined - return { lastRun: Number(s.lastRun) || 0, promptAt: at(s.promptAt), - toolAt: at(s.toolAt), closed: s.closed === true } + const [promptAt, toolAt, postFixAt] = + [at(s.promptAt), at(s.toolAt), at(s.postFixAt)] + // closed is derived: a stored flag without all three marks reads open + const closed = promptAt !== undefined && toolAt !== undefined && + postFixAt !== undefined + return { lastRun: Number(s.lastRun) || 0, promptAt, toolAt, postFixAt, + closed } } catch { throw new Error(`state.json unreadable (${file}): fix or delete it`) } @@ -135,7 +158,8 @@ function fitError(d: Draft, it: Item | undefined): string { ? '' : `item ${d.item}: only checked or test take id -` } if (ENTRY_LABELS.includes(d.label)) { - return plain && it.ids.includes(d.id) + if (!plain) return `item ${d.item}: ${d.label} takes no cause` + return it.ids.includes(d.id) ? '' : `item ${d.item}: ${d.id} was not injected there` } const ok = d.label === 'missed' && it.channel === 'prompt' && diff --git a/lib/memory-feedback/metrics.test.ts b/lib/memory-feedback/metrics.test.ts index c64aedd..2536884 100644 --- a/lib/memory-feedback/metrics.test.ts +++ b/lib/memory-feedback/metrics.test.ts @@ -1,12 +1,12 @@ import { expect, test } from 'bun:test' -import { DAY_MS, GATE_DAYS } from './thresholds.ts' +import { DAY_MS, GATE1_FIX_AT, GATE_DAYS } from './thresholds.ts' import { actionsOf, clockStart, gateVerdict, metricsOf, p90, precision, renderFeedback, type Metrics, } from './metrics.ts' import type { Label, Row } from './labels.ts' -let clock = 1000 +let clock = GATE1_FIX_AT + 1000 // every default row is post-fix /** A labels.tsv row; each call gets its own `at` unless given. */ function row(over: Partial & { label: Label }): Row { clock += 1 @@ -129,3 +129,127 @@ test('M10: FEEDBACK.md shows n per channel and trigger', () => { expect(text).toContain('| prompt precision | 33 % (n=3) |') expect(text).toContain('| tool precision | n/a (n=0) |') }) + +const shift = (rs: Row[], at: number) => rs.map(r => ({ ...r, at })) +const PRE = Date.parse('2026-11-12T15:00:00Z') + +test('M11: the fix instant splits prompt precision, not noisy', () => { + const old = shift(ask(['irrelevant', 'irrelevant', 'irrelevant']), PRE) + const mm = metricsOf([...old, ...ask(['relevant'])]) + expect(mm.precision).toBe(1) + expect(mm.n.prompt).toBe(1) + expect(mm.preFix).toBe(3) + expect(mm.promptN).toBe(2) // the gate count reads every row + const thrice = [1, 2, 3].flatMap(() => shift(ask(['irrelevant']), PRE)) + expect(metricsOf(thrice).noisy).toEqual(['r#000000 LRN-001']) + const edge = [...shift(ask(['irrelevant']), GATE1_FIX_AT - 1), + ...shift(ask(['relevant']), GATE1_FIX_AT)] + expect(metricsOf(edge).precision).toBe(1) + expect(metricsOf(edge).n.prompt).toBe(1) +}) + +test('M12: the miss share reads post-fix asks only', () => { + const miss = (at: number) => [row({ at, id: '', label: 'ask' }), + row({ at, id: 'BDR-009', label: 'missed', cause: 'overlap' })] + const mm = metricsOf([...miss(GATE1_FIX_AT - 1), ...miss(GATE1_FIX_AT)]) + expect(mm.sampled).toBe(1) + expect(mm.missShare).toBe(1) +}) + +/** Metrics whose every gate is reached, with `over` on top. */ +const reached = (over: Partial): Metrics => ({ + ...metricsOf([]), start: 0, promptN: 20, toolN: 10, postAsks: 20, ...over, +}) +const acts = (over: Partial) => { + const mm = reached(over) + return actionsOf(mm, gateVerdict(mm, 1)) +} +const sized = (i: number, k: number, f: number) => ({ + idle: { rate: i, n: 5 }, keyword: { rate: k, n: 5 }, + fresh: { rate: f, n: 5 }, +}) + +test('M13: each comparator at its threshold and one step past', () => { + const has = (over: Partial, part: string) => + acts(over).some(a => a.includes(part)) + expect(has({ late: 0.25 }, 'timeout + paused')).toBe(false) + expect(has({ late: 0.26 }, 'timeout + paused')).toBe(true) + expect(has({ msP90: 2500 }, 'rerank p90')).toBe(false) + expect(has({ msP90: 2499 }, 'rerank p90')).toBe(true) + expect(has({ tool: 0.5 }, 'tool precision')).toBe(false) + expect(has({ tool: 0.49 }, 'tool precision')).toBe(true) + expect(has({ missShare: 1 / 3 }, 'misses in 1 ask')).toBe(true) + expect(has({ missShare: 0.33 }, 'misses in 1 ask')).toBe(false) + const by = sized(0.4, 0.9, 0.9) + expect(has({ precision: 0.64, byTrigger: by }, 'prompt precision')) + .toBe(false) + expect(has({ precision: 0.63, byTrigger: by }, 'prompt precision')) + .toBe(true) + const low = (r: number) => sized(r, 0.9, 0.9) + expect(has({ precision: 0.5, byTrigger: low(0.5) }, 'prompt precision')) + .toBe(false) + expect(has({ precision: 0.5, byTrigger: low(0.49) }, 'prompt precision')) + .toBe(true) +}) + +test('M14: the worst trigger ties go idle, keyword, fresh', () => { + const pick = (by: ReturnType) => acts({ precision: 0.3, + byTrigger: by }).find(a => a.startsWith('prompt precision')) + expect(pick(sized(0.25, 0.25, 0.25))).toContain('the idle trigger') + expect(pick(sized(0.9, 0.25, 0.25))).toContain('the keyword trigger') + expect(pick(sized(0.25, 0.2, 0.9))).toContain('the keyword trigger') + expect(pick(sized(0.9, 0.9, 0.1))).toContain('the fresh trigger') +}) + +test('M15: promotion needs 3 hits in 2 sessions, noisy 3 never hit', () => { + const rel = (s: string) => ask(['relevant'], { session: s }) + expect(metricsOf([...rel('a'), ...rel('a'), ...rel('b')]).promote) + .toEqual(['r#000000 LRN-001']) + expect(metricsOf([...rel('a'), ...rel('b')]).promote).toEqual([]) + const irr = () => ask(['irrelevant']) + expect(metricsOf([...irr(), ...irr(), ...irr()]).noisy).toHaveLength(1) + expect(metricsOf([...irr(), ...irr()]).noisy).toEqual([]) +}) + +test('M16: the post-fix floor gates the prompt actions, 19 vs 20', () => { + const idle = (n: number) => Array.from({ length: n }, () => + ask(['irrelevant'], { trigger: 'idle' })).flat() + const run = (n: number) => { + const mm = { ...metricsOf(idle(n)), start: 0, promptN: 20 } + return actionsOf(mm, gateVerdict(mm, 1)) + } + expect(run(19).some(a => a.startsWith('prompt precision'))).toBe(false) + expect(run(20).some(a => a.startsWith('prompt precision'))).toBe(true) + const text = renderFeedback(metricsOf(idle(19)), gateVerdict( + metricsOf(idle(19)), 1), 0) + expect(text).toContain('Post-fix prompt asks: 19/20 (since ' + + '2026-11-12T19:53:59.000Z), pre-fix entries excluded: 0') +}) + +test('M17: below the floor the miss action waits, late and p90 fire', () => { + const asks = Array.from({ length: 19 }, () => + ask(['irrelevant'], { trigger: 'idle', ms: 2000 })).flat() + const miss = row({ id: 'BDR-009', label: 'missed', cause: 'overlap', + at: asks[0]!.at }) + const late = Array.from({ length: 8 }, () => + row({ id: '', label: 'ask', outcome: 'timeout' })) + const mm = { ...metricsOf([...asks, miss, ...late]), start: 0, promptN: 20 } + expect(mm.missShare).toBeGreaterThanOrEqual(1 / 3) + const out = actionsOf(mm, gateVerdict(mm, 1)) + expect(out.some(a => a.startsWith('misses in 1 ask'))).toBe(false) + expect(out.some(a => a.startsWith('timeout + paused'))).toBe(true) + expect(out.some(a => a.startsWith('rerank p90'))).toBe(true) +}) + +test('M18: an ask with 2 entries counts once toward the floor', () => { + const two = Array.from({ length: 19 }, () => + ask(['relevant', 'relevant'])).flat() + expect(metricsOf(two).postAsks).toBe(19) + expect(metricsOf([...two, ...ask(['relevant'])]).postAsks).toBe(20) +}) + +test('M19: a pre-fix row still counts toward promote', () => { + const old = (s: string) => shift(ask(['relevant'], { session: s }), PRE) + expect(metricsOf([...old('a'), ...old('a'), ...old('b')]).promote) + .toEqual(['r#000000 LRN-001']) +}) diff --git a/lib/memory-feedback/metrics.ts b/lib/memory-feedback/metrics.ts index ff025af..b5c3599 100644 --- a/lib/memory-feedback/metrics.ts +++ b/lib/memory-feedback/metrics.ts @@ -2,7 +2,7 @@ // FEEDBACK.md text, computed from labels.tsv rows only (transcripts expire // at 7 days, the rows stay). Spec §4. import { - DAY_MS, GATE_DAYS, PROMPT_GATE_N, T, TOOL_GATE_N, + DAY_MS, GATE1_FIX_AT, GATE_DAYS, PROMPT_GATE_N, T, TOOL_GATE_N, } from './thresholds.ts' import { keyOf, type Row } from './labels.ts' @@ -13,10 +13,12 @@ type Rate = number | undefined type Sized = { rate: Rate; n: number } export type Metrics = { start?: number; promptN: number; toolN: number; lost: number - precision: Rate; byTrigger: Record; thirdPick: Rate + precision: Rate; byTrigger: Record late: Rate; msP90: Rate; tool: Rate; missShare: Rate; sampled: number noisy: string[]; promote: string[] - n: { prompt: number; third: number; tool: number } // entries + n: { prompt: number; tool: number } // entries + postAsks: number // post-fix prompt asks labelled: the floor of the actions + preFix: number // pre-fix prompt entries left out of the prompt signals } const share = (k: number, n: number): Rate => n === 0 ? undefined : k / n @@ -66,7 +68,6 @@ function byTrigger(entries: Row[]): Record { /** Entry counts behind the three channel precisions. */ const countsOf = (entries: Row[]) => ({ prompt: entries.filter(r => r.channel === 'prompt').length, - third: entries.filter(r => r.channel === 'prompt' && r.pick === 3).length, tool: entries.filter(r => r.channel === 'tool').length, }) @@ -109,25 +110,29 @@ function promote(entries: Row[]): string[] { const itemsOf = (entries: Row[], ch: string) => new Set(entries.filter(r => r.channel === ch).map(keyOf)).size -/** Every 4d signal, from labels.tsv rows. */ +const postFix = (r: Row) => r.at >= GATE1_FIX_AT + +/** Every 4d signal, from labels.tsv rows. Prompt precision, triggers and + * the miss share read post-fix rows (BDR-146); the rest read all rows. */ export function metricsOf(all: Row[]): Metrics { const rows = liveRows(all) const entries = rows.filter(isEntry) - const prompt = entries.filter(r => r.channel === 'prompt') + const recent = entries.filter(r => r.channel === 'prompt' && postFix(r)) const asks = rows.filter(r => r.channel === 'prompt' && (r.label === 'ask' || r.label === 'lost')) - const miss = misses(rows) + const miss = misses(rows.filter(postFix)) return { start: clockStart(all, []), promptN: itemsOf(entries, 'prompt'), toolN: itemsOf(entries, 'tool'), lost: rows.filter(r => r.label === 'lost').length, - precision: precision(prompt), byTrigger: byTrigger(prompt), - thirdPick: precision(prompt.filter(r => r.pick === 3)), + precision: precision(recent), byTrigger: byTrigger(recent), late: lateShare(asks), msP90: p90(asks.flatMap(r => r.ms ?? [])), tool: precision(entries.filter(r => r.channel === 'tool')), missShare: miss.rate, sampled: miss.sampled, noisy: noisy(entries), promote: promote(entries), - n: countsOf(entries), + n: { ...countsOf(entries), prompt: recent.length }, + postAsks: itemsOf(recent, 'prompt'), + preFix: countsOf(entries).prompt - recent.length, } } @@ -158,13 +163,11 @@ function worstTrigger(by: Record): string | undefined { function promptActions(m: Metrics): string[] { const out: string[] = [] const worst = worstTrigger(m.byTrigger) - if (under(m.precision, T.promptKeep) && worst !== undefined) { + const floor = m.postAsks >= PROMPT_GATE_N + if (floor && under(m.precision, T.promptKeep) && worst !== undefined) { out.push(`prompt precision under ${pct(T.promptKeep)}: ` + `tighten or drop the ${worst} trigger`) } - if (under(m.thirdPick, T.thirdPick)) { - out.push(`3rd pick under ${pct(T.thirdPick)}: cap recall at 2 entries`) - } if ((m.late ?? 0) > T.lateShare) { out.push(`timeout + paused over ${pct(T.lateShare)}: ` + 'revisit RERANK_MS or skip the rerank on short lists') @@ -172,7 +175,7 @@ function promptActions(m: Metrics): string[] { if (under(m.msP90, T.msP90)) { out.push(`rerank p90 under ${T.msP90} ms: lower RERANK_MS`) } - if ((m.missShare ?? 0) >= T.missShare) { + if (floor && (m.missShare ?? 0) >= T.missShare) { out.push('misses in 1 ask in 3 or more: diagnose by cause') } return out @@ -197,6 +200,11 @@ function toolLine(m: Metrics, v: Verdict): string { : `Tool gate: ${v.tool} (${m.toolN}/${TOOL_GATE_N})` } +const postFixLine = (m: Metrics) => + `Post-fix prompt asks: ${m.postAsks}/${PROMPT_GATE_N} (since ` + + `${new Date(GATE1_FIX_AT).toISOString()}), pre-fix entries excluded: ` + + `${m.preFix}` + /** FEEDBACK.md: counts and shares; repo tags only in the last lists. */ export function renderFeedback(m: Metrics, v: Verdict, now: number) { const acts = actionsOf(m, v) @@ -204,12 +212,11 @@ export function renderFeedback(m: Metrics, v: Verdict, now: number) { `# Memory feedback (${day(now)})`, '', `Gate clock start: ${m.start === undefined ? 'not started' : day(m.start)}`, `Prompt gate: ${v.prompt} (${m.promptN}/${PROMPT_GATE_N})`, - toolLine(m, v), `Lost records: ${m.lost}`, '', + toolLine(m, v), `Lost records: ${m.lost}`, postFixLine(m), '', '| Signal | Value |', '|---|---|', `| prompt precision | ${pct(m.precision)} (n=${m.n.prompt}) |`, ...Object.entries(m.byTrigger).map(([k, x]) => `| precision, ${k} | ${pct(x.rate)} (n=${x.n}) |`), - `| 3rd pick precision | ${pct(m.thirdPick)} (n=${m.n.third}) |`, `| timeout + paused | ${pct(m.late)} |`, `| rerank ms p90 | ${m.msP90 ?? 'n/a'} |`, `| tool precision | ${pct(m.tool)} (n=${m.n.tool}) |`, diff --git a/lib/memory-feedback/reader.test.ts b/lib/memory-feedback/reader.test.ts index c10de45..a1e3414 100644 --- a/lib/memory-feedback/reader.test.ts +++ b/lib/memory-feedback/reader.test.ts @@ -3,6 +3,8 @@ import { spawnSync } from 'node:child_process' import * as fs from 'node:fs' import * as os from 'node:os' import * as path from 'node:path' +import { GATE1_FIX_AT } from './thresholds.ts' +import { tagOf } from './labels.ts' import { slugOf } from './transcripts.ts' import { apply, checkDir, population, readLogs, safePath, sample, storeFile, @@ -17,7 +19,7 @@ const mk = (pfx: string) => { dirs.push(d) return d } -const T0 = Date.parse('2026-11-10T10:00:00.000Z') +const T0 = GATE1_FIX_AT + 20 * 3600000 // post-fix: the floor counts it const NOW = T0 + 2 * 3600000 // every fixture record is 2 h old const iso = (ms: number) => new Date(T0 + ms).toISOString() // built at run time: no secret-shaped literal in the repo (BLK-041) @@ -156,7 +158,7 @@ test('R8: the CLI refuses a dir inside a git work tree', () => { test('R10: a tampered items.json is refused, nothing written', () => { for (const bad of [{ root: '/repo/x' }, { root: 'a\tb#abcdef' }, { session: '../s' }, { channel: 'x' }, { ids: ['../x'] }, - { cited: ['BDR-999'] }, { at: -1 }, { lost: 'no' }]) { + { cited: ['BDR-999'] }, { at: -1 }, { at: 1e21 }, { lost: 'no' }]) { const { p, work } = fixture(STORE) worksheet(work, p, NOW) const f = path.join(work, 'items.json') @@ -181,3 +183,135 @@ test('safePath keeps plain paths and withholds shell-active ones', () => { for (const bad of ['/tmp/$(id)', '/tmp/`id`', "/tmp/a'b", '/a/../b']) expect(safePath(bad)).toBe(w) }) + +/** The 3 transcript records of the prompt asked at `T0 + ms`. */ +const turn = (ms: number, tag: string) => [ + { type: 'user', timestamp: iso(ms - 50), origin: { kind: 'human' }, + message: { content: `prompt-${tag}` } }, + { type: 'attachment', timestamp: iso(ms + 400), attachment: { + type: 'hook_additional_context', hookName: 'prompt.submit', + content: ['memory recall:\nBDR-001 x'] } }, + { type: 'assistant', timestamp: iso(ms + 900), + message: { id: `m${ms}`, content: [{ type: 'text', text: 'ok' }] } }, +] +const jsonl = (recs: object[], torn = 0) => recs.map(r => JSON.stringify(r)) + .concat(Array(torn).fill('{"torn')).join('\n') + '\n' +const put = (p: Paths, session: string, text: string) => fs.writeFileSync( + path.join(p.projects, slugOf('/repo/x'), `${session}.jsonl`), text) +const sheetOf = (work: string) => + fs.readFileSync(path.join(work, 'worksheet.md'), 'utf8') + +const promptAt = (sheet: string, t: string) => + sheet.indexOf(`Prompt: prompt-${t}`) +const itemsOf = (work: string): { n: number }[] => JSON.parse( + fs.readFileSync(path.join(work, 'items.json'), 'utf8')) + +test('R11: A,B,A sessions: each file read once, order kept', () => { + const { p, work } = fixture({ 'l:/repo/x': [ + REC(T0, { session: 'sA' }), REC(T0 + 60000, { session: 'sB' }), + REC(T0 + 120000, { session: 'sA' })] }) + put(p, 'sA', jsonl([...turn(0, 'A1'), ...turn(120000, 'A2')], 1)) + put(p, 'sB', jsonl(turn(60000, 'B1'))) + expect(worksheet(work, p, NOW)).toBe(3) + const sheet = sheetOf(work) + expect(sheet).toContain('Skipped 1 torn transcript lines.') + const heads = sheet.split('\n').filter(l => l.startsWith('## Item')) + expect(heads.map(h => h.slice(0, 9))).toEqual(['## Item 1', + '## Item 2', '## Item 3']) + expect(heads.map(h => /s:(s.)/.exec(h)?.[1])).toEqual(['sA', 'sB', 'sA']) + expect(promptAt(sheet, 'A1')).toBeLessThan(promptAt(sheet, 'B1')) + expect(promptAt(sheet, 'B1')).toBeLessThan(promptAt(sheet, 'A2')) + expect(itemsOf(work).map(i => i.n)).toEqual([1, 2, 3]) +}) + +test('R12: torn lines are summed across files', () => { + const { p, work } = fixture({ 'l:/repo/x': [ + REC(T0, { session: 'sA' }), REC(T0 + 60000, { session: 'sB' })] }) + put(p, 'sA', jsonl(turn(0, 'A'), 1)) + put(p, 'sB', jsonl(turn(60000, 'B'), 2)) + worksheet(work, p, NOW) + expect(sheetOf(work)).toContain('Skipped 3 torn transcript lines.') +}) + +test('R13: a root at the log cap, newer than the last run, is noted', () => { + const full = Array.from({ length: 200 }, (_, i) => + REC(T0 + i * 1000, { ids: [], outcome: 'none' })) + const { p, work } = fixture({ 'l:/repo/x': full }) + fs.mkdirSync(p.out) + fs.writeFileSync(path.join(p.out, 'state.json'), + JSON.stringify({ lastRun: T0 - 1, closed: false })) + worksheet(work, p, NOW) + expect(sheetOf(work)).toContain( + `Possible eviction before labelling: ${tagOf('/repo/x')}`) +}) + +const use = (ms: number, id: string, command: string) => ({ + type: 'assistant', timestamp: iso(ms), message: { id: `a${id}`, + content: [{ type: 'tool_use', id, name: 'Bash', + input: { command } }] } }) +const result = (ms: number, id: string, err: boolean) => ({ + type: 'user', timestamp: iso(ms), message: { content: [ + { type: 'tool_result', tool_use_id: id, is_error: err }] } }) +const toolRec = (at: number, session: string) => REC(at, { session, + channel: 'tool', ids: ['BLK-003'], trigger: undefined, + outcome: undefined, cands: 1, ms: undefined }) + +test('R14: a tool item shows command, known line, next results', () => { + const { p, work } = fixture({ 'l:/repo/x': [ + toolRec(T0, 'sA'), toolRec(T0 + 60000, 'sB')] }) + put(p, 'sA', jsonl([use(-100, 'u1', 'make test'), result(-50, 'u1', true), + { type: 'attachment', timestamp: iso(400), attachment: { + type: 'hook_additional_context', hookName: 'tool.call', + toolUseID: 'u1-context', content: ['memory: known BLK-003 x'] } }, + use(500, 'u2', 'ls'), result(600, 'u2', false)])) + expect(worksheet(work, p, NOW)).toBe(2) + expect(sheetOf(work)).toContain('Command: make test') + expect(sheetOf(work)).toContain('Known: memory: known BLK-003 x') + expect(sheetOf(work)).toContain('Next Bash results: ok') + expect(sheetOf(work)).toContain('LOST') // sB has no transcript +}) + +test('R15: the post-fix floor is NEW once, state keeps postFixAt', () => { + const recs = Array.from({ length: 20 }, (_, i) => + REC(T0 + i * 1000, { session: `s${i}`, ids: ['BDR-001'] })) + const { p, work } = fixture({ 'l:/repo/x': recs }) + recs.forEach((r, i) => put(p, r.session, jsonl(turn(i * 1000, 'f')))) + worksheet(work, p, NOW) + draft(work, Array.from({ length: 20 }, (_, i) => + `${i + 1}\tBDR-001\trelevant`).join('\n')) + const first = apply(work, p, NOW) + expect(first).toContain('NEW: post-fix prompt sample reached (20/20 asks)') + expect(first).toContain('Post-fix prompt asks: 20/20 (since ' + + '2026-11-12T19:53:59.000Z)') + const st = JSON.parse(fs.readFileSync(path.join(p.out, 'state.json'), + 'utf8')) + expect(st.postFixAt).toBe(NOW) + worksheet(work, p, NOW + 1) + draft(work, '') + expect(apply(work, p, NOW + 1)).not.toContain('post-fix prompt sample') +}) + +/** Applies `n` relevant asks over an old state file: both gate-1 gates + * recorded, no postFixAt. Returns the state written. */ +function closedAfter(n: number, closed = false): { closed: boolean } { + const recs = Array.from({ length: n }, (_, i) => + REC(T0 + i * 1000, { session: `s${i}`, ids: ['BDR-001'] })) + const { p, work } = fixture({ 'l:/repo/x': recs }) + fs.mkdirSync(p.out) + fs.writeFileSync(path.join(p.out, 'state.json'), JSON.stringify( + { lastRun: T0 - 1, promptAt: T0 - 2, toolAt: T0 - 2, closed })) + recs.forEach((r, i) => put(p, r.session, jsonl(turn(i * 1000, 'f')))) + worksheet(work, p, NOW) + draft(work, recs.map((_, i) => `${i + 1}\tBDR-001\trelevant`).join('\n')) + apply(work, p, NOW) + return JSON.parse(fs.readFileSync(path.join(p.out, 'state.json'), 'utf8')) +} + +test('R16: closed waits for the post-fix floor, old state included', () => { + expect(closedAfter(19).closed).toBe(false) + expect(closedAfter(20).closed).toBe(true) +}) + +test('R17: a legacy closed:true without postFixAt is rewritten open', () => { + expect(closedAfter(1, true).closed).toBe(false) +}) diff --git a/lib/memory-feedback/reader.ts b/lib/memory-feedback/reader.ts index d26b81c..5aab92d 100644 --- a/lib/memory-feedback/reader.ts +++ b/lib/memory-feedback/reader.ts @@ -16,8 +16,10 @@ import { tagOf, writeSafe, writeState, type Draft, type Item, type Row, type State, } from './labels.ts' +import { PROMPT_GATE_N } from './thresholds.ts' import { - clockStart, gateVerdict, metricsOf, renderFeedback, type Verdict, + clockStart, gateVerdict, metricsOf, renderFeedback, type Metrics, + type Verdict, } from './metrics.ts' import { joinPrompt, joinTool, records, sessionFiles, type Loaded, @@ -158,23 +160,39 @@ function toolItem( return { item: { ...b, cited: [], lost: false }, text } } -/** A per-run transcript loader: each file read once (R8); a record - * reads its main thread or its own sub-agent file only; torn lines - * summed once per file. */ +const fileOf = (projects: string, l: Logged): string => + sessionFiles(projects, l.root, l.rec.session, l.rec.agent)[0] ?? '' + +/** A per-run transcript loader with one slot: the work is sorted by file, + * so each file is parsed once and dropped when the next one loads (memory + * stays at one transcript); a record reads its main thread or its own + * sub-agent file only; torn lines are summed at each load. */ function loaderFor(projects: string): Loader { - const files = new Map() - const read = (f: string) => { - const got = files.get(f) ?? records(f) - files.set(f, got) - return got - } + let slot: { file: string; got: Loaded } | undefined + let torn = 0 const load = (l: Logged): Loaded => { - const r = l.rec - const fs_ = sessionFiles(projects, l.root, r.session, r.agent) - return { torn: 0, recs: fs_.flatMap(f => read(f).recs) } + const file = fileOf(projects, l) + if (file === '') return { torn: 0, recs: [] } + if (slot?.file !== file) { + slot = { file, got: records(file) } + torn += slot.got.torn + } + return slot.got } - const torn = () => [...files.values()].reduce((t, x) => t + x.torn, 0) - return { load, torn } + return { load, torn: () => torn } +} + +const byFile = (a: { file: string; n: number }, b: typeof a) => + a.file === b.file ? a.n - b.n : a.file < b.file ? -1 : 1 + +/** Every record joined, file by file; the items come back in `n` order. */ +function joinAll(pop: Logged[], projects: string) { + const loader = loaderFor(projects) + const work = pop.map((l, i) => + ({ l, n: i + 1, file: fileOf(projects, l) })).sort(byFile) + const joined = work.map(w => joinOne(w.l, w.n, loader.load)) + .sort((a, b) => a.item.n - b.item.n) + return { joined, torn: loader.torn() } } /** One worksheet item and its redacted text. */ @@ -212,9 +230,8 @@ function collect(p: Paths, now: number) { const logs = readLogs(storeFile(p.store)) const pop = population(logs, new Set(rows.map(keyOf)), clockStart(rows, logAtsOf(logs)), now) - const loader = loaderFor(p.projects) - const joined = pop.map((l, i) => joinOne(l, i + 1, loader.load)) - return { joined, head: sheetHead(pop.length, loader.torn(), + const { joined, torn } = joinAll(pop, p.projects) + return { joined, head: sheetHead(pop.length, torn, evictions(logs, readState(stateOf(p)).lastRun)) } } @@ -238,21 +255,29 @@ const draftOf = (dir: string) => { return fs.existsSync(f) ? fs.readFileSync(f, 'utf8') : '' } -/** The state after this run, and a NEW line per gate first reached. */ -function nextState(st: State, v: Verdict, now: number) { +/** The state after this run, and a NEW line per gate or floor first + * reached. */ +function nextState(st: State, m: Metrics, v: Verdict, now: number) { const news: string[] = [] - let { promptAt, toolAt } = st + let { promptAt, toolAt, postFixAt } = st if (promptAt === undefined && v.prompt === 'reached') { promptAt = now news.push('NEW: prompt gate reached') } - const toolDone = v.tool === 'reached' || v.tool === 'insufficient' - if (toolAt === undefined && toolDone) { + if (toolAt === undefined && + (v.tool === 'reached' || v.tool === 'insufficient')) { toolAt = now news.push(`NEW: tool gate ${v.tool}`) } - const closed = promptAt !== undefined && toolAt !== undefined - return { state: { lastRun: now, promptAt, toolAt, closed }, news } + if (postFixAt === undefined && m.postAsks >= PROMPT_GATE_N) { + postFixAt = now + news.push('NEW: post-fix prompt sample reached ' + + `(${m.postAsks}/${PROMPT_GATE_N} asks)`) + } + const closed = promptAt !== undefined && toolAt !== undefined && + postFixAt !== undefined + return { state: { lastRun: now, promptAt, toolAt, postFixAt, closed }, + news } } const assertFresh = (items: Item[], file: string) => { @@ -275,7 +300,7 @@ function commit( ): string { assertFresh(items, labelsOf(p)) const { fresh, m, v } = projected(items, drafts, p, now) - const next = nextState(readState(stateOf(p)), v, now) + const next = nextState(readState(stateOf(p)), m, v, now) const feedback = renderFeedback(m, v, now) if (fresh.length > 0) appendRows(labelsOf(p), fresh) writeState(stateOf(p), next.state) @@ -285,6 +310,7 @@ function commit( const isCount = (v: unknown) => typeof v === 'number' && Number.isInteger(v) && v >= 0 +const isSafeCount = (v: unknown) => isCount(v) && Number.isSafeInteger(v) const optCount = (v: unknown) => v === undefined || isCount(v) const idsOk = (v: unknown): v is string[] => Array.isArray(v) && v.every(x => typeof x === 'string' && ENTRY_ID.test(x)) @@ -305,7 +331,7 @@ function itemError(raw: unknown): string { ['ids', idsOk(i.ids)], ['cited', idsOk(i.cited) && i.cited.every(c => (i.ids as string[]).includes(c))], - ['numbers', isCount(i.at) && isCount(i.n) && optCount(i.cands) && + ['numbers', isSafeCount(i.at) && isCount(i.n) && optCount(i.cands) && optCount(i.ms)], ['lost', typeof i.lost === 'boolean'], ] diff --git a/lib/memory-feedback/thresholds.ts b/lib/memory-feedback/thresholds.ts index c7456f7..04a7e0f 100644 --- a/lib/memory-feedback/thresholds.ts +++ b/lib/memory-feedback/thresholds.ts @@ -5,11 +5,13 @@ export const DAY_MS = 86400000 export const GATE_DAYS = 35 // the 5-week cap from the gate clock start export const PROMPT_GATE_N = 20 // labelled injected prompt asks export const TOOL_GATE_N = 10 // labelled tool injections +// Merge d3d5e6c of 7e87d77 (idle trigger >= 8 words, cap 2 entries): prompt +// rows before it still reflect the old triggers (BDR-146 residual). +export const GATE1_FIX_AT = Date.parse('2026-11-12T19:53:59Z') export const T = { promptKeep: 0.64, // relevant / labelled entries (4c baseline 14/22) triggerFloor: 0.5, // worst trigger kind under this gets tightened - thirdPick: 0.4, // 3rd pick precision under this caps picks at 2 lateShare: 0.25, // timeout + paused among asks with candidates msP90: 2500, // rerank p90 under this lowers RERANK_MS toolKeep: 0.5, // tool precision under this narrows the match diff --git a/lib/memory-feedback/transcripts.ts b/lib/memory-feedback/transcripts.ts index 78f51bb..15a4dca 100644 --- a/lib/memory-feedback/transcripts.ts +++ b/lib/memory-feedback/transcripts.ts @@ -126,14 +126,11 @@ const textsOf = (r: Rec): [string, string][] => blocksOf(r) /** Main-thread assistant text after `ts` up to the next prompt. */ function replyAfter(recs: Rec[], ts: number): string { const end = nextPromptAt(recs, ts) - const seen = new Set() - return recs + const texts = recs .filter(r => r.type === 'assistant' && r.isSidechain !== true) .filter(r => tsOf(r) > ts && tsOf(r) < end) .flatMap(textsOf) - .filter(([key]) => !seen.has(key) && !!seen.add(key)) - .map(([, text]) => text) - .join('\n') + return [...new Map(texts).values()].join('\n') // first position kept; the value is derived from the key } export type PromptJoin = { prompt: string; injected: string; reply: string } diff --git a/lib/tests/memory-feedback-remind.test.sh b/lib/tests/memory-feedback-remind.test.sh index 37dc40e..04bb302 100755 --- a/lib/tests/memory-feedback-remind.test.sh +++ b/lib/tests/memory-feedback-remind.test.sh @@ -1,8 +1,8 @@ #!/usr/bin/env bash # lib/tests/memory-feedback-remind.test.sh — the phase 4d banner line: # shown when /memory-feedback never ran, or ran 5+ days ago and the store -# changed since; silent once both gates are recorded, with recall off, or -# on any error. +# changed since; silent once all three gate marks are recorded, with recall +# off, or on any error. set -u ROOT="$(cd "$(dirname "$0")/../.." && pwd)" LIB="$ROOT/lib/memory-feedback-remind.sh" diff --git a/mods/memory/hooks/register.test.ts b/mods/memory/hooks/register.test.ts index 44dd41e..676635c 100644 --- a/mods/memory/hooks/register.test.ts +++ b/mods/memory/hooks/register.test.ts @@ -2658,6 +2658,42 @@ test('IR11: an injection leaves one log record; 205 keep 200', async ( expect(await loadLog(storeApi(w), ROOT)).toHaveLength(200) }) +/** The key set of a log record is pinned: a text field must fail. */ +const KEYS = ['agent', 'at', 'cands', 'channel', 'ids', 'ms', 'outcome', + 'session', 'trigger'] + +test('IR11b: an injected record holds exactly these keys', async ( + $, on, +) => { + const { w } = await bootRecall($, on) + await start($) + await say($, GO) + const log = await loadLog(storeApi(w), ROOT) + expect(log.map(r => Object.keys(r).sort())).toEqual([KEYS]) +}) + +test('IR11c: a none record holds exactly these keys', async ($, on) => { + const { w } = await bootRecall($, on) + w.model = { text: 'none' } + await start($) + await say($, GO) + const log = await loadLog(storeApi(w), ROOT) + expect(log.map(r => Object.keys(r).sort())).toEqual([KEYS]) +}) + +test('IR11d: a no-candidate record holds these keys, no ms', async ( + $, on, +) => { + const { w } = await bootRecall($, on) + w.proc = { stdout: rowsOut([['BLK-001', 'x'], ['LRN-002', 'y']]) + .replace(/open/g, 'superseded') } + await start($) + await say($, GO) + const log = await loadLog(storeApi(w), ROOT) + expect(log.map(r => Object.keys(r).sort())) + .toEqual([KEYS.filter(k => k !== 'ms')]) +}) + test('IR12: a pending milestone and a trigger give [note, block]', async ( $, on, ) => {