From 1c53aa05b39d1af742eae791f05998f3ff9c1a72 Mon Sep 17 00:00:00 2001 From: bchanot Date: Sun, 8 Nov 2026 17:34:57 +0100 Subject: [PATCH] feat(memory-feedback): worksheet, apply and sample reader --- lib/memory-feedback/reader.test.ts | 155 +++++++++++++++ lib/memory-feedback/reader.ts | 302 +++++++++++++++++++++++++++++ 2 files changed, 457 insertions(+) create mode 100644 lib/memory-feedback/reader.test.ts create mode 100644 lib/memory-feedback/reader.ts diff --git a/lib/memory-feedback/reader.test.ts b/lib/memory-feedback/reader.test.ts new file mode 100644 index 0000000..67ea962 --- /dev/null +++ b/lib/memory-feedback/reader.test.ts @@ -0,0 +1,155 @@ +import { afterAll, expect, test } from 'bun:test' +import { spawnSync } from 'node:child_process' +import * as fs from 'node:fs' +import * as os from 'node:os' +import * as path from 'node:path' +import { slugOf } from './transcripts.ts' +import { + apply, checkDir, population, readLogs, sample, storeFile, worksheet, + type Paths, +} from './reader.ts' + +const dirs: string[] = [] +afterAll(() => dirs.forEach(d => fs.rmSync(d, { recursive: true }))) +const mk = (pfx: string) => { + const d = fs.mkdtempSync(path.join(os.tmpdir(), pfx)) + dirs.push(d) + return d +} +const T0 = Date.parse('2026-11-10T10:00:00.000Z') +const NOW = T0 + 2 * 3600000 // every fixture record is 2 h old +const iso = (ms: number) => new Date(T0 + ms).toISOString() +// built at run time: no secret-shaped literal in the repo (BLK-041) +const PLANTED = ['API', 'TOKEN'].join('_') + '=' + 'hunter' + '2value' + +/** A fixture home: store, one transcript, an out dir, a work dir. */ +function fixture(store: unknown): { p: Paths; work: string } { + const home = mk('mf-') + const p = { store: path.join(home, 'store'), + projects: path.join(home, 'projects'), out: path.join(home, 'mf') } + fs.mkdirSync(p.store) + fs.writeFileSync(path.join(p.store, 'memory_skills-dir-abc.json'), + typeof store === 'string' ? store : JSON.stringify(store)) + const dir = path.join(p.projects, slugOf('/repo/x')) + fs.mkdirSync(dir, { recursive: true }) + fs.writeFileSync(path.join(dir, 's1.jsonl'), [ + { type: 'user', timestamp: iso(-50), origin: { kind: 'human' }, + message: { content: `design the reader gate with ${PLANTED}` } }, + { type: 'attachment', timestamp: iso(400), attachment: { + type: 'hook_additional_context', hookName: 'prompt.submit', + content: ['memory recall:\nBDR-001 x'] } }, + { type: 'assistant', timestamp: iso(900), + message: { id: 'm', content: [{ type: 'text', text: 'per BDR-001' }] } }, + ].map(r => JSON.stringify(r)).join('\n') + '\n') + return { p, work: mk('mf-work-') } +} +const REC = (at: number, over = {}) => ({ at, session: 's1', agent: 'main', + channel: 'prompt', ids: ['BDR-001'], trigger: 'fresh', + outcome: 'injected', cands: 5, ms: 800, ...over }) +const STORE = { 'l:/repo/x': [ + REC(T0 - 9e6, { cands: undefined }), // before 4d.1: out of population + REC(T0), REC(T0 + 60000, { session: 'gone' }), // the second one lost +], 'p:/repo/x': { sessions: [] } } +const draft = (work: string, text: string) => + fs.writeFileSync(path.join(work, 'labels-draft.tsv'), text) + +test('R1: no store file, or a torn one, fails loudly', () => { + const { p } = fixture('{"l:/x": [tor') + expect(() => readLogs(storeFile(p.store))).toThrow('unreadable') + expect(() => storeFile(path.join(p.store, 'none'))).toThrow('no memory') +}) + +test('R2: population from the clock start, unlabelled, 30 min old', () => { + const { p } = fixture(STORE) + const logs = readLogs(storeFile(p.store)) + expect(population(logs, new Set(), T0, NOW).map(l => l.rec.at)) + .toEqual([T0, T0 + 60000]) + expect(population(logs, new Set([`s1|${T0}|prompt`]), T0, NOW)) + .toHaveLength(1) + expect(population(logs, new Set(), T0, T0 + 600000)).toHaveLength(0) + expect(population(logs, new Set(), undefined, NOW)).toEqual([]) +}) + +test('R3: worksheet: 0600 files, redacted text, a lost item, a skeleton', + () => { + const { p, work } = fixture(STORE) + expect(worksheet(work, p, NOW)).toBe(2) + const sheet = fs.readFileSync(path.join(work, 'worksheet.md'), 'utf8') + expect(sheet).toContain('design the reader gate') + expect(sheet).toContain('(/repo/x)') + expect(sheet).not.toContain('hunter2value') + expect(sheet).toContain('LOST') + for (const f of ['worksheet.md', 'items.json', 'labels-draft.tsv']) { + expect(fs.statSync(path.join(work, f)).mode & 0o777).toBe(0o600) + } + expect(fs.readFileSync(path.join(work, 'labels-draft.tsv'), 'utf8')) + .toContain('1\tBDR-001\t') + const items = JSON.parse(fs.readFileSync( + path.join(work, 'items.json'), 'utf8')) + expect(items.map((i: { cited: string[]; lost: boolean }) => + [i.cited, i.lost])).toEqual([[['BDR-001'], false], [[], true]]) + }) + +test('R4: a bad draft refuses the whole apply, nothing written', () => { + const { p, work } = fixture(STORE) + worksheet(work, p, NOW) + draft(work, '1\tBDR-001\tmaybe') + expect(() => apply(work, p, NOW)).toThrow('draft refused') + expect(fs.existsSync(p.out)).toBe(false) +}) + +test('R5: apply writes labels, state, FEEDBACK; a second apply refuses', + () => { + const { p, work } = fixture(STORE) + worksheet(work, p, NOW) + draft(work, '1\tBDR-001\trelevant\n') + const fb = apply(work, p, NOW) + expect(fb).toContain('Prompt gate: open (1/20)') + expect(fb).toContain('Lost records: 1') + expect(fb).not.toContain('design the reader gate') + const labels = fs.readFileSync(path.join(p.out, 'labels.tsv'), 'utf8') + expect(labels).not.toContain('design') + expect(labels).not.toContain('/repo/x') + expect(labels).toMatch(/\tx#[0-9a-f]{6}\t/) + expect(JSON.parse(fs.readFileSync(path.join(p.out, 'state.json'), + 'utf8')).lastRun).toBe(NOW) + expect(() => apply(work, p, NOW + 1)).toThrow('already applied') + expect(worksheet(work, p, NOW + 2)).toBe(0) + }) + +test('R6: a first run with nothing to label records the run only', () => { + const { p, work } = fixture({ 'l:/repo/x': [REC(T0, { cands: undefined })] }) + expect(worksheet(work, p, NOW)).toBe(0) + expect(fs.readFileSync(path.join(work, 'worksheet.md'), 'utf8')) + .toContain('No unlabelled record') + expect(apply(work, p, NOW)).toContain('Gate clock start: not started') + expect(fs.existsSync(path.join(p.out, 'state.json'))).toBe(true) +}) + +test('R7: a gate verdict is NEW once, then recorded', () => { + const { p, work } = fixture(STORE) + const late = T0 + 36 * 86400000 // past the 5-week cap + worksheet(work, p, late) + draft(work, '1\tBDR-001\trelevant\n') + expect(apply(work, p, late)).toContain('NEW: prompt gate reached') + worksheet(work, p, late + 1) + draft(work, '') + const again = apply(work, p, late + 1) + expect(again).toContain('Prompt gate: reached') + expect(again).not.toContain('NEW:') +}) + +test('R8: the CLI refuses a dir inside a git work tree', () => { + const repo = mk('mf-repo-') + spawnSync('git', ['-C', repo, 'init', '-q']) + expect(checkDir(repo)).toContain('git work tree') + expect(checkDir('relative/dir')).not.toBe('') + expect(checkDir(mk('mf-ok-'))).toBe('') +}) + +test('R9: sample picks entry lines of the draft', () => { + const { p, work } = fixture(STORE) + worksheet(work, p, NOW) + draft(work, '1\tBDR-001\trelevant\n1\t-\tchecked\n') + expect(sample(work, 5)).toEqual(['1\tBDR-001\trelevant']) +}) diff --git a/lib/memory-feedback/reader.ts b/lib/memory-feedback/reader.ts new file mode 100644 index 0000000..10bde18 --- /dev/null +++ b/lib/memory-feedback/reader.ts @@ -0,0 +1,302 @@ +#!/usr/bin/env bun +// lib/memory-feedback/reader.ts: phase 4d reader. `worksheet ` joins +// the unlabelled injection-log records to their transcripts; `apply ` +// validates the labeller's draft, appends labels.tsv and regenerates +// FEEDBACK.md; `sample ` picks entries for the spot check. Redacted +// text only ever lands in , the session scratchpad. +import { spawnSync } from 'node:child_process' +import * as fs from 'node:fs' +import * as os from 'node:os' +import * as path from 'node:path' +import { + LOG_PREFIX, MAX_LOG, isLogRecord, redact, type LogRecord, +} from '../../mods/memory/hooks/register.ts' +import { + appendRows, keyOf, parseDraft, readRows, readState, rowsOf, skeleton, + tagOf, writeSafe, writeState, type Item, type State, +} from './labels.ts' +import { + clockStart, gateVerdict, metricsOf, renderFeedback, type Verdict, +} from './metrics.ts' +import { + joinPrompt, joinTool, records, sessionFiles, type Loaded, +} from './transcripts.ts' + +export type Paths = { store: string; projects: string; out: string } +export type Logged = { root: string; rec: LogRecord } +type Base = Omit +type Joined = { item: Item; text: string } +type Load = (l: Logged) => Loaded +type Loader = { load: Load; torn: () => number } + +const AGE_FLOOR_MS = 1800000 // a younger record may still be mid-turn +const PROMPT_MAX = 600 +const REPLY_MAX = 800 +const DRAFT_HELP = [ + 'Fill the 3rd column of labels-draft.tsv: `relevant` or `irrelevant`.', + 'A whole proof item: ` - test`. A miss: ` missed', + '` with cause `no-overlap` or `overlap`; a searched ask with no', + 'miss: ` - checked`. Lines are TAB separated.', +].join('\n') + +export function defaultPaths(): Paths { + const c = path.join(os.homedir(), '.claude') + return { store: path.join(c, 'plugins', 'store'), + projects: path.join(c, 'projects'), + out: path.join(c, 'memory-feedback') } +} + +/** The newest memory store file; throws when there is none. */ +export function storeFile(dir: string): string { + const files = fs.existsSync(dir) ? fs.readdirSync(dir) + .filter(f => f.startsWith('memory_skills-dir-') && f.endsWith('.json')) + .map(f => path.join(dir, f)) : [] + if (files.length === 0) throw new Error(`no memory store file in ${dir}`) + const mtime = (f: string) => fs.statSync(f).mtimeMs + return files.sort((a, b) => mtime(b) - mtime(a))[0] as string +} + +/** Every valid log record of every root, oldest first. */ +export function readLogs(file: string): Logged[] { + let data: unknown + try { + data = JSON.parse(fs.readFileSync(file, 'utf8')) + } catch { + throw new Error(`store file unreadable (${file}): retry in a moment`) + } + if (typeof data !== 'object' || data === null) { + throw new Error(`store file is not an object: ${file}`) + } + const out: Logged[] = [] + for (const [k, v] of Object.entries(data)) { + if (!k.startsWith(LOG_PREFIX) || !Array.isArray(v)) continue + const root = k.slice(LOG_PREFIX.length) + for (const rec of v.filter(isLogRecord)) out.push({ root, rec }) + } + return out.sort((a, b) => a.rec.at - b.rec.at) +} + +/** Unlabelled records from the clock start, AGE_FLOOR_MS old or more. */ +export function population( + logs: Logged[], done: Set, start: number | undefined, + now: number, +): Logged[] { + if (start === undefined) return [] + return logs.filter(l => l.rec.at >= start && + l.rec.at <= now - AGE_FLOOR_MS && !done.has(keyOf(l.rec))) +} + +/** Roots at the log cap whose oldest record is newer than the last run. */ +function evictions(logs: Logged[], lastRun: number): string[] { + if (lastRun === 0) return [] + const roots = [...new Set(logs.map(l => l.root))] + return roots.filter(root => { + const mine = logs.filter(l => l.root === root) + return mine.length >= MAX_LOG && (mine[0]?.rec.at ?? 0) > lastRun + }).map(tagOf) +} + +const clip = (s: string, n: number) => { + const r = redact(s) // redact first: a secret straddling the cut escapes + return r.length <= n ? r : `${r.slice(0, n)}…` +} +const citedIn = (ids: string[], text: string) => + ids.filter(id => new RegExp(`\\b${id}\\b`).test(text)) + +/** An item header; the absolute repo path is for STEP 4 (scratchpad + * only: labels.tsv keeps the tag). */ +function header(b: Base, repo: string, lost: boolean): string { + const ids = b.ids.join(' ') || '-' + return `## Item ${b.n}: ${b.channel}/${b.trigger}/${b.outcome}` + + ` · cands ${b.cands ?? '-'} · ms ${b.ms ?? '-'}` + + ` · ${b.root} (${repo})` + + ` · ids ${ids}${lost ? ' · LOST (no transcript turn)' : ''}` +} + +const lostItem = (b: Base, repo: string): Joined => + ({ item: { ...b, cited: [], lost: true }, text: header(b, repo, true) }) + +function promptItem( + b: Base, repo: string, l: Loaded, rec: LogRecord, +): Joined { + const j = joinPrompt(l.recs, rec) + if (j === undefined) return lostItem(b, repo) + const cited = citedIn(b.ids, j.reply) + const text = [header(b, repo, false), + `Prompt: ${clip(j.prompt, PROMPT_MAX)}`, + `Injected:\n${redact(j.injected) || '-'}`, + `Reply: ${clip(j.reply, REPLY_MAX)}`, + `Cited: ${cited.join(', ') || 'none'}`].join('\n\n') + return { item: { ...b, cited, lost: false }, text } +} + +function toolItem( + b: Base, repo: string, l: Loaded, rec: LogRecord, +): Joined { + const j = joinTool(l.recs, rec) + if (j === undefined) return lostItem(b, repo) + const next = j.next.map(f => f ? 'failed' : 'ok').join(', ') || 'none' + const text = [header(b, repo, false), + `Command: ${clip(j.command, PROMPT_MAX)}`, + `Known: ${redact(j.known)}`, `Next Bash results: ${next}`] + .join('\n\n') + return { item: { ...b, cited: [], lost: false }, text } +} + +/** A per-run transcript loader: each file read once (R8); a record + * reads its main thread or its own sub-agent file only; torn lines + * summed once per file. */ +function loaderFor(projects: string): Loader { + const files = new Map() + const read = (f: string) => { + const got = files.get(f) ?? records(f) + files.set(f, got) + return got + } + const load = (l: Logged): Loaded => { + const r = l.rec + const fs_ = sessionFiles(projects, l.root, r.session, r.agent) + return { torn: 0, recs: fs_.flatMap(f => read(f).recs) } + } + const torn = () => [...files.values()].reduce((t, x) => t + x.torn, 0) + return { load, torn } +} + +/** One worksheet item and its redacted text. */ +function joinOne(l: Logged, n: number, load: Load): Joined { + const r = l.rec + const b: Base = { n, root: tagOf(l.root), session: r.session, at: r.at, + channel: r.channel, trigger: r.trigger ?? '', outcome: r.outcome ?? '', + cands: r.cands, ms: r.ms, ids: r.ids } + const loaded = load(l) + return r.channel === 'prompt' + ? promptItem(b, l.root, loaded, r) : toolItem(b, l.root, loaded, r) +} + +const labelsOf = (p: Paths) => path.join(p.out, 'labels.tsv') +const stateOf = (p: Paths) => path.join(p.out, 'state.json') + +/** The worksheet's first lines: counts, torn lines, possible evictions. */ +function sheetHead(n: number, torn: number, evicted: string[]): string { + const lines = [n === 0 + ? 'No unlabelled record since the gate clock start.' + : `${n} records to label.`] + if (torn > 0) lines.push(`Skipped ${torn} torn transcript lines.`) + if (evicted.length > 0) { + lines.push(`Possible eviction before labelling: ${evicted.join(', ')}`) + } + return lines.join('\n') +} + +/** Writes worksheet.md, items.json, labels-draft.tsv (0600); the count. */ +export function worksheet(dir: string, p: Paths, now: number): number { + const rows = readRows(labelsOf(p)) + const logs = readLogs(storeFile(p.store)) + const logAts = logs.filter(l => l.rec.cands !== undefined) + .map(l => l.rec.at) + const pop = population(logs, new Set(rows.map(keyOf)), + clockStart(rows, logAts), now) + const loader = loaderFor(p.projects) + const joined = pop.map((l, i) => joinOne(l, i + 1, loader.load)) + const head = sheetHead(pop.length, loader.torn(), + evictions(logs, readState(stateOf(p)).lastRun)) + writeSafe(path.join(dir, 'worksheet.md'), ['# Memory feedback worksheet', + head, DRAFT_HELP, ...joined.map(j => j.text)].join('\n\n') + '\n') + const items = joined.map(j => j.item) + writeSafe(path.join(dir, 'items.json'), JSON.stringify(items)) + writeSafe(path.join(dir, 'labels-draft.tsv'), skeleton(items)) + return pop.length +} + +const draftOf = (dir: string) => { + const f = path.join(dir, 'labels-draft.tsv') + return fs.existsSync(f) ? fs.readFileSync(f, 'utf8') : '' +} + +/** The state after this run, and a NEW line per gate first reached. */ +function nextState(st: State, v: Verdict, now: number) { + const news: string[] = [] + let { promptAt, toolAt } = st + if (promptAt === undefined && v.prompt === 'reached') { + promptAt = now + news.push('NEW: prompt gate reached') + } + const toolDone = v.tool === 'reached' || v.tool === 'insufficient' + if (toolAt === undefined && toolDone) { + toolAt = now + news.push(`NEW: tool gate ${v.tool}`) + } + const closed = promptAt !== undefined && toolAt !== undefined + return { state: { lastRun: now, promptAt, toolAt, closed }, news } +} + +/** Validates the draft; writes state, then labels, then FEEDBACK.md. */ +export function apply(dir: string, p: Paths, now: number): string { + const items: Item[] = JSON.parse( + fs.readFileSync(path.join(dir, 'items.json'), 'utf8')) + const drafts = parseDraft(draftOf(dir), items) + if (typeof drafts === 'string') throw new Error(`draft refused: ${drafts}`) + const old = readRows(labelsOf(p)) + const done = new Set(old.map(keyOf)) + if (items.some(i => done.has(keyOf(i)))) { + throw new Error('items already applied: run worksheet again') + } + const fresh = items.flatMap(i => rowsOf(i, drafts)) + const m = metricsOf([...old, ...fresh]) + const v = gateVerdict(m, now) + const { state, news } = nextState(readState(stateOf(p)), v, now) + const feedback = renderFeedback(m, v, now) + writeState(stateOf(p), state) + if (fresh.length > 0) appendRows(labelsOf(p), fresh) + writeSafe(path.join(p.out, 'FEEDBACK.md'), feedback) + return feedback + news.map(n => n + '\n').join('') +} + +/** Up to `n` entry lines of the draft, picked at random. */ +export function sample(dir: string, n: number): string[] { + const lines = draftOf(dir).split('\n') + .filter(l => /\t(relevant|irrelevant)$/.test(l)) + for (let i = lines.length - 1; i > 0; i--) { + const j = Math.floor(Math.random() * (i + 1)) + ;[lines[i], lines[j]] = [lines[j] as string, lines[i] as string] + } + return lines.slice(0, n) +} + +/** '' for a usable dir; else why not (text must never reach a repo). */ +export function checkDir(dir: string | undefined): string { + if (dir === undefined || !path.isAbsolute(dir)) return 'not absolute' + if (!fs.existsSync(dir)) return 'does not exist' + const git = spawnSync('git', ['-C', dir, 'rev-parse', + '--is-inside-work-tree'], { encoding: 'utf8' }) + return git.stdout.trim() === 'true' ? 'inside a git work tree' : '' +} + +function run(cmd: string, dir: string): void { + const p = defaultPaths() + if (cmd === 'worksheet') { + const n = worksheet(dir, p, Date.now()) + console.log(`${n} records to label: ${dir}/worksheet.md`) + } else if (cmd === 'apply') { + process.stdout.write(apply(dir, p, Date.now())) + } else console.log(sample(dir, 5).join('\n')) +} + +function main(argv: string[]): number { + const [cmd, dir] = argv + const why = checkDir(dir) + if (!['worksheet', 'apply', 'sample'].includes(cmd ?? '') || why !== '') { + console.error('usage: reader.ts worksheet|apply|sample ' + + `(absolute, outside any git work tree)${why ? `: ${why}` : ''}`) + return 2 + } + try { + run(cmd as string, dir as string) + return 0 + } catch (e) { + console.error(`memory-feedback: ${(e as Error).message}`) + return 1 + } +} + +if (import.meta.main) process.exit(main(process.argv.slice(2)))