#!/usr/bin/env bun // lib/memory-feedback/reader.ts: phase 4d reader. `worksheet ` joins // the unlabelled injection-log records to their transcripts; `apply ` // validates the labeller's draft, appends labels.tsv and regenerates // FEEDBACK.md; `sample ` picks entries for the spot check. Redacted // text only ever lands in , the session scratchpad. import { spawnSync } from 'node:child_process' import * as fs from 'node:fs' import * as os from 'node:os' import * as path from 'node:path' import { LOG_PREFIX, MAX_LOG, OUTCOMES, TRIGGERS, isLogRecord, redact, type LogRecord, } from '../../mods/memory/hooks/register.ts' import { appendRows, keyOf, parseDraft, readRows, readState, rowsOf, skeleton, tagOf, writeSafe, writeState, type Draft, type Item, type Row, type State, } from './labels.ts' import { PROMPT_GATE_N } from './thresholds.ts' import { clockStart, gateVerdict, metricsOf, renderFeedback, type Metrics, type Verdict, } from './metrics.ts' import { joinPrompt, joinTool, records, sessionFiles, type Loaded, } from './transcripts.ts' export type Paths = { store: string; projects: string; out: string } export type Logged = { root: string; rec: LogRecord } type Base = Omit type Joined = { item: Item; text: string } type Load = (l: Logged) => Loaded type Loader = { load: Load; torn: () => number } const AGE_FLOOR_MS = 1800000 // a younger record may still be mid-turn const PROMPT_MAX = 600 const REPLY_MAX = 800 const DRAFT_HELP = [ 'Fill the 3rd column of labels-draft.tsv: `relevant` or `irrelevant`.', 'A whole proof item: ` - test`. A miss: ` missed', '` with cause `no-overlap` or `overlap`; a searched ask with no', 'miss: ` - checked`. Lines are TAB separated.', ].join('\n') export function defaultPaths(): Paths { const c = path.join(os.homedir(), '.claude') return { store: path.join(c, 'plugins', 'store'), projects: path.join(c, 'projects'), out: path.join(c, 'memory-feedback') } } /** The newest memory store file; throws when there is none. */ export function storeFile(dir: string): string { const files = fs.existsSync(dir) ? fs.readdirSync(dir) .filter(f => f.startsWith('memory_skills-dir-') && f.endsWith('.json')) .map(f => path.join(dir, f)) : [] if (files.length === 0) throw new Error(`no memory store file in ${dir}`) const mtime = (f: string) => fs.statSync(f).mtimeMs return files.sort((a, b) => mtime(b) - mtime(a))[0] as string } function readStore(file: string): object { let data: unknown try { data = JSON.parse(fs.readFileSync(file, 'utf8')) } catch { throw new Error(`store file unreadable (${file}): retry in a moment`) } if (typeof data !== 'object' || data === null) { throw new Error(`store file is not an object: ${file}`) } return data } /** Every valid log record of every root, oldest first. */ export function readLogs(file: string): Logged[] { return Object.entries(readStore(file)) .filter(([k, v]) => k.startsWith(LOG_PREFIX) && Array.isArray(v)) .flatMap(([k, v]) => (v as unknown[]).filter(isLogRecord) .map(rec => ({ root: k.slice(LOG_PREFIX.length), rec }))) .sort((a, b) => a.rec.at - b.rec.at) } /** Unlabelled records from the clock start, AGE_FLOOR_MS old or more. */ export function population( logs: Logged[], done: Set, start: number | undefined, now: number, ): Logged[] { if (start === undefined) return [] return logs.filter(l => l.rec.at >= start && l.rec.at <= now - AGE_FLOOR_MS && !done.has(keyOf(l.rec))) } /** Roots at the log cap whose oldest record is newer than the last run. */ function evictions(logs: Logged[], lastRun: number): string[] { if (lastRun === 0) return [] const roots = [...new Set(logs.map(l => l.root))] return roots.filter(root => { const mine = logs.filter(l => l.root === root) return mine.length >= MAX_LOG && (mine[0]?.rec.at ?? 0) > lastRun }).map(tagOf) } const clip = (s: string, n: number) => { const r = redact(s) // redact first: a secret straddling the cut escapes return r.length <= n ? r : `${r.slice(0, n)}…` } const ENTRY_ID = /^(BDR|LRN|BLK)-\d{3,}$/ const citedIn = (ids: string[], text: string) => ids.filter(id => ENTRY_ID.test(id) && new RegExp(`\\b${id}\\b`).test(text)) const WITHHELD = 'path withheld: unsafe characters' /** `p` if it is safe to paste inside single quotes in a shell command. */ export function safePath(p: string): string { const ok = /^\/[A-Za-z0-9._\/ -]*$/.test(p) && !p.split('/').includes('..') return ok ? p : WITHHELD } /** An item header; the absolute repo path is for STEP 4 (scratchpad * only: labels.tsv keeps the tag). */ function header(b: Base, repo: string, lost: boolean): string { const ids = b.ids.join(' ') || '-' return `## Item ${b.n}: ${b.channel}/${b.trigger}/${b.outcome}` + ` · cands ${b.cands ?? '-'} · ms ${b.ms ?? '-'}` + ` · s:${b.session.slice(0, 8)}` + ` · ${new Date(b.at).toISOString().slice(11, 16)} UTC` + ` · ${b.root} (${safePath(repo)})` + ` · ids ${ids}${lost ? ' · LOST (no transcript turn)' : ''}` } const lostItem = (b: Base, repo: string): Joined => ({ item: { ...b, cited: [], lost: true }, text: header(b, repo, true) }) function promptItem( b: Base, repo: string, l: Loaded, rec: LogRecord, ): Joined { const j = joinPrompt(l.recs, rec) if (j === undefined) return lostItem(b, repo) const cited = citedIn(b.ids, j.reply) const text = [header(b, repo, false), `Prompt: ${clip(j.prompt, PROMPT_MAX)}`, `Injected:\n${redact(j.injected) || '-'}`, `Reply: ${clip(j.reply, REPLY_MAX)}`, `Cited: ${cited.join(', ') || 'none'}`].join('\n\n') return { item: { ...b, cited, lost: false }, text } } function toolItem( b: Base, repo: string, l: Loaded, rec: LogRecord, ): Joined { const j = joinTool(l.recs, rec) if (j === undefined) return lostItem(b, repo) const next = j.next.map(f => f ? 'failed' : 'ok').join(', ') || 'none' const text = [header(b, repo, false), `Command: ${clip(j.command, PROMPT_MAX)}`, `Known: ${redact(j.known)}`, `Next Bash results: ${next}`] .join('\n\n') return { item: { ...b, cited: [], lost: false }, text } } const fileOf = (projects: string, l: Logged): string => sessionFiles(projects, l.root, l.rec.session, l.rec.agent)[0] ?? '' /** A per-run transcript loader with one slot: the work is sorted by file, * so each file is parsed once and dropped when the next one loads (memory * stays at one transcript); a record reads its main thread or its own * sub-agent file only; torn lines are summed at each load. */ function loaderFor(projects: string): Loader { let slot: { file: string; got: Loaded } | undefined let torn = 0 const load = (l: Logged): Loaded => { const file = fileOf(projects, l) if (file === '') return { torn: 0, recs: [] } if (slot?.file !== file) { slot = { file, got: records(file) } torn += slot.got.torn } return slot.got } return { load, torn: () => torn } } const byFile = (a: { file: string; n: number }, b: typeof a) => a.file === b.file ? a.n - b.n : a.file < b.file ? -1 : 1 /** Every record joined, file by file; the items come back in `n` order. */ function joinAll(pop: Logged[], projects: string) { const loader = loaderFor(projects) const work = pop.map((l, i) => ({ l, n: i + 1, file: fileOf(projects, l) })).sort(byFile) const joined = work.map(w => joinOne(w.l, w.n, loader.load)) .sort((a, b) => a.item.n - b.item.n) return { joined, torn: loader.torn() } } /** One worksheet item and its redacted text. */ function joinOne(l: Logged, n: number, load: Load): Joined { const r = l.rec const b: Base = { n, root: tagOf(l.root), session: r.session, at: r.at, channel: r.channel, trigger: r.trigger ?? '', outcome: r.outcome ?? '', cands: r.cands, ms: r.ms, ids: r.ids } const loaded = load(l) return r.channel === 'prompt' ? promptItem(b, l.root, loaded, r) : toolItem(b, l.root, loaded, r) } const labelsOf = (p: Paths) => path.join(p.out, 'labels.tsv') const stateOf = (p: Paths) => path.join(p.out, 'state.json') /** The worksheet's first lines: counts, torn lines, possible evictions. */ function sheetHead(n: number, torn: number, evicted: string[]): string { const lines = [n === 0 ? 'No unlabelled record since the gate clock start.' : `${n} records to label.`] if (torn > 0) lines.push(`Skipped ${torn} torn transcript lines.`) if (evicted.length > 0) { lines.push(`Possible eviction before labelling: ${evicted.join(', ')}`) } return lines.join('\n') } const logAtsOf = (logs: Logged[]) => logs.filter(l => l.rec.cands !== undefined).map(l => l.rec.at) /** Joins the unlabelled population to its transcripts. */ function collect(p: Paths, now: number) { const rows = readRows(labelsOf(p)) const logs = readLogs(storeFile(p.store)) const pop = population(logs, new Set(rows.map(keyOf)), clockStart(rows, logAtsOf(logs)), now) const { joined, torn } = joinAll(pop, p.projects) return { joined, head: sheetHead(pop.length, torn, evictions(logs, readState(stateOf(p)).lastRun)) } } function writeSheet(dir: string, joined: Joined[], head: string): void { writeSafe(path.join(dir, 'worksheet.md'), ['# Memory feedback worksheet', head, DRAFT_HELP, ...joined.map(j => j.text)].join('\n\n') + '\n') const items = joined.map(j => j.item) writeSafe(path.join(dir, 'items.json'), JSON.stringify(items)) writeSafe(path.join(dir, 'labels-draft.tsv'), skeleton(items)) } /** Writes worksheet.md, items.json, labels-draft.tsv (0600); the count. */ export function worksheet(dir: string, p: Paths, now: number): number { const { joined, head } = collect(p, now) writeSheet(dir, joined, head) return joined.length } const draftOf = (dir: string) => { const f = path.join(dir, 'labels-draft.tsv') return fs.existsSync(f) ? fs.readFileSync(f, 'utf8') : '' } /** The state after this run, and a NEW line per gate or floor first * reached. */ function nextState(st: State, m: Metrics, v: Verdict, now: number) { const news: string[] = [] let { promptAt, toolAt, postFixAt } = st if (promptAt === undefined && v.prompt === 'reached') { promptAt = now news.push('NEW: prompt gate reached') } if (toolAt === undefined && (v.tool === 'reached' || v.tool === 'insufficient')) { toolAt = now news.push(`NEW: tool gate ${v.tool}`) } if (postFixAt === undefined && m.postAsks >= PROMPT_GATE_N) { postFixAt = now news.push('NEW: post-fix prompt sample reached ' + `(${m.postAsks}/${PROMPT_GATE_N} asks)`) } const closed = promptAt !== undefined && toolAt !== undefined && postFixAt !== undefined return { state: { lastRun: now, promptAt, toolAt, postFixAt, closed }, news } } const assertFresh = (items: Item[], file: string) => { const done = new Set(readRows(file).map(keyOf)) if (items.some(i => done.has(keyOf(i)))) { throw new Error('items already applied: run worksheet again') } } /** The new rows, and the metrics and verdict once they are added. */ function projected(items: Item[], drafts: Draft[], p: Paths, now: number) { const fresh = items.flatMap(i => rowsOf(i, drafts)) const m = metricsOf([...readRows(labelsOf(p)), ...fresh]) return { fresh, m, v: gateVerdict(m, now) } } /** Appends labels, then writes state, then FEEDBACK.md. */ function commit( p: Paths, items: Item[], drafts: Draft[], now: number, ): string { assertFresh(items, labelsOf(p)) const { fresh, m, v } = projected(items, drafts, p, now) const next = nextState(readState(stateOf(p)), m, v, now) const feedback = renderFeedback(m, v, now) if (fresh.length > 0) appendRows(labelsOf(p), fresh) writeState(stateOf(p), next.state) writeSafe(path.join(p.out, 'FEEDBACK.md'), feedback) return feedback + next.news.map(n => n + '\n').join('') } const isCount = (v: unknown) => typeof v === 'number' && Number.isInteger(v) && v >= 0 const isSafeCount = (v: unknown) => isCount(v) && Number.isSafeInteger(v) const optCount = (v: unknown) => v === undefined || isCount(v) const idsOk = (v: unknown): v is string[] => Array.isArray(v) && v.every(x => typeof x === 'string' && ENTRY_ID.test(x)) const inEnum = (v: unknown, set: string[]) => v === '' || (typeof v === 'string' && set.includes(v)) /** Why a parsed items.json element is unusable; '' when it is sound. */ function itemError(raw: unknown): string { const i = (typeof raw === 'object' && raw !== null ? raw : {}) as Record const text = (v: unknown, re: RegExp) => typeof v === 'string' && re.test(v) const checks: [string, boolean][] = [ ['root', text(i.root, /^[^\t\n/]+#[0-9a-f]{6}$/)], ['session', text(i.session, /^[A-Za-z0-9_-]+$/)], ['channel', i.channel === 'prompt' || i.channel === 'tool'], ['trigger', inEnum(i.trigger, TRIGGERS)], ['outcome', inEnum(i.outcome, OUTCOMES)], ['ids', idsOk(i.ids)], ['cited', idsOk(i.cited) && i.cited.every(c => (i.ids as string[]).includes(c))], ['numbers', isSafeCount(i.at) && isCount(i.n) && optCount(i.cands) && optCount(i.ms)], ['lost', typeof i.lost === 'boolean'], ] return checks.find(([, ok]) => !ok)?.[0] ?? '' } /** The items.json content, every element validated (it is on disk in a * shared scratchpad: treat it as untrusted input). */ function readItems(dir: string): Item[] { const items: unknown = JSON.parse( fs.readFileSync(path.join(dir, 'items.json'), 'utf8')) if (!Array.isArray(items)) throw new Error('items.json invalid: not a list') for (const it of items) { const bad = itemError(it) if (bad !== '') throw new Error(`items.json invalid: bad ${bad}`) } return items as Item[] } /** Validates the draft; appends labels, writes state, then FEEDBACK.md * (a failed state write leaves the NEW line to print on the next run). */ export function apply(dir: string, p: Paths, now: number): string { const items = readItems(dir) const drafts = parseDraft(draftOf(dir), items) if (typeof drafts === 'string') throw new Error(`draft refused: ${drafts}`) return commit(p, items, drafts, now) } /** Up to `n` entry lines of the draft, picked at random. */ export function sample(dir: string, n: number): string[] { const lines = draftOf(dir).split('\n') .filter(l => /\t(relevant|irrelevant)$/.test(l)) for (let i = lines.length - 1; i > 0; i--) { const j = Math.floor(Math.random() * (i + 1)) ;[lines[i], lines[j]] = [lines[j] as string, lines[i] as string] } return lines.slice(0, n) } /** '' only for a git answer of "outside a work tree": exit 0 and * `false`, or exit 128 and `not a git repository`. Fails closed. */ function gitVerdict(dir: string): string { const git = spawnSync('git', ['-C', dir, 'rev-parse', '--is-inside-work-tree'], { encoding: 'utf8' }) if (git.error !== undefined) return 'git unavailable' if (git.status === 0 && String(git.stdout).trim() === 'false') return '' if (git.status === 0) return 'inside a git work tree' if (git.status === 128 && String(git.stderr).includes('not a git repository')) return '' return 'git check inconclusive' } /** '' for a usable dir; else why not (text must never reach a repo). */ export function checkDir(dir: string | undefined): string { if (dir === undefined || !path.isAbsolute(dir)) return 'not absolute' if (!fs.existsSync(dir)) return 'does not exist' const st = fs.statSync(dir) if (st.uid !== process.getuid?.()) return 'not owned by you' if ((st.mode & 0o022) !== 0) return 'group or world writable' return gitVerdict(dir) } function run(cmd: string, dir: string): void { const p = defaultPaths() if (cmd === 'worksheet') { const n = worksheet(dir, p, Date.now()) console.log(`${n} records to label: ${dir}/worksheet.md`) } else if (cmd === 'apply') { process.stdout.write(apply(dir, p, Date.now())) } else console.log(sample(dir, 5).join('\n')) } function main(argv: string[]): number { const [cmd, dir] = argv const why = checkDir(dir) if (!['worksheet', 'apply', 'sample'].includes(cmd ?? '') || why !== '') { console.error('usage: reader.ts worksheet|apply|sample ' + `(absolute, outside any git work tree)${why ? `: ${why}` : ''}`) return 2 } try { run(cmd as string, dir as string) return 0 } catch (e) { console.error(`memory-feedback: ${(e as Error).message}`) return 1 } } if (import.meta.main) process.exit(main(process.argv.slice(2)))