diff --git a/lib/memory-feedback/advisor.test.ts b/lib/memory-feedback/advisor.test.ts new file mode 100644 index 0000000..1b70643 --- /dev/null +++ b/lib/memory-feedback/advisor.test.ts @@ -0,0 +1,239 @@ +import { afterAll, expect, test } from 'bun:test' +import * as fs from 'node:fs' +import * as os from 'node:os' +import * as path from 'node:path' +import { + MAIN, UNKNOWN_AGENT, advisorSection, advisorUsage, callerOf, compact, + renderAdvisor, type Tally, +} from './advisor.ts' + +const dirs: string[] = [] +afterAll(() => dirs.forEach(d => fs.rmSync(d, { recursive: true }))) +const mk = () => { + const d = fs.mkdtempSync(path.join(os.tmpdir(), 'mf-adv-')) + dirs.push(d) + return d +} +const DAY = 86400000 +const FROM = Date.parse('2026-01-01T00:00:00Z') +const TO = FROM + 10 * DAY +const SLUG = '-home-secret-slug' +const iso = (ms: number) => new Date(FROM + ms).toISOString() +const adv = (input: number, model = 'claude-fable-5-2') => + ({ type: 'advisor_message', model, input_tokens: input }) +const row = (ms: number, id: string, its: object[], type = 'assistant') => + ({ type, timestamp: iso(ms), message: { id, usage: { iterations: its } } }) +const lines = (recs: object[]) => + recs.map(r => JSON.stringify(r)).join('\n') + '\n' + +/** A projects dir with one project dir; returns both paths. */ +function fixture() { + const projects = mk() + const proj = path.join(projects, SLUG) + fs.mkdirSync(proj) + return { projects, proj } +} +const put = (file: string, recs: object[]) => + fs.writeFileSync(file, lines(recs)) +const subDir = (proj: string, session = 's1') => { + const d = path.join(proj, session, 'subagents') + fs.mkdirSync(d, { recursive: true }) + return d +} +const only = (t: Tally) => t.uses.map(u => + [u.caller, u.model, u.calls, u.input]) + +test('1: one main row, duplicate rows deduped by message.id', () => { + const { projects, proj } = fixture() + put(path.join(proj, 's1.jsonl'), [ + row(10, 'm1', [adv(1500)]), row(10, 'm1', [adv(1500)])]) + expect(only(advisorUsage(projects, FROM, TO))) + .toEqual([[MAIN, 'claude-fable-5-2', 1, 1500]]) +}) + +test('2: rows outside (from, to] are not counted', () => { + const { projects, proj } = fixture() + put(path.join(proj, 's1.jsonl'), [row(0, 'a', [adv(1)]), + row(-5, 'b', [adv(1)]), row(11 * DAY, 'c', [adv(1)]), + row(1000, 'd', [adv(7)])]) + expect(only(advisorUsage(projects, FROM, TO))[0]?.[3]).toBe(7) +}) + +test('3: a user record and a torn line are ignored', () => { + const { projects, proj } = fixture() + fs.writeFileSync(path.join(proj, 's1.jsonl'), + lines([row(10, 'u', [adv(5)], 'user')]) + '{"torn advisor_message\n') + const t = advisorUsage(projects, FROM, TO) + expect(t.uses).toEqual([]) + expect(t.unparsed).toBe(0) +}) + +test('4: two advisor iterations in one message are two calls', () => { + const { projects, proj } = fixture() + put(path.join(proj, 's1.jsonl'), [row(10, 'm', [adv(1), adv(2)])]) + expect(only(advisorUsage(projects, FROM, TO))) + .toEqual([[MAIN, 'claude-fable-5-2', 2, 3]]) +}) + +test('5: an iteration without model is `unknown`', () => { + const { projects, proj } = fixture() + const bare = { type: 'advisor_message', input_tokens: 4 } + put(path.join(proj, 's1.jsonl'), [row(10, 'm', [bare])]) + expect(advisorUsage(projects, FROM, TO).uses[0]?.model).toBe('unknown') +}) + +/** One sub-agent transcript per entry; its meta file only when given. */ +function putAgents(sub: string, metas: Record) { + for (const [id, meta] of Object.entries(metas)) { + put(path.join(sub, `agent-${id}.jsonl`), [row(10, id, [adv(1)])]) + if (meta) fs.writeFileSync(path.join(sub, `agent-${id}.meta.json`), meta) + } +} + +test('6: sub-agent callers come from the meta agentType', () => { + const { projects, proj } = fixture() + putAgents(subDir(proj), { + a1: '{"agentType":"verifier"}', a2: undefined, + a3: '{"agentType":"../x"}' }) + const t = advisorUsage(projects, FROM, TO) + expect(t.uses.map(u => [u.caller, u.calls]).sort()) + .toEqual([['verifier', 1], [UNKNOWN_AGENT, 2]].sort()) +}) + +test('7: a transcript with mtime <= from is skipped', () => { + const { projects, proj } = fixture() + const f = path.join(proj, 's1.jsonl') + put(f, [row(10, 'm', [adv(1)])]) + fs.utimesSync(f, new Date(FROM - 1000), new Date(FROM - 1000)) + const t = advisorUsage(projects, FROM, TO) + expect(t.uses).toEqual([]) + expect(t.files).toBe(0) +}) + +test('8: missing projects dir, and a window with no advisor row', () => { + const none = path.join(mk(), 'absent') + expect(advisorUsage(none, FROM, TO)) + .toEqual({ uses: [], unparsed: 0, files: 0 }) + const s = advisorSection(none, FROM, TO) + expect(s).toContain('## Advisor usage (2026-01-01 → 2026-01-11)') + expect(s).toContain('No transcript modified since 2026-01-01') + expect(s).toContain('check the path.') + const { projects, proj } = fixture() + put(path.join(proj, 's1.jsonl'), [row(10, 'm', [])]) + expect(advisorSection(projects, FROM, TO)) + .toContain('No advisor call in the window.') +}) + +test('9: render: all on disk, compact, total row, sort order', () => { + expect([999, 1500, 999950, 2500000].map(compact)) + .toEqual(['999', '1.5k', '1.0M', '2.5M']) + const uses = [ + { caller: 'verifier', model: 'm', calls: 9, input: 1000 }, + { caller: MAIN, model: 'm', calls: 2, input: 1500 }, + { caller: 'planner', model: 'm', calls: 3, input: 500 }] + const t = { uses: [...uses].sort((a, b) => + Number(b.caller === MAIN) - Number(a.caller === MAIN) || + b.calls - a.calls), unparsed: 0, files: 3 } + const s = renderAdvisor(t, 0, TO) + expect(s.startsWith('## Advisor usage (all on disk → 2026-01-11)\n')) + .toBe(true) + expect(s).toContain('| total | | 14 | 3.0k |') + expect(s.indexOf('| main loop')).toBeLessThan(s.indexOf('| verifier')) + expect(s.indexOf('| verifier')).toBeLessThan(s.indexOf('| planner')) + expect(s.endsWith('\n')).toBe(true) +}) + +test('9b: advisorUsage sorts main loop first, then calls desc', () => { + const { projects, proj } = fixture() + put(path.join(proj, 's1.jsonl'), [row(10, 'm', [adv(1)])]) + const sub = subDir(proj) + put(path.join(sub, 'agent-a.jsonl'), + [row(10, 'x', [adv(1)]), row(20, 'y', [adv(1)])]) + fs.writeFileSync(path.join(sub, 'agent-a.meta.json'), + '{"agentType":"verifier"}') + expect(advisorUsage(projects, FROM, TO).uses.map(u => u.caller)) + .toEqual([MAIN, 'verifier']) +}) + +test('10: the section holds no project slug or path', () => { + const { projects, proj } = fixture() + put(path.join(proj, 's1.jsonl'), [row(10, 'm', [adv(1)])]) + const s = advisorSection(projects, FROM, TO) + expect(s).toContain('| main loop |') + expect(s).not.toContain('secret-slug') + expect(s).not.toContain(projects) +}) + +test('11: an advisor block with no iteration is not a call', () => { + const { projects, proj } = fixture() + const blockOnly = { type: 'assistant', timestamp: iso(10), message: { + id: 'm', content: [{ type: 'server_tool_use', name: 'advisor' }], + usage: { iterations: [] } } } + fs.writeFileSync(path.join(proj, 's1.jsonl'), lines([blockOnly]) + + '{"type":"assistant","note":"advisor_message"}\n') + const t = advisorUsage(projects, FROM, TO) + expect(t.uses).toEqual([]) + expect(t.unparsed).toBe(0) +}) + +const drift = { type: 'assistant', timestamp: iso(10), message: { + id: 'd', usage: { advisor_message: { input_tokens: 1 } } } } + +test('12: shape drift is counted and replaces the plain zero', () => { + const { projects, proj } = fixture() + put(path.join(proj, 's1.jsonl'), [drift]) + const t = advisorUsage(projects, FROM, TO) + expect(t.unparsed).toBe(1) + const s = renderAdvisor(t, FROM, TO) + expect(s).toContain('1 transcript lines mention advisor_message, none ' + + 'parsed: check the transcript shape.') + expect(s).not.toContain('No advisor call') +}) + +test('13: the retention note appears past 7 days only', () => { + const t: Tally = { uses: [], unparsed: 0, files: 1 } + expect(renderAdvisor(t, TO - 10 * DAY, TO)).toContain( + 'Transcripts kept 7 d: calls before 2026-01-04 may be missing.') + expect(renderAdvisor(t, TO - 3 * DAY, TO)).not.toContain('Transcripts kept') + expect(renderAdvisor(t, 0, TO)).not.toContain('Transcripts kept') +}) + +test('14: an unreadable sub-directory is skipped, the rest counts', () => { + const { projects, proj } = fixture() + put(path.join(proj, 's1.jsonl'), [row(10, 'm', [adv(1)])]) + const sub = subDir(proj, 's2') + put(path.join(sub, 'agent-a.jsonl'), [row(10, 'z', [adv(1)])]) + const locked = path.join(proj, 's2') + fs.chmodSync(locked, 0o000) + try { + const t = advisorUsage(projects, FROM, TO) + if (process.getuid?.() !== 0) expect(only(t)) + .toEqual([[MAIN, 'claude-fable-5-2', 1, 1]]) + } finally { + fs.chmodSync(locked, 0o700) + } +}) + +test('15: a hand-edited from never throws', () => { + const none = path.join(mk(), 'absent') + for (const bad of [Infinity, 1e17, NaN]) { + expect(advisorSection(none, bad, TO)).toContain('(all on disk → ') + } +}) + +test('16: drift under a table prints the table then the drift line', () => { + const { projects, proj } = fixture() + put(path.join(proj, 's1.jsonl'), [row(10, 'm', [adv(1)]), drift]) + const s = advisorSection(projects, FROM, TO) + expect(s.indexOf('| total |')).toBeGreaterThan(0) + expect(s.indexOf('none parsed')).toBeGreaterThan(s.indexOf('| total |')) +}) + +test('17: a symlinked meta file is not trusted', () => { + const { proj } = fixture() + const sub = subDir(proj) + const real = path.join(mk(), 'meta.json') + fs.writeFileSync(real, '{"agentType":"verifier"}') + fs.symlinkSync(real, path.join(sub, 'agent-a.meta.json')) + expect(callerOf(path.join(sub, 'agent-a.jsonl'))).toBe(UNKNOWN_AGENT) +}) diff --git a/lib/memory-feedback/advisor.ts b/lib/memory-feedback/advisor.ts new file mode 100644 index 0000000..eb731a4 --- /dev/null +++ b/lib/memory-feedback/advisor.ts @@ -0,0 +1,210 @@ +// lib/memory-feedback/advisor.ts: advisor usage read from Claude Code +// transcripts (Run C, LRN-295): served calls and input tokens per caller +// and model, deduped by message.id, for the FEEDBACK.md section. +import * as fs from 'node:fs' +import * as path from 'node:path' + +export type Use = { + caller: string; model: string; calls: number; input: number +} +export type Tally = { uses: Use[]; unparsed: number; files: number } +export type Source = { file: string; caller: string } +type Obj = Record +type Acc = { seen: Set; uses: Map; unparsed: number } + +export const MAIN = 'main loop' +export const UNKNOWN_AGENT = 'sub-agent (unknown)' +export const RETENTION_DAYS = 7 + +const DAY_MS = 86400000 +const META_MAX = 65536 +const MARK = 'advisor_message' +const AGENT_RE = /^[A-Za-z0-9_.:-]{1,64}$/ +const MODEL_RE = /^[A-Za-z0-9._:-]{1,80}$/ +const UNAVAILABLE = 'Unavailable: transcripts unreadable.' + +const day = (ms: number) => new Date(ms).toISOString().slice(0, 10) +const since = (from: number) => from === 0 ? 'all on disk' : day(from) +const heading = (from: number, to: number) => + `## Advisor usage (${since(from)} → ${day(to)})` +const obj = (v: unknown): Obj | undefined => + typeof v === 'object' && v !== null && !Array.isArray(v) + ? v as Obj : undefined + +function lstatOf(p: string): fs.Stats | undefined { + try { + return fs.lstatSync(p) + } catch { + return undefined + } +} + +/** Directory entries; [] when unreadable or gone, never throws. */ +function listDir(dir: string): string[] { + try { + return fs.readdirSync(dir) + } catch { + return [] + } +} + +const isRealDir = (p: string) => lstatOf(p)?.isDirectory() === true + +/** Regular `*.jsonl` files of `dir` newer than `from`. */ +function jsonlIn( + dir: string, prefix: string, from: number, callerOf: (f: string) => string, +): Source[] { + return listDir(dir) + .filter(n => n.startsWith(prefix) && n.endsWith('.jsonl')) + .map(n => path.join(dir, n)) + .flatMap(file => { + const st = lstatOf(file) + return st?.isFile() && st.mtimeMs > from + ? [{ file, caller: callerOf(file) }] : [] + }) +} + +/** Every transcript under `projects` modified after `from`, with its + * caller: main threads and `/subagents/agent-*.jsonl`. */ +export function transcriptsOf(projects: string, from: number): Source[] { + return listDir(projects).map(n => path.join(projects, n)) + .filter(isRealDir).flatMap(proj => { + const subs = listDir(proj).map(n => path.join(proj, n)) + .filter(isRealDir).map(s => path.join(s, 'subagents')) + .filter(isRealDir) + return [...jsonlIn(proj, '', from, () => MAIN), + ...subs.flatMap(s => jsonlIn(s, 'agent-', from, callerOf))] + }) +} + +/** The agent type of a sub-agent transcript, from its sibling meta file; + * UNKNOWN_AGENT for anything missing, odd or oversized. */ +export function callerOf(jsonl: string): string { + const meta = jsonl.replace(/\.jsonl$/, '') + '.meta.json' + const st = lstatOf(meta) + if (!st?.isFile() || st.size > META_MAX) return UNKNOWN_AGENT + try { + const type = obj(JSON.parse(fs.readFileSync(meta, 'utf8')))?.agentType + return typeof type === 'string' && AGENT_RE.test(type) + ? type : UNKNOWN_AGENT + } catch { + return UNKNOWN_AGENT + } +} + +/** The assistant messages of a file that mention advisor_message and + * fall in (from, to]; [] for an unreadable file. */ +function windowed(file: string, from: number, to: number): Obj[] { + let text: string + try { + text = fs.readFileSync(file, 'utf8') + } catch { + return [] + } + return text.split('\n').filter(l => l.includes(MARK)).flatMap(line => { + try { + const rec = obj(JSON.parse(line)) + const at = Date.parse(String(rec?.timestamp)) + const msg = obj(rec?.message) + return rec?.type === 'assistant' && msg !== undefined && + at > from && at <= to ? [msg] : [] + } catch { + return [] + } + }) +} + +function bump(uses: Map, caller: string, it: Obj): void { + const model = typeof it.model === 'string' && MODEL_RE.test(it.model) + ? it.model : 'unknown' + const key = `${caller}\t${model}` + const use = uses.get(key) ?? { caller, model, calls: 0, input: 0 } + use.calls += 1 + use.input += Number.isSafeInteger(it.input_tokens) && + (it.input_tokens as number) >= 0 ? it.input_tokens as number : 0 + uses.set(key, use) +} + +/** Counts one message once: its served advisor iterations, or a drift. */ +function tallyOne(msg: Obj, caller: string, acc: Acc): void { + const id = msg.id + if (typeof id !== 'string' || id === '' || acc.seen.has(id)) return + acc.seen.add(id) + const its = obj(msg.usage)?.iterations + const calls = (Array.isArray(its) ? its : []).map(obj) + .filter((i): i is Obj => i?.type === MARK) + calls.forEach(it => bump(acc.uses, caller, it)) + if (calls.length === 0 && + JSON.stringify(msg.usage ?? null).includes(MARK)) acc.unparsed += 1 +} + +const byUse = (a: Use, b: Use) => + Number(b.caller === MAIN) - Number(a.caller === MAIN) || + b.calls - a.calls || a.caller.localeCompare(b.caller) || + a.model.localeCompare(b.model) + +/** Served advisor calls and input tokens in (from, to], per caller and + * model, over every transcript under `projects`. */ +export function advisorUsage( + projects: string, from: number, to: number, +): Tally { + const sources = transcriptsOf(projects, from) + const acc: Acc = { seen: new Set(), uses: new Map(), unparsed: 0 } + for (const s of sources) { + windowed(s.file, from, to).forEach(m => tallyOne(m, s.caller, acc)) + } + return { uses: [...acc.uses.values()].sort(byUse), + unparsed: acc.unparsed, files: sources.length } +} + +/** 999 → `999`, 1500 → `1.5k`, 2500000 → `2.5M`. */ +export function compact(n: number): string { + if (n < 1000) return String(n) + return n < 999950 ? `${(n / 1e3).toFixed(1)}k` : `${(n / 1e6).toFixed(1)}M` +} + +function table(uses: Use[]): string { + const sum = (f: (u: Use) => number) => uses.reduce((a, u) => a + f(u), 0) + const rows = uses.map(u => + `| ${u.caller} | ${u.model} | ${u.calls} | ${compact(u.input)} |`) + return ['| caller | model | calls | input tok |', '|---|---|---|---|', + ...rows, `| total | | ${sum(u => u.calls)} | ` + + `${compact(sum(u => u.input))} |`].join('\n') +} + +function bodyOf(t: Tally, from: number): string { + const drift = `${t.unparsed} transcript lines mention ${MARK}, none ` + + 'parsed: check the transcript shape.' + if (t.uses.length > 0) { + return t.unparsed > 0 ? `${table(t.uses)}\n\n${drift}` : table(t.uses) + } + if (t.files === 0) { + return `No transcript modified since ${since(from)} under the ` + + 'projects dir: check the path.' + } + return t.unparsed > 0 ? drift : 'No advisor call in the window.' +} + +/** The `## Advisor usage` markdown section, ending with a newline. */ +export function renderAdvisor(t: Tally, from: number, to: number): string { + const blocks = [heading(from, to)] + if (from > 0 && to - from > RETENTION_DAYS * DAY_MS) { + blocks.push(`Transcripts kept ${RETENTION_DAYS} d: calls before ` + + `${day(to - RETENTION_DAYS * DAY_MS)} may be missing.`) + } + blocks.push(bodyOf(t, from)) + return blocks.join('\n\n') + '\n' +} + +/** The section for FEEDBACK.md; a fixed Unavailable text on any failure + * (an error message could carry a path). Never throws. */ +export function advisorSection( + projects: string, from: number, to: number, +): string { + const f = Number.isFinite(from) && from >= 0 && from <= to ? from : 0 + try { + return renderAdvisor(advisorUsage(projects, f, to), f, to) + } catch { + return `${heading(f, to)}\n\n${UNAVAILABLE}\n` + } +} diff --git a/lib/memory-feedback/reader.test.ts b/lib/memory-feedback/reader.test.ts index a1e3414..f081f89 100644 --- a/lib/memory-feedback/reader.test.ts +++ b/lib/memory-feedback/reader.test.ts @@ -109,6 +109,9 @@ test('R5: apply writes labels, state, FEEDBACK; a second apply refuses', const fb = apply(work, p, NOW) expect(fb).toContain('Prompt gate: open (1/20)') expect(fb).toContain('Lost records: 1') + expect(fb).toContain('## Advisor usage') + expect(fs.readFileSync(path.join(p.out, 'FEEDBACK.md'), 'utf8')) + .toContain('## Advisor usage') expect(fb).not.toContain('design the reader gate') const labels = fs.readFileSync(path.join(p.out, 'labels.tsv'), 'utf8') expect(labels).not.toContain('design') diff --git a/lib/memory-feedback/reader.ts b/lib/memory-feedback/reader.ts index 5aab92d..1a099d7 100644 --- a/lib/memory-feedback/reader.ts +++ b/lib/memory-feedback/reader.ts @@ -16,6 +16,7 @@ import { tagOf, writeSafe, writeState, type Draft, type Item, type Row, type State, } from './labels.ts' +import { advisorSection } from './advisor.ts' import { PROMPT_GATE_N } from './thresholds.ts' import { clockStart, gateVerdict, metricsOf, renderFeedback, type Metrics, @@ -301,7 +302,8 @@ function commit( assertFresh(items, labelsOf(p)) const { fresh, m, v } = projected(items, drafts, p, now) const next = nextState(readState(stateOf(p)), m, v, now) - const feedback = renderFeedback(m, v, now) + const feedback = renderFeedback(m, v, now) + '\n' + + advisorSection(p.projects, readState(stateOf(p)).lastRun, now) if (fresh.length > 0) appendRows(labelsOf(p), fresh) writeState(stateOf(p), next.state) writeSafe(path.join(p.out, 'FEEDBACK.md'), feedback) diff --git a/skills/memory-feedback/SKILL.md b/skills/memory-feedback/SKILL.md index 1b28a19..097e332 100644 --- a/skills/memory-feedback/SKILL.md +++ b/skills/memory-feedback/SKILL.md @@ -73,7 +73,9 @@ MFD="$(cd -P "$HOME/.claude/lib" && pwd)/memory-feedback" rc 1 `draft refused: ` → fix the draft, rerun. It prints FEEDBACK.md: show the gate lines (`Post-fix prompt asks: k/20` included: -actions wait for 20 post-fix asks) and the Actions list to the user. +actions wait for 20 post-fix asks), the Actions list and the +`## Advisor usage` table (advisor calls since the last run per caller and +served model) to the user. ## STEP 6 — A GATE REACHED FOR THE FIRST TIME