Author SHA1 Message Date
bchanot c6f2dad39a feat(memory-feedback): advisor usage section in FEEDBACK.md
apply appends served advisor calls since the last run, per caller (main
loop or sub-agent type) and model, with input tokens, deduped by message.id.
Retention, no-transcript and shape-drift lines replace false zeros.
2026-11-19 17:56:46 +01:00
5 changed files with 458 additions and 2 deletions

No files matched your search

+239
View File
@@ -0,0 +1,239 @@
import { afterAll, expect, test } from 'bun:test'
import * as fs from 'node:fs'
import * as os from 'node:os'
import * as path from 'node:path'
import {
MAIN, UNKNOWN_AGENT, advisorSection, advisorUsage, callerOf, compact,
renderAdvisor, type Tally,
} from './advisor.ts'
const dirs: string[] = []
afterAll(() => dirs.forEach(d => fs.rmSync(d, { recursive: true })))
const mk = () => {
const d = fs.mkdtempSync(path.join(os.tmpdir(), 'mf-adv-'))
dirs.push(d)
return d
}
const DAY = 86400000
const FROM = Date.parse('2026-01-01T00:00:00Z')
const TO = FROM + 10 * DAY
const SLUG = '-home-secret-slug'
const iso = (ms: number) => new Date(FROM + ms).toISOString()
const adv = (input: number, model = 'claude-fable-5-2') =>
({ type: 'advisor_message', model, input_tokens: input })
const row = (ms: number, id: string, its: object[], type = 'assistant') =>
({ type, timestamp: iso(ms), message: { id, usage: { iterations: its } } })
const lines = (recs: object[]) =>
recs.map(r => JSON.stringify(r)).join('\n') + '\n'
/** A projects dir with one project dir; returns both paths. */
function fixture() {
const projects = mk()
const proj = path.join(projects, SLUG)
fs.mkdirSync(proj)
return { projects, proj }
}
const put = (file: string, recs: object[]) =>
fs.writeFileSync(file, lines(recs))
const subDir = (proj: string, session = 's1') => {
const d = path.join(proj, session, 'subagents')
fs.mkdirSync(d, { recursive: true })
return d
}
const only = (t: Tally) => t.uses.map(u =>
[u.caller, u.model, u.calls, u.input])
test('1: one main row, duplicate rows deduped by message.id', () => {
const { projects, proj } = fixture()
put(path.join(proj, 's1.jsonl'), [
row(10, 'm1', [adv(1500)]), row(10, 'm1', [adv(1500)])])
expect(only(advisorUsage(projects, FROM, TO)))
.toEqual([[MAIN, 'claude-fable-5-2', 1, 1500]])
})
test('2: rows outside (from, to] are not counted', () => {
const { projects, proj } = fixture()
put(path.join(proj, 's1.jsonl'), [row(0, 'a', [adv(1)]),
row(-5, 'b', [adv(1)]), row(11 * DAY, 'c', [adv(1)]),
row(1000, 'd', [adv(7)])])
expect(only(advisorUsage(projects, FROM, TO))[0]?.[3]).toBe(7)
})
test('3: a user record and a torn line are ignored', () => {
const { projects, proj } = fixture()
fs.writeFileSync(path.join(proj, 's1.jsonl'),
lines([row(10, 'u', [adv(5)], 'user')]) + '{"torn advisor_message\n')
const t = advisorUsage(projects, FROM, TO)
expect(t.uses).toEqual([])
expect(t.unparsed).toBe(0)
})
test('4: two advisor iterations in one message are two calls', () => {
const { projects, proj } = fixture()
put(path.join(proj, 's1.jsonl'), [row(10, 'm', [adv(1), adv(2)])])
expect(only(advisorUsage(projects, FROM, TO)))
.toEqual([[MAIN, 'claude-fable-5-2', 2, 3]])
})
test('5: an iteration without model is `unknown`', () => {
const { projects, proj } = fixture()
const bare = { type: 'advisor_message', input_tokens: 4 }
put(path.join(proj, 's1.jsonl'), [row(10, 'm', [bare])])
expect(advisorUsage(projects, FROM, TO).uses[0]?.model).toBe('unknown')
})
/** One sub-agent transcript per entry; its meta file only when given. */
function putAgents(sub: string, metas: Record<string, string | undefined>) {
for (const [id, meta] of Object.entries(metas)) {
put(path.join(sub, `agent-${id}.jsonl`), [row(10, id, [adv(1)])])
if (meta) fs.writeFileSync(path.join(sub, `agent-${id}.meta.json`), meta)
}
}
test('6: sub-agent callers come from the meta agentType', () => {
const { projects, proj } = fixture()
putAgents(subDir(proj), {
a1: '{"agentType":"verifier"}', a2: undefined,
a3: '{"agentType":"../x"}' })
const t = advisorUsage(projects, FROM, TO)
expect(t.uses.map(u => [u.caller, u.calls]).sort())
.toEqual([['verifier', 1], [UNKNOWN_AGENT, 2]].sort())
})
test('7: a transcript with mtime <= from is skipped', () => {
const { projects, proj } = fixture()
const f = path.join(proj, 's1.jsonl')
put(f, [row(10, 'm', [adv(1)])])
fs.utimesSync(f, new Date(FROM - 1000), new Date(FROM - 1000))
const t = advisorUsage(projects, FROM, TO)
expect(t.uses).toEqual([])
expect(t.files).toBe(0)
})
test('8: missing projects dir, and a window with no advisor row', () => {
const none = path.join(mk(), 'absent')
expect(advisorUsage(none, FROM, TO))
.toEqual({ uses: [], unparsed: 0, files: 0 })
const s = advisorSection(none, FROM, TO)
expect(s).toContain('## Advisor usage (2026-01-01 → 2026-01-11)')
expect(s).toContain('No transcript modified since 2026-01-01')
expect(s).toContain('check the path.')
const { projects, proj } = fixture()
put(path.join(proj, 's1.jsonl'), [row(10, 'm', [])])
expect(advisorSection(projects, FROM, TO))
.toContain('No advisor call in the window.')
})
test('9: render: all on disk, compact, total row, sort order', () => {
expect([999, 1500, 999950, 2500000].map(compact))
.toEqual(['999', '1.5k', '1.0M', '2.5M'])
const uses = [
{ caller: 'verifier', model: 'm', calls: 9, input: 1000 },
{ caller: MAIN, model: 'm', calls: 2, input: 1500 },
{ caller: 'planner', model: 'm', calls: 3, input: 500 }]
const t = { uses: [...uses].sort((a, b) =>
Number(b.caller === MAIN) - Number(a.caller === MAIN) ||
b.calls - a.calls), unparsed: 0, files: 3 }
const s = renderAdvisor(t, 0, TO)
expect(s.startsWith('## Advisor usage (all on disk → 2026-01-11)\n'))
.toBe(true)
expect(s).toContain('| total | | 14 | 3.0k |')
expect(s.indexOf('| main loop')).toBeLessThan(s.indexOf('| verifier'))
expect(s.indexOf('| verifier')).toBeLessThan(s.indexOf('| planner'))
expect(s.endsWith('\n')).toBe(true)
})
test('9b: advisorUsage sorts main loop first, then calls desc', () => {
const { projects, proj } = fixture()
put(path.join(proj, 's1.jsonl'), [row(10, 'm', [adv(1)])])
const sub = subDir(proj)
put(path.join(sub, 'agent-a.jsonl'),
[row(10, 'x', [adv(1)]), row(20, 'y', [adv(1)])])
fs.writeFileSync(path.join(sub, 'agent-a.meta.json'),
'{"agentType":"verifier"}')
expect(advisorUsage(projects, FROM, TO).uses.map(u => u.caller))
.toEqual([MAIN, 'verifier'])
})
test('10: the section holds no project slug or path', () => {
const { projects, proj } = fixture()
put(path.join(proj, 's1.jsonl'), [row(10, 'm', [adv(1)])])
const s = advisorSection(projects, FROM, TO)
expect(s).toContain('| main loop |')
expect(s).not.toContain('secret-slug')
expect(s).not.toContain(projects)
})
test('11: an advisor block with no iteration is not a call', () => {
const { projects, proj } = fixture()
const blockOnly = { type: 'assistant', timestamp: iso(10), message: {
id: 'm', content: [{ type: 'server_tool_use', name: 'advisor' }],
usage: { iterations: [] } } }
fs.writeFileSync(path.join(proj, 's1.jsonl'), lines([blockOnly]) +
'{"type":"assistant","note":"advisor_message"}\n')
const t = advisorUsage(projects, FROM, TO)
expect(t.uses).toEqual([])
expect(t.unparsed).toBe(0)
})
const drift = { type: 'assistant', timestamp: iso(10), message: {
id: 'd', usage: { advisor_message: { input_tokens: 1 } } } }
test('12: shape drift is counted and replaces the plain zero', () => {
const { projects, proj } = fixture()
put(path.join(proj, 's1.jsonl'), [drift])
const t = advisorUsage(projects, FROM, TO)
expect(t.unparsed).toBe(1)
const s = renderAdvisor(t, FROM, TO)
expect(s).toContain('1 transcript lines mention advisor_message, none ' +
'parsed: check the transcript shape.')
expect(s).not.toContain('No advisor call')
})
test('13: the retention note appears past 7 days only', () => {
const t: Tally = { uses: [], unparsed: 0, files: 1 }
expect(renderAdvisor(t, TO - 10 * DAY, TO)).toContain(
'Transcripts kept 7 d: calls before 2026-01-04 may be missing.')
expect(renderAdvisor(t, TO - 3 * DAY, TO)).not.toContain('Transcripts kept')
expect(renderAdvisor(t, 0, TO)).not.toContain('Transcripts kept')
})
test('14: an unreadable sub-directory is skipped, the rest counts', () => {
const { projects, proj } = fixture()
put(path.join(proj, 's1.jsonl'), [row(10, 'm', [adv(1)])])
const sub = subDir(proj, 's2')
put(path.join(sub, 'agent-a.jsonl'), [row(10, 'z', [adv(1)])])
const locked = path.join(proj, 's2')
fs.chmodSync(locked, 0o000)
try {
const t = advisorUsage(projects, FROM, TO)
if (process.getuid?.() !== 0) expect(only(t))
.toEqual([[MAIN, 'claude-fable-5-2', 1, 1]])
} finally {
fs.chmodSync(locked, 0o700)
}
})
test('15: a hand-edited from never throws', () => {
const none = path.join(mk(), 'absent')
for (const bad of [Infinity, 1e17, NaN]) {
expect(advisorSection(none, bad, TO)).toContain('(all on disk → ')
}
})
test('16: drift under a table prints the table then the drift line', () => {
const { projects, proj } = fixture()
put(path.join(proj, 's1.jsonl'), [row(10, 'm', [adv(1)]), drift])
const s = advisorSection(projects, FROM, TO)
expect(s.indexOf('| total |')).toBeGreaterThan(0)
expect(s.indexOf('none parsed')).toBeGreaterThan(s.indexOf('| total |'))
})
test('17: a symlinked meta file is not trusted', () => {
const { proj } = fixture()
const sub = subDir(proj)
const real = path.join(mk(), 'meta.json')
fs.writeFileSync(real, '{"agentType":"verifier"}')
fs.symlinkSync(real, path.join(sub, 'agent-a.meta.json'))
expect(callerOf(path.join(sub, 'agent-a.jsonl'))).toBe(UNKNOWN_AGENT)
})
+210
View File
@@ -0,0 +1,210 @@
// lib/memory-feedback/advisor.ts: advisor usage read from Claude Code
// transcripts (Run C, LRN-295): served calls and input tokens per caller
// and model, deduped by message.id, for the FEEDBACK.md section.
import * as fs from 'node:fs'
import * as path from 'node:path'
export type Use = {
caller: string; model: string; calls: number; input: number
}
export type Tally = { uses: Use[]; unparsed: number; files: number }
export type Source = { file: string; caller: string }
type Obj = Record<string, unknown>
type Acc = { seen: Set<string>; uses: Map<string, Use>; unparsed: number }
export const MAIN = 'main loop'
export const UNKNOWN_AGENT = 'sub-agent (unknown)'
export const RETENTION_DAYS = 7
const DAY_MS = 86400000
const META_MAX = 65536
const MARK = 'advisor_message'
const AGENT_RE = /^[A-Za-z0-9_.:-]{1,64}$/
const MODEL_RE = /^[A-Za-z0-9._:-]{1,80}$/
const UNAVAILABLE = 'Unavailable: transcripts unreadable.'
const day = (ms: number) => new Date(ms).toISOString().slice(0, 10)
const since = (from: number) => from === 0 ? 'all on disk' : day(from)
const heading = (from: number, to: number) =>
`## Advisor usage (${since(from)} → ${day(to)})`
const obj = (v: unknown): Obj | undefined =>
typeof v === 'object' && v !== null && !Array.isArray(v)
? v as Obj : undefined
function lstatOf(p: string): fs.Stats | undefined {
try {
return fs.lstatSync(p)
} catch {
return undefined
}
}
/** Directory entries; [] when unreadable or gone, never throws. */
function listDir(dir: string): string[] {
try {
return fs.readdirSync(dir)
} catch {
return []
}
}
const isRealDir = (p: string) => lstatOf(p)?.isDirectory() === true
/** Regular `<prefix>*.jsonl` files of `dir` newer than `from`. */
function jsonlIn(
dir: string, prefix: string, from: number, callerOf: (f: string) => string,
): Source[] {
return listDir(dir)
.filter(n => n.startsWith(prefix) && n.endsWith('.jsonl'))
.map(n => path.join(dir, n))
.flatMap(file => {
const st = lstatOf(file)
return st?.isFile() && st.mtimeMs > from
? [{ file, caller: callerOf(file) }] : []
})
}
/** Every transcript under `projects` modified after `from`, with its
* caller: main threads and `<session>/subagents/agent-*.jsonl`. */
export function transcriptsOf(projects: string, from: number): Source[] {
return listDir(projects).map(n => path.join(projects, n))
.filter(isRealDir).flatMap(proj => {
const subs = listDir(proj).map(n => path.join(proj, n))
.filter(isRealDir).map(s => path.join(s, 'subagents'))
.filter(isRealDir)
return [...jsonlIn(proj, '', from, () => MAIN),
...subs.flatMap(s => jsonlIn(s, 'agent-', from, callerOf))]
})
}
/** The agent type of a sub-agent transcript, from its sibling meta file;
* UNKNOWN_AGENT for anything missing, odd or oversized. */
export function callerOf(jsonl: string): string {
const meta = jsonl.replace(/\.jsonl$/, '') + '.meta.json'
const st = lstatOf(meta)
if (!st?.isFile() || st.size > META_MAX) return UNKNOWN_AGENT
try {
const type = obj(JSON.parse(fs.readFileSync(meta, 'utf8')))?.agentType
return typeof type === 'string' && AGENT_RE.test(type)
? type : UNKNOWN_AGENT
} catch {
return UNKNOWN_AGENT
}
}
/** The assistant messages of a file that mention advisor_message and
* fall in (from, to]; [] for an unreadable file. */
function windowed(file: string, from: number, to: number): Obj[] {
let text: string
try {
text = fs.readFileSync(file, 'utf8')
} catch {
return []
}
return text.split('\n').filter(l => l.includes(MARK)).flatMap(line => {
try {
const rec = obj(JSON.parse(line))
const at = Date.parse(String(rec?.timestamp))
const msg = obj(rec?.message)
return rec?.type === 'assistant' && msg !== undefined &&
at > from && at <= to ? [msg] : []
} catch {
return []
}
})
}
function bump(uses: Map<string, Use>, caller: string, it: Obj): void {
const model = typeof it.model === 'string' && MODEL_RE.test(it.model)
? it.model : 'unknown'
const key = `${caller}\t${model}`
const use = uses.get(key) ?? { caller, model, calls: 0, input: 0 }
use.calls += 1
use.input += Number.isSafeInteger(it.input_tokens) &&
(it.input_tokens as number) >= 0 ? it.input_tokens as number : 0
uses.set(key, use)
}
/** Counts one message once: its served advisor iterations, or a drift. */
function tallyOne(msg: Obj, caller: string, acc: Acc): void {
const id = msg.id
if (typeof id !== 'string' || id === '' || acc.seen.has(id)) return
acc.seen.add(id)
const its = obj(msg.usage)?.iterations
const calls = (Array.isArray(its) ? its : []).map(obj)
.filter((i): i is Obj => i?.type === MARK)
calls.forEach(it => bump(acc.uses, caller, it))
if (calls.length === 0 &&
JSON.stringify(msg.usage ?? null).includes(MARK)) acc.unparsed += 1
}
const byUse = (a: Use, b: Use) =>
Number(b.caller === MAIN) - Number(a.caller === MAIN) ||
b.calls - a.calls || a.caller.localeCompare(b.caller) ||
a.model.localeCompare(b.model)
/** Served advisor calls and input tokens in (from, to], per caller and
* model, over every transcript under `projects`. */
export function advisorUsage(
projects: string, from: number, to: number,
): Tally {
const sources = transcriptsOf(projects, from)
const acc: Acc = { seen: new Set(), uses: new Map(), unparsed: 0 }
for (const s of sources) {
windowed(s.file, from, to).forEach(m => tallyOne(m, s.caller, acc))
}
return { uses: [...acc.uses.values()].sort(byUse),
unparsed: acc.unparsed, files: sources.length }
}
/** 999 → `999`, 1500 → `1.5k`, 2500000 → `2.5M`. */
export function compact(n: number): string {
if (n < 1000) return String(n)
return n < 999950 ? `${(n / 1e3).toFixed(1)}k` : `${(n / 1e6).toFixed(1)}M`
}
function table(uses: Use[]): string {
const sum = (f: (u: Use) => number) => uses.reduce((a, u) => a + f(u), 0)
const rows = uses.map(u =>
`| ${u.caller} | ${u.model} | ${u.calls} | ${compact(u.input)} |`)
return ['| caller | model | calls | input tok |', '|---|---|---|---|',
...rows, `| total | | ${sum(u => u.calls)} | ` +
`${compact(sum(u => u.input))} |`].join('\n')
}
function bodyOf(t: Tally, from: number): string {
const drift = `${t.unparsed} transcript lines mention ${MARK}, none ` +
'parsed: check the transcript shape.'
if (t.uses.length > 0) {
return t.unparsed > 0 ? `${table(t.uses)}\n\n${drift}` : table(t.uses)
}
if (t.files === 0) {
return `No transcript modified since ${since(from)} under the ` +
'projects dir: check the path.'
}
return t.unparsed > 0 ? drift : 'No advisor call in the window.'
}
/** The `## Advisor usage` markdown section, ending with a newline. */
export function renderAdvisor(t: Tally, from: number, to: number): string {
const blocks = [heading(from, to)]
if (from > 0 && to - from > RETENTION_DAYS * DAY_MS) {
blocks.push(`Transcripts kept ${RETENTION_DAYS} d: calls before ` +
`${day(to - RETENTION_DAYS * DAY_MS)} may be missing.`)
}
blocks.push(bodyOf(t, from))
return blocks.join('\n\n') + '\n'
}
/** The section for FEEDBACK.md; a fixed Unavailable text on any failure
* (an error message could carry a path). Never throws. */
export function advisorSection(
projects: string, from: number, to: number,
): string {
const f = Number.isFinite(from) && from >= 0 && from <= to ? from : 0
try {
return renderAdvisor(advisorUsage(projects, f, to), f, to)
} catch {
return `${heading(f, to)}\n\n${UNAVAILABLE}\n`
}
}
+3
View File
@@ -109,6 +109,9 @@ test('R5: apply writes labels, state, FEEDBACK; a second apply refuses',
const fb = apply(work, p, NOW)
expect(fb).toContain('Prompt gate: open (1/20)')
expect(fb).toContain('Lost records: 1')
expect(fb).toContain('## Advisor usage')
expect(fs.readFileSync(path.join(p.out, 'FEEDBACK.md'), 'utf8'))
.toContain('## Advisor usage')
expect(fb).not.toContain('design the reader gate')
const labels = fs.readFileSync(path.join(p.out, 'labels.tsv'), 'utf8')
expect(labels).not.toContain('design')
+3 -1
View File
@@ -16,6 +16,7 @@ import {
tagOf, writeSafe, writeState, type Draft, type Item, type Row,
type State,
} from './labels.ts'
import { advisorSection } from './advisor.ts'
import { PROMPT_GATE_N } from './thresholds.ts'
import {
clockStart, gateVerdict, metricsOf, renderFeedback, type Metrics,
@@ -301,7 +302,8 @@ function commit(
assertFresh(items, labelsOf(p))
const { fresh, m, v } = projected(items, drafts, p, now)
const next = nextState(readState(stateOf(p)), m, v, now)
const feedback = renderFeedback(m, v, now)
const feedback = renderFeedback(m, v, now) + '\n' +
advisorSection(p.projects, readState(stateOf(p)).lastRun, now)
if (fresh.length > 0) appendRows(labelsOf(p), fresh)
writeState(stateOf(p), next.state)
writeSafe(path.join(p.out, 'FEEDBACK.md'), feedback)
+3 -1
View File
@@ -73,7 +73,9 @@ MFD="$(cd -P "$HOME/.claude/lib" && pwd)/memory-feedback"
rc 1 `draft refused: <reason>` → fix the draft, rerun. It prints
FEEDBACK.md: show the gate lines (`Post-fix prompt asks: k/20` included:
actions wait for 20 post-fix asks) and the Actions list to the user.
actions wait for 20 post-fix asks), the Actions list and the
`## Advisor usage` table (advisor calls since the last run per caller and
served model) to the user.
## STEP 6 — A GATE REACHED FOR THE FIRST TIME