Precision, triggers and the miss share count only rows after the gate-1
fix merge (d3d5e6c, 2026-11-12T19:53:59Z); their actions wait for 20
post-fix asks, a one-off NEW line marks the sample and the reminder
keeps prompting until then. The 3rd-pick signal is gone. The reader
holds one parsed transcript at a time. labels.tsv rows and state.json
are validated on read. Exact key sets of log records pinned.
425 lines
16 KiB
TypeScript
425 lines
16 KiB
TypeScript
#!/usr/bin/env bun
|
|
// lib/memory-feedback/reader.ts: phase 4d reader. `worksheet <dir>` joins
|
|
// the unlabelled injection-log records to their transcripts; `apply <dir>`
|
|
// validates the labeller's draft, appends labels.tsv and regenerates
|
|
// FEEDBACK.md; `sample <dir>` picks entries for the spot check. Redacted
|
|
// text only ever lands in <dir>, the session scratchpad.
|
|
import { spawnSync } from 'node:child_process'
|
|
import * as fs from 'node:fs'
|
|
import * as os from 'node:os'
|
|
import * as path from 'node:path'
|
|
import {
|
|
LOG_PREFIX, MAX_LOG, OUTCOMES, TRIGGERS, isLogRecord, redact, type LogRecord,
|
|
} from '../../mods/memory/hooks/register.ts'
|
|
import {
|
|
appendRows, keyOf, parseDraft, readRows, readState, rowsOf, skeleton,
|
|
tagOf, writeSafe, writeState, type Draft, type Item, type Row,
|
|
type State,
|
|
} from './labels.ts'
|
|
import { PROMPT_GATE_N } from './thresholds.ts'
|
|
import {
|
|
clockStart, gateVerdict, metricsOf, renderFeedback, type Metrics,
|
|
type Verdict,
|
|
} from './metrics.ts'
|
|
import {
|
|
joinPrompt, joinTool, records, sessionFiles, type Loaded,
|
|
} from './transcripts.ts'
|
|
|
|
export type Paths = { store: string; projects: string; out: string }
|
|
export type Logged = { root: string; rec: LogRecord }
|
|
type Base = Omit<Item, 'cited' | 'lost'>
|
|
type Joined = { item: Item; text: string }
|
|
type Load = (l: Logged) => Loaded
|
|
type Loader = { load: Load; torn: () => number }
|
|
|
|
const AGE_FLOOR_MS = 1800000 // a younger record may still be mid-turn
|
|
const PROMPT_MAX = 600
|
|
const REPLY_MAX = 800
|
|
const DRAFT_HELP = [
|
|
'Fill the 3rd column of labels-draft.tsv: `relevant` or `irrelevant`.',
|
|
'A whole proof item: `<item> - test`. A miss: `<item> <id> missed',
|
|
'<cause>` with cause `no-overlap` or `overlap`; a searched ask with no',
|
|
'miss: `<item> - checked`. Lines are TAB separated.',
|
|
].join('\n')
|
|
|
|
export function defaultPaths(): Paths {
|
|
const c = path.join(os.homedir(), '.claude')
|
|
return { store: path.join(c, 'plugins', 'store'),
|
|
projects: path.join(c, 'projects'),
|
|
out: path.join(c, 'memory-feedback') }
|
|
}
|
|
|
|
/** The newest memory store file; throws when there is none. */
|
|
export function storeFile(dir: string): string {
|
|
const files = fs.existsSync(dir) ? fs.readdirSync(dir)
|
|
.filter(f => f.startsWith('memory_skills-dir-') && f.endsWith('.json'))
|
|
.map(f => path.join(dir, f)) : []
|
|
if (files.length === 0) throw new Error(`no memory store file in ${dir}`)
|
|
const mtime = (f: string) => fs.statSync(f).mtimeMs
|
|
return files.sort((a, b) => mtime(b) - mtime(a))[0] as string
|
|
}
|
|
|
|
function readStore(file: string): object {
|
|
let data: unknown
|
|
try {
|
|
data = JSON.parse(fs.readFileSync(file, 'utf8'))
|
|
} catch {
|
|
throw new Error(`store file unreadable (${file}): retry in a moment`)
|
|
}
|
|
if (typeof data !== 'object' || data === null) {
|
|
throw new Error(`store file is not an object: ${file}`)
|
|
}
|
|
return data
|
|
}
|
|
|
|
/** Every valid log record of every root, oldest first. */
|
|
export function readLogs(file: string): Logged[] {
|
|
return Object.entries(readStore(file))
|
|
.filter(([k, v]) => k.startsWith(LOG_PREFIX) && Array.isArray(v))
|
|
.flatMap(([k, v]) => (v as unknown[]).filter(isLogRecord)
|
|
.map(rec => ({ root: k.slice(LOG_PREFIX.length), rec })))
|
|
.sort((a, b) => a.rec.at - b.rec.at)
|
|
}
|
|
|
|
/** Unlabelled records from the clock start, AGE_FLOOR_MS old or more. */
|
|
export function population(
|
|
logs: Logged[], done: Set<string>, start: number | undefined,
|
|
now: number,
|
|
): Logged[] {
|
|
if (start === undefined) return []
|
|
return logs.filter(l => l.rec.at >= start &&
|
|
l.rec.at <= now - AGE_FLOOR_MS && !done.has(keyOf(l.rec)))
|
|
}
|
|
|
|
/** Roots at the log cap whose oldest record is newer than the last run. */
|
|
function evictions(logs: Logged[], lastRun: number): string[] {
|
|
if (lastRun === 0) return []
|
|
const roots = [...new Set(logs.map(l => l.root))]
|
|
return roots.filter(root => {
|
|
const mine = logs.filter(l => l.root === root)
|
|
return mine.length >= MAX_LOG && (mine[0]?.rec.at ?? 0) > lastRun
|
|
}).map(tagOf)
|
|
}
|
|
|
|
const clip = (s: string, n: number) => {
|
|
const r = redact(s) // redact first: a secret straddling the cut escapes
|
|
return r.length <= n ? r : `${r.slice(0, n)}…`
|
|
}
|
|
const ENTRY_ID = /^(BDR|LRN|BLK)-\d{3,}$/
|
|
const citedIn = (ids: string[], text: string) => ids.filter(id =>
|
|
ENTRY_ID.test(id) && new RegExp(`\\b${id}\\b`).test(text))
|
|
|
|
const WITHHELD = 'path withheld: unsafe characters'
|
|
|
|
/** `p` if it is safe to paste inside single quotes in a shell command. */
|
|
export function safePath(p: string): string {
|
|
const ok = /^\/[A-Za-z0-9._\/ -]*$/.test(p) &&
|
|
!p.split('/').includes('..')
|
|
return ok ? p : WITHHELD
|
|
}
|
|
|
|
/** An item header; the absolute repo path is for STEP 4 (scratchpad
|
|
* only: labels.tsv keeps the tag). */
|
|
function header(b: Base, repo: string, lost: boolean): string {
|
|
const ids = b.ids.join(' ') || '-'
|
|
return `## Item ${b.n}: ${b.channel}/${b.trigger}/${b.outcome}` +
|
|
` · cands ${b.cands ?? '-'} · ms ${b.ms ?? '-'}` +
|
|
` · s:${b.session.slice(0, 8)}` +
|
|
` · ${new Date(b.at).toISOString().slice(11, 16)} UTC` +
|
|
` · ${b.root} (${safePath(repo)})` +
|
|
` · ids ${ids}${lost ? ' · LOST (no transcript turn)' : ''}`
|
|
}
|
|
|
|
const lostItem = (b: Base, repo: string): Joined =>
|
|
({ item: { ...b, cited: [], lost: true }, text: header(b, repo, true) })
|
|
|
|
function promptItem(
|
|
b: Base, repo: string, l: Loaded, rec: LogRecord,
|
|
): Joined {
|
|
const j = joinPrompt(l.recs, rec)
|
|
if (j === undefined) return lostItem(b, repo)
|
|
const cited = citedIn(b.ids, j.reply)
|
|
const text = [header(b, repo, false),
|
|
`Prompt: ${clip(j.prompt, PROMPT_MAX)}`,
|
|
`Injected:\n${redact(j.injected) || '-'}`,
|
|
`Reply: ${clip(j.reply, REPLY_MAX)}`,
|
|
`Cited: ${cited.join(', ') || 'none'}`].join('\n\n')
|
|
return { item: { ...b, cited, lost: false }, text }
|
|
}
|
|
|
|
function toolItem(
|
|
b: Base, repo: string, l: Loaded, rec: LogRecord,
|
|
): Joined {
|
|
const j = joinTool(l.recs, rec)
|
|
if (j === undefined) return lostItem(b, repo)
|
|
const next = j.next.map(f => f ? 'failed' : 'ok').join(', ') || 'none'
|
|
const text = [header(b, repo, false),
|
|
`Command: ${clip(j.command, PROMPT_MAX)}`,
|
|
`Known: ${redact(j.known)}`, `Next Bash results: ${next}`]
|
|
.join('\n\n')
|
|
return { item: { ...b, cited: [], lost: false }, text }
|
|
}
|
|
|
|
const fileOf = (projects: string, l: Logged): string =>
|
|
sessionFiles(projects, l.root, l.rec.session, l.rec.agent)[0] ?? ''
|
|
|
|
/** A per-run transcript loader with one slot: the work is sorted by file,
|
|
* so each file is parsed once and dropped when the next one loads (memory
|
|
* stays at one transcript); a record reads its main thread or its own
|
|
* sub-agent file only; torn lines are summed at each load. */
|
|
function loaderFor(projects: string): Loader {
|
|
let slot: { file: string; got: Loaded } | undefined
|
|
let torn = 0
|
|
const load = (l: Logged): Loaded => {
|
|
const file = fileOf(projects, l)
|
|
if (file === '') return { torn: 0, recs: [] }
|
|
if (slot?.file !== file) {
|
|
slot = { file, got: records(file) }
|
|
torn += slot.got.torn
|
|
}
|
|
return slot.got
|
|
}
|
|
return { load, torn: () => torn }
|
|
}
|
|
|
|
const byFile = (a: { file: string; n: number }, b: typeof a) =>
|
|
a.file === b.file ? a.n - b.n : a.file < b.file ? -1 : 1
|
|
|
|
/** Every record joined, file by file; the items come back in `n` order. */
|
|
function joinAll(pop: Logged[], projects: string) {
|
|
const loader = loaderFor(projects)
|
|
const work = pop.map((l, i) =>
|
|
({ l, n: i + 1, file: fileOf(projects, l) })).sort(byFile)
|
|
const joined = work.map(w => joinOne(w.l, w.n, loader.load))
|
|
.sort((a, b) => a.item.n - b.item.n)
|
|
return { joined, torn: loader.torn() }
|
|
}
|
|
|
|
/** One worksheet item and its redacted text. */
|
|
function joinOne(l: Logged, n: number, load: Load): Joined {
|
|
const r = l.rec
|
|
const b: Base = { n, root: tagOf(l.root), session: r.session, at: r.at,
|
|
channel: r.channel, trigger: r.trigger ?? '', outcome: r.outcome ?? '',
|
|
cands: r.cands, ms: r.ms, ids: r.ids }
|
|
const loaded = load(l)
|
|
return r.channel === 'prompt'
|
|
? promptItem(b, l.root, loaded, r) : toolItem(b, l.root, loaded, r)
|
|
}
|
|
|
|
const labelsOf = (p: Paths) => path.join(p.out, 'labels.tsv')
|
|
const stateOf = (p: Paths) => path.join(p.out, 'state.json')
|
|
|
|
/** The worksheet's first lines: counts, torn lines, possible evictions. */
|
|
function sheetHead(n: number, torn: number, evicted: string[]): string {
|
|
const lines = [n === 0
|
|
? 'No unlabelled record since the gate clock start.'
|
|
: `${n} records to label.`]
|
|
if (torn > 0) lines.push(`Skipped ${torn} torn transcript lines.`)
|
|
if (evicted.length > 0) {
|
|
lines.push(`Possible eviction before labelling: ${evicted.join(', ')}`)
|
|
}
|
|
return lines.join('\n')
|
|
}
|
|
|
|
const logAtsOf = (logs: Logged[]) =>
|
|
logs.filter(l => l.rec.cands !== undefined).map(l => l.rec.at)
|
|
|
|
/** Joins the unlabelled population to its transcripts. */
|
|
function collect(p: Paths, now: number) {
|
|
const rows = readRows(labelsOf(p))
|
|
const logs = readLogs(storeFile(p.store))
|
|
const pop = population(logs, new Set(rows.map(keyOf)),
|
|
clockStart(rows, logAtsOf(logs)), now)
|
|
const { joined, torn } = joinAll(pop, p.projects)
|
|
return { joined, head: sheetHead(pop.length, torn,
|
|
evictions(logs, readState(stateOf(p)).lastRun)) }
|
|
}
|
|
|
|
function writeSheet(dir: string, joined: Joined[], head: string): void {
|
|
writeSafe(path.join(dir, 'worksheet.md'), ['# Memory feedback worksheet',
|
|
head, DRAFT_HELP, ...joined.map(j => j.text)].join('\n\n') + '\n')
|
|
const items = joined.map(j => j.item)
|
|
writeSafe(path.join(dir, 'items.json'), JSON.stringify(items))
|
|
writeSafe(path.join(dir, 'labels-draft.tsv'), skeleton(items))
|
|
}
|
|
|
|
/** Writes worksheet.md, items.json, labels-draft.tsv (0600); the count. */
|
|
export function worksheet(dir: string, p: Paths, now: number): number {
|
|
const { joined, head } = collect(p, now)
|
|
writeSheet(dir, joined, head)
|
|
return joined.length
|
|
}
|
|
|
|
const draftOf = (dir: string) => {
|
|
const f = path.join(dir, 'labels-draft.tsv')
|
|
return fs.existsSync(f) ? fs.readFileSync(f, 'utf8') : ''
|
|
}
|
|
|
|
/** The state after this run, and a NEW line per gate or floor first
|
|
* reached. */
|
|
function nextState(st: State, m: Metrics, v: Verdict, now: number) {
|
|
const news: string[] = []
|
|
let { promptAt, toolAt, postFixAt } = st
|
|
if (promptAt === undefined && v.prompt === 'reached') {
|
|
promptAt = now
|
|
news.push('NEW: prompt gate reached')
|
|
}
|
|
if (toolAt === undefined &&
|
|
(v.tool === 'reached' || v.tool === 'insufficient')) {
|
|
toolAt = now
|
|
news.push(`NEW: tool gate ${v.tool}`)
|
|
}
|
|
if (postFixAt === undefined && m.postAsks >= PROMPT_GATE_N) {
|
|
postFixAt = now
|
|
news.push('NEW: post-fix prompt sample reached ' +
|
|
`(${m.postAsks}/${PROMPT_GATE_N} asks)`)
|
|
}
|
|
const closed = promptAt !== undefined && toolAt !== undefined &&
|
|
postFixAt !== undefined
|
|
return { state: { lastRun: now, promptAt, toolAt, postFixAt, closed },
|
|
news }
|
|
}
|
|
|
|
const assertFresh = (items: Item[], file: string) => {
|
|
const done = new Set(readRows(file).map(keyOf))
|
|
if (items.some(i => done.has(keyOf(i)))) {
|
|
throw new Error('items already applied: run worksheet again')
|
|
}
|
|
}
|
|
|
|
/** The new rows, and the metrics and verdict once they are added. */
|
|
function projected(items: Item[], drafts: Draft[], p: Paths, now: number) {
|
|
const fresh = items.flatMap(i => rowsOf(i, drafts))
|
|
const m = metricsOf([...readRows(labelsOf(p)), ...fresh])
|
|
return { fresh, m, v: gateVerdict(m, now) }
|
|
}
|
|
|
|
/** Appends labels, then writes state, then FEEDBACK.md. */
|
|
function commit(
|
|
p: Paths, items: Item[], drafts: Draft[], now: number,
|
|
): string {
|
|
assertFresh(items, labelsOf(p))
|
|
const { fresh, m, v } = projected(items, drafts, p, now)
|
|
const next = nextState(readState(stateOf(p)), m, v, now)
|
|
const feedback = renderFeedback(m, v, now)
|
|
if (fresh.length > 0) appendRows(labelsOf(p), fresh)
|
|
writeState(stateOf(p), next.state)
|
|
writeSafe(path.join(p.out, 'FEEDBACK.md'), feedback)
|
|
return feedback + next.news.map(n => n + '\n').join('')
|
|
}
|
|
|
|
const isCount = (v: unknown) =>
|
|
typeof v === 'number' && Number.isInteger(v) && v >= 0
|
|
const isSafeCount = (v: unknown) => isCount(v) && Number.isSafeInteger(v)
|
|
const optCount = (v: unknown) => v === undefined || isCount(v)
|
|
const idsOk = (v: unknown): v is string[] => Array.isArray(v) &&
|
|
v.every(x => typeof x === 'string' && ENTRY_ID.test(x))
|
|
const inEnum = (v: unknown, set: string[]) =>
|
|
v === '' || (typeof v === 'string' && set.includes(v))
|
|
|
|
/** Why a parsed items.json element is unusable; '' when it is sound. */
|
|
function itemError(raw: unknown): string {
|
|
const i = (typeof raw === 'object' && raw !== null ? raw : {}) as
|
|
Record<string, unknown>
|
|
const text = (v: unknown, re: RegExp) => typeof v === 'string' && re.test(v)
|
|
const checks: [string, boolean][] = [
|
|
['root', text(i.root, /^[^\t\n/]+#[0-9a-f]{6}$/)],
|
|
['session', text(i.session, /^[A-Za-z0-9_-]+$/)],
|
|
['channel', i.channel === 'prompt' || i.channel === 'tool'],
|
|
['trigger', inEnum(i.trigger, TRIGGERS)],
|
|
['outcome', inEnum(i.outcome, OUTCOMES)],
|
|
['ids', idsOk(i.ids)],
|
|
['cited', idsOk(i.cited) && i.cited.every(c =>
|
|
(i.ids as string[]).includes(c))],
|
|
['numbers', isSafeCount(i.at) && isCount(i.n) && optCount(i.cands) &&
|
|
optCount(i.ms)],
|
|
['lost', typeof i.lost === 'boolean'],
|
|
]
|
|
return checks.find(([, ok]) => !ok)?.[0] ?? ''
|
|
}
|
|
|
|
/** The items.json content, every element validated (it is on disk in a
|
|
* shared scratchpad: treat it as untrusted input). */
|
|
function readItems(dir: string): Item[] {
|
|
const items: unknown = JSON.parse(
|
|
fs.readFileSync(path.join(dir, 'items.json'), 'utf8'))
|
|
if (!Array.isArray(items)) throw new Error('items.json invalid: not a list')
|
|
for (const it of items) {
|
|
const bad = itemError(it)
|
|
if (bad !== '') throw new Error(`items.json invalid: bad ${bad}`)
|
|
}
|
|
return items as Item[]
|
|
}
|
|
|
|
/** Validates the draft; appends labels, writes state, then FEEDBACK.md
|
|
* (a failed state write leaves the NEW line to print on the next run). */
|
|
export function apply(dir: string, p: Paths, now: number): string {
|
|
const items = readItems(dir)
|
|
const drafts = parseDraft(draftOf(dir), items)
|
|
if (typeof drafts === 'string') throw new Error(`draft refused: ${drafts}`)
|
|
return commit(p, items, drafts, now)
|
|
}
|
|
|
|
/** Up to `n` entry lines of the draft, picked at random. */
|
|
export function sample(dir: string, n: number): string[] {
|
|
const lines = draftOf(dir).split('\n')
|
|
.filter(l => /\t(relevant|irrelevant)$/.test(l))
|
|
for (let i = lines.length - 1; i > 0; i--) {
|
|
const j = Math.floor(Math.random() * (i + 1))
|
|
;[lines[i], lines[j]] = [lines[j] as string, lines[i] as string]
|
|
}
|
|
return lines.slice(0, n)
|
|
}
|
|
|
|
/** '' only for a git answer of "outside a work tree": exit 0 and
|
|
* `false`, or exit 128 and `not a git repository`. Fails closed. */
|
|
function gitVerdict(dir: string): string {
|
|
const git = spawnSync('git', ['-C', dir, 'rev-parse',
|
|
'--is-inside-work-tree'], { encoding: 'utf8' })
|
|
if (git.error !== undefined) return 'git unavailable'
|
|
if (git.status === 0 && String(git.stdout).trim() === 'false') return ''
|
|
if (git.status === 0) return 'inside a git work tree'
|
|
if (git.status === 128 &&
|
|
String(git.stderr).includes('not a git repository')) return ''
|
|
return 'git check inconclusive'
|
|
}
|
|
|
|
/** '' for a usable dir; else why not (text must never reach a repo). */
|
|
export function checkDir(dir: string | undefined): string {
|
|
if (dir === undefined || !path.isAbsolute(dir)) return 'not absolute'
|
|
if (!fs.existsSync(dir)) return 'does not exist'
|
|
const st = fs.statSync(dir)
|
|
if (st.uid !== process.getuid?.()) return 'not owned by you'
|
|
if ((st.mode & 0o022) !== 0) return 'group or world writable'
|
|
return gitVerdict(dir)
|
|
}
|
|
|
|
function run(cmd: string, dir: string): void {
|
|
const p = defaultPaths()
|
|
if (cmd === 'worksheet') {
|
|
const n = worksheet(dir, p, Date.now())
|
|
console.log(`${n} records to label: ${dir}/worksheet.md`)
|
|
} else if (cmd === 'apply') {
|
|
process.stdout.write(apply(dir, p, Date.now()))
|
|
} else console.log(sample(dir, 5).join('\n'))
|
|
}
|
|
|
|
function main(argv: string[]): number {
|
|
const [cmd, dir] = argv
|
|
const why = checkDir(dir)
|
|
if (!['worksheet', 'apply', 'sample'].includes(cmd ?? '') || why !== '') {
|
|
console.error('usage: reader.ts worksheet|apply|sample <dir> ' +
|
|
`(absolute, outside any git work tree)${why ? `: ${why}` : ''}`)
|
|
return 2
|
|
}
|
|
try {
|
|
run(cmd as string, dir as string)
|
|
return 0
|
|
} catch (e) {
|
|
console.error(`memory-feedback: ${(e as Error).message}`)
|
|
return 1
|
|
}
|
|
}
|
|
|
|
if (import.meta.main) process.exit(main(process.argv.slice(2)))
|