forked from bchanot/claude
fix(memory-feedback): final review: locals, per-trigger n, write order
This commit is contained in:
1 parent
c00649f516
commit
271172564c
8 files changed
+167
-78
No files matched your search
@@ -118,3 +118,14 @@ test('M9: FEEDBACK.md counts, no repo tag without a noisy entry', () => {
|
|||||||
expect(text).not.toContain('client#abcdef')
|
expect(text).not.toContain('client#abcdef')
|
||||||
expect(text).toContain('Prompt gate: open (1/20)')
|
expect(text).toContain('Prompt gate: open (1/20)')
|
||||||
})
|
})
|
||||||
|
|
||||||
|
test('M10: FEEDBACK.md shows n per channel and trigger', () => {
|
||||||
|
const rows = [...ask(['relevant', 'irrelevant']),
|
||||||
|
...ask(['irrelevant'], { trigger: 'idle' })]
|
||||||
|
const mm = metricsOf(rows)
|
||||||
|
const text = renderFeedback(mm, gateVerdict(mm, 1), 0)
|
||||||
|
expect(text).toContain('| precision, idle | 0 % (n=1) |')
|
||||||
|
expect(text).toContain('| precision, fresh | 50 % (n=2) |')
|
||||||
|
expect(text).toContain('| prompt precision | 33 % (n=3) |')
|
||||||
|
expect(text).toContain('| tool precision | n/a (n=0) |')
|
||||||
|
})
|
||||||
@@ -10,11 +10,13 @@ export type Gate = 'open' | 'reached'
|
|||||||
export type ToolGate = Gate | 'extended' | 'insufficient'
|
export type ToolGate = Gate | 'extended' | 'insufficient'
|
||||||
export type Verdict = { prompt: Gate; tool: ToolGate }
|
export type Verdict = { prompt: Gate; tool: ToolGate }
|
||||||
type Rate = number | undefined
|
type Rate = number | undefined
|
||||||
|
type Sized = { rate: Rate; n: number }
|
||||||
export type Metrics = {
|
export type Metrics = {
|
||||||
start?: number; promptN: number; toolN: number; lost: number
|
start?: number; promptN: number; toolN: number; lost: number
|
||||||
precision: Rate; byTrigger: Record<string, Rate>; thirdPick: Rate
|
precision: Rate; byTrigger: Record<string, Sized>; thirdPick: Rate
|
||||||
late: Rate; msP90: Rate; tool: Rate; missShare: Rate; sampled: number
|
late: Rate; msP90: Rate; tool: Rate; missShare: Rate; sampled: number
|
||||||
noisy: string[]; promote: string[]
|
noisy: string[]; promote: string[]
|
||||||
|
n: { prompt: number; third: number; tool: number } // entries
|
||||||
}
|
}
|
||||||
|
|
||||||
const share = (k: number, n: number): Rate => n === 0 ? undefined : k / n
|
const share = (k: number, n: number): Rate => n === 0 ? undefined : k / n
|
||||||
@@ -52,14 +54,22 @@ export function p90(values: number[]): Rate {
|
|||||||
return s[Math.ceil(0.9 * s.length) - 1]
|
return s[Math.ceil(0.9 * s.length) - 1]
|
||||||
}
|
}
|
||||||
|
|
||||||
function byTrigger(entries: Row[]): Record<string, Rate> {
|
function byTrigger(entries: Row[]): Record<string, Sized> {
|
||||||
const out: Record<string, Rate> = {}
|
const out: Record<string, Sized> = {}
|
||||||
for (const k of ['fresh', 'idle', 'keyword']) {
|
for (const k of ['fresh', 'idle', 'keyword']) {
|
||||||
out[k] = precision(entries.filter(r => r.trigger === k))
|
const mine = entries.filter(r => r.trigger === k)
|
||||||
|
out[k] = { rate: precision(mine), n: mine.length }
|
||||||
}
|
}
|
||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Entry counts behind the three channel precisions. */
|
||||||
|
const countsOf = (entries: Row[]) => ({
|
||||||
|
prompt: entries.filter(r => r.channel === 'prompt').length,
|
||||||
|
third: entries.filter(r => r.channel === 'prompt' && r.pick === 3).length,
|
||||||
|
tool: entries.filter(r => r.channel === 'tool').length,
|
||||||
|
})
|
||||||
|
|
||||||
/** timeout + paused among asks (lost ones too) that had candidates. */
|
/** timeout + paused among asks (lost ones too) that had candidates. */
|
||||||
function lateShare(asks: Row[]): Rate {
|
function lateShare(asks: Row[]): Rate {
|
||||||
const had = asks.filter(r => (r.cands ?? 0) > 0)
|
const had = asks.filter(r => (r.cands ?? 0) > 0)
|
||||||
@@ -117,6 +127,7 @@ export function metricsOf(all: Row[]): Metrics {
|
|||||||
tool: precision(entries.filter(r => r.channel === 'tool')),
|
tool: precision(entries.filter(r => r.channel === 'tool')),
|
||||||
missShare: miss.rate, sampled: miss.sampled,
|
missShare: miss.rate, sampled: miss.sampled,
|
||||||
noisy: noisy(entries), promote: promote(entries),
|
noisy: noisy(entries), promote: promote(entries),
|
||||||
|
n: countsOf(entries),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -137,10 +148,10 @@ export function gateVerdict(m: Metrics, now: number): Verdict {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/** The worst trigger under the floor; ties go idle, keyword, fresh. */
|
/** The worst trigger under the floor; ties go idle, keyword, fresh. */
|
||||||
function worstTrigger(by: Record<string, Rate>): string | undefined {
|
function worstTrigger(by: Record<string, Sized>): string | undefined {
|
||||||
const low = ['idle', 'keyword', 'fresh']
|
const low = ['idle', 'keyword', 'fresh']
|
||||||
.filter(k => under(by[k], T.triggerFloor))
|
.filter(k => under(by[k]?.rate, T.triggerFloor))
|
||||||
low.sort((a, b) => (by[a] ?? 0) - (by[b] ?? 0))
|
low.sort((a, b) => (by[a]?.rate ?? 0) - (by[b]?.rate ?? 0))
|
||||||
return low[0]
|
return low[0]
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -195,13 +206,13 @@ export function renderFeedback(m: Metrics, v: Verdict, now: number) {
|
|||||||
`Prompt gate: ${v.prompt} (${m.promptN}/${PROMPT_GATE_N})`,
|
`Prompt gate: ${v.prompt} (${m.promptN}/${PROMPT_GATE_N})`,
|
||||||
toolLine(m, v), `Lost records: ${m.lost}`, '',
|
toolLine(m, v), `Lost records: ${m.lost}`, '',
|
||||||
'| Signal | Value |', '|---|---|',
|
'| Signal | Value |', '|---|---|',
|
||||||
`| prompt precision | ${pct(m.precision)} |`,
|
`| prompt precision | ${pct(m.precision)} (n=${m.n.prompt}) |`,
|
||||||
...Object.entries(m.byTrigger).map(([k, x]) =>
|
...Object.entries(m.byTrigger).map(([k, x]) =>
|
||||||
`| precision, ${k} | ${pct(x)} |`),
|
`| precision, ${k} | ${pct(x.rate)} (n=${x.n}) |`),
|
||||||
`| 3rd pick precision | ${pct(m.thirdPick)} |`,
|
`| 3rd pick precision | ${pct(m.thirdPick)} (n=${m.n.third}) |`,
|
||||||
`| timeout + paused | ${pct(m.late)} |`,
|
`| timeout + paused | ${pct(m.late)} |`,
|
||||||
`| rerank ms p90 | ${m.msP90 ?? 'n/a'} |`,
|
`| rerank ms p90 | ${m.msP90 ?? 'n/a'} |`,
|
||||||
`| tool precision | ${pct(m.tool)} |`,
|
`| tool precision | ${pct(m.tool)} (n=${m.n.tool}) |`,
|
||||||
`| asks with a miss | ${pct(m.missShare)} of ${m.sampled} |`, '',
|
`| asks with a miss | ${pct(m.missShare)} of ${m.sampled} |`, '',
|
||||||
`Noisy: ${m.noisy.join(', ') || 'none'}`,
|
`Noisy: ${m.noisy.join(', ') || 'none'}`,
|
||||||
`Promotion candidates: ${m.promote.join(', ') || 'none'}`, '',
|
`Promotion candidates: ${m.promote.join(', ') || 'none'}`, '',
|
||||||
|
|||||||
@@ -13,7 +13,8 @@ import {
|
|||||||
} from '../../mods/memory/hooks/register.ts'
|
} from '../../mods/memory/hooks/register.ts'
|
||||||
import {
|
import {
|
||||||
appendRows, keyOf, parseDraft, readRows, readState, rowsOf, skeleton,
|
appendRows, keyOf, parseDraft, readRows, readState, rowsOf, skeleton,
|
||||||
tagOf, writeSafe, writeState, type Item, type State,
|
tagOf, writeSafe, writeState, type Draft, type Item, type Row,
|
||||||
|
type State,
|
||||||
} from './labels.ts'
|
} from './labels.ts'
|
||||||
import {
|
import {
|
||||||
clockStart, gateVerdict, metricsOf, renderFeedback, type Verdict,
|
clockStart, gateVerdict, metricsOf, renderFeedback, type Verdict,
|
||||||
@@ -56,8 +57,7 @@ export function storeFile(dir: string): string {
|
|||||||
return files.sort((a, b) => mtime(b) - mtime(a))[0] as string
|
return files.sort((a, b) => mtime(b) - mtime(a))[0] as string
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Every valid log record of every root, oldest first. */
|
function readStore(file: string): object {
|
||||||
export function readLogs(file: string): Logged[] {
|
|
||||||
let data: unknown
|
let data: unknown
|
||||||
try {
|
try {
|
||||||
data = JSON.parse(fs.readFileSync(file, 'utf8'))
|
data = JSON.parse(fs.readFileSync(file, 'utf8'))
|
||||||
@@ -67,13 +67,16 @@ export function readLogs(file: string): Logged[] {
|
|||||||
if (typeof data !== 'object' || data === null) {
|
if (typeof data !== 'object' || data === null) {
|
||||||
throw new Error(`store file is not an object: ${file}`)
|
throw new Error(`store file is not an object: ${file}`)
|
||||||
}
|
}
|
||||||
const out: Logged[] = []
|
return data
|
||||||
for (const [k, v] of Object.entries(data)) {
|
}
|
||||||
if (!k.startsWith(LOG_PREFIX) || !Array.isArray(v)) continue
|
|
||||||
const root = k.slice(LOG_PREFIX.length)
|
/** Every valid log record of every root, oldest first. */
|
||||||
for (const rec of v.filter(isLogRecord)) out.push({ root, rec })
|
export function readLogs(file: string): Logged[] {
|
||||||
}
|
return Object.entries(readStore(file))
|
||||||
return out.sort((a, b) => a.rec.at - b.rec.at)
|
.filter(([k, v]) => k.startsWith(LOG_PREFIX) && Array.isArray(v))
|
||||||
|
.flatMap(([k, v]) => (v as unknown[]).filter(isLogRecord)
|
||||||
|
.map(rec => ({ root: k.slice(LOG_PREFIX.length), rec })))
|
||||||
|
.sort((a, b) => a.rec.at - b.rec.at)
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Unlabelled records from the clock start, AGE_FLOOR_MS old or more. */
|
/** Unlabelled records from the clock start, AGE_FLOOR_MS old or more. */
|
||||||
@@ -109,6 +112,8 @@ function header(b: Base, repo: string, lost: boolean): string {
|
|||||||
const ids = b.ids.join(' ') || '-'
|
const ids = b.ids.join(' ') || '-'
|
||||||
return `## Item ${b.n}: ${b.channel}/${b.trigger}/${b.outcome}` +
|
return `## Item ${b.n}: ${b.channel}/${b.trigger}/${b.outcome}` +
|
||||||
` · cands ${b.cands ?? '-'} · ms ${b.ms ?? '-'}` +
|
` · cands ${b.cands ?? '-'} · ms ${b.ms ?? '-'}` +
|
||||||
|
` · s:${b.session.slice(0, 8)}` +
|
||||||
|
` · ${new Date(b.at).toISOString().slice(11, 16)} UTC` +
|
||||||
` · ${b.root} (${repo})` +
|
` · ${b.root} (${repo})` +
|
||||||
` · ids ${ids}${lost ? ' · LOST (no transcript turn)' : ''}`
|
` · ids ${ids}${lost ? ' · LOST (no transcript turn)' : ''}`
|
||||||
}
|
}
|
||||||
@@ -188,24 +193,34 @@ function sheetHead(n: number, torn: number, evicted: string[]): string {
|
|||||||
return lines.join('\n')
|
return lines.join('\n')
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Writes worksheet.md, items.json, labels-draft.tsv (0600); the count. */
|
const logAtsOf = (logs: Logged[]) =>
|
||||||
export function worksheet(dir: string, p: Paths, now: number): number {
|
logs.filter(l => l.rec.cands !== undefined).map(l => l.rec.at)
|
||||||
|
|
||||||
|
/** Joins the unlabelled population to its transcripts. */
|
||||||
|
function collect(p: Paths, now: number) {
|
||||||
const rows = readRows(labelsOf(p))
|
const rows = readRows(labelsOf(p))
|
||||||
const logs = readLogs(storeFile(p.store))
|
const logs = readLogs(storeFile(p.store))
|
||||||
const logAts = logs.filter(l => l.rec.cands !== undefined)
|
|
||||||
.map(l => l.rec.at)
|
|
||||||
const pop = population(logs, new Set(rows.map(keyOf)),
|
const pop = population(logs, new Set(rows.map(keyOf)),
|
||||||
clockStart(rows, logAts), now)
|
clockStart(rows, logAtsOf(logs)), now)
|
||||||
const loader = loaderFor(p.projects)
|
const loader = loaderFor(p.projects)
|
||||||
const joined = pop.map((l, i) => joinOne(l, i + 1, loader.load))
|
const joined = pop.map((l, i) => joinOne(l, i + 1, loader.load))
|
||||||
const head = sheetHead(pop.length, loader.torn(),
|
return { joined, head: sheetHead(pop.length, loader.torn(),
|
||||||
evictions(logs, readState(stateOf(p)).lastRun))
|
evictions(logs, readState(stateOf(p)).lastRun)) }
|
||||||
|
}
|
||||||
|
|
||||||
|
function writeSheet(dir: string, joined: Joined[], head: string): void {
|
||||||
writeSafe(path.join(dir, 'worksheet.md'), ['# Memory feedback worksheet',
|
writeSafe(path.join(dir, 'worksheet.md'), ['# Memory feedback worksheet',
|
||||||
head, DRAFT_HELP, ...joined.map(j => j.text)].join('\n\n') + '\n')
|
head, DRAFT_HELP, ...joined.map(j => j.text)].join('\n\n') + '\n')
|
||||||
const items = joined.map(j => j.item)
|
const items = joined.map(j => j.item)
|
||||||
writeSafe(path.join(dir, 'items.json'), JSON.stringify(items))
|
writeSafe(path.join(dir, 'items.json'), JSON.stringify(items))
|
||||||
writeSafe(path.join(dir, 'labels-draft.tsv'), skeleton(items))
|
writeSafe(path.join(dir, 'labels-draft.tsv'), skeleton(items))
|
||||||
return pop.length
|
}
|
||||||
|
|
||||||
|
/** Writes worksheet.md, items.json, labels-draft.tsv (0600); the count. */
|
||||||
|
export function worksheet(dir: string, p: Paths, now: number): number {
|
||||||
|
const { joined, head } = collect(p, now)
|
||||||
|
writeSheet(dir, joined, head)
|
||||||
|
return joined.length
|
||||||
}
|
}
|
||||||
|
|
||||||
const draftOf = (dir: string) => {
|
const draftOf = (dir: string) => {
|
||||||
@@ -230,26 +245,45 @@ function nextState(st: State, v: Verdict, now: number) {
|
|||||||
return { state: { lastRun: now, promptAt, toolAt, closed }, news }
|
return { state: { lastRun: now, promptAt, toolAt, closed }, news }
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Validates the draft; writes state, then labels, then FEEDBACK.md. */
|
const assertFresh = (items: Item[], old: Row[]) => {
|
||||||
|
const done = new Set(old.map(keyOf))
|
||||||
|
if (items.some(i => done.has(keyOf(i)))) {
|
||||||
|
throw new Error('items already applied: run worksheet again')
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The new rows, and the metrics and verdict once they are added. */
|
||||||
|
function projected(
|
||||||
|
items: Item[], drafts: Draft[], old: Row[], now: number,
|
||||||
|
) {
|
||||||
|
const fresh = items.flatMap(i => rowsOf(i, drafts))
|
||||||
|
const m = metricsOf([...old, ...fresh])
|
||||||
|
return { fresh, m, v: gateVerdict(m, now) }
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Appends labels, then writes state, then FEEDBACK.md. */
|
||||||
|
function commit(
|
||||||
|
p: Paths, items: Item[], drafts: Draft[], now: number,
|
||||||
|
): string {
|
||||||
|
const old = readRows(labelsOf(p))
|
||||||
|
assertFresh(items, old)
|
||||||
|
const { fresh, m, v } = projected(items, drafts, old, now)
|
||||||
|
const next = nextState(readState(stateOf(p)), v, now)
|
||||||
|
const feedback = renderFeedback(m, v, now)
|
||||||
|
if (fresh.length > 0) appendRows(labelsOf(p), fresh)
|
||||||
|
writeState(stateOf(p), next.state)
|
||||||
|
writeSafe(path.join(p.out, 'FEEDBACK.md'), feedback)
|
||||||
|
return feedback + next.news.map(n => n + '\n').join('')
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Validates the draft; appends labels, writes state, then FEEDBACK.md
|
||||||
|
* (a failed state write leaves the NEW line to print on the next run). */
|
||||||
export function apply(dir: string, p: Paths, now: number): string {
|
export function apply(dir: string, p: Paths, now: number): string {
|
||||||
const items: Item[] = JSON.parse(
|
const items: Item[] = JSON.parse(
|
||||||
fs.readFileSync(path.join(dir, 'items.json'), 'utf8'))
|
fs.readFileSync(path.join(dir, 'items.json'), 'utf8'))
|
||||||
const drafts = parseDraft(draftOf(dir), items)
|
const drafts = parseDraft(draftOf(dir), items)
|
||||||
if (typeof drafts === 'string') throw new Error(`draft refused: ${drafts}`)
|
if (typeof drafts === 'string') throw new Error(`draft refused: ${drafts}`)
|
||||||
const old = readRows(labelsOf(p))
|
return commit(p, items, drafts, now)
|
||||||
const done = new Set(old.map(keyOf))
|
|
||||||
if (items.some(i => done.has(keyOf(i)))) {
|
|
||||||
throw new Error('items already applied: run worksheet again')
|
|
||||||
}
|
|
||||||
const fresh = items.flatMap(i => rowsOf(i, drafts))
|
|
||||||
const m = metricsOf([...old, ...fresh])
|
|
||||||
const v = gateVerdict(m, now)
|
|
||||||
const { state, news } = nextState(readState(stateOf(p)), v, now)
|
|
||||||
const feedback = renderFeedback(m, v, now)
|
|
||||||
writeState(stateOf(p), state)
|
|
||||||
if (fresh.length > 0) appendRows(labelsOf(p), fresh)
|
|
||||||
writeSafe(path.join(p.out, 'FEEDBACK.md'), feedback)
|
|
||||||
return feedback + news.map(n => n + '\n').join('')
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Up to `n` entry lines of the draft, picked at random. */
|
/** Up to `n` entry lines of the draft, picked at random. */
|
||||||
@@ -269,6 +303,9 @@ export function checkDir(dir: string | undefined): string {
|
|||||||
if (!fs.existsSync(dir)) return 'does not exist'
|
if (!fs.existsSync(dir)) return 'does not exist'
|
||||||
const git = spawnSync('git', ['-C', dir, 'rev-parse',
|
const git = spawnSync('git', ['-C', dir, 'rev-parse',
|
||||||
'--is-inside-work-tree'], { encoding: 'utf8' })
|
'--is-inside-work-tree'], { encoding: 'utf8' })
|
||||||
|
if (git.error !== undefined || typeof git.stdout !== 'string') {
|
||||||
|
return 'git unavailable'
|
||||||
|
}
|
||||||
return git.stdout.trim() === 'true' ? 'inside a git work tree' : ''
|
return git.stdout.trim() === 'true' ? 'inside a git work tree' : ''
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -41,6 +41,9 @@ test('X1: slug, the main or one sub-agent file, torn lines counted', () => {
|
|||||||
expect(sessionFiles(p, '/r', 's', 'a1').map(f => path.basename(f)))
|
expect(sessionFiles(p, '/r', 's', 'a1').map(f => path.basename(f)))
|
||||||
.toEqual(['agent-a1.jsonl'])
|
.toEqual(['agent-a1.jsonl'])
|
||||||
expect(records(main[0]!)).toEqual({ recs: [{ a: 1 }], torn: 1 })
|
expect(records(main[0]!)).toEqual({ recs: [{ a: 1 }], torn: 1 })
|
||||||
|
fs.writeFileSync(path.join(dir, 'n.jsonl'), '{"a":1}\nnull\n[1]\n7\n')
|
||||||
|
expect(records(path.join(dir, 'n.jsonl')))
|
||||||
|
.toEqual({ recs: [{ a: 1 }], torn: 3 })
|
||||||
expect(sessionFiles(p, '/r', 'gone', 'main')).toEqual([])
|
expect(sessionFiles(p, '/r', 'gone', 'main')).toEqual([])
|
||||||
})
|
})
|
||||||
|
|
||||||
|
|||||||
@@ -28,13 +28,19 @@ export function sessionFiles(
|
|||||||
return fs.existsSync(file) ? [file] : []
|
return fs.existsSync(file) ? [file] : []
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Parsed lines of a transcript; torn or partial lines are counted. */
|
/** Parsed lines of a transcript; torn, partial or non-object lines
|
||||||
|
* (`null`, a number, an array) are counted as torn. */
|
||||||
export function records(file: string): Loaded {
|
export function records(file: string): Loaded {
|
||||||
const recs: Rec[] = []
|
const recs: Rec[] = []
|
||||||
let torn = 0
|
let torn = 0
|
||||||
for (const raw of fs.readFileSync(file, 'utf8').split('\n')) {
|
for (const raw of fs.readFileSync(file, 'utf8').split('\n')) {
|
||||||
if (raw.trim() === '') continue
|
if (raw.trim() === '') continue
|
||||||
try { recs.push(JSON.parse(raw)) } catch { torn += 1 }
|
try {
|
||||||
|
const v = JSON.parse(raw)
|
||||||
|
if (v === null || typeof v !== 'object' || Array.isArray(v)) {
|
||||||
|
torn += 1
|
||||||
|
} else recs.push(v)
|
||||||
|
} catch { torn += 1 }
|
||||||
}
|
}
|
||||||
return { recs, torn }
|
return { recs, torn }
|
||||||
}
|
}
|
||||||
@@ -90,7 +96,8 @@ function promptNear(recs: Rec[], ts: number): Prompt | undefined {
|
|||||||
for (const r of recs) {
|
for (const r of recs) {
|
||||||
const text = composerText(r)
|
const text = composerText(r)
|
||||||
const t = tsOf(r)
|
const t = tsOf(r)
|
||||||
if (text === undefined || Math.abs(t - ts) > NEAR_MS) continue
|
if (text === undefined || !Number.isFinite(t)) continue
|
||||||
|
if (Math.abs(t - ts) > NEAR_MS) continue
|
||||||
if (best === undefined || Math.abs(t - ts) < Math.abs(best.t - ts)) {
|
if (best === undefined || Math.abs(t - ts) < Math.abs(best.t - ts)) {
|
||||||
best = { t, text }
|
best = { t, text }
|
||||||
}
|
}
|
||||||
@@ -108,22 +115,22 @@ function nextPromptAt(recs: Rec[], ts: number): number {
|
|||||||
return end
|
return end
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** The [dedupe key, text] of each text block of an assistant record. */
|
||||||
|
const textsOf = (r: Rec): [string, string][] => blocksOf(r)
|
||||||
|
.filter(b => b.type === 'text')
|
||||||
|
.map(b => [`${r.message?.id}:${b.text}`, String(b.text)]) // EVAL-037
|
||||||
|
|
||||||
/** Main-thread assistant text after `ts` up to the next prompt. */
|
/** Main-thread assistant text after `ts` up to the next prompt. */
|
||||||
function replyAfter(recs: Rec[], ts: number): string {
|
function replyAfter(recs: Rec[], ts: number): string {
|
||||||
const end = nextPromptAt(recs, ts)
|
const end = nextPromptAt(recs, ts)
|
||||||
const seen = new Set<string>()
|
const seen = new Set<string>()
|
||||||
const parts: string[] = []
|
return recs
|
||||||
for (const r of recs) {
|
.filter(r => r.type === 'assistant' && r.isSidechain !== true)
|
||||||
const t = tsOf(r)
|
.filter(r => tsOf(r) > ts && tsOf(r) < end)
|
||||||
if (r.type !== 'assistant' || r.isSidechain === true) continue
|
.flatMap(textsOf)
|
||||||
if (!(t > ts && t < end)) continue
|
.filter(([key]) => !seen.has(key) && !!seen.add(key))
|
||||||
for (const b of blocksOf(r).filter(b => b.type === 'text')) {
|
.map(([, text]) => text)
|
||||||
const key = `${r.message?.id}:${b.text}` // EVAL-037
|
.join('\n')
|
||||||
if (!seen.has(key)) parts.push(String(b.text))
|
|
||||||
seen.add(key)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return parts.join('\n')
|
|
||||||
}
|
}
|
||||||
|
|
||||||
export type PromptJoin = { prompt: string; injected: string; reply: string }
|
export type PromptJoin = { prompt: string; injected: string; reply: string }
|
||||||
@@ -178,7 +185,7 @@ export type ToolJoin = { command: string; known: string; next: boolean[] }
|
|||||||
|
|
||||||
/** The failure a known line answered and the next 3 Bash results. */
|
/** The failure a known line answered and the next 3 Bash results. */
|
||||||
export function joinTool(recs: Rec[], rec: LogRecord): ToolJoin | undefined {
|
export function joinTool(recs: Rec[], rec: LogRecord): ToolJoin | undefined {
|
||||||
const needle = KNOWN_HEAD + (rec.ids[0] ?? '')
|
const needle = `${KNOWN_HEAD}${rec.ids[0] ?? ''} ` // BLK-003 != BLK-0031
|
||||||
const att = attachmentAfter(recs, rec.at, 'tool.call', needle)
|
const att = attachmentAfter(recs, rec.at, 'tool.call', needle)
|
||||||
const raw = att?.attachment?.toolUseID
|
const raw = att?.attachment?.toolUseID
|
||||||
if (att === undefined || typeof raw !== 'string') return undefined
|
if (att === undefined || typeof raw !== 'string') return undefined
|
||||||
|
|||||||
@@ -13,8 +13,10 @@ check() { if [ "$2" = "$3" ]; then pass=$((pass+1)); else fail=$((fail+1));
|
|||||||
printf 'FAIL %s: got[%s] want[%s]\n' "$1" "$2" "$3"; fi; }
|
printf 'FAIL %s: got[%s] want[%s]\n' "$1" "$2" "$3"; fi; }
|
||||||
# home <name>: a fixture HOME with a store file (unless "nostore")
|
# home <name>: a fixture HOME with a store file (unless "nostore")
|
||||||
home() {
|
home() {
|
||||||
mkdir -p "$WORK/$1/.claude/plugins/store" "$WORK/$1/.claude/memory-feedback"
|
mkdir -p "$WORK/$1/.claude/plugins/store" \
|
||||||
[ "$1" = nostore ] || : > "$WORK/$1/.claude/plugins/store/memory_skills-dir-a.json"
|
"$WORK/$1/.claude/memory-feedback"
|
||||||
|
[ "$1" = nostore ] ||
|
||||||
|
: > "$WORK/$1/.claude/plugins/store/memory_skills-dir-a.json"
|
||||||
echo "$WORK/$1"
|
echo "$WORK/$1"
|
||||||
}
|
}
|
||||||
OPEN='{"lastRun":1,"closed":false}'
|
OPEN='{"lastRun":1,"closed":false}'
|
||||||
@@ -24,24 +26,29 @@ old() {
|
|||||||
touch -t 202601010000 "$1/.claude/memory-feedback/state.json"
|
touch -t 202601010000 "$1/.claude/memory-feedback/state.json"
|
||||||
}
|
}
|
||||||
run() { HOME="$1" bash "$LIB" 2>&1; echo "rc=$?"; }
|
run() { HOME="$1" bash "$LIB" 2>&1; echo "rc=$?"; }
|
||||||
DUE='memory feedback: last run 5+ d ago, transcripts expire at 7 → /memory-feedback'
|
DUE='memory feedback: last run 5+ d ago, '
|
||||||
|
DUE+='transcripts expire at 7 → /memory-feedback'
|
||||||
|
|
||||||
check MR1-no-store "$(run "$(home nostore)")" "rc=0"
|
check MR1-no-store "$(run "$(home nostore)")" "rc=0"
|
||||||
check MR2-never "$(run "$(home never)")" "memory feedback: never run → /memory-feedback
|
NEVER='memory feedback: never run → /memory-feedback'
|
||||||
|
check MR2-never "$(run "$(home never)")" "$NEVER
|
||||||
rc=0"
|
rc=0"
|
||||||
h=$(home due); old "$h" "$OPEN"
|
h=$(home due); old "$h" "$OPEN"
|
||||||
check MR3-due "$(run "$h")" "$DUE
|
check MR3-due "$(run "$h")" "$DUE
|
||||||
rc=0"
|
rc=0"
|
||||||
h=$(home quiet); old "$h" "$OPEN"
|
h=$(home quiet); old "$h" "$OPEN"
|
||||||
touch -t 202512010000 "$h/.claude/plugins/store/memory_skills-dir-a.json"
|
touch -t 202512010000 \
|
||||||
|
"$h/.claude/plugins/store/memory_skills-dir-a.json"
|
||||||
check MR4-store-unchanged "$(run "$h")" "rc=0"
|
check MR4-store-unchanged "$(run "$h")" "rc=0"
|
||||||
h=$(home fresh)
|
h=$(home fresh)
|
||||||
printf '%s' "$OPEN" > "$h/.claude/memory-feedback/state.json"
|
printf '%s' "$OPEN" > "$h/.claude/memory-feedback/state.json"
|
||||||
touch -t 209901010000 "$h/.claude/plugins/store/memory_skills-dir-a.json"
|
touch -t 209901010000 \
|
||||||
|
"$h/.claude/plugins/store/memory_skills-dir-a.json"
|
||||||
check MR5-ran-today "$(run "$h")" "rc=0"
|
check MR5-ran-today "$(run "$h")" "rc=0"
|
||||||
h=$(home closed); old "$h" '{"lastRun":1,"closed":true}'
|
h=$(home closed); old "$h" '{"lastRun":1,"closed":true}'
|
||||||
check MR6-closed "$(run "$h")" "rc=0"
|
check MR6-closed "$(run "$h")" "rc=0"
|
||||||
h=$(home off); old "$h" "$OPEN"; printf '{"recall": false}' > "$h/.claude/memory.json"
|
h=$(home off); old "$h" "$OPEN"
|
||||||
|
printf '{"recall": false}' > "$h/.claude/memory.json"
|
||||||
check MR7-recall-off "$(run "$h")" "rc=0"
|
check MR7-recall-off "$(run "$h")" "rc=0"
|
||||||
h=$(home locked); old "$h" "$OPEN"
|
h=$(home locked); old "$h" "$OPEN"
|
||||||
chmod 000 "$h/.claude/memory-feedback/state.json"
|
chmod 000 "$h/.claude/memory-feedback/state.json"
|
||||||
|
|||||||
@@ -1734,6 +1734,22 @@ function claimPicks(st: State, key: string, picked: Entry[]): string[] {
|
|||||||
return taken
|
return taken
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Claims the picks of a rerank answer; the block to inject. */
|
||||||
|
function injectPicks(
|
||||||
|
st: State,
|
||||||
|
key: string,
|
||||||
|
answer: string,
|
||||||
|
cands: Entry[],
|
||||||
|
): Picked {
|
||||||
|
const picked = picksOf(answer, cands)
|
||||||
|
if (picked.length === 0) return nothing('none')
|
||||||
|
const taken = claimPicks(st, key, picked)
|
||||||
|
const block = recallBlock(
|
||||||
|
picked.filter(e => taken.includes(e.id)).map(recallLine), RECALL_CAP)
|
||||||
|
if (block === '') return nothing('refused')
|
||||||
|
return { block, ids: taken, outcome: 'injected' }
|
||||||
|
}
|
||||||
|
|
||||||
/** Asks haiku among the candidates and claims its picks. */
|
/** Asks haiku among the candidates and claims its picks. */
|
||||||
async function pickRecall(
|
async function pickRecall(
|
||||||
$: Api,
|
$: Api,
|
||||||
@@ -1748,13 +1764,7 @@ async function pickRecall(
|
|||||||
// a clock step back must not make the record invalid (isCount)
|
// a clock step back must not make the record invalid (isCount)
|
||||||
const ms = Math.max(0, Math.round((await $.clock.now()) - now))
|
const ms = Math.max(0, Math.round((await $.clock.now()) - now))
|
||||||
if (answer === undefined) return { ...nothing('timeout'), ms }
|
if (answer === undefined) return { ...nothing('timeout'), ms }
|
||||||
const picked = picksOf(answer, cands)
|
return { ...injectPicks(st, key, answer, cands), ms }
|
||||||
if (picked.length === 0) return { ...nothing('none'), ms }
|
|
||||||
const taken = claimPicks(st, key, picked)
|
|
||||||
const block = recallBlock(
|
|
||||||
picked.filter(e => taken.includes(e.id)).map(recallLine), RECALL_CAP)
|
|
||||||
if (block === '') return { ...nothing('refused'), ms }
|
|
||||||
return { block, ids: taken, outcome: 'injected', ms }
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
@@ -33,8 +33,10 @@ Read `<dir>/worksheet.md`. Fill the third column of the skeleton
|
|||||||
- `relevant`: the entry bears on what the prompt asked or what the reply
|
- `relevant`: the entry bears on what the prompt asked or what the reply
|
||||||
did; `irrelevant` otherwise. Judge the entry against the ask, not the
|
did; `irrelevant` otherwise. Judge the entry against the ask, not the
|
||||||
reply's quality.
|
reply's quality.
|
||||||
- A proof, bench or recall-testing session: replace that item's lines by
|
- A proof, bench or recall-testing session: every item of that session,
|
||||||
one `<item> - test` line.
|
with lines or not (none, timeout and paused items included), gets one
|
||||||
|
`<item> - test` line in place of its lines. The header's `s:` shows
|
||||||
|
the session, the time the order.
|
||||||
Lost items have no line.
|
Lost items have no line.
|
||||||
|
|
||||||
## STEP 3 — SPOT CHECK
|
## STEP 3 — SPOT CHECK
|
||||||
@@ -54,8 +56,9 @@ user confirms or flips each. Apply the flips to the draft (LRN-168).
|
|||||||
|
|
||||||
On up to 5 prompt items whose outcome is injected or none, search the
|
On up to 5 prompt items whose outcome is injected or none, search the
|
||||||
registries of that item's repo, whose absolute path is in the item header
|
registries of that item's repo, whose absolute path is in the item header
|
||||||
(`bash ~/.claude/lib/memory.sh -C <path> index` + entry bodies) for live entries that bore on the ask and were not
|
(`bash ~/.claude/lib/memory.sh -C <path> index` + entry bodies) for
|
||||||
injected. Each one: `<item> <id> missed <cause>`, cause `no-overlap` (no
|
live entries that bore on the ask and were not injected. Each one:
|
||||||
|
`<item> <id> missed <cause>`, cause `no-overlap` (no
|
||||||
content word shared with the prompt) or `overlap`. A searched item with no
|
content word shared with the prompt) or `overlap`. A searched item with no
|
||||||
miss: `<item> - checked`. Example line: `3 LRN-901 missed no-overlap`.
|
miss: `<item> - checked`. Example line: `3 LRN-901 missed no-overlap`.
|
||||||
|
|
||||||
|
|||||||
Reference in new issue
Block a user