forked from bchanot/claude
fix(memory-feedback): final review: locals, per-trigger n, write order
This commit is contained in:
1 parent
c00649f516
commit
271172564c
8 files changed
+167
-78
No files matched your search
@@ -118,3 +118,14 @@ test('M9: FEEDBACK.md counts, no repo tag without a noisy entry', () => {
|
||||
expect(text).not.toContain('client#abcdef')
|
||||
expect(text).toContain('Prompt gate: open (1/20)')
|
||||
})
|
||||
|
||||
test('M10: FEEDBACK.md shows n per channel and trigger', () => {
|
||||
const rows = [...ask(['relevant', 'irrelevant']),
|
||||
...ask(['irrelevant'], { trigger: 'idle' })]
|
||||
const mm = metricsOf(rows)
|
||||
const text = renderFeedback(mm, gateVerdict(mm, 1), 0)
|
||||
expect(text).toContain('| precision, idle | 0 % (n=1) |')
|
||||
expect(text).toContain('| precision, fresh | 50 % (n=2) |')
|
||||
expect(text).toContain('| prompt precision | 33 % (n=3) |')
|
||||
expect(text).toContain('| tool precision | n/a (n=0) |')
|
||||
})
|
||||
@@ -10,11 +10,13 @@ export type Gate = 'open' | 'reached'
|
||||
export type ToolGate = Gate | 'extended' | 'insufficient'
|
||||
export type Verdict = { prompt: Gate; tool: ToolGate }
|
||||
type Rate = number | undefined
|
||||
type Sized = { rate: Rate; n: number }
|
||||
export type Metrics = {
|
||||
start?: number; promptN: number; toolN: number; lost: number
|
||||
precision: Rate; byTrigger: Record<string, Rate>; thirdPick: Rate
|
||||
precision: Rate; byTrigger: Record<string, Sized>; thirdPick: Rate
|
||||
late: Rate; msP90: Rate; tool: Rate; missShare: Rate; sampled: number
|
||||
noisy: string[]; promote: string[]
|
||||
n: { prompt: number; third: number; tool: number } // entries
|
||||
}
|
||||
|
||||
const share = (k: number, n: number): Rate => n === 0 ? undefined : k / n
|
||||
@@ -52,14 +54,22 @@ export function p90(values: number[]): Rate {
|
||||
return s[Math.ceil(0.9 * s.length) - 1]
|
||||
}
|
||||
|
||||
function byTrigger(entries: Row[]): Record<string, Rate> {
|
||||
const out: Record<string, Rate> = {}
|
||||
function byTrigger(entries: Row[]): Record<string, Sized> {
|
||||
const out: Record<string, Sized> = {}
|
||||
for (const k of ['fresh', 'idle', 'keyword']) {
|
||||
out[k] = precision(entries.filter(r => r.trigger === k))
|
||||
const mine = entries.filter(r => r.trigger === k)
|
||||
out[k] = { rate: precision(mine), n: mine.length }
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
/** Entry counts behind the three channel precisions. */
|
||||
const countsOf = (entries: Row[]) => ({
|
||||
prompt: entries.filter(r => r.channel === 'prompt').length,
|
||||
third: entries.filter(r => r.channel === 'prompt' && r.pick === 3).length,
|
||||
tool: entries.filter(r => r.channel === 'tool').length,
|
||||
})
|
||||
|
||||
/** timeout + paused among asks (lost ones too) that had candidates. */
|
||||
function lateShare(asks: Row[]): Rate {
|
||||
const had = asks.filter(r => (r.cands ?? 0) > 0)
|
||||
@@ -117,6 +127,7 @@ export function metricsOf(all: Row[]): Metrics {
|
||||
tool: precision(entries.filter(r => r.channel === 'tool')),
|
||||
missShare: miss.rate, sampled: miss.sampled,
|
||||
noisy: noisy(entries), promote: promote(entries),
|
||||
n: countsOf(entries),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -137,10 +148,10 @@ export function gateVerdict(m: Metrics, now: number): Verdict {
|
||||
}
|
||||
|
||||
/** The worst trigger under the floor; ties go idle, keyword, fresh. */
|
||||
function worstTrigger(by: Record<string, Rate>): string | undefined {
|
||||
function worstTrigger(by: Record<string, Sized>): string | undefined {
|
||||
const low = ['idle', 'keyword', 'fresh']
|
||||
.filter(k => under(by[k], T.triggerFloor))
|
||||
low.sort((a, b) => (by[a] ?? 0) - (by[b] ?? 0))
|
||||
.filter(k => under(by[k]?.rate, T.triggerFloor))
|
||||
low.sort((a, b) => (by[a]?.rate ?? 0) - (by[b]?.rate ?? 0))
|
||||
return low[0]
|
||||
}
|
||||
|
||||
@@ -195,13 +206,13 @@ export function renderFeedback(m: Metrics, v: Verdict, now: number) {
|
||||
`Prompt gate: ${v.prompt} (${m.promptN}/${PROMPT_GATE_N})`,
|
||||
toolLine(m, v), `Lost records: ${m.lost}`, '',
|
||||
'| Signal | Value |', '|---|---|',
|
||||
`| prompt precision | ${pct(m.precision)} |`,
|
||||
`| prompt precision | ${pct(m.precision)} (n=${m.n.prompt}) |`,
|
||||
...Object.entries(m.byTrigger).map(([k, x]) =>
|
||||
`| precision, ${k} | ${pct(x)} |`),
|
||||
`| 3rd pick precision | ${pct(m.thirdPick)} |`,
|
||||
`| precision, ${k} | ${pct(x.rate)} (n=${x.n}) |`),
|
||||
`| 3rd pick precision | ${pct(m.thirdPick)} (n=${m.n.third}) |`,
|
||||
`| timeout + paused | ${pct(m.late)} |`,
|
||||
`| rerank ms p90 | ${m.msP90 ?? 'n/a'} |`,
|
||||
`| tool precision | ${pct(m.tool)} |`,
|
||||
`| tool precision | ${pct(m.tool)} (n=${m.n.tool}) |`,
|
||||
`| asks with a miss | ${pct(m.missShare)} of ${m.sampled} |`, '',
|
||||
`Noisy: ${m.noisy.join(', ') || 'none'}`,
|
||||
`Promotion candidates: ${m.promote.join(', ') || 'none'}`, '',
|
||||
|
||||
@@ -13,7 +13,8 @@ import {
|
||||
} from '../../mods/memory/hooks/register.ts'
|
||||
import {
|
||||
appendRows, keyOf, parseDraft, readRows, readState, rowsOf, skeleton,
|
||||
tagOf, writeSafe, writeState, type Item, type State,
|
||||
tagOf, writeSafe, writeState, type Draft, type Item, type Row,
|
||||
type State,
|
||||
} from './labels.ts'
|
||||
import {
|
||||
clockStart, gateVerdict, metricsOf, renderFeedback, type Verdict,
|
||||
@@ -56,8 +57,7 @@ export function storeFile(dir: string): string {
|
||||
return files.sort((a, b) => mtime(b) - mtime(a))[0] as string
|
||||
}
|
||||
|
||||
/** Every valid log record of every root, oldest first. */
|
||||
export function readLogs(file: string): Logged[] {
|
||||
function readStore(file: string): object {
|
||||
let data: unknown
|
||||
try {
|
||||
data = JSON.parse(fs.readFileSync(file, 'utf8'))
|
||||
@@ -67,13 +67,16 @@ export function readLogs(file: string): Logged[] {
|
||||
if (typeof data !== 'object' || data === null) {
|
||||
throw new Error(`store file is not an object: ${file}`)
|
||||
}
|
||||
const out: Logged[] = []
|
||||
for (const [k, v] of Object.entries(data)) {
|
||||
if (!k.startsWith(LOG_PREFIX) || !Array.isArray(v)) continue
|
||||
const root = k.slice(LOG_PREFIX.length)
|
||||
for (const rec of v.filter(isLogRecord)) out.push({ root, rec })
|
||||
}
|
||||
return out.sort((a, b) => a.rec.at - b.rec.at)
|
||||
return data
|
||||
}
|
||||
|
||||
/** Every valid log record of every root, oldest first. */
|
||||
export function readLogs(file: string): Logged[] {
|
||||
return Object.entries(readStore(file))
|
||||
.filter(([k, v]) => k.startsWith(LOG_PREFIX) && Array.isArray(v))
|
||||
.flatMap(([k, v]) => (v as unknown[]).filter(isLogRecord)
|
||||
.map(rec => ({ root: k.slice(LOG_PREFIX.length), rec })))
|
||||
.sort((a, b) => a.rec.at - b.rec.at)
|
||||
}
|
||||
|
||||
/** Unlabelled records from the clock start, AGE_FLOOR_MS old or more. */
|
||||
@@ -109,6 +112,8 @@ function header(b: Base, repo: string, lost: boolean): string {
|
||||
const ids = b.ids.join(' ') || '-'
|
||||
return `## Item ${b.n}: ${b.channel}/${b.trigger}/${b.outcome}` +
|
||||
` · cands ${b.cands ?? '-'} · ms ${b.ms ?? '-'}` +
|
||||
` · s:${b.session.slice(0, 8)}` +
|
||||
` · ${new Date(b.at).toISOString().slice(11, 16)} UTC` +
|
||||
` · ${b.root} (${repo})` +
|
||||
` · ids ${ids}${lost ? ' · LOST (no transcript turn)' : ''}`
|
||||
}
|
||||
@@ -188,24 +193,34 @@ function sheetHead(n: number, torn: number, evicted: string[]): string {
|
||||
return lines.join('\n')
|
||||
}
|
||||
|
||||
/** Writes worksheet.md, items.json, labels-draft.tsv (0600); the count. */
|
||||
export function worksheet(dir: string, p: Paths, now: number): number {
|
||||
const logAtsOf = (logs: Logged[]) =>
|
||||
logs.filter(l => l.rec.cands !== undefined).map(l => l.rec.at)
|
||||
|
||||
/** Joins the unlabelled population to its transcripts. */
|
||||
function collect(p: Paths, now: number) {
|
||||
const rows = readRows(labelsOf(p))
|
||||
const logs = readLogs(storeFile(p.store))
|
||||
const logAts = logs.filter(l => l.rec.cands !== undefined)
|
||||
.map(l => l.rec.at)
|
||||
const pop = population(logs, new Set(rows.map(keyOf)),
|
||||
clockStart(rows, logAts), now)
|
||||
clockStart(rows, logAtsOf(logs)), now)
|
||||
const loader = loaderFor(p.projects)
|
||||
const joined = pop.map((l, i) => joinOne(l, i + 1, loader.load))
|
||||
const head = sheetHead(pop.length, loader.torn(),
|
||||
evictions(logs, readState(stateOf(p)).lastRun))
|
||||
return { joined, head: sheetHead(pop.length, loader.torn(),
|
||||
evictions(logs, readState(stateOf(p)).lastRun)) }
|
||||
}
|
||||
|
||||
function writeSheet(dir: string, joined: Joined[], head: string): void {
|
||||
writeSafe(path.join(dir, 'worksheet.md'), ['# Memory feedback worksheet',
|
||||
head, DRAFT_HELP, ...joined.map(j => j.text)].join('\n\n') + '\n')
|
||||
const items = joined.map(j => j.item)
|
||||
writeSafe(path.join(dir, 'items.json'), JSON.stringify(items))
|
||||
writeSafe(path.join(dir, 'labels-draft.tsv'), skeleton(items))
|
||||
return pop.length
|
||||
}
|
||||
|
||||
/** Writes worksheet.md, items.json, labels-draft.tsv (0600); the count. */
|
||||
export function worksheet(dir: string, p: Paths, now: number): number {
|
||||
const { joined, head } = collect(p, now)
|
||||
writeSheet(dir, joined, head)
|
||||
return joined.length
|
||||
}
|
||||
|
||||
const draftOf = (dir: string) => {
|
||||
@@ -230,26 +245,45 @@ function nextState(st: State, v: Verdict, now: number) {
|
||||
return { state: { lastRun: now, promptAt, toolAt, closed }, news }
|
||||
}
|
||||
|
||||
/** Validates the draft; writes state, then labels, then FEEDBACK.md. */
|
||||
const assertFresh = (items: Item[], old: Row[]) => {
|
||||
const done = new Set(old.map(keyOf))
|
||||
if (items.some(i => done.has(keyOf(i)))) {
|
||||
throw new Error('items already applied: run worksheet again')
|
||||
}
|
||||
}
|
||||
|
||||
/** The new rows, and the metrics and verdict once they are added. */
|
||||
function projected(
|
||||
items: Item[], drafts: Draft[], old: Row[], now: number,
|
||||
) {
|
||||
const fresh = items.flatMap(i => rowsOf(i, drafts))
|
||||
const m = metricsOf([...old, ...fresh])
|
||||
return { fresh, m, v: gateVerdict(m, now) }
|
||||
}
|
||||
|
||||
/** Appends labels, then writes state, then FEEDBACK.md. */
|
||||
function commit(
|
||||
p: Paths, items: Item[], drafts: Draft[], now: number,
|
||||
): string {
|
||||
const old = readRows(labelsOf(p))
|
||||
assertFresh(items, old)
|
||||
const { fresh, m, v } = projected(items, drafts, old, now)
|
||||
const next = nextState(readState(stateOf(p)), v, now)
|
||||
const feedback = renderFeedback(m, v, now)
|
||||
if (fresh.length > 0) appendRows(labelsOf(p), fresh)
|
||||
writeState(stateOf(p), next.state)
|
||||
writeSafe(path.join(p.out, 'FEEDBACK.md'), feedback)
|
||||
return feedback + next.news.map(n => n + '\n').join('')
|
||||
}
|
||||
|
||||
/** Validates the draft; appends labels, writes state, then FEEDBACK.md
|
||||
* (a failed state write leaves the NEW line to print on the next run). */
|
||||
export function apply(dir: string, p: Paths, now: number): string {
|
||||
const items: Item[] = JSON.parse(
|
||||
fs.readFileSync(path.join(dir, 'items.json'), 'utf8'))
|
||||
const drafts = parseDraft(draftOf(dir), items)
|
||||
if (typeof drafts === 'string') throw new Error(`draft refused: ${drafts}`)
|
||||
const old = readRows(labelsOf(p))
|
||||
const done = new Set(old.map(keyOf))
|
||||
if (items.some(i => done.has(keyOf(i)))) {
|
||||
throw new Error('items already applied: run worksheet again')
|
||||
}
|
||||
const fresh = items.flatMap(i => rowsOf(i, drafts))
|
||||
const m = metricsOf([...old, ...fresh])
|
||||
const v = gateVerdict(m, now)
|
||||
const { state, news } = nextState(readState(stateOf(p)), v, now)
|
||||
const feedback = renderFeedback(m, v, now)
|
||||
writeState(stateOf(p), state)
|
||||
if (fresh.length > 0) appendRows(labelsOf(p), fresh)
|
||||
writeSafe(path.join(p.out, 'FEEDBACK.md'), feedback)
|
||||
return feedback + news.map(n => n + '\n').join('')
|
||||
return commit(p, items, drafts, now)
|
||||
}
|
||||
|
||||
/** Up to `n` entry lines of the draft, picked at random. */
|
||||
@@ -269,6 +303,9 @@ export function checkDir(dir: string | undefined): string {
|
||||
if (!fs.existsSync(dir)) return 'does not exist'
|
||||
const git = spawnSync('git', ['-C', dir, 'rev-parse',
|
||||
'--is-inside-work-tree'], { encoding: 'utf8' })
|
||||
if (git.error !== undefined || typeof git.stdout !== 'string') {
|
||||
return 'git unavailable'
|
||||
}
|
||||
return git.stdout.trim() === 'true' ? 'inside a git work tree' : ''
|
||||
}
|
||||
|
||||
|
||||
@@ -41,6 +41,9 @@ test('X1: slug, the main or one sub-agent file, torn lines counted', () => {
|
||||
expect(sessionFiles(p, '/r', 's', 'a1').map(f => path.basename(f)))
|
||||
.toEqual(['agent-a1.jsonl'])
|
||||
expect(records(main[0]!)).toEqual({ recs: [{ a: 1 }], torn: 1 })
|
||||
fs.writeFileSync(path.join(dir, 'n.jsonl'), '{"a":1}\nnull\n[1]\n7\n')
|
||||
expect(records(path.join(dir, 'n.jsonl')))
|
||||
.toEqual({ recs: [{ a: 1 }], torn: 3 })
|
||||
expect(sessionFiles(p, '/r', 'gone', 'main')).toEqual([])
|
||||
})
|
||||
|
||||
|
||||
@@ -28,13 +28,19 @@ export function sessionFiles(
|
||||
return fs.existsSync(file) ? [file] : []
|
||||
}
|
||||
|
||||
/** Parsed lines of a transcript; torn or partial lines are counted. */
|
||||
/** Parsed lines of a transcript; torn, partial or non-object lines
|
||||
* (`null`, a number, an array) are counted as torn. */
|
||||
export function records(file: string): Loaded {
|
||||
const recs: Rec[] = []
|
||||
let torn = 0
|
||||
for (const raw of fs.readFileSync(file, 'utf8').split('\n')) {
|
||||
if (raw.trim() === '') continue
|
||||
try { recs.push(JSON.parse(raw)) } catch { torn += 1 }
|
||||
try {
|
||||
const v = JSON.parse(raw)
|
||||
if (v === null || typeof v !== 'object' || Array.isArray(v)) {
|
||||
torn += 1
|
||||
} else recs.push(v)
|
||||
} catch { torn += 1 }
|
||||
}
|
||||
return { recs, torn }
|
||||
}
|
||||
@@ -90,7 +96,8 @@ function promptNear(recs: Rec[], ts: number): Prompt | undefined {
|
||||
for (const r of recs) {
|
||||
const text = composerText(r)
|
||||
const t = tsOf(r)
|
||||
if (text === undefined || Math.abs(t - ts) > NEAR_MS) continue
|
||||
if (text === undefined || !Number.isFinite(t)) continue
|
||||
if (Math.abs(t - ts) > NEAR_MS) continue
|
||||
if (best === undefined || Math.abs(t - ts) < Math.abs(best.t - ts)) {
|
||||
best = { t, text }
|
||||
}
|
||||
@@ -108,22 +115,22 @@ function nextPromptAt(recs: Rec[], ts: number): number {
|
||||
return end
|
||||
}
|
||||
|
||||
/** The [dedupe key, text] of each text block of an assistant record. */
|
||||
const textsOf = (r: Rec): [string, string][] => blocksOf(r)
|
||||
.filter(b => b.type === 'text')
|
||||
.map(b => [`${r.message?.id}:${b.text}`, String(b.text)]) // EVAL-037
|
||||
|
||||
/** Main-thread assistant text after `ts` up to the next prompt. */
|
||||
function replyAfter(recs: Rec[], ts: number): string {
|
||||
const end = nextPromptAt(recs, ts)
|
||||
const seen = new Set<string>()
|
||||
const parts: string[] = []
|
||||
for (const r of recs) {
|
||||
const t = tsOf(r)
|
||||
if (r.type !== 'assistant' || r.isSidechain === true) continue
|
||||
if (!(t > ts && t < end)) continue
|
||||
for (const b of blocksOf(r).filter(b => b.type === 'text')) {
|
||||
const key = `${r.message?.id}:${b.text}` // EVAL-037
|
||||
if (!seen.has(key)) parts.push(String(b.text))
|
||||
seen.add(key)
|
||||
}
|
||||
}
|
||||
return parts.join('\n')
|
||||
return recs
|
||||
.filter(r => r.type === 'assistant' && r.isSidechain !== true)
|
||||
.filter(r => tsOf(r) > ts && tsOf(r) < end)
|
||||
.flatMap(textsOf)
|
||||
.filter(([key]) => !seen.has(key) && !!seen.add(key))
|
||||
.map(([, text]) => text)
|
||||
.join('\n')
|
||||
}
|
||||
|
||||
export type PromptJoin = { prompt: string; injected: string; reply: string }
|
||||
@@ -178,7 +185,7 @@ export type ToolJoin = { command: string; known: string; next: boolean[] }
|
||||
|
||||
/** The failure a known line answered and the next 3 Bash results. */
|
||||
export function joinTool(recs: Rec[], rec: LogRecord): ToolJoin | undefined {
|
||||
const needle = KNOWN_HEAD + (rec.ids[0] ?? '')
|
||||
const needle = `${KNOWN_HEAD}${rec.ids[0] ?? ''} ` // BLK-003 != BLK-0031
|
||||
const att = attachmentAfter(recs, rec.at, 'tool.call', needle)
|
||||
const raw = att?.attachment?.toolUseID
|
||||
if (att === undefined || typeof raw !== 'string') return undefined
|
||||
|
||||
@@ -13,8 +13,10 @@ check() { if [ "$2" = "$3" ]; then pass=$((pass+1)); else fail=$((fail+1));
|
||||
printf 'FAIL %s: got[%s] want[%s]\n' "$1" "$2" "$3"; fi; }
|
||||
# home <name>: a fixture HOME with a store file (unless "nostore")
|
||||
home() {
|
||||
mkdir -p "$WORK/$1/.claude/plugins/store" "$WORK/$1/.claude/memory-feedback"
|
||||
[ "$1" = nostore ] || : > "$WORK/$1/.claude/plugins/store/memory_skills-dir-a.json"
|
||||
mkdir -p "$WORK/$1/.claude/plugins/store" \
|
||||
"$WORK/$1/.claude/memory-feedback"
|
||||
[ "$1" = nostore ] ||
|
||||
: > "$WORK/$1/.claude/plugins/store/memory_skills-dir-a.json"
|
||||
echo "$WORK/$1"
|
||||
}
|
||||
OPEN='{"lastRun":1,"closed":false}'
|
||||
@@ -24,24 +26,29 @@ old() {
|
||||
touch -t 202601010000 "$1/.claude/memory-feedback/state.json"
|
||||
}
|
||||
run() { HOME="$1" bash "$LIB" 2>&1; echo "rc=$?"; }
|
||||
DUE='memory feedback: last run 5+ d ago, transcripts expire at 7 → /memory-feedback'
|
||||
DUE='memory feedback: last run 5+ d ago, '
|
||||
DUE+='transcripts expire at 7 → /memory-feedback'
|
||||
|
||||
check MR1-no-store "$(run "$(home nostore)")" "rc=0"
|
||||
check MR2-never "$(run "$(home never)")" "memory feedback: never run → /memory-feedback
|
||||
NEVER='memory feedback: never run → /memory-feedback'
|
||||
check MR2-never "$(run "$(home never)")" "$NEVER
|
||||
rc=0"
|
||||
h=$(home due); old "$h" "$OPEN"
|
||||
check MR3-due "$(run "$h")" "$DUE
|
||||
rc=0"
|
||||
h=$(home quiet); old "$h" "$OPEN"
|
||||
touch -t 202512010000 "$h/.claude/plugins/store/memory_skills-dir-a.json"
|
||||
touch -t 202512010000 \
|
||||
"$h/.claude/plugins/store/memory_skills-dir-a.json"
|
||||
check MR4-store-unchanged "$(run "$h")" "rc=0"
|
||||
h=$(home fresh)
|
||||
printf '%s' "$OPEN" > "$h/.claude/memory-feedback/state.json"
|
||||
touch -t 209901010000 "$h/.claude/plugins/store/memory_skills-dir-a.json"
|
||||
touch -t 209901010000 \
|
||||
"$h/.claude/plugins/store/memory_skills-dir-a.json"
|
||||
check MR5-ran-today "$(run "$h")" "rc=0"
|
||||
h=$(home closed); old "$h" '{"lastRun":1,"closed":true}'
|
||||
check MR6-closed "$(run "$h")" "rc=0"
|
||||
h=$(home off); old "$h" "$OPEN"; printf '{"recall": false}' > "$h/.claude/memory.json"
|
||||
h=$(home off); old "$h" "$OPEN"
|
||||
printf '{"recall": false}' > "$h/.claude/memory.json"
|
||||
check MR7-recall-off "$(run "$h")" "rc=0"
|
||||
h=$(home locked); old "$h" "$OPEN"
|
||||
chmod 000 "$h/.claude/memory-feedback/state.json"
|
||||
|
||||
@@ -1734,6 +1734,22 @@ function claimPicks(st: State, key: string, picked: Entry[]): string[] {
|
||||
return taken
|
||||
}
|
||||
|
||||
/** Claims the picks of a rerank answer; the block to inject. */
|
||||
function injectPicks(
|
||||
st: State,
|
||||
key: string,
|
||||
answer: string,
|
||||
cands: Entry[],
|
||||
): Picked {
|
||||
const picked = picksOf(answer, cands)
|
||||
if (picked.length === 0) return nothing('none')
|
||||
const taken = claimPicks(st, key, picked)
|
||||
const block = recallBlock(
|
||||
picked.filter(e => taken.includes(e.id)).map(recallLine), RECALL_CAP)
|
||||
if (block === '') return nothing('refused')
|
||||
return { block, ids: taken, outcome: 'injected' }
|
||||
}
|
||||
|
||||
/** Asks haiku among the candidates and claims its picks. */
|
||||
async function pickRecall(
|
||||
$: Api,
|
||||
@@ -1748,13 +1764,7 @@ async function pickRecall(
|
||||
// a clock step back must not make the record invalid (isCount)
|
||||
const ms = Math.max(0, Math.round((await $.clock.now()) - now))
|
||||
if (answer === undefined) return { ...nothing('timeout'), ms }
|
||||
const picked = picksOf(answer, cands)
|
||||
if (picked.length === 0) return { ...nothing('none'), ms }
|
||||
const taken = claimPicks(st, key, picked)
|
||||
const block = recallBlock(
|
||||
picked.filter(e => taken.includes(e.id)).map(recallLine), RECALL_CAP)
|
||||
if (block === '') return { ...nothing('refused'), ms }
|
||||
return { block, ids: taken, outcome: 'injected', ms }
|
||||
return { ...injectPicks(st, key, answer, cands), ms }
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -33,8 +33,10 @@ Read `<dir>/worksheet.md`. Fill the third column of the skeleton
|
||||
- `relevant`: the entry bears on what the prompt asked or what the reply
|
||||
did; `irrelevant` otherwise. Judge the entry against the ask, not the
|
||||
reply's quality.
|
||||
- A proof, bench or recall-testing session: replace that item's lines by
|
||||
one `<item> - test` line.
|
||||
- A proof, bench or recall-testing session: every item of that session,
|
||||
with lines or not (none, timeout and paused items included), gets one
|
||||
`<item> - test` line in place of its lines. The header's `s:` shows
|
||||
the session, the time the order.
|
||||
Lost items have no line.
|
||||
|
||||
## STEP 3 — SPOT CHECK
|
||||
@@ -54,8 +56,9 @@ user confirms or flips each. Apply the flips to the draft (LRN-168).
|
||||
|
||||
On up to 5 prompt items whose outcome is injected or none, search the
|
||||
registries of that item's repo, whose absolute path is in the item header
|
||||
(`bash ~/.claude/lib/memory.sh -C <path> index` + entry bodies) for live entries that bore on the ask and were not
|
||||
injected. Each one: `<item> <id> missed <cause>`, cause `no-overlap` (no
|
||||
(`bash ~/.claude/lib/memory.sh -C <path> index` + entry bodies) for
|
||||
live entries that bore on the ask and were not injected. Each one:
|
||||
`<item> <id> missed <cause>`, cause `no-overlap` (no
|
||||
content word shared with the prompt) or `overlap`. A searched item with no
|
||||
miss: `<item> - checked`. Example line: `3 LRN-901 missed no-overlap`.
|
||||
|
||||
|
||||
Reference in new issue
Block a user