fix(memory-feedback): post-fix precision split, 3rd pick retired, bounded transcript cache, strict labels

Precision, triggers and the miss share count only rows after the gate-1
fix merge (d3d5e6c, 2026-11-12T19:53:59Z); their actions wait for 20
post-fix asks, a one-off NEW line marks the sample and the reminder
keeps prompting until then. The 3rd-pick signal is gone. The reader
holds one parsed transcript at a time. labels.tsv rows and state.json
are validated on read. Exact key sets of log records pinned.
This commit is contained in:
bchanot committed 2026-11-13 19:47:37 +01:00
1 parent e132488aef
commit 9f9412aaba
11 files changed
+478 -67

No files matched your search

+24 -17
View File
@@ -2,7 +2,7 @@
// FEEDBACK.md text, computed from labels.tsv rows only (transcripts expire
// at 7 days, the rows stay). Spec §4.
import {
DAY_MS, GATE_DAYS, PROMPT_GATE_N, T, TOOL_GATE_N,
DAY_MS, GATE1_FIX_AT, GATE_DAYS, PROMPT_GATE_N, T, TOOL_GATE_N,
} from './thresholds.ts'
import { keyOf, type Row } from './labels.ts'
@@ -13,10 +13,12 @@ type Rate = number | undefined
type Sized = { rate: Rate; n: number }
export type Metrics = {
start?: number; promptN: number; toolN: number; lost: number
precision: Rate; byTrigger: Record<string, Sized>; thirdPick: Rate
precision: Rate; byTrigger: Record<string, Sized>
late: Rate; msP90: Rate; tool: Rate; missShare: Rate; sampled: number
noisy: string[]; promote: string[]
n: { prompt: number; third: number; tool: number } // entries
n: { prompt: number; tool: number } // entries
postAsks: number // post-fix prompt asks labelled: the floor of the actions
preFix: number // pre-fix prompt entries left out of the prompt signals
}
const share = (k: number, n: number): Rate => n === 0 ? undefined : k / n
@@ -66,7 +68,6 @@ function byTrigger(entries: Row[]): Record<string, Sized> {
/** Entry counts behind the three channel precisions. */
const countsOf = (entries: Row[]) => ({
prompt: entries.filter(r => r.channel === 'prompt').length,
third: entries.filter(r => r.channel === 'prompt' && r.pick === 3).length,
tool: entries.filter(r => r.channel === 'tool').length,
})
@@ -109,25 +110,29 @@ function promote(entries: Row[]): string[] {
const itemsOf = (entries: Row[], ch: string) =>
new Set(entries.filter(r => r.channel === ch).map(keyOf)).size
/** Every 4d signal, from labels.tsv rows. */
const postFix = (r: Row) => r.at >= GATE1_FIX_AT
/** Every 4d signal, from labels.tsv rows. Prompt precision, triggers and
* the miss share read post-fix rows (BDR-146); the rest read all rows. */
export function metricsOf(all: Row[]): Metrics {
const rows = liveRows(all)
const entries = rows.filter(isEntry)
const prompt = entries.filter(r => r.channel === 'prompt')
const recent = entries.filter(r => r.channel === 'prompt' && postFix(r))
const asks = rows.filter(r => r.channel === 'prompt' &&
(r.label === 'ask' || r.label === 'lost'))
const miss = misses(rows)
const miss = misses(rows.filter(postFix))
return {
start: clockStart(all, []),
promptN: itemsOf(entries, 'prompt'), toolN: itemsOf(entries, 'tool'),
lost: rows.filter(r => r.label === 'lost').length,
precision: precision(prompt), byTrigger: byTrigger(prompt),
thirdPick: precision(prompt.filter(r => r.pick === 3)),
precision: precision(recent), byTrigger: byTrigger(recent),
late: lateShare(asks), msP90: p90(asks.flatMap(r => r.ms ?? [])),
tool: precision(entries.filter(r => r.channel === 'tool')),
missShare: miss.rate, sampled: miss.sampled,
noisy: noisy(entries), promote: promote(entries),
n: countsOf(entries),
n: { ...countsOf(entries), prompt: recent.length },
postAsks: itemsOf(recent, 'prompt'),
preFix: countsOf(entries).prompt - recent.length,
}
}
@@ -158,13 +163,11 @@ function worstTrigger(by: Record<string, Sized>): string | undefined {
function promptActions(m: Metrics): string[] {
const out: string[] = []
const worst = worstTrigger(m.byTrigger)
if (under(m.precision, T.promptKeep) && worst !== undefined) {
const floor = m.postAsks >= PROMPT_GATE_N
if (floor && under(m.precision, T.promptKeep) && worst !== undefined) {
out.push(`prompt precision under ${pct(T.promptKeep)}: ` +
`tighten or drop the ${worst} trigger`)
}
if (under(m.thirdPick, T.thirdPick)) {
out.push(`3rd pick under ${pct(T.thirdPick)}: cap recall at 2 entries`)
}
if ((m.late ?? 0) > T.lateShare) {
out.push(`timeout + paused over ${pct(T.lateShare)}: ` +
'revisit RERANK_MS or skip the rerank on short lists')
@@ -172,7 +175,7 @@ function promptActions(m: Metrics): string[] {
if (under(m.msP90, T.msP90)) {
out.push(`rerank p90 under ${T.msP90} ms: lower RERANK_MS`)
}
if ((m.missShare ?? 0) >= T.missShare) {
if (floor && (m.missShare ?? 0) >= T.missShare) {
out.push('misses in 1 ask in 3 or more: diagnose by cause')
}
return out
@@ -197,6 +200,11 @@ function toolLine(m: Metrics, v: Verdict): string {
: `Tool gate: ${v.tool} (${m.toolN}/${TOOL_GATE_N})`
}
const postFixLine = (m: Metrics) =>
`Post-fix prompt asks: ${m.postAsks}/${PROMPT_GATE_N} (since ` +
`${new Date(GATE1_FIX_AT).toISOString()}), pre-fix entries excluded: ` +
`${m.preFix}`
/** FEEDBACK.md: counts and shares; repo tags only in the last lists. */
export function renderFeedback(m: Metrics, v: Verdict, now: number) {
const acts = actionsOf(m, v)
@@ -204,12 +212,11 @@ export function renderFeedback(m: Metrics, v: Verdict, now: number) {
`# Memory feedback (${day(now)})`, '',
`Gate clock start: ${m.start === undefined ? 'not started' : day(m.start)}`,
`Prompt gate: ${v.prompt} (${m.promptN}/${PROMPT_GATE_N})`,
toolLine(m, v), `Lost records: ${m.lost}`, '',
toolLine(m, v), `Lost records: ${m.lost}`, postFixLine(m), '',
'| Signal | Value |', '|---|---|',
`| prompt precision | ${pct(m.precision)} (n=${m.n.prompt}) |`,
...Object.entries(m.byTrigger).map(([k, x]) =>
`| precision, ${k} | ${pct(x.rate)} (n=${x.n}) |`),
`| 3rd pick precision | ${pct(m.thirdPick)} (n=${m.n.third}) |`,
`| timeout + paused | ${pct(m.late)} |`,
`| rerank ms p90 | ${m.msP90 ?? 'n/a'} |`,
`| tool precision | ${pct(m.tool)} (n=${m.n.tool}) |`,