diff --git a/lib/memory-feedback/metrics.test.ts b/lib/memory-feedback/metrics.test.ts new file mode 100644 index 0000000..01514bb --- /dev/null +++ b/lib/memory-feedback/metrics.test.ts @@ -0,0 +1,120 @@ +import { expect, test } from 'bun:test' +import { DAY_MS, GATE_DAYS } from './thresholds.ts' +import { + actionsOf, clockStart, gateVerdict, metricsOf, p90, precision, + renderFeedback, type Metrics, +} from './metrics.ts' +import type { Label, Row } from './labels.ts' + +let clock = 1000 +/** A labels.tsv row; each call gets its own `at` unless given. */ +function row(over: Partial & { label: Label }): Row { + clock += 1 + return { at: clock, root: 'r#000000', session: 's1', channel: 'prompt', + trigger: 'fresh', outcome: 'injected', cands: 3, ms: 900, + id: 'BDR-001', cause: '', ...over } +} +/** One injected ask with entries labelled `labels`, picks 1..n. */ +function ask(labels: Label[], over: Partial = {}): Row[] { + const at = (clock += 10) + const head = row({ ...over, at, id: '', label: 'ask' }) + return [head, ...labels.map((label, i) => row({ ...over, at, label, + id: `LRN-00${i + 1}`, pick: i + 1 }))] +} + +test('M1: precision, p90 and the clock start rule', () => { + const r = ask(['relevant', 'irrelevant', 'relevant', 'relevant']) + expect(precision(r)).toBe(0.75) + expect(precision([])).toBeUndefined() + expect(p90([5, 1, 9, 3, 7, 2, 8, 4, 6, 10])).toBe(9) + expect(p90([])).toBeUndefined() + expect(clockStart(r, [1])).toBe(r[0]?.at) + expect(clockStart([], [7, 3])).toBe(3) + expect(clockStart([], [])).toBeUndefined() +}) + +test('M2: a `- test` row excludes its whole item', () => { + const t = ask(['irrelevant']) + const rows = [...ask(['relevant']), ...t, + row({ at: t[0]!.at, id: '', label: 'test' })] + const m = metricsOf(rows) + expect(m.precision).toBe(1) + expect(m.promptN).toBe(1) +}) + +test('M3: noisy and promotion keyed by repo and ID', () => { + const noisy = [1, 2, 3].flatMap(() => ask(['irrelevant'])) + expect(metricsOf(noisy).noisy).toEqual(['r#000000 LRN-001']) + expect(metricsOf(noisy.slice(0, 4)).noisy).toEqual([]) + const split = [...noisy.slice(0, 4), ...ask(['irrelevant'], + { root: 'other#111111' })] + expect(metricsOf(split).noisy).toEqual([]) + const one = [1, 2, 3].flatMap(() => ask(['relevant'])) + expect(metricsOf(one).promote).toEqual([]) + const two = [...one, ...ask(['relevant'], { session: 's2' })] + expect(metricsOf(two).promote).toEqual(['r#000000 LRN-001']) +}) + +test('M4: late share counts asks with candidates, lost ones too', () => { + const rows = [ + row({ id: '', label: 'ask', outcome: 'timeout' }), + row({ id: '', label: 'lost', outcome: 'paused' }), + row({ id: '', label: 'ask', outcome: 'none', cands: 0 }), + ...ask(['relevant']), + ] + expect(metricsOf(rows).late).toBeCloseTo(2 / 3) + expect(metricsOf(rows).lost).toBe(1) +}) + +/** Metrics with `n` prompt asks and `t` tool injections, clock at 0. */ +function m(n: number, t: number): Metrics { + const p = Array.from({ length: n }, () => ask(['relevant'])).flat() + const tl = Array.from({ length: t }, () => ask(['relevant'], + { channel: 'tool', trigger: 'tool', cands: undefined })).flat() + return { ...metricsOf([...p, ...tl]), start: 0 } +} +const CAP = GATE_DAYS * DAY_MS + +test('M5: the prompt gate at 20 asks or at the cap', () => { + expect(gateVerdict(m(19, 0), 1).prompt).toBe('open') + expect(gateVerdict(m(20, 0), 1).prompt).toBe('reached') + expect(gateVerdict(m(3, 0), CAP).prompt).toBe('reached') + const none = { ...metricsOf([]), start: undefined } + expect(gateVerdict(none, CAP * 9)) + .toEqual({ prompt: 'open', tool: 'open' }) +}) + +test('M6: the tool gate extends once by time, then closes', () => { + expect(gateVerdict(m(0, 10), 1).tool).toBe('reached') + expect(gateVerdict(m(0, 9), CAP - 1).tool).toBe('open') + expect(gateVerdict(m(0, 9), CAP).tool).toBe('extended') + expect(gateVerdict(m(0, 9), 2 * CAP - 1).tool).toBe('extended') + expect(gateVerdict(m(0, 9), 2 * CAP).tool).toBe('insufficient') +}) + +test('M7: actions only once a gate is reached', () => { + const idle = Array.from({ length: 20 }, (_, i) => + ask([i % 3 === 0 ? 'relevant' : 'irrelevant'], { trigger: 'idle' })) + .flat() // 7/20 relevant: under 64 % overall and under 50 % for idle + const mm = { ...metricsOf(idle), start: 0 } + expect(actionsOf(mm, gateVerdict(mm, 1))) + .toContain('prompt precision under 64 %: tighten or drop the idle trigger') + const open = { ...mm, promptN: 2 } + expect(actionsOf(open, gateVerdict(open, 1))).toEqual([]) +}) + +test('M8: rerank latency action', () => { + const slow = Array.from({ length: 20 }, () => + ask(['relevant'], { ms: 2400 })).flat() + const mm = { ...metricsOf(slow), start: 0 } + expect(actionsOf(mm, gateVerdict(mm, 1))) + .toContain('rerank p90 under 2500 ms: lower RERANK_MS') +}) + +test('M9: FEEDBACK.md counts, no repo tag without a noisy entry', () => { + const rows = ask(['relevant'], { root: 'client#abcdef' }) + const mm = metricsOf(rows) + const text = renderFeedback(mm, gateVerdict(mm, 1), 0) + expect(text).not.toContain('client#abcdef') + expect(text).toContain('Prompt gate: open (1/20)') +}) diff --git a/lib/memory-feedback/metrics.ts b/lib/memory-feedback/metrics.ts new file mode 100644 index 0000000..3552df4 --- /dev/null +++ b/lib/memory-feedback/metrics.ts @@ -0,0 +1,210 @@ +// lib/memory-feedback/metrics.ts: phase 4d metrics, gate verdicts and the +// FEEDBACK.md text, computed from labels.tsv rows only (transcripts expire +// at 7 days, the rows stay). Spec §4. +import { + DAY_MS, GATE_DAYS, PROMPT_GATE_N, T, TOOL_GATE_N, +} from './thresholds.ts' +import { keyOf, type Row } from './labels.ts' + +export type Gate = 'open' | 'reached' +export type ToolGate = Gate | 'extended' | 'insufficient' +export type Verdict = { prompt: Gate; tool: ToolGate } +type Rate = number | undefined +export type Metrics = { + start?: number; promptN: number; toolN: number; lost: number + precision: Rate; byTrigger: Record; thirdPick: Rate + late: Rate; msP90: Rate; tool: Rate; missShare: Rate; sampled: number + noisy: string[]; promote: string[] +} + +const share = (k: number, n: number): Rate => n === 0 ? undefined : k / n +const isEntry = (r: Row) => + r.label === 'relevant' || r.label === 'irrelevant' +const under = (v: Rate, t: number) => v !== undefined && v < t +const pct = (v: Rate) => v === undefined ? 'n/a' : `${Math.round(v * 100)} %` +const day = (ms: number) => new Date(ms).toISOString().slice(0, 10) +const entryKey = (r: Row) => `${r.root} ${r.id}` + +/** The gate clock start: the first labelled row with cands, else the + * first log record with cands (one rule for reader and metrics). */ +export function clockStart(rows: Row[], logAts: number[]): Rate { + const labelled = rows.filter(r => r.cands !== undefined).map(r => r.at) + const ats = labelled.length > 0 ? labelled : logAts + return ats.length === 0 ? undefined : Math.min(...ats) +} + +/** Rows of items with no `test` row (LRN-241). */ +function liveRows(rows: Row[]): Row[] { + const test = new Set(rows.filter(r => r.label === 'test').map(keyOf)) + return rows.filter(r => !test.has(keyOf(r))) +} + +/** relevant / (relevant + irrelevant); undefined with no entry. */ +export function precision(rows: Row[]): Rate { + const e = rows.filter(isEntry) + return share(e.filter(r => r.label === 'relevant').length, e.length) +} + +/** The 90th percentile, nearest rank; undefined with no value. */ +export function p90(values: number[]): Rate { + if (values.length === 0) return undefined + const s = [...values].sort((a, b) => a - b) + return s[Math.ceil(0.9 * s.length) - 1] +} + +function byTrigger(entries: Row[]): Record { + const out: Record = {} + for (const k of ['fresh', 'idle', 'keyword']) { + out[k] = precision(entries.filter(r => r.trigger === k)) + } + return out +} + +/** timeout + paused among asks (lost ones too) that had candidates. */ +function lateShare(asks: Row[]): Rate { + const had = asks.filter(r => (r.cands ?? 0) > 0) + const late = had.filter(r => + r.outcome === 'timeout' || r.outcome === 'paused') + return share(late.length, had.length) +} + +/** Share of sampled asks (a missed or checked row) with a miss. */ +function misses(rows: Row[]): { rate: Rate; sampled: number } { + const sampled = new Set(rows + .filter(r => r.label === 'missed' || r.label === 'checked').map(keyOf)) + const missed = new Set(rows.filter(r => r.label === 'missed').map(keyOf)) + return { rate: share(missed.size, sampled.size), sampled: sampled.size } +} + +/** ` ` injected noisyHits times or more, never relevant. */ +function noisy(entries: Row[]): string[] { + return [...new Set(entries.map(entryKey))].filter(k => { + const mine = entries.filter(r => entryKey(r) === k) + return mine.length >= T.noisyHits && + mine.every(r => r.label !== 'relevant') + }).sort() +} + +/** ` ` relevant promoteHits times in promoteSessions. */ +function promote(entries: Row[]): string[] { + const rel = entries.filter(r => r.label === 'relevant') + return [...new Set(rel.map(entryKey))].filter(k => { + const mine = rel.filter(r => entryKey(r) === k) + return mine.length >= T.promoteHits && + new Set(mine.map(r => r.session)).size >= T.promoteSessions + }).sort() +} + +/** Distinct labelled items of a channel. */ +const itemsOf = (entries: Row[], ch: string) => + new Set(entries.filter(r => r.channel === ch).map(keyOf)).size + +/** Every 4d signal, from labels.tsv rows. */ +export function metricsOf(all: Row[]): Metrics { + const rows = liveRows(all) + const entries = rows.filter(isEntry) + const prompt = entries.filter(r => r.channel === 'prompt') + const asks = rows.filter(r => r.channel === 'prompt' && + (r.label === 'ask' || r.label === 'lost')) + const miss = misses(rows) + return { + start: clockStart(all, []), + promptN: itemsOf(entries, 'prompt'), toolN: itemsOf(entries, 'tool'), + lost: rows.filter(r => r.label === 'lost').length, + precision: precision(prompt), byTrigger: byTrigger(prompt), + thirdPick: precision(prompt.filter(r => r.pick === 3)), + late: lateShare(asks), msP90: p90(asks.flatMap(r => r.ms ?? [])), + tool: precision(entries.filter(r => r.channel === 'tool')), + missShare: miss.rate, sampled: miss.sampled, + noisy: noisy(entries), promote: promote(entries), + } +} + +/** The tool gate by count, else by time: one extension, then closed. */ +function toolGate(n: number, now: number, cap: number): ToolGate { + if (n >= TOOL_GATE_N) return 'reached' + if (now < cap) return 'open' + return now < cap + GATE_DAYS * DAY_MS ? 'extended' : 'insufficient' +} + +/** Each channel's gate; the prompt gate never waits for the tool gate. */ +export function gateVerdict(m: Metrics, now: number): Verdict { + if (m.start === undefined) return { prompt: 'open', tool: 'open' } + const cap = m.start + GATE_DAYS * DAY_MS + const prompt: Gate = + m.promptN >= PROMPT_GATE_N || now >= cap ? 'reached' : 'open' + return { prompt, tool: toolGate(m.toolN, now, cap) } +} + +/** The worst trigger under the floor; ties go idle, keyword, fresh. */ +function worstTrigger(by: Record): string | undefined { + const low = ['idle', 'keyword', 'fresh'] + .filter(k => under(by[k], T.triggerFloor)) + low.sort((a, b) => (by[a] ?? 0) - (by[b] ?? 0)) + return low[0] +} + +function promptActions(m: Metrics): string[] { + const out: string[] = [] + const worst = worstTrigger(m.byTrigger) + if (under(m.precision, T.promptKeep) && worst !== undefined) { + out.push(`prompt precision under ${pct(T.promptKeep)}: ` + + `tighten or drop the ${worst} trigger`) + } + if (under(m.thirdPick, T.thirdPick)) { + out.push(`3rd pick under ${pct(T.thirdPick)}: cap recall at 2 entries`) + } + if ((m.late ?? 0) > T.lateShare) { + out.push(`timeout + paused over ${pct(T.lateShare)}: ` + + 'revisit RERANK_MS or skip the rerank on short lists') + } + if (under(m.msP90, T.msP90)) { + out.push(`rerank p90 under ${T.msP90} ms: lower RERANK_MS`) + } + if ((m.missShare ?? 0) >= T.missShare) { + out.push('misses in 1 ask in 3 or more: diagnose by cause') + } + return out +} + +/** The threshold actions, once at least one gate is reached. */ +export function actionsOf(m: Metrics, v: Verdict): string[] { + if (v.prompt !== 'reached' && v.tool !== 'reached') return [] + const out = v.prompt === 'reached' ? promptActions(m) : [] + if (v.tool === 'reached' && under(m.tool, T.toolKeep)) { + out.push(`tool precision under ${pct(T.toolKeep)}: ` + + 'require 2 matched 4-grams, or exclude the entry') + } + out.push(...m.noisy.map(k => `noisy: /prune-memory rewrites ${k}`)) + out.push(...m.promote.map(k => `promotion candidate (BDR-100): ${k}`)) + return out +} + +function toolLine(m: Metrics, v: Verdict): string { + return v.tool === 'insufficient' + ? `Tool gate: insufficient sample n=${m.toolN}, closed` + : `Tool gate: ${v.tool} (${m.toolN}/${TOOL_GATE_N})` +} + +/** FEEDBACK.md: counts and shares; repo tags only in the last lists. */ +export function renderFeedback(m: Metrics, v: Verdict, now: number) { + const acts = actionsOf(m, v) + return [ + `# Memory feedback (${day(now)})`, '', + `Gate clock start: ${m.start === undefined ? 'not started' : day(m.start)}`, + `Prompt gate: ${v.prompt} (${m.promptN}/${PROMPT_GATE_N})`, + toolLine(m, v), `Lost records: ${m.lost}`, '', + '| Signal | Value |', '|---|---|', + `| prompt precision | ${pct(m.precision)} |`, + ...Object.entries(m.byTrigger).map(([k, x]) => + `| precision, ${k} | ${pct(x)} |`), + `| 3rd pick precision | ${pct(m.thirdPick)} |`, + `| timeout + paused | ${pct(m.late)} |`, + `| rerank ms p90 | ${m.msP90 ?? 'n/a'} |`, + `| tool precision | ${pct(m.tool)} |`, + `| asks with a miss | ${pct(m.missShare)} of ${m.sampled} |`, '', + `Noisy: ${m.noisy.join(', ') || 'none'}`, + `Promotion candidates: ${m.promote.join(', ') || 'none'}`, '', + 'Actions:', ...(acts.length ? acts.map(a => `- ${a}`) : ['- none']), + ].join('\n') + '\n' +} diff --git a/lib/memory-feedback/thresholds.ts b/lib/memory-feedback/thresholds.ts new file mode 100644 index 0000000..c7456f7 --- /dev/null +++ b/lib/memory-feedback/thresholds.ts @@ -0,0 +1,20 @@ +// lib/memory-feedback/thresholds.ts: phase 4d gate numbers and threshold +// actions, fixed before the data (BDR-100: one file; the reader and a +// future /prune-memory step read it). Spec §4. +export const DAY_MS = 86400000 +export const GATE_DAYS = 35 // the 5-week cap from the gate clock start +export const PROMPT_GATE_N = 20 // labelled injected prompt asks +export const TOOL_GATE_N = 10 // labelled tool injections + +export const T = { + promptKeep: 0.64, // relevant / labelled entries (4c baseline 14/22) + triggerFloor: 0.5, // worst trigger kind under this gets tightened + thirdPick: 0.4, // 3rd pick precision under this caps picks at 2 + lateShare: 0.25, // timeout + paused among asks with candidates + msP90: 2500, // rerank p90 under this lowers RERANK_MS + toolKeep: 0.5, // tool precision under this narrows the match + missShare: 1 / 3, // sampled asks with a miss, at or over this + noisyHits: 3, // injected this often and never relevant: noisy + promoteHits: 3, // relevant this often... + promoteSessions: 2, // ...in this many sessions: promotion candidate +} as const