From f624ce101c18c2c323656840a14952f0eb2165c4 Mon Sep 17 00:00:00 2001 From: bchanot Date: Sun, 8 Nov 2026 17:25:07 +0100 Subject: [PATCH] feat(memory-feedback): append-only labels, draft and state --- lib/memory-feedback/labels.test.ts | 110 ++++++++++++++++ lib/memory-feedback/labels.ts | 197 +++++++++++++++++++++++++++++ lib/tests/memory-feedback.test.sh | 9 ++ 3 files changed, 316 insertions(+) create mode 100644 lib/memory-feedback/labels.test.ts create mode 100644 lib/memory-feedback/labels.ts create mode 100755 lib/tests/memory-feedback.test.sh diff --git a/lib/memory-feedback/labels.test.ts b/lib/memory-feedback/labels.test.ts new file mode 100644 index 0000000..3d9595b --- /dev/null +++ b/lib/memory-feedback/labels.test.ts @@ -0,0 +1,110 @@ +import { afterAll, expect, test } from 'bun:test' +import * as fs from 'node:fs' +import * as os from 'node:os' +import * as path from 'node:path' +import { + appendRows, fromLine, parseDraft, readRows, readState, rowsOf, skeleton, + tagOf, toLine, writeState, type Item, type Row, +} from './labels.ts' + +const dirs: string[] = [] +const tmp = () => { + const d = fs.mkdtempSync(path.join(os.tmpdir(), 'mf-')) + dirs.push(d) + return d +} +afterAll(() => dirs.forEach(d => fs.rmSync(d, { recursive: true }))) + +const item = (n: number, over: Partial = {}): Item => ({ + n, root: 'r#000000', session: `s${n}`, at: 1000 + n, channel: 'prompt', + trigger: 'fresh', outcome: 'injected', cands: 4, ms: 900, + ids: ['BDR-001', 'LRN-002'], cited: ['BDR-001'], lost: false, ...over, +}) +const ITEMS = [item(1), item(2, { ids: [], outcome: 'none' })] +const OK = '1\tBDR-001\trelevant\n1\tLRN-002\tirrelevant' + +test('L1: a row survives the TSV round trip', () => { + const r: Row = { at: 5, root: 'r#000000', session: 's', + channel: 'tool', trigger: 'tool', outcome: 'injected', + cands: undefined, ms: undefined, pick: 1, id: 'BLK-003', + label: 'relevant', cause: '' } + expect(fromLine(toLine(r))).toEqual(r) + expect(fromLine('at\troot')).toBeUndefined() +}) + +test('L2: a complete draft parses', () => { + const d = parseDraft([OK, '2\tBDR-009\tmissed\tno-overlap', + '1\t-\tchecked', '# note'].join('\n'), ITEMS) + expect(Array.isArray(d) && d.length).toBe(4) +}) + +test('L3: every bad draft is refused with a reason', () => { + const bad = [ + '1\tBDR-001\tmaybe\n1\tLRN-002\trelevant', // unknown label + `${OK}\n9\tBDR-001\trelevant`, // unknown item + `${OK}\n1\tBLK-777\trelevant`, // not injected there + '1\tBDR-001\trelevant', // LRN-002 unlabelled + `${OK}\n1\tBDR-001\trelevant`, // labelled twice + `${OK}\n2\tBDR-001\tmissed\tbad`, // unknown cause + `${OK}\n1\tLRN-002\tmissed\toverlap`, // missed but injected + '1\tBDR-001\ttest\n1\tLRN-002\trelevant', // per-id test + '1\tBDR-001\t\n1\tLRN-002\trelevant', // blank skeleton label + ] + for (const text of bad) { + expect(typeof parseDraft(text, ITEMS)).toBe('string') + } +}) + +test('L4: `- test` marks a whole item, no entry label needed', () => { + expect(Array.isArray(parseDraft('1\t-\ttest', ITEMS))).toBe(true) +}) + +test('L5: a lost item takes no label', () => { + const lost = [item(1, { lost: true })] + expect(typeof parseDraft('1\tBDR-001\trelevant', lost)).toBe('string') + expect(parseDraft('', lost)).toEqual([]) +}) + +test('L6: rowsOf writes an ask row then the entry rows', () => { + const d = parseDraft(OK, [item(1)]) + if (typeof d === 'string') throw new Error(d) + expect(rowsOf(item(1), d).map(r => [r.label, r.id, r.pick])).toEqual([ + ['ask', '', undefined], ['relevant', 'BDR-001', 1], + ['irrelevant', 'LRN-002', 2], + ]) + expect(rowsOf(item(3, { lost: true }), [])[0]?.label).toBe('lost') +}) + +test('L7: append-only labels, a backup, a loud damaged line', () => { + const dir = path.join(tmp(), 'mf') + const file = path.join(dir, 'labels.tsv') + appendRows(file, rowsOf(item(1), [])) + appendRows(file, rowsOf(item(2), [])) + expect(fs.statSync(dir).mode & 0o777).toBe(0o700) + expect(fs.statSync(file).mode & 0o777).toBe(0o600) + expect(readRows(file)).toHaveLength(2) + expect(readRows(`${file}.bak`)).toHaveLength(1) + fs.appendFileSync(file, 'broken line\n') + expect(() => readRows(file)).toThrow('labels.tsv.bak') +}) + +test('L8: state defaults, round trip, a corrupt file throws', () => { + const f = path.join(tmp(), 'state.json') + expect(readState(f)).toEqual({ lastRun: 0, closed: false }) + writeState(f, { lastRun: 9, promptAt: 9, closed: false }) + expect(readState(f)).toEqual({ lastRun: 9, promptAt: 9, closed: false }) + fs.writeFileSync(f, '{oops') + expect(() => readState(f)).toThrow('state.json') +}) + +test('L9: repo tags keep same-named repos apart', () => { + expect(tagOf('/a/x')).toMatch(/^x#[0-9a-f]{6}$/) + expect(tagOf('/a/x')).toBe(tagOf('/a/x')) + expect(tagOf('/a/x')).not.toBe(tagOf('/b/x')) +}) + +test('L10: the skeleton draft lists each injected id once', () => { + const lines = skeleton([item(1), item(2, { ids: [] }), + item(3, { lost: true })]).split('\n').filter(l => !l.startsWith('#')) + expect(lines.filter(l => l !== '')).toEqual(['1\tBDR-001\t', '1\tLRN-002\t']) +}) diff --git a/lib/memory-feedback/labels.ts b/lib/memory-feedback/labels.ts new file mode 100644 index 0000000..27c665c --- /dev/null +++ b/lib/memory-feedback/labels.ts @@ -0,0 +1,197 @@ +// lib/memory-feedback/labels.ts: labels.tsv rows (append-only), the +// labeller's draft and state.json (phase 4d). IDs, enums and repo tags +// only, never text (LRN-124). +import { createHash } from 'node:crypto' +import * as fs from 'node:fs' +import * as path from 'node:path' + +export type Channel = 'prompt' | 'tool' +export type Label = 'relevant' | 'irrelevant' | 'test' | 'missed' | + 'checked' | 'ask' | 'lost' +const DRAFT_LABELS = ['relevant', 'irrelevant', 'test', 'missed', 'checked'] +const ENTRY_LABELS = ['relevant', 'irrelevant'] +const CAUSES = ['no-overlap', 'overlap'] +const ID_RE = /^(BDR|LRN|BLK)-\d{3,}$/ + +/** One worksheet item: a log record and what the join found. */ +export type Item = { + n: number; root: string; session: string; at: number; channel: Channel + trigger: string; outcome: string; cands?: number; ms?: number + ids: string[]; cited: string[]; lost: boolean +} +export type Row = { + at: number; root: string; session: string; channel: Channel + trigger: string; outcome: string; cands?: number; ms?: number + pick?: number; id: string; label: Label; cause: string +} +export type Draft = { item: number; id: string; label: Label; cause: string } +/** promptAt/toolAt: when each gate verdict was first reached. */ +export type State = { + lastRun: number; promptAt?: number; toolAt?: number; closed: boolean +} + +const COLS = ['at', 'root', 'session', 'channel', 'trigger', 'outcome', + 'cands', 'ms', 'pick', 'id', 'label', 'cause'] +const HEADER = COLS.join('\t') + '\n' + +/** A repo tag: basename + 6 hex of the path (same names stay apart). */ +export const tagOf = (root: string) => `${path.basename(root)}#` + + createHash('sha1').update(root).digest('hex').slice(0, 6) + +/** One record's identity: session ids are unique across repos. */ +export const keyOf = (r: { session: string; at: number; channel: string }) => + `${r.session}|${r.at}|${r.channel}` + +const num = (v: number | undefined) => v === undefined ? '' : String(v) +const opt = (s: string | undefined) => + s === undefined || s === '' ? undefined : Number(s) + +export function toLine(r: Row): string { + return [r.at, r.root, r.session, r.channel, r.trigger, r.outcome, + num(r.cands), num(r.ms), num(r.pick), r.id, r.label, r.cause] + .join('\t') +} + +/** A labels.tsv line as a row; the header or a short line: undefined. */ +export function fromLine(line: string): Row | undefined { + const f = line.split('\t') + if (f.length !== COLS.length || f[0] === 'at') return undefined + return { + at: Number(f[0]), root: f[1] ?? '', session: f[2] ?? '', + channel: f[3] as Channel, trigger: f[4] ?? '', outcome: f[5] ?? '', + cands: opt(f[6]), ms: opt(f[7]), pick: opt(f[8]), id: f[9] ?? '', + label: f[10] as Label, cause: f[11] ?? '', + } +} + +const mkdir = (file: string) => + fs.mkdirSync(path.dirname(file), { recursive: true, mode: 0o700 }) + +/** Writes through a 0600 temp file and a rename; the dir is made 0700. */ +export function writeSafe(file: string, text: string): void { + mkdir(file) + const tmp = `${file}.${process.pid}.tmp` + fs.writeFileSync(tmp, text, { mode: 0o600 }) + fs.renameSync(tmp, file) +} + +/** Every row; a damaged line fails loudly, it is never dropped. */ +export function readRows(file: string): Row[] { + if (!fs.existsSync(file)) return [] + const lines = fs.readFileSync(file, 'utf8').split('\n') + return lines.flatMap((line, i) => { + if (line === '' || line === HEADER.trimEnd()) return [] + const r = fromLine(line) + if (r !== undefined) return [r] + throw new Error(`labels.tsv line ${i + 1} unreadable: fix it by hand ` + + 'or restore labels.tsv.bak') + }) +} + +/** Appends rows; the previous file is copied to labels.tsv.bak first. */ +export function appendRows(file: string, rows: Row[]): void { + mkdir(file) + if (fs.existsSync(file)) fs.copyFileSync(file, `${file}.bak`) + else fs.writeFileSync(file, HEADER, { mode: 0o600 }) + fs.appendFileSync(file, rows.map(r => toLine(r) + '\n').join('')) +} + +/** state.json; absent: a first run; unparsable: a loud error. */ +export function readState(file: string): State { + if (!fs.existsSync(file)) return { lastRun: 0, closed: false } + try { + const s = JSON.parse(fs.readFileSync(file, 'utf8')) + const at = (v: unknown) => typeof v === 'number' ? v : undefined + return { lastRun: Number(s.lastRun) || 0, promptAt: at(s.promptAt), + toolAt: at(s.toolAt), closed: s.closed === true } + } catch { + throw new Error(`state.json unreadable (${file}): fix or delete it`) + } +} + +export const writeState = (file: string, s: State) => + writeSafe(file, JSON.stringify(s) + '\n') + +/** One draft line `itemidlabel[cause]`, or why not. */ +function parseLine(line: string): Draft | string { + const f = line.split('\t') + if (f.length < 3 || f.length > 4) return `bad field count: ${line}` + const item = Number(f[0]) + const label = f[2] ?? '' + if (!Number.isInteger(item)) return `bad item: ${f[0]}` + if (!DRAFT_LABELS.includes(label)) return `bad label: '${label}'` + return { item, id: f[1] ?? '', label: label as Label, cause: f[3] ?? '' } +} + +/** Why `d` does not fit item `it`; '' when it fits. */ +function fitError(d: Draft, it: Item | undefined): string { + if (it === undefined || it.lost) return `unknown or lost item ${d.item}` + const plain = d.cause === '' + if (d.id === '-') { + return plain && (d.label === 'checked' || d.label === 'test') + ? '' : `item ${d.item}: only checked or test take id -` + } + if (ENTRY_LABELS.includes(d.label)) { + return plain && it.ids.includes(d.id) + ? '' : `item ${d.item}: ${d.id} was not injected there` + } + const ok = d.label === 'missed' && it.channel === 'prompt' && + ID_RE.test(d.id) && !it.ids.includes(d.id) && CAUSES.includes(d.cause) + return ok ? '' : `item ${d.item}: bad ${d.label} row ${d.id} ${d.cause}` +} + +/** Every injected entry of every joined item labelled exactly once. */ +function completeError(drafts: Draft[], items: Item[]): string { + const whole = new Set(drafts.filter(d => d.id === '-' && + d.label === 'test').map(d => d.item)) + const got = drafts.filter(d => ENTRY_LABELS.includes(d.label)) + .map(d => `${d.item}:${d.id}`) + if (new Set(got).size !== got.length) return 'an entry is labelled twice' + for (const it of items) { + if (it.lost || whole.has(it.n)) continue + const miss = it.ids.find(id => !got.includes(`${it.n}:${id}`)) + if (miss !== undefined) return `item ${it.n}: ${miss} unlabelled` + } + return '' +} + +/** The validated draft, or the first problem found. */ +export function parseDraft(text: string, items: Item[]): Draft[] | string { + const byN = new Map(items.map(i => [i.n, i])) + const drafts: Draft[] = [] + for (const line of text.split('\n')) { + if (line.trim() === '' || line.startsWith('#')) continue + const d = parseLine(line) + if (typeof d === 'string') return d + const err = fitError(d, byN.get(d.item)) + if (err !== '') return err + drafts.push(d) + } + const err = completeError(drafts, items) + return err === '' ? drafts : err +} + +const base = (it: Item) => ({ + at: it.at, root: it.root, session: it.session, channel: it.channel, + trigger: it.trigger, outcome: it.outcome, cands: it.cands, ms: it.ms, +}) + +/** The labels.tsv rows of one item: its ask row, then its draft rows. */ +export function rowsOf(it: Item, drafts: Draft[]): Row[] { + const head: Row = { ...base(it), id: '', + label: it.lost ? 'lost' : 'ask', cause: '' } + const mine = drafts.filter(d => d.item === it.n).map((d): Row => ({ + ...base(it), id: d.id === '-' ? '' : d.id, + pick: it.ids.includes(d.id) ? it.ids.indexOf(d.id) + 1 : undefined, + label: d.label, cause: d.cause, + })) + return [head, ...mine] +} + +/** A draft to fill in: one ` ` line per injected id. */ +export function skeleton(items: Item[]): string { + const lines = items.filter(i => !i.lost) + .flatMap(i => i.ids.map(id => `${i.n}\t${id}\t`)) + return ['# fill the 3rd column: relevant or irrelevant', ...lines] + .join('\n') + '\n' +} diff --git a/lib/tests/memory-feedback.test.sh b/lib/tests/memory-feedback.test.sh new file mode 100755 index 0000000..fb87843 --- /dev/null +++ b/lib/tests/memory-feedback.test.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +# lib/tests/memory-feedback.test.sh — the bun unit suite of +# lib/memory-feedback (phase 4d reader, labels, metrics, transcripts). +# Fails when bun is absent: it never passes vacuously. +set -u +ROOT="$(cd "$(dirname "$0")/../.." && pwd)" +command -v bun >/dev/null 2>&1 || { echo "FAIL bun not on PATH"; exit 1; } +cd "$ROOT/lib/memory-feedback" || exit 1 +bun test 2>&1