From 047806145753ce90d6e461f6c03dd7830fff9ee2 Mon Sep 17 00:00:00 2001 From: bchanot Date: Wed, 4 Nov 2026 12:14:58 +0100 Subject: [PATCH] fix(memory): bound index regexes, skip unreadable registries and malformed entries --- mods/memory/hooks/register.test.ts | 74 ++++++++++++++++++++++++++++-- mods/memory/hooks/register.ts | 68 ++++++++++++++++++--------- 2 files changed, 116 insertions(+), 26 deletions(-) diff --git a/mods/memory/hooks/register.test.ts b/mods/memory/hooks/register.test.ts index 3768c2f..be4a8b3 100644 --- a/mods/memory/hooks/register.test.ts +++ b/mods/memory/hooks/register.test.ts @@ -3,7 +3,7 @@ import type { Engine, MockClock } from 'claude-code/testing' import type { On } from 'claude-code' import { cap, tail, redact, normalize, failureLine, sessionOf, addFailure, - entriesOf, isSuperseded, grams, buildIndex, matchOf, readRegistry, + entriesOf, isSuperseded, grams, buildIndex, matchOf, } from './register' const HOME = '/home/t' @@ -24,6 +24,7 @@ type World = { logs: string[] // every $.ui.log line tools: string[] // every tool name registered files: Map // other files (null: a directory) + unreadable: Set // paths whose fs.read throws } type Opts = { @@ -32,6 +33,7 @@ type Opts = { seed?: Record // the store at the start storeFails?: boolean // store.set rejects (a hand-made store) files?: Record // registries under the root + unreadable?: string[] // paths whose fs.read throws } /** @@ -61,6 +63,7 @@ async function boot($: Engine, on: On, o: Opts = {}): Promise { kv: new Map(Object.entries(o.seed ?? {})), sets: [], logs: [], tools: [], files: new Map(Object.entries(o.files ?? {})), + unreadable: new Set(o.unreadable ?? []), } handStore(on, w, o.storeFails ?? false) on('tool.register', (_a, e) => { @@ -68,6 +71,9 @@ async function boot($: Engine, on: On, o: Opts = {}): Promise { return { value: undefined } as never }) on('env.get', () => ({ value: HOME })) + on('fs.exists', (_a, e) => ({ + value: w.files.has(e.path) || e.path === SWITCH, + })) on('fs.stat', (_a, e) => { if (w.files.has(e.path)) { const text = w.files.get(e.path) @@ -85,6 +91,7 @@ async function boot($: Engine, on: On, o: Opts = {}): Promise { } } }) on('fs.read', (_a, e) => { + if (w.unreadable.has(e.path)) throw new Error('EIO') if (w.files.has(e.path)) return { value: w.files.get(e.path) ?? '' } if (e.path !== SWITCH || w.switchText === undefined) { throw new Error('ENOENT') @@ -620,13 +627,11 @@ test('isSuperseded: the last status wins, else the heading tag', () => { expect(isSuperseded('## LRN-1 [open] a\n')).toBe(false) }) -test('readRegistry: a directory and an over-size file read as empty', async ( +test('a directory and an over-size file are skipped on the index line', async ( $, on, ) => { const big = 'x'.repeat(4194305) const w = await boot($, on, { files: { [BLK]: null, [LRN]: big } }) - expect(await readRegistry($ as never, BLK)).toBe('') - expect(await readRegistry($ as never, LRN)).toBe('') await failing($, w, EXIT1 + 'fatal: bad thing') expect(await pending($)).toContain( 'index: skipped blockers.md (not a file), learnings.md (over 4 MiB)') @@ -743,3 +748,64 @@ test('the tool never writes the store', async ($, on) => { await pending($) expect(w.sets).toEqual([]) }) + +test('TIME1: an index over 80000 newlines builds in under 500 ms', () => { + const body = '## BLK-001 x\n' + '\n'.repeat(80000) + '- **Status**: open\n' + + '\r\n'.repeat(40000) + '`fatal: unable to access remote`\n' + const t0 = now() + const index = buildIndex([body]) + expect(now() - t0).toBeLessThan(500) + expect(index.size).toBe(2) +}) + +test('a registry whose read throws is named unreadable', async ($, on) => { + const w = await boot($, on, { files: REGS, unreadable: [BLK] }) + await failing($, w, EXIT1 + 'fatal: unable to access remote host') + const text = await pending($) + expect(text).toContain( + 'index: read learnings.md; skipped blockers.md (unreadable)') + expect(text).toContain('- NEW \u00d71') +}) + +test('a lone CR in the excerpt shows as the newline mark', async ($, on) => { + const w = await boot($, on) + await failing($, w, EXIT1 + 'a\rb\r\nc\nfatal: end') + expect(await pending($)).toContain( + 'excerpt: Exit code 1\u23cea\u23ceb\u23cec\u23cefatal: end') +}) + +test('malformed stored elements are skipped, valid ones listed', async ( + $, on, +) => { + const ok = { key: 'good line here ok', line: 'good line here ok', + excerpt: '', command: 'c', agent: false, at: 1, count: 1 } + const seed = { [`p:${ROOT}`]: { sessions: [{ + id: 's1', at: 1, red: [], + failures: [{ ...ok, key: 42 }, null, ok], + milestones: [{ kind: 'commit', label: 'x', at: 'bad' }, null, + { kind: 'commit', label: 'fine', at: 60000 }], + }] } } + await boot($, on, { seed }) + const text = await pending($) + expect(text).toContain('- commit fine (00:01)') + expect(text).toContain('- NEW \u00d71 good line here ok') + expect(text).not.toContain('bad') +}) + +test('a registry of exactly 4 MiB is read, one byte more skipped', async ( + $, on, +) => { + const w = await boot($, on, { files: { + [BLK]: 'x'.repeat(4194304), [LRN]: 'x'.repeat(4194305), + } }) + await failing($, w, EXIT1 + 'fatal: bad thing') + expect(await pending($)).toContain( + 'index: read blockers.md; skipped learnings.md (over 4 MiB)') +}) + +test('a span of 12 chars is indexed, one of 11 is not', () => { + const at12 = buildIndex(['## BLK-001 x\n`aa bb cc ddd`\n']) + const at11 = buildIndex(['## BLK-001 x\n`aa bb cc dd`\n']) + expect(at12.size).toBe(1) + expect(at11.size).toBe(0) +}) diff --git a/mods/memory/hooks/register.ts b/mods/memory/hooks/register.ts index cda6d77..4bfc3a5 100644 --- a/mods/memory/hooks/register.ts +++ b/mods/memory/hooks/register.ts @@ -286,29 +286,35 @@ const MIN_SPAN = 12 const GRAM_WORDS = 4 const HEADING = /^## ((?:BLK|LRN)-\d+)\b.*$/gm const SPAN = /`([^`\n]+)`/g -const STATUS = /^\s*-\s*\*\*Stat(?:us|ut)[^*]*\*\*\s*:?\s*([A-Za-z-]+)/gim +// [ \t] not \s: a multiline regex must not cross newlines (quadratic). +const STATUS = + /^[ \t]*-[ \t]*\*\*Stat(?:us|ut)[^*\n]*\*\*[ \t]*:?[ \t]*([A-Za-z-]+)/gim type Entry = { id: string; body: string } -/** Why a registry is not read, or `undefined` when it is readable. */ -async function skipReason($: Api, path: string): Promise { +type Read = { text: string } | { why: string } + +/** One stat, one read: the text, or why the registry is skipped. */ +// a rejected stat means missing only when exists says so +async function tryRead($: Api, path: string): Promise { try { const info = await $.fs.stat(path) - if (info.kind !== 'file') return 'not a file' - return info.size > MAX_REGISTRY ? 'over 4 MiB' : undefined + if (info.kind !== 'file') return { why: 'not a file' } + if (info.size > MAX_REGISTRY) return { why: 'over 4 MiB' } } catch { - return 'missing' + return { why: (await $.fs.exists(path)) ? 'unreadable' : 'missing' } + } + try { + return { text: await $.fs.read(path) } + } catch { + return { why: 'unreadable' } } } -/** A registry's text; `''` when missing, not a file or over 4 MiB. */ +/** A registry's text; `''` when skipped (missing, not a file, too big). */ export async function readRegistry($: Api, path: string): Promise { - if ((await skipReason($, path)) !== undefined) return '' - try { - return await $.fs.read(path) - } catch { - return '' - } + const r = await tryRead($, path) + return 'text' in r ? r.text : '' } /** The BLK/LRN entries; a body runs from its heading to the next one. */ @@ -383,12 +389,12 @@ async function loadIndex($: Api, root: string): Promise { const skipped: string[] = [] for (const name of REGISTRIES) { const path = `${root}/.claude/memory/${name}` - const why = await skipReason($, path) - if (why !== undefined) { - skipped.push(`${name} (${why})`) + const r = await tryRead($, path) + if ('why' in r) { + skipped.push(`${name} (${r.why})`) continue } - texts.push(await readRegistry($, path)) + texts.push(r.text) read.push(name) } const parts = [ @@ -407,13 +413,30 @@ const EXCERPT_SHOWN = 300 const OFF_TEXT = 'memory capture is off (~/.claude/memory.json)' const NOTHING_TEXT = 'memory_pending: nothing captured' +const isText = (...v: unknown[]) => v.every(x => typeof x === 'string') + +/** The store is data: a malformed element is skipped, never thrown on. */ +function isFailure(v: unknown): v is Failure { + const f = v as Partial | null + return typeof f === 'object' && f !== null && + isText(f.key, f.line, f.excerpt, f.command) && + Number.isFinite(f.count) +} + +function isMilestone(v: unknown): v is Milestone { + const m = v as Partial | null + return typeof m === 'object' && m !== null && isText(m.kind, m.label) && + Number.isFinite(m.at) && Math.abs(m.at as number) < 8.64e15 +} + type Row = { f: Failure; id: string | undefined; earlier: boolean } /** Current-session NEW, current-session known, then earlier sessions. */ function rowsOf(p: Project, curId: string, index: Map) { - const rows: Row[] = p.sessions.flatMap(s => s.failures.map(f => ({ - f, id: matchOf(index, f.key), earlier: s.id !== curId, - }))) + const rows: Row[] = p.sessions.flatMap(s => + s.failures.filter(isFailure).map(f => ({ + f, id: matchOf(index, f.key), earlier: s.id !== curId, + }))) const rank = (r: Row) => (r.earlier ? 2 : r.id === undefined ? 0 : 1) return rows.sort((a, b) => rank(a) - rank(b)) } @@ -424,7 +447,7 @@ function failureText({ f, id, earlier }: Row): string { (earlier ? ' (earlier session)' : '') const head = `- ${tag} ×${f.count}${marks} ${f.line} — command: ${f.command}` if (f.excerpt === '') return head - const shown = tail(f.excerpt, EXCERPT_SHOWN).replace(/\r?\n/g, '⏎') + const shown = tail(f.excerpt, EXCERPT_SHOWN).replace(/\r\n?|\n/g, '⏎') return `${head}\n excerpt: ${shown}` } @@ -453,7 +476,8 @@ export function formatPending( rx: Registries, ): string { const rows = rowsOf(p, curId, rx.index) - const ms = p.sessions.find(s => s.id === curId)?.milestones ?? [] + const cur = p.sessions.find(s => s.id === curId) + const ms = (cur?.milestones ?? []).filter(isMilestone) if (rows.length === 0 && ms.length === 0) return NOTHING_TEXT const name = root.split('/').filter(Boolean).pop() ?? root const out = [`memory_pending (project ${name}, ${p.sessions.length} ` +