diff --git a/mods/memory/hooks/register.test.ts b/mods/memory/hooks/register.test.ts index bdf5c43..b99d31d 100644 --- a/mods/memory/hooks/register.test.ts +++ b/mods/memory/hooks/register.test.ts @@ -28,6 +28,8 @@ type World = { unreadable: Set // paths whose fs.read throws statFails: Set // paths whose fs.stat throws though they exist existsFails: Set // paths whose fs.exists throws + gets: number // store.get calls + getsFail: boolean // store.get throws } type Opts = { @@ -47,7 +49,11 @@ type Opts = { * store.set hook to observe or fail it. */ function handStore(on: On, w: World, fails: boolean): void { - on('store.get', (_a, e) => ({ value: w.kv.get(e.key) })) + on('store.get', (_a, e) => { + w.gets++ + if (w.getsFail) throw new Error('boom') + return { value: w.kv.get(e.key) } + }) on('store.keys', () => ({ value: [...w.kv.keys()] })) on('store.delete', (_a, e) => ({ value: void w.kv.delete(e.key) })) on('store.set', (_a, e) => { @@ -71,6 +77,7 @@ async function boot($: Engine, on: On, o: Opts = {}): Promise { unreadable: new Set(o.unreadable ?? []), statFails: new Set(o.statFails ?? []), existsFails: new Set(o.existsFails ?? []), + gets: 0, getsFail: false, } handStore(on, w, o.storeFails ?? false) on('tool.register', (_a, e) => { @@ -308,7 +315,7 @@ test('CAP2: Exit code alone stores nothing', async ($, on) => { expect(w.sets).toEqual([]) }) -test('CAP3: excluded texts and a no-error-word line store nothing', async ( +test('CAP3: excluded texts and a letterless line store nothing', async ( $, on, ) => { const w = await boot($, on) @@ -682,6 +689,13 @@ test('an empty root answers no project root', async ($, on) => { expect(await pending($)).toBe('memory_pending: no project root') }) +/** HH:MM of the local clock, from the Date API (not the code under test). */ +const localHhmm = (at: number) => { + const d = new Date(at) + const two = (n: number) => String(n).padStart(2, '0') + return `${two(d.getHours())}:${two(d.getMinutes())}` +} + test('milestones, sub-agent, earlier session and order', async ($, on) => { const mk = (id: string, at: number, failures: unknown[], ms: unknown[]) => ({ id, at, failures, milestones: ms, red: [] }) @@ -700,7 +714,7 @@ test('milestones, sub-agent, earlier session and order', async ($, on) => { 'memory_pending (project app, 2 session(s))', 'index: read blockers.md, learnings.md', 'milestones (this session):', - '- commit fix: x (14:02)', + '- commit fix: x (' + localHhmm(3600000 * 14 + 120000) + ')', 'failures:', '- NEW \u00d72 (sub-agent) brand new line here ok \u2014 command: c', ' excerpt: a\u23ceb', @@ -826,7 +840,7 @@ test('malformed stored elements are skipped, valid ones listed', async ( }] } } await boot($, on, { seed }) const text = await pending($) - expect(text).toContain('- commit fine (00:01)') + expect(text).toContain('- commit fine (' + localHhmm(60000) + ')') expect(text).toContain('- NEW \u00d71 good line here ok') expect(text).not.toContain('bad') }) @@ -1057,7 +1071,7 @@ test('a stored commit milestone shows in the memory_pending answer', async ( 'memory_pending (project app, 1 session(s))', 'index: skipped blockers.md (missing), learnings.md (missing)', 'milestones (this session):', - '- commit fix(x): y (00:00)', + '- commit fix(x): y (' + localHhmm(1000) + ')', ].join('\n')) }) @@ -1220,3 +1234,119 @@ test('pending keeps the 20 latest milestones', async ($, on) => { await composer($) expect(seen[0]?.[0]).toContain('memory: commit fix: n2 ') }) + +test('I2: names with digits are redacted', () => { + expect(redact('OAUTH2_CLIENT_' + 'SECRET=abc')) + .toBe('OAUTH2_CLIENT_SECRET=') + expect(redact('S3_' + 'SECRET=abc')).toBe('S3_SECRET=') + expect(redact('TOKEN' + '_V2=abc')).toBe('TOKEN_V2=') +}) + +const JWT = 'eyJ' + 'a'.repeat(20) + '.eyJ' + 'b'.repeat(20) + '.' + + 'c'.repeat(20) + +test('I2: a JWT is redacted alone and inside an env value', () => { + expect(redact('t ' + JWT + ' z')).toBe('t z') + expect(redact('SUPABASE_SERVICE_ROLE_KEY=' + JWT)) + .toBe('SUPABASE_SERVICE_ROLE_KEY=') + expect(redact('MY_' + 'TOKEN=' + JWT)).toBe('MY_TOKEN=') +}) + +test('I2: credentials on the command line are redacted', () => { + expect(redact('curl -u ' + 'bob:hunter2 https://x')) + .toBe('curl -u https://x') + expect(redact('curl --user ' + 'bob:hunter2 https://x')) + .toBe('curl --user https://x') + expect(redact('mysql --pass' + 'word hunter2 db')) + .toBe('mysql --password db') + expect(redact('mysql --pass' + 'word=hunter2 db')) + .toBe('mysql --password db') + expect(redact('Authorization: Ba' + 'sic dXNlcjpwdw== z')) + .toBe('Authorization: Basic z') + expect(redact('Authorization: Bea' + 'rer abcdefgh12345 z')) + .toBe('Authorization: Bearer z') + expect(redact('Authorization: to' + 'ken abc123 z')) + .toBe('Authorization: token z') +}) + +test('IDX9: a placeholder-only gram is not indexed nor matched', () => { + const span = '`connect 10.0.0.1:5432 refused 12 34`' + const index = buildIndex(['## LRN-074 x\n' + span + '\n']) + const key = normalize('Error at 2026-11-04 12:00:00.123 timeout') + expect(matchOf(index, key)).toBeUndefined() + expect(grams('# # # # x')).toEqual(['# # # x']) + expect(grams('a # b')) + .toEqual(['a # ', '# b']) +}) + +test('M1: a letterless last line is dropped, one with letters kept', async ( + $, on, +) => { + const w = await boot($, on) + await failing($, w, 'Exit code 1\n> 2') + expect(w.sets).toEqual([]) + await failing($, w, 'Exit code 1\n> 2 lines differ') + expect(failuresOf(w).map(f => f.line)).toEqual(['> 2 lines differ']) +}) + +test('M2: a plain successful ls touches the store not at all', async ( + $, on, +) => { + const w = await boot($, on) + await bash($, 'ls') + expect(w.gets).toBe(0) + expect(w.sets).toEqual([]) +}) + +test('M3: a 70000-char failing text is captured', async ($, on) => { + const w = await boot($, on) + await failing($, w, 'Exit code 1\n' + 'x\n'.repeat(35000) + 'fatal: end') + expect(failuresOf(w).map(f => f.line)).toEqual(['fatal: end']) +}) + +test('M4: a heredoc command shows on one answer line', async ($, on) => { + const w = await boot($, on) + await failing($, w, EXIT1 + 'fatal: no', HEREDOC('feat: z')) + const out = await pending($) + expect(out).toContain('command: git commit -m "$(cat <<\'EOF\'\u23cefeat: z') + expect(out.split('\n').filter(l => l.startsWith('- NEW'))).toHaveLength(1) + expect(out.split('\n')).toHaveLength(5) +}) + +test('M5: the milestone time is the local one', async ($, on) => { + const w = await boot($, on) + await run($, w, 'git commit -m "fix: y"') + expect(await pending($)).toContain(`(${localHhmm(1000)})`) +}) + +test('NT5: a bare finish reads once in the note', async ($, on) => { + const { w, seen } = await bootNotes($, on) + await run($, w, FINISH_CMD) + await composer($) + expect(seen).toEqual([[ + 'memory: finish \u2014 0 new failure mode(s) this session;' + + ' /capitalize when ready.', + ]]) +}) + +test('M12: a throwing note step passes the prompt unchanged', async ( + $, on, +) => { + const { w, seen } = await bootNotes($, on) + await run($, w, 'git commit -m "fix: y"') + w.getsFail = true + await composer($) + expect(seen).toEqual([undefined]) + expect(w.logs.filter(l => l.includes('prompt.submit'))).toHaveLength(1) +}) + +test('session.end lets a hook error log again', async ($, on) => { + const w = await boot($, on, { rootFails: true }) + await bash($, 'ls') + await bash($, 'ls') + const warns = () => w.logs.filter(l => l.startsWith('memory: Bash hook')) + expect(warns()).toHaveLength(1) + await endSession($) + await bash($, 'ls') + expect(warns()).toHaveLength(2) +}) diff --git a/mods/memory/hooks/register.ts b/mods/memory/hooks/register.ts index c16c16a..e4e0a67 100644 --- a/mods/memory/hooks/register.ts +++ b/mods/memory/hooks/register.ts @@ -23,12 +23,18 @@ const MAX_REGEX_INPUT = 65536 const PEM = '-----' const PEM_KIND = '(?:[A-Z]+ )?PRIVATE KEY' -// Redaction patterns, applied in order. Repeats are bounded where the +// Redaction patterns, applied in order: the JWT shape first (so a secret +// named variable still ends as `NAME=` through pattern 1, and any +// other name keeps `NAME=`), then the name rule, the vendor +// shapes, PEM, Bearer, URL credentials, and last the command-line shapes +// (-u user:pass, --password, Authorization). Repeats are bounded where the // input is free text, so no pattern can run away on a 64 KiB input. const REDACTIONS: [RegExp, string][] = [ + [/eyJ[A-Za-z0-9_-]{8,}\.eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}/g, + ''], [new RegExp( - '\\b([A-Za-z_]{0,60}(?:TOKEN|API_KEY|SECRET|PASSWORD|PASSWD)' + - '[A-Za-z_]{0,60})=\\S+', + '\\b([A-Za-z0-9_]{0,60}(?:TOKEN|API_KEY|SECRET|PASSWORD|PASSWD)' + + '[A-Za-z0-9_]{0,60})=\\S+', 'gi', ), '$1='], [/sk_live_[0-9A-Za-z]{10,}/g, ''], @@ -44,6 +50,10 @@ const REDACTIONS: [RegExp, string][] = [ ), ''], [/\bBearer\s+[A-Za-z0-9._~+/=-]{8,}/g, 'Bearer '], [/([a-z][a-z0-9+.-]{0,30}:\/\/)[^\s/@:]+:[^\s/@]+@/gi, '$1@'], + [/(?<=^|\s)(-u|--user)[ \t]{1,5}\S{1,200}:\S{1,200}/g, '$1 '], + [/--password(?:=|[ \t]{1,5})\S{1,200}/g, '--password '], + [/Authorization:[ \t]{0,5}(Basic|Bearer|token)[ \t]{1,5}\S{1,400}/gi, + 'Authorization: $1 '], ] const HEX_RUN = /\b[0-9a-f]{7,}\b/g @@ -128,17 +138,26 @@ const RTK_TRAILER = /** * The line that tells what failed, from an already redacted answer: the * last line after the first of an `Exit code N` text. `undefined` when the - * text is another kind, has no such line, or the redacted line is not - * error-shaped (`ERRORISH`); no earlier line is tried. + * text is another kind, has no such line, or the redacted line holds no + * letter or is not error-shaped (`ERRORISH`); no earlier line is tried. */ export function failureLine(text: string): string | undefined { - if (!EXIT_CODE.test(text)) return undefined + return EXIT_CODE.test(text) ? lastLine(text) : undefined +} + +/** The error-shaped last line of a redacted text with a letter in it. */ +function lastLine(text: string): string | undefined { const lines = text.split('\n') .map(l => l.slice(-LINE_WINDOW).replace(RTK_TRAILER, '').trim()) .filter(l => l !== '') if (lines.length < 2) return undefined const line = (lines[lines.length - 1] ?? '').slice(0, MAX_LINE) - return ERRORISH.test(line) ? line : undefined + return hasLetter(line) && ERRORISH.test(line) ? line : undefined +} + +/** A letter outside the ``, `` and `#` placeholders. */ +function hasLetter(line: string): boolean { + return /\p{L}/u.test(normalize(line).replace(/|/g, '')) } function isSession(v: unknown): v is Session { @@ -206,8 +225,9 @@ function buildFailure( agent: boolean, at: number, ): Failure | undefined { + if (!EXIT_CODE.test(text)) return undefined const red = redact(tail(text, MAX_REGEX_INPUT)) - const line = failureLine(red) + const line = lastLine(red) if (line === undefined) return undefined return fitEntry({ key: normalize(line), @@ -345,13 +365,20 @@ export function isSuperseded(body: string): boolean { return /\[superseded\]/i.test(body.split('\n', 1)[0] ?? '') } -/** Every window of 4 words; a key of fewer words has none (always NEW). */ +const PLACEHOLDERS = new Set(['#', '', '']) + +/** + * Every window of 4 words, except one made only of placeholders (it + * carries no content); a key of fewer words has none (always NEW). + */ export function grams(normLine: string): string[] { const words = normLine.split(/[^a-z0-9<>#]+/).filter(Boolean) if (words.length < GRAM_WORDS) return [] return words .slice(0, words.length - GRAM_WORDS + 1) - .map((_, i) => words.slice(i, i + GRAM_WORDS).join(' ')) + .map((_, i) => words.slice(i, i + GRAM_WORDS)) + .filter(win => !win.every(w => PLACEHOLDERS.has(w))) + .map(win => win.join(' ')) } function spansOf(body: string): string[] { @@ -422,6 +449,8 @@ const EXCERPT_SHOWN = 300 const OFF_TEXT = 'memory capture is off (~/.claude/memory.json)' const NOTHING_TEXT = 'memory_pending: nothing captured' +const NEWLINES = /\r\n?|\n/g + const isText = (...v: unknown[]) => v.every(x => typeof x === 'string') /** The store is data: a malformed element is skipped, never thrown on. */ @@ -454,14 +483,17 @@ function failureText({ f, id, earlier }: Row): string { const tag = id === undefined ? 'NEW' : `known ${id}` const marks = (f.agent ? ' (sub-agent)' : '') + (earlier ? ' (earlier session)' : '') - const head = `- ${tag} ×${f.count}${marks} ${f.line} — command: ${f.command}` + const cmd = f.command.replace(NEWLINES, '⏎') + const head = `- ${tag} ×${f.count}${marks} ${f.line} — command: ${cmd}` if (f.excerpt === '') return head - const shown = tail(f.excerpt, EXCERPT_SHOWN).replace(/\r\n?|\n/g, '⏎') + const shown = tail(f.excerpt, EXCERPT_SHOWN).replace(NEWLINES, '⏎') return `${head}\n excerpt: ${shown}` } function milestoneText(m: Milestone): string { - const hhmm = new Date(m.at).toISOString().slice(11, 16) + const d = new Date(m.at) + const two = (n: number) => String(n).padStart(2, '0') + const hhmm = `${two(d.getHours())}:${two(d.getMinutes())}` return `- ${m.kind} ${m.label} (${hhmm})` } @@ -698,6 +730,7 @@ function registerSession(on: On, st: State): void { on('session.end', async ($, e, next) => { st.enabled = null st.pending = [] + st.warned.clear() return next(e) }).catch(($, e, next) => { warnOnce(st, $, 'session.end', next.error.kind) @@ -757,6 +790,10 @@ function recordRed($: Api, st: State, at: Where, key: string, now: number) { })) } +/** `commit fix: x`; a bare `finish` is not said twice. */ +const noteLabel = (m: Milestone) => + m.label === m.kind ? m.kind : `${m.kind} ${m.label}` + /** Commit and finish milestones of a success; green when the suite was red. */ function recordSuccess( $: Api, @@ -770,6 +807,7 @@ function recordSuccess( ['finish', finishLabel(command)], ].filter((m): m is [Milestone['kind'], string] => m[1] !== undefined) const key = suiteKeyOf(command) + if (found.length === 0 && key === undefined) return Promise.resolve() return queueWrite(st, async () => { const cur = (await loadProject($, at.root)).sessions .find(s => s.id === at.id) @@ -781,8 +819,7 @@ function recordSuccess( s.milestones = [...s.milestones, ...added] if (green) s.red = s.red.filter(r => r !== key) }) - st.pending = [...st.pending, ...added.map(m => `${m.kind} ${m.label}`)] - .slice(-MAX_PENDING) + st.pending = [...st.pending, ...added.map(noteLabel)].slice(-MAX_PENDING) }) }