fix(memory): sweep before save, retry on a rejected set, foreign repos get no recall
This commit is contained in:
1 parent
a8847d7525
commit
fb2a4ff5ba
2 files changed
+179
-10
No files matched your search
@@ -64,6 +64,11 @@ type World = {
|
|||||||
getsFail: boolean // store.get throws
|
getsFail: boolean // store.get throws
|
||||||
model: Model // what `model.complete` does now
|
model: Model // what `model.complete` does now
|
||||||
asks: Ask[] // every model.complete call
|
asks: Ask[] // every model.complete call
|
||||||
|
storeCap: number // store.set rejects when the store would pass this size
|
||||||
|
rejected: number // store.set calls rejected (full store or rejectSets)
|
||||||
|
rejectSets: number // the next store.set calls to reject whatever the size
|
||||||
|
gitProtect: string | undefined // `git config gitflow.protect` (absent: exit 1)
|
||||||
|
gitRuns: string[][] // every git process.run argv
|
||||||
}
|
}
|
||||||
|
|
||||||
type Opts = {
|
type Opts = {
|
||||||
@@ -75,6 +80,16 @@ type Opts = {
|
|||||||
unreadable?: string[] // paths whose fs.read throws
|
unreadable?: string[] // paths whose fs.read throws
|
||||||
statFails?: string[] // paths whose fs.stat throws though they exist
|
statFails?: string[] // paths whose fs.stat throws though they exist
|
||||||
existsFails?: string[] // paths whose fs.exists throws
|
existsFails?: string[] // paths whose fs.exists throws
|
||||||
|
storeCap?: number // total serialized bytes the store accepts
|
||||||
|
rejectSets?: number // reject this many store.set calls, then accept
|
||||||
|
gitProtect?: string // what `git config --get gitflow.protect` prints
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The bytes the store holds, counting `key` as empty (it is rewritten). */
|
||||||
|
function storedSize(w: World, key: string): number {
|
||||||
|
let n = 0
|
||||||
|
for (const [k, v] of w.kv) if (k !== key) n += JSON.stringify(v).length
|
||||||
|
return n
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -92,6 +107,13 @@ function handStore(on: On, w: World, fails: boolean): void {
|
|||||||
on('store.delete', (_a, e) => ({ value: void w.kv.delete(e.key) }))
|
on('store.delete', (_a, e) => ({ value: void w.kv.delete(e.key) }))
|
||||||
on('store.set', (_a, e) => {
|
on('store.set', (_a, e) => {
|
||||||
if (fails) throw new Error('disk full')
|
if (fails) throw new Error('disk full')
|
||||||
|
const over = storedSize(w, e.key) + JSON.stringify(e.value).length >
|
||||||
|
w.storeCap
|
||||||
|
if (over || w.rejectSets > 0) {
|
||||||
|
w.rejectSets = Math.max(0, w.rejectSets - 1)
|
||||||
|
w.rejected++
|
||||||
|
throw new Error('store full')
|
||||||
|
}
|
||||||
w.sets.push(e.key)
|
w.sets.push(e.key)
|
||||||
w.kv.set(e.key, JSON.parse(JSON.stringify(e.value)))
|
w.kv.set(e.key, JSON.parse(JSON.stringify(e.value)))
|
||||||
return { value: undefined }
|
return { value: undefined }
|
||||||
@@ -113,6 +135,9 @@ async function boot($: Engine, on: On, o: Opts = {}): Promise<World> {
|
|||||||
existsFails: new Set(o.existsFails ?? []),
|
existsFails: new Set(o.existsFails ?? []),
|
||||||
proc: { stdout: INDEX_OUT }, runs: [], home: HOME,
|
proc: { stdout: INDEX_OUT }, runs: [], home: HOME,
|
||||||
gets: 0, getsFail: false, model: { text: 'none' }, asks: [],
|
gets: 0, getsFail: false, model: { text: 'none' }, asks: [],
|
||||||
|
storeCap: o.storeCap ?? Infinity, rejected: 0,
|
||||||
|
rejectSets: o.rejectSets ?? 0,
|
||||||
|
gitProtect: o.gitProtect, gitRuns: [],
|
||||||
}
|
}
|
||||||
handStore(on, w, o.storeFails ?? false)
|
handStore(on, w, o.storeFails ?? false)
|
||||||
on('tool.register', (_a, e) => {
|
on('tool.register', (_a, e) => {
|
||||||
@@ -151,6 +176,13 @@ async function boot($: Engine, on: On, o: Opts = {}): Promise<World> {
|
|||||||
return { value: w.switchText }
|
return { value: w.switchText }
|
||||||
})
|
})
|
||||||
on('process.run', async (_a, e) => {
|
on('process.run', async (_a, e) => {
|
||||||
|
if (e.argv[0] === 'git') {
|
||||||
|
w.gitRuns.push([...e.argv])
|
||||||
|
const out = w.gitProtect
|
||||||
|
return { value: { exitCode: out === undefined ? 1 : 0,
|
||||||
|
stdout: out === undefined ? '' : `${out}\n`, stderr: '',
|
||||||
|
isStdoutTruncated: false, isStderrTruncated: false } }
|
||||||
|
}
|
||||||
w.runs.push([...e.argv])
|
w.runs.push([...e.argv])
|
||||||
if (w.proc === 'throw') throw new Error('spawn failed')
|
if (w.proc === 'throw') throw new Error('spawn failed')
|
||||||
if (w.proc === 'hang') await new Promise(() => undefined)
|
if (w.proc === 'hang') await new Promise(() => undefined)
|
||||||
@@ -2968,3 +3000,97 @@ test('PC2: a throwing recall still lets the note attach', async ($, on) => {
|
|||||||
expect(added[0]).toContain('memory: commit fix: y')
|
expect(added[0]).toContain('memory: commit fix: y')
|
||||||
expect(w.logs.filter(l => l.includes('prompt.submit'))).toEqual([])
|
expect(w.logs.filter(l => l.includes('prompt.submit'))).toEqual([])
|
||||||
})
|
})
|
||||||
|
|
||||||
|
// ---- store sweep before the save, one retry on a rejected set ----------
|
||||||
|
|
||||||
|
const ENGINE_BYTES = 4194304
|
||||||
|
const fill = (n: number) => 'x'.repeat(n - 2) // n serialized bytes
|
||||||
|
|
||||||
|
test('SW1: a full store and a new session: swept first, the save lands',
|
||||||
|
async ($, on) => {
|
||||||
|
const w = await boot($, on, {
|
||||||
|
storeCap: ENGINE_BYTES,
|
||||||
|
seed: { 'l:/a': fill(1400000), 'l:/b': fill(1400000),
|
||||||
|
'l:/c': fill(1394240) },
|
||||||
|
})
|
||||||
|
await failing($, w, EXIT1 + 'fatal: bad thing')
|
||||||
|
expect(failuresOf(w)).toHaveLength(1)
|
||||||
|
expect(w.kv.has('l:/a')).toBe(false)
|
||||||
|
expect(w.rejected).toBe(0)
|
||||||
|
})
|
||||||
|
|
||||||
|
test('SW2: a rejected set sweeps and retries once, the retry lands',
|
||||||
|
async ($, on) => {
|
||||||
|
const w = await boot($, on, { rejectSets: 1 })
|
||||||
|
await failing($, w, EXIT1 + 'fatal: bad thing')
|
||||||
|
expect(w.rejected).toBe(1)
|
||||||
|
expect(failuresOf(w)).toHaveLength(1)
|
||||||
|
})
|
||||||
|
|
||||||
|
test('SW3: a set rejected twice is not retried a third time', async (
|
||||||
|
$, on,
|
||||||
|
) => {
|
||||||
|
const w = await boot($, on, { rejectSets: 5 })
|
||||||
|
const out = await failing($, w, EXIT1 + 'fatal: bad thing')
|
||||||
|
expect(out).toEqual(w.answer)
|
||||||
|
expect(w.rejected).toBe(2)
|
||||||
|
})
|
||||||
|
|
||||||
|
test('SW4: a rejected log set retries once and lands', async ($, on) => {
|
||||||
|
const { w } = await bootRecall($, on, { rejectSets: 1 })
|
||||||
|
await start($)
|
||||||
|
await say($, GO)
|
||||||
|
expect(w.rejected).toBe(1)
|
||||||
|
expect(await askLog(w)).toEqual([['fresh', 'injected', ['BLK-001']]])
|
||||||
|
})
|
||||||
|
|
||||||
|
// ---- a foreign repository gets no recall and no known line -------------
|
||||||
|
|
||||||
|
test('FR1: gitflow.protect false: no known line, capture still works',
|
||||||
|
async ($, on) => {
|
||||||
|
const w = await bootKnown($, on, { gitProtect: 'false' })
|
||||||
|
const out = await failing($, w, EXIT1 + KN_SPAN)
|
||||||
|
expect(out).toEqual(w.answer)
|
||||||
|
expect(failuresOf(w)).toHaveLength(1)
|
||||||
|
expect(w.gitRuns[0]).toEqual(
|
||||||
|
['git', '-C', ROOT, 'config', '--get', 'gitflow.protect'])
|
||||||
|
})
|
||||||
|
|
||||||
|
test('FR2: gitflow.protect false: no recall, memory_pending answers',
|
||||||
|
async ($, on) => {
|
||||||
|
const { w, seen } = await bootRecall($, on, { gitProtect: 'false' })
|
||||||
|
await start($)
|
||||||
|
await run($, w, 'git commit -m "fix: y"')
|
||||||
|
await say($, GO)
|
||||||
|
expect(seen[0]?.some(c => c.startsWith('memory recall:')) ?? false)
|
||||||
|
.toBe(false)
|
||||||
|
expect(w.asks).toHaveLength(0)
|
||||||
|
expect(await pending($)).toContain('fix: y')
|
||||||
|
})
|
||||||
|
|
||||||
|
const protectCases: [string, Opts][] = [
|
||||||
|
['true', { gitProtect: 'true' }], ['absent', {}],
|
||||||
|
]
|
||||||
|
for (const [name, opts] of protectCases) {
|
||||||
|
test(`FR3: gitflow.protect ${name}: known line unchanged`, async ($, on) => {
|
||||||
|
const w = await bootKnown($, on, opts)
|
||||||
|
const out = await failing($, w, EXIT1 + KN_SPAN)
|
||||||
|
expect(out).toEqual({ ...(w.answer as object), context: [KN_LINE] })
|
||||||
|
})
|
||||||
|
|
||||||
|
test(`FR3: gitflow.protect ${name}: recall unchanged`, async ($, on) => {
|
||||||
|
const { seen } = await bootRecall($, on, opts)
|
||||||
|
await start($)
|
||||||
|
await say($, GO)
|
||||||
|
expect(seen[0]?.[0]).toBe(BLOCK1)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
test('FR4: the protect read happens once per root and session', async (
|
||||||
|
$, on,
|
||||||
|
) => {
|
||||||
|
const w = await bootKnown($, on, { gitProtect: 'true' })
|
||||||
|
await failing($, w, EXIT1 + KN_SPAN)
|
||||||
|
await failing($, w, EXIT1 + 'fatal: other zebra quartz', 'make b')
|
||||||
|
expect(w.gitRuns).toHaveLength(1)
|
||||||
|
})
|
||||||
@@ -20,6 +20,7 @@ type State = {
|
|||||||
fresh: Set<string> // sessions whose next qualifying prompt starts a context
|
fresh: Set<string> // sessions whose next qualifying prompt starts a context
|
||||||
lastPrompt: Map<string, number> // session id -> time of its last prompt
|
lastPrompt: Map<string, number> // session id -> time of its last prompt
|
||||||
rerank: { fails: number; until: number } // failures in a row, pause end
|
rerank: { fails: number; until: number } // failures in a row, pause end
|
||||||
|
foreign: Map<string, boolean> // project root -> gitflow.protect is false
|
||||||
}
|
}
|
||||||
|
|
||||||
export const IDLE_MS = 1800000
|
export const IDLE_MS = 1800000
|
||||||
@@ -673,6 +674,20 @@ async function statusMap($: Api, root: string): Promise<StatusMap | undefined> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* True when the project's git config marks it foreign (`gitflow.protect`
|
||||||
|
* is `false`). Read once per root and session; any failure: not foreign.
|
||||||
|
*/
|
||||||
|
async function isForeign($: Api, st: State, root: string): Promise<boolean> {
|
||||||
|
const known = st.foreign.get(root)
|
||||||
|
if (known !== undefined) return known
|
||||||
|
const out = await runWithin(
|
||||||
|
$, ['git', '-C', root, 'config', '--get', 'gitflow.protect'])
|
||||||
|
const foreign = out?.trim() === 'false'
|
||||||
|
st.foreign.set(root, foreign)
|
||||||
|
return foreign
|
||||||
|
}
|
||||||
|
|
||||||
// Function words, 3+ letters first, then the 2-letter FR and EN ones: TOKEN
|
// Function words, 3+ letters first, then the 2-letter FR and EN ones: TOKEN
|
||||||
// takes 2+ letters, so without them "ok tu peux continuer la suite stp"
|
// takes 2+ letters, so without them "ok tu peux continuer la suite stp"
|
||||||
// counts 7 content words and passes the floor. At most 120 words.
|
// counts 7 content words and passes the floor. At most 120 words.
|
||||||
@@ -1004,6 +1019,19 @@ function fitLog(log: LogRecord[]): LogRecord[] {
|
|||||||
return kept
|
return kept
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Runs `write`; if it rejects, sweeps once and runs it a second time. */
|
||||||
|
async function retryOnce(
|
||||||
|
sweep: () => Promise<void>,
|
||||||
|
write: () => Promise<void>,
|
||||||
|
): Promise<void> {
|
||||||
|
try {
|
||||||
|
await write()
|
||||||
|
} catch {
|
||||||
|
await sweep()
|
||||||
|
await write()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/** Appends an injection record on the write chain; the oldest fall off. */
|
/** Appends an injection record on the write chain; the oldest fall off. */
|
||||||
export function appendLog(
|
export function appendLog(
|
||||||
$: Api,
|
$: Api,
|
||||||
@@ -1014,27 +1042,38 @@ export function appendLog(
|
|||||||
return queueWrite(st, async () => {
|
return queueWrite(st, async () => {
|
||||||
const log = await loadLog($, root)
|
const log = await loadLog($, root)
|
||||||
if (log.length === 0) await sweepLogs($, logKey(root))
|
if (log.length === 0) await sweepLogs($, logKey(root))
|
||||||
await $.store.set(logKey(root), fitLog([...log, record]))
|
await retryOnce(
|
||||||
|
() => sweepLogs($, logKey(root)),
|
||||||
|
() => $.store.set(logKey(root), fitLog([...log, record])),
|
||||||
|
)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
type Where = { root: string; id: string }
|
type Where = { root: string; id: string }
|
||||||
|
|
||||||
/** Load, change the current session, save; sweeps when it is created. */
|
const sweepAll = async ($: Api, at: Where) => {
|
||||||
|
await sweepStore($, storeKey(at.root), at.id)
|
||||||
|
await sweepLogs($, logKey(at.root))
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Load, change the current session, save. A new session sweeps the store
|
||||||
|
* first, so the save fits; a rejected save sweeps and retries once.
|
||||||
|
*/
|
||||||
async function mutateSession(
|
async function mutateSession(
|
||||||
$: Api,
|
$: Api,
|
||||||
at: Where,
|
at: Where,
|
||||||
now: number,
|
now: number,
|
||||||
change: (s: Session) => void,
|
change: (s: Session) => void,
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
const p = await loadProject($, at.root)
|
const save = async () => {
|
||||||
const created = !p.sessions.some(s => s.id === at.id)
|
const p = await loadProject($, at.root)
|
||||||
change(sessionOf(p, at.id, now))
|
change(sessionOf(p, at.id, now))
|
||||||
await saveProject($, at.root, p)
|
await saveProject($, at.root, p)
|
||||||
if (created) {
|
|
||||||
await sweepStore($, storeKey(at.root), at.id)
|
|
||||||
await sweepLogs($, logKey(at.root))
|
|
||||||
}
|
}
|
||||||
|
const known = await loadProject($, at.root)
|
||||||
|
if (!known.sessions.some(s => s.id === at.id)) await sweepAll($, at)
|
||||||
|
await retryOnce(() => sweepAll($, at), save)
|
||||||
}
|
}
|
||||||
|
|
||||||
const recordFailure = ($: Api, at: Where, f: Failure) =>
|
const recordFailure = ($: Api, at: Where, f: Failure) =>
|
||||||
@@ -1122,6 +1161,7 @@ export function newState(): State {
|
|||||||
fresh: new Set(),
|
fresh: new Set(),
|
||||||
lastPrompt: new Map(),
|
lastPrompt: new Map(),
|
||||||
rerank: { fails: 0, until: 0 },
|
rerank: { fails: 0, until: 0 },
|
||||||
|
foreign: new Map(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1493,6 +1533,7 @@ async function knownLine(
|
|||||||
agent: string | undefined,
|
agent: string | undefined,
|
||||||
): Promise<string> {
|
): Promise<string> {
|
||||||
if (!(await ensureRecall($, st))) return ''
|
if (!(await ensureRecall($, st))) return ''
|
||||||
|
if (await isForeign($, st, at.root)) return ''
|
||||||
const entry = await liveCandidate($, at.root, f.key)
|
const entry = await liveCandidate($, at.root, f.key)
|
||||||
if (entry === undefined) return ''
|
if (entry === undefined) return ''
|
||||||
const line = `memory: known ${recallLine(entry)}`
|
const line = `memory: known ${recallLine(entry)}`
|
||||||
@@ -1777,7 +1818,9 @@ export async function promptContext(
|
|||||||
const idle = last === undefined ? 0 : now - last // unknown: not idle
|
const idle = last === undefined ? 0 : now - last // unknown: not idle
|
||||||
st.lastPrompt.set(at.id, now)
|
st.lastPrompt.set(at.id, now)
|
||||||
st.spent.delete(contextKey(at.id, undefined))
|
st.spent.delete(contextKey(at.id, undefined))
|
||||||
const trigger = (await ensureRecall($, st))
|
const recalling = (await ensureRecall($, st)) &&
|
||||||
|
!(await isForeign($, st, at.root))
|
||||||
|
const trigger = recalling
|
||||||
? triggerKind(e.text, st.fresh.has(at.id), idle)
|
? triggerKind(e.text, st.fresh.has(at.id), idle)
|
||||||
: undefined
|
: undefined
|
||||||
if (trigger !== undefined) st.fresh.delete(at.id)
|
if (trigger !== undefined) st.fresh.delete(at.id)
|
||||||
|
|||||||
Reference in new issue
Block a user