chore(memory): routing probe bench and its measured results
This commit is contained in:
1 parent
625b90bcbf
commit
8e575c7e46
5 files changed
+71
No files matched your search
@@ -0,0 +1 @@
|
||||
{ "name": "routeprobe", "version": "0.1.0", "description": "Records routing decisions and blocks every effect (throwaway)" }
|
||||
@@ -0,0 +1,25 @@
|
||||
# routing probe (bench for skill auto-routing, 2026-10-25)
|
||||
|
||||
A throwaway mod that records the first routing decision of a fresh headless
|
||||
session and blocks every effect: Skill and Agent calls are recorded then
|
||||
denied, a code Edit/Write is recorded then denied, doc writes (.md,
|
||||
`.claude/**`) are allowed, Bash is softly denied, Read/Grep/Glob pass.
|
||||
Run each case in a throwaway clone (never the real repo), output path in
|
||||
`ROUTEPROBE_OUT`:
|
||||
|
||||
ROUTEPROBE_OUT=/tmp/r.json claude -p --plugin-dir .claude/tasks/plans/2026-10-25-routing-probe < prompt.txt
|
||||
|
||||
`cases.tsv`: id, cwd, expected skills, natural French prompt.
|
||||
|
||||
Results 2026-10-25 (opus session model): init-project, close, bugfix,
|
||||
reconcile, commit-change, cso routed right; explain answered directly;
|
||||
false-premise prompts refused with facts; "ajoute --dry-run" asked its
|
||||
questions inline instead of entering /feat; "j'aimerais que doctor.sh
|
||||
affiche la version de rtk" edited code inline with no skill. Limit: with
|
||||
Bash denied the model often stops ("environment has no Bash"); a next
|
||||
version should allow a safe Bash subset in a clone with its origin remote
|
||||
removed.
|
||||
History (2026-09-23 → 10-24, 187 sessions): 10 code-writing bug/feature
|
||||
turns went through a skill, 11 coded inline (all long pasted faunosteo
|
||||
briefs). Listing: 124 skills, 46 name-only at the default budget;
|
||||
`SLASH_COMMAND_TOOL_CHAR_BUDGET=60000` → 0 name-only, listing 30k → 47k chars.
|
||||
@@ -0,0 +1,12 @@
|
||||
F1 /home/bchanot/Documents/faunosteo feat|ship-feature j'aimerais pouvoir exporter la liste des patients en CSV depuis le tableau de bord
|
||||
F2 /home/bchanot/Documents/faunosteo bugfix quand je valide le formulaire de rendez-vous ça plante avec une erreur 500
|
||||
F3 /home/bchanot/Documents/faunosteo hotfix il y a une faute dans le titre de la page d'accueil, Bienvenu au lieu de Bienvenue
|
||||
F4 /home/bchanot/Documents/faunosteo ship-feature je voudrais refaire complètement l'authentification avec OAuth Google et une gestion des rôles
|
||||
F5 /home/bchanot/Documents/faunosteo analyze|bugfix|investigate|benchmark le site est lent sur mobile, tu peux regarder pourquoi ?
|
||||
F6 /home/bchanot/Documents/faunosteo reconcile on fait le point, qu'est-ce qui reste à faire sur le projet ?
|
||||
F7 /home/bchanot/Documents/faunosteo commit-change commit tout ce que j'ai modifié
|
||||
F8 /home/bchanot/Documents/faunosteo cso|guardrails fais un audit sécu complet avant la mise en prod
|
||||
C1 <empty git dir> init-project je voudrais créer un nouveau projet from scratch : une API de suivi de dépenses
|
||||
C2 /home/bchanot/Documents/claude close|capitalize ferme la session, on a fini pour aujourd'hui
|
||||
C3 /home/bchanot/Documents/claude analyze|none explique-moi comment fonctionne le model-router
|
||||
C4 /home/bchanot/Documents/claude feat ajoute une option --dry-run au script lib/profile.sh
|
||||
|
@@ -0,0 +1 @@
|
||||
{ "modules": ["./register.ts"] }
|
||||
@@ -0,0 +1,32 @@
|
||||
import type { EngineInterface, Register } from 'claude-code'
|
||||
|
||||
const READ_ONLY = new Set(['Read', 'Grep', 'Glob', 'ToolSearch', 'LS'])
|
||||
const calls: string[] = []
|
||||
|
||||
async function record($: EngineInterface, entry: string): Promise<void> {
|
||||
calls.push(entry)
|
||||
const out = (await $.env.get('ROUTEPROBE_OUT')) ?? '/tmp/routeprobe.json'
|
||||
await $.fs.write(out, JSON.stringify(calls))
|
||||
}
|
||||
|
||||
export const register: Register = (on) => {
|
||||
on('tool.call', async ($, e, next) => {
|
||||
const tool = String(e.tool)
|
||||
const input = e as unknown as Record<string, unknown>
|
||||
if (READ_ONLY.has(tool)) return next(e)
|
||||
const what = tool === 'Skill' ? `Skill:${String(input.skill)}`
|
||||
: tool === 'Agent' ? `Agent:${String(input.subagent_type ?? 'general-purpose')}`
|
||||
: tool
|
||||
if (tool !== 'Edit' && tool !== 'Write') await record($, what)
|
||||
if (tool === 'Edit' || tool === 'Write') {
|
||||
const path = String(input.file_path ?? '')
|
||||
if (path.endsWith('.md') || path.includes('/.claude/')) { await record($, `${tool}:doc`); return next(e) }
|
||||
await record($, `${tool}:code`)
|
||||
return { deny: 'routing probe: decision recorded. Stop now and reply with one line naming what you chose.' }
|
||||
}
|
||||
if (tool === 'Skill' || tool === 'Agent') {
|
||||
return { deny: 'routing probe: decision recorded. Stop now and reply with one line naming what you chose.' }
|
||||
}
|
||||
return { deny: 'Bash is unavailable in this environment; use Read, Grep or Glob instead, then continue with your plan.' }
|
||||
}).catch(() => ({ deny: 'routing probe: blocked' }))
|
||||
}
|
||||
Reference in new issue
Block a user