From 8e575c7e46ad01850e070260e38877623a1a8e17 Mon Sep 17 00:00:00 2001 From: bchanot Date: Sun, 25 Oct 2026 01:36:30 +0200 Subject: [PATCH] chore(memory): routing probe bench and its measured results --- .../.claude-plugin/plugin.json | 1 + .../plans/2026-10-25-routing-probe/README.md | 25 +++++++++++++++ .../plans/2026-10-25-routing-probe/cases.tsv | 12 +++++++ .../2026-10-25-routing-probe/hooks/hooks.json | 1 + .../hooks/register.ts | 32 +++++++++++++++++++ 5 files changed, 71 insertions(+) create mode 100644 .claude/tasks/plans/2026-10-25-routing-probe/.claude-plugin/plugin.json create mode 100644 .claude/tasks/plans/2026-10-25-routing-probe/README.md create mode 100644 .claude/tasks/plans/2026-10-25-routing-probe/cases.tsv create mode 100644 .claude/tasks/plans/2026-10-25-routing-probe/hooks/hooks.json create mode 100644 .claude/tasks/plans/2026-10-25-routing-probe/hooks/register.ts diff --git a/.claude/tasks/plans/2026-10-25-routing-probe/.claude-plugin/plugin.json b/.claude/tasks/plans/2026-10-25-routing-probe/.claude-plugin/plugin.json new file mode 100644 index 0000000..a3d6c4a --- /dev/null +++ b/.claude/tasks/plans/2026-10-25-routing-probe/.claude-plugin/plugin.json @@ -0,0 +1 @@ +{ "name": "routeprobe", "version": "0.1.0", "description": "Records routing decisions and blocks every effect (throwaway)" } diff --git a/.claude/tasks/plans/2026-10-25-routing-probe/README.md b/.claude/tasks/plans/2026-10-25-routing-probe/README.md new file mode 100644 index 0000000..edb2d19 --- /dev/null +++ b/.claude/tasks/plans/2026-10-25-routing-probe/README.md @@ -0,0 +1,25 @@ +# routing probe (bench for skill auto-routing, 2026-10-25) + +A throwaway mod that records the first routing decision of a fresh headless +session and blocks every effect: Skill and Agent calls are recorded then +denied, a code Edit/Write is recorded then denied, doc writes (.md, +`.claude/**`) are allowed, Bash is softly denied, Read/Grep/Glob pass. +Run each case in a throwaway clone (never the real repo), output path in +`ROUTEPROBE_OUT`: + + ROUTEPROBE_OUT=/tmp/r.json claude -p --plugin-dir .claude/tasks/plans/2026-10-25-routing-probe < prompt.txt + +`cases.tsv`: id, cwd, expected skills, natural French prompt. + +Results 2026-10-25 (opus session model): init-project, close, bugfix, +reconcile, commit-change, cso routed right; explain answered directly; +false-premise prompts refused with facts; "ajoute --dry-run" asked its +questions inline instead of entering /feat; "j'aimerais que doctor.sh +affiche la version de rtk" edited code inline with no skill. Limit: with +Bash denied the model often stops ("environment has no Bash"); a next +version should allow a safe Bash subset in a clone with its origin remote +removed. +History (2026-09-23 → 10-24, 187 sessions): 10 code-writing bug/feature +turns went through a skill, 11 coded inline (all long pasted faunosteo +briefs). Listing: 124 skills, 46 name-only at the default budget; +`SLASH_COMMAND_TOOL_CHAR_BUDGET=60000` → 0 name-only, listing 30k → 47k chars. diff --git a/.claude/tasks/plans/2026-10-25-routing-probe/cases.tsv b/.claude/tasks/plans/2026-10-25-routing-probe/cases.tsv new file mode 100644 index 0000000..97baf12 --- /dev/null +++ b/.claude/tasks/plans/2026-10-25-routing-probe/cases.tsv @@ -0,0 +1,12 @@ +F1 /home/bchanot/Documents/faunosteo feat|ship-feature j'aimerais pouvoir exporter la liste des patients en CSV depuis le tableau de bord +F2 /home/bchanot/Documents/faunosteo bugfix quand je valide le formulaire de rendez-vous ça plante avec une erreur 500 +F3 /home/bchanot/Documents/faunosteo hotfix il y a une faute dans le titre de la page d'accueil, Bienvenu au lieu de Bienvenue +F4 /home/bchanot/Documents/faunosteo ship-feature je voudrais refaire complètement l'authentification avec OAuth Google et une gestion des rôles +F5 /home/bchanot/Documents/faunosteo analyze|bugfix|investigate|benchmark le site est lent sur mobile, tu peux regarder pourquoi ? +F6 /home/bchanot/Documents/faunosteo reconcile on fait le point, qu'est-ce qui reste à faire sur le projet ? +F7 /home/bchanot/Documents/faunosteo commit-change commit tout ce que j'ai modifié +F8 /home/bchanot/Documents/faunosteo cso|guardrails fais un audit sécu complet avant la mise en prod +C1 init-project je voudrais créer un nouveau projet from scratch : une API de suivi de dépenses +C2 /home/bchanot/Documents/claude close|capitalize ferme la session, on a fini pour aujourd'hui +C3 /home/bchanot/Documents/claude analyze|none explique-moi comment fonctionne le model-router +C4 /home/bchanot/Documents/claude feat ajoute une option --dry-run au script lib/profile.sh diff --git a/.claude/tasks/plans/2026-10-25-routing-probe/hooks/hooks.json b/.claude/tasks/plans/2026-10-25-routing-probe/hooks/hooks.json new file mode 100644 index 0000000..5fcba7c --- /dev/null +++ b/.claude/tasks/plans/2026-10-25-routing-probe/hooks/hooks.json @@ -0,0 +1 @@ +{ "modules": ["./register.ts"] } diff --git a/.claude/tasks/plans/2026-10-25-routing-probe/hooks/register.ts b/.claude/tasks/plans/2026-10-25-routing-probe/hooks/register.ts new file mode 100644 index 0000000..647a8e6 --- /dev/null +++ b/.claude/tasks/plans/2026-10-25-routing-probe/hooks/register.ts @@ -0,0 +1,32 @@ +import type { EngineInterface, Register } from 'claude-code' + +const READ_ONLY = new Set(['Read', 'Grep', 'Glob', 'ToolSearch', 'LS']) +const calls: string[] = [] + +async function record($: EngineInterface, entry: string): Promise { + calls.push(entry) + const out = (await $.env.get('ROUTEPROBE_OUT')) ?? '/tmp/routeprobe.json' + await $.fs.write(out, JSON.stringify(calls)) +} + +export const register: Register = (on) => { + on('tool.call', async ($, e, next) => { + const tool = String(e.tool) + const input = e as unknown as Record + if (READ_ONLY.has(tool)) return next(e) + const what = tool === 'Skill' ? `Skill:${String(input.skill)}` + : tool === 'Agent' ? `Agent:${String(input.subagent_type ?? 'general-purpose')}` + : tool + if (tool !== 'Edit' && tool !== 'Write') await record($, what) + if (tool === 'Edit' || tool === 'Write') { + const path = String(input.file_path ?? '') + if (path.endsWith('.md') || path.includes('/.claude/')) { await record($, `${tool}:doc`); return next(e) } + await record($, `${tool}:code`) + return { deny: 'routing probe: decision recorded. Stop now and reply with one line naming what you chose.' } + } + if (tool === 'Skill' || tool === 'Agent') { + return { deny: 'routing probe: decision recorded. Stop now and reply with one line naming what you chose.' } + } + return { deny: 'Bash is unavailable in this environment; use Read, Grep or Glob instead, then continue with your plan.' } + }).catch(() => ({ deny: 'routing probe: blocked' })) +}