feat(effort): entry level on every skill next to its model pin (BDR-108)

- lib/effort-pins.txt (map) + lib/effort-pins.sh (idempotent re-apply)
  replace the hardcoded brainstorming/writing-plans loop; called after the
  last vendoring step of install-plugins.sh AND update-all.sh (the resync
  dropped the pins until the next make plugin)
- design stack high uniform (last loaded wins), superpowers, agent-skills,
  21st pack pinned from the map; skills-perso low, pdf-translate medium,
  site-motion high
- doctrine: design stack loads paired with the first Read; one level per
  stack (CLAUDE.global.md, lib/effort-shift.md)
- lib/effort-audit.py prints thinking coverage per scope (sub-agent records
  carry no thinking count on ~94 % of requests)
- census map-driven + fixture suite lib/tests/effort-pins.test.sh; docs
  README/USAGE/CHANGELOG; contract + TODO plan
This commit is contained in:
bastien
2026-09-29 13:15:41 +02:00
parent 3fcc0c8211
commit c859ae256f
17 changed files with 376 additions and 33 deletions
+39 -14
View File
@@ -10,18 +10,21 @@ import sys
# Weights relative to input price.
WEIGHTS = {"in": 1.0, "cc": 1.25, "cr": 0.1, "out": 5.0}
FIELDS = ("in", "cc", "cr", "out", "think")
FIELDS = ("in", "cc", "cr", "out", "think", "nodet")
def usage_row(usage):
"""Map one API usage block to the five counted fields."""
details = usage.get("output_tokens_details") or {}
"""Map one API usage block to the counted fields. `nodet` marks a
record whose usage carries no output_tokens_details at all: no thinking
count was recorded (most sub-agent records), so `think` understates."""
details = usage.get("output_tokens_details")
return {
"in": usage.get("input_tokens", 0) or 0,
"cc": usage.get("cache_creation_input_tokens", 0) or 0,
"cr": usage.get("cache_read_input_tokens", 0) or 0,
"out": usage.get("output_tokens", 0) or 0,
"think": details.get("thinking_tokens", 0) or 0,
"think": (details or {}).get("thinking_tokens", 0) or 0,
"nodet": 0 if details else 1,
}
@@ -57,21 +60,27 @@ def weighted(counter):
return sum(counter[f] * WEIGHTS[f] for f in WEIGHTS)
def report(agg):
"""Print the per-key table, then the main/sub split and the thinking
share."""
total = collections.Counter()
for counter in agg.values():
total.update(counter)
total_w = weighted(total) or 1
def coverage(counter):
"""Share of requests whose usage carries a thinking count."""
return 100 * (1 - counter["nodet"] / max(counter["msgs"], 1))
def print_rows(agg, total_w):
"""One line per (scope, model, effort), costliest first."""
print(f"{'scope':5} {'model':22} {'effort':7} {'msgs':>6} {'think/msg':>9} "
f"{'think_tok':>10} {'out_tok':>10} {'cache_read':>12} {'%wcost':>7}")
f"{'think_tok':>10} {'out_tok':>10} {'cache_read':>12} {'%wcost':>7} "
f"{'%counted':>8}")
ranked = sorted(agg.items(), key=lambda kv: -weighted(kv[1]))
for (scope, model, effort), c in ranked:
per_msg = c["think"] / max(c["msgs"], 1)
print(f"{scope:5} {model:22} {effort:7} {c['msgs']:6d} "
f"{per_msg:9.0f} {c['think']:10d} {c['out']:10d} "
f"{c['cr']:12d} {100 * weighted(c) / total_w:6.1f}%")
f"{c['cr']:12d} {100 * weighted(c) / total_w:6.1f}% "
f"{coverage(c):7.0f}%")
def print_scopes(agg, total, total_w):
"""Main/sub split, thinking share and the coverage caveat."""
by_scope = collections.defaultdict(collections.Counter)
for (scope, _, _), c in agg.items():
by_scope[scope].update(c)
@@ -79,11 +88,27 @@ def report(agg):
print(f" {scope:5} weighted-cost "
f"{100 * weighted(c) / total_w:5.1f}% thinking "
f"{100 * c['think'] / max(total['think'], 1):5.1f}% "
f"requests {c['msgs']}")
f"requests {c['msgs']} thinking counted on "
f"{coverage(c):.0f}% of them")
print(f" thinking = "
f"{100 * total['think'] * WEIGHTS['out'] / total_w:.1f}% "
f"of weighted cost; cache reads = "
f"{100 * total['cr'] * WEIGHTS['cr'] / total_w:.1f}%")
low = [s for s, c in by_scope.items() if coverage(c) < 50]
if low:
print(f" CAVEAT: {', '.join(low)} records mostly carry no thinking "
f"count — their think columns are a floor, not a measure")
def report(agg):
"""Print the per-key table, then the main/sub split and the thinking
share."""
total = collections.Counter()
for counter in agg.values():
total.update(counter)
total_w = weighted(total) or 1
print_rows(agg, total_w)
print_scopes(agg, total, total_w)
def main():
+75
View File
@@ -0,0 +1,75 @@
#!/usr/bin/env bash
# lib/effort-pins.sh — re-apply the entry effort level on vendored skills
# (BDR-107 second axis, extended to every vendored external by BDR-108).
# Upstream copies carry no `effort:` and every vendoring step rewrites
# SKILL.md, so the level lives in lib/effort-pins.txt and this helper puts
# it back after the last vendoring step of install-plugins.sh and
# update-all.sh. Idempotent: same level → untouched, other level →
# replaced inside the frontmatter only, skill not vendored → skipped,
# malformed map line → rejected loudly, never applied. Placement inside the
# frontmatter has no effect on the harness, which reads the key anywhere.
#
# Usage: source it, then `apply_effort_pins [repo-root]`
# or standalone: bash lib/effort-pins.sh [repo-root]
# Exit 1 when at least one map line was rejected.
EFFORT_PINS_REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
EFFORT_PIN_LEVEL_RE='^(low|medium|high|xhigh|max)$'
EFFORT_PIN_NAME_RE='^[A-Za-z0-9][A-Za-z0-9._-]*$'
# Callers (install-plugins.sh, update-all.sh) define these; standalone
# runs get plain fallbacks.
declare -F ok >/dev/null || ok() { printf ' ok %s\n' "$*"; }
declare -F info >/dev/null || info() { printf ' info %s\n' "$*"; }
declare -F err >/dev/null || err() { printf ' ERR %s\n' "$*" >&2; }
# _effort_pin_current <skill-file> → prints the frontmatter effort, if any
_effort_pin_current() {
awk 'NR==1&&/^---$/{p=1;next} p&&/^---$/{exit} p' "$1" \
| sed -n 's/^effort: //p' | head -1
}
# _effort_pin_write <skill-file> <name> <level> — replace the frontmatter
# `effort:` line, or insert one after `name: <name>` (before the closing
# `---` when the frontmatter has no name line). Body lines never change.
_effort_pin_write() {
local file="$1" name="$2" level="$3"
awk -v n="$name" -v lvl="$level" '
NR==1 && /^---$/ { fm=1; print; next }
fm && /^---$/ {
if (!done) { print "effort: " lvl; done=1 }
fm=0; print; next
}
fm && /^effort: / { if (!done) { print "effort: " lvl; done=1 }; next }
fm && $0 == "name: " n { print; if (!done) { print "effort: " lvl; done=1 }; next }
{ print }
' "$file" > "$file.tmp" && mv "$file.tmp" "$file"
}
# apply_effort_pins [repo-root] — walk the map, pin every vendored skill
apply_effort_pins() {
local repo="${1:-$EFFORT_PINS_REPO}" map name level rest file
local applied=0 kept=0 rejected=0
map="$repo/lib/effort-pins.txt"
[ -f "$map" ] || { err "effort-pins: map missing: $map"; return 1; }
while read -r name level rest; do
case "$name" in ''|'#'*) continue ;; esac
if [ -n "$rest" ] || ! [[ "$name" =~ $EFFORT_PIN_NAME_RE ]] \
|| ! [[ "$level" =~ $EFFORT_PIN_LEVEL_RE ]]; then
err "effort-pins: rejected map line '$name $level $rest'"
rejected=$((rejected + 1)); continue
fi
file="$repo/skills-external/$name/SKILL.md"
[ -f "$file" ] || continue
if [ "$(_effort_pin_current "$file")" = "$level" ]; then
kept=$((kept + 1)); continue
fi
_effort_pin_write "$file" "$name" "$level" && applied=$((applied + 1))
done < "$map"
ok "effort-pins: $applied applied, $kept already at level"
[ "$rejected" -eq 0 ]
}
if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
apply_effort_pins "$@"
fi
+46
View File
@@ -0,0 +1,46 @@
# lib/effort-pins.txt — entry effort level of the vendored skills
# (skills-external/<name>/SKILL.md). Upstream copies carry no `effort:` and
# every resync rewrites SKILL.md, so the pin lives here and
# lib/effort-pins.sh re-applies it after the last vendoring step of
# install-plugins.sh and update-all.sh. One line = `<skill> <level>`,
# level in low|medium|high|xhigh|max. The census
# lib/tests/effort-routing.test.sh checks every vendored file against this
# map. Rungs (BDR-107, BDR-108): low = fix a line, run a script · medium =
# day-to-day · high = refactor, resisting bug · xhigh = architecture, audit
# before validation · max = stuck.
#
# superpowers (obra/superpowers, plugins.lock.json "superpowers")
brainstorming xhigh
writing-plans xhigh
requesting-code-review xhigh
subagent-driven-development high
writing-skills high
test-driven-development medium
using-git-worktrees low
#
# agent-skills (addyosmani/agent-skills, plugins.lock.json "agent-skills")
deprecation-and-migration high
ci-cd-and-automation medium
observability-and-instrumentation medium
#
# design stack — ONE level for every member: these skills load stacked in a
# single UI build and the last loaded wins (lib/effort-shift.md), so two
# levels in the stack would make the effort depend on load order.
# skills/site-motion (repo-authored) pins the same level in its frontmatter.
frontend-design high
emil-design-eng high
design-motion-principles high
21st-ui-build high
scroll-world-storytelling high
build-threejs-scroll-worlds high
scroll-scrubbed-visual-sequence high
scroll-scrubbed-word-reveal high
scroll-progress-timeline high
#
# 21st pack (`21st skills install`): tooling low, generation high, critique xhigh
21st-cli-use low
21st-registry low
21st-design-sync low
21st-ai high
21st-ui-explore high
21st-ui-review xhigh
+5
View File
@@ -24,6 +24,11 @@ max (stuck error, judged need).
Claude loads alone, such as `brainstorming` or `writing-plans`, applies
nothing). Last loaded wins, both directions. The prompt cache survives a
shift.
- **Stacked skills share one level**: skills that load together in one
build (the design stack) all pin the same level, since the last loaded
wins. Vendored externals get their level from `lib/effort-pins.txt`,
re-applied by `lib/effort-pins.sh` after every vendoring step; repo
skills carry it in their frontmatter.
- Dispatched agents run on their own `effort:` pin, never on a shift.
Unpinned agents inherit the level in force at dispatch.
- Headless sessions (`-p`, `claude agents`, SDK) ignore skill-level effort:
+60
View File
@@ -0,0 +1,60 @@
#!/usr/bin/env bash
# lib/tests/effort-pins.test.sh — lib/effort-pins.sh's apply_effort_pins():
# insert after `name:`, keep an equal level untouched, replace a different
# level inside the frontmatter only (a prose `effort:` in the body stays),
# skip a skill not vendored, insert before the closing `---` when the
# frontmatter has no name line, run idempotently, reject a bad level, a
# traversal name and a three-field line before writing anything, and
# parse the real map without error. All on a throwaway fixture repo.
set -u
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
LIB="$ROOT/lib/effort-pins.sh"
pass=0; fail=0
check() { if [ "$2" = "$3" ]; then pass=$((pass+1)); echo "PASS $1"
else fail=$((fail+1)); echo "FAIL $1: got[$2] want[$3]"; fi; }
fm_effort() { awk 'NR==1&&/^---$/{p=1;next} p&&/^---$/{exit} p' "$1" \
| sed -n 's/^effort: //p' | head -1; }
WORK="$(mktemp -d)"; trap 'rm -rf "$WORK"' EXIT
REPO="$WORK/repo"; EXT="$REPO/skills-external"
mkdir -p "$REPO/lib" "$EXT/alpha" "$EXT/beta" "$EXT/gamma" "$EXT/noname"
printf -- '---\nname: alpha\ndescription: a\n---\nbody\n' > "$EXT/alpha/SKILL.md"
printf -- '---\nname: beta\neffort: low\n---\nprose says effort: max here\n' > "$EXT/beta/SKILL.md"
printf -- '---\nname: gamma\neffort: low\n---\nbody\n' > "$EXT/gamma/SKILL.md"
printf -- '---\ndescription: no name line\n---\nbody\n' > "$EXT/noname/SKILL.md"
printf '# map\nalpha high\nbeta medium\ngamma low\nghost xhigh\nnoname low\n' > "$REPO/lib/effort-pins.txt"
gamma_before="$(cat "$EXT/gamma/SKILL.md")"
bash "$LIB" "$REPO" >/dev/null 2>&1; check T1-rc-clean "$?" 0
check T2-insert-after-name "$(sed -n '3p' "$EXT/alpha/SKILL.md")" "effort: high"
check T3-replace-in-frontmatter "$(fm_effort "$EXT/beta/SKILL.md")" "medium"
check T3b-body-prose-untouched "$(grep -c 'effort: max' "$EXT/beta/SKILL.md")" 1
check T3c-single-effort-line "$(grep -c '^effort:' "$EXT/beta/SKILL.md")" 1
check T4-equal-level-untouched "$(cat "$EXT/gamma/SKILL.md")" "$gamma_before"
check T5-missing-skill-skipped "$([ -e "$EXT/ghost" ] && echo created || echo absent)" absent
check T6-no-name-inserts-before-closing "$(sed -n '3p' "$EXT/noname/SKILL.md")" "effort: low"
check T6b-no-name-still-frontmatter "$(fm_effort "$EXT/noname/SKILL.md")" "low"
snap="$(cat "$EXT"/*/SKILL.md)"
bash "$LIB" "$REPO" >/dev/null 2>&1
check T7-idempotent "$(cat "$EXT"/*/SKILL.md)" "$snap"
check T7b-no-tmp-left "$(find "$EXT" -name '*.tmp' | wc -l)" 0
# rejections: nothing written, rc 1
for bad in 'alpha turbo' '../evil high' 'alpha high extra'; do
printf '%s\n' "$bad" > "$REPO/lib/effort-pins.txt"
out="$(bash "$LIB" "$REPO" 2>&1)"; rc=$?
check "T8-rejected[$bad]-rc" "$rc" 1
check "T8-rejected[$bad]-named" "$(printf '%s' "$out" | grep -c 'rejected map line')" 1
done
check T8b-tree-unchanged-after-rejections "$(cat "$EXT"/*/SKILL.md)" "$snap"
check T8c-no-evil-dir "$([ -e "$WORK/evil" ] && echo created || echo absent)" absent
# the real map parses: fixture repo with the real map and no vendored skill
mkdir -p "$WORK/real/lib" "$WORK/real/skills-external"
cp "$ROOT/lib/effort-pins.txt" "$WORK/real/lib/"
out="$(bash "$LIB" "$WORK/real" 2>&1)"; check T9-real-map-parses "$?" 0
check T9b-real-map-nothing-applied "$(printf '%s' "$out" | grep -c '0 applied, 0 already')" 1
check T10-missing-map-rc "$(bash "$LIB" "$WORK/nowhere" >/dev/null 2>&1; echo $?)" 1
echo "effort-pins: $pass pass, $fail fail"
[ "$fail" -eq 0 ]
+24 -9
View File
@@ -1,7 +1,7 @@
#!/usr/bin/env bash
# lib/tests/effort-routing.test.sh — census: effort tiering (BDR-107)
# agent pins, skill entry levels, shifter skills, orchestrator wiring, settings.
# shellcheck disable=SC2015 # A && ok || ko is deliberate here: ok/ko never fail, so C never masks a true A
# shellcheck disable=SC2015,SC2016 # A && ok || ko is deliberate (ok/ko never fail); '$REPO' locks are literal source text
set -u
R="$(cd "$(dirname "$0")/../.." && pwd)"
pass=0; fail=0
@@ -46,15 +46,30 @@ for s in status commit-change release-candidate doc capitalize close reconcile d
for s in gitflow prune-memory; do fm_has_effort "skills/$s/SKILL.md" medium; done
for s in feat hotfix bugfix refactor web-validate harden seo geo; do fm_has_effort "skills/$s/SKILL.md" high; done
for s in ship-feature init-project onboard tour audit-delta analyze code-clean client-handover; do fm_has_effort "skills/$s/SKILL.md" xhigh; done
# BDR-108 round: the three repo skills that had no level
fm_has_effort "skills/skills-perso/SKILL.md" low
fm_has_effort "skills/pdf-translate/SKILL.md" medium
fm_has_effort "skills/site-motion/SKILL.md" high
# ── 9) vendored superpowers carry xhigh (spec D3). The files live in skills-external/ (gitignored,
# machine-owned), so the durable artifact is the install-plugins.sh re-apply; the frontmatter
# check skips VISIBLY when the skill is not vendored yet (fresh clone before make plugin).
for s in brainstorming writing-plans; do
if [ -f "$R/skills-external/$s/SKILL.md" ]; then fm_has_effort "skills-external/$s/SKILL.md" xhigh
# ── 9) vendored externals carry the level of lib/effort-pins.txt (BDR-108). The files live in
# skills-external/ (gitignored, machine-owned): the durable artifact is the map + the re-apply
# after the last vendoring step of install-plugins.sh AND update-all.sh; a skill not vendored
# yet SKIPs visibly (fresh clone before make plugin).
while read -r s lvl _; do
case "$s" in ''|'#'*) continue ;; esac
if [ -f "$R/skills-external/$s/SKILL.md" ]; then fm_has_effort "skills-external/$s/SKILL.md" "$lvl"
else printf 'SKIP skills-external/%s/SKILL.md not vendored yet (run make plugin)\n' "$s"; fi
done
has "install-plugins.sh" 'effort: xhigh'
done < "$R/lib/effort-pins.txt"
has "lib/effort-pins.txt" 'brainstorming xhigh'; has "lib/effort-pins.txt" 'writing-plans xhigh'
has "install-plugins.sh" 'apply_effort_pins "$REPO"'; has "update-all.sh" 'apply_effort_pins "$REPO"'
lacks "install-plugins.sh" 'for _s in brainstorming writing-plans; do'
[ -x "$R/lib/effort-pins.sh" ] && ok || ko "lib/effort-pins.sh missing or not executable"
# 9b) design stack = ONE level (last loaded wins); site-motion (repo skill) pins the same one
stack_levels() { awk '/^# design stack/{f=1;next} f&&/^#$/{f=0} f&&!/^#/&&NF==2{print $2}' "$R/lib/effort-pins.txt" | sort -u; }
[ "$(stack_levels | wc -l)" -eq 1 ] && ok || ko "design stack must share ONE level in lib/effort-pins.txt (got: $(stack_levels | tr '\n' ' '))"
[ "$(stack_levels | wc -l)" -ge 1 ] && fm_has_effort "skills/site-motion/SKILL.md" "$(stack_levels | head -1)"
has "lib/effort-shift.md" 'Stacked skills share one level'
has "CLAUDE.global.md" 'lib/effort-pins.txt'
# ── 5) shifter skills + include (spec D4)
for l in low medium high xhigh max; do fm_has_effort "skills/effort-$l/SKILL.md" "$l"; has "skills/effort-$l/SKILL.md" "name: effort-$l"; done
@@ -101,7 +116,7 @@ has "lib/effort-shift.md" 'Before any built-in or unpinned dispatch'
has "lib/model-gate.md" 'built-ins inherit the effort in force'
has "skills/ship-feature/SKILL.md" 'effort-shift: error recovery'
for s in feat hotfix bugfix seo geo harden web-validate ship-feature init-project onboard code-clean audit-delta; do has "skills/$s/SKILL.md" 'effort-shift: own level before the challenge'; done
has "install-plugins.sh" 'for _s in brainstorming writing-plans; do'
has "update-all.sh" 'source "$REPO/lib/effort-pins.sh"'
# ── summary (later tasks insert their locks ABOVE this line)
printf 'effort-routing census: %d pass, %d fail\n' "$pass" "$fail"