forked from bchanot/claude
feat(effort): entry level on every skill next to its model pin (BDR-108)
- lib/effort-pins.txt (map) + lib/effort-pins.sh (idempotent re-apply) replace the hardcoded brainstorming/writing-plans loop; called after the last vendoring step of install-plugins.sh AND update-all.sh (the resync dropped the pins until the next make plugin) - design stack high uniform (last loaded wins), superpowers, agent-skills, 21st pack pinned from the map; skills-perso low, pdf-translate medium, site-motion high - doctrine: design stack loads paired with the first Read; one level per stack (CLAUDE.global.md, lib/effort-shift.md) - lib/effort-audit.py prints thinking coverage per scope (sub-agent records carry no thinking count on ~94 % of requests) - census map-driven + fixture suite lib/tests/effort-pins.test.sh; docs README/USAGE/CHANGELOG; contract + TODO plan
This commit is contained in:
+39
-14
@@ -10,18 +10,21 @@ import sys
|
||||
|
||||
# Weights relative to input price.
|
||||
WEIGHTS = {"in": 1.0, "cc": 1.25, "cr": 0.1, "out": 5.0}
|
||||
FIELDS = ("in", "cc", "cr", "out", "think")
|
||||
FIELDS = ("in", "cc", "cr", "out", "think", "nodet")
|
||||
|
||||
|
||||
def usage_row(usage):
|
||||
"""Map one API usage block to the five counted fields."""
|
||||
details = usage.get("output_tokens_details") or {}
|
||||
"""Map one API usage block to the counted fields. `nodet` marks a
|
||||
record whose usage carries no output_tokens_details at all: no thinking
|
||||
count was recorded (most sub-agent records), so `think` understates."""
|
||||
details = usage.get("output_tokens_details")
|
||||
return {
|
||||
"in": usage.get("input_tokens", 0) or 0,
|
||||
"cc": usage.get("cache_creation_input_tokens", 0) or 0,
|
||||
"cr": usage.get("cache_read_input_tokens", 0) or 0,
|
||||
"out": usage.get("output_tokens", 0) or 0,
|
||||
"think": details.get("thinking_tokens", 0) or 0,
|
||||
"think": (details or {}).get("thinking_tokens", 0) or 0,
|
||||
"nodet": 0 if details else 1,
|
||||
}
|
||||
|
||||
|
||||
@@ -57,21 +60,27 @@ def weighted(counter):
|
||||
return sum(counter[f] * WEIGHTS[f] for f in WEIGHTS)
|
||||
|
||||
|
||||
def report(agg):
|
||||
"""Print the per-key table, then the main/sub split and the thinking
|
||||
share."""
|
||||
total = collections.Counter()
|
||||
for counter in agg.values():
|
||||
total.update(counter)
|
||||
total_w = weighted(total) or 1
|
||||
def coverage(counter):
|
||||
"""Share of requests whose usage carries a thinking count."""
|
||||
return 100 * (1 - counter["nodet"] / max(counter["msgs"], 1))
|
||||
|
||||
|
||||
def print_rows(agg, total_w):
|
||||
"""One line per (scope, model, effort), costliest first."""
|
||||
print(f"{'scope':5} {'model':22} {'effort':7} {'msgs':>6} {'think/msg':>9} "
|
||||
f"{'think_tok':>10} {'out_tok':>10} {'cache_read':>12} {'%wcost':>7}")
|
||||
f"{'think_tok':>10} {'out_tok':>10} {'cache_read':>12} {'%wcost':>7} "
|
||||
f"{'%counted':>8}")
|
||||
ranked = sorted(agg.items(), key=lambda kv: -weighted(kv[1]))
|
||||
for (scope, model, effort), c in ranked:
|
||||
per_msg = c["think"] / max(c["msgs"], 1)
|
||||
print(f"{scope:5} {model:22} {effort:7} {c['msgs']:6d} "
|
||||
f"{per_msg:9.0f} {c['think']:10d} {c['out']:10d} "
|
||||
f"{c['cr']:12d} {100 * weighted(c) / total_w:6.1f}%")
|
||||
f"{c['cr']:12d} {100 * weighted(c) / total_w:6.1f}% "
|
||||
f"{coverage(c):7.0f}%")
|
||||
|
||||
|
||||
def print_scopes(agg, total, total_w):
|
||||
"""Main/sub split, thinking share and the coverage caveat."""
|
||||
by_scope = collections.defaultdict(collections.Counter)
|
||||
for (scope, _, _), c in agg.items():
|
||||
by_scope[scope].update(c)
|
||||
@@ -79,11 +88,27 @@ def report(agg):
|
||||
print(f" {scope:5} weighted-cost "
|
||||
f"{100 * weighted(c) / total_w:5.1f}% thinking "
|
||||
f"{100 * c['think'] / max(total['think'], 1):5.1f}% "
|
||||
f"requests {c['msgs']}")
|
||||
f"requests {c['msgs']} thinking counted on "
|
||||
f"{coverage(c):.0f}% of them")
|
||||
print(f" thinking = "
|
||||
f"{100 * total['think'] * WEIGHTS['out'] / total_w:.1f}% "
|
||||
f"of weighted cost; cache reads = "
|
||||
f"{100 * total['cr'] * WEIGHTS['cr'] / total_w:.1f}%")
|
||||
low = [s for s, c in by_scope.items() if coverage(c) < 50]
|
||||
if low:
|
||||
print(f" CAVEAT: {', '.join(low)} records mostly carry no thinking "
|
||||
f"count — their think columns are a floor, not a measure")
|
||||
|
||||
|
||||
def report(agg):
|
||||
"""Print the per-key table, then the main/sub split and the thinking
|
||||
share."""
|
||||
total = collections.Counter()
|
||||
for counter in agg.values():
|
||||
total.update(counter)
|
||||
total_w = weighted(total) or 1
|
||||
print_rows(agg, total_w)
|
||||
print_scopes(agg, total, total_w)
|
||||
|
||||
|
||||
def main():
|
||||
|
||||
Executable
+75
@@ -0,0 +1,75 @@
|
||||
#!/usr/bin/env bash
|
||||
# lib/effort-pins.sh — re-apply the entry effort level on vendored skills
|
||||
# (BDR-107 second axis, extended to every vendored external by BDR-108).
|
||||
# Upstream copies carry no `effort:` and every vendoring step rewrites
|
||||
# SKILL.md, so the level lives in lib/effort-pins.txt and this helper puts
|
||||
# it back after the last vendoring step of install-plugins.sh and
|
||||
# update-all.sh. Idempotent: same level → untouched, other level →
|
||||
# replaced inside the frontmatter only, skill not vendored → skipped,
|
||||
# malformed map line → rejected loudly, never applied. Placement inside the
|
||||
# frontmatter has no effect on the harness, which reads the key anywhere.
|
||||
#
|
||||
# Usage: source it, then `apply_effort_pins [repo-root]`
|
||||
# or standalone: bash lib/effort-pins.sh [repo-root]
|
||||
# Exit 1 when at least one map line was rejected.
|
||||
|
||||
EFFORT_PINS_REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
EFFORT_PIN_LEVEL_RE='^(low|medium|high|xhigh|max)$'
|
||||
EFFORT_PIN_NAME_RE='^[A-Za-z0-9][A-Za-z0-9._-]*$'
|
||||
|
||||
# Callers (install-plugins.sh, update-all.sh) define these; standalone
|
||||
# runs get plain fallbacks.
|
||||
declare -F ok >/dev/null || ok() { printf ' ok %s\n' "$*"; }
|
||||
declare -F info >/dev/null || info() { printf ' info %s\n' "$*"; }
|
||||
declare -F err >/dev/null || err() { printf ' ERR %s\n' "$*" >&2; }
|
||||
|
||||
# _effort_pin_current <skill-file> → prints the frontmatter effort, if any
|
||||
_effort_pin_current() {
|
||||
awk 'NR==1&&/^---$/{p=1;next} p&&/^---$/{exit} p' "$1" \
|
||||
| sed -n 's/^effort: //p' | head -1
|
||||
}
|
||||
|
||||
# _effort_pin_write <skill-file> <name> <level> — replace the frontmatter
|
||||
# `effort:` line, or insert one after `name: <name>` (before the closing
|
||||
# `---` when the frontmatter has no name line). Body lines never change.
|
||||
_effort_pin_write() {
|
||||
local file="$1" name="$2" level="$3"
|
||||
awk -v n="$name" -v lvl="$level" '
|
||||
NR==1 && /^---$/ { fm=1; print; next }
|
||||
fm && /^---$/ {
|
||||
if (!done) { print "effort: " lvl; done=1 }
|
||||
fm=0; print; next
|
||||
}
|
||||
fm && /^effort: / { if (!done) { print "effort: " lvl; done=1 }; next }
|
||||
fm && $0 == "name: " n { print; if (!done) { print "effort: " lvl; done=1 }; next }
|
||||
{ print }
|
||||
' "$file" > "$file.tmp" && mv "$file.tmp" "$file"
|
||||
}
|
||||
|
||||
# apply_effort_pins [repo-root] — walk the map, pin every vendored skill
|
||||
apply_effort_pins() {
|
||||
local repo="${1:-$EFFORT_PINS_REPO}" map name level rest file
|
||||
local applied=0 kept=0 rejected=0
|
||||
map="$repo/lib/effort-pins.txt"
|
||||
[ -f "$map" ] || { err "effort-pins: map missing: $map"; return 1; }
|
||||
while read -r name level rest; do
|
||||
case "$name" in ''|'#'*) continue ;; esac
|
||||
if [ -n "$rest" ] || ! [[ "$name" =~ $EFFORT_PIN_NAME_RE ]] \
|
||||
|| ! [[ "$level" =~ $EFFORT_PIN_LEVEL_RE ]]; then
|
||||
err "effort-pins: rejected map line '$name $level $rest'"
|
||||
rejected=$((rejected + 1)); continue
|
||||
fi
|
||||
file="$repo/skills-external/$name/SKILL.md"
|
||||
[ -f "$file" ] || continue
|
||||
if [ "$(_effort_pin_current "$file")" = "$level" ]; then
|
||||
kept=$((kept + 1)); continue
|
||||
fi
|
||||
_effort_pin_write "$file" "$name" "$level" && applied=$((applied + 1))
|
||||
done < "$map"
|
||||
ok "effort-pins: $applied applied, $kept already at level"
|
||||
[ "$rejected" -eq 0 ]
|
||||
}
|
||||
|
||||
if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
|
||||
apply_effort_pins "$@"
|
||||
fi
|
||||
@@ -0,0 +1,46 @@
|
||||
# lib/effort-pins.txt — entry effort level of the vendored skills
|
||||
# (skills-external/<name>/SKILL.md). Upstream copies carry no `effort:` and
|
||||
# every resync rewrites SKILL.md, so the pin lives here and
|
||||
# lib/effort-pins.sh re-applies it after the last vendoring step of
|
||||
# install-plugins.sh and update-all.sh. One line = `<skill> <level>`,
|
||||
# level in low|medium|high|xhigh|max. The census
|
||||
# lib/tests/effort-routing.test.sh checks every vendored file against this
|
||||
# map. Rungs (BDR-107, BDR-108): low = fix a line, run a script · medium =
|
||||
# day-to-day · high = refactor, resisting bug · xhigh = architecture, audit
|
||||
# before validation · max = stuck.
|
||||
#
|
||||
# superpowers (obra/superpowers, plugins.lock.json "superpowers")
|
||||
brainstorming xhigh
|
||||
writing-plans xhigh
|
||||
requesting-code-review xhigh
|
||||
subagent-driven-development high
|
||||
writing-skills high
|
||||
test-driven-development medium
|
||||
using-git-worktrees low
|
||||
#
|
||||
# agent-skills (addyosmani/agent-skills, plugins.lock.json "agent-skills")
|
||||
deprecation-and-migration high
|
||||
ci-cd-and-automation medium
|
||||
observability-and-instrumentation medium
|
||||
#
|
||||
# design stack — ONE level for every member: these skills load stacked in a
|
||||
# single UI build and the last loaded wins (lib/effort-shift.md), so two
|
||||
# levels in the stack would make the effort depend on load order.
|
||||
# skills/site-motion (repo-authored) pins the same level in its frontmatter.
|
||||
frontend-design high
|
||||
emil-design-eng high
|
||||
design-motion-principles high
|
||||
21st-ui-build high
|
||||
scroll-world-storytelling high
|
||||
build-threejs-scroll-worlds high
|
||||
scroll-scrubbed-visual-sequence high
|
||||
scroll-scrubbed-word-reveal high
|
||||
scroll-progress-timeline high
|
||||
#
|
||||
# 21st pack (`21st skills install`): tooling low, generation high, critique xhigh
|
||||
21st-cli-use low
|
||||
21st-registry low
|
||||
21st-design-sync low
|
||||
21st-ai high
|
||||
21st-ui-explore high
|
||||
21st-ui-review xhigh
|
||||
@@ -24,6 +24,11 @@ max (stuck error, judged need).
|
||||
Claude loads alone, such as `brainstorming` or `writing-plans`, applies
|
||||
nothing). Last loaded wins, both directions. The prompt cache survives a
|
||||
shift.
|
||||
- **Stacked skills share one level**: skills that load together in one
|
||||
build (the design stack) all pin the same level, since the last loaded
|
||||
wins. Vendored externals get their level from `lib/effort-pins.txt`,
|
||||
re-applied by `lib/effort-pins.sh` after every vendoring step; repo
|
||||
skills carry it in their frontmatter.
|
||||
- Dispatched agents run on their own `effort:` pin, never on a shift.
|
||||
Unpinned agents inherit the level in force at dispatch.
|
||||
- Headless sessions (`-p`, `claude agents`, SDK) ignore skill-level effort:
|
||||
|
||||
Executable
+60
@@ -0,0 +1,60 @@
|
||||
#!/usr/bin/env bash
|
||||
# lib/tests/effort-pins.test.sh — lib/effort-pins.sh's apply_effort_pins():
|
||||
# insert after `name:`, keep an equal level untouched, replace a different
|
||||
# level inside the frontmatter only (a prose `effort:` in the body stays),
|
||||
# skip a skill not vendored, insert before the closing `---` when the
|
||||
# frontmatter has no name line, run idempotently, reject a bad level, a
|
||||
# traversal name and a three-field line before writing anything, and
|
||||
# parse the real map without error. All on a throwaway fixture repo.
|
||||
set -u
|
||||
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
|
||||
LIB="$ROOT/lib/effort-pins.sh"
|
||||
pass=0; fail=0
|
||||
check() { if [ "$2" = "$3" ]; then pass=$((pass+1)); echo "PASS $1"
|
||||
else fail=$((fail+1)); echo "FAIL $1: got[$2] want[$3]"; fi; }
|
||||
fm_effort() { awk 'NR==1&&/^---$/{p=1;next} p&&/^---$/{exit} p' "$1" \
|
||||
| sed -n 's/^effort: //p' | head -1; }
|
||||
|
||||
WORK="$(mktemp -d)"; trap 'rm -rf "$WORK"' EXIT
|
||||
REPO="$WORK/repo"; EXT="$REPO/skills-external"
|
||||
mkdir -p "$REPO/lib" "$EXT/alpha" "$EXT/beta" "$EXT/gamma" "$EXT/noname"
|
||||
printf -- '---\nname: alpha\ndescription: a\n---\nbody\n' > "$EXT/alpha/SKILL.md"
|
||||
printf -- '---\nname: beta\neffort: low\n---\nprose says effort: max here\n' > "$EXT/beta/SKILL.md"
|
||||
printf -- '---\nname: gamma\neffort: low\n---\nbody\n' > "$EXT/gamma/SKILL.md"
|
||||
printf -- '---\ndescription: no name line\n---\nbody\n' > "$EXT/noname/SKILL.md"
|
||||
printf '# map\nalpha high\nbeta medium\ngamma low\nghost xhigh\nnoname low\n' > "$REPO/lib/effort-pins.txt"
|
||||
gamma_before="$(cat "$EXT/gamma/SKILL.md")"
|
||||
|
||||
bash "$LIB" "$REPO" >/dev/null 2>&1; check T1-rc-clean "$?" 0
|
||||
check T2-insert-after-name "$(sed -n '3p' "$EXT/alpha/SKILL.md")" "effort: high"
|
||||
check T3-replace-in-frontmatter "$(fm_effort "$EXT/beta/SKILL.md")" "medium"
|
||||
check T3b-body-prose-untouched "$(grep -c 'effort: max' "$EXT/beta/SKILL.md")" 1
|
||||
check T3c-single-effort-line "$(grep -c '^effort:' "$EXT/beta/SKILL.md")" 1
|
||||
check T4-equal-level-untouched "$(cat "$EXT/gamma/SKILL.md")" "$gamma_before"
|
||||
check T5-missing-skill-skipped "$([ -e "$EXT/ghost" ] && echo created || echo absent)" absent
|
||||
check T6-no-name-inserts-before-closing "$(sed -n '3p' "$EXT/noname/SKILL.md")" "effort: low"
|
||||
check T6b-no-name-still-frontmatter "$(fm_effort "$EXT/noname/SKILL.md")" "low"
|
||||
snap="$(cat "$EXT"/*/SKILL.md)"
|
||||
bash "$LIB" "$REPO" >/dev/null 2>&1
|
||||
check T7-idempotent "$(cat "$EXT"/*/SKILL.md)" "$snap"
|
||||
check T7b-no-tmp-left "$(find "$EXT" -name '*.tmp' | wc -l)" 0
|
||||
|
||||
# rejections: nothing written, rc 1
|
||||
for bad in 'alpha turbo' '../evil high' 'alpha high extra'; do
|
||||
printf '%s\n' "$bad" > "$REPO/lib/effort-pins.txt"
|
||||
out="$(bash "$LIB" "$REPO" 2>&1)"; rc=$?
|
||||
check "T8-rejected[$bad]-rc" "$rc" 1
|
||||
check "T8-rejected[$bad]-named" "$(printf '%s' "$out" | grep -c 'rejected map line')" 1
|
||||
done
|
||||
check T8b-tree-unchanged-after-rejections "$(cat "$EXT"/*/SKILL.md)" "$snap"
|
||||
check T8c-no-evil-dir "$([ -e "$WORK/evil" ] && echo created || echo absent)" absent
|
||||
|
||||
# the real map parses: fixture repo with the real map and no vendored skill
|
||||
mkdir -p "$WORK/real/lib" "$WORK/real/skills-external"
|
||||
cp "$ROOT/lib/effort-pins.txt" "$WORK/real/lib/"
|
||||
out="$(bash "$LIB" "$WORK/real" 2>&1)"; check T9-real-map-parses "$?" 0
|
||||
check T9b-real-map-nothing-applied "$(printf '%s' "$out" | grep -c '0 applied, 0 already')" 1
|
||||
check T10-missing-map-rc "$(bash "$LIB" "$WORK/nowhere" >/dev/null 2>&1; echo $?)" 1
|
||||
|
||||
echo "effort-pins: $pass pass, $fail fail"
|
||||
[ "$fail" -eq 0 ]
|
||||
@@ -1,7 +1,7 @@
|
||||
#!/usr/bin/env bash
|
||||
# lib/tests/effort-routing.test.sh — census: effort tiering (BDR-107)
|
||||
# agent pins, skill entry levels, shifter skills, orchestrator wiring, settings.
|
||||
# shellcheck disable=SC2015 # A && ok || ko is deliberate here: ok/ko never fail, so C never masks a true A
|
||||
# shellcheck disable=SC2015,SC2016 # A && ok || ko is deliberate (ok/ko never fail); '$REPO' locks are literal source text
|
||||
set -u
|
||||
R="$(cd "$(dirname "$0")/../.." && pwd)"
|
||||
pass=0; fail=0
|
||||
@@ -46,15 +46,30 @@ for s in status commit-change release-candidate doc capitalize close reconcile d
|
||||
for s in gitflow prune-memory; do fm_has_effort "skills/$s/SKILL.md" medium; done
|
||||
for s in feat hotfix bugfix refactor web-validate harden seo geo; do fm_has_effort "skills/$s/SKILL.md" high; done
|
||||
for s in ship-feature init-project onboard tour audit-delta analyze code-clean client-handover; do fm_has_effort "skills/$s/SKILL.md" xhigh; done
|
||||
# BDR-108 round: the three repo skills that had no level
|
||||
fm_has_effort "skills/skills-perso/SKILL.md" low
|
||||
fm_has_effort "skills/pdf-translate/SKILL.md" medium
|
||||
fm_has_effort "skills/site-motion/SKILL.md" high
|
||||
|
||||
# ── 9) vendored superpowers carry xhigh (spec D3). The files live in skills-external/ (gitignored,
|
||||
# machine-owned), so the durable artifact is the install-plugins.sh re-apply; the frontmatter
|
||||
# check skips VISIBLY when the skill is not vendored yet (fresh clone before make plugin).
|
||||
for s in brainstorming writing-plans; do
|
||||
if [ -f "$R/skills-external/$s/SKILL.md" ]; then fm_has_effort "skills-external/$s/SKILL.md" xhigh
|
||||
# ── 9) vendored externals carry the level of lib/effort-pins.txt (BDR-108). The files live in
|
||||
# skills-external/ (gitignored, machine-owned): the durable artifact is the map + the re-apply
|
||||
# after the last vendoring step of install-plugins.sh AND update-all.sh; a skill not vendored
|
||||
# yet SKIPs visibly (fresh clone before make plugin).
|
||||
while read -r s lvl _; do
|
||||
case "$s" in ''|'#'*) continue ;; esac
|
||||
if [ -f "$R/skills-external/$s/SKILL.md" ]; then fm_has_effort "skills-external/$s/SKILL.md" "$lvl"
|
||||
else printf 'SKIP skills-external/%s/SKILL.md not vendored yet (run make plugin)\n' "$s"; fi
|
||||
done
|
||||
has "install-plugins.sh" 'effort: xhigh'
|
||||
done < "$R/lib/effort-pins.txt"
|
||||
has "lib/effort-pins.txt" 'brainstorming xhigh'; has "lib/effort-pins.txt" 'writing-plans xhigh'
|
||||
has "install-plugins.sh" 'apply_effort_pins "$REPO"'; has "update-all.sh" 'apply_effort_pins "$REPO"'
|
||||
lacks "install-plugins.sh" 'for _s in brainstorming writing-plans; do'
|
||||
[ -x "$R/lib/effort-pins.sh" ] && ok || ko "lib/effort-pins.sh missing or not executable"
|
||||
# 9b) design stack = ONE level (last loaded wins); site-motion (repo skill) pins the same one
|
||||
stack_levels() { awk '/^# design stack/{f=1;next} f&&/^#$/{f=0} f&&!/^#/&&NF==2{print $2}' "$R/lib/effort-pins.txt" | sort -u; }
|
||||
[ "$(stack_levels | wc -l)" -eq 1 ] && ok || ko "design stack must share ONE level in lib/effort-pins.txt (got: $(stack_levels | tr '\n' ' '))"
|
||||
[ "$(stack_levels | wc -l)" -ge 1 ] && fm_has_effort "skills/site-motion/SKILL.md" "$(stack_levels | head -1)"
|
||||
has "lib/effort-shift.md" 'Stacked skills share one level'
|
||||
has "CLAUDE.global.md" 'lib/effort-pins.txt'
|
||||
|
||||
# ── 5) shifter skills + include (spec D4)
|
||||
for l in low medium high xhigh max; do fm_has_effort "skills/effort-$l/SKILL.md" "$l"; has "skills/effort-$l/SKILL.md" "name: effort-$l"; done
|
||||
@@ -101,7 +116,7 @@ has "lib/effort-shift.md" 'Before any built-in or unpinned dispatch'
|
||||
has "lib/model-gate.md" 'built-ins inherit the effort in force'
|
||||
has "skills/ship-feature/SKILL.md" 'effort-shift: error recovery'
|
||||
for s in feat hotfix bugfix seo geo harden web-validate ship-feature init-project onboard code-clean audit-delta; do has "skills/$s/SKILL.md" 'effort-shift: own level before the challenge'; done
|
||||
has "install-plugins.sh" 'for _s in brainstorming writing-plans; do'
|
||||
has "update-all.sh" 'source "$REPO/lib/effort-pins.sh"'
|
||||
|
||||
# ── summary (later tasks insert their locks ABOVE this line)
|
||||
printf 'effort-routing census: %d pass, %d fail\n' "$pass" "$fail"
|
||||
|
||||
Reference in New Issue
Block a user