feat(vendor): vendor_pinned_tree fetches whole upstream trees at a pinned commit

This commit is contained in:
bchanot committed 2026-11-22 04:08:34 +01:00
1 parent 28460c8169
commit 1213af8b89
2 files changed
+432

No files matched your search

+153
View File
@@ -0,0 +1,153 @@
#!/usr/bin/env bash
# lib/tests/vendor-tree.test.sh — lib/vendor-skills.sh vendor_pinned_tree():
# pinned git fetch from a file:// fixture repo, allowlisted copy, INDEX
# (README description, else H1) / NOTICE / .vendored-commit generation,
# upstream SKILL.md never copied, skip at the pin, a pin bump prunes a
# guide dropped upstream, an absent copy is installed, a commit not
# upstream is refused with nothing written, unsafe lock paths rejected
# before any fetch, an allowlist edit at the same commit re-fetched, a
# dest owned by someone else never replaced, a symlink in the fetched
# content or a listed path leaving the checkout refused.
set -u
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
LIB="$ROOT/lib/vendor-skills.sh"
pass=0; fail=0
check_bool() {
if [ "$2" = 1 ]; then pass=$((pass+1)); echo "PASS $1"
else fail=$((fail+1)); echo "FAIL $1"; fi
}
WORK="$(mktemp -d)"; trap 'rm -rf "$WORK"' EXIT
UP="$WORK/upstream"; REPO="$WORK/repo"
mkdir -p "$REPO/skills-external"
g() { git -C "$UP" -c user.email=t@t -c user.name=t "$@"; }
# Fixture upstream: modern-web-guidance layout (SKILL.md + guides/), a
# README listing one guide, and a chrome-extensions skill with a
# references/ file.
mkdir -p "$UP/skills/modern-web-guidance/guides/css" \
"$UP/skills/modern-web-guidance/guides/html" \
"$UP/skills/chrome-extensions/references/extensions"
printf 'npx -y modern-web-guidance@latest search\n' \
> "$UP/skills/modern-web-guidance/SKILL.md"
printf '# Animate to intrinsic sizes\nbody\n' \
> "$UP/skills/modern-web-guidance/guides/css/animate-to-intrinsic-sizes.md"
printf '# Dialog\nbody\n' > "$UP/skills/modern-web-guidance/guides/html/dialog.md"
printf '# Old guide\nbody\n' > "$UP/skills/modern-web-guidance/guides/html/old-guide.md"
printf -- '- **[dialog](https://x/guides/html/dialog.md)**: Native modal dialog.\n' \
> "$UP/README.md"
printf -- '---\nname: chrome-extensions\n---\nbody\n' \
> "$UP/skills/chrome-extensions/SKILL.md"
printf 'ref\n' > "$UP/skills/chrome-extensions/references/extensions/api.md"
printf 'Apache License\n' > "$UP/LICENSE"
git init -q "$UP"; g add -A; g commit -qm one
SHA1="$(git -C "$UP" rev-parse HEAD)"
write_lock() { # write_lock <commit> [extra-tree-json]
cat > "$REPO/plugins.lock.json" <<JSON
{
"mwg-key": {
"source": "https://github.com/acme/mwg",
"commit": "$1",
"tag": "v1.0.1",
"license": "Apache-2.0 (code), CC-BY 4.0 (guides)",
"managed_by": "git-tree",
"trees": {
"mwg": {"guides": "skills/modern-web-guidance/guides", "LICENSE": "LICENSE", "README.md": "README.md"},
"chrome-extensions": {".": "skills/chrome-extensions"}${2:-}
}
}
}
JSON
}
# Same pattern as vendor-skills.test.sh: export once, source once (the lib
# reads VENDOR_SKILLS_REPO_OVERRIDE at source time), override per call.
export VENDOR_SKILLS_REPO_OVERRIDE="$REPO" VENDOR_BASE_URL="file://$UP"
# shellcheck source=../vendor-skills.sh disable=SC1091
source "$LIB"
run() { vendor_pinned_tree "$@" >"$WORK/out" 2>&1; }
M="$REPO/skills-external/mwg"; C="$REPO/skills-external/chrome-extensions"
# T1 install: allowlisted copy, generated files, no upstream SKILL.md.
write_lock "$SHA1"; run mwg-key; rc=$?
check_bool T1-rc "$([ $rc -eq 0 ] && echo 1)"
check_bool T1-guide "$([ -f "$M/guides/html/dialog.md" ] && echo 1)"
check_bool T1-license "$([ -f "$M/LICENSE" ] && echo 1)"
check_bool T1-no-upstream-skill "$([ ! -e "$M/SKILL.md" ] && echo 1)"
check_bool T1-index-desc "$(grep -qx -- '- dialog | html | Native modal dialog.' "$M/INDEX.md" && echo 1)"
check_bool T1-index-h1 "$(grep -qx -- '- animate-to-intrinsic-sizes | css | Animate to intrinsic sizes' "$M/INDEX.md" && echo 1)"
check_bool T1-index-count "$([ "$(grep -c '^- ' "$M/INDEX.md")" -eq 3 ] && echo 1)"
check_bool T1-notice "$(grep -q "$SHA1" "$M/NOTICE.md" && grep -q 'CC-BY 4.0' "$M/NOTICE.md" && echo 1)"
check_bool T1-marker "$([ "$(cat "$M/.vendored-commit")" = "$SHA1" ] && echo 1)"
check_bool T1-ce-contents "$([ -f "$C/SKILL.md" ] && [ -f "$C/references/extensions/api.md" ] && echo 1)"
check_bool T1-no-stage-left "$(compgen -G "$REPO/skills-external/.vendor-stage.*" >/dev/null || echo 1)"
# T2 at the pin: skip, no fetch (source made unreachable).
VENDOR_BASE_URL="file://$WORK/nowhere" vendor_pinned_tree mwg-key >"$WORK/out" 2>&1; rc=$?
check_bool T2-skip-rc "$([ $rc -eq 0 ] && grep -q 'already at pin' "$WORK/out" && echo 1)"
# T3 pin bump after an upstream drop: old-guide disappears locally.
g rm -q skills/modern-web-guidance/guides/html/old-guide.md; g commit -qm two
SHA2="$(git -C "$UP" rev-parse HEAD)"
write_lock "$SHA2"; run mwg-key; rc=$?
check_bool T3-rc "$([ $rc -eq 0 ] && echo 1)"
check_bool T3-pruned "$([ ! -e "$M/guides/html/old-guide.md" ] && echo 1)"
check_bool T3-marker "$([ "$(cat "$M/.vendored-commit")" = "$SHA2" ] && echo 1)"
# T4 an absent copy is installed (never-installed machine).
rm -rf "$REPO/skills-external/chrome-extensions"
run mwg-key; rc=$?
check_bool T4-installs-absent "$([ $rc -eq 0 ] && [ -f "$C/SKILL.md" ] && echo 1)"
# T12 an allowlist edit at the same commit is applied (README dropped:
# the dialog line falls back to its H1), then restored.
sed 's/, "README.md": "README.md"//' "$REPO/plugins.lock.json" > "$WORK/lock"
cp "$WORK/lock" "$REPO/plugins.lock.json"; run mwg-key; rc=$?
check_bool T12-spec-refetch "$([ $rc -eq 0 ] && [ ! -e "$M/README.md" ] && grep -qx -- '- dialog | html | Dialog' "$M/INDEX.md" && echo 1)"
write_lock "$SHA2"; run mwg-key; rc=$?
check_bool T12-spec-restored "$([ $rc -eq 0 ] && [ -f "$M/README.md" ] && grep -q 'README.md' "$M/.vendored-spec" && echo 1)"
# T5 commit not in upstream: rc 1, existing copy untouched.
BAD="0123456789abcdef0123456789abcdef01234567"
write_lock "$BAD"; run mwg-key; rc=$?
check_bool T5-rc "$([ $rc -eq 1 ] && echo 1)"
check_bool T5-untouched "$([ "$(cat "$M/.vendored-commit")" = "$SHA2" ] && echo 1)"
# T6 unsafe repo path rejected before any fetch.
write_lock "$SHA2" ', "evil": {"x": "../etc"}'; run mwg-key; rc=$?
check_bool T6-traversal "$([ $rc -eq 1 ] && grep -q "rejected" "$WORK/out" && [ ! -e "$REPO/skills-external/evil" ] && echo 1)"
# T7 bad dest name rejected.
write_lock "$SHA2" ', "a/b": {"x": "LICENSE"}'; run mwg-key; rc=$?
check_bool T7-dest "$([ $rc -eq 1 ] && grep -q "rejected" "$WORK/out" && echo 1)"
# T8 a missing upstream path aborts with nothing swapped.
write_lock "$SHA2" ', "ghost": {"x": "no/such/path"}'; run mwg-key; rc=$?
check_bool T8-missing-path "$([ $rc -eq 1 ] && [ ! -e "$REPO/skills-external/ghost" ] && [ "$(cat "$M/.vendored-commit")" = "$SHA2" ] && echo 1)"
# T9 a non-file:// VENDOR_BASE_URL is ignored with a warn.
write_lock "$SHA2"
VENDOR_BASE_URL="https://evil.example" vendor_pinned_tree mwg-key >"$WORK/out" 2>&1
check_bool T9-override-ignored "$(grep -q 'VENDOR_BASE_URL ignored' "$WORK/out" && echo 1)"
# T10 a dest that exists without .vendored-commit is never replaced.
mkdir -p "$REPO/skills-external/owned"; printf 'x\n' > "$REPO/skills-external/owned/keep"
write_lock "$SHA2" ', "owned": {"x": "LICENSE"}'; run mwg-key; rc=$?
check_bool T10-owned "$([ $rc -eq 1 ] && grep -q 'refusing to replace' "$WORK/out" && [ -f "$REPO/skills-external/owned/keep" ] && [ ! -e "$REPO/skills-external/owned/x" ] && echo 1)"
# T11 a symlink in the fetched content aborts with nothing swapped.
ln -s ../../SKILL.md "$UP/skills/chrome-extensions/references/link.md"
g add -A; g commit -qm three
SHA3="$(git -C "$UP" rev-parse HEAD)"
write_lock "$SHA3"; run mwg-key; rc=$?
check_bool T11-symlink "$([ $rc -eq 1 ] && [ ! -L "$C/references/link.md" ] && [ "$(cat "$C/.vendored-commit")" = "$SHA2" ] && echo 1)"
# T13 a listed path that resolves outside the checkout is refused.
g rm -q skills/chrome-extensions/references/link.md
mkdir -p "$WORK/outside"; printf 'x\n' > "$WORK/outside/f"
ln -s "$WORK/outside" "$UP/skills/escape"; g add -A; g commit -qm four
SHA4="$(git -C "$UP" rev-parse HEAD)"
write_lock "$SHA4" ', "esc": {".": "skills/escape"}'; run mwg-key; rc=$?
check_bool T13-escape "$([ $rc -eq 1 ] && grep -q 'leaves the checkout' "$WORK/out" && [ ! -e "$REPO/skills-external/esc" ] && echo 1)"
echo "vendor-tree: $pass pass, $fail fail"
[ "$fail" -eq 0 ]
+279
View File
@@ -234,3 +234,282 @@ vendor_pinned_skills() {
fi fi
done < <(tail -n +3 <<<"$lock_out") done < <(tail -n +3 <<<"$lock_out")
} }
# ============================================================
# vendor_pinned_tree — whole upstream trees at one pinned commit, by git
# ============================================================
# Lock entry shape (plugins.lock.json, "managed_by": "git-tree"):
# "<key>": {
# "source": "https://github.com/<owner>/<repo>",
# "commit": "<40-hex sha>", "tag": "<tag name, informational>",
# "license": "<one-line license note>",
# "trees": { "<dest>": { "<target>" | ".": "<repo path>", ... }, ... }
# }
# Each <dest> becomes skills-external/<dest>/. "<target>": "<path>" copies
# the repo path as <dest>/<target>; ".": "<path>" copies the path's
# contents into <dest>/. Only listed paths are copied (allowlist). The
# fetch is one `git fetch --depth 1 <source> <commit>`; HEAD must equal
# the pin or nothing is written. Every dest is staged complete under a
# mktemp dir in skills-external/, then swapped in whole, so a file
# dropped upstream disappears on a pin bump. Per dest it writes
# .vendored-commit, .vendored-spec (its lines of the trees allowlist),
# NOTICE.md, and INDEX.md when the dest holds guides/. One mode: skip when
# every dest holds the pinned commit and the current allowlist, else fetch
# (a pin bump, an allowlist edit, a missing copy). A hand-damaged copy
# with intact markers is not detected: delete the dest and rerun. A dest
# that exists without .vendored-commit belongs to someone else and is
# never replaced; a listed path that resolves outside the checkout, or a
# symlink in the copied content, aborts the run.
# _vendor_read_tree_lock <lockfile> <key> — prints source, commit, tag,
# license, then one "<dest>\t<target>\t<path>" line per item (sorted).
# rc 1 silent when the entry is absent or incomplete; rc 1 with one
# "INVALID\t<field>=<value>" or "INVALID\t<value>" line on a bad field.
# Only values that reach a path or a URL are allowlisted; tag and license
# are free text without control characters.
_vendor_read_tree_lock() {
python3 - "$1" "$2" <<'PY'
import json, re, sys
SAFE = re.compile(r'^[A-Za-z0-9._/-]+$')
NAME = re.compile(r'^[A-Za-z0-9_-][A-Za-z0-9._-]*$')
COMMIT_RE = re.compile(r'^[0-9a-f]{40}$')
SOURCE_RE = re.compile(
r'^https://github\.com/[A-Za-z0-9._-]+/[A-Za-z0-9._-]+/?$')
def unsafe(value):
return ".." in value or value.startswith("/") or not SAFE.fullmatch(value)
def free_text_bad(value):
return not isinstance(value, str) or any(ord(c) < 32 for c in value)
def bad(text):
print(f"INVALID\t{text}")
sys.exit(1)
lockfile, key = sys.argv[1], sys.argv[2]
with open(lockfile) as f:
entry = json.load(f).get(key, {})
source, sha = entry.get("source", ""), entry.get("commit", "")
tag, note = entry.get("tag", ""), entry.get("license", "")
trees = entry.get("trees", {})
if not source or not sha or not isinstance(trees, dict) or not trees:
sys.exit(1)
if not COMMIT_RE.fullmatch(sha):
bad(f"commit={sha}")
if not SOURCE_RE.fullmatch(source):
bad(f"source={source}")
if free_text_bad(tag):
bad("tag")
if free_text_bad(note):
bad("license")
rows = []
for dest, items in trees.items():
if not NAME.fullmatch(dest) or not isinstance(items, dict) or not items:
bad(f"dest={dest}")
for target, path in items.items():
if target != "." and not NAME.fullmatch(target):
bad(target)
if not isinstance(path, str) or unsafe(path):
bad(str(path))
rows.append((dest, target, path))
print(source)
print(sha)
print(tag)
print(note)
for dest, target, path in sorted(rows):
print(f"{dest}\t{target}\t{path}")
PY
}
# _vendor_tree_current <root> <sha> <items> — rc 0 when every dest of
# <items> holds a .vendored-commit equal to <sha> and a .vendored-spec
# equal to its own "<dest>\t<target>\t<path>" lines.
_vendor_tree_current() {
local root="$1" sha="$2" items="$3" dest
while IFS= read -r dest; do
[ -n "$dest" ] || continue
[ "$(cat "$root/$dest/.vendored-commit" 2>/dev/null)" = "$sha" ] \
|| return 1
[ "$(cat "$root/$dest/.vendored-spec" 2>/dev/null)" \
= "$(awk -F'\t' -v d="$dest" '$1 == d' <<<"$items")" ] || return 1
done < <(cut -f1 <<<"$items" | sort -u)
}
# _vendor_tree_owned <root> <dests> — rc 1 (err names it) when a dest
# exists without .vendored-commit: a directory another installer or the
# repo owns is never replaced.
_vendor_tree_owned() {
local root="$1" dest
while IFS= read -r dest; do
[ -n "$dest" ] || continue
if [ -e "$root/$dest" ] && [ ! -f "$root/$dest/.vendored-commit" ]; then
err "skills-external/$dest exists without .vendored-commit — \
not vendored here, refusing to replace it"
return 1
fi
done <<<"$2"
}
# _vendor_tree_checkout <dir> <url> <sha> — shallow fetch of <sha> into a
# fresh repo at <dir>; rc 0 only when the checked-out HEAD is <sha>.
_vendor_tree_checkout() {
local dir="$1" url="$2" sha="$3"
git init -q "$dir" >/dev/null 2>&1 || return 1
# No credential prompt of any kind (same flags as
# lib/higgsfield-skills.sh): a private or deleted upstream fails at
# once, not on a terminal, an askpass program or a credential helper.
GIT_TERMINAL_PROMPT=0 GIT_ASKPASS='' SSH_ASKPASS='' \
git -C "$dir" -c credential.helper= -c core.askPass= \
fetch -q --depth 1 -- "$url" "$sha" </dev/null >/dev/null 2>&1 \
|| return 1
git -C "$dir" -c advice.detachedHead=false checkout -q FETCH_HEAD \
>/dev/null 2>&1 || return 1
[ "$(git -C "$dir" rev-parse HEAD 2>/dev/null)" = "$sha" ]
}
# _vendor_tree_summary <readme> <id> — the upstream README's one-line
# description of guide <id> (`- **[<id>](…)**: <text>`), empty when the
# README or the line is absent.
_vendor_tree_summary() {
[ -f "$1" ] || return 0
grep -m1 -F -- "- **[$2](" "$1" \
| sed -n 's/^- \*\*\[[^]]*\]([^)]*)\*\*: //p' || true
}
# _vendor_tree_index <dest_dir> — writes <dest_dir>/INDEX.md, one
# "- <id> | <category> | <summary>" line per guides/<category>/<id>.md;
# the summary is the README description, else the guide's H1 title.
_vendor_tree_index() {
local dir="$1" f id category summary
[ -d "$dir/guides" ] || return 0
{
cat <<'EOF'
# Guide index
One line per guide: `id | category | summary` (the upstream README's
description, else the guide's title). Read `guides/<category>/<id>.md`.
EOF
find "$dir/guides" -type f -name '*.md' | LC_ALL=C sort \
| while IFS= read -r f; do
id="$(basename "$f" .md)"
category="$(basename "$(dirname "$f")")"
summary="$(_vendor_tree_summary "$dir/README.md" "$id")"
[ -n "$summary" ] \
|| summary="$(grep -m1 '^# ' "$f" | sed 's/^# //')"
printf -- '- %s | %s | %s\n' "$id" "$category" "${summary:-$id}"
done
} > "$dir/INDEX.md"
}
# _vendor_tree_stage <src> <stage> <items> — copies every
# "<dest>\t<target>\t<path>" item from the checkout <src> into
# <stage>/<dest>/ and records each dest's lines in .vendored-spec; rc 1
# (err names the path) when a path is missing, resolves outside the
# checkout, or the copied content holds a symlink.
_vendor_tree_stage() {
local src="$1" stage="$2" dest target path
while IFS=$'\t' read -r dest target path; do
[ -n "$dest" ] || continue
mkdir -p "$stage/$dest"
if [ ! -e "$src/$path" ]; then
err "upstream path missing at the pin: $path"
return 1
fi
case "$(realpath -e -- "$src/$path")" in
"$(realpath -e -- "$src")"/*) ;;
*) err "upstream path leaves the checkout through a symlink: $path"
return 1 ;;
esac
if [ "$target" = "." ]; then
cp -R "$src/$path/." "$stage/$dest/" || return 1
else
cp -R "$src/$path" "$stage/$dest/$target" || return 1
fi
done <<<"$3"
if [ -n "$(find "$stage" -type l -print -quit)" ]; then
err "upstream content holds a symlink at the pin — refusing"
return 1
fi
awk -F'\t' -v s="$stage" 'NF { print > (s "/" $1 "/.vendored-spec") }' \
<<<"$3"
}
# _vendor_tree_finish <stage> <dests> <source> <sha> <tag> <note> —
# writes INDEX.md, NOTICE.md and .vendored-commit into each staged dest.
_vendor_tree_finish() {
local stage="$1" dest
while IFS= read -r dest; do
[ -n "$dest" ] || continue
_vendor_tree_index "$stage/$dest"
printf '# Vendored content\n\nSource: %s (tag %s, commit %s).\n' \
"$3" "${5:-none}" "$4" > "$stage/$dest/NOTICE.md"
printf 'Fetched by lib/vendor-skills.sh vendor_pinned_tree;' \
>> "$stage/$dest/NOTICE.md"
printf ' never edited here. License: %s.\n' "$6" >> "$stage/$dest/NOTICE.md"
printf '%s\n' "$4" > "$stage/$dest/.vendored-commit"
done <<<"$2"
}
# _vendor_tree_swap <root> <stage> <dests> — moves each staged dest into
# <root>/<dest>, the previous copy parked under <stage>/.old-<dest> and
# moved back when the new one cannot be put in place.
_vendor_tree_swap() {
local root="$1" stage="$2" dest
while IFS= read -r dest; do
[ -n "$dest" ] || continue
if [ -e "$root/$dest" ]; then
mv "$root/$dest" "$stage/.old-$dest" || return 1
fi
if ! mv "$stage/$dest" "$root/$dest"; then
[ ! -e "$stage/.old-$dest" ] || mv "$stage/.old-$dest" "$root/$dest"
return 1
fi
done <<<"$3"
}
# vendor_pinned_tree <lock-key> — see the block comment above. rc 0 when
# every dest is at the pin and allowlist (already or after the fetch),
# rc 1 otherwise.
vendor_pinned_tree() {
local key="$1" out root="$VENDOR_REPO/skills-external"
local base="${VENDOR_BASE_URL:-}"
if [ -n "$base" ] && [[ "$base" != file://* ]]; then
warn "VENDOR_BASE_URL ignored (must start with file://): $base"
base=""
fi
if ! out="$(_vendor_read_tree_lock "$VENDOR_REPO/plugins.lock.json" "$key")"; then
_vendor_report_lock_error "$key" "$out"
return 1
fi
local source sha tag note items dests
{ read -r source; read -r sha; read -r tag; read -r note; } <<<"$out"
items="$(tail -n +5 <<<"$out")"
dests="$(cut -f1 <<<"$items" | sort -u)"
if _vendor_tree_current "$root" "$sha" "$items"; then
ok "$key already at pin ${sha:0:7}"
return 0
fi
_vendor_tree_owned "$root" "$dests" || return 1
local tmp stage
mkdir -p "$root" && tmp="$(mktemp -d)" || return 1
stage="$(mktemp -d "$root/.vendor-stage.XXXXXX")" \
|| { rm -rf "${tmp:?}"; return 1; }
if _vendor_tree_checkout "$tmp/src" "${base:-$source}" "$sha" \
&& _vendor_tree_stage "$tmp/src" "$stage" "$items" \
&& _vendor_tree_finish "$stage" "$dests" "$source" "$sha" "$tag" "$note" \
&& _vendor_tree_swap "$root" "$stage" "$dests"; then
ok "$key vendored (pinned @ ${sha:0:7}): $(tr '\n' ' ' <<<"$dests")"
rm -rf "${tmp:?}" "${stage:?}"
return 0
fi
err "$key: fetch, stage or swap failed at ${sha:0:7}; a dest not \
swapped keeps its previous copy (doctor names any not at the pin)"
rm -rf "${tmp:?}" "${stage:?}"
return 1
}