diff --git a/lib/tests/vendor-tree.test.sh b/lib/tests/vendor-tree.test.sh new file mode 100755 index 0000000..e50e2b2 --- /dev/null +++ b/lib/tests/vendor-tree.test.sh @@ -0,0 +1,153 @@ +#!/usr/bin/env bash +# lib/tests/vendor-tree.test.sh — lib/vendor-skills.sh vendor_pinned_tree(): +# pinned git fetch from a file:// fixture repo, allowlisted copy, INDEX +# (README description, else H1) / NOTICE / .vendored-commit generation, +# upstream SKILL.md never copied, skip at the pin, a pin bump prunes a +# guide dropped upstream, an absent copy is installed, a commit not +# upstream is refused with nothing written, unsafe lock paths rejected +# before any fetch, an allowlist edit at the same commit re-fetched, a +# dest owned by someone else never replaced, a symlink in the fetched +# content or a listed path leaving the checkout refused. +set -u +ROOT="$(cd "$(dirname "$0")/../.." && pwd)" +LIB="$ROOT/lib/vendor-skills.sh" +pass=0; fail=0 +check_bool() { + if [ "$2" = 1 ]; then pass=$((pass+1)); echo "PASS $1" + else fail=$((fail+1)); echo "FAIL $1"; fi +} +WORK="$(mktemp -d)"; trap 'rm -rf "$WORK"' EXIT +UP="$WORK/upstream"; REPO="$WORK/repo" +mkdir -p "$REPO/skills-external" +g() { git -C "$UP" -c user.email=t@t -c user.name=t "$@"; } + +# Fixture upstream: modern-web-guidance layout (SKILL.md + guides/), a +# README listing one guide, and a chrome-extensions skill with a +# references/ file. +mkdir -p "$UP/skills/modern-web-guidance/guides/css" \ + "$UP/skills/modern-web-guidance/guides/html" \ + "$UP/skills/chrome-extensions/references/extensions" +printf 'npx -y modern-web-guidance@latest search\n' \ + > "$UP/skills/modern-web-guidance/SKILL.md" +printf '# Animate to intrinsic sizes\nbody\n' \ + > "$UP/skills/modern-web-guidance/guides/css/animate-to-intrinsic-sizes.md" +printf '# Dialog\nbody\n' > "$UP/skills/modern-web-guidance/guides/html/dialog.md" +printf '# Old guide\nbody\n' > "$UP/skills/modern-web-guidance/guides/html/old-guide.md" +printf -- '- **[dialog](https://x/guides/html/dialog.md)**: Native modal dialog.\n' \ + > "$UP/README.md" +printf -- '---\nname: chrome-extensions\n---\nbody\n' \ + > "$UP/skills/chrome-extensions/SKILL.md" +printf 'ref\n' > "$UP/skills/chrome-extensions/references/extensions/api.md" +printf 'Apache License\n' > "$UP/LICENSE" +git init -q "$UP"; g add -A; g commit -qm one +SHA1="$(git -C "$UP" rev-parse HEAD)" + +write_lock() { # write_lock [extra-tree-json] + cat > "$REPO/plugins.lock.json" <"$WORK/out" 2>&1; } +M="$REPO/skills-external/mwg"; C="$REPO/skills-external/chrome-extensions" + +# T1 install: allowlisted copy, generated files, no upstream SKILL.md. +write_lock "$SHA1"; run mwg-key; rc=$? +check_bool T1-rc "$([ $rc -eq 0 ] && echo 1)" +check_bool T1-guide "$([ -f "$M/guides/html/dialog.md" ] && echo 1)" +check_bool T1-license "$([ -f "$M/LICENSE" ] && echo 1)" +check_bool T1-no-upstream-skill "$([ ! -e "$M/SKILL.md" ] && echo 1)" +check_bool T1-index-desc "$(grep -qx -- '- dialog | html | Native modal dialog.' "$M/INDEX.md" && echo 1)" +check_bool T1-index-h1 "$(grep -qx -- '- animate-to-intrinsic-sizes | css | Animate to intrinsic sizes' "$M/INDEX.md" && echo 1)" +check_bool T1-index-count "$([ "$(grep -c '^- ' "$M/INDEX.md")" -eq 3 ] && echo 1)" +check_bool T1-notice "$(grep -q "$SHA1" "$M/NOTICE.md" && grep -q 'CC-BY 4.0' "$M/NOTICE.md" && echo 1)" +check_bool T1-marker "$([ "$(cat "$M/.vendored-commit")" = "$SHA1" ] && echo 1)" +check_bool T1-ce-contents "$([ -f "$C/SKILL.md" ] && [ -f "$C/references/extensions/api.md" ] && echo 1)" +check_bool T1-no-stage-left "$(compgen -G "$REPO/skills-external/.vendor-stage.*" >/dev/null || echo 1)" + +# T2 at the pin: skip, no fetch (source made unreachable). +VENDOR_BASE_URL="file://$WORK/nowhere" vendor_pinned_tree mwg-key >"$WORK/out" 2>&1; rc=$? +check_bool T2-skip-rc "$([ $rc -eq 0 ] && grep -q 'already at pin' "$WORK/out" && echo 1)" + +# T3 pin bump after an upstream drop: old-guide disappears locally. +g rm -q skills/modern-web-guidance/guides/html/old-guide.md; g commit -qm two +SHA2="$(git -C "$UP" rev-parse HEAD)" +write_lock "$SHA2"; run mwg-key; rc=$? +check_bool T3-rc "$([ $rc -eq 0 ] && echo 1)" +check_bool T3-pruned "$([ ! -e "$M/guides/html/old-guide.md" ] && echo 1)" +check_bool T3-marker "$([ "$(cat "$M/.vendored-commit")" = "$SHA2" ] && echo 1)" + +# T4 an absent copy is installed (never-installed machine). +rm -rf "$REPO/skills-external/chrome-extensions" +run mwg-key; rc=$? +check_bool T4-installs-absent "$([ $rc -eq 0 ] && [ -f "$C/SKILL.md" ] && echo 1)" + +# T12 an allowlist edit at the same commit is applied (README dropped: +# the dialog line falls back to its H1), then restored. +sed 's/, "README.md": "README.md"//' "$REPO/plugins.lock.json" > "$WORK/lock" +cp "$WORK/lock" "$REPO/plugins.lock.json"; run mwg-key; rc=$? +check_bool T12-spec-refetch "$([ $rc -eq 0 ] && [ ! -e "$M/README.md" ] && grep -qx -- '- dialog | html | Dialog' "$M/INDEX.md" && echo 1)" +write_lock "$SHA2"; run mwg-key; rc=$? +check_bool T12-spec-restored "$([ $rc -eq 0 ] && [ -f "$M/README.md" ] && grep -q 'README.md' "$M/.vendored-spec" && echo 1)" + +# T5 commit not in upstream: rc 1, existing copy untouched. +BAD="0123456789abcdef0123456789abcdef01234567" +write_lock "$BAD"; run mwg-key; rc=$? +check_bool T5-rc "$([ $rc -eq 1 ] && echo 1)" +check_bool T5-untouched "$([ "$(cat "$M/.vendored-commit")" = "$SHA2" ] && echo 1)" + +# T6 unsafe repo path rejected before any fetch. +write_lock "$SHA2" ', "evil": {"x": "../etc"}'; run mwg-key; rc=$? +check_bool T6-traversal "$([ $rc -eq 1 ] && grep -q "rejected" "$WORK/out" && [ ! -e "$REPO/skills-external/evil" ] && echo 1)" + +# T7 bad dest name rejected. +write_lock "$SHA2" ', "a/b": {"x": "LICENSE"}'; run mwg-key; rc=$? +check_bool T7-dest "$([ $rc -eq 1 ] && grep -q "rejected" "$WORK/out" && echo 1)" + +# T8 a missing upstream path aborts with nothing swapped. +write_lock "$SHA2" ', "ghost": {"x": "no/such/path"}'; run mwg-key; rc=$? +check_bool T8-missing-path "$([ $rc -eq 1 ] && [ ! -e "$REPO/skills-external/ghost" ] && [ "$(cat "$M/.vendored-commit")" = "$SHA2" ] && echo 1)" + +# T9 a non-file:// VENDOR_BASE_URL is ignored with a warn. +write_lock "$SHA2" +VENDOR_BASE_URL="https://evil.example" vendor_pinned_tree mwg-key >"$WORK/out" 2>&1 +check_bool T9-override-ignored "$(grep -q 'VENDOR_BASE_URL ignored' "$WORK/out" && echo 1)" + +# T10 a dest that exists without .vendored-commit is never replaced. +mkdir -p "$REPO/skills-external/owned"; printf 'x\n' > "$REPO/skills-external/owned/keep" +write_lock "$SHA2" ', "owned": {"x": "LICENSE"}'; run mwg-key; rc=$? +check_bool T10-owned "$([ $rc -eq 1 ] && grep -q 'refusing to replace' "$WORK/out" && [ -f "$REPO/skills-external/owned/keep" ] && [ ! -e "$REPO/skills-external/owned/x" ] && echo 1)" + +# T11 a symlink in the fetched content aborts with nothing swapped. +ln -s ../../SKILL.md "$UP/skills/chrome-extensions/references/link.md" +g add -A; g commit -qm three +SHA3="$(git -C "$UP" rev-parse HEAD)" +write_lock "$SHA3"; run mwg-key; rc=$? +check_bool T11-symlink "$([ $rc -eq 1 ] && [ ! -L "$C/references/link.md" ] && [ "$(cat "$C/.vendored-commit")" = "$SHA2" ] && echo 1)" + +# T13 a listed path that resolves outside the checkout is refused. +g rm -q skills/chrome-extensions/references/link.md +mkdir -p "$WORK/outside"; printf 'x\n' > "$WORK/outside/f" +ln -s "$WORK/outside" "$UP/skills/escape"; g add -A; g commit -qm four +SHA4="$(git -C "$UP" rev-parse HEAD)" +write_lock "$SHA4" ', "esc": {".": "skills/escape"}'; run mwg-key; rc=$? +check_bool T13-escape "$([ $rc -eq 1 ] && grep -q 'leaves the checkout' "$WORK/out" && [ ! -e "$REPO/skills-external/esc" ] && echo 1)" + +echo "vendor-tree: $pass pass, $fail fail" +[ "$fail" -eq 0 ] diff --git a/lib/vendor-skills.sh b/lib/vendor-skills.sh index 72f4af7..9b2b59b 100644 --- a/lib/vendor-skills.sh +++ b/lib/vendor-skills.sh @@ -234,3 +234,282 @@ vendor_pinned_skills() { fi done < <(tail -n +3 <<<"$lock_out") } + +# ============================================================ +# vendor_pinned_tree — whole upstream trees at one pinned commit, by git +# ============================================================ +# Lock entry shape (plugins.lock.json, "managed_by": "git-tree"): +# "": { +# "source": "https://github.com//", +# "commit": "<40-hex sha>", "tag": "", +# "license": "", +# "trees": { "": { "" | ".": "", ... }, ... } +# } +# Each becomes skills-external//. "": "" copies +# the repo path as /; ".": "" copies the path's +# contents into /. Only listed paths are copied (allowlist). The +# fetch is one `git fetch --depth 1 `; HEAD must equal +# the pin or nothing is written. Every dest is staged complete under a +# mktemp dir in skills-external/, then swapped in whole, so a file +# dropped upstream disappears on a pin bump. Per dest it writes +# .vendored-commit, .vendored-spec (its lines of the trees allowlist), +# NOTICE.md, and INDEX.md when the dest holds guides/. One mode: skip when +# every dest holds the pinned commit and the current allowlist, else fetch +# (a pin bump, an allowlist edit, a missing copy). A hand-damaged copy +# with intact markers is not detected: delete the dest and rerun. A dest +# that exists without .vendored-commit belongs to someone else and is +# never replaced; a listed path that resolves outside the checkout, or a +# symlink in the copied content, aborts the run. + +# _vendor_read_tree_lock — prints source, commit, tag, +# license, then one "\t\t" line per item (sorted). +# rc 1 silent when the entry is absent or incomplete; rc 1 with one +# "INVALID\t=" or "INVALID\t" line on a bad field. +# Only values that reach a path or a URL are allowlisted; tag and license +# are free text without control characters. +_vendor_read_tree_lock() { + python3 - "$1" "$2" <<'PY' +import json, re, sys + +SAFE = re.compile(r'^[A-Za-z0-9._/-]+$') +NAME = re.compile(r'^[A-Za-z0-9_-][A-Za-z0-9._-]*$') +COMMIT_RE = re.compile(r'^[0-9a-f]{40}$') +SOURCE_RE = re.compile( + r'^https://github\.com/[A-Za-z0-9._-]+/[A-Za-z0-9._-]+/?$') + + +def unsafe(value): + return ".." in value or value.startswith("/") or not SAFE.fullmatch(value) + + +def free_text_bad(value): + return not isinstance(value, str) or any(ord(c) < 32 for c in value) + + +def bad(text): + print(f"INVALID\t{text}") + sys.exit(1) + + +lockfile, key = sys.argv[1], sys.argv[2] +with open(lockfile) as f: + entry = json.load(f).get(key, {}) +source, sha = entry.get("source", ""), entry.get("commit", "") +tag, note = entry.get("tag", ""), entry.get("license", "") +trees = entry.get("trees", {}) +if not source or not sha or not isinstance(trees, dict) or not trees: + sys.exit(1) +if not COMMIT_RE.fullmatch(sha): + bad(f"commit={sha}") +if not SOURCE_RE.fullmatch(source): + bad(f"source={source}") +if free_text_bad(tag): + bad("tag") +if free_text_bad(note): + bad("license") +rows = [] +for dest, items in trees.items(): + if not NAME.fullmatch(dest) or not isinstance(items, dict) or not items: + bad(f"dest={dest}") + for target, path in items.items(): + if target != "." and not NAME.fullmatch(target): + bad(target) + if not isinstance(path, str) or unsafe(path): + bad(str(path)) + rows.append((dest, target, path)) +print(source) +print(sha) +print(tag) +print(note) +for dest, target, path in sorted(rows): + print(f"{dest}\t{target}\t{path}") +PY +} + +# _vendor_tree_current — rc 0 when every dest of +# holds a .vendored-commit equal to and a .vendored-spec +# equal to its own "\t\t" lines. +_vendor_tree_current() { + local root="$1" sha="$2" items="$3" dest + while IFS= read -r dest; do + [ -n "$dest" ] || continue + [ "$(cat "$root/$dest/.vendored-commit" 2>/dev/null)" = "$sha" ] \ + || return 1 + [ "$(cat "$root/$dest/.vendored-spec" 2>/dev/null)" \ + = "$(awk -F'\t' -v d="$dest" '$1 == d' <<<"$items")" ] || return 1 + done < <(cut -f1 <<<"$items" | sort -u) +} + +# _vendor_tree_owned — rc 1 (err names it) when a dest +# exists without .vendored-commit: a directory another installer or the +# repo owns is never replaced. +_vendor_tree_owned() { + local root="$1" dest + while IFS= read -r dest; do + [ -n "$dest" ] || continue + if [ -e "$root/$dest" ] && [ ! -f "$root/$dest/.vendored-commit" ]; then + err "skills-external/$dest exists without .vendored-commit — \ +not vendored here, refusing to replace it" + return 1 + fi + done <<<"$2" +} + +# _vendor_tree_checkout — shallow fetch of into a +# fresh repo at ; rc 0 only when the checked-out HEAD is . +_vendor_tree_checkout() { + local dir="$1" url="$2" sha="$3" + git init -q "$dir" >/dev/null 2>&1 || return 1 + # No credential prompt of any kind (same flags as + # lib/higgsfield-skills.sh): a private or deleted upstream fails at + # once, not on a terminal, an askpass program or a credential helper. + GIT_TERMINAL_PROMPT=0 GIT_ASKPASS='' SSH_ASKPASS='' \ + git -C "$dir" -c credential.helper= -c core.askPass= \ + fetch -q --depth 1 -- "$url" "$sha" /dev/null 2>&1 \ + || return 1 + git -C "$dir" -c advice.detachedHead=false checkout -q FETCH_HEAD \ + >/dev/null 2>&1 || return 1 + [ "$(git -C "$dir" rev-parse HEAD 2>/dev/null)" = "$sha" ] +} + +# _vendor_tree_summary — the upstream README's one-line +# description of guide (`- **[](…)**: `), empty when the +# README or the line is absent. +_vendor_tree_summary() { + [ -f "$1" ] || return 0 + grep -m1 -F -- "- **[$2](" "$1" \ + | sed -n 's/^- \*\*\[[^]]*\]([^)]*)\*\*: //p' || true +} + +# _vendor_tree_index — writes /INDEX.md, one +# "- | | " line per guides//.md; +# the summary is the README description, else the guide's H1 title. +_vendor_tree_index() { + local dir="$1" f id category summary + [ -d "$dir/guides" ] || return 0 + { + cat <<'EOF' +# Guide index + +One line per guide: `id | category | summary` (the upstream README's +description, else the guide's title). Read `guides//.md`. + +EOF + find "$dir/guides" -type f -name '*.md' | LC_ALL=C sort \ + | while IFS= read -r f; do + id="$(basename "$f" .md)" + category="$(basename "$(dirname "$f")")" + summary="$(_vendor_tree_summary "$dir/README.md" "$id")" + [ -n "$summary" ] \ + || summary="$(grep -m1 '^# ' "$f" | sed 's/^# //')" + printf -- '- %s | %s | %s\n' "$id" "$category" "${summary:-$id}" + done + } > "$dir/INDEX.md" +} + +# _vendor_tree_stage — copies every +# "\t\t" item from the checkout into +# // and records each dest's lines in .vendored-spec; rc 1 +# (err names the path) when a path is missing, resolves outside the +# checkout, or the copied content holds a symlink. +_vendor_tree_stage() { + local src="$1" stage="$2" dest target path + while IFS=$'\t' read -r dest target path; do + [ -n "$dest" ] || continue + mkdir -p "$stage/$dest" + if [ ! -e "$src/$path" ]; then + err "upstream path missing at the pin: $path" + return 1 + fi + case "$(realpath -e -- "$src/$path")" in + "$(realpath -e -- "$src")"/*) ;; + *) err "upstream path leaves the checkout through a symlink: $path" + return 1 ;; + esac + if [ "$target" = "." ]; then + cp -R "$src/$path/." "$stage/$dest/" || return 1 + else + cp -R "$src/$path" "$stage/$dest/$target" || return 1 + fi + done <<<"$3" + if [ -n "$(find "$stage" -type l -print -quit)" ]; then + err "upstream content holds a symlink at the pin — refusing" + return 1 + fi + awk -F'\t' -v s="$stage" 'NF { print > (s "/" $1 "/.vendored-spec") }' \ + <<<"$3" +} + +# _vendor_tree_finish — +# writes INDEX.md, NOTICE.md and .vendored-commit into each staged dest. +_vendor_tree_finish() { + local stage="$1" dest + while IFS= read -r dest; do + [ -n "$dest" ] || continue + _vendor_tree_index "$stage/$dest" + printf '# Vendored content\n\nSource: %s (tag %s, commit %s).\n' \ + "$3" "${5:-none}" "$4" > "$stage/$dest/NOTICE.md" + printf 'Fetched by lib/vendor-skills.sh vendor_pinned_tree;' \ + >> "$stage/$dest/NOTICE.md" + printf ' never edited here. License: %s.\n' "$6" >> "$stage/$dest/NOTICE.md" + printf '%s\n' "$4" > "$stage/$dest/.vendored-commit" + done <<<"$2" +} + +# _vendor_tree_swap — moves each staged dest into +# /, the previous copy parked under /.old- and +# moved back when the new one cannot be put in place. +_vendor_tree_swap() { + local root="$1" stage="$2" dest + while IFS= read -r dest; do + [ -n "$dest" ] || continue + if [ -e "$root/$dest" ]; then + mv "$root/$dest" "$stage/.old-$dest" || return 1 + fi + if ! mv "$stage/$dest" "$root/$dest"; then + [ ! -e "$stage/.old-$dest" ] || mv "$stage/.old-$dest" "$root/$dest" + return 1 + fi + done <<<"$3" +} + +# vendor_pinned_tree — see the block comment above. rc 0 when +# every dest is at the pin and allowlist (already or after the fetch), +# rc 1 otherwise. +vendor_pinned_tree() { + local key="$1" out root="$VENDOR_REPO/skills-external" + local base="${VENDOR_BASE_URL:-}" + if [ -n "$base" ] && [[ "$base" != file://* ]]; then + warn "VENDOR_BASE_URL ignored (must start with file://): $base" + base="" + fi + if ! out="$(_vendor_read_tree_lock "$VENDOR_REPO/plugins.lock.json" "$key")"; then + _vendor_report_lock_error "$key" "$out" + return 1 + fi + local source sha tag note items dests + { read -r source; read -r sha; read -r tag; read -r note; } <<<"$out" + items="$(tail -n +5 <<<"$out")" + dests="$(cut -f1 <<<"$items" | sort -u)" + if _vendor_tree_current "$root" "$sha" "$items"; then + ok "$key already at pin ${sha:0:7}" + return 0 + fi + _vendor_tree_owned "$root" "$dests" || return 1 + local tmp stage + mkdir -p "$root" && tmp="$(mktemp -d)" || return 1 + stage="$(mktemp -d "$root/.vendor-stage.XXXXXX")" \ + || { rm -rf "${tmp:?}"; return 1; } + if _vendor_tree_checkout "$tmp/src" "${base:-$source}" "$sha" \ + && _vendor_tree_stage "$tmp/src" "$stage" "$items" \ + && _vendor_tree_finish "$stage" "$dests" "$source" "$sha" "$tag" "$note" \ + && _vendor_tree_swap "$root" "$stage" "$dests"; then + ok "$key vendored (pinned @ ${sha:0:7}): $(tr '\n' ' ' <<<"$dests")" + rm -rf "${tmp:?}" "${stage:?}" + return 0 + fi + err "$key: fetch, stage or swap failed at ${sha:0:7}; a dest not \ +swapped keeps its previous copy (doctor names any not at the pin)" + rm -rf "${tmp:?}" "${stage:?}" + return 1 +}