From c23cdb1aaf6698ab62469e77cdace40af3217684 Mon Sep 17 00:00:00 2001 From: Guardian Date: Mon, 2 Mar 2026 02:11:02 +0000 Subject: [PATCH] feat: add full auto-collection script (skills.sh + GitHub + ClaWHub) --- scripts/collect.sh | 356 +++++++++++++++++++++++++++++++++++++++------ 1 file changed, 315 insertions(+), 41 deletions(-) mode change 100644 => 100755 scripts/collect.sh diff --git a/scripts/collect.sh b/scripts/collect.sh old mode 100644 new mode 100755 index 075a5dfc..b201c18b --- a/scripts/collect.sh +++ b/scripts/collect.sh @@ -1,51 +1,325 @@ #!/bin/bash -# collect.sh - Weekly OpenClaw Master Skills collection script -# Scans ClaWHub + GitHub for new skills, outputs candidates for review +# collect.sh - Weekly OpenClaw Master Skills collector +# Sources: skills.sh (top 200) + GitHub (openclaw-skill topic) + ClaWHub CLI +# Usage: ./collect.sh [--dry-run] +# Env: GITHUB_TOKEN (required for GitHub scan) -SKILLS_DIR="$(dirname "$0")/../skills" -PENDING_DIR="$(dirname "$0")/../pending" +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)" +SKILLS_DIR="$REPO_ROOT/skills" +PENDING_DIR="$REPO_ROOT/pending" +CHANGELOG="$REPO_ROOT/CHANGELOG.md" +README="$REPO_ROOT/README.md" LOG_FILE="/tmp/openclaw-collect-$(date +%Y-%m-%d).log" - -log() { echo "[$(date '+%H:%M:%S')] $1" | tee -a "$LOG_FILE"; } - -log "=== OpenClaw Master Skills Weekly Collector ===" -log "Date: $(date '+%Y-%m-%d')" - -# ── 1. Scan ClaWHub for latest skills ──────────────────────────────────────── -log "Scanning ClaWHub..." -if command -v clawhub &>/dev/null; then - clawhub explore --limit 50 2>/dev/null | tee -a "$LOG_FILE" || log "ClaWHub explore failed" -else - log "clawhub CLI not installed, skipping" -fi - -# ── 2. Scan GitHub for repos tagged openclaw-skill ─────────────────────────── -log "Scanning GitHub (openclaw-skill topic)..." +DRY_RUN="${1:-}" GH_TOKEN="${GITHUB_TOKEN:-}" -if [ -n "$GH_TOKEN" ]; then - curl -s \ - -H "Authorization: token $GH_TOKEN" \ - "https://api.github.com/search/repositories?q=topic:openclaw-skill&sort=updated&per_page=20" \ - | python3 -c " +WEEK=$(date +%Y-%m-%d) + +mkdir -p "$PENDING_DIR" + +log() { echo "[$(date '+%H:%M:%S')] $1" | tee -a "$LOG_FILE"; } +info() { log "ℹ️ $1"; } +ok() { log "✅ $1"; } +warn() { log "⚠️ $1"; } + +# ── helpers ────────────────────────────────────────────────────────────────── + +fetch_json() { + curl -s --max-time 15 \ + ${GH_TOKEN:+-H "Authorization: token $GH_TOKEN"} \ + "$@" +} + +# Clone or update a GitHub repo skill folder into pending/ +fetch_github_skill() { + local repo="$1" # e.g. vercel-labs/agent-skills + local skill="$2" # e.g. vercel-react-best-practices + local dest="$PENDING_DIR/${skill}" + + if [ -d "$dest" ]; then + info "Already fetched: $skill" + return 0 + fi + + local url="https://api.github.com/repos/${repo}/contents/${skill}" + local files + files=$(fetch_json "$url" 2>/dev/null) || { warn "Cannot fetch $repo/$skill"; return 1; } + + local has_skill_md + has_skill_md=$(echo "$files" | python3 -c " +import sys, json +try: + data = json.load(sys.stdin) + names = [f['name'] for f in data if isinstance(data, list)] + print('yes' if 'SKILL.md' in names else 'no') +except: print('no') +" 2>/dev/null) + + if [ "$has_skill_md" = "yes" ]; then + mkdir -p "$dest" + # Download each file + echo "$files" | python3 -c " +import sys, json, urllib.request, os +data = json.load(sys.stdin) +dest = sys.argv[1] +for f in data: + if f.get('type') == 'file' and f.get('download_url'): + out = os.path.join(dest, f['name']) + urllib.request.urlretrieve(f['download_url'], out) + print(f' → {f[\"name\"]}') +" "$dest" 2>/dev/null && ok "Fetched: $skill (from $repo)" || warn "Failed to download files for $skill" + else + warn "No SKILL.md in $repo/$skill, skipping" + fi +} + +# ── Source 1: skills.sh top 200 ────────────────────────────────────────────── +scan_skills_sh() { + info "=== Scanning skills.sh top 200 ===" + + local html + html=$(curl -s --max-time 30 "https://skills.sh/" 2>/dev/null) || { warn "skills.sh unreachable"; return; } + + # Extract repo/skill pairs from the leaderboard + echo "$html" | python3 -c " +import re, sys +html = sys.stdin.read() +# Pattern: /owner/repo/skill-name in links +matches = re.findall(r'/([a-z0-9_-]+)/([a-z0-9_.-]+)/([a-z0-9_:-]+)', html) +seen = set() +for owner, repo, skill in matches: + key = f'{owner}/{repo}/{skill}' + if key not in seen and owner not in ('http', 'https', 'www'): + seen.add(key) + print(f'{owner}/{repo}\t{skill}') +" 2>/dev/null | head -60 | while IFS=$'\t' read -r repo skill; do + fetch_github_skill "$repo" "$skill" || true + done +} + +# ── Source 2: GitHub topic:openclaw-skill ──────────────────────────────────── +scan_github_topic() { + info "=== Scanning GitHub topic: openclaw-skill ===" + + if [ -z "$GH_TOKEN" ]; then + warn "GITHUB_TOKEN not set, skipping GitHub scan (rate limits apply)" + return + fi + + local results + results=$(fetch_json \ + "https://api.github.com/search/repositories?q=topic:openclaw-skill&sort=stars&per_page=30" \ + 2>/dev/null) || { warn "GitHub search failed"; return; } + + echo "$results" | python3 -c " import sys, json data = json.load(sys.stdin) for r in data.get('items', []): - print(f\" {r['full_name']} — {r['description']} ({r['stargazers_count']} ⭐)\") -" 2>/dev/null | tee -a "$LOG_FILE" -else - log "GITHUB_TOKEN not set, skipping GitHub scan" -fi + print(r['full_name'], r.get('description',''), r.get('stargazers_count',0)) +" 2>/dev/null | while read -r full_name desc stars; do + info " Found: $full_name (⭐ $stars)" + skill_name=$(echo "$full_name" | sed 's|.*/||') + fetch_github_skill "$full_name" "$skill_name" || true + done -# ── 3. Validate existing skills ────────────────────────────────────────────── -log "Validating existing skills in collection..." -for skill_dir in "$SKILLS_DIR"/*/; do - skill_name=$(basename "$skill_dir") - if [ -f "$skill_dir/SKILL.md" ]; then - log " ✅ $skill_name — OK" - else - log " ❌ $skill_name — missing SKILL.md!" + # Also search for repos containing SKILL.md with openclaw in description + fetch_json \ + "https://api.github.com/search/repositories?q=openclaw+skill+in:description&sort=stars&per_page=20" \ + 2>/dev/null | python3 -c " +import sys, json +data = json.load(sys.stdin) +for r in data.get('items', []): + print(r['full_name']) +" 2>/dev/null | while read -r full_name; do + skill_name=$(echo "$full_name" | sed 's|.*/||') + fetch_github_skill "$full_name" "$skill_name" || true + done +} + +# ── Source 3: ClaWHub CLI ───────────────────────────────────────────────────── +scan_clawhub() { + info "=== Scanning ClaWHub ===" + + if ! command -v clawhub &>/dev/null; then + warn "clawhub CLI not installed (npm i -g clawhub), skipping" + return fi -done -log "=== Done. Review $LOG_FILE for candidates ===" -log "Add approved skills to $SKILLS_DIR/ and update CHANGELOG.md" + clawhub explore --limit 50 2>/dev/null | while read -r line; do + info " ClaWHub: $line" + slug=$(echo "$line" | awk '{print $1}') + [ -z "$slug" ] && continue + dest="$PENDING_DIR/$slug" + if [ ! -d "$dest" ]; then + clawhub install "$slug" --workdir "$PENDING_DIR" --force 2>/dev/null && ok "Installed from ClaWHub: $slug" || true + fi + done +} + +# ── Validate pending skills ─────────────────────────────────────────────────── +validate_pending() { + info "=== Validating pending skills ===" + local approved=0 + local rejected=0 + + for skill_dir in "$PENDING_DIR"/*/; do + [ -d "$skill_dir" ] || continue + skill_name=$(basename "$skill_dir") + + if [ -f "$skill_dir/SKILL.md" ]; then + # Check frontmatter has name + description + valid=$(python3 -c " +import sys +content = open('$skill_dir/SKILL.md').read() +has_name = 'name:' in content +has_desc = 'description:' in content +print('ok' if has_name and has_desc else 'fail') +" 2>/dev/null) + if [ "$valid" = "ok" ]; then + if [ "$DRY_RUN" = "--dry-run" ]; then + ok "[DRY RUN] Would approve: $skill_name" + else + cp -r "$skill_dir" "$SKILLS_DIR/$skill_name" && ok "Approved: $skill_name" + fi + approved=$((approved+1)) + else + warn "Invalid SKILL.md (missing name/description): $skill_name" + rejected=$((rejected+1)) + fi + else + warn "No SKILL.md found: $skill_name" + rejected=$((rejected+1)) + fi + done + + info "Results: ✅ $approved approved, ❌ $rejected rejected" + echo "$approved" +} + +# ── Update CHANGELOG ───────────────────────────────────────────────────────── +update_changelog() { + local count="$1" + info "=== Updating CHANGELOG ===" + + local new_skills="" + for skill_dir in "$SKILLS_DIR"/*/; do + [ -d "$skill_dir" ] || continue + skill_name=$(basename "$skill_dir") + desc=$(python3 -c " +import re +try: + content = open('$skill_dir/SKILL.md').read() + m = re.search(r'description:\s*(.+)', content) + print(m.group(1).strip()[:80] if m else '') +except: print('') +" 2>/dev/null) + new_skills="$new_skills\n| \`$skill_name\` | $desc |" + done + + local entry="## [Week of $WEEK]\n\n### Skills Added This Week: $count\n\n| Skill | Description |\n|---|---|\n$new_skills\n\n---\n" + + # Prepend to changelog after header + python3 -c " +import re, sys +entry = sys.argv[1] +with open('$CHANGELOG', 'r') as f: + content = f.read() +# Insert after first '---' +content = content.replace('---\n', '---\n\n' + entry, 1) +with open('$CHANGELOG', 'w') as f: + f.write(content) +" "$entry" 2>/dev/null && ok "CHANGELOG updated" +} + +# ── Update README skill index ───────────────────────────────────────────────── +update_readme() { + info "=== Updating README skill index ===" + local rows="" + local count=0 + + for skill_dir in "$SKILLS_DIR"/*/; do + [ -d "$skill_dir" ] || continue + skill_name=$(basename "$skill_dir") + desc=$(python3 -c " +import re +try: + content = open('$skill_dir/SKILL.md').read() + m = re.search(r'description:\s*(.+)', content) + print(m.group(1).strip()[:70] if m else '') +except: print('') +" 2>/dev/null) + rows="$rows\n| [\`$skill_name\`](skills/$skill_name/) | $desc | — | — | — |" + count=$((count+1)) + done + + python3 -c " +import re, sys +rows = sys.argv[1] +count = sys.argv[2] +with open('$README', 'r') as f: + content = f.read() +# Replace table content between | Skill | header and next --- +table_header = '| Skill | Description | Category | Source | Added |' +table_sep = '|---|---|---|---|---|' +new_table = table_header + '\n' + table_sep + rows +content = re.sub( + r'\| Skill \| Description \| Category \| Source \| Added \|.*?(?=\n---|\n##)', + new_table, + content, flags=re.DOTALL +) +with open('$README', 'w') as f: + f.write(content) +print(f'README updated: {count} skills in index') +" "$rows" "$count" 2>/dev/null && ok "README updated ($count skills)" +} + +# ── Git commit & push ───────────────────────────────────────────────────────── +git_push() { + local count="$1" + info "=== Committing and pushing ===" + + cd "$REPO_ROOT" + git add -A + if git diff --cached --quiet; then + info "No changes to commit" + return + fi + + git commit -m "chore: weekly update $WEEK — $count new skills collected" + + if [ -n "$GH_TOKEN" ]; then + remote_url=$(git remote get-url origin 2>/dev/null | sed "s|https://|https://$GH_TOKEN@|") + git push "$remote_url" main 2>/dev/null && ok "Pushed to GitHub" || warn "Push failed" + else + warn "GITHUB_TOKEN not set, skipping push" + fi +} + +# ── Main ────────────────────────────────────────────────────────────────────── +main() { + log "======================================" + log "OpenClaw Master Skills Weekly Collector" + log "Week: $WEEK | Dry-run: ${DRY_RUN:-no}" + log "======================================" + + scan_skills_sh + scan_github_topic + scan_clawhub + + count=$(validate_pending) + + if [ "$DRY_RUN" != "--dry-run" ]; then + update_changelog "$count" + update_readme + git_push "$count" + # Clear pending after approval + rm -rf "$PENDING_DIR"/* + fi + + log "======================================" + log "Done! Log: $LOG_FILE" + log "======================================" +} + +main