#!/usr/bin/env bun /** * skillify-check — Post-task audit. * * Runs after any task that produced new code/features. Checks whether * the work is "properly skilled" per the 10-item checklist in * skills/skillify/SKILL.md and returns a score + recommendation. * * Usage: * bun run scripts/skillify-check.ts * bun run scripts/skillify-check.ts scripts/frameio-scraper.ts * bun run scripts/skillify-check.ts --recent # check recently-modified * bun run scripts/skillify-check.ts --json # machine-readable output * * Returns JSON when --json is passed: { path, score, total, items, * recommendation }. Exit code is 0 when score == total, 1 otherwise. * * Ported from ~/git/your-openclaw/workspace/scripts/skillify-check.mjs * (genericized: paths computed from $PROJECT_ROOT + runtime test-dir * detection; replaces the manual `grep AGENTS.md` check with a reference * to `gbrain check-resolvable` which validates the resolver better). */ import { existsSync, readFileSync, readdirSync, statSync } from 'fs'; import { join, basename, dirname, resolve } from 'path'; function projectRoot(): string { // Walk up from cwd until we find a package.json — that's the repo root. let dir = process.cwd(); for (let i = 0; i < 20; i++) { if (existsSync(join(dir, 'package.json'))) return dir; const parent = dirname(dir); if (parent === dir) break; dir = parent; } return process.cwd(); } const ROOT = projectRoot(); const SKILLS_DIR = join(ROOT, 'skills'); const RESOLVER_MD = join(SKILLS_DIR, 'RESOLVER.md'); // Test dir detection: prefer test/, then __tests__/, then tests/, then spec/. function detectTestDir(): string | null { for (const candidate of ['test', '__tests__', 'tests', 'spec']) { const p = join(ROOT, candidate); if (existsSync(p)) return p; } return null; } const TESTS_DIR = detectTestDir(); interface CheckItem { name: string; passed: boolean; required: boolean; detail?: string; } function check(name: string, passed: boolean, detail?: string): CheckItem { return { name, passed, required: true, detail }; } function checkOptional(name: string, passed: boolean, detail?: string): CheckItem { return { name, passed, required: false, detail }; } /** * Guess the skill-directory name from a script path. * scripts/frameio-scraper.ts → frameio-scraper * src/commands/publish.ts → publish * skills/foo/something.ts → foo */ function inferSkillName(scriptPath: string): string { // If the path is inside skills/, the second segment is the skill name. const abs = resolve(scriptPath); const inSkills = abs.match(/skills\/([^/]+)\//); if (inSkills) return inSkills[1]; const base = basename(scriptPath).replace(/\.(ts|mjs|js|py)$/, ''); // Check for an existing skill dir that matches. if (existsSync(SKILLS_DIR)) { for (const d of readdirSync(SKILLS_DIR)) { if (d === base) return d; // Fuzzy: script base stripped of common suffixes matches a dir name. const normalized = base.replace(/[-_]?(scraper|monitor|check|poll|sync|ingest|core)$/, ''); if (d === normalized || d.replace(/-/g, '') === normalized.replace(/[-_]/g, '')) return d; } } return base; } function findRelatedTests(scriptPath: string): string[] { if (!TESTS_DIR) return []; const base = basename(scriptPath).replace(/\.(ts|mjs|js|py)$/, ''); const patterns = [ `${base}.test.ts`, `${base}.test.mjs`, `${base}.test.js`, `test-${base}.ts`, `${base.replace(/-/g, '_')}.test.ts`, ]; const out: string[] = []; for (const p of patterns) { const f = join(TESTS_DIR, p); if (existsSync(f)) out.push(f); } // Fuzzy partial match. for (const f of readdirSync(TESTS_DIR)) { const normalized = f.replace(/-/g, '').replace('.test.ts', '').replace('.test.mjs', '').replace('test-', '').toLowerCase(); const nbase = base.replace(/-/g, '').toLowerCase(); if (normalized.includes(nbase) || nbase.includes(normalized)) { const fp = join(TESTS_DIR, f); if (!out.includes(fp)) out.push(fp); } } return out; } function isInResolver(skillName: string, scriptPath: string): boolean { if (!existsSync(RESOLVER_MD)) return false; const content = readFileSync(RESOLVER_MD, 'utf-8'); const base = basename(scriptPath).replace(/\.(ts|mjs|js|py)$/, ''); return content.includes(`skills/${skillName}`) || content.includes(skillName) || content.includes(base); } function runCheck(target: string): { path: string; skillName: string; items: CheckItem[]; score: number; total: number; recommendation: string; } { const abs = resolve(target); const skillName = inferSkillName(target); const skillMd = join(SKILLS_DIR, skillName, 'SKILL.md'); const items: CheckItem[] = []; // 1. SKILL.md exists items.push(check('SKILL.md exists', existsSync(skillMd), skillMd)); // 2. Code exists at target path items.push(check('Code file exists', existsSync(abs), abs)); // 3. Unit tests const unitTests = findRelatedTests(target); items.push(check('Unit tests', unitTests.length > 0, unitTests[0] ?? 'no matching *.test.ts in ' + (TESTS_DIR ?? '(no test dir)'))); // 4. Integration tests (heuristic: has a test that lives under test/e2e/) const e2eDir = TESTS_DIR ? join(TESTS_DIR, 'e2e') : null; const hasE2E = !!e2eDir && existsSync(e2eDir) && readdirSync(e2eDir).some(f => f.includes(skillName) || f.includes(basename(target).replace(/\.(ts|mjs|js|py)$/, '')), ); items.push(checkOptional('Integration tests (E2E)', hasE2E, e2eDir ?? 'no e2e dir')); // 5. LLM evals — heuristic: a file named *eval*.test.* in test dir referencing the skill name. let hasEvals = false; if (TESTS_DIR) { for (const f of readdirSync(TESTS_DIR)) { if (/eval/i.test(f) && (f.includes(skillName) || f.includes(basename(target)))) { hasEvals = true; break; } } } items.push(checkOptional('LLM evals', hasEvals)); // 6. Resolver entry items.push(check('Resolver entry', isInResolver(skillName, target))); // 7. Resolver trigger eval — heuristic: a resolver test that mentions skillName. let hasTriggerEval = false; if (TESTS_DIR) { const resolverTest = join(TESTS_DIR, 'resolver.test.ts'); if (existsSync(resolverTest)) { const content = readFileSync(resolverTest, 'utf-8'); hasTriggerEval = content.includes(skillName); } } items.push(checkOptional('Resolver trigger eval', hasTriggerEval)); // 8. check-resolvable — we don't run it here (side effects + cost); we // report whether the SKILL.md exists at all, which is the ground-truth // input check-resolvable would consume. items.push(checkOptional('check-resolvable input present', existsSync(skillMd) && existsSync(RESOLVER_MD), 'run: gbrain check-resolvable')); // 9. E2E — same as item 4 but required. items.push(check('E2E test (either under e2e/ or integration test)', hasE2E, 'try /qa or test/e2e/')); // 10. Brain filing — heuristic: if script mentions `addPage`, `upsertPage`, // or `addBrainPage` then brain/RESOLVER.md should list a matching dir. let writesBrain = false; if (existsSync(abs)) { try { const src = readFileSync(abs, 'utf-8'); writesBrain = /addPage|upsertPage|addBrainPage|putPage/.test(src); } catch { /* skip */ } } const brainResolver = join(ROOT, 'brain', 'RESOLVER.md'); const hasBrainEntry = writesBrain && existsSync(brainResolver) && readFileSync(brainResolver, 'utf-8').includes(skillName); items.push(checkOptional('Brain filing (RESOLVER entry for brain writes)', !writesBrain || hasBrainEntry, writesBrain ? (hasBrainEntry ? 'entry present' : 'writes brain but no brain/RESOLVER.md entry') : 'n/a')); // Score: required items pass; optional items contribute only if they pass. const passed = items.filter(i => i.passed).length; const total = items.length; const missing = items.filter(i => !i.passed && i.required).map(i => i.name); let recommendation: string; if (missing.length === 0) { recommendation = 'properly skilled'; } else if (missing.length <= 2) { recommendation = `close — create: ${missing.join(', ')}`; } else { recommendation = `needs skillify — run /skillify on ${target}; missing: ${missing.join(', ')}`; } return { path: target, skillName, items, score: passed, total, recommendation }; } function recentlyModified(days: number = 7): string[] { const candidates: string[] = []; const cutoff = Date.now() - days * 24 * 60 * 60 * 1000; const roots = ['src/commands', 'src/core', 'scripts'].map(r => join(ROOT, r)).filter(existsSync); for (const root of roots) { try { for (const f of readdirSync(root)) { if (!f.match(/\.(ts|mjs|js|py)$/)) continue; const fp = join(root, f); try { const st = statSync(fp); if (st.isFile() && st.mtimeMs >= cutoff) candidates.push(fp); } catch { /* skip */ } } } catch { /* skip */ } } return candidates; } function main() { const args = process.argv.slice(2); const json = args.includes('--json'); const recent = args.includes('--recent'); const help = args.includes('--help') || args.includes('-h'); if (help || (args.length === 0)) { console.log(`skillify-check — 10-item checklist audit for gbrain features. Usage: bun run scripts/skillify-check.ts bun run scripts/skillify-check.ts --recent Check files modified in the last 7 days. bun run scripts/skillify-check.ts --json Emit JSON. Exit code 0 when everything required passes; 1 otherwise. `); process.exit(args.length === 0 ? 1 : 0); } const targets = recent ? recentlyModified(7) : args.filter(a => !a.startsWith('--')); if (targets.length === 0) { console.error('No targets. Pass a path or --recent.'); process.exit(1); } const results = targets.map(runCheck); if (json) { console.log(JSON.stringify(results, null, 2)); } else { for (const r of results) { console.log(`\n${r.path} [${r.skillName}] ${r.score}/${r.total}`); for (const item of r.items) { const mark = item.passed ? '✓' : (item.required ? '✗' : '·'); const tag = item.required ? '' : ' (optional)'; const detail = item.detail ? ` — ${item.detail}` : ''; console.log(` ${mark} ${item.name}${tag}${detail}`); } console.log(` → ${r.recommendation}`); } } // Exit code: non-zero if any result has missing required items. const anyFailed = results.some(r => r.items.some(i => !i.passed && i.required)); process.exit(anyFailed ? 1 : 0); } main();