gbrain/test/skillify-scaffold.test.ts

/**
 * Tests for src/core/skillify/generator.ts (W4).
 * Mechanical scaffold plan + apply, idempotency (D-CX-7), stub sentinel.
 */

import { describe, expect, it, afterEach } from 'bun:test';
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'fs';
import { join } from 'path';
import { tmpdir } from 'os';

import {
  applyScaffold,
  planScaffold,
  SkillifyScaffoldError,
  SKILL_NAME_PATTERN,
} from '../src/core/skillify/generator.ts';
import { SKILLIFY_STUB_MARKER } from '../src/core/skillify/templates.ts';

const created: string[] = [];

function scratchRepo(): { root: string; skillsDir: string } {
  const root = mkdtempSync(join(tmpdir(), 'skillify-repo-'));
  created.push(root);
  const skillsDir = join(root, 'skills');
  mkdirSync(skillsDir, { recursive: true });
  mkdirSync(join(root, 'test'), { recursive: true });
  writeFileSync(
    join(skillsDir, 'RESOLVER.md'),
    '# RESOLVER\n\n## Brain operations\n\n| Trigger | Skill |\n|---------|-------|\n| "existing thing" | `skills/existing/SKILL.md` |\n',
  );
  return { root, skillsDir };
}

afterEach(() => {
  while (created.length) {
    const d = created.pop();
    if (d && existsSync(d)) rmSync(d, { recursive: true, force: true });
  }
});

describe('SKILL_NAME_PATTERN', () => {
  it('accepts valid kebab-case', () => {
    expect(SKILL_NAME_PATTERN.test('context-now')).toBe(true);
    expect(SKILL_NAME_PATTERN.test('a')).toBe(true);
    expect(SKILL_NAME_PATTERN.test('calendar-recall-v2')).toBe(true);
  });
  it('rejects uppercase, spaces, underscores, leading digits', () => {
    expect(SKILL_NAME_PATTERN.test('ContextNow')).toBe(false);
    expect(SKILL_NAME_PATTERN.test('context now')).toBe(false);
    expect(SKILL_NAME_PATTERN.test('context_now')).toBe(false);
    expect(SKILL_NAME_PATTERN.test('2-skill')).toBe(false);
  });
});

describe('planScaffold', () => {
  it('throws SkillifyScaffoldError on invalid name', () => {
    const { root, skillsDir } = scratchRepo();
    expect(() =>
      planScaffold({
        skillsDir,
        repoRoot: root,
        vars: {
          name: 'Bad Name',
          description: 'x',
          triggers: [],
          writesTo: [],
          writesPages: false,
          mutating: false,
        },
      }),
    ).toThrow(SkillifyScaffoldError);
  });

  it('plans 4 files + resolver append for a new skill', () => {
    const { root, skillsDir } = scratchRepo();
    const plan = planScaffold({
      skillsDir,
      repoRoot: root,
      vars: {
        name: 'hello-world',
        description: 'say hello',
        triggers: ['say hello', 'greet me'],
        writesTo: [],
        writesPages: false,
        mutating: false,
      },
    });
    expect(plan.files.length).toBe(4);
    const paths = plan.files.map(f => f.path);
    expect(paths).toContain(join(skillsDir, 'hello-world', 'SKILL.md'));
    expect(paths).toContain(
      join(skillsDir, 'hello-world', 'scripts', 'hello-world.mjs'),
    );
    expect(paths).toContain(join(skillsDir, 'hello-world', 'routing-eval.jsonl'));
    expect(paths).toContain(join(root, 'test', 'hello-world.test.ts'));
    expect(plan.files.every(f => f.kind === 'new')).toBe(true);
    expect(plan.resolverFile).toBe(join(skillsDir, 'RESOLVER.md'));
    expect(plan.resolverAppend).not.toBeNull();
    expect(plan.resolverAppend!).toContain('`skills/hello-world/SKILL.md`');
  });

  it('SKILL.md includes the SKILLIFY_STUB sentinel', () => {
    const { root, skillsDir } = scratchRepo();
    const plan = planScaffold({
      skillsDir,
      repoRoot: root,
      vars: {
        name: 'foo',
        description: 'foo skill',
        triggers: [],
        writesTo: [],
        writesPages: false,
        mutating: false,
      },
    });
    const skillMd = plan.files.find(f => f.path.endsWith('SKILL.md'))!;
    expect(skillMd.content).toContain(SKILLIFY_STUB_MARKER);
  });

  it('script stub includes the SKILLIFY_STUB sentinel (D-CX-9 gate hook)', () => {
    const { root, skillsDir } = scratchRepo();
    const plan = planScaffold({
      skillsDir,
      repoRoot: root,
      vars: {
        name: 'foo',
        description: 'foo skill',
        triggers: [],
        writesTo: [],
        writesPages: false,
        mutating: false,
      },
    });
    const script = plan.files.find(f => f.path.endsWith('foo.mjs'))!;
    expect(script.content).toContain(SKILLIFY_STUB_MARKER);
  });

  it('refuses to scaffold over an existing file without --force', () => {
    const { root, skillsDir } = scratchRepo();
    mkdirSync(join(skillsDir, 'existing'), { recursive: true });
    writeFileSync(
      join(skillsDir, 'existing', 'SKILL.md'),
      '---\nname: existing\n---\n',
    );
    expect(() =>
      planScaffold({
        skillsDir,
        repoRoot: root,
        vars: {
          name: 'existing',
          description: 'x',
          triggers: [],
          writesTo: [],
          writesPages: false,
          mutating: false,
        },
      }),
    ).toThrow(SkillifyScaffoldError);
  });

  it('--force marks existing files as overwrite', () => {
    const { root, skillsDir } = scratchRepo();
    mkdirSync(join(skillsDir, 'existing'), { recursive: true });
    writeFileSync(
      join(skillsDir, 'existing', 'SKILL.md'),
      '---\nname: existing\n---\n',
    );
    const plan = planScaffold({
      skillsDir,
      repoRoot: root,
      force: true,
      vars: {
        name: 'existing',
        description: 'x',
        triggers: ['foo'],
        writesTo: [],
        writesPages: false,
        mutating: false,
      },
    });
    const skillMd = plan.files.find(f => f.path.endsWith('SKILL.md'))!;
    expect(skillMd.kind).toBe('overwrite');
  });

  it('D-CX-7: resolverAppend is null when row already present (idempotent)', () => {
    const { root, skillsDir } = scratchRepo();
    // Prime the resolver with an existing row for the skill we're about
    // to scaffold. The plan must NOT queue a duplicate append, even
    // when --force regenerates files.
    const resolverPath = join(skillsDir, 'RESOLVER.md');
    const before = readFileSync(resolverPath, 'utf-8');
    writeFileSync(
      resolverPath,
      before +
        '\n## Uncategorized\n\n| Trigger | Skill |\n|---------|-------|\n' +
        '| "do thing" | `skills/demo/SKILL.md` |\n',
    );
    const plan = planScaffold({
      skillsDir,
      repoRoot: root,
      vars: {
        name: 'demo',
        description: 'demo',
        triggers: ['do thing'],
        writesTo: [],
        writesPages: false,
        mutating: false,
      },
    });
    expect(plan.resolverAppend).toBeNull();
  });

  it('detects bare-path resolver row (no backticks) → no duplicate append', () => {
    // User hand-edited the resolver to drop backticks. The original
    // backtick-only matcher missed this; broadened matcher catches it.
    const { root, skillsDir } = scratchRepo();
    const resolverPath = join(skillsDir, 'RESOLVER.md');
    const before = readFileSync(resolverPath, 'utf-8');
    writeFileSync(
      resolverPath,
      before +
        '\n## Uncategorized\n\n| Trigger | Skill |\n|---------|-------|\n' +
        '| "do thing" | skills/demo/SKILL.md |\n',
    );
    const plan = planScaffold({
      skillsDir,
      repoRoot: root,
      vars: { name: 'demo', description: 'd', triggers: ['do thing'], writesTo: [], writesPages: false, mutating: false },
    });
    expect(plan.resolverAppend).toBeNull();
  });

  it('detects double-quoted resolver row → no duplicate append', () => {
    const { root, skillsDir } = scratchRepo();
    const resolverPath = join(skillsDir, 'RESOLVER.md');
    const before = readFileSync(resolverPath, 'utf-8');
    writeFileSync(
      resolverPath,
      before +
        '\n## Uncategorized\n\n| Trigger | Skill |\n|---------|-------|\n' +
        '| "do thing" | "skills/demo/SKILL.md" |\n',
    );
    const plan = planScaffold({
      skillsDir,
      repoRoot: root,
      vars: { name: 'demo', description: 'd', triggers: ['do thing'], writesTo: [], writesPages: false, mutating: false },
    });
    expect(plan.resolverAppend).toBeNull();
  });

  it('detects single-quoted resolver row → no duplicate append', () => {
    const { root, skillsDir } = scratchRepo();
    const resolverPath = join(skillsDir, 'RESOLVER.md');
    const before = readFileSync(resolverPath, 'utf-8');
    writeFileSync(
      resolverPath,
      before +
        "\n## Uncategorized\n\n| Trigger | Skill |\n|---------|-------|\n" +
        "| \"do thing\" | 'skills/demo/SKILL.md' |\n",
    );
    const plan = planScaffold({
      skillsDir,
      repoRoot: root,
      vars: { name: 'demo', description: 'd', triggers: ['do thing'], writesTo: [], writesPages: false, mutating: false },
    });
    expect(plan.resolverAppend).toBeNull();
  });

  it('does NOT false-match a longer skill name with a shared prefix', () => {
    // If "demo" is the target but resolver only references
    // "demo-extended", we MUST treat that as no existing row (different
    // skill). Broadened matcher uses anchored boundaries to prevent this.
    const { root, skillsDir } = scratchRepo();
    const resolverPath = join(skillsDir, 'RESOLVER.md');
    const before = readFileSync(resolverPath, 'utf-8');
    writeFileSync(
      resolverPath,
      before +
        '\n## Uncategorized\n\n| Trigger | Skill |\n|---------|-------|\n' +
        '| "do extended" | `skills/demo-extended/SKILL.md` |\n',
    );
    const plan = planScaffold({
      skillsDir,
      repoRoot: root,
      vars: { name: 'demo', description: 'd', triggers: ['do thing'], writesTo: [], writesPages: false, mutating: false },
    });
    expect(plan.resolverAppend).not.toBeNull();
  });

  it('handles --triggers omitted by seeding TBD placeholder', () => {
    const { root, skillsDir } = scratchRepo();
    const plan = planScaffold({
      skillsDir,
      repoRoot: root,
      vars: {
        name: 'empty-triggers',
        description: 'test',
        triggers: [],
        writesTo: [],
        writesPages: false,
        mutating: false,
      },
    });
    const skillMd = plan.files.find(f => f.path.endsWith('SKILL.md'))!;
    expect(skillMd.content).toContain('TBD-trigger');
  });

  it('writes_pages + writes_to flow through to frontmatter', () => {
    const { root, skillsDir } = scratchRepo();
    const plan = planScaffold({
      skillsDir,
      repoRoot: root,
      vars: {
        name: 'writer',
        description: 'writer',
        triggers: ['write me'],
        writesTo: ['people/', 'companies/'],
        writesPages: true,
        mutating: true,
      },
    });
    const skillMd = plan.files.find(f => f.path.endsWith('SKILL.md'))!;
    expect(skillMd.content).toContain('writes_pages: true');
    expect(skillMd.content).toContain('- people/');
    expect(skillMd.content).toContain('- companies/');
    expect(skillMd.content).toContain('mutating: true');
  });
});

describe('applyScaffold', () => {
  it('writes all planned files and appends the resolver row', () => {
    const { root, skillsDir } = scratchRepo();
    const plan = planScaffold({
      skillsDir,
      repoRoot: root,
      vars: {
        name: 'hello',
        description: 'hi',
        triggers: ['say hi'],
        writesTo: [],
        writesPages: false,
        mutating: false,
      },
    });
    applyScaffold(plan);
    for (const f of plan.files) {
      expect(existsSync(f.path)).toBe(true);
    }
    const resolver = readFileSync(join(skillsDir, 'RESOLVER.md'), 'utf-8');
    expect(resolver).toContain('`skills/hello/SKILL.md`');
  });

  it('second apply with same name + --force overwrites files but does NOT duplicate resolver row (D-CX-7)', () => {
    const { root, skillsDir } = scratchRepo();

    const firstPlan = planScaffold({
      skillsDir,
      repoRoot: root,
      vars: {
        name: 'idem',
        description: 'first',
        triggers: ['t'],
        writesTo: [],
        writesPages: false,
        mutating: false,
      },
    });
    applyScaffold(firstPlan);

    const secondPlan = planScaffold({
      skillsDir,
      repoRoot: root,
      force: true,
      vars: {
        name: 'idem',
        description: 'second',
        triggers: ['t'],
        writesTo: [],
        writesPages: false,
        mutating: false,
      },
    });
    expect(secondPlan.resolverAppend).toBeNull();
    applyScaffold(secondPlan);

    const resolver = readFileSync(join(skillsDir, 'RESOLVER.md'), 'utf-8');
    const count = (resolver.match(/`skills\/idem\/SKILL\.md`/g) ?? []).length;
    expect(count).toBe(1);
  });

  it('applies against an AGENTS.md-layout workspace (W1 interop)', () => {
    const root = mkdtempSync(join(tmpdir(), 'skillify-openclaw-'));
    created.push(root);
    const skillsDir = join(root, 'workspace', 'skills');
    mkdirSync(skillsDir, { recursive: true });
    mkdirSync(join(root, 'test'), { recursive: true });
    // AGENTS.md at workspace root, NOT inside skills/.
    writeFileSync(
      join(root, 'workspace', 'AGENTS.md'),
      '# AGENTS\n\n## Ops\n\n| Trigger | Skill |\n|---------|-------|\n',
    );
    const plan = planScaffold({
      skillsDir,
      repoRoot: root,
      vars: {
        name: 'openclaw-demo',
        description: 'demo',
        triggers: ['do it'],
        writesTo: [],
        writesPages: false,
        mutating: false,
      },
    });
    expect(plan.resolverFile).toBe(join(root, 'workspace', 'AGENTS.md'));
    expect(plan.resolverAppend).not.toBeNull();
    applyScaffold(plan);
    const agents = readFileSync(join(root, 'workspace', 'AGENTS.md'), 'utf-8');
    expect(agents).toContain('`skills/openclaw-demo/SKILL.md`');
  });
});

describe('11-item scaffold contract (T9 + Phase 3 cross-modal eval)', () => {
  it('scaffolded SKILL.md teaches the cross-modal eval Phase 3', () => {
    // T9 (plans/radiant-napping-lerdorf.md): non-mutating verification of
    // the 11-item contract bump. Calls planScaffold + applyScaffold against
    // an in-memory tempdir, then asserts the produced SKILL.md mentions
    // cross-modal eval. No `gbrain skillify scaffold demo-eleven` shell-out
    // (which would mutate the real repo and require --description).
    const { root, skillsDir } = scratchRepo();
    const plan = planScaffold({
      skillsDir,
      vars: {
        name: 'phase-three-demo',
        description: 'demo skill that verifies Phase 3 lands in scaffold output',
        triggers: ['phase three demo'],
        writesTo: [],
        writesPages: false,
        mutating: false,
      },
    });
    applyScaffold(plan);

    const skillMdPath = join(skillsDir, 'phase-three-demo', 'SKILL.md');
    const body = readFileSync(skillMdPath, 'utf-8');

    expect(body).toContain('## Phase 3: Cross-modal eval');
    expect(body).toContain('gbrain eval cross-modal');
    expect(body).toContain('skills/phase-three-demo/SKILL.md');
    // Receipts naming convention is documented in the scaffold so the
    // implementer knows where to look.
    expect(body).toContain('eval-receipts');
    expect(body).toContain('<sha8>');
    // Pass criterion is documented in the scaffold (Q2 floor + Q3 floor).
    expect(body).toMatch(/dim mean\s*>=\s*7/i);
    expect(body).toMatch(/no model scored any dim\s*<\s*5/i);
    // Resolver row is appended once and only once — the verification step
    // does not need any cleanup beyond the tempdir afterEach drops.
    const resolverContent = readFileSync(join(skillsDir, 'RESOLVER.md'), 'utf-8');
    const occurrences = resolverContent.split('skills/phase-three-demo/SKILL.md').length - 1;
    expect(occurrences).toBe(1);
  });
});