From b6e8c5b03674a9b84d22221f9b792dec8976306e Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Mon, 1 Jun 2026 20:37:55 -0700 Subject: [PATCH] test(skillopt): held-out gate, ENFORCE, one-shot rewrite, runtime + receipt honesty New test/skillopt/rollout.test.ts (rollout had zero coverage). Held-out ENFORCE unit cases + one-shot-rewrite fence handling (whole-response unwrap, embedded-fence preserved, error path). E2E: F11 held-out BLOCKS/ALLOWS, bundled no-mutate write, reflectMode/disableValidationGate/optimizerMode, maxRuntimeMin abort, receipt baseline/test-score honesty, held-out/benchmark disjointness, D2 no-DB-pollution. --- test/e2e/skillopt-loop.serial.test.ts | 327 +++++++++++++++++++++++++- test/skillopt/held-out.test.ts | 34 +++ test/skillopt/reflect.test.ts | 42 +++- test/skillopt/rollout.test.ts | 123 ++++++++++ 4 files changed, 520 insertions(+), 6 deletions(-) create mode 100644 test/skillopt/rollout.test.ts diff --git a/test/e2e/skillopt-loop.serial.test.ts b/test/e2e/skillopt-loop.serial.test.ts index 05cd96c65..5b5038b3d 100644 --- a/test/e2e/skillopt-loop.serial.test.ts +++ b/test/e2e/skillopt-loop.serial.test.ts @@ -134,7 +134,30 @@ interface Fixture { cleanup: () => void; } -function setupFixture(skillBody: string = SKILL_PEOPLE_ONLY): Fixture { +/** 50 tasks all checking `contains: Citations` — baseline People-only fails them all. */ +const CITATIONS_BENCHMARK = Array.from({ length: 50 }, (_, i) => { + const n = String(i + 1).padStart(3, '0'); + return { + task_id: `cit-${n}`, + task: `Process task ${i + 1}`, + judge: { kind: 'rule' as const, checks: [{ op: 'contains' as const, arg: 'Citations' }] }, + }; +}); + +/** Held-out set checking `contains: People` — baseline passes, a People-dropping candidate fails. */ +const PEOPLE_HELDOUT = Array.from({ length: 6 }, (_, i) => { + const n = String(i + 1).padStart(3, '0'); + return { + task_id: `ho-${n}`, + task: `Held-out task ${i + 1}`, + judge: { kind: 'rule' as const, checks: [{ op: 'contains' as const, arg: 'People' }] }, + }; +}); + +function setupFixture( + skillBody: string = SKILL_PEOPLE_ONLY, + benchmark: ReadonlyArray<{ task_id: string; task: string; judge: unknown }> = SAMPLE_BENCHMARK, +): Fixture { const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'skillopt-loop-e2e-')); const skillDir = path.join(tmp, SKILL); fs.mkdirSync(skillDir, { recursive: true }); @@ -142,7 +165,7 @@ function setupFixture(skillBody: string = SKILL_PEOPLE_ONLY): Fixture { const benchmarkPath = path.join(skillDir, 'skillopt-benchmark.jsonl'); fs.writeFileSync( benchmarkPath, - SAMPLE_BENCHMARK.map((t) => JSON.stringify(t)).join('\n') + '\n', + benchmark.map((t) => JSON.stringify(t)).join('\n') + '\n', ); return { skillsDir: tmp, @@ -153,6 +176,13 @@ function setupFixture(skillBody: string = SKILL_PEOPLE_ONLY): Fixture { }; } +/** Write a held-out JSONL into the fixture's skill dir; return its path. */ +function writeHeldOut(fixture: Fixture, tasks: ReadonlyArray<{ task_id: string; task: string; judge: unknown }>): string { + const p = path.join(fixture.skillsDir, SKILL, 'held-out.jsonl'); + fs.writeFileSync(p, tasks.map((t) => JSON.stringify(t)).join('\n') + '\n'); + return p; +} + // ─── Stub builder ─────────────────────────────────────────────────────────── interface StubOpts { @@ -176,10 +206,17 @@ interface StubOpts { * fast against a tight cap). */ perCallUsage?: { input: number; output: number }; + /** Raw body returned for ONE-SHOT REWRITE optimizer calls (optimizerMode test). */ + oneShotBody?: string; + /** Counters incremented as the stub observes each call kind (ablation tests). */ + stats?: { successReflectCalls: number; oneShotCalls: number }; } -const REFLECT_OPTIMIZER_PREFIX = "You are SkillOpt's optimizer."; +// No trailing period: matches the FAILURE/SUCCESS reflect systems ("...optimizer.") +// AND the one-shot system ("...optimizer in ONE-SHOT REWRITE mode."). +const REFLECT_OPTIMIZER_PREFIX = "You are SkillOpt's optimizer"; const FAILURE_REFLECT_MARKER = 'FAILURE TRAJECTORIES'; +const ONE_SHOT_MARKER = 'ONE-SHOT REWRITE'; function defaultTargetText(skillText: string): string { // Faithful agent: read the skill's body, emit sections that exist there. @@ -219,10 +256,16 @@ function installStub(opts: StubOpts): void { if (isOptimizerCall) { const model = chatOpts.model ?? 'anthropic:claude-opus-4-7'; + // ONE-SHOT REWRITE mode returns a raw body, not edits JSON. + if (sys.includes(ONE_SHOT_MARKER)) { + if (opts.stats) opts.stats.oneShotCalls += 1; + return makeChatResult(opts.oneShotBody ?? '', model, usage); + } if (opts.optimizerRaw !== undefined) { return makeChatResult(opts.optimizerRaw, model, usage); } const isFailureMode = sys.includes(FAILURE_REFLECT_MARKER); + if (!isFailureMode && opts.stats) opts.stats.successReflectCalls += 1; const edit = isFailureMode ? opts.failureEdit : opts.successEdit; const text = JSON.stringify({ edits: edit ? [edit] : [] }); return makeChatResult(text, model, usage); @@ -245,6 +288,12 @@ interface RunOptsOverride { maxCostUsd?: number; epochs?: number; batchSize?: number; + noMutate?: boolean; + heldOutPath?: string; + optimizerMode?: 'reflect' | 'one-shot-rewrite'; + reflectMode?: 'both' | 'failure-only'; + disableValidationGate?: boolean; + maxRuntimeMin?: number; } async function runOnce(fixture: Fixture, over: RunOptsOverride = {}) { @@ -263,13 +312,17 @@ async function runOnce(fixture: Fixture, over: RunOptsOverride = {}) { judgeModel: 'anthropic:claude-sonnet-4-6', mode: 'patch', dryRun: false, - noMutate: false, + noMutate: over.noMutate ?? false, allowMutateBundled: true, bootstrapReviewed: false, json: true, maxCostUsd: over.maxCostUsd ?? 100, - maxRuntimeMin: 1, + maxRuntimeMin: over.maxRuntimeMin ?? 1, force: true, // bypass dirty-tree (tempdir isn't a git repo) + ...(over.heldOutPath ? { heldOutPath: over.heldOutPath } : {}), + ...(over.optimizerMode ? { optimizerMode: over.optimizerMode } : {}), + ...(over.reflectMode ? { reflectMode: over.reflectMode } : {}), + ...(over.disableValidationGate ? { disableValidationGate: over.disableValidationGate } : {}), }); } @@ -629,3 +682,267 @@ describe('skillopt full-loop E2E (happy path + broken cases)', () => { } }); }); + +// ─── T3: held-out gate + ablation opts + no-DB-pollution ───────────────────── + +describe('skillopt T3 — F11 held-out gate, ablation opts, no-DB-pollution', () => { + test('F11 held-out BLOCKS: candidate passes D_sel but regresses held-out → no commit', async () => { + // baseline People-only fails the Citations benchmark; the failure edit + // REPLACES People with Citations → candidate passes D_sel (Citations) but + // tanks the held-out (People) → held-out gate refuses the commit. + const fixture = setupFixture(SKILL_PEOPLE_ONLY, CITATIONS_BENCHMARK); + const heldOutPath = writeHeldOut(fixture, PEOPLE_HELDOUT); + try { + installStub({ + failureEdit: { + op: 'replace', + target: '## People\nList people mentioned.', + replacement: '## Citations\nCite the source.', + reason: 'swap People for Citations to pass the benchmark', + }, + successEdit: null, + }); + try { + await withEnv({ GBRAIN_AUDIT_DIR: fixture.skillsDir }, async () => { + const result = await runOnce(fixture, { heldOutPath }); + // Held-out gate blocked the promotion. + expect(result.outcome).toBe('no_improvement'); + // SKILL.md unchanged: still People-only, never swapped to Citations. + const skill = fs.readFileSync(skillPath(fixture.skillsDir, SKILL), 'utf8'); + expect(skill).toContain('## People'); + expect(skill).not.toContain('## Citations'); + // No committed history row. + expect(loadHistory(fixture.skillsDir, SKILL).filter((r) => r.status === 'committed')).toHaveLength(0); + }); + } finally { uninstallStub(); } + } finally { fixture.cleanup(); } + }); + + test('F11 held-out ALLOWS: candidate improves D_sel AND holds held-out → commit', async () => { + // ADD Citations (keep People): passes D_sel (Citations) and keeps held-out + // (People) at baseline → held-out gate allows → commit. + const fixture = setupFixture(SKILL_PEOPLE_ONLY, CITATIONS_BENCHMARK); + const heldOutPath = writeHeldOut(fixture, PEOPLE_HELDOUT); + try { + installStub({ + failureEdit: { op: 'add', anchor: 'People', content: '## Citations\nCite the source.', reason: 'add Citations' }, + successEdit: null, + }); + try { + await withEnv({ GBRAIN_AUDIT_DIR: fixture.skillsDir }, async () => { + const result = await runOnce(fixture, { heldOutPath }); + expect(result.outcome).toBe('accepted'); + const skill = fs.readFileSync(skillPath(fixture.skillsDir, SKILL), 'utf8'); + expect(skill).toContain('## People'); + expect(skill).toContain('## Citations'); + expect(loadHistory(fixture.skillsDir, SKILL).filter((r) => r.status === 'committed')).toHaveLength(1); + }); + } finally { uninstallStub(); } + } finally { fixture.cleanup(); } + }); + + test('--no-mutate writes proposed.md (best.md), leaves SKILL.md untouched', async () => { + const fixture = setupFixture(SKILL_PEOPLE_ONLY, CITATIONS_BENCHMARK); + try { + installStub({ + failureEdit: { op: 'add', anchor: 'People', content: '## Citations\nCite the source.', reason: 'add Citations' }, + successEdit: null, + }); + try { + await withEnv({ GBRAIN_AUDIT_DIR: fixture.skillsDir }, async () => { + const result = await runOnce(fixture, { noMutate: true }); + expect(result.outcome).toBe('accepted'); + expect(result.mutatedSkillFile).toBe(false); + expect(result.proposedPath).toBeDefined(); + // proposed.md (best.md) exists and carries the improvement. + expect(fs.existsSync(result.proposedPath!)).toBe(true); + expect(fs.readFileSync(result.proposedPath!, 'utf8')).toContain('## Citations'); + // SKILL.md on disk is UNCHANGED (still People-only). + const skill = fs.readFileSync(skillPath(fixture.skillsDir, SKILL), 'utf8'); + expect(skill).not.toContain('## Citations'); + }); + } finally { uninstallStub(); } + } finally { fixture.cleanup(); } + }); + + test('optimizerMode one-shot-rewrite: single rewrite, no epoch loop, receipt records mode', async () => { + const fixture = setupFixture(SKILL_PEOPLE_ONLY, CITATIONS_BENCHMARK); + const stats = { successReflectCalls: 0, oneShotCalls: 0 }; + try { + installStub({ + stats, + // Body-only rewrite (frontmatter re-attached by the orchestrator). + oneShotBody: '# E2E Loop Test Skill\n\nProduce a structured output.\n\n## People\nList people.\n\n## Citations\nCite the source.\n', + }); + try { + await withEnv({ GBRAIN_AUDIT_DIR: fixture.skillsDir }, async () => { + const result = await runOnce(fixture, { optimizerMode: 'one-shot-rewrite' }); + expect(result.outcome).toBe('accepted'); + expect(result.receipt.optimizer_mode).toBe('one-shot-rewrite'); + // Exactly ONE optimizer rewrite call — no epoch loop. + expect(stats.oneShotCalls).toBe(1); + expect(result.receipt.total_steps).toBe(1); + const skill = fs.readFileSync(skillPath(fixture.skillsDir, SKILL), 'utf8'); + expect(skill).toContain('## Citations'); + // Frontmatter preserved (D5). + expect(skill).toContain('name: e2e-loop-skill'); + }); + } finally { uninstallStub(); } + } finally { fixture.cleanup(); } + }); + + test('reflectMode failure-only SKIPS the success reflect call; default fires it', async () => { + // Mixed benchmark → baseline has both successes (People tasks) and failures + // (Citations tasks), so default mode WOULD fire the success reflect. + const failOnly = { successReflectCalls: 0, oneShotCalls: 0 }; + const fixtureA = setupFixture(SKILL_PEOPLE_ONLY, SAMPLE_BENCHMARK); + try { + installStub({ + stats: failOnly, + failureEdit: { op: 'add', anchor: 'People', content: '## Citations\nCite.', reason: 'add' }, + successEdit: { op: 'add', anchor: 'People', content: '', reason: 'note' }, + }); + try { + await withEnv({ GBRAIN_AUDIT_DIR: fixtureA.skillsDir }, async () => { + await runOnce(fixtureA, { reflectMode: 'failure-only' }); + expect(failOnly.successReflectCalls).toBe(0); + }); + } finally { uninstallStub(); } + } finally { fixtureA.cleanup(); } + + const both = { successReflectCalls: 0, oneShotCalls: 0 }; + const fixtureB = setupFixture(SKILL_PEOPLE_ONLY, SAMPLE_BENCHMARK); + try { + installStub({ + stats: both, + failureEdit: { op: 'add', anchor: 'People', content: '## Citations\nCite.', reason: 'add' }, + successEdit: { op: 'add', anchor: 'People', content: '', reason: 'note' }, + }); + try { + await withEnv({ GBRAIN_AUDIT_DIR: fixtureB.skillsDir }, async () => { + await runOnce(fixtureB); + expect(both.successReflectCalls).toBeGreaterThan(0); + }); + } finally { uninstallStub(); } + } finally { fixtureB.cleanup(); } + }); + + test('disableValidationGate greedy-accepts a no-improvement edit the gate would reject', async () => { + // BOTH_SECTIONS already scores 1.0; a benign success edit yields delta 0, + // which the D12 gate rejects — unless disableValidationGate greedy-accepts. + const benignEdit = { op: 'add' as const, anchor: 'Citations', content: 'Extra note.', reason: 'benign' }; + + const fixtureGated = setupFixture(SKILL_BOTH_SECTIONS, SAMPLE_BENCHMARK); + try { + installStub({ successEdit: benignEdit, failureEdit: null }); + try { + await withEnv({ GBRAIN_AUDIT_DIR: fixtureGated.skillsDir }, async () => { + const result = await runOnce(fixtureGated); + expect(result.outcome).toBe('no_improvement'); // gate rejected + }); + } finally { uninstallStub(); } + } finally { fixtureGated.cleanup(); } + + const fixtureGreedy = setupFixture(SKILL_BOTH_SECTIONS, SAMPLE_BENCHMARK); + try { + installStub({ successEdit: benignEdit, failureEdit: null }); + try { + await withEnv({ GBRAIN_AUDIT_DIR: fixtureGreedy.skillsDir }, async () => { + const result = await runOnce(fixtureGreedy, { disableValidationGate: true }); + expect(result.outcome).toBe('accepted'); // greedy + expect(result.receipt.validation_gate_disabled).toBe(true); + }); + } finally { uninstallStub(); } + } finally { fixtureGreedy.cleanup(); } + }); + + test('D2 no-DB-pollution: subagent_messages count unchanged across a full run', async () => { + const fixture = setupFixture(SKILL_PEOPLE_ONLY, CITATIONS_BENCHMARK); + const countMessages = async (): Promise => { + const r = await engine.executeRaw('SELECT COUNT(*) AS c FROM subagent_messages', []); + const rows = Array.isArray(r) ? r : ((r as { rows?: unknown[] }).rows ?? []); + return Number((rows[0] as { c?: number | string })?.c ?? 0); + }; + try { + const before = await countMessages(); + installStub({ + failureEdit: { op: 'add', anchor: 'People', content: '## Citations\nCite the source.', reason: 'add' }, + successEdit: null, + }); + try { + await withEnv({ GBRAIN_AUDIT_DIR: fixture.skillsDir }, async () => { + await runOnce(fixture); + }); + } finally { uninstallStub(); } + const after = await countMessages(); + // Rollouts use gateway.toolLoop with no-op persistence (D2) → zero rows written. + expect(after).toBe(before); + } finally { fixture.cleanup(); } + }); +}); + +// ─── T3 (review follow-ups): maxRuntimeMin abort + receipt honesty ─────────── + +describe('skillopt T3 — runtime deadline + receipt score honesty', () => { + test('maxRuntimeMin deadline aborts cleanly with no commit', async () => { + const fixture = setupFixture(SKILL_PEOPLE_ONLY, CITATIONS_BENCHMARK); + try { + installStub({ + failureEdit: { op: 'add', anchor: 'People', content: '## Citations\nCite the source.', reason: 'add' }, + successEdit: null, + }); + try { + await withEnv({ GBRAIN_AUDIT_DIR: fixture.skillsDir }, async () => { + // maxRuntimeMin:0 → deadline == run start; the first step's deadline + // check fires after the baseline eval has already elapsed → abort. + const result = await runOnce(fixture, { maxRuntimeMin: 0 }); + expect(result.outcome).toBe('aborted'); + // No commit: SKILL.md unchanged, no committed history row. + const skill = fs.readFileSync(skillPath(fixture.skillsDir, SKILL), 'utf8'); + expect(skill).not.toContain('## Citations'); + expect(loadHistory(fixture.skillsDir, SKILL).filter((r) => r.status === 'committed')).toHaveLength(0); + }); + } finally { uninstallStub(); } + } finally { fixture.cleanup(); } + }); + + test('receipt records the REAL baseline_sel_score + final-test scores (regression: was hardcoded 0)', async () => { + // BOTH_SECTIONS already scores ~1.0 on the alternating benchmark, so a real + // baseline read is ~1.0 — a hardcoded-0 receipt would fail this immediately. + const fixture = setupFixture(SKILL_BOTH_SECTIONS, SAMPLE_BENCHMARK); + try { + installStub({ successEdit: null, failureEdit: null }); // baseline already perfect; no edits + try { + await withEnv({ GBRAIN_AUDIT_DIR: fixture.skillsDir }, async () => { + const r = (await runOnce(fixture)).receipt; + // Real baseline, not the old hardcoded 0. + expect(r.baseline_sel_score).toBeGreaterThan(0.9); + // Final-test eval populated both test scores (D_test non-empty under 4:1:5). + expect(typeof r.test_score).toBe('number'); + expect(typeof r.baseline_test_score).toBe('number'); + expect(r.baseline_test_score!).toBeGreaterThan(0.9); + }); + } finally { uninstallStub(); } + } finally { fixture.cleanup(); } + }); +}); + +describe('skillopt T3 — held-out independence guard', () => { + test('held-out sharing task_ids with the benchmark is rejected (gaming defense)', async () => { + const fixture = setupFixture(SKILL_PEOPLE_ONLY, CITATIONS_BENCHMARK); + // Held-out file reuses benchmark task_ids (cit-001..006) → must be rejected + // before any optimization, since an overlapping held-out can't catch overfit. + const heldOutPath = writeHeldOut(fixture, CITATIONS_BENCHMARK.slice(0, 6)); + try { + installStub({ + failureEdit: { op: 'add', anchor: 'People', content: '## Citations\nCite.', reason: 'add' }, + successEdit: null, + }); + try { + await withEnv({ GBRAIN_AUDIT_DIR: fixture.skillsDir }, async () => { + await expect(runOnce(fixture, { heldOutPath })).rejects.toThrow(/independent|shares .* task_id/i); + }); + } finally { uninstallStub(); } + } finally { fixture.cleanup(); } + }); +}); diff --git a/test/skillopt/held-out.test.ts b/test/skillopt/held-out.test.ts index c1b12a1ae..af9a17bf6 100644 --- a/test/skillopt/held-out.test.ts +++ b/test/skillopt/held-out.test.ts @@ -15,7 +15,9 @@ import { capturesDir, loadHeldOut, runHeldOutGate, + MIN_HELD_OUT_SIZE, } from '../../src/core/skillopt/held-out.ts'; +import { assertBundledMutationHeldOut } from '../../src/core/skillopt/bundled-skill-gate.ts'; let tmp: string; @@ -109,3 +111,35 @@ describe('F11 runHeldOutGate vacuous case', () => { expect(result.candidateScore).toBe(0); }); }); + +describe('D16 assertBundledMutationHeldOut (core ENFORCE)', () => { + test('bundled + mutate + too-small held-out → throws (hard refuse)', () => { + expect(() => assertBundledMutationHeldOut({ + isBundled: true, willMutate: true, heldOutCount: MIN_HELD_OUT_SIZE - 1, skillName: 'brain-ops', + })).toThrow(/held-out/i); + }); + + test('bundled + mutate + empty held-out → throws (vacuous-pass hole closed)', () => { + expect(() => assertBundledMutationHeldOut({ + isBundled: true, willMutate: true, heldOutCount: 0, skillName: 'brain-ops', + })).toThrow(/non-empty held-out/i); + }); + + test('bundled + mutate + sufficient held-out → no throw', () => { + expect(() => assertBundledMutationHeldOut({ + isBundled: true, willMutate: true, heldOutCount: MIN_HELD_OUT_SIZE, skillName: 'brain-ops', + })).not.toThrow(); + }); + + test('not bundled → no throw even with zero held-out (user skills are free)', () => { + expect(() => assertBundledMutationHeldOut({ + isBundled: false, willMutate: true, heldOutCount: 0, skillName: 'my-skill', + })).not.toThrow(); + }); + + test('bundled but NOT mutating (no-mutate / proposed.md path) → no throw', () => { + expect(() => assertBundledMutationHeldOut({ + isBundled: true, willMutate: false, heldOutCount: 0, skillName: 'brain-ops', + })).not.toThrow(); + }); +}); diff --git a/test/skillopt/reflect.test.ts b/test/skillopt/reflect.test.ts index 99a058501..9eb431307 100644 --- a/test/skillopt/reflect.test.ts +++ b/test/skillopt/reflect.test.ts @@ -18,7 +18,7 @@ */ import { describe, expect, test } from 'bun:test'; -import { parseEditsResponse, runReflect } from '../../src/core/skillopt/reflect.ts'; +import { parseEditsResponse, runReflect, runOneShotRewrite } from '../../src/core/skillopt/reflect.ts'; import type { ChatOpts, ChatResult } from '../../src/core/ai/gateway.ts'; import type { ScoredRollout, Trajectory } from '../../src/core/skillopt/types.ts'; @@ -287,3 +287,43 @@ describe('runReflect (D7 two-call contract)', () => { expect(observedUserMsg).toContain('validation_gate_below_baseline'); }); }); + +// ─── runOneShotRewrite (cat31 config C baseline) ──────────────────────────── + +function oneShotChatStub(text: string): NonNullable[0]['chatFn']> { + return (async (_opts: ChatOpts): Promise => ({ + text, + blocks: [{ type: 'text', text }], + stopReason: 'end', + usage: { input_tokens: 1, output_tokens: 1, cache_read_tokens: 0, cache_creation_tokens: 0 }, + model: 'm', + providerId: 'anthropic', + })) as NonNullable[0]['chatFn']>; +} + +const ONE_SHOT_BASE = { skillBodyText: '# Skill', successes: [] as ScoredRollout[], failures: [] as ScoredRollout[], rejected: [], optimizerModel: 'm' }; + +describe('runOneShotRewrite', () => { + test('unwraps a whole-response code fence', async () => { + const body = '# Skill\n\n## People\nList people.'; + const r = await runOneShotRewrite({ ...ONE_SHOT_BASE, chatFn: oneShotChatStub('```markdown\n' + body + '\n```') }); + expect(r.newBody).toBe(body); + expect(r.error).toBeUndefined(); + }); + + test('preserves an EMBEDDED code fence — does not truncate to the first block', async () => { + const body = '# Skill\n\nRun this:\n```bash\nls -la\n```\n\n## People\nList people.'; + const r = await runOneShotRewrite({ ...ONE_SHOT_BASE, chatFn: oneShotChatStub(body) }); + expect(r.newBody).toBe(body); + expect(r.newBody).toContain('## People'); // regression: old non-anchored regex truncated to the bash block + }); + + test('chat error returns empty newBody + error (caller treats as no-change)', async () => { + const r = await runOneShotRewrite({ + ...ONE_SHOT_BASE, + chatFn: (async () => { throw new Error('boom'); }) as NonNullable[0]['chatFn']>, + }); + expect(r.newBody).toBe(''); + expect(r.error).toContain('one_shot_rewrite_failed'); + }); +}); diff --git a/test/skillopt/rollout.test.ts b/test/skillopt/rollout.test.ts new file mode 100644 index 000000000..c4f1d6fee --- /dev/null +++ b/test/skillopt/rollout.test.ts @@ -0,0 +1,123 @@ +/** + * SkillOpt rollout tests (the `toolLoopFn` DI seam — previously zero coverage). + * + * Covers: trajectory shape capture, READ_ONLY_BRAIN_TOOLS zero-write invariant, + * default read-only tool registry (no put_page/submit_job), --write-capture + * routing to the virtual write registry, and tool-call ordering via the + * onToolCallStart callback. Hermetic: the toolLoop transport is stubbed, so no + * LLM calls, no DB, no API keys. Handlers are never executed (the stub returns + * messages directly), so a `{}`-shaped engine is sufficient. + */ + +import { describe, expect, test } from 'bun:test'; +import { runRollout, READ_ONLY_BRAIN_TOOLS } from '../../src/core/skillopt/rollout.ts'; +import type { BenchmarkTask } from '../../src/core/skillopt/types.ts'; + +const TASK: BenchmarkTask = { + task_id: 't1', + task: 'do the thing', + judge: { kind: 'rule', checks: [] }, +}; + +/** Build a stubbed toolLoop that records the tool defs it was handed. */ +function makeStubLoop(opts: { capturedTools?: Array<{ name: string }>; finalText?: string }) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + return async (loopOpts: any) => { + if (opts.capturedTools) opts.capturedTools.push(...loopOpts.tools); + return { + messages: [ + { role: 'user', content: 'do the thing' }, + { role: 'assistant', content: opts.finalText ?? 'done' }, + ], + totalUsage: { input_tokens: 10, output_tokens: 5, cache_read_tokens: 0, cache_creation_tokens: 0 }, + totalTurns: 2, + stopReason: 'end', + }; + }; +} + +describe('rollout — READ_ONLY_BRAIN_TOOLS zero-write invariant', () => { + test('excludes put_page / submit_job / file_upload, includes a read op', () => { + expect(READ_ONLY_BRAIN_TOOLS.has('put_page')).toBe(false); + expect(READ_ONLY_BRAIN_TOOLS.has('submit_job')).toBe(false); + expect(READ_ONLY_BRAIN_TOOLS.has('file_upload')).toBe(false); + expect(READ_ONLY_BRAIN_TOOLS.has('search')).toBe(true); + }); +}); + +describe('rollout — trajectory capture', () => { + test('returns a Trajectory with the expected shape + threaded usage/stop_reason', async () => { + const traj = await runRollout({ + engine: {} as never, + skillText: 'skill body', + task: TASK, + targetModel: 'anthropic:claude-sonnet-4-6', + toolLoopFn: makeStubLoop({ finalText: 'the answer' }) as never, + }); + expect(traj.task_id).toBe('t1'); + expect(traj.task).toBe('do the thing'); + expect(traj.final_text).toBe('the answer'); + expect(traj.stop_reason).toBe('end'); + expect(traj.turns).toBe(2); + expect(traj.usage.input_tokens).toBe(10); + expect(traj.usage.output_tokens).toBe(5); + expect(typeof traj.duration_ms).toBe('number'); + expect(Array.isArray(traj.tool_calls)).toBe(true); + }); +}); + +describe('rollout — tool registry routing', () => { + test('default rollout passes ONLY read-only tools (no write defs)', async () => { + const capturedTools: Array<{ name: string }> = []; + await runRollout({ + engine: {} as never, + skillText: 's', + task: TASK, + targetModel: 'm', + toolLoopFn: makeStubLoop({ capturedTools }) as never, + }); + const names = capturedTools.map((d) => d.name); + expect(names).toContain('brain_search'); + expect(names).not.toContain('brain_put_page'); + expect(names).not.toContain('brain_submit_job'); + expect(names).not.toContain('brain_file_upload'); + }); + + test('--write-capture routes to the virtual write registry (put_page present, captured not real)', async () => { + const capturedTools: Array<{ name: string }> = []; + await runRollout({ + engine: {} as never, + skillText: 's', + task: TASK, + targetModel: 'm', + writeCapture: true, + toolLoopFn: makeStubLoop({ capturedTools }) as never, + }); + const names = capturedTools.map((d) => d.name); + expect(names).toContain('brain_put_page'); + expect(names).toContain('brain_submit_job'); + expect(names).toContain('brain_file_upload'); + }); +}); + +describe('rollout — tool-call ordering', () => { + test('onToolCallStart records calls in order with the brain_ prefix stripped', async () => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const loop = async (loopOpts: any) => { + await loopOpts.onToolCallStart?.(0, 0, 0, 'brain_search', { q: 'x' }, 'pc1'); + await loopOpts.onToolCallStart?.(0, 1, 1, 'brain_get_page', { slug: 'y' }, 'pc2'); + return { + messages: [{ role: 'assistant', content: 'ok' }], + totalUsage: { input_tokens: 1, output_tokens: 1, cache_read_tokens: 0, cache_creation_tokens: 0 }, + totalTurns: 1, + stopReason: 'end', + }; + }; + const traj = await runRollout({ + engine: {} as never, skillText: 's', task: TASK, targetModel: 'm', toolLoopFn: loop as never, + }); + expect(traj.tool_calls).toHaveLength(2); + expect(traj.tool_calls[0]!.name).toBe('search'); + expect(traj.tool_calls[1]!.name).toBe('get_page'); + }); +});