mirror of
https://github.com/garrytan/gbrain.git
synced 2026-07-27 22:15:33 +00:00
test(skillopt): held-out gate, ENFORCE, one-shot rewrite, runtime + receipt honesty
New test/skillopt/rollout.test.ts (rollout had zero coverage). Held-out ENFORCE unit cases + one-shot-rewrite fence handling (whole-response unwrap, embedded-fence preserved, error path). E2E: F11 held-out BLOCKS/ALLOWS, bundled no-mutate write, reflectMode/disableValidationGate/optimizerMode, maxRuntimeMin abort, receipt baseline/test-score honesty, held-out/benchmark disjointness, D2 no-DB-pollution.
This commit is contained in:
Garry Tan
@@ -134,7 +134,30 @@ interface Fixture {
|
||||
cleanup: () => void;
|
||||
}
|
||||
|
||||
function setupFixture(skillBody: string = SKILL_PEOPLE_ONLY): Fixture {
|
||||
/** 50 tasks all checking `contains: Citations` — baseline People-only fails them all. */
|
||||
const CITATIONS_BENCHMARK = Array.from({ length: 50 }, (_, i) => {
|
||||
const n = String(i + 1).padStart(3, '0');
|
||||
return {
|
||||
task_id: `cit-${n}`,
|
||||
task: `Process task ${i + 1}`,
|
||||
judge: { kind: 'rule' as const, checks: [{ op: 'contains' as const, arg: 'Citations' }] },
|
||||
};
|
||||
});
|
||||
|
||||
/** Held-out set checking `contains: People` — baseline passes, a People-dropping candidate fails. */
|
||||
const PEOPLE_HELDOUT = Array.from({ length: 6 }, (_, i) => {
|
||||
const n = String(i + 1).padStart(3, '0');
|
||||
return {
|
||||
task_id: `ho-${n}`,
|
||||
task: `Held-out task ${i + 1}`,
|
||||
judge: { kind: 'rule' as const, checks: [{ op: 'contains' as const, arg: 'People' }] },
|
||||
};
|
||||
});
|
||||
|
||||
function setupFixture(
|
||||
skillBody: string = SKILL_PEOPLE_ONLY,
|
||||
benchmark: ReadonlyArray<{ task_id: string; task: string; judge: unknown }> = SAMPLE_BENCHMARK,
|
||||
): Fixture {
|
||||
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'skillopt-loop-e2e-'));
|
||||
const skillDir = path.join(tmp, SKILL);
|
||||
fs.mkdirSync(skillDir, { recursive: true });
|
||||
@@ -142,7 +165,7 @@ function setupFixture(skillBody: string = SKILL_PEOPLE_ONLY): Fixture {
|
||||
const benchmarkPath = path.join(skillDir, 'skillopt-benchmark.jsonl');
|
||||
fs.writeFileSync(
|
||||
benchmarkPath,
|
||||
SAMPLE_BENCHMARK.map((t) => JSON.stringify(t)).join('\n') + '\n',
|
||||
benchmark.map((t) => JSON.stringify(t)).join('\n') + '\n',
|
||||
);
|
||||
return {
|
||||
skillsDir: tmp,
|
||||
@@ -153,6 +176,13 @@ function setupFixture(skillBody: string = SKILL_PEOPLE_ONLY): Fixture {
|
||||
};
|
||||
}
|
||||
|
||||
/** Write a held-out JSONL into the fixture's skill dir; return its path. */
|
||||
function writeHeldOut(fixture: Fixture, tasks: ReadonlyArray<{ task_id: string; task: string; judge: unknown }>): string {
|
||||
const p = path.join(fixture.skillsDir, SKILL, 'held-out.jsonl');
|
||||
fs.writeFileSync(p, tasks.map((t) => JSON.stringify(t)).join('\n') + '\n');
|
||||
return p;
|
||||
}
|
||||
|
||||
// ─── Stub builder ───────────────────────────────────────────────────────────
|
||||
|
||||
interface StubOpts {
|
||||
@@ -176,10 +206,17 @@ interface StubOpts {
|
||||
* fast against a tight cap).
|
||||
*/
|
||||
perCallUsage?: { input: number; output: number };
|
||||
/** Raw body returned for ONE-SHOT REWRITE optimizer calls (optimizerMode test). */
|
||||
oneShotBody?: string;
|
||||
/** Counters incremented as the stub observes each call kind (ablation tests). */
|
||||
stats?: { successReflectCalls: number; oneShotCalls: number };
|
||||
}
|
||||
|
||||
const REFLECT_OPTIMIZER_PREFIX = "You are SkillOpt's optimizer.";
|
||||
// No trailing period: matches the FAILURE/SUCCESS reflect systems ("...optimizer.")
|
||||
// AND the one-shot system ("...optimizer in ONE-SHOT REWRITE mode.").
|
||||
const REFLECT_OPTIMIZER_PREFIX = "You are SkillOpt's optimizer";
|
||||
const FAILURE_REFLECT_MARKER = 'FAILURE TRAJECTORIES';
|
||||
const ONE_SHOT_MARKER = 'ONE-SHOT REWRITE';
|
||||
|
||||
function defaultTargetText(skillText: string): string {
|
||||
// Faithful agent: read the skill's body, emit sections that exist there.
|
||||
@@ -219,10 +256,16 @@ function installStub(opts: StubOpts): void {
|
||||
|
||||
if (isOptimizerCall) {
|
||||
const model = chatOpts.model ?? 'anthropic:claude-opus-4-7';
|
||||
// ONE-SHOT REWRITE mode returns a raw body, not edits JSON.
|
||||
if (sys.includes(ONE_SHOT_MARKER)) {
|
||||
if (opts.stats) opts.stats.oneShotCalls += 1;
|
||||
return makeChatResult(opts.oneShotBody ?? '', model, usage);
|
||||
}
|
||||
if (opts.optimizerRaw !== undefined) {
|
||||
return makeChatResult(opts.optimizerRaw, model, usage);
|
||||
}
|
||||
const isFailureMode = sys.includes(FAILURE_REFLECT_MARKER);
|
||||
if (!isFailureMode && opts.stats) opts.stats.successReflectCalls += 1;
|
||||
const edit = isFailureMode ? opts.failureEdit : opts.successEdit;
|
||||
const text = JSON.stringify({ edits: edit ? [edit] : [] });
|
||||
return makeChatResult(text, model, usage);
|
||||
@@ -245,6 +288,12 @@ interface RunOptsOverride {
|
||||
maxCostUsd?: number;
|
||||
epochs?: number;
|
||||
batchSize?: number;
|
||||
noMutate?: boolean;
|
||||
heldOutPath?: string;
|
||||
optimizerMode?: 'reflect' | 'one-shot-rewrite';
|
||||
reflectMode?: 'both' | 'failure-only';
|
||||
disableValidationGate?: boolean;
|
||||
maxRuntimeMin?: number;
|
||||
}
|
||||
|
||||
async function runOnce(fixture: Fixture, over: RunOptsOverride = {}) {
|
||||
@@ -263,13 +312,17 @@ async function runOnce(fixture: Fixture, over: RunOptsOverride = {}) {
|
||||
judgeModel: 'anthropic:claude-sonnet-4-6',
|
||||
mode: 'patch',
|
||||
dryRun: false,
|
||||
noMutate: false,
|
||||
noMutate: over.noMutate ?? false,
|
||||
allowMutateBundled: true,
|
||||
bootstrapReviewed: false,
|
||||
json: true,
|
||||
maxCostUsd: over.maxCostUsd ?? 100,
|
||||
maxRuntimeMin: 1,
|
||||
maxRuntimeMin: over.maxRuntimeMin ?? 1,
|
||||
force: true, // bypass dirty-tree (tempdir isn't a git repo)
|
||||
...(over.heldOutPath ? { heldOutPath: over.heldOutPath } : {}),
|
||||
...(over.optimizerMode ? { optimizerMode: over.optimizerMode } : {}),
|
||||
...(over.reflectMode ? { reflectMode: over.reflectMode } : {}),
|
||||
...(over.disableValidationGate ? { disableValidationGate: over.disableValidationGate } : {}),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -629,3 +682,267 @@ describe('skillopt full-loop E2E (happy path + broken cases)', () => {
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// ─── T3: held-out gate + ablation opts + no-DB-pollution ─────────────────────
|
||||
|
||||
describe('skillopt T3 — F11 held-out gate, ablation opts, no-DB-pollution', () => {
|
||||
test('F11 held-out BLOCKS: candidate passes D_sel but regresses held-out → no commit', async () => {
|
||||
// baseline People-only fails the Citations benchmark; the failure edit
|
||||
// REPLACES People with Citations → candidate passes D_sel (Citations) but
|
||||
// tanks the held-out (People) → held-out gate refuses the commit.
|
||||
const fixture = setupFixture(SKILL_PEOPLE_ONLY, CITATIONS_BENCHMARK);
|
||||
const heldOutPath = writeHeldOut(fixture, PEOPLE_HELDOUT);
|
||||
try {
|
||||
installStub({
|
||||
failureEdit: {
|
||||
op: 'replace',
|
||||
target: '## People\nList people mentioned.',
|
||||
replacement: '## Citations\nCite the source.',
|
||||
reason: 'swap People for Citations to pass the benchmark',
|
||||
},
|
||||
successEdit: null,
|
||||
});
|
||||
try {
|
||||
await withEnv({ GBRAIN_AUDIT_DIR: fixture.skillsDir }, async () => {
|
||||
const result = await runOnce(fixture, { heldOutPath });
|
||||
// Held-out gate blocked the promotion.
|
||||
expect(result.outcome).toBe('no_improvement');
|
||||
// SKILL.md unchanged: still People-only, never swapped to Citations.
|
||||
const skill = fs.readFileSync(skillPath(fixture.skillsDir, SKILL), 'utf8');
|
||||
expect(skill).toContain('## People');
|
||||
expect(skill).not.toContain('## Citations');
|
||||
// No committed history row.
|
||||
expect(loadHistory(fixture.skillsDir, SKILL).filter((r) => r.status === 'committed')).toHaveLength(0);
|
||||
});
|
||||
} finally { uninstallStub(); }
|
||||
} finally { fixture.cleanup(); }
|
||||
});
|
||||
|
||||
test('F11 held-out ALLOWS: candidate improves D_sel AND holds held-out → commit', async () => {
|
||||
// ADD Citations (keep People): passes D_sel (Citations) and keeps held-out
|
||||
// (People) at baseline → held-out gate allows → commit.
|
||||
const fixture = setupFixture(SKILL_PEOPLE_ONLY, CITATIONS_BENCHMARK);
|
||||
const heldOutPath = writeHeldOut(fixture, PEOPLE_HELDOUT);
|
||||
try {
|
||||
installStub({
|
||||
failureEdit: { op: 'add', anchor: 'People', content: '## Citations\nCite the source.', reason: 'add Citations' },
|
||||
successEdit: null,
|
||||
});
|
||||
try {
|
||||
await withEnv({ GBRAIN_AUDIT_DIR: fixture.skillsDir }, async () => {
|
||||
const result = await runOnce(fixture, { heldOutPath });
|
||||
expect(result.outcome).toBe('accepted');
|
||||
const skill = fs.readFileSync(skillPath(fixture.skillsDir, SKILL), 'utf8');
|
||||
expect(skill).toContain('## People');
|
||||
expect(skill).toContain('## Citations');
|
||||
expect(loadHistory(fixture.skillsDir, SKILL).filter((r) => r.status === 'committed')).toHaveLength(1);
|
||||
});
|
||||
} finally { uninstallStub(); }
|
||||
} finally { fixture.cleanup(); }
|
||||
});
|
||||
|
||||
test('--no-mutate writes proposed.md (best.md), leaves SKILL.md untouched', async () => {
|
||||
const fixture = setupFixture(SKILL_PEOPLE_ONLY, CITATIONS_BENCHMARK);
|
||||
try {
|
||||
installStub({
|
||||
failureEdit: { op: 'add', anchor: 'People', content: '## Citations\nCite the source.', reason: 'add Citations' },
|
||||
successEdit: null,
|
||||
});
|
||||
try {
|
||||
await withEnv({ GBRAIN_AUDIT_DIR: fixture.skillsDir }, async () => {
|
||||
const result = await runOnce(fixture, { noMutate: true });
|
||||
expect(result.outcome).toBe('accepted');
|
||||
expect(result.mutatedSkillFile).toBe(false);
|
||||
expect(result.proposedPath).toBeDefined();
|
||||
// proposed.md (best.md) exists and carries the improvement.
|
||||
expect(fs.existsSync(result.proposedPath!)).toBe(true);
|
||||
expect(fs.readFileSync(result.proposedPath!, 'utf8')).toContain('## Citations');
|
||||
// SKILL.md on disk is UNCHANGED (still People-only).
|
||||
const skill = fs.readFileSync(skillPath(fixture.skillsDir, SKILL), 'utf8');
|
||||
expect(skill).not.toContain('## Citations');
|
||||
});
|
||||
} finally { uninstallStub(); }
|
||||
} finally { fixture.cleanup(); }
|
||||
});
|
||||
|
||||
test('optimizerMode one-shot-rewrite: single rewrite, no epoch loop, receipt records mode', async () => {
|
||||
const fixture = setupFixture(SKILL_PEOPLE_ONLY, CITATIONS_BENCHMARK);
|
||||
const stats = { successReflectCalls: 0, oneShotCalls: 0 };
|
||||
try {
|
||||
installStub({
|
||||
stats,
|
||||
// Body-only rewrite (frontmatter re-attached by the orchestrator).
|
||||
oneShotBody: '# E2E Loop Test Skill\n\nProduce a structured output.\n\n## People\nList people.\n\n## Citations\nCite the source.\n',
|
||||
});
|
||||
try {
|
||||
await withEnv({ GBRAIN_AUDIT_DIR: fixture.skillsDir }, async () => {
|
||||
const result = await runOnce(fixture, { optimizerMode: 'one-shot-rewrite' });
|
||||
expect(result.outcome).toBe('accepted');
|
||||
expect(result.receipt.optimizer_mode).toBe('one-shot-rewrite');
|
||||
// Exactly ONE optimizer rewrite call — no epoch loop.
|
||||
expect(stats.oneShotCalls).toBe(1);
|
||||
expect(result.receipt.total_steps).toBe(1);
|
||||
const skill = fs.readFileSync(skillPath(fixture.skillsDir, SKILL), 'utf8');
|
||||
expect(skill).toContain('## Citations');
|
||||
// Frontmatter preserved (D5).
|
||||
expect(skill).toContain('name: e2e-loop-skill');
|
||||
});
|
||||
} finally { uninstallStub(); }
|
||||
} finally { fixture.cleanup(); }
|
||||
});
|
||||
|
||||
test('reflectMode failure-only SKIPS the success reflect call; default fires it', async () => {
|
||||
// Mixed benchmark → baseline has both successes (People tasks) and failures
|
||||
// (Citations tasks), so default mode WOULD fire the success reflect.
|
||||
const failOnly = { successReflectCalls: 0, oneShotCalls: 0 };
|
||||
const fixtureA = setupFixture(SKILL_PEOPLE_ONLY, SAMPLE_BENCHMARK);
|
||||
try {
|
||||
installStub({
|
||||
stats: failOnly,
|
||||
failureEdit: { op: 'add', anchor: 'People', content: '## Citations\nCite.', reason: 'add' },
|
||||
successEdit: { op: 'add', anchor: 'People', content: '<!-- success note -->', reason: 'note' },
|
||||
});
|
||||
try {
|
||||
await withEnv({ GBRAIN_AUDIT_DIR: fixtureA.skillsDir }, async () => {
|
||||
await runOnce(fixtureA, { reflectMode: 'failure-only' });
|
||||
expect(failOnly.successReflectCalls).toBe(0);
|
||||
});
|
||||
} finally { uninstallStub(); }
|
||||
} finally { fixtureA.cleanup(); }
|
||||
|
||||
const both = { successReflectCalls: 0, oneShotCalls: 0 };
|
||||
const fixtureB = setupFixture(SKILL_PEOPLE_ONLY, SAMPLE_BENCHMARK);
|
||||
try {
|
||||
installStub({
|
||||
stats: both,
|
||||
failureEdit: { op: 'add', anchor: 'People', content: '## Citations\nCite.', reason: 'add' },
|
||||
successEdit: { op: 'add', anchor: 'People', content: '<!-- success note -->', reason: 'note' },
|
||||
});
|
||||
try {
|
||||
await withEnv({ GBRAIN_AUDIT_DIR: fixtureB.skillsDir }, async () => {
|
||||
await runOnce(fixtureB);
|
||||
expect(both.successReflectCalls).toBeGreaterThan(0);
|
||||
});
|
||||
} finally { uninstallStub(); }
|
||||
} finally { fixtureB.cleanup(); }
|
||||
});
|
||||
|
||||
test('disableValidationGate greedy-accepts a no-improvement edit the gate would reject', async () => {
|
||||
// BOTH_SECTIONS already scores 1.0; a benign success edit yields delta 0,
|
||||
// which the D12 gate rejects — unless disableValidationGate greedy-accepts.
|
||||
const benignEdit = { op: 'add' as const, anchor: 'Citations', content: 'Extra note.', reason: 'benign' };
|
||||
|
||||
const fixtureGated = setupFixture(SKILL_BOTH_SECTIONS, SAMPLE_BENCHMARK);
|
||||
try {
|
||||
installStub({ successEdit: benignEdit, failureEdit: null });
|
||||
try {
|
||||
await withEnv({ GBRAIN_AUDIT_DIR: fixtureGated.skillsDir }, async () => {
|
||||
const result = await runOnce(fixtureGated);
|
||||
expect(result.outcome).toBe('no_improvement'); // gate rejected
|
||||
});
|
||||
} finally { uninstallStub(); }
|
||||
} finally { fixtureGated.cleanup(); }
|
||||
|
||||
const fixtureGreedy = setupFixture(SKILL_BOTH_SECTIONS, SAMPLE_BENCHMARK);
|
||||
try {
|
||||
installStub({ successEdit: benignEdit, failureEdit: null });
|
||||
try {
|
||||
await withEnv({ GBRAIN_AUDIT_DIR: fixtureGreedy.skillsDir }, async () => {
|
||||
const result = await runOnce(fixtureGreedy, { disableValidationGate: true });
|
||||
expect(result.outcome).toBe('accepted'); // greedy
|
||||
expect(result.receipt.validation_gate_disabled).toBe(true);
|
||||
});
|
||||
} finally { uninstallStub(); }
|
||||
} finally { fixtureGreedy.cleanup(); }
|
||||
});
|
||||
|
||||
test('D2 no-DB-pollution: subagent_messages count unchanged across a full run', async () => {
|
||||
const fixture = setupFixture(SKILL_PEOPLE_ONLY, CITATIONS_BENCHMARK);
|
||||
const countMessages = async (): Promise<number> => {
|
||||
const r = await engine.executeRaw('SELECT COUNT(*) AS c FROM subagent_messages', []);
|
||||
const rows = Array.isArray(r) ? r : ((r as { rows?: unknown[] }).rows ?? []);
|
||||
return Number((rows[0] as { c?: number | string })?.c ?? 0);
|
||||
};
|
||||
try {
|
||||
const before = await countMessages();
|
||||
installStub({
|
||||
failureEdit: { op: 'add', anchor: 'People', content: '## Citations\nCite the source.', reason: 'add' },
|
||||
successEdit: null,
|
||||
});
|
||||
try {
|
||||
await withEnv({ GBRAIN_AUDIT_DIR: fixture.skillsDir }, async () => {
|
||||
await runOnce(fixture);
|
||||
});
|
||||
} finally { uninstallStub(); }
|
||||
const after = await countMessages();
|
||||
// Rollouts use gateway.toolLoop with no-op persistence (D2) → zero rows written.
|
||||
expect(after).toBe(before);
|
||||
} finally { fixture.cleanup(); }
|
||||
});
|
||||
});
|
||||
|
||||
// ─── T3 (review follow-ups): maxRuntimeMin abort + receipt honesty ───────────
|
||||
|
||||
describe('skillopt T3 — runtime deadline + receipt score honesty', () => {
|
||||
test('maxRuntimeMin deadline aborts cleanly with no commit', async () => {
|
||||
const fixture = setupFixture(SKILL_PEOPLE_ONLY, CITATIONS_BENCHMARK);
|
||||
try {
|
||||
installStub({
|
||||
failureEdit: { op: 'add', anchor: 'People', content: '## Citations\nCite the source.', reason: 'add' },
|
||||
successEdit: null,
|
||||
});
|
||||
try {
|
||||
await withEnv({ GBRAIN_AUDIT_DIR: fixture.skillsDir }, async () => {
|
||||
// maxRuntimeMin:0 → deadline == run start; the first step's deadline
|
||||
// check fires after the baseline eval has already elapsed → abort.
|
||||
const result = await runOnce(fixture, { maxRuntimeMin: 0 });
|
||||
expect(result.outcome).toBe('aborted');
|
||||
// No commit: SKILL.md unchanged, no committed history row.
|
||||
const skill = fs.readFileSync(skillPath(fixture.skillsDir, SKILL), 'utf8');
|
||||
expect(skill).not.toContain('## Citations');
|
||||
expect(loadHistory(fixture.skillsDir, SKILL).filter((r) => r.status === 'committed')).toHaveLength(0);
|
||||
});
|
||||
} finally { uninstallStub(); }
|
||||
} finally { fixture.cleanup(); }
|
||||
});
|
||||
|
||||
test('receipt records the REAL baseline_sel_score + final-test scores (regression: was hardcoded 0)', async () => {
|
||||
// BOTH_SECTIONS already scores ~1.0 on the alternating benchmark, so a real
|
||||
// baseline read is ~1.0 — a hardcoded-0 receipt would fail this immediately.
|
||||
const fixture = setupFixture(SKILL_BOTH_SECTIONS, SAMPLE_BENCHMARK);
|
||||
try {
|
||||
installStub({ successEdit: null, failureEdit: null }); // baseline already perfect; no edits
|
||||
try {
|
||||
await withEnv({ GBRAIN_AUDIT_DIR: fixture.skillsDir }, async () => {
|
||||
const r = (await runOnce(fixture)).receipt;
|
||||
// Real baseline, not the old hardcoded 0.
|
||||
expect(r.baseline_sel_score).toBeGreaterThan(0.9);
|
||||
// Final-test eval populated both test scores (D_test non-empty under 4:1:5).
|
||||
expect(typeof r.test_score).toBe('number');
|
||||
expect(typeof r.baseline_test_score).toBe('number');
|
||||
expect(r.baseline_test_score!).toBeGreaterThan(0.9);
|
||||
});
|
||||
} finally { uninstallStub(); }
|
||||
} finally { fixture.cleanup(); }
|
||||
});
|
||||
});
|
||||
|
||||
describe('skillopt T3 — held-out independence guard', () => {
|
||||
test('held-out sharing task_ids with the benchmark is rejected (gaming defense)', async () => {
|
||||
const fixture = setupFixture(SKILL_PEOPLE_ONLY, CITATIONS_BENCHMARK);
|
||||
// Held-out file reuses benchmark task_ids (cit-001..006) → must be rejected
|
||||
// before any optimization, since an overlapping held-out can't catch overfit.
|
||||
const heldOutPath = writeHeldOut(fixture, CITATIONS_BENCHMARK.slice(0, 6));
|
||||
try {
|
||||
installStub({
|
||||
failureEdit: { op: 'add', anchor: 'People', content: '## Citations\nCite.', reason: 'add' },
|
||||
successEdit: null,
|
||||
});
|
||||
try {
|
||||
await withEnv({ GBRAIN_AUDIT_DIR: fixture.skillsDir }, async () => {
|
||||
await expect(runOnce(fixture, { heldOutPath })).rejects.toThrow(/independent|shares .* task_id/i);
|
||||
});
|
||||
} finally { uninstallStub(); }
|
||||
} finally { fixture.cleanup(); }
|
||||
});
|
||||
});
|
||||
|
||||
@@ -15,7 +15,9 @@ import {
|
||||
capturesDir,
|
||||
loadHeldOut,
|
||||
runHeldOutGate,
|
||||
MIN_HELD_OUT_SIZE,
|
||||
} from '../../src/core/skillopt/held-out.ts';
|
||||
import { assertBundledMutationHeldOut } from '../../src/core/skillopt/bundled-skill-gate.ts';
|
||||
|
||||
let tmp: string;
|
||||
|
||||
@@ -109,3 +111,35 @@ describe('F11 runHeldOutGate vacuous case', () => {
|
||||
expect(result.candidateScore).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('D16 assertBundledMutationHeldOut (core ENFORCE)', () => {
|
||||
test('bundled + mutate + too-small held-out → throws (hard refuse)', () => {
|
||||
expect(() => assertBundledMutationHeldOut({
|
||||
isBundled: true, willMutate: true, heldOutCount: MIN_HELD_OUT_SIZE - 1, skillName: 'brain-ops',
|
||||
})).toThrow(/held-out/i);
|
||||
});
|
||||
|
||||
test('bundled + mutate + empty held-out → throws (vacuous-pass hole closed)', () => {
|
||||
expect(() => assertBundledMutationHeldOut({
|
||||
isBundled: true, willMutate: true, heldOutCount: 0, skillName: 'brain-ops',
|
||||
})).toThrow(/non-empty held-out/i);
|
||||
});
|
||||
|
||||
test('bundled + mutate + sufficient held-out → no throw', () => {
|
||||
expect(() => assertBundledMutationHeldOut({
|
||||
isBundled: true, willMutate: true, heldOutCount: MIN_HELD_OUT_SIZE, skillName: 'brain-ops',
|
||||
})).not.toThrow();
|
||||
});
|
||||
|
||||
test('not bundled → no throw even with zero held-out (user skills are free)', () => {
|
||||
expect(() => assertBundledMutationHeldOut({
|
||||
isBundled: false, willMutate: true, heldOutCount: 0, skillName: 'my-skill',
|
||||
})).not.toThrow();
|
||||
});
|
||||
|
||||
test('bundled but NOT mutating (no-mutate / proposed.md path) → no throw', () => {
|
||||
expect(() => assertBundledMutationHeldOut({
|
||||
isBundled: true, willMutate: false, heldOutCount: 0, skillName: 'brain-ops',
|
||||
})).not.toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -18,7 +18,7 @@
|
||||
*/
|
||||
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import { parseEditsResponse, runReflect } from '../../src/core/skillopt/reflect.ts';
|
||||
import { parseEditsResponse, runReflect, runOneShotRewrite } from '../../src/core/skillopt/reflect.ts';
|
||||
import type { ChatOpts, ChatResult } from '../../src/core/ai/gateway.ts';
|
||||
import type { ScoredRollout, Trajectory } from '../../src/core/skillopt/types.ts';
|
||||
|
||||
@@ -287,3 +287,43 @@ describe('runReflect (D7 two-call contract)', () => {
|
||||
expect(observedUserMsg).toContain('validation_gate_below_baseline');
|
||||
});
|
||||
});
|
||||
|
||||
// ─── runOneShotRewrite (cat31 config C baseline) ────────────────────────────
|
||||
|
||||
function oneShotChatStub(text: string): NonNullable<Parameters<typeof runOneShotRewrite>[0]['chatFn']> {
|
||||
return (async (_opts: ChatOpts): Promise<ChatResult> => ({
|
||||
text,
|
||||
blocks: [{ type: 'text', text }],
|
||||
stopReason: 'end',
|
||||
usage: { input_tokens: 1, output_tokens: 1, cache_read_tokens: 0, cache_creation_tokens: 0 },
|
||||
model: 'm',
|
||||
providerId: 'anthropic',
|
||||
})) as NonNullable<Parameters<typeof runOneShotRewrite>[0]['chatFn']>;
|
||||
}
|
||||
|
||||
const ONE_SHOT_BASE = { skillBodyText: '# Skill', successes: [] as ScoredRollout[], failures: [] as ScoredRollout[], rejected: [], optimizerModel: 'm' };
|
||||
|
||||
describe('runOneShotRewrite', () => {
|
||||
test('unwraps a whole-response code fence', async () => {
|
||||
const body = '# Skill\n\n## People\nList people.';
|
||||
const r = await runOneShotRewrite({ ...ONE_SHOT_BASE, chatFn: oneShotChatStub('```markdown\n' + body + '\n```') });
|
||||
expect(r.newBody).toBe(body);
|
||||
expect(r.error).toBeUndefined();
|
||||
});
|
||||
|
||||
test('preserves an EMBEDDED code fence — does not truncate to the first block', async () => {
|
||||
const body = '# Skill\n\nRun this:\n```bash\nls -la\n```\n\n## People\nList people.';
|
||||
const r = await runOneShotRewrite({ ...ONE_SHOT_BASE, chatFn: oneShotChatStub(body) });
|
||||
expect(r.newBody).toBe(body);
|
||||
expect(r.newBody).toContain('## People'); // regression: old non-anchored regex truncated to the bash block
|
||||
});
|
||||
|
||||
test('chat error returns empty newBody + error (caller treats as no-change)', async () => {
|
||||
const r = await runOneShotRewrite({
|
||||
...ONE_SHOT_BASE,
|
||||
chatFn: (async () => { throw new Error('boom'); }) as NonNullable<Parameters<typeof runOneShotRewrite>[0]['chatFn']>,
|
||||
});
|
||||
expect(r.newBody).toBe('');
|
||||
expect(r.error).toContain('one_shot_rewrite_failed');
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,123 @@
|
||||
/**
|
||||
* SkillOpt rollout tests (the `toolLoopFn` DI seam — previously zero coverage).
|
||||
*
|
||||
* Covers: trajectory shape capture, READ_ONLY_BRAIN_TOOLS zero-write invariant,
|
||||
* default read-only tool registry (no put_page/submit_job), --write-capture
|
||||
* routing to the virtual write registry, and tool-call ordering via the
|
||||
* onToolCallStart callback. Hermetic: the toolLoop transport is stubbed, so no
|
||||
* LLM calls, no DB, no API keys. Handlers are never executed (the stub returns
|
||||
* messages directly), so a `{}`-shaped engine is sufficient.
|
||||
*/
|
||||
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import { runRollout, READ_ONLY_BRAIN_TOOLS } from '../../src/core/skillopt/rollout.ts';
|
||||
import type { BenchmarkTask } from '../../src/core/skillopt/types.ts';
|
||||
|
||||
const TASK: BenchmarkTask = {
|
||||
task_id: 't1',
|
||||
task: 'do the thing',
|
||||
judge: { kind: 'rule', checks: [] },
|
||||
};
|
||||
|
||||
/** Build a stubbed toolLoop that records the tool defs it was handed. */
|
||||
function makeStubLoop(opts: { capturedTools?: Array<{ name: string }>; finalText?: string }) {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
return async (loopOpts: any) => {
|
||||
if (opts.capturedTools) opts.capturedTools.push(...loopOpts.tools);
|
||||
return {
|
||||
messages: [
|
||||
{ role: 'user', content: 'do the thing' },
|
||||
{ role: 'assistant', content: opts.finalText ?? 'done' },
|
||||
],
|
||||
totalUsage: { input_tokens: 10, output_tokens: 5, cache_read_tokens: 0, cache_creation_tokens: 0 },
|
||||
totalTurns: 2,
|
||||
stopReason: 'end',
|
||||
};
|
||||
};
|
||||
}
|
||||
|
||||
describe('rollout — READ_ONLY_BRAIN_TOOLS zero-write invariant', () => {
|
||||
test('excludes put_page / submit_job / file_upload, includes a read op', () => {
|
||||
expect(READ_ONLY_BRAIN_TOOLS.has('put_page')).toBe(false);
|
||||
expect(READ_ONLY_BRAIN_TOOLS.has('submit_job')).toBe(false);
|
||||
expect(READ_ONLY_BRAIN_TOOLS.has('file_upload')).toBe(false);
|
||||
expect(READ_ONLY_BRAIN_TOOLS.has('search')).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('rollout — trajectory capture', () => {
|
||||
test('returns a Trajectory with the expected shape + threaded usage/stop_reason', async () => {
|
||||
const traj = await runRollout({
|
||||
engine: {} as never,
|
||||
skillText: 'skill body',
|
||||
task: TASK,
|
||||
targetModel: 'anthropic:claude-sonnet-4-6',
|
||||
toolLoopFn: makeStubLoop({ finalText: 'the answer' }) as never,
|
||||
});
|
||||
expect(traj.task_id).toBe('t1');
|
||||
expect(traj.task).toBe('do the thing');
|
||||
expect(traj.final_text).toBe('the answer');
|
||||
expect(traj.stop_reason).toBe('end');
|
||||
expect(traj.turns).toBe(2);
|
||||
expect(traj.usage.input_tokens).toBe(10);
|
||||
expect(traj.usage.output_tokens).toBe(5);
|
||||
expect(typeof traj.duration_ms).toBe('number');
|
||||
expect(Array.isArray(traj.tool_calls)).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('rollout — tool registry routing', () => {
|
||||
test('default rollout passes ONLY read-only tools (no write defs)', async () => {
|
||||
const capturedTools: Array<{ name: string }> = [];
|
||||
await runRollout({
|
||||
engine: {} as never,
|
||||
skillText: 's',
|
||||
task: TASK,
|
||||
targetModel: 'm',
|
||||
toolLoopFn: makeStubLoop({ capturedTools }) as never,
|
||||
});
|
||||
const names = capturedTools.map((d) => d.name);
|
||||
expect(names).toContain('brain_search');
|
||||
expect(names).not.toContain('brain_put_page');
|
||||
expect(names).not.toContain('brain_submit_job');
|
||||
expect(names).not.toContain('brain_file_upload');
|
||||
});
|
||||
|
||||
test('--write-capture routes to the virtual write registry (put_page present, captured not real)', async () => {
|
||||
const capturedTools: Array<{ name: string }> = [];
|
||||
await runRollout({
|
||||
engine: {} as never,
|
||||
skillText: 's',
|
||||
task: TASK,
|
||||
targetModel: 'm',
|
||||
writeCapture: true,
|
||||
toolLoopFn: makeStubLoop({ capturedTools }) as never,
|
||||
});
|
||||
const names = capturedTools.map((d) => d.name);
|
||||
expect(names).toContain('brain_put_page');
|
||||
expect(names).toContain('brain_submit_job');
|
||||
expect(names).toContain('brain_file_upload');
|
||||
});
|
||||
});
|
||||
|
||||
describe('rollout — tool-call ordering', () => {
|
||||
test('onToolCallStart records calls in order with the brain_ prefix stripped', async () => {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const loop = async (loopOpts: any) => {
|
||||
await loopOpts.onToolCallStart?.(0, 0, 0, 'brain_search', { q: 'x' }, 'pc1');
|
||||
await loopOpts.onToolCallStart?.(0, 1, 1, 'brain_get_page', { slug: 'y' }, 'pc2');
|
||||
return {
|
||||
messages: [{ role: 'assistant', content: 'ok' }],
|
||||
totalUsage: { input_tokens: 1, output_tokens: 1, cache_read_tokens: 0, cache_creation_tokens: 0 },
|
||||
totalTurns: 1,
|
||||
stopReason: 'end',
|
||||
};
|
||||
};
|
||||
const traj = await runRollout({
|
||||
engine: {} as never, skillText: 's', task: TASK, targetModel: 'm', toolLoopFn: loop as never,
|
||||
});
|
||||
expect(traj.tool_calls).toHaveLength(2);
|
||||
expect(traj.tool_calls[0]!.name).toBe('search');
|
||||
expect(traj.tool_calls[1]!.name).toBe('get_page');
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user