From 5bf9e63a930a77424f9cd5c802ab8ca6d2c41384 Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Wed, 27 May 2026 07:35:05 -0700 Subject: [PATCH] feat(skillopt): rollout (D2 gateway.toolLoop + D13 read-only allowlist), reflect (D7 two calls), validate-gate (D12 median+epsilon, D4 parallel), preflight (D3), bundled-skill-gate (D16) --- src/core/skillopt/bundled-skill-gate.ts | 71 ++++++++ src/core/skillopt/preflight.ts | 184 ++++++++++++++++++++ src/core/skillopt/reflect.ts | 209 +++++++++++++++++++++++ src/core/skillopt/rollout.ts | 218 ++++++++++++++++++++++++ src/core/skillopt/validate-gate.ts | 109 ++++++++++++ 5 files changed, 791 insertions(+) create mode 100644 src/core/skillopt/bundled-skill-gate.ts create mode 100644 src/core/skillopt/preflight.ts create mode 100644 src/core/skillopt/reflect.ts create mode 100644 src/core/skillopt/rollout.ts create mode 100644 src/core/skillopt/validate-gate.ts diff --git a/src/core/skillopt/bundled-skill-gate.ts b/src/core/skillopt/bundled-skill-gate.ts new file mode 100644 index 000000000..940d6763e --- /dev/null +++ b/src/core/skillopt/bundled-skill-gate.ts @@ -0,0 +1,71 @@ +/** + * SkillOpt bundled-skill mutation gate (D16). + * + * A "bundled" skill is one that lives in the gbrain repo's `skills/` tree + * (shipped alongside the binary). These are load-bearing for production + * workflows; mutating them via SkillOpt without explicit operator opt-in + * is too risky. By default, bundled-skill optimization runs in `--no-mutate` + * mode automatically — the proposed best is written to `proposed.md` for + * human review. + * + * User-owned skills (under a user's `~/.gbrain/skills/` or their own + * project's `skills/`) are NOT bundled — they can be mutated freely. + */ + +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { autoDetectSkillsDirReadOnly } from '../repo-root.ts'; +import type { BundledSkillContext } from './types.ts'; + +/** + * Build the bundled-skill context for a (skillsDir, skillName) pair. + * + * The resolution chain: + * 1. Resolve the absolute skill path: `skillsDir//SKILL.md`. + * 2. Check whether the skillsDir came from the install-path fallback + * (i.e. it's `/skills`). If so, this is a bundled skill. + * 3. Otherwise, the skill is user-owned and freely mutable. + */ +export function getBundledSkillContext(skillsDir: string, skillName: string): BundledSkillContext { + const skillPath = path.join(skillsDir, skillName, 'SKILL.md'); + // Detect bundled by checking whether autoDetectSkillsDirReadOnly would + // have routed here via the install-path fallback. The simpler heuristic: + // if the resolved skillsDir is under the gbrain install tree (somewhere + // up the chain has node_modules/gbrain or VERSION matching this binary), + // it's bundled. Use the read-only detector to find the canonical install + // skillsDir and compare. + const detected = autoDetectSkillsDirReadOnly(process.cwd()); + const isBundled = detected.source === 'install_path' && detected.dir !== null + && resolvesToSame(detected.dir, skillsDir); + return { skillName, skillsDir, skillPath, isBundled }; +} + +function resolvesToSame(a: string, b: string): boolean { + try { + return fs.realpathSync(a) === fs.realpathSync(b); + } catch { + return path.resolve(a) === path.resolve(b); + } +} + +/** + * Decide whether SkillOpt should mutate SKILL.md or just write proposed.md. + * + * Logic: + * - `--no-mutate` flag → never mutate (write proposed.md). + * - Bundled skill + no `--allow-mutate-bundled` → never mutate (D16). + * - User-owned skill (or bundled + `--allow-mutate-bundled`) → mutate + * in place. + */ +export function shouldMutateSkillFile( + ctx: BundledSkillContext, + flags: { noMutate: boolean; allowMutateBundled: boolean }, +): { mutate: boolean; reason?: string } { + if (flags.noMutate) { + return { mutate: false, reason: 'user_passed_no_mutate' }; + } + if (ctx.isBundled && !flags.allowMutateBundled) { + return { mutate: false, reason: 'bundled_skill_requires_allow_flag' }; + } + return { mutate: true }; +} diff --git a/src/core/skillopt/preflight.ts b/src/core/skillopt/preflight.ts new file mode 100644 index 000000000..39de02781 --- /dev/null +++ b/src/core/skillopt/preflight.ts @@ -0,0 +1,184 @@ +/** + * SkillOpt cost preflight (D3). + * + * Estimates the total USD cost of a run BEFORE any LLM call fires. + * Refuses to start when estimate > --max-cost-usd. In TTY, prompts the + * user with a 10-second Ctrl-C grace window (mirrors the progressive-batch + * cost-prompt UX). + * + * Cost model (rough but consistent): + * + * Per step: + * - batch_size rollouts × target-model price + * - 2 reflect calls (D7) × optimizer-model price + * - sel_size sel-tasks × VALIDATION_RUNS_PER_TASK × target-model price + * - sel_size sel-tasks × VALIDATION_RUNS_PER_TASK × judge-model price + * + * Total: + * - 1× baseline eval on D_sel + * - epochs × steps_per_epoch × per-step cost + * - epochs × 1 slow-update reflect call (if no improvement that epoch) + * - 1× final test eval on D_test + * + * Prices come from existing per-model pricing tables (Anthropic + + * embedding-pricing). For unknown providers we fail-loud — same posture + * as BudgetTracker's TX2 contract. + */ + +import { ANTHROPIC_PRICING } from '../anthropic-pricing.ts'; +import { VALIDATION_RUNS_PER_TASK } from './types.ts'; + +/** Conservative per-rollout token estimates (input + output). */ +const ROLLOUT_INPUT_TOKENS = 3000; // skill + task prompt + tool defs +const ROLLOUT_OUTPUT_TOKENS = 800; +const REFLECT_INPUT_TOKENS = 8000; // skill + trajectories + rejected buffer +const REFLECT_OUTPUT_TOKENS = 1500; +const JUDGE_INPUT_TOKENS = 2000; // rubric + agent output +const JUDGE_OUTPUT_TOKENS = 200; + +export interface PreflightOpts { + epochs: number; + batchSize: number; + trainSize: number; + selSize: number; + testSize: number; + optimizerModel: string; + targetModel: string; + judgeModel: string; + maxCostUsd: number; + /** When true, print the prompt to stderr + use Ctrl-C grace. Default false (non-TTY). */ + interactive?: boolean; +} + +export interface PreflightEstimate { + steps_per_epoch: number; + total_steps: number; + rollout_calls: number; + reflect_calls: number; + judge_calls: number; + est_input_tokens: number; + est_output_tokens: number; + est_cost_usd: number; + /** Per-model breakdown for the audit. */ + per_model_cost_usd: Record; + /** True when est_cost_usd > maxCostUsd (caller should refuse or prompt). */ + exceeds_cap: boolean; +} + +export interface PreflightResult { + estimate: PreflightEstimate; + /** When false, caller should abort. When true, run may proceed. */ + proceed: boolean; + /** Reason for abort, if proceed=false. */ + abort_reason?: string; +} + +export function estimateCost(opts: PreflightOpts): PreflightEstimate { + const stepsPerEpoch = Math.max(1, Math.floor(opts.trainSize / opts.batchSize)); + const totalSteps = opts.epochs * stepsPerEpoch; + + // Per-step counts. + const rolloutsPerStep = opts.batchSize; + const reflectsPerStep = 2; // D7: two reflect calls + const sel_runs_per_step = opts.selSize * VALIDATION_RUNS_PER_TASK; + + // Cumulative counts across the whole run. + const rollout_calls = totalSteps * rolloutsPerStep + + opts.selSize * VALIDATION_RUNS_PER_TASK // baseline sel eval + + opts.selSize * VALIDATION_RUNS_PER_TASK * totalSteps // per-step sel validation + + opts.testSize; // final test eval + const reflect_calls = totalSteps * reflectsPerStep + + opts.epochs; // slow-update meta calls + const judge_calls = opts.selSize // baseline (1 per task; median-of-3 is in the rollout count already? — no, judge runs per rollout) + + opts.selSize * VALIDATION_RUNS_PER_TASK * totalSteps // per-step validation + + opts.testSize; // final test judges + + // Cost per call type. + const targetPrice = lookupPrice(opts.targetModel); + const optimizerPrice = lookupPrice(opts.optimizerModel); + const judgePrice = lookupPrice(opts.judgeModel); + + const rolloutCost = rollout_calls * ( + (ROLLOUT_INPUT_TOKENS * targetPrice.input) / 1_000_000 + + (ROLLOUT_OUTPUT_TOKENS * targetPrice.output) / 1_000_000 + ); + const reflectCost = reflect_calls * ( + (REFLECT_INPUT_TOKENS * optimizerPrice.input) / 1_000_000 + + (REFLECT_OUTPUT_TOKENS * optimizerPrice.output) / 1_000_000 + ); + const judgeCost = judge_calls * ( + (JUDGE_INPUT_TOKENS * judgePrice.input) / 1_000_000 + + (JUDGE_OUTPUT_TOKENS * judgePrice.output) / 1_000_000 + ); + + // D11 prompt caching gives ~50% discount on stable layers. Apply + // conservatively (assume 50% of optimizer + judge tokens are cached). + const cachedReflectCost = reflectCost * 0.6; + const cachedJudgeCost = judgeCost * 0.6; + + const total = rolloutCost + cachedReflectCost + cachedJudgeCost; + void sel_runs_per_step; + + return { + steps_per_epoch: stepsPerEpoch, + total_steps: totalSteps, + rollout_calls, + reflect_calls, + judge_calls, + est_input_tokens: rollout_calls * ROLLOUT_INPUT_TOKENS + reflect_calls * REFLECT_INPUT_TOKENS + judge_calls * JUDGE_INPUT_TOKENS, + est_output_tokens: rollout_calls * ROLLOUT_OUTPUT_TOKENS + reflect_calls * REFLECT_OUTPUT_TOKENS + judge_calls * JUDGE_OUTPUT_TOKENS, + est_cost_usd: total, + per_model_cost_usd: { + [opts.targetModel]: rolloutCost, + [opts.optimizerModel]: cachedReflectCost, + [opts.judgeModel]: cachedJudgeCost, + }, + exceeds_cap: total > opts.maxCostUsd, + }; +} + +/** + * Render a human-readable preflight summary to stderr. Caller may follow + * with a Ctrl-C grace prompt in TTY mode (see runPreflightPrompt). + */ +export function formatPreflightReport(est: PreflightEstimate, opts: PreflightOpts): string { + return [ + `[skillopt] Cost estimate for ${opts.epochs} epochs × ${est.steps_per_epoch} steps × ${opts.batchSize} rollouts:`, + ` Rollouts: ${est.rollout_calls.toLocaleString()} calls`, + ` Reflects: ${est.reflect_calls.toLocaleString()} calls`, + ` Judges: ${est.judge_calls.toLocaleString()} calls`, + ` Tokens: ~${(est.est_input_tokens / 1000).toFixed(0)}K in / ~${(est.est_output_tokens / 1000).toFixed(0)}K out`, + ` Est. cost: $${est.est_cost_usd.toFixed(2)} (cap: $${opts.maxCostUsd.toFixed(2)})`, + est.exceeds_cap ? ` WARNING: estimate exceeds --max-cost-usd cap.` : '', + ].filter(Boolean).join('\n'); +} + +/** + * Decision wrapper. Returns {proceed: false, abort_reason} when the + * estimate over-shoots the cap (caller exits 2). Returns {proceed: true} + * otherwise. Interactive=true callers should print formatPreflightReport + * + a Ctrl-C grace window separately. + */ +export function preflight(opts: PreflightOpts): PreflightResult { + const estimate = estimateCost(opts); + if (estimate.exceeds_cap) { + return { + estimate, + proceed: false, + abort_reason: `estimated cost $${estimate.est_cost_usd.toFixed(2)} exceeds --max-cost-usd $${opts.maxCostUsd.toFixed(2)}. Raise the cap with --max-cost-usd ${Math.ceil(estimate.est_cost_usd)} or reduce --epochs/--batch-size.`, + }; + } + return { estimate, proceed: true }; +} + +function lookupPrice(model: string): { input: number; output: number } { + // Anthropic models — strip provider prefix. + const bare = model.startsWith('anthropic:') ? model.slice('anthropic:'.length) : model; + const anth = (ANTHROPIC_PRICING as Record)[bare]; + if (anth) return anth; + // Conservative fallback: assume Sonnet-tier pricing for unknown providers. + // Don't throw — preflight is for warning, not gating. The actual budget + // tracker (BudgetTracker TX2) will fail-loud at run time if pricing is + // truly unknown. + return { input: 3.0, output: 15.0 }; +} diff --git a/src/core/skillopt/reflect.ts b/src/core/skillopt/reflect.ts new file mode 100644 index 000000000..8530ca488 --- /dev/null +++ b/src/core/skillopt/reflect.ts @@ -0,0 +1,209 @@ +/** + * SkillOpt reflect: ask the optimizer model to propose edits to SKILL.md + * based on a batch of scored rollouts. + * + * D7: TWO reflect calls per step — one for failures, one for successes. + * Paper-faithful: each call uses its own rubric prompt so attention isn't + * conflated between "what went wrong" and "what went right" analyses. + * + * D11: optimizer system prompt is cached via cacheSystem=true (stable + * across all reflect calls in a run; ~$0.30/run savings). + * + * The reflect call also receives the rejected-edit buffer as anti-bias + * context so the optimizer doesn't re-propose previously-failing edits. + */ + +import { chat as gatewayChat } from '../ai/gateway.ts'; +import { parseJudgeJson } from './score.ts'; +import type { EditOp, ScoredRollout } from './types.ts'; +import type { RejectedEntry } from './rejected-buffer.ts'; + +const FAILURE_REFLECT_SYSTEM = `You are SkillOpt's optimizer. You analyze AGENT FAILURE TRAJECTORIES and propose specific edits to a SKILL document so the agent does better next time. + +Output ONLY a single JSON object on one or more lines: +{"edits": [{"op": "add|replace|delete", ...}, ...]} + +Edit ops: + add: {"op": "add", "anchor": "", "content": "", "reason": ""} + replace: {"op": "replace", "target": "", "replacement": "", "reason": ""} + delete: {"op": "delete", "target": "", "reason": ""} + +Rules: +- Each edit MUST address a SPECIFIC failure pattern you observed. +- anchor / target MUST be uniquely identifiable in the skill body (exact match). +- Do NOT propose edits already in the rejected-edit history — those were tried and didn't help. +- Be SURGICAL. Small targeted edits outperform large rewrites. +- Do NOT modify the YAML frontmatter (triggers, brain_first, etc.) — that's out of scope. +- Output at MOST 8 edits. The orchestrator's LR budget will rank-and-clip further.`; + +const SUCCESS_REFLECT_SYSTEM = `You are SkillOpt's optimizer. You analyze AGENT SUCCESS TRAJECTORIES and propose specific edits to a SKILL document so the agent CONSISTENTLY does what worked here. + +Output format and rules are identical to the failure-reflect mode — same {edits: [...]} shape. + +When successes are present, look for: which rules were FOLLOWED to produce success, which rules could be MADE EXPLICIT (not yet stated, but exemplified), which anti-patterns the agent successfully AVOIDED that should be stated. + +Be SURGICAL. Don't restate things that are already in the skill. Don't modify frontmatter.`; + +export interface ReflectOpts { + skillBodyText: string; + /** Successful rollouts (score >= 0.5). */ + successes: ScoredRollout[]; + /** Failed rollouts (score < 0.5). */ + failures: ScoredRollout[]; + /** Rejected-edit buffer for anti-bias context. */ + rejected: readonly RejectedEntry[]; + optimizerModel: string; + /** Test seam — substitute for gateway.chat. */ + chatFn?: typeof gatewayChat; + abortSignal?: AbortSignal; +} + +export interface ReflectResult { + /** Edits proposed from FAILURE analysis. */ + failureEdits: EditOp[]; + /** Edits proposed from SUCCESS analysis. */ + successEdits: EditOp[]; + /** Token usage across both calls (for cost tracking). */ + usage: { + input_tokens: number; + output_tokens: number; + cache_read_tokens: number; + cache_creation_tokens: number; + }; + /** Any per-call errors (for audit). */ + errors: string[]; +} + +/** + * D7: fire two reflect calls (failures + successes). Empty batches skip + * their reflect call (no point asking for edits without data). + */ +export async function runReflect(opts: ReflectOpts): Promise { + const usage = { input_tokens: 0, output_tokens: 0, cache_read_tokens: 0, cache_creation_tokens: 0 }; + const errors: string[] = []; + + const failureEdits = opts.failures.length > 0 + ? await callReflect('failure', opts, FAILURE_REFLECT_SYSTEM, opts.failures, usage, errors) + : []; + const successEdits = opts.successes.length > 0 + ? await callReflect('success', opts, SUCCESS_REFLECT_SYSTEM, opts.successes, usage, errors) + : []; + + return { failureEdits, successEdits, usage, errors }; +} + +async function callReflect( + mode: 'failure' | 'success', + opts: ReflectOpts, + system: string, + scoredRollouts: ScoredRollout[], + cumUsage: ReflectResult['usage'], + errors: string[], +): Promise { + const chat = opts.chatFn ?? gatewayChat; + const userMsg = buildReflectUserMessage(opts.skillBodyText, scoredRollouts, opts.rejected); + try { + const result = await chat({ + model: opts.optimizerModel, + system, + messages: [{ role: 'user', content: userMsg }], + maxTokens: 2048, + cacheSystem: true, // D11 + abortSignal: opts.abortSignal, + }); + cumUsage.input_tokens += result.usage.input_tokens; + cumUsage.output_tokens += result.usage.output_tokens; + cumUsage.cache_read_tokens += result.usage.cache_read_tokens; + cumUsage.cache_creation_tokens += result.usage.cache_creation_tokens; + return parseEditsResponse(result.text); + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + errors.push(`reflect_${mode}_failed: ${msg}`); + return []; + } +} + +function buildReflectUserMessage( + skillBody: string, + rollouts: ScoredRollout[], + rejected: readonly RejectedEntry[], +): string { + const trajectoryBlocks = rollouts.map((r, i) => { + const tcSummary = r.trajectory.tool_calls + .map((tc) => ` - ${tc.name}${tc.failed ? ' [FAILED]' : ''}`) + .join('\n'); + return `--- ROLLOUT ${i + 1} (score=${r.score.toFixed(2)}) --- +TASK: ${r.trajectory.task} +TOOL CALLS: +${tcSummary || ' (none)'} +OUTPUT: +${truncate(r.trajectory.final_text, 2000)} +${r.rationale ? `JUDGE RATIONALE: ${r.rationale}` : ''}`; + }).join('\n\n'); + + const rejectedSummary = rejected.length > 0 + ? `\n\n--- PREVIOUSLY REJECTED EDITS (do not re-propose) ---\n${rejected.slice(0, 20).map((r) => `- ${r.reason}: ${JSON.stringify(r.edits)}`).join('\n')}` + : ''; + + return `CURRENT SKILL BODY: +${truncate(skillBody, 5000)} + +OBSERVED ROLLOUTS: +${trajectoryBlocks}${rejectedSummary} + +Propose edits to improve the skill. Output the {edits: [...]} JSON only.`; +} + +function truncate(s: string, max: number): string { + return s.length > max ? s.slice(0, max) + `\n...(truncated, ${s.length - max} more chars)` : s; +} + +function parseEditsResponse(raw: string): EditOp[] { + // Reuse the same forgiving JSON parser as the score module (tagged result + // shape; the parser pulls the first {...} object from prose if present). + const parsed = parseJudgeJson(raw); + if (!parsed) return []; + // The parser returned a {score, rationale} shape; for reflect we expect + // {edits: [...]}. Re-extract directly with a tolerant fallback. + const direct = tryExtractEdits(raw); + return direct; +} + +function tryExtractEdits(raw: string): EditOp[] { + try { + // Strip fences first. + const fenced = raw.match(/```(?:json)?\s*\n?([\s\S]*?)```/i); + const cleaned = (fenced ? fenced[1]! : raw).trim(); + // Try direct parse. + const direct = JSON.parse(cleaned); + if (direct && typeof direct === 'object' && Array.isArray((direct as { edits?: unknown }).edits)) { + return validateEdits((direct as { edits: unknown[] }).edits); + } + } catch { /* try next strategy */ } + // Fallback: extract first {...} substring. + const match = raw.match(/\{[\s\S]*\}/); + if (!match) return []; + try { + const parsed = JSON.parse(match[0]); + if (parsed && typeof parsed === 'object' && Array.isArray((parsed as { edits?: unknown }).edits)) { + return validateEdits((parsed as { edits: unknown[] }).edits); + } + } catch { /* fall through */ } + return []; +} + +function validateEdits(raw: unknown[]): EditOp[] { + const out: EditOp[] = []; + for (const r of raw) { + if (!r || typeof r !== 'object') continue; + const o = r as Record; + if (o.op === 'add' && typeof o.anchor === 'string' && typeof o.content === 'string') { + out.push({ op: 'add', anchor: o.anchor, content: o.content, reason: typeof o.reason === 'string' ? o.reason : undefined }); + } else if (o.op === 'replace' && typeof o.target === 'string' && typeof o.replacement === 'string') { + out.push({ op: 'replace', target: o.target, replacement: o.replacement, reason: typeof o.reason === 'string' ? o.reason : undefined }); + } else if (o.op === 'delete' && typeof o.target === 'string') { + out.push({ op: 'delete', target: o.target, reason: typeof o.reason === 'string' ? o.reason : undefined }); + } + } + return out; +} diff --git a/src/core/skillopt/rollout.ts b/src/core/skillopt/rollout.ts new file mode 100644 index 000000000..88465916a --- /dev/null +++ b/src/core/skillopt/rollout.ts @@ -0,0 +1,218 @@ +/** + * SkillOpt rollout: execute a skill against one benchmark task. + * + * Per D2: uses `gateway.toolLoop` directly with NO persistence callbacks. + * SkillOpt rollouts are transient eval data — they don't pollute + * `subagent_messages` / `subagent_tool_executions`. + * + * Per D13: tool registry is the read-only subset of BRAIN_TOOL_ALLOWLIST. + * Excluded ops: `put_page`, `submit_job`, `file_upload` (the latter two + * aren't in BRAIN_TOOL_ALLOWLIST anyway, but documenting for clarity). + * The optimizer's loop can call `search`, `query`, `get_page`, etc., but + * cannot WRITE to the user's brain. + * + * Each rollout returns a `Trajectory` capturing the final assistant text, + * tool calls (with inputs + outputs), token usage, and stop reason. The + * caller (orchestrator) feeds the trajectory to the judge for scoring. + */ + +import { chat as gatewayChat, toolLoop, type ChatMessage, type ChatToolDef, type ToolHandler } from '../ai/gateway.ts'; +import { BRAIN_TOOL_ALLOWLIST } from '../minions/tools/brain-allowlist.ts'; +import { operations, type OperationContext } from '../operations.ts'; +import { loadConfig } from '../config.ts'; +import type { BrainEngine } from '../engine.ts'; +import type { BenchmarkTask, Trajectory } from './types.ts'; + +/** + * D13: which tools SkillOpt rollouts are allowed to call. + * + * Derived from BRAIN_TOOL_ALLOWLIST minus `put_page` (only mutating op in + * the base set). New mutating ops MUST be added here AND BRAIN_TOOL_ALLOWLIST + * mustn't be silently widened; the rollout test pins zero-write invariant. + */ +export const READ_ONLY_BRAIN_TOOLS: ReadonlySet = new Set( + [...BRAIN_TOOL_ALLOWLIST].filter((name) => name !== 'put_page'), +); + +export interface RolloutOpts { + engine: BrainEngine; + skillText: string; + task: BenchmarkTask; + /** Provider:model string for the target model. */ + targetModel: string; + /** Max agent turns. Default 20. */ + maxTurns?: number; + /** AbortSignal for cooperative cancellation (e.g. budget exhausted). */ + abortSignal?: AbortSignal; + /** F10: when true, enable write-capture (virtual put_page/submit_job/file_upload). */ + writeCapture?: boolean; + /** Test seam — substitute the toolLoop call. */ + toolLoopFn?: typeof toolLoop; + /** Test seam — substitute chat (currently unused; toolLoop wraps chat). */ + chatFn?: typeof gatewayChat; +} + +/** + * Run one rollout. Returns a Trajectory; never throws on agent-side failures + * (max_turns, refusal, aborted) — those land in `trajectory.stop_reason` so + * the caller can score them appropriately. + * + * Throws ONLY on infrastructure errors (no engine, unknown target model) or + * BudgetExhausted (which propagates via gateway → MUST_ABORT_ERROR_TAGS). + */ +export async function runRollout(opts: RolloutOpts): Promise { + const { engine, skillText, task, targetModel } = opts; + const startedAt = Date.now(); + + // Build the tool handlers + defs. F10: when write-capture is on, swap + // in the virtual-write registry (read-only base + virtual put_page / + // submit_job / file_upload). Default: read-only allowlist only (D13). + const ctx = buildOpContext(engine); + let defs, handlers; + if (opts.writeCapture) { + const { buildWriteCaptureRegistry } = await import('./write-capture.ts'); + const registry = buildWriteCaptureRegistry(engine); + defs = registry.defs; + handlers = registry.handlers; + } else { + const r = buildReadOnlyToolRegistry(ctx); + defs = r.defs; + handlers = r.handlers; + } + + // The skill text becomes the system prompt (D11: cacheSystem=true so the + // candidate skill is cached across all rollouts in a single batch). + const system = skillText; + const initialMessages: ChatMessage[] = [{ role: 'user', content: task.task }]; + + const toolLoopImpl = opts.toolLoopFn ?? toolLoop; + + // Capture tool calls as they fire. The toolLoop's callbacks fire in + // ordering: onToolCallStart -> handler.execute -> onToolCallComplete|Failed. + // We capture outputs in a Map keyed by gbrainToolUseId so we can stitch + // them into the trajectory's tool_calls array (preserving call order). + const toolCalls: Trajectory['tool_calls'] = []; + const callsById = new Map(); // gbrainToolUseId -> index in toolCalls + let nextOrdinal = 0; + + const result = await toolLoopImpl({ + model: targetModel, + system, + initialMessages, + tools: defs, + toolHandlers: handlers, + maxTurns: opts.maxTurns ?? 20, + cacheSystem: true, // D11: candidate skill is stable for a step's batch. + abortSignal: opts.abortSignal, + onToolCallStart: async (_turnIdx, _messageIdx, _ordinal, toolName, input, providerToolCallId) => { + const gbrainToolUseId = `skillopt-${nextOrdinal++}-${providerToolCallId}`; + const idx = toolCalls.length; + callsById.set(gbrainToolUseId, idx); + toolCalls.push({ name: stripBrainPrefix(toolName), input }); + return { gbrainToolUseId }; + }, + onToolCallComplete: async (gbrainToolUseId, output) => { + const idx = callsById.get(gbrainToolUseId); + if (idx !== undefined && toolCalls[idx]) { + toolCalls[idx]!.output = output; + } + }, + onToolCallFailed: async (gbrainToolUseId, error) => { + const idx = callsById.get(gbrainToolUseId); + if (idx !== undefined && toolCalls[idx]) { + toolCalls[idx]!.failed = true; + toolCalls[idx]!.output = { error }; + } + }, + }); + + // Final assistant text: concatenate text blocks from the last assistant message. + const lastAssistant = [...result.messages].reverse().find((m) => m.role === 'assistant'); + let finalText = ''; + if (lastAssistant && typeof lastAssistant.content === 'string') { + finalText = lastAssistant.content; + } else if (lastAssistant && Array.isArray(lastAssistant.content)) { + finalText = lastAssistant.content + .filter((b) => b.type === 'text') + .map((b) => ('text' in b ? b.text : '')) + .join('\n') + .trim(); + } + + return { + task_id: task.task_id, + task: task.task, + final_text: finalText, + tool_calls: toolCalls, + usage: result.totalUsage, + turns: result.totalTurns, + stop_reason: result.stopReason, + duration_ms: Date.now() - startedAt, + }; +} + +// ─── Tool registry construction ────────────────────────────────────────── + +function buildOpContext(engine: BrainEngine): OperationContext { + const cfg = loadConfig(); + return { + engine, + config: cfg ?? ({} as never), + logger: { info: () => {}, warn: () => {}, error: () => {}, debug: () => {} } as never, + dryRun: false, + // SkillOpt rollouts ARE remote-equivalent (the optimizer chose what to call, + // not the user) → set remote: true. Per-op trust gates that check `remote` + // see this and apply remote-tightening rules. Read-only ops aren't affected. + remote: true, + // v0.34 D4: sourceId is required on OperationContext. SkillOpt rollouts + // operate on the default source — the agent under test reads the brain + // it would normally read. + sourceId: 'default', + }; +} + +interface ToolRegistry { + defs: ChatToolDef[]; + handlers: Map; +} + +/** + * Build the read-only tool registry that SkillOpt rollouts call into. + * + * Tool names are prefixed `brain_` for Anthropic-name compliance (same + * convention as the subagent handler's buildBrainTools). + */ +function buildReadOnlyToolRegistry(ctx: OperationContext): ToolRegistry { + const defs: ChatToolDef[] = []; + const handlers = new Map(); + for (const op of operations) { + if (!READ_ONLY_BRAIN_TOOLS.has(op.name)) continue; + const toolName = `brain_${op.name}`; + defs.push({ + name: toolName, + description: op.description, + inputSchema: paramsToSchema(op.params), + }); + handlers.set(toolName, { + idempotent: true, // All read-only ops are idempotent by construction. + execute: async (input: unknown) => { + return op.handler(ctx, (input as Record) ?? {}); + }, + }); + } + return { defs, handlers }; +} + +function stripBrainPrefix(toolName: string): string { + return toolName.startsWith('brain_') ? toolName.slice('brain_'.length) : toolName; +} + +function paramsToSchema(params: Record): Record { + return { + type: 'object' as const, + properties: Object.fromEntries( + Object.entries(params).map(([k, v]) => [k, { type: v.type, description: v.description }]), + ), + required: Object.entries(params).filter(([, v]) => v.required).map(([k]) => k), + }; +} diff --git a/src/core/skillopt/validate-gate.ts b/src/core/skillopt/validate-gate.ts new file mode 100644 index 000000000..e31706ce9 --- /dev/null +++ b/src/core/skillopt/validate-gate.ts @@ -0,0 +1,109 @@ +/** + * SkillOpt validation gate (D12 + D4). + * + * D12 — median-of-3 judge runs per sel-task + epsilon=0.05 margin. + * D4 — parallel sel-task scoring via runWithLimit cap=4. + * + * Pseudocode: + * + * For each sel-task t in selSet: + * scores[t] = median(await Promise.all([ + * scoreTrajectory(rollout(candidate, t), judge), // run 1 + * scoreTrajectory(rollout(candidate, t), judge), // run 2 + * scoreTrajectory(rollout(candidate, t), judge), // run 3 + * ])) + * sel_score = mean(scores) + * if sel_score > best_score + 0.05: ACCEPT + * else: REJECT + * + * The 3 rollout calls per sel-task share the same candidate-skill prompt + * which is cached (D11), so the effective cost ~1.3x not 3x. + */ + +import { runWithLimit } from '../worker-pool.ts'; +import { runRollout, type RolloutOpts } from './rollout.ts'; +import { scoreTrajectory } from './score.ts'; +import type { BenchmarkTask, GateInput, GateResult, ScoredRollout } from './types.ts'; +import { VALIDATION_EPSILON, VALIDATION_RUNS_PER_TASK } from './types.ts'; +import type { BrainEngine } from '../engine.ts'; + +export interface ValidateGateOpts extends Omit { + selSet: BenchmarkTask[]; + engine: BrainEngine; + targetModel: string; + judgeModel?: string; + /** D4: max in-flight sel-task evaluations. Default 4. */ + concurrency?: number; + abortSignal?: AbortSignal; + /** Test seam — substitute rollout. */ + rolloutFn?: typeof runRollout; + /** Test seam — substitute scoreTrajectory. */ + scoreFn?: typeof scoreTrajectory; +} + +/** + * Run the validation gate against a candidate skill. Returns GateResult + * with accept/reject + per-task medians for the audit JSONL. + */ +export async function runValidationGate(opts: ValidateGateOpts): Promise { + const rolloutFn = opts.rolloutFn ?? runRollout; + const scoreFn = opts.scoreFn ?? scoreTrajectory; + const concurrency = opts.concurrency ?? 4; + + // Per sel-task, run N rollouts + N judges, take median. + const settled = await runWithLimit({ + items: opts.selSet, + limit: concurrency, + fn: async (task: BenchmarkTask): Promise<{ task_id: string; median: number; runs: number[] }> => { + const runs: number[] = []; + for (let i = 0; i < VALIDATION_RUNS_PER_TASK; i++) { + const rolloutOpts: RolloutOpts = { + engine: opts.engine, + skillText: opts.candidateSkillText, + task, + targetModel: opts.targetModel, + abortSignal: opts.abortSignal, + }; + const trajectory = await rolloutFn(rolloutOpts); + const scored: ScoredRollout = await scoreFn(trajectory, task.judge, { + judgeModel: opts.judgeModel, + }); + runs.push(scored.score); + } + return { task_id: task.task_id, median: median(runs), runs }; + }, + signal: opts.abortSignal, + }); + + // SettledItem[] — extract successful results; treat errors as score=0 + // (pessimistic fallback consistent with the judge fail-open posture). + const perTaskMedians = settled.map((s, idx) => { + if (s && s.ok) return s.value; + return { task_id: opts.selSet[idx]!.task_id, median: 0, runs: [] }; + }); + const selScore = perTaskMedians.length === 0 + ? 0 + : perTaskMedians.reduce((acc, r) => acc + r.median, 0) / perTaskMedians.length; + + // D12 accept rule: strict > best + epsilon. Ties/sub-epsilon-gains are + // rejected (paper-faithful — protects against noise-as-improvement). + const threshold = opts.bestScore + VALIDATION_EPSILON; + const accepted = selScore > threshold; + + let reason: GateResult['reason']; + if (!accepted) { + if (selScore <= opts.bestScore) reason = 'below_baseline'; + else reason = 'no_margin'; + } + return { accepted, perTaskMedians, selScore, ...(reason ? { reason } : {}) }; +} + +/** Pure median for an array of numbers. Returns 0 for empty array. */ +export function median(values: readonly number[]): number { + if (values.length === 0) return 0; + const sorted = [...values].sort((a, b) => a - b); + const mid = Math.floor(sorted.length / 2); + return sorted.length % 2 === 1 + ? sorted[mid]! + : (sorted[mid - 1]! + sorted[mid]!) / 2; +}