feat(skillopt): rollout (D2 gateway.toolLoop + D13 read-only allowlist), reflect (D7 two calls), validate-gate (D12 median+epsilon, D4 parallel), preflight (D3), bundled-skill-gate (D16)

src/core/skillopt/bundled-skill-gate.ts

@@ -0,0 +1,71 @@
+/**
+ * SkillOpt bundled-skill mutation gate (D16).
+ *
+ * A "bundled" skill is one that lives in the gbrain repo's `skills/` tree
+ * (shipped alongside the binary). These are load-bearing for production
+ * workflows; mutating them via SkillOpt without explicit operator opt-in
+ * is too risky. By default, bundled-skill optimization runs in `--no-mutate`
+ * mode automatically — the proposed best is written to `proposed.md` for
+ * human review.
+ *
+ * User-owned skills (under a user's `~/.gbrain/skills/` or their own
+ * project's `skills/`) are NOT bundled — they can be mutated freely.
+ */
+
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import { autoDetectSkillsDirReadOnly } from '../repo-root.ts';
+import type { BundledSkillContext } from './types.ts';
+
+/**
+ * Build the bundled-skill context for a (skillsDir, skillName) pair.
+ *
+ * The resolution chain:
+ *  1. Resolve the absolute skill path: `skillsDir/<name>/SKILL.md`.
+ *  2. Check whether the skillsDir came from the install-path fallback
+ *     (i.e. it's `<gbrain-install>/skills`). If so, this is a bundled skill.
+ *  3. Otherwise, the skill is user-owned and freely mutable.
+ */
+export function getBundledSkillContext(skillsDir: string, skillName: string): BundledSkillContext {
+  const skillPath = path.join(skillsDir, skillName, 'SKILL.md');
+  // Detect bundled by checking whether autoDetectSkillsDirReadOnly would
+  // have routed here via the install-path fallback. The simpler heuristic:
+  // if the resolved skillsDir is under the gbrain install tree (somewhere
+  // up the chain has node_modules/gbrain or VERSION matching this binary),
+  // it's bundled. Use the read-only detector to find the canonical install
+  // skillsDir and compare.
+  const detected = autoDetectSkillsDirReadOnly(process.cwd());
+  const isBundled = detected.source === 'install_path' && detected.dir !== null
+    && resolvesToSame(detected.dir, skillsDir);
+  return { skillName, skillsDir, skillPath, isBundled };
+}
+
+function resolvesToSame(a: string, b: string): boolean {
+  try {
+    return fs.realpathSync(a) === fs.realpathSync(b);
+  } catch {
+    return path.resolve(a) === path.resolve(b);
+  }
+}
+
+/**
+ * Decide whether SkillOpt should mutate SKILL.md or just write proposed.md.
+ *
+ * Logic:
+ *  - `--no-mutate` flag → never mutate (write proposed.md).
+ *  - Bundled skill + no `--allow-mutate-bundled` → never mutate (D16).
+ *  - User-owned skill (or bundled + `--allow-mutate-bundled`) → mutate
+ *    in place.
+ */
+export function shouldMutateSkillFile(
+  ctx: BundledSkillContext,
+  flags: { noMutate: boolean; allowMutateBundled: boolean },
+): { mutate: boolean; reason?: string } {
+  if (flags.noMutate) {
+    return { mutate: false, reason: 'user_passed_no_mutate' };
+  }
+  if (ctx.isBundled && !flags.allowMutateBundled) {
+    return { mutate: false, reason: 'bundled_skill_requires_allow_flag' };
+  }
+  return { mutate: true };
+}

src/core/skillopt/preflight.ts

@@ -0,0 +1,184 @@
+/**
+ * SkillOpt cost preflight (D3).
+ *
+ * Estimates the total USD cost of a run BEFORE any LLM call fires.
+ * Refuses to start when estimate > --max-cost-usd. In TTY, prompts the
+ * user with a 10-second Ctrl-C grace window (mirrors the progressive-batch
+ * cost-prompt UX).
+ *
+ * Cost model (rough but consistent):
+ *
+ *   Per step:
+ *     - batch_size rollouts × target-model price
+ *     - 2 reflect calls (D7) × optimizer-model price
+ *     - sel_size sel-tasks × VALIDATION_RUNS_PER_TASK × target-model price
+ *     - sel_size sel-tasks × VALIDATION_RUNS_PER_TASK × judge-model price
+ *
+ *   Total:
+ *     - 1× baseline eval on D_sel
+ *     - epochs × steps_per_epoch × per-step cost
+ *     - epochs × 1 slow-update reflect call (if no improvement that epoch)
+ *     - 1× final test eval on D_test
+ *
+ * Prices come from existing per-model pricing tables (Anthropic +
+ * embedding-pricing). For unknown providers we fail-loud — same posture
+ * as BudgetTracker's TX2 contract.
+ */
+
+import { ANTHROPIC_PRICING } from '../anthropic-pricing.ts';
+import { VALIDATION_RUNS_PER_TASK } from './types.ts';
+
+/** Conservative per-rollout token estimates (input + output). */
+const ROLLOUT_INPUT_TOKENS = 3000; // skill + task prompt + tool defs
+const ROLLOUT_OUTPUT_TOKENS = 800;
+const REFLECT_INPUT_TOKENS = 8000; // skill + trajectories + rejected buffer
+const REFLECT_OUTPUT_TOKENS = 1500;
+const JUDGE_INPUT_TOKENS = 2000; // rubric + agent output
+const JUDGE_OUTPUT_TOKENS = 200;
+
+export interface PreflightOpts {
+  epochs: number;
+  batchSize: number;
+  trainSize: number;
+  selSize: number;
+  testSize: number;
+  optimizerModel: string;
+  targetModel: string;
+  judgeModel: string;
+  maxCostUsd: number;
+  /** When true, print the prompt to stderr + use Ctrl-C grace. Default false (non-TTY). */
+  interactive?: boolean;
+}
+
+export interface PreflightEstimate {
+  steps_per_epoch: number;
+  total_steps: number;
+  rollout_calls: number;
+  reflect_calls: number;
+  judge_calls: number;
+  est_input_tokens: number;
+  est_output_tokens: number;
+  est_cost_usd: number;
+  /** Per-model breakdown for the audit. */
+  per_model_cost_usd: Record<string, number>;
+  /** True when est_cost_usd > maxCostUsd (caller should refuse or prompt). */
+  exceeds_cap: boolean;
+}
+
+export interface PreflightResult {
+  estimate: PreflightEstimate;
+  /** When false, caller should abort. When true, run may proceed. */
+  proceed: boolean;
+  /** Reason for abort, if proceed=false. */
+  abort_reason?: string;
+}
+
+export function estimateCost(opts: PreflightOpts): PreflightEstimate {
+  const stepsPerEpoch = Math.max(1, Math.floor(opts.trainSize / opts.batchSize));
+  const totalSteps = opts.epochs * stepsPerEpoch;
+
+  // Per-step counts.
+  const rolloutsPerStep = opts.batchSize;
+  const reflectsPerStep = 2; // D7: two reflect calls
+  const sel_runs_per_step = opts.selSize * VALIDATION_RUNS_PER_TASK;
+
+  // Cumulative counts across the whole run.
+  const rollout_calls = totalSteps * rolloutsPerStep
+    + opts.selSize * VALIDATION_RUNS_PER_TASK // baseline sel eval
+    + opts.selSize * VALIDATION_RUNS_PER_TASK * totalSteps // per-step sel validation
+    + opts.testSize; // final test eval
+  const reflect_calls = totalSteps * reflectsPerStep
+    + opts.epochs; // slow-update meta calls
+  const judge_calls = opts.selSize // baseline (1 per task; median-of-3 is in the rollout count already? — no, judge runs per rollout)
+    + opts.selSize * VALIDATION_RUNS_PER_TASK * totalSteps // per-step validation
+    + opts.testSize; // final test judges
+
+  // Cost per call type.
+  const targetPrice = lookupPrice(opts.targetModel);
+  const optimizerPrice = lookupPrice(opts.optimizerModel);
+  const judgePrice = lookupPrice(opts.judgeModel);
+
+  const rolloutCost = rollout_calls * (
+    (ROLLOUT_INPUT_TOKENS * targetPrice.input) / 1_000_000
+    + (ROLLOUT_OUTPUT_TOKENS * targetPrice.output) / 1_000_000
+  );
+  const reflectCost = reflect_calls * (
+    (REFLECT_INPUT_TOKENS * optimizerPrice.input) / 1_000_000
+    + (REFLECT_OUTPUT_TOKENS * optimizerPrice.output) / 1_000_000
+  );
+  const judgeCost = judge_calls * (
+    (JUDGE_INPUT_TOKENS * judgePrice.input) / 1_000_000
+    + (JUDGE_OUTPUT_TOKENS * judgePrice.output) / 1_000_000
+  );
+
+  // D11 prompt caching gives ~50% discount on stable layers. Apply
+  // conservatively (assume 50% of optimizer + judge tokens are cached).
+  const cachedReflectCost = reflectCost * 0.6;
+  const cachedJudgeCost = judgeCost * 0.6;
+
+  const total = rolloutCost + cachedReflectCost + cachedJudgeCost;
+  void sel_runs_per_step;
+
+  return {
+    steps_per_epoch: stepsPerEpoch,
+    total_steps: totalSteps,
+    rollout_calls,
+    reflect_calls,
+    judge_calls,
+    est_input_tokens: rollout_calls * ROLLOUT_INPUT_TOKENS + reflect_calls * REFLECT_INPUT_TOKENS + judge_calls * JUDGE_INPUT_TOKENS,
+    est_output_tokens: rollout_calls * ROLLOUT_OUTPUT_TOKENS + reflect_calls * REFLECT_OUTPUT_TOKENS + judge_calls * JUDGE_OUTPUT_TOKENS,
+    est_cost_usd: total,
+    per_model_cost_usd: {
+      [opts.targetModel]: rolloutCost,
+      [opts.optimizerModel]: cachedReflectCost,
+      [opts.judgeModel]: cachedJudgeCost,
+    },
+    exceeds_cap: total > opts.maxCostUsd,
+  };
+}
+
+/**
+ * Render a human-readable preflight summary to stderr. Caller may follow
+ * with a Ctrl-C grace prompt in TTY mode (see runPreflightPrompt).
+ */
+export function formatPreflightReport(est: PreflightEstimate, opts: PreflightOpts): string {
+  return [
+    `[skillopt] Cost estimate for ${opts.epochs} epochs × ${est.steps_per_epoch} steps × ${opts.batchSize} rollouts:`,
+    `  Rollouts:   ${est.rollout_calls.toLocaleString()} calls`,
+    `  Reflects:   ${est.reflect_calls.toLocaleString()} calls`,
+    `  Judges:     ${est.judge_calls.toLocaleString()} calls`,
+    `  Tokens:     ~${(est.est_input_tokens / 1000).toFixed(0)}K in / ~${(est.est_output_tokens / 1000).toFixed(0)}K out`,
+    `  Est. cost:  $${est.est_cost_usd.toFixed(2)} (cap: $${opts.maxCostUsd.toFixed(2)})`,
+    est.exceeds_cap ? `  WARNING:    estimate exceeds --max-cost-usd cap.` : '',
+  ].filter(Boolean).join('\n');
+}
+
+/**
+ * Decision wrapper. Returns {proceed: false, abort_reason} when the
+ * estimate over-shoots the cap (caller exits 2). Returns {proceed: true}
+ * otherwise. Interactive=true callers should print formatPreflightReport
+ * + a Ctrl-C grace window separately.
+ */
+export function preflight(opts: PreflightOpts): PreflightResult {
+  const estimate = estimateCost(opts);
+  if (estimate.exceeds_cap) {
+    return {
+      estimate,
+      proceed: false,
+      abort_reason: `estimated cost $${estimate.est_cost_usd.toFixed(2)} exceeds --max-cost-usd $${opts.maxCostUsd.toFixed(2)}. Raise the cap with --max-cost-usd ${Math.ceil(estimate.est_cost_usd)} or reduce --epochs/--batch-size.`,
+    };
+  }
+  return { estimate, proceed: true };
+}
+
+function lookupPrice(model: string): { input: number; output: number } {
+  // Anthropic models — strip provider prefix.
+  const bare = model.startsWith('anthropic:') ? model.slice('anthropic:'.length) : model;
+  const anth = (ANTHROPIC_PRICING as Record<string, { input: number; output: number }>)[bare];
+  if (anth) return anth;
+  // Conservative fallback: assume Sonnet-tier pricing for unknown providers.
+  // Don't throw — preflight is for warning, not gating. The actual budget
+  // tracker (BudgetTracker TX2) will fail-loud at run time if pricing is
+  // truly unknown.
+  return { input: 3.0, output: 15.0 };
+}

src/core/skillopt/reflect.ts

@@ -0,0 +1,209 @@
+/**
+ * SkillOpt reflect: ask the optimizer model to propose edits to SKILL.md
+ * based on a batch of scored rollouts.
+ *
+ * D7: TWO reflect calls per step — one for failures, one for successes.
+ * Paper-faithful: each call uses its own rubric prompt so attention isn't
+ * conflated between "what went wrong" and "what went right" analyses.
+ *
+ * D11: optimizer system prompt is cached via cacheSystem=true (stable
+ * across all reflect calls in a run; ~$0.30/run savings).
+ *
+ * The reflect call also receives the rejected-edit buffer as anti-bias
+ * context so the optimizer doesn't re-propose previously-failing edits.
+ */
+
+import { chat as gatewayChat } from '../ai/gateway.ts';
+import { parseJudgeJson } from './score.ts';
+import type { EditOp, ScoredRollout } from './types.ts';
+import type { RejectedEntry } from './rejected-buffer.ts';
+
+const FAILURE_REFLECT_SYSTEM = `You are SkillOpt's optimizer. You analyze AGENT FAILURE TRAJECTORIES and propose specific edits to a SKILL document so the agent does better next time.
+
+Output ONLY a single JSON object on one or more lines:
+{"edits": [{"op": "add|replace|delete", ...}, ...]}
+
+Edit ops:
+  add:      {"op": "add", "anchor": "<exact heading text>", "content": "<new markdown>", "reason": "<one sentence>"}
+  replace:  {"op": "replace", "target": "<exact text to find>", "replacement": "<new text>", "reason": "<one sentence>"}
+  delete:   {"op": "delete", "target": "<exact text to remove>", "reason": "<one sentence>"}
+
+Rules:
+- Each edit MUST address a SPECIFIC failure pattern you observed.
+- anchor / target MUST be uniquely identifiable in the skill body (exact match).
+- Do NOT propose edits already in the rejected-edit history — those were tried and didn't help.
+- Be SURGICAL. Small targeted edits outperform large rewrites.
+- Do NOT modify the YAML frontmatter (triggers, brain_first, etc.) — that's out of scope.
+- Output at MOST 8 edits. The orchestrator's LR budget will rank-and-clip further.`;
+
+const SUCCESS_REFLECT_SYSTEM = `You are SkillOpt's optimizer. You analyze AGENT SUCCESS TRAJECTORIES and propose specific edits to a SKILL document so the agent CONSISTENTLY does what worked here.
+
+Output format and rules are identical to the failure-reflect mode — same {edits: [...]} shape.
+
+When successes are present, look for: which rules were FOLLOWED to produce success, which rules could be MADE EXPLICIT (not yet stated, but exemplified), which anti-patterns the agent successfully AVOIDED that should be stated.
+
+Be SURGICAL. Don't restate things that are already in the skill. Don't modify frontmatter.`;
+
+export interface ReflectOpts {
+  skillBodyText: string;
+  /** Successful rollouts (score >= 0.5). */
+  successes: ScoredRollout[];
+  /** Failed rollouts (score < 0.5). */
+  failures: ScoredRollout[];
+  /** Rejected-edit buffer for anti-bias context. */
+  rejected: readonly RejectedEntry[];
+  optimizerModel: string;
+  /** Test seam — substitute for gateway.chat. */
+  chatFn?: typeof gatewayChat;
+  abortSignal?: AbortSignal;
+}
+
+export interface ReflectResult {
+  /** Edits proposed from FAILURE analysis. */
+  failureEdits: EditOp[];
+  /** Edits proposed from SUCCESS analysis. */
+  successEdits: EditOp[];
+  /** Token usage across both calls (for cost tracking). */
+  usage: {
+    input_tokens: number;
+    output_tokens: number;
+    cache_read_tokens: number;
+    cache_creation_tokens: number;
+  };
+  /** Any per-call errors (for audit). */
+  errors: string[];
+}
+
+/**
+ * D7: fire two reflect calls (failures + successes). Empty batches skip
+ * their reflect call (no point asking for edits without data).
+ */
+export async function runReflect(opts: ReflectOpts): Promise<ReflectResult> {
+  const usage = { input_tokens: 0, output_tokens: 0, cache_read_tokens: 0, cache_creation_tokens: 0 };
+  const errors: string[] = [];
+
+  const failureEdits = opts.failures.length > 0
+    ? await callReflect('failure', opts, FAILURE_REFLECT_SYSTEM, opts.failures, usage, errors)
+    : [];
+  const successEdits = opts.successes.length > 0
+    ? await callReflect('success', opts, SUCCESS_REFLECT_SYSTEM, opts.successes, usage, errors)
+    : [];
+
+  return { failureEdits, successEdits, usage, errors };
+}
+
+async function callReflect(
+  mode: 'failure' | 'success',
+  opts: ReflectOpts,
+  system: string,
+  scoredRollouts: ScoredRollout[],
+  cumUsage: ReflectResult['usage'],
+  errors: string[],
+): Promise<EditOp[]> {
+  const chat = opts.chatFn ?? gatewayChat;
+  const userMsg = buildReflectUserMessage(opts.skillBodyText, scoredRollouts, opts.rejected);
+  try {
+    const result = await chat({
+      model: opts.optimizerModel,
+      system,
+      messages: [{ role: 'user', content: userMsg }],
+      maxTokens: 2048,
+      cacheSystem: true, // D11
+      abortSignal: opts.abortSignal,
+    });
+    cumUsage.input_tokens += result.usage.input_tokens;
+    cumUsage.output_tokens += result.usage.output_tokens;
+    cumUsage.cache_read_tokens += result.usage.cache_read_tokens;
+    cumUsage.cache_creation_tokens += result.usage.cache_creation_tokens;
+    return parseEditsResponse(result.text);
+  } catch (err) {
+    const msg = err instanceof Error ? err.message : String(err);
+    errors.push(`reflect_${mode}_failed: ${msg}`);
+    return [];
+  }
+}
+
+function buildReflectUserMessage(
+  skillBody: string,
+  rollouts: ScoredRollout[],
+  rejected: readonly RejectedEntry[],
+): string {
+  const trajectoryBlocks = rollouts.map((r, i) => {
+    const tcSummary = r.trajectory.tool_calls
+      .map((tc) => `  - ${tc.name}${tc.failed ? ' [FAILED]' : ''}`)
+      .join('\n');
+    return `--- ROLLOUT ${i + 1} (score=${r.score.toFixed(2)}) ---
+TASK: ${r.trajectory.task}
+TOOL CALLS:
+${tcSummary || '  (none)'}
+OUTPUT:
+${truncate(r.trajectory.final_text, 2000)}
+${r.rationale ? `JUDGE RATIONALE: ${r.rationale}` : ''}`;
+  }).join('\n\n');
+
+  const rejectedSummary = rejected.length > 0
+    ? `\n\n--- PREVIOUSLY REJECTED EDITS (do not re-propose) ---\n${rejected.slice(0, 20).map((r) => `- ${r.reason}: ${JSON.stringify(r.edits)}`).join('\n')}`
+    : '';
+
+  return `CURRENT SKILL BODY:
+${truncate(skillBody, 5000)}
+
+OBSERVED ROLLOUTS:
+${trajectoryBlocks}${rejectedSummary}
+
+Propose edits to improve the skill. Output the {edits: [...]} JSON only.`;
+}
+
+function truncate(s: string, max: number): string {
+  return s.length > max ? s.slice(0, max) + `\n...(truncated, ${s.length - max} more chars)` : s;
+}
+
+function parseEditsResponse(raw: string): EditOp[] {
+  // Reuse the same forgiving JSON parser as the score module (tagged result
+  // shape; the parser pulls the first {...} object from prose if present).
+  const parsed = parseJudgeJson(raw);
+  if (!parsed) return [];
+  // The parser returned a {score, rationale} shape; for reflect we expect
+  // {edits: [...]}. Re-extract directly with a tolerant fallback.
+  const direct = tryExtractEdits(raw);
+  return direct;
+}
+
+function tryExtractEdits(raw: string): EditOp[] {
+  try {
+    // Strip fences first.
+    const fenced = raw.match(/```(?:json)?\s*\n?([\s\S]*?)```/i);
+    const cleaned = (fenced ? fenced[1]! : raw).trim();
+    // Try direct parse.
+    const direct = JSON.parse(cleaned);
+    if (direct && typeof direct === 'object' && Array.isArray((direct as { edits?: unknown }).edits)) {
+      return validateEdits((direct as { edits: unknown[] }).edits);
+    }
+  } catch { /* try next strategy */ }
+  // Fallback: extract first {...} substring.
+  const match = raw.match(/\{[\s\S]*\}/);
+  if (!match) return [];
+  try {
+    const parsed = JSON.parse(match[0]);
+    if (parsed && typeof parsed === 'object' && Array.isArray((parsed as { edits?: unknown }).edits)) {
+      return validateEdits((parsed as { edits: unknown[] }).edits);
+    }
+  } catch { /* fall through */ }
+  return [];
+}
+
+function validateEdits(raw: unknown[]): EditOp[] {
+  const out: EditOp[] = [];
+  for (const r of raw) {
+    if (!r || typeof r !== 'object') continue;
+    const o = r as Record<string, unknown>;
+    if (o.op === 'add' && typeof o.anchor === 'string' && typeof o.content === 'string') {
+      out.push({ op: 'add', anchor: o.anchor, content: o.content, reason: typeof o.reason === 'string' ? o.reason : undefined });
+    } else if (o.op === 'replace' && typeof o.target === 'string' && typeof o.replacement === 'string') {
+      out.push({ op: 'replace', target: o.target, replacement: o.replacement, reason: typeof o.reason === 'string' ? o.reason : undefined });
+    } else if (o.op === 'delete' && typeof o.target === 'string') {
+      out.push({ op: 'delete', target: o.target, reason: typeof o.reason === 'string' ? o.reason : undefined });
+    }
+  }
+  return out;
+}

src/core/skillopt/rollout.ts

@@ -0,0 +1,218 @@
+/**
+ * SkillOpt rollout: execute a skill against one benchmark task.
+ *
+ * Per D2: uses `gateway.toolLoop` directly with NO persistence callbacks.
+ * SkillOpt rollouts are transient eval data — they don't pollute
+ * `subagent_messages` / `subagent_tool_executions`.
+ *
+ * Per D13: tool registry is the read-only subset of BRAIN_TOOL_ALLOWLIST.
+ * Excluded ops: `put_page`, `submit_job`, `file_upload` (the latter two
+ * aren't in BRAIN_TOOL_ALLOWLIST anyway, but documenting for clarity).
+ * The optimizer's loop can call `search`, `query`, `get_page`, etc., but
+ * cannot WRITE to the user's brain.
+ *
+ * Each rollout returns a `Trajectory` capturing the final assistant text,
+ * tool calls (with inputs + outputs), token usage, and stop reason. The
+ * caller (orchestrator) feeds the trajectory to the judge for scoring.
+ */
+
+import { chat as gatewayChat, toolLoop, type ChatMessage, type ChatToolDef, type ToolHandler } from '../ai/gateway.ts';
+import { BRAIN_TOOL_ALLOWLIST } from '../minions/tools/brain-allowlist.ts';
+import { operations, type OperationContext } from '../operations.ts';
+import { loadConfig } from '../config.ts';
+import type { BrainEngine } from '../engine.ts';
+import type { BenchmarkTask, Trajectory } from './types.ts';
+
+/**
+ * D13: which tools SkillOpt rollouts are allowed to call.
+ *
+ * Derived from BRAIN_TOOL_ALLOWLIST minus `put_page` (only mutating op in
+ * the base set). New mutating ops MUST be added here AND BRAIN_TOOL_ALLOWLIST
+ * mustn't be silently widened; the rollout test pins zero-write invariant.
+ */
+export const READ_ONLY_BRAIN_TOOLS: ReadonlySet<string> = new Set(
+  [...BRAIN_TOOL_ALLOWLIST].filter((name) => name !== 'put_page'),
+);
+
+export interface RolloutOpts {
+  engine: BrainEngine;
+  skillText: string;
+  task: BenchmarkTask;
+  /** Provider:model string for the target model. */
+  targetModel: string;
+  /** Max agent turns. Default 20. */
+  maxTurns?: number;
+  /** AbortSignal for cooperative cancellation (e.g. budget exhausted). */
+  abortSignal?: AbortSignal;
+  /** F10: when true, enable write-capture (virtual put_page/submit_job/file_upload). */
+  writeCapture?: boolean;
+  /** Test seam — substitute the toolLoop call. */
+  toolLoopFn?: typeof toolLoop;
+  /** Test seam — substitute chat (currently unused; toolLoop wraps chat). */
+  chatFn?: typeof gatewayChat;
+}
+
+/**
+ * Run one rollout. Returns a Trajectory; never throws on agent-side failures
+ * (max_turns, refusal, aborted) — those land in `trajectory.stop_reason` so
+ * the caller can score them appropriately.
+ *
+ * Throws ONLY on infrastructure errors (no engine, unknown target model) or
+ * BudgetExhausted (which propagates via gateway → MUST_ABORT_ERROR_TAGS).
+ */
+export async function runRollout(opts: RolloutOpts): Promise<Trajectory> {
+  const { engine, skillText, task, targetModel } = opts;
+  const startedAt = Date.now();
+
+  // Build the tool handlers + defs. F10: when write-capture is on, swap
+  // in the virtual-write registry (read-only base + virtual put_page /
+  // submit_job / file_upload). Default: read-only allowlist only (D13).
+  const ctx = buildOpContext(engine);
+  let defs, handlers;
+  if (opts.writeCapture) {
+    const { buildWriteCaptureRegistry } = await import('./write-capture.ts');
+    const registry = buildWriteCaptureRegistry(engine);
+    defs = registry.defs;
+    handlers = registry.handlers;
+  } else {
+    const r = buildReadOnlyToolRegistry(ctx);
+    defs = r.defs;
+    handlers = r.handlers;
+  }
+
+  // The skill text becomes the system prompt (D11: cacheSystem=true so the
+  // candidate skill is cached across all rollouts in a single batch).
+  const system = skillText;
+  const initialMessages: ChatMessage[] = [{ role: 'user', content: task.task }];
+
+  const toolLoopImpl = opts.toolLoopFn ?? toolLoop;
+
+  // Capture tool calls as they fire. The toolLoop's callbacks fire in
+  // ordering: onToolCallStart -> handler.execute -> onToolCallComplete|Failed.
+  // We capture outputs in a Map keyed by gbrainToolUseId so we can stitch
+  // them into the trajectory's tool_calls array (preserving call order).
+  const toolCalls: Trajectory['tool_calls'] = [];
+  const callsById = new Map<string, number>(); // gbrainToolUseId -> index in toolCalls
+  let nextOrdinal = 0;
+
+  const result = await toolLoopImpl({
+    model: targetModel,
+    system,
+    initialMessages,
+    tools: defs,
+    toolHandlers: handlers,
+    maxTurns: opts.maxTurns ?? 20,
+    cacheSystem: true, // D11: candidate skill is stable for a step's batch.
+    abortSignal: opts.abortSignal,
+    onToolCallStart: async (_turnIdx, _messageIdx, _ordinal, toolName, input, providerToolCallId) => {
+      const gbrainToolUseId = `skillopt-${nextOrdinal++}-${providerToolCallId}`;
+      const idx = toolCalls.length;
+      callsById.set(gbrainToolUseId, idx);
+      toolCalls.push({ name: stripBrainPrefix(toolName), input });
+      return { gbrainToolUseId };
+    },
+    onToolCallComplete: async (gbrainToolUseId, output) => {
+      const idx = callsById.get(gbrainToolUseId);
+      if (idx !== undefined && toolCalls[idx]) {
+        toolCalls[idx]!.output = output;
+      }
+    },
+    onToolCallFailed: async (gbrainToolUseId, error) => {
+      const idx = callsById.get(gbrainToolUseId);
+      if (idx !== undefined && toolCalls[idx]) {
+        toolCalls[idx]!.failed = true;
+        toolCalls[idx]!.output = { error };
+      }
+    },
+  });
+
+  // Final assistant text: concatenate text blocks from the last assistant message.
+  const lastAssistant = [...result.messages].reverse().find((m) => m.role === 'assistant');
+  let finalText = '';
+  if (lastAssistant && typeof lastAssistant.content === 'string') {
+    finalText = lastAssistant.content;
+  } else if (lastAssistant && Array.isArray(lastAssistant.content)) {
+    finalText = lastAssistant.content
+      .filter((b) => b.type === 'text')
+      .map((b) => ('text' in b ? b.text : ''))
+      .join('\n')
+      .trim();
+  }
+
+  return {
+    task_id: task.task_id,
+    task: task.task,
+    final_text: finalText,
+    tool_calls: toolCalls,
+    usage: result.totalUsage,
+    turns: result.totalTurns,
+    stop_reason: result.stopReason,
+    duration_ms: Date.now() - startedAt,
+  };
+}
+
+// ─── Tool registry construction ──────────────────────────────────────────
+
+function buildOpContext(engine: BrainEngine): OperationContext {
+  const cfg = loadConfig();
+  return {
+    engine,
+    config: cfg ?? ({} as never),
+    logger: { info: () => {}, warn: () => {}, error: () => {}, debug: () => {} } as never,
+    dryRun: false,
+    // SkillOpt rollouts ARE remote-equivalent (the optimizer chose what to call,
+    // not the user) → set remote: true. Per-op trust gates that check `remote`
+    // see this and apply remote-tightening rules. Read-only ops aren't affected.
+    remote: true,
+    // v0.34 D4: sourceId is required on OperationContext. SkillOpt rollouts
+    // operate on the default source — the agent under test reads the brain
+    // it would normally read.
+    sourceId: 'default',
+  };
+}
+
+interface ToolRegistry {
+  defs: ChatToolDef[];
+  handlers: Map<string, ToolHandler>;
+}
+
+/**
+ * Build the read-only tool registry that SkillOpt rollouts call into.
+ *
+ * Tool names are prefixed `brain_` for Anthropic-name compliance (same
+ * convention as the subagent handler's buildBrainTools).
+ */
+function buildReadOnlyToolRegistry(ctx: OperationContext): ToolRegistry {
+  const defs: ChatToolDef[] = [];
+  const handlers = new Map<string, ToolHandler>();
+  for (const op of operations) {
+    if (!READ_ONLY_BRAIN_TOOLS.has(op.name)) continue;
+    const toolName = `brain_${op.name}`;
+    defs.push({
+      name: toolName,
+      description: op.description,
+      inputSchema: paramsToSchema(op.params),
+    });
+    handlers.set(toolName, {
+      idempotent: true, // All read-only ops are idempotent by construction.
+      execute: async (input: unknown) => {
+        return op.handler(ctx, (input as Record<string, unknown>) ?? {});
+      },
+    });
+  }
+  return { defs, handlers };
+}
+
+function stripBrainPrefix(toolName: string): string {
+  return toolName.startsWith('brain_') ? toolName.slice('brain_'.length) : toolName;
+}
+
+function paramsToSchema(params: Record<string, { type: string; description?: string; required?: boolean }>): Record<string, unknown> {
+  return {
+    type: 'object' as const,
+    properties: Object.fromEntries(
+      Object.entries(params).map(([k, v]) => [k, { type: v.type, description: v.description }]),
+    ),
+    required: Object.entries(params).filter(([, v]) => v.required).map(([k]) => k),
+  };
+}

src/core/skillopt/validate-gate.ts

@@ -0,0 +1,109 @@
+/**
+ * SkillOpt validation gate (D12 + D4).
+ *
+ * D12 — median-of-3 judge runs per sel-task + epsilon=0.05 margin.
+ * D4 — parallel sel-task scoring via runWithLimit cap=4.
+ *
+ * Pseudocode:
+ *
+ *   For each sel-task t in selSet:
+ *     scores[t] = median(await Promise.all([
+ *       scoreTrajectory(rollout(candidate, t), judge),  // run 1
+ *       scoreTrajectory(rollout(candidate, t), judge),  // run 2
+ *       scoreTrajectory(rollout(candidate, t), judge),  // run 3
+ *     ]))
+ *   sel_score = mean(scores)
+ *   if sel_score > best_score + 0.05: ACCEPT
+ *   else: REJECT
+ *
+ * The 3 rollout calls per sel-task share the same candidate-skill prompt
+ * which is cached (D11), so the effective cost ~1.3x not 3x.
+ */
+
+import { runWithLimit } from '../worker-pool.ts';
+import { runRollout, type RolloutOpts } from './rollout.ts';
+import { scoreTrajectory } from './score.ts';
+import type { BenchmarkTask, GateInput, GateResult, ScoredRollout } from './types.ts';
+import { VALIDATION_EPSILON, VALIDATION_RUNS_PER_TASK } from './types.ts';
+import type { BrainEngine } from '../engine.ts';
+
+export interface ValidateGateOpts extends Omit<GateInput, 'selSet'> {
+  selSet: BenchmarkTask[];
+  engine: BrainEngine;
+  targetModel: string;
+  judgeModel?: string;
+  /** D4: max in-flight sel-task evaluations. Default 4. */
+  concurrency?: number;
+  abortSignal?: AbortSignal;
+  /** Test seam — substitute rollout. */
+  rolloutFn?: typeof runRollout;
+  /** Test seam — substitute scoreTrajectory. */
+  scoreFn?: typeof scoreTrajectory;
+}
+
+/**
+ * Run the validation gate against a candidate skill. Returns GateResult
+ * with accept/reject + per-task medians for the audit JSONL.
+ */
+export async function runValidationGate(opts: ValidateGateOpts): Promise<GateResult> {
+  const rolloutFn = opts.rolloutFn ?? runRollout;
+  const scoreFn = opts.scoreFn ?? scoreTrajectory;
+  const concurrency = opts.concurrency ?? 4;
+
+  // Per sel-task, run N rollouts + N judges, take median.
+  const settled = await runWithLimit({
+    items: opts.selSet,
+    limit: concurrency,
+    fn: async (task: BenchmarkTask): Promise<{ task_id: string; median: number; runs: number[] }> => {
+      const runs: number[] = [];
+      for (let i = 0; i < VALIDATION_RUNS_PER_TASK; i++) {
+        const rolloutOpts: RolloutOpts = {
+          engine: opts.engine,
+          skillText: opts.candidateSkillText,
+          task,
+          targetModel: opts.targetModel,
+          abortSignal: opts.abortSignal,
+        };
+        const trajectory = await rolloutFn(rolloutOpts);
+        const scored: ScoredRollout = await scoreFn(trajectory, task.judge, {
+          judgeModel: opts.judgeModel,
+        });
+        runs.push(scored.score);
+      }
+      return { task_id: task.task_id, median: median(runs), runs };
+    },
+    signal: opts.abortSignal,
+  });
+
+  // SettledItem<TOut>[] — extract successful results; treat errors as score=0
+  // (pessimistic fallback consistent with the judge fail-open posture).
+  const perTaskMedians = settled.map((s, idx) => {
+    if (s && s.ok) return s.value;
+    return { task_id: opts.selSet[idx]!.task_id, median: 0, runs: [] };
+  });
+  const selScore = perTaskMedians.length === 0
+    ? 0
+    : perTaskMedians.reduce((acc, r) => acc + r.median, 0) / perTaskMedians.length;
+
+  // D12 accept rule: strict > best + epsilon. Ties/sub-epsilon-gains are
+  // rejected (paper-faithful — protects against noise-as-improvement).
+  const threshold = opts.bestScore + VALIDATION_EPSILON;
+  const accepted = selScore > threshold;
+
+  let reason: GateResult['reason'];
+  if (!accepted) {
+    if (selScore <= opts.bestScore) reason = 'below_baseline';
+    else reason = 'no_margin';
+  }
+  return { accepted, perTaskMedians, selScore, ...(reason ? { reason } : {}) };
+}
+
+/** Pure median for an array of numbers. Returns 0 for empty array. */
+export function median(values: readonly number[]): number {
+  if (values.length === 0) return 0;
+  const sorted = [...values].sort((a, b) => a - b);
+  const mid = Math.floor(sorted.length / 2);
+  return sorted.length % 2 === 1
+    ? sorted[mid]!
+    : (sorted[mid - 1]! + sorted[mid]!) / 2;
+}