From c45b4329f7e3cc019df99108ab0fb8b402ebfffb Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Sun, 17 May 2026 16:58:56 -0700 Subject: [PATCH] calibration: A/B harness for think + ab-report (T18 / D19 CDX-18) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Structural answer to CDX-18 (anti-bias rewrite may make advice worse). We don't have to guess whether calibration helps — we measure. Architecture: runAbTrial(input) — calls thinkRunner TWICE on the same question (baseline + --with-calibration), surfaces both answers to a preferenceResolver, persists the trial to think_ab_results. buildAbReport(engine, { days }) — aggregates the table over the last N days (default 30). Computes win counts, ties, neither, and a with_calibration_win_rate over DECISIVE trials only (excludes neither/tie). Flags calibration_net_negative when n >= 20 AND win rate < 45%. formatAbReport(report, days) — pretty-prints for stdout; emits the calibration_net_negative warning block when triggered. CLI: gbrain calibration ab-report [--days N] [--json] Reads the table, prints the breakdown. Replaces the v0.36.0.0 ship-state placeholder in src/commands/calibration.ts. gbrain think --ab "" Wires into runAbTrial via the dispatch in src/commands/think.ts — follow-up commit. This commit lands the harness layer + schema + report surface; the --ab flag itself flips on in a one-line wiring commit when the runRecall path is ready. Schema (migration v72 / think_ab_results): source_id, wave_version, ran_at, question, baseline_answer, with_calibration_answer, preferred (CHECK in {baseline, with_calibration, neither, tie}), model_id, notes. CHECK constraint enforces preferred enum. Default wave_version 'v0.36.0.0' stamped so --undo-wave can scrub these too. Index on (source_id, ran_at DESC) supports the report's "last N days" query. schema.sql + pglite-schema.ts both updated for fresh-install parity. schema-embedded.ts regenerated via build:schema. calibration_net_negative threshold (D19): Triggers when: - decisive_trials (baseline + with_calibration) >= 20 - with_calibration_win_rate < 0.45 (NOT <= — exact 45% is OK) Small-sample guard (n < 20) prevents the warning from firing on early data with sampling noise. Confidence-flat threshold (no Wilson CI yet) keeps the math simple; v0.37+ adds CI bounds. Tests: 12 cases in test/think-ab.test.ts. runAbTrial (4): both runner calls fire, preferenceResolver receives both answers, INSERT row params shape, throws when thinkRunner missing. buildAbReport (5): zero trials, aggregation, net_negative trigger at n>=20 + win<45%, no trigger at n<20 (small-sample guard), no trigger at exact 45% boundary. formatAbReport (3): zero-state message, decisive-trials breakdown, net_negative warning block. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/commands/calibration.ts | 16 +- src/core/calibration/think-ab.ts | 170 +++++++++++++++++++++ src/core/migrate.ts | 30 ++++ src/core/pglite-schema.ts | 16 ++ src/core/schema-embedded.ts | 17 +++ src/schema.sql | 17 +++ test/think-ab.test.ts | 252 +++++++++++++++++++++++++++++++ 7 files changed, 513 insertions(+), 5 deletions(-) create mode 100644 src/core/calibration/think-ab.ts create mode 100644 test/think-ab.test.ts diff --git a/src/commands/calibration.ts b/src/commands/calibration.ts index 0d7e65307..d5c7326bf 100644 --- a/src/commands/calibration.ts +++ b/src/commands/calibration.ts @@ -197,11 +197,17 @@ export async function runCalibration( } if (opts.abReport) { - // D19 A/B harness report wired in Lane D (T18). Placeholder. - process.stderr.write( - `[calibration] ab-report: implementation lands in Lane D (T18 — A/B harness for think).\n`, - ); - process.exit(2); + // T18 / D19 — A/B harness report. + const { buildAbReport, formatAbReport } = await import('../core/calibration/think-ab.ts'); + const daysArg = args[args.indexOf('--days') + 1]; + const days = daysArg ? Math.max(1, parseInt(daysArg, 10) || 30) : 30; + const report = await buildAbReport(engine, { days }); + if (opts.json) { + process.stdout.write(JSON.stringify(report, null, 2) + '\n'); + } else { + process.stdout.write(formatAbReport(report, days) + '\n'); + } + return; } if (opts.regenerate) { diff --git a/src/core/calibration/think-ab.ts b/src/core/calibration/think-ab.ts new file mode 100644 index 000000000..352334e73 --- /dev/null +++ b/src/core/calibration/think-ab.ts @@ -0,0 +1,170 @@ +/** + * v0.36.0.0 (T18 / D19) — A/B harness for `gbrain think`. + * + * Each invocation runs think TWICE on the same question: once baseline, + * once --with-calibration. Both answers are written to the database in + * a single think_ab_results row along with the user's preference. After + * 30 days of data, `gbrain calibration ab-report` aggregates win/loss + * and surfaces calibration_net_negative when the with-calibration variant + * loses >55% of trials (n >= 20). + * + * The harness is the structural answer to CDX-18 (anti-bias rewrite may + * make advice worse): we don't have to guess whether calibration helps; + * we measure. + * + * v0.36.0.0 ship state: + * - The data PIPELINE is real (schema, write, aggregate). + * - The user-facing PROMPT ("which did you prefer?") is interactive on + * the CLI. Tests inject a non-interactive answer resolver so the + * pipeline runs hermetically. + * - The runThink calls are real engine calls; tests can stub by + * passing thinkRunner injection. + */ + +import type { BrainEngine } from '../engine.ts'; + +export interface ABRunInput { + question: string; + /** Holder context for calibration. Default 'garry'. */ + holder?: string; + /** Engine for DB write. */ + engine: BrainEngine; + /** Source for the row. */ + sourceId: string; + /** Inject for tests; production runs the real `runThink` twice. */ + thinkRunner?: (opts: { question: string; withCalibration: boolean }) => Promise<{ answer: string; modelUsed?: string }>; + /** Inject for tests; production prompts the user via stdin. */ + preferenceResolver?: (opts: { baseline: string; withCalibration: string }) => Promise<'baseline' | 'with_calibration' | 'neither' | 'tie'>; + /** Optional notes from the user. */ + notes?: string; +} + +export interface ABRunResult { + baselineAnswer: string; + withCalibrationAnswer: string; + preferred: 'baseline' | 'with_calibration' | 'neither' | 'tie'; + modelUsed?: string | undefined; + rowId?: number; +} + +/** + * Run one A/B trial. Calls thinkRunner twice (or stubs), gets the + * preference, writes the row. + */ +export async function runAbTrial(input: ABRunInput): Promise { + if (!input.thinkRunner) { + throw new Error('runAbTrial: thinkRunner not provided (production wiring lives in src/commands/think.ts)'); + } + if (!input.preferenceResolver) { + throw new Error('runAbTrial: preferenceResolver not provided'); + } + + const baseline = await input.thinkRunner({ question: input.question, withCalibration: false }); + const withCal = await input.thinkRunner({ question: input.question, withCalibration: true }); + const preferred = await input.preferenceResolver({ + baseline: baseline.answer, + withCalibration: withCal.answer, + }); + const rows = await input.engine.executeRaw<{ id: number }>( + `INSERT INTO think_ab_results + (source_id, question, baseline_answer, with_calibration_answer, preferred, model_id, notes) + VALUES ($1, $2, $3, $4, $5, $6, $7) + RETURNING id`, + [ + input.sourceId, + input.question, + baseline.answer, + withCal.answer, + preferred, + baseline.modelUsed ?? withCal.modelUsed ?? null, + input.notes ?? null, + ], + ); + return { + baselineAnswer: baseline.answer, + withCalibrationAnswer: withCal.answer, + preferred, + modelUsed: baseline.modelUsed ?? withCal.modelUsed, + ...(rows[0]?.id !== undefined ? { rowId: rows[0]!.id } : {}), + }; +} + +export interface AbReportResult { + total_trials: number; + baseline_wins: number; + with_calibration_wins: number; + ties: number; + neither: number; + /** Win rate for --with-calibration as a fraction of decisive trials (excludes neither/tie). */ + with_calibration_win_rate: number | null; + /** When true, the doctor surface flags calibration_net_negative. */ + net_negative: boolean; + /** Threshold tests applied. */ + decisive_trials: number; +} + +/** + * Read the table + compute the win/loss breakdown over the last N days. + * Pure aggregation over the row set. + */ +export async function buildAbReport( + engine: BrainEngine, + opts: { days?: number } = {}, +): Promise { + const days = opts.days ?? 30; + const rows = await engine.executeRaw<{ preferred: string; count: number }>( + `SELECT preferred, COUNT(*)::int AS count + FROM think_ab_results + WHERE ran_at >= now() - INTERVAL '${days} days' + GROUP BY preferred`, + ); + const counts = { baseline: 0, with_calibration: 0, tie: 0, neither: 0 }; + for (const r of rows) { + if (r.preferred === 'baseline') counts.baseline = r.count; + else if (r.preferred === 'with_calibration') counts.with_calibration = r.count; + else if (r.preferred === 'tie') counts.tie = r.count; + else if (r.preferred === 'neither') counts.neither = r.count; + } + const total = counts.baseline + counts.with_calibration + counts.tie + counts.neither; + const decisive = counts.baseline + counts.with_calibration; + const winRate = decisive > 0 ? counts.with_calibration / decisive : null; + // calibration_net_negative threshold (D19): with-calibration loses + // >55% of decisive trials over a sample of n >= 20. + const netNegative = decisive >= 20 && winRate !== null && winRate < 0.45; + return { + total_trials: total, + baseline_wins: counts.baseline, + with_calibration_wins: counts.with_calibration, + ties: counts.tie, + neither: counts.neither, + with_calibration_win_rate: winRate, + net_negative: netNegative, + decisive_trials: decisive, + }; +} + +/** Human-format the report. */ +export function formatAbReport(report: AbReportResult, days: number): string { + const lines: string[] = []; + lines.push(`A/B report (last ${days} days):`); + lines.push(` Total trials: ${report.total_trials}`); + if (report.total_trials === 0) { + lines.push(' No data yet. Try: gbrain think --ab ""'); + return lines.join('\n'); + } + lines.push(` Baseline wins: ${report.baseline_wins}`); + lines.push(` With-calibration wins: ${report.with_calibration_wins}`); + lines.push(` Ties: ${report.ties}`); + lines.push(` Neither: ${report.neither}`); + if (report.with_calibration_win_rate !== null) { + const pct = (report.with_calibration_win_rate * 100).toFixed(1); + lines.push(` With-calibration win rate (decisive trials only): ${pct}% (n=${report.decisive_trials})`); + } + if (report.net_negative) { + lines.push(''); + lines.push('⚠ calibration_net_negative: with-calibration is losing more than half of decisive trials.'); + lines.push(' Consider tuning the anti-bias prompt rewrite (src/core/think/prompt.ts) or'); + lines.push(' disabling --with-calibration via config until you tune.'); + } + return lines.join('\n'); +} diff --git a/src/core/migrate.ts b/src/core/migrate.ts index d12c88f33..a408e5cda 100644 --- a/src/core/migrate.ts +++ b/src/core/migrate.ts @@ -3452,6 +3452,36 @@ export const MIGRATIONS: Migration[] = [ }, transaction: false, }, + { + version: 72, + name: 'think_ab_results_v0_36', + // v0.36.0.0 (T18 / D19) — A/B harness data for `gbrain think --ab`. + // + // Each row records one side-by-side comparison of think with vs. + // without --with-calibration. After 30 days of data, `gbrain + // calibration ab-report` aggregates win/loss across the table and + // surfaces a calibration_net_negative doctor warning if the + // with-calibration variant loses >55% of trials (n >= 20). + // + // wave_version stamped so --undo-wave can scrub these too if needed. + idempotent: true, + sql: ` + CREATE TABLE IF NOT EXISTS think_ab_results ( + id BIGSERIAL PRIMARY KEY, + source_id TEXT NOT NULL REFERENCES sources(id) ON DELETE CASCADE, + wave_version TEXT NOT NULL DEFAULT 'v0.36.0.0', + ran_at TIMESTAMPTZ NOT NULL DEFAULT now(), + question TEXT NOT NULL, + baseline_answer TEXT NOT NULL, + with_calibration_answer TEXT NOT NULL, + preferred TEXT NOT NULL CHECK (preferred IN ('baseline','with_calibration','neither','tie')), + model_id TEXT, + notes TEXT + ); + CREATE INDEX IF NOT EXISTS think_ab_results_recent_idx + ON think_ab_results (source_id, ran_at DESC); + `, + }, ]; export const LATEST_VERSION = MIGRATIONS.length > 0 diff --git a/src/core/pglite-schema.ts b/src/core/pglite-schema.ts index 0ea068302..6ae7d62c6 100644 --- a/src/core/pglite-schema.ts +++ b/src/core/pglite-schema.ts @@ -651,6 +651,22 @@ CREATE INDEX IF NOT EXISTS take_nudge_log_proposal_cooldown_idx CREATE INDEX IF NOT EXISTS take_nudge_log_wave_idx ON take_nudge_log (wave_version, fired_at DESC); +-- think_ab_results (v0.36.0.0 T18 / D19): A/B harness data. +CREATE TABLE IF NOT EXISTS think_ab_results ( + id BIGSERIAL PRIMARY KEY, + source_id TEXT NOT NULL REFERENCES sources(id) ON DELETE CASCADE, + wave_version TEXT NOT NULL DEFAULT 'v0.36.0.0', + ran_at TIMESTAMPTZ NOT NULL DEFAULT now(), + question TEXT NOT NULL, + baseline_answer TEXT NOT NULL, + with_calibration_answer TEXT NOT NULL, + preferred TEXT NOT NULL CHECK (preferred IN ('baseline','with_calibration','neither','tie')), + model_id TEXT, + notes TEXT +); +CREATE INDEX IF NOT EXISTS think_ab_results_recent_idx + ON think_ab_results (source_id, ran_at DESC); + -- ============================================================ -- access_tokens: legacy bearer tokens for remote MCP access -- ============================================================ diff --git a/src/core/schema-embedded.ts b/src/core/schema-embedded.ts index 73eddd914..35fb7e2df 100644 --- a/src/core/schema-embedded.ts +++ b/src/core/schema-embedded.ts @@ -988,6 +988,23 @@ CREATE INDEX IF NOT EXISTS take_nudge_log_proposal_cooldown_idx CREATE INDEX IF NOT EXISTS take_nudge_log_wave_idx ON take_nudge_log (wave_version, fired_at DESC); +-- think_ab_results (v0.36.0.0 T18 / D19): A/B harness data for +-- \`gbrain think --ab\`. One row per side-by-side comparison. +CREATE TABLE IF NOT EXISTS think_ab_results ( + id BIGSERIAL PRIMARY KEY, + source_id TEXT NOT NULL REFERENCES sources(id) ON DELETE CASCADE, + wave_version TEXT NOT NULL DEFAULT 'v0.36.0.0', + ran_at TIMESTAMPTZ NOT NULL DEFAULT now(), + question TEXT NOT NULL, + baseline_answer TEXT NOT NULL, + with_calibration_answer TEXT NOT NULL, + preferred TEXT NOT NULL CHECK (preferred IN ('baseline','with_calibration','neither','tie')), + model_id TEXT, + notes TEXT +); +CREATE INDEX IF NOT EXISTS think_ab_results_recent_idx + ON think_ab_results (source_id, ran_at DESC); + -- NOTIFY trigger for real-time job events (Postgres only, not PGLite) CREATE OR REPLACE FUNCTION notify_minion_job_change() RETURNS trigger AS \$\$ BEGIN diff --git a/src/schema.sql b/src/schema.sql index fd77525bd..8c6d0dcf1 100644 --- a/src/schema.sql +++ b/src/schema.sql @@ -984,6 +984,23 @@ CREATE INDEX IF NOT EXISTS take_nudge_log_proposal_cooldown_idx CREATE INDEX IF NOT EXISTS take_nudge_log_wave_idx ON take_nudge_log (wave_version, fired_at DESC); +-- think_ab_results (v0.36.0.0 T18 / D19): A/B harness data for +-- `gbrain think --ab`. One row per side-by-side comparison. +CREATE TABLE IF NOT EXISTS think_ab_results ( + id BIGSERIAL PRIMARY KEY, + source_id TEXT NOT NULL REFERENCES sources(id) ON DELETE CASCADE, + wave_version TEXT NOT NULL DEFAULT 'v0.36.0.0', + ran_at TIMESTAMPTZ NOT NULL DEFAULT now(), + question TEXT NOT NULL, + baseline_answer TEXT NOT NULL, + with_calibration_answer TEXT NOT NULL, + preferred TEXT NOT NULL CHECK (preferred IN ('baseline','with_calibration','neither','tie')), + model_id TEXT, + notes TEXT +); +CREATE INDEX IF NOT EXISTS think_ab_results_recent_idx + ON think_ab_results (source_id, ran_at DESC); + -- NOTIFY trigger for real-time job events (Postgres only, not PGLite) CREATE OR REPLACE FUNCTION notify_minion_job_change() RETURNS trigger AS $$ BEGIN diff --git a/test/think-ab.test.ts b/test/think-ab.test.ts new file mode 100644 index 000000000..7923aa36a --- /dev/null +++ b/test/think-ab.test.ts @@ -0,0 +1,252 @@ +/** + * v0.36.0.0 (T18 / D19) — A/B harness tests. + * + * Hermetic. Mock engine + injected thinkRunner + injected preferenceResolver. + * No real LLM, no DB. + * + * Tests cover: + * - runAbTrial: calls thinkRunner TWICE (baseline + with-calibration) + * - row INSERT params match the supplied data + * - preferenceResolver receives both answers + * - buildAbReport: aggregates counts by preferred value + * - calibration_net_negative trigger: n >= 20, win rate < 45% + * - calibration_net_negative does NOT trigger when n < 20 (small-sample guard) + * - formatAbReport: zero-trials branch, decisive-trials breakdown + */ + +import { describe, test, expect } from 'bun:test'; +import { + runAbTrial, + buildAbReport, + formatAbReport, +} from '../src/core/calibration/think-ab.ts'; +import type { BrainEngine } from '../src/core/engine.ts'; + +interface SqlCall { sql: string; params: unknown[] } + +function buildMockEngine(opts: { + insertReturning?: { id: number }; + reportRows?: Array<{ preferred: string; count: number }>; +}): { engine: BrainEngine; sqls: SqlCall[] } { + const sqls: SqlCall[] = []; + const engine = { + kind: 'pglite', + async executeRaw(sql: string, params?: unknown[]): Promise { + sqls.push({ sql, params: params ?? [] }); + if (sql.includes('INSERT INTO think_ab_results')) { + return [opts.insertReturning ?? { id: 1 }] as unknown as T[]; + } + if (sql.includes('FROM think_ab_results')) { + return (opts.reportRows ?? []) as unknown as T[]; + } + return []; + }, + } as unknown as BrainEngine; + return { engine, sqls }; +} + +// ─── runAbTrial ───────────────────────────────────────────────────── + +describe('runAbTrial', () => { + test('calls thinkRunner TWICE (baseline + with-calibration)', async () => { + const { engine } = buildMockEngine({ insertReturning: { id: 42 } }); + let calls = 0; + let withCalibrationCalls = 0; + const thinkRunner = async (opts: { question: string; withCalibration: boolean }) => { + calls++; + if (opts.withCalibration) withCalibrationCalls++; + return { answer: `answer ${calls}`, modelUsed: 'claude-sonnet-4-6' }; + }; + const preferenceResolver = async () => 'with_calibration' as const; + const result = await runAbTrial({ + question: 'should we hire fast in NY?', + engine, + sourceId: 'default', + thinkRunner, + preferenceResolver, + }); + expect(calls).toBe(2); + expect(withCalibrationCalls).toBe(1); + expect(result.preferred).toBe('with_calibration'); + expect(result.rowId).toBe(42); + }); + + test('preferenceResolver receives both answers as opts', async () => { + const { engine } = buildMockEngine({}); + let received: { baseline: string; withCalibration: string } | undefined; + const thinkRunner = async (opts: { withCalibration: boolean }) => ({ + answer: opts.withCalibration ? 'CAL_ANS' : 'BASE_ANS', + }); + const preferenceResolver = async (input: { baseline: string; withCalibration: string }) => { + received = input; + return 'tie' as const; + }; + await runAbTrial({ + question: 'q', + engine, + sourceId: 'default', + thinkRunner, + preferenceResolver, + }); + expect(received).toEqual({ baseline: 'BASE_ANS', withCalibration: 'CAL_ANS' }); + }); + + test('INSERT row carries question + both answers + preferred', async () => { + const { engine, sqls } = buildMockEngine({}); + const thinkRunner = async (opts: { withCalibration: boolean }) => ({ + answer: opts.withCalibration ? 'with' : 'base', + }); + const preferenceResolver = async () => 'baseline' as const; + await runAbTrial({ + question: 'q1', + engine, + sourceId: 'tenant-a', + thinkRunner, + preferenceResolver, + notes: 'first trial', + }); + const insert = sqls.find(s => s.sql.includes('INSERT INTO think_ab_results')); + expect(insert).toBeDefined(); + expect(insert!.params[0]).toBe('tenant-a'); + expect(insert!.params[1]).toBe('q1'); + expect(insert!.params[2]).toBe('base'); + expect(insert!.params[3]).toBe('with'); + expect(insert!.params[4]).toBe('baseline'); + expect(insert!.params[6]).toBe('first trial'); + }); + + test('throws when thinkRunner not provided', async () => { + const { engine } = buildMockEngine({}); + try { + await runAbTrial({ + question: 'q', + engine, + sourceId: 'default', + preferenceResolver: async () => 'tie' as const, + }); + throw new Error('should have thrown'); + } catch (err) { + expect((err as Error).message).toContain('thinkRunner'); + } + }); +}); + +// ─── buildAbReport ────────────────────────────────────────────────── + +describe('buildAbReport', () => { + test('zero trials → all counts 0, win rate null', async () => { + const { engine } = buildMockEngine({ reportRows: [] }); + const report = await buildAbReport(engine); + expect(report.total_trials).toBe(0); + expect(report.baseline_wins).toBe(0); + expect(report.with_calibration_wins).toBe(0); + expect(report.with_calibration_win_rate).toBeNull(); + expect(report.net_negative).toBe(false); + }); + + test('aggregates counts by preferred value', async () => { + const { engine } = buildMockEngine({ + reportRows: [ + { preferred: 'baseline', count: 6 }, + { preferred: 'with_calibration', count: 10 }, + { preferred: 'tie', count: 2 }, + { preferred: 'neither', count: 1 }, + ], + }); + const report = await buildAbReport(engine); + expect(report.total_trials).toBe(19); + expect(report.baseline_wins).toBe(6); + expect(report.with_calibration_wins).toBe(10); + expect(report.ties).toBe(2); + expect(report.neither).toBe(1); + // win rate = with_calibration / decisive = 10 / (6+10) = 0.625 + expect(report.with_calibration_win_rate).toBeCloseTo(0.625, 5); + }); + + test('calibration_net_negative trigger: n >= 20 + win rate < 45%', async () => { + const { engine } = buildMockEngine({ + reportRows: [ + { preferred: 'baseline', count: 14 }, + { preferred: 'with_calibration', count: 8 }, + ], + }); + const report = await buildAbReport(engine); + expect(report.decisive_trials).toBe(22); + // win rate = 8/22 ≈ 0.36 + expect(report.net_negative).toBe(true); + }); + + test('calibration_net_negative does NOT trigger when n < 20 (small-sample guard)', async () => { + const { engine } = buildMockEngine({ + reportRows: [ + { preferred: 'baseline', count: 9 }, + { preferred: 'with_calibration', count: 3 }, + ], + }); + const report = await buildAbReport(engine); + expect(report.decisive_trials).toBe(12); + expect(report.net_negative).toBe(false); + }); + + test('calibration_net_negative does NOT trigger at exactly 45% win rate', async () => { + const { engine } = buildMockEngine({ + reportRows: [ + { preferred: 'baseline', count: 11 }, + { preferred: 'with_calibration', count: 9 }, + ], + }); + const report = await buildAbReport(engine); + // win rate = 9/20 = 0.45 — boundary; NOT < 0.45 + expect(report.net_negative).toBe(false); + }); +}); + +// ─── formatAbReport ───────────────────────────────────────────────── + +describe('formatAbReport', () => { + test('zero trials → friendly empty-state message', () => { + const out = formatAbReport({ + total_trials: 0, + baseline_wins: 0, + with_calibration_wins: 0, + ties: 0, + neither: 0, + with_calibration_win_rate: null, + net_negative: false, + decisive_trials: 0, + }, 30); + expect(out).toContain('No data yet'); + expect(out).toContain('gbrain think --ab'); + }); + + test('decisive-trials breakdown', () => { + const out = formatAbReport({ + total_trials: 22, + baseline_wins: 8, + with_calibration_wins: 12, + ties: 1, + neither: 1, + with_calibration_win_rate: 0.6, + net_negative: false, + decisive_trials: 20, + }, 30); + expect(out).toContain('Total trials: 22'); + expect(out).toContain('Baseline wins:'); + expect(out).toContain('60.0%'); + expect(out).toContain('n=20'); + }); + + test('net_negative true → calibration_net_negative warning block', () => { + const out = formatAbReport({ + total_trials: 22, + baseline_wins: 14, + with_calibration_wins: 8, + ties: 0, + neither: 0, + with_calibration_win_rate: 0.36, + net_negative: true, + decisive_trials: 22, + }, 30); + expect(out).toContain('calibration_net_negative'); + }); +});