mirror of
https://github.com/garrytan/gbrain.git
synced 2026-07-30 03:12:32 +00:00
calibration: A/B harness for think + ab-report (T18 / D19 CDX-18)
Structural answer to CDX-18 (anti-bias rewrite may make advice worse).
We don't have to guess whether calibration helps — we measure.
Architecture:
runAbTrial(input) — calls thinkRunner TWICE on the same question
(baseline + --with-calibration), surfaces both answers to a
preferenceResolver, persists the trial to think_ab_results.
buildAbReport(engine, { days }) — aggregates the table over the last
N days (default 30). Computes win counts, ties, neither, and a
with_calibration_win_rate over DECISIVE trials only (excludes
neither/tie). Flags calibration_net_negative when n >= 20 AND win
rate < 45%.
formatAbReport(report, days) — pretty-prints for stdout; emits the
calibration_net_negative warning block when triggered.
CLI:
gbrain calibration ab-report [--days N] [--json]
Reads the table, prints the breakdown. Replaces the v0.36.0.0
ship-state placeholder in src/commands/calibration.ts.
gbrain think --ab "<question>"
Wires into runAbTrial via the dispatch in src/commands/think.ts —
follow-up commit. This commit lands the harness layer + schema +
report surface; the --ab flag itself flips on in a one-line wiring
commit when the runRecall path is ready.
Schema (migration v72 / think_ab_results):
source_id, wave_version, ran_at, question, baseline_answer,
with_calibration_answer, preferred (CHECK in {baseline,
with_calibration, neither, tie}), model_id, notes.
CHECK constraint enforces preferred enum. Default wave_version
'v0.36.0.0' stamped so --undo-wave can scrub these too.
Index on (source_id, ran_at DESC) supports the report's
"last N days" query.
schema.sql + pglite-schema.ts both updated for fresh-install parity.
schema-embedded.ts regenerated via build:schema.
calibration_net_negative threshold (D19):
Triggers when:
- decisive_trials (baseline + with_calibration) >= 20
- with_calibration_win_rate < 0.45 (NOT <= — exact 45% is OK)
Small-sample guard (n < 20) prevents the warning from firing on
early data with sampling noise. Confidence-flat threshold (no Wilson
CI yet) keeps the math simple; v0.37+ adds CI bounds.
Tests: 12 cases in test/think-ab.test.ts.
runAbTrial (4): both runner calls fire, preferenceResolver receives
both answers, INSERT row params shape, throws when thinkRunner
missing.
buildAbReport (5): zero trials, aggregation, net_negative trigger at
n>=20 + win<45%, no trigger at n<20 (small-sample guard), no
trigger at exact 45% boundary.
formatAbReport (3): zero-state message, decisive-trials breakdown,
net_negative warning block.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.7
parent
7eddb4b1d9
commit
c45b4329f7
@@ -197,11 +197,17 @@ export async function runCalibration(
|
||||
}
|
||||
|
||||
if (opts.abReport) {
|
||||
// D19 A/B harness report wired in Lane D (T18). Placeholder.
|
||||
process.stderr.write(
|
||||
`[calibration] ab-report: implementation lands in Lane D (T18 — A/B harness for think).\n`,
|
||||
);
|
||||
process.exit(2);
|
||||
// T18 / D19 — A/B harness report.
|
||||
const { buildAbReport, formatAbReport } = await import('../core/calibration/think-ab.ts');
|
||||
const daysArg = args[args.indexOf('--days') + 1];
|
||||
const days = daysArg ? Math.max(1, parseInt(daysArg, 10) || 30) : 30;
|
||||
const report = await buildAbReport(engine, { days });
|
||||
if (opts.json) {
|
||||
process.stdout.write(JSON.stringify(report, null, 2) + '\n');
|
||||
} else {
|
||||
process.stdout.write(formatAbReport(report, days) + '\n');
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (opts.regenerate) {
|
||||
|
||||
@@ -0,0 +1,170 @@
|
||||
/**
|
||||
* v0.36.0.0 (T18 / D19) — A/B harness for `gbrain think`.
|
||||
*
|
||||
* Each invocation runs think TWICE on the same question: once baseline,
|
||||
* once --with-calibration. Both answers are written to the database in
|
||||
* a single think_ab_results row along with the user's preference. After
|
||||
* 30 days of data, `gbrain calibration ab-report` aggregates win/loss
|
||||
* and surfaces calibration_net_negative when the with-calibration variant
|
||||
* loses >55% of trials (n >= 20).
|
||||
*
|
||||
* The harness is the structural answer to CDX-18 (anti-bias rewrite may
|
||||
* make advice worse): we don't have to guess whether calibration helps;
|
||||
* we measure.
|
||||
*
|
||||
* v0.36.0.0 ship state:
|
||||
* - The data PIPELINE is real (schema, write, aggregate).
|
||||
* - The user-facing PROMPT ("which did you prefer?") is interactive on
|
||||
* the CLI. Tests inject a non-interactive answer resolver so the
|
||||
* pipeline runs hermetically.
|
||||
* - The runThink calls are real engine calls; tests can stub by
|
||||
* passing thinkRunner injection.
|
||||
*/
|
||||
|
||||
import type { BrainEngine } from '../engine.ts';
|
||||
|
||||
export interface ABRunInput {
|
||||
question: string;
|
||||
/** Holder context for calibration. Default 'garry'. */
|
||||
holder?: string;
|
||||
/** Engine for DB write. */
|
||||
engine: BrainEngine;
|
||||
/** Source for the row. */
|
||||
sourceId: string;
|
||||
/** Inject for tests; production runs the real `runThink` twice. */
|
||||
thinkRunner?: (opts: { question: string; withCalibration: boolean }) => Promise<{ answer: string; modelUsed?: string }>;
|
||||
/** Inject for tests; production prompts the user via stdin. */
|
||||
preferenceResolver?: (opts: { baseline: string; withCalibration: string }) => Promise<'baseline' | 'with_calibration' | 'neither' | 'tie'>;
|
||||
/** Optional notes from the user. */
|
||||
notes?: string;
|
||||
}
|
||||
|
||||
export interface ABRunResult {
|
||||
baselineAnswer: string;
|
||||
withCalibrationAnswer: string;
|
||||
preferred: 'baseline' | 'with_calibration' | 'neither' | 'tie';
|
||||
modelUsed?: string | undefined;
|
||||
rowId?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Run one A/B trial. Calls thinkRunner twice (or stubs), gets the
|
||||
* preference, writes the row.
|
||||
*/
|
||||
export async function runAbTrial(input: ABRunInput): Promise<ABRunResult> {
|
||||
if (!input.thinkRunner) {
|
||||
throw new Error('runAbTrial: thinkRunner not provided (production wiring lives in src/commands/think.ts)');
|
||||
}
|
||||
if (!input.preferenceResolver) {
|
||||
throw new Error('runAbTrial: preferenceResolver not provided');
|
||||
}
|
||||
|
||||
const baseline = await input.thinkRunner({ question: input.question, withCalibration: false });
|
||||
const withCal = await input.thinkRunner({ question: input.question, withCalibration: true });
|
||||
const preferred = await input.preferenceResolver({
|
||||
baseline: baseline.answer,
|
||||
withCalibration: withCal.answer,
|
||||
});
|
||||
const rows = await input.engine.executeRaw<{ id: number }>(
|
||||
`INSERT INTO think_ab_results
|
||||
(source_id, question, baseline_answer, with_calibration_answer, preferred, model_id, notes)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7)
|
||||
RETURNING id`,
|
||||
[
|
||||
input.sourceId,
|
||||
input.question,
|
||||
baseline.answer,
|
||||
withCal.answer,
|
||||
preferred,
|
||||
baseline.modelUsed ?? withCal.modelUsed ?? null,
|
||||
input.notes ?? null,
|
||||
],
|
||||
);
|
||||
return {
|
||||
baselineAnswer: baseline.answer,
|
||||
withCalibrationAnswer: withCal.answer,
|
||||
preferred,
|
||||
modelUsed: baseline.modelUsed ?? withCal.modelUsed,
|
||||
...(rows[0]?.id !== undefined ? { rowId: rows[0]!.id } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
export interface AbReportResult {
|
||||
total_trials: number;
|
||||
baseline_wins: number;
|
||||
with_calibration_wins: number;
|
||||
ties: number;
|
||||
neither: number;
|
||||
/** Win rate for --with-calibration as a fraction of decisive trials (excludes neither/tie). */
|
||||
with_calibration_win_rate: number | null;
|
||||
/** When true, the doctor surface flags calibration_net_negative. */
|
||||
net_negative: boolean;
|
||||
/** Threshold tests applied. */
|
||||
decisive_trials: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Read the table + compute the win/loss breakdown over the last N days.
|
||||
* Pure aggregation over the row set.
|
||||
*/
|
||||
export async function buildAbReport(
|
||||
engine: BrainEngine,
|
||||
opts: { days?: number } = {},
|
||||
): Promise<AbReportResult> {
|
||||
const days = opts.days ?? 30;
|
||||
const rows = await engine.executeRaw<{ preferred: string; count: number }>(
|
||||
`SELECT preferred, COUNT(*)::int AS count
|
||||
FROM think_ab_results
|
||||
WHERE ran_at >= now() - INTERVAL '${days} days'
|
||||
GROUP BY preferred`,
|
||||
);
|
||||
const counts = { baseline: 0, with_calibration: 0, tie: 0, neither: 0 };
|
||||
for (const r of rows) {
|
||||
if (r.preferred === 'baseline') counts.baseline = r.count;
|
||||
else if (r.preferred === 'with_calibration') counts.with_calibration = r.count;
|
||||
else if (r.preferred === 'tie') counts.tie = r.count;
|
||||
else if (r.preferred === 'neither') counts.neither = r.count;
|
||||
}
|
||||
const total = counts.baseline + counts.with_calibration + counts.tie + counts.neither;
|
||||
const decisive = counts.baseline + counts.with_calibration;
|
||||
const winRate = decisive > 0 ? counts.with_calibration / decisive : null;
|
||||
// calibration_net_negative threshold (D19): with-calibration loses
|
||||
// >55% of decisive trials over a sample of n >= 20.
|
||||
const netNegative = decisive >= 20 && winRate !== null && winRate < 0.45;
|
||||
return {
|
||||
total_trials: total,
|
||||
baseline_wins: counts.baseline,
|
||||
with_calibration_wins: counts.with_calibration,
|
||||
ties: counts.tie,
|
||||
neither: counts.neither,
|
||||
with_calibration_win_rate: winRate,
|
||||
net_negative: netNegative,
|
||||
decisive_trials: decisive,
|
||||
};
|
||||
}
|
||||
|
||||
/** Human-format the report. */
|
||||
export function formatAbReport(report: AbReportResult, days: number): string {
|
||||
const lines: string[] = [];
|
||||
lines.push(`A/B report (last ${days} days):`);
|
||||
lines.push(` Total trials: ${report.total_trials}`);
|
||||
if (report.total_trials === 0) {
|
||||
lines.push(' No data yet. Try: gbrain think --ab "<question>"');
|
||||
return lines.join('\n');
|
||||
}
|
||||
lines.push(` Baseline wins: ${report.baseline_wins}`);
|
||||
lines.push(` With-calibration wins: ${report.with_calibration_wins}`);
|
||||
lines.push(` Ties: ${report.ties}`);
|
||||
lines.push(` Neither: ${report.neither}`);
|
||||
if (report.with_calibration_win_rate !== null) {
|
||||
const pct = (report.with_calibration_win_rate * 100).toFixed(1);
|
||||
lines.push(` With-calibration win rate (decisive trials only): ${pct}% (n=${report.decisive_trials})`);
|
||||
}
|
||||
if (report.net_negative) {
|
||||
lines.push('');
|
||||
lines.push('⚠ calibration_net_negative: with-calibration is losing more than half of decisive trials.');
|
||||
lines.push(' Consider tuning the anti-bias prompt rewrite (src/core/think/prompt.ts) or');
|
||||
lines.push(' disabling --with-calibration via config until you tune.');
|
||||
}
|
||||
return lines.join('\n');
|
||||
}
|
||||
@@ -3452,6 +3452,36 @@ export const MIGRATIONS: Migration[] = [
|
||||
},
|
||||
transaction: false,
|
||||
},
|
||||
{
|
||||
version: 72,
|
||||
name: 'think_ab_results_v0_36',
|
||||
// v0.36.0.0 (T18 / D19) — A/B harness data for `gbrain think --ab`.
|
||||
//
|
||||
// Each row records one side-by-side comparison of think with vs.
|
||||
// without --with-calibration. After 30 days of data, `gbrain
|
||||
// calibration ab-report` aggregates win/loss across the table and
|
||||
// surfaces a calibration_net_negative doctor warning if the
|
||||
// with-calibration variant loses >55% of trials (n >= 20).
|
||||
//
|
||||
// wave_version stamped so --undo-wave can scrub these too if needed.
|
||||
idempotent: true,
|
||||
sql: `
|
||||
CREATE TABLE IF NOT EXISTS think_ab_results (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
source_id TEXT NOT NULL REFERENCES sources(id) ON DELETE CASCADE,
|
||||
wave_version TEXT NOT NULL DEFAULT 'v0.36.0.0',
|
||||
ran_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
question TEXT NOT NULL,
|
||||
baseline_answer TEXT NOT NULL,
|
||||
with_calibration_answer TEXT NOT NULL,
|
||||
preferred TEXT NOT NULL CHECK (preferred IN ('baseline','with_calibration','neither','tie')),
|
||||
model_id TEXT,
|
||||
notes TEXT
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS think_ab_results_recent_idx
|
||||
ON think_ab_results (source_id, ran_at DESC);
|
||||
`,
|
||||
},
|
||||
];
|
||||
|
||||
export const LATEST_VERSION = MIGRATIONS.length > 0
|
||||
|
||||
@@ -651,6 +651,22 @@ CREATE INDEX IF NOT EXISTS take_nudge_log_proposal_cooldown_idx
|
||||
CREATE INDEX IF NOT EXISTS take_nudge_log_wave_idx
|
||||
ON take_nudge_log (wave_version, fired_at DESC);
|
||||
|
||||
-- think_ab_results (v0.36.0.0 T18 / D19): A/B harness data.
|
||||
CREATE TABLE IF NOT EXISTS think_ab_results (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
source_id TEXT NOT NULL REFERENCES sources(id) ON DELETE CASCADE,
|
||||
wave_version TEXT NOT NULL DEFAULT 'v0.36.0.0',
|
||||
ran_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
question TEXT NOT NULL,
|
||||
baseline_answer TEXT NOT NULL,
|
||||
with_calibration_answer TEXT NOT NULL,
|
||||
preferred TEXT NOT NULL CHECK (preferred IN ('baseline','with_calibration','neither','tie')),
|
||||
model_id TEXT,
|
||||
notes TEXT
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS think_ab_results_recent_idx
|
||||
ON think_ab_results (source_id, ran_at DESC);
|
||||
|
||||
-- ============================================================
|
||||
-- access_tokens: legacy bearer tokens for remote MCP access
|
||||
-- ============================================================
|
||||
|
||||
@@ -988,6 +988,23 @@ CREATE INDEX IF NOT EXISTS take_nudge_log_proposal_cooldown_idx
|
||||
CREATE INDEX IF NOT EXISTS take_nudge_log_wave_idx
|
||||
ON take_nudge_log (wave_version, fired_at DESC);
|
||||
|
||||
-- think_ab_results (v0.36.0.0 T18 / D19): A/B harness data for
|
||||
-- \`gbrain think --ab\`. One row per side-by-side comparison.
|
||||
CREATE TABLE IF NOT EXISTS think_ab_results (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
source_id TEXT NOT NULL REFERENCES sources(id) ON DELETE CASCADE,
|
||||
wave_version TEXT NOT NULL DEFAULT 'v0.36.0.0',
|
||||
ran_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
question TEXT NOT NULL,
|
||||
baseline_answer TEXT NOT NULL,
|
||||
with_calibration_answer TEXT NOT NULL,
|
||||
preferred TEXT NOT NULL CHECK (preferred IN ('baseline','with_calibration','neither','tie')),
|
||||
model_id TEXT,
|
||||
notes TEXT
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS think_ab_results_recent_idx
|
||||
ON think_ab_results (source_id, ran_at DESC);
|
||||
|
||||
-- NOTIFY trigger for real-time job events (Postgres only, not PGLite)
|
||||
CREATE OR REPLACE FUNCTION notify_minion_job_change() RETURNS trigger AS \$\$
|
||||
BEGIN
|
||||
|
||||
@@ -984,6 +984,23 @@ CREATE INDEX IF NOT EXISTS take_nudge_log_proposal_cooldown_idx
|
||||
CREATE INDEX IF NOT EXISTS take_nudge_log_wave_idx
|
||||
ON take_nudge_log (wave_version, fired_at DESC);
|
||||
|
||||
-- think_ab_results (v0.36.0.0 T18 / D19): A/B harness data for
|
||||
-- `gbrain think --ab`. One row per side-by-side comparison.
|
||||
CREATE TABLE IF NOT EXISTS think_ab_results (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
source_id TEXT NOT NULL REFERENCES sources(id) ON DELETE CASCADE,
|
||||
wave_version TEXT NOT NULL DEFAULT 'v0.36.0.0',
|
||||
ran_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
question TEXT NOT NULL,
|
||||
baseline_answer TEXT NOT NULL,
|
||||
with_calibration_answer TEXT NOT NULL,
|
||||
preferred TEXT NOT NULL CHECK (preferred IN ('baseline','with_calibration','neither','tie')),
|
||||
model_id TEXT,
|
||||
notes TEXT
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS think_ab_results_recent_idx
|
||||
ON think_ab_results (source_id, ran_at DESC);
|
||||
|
||||
-- NOTIFY trigger for real-time job events (Postgres only, not PGLite)
|
||||
CREATE OR REPLACE FUNCTION notify_minion_job_change() RETURNS trigger AS $$
|
||||
BEGIN
|
||||
|
||||
@@ -0,0 +1,252 @@
|
||||
/**
|
||||
* v0.36.0.0 (T18 / D19) — A/B harness tests.
|
||||
*
|
||||
* Hermetic. Mock engine + injected thinkRunner + injected preferenceResolver.
|
||||
* No real LLM, no DB.
|
||||
*
|
||||
* Tests cover:
|
||||
* - runAbTrial: calls thinkRunner TWICE (baseline + with-calibration)
|
||||
* - row INSERT params match the supplied data
|
||||
* - preferenceResolver receives both answers
|
||||
* - buildAbReport: aggregates counts by preferred value
|
||||
* - calibration_net_negative trigger: n >= 20, win rate < 45%
|
||||
* - calibration_net_negative does NOT trigger when n < 20 (small-sample guard)
|
||||
* - formatAbReport: zero-trials branch, decisive-trials breakdown
|
||||
*/
|
||||
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import {
|
||||
runAbTrial,
|
||||
buildAbReport,
|
||||
formatAbReport,
|
||||
} from '../src/core/calibration/think-ab.ts';
|
||||
import type { BrainEngine } from '../src/core/engine.ts';
|
||||
|
||||
interface SqlCall { sql: string; params: unknown[] }
|
||||
|
||||
function buildMockEngine(opts: {
|
||||
insertReturning?: { id: number };
|
||||
reportRows?: Array<{ preferred: string; count: number }>;
|
||||
}): { engine: BrainEngine; sqls: SqlCall[] } {
|
||||
const sqls: SqlCall[] = [];
|
||||
const engine = {
|
||||
kind: 'pglite',
|
||||
async executeRaw<T>(sql: string, params?: unknown[]): Promise<T[]> {
|
||||
sqls.push({ sql, params: params ?? [] });
|
||||
if (sql.includes('INSERT INTO think_ab_results')) {
|
||||
return [opts.insertReturning ?? { id: 1 }] as unknown as T[];
|
||||
}
|
||||
if (sql.includes('FROM think_ab_results')) {
|
||||
return (opts.reportRows ?? []) as unknown as T[];
|
||||
}
|
||||
return [];
|
||||
},
|
||||
} as unknown as BrainEngine;
|
||||
return { engine, sqls };
|
||||
}
|
||||
|
||||
// ─── runAbTrial ─────────────────────────────────────────────────────
|
||||
|
||||
describe('runAbTrial', () => {
|
||||
test('calls thinkRunner TWICE (baseline + with-calibration)', async () => {
|
||||
const { engine } = buildMockEngine({ insertReturning: { id: 42 } });
|
||||
let calls = 0;
|
||||
let withCalibrationCalls = 0;
|
||||
const thinkRunner = async (opts: { question: string; withCalibration: boolean }) => {
|
||||
calls++;
|
||||
if (opts.withCalibration) withCalibrationCalls++;
|
||||
return { answer: `answer ${calls}`, modelUsed: 'claude-sonnet-4-6' };
|
||||
};
|
||||
const preferenceResolver = async () => 'with_calibration' as const;
|
||||
const result = await runAbTrial({
|
||||
question: 'should we hire fast in NY?',
|
||||
engine,
|
||||
sourceId: 'default',
|
||||
thinkRunner,
|
||||
preferenceResolver,
|
||||
});
|
||||
expect(calls).toBe(2);
|
||||
expect(withCalibrationCalls).toBe(1);
|
||||
expect(result.preferred).toBe('with_calibration');
|
||||
expect(result.rowId).toBe(42);
|
||||
});
|
||||
|
||||
test('preferenceResolver receives both answers as opts', async () => {
|
||||
const { engine } = buildMockEngine({});
|
||||
let received: { baseline: string; withCalibration: string } | undefined;
|
||||
const thinkRunner = async (opts: { withCalibration: boolean }) => ({
|
||||
answer: opts.withCalibration ? 'CAL_ANS' : 'BASE_ANS',
|
||||
});
|
||||
const preferenceResolver = async (input: { baseline: string; withCalibration: string }) => {
|
||||
received = input;
|
||||
return 'tie' as const;
|
||||
};
|
||||
await runAbTrial({
|
||||
question: 'q',
|
||||
engine,
|
||||
sourceId: 'default',
|
||||
thinkRunner,
|
||||
preferenceResolver,
|
||||
});
|
||||
expect(received).toEqual({ baseline: 'BASE_ANS', withCalibration: 'CAL_ANS' });
|
||||
});
|
||||
|
||||
test('INSERT row carries question + both answers + preferred', async () => {
|
||||
const { engine, sqls } = buildMockEngine({});
|
||||
const thinkRunner = async (opts: { withCalibration: boolean }) => ({
|
||||
answer: opts.withCalibration ? 'with' : 'base',
|
||||
});
|
||||
const preferenceResolver = async () => 'baseline' as const;
|
||||
await runAbTrial({
|
||||
question: 'q1',
|
||||
engine,
|
||||
sourceId: 'tenant-a',
|
||||
thinkRunner,
|
||||
preferenceResolver,
|
||||
notes: 'first trial',
|
||||
});
|
||||
const insert = sqls.find(s => s.sql.includes('INSERT INTO think_ab_results'));
|
||||
expect(insert).toBeDefined();
|
||||
expect(insert!.params[0]).toBe('tenant-a');
|
||||
expect(insert!.params[1]).toBe('q1');
|
||||
expect(insert!.params[2]).toBe('base');
|
||||
expect(insert!.params[3]).toBe('with');
|
||||
expect(insert!.params[4]).toBe('baseline');
|
||||
expect(insert!.params[6]).toBe('first trial');
|
||||
});
|
||||
|
||||
test('throws when thinkRunner not provided', async () => {
|
||||
const { engine } = buildMockEngine({});
|
||||
try {
|
||||
await runAbTrial({
|
||||
question: 'q',
|
||||
engine,
|
||||
sourceId: 'default',
|
||||
preferenceResolver: async () => 'tie' as const,
|
||||
});
|
||||
throw new Error('should have thrown');
|
||||
} catch (err) {
|
||||
expect((err as Error).message).toContain('thinkRunner');
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// ─── buildAbReport ──────────────────────────────────────────────────
|
||||
|
||||
describe('buildAbReport', () => {
|
||||
test('zero trials → all counts 0, win rate null', async () => {
|
||||
const { engine } = buildMockEngine({ reportRows: [] });
|
||||
const report = await buildAbReport(engine);
|
||||
expect(report.total_trials).toBe(0);
|
||||
expect(report.baseline_wins).toBe(0);
|
||||
expect(report.with_calibration_wins).toBe(0);
|
||||
expect(report.with_calibration_win_rate).toBeNull();
|
||||
expect(report.net_negative).toBe(false);
|
||||
});
|
||||
|
||||
test('aggregates counts by preferred value', async () => {
|
||||
const { engine } = buildMockEngine({
|
||||
reportRows: [
|
||||
{ preferred: 'baseline', count: 6 },
|
||||
{ preferred: 'with_calibration', count: 10 },
|
||||
{ preferred: 'tie', count: 2 },
|
||||
{ preferred: 'neither', count: 1 },
|
||||
],
|
||||
});
|
||||
const report = await buildAbReport(engine);
|
||||
expect(report.total_trials).toBe(19);
|
||||
expect(report.baseline_wins).toBe(6);
|
||||
expect(report.with_calibration_wins).toBe(10);
|
||||
expect(report.ties).toBe(2);
|
||||
expect(report.neither).toBe(1);
|
||||
// win rate = with_calibration / decisive = 10 / (6+10) = 0.625
|
||||
expect(report.with_calibration_win_rate).toBeCloseTo(0.625, 5);
|
||||
});
|
||||
|
||||
test('calibration_net_negative trigger: n >= 20 + win rate < 45%', async () => {
|
||||
const { engine } = buildMockEngine({
|
||||
reportRows: [
|
||||
{ preferred: 'baseline', count: 14 },
|
||||
{ preferred: 'with_calibration', count: 8 },
|
||||
],
|
||||
});
|
||||
const report = await buildAbReport(engine);
|
||||
expect(report.decisive_trials).toBe(22);
|
||||
// win rate = 8/22 ≈ 0.36
|
||||
expect(report.net_negative).toBe(true);
|
||||
});
|
||||
|
||||
test('calibration_net_negative does NOT trigger when n < 20 (small-sample guard)', async () => {
|
||||
const { engine } = buildMockEngine({
|
||||
reportRows: [
|
||||
{ preferred: 'baseline', count: 9 },
|
||||
{ preferred: 'with_calibration', count: 3 },
|
||||
],
|
||||
});
|
||||
const report = await buildAbReport(engine);
|
||||
expect(report.decisive_trials).toBe(12);
|
||||
expect(report.net_negative).toBe(false);
|
||||
});
|
||||
|
||||
test('calibration_net_negative does NOT trigger at exactly 45% win rate', async () => {
|
||||
const { engine } = buildMockEngine({
|
||||
reportRows: [
|
||||
{ preferred: 'baseline', count: 11 },
|
||||
{ preferred: 'with_calibration', count: 9 },
|
||||
],
|
||||
});
|
||||
const report = await buildAbReport(engine);
|
||||
// win rate = 9/20 = 0.45 — boundary; NOT < 0.45
|
||||
expect(report.net_negative).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
// ─── formatAbReport ─────────────────────────────────────────────────
|
||||
|
||||
describe('formatAbReport', () => {
|
||||
test('zero trials → friendly empty-state message', () => {
|
||||
const out = formatAbReport({
|
||||
total_trials: 0,
|
||||
baseline_wins: 0,
|
||||
with_calibration_wins: 0,
|
||||
ties: 0,
|
||||
neither: 0,
|
||||
with_calibration_win_rate: null,
|
||||
net_negative: false,
|
||||
decisive_trials: 0,
|
||||
}, 30);
|
||||
expect(out).toContain('No data yet');
|
||||
expect(out).toContain('gbrain think --ab');
|
||||
});
|
||||
|
||||
test('decisive-trials breakdown', () => {
|
||||
const out = formatAbReport({
|
||||
total_trials: 22,
|
||||
baseline_wins: 8,
|
||||
with_calibration_wins: 12,
|
||||
ties: 1,
|
||||
neither: 1,
|
||||
with_calibration_win_rate: 0.6,
|
||||
net_negative: false,
|
||||
decisive_trials: 20,
|
||||
}, 30);
|
||||
expect(out).toContain('Total trials: 22');
|
||||
expect(out).toContain('Baseline wins:');
|
||||
expect(out).toContain('60.0%');
|
||||
expect(out).toContain('n=20');
|
||||
});
|
||||
|
||||
test('net_negative true → calibration_net_negative warning block', () => {
|
||||
const out = formatAbReport({
|
||||
total_trials: 22,
|
||||
baseline_wins: 14,
|
||||
with_calibration_wins: 8,
|
||||
ties: 0,
|
||||
neither: 0,
|
||||
with_calibration_win_rate: 0.36,
|
||||
net_negative: true,
|
||||
decisive_trials: 22,
|
||||
}, 30);
|
||||
expect(out).toContain('calibration_net_negative');
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user