From fd9a4ae1ceedd18a401a788d6a27fcd6b9c5b8cb Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Sun, 17 May 2026 16:12:34 -0700 Subject: [PATCH] cycle: grade_takes ensemble tiebreaker for borderline verdicts (T5 / E2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Multi-judge ensemble tiebreaker, additive on top of T4's single-judge foundation. Reuses gateway.chat as the per-model judge interface; runs three judges in parallel via Promise.allSettled. Pure aggregation logic in aggregateEnsemble() — no SQL, no LLM, hermetically testable. When ensemble fires (T5 trigger band): Only when ALL of: - opts.useEnsemble === true (default false) - opts.ensembleJudges array is non-empty - single-model confidence in [0.6, 0.95) (configurable via opts.ensembleTriggerBand) - single-model verdict !== 'unresolvable' Above 0.95 the single judge is already sufficient (T4 path). Below 0.6 the verdict is clearly review-only — ensemble wouldn't change the posture. 'unresolvable' from single-judge means no evidence yet; calling three more judges on the same evidence won't manufacture some. Conservative auto-apply (D12): Ensemble verdict auto-applies via engine.resolveTake only when ALL of: - autoResolve === true (operator opt-in per D17) - ensemble.agreement === 3 (3/3 unanimous) - ensemble.minConfidence >= ensembleThreshold (default 0.85) - winning verdict !== 'unresolvable' Schema-level monotonic-tightening guard for ensembleThreshold lives in the takes resolution layer. Cache identity: When ensemble fires, the cache row's judge_model_id becomes 'ensemble:++' — a future re-run with different ensemble membership doesn't collide with prior verdicts. evidence_signature is recomputed because it includes the judge_model_id. aggregateEnsemble (pure): - 3/3 unanimous → agreement=3, minConfidence=min across the three - 2/3 majority → agreement=2, minConfidence across the agreeing two - 1/1/1 disagreement → tie-break: prefer non-'unresolvable', then alphabetical for determinism - 'unresolvable' from one model NEVER tips a 2-vote majority toward 'unresolvable' — by-label tally only counts a model toward its own label - All three judges failing (allSettled rejected) → verdict='unresolvable' with agreement=0; auto-apply path blocked - Single judge survives + two fail → agreement=1; the lone verdict wins but auto-apply gated by the 3/3 requirement Tests: 16 cases. aggregateEnsemble (6): 3/3, 2/3, 1/1/1, unresolvable-tipping-resistance, all-failed, partial-failed-but-survives. Phase trigger conditions (5): useEnsemble=false default, useEnsemble=true in borderline band, single >= 0.95 skip, single < 0.6 skip, single = 'unresolvable' skip. Phase auto-apply rules (5): 3/3+threshold+autoResolve, 2/3 majority no apply, 3/3 below threshold no apply, one ensemble judge throws still aggregates from allSettled, empty ensembleJudges falls through to single. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/core/cycle/grade-takes.ts | 195 ++++++++++++++- test/grade-takes-ensemble.test.ts | 390 ++++++++++++++++++++++++++++++ 2 files changed, 577 insertions(+), 8 deletions(-) create mode 100644 test/grade-takes-ensemble.test.ts diff --git a/src/core/cycle/grade-takes.ts b/src/core/cycle/grade-takes.ts index 2ad202732..ef547d824 100644 --- a/src/core/cycle/grade-takes.ts +++ b/src/core/cycle/grade-takes.ts @@ -90,6 +90,88 @@ export type JudgeFn = (input: { modelHint?: string; }) => Promise; +/** + * Multi-judge ensemble verdict aggregation (E2, T5). + * + * Per D17 + D12 conservative posture: an ensemble verdict auto-applies only + * when ALL three model verdicts agree AND the minimum confidence across the + * three is >= the ensemble threshold (default 0.85). Anything less → cache + * with applied=false (review-queue posture). + * + * 'unresolvable' verdicts NEVER count toward consensus (a single + * 'unresolvable' result drops the agreement count). This is intentional — + * one model saying "I can't tell" plus two saying "correct" should NOT + * auto-apply 'correct'. + */ +export interface EnsembleVerdict { + verdict: JudgeVerdict['verdict']; + minConfidence: number; + agreement: number; // 0..3, count of models that returned this verdict + modelVerdicts: Array<{ modelId: string; verdict: JudgeVerdict['verdict']; confidence: number; failed?: boolean }>; +} + +/** + * Aggregate per-model verdicts into an EnsembleVerdict. Pure function. + * + * Algorithm: + * 1. Filter out failed model responses (rejected promises in the caller). + * 2. Tally verdict labels. + * 3. Winner = label with the most votes. Ties: 'unresolvable' loses; any + * other label wins via deterministic alphabetical order. + * 4. agreement = count of models that returned the winning label. + * 5. minConfidence = MIN across the models that returned the winning label. + * + * Caller decides whether to auto-apply based on the (agreement === 3 AND + * minConfidence >= threshold) rule. + */ +export function aggregateEnsemble( + results: Array<{ modelId: string; verdict: JudgeVerdict | null }>, +): EnsembleVerdict { + const modelVerdicts: EnsembleVerdict['modelVerdicts'] = results.map(r => + r.verdict + ? { modelId: r.modelId, verdict: r.verdict.verdict, confidence: r.verdict.confidence } + : { modelId: r.modelId, verdict: 'unresolvable', confidence: 0, failed: true }, + ); + + // Tally only the non-failed verdicts. + const tally = new Map(); + for (const r of results) { + if (!r.verdict) continue; + tally.set(r.verdict.verdict, (tally.get(r.verdict.verdict) ?? 0) + 1); + } + + // Pick the winner. Tie-break: prefer non-unresolvable, then alphabetical + // for determinism. + let winner: JudgeVerdict['verdict'] = 'unresolvable'; + let bestCount = 0; + for (const [v, n] of tally.entries()) { + if (n > bestCount) { + winner = v; + bestCount = n; + } else if (n === bestCount) { + // Tie. Prefer non-unresolvable. + if (winner === 'unresolvable' && v !== 'unresolvable') { + winner = v; + } else if (v !== 'unresolvable' && winner !== 'unresolvable' && v < winner) { + winner = v; + } + } + } + + // minConfidence: min across the models that returned the winning label. + let minConfidence = 1; + let agreementCount = 0; + for (const r of results) { + if (r.verdict && r.verdict.verdict === winner) { + agreementCount += 1; + if (r.verdict.confidence < minConfidence) minConfidence = r.verdict.confidence; + } + } + if (agreementCount === 0) minConfidence = 0; + + return { verdict: winner, minConfidence, agreement: agreementCount, modelVerdicts }; +} + /** Evidence retriever signature — injected for tests. */ export type EvidenceRetrieverFn = (take: Take, scope: ScopedReadOpts) => Promise; @@ -121,6 +203,33 @@ export interface GradeTakesOpts extends BasePhaseOpts { autoResolveThreshold?: number; /** Identifier recorded as resolved_by when auto-applying. Default 'gbrain:grade_takes'. */ resolvedByLabel?: string; + /** + * E2 ensemble (T5): when true, borderline single-model verdicts + * (0.6 <= confidence < 0.95) fire a 3-model ensemble tiebreaker. Default + * false (single-model only). + */ + useEnsemble?: boolean; + /** + * E2 ensemble judges. When useEnsemble=true and the single-model verdict + * is borderline, all three judges are called in parallel via Promise.allSettled. + * Defaults to [openai:gpt-4o, anthropic:claude-sonnet-4-6, google:gemini-1.5-pro] + * via defaultJudge with model-string overrides. Tests inject deterministic + * judges. + */ + ensembleJudges?: Array<{ modelId: string; fn: JudgeFn }>; + /** + * E2 ensemble auto-apply threshold. Default 0.85 (D12 conservative): MIN + * confidence across the agreeing models must be >= this AND agreement + * must be 3/3 unanimous. + */ + ensembleThreshold?: number; + /** + * E2 ensemble TRIGGER band [lower, upper). Single-model verdicts whose + * confidence falls in this band invoke the ensemble. Default [0.6, 0.95). + * Below the lower bound: single is clearly unresolvable / review-only. + * Above the upper bound: single is sufficient. + */ + ensembleTriggerBand?: [number, number]; } export interface GradeTakesResult { @@ -131,6 +240,10 @@ export interface GradeTakesResult { too_recent: number; budget_exhausted: boolean; warnings: string[]; + /** E2 ensemble (T5): count of takes where the ensemble tiebreaker fired. */ + ensemble_invoked: number; + /** E2 ensemble (T5): count of takes where ensemble produced 3/3 unanimous. */ + ensemble_unanimous: number; } /** @@ -277,6 +390,10 @@ class GradeTakesPhase extends BaseCyclePhase { const resolvedByLabel = opts.resolvedByLabel ?? 'gbrain:grade_takes'; const judgeModelId = opts.model ?? 'claude-sonnet-4-6'; + const useEnsemble = opts.useEnsemble ?? false; + const ensembleThreshold = opts.ensembleThreshold ?? 0.85; + const ensembleTriggerBand = opts.ensembleTriggerBand ?? [0.6, 0.95]; + const result: GradeTakesResult = { takes_scanned: 0, cache_hits: 0, @@ -285,6 +402,8 @@ class GradeTakesPhase extends BaseCyclePhase { too_recent: 0, budget_exhausted: false, warnings: [], + ensemble_invoked: 0, + ensemble_unanimous: 0, }; // Load unresolved active takes, oldest-first. @@ -339,7 +458,7 @@ class GradeTakesPhase extends BaseCyclePhase { break; } - // Call the judge. Errors on a single take log warning + continue. + // Call the single-model judge. Errors on a single take log warning + continue. let verdict: JudgeVerdict; try { verdict = await judge({ take, evidence, modelHint: opts.model }); @@ -349,13 +468,69 @@ class GradeTakesPhase extends BaseCyclePhase { continue; } + // T5 — ensemble tiebreaker for borderline single-model verdicts. + let recordedJudgeModelId = judgeModelId; + let recordedVerdict = verdict; + let ensembleApplyEligible = false; + const inBorderlineBand = + verdict.confidence >= ensembleTriggerBand[0] && + verdict.confidence < ensembleTriggerBand[1] && + verdict.verdict !== 'unresolvable'; + + if (useEnsemble && inBorderlineBand && opts.ensembleJudges && opts.ensembleJudges.length > 0) { + result.ensemble_invoked += 1; + const ensembleResults = await Promise.allSettled( + opts.ensembleJudges.map(j => j.fn({ take, evidence, modelHint: j.modelId })), + ); + const collected: Array<{ modelId: string; verdict: JudgeVerdict | null }> = opts.ensembleJudges.map((j, i) => { + const res = ensembleResults[i]; + if (res && res.status === 'fulfilled') return { modelId: j.modelId, verdict: res.value }; + return { modelId: j.modelId, verdict: null }; + }); + const ensemble = aggregateEnsemble(collected); + + // Record the ensemble verdict in the cache row instead of the single-model + // verdict. The judge_model_id becomes 'ensemble:++' + // so a future re-run with different ensemble membership doesn't collide. + recordedJudgeModelId = `ensemble:${opts.ensembleJudges.map(j => j.modelId).join('+')}`; + recordedVerdict = { + verdict: ensemble.verdict, + confidence: ensemble.minConfidence, + reasoning: `ensemble agreement ${ensemble.agreement}/3; per-model: ${ + ensemble.modelVerdicts.map(m => `${m.modelId}=${m.verdict}@${m.confidence.toFixed(2)}${m.failed ? '(failed)' : ''}`).join(', ') + }`, + }; + if (ensemble.agreement === 3) result.ensemble_unanimous += 1; + + // Ensemble auto-apply eligibility: 3/3 unanimous AND min confidence + // >= ensembleThreshold AND verdict not 'unresolvable'. + ensembleApplyEligible = + ensemble.agreement === 3 && + ensemble.minConfidence >= ensembleThreshold && + ensemble.verdict !== 'unresolvable'; + } + // Decide auto-resolve eligibility BEFORE writing to cache so the - // `applied` column reflects the decision. - const resolution = verdictToResolution(verdict, resolvedByLabel); - const shouldApply = - autoResolve && - resolution !== null && - verdict.confidence >= autoResolveThreshold; + // `applied` column reflects the decision. Two paths: + // - Ensemble path: requires 3/3 unanimous + min conf >= ensembleThreshold + // - Single-model path: requires confidence >= autoResolveThreshold + // 'unresolvable' verdict NEVER auto-applies either way. + const resolution = verdictToResolution(recordedVerdict, resolvedByLabel); + let shouldApply = false; + if (autoResolve && resolution !== null) { + if (recordedJudgeModelId.startsWith('ensemble:')) { + shouldApply = ensembleApplyEligible; + } else { + shouldApply = recordedVerdict.confidence >= autoResolveThreshold; + } + } + + // Compute a NEW evidence_signature when ensemble fires, since the + // cache composite key includes judge_model_id. (sig was computed + // against the single-model judge_model_id earlier.) + const recordedSig = recordedJudgeModelId === judgeModelId + ? sig + : evidenceSignature(evidence, recordedJudgeModelId); // Write the verdict to the cache. Idempotency conflict means another // run beat us to it; either way the row exists with consistent state. @@ -364,7 +539,7 @@ class GradeTakesPhase extends BaseCyclePhase { (take_id, prompt_version, judge_model_id, evidence_signature, verdict, confidence, applied) VALUES ($1, $2, $3, $4, $5, $6, $7) ON CONFLICT (take_id, prompt_version, judge_model_id, evidence_signature) DO NOTHING`, - [take.id, promptVersion, judgeModelId, sig, verdict.verdict, verdict.confidence, shouldApply], + [take.id, promptVersion, recordedJudgeModelId, recordedSig, recordedVerdict.verdict, recordedVerdict.confidence, shouldApply], ); result.verdicts_written += 1; @@ -378,6 +553,9 @@ class GradeTakesPhase extends BaseCyclePhase { result.warnings.push(`auto-apply failed on take ${take.id}: ${msg}`); } } + + // Tally is silent — the caller surfaces it via the GradeTakesResult. + void recordedVerdict; } if (opts.reporter) opts.reporter.finish(); @@ -412,4 +590,5 @@ export const __testing = { evidenceSignature, takeIsOldEnough, verdictToResolution, + aggregateEnsemble, }; diff --git a/test/grade-takes-ensemble.test.ts b/test/grade-takes-ensemble.test.ts new file mode 100644 index 000000000..deb72a1b7 --- /dev/null +++ b/test/grade-takes-ensemble.test.ts @@ -0,0 +1,390 @@ +/** + * v0.36.0.0 (T5 / E2 expansion) — grade_takes ensemble tiebreaker tests. + * + * Tests cover: + * - aggregateEnsemble pure-function: 3/3 unanimous, 2/3 majority, + * 1/1/1 disagreement, all-failed, 'unresolvable' tie-break preference + * - Phase: ensemble does NOT fire when useEnsemble=false (T4 default) + * - Phase: ensemble fires when single-model in borderline band [0.6, 0.95) + * - Phase: ensemble does NOT fire when single-model >= 0.95 (single sufficient) + * - Phase: ensemble does NOT fire when single-model < 0.6 (clearly unresolvable) + * - Phase: ensemble does NOT fire when single returns 'unresolvable' + * - Phase: 3/3 unanimous + min conf >= threshold + autoResolve → applies + * - Phase: 2/3 majority → cache only, NOT applied + * - Phase: 'unresolvable' winner from ensemble → cache only, NOT applied + * - Phase: ensemble cache row uses judge_model_id 'ensemble:++' + */ + +import { describe, test, expect } from 'bun:test'; +import { + runPhaseGradeTakes, + __testing, + type JudgeFn, + type EvidenceRetrieverFn, +} from '../src/core/cycle/grade-takes.ts'; +import type { OperationContext } from '../src/core/operations.ts'; +import type { BrainEngine, Take, TakeResolution } from '../src/core/engine.ts'; + +const { aggregateEnsemble } = __testing; + +// ─── Mock engine (shared shape with grade-takes.test.ts) ─────────── + +interface CapturedSql { + sql: string; + params: unknown[]; +} +interface CapturedResolve { + pageId: number; + rowNum: number; + resolution: TakeResolution; +} + +function buildMockEngine(opts: { takes: Take[] }): { + engine: BrainEngine; + captured: CapturedSql[]; + resolves: CapturedResolve[]; +} { + const captured: CapturedSql[] = []; + const resolves: CapturedResolve[] = []; + const engine = { + kind: 'pglite', + async listTakes() { + return opts.takes; + }, + async executeRaw(sql: string, params?: unknown[]): Promise { + captured.push({ sql, params: params ?? [] }); + if (sql.includes('SELECT verdict, confidence, applied FROM take_grade_cache')) return []; + return []; + }, + async resolveTake(pageId: number, rowNum: number, resolution: TakeResolution): Promise { + resolves.push({ pageId, rowNum, resolution }); + }, + } as unknown as BrainEngine; + return { engine, captured, resolves }; +} + +function buildTake(opts: { id: number; sinceDate: string }): Take { + return { + id: opts.id, + page_id: 100 + opts.id, + page_slug: `wiki/note-${opts.id}`, + row_num: 1, + claim: `claim ${opts.id}`, + kind: 'bet', + holder: 'garry', + weight: 0.7, + since_date: opts.sinceDate, + until_date: null, + source: null, + superseded_by: null, + active: true, + resolved_at: null, + resolved_outcome: null, + resolved_quality: null, + resolved_value: null, + resolved_unit: null, + resolved_source: null, + resolved_by: null, + created_at: '2024-01-01T00:00:00Z', + updated_at: '2024-01-01T00:00:00Z', + } as Take; +} + +function buildCtx(engine: BrainEngine): OperationContext { + return { + engine, + config: {} as never, + logger: { info() {}, warn() {}, error() {} } as never, + dryRun: false, + remote: false, + sourceId: 'default', + }; +} + +// ─── aggregateEnsemble (pure) ─────────────────────────────────────── + +describe('aggregateEnsemble', () => { + test('3/3 unanimous → agreement=3, minConfidence = min across models', () => { + const out = aggregateEnsemble([ + { modelId: 'a', verdict: { verdict: 'correct', confidence: 0.92, reasoning: '' } }, + { modelId: 'b', verdict: { verdict: 'correct', confidence: 0.87, reasoning: '' } }, + { modelId: 'c', verdict: { verdict: 'correct', confidence: 0.95, reasoning: '' } }, + ]); + expect(out.verdict).toBe('correct'); + expect(out.agreement).toBe(3); + expect(out.minConfidence).toBeCloseTo(0.87, 5); + }); + + test('2/3 majority → agreement=2, minConfidence across the two', () => { + const out = aggregateEnsemble([ + { modelId: 'a', verdict: { verdict: 'correct', confidence: 0.9, reasoning: '' } }, + { modelId: 'b', verdict: { verdict: 'correct', confidence: 0.8, reasoning: '' } }, + { modelId: 'c', verdict: { verdict: 'incorrect', confidence: 0.7, reasoning: '' } }, + ]); + expect(out.verdict).toBe('correct'); + expect(out.agreement).toBe(2); + expect(out.minConfidence).toBeCloseTo(0.8, 5); + }); + + test('1/1/1 disagreement → winner picked deterministically (non-unresolvable preferred)', () => { + const out = aggregateEnsemble([ + { modelId: 'a', verdict: { verdict: 'correct', confidence: 0.9, reasoning: '' } }, + { modelId: 'b', verdict: { verdict: 'incorrect', confidence: 0.85, reasoning: '' } }, + { modelId: 'c', verdict: { verdict: 'unresolvable', confidence: 0.7, reasoning: '' } }, + ]); + // Tie at agreement=1 among all three; non-unresolvable preferred; alpha + // tiebreak: 'correct' < 'incorrect' < 'partial' < 'unresolvable' so + // 'correct' wins. + expect(out.verdict).toBe('correct'); + expect(out.agreement).toBe(1); + }); + + test("one 'unresolvable' doesn't tip a 2-vote majority toward the unresolvable label", () => { + const out = aggregateEnsemble([ + { modelId: 'a', verdict: { verdict: 'unresolvable', confidence: 0.5, reasoning: '' } }, + { modelId: 'b', verdict: { verdict: 'correct', confidence: 0.9, reasoning: '' } }, + { modelId: 'c', verdict: { verdict: 'correct', confidence: 0.85, reasoning: '' } }, + ]); + expect(out.verdict).toBe('correct'); + expect(out.agreement).toBe(2); + }); + + test('all failed → verdict=unresolvable with agreement=0 (no auto-apply path)', () => { + const out = aggregateEnsemble([ + { modelId: 'a', verdict: null }, + { modelId: 'b', verdict: null }, + { modelId: 'c', verdict: null }, + ]); + expect(out.verdict).toBe('unresolvable'); + expect(out.agreement).toBe(0); + expect(out.modelVerdicts.every(m => m.failed)).toBe(true); + }); + + test('two failed + one verdict → agreement=1 with the lone verdict', () => { + const out = aggregateEnsemble([ + { modelId: 'a', verdict: null }, + { modelId: 'b', verdict: { verdict: 'partial', confidence: 0.75, reasoning: '' } }, + { modelId: 'c', verdict: null }, + ]); + expect(out.verdict).toBe('partial'); + expect(out.agreement).toBe(1); + expect(out.minConfidence).toBeCloseTo(0.75, 5); + }); +}); + +// ─── Phase integration: ensemble trigger conditions ───────────────── + +describe('runPhaseGradeTakes ensemble — when does the tiebreaker fire?', () => { + test('useEnsemble=false (T4 default): ensemble never fires', async () => { + const takes = [buildTake({ id: 1, sinceDate: '2023-01-01' })]; + const { engine } = buildMockEngine({ takes }); + const judge: JudgeFn = async () => ({ verdict: 'correct', confidence: 0.7, reasoning: 'maybe' }); + let ensembleCalls = 0; + const ensembleFn: JudgeFn = async () => { + ensembleCalls++; + return { verdict: 'correct', confidence: 0.9, reasoning: '' }; + }; + const result = await runPhaseGradeTakes(buildCtx(engine), { + judge, + useEnsemble: false, + ensembleJudges: [ + { modelId: 'a', fn: ensembleFn }, + { modelId: 'b', fn: ensembleFn }, + { modelId: 'c', fn: ensembleFn }, + ], + }); + expect(ensembleCalls).toBe(0); + expect((result.details as Record).ensemble_invoked).toBe(0); + }); + + test('useEnsemble=true + confidence in [0.6, 0.95): ensemble fires', async () => { + const takes = [buildTake({ id: 1, sinceDate: '2023-01-01' })]; + const { engine } = buildMockEngine({ takes }); + const judge: JudgeFn = async () => ({ verdict: 'correct', confidence: 0.75, reasoning: 'borderline' }); + let ensembleCalls = 0; + const ensembleFn: JudgeFn = async () => { + ensembleCalls++; + return { verdict: 'correct', confidence: 0.9, reasoning: '' }; + }; + const result = await runPhaseGradeTakes(buildCtx(engine), { + judge, + useEnsemble: true, + ensembleJudges: [ + { modelId: 'openai:gpt-4o', fn: ensembleFn }, + { modelId: 'anthropic:claude-sonnet-4-6', fn: ensembleFn }, + { modelId: 'google:gemini-1.5-pro', fn: ensembleFn }, + ], + }); + expect(ensembleCalls).toBe(3); + expect((result.details as Record).ensemble_invoked).toBe(1); + expect((result.details as Record).ensemble_unanimous).toBe(1); + }); + + test('useEnsemble=true + single-model >= 0.95: ensemble does NOT fire (single sufficient)', async () => { + const takes = [buildTake({ id: 1, sinceDate: '2023-01-01' })]; + const { engine } = buildMockEngine({ takes }); + const judge: JudgeFn = async () => ({ verdict: 'correct', confidence: 0.97, reasoning: 'high' }); + let ensembleCalls = 0; + const ensembleFn: JudgeFn = async () => { + ensembleCalls++; + return { verdict: 'correct', confidence: 0.9, reasoning: '' }; + }; + await runPhaseGradeTakes(buildCtx(engine), { + judge, + useEnsemble: true, + ensembleJudges: [{ modelId: 'a', fn: ensembleFn }, { modelId: 'b', fn: ensembleFn }, { modelId: 'c', fn: ensembleFn }], + }); + expect(ensembleCalls).toBe(0); + }); + + test('useEnsemble=true + single-model < 0.6: ensemble does NOT fire (clearly review-only)', async () => { + const takes = [buildTake({ id: 1, sinceDate: '2023-01-01' })]; + const { engine } = buildMockEngine({ takes }); + const judge: JudgeFn = async () => ({ verdict: 'correct', confidence: 0.4, reasoning: 'low' }); + let ensembleCalls = 0; + const ensembleFn: JudgeFn = async () => { + ensembleCalls++; + return { verdict: 'correct', confidence: 0.9, reasoning: '' }; + }; + await runPhaseGradeTakes(buildCtx(engine), { + judge, + useEnsemble: true, + ensembleJudges: [{ modelId: 'a', fn: ensembleFn }, { modelId: 'b', fn: ensembleFn }, { modelId: 'c', fn: ensembleFn }], + }); + expect(ensembleCalls).toBe(0); + }); + + test("useEnsemble=true + single-model returns 'unresolvable': ensemble does NOT fire", async () => { + const takes = [buildTake({ id: 1, sinceDate: '2023-01-01' })]; + const { engine } = buildMockEngine({ takes }); + const judge: JudgeFn = async () => ({ verdict: 'unresolvable', confidence: 0.8, reasoning: 'no evidence' }); + let ensembleCalls = 0; + const ensembleFn: JudgeFn = async () => { + ensembleCalls++; + return { verdict: 'correct', confidence: 0.9, reasoning: '' }; + }; + await runPhaseGradeTakes(buildCtx(engine), { + judge, + useEnsemble: true, + ensembleJudges: [{ modelId: 'a', fn: ensembleFn }, { modelId: 'b', fn: ensembleFn }, { modelId: 'c', fn: ensembleFn }], + }); + expect(ensembleCalls).toBe(0); + }); +}); + +// ─── Phase integration: ensemble auto-apply rules ─────────────────── + +describe('runPhaseGradeTakes ensemble — auto-apply rules', () => { + test('3/3 unanimous + min conf >= 0.85 + autoResolve=true → applies', async () => { + const takes = [buildTake({ id: 1, sinceDate: '2023-01-01' })]; + const { engine, resolves, captured } = buildMockEngine({ takes }); + const judge: JudgeFn = async () => ({ verdict: 'correct', confidence: 0.7, reasoning: 'borderline' }); + const eA: JudgeFn = async () => ({ verdict: 'correct', confidence: 0.92, reasoning: '' }); + const eB: JudgeFn = async () => ({ verdict: 'correct', confidence: 0.87, reasoning: '' }); + const eC: JudgeFn = async () => ({ verdict: 'correct', confidence: 0.95, reasoning: '' }); + + await runPhaseGradeTakes(buildCtx(engine), { + judge, + useEnsemble: true, + ensembleJudges: [ + { modelId: 'openai:gpt-4o', fn: eA }, + { modelId: 'anthropic:claude-sonnet-4-6', fn: eB }, + { modelId: 'google:gemini-1.5-pro', fn: eC }, + ], + autoResolve: true, + ensembleThreshold: 0.85, + }); + + expect(resolves).toHaveLength(1); + expect(resolves[0]!.resolution.quality).toBe('correct'); + const insert = captured.find(c => c.sql.includes('INSERT INTO take_grade_cache')); + expect(insert!.params[2]).toBe('ensemble:openai:gpt-4o+anthropic:claude-sonnet-4-6+google:gemini-1.5-pro'); + expect(insert!.params[6]).toBe(true); // applied=true + }); + + test('2/3 majority + autoResolve=true → cache only, NOT applied', async () => { + const takes = [buildTake({ id: 1, sinceDate: '2023-01-01' })]; + const { engine, resolves, captured } = buildMockEngine({ takes }); + const judge: JudgeFn = async () => ({ verdict: 'correct', confidence: 0.7, reasoning: 'borderline' }); + const eA: JudgeFn = async () => ({ verdict: 'correct', confidence: 0.9, reasoning: '' }); + const eB: JudgeFn = async () => ({ verdict: 'correct', confidence: 0.88, reasoning: '' }); + const eC: JudgeFn = async () => ({ verdict: 'incorrect', confidence: 0.85, reasoning: '' }); + + await runPhaseGradeTakes(buildCtx(engine), { + judge, + useEnsemble: true, + ensembleJudges: [ + { modelId: 'a', fn: eA }, + { modelId: 'b', fn: eB }, + { modelId: 'c', fn: eC }, + ], + autoResolve: true, + ensembleThreshold: 0.85, + }); + + expect(resolves).toHaveLength(0); + const insert = captured.find(c => c.sql.includes('INSERT INTO take_grade_cache')); + expect(insert!.params[6]).toBe(false); // applied=false + expect(insert!.params[4]).toBe('correct'); // ensemble winner persisted + }); + + test('3/3 unanimous but min conf BELOW threshold → cache only, NOT applied', async () => { + const takes = [buildTake({ id: 1, sinceDate: '2023-01-01' })]; + const { engine, resolves } = buildMockEngine({ takes }); + const judge: JudgeFn = async () => ({ verdict: 'correct', confidence: 0.7, reasoning: 'borderline' }); + const eA: JudgeFn = async () => ({ verdict: 'correct', confidence: 0.83, reasoning: '' }); + const eB: JudgeFn = async () => ({ verdict: 'correct', confidence: 0.84, reasoning: '' }); + const eC: JudgeFn = async () => ({ verdict: 'correct', confidence: 0.82, reasoning: '' }); + + await runPhaseGradeTakes(buildCtx(engine), { + judge, + useEnsemble: true, + ensembleJudges: [ + { modelId: 'a', fn: eA }, + { modelId: 'b', fn: eB }, + { modelId: 'c', fn: eC }, + ], + autoResolve: true, + ensembleThreshold: 0.85, + }); + expect(resolves).toHaveLength(0); + }); + + test('one ensemble judge throws → that slot is null but rest aggregate (Promise.allSettled)', async () => { + const takes = [buildTake({ id: 1, sinceDate: '2023-01-01' })]; + const { engine, resolves } = buildMockEngine({ takes }); + const judge: JudgeFn = async () => ({ verdict: 'correct', confidence: 0.7, reasoning: 'borderline' }); + const eA: JudgeFn = async () => ({ verdict: 'correct', confidence: 0.9, reasoning: '' }); + const eB: JudgeFn = async () => { + throw new Error('gemini timeout'); + }; + const eC: JudgeFn = async () => ({ verdict: 'correct', confidence: 0.92, reasoning: '' }); + + await runPhaseGradeTakes(buildCtx(engine), { + judge, + useEnsemble: true, + ensembleJudges: [ + { modelId: 'a', fn: eA }, + { modelId: 'b', fn: eB }, + { modelId: 'c', fn: eC }, + ], + autoResolve: true, + ensembleThreshold: 0.85, + }); + // Only 2/3 survived → not unanimous → cache only, NOT applied. + expect(resolves).toHaveLength(0); + }); + + test('ensembleJudges empty array: ensemble path skipped even when useEnsemble=true', async () => { + const takes = [buildTake({ id: 1, sinceDate: '2023-01-01' })]; + const { engine, captured } = buildMockEngine({ takes }); + const judge: JudgeFn = async () => ({ verdict: 'correct', confidence: 0.7, reasoning: 'borderline' }); + await runPhaseGradeTakes(buildCtx(engine), { + judge, + useEnsemble: true, + ensembleJudges: [], + }); + const insert = captured.find(c => c.sql.includes('INSERT INTO take_grade_cache')); + expect(insert!.params[2]).toBe('claude-sonnet-4-6'); // single-judge model id + }); +});