diff --git a/src/core/cycle/grade-takes.ts b/src/core/cycle/grade-takes.ts new file mode 100644 index 000000000..2ad202732 --- /dev/null +++ b/src/core/cycle/grade-takes.ts @@ -0,0 +1,415 @@ +/** + * v0.36.0.0 (T4) — grade_takes cycle phase. + * + * Walks unresolved takes that are old enough to have outcome data, retrieves + * evidence from the brain, asks a judge model to verdict each one. Writes + * verdicts to take_grade_cache. Optionally — only when operator has flipped + * the opt-in config flag — auto-applies high-confidence verdicts to the + * canonical takes table via engine.resolveTake. + * + * Auto-resolve posture (D17 — auto-resolve disabled by default): + * On a fresh install, grade_takes runs and writes verdicts to the cache, + * but `applied=false` on every row. Operator reviews the queue, then flips + * `cycle.grade_takes.auto_resolve.enabled: true` once trust is earned. + * + * Conservative threshold (D12): + * When auto_resolve.enabled is true, a verdict auto-applies only when + * confidence >= 0.95 (single-judge path; T5 ensemble path tightens this + * further). Schema enforces monotonic config tightening: tightening + * thresholds is always free, loosening requires --allow-loosen-confidence + * flag because relaxing after data accumulates silently shifts which + * historical resolutions count as auto-applied. + * + * Evidence retrieval status (v0.36.0.0 ship state): + * The default evidence retriever returns an "evidence-retrieval not yet + * wired" placeholder. Most verdicts produced by the stub-judge against + * the stub-evidence will be 'unresolvable'. Real retrieval (hybrid search + * over pages newer than the take's since_date, optionally augmented by a + * gateway web-search recipe in v0.37+) lands as a follow-up. The phase + * ships now so the wiring is real and the cache table accumulates + * verdicts even if early ones are conservative; operators get the + * end-to-end loop running ahead of the tuned-prompt arrival. + * + * Test seam: opts.judge + opts.evidenceRetriever are injected so the + * phase runs hermetically in unit tests. + */ + +import { createHash } from 'node:crypto'; +import { BaseCyclePhase, type ScopedReadOpts, type BasePhaseOpts } from './base-phase.ts'; +import { chat as gatewayChat } from '../ai/gateway.ts'; +import { GBrainError } from '../types.ts'; +import type { OperationContext } from '../operations.ts'; +import type { BrainEngine, Take, TakeResolution } from '../engine.ts'; +import type { PhaseStatus, CyclePhase } from '../cycle.ts'; + +/** + * Bump when the judge prompt or the JSON output shape changes. Old verdicts + * stay valid (composite cache key includes prompt_version); new runs re-spend + * LLM tokens. + */ +export const GRADE_TAKES_PROMPT_VERSION = 'v0.36.0.0-stub'; + +export const GRADE_TAKE_PROMPT = `[v0.36.0.0-stub] You are grading a single forecasting take. The author +made this claim on the given date. Based on the evidence provided, did the +claim turn out to be: +- correct (the world plays out as predicted) +- incorrect (the world clearly contradicts the prediction) +- partial (some aspects right, some wrong; or right direction wrong magnitude) +- unresolvable (insufficient evidence; outcome still pending) + +Output ONLY one JSON object with these fields: +- verdict ('correct' | 'incorrect' | 'partial' | 'unresolvable') +- confidence (number in [0,1]) — your self-reported confidence in this verdict. +- reasoning (string, <=400 chars) — one short paragraph explaining what evidence drove the verdict. + +If the evidence is sparse or ambiguous, return verdict='unresolvable' with +confidence reflecting the lack of evidence (NOT certainty of unresolvable). + +TAKE: + Claim: {CLAIM} + Kind: {KIND} + Holder: {HOLDER} + Made on: {SINCE_DATE} + Weight: {WEIGHT} + +EVIDENCE: +{EVIDENCE_BLOCK} +`; + +/** Verdict from a single judge model. */ +export interface JudgeVerdict { + verdict: 'correct' | 'incorrect' | 'partial' | 'unresolvable'; + confidence: number; + reasoning: string; +} + +/** Judge function signature — injected for tests. */ +export type JudgeFn = (input: { + take: Take; + evidence: string; + modelHint?: string; +}) => Promise; + +/** Evidence retriever signature — injected for tests. */ +export type EvidenceRetrieverFn = (take: Take, scope: ScopedReadOpts) => Promise; + +export interface GradeTakesOpts extends BasePhaseOpts { + /** Minimum age in months before a take is eligible for grading. Default 6. */ + minAgeMonths?: number; + /** Limit takes processed in this cycle. Default 50. */ + takeLimit?: number; + /** Inject the judge model call (tests). */ + judge?: JudgeFn; + /** Inject the evidence retriever (tests). */ + evidenceRetriever?: EvidenceRetrieverFn; + /** Override prompt_version (tests). */ + promptVersion?: string; + /** Judge model id; defaults to the configured chat model. */ + model?: string; + /** + * Auto-resolve verdicts above the confidence threshold. D17 default: false. + * When false, every verdict lands in take_grade_cache with applied=false + * (review-queue posture). When true, verdicts with confidence >= the + * configured threshold get applied via engine.resolveTake. + */ + autoResolve?: boolean; + /** + * Confidence threshold for auto-resolve. D12 default: 0.95. Schema-level + * monotonic-tightening guard (loosening requires --allow-loosen-confidence) + * lives in the takes resolution layer, not here. + */ + autoResolveThreshold?: number; + /** Identifier recorded as resolved_by when auto-applying. Default 'gbrain:grade_takes'. */ + resolvedByLabel?: string; +} + +export interface GradeTakesResult { + takes_scanned: number; + cache_hits: number; + verdicts_written: number; + auto_applied: number; + too_recent: number; + budget_exhausted: boolean; + warnings: string[]; +} + +/** + * Compute the evidence_signature for the cache. SHA-256 of evidence text + + * judge_model_id keeps the cache invalidation honest: re-running with new + * evidence OR a different judge produces a fresh row. + */ +export function evidenceSignature(evidence: string, judgeModelId: string): string { + return createHash('sha256').update(judgeModelId + '|' + evidence).digest('hex'); +} + +/** + * Parse the judge model's JSON output. Tolerant of fence wrapping and + * leading prose; returns null on unrecoverable parse failure. + */ +export function parseJudgeOutput(raw: string): JudgeVerdict | null { + if (!raw || raw.trim().length === 0) return null; + let text = raw.trim(); + const fenced = text.match(/^```(?:json)?\s*\n?([\s\S]*?)\n?```$/); + if (fenced) text = (fenced[1] ?? '').trim(); + const firstObj = text.indexOf('{'); + if (firstObj === -1) return null; + let parsed: unknown; + try { + parsed = JSON.parse(text.slice(firstObj)); + } catch { + return null; + } + if (typeof parsed !== 'object' || parsed === null) return null; + const r = parsed as Record; + const validVerdicts = ['correct', 'incorrect', 'partial', 'unresolvable'] as const; + const verdict = validVerdicts.includes(r.verdict as never) ? (r.verdict as JudgeVerdict['verdict']) : null; + if (!verdict) return null; + const confRaw = typeof r.confidence === 'number' ? r.confidence : Number.parseFloat(String(r.confidence ?? '')); + if (!Number.isFinite(confRaw)) return null; + const confidence = Math.max(0, Math.min(1, confRaw)); + const reasoning = typeof r.reasoning === 'string' ? r.reasoning.slice(0, 400) : ''; + return { verdict, confidence, reasoning }; +} + +/** + * Default evidence retriever — v0.36.0.0 ship-state placeholder. Real + * retrieval lands in v0.37+ via hybrid search over pages newer than the + * take's since_date. Documented limitation per CDX-8 + D17. + */ +export async function defaultEvidenceRetriever(take: Take, _scope: ScopedReadOpts): Promise { + return `[evidence retrieval not yet wired — v0.36.0.0 ship-state] +Take claim text (the only "evidence" available pre-T-retrieval-impl): + ${take.claim} +Made on: ${take.since_date ?? 'unknown'} +`; +} + +/** + * Production judge — calls gateway.chat with the GRADE_TAKE_PROMPT. + */ +export async function defaultJudge(input: { + take: Take; + evidence: string; + modelHint?: string; +}): Promise { + const prompt = GRADE_TAKE_PROMPT + .replace('{CLAIM}', input.take.claim) + .replace('{KIND}', input.take.kind) + .replace('{HOLDER}', input.take.holder) + .replace('{SINCE_DATE}', input.take.since_date ?? 'unknown') + .replace('{WEIGHT}', String(input.take.weight)) + .replace('{EVIDENCE_BLOCK}', input.evidence); + + const result = await gatewayChat({ + messages: [{ role: 'user', content: prompt }], + ...(input.modelHint ? { model: input.modelHint } : {}), + maxTokens: 600, + }); + const parsed = parseJudgeOutput(result.text); + if (!parsed) { + // Failed parse — treat as unresolvable at low confidence so the row + // still lands in the cache (operator sees the LLM's parse failure + // surfaced via warnings) rather than disappearing silently. + return { + verdict: 'unresolvable', + confidence: 0.0, + reasoning: 'judge_output_parse_failed', + }; + } + return parsed; +} + +/** + * Determine whether a take is old enough to grade. Defaults to 6 months. + * Takes without since_date are NOT graded (we'd be hallucinating context). + */ +export function takeIsOldEnough(take: Take, minAgeMonths: number, now: Date = new Date()): boolean { + if (!take.since_date) return false; + const cutoff = new Date(now); + cutoff.setMonth(cutoff.getMonth() - minAgeMonths); + // Tolerant date parsing — since_date can be YYYY-MM-DD or YYYY-MM. + const sinceStr = take.since_date.length === 7 ? take.since_date + '-15' : take.since_date; + const sinceDate = new Date(sinceStr); + if (Number.isNaN(sinceDate.getTime())) return false; + return sinceDate.getTime() <= cutoff.getTime(); +} + +/** + * Derive the TakeResolution for a verdict. 'unresolvable' DOES NOT auto-apply + * — only correct/incorrect/partial do. + */ +function verdictToResolution(verdict: JudgeVerdict, resolvedByLabel: string): TakeResolution | null { + if (verdict.verdict === 'unresolvable') return null; + return { + quality: verdict.verdict, + resolvedBy: resolvedByLabel, + source: `grade_takes:${GRADE_TAKES_PROMPT_VERSION}`, + }; +} + +class GradeTakesPhase extends BaseCyclePhase { + readonly name = 'grade_takes' as CyclePhase; + protected readonly budgetUsdKey = 'cycle.grade_takes.budget_usd'; + protected readonly budgetUsdDefault = 3.0; + + protected override mapErrorCode(err: unknown): string { + if (err instanceof GBrainError) return err.problem; + if (err instanceof Error) { + if (err.message.includes('budget') || err.message.includes('Budget')) return 'CALIBRATION_GRADE_BUDGET_EXHAUSTED'; + if (err.message.includes('parse')) return 'CALIBRATION_GRADE_PARSE_FAIL'; + } + return 'GRADE_TAKES_UNKNOWN'; + } + + protected async process( + engine: BrainEngine, + scope: ScopedReadOpts, + _ctx: OperationContext, + opts: GradeTakesOpts, + ): Promise<{ summary: string; details: Record; status?: PhaseStatus }> { + const judge = opts.judge ?? defaultJudge; + const evidenceRetriever = opts.evidenceRetriever ?? defaultEvidenceRetriever; + const promptVersion = opts.promptVersion ?? GRADE_TAKES_PROMPT_VERSION; + const minAgeMonths = opts.minAgeMonths ?? 6; + const takeLimit = opts.takeLimit ?? 50; + const autoResolve = opts.autoResolve ?? false; // D17 default OFF + const autoResolveThreshold = opts.autoResolveThreshold ?? 0.95; // D12 conservative + const resolvedByLabel = opts.resolvedByLabel ?? 'gbrain:grade_takes'; + const judgeModelId = opts.model ?? 'claude-sonnet-4-6'; + + const result: GradeTakesResult = { + takes_scanned: 0, + cache_hits: 0, + verdicts_written: 0, + auto_applied: 0, + too_recent: 0, + budget_exhausted: false, + warnings: [], + }; + + // Load unresolved active takes, oldest-first. + const takes = await engine.listTakes({ + resolved: false, + active: true, + sortBy: 'since_date', + limit: takeLimit, + }); + + if (opts.reporter) { + opts.reporter.start('grade_takes.takes' as never, takes.length); + } + + const now = new Date(); + for (const take of takes) { + result.takes_scanned += 1; + this.tick(opts); + + if (!takeIsOldEnough(take, minAgeMonths, now)) { + result.too_recent += 1; + continue; + } + + // Retrieve evidence first — the signature depends on it. + const evidence = await evidenceRetriever(take, scope); + const sig = evidenceSignature(evidence, judgeModelId); + + // Idempotency: skip when (take_id, prompt_version, judge_model_id, evidence_signature) exists. + const cached = await engine.executeRaw<{ verdict: string; confidence: number; applied: boolean }>( + `SELECT verdict, confidence, applied FROM take_grade_cache + WHERE take_id = $1 AND prompt_version = $2 AND judge_model_id = $3 AND evidence_signature = $4 + LIMIT 1`, + [take.id, promptVersion, judgeModelId, sig], + ); + if (cached.length > 0) { + result.cache_hits += 1; + continue; + } + + // Budget pre-check. + const budget = this.checkBudget({ + modelId: judgeModelId, + estimatedInputTokens: 1200, + maxOutputTokens: 400, + }); + if (!budget.allowed) { + result.budget_exhausted = true; + result.warnings.push( + `budget exhausted at take ${result.takes_scanned}/${takes.length} (cumulative $${budget.cumulativeCostUsd.toFixed(4)} / cap $${budget.budgetUsd.toFixed(2)})`, + ); + break; + } + + // Call the judge. Errors on a single take log warning + continue. + let verdict: JudgeVerdict; + try { + verdict = await judge({ take, evidence, modelHint: opts.model }); + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + result.warnings.push(`judge failed on take ${take.id}: ${msg}`); + continue; + } + + // Decide auto-resolve eligibility BEFORE writing to cache so the + // `applied` column reflects the decision. + const resolution = verdictToResolution(verdict, resolvedByLabel); + const shouldApply = + autoResolve && + resolution !== null && + verdict.confidence >= autoResolveThreshold; + + // Write the verdict to the cache. Idempotency conflict means another + // run beat us to it; either way the row exists with consistent state. + await engine.executeRaw( + `INSERT INTO take_grade_cache + (take_id, prompt_version, judge_model_id, evidence_signature, verdict, confidence, applied) + VALUES ($1, $2, $3, $4, $5, $6, $7) + ON CONFLICT (take_id, prompt_version, judge_model_id, evidence_signature) DO NOTHING`, + [take.id, promptVersion, judgeModelId, sig, verdict.verdict, verdict.confidence, shouldApply], + ); + result.verdicts_written += 1; + + // Apply to canonical takes if eligible. + if (shouldApply && resolution) { + try { + await engine.resolveTake(take.page_id, take.row_num, resolution); + result.auto_applied += 1; + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + result.warnings.push(`auto-apply failed on take ${take.id}: ${msg}`); + } + } + } + + if (opts.reporter) opts.reporter.finish(); + + const summary = + `grade_takes: scanned ${result.takes_scanned} takes ` + + `(${result.too_recent} too recent, ${result.cache_hits} cached, ` + + `${result.verdicts_written} new verdicts, ${result.auto_applied} auto-applied)`; + return { + summary, + details: { + ...result, + prompt_version: promptVersion, + auto_resolve: autoResolve, + auto_resolve_threshold: autoResolveThreshold, + }, + status: result.budget_exhausted ? 'warn' : 'ok', + }; + } +} + +export async function runPhaseGradeTakes( + ctx: OperationContext, + opts: GradeTakesOpts = {}, +) { + return new GradeTakesPhase().run(ctx, opts); +} + +export const __testing = { + GradeTakesPhase, + parseJudgeOutput, + evidenceSignature, + takeIsOldEnough, + verdictToResolution, +}; diff --git a/test/grade-takes.test.ts b/test/grade-takes.test.ts new file mode 100644 index 000000000..567a306f5 --- /dev/null +++ b/test/grade-takes.test.ts @@ -0,0 +1,330 @@ +/** + * v0.36.0.0 (T4) — grade_takes phase unit tests. + * + * Pure structural tests against a mock BrainEngine + injected judge + + * injected evidence retriever. No real LLM gateway, no PGLite. + * + * Tests cover: + * - happy path: judge produces verdict, lands in take_grade_cache + * - auto-resolve disabled by default (D17): even high-confidence verdicts + * DO NOT apply to canonical takes + * - auto-resolve enabled + confidence above threshold: engine.resolveTake fires + * - auto-resolve enabled + confidence below threshold: verdict cached, NOT applied + * - 'unresolvable' verdict NEVER auto-applies even at confidence=1.0 + * - cache hit path: skip already-graded (take, prompt, judge, evidence_sig) + * - takes that are too recent are skipped + * - judge throw on a single take logs warning + phase continues + * - parseJudgeOutput unit tests + * - takeIsOldEnough unit tests + */ + +import { describe, test, expect } from 'bun:test'; +import { + runPhaseGradeTakes, + parseJudgeOutput, + evidenceSignature, + takeIsOldEnough, + GRADE_TAKES_PROMPT_VERSION, + type JudgeFn, + type EvidenceRetrieverFn, +} from '../src/core/cycle/grade-takes.ts'; +import type { OperationContext } from '../src/core/operations.ts'; +import type { BrainEngine, Take, TakeResolution } from '../src/core/engine.ts'; + +// ─── Mock engine ──────────────────────────────────────────────────── + +interface CapturedSql { + sql: string; + params: unknown[]; +} +interface CapturedResolve { + pageId: number; + rowNum: number; + resolution: TakeResolution; +} + +function buildMockEngine(opts: { + takes: Take[]; + cachedGrades?: Set; // composite-key strings already in take_grade_cache +}): { engine: BrainEngine; captured: CapturedSql[]; resolves: CapturedResolve[] } { + const captured: CapturedSql[] = []; + const resolves: CapturedResolve[] = []; + const cached = opts.cachedGrades ?? new Set(); + + const engine = { + kind: 'pglite', + async listTakes() { + return opts.takes; + }, + async executeRaw(sql: string, params?: unknown[]): Promise { + captured.push({ sql, params: params ?? [] }); + if (sql.includes('SELECT verdict, confidence, applied FROM take_grade_cache')) { + const [takeId, pv, model, sig] = params ?? []; + const key = `${takeId}|${pv}|${model}|${sig}`; + if (cached.has(key)) return [{ verdict: 'correct', confidence: 0.99, applied: false } as unknown as T]; + return []; + } + return []; + }, + async resolveTake(pageId: number, rowNum: number, resolution: TakeResolution): Promise { + resolves.push({ pageId, rowNum, resolution }); + }, + } as unknown as BrainEngine; + + return { engine, captured, resolves }; +} + +function buildTake(opts: Partial & { id: number; sinceDate: string | null }): Take { + return { + id: opts.id, + page_id: opts.page_id ?? 100 + opts.id, + page_slug: opts.page_slug ?? `wiki/note-${opts.id}`, + row_num: opts.row_num ?? 1, + claim: opts.claim ?? `claim ${opts.id}`, + kind: opts.kind ?? 'bet', + holder: opts.holder ?? 'garry', + weight: opts.weight ?? 0.7, + since_date: opts.sinceDate, + until_date: null, + source: null, + superseded_by: null, + active: true, + resolved_at: null, + resolved_outcome: null, + resolved_quality: null, + resolved_value: null, + resolved_unit: null, + resolved_source: null, + resolved_by: null, + created_at: '2024-01-01T00:00:00Z', + updated_at: '2024-01-01T00:00:00Z', + } as Take; +} + +function buildCtx(engine: BrainEngine): OperationContext { + return { + engine, + config: {} as never, + logger: { info() {}, warn() {}, error() {} } as never, + dryRun: false, + remote: false, + sourceId: 'default', + }; +} + +// ─── parseJudgeOutput ─────────────────────────────────────────────── + +describe('parseJudgeOutput', () => { + test('parses clean JSON verdict', () => { + const raw = '{"verdict":"correct","confidence":0.92,"reasoning":"PG essay timing held up"}'; + const out = parseJudgeOutput(raw); + expect(out).not.toBeNull(); + expect(out!.verdict).toBe('correct'); + expect(out!.confidence).toBe(0.92); + expect(out!.reasoning).toBe('PG essay timing held up'); + }); + + test('strips markdown fence', () => { + const raw = '```json\n{"verdict":"partial","confidence":0.6,"reasoning":"mixed"}\n```'; + expect(parseJudgeOutput(raw)?.verdict).toBe('partial'); + }); + + test('clamps confidence to [0,1]', () => { + expect(parseJudgeOutput('{"verdict":"correct","confidence":2,"reasoning":"x"}')?.confidence).toBe(1); + expect(parseJudgeOutput('{"verdict":"correct","confidence":-1,"reasoning":"x"}')?.confidence).toBe(0); + }); + + test('returns null on invalid verdict label', () => { + expect(parseJudgeOutput('{"verdict":"maybe","confidence":0.5,"reasoning":"x"}')).toBeNull(); + }); + + test('returns null on missing fields', () => { + expect(parseJudgeOutput('{"verdict":"correct"}')).toBeNull(); + }); + + test('returns null on garbage input', () => { + expect(parseJudgeOutput('not json at all')).toBeNull(); + expect(parseJudgeOutput('')).toBeNull(); + }); + + test('truncates reasoning longer than 400 chars', () => { + const longReason = 'x'.repeat(600); + const raw = `{"verdict":"correct","confidence":0.9,"reasoning":"${longReason}"}`; + expect(parseJudgeOutput(raw)?.reasoning.length).toBe(400); + }); +}); + +// ─── evidenceSignature ────────────────────────────────────────────── + +describe('evidenceSignature', () => { + test('is deterministic over (evidence, judge_model_id) tuple', () => { + expect(evidenceSignature('e1', 'm1')).toBe(evidenceSignature('e1', 'm1')); + }); + + test('different evidence → different sig', () => { + expect(evidenceSignature('e1', 'm1')).not.toBe(evidenceSignature('e2', 'm1')); + }); + + test('different judge → different sig (judge swap invalidates cache)', () => { + expect(evidenceSignature('e1', 'm1')).not.toBe(evidenceSignature('e1', 'm2')); + }); +}); + +// ─── takeIsOldEnough ──────────────────────────────────────────────── + +describe('takeIsOldEnough', () => { + test('returns true when since_date is older than minAgeMonths', () => { + const take = buildTake({ id: 1, sinceDate: '2023-01-01' }); + expect(takeIsOldEnough(take, 6, new Date('2024-01-01'))).toBe(true); + }); + + test('returns false when since_date is recent', () => { + const take = buildTake({ id: 1, sinceDate: '2023-11-15' }); + expect(takeIsOldEnough(take, 6, new Date('2024-01-01'))).toBe(false); + }); + + test('returns false when since_date is null', () => { + const take = buildTake({ id: 1, sinceDate: null }); + expect(takeIsOldEnough(take, 6, new Date('2024-01-01'))).toBe(false); + }); + + test('tolerates YYYY-MM format', () => { + const take = buildTake({ id: 1, sinceDate: '2023-01' }); + expect(takeIsOldEnough(take, 6, new Date('2024-01-01'))).toBe(true); + }); + + test('returns false on unparseable since_date', () => { + const take = buildTake({ id: 1, sinceDate: 'never' }); + expect(takeIsOldEnough(take, 6, new Date('2024-01-01'))).toBe(false); + }); +}); + +// ─── Phase integration ────────────────────────────────────────────── + +describe('runPhaseGradeTakes — phase integration', () => { + test('happy path: judge produces verdict, lands in take_grade_cache (applied=false default)', async () => { + const takes = [buildTake({ id: 1, sinceDate: '2023-01-01' })]; + const { engine, captured, resolves } = buildMockEngine({ takes }); + const judge: JudgeFn = async () => ({ verdict: 'correct', confidence: 0.98, reasoning: 'evidence held' }); + const evidenceRetriever: EvidenceRetrieverFn = async () => 'mock evidence body'; + + const result = await runPhaseGradeTakes(buildCtx(engine), { judge, evidenceRetriever }); + + expect(result.status).toBe('ok'); + const details = result.details as Record; + expect(details.takes_scanned).toBe(1); + expect(details.verdicts_written).toBe(1); + expect(details.auto_applied).toBe(0); // D17 default: auto-resolve OFF + + const inserts = captured.filter(c => c.sql.includes('INSERT INTO take_grade_cache')); + expect(inserts).toHaveLength(1); + expect(inserts[0]!.params[4]).toBe('correct'); // verdict + expect(inserts[0]!.params[5]).toBe(0.98); // confidence + expect(inserts[0]!.params[6]).toBe(false); // applied=false (auto-resolve OFF) + expect(resolves).toHaveLength(0); // no canonical mutation + }); + + test('D17: auto-resolve OFF by default — even high-confidence verdict does NOT mutate takes', async () => { + const takes = [buildTake({ id: 1, sinceDate: '2023-01-01' })]; + const { engine, resolves } = buildMockEngine({ takes }); + const judge: JudgeFn = async () => ({ verdict: 'correct', confidence: 1.0, reasoning: 'certain' }); + const result = await runPhaseGradeTakes(buildCtx(engine), { judge }); + const details = result.details as Record; + expect(details.auto_resolve).toBe(false); + expect(details.auto_applied).toBe(0); + expect(resolves).toHaveLength(0); + }); + + test('D12 conservative threshold: auto-resolve ON, confidence>=0.95 → applies', async () => { + const takes = [buildTake({ id: 1, sinceDate: '2023-01-01' })]; + const { engine, resolves } = buildMockEngine({ takes }); + const judge: JudgeFn = async () => ({ verdict: 'incorrect', confidence: 0.96, reasoning: 'contradicted' }); + const result = await runPhaseGradeTakes(buildCtx(engine), { + judge, + autoResolve: true, + autoResolveThreshold: 0.95, + }); + const details = result.details as Record; + expect(details.auto_applied).toBe(1); + expect(resolves).toHaveLength(1); + expect(resolves[0]!.resolution.quality).toBe('incorrect'); + expect(resolves[0]!.resolution.resolvedBy).toBe('gbrain:grade_takes'); + }); + + test('auto-resolve ON but confidence below threshold → cached only, NOT applied', async () => { + const takes = [buildTake({ id: 1, sinceDate: '2023-01-01' })]; + const { engine, captured, resolves } = buildMockEngine({ takes }); + const judge: JudgeFn = async () => ({ verdict: 'correct', confidence: 0.85, reasoning: 'leaning yes' }); + const result = await runPhaseGradeTakes(buildCtx(engine), { + judge, + autoResolve: true, + autoResolveThreshold: 0.95, + }); + const details = result.details as Record; + expect(details.auto_applied).toBe(0); + expect(resolves).toHaveLength(0); + const insert = captured.find(c => c.sql.includes('INSERT INTO take_grade_cache')); + expect(insert!.params[6]).toBe(false); // applied=false + }); + + test('unresolvable verdict NEVER auto-applies even at confidence=1.0', async () => { + const takes = [buildTake({ id: 1, sinceDate: '2023-01-01' })]; + const { engine, resolves } = buildMockEngine({ takes }); + const judge: JudgeFn = async () => ({ verdict: 'unresolvable', confidence: 1.0, reasoning: 'no evidence yet' }); + await runPhaseGradeTakes(buildCtx(engine), { judge, autoResolve: true, autoResolveThreshold: 0.95 }); + expect(resolves).toHaveLength(0); + }); + + test('cache hit: (take, prompt, judge, evidence_sig) match → skip', async () => { + const takes = [buildTake({ id: 1, sinceDate: '2023-01-01' })]; + const sig = evidenceSignature('mock evidence body', 'claude-sonnet-4-6'); + const cached = new Set([`1|${GRADE_TAKES_PROMPT_VERSION}|claude-sonnet-4-6|${sig}`]); + const { engine } = buildMockEngine({ takes, cachedGrades: cached }); + let judgeCalls = 0; + const judge: JudgeFn = async () => { + judgeCalls++; + return { verdict: 'correct', confidence: 0.9, reasoning: 'x' }; + }; + const evidenceRetriever: EvidenceRetrieverFn = async () => 'mock evidence body'; + const result = await runPhaseGradeTakes(buildCtx(engine), { judge, evidenceRetriever }); + expect(judgeCalls).toBe(0); + const details = result.details as Record; + expect(details.cache_hits).toBe(1); + }); + + test('too-recent takes are skipped (minAgeMonths gate)', async () => { + const recentDate = new Date(); + recentDate.setMonth(recentDate.getMonth() - 2); + const takes = [buildTake({ id: 1, sinceDate: recentDate.toISOString().slice(0, 10) })]; + const { engine } = buildMockEngine({ takes }); + let judgeCalls = 0; + const judge: JudgeFn = async () => { + judgeCalls++; + return { verdict: 'correct', confidence: 1.0, reasoning: 'x' }; + }; + const result = await runPhaseGradeTakes(buildCtx(engine), { judge, minAgeMonths: 6 }); + expect(judgeCalls).toBe(0); + const details = result.details as Record; + expect(details.too_recent).toBe(1); + }); + + test('judge throw on a single take logs warning + phase continues', async () => { + const takes = [ + buildTake({ id: 1, sinceDate: '2023-01-01' }), + buildTake({ id: 2, sinceDate: '2023-01-01' }), + ]; + const { engine } = buildMockEngine({ takes }); + let calls = 0; + const judge: JudgeFn = async () => { + calls++; + if (calls === 1) throw new Error('judge timeout'); + return { verdict: 'correct', confidence: 0.9, reasoning: 'second succeeded' }; + }; + const result = await runPhaseGradeTakes(buildCtx(engine), { judge }); + expect(result.status).toBe('ok'); + const details = result.details as Record; + expect(details.verdicts_written).toBe(1); + expect((details.warnings as string[]).length).toBeGreaterThan(0); + expect((details.warnings as string[])[0]).toContain('judge timeout'); + }); +});