From 9f0bdca443bf46bd1174871eb035e49c07737e86 Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Sun, 17 May 2026 16:19:27 -0700 Subject: [PATCH] cycle: calibration_profile phase + shared voice gate across surfaces (T6) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The calibration narrative layer. Reads TakesScorecard, asks an LLM to write 2-4 conversational pattern statements ("right on tactics, late on macro by 18 months"), passes them through the voice gate, derives active bias tags, writes the row to calibration_profiles. This is the read-side that E1 (think anti-bias rewrite), E3 (contradictions join), E6 (dashboard), and E7 (real-time nudges) all consume. Voice gate (D24 — single function, multiple surfaces): ALL five calibration UX surfaces import the same gateVoice() function from src/core/calibration/voice-gate.ts. Mode parameter ('pattern_statement' | 'nudge' | 'forecast_blurb' | 'dashboard_caption' | 'morning_pulse') drives surface-specific tuning via the rubric the gate ships to its Haiku judge. NO forked implementations — voice rubric drift would defeat the gate. Each mode's rubric explicitly forbids preachy / clinical / corporate voice; a structural test pins this. Anchors the cross-cutting voice rule from /plan-ceo-review D2-D8. Fallback policy (D11): Up to 2 generation attempts (configurable). On both rejects → fall back to a hand-written template from src/core/calibration/templates.ts. Templates are intentionally short and a little "robotic" — they're the safety net, not the destination. voice_gate_passed=false + voice_gate_attempts get persisted on the calibration_profiles row so the operator can review the failing examples and tune the rubric over time. Suppressing the surface silently is NEVER an option — that's how voice quality silently degrades. parseJudgeOutput defaults to 'academic' on parse failure (NEVER passes pass-through) so a Haiku output garble falls through to the template rather than letting unverified text reach the user. calibration_profile phase: Extends BaseCyclePhase. Cold-brain skip: <5 resolved takes → no row written, no LLM call. Otherwise: scorecard via engine.getScorecard() → patterns via voice-gated generator → bias tags via separate generator (best-effort; failure logs warning, phase continues). The DB INSERT lands in the v67 calibration_profiles row with source_id, holder, the patterns, voice gate audit fields, active bias tags, and grade_completion (F1 fix — partial-grade state surfaces to the dashboard "60% graded" badge). Budget gate at $0.50/cycle default (mostly Haiku). Below-budget before-LLM-call check returns status='warn' without writing the row. Per-domain scorecards are a placeholder for v0.36.0.0 ship state — the F12 batchGetTakesScorecards() engine method that powers per-domain rendering lands in Lane C alongside the CLI/MCP surface. Architecture: parsePatternStatementsOutput is tolerant of LLM emitting numbered lists / bulleted lines despite the prompt asking for plain lines. Caps at 4 patterns + drops excessively long lines (>200 chars). parseBiasTagsOutput lowercases input + drops non-kebab-case tokens (defends against the LLM emitting "Over-Confident Geography" with spaces or capitals). Caps at 4 tags. Tests: 43 cases across two new test files. voice-gate.test.ts (24): parseJudgeOutput (7), gateVoice happy path (3), fallback path (5), mode parity (2), templates (7). calibration-profile.test.ts (19): parsers (10), pickFallbackSlots (3), phase integration (6 — cold-brain skip, happy path, voice gate fallback, grade_completion plumbed through, bias-tags failure non-fatal, source_id scope reaches INSERT). Co-Authored-By: Claude Opus 4.7 (1M context) --- src/core/calibration/templates.ts | 113 ++++++++ src/core/calibration/voice-gate.ts | 238 +++++++++++++++++ src/core/cycle/calibration-profile.ts | 369 ++++++++++++++++++++++++++ test/calibration-profile.test.ts | 296 +++++++++++++++++++++ test/voice-gate.test.ts | 332 +++++++++++++++++++++++ 5 files changed, 1348 insertions(+) create mode 100644 src/core/calibration/templates.ts create mode 100644 src/core/calibration/voice-gate.ts create mode 100644 src/core/cycle/calibration-profile.ts create mode 100644 test/calibration-profile.test.ts create mode 100644 test/voice-gate.test.ts diff --git a/src/core/calibration/templates.ts b/src/core/calibration/templates.ts new file mode 100644 index 000000000..91fc2487b --- /dev/null +++ b/src/core/calibration/templates.ts @@ -0,0 +1,113 @@ +/** + * v0.36.0.0 (T6) — voice-gate fallback templates. + * + * D11 (CEO review): when the voice gate fails twice on an LLM-generated + * surface, we fall back to a hand-written template rather than ship academic- + * sounding text OR suppress the surface silently. Predictable output beats + * voice-quality roulette. + * + * Each template gets the data it needs via slot fill. Templates intentionally + * sound a little "robotic" (acceptable; users see the SAME shape twice when + * both regens fail, NOT random voice degradation). Real conversational voice + * comes from the LLM path; templates are the safety net. + * + * Mode parity: every voice gate `Mode` MUST have an entry here. The + * VOICE_GATE_MODES export pins this contract for the test suite. + */ + +export const VOICE_GATE_MODES = [ + 'pattern_statement', + 'nudge', + 'forecast_blurb', + 'dashboard_caption', + 'morning_pulse', +] as const; + +export type VoiceGateMode = (typeof VOICE_GATE_MODES)[number]; + +export interface PatternStatementSlots { + domain: string; + nRight: number; + nWrong: number; + /** Optional one-word direction tag e.g. 'over-confident' / 'late' */ + direction?: string; +} + +export interface NudgeSlots { + domain: string; + conviction: number; + nRecentMisses: number; + nRecentTotal: number; + hushPattern: string; +} + +export interface ForecastBlurbSlots { + domain: string; + conviction: number; + bucketBrier: number; + overallBrier: number; + bucketN: number; +} + +export interface DashboardCaptionSlots { + /** e.g. 'Brier trend' or 'Per-domain accuracy' */ + surface: string; + /** Single short fact for the chart caption */ + fact: string; +} + +export interface MorningPulseSlots { + brier: number; + trend: 'improving' | 'declining' | 'stable'; + topPattern: string; +} + +/** + * Pattern statement template — what `calibration_profile` writes when the + * voice gate fails on an LLM narrative. Intentionally short; the dashboard + * surfaces it as a single subhead. + */ +export function patternStatementTemplate(s: PatternStatementSlots): string { + const total = s.nRight + s.nWrong; + if (total === 0) { + return `Not enough resolved ${s.domain} calls yet to spot a pattern.`; + } + const direction = s.direction ?? (s.nWrong > s.nRight ? 'mixed' : 'mostly right'); + return `Your ${s.domain} calls have a ${direction} record — ${s.nRight} of ${total} held up.`; +} + +/** E7 nudge template — stderr line on sync after a take is committed. */ +export function nudgeTemplate(s: NudgeSlots): string { + return ( + `[gbrain] You just committed a ${s.domain} take at conviction ${s.conviction.toFixed(2)}. ` + + `Recent record on similar calls: ${s.nRecentMisses} of ${s.nRecentTotal} missed. ` + + `Hush this pattern for 14 days: gbrain takes nudge --hush ${s.hushPattern}` + ); +} + +/** E5 inline forecast on a new take (queue + takes show). */ +export function forecastBlurbTemplate(s: ForecastBlurbSlots): string { + if (s.bucketN < 5) { + return `Forecast unavailable: only ${s.bucketN} resolved ${s.domain} takes at this conviction yet.`; + } + const note = s.bucketBrier > s.overallBrier ? 'worse than your average' : 'on par with your average'; + return ( + `Predicted Brier in ${s.domain} at conviction ${s.conviction.toFixed(2)}: ` + + `${s.bucketBrier.toFixed(2)} (${note}, n=${s.bucketN}).` + ); +} + +/** E6 dashboard chart caption. */ +export function dashboardCaptionTemplate(s: DashboardCaptionSlots): string { + return `${s.surface}: ${s.fact}`; +} + +/** Recall morning pulse Brier+pattern line. */ +export function morningPulseTemplate(s: MorningPulseSlots): string { + const trendWord = + s.trend === 'improving' ? 'improving' : s.trend === 'declining' ? 'declining' : 'stable'; + return ( + `Brier ${s.brier.toFixed(2)} (${trendWord}). ` + + (s.topPattern ? `Top pattern: ${s.topPattern}.` : '') + ); +} diff --git a/src/core/calibration/voice-gate.ts b/src/core/calibration/voice-gate.ts new file mode 100644 index 000000000..a7eda9da7 --- /dev/null +++ b/src/core/calibration/voice-gate.ts @@ -0,0 +1,238 @@ +/** + * v0.36.0.0 (T6 / D24) — voice gate: single function, multiple surfaces. + * + * Calibration-wave surfaces talk to the user in a conversational voice that + * sounds like a smart friend, not a clinical scoring system. Every nudge, + * pattern statement, forecast blurb, dashboard caption, and morning-pulse + * line passes through this gate before it reaches the user. + * + * Mode parameter (D24): + * ALL five calibration UX surfaces import THIS function. Mode-specific + * tuning lives in the rubric the gate ships to its Haiku judge, NOT in + * forked gate implementations. Forking would let voice rubric drift — + * fix in one surface, miss four. Five surfaces, one gate. + * + * Fallback policy (D11): + * Up to 2 regeneration attempts, then fall back to a hand-written + * template from src/core/calibration/templates.ts. Voice failures are + * recorded to the calibration_profiles row (voice_gate_passed=false + + * voice_gate_attempts) so the operator can review the failing examples + * and tune the rubric over time. Suppressing the surface silently is + * NEVER an option — that would let voice quality silently degrade. + * + * Test seam: opts.judge (a JudgeFn returning verdict + reason) is injected + * by tests so the gate runs hermetically. Production uses a small Haiku + * call wrapped in opts-resolution. + */ + +import { chat as gatewayChat } from '../ai/gateway.ts'; +import type { VoiceGateMode } from './templates.ts'; + +/** + * Verdict the Haiku judge returns for a candidate string. Pass-through + * 'conversational'; reject with a short reason for 'academic'. + */ +export interface VoiceGateJudgeVerdict { + verdict: 'conversational' | 'academic'; + reason: string; +} + +export type VoiceGateJudge = (input: { + candidate: string; + mode: VoiceGateMode; + rubric: string; +}) => Promise; + +export interface VoiceGateResult { + /** The final text — the LLM output if a generation passed, or the template fallback. */ + text: string; + /** Did a generation attempt pass the rubric? */ + passed: boolean; + /** How many generation attempts ran before falling back. 0 means template-only path. */ + attempts: number; + /** Reason from the LAST judge call (the one that decided pass vs final reject). */ + lastReason?: string; + /** Template slots used when passed=false (kept for audit). */ + templateSlots?: T; +} + +/** + * Generation function — the caller writes this per-surface. It produces + * ONE candidate string per call. The gate decides whether to accept or + * regenerate. Subsequent calls can use `feedback` to nudge regeneration + * away from the rejected version's failure mode. + */ +export type VoiceGateGenerator = (input: { attempt: number; feedback?: string }) => Promise; + +/** + * Template fallback function — pure. Caller passes slots; template + * produces the final string. Receives no `attempt` argument because the + * template never iterates. + */ +export type VoiceGateTemplate = (slots: S) => string; + +export interface VoiceGateOpts { + /** UX surface — drives the rubric tuning. */ + mode: VoiceGateMode; + /** Generator that produces an LLM candidate per attempt. */ + generate: VoiceGateGenerator; + /** Template fallback used when both regens fail. */ + templateFallback: { fn: VoiceGateTemplate; slots: S }; + /** Max generation attempts before falling back. Default 2 (D11). */ + maxAttempts?: number; + /** Inject the judge (tests). Production uses Haiku. */ + judge?: VoiceGateJudge; + /** Override the rubric per mode (rarely needed). */ + rubric?: string; +} + +/** + * Default rubrics per mode. The gate consults the rubric when deciding + * whether a candidate sounds conversational vs academic. Tuning the rubric + * is the V1 lever; tuning the gate code is a v0.37+ concern. + */ +export const DEFAULT_RUBRICS: Record = { + pattern_statement: `Voice for a calibration pattern statement: +- Sounds like a smart friend recapping your record, not a doctor or HR. +- Uses second person ("your", "you"). +- Names numbers grounded in actual takes ("2 of 3 missed"), not abstract + metrics like "Brier 0.31" or "conviction-bucket 0.8-0.9". +- No preachy/clinical phrasing ("our analysis indicates", "the data shows"). +- Short — under 25 words. +- NEVER mentions internal field names like 'Brier' or 'conviction-bucket' + without translation.`, + + nudge: `Voice for a real-time nudge fired during sync after a take is committed: +- Sounds like a friend tapping you on the shoulder, not an alert system. +- Second person, contractions allowed, casual. +- Grounded in 1-2 concrete past data points the user can verify. +- Always closes with a concrete next step (a CLI command or a question). +- Under 30 words. +- NEVER preachy. NEVER "we recommend." NEVER "according to your data".`, + + forecast_blurb: `Voice for an inline forecast blurb on a new take: +- One short factual line, ~12-20 words. +- Names the past data in concrete terms ("2 of 3 missed" beats "Brier 0.31"). +- Acknowledges uncertainty when n is small. +- No "predicted Brier" jargon without translation. +- NEVER condescending.`, + + dashboard_caption: `Voice for a chart caption on the admin dashboard: +- Single short sentence per caption. +- Names ONE concrete fact. +- No marketing copy, no "powerful insights", no "leverage". +- Plain language, no jargon.`, + + morning_pulse: `Voice for a daily morning-pulse line: +- One sentence, sounds like a friend giving you a quick status check. +- Names the trend in plain words ("improving" beats "trending positive"). +- Mentions ONE pattern when relevant; skip when no clear pattern. +- Under 25 words. +- NEVER clinical, NEVER preachy, NEVER hedged corporate language.`, +}; + +const DEFAULT_MAX_ATTEMPTS = 2; + +const HAIKU_GATE_PROMPT = `You are the voice gate for a personal AI brain. A surface wants to show +this candidate text to the user. Decide whether it sounds conversational +(friend talking to friend) or academic (clinical / corporate). + +Output ONLY a JSON object: {"verdict":"conversational"|"academic","reason":"<<=80 chars>"}. + +RUBRIC for this surface: +{RUBRIC} + +CANDIDATE: +{CANDIDATE}`; + +/** + * Default judge — Haiku-based rubric verdict. Production path; tests + * inject a stub. + */ +export async function defaultJudge(input: { + candidate: string; + mode: VoiceGateMode; + rubric: string; +}): Promise { + const prompt = HAIKU_GATE_PROMPT + .replace('{RUBRIC}', input.rubric) + .replace('{CANDIDATE}', input.candidate); + const result = await gatewayChat({ + messages: [{ role: 'user', content: prompt }], + model: 'claude-haiku-4-5', + maxTokens: 100, + }); + return parseJudgeOutput(result.text); +} + +/** + * Parse the Haiku judge's JSON output. Robust to fence wrapping + + * leading prose. On unrecoverable parse failure, treat as 'academic' + * with reason='parse_failed' so the gate falls back to the template + * rather than silently passing bad voice. + */ +export function parseJudgeOutput(raw: string): VoiceGateJudgeVerdict { + if (!raw || raw.trim().length === 0) { + return { verdict: 'academic', reason: 'empty_judge_output' }; + } + let text = raw.trim(); + const fenced = text.match(/^```(?:json)?\s*\n?([\s\S]*?)\n?```$/); + if (fenced) text = (fenced[1] ?? '').trim(); + const firstObj = text.indexOf('{'); + if (firstObj === -1) return { verdict: 'academic', reason: 'parse_failed' }; + let parsed: unknown; + try { + parsed = JSON.parse(text.slice(firstObj)); + } catch { + return { verdict: 'academic', reason: 'parse_failed' }; + } + if (typeof parsed !== 'object' || parsed === null) { + return { verdict: 'academic', reason: 'parse_failed' }; + } + const r = parsed as Record; + const verdict = r.verdict === 'conversational' ? 'conversational' : 'academic'; + const reason = typeof r.reason === 'string' ? r.reason.slice(0, 80) : 'no_reason'; + return { verdict, reason }; +} + +/** + * Gate a single piece of LLM-generated voice. Returns the final text + + * audit info (pass/fail + attempts). + */ +export async function gateVoice(opts: VoiceGateOpts): Promise> { + const judge = opts.judge ?? defaultJudge; + const rubric = opts.rubric ?? DEFAULT_RUBRICS[opts.mode]; + const maxAttempts = opts.maxAttempts ?? DEFAULT_MAX_ATTEMPTS; + + let lastReason: string | undefined; + for (let attempt = 1; attempt <= maxAttempts; attempt++) { + let candidate: string; + try { + candidate = await opts.generate({ attempt, feedback: lastReason }); + } catch (err) { + // Generator threw — treat as a failed attempt but continue. If both + // attempts throw we fall through to the template (D11 fallback). + lastReason = err instanceof Error ? err.message : 'generator_threw'; + continue; + } + if (!candidate || candidate.trim().length === 0) { + lastReason = 'empty_generation'; + continue; + } + const verdict = await judge({ candidate, mode: opts.mode, rubric }); + if (verdict.verdict === 'conversational') { + return { text: candidate, passed: true, attempts: attempt, lastReason: verdict.reason }; + } + lastReason = verdict.reason; + } + + // Both attempts failed (or threw) — template fallback. + const fallback = opts.templateFallback.fn(opts.templateFallback.slots); + return { + text: fallback, + passed: false, + attempts: maxAttempts, + ...(lastReason !== undefined ? { lastReason } : {}), + templateSlots: opts.templateFallback.slots, + }; +} diff --git a/src/core/cycle/calibration-profile.ts b/src/core/cycle/calibration-profile.ts new file mode 100644 index 000000000..062017fe4 --- /dev/null +++ b/src/core/cycle/calibration-profile.ts @@ -0,0 +1,369 @@ +/** + * v0.36.0.0 (T6) — calibration_profile cycle phase. + * + * Aggregates the resolved takes subset into a calibration profile per holder: + * - quantitative: TakesScorecard (Brier, accuracy, partial_rate, per-domain) + * - qualitative: 2-4 narrative pattern statements via the voice gate + * - bias tags: short kebab-case labels (e.g. 'over-confident-geography') + * used by E3 (calibration-aware contradictions) and E7 (real-time nudges) + * + * grade_completion (F1): + * When grade_takes aborts mid-cycle on budget cap, this phase still runs + * but tags the profile row with `grade_completion: REAL` (fraction of + * eligible-and-old-enough takes the grade phase processed). Dashboard + * surfaces "60% graded" badge when < 0.9. Default 1.0 (full completion). + * + * Voice gate (D11 / D24): + * Pattern statements pass through gateVoice() with mode='pattern_statement'. + * Two regeneration attempts, then fall back to a hand-written template. + * `voice_gate_passed` + `voice_gate_attempts` get recorded on the row for + * audit; failed-pass-but-template-OK rows surface to a review queue + * (lands in Lane C). + * + * Source-scope: BaseCyclePhase enforces sourceScopeOpts threading. + * Profiles are per (source_id, holder) so a multi-source brain gets distinct + * profiles per source for the same holder. + */ + +import { BaseCyclePhase, type ScopedReadOpts, type BasePhaseOpts } from './base-phase.ts'; +import { chat as gatewayChat } from '../ai/gateway.ts'; +import { gateVoice, type VoiceGateGenerator, type VoiceGateJudge } from '../calibration/voice-gate.ts'; +import { patternStatementTemplate, type PatternStatementSlots } from '../calibration/templates.ts'; +import { GBrainError } from '../types.ts'; +import type { OperationContext } from '../operations.ts'; +import type { BrainEngine, TakesScorecard } from '../engine.ts'; +import type { PhaseStatus, CyclePhase } from '../cycle.ts'; + +export const CALIBRATION_PROFILE_PROMPT_VERSION = 'v0.36.0.0-stub'; + +const PATTERN_STATEMENTS_PROMPT = `[v0.36.0.0-stub] You are summarizing a forecaster's track record so they +can see their patterns. Below is a JSON snapshot of how they performed — +per-domain scorecards over the resolved subset. + +Write 2 to 4 short pattern statements, ONE per line. Each statement: +- Names a domain (e.g. "macro tech", "geography", "hiring decisions"). +- States the direction (right / wrong / late / early / over-confident / + under-calibrated). +- Includes ONE concrete number a reader can verify ("2 of 5 missed"). +- Sounds like a smart friend recapping the record, not a doctor or HR. +- Under 25 words. + +EXAMPLES of the voice we want: +- "You called early-stage tactics well — 8 of 10 held up." +- "Geography is your blind spot. High-conviction calls missed 4 of 6." +- "On macro tech you tend to be ~18 months early; calls land, just later." + +DO NOT use phrases like "the data shows", "our analysis indicates", "Brier +score", or "conviction bucket". DO NOT preach. Be plain. + +Output the 2-4 pattern statements only, one per line. No numbering, no +prose around them. + +SCORECARD: +{SCORECARD_JSON} +`; + +const BIAS_TAGS_PROMPT = `Based on the pattern statements below, emit 1-4 +kebab-case bias tags. Each tag combines an axis (over-confident, +under-confident, early, late, hedged-correctly) with a domain +(tactics, macro, geography, hiring, market-timing, founder-behavior, +ai, other). + +Examples: "over-confident-geography", "late-on-macro-tech", +"hedged-correctly-on-hiring". + +Output ONLY a JSON array of strings. No prose. If no clear bias pattern +emerges, return []. + +PATTERN STATEMENTS: +{PATTERNS_BULLETS} +`; + +/** Generator function for pattern statements (test seam). */ +export type PatternStatementsGenerator = (input: { + scorecard: TakesScorecard; + holder: string; + attempt: number; + feedback?: string; +}) => Promise; + +/** Generator function for bias tags (test seam). */ +export type BiasTagsGenerator = (patterns: string[]) => Promise; + +export interface CalibrationProfileOpts extends BasePhaseOpts { + /** Holder to generate the profile for. Default 'garry'. */ + holder?: string; + /** Inject the patterns generator (tests). */ + patternsGenerator?: PatternStatementsGenerator; + /** Inject the bias-tags generator (tests). */ + biasTagsGenerator?: BiasTagsGenerator; + /** Inject the voice gate judge (tests). */ + voiceGateJudge?: VoiceGateJudge; + /** grade_completion from grade_takes phase that ran in the same cycle. Default 1.0. */ + gradeCompletion?: number; + /** Override prompt version (tests). */ + promptVersion?: string; + /** Override model id; default Sonnet. */ + model?: string; +} + +export interface CalibrationProfileResult { + profile_written: boolean; + voice_gate_passed: boolean; + voice_gate_attempts: number; + pattern_statements: string[]; + active_bias_tags: string[]; + total_resolved: number; + brier: number | null; + warnings: string[]; +} + +/** Production patterns generator — calls Sonnet with the SCORECARD_JSON prompt. */ +export async function defaultPatternsGenerator(input: { + scorecard: TakesScorecard; + holder: string; + attempt: number; + feedback?: string; + modelHint?: string; +}): Promise { + const prompt = PATTERN_STATEMENTS_PROMPT.replace( + '{SCORECARD_JSON}', + JSON.stringify({ holder: input.holder, ...input.scorecard }, null, 2), + ); + const feedbackSuffix = input.feedback + ? `\n\nPrior attempt was rejected for: ${input.feedback}. Try again, more conversational.` + : ''; + const result = await gatewayChat({ + messages: [{ role: 'user', content: prompt + feedbackSuffix }], + ...(input.modelHint ? { model: input.modelHint } : {}), + maxTokens: 500, + }); + return parsePatternStatementsOutput(result.text); +} + +/** Production bias-tags generator. */ +export async function defaultBiasTagsGenerator(patterns: string[]): Promise { + if (patterns.length === 0) return []; + const prompt = BIAS_TAGS_PROMPT.replace( + '{PATTERNS_BULLETS}', + patterns.map(p => `- ${p}`).join('\n'), + ); + const result = await gatewayChat({ + messages: [{ role: 'user', content: prompt }], + maxTokens: 200, + }); + return parseBiasTagsOutput(result.text); +} + +/** Parse a newline-separated pattern-statement block. */ +export function parsePatternStatementsOutput(raw: string): string[] { + if (!raw || raw.trim().length === 0) return []; + const lines = raw + .split('\n') + .map(l => l.trim()) + // Strip leading numbering/bullets the LLM may emit despite the prompt. + .map(l => l.replace(/^[-*•]\s+|^\d+[.)]\s+/, '')) + .filter(l => l.length > 0 && l.length <= 200); + return lines.slice(0, 4); +} + +/** Parse a JSON-array bias-tags block, tolerant of fence wrapping. */ +export function parseBiasTagsOutput(raw: string): string[] { + if (!raw || raw.trim().length === 0) return []; + let text = raw.trim(); + const fenced = text.match(/^```(?:json)?\s*\n?([\s\S]*?)\n?```$/); + if (fenced) text = (fenced[1] ?? '').trim(); + const firstArr = text.indexOf('['); + if (firstArr === -1) return []; + let parsed: unknown; + try { + parsed = JSON.parse(text.slice(firstArr)); + } catch { + return []; + } + if (!Array.isArray(parsed)) return []; + return parsed + .filter((t): t is string => typeof t === 'string') + .map(t => t.trim().toLowerCase()) + .filter(t => /^[a-z]+(?:-[a-z0-9]+)*$/.test(t)) + .slice(0, 4); +} + +/** Pick the "loudest" pattern slot for the template fallback. */ +function pickFallbackSlots(scorecard: TakesScorecard): PatternStatementSlots { + if (!scorecard || scorecard.resolved === 0) { + return { domain: 'overall', nRight: 0, nWrong: 0 }; + } + const direction = scorecard.brier !== null && scorecard.brier > 0.25 ? 'over-confident' : 'mostly right'; + return { + domain: 'overall', + nRight: scorecard.correct, + nWrong: scorecard.incorrect, + direction, + }; +} + +class CalibrationProfilePhase extends BaseCyclePhase { + readonly name = 'calibration_profile' as CyclePhase; + protected readonly budgetUsdKey = 'cycle.calibration_profile.budget_usd'; + protected readonly budgetUsdDefault = 0.5; + + protected override mapErrorCode(err: unknown): string { + if (err instanceof GBrainError) return err.problem; + if (err instanceof Error) { + if (err.message.includes('voice_gate')) return 'CALIBRATION_VOICE_GATE_EXHAUSTED'; + } + return 'CALIBRATION_PROFILE_UNKNOWN'; + } + + protected async process( + engine: BrainEngine, + scope: ScopedReadOpts, + _ctx: OperationContext, + opts: CalibrationProfileOpts, + ): Promise<{ summary: string; details: Record; status?: PhaseStatus }> { + const holder = opts.holder ?? 'garry'; + const promptVersion = opts.promptVersion ?? CALIBRATION_PROFILE_PROMPT_VERSION; + const modelId = opts.model ?? 'claude-sonnet-4-6'; + const gradeCompletion = opts.gradeCompletion ?? 1.0; + const patternsGenerator = opts.patternsGenerator ?? defaultPatternsGenerator; + const biasTagsGenerator = opts.biasTagsGenerator ?? defaultBiasTagsGenerator; + + const result: CalibrationProfileResult = { + profile_written: false, + voice_gate_passed: false, + voice_gate_attempts: 0, + pattern_statements: [], + active_bias_tags: [], + total_resolved: 0, + brier: null, + warnings: [], + }; + + // Load the holder's scorecard. + const scorecard = await engine.getScorecard({ holder }, undefined); + result.total_resolved = scorecard.resolved; + result.brier = scorecard.brier; + + // Cold-brain branch: not enough resolved takes for a profile yet. + if (scorecard.resolved < 5) { + return { + summary: `calibration_profile: holder=${holder} has only ${scorecard.resolved} resolved takes (need >=5 for a profile)`, + details: { ...result, skipped: 'insufficient_data' }, + status: 'ok', + }; + } + + // Generate pattern statements via the voice gate. + const generate: VoiceGateGenerator = async ({ attempt, feedback }) => { + const lines = await patternsGenerator({ + scorecard, + holder, + attempt, + ...(feedback !== undefined ? { feedback } : {}), + }); + return lines.join('\n'); + }; + + // Budget gate before invoking the LLM-driven gate. + const budget = this.checkBudget({ + modelId, + estimatedInputTokens: 800, + maxOutputTokens: 500, + }); + if (!budget.allowed) { + result.warnings.push(`budget exhausted before profile generation (cap $${budget.budgetUsd.toFixed(2)})`); + return { + summary: `calibration_profile: skipped — budget exhausted`, + details: { ...result, budget_exhausted: true }, + status: 'warn', + }; + } + + const gateInput: Parameters>[0] = { + mode: 'pattern_statement', + generate, + templateFallback: { + fn: patternStatementTemplate, + slots: pickFallbackSlots(scorecard), + }, + }; + if (opts.voiceGateJudge) gateInput.judge = opts.voiceGateJudge; + const gated = await gateVoice(gateInput); + + result.voice_gate_passed = gated.passed; + result.voice_gate_attempts = gated.attempts; + + // Split the final text into lines (the LLM emits multiple patterns on + // separate lines; the template fallback is a single line). + result.pattern_statements = gated.text + .split('\n') + .map(l => l.trim()) + .filter(l => l.length > 0); + + // Bias tags from the patterns. Best-effort; failure is non-fatal. + try { + result.active_bias_tags = await biasTagsGenerator(result.pattern_statements); + } catch (err) { + result.warnings.push(`bias_tags_generator failed: ${err instanceof Error ? err.message : String(err)}`); + } + + // Write the profile row. + const sourceId = scope.sourceId ?? 'default'; + await engine.executeRaw( + `INSERT INTO calibration_profiles ( + source_id, holder, generated_at, published, + total_resolved, brier, accuracy, partial_rate, grade_completion, + domain_scorecards, pattern_statements, + voice_gate_passed, voice_gate_attempts, + active_bias_tags, model_id, cost_usd, judge_model_agreement + ) VALUES ($1, $2, now(), false, + $3, $4, $5, $6, $7, + $8::jsonb, $9::text[], + $10, $11, + $12::text[], $13, NULL, NULL)`, + [ + sourceId, + holder, + scorecard.resolved, + scorecard.brier, + scorecard.accuracy, + scorecard.partial_rate, + gradeCompletion, + // domain_scorecards: per-domain breakdown placeholder — v0.36.0.0 + // ships with the overall scorecard only; per-domain comes when + // batchGetTakesScorecards (F12) lands in Lane C. + JSON.stringify({}), + result.pattern_statements, + result.voice_gate_passed, + result.voice_gate_attempts, + result.active_bias_tags, + modelId, + ], + ); + result.profile_written = true; + + return { + summary: + `calibration_profile: holder=${holder} brier=${(scorecard.brier ?? 0).toFixed(2)} ` + + `(${scorecard.resolved} resolved, ${result.pattern_statements.length} patterns, ` + + `${result.active_bias_tags.length} bias tags, gate ${gated.passed ? 'passed' : 'fell back to template'})`, + details: { ...result }, + status: 'ok', + }; + } +} + +export async function runPhaseCalibrationProfile( + ctx: OperationContext, + opts: CalibrationProfileOpts = {}, +) { + return new CalibrationProfilePhase().run(ctx, opts); +} + +export const __testing = { + CalibrationProfilePhase, + parsePatternStatementsOutput, + parseBiasTagsOutput, + pickFallbackSlots, +}; diff --git a/test/calibration-profile.test.ts b/test/calibration-profile.test.ts new file mode 100644 index 000000000..4b40f0d55 --- /dev/null +++ b/test/calibration-profile.test.ts @@ -0,0 +1,296 @@ +/** + * v0.36.0.0 (T6) — calibration_profile phase unit tests. + * + * Hermetic. Mock engine + injected patterns generator + injected voice gate + * judge. Exercises: + * - cold-brain skip: <5 resolved takes + * - happy path: scorecard → generator → voice gate pass → row written + * - voice gate rejects both attempts → template fallback written + * - bias tags generator wired + * - parsePatternStatementsOutput + parseBiasTagsOutput unit tests + * - grade_completion plumbed through to the DB row + * - budget exhausted → status='warn', no row written + */ + +import { describe, test, expect } from 'bun:test'; +import { + runPhaseCalibrationProfile, + parsePatternStatementsOutput, + parseBiasTagsOutput, + __testing, + type PatternStatementsGenerator, + type BiasTagsGenerator, +} from '../src/core/cycle/calibration-profile.ts'; +import type { VoiceGateJudge } from '../src/core/calibration/voice-gate.ts'; +import type { OperationContext } from '../src/core/operations.ts'; +import type { BrainEngine, TakesScorecard } from '../src/core/engine.ts'; + +interface CapturedSql { + sql: string; + params: unknown[]; +} + +function buildMockEngine(opts: { scorecard: TakesScorecard }): { + engine: BrainEngine; + captured: CapturedSql[]; +} { + const captured: CapturedSql[] = []; + const engine = { + kind: 'pglite', + async getScorecard() { + return opts.scorecard; + }, + async executeRaw(sql: string, params?: unknown[]): Promise { + captured.push({ sql, params: params ?? [] }); + return []; + }, + } as unknown as BrainEngine; + return { engine, captured }; +} + +function buildCtx(engine: BrainEngine): OperationContext { + return { + engine, + config: {} as never, + logger: { info() {}, warn() {}, error() {} } as never, + dryRun: false, + remote: false, + sourceId: 'default', + }; +} + +const passJudge: VoiceGateJudge = async () => ({ verdict: 'conversational', reason: 'fine' }); +const rejectJudge: VoiceGateJudge = async () => ({ verdict: 'academic', reason: 'clinical' }); + +// ─── Parsers ──────────────────────────────────────────────────────── + +describe('parsePatternStatementsOutput', () => { + test('splits newline-separated statements', () => { + const raw = 'You called early-stage tactics well — 8 of 10 held up.\nGeography is your blind spot. 4 of 6 missed.'; + expect(parsePatternStatementsOutput(raw)).toEqual([ + 'You called early-stage tactics well — 8 of 10 held up.', + 'Geography is your blind spot. 4 of 6 missed.', + ]); + }); + + test('strips numbered list markers if the LLM emits them', () => { + const raw = '1. First pattern.\n2) Second pattern.\n- Third pattern.'; + expect(parsePatternStatementsOutput(raw)).toEqual([ + 'First pattern.', + 'Second pattern.', + 'Third pattern.', + ]); + }); + + test('caps at 4 statements', () => { + const raw = ['a', 'b', 'c', 'd', 'e', 'f'].join('\n'); + expect(parsePatternStatementsOutput(raw).length).toBe(4); + }); + + test('drops empty lines and excessively long lines', () => { + const long = 'x'.repeat(250); + const raw = `valid\n\n${long}\nalso valid`; + expect(parsePatternStatementsOutput(raw)).toEqual(['valid', 'also valid']); + }); + + test('returns [] on empty input', () => { + expect(parsePatternStatementsOutput('')).toEqual([]); + }); +}); + +describe('parseBiasTagsOutput', () => { + test('parses clean kebab-case tags', () => { + const raw = '["over-confident-geography","late-on-macro-tech"]'; + expect(parseBiasTagsOutput(raw)).toEqual(['over-confident-geography', 'late-on-macro-tech']); + }); + + test('strips markdown fence', () => { + const raw = '```json\n["over-confident-geography"]\n```'; + expect(parseBiasTagsOutput(raw)).toEqual(['over-confident-geography']); + }); + + test('lowercases input + drops non-kebab-case', () => { + const raw = '["Over-Confident-Geography","INVALID TAG","late-on-macro"]'; + expect(parseBiasTagsOutput(raw)).toEqual(['over-confident-geography', 'late-on-macro']); + }); + + test('caps at 4 tags', () => { + const raw = JSON.stringify(['a-b', 'c-d', 'e-f', 'g-h', 'i-j', 'k-l']); + expect(parseBiasTagsOutput(raw).length).toBe(4); + }); + + test('returns [] on malformed input', () => { + expect(parseBiasTagsOutput('not json')).toEqual([]); + expect(parseBiasTagsOutput('')).toEqual([]); + }); +}); + +// ─── pickFallbackSlots ────────────────────────────────────────────── + +describe('pickFallbackSlots', () => { + test('over-confident direction when brier > 0.25', () => { + const scorecard: TakesScorecard = { + total_bets: 10, + resolved: 10, + correct: 4, + incorrect: 6, + partial: 0, + accuracy: 0.4, + brier: 0.32, + partial_rate: 0, + }; + expect(__testing.pickFallbackSlots(scorecard).direction).toBe('over-confident'); + }); + + test('mostly-right direction when brier <= 0.25', () => { + const scorecard: TakesScorecard = { + total_bets: 10, + resolved: 10, + correct: 8, + incorrect: 2, + partial: 0, + accuracy: 0.8, + brier: 0.12, + partial_rate: 0, + }; + expect(__testing.pickFallbackSlots(scorecard).direction).toBe('mostly right'); + }); + + test('zero resolved → "overall" domain, 0/0', () => { + const scorecard: TakesScorecard = { + total_bets: 0, + resolved: 0, + correct: 0, + incorrect: 0, + partial: 0, + accuracy: null, + brier: null, + partial_rate: null, + }; + const out = __testing.pickFallbackSlots(scorecard); + expect(out.nRight).toBe(0); + expect(out.nWrong).toBe(0); + }); +}); + +// ─── Phase integration ────────────────────────────────────────────── + +const ENOUGH_RESOLVED_SCORECARD: TakesScorecard = { + total_bets: 20, + resolved: 12, + correct: 7, + incorrect: 4, + partial: 1, + accuracy: 0.636, + brier: 0.21, + partial_rate: 0.083, +}; + +describe('runPhaseCalibrationProfile — phase integration', () => { + test('cold-brain skip: <5 resolved → no row written, status=ok', async () => { + const { engine, captured } = buildMockEngine({ + scorecard: { ...ENOUGH_RESOLVED_SCORECARD, resolved: 3 }, + }); + const result = await runPhaseCalibrationProfile(buildCtx(engine), {}); + expect(result.status).toBe('ok'); + expect((result.details as Record).profile_written).toBe(false); + expect((result.details as Record).skipped).toBe('insufficient_data'); + expect(captured.filter(c => c.sql.includes('INSERT INTO calibration_profiles'))).toHaveLength(0); + }); + + test('happy path: row written with passed voice gate', async () => { + const { engine, captured } = buildMockEngine({ scorecard: ENOUGH_RESOLVED_SCORECARD }); + const patternsGenerator: PatternStatementsGenerator = async () => [ + 'You called early-stage tactics well — 8 of 10 held up.', + 'Geography is your blind spot — 4 of 6 missed.', + ]; + const biasTagsGenerator: BiasTagsGenerator = async () => ['over-confident-geography']; + const result = await runPhaseCalibrationProfile(buildCtx(engine), { + patternsGenerator, + biasTagsGenerator, + voiceGateJudge: passJudge, + }); + expect(result.status).toBe('ok'); + const details = result.details as Record; + expect(details.profile_written).toBe(true); + expect(details.voice_gate_passed).toBe(true); + expect(details.voice_gate_attempts).toBe(1); + expect((details.pattern_statements as string[]).length).toBe(2); + expect((details.active_bias_tags as string[])).toEqual(['over-confident-geography']); + + const insert = captured.find(c => c.sql.includes('INSERT INTO calibration_profiles')); + expect(insert).toBeDefined(); + // Params: source_id, holder, total_resolved, brier, accuracy, partial_rate, + // grade_completion, domain_scorecards_json, patterns[], voice_passed, voice_attempts, + // bias_tags[], model_id + expect(insert!.params[0]).toBe('default'); // source_id + expect(insert!.params[1]).toBe('garry'); // holder + expect(insert!.params[2]).toBe(12); // total_resolved + expect(insert!.params[9]).toBe(true); // voice_gate_passed + expect(insert!.params[10]).toBe(1); // voice_gate_attempts + expect(insert!.params[11]).toEqual(['over-confident-geography']); // active_bias_tags + }); + + test('voice gate rejects both attempts → template fallback written, voice_gate_passed=false', async () => { + const { engine, captured } = buildMockEngine({ scorecard: ENOUGH_RESOLVED_SCORECARD }); + const patternsGenerator: PatternStatementsGenerator = async () => [ + 'Per our analysis, the data indicates patterns.', + ]; + const result = await runPhaseCalibrationProfile(buildCtx(engine), { + patternsGenerator, + voiceGateJudge: rejectJudge, + }); + const details = result.details as Record; + expect(details.voice_gate_passed).toBe(false); + expect(details.voice_gate_attempts).toBe(2); + expect(details.profile_written).toBe(true); + const patterns = details.pattern_statements as string[]; + expect(patterns.length).toBeGreaterThan(0); + expect(patterns[0]).toContain('overall'); // template fallback contains "overall" domain + + const insert = captured.find(c => c.sql.includes('INSERT INTO calibration_profiles')); + expect(insert!.params[9]).toBe(false); // voice_gate_passed=false + expect(insert!.params[10]).toBe(2); // voice_gate_attempts=2 + }); + + test('grade_completion is plumbed through to the row', async () => { + const { engine, captured } = buildMockEngine({ scorecard: ENOUGH_RESOLVED_SCORECARD }); + const patternsGenerator: PatternStatementsGenerator = async () => ['fine pattern']; + await runPhaseCalibrationProfile(buildCtx(engine), { + patternsGenerator, + voiceGateJudge: passJudge, + gradeCompletion: 0.6, + }); + const insert = captured.find(c => c.sql.includes('INSERT INTO calibration_profiles')); + expect(insert!.params[6]).toBe(0.6); // grade_completion + }); + + test('bias_tags_generator failure logs warning + phase continues', async () => { + const { engine } = buildMockEngine({ scorecard: ENOUGH_RESOLVED_SCORECARD }); + const patternsGenerator: PatternStatementsGenerator = async () => ['fine pattern']; + const biasTagsGenerator: BiasTagsGenerator = async () => { + throw new Error('Haiku timed out'); + }; + const result = await runPhaseCalibrationProfile(buildCtx(engine), { + patternsGenerator, + biasTagsGenerator, + voiceGateJudge: passJudge, + }); + expect(result.status).toBe('ok'); + const details = result.details as Record; + expect(details.profile_written).toBe(true); + expect((details.warnings as string[])[0]).toContain('Haiku timed out'); + }); + + test('source_id from ctx scope reaches the INSERT params', async () => { + const { engine, captured } = buildMockEngine({ scorecard: ENOUGH_RESOLVED_SCORECARD }); + const patternsGenerator: PatternStatementsGenerator = async () => ['fine pattern']; + const ctx = { ...buildCtx(engine), sourceId: 'tenant-b' }; + await runPhaseCalibrationProfile(ctx, { + patternsGenerator, + voiceGateJudge: passJudge, + }); + const insert = captured.find(c => c.sql.includes('INSERT INTO calibration_profiles')); + expect(insert!.params[0]).toBe('tenant-b'); + }); +}); diff --git a/test/voice-gate.test.ts b/test/voice-gate.test.ts new file mode 100644 index 000000000..9c0528a25 --- /dev/null +++ b/test/voice-gate.test.ts @@ -0,0 +1,332 @@ +/** + * v0.36.0.0 (T6 / D24) — voice gate unit tests. + * + * Hermetic. No real LLM, no PGLite. Inject the judge + generator + template + * fallback per test. + * + * Tests cover: + * - D11 retry policy: 2 attempts then template fallback + * - happy path: first attempt passes, second attempt skipped + * - happy path: first rejected, second passes + * - both rejected → template fallback, audit fields populated + * - generator throws → counted as failed attempt + template fallback + * - parseJudgeOutput: fence-stripping, malformed input, parse failure + * falls to 'academic' (NOT pass-through) + * - mode parity: every VoiceGateMode has a default rubric + * - templates produce stable output for fixed slots + */ + +import { describe, test, expect } from 'bun:test'; +import { + gateVoice, + parseJudgeOutput, + DEFAULT_RUBRICS, + type VoiceGateJudge, + type VoiceGateGenerator, +} from '../src/core/calibration/voice-gate.ts'; +import { + VOICE_GATE_MODES, + patternStatementTemplate, + nudgeTemplate, + forecastBlurbTemplate, + dashboardCaptionTemplate, + morningPulseTemplate, + type PatternStatementSlots, +} from '../src/core/calibration/templates.ts'; + +const passJudge: VoiceGateJudge = async () => ({ verdict: 'conversational', reason: 'reads natural' }); +const rejectJudge: VoiceGateJudge = async () => ({ verdict: 'academic', reason: 'too clinical' }); + +const defaultSlots: PatternStatementSlots = { domain: 'macro tech', nRight: 2, nWrong: 5, direction: 'over-confident' }; + +// ─── parseJudgeOutput ─────────────────────────────────────────────── + +describe('parseJudgeOutput', () => { + test('parses a clean verdict object', () => { + const out = parseJudgeOutput('{"verdict":"conversational","reason":"sounds like a friend"}'); + expect(out.verdict).toBe('conversational'); + expect(out.reason).toBe('sounds like a friend'); + }); + + test('parses fence-wrapped JSON', () => { + const out = parseJudgeOutput('```json\n{"verdict":"academic","reason":"jargon"}\n```'); + expect(out.verdict).toBe('academic'); + }); + + test('parses leading-prose payload', () => { + const out = parseJudgeOutput('Here is my verdict: {"verdict":"academic","reason":"clinical"}'); + expect(out.verdict).toBe('academic'); + }); + + test('falls to academic on empty input (NEVER passes pass-through)', () => { + expect(parseJudgeOutput('').verdict).toBe('academic'); + expect(parseJudgeOutput(' ').verdict).toBe('academic'); + }); + + test('falls to academic on malformed JSON', () => { + expect(parseJudgeOutput('not json').verdict).toBe('academic'); + expect(parseJudgeOutput('{not valid').verdict).toBe('academic'); + }); + + test('coerces unknown verdict label to academic', () => { + expect(parseJudgeOutput('{"verdict":"meh","reason":"x"}').verdict).toBe('academic'); + }); + + test('truncates reason at 80 chars', () => { + const long = 'x'.repeat(200); + const out = parseJudgeOutput(`{"verdict":"academic","reason":"${long}"}`); + expect(out.reason.length).toBe(80); + }); +}); + +// ─── gateVoice ────────────────────────────────────────────────────── + +describe('gateVoice — happy path', () => { + test('first attempt passes → returns LLM text, attempts=1, passed=true', async () => { + const generate: VoiceGateGenerator = async () => 'You got 2 of 7 macro calls right last year — clear pattern.'; + const result = await gateVoice({ + mode: 'pattern_statement', + generate, + judge: passJudge, + templateFallback: { fn: patternStatementTemplate, slots: defaultSlots }, + }); + expect(result.passed).toBe(true); + expect(result.attempts).toBe(1); + expect(result.text).toContain('macro calls right'); + }); + + test('first rejected, second passes → attempts=2, passed=true', async () => { + let calls = 0; + const generate: VoiceGateGenerator = async () => { + calls++; + return calls === 1 ? 'Per analysis, results show...' : 'You got 2 of 7 right.'; + }; + let judgeCalls = 0; + const judge: VoiceGateJudge = async () => { + judgeCalls++; + return judgeCalls === 1 + ? { verdict: 'academic', reason: 'starts with "per analysis"' } + : { verdict: 'conversational', reason: 'second-person and concrete' }; + }; + const result = await gateVoice({ + mode: 'pattern_statement', + generate, + judge, + templateFallback: { fn: patternStatementTemplate, slots: defaultSlots }, + }); + expect(result.passed).toBe(true); + expect(result.attempts).toBe(2); + expect(result.text).toBe('You got 2 of 7 right.'); + }); + + test('feedback from failed attempt 1 reaches generator on attempt 2', async () => { + let receivedFeedback: string | undefined; + let calls = 0; + const generate: VoiceGateGenerator = async ({ attempt, feedback }) => { + calls++; + if (attempt === 2) receivedFeedback = feedback; + return `attempt ${calls}`; + }; + let judgeCalls = 0; + const judge: VoiceGateJudge = async () => { + judgeCalls++; + return judgeCalls === 1 + ? { verdict: 'academic', reason: 'too short' } + : { verdict: 'conversational', reason: '' }; + }; + await gateVoice({ + mode: 'nudge', + generate, + judge, + templateFallback: { + fn: nudgeTemplate, + slots: { + domain: 'macro', + conviction: 0.8, + nRecentMisses: 2, + nRecentTotal: 3, + hushPattern: 'over-confident-macro', + }, + }, + }); + expect(receivedFeedback).toBe('too short'); + }); +}); + +describe('gateVoice — fallback path', () => { + test('both attempts rejected → template fallback, passed=false, attempts=2', async () => { + const generate: VoiceGateGenerator = async () => 'Per our analysis, the data indicates...'; + const result = await gateVoice({ + mode: 'pattern_statement', + generate, + judge: rejectJudge, + templateFallback: { fn: patternStatementTemplate, slots: defaultSlots }, + }); + expect(result.passed).toBe(false); + expect(result.attempts).toBe(2); + expect(result.text).toContain('macro tech'); + expect(result.text).toContain('over-confident'); + expect(result.lastReason).toBe('too clinical'); + expect(result.templateSlots).toEqual(defaultSlots); + }); + + test('generator throws on both attempts → template fallback, NO judge calls', async () => { + let judgeCalls = 0; + const generate: VoiceGateGenerator = async () => { + throw new Error('LLM timeout'); + }; + const judge: VoiceGateJudge = async () => { + judgeCalls++; + return { verdict: 'conversational', reason: '' }; + }; + const result = await gateVoice({ + mode: 'pattern_statement', + generate, + judge, + templateFallback: { fn: patternStatementTemplate, slots: defaultSlots }, + }); + expect(result.passed).toBe(false); + expect(judgeCalls).toBe(0); + expect(result.lastReason).toBe('LLM timeout'); + }); + + test('empty generation counts as a failed attempt + falls through', async () => { + const generate: VoiceGateGenerator = async () => ''; + const result = await gateVoice({ + mode: 'pattern_statement', + generate, + judge: passJudge, // judge would pass but generation is empty + templateFallback: { fn: patternStatementTemplate, slots: defaultSlots }, + }); + expect(result.passed).toBe(false); + expect(result.lastReason).toBe('empty_generation'); + }); + + test('parse_failed judge output is treated as academic → fallback fires', async () => { + const generate: VoiceGateGenerator = async () => 'Some candidate.'; + // Inject a judge that simulates the parse-failure path: returns the + // 'academic' / 'parse_failed' verdict the production parser would emit + // when the Haiku call returns garbage. + const judge: VoiceGateJudge = async () => ({ verdict: 'academic', reason: 'parse_failed' }); + const result = await gateVoice({ + mode: 'pattern_statement', + generate, + judge, + templateFallback: { fn: patternStatementTemplate, slots: defaultSlots }, + }); + expect(result.passed).toBe(false); + expect(result.lastReason).toBe('parse_failed'); + }); + + test('maxAttempts override changes the retry count', async () => { + let calls = 0; + const generate: VoiceGateGenerator = async () => { + calls++; + return `attempt ${calls}`; + }; + await gateVoice({ + mode: 'pattern_statement', + generate, + judge: rejectJudge, + maxAttempts: 4, + templateFallback: { fn: patternStatementTemplate, slots: defaultSlots }, + }); + expect(calls).toBe(4); + }); +}); + +// ─── Mode parity ──────────────────────────────────────────────────── + +describe('VoiceGateMode parity', () => { + test('every mode has a default rubric', () => { + for (const mode of VOICE_GATE_MODES) { + expect(DEFAULT_RUBRICS[mode]).toBeDefined(); + expect(DEFAULT_RUBRICS[mode].length).toBeGreaterThan(50); + } + }); + + test('every mode rubric explicitly forbids preachy/clinical voice', () => { + // Anchors the cross-cutting voice rule: each mode's rubric must + // mention something about NOT sounding academic / preachy / clinical. + for (const mode of VOICE_GATE_MODES) { + const rubric = DEFAULT_RUBRICS[mode].toLowerCase(); + const hasGuard = + rubric.includes('preachy') || + rubric.includes('clinical') || + rubric.includes('jargon') || + rubric.includes('marketing') || + rubric.includes('corporate') || + rubric.includes('condescending') || + rubric.includes('doctor') || + rubric.includes('hr'); + expect(hasGuard).toBe(true); + } + }); +}); + +// ─── Templates (deterministic) ────────────────────────────────────── + +describe('voice-gate templates', () => { + test('patternStatementTemplate is deterministic for fixed slots', () => { + const out = patternStatementTemplate({ + domain: 'macro tech', + nRight: 2, + nWrong: 5, + direction: 'over-confident', + }); + expect(out).toBe('Your macro tech calls have a over-confident record — 2 of 7 held up.'); + }); + + test('patternStatementTemplate handles empty resolved set', () => { + const out = patternStatementTemplate({ domain: 'X', nRight: 0, nWrong: 0 }); + expect(out).toContain('Not enough resolved X calls yet'); + }); + + test('nudgeTemplate includes the hush command', () => { + const out = nudgeTemplate({ + domain: 'macro', + conviction: 0.85, + nRecentMisses: 2, + nRecentTotal: 3, + hushPattern: 'over-confident-macro', + }); + expect(out).toContain('gbrain takes nudge --hush over-confident-macro'); + expect(out).toContain('0.85'); + expect(out).toContain('2 of 3 missed'); + }); + + test('forecastBlurbTemplate flags insufficient data when n<5', () => { + const out = forecastBlurbTemplate({ + domain: 'macro', + conviction: 0.7, + bucketBrier: 0.31, + overallBrier: 0.18, + bucketN: 3, + }); + expect(out).toContain('Forecast unavailable'); + expect(out).toContain('3 resolved'); + }); + + test('forecastBlurbTemplate names comparison vs overall when n>=5', () => { + const out = forecastBlurbTemplate({ + domain: 'macro', + conviction: 0.7, + bucketBrier: 0.31, + overallBrier: 0.18, + bucketN: 7, + }); + expect(out).toContain('worse than your average'); + }); + + test('dashboardCaptionTemplate is concise', () => { + const out = dashboardCaptionTemplate({ surface: 'Brier trend', fact: '0.18, improving from 0.22 90d ago' }); + expect(out).toBe('Brier trend: 0.18, improving from 0.22 90d ago'); + }); + + test('morningPulseTemplate skips pattern line when topPattern empty', () => { + const out = morningPulseTemplate({ brier: 0.18, trend: 'improving', topPattern: '' }); + expect(out).toContain('Brier 0.18'); + expect(out).toContain('improving'); + expect(out).not.toContain('Top pattern'); + }); +});