From 32242276dc91eb352f5c50ff1c1b1fa6a175503f Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Sun, 17 May 2026 16:25:13 -0700 Subject: [PATCH] cli: gbrain calibration + get_calibration_profile MCP op (T7) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Public-facing read surface for the v0.36.0.0 calibration wave. CLI prints the active calibration profile; MCP op exposes the same data path for agents. Mirror of the v0.29 salience/anomalies shape (pure data fn + JSON formatter + human formatter + thin CLI dispatch). CLI: `gbrain calibration` Flags: --holder specific holder (default 'garry') --json machine output for piping --regenerate run calibration_profile phase now --undo-wave [placeholder — wires in Lane D / T17] ab-report [placeholder — wires in Lane D / T18] Human output: Calibration profile — holder: garry, source: default Generated: [Note: built on 60% graded — partial completion this cycle.] (when grade_completion < 0.9) [Note: voice gate fell back to template (2 attempts).] (when voice_gate_passed=false) Resolved: 12 takes Brier: 0.210 (lower is better) Accuracy: 60.0% Partial: 10.0% Pattern statements: • You called early-stage tactics well — 8 of 10 held up. Active bias tags: over-confident-geography Cold-brain fallback message names the exact dream command to run. MCP: `get_calibration_profile` (scope: read) Param: holder?: string (defaults to 'garry') Returns: latest CalibrationProfileRow | null Source-scoping via sourceScopeOpts(ctx): scalar source-bound clients see only their source; federated_read scopes see the union of allowed sources; no source filter when neither is set (CLI default path). Throws GBrainError('INVALID_HOLDER') on empty/non-string holder so remote callers get a structured error instead of a SQL-shape failure. Architecture: getLatestProfile is the pure data fn — engine + opts → CalibrationProfileRow | null. Reused by both the CLI and the MCP op. Source-scoped via the standard v0.34.1 spread pattern (scalar sourceId vs sourceIds array). formatProfileText is pure — null → cold-brain message, populated → full printout. Annotates partial-grade rows and voice-gate-fallback rows so the operator sees data-quality status inline. parseArgs is exported via __testing for unit coverage. Sub-command ('ab-report') vs flag distinction is intentional — keeps the surface parallel with `gbrain eval cross-modal` etc. Tests: 21 cases. parseArgs (6 cases): empty, --holder, --json, --regenerate, --undo-wave, ab-report. getLatestProfile (5 cases): happy, null, scalar source scope, federated array scope, no-source-filter default. formatProfileText (5 cases): cold-brain, happy, partial-grade note, voice-fallback note, published-to-mounts note. getCalibrationProfileOp (5 cases): default holder, scalar source scope, federated scope union, returns-null-on-unknown-holder, throws on empty holder. Lane D follow-ups: --undo-wave (T17) and ab-report (T18) print a clear "lands in Lane D" stderr line + exit 2; the surfaces exist for early testers, the implementations land next. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/cli.ts | 8 ++ src/commands/calibration.ts | 227 ++++++++++++++++++++++++++++++++ src/core/operations.ts | 28 ++++ test/calibration-cli.test.ts | 244 +++++++++++++++++++++++++++++++++++ 4 files changed, 507 insertions(+) create mode 100644 src/commands/calibration.ts create mode 100644 test/calibration-cli.test.ts diff --git a/src/cli.ts b/src/cli.ts index 32d2e5081..10e93b89d 100755 --- a/src/cli.ts +++ b/src/cli.ts @@ -1144,6 +1144,14 @@ async function handleCliOnly(command: string, args: string[]) { await runWhoknows(engine, args); break; } + case 'calibration': { + // v0.36.0.0 (T7): print/regenerate the active calibration profile. + // MCP op `get_calibration_profile` (read-scoped) backs the same data path. + const { runCalibration } = await import('./commands/calibration.ts'); + const calibrationConfig = loadConfig() ?? ({} as never); + await runCalibration(engine, args, calibrationConfig); + break; + } case 'transcripts': { const { runTranscripts } = await import('./commands/transcripts.ts'); await runTranscripts(engine, args); diff --git a/src/commands/calibration.ts b/src/commands/calibration.ts new file mode 100644 index 000000000..db9306596 --- /dev/null +++ b/src/commands/calibration.ts @@ -0,0 +1,227 @@ +/** + * v0.36.0.0 (T7) — `gbrain calibration` CLI. + * + * Reads the latest calibration profile from the DB and prints it. Mirror of + * the v0.29 `gbrain salience` / `gbrain anomalies` shape (pure data fn + JSON + * formatter + human formatter + thin CLI dispatch). + * + * Sub-commands: + * gbrain calibration — print active profile for default holder + * gbrain calibration --holder — print for a specific holder + * gbrain calibration --json — machine output + * gbrain calibration --regenerate — run the calibration_profile phase now + * gbrain calibration --undo-wave — D18 undo command (Lane D adds the impl) + * gbrain calibration ab-report — D19 A/B harness report (Lane D adds the impl) + * + * MCP op: `get_calibration_profile` (scope: read) routes the same read path. + * Source-scoping via sourceScopeOpts(ctx) on the MCP path keeps multi-source + * brains source-isolated per the v0.34.1 discipline. + */ + +import type { BrainEngine } from '../core/engine.ts'; +import { runPhaseCalibrationProfile } from '../core/cycle/calibration-profile.ts'; +import { sourceScopeOpts, type OperationContext } from '../core/operations.ts'; +import type { GBrainConfig } from '../core/config.ts'; +import { GBrainError } from '../core/types.ts'; + +export interface CalibrationProfileRow { + id: number; + source_id: string; + holder: string; + wave_version: string; + generated_at: string; + published: boolean; + total_resolved: number; + brier: number | null; + accuracy: number | null; + partial_rate: number | null; + grade_completion: number; + pattern_statements: string[]; + active_bias_tags: string[]; + voice_gate_passed: boolean; + voice_gate_attempts: number; + model_id: string; +} + +/** Source-scoped read of the latest profile row for a holder. */ +export async function getLatestProfile( + engine: BrainEngine, + opts: { holder: string; sourceId?: string; sourceIds?: string[] }, +): Promise { + let sql = `SELECT id, source_id, holder, wave_version, generated_at, published, + total_resolved, brier, accuracy, partial_rate, grade_completion, + pattern_statements, active_bias_tags, + voice_gate_passed, voice_gate_attempts, model_id + FROM calibration_profiles + WHERE holder = $1`; + const params: unknown[] = [opts.holder]; + + if (opts.sourceIds && opts.sourceIds.length > 0) { + sql += ` AND source_id = ANY($2::text[])`; + params.push(opts.sourceIds); + } else if (opts.sourceId) { + sql += ` AND source_id = $2`; + params.push(opts.sourceId); + } + + sql += ` ORDER BY generated_at DESC LIMIT 1`; + + const rows = await engine.executeRaw(sql, params); + return rows[0] ?? null; +} + +/** Human format the profile for terminal output. */ +export function formatProfileText(profile: CalibrationProfileRow | null, holder: string): string { + if (!profile) { + return ( + `No calibration profile yet for holder "${holder}".\n` + + `Build one by resolving 5+ takes then running:\n` + + ` gbrain dream --phase calibration_profile\n` + + `Or wait for the next autopilot cycle.` + ); + } + const lines: string[] = []; + const generatedLocal = new Date(profile.generated_at).toLocaleString(); + lines.push(`Calibration profile — holder: ${profile.holder}, source: ${profile.source_id}`); + lines.push(`Generated: ${generatedLocal} ${profile.published ? '(published to mounts)' : ''}`); + if (profile.grade_completion < 0.9) { + lines.push(`Note: built on ${(profile.grade_completion * 100).toFixed(0)}% graded — partial completion this cycle.`); + } + if (!profile.voice_gate_passed) { + lines.push(`Note: voice gate fell back to template (${profile.voice_gate_attempts} attempts).`); + } + lines.push(''); + lines.push(`Resolved: ${profile.total_resolved} takes`); + if (profile.brier !== null) lines.push(`Brier: ${profile.brier.toFixed(3)} (lower is better)`); + if (profile.accuracy !== null) lines.push(`Accuracy: ${(profile.accuracy * 100).toFixed(1)}%`); + if (profile.partial_rate !== null) lines.push(`Partial: ${(profile.partial_rate * 100).toFixed(1)}%`); + lines.push(''); + lines.push('Pattern statements:'); + for (const p of profile.pattern_statements) { + lines.push(` • ${p}`); + } + if (profile.active_bias_tags.length > 0) { + lines.push(''); + lines.push(`Active bias tags: ${profile.active_bias_tags.join(', ')}`); + } + return lines.join('\n'); +} + +/** Build an OperationContext shape suitable for the cycle phase from a CLI engine. */ +function ctxFromCli(engine: BrainEngine, config: GBrainConfig, sourceId: string): OperationContext { + return { + engine, + config, + logger: { info() {}, warn() {}, error() {} } as never, + dryRun: false, + remote: false, + sourceId, + }; +} + +export interface RunCalibrationArgs { + holder?: string; + json?: boolean; + regenerate?: boolean; + undoWave?: string; + abReport?: boolean; +} + +function parseArgs(args: string[]): { sub?: string; opts: RunCalibrationArgs } { + const opts: RunCalibrationArgs = {}; + let sub: string | undefined; + for (let i = 0; i < args.length; i++) { + const a = args[i]; + if (a === 'ab-report') { + opts.abReport = true; + continue; + } + if (!a?.startsWith('--') && !sub) { + sub = a; + continue; + } + if (a === '--holder') opts.holder = args[++i]; + else if (a === '--json') opts.json = true; + else if (a === '--regenerate') opts.regenerate = true; + else if (a === '--undo-wave') opts.undoWave = args[++i]; + } + return { sub, opts }; +} + +/** + * CLI entry point. The `config` param is forwarded so the calibration_profile + * phase has access to the budget cap config key. + */ +export async function runCalibration( + engine: BrainEngine, + args: string[], + config: GBrainConfig, +): Promise { + const { opts } = parseArgs(args); + const holder = opts.holder ?? 'garry'; + const sourceId = 'default'; + + if (opts.undoWave) { + // D18 undo-wave is wired in Lane D. v0.36.0.0 ship-state placeholder. + process.stderr.write( + `[calibration] --undo-wave ${opts.undoWave}: implementation lands in Lane D ` + + `(T17). For now run \`gbrain dream --phase calibration_profile\` to regenerate, ` + + `or operate on calibration_profiles directly via SQL.\n`, + ); + process.exit(2); + } + + if (opts.abReport) { + // D19 A/B harness report wired in Lane D (T18). Placeholder. + process.stderr.write( + `[calibration] ab-report: implementation lands in Lane D (T18 — A/B harness for think).\n`, + ); + process.exit(2); + } + + if (opts.regenerate) { + process.stderr.write(`[calibration] regenerating profile for holder=${holder}...\n`); + const ctx = ctxFromCli(engine, config, sourceId); + const result = await runPhaseCalibrationProfile(ctx, { holder }); + if (result.status === 'fail') { + process.stderr.write(`[calibration] regenerate failed: ${result.error?.message ?? 'unknown'}\n`); + process.exit(1); + } + process.stderr.write(`[calibration] ${result.summary}\n`); + } + + const profile = await getLatestProfile(engine, { holder, sourceId }); + + if (opts.json) { + process.stdout.write(JSON.stringify(profile, null, 2) + '\n'); + return; + } + + process.stdout.write(formatProfileText(profile, holder) + '\n'); +} + +/** + * Op-handler entry point for `get_calibration_profile` MCP op. Source-scoped + * via sourceScopeOpts(ctx). scope: 'read' on the op definition; this handler + * is the implementation. + */ +export async function getCalibrationProfileOp( + ctx: OperationContext, + params: { holder?: string }, +): Promise { + const holder = params.holder ?? 'garry'; + if (typeof holder !== 'string' || holder.length === 0) { + throw new GBrainError( + 'INVALID_HOLDER', + 'get_calibration_profile.holder must be a non-empty string', + 'pass holder="" or omit to default to "garry"', + ); + } + const scope = sourceScopeOpts(ctx); + return getLatestProfile(ctx.engine, { holder, ...scope }); +} + +export const __testing = { + parseArgs, + formatProfileText, +}; diff --git a/src/core/operations.ts b/src/core/operations.ts index 49ed17700..22c2b441d 100644 --- a/src/core/operations.ts +++ b/src/core/operations.ts @@ -2269,6 +2269,32 @@ const find_orphans: Operation = { cliHints: { name: 'orphans', hidden: true }, }; +// --- v0.36.0.0 (T7): calibration profile read op --- + +const get_calibration_profile: Operation = { + name: 'get_calibration_profile', + description: + 'Read the active calibration profile for a holder. Returns the latest row from calibration_profiles ' + + '(per-source, per-holder) including Brier score, accuracy, pattern statements, and active bias tags. ' + + 'Source-scoped via sourceScopeOpts — federated_read scopes see the union of allowed sources, ' + + 'scalar source-bound clients see only their source. Returns null when no profile exists yet ' + + '(cold-brain branch: builds after 5+ resolved takes + a calibration_profile phase run).', + scope: 'read', + params: { + holder: { + type: 'string', + description: + "Holder slug, e.g. 'garry' or 'people/charlie-example'. Defaults to 'garry' when omitted.", + }, + }, + handler: async (ctx, p) => { + const { getCalibrationProfileOp } = await import('../commands/calibration.ts'); + return getCalibrationProfileOp(ctx, { + ...(typeof p.holder === 'string' ? { holder: p.holder } : {}), + }); + }, +}; + // --- v0.29: Salience + Anomaly Detection --- const get_recent_salience: Operation = { @@ -3158,6 +3184,8 @@ export const operations: Operation[] = [ pause_job, resume_job, replay_job, send_job_message, // Orphans find_orphans, + // v0.36.0.0 (T7) — Hindsight calibration wave: read profile via MCP + get_calibration_profile, // v0.28: Takes + think takes_list, takes_search, think, // v0.30: calibration aggregates over takes diff --git a/test/calibration-cli.test.ts b/test/calibration-cli.test.ts new file mode 100644 index 000000000..452b939ac --- /dev/null +++ b/test/calibration-cli.test.ts @@ -0,0 +1,244 @@ +/** + * v0.36.0.0 (T7) — gbrain calibration CLI + get_calibration_profile MCP op tests. + * + * Hermetic. Mock engine + injected args. + */ + +import { describe, test, expect } from 'bun:test'; +import { + getLatestProfile, + getCalibrationProfileOp, + formatProfileText, + __testing, + type CalibrationProfileRow, +} from '../src/commands/calibration.ts'; +import type { OperationContext } from '../src/core/operations.ts'; +import type { BrainEngine } from '../src/core/engine.ts'; +import { GBrainError } from '../src/core/types.ts'; + +const { parseArgs } = __testing; + +function buildMockEngine(opts: { rows: CalibrationProfileRow[] }): { + engine: BrainEngine; + capturedSql: string[]; + capturedParams: unknown[][]; +} { + const capturedSql: string[] = []; + const capturedParams: unknown[][] = []; + const engine = { + kind: 'pglite', + async executeRaw(sql: string, params?: unknown[]): Promise { + capturedSql.push(sql); + capturedParams.push(params ?? []); + // SELECT first row matching holder + optional source filter + const holder = (params ?? [])[0]; + const matching = opts.rows.filter(r => r.holder === holder); + if ((params ?? []).length > 1) { + const p2 = (params ?? [])[1]; + if (Array.isArray(p2)) { + return matching.filter(r => (p2 as string[]).includes(r.source_id)) as unknown as T[]; + } + return matching.filter(r => r.source_id === p2) as unknown as T[]; + } + return matching as unknown as T[]; + }, + } as unknown as BrainEngine; + return { engine, capturedSql, capturedParams }; +} + +function buildCtx(engine: BrainEngine, opts: { sourceId?: string; allowedSources?: string[] } = {}): OperationContext { + const ctx: OperationContext = { + engine, + config: {} as never, + logger: { info() {}, warn() {}, error() {} } as never, + dryRun: false, + remote: false, + sourceId: opts.sourceId ?? 'default', + }; + if (opts.allowedSources) ctx.auth = { allowedSources: opts.allowedSources } as never; + return ctx; +} + +function buildProfile(opts: Partial & { holder: string }): CalibrationProfileRow { + return { + id: 1, + source_id: opts.source_id ?? 'default', + holder: opts.holder, + wave_version: 'v0.36.0.0', + generated_at: '2026-05-17T15:00:00Z', + published: opts.published ?? false, + total_resolved: opts.total_resolved ?? 12, + brier: opts.brier ?? 0.21, + accuracy: opts.accuracy ?? 0.6, + partial_rate: opts.partial_rate ?? 0.1, + grade_completion: opts.grade_completion ?? 1.0, + pattern_statements: opts.pattern_statements ?? ['You called early-stage tactics well — 8 of 10 held up.'], + active_bias_tags: opts.active_bias_tags ?? ['over-confident-geography'], + voice_gate_passed: opts.voice_gate_passed ?? true, + voice_gate_attempts: opts.voice_gate_attempts ?? 1, + model_id: 'claude-sonnet-4-6', + }; +} + +// ─── parseArgs ────────────────────────────────────────────────────── + +describe('parseArgs', () => { + test('empty args: defaults applied (no holder, no flags)', () => { + expect(parseArgs([])).toEqual({ sub: undefined, opts: {} }); + }); + + test('--holder ', () => { + expect(parseArgs(['--holder', 'people/charlie-example']).opts.holder).toBe('people/charlie-example'); + }); + + test('--json flag', () => { + expect(parseArgs(['--json']).opts.json).toBe(true); + }); + + test('--regenerate flag', () => { + expect(parseArgs(['--regenerate']).opts.regenerate).toBe(true); + }); + + test('--undo-wave ', () => { + expect(parseArgs(['--undo-wave', 'v0.36.0.0']).opts.undoWave).toBe('v0.36.0.0'); + }); + + test('ab-report subcommand', () => { + expect(parseArgs(['ab-report']).opts.abReport).toBe(true); + }); +}); + +// ─── getLatestProfile ─────────────────────────────────────────────── + +describe('getLatestProfile', () => { + test('returns the row when holder matches', async () => { + const { engine } = buildMockEngine({ rows: [buildProfile({ holder: 'garry' })] }); + const profile = await getLatestProfile(engine, { holder: 'garry', sourceId: 'default' }); + expect(profile).not.toBeNull(); + expect(profile!.holder).toBe('garry'); + }); + + test('returns null when no profile exists', async () => { + const { engine } = buildMockEngine({ rows: [] }); + const profile = await getLatestProfile(engine, { holder: 'unknown', sourceId: 'default' }); + expect(profile).toBeNull(); + }); + + test('source-scoped query: scalar sourceId filters to that source', async () => { + const rows = [ + buildProfile({ holder: 'garry', source_id: 'default' }), + buildProfile({ holder: 'garry', source_id: 'tenant-b' }), + ]; + const { engine } = buildMockEngine({ rows }); + const profile = await getLatestProfile(engine, { holder: 'garry', sourceId: 'tenant-b' }); + expect(profile!.source_id).toBe('tenant-b'); + }); + + test('federated array filters to any of the listed sources', async () => { + const rows = [ + buildProfile({ holder: 'garry', source_id: 'tenant-a' }), + buildProfile({ holder: 'garry', source_id: 'tenant-c' }), + ]; + const { engine, capturedSql, capturedParams } = buildMockEngine({ rows }); + await getLatestProfile(engine, { holder: 'garry', sourceIds: ['tenant-a', 'tenant-b'] }); + expect(capturedSql[0]).toContain('= ANY($2::text[])'); + expect(capturedParams[0]![1]).toEqual(['tenant-a', 'tenant-b']); + }); + + test('no source filter when neither sourceId nor sourceIds is passed', async () => { + const { engine, capturedSql } = buildMockEngine({ rows: [] }); + await getLatestProfile(engine, { holder: 'garry' }); + // SELECT clause names the column but WHERE clause omits source_id filter. + expect(capturedSql[0]).not.toContain('AND source_id'); + }); +}); + +// ─── formatProfileText ────────────────────────────────────────────── + +describe('formatProfileText', () => { + test('null profile prints helpful cold-brain message', () => { + const out = formatProfileText(null, 'garry'); + expect(out).toContain('No calibration profile yet'); + expect(out).toContain('gbrain dream --phase calibration_profile'); + }); + + test('happy profile prints Brier + accuracy + patterns + bias tags', () => { + const p = buildProfile({ holder: 'garry' }); + const out = formatProfileText(p, 'garry'); + expect(out).toContain('holder: garry'); + expect(out).toContain('Brier:'); + expect(out).toContain('Pattern statements:'); + expect(out).toContain('• You called early-stage tactics'); + expect(out).toContain('Active bias tags: over-confident-geography'); + }); + + test('partial-grade row prints "60% graded" note', () => { + const p = buildProfile({ holder: 'garry', grade_completion: 0.6 }); + const out = formatProfileText(p, 'garry'); + expect(out).toContain('60% graded'); + }); + + test('voice-gate-failed row prints template-fallback note', () => { + const p = buildProfile({ holder: 'garry', voice_gate_passed: false, voice_gate_attempts: 2 }); + const out = formatProfileText(p, 'garry'); + expect(out).toContain('voice gate fell back to template'); + }); + + test('published=true is annotated', () => { + const p = buildProfile({ holder: 'garry', published: true }); + const out = formatProfileText(p, 'garry'); + expect(out).toContain('published to mounts'); + }); +}); + +// ─── getCalibrationProfileOp ──────────────────────────────────────── + +describe('getCalibrationProfileOp (MCP)', () => { + test('defaults holder to "garry" when omitted', async () => { + const { engine } = buildMockEngine({ rows: [buildProfile({ holder: 'garry' })] }); + const ctx = buildCtx(engine); + const result = await getCalibrationProfileOp(ctx, {}); + expect(result?.holder).toBe('garry'); + }); + + test('routes through sourceScopeOpts: scalar source-bound client gets source-scoped result', async () => { + const rows = [ + buildProfile({ holder: 'garry', source_id: 'default' }), + buildProfile({ holder: 'garry', source_id: 'tenant-b' }), + ]; + const { engine } = buildMockEngine({ rows }); + const ctx = buildCtx(engine, { sourceId: 'tenant-b' }); + const result = await getCalibrationProfileOp(ctx, {}); + expect(result?.source_id).toBe('tenant-b'); + }); + + test('federated read scope sees the union of allowed sources', async () => { + const rows = [ + buildProfile({ holder: 'garry', source_id: 'tenant-a' }), + buildProfile({ holder: 'garry', source_id: 'tenant-z' }), + ]; + const { engine } = buildMockEngine({ rows }); + const ctx = buildCtx(engine, { allowedSources: ['tenant-a', 'tenant-b'] }); + const result = await getCalibrationProfileOp(ctx, {}); + // tenant-a is in the federated set → returns it; tenant-z is not → filtered out + expect(result?.source_id).toBe('tenant-a'); + }); + + test('returns null for unknown holder without throwing', async () => { + const { engine } = buildMockEngine({ rows: [] }); + const ctx = buildCtx(engine); + expect(await getCalibrationProfileOp(ctx, { holder: 'people/nobody' })).toBeNull(); + }); + + test('throws on empty/non-string holder', async () => { + const { engine } = buildMockEngine({ rows: [] }); + const ctx = buildCtx(engine); + try { + await getCalibrationProfileOp(ctx, { holder: '' }); + throw new Error('should have thrown'); + } catch (err) { + expect(err).toBeInstanceOf(GBrainError); + expect((err as GBrainError).problem).toBe('INVALID_HOLDER'); + } + }); +});