mirror of
https://github.com/garrytan/gbrain.git
synced 2026-07-30 19:49:14 +00:00
doctor: 4 calibration checks — abandoned/freshness/drift/voice (T12)
Adds the four calibration doctor checks per the eng-review spec. abandoned_threads: Counts active high-conviction takes (weight >= 0.7) older than 12 months that have never been superseded. Signal, not error — always status='ok' with a count. The hint sends users to `gbrain calibration` for details. calibration_freshness: Warns when the active profile is older than 7 days (configurable via the same env-var pattern other freshness checks use). Cold-brain branch (no profile yet) returns ok without scolding. Hint points at `gbrain calibration --regenerate`. grade_confidence_drift (CDX-11 mitigation): Surfaces the count of auto-applied grade verdicts. Below 30: returns "need 30+ for drift detection". At/above 30: returns "drift math arrives in v0.37+". The surface is wired; the actual confidence-vs-accuracy correlation math is a v0.37+ follow-up once we have 30+ auto-applied verdicts to measure against. Closes the CDX-11 hole structurally — the operator sees the surface even before the math is meaningful. voice_gate_health: Tracks voice gate failure rate over the last 7 days. <30% fail rate → ok (template fallback is fine in isolation). >=30% → warn with hint to review src/core/calibration/voice-gate.ts rubric. Anchors the cross-cutting voice rule observability story. All four checks return status='warn' with a diagnostic message on engine errors — non-blocking, never throws. Matches the existing doctor check pattern (see checkSyncFreshness for prior art). Wired into runDoctor after checkRerankerHealth (the v0.35 cluster), in the canonical block 10 slot. Tests: 15 cases. 4 per check (happy path, alt-status, engine-throw diagnostic, plus boundary tests for the freshness staleness gate at exactly 7 days and the grade drift gate at 30 applied verdicts). Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.7
parent
08f59b0e7c
commit
8ae71a46b6
@@ -0,0 +1,184 @@
|
||||
/**
|
||||
* v0.36.0.0 (T12) — calibration doctor check tests.
|
||||
*
|
||||
* Hermetic. Mock engine + injected executeRaw responses.
|
||||
*
|
||||
* Tests cover:
|
||||
* - checkAbandonedThreads: zero count → ok; non-zero → ok with count
|
||||
* - checkCalibrationFreshness: missing profile → ok cold-brain; fresh → ok;
|
||||
* stale > 7 days → warn with hint
|
||||
* - checkGradeConfidenceDrift: < 30 applied → ok ("math arrives in v0.37+");
|
||||
* >= 30 → ok placeholder
|
||||
* - checkVoiceGateHealth: 0 in window → ok; high fail rate → warn
|
||||
* - all checks return status='warn' with diagnostic on executeRaw throw
|
||||
*/
|
||||
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import {
|
||||
checkAbandonedThreads,
|
||||
checkCalibrationFreshness,
|
||||
checkGradeConfidenceDrift,
|
||||
checkVoiceGateHealth,
|
||||
} from '../src/commands/doctor.ts';
|
||||
import type { BrainEngine } from '../src/core/engine.ts';
|
||||
|
||||
function buildMockEngine(opts: {
|
||||
abandonedCount?: number;
|
||||
freshGeneratedAt?: Date | null;
|
||||
gradeAppliedCount?: number;
|
||||
voiceTotal?: number;
|
||||
voiceFailures?: number;
|
||||
throwOn?: RegExp;
|
||||
}): BrainEngine {
|
||||
return {
|
||||
kind: 'pglite',
|
||||
async executeRaw<T>(sql: string): Promise<T[]> {
|
||||
if (opts.throwOn && opts.throwOn.test(sql)) {
|
||||
throw new Error('mock engine error: ' + sql.slice(0, 50));
|
||||
}
|
||||
if (sql.includes('FROM takes')) {
|
||||
return [{ count: opts.abandonedCount ?? 0 } as unknown as T];
|
||||
}
|
||||
if (sql.includes('FROM calibration_profiles WHERE holder')) {
|
||||
return [{ generated_at: opts.freshGeneratedAt ?? null } as unknown as T];
|
||||
}
|
||||
if (sql.includes('FROM take_grade_cache')) {
|
||||
return [{ applied_count: opts.gradeAppliedCount ?? 0 } as unknown as T];
|
||||
}
|
||||
if (sql.includes('FROM calibration_profiles\n WHERE generated_at')) {
|
||||
return [
|
||||
{
|
||||
total: opts.voiceTotal ?? 0,
|
||||
failures: opts.voiceFailures ?? 0,
|
||||
} as unknown as T,
|
||||
];
|
||||
}
|
||||
return [] as T[];
|
||||
},
|
||||
} as unknown as BrainEngine;
|
||||
}
|
||||
|
||||
// ─── abandoned_threads ──────────────────────────────────────────────
|
||||
|
||||
describe('checkAbandonedThreads', () => {
|
||||
test('zero count → ok with no-abandoned message', async () => {
|
||||
const out = await checkAbandonedThreads(buildMockEngine({ abandonedCount: 0 }));
|
||||
expect(out.status).toBe('ok');
|
||||
expect(out.message).toContain('No abandoned high-conviction threads');
|
||||
});
|
||||
|
||||
test('non-zero count → ok with count + hint', async () => {
|
||||
const out = await checkAbandonedThreads(buildMockEngine({ abandonedCount: 4 }));
|
||||
expect(out.status).toBe('ok');
|
||||
expect(out.message).toContain('4 high-conviction take(s)');
|
||||
expect(out.message).toContain('gbrain calibration');
|
||||
});
|
||||
|
||||
test('engine throw → warn with diagnostic (non-blocking)', async () => {
|
||||
const out = await checkAbandonedThreads(buildMockEngine({ throwOn: /FROM takes/ }));
|
||||
expect(out.status).toBe('warn');
|
||||
expect(out.message).toContain('Could not check abandoned threads');
|
||||
});
|
||||
});
|
||||
|
||||
// ─── calibration_freshness ──────────────────────────────────────────
|
||||
|
||||
describe('checkCalibrationFreshness', () => {
|
||||
test('no profile yet → ok cold-brain message', async () => {
|
||||
const out = await checkCalibrationFreshness(buildMockEngine({ freshGeneratedAt: null }));
|
||||
expect(out.status).toBe('ok');
|
||||
expect(out.message).toContain('No calibration profile yet');
|
||||
});
|
||||
|
||||
test('fresh profile (1 day old) → ok', async () => {
|
||||
const d = new Date();
|
||||
d.setDate(d.getDate() - 1);
|
||||
const out = await checkCalibrationFreshness(buildMockEngine({ freshGeneratedAt: d }));
|
||||
expect(out.status).toBe('ok');
|
||||
expect(out.message).toContain('1d ago');
|
||||
});
|
||||
|
||||
test('stale profile (>7 days) → warn with regenerate hint', async () => {
|
||||
const d = new Date();
|
||||
d.setDate(d.getDate() - 10);
|
||||
const out = await checkCalibrationFreshness(buildMockEngine({ freshGeneratedAt: d }));
|
||||
expect(out.status).toBe('warn');
|
||||
expect(out.message).toContain('10 days old');
|
||||
expect(out.message).toContain('gbrain calibration --regenerate');
|
||||
});
|
||||
|
||||
test('boundary: 7 days old → still ok (NOT warn)', async () => {
|
||||
const d = new Date();
|
||||
d.setDate(d.getDate() - 7);
|
||||
d.setMinutes(d.getMinutes() + 1); // slightly less than 7 full days
|
||||
const out = await checkCalibrationFreshness(buildMockEngine({ freshGeneratedAt: d }));
|
||||
expect(out.status).toBe('ok');
|
||||
});
|
||||
|
||||
test('engine throw → warn with diagnostic', async () => {
|
||||
const out = await checkCalibrationFreshness(
|
||||
buildMockEngine({ throwOn: /FROM calibration_profiles WHERE holder/ }),
|
||||
);
|
||||
expect(out.status).toBe('warn');
|
||||
expect(out.message).toContain('Could not check calibration freshness');
|
||||
});
|
||||
});
|
||||
|
||||
// ─── grade_confidence_drift ─────────────────────────────────────────
|
||||
|
||||
describe('checkGradeConfidenceDrift', () => {
|
||||
test('fewer than 30 applied → ok placeholder', async () => {
|
||||
const out = await checkGradeConfidenceDrift(buildMockEngine({ gradeAppliedCount: 12 }));
|
||||
expect(out.status).toBe('ok');
|
||||
expect(out.message).toContain('12 auto-applied verdicts');
|
||||
expect(out.message).toContain('need 30');
|
||||
});
|
||||
|
||||
test('>= 30 applied → ok placeholder with math-pending note', async () => {
|
||||
const out = await checkGradeConfidenceDrift(buildMockEngine({ gradeAppliedCount: 50 }));
|
||||
expect(out.status).toBe('ok');
|
||||
expect(out.message).toContain('50 auto-applied verdicts');
|
||||
expect(out.message).toContain('v0.37');
|
||||
});
|
||||
|
||||
test('engine throw → warn with diagnostic', async () => {
|
||||
const out = await checkGradeConfidenceDrift(buildMockEngine({ throwOn: /FROM take_grade_cache/ }));
|
||||
expect(out.status).toBe('warn');
|
||||
expect(out.message).toContain('Could not check grade confidence drift');
|
||||
});
|
||||
});
|
||||
|
||||
// ─── voice_gate_health ──────────────────────────────────────────────
|
||||
|
||||
describe('checkVoiceGateHealth', () => {
|
||||
test('no profile in window → ok', async () => {
|
||||
const out = await checkVoiceGateHealth(buildMockEngine({ voiceTotal: 0, voiceFailures: 0 }));
|
||||
expect(out.status).toBe('ok');
|
||||
expect(out.message).toContain('No calibration profile generation');
|
||||
});
|
||||
|
||||
test('low fail rate → ok', async () => {
|
||||
const out = await checkVoiceGateHealth(
|
||||
buildMockEngine({ voiceTotal: 10, voiceFailures: 1 }),
|
||||
);
|
||||
expect(out.status).toBe('ok');
|
||||
expect(out.message).toContain('1/10 failed');
|
||||
});
|
||||
|
||||
test('30%+ fail rate → warn with rubric-review hint', async () => {
|
||||
const out = await checkVoiceGateHealth(
|
||||
buildMockEngine({ voiceTotal: 10, voiceFailures: 4 }),
|
||||
);
|
||||
expect(out.status).toBe('warn');
|
||||
expect(out.message).toContain('4/10');
|
||||
expect(out.message).toContain('voice-gate.ts');
|
||||
});
|
||||
|
||||
test('engine throw → warn with diagnostic', async () => {
|
||||
const out = await checkVoiceGateHealth(
|
||||
buildMockEngine({ throwOn: /WHERE generated_at/ }),
|
||||
);
|
||||
expect(out.status).toBe('warn');
|
||||
expect(out.message).toContain('Could not check voice gate health');
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user