diff --git a/src/commands/autopilot.ts b/src/commands/autopilot.ts index 82979fb12..009c4ecbe 100644 --- a/src/commands/autopilot.ts +++ b/src/commands/autopilot.ts @@ -527,6 +527,9 @@ export async function runAutopilot(engine: BrainEngine, args: string[]) { process.on('SIGINT', () => { void shutdown('SIGINT'); }); let consecutiveErrors = 0; + // Parser-probe fixture warning is once-per-process, not once-per-cycle + // (compiled-binary installs have no source tree; don't spam the log). + let parserProbeFixtureWarned = false; // v0.37.7.0 #1162 — counter for consecutive reconnect failures. // Reset on every successful health probe or reconnect. Threshold // controlled by GBRAIN_AUTOPILOT_MAX_RECONNECT_FAILS env (default 30). @@ -1073,17 +1076,36 @@ export async function runAutopilot(engine: BrainEngine, args: string[]) { // loop. Probe runs even when cycleOk=false (probe may surface signal // explaining why the cycle is failing). try { - const probeEnabled = cfg?.autopilot?.nightly_quality_probe?.enabled === true; + const { resolveProbeEnabled, resolveProbeMaxUsd, runNightlyQualityProbe } = await import('../core/cycle/nightly-quality-probe.ts'); + // Dual-plane read: `gbrain config set` (what the doctor enable hint + // prints) writes the DB plane; ~/.gbrain/config.json is the fallback. + let dbEnabled: string | null = null; + let dbMaxUsd: string | null = null; + try { + dbEnabled = await engine.getConfig('autopilot.nightly_quality_probe.enabled'); + dbMaxUsd = await engine.getConfig('autopilot.nightly_quality_probe.max_usd'); + } catch { /* DB unavailable → file plane only */ } + const probeEnabled = resolveProbeEnabled(dbEnabled, cfg?.autopilot?.nightly_quality_probe?.enabled); if (probeEnabled) { - const { runNightlyQualityProbe } = await import('../core/cycle/nightly-quality-probe.ts'); const { runLongMemEvalForProbe, runCrossModalBatchForProbe } = await import('../core/cycle/nightly-probe-adapters.ts'); const { isAvailable } = await import('../core/ai/gateway.ts'); - const maxUsd = Number(cfg?.autopilot?.nightly_quality_probe?.max_usd ?? 5); + const { existsSync } = await import('node:fs'); + const { fileURLToPath } = await import('node:url'); + const { join } = await import('node:path'); + const maxUsd = resolveProbeMaxUsd(dbMaxUsd, cfg?.autopilot?.nightly_quality_probe?.max_usd); + // The committed fixture (test/fixtures/longmemeval-nightly.jsonl) + // lives in the gbrain PACKAGE, not the brain repo — repoPath is + // sync.repo_path (the user's brain), where the fixture never + // exists, so the probe error'd on every real install. Resolve the + // package root from the module location; keep repoPath as the + // fallback for setups that vendor the fixture into the brain repo. + const pkgRoot = fileURLToPath(new URL('../..', import.meta.url)); + const fixtureAtPkgRoot = existsSync(join(pkgRoot, 'test', 'fixtures', 'longmemeval-nightly.jsonl')); await runNightlyQualityProbe({ isEnabled: () => true, // already gated above; phase re-checks for defense-in-depth hasEmbeddingProvider: () => isAvailable('embedding'), resolveMaxUsd: () => maxUsd, - resolveRepoRoot: () => repoPath ?? gbrainHomePath('.'), + resolveRepoRoot: () => (fixtureAtPkgRoot ? pkgRoot : repoPath ?? gbrainHomePath('.')), runLongMemEval: runLongMemEvalForProbe, runCrossModalBatch: runCrossModalBatchForProbe, now: () => new Date(), @@ -1095,6 +1117,62 @@ export async function runAutopilot(engine: BrainEngine, args: string[]) { // informational; autopilot loop continues. } + // 4.6 — Nightly conversation-parser probe (v0.41.16.0 phase module; + // the scheduler wire-up was deferred at ship and is added here). Same + // posture as 4.5: the phase owns its gates (enabled/mode-gate, LLM + // key), the wiring owns invocation + the audit row, and a probe + // failure NEVER crashes the autopilot loop. Per D10 the probe is + // default-ON for search.mode=tokenmax, opt-in otherwise. + try { + const { runConversationParserNightlyProbe } = await import('../core/conversation-parser/nightly-probe.ts'); + const { logParserProbeEvent, parserProbeRanWithin } = await import('../core/audit-parser-probe.ts'); + const { isAvailable } = await import('../core/ai/gateway.ts'); + const { existsSync } = await import('node:fs'); + const { fileURLToPath } = await import('node:url'); + const { join } = await import('node:path'); + // Flag reads dual-plane: the DB row (`gbrain config set …`) wins, + // ~/.gbrain/config.json is the fallback. search.mode lives on the + // DB plane only (mode.ts owns it). + let parserDbEnabled: string | null = null; + let dbSearchMode: string | null = null; + try { + parserDbEnabled = await engine.getConfig('autopilot.conversation_parser_probe.enabled'); + dbSearchMode = await engine.getConfig('search.mode'); + } catch { /* DB unavailable → file plane only */ } + const parserEnabled = parserDbEnabled != null + ? parserDbEnabled === 'true' + : cfg?.autopilot?.conversation_parser_probe?.enabled === true; + const searchMode = dbSearchMode ?? ''; + // Fixtures are committed in the gbrain package (test/fixtures/…), + // NOT the brain repo — resolve from the module location. Compiled + // binaries carry no source tree: skip quietly instead of writing + // failure rows that would flip doctor to WARN on every binary install. + const pkgRoot = fileURLToPath(new URL('../..', import.meta.url)); + const fixturePath = join(pkgRoot, 'test', 'fixtures', 'conversation-formats', 'all.jsonl'); + const adversarialPath = join(pkgRoot, 'test', 'fixtures', 'conversation-formats', 'adversarial.jsonl'); + const shouldInvoke = parserEnabled || searchMode === 'tokenmax'; + if (shouldInvoke && existsSync(fixturePath) && existsSync(adversarialPath)) { + const result = await runConversationParserNightlyProbe({ + isEnabled: () => parserEnabled, + searchMode: () => searchMode, + hasLlmKey: () => isAvailable('chat'), + resolveFixturePath: () => fixturePath, + resolveAdversarialPath: () => adversarialPath, + now: () => new Date(), + shouldSkipForRateLimit: () => parserProbeRanWithin(24 * 60 * 60 * 1000), + }); + // rate_limited is a non-run: the loop ticks every few minutes, so + // logging every skip would flood the audit file with no-signal rows. + if (result.outcome !== 'rate_limited') logParserProbeEvent(result); + } else if (shouldInvoke && !parserProbeFixtureWarned) { + parserProbeFixtureWarned = true; + console.error(`[parser-probe] fixtures not found under ${pkgRoot}; skipping (probe needs a source-checkout install)`); + } + } catch (e) { + logError('autopilot.parser_probe', e); + // Informational, like 4.5: do NOT bump consecutiveErrors. + } + // Wait for next cycle await new Promise(r => setTimeout(r, interval * 1000)); } diff --git a/src/commands/doctor.ts b/src/commands/doctor.ts index c9457d318..eb41fb660 100644 --- a/src/commands/doctor.ts +++ b/src/commands/doctor.ts @@ -2960,6 +2960,54 @@ function _resolveSyncFreshnessHours(varName: string, fallback: number): number { * branch (disabled / enabled-no-events / enabled-all-pass / enabled-with-failures) * without spinning up the audit JSONL or a real config file. */ +/** + * Pure function form of the conversation_parser_probe_health check. + * Mirrors computeNightlyQualityProbeHealthCheck: skip-with-hint when the + * probe is off and silent, surface the last 7 days of audit events when + * it has run, WARN on any non-pass outcome. + * + * `effectiveEnabled` folds the D10 mode-gate in: explicitly enabled OR + * search.mode=tokenmax (where the probe is default-on). + */ +export function computeConversationParserProbeHealthCheck( + effectiveEnabled: boolean, + events: ReadonlyArray<{ outcome: string; ts: string; reason?: string }>, +): Check { + const name = 'conversation_parser_probe_health'; + if (!effectiveEnabled && events.length === 0) { + return { + name, + status: 'ok', + message: + 'disabled (opt-in; default-on only for search.mode=tokenmax). Enable with: ' + + '`gbrain config set autopilot.conversation_parser_probe.enabled true`', + }; + } + if (events.length === 0) { + return { + name, + status: 'ok', + message: 'enabled but no probe events in the last 7 days (next run by autopilot; fixtures require a source-checkout install).', + }; + } + const bad = events.filter(e => e.outcome !== 'pass'); + const latest = events[events.length - 1]!; + if (bad.length > 0) { + return { + name, + status: 'warn', + message: + `${bad.length}/${events.length} probe run(s) in the last 7 days did not pass; ` + + `latest: ${latest.outcome}${latest.reason ? ` (${latest.reason})` : ''}`, + }; + } + return { + name, + status: 'ok', + message: `${events.length} probe run(s) in the last 7 days, all pass (latest ${latest.ts}).`, + }; +} + export function computeNightlyQualityProbeHealthCheck( probeEnabled: boolean, events: ReadonlyArray<{ outcome: string; ts: string; detail?: string }>, @@ -4843,10 +4891,17 @@ export async function buildChecks( try { const { readRecentQualityProbeEvents } = await import('../core/audit-quality-probe.ts'); const { loadConfig } = await import('../core/config.ts'); + const { resolveProbeEnabled } = await import('../core/cycle/nightly-quality-probe.ts'); let probeEnabled = false; try { + // Dual-plane read, matching the autopilot gate: the DB row (what the + // enable hint's `gbrain config set` writes) wins; file plane fallback. + let dbVal: string | null = null; + try { + dbVal = engine ? await engine.getConfig('autopilot.nightly_quality_probe.enabled') : null; + } catch { /* DB unavailable → file plane only */ } const cfg = loadConfig(); - probeEnabled = Boolean((cfg as any)?.autopilot?.nightly_quality_probe?.enabled); + probeEnabled = resolveProbeEnabled(dbVal, (cfg as any)?.autopilot?.nightly_quality_probe?.enabled); } catch { /* config unavailable → treat as disabled */ } const events = readRecentQualityProbeEvents(7); const check = computeNightlyQualityProbeHealthCheck(probeEnabled, events); @@ -5030,19 +5085,29 @@ export async function buildChecks( // 3d.5 v0.41.13.0 — conversation_parser_probe_health. Mode-gated // per D10: ON when search.mode=tokenmax, opt-in for other modes. - // Surface the last 7 days of nightly-probe events; warn on FAIL / - // BUDGET_EXCEEDED / adversarial_false_positive. - // - // v0.41.13.0 ships the probe as opt-in (autopilot wiring deferred - // to T7 in the cathedral plan); this check skips with an enable - // hint until the probe has at least one audit event written. - checks.push({ - name: 'conversation_parser_probe_health', - status: 'ok', - message: - 'Skipped (nightly probe is opt-in; enable with ' + - '`gbrain config set autopilot.conversation_parser_probe.enabled true`)', - }); + // Surfaces the last 7 days of nightly-probe audit events; warn on any + // non-pass outcome (fail / budget_exceeded / adversarial_false_positive). + // (Until the autopilot wire-up this was a hardcoded "Skipped" stub.) + try { + const { readRecentParserProbeEvents } = await import('../core/audit-parser-probe.ts'); + let parserProbeEnabled = false; + try { + let dbVal: string | null = null; + let dbMode: string | null = null; + try { + dbVal = engine ? await engine.getConfig('autopilot.conversation_parser_probe.enabled') : null; + dbMode = engine ? await engine.getConfig('search.mode') : null; + } catch { /* DB unavailable → file plane only */ } + const { loadConfig } = await import('../core/config.ts'); + const fileVal = (loadConfig() as any)?.autopilot?.conversation_parser_probe?.enabled; + const flagOn = dbVal != null ? dbVal === 'true' : fileVal === true; + parserProbeEnabled = flagOn || dbMode === 'tokenmax'; + } catch { /* config unavailable → treat as disabled */ } + const parserEvents = readRecentParserProbeEvents(7); + checks.push(computeConversationParserProbeHealthCheck(parserProbeEnabled, parserEvents)); + } catch { + // Best-effort; audit-log read failure shouldn't stop doctor. + } // 3e. home_dir_in_worktree (v0.35.8.0). Walks up from `gbrainPath()` // looking for a `.git` directory OR file. If found, warns: `~/.gbrain/` diff --git a/src/commands/eval-cross-modal.ts b/src/commands/eval-cross-modal.ts index c466076b9..b06d69146 100644 --- a/src/commands/eval-cross-modal.ts +++ b/src/commands/eval-cross-modal.ts @@ -468,6 +468,14 @@ interface BatchRow { question_id: string; question: string; hypothesis: string; + /** + * Gold answer from the benchmark dataset, when the upstream eval emits + * it (eval-longmemeval does). Folded into the judge task so CORRECTNESS + * is verifiable — without it a judge panel that sees only + * {question, hypothesis} cannot validate a terse factual answer against + * a haystack it never saw. + */ + answer?: string; } /** @@ -581,6 +589,7 @@ function readBatchRows(path: string): BatchReadResult { question_id: typeof obj.question_id === 'string' ? obj.question_id : `line-${lineNo}`, question: obj.question, hypothesis: obj.hypothesis, + ...(typeof obj.answer === 'string' && obj.answer.length > 0 ? { answer: obj.answer } : {}), }); } if (summarySkipped > 0) { @@ -697,7 +706,11 @@ async function runBatchMode(parsed: ParsedArgs, opts: RunCrossModalOpts): Promis fn: async (row, idx) => { process.stderr.write(`[eval cross-modal batch] ${idx + 1}/${rows.length} ${row.question_id} starting...\n`); return await runEvalFn({ - task: row.question, + // With a gold answer the judges can actually verify correctness; + // without one they see only {question, hypothesis} and cannot. + task: row.answer + ? `${row.question}\n\nExpected answer (gold label from the benchmark dataset): ${row.answer}` + : row.question, output: row.hypothesis, slug: row.question_id, dimensions, diff --git a/src/commands/eval-longmemeval.ts b/src/commands/eval-longmemeval.ts index 5090a0581..320b1b56f 100644 --- a/src/commands/eval-longmemeval.ts +++ b/src/commands/eval-longmemeval.ts @@ -33,6 +33,7 @@ import { type AliasMap, } from '../eval/longmemeval/extract.ts'; import { extractCandidateEntities } from '../core/think/entity-extract.ts'; +import { splitProviderModelId } from '../core/model-id.ts'; import { resolveEntitySlugWithSource, type ResolutionSource } from '../core/entities/resolve.ts'; import { formatTrajectoryBlock } from '../core/trajectory-format.ts'; @@ -469,14 +470,22 @@ export async function runEvalLongMemEval(args: string[], runOpts: RunOpts = {}): }); // Wrap Anthropic SDK so its `.messages.create` shape matches ThinkLLMClient. - // Same pattern as src/core/think/index.ts:247-249. + // Same pattern as src/core/think/index.ts:247-249 — EXCEPT think's default + // client routes through the gateway, which parses `provider:model` recipe + // ids. This eval's client is a raw SDK by design (hermetic, no gateway + // dependency), and resolveModel returns RECIPE ids (`anthropic:claude-…`); + // passing one through unstripped 404s every answer/extractor call, which + // surfaces downstream as all-upstream_error batches in the nightly probe. + const toSdkModel = (m: string): string => splitProviderModelId(m).model || m; const realClient = new Anthropic(); const client: ThinkLLMClient = runOpts.client ?? { - create: (params, callOpts) => realClient.messages.create(params, callOpts), + create: (params, callOpts) => + realClient.messages.create({ ...params, model: toSdkModel(params.model) }, callOpts), }; // v0.40.2.0 — separate extractor client (defaults to same SDK). const extractorClient: ThinkLLMClient = runOpts.extractorClient ?? { - create: (params, callOpts) => realClient.messages.create(params, callOpts), + create: (params, callOpts) => + realClient.messages.create({ ...params, model: toSdkModel(params.model) }, callOpts), }; const trajectoryEnabled = !opts.noTrajectory; const extractorModel = trajectoryEnabled @@ -751,6 +760,11 @@ async function runOneQuestion( // v0.40.1.0 (Track D / T2) — copy question_type into the row so the // by_type_summary can be rebuilt from the file on resume runs. question_type: q.question_type, + // Gold answer for downstream consumers that verify correctness (the + // cross-modal --batch judge folds it into the task; evaluate_qa.py + // ignores unknown fields). Without it a judge can't validate a terse + // factual hypothesis against a haystack it never saw. + ...(q.answer !== undefined ? { answer: q.answer } : {}), hypothesis, retrieved_session_ids: retrievedSessionIds, ...(recallHit !== undefined ? { recall_hit: recallHit } : {}), diff --git a/src/core/audit-parser-probe.ts b/src/core/audit-parser-probe.ts new file mode 100644 index 000000000..ac83d985d --- /dev/null +++ b/src/core/audit-parser-probe.ts @@ -0,0 +1,63 @@ +/** + * Nightly conversation-parser probe audit trail. + * + * One event per REAL probe run lands in + * `~/.gbrain/audit/parser-probe-YYYY-Www.jsonl` (ISO-week rotation via the + * shared audit-writer primitive; honors `GBRAIN_AUDIT_DIR`). + * Scheduler-cadence skips (`rate_limited`) are NOT logged — the autopilot + * loop ticks every few minutes, so logging every skip would flood the + * audit file with rows that carry no signal. + * + * Read by `gbrain doctor`'s `conversation_parser_probe_health` check and + * by the autopilot wiring's 24h rate-limit gate (`parserProbeRanWithin`). + */ + +import { createAuditWriter } from './audit/audit-writer.ts'; +import type { NightlyProbeResult } from './conversation-parser/nightly-probe.ts'; + +export type ParserProbeAuditEvent = NightlyProbeResult; + +const writer = createAuditWriter({ + featureName: 'parser-probe', + errorLabel: 'gbrain', + errorMessagePrefix: 'parser-probe audit ', + errorTrailer: '; probe continues', +}); + +/** Append one parser-probe event. Best-effort; never throws. */ +export function logParserProbeEvent(event: ParserProbeAuditEvent): void { + writer.log(event); +} + +/** + * Read recent parser-probe events (current + previous ISO week, filtered + * to the window). Missing files and corrupt rows are skipped silently. + */ +export function readRecentParserProbeEvents( + days = 7, + now: Date = new Date(), +): ParserProbeAuditEvent[] { + return writer.readRecent(days, now); +} + +/** Exposed for tests pinning the rotation edge cases. */ +export function computeParserProbeAuditFilename(now: Date = new Date()): string { + return writer.computeFilename(now); +} + +/** + * 24h rate-limit gate for the autopilot wiring: true when any audited run + * happened within `windowMs` of `now`. Only REAL outcomes are audited (see + * module header), so a pass/fail today blocks re-runs until tomorrow while + * scheduler-cadence skips never extend the window. + */ +export function parserProbeRanWithin( + windowMs: number, + now: Date = new Date(), +): boolean { + const cutoff = now.getTime() - windowMs; + return readRecentParserProbeEvents(2, now).some((ev) => { + const ts = Date.parse(ev.ts); + return Number.isFinite(ts) && ts >= cutoff; + }); +} diff --git a/src/core/config.ts b/src/core/config.ts index 4ca00cdc9..9ee65a3e9 100644 --- a/src/core/config.ts +++ b/src/core/config.ts @@ -105,6 +105,16 @@ export interface GBrainConfig { */ max_usd?: number; }; + /** + * v0.41.16.0 — nightly conversation-parser probe. Per D10: default ON + * for `search.mode=tokenmax` brains, opt-in for conservative/balanced. + * ~$0.05/night with the committed fixtures × Haiku polish. Gated + * INSIDE the autopilot tick body, like nightly_quality_probe. + */ + conversation_parser_probe?: { + /** Enable for non-tokenmax modes. Defaults to false. */ + enabled?: boolean; + }; /** * v0.42.x (#1685 GAP D) — extract_atoms backlog auto-drain. Default ON so a * pack-gated silent backlog never piles up unseen; daily-spend-capped so the diff --git a/src/core/conversation-parser/nightly-probe.ts b/src/core/conversation-parser/nightly-probe.ts index a386650a1..84f19eecd 100644 --- a/src/core/conversation-parser/nightly-probe.ts +++ b/src/core/conversation-parser/nightly-probe.ts @@ -17,11 +17,11 @@ * Cost: ~$0.05/night with default fixtures × Haiku polish. Bounded * by the active BudgetTracker the autopilot loop creates per-tick. * - * **Wiring into the autopilot loop is deferred to a follow-up** - * (filed in TODOS.md). v0.41.16.0 ships the phase as a callable - * module so doctor + future cron drivers can invoke it; the - * scheduler wire-up follows the same shape as - * `src/core/cycle/nightly-quality-probe.ts` (v0.40.1.0 Track D / T6). + * Wired into the autopilot loop (step 4.6 in autopilot.ts), following + * the same shape as `src/core/cycle/nightly-quality-probe.ts` + * (v0.40.1.0 Track D / T6): the wiring resolves fixtures from the + * gbrain package root, writes real outcomes to the parser-probe audit + * trail (`audit-parser-probe.ts`), and never crashes the loop. * * Test seam: all dependencies are injected via NightlyProbeDeps so * unit tests don't touch real LLMs or real fixtures. diff --git a/src/core/cross-modal-eval/runner.ts b/src/core/cross-modal-eval/runner.ts index 9fdaa4920..a8a08c819 100644 --- a/src/core/cross-modal-eval/runner.ts +++ b/src/core/cross-modal-eval/runner.ts @@ -44,7 +44,12 @@ export const DEFAULT_DIMENSIONS: string[] = [ * `--slot-a-model`, `--slot-b-model`, `--slot-c-model` on the CLI. */ export const DEFAULT_SLOTS: SlotConfig[] = [ - { id: 'A', model: 'openai:gpt-4o' }, + // Every default MUST be listed in its recipe's chat touchpoint (pinned by + // test/cross-modal-default-slots.test.ts) — `openai:gpt-4o` sat here after + // the OpenAI recipe dropped it, so slot A errored "not listed for OpenAI + // chat" on every install and the 3-slot panel could never reach its + // 2-model quorum without a Google key (verdict: permanently inconclusive). + { id: 'A', model: 'openai:gpt-5.2' }, { id: 'B', model: 'anthropic:claude-opus-4-7' }, { id: 'C', model: 'google:gemini-1.5-pro' }, ]; diff --git a/src/core/cycle/nightly-probe-adapters.ts b/src/core/cycle/nightly-probe-adapters.ts index 2ce34e1af..8b274c58e 100644 --- a/src/core/cycle/nightly-probe-adapters.ts +++ b/src/core/cycle/nightly-probe-adapters.ts @@ -70,6 +70,28 @@ export async function runLongMemEvalForProbe(args: LongMemEvalProbeArgs): Promis * the batch input) or unparseable (cross-modal wrote garbage). Both * cases are paste-ready in the error message. */ +/** + * QA-shaped judge dimensions for the nightly probe. The batch judge's + * DEFAULT_DIMENSIONS rubric (DEPTH / SOURCING / SPECIFICITY / …) is built + * for rich agent responses; LongMemEval hypotheses are deliberately terse + * factual answers ("in widget-co") that can never score ≥7 on DEPTH or + * SOURCING — so with the default rubric the probe FAILs every night even + * when retrieval + answering are perfectly healthy. The probe owns its + * invocation of the eval tool and passes dimensions matching the + * fixture's QA shape instead. + * + * NOTE: the `--dimensions` CLI flag splits on commas, so these dimension + * descriptions must stay comma-free. + */ +export const PROBE_QA_DIMENSIONS: string[] = [ + // No faithfulness/grounding dimension on purpose: the judge never sees + // the haystack, so any accurate detail beyond the terse gold label reads + // as "invented" and correct answers fail (verified empirically — a + // correct "before + dates" answer scored 4/10 on such a dimension). + 'CORRECTNESS — Does the hypothesis state the same fact as the expected answer? A terse direct answer is ideal.', + 'DIRECTNESS — Does it answer THIS question without hedging or padding or answering something else?', +]; + export async function runCrossModalBatchForProbe( args: CrossModalProbeArgs, ): Promise<{ exitCode: number; summary: CrossModalBatchSummary }> { @@ -81,6 +103,8 @@ export async function runCrossModalBatchForProbe( args.summaryPath, '--max-usd', String(args.maxUsd), + '--dimensions', + PROBE_QA_DIMENSIONS.join(','), '--yes', '--json', ]); diff --git a/src/core/cycle/nightly-quality-probe.ts b/src/core/cycle/nightly-quality-probe.ts index caf7a1f53..ef3de08d7 100644 --- a/src/core/cycle/nightly-quality-probe.ts +++ b/src/core/cycle/nightly-quality-probe.ts @@ -62,6 +62,42 @@ export interface NightlyProbeDeps { now: () => Date; } +/** + * Dual-plane flag resolution (same precedent as `mcp.publish_skills` in + * serve-http.ts): the DB config row — what `gbrain config set` writes — + * wins when present; the file plane (~/.gbrain/config.json) is the + * fallback. Doctor's paste-ready enable hint says `gbrain config set + * autopilot.nightly_quality_probe.enabled true`, so the gate MUST read + * the DB plane — a file-only read turns that hint into a silent no-op. + */ +export function resolveProbeEnabled( + dbVal: string | null | undefined, + fileVal: unknown, +): boolean { + if (dbVal != null) return dbVal === 'true'; + return fileVal === true; +} + +/** + * Same dual-plane rule for the per-run cost cap. Malformed or negative + * values on either plane fall through to the next plane / the default. + */ +export function resolveProbeMaxUsd( + dbVal: string | null | undefined, + fileVal: unknown, + fallback: number = DEFAULT_MAX_USD, +): number { + if (dbVal != null) { + const n = Number(dbVal); + if (Number.isFinite(n) && n >= 0) return n; + } + if (fileVal != null) { + const n = Number(fileVal); + if (Number.isFinite(n) && n >= 0) return n; + } + return fallback; +} + /** * Pure function: decide whether the probe should run given the audit * history. Returns reason when skipping. @@ -101,21 +137,17 @@ export async function runNightlyQualityProbe(deps: NightlyProbeDeps): Promise { + auditDir = mkdtempSync(join(tmpdir(), 'parser-probe-audit-')); + savedEnv = process.env.GBRAIN_AUDIT_DIR; + process.env.GBRAIN_AUDIT_DIR = auditDir; +}); + +afterEach(() => { + if (savedEnv === undefined) delete process.env.GBRAIN_AUDIT_DIR; + else process.env.GBRAIN_AUDIT_DIR = savedEnv; + rmSync(auditDir, { recursive: true, force: true }); +}); + +function makeEvent(overrides: Partial = {}): ParserProbeAuditEvent { + return { + schema_version: 1, + ts: new Date().toISOString(), + outcome: 'pass', + fixtures_total: 12, + fixtures_passed: 12, + recall_mean: 0.98, + participants_recall_mean: 0.97, + adversarial_false_positives: 0, + failed_fixture_ids: [], + ...overrides, + }; +} + +describe('parser-probe audit trail', () => { + test('log + readRecent round-trip', () => { + logParserProbeEvent(makeEvent({ outcome: 'fail', reason: '2 fixture(s) failed' })); + const events = readRecentParserProbeEvents(7); + expect(events.length).toBe(1); + expect(events[0]!.outcome).toBe('fail'); + expect(events[0]!.reason).toBe('2 fixture(s) failed'); + const files = readdirSync(auditDir); + expect(files.length).toBe(1); + expect(files[0]).toMatch(/^parser-probe-\d{4}-W\d{2}\.jsonl$/); + }); + + test('filename uses ISO-week rotation with the parser-probe prefix', () => { + // Year-boundary edge pinned by the shared writer's own tests; here we + // pin the prefix wiring. + expect(computeParserProbeAuditFilename(new Date('2026-07-06T12:00:00Z'))).toBe( + 'parser-probe-2026-W28.jsonl', + ); + }); + + test('readRecent filters by window', () => { + const old = new Date(Date.now() - 10 * 86400000).toISOString(); + logParserProbeEvent(makeEvent({ ts: old })); + expect(readRecentParserProbeEvents(7).length).toBe(0); + }); +}); + +describe('parserProbeRanWithin — 24h rate-limit gate', () => { + const DAY_MS = 24 * 60 * 60 * 1000; + + test('false when no runs are audited', () => { + expect(parserProbeRanWithin(DAY_MS)).toBe(false); + }); + + test('true when a run landed within the window', () => { + logParserProbeEvent(makeEvent({ ts: new Date(Date.now() - 60_000).toISOString() })); + expect(parserProbeRanWithin(DAY_MS)).toBe(true); + }); + + test('false when the last run is older than the window', () => { + logParserProbeEvent(makeEvent({ ts: new Date(Date.now() - 25 * 3600_000).toISOString() })); + expect(parserProbeRanWithin(DAY_MS)).toBe(false); + }); + + test('non-pass outcomes also hold the window (mirrors quality-probe semantics)', () => { + logParserProbeEvent(makeEvent({ + outcome: 'no_embedding_key', + ts: new Date(Date.now() - 3600_000).toISOString(), + })); + expect(parserProbeRanWithin(DAY_MS)).toBe(true); + }); +}); diff --git a/test/autopilot-nightly-probe-wiring.test.ts b/test/autopilot-nightly-probe-wiring.test.ts index 207beba8e..b63db3faa 100644 --- a/test/autopilot-nightly-probe-wiring.test.ts +++ b/test/autopilot-nightly-probe-wiring.test.ts @@ -31,10 +31,15 @@ describe('autopilot wiring: nightly quality probe', () => { expect(SOURCE).toContain(`runCrossModalBatchForProbe`); }); - test('feature flag gate present: cfg.autopilot.nightly_quality_probe.enabled', () => { + test('feature flag gate present: dual-plane read (DB row wins, file plane fallback)', () => { // Per D10: the scheduler ONLY checks the feature flag. The 24h rate-limit // lives inside runNightlyQualityProbe itself (no scheduler-side precheck). - expect(SOURCE).toContain(`nightly_quality_probe?.enabled === true`); + // The flag resolves through resolveProbeEnabled so `gbrain config set + // autopilot.nightly_quality_probe.enabled true` (the doctor hint, DB + // plane) and ~/.gbrain/config.json (file plane) BOTH work — a file-only + // read made the printed hint a silent no-op. + expect(SOURCE).toContain(`getConfig('autopilot.nightly_quality_probe.enabled')`); + expect(SOURCE).toMatch(/resolveProbeEnabled\(dbEnabled,\s*cfg\?\.autopilot\?\.nightly_quality_probe\?\.enabled\)/); }); test('NO scheduler-side rate-limit check (D10 simplification)', () => { @@ -64,12 +69,23 @@ describe('autopilot wiring: nightly quality probe', () => { expect(SOURCE).toContain(`now:`); }); + test('resolveRepoRoot prefers the gbrain package root (committed fixture home), not the brain repoPath', () => { + // The DI harness in nightly-quality-probe.test.ts passes process.cwd() + // (= the gbrain repo in CI), which papered over the wiring passing + // repoPath (= sync.repo_path, the user's BRAIN repo, where the fixture + // never exists). Pin the package-root resolution + existence check. + expect(SOURCE).toMatch(/fileURLToPath\(new URL\('\.\.\/\.\.', import\.meta\.url\)\)/); + expect(SOURCE).toContain(`'longmemeval-nightly.jsonl'`); + expect(SOURCE).toMatch(/fixtureAtPkgRoot \? pkgRoot : repoPath/); + }); + test('hasEmbeddingProvider reads from gateway.isAvailable("embedding") (codex round-2 #12 — in-process, not subprocess)', () => { expect(SOURCE).toContain(`isAvailable('embedding')`); expect(SOURCE).toContain(`gateway`); }); - test('max_usd default = 5 when config unset (matches plan default per D10)', () => { - expect(SOURCE).toMatch(/max_usd\s*\?\?\s*5/); + test('max_usd resolves dual-plane (default = 5 pinned by resolveProbeMaxUsd unit tests)', () => { + expect(SOURCE).toContain(`getConfig('autopilot.nightly_quality_probe.max_usd')`); + expect(SOURCE).toMatch(/resolveProbeMaxUsd\(dbMaxUsd,\s*cfg\?\.autopilot\?\.nightly_quality_probe\?\.max_usd\)/); }); }); diff --git a/test/autopilot-parser-probe-wiring.test.ts b/test/autopilot-parser-probe-wiring.test.ts new file mode 100644 index 000000000..9ff408c7a --- /dev/null +++ b/test/autopilot-parser-probe-wiring.test.ts @@ -0,0 +1,77 @@ +/** + * Source-shape regression tests for the autopilot wiring of + * `runConversationParserNightlyProbe` (step 4.6). + * + * Same rationale as autopilot-nightly-probe-wiring.test.ts: the loop is + * hard to drive end-to-end, so these pin the structural protections — + * the dual-plane flag read, the D10 tokenmax mode-gate, the package-root + * fixture resolution, the audit-flood guard, and the try/catch posture. + * + * The probe's own gate/scoring logic is pinned by the module's unit + * tests; the audit trail by audit-parser-probe.serial.test.ts. + */ + +import { describe, test, expect } from 'bun:test'; +import { readFileSync } from 'node:fs'; +import { resolve } from 'node:path'; + +const AUTOPILOT_SRC = resolve('src/commands/autopilot.ts'); +const SOURCE = readFileSync(AUTOPILOT_SRC, 'utf-8'); + +describe('autopilot wiring: conversation-parser probe', () => { + test('invokes the phase module and the audit trail', () => { + expect(SOURCE).toContain(`runConversationParserNightlyProbe`); + expect(SOURCE).toContain(`conversation-parser/nightly-probe`); + expect(SOURCE).toContain(`logParserProbeEvent`); + expect(SOURCE).toContain(`audit-parser-probe`); + }); + + test('flag reads dual-plane: DB row (gbrain config set) wins, file plane fallback', () => { + expect(SOURCE).toContain(`getConfig('autopilot.conversation_parser_probe.enabled')`); + expect(SOURCE).toContain(`cfg?.autopilot?.conversation_parser_probe?.enabled === true`); + }); + + test('D10 mode-gate present: tokenmax brains run the probe by default', () => { + expect(SOURCE).toMatch(/parserEnabled \|\| searchMode === 'tokenmax'/); + }); + + test('fixtures resolve from the gbrain package root, NOT the brain repoPath', () => { + // The committed fixtures live in the gbrain source tree; resolving + // them against sync.repo_path would point into the user's brain repo. + expect(SOURCE).toMatch(/fileURLToPath\(new URL\('\.\.\/\.\.', import\.meta\.url\)\)/); + expect(SOURCE).toContain(`'conversation-formats', 'all.jsonl'`); + expect(SOURCE).toContain(`'conversation-formats', 'adversarial.jsonl'`); + }); + + test('missing fixtures skip quietly (no audit row, once-per-process stderr note)', () => { + // Compiled-binary installs carry no source tree; writing failure rows + // would flip doctor to WARN on every binary install. + expect(SOURCE).toContain(`parserProbeFixtureWarned`); + }); + + test('rate_limited outcomes are NOT audit-logged (flood guard)', () => { + expect(SOURCE).toMatch(/outcome !== 'rate_limited'\) logParserProbeEvent\(result\)/); + }); + + test('rate-limit gate delegates to the audit module, not inline event reads', () => { + expect(SOURCE).toContain(`parserProbeRanWithin(24 * 60 * 60 * 1000)`); + }); + + test('LLM-key gate reads gateway.isAvailable("chat") in-process', () => { + expect(SOURCE).toContain(`isAvailable('chat')`); + }); + + test('probe call wrapped in try/catch that does NOT bump consecutiveErrors', () => { + expect(SOURCE).toMatch(/catch[\s\S]*?autopilot\.parser_probe[\s\S]*?do NOT bump consecutiveErrors/); + }); + + test('DI shape: the exact 7 fields of the parser probe NightlyProbeDeps', () => { + expect(SOURCE).toContain(`isEnabled:`); + expect(SOURCE).toContain(`searchMode:`); + expect(SOURCE).toContain(`hasLlmKey:`); + expect(SOURCE).toContain(`resolveFixturePath:`); + expect(SOURCE).toContain(`resolveAdversarialPath:`); + expect(SOURCE).toContain(`shouldSkipForRateLimit:`); + expect(SOURCE).toContain(`now:`); + }); +}); diff --git a/test/cross-modal-default-slots.test.ts b/test/cross-modal-default-slots.test.ts new file mode 100644 index 000000000..16079e4d9 --- /dev/null +++ b/test/cross-modal-default-slots.test.ts @@ -0,0 +1,35 @@ +/** + * Consistency guard: every cross-modal DEFAULT_SLOTS model must be listed + * in its recipe's chat touchpoint. `openai:gpt-4o` drifted out of the + * OpenAI recipe while remaining the slot-A default — the gateway then + * rejected slot A ("not listed for OpenAI chat") on every install, and the + * 3-slot judge panel could never reach its 2-model quorum without a Google + * key, pinning every batch verdict at inconclusive (which the nightly + * quality probe surfaces as a doctor WARN). + */ +import { describe, expect, test } from 'bun:test'; + +import { DEFAULT_SLOTS } from '../src/core/cross-modal-eval/runner.ts'; +import { getRecipe } from '../src/core/ai/recipes/index.ts'; +import { splitProviderModelId } from '../src/core/model-id.ts'; + +describe('cross-modal DEFAULT_SLOTS ↔ recipe consistency', () => { + test('every default slot model is listed in its recipe chat touchpoint', () => { + for (const slot of DEFAULT_SLOTS) { + const { provider, model } = splitProviderModelId(slot.model); + expect(provider).not.toBeNull(); + const recipe = getRecipe(provider!); + expect(recipe, `slot ${slot.id}: unknown recipe "${provider}"`).toBeDefined(); + const chatModels = recipe!.touchpoints.chat?.models ?? []; + expect( + chatModels, + `slot ${slot.id}: "${model}" not listed for ${provider} chat — the judge slot can never run`, + ).toContain(model); + } + }); + + test('slots span three distinct providers (uncorrelated blind spots)', () => { + const providers = new Set(DEFAULT_SLOTS.map(s => splitProviderModelId(s.model).provider)); + expect(providers.size).toBe(3); + }); +}); diff --git a/test/doctor-parser-probe-check.test.ts b/test/doctor-parser-probe-check.test.ts new file mode 100644 index 000000000..c35bf6945 --- /dev/null +++ b/test/doctor-parser-probe-check.test.ts @@ -0,0 +1,51 @@ +/** + * Tests for computeConversationParserProbeHealthCheck — the pure function + * behind doctor's conversation_parser_probe_health check, which replaced + * the v0.41.13.0 hardcoded "Skipped" stub when the autopilot wiring + * landed. Mirrors the branch coverage style of the quality-probe check. + */ +import { describe, expect, test } from 'bun:test'; + +import { computeConversationParserProbeHealthCheck } from '../src/commands/doctor.ts'; + +const ev = (outcome: string, reason?: string, ts = new Date().toISOString()) => ({ + outcome, + ts, + ...(reason !== undefined ? { reason } : {}), +}); + +describe('computeConversationParserProbeHealthCheck', () => { + test('disabled + no events → ok with paste-ready enable hint', () => { + const check = computeConversationParserProbeHealthCheck(false, []); + expect(check.status).toBe('ok'); + expect(check.message).toContain('gbrain config set autopilot.conversation_parser_probe.enabled true'); + }); + + test('enabled + no events yet → ok, next run by autopilot', () => { + const check = computeConversationParserProbeHealthCheck(true, []); + expect(check.status).toBe('ok'); + expect(check.message).toContain('no probe events'); + }); + + test('disabled flag but events exist (tokenmax mode-gate ran it) → events win over the hint', () => { + const check = computeConversationParserProbeHealthCheck(false, [ev('pass')]); + expect(check.status).toBe('ok'); + expect(check.message).toContain('all pass'); + }); + + test('any non-pass outcome in the window → warn, latest surfaced with reason', () => { + const check = computeConversationParserProbeHealthCheck(true, [ + ev('pass'), + ev('adversarial_false_positive', '1 adversarial fixture(s) parsed to non-empty'), + ]); + expect(check.status).toBe('warn'); + expect(check.message).toContain('adversarial_false_positive'); + expect(check.message).toContain('parsed to non-empty'); + }); + + test('all pass → ok with run count', () => { + const check = computeConversationParserProbeHealthCheck(true, [ev('pass'), ev('pass')]); + expect(check.status).toBe('ok'); + expect(check.message).toContain('2 probe run(s)'); + }); +}); diff --git a/test/nightly-probe-config-plane.test.ts b/test/nightly-probe-config-plane.test.ts new file mode 100644 index 000000000..1725bc6f1 --- /dev/null +++ b/test/nightly-probe-config-plane.test.ts @@ -0,0 +1,74 @@ +// Regression test for the nightly-quality-probe config-plane split-brain. +// +// The doctor check prints a paste-ready enable hint — `gbrain config set +// autopilot.nightly_quality_probe.enabled true` — which writes the DB config +// plane. But both the autopilot gate and the doctor check used to read ONLY +// the file plane (~/.gbrain/config.json via loadConfig), so following the +// hint was a silent no-op: the probe never ran and doctor kept reporting +// "disabled (opt-in)". +// +// resolveProbeEnabled / resolveProbeMaxUsd pin the dual-plane rule (same +// precedent as `mcp.publish_skills` in serve-http.ts): DB row wins when +// present, file plane is the fallback. +import { describe, expect, test } from 'bun:test'; + +import { + resolveProbeEnabled, + resolveProbeMaxUsd, +} from '../src/core/cycle/nightly-quality-probe.ts'; + +describe('resolveProbeEnabled — dual-plane flag resolution', () => { + test('DB plane "true" enables regardless of file plane (the doctor hint path)', () => { + expect(resolveProbeEnabled('true', undefined)).toBe(true); + expect(resolveProbeEnabled('true', false)).toBe(true); + }); + + test('explicit DB "false" wins over file-plane true (config set off sticks)', () => { + expect(resolveProbeEnabled('false', true)).toBe(false); + }); + + test('file plane is the fallback when no DB row exists', () => { + expect(resolveProbeEnabled(null, true)).toBe(true); + expect(resolveProbeEnabled(undefined, true)).toBe(true); + expect(resolveProbeEnabled(null, undefined)).toBe(false); + expect(resolveProbeEnabled(null, false)).toBe(false); + }); + + test('file plane stays strict boolean — string "true" in config.json does not enable', () => { + // Matches the pre-fix autopilot gate (`=== true`); the doctor check used + // Boolean(...) and could disagree with autopilot on a string value. + // Both call sites now share this helper, so they can no longer diverge. + expect(resolveProbeEnabled(null, 'true')).toBe(false); + expect(resolveProbeEnabled(null, 1)).toBe(false); + }); + + test('non-"true" DB strings are off (mcp.publish_skills semantics)', () => { + expect(resolveProbeEnabled('1', true)).toBe(false); + expect(resolveProbeEnabled('yes', true)).toBe(false); + expect(resolveProbeEnabled('', true)).toBe(false); + }); +}); + +describe('resolveProbeMaxUsd — dual-plane cost cap resolution', () => { + test('DB plane wins when parseable', () => { + expect(resolveProbeMaxUsd('2.5', 10)).toBe(2.5); + expect(resolveProbeMaxUsd('0', 10)).toBe(0); + }); + + test('malformed or negative DB value falls through to file plane', () => { + expect(resolveProbeMaxUsd('banana', 3)).toBe(3); + expect(resolveProbeMaxUsd('-1', 3)).toBe(3); + }); + + test('file plane used when no DB row; default when both absent/invalid', () => { + expect(resolveProbeMaxUsd(null, 7)).toBe(7); + expect(resolveProbeMaxUsd(null, '4')).toBe(4); + expect(resolveProbeMaxUsd(null, undefined)).toBe(5); + expect(resolveProbeMaxUsd(null, 'banana')).toBe(5); + expect(resolveProbeMaxUsd(undefined, -2)).toBe(5); + }); + + test('explicit fallback override is honored', () => { + expect(resolveProbeMaxUsd(null, undefined, 12)).toBe(12); + }); +}); diff --git a/test/nightly-quality-probe.test.ts b/test/nightly-quality-probe.test.ts index 5f145e903..e7839e0ab 100644 --- a/test/nightly-quality-probe.test.ts +++ b/test/nightly-quality-probe.test.ts @@ -132,17 +132,20 @@ describe('runNightlyQualityProbe (DI stub harness)', () => { }); }); - test('enabled + recent run within 24h → outcome: rate_limited', async () => { + test('enabled + recent run within 24h → outcome: rate_limited, NO audit row', async () => { // Pre-seed a recent audit event by running the probe once first. await withEnv({ GBRAIN_AUDIT_DIR: auditTmp }, async () => { // First run succeeds. await runNightlyQualityProbe(makeDeps()); - // Second run, same hour → rate_limited. + // Second run, same hour → rate_limited. A skip is a non-event: the + // autopilot loop invokes the probe every cycle (~5-10 min), so + // logging each skip would flood the audit file and flip doctor's + // any-non-pass-is-bad filter to a permanent WARN. const r2 = await runNightlyQualityProbe(makeDeps()); expect(r2.outcome).toBe('rate_limited'); const events = await readEvents(); - expect(events.length).toBe(2); - expect(events[1].outcome).toBe('rate_limited'); + expect(events.length).toBe(1); + expect(events[0].outcome).toBe('pass'); }); });