diff --git a/src/commands/autopilot.ts b/src/commands/autopilot.ts index 927b571e5..b484fbacf 100644 --- a/src/commands/autopilot.ts +++ b/src/commands/autopilot.ts @@ -1022,17 +1022,36 @@ export async function runAutopilot(engine: BrainEngine, args: string[]) { // loop. Probe runs even when cycleOk=false (probe may surface signal // explaining why the cycle is failing). try { - const probeEnabled = cfg?.autopilot?.nightly_quality_probe?.enabled === true; + const { resolveProbeEnabled, resolveProbeMaxUsd, runNightlyQualityProbe } = await import('../core/cycle/nightly-quality-probe.ts'); + // Dual-plane read: `gbrain config set` (what the doctor enable hint + // prints) writes the DB plane; ~/.gbrain/config.json is the fallback. + let dbEnabled: string | null = null; + let dbMaxUsd: string | null = null; + try { + dbEnabled = await engine.getConfig('autopilot.nightly_quality_probe.enabled'); + dbMaxUsd = await engine.getConfig('autopilot.nightly_quality_probe.max_usd'); + } catch { /* DB unavailable → file plane only */ } + const probeEnabled = resolveProbeEnabled(dbEnabled, cfg?.autopilot?.nightly_quality_probe?.enabled); if (probeEnabled) { - const { runNightlyQualityProbe } = await import('../core/cycle/nightly-quality-probe.ts'); const { runLongMemEvalForProbe, runCrossModalBatchForProbe } = await import('../core/cycle/nightly-probe-adapters.ts'); const { isAvailable } = await import('../core/ai/gateway.ts'); - const maxUsd = Number(cfg?.autopilot?.nightly_quality_probe?.max_usd ?? 5); + const { existsSync } = await import('node:fs'); + const { fileURLToPath } = await import('node:url'); + const { join } = await import('node:path'); + const maxUsd = resolveProbeMaxUsd(dbMaxUsd, cfg?.autopilot?.nightly_quality_probe?.max_usd); + // The committed fixture (test/fixtures/longmemeval-nightly.jsonl) + // lives in the gbrain PACKAGE, not the brain repo — repoPath is + // sync.repo_path (the user's brain), where the fixture never + // exists, so the probe error'd on every real install. Resolve the + // package root from the module location; keep repoPath as the + // fallback for setups that vendor the fixture into the brain repo. + const pkgRoot = fileURLToPath(new URL('../..', import.meta.url)); + const fixtureAtPkgRoot = existsSync(join(pkgRoot, 'test', 'fixtures', 'longmemeval-nightly.jsonl')); await runNightlyQualityProbe({ isEnabled: () => true, // already gated above; phase re-checks for defense-in-depth hasEmbeddingProvider: () => isAvailable('embedding'), resolveMaxUsd: () => maxUsd, - resolveRepoRoot: () => repoPath ?? gbrainHomePath('.'), + resolveRepoRoot: () => (fixtureAtPkgRoot ? pkgRoot : repoPath ?? gbrainHomePath('.')), runLongMemEval: runLongMemEvalForProbe, runCrossModalBatch: runCrossModalBatchForProbe, now: () => new Date(), diff --git a/src/commands/doctor.ts b/src/commands/doctor.ts index 9b0dc3a4b..5b4a1f379 100644 --- a/src/commands/doctor.ts +++ b/src/commands/doctor.ts @@ -4779,10 +4779,17 @@ export async function buildChecks( try { const { readRecentQualityProbeEvents } = await import('../core/audit-quality-probe.ts'); const { loadConfig } = await import('../core/config.ts'); + const { resolveProbeEnabled } = await import('../core/cycle/nightly-quality-probe.ts'); let probeEnabled = false; try { + // Dual-plane read, matching the autopilot gate: the DB row (what the + // enable hint's `gbrain config set` writes) wins; file plane fallback. + let dbVal: string | null = null; + try { + dbVal = engine ? await engine.getConfig('autopilot.nightly_quality_probe.enabled') : null; + } catch { /* DB unavailable → file plane only */ } const cfg = loadConfig(); - probeEnabled = Boolean((cfg as any)?.autopilot?.nightly_quality_probe?.enabled); + probeEnabled = resolveProbeEnabled(dbVal, (cfg as any)?.autopilot?.nightly_quality_probe?.enabled); } catch { /* config unavailable → treat as disabled */ } const events = readRecentQualityProbeEvents(7); const check = computeNightlyQualityProbeHealthCheck(probeEnabled, events); diff --git a/src/commands/eval-cross-modal.ts b/src/commands/eval-cross-modal.ts index 4d3898fce..ed7edbdaa 100644 --- a/src/commands/eval-cross-modal.ts +++ b/src/commands/eval-cross-modal.ts @@ -466,6 +466,14 @@ interface BatchRow { question_id: string; question: string; hypothesis: string; + /** + * Gold answer from the benchmark dataset, when the upstream eval emits + * it (eval-longmemeval does). Folded into the judge task so CORRECTNESS + * is verifiable — without it a judge panel that sees only + * {question, hypothesis} cannot validate a terse factual answer against + * a haystack it never saw. + */ + answer?: string; } /** @@ -579,6 +587,7 @@ function readBatchRows(path: string): BatchReadResult { question_id: typeof obj.question_id === 'string' ? obj.question_id : `line-${lineNo}`, question: obj.question, hypothesis: obj.hypothesis, + ...(typeof obj.answer === 'string' && obj.answer.length > 0 ? { answer: obj.answer } : {}), }); } if (summarySkipped > 0) { @@ -695,7 +704,11 @@ async function runBatchMode(parsed: ParsedArgs, opts: RunCrossModalOpts): Promis fn: async (row, idx) => { process.stderr.write(`[eval cross-modal batch] ${idx + 1}/${rows.length} ${row.question_id} starting...\n`); return await runEvalFn({ - task: row.question, + // With a gold answer the judges can actually verify correctness; + // without one they see only {question, hypothesis} and cannot. + task: row.answer + ? `${row.question}\n\nExpected answer (gold label from the benchmark dataset): ${row.answer}` + : row.question, output: row.hypothesis, slug: row.question_id, dimensions, diff --git a/src/commands/eval-longmemeval.ts b/src/commands/eval-longmemeval.ts index 5090a0581..320b1b56f 100644 --- a/src/commands/eval-longmemeval.ts +++ b/src/commands/eval-longmemeval.ts @@ -33,6 +33,7 @@ import { type AliasMap, } from '../eval/longmemeval/extract.ts'; import { extractCandidateEntities } from '../core/think/entity-extract.ts'; +import { splitProviderModelId } from '../core/model-id.ts'; import { resolveEntitySlugWithSource, type ResolutionSource } from '../core/entities/resolve.ts'; import { formatTrajectoryBlock } from '../core/trajectory-format.ts'; @@ -469,14 +470,22 @@ export async function runEvalLongMemEval(args: string[], runOpts: RunOpts = {}): }); // Wrap Anthropic SDK so its `.messages.create` shape matches ThinkLLMClient. - // Same pattern as src/core/think/index.ts:247-249. + // Same pattern as src/core/think/index.ts:247-249 — EXCEPT think's default + // client routes through the gateway, which parses `provider:model` recipe + // ids. This eval's client is a raw SDK by design (hermetic, no gateway + // dependency), and resolveModel returns RECIPE ids (`anthropic:claude-…`); + // passing one through unstripped 404s every answer/extractor call, which + // surfaces downstream as all-upstream_error batches in the nightly probe. + const toSdkModel = (m: string): string => splitProviderModelId(m).model || m; const realClient = new Anthropic(); const client: ThinkLLMClient = runOpts.client ?? { - create: (params, callOpts) => realClient.messages.create(params, callOpts), + create: (params, callOpts) => + realClient.messages.create({ ...params, model: toSdkModel(params.model) }, callOpts), }; // v0.40.2.0 — separate extractor client (defaults to same SDK). const extractorClient: ThinkLLMClient = runOpts.extractorClient ?? { - create: (params, callOpts) => realClient.messages.create(params, callOpts), + create: (params, callOpts) => + realClient.messages.create({ ...params, model: toSdkModel(params.model) }, callOpts), }; const trajectoryEnabled = !opts.noTrajectory; const extractorModel = trajectoryEnabled @@ -751,6 +760,11 @@ async function runOneQuestion( // v0.40.1.0 (Track D / T2) — copy question_type into the row so the // by_type_summary can be rebuilt from the file on resume runs. question_type: q.question_type, + // Gold answer for downstream consumers that verify correctness (the + // cross-modal --batch judge folds it into the task; evaluate_qa.py + // ignores unknown fields). Without it a judge can't validate a terse + // factual hypothesis against a haystack it never saw. + ...(q.answer !== undefined ? { answer: q.answer } : {}), hypothesis, retrieved_session_ids: retrievedSessionIds, ...(recallHit !== undefined ? { recall_hit: recallHit } : {}), diff --git a/src/core/cross-modal-eval/runner.ts b/src/core/cross-modal-eval/runner.ts index 9fdaa4920..a8a08c819 100644 --- a/src/core/cross-modal-eval/runner.ts +++ b/src/core/cross-modal-eval/runner.ts @@ -44,7 +44,12 @@ export const DEFAULT_DIMENSIONS: string[] = [ * `--slot-a-model`, `--slot-b-model`, `--slot-c-model` on the CLI. */ export const DEFAULT_SLOTS: SlotConfig[] = [ - { id: 'A', model: 'openai:gpt-4o' }, + // Every default MUST be listed in its recipe's chat touchpoint (pinned by + // test/cross-modal-default-slots.test.ts) — `openai:gpt-4o` sat here after + // the OpenAI recipe dropped it, so slot A errored "not listed for OpenAI + // chat" on every install and the 3-slot panel could never reach its + // 2-model quorum without a Google key (verdict: permanently inconclusive). + { id: 'A', model: 'openai:gpt-5.2' }, { id: 'B', model: 'anthropic:claude-opus-4-7' }, { id: 'C', model: 'google:gemini-1.5-pro' }, ]; diff --git a/src/core/cycle/nightly-probe-adapters.ts b/src/core/cycle/nightly-probe-adapters.ts index 2ce34e1af..8b274c58e 100644 --- a/src/core/cycle/nightly-probe-adapters.ts +++ b/src/core/cycle/nightly-probe-adapters.ts @@ -70,6 +70,28 @@ export async function runLongMemEvalForProbe(args: LongMemEvalProbeArgs): Promis * the batch input) or unparseable (cross-modal wrote garbage). Both * cases are paste-ready in the error message. */ +/** + * QA-shaped judge dimensions for the nightly probe. The batch judge's + * DEFAULT_DIMENSIONS rubric (DEPTH / SOURCING / SPECIFICITY / …) is built + * for rich agent responses; LongMemEval hypotheses are deliberately terse + * factual answers ("in widget-co") that can never score ≥7 on DEPTH or + * SOURCING — so with the default rubric the probe FAILs every night even + * when retrieval + answering are perfectly healthy. The probe owns its + * invocation of the eval tool and passes dimensions matching the + * fixture's QA shape instead. + * + * NOTE: the `--dimensions` CLI flag splits on commas, so these dimension + * descriptions must stay comma-free. + */ +export const PROBE_QA_DIMENSIONS: string[] = [ + // No faithfulness/grounding dimension on purpose: the judge never sees + // the haystack, so any accurate detail beyond the terse gold label reads + // as "invented" and correct answers fail (verified empirically — a + // correct "before + dates" answer scored 4/10 on such a dimension). + 'CORRECTNESS — Does the hypothesis state the same fact as the expected answer? A terse direct answer is ideal.', + 'DIRECTNESS — Does it answer THIS question without hedging or padding or answering something else?', +]; + export async function runCrossModalBatchForProbe( args: CrossModalProbeArgs, ): Promise<{ exitCode: number; summary: CrossModalBatchSummary }> { @@ -81,6 +103,8 @@ export async function runCrossModalBatchForProbe( args.summaryPath, '--max-usd', String(args.maxUsd), + '--dimensions', + PROBE_QA_DIMENSIONS.join(','), '--yes', '--json', ]); diff --git a/src/core/cycle/nightly-quality-probe.ts b/src/core/cycle/nightly-quality-probe.ts index caf7a1f53..ef3de08d7 100644 --- a/src/core/cycle/nightly-quality-probe.ts +++ b/src/core/cycle/nightly-quality-probe.ts @@ -62,6 +62,42 @@ export interface NightlyProbeDeps { now: () => Date; } +/** + * Dual-plane flag resolution (same precedent as `mcp.publish_skills` in + * serve-http.ts): the DB config row — what `gbrain config set` writes — + * wins when present; the file plane (~/.gbrain/config.json) is the + * fallback. Doctor's paste-ready enable hint says `gbrain config set + * autopilot.nightly_quality_probe.enabled true`, so the gate MUST read + * the DB plane — a file-only read turns that hint into a silent no-op. + */ +export function resolveProbeEnabled( + dbVal: string | null | undefined, + fileVal: unknown, +): boolean { + if (dbVal != null) return dbVal === 'true'; + return fileVal === true; +} + +/** + * Same dual-plane rule for the per-run cost cap. Malformed or negative + * values on either plane fall through to the next plane / the default. + */ +export function resolveProbeMaxUsd( + dbVal: string | null | undefined, + fileVal: unknown, + fallback: number = DEFAULT_MAX_USD, +): number { + if (dbVal != null) { + const n = Number(dbVal); + if (Number.isFinite(n) && n >= 0) return n; + } + if (fileVal != null) { + const n = Number(fileVal); + if (Number.isFinite(n) && n >= 0) return n; + } + return fallback; +} + /** * Pure function: decide whether the probe should run given the audit * history. Returns reason when skipping. @@ -101,21 +137,17 @@ export async function runNightlyQualityProbe(deps: NightlyProbeDeps): Promise { expect(SOURCE).toContain(`runCrossModalBatchForProbe`); }); - test('feature flag gate present: cfg.autopilot.nightly_quality_probe.enabled', () => { + test('feature flag gate present: dual-plane read (DB row wins, file plane fallback)', () => { // Per D10: the scheduler ONLY checks the feature flag. The 24h rate-limit // lives inside runNightlyQualityProbe itself (no scheduler-side precheck). - expect(SOURCE).toContain(`nightly_quality_probe?.enabled === true`); + // The flag resolves through resolveProbeEnabled so `gbrain config set + // autopilot.nightly_quality_probe.enabled true` (the doctor hint, DB + // plane) and ~/.gbrain/config.json (file plane) BOTH work — a file-only + // read made the printed hint a silent no-op. + expect(SOURCE).toContain(`getConfig('autopilot.nightly_quality_probe.enabled')`); + expect(SOURCE).toMatch(/resolveProbeEnabled\(dbEnabled,\s*cfg\?\.autopilot\?\.nightly_quality_probe\?\.enabled\)/); }); test('NO scheduler-side rate-limit check (D10 simplification)', () => { @@ -64,12 +69,23 @@ describe('autopilot wiring: nightly quality probe', () => { expect(SOURCE).toContain(`now:`); }); + test('resolveRepoRoot prefers the gbrain package root (committed fixture home), not the brain repoPath', () => { + // The DI harness in nightly-quality-probe.test.ts passes process.cwd() + // (= the gbrain repo in CI), which papered over the wiring passing + // repoPath (= sync.repo_path, the user's BRAIN repo, where the fixture + // never exists). Pin the package-root resolution + existence check. + expect(SOURCE).toMatch(/fileURLToPath\(new URL\('\.\.\/\.\.', import\.meta\.url\)\)/); + expect(SOURCE).toContain(`'longmemeval-nightly.jsonl'`); + expect(SOURCE).toMatch(/fixtureAtPkgRoot \? pkgRoot : repoPath/); + }); + test('hasEmbeddingProvider reads from gateway.isAvailable("embedding") (codex round-2 #12 — in-process, not subprocess)', () => { expect(SOURCE).toContain(`isAvailable('embedding')`); expect(SOURCE).toContain(`gateway`); }); - test('max_usd default = 5 when config unset (matches plan default per D10)', () => { - expect(SOURCE).toMatch(/max_usd\s*\?\?\s*5/); + test('max_usd resolves dual-plane (default = 5 pinned by resolveProbeMaxUsd unit tests)', () => { + expect(SOURCE).toContain(`getConfig('autopilot.nightly_quality_probe.max_usd')`); + expect(SOURCE).toMatch(/resolveProbeMaxUsd\(dbMaxUsd,\s*cfg\?\.autopilot\?\.nightly_quality_probe\?\.max_usd\)/); }); }); diff --git a/test/cross-modal-default-slots.test.ts b/test/cross-modal-default-slots.test.ts new file mode 100644 index 000000000..16079e4d9 --- /dev/null +++ b/test/cross-modal-default-slots.test.ts @@ -0,0 +1,35 @@ +/** + * Consistency guard: every cross-modal DEFAULT_SLOTS model must be listed + * in its recipe's chat touchpoint. `openai:gpt-4o` drifted out of the + * OpenAI recipe while remaining the slot-A default — the gateway then + * rejected slot A ("not listed for OpenAI chat") on every install, and the + * 3-slot judge panel could never reach its 2-model quorum without a Google + * key, pinning every batch verdict at inconclusive (which the nightly + * quality probe surfaces as a doctor WARN). + */ +import { describe, expect, test } from 'bun:test'; + +import { DEFAULT_SLOTS } from '../src/core/cross-modal-eval/runner.ts'; +import { getRecipe } from '../src/core/ai/recipes/index.ts'; +import { splitProviderModelId } from '../src/core/model-id.ts'; + +describe('cross-modal DEFAULT_SLOTS ↔ recipe consistency', () => { + test('every default slot model is listed in its recipe chat touchpoint', () => { + for (const slot of DEFAULT_SLOTS) { + const { provider, model } = splitProviderModelId(slot.model); + expect(provider).not.toBeNull(); + const recipe = getRecipe(provider!); + expect(recipe, `slot ${slot.id}: unknown recipe "${provider}"`).toBeDefined(); + const chatModels = recipe!.touchpoints.chat?.models ?? []; + expect( + chatModels, + `slot ${slot.id}: "${model}" not listed for ${provider} chat — the judge slot can never run`, + ).toContain(model); + } + }); + + test('slots span three distinct providers (uncorrelated blind spots)', () => { + const providers = new Set(DEFAULT_SLOTS.map(s => splitProviderModelId(s.model).provider)); + expect(providers.size).toBe(3); + }); +}); diff --git a/test/engine-find-trajectory.test.ts b/test/engine-find-trajectory.test.ts index 4415c4a4b..524cefeb7 100644 --- a/test/engine-find-trajectory.test.ts +++ b/test/engine-find-trajectory.test.ts @@ -15,7 +15,7 @@ import { describe, test, expect, beforeAll, afterAll, beforeEach } from 'bun:test'; import { PGLiteEngine } from '../src/core/pglite-engine.ts'; -import { configureGateway } from '../src/core/ai/gateway.ts'; +import { configureGateway, resetGateway } from '../src/core/ai/gateway.ts'; import { detectRegressions, computeDriftScore, @@ -27,16 +27,19 @@ import type { TrajectoryPoint } from '../src/core/engine.ts'; let engine: PGLiteEngine; beforeAll(async () => { - // This file hardcodes 1536-d vectors (vecForMetric). initSchema sizes the - // facts halfvec column from the AMBIENT gateway state, and a beforeAll runs - // before the legacy-embedding-preload's per-test restore — so a preceding - // file in the shard that left the gateway reset or non-1536 would seed a - // 1280-d schema and every insert here would fail with a width mismatch. - // Pin the schema shape explicitly (the pattern bunfig's preload documents). + // Pin the embedding dim to 1536 BEFORE initSchema. This file hardcodes + // Float32Array(1536) vectors, but initSchema sizes vector columns from + // process-global gateway state (getEmbeddingDimensions(), default 1280 = + // zeroentropyai). Whether this file passes therefore depended on which + // test files happened to run before it in the shard: a predecessor that + // leaves the gateway configured without dims (or a bare CI env) yields + // vector(1280) and every insert here dies with "expected 1280 dimensions, + // not 1536". Same fix + rationale as cosine-rescore-column.test.ts, which + // documents this exact class. configureGateway({ embedding_model: 'openai:text-embedding-3-large', embedding_dimensions: 1536, - env: { ...process.env }, + env: { OPENAI_API_KEY: 'sk-test-find-trajectory' }, }); engine = new PGLiteEngine(); await engine.connect({}); @@ -45,6 +48,7 @@ beforeAll(async () => { afterAll(async () => { await engine.disconnect(); + resetGateway(); }); beforeEach(async () => { diff --git a/test/nightly-probe-config-plane.test.ts b/test/nightly-probe-config-plane.test.ts new file mode 100644 index 000000000..1725bc6f1 --- /dev/null +++ b/test/nightly-probe-config-plane.test.ts @@ -0,0 +1,74 @@ +// Regression test for the nightly-quality-probe config-plane split-brain. +// +// The doctor check prints a paste-ready enable hint — `gbrain config set +// autopilot.nightly_quality_probe.enabled true` — which writes the DB config +// plane. But both the autopilot gate and the doctor check used to read ONLY +// the file plane (~/.gbrain/config.json via loadConfig), so following the +// hint was a silent no-op: the probe never ran and doctor kept reporting +// "disabled (opt-in)". +// +// resolveProbeEnabled / resolveProbeMaxUsd pin the dual-plane rule (same +// precedent as `mcp.publish_skills` in serve-http.ts): DB row wins when +// present, file plane is the fallback. +import { describe, expect, test } from 'bun:test'; + +import { + resolveProbeEnabled, + resolveProbeMaxUsd, +} from '../src/core/cycle/nightly-quality-probe.ts'; + +describe('resolveProbeEnabled — dual-plane flag resolution', () => { + test('DB plane "true" enables regardless of file plane (the doctor hint path)', () => { + expect(resolveProbeEnabled('true', undefined)).toBe(true); + expect(resolveProbeEnabled('true', false)).toBe(true); + }); + + test('explicit DB "false" wins over file-plane true (config set off sticks)', () => { + expect(resolveProbeEnabled('false', true)).toBe(false); + }); + + test('file plane is the fallback when no DB row exists', () => { + expect(resolveProbeEnabled(null, true)).toBe(true); + expect(resolveProbeEnabled(undefined, true)).toBe(true); + expect(resolveProbeEnabled(null, undefined)).toBe(false); + expect(resolveProbeEnabled(null, false)).toBe(false); + }); + + test('file plane stays strict boolean — string "true" in config.json does not enable', () => { + // Matches the pre-fix autopilot gate (`=== true`); the doctor check used + // Boolean(...) and could disagree with autopilot on a string value. + // Both call sites now share this helper, so they can no longer diverge. + expect(resolveProbeEnabled(null, 'true')).toBe(false); + expect(resolveProbeEnabled(null, 1)).toBe(false); + }); + + test('non-"true" DB strings are off (mcp.publish_skills semantics)', () => { + expect(resolveProbeEnabled('1', true)).toBe(false); + expect(resolveProbeEnabled('yes', true)).toBe(false); + expect(resolveProbeEnabled('', true)).toBe(false); + }); +}); + +describe('resolveProbeMaxUsd — dual-plane cost cap resolution', () => { + test('DB plane wins when parseable', () => { + expect(resolveProbeMaxUsd('2.5', 10)).toBe(2.5); + expect(resolveProbeMaxUsd('0', 10)).toBe(0); + }); + + test('malformed or negative DB value falls through to file plane', () => { + expect(resolveProbeMaxUsd('banana', 3)).toBe(3); + expect(resolveProbeMaxUsd('-1', 3)).toBe(3); + }); + + test('file plane used when no DB row; default when both absent/invalid', () => { + expect(resolveProbeMaxUsd(null, 7)).toBe(7); + expect(resolveProbeMaxUsd(null, '4')).toBe(4); + expect(resolveProbeMaxUsd(null, undefined)).toBe(5); + expect(resolveProbeMaxUsd(null, 'banana')).toBe(5); + expect(resolveProbeMaxUsd(undefined, -2)).toBe(5); + }); + + test('explicit fallback override is honored', () => { + expect(resolveProbeMaxUsd(null, undefined, 12)).toBe(12); + }); +}); diff --git a/test/nightly-quality-probe.test.ts b/test/nightly-quality-probe.test.ts index 5f145e903..e7839e0ab 100644 --- a/test/nightly-quality-probe.test.ts +++ b/test/nightly-quality-probe.test.ts @@ -132,17 +132,20 @@ describe('runNightlyQualityProbe (DI stub harness)', () => { }); }); - test('enabled + recent run within 24h → outcome: rate_limited', async () => { + test('enabled + recent run within 24h → outcome: rate_limited, NO audit row', async () => { // Pre-seed a recent audit event by running the probe once first. await withEnv({ GBRAIN_AUDIT_DIR: auditTmp }, async () => { // First run succeeds. await runNightlyQualityProbe(makeDeps()); - // Second run, same hour → rate_limited. + // Second run, same hour → rate_limited. A skip is a non-event: the + // autopilot loop invokes the probe every cycle (~5-10 min), so + // logging each skip would flood the audit file and flip doctor's + // any-non-pass-is-bad filter to a permanent WARN. const r2 = await runNightlyQualityProbe(makeDeps()); expect(r2.outcome).toBe('rate_limited'); const events = await readEvents(); - expect(events.length).toBe(2); - expect(events[1].outcome).toBe('rate_limited'); + expect(events.length).toBe(1); + expect(events[0].outcome).toBe('pass'); }); });