From 3265dc2c93831ab111137c566a2b56a9e072f04d Mon Sep 17 00:00:00 2001 From: Paolo Belcastro Date: Mon, 6 Jul 2026 11:50:45 +0200 Subject: [PATCH 1/4] fix(autopilot,doctor): nightly quality probe honors gbrain config set (dual-plane flag read) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The doctor check prints a paste-ready enable hint — gbrain config set autopilot.nightly_quality_probe.enabled true — which writes the DB config plane. But the autopilot gate and the doctor check both read only the file plane (~/.gbrain/config.json), so following the printed hint was a silent no-op: the probe never ran and doctor kept reporting disabled. Fix: resolveProbeEnabled / resolveProbeMaxUsd in nightly-quality-probe.ts apply the dual-plane rule already used by mcp.publish_skills (serve-http.ts): the DB row wins when present, the file plane is the fallback. Both call sites (autopilot loop gate + doctor health check) now share the helpers, so they can no longer diverge (doctor's old Boolean() read also disagreed with autopilot's === true on string values). Covers the max_usd cap with the same rule. Regression tests pin the helper semantics and the new wiring shape. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01FQgByq4aqQq2PP8UHCdnfk --- src/commands/autopilot.ts | 14 +++- src/commands/doctor.ts | 9 ++- src/core/cycle/nightly-quality-probe.ts | 36 ++++++++++ test/autopilot-nightly-probe-wiring.test.ts | 14 ++-- test/nightly-probe-config-plane.test.ts | 74 +++++++++++++++++++++ 5 files changed, 139 insertions(+), 8 deletions(-) create mode 100644 test/nightly-probe-config-plane.test.ts diff --git a/src/commands/autopilot.ts b/src/commands/autopilot.ts index 927b571e5..1705137f8 100644 --- a/src/commands/autopilot.ts +++ b/src/commands/autopilot.ts @@ -1022,12 +1022,20 @@ export async function runAutopilot(engine: BrainEngine, args: string[]) { // loop. Probe runs even when cycleOk=false (probe may surface signal // explaining why the cycle is failing). try { - const probeEnabled = cfg?.autopilot?.nightly_quality_probe?.enabled === true; + const { resolveProbeEnabled, resolveProbeMaxUsd, runNightlyQualityProbe } = await import('../core/cycle/nightly-quality-probe.ts'); + // Dual-plane read: `gbrain config set` (what the doctor enable hint + // prints) writes the DB plane; ~/.gbrain/config.json is the fallback. + let dbEnabled: string | null = null; + let dbMaxUsd: string | null = null; + try { + dbEnabled = await engine.getConfig('autopilot.nightly_quality_probe.enabled'); + dbMaxUsd = await engine.getConfig('autopilot.nightly_quality_probe.max_usd'); + } catch { /* DB unavailable → file plane only */ } + const probeEnabled = resolveProbeEnabled(dbEnabled, cfg?.autopilot?.nightly_quality_probe?.enabled); if (probeEnabled) { - const { runNightlyQualityProbe } = await import('../core/cycle/nightly-quality-probe.ts'); const { runLongMemEvalForProbe, runCrossModalBatchForProbe } = await import('../core/cycle/nightly-probe-adapters.ts'); const { isAvailable } = await import('../core/ai/gateway.ts'); - const maxUsd = Number(cfg?.autopilot?.nightly_quality_probe?.max_usd ?? 5); + const maxUsd = resolveProbeMaxUsd(dbMaxUsd, cfg?.autopilot?.nightly_quality_probe?.max_usd); await runNightlyQualityProbe({ isEnabled: () => true, // already gated above; phase re-checks for defense-in-depth hasEmbeddingProvider: () => isAvailable('embedding'), diff --git a/src/commands/doctor.ts b/src/commands/doctor.ts index 9b0dc3a4b..5b4a1f379 100644 --- a/src/commands/doctor.ts +++ b/src/commands/doctor.ts @@ -4779,10 +4779,17 @@ export async function buildChecks( try { const { readRecentQualityProbeEvents } = await import('../core/audit-quality-probe.ts'); const { loadConfig } = await import('../core/config.ts'); + const { resolveProbeEnabled } = await import('../core/cycle/nightly-quality-probe.ts'); let probeEnabled = false; try { + // Dual-plane read, matching the autopilot gate: the DB row (what the + // enable hint's `gbrain config set` writes) wins; file plane fallback. + let dbVal: string | null = null; + try { + dbVal = engine ? await engine.getConfig('autopilot.nightly_quality_probe.enabled') : null; + } catch { /* DB unavailable → file plane only */ } const cfg = loadConfig(); - probeEnabled = Boolean((cfg as any)?.autopilot?.nightly_quality_probe?.enabled); + probeEnabled = resolveProbeEnabled(dbVal, (cfg as any)?.autopilot?.nightly_quality_probe?.enabled); } catch { /* config unavailable → treat as disabled */ } const events = readRecentQualityProbeEvents(7); const check = computeNightlyQualityProbeHealthCheck(probeEnabled, events); diff --git a/src/core/cycle/nightly-quality-probe.ts b/src/core/cycle/nightly-quality-probe.ts index caf7a1f53..726ac3fa9 100644 --- a/src/core/cycle/nightly-quality-probe.ts +++ b/src/core/cycle/nightly-quality-probe.ts @@ -62,6 +62,42 @@ export interface NightlyProbeDeps { now: () => Date; } +/** + * Dual-plane flag resolution (same precedent as `mcp.publish_skills` in + * serve-http.ts): the DB config row — what `gbrain config set` writes — + * wins when present; the file plane (~/.gbrain/config.json) is the + * fallback. Doctor's paste-ready enable hint says `gbrain config set + * autopilot.nightly_quality_probe.enabled true`, so the gate MUST read + * the DB plane — a file-only read turns that hint into a silent no-op. + */ +export function resolveProbeEnabled( + dbVal: string | null | undefined, + fileVal: unknown, +): boolean { + if (dbVal != null) return dbVal === 'true'; + return fileVal === true; +} + +/** + * Same dual-plane rule for the per-run cost cap. Malformed or negative + * values on either plane fall through to the next plane / the default. + */ +export function resolveProbeMaxUsd( + dbVal: string | null | undefined, + fileVal: unknown, + fallback: number = DEFAULT_MAX_USD, +): number { + if (dbVal != null) { + const n = Number(dbVal); + if (Number.isFinite(n) && n >= 0) return n; + } + if (fileVal != null) { + const n = Number(fileVal); + if (Number.isFinite(n) && n >= 0) return n; + } + return fallback; +} + /** * Pure function: decide whether the probe should run given the audit * history. Returns reason when skipping. diff --git a/test/autopilot-nightly-probe-wiring.test.ts b/test/autopilot-nightly-probe-wiring.test.ts index 207beba8e..15afb6d65 100644 --- a/test/autopilot-nightly-probe-wiring.test.ts +++ b/test/autopilot-nightly-probe-wiring.test.ts @@ -31,10 +31,15 @@ describe('autopilot wiring: nightly quality probe', () => { expect(SOURCE).toContain(`runCrossModalBatchForProbe`); }); - test('feature flag gate present: cfg.autopilot.nightly_quality_probe.enabled', () => { + test('feature flag gate present: dual-plane read (DB row wins, file plane fallback)', () => { // Per D10: the scheduler ONLY checks the feature flag. The 24h rate-limit // lives inside runNightlyQualityProbe itself (no scheduler-side precheck). - expect(SOURCE).toContain(`nightly_quality_probe?.enabled === true`); + // The flag resolves through resolveProbeEnabled so `gbrain config set + // autopilot.nightly_quality_probe.enabled true` (the doctor hint, DB + // plane) and ~/.gbrain/config.json (file plane) BOTH work — a file-only + // read made the printed hint a silent no-op. + expect(SOURCE).toContain(`getConfig('autopilot.nightly_quality_probe.enabled')`); + expect(SOURCE).toMatch(/resolveProbeEnabled\(dbEnabled,\s*cfg\?\.autopilot\?\.nightly_quality_probe\?\.enabled\)/); }); test('NO scheduler-side rate-limit check (D10 simplification)', () => { @@ -69,7 +74,8 @@ describe('autopilot wiring: nightly quality probe', () => { expect(SOURCE).toContain(`gateway`); }); - test('max_usd default = 5 when config unset (matches plan default per D10)', () => { - expect(SOURCE).toMatch(/max_usd\s*\?\?\s*5/); + test('max_usd resolves dual-plane (default = 5 pinned by resolveProbeMaxUsd unit tests)', () => { + expect(SOURCE).toContain(`getConfig('autopilot.nightly_quality_probe.max_usd')`); + expect(SOURCE).toMatch(/resolveProbeMaxUsd\(dbMaxUsd,\s*cfg\?\.autopilot\?\.nightly_quality_probe\?\.max_usd\)/); }); }); diff --git a/test/nightly-probe-config-plane.test.ts b/test/nightly-probe-config-plane.test.ts new file mode 100644 index 000000000..1725bc6f1 --- /dev/null +++ b/test/nightly-probe-config-plane.test.ts @@ -0,0 +1,74 @@ +// Regression test for the nightly-quality-probe config-plane split-brain. +// +// The doctor check prints a paste-ready enable hint — `gbrain config set +// autopilot.nightly_quality_probe.enabled true` — which writes the DB config +// plane. But both the autopilot gate and the doctor check used to read ONLY +// the file plane (~/.gbrain/config.json via loadConfig), so following the +// hint was a silent no-op: the probe never ran and doctor kept reporting +// "disabled (opt-in)". +// +// resolveProbeEnabled / resolveProbeMaxUsd pin the dual-plane rule (same +// precedent as `mcp.publish_skills` in serve-http.ts): DB row wins when +// present, file plane is the fallback. +import { describe, expect, test } from 'bun:test'; + +import { + resolveProbeEnabled, + resolveProbeMaxUsd, +} from '../src/core/cycle/nightly-quality-probe.ts'; + +describe('resolveProbeEnabled — dual-plane flag resolution', () => { + test('DB plane "true" enables regardless of file plane (the doctor hint path)', () => { + expect(resolveProbeEnabled('true', undefined)).toBe(true); + expect(resolveProbeEnabled('true', false)).toBe(true); + }); + + test('explicit DB "false" wins over file-plane true (config set off sticks)', () => { + expect(resolveProbeEnabled('false', true)).toBe(false); + }); + + test('file plane is the fallback when no DB row exists', () => { + expect(resolveProbeEnabled(null, true)).toBe(true); + expect(resolveProbeEnabled(undefined, true)).toBe(true); + expect(resolveProbeEnabled(null, undefined)).toBe(false); + expect(resolveProbeEnabled(null, false)).toBe(false); + }); + + test('file plane stays strict boolean — string "true" in config.json does not enable', () => { + // Matches the pre-fix autopilot gate (`=== true`); the doctor check used + // Boolean(...) and could disagree with autopilot on a string value. + // Both call sites now share this helper, so they can no longer diverge. + expect(resolveProbeEnabled(null, 'true')).toBe(false); + expect(resolveProbeEnabled(null, 1)).toBe(false); + }); + + test('non-"true" DB strings are off (mcp.publish_skills semantics)', () => { + expect(resolveProbeEnabled('1', true)).toBe(false); + expect(resolveProbeEnabled('yes', true)).toBe(false); + expect(resolveProbeEnabled('', true)).toBe(false); + }); +}); + +describe('resolveProbeMaxUsd — dual-plane cost cap resolution', () => { + test('DB plane wins when parseable', () => { + expect(resolveProbeMaxUsd('2.5', 10)).toBe(2.5); + expect(resolveProbeMaxUsd('0', 10)).toBe(0); + }); + + test('malformed or negative DB value falls through to file plane', () => { + expect(resolveProbeMaxUsd('banana', 3)).toBe(3); + expect(resolveProbeMaxUsd('-1', 3)).toBe(3); + }); + + test('file plane used when no DB row; default when both absent/invalid', () => { + expect(resolveProbeMaxUsd(null, 7)).toBe(7); + expect(resolveProbeMaxUsd(null, '4')).toBe(4); + expect(resolveProbeMaxUsd(null, undefined)).toBe(5); + expect(resolveProbeMaxUsd(null, 'banana')).toBe(5); + expect(resolveProbeMaxUsd(undefined, -2)).toBe(5); + }); + + test('explicit fallback override is honored', () => { + expect(resolveProbeMaxUsd(null, undefined, 12)).toBe(12); + }); +}); From bce5f20c258e66a43928380195d3be865f429785 Mon Sep 17 00:00:00 2001 From: Paolo Belcastro Date: Mon, 6 Jul 2026 12:00:25 +0200 Subject: [PATCH 2/4] fix(autopilot): quality probe finds its fixture + rate-limit skips stop flooding the audit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two more blockers on the same enable path, found while verifying the config-plane fix end-to-end: 1. Fixture root: the wiring passed resolveRepoRoot = repoPath (sync.repo_path, the user's BRAIN repo) but the committed fixture lives in the gbrain package. On every real install the probe error'd with fixture-not-found. The DI test harness passes process.cwd() — the gbrain repo in CI — which papered over it. Now resolves the package root from the module location, with repoPath kept as fallback. 2. rate_limited flood: the loop invokes the probe every cycle (~5-10 min); each within-24h skip wrote a rate_limited audit row (~hundreds/day), and doctor counts any non-pass outcome as bad — so enabling the probe flipped nightly_quality_probe_health to a permanent WARN by design. Skips are now non-events: no audit row; the real runs gate the 24h window. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01FQgByq4aqQq2PP8UHCdnfk --- src/commands/autopilot.ts | 13 ++++++++++++- src/core/cycle/nightly-quality-probe.ts | 18 +++++++----------- test/autopilot-nightly-probe-wiring.test.ts | 10 ++++++++++ test/nightly-quality-probe.test.ts | 11 +++++++---- 4 files changed, 36 insertions(+), 16 deletions(-) diff --git a/src/commands/autopilot.ts b/src/commands/autopilot.ts index 1705137f8..b484fbacf 100644 --- a/src/commands/autopilot.ts +++ b/src/commands/autopilot.ts @@ -1035,12 +1035,23 @@ export async function runAutopilot(engine: BrainEngine, args: string[]) { if (probeEnabled) { const { runLongMemEvalForProbe, runCrossModalBatchForProbe } = await import('../core/cycle/nightly-probe-adapters.ts'); const { isAvailable } = await import('../core/ai/gateway.ts'); + const { existsSync } = await import('node:fs'); + const { fileURLToPath } = await import('node:url'); + const { join } = await import('node:path'); const maxUsd = resolveProbeMaxUsd(dbMaxUsd, cfg?.autopilot?.nightly_quality_probe?.max_usd); + // The committed fixture (test/fixtures/longmemeval-nightly.jsonl) + // lives in the gbrain PACKAGE, not the brain repo — repoPath is + // sync.repo_path (the user's brain), where the fixture never + // exists, so the probe error'd on every real install. Resolve the + // package root from the module location; keep repoPath as the + // fallback for setups that vendor the fixture into the brain repo. + const pkgRoot = fileURLToPath(new URL('../..', import.meta.url)); + const fixtureAtPkgRoot = existsSync(join(pkgRoot, 'test', 'fixtures', 'longmemeval-nightly.jsonl')); await runNightlyQualityProbe({ isEnabled: () => true, // already gated above; phase re-checks for defense-in-depth hasEmbeddingProvider: () => isAvailable('embedding'), resolveMaxUsd: () => maxUsd, - resolveRepoRoot: () => repoPath ?? gbrainHomePath('.'), + resolveRepoRoot: () => (fixtureAtPkgRoot ? pkgRoot : repoPath ?? gbrainHomePath('.')), runLongMemEval: runLongMemEvalForProbe, runCrossModalBatch: runCrossModalBatchForProbe, now: () => new Date(), diff --git a/src/core/cycle/nightly-quality-probe.ts b/src/core/cycle/nightly-quality-probe.ts index 726ac3fa9..ef3de08d7 100644 --- a/src/core/cycle/nightly-quality-probe.ts +++ b/src/core/cycle/nightly-quality-probe.ts @@ -137,21 +137,17 @@ export async function runNightlyQualityProbe(deps: NightlyProbeDeps): Promise { expect(SOURCE).toContain(`now:`); }); + test('resolveRepoRoot prefers the gbrain package root (committed fixture home), not the brain repoPath', () => { + // The DI harness in nightly-quality-probe.test.ts passes process.cwd() + // (= the gbrain repo in CI), which papered over the wiring passing + // repoPath (= sync.repo_path, the user's BRAIN repo, where the fixture + // never exists). Pin the package-root resolution + existence check. + expect(SOURCE).toMatch(/fileURLToPath\(new URL\('\.\.\/\.\.', import\.meta\.url\)\)/); + expect(SOURCE).toContain(`'longmemeval-nightly.jsonl'`); + expect(SOURCE).toMatch(/fixtureAtPkgRoot \? pkgRoot : repoPath/); + }); + test('hasEmbeddingProvider reads from gateway.isAvailable("embedding") (codex round-2 #12 — in-process, not subprocess)', () => { expect(SOURCE).toContain(`isAvailable('embedding')`); expect(SOURCE).toContain(`gateway`); diff --git a/test/nightly-quality-probe.test.ts b/test/nightly-quality-probe.test.ts index 5f145e903..e7839e0ab 100644 --- a/test/nightly-quality-probe.test.ts +++ b/test/nightly-quality-probe.test.ts @@ -132,17 +132,20 @@ describe('runNightlyQualityProbe (DI stub harness)', () => { }); }); - test('enabled + recent run within 24h → outcome: rate_limited', async () => { + test('enabled + recent run within 24h → outcome: rate_limited, NO audit row', async () => { // Pre-seed a recent audit event by running the probe once first. await withEnv({ GBRAIN_AUDIT_DIR: auditTmp }, async () => { // First run succeeds. await runNightlyQualityProbe(makeDeps()); - // Second run, same hour → rate_limited. + // Second run, same hour → rate_limited. A skip is a non-event: the + // autopilot loop invokes the probe every cycle (~5-10 min), so + // logging each skip would flood the audit file and flip doctor's + // any-non-pass-is-bad filter to a permanent WARN. const r2 = await runNightlyQualityProbe(makeDeps()); expect(r2.outcome).toBe('rate_limited'); const events = await readEvents(); - expect(events.length).toBe(2); - expect(events[1].outcome).toBe('rate_limited'); + expect(events.length).toBe(1); + expect(events[0].outcome).toBe('pass'); }); }); From e6d392ad088ae9cfc13c0ac37af64cfa6006d751 Mon Sep 17 00:00:00 2001 From: Paolo Belcastro Date: Mon, 6 Jul 2026 12:23:28 +0200 Subject: [PATCH 3/4] =?UTF-8?q?fix(eval,probe):=20nightly=20probe=20reache?= =?UTF-8?q?s=20real=20verdicts=20=E2=80=94=20SDK=20model=20strip,=20live?= =?UTF-8?q?=20default=20slots,=20gold-answer=20QA=20judging?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three more layers of the same never-exercised enable path, each found by running the probe end-to-end after the previous fix: 1. eval-longmemeval passed resolveModel's RECIPE ids (anthropic:claude-…) straight to the raw Anthropic SDK — the API 404s on the prefixed string, so every answer/extractor call failed and batches were all-upstream_error. (think avoids this by routing through the gateway; this eval's raw-SDK client is deliberate — hermetic — so strip via splitProviderModelId.) 2. DEFAULT_SLOTS pinned openai:gpt-4o after the OpenAI recipe dropped it, so judge slot A errored 'not listed for OpenAI chat' on every install and the panel could not reach its 2-model quorum without a Google key — verdicts were permanently inconclusive. Bumped to a listed model and added a slots↔recipes consistency test so defaults can't drift again. 3. The judge panel scored terse LongMemEval facts ('in widget-co') on a rich-response rubric (DEPTH/SOURCING ≥7) with no gold answer and no haystack — guaranteed FAIL even when healthy. eval-longmemeval now emits the dataset's gold answer in each row; batch mode folds it into the judge task when present; the probe adapter passes QA-shaped dimensions (CORRECTNESS + DIRECTNESS; deliberately no grounding dimension — judges who never see the haystack read correct extra detail as invented). Verified end-to-end on a real brain: 9/10 fixture questions pass; the one failure is a genuine multi-session aggregation miss (2 of 3 evidence sessions retrieved), i.e. the probe now surfaces real signal instead of plumbing artifacts. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01FQgByq4aqQq2PP8UHCdnfk --- src/commands/eval-cross-modal.ts | 15 +++++++++- src/commands/eval-longmemeval.ts | 20 ++++++++++++-- src/core/cross-modal-eval/runner.ts | 7 ++++- src/core/cycle/nightly-probe-adapters.ts | 24 ++++++++++++++++ test/cross-modal-default-slots.test.ts | 35 ++++++++++++++++++++++++ 5 files changed, 96 insertions(+), 5 deletions(-) create mode 100644 test/cross-modal-default-slots.test.ts diff --git a/src/commands/eval-cross-modal.ts b/src/commands/eval-cross-modal.ts index 4d3898fce..ed7edbdaa 100644 --- a/src/commands/eval-cross-modal.ts +++ b/src/commands/eval-cross-modal.ts @@ -466,6 +466,14 @@ interface BatchRow { question_id: string; question: string; hypothesis: string; + /** + * Gold answer from the benchmark dataset, when the upstream eval emits + * it (eval-longmemeval does). Folded into the judge task so CORRECTNESS + * is verifiable — without it a judge panel that sees only + * {question, hypothesis} cannot validate a terse factual answer against + * a haystack it never saw. + */ + answer?: string; } /** @@ -579,6 +587,7 @@ function readBatchRows(path: string): BatchReadResult { question_id: typeof obj.question_id === 'string' ? obj.question_id : `line-${lineNo}`, question: obj.question, hypothesis: obj.hypothesis, + ...(typeof obj.answer === 'string' && obj.answer.length > 0 ? { answer: obj.answer } : {}), }); } if (summarySkipped > 0) { @@ -695,7 +704,11 @@ async function runBatchMode(parsed: ParsedArgs, opts: RunCrossModalOpts): Promis fn: async (row, idx) => { process.stderr.write(`[eval cross-modal batch] ${idx + 1}/${rows.length} ${row.question_id} starting...\n`); return await runEvalFn({ - task: row.question, + // With a gold answer the judges can actually verify correctness; + // without one they see only {question, hypothesis} and cannot. + task: row.answer + ? `${row.question}\n\nExpected answer (gold label from the benchmark dataset): ${row.answer}` + : row.question, output: row.hypothesis, slug: row.question_id, dimensions, diff --git a/src/commands/eval-longmemeval.ts b/src/commands/eval-longmemeval.ts index 5090a0581..320b1b56f 100644 --- a/src/commands/eval-longmemeval.ts +++ b/src/commands/eval-longmemeval.ts @@ -33,6 +33,7 @@ import { type AliasMap, } from '../eval/longmemeval/extract.ts'; import { extractCandidateEntities } from '../core/think/entity-extract.ts'; +import { splitProviderModelId } from '../core/model-id.ts'; import { resolveEntitySlugWithSource, type ResolutionSource } from '../core/entities/resolve.ts'; import { formatTrajectoryBlock } from '../core/trajectory-format.ts'; @@ -469,14 +470,22 @@ export async function runEvalLongMemEval(args: string[], runOpts: RunOpts = {}): }); // Wrap Anthropic SDK so its `.messages.create` shape matches ThinkLLMClient. - // Same pattern as src/core/think/index.ts:247-249. + // Same pattern as src/core/think/index.ts:247-249 — EXCEPT think's default + // client routes through the gateway, which parses `provider:model` recipe + // ids. This eval's client is a raw SDK by design (hermetic, no gateway + // dependency), and resolveModel returns RECIPE ids (`anthropic:claude-…`); + // passing one through unstripped 404s every answer/extractor call, which + // surfaces downstream as all-upstream_error batches in the nightly probe. + const toSdkModel = (m: string): string => splitProviderModelId(m).model || m; const realClient = new Anthropic(); const client: ThinkLLMClient = runOpts.client ?? { - create: (params, callOpts) => realClient.messages.create(params, callOpts), + create: (params, callOpts) => + realClient.messages.create({ ...params, model: toSdkModel(params.model) }, callOpts), }; // v0.40.2.0 — separate extractor client (defaults to same SDK). const extractorClient: ThinkLLMClient = runOpts.extractorClient ?? { - create: (params, callOpts) => realClient.messages.create(params, callOpts), + create: (params, callOpts) => + realClient.messages.create({ ...params, model: toSdkModel(params.model) }, callOpts), }; const trajectoryEnabled = !opts.noTrajectory; const extractorModel = trajectoryEnabled @@ -751,6 +760,11 @@ async function runOneQuestion( // v0.40.1.0 (Track D / T2) — copy question_type into the row so the // by_type_summary can be rebuilt from the file on resume runs. question_type: q.question_type, + // Gold answer for downstream consumers that verify correctness (the + // cross-modal --batch judge folds it into the task; evaluate_qa.py + // ignores unknown fields). Without it a judge can't validate a terse + // factual hypothesis against a haystack it never saw. + ...(q.answer !== undefined ? { answer: q.answer } : {}), hypothesis, retrieved_session_ids: retrievedSessionIds, ...(recallHit !== undefined ? { recall_hit: recallHit } : {}), diff --git a/src/core/cross-modal-eval/runner.ts b/src/core/cross-modal-eval/runner.ts index 9fdaa4920..a8a08c819 100644 --- a/src/core/cross-modal-eval/runner.ts +++ b/src/core/cross-modal-eval/runner.ts @@ -44,7 +44,12 @@ export const DEFAULT_DIMENSIONS: string[] = [ * `--slot-a-model`, `--slot-b-model`, `--slot-c-model` on the CLI. */ export const DEFAULT_SLOTS: SlotConfig[] = [ - { id: 'A', model: 'openai:gpt-4o' }, + // Every default MUST be listed in its recipe's chat touchpoint (pinned by + // test/cross-modal-default-slots.test.ts) — `openai:gpt-4o` sat here after + // the OpenAI recipe dropped it, so slot A errored "not listed for OpenAI + // chat" on every install and the 3-slot panel could never reach its + // 2-model quorum without a Google key (verdict: permanently inconclusive). + { id: 'A', model: 'openai:gpt-5.2' }, { id: 'B', model: 'anthropic:claude-opus-4-7' }, { id: 'C', model: 'google:gemini-1.5-pro' }, ]; diff --git a/src/core/cycle/nightly-probe-adapters.ts b/src/core/cycle/nightly-probe-adapters.ts index 2ce34e1af..8b274c58e 100644 --- a/src/core/cycle/nightly-probe-adapters.ts +++ b/src/core/cycle/nightly-probe-adapters.ts @@ -70,6 +70,28 @@ export async function runLongMemEvalForProbe(args: LongMemEvalProbeArgs): Promis * the batch input) or unparseable (cross-modal wrote garbage). Both * cases are paste-ready in the error message. */ +/** + * QA-shaped judge dimensions for the nightly probe. The batch judge's + * DEFAULT_DIMENSIONS rubric (DEPTH / SOURCING / SPECIFICITY / …) is built + * for rich agent responses; LongMemEval hypotheses are deliberately terse + * factual answers ("in widget-co") that can never score ≥7 on DEPTH or + * SOURCING — so with the default rubric the probe FAILs every night even + * when retrieval + answering are perfectly healthy. The probe owns its + * invocation of the eval tool and passes dimensions matching the + * fixture's QA shape instead. + * + * NOTE: the `--dimensions` CLI flag splits on commas, so these dimension + * descriptions must stay comma-free. + */ +export const PROBE_QA_DIMENSIONS: string[] = [ + // No faithfulness/grounding dimension on purpose: the judge never sees + // the haystack, so any accurate detail beyond the terse gold label reads + // as "invented" and correct answers fail (verified empirically — a + // correct "before + dates" answer scored 4/10 on such a dimension). + 'CORRECTNESS — Does the hypothesis state the same fact as the expected answer? A terse direct answer is ideal.', + 'DIRECTNESS — Does it answer THIS question without hedging or padding or answering something else?', +]; + export async function runCrossModalBatchForProbe( args: CrossModalProbeArgs, ): Promise<{ exitCode: number; summary: CrossModalBatchSummary }> { @@ -81,6 +103,8 @@ export async function runCrossModalBatchForProbe( args.summaryPath, '--max-usd', String(args.maxUsd), + '--dimensions', + PROBE_QA_DIMENSIONS.join(','), '--yes', '--json', ]); diff --git a/test/cross-modal-default-slots.test.ts b/test/cross-modal-default-slots.test.ts new file mode 100644 index 000000000..16079e4d9 --- /dev/null +++ b/test/cross-modal-default-slots.test.ts @@ -0,0 +1,35 @@ +/** + * Consistency guard: every cross-modal DEFAULT_SLOTS model must be listed + * in its recipe's chat touchpoint. `openai:gpt-4o` drifted out of the + * OpenAI recipe while remaining the slot-A default — the gateway then + * rejected slot A ("not listed for OpenAI chat") on every install, and the + * 3-slot judge panel could never reach its 2-model quorum without a Google + * key, pinning every batch verdict at inconclusive (which the nightly + * quality probe surfaces as a doctor WARN). + */ +import { describe, expect, test } from 'bun:test'; + +import { DEFAULT_SLOTS } from '../src/core/cross-modal-eval/runner.ts'; +import { getRecipe } from '../src/core/ai/recipes/index.ts'; +import { splitProviderModelId } from '../src/core/model-id.ts'; + +describe('cross-modal DEFAULT_SLOTS ↔ recipe consistency', () => { + test('every default slot model is listed in its recipe chat touchpoint', () => { + for (const slot of DEFAULT_SLOTS) { + const { provider, model } = splitProviderModelId(slot.model); + expect(provider).not.toBeNull(); + const recipe = getRecipe(provider!); + expect(recipe, `slot ${slot.id}: unknown recipe "${provider}"`).toBeDefined(); + const chatModels = recipe!.touchpoints.chat?.models ?? []; + expect( + chatModels, + `slot ${slot.id}: "${model}" not listed for ${provider} chat — the judge slot can never run`, + ).toContain(model); + } + }); + + test('slots span three distinct providers (uncorrelated blind spots)', () => { + const providers = new Set(DEFAULT_SLOTS.map(s => splitProviderModelId(s.model).provider)); + expect(providers.size).toBe(3); + }); +}); From 22a9c985d2e51516141c530fdc34b270786680f3 Mon Sep 17 00:00:00 2001 From: Paolo Belcastro Date: Mon, 6 Jul 2026 14:11:00 +0200 Subject: [PATCH 4/4] =?UTF-8?q?test(find-trajectory):=20pin=20embedding=20?= =?UTF-8?q?dims=20=E2=80=94=20kill=20the=20shard-order=201280/1536=20flake?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit engine-find-trajectory.test.ts hardcodes Float32Array(1536) vectors but let initSchema size its vector columns from process-global gateway state (getEmbeddingDimensions(), default 1280). Whether the file passed depended on which test files ran before it in the shard: any predecessor leaving the gateway configured without dims (or a bare CI env) yields vector(1280) and every insert dies with 'expected 1280 dimensions, not 1536'. Adding test files to the repo reshuffles the weight-packed shards, so unrelated PRs trip it (seen on #2629/#2630 CI, test (5)). Same fix + rationale as cosine-rescore-column.test.ts, which documents this exact class: configureGateway(1536) in beforeAll, resetGateway in afterAll. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01FQgByq4aqQq2PP8UHCdnfk --- test/engine-find-trajectory.test.ts | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/test/engine-find-trajectory.test.ts b/test/engine-find-trajectory.test.ts index ab34ef61d..524cefeb7 100644 --- a/test/engine-find-trajectory.test.ts +++ b/test/engine-find-trajectory.test.ts @@ -15,6 +15,7 @@ import { describe, test, expect, beforeAll, afterAll, beforeEach } from 'bun:test'; import { PGLiteEngine } from '../src/core/pglite-engine.ts'; +import { configureGateway, resetGateway } from '../src/core/ai/gateway.ts'; import { detectRegressions, computeDriftScore, @@ -26,6 +27,20 @@ import type { TrajectoryPoint } from '../src/core/engine.ts'; let engine: PGLiteEngine; beforeAll(async () => { + // Pin the embedding dim to 1536 BEFORE initSchema. This file hardcodes + // Float32Array(1536) vectors, but initSchema sizes vector columns from + // process-global gateway state (getEmbeddingDimensions(), default 1280 = + // zeroentropyai). Whether this file passes therefore depended on which + // test files happened to run before it in the shard: a predecessor that + // leaves the gateway configured without dims (or a bare CI env) yields + // vector(1280) and every insert here dies with "expected 1280 dimensions, + // not 1536". Same fix + rationale as cosine-rescore-column.test.ts, which + // documents this exact class. + configureGateway({ + embedding_model: 'openai:text-embedding-3-large', + embedding_dimensions: 1536, + env: { OPENAI_API_KEY: 'sk-test-find-trajectory' }, + }); engine = new PGLiteEngine(); await engine.connect({}); await engine.initSchema(); @@ -33,6 +48,7 @@ beforeAll(async () => { afterAll(async () => { await engine.disconnect(); + resetGateway(); }); beforeEach(async () => {