mirror of
https://github.com/garrytan/gbrain.git
synced 2026-07-31 04:07:52 +00:00
b91350d778
* fix(autopilot,eval): nightly quality probe enable path works end-to-end + wire conversation-parser probe Takeover of #2629 and #2630 (rebased onto master; dropped the test/engine-find-trajectory.test.ts hunk both PRs carried — master already ships the equivalent gateway-dims fix). #2629 — nightly quality probe enable path: - autopilot + doctor read the probe flag dual-plane (DB config row from 'gbrain config set' wins, ~/.gbrain/config.json fallback) via new resolveProbeEnabled/resolveProbeMaxUsd helpers - resolveRepoRoot prefers the gbrain package root where the committed fixture lives, not the brain repoPath - rate_limited skips no longer write an audit row every autopilot cycle - eval-longmemeval strips 'provider:' recipe ids before raw Anthropic SDK calls and emits the gold answer for downstream judges - cross-modal batch folds the gold answer into the judge task; probe passes QA-shaped dimensions instead of the agent-response rubric - DEFAULT_SLOTS slot A moves to openai:gpt-5.2 (gpt-4o left the recipe); new consistency test pins every default slot to its recipe #2630 — conversation-parser nightly probe wire-up: - autopilot step 4.6 invokes runConversationParserNightlyProbe (dual-plane flag + D10 tokenmax mode-gate, package-root fixtures, 24h gate, audit trail via new src/core/audit-parser-probe.ts) - doctor's conversation_parser_probe_health replaces the hardcoded 'Skipped' stub with a real pure-function check over the audit trail Co-authored-by: p3ob7o <p3ob7o@users.noreply.github.com> Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * fix(pricing): add openai:gpt-5.2 canonical entry for the new default slot A DEFAULT_SLOTS slot A moved to openai:gpt-5.2, which had no CANONICAL_PRICING entry — estimateCost silently dropped slot A from the --max-usd pre-flight and est_cost_usd audit rows (~1/3 under-count on the default panel). Rates from the OpenAI recipe chat touchpoint (verified 2026-04-20). Also refresh the --slot-a-model help text default and pin a pricing-presence assertion in the DEFAULT_SLOTS consistency test. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> --------- Co-authored-by: Sinabina <sinabina@Sinabinas-MacBook-Pro-4.local> Co-authored-by: p3ob7o <p3ob7o@users.noreply.github.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com> Co-authored-by: Garry Tan <garrytan@gmail.com>
48 lines
2.1 KiB
TypeScript
48 lines
2.1 KiB
TypeScript
/**
|
|
* Consistency guard: every cross-modal DEFAULT_SLOTS model must be listed
|
|
* in its recipe's chat touchpoint. `openai:gpt-4o` drifted out of the
|
|
* OpenAI recipe while remaining the slot-A default — the gateway then
|
|
* rejected slot A ("not listed for OpenAI chat") on every install, and the
|
|
* 3-slot judge panel could never reach its 2-model quorum without a Google
|
|
* key, pinning every batch verdict at inconclusive (which the nightly
|
|
* quality probe surfaces as a doctor WARN).
|
|
*/
|
|
import { describe, expect, test } from 'bun:test';
|
|
|
|
import { DEFAULT_SLOTS } from '../src/core/cross-modal-eval/runner.ts';
|
|
import { getRecipe } from '../src/core/ai/recipes/index.ts';
|
|
import { splitProviderModelId } from '../src/core/model-id.ts';
|
|
import { canonicalLookup } from '../src/core/model-pricing.ts';
|
|
|
|
describe('cross-modal DEFAULT_SLOTS ↔ recipe consistency', () => {
|
|
test('every default slot model is listed in its recipe chat touchpoint', () => {
|
|
for (const slot of DEFAULT_SLOTS) {
|
|
const { provider, model } = splitProviderModelId(slot.model);
|
|
expect(provider).not.toBeNull();
|
|
const recipe = getRecipe(provider!);
|
|
expect(recipe, `slot ${slot.id}: unknown recipe "${provider}"`).toBeDefined();
|
|
const chatModels = recipe!.touchpoints.chat?.models ?? [];
|
|
expect(
|
|
chatModels,
|
|
`slot ${slot.id}: "${model}" not listed for ${provider} chat — the judge slot can never run`,
|
|
).toContain(model);
|
|
}
|
|
});
|
|
|
|
test('every default slot model has a canonical pricing entry', () => {
|
|
// Without one, estimateCost silently drops the slot from the
|
|
// --max-usd pre-flight and est_cost_usd audit rows (~1/3 under-count).
|
|
for (const slot of DEFAULT_SLOTS) {
|
|
expect(
|
|
canonicalLookup(slot.model),
|
|
`slot ${slot.id}: "${slot.model}" missing from CANONICAL_PRICING`,
|
|
).toBeDefined();
|
|
}
|
|
});
|
|
|
|
test('slots span three distinct providers (uncorrelated blind spots)', () => {
|
|
const providers = new Set(DEFAULT_SLOTS.map(s => splitProviderModelId(s.model).provider));
|
|
expect(providers.size).toBe(3);
|
|
});
|
|
});
|