mirror of
https://github.com/garrytan/gbrain.git
synced 2026-07-30 19:49:14 +00:00
Garry TanandClaude Opus 5
398b17aa30
Part A (#3500): dashscope_api_key and google_api_key are now GBrainConfig file-plane fields folded into the gateway env by buildGatewayConfig (DASHSCOPE_API_KEY / GOOGLE_GENERATIVE_AI_API_KEY), with process-env GEMINI_API_KEY accepted as an alias for the canonical Google name. Honors the deferral note in brain-score-recommendations.ts by adding the matching HOSTED_EMBED_KEY_CONFIG entries in the same change. A new sweep guard asserts every KNOWN_CONFIG_KEYS *_api_key field reaches the gateway env (the #121/#2662/#3500 recurring bug class). Part B (#3510): google:gemini-1.5-pro (retired by Google) removed from the google recipe chat allowlist first — so recipe-vs-default guard tests catch every dead default — then replaced at the default sites: takes-quality DEFAULT_MODEL_PANEL and grade-takes ensemble docstring use google:gemini-2.0-flash (and openai:gpt-5.2 for the equally-dead gpt-4o), the takes-quality pricing allowlist swaps in gemini-2.0-flash + gpt-5.2, and cross-modal slot C moves to deepseek:deepseek-v4-pro (same replacement as PR #3501). New test/default-model-panels.test.ts pins the takes-quality panel to recipe chat lists, canonical pricing, the budget allowlist, and three distinct providers. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
49 lines
2.3 KiB
TypeScript
49 lines
2.3 KiB
TypeScript
/**
|
|
* Consistency guard for the takes-quality DEFAULT_MODEL_PANEL — the sibling
|
|
* of test/cross-modal-default-slots.test.ts (#3510).
|
|
*
|
|
* `google:gemini-1.5-pro` sat in the panel after Google retired it, and
|
|
* `openai:gpt-4o` after the OpenAI recipe dropped it from its chat list —
|
|
* either way the gateway rejects the slot on every default run. The guard
|
|
* only works if recipes list LIVE models: removing a dead model from its
|
|
* recipe makes every hardcoded default that still names it fail here at
|
|
* once. Do not re-add retired models to a recipe to quiet this test.
|
|
*/
|
|
import { describe, expect, test } from 'bun:test';
|
|
|
|
import { DEFAULT_MODEL_PANEL } from '../src/core/takes-quality-eval/runner.ts';
|
|
import { getPricing } from '../src/core/takes-quality-eval/pricing.ts';
|
|
import { getRecipe } from '../src/core/ai/recipes/index.ts';
|
|
import { splitProviderModelId } from '../src/core/model-id.ts';
|
|
import { canonicalLookup } from '../src/core/model-pricing.ts';
|
|
|
|
describe('takes-quality DEFAULT_MODEL_PANEL ↔ recipe consistency', () => {
|
|
test('every default panel model is listed in its recipe chat touchpoint', () => {
|
|
for (const id of DEFAULT_MODEL_PANEL) {
|
|
const { provider, model } = splitProviderModelId(id);
|
|
expect(provider).not.toBeNull();
|
|
const recipe = getRecipe(provider!);
|
|
expect(recipe, `unknown recipe "${provider}"`).toBeDefined();
|
|
expect(
|
|
recipe!.touchpoints.chat?.models ?? [],
|
|
`"${model}" not listed for ${provider} chat — the default panel can never run`,
|
|
).toContain(model);
|
|
}
|
|
});
|
|
|
|
test('every default panel model prices via canonical AND the takes-quality allowlist', () => {
|
|
for (const id of DEFAULT_MODEL_PANEL) {
|
|
expect(canonicalLookup(id), `"${id}" missing from CANONICAL_PRICING`).toBeDefined();
|
|
// getPricing throws PricingNotFoundError if the model is missing from
|
|
// SUPPORTED_MODELS — a default that can't be budget-gated aborts every
|
|
// `--budget-usd` run before the first call.
|
|
expect(getPricing(id)).toBeDefined();
|
|
}
|
|
});
|
|
|
|
test('panel spans three distinct providers (uncorrelated blind spots)', () => {
|
|
const providers = new Set(DEFAULT_MODEL_PANEL.map((id) => splitProviderModelId(id).provider));
|
|
expect(providers.size).toBe(3);
|
|
});
|
|
});
|