mirror of
https://github.com/garrytan/gbrain.git
synced 2026-07-31 04:07:52 +00:00
fix(ai): tier-configured models reach the recipe allowlist — refresh Anthropic models, register tier resolutions, honest probe labels (#2800)
Setting `models.tier.deep anthropic:claude-opus-4-8` silently disabled think and auto_think: the Anthropic recipe's chat allowlist stopped at Opus 4.7, the tier-resolved model never joined the extended set that assertTouchpoint's contract promises for config-chosen models, and the resulting probe failure was stamped NO_ANTHROPIC_API_KEY — sending the operator to debug env/keychain when the fix was the model id. Three fixes, one per layer: - recipes/anthropic.ts: add claude-fable-5, claude-opus-4-8, and claude-sonnet-5 to chat models; claude-sonnet-5 to expansion models. - gateway.ts reconfigureGatewayWithEngine: resolve all four tiers and register the results as extended models, honoring the documented contract for models.default / models.tier.* (model-resolver.ts docstring). A tier-only model now validates like a chat/expansion one. - think/index.ts: when the gateway client can't be built, re-probe and label honestly — MODEL_NOT_USABLE:<reason> for unknown_model / unknown_provider, NO_ANTHROPIC_API_KEY only for the actual missing-key case; the stub answer carries the probe detail and fix hint. Tests: recipe-list presence pins; a new gateway-tier-extended-models suite proving a fictional tier model validates post-reconfigure (and an unconfigured one still doesn't); think-pipeline coverage for the honest label (unknown_model beats missing-key even keyless); the existing non-explicit bogus-provider test updated from the old catch-all label to the honest one (no-throw contract unchanged). Verified live: a brain with tier.deep=claude-opus-4-8 had think degrade to gather-only with the misleading key warning; with this change the probe passes and synthesis runs. Co-authored-by: Paolo Belcastro <p3ob7o@users.noreply.github.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Paolo Belcastro
Claude Fable 5
parent
d61808d806
commit
447e57ec41
@@ -521,6 +521,20 @@ export async function reconfigureGatewayWithEngine(engine: BrainEngine): Promise
|
||||
const expansionFull = newExpansion.includes(':') ? newExpansion : prefixWithProviderFrom(cfg.expansion_model ?? DEFAULT_EXPANSION_MODEL, newExpansion);
|
||||
const chatFull = newChat.includes(':') ? newChat : prefixWithProviderFrom(cfg.chat_model ?? DEFAULT_CHAT_MODEL, newChat);
|
||||
|
||||
// ALSO resolve the four tier models and register them as extended models.
|
||||
// assertTouchpoint's contract (model-resolver.ts) says config-chosen models —
|
||||
// `models.default` and `models.tier.*` included — bypass the native recipe
|
||||
// allowlist, but pre-fix only chat/expansion/embedding/reranker were
|
||||
// registered. A model reachable ONLY through a tier (e.g. `models.tier.deep`
|
||||
// set to an Opus newer than the recipe list) failed `probeChatModel` at call
|
||||
// time and silently degraded think/auto_think to the gather-only stub.
|
||||
// Resolving per-tier also honors `models.default` (it sits above tiers in
|
||||
// the resolveModel chain).
|
||||
const tierModels: string[] = [];
|
||||
for (const tier of ['utility', 'reasoning', 'deep', 'subagent'] as const) {
|
||||
tierModels.push(await resolveModel(engine, { tier, fallback: TIER_DEFAULTS[tier] }));
|
||||
}
|
||||
|
||||
_config = { ...cfg, expansion_model: expansionFull, chat_model: chatFull };
|
||||
_modelCache.clear();
|
||||
_shrinkState.clear();
|
||||
@@ -532,6 +546,7 @@ export async function reconfigureGatewayWithEngine(engine: BrainEngine): Promise
|
||||
_config.chat_model,
|
||||
_config.reranker_model,
|
||||
...(_config.chat_fallback_chain ?? []),
|
||||
...tierModels,
|
||||
]) {
|
||||
if (m) registerExtendedModel(m);
|
||||
}
|
||||
|
||||
@@ -17,13 +17,16 @@ export const anthropic: Recipe = {
|
||||
touchpoints: {
|
||||
// No embedding model available.
|
||||
expansion: {
|
||||
models: ['claude-haiku-4-5-20251001', 'claude-sonnet-4-6'],
|
||||
models: ['claude-haiku-4-5-20251001', 'claude-sonnet-5', 'claude-sonnet-4-6'],
|
||||
cost_per_1m_tokens_usd: 0.25,
|
||||
price_last_verified: '2026-05-10',
|
||||
},
|
||||
chat: {
|
||||
models: [
|
||||
'claude-fable-5',
|
||||
'claude-opus-4-8',
|
||||
'claude-opus-4-7',
|
||||
'claude-sonnet-5',
|
||||
'claude-sonnet-4-6',
|
||||
'claude-haiku-4-5-20251001',
|
||||
],
|
||||
|
||||
+21
-3
@@ -454,13 +454,31 @@ export async function runThink(
|
||||
// Closes #952 (think over MCP returns "no LLM available").
|
||||
const client = opts.client ?? await tryBuildGatewayClient(modelUsed, { explicitModel: opts.modelExplicit });
|
||||
if (!client) {
|
||||
warnings.push('NO_ANTHROPIC_API_KEY');
|
||||
// Label the failure honestly: a missing key and an unusable model id are
|
||||
// different incidents with different fixes. Pre-fix EVERY null client was
|
||||
// stamped NO_ANTHROPIC_API_KEY, which sent operators chasing env/keychain
|
||||
// problems when the real cause was a model id the recipe didn't know
|
||||
// (e.g. a tier-configured model newer than the recipe list). The re-probe
|
||||
// is pure and cheap (no IO): same predicate tryBuildGatewayClient used.
|
||||
const probe = probeChatModel(normalizeModelId(modelUsed));
|
||||
const modelProblem = !probe.ok && probe.reason !== 'unavailable';
|
||||
warnings.push(
|
||||
modelProblem ? `MODEL_NOT_USABLE:${(probe as { reason: string }).reason}` : 'NO_ANTHROPIC_API_KEY',
|
||||
);
|
||||
const detail = !probe.ok ? probe.detail : '';
|
||||
const fix = !probe.ok && probe.fix ? ` Fix: ${probe.fix}` : '';
|
||||
// Degrade gracefully: return the gather without synthesis. Better than throwing.
|
||||
return {
|
||||
question: opts.question,
|
||||
answer: '(no LLM available — set ANTHROPIC_API_KEY or pass `client`)',
|
||||
answer: modelProblem
|
||||
? `(model "${modelUsed}" not usable — ${detail}${fix})`
|
||||
: '(no LLM available — set ANTHROPIC_API_KEY or pass `client`)',
|
||||
citations: [],
|
||||
gaps: ['no LLM available; gather succeeded but synthesis skipped'],
|
||||
gaps: [
|
||||
modelProblem
|
||||
? `model "${modelUsed}" not usable (${(probe as { reason: string }).reason}); gather succeeded but synthesis skipped`
|
||||
: 'no LLM available; gather succeeded but synthesis skipped',
|
||||
],
|
||||
pagesGathered: gather.pages.length,
|
||||
takesGathered: gather.takes.length,
|
||||
graphHits: gather.graphSlugs.length,
|
||||
|
||||
@@ -35,6 +35,20 @@ describe('Anthropic recipe model IDs', () => {
|
||||
expect(anthropic.aliases?.['claude-sonnet-4-6-20250929']).toBe('claude-sonnet-4-6');
|
||||
});
|
||||
|
||||
it('current-generation models are listed for chat (Fable 5 / Opus 4.8 / Sonnet 5)', () => {
|
||||
// Regression guard for the tier-config incident: a brain with
|
||||
// `models.tier.deep = anthropic:claude-opus-4-8` had think/auto_think
|
||||
// silently degrade because the recipe list stopped at Opus 4.7.
|
||||
const chatModels = anthropic.touchpoints?.chat?.models ?? [];
|
||||
expect(chatModels).toContain('claude-fable-5');
|
||||
expect(chatModels).toContain('claude-opus-4-8');
|
||||
expect(chatModels).toContain('claude-sonnet-5');
|
||||
});
|
||||
|
||||
it('Sonnet 5 is listed for expansion', () => {
|
||||
expect(anthropic.touchpoints?.expansion?.models ?? []).toContain('claude-sonnet-5');
|
||||
});
|
||||
|
||||
it('all listed models follow naming conventions', () => {
|
||||
const allModels = [
|
||||
...(anthropic.touchpoints?.chat?.models ?? []),
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
/**
|
||||
* reconfigureGatewayWithEngine — tier-resolved models join the extended set.
|
||||
*
|
||||
* assertTouchpoint's extended-models contract (model-resolver.ts) says models
|
||||
* the user opted into via config — `models.default` and `models.tier.*`
|
||||
* included — bypass the native recipe allowlist. Pre-fix, only chat/expansion/
|
||||
* embedding/reranker were registered, so a model reachable ONLY through a tier
|
||||
* (e.g. `models.tier.deep` set to an Opus newer than the recipe list) failed
|
||||
* `probeChatModel` and silently degraded think/auto_think to the gather-only
|
||||
* stub — mislabeled NO_ANTHROPIC_API_KEY.
|
||||
*
|
||||
* Uses a deliberately fictional model id so the test stays valid no matter how
|
||||
* current the recipe list is.
|
||||
*/
|
||||
import { describe, test, expect, afterEach } from 'bun:test';
|
||||
import {
|
||||
configureGateway,
|
||||
reconfigureGatewayWithEngine,
|
||||
resetGateway,
|
||||
validateModelId,
|
||||
} from '../src/core/ai/gateway.ts';
|
||||
import type { BrainEngine } from '../src/core/engine.ts';
|
||||
|
||||
function stubEngine(config: Record<string, string>): BrainEngine {
|
||||
return { getConfig: async (k: string) => config[k] ?? null } as unknown as BrainEngine;
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
resetGateway();
|
||||
});
|
||||
|
||||
describe('reconfigureGatewayWithEngine — tier models extend the allowlist', () => {
|
||||
test('a models.tier.deep model unknown to the recipe validates after reconfigure', async () => {
|
||||
configureGateway({
|
||||
embedding_model: 'openai:text-embedding-3-large',
|
||||
embedding_dimensions: 1536,
|
||||
env: { ANTHROPIC_API_KEY: 'sk-fake', OPENAI_API_KEY: 'sk-fake' },
|
||||
});
|
||||
// Pre-reconfigure: an id absent from the recipe allowlist is rejected.
|
||||
expect(validateModelId('anthropic:claude-hypothetical-9').ok).toBe(false);
|
||||
|
||||
await reconfigureGatewayWithEngine(
|
||||
stubEngine({ 'models.tier.deep': 'anthropic:claude-hypothetical-9' }),
|
||||
);
|
||||
|
||||
// Post-reconfigure: the tier-configured model is in the extended set.
|
||||
expect(validateModelId('anthropic:claude-hypothetical-9').ok).toBe(true);
|
||||
// An id configured NOWHERE stays rejected — the allowlist still bites.
|
||||
expect(validateModelId('anthropic:claude-never-configured-1').ok).toBe(false);
|
||||
});
|
||||
|
||||
test('models.default reaches the extended set through tier resolution', async () => {
|
||||
configureGateway({
|
||||
embedding_model: 'openai:text-embedding-3-large',
|
||||
embedding_dimensions: 1536,
|
||||
env: { ANTHROPIC_API_KEY: 'sk-fake', OPENAI_API_KEY: 'sk-fake' },
|
||||
});
|
||||
await reconfigureGatewayWithEngine(
|
||||
stubEngine({ 'models.default': 'anthropic:claude-hypothetical-10' }),
|
||||
);
|
||||
expect(validateModelId('anthropic:claude-hypothetical-10').ok).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -186,6 +186,23 @@ describe('think gateway adapter — #1698 slash form + explicit-model fork', ()
|
||||
});
|
||||
});
|
||||
|
||||
describe('think gateway adapter — current-generation recipe models', () => {
|
||||
test('builds clients for Opus 4.8 / Sonnet 5 / Fable 5 (recipe-list refresh)', async () => {
|
||||
// Regression guard: these GA models were absent from the recipe allowlist,
|
||||
// so a tier-configured deep model degraded think to the no-LLM stub.
|
||||
await withEnv({ ANTHROPIC_API_KEY: 'sk-test-fake' }, async () => {
|
||||
for (const id of [
|
||||
'anthropic:claude-opus-4-8',
|
||||
'anthropic:claude-sonnet-5',
|
||||
'anthropic:claude-fable-5',
|
||||
]) {
|
||||
const client = await __thinkAdapter.tryBuildGatewayClient(id);
|
||||
expect(client).not.toBeNull();
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('think gateway adapter — graceful fallback shape', () => {
|
||||
test('buildGracefulMessage produces a parseable Anthropic.Message-shaped object', () => {
|
||||
const m = __thinkAdapter.buildGracefulMessage('anthropic:claude-opus-4-7');
|
||||
|
||||
@@ -217,6 +217,24 @@ describe('runThink (with stub client)', () => {
|
||||
expect(result.rounds).toBe(0);
|
||||
});
|
||||
|
||||
test('labels an unusable CONFIGURED model honestly (MODEL_NOT_USABLE, not NO_ANTHROPIC_API_KEY)', async () => {
|
||||
// Regression guard: a configured model the recipe rejects (unknown_model)
|
||||
// used to be stamped NO_ANTHROPIC_API_KEY, sending operators to debug
|
||||
// env/keychain when the fix was the model id. Model validity beats the key
|
||||
// check in probeChatModel, so the honest label holds even keyless.
|
||||
await engine.setConfig('models.think', 'anthropic:claude-bogus-9');
|
||||
try {
|
||||
const result = await withoutAnthropicKey(() => runThink(engine, { question: 'bad model test' }));
|
||||
expect(result.warnings).toContain('MODEL_NOT_USABLE:unknown_model');
|
||||
expect(result.warnings).not.toContain('NO_ANTHROPIC_API_KEY');
|
||||
expect(result.answer).toContain('not usable');
|
||||
expect(result.rounds).toBe(0);
|
||||
expect(result.synthesisOk).toBe(false);
|
||||
} finally {
|
||||
await engine.unsetConfig('models.think');
|
||||
}
|
||||
});
|
||||
|
||||
test('persistSynthesis writes synthesis page + evidence rows', async () => {
|
||||
const stubClient: ThinkLLMClient = {
|
||||
create: async () => ({
|
||||
@@ -285,8 +303,11 @@ describe('runThink — #1698 explicit-model hard error', () => {
|
||||
test('NON-explicit bad model does NOT throw — graceful degrade (no modelExplicit)', async () => {
|
||||
// model present but modelExplicit unset → early gate skipped; builder returns null.
|
||||
// Hermetic no-key so the assertion can't be perturbed by a configured key.
|
||||
// Post-honest-labeling: an unknown PROVIDER is a model problem, not a key
|
||||
// problem — the warning names it instead of the old NO_ANTHROPIC_API_KEY
|
||||
// catch-all. The graceful no-throw contract is unchanged.
|
||||
const result = await withoutAnthropicKey(() => runThink(engine, { question: 'nonexplicit bad', model: 'bogusprovider:foo' }));
|
||||
expect(result.warnings).toContain('NO_ANTHROPIC_API_KEY');
|
||||
expect(result.warnings).toContain('MODEL_NOT_USABLE:unknown_provider');
|
||||
expect(result.synthesisOk).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user