fix(ai): tier-configured models reach the recipe allowlist — refresh Anthropic models, register tier resolutions, honest probe labels (#2800)

Setting `models.tier.deep anthropic:claude-opus-4-8` silently disabled
think and auto_think: the Anthropic recipe's chat allowlist stopped at
Opus 4.7, the tier-resolved model never joined the extended set that
assertTouchpoint's contract promises for config-chosen models, and the
resulting probe failure was stamped NO_ANTHROPIC_API_KEY — sending the
operator to debug env/keychain when the fix was the model id. Three
fixes, one per layer:

- recipes/anthropic.ts: add claude-fable-5, claude-opus-4-8, and
  claude-sonnet-5 to chat models; claude-sonnet-5 to expansion models.
- gateway.ts reconfigureGatewayWithEngine: resolve all four tiers and
  register the results as extended models, honoring the documented
  contract for models.default / models.tier.* (model-resolver.ts
  docstring). A tier-only model now validates like a chat/expansion one.
- think/index.ts: when the gateway client can't be built, re-probe and
  label honestly — MODEL_NOT_USABLE:<reason> for unknown_model /
  unknown_provider, NO_ANTHROPIC_API_KEY only for the actual missing-key
  case; the stub answer carries the probe detail and fix hint.

Tests: recipe-list presence pins; a new gateway-tier-extended-models
suite proving a fictional tier model validates post-reconfigure (and an
unconfigured one still doesn't); think-pipeline coverage for the honest
label (unknown_model beats missing-key even keyless); the existing
non-explicit bogus-provider test updated from the old catch-all label to
the honest one (no-throw contract unchanged).

Verified live: a brain with tier.deep=claude-opus-4-8 had think degrade
to gather-only with the misleading key warning; with this change the
probe passes and synthesis runs.

Co-authored-by: Paolo Belcastro <p3ob7o@users.noreply.github.com>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Paolo Belcastro
2026-07-21 13:06:32 -07:00
committed by GitHub
co-authored by Paolo Belcastro Claude Fable 5
parent d61808d806
commit 447e57ec41
7 changed files with 156 additions and 5 deletions
+15
View File
@@ -521,6 +521,20 @@ export async function reconfigureGatewayWithEngine(engine: BrainEngine): Promise
const expansionFull = newExpansion.includes(':') ? newExpansion : prefixWithProviderFrom(cfg.expansion_model ?? DEFAULT_EXPANSION_MODEL, newExpansion);
const chatFull = newChat.includes(':') ? newChat : prefixWithProviderFrom(cfg.chat_model ?? DEFAULT_CHAT_MODEL, newChat);
// ALSO resolve the four tier models and register them as extended models.
// assertTouchpoint's contract (model-resolver.ts) says config-chosen models —
// `models.default` and `models.tier.*` included — bypass the native recipe
// allowlist, but pre-fix only chat/expansion/embedding/reranker were
// registered. A model reachable ONLY through a tier (e.g. `models.tier.deep`
// set to an Opus newer than the recipe list) failed `probeChatModel` at call
// time and silently degraded think/auto_think to the gather-only stub.
// Resolving per-tier also honors `models.default` (it sits above tiers in
// the resolveModel chain).
const tierModels: string[] = [];
for (const tier of ['utility', 'reasoning', 'deep', 'subagent'] as const) {
tierModels.push(await resolveModel(engine, { tier, fallback: TIER_DEFAULTS[tier] }));
}
_config = { ...cfg, expansion_model: expansionFull, chat_model: chatFull };
_modelCache.clear();
_shrinkState.clear();
@@ -532,6 +546,7 @@ export async function reconfigureGatewayWithEngine(engine: BrainEngine): Promise
_config.chat_model,
_config.reranker_model,
...(_config.chat_fallback_chain ?? []),
...tierModels,
]) {
if (m) registerExtendedModel(m);
}
+4 -1
View File
@@ -17,13 +17,16 @@ export const anthropic: Recipe = {
touchpoints: {
// No embedding model available.
expansion: {
models: ['claude-haiku-4-5-20251001', 'claude-sonnet-4-6'],
models: ['claude-haiku-4-5-20251001', 'claude-sonnet-5', 'claude-sonnet-4-6'],
cost_per_1m_tokens_usd: 0.25,
price_last_verified: '2026-05-10',
},
chat: {
models: [
'claude-fable-5',
'claude-opus-4-8',
'claude-opus-4-7',
'claude-sonnet-5',
'claude-sonnet-4-6',
'claude-haiku-4-5-20251001',
],
+21 -3
View File
@@ -454,13 +454,31 @@ export async function runThink(
// Closes #952 (think over MCP returns "no LLM available").
const client = opts.client ?? await tryBuildGatewayClient(modelUsed, { explicitModel: opts.modelExplicit });
if (!client) {
warnings.push('NO_ANTHROPIC_API_KEY');
// Label the failure honestly: a missing key and an unusable model id are
// different incidents with different fixes. Pre-fix EVERY null client was
// stamped NO_ANTHROPIC_API_KEY, which sent operators chasing env/keychain
// problems when the real cause was a model id the recipe didn't know
// (e.g. a tier-configured model newer than the recipe list). The re-probe
// is pure and cheap (no IO): same predicate tryBuildGatewayClient used.
const probe = probeChatModel(normalizeModelId(modelUsed));
const modelProblem = !probe.ok && probe.reason !== 'unavailable';
warnings.push(
modelProblem ? `MODEL_NOT_USABLE:${(probe as { reason: string }).reason}` : 'NO_ANTHROPIC_API_KEY',
);
const detail = !probe.ok ? probe.detail : '';
const fix = !probe.ok && probe.fix ? ` Fix: ${probe.fix}` : '';
// Degrade gracefully: return the gather without synthesis. Better than throwing.
return {
question: opts.question,
answer: '(no LLM available — set ANTHROPIC_API_KEY or pass `client`)',
answer: modelProblem
? `(model "${modelUsed}" not usable — ${detail}${fix})`
: '(no LLM available — set ANTHROPIC_API_KEY or pass `client`)',
citations: [],
gaps: ['no LLM available; gather succeeded but synthesis skipped'],
gaps: [
modelProblem
? `model "${modelUsed}" not usable (${(probe as { reason: string }).reason}); gather succeeded but synthesis skipped`
: 'no LLM available; gather succeeded but synthesis skipped',
],
pagesGathered: gather.pages.length,
takesGathered: gather.takes.length,
graphHits: gather.graphSlugs.length,
+14
View File
@@ -35,6 +35,20 @@ describe('Anthropic recipe model IDs', () => {
expect(anthropic.aliases?.['claude-sonnet-4-6-20250929']).toBe('claude-sonnet-4-6');
});
it('current-generation models are listed for chat (Fable 5 / Opus 4.8 / Sonnet 5)', () => {
// Regression guard for the tier-config incident: a brain with
// `models.tier.deep = anthropic:claude-opus-4-8` had think/auto_think
// silently degrade because the recipe list stopped at Opus 4.7.
const chatModels = anthropic.touchpoints?.chat?.models ?? [];
expect(chatModels).toContain('claude-fable-5');
expect(chatModels).toContain('claude-opus-4-8');
expect(chatModels).toContain('claude-sonnet-5');
});
it('Sonnet 5 is listed for expansion', () => {
expect(anthropic.touchpoints?.expansion?.models ?? []).toContain('claude-sonnet-5');
});
it('all listed models follow naming conventions', () => {
const allModels = [
...(anthropic.touchpoints?.chat?.models ?? []),
+63
View File
@@ -0,0 +1,63 @@
/**
* reconfigureGatewayWithEngine — tier-resolved models join the extended set.
*
* assertTouchpoint's extended-models contract (model-resolver.ts) says models
* the user opted into via config — `models.default` and `models.tier.*`
* included — bypass the native recipe allowlist. Pre-fix, only chat/expansion/
* embedding/reranker were registered, so a model reachable ONLY through a tier
* (e.g. `models.tier.deep` set to an Opus newer than the recipe list) failed
* `probeChatModel` and silently degraded think/auto_think to the gather-only
* stub — mislabeled NO_ANTHROPIC_API_KEY.
*
* Uses a deliberately fictional model id so the test stays valid no matter how
* current the recipe list is.
*/
import { describe, test, expect, afterEach } from 'bun:test';
import {
configureGateway,
reconfigureGatewayWithEngine,
resetGateway,
validateModelId,
} from '../src/core/ai/gateway.ts';
import type { BrainEngine } from '../src/core/engine.ts';
function stubEngine(config: Record<string, string>): BrainEngine {
return { getConfig: async (k: string) => config[k] ?? null } as unknown as BrainEngine;
}
afterEach(() => {
resetGateway();
});
describe('reconfigureGatewayWithEngine — tier models extend the allowlist', () => {
test('a models.tier.deep model unknown to the recipe validates after reconfigure', async () => {
configureGateway({
embedding_model: 'openai:text-embedding-3-large',
embedding_dimensions: 1536,
env: { ANTHROPIC_API_KEY: 'sk-fake', OPENAI_API_KEY: 'sk-fake' },
});
// Pre-reconfigure: an id absent from the recipe allowlist is rejected.
expect(validateModelId('anthropic:claude-hypothetical-9').ok).toBe(false);
await reconfigureGatewayWithEngine(
stubEngine({ 'models.tier.deep': 'anthropic:claude-hypothetical-9' }),
);
// Post-reconfigure: the tier-configured model is in the extended set.
expect(validateModelId('anthropic:claude-hypothetical-9').ok).toBe(true);
// An id configured NOWHERE stays rejected — the allowlist still bites.
expect(validateModelId('anthropic:claude-never-configured-1').ok).toBe(false);
});
test('models.default reaches the extended set through tier resolution', async () => {
configureGateway({
embedding_model: 'openai:text-embedding-3-large',
embedding_dimensions: 1536,
env: { ANTHROPIC_API_KEY: 'sk-fake', OPENAI_API_KEY: 'sk-fake' },
});
await reconfigureGatewayWithEngine(
stubEngine({ 'models.default': 'anthropic:claude-hypothetical-10' }),
);
expect(validateModelId('anthropic:claude-hypothetical-10').ok).toBe(true);
});
});
+17
View File
@@ -186,6 +186,23 @@ describe('think gateway adapter — #1698 slash form + explicit-model fork', ()
});
});
describe('think gateway adapter — current-generation recipe models', () => {
test('builds clients for Opus 4.8 / Sonnet 5 / Fable 5 (recipe-list refresh)', async () => {
// Regression guard: these GA models were absent from the recipe allowlist,
// so a tier-configured deep model degraded think to the no-LLM stub.
await withEnv({ ANTHROPIC_API_KEY: 'sk-test-fake' }, async () => {
for (const id of [
'anthropic:claude-opus-4-8',
'anthropic:claude-sonnet-5',
'anthropic:claude-fable-5',
]) {
const client = await __thinkAdapter.tryBuildGatewayClient(id);
expect(client).not.toBeNull();
}
});
});
});
describe('think gateway adapter — graceful fallback shape', () => {
test('buildGracefulMessage produces a parseable Anthropic.Message-shaped object', () => {
const m = __thinkAdapter.buildGracefulMessage('anthropic:claude-opus-4-7');
+22 -1
View File
@@ -217,6 +217,24 @@ describe('runThink (with stub client)', () => {
expect(result.rounds).toBe(0);
});
test('labels an unusable CONFIGURED model honestly (MODEL_NOT_USABLE, not NO_ANTHROPIC_API_KEY)', async () => {
// Regression guard: a configured model the recipe rejects (unknown_model)
// used to be stamped NO_ANTHROPIC_API_KEY, sending operators to debug
// env/keychain when the fix was the model id. Model validity beats the key
// check in probeChatModel, so the honest label holds even keyless.
await engine.setConfig('models.think', 'anthropic:claude-bogus-9');
try {
const result = await withoutAnthropicKey(() => runThink(engine, { question: 'bad model test' }));
expect(result.warnings).toContain('MODEL_NOT_USABLE:unknown_model');
expect(result.warnings).not.toContain('NO_ANTHROPIC_API_KEY');
expect(result.answer).toContain('not usable');
expect(result.rounds).toBe(0);
expect(result.synthesisOk).toBe(false);
} finally {
await engine.unsetConfig('models.think');
}
});
test('persistSynthesis writes synthesis page + evidence rows', async () => {
const stubClient: ThinkLLMClient = {
create: async () => ({
@@ -285,8 +303,11 @@ describe('runThink — #1698 explicit-model hard error', () => {
test('NON-explicit bad model does NOT throw — graceful degrade (no modelExplicit)', async () => {
// model present but modelExplicit unset → early gate skipped; builder returns null.
// Hermetic no-key so the assertion can't be perturbed by a configured key.
// Post-honest-labeling: an unknown PROVIDER is a model problem, not a key
// problem — the warning names it instead of the old NO_ANTHROPIC_API_KEY
// catch-all. The graceful no-throw contract is unchanged.
const result = await withoutAnthropicKey(() => runThink(engine, { question: 'nonexplicit bad', model: 'bogusprovider:foo' }));
expect(result.warnings).toContain('NO_ANTHROPIC_API_KEY');
expect(result.warnings).toContain('MODEL_NOT_USABLE:unknown_provider');
expect(result.synthesisOk).toBe(false);
});
});