From 447e57ec41b40197ac90bed60887d8706f2c24ca Mon Sep 17 00:00:00 2001 From: Paolo Belcastro <1436372+p3ob7o@users.noreply.github.com> Date: Tue, 21 Jul 2026 22:06:32 +0200 Subject: [PATCH] =?UTF-8?q?fix(ai):=20tier-configured=20models=20reach=20t?= =?UTF-8?q?he=20recipe=20allowlist=20=E2=80=94=20refresh=20Anthropic=20mod?= =?UTF-8?q?els,=20register=20tier=20resolutions,=20honest=20probe=20labels?= =?UTF-8?q?=20(#2800)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Setting `models.tier.deep anthropic:claude-opus-4-8` silently disabled think and auto_think: the Anthropic recipe's chat allowlist stopped at Opus 4.7, the tier-resolved model never joined the extended set that assertTouchpoint's contract promises for config-chosen models, and the resulting probe failure was stamped NO_ANTHROPIC_API_KEY — sending the operator to debug env/keychain when the fix was the model id. Three fixes, one per layer: - recipes/anthropic.ts: add claude-fable-5, claude-opus-4-8, and claude-sonnet-5 to chat models; claude-sonnet-5 to expansion models. - gateway.ts reconfigureGatewayWithEngine: resolve all four tiers and register the results as extended models, honoring the documented contract for models.default / models.tier.* (model-resolver.ts docstring). A tier-only model now validates like a chat/expansion one. - think/index.ts: when the gateway client can't be built, re-probe and label honestly — MODEL_NOT_USABLE: for unknown_model / unknown_provider, NO_ANTHROPIC_API_KEY only for the actual missing-key case; the stub answer carries the probe detail and fix hint. Tests: recipe-list presence pins; a new gateway-tier-extended-models suite proving a fictional tier model validates post-reconfigure (and an unconfigured one still doesn't); think-pipeline coverage for the honest label (unknown_model beats missing-key even keyless); the existing non-explicit bogus-provider test updated from the old catch-all label to the honest one (no-throw contract unchanged). Verified live: a brain with tier.deep=claude-opus-4-8 had think degrade to gather-only with the misleading key warning; with this change the probe passes and synthesis runs. Co-authored-by: Paolo Belcastro Co-authored-by: Claude Fable 5 --- src/core/ai/gateway.ts | 15 ++++++ src/core/ai/recipes/anthropic.ts | 5 +- src/core/think/index.ts | 24 +++++++-- test/anthropic-model-ids.test.ts | 14 +++++ test/gateway-tier-extended-models.test.ts | 63 +++++++++++++++++++++++ test/think-gateway-adapter.test.ts | 17 ++++++ test/think-pipeline.serial.test.ts | 23 ++++++++- 7 files changed, 156 insertions(+), 5 deletions(-) create mode 100644 test/gateway-tier-extended-models.test.ts diff --git a/src/core/ai/gateway.ts b/src/core/ai/gateway.ts index 3752a7919..8a2674484 100644 --- a/src/core/ai/gateway.ts +++ b/src/core/ai/gateway.ts @@ -521,6 +521,20 @@ export async function reconfigureGatewayWithEngine(engine: BrainEngine): Promise const expansionFull = newExpansion.includes(':') ? newExpansion : prefixWithProviderFrom(cfg.expansion_model ?? DEFAULT_EXPANSION_MODEL, newExpansion); const chatFull = newChat.includes(':') ? newChat : prefixWithProviderFrom(cfg.chat_model ?? DEFAULT_CHAT_MODEL, newChat); + // ALSO resolve the four tier models and register them as extended models. + // assertTouchpoint's contract (model-resolver.ts) says config-chosen models — + // `models.default` and `models.tier.*` included — bypass the native recipe + // allowlist, but pre-fix only chat/expansion/embedding/reranker were + // registered. A model reachable ONLY through a tier (e.g. `models.tier.deep` + // set to an Opus newer than the recipe list) failed `probeChatModel` at call + // time and silently degraded think/auto_think to the gather-only stub. + // Resolving per-tier also honors `models.default` (it sits above tiers in + // the resolveModel chain). + const tierModels: string[] = []; + for (const tier of ['utility', 'reasoning', 'deep', 'subagent'] as const) { + tierModels.push(await resolveModel(engine, { tier, fallback: TIER_DEFAULTS[tier] })); + } + _config = { ...cfg, expansion_model: expansionFull, chat_model: chatFull }; _modelCache.clear(); _shrinkState.clear(); @@ -532,6 +546,7 @@ export async function reconfigureGatewayWithEngine(engine: BrainEngine): Promise _config.chat_model, _config.reranker_model, ...(_config.chat_fallback_chain ?? []), + ...tierModels, ]) { if (m) registerExtendedModel(m); } diff --git a/src/core/ai/recipes/anthropic.ts b/src/core/ai/recipes/anthropic.ts index 4d09bcf7f..dda33ac7b 100644 --- a/src/core/ai/recipes/anthropic.ts +++ b/src/core/ai/recipes/anthropic.ts @@ -17,13 +17,16 @@ export const anthropic: Recipe = { touchpoints: { // No embedding model available. expansion: { - models: ['claude-haiku-4-5-20251001', 'claude-sonnet-4-6'], + models: ['claude-haiku-4-5-20251001', 'claude-sonnet-5', 'claude-sonnet-4-6'], cost_per_1m_tokens_usd: 0.25, price_last_verified: '2026-05-10', }, chat: { models: [ + 'claude-fable-5', + 'claude-opus-4-8', 'claude-opus-4-7', + 'claude-sonnet-5', 'claude-sonnet-4-6', 'claude-haiku-4-5-20251001', ], diff --git a/src/core/think/index.ts b/src/core/think/index.ts index 9c31c81dc..e11d7475f 100644 --- a/src/core/think/index.ts +++ b/src/core/think/index.ts @@ -454,13 +454,31 @@ export async function runThink( // Closes #952 (think over MCP returns "no LLM available"). const client = opts.client ?? await tryBuildGatewayClient(modelUsed, { explicitModel: opts.modelExplicit }); if (!client) { - warnings.push('NO_ANTHROPIC_API_KEY'); + // Label the failure honestly: a missing key and an unusable model id are + // different incidents with different fixes. Pre-fix EVERY null client was + // stamped NO_ANTHROPIC_API_KEY, which sent operators chasing env/keychain + // problems when the real cause was a model id the recipe didn't know + // (e.g. a tier-configured model newer than the recipe list). The re-probe + // is pure and cheap (no IO): same predicate tryBuildGatewayClient used. + const probe = probeChatModel(normalizeModelId(modelUsed)); + const modelProblem = !probe.ok && probe.reason !== 'unavailable'; + warnings.push( + modelProblem ? `MODEL_NOT_USABLE:${(probe as { reason: string }).reason}` : 'NO_ANTHROPIC_API_KEY', + ); + const detail = !probe.ok ? probe.detail : ''; + const fix = !probe.ok && probe.fix ? ` Fix: ${probe.fix}` : ''; // Degrade gracefully: return the gather without synthesis. Better than throwing. return { question: opts.question, - answer: '(no LLM available — set ANTHROPIC_API_KEY or pass `client`)', + answer: modelProblem + ? `(model "${modelUsed}" not usable — ${detail}${fix})` + : '(no LLM available — set ANTHROPIC_API_KEY or pass `client`)', citations: [], - gaps: ['no LLM available; gather succeeded but synthesis skipped'], + gaps: [ + modelProblem + ? `model "${modelUsed}" not usable (${(probe as { reason: string }).reason}); gather succeeded but synthesis skipped` + : 'no LLM available; gather succeeded but synthesis skipped', + ], pagesGathered: gather.pages.length, takesGathered: gather.takes.length, graphHits: gather.graphSlugs.length, diff --git a/test/anthropic-model-ids.test.ts b/test/anthropic-model-ids.test.ts index 5ed742649..a7cf46ea3 100644 --- a/test/anthropic-model-ids.test.ts +++ b/test/anthropic-model-ids.test.ts @@ -35,6 +35,20 @@ describe('Anthropic recipe model IDs', () => { expect(anthropic.aliases?.['claude-sonnet-4-6-20250929']).toBe('claude-sonnet-4-6'); }); + it('current-generation models are listed for chat (Fable 5 / Opus 4.8 / Sonnet 5)', () => { + // Regression guard for the tier-config incident: a brain with + // `models.tier.deep = anthropic:claude-opus-4-8` had think/auto_think + // silently degrade because the recipe list stopped at Opus 4.7. + const chatModels = anthropic.touchpoints?.chat?.models ?? []; + expect(chatModels).toContain('claude-fable-5'); + expect(chatModels).toContain('claude-opus-4-8'); + expect(chatModels).toContain('claude-sonnet-5'); + }); + + it('Sonnet 5 is listed for expansion', () => { + expect(anthropic.touchpoints?.expansion?.models ?? []).toContain('claude-sonnet-5'); + }); + it('all listed models follow naming conventions', () => { const allModels = [ ...(anthropic.touchpoints?.chat?.models ?? []), diff --git a/test/gateway-tier-extended-models.test.ts b/test/gateway-tier-extended-models.test.ts new file mode 100644 index 000000000..717ab2baf --- /dev/null +++ b/test/gateway-tier-extended-models.test.ts @@ -0,0 +1,63 @@ +/** + * reconfigureGatewayWithEngine — tier-resolved models join the extended set. + * + * assertTouchpoint's extended-models contract (model-resolver.ts) says models + * the user opted into via config — `models.default` and `models.tier.*` + * included — bypass the native recipe allowlist. Pre-fix, only chat/expansion/ + * embedding/reranker were registered, so a model reachable ONLY through a tier + * (e.g. `models.tier.deep` set to an Opus newer than the recipe list) failed + * `probeChatModel` and silently degraded think/auto_think to the gather-only + * stub — mislabeled NO_ANTHROPIC_API_KEY. + * + * Uses a deliberately fictional model id so the test stays valid no matter how + * current the recipe list is. + */ +import { describe, test, expect, afterEach } from 'bun:test'; +import { + configureGateway, + reconfigureGatewayWithEngine, + resetGateway, + validateModelId, +} from '../src/core/ai/gateway.ts'; +import type { BrainEngine } from '../src/core/engine.ts'; + +function stubEngine(config: Record): BrainEngine { + return { getConfig: async (k: string) => config[k] ?? null } as unknown as BrainEngine; +} + +afterEach(() => { + resetGateway(); +}); + +describe('reconfigureGatewayWithEngine — tier models extend the allowlist', () => { + test('a models.tier.deep model unknown to the recipe validates after reconfigure', async () => { + configureGateway({ + embedding_model: 'openai:text-embedding-3-large', + embedding_dimensions: 1536, + env: { ANTHROPIC_API_KEY: 'sk-fake', OPENAI_API_KEY: 'sk-fake' }, + }); + // Pre-reconfigure: an id absent from the recipe allowlist is rejected. + expect(validateModelId('anthropic:claude-hypothetical-9').ok).toBe(false); + + await reconfigureGatewayWithEngine( + stubEngine({ 'models.tier.deep': 'anthropic:claude-hypothetical-9' }), + ); + + // Post-reconfigure: the tier-configured model is in the extended set. + expect(validateModelId('anthropic:claude-hypothetical-9').ok).toBe(true); + // An id configured NOWHERE stays rejected — the allowlist still bites. + expect(validateModelId('anthropic:claude-never-configured-1').ok).toBe(false); + }); + + test('models.default reaches the extended set through tier resolution', async () => { + configureGateway({ + embedding_model: 'openai:text-embedding-3-large', + embedding_dimensions: 1536, + env: { ANTHROPIC_API_KEY: 'sk-fake', OPENAI_API_KEY: 'sk-fake' }, + }); + await reconfigureGatewayWithEngine( + stubEngine({ 'models.default': 'anthropic:claude-hypothetical-10' }), + ); + expect(validateModelId('anthropic:claude-hypothetical-10').ok).toBe(true); + }); +}); diff --git a/test/think-gateway-adapter.test.ts b/test/think-gateway-adapter.test.ts index dbe830ce6..a08146359 100644 --- a/test/think-gateway-adapter.test.ts +++ b/test/think-gateway-adapter.test.ts @@ -186,6 +186,23 @@ describe('think gateway adapter — #1698 slash form + explicit-model fork', () }); }); +describe('think gateway adapter — current-generation recipe models', () => { + test('builds clients for Opus 4.8 / Sonnet 5 / Fable 5 (recipe-list refresh)', async () => { + // Regression guard: these GA models were absent from the recipe allowlist, + // so a tier-configured deep model degraded think to the no-LLM stub. + await withEnv({ ANTHROPIC_API_KEY: 'sk-test-fake' }, async () => { + for (const id of [ + 'anthropic:claude-opus-4-8', + 'anthropic:claude-sonnet-5', + 'anthropic:claude-fable-5', + ]) { + const client = await __thinkAdapter.tryBuildGatewayClient(id); + expect(client).not.toBeNull(); + } + }); + }); +}); + describe('think gateway adapter — graceful fallback shape', () => { test('buildGracefulMessage produces a parseable Anthropic.Message-shaped object', () => { const m = __thinkAdapter.buildGracefulMessage('anthropic:claude-opus-4-7'); diff --git a/test/think-pipeline.serial.test.ts b/test/think-pipeline.serial.test.ts index d2e1a5442..8c85e6d0b 100644 --- a/test/think-pipeline.serial.test.ts +++ b/test/think-pipeline.serial.test.ts @@ -217,6 +217,24 @@ describe('runThink (with stub client)', () => { expect(result.rounds).toBe(0); }); + test('labels an unusable CONFIGURED model honestly (MODEL_NOT_USABLE, not NO_ANTHROPIC_API_KEY)', async () => { + // Regression guard: a configured model the recipe rejects (unknown_model) + // used to be stamped NO_ANTHROPIC_API_KEY, sending operators to debug + // env/keychain when the fix was the model id. Model validity beats the key + // check in probeChatModel, so the honest label holds even keyless. + await engine.setConfig('models.think', 'anthropic:claude-bogus-9'); + try { + const result = await withoutAnthropicKey(() => runThink(engine, { question: 'bad model test' })); + expect(result.warnings).toContain('MODEL_NOT_USABLE:unknown_model'); + expect(result.warnings).not.toContain('NO_ANTHROPIC_API_KEY'); + expect(result.answer).toContain('not usable'); + expect(result.rounds).toBe(0); + expect(result.synthesisOk).toBe(false); + } finally { + await engine.unsetConfig('models.think'); + } + }); + test('persistSynthesis writes synthesis page + evidence rows', async () => { const stubClient: ThinkLLMClient = { create: async () => ({ @@ -285,8 +303,11 @@ describe('runThink — #1698 explicit-model hard error', () => { test('NON-explicit bad model does NOT throw — graceful degrade (no modelExplicit)', async () => { // model present but modelExplicit unset → early gate skipped; builder returns null. // Hermetic no-key so the assertion can't be perturbed by a configured key. + // Post-honest-labeling: an unknown PROVIDER is a model problem, not a key + // problem — the warning names it instead of the old NO_ANTHROPIC_API_KEY + // catch-all. The graceful no-throw contract is unchanged. const result = await withoutAnthropicKey(() => runThink(engine, { question: 'nonexplicit bad', model: 'bogusprovider:foo' })); - expect(result.warnings).toContain('NO_ANTHROPIC_API_KEY'); + expect(result.warnings).toContain('MODEL_NOT_USABLE:unknown_provider'); expect(result.synthesisOk).toBe(false); }); });