From 180f4fcb8a8d0f7b402accdde58f8eccd1aba0ff Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Sun, 10 May 2026 00:29:31 -0700 Subject: [PATCH] =?UTF-8?q?feat(ai):=20adjacent=20fixes=20=E2=80=94=20no?= =?UTF-8?q?=5Fbatch=5Fcap=20(#779)=20+=20config-key=20fallbacks=20(#121)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two small ergonomics fixes folded together (#765 deferred — see TODOS.md follow-up; the CJK PGLite extraction was bigger than the plan estimated). #779 reworked (alexandreroumieu-codeapprentice): silence the missing-max_batch_tokens startup warning for recipes with genuinely dynamic batch capacity. New `EmbeddingTouchpoint.no_batch_cap?: true` field. Set on ollama (capacity depends on locally loaded model + OLLAMA_NUM_PARALLEL), litellm-proxy (depends on backend), llama-server (set by --ctx-size at server launch). Three less stderr warnings on every gateway configure; google still warns (it's a real fixed-cap provider that ought to ship a max_batch_tokens declaration). Bonus: litellm-proxy now declares `user_provided_models: true`, removing the last consumer of the legacy `recipe.id === 'litellm'` hardcode in gateway.ts:223 (D8=A wire-through completion). #121 reworked (vinsew): self-contained API keys. Two parts: 1. config.ts: ANTHROPIC_API_KEY env merge was silently missing. loadConfig() merged OPENAI_API_KEY but not ANTHROPIC_API_KEY into the file-config-shape result. One-line addition. 2. cli.ts:buildGatewayConfig: when ~/.gbrain/config.json declares openai_api_key / anthropic_api_key but the process env doesn't have those env vars set (common for launchd-spawned daemons, agent subprocess tools, containers that don't propagate ~/.zshrc), fold the config-file values into the gateway env snapshot. Process env still wins (loaded last) so per-process overrides keep working. Tests (4 cases in test/ai/no-batch-cap-suppression.test.ts): - Ollama / LiteLLM / llama-server all declare no_batch_cap: true - configureGateway does NOT warn for those three - configureGateway STILL warns for google (regression guard) - Cross-cutting invariant: empty-models recipes declare user_provided_models Tests: bun test test/ai/ — 128/128 (4 new + 124 prior). Plan: ~/.claude/plans/ok-lets-turn-this-enumerated-sonnet.md (commit 9 of 11). #765 (Hunyuan PGLite + CJK keyword fallback) deferred to TODOS.md follow-up; the CJK extraction (~150 lines + scoring logic + tests) is larger than the wave's adjacent-fix lane should carry. Closes that PR with a deferral note. Co-Authored-By: alexandreroumieu-codeapprentice Co-Authored-By: vinsew Co-Authored-By: Claude Opus 4.7 (1M context) --- src/cli.ts | 11 +++- src/core/ai/gateway.ts | 4 ++ src/core/ai/recipes/litellm-proxy.ts | 4 ++ src/core/ai/recipes/llama-server.ts | 5 +- src/core/ai/recipes/ollama.ts | 3 + src/core/ai/types.ts | 13 ++++ src/core/config.ts | 1 + test/ai/no-batch-cap-suppression.test.ts | 79 ++++++++++++++++++++++++ 8 files changed, 118 insertions(+), 2 deletions(-) create mode 100644 test/ai/no-batch-cap-suppression.test.ts diff --git a/src/cli.ts b/src/cli.ts index 9e4d802c1..6016b0265 100755 --- a/src/cli.ts +++ b/src/cli.ts @@ -1116,6 +1116,15 @@ async function handleCliOnly(command: string, args: string[]) { // but not the other previously required remembering to mirror the change; // the helper makes that structural. function buildGatewayConfig(c: GBrainConfig): AIGatewayConfig { + // v0.32 (#121 reworked): when ~/.gbrain/config.json declares + // openai_api_key / anthropic_api_key, fold them into the gateway env so + // recipes that read OPENAI_API_KEY / ANTHROPIC_API_KEY find them. Process + // env still wins (it's loaded last) — this is a fallback for daemons / + // launchd-spawned subprocesses that don't propagate ~/.zshrc-sourced keys. + const envFromConfig: Record = {}; + if (c.openai_api_key) envFromConfig.OPENAI_API_KEY = c.openai_api_key; + if (c.anthropic_api_key) envFromConfig.ANTHROPIC_API_KEY = c.anthropic_api_key; + return { embedding_model: c.embedding_model, embedding_dimensions: c.embedding_dimensions, @@ -1124,7 +1133,7 @@ function buildGatewayConfig(c: GBrainConfig): AIGatewayConfig { chat_model: c.chat_model, chat_fallback_chain: c.chat_fallback_chain, base_urls: c.provider_base_urls, - env: { ...process.env }, + env: { ...envFromConfig, ...process.env }, // process.env wins }; } diff --git a/src/core/ai/gateway.ts b/src/core/ai/gateway.ts index 0b54552c3..affba0ab6 100644 --- a/src/core/ai/gateway.ts +++ b/src/core/ai/gateway.ts @@ -234,6 +234,10 @@ function warnRecipesMissingBatchTokens(): void { // recipe; suppress the warning for it. Every other recipe missing the // field is suspicious. if (recipe.id === 'openai') continue; + // v0.32 (#779): explicit opt-out for dynamic-cap recipes (Ollama, + // LiteLLM proxy, llama-server) — they ship without a static cap because + // the cap depends on a user-launched server. Warning is noise for them. + if (embedding.no_batch_cap === true) continue; if (_warnedRecipes.has(recipe.id)) continue; _warnedRecipes.add(recipe.id); // eslint-disable-next-line no-console diff --git a/src/core/ai/recipes/litellm-proxy.ts b/src/core/ai/recipes/litellm-proxy.ts index 8f7da2dea..d7ee4e946 100644 --- a/src/core/ai/recipes/litellm-proxy.ts +++ b/src/core/ai/recipes/litellm-proxy.ts @@ -23,9 +23,13 @@ export const litellmProxy: Recipe = { embedding: { // Models depend on the proxy's config; declare empties so wizard prompts user. models: [], + user_provided_models: true, // v0.32 D8=A wire-through for the litellm hardcode default_dims: 0, // user must declare --embedding-dimensions explicitly cost_per_1m_tokens_usd: undefined, price_last_verified: '2026-04-20', + // LiteLLM's batch capacity is determined by the backend it proxies; + // no static cap to declare here. v0.32 (#779). + no_batch_cap: true, }, }, setup_hint: 'Run LiteLLM (https://docs.litellm.ai) in front of any provider; set LITELLM_BASE_URL + pass --embedding-model litellm: and --embedding-dimensions .', diff --git a/src/core/ai/recipes/llama-server.ts b/src/core/ai/recipes/llama-server.ts index 34953a3ae..aad2dcafa 100644 --- a/src/core/ai/recipes/llama-server.ts +++ b/src/core/ai/recipes/llama-server.ts @@ -33,7 +33,10 @@ export const llamaServer: Recipe = { user_provided_models: true, default_dims: 0, // forces explicit --embedding-dimensions cost_per_1m_tokens_usd: 0, - price_last_verified: '2026-05-09', + price_last_verified: '2026-05-10', + // llama-server's batch capacity is set by `--ctx-size` at launch + // time; no static cap to declare. v0.32 (#779). + no_batch_cap: true, }, }, /** diff --git a/src/core/ai/recipes/ollama.ts b/src/core/ai/recipes/ollama.ts index 31fac0ae6..0f355cdae 100644 --- a/src/core/ai/recipes/ollama.ts +++ b/src/core/ai/recipes/ollama.ts @@ -17,6 +17,9 @@ export const ollama: Recipe = { default_dims: 768, // nomic-embed-text native dim cost_per_1m_tokens_usd: 0, price_last_verified: '2026-04-20', + // Ollama's batch capacity depends on the locally loaded model + the + // OLLAMA_NUM_PARALLEL config; no static cap to declare. v0.32 (#779). + no_batch_cap: true, }, }, setup_hint: 'Install Ollama from https://ollama.ai, then `ollama pull nomic-embed-text` and `ollama serve`.', diff --git a/src/core/ai/types.ts b/src/core/ai/types.ts index b995a5e64..88cb8d1a9 100644 --- a/src/core/ai/types.ts +++ b/src/core/ai/types.ts @@ -86,6 +86,19 @@ export interface EmbeddingTouchpoint { * for shorthand `--model ` and prints a setup hint. */ user_provided_models?: true; + /** + * v0.32 (#779 reworked): explicit opt-out of the missing-max_batch_tokens + * startup warning. Set to `true` for recipes whose batch capacity is + * genuinely dynamic (Ollama: depends on user-loaded model; LiteLLM proxy: + * depends on backend; llama.cpp: depends on `--ctx-size` at server launch). + * + * Without this flag, missing `max_batch_tokens` triggers a once-per-process + * stderr warning so future recipes that forget the cap (and would + * silently rely on recursive-halving) don't ship un-noticed. Recipes that + * declare `no_batch_cap: true` are explicitly opting out — the warning is + * noise for them. + */ + no_batch_cap?: true; } /** diff --git a/src/core/config.ts b/src/core/config.ts index c0c6f09e4..0129ab171 100644 --- a/src/core/config.ts +++ b/src/core/config.ts @@ -144,6 +144,7 @@ export function loadConfig(): GBrainConfig | null { engine: inferredEngine, ...(dbUrl ? { database_url: dbUrl } : {}), ...(process.env.OPENAI_API_KEY ? { openai_api_key: process.env.OPENAI_API_KEY } : {}), + ...(process.env.ANTHROPIC_API_KEY ? { anthropic_api_key: process.env.ANTHROPIC_API_KEY } : {}), ...(process.env.GBRAIN_EMBEDDING_MODEL ? { embedding_model: process.env.GBRAIN_EMBEDDING_MODEL } : {}), ...(process.env.GBRAIN_EMBEDDING_DIMENSIONS ? { embedding_dimensions: parseInt(process.env.GBRAIN_EMBEDDING_DIMENSIONS, 10) } : {}), ...(process.env.GBRAIN_EXPANSION_MODEL ? { expansion_model: process.env.GBRAIN_EXPANSION_MODEL } : {}), diff --git a/test/ai/no-batch-cap-suppression.test.ts b/test/ai/no-batch-cap-suppression.test.ts new file mode 100644 index 000000000..9bd3e69b7 --- /dev/null +++ b/test/ai/no-batch-cap-suppression.test.ts @@ -0,0 +1,79 @@ +/** + * #779 + #121 adjacent fixes (Commit 9 of v0.32 wave). + * + * Coverage: + * - Recipes with `embedding.no_batch_cap: true` suppress the + * missing-max_batch_tokens startup warning (#779) + * - Real-provider recipes without the flag still warn (regression guard) + * - listRecipes returns expected dynamic-cap recipes (ollama, litellm, + * llama-server) all flagged + */ + +import { afterAll, beforeAll, describe, expect, mock, test } from 'bun:test'; +import { configureGateway, resetGateway } from '../../src/core/ai/gateway.ts'; +import { listRecipes, getRecipe } from '../../src/core/ai/recipes/index.ts'; + +describe('v0.32 #779: no_batch_cap suppresses the missing-max_batch_tokens warning', () => { + let warnSpy: ReturnType; + let realWarn: typeof console.warn; + + beforeAll(() => { + realWarn = console.warn; + warnSpy = mock(() => {}); + console.warn = warnSpy as any; + }); + + afterAll(() => { + console.warn = realWarn; + resetGateway(); + }); + + test('Ollama, LiteLLM, llama-server all declare no_batch_cap: true', () => { + for (const id of ['ollama', 'litellm', 'llama-server']) { + const r = getRecipe(id); + expect(r, `${id} not registered`).toBeDefined(); + expect( + r!.touchpoints.embedding?.no_batch_cap, + `${id} should declare no_batch_cap: true`, + ).toBe(true); + } + }); + + test('configureGateway does NOT warn for ollama/litellm/llama-server', () => { + warnSpy.mockClear(); + resetGateway(); + configureGateway({ env: {} }); + const messages = warnSpy.mock.calls.map(c => String(c[0] ?? '')); + for (const id of ['ollama', 'litellm', 'llama-server']) { + expect( + messages.some(m => m.includes(`"${id}"`)), + `should NOT warn for ${id}`, + ).toBe(false); + } + }); + + test('configureGateway STILL warns for google (real provider, no cap declared)', () => { + warnSpy.mockClear(); + resetGateway(); + configureGateway({ env: {} }); + const messages = warnSpy.mock.calls.map(c => String(c[0] ?? '')); + expect( + messages.some(m => m.includes('"google"') && m.includes('without max_batch_tokens')), + 'google should warn (it has fixed-cap models)', + ).toBe(true); + }); + + test('every recipe with empty models[] declares user_provided_models OR has openai-fast-path', () => { + // Cross-cutting invariant: contracts should not silently disagree. + for (const r of listRecipes()) { + const e = r.touchpoints.embedding; + if (!e) continue; + if (e.models.length === 0) { + expect( + e.user_provided_models === true || r.id === 'litellm', + `${r.id} has empty models[] — must declare user_provided_models: true`, + ).toBe(true); + } + } + }); +});