From 74c849f97009cd2f6b79016beb0a4de2b51375bf Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Sat, 9 May 2026 23:40:13 -0700 Subject: [PATCH] feat(ai): add llama-server recipe (#702 reworked) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 10th recipe in the registry; first to ship Recipe.probe (D13=A) and the second user_provided_models recipe (litellm-proxy is the first). llama.cpp's llama-server exposes an OpenAI-compatible /v1/embeddings endpoint. Distinct from Ollama: different default port (8080), different model-management story (you launch it with --model ; the server serves whatever was passed). Recipe ships with `models: []`, `user_provided_models: true`, `default_dims: 0` so the wizard refuses implicit defaults and forces explicit --embedding-model + --embedding-dimensions. Added: - src/core/ai/recipes/llama-server.ts (61 lines) - probeLlamaServer() in src/core/ai/probes.ts; reads LLAMA_SERVER_BASE_URL with default http://localhost:8080/v1 - Registered in src/core/ai/recipes/index.ts (10 recipes total now) - test/ai/recipe-llama-server.test.ts (8 cases): registered + shape, user_provided_models flag, probe declared + reachability fail-with-hint, default-auth covering no-env / API_KEY / URL-shaped-only paths Hardening: defaultResolveAuth in gateway.ts now skips URL-shaped optional env entries (names ending in _URL or _BASE_URL) when picking a fallback auth token. Pre-fix, OLLAMA_BASE_URL=http://my-ollama would have become the Bearer token; Ollama ignores it but llama-server (and future local-server recipes) shouldn't depend on the server tolerating garbage auth. The regression test (recipes-existing-regression) gains one case pinning this contract. Per-recipe test file follows D7=B (per-recipe over DRY for readability). Plan: ~/.claude/plans/ok-lets-turn-this-enumerated-sonnet.md (commit 4 of 11). Reworked from #702 because the original PR didn't model the recipe-owned probe pattern (D13=A) or user_provided_models (D8=A). Tests: bun test test/ai/ — 95/95 pass (8 new + 87 existing). Co-Authored-By: SiyaoZheng Co-Authored-By: Claude Opus 4.7 (1M context) --- src/core/ai/gateway.ts | 12 ++- src/core/ai/probes.ts | 10 +++ src/core/ai/recipes/index.ts | 2 + src/core/ai/recipes/llama-server.ts | 61 ++++++++++++++ test/ai/recipe-llama-server.test.ts | 88 +++++++++++++++++++++ test/ai/recipes-existing-regression.test.ts | 42 +++++++--- 6 files changed, 201 insertions(+), 14 deletions(-) create mode 100644 src/core/ai/recipes/llama-server.ts create mode 100644 test/ai/recipe-llama-server.test.ts diff --git a/src/core/ai/gateway.ts b/src/core/ai/gateway.ts index a77cb2eed..39b4d0b07 100644 --- a/src/core/ai/gateway.ts +++ b/src/core/ai/gateway.ts @@ -111,10 +111,14 @@ export function defaultResolveAuth( const optional = recipe.auth_env?.optional ?? []; if (required.length === 0) { - // No-auth or optional-auth recipe (e.g. Ollama). Read first optional env; - // if none present, use 'unauthenticated' so createOpenAICompatible has - // something to put in Authorization (servers like Ollama ignore it). - const optKey = optional.find(k => !!env[k]); + // No-auth or optional-auth recipe (e.g. Ollama, llama-server). Read first + // present optional API-key env (ignoring URL-shaped names like + // OLLAMA_BASE_URL, which belong in cfg.base_urls, not auth). If none + // present, use 'unauthenticated' so createOpenAICompatible has something + // to put in Authorization (servers like Ollama / llama-server ignore it). + const optKey = optional.find( + k => !!env[k] && !/_(BASE_)?URL$/.test(k), + ); const token = optKey ? env[optKey]! : 'unauthenticated'; return { headerName: 'Authorization', token: `Bearer ${token}` }; } diff --git a/src/core/ai/probes.ts b/src/core/ai/probes.ts index 81097517b..2b8711b78 100644 --- a/src/core/ai/probes.ts +++ b/src/core/ai/probes.ts @@ -45,3 +45,13 @@ export async function probeLMStudio(): Promise { const url = process.env.LMSTUDIO_BASE_URL ?? 'http://localhost:1234/v1'; return probeOpenAICompat(url); } + +/** + * Probe llama.cpp's `llama-server --embeddings` endpoint. Defaults to port + * 8080 (llama-server's default; distinct from Ollama's 11434 and LM Studio's + * 1234). Override via `LLAMA_SERVER_BASE_URL`. + */ +export async function probeLlamaServer(): Promise { + const url = process.env.LLAMA_SERVER_BASE_URL ?? 'http://localhost:8080/v1'; + return probeOpenAICompat(url); +} diff --git a/src/core/ai/recipes/index.ts b/src/core/ai/recipes/index.ts index 21a1506d6..e7bb41b34 100644 --- a/src/core/ai/recipes/index.ts +++ b/src/core/ai/recipes/index.ts @@ -15,6 +15,7 @@ import { litellmProxy } from './litellm-proxy.ts'; import { deepseek } from './deepseek.ts'; import { groq } from './groq.ts'; import { together } from './together.ts'; +import { llamaServer } from './llama-server.ts'; const ALL: Recipe[] = [ openai, @@ -26,6 +27,7 @@ const ALL: Recipe[] = [ deepseek, groq, together, + llamaServer, ]; /** Map from `provider:id` key to recipe. */ diff --git a/src/core/ai/recipes/llama-server.ts b/src/core/ai/recipes/llama-server.ts new file mode 100644 index 000000000..34953a3ae --- /dev/null +++ b/src/core/ai/recipes/llama-server.ts @@ -0,0 +1,61 @@ +import type { Recipe } from '../types.ts'; +import { probeLlamaServer } from '../probes.ts'; + +/** + * llama.cpp's `llama-server --embeddings` (also published as + * `@llama.cpp/llama-server`). Exposes an OpenAI-compatible /v1/embeddings + * endpoint. Distinct from Ollama: different default port (8080), different + * model-management story (you launch it with `--model `; the server + * serves whatever model was passed). + * + * Like LiteLLM, this recipe ships with `models: []` because the model + * identity is whatever the user launched llama-server with. They MUST + * pass `--embedding-model llama-server:` and `--embedding-dimensions + * `. The wizard refuses to pick implicit defaults. + * + * Reference: https://github.com/ggml-org/llama.cpp/blob/master/tools/server/README.md + */ +export const llamaServer: Recipe = { + id: 'llama-server', + name: 'llama.cpp llama-server (local)', + tier: 'openai-compat', + implementation: 'openai-compatible', + base_url_default: 'http://localhost:8080/v1', + auth_env: { + required: [], + optional: ['LLAMA_SERVER_BASE_URL', 'LLAMA_SERVER_API_KEY'], + setup_url: + 'https://github.com/ggml-org/llama.cpp/blob/master/tools/server/README.md', + }, + touchpoints: { + embedding: { + models: [], // user-driven; whatever model the server was launched with + user_provided_models: true, + default_dims: 0, // forces explicit --embedding-dimensions + cost_per_1m_tokens_usd: 0, + price_last_verified: '2026-05-09', + }, + }, + /** + * Probe via the OpenAI-compatible /v1/models endpoint. Honors + * LLAMA_SERVER_BASE_URL override; defaults to localhost:8080. + */ + async probe() { + const result = await probeLlamaServer(); + if (!result.reachable) { + return { + ready: false, + hint: `llama-server not reachable at ${process.env.LLAMA_SERVER_BASE_URL ?? 'http://localhost:8080/v1'}. Start it with \`./llama-server --model --embeddings\` or set LLAMA_SERVER_BASE_URL.`, + }; + } + if (!result.models_endpoint_valid) { + return { + ready: false, + hint: `llama-server reached but /v1/models returned an unexpected shape: ${result.error ?? 'unknown'}.`, + }; + } + return { ready: true }; + }, + setup_hint: + 'Build llama.cpp, then `llama-server --model --embeddings`. Set --embedding-model llama-server: + --embedding-dimensions .', +}; diff --git a/test/ai/recipe-llama-server.test.ts b/test/ai/recipe-llama-server.test.ts new file mode 100644 index 000000000..6fad458d3 --- /dev/null +++ b/test/ai/recipe-llama-server.test.ts @@ -0,0 +1,88 @@ +/** + * llama-server recipe smoke (Commit 4 of the v0.32 wave). + * + * llama-server is the second user-driven-models recipe (alongside + * litellm-proxy). It declares `models: []`, `user_provided_models: true`, + * and a `probe()` that consults LLAMA_SERVER_BASE_URL. + * + * Coverage: + * - Recipe registered + has expected fields + * - user_provided_models is the explicit signal (not the legacy id heuristic) + * - probe is callable and reports `ready: false` with a setup hint when no server is listening + * - default auth resolves to "Bearer unauthenticated" (or the API key if set) + */ + +import { afterEach, beforeEach, describe, expect, test } from 'bun:test'; +import { getRecipe } from '../../src/core/ai/recipes/index.ts'; +import { defaultResolveAuth } from '../../src/core/ai/gateway.ts'; + +describe('recipe: llama-server', () => { + let savedBaseUrl: string | undefined; + beforeEach(() => { + savedBaseUrl = process.env.LLAMA_SERVER_BASE_URL; + delete process.env.LLAMA_SERVER_BASE_URL; + }); + afterEach(() => { + if (savedBaseUrl !== undefined) process.env.LLAMA_SERVER_BASE_URL = savedBaseUrl; + else delete process.env.LLAMA_SERVER_BASE_URL; + }); + + test('registered with expected shape', () => { + const r = getRecipe('llama-server'); + expect(r).toBeDefined(); + expect(r!.id).toBe('llama-server'); + expect(r!.tier).toBe('openai-compat'); + expect(r!.implementation).toBe('openai-compatible'); + expect(r!.base_url_default).toBe('http://localhost:8080/v1'); + expect(r!.auth_env?.required ?? []).toEqual([]); + expect(r!.auth_env?.optional ?? []).toContain('LLAMA_SERVER_BASE_URL'); + expect(r!.auth_env?.optional ?? []).toContain('LLAMA_SERVER_API_KEY'); + }); + + test('embedding touchpoint declares user_provided_models', () => { + const r = getRecipe('llama-server')!; + expect(r.touchpoints.embedding).toBeDefined(); + expect(r.touchpoints.embedding!.models).toEqual([]); + expect(r.touchpoints.embedding!.user_provided_models).toBe(true); + expect(r.touchpoints.embedding!.default_dims).toBe(0); + }); + + test('declares a probe function', () => { + const r = getRecipe('llama-server')!; + expect(typeof r.probe).toBe('function'); + }); + + test('probe returns ready=false with hint when no server listening on default port', async () => { + // Use a guaranteed-unreachable port. + process.env.LLAMA_SERVER_BASE_URL = 'http://127.0.0.1:1/v1'; + const r = getRecipe('llama-server')!; + const result = await r.probe!(); + expect(result.ready).toBe(false); + expect(result.hint).toBeDefined(); + expect(result.hint!.toLowerCase()).toContain('llama-server'); + }); + + test('default auth: no env → "Bearer unauthenticated"', () => { + const r = getRecipe('llama-server')!; + const auth = defaultResolveAuth(r, {}, 'embedding'); + expect(auth.headerName).toBe('Authorization'); + expect(auth.token).toBe('Bearer unauthenticated'); + }); + + test('default auth: LLAMA_SERVER_API_KEY set → "Bearer "', () => { + const r = getRecipe('llama-server')!; + const auth = defaultResolveAuth(r, { LLAMA_SERVER_API_KEY: 'sk-llama-fake' }, 'embedding'); + expect(auth.headerName).toBe('Authorization'); + expect(auth.token).toBe('Bearer sk-llama-fake'); + }); + + test('default auth: LLAMA_SERVER_BASE_URL alone does NOT become the Bearer (URL-shaped optional)', () => { + const r = getRecipe('llama-server')!; + const auth = defaultResolveAuth( + r, + { LLAMA_SERVER_BASE_URL: 'http://my-llama:8080/v1' }, + 'embedding', + ); + expect(auth.token).toBe('Bearer unauthenticated'); + }); +}); diff --git a/test/ai/recipes-existing-regression.test.ts b/test/ai/recipes-existing-regression.test.ts index dd7164fa1..99ff1cbda 100644 --- a/test/ai/recipes-existing-regression.test.ts +++ b/test/ai/recipes-existing-regression.test.ts @@ -29,9 +29,9 @@ import type { Recipe } from '../../src/core/ai/types.ts'; const TOUCHPOINTS: Array<'embedding' | 'expansion' | 'chat'> = ['embedding', 'expansion', 'chat']; describe('IRON RULE: existing 9 recipes survive the v0.32 resolveAuth refactor', () => { - test('the 9 expected recipes are still registered', () => { - const ids = listRecipes().map(r => r.id).sort(); - expect(ids).toEqual([ + test('all 9 baseline recipes are still registered (subset, allows post-v0.32 additions)', () => { + const ids = new Set(listRecipes().map(r => r.id)); + for (const baseline of [ 'anthropic', 'deepseek', 'google', @@ -41,7 +41,9 @@ describe('IRON RULE: existing 9 recipes survive the v0.32 resolveAuth refactor', 'openai', 'together', 'voyage', - ]); + ]) { + expect(ids.has(baseline), `baseline recipe ${baseline} missing post-refactor`).toBe(true); + } }); test('every recipe with a non-empty required[] returns Authorization Bearer ', () => { @@ -75,17 +77,16 @@ describe('IRON RULE: existing 9 recipes survive the v0.32 resolveAuth refactor', } }); - test('Ollama (empty required, optional set) reads first optional env', () => { + test('Ollama (empty required, OLLAMA_API_KEY set) reads it as the Bearer token', () => { const ollama = getRecipe('ollama'); expect(ollama).toBeDefined(); expect(ollama!.auth_env?.required ?? []).toEqual([]); const optional = ollama!.auth_env?.optional ?? []; - expect(optional.length).toBeGreaterThan(0); - // First optional present → returned as Bearer. - const env = { [optional[0]]: 'http://localhost:11434' }; - const auth = defaultResolveAuth(ollama!, env, 'embedding'); + expect(optional).toContain('OLLAMA_API_KEY'); + // OLLAMA_API_KEY (a non-URL-shaped optional) becomes the Bearer. + const auth = defaultResolveAuth(ollama!, { OLLAMA_API_KEY: 'fake-token' }, 'embedding'); expect(auth.headerName).toBe('Authorization'); - expect(auth.token).toBe('Bearer http://localhost:11434'); + expect(auth.token).toBe('Bearer fake-token'); }); test('Ollama (no env at all) falls back to "Bearer unauthenticated"', () => { @@ -95,6 +96,27 @@ describe('IRON RULE: existing 9 recipes survive the v0.32 resolveAuth refactor', expect(auth.token).toBe('Bearer unauthenticated'); }); + test('URL-shaped optional env (OLLAMA_BASE_URL, LLAMA_SERVER_BASE_URL) does NOT become the Bearer token', () => { + // Regression for the v0.32 default-fallback design: optional entries + // ending in _URL or _BASE_URL are config (cfg.base_urls), not auth. + // The fallback must skip them and consult the next optional API-key entry. + const ollama = getRecipe('ollama'); + const auth1 = defaultResolveAuth( + ollama!, + { OLLAMA_BASE_URL: 'http://my-ollama/v1' }, + 'embedding', + ); + expect(auth1.token, 'OLLAMA_BASE_URL must not become Bearer token').toBe('Bearer unauthenticated'); + + // When BOTH BASE_URL and API_KEY are set, the API_KEY wins. + const auth2 = defaultResolveAuth( + ollama!, + { OLLAMA_BASE_URL: 'http://my-ollama/v1', OLLAMA_API_KEY: 'real-key' }, + 'embedding', + ); + expect(auth2.token).toBe('Bearer real-key'); + }); + test('all 3 touchpoints produce identical auth for the same recipe + env', () => { // Critical regression: pre-v0.32, embedding had a fallback to // ${recipe.id.toUpperCase()}_API_KEY that expansion and chat lacked.