mirror of
https://github.com/garrytan/gbrain.git
synced 2026-07-27 22:15:33 +00:00
fix(ai): OpenRouter family-scoped prompt caching + expansion on chat-capable openai-compat recipes
Takeover of #1988 (OpenRouter prompt caching), reimplemented on current master: supports_prompt_cache may now be a per-model-id predicate; the OpenRouter recipe marks openai/* chat and anthropic/claude-* routes cacheable. Claude routes get an explicit cache_control on the system content block via the recipe compat fetch shim (OpenRouter's documented per-block format, not a top-level body field), signaled through a private in-process marker header instead of the promptCacheKey sentinel that now collides with the real OpenAI prompt_cache_key derivation. Cache reads on OpenAI-compatible routes surface via the SDK's cachedInputTokens. Root fix for #1135: deepseek, groq, and together now declare expansion touchpoints (their expansion path is the same plain OpenAI-compatible languageModel call as chat), so an explicit expansion_model pointed at them no longer silently yields zero expansion. Co-authored-by: tmchow <tmchow@users.noreply.github.com> Co-authored-by: warkcod <warkcod@users.noreply.github.com> Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
tmchow
warkcod
Claude Fable 5
parent
0612b0daa8
commit
00caa22c96
@@ -88,9 +88,13 @@ export function getProviderCapabilities(modelString: string): ProviderCapabiliti
|
||||
// boundary; this function returns capabilities for whatever the user asked
|
||||
// for, on the assumption it'll be validated elsewhere.
|
||||
|
||||
const promptCache = chat.supports_prompt_cache;
|
||||
|
||||
return {
|
||||
supportsToolCalling: chat.supports_tools === true,
|
||||
supportsPromptCaching: chat.supports_prompt_cache === true,
|
||||
supportsPromptCaching: typeof promptCache === 'function'
|
||||
? promptCache(parsed.modelId)
|
||||
: promptCache === true,
|
||||
// No recipe exposes parallel-tools-specifically yet; gate on supports_tools.
|
||||
// Subsequent waves can split this into its own recipe field if a provider
|
||||
// ever supports tools without parallel dispatch.
|
||||
@@ -101,11 +105,6 @@ export function getProviderCapabilities(modelString: string): ProviderCapabiliti
|
||||
supportsThinking: false,
|
||||
maxContext: chat.max_context_tokens ?? 128_000,
|
||||
};
|
||||
|
||||
// The `parsed` binding is intentionally unused — `resolveRecipe` is called
|
||||
// here for its validation side-effects (throws on unknown provider). Keeping
|
||||
// the destructure makes future per-model capability overrides cheap.
|
||||
void parsed;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
+31
-2
@@ -47,6 +47,10 @@ import type {
|
||||
TouchpointKind,
|
||||
} from './types.ts';
|
||||
import { resolveRecipe, assertTouchpoint, parseModelId } from './model-resolver.ts';
|
||||
import {
|
||||
OPENROUTER_CACHE_HEADER,
|
||||
openrouterRequiresExplicitPromptCache,
|
||||
} from './recipes/openrouter.ts';
|
||||
import { resolveModel, TIER_DEFAULTS } from '../model-config.ts';
|
||||
import type { BrainEngine } from '../engine.ts';
|
||||
import { dimsProviderOptions } from './dims.ts';
|
||||
@@ -2709,6 +2713,17 @@ export function probeChatModel(modelStr: string): ChatModelProbe {
|
||||
return { ok: true };
|
||||
}
|
||||
|
||||
/**
|
||||
* Per-model prompt-cache capability: `supports_prompt_cache` may be a static
|
||||
* boolean (native providers) or a per-model-id predicate (OpenRouter's
|
||||
* family-scoped caching).
|
||||
*/
|
||||
function chatSupportsPromptCache(recipe: Recipe, modelId: string): boolean {
|
||||
const support = recipe.touchpoints.chat?.supports_prompt_cache;
|
||||
if (typeof support === 'function') return support(modelId);
|
||||
return support === true;
|
||||
}
|
||||
|
||||
async function resolveChatProvider(modelStr: string): Promise<{ model: any; recipe: Recipe; modelId: string }> {
|
||||
const { parsed, recipe } = resolveRecipe(modelStr);
|
||||
assertTouchpoint(recipe, 'chat', parsed.modelId, getExtendedModelsForProvider(parsed.providerId));
|
||||
@@ -3030,9 +3045,19 @@ export async function chat(opts: ChatOpts): Promise<ChatResult> {
|
||||
const { model, recipe, modelId } = await resolveChatProvider(modelStr);
|
||||
const cfg = requireConfig();
|
||||
|
||||
const supportsCache = recipe.touchpoints.chat?.supports_prompt_cache === true;
|
||||
const supportsCache = chatSupportsPromptCache(recipe, modelId);
|
||||
const useCache = !!opts.cacheSystem && supportsCache;
|
||||
|
||||
// OpenRouter Claude routes need an explicit `cache_control` on the system
|
||||
// content block, but the openai-compatible adapter drops anthropic-namespace
|
||||
// providerOptions before building the wire body. Signal intent via a private
|
||||
// header; the recipe's compat fetch shim rewrites the body and strips the
|
||||
// header before the request leaves the process. OpenAI routes through
|
||||
// OpenRouter cache automatically — no marker needed.
|
||||
const requestHeaders = useCache && recipe.id === 'openrouter' && openrouterRequiresExplicitPromptCache(modelId)
|
||||
? { [OPENROUTER_CACHE_HEADER]: '1' }
|
||||
: undefined;
|
||||
|
||||
const tools = toAISDKTools(opts.tools);
|
||||
|
||||
const providerOptions: Record<string, any> = {};
|
||||
@@ -3144,6 +3169,7 @@ export async function chat(opts: ChatOpts): Promise<ChatResult> {
|
||||
// shorter wins). Covers native-anthropic (the default provider + facts Haiku).
|
||||
abortSignal: withDefaultTimeout(opts.abortSignal, AI_CHAT_TIMEOUT_MS),
|
||||
providerOptions: Object.keys(providerOptions).length > 0 ? providerOptions : undefined,
|
||||
...(requestHeaders ? { headers: requestHeaders } : {}),
|
||||
});
|
||||
|
||||
// Normalize blocks. Vercel SDK gives us `result.content` (an array of typed
|
||||
@@ -3192,7 +3218,10 @@ export async function chat(opts: ChatOpts): Promise<ChatResult> {
|
||||
usage: {
|
||||
input_tokens: inTok,
|
||||
output_tokens: outTok,
|
||||
cache_read_tokens: Number(anthropicCache.cacheReadInputTokens ?? anthropicCache.cache_read_input_tokens ?? 0),
|
||||
// `usage.cachedInputTokens` is the AI SDK's provider-neutral cache-read
|
||||
// count — it's how OpenAI-compatible routes (OpenRouter's
|
||||
// prompt_tokens_details.cached_tokens) surface cache hits.
|
||||
cache_read_tokens: Number(anthropicCache.cacheReadInputTokens ?? anthropicCache.cache_read_input_tokens ?? usage.cachedInputTokens ?? 0),
|
||||
cache_creation_tokens: Number(anthropicCache.cacheCreationInputTokens ?? anthropicCache.cache_creation_input_tokens ?? 0),
|
||||
},
|
||||
model: `${recipe.id}:${modelId}`,
|
||||
|
||||
@@ -76,6 +76,15 @@ export const deepseek: Recipe = {
|
||||
setup_url: 'https://platform.deepseek.com/api_keys',
|
||||
},
|
||||
touchpoints: {
|
||||
// Query expansion reuses the same OpenAI-compatible chat endpoint (the
|
||||
// gateway's expansion path is a plain languageModel call). Without this
|
||||
// declaration an explicit `expansion_model: deepseek:...` silently
|
||||
// yields no expansion (#1135).
|
||||
expansion: {
|
||||
models: ['deepseek-chat'],
|
||||
cost_per_1m_tokens_usd: 0.14,
|
||||
price_last_verified: '2026-04-20',
|
||||
},
|
||||
chat: {
|
||||
models: ['deepseek-chat', 'deepseek-reasoner'],
|
||||
supports_tools: true,
|
||||
|
||||
@@ -16,6 +16,14 @@ export const groq: Recipe = {
|
||||
setup_url: 'https://console.groq.com/keys',
|
||||
},
|
||||
touchpoints: {
|
||||
// Same OpenAI-compatible endpoint as chat; declared so an explicit
|
||||
// `expansion_model: groq:...` resolves instead of silently dropping
|
||||
// expansion (#1135). 8b-instant is the natural expansion pick (cheap,
|
||||
// fast, no tool-calling needed for multi-query rewrites).
|
||||
expansion: {
|
||||
models: ['llama-3.1-8b-instant', 'llama-3.3-70b-versatile'],
|
||||
price_last_verified: '2026-04-20',
|
||||
},
|
||||
chat: {
|
||||
models: [
|
||||
'llama-3.3-70b-versatile',
|
||||
|
||||
@@ -1,5 +1,101 @@
|
||||
import type { Recipe } from '../types.ts';
|
||||
|
||||
/**
|
||||
* Private in-process marker header. `gateway.chat()` sets it when the caller
|
||||
* asked for prompt caching (`cacheSystem`) on an OpenRouter route that needs
|
||||
* an explicit `cache_control` (Anthropic Claude). The compat fetch shim below
|
||||
* strips it and rewrites the body; the header NEVER leaves the process.
|
||||
*
|
||||
* Why a header and not providerOptions: the AI SDK's openai-compatible
|
||||
* adapter validates providerOptions against a fixed schema and silently
|
||||
* drops anthropic-namespace fields before building the wire body (same class
|
||||
* of problem as the embedding `input_type` ALS in gateway.ts). Headers pass
|
||||
* through untouched.
|
||||
*/
|
||||
export const OPENROUTER_CACHE_HEADER = 'x-gbrain-anthropic-prompt-cache';
|
||||
|
||||
/**
|
||||
* Family-scoped prompt-cache capability (per OpenRouter docs):
|
||||
* - OpenAI chat routes cache automatically (no request mutation needed).
|
||||
* - Anthropic Claude routes cache when the request carries `cache_control`
|
||||
* on a content block (applied by the fetch shim below).
|
||||
* Everything else is not marked cacheable — deliberately narrow rather than
|
||||
* blessing every routed model family forever.
|
||||
*/
|
||||
export function openrouterSupportsPromptCache(modelId: string): boolean {
|
||||
const normalized = modelId.trim().toLowerCase();
|
||||
if (normalized.startsWith('openai/gpt-') || /^openai\/o\d/.test(normalized)) return true;
|
||||
if (normalized.startsWith('anthropic/claude-')) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
/** Only Anthropic Claude routes need an explicit cache_control block. */
|
||||
export function openrouterRequiresExplicitPromptCache(modelId: string): boolean {
|
||||
return modelId.trim().toLowerCase().startsWith('anthropic/claude-');
|
||||
}
|
||||
|
||||
/**
|
||||
* Rewrite the last system message's string content into OpenRouter's
|
||||
* documented Anthropic caching shape: a content-part array carrying
|
||||
* `cache_control: { type: 'ephemeral' }` on the text block. (A top-level
|
||||
* body `cache_control` is NOT the OpenRouter format — OR forwards per-block
|
||||
* markers only.) Returns the input unchanged when it doesn't apply.
|
||||
*/
|
||||
function withSystemCacheControl(body: unknown): unknown {
|
||||
if (!body || typeof body !== 'object' || Array.isArray(body)) return body;
|
||||
const record = body as Record<string, unknown>;
|
||||
const model = typeof record.model === 'string' ? record.model : '';
|
||||
if (!openrouterRequiresExplicitPromptCache(model)) return body;
|
||||
const messages = Array.isArray(record.messages) ? record.messages : undefined;
|
||||
if (!messages) return body;
|
||||
let idx = -1;
|
||||
for (let i = 0; i < messages.length; i++) {
|
||||
const m = messages[i];
|
||||
if (m && typeof m === 'object' && (m as Record<string, unknown>).role === 'system') idx = i;
|
||||
}
|
||||
if (idx === -1) return body;
|
||||
const sys = messages[idx] as Record<string, unknown>;
|
||||
if (typeof sys.content !== 'string' || sys.content.length === 0) return body;
|
||||
const next = messages.slice();
|
||||
next[idx] = {
|
||||
...sys,
|
||||
content: [{ type: 'text', text: sys.content, cache_control: { type: 'ephemeral' } }],
|
||||
};
|
||||
return { ...record, messages: next };
|
||||
}
|
||||
|
||||
/**
|
||||
* Compat fetch: honors the OPENROUTER_CACHE_HEADER marker by splicing an
|
||||
* Anthropic cache_control breakpoint onto the system block, then strips the
|
||||
* marker. Fail-open: any parse problem sends the original body unchanged.
|
||||
*
|
||||
* @internal exported for tests. Cast through `unknown` because TS's
|
||||
* `typeof fetch` includes a `preconnect` member (matches azure-openai.ts).
|
||||
*/
|
||||
export const openrouterCompatFetch = (async (
|
||||
input: RequestInfo | URL,
|
||||
init?: RequestInit,
|
||||
): Promise<Response> => {
|
||||
if (!init?.headers) return fetch(input as any, init as any);
|
||||
const headers = new Headers(init.headers as any);
|
||||
if (!headers.has(OPENROUTER_CACHE_HEADER)) return fetch(input as any, init as any);
|
||||
headers.delete(OPENROUTER_CACHE_HEADER);
|
||||
let body = init.body;
|
||||
if (typeof body === 'string') {
|
||||
try {
|
||||
const parsed = JSON.parse(body);
|
||||
const rewritten = withSystemCacheControl(parsed);
|
||||
if (rewritten !== parsed) {
|
||||
body = JSON.stringify(rewritten);
|
||||
headers.delete('content-length');
|
||||
}
|
||||
} catch {
|
||||
// Non-JSON body: let the provider surface the original problem.
|
||||
}
|
||||
}
|
||||
return fetch(input as any, { ...init, headers, body } as any);
|
||||
}) as unknown as typeof fetch;
|
||||
|
||||
/**
|
||||
* OpenRouter — single-key fan-out to OpenAI, Anthropic, Google, DeepSeek, and
|
||||
* dozens of other providers via a single OpenAI-compatible endpoint at
|
||||
@@ -93,7 +189,9 @@ export const openrouter: Recipe = {
|
||||
supports_tools: true,
|
||||
// Informational only — real gate is isAnthropicProvider() upstream.
|
||||
supports_subagent_loop: false,
|
||||
supports_prompt_cache: false,
|
||||
// Family-scoped: OpenAI routes cache automatically; Anthropic routes
|
||||
// cache via the compat fetch shim's cache_control rewrite.
|
||||
supports_prompt_cache: openrouterSupportsPromptCache,
|
||||
// No max_context_tokens: catalog spans 128K to 1M+; a single recipe-wide
|
||||
// value is either unsafe for smaller models or wasteful for larger ones.
|
||||
// Let upstream errors surface per-model.
|
||||
@@ -102,4 +200,5 @@ export const openrouter: Recipe = {
|
||||
},
|
||||
setup_hint:
|
||||
'Get an API key at https://openrouter.ai/settings/keys, then `export OPENROUTER_API_KEY=...` and use `openrouter:<provider>/<model>`. Optional overrides: OPENROUTER_BASE_URL (proxy), OPENROUTER_REFERER (attribution URL), OPENROUTER_TITLE (attribution name).',
|
||||
compat: { fetch: openrouterCompatFetch },
|
||||
};
|
||||
|
||||
@@ -16,6 +16,13 @@ export const together: Recipe = {
|
||||
setup_url: 'https://api.together.ai/settings/api-keys',
|
||||
},
|
||||
touchpoints: {
|
||||
// Same OpenAI-compatible endpoint as chat; declared so an explicit
|
||||
// `expansion_model: together:...` resolves instead of silently dropping
|
||||
// expansion (#1135).
|
||||
expansion: {
|
||||
models: ['meta-llama/Llama-3.3-70B-Instruct-Turbo', 'Qwen/Qwen2.5-72B-Instruct-Turbo'],
|
||||
price_last_verified: '2026-04-20',
|
||||
},
|
||||
chat: {
|
||||
models: [
|
||||
'Qwen/Qwen2.5-72B-Instruct-Turbo',
|
||||
|
||||
@@ -222,8 +222,13 @@ export interface ChatTouchpoint {
|
||||
* Strictly stronger than supports_tools.
|
||||
*/
|
||||
supports_subagent_loop: boolean;
|
||||
/** Anthropic-style ephemeral prompt cache markers honored. */
|
||||
supports_prompt_cache?: boolean;
|
||||
/**
|
||||
* Prompt caching honored for this chat touchpoint. Static booleans cover
|
||||
* native providers; openai-compatible aggregators may decide per model id
|
||||
* (e.g. OpenRouter caches OpenAI and Anthropic routes but not every routed
|
||||
* model family).
|
||||
*/
|
||||
supports_prompt_cache?: boolean | ((modelId: string) => boolean);
|
||||
max_context_tokens?: number;
|
||||
cost_per_1m_input_usd?: number;
|
||||
cost_per_1m_output_usd?: number;
|
||||
|
||||
@@ -24,6 +24,22 @@ describe('getProviderCapabilities (v0.38 Slice 1 — D6/D7 recipe-driven capabil
|
||||
expect(caps.maxContext).toBe(1000000); // Gemini 1.5 Pro
|
||||
});
|
||||
|
||||
it('marks OpenRouter OpenAI/Anthropic routes as cache-capable (per-model predicate)', () => {
|
||||
const openaiCaps = getProviderCapabilities('openrouter:openai/gpt-5.2');
|
||||
expect(openaiCaps.supportsToolCalling).toBe(true);
|
||||
expect(openaiCaps.supportsPromptCaching).toBe(true);
|
||||
|
||||
const anthropicCaps = getProviderCapabilities('openrouter:anthropic/claude-sonnet-4.6');
|
||||
expect(anthropicCaps.supportsToolCalling).toBe(true);
|
||||
expect(anthropicCaps.supportsPromptCaching).toBe(true);
|
||||
});
|
||||
|
||||
it('does not mark every OpenRouter route as cache-capable', () => {
|
||||
const caps = getProviderCapabilities('openrouter:deepseek/deepseek-chat');
|
||||
expect(caps.supportsToolCalling).toBe(true);
|
||||
expect(caps.supportsPromptCaching).toBe(false);
|
||||
});
|
||||
|
||||
it('honors Anthropic alias (undated → dated)', () => {
|
||||
const caps = getProviderCapabilities('anthropic:claude-haiku-4-5');
|
||||
expect(caps.supportsToolCalling).toBe(true);
|
||||
@@ -58,6 +74,12 @@ describe('classifyCapabilities (D6 — three-tier capability verdict)', () => {
|
||||
expect(classifyCapabilities('google:gemini-1.5-pro')).toBe('degraded:no_caching');
|
||||
});
|
||||
|
||||
it('returns ok for cacheable OpenRouter routes, degraded:no_caching otherwise', () => {
|
||||
expect(classifyCapabilities('openrouter:openai/gpt-5.2')).toBe('ok');
|
||||
expect(classifyCapabilities('openrouter:anthropic/claude-sonnet-4.6')).toBe('ok');
|
||||
expect(classifyCapabilities('openrouter:deepseek/deepseek-chat')).toBe('degraded:no_caching');
|
||||
});
|
||||
|
||||
it('returns unknown for unrecognized providers', () => {
|
||||
expect(classifyCapabilities('madeup:something')).toBe('unknown');
|
||||
});
|
||||
|
||||
@@ -29,13 +29,19 @@
|
||||
* the bug made you believe was sufficient.
|
||||
*/
|
||||
|
||||
import { describe, test, expect, beforeEach } from 'bun:test';
|
||||
import { describe, test, expect, beforeEach, afterAll } from 'bun:test';
|
||||
import {
|
||||
chat,
|
||||
configureGateway,
|
||||
resetGateway,
|
||||
__setGenerateTextTransportForTests,
|
||||
} from '../../src/core/ai/gateway.ts';
|
||||
import { OPENROUTER_CACHE_HEADER } from '../../src/core/ai/recipes/openrouter.ts';
|
||||
|
||||
afterAll(() => {
|
||||
resetGateway();
|
||||
__setGenerateTextTransportForTests(null);
|
||||
});
|
||||
|
||||
describe('gbrain#2490 — Anthropic cache breakpoint placement', () => {
|
||||
beforeEach(() => {
|
||||
@@ -193,3 +199,55 @@ describe('gbrain#2490 — Anthropic cache breakpoint placement', () => {
|
||||
expect(captured.tools?.search?.providerOptions?.anthropic?.cacheControl).toEqual(expected);
|
||||
});
|
||||
});
|
||||
|
||||
describe('OpenRouter prompt caching (takeover of PR #1988)', () => {
|
||||
beforeEach(() => {
|
||||
resetGateway();
|
||||
__setGenerateTextTransportForTests(null);
|
||||
});
|
||||
|
||||
async function captureOpenRouterArgs(model: string, cacheSystem: boolean): Promise<any> {
|
||||
let captured: any;
|
||||
__setGenerateTextTransportForTests(async (args: any) => {
|
||||
captured = args;
|
||||
return {
|
||||
content: [{ type: 'text', text: 'ok' }],
|
||||
finishReason: 'stop',
|
||||
usage: { inputTokens: 1, outputTokens: 1 },
|
||||
} as any;
|
||||
});
|
||||
configureGateway({
|
||||
chat_model: model,
|
||||
env: { OPENROUTER_API_KEY: 'fake' },
|
||||
});
|
||||
await chat({
|
||||
model,
|
||||
system: 'stable system prompt',
|
||||
cacheSystem,
|
||||
messages: [{ role: 'user', content: 'hello' }],
|
||||
});
|
||||
return captured;
|
||||
}
|
||||
|
||||
test('cacheSystem:true on an OpenRouter Claude route threads the private marker header to the compat fetch shim', async () => {
|
||||
const args = await captureOpenRouterArgs('openrouter:anthropic/claude-sonnet-4.6', true);
|
||||
expect(args.headers).toEqual({ [OPENROUTER_CACHE_HEADER]: '1' });
|
||||
});
|
||||
|
||||
test('cacheSystem:false on an OpenRouter Claude route sends no marker header', async () => {
|
||||
const args = await captureOpenRouterArgs('openrouter:anthropic/claude-sonnet-4.6', false);
|
||||
expect(args.headers).toBeUndefined();
|
||||
});
|
||||
|
||||
test('cacheSystem:true on an OpenRouter OpenAI route needs no marker (OR caches OpenAI automatically)', async () => {
|
||||
const args = await captureOpenRouterArgs('openrouter:openai/gpt-5.2', true);
|
||||
expect(args.headers).toBeUndefined();
|
||||
});
|
||||
|
||||
test('cacheSystem:true on a non-cacheable OpenRouter route is silently ignored', async () => {
|
||||
const args = await captureOpenRouterArgs('openrouter:deepseek/deepseek-chat', true);
|
||||
expect(args.headers).toBeUndefined();
|
||||
// useCache is false → system stays a bare string.
|
||||
expect(args.system).toBe('stable system prompt');
|
||||
});
|
||||
});
|
||||
|
||||
@@ -42,11 +42,15 @@ describe('chat touchpoint — recipe registry', () => {
|
||||
}
|
||||
});
|
||||
|
||||
test('only Anthropic claims supports_prompt_cache=true', () => {
|
||||
test('only Anthropic and model-family-gated OpenRouter claim supports_prompt_cache', () => {
|
||||
for (const r of listRecipes()) {
|
||||
if (!r.touchpoints.chat) continue;
|
||||
if (r.id === 'anthropic') {
|
||||
expect(r.touchpoints.chat.supports_prompt_cache).toBe(true);
|
||||
} else if (r.id === 'openrouter') {
|
||||
// Family-scoped predicate (openai/* + anthropic/claude-*), never a
|
||||
// blanket true — see recipe-openrouter.test.ts for the model matrix.
|
||||
expect(typeof r.touchpoints.chat.supports_prompt_cache).toBe('function');
|
||||
} else {
|
||||
expect(r.touchpoints.chat.supports_prompt_cache ?? false).toBe(false);
|
||||
}
|
||||
|
||||
@@ -110,6 +110,22 @@ describe('gateway.isAvailable (silent-drop regression surface)', () => {
|
||||
});
|
||||
expect(isAvailable('expansion')).toBe(true);
|
||||
});
|
||||
|
||||
// #1135 — an explicit expansion_model pointed at a chat-capable
|
||||
// OpenAI-compatible provider used to silently yield no expansion because
|
||||
// the recipe declared no expansion touchpoint.
|
||||
test('expansion available for chat-capable openai-compat providers (deepseek/groq/together)', () => {
|
||||
const cases: Array<[string, Record<string, string>]> = [
|
||||
['deepseek:deepseek-chat', { DEEPSEEK_API_KEY: 'fake' }],
|
||||
['groq:llama-3.1-8b-instant', { GROQ_API_KEY: 'fake' }],
|
||||
['together:meta-llama/Llama-3.3-70B-Instruct-Turbo', { TOGETHER_API_KEY: 'fake' }],
|
||||
];
|
||||
for (const [model, env] of cases) {
|
||||
resetGateway();
|
||||
configureGateway({ expansion_model: model, env });
|
||||
expect(isAvailable('expansion'), `${model} expansion should be available`).toBe(true);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('model-resolver', () => {
|
||||
|
||||
@@ -11,6 +11,12 @@
|
||||
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import { getRecipe } from '../../src/core/ai/recipes/index.ts';
|
||||
import {
|
||||
OPENROUTER_CACHE_HEADER,
|
||||
openrouterCompatFetch,
|
||||
openrouterRequiresExplicitPromptCache,
|
||||
openrouterSupportsPromptCache,
|
||||
} from '../../src/core/ai/recipes/openrouter.ts';
|
||||
import { defaultResolveAuth } from '../../src/core/ai/gateway.ts';
|
||||
import { assertTouchpoint } from '../../src/core/ai/model-resolver.ts';
|
||||
import { AIConfigError } from '../../src/core/ai/errors.ts';
|
||||
@@ -135,4 +141,78 @@ describe('recipe: openrouter', () => {
|
||||
expect(r.setup_hint).toContain('OPENROUTER_REFERER');
|
||||
expect(r.setup_hint).toContain('OPENROUTER_TITLE');
|
||||
});
|
||||
|
||||
// 12-15 — prompt caching (takeover of PR #1988).
|
||||
|
||||
test('12. prompt cache capability is family-scoped, not a blanket claim', () => {
|
||||
const r = getRecipe('openrouter')!;
|
||||
expect(r.touchpoints.chat!.supports_prompt_cache).toBe(openrouterSupportsPromptCache);
|
||||
|
||||
expect(openrouterSupportsPromptCache('openai/gpt-5.2')).toBe(true);
|
||||
expect(openrouterSupportsPromptCache('openai/gpt-5.2-chat')).toBe(true);
|
||||
expect(openrouterSupportsPromptCache('openai/o4-mini')).toBe(true);
|
||||
expect(openrouterSupportsPromptCache('openai/text-embedding-3-small')).toBe(false);
|
||||
expect(openrouterSupportsPromptCache('anthropic/claude-sonnet-4.6')).toBe(true);
|
||||
expect(openrouterSupportsPromptCache('anthropic/claude-opus-4.7')).toBe(true);
|
||||
expect(openrouterSupportsPromptCache('deepseek/deepseek-chat')).toBe(false);
|
||||
expect(openrouterSupportsPromptCache('google/gemini-3-flash-preview')).toBe(false);
|
||||
});
|
||||
|
||||
test('13. only Anthropic Claude routes require the explicit cache_control rewrite', () => {
|
||||
expect(openrouterRequiresExplicitPromptCache('anthropic/claude-sonnet-4.6')).toBe(true);
|
||||
expect(openrouterRequiresExplicitPromptCache('openai/gpt-5.2')).toBe(false);
|
||||
expect(openrouterRequiresExplicitPromptCache('deepseek/deepseek-chat')).toBe(false);
|
||||
});
|
||||
|
||||
test('14. recipe installs the cache compat fetch shim', () => {
|
||||
const r = getRecipe('openrouter')!;
|
||||
expect(r.compat?.fetch).toBe(openrouterCompatFetch);
|
||||
});
|
||||
|
||||
test('15. fetch shim rewrites system content-block cache_control for Claude routes and always strips the marker header', async () => {
|
||||
const originalFetch = globalThis.fetch;
|
||||
const calls: Array<{ input: RequestInfo | URL; init?: RequestInit }> = [];
|
||||
globalThis.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => {
|
||||
calls.push({ input, init });
|
||||
return new Response('{}', { status: 200, headers: { 'content-type': 'application/json' } });
|
||||
}) as typeof fetch;
|
||||
|
||||
try {
|
||||
const post = (model: string, withMarker: boolean) =>
|
||||
openrouterCompatFetch('https://openrouter.ai/api/v1/chat/completions', {
|
||||
method: 'POST',
|
||||
headers: withMarker ? { [OPENROUTER_CACHE_HEADER]: '1' } : {},
|
||||
body: JSON.stringify({
|
||||
model,
|
||||
messages: [
|
||||
{ role: 'system', content: 'stable system prompt' },
|
||||
{ role: 'user', content: 'hello' },
|
||||
],
|
||||
}),
|
||||
});
|
||||
|
||||
// Marker + Claude route → system content becomes a cache_control block.
|
||||
await post('anthropic/claude-sonnet-4.6', true);
|
||||
const rewritten = JSON.parse(calls[0].init!.body as string);
|
||||
expect(rewritten.messages[0].content).toEqual([
|
||||
{ type: 'text', text: 'stable system prompt', cache_control: { type: 'ephemeral' } },
|
||||
]);
|
||||
expect(rewritten.messages[1]).toEqual({ role: 'user', content: 'hello' });
|
||||
// Marker never leaves the process.
|
||||
expect(new Headers(calls[0].init!.headers as any).has(OPENROUTER_CACHE_HEADER)).toBe(false);
|
||||
|
||||
// Marker + non-Claude route → body untouched, marker still stripped.
|
||||
await post('openai/gpt-5.2', true);
|
||||
const untouched = JSON.parse(calls[1].init!.body as string);
|
||||
expect(untouched.messages[0]).toEqual({ role: 'system', content: 'stable system prompt' });
|
||||
expect(new Headers(calls[1].init!.headers as any).has(OPENROUTER_CACHE_HEADER)).toBe(false);
|
||||
|
||||
// No marker → body untouched even on a Claude route.
|
||||
await post('anthropic/claude-sonnet-4.6', false);
|
||||
const noMarker = JSON.parse(calls[2].init!.body as string);
|
||||
expect(noMarker.messages[0]).toEqual({ role: 'system', content: 'stable system prompt' });
|
||||
} finally {
|
||||
globalThis.fetch = originalFetch;
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user