mirror of
https://github.com/garrytan/gbrain.git
synced 2026-07-30 19:49:14 +00:00
feat(ai): prompt caching for OpenRouter Anthropic routes (#1987)
Rework of #1988 against the post-#2981 cache-breakpoint placement. - ChatTouchpoint.supports_prompt_cache may now be function-valued (per-model-id); OpenRouter scopes it to anthropic/claude-* routes, so those stop classifying as degraded:no_caching. capabilities.ts + gateway's supportsCache gate handle both forms (fail-closed). - The system-block cache breakpoint additionally rides providerOptions.openaiCompatible on openai-compatible recipes (the anthropic key never reaches the compat wire); a new chat-scoped recipe fetch shim (compat.chatFetch) lifts the marker into OpenRouter's documented content-part cache_control shape before the request leaves the process. chatFetch deliberately does NOT displace the embedding path's asymmetric input_type shim the way compat.fetch would. Co-authored-by: tmchow <tmchow@users.noreply.github.com> Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
tmchow
Claude Fable 5
parent
81529ca25c
commit
2de17dd3ee
@@ -24,6 +24,17 @@ describe('getProviderCapabilities (v0.38 Slice 1 — D6/D7 recipe-driven capabil
|
||||
expect(caps.maxContext).toBe(1000000); // Gemini 1.5 Pro
|
||||
});
|
||||
|
||||
it('marks OpenRouter Anthropic routes as cache-capable (#1987 function-valued capability)', () => {
|
||||
const caps = getProviderCapabilities('openrouter:anthropic/claude-sonnet-4.6');
|
||||
expect(caps.supportsToolCalling).toBe(true);
|
||||
expect(caps.supportsPromptCaching).toBe(true);
|
||||
});
|
||||
|
||||
it('does not mark every OpenRouter route as cache-capable', () => {
|
||||
const caps = getProviderCapabilities('openrouter:deepseek/deepseek-chat');
|
||||
expect(caps.supportsPromptCaching).toBe(false);
|
||||
});
|
||||
|
||||
it('honors Anthropic alias (undated → dated)', () => {
|
||||
const caps = getProviderCapabilities('anthropic:claude-haiku-4-5');
|
||||
expect(caps.supportsToolCalling).toBe(true);
|
||||
@@ -54,6 +65,11 @@ describe('classifyCapabilities (D6 — three-tier capability verdict)', () => {
|
||||
expect(classifyCapabilities('openai:gpt-5.2')).toBe('degraded:no_caching');
|
||||
});
|
||||
|
||||
it('returns ok for OpenRouter Anthropic routes; other routes stay degraded:no_caching (#1987)', () => {
|
||||
expect(classifyCapabilities('openrouter:anthropic/claude-sonnet-4.6')).toBe('ok');
|
||||
expect(classifyCapabilities('openrouter:deepseek/deepseek-chat')).toBe('degraded:no_caching');
|
||||
});
|
||||
|
||||
it('returns degraded:no_caching for Google Gemini', () => {
|
||||
expect(classifyCapabilities('google:gemini-1.5-pro')).toBe('degraded:no_caching');
|
||||
});
|
||||
|
||||
@@ -29,7 +29,7 @@
|
||||
* the bug made you believe was sufficient.
|
||||
*/
|
||||
|
||||
import { describe, test, expect, beforeEach } from 'bun:test';
|
||||
import { describe, test, expect, beforeEach, afterAll } from 'bun:test';
|
||||
import {
|
||||
chat,
|
||||
configureGateway,
|
||||
@@ -43,6 +43,11 @@ describe('gbrain#2490 — Anthropic cache breakpoint placement', () => {
|
||||
__setGenerateTextTransportForTests(null);
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
resetGateway();
|
||||
__setGenerateTextTransportForTests(null);
|
||||
});
|
||||
|
||||
async function captureTransportArgs(
|
||||
opts: Partial<Parameters<typeof chat>[0]> = {},
|
||||
): Promise<any> {
|
||||
@@ -192,4 +197,51 @@ describe('gbrain#2490 — Anthropic cache breakpoint placement', () => {
|
||||
expect((captured.system as any)?.providerOptions?.anthropic?.cacheControl).toEqual(expected);
|
||||
expect(captured.tools?.search?.providerOptions?.anthropic?.cacheControl).toEqual(expected);
|
||||
});
|
||||
|
||||
// #1987 — OpenRouter Anthropic routes support prompt caching. The
|
||||
// capability is per-model-family (function-valued supports_prompt_cache),
|
||||
// and the system-block marker must ALSO ride `openaiCompatible` metadata:
|
||||
// `providerOptions.anthropic` never leaves the process on the
|
||||
// openai-compatible wire, so without the extra key no cache_control could
|
||||
// ever reach OpenRouter.
|
||||
async function captureOpenRouterArgs(model: string): Promise<any> {
|
||||
let captured: any;
|
||||
__setGenerateTextTransportForTests(async (args: any) => {
|
||||
captured = args;
|
||||
return {
|
||||
content: [{ type: 'text', text: 'ok' }],
|
||||
finishReason: 'stop',
|
||||
usage: { inputTokens: 1, outputTokens: 1 },
|
||||
} as any;
|
||||
});
|
||||
configureGateway({
|
||||
chat_model: model,
|
||||
env: { OPENROUTER_API_KEY: 'fake' },
|
||||
});
|
||||
await chat({
|
||||
model,
|
||||
system: 'SYS',
|
||||
cacheSystem: true,
|
||||
messages: [{ role: 'user', content: 'hello' }],
|
||||
});
|
||||
return captured;
|
||||
}
|
||||
|
||||
test('cacheSystem:true on openrouter:anthropic/claude-* plants anthropic AND openaiCompatible markers on the system block (#1987)', async () => {
|
||||
const args = await captureOpenRouterArgs('openrouter:anthropic/claude-sonnet-4.6');
|
||||
expect(args.system).toEqual({
|
||||
role: 'system',
|
||||
content: 'SYS',
|
||||
providerOptions: {
|
||||
anthropic: { cacheControl: { type: 'ephemeral' } },
|
||||
openaiCompatible: { cache_control: { type: 'ephemeral' } },
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
test('cacheSystem:true on a non-Claude OpenRouter route stays uncached (per-model-family gate)', async () => {
|
||||
const args = await captureOpenRouterArgs('openrouter:deepseek/deepseek-chat');
|
||||
expect(args.system).toBe('SYS');
|
||||
expect(args.providerOptions?.anthropic).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -45,11 +45,14 @@ describe('chat touchpoint — recipe registry', () => {
|
||||
}
|
||||
});
|
||||
|
||||
test('only Anthropic claims supports_prompt_cache=true', () => {
|
||||
test('only Anthropic and model-family-gated OpenRouter claim supports_prompt_cache', () => {
|
||||
for (const r of listRecipes()) {
|
||||
if (!r.touchpoints.chat) continue;
|
||||
if (r.id === 'anthropic') {
|
||||
expect(r.touchpoints.chat.supports_prompt_cache).toBe(true);
|
||||
} else if (r.id === 'openrouter') {
|
||||
// #1987: per-model-family (anthropic/claude-* routes only).
|
||||
expect(typeof r.touchpoints.chat.supports_prompt_cache).toBe('function');
|
||||
} else {
|
||||
expect(r.touchpoints.chat.supports_prompt_cache ?? false).toBe(false);
|
||||
}
|
||||
|
||||
@@ -11,6 +11,11 @@
|
||||
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import { getRecipe } from '../../src/core/ai/recipes/index.ts';
|
||||
import {
|
||||
openrouterSupportsPromptCache,
|
||||
liftMessageCacheControl,
|
||||
openrouterCacheControlFetch,
|
||||
} from '../../src/core/ai/recipes/openrouter.ts';
|
||||
import { defaultResolveAuth } from '../../src/core/ai/gateway.ts';
|
||||
import { assertTouchpoint } from '../../src/core/ai/model-resolver.ts';
|
||||
import { AIConfigError } from '../../src/core/ai/errors.ts';
|
||||
@@ -135,4 +140,87 @@ describe('recipe: openrouter', () => {
|
||||
expect(r.setup_hint).toContain('OPENROUTER_REFERER');
|
||||
expect(r.setup_hint).toContain('OPENROUTER_TITLE');
|
||||
});
|
||||
|
||||
test('12. prompt cache capability is scoped to anthropic/claude-* routes (#1987)', () => {
|
||||
expect(openrouterSupportsPromptCache('anthropic/claude-sonnet-4.6')).toBe(true);
|
||||
expect(openrouterSupportsPromptCache('anthropic/claude-opus-4.7')).toBe(true);
|
||||
expect(openrouterSupportsPromptCache('ANTHROPIC/Claude-Haiku-4.5')).toBe(true); // case-insensitive
|
||||
expect(openrouterSupportsPromptCache('openai/gpt-5.2')).toBe(false);
|
||||
expect(openrouterSupportsPromptCache('deepseek/deepseek-chat')).toBe(false);
|
||||
expect(openrouterSupportsPromptCache('google/gemini-3-flash-preview')).toBe(false);
|
||||
});
|
||||
|
||||
test('13. liftMessageCacheControl moves a message-level marker into the content part', () => {
|
||||
const body: any = {
|
||||
model: 'anthropic/claude-sonnet-4.6',
|
||||
messages: [
|
||||
{ role: 'system', content: 'SYS', cache_control: { type: 'ephemeral' } },
|
||||
{ role: 'user', content: 'hello' },
|
||||
],
|
||||
};
|
||||
expect(liftMessageCacheControl(body)).toBe(true);
|
||||
expect(body.messages[0]).toEqual({
|
||||
role: 'system',
|
||||
content: [{ type: 'text', text: 'SYS', cache_control: { type: 'ephemeral' } }],
|
||||
});
|
||||
// The user message (no marker) is untouched.
|
||||
expect(body.messages[1]).toEqual({ role: 'user', content: 'hello' });
|
||||
});
|
||||
|
||||
test('14. liftMessageCacheControl is a no-op when no marker rides the body', () => {
|
||||
const body = {
|
||||
model: 'openai/gpt-5.2',
|
||||
messages: [
|
||||
{ role: 'system', content: 'SYS' },
|
||||
{ role: 'user', content: 'hello' },
|
||||
],
|
||||
};
|
||||
expect(liftMessageCacheControl(body)).toBe(false);
|
||||
expect(body.messages[0]).toEqual({ role: 'system', content: 'SYS' });
|
||||
expect(liftMessageCacheControl(undefined)).toBe(false);
|
||||
expect(liftMessageCacheControl({ input: 'embedding body, no messages' })).toBe(false);
|
||||
});
|
||||
|
||||
test('15. cache fetch shim rewrites the outbound body and recomputes content-length', async () => {
|
||||
const originalFetch = globalThis.fetch;
|
||||
const calls: Array<{ init?: RequestInit }> = [];
|
||||
globalThis.fetch = (async (_input: RequestInfo | URL, init?: RequestInit) => {
|
||||
calls.push({ init });
|
||||
return new Response('{}', { status: 200 });
|
||||
}) as typeof fetch;
|
||||
try {
|
||||
await openrouterCacheControlFetch('https://openrouter.ai/api/v1/chat/completions', {
|
||||
method: 'POST',
|
||||
headers: { 'content-length': '999', 'content-type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
model: 'anthropic/claude-sonnet-4.6',
|
||||
messages: [{ role: 'system', content: 'SYS', cache_control: { type: 'ephemeral' } }],
|
||||
}),
|
||||
});
|
||||
const rewritten = JSON.parse(calls[0].init!.body as string);
|
||||
expect(rewritten.messages[0].content).toEqual([
|
||||
{ type: 'text', text: 'SYS', cache_control: { type: 'ephemeral' } },
|
||||
]);
|
||||
expect(rewritten.messages[0].cache_control).toBeUndefined();
|
||||
expect(new Headers(calls[0].init!.headers).get('content-length')).toBeNull();
|
||||
|
||||
// Marker-free body passes through byte-identical (fail-open contract).
|
||||
const plain = JSON.stringify({ model: 'openai/gpt-5.2', messages: [{ role: 'user', content: 'hi' }] });
|
||||
await openrouterCacheControlFetch('https://openrouter.ai/api/v1/chat/completions', {
|
||||
method: 'POST',
|
||||
body: plain,
|
||||
});
|
||||
expect(calls[1].init!.body).toBe(plain);
|
||||
} finally {
|
||||
globalThis.fetch = originalFetch;
|
||||
}
|
||||
});
|
||||
|
||||
test('16. cache shim is installed as compat.chatFetch (chat path only, embedding shim preserved)', () => {
|
||||
const r = getRecipe('openrouter')!;
|
||||
expect(r.compat?.chatFetch).toBe(openrouterCacheControlFetch);
|
||||
// Deliberately NOT compat.fetch: that would displace the gateway's
|
||||
// asymmetric input_type shim on the embedding path.
|
||||
expect(r.compat?.fetch).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user