feat(ai): prompt caching for OpenRouter Anthropic routes (#1987)

Rework of #1988 against the post-#2981 cache-breakpoint placement.

- ChatTouchpoint.supports_prompt_cache may now be function-valued
  (per-model-id); OpenRouter scopes it to anthropic/claude-* routes, so
  those stop classifying as degraded:no_caching. capabilities.ts +
  gateway's supportsCache gate handle both forms (fail-closed).
- The system-block cache breakpoint additionally rides
  providerOptions.openaiCompatible on openai-compatible recipes (the
  anthropic key never reaches the compat wire); a new chat-scoped
  recipe fetch shim (compat.chatFetch) lifts the marker into
  OpenRouter's documented content-part cache_control shape before the
  request leaves the process. chatFetch deliberately does NOT displace
  the embedding path's asymmetric input_type shim the way compat.fetch
  would.

Co-authored-by: tmchow <tmchow@users.noreply.github.com>
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Garry Tan
2026-07-21 14:53:41 -07:00
co-authored by tmchow Claude Fable 5
parent 81529ca25c
commit 2de17dd3ee
8 changed files with 292 additions and 10 deletions
+16
View File
@@ -24,6 +24,17 @@ describe('getProviderCapabilities (v0.38 Slice 1 — D6/D7 recipe-driven capabil
expect(caps.maxContext).toBe(1000000); // Gemini 1.5 Pro
});
it('marks OpenRouter Anthropic routes as cache-capable (#1987 function-valued capability)', () => {
const caps = getProviderCapabilities('openrouter:anthropic/claude-sonnet-4.6');
expect(caps.supportsToolCalling).toBe(true);
expect(caps.supportsPromptCaching).toBe(true);
});
it('does not mark every OpenRouter route as cache-capable', () => {
const caps = getProviderCapabilities('openrouter:deepseek/deepseek-chat');
expect(caps.supportsPromptCaching).toBe(false);
});
it('honors Anthropic alias (undated → dated)', () => {
const caps = getProviderCapabilities('anthropic:claude-haiku-4-5');
expect(caps.supportsToolCalling).toBe(true);
@@ -54,6 +65,11 @@ describe('classifyCapabilities (D6 — three-tier capability verdict)', () => {
expect(classifyCapabilities('openai:gpt-5.2')).toBe('degraded:no_caching');
});
it('returns ok for OpenRouter Anthropic routes; other routes stay degraded:no_caching (#1987)', () => {
expect(classifyCapabilities('openrouter:anthropic/claude-sonnet-4.6')).toBe('ok');
expect(classifyCapabilities('openrouter:deepseek/deepseek-chat')).toBe('degraded:no_caching');
});
it('returns degraded:no_caching for Google Gemini', () => {
expect(classifyCapabilities('google:gemini-1.5-pro')).toBe('degraded:no_caching');
});
+53 -1
View File
@@ -29,7 +29,7 @@
* the bug made you believe was sufficient.
*/
import { describe, test, expect, beforeEach } from 'bun:test';
import { describe, test, expect, beforeEach, afterAll } from 'bun:test';
import {
chat,
configureGateway,
@@ -43,6 +43,11 @@ describe('gbrain#2490 — Anthropic cache breakpoint placement', () => {
__setGenerateTextTransportForTests(null);
});
afterAll(() => {
resetGateway();
__setGenerateTextTransportForTests(null);
});
async function captureTransportArgs(
opts: Partial<Parameters<typeof chat>[0]> = {},
): Promise<any> {
@@ -192,4 +197,51 @@ describe('gbrain#2490 — Anthropic cache breakpoint placement', () => {
expect((captured.system as any)?.providerOptions?.anthropic?.cacheControl).toEqual(expected);
expect(captured.tools?.search?.providerOptions?.anthropic?.cacheControl).toEqual(expected);
});
// #1987 — OpenRouter Anthropic routes support prompt caching. The
// capability is per-model-family (function-valued supports_prompt_cache),
// and the system-block marker must ALSO ride `openaiCompatible` metadata:
// `providerOptions.anthropic` never leaves the process on the
// openai-compatible wire, so without the extra key no cache_control could
// ever reach OpenRouter.
async function captureOpenRouterArgs(model: string): Promise<any> {
let captured: any;
__setGenerateTextTransportForTests(async (args: any) => {
captured = args;
return {
content: [{ type: 'text', text: 'ok' }],
finishReason: 'stop',
usage: { inputTokens: 1, outputTokens: 1 },
} as any;
});
configureGateway({
chat_model: model,
env: { OPENROUTER_API_KEY: 'fake' },
});
await chat({
model,
system: 'SYS',
cacheSystem: true,
messages: [{ role: 'user', content: 'hello' }],
});
return captured;
}
test('cacheSystem:true on openrouter:anthropic/claude-* plants anthropic AND openaiCompatible markers on the system block (#1987)', async () => {
const args = await captureOpenRouterArgs('openrouter:anthropic/claude-sonnet-4.6');
expect(args.system).toEqual({
role: 'system',
content: 'SYS',
providerOptions: {
anthropic: { cacheControl: { type: 'ephemeral' } },
openaiCompatible: { cache_control: { type: 'ephemeral' } },
},
});
});
test('cacheSystem:true on a non-Claude OpenRouter route stays uncached (per-model-family gate)', async () => {
const args = await captureOpenRouterArgs('openrouter:deepseek/deepseek-chat');
expect(args.system).toBe('SYS');
expect(args.providerOptions?.anthropic).toBeUndefined();
});
});
+4 -1
View File
@@ -45,11 +45,14 @@ describe('chat touchpoint — recipe registry', () => {
}
});
test('only Anthropic claims supports_prompt_cache=true', () => {
test('only Anthropic and model-family-gated OpenRouter claim supports_prompt_cache', () => {
for (const r of listRecipes()) {
if (!r.touchpoints.chat) continue;
if (r.id === 'anthropic') {
expect(r.touchpoints.chat.supports_prompt_cache).toBe(true);
} else if (r.id === 'openrouter') {
// #1987: per-model-family (anthropic/claude-* routes only).
expect(typeof r.touchpoints.chat.supports_prompt_cache).toBe('function');
} else {
expect(r.touchpoints.chat.supports_prompt_cache ?? false).toBe(false);
}
+88
View File
@@ -11,6 +11,11 @@
import { describe, expect, test } from 'bun:test';
import { getRecipe } from '../../src/core/ai/recipes/index.ts';
import {
openrouterSupportsPromptCache,
liftMessageCacheControl,
openrouterCacheControlFetch,
} from '../../src/core/ai/recipes/openrouter.ts';
import { defaultResolveAuth } from '../../src/core/ai/gateway.ts';
import { assertTouchpoint } from '../../src/core/ai/model-resolver.ts';
import { AIConfigError } from '../../src/core/ai/errors.ts';
@@ -135,4 +140,87 @@ describe('recipe: openrouter', () => {
expect(r.setup_hint).toContain('OPENROUTER_REFERER');
expect(r.setup_hint).toContain('OPENROUTER_TITLE');
});
test('12. prompt cache capability is scoped to anthropic/claude-* routes (#1987)', () => {
expect(openrouterSupportsPromptCache('anthropic/claude-sonnet-4.6')).toBe(true);
expect(openrouterSupportsPromptCache('anthropic/claude-opus-4.7')).toBe(true);
expect(openrouterSupportsPromptCache('ANTHROPIC/Claude-Haiku-4.5')).toBe(true); // case-insensitive
expect(openrouterSupportsPromptCache('openai/gpt-5.2')).toBe(false);
expect(openrouterSupportsPromptCache('deepseek/deepseek-chat')).toBe(false);
expect(openrouterSupportsPromptCache('google/gemini-3-flash-preview')).toBe(false);
});
test('13. liftMessageCacheControl moves a message-level marker into the content part', () => {
const body: any = {
model: 'anthropic/claude-sonnet-4.6',
messages: [
{ role: 'system', content: 'SYS', cache_control: { type: 'ephemeral' } },
{ role: 'user', content: 'hello' },
],
};
expect(liftMessageCacheControl(body)).toBe(true);
expect(body.messages[0]).toEqual({
role: 'system',
content: [{ type: 'text', text: 'SYS', cache_control: { type: 'ephemeral' } }],
});
// The user message (no marker) is untouched.
expect(body.messages[1]).toEqual({ role: 'user', content: 'hello' });
});
test('14. liftMessageCacheControl is a no-op when no marker rides the body', () => {
const body = {
model: 'openai/gpt-5.2',
messages: [
{ role: 'system', content: 'SYS' },
{ role: 'user', content: 'hello' },
],
};
expect(liftMessageCacheControl(body)).toBe(false);
expect(body.messages[0]).toEqual({ role: 'system', content: 'SYS' });
expect(liftMessageCacheControl(undefined)).toBe(false);
expect(liftMessageCacheControl({ input: 'embedding body, no messages' })).toBe(false);
});
test('15. cache fetch shim rewrites the outbound body and recomputes content-length', async () => {
const originalFetch = globalThis.fetch;
const calls: Array<{ init?: RequestInit }> = [];
globalThis.fetch = (async (_input: RequestInfo | URL, init?: RequestInit) => {
calls.push({ init });
return new Response('{}', { status: 200 });
}) as typeof fetch;
try {
await openrouterCacheControlFetch('https://openrouter.ai/api/v1/chat/completions', {
method: 'POST',
headers: { 'content-length': '999', 'content-type': 'application/json' },
body: JSON.stringify({
model: 'anthropic/claude-sonnet-4.6',
messages: [{ role: 'system', content: 'SYS', cache_control: { type: 'ephemeral' } }],
}),
});
const rewritten = JSON.parse(calls[0].init!.body as string);
expect(rewritten.messages[0].content).toEqual([
{ type: 'text', text: 'SYS', cache_control: { type: 'ephemeral' } },
]);
expect(rewritten.messages[0].cache_control).toBeUndefined();
expect(new Headers(calls[0].init!.headers).get('content-length')).toBeNull();
// Marker-free body passes through byte-identical (fail-open contract).
const plain = JSON.stringify({ model: 'openai/gpt-5.2', messages: [{ role: 'user', content: 'hi' }] });
await openrouterCacheControlFetch('https://openrouter.ai/api/v1/chat/completions', {
method: 'POST',
body: plain,
});
expect(calls[1].init!.body).toBe(plain);
} finally {
globalThis.fetch = originalFetch;
}
});
test('16. cache shim is installed as compat.chatFetch (chat path only, embedding shim preserved)', () => {
const r = getRecipe('openrouter')!;
expect(r.compat?.chatFetch).toBe(openrouterCacheControlFetch);
// Deliberately NOT compat.fetch: that would displace the gateway's
// asymmetric input_type shim on the embedding path.
expect(r.compat?.fetch).toBeUndefined();
});
});