From 2df41a84c9cbca6bea4c6d59a6f93ec9247cfa02 Mon Sep 17 00:00:00 2001 From: Time Attakc <89218912+time-attack@users.noreply.github.com> Date: Fri, 17 Jul 2026 14:18:48 -0700 Subject: [PATCH] feat(ai-gateway): derive OpenAI prompt_cache_key for native-OpenAI chat models (takeover of #2442) (#2933) Ports the still-unmerged half of PR #2442. OpenAI caches prompt prefixes automatically, but a stable prompt_cache_key keeps requests that share a prefix on the same inference engine, lifting the automatic-cache hit rate. chat() now derives a stable key from the system prompt + sorted tool names for native-openai models and passes it via providerOptions.openai. promptCacheKey. Config provider_chat_options still overrides the derived key; anthropic/google/openai-compatible providers are untouched. The Anthropic half of #2442 (cache_control "silent no-op") is superseded: @ai-sdk/anthropic 3.0.74 forwards call-level providerOptions.anthropic. cacheControl as the Messages API's request-level cache_control (automatic prefix caching), so master's existing marker is live on current deps. Co-authored-by: Sinabina Co-authored-by: CoachRyanNguyen Co-authored-by: Claude Fable 5 --- src/core/ai/gateway.ts | 39 ++++++ .../gateway-openai-prompt-cache-key.test.ts | 123 ++++++++++++++++++ 2 files changed, 162 insertions(+) create mode 100644 test/ai/gateway-openai-prompt-cache-key.test.ts diff --git a/src/core/ai/gateway.ts b/src/core/ai/gateway.ts index 8a169e5fd..789c2db1e 100644 --- a/src/core/ai/gateway.ts +++ b/src/core/ai/gateway.ts @@ -23,6 +23,7 @@ import { embed as aiEmbed, embedMany, generateObject, generateText, jsonSchema } from 'ai'; import { AsyncLocalStorage } from 'node:async_hooks'; +import { createHash } from 'node:crypto'; import { listRecipes } from './recipes/index.ts'; import { createOpenAI } from '@ai-sdk/openai'; import { createGoogleGenerativeAI } from '@ai-sdk/google'; @@ -2849,6 +2850,30 @@ async function classifyGatewayGuardrail(input: { } } +/** + * Derive OpenAI's `prompt_cache_key` (the AI SDK's `providerOptions.openai. + * promptCacheKey`). It's a ROUTING hint, not a cache breakpoint: OpenAI caches + * prefixes automatically, and a stable key makes requests sharing a prefix + * land on the same engine, raising the hit rate (OpenAI cites 60%→87%). + * + * Hash the system prompt + sorted tool names — that's the stable prefix + * gbrain's repeated loops (enrich, page-summary, skillopt, subagent) actually + * share. Returns undefined when there's no system prompt (nothing stable to + * key on), so one-off requests don't get pinned to a single engine. An + * explicit key can still be set per provider/model via + * `provider_chat_options` config, which overrides the derived key. + * + * @internal exported for tests; not part of the public gateway API. + */ +export function openAIPromptCacheKey(args: { + system?: string; + toolNames?: string[]; +}): string | undefined { + if (!args.system) return undefined; + const basis = `${args.system} ${(args.toolNames ?? []).slice().sort().join(',')}`; + return `gbrain:${createHash('sha256').update(basis).digest('hex').slice(0, 32)}`; +} + export function toAISDKTools(tools: ChatToolDef[] | undefined): Record | undefined { if (!tools || tools.length === 0) return undefined; return tools.reduce((acc, t) => { @@ -2959,6 +2984,20 @@ export async function chat(opts: ChatOpts): Promise { if (useCache) { providerOptions.anthropic = { cacheControl: { type: 'ephemeral' } }; } + // OpenAI prompt_cache_key (native-openai only): a stable per-prefix routing + // hint that keeps requests sharing a system prompt + tool set on the same + // inference engine, lifting OpenAI's automatic prefix-cache hit rate. The + // openai-compatible path (litellm/azure/groq/...) ignores + // providerOptions.openai, so it gets nothing. Applied BEFORE the configured + // provider options so `provider_chat_options.openai.promptCacheKey` from + // config still overrides the derived key. + if (recipe.implementation === 'native-openai') { + const promptCacheKey = openAIPromptCacheKey({ + system: opts.system, + toolNames: (opts.tools ?? []).map(t => t.name), + }); + if (promptCacheKey) providerOptions.openai = { promptCacheKey }; + } applyConfiguredChatProviderOptions(providerOptions, cfg, recipe.id, modelId); let _budgetRecorded = false; diff --git a/test/ai/gateway-openai-prompt-cache-key.test.ts b/test/ai/gateway-openai-prompt-cache-key.test.ts new file mode 100644 index 000000000..8d49e56b2 --- /dev/null +++ b/test/ai/gateway-openai-prompt-cache-key.test.ts @@ -0,0 +1,123 @@ +/** + * OpenAI `prompt_cache_key` routing hint (takeover of PR #2442's remaining + * half, originally by @CoachRyanNguyen). + * + * OpenAI caches prompt prefixes automatically; a stable `prompt_cache_key` + * keeps requests that share a prefix on the same inference engine, lifting the + * automatic-cache hit rate. `chat()` derives one from the system prompt + tool + * names for native-OpenAI models and passes it via + * `providerOptions.openai.promptCacheKey` (which @ai-sdk/openai maps to the + * request's `prompt_cache_key`). + * + * Pins: + * - key derivation is stable (tool ORDER doesn't matter), sensitive to + * system/tool-set changes, and absent without a system prompt + * - the chat() wiring only fires for native-openai (anthropic/compat get + * nothing), and config `provider_chat_options` overrides the derived key + */ + +import { describe, test, expect, beforeEach } from 'bun:test'; +import { + chat, + configureGateway, + openAIPromptCacheKey, + resetGateway, + __setGenerateTextTransportForTests, +} from '../../src/core/ai/gateway.ts'; + +describe('openAIPromptCacheKey — derivation', () => { + test('same system + same tools → identical stable key (sticky routing)', () => { + const a = openAIPromptCacheKey({ system: 'SYS', toolNames: ['search', 'put_page'] }); + const b = openAIPromptCacheKey({ system: 'SYS', toolNames: ['put_page', 'search'] }); + expect(a).toBe(b as string); // tool ORDER must not change the key + expect(a).toMatch(/^gbrain:[0-9a-f]{32}$/); + }); + + test('different system → different key', () => { + const a = openAIPromptCacheKey({ system: 'SYS A', toolNames: [] }); + const b = openAIPromptCacheKey({ system: 'SYS B', toolNames: [] }); + expect(a).not.toBe(b as string); + }); + + test('different tool set → different key', () => { + const a = openAIPromptCacheKey({ system: 'SYS', toolNames: ['search'] }); + const b = openAIPromptCacheKey({ system: 'SYS', toolNames: ['search', 'put_page'] }); + expect(a).not.toBe(b as string); + }); + + test('no system prompt → undefined (do not pin one-off requests)', () => { + expect(openAIPromptCacheKey({ system: undefined, toolNames: ['search'] })).toBeUndefined(); + }); +}); + +describe('chat() wiring — prompt_cache_key per provider', () => { + beforeEach(() => { + resetGateway(); + __setGenerateTextTransportForTests(null); + }); + + async function captureProviderOptions( + config: Parameters[0], + opts: Partial[0]> = {}, + ): Promise | undefined> { + let captured: Record | undefined; + __setGenerateTextTransportForTests(async (args: any) => { + captured = args.providerOptions; + return { + content: [{ type: 'text', text: 'ok' }], + finishReason: 'stop', + usage: { inputTokens: 1, outputTokens: 1 }, + } as any; + }); + configureGateway(config); + await chat({ + model: config.chat_model ?? 'anthropic:claude-sonnet-4-6', + messages: [{ role: 'user', content: 'hello' }], + ...opts, + }); + return captured; + } + + test('native-openai with a system prompt → providerOptions.openai.promptCacheKey', async () => { + const providerOptions = await captureProviderOptions( + { chat_model: 'openai:gpt-4o-mini', env: { OPENAI_API_KEY: 'fake' } }, + { system: 'SYS' }, + ); + expect(providerOptions?.openai?.promptCacheKey).toMatch(/^gbrain:[0-9a-f]{32}$/); + }); + + test('native-openai without a system prompt → no providerOptions at all', async () => { + const providerOptions = await captureProviderOptions( + { chat_model: 'openai:gpt-4o-mini', env: { OPENAI_API_KEY: 'fake' } }, + ); + expect(providerOptions).toBeUndefined(); + }); + + test('native-anthropic never gets an openai promptCacheKey', async () => { + const providerOptions = await captureProviderOptions( + { chat_model: 'anthropic:claude-sonnet-4-6', env: { ANTHROPIC_API_KEY: 'fake' } }, + { system: 'SYS' }, + ); + expect(providerOptions?.openai).toBeUndefined(); + }); + + test('openai-compatible (deepseek) never gets promptCacheKey (provider ignores providerOptions.openai)', async () => { + const providerOptions = await captureProviderOptions( + { chat_model: 'deepseek:deepseek-chat', env: { DEEPSEEK_API_KEY: 'fake' } }, + { system: 'SYS' }, + ); + expect(providerOptions?.openai).toBeUndefined(); + }); + + test('config provider_chat_options overrides the derived key', async () => { + const providerOptions = await captureProviderOptions( + { + chat_model: 'openai:gpt-4o-mini', + env: { OPENAI_API_KEY: 'fake' }, + provider_chat_options: { openai: { promptCacheKey: 'session-42' } }, + }, + { system: 'SYS' }, + ); + expect(providerOptions?.openai?.promptCacheKey).toBe('session-42'); + }); +});