diff --git a/docs/agent/prompt-audit.md b/docs/agent/prompt-audit.md index cfa8f9465..4e0e56fa6 100644 --- a/docs/agent/prompt-audit.md +++ b/docs/agent/prompt-audit.md @@ -128,3 +128,38 @@ and dynamic build paths, plus an assertion in the narrow-renderer test. GGML_NATIVE=OFF cargo test --manifest-path Cargo.toml --lib agent::prompts:: GGML_NATIVE=OFF cargo test --manifest-path Cargo.toml --lib agent_registry::agents ``` + +--- + +## Provider prompt-cache behaviour (#3939) + +The byte-stable prompt prefix above is what makes KV-cache reuse possible. How +much of that reuse a user actually gets depends on the backend they route to. +`Provider::prompt_cache_capabilities()` (`src/openhuman/inference/provider/traits.rs`, +`PromptCacheCapabilities`) makes that contract explicit per provider: + +| Provider | automatic prefix cache | reports cached input tokens | explicit cache-control | thread/session grouping | +|---|---|---|---|---| +| OpenHuman backend | ✓ | ✓ | — | ✓ (`thread_id` extension) | +| OpenAI / OpenRouter / GMI (OpenAI-compatible) | ✓ | ✓ | — | — (prefix identity) | +| Other / custom / LM Studio compatible | conservative default — all `false` | | | | + +Notes: + +- **Conservative by default.** Unknown or custom OpenAI-compatible slugs report + no caching, so we never assume cache hits or send cache-only request fields a + provider may not honour. Opting a provider in is a one-line edit to + `prompt_cache_for_compatible_slug` (`compatible.rs`) once verified. +- **Usage normalization is provider-agnostic.** `extract_usage` already folds the + OpenAI `usage.prompt_tokens_details.cached_tokens` shape and the + `openhuman.usage.cached_input_tokens` extension into + `UsageInfo.cached_input_tokens`, so cached-prefix cost accounting + (`src/openhuman/agent/cost.rs`) is exact wherever the provider reports it. +- **No OpenHuman-only leakage.** `explicit_cache_control` stays `false` for every + OpenAI-compatible provider (the chat-completions API has no such field), and + `thread_id` grouping is declared only on `OpenHumanBackendProvider`. + +Follow-ups (not in this slice): explicit cache-control request shaping for +providers that support it (e.g. Anthropic `cache_control`), a `ChatRequest` +cache-boundary marker, and extending cached-token parsing to non-OpenAI usage +shapes (e.g. DeepSeek `prompt_cache_hit_tokens`). diff --git a/src/openhuman/inference/provider/compatible.rs b/src/openhuman/inference/provider/compatible.rs index 412d1cb8a..a0083ca22 100644 --- a/src/openhuman/inference/provider/compatible.rs +++ b/src/openhuman/inference/provider/compatible.rs @@ -297,6 +297,56 @@ impl OpenAiCompatibleProvider { } } +/// Prompt-cache behaviour for an OpenAI-compatible provider, keyed on its +/// configured slug (#3939). +/// +/// Conservative by default: only providers we have verified to both (a) cache +/// identical request prefixes server-side and (b) report cached input tokens +/// via the OpenAI `prompt_tokens_details.cached_tokens` shape that +/// [`OpenAiCompatibleProvider`]'s usage extractor already normalises are opted +/// in. Unknown / custom slugs (including local LM Studio endpoints, which do no +/// server-side billing-grade caching) get the all-`false` default, so we never +/// advertise caching a custom endpoint may not do. `explicit_cache_control` +/// stays `false` for every OpenAI-compatible provider — the chat-completions +/// API has no cache-control field — and `cache_key_grouping` is reserved for +/// the OpenHuman backend's `thread_id` extension, declared on +/// `OpenHumanBackendProvider` directly. +pub(crate) fn prompt_cache_for_compatible_slug( + slug: &str, +) -> super::traits::PromptCacheCapabilities { + // Compare on the leading slug segment so a user-renamed `openai-eu` or a + // `openai:gpt-5.1` style slug still resolves to the `openai` family. + let normalized = slug.trim().to_ascii_lowercase(); + let family = normalized + .split(|c| c == ':' || c == '/' || c == '-') + .next() + .unwrap_or("") + .trim(); + + // Verified OpenAI-style implicit prefix cache + cached-token usage reporting. + let openai_style_cache = matches!(family, "openai" | "openrouter" | "gmi"); + + let caps = super::traits::PromptCacheCapabilities { + automatic_prefix_cache: openai_style_cache, + explicit_cache_control: false, + usage_reports_cached_input: openai_style_cache, + cache_key_grouping: false, + }; + + tracing::debug!( + domain = "llm_provider", + operation = "prompt_cache_capability", + provider = %family, + automatic_prefix_cache = caps.automatic_prefix_cache, + explicit_cache_control = caps.explicit_cache_control, + usage_reports_cached_input = caps.usage_reports_cached_input, + cache_key_grouping = caps.cache_key_grouping, + "[llm_provider] prompt-cache capability selected for compatible provider" + ); + + caps +} + #[cfg(test)] #[path = "compatible_tests.rs"] mod tests; diff --git a/src/openhuman/inference/provider/compatible_provider_impl.rs b/src/openhuman/inference/provider/compatible_provider_impl.rs index 82fb0ffc5..023248c4d 100644 --- a/src/openhuman/inference/provider/compatible_provider_impl.rs +++ b/src/openhuman/inference/provider/compatible_provider_impl.rs @@ -23,6 +23,15 @@ impl Provider for OpenAiCompatibleProvider { } } + fn prompt_cache_capabilities( + &self, + ) -> crate::openhuman::inference::provider::traits::PromptCacheCapabilities { + // Derive from the configured slug — conservative for unknown / custom + // providers (#3939). The OpenHuman backend wraps this provider but + // declares its own grouping-aware caps on `OpenHumanBackendProvider`. + super::prompt_cache_for_compatible_slug(&self.name) + } + async fn chat_with_system( &self, system_prompt: Option<&str>, diff --git a/src/openhuman/inference/provider/compatible_tests.rs b/src/openhuman/inference/provider/compatible_tests.rs index 7bebc9523..753911c14 100644 --- a/src/openhuman/inference/provider/compatible_tests.rs +++ b/src/openhuman/inference/provider/compatible_tests.rs @@ -3675,3 +3675,112 @@ async fn effective_context_window_lmstudio_falls_back_when_native_unavailable() Some(8_192) ); } + +// ---------------------------------------------------------- +// Prompt-cache capability model (#3939) +// ---------------------------------------------------------- + +#[test] +fn prompt_cache_caps_openai_style_for_known_slugs() { + for slug in ["openai", "openrouter", "gmi"] { + let caps = super::prompt_cache_for_compatible_slug(slug); + assert!( + caps.automatic_prefix_cache, + "{slug} should advertise automatic prefix cache" + ); + assert!( + caps.usage_reports_cached_input, + "{slug} should report cached input tokens" + ); + assert!( + !caps.explicit_cache_control, + "OpenAI-compatible chat API has no cache-control field" + ); + assert!( + !caps.cache_key_grouping, + "thread/session grouping is OpenHuman-backend-only" + ); + } +} + +#[test] +fn prompt_cache_caps_match_slug_family_variants() { + // Case-insensitive, leading-segment family match so renamed/suffixed slugs + // still resolve to the verified family. + for slug in ["OpenAI", "openai:gpt-5.1", "openai/responses", "openai-eu"] { + let caps = super::prompt_cache_for_compatible_slug(slug); + assert!( + caps.automatic_prefix_cache && caps.usage_reports_cached_input, + "{slug} should resolve to the openai family" + ); + } +} + +#[test] +fn prompt_cache_caps_conservative_for_unknown_or_custom_slugs() { + // Custom / local / unverified providers must not advertise caching — they + // get the all-false default so we never send or assume unsupported behaviour. + let conservative = + crate::openhuman::inference::provider::traits::PromptCacheCapabilities::default(); + for slug in ["custom_openai", "lmstudio", "deepseek", "mystery-proxy", ""] { + assert_eq!( + super::prompt_cache_for_compatible_slug(slug), + conservative, + "{slug} must stay conservative" + ); + } +} + +#[test] +fn compatible_provider_declares_prompt_cache_from_its_slug() { + let conservative = + crate::openhuman::inference::provider::traits::PromptCacheCapabilities::default(); + + let openai = make_provider("openai", "https://api.openai.com", Some("k")); + let caps = openai.prompt_cache_capabilities(); + assert!( + caps.automatic_prefix_cache && caps.usage_reports_cached_input, + "openai provider must advertise OpenAI-style caching" + ); + + let custom = make_provider("custom_openai", "https://proxy.example", Some("k")); + assert_eq!( + custom.prompt_cache_capabilities(), + conservative, + "unknown custom provider must stay conservative" + ); +} + +#[test] +fn extract_usage_normalizes_openai_cached_prompt_tokens() { + // Regression: an OpenAI-compatible usage block carrying cached prefix tokens + // (`prompt_tokens_details.cached_tokens`) must normalize into + // `UsageInfo.cached_input_tokens` so cached-prefix cost accounting is exact. + let json = r#"{ + "choices":[{"message":{"role":"assistant","content":"hi"}}], + "usage":{"prompt_tokens":1000,"completion_tokens":20,"total_tokens":1020, + "prompt_tokens_details":{"cached_tokens":768}} + }"#; + let resp: ApiChatResponse = serde_json::from_str(json).expect("parse api response"); + let usage = OpenAiCompatibleProvider::extract_usage(&resp).expect("usage present"); + assert_eq!(usage.input_tokens, 1000); + assert_eq!(usage.output_tokens, 20); + assert_eq!( + usage.cached_input_tokens, 768, + "cached prefix tokens must be normalized into cached_input_tokens" + ); +} + +#[test] +fn extract_usage_defaults_cached_tokens_to_zero_when_absent() { + // A provider that omits cache details must yield cached_input_tokens = 0, + // keeping cost accounting coherent (full prompt charged at the input rate). + let json = r#"{ + "choices":[{"message":{"role":"assistant","content":"hi"}}], + "usage":{"prompt_tokens":500,"completion_tokens":10,"total_tokens":510} + }"#; + let resp: ApiChatResponse = serde_json::from_str(json).expect("parse api response"); + let usage = OpenAiCompatibleProvider::extract_usage(&resp).expect("usage present"); + assert_eq!(usage.cached_input_tokens, 0); + assert_eq!(usage.input_tokens, 500); +} diff --git a/src/openhuman/inference/provider/factory.rs b/src/openhuman/inference/provider/factory.rs index 05cbca05f..979c66230 100644 --- a/src/openhuman/inference/provider/factory.rs +++ b/src/openhuman/inference/provider/factory.rs @@ -418,7 +418,7 @@ pub(crate) fn resolve_byok_fallback_provider_string(config: &Config) -> Option ProviderCapabilities { self.0.capabilities() } + fn prompt_cache_capabilities(&self) -> PromptCacheCapabilities { + self.0.prompt_cache_capabilities() + } async fn chat_with_system( &self, system_prompt: Option<&str>, diff --git a/src/openhuman/inference/provider/mod.rs b/src/openhuman/inference/provider/mod.rs index 530a75150..fe00ec935 100644 --- a/src/openhuman/inference/provider/mod.rs +++ b/src/openhuman/inference/provider/mod.rs @@ -27,8 +27,8 @@ pub mod traits; #[allow(unused_imports)] pub use traits::{ - ChatMessage, ChatRequest, ChatResponse, ConversationMessage, Provider, ProviderCapabilityError, - ProviderDelta, ToolCall, ToolResultMessage, UsageInfo, + ChatMessage, ChatRequest, ChatResponse, ConversationMessage, PromptCacheCapabilities, Provider, + ProviderCapabilityError, ProviderDelta, ToolCall, ToolResultMessage, UsageInfo, }; pub use billing_error::is_budget_exhausted_message; diff --git a/src/openhuman/inference/provider/openhuman_backend.rs b/src/openhuman/inference/provider/openhuman_backend.rs index afe207b47..d49586e95 100644 --- a/src/openhuman/inference/provider/openhuman_backend.rs +++ b/src/openhuman/inference/provider/openhuman_backend.rs @@ -138,6 +138,21 @@ impl Provider for OpenHumanBackendProvider { } } + fn prompt_cache_capabilities( + &self, + ) -> crate::openhuman::inference::provider::traits::PromptCacheCapabilities { + // The hosted backend caches byte-stable prefixes, reports cached input + // tokens via `openhuman.usage.cached_input_tokens`, and groups calls by + // the `thread_id` extension for cache locality + InferenceLog grouping + // (#3939). It does not accept explicit cache-control markers. + crate::openhuman::inference::provider::traits::PromptCacheCapabilities { + automatic_prefix_cache: true, + explicit_cache_control: false, + usage_reports_cached_input: true, + cache_key_grouping: true, + } + } + async fn chat_with_system( &self, system_prompt: Option<&str>, @@ -216,6 +231,19 @@ mod tests { // intercept blank / whitespace-only values before they hit the wire and // substitute the canonical default tier. + #[test] + fn backend_declares_grouping_aware_prompt_cache() { + // #3939: the hosted backend caches prefixes, reports cached input + // tokens, and groups by thread_id — but accepts no explicit + // cache-control field. + let provider = OpenHumanBackendProvider::new(None, &ProviderRuntimeOptions::default()); + let caps = provider.prompt_cache_capabilities(); + assert!(caps.automatic_prefix_cache); + assert!(caps.usage_reports_cached_input); + assert!(caps.cache_key_grouping); + assert!(!caps.explicit_cache_control); + } + #[test] fn resolve_model_substitutes_default_for_empty() { assert_eq!( diff --git a/src/openhuman/inference/provider/traits.rs b/src/openhuman/inference/provider/traits.rs index 2dd2c999a..36e7471a4 100644 --- a/src/openhuman/inference/provider/traits.rs +++ b/src/openhuman/inference/provider/traits.rs @@ -317,6 +317,40 @@ pub struct ProviderCapabilities { pub vision: bool, } +/// Prompt / KV-cache behaviour a provider supports. +/// +/// Sibling to [`ProviderCapabilities`], surfaced via +/// [`Provider::prompt_cache_capabilities`] so the agent and cost layers can +/// pick a stable cache-key strategy and calibrate cached-token telemetry per +/// provider. Every field defaults to `false` (conservative): an unknown or +/// custom OpenAI-compatible provider is assumed to support no caching, so we +/// never infer cache behaviour — or send cache-only request fields — that the +/// upstream may not honour (#3939). +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct PromptCacheCapabilities { + /// Provider transparently caches identical request prefixes server-side + /// with no client action — e.g. OpenAI / DeepSeek / Anthropic implicit + /// caching. A byte-stable prompt prefix then earns cache hits for free, so + /// preserving the prefix is worthwhile for this provider. + pub automatic_prefix_cache: bool, + /// Provider accepts explicit cache-control / cache-boundary markers in the + /// request body (e.g. Anthropic `cache_control`). OpenAI-compatible chat + /// APIs do not, so this stays `false` for them — we must not send such + /// fields to a provider that would reject or ignore them. + pub explicit_cache_control: bool, + /// Provider returns cached-input-token counts in its usage block + /// (`prompt_tokens_details.cached_tokens` or + /// `openhuman.usage.cached_input_tokens`), so [`UsageInfo::cached_input_tokens`] + /// is populated and cached-prefix cost accounting is exact rather than + /// estimated. + pub usage_reports_cached_input: bool, + /// Provider supports grouping calls by a stable logical key (thread / + /// session) for cache locality — today only the OpenHuman backend, via its + /// `thread_id` extension. Third-party providers rely on prefix identity + /// instead and must not receive OpenHuman-only grouping fields. + pub cache_key_grouping: bool, +} + /// Provider-specific tool payload formats. /// /// Different LLM providers require different formats for tool definitions. @@ -366,6 +400,19 @@ pub trait Provider: Send + Sync { ProviderCapabilities::default() } + /// Declare the provider's prompt / KV-cache behaviour. + /// + /// Default is the conservative all-`false` [`PromptCacheCapabilities`]: + /// callers must not assume any caching for a provider that hasn't opted in. + /// Providers that cache prefixes server-side, report cached input tokens, + /// or support thread/session grouping override this to advertise it so the + /// agent + cost layers get accurate cache telemetry and a stable cache-key + /// strategy without leaking OpenHuman internals to providers that don't + /// need them (#3939). + fn prompt_cache_capabilities(&self) -> PromptCacheCapabilities { + PromptCacheCapabilities::default() + } + /// Convert tool specifications to provider-native format. /// /// Default implementation returns `PromptGuided` payload, which injects