diff --git a/app/src/components/settings/panels/ModelHealthPanel.tsx b/app/src/components/settings/panels/ModelHealthPanel.tsx index 0c54b399e..853c3b120 100644 --- a/app/src/components/settings/panels/ModelHealthPanel.tsx +++ b/app/src/components/settings/panels/ModelHealthPanel.tsx @@ -12,7 +12,11 @@ const log = debug('openhuman:model-health'); interface ModelEntry { id: string; provider: string; + /** USD per 1M input tokens (0/absent ⇒ unknown). */ + cost_per_1m_input?: number; cost_per_1m_output: number; + /** Max context window in tokens (0/absent ⇒ unknown). */ + context_window?: number; vision: boolean; quality_score: number | null; hallucination_rate: number | null; @@ -20,6 +24,17 @@ interface ModelEntry { tasks_evaluated: number; } +/** Compact token-count label, e.g. 1_000_000 → "1M", 128_000 → "128K". */ +function formatContextWindow(tokens: number | undefined): string { + if (!tokens) return '—'; + if (tokens >= 1_000_000) { + const m = tokens / 1_000_000; + return `${Number.isInteger(m) ? m : m.toFixed(1)}M`; + } + if (tokens >= 1_000) return `${Math.round(tokens / 1_000)}K`; + return String(tokens); +} + interface HealthConfig { hallucination_threshold: number; min_tasks_for_rating: number; @@ -244,7 +259,14 @@ const ModelHealthPanel = () => {
{m.id}
-
{m.provider}
+
+ {m.provider} + {m.context_window ? ( + + · {formatContextWindow(m.context_window)} ctx + + ) : null} +
{qualityStars(m.quality_score)} @@ -261,7 +283,11 @@ const ModelHealthPanel = () => { '—' )} - ${m.cost_per_1m_output.toFixed(2)} + + {m.cost_per_1m_input + ? `$${m.cost_per_1m_input.toFixed(2)} / $${m.cost_per_1m_output.toFixed(2)}` + : `$${m.cost_per_1m_output.toFixed(2)}`} + {m.agents_using} { expect(screen.getByText('bad-model')).toBeTruthy(); }); + it('shows context window and input/output pricing', async () => { + await mockRpc(MOCK_RESPONSE); + renderWithProviders(); + await waitFor(() => { + expect(screen.getByText('deepseek-v3')).toBeTruthy(); + }); + // Context window rendered as a compact token-count label next to provider. + expect(screen.getByText(/128K ctx/)).toBeTruthy(); + // Cost cell shows "input / output" when input pricing is present. + expect(screen.getByText(/\$0\.14 \/ \$0\.33/)).toBeTruthy(); + }); + it('shows correct status badges', async () => { await mockRpc(MOCK_RESPONSE); renderWithProviders(); diff --git a/app/src/services/api/__tests__/aiSettingsApi.test.ts b/app/src/services/api/__tests__/aiSettingsApi.test.ts index 3f08d351c..e16047b14 100644 --- a/app/src/services/api/__tests__/aiSettingsApi.test.ts +++ b/app/src/services/api/__tests__/aiSettingsApi.test.ts @@ -741,7 +741,15 @@ describe('saveAISettings', () => { await saveAISettings(prev, next); const patch = mockOpenhumanUpdateModelSettings.mock.calls[0][0]; expect(patch.model_registry).toEqual([ - { id: 'my-llava', provider: 'openai', cost_per_1m_output: 0, vision: true }, + { + id: 'my-llava', + provider: 'openai', + cost_per_1m_input: 0, + cost_per_1m_cached_input: 0, + cost_per_1m_output: 0, + context_window: 0, + vision: true, + }, ]); }); @@ -1049,7 +1057,15 @@ describe('model registry vision helpers', () => { it('upsertModelRegistryVision adds, flips, and removes entries', () => { const added = upsertModelRegistryVision([], 'openai', 'my-llava', true); expect(added).toEqual([ - { id: 'my-llava', provider: 'openai', cost_per_1m_output: 0, vision: true }, + { + id: 'my-llava', + provider: 'openai', + cost_per_1m_input: 0, + cost_per_1m_cached_input: 0, + cost_per_1m_output: 0, + context_window: 0, + vision: true, + }, ]); // vision:false removes the entry (absence ⇒ no vision). const removed = upsertModelRegistryVision(reg, 'openai', 'gpt-4o', false); diff --git a/app/src/services/api/aiSettingsApi.ts b/app/src/services/api/aiSettingsApi.ts index 3e4eed9f4..54e55ebb3 100644 --- a/app/src/services/api/aiSettingsApi.ts +++ b/app/src/services/api/aiSettingsApi.ts @@ -196,7 +196,15 @@ export function upsertModelRegistryVision( const existing = base.find(e => e.provider === provider && e.id === id); return [ ...without, - { id, provider, cost_per_1m_output: existing?.cost_per_1m_output ?? 0, vision: true }, + { + id, + provider, + cost_per_1m_input: existing?.cost_per_1m_input ?? 0, + cost_per_1m_cached_input: existing?.cost_per_1m_cached_input ?? 0, + cost_per_1m_output: existing?.cost_per_1m_output ?? 0, + context_window: existing?.context_window ?? 0, + vision: true, + }, ]; } @@ -391,10 +399,24 @@ export async function saveAISettings(prev: AISettings, next: AISettings): Promis // Per-model registry (vision flags): any change → send the full list. if (!modelRegistriesEqual(prev.modelRegistry, next.modelRegistry)) { patch.model_registry = next.modelRegistry.map( - ({ id, provider, cost_per_1m_output, vision }) => ({ + ({ id, provider, + cost_per_1m_input, + cost_per_1m_cached_input, cost_per_1m_output, + context_window, + vision, + }) => ({ + id, + provider, + // Preserve catalog-prefilled prices + context window through the + // round-trip; omitting them would let the Rust serde defaults zero + // them out. + cost_per_1m_input: cost_per_1m_input ?? 0, + cost_per_1m_cached_input: cost_per_1m_cached_input ?? 0, + cost_per_1m_output, + context_window: context_window ?? 0, vision, }) ); @@ -415,7 +437,14 @@ function modelRegistriesEqual(a: ModelRegistryEntry[], b: ModelRegistryEntry[]): const bByKey = new Map(b.map(e => [key(e), e])); return a.every(e => { const m = bByKey.get(key(e)); - return !!m && m.vision === e.vision && m.cost_per_1m_output === e.cost_per_1m_output; + return ( + !!m && + m.vision === e.vision && + m.cost_per_1m_output === e.cost_per_1m_output && + (m.cost_per_1m_input ?? 0) === (e.cost_per_1m_input ?? 0) && + (m.cost_per_1m_cached_input ?? 0) === (e.cost_per_1m_cached_input ?? 0) && + (m.context_window ?? 0) === (e.context_window ?? 0) + ); }); } diff --git a/app/src/utils/tauriCommands/config.ts b/app/src/utils/tauriCommands/config.ts index 2776ec484..bab210aac 100644 --- a/app/src/utils/tauriCommands/config.ts +++ b/app/src/utils/tauriCommands/config.ts @@ -57,7 +57,16 @@ export interface CloudProviderCreds { export interface ModelRegistryEntry { id: string; provider: string; + /** USD per 1M input tokens. `0`/absent ⇒ unknown. Pre-filled for known + * vendor models from the Rust pricing catalog (`cost/catalog.rs`). */ + cost_per_1m_input?: number; + /** USD per 1M cached-prefix input tokens. `0`/absent ⇒ unknown. */ + cost_per_1m_cached_input?: number; cost_per_1m_output: number; + /** Max context window in tokens. `0`/absent ⇒ unknown. Pre-filled for known + * vendor models from the Rust pricing catalog. Used to budget prompts and + * trigger compaction — providers differ widely (128K–1M+). */ + context_window?: number; vision: boolean; } diff --git a/src/openhuman/agent/cost.rs b/src/openhuman/agent/cost.rs index 0fc103f77..0a4c5c6ff 100644 --- a/src/openhuman/agent/cost.rs +++ b/src/openhuman/agent/cost.rs @@ -111,13 +111,27 @@ pub const PRICING_TABLE: &[ModelPricing] = &[ /// Look up pricing for a model name, falling back to [`FALLBACK_PRICING`]. /// -/// Matching is exact on the canonical tier name and case-insensitive on -/// concrete vendor names (so `"claude-opus"` still hits the -/// reasoning-tier row when callers pass an underlying model string). +/// Resolution order: +/// 1. Exact match on a canonical OpenHuman tier name (`agentic-v1`, …). +/// 2. The concrete-vendor-model pricing catalog +/// ([`crate::openhuman::cost::catalog`]) — accurate per-model rates for +/// `claude-*`, `gpt-*`, `gemini-*`, `deepseek-*`, `kimi-*`, `qwen-*`, +/// `mistral-*`, including OpenRouter-style `vendor/model` ids. +/// 3. Coarse case-insensitive vendor-name heuristics (so an unrecognised +/// `"…opus…"` string still maps to the reasoning tier). +/// 4. [`FALLBACK_PRICING`]. pub fn lookup_pricing(model: &str) -> ModelPricing { if let Some(row) = PRICING_TABLE.iter().find(|row| row.model == model) { return *row; } + if let Some(price) = crate::openhuman::cost::catalog::lookup(model) { + return ModelPricing { + model: price.model_id, + input_per_mtok_usd: price.input_per_mtok_usd, + cached_input_per_mtok_usd: price.cached_input_per_mtok_usd, + output_per_mtok_usd: price.output_per_mtok_usd, + }; + } let lower = model.to_ascii_lowercase(); let by_tier = |tier: &str| { PRICING_TABLE diff --git a/src/openhuman/config/ops/loader.rs b/src/openhuman/config/ops/loader.rs index 51b21866c..ad2e6021e 100644 --- a/src/openhuman/config/ops/loader.rs +++ b/src/openhuman/config/ops/loader.rs @@ -81,12 +81,52 @@ pub async fn reload_config_snapshot_with_timeout(snapshot: &Config) -> Result 0 { + log::debug!("[config] backfilled pricing on {filled} model_registry entries from catalog"); + } } /// Returns the default workspace directory fallback (~/.openhuman/workspace). @@ -294,7 +334,10 @@ pub fn client_config_json(config: &Config) -> serde_json::Value { serde_json::json!({ "id": m.id, "provider": m.provider, + "cost_per_1m_input": m.cost_per_1m_input, + "cost_per_1m_cached_input": m.cost_per_1m_cached_input, "cost_per_1m_output": m.cost_per_1m_output, + "context_window": m.context_window, "vision": m.vision, }) }) @@ -591,6 +634,67 @@ pub async fn get_data_paths() -> Result, String> { )) } +#[cfg(test)] +mod model_registry_seed_tests { + use super::*; + + #[test] + fn seeds_empty_registry_from_catalog() { + let mut config = Config { + model_registry: Vec::new(), + ..Default::default() + }; + seed_and_enrich_model_registry(&mut config); + assert!( + !config.model_registry.is_empty(), + "empty registry should be seeded from the catalog" + ); + // Every seeded entry carries pricing + a context window. + for entry in &config.model_registry { + assert!(entry.cost_per_1m_input > 0.0, "{}", entry.id); + assert!(entry.cost_per_1m_output > 0.0, "{}", entry.id); + assert!(entry.context_window > 0, "{}", entry.id); + } + } + + #[test] + fn backfills_existing_entries_but_preserves_user_values_and_count() { + let mut config = Config { + model_registry: vec![ + // Known model, missing prices → backfilled. + crate::openhuman::config::schema::ModelRegistryEntry { + id: "claude-opus-4-8".to_string(), + provider: "anthropic".to_string(), + cost_per_1m_output: 99.0, // user override — must survive + vision: true, + ..Default::default() + }, + // Unknown model → left untouched. + crate::openhuman::config::schema::ModelRegistryEntry { + id: "my-byok-model".to_string(), + provider: "custom".to_string(), + ..Default::default() + }, + ], + ..Default::default() + }; + seed_and_enrich_model_registry(&mut config); + + assert_eq!( + config.model_registry.len(), + 2, + "must not seed when non-empty" + ); + let opus = &config.model_registry[0]; + assert_eq!(opus.cost_per_1m_input, 5.00, "backfilled"); + assert_eq!(opus.context_window, 1_000_000, "backfilled"); + assert_eq!(opus.cost_per_1m_output, 99.0, "user value preserved"); + let byok = &config.model_registry[1]; + assert_eq!(byok.cost_per_1m_input, 0.0, "unknown model untouched"); + assert_eq!(byok.context_window, 0); + } +} + #[cfg(test)] mod loader_io_chain_tests { use super::*; diff --git a/src/openhuman/config/ops_tests.rs b/src/openhuman/config/ops_tests.rs index da16ccbe2..fb4a035ab 100644 --- a/src/openhuman/config/ops_tests.rs +++ b/src/openhuman/config/ops_tests.rs @@ -658,6 +658,7 @@ async fn apply_model_settings_replaces_model_registry_when_some_and_keeps_when_n provider: "openai".into(), cost_per_1m_output: 0.0, vision: true, + ..Default::default() }]), ..Default::default() }; @@ -707,6 +708,7 @@ async fn apply_model_settings_trims_model_registry_ids() { provider: "openai".into(), cost_per_1m_output: 0.0, vision: true, + ..Default::default() }]), ..Default::default() }; diff --git a/src/openhuman/config/schema/types.rs b/src/openhuman/config/schema/types.rs index c8ba71ecf..2917210cb 100644 --- a/src/openhuman/config/schema/types.rs +++ b/src/openhuman/config/schema/types.rs @@ -36,12 +36,30 @@ pub const DEFAULT_MEMORY_SYNC_INTERVAL_SECS: u64 = 86_400; /// "Manual only" is represented separately by `Some(0)`. See issue #3302. pub const MEMORY_SYNC_INTERVAL_PRESETS_SECS: [u64; 3] = [14_400, 43_200, 86_400]; -#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +#[derive(Debug, Clone, Default, Serialize, Deserialize, JsonSchema)] pub struct ModelRegistryEntry { pub id: String, pub provider: String, + /// Standard prompt rate, USD per **million input tokens**. Used (together + /// with [`Self::cost_per_1m_output`]) to estimate request cost when the + /// provider doesn't echo an authoritative `charged_amount_usd`. `0.0` means + /// "unknown" — callers fall back to the tier/catalog estimate. Pre-filled + /// for known vendor models from [`crate::openhuman::cost::catalog`]. + #[serde(default)] + pub cost_per_1m_input: f64, + /// Cached-prefix prompt rate, USD per million cached input tokens (KV-cache + /// read hits on supporting backends). `0.0` means "unknown". + #[serde(default)] + pub cost_per_1m_cached_input: f64, + /// Completion rate, USD per **million output tokens**. #[serde(default)] pub cost_per_1m_output: f64, + /// Maximum context window in tokens (published max input). `0` means + /// "unknown". Providers differ widely (128K–1M+); callers use this to + /// budget prompts, trigger compaction, and route work. Pre-filled for known + /// vendor models from [`crate::openhuman::cost::catalog`]. + #[serde(default)] + pub context_window: u32, #[serde(default)] pub vision: bool, } diff --git a/src/openhuman/cost/catalog.rs b/src/openhuman/cost/catalog.rs new file mode 100644 index 000000000..1d2ef2dc9 --- /dev/null +++ b/src/openhuman/cost/catalog.rs @@ -0,0 +1,554 @@ +//! Static pricing + context-window catalog for known LLM models. +//! +//! This is the single source of truth for **pre-filled** per-model metadata of +//! the default models the product can route to: +//! +//! - per-token pricing (input + cached-input + output, USD per million tokens), +//! - the model's **context window** (max input tokens) — different providers +//! ship very different windows, and callers need it to budget prompts, +//! trigger compaction, and route work. +//! +//! It exists so the client can estimate request cost and reason about context +//! limits for any provider, used to: +//! +//! - pre-fill [`crate::openhuman::config::schema::ModelRegistryEntry`] rows so +//! the Model Health dashboard shows real numbers instead of zeros, and +//! - power the fallback estimate in +//! [`crate::openhuman::agent::cost::lookup_pricing`] when a backend doesn't +//! echo an authoritative `charged_amount_usd`. +//! +//! ## Authority & freshness +//! +//! These are **best-effort published values** captured at [`PRICING_AS_OF`]. +//! The provider-reported `charged_amount_usd` always wins for cost when +//! present; the catalog is only a floor estimate. Context windows are the +//! published maximums and may differ from what a given deployment/tier exposes. +//! Prices and windows drift — when a provider changes them or a new default +//! model ships, update the matching row here (and bump [`PRICING_AS_OF`]). The +//! table is intentionally a plain `const` slice with no I/O so it's cheap to +//! consult on every lookup. +//! +//! ## Matching +//! +//! [`lookup`] resolves a concrete model string to a row. It normalises case, +//! strips a leading `vendor/` segment (OpenRouter-style ids like +//! `anthropic/claude-opus-4-8`) and trailing decorations (`:tag`, `@date`, +//! `[1m]`), and finally does a longest-substring match so dated/suffixed ids +//! (`claude-opus-4-8[1m]`, `gpt-5.4-2026-05-01`) still resolve. + +use crate::openhuman::config::schema::ModelRegistryEntry; + +/// Month the published values below were last verified. Bump when refreshing. +pub const PRICING_AS_OF: &str = "2026-06"; + +/// A single model's published per-million-token rates (USD) and context window. +#[derive(Debug, Clone, Copy)] +pub struct ModelPrice { + /// Canonical provider slug, matching the `cloud_providers` type strings + /// (`anthropic`, `openai`, `google`, `deepseek`, `moonshot`, `qwen`, + /// `mistral`). Used as the `provider` field when pre-filling registry rows. + pub provider: &'static str, + /// Canonical, lower-case model id used for matching. Keep these distinctive + /// (no bare `gpt-5`) so substring matching stays unambiguous. + pub model_id: &'static str, + /// USD per million standard (cache-miss) input tokens. + pub input_per_mtok_usd: f64, + /// USD per million cached-prefix input tokens. Best-effort: exact where the + /// provider publishes it, otherwise the provider's typical cache discount. + pub cached_input_per_mtok_usd: f64, + /// USD per million output tokens. + pub output_per_mtok_usd: f64, + /// Maximum context window in tokens (published max input). Providers differ + /// widely (128K–1M+); callers budget prompts / trigger compaction off this. + pub context_window: u32, +} + +/// Published list prices and context windows for the default models the product +/// can route to. +/// +/// Sources (captured [`PRICING_AS_OF`]): vendor pricing/model pages. Anthropic +/// price rows are authoritative (cached = 0.1× input, the documented cache-read +/// rate); other providers' cached rates use the published discount where known +/// and a conservative provider-typical fraction otherwise. Context windows are +/// the published maximums. +pub const KNOWN_MODEL_PRICING: &[ModelPrice] = &[ + // ── Anthropic (authoritative prices; cache read = 0.1× input) ──────────── + ModelPrice { + provider: "anthropic", + model_id: "claude-fable-5", + input_per_mtok_usd: 10.00, + cached_input_per_mtok_usd: 1.00, + output_per_mtok_usd: 50.00, + context_window: 1_000_000, + }, + ModelPrice { + provider: "anthropic", + model_id: "claude-opus-4-8", + input_per_mtok_usd: 5.00, + cached_input_per_mtok_usd: 0.50, + output_per_mtok_usd: 25.00, + context_window: 1_000_000, + }, + ModelPrice { + provider: "anthropic", + model_id: "claude-opus-4-7", + input_per_mtok_usd: 5.00, + cached_input_per_mtok_usd: 0.50, + output_per_mtok_usd: 25.00, + context_window: 1_000_000, + }, + ModelPrice { + provider: "anthropic", + model_id: "claude-opus-4-6", + input_per_mtok_usd: 5.00, + cached_input_per_mtok_usd: 0.50, + output_per_mtok_usd: 25.00, + context_window: 1_000_000, + }, + ModelPrice { + provider: "anthropic", + model_id: "claude-opus-4-5", + input_per_mtok_usd: 5.00, + cached_input_per_mtok_usd: 0.50, + output_per_mtok_usd: 25.00, + context_window: 200_000, + }, + ModelPrice { + provider: "anthropic", + model_id: "claude-sonnet-4-6", + input_per_mtok_usd: 3.00, + cached_input_per_mtok_usd: 0.30, + output_per_mtok_usd: 15.00, + context_window: 1_000_000, + }, + ModelPrice { + provider: "anthropic", + model_id: "claude-sonnet-4-5", + input_per_mtok_usd: 3.00, + cached_input_per_mtok_usd: 0.30, + output_per_mtok_usd: 15.00, + context_window: 200_000, + }, + ModelPrice { + provider: "anthropic", + model_id: "claude-haiku-4-5", + input_per_mtok_usd: 1.00, + cached_input_per_mtok_usd: 0.10, + output_per_mtok_usd: 5.00, + context_window: 200_000, + }, + // ── OpenAI (cache read ≈ 0.25× input — published 75% off) ──────────────── + ModelPrice { + provider: "openai", + model_id: "gpt-5.5", + input_per_mtok_usd: 5.00, + cached_input_per_mtok_usd: 1.25, + output_per_mtok_usd: 30.00, + context_window: 1_000_000, + }, + ModelPrice { + provider: "openai", + model_id: "gpt-5.4", + input_per_mtok_usd: 2.50, + cached_input_per_mtok_usd: 0.625, + output_per_mtok_usd: 15.00, + context_window: 1_000_000, + }, + ModelPrice { + provider: "openai", + model_id: "gpt-5.4-mini", + input_per_mtok_usd: 0.75, + cached_input_per_mtok_usd: 0.1875, + output_per_mtok_usd: 4.50, + context_window: 400_000, + }, + ModelPrice { + provider: "openai", + model_id: "gpt-5.4-nano", + input_per_mtok_usd: 0.20, + cached_input_per_mtok_usd: 0.05, + output_per_mtok_usd: 1.25, + context_window: 400_000, + }, + ModelPrice { + provider: "openai", + model_id: "gpt-4.1", + input_per_mtok_usd: 2.00, + cached_input_per_mtok_usd: 0.50, + output_per_mtok_usd: 8.00, + context_window: 1_000_000, + }, + ModelPrice { + provider: "openai", + model_id: "gpt-4.1-mini", + input_per_mtok_usd: 0.40, + cached_input_per_mtok_usd: 0.10, + output_per_mtok_usd: 1.60, + context_window: 1_000_000, + }, + ModelPrice { + provider: "openai", + model_id: "o3", + input_per_mtok_usd: 2.00, + cached_input_per_mtok_usd: 0.50, + output_per_mtok_usd: 8.00, + context_window: 200_000, + }, + // ── Google Gemini (cache read ≈ 0.25× input; 1M-token windows) ─────────── + ModelPrice { + provider: "google", + model_id: "gemini-2.5-pro", + input_per_mtok_usd: 1.25, + cached_input_per_mtok_usd: 0.3125, + output_per_mtok_usd: 10.00, + context_window: 1_048_576, + }, + ModelPrice { + provider: "google", + model_id: "gemini-2.5-flash", + input_per_mtok_usd: 0.30, + cached_input_per_mtok_usd: 0.075, + output_per_mtok_usd: 2.50, + context_window: 1_048_576, + }, + ModelPrice { + provider: "google", + model_id: "gemini-2.5-flash-lite", + input_per_mtok_usd: 0.10, + cached_input_per_mtok_usd: 0.025, + output_per_mtok_usd: 0.40, + context_window: 1_048_576, + }, + // ── DeepSeek (cache hit = 0.1× input, published) ───────────────────────── + ModelPrice { + provider: "deepseek", + model_id: "deepseek-chat", + input_per_mtok_usd: 0.14, + cached_input_per_mtok_usd: 0.014, + output_per_mtok_usd: 0.28, + context_window: 128_000, + }, + ModelPrice { + provider: "deepseek", + model_id: "deepseek-reasoner", + input_per_mtok_usd: 0.55, + cached_input_per_mtok_usd: 0.055, + output_per_mtok_usd: 2.19, + context_window: 128_000, + }, + // ── Moonshot Kimi (cache hit published) ────────────────────────────────── + ModelPrice { + provider: "moonshot", + model_id: "kimi-k2.6", + input_per_mtok_usd: 0.95, + cached_input_per_mtok_usd: 0.16, + output_per_mtok_usd: 4.00, + context_window: 256_000, + }, + ModelPrice { + provider: "moonshot", + model_id: "kimi-k2.5", + input_per_mtok_usd: 0.60, + cached_input_per_mtok_usd: 0.10, + output_per_mtok_usd: 3.00, + context_window: 256_000, + }, + // ── Qwen / Alibaba (cache read ≈ 0.1× input) ───────────────────────────── + ModelPrice { + provider: "qwen", + model_id: "qwen3-max", + input_per_mtok_usd: 1.20, + cached_input_per_mtok_usd: 0.12, + output_per_mtok_usd: 6.00, + context_window: 256_000, + }, + ModelPrice { + provider: "qwen", + model_id: "qwen-max", + input_per_mtok_usd: 1.20, + cached_input_per_mtok_usd: 0.12, + output_per_mtok_usd: 6.00, + context_window: 256_000, + }, + ModelPrice { + provider: "qwen", + model_id: "qwen-plus", + input_per_mtok_usd: 0.40, + cached_input_per_mtok_usd: 0.04, + output_per_mtok_usd: 1.20, + context_window: 256_000, + }, + ModelPrice { + provider: "qwen", + model_id: "qwen-flash", + input_per_mtok_usd: 0.05, + cached_input_per_mtok_usd: 0.005, + output_per_mtok_usd: 0.40, + context_window: 256_000, + }, + // ── Mistral (cache read ≈ 0.1× input) ──────────────────────────────────── + ModelPrice { + provider: "mistral", + model_id: "mistral-large", + input_per_mtok_usd: 2.00, + cached_input_per_mtok_usd: 0.20, + output_per_mtok_usd: 6.00, + context_window: 128_000, + }, + ModelPrice { + provider: "mistral", + model_id: "mistral-medium", + input_per_mtok_usd: 0.40, + cached_input_per_mtok_usd: 0.04, + output_per_mtok_usd: 2.00, + context_window: 128_000, + }, + ModelPrice { + provider: "mistral", + model_id: "mistral-small", + input_per_mtok_usd: 0.20, + cached_input_per_mtok_usd: 0.02, + output_per_mtok_usd: 0.60, + context_window: 128_000, + }, + ModelPrice { + provider: "mistral", + model_id: "codestral", + input_per_mtok_usd: 0.30, + cached_input_per_mtok_usd: 0.03, + output_per_mtok_usd: 0.90, + context_window: 256_000, + }, + ModelPrice { + provider: "mistral", + model_id: "ministral-8b", + input_per_mtok_usd: 0.10, + cached_input_per_mtok_usd: 0.01, + output_per_mtok_usd: 0.10, + context_window: 128_000, + }, +]; + +/// Normalise a model string for matching: lower-case, trim, drop a trailing +/// `:tag` / `@date` decoration and a `[...]` suffix. +fn normalize(model: &str) -> String { + let mut s = model.trim().to_ascii_lowercase(); + // Strip a `[1m]`-style context-window suffix. + if let Some(idx) = s.find('[') { + s.truncate(idx); + } + // Strip `:tag` (e.g. ollama-style) and `@date` (Vertex-style) decorations. + for sep in [':', '@'] { + if let Some(idx) = s.find(sep) { + s.truncate(idx); + } + } + s.trim().to_string() +} + +/// Resolve a concrete model string to its catalogued row, if known. +/// +/// Match order: exact canonical id → id with a leading `vendor/` segment +/// stripped → longest canonical id that is a substring of the normalised +/// request (handles dated/suffixed ids). Returns `None` for unknown models — +/// callers should fall back to a tier estimate. +pub fn lookup(model: &str) -> Option<&'static ModelPrice> { + let norm = normalize(model); + if norm.is_empty() { + return None; + } + if let Some(p) = KNOWN_MODEL_PRICING.iter().find(|p| p.model_id == norm) { + return Some(p); + } + let bare = norm.rsplit('/').next().unwrap_or(norm.as_str()); + if let Some(p) = KNOWN_MODEL_PRICING.iter().find(|p| p.model_id == bare) { + return Some(p); + } + KNOWN_MODEL_PRICING + .iter() + .filter(|p| norm.contains(p.model_id) || bare.contains(p.model_id)) + .max_by_key(|p| p.model_id.len()) +} + +/// Published maximum context window (tokens) for a model, if catalogued. +/// +/// Convenience wrapper over [`lookup`] for callers that only need the window +/// to budget prompts / trigger compaction / pick a route. `None` ⇒ unknown. +pub fn context_window(model: &str) -> Option { + lookup(model).map(|p| p.context_window) +} + +/// Build a default registry, one [`ModelRegistryEntry`] per catalogued model +/// with prices and context window pre-filled. Used to seed an empty +/// `config.model_registry`. +pub fn default_registry_entries() -> Vec { + KNOWN_MODEL_PRICING + .iter() + .map(|p| ModelRegistryEntry { + id: p.model_id.to_string(), + provider: p.provider.to_string(), + cost_per_1m_input: p.input_per_mtok_usd, + cost_per_1m_cached_input: p.cached_input_per_mtok_usd, + cost_per_1m_output: p.output_per_mtok_usd, + context_window: p.context_window, + vision: false, + }) + .collect() +} + +/// Pre-fill any **missing** (zero) price or context-window field on a registry +/// entry from the catalog, matching on its `id`. Leaves user-supplied non-zero +/// values and the `vision` flag untouched. Returns `true` when a field was +/// filled in. +pub fn enrich_entry(entry: &mut ModelRegistryEntry) -> bool { + let Some(price) = lookup(&entry.id) else { + return false; + }; + let mut changed = false; + if entry.cost_per_1m_input == 0.0 { + entry.cost_per_1m_input = price.input_per_mtok_usd; + changed = true; + } + if entry.cost_per_1m_cached_input == 0.0 { + entry.cost_per_1m_cached_input = price.cached_input_per_mtok_usd; + changed = true; + } + if entry.cost_per_1m_output == 0.0 { + entry.cost_per_1m_output = price.output_per_mtok_usd; + changed = true; + } + if entry.context_window == 0 { + entry.context_window = price.context_window; + changed = true; + } + changed +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn exact_lookup_resolves_canonical_ids() { + let p = lookup("claude-opus-4-8").expect("anthropic row"); + assert_eq!(p.provider, "anthropic"); + assert_eq!(p.input_per_mtok_usd, 5.00); + assert_eq!(p.output_per_mtok_usd, 25.00); + assert_eq!(p.cached_input_per_mtok_usd, 0.50); + assert_eq!(p.context_window, 1_000_000); + } + + #[test] + fn lookup_is_case_insensitive() { + assert_eq!(lookup("GPT-4.1").unwrap().model_id, "gpt-4.1"); + } + + #[test] + fn lookup_strips_vendor_prefix_openrouter_style() { + assert_eq!( + lookup("anthropic/claude-sonnet-4-6").unwrap().model_id, + "claude-sonnet-4-6" + ); + assert_eq!( + lookup("deepseek/deepseek-chat").unwrap().model_id, + "deepseek-chat" + ); + assert_eq!(lookup("qwen/qwen3-max").unwrap().model_id, "qwen3-max"); + } + + #[test] + fn lookup_strips_context_and_tag_decorations() { + assert_eq!( + lookup("claude-opus-4-8[1m]").unwrap().model_id, + "claude-opus-4-8" + ); + assert_eq!(lookup("kimi-k2.6:turbo").unwrap().model_id, "kimi-k2.6"); + assert_eq!( + lookup("claude-opus-4-5@20251101").unwrap().model_id, + "claude-opus-4-5" + ); + } + + #[test] + fn lookup_longest_substring_wins_for_suffixed_ids() { + // A dated/suffixed id should resolve to the most specific row. + assert_eq!( + lookup("gpt-5.4-mini-2026-05-01").unwrap().model_id, + "gpt-5.4-mini" + ); + } + + #[test] + fn lookup_returns_none_for_unknown() { + assert!(lookup("totally-made-up-model").is_none()); + assert!(lookup("").is_none()); + assert!( + lookup("agentic-v1").is_none(), + "abstract tiers aren't vendor models" + ); + } + + #[test] + fn context_window_helper_resolves_known_models() { + assert_eq!(context_window("claude-opus-4-8"), Some(1_000_000)); + assert_eq!(context_window("openai/gpt-4.1-mini"), Some(1_000_000)); + assert_eq!(context_window("deepseek-chat"), Some(128_000)); + assert_eq!(context_window("totally-made-up"), None); + } + + #[test] + fn default_registry_entries_are_fully_populated() { + let entries = default_registry_entries(); + assert_eq!(entries.len(), KNOWN_MODEL_PRICING.len()); + for e in &entries { + assert!(e.cost_per_1m_input > 0.0, "{} missing input price", e.id); + assert!(e.cost_per_1m_output > 0.0, "{} missing output price", e.id); + assert!(e.context_window > 0, "{} missing context window", e.id); + assert!(!e.provider.is_empty()); + } + } + + #[test] + fn enrich_fills_zeros_but_preserves_user_values() { + let mut e = ModelRegistryEntry { + id: "claude-opus-4-8".to_string(), + provider: "anthropic".to_string(), + cost_per_1m_input: 0.0, + cost_per_1m_cached_input: 0.0, + cost_per_1m_output: 99.0, // user override — must survive + context_window: 0, + vision: true, + }; + assert!(enrich_entry(&mut e)); + assert_eq!(e.cost_per_1m_input, 5.00); + assert_eq!(e.cost_per_1m_cached_input, 0.50); + assert_eq!(e.cost_per_1m_output, 99.0, "user value preserved"); + assert_eq!(e.context_window, 1_000_000); + assert!(e.vision, "vision flag untouched"); + } + + #[test] + fn enrich_unknown_model_is_noop() { + let mut e = ModelRegistryEntry { + id: "unknown-model".to_string(), + ..Default::default() + }; + assert!(!enrich_entry(&mut e)); + assert_eq!(e.cost_per_1m_input, 0.0); + assert_eq!(e.context_window, 0); + } + + #[test] + fn every_row_has_sane_values() { + for p in KNOWN_MODEL_PRICING { + assert!(p.input_per_mtok_usd > 0.0, "{}", p.model_id); + assert!(p.output_per_mtok_usd > 0.0, "{}", p.model_id); + assert!(p.context_window > 0, "{}", p.model_id); + assert!( + p.cached_input_per_mtok_usd <= p.input_per_mtok_usd, + "{} cached should not exceed input", + p.model_id + ); + } + } +} diff --git a/src/openhuman/cost/mod.rs b/src/openhuman/cost/mod.rs index df659e346..812e643e7 100644 --- a/src/openhuman/cost/mod.rs +++ b/src/openhuman/cost/mod.rs @@ -1,3 +1,4 @@ +pub mod catalog; mod global; mod rpc; mod schemas; @@ -5,6 +6,10 @@ pub mod tools; pub mod tracker; pub mod types; +pub use catalog::{ + context_window as model_context_window, default_registry_entries, enrich_entry, + lookup as lookup_model_price, ModelPrice, +}; pub use global::{init_global, record_provider_usage, try_global}; pub use schemas::{ all_controller_schemas as all_cost_controller_schemas, diff --git a/src/openhuman/dashboard/ops.rs b/src/openhuman/dashboard/ops.rs index a7fc41edf..cebf8022a 100644 --- a/src/openhuman/dashboard/ops.rs +++ b/src/openhuman/dashboard/ops.rs @@ -28,7 +28,10 @@ pub fn model_health(config: &Config) -> Result, .map(|entry| ModelHealthEntry { id: entry.id.clone(), provider: entry.provider.clone(), + cost_per_1m_input: entry.cost_per_1m_input, + cost_per_1m_cached_input: entry.cost_per_1m_cached_input, cost_per_1m_output: entry.cost_per_1m_output, + context_window: entry.context_window, vision: entry.vision, // Placeholder metrics — see module-level docs. quality_score: None, @@ -70,12 +73,14 @@ mod tests { provider: "SiliconFlow".to_string(), cost_per_1m_output: 0.33, vision: false, + ..Default::default() }, crate::openhuman::config::schema::ModelRegistryEntry { id: "qwen-2.5-8b".to_string(), provider: "OpenRouter".to_string(), cost_per_1m_output: 0.09, vision: true, + ..Default::default() }, ]; cfg diff --git a/src/openhuman/dashboard/schemas.rs b/src/openhuman/dashboard/schemas.rs index 23eaa45f5..dcb12b2fe 100644 --- a/src/openhuman/dashboard/schemas.rs +++ b/src/openhuman/dashboard/schemas.rs @@ -80,12 +80,32 @@ fn model_health_entry_fields() -> Vec { comment: "Provider label, e.g. SiliconFlow, OpenRouter.", required: true, }, + FieldSchema { + name: "cost_per_1m_input", + ty: TypeSchema::F64, + comment: "USD cost per 1M input tokens (0 when unknown). Pre-filled \ + from the pricing catalog for known vendor models.", + required: true, + }, + FieldSchema { + name: "cost_per_1m_cached_input", + ty: TypeSchema::F64, + comment: "USD cost per 1M cached-prefix input tokens (0 when unknown).", + required: true, + }, FieldSchema { name: "cost_per_1m_output", ty: TypeSchema::F64, comment: "USD cost per 1M output tokens from local config.", required: true, }, + FieldSchema { + name: "context_window", + ty: TypeSchema::U64, + comment: "Maximum context window in tokens (0 when unknown). \ + Pre-filled from the pricing catalog for known vendor models.", + required: true, + }, FieldSchema { name: "vision", ty: TypeSchema::Bool, diff --git a/src/openhuman/dashboard/types.rs b/src/openhuman/dashboard/types.rs index 3ff420d0a..3015faa36 100644 --- a/src/openhuman/dashboard/types.rs +++ b/src/openhuman/dashboard/types.rs @@ -5,15 +5,22 @@ use serde::{Deserialize, Serialize}; /// One row in the model health comparison table. /// /// Mirrors the JSON shape consumed by the frontend -/// `ModelHealthPanel` — `id`/`provider`/`cost_per_1m_output`/`vision` -/// come from `Config::model_registry`; the four metric fields are -/// emitted as placeholder (`null` / `0`) values until a local telemetry -/// pipeline is wired in. +/// `ModelHealthPanel` — `id`/`provider`/`cost_per_1m_input`/ +/// `cost_per_1m_cached_input`/`cost_per_1m_output`/`vision` come from +/// `Config::model_registry` (pricing pre-filled from the cost catalog); +/// the four metric fields are emitted as placeholder (`null` / `0`) +/// values until a local telemetry pipeline is wired in. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub struct ModelHealthEntry { pub id: String, pub provider: String, + /// USD per 1M input tokens (`0.0` when unknown). + pub cost_per_1m_input: f64, + /// USD per 1M cached-prefix input tokens (`0.0` when unknown). + pub cost_per_1m_cached_input: f64, pub cost_per_1m_output: f64, + /// Maximum context window in tokens (`0` when unknown). + pub context_window: u32, pub vision: bool, pub quality_score: Option, pub hallucination_rate: Option, diff --git a/src/openhuman/inference/model_context.rs b/src/openhuman/inference/model_context.rs index 519ad13df..de9af34b8 100644 --- a/src/openhuman/inference/model_context.rs +++ b/src/openhuman/inference/model_context.rs @@ -314,12 +314,14 @@ mod tests { provider: "openai".into(), cost_per_1m_output: 0.0, vision: true, + ..Default::default() }, ModelRegistryEntry { id: "text-only".into(), provider: "openai".into(), cost_per_1m_output: 0.0, vision: false, + ..Default::default() }, ]; assert!(model_vision_enabled("my-llava", &config)); @@ -337,6 +339,7 @@ mod tests { provider: "openai".into(), cost_per_1m_output: 0.0, vision: true, + ..Default::default() }]; // `reasoning-v1` is the one vision-capable managed tier; the rest are not. assert!(model_supports_vision("reasoning-v1", &config)); diff --git a/tests/config_auth_app_state_connectivity_e2e.rs b/tests/config_auth_app_state_connectivity_e2e.rs index 364058173..2c4d1ec03 100644 --- a/tests/config_auth_app_state_connectivity_e2e.rs +++ b/tests/config_auth_app_state_connectivity_e2e.rs @@ -2968,6 +2968,64 @@ async fn config_controller_mutations_round_trip_over_json_rpc() { "client config must not echo local API keys: {client_payload}" ); + // The model registry is seeded + price/context-window-enriched from the + // static pricing catalog on load (in-memory), so a fresh workspace surfaces + // real numbers over RPC. Validates the full path: catalog → seed-on-load → + // client_config_json → JSON-RPC. (The earlier update_model_settings call did + // not include `model_registry`, so the seeded registry is left intact.) + let registry = client_payload + .get("model_registry") + .and_then(Value::as_array) + .expect("client config should expose model_registry"); + assert!( + !registry.is_empty(), + "model_registry should be auto-seeded from the pricing catalog: {client_payload}" + ); + let opus = registry + .iter() + .find(|m| m.get("id").and_then(Value::as_str) == Some("claude-opus-4-8")) + .unwrap_or_else(|| panic!("seeded registry should contain claude-opus-4-8: {registry:?}")); + assert_eq!( + opus.get("provider").and_then(Value::as_str), + Some("anthropic") + ); + assert_eq!( + opus.get("cost_per_1m_input").and_then(Value::as_f64), + Some(5.0), + "input price should be pre-filled from the catalog: {opus:?}" + ); + assert_eq!( + opus.get("cost_per_1m_output").and_then(Value::as_f64), + Some(25.0), + "output price should be pre-filled from the catalog: {opus:?}" + ); + assert_eq!( + opus.get("context_window").and_then(Value::as_u64), + Some(1_000_000), + "context window should be pre-filled from the catalog: {opus:?}" + ); + // Every seeded entry carries non-zero pricing + context window — the whole + // point of the catalog pre-fill. + for entry in registry { + let id = entry.get("id").and_then(Value::as_str).unwrap_or(""); + assert!( + entry + .get("cost_per_1m_output") + .and_then(Value::as_f64) + .unwrap_or(0.0) + > 0.0, + "{id} missing output price: {entry:?}" + ); + assert!( + entry + .get("context_window") + .and_then(Value::as_u64) + .unwrap_or(0) + > 0, + "{id} missing context window: {entry:?}" + ); + } + let memory = rpc( &harness.rpc_base, 10_004,