diff --git a/src/openhuman/agent/harness/engine/core.rs b/src/openhuman/agent/harness/engine/core.rs index d0f12efe1..9c58b39a2 100644 --- a/src/openhuman/agent/harness/engine/core.rs +++ b/src/openhuman/agent/harness/engine/core.rs @@ -25,7 +25,6 @@ use crate::openhuman::agent::cost::TurnCost; use crate::openhuman::agent::multimodal; use crate::openhuman::agent::stop_hooks::{current_stop_hooks, StopDecision, TurnState}; use crate::openhuman::context::guard::{ContextCheckResult, ContextGuard}; -use crate::openhuman::inference::model_context::context_window_for_model; use crate::openhuman::inference::provider::{ ChatMessage, ChatRequest, Provider, ProviderCapabilityError, }; @@ -92,7 +91,25 @@ pub(crate) async fn run_turn_engine( early_exit_tool_names: &[&str], run_queue: Option>, ) -> Result { - let mut context_guard = context_window_for_model(model) + // Resolve the model's context window once per turn. Local providers (e.g. + // LM Studio) report their *runtime-loaded* window here, which can be far + // smaller than the model's trained maximum in the static table — trimming + // to the max would overflow the loaded `n_ctx` (#3550 / TAURI-RUST-6V0). + let effective_context_window = provider.effective_context_window(model).await; + match effective_context_window { + Some(context_window) => tracing::debug!( + provider = provider_name, + model, + context_window, + "[agent_loop] effective context window resolved" + ), + None => tracing::debug!( + provider = provider_name, + model, + "[agent_loop] effective context window unavailable; pre-dispatch trimming disabled this turn" + ), + } + let mut context_guard = effective_context_window .map(ContextGuard::with_context_window) .unwrap_or_else(ContextGuard::new); let mut turn_cost = TurnCost::new(); @@ -174,7 +191,7 @@ pub(crate) async fn run_turn_engine( } } - if let Some(context_window) = context_window_for_model(model) { + if let Some(context_window) = effective_context_window { let budget_outcome = trim_chat_messages_to_budget(history, context_window); if budget_outcome.trimmed { log::warn!( @@ -291,7 +308,7 @@ pub(crate) async fn run_turn_engine( // *original* marker text, not the rendered // [FILE-EXTRACTED]/[FILE-ATTACHED]/[IMAGE:data:…] blocks. let mut prepared_messages_vec = prepared_messages.messages; - if let Some(context_window) = context_window_for_model(model) { + if let Some(context_window) = effective_context_window { let budget_outcome = trim_chat_messages_to_budget(&mut prepared_messages_vec, context_window); if budget_outcome.trimmed { diff --git a/src/openhuman/agent/harness/session/turn/core.rs b/src/openhuman/agent/harness/session/turn/core.rs index dd89d8aa2..65545670d 100644 --- a/src/openhuman/agent/harness/session/turn/core.rs +++ b/src/openhuman/agent/harness/session/turn/core.rs @@ -414,10 +414,19 @@ impl Agent { }; let turn_run_queue = self.run_queue.clone(); let cached_prefix = self.cached_transcript_messages.take(); + // Resolve the context window once per turn through the provider so + // local providers (LM Studio) trim to their runtime-loaded n_ctx + // rather than the trained-max table (#3550 / TAURI-RUST-6V0). + // Must run before `agent: self` takes the &mut borrow below. + let turn_context_window = self + .provider + .effective_context_window(&effective_model) + .await; let mut observer = AgentObserver { agent: self, artifact_store, effective_model: effective_model.clone(), + context_window: turn_context_window, cumulative_input: 0, cumulative_output: 0, cumulative_cached: 0, diff --git a/src/openhuman/agent/harness/session/turn_engine_adapter.rs b/src/openhuman/agent/harness/session/turn_engine_adapter.rs index ce7256bc3..f16d790ac 100644 --- a/src/openhuman/agent/harness/session/turn_engine_adapter.rs +++ b/src/openhuman/agent/harness/session/turn_engine_adapter.rs @@ -44,7 +44,6 @@ use crate::openhuman::agent::progress::AgentProgress; use crate::openhuman::agent::tool_policy::ToolPolicy; use crate::openhuman::agent_tool_policy::ToolPolicySession; use crate::openhuman::context::ReductionOutcome; -use crate::openhuman::inference::model_context::context_window_for_model; use crate::openhuman::inference::provider::{ ChatMessage, ChatRequest, ConversationMessage, Provider, ProviderDelta, ToolCall, UsageInfo, }; @@ -178,6 +177,11 @@ pub(super) struct AgentObserver<'a> { pub agent: &'a mut Agent, pub artifact_store: Option, pub effective_model: String, + /// Effective context window (tokens) for `effective_model`, resolved once + /// per turn via the provider so local providers (e.g. LM Studio) trim to + /// their *runtime-loaded* `n_ctx` rather than the model's trained maximum + /// (#3550 / Sentry TAURI-RUST-6V0). `None` → skip pre-dispatch trimming. + pub context_window: Option, pub cumulative_input: u64, pub cumulative_output: u64, pub cumulative_cached: u64, @@ -243,7 +247,7 @@ impl TurnObserver for AgentObserver<'_> { } // Pre-dispatch token-budget trim on the typed history. - if let Some(context_window) = context_window_for_model(&self.effective_model) { + if let Some(context_window) = self.context_window { super::super::token_budget::trim_conversation_history_to_budget( &mut self.agent.history, context_window, @@ -284,7 +288,7 @@ impl TurnObserver for AgentObserver<'_> { // Second-pass trim on the materialized provider messages (mirrors the // legacy `Agent::turn`, which trimmed both the typed history and the // built `ChatMessage` list). - if let Some(context_window) = context_window_for_model(&self.effective_model) { + if let Some(context_window) = self.context_window { super::super::token_budget::trim_chat_messages_to_budget(buf, context_window); } Ok(()) diff --git a/src/openhuman/inference/local/lm_studio.rs b/src/openhuman/inference/local/lm_studio.rs index c7a8ce679..511616d1f 100644 --- a/src/openhuman/inference/local/lm_studio.rs +++ b/src/openhuman/inference/local/lm_studio.rs @@ -244,10 +244,114 @@ pub(crate) struct LmStudioUsage { pub completion_tokens: Option, } +/// LM Studio **native** REST (`GET /api/v0/models`) model entry. +/// +/// Unlike the OpenAI-compatible `/v1/models` (which returns only +/// `id`/`object`/`owned_by`), the native API reports the model's context +/// window — the value the agent harness must budget against to avoid an +/// `n_ctx` overflow when the user loaded the model with a small context +/// (issue #3550 / Sentry TAURI-RUST-6V0). +#[derive(Debug, Clone, Deserialize)] +pub(crate) struct LmStudioNativeModel { + pub id: String, + /// Context window the model is *currently loaded* with — the runtime's + /// hard limit. Authoritative for budgeting. (LM Studio also returns a + /// `state` field, which we ignore — we prefer the loaded window whenever + /// present regardless of load state.) + #[serde(default)] + pub loaded_context_length: Option, + /// Model's declared maximum context. Fallback when not currently loaded. + #[serde(default)] + pub max_context_length: Option, +} + +#[derive(Debug, Deserialize)] +pub(crate) struct LmStudioNativeModelsResponse { + #[serde(default)] + pub data: Vec, +} + +/// Map a normalized `…/v1` base URL to the LM Studio native models endpoint +/// `…/api/v0/models` (a sibling of `/v1`, served at the host root). +pub(crate) fn lm_studio_native_models_url(v1_base_url: &str) -> String { + let root = v1_base_url + .trim_end_matches('/') + .trim_end_matches("/v1") + .trim_end_matches('/'); + format!("{root}/api/v0/models") +} + +/// Resolve the context window LM Studio reports for `model_id` from a native +/// `/api/v0/models` payload: prefer the *loaded* context (the limit the +/// runtime actually enforces), else the model's declared maximum. Zero/absent +/// values are treated as unknown. Returns `None` when the model isn't present +/// or reports no usable window. +pub(crate) fn lm_studio_context_window_for( + resp: &LmStudioNativeModelsResponse, + model_id: &str, +) -> Option { + resp.data.iter().find(|m| m.id == model_id).and_then(|m| { + m.loaded_context_length + .filter(|&v| v > 0) + .or(m.max_context_length.filter(|&v| v > 0)) + }) +} + #[cfg(test)] mod tests { use super::*; + #[test] + fn native_models_url_derived_from_v1_base() { + assert_eq!( + lm_studio_native_models_url("http://localhost:1234/v1"), + "http://localhost:1234/api/v0/models" + ); + // Trailing slash tolerated. + assert_eq!( + lm_studio_native_models_url("http://127.0.0.1:1234/v1/"), + "http://127.0.0.1:1234/api/v0/models" + ); + // Remote host with path prefix. + assert_eq!( + lm_studio_native_models_url("https://lm.example.com/lmstudio/v1"), + "https://lm.example.com/lmstudio/api/v0/models" + ); + } + + #[test] + fn context_window_prefers_loaded_then_max() { + let resp: LmStudioNativeModelsResponse = serde_json::from_str( + r#"{"data":[ + {"id":"qwen2.5-7b","state":"loaded","loaded_context_length":4096,"max_context_length":32768}, + {"id":"phi-4","state":"not-loaded","max_context_length":16384} + ]}"#, + ) + .unwrap(); + // Loaded model → the runtime-enforced loaded window, NOT the trained max. + assert_eq!( + lm_studio_context_window_for(&resp, "qwen2.5-7b"), + Some(4096) + ); + // Not-loaded model → declared max as fallback. + assert_eq!(lm_studio_context_window_for(&resp, "phi-4"), Some(16384)); + // Unknown model id → None (caller falls back to profile default). + assert_eq!(lm_studio_context_window_for(&resp, "missing"), None); + } + + #[test] + fn context_window_treats_zero_and_absent_as_unknown() { + let resp: LmStudioNativeModelsResponse = serde_json::from_str( + r#"{"data":[ + {"id":"zeroed","loaded_context_length":0,"max_context_length":0}, + {"id":"bare"} + ]}"#, + ) + .unwrap(); + assert_eq!(lm_studio_context_window_for(&resp, "zeroed"), None); + assert_eq!(lm_studio_context_window_for(&resp, "bare"), None); + } + #[test] fn normalize_lm_studio_base_url_defaults_scheme_and_v1() { assert_eq!( diff --git a/src/openhuman/inference/provider/compatible_provider_impl.rs b/src/openhuman/inference/provider/compatible_provider_impl.rs index ba11df5db..21a949340 100644 --- a/src/openhuman/inference/provider/compatible_provider_impl.rs +++ b/src/openhuman/inference/provider/compatible_provider_impl.rs @@ -1184,4 +1184,88 @@ impl Provider for OpenAiCompatibleProvider { } Ok(()) } + + /// Resolve the effective context window for pre-dispatch trimming. + /// + /// For cloud (non-local) providers this is the static model table. For + /// local providers the trained-maximum table over-estimates the window + /// the runtime actually enforces — LM Studio lets the user load a model + /// with a smaller `n_ctx`, and budgeting against the max overflows it + /// (#3550 / Sentry TAURI-RUST-6V0). So for LM Studio we query the native + /// `/api/v0/models` endpoint for the model's loaded context window, and + /// fall back to the provider-profile default when it's unavailable. + async fn effective_context_window(&self, model: &str) -> Option { + use crate::openhuman::inference::local::profile::LocalProviderKind; + let Some(kind) = self.local_provider_kind else { + return crate::openhuman::inference::context_window_for_model(model); + }; + // LM Studio: the OpenAI-compat /v1/models hides the context window, but + // the native /api/v0/models reports the loaded n_ctx. Prefer it. + if kind == LocalProviderKind::LmStudio { + if let Some(window) = self.lm_studio_loaded_context_window(model).await { + return Some(window); + } + } + // Local fallback: pattern table → provider-profile default. Ensures + // unknown local models still get trimmed instead of skipped. + crate::openhuman::inference::model_context::context_window_for_model_with_local_fallback( + model, + Some(kind), + ) + } +} + +impl OpenAiCompatibleProvider { + /// Best-effort fetch of the model's loaded context window from LM Studio's + /// native `/api/v0/models` endpoint. Returns `None` on any transport / + /// parse error or when the model reports no window — callers then fall + /// back to the profile default. Never panics, never propagates errors. + async fn lm_studio_loaded_context_window(&self, model: &str) -> Option { + use crate::openhuman::inference::local::lm_studio; + let url = lm_studio::lm_studio_native_models_url(&self.base_url); + let resp = match self + .apply_auth_header(self.http_client().get(&url), self.credential.as_deref()) + .send() + .await + { + Ok(resp) => resp, + Err(err) => { + tracing::debug!( + provider = %self.name, + model, + error = %err, + "[lm-studio] native models probe transport error; using profile default context window" + ); + return None; + } + }; + if !resp.status().is_success() { + tracing::debug!( + provider = %self.name, + status = resp.status().as_u16(), + "[lm-studio] native models probe non-success; using profile default context window" + ); + return None; + } + let parsed: lm_studio::LmStudioNativeModelsResponse = match resp.json().await { + Ok(parsed) => parsed, + Err(err) => { + tracing::debug!( + provider = %self.name, + model, + error = %err, + "[lm-studio] native models probe parse error; using profile default context window" + ); + return None; + } + }; + let window = lm_studio::lm_studio_context_window_for(&parsed, model); + tracing::debug!( + provider = %self.name, + model, + loaded_context_window = window.unwrap_or(0), + "[lm-studio] resolved loaded context window for pre-dispatch trimming" + ); + window + } } diff --git a/src/openhuman/inference/provider/compatible_tests.rs b/src/openhuman/inference/provider/compatible_tests.rs index 1e123a333..fa2238f31 100644 --- a/src/openhuman/inference/provider/compatible_tests.rs +++ b/src/openhuman/inference/provider/compatible_tests.rs @@ -3014,3 +3014,51 @@ fn stream_repeat_detector_ignores_varied_and_short_lines() { assert!(!d.observe("}\n"), "short repeated lines must not trip"); } } + +// ── effective_context_window (#3550 / Sentry TAURI-RUST-6V0) ─────────────── + +#[tokio::test] +async fn effective_context_window_cloud_uses_static_table() { + // No local provider kind → static trained-max table, unchanged behavior. + let p = make_provider("openai", "https://api.openai.com/v1", Some("k")); + assert_eq!(p.effective_context_window("gpt-4o").await, Some(128_000)); + // Unknown cloud model → None (skip trimming), as before. + assert_eq!(p.effective_context_window("totally-unknown").await, None); +} + +#[tokio::test] +async fn effective_context_window_lmstudio_uses_loaded_window() { + use crate::openhuman::inference::local::profile::LocalProviderKind; + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/api/v0/models")) + .respond_with(ResponseTemplate::new(200).set_body_raw( + r#"{"data":[{"id":"qwen2.5-7b","loaded_context_length":4096,"max_context_length":32768}]}"#, + "application/json", + )) + .mount(&server) + .await; + let p = OpenAiCompatibleProvider::new("lmstudio", &server.uri(), None, AuthStyle::None) + .with_local_provider_kind(LocalProviderKind::LmStudio); + // Trim to the runtime-loaded n_ctx (4096), NOT the model's trained max. + assert_eq!(p.effective_context_window("qwen2.5-7b").await, Some(4096)); +} + +#[tokio::test] +async fn effective_context_window_lmstudio_falls_back_when_native_unavailable() { + use crate::openhuman::inference::local::profile::LocalProviderKind; + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/api/v0/models")) + .respond_with(ResponseTemplate::new(404)) + .mount(&server) + .await; + let p = OpenAiCompatibleProvider::new("lmstudio", &server.uri(), None, AuthStyle::None) + .with_local_provider_kind(LocalProviderKind::LmStudio); + // Native probe fails → fall back to the LM Studio profile default (8192), + // so unknown local models still get trimmed instead of skipped. + assert_eq!( + p.effective_context_window("unknown-local-model").await, + Some(8_192) + ); +} diff --git a/src/openhuman/inference/provider/traits.rs b/src/openhuman/inference/provider/traits.rs index 5af662d74..90954a0f2 100644 --- a/src/openhuman/inference/provider/traits.rs +++ b/src/openhuman/inference/provider/traits.rs @@ -483,6 +483,21 @@ pub trait Provider: Send + Sync { self.capabilities().vision } + /// Effective context window (in tokens) for `model`, used for + /// pre-dispatch history trimming. + /// + /// Defaults to the static model table + /// ([`crate::openhuman::inference::context_window_for_model`]), which + /// reflects a model's *trained maximum* context. Local providers + /// override this to report the model's **runtime-loaded** window — e.g. + /// LM Studio lets the user load a model with a smaller `n_ctx` than its + /// trained maximum, and budgeting against the max overflows the loaded + /// window so the request is rejected (issue #3550 / Sentry + /// TAURI-RUST-6V0). `None` means "unknown — skip pre-dispatch trimming". + async fn effective_context_window(&self, model: &str) -> Option { + crate::openhuman::inference::context_window_for_model(model) + } + /// Warm up the HTTP connection pool (TLS handshake, DNS, HTTP/2 setup). /// Default implementation is a no-op; providers with HTTP clients should override. async fn warmup(&self) -> anyhow::Result<()> {