diff --git a/src/openhuman/about_app/catalog_data.rs b/src/openhuman/about_app/catalog_data.rs index f1d1dd066..9c116e9ba 100644 --- a/src/openhuman/about_app/catalog_data.rs +++ b/src/openhuman/about_app/catalog_data.rs @@ -712,8 +712,8 @@ pub(super) const CAPABILITIES: &[Capability] = &[ name: "Configure Local Provider", domain: "local_ai", category: CapabilityCategory::LocalAI, - description: "Select Ollama or LM Studio as the local model provider and configure the local server endpoint.", - how_to: "Settings > AI > providers, or Settings > Local AI Model > Ollama server URL", + description: "Select Ollama, LM Studio, MLX, or a generic local OpenAI-compatible server as the local model provider and configure the endpoint.", + how_to: "Settings > AI > providers, or use provider strings: ollama:, lmstudio:, mlx:, local-openai:", status: CapabilityStatus::Beta, privacy: None, }, diff --git a/src/openhuman/config/schema/local_ai.rs b/src/openhuman/config/schema/local_ai.rs index 4b039f48d..496e7ecd1 100644 --- a/src/openhuman/config/schema/local_ai.rs +++ b/src/openhuman/config/schema/local_ai.rs @@ -118,6 +118,12 @@ pub struct LocalAiConfig { /// local LLM to fix grammar/punctuation using conversation context. #[serde(default = "default_voice_llm_cleanup_enabled")] pub voice_llm_cleanup_enabled: bool, + /// Ollama `options.num_ctx` override. When set, every chat request to + /// an Ollama provider includes `"options": {"num_ctx": }` so + /// the model allocates at least this much KV-cache. Ollama defaults + /// to 2048 for many models which is too small for agentic use. + #[serde(default)] + pub num_ctx: Option, /// Per-feature flags. Each gate is AND-ed with `runtime_enabled`. /// All default to `false` (cloud path). #[serde(default)] @@ -290,6 +296,7 @@ impl Default for LocalAiConfig { ollama_binary_path: None, whisper_in_process: default_whisper_in_process(), voice_llm_cleanup_enabled: default_voice_llm_cleanup_enabled(), + num_ctx: None, usage: LocalAiUsage::default(), } } diff --git a/src/openhuman/inference/local/mod.rs b/src/openhuman/inference/local/mod.rs index d4913e0ad..32296a8fd 100644 --- a/src/openhuman/inference/local/mod.rs +++ b/src/openhuman/inference/local/mod.rs @@ -36,6 +36,7 @@ pub(crate) mod lm_studio; pub(crate) mod model_requirements; mod ollama; mod process_util; +pub mod profile; pub(crate) mod provider; pub(crate) use model_requirements::{evaluate_context, ContextEligibility, MIN_CONTEXT_TOKENS}; pub(crate) use ollama::{ diff --git a/src/openhuman/inference/local/profile.rs b/src/openhuman/inference/local/profile.rs new file mode 100644 index 000000000..26044b094 --- /dev/null +++ b/src/openhuman/inference/local/profile.rs @@ -0,0 +1,299 @@ +//! Local provider profiles — capability metadata for local inference runtimes. +//! +//! Instead of treating all local OpenAI-compatible providers identically, each +//! provider type (Ollama, LM Studio, MLX-compatible, generic local OpenAI) gets +//! a profile that declares its capabilities, quirks, and default context window. +//! The factory and agent harness consult these profiles for: +//! +//! - Tool dispatch strategy (native vs prompt-guided) +//! - Context window defaults for unknown model names +//! - Request body extras (`options.num_ctx`, `think` field suppression) +//! - Temperature handling + +use serde::{Deserialize, Serialize}; + +/// Identifies a local provider type. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum LocalProviderKind { + Ollama, + LmStudio, + /// MLX-compatible local server (e.g. `mlx_lm.server`). + Mlx, + /// Generic local OpenAI-compatible endpoint (llama.cpp, vLLM, etc.). + LocalOpenai, +} + +impl LocalProviderKind { + pub fn as_str(self) -> &'static str { + match self { + Self::Ollama => "ollama", + Self::LmStudio => "lmstudio", + Self::Mlx => "mlx", + Self::LocalOpenai => "local-openai", + } + } + + pub fn display_name(self) -> &'static str { + match self { + Self::Ollama => "Ollama", + Self::LmStudio => "LM Studio", + Self::Mlx => "MLX", + Self::LocalOpenai => "Local OpenAI", + } + } + + /// Parse a provider kind from a string, accepting common aliases. + pub fn from_str_loose(s: &str) -> Option { + match s.trim().to_ascii_lowercase().as_str() { + "ollama" => Some(Self::Ollama), + "lmstudio" | "lm-studio" | "lm_studio" => Some(Self::LmStudio), + "mlx" | "mlx-server" | "mlx_lm" => Some(Self::Mlx), + "local-openai" | "local_openai" | "llamacpp" | "llama.cpp" | "vllm" => { + Some(Self::LocalOpenai) + } + _ => None, + } + } +} + +/// How the provider handles tool calling. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ToolSupport { + /// Provider reliably supports native OpenAI-style tool calling. + Native, + /// Provider does NOT support native tools — use prompt-guided dispatch. + PromptGuided, + /// Support depends on the specific model; probe or consult model profile. + ModelDependent, +} + +/// Extra request body options for a local provider. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct RequestQuirks { + /// Ollama `options.num_ctx` override. When set, injected into the + /// request body as `{"options": {"num_ctx": }}`. + pub num_ctx: Option, + /// When true, suppress reasoning/thinking fields in the request. + /// Some Ollama models reject requests containing `think` parameters. + pub suppress_thinking: bool, + /// When true, omit the `temperature` field entirely (model uses its + /// own default). Distinct from `temperature_unsupported_models` which + /// is pattern-based — this is a blanket provider-level override. + pub omit_temperature: bool, + /// When true, merge system messages into user messages (provider + /// rejects `role: system`). + pub merge_system_into_user: bool, +} + +/// Static capability profile for a local provider type. +#[derive(Debug, Clone)] +pub struct LocalProviderProfile { + pub kind: LocalProviderKind, + /// Default tool support level for this provider type. + pub tool_support: ToolSupport, + /// Default context window (tokens) when the model name is not + /// recognized by `context_window_for_model`. `None` means "unknown, + /// skip preflight trimming". + pub default_context_window: Option, + /// Whether the provider supports the Responses API (`/v1/responses`). + pub supports_responses_api: bool, + /// Whether the provider supports SSE streaming. + pub supports_streaming: bool, + /// Default request quirks for this provider type. + pub default_quirks: RequestQuirks, + /// Default base URL when none is configured. + pub default_base_url: &'static str, + /// Environment variable name for base URL override. + pub base_url_env: &'static str, +} + +/// Ollama profile: conservative defaults, no native tools. +pub const OLLAMA_PROFILE: LocalProviderProfile = LocalProviderProfile { + kind: LocalProviderKind::Ollama, + tool_support: ToolSupport::PromptGuided, + default_context_window: Some(8_192), + supports_responses_api: false, + supports_streaming: true, + default_quirks: RequestQuirks { + num_ctx: None, + suppress_thinking: false, + omit_temperature: false, + merge_system_into_user: false, + }, + default_base_url: "http://127.0.0.1:11434", + base_url_env: "OLLAMA_HOST", +}; + +/// LM Studio profile: conservative defaults, no native tools. +pub const LM_STUDIO_PROFILE: LocalProviderProfile = LocalProviderProfile { + kind: LocalProviderKind::LmStudio, + tool_support: ToolSupport::PromptGuided, + default_context_window: Some(8_192), + supports_responses_api: false, + supports_streaming: true, + default_quirks: RequestQuirks { + num_ctx: None, + suppress_thinking: false, + omit_temperature: false, + merge_system_into_user: false, + }, + default_base_url: "http://127.0.0.1:1234/v1", + base_url_env: "LM_STUDIO_URL", +}; + +/// MLX-compatible server profile (mlx_lm.server, etc.). +pub const MLX_PROFILE: LocalProviderProfile = LocalProviderProfile { + kind: LocalProviderKind::Mlx, + tool_support: ToolSupport::PromptGuided, + default_context_window: Some(4_096), + supports_responses_api: false, + supports_streaming: true, + default_quirks: RequestQuirks { + num_ctx: None, + suppress_thinking: false, + omit_temperature: false, + merge_system_into_user: false, + }, + default_base_url: "http://127.0.0.1:8080/v1", + base_url_env: "MLX_SERVER_URL", +}; + +/// Generic local OpenAI-compatible server (llama.cpp, vLLM, etc.). +pub const LOCAL_OPENAI_PROFILE: LocalProviderProfile = LocalProviderProfile { + kind: LocalProviderKind::LocalOpenai, + tool_support: ToolSupport::PromptGuided, + default_context_window: None, + supports_responses_api: false, + supports_streaming: true, + default_quirks: RequestQuirks { + num_ctx: None, + suppress_thinking: false, + omit_temperature: false, + merge_system_into_user: false, + }, + default_base_url: "http://127.0.0.1:8080/v1", + base_url_env: "LOCAL_OPENAI_URL", +}; + +/// Look up the static profile for a provider kind. +pub fn profile_for_kind(kind: LocalProviderKind) -> &'static LocalProviderProfile { + match kind { + LocalProviderKind::Ollama => &OLLAMA_PROFILE, + LocalProviderKind::LmStudio => &LM_STUDIO_PROFILE, + LocalProviderKind::Mlx => &MLX_PROFILE, + LocalProviderKind::LocalOpenai => &LOCAL_OPENAI_PROFILE, + } +} + +/// Resolve the provider kind from a provider string prefix. +/// +/// Returns `None` for cloud/openhuman/unknown providers. +pub fn kind_from_provider_string(provider: &str) -> Option { + let p = provider.trim().to_ascii_lowercase(); + if p.starts_with("ollama:") || p == "ollama" { + Some(LocalProviderKind::Ollama) + } else if p.starts_with("lmstudio:") + || p.starts_with("lm-studio:") + || p.starts_with("lm_studio:") + { + Some(LocalProviderKind::LmStudio) + } else if p.starts_with("mlx:") { + Some(LocalProviderKind::Mlx) + } else if p.starts_with("local-openai:") || p.starts_with("local_openai:") { + Some(LocalProviderKind::LocalOpenai) + } else { + None + } +} + +/// Returns `true` when the provider string resolves to any local provider. +pub fn is_local_provider_string(provider: &str) -> bool { + kind_from_provider_string(provider).is_some() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn kind_from_str_loose_accepts_aliases() { + assert_eq!( + LocalProviderKind::from_str_loose("ollama"), + Some(LocalProviderKind::Ollama) + ); + assert_eq!( + LocalProviderKind::from_str_loose("LM-Studio"), + Some(LocalProviderKind::LmStudio) + ); + assert_eq!( + LocalProviderKind::from_str_loose("lm_studio"), + Some(LocalProviderKind::LmStudio) + ); + assert_eq!( + LocalProviderKind::from_str_loose("mlx"), + Some(LocalProviderKind::Mlx) + ); + assert_eq!( + LocalProviderKind::from_str_loose("mlx-server"), + Some(LocalProviderKind::Mlx) + ); + assert_eq!( + LocalProviderKind::from_str_loose("llamacpp"), + Some(LocalProviderKind::LocalOpenai) + ); + assert_eq!( + LocalProviderKind::from_str_loose("vllm"), + Some(LocalProviderKind::LocalOpenai) + ); + assert_eq!(LocalProviderKind::from_str_loose("unknown"), None); + } + + #[test] + fn kind_from_provider_string_parses_prefixes() { + assert_eq!( + kind_from_provider_string("ollama:qwen3:14b"), + Some(LocalProviderKind::Ollama) + ); + assert_eq!( + kind_from_provider_string("lmstudio:mistral"), + Some(LocalProviderKind::LmStudio) + ); + assert_eq!( + kind_from_provider_string("mlx:llama-3.1-8b"), + Some(LocalProviderKind::Mlx) + ); + assert_eq!( + kind_from_provider_string("local-openai:qwen2"), + Some(LocalProviderKind::LocalOpenai) + ); + assert_eq!(kind_from_provider_string("openai:gpt-4o"), None); + assert_eq!(kind_from_provider_string("openhuman"), None); + } + + #[test] + fn is_local_identifies_local_strings() { + assert!(is_local_provider_string("ollama:phi3")); + assert!(is_local_provider_string("mlx:model")); + assert!(!is_local_provider_string("openai:gpt-4")); + assert!(!is_local_provider_string("openhuman")); + } + + #[test] + fn profiles_have_correct_defaults() { + let ollama = profile_for_kind(LocalProviderKind::Ollama); + assert_eq!(ollama.tool_support, ToolSupport::PromptGuided); + assert_eq!(ollama.default_context_window, Some(8_192)); + assert!(!ollama.supports_responses_api); + + let mlx = profile_for_kind(LocalProviderKind::Mlx); + assert_eq!(mlx.default_context_window, Some(4_096)); + } + + #[test] + fn ollama_profile_is_conservative_on_tools() { + let profile = profile_for_kind(LocalProviderKind::Ollama); + assert_eq!(profile.tool_support, ToolSupport::PromptGuided); + } +} diff --git a/src/openhuman/inference/model_context.rs b/src/openhuman/inference/model_context.rs index 5b6386284..6642ea9b1 100644 --- a/src/openhuman/inference/model_context.rs +++ b/src/openhuman/inference/model_context.rs @@ -107,9 +107,73 @@ fn tier_context_window(model: &str) -> Option { } } +/// Resolve context window with local provider profile fallback. +/// +/// When `context_window_for_model` returns `None` (unknown model name — +/// common for local models like `qwen3:14b`, `phi3:mini`, etc.) this +/// function falls back to the provider profile's declared default context +/// window. This ensures preflight trimming still works for local models +/// even when the exact model name isn't in the static pattern table. +pub fn context_window_for_model_with_local_fallback( + model: &str, + local_kind: Option, +) -> Option { + if let Some(window) = context_window_for_model(model) { + return Some(window); + } + // Fall back to the local provider profile's default context window. + if let Some(kind) = local_kind { + let profile = crate::openhuman::inference::local::profile::profile_for_kind(kind); + if let Some(default_ctx) = profile.default_context_window { + tracing::debug!( + model, + provider = kind.as_str(), + context_window = default_ctx, + "[model_context] using local provider profile default context window" + ); + return Some(default_ctx); + } + } + None +} + #[cfg(test)] mod tests { use super::*; + use crate::openhuman::inference::local::profile::LocalProviderKind; + + #[test] + fn local_fallback_uses_profile_default() { + // Unknown model with Ollama profile → 8192 default + assert_eq!( + context_window_for_model_with_local_fallback( + "qwen3:14b", + Some(LocalProviderKind::Ollama) + ), + Some(8_192) + ); + // Unknown model with MLX profile → 4096 default + assert_eq!( + context_window_for_model_with_local_fallback( + "my-custom-model", + Some(LocalProviderKind::Mlx) + ), + Some(4_096) + ); + // Unknown model with no local provider → None + assert_eq!( + context_window_for_model_with_local_fallback("qwen3:14b", None), + None + ); + // Known model ignores local fallback + assert_eq!( + context_window_for_model_with_local_fallback( + "llama3:8b", + Some(LocalProviderKind::Ollama) + ), + Some(128_000) + ); + } #[test] fn tier_aliases_resolve() { diff --git a/src/openhuman/inference/provider/compatible.rs b/src/openhuman/inference/provider/compatible.rs index a82fa972f..6e38ea151 100644 --- a/src/openhuman/inference/provider/compatible.rs +++ b/src/openhuman/inference/provider/compatible.rs @@ -88,6 +88,14 @@ pub struct OpenAiCompatibleProvider { /// HTTP 400 on `tools` for many models — making prompt-guided text /// tool specs the only path that works across the Ollama model zoo. native_tool_calling: bool, + /// Ollama-specific `options.num_ctx` override. When set, every request + /// to this provider includes `"options": {"num_ctx": }` in the + /// body so Ollama allocates the requested KV-cache size. + pub(crate) ollama_num_ctx: Option, + /// The local provider kind, if this is a local provider. + /// Used for profile-aware context window resolution and diagnostics. + pub(crate) local_provider_kind: + Option, } /// How the provider expects the API key to be sent. @@ -241,6 +249,8 @@ impl OpenAiCompatibleProvider { temperature_unsupported_models: Vec::new(), temperature_override: None, native_tool_calling: true, + ollama_num_ctx: None, + local_provider_kind: None, } } @@ -269,6 +279,23 @@ impl OpenAiCompatibleProvider { self } + /// Set the Ollama `options.num_ctx` override. When set, the provider + /// includes `"options": {"num_ctx": }` in every request body. + pub fn with_ollama_num_ctx(mut self, num_ctx: Option) -> Self { + self.ollama_num_ctx = num_ctx; + self + } + + /// Tag this provider with its local provider kind for profile-aware + /// context window resolution and diagnostics. + pub fn with_local_provider_kind( + mut self, + kind: crate::openhuman::inference::local::profile::LocalProviderKind, + ) -> Self { + self.local_provider_kind = Some(kind); + self + } + pub fn with_extra_header(mut self, name: impl Into, value: impl Into) -> Self { let name = name.into(); let value = value.into(); @@ -1832,6 +1859,7 @@ impl Provider for OpenAiCompatibleProvider { stream_options: Some(OpenAiStreamOptions { include_usage: true, }), + options: self.build_ollama_options(), }; let stream_dump_seq = reserve_dump_seq(); dump_prompt_if_enabled(&self.name, model, stream_dump_seq, &native_request); @@ -1902,6 +1930,7 @@ impl Provider for OpenAiCompatibleProvider { tools, thread_id, stream_options: None, + options: self.build_ollama_options(), }; let dump_seq = reserve_dump_seq(); dump_prompt_if_enabled(&self.name, model, dump_seq, &native_request); diff --git a/src/openhuman/inference/provider/compatible_request.rs b/src/openhuman/inference/provider/compatible_request.rs index ed25fd9fb..7402db743 100644 --- a/src/openhuman/inference/provider/compatible_request.rs +++ b/src/openhuman/inference/provider/compatible_request.rs @@ -9,6 +9,15 @@ use crate::openhuman::inference::provider::{temperature, thread_context}; use super::{AuthStyle, OpenAiCompatibleProvider}; impl OpenAiCompatibleProvider { + /// Build the Ollama-specific `options` block for the request body. + /// Returns `None` when no `num_ctx` override is configured. + pub(super) fn build_ollama_options(&self) -> Option { + self.ollama_num_ctx + .map(|num_ctx| super::compatible_types::OllamaOptions { + num_ctx: Some(num_ctx), + }) + } + /// Resolve the effective temperature for `model`. Returns `None` when the /// model matches a pattern in `temperature_unsupported_models` (causing the /// field to be omitted from the serialised request). Otherwise yields the diff --git a/src/openhuman/inference/provider/compatible_tests.rs b/src/openhuman/inference/provider/compatible_tests.rs index a02976a93..1b0d93950 100644 --- a/src/openhuman/inference/provider/compatible_tests.rs +++ b/src/openhuman/inference/provider/compatible_tests.rs @@ -65,6 +65,7 @@ fn native_request_emits_thread_id_when_present() { tool_choice: None, thread_id: Some("thread-abc".to_string()), stream_options: None, + options: None, }; let json = serde_json::to_value(&req).unwrap(); assert_eq!( @@ -82,6 +83,7 @@ fn native_request_emits_thread_id_when_present() { tool_choice: None, thread_id: None, stream_options: None, + options: None, }; let json_no_thread = serde_json::to_value(&req_no_thread).unwrap(); assert!( @@ -108,6 +110,7 @@ fn streaming_request_sets_stream_options_include_usage() { stream_options: Some(super::compatible_types::OpenAiStreamOptions { include_usage: true, }), + options: None, }; let json = serde_json::to_value(&req).unwrap(); assert_eq!( @@ -129,6 +132,7 @@ fn non_streaming_request_omits_stream_options() { tool_choice: None, thread_id: None, stream_options: None, + options: None, }; let json = serde_json::to_value(&req).unwrap(); assert!( @@ -137,6 +141,49 @@ fn non_streaming_request_omits_stream_options() { ); } +#[test] +fn ollama_options_num_ctx_serializes_correctly() { + let req = super::NativeChatRequest { + model: "qwen3:14b".to_string(), + messages: Vec::new(), + temperature: Some(0.7), + stream: Some(false), + tools: None, + tool_choice: None, + thread_id: None, + stream_options: None, + options: Some(super::compatible_types::OllamaOptions { + num_ctx: Some(32768), + }), + }; + let json = serde_json::to_value(&req).unwrap(); + assert_eq!( + json.pointer("/options/num_ctx").and_then(|v| v.as_u64()), + Some(32768), + "Ollama num_ctx must appear at options.num_ctx in serialized body" + ); +} + +#[test] +fn ollama_options_none_is_omitted() { + let req = super::NativeChatRequest { + model: "gpt-4o".to_string(), + messages: Vec::new(), + temperature: Some(0.7), + stream: Some(false), + tools: None, + tool_choice: None, + thread_id: None, + stream_options: None, + options: None, + }; + let json = serde_json::to_value(&req).unwrap(); + assert!( + json.get("options").is_none(), + "options field must be omitted when None (non-Ollama providers)" + ); +} + #[tokio::test] async fn outbound_thread_id_is_gated_per_provider() { use crate::openhuman::inference::provider::thread_context::with_thread_id; @@ -544,6 +591,7 @@ async fn streaming_chat_config_rejection_propagates_error_without_sentry_report( stream_options: Some(super::compatible_types::OpenAiStreamOptions { include_usage: true, }), + options: None, }; let (delta_tx, _delta_rx) = tokio::sync::mpsc::channel(8); diff --git a/src/openhuman/inference/provider/compatible_types.rs b/src/openhuman/inference/provider/compatible_types.rs index 2f23ebe11..892a30ee3 100644 --- a/src/openhuman/inference/provider/compatible_types.rs +++ b/src/openhuman/inference/provider/compatible_types.rs @@ -58,6 +58,20 @@ pub(crate) struct NativeChatRequest { /// streamed sessions (typically the orchestrator). #[serde(skip_serializing_if = "Option::is_none")] pub(crate) stream_options: Option, + /// Ollama-specific `options` block (e.g. `{"num_ctx": 32768}`). + /// Injected by the factory when the provider profile declares a + /// `num_ctx` override. Ignored (skipped) for non-Ollama providers. + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) options: Option, +} + +/// Ollama-specific request options passed in the `options` field. +#[derive(Debug, Clone, Serialize)] +pub(crate) struct OllamaOptions { + /// Context window size override. Ollama defaults to 2048 for many + /// models; setting this ensures the model allocates enough KV-cache. + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) num_ctx: Option, } /// OpenAI-spec `stream_options` payload (sent on the wire). Distinct from diff --git a/src/openhuman/inference/provider/factory.rs b/src/openhuman/inference/provider/factory.rs index d01377418..f6b3135ba 100644 --- a/src/openhuman/inference/provider/factory.rs +++ b/src/openhuman/inference/provider/factory.rs @@ -11,6 +11,9 @@ //! "cloud" / missing → primary_cloud; legacy custom inference_url wins when //! primary still points at OpenHuman after migration //! "ollama:[@]" → local Ollama at config.local_ai.base_url +//! "lmstudio:[@]" → local LM Studio +//! "mlx:[@]" → local MLX-compatible server +//! "local-openai:[@]"→ generic local OpenAI-compatible //! ":[@]" → cloud_providers entry keyed by slug; //! builds OpenAiCompatibleProvider (Bearer) or //! Anthropic flavour depending on auth_style. @@ -42,6 +45,10 @@ pub const PROVIDER_OPENHUMAN: &str = "openhuman"; pub const OLLAMA_PROVIDER_PREFIX: &str = "ollama:"; /// Prefix for LM Studio-local providers: `"lmstudio:"`. pub const LM_STUDIO_PROVIDER_PREFIX: &str = "lmstudio:"; +/// Prefix for MLX-compatible local providers: `"mlx:"`. +pub const MLX_PROVIDER_PREFIX: &str = "mlx:"; +/// Prefix for generic local OpenAI-compatible providers: `"local-openai:"`. +pub const LOCAL_OPENAI_PROVIDER_PREFIX: &str = "local-openai:"; /// Prefix for the Claude Agent SDK subprocess provider: `"claude_agent_sdk:"`. pub const CLAUDE_AGENT_SDK_PREFIX: &str = "claude_agent_sdk:"; /// Sentinel for the Claude Agent SDK provider without a model suffix. @@ -158,6 +165,25 @@ pub fn provider_for_role(role: &str, config: &Config) -> String { return byok; } } + + // Diagnostic: when the user has a local provider configured for chat + // but this background workload is falling through to cloud, emit a + // warning so it's visible in logs (no silent fallback). + if !matches!(role, "chat" | "reasoning" | "coding") { + if let Some(chat) = config.chat_provider.as_deref() { + if crate::openhuman::inference::local::profile::is_local_provider_string(chat) { + log::info!( + "[providers][local-fallback] role={} using managed backend (chat is \ + local '{}' but background workloads require cloud — set \ + {}_provider explicitly to override)", + role, + chat, + role + ); + } + } + } + resolve_primary_cloud_provider_string(config) } else { s.to_string() @@ -187,7 +213,11 @@ pub(crate) fn resolve_byok_fallback_provider_string(config: &Config) -> Option'", + p, + role + ); + } + return make_mlx_provider(&model, temperature_override, config); + } + + if let Some(model_with_temp) = p.strip_prefix(LOCAL_OPENAI_PROVIDER_PREFIX) { + let (model, temperature_override) = split_model_and_temperature(model_with_temp); + if model.is_empty() { + anyhow::bail!( + "[chat-factory] provider string '{}' for role '{}' has an empty model — \ + use 'local-openai:'", + p, + role + ); + } + return make_local_openai_provider(&model, temperature_override, config); + } + if p == CLAUDE_AGENT_SDK_PROVIDER || p.starts_with(CLAUDE_AGENT_SDK_PREFIX) { let model = if let Some(m) = p.strip_prefix(CLAUDE_AGENT_SDK_PREFIX) { m.trim().to_string() @@ -376,8 +432,8 @@ pub fn create_chat_provider_from_string( // than an opaque parse failure. anyhow::bail!( "[chat-factory] unrecognised provider string '{}' for role '{}'. \ - Valid forms: openhuman, ollama:, lmstudio:, claude_agent_sdk, \ - claude_agent_sdk:, :. \ + Valid forms: openhuman, ollama:, lmstudio:, mlx:, \ + local-openai:, claude_agent_sdk, claude_agent_sdk:, :. \ Configured slugs: [{}]", p, role, @@ -437,8 +493,31 @@ pub(crate) fn create_local_chat_provider_from_string( return make_lm_studio_provider(&model, temperature_override, config); } + if let Some(model_with_temp) = p.strip_prefix(MLX_PROVIDER_PREFIX) { + let (model, temperature_override) = split_model_and_temperature(model_with_temp); + if model.is_empty() { + anyhow::bail!( + "[chat-factory] provider string '{}' has an empty model — use 'mlx:'", + p + ); + } + return make_mlx_provider(&model, temperature_override, config); + } + + if let Some(model_with_temp) = p.strip_prefix(LOCAL_OPENAI_PROVIDER_PREFIX) { + let (model, temperature_override) = split_model_and_temperature(model_with_temp); + if model.is_empty() { + anyhow::bail!( + "[chat-factory] provider string '{}' has an empty model — use 'local-openai:'", + p + ); + } + return make_local_openai_provider(&model, temperature_override, config); + } + anyhow::bail!( - "[chat-factory] '{}' is not a supported local provider string. Valid local forms: ollama:, lmstudio:", + "[chat-factory] '{}' is not a supported local provider string. Valid local forms: \ + ollama:, lmstudio:, mlx:, local-openai:", p ); } @@ -759,15 +838,20 @@ fn make_ollama_provider( temperature_override: Option, config: &Config, ) -> anyhow::Result<(Box, String)> { + use crate::openhuman::inference::local::profile::LocalProviderKind; + let base_url = crate::openhuman::inference::local::ollama_base_url_from_config(config); let normalized_base_url = base_url.trim_end_matches('/').trim_end_matches("/v1"); // Ollama exposes an OpenAI-compatible endpoint at /v1. let endpoint = format!("{normalized_base_url}/v1"); + let num_ctx = config.local_ai.num_ctx; log::info!( - "[providers][chat-factory] building ollama provider model={} endpoint_host={} temp_override={:?}", + "[providers][chat-factory] building ollama provider model={} endpoint_host={} \ + temp_override={:?} num_ctx={:?}", model, redact_endpoint(&endpoint), - temperature_override + temperature_override, + num_ctx, ); // Ollama does not expose the Responses API (/v1/responses) — passing // `false` prevents a guaranteed-404 fallback attempt and the Sentry @@ -788,7 +872,9 @@ fn make_ollama_provider( ) .with_temperature_unsupported_models(config.temperature_unsupported_models.clone()) .with_temperature_override(temperature_override) - .with_native_tool_calling(false); + .with_native_tool_calling(false) + .with_ollama_num_ctx(num_ctx) + .with_local_provider_kind(LocalProviderKind::Ollama); Ok((Box::new(provider), model.to_string())) } @@ -798,6 +884,8 @@ fn make_lm_studio_provider( temperature_override: Option, config: &Config, ) -> anyhow::Result<(Box, String)> { + use crate::openhuman::inference::local::profile::LocalProviderKind; + let endpoint = crate::openhuman::inference::local::lm_studio::lm_studio_base_url(config); let api_key = config.local_ai.api_key.as_deref().unwrap_or(""); log::info!( @@ -807,20 +895,109 @@ fn make_lm_studio_provider( temperature_override ); // LM Studio does not expose the Responses API — same rationale as Ollama. - let p = make_openai_compatible_provider_with_config( + let auth = if api_key.trim().is_empty() { + CompatAuthStyle::None + } else { + CompatAuthStyle::Bearer + }; + let provider = OpenAiCompatibleProvider::new_no_responses_fallback( "lmstudio", &endpoint, - api_key, if api_key.trim().is_empty() { - CompatAuthStyle::None + None } else { - CompatAuthStyle::Bearer + Some(api_key) }, - &config.temperature_unsupported_models, - temperature_override, - false, - )?; - Ok((p, model.to_string())) + auth, + ) + .with_temperature_unsupported_models(config.temperature_unsupported_models.clone()) + .with_temperature_override(temperature_override) + .with_native_tool_calling(false) + .with_local_provider_kind(LocalProviderKind::LmStudio); + Ok((Box::new(provider), model.to_string())) +} + +/// Build an MLX-compatible local provider. +/// +/// MLX servers (e.g. `mlx_lm.server`) expose an OpenAI-compatible endpoint. +/// Default URL: `http://127.0.0.1:8080/v1` (override via `MLX_SERVER_URL` env +/// or `local_ai.base_url` when provider is set to "mlx"). +fn make_mlx_provider( + model: &str, + temperature_override: Option, + config: &Config, +) -> anyhow::Result<(Box, String)> { + use crate::openhuman::inference::local::profile::{LocalProviderKind, MLX_PROFILE}; + + let endpoint = std::env::var("MLX_SERVER_URL") + .ok() + .filter(|s| !s.trim().is_empty()) + .or_else(|| config.local_ai.base_url.clone()) + .unwrap_or_else(|| MLX_PROFILE.default_base_url.to_string()); + log::info!( + "[providers][chat-factory] building mlx provider model={} endpoint_host={} temp_override={:?}", + model, + redact_endpoint(&endpoint), + temperature_override + ); + let provider = OpenAiCompatibleProvider::new_no_responses_fallback( + "mlx", + &endpoint, + None, + CompatAuthStyle::None, + ) + .with_temperature_unsupported_models(config.temperature_unsupported_models.clone()) + .with_temperature_override(temperature_override) + .with_native_tool_calling(false) + .with_local_provider_kind(LocalProviderKind::Mlx); + Ok((Box::new(provider), model.to_string())) +} + +/// Build a generic local OpenAI-compatible provider. +/// +/// Points at any local server that speaks the OpenAI chat-completions API +/// (llama.cpp, vLLM, text-generation-inference, etc.). +/// Default URL: `http://127.0.0.1:8080/v1` (override via `LOCAL_OPENAI_URL` +/// env or `local_ai.base_url`). +fn make_local_openai_provider( + model: &str, + temperature_override: Option, + config: &Config, +) -> anyhow::Result<(Box, String)> { + use crate::openhuman::inference::local::profile::{LocalProviderKind, LOCAL_OPENAI_PROFILE}; + + let endpoint = std::env::var("LOCAL_OPENAI_URL") + .ok() + .filter(|s| !s.trim().is_empty()) + .or_else(|| config.local_ai.base_url.clone()) + .unwrap_or_else(|| LOCAL_OPENAI_PROFILE.default_base_url.to_string()); + let api_key = config.local_ai.api_key.as_deref().unwrap_or(""); + log::info!( + "[providers][chat-factory] building local-openai provider model={} endpoint_host={} temp_override={:?}", + model, + redact_endpoint(&endpoint), + temperature_override + ); + let auth = if api_key.trim().is_empty() { + CompatAuthStyle::None + } else { + CompatAuthStyle::Bearer + }; + let provider = OpenAiCompatibleProvider::new_no_responses_fallback( + "local-openai", + &endpoint, + if api_key.trim().is_empty() { + None + } else { + Some(api_key) + }, + auth, + ) + .with_temperature_unsupported_models(config.temperature_unsupported_models.clone()) + .with_temperature_override(temperature_override) + .with_native_tool_calling(false) + .with_local_provider_kind(LocalProviderKind::LocalOpenai); + Ok((Box::new(provider), model.to_string())) } /// Look up a `cloud_providers` entry by slug and build the provider. diff --git a/src/openhuman/inference/provider/factory_tests.rs b/src/openhuman/inference/provider/factory_tests.rs index 25931efad..85a7a9434 100644 --- a/src/openhuman/inference/provider/factory_tests.rs +++ b/src/openhuman/inference/provider/factory_tests.rs @@ -230,18 +230,20 @@ fn ollama_provider_opts_out_of_native_tool_calling() { } #[test] -fn lmstudio_provider_keeps_native_tool_calling_enabled() { - // LM Studio's OpenAI-compat endpoint supports the `tools` parameter for - // models that expose function calling. Only Ollama gets opted out by - // default — the LM Studio path stays on the native schema. +fn lmstudio_provider_defaults_to_prompt_guided_tools() { + // All local providers (Ollama, LM Studio, MLX, local-openai) default to + // prompt-guided tool dispatch (#3246). This prevents HTTP 400 errors + // from models that don't support the native `tools` parameter. Users + // can override via `config.agent.tool_dispatcher = "native"` if their + // model supports it. let mut config = Config::default(); config.local_ai.base_url = Some("http://127.0.0.1:1234".to_string()); let (provider, _model) = create_chat_provider_from_string("chat", "lmstudio:google/gemma-4-e4b", &config) .expect("lmstudio: must build"); assert!( - provider.capabilities().native_tool_calling, - "lmstudio provider must keep native_tool_calling=true; only the ollama branch opts out" + !provider.capabilities().native_tool_calling, + "lmstudio provider must default to native_tool_calling=false (conservative local dispatch)" ); } @@ -1686,3 +1688,77 @@ fn config_api_key_fallback_inert_without_inference_url() { "without inference_url there is no legacy slug — fallback must stay inert", ); } + +// ── Local provider profile tests ───────────────────────────────────────────── + +#[test] +fn mlx_provider_string_resolves() { + let config = Config::default(); + let result = create_chat_provider_from_string("chat", "mlx:llama-3.1-8b", &config); + assert!(result.is_ok(), "mlx provider must resolve"); + let (_, model) = result.unwrap(); + assert_eq!(model, "llama-3.1-8b"); +} + +#[test] +fn local_openai_provider_string_resolves() { + let config = Config::default(); + let result = create_chat_provider_from_string("chat", "local-openai:phi3", &config); + assert!(result.is_ok(), "local-openai provider must resolve"); + let (_, model) = result.unwrap(); + assert_eq!(model, "phi3"); +} + +#[test] +fn mlx_provider_empty_model_errors() { + let config = Config::default(); + let result = create_chat_provider_from_string("chat", "mlx:", &config); + let err = result.err().expect("mlx: with empty model must error"); + assert!(err.to_string().contains("empty model")); +} + +#[test] +fn local_openai_provider_empty_model_errors() { + let config = Config::default(); + let result = create_chat_provider_from_string("chat", "local-openai:", &config); + let err = result + .err() + .expect("local-openai: with empty model must error"); + assert!(err.to_string().contains("empty model")); +} + +#[test] +fn ollama_provider_passes_num_ctx() { + let mut config = Config::default(); + config.local_ai.num_ctx = Some(32768); + let result = create_chat_provider_from_string("chat", "ollama:qwen3:14b", &config); + assert!(result.is_ok()); + // The provider is constructed — num_ctx is set on the provider instance. + // Full integration test verifying the serialized body is in the JSON-RPC + // E2E suite; here we just confirm the factory doesn't reject it. +} + +#[test] +fn byok_fallback_skips_mlx_and_local_openai() { + let mut config = Config::default(); + config.chat_provider = Some("mlx:llama3".to_string()); + config.reasoning_provider = Some("local-openai:phi3".to_string()); + // Neither should be picked up as a BYOK fallback + let result = resolve_byok_fallback_provider_string(&config); + assert!( + result.is_none(), + "local providers must not be BYOK fallbacks" + ); +} + +#[test] +fn local_provider_string_detection() { + use crate::openhuman::inference::local::profile::is_local_provider_string; + assert!(is_local_provider_string("ollama:phi3")); + assert!(is_local_provider_string("lmstudio:model")); + assert!(is_local_provider_string("mlx:llama")); + assert!(is_local_provider_string("local-openai:qwen2")); + assert!(!is_local_provider_string("openai:gpt-4o")); + assert!(!is_local_provider_string("openhuman")); + assert!(!is_local_provider_string("cloud")); +}