From f82a302d386a2bce34b6e4900fbde058f000747b Mon Sep 17 00:00:00 2001 From: Zavian Wang <36817799+Zavianx@users.noreply.github.com> Date: Wed, 20 May 2026 12:10:15 +0800 Subject: [PATCH] feat(mcp): add SearXNG search tool (#1988) Co-authored-by: Steven Enamakel --- .env.example | 14 + docs/TEST-COVERAGE-MATRIX.md | 1 + gitbooks/developing/mcp-server.md | 33 +- scripts/feature-ids.json | 1 + src/openhuman/about_app/catalog.rs | 16 + src/openhuman/about_app/catalog_tests.rs | 1 + .../channels/tests/runtime_dispatch.rs | 68 +-- src/openhuman/composio/action_tool.rs | 50 +- src/openhuman/composio/tools_tests.rs | 82 +-- src/openhuman/config/mod.rs | 2 +- src/openhuman/config/schema/load.rs | 42 ++ src/openhuman/config/schema/load_tests.rs | 84 +++ src/openhuman/config/schema/mod.rs | 4 +- src/openhuman/config/schema/tools.rs | 48 ++ src/openhuman/config/schema/types.rs | 4 + src/openhuman/integrations/mod.rs | 12 +- src/openhuman/integrations/searxng.rs | 574 ++++++++++++++++++ src/openhuman/mcp_server/protocol.rs | 39 +- src/openhuman/mcp_server/tools.rs | 189 +++++- src/openhuman/tools/ops.rs | 21 + src/openhuman/tools/ops_tests.rs | 8 +- src/openhuman/tools/schemas.rs | 168 ++++- 22 files changed, 1313 insertions(+), 148 deletions(-) create mode 100644 src/openhuman/integrations/searxng.rs diff --git a/.env.example b/.env.example index 5fd41ea03..698664938 100644 --- a/.env.example +++ b/.env.example @@ -110,6 +110,20 @@ OPENHUMAN_WEB_SEARCH_TIMEOUT_SECS=10 # [optional] Default max results per query (1-20, default 10) # OPENHUMAN_SELTZ_MAX_RESULTS=10 +# --------------------------------------------------------------------------- +# SearXNG search (self-hosted — https://docs.searxng.org) +# --------------------------------------------------------------------------- +# [optional] Enable the searxng_search agent/MCP tool. Default: false. +# OPENHUMAN_SEARXNG_ENABLED=false +# [optional] Base URL for your SearXNG instance. Default: http://localhost:8080 +# OPENHUMAN_SEARXNG_BASE_URL=http://localhost:8080 +# [optional] Default max results per query (1-50, default 10) +# OPENHUMAN_SEARXNG_MAX_RESULTS=10 +# [optional] Default language when a tool call omits language. Default: en +# OPENHUMAN_SEARXNG_DEFAULT_LANGUAGE=en +# [optional] Request timeout in seconds. Default: 10 +# OPENHUMAN_SEARXNG_TIMEOUT_SECONDS=10 + # --------------------------------------------------------------------------- # Proxy # --------------------------------------------------------------------------- diff --git a/docs/TEST-COVERAGE-MATRIX.md b/docs/TEST-COVERAGE-MATRIX.md index 43bb6cd33..9704f6c85 100644 --- a/docs/TEST-COVERAGE-MATRIX.md +++ b/docs/TEST-COVERAGE-MATRIX.md @@ -398,6 +398,7 @@ Canonical mapping of every product feature to its test source(s). Drives gap-fil | 11.1.3 | Analyze Trigger | WD | `app/test/e2e/specs/insights-dashboard.spec.ts` mounts the route (this PR); explicit analyze-handler invocation TBD | 🟡 | Route mounts and search/filter UI assert — full analyze trigger flow tracked as follow-up | | 11.1.4 | MCP stdio server | RU | `src/openhuman/mcp_server/` | ✅ | Read-only initialize/tools/list/tools/call plus stdio framing; binary smoke in PR validation | | 11.1.5 | Global tool registry | RI | `src/openhuman/tool_registry/`, `tests/json_rpc_e2e.rs` | ✅ | Read-only MCP/controller discovery with routes, schemas, version, allowed agents, and health | +| 11.1.6 | SearXNG MCP search | RU | `src/openhuman/integrations/searxng.rs`, `src/openhuman/mcp_server/tools.rs`, `src/openhuman/tools/schemas.rs` | ✅ | Self-hosted search config, normalized results, MCP argument validation, and mocked HTTP execution | ### 11.2 Insights Dashboard diff --git a/gitbooks/developing/mcp-server.md b/gitbooks/developing/mcp-server.md index c74b8f38c..180c3ad06 100644 --- a/gitbooks/developing/mcp-server.md +++ b/gitbooks/developing/mcp-server.md @@ -23,6 +23,7 @@ controller registry plus the core security policy read gate: | MCP tool | Backing RPC | Purpose | | --- | --- | --- | +| `searxng_search`* | `openhuman.tools_searxng_search` | Search a configured self-hosted SearXNG instance. | | `memory.search` | `openhuman.memory_tree_search` | Keyword search over memory-tree chunks. | | `memory.recall` | `openhuman.memory_tree_recall` | Semantic recall over memory-tree summaries/chunks. | | `tree.read_chunk` | `openhuman.memory_tree_get_chunk` | Read one chunk returned by search or recall. | @@ -30,11 +31,35 @@ controller registry plus the core security policy read gate: | `tree.top_entities` | `openhuman.memory_tree_top_entities` | Most-referenced canonical entities, optionally filtered by kind. | | `tree.list_sources` | `openhuman.memory_tree_list_sources` | Distinct ingest sources with chunk counts and last-activity timestamps. | +* `searxng_search` is present only when SearXNG is enabled. + +`searxng_search` is added to the MCP catalog when SearXNG is enabled. It accepts +`query`, optional `categories` (`web`, `news`, `images`), optional `language`, +and optional `max_results` (1-50). `memory.search` and `memory.recall` accept `query` plus optional `k` (default -10, capped at 50). `tree.read_chunk` accepts `chunk_id`. `tree.browse` -accepts optional `source_kinds`, `source_ids`, `entity_ids`, `since_ms`, -`until_ms`, `query`, `k`, and `offset`. `tree.top_entities` accepts optional -`kind` and `k`. `tree.list_sources` accepts an optional `user_email_hint`. +10, capped at 50). `tree.read_chunk` accepts `chunk_id`. `tree.browse` accepts +optional `source_kinds`, `source_ids`, `entity_ids`, `since_ms`, `until_ms`, +`query`, `k`, and `offset`. `tree.top_entities` accepts optional `kind` and +`k`. `tree.list_sources` accepts an optional `user_email_hint`. + +Enable SearXNG in `config.toml` or via environment: + +```toml +[searxng] +enabled = true +base_url = "http://localhost:8080" +max_results = 10 +default_language = "en" +timeout_seconds = 10 +``` + +```bash +OPENHUMAN_SEARXNG_ENABLED=true +OPENHUMAN_SEARXNG_BASE_URL=http://localhost:8080 +OPENHUMAN_SEARXNG_MAX_RESULTS=10 +OPENHUMAN_SEARXNG_DEFAULT_LANGUAGE=en +OPENHUMAN_SEARXNG_TIMEOUT_SECONDS=10 +``` ## Tool Registry diff --git a/scripts/feature-ids.json b/scripts/feature-ids.json index 7f9d7e257..83f50c859 100644 --- a/scripts/feature-ids.json +++ b/scripts/feature-ids.json @@ -121,6 +121,7 @@ "11.1.3", "11.1.4", "11.1.5", + "11.1.6", "11.2.1", "11.2.2", "11.2.3", diff --git a/src/openhuman/about_app/catalog.rs b/src/openhuman/about_app/catalog.rs index 5e5876b82..1524f9f1c 100644 --- a/src/openhuman/about_app/catalog.rs +++ b/src/openhuman/about_app/catalog.rs @@ -47,6 +47,12 @@ const GITHUB_RELEASES_METADATA: Option = Some(CapabilityPriva destinations: &["GitHub Releases"], }); +const SEARXNG_RAW_TO_CONFIGURED_INSTANCE: Option = Some(CapabilityPrivacy { + leaves_device: true, + data_kind: PrivacyDataKind::Raw, + destinations: &["Configured SearXNG instance"], +}); + // Direct-mode Composio: the user's API key and tool arguments leave the // device — they are sent to backend.composio.dev, not the OpenHuman backend. // LOCAL_CREDENTIALS was incorrect here because leaves_device must be true. @@ -298,6 +304,16 @@ const CAPABILITIES: &[Capability] = &[ status: CapabilityStatus::Beta, privacy: LOCAL_RAW, }, + Capability { + id: "intelligence.searxng_search", + name: "SearXNG Search", + domain: "intelligence", + category: CapabilityCategory::Intelligence, + description: "Search a configured self-hosted SearXNG instance from agent and MCP tools, returning normalized title, URL, snippet, and source results.", + how_to: "Set `[searxng] enabled = true` and `base_url` in config.toml, or use OPENHUMAN_SEARXNG_* environment variables.", + status: CapabilityStatus::Beta, + privacy: SEARXNG_RAW_TO_CONFIGURED_INSTANCE, + }, Capability { id: "intelligence.tool_registry", name: "Tool Registry", diff --git a/src/openhuman/about_app/catalog_tests.rs b/src/openhuman/about_app/catalog_tests.rs index 26532fb82..f8c75f835 100644 --- a/src/openhuman/about_app/catalog_tests.rs +++ b/src/openhuman/about_app/catalog_tests.rs @@ -101,6 +101,7 @@ fn catalog_includes_additional_user_facing_surfaces() { "meet.join_call", "meet_agent.live_loop", "intelligence.mcp_server", + "intelligence.searxng_search", "intelligence.tool_registry", "conversation.subagent_mascots", ] { diff --git a/src/openhuman/channels/tests/runtime_dispatch.rs b/src/openhuman/channels/tests/runtime_dispatch.rs index 07826c2bc..c469525ce 100644 --- a/src/openhuman/channels/tests/runtime_dispatch.rs +++ b/src/openhuman/channels/tests/runtime_dispatch.rs @@ -5,22 +5,33 @@ use super::common::{use_real_agent_handler, NoopMemory, RecordingChannel, SlowPr use crate::openhuman::agent::bus::{mock_agent_run_turn, AgentTurnRequest, AgentTurnResponse}; use crate::openhuman::inference::provider; use std::collections::HashMap; -use std::sync::atomic::Ordering; +use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Arc, Mutex}; -use std::time::{Duration, Instant}; +use std::time::Duration; #[tokio::test] async fn message_dispatch_processes_messages_in_parallel() { - // Install a deterministic stub that takes 250ms per turn. Two messages - // should complete materially faster than a fully sequential path. In - // practice the test harness adds non-trivial fixed overhead (channel - // typing/reply work, bus dispatch, CI scheduling), so the assertion - // targets "well below sequential" instead of the ideal 250ms floor. - let _bus_guard = mock_agent_run_turn(|_req: AgentTurnRequest| async move { - tokio::time::sleep(Duration::from_millis(250)).await; - Ok(AgentTurnResponse { - text: "echo: stub".to_string(), - }) + // Install a deterministic stub that takes 250ms per turn and records + // the peak number of in-flight turns. This proves concurrency directly + // without relying on wall-clock thresholds that can wobble in CI. + let in_flight = Arc::new(AtomicUsize::new(0)); + let peak_in_flight = Arc::new(AtomicUsize::new(0)); + let _bus_guard = mock_agent_run_turn({ + let in_flight = in_flight.clone(); + let peak_in_flight = peak_in_flight.clone(); + move |_req: AgentTurnRequest| { + let in_flight = in_flight.clone(); + let peak_in_flight = peak_in_flight.clone(); + async move { + let current = in_flight.fetch_add(1, Ordering::SeqCst) + 1; + peak_in_flight.fetch_max(current, Ordering::SeqCst); + tokio::time::sleep(Duration::from_millis(250)).await; + in_flight.fetch_sub(1, Ordering::SeqCst); + Ok(AgentTurnResponse { + text: "echo: stub".to_string(), + }) + } + } }) .await; @@ -60,27 +71,6 @@ async fn message_dispatch_processes_messages_in_parallel() { (channel_impl, runtime_ctx) }; - let (baseline_channel, baseline_ctx) = build_runtime(); - let (baseline_tx, baseline_rx) = tokio::sync::mpsc::channel::(2); - baseline_tx - .send(traits::ChannelMessage { - id: "baseline".to_string(), - sender: "alice".to_string(), - reply_target: "alice".to_string(), - content: "hello".to_string(), - channel: "test-channel".to_string(), - timestamp: 1, - thread_ts: None, - }) - .await - .unwrap(); - drop(baseline_tx); - - let baseline_started = Instant::now(); - run_message_dispatch_loop(baseline_rx, baseline_ctx, 1).await; - let baseline_elapsed = baseline_started.elapsed(); - assert_eq!(baseline_channel.sent_messages.lock().await.len(), 1); - let (parallel_channel, parallel_ctx) = build_runtime(); let (tx, rx) = tokio::sync::mpsc::channel::(4); tx.send(traits::ChannelMessage { @@ -107,18 +97,8 @@ async fn message_dispatch_processes_messages_in_parallel() { .unwrap(); drop(tx); - let started = Instant::now(); run_message_dispatch_loop(rx, parallel_ctx, 2).await; - let elapsed = started.elapsed(); - - let allowed_parallel = baseline_elapsed + Duration::from_millis(180); - assert!( - elapsed < allowed_parallel, - "expected parallel dispatch (< {:?}) based on single-message baseline {:?}, got {:?}", - allowed_parallel, - baseline_elapsed, - elapsed - ); + assert_eq!(peak_in_flight.load(Ordering::SeqCst), 2); let sent_messages = parallel_channel.sent_messages.lock().await; assert_eq!(sent_messages.len(), 2); diff --git a/src/openhuman/composio/action_tool.rs b/src/openhuman/composio/action_tool.rs index ffa22fb7e..183a184bb 100644 --- a/src/openhuman/composio/action_tool.rs +++ b/src/openhuman/composio/action_tool.rs @@ -262,6 +262,36 @@ impl Tool for ComposioActionTool { mod tests { use super::*; use crate::openhuman::agent::harness::with_current_sandbox_mode; + use std::path::Path; + + struct WorkspaceEnvGuard { + previous: Option, + } + + impl WorkspaceEnvGuard { + fn set(path: &Path) -> Self { + let previous = std::env::var_os("OPENHUMAN_WORKSPACE"); + Self::set_current(path); + Self { previous } + } + + fn set_current(path: &Path) { + unsafe { + std::env::set_var("OPENHUMAN_WORKSPACE", path); + } + } + } + + impl Drop for WorkspaceEnvGuard { + fn drop(&mut self) { + unsafe { + match self.previous.take() { + Some(value) => std::env::set_var("OPENHUMAN_WORKSPACE", value), + None => std::env::remove_var("OPENHUMAN_WORKSPACE"), + } + } + } + } /// Build a minimal `Arc` with `composio.mode = "backend"` /// (the default). The sandbox gate runs *before* any HTTP call or @@ -352,9 +382,7 @@ mod tests { let _env_guard = TEST_ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner()); let tmp = tempfile::tempdir().expect("tempdir"); - unsafe { - std::env::set_var("OPENHUMAN_WORKSPACE", tmp.path()); - } + let _workspace_guard = WorkspaceEnvGuard::set(tmp.path()); let mut config = Config::default(); config.config_path = tmp.path().join("config.toml"); @@ -373,10 +401,6 @@ mod tests { !msg.contains("strict read-only"), "unset sandbox must never trigger the gate, got: {msg}" ); - - unsafe { - std::env::remove_var("OPENHUMAN_WORKSPACE"); - } } // ── Factory routing (#1710) ────────────────────────────────────── @@ -469,9 +493,7 @@ mod tests { // ── Backend half ──────────────────────────────────────────── let tmp_backend = tempfile::tempdir().expect("tempdir backend"); - unsafe { - std::env::set_var("OPENHUMAN_WORKSPACE", tmp_backend.path()); - } + let _workspace_guard = WorkspaceEnvGuard::set(tmp_backend.path()); let mut backend_config = Config::default(); backend_config.config_path = tmp_backend.path().join("config.toml"); backend_config.workspace_dir = tmp_backend.path().join("workspace"); @@ -496,9 +518,7 @@ mod tests { // ── Direct half ───────────────────────────────────────────── let tmp_direct = tempfile::tempdir().expect("tempdir direct"); - unsafe { - std::env::set_var("OPENHUMAN_WORKSPACE", tmp_direct.path()); - } + WorkspaceEnvGuard::set_current(tmp_direct.path()); let mut direct_config = Config::default(); direct_config.config_path = tmp_direct.path().join("config.toml"); direct_config.workspace_dir = tmp_direct.path().join("workspace"); @@ -528,9 +548,5 @@ mod tests { !direct_msg.contains("no backend session"), "direct-mode tool must not surface backend-session artifacts: {direct_msg}" ); - - unsafe { - std::env::remove_var("OPENHUMAN_WORKSPACE"); - } } } diff --git a/src/openhuman/composio/tools_tests.rs b/src/openhuman/composio/tools_tests.rs index 1b814a943..219709eb5 100644 --- a/src/openhuman/composio/tools_tests.rs +++ b/src/openhuman/composio/tools_tests.rs @@ -1,6 +1,32 @@ use super::*; +use std::path::Path; use std::sync::Arc; +struct WorkspaceEnvGuard { + previous: Option, +} + +impl WorkspaceEnvGuard { + fn set(path: &Path) -> Self { + let previous = std::env::var_os("OPENHUMAN_WORKSPACE"); + unsafe { + std::env::set_var("OPENHUMAN_WORKSPACE", path); + } + Self { previous } + } +} + +impl Drop for WorkspaceEnvGuard { + fn drop(&mut self) { + unsafe { + match self.previous.take() { + Some(value) => std::env::set_var("OPENHUMAN_WORKSPACE", value), + None => std::env::remove_var("OPENHUMAN_WORKSPACE"), + } + } + } +} + /// Minimal `Arc` for the agent-tool constructors. All five /// composio agent tools now resolve their client per call through /// `create_composio_client(&config)` rather than holding a pre-baked @@ -310,9 +336,7 @@ async fn sandbox_read_only_passes_through_read_scope_actions_to_downstream_gates let _env_guard = TEST_ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner()); let tmp = tempfile::tempdir().expect("tempdir"); - unsafe { - std::env::set_var("OPENHUMAN_WORKSPACE", tmp.path()); - } + let _workspace_guard = WorkspaceEnvGuard::set(tmp.path()); let mut config = crate::openhuman::config::Config::default(); config.config_path = tmp.path().join("config.toml"); @@ -332,10 +356,6 @@ async fn sandbox_read_only_passes_through_read_scope_actions_to_downstream_gates !msg.contains("strict read-only"), "read-scoped slug must not hit the sandbox gate, got: {msg}" ); - - unsafe { - std::env::remove_var("OPENHUMAN_WORKSPACE"); - } } #[tokio::test] @@ -353,9 +373,7 @@ async fn sandbox_unset_leaves_all_scopes_to_downstream_gates() { let _env_guard = TEST_ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner()); let tmp = tempfile::tempdir().expect("tempdir"); - unsafe { - std::env::set_var("OPENHUMAN_WORKSPACE", tmp.path()); - } + let _workspace_guard = WorkspaceEnvGuard::set(tmp.path()); let mut config = crate::openhuman::config::Config::default(); config.config_path = tmp.path().join("config.toml"); @@ -372,10 +390,6 @@ async fn sandbox_unset_leaves_all_scopes_to_downstream_gates() { !msg.contains("strict read-only"), "no sandbox scope must never trigger the gate, got: {msg}" ); - - unsafe { - std::env::remove_var("OPENHUMAN_WORKSPACE"); - } } #[tokio::test] @@ -394,9 +408,7 @@ async fn sandbox_sandboxed_mode_does_not_trigger_readonly_gate() { let _env_guard = TEST_ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner()); let tmp = tempfile::tempdir().expect("tempdir"); - unsafe { - std::env::set_var("OPENHUMAN_WORKSPACE", tmp.path()); - } + let _workspace_guard = WorkspaceEnvGuard::set(tmp.path()); let mut config = crate::openhuman::config::Config::default(); config.config_path = tmp.path().join("config.toml"); @@ -418,10 +430,6 @@ async fn sandbox_sandboxed_mode_does_not_trigger_readonly_gate() { !msg.contains("strict read-only"), "Sandboxed mode must not trigger the read-only gate, got: {msg}" ); - - unsafe { - std::env::remove_var("OPENHUMAN_WORKSPACE"); - } } // ── render_tools_markdown ─────────────────────────────────────────── @@ -647,9 +655,7 @@ async fn list_tools_in_direct_mode_returns_empty_without_hitting_backend() { let _env_guard = TEST_ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner()); let tmp = tempfile::tempdir().expect("tempdir"); - unsafe { - std::env::set_var("OPENHUMAN_WORKSPACE", tmp.path()); - } + let _workspace_guard = WorkspaceEnvGuard::set(tmp.path()); let mut config = crate::openhuman::config::Config::default(); config.config_path = tmp.path().join("config.toml"); @@ -681,10 +687,6 @@ async fn list_tools_in_direct_mode_returns_empty_without_hitting_backend() { body.contains("\"tools\":[]") || body.contains("\"tools\": []"), "direct-mode list_tools body should contain an empty tools array: {body}" ); - - unsafe { - std::env::remove_var("OPENHUMAN_WORKSPACE"); - } } #[tokio::test] @@ -708,9 +710,7 @@ async fn execute_tool_per_call_factory_means_no_baked_client() { let _env_guard = TEST_ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner()); let tmp = tempfile::tempdir().unwrap(); - unsafe { - std::env::set_var("OPENHUMAN_WORKSPACE", tmp.path()); - } + let _workspace_guard = WorkspaceEnvGuard::set(tmp.path()); let mut config = crate::openhuman::config::Config::default(); config.config_path = tmp.path().join("config.toml"); @@ -738,10 +738,6 @@ async fn execute_tool_per_call_factory_means_no_baked_client() { !msg.contains("staging-api") && !msg.contains("agent-integrations"), "must not leak backend-tenant routing artifacts in direct mode: {msg}" ); - - unsafe { - std::env::remove_var("OPENHUMAN_WORKSPACE"); - } } #[tokio::test] @@ -760,9 +756,7 @@ async fn list_toolkits_in_direct_mode_returns_empty_without_hitting_backend() { let _env_guard = TEST_ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner()); let tmp = tempfile::tempdir().expect("tempdir"); - unsafe { - std::env::set_var("OPENHUMAN_WORKSPACE", tmp.path()); - } + let _workspace_guard = WorkspaceEnvGuard::set(tmp.path()); let mut config = crate::openhuman::config::Config::default(); config.config_path = tmp.path().join("config.toml"); @@ -793,10 +787,6 @@ async fn list_toolkits_in_direct_mode_returns_empty_without_hitting_backend() { body.contains("\"toolkits\":[]") || body.contains("\"toolkits\": []"), "direct-mode list_toolkits body should contain an empty toolkits array: {body}" ); - - unsafe { - std::env::remove_var("OPENHUMAN_WORKSPACE"); - } } #[test] @@ -833,9 +823,7 @@ async fn authorize_in_direct_mode_refuses_with_app_composio_dev_hint() { let _env_guard = TEST_ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner()); let tmp = tempfile::tempdir().expect("tempdir"); - unsafe { - std::env::set_var("OPENHUMAN_WORKSPACE", tmp.path()); - } + let _workspace_guard = WorkspaceEnvGuard::set(tmp.path()); let mut config = crate::openhuman::config::Config::default(); config.config_path = tmp.path().join("config.toml"); @@ -862,8 +850,4 @@ async fn authorize_in_direct_mode_refuses_with_app_composio_dev_hint() { !msg.contains("staging-api") && !msg.contains("agent-integrations"), "must not leak backend-tenant routing artifacts in direct mode: {msg}" ); - - unsafe { - std::env::remove_var("OPENHUMAN_WORKSPACE"); - } } diff --git a/src/openhuman/config/mod.rs b/src/openhuman/config/mod.rs index 312a35bbc..e9435e347 100644 --- a/src/openhuman/config/mod.rs +++ b/src/openhuman/config/mod.rs @@ -34,7 +34,7 @@ pub use schema::{ MultimodalConfig, ObservabilityConfig, OrchestratorModelConfig, PolymarketClobCredentials, PolymarketConfig, ProxyConfig, ProxyScope, ReflectionSource, ReliabilityConfig, ResourceLimitsConfig, RuntimeConfig, SandboxBackend, SandboxConfig, SchedulerConfig, - SchedulerGateConfig, SchedulerGateMode, ScreenIntelligenceConfig, SecretsConfig, + SchedulerGateConfig, SchedulerGateMode, ScreenIntelligenceConfig, SearxngConfig, SecretsConfig, SecurityConfig, SlackConfig, StorageConfig, StorageProviderConfig, StorageProviderSection, StreamMode, TeamModelConfig, TelegramConfig, UpdateConfig, UpdateRestartStrategy, VoiceActivationMode, VoiceServerConfig, WebSearchConfig, WebhookConfig, diff --git a/src/openhuman/config/schema/load.rs b/src/openhuman/config/schema/load.rs index 4f0c65e27..1de44a930 100644 --- a/src/openhuman/config/schema/load.rs +++ b/src/openhuman/config/schema/load.rs @@ -1290,6 +1290,48 @@ impl Config { } } + // SearXNG self-hosted search. Unlike Seltz, this needs no API key; + // keep it opt-in because it reaches a user-controlled HTTP endpoint. + if let Some(flag) = env.get_any(&["OPENHUMAN_SEARXNG_ENABLED", "SEARXNG_ENABLED"]) { + if let Some(enabled) = parse_env_bool("OPENHUMAN_SEARXNG_ENABLED", &flag) { + self.searxng.enabled = enabled; + } + } + if let Some(url) = env.get_any(&["OPENHUMAN_SEARXNG_BASE_URL", "SEARXNG_BASE_URL"]) { + let url = url.trim(); + if !url.is_empty() { + self.searxng.base_url = url.to_string(); + } + } + if let Some(max) = env.get_any(&["OPENHUMAN_SEARXNG_MAX_RESULTS", "SEARXNG_MAX_RESULTS"]) { + if let Ok(n) = max.parse::() { + if (1..=50).contains(&n) { + self.searxng.max_results = n; + } + } + } + if let Some(language) = env.get_any(&[ + "OPENHUMAN_SEARXNG_DEFAULT_LANGUAGE", + "SEARXNG_DEFAULT_LANGUAGE", + ]) { + let language = language.trim(); + if !language.is_empty() { + self.searxng.default_language = language.to_string(); + } + } + if let Some(timeout_secs) = env.get_any(&[ + "OPENHUMAN_SEARXNG_TIMEOUT_SECS", + "OPENHUMAN_SEARXNG_TIMEOUT_SECONDS", + "SEARXNG_TIMEOUT_SECS", + "SEARXNG_TIMEOUT_SECONDS", + ]) { + if let Ok(timeout_secs) = timeout_secs.parse::() { + if timeout_secs > 0 { + self.searxng.timeout_secs = timeout_secs; + } + } + } + // `OPENHUMAN_WEB_SEARCH_ENABLED` is intentionally ignored — // web search is unconditionally registered in the tool set. // Only the result/timeout budget knobs remain environment-configurable. diff --git a/src/openhuman/config/schema/load_tests.rs b/src/openhuman/config/schema/load_tests.rs index 346f1338f..0f64ec856 100644 --- a/src/openhuman/config/schema/load_tests.rs +++ b/src/openhuman/config/schema/load_tests.rs @@ -249,6 +249,56 @@ fn apply_env_overrides_web_search_max_results_and_timeout_clamped() { ]); } +#[test] +fn apply_env_overrides_searxng_config() { + let _g = env_lock(); + clear_env(&[ + "OPENHUMAN_SEARXNG_ENABLED", + "SEARXNG_ENABLED", + "OPENHUMAN_SEARXNG_BASE_URL", + "SEARXNG_BASE_URL", + "OPENHUMAN_SEARXNG_MAX_RESULTS", + "SEARXNG_MAX_RESULTS", + "OPENHUMAN_SEARXNG_DEFAULT_LANGUAGE", + "SEARXNG_DEFAULT_LANGUAGE", + "OPENHUMAN_SEARXNG_TIMEOUT_SECS", + "OPENHUMAN_SEARXNG_TIMEOUT_SECONDS", + "SEARXNG_TIMEOUT_SECS", + "SEARXNG_TIMEOUT_SECONDS", + ]); + + let mut cfg = Config::default(); + unsafe { + std::env::set_var("OPENHUMAN_SEARXNG_ENABLED", "yes"); + std::env::set_var("OPENHUMAN_SEARXNG_BASE_URL", "http://127.0.0.1:8081"); + std::env::set_var("OPENHUMAN_SEARXNG_MAX_RESULTS", "25"); + std::env::set_var("OPENHUMAN_SEARXNG_DEFAULT_LANGUAGE", "zh-CN"); + std::env::set_var("OPENHUMAN_SEARXNG_TIMEOUT_SECONDS", "12"); + } + + cfg.apply_env_overrides(); + + assert!(cfg.searxng.enabled); + assert_eq!(cfg.searxng.base_url, "http://127.0.0.1:8081"); + assert_eq!(cfg.searxng.max_results, 25); + assert_eq!(cfg.searxng.default_language, "zh-CN"); + assert_eq!(cfg.searxng.timeout_secs, 12); + clear_env(&[ + "OPENHUMAN_SEARXNG_ENABLED", + "OPENHUMAN_SEARXNG_BASE_URL", + "OPENHUMAN_SEARXNG_MAX_RESULTS", + "OPENHUMAN_SEARXNG_DEFAULT_LANGUAGE", + "OPENHUMAN_SEARXNG_TIMEOUT_SECONDS", + ]); +} + +#[test] +fn searxng_timeout_seconds_alias_deserializes() { + let cfg: crate::openhuman::config::SearxngConfig = + toml::from_str(r#"timeout_seconds = 7"#).expect("deserialize searxng config"); + assert_eq!(cfg.timeout_secs, 7); +} + #[test] fn apply_env_overrides_picks_up_sentry_dsn() { let _g = env_lock(); @@ -564,6 +614,40 @@ fn env_overlay_web_search_limits_validated() { assert_eq!(cfg.web_search.max_results, 4); } +#[test] +fn env_overlay_searxng_config_validated() { + let mut cfg = Config::default(); + + cfg.apply_env_overlay_with( + &HashMapEnv::new() + .with("OPENHUMAN_SEARXNG_ENABLED", "true") + .with("OPENHUMAN_SEARXNG_BASE_URL", "http://127.0.0.1:8888") + .with("OPENHUMAN_SEARXNG_MAX_RESULTS", "40") + .with("OPENHUMAN_SEARXNG_DEFAULT_LANGUAGE", "fr") + .with("OPENHUMAN_SEARXNG_TIMEOUT_SECS", "9"), + ); + + assert!(cfg.searxng.enabled); + assert_eq!(cfg.searxng.base_url, "http://127.0.0.1:8888"); + assert_eq!(cfg.searxng.max_results, 40); + assert_eq!(cfg.searxng.default_language, "fr"); + assert_eq!(cfg.searxng.timeout_secs, 9); + + cfg.apply_env_overlay_with( + &HashMapEnv::new() + .with("OPENHUMAN_SEARXNG_ENABLED", "no") + .with("OPENHUMAN_SEARXNG_MAX_RESULTS", "0") + .with("OPENHUMAN_SEARXNG_TIMEOUT_SECS", "0"), + ); + + assert!(!cfg.searxng.enabled); + assert_eq!(cfg.searxng.max_results, 40); + assert_eq!(cfg.searxng.timeout_secs, 9); + + cfg.apply_env_overlay_with(&HashMapEnv::new().with("SEARXNG_TIMEOUT_SECONDS", "11")); + assert_eq!(cfg.searxng.timeout_secs, 11); +} + #[test] fn env_overlay_proxy_url_enables_proxy_when_not_explicit() { let mut cfg = Config::default(); diff --git a/src/openhuman/config/schema/mod.rs b/src/openhuman/config/schema/mod.rs index 2b921a097..be94e882d 100644 --- a/src/openhuman/config/schema/mod.rs +++ b/src/openhuman/config/schema/mod.rs @@ -75,8 +75,8 @@ pub use tools::{ BrowserComputerUseConfig, BrowserConfig, ComposioConfig, ComputerControlConfig, CurlConfig, GitbooksConfig, HttpRequestConfig, IntegrationToggle, IntegrationsConfig, McpAuthConfig, McpClientConfig, McpClientIdentityConfig, McpServerConfig, MultimodalConfig, - PolymarketClobCredentials, PolymarketConfig, SecretsConfig, SeltzConfig, WebSearchConfig, - COMPOSIO_MODE_BACKEND, COMPOSIO_MODE_DIRECT, + PolymarketClobCredentials, PolymarketConfig, SearxngConfig, SecretsConfig, SeltzConfig, + WebSearchConfig, COMPOSIO_MODE_BACKEND, COMPOSIO_MODE_DIRECT, }; pub use update::{UpdateConfig, UpdateRestartStrategy}; mod voice_server; diff --git a/src/openhuman/config/schema/tools.rs b/src/openhuman/config/schema/tools.rs index fddfd6fe0..1050dc657 100644 --- a/src/openhuman/config/schema/tools.rs +++ b/src/openhuman/config/schema/tools.rs @@ -417,6 +417,54 @@ impl Default for SeltzConfig { } } +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +#[serde(default)] +pub struct SearxngConfig { + /// When `true`, register `searxng_search` as an agent and MCP tool. + #[serde(default)] + pub enabled: bool, + /// Base URL for the user's SearXNG instance. + #[serde(default = "default_searxng_base_url")] + pub base_url: String, + /// Max results per query (1-50, default 10). + #[serde(default = "default_searxng_max_results")] + pub max_results: usize, + /// Language code passed to SearXNG when a call omits `language`. + #[serde(default = "default_searxng_language")] + pub default_language: String, + /// Per-request timeout in seconds (default 10). + #[serde(default = "default_searxng_timeout_secs", alias = "timeout_seconds")] + pub timeout_secs: u64, +} + +fn default_searxng_base_url() -> String { + "http://localhost:8080".into() +} + +fn default_searxng_max_results() -> usize { + 10 +} + +fn default_searxng_language() -> String { + "en".into() +} + +fn default_searxng_timeout_secs() -> u64 { + 10 +} + +impl Default for SearxngConfig { + fn default() -> Self { + Self { + enabled: false, + base_url: default_searxng_base_url(), + max_results: default_searxng_max_results(), + default_language: default_searxng_language(), + timeout_secs: default_searxng_timeout_secs(), + } + } +} + #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] #[serde(default)] pub struct WebSearchConfig { diff --git a/src/openhuman/config/schema/types.rs b/src/openhuman/config/schema/types.rs index cdbee8f90..d91751ec1 100644 --- a/src/openhuman/config/schema/types.rs +++ b/src/openhuman/config/schema/types.rs @@ -175,6 +175,9 @@ pub struct Config { #[serde(default)] pub seltz: SeltzConfig, + #[serde(default)] + pub searxng: SearxngConfig, + #[serde(default)] pub web_search: WebSearchConfig, @@ -549,6 +552,7 @@ impl Default for Config { mcp_client: McpClientConfig::default(), multimodal: MultimodalConfig::default(), seltz: SeltzConfig::default(), + searxng: SearxngConfig::default(), web_search: WebSearchConfig::default(), proxy: ProxyConfig::default(), cost: CostConfig::default(), diff --git a/src/openhuman/integrations/mod.rs b/src/openhuman/integrations/mod.rs index 4d0f1b8f2..a57e1bad9 100644 --- a/src/openhuman/integrations/mod.rs +++ b/src/openhuman/integrations/mod.rs @@ -1,13 +1,16 @@ -//! Agent integration tools that proxy through the backend API. +//! Agent integration tools. //! -//! Each tool calls a backend endpoint (authenticated via JWT Bearer token) which -//! handles external API calls, billing, rate limiting, and markup. The client -//! never talks to external services directly. +//! Most integrations proxy through backend endpoints authenticated with the +//! user's session token, so billing, rate limiting, and provider markup stay +//! server-side. Some integrations, such as SearXNG, call user-configured +//! endpoints directly when enabled; those callers must keep configured base URLs +//! trusted because requests leave the local core process. pub mod apify; pub mod client; pub mod google_places; pub mod parallel; +pub mod searxng; pub mod seltz; pub mod stock_prices; pub mod tinyfish; @@ -21,6 +24,7 @@ pub use parallel::{ ParallelChatTool, ParallelDatasetTool, ParallelEnrichTool, ParallelExtractTool, ParallelResearchTool, ParallelSearchTool, }; +pub use searxng::{SearxngSearchArgs, SearxngSearchResponse, SearxngSearchTool}; pub use seltz::SeltzSearchTool; pub use stock_prices::{ StockCommodityTool, StockCryptoSeriesTool, StockExchangeRateTool, StockOptionsTool, diff --git a/src/openhuman/integrations/searxng.rs b/src/openhuman/integrations/searxng.rs new file mode 100644 index 000000000..6eb878b14 --- /dev/null +++ b/src/openhuman/integrations/searxng.rs @@ -0,0 +1,574 @@ +//! SearXNG search integration for self-hosted, private web search. +//! +//! SearXNG exposes JSON results from `GET /search?format=json`. This wrapper +//! keeps the output shape small and stable for agents and MCP clients: +//! `{ title, url, snippet, source }`. + +use crate::openhuman::tools::traits::{Tool, ToolCallOptions, ToolCategory, ToolResult}; +use crate::openhuman::util::utf8_safe_prefix_at_byte_boundary; +use anyhow::Context; +use async_trait::async_trait; +use serde::{Deserialize, Serialize}; +use serde_json::json; +use std::sync::OnceLock; +use std::time::Duration; + +const DEFAULT_SOURCE: &str = "searxng"; +/// Maximum number of SearXNG results accepted by the public tool surface. +pub const MAX_RESULTS: usize = 50; + +static SHARED_HTTP_CLIENT: OnceLock = OnceLock::new(); + +fn shared_http_client() -> reqwest::Client { + SHARED_HTTP_CLIENT + .get_or_init(|| { + tracing::debug!("[searxng] initializing shared HTTP client"); + reqwest::Client::builder() + .use_rustls_tls() + .build() + .expect("failed to build shared SearXNG HTTP client") + }) + .clone() +} + +#[derive(Debug, Clone, PartialEq, Eq)] +/// Search arguments accepted by the SearXNG tool. +pub struct SearxngSearchArgs { + /// Search query sent as SearXNG's `q` parameter. + pub query: String, + /// Optional SearXNG categories. `web` is normalized to `general`. + pub categories: Vec, + /// Optional language code. Falls back to the configured default when absent. + pub language: Option, + /// Optional per-call result limit, clamped to the supported maximum. + pub max_results: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +/// Normalized response returned to agents, RPC callers, and MCP clients. +pub struct SearxngSearchResponse { + /// Trimmed query used for the search. + pub query: String, + /// Normalized search results. + pub results: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +/// One normalized SearXNG result. +pub struct SearxngSearchResult { + /// Result title, or the URL when SearXNG omits a title. + pub title: String, + /// Absolute result URL. + pub url: String, + /// Result excerpt, if SearXNG returned one. + pub snippet: String, + /// Search engine/source name. + pub source: String, +} + +#[derive(Debug, Deserialize)] +struct RawSearxngResponse { + #[serde(default)] + results: Vec, +} + +#[derive(Debug, Deserialize)] +struct RawSearxngResult { + #[serde(default)] + title: Option, + #[serde(default)] + url: Option, + #[serde(default)] + content: Option, + #[serde(default)] + snippet: Option, + #[serde(default)] + engine: Option, + #[serde(default)] + engines: Vec, +} + +pub struct SearxngSearchTool { + base_url: String, + max_results: usize, + default_language: String, + timeout_secs: u64, + http_client: reqwest::Client, +} + +impl SearxngSearchTool { + /// Build a SearXNG search tool for a user-configured endpoint. + pub fn new( + base_url: String, + max_results: usize, + default_language: String, + timeout_secs: u64, + ) -> Self { + Self::with_http_client( + base_url, + max_results, + default_language, + timeout_secs, + shared_http_client(), + ) + } + + /// Build a SearXNG search tool with a caller-provided HTTP client. + pub fn with_http_client( + base_url: String, + max_results: usize, + default_language: String, + timeout_secs: u64, + http_client: reqwest::Client, + ) -> Self { + let timeout = timeout_secs.max(1); + + Self { + base_url, + max_results: max_results.clamp(1, MAX_RESULTS), + default_language, + timeout_secs: timeout, + http_client, + } + } + + /// Execute a SearXNG JSON search and normalize the returned result rows. + pub async fn search(&self, args: SearxngSearchArgs) -> anyhow::Result { + let query = args.query.trim(); + if query.is_empty() { + anyhow::bail!("SearXNG search query cannot be empty"); + } + + let max_results = args + .max_results + .unwrap_or(self.max_results) + .clamp(1, MAX_RESULTS); + let categories = normalize_categories(args.categories)?; + let language = args + .language + .as_deref() + .map(str::trim) + .filter(|value| !value.is_empty()) + .unwrap_or_else(|| self.default_language.trim()); + + let mut url = self.search_endpoint_url()?; + { + let mut pairs = url.query_pairs_mut(); + pairs.append_pair("q", query); + pairs.append_pair("format", "json"); + if !categories.is_empty() { + pairs.append_pair("categories", &categories.join(",")); + } + if !language.is_empty() { + pairs.append_pair("language", language); + } + } + + tracing::debug!( + query_len = query.chars().count(), + max_results, + categories = ?categories, + timeout_secs = self.timeout_secs, + "[searxng] GET /search" + ); + + let response = self + .http_client + .get(url.clone()) + .timeout(Duration::from_secs(self.timeout_secs)) + .send() + .await + .map_err(|err| { + tracing::warn!(error = %err, "[searxng] request failed"); + anyhow::anyhow!("SearXNG request failed: {err}") + })?; + + let status = response.status(); + if !status.is_success() { + let body = response.text().await.unwrap_or_default(); + let detail = utf8_safe_prefix_at_byte_boundary(&body, 500); + tracing::warn!(status = %status, "[searxng] non-2xx response: {detail}"); + anyhow::bail!("SearXNG returned {status}: {detail}"); + } + + let raw: RawSearxngResponse = response.json().await.map_err(|err| { + tracing::warn!(error = %err, "[searxng] failed to parse JSON response"); + anyhow::anyhow!("Failed to parse SearXNG response: {err}") + })?; + + let results = normalize_results(raw, max_results); + tracing::debug!(result_count = results.len(), "[searxng] search complete"); + + Ok(SearxngSearchResponse { + query: query.to_string(), + results, + }) + } + + fn search_endpoint_url(&self) -> anyhow::Result { + let base = self.base_url.trim().trim_end_matches('/'); + if base.is_empty() { + anyhow::bail!("SearXNG base_url cannot be empty"); + } + let endpoint = if base.ends_with("/search") { + base.to_string() + } else { + format!("{base}/search") + }; + reqwest::Url::parse(&endpoint) + .with_context(|| format!("invalid SearXNG base_url `{}`", self.base_url)) + } + + fn render_markdown(response: &SearxngSearchResponse) -> String { + if response.results.is_empty() { + return format!("No SearXNG results for `{}`.", response.query); + } + + let mut out = format!("# SearXNG results for `{}`\n", response.query); + for (index, result) in response.results.iter().enumerate() { + out.push_str(&format!( + "\n{}. [{}]({})\n", + index + 1, + result.title, + result.url + )); + if !result.snippet.is_empty() { + out.push_str(&format!(" {}\n", result.snippet)); + } + out.push_str(&format!(" Source: {}\n", result.source)); + } + out + } +} + +#[async_trait] +impl Tool for SearxngSearchTool { + fn name(&self) -> &str { + "searxng_search" + } + + fn description(&self) -> &str { + "Search a user-configured SearXNG instance. Returns private, self-hosted web search results normalized as title, URL, snippet, and source." + } + + fn parameters_schema(&self) -> serde_json::Value { + json!({ + "type": "object", + "properties": { + "query": { + "type": "string", + "description": "Search query string." + }, + "categories": { + "type": "array", + "items": { + "type": "string", + "enum": ["web", "general", "news", "images"] + }, + "description": "Optional SearXNG categories. `web` maps to SearXNG `general`." + }, + "language": { + "type": "string", + "description": "Optional language code, e.g. `en`, `zh-CN`, `fr`." + }, + "max_results": { + "type": "integer", + "minimum": 1, + "maximum": MAX_RESULTS, + "description": "Maximum number of results to return." + } + }, + "required": ["query"], + "additionalProperties": false + }) + } + + fn supports_markdown(&self) -> bool { + true + } + + fn category(&self) -> ToolCategory { + ToolCategory::Skill + } + + async fn execute(&self, args: serde_json::Value) -> anyhow::Result { + self.execute_with_options(args, ToolCallOptions::default()) + .await + } + + async fn execute_with_options( + &self, + args: serde_json::Value, + options: ToolCallOptions, + ) -> anyhow::Result { + let search_args = parse_search_args(args)?; + let response = self.search(search_args).await?; + let payload = json!({ + "query": response.query, + "results": response.results, + }); + let markdown = Self::render_markdown(&response); + let mut result = ToolResult::json(payload); + if options.prefer_markdown { + result.markdown_formatted = Some(markdown); + } + Ok(result) + } +} + +/// Normalize user-facing category aliases into SearXNG category names. +pub fn normalize_categories(categories: Vec) -> anyhow::Result> { + let mut normalized = Vec::new(); + for category in categories { + let trimmed = category.trim(); + if trimmed.is_empty() { + continue; + } + let mapped = match trimmed.to_ascii_lowercase().as_str() { + "web" | "general" => "general", + "news" => "news", + "images" => "images", + other => anyhow::bail!( + "unsupported SearXNG category `{other}`; expected web, news, or images" + ), + }; + if !normalized.iter().any(|existing| existing == mapped) { + normalized.push(mapped.to_string()); + } + } + Ok(normalized) +} + +fn normalize_results(raw: RawSearxngResponse, max_results: usize) -> Vec { + raw.results + .into_iter() + .filter_map(|item| { + let url = item.url.unwrap_or_default().trim().to_string(); + if url.is_empty() { + return None; + } + let title = item.title.unwrap_or_default().trim().to_string(); + let snippet = first_non_empty_trimmed([item.content, item.snippet]); + let source = item + .engine + .filter(|value| !value.trim().is_empty()) + .or_else(|| { + item.engines + .into_iter() + .find(|value| !value.trim().is_empty()) + }) + .unwrap_or_else(|| DEFAULT_SOURCE.to_string()) + .trim() + .to_string(); + Some(SearxngSearchResult { + title: if title.is_empty() { url.clone() } else { title }, + url, + snippet, + source, + }) + }) + .take(max_results) + .collect() +} + +fn first_non_empty_trimmed(values: impl IntoIterator>) -> String { + values + .into_iter() + .flatten() + .map(|value| value.trim().to_string()) + .find(|value| !value.is_empty()) + .unwrap_or_default() +} + +fn parse_search_args(args: serde_json::Value) -> anyhow::Result { + let object = args + .as_object() + .ok_or_else(|| anyhow::anyhow!("SearXNG arguments must be an object"))?; + let query = object + .get("query") + .and_then(|value| value.as_str()) + .map(str::trim) + .filter(|value| !value.is_empty()) + .ok_or_else(|| anyhow::anyhow!("Missing required parameter: query"))? + .to_string(); + let categories = match object.get("categories") { + Some(value) if value.is_null() => Vec::new(), + Some(value) => value + .as_array() + .ok_or_else(|| anyhow::anyhow!("categories must be an array of strings"))? + .iter() + .map(|item| { + item.as_str() + .map(str::to_string) + .ok_or_else(|| anyhow::anyhow!("categories must contain only strings")) + }) + .collect::>>()?, + None => Vec::new(), + }; + let language = match object.get("language") { + Some(value) => { + let language = value + .as_str() + .map(str::trim) + .filter(|value| !value.is_empty()) + .ok_or_else(|| anyhow::anyhow!("language must be a non-empty string"))?; + Some(language.to_string()) + } + None => None, + }; + let max_results = match object.get("max_results") { + Some(value) => { + let max_results = value + .as_u64() + .ok_or_else(|| anyhow::anyhow!("max_results must be a positive integer"))?; + Some(max_results.clamp(1, MAX_RESULTS as u64) as usize) + } + None => None, + }; + + Ok(SearxngSearchArgs { + query, + categories, + language, + max_results, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn tool(base_url: String) -> SearxngSearchTool { + SearxngSearchTool::new(base_url, 10, "en".into(), 5) + } + + #[test] + fn normalizes_categories_and_maps_web_to_general() { + let categories = normalize_categories(vec![ + "web".into(), + "news".into(), + "general".into(), + " images ".into(), + ]) + .expect("categories"); + assert_eq!(categories, vec!["general", "news", "images"]); + } + + #[test] + fn rejects_unknown_category() { + let err = normalize_categories(vec!["videos".into()]).expect_err("must reject"); + assert!(err.to_string().contains("unsupported SearXNG category")); + } + + #[test] + fn normalize_results_falls_back_to_snippet_when_content_is_blank() { + let results = normalize_results( + RawSearxngResponse { + results: vec![RawSearxngResult { + title: Some("Result".into()), + url: Some("https://example.com".into()), + content: Some(" ".into()), + snippet: Some("Useful fallback snippet".into()), + engine: Some("engine".into()), + engines: Vec::new(), + }], + }, + 5, + ); + + assert_eq!(results[0].snippet, "Useful fallback snippet"); + } + + #[test] + fn parse_search_args_rejects_malformed_optional_values() { + let language_err = parse_search_args(json!({ + "query": "privacy search", + "language": 1 + })) + .expect_err("language must reject wrong type"); + assert!(language_err + .to_string() + .contains("language must be a non-empty string")); + + let max_results_err = parse_search_args(json!({ + "query": "privacy search", + "max_results": "10" + })) + .expect_err("max_results must reject wrong type"); + assert!(max_results_err + .to_string() + .contains("max_results must be a positive integer")); + } + + #[test] + fn parameters_schema_includes_mcp_expected_fields() { + let schema = tool("http://localhost:8080".into()).parameters_schema(); + assert_eq!(schema["type"], "object"); + assert!(schema["properties"]["query"].is_object()); + assert!(schema["properties"]["categories"].is_object()); + assert!(schema["properties"]["language"].is_object()); + assert!(schema["properties"]["max_results"].is_object()); + } + + #[tokio::test] + async fn search_calls_json_endpoint_and_normalizes_results() { + use axum::{extract::Query, routing::get, Json, Router}; + use std::collections::HashMap; + + let app = Router::new().route( + "/search", + get(|Query(params): Query>| async move { + assert_eq!(params.get("q").map(String::as_str), Some("test query")); + assert_eq!(params.get("format").map(String::as_str), Some("json")); + assert_eq!( + params.get("categories").map(String::as_str), + Some("general,news") + ); + assert_eq!(params.get("language").map(String::as_str), Some("en")); + Json(json!({ + "results": [ + { + "title": "First result", + "url": "https://example.com/one", + "content": "A useful snippet.", + "engine": "duckduckgo" + }, + { + "title": "Missing URL should be skipped", + "content": "No URL" + }, + { + "url": "https://example.com/two", + "snippet": "Fallback snippet.", + "engines": ["brave"] + } + ] + })) + }), + ); + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }); + + let response = tool(format!("http://127.0.0.1:{}", addr.port())) + .search(SearxngSearchArgs { + query: " test query ".into(), + categories: vec!["web".into(), "news".into()], + language: None, + max_results: Some(5), + }) + .await + .expect("search"); + + assert_eq!(response.query, "test query"); + assert_eq!(response.results.len(), 2); + assert_eq!(response.results[0].title, "First result"); + assert_eq!(response.results[0].source, "duckduckgo"); + assert_eq!(response.results[1].title, "https://example.com/two"); + assert_eq!(response.results[1].snippet, "Fallback snippet."); + assert_eq!(response.results[1].source, "brave"); + } +} diff --git a/src/openhuman/mcp_server/protocol.rs b/src/openhuman/mcp_server/protocol.rs index 00b2fb028..85f7cdcce 100644 --- a/src/openhuman/mcp_server/protocol.rs +++ b/src/openhuman/mcp_server/protocol.rs @@ -139,7 +139,7 @@ async fn handle_request(id: Value, method: &str, params: Value) -> Value { match method { "initialize" => success_response(id, initialize_result(params)), "ping" => success_response(id, json!({})), - "tools/list" => success_response(id, tools::list_tools_result()), + "tools/list" => success_response(id, tools::list_tools_result().await), "tools/call" => match parse_tool_call_params(params) { Ok((name, arguments)) => { log::debug!( @@ -227,7 +227,7 @@ fn initialize_result(params: Value) -> Value { "name": "openhuman-core", "version": env!("CARGO_PKG_VERSION") }, - "instructions": "OpenHuman MCP exposes first-level core integration: inspect the live tool catalog with core.list_tools or core.tool_instructions, inspect subagents with agent.list_subagents, run a standalone subagent with agent.run_subagent, and use memory.search or memory.recall plus tree.read_chunk for local memory reads." + "instructions": "OpenHuman MCP exposes first-level core integration: inspect the live tool catalog with core.list_tools or core.tool_instructions, inspect subagents with agent.list_subagents, run a standalone subagent with agent.run_subagent, use searxng_search when self-hosted search is enabled, and use memory.search or memory.recall plus tree.read_chunk for local memory reads." }) } @@ -338,21 +338,26 @@ mod tests { .iter() .map(|tool| tool["name"].as_str().expect("name")) .collect::>(); - assert_eq!( - names, - vec![ - "core.list_tools", - "core.tool_instructions", - "agent.list_subagents", - "agent.run_subagent", - "memory.search", - "memory.recall", - "tree.read_chunk", - "tree.browse", - "tree.top_entities", - "tree.list_sources", - ] - ); + let mut base_names = names + .iter() + .copied() + .filter(|name| *name != "searxng_search") + .collect::>(); + let mut expected_base_names = vec![ + "core.list_tools", + "core.tool_instructions", + "agent.list_subagents", + "agent.run_subagent", + "memory.search", + "memory.recall", + "tree.read_chunk", + "tree.browse", + "tree.top_entities", + "tree.list_sources", + ]; + base_names.sort_unstable(); + expected_base_names.sort_unstable(); + assert_eq!(base_names, expected_base_names); } #[tokio::test] diff --git a/src/openhuman/mcp_server/tools.rs b/src/openhuman/mcp_server/tools.rs index 6dc2263f8..52feb6d4e 100644 --- a/src/openhuman/mcp_server/tools.rs +++ b/src/openhuman/mcp_server/tools.rs @@ -5,11 +5,13 @@ use crate::openhuman::agent::harness::AgentDefinitionRegistry; use crate::openhuman::agent::Agent; use crate::openhuman::config::rpc as config_rpc; use crate::openhuman::inference::provider::traits::build_tool_instructions_text; +use crate::openhuman::integrations::searxng::MAX_RESULTS as SEARXNG_MAX_RESULTS; use crate::openhuman::security::{SecurityPolicy, ToolOperation}; const DEFAULT_LIMIT: u64 = 10; const MAX_LIMIT: u64 = 50; const QUERY_ARGUMENTS: &[&str] = &["query", "k"]; +const SEARXNG_SEARCH_ARGUMENTS: &[&str] = &["query", "categories", "language", "max_results"]; const TREE_READ_CHUNK_ARGUMENTS: &[&str] = &["chunk_id"]; const SUBAGENT_RUN_ARGUMENTS: &[&str] = &["agent_id", "prompt"]; const TREE_BROWSE_ARGUMENTS: &[&str] = &[ @@ -72,6 +74,12 @@ impl ToolCallError { } pub fn tool_specs() -> Vec { + let mut specs = base_tool_specs(); + specs.push(searxng_tool_spec()); + specs +} + +fn base_tool_specs() -> Vec { vec![ McpToolSpec { name: "core.list_tools", @@ -182,6 +190,16 @@ pub fn tool_specs() -> Vec { ] } +fn searxng_tool_spec() -> McpToolSpec { + McpToolSpec { + name: "searxng_search", + title: "SearXNG Search", + description: "Search the configured self-hosted SearXNG instance and return normalized title, URL, snippet, and source results. Requires searxng.enabled=true in OpenHuman config.", + rpc_method: Some("openhuman.tools_searxng_search"), + input_schema: searxng_search_schema(), + } +} + fn tree_browse_schema() -> Value { json!({ "type": "object", @@ -269,8 +287,62 @@ fn tree_list_sources_schema() -> Value { }) } -pub fn list_tools_result() -> Value { - let tools = tool_specs() +fn searxng_search_schema() -> Value { + json!({ + "type": "object", + "properties": { + "query": { + "type": "string", + "minLength": 1, + "description": "Search query string." + }, + "categories": { + "type": "array", + "items": { + "type": "string", + "enum": ["web", "general", "news", "images"] + }, + "description": "Optional SearXNG categories. `web` maps to SearXNG `general`." + }, + "language": { + "type": "string", + "minLength": 1, + "description": "Optional language code, e.g. `en`, `zh-CN`, or `fr`." + }, + "max_results": { + "type": "integer", + "minimum": 1, + "maximum": SEARXNG_MAX_RESULTS, + "description": format!("Maximum results to return. Defaults to searxng.max_results; capped at {SEARXNG_MAX_RESULTS}.") + } + }, + "required": ["query"], + "additionalProperties": false + }) +} + +pub async fn list_tools_result() -> Value { + match config_rpc::load_config_with_timeout().await { + Ok(config) => list_tools_result_for_config(&config), + Err(err) => { + log::warn!( + "[mcp_server] tools/list config load failed; omitting config-gated tools: {err}" + ); + list_tools_result_from_specs(base_tool_specs()) + } + } +} + +fn list_tools_result_for_config(config: &crate::openhuman::config::Config) -> Value { + let mut specs = base_tool_specs(); + if config.searxng.enabled { + specs.push(searxng_tool_spec()); + } + list_tools_result_from_specs(specs) +} + +fn list_tools_result_from_specs(specs: Vec) -> Value { + let tools = specs .into_iter() .map(|tool| { json!({ @@ -415,6 +487,24 @@ fn build_rpc_params( ("k".to_string(), Value::from(limit)), ])) } + "searxng_search" => { + reject_unexpected_arguments(&args, SEARXNG_SEARCH_ARGUMENTS)?; + let query = required_non_empty_string(&args, "query")?; + let mut params = Map::new(); + params.insert("query".to_string(), Value::String(query)); + if let Some(categories) = optional_string_array(&args, "categories")? { + crate::openhuman::integrations::searxng::normalize_categories(categories.clone()) + .map_err(|err| ToolCallError::InvalidParams(err.to_string()))?; + params.insert("categories".to_string(), Value::from(categories)); + } + if let Some(language) = optional_non_empty_string(&args, "language")? { + params.insert("language".to_string(), Value::String(language)); + } + if let Some(max_results) = optional_max_results(&args, "max_results")? { + params.insert("max_results".to_string(), Value::from(max_results)); + } + Ok(params) + } "tree.read_chunk" => { reject_unexpected_arguments(&args, TREE_READ_CHUNK_ARGUMENTS)?; let chunk_id = required_non_empty_string(&args, "chunk_id")?; @@ -648,6 +738,34 @@ fn optional_limit(args: &Map) -> Result { Ok(limit) } +fn optional_max_results( + args: &Map, + key: &str, +) -> Result, ToolCallError> { + let Some(value) = args.get(key) else { + return Ok(None); + }; + if value.is_null() { + return Ok(None); + } + let Some(limit) = value.as_u64() else { + return Err(ToolCallError::InvalidParams(format!( + "argument `{key}` must be a positive integer" + ))); + }; + if limit == 0 { + return Err(ToolCallError::InvalidParams(format!( + "argument `{key}` must be greater than zero" + ))); + } + if limit > SEARXNG_MAX_RESULTS as u64 { + return Err(ToolCallError::InvalidParams(format!( + "argument `{key}` must not exceed {SEARXNG_MAX_RESULTS} (got {limit})" + ))); + } + Ok(Some(limit)) +} + fn validate_controller_params( spec: &McpToolSpec, params: &Map, @@ -885,8 +1003,9 @@ mod tests { use super::*; #[test] - fn list_tools_exposes_first_level_mcp_surface() { - let result = list_tools_result(); + fn list_tools_exposes_base_mcp_surface_when_searxng_disabled() { + let config = crate::openhuman::config::Config::default(); + let result = list_tools_result_for_config(&config); let names = result["tools"] .as_array() .expect("tools array") @@ -911,6 +1030,21 @@ mod tests { ); } + #[test] + fn list_tools_includes_searxng_when_enabled() { + let mut config = crate::openhuman::config::Config::default(); + config.searxng.enabled = true; + let result = list_tools_result_for_config(&config); + let names = result["tools"] + .as_array() + .expect("tools array") + .iter() + .map(|tool| tool["name"].as_str().expect("tool name")) + .collect::>(); + + assert!(names.contains(&"searxng_search")); + } + #[test] fn mapped_rpc_methods_are_registered() { for spec in tool_specs() { @@ -977,6 +1111,53 @@ mod tests { assert_eq!(params["k"], DEFAULT_LIMIT); } + #[test] + fn searxng_search_params_accept_optional_fields() { + let params = build_rpc_params( + "searxng_search", + json!({ + "query": " rust async ", + "categories": ["web", "news"], + "language": " en ", + "max_results": 12 + }), + ) + .expect("params"); + + assert_eq!(params["query"], "rust async"); + assert_eq!(params["categories"], json!(["web", "news"])); + assert_eq!(params["language"], "en"); + assert_eq!(params["max_results"], 12); + } + + #[test] + fn searxng_search_rejects_unknown_category() { + let err = build_rpc_params( + "searxng_search", + json!({ + "query": "rust", + "categories": ["videos"] + }), + ) + .expect_err("must reject"); + + assert!(err.message().contains("unsupported SearXNG category")); + } + + #[test] + fn searxng_search_rejects_max_results_above_max() { + let err = build_rpc_params( + "searxng_search", + json!({ + "query": "rust", + "max_results": SEARXNG_MAX_RESULTS + 1 + }), + ) + .expect_err("must reject"); + + assert!(err.message().contains("must not exceed")); + } + #[test] fn memory_search_rejects_k_above_max() { // Reject (don't silent-clamp) so the LLM can self-correct on the next diff --git a/src/openhuman/tools/ops.rs b/src/openhuman/tools/ops.rs index 682f66764..1e63490ae 100644 --- a/src/openhuman/tools/ops.rs +++ b/src/openhuman/tools/ops.rs @@ -343,6 +343,27 @@ pub fn all_tools_with_runtime( tracing::debug!("[seltz] disabled — set SELTZ_API_KEY to enable"); } + // SearXNG — self-hosted web search, gated on `searxng.enabled`. + // This is useful for users who want current web results without routing + // queries through OpenHuman's backend or a hosted search API. + if root_config.searxng.enabled { + tools.push(Box::new( + crate::openhuman::integrations::SearxngSearchTool::new( + root_config.searxng.base_url.clone(), + root_config.searxng.max_results, + root_config.searxng.default_language.clone(), + root_config.searxng.timeout_secs, + ), + )); + tracing::debug!( + base_url = %root_config.searxng.base_url, + max_results = root_config.searxng.max_results, + "[searxng] registered searxng_search tool" + ); + } else { + tracing::debug!("[searxng] disabled — set searxng.enabled=true to enable"); + } + // Managed Node.js exec tools — gated on `root_config.node.enabled`. // Both share the same `NodeBootstrap` as ShellTool so the download + // extract + install pipeline runs at most once per session. diff --git a/src/openhuman/tools/ops_tests.rs b/src/openhuman/tools/ops_tests.rs index ca5d54303..60d64ed2d 100644 --- a/src/openhuman/tools/ops_tests.rs +++ b/src/openhuman/tools/ops_tests.rs @@ -926,7 +926,7 @@ fn all_tools_registers_integration_families_when_enabled_and_signed_in() { } #[test] -fn all_tools_registers_seltz_lsp_and_tool_stats_when_enabled() { +fn all_tools_registers_optional_search_lsp_and_tool_stats_when_enabled() { let tmp = TempDir::new().unwrap(); let security = Arc::new(SecurityPolicy::default()); let mem = test_memory(&tmp); @@ -934,6 +934,7 @@ fn all_tools_registers_seltz_lsp_and_tool_stats_when_enabled() { let http = crate::openhuman::config::HttpRequestConfig::default(); let mut cfg = test_config(&tmp); cfg.seltz.enabled = true; + cfg.searxng.enabled = true; cfg.learning.enabled = true; cfg.learning.tool_tracking_enabled = true; @@ -958,7 +959,10 @@ fn all_tools_registers_seltz_lsp_and_tool_stats_when_enabled() { &cfg, ); let names = tool_names(&tools); - assert_contains_all(&names, &["seltz_search", "lsp", "tool_stats"]); + assert_contains_all( + &names, + &["seltz_search", "searxng_search", "lsp", "tool_stats"], + ); unsafe { std::env::remove_var(crate::openhuman::tools::implementations::LSP_ENABLED_ENV); diff --git a/src/openhuman/tools/schemas.rs b/src/openhuman/tools/schemas.rs index b23a2b820..f3d1d7343 100644 --- a/src/openhuman/tools/schemas.rs +++ b/src/openhuman/tools/schemas.rs @@ -11,6 +11,7 @@ use serde_json::{json, Map, Value}; use crate::core::all::{ControllerFuture, RegisteredController}; use crate::core::{ControllerSchema, FieldSchema, TypeSchema}; use crate::openhuman::config::rpc as config_rpc; +use crate::openhuman::integrations::searxng::MAX_RESULTS as SEARXNG_MAX_RESULTS; use crate::openhuman::tools::traits::Tool; use crate::rpc::RpcOutcome; @@ -19,6 +20,7 @@ pub fn all_controller_schemas() -> Vec { tools_schemas("tools_composio_execute"), tools_schemas("tools_web_search"), tools_schemas("tools_seltz_search"), + tools_schemas("tools_searxng_search"), tools_schemas("tools_apify_linkedin_scrape"), tools_schemas("tools_polymarket_execute"), ] @@ -38,6 +40,10 @@ pub fn all_registered_controllers() -> Vec { schema: tools_schemas("tools_seltz_search"), handler: handle_seltz_search, }, + RegisteredController { + schema: tools_schemas("tools_searxng_search"), + handler: handle_searxng_search, + }, RegisteredController { schema: tools_schemas("tools_apify_linkedin_scrape"), handler: handle_apify_linkedin_scrape, @@ -194,6 +200,50 @@ pub fn tools_schemas(function: &str) -> ControllerSchema { required: true, }], }, + "tools_searxng_search" => ControllerSchema { + namespace: "tools", + function: "searxng_search", + description: + "Web search via a user-configured SearXNG instance. Returns normalized \ + results with title, URL, snippet, and source. Intended for private, \ + self-hosted search without routing queries through the OpenHuman backend.", + inputs: vec![ + FieldSchema { + name: "query", + ty: TypeSchema::String, + comment: "Search query string.", + required: true, + }, + FieldSchema { + name: "categories", + ty: TypeSchema::Option(Box::new(TypeSchema::Array(Box::new( + TypeSchema::Enum { + variants: vec!["web", "general", "news", "images"], + }, + )))), + comment: "Optional SearXNG categories. `web` maps to SearXNG `general`.", + required: false, + }, + FieldSchema { + name: "language", + ty: TypeSchema::Option(Box::new(TypeSchema::String)), + comment: "Optional language code, e.g. `en`, `zh-CN`, or `fr`.", + required: false, + }, + FieldSchema { + name: "max_results", + ty: TypeSchema::Option(Box::new(TypeSchema::U64)), + comment: "Max results (1-50, default from searxng.max_results).", + required: false, + }, + ], + outputs: vec![FieldSchema { + name: "results", + ty: TypeSchema::Array(Box::new(TypeSchema::Json)), + comment: "Each item: {title, url, snippet, source}.", + required: true, + }], + }, "tools_apify_linkedin_scrape" => ControllerSchema { namespace: "tools", function: "apify_linkedin_scrape", @@ -464,6 +514,74 @@ fn handle_seltz_search(params: Map) -> ControllerFuture { }) } +fn handle_searxng_search(params: Map) -> ControllerFuture { + Box::pin(async move { + let query = params + .get("query") + .and_then(Value::as_str) + .map(str::trim) + .filter(|s| !s.is_empty()) + .map(str::to_string) + .ok_or_else(|| "missing or empty `query`".to_string())?; + let max_results = params + .get("max_results") + .and_then(Value::as_u64) + .map(|n| n.clamp(1, SEARXNG_MAX_RESULTS as u64) as usize); + let categories = optional_string_array(¶ms, "categories")?; + let language = params + .get("language") + .and_then(Value::as_str) + .map(str::trim) + .filter(|s| !s.is_empty()) + .map(str::to_string); + + let config = config_rpc::load_config_with_timeout().await?; + if !config.searxng.enabled { + tracing::debug!("[rpc][tools.searxng_search] searxng disabled — rejecting"); + return Err( + "SearXNG search is not enabled. Set searxng.enabled=true or OPENHUMAN_SEARXNG_ENABLED=true." + .to_string(), + ); + } + + tracing::debug!( + query_len = query.chars().count(), + max_results = max_results.unwrap_or(config.searxng.max_results), + category_count = categories.len(), + has_language = language.is_some(), + base_url = %config.searxng.base_url, + "[rpc][tools.searxng_search] start" + ); + + let tool = crate::openhuman::integrations::SearxngSearchTool::new( + config.searxng.base_url.clone(), + config.searxng.max_results, + config.searxng.default_language.clone(), + config.searxng.timeout_secs, + ); + + let response = tool + .search(crate::openhuman::integrations::SearxngSearchArgs { + query, + categories, + language, + max_results, + }) + .await + .map_err(|e| format!("searxng search failed: {e:#}"))?; + + let result_count = response.results.len(); + let payload = json!({ + "query": response.query, + "results": response.results, + }); + let log = vec![format!( + "[rpc][tools.searxng_search] success results={result_count}" + )]; + RpcOutcome::new(payload, log).into_cli_compatible_json() + }) +} + fn handle_apify_linkedin_scrape(params: Map) -> ControllerFuture { Box::pin(async move { let profile_url = params @@ -500,6 +618,28 @@ fn handle_apify_linkedin_scrape(params: Map) -> ControllerFuture }) } +fn optional_string_array(params: &Map, key: &str) -> Result, String> { + let Some(value) = params.get(key) else { + return Ok(Vec::new()); + }; + if value.is_null() { + return Ok(Vec::new()); + } + let items = value + .as_array() + .ok_or_else(|| format!("`{key}` must be an array of strings"))?; + items + .iter() + .filter_map(|item| match item.as_str() { + Some(value) => { + let trimmed = value.trim(); + (!trimmed.is_empty()).then(|| Ok(trimmed.to_string())) + } + None => Some(Err(format!("`{key}` must contain only strings"))), + }) + .collect() +} + fn handle_polymarket_execute(params: Map) -> ControllerFuture { Box::pin(async move { let action = params @@ -573,13 +713,13 @@ mod tests { use super::*; #[test] - fn all_schemas_returns_five() { - assert_eq!(all_controller_schemas().len(), 5); + fn all_schemas_returns_six() { + assert_eq!(all_controller_schemas().len(), 6); } #[test] - fn all_controllers_returns_five() { - assert_eq!(all_registered_controllers().len(), 5); + fn all_controllers_returns_six() { + assert_eq!(all_registered_controllers().len(), 6); } #[test] @@ -611,6 +751,26 @@ mod tests { assert!(s.inputs.iter().any(|f| f.name == "scope")); } + #[test] + fn searxng_search_schema_shape() { + let s = tools_schemas("tools_searxng_search"); + assert_eq!(s.namespace, "tools"); + assert_eq!(s.function, "searxng_search"); + assert!(s.inputs.iter().any(|f| f.name == "query" && f.required)); + assert!(s.inputs.iter().any(|f| f.name == "categories")); + assert!(s.inputs.iter().any(|f| f.name == "language")); + } + + #[test] + fn optional_string_array_trims_and_drops_blank_entries() { + let params = + Map::from_iter([("categories".to_string(), json!([" web ", "", " ", "news"]))]); + + let values = optional_string_array(¶ms, "categories").expect("string array"); + + assert_eq!(values, vec!["web", "news"]); + } + #[test] fn web_search_schema_shape() { let s = tools_schemas("tools_web_search");