diff --git a/src/openhuman/agent/agents/code_executor/agent.toml b/src/openhuman/agent/agents/code_executor/agent.toml index 69405214b..1c11321d9 100644 --- a/src/openhuman/agent/agents/code_executor/agent.toml +++ b/src/openhuman/agent/agents/code_executor/agent.toml @@ -14,4 +14,4 @@ omit_skills_catalog = true hint = "coding" [tools] -named = ["shell", "file_read", "file_write", "git_operations", "node_exec", "npm_exec"] +named = ["shell", "file_read", "file_write", "git_operations", "node_exec", "npm_exec", "curl"] diff --git a/src/openhuman/agent/agents/help/agent.toml b/src/openhuman/agent/agents/help/agent.toml new file mode 100644 index 000000000..977240374 --- /dev/null +++ b/src/openhuman/agent/agents/help/agent.toml @@ -0,0 +1,24 @@ +id = "help" +display_name = "Help" +delegate_name = "ask_docs" +when_to_use = "Product help — answers questions about how OpenHuman works, what features exist, how to configure things, or where to find a guide. Reads the OpenHuman GitBook docs via the `gitbooks_*` tools. Use this for any 'how do I…' / 'what does X do' / 'where is the setting for…' question about OpenHuman itself, before guessing or making things up." +temperature = 0.3 +max_iterations = 6 +sandbox_mode = "read_only" + +# Drop the standard identity/safety/skills/profile boilerplate — this +# agent has a narrow, single-purpose voice and no need for the full +# orchestrator-style preamble. Memory context is kept on so we can +# personalise references ("you mentioned earlier that you use X"). +omit_identity = true +omit_memory_context = false +omit_safety_preamble = true +omit_skills_catalog = true +omit_profile = false +omit_memory_md = false + +[model] +hint = "agentic" + +[tools] +named = ["gitbooks_search", "gitbooks_get_page", "memory_recall"] diff --git a/src/openhuman/agent/agents/help/mod.rs b/src/openhuman/agent/agents/help/mod.rs new file mode 100644 index 000000000..8bf84783c --- /dev/null +++ b/src/openhuman/agent/agents/help/mod.rs @@ -0,0 +1 @@ +pub mod prompt; diff --git a/src/openhuman/agent/agents/help/prompt.md b/src/openhuman/agent/agents/help/prompt.md new file mode 100644 index 000000000..98b9a533c --- /dev/null +++ b/src/openhuman/agent/agents/help/prompt.md @@ -0,0 +1,51 @@ +# Help Agent + +You are the **Help** agent — OpenHuman's product docs specialist. Your job is to answer questions about **OpenHuman itself** by searching the official documentation and giving the user a direct, grounded answer with links to the relevant pages. + +## Tone + +- **Direct and concrete.** Answer the question. Don't restate it back. +- **Cite the docs.** When you use information from a search hit or page, include the page link in your reply so the user can read more. +- **Short.** A sentence or two when that's enough; a tight bulleted list when there are real steps. +- **Honest about gaps.** If the docs don't cover something, say so plainly — do not invent features, flags, or commands. + +## How to work + +You have three tools: + +- `gitbooks_search { query }` — returns excerpts from the OpenHuman GitBook docs along with page titles and URLs. Always start here. +- `gitbooks_get_page { url }` — fetches the full markdown of a page. Use it only when the search excerpt does not contain enough detail to answer the question. +- `memory_recall { query, ... }` — pulls relevant past context about this user. Use sparingly, only when the user's question depends on something they told you before. + +### Standard flow + +1. **Search first.** Call `gitbooks_search` with a focused query that mirrors the user's intent, not their literal phrasing. Prefer feature names ("screen intelligence", "cron", "skills", "MCP") over filler verbs. +2. **Read the excerpts.** If one of them clearly answers the question, write the answer in your own words and link the page. Done. +3. **Drill in if needed.** If the excerpts are too partial, call `gitbooks_get_page` on the most promising URL, then answer. +4. **Refine the search.** If the first query missed, reformulate (different keywords, narrower scope) and try once more before admitting you cannot find it. + +### What you do NOT do + +- Do not run shell commands, write files, edit configuration, or call other tools. Help is read-only — you point to docs, you do not change the system. +- Do not invent commands, config keys, env vars, or feature names. If GitBook does not mention it, treat it as not documented. +- Do not delegate by spawning sub-agents. Stay in your lane. + +## Output shape + +When the answer is short: + +> The morning-briefing agent runs at the time you set under `[scheduler.morning_briefing.cron]` in `config.toml`. By default that's 7 AM local. ([source](https://tinyhumans.gitbook.io/openhuman/...)) + +When there are steps, use a tight numbered list and link the source at the end: + +> 1. Open Settings → Skills. +> 2. Click **Connect** next to Gmail. +> 3. Authorize in the popup. +> +> ([source](https://tinyhumans.gitbook.io/openhuman/...)) + +When the docs do not cover the question: + +> The OpenHuman docs don't cover that. You may want to check the GitHub repo or ask in the community channel. + +Keep it that simple. diff --git a/src/openhuman/agent/agents/help/prompt.rs b/src/openhuman/agent/agents/help/prompt.rs new file mode 100644 index 000000000..8d738de80 --- /dev/null +++ b/src/openhuman/agent/agents/help/prompt.rs @@ -0,0 +1,68 @@ +//! System prompt builder for the `help` built-in agent. +//! +//! Help is a read-only docs-grounded agent. The body is straightforward +//! — render the archetype, then the standard tools + workspace blocks +//! so the model sees the `gitbooks_*` schemas the runtime injected. + +use crate::openhuman::context::prompt::{ + render_tools, render_user_files, render_workspace, PromptContext, +}; +use anyhow::Result; + +const ARCHETYPE: &str = include_str!("prompt.md"); + +pub fn build(ctx: &PromptContext<'_>) -> Result { + let mut out = String::with_capacity(4096); + out.push_str(ARCHETYPE.trim_end()); + out.push_str("\n\n"); + + let user_files = render_user_files(ctx)?; + if !user_files.trim().is_empty() { + out.push_str(user_files.trim_end()); + out.push_str("\n\n"); + } + + let tools = render_tools(ctx)?; + if !tools.trim().is_empty() { + out.push_str(tools.trim_end()); + out.push_str("\n\n"); + } + + let workspace = render_workspace(ctx)?; + if !workspace.trim().is_empty() { + out.push_str(workspace.trim_end()); + out.push('\n'); + } + + Ok(out) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::openhuman::context::prompt::{LearnedContextData, ToolCallFormat}; + use std::collections::HashSet; + + #[test] + fn build_returns_nonempty_body() { + let visible: HashSet = HashSet::new(); + let ctx = PromptContext { + workspace_dir: std::path::Path::new("."), + model_name: "test", + agent_id: "help", + tools: &[], + skills: &[], + dispatcher_instructions: "", + learned: LearnedContextData::default(), + visible_tool_names: &visible, + tool_call_format: ToolCallFormat::PFormat, + connected_integrations: &[], + connected_identities_md: String::new(), + include_profile: false, + include_memory_md: false, + }; + let body = build(&ctx).unwrap(); + assert!(!body.is_empty()); + assert!(body.contains("Help Agent")); + } +} diff --git a/src/openhuman/agent/agents/loader.rs b/src/openhuman/agent/agents/loader.rs index f0dc1cbef..cf78c9eda 100644 --- a/src/openhuman/agent/agents/loader.rs +++ b/src/openhuman/agent/agents/loader.rs @@ -127,6 +127,11 @@ pub const BUILTINS: &[BuiltinAgent] = &[ toml: include_str!("summarizer/agent.toml"), prompt_fn: super::summarizer::prompt::build, }, + BuiltinAgent { + id: "help", + toml: include_str!("help/agent.toml"), + prompt_fn: super::help::prompt::build, + }, ]; /// Parse every entry in [`BUILTINS`] into an [`AgentDefinition`]. @@ -172,7 +177,7 @@ mod tests { fn all_builtins_parse() { let defs = load_builtins().expect("built-in TOML must parse"); assert_eq!(defs.len(), BUILTINS.len()); - assert_eq!(defs.len(), 14, "expected 14 built-in agents"); + assert_eq!(defs.len(), 15, "expected 15 built-in agents"); } #[test] @@ -365,13 +370,89 @@ mod tests { assert_eq!(def.max_iterations, 8); } + #[test] + fn help_uses_gitbooks_tools_and_is_read_only() { + let def = find("help"); + assert_eq!(def.sandbox_mode, SandboxMode::ReadOnly); + match &def.tools { + ToolScope::Named(tools) => { + assert!( + tools.iter().any(|t| t == "gitbooks_search"), + "help needs gitbooks_search" + ); + assert!( + tools.iter().any(|t| t == "gitbooks_get_page"), + "help needs gitbooks_get_page" + ); + assert!( + tools.iter().any(|t| t == "memory_recall"), + "help needs memory_recall for personalisation" + ); + // Help is docs-only — no write/exec tools. + assert!(!tools.iter().any(|t| t == "shell")); + assert!(!tools.iter().any(|t| t == "file_write")); + assert!(!tools.iter().any(|t| t == "curl")); + assert!(!tools.iter().any(|t| t == "spawn_subagent")); + } + ToolScope::Wildcard => panic!("help must have a Named tool scope"), + } + assert!(def.omit_identity); + assert!(def.omit_safety_preamble); + assert!(!def.omit_memory_context); + } + + #[test] + fn researcher_has_curl_for_artifact_downloads() { + let def = find("researcher"); + match &def.tools { + ToolScope::Named(tools) => { + assert!( + tools.iter().any(|t| t == "curl"), + "researcher needs curl for artifact downloads" + ); + assert!( + tools.iter().any(|t| t == "http_request"), + "researcher still needs http_request" + ); + } + ToolScope::Wildcard => panic!("researcher must have Named tool scope"), + } + } + + #[test] + fn code_executor_has_curl_for_artifact_downloads() { + let def = find("code_executor"); + match &def.tools { + ToolScope::Named(tools) => { + assert!( + tools.iter().any(|t| t == "curl"), + "code_executor needs curl for artifact/dataset fetches" + ); + } + ToolScope::Wildcard => panic!("code_executor must have Named tool scope"), + } + } + + #[test] + fn orchestrator_does_not_get_curl() { + // Per design: curl is a `Write` permission tool that writes + // to the workspace. The orchestrator delegates rather than + // executing — code_executor / researcher own actual downloads. + let def = find("orchestrator"); + if let ToolScope::Named(tools) = &def.tools { + assert!( + !tools.iter().any(|t| t == "curl"), + "orchestrator must not have curl — it should delegate" + ); + } + } + #[test] fn welcome_has_onboarding_and_memory_tools() { let def = find("welcome"); assert_eq!(def.sandbox_mode, SandboxMode::ReadOnly); match &def.tools { ToolScope::Named(tools) => { - assert_eq!(tools.len(), 4, "welcome should have exactly four tools"); assert!( tools.iter().any(|t| t == "check_onboarding_status"), "welcome needs check_onboarding_status" @@ -388,6 +469,18 @@ mod tests { tools.iter().any(|t| t == "composio_authorize"), "welcome needs composio_authorize" ); + assert!( + tools.iter().any(|t| t == "gitbooks_search"), + "welcome needs gitbooks_search to answer 'how does X work' during onboarding" + ); + assert!( + tools.iter().any(|t| t == "gitbooks_get_page"), + "welcome needs gitbooks_get_page for full-page lookups" + ); + // Welcome must not gain write/exec power; onboarding stays read-only. + assert!(!tools.iter().any(|t| t == "shell")); + assert!(!tools.iter().any(|t| t == "file_write")); + assert!(!tools.iter().any(|t| t == "curl")); } ToolScope::Wildcard => panic!("welcome must have a Named tool scope"), } diff --git a/src/openhuman/agent/agents/mod.rs b/src/openhuman/agent/agents/mod.rs index 9d519e4b5..b16562b65 100644 --- a/src/openhuman/agent/agents/mod.rs +++ b/src/openhuman/agent/agents/mod.rs @@ -7,6 +7,7 @@ mod loader; pub mod archivist; pub mod code_executor; pub mod critic; +pub mod help; pub mod integrations_agent; pub mod morning_briefing; pub mod orchestrator; diff --git a/src/openhuman/agent/agents/researcher/agent.toml b/src/openhuman/agent/agents/researcher/agent.toml index 46be95e70..db5a77a70 100644 --- a/src/openhuman/agent/agents/researcher/agent.toml +++ b/src/openhuman/agent/agents/researcher/agent.toml @@ -14,4 +14,4 @@ omit_skills_catalog = true hint = "agentic" [tools] -named = ["http_request", "web_search", "file_read", "memory_recall"] +named = ["http_request", "curl", "web_search", "file_read", "memory_recall"] diff --git a/src/openhuman/agent/agents/welcome/agent.toml b/src/openhuman/agent/agents/welcome/agent.toml index fd58bc2d9..6f55dd197 100644 --- a/src/openhuman/agent/agents/welcome/agent.toml +++ b/src/openhuman/agent/agents/welcome/agent.toml @@ -30,9 +30,13 @@ hint = "agentic" # complete_onboarding: finalize the welcome flow once ready_to_complete is true. # memory_recall: pull additional user details beyond injected context. # composio_authorize: start an OAuth flow for a toolkit (e.g. gmail). +# gitbooks_*: answer "how does X work" / "what can OpenHuman do" questions +# during onboarding from the real product docs instead of guessing. named = [ "check_onboarding_status", "complete_onboarding", "memory_recall", "composio_authorize", + "gitbooks_search", + "gitbooks_get_page", ] diff --git a/src/openhuman/agent/agents/welcome/prompt.md b/src/openhuman/agent/agents/welcome/prompt.md index 7b4fb4286..0e255c2a4 100644 --- a/src/openhuman/agent/agents/welcome/prompt.md +++ b/src/openhuman/agent/agents/welcome/prompt.md @@ -11,7 +11,9 @@ You are the **Welcome** agent — the first agent a new user talks to in OpenHum ## Tools you must use correctly -You have `check_onboarding_status`, `complete_onboarding`, `memory_recall`, and `composio_authorize`. +You have `check_onboarding_status`, `complete_onboarding`, `memory_recall`, `composio_authorize`, `gitbooks_search`, and `gitbooks_get_page`. + +If the user asks a "how does X work" / "what can OpenHuman do" / "where is the setting for…" question during onboarding, call `gitbooks_search` first and ground your answer in the real docs — do **not** guess feature names, flags, or commands. Use `gitbooks_get_page` only when the search excerpt is too partial. Cite the page URL inline so they can read more. Keep doc-grounded replies short and steer the conversation back to setup once they're satisfied — your primary job is still onboarding, not an open-ended Q&A loop. | Tool | What it does | |------|--------------| diff --git a/src/openhuman/config/mod.rs b/src/openhuman/config/mod.rs index 67580ef16..117b3723b 100644 --- a/src/openhuman/config/mod.rs +++ b/src/openhuman/config/mod.rs @@ -26,15 +26,16 @@ pub use schema::{ build_runtime_proxy_client_with_timeouts, runtime_proxy_config, set_runtime_proxy_config, AgentConfig, AuditConfig, AutocompleteConfig, AutonomyConfig, BrowserComputerUseConfig, BrowserConfig, ChannelsConfig, ComposioConfig, Config, ContextConfig, CostConfig, CronConfig, - DelegateAgentConfig, DictationActivationMode, DictationConfig, DiscordConfig, - DockerRuntimeConfig, EmbeddingRouteConfig, HeartbeatConfig, HttpRequestConfig, IMessageConfig, - IntegrationToggle, IntegrationsConfig, LarkConfig, LearningConfig, LocalAiConfig, MatrixConfig, - MemoryConfig, ModelRouteConfig, MultimodalConfig, ObservabilityConfig, ProxyConfig, ProxyScope, - ReflectionSource, ReliabilityConfig, ResourceLimitsConfig, RuntimeConfig, SandboxBackend, - SandboxConfig, SchedulerConfig, ScreenIntelligenceConfig, SecretsConfig, SecurityConfig, - SlackConfig, StorageConfig, StorageProviderConfig, StorageProviderSection, StreamMode, - TelegramConfig, UpdateConfig, VoiceActivationMode, VoiceServerConfig, WebSearchConfig, - WebhookConfig, DEFAULT_MODEL, MODEL_AGENTIC_V1, MODEL_CODING_V1, MODEL_REASONING_V1, + CurlConfig, DelegateAgentConfig, DictationActivationMode, DictationConfig, DiscordConfig, + DockerRuntimeConfig, EmbeddingRouteConfig, GitbooksConfig, HeartbeatConfig, HttpRequestConfig, + IMessageConfig, IntegrationToggle, IntegrationsConfig, LarkConfig, LearningConfig, + LocalAiConfig, MatrixConfig, MemoryConfig, ModelRouteConfig, MultimodalConfig, + ObservabilityConfig, ProxyConfig, ProxyScope, ReflectionSource, ReliabilityConfig, + ResourceLimitsConfig, RuntimeConfig, SandboxBackend, SandboxConfig, SchedulerConfig, + ScreenIntelligenceConfig, SecretsConfig, SecurityConfig, SlackConfig, StorageConfig, + StorageProviderConfig, StorageProviderSection, StreamMode, TelegramConfig, UpdateConfig, + VoiceActivationMode, VoiceServerConfig, WebSearchConfig, WebhookConfig, DEFAULT_MODEL, + MODEL_AGENTIC_V1, MODEL_CODING_V1, MODEL_REASONING_V1, }; pub use schema::{ clear_active_user, default_root_openhuman_dir, pre_login_user_dir, read_active_user_id, diff --git a/src/openhuman/config/schema/mod.rs b/src/openhuman/config/schema/mod.rs index 0c77cb7ba..b3e8f3e08 100644 --- a/src/openhuman/config/schema/mod.rs +++ b/src/openhuman/config/schema/mod.rs @@ -57,9 +57,9 @@ pub use storage_memory::{ MemoryConfig, MemoryTreeConfig, StorageConfig, StorageProviderConfig, StorageProviderSection, }; pub use tools::{ - BrowserComputerUseConfig, BrowserConfig, ComposioConfig, ComputerControlConfig, - HttpRequestConfig, IntegrationToggle, IntegrationsConfig, MultimodalConfig, SecretsConfig, - WebSearchConfig, + BrowserComputerUseConfig, BrowserConfig, ComposioConfig, ComputerControlConfig, CurlConfig, + GitbooksConfig, HttpRequestConfig, IntegrationToggle, IntegrationsConfig, MultimodalConfig, + SecretsConfig, WebSearchConfig, }; pub use update::UpdateConfig; mod voice_server; diff --git a/src/openhuman/config/schema/tools.rs b/src/openhuman/config/schema/tools.rs index aa4fda071..4e4c9eaa0 100644 --- a/src/openhuman/config/schema/tools.rs +++ b/src/openhuman/config/schema/tools.rs @@ -148,6 +148,75 @@ fn default_http_timeout_secs() -> u64 { 30 } +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +pub struct CurlConfig { + /// Subdirectory under `workspace_dir` where downloads land. Inputs + /// are resolved relative to this root; absolute paths and `..` + /// segments are rejected. + #[serde(default = "default_curl_dest_subdir")] + pub dest_subdir: String, + /// Hard byte ceiling per download. Streaming aborts and the + /// partial file is removed if exceeded. + #[serde(default = "default_curl_max_download_bytes")] + pub max_download_bytes: u64, + /// Per-request timeout in seconds. + #[serde(default = "default_curl_timeout_secs")] + pub timeout_secs: u64, +} + +fn default_curl_dest_subdir() -> String { + "downloads".into() +} + +fn default_curl_max_download_bytes() -> u64 { + 50 * 1024 * 1024 +} + +fn default_curl_timeout_secs() -> u64 { + 120 +} + +impl Default for CurlConfig { + fn default() -> Self { + Self { + dest_subdir: default_curl_dest_subdir(), + max_download_bytes: default_curl_max_download_bytes(), + timeout_secs: default_curl_timeout_secs(), + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +pub struct GitbooksConfig { + /// When `true`, register `gitbooks_search` and `gitbooks_get_page`. + #[serde(default = "defaults::default_true")] + pub enabled: bool, + /// MCP endpoint URL for the OpenHuman GitBook docs. + #[serde(default = "default_gitbooks_endpoint")] + pub endpoint: String, + /// Per-request timeout in seconds. + #[serde(default = "default_gitbooks_timeout_secs")] + pub timeout_secs: u64, +} + +fn default_gitbooks_endpoint() -> String { + "https://tinyhumans.gitbook.io/openhuman/~gitbook/mcp".into() +} + +fn default_gitbooks_timeout_secs() -> u64 { + 30 +} + +impl Default for GitbooksConfig { + fn default() -> Self { + Self { + enabled: defaults::default_true(), + endpoint: default_gitbooks_endpoint(), + timeout_secs: default_gitbooks_timeout_secs(), + } + } +} + #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] pub struct WebSearchConfig { #[serde(default = "default_web_search_max_results")] diff --git a/src/openhuman/config/schema/types.rs b/src/openhuman/config/schema/types.rs index edd442b9f..e834d0ebc 100644 --- a/src/openhuman/config/schema/types.rs +++ b/src/openhuman/config/schema/types.rs @@ -103,6 +103,12 @@ pub struct Config { #[serde(default)] pub http_request: HttpRequestConfig, + #[serde(default)] + pub curl: CurlConfig, + + #[serde(default)] + pub gitbooks: GitbooksConfig, + #[serde(default)] pub multimodal: MultimodalConfig, @@ -240,6 +246,8 @@ impl Default for Config { secrets: SecretsConfig::default(), browser: BrowserConfig::default(), http_request: HttpRequestConfig::default(), + curl: CurlConfig::default(), + gitbooks: GitbooksConfig::default(), multimodal: MultimodalConfig::default(), web_search: WebSearchConfig::default(), proxy: ProxyConfig::default(), diff --git a/src/openhuman/tools/impl/network/curl.rs b/src/openhuman/tools/impl/network/curl.rs new file mode 100644 index 000000000..1cdfd67ee --- /dev/null +++ b/src/openhuman/tools/impl/network/curl.rs @@ -0,0 +1,544 @@ +//! `curl` — download files from the web to a path under the workspace. +//! +//! Distinct from `http_request`: instead of returning the body inline +//! (size-capped), `curl` streams to disk with a hard byte ceiling. Same +//! SSRF/allowlist guards (shared via `url_guard`), shares +//! `http_request.allowed_domains` so there is one allowlist to reason +//! about. + +use super::url_guard::{normalize_allowed_domains, validate_url}; +use crate::openhuman::security::SecurityPolicy; +use crate::openhuman::tools::traits::{PermissionLevel, Tool, ToolResult}; +use async_trait::async_trait; +use futures_util::StreamExt; +use sha2::{Digest, Sha256}; +use std::path::{Component, Path, PathBuf}; +use std::sync::Arc; +use std::time::Duration; +use tokio::fs; +use tokio::io::AsyncWriteExt; + +pub struct CurlTool { + security: Arc, + allowed_domains: Vec, + workspace_dir: PathBuf, + dest_subdir: String, + max_download_bytes: u64, + timeout_secs: u64, +} + +impl CurlTool { + pub fn new( + security: Arc, + allowed_domains: Vec, + workspace_dir: PathBuf, + dest_subdir: String, + max_download_bytes: u64, + timeout_secs: u64, + ) -> Self { + Self { + security, + allowed_domains: normalize_allowed_domains(allowed_domains), + workspace_dir, + dest_subdir: sanitize_dest_subdir(&dest_subdir), + max_download_bytes, + timeout_secs, + } + } + + /// Resolve a user-supplied dest path to an absolute path inside + /// `/`. Rejects absolute paths, `..` + /// segments, and any other escape attempts. + fn resolve_dest(&self, dest: &str) -> anyhow::Result { + let trimmed = dest.trim(); + if trimmed.is_empty() { + anyhow::bail!("dest_path cannot be empty"); + } + + let p = Path::new(trimmed); + if p.is_absolute() { + anyhow::bail!("dest_path must be relative — got absolute path"); + } + + for component in p.components() { + match component { + Component::Normal(_) => {} + Component::CurDir => {} + Component::ParentDir => { + anyhow::bail!("dest_path may not contain '..'"); + } + Component::Prefix(_) | Component::RootDir => { + anyhow::bail!("dest_path must be relative"); + } + } + } + + let root = self.workspace_dir.join(&self.dest_subdir); + let resolved = root.join(p); + + // Belt-and-braces: ensure the resolved path still lives under root. + // Lexical check is sufficient because we already rejected `..`. + if !resolved.starts_with(&root) { + anyhow::bail!("dest_path resolves outside the downloads root"); + } + + Ok(resolved) + } + + fn validate_url(&self, raw_url: &str) -> anyhow::Result { + validate_url(raw_url, &self.allowed_domains) + } + + fn default_filename_from_url(url: &str) -> String { + let after_scheme = url.split_once("://").map(|(_, rest)| rest).unwrap_or(url); + let path_part = after_scheme.split_once('/').map(|(_, p)| p).unwrap_or(""); + let last = path_part + .split('?') + .next() + .unwrap_or("") + .rsplit('/') + .next() + .unwrap_or(""); + let cleaned: String = last + .chars() + .filter(|c| c.is_alphanumeric() || matches!(c, '.' | '-' | '_')) + .collect(); + if cleaned.is_empty() { + "download.bin".into() + } else { + cleaned + } + } +} + +#[async_trait] +impl Tool for CurlTool { + fn name(&self) -> &str { + "curl" + } + + fn description(&self) -> &str { + "Download a file from an http(s) URL into the workspace. The body is streamed to disk \ + with a hard byte ceiling. Same allowlist as `http_request`. Returns the saved path, \ + bytes written, content-type, and SHA-256 of the file." + } + + fn parameters_schema(&self) -> serde_json::Value { + serde_json::json!({ + "type": "object", + "properties": { + "url": { + "type": "string", + "description": "HTTP or HTTPS URL of the file to download" + }, + "dest_path": { + "type": "string", + "description": "Destination path relative to the downloads root inside the workspace. No '..' or absolute paths. If omitted, the filename is inferred from the URL." + }, + "headers": { + "type": "object", + "description": "Optional HTTP headers (e.g. {\"Authorization\": \"Bearer …\"})", + "default": {} + } + }, + "required": ["url"] + }) + } + + fn permission_level(&self) -> PermissionLevel { + PermissionLevel::Write + } + + async fn execute(&self, args: serde_json::Value) -> anyhow::Result { + let url = args + .get("url") + .and_then(|v| v.as_str()) + .ok_or_else(|| anyhow::anyhow!("Missing 'url' parameter"))?; + + let dest_arg = args.get("dest_path").and_then(|v| v.as_str()); + let headers_val = args + .get("headers") + .cloned() + .unwrap_or_else(|| serde_json::json!({})); + + if !self.security.can_act() { + tracing::debug!(target: "[curl]", url = %url, "blocked: autonomy read-only"); + return Ok(ToolResult::error("Action blocked: autonomy is read-only")); + } + if !self.security.record_action() { + tracing::debug!(target: "[curl]", url = %url, "blocked: rate limit"); + return Ok(ToolResult::error("Action blocked: rate limit exceeded")); + } + + let url = match self.validate_url(url) { + Ok(v) => v, + Err(e) => { + tracing::debug!(target: "[curl]", url = %url, reason = %e, "url validation failed"); + return Ok(ToolResult::error(e.to_string())); + } + }; + + let dest = match dest_arg { + Some(d) => d.to_string(), + None => Self::default_filename_from_url(&url), + }; + let dest_path = match self.resolve_dest(&dest) { + Ok(p) => p, + Err(e) => { + tracing::debug!(target: "[curl]", url = %url, dest = %dest, reason = %e, "dest_path rejected"); + return Ok(ToolResult::error(e.to_string())); + } + }; + + if let Some(parent) = dest_path.parent() { + if let Err(e) = fs::create_dir_all(parent).await { + tracing::error!(target: "[curl]", url = %url, dest = %dest_path.display(), reason = %e, "create_dir_all failed"); + return Ok(ToolResult::error(format!( + "Failed to create destination directory: {e}" + ))); + } + } + + let builder = reqwest::Client::builder() + .timeout(Duration::from_secs(self.timeout_secs)) + .connect_timeout(Duration::from_secs(10)) + .redirect(reqwest::redirect::Policy::none()); + let builder = + crate::openhuman::config::apply_runtime_proxy_to_builder(builder, "tool.curl"); + let client = match builder.build() { + Ok(c) => c, + Err(e) => { + tracing::error!(target: "[curl]", reason = %e, "HTTP client build failed"); + return Ok(ToolResult::error(format!("HTTP client build failed: {e}"))); + } + }; + + let mut request = client.get(&url); + if let Some(obj) = headers_val.as_object() { + for (k, v) in obj { + if let Some(s) = v.as_str() { + request = request.header(k, s); + } + } + } + + tracing::debug!(target: "[curl]", url = %url, dest = %dest_path.display(), "starting download"); + + let response = match request.send().await { + Ok(r) => r, + Err(e) => { + tracing::error!(target: "[curl]", url = %url, reason = %e, "request send failed"); + return Ok(ToolResult::error(format!("Request failed: {e}"))); + } + }; + + let status = response.status(); + if !status.is_success() { + tracing::debug!(target: "[curl]", url = %url, status = %status.as_u16(), "non-success HTTP status"); + return Ok(ToolResult::error(format!( + "HTTP {} {}", + status.as_u16(), + status.canonical_reason().unwrap_or("Unknown") + ))); + } + + let content_type = response + .headers() + .get(reqwest::header::CONTENT_TYPE) + .and_then(|v| v.to_str().ok()) + .unwrap_or("application/octet-stream") + .to_string(); + + let mut file = match fs::File::create(&dest_path).await { + Ok(f) => f, + Err(e) => { + tracing::error!(target: "[curl]", dest = %dest_path.display(), reason = %e, "fs::File::create failed"); + return Ok(ToolResult::error(format!( + "Failed to create destination file: {e}" + ))); + } + }; + + let mut hasher = Sha256::new(); + let mut bytes_written: u64 = 0; + let mut stream = response.bytes_stream(); + + while let Some(chunk) = stream.next().await { + let chunk = match chunk { + Ok(c) => c, + Err(e) => { + drop(file); + if let Err(rm) = fs::remove_file(&dest_path).await { + tracing::debug!(target: "[curl]", dest = %dest_path.display(), reason = %rm, "cleanup remove_file failed"); + } + tracing::error!(target: "[curl]", url = %url, bytes_written, reason = %e, "stream error"); + return Ok(ToolResult::error(format!("Stream error: {e}"))); + } + }; + + if bytes_written.saturating_add(chunk.len() as u64) > self.max_download_bytes { + let _ = file.flush().await; + drop(file); + if let Err(rm) = fs::remove_file(&dest_path).await { + tracing::debug!(target: "[curl]", dest = %dest_path.display(), reason = %rm, "cleanup remove_file failed"); + } + tracing::error!(target: "[curl]", url = %url, bytes_written, max = self.max_download_bytes, "size cap exceeded — download aborted"); + return Ok(ToolResult::error(format!( + "Download exceeded max_download_bytes ({} bytes)", + self.max_download_bytes + ))); + } + + if let Err(e) = file.write_all(&chunk).await { + drop(file); + if let Err(rm) = fs::remove_file(&dest_path).await { + tracing::debug!(target: "[curl]", dest = %dest_path.display(), reason = %rm, "cleanup remove_file failed"); + } + tracing::error!(target: "[curl]", dest = %dest_path.display(), bytes_written, reason = %e, "write_all failed"); + return Ok(ToolResult::error(format!("Write failed: {e}"))); + } + hasher.update(&chunk); + bytes_written += chunk.len() as u64; + } + + if let Err(e) = file.flush().await { + drop(file); + if let Err(rm) = fs::remove_file(&dest_path).await { + tracing::debug!(target: "[curl]", dest = %dest_path.display(), reason = %rm, "cleanup remove_file failed"); + } + tracing::error!(target: "[curl]", dest = %dest_path.display(), bytes_written, reason = %e, "flush failed"); + return Ok(ToolResult::error(format!("Flush failed: {e}"))); + } + + let sha256 = format!("{:x}", hasher.finalize()); + + tracing::debug!( + target: "[curl]", + url = %url, + dest = %dest_path.display(), + bytes = bytes_written, + content_type = %content_type, + sha256 = %sha256, + "download complete" + ); + + let payload = serde_json::json!({ + "path": dest_path.display().to_string(), + "bytes_written": bytes_written, + "content_type": content_type, + "sha256": sha256, + }); + Ok(ToolResult::success(payload.to_string())) + } +} + +/// Sanitize the configured `dest_subdir` so a malicious or misconfigured +/// `[curl].dest_subdir` cannot escape the workspace via absolute paths +/// or `..` segments. Drops disallowed components rather than panicking; +/// falls back to `"downloads"` if everything is filtered out. +fn sanitize_dest_subdir(raw: &str) -> String { + let trimmed = raw.trim(); + if trimmed.is_empty() { + return "downloads".into(); + } + let p = Path::new(trimmed); + let mut buf = PathBuf::new(); + for component in p.components() { + match component { + Component::Normal(c) => buf.push(c), + // Drop everything else: absolute roots, prefixes, parent dirs, cur dirs. + _ => continue, + } + } + if buf.as_os_str().is_empty() { + return "downloads".into(); + } + buf.to_string_lossy().into_owned() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::openhuman::security::SecurityPolicy; + use tempfile::TempDir; + + fn tool(tmp: &TempDir, allow: Vec<&str>) -> CurlTool { + CurlTool::new( + Arc::new(SecurityPolicy::default()), + allow.into_iter().map(String::from).collect(), + tmp.path().to_path_buf(), + "downloads".into(), + 1024 * 1024, + 30, + ) + } + + #[test] + fn sanitize_dest_subdir_strips_absolute_paths() { + assert_eq!(sanitize_dest_subdir("/etc/passwd"), "etc/passwd"); + assert_eq!(sanitize_dest_subdir("//foo"), "foo"); + } + + #[test] + fn sanitize_dest_subdir_strips_parent_segments() { + assert_eq!(sanitize_dest_subdir("../../etc"), "etc"); + assert_eq!(sanitize_dest_subdir("a/../b"), "a/b"); + } + + #[test] + fn sanitize_dest_subdir_falls_back_to_downloads() { + assert_eq!(sanitize_dest_subdir(""), "downloads"); + assert_eq!(sanitize_dest_subdir(" "), "downloads"); + assert_eq!(sanitize_dest_subdir(".."), "downloads"); + assert_eq!(sanitize_dest_subdir("/"), "downloads"); + } + + #[test] + fn sanitize_dest_subdir_keeps_normal_paths() { + assert_eq!(sanitize_dest_subdir("downloads"), "downloads"); + assert_eq!(sanitize_dest_subdir("artifacts/build"), "artifacts/build"); + } + + #[test] + fn new_sanitizes_malicious_dest_subdir() { + let tmp = TempDir::new().unwrap(); + let t = CurlTool::new( + Arc::new(SecurityPolicy::default()), + vec!["example.com".into()], + tmp.path().to_path_buf(), + "../../etc".into(), + 1024, + 30, + ); + let resolved = t.resolve_dest("file.txt").unwrap(); + // Sanitizer reduced "../../etc" to "etc"; resolution must stay under workspace. + assert!(resolved.starts_with(tmp.path().join("etc"))); + assert!(resolved.starts_with(tmp.path())); + } + + #[test] + fn resolve_dest_normal() { + let tmp = TempDir::new().unwrap(); + let t = tool(&tmp, vec!["example.com"]); + let p = t.resolve_dest("foo/bar.txt").unwrap(); + assert!(p.starts_with(tmp.path().join("downloads"))); + assert!(p.ends_with("foo/bar.txt")); + } + + #[test] + fn resolve_dest_rejects_absolute() { + let tmp = TempDir::new().unwrap(); + let t = tool(&tmp, vec!["example.com"]); + let err = t.resolve_dest("/etc/passwd").unwrap_err().to_string(); + assert!(err.contains("relative")); + } + + #[test] + fn resolve_dest_rejects_parent_dir() { + let tmp = TempDir::new().unwrap(); + let t = tool(&tmp, vec!["example.com"]); + let err = t.resolve_dest("../etc/passwd").unwrap_err().to_string(); + assert!(err.contains("..")); + } + + #[test] + fn resolve_dest_rejects_nested_parent_dir() { + let tmp = TempDir::new().unwrap(); + let t = tool(&tmp, vec!["example.com"]); + let err = t.resolve_dest("a/../../b").unwrap_err().to_string(); + assert!(err.contains("..")); + } + + #[test] + fn resolve_dest_rejects_empty() { + let tmp = TempDir::new().unwrap(); + let t = tool(&tmp, vec!["example.com"]); + assert!(t.resolve_dest("").is_err()); + assert!(t.resolve_dest(" ").is_err()); + } + + #[test] + fn default_filename_from_url_basic() { + assert_eq!( + CurlTool::default_filename_from_url("https://example.com/foo/bar.zip"), + "bar.zip" + ); + } + + #[test] + fn default_filename_from_url_query_stripped() { + assert_eq!( + CurlTool::default_filename_from_url("https://example.com/file.tar.gz?token=x"), + "file.tar.gz" + ); + } + + #[test] + fn default_filename_from_url_root_falls_back() { + assert_eq!( + CurlTool::default_filename_from_url("https://example.com/"), + "download.bin" + ); + } + + #[tokio::test] + async fn execute_blocks_when_rate_limited() { + let tmp = TempDir::new().unwrap(); + let security = Arc::new(SecurityPolicy { + max_actions_per_hour: 0, + ..SecurityPolicy::default() + }); + let t = CurlTool::new( + security, + vec!["example.com".into()], + tmp.path().into(), + "downloads".into(), + 1024, + 30, + ); + let result = t + .execute(serde_json::json!({"url": "https://example.com/x"})) + .await + .unwrap(); + assert!(result.is_error); + assert!(result.output().contains("rate limit")); + } + + /// Live integration smoke: downloads example.com (a tiny, stable + /// public page). Gated behind `OPENHUMAN_CURL_LIVE_TEST=1` so CI / + /// offline runs don't depend on the network. + #[tokio::test] + async fn live_download_example_com() { + if std::env::var("OPENHUMAN_CURL_LIVE_TEST").ok().as_deref() != Some("1") { + return; + } + let tmp = TempDir::new().unwrap(); + let t = tool(&tmp, vec!["example.com"]); + let result = t + .execute(serde_json::json!({ + "url": "https://example.com/", + "dest_path": "example.html" + })) + .await + .unwrap(); + assert!(!result.is_error, "live curl errored: {}", result.output()); + let payload: serde_json::Value = serde_json::from_str(&result.output()).unwrap(); + let bytes = payload["bytes_written"].as_u64().unwrap(); + assert!(bytes > 100, "unexpectedly small download: {bytes} bytes"); + let path = payload["path"].as_str().unwrap(); + let content = std::fs::read_to_string(path).unwrap(); + assert!(content.to_lowercase().contains("example domain")); + } + + #[tokio::test] + async fn execute_rejects_allowlist_miss() { + let tmp = TempDir::new().unwrap(); + let t = tool(&tmp, vec!["example.com"]); + let result = t + .execute(serde_json::json!({"url": "https://other.example.org/x"})) + .await + .unwrap(); + assert!(result.is_error); + assert!(result.output().contains("allowed_domains")); + } +} diff --git a/src/openhuman/tools/impl/network/gitbooks.rs b/src/openhuman/tools/impl/network/gitbooks.rs new file mode 100644 index 000000000..45ac08d30 --- /dev/null +++ b/src/openhuman/tools/impl/network/gitbooks.rs @@ -0,0 +1,425 @@ +//! `gitbooks` — answer questions about OpenHuman by talking to the +//! GitBook MCP server. +//! +//! GitBook hosts a stateless Streamable-HTTP MCP server that exposes +//! exactly two tools: +//! +//! - `searchDocumentation { query }` — returns excerpts + page links +//! - `getPage { url }` — returns the full markdown of a page +//! +//! We mirror them as `gitbooks_search` and `gitbooks_get_page`. The +//! server returns each JSON-RPC response in a single +//! `event: message\ndata: {…}` SSE frame, so we do a tiny inline +//! parse — no need to pull in a full SSE/MCP client crate yet. + +use crate::openhuman::tools::traits::{PermissionLevel, Tool, ToolResult}; +use async_trait::async_trait; +use serde_json::{json, Value}; +use std::sync::atomic::{AtomicI64, Ordering}; +use std::sync::Arc; +use std::time::Duration; + +/// Minimal MCP-over-HTTP client for stateless servers (no session id). +struct McpHttpClient { + endpoint: String, + timeout_secs: u64, + next_id: AtomicI64, +} + +impl McpHttpClient { + fn new(endpoint: String, timeout_secs: u64) -> Self { + Self { + endpoint, + timeout_secs, + next_id: AtomicI64::new(1), + } + } + + async fn call_tool(&self, name: &str, arguments: Value) -> anyhow::Result { + let id = self.next_id.fetch_add(1, Ordering::Relaxed); + let body = json!({ + "jsonrpc": "2.0", + "id": id, + "method": "tools/call", + "params": { "name": name, "arguments": arguments } + }); + + let builder = reqwest::Client::builder() + .timeout(Duration::from_secs(self.timeout_secs)) + .connect_timeout(Duration::from_secs(10)) + .redirect(reqwest::redirect::Policy::none()); + let builder = + crate::openhuman::config::apply_runtime_proxy_to_builder(builder, "tool.gitbooks"); + let client = builder.build()?; + + // Log only the redacted host so query strings / path params / + // any future bearer-token-bearing endpoints don't leak into + // logs aggregated for triage. + tracing::debug!( + target: "[gitbooks]", + endpoint = %redact_endpoint(&self.endpoint), + tool = %name, + "MCP tools/call" + ); + + let resp = client + .post(&self.endpoint) + .header("Content-Type", "application/json") + .header("Accept", "application/json, text/event-stream") + .body(serde_json::to_vec(&body)?) + .send() + .await?; + + let status = resp.status(); + let content_type = resp + .headers() + .get(reqwest::header::CONTENT_TYPE) + .and_then(|v| v.to_str().ok()) + .unwrap_or("") + .to_string(); + let text = resp.text().await?; + + if !status.is_success() { + anyhow::bail!("MCP HTTP {} — {}", status.as_u16(), text); + } + + let payload: Value = if content_type.starts_with("text/event-stream") { + parse_sse_message(&text)? + } else { + serde_json::from_str(&text).map_err(|e| { + anyhow::anyhow!("Failed to parse MCP JSON response: {e} — body: {text}") + })? + }; + + if let Some(err) = payload.get("error") { + anyhow::bail!("MCP error: {err}"); + } + let result = payload + .get("result") + .ok_or_else(|| anyhow::anyhow!("MCP response missing 'result': {payload}"))? + .clone(); + Ok(result) + } +} + +/// Reduce a configured endpoint URL to `scheme://host[:port]` for +/// safe logging. Anything that doesn't parse as a recognisable +/// http(s) URL is reported as `` rather than echoed. +fn redact_endpoint(raw: &str) -> String { + let trimmed = raw.trim(); + let (scheme, rest) = if let Some(r) = trimmed.strip_prefix("https://") { + ("https", r) + } else if let Some(r) = trimmed.strip_prefix("http://") { + ("http", r) + } else { + return "".into(); + }; + let authority = rest.split(['/', '?', '#']).next().unwrap_or(""); + if authority.is_empty() || authority.contains('@') { + return "".into(); + } + format!("{scheme}://{authority}") +} + +/// Parse the first `data: {…}` line from a Streamable-HTTP SSE +/// response. The GitBook server emits exactly one frame per JSON-RPC +/// request, so we do not need a full SSE state machine. +fn parse_sse_message(body: &str) -> anyhow::Result { + for line in body.lines() { + let line = line.trim_end_matches('\r'); + if let Some(data) = line.strip_prefix("data:") { + let data = data.trim_start(); + if data.is_empty() { + continue; + } + return serde_json::from_str(data).map_err(|e| { + anyhow::anyhow!("Failed to parse SSE data frame: {e} — line: {data}") + }); + } + } + anyhow::bail!("No SSE data frame found in MCP response: {body}") +} + +/// Render an MCP `tools/call` result into a single string for the +/// agent. MCP returns `{ content: [{ type, text }, …], isError? }`. +fn render_tool_result(result: &Value) -> ToolResult { + let is_error = result + .get("isError") + .and_then(Value::as_bool) + .unwrap_or(false); + + let mut out = String::new(); + if let Some(content) = result.get("content").and_then(Value::as_array) { + for block in content { + if let Some(t) = block.get("text").and_then(Value::as_str) { + if !out.is_empty() { + out.push_str("\n\n"); + } + out.push_str(t); + } + } + } + if out.is_empty() { + out = result.to_string(); + } + + if is_error { + ToolResult::error(out) + } else { + ToolResult::success(out) + } +} + +// ── Search ───────────────────────────────────────────────────────── + +pub struct GitbooksSearchTool { + client: Arc, +} + +impl GitbooksSearchTool { + pub fn new(endpoint: String, timeout_secs: u64) -> Self { + Self { + client: Arc::new(McpHttpClient::new(endpoint, timeout_secs)), + } + } +} + +#[async_trait] +impl Tool for GitbooksSearchTool { + fn name(&self) -> &str { + "gitbooks_search" + } + + fn description(&self) -> &str { + "Search the OpenHuman product documentation. Use this to answer questions about how \ + OpenHuman works, find features, look up configuration, or locate guides. Returns \ + excerpts with page titles and links — follow up with `gitbooks_get_page` for the \ + full markdown of a page." + } + + fn parameters_schema(&self) -> Value { + json!({ + "type": "object", + "properties": { + "query": { + "type": "string", + "description": "Natural-language question or keyword query about OpenHuman." + } + }, + "required": ["query"] + }) + } + + fn permission_level(&self) -> PermissionLevel { + PermissionLevel::ReadOnly + } + + async fn execute(&self, args: Value) -> anyhow::Result { + let query = args + .get("query") + .and_then(Value::as_str) + .ok_or_else(|| anyhow::anyhow!("Missing 'query' parameter"))?; + if query.trim().is_empty() { + return Ok(ToolResult::error("query cannot be empty")); + } + + match self + .client + .call_tool("searchDocumentation", json!({ "query": query })) + .await + { + Ok(result) => Ok(render_tool_result(&result)), + Err(e) => Ok(ToolResult::error(format!("gitbooks_search failed: {e}"))), + } + } +} + +// ── Get page ────────────────────────────────────────────────────── + +pub struct GitbooksGetPageTool { + client: Arc, +} + +impl GitbooksGetPageTool { + pub fn new(endpoint: String, timeout_secs: u64) -> Self { + Self { + client: Arc::new(McpHttpClient::new(endpoint, timeout_secs)), + } + } +} + +#[async_trait] +impl Tool for GitbooksGetPageTool { + fn name(&self) -> &str { + "gitbooks_get_page" + } + + fn description(&self) -> &str { + "Fetch the full markdown of a specific OpenHuman documentation page by URL. Pair this \ + with `gitbooks_search` — search returns partial excerpts; use this to get the \ + complete page when more detail is needed." + } + + fn parameters_schema(&self) -> Value { + json!({ + "type": "object", + "properties": { + "url": { + "type": "string", + "description": "The full URL of the OpenHuman documentation page (e.g. https://tinyhumans.gitbook.io/openhuman/getting-started)." + } + }, + "required": ["url"] + }) + } + + fn permission_level(&self) -> PermissionLevel { + PermissionLevel::ReadOnly + } + + async fn execute(&self, args: Value) -> anyhow::Result { + let url = args + .get("url") + .and_then(Value::as_str) + .ok_or_else(|| anyhow::anyhow!("Missing 'url' parameter"))?; + if url.trim().is_empty() { + return Ok(ToolResult::error("url cannot be empty")); + } + + match self + .client + .call_tool("getPage", json!({ "url": url })) + .await + { + Ok(result) => Ok(render_tool_result(&result)), + Err(e) => Ok(ToolResult::error(format!("gitbooks_get_page failed: {e}"))), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn redact_endpoint_keeps_only_origin() { + assert_eq!( + redact_endpoint("https://tinyhumans.gitbook.io/openhuman/~gitbook/mcp"), + "https://tinyhumans.gitbook.io" + ); + assert_eq!( + redact_endpoint("http://example.com:8080/path?token=secret"), + "http://example.com:8080" + ); + } + + #[test] + fn redact_endpoint_rejects_userinfo_and_unknown_schemes() { + assert_eq!( + redact_endpoint("https://user:pass@example.com/x"), + "" + ); + assert_eq!(redact_endpoint("ftp://example.com"), ""); + assert_eq!(redact_endpoint(""), ""); + } + + #[test] + fn parse_sse_extracts_data_frame() { + let body = "event: message\ndata: {\"jsonrpc\":\"2.0\",\"id\":1,\"result\":{\"x\":1}}\n\n"; + let v = parse_sse_message(body).unwrap(); + assert_eq!(v["result"]["x"], 1); + } + + #[test] + fn parse_sse_handles_crlf() { + let body = "event: message\r\ndata: {\"jsonrpc\":\"2.0\",\"id\":1,\"result\":{}}\r\n\r\n"; + assert!(parse_sse_message(body).is_ok()); + } + + #[test] + fn parse_sse_errors_when_no_data_line() { + let body = "event: ping\n\n"; + assert!(parse_sse_message(body).is_err()); + } + + #[test] + fn parse_sse_errors_on_invalid_json() { + let body = "data: not-json\n\n"; + assert!(parse_sse_message(body).is_err()); + } + + #[test] + fn render_tool_result_concatenates_text_blocks() { + let r = json!({ + "content": [ + {"type": "text", "text": "first"}, + {"type": "text", "text": "second"} + ] + }); + let out = render_tool_result(&r); + assert!(!out.is_error); + assert!(out.output().contains("first")); + assert!(out.output().contains("second")); + } + + #[test] + fn render_tool_result_marks_errors() { + let r = json!({ + "content": [{"type": "text", "text": "boom"}], + "isError": true + }); + let out = render_tool_result(&r); + assert!(out.is_error); + assert!(out.output().contains("boom")); + } + + #[test] + fn render_tool_result_falls_back_to_raw_json() { + let r = json!({"weird": "shape"}); + let out = render_tool_result(&r); + assert!(out.output().contains("weird")); + } + + #[tokio::test] + async fn search_rejects_empty_query() { + let t = GitbooksSearchTool::new("https://example.com/mcp".into(), 5); + let result = t.execute(json!({"query": " "})).await.unwrap(); + assert!(result.is_error); + assert!(result.output().contains("empty")); + } + + #[tokio::test] + async fn get_page_rejects_empty_url() { + let t = GitbooksGetPageTool::new("https://example.com/mcp".into(), 5); + let result = t.execute(json!({"url": ""})).await.unwrap(); + assert!(result.is_error); + } + + /// Live integration test against the real GitBook MCP endpoint. + /// Gated behind `OPENHUMAN_GITBOOKS_LIVE_TEST=1` so CI / offline + /// builds don't depend on the public network. + #[tokio::test] + async fn live_search_smoke() { + if std::env::var("OPENHUMAN_GITBOOKS_LIVE_TEST") + .ok() + .as_deref() + != Some("1") + { + return; + } + let t = GitbooksSearchTool::new( + "https://tinyhumans.gitbook.io/openhuman/~gitbook/mcp".into(), + 30, + ); + let result = t + .execute(json!({"query": "what is openhuman"})) + .await + .unwrap(); + assert!( + !result.is_error, + "live search returned error: {}", + result.output() + ); + assert!(!result.output().is_empty()); + } +} diff --git a/src/openhuman/tools/impl/network/http_request.rs b/src/openhuman/tools/impl/network/http_request.rs index 2f943a509..0ec2967a2 100644 --- a/src/openhuman/tools/impl/network/http_request.rs +++ b/src/openhuman/tools/impl/network/http_request.rs @@ -1,3 +1,4 @@ +use super::url_guard::{normalize_allowed_domains, validate_url}; use crate::openhuman::security::SecurityPolicy; use crate::openhuman::tools::traits::{Tool, ToolResult}; use async_trait::async_trait; @@ -30,37 +31,7 @@ impl HttpRequestTool { } fn validate_url(&self, raw_url: &str) -> anyhow::Result { - let url = raw_url.trim(); - - if url.is_empty() { - anyhow::bail!("URL cannot be empty"); - } - - if url.chars().any(char::is_whitespace) { - anyhow::bail!("URL cannot contain whitespace"); - } - - if !url.starts_with("http://") && !url.starts_with("https://") { - anyhow::bail!("Only http:// and https:// URLs are allowed"); - } - - if self.allowed_domains.is_empty() { - anyhow::bail!( - "HTTP request tool is enabled but no allowed_domains are configured. Add [http_request].allowed_domains in config.toml" - ); - } - - let host = extract_host(url)?; - - if is_private_or_local_host(&host) { - anyhow::bail!("Blocked local/private host: {host}"); - } - - if !host_matches_allowlist(&host, &self.allowed_domains) { - anyhow::bail!("Host '{host}' is not in http_request.allowed_domains"); - } - - Ok(url.to_string()) + validate_url(raw_url, &self.allowed_domains) } fn validate_method(&self, method: &str) -> anyhow::Result { @@ -225,7 +196,6 @@ impl Tool for HttpRequestTool { let status = response.status(); let status_code = status.as_u16(); - // Get response headers (redact sensitive ones) let response_headers = response.headers().iter(); let headers_text = response_headers .map(|(k, _)| { @@ -239,7 +209,6 @@ impl Tool for HttpRequestTool { .collect::>() .join(", "); - // Get response body with size limit let response_text = match response.text().await { Ok(text) => self.truncate_response(&text), Err(e) => format!("[Failed to read response body: {e}]"), @@ -264,149 +233,6 @@ impl Tool for HttpRequestTool { } } -// Helper functions similar to browser_open.rs - -fn normalize_allowed_domains(domains: Vec) -> Vec { - let mut normalized = domains - .into_iter() - .filter_map(|d| normalize_domain(&d)) - .collect::>(); - normalized.sort_unstable(); - normalized.dedup(); - normalized -} - -fn normalize_domain(raw: &str) -> Option { - let mut d = raw.trim().to_lowercase(); - if d.is_empty() { - return None; - } - - if let Some(stripped) = d.strip_prefix("https://") { - d = stripped.to_string(); - } else if let Some(stripped) = d.strip_prefix("http://") { - d = stripped.to_string(); - } - - if let Some((host, _)) = d.split_once('/') { - d = host.to_string(); - } - - d = d.trim_start_matches('.').trim_end_matches('.').to_string(); - - if let Some((host, _)) = d.split_once(':') { - d = host.to_string(); - } - - if d.is_empty() || d.chars().any(char::is_whitespace) { - return None; - } - - Some(d) -} - -fn extract_host(url: &str) -> anyhow::Result { - let rest = url - .strip_prefix("http://") - .or_else(|| url.strip_prefix("https://")) - .ok_or_else(|| anyhow::anyhow!("Only http:// and https:// URLs are allowed"))?; - - let authority = rest - .split(['/', '?', '#']) - .next() - .ok_or_else(|| anyhow::anyhow!("Invalid URL"))?; - - if authority.is_empty() { - anyhow::bail!("URL must include a host"); - } - - if authority.contains('@') { - anyhow::bail!("URL userinfo is not allowed"); - } - - if authority.starts_with('[') { - anyhow::bail!("IPv6 hosts are not supported in http_request"); - } - - let host = authority - .split(':') - .next() - .unwrap_or_default() - .trim() - .trim_end_matches('.') - .to_lowercase(); - - if host.is_empty() { - anyhow::bail!("URL must include a valid host"); - } - - Ok(host) -} - -fn host_matches_allowlist(host: &str, allowed_domains: &[String]) -> bool { - allowed_domains.iter().any(|domain| { - host == domain - || host - .strip_suffix(domain) - .is_some_and(|prefix| prefix.ends_with('.')) - }) -} - -fn is_private_or_local_host(host: &str) -> bool { - // Strip brackets from IPv6 addresses like [::1] - let bare = host - .strip_prefix('[') - .and_then(|h| h.strip_suffix(']')) - .unwrap_or(host); - - let has_local_tld = bare - .rsplit('.') - .next() - .is_some_and(|label| label == "local"); - - if bare == "localhost" || bare.ends_with(".localhost") || has_local_tld { - return true; - } - - if let Ok(ip) = bare.parse::() { - return match ip { - std::net::IpAddr::V4(v4) => is_non_global_v4(v4), - std::net::IpAddr::V6(v6) => is_non_global_v6(v6), - }; - } - - false -} - -/// Returns true if the IPv4 address is not globally routable. -fn is_non_global_v4(v4: std::net::Ipv4Addr) -> bool { - let [a, b, c, _] = v4.octets(); - v4.is_loopback() // 127.0.0.0/8 - || v4.is_private() // 10/8, 172.16/12, 192.168/16 - || v4.is_link_local() // 169.254.0.0/16 - || v4.is_unspecified() // 0.0.0.0 - || v4.is_broadcast() // 255.255.255.255 - || v4.is_multicast() // 224.0.0.0/4 - || (a == 100 && (64..=127).contains(&b)) // Shared address space (RFC 6598) - || a >= 240 // Reserved (240.0.0.0/4, except broadcast) - || (a == 192 && b == 0 && (c == 0 || c == 2)) // IETF assignments + TEST-NET-1 - || (a == 198 && b == 51) // Documentation (198.51.100.0/24) - || (a == 203 && b == 0) // Documentation (203.0.113.0/24) - || (a == 198 && (18..=19).contains(&b)) // Benchmarking (198.18.0.0/15) -} - -/// Returns true if the IPv6 address is not globally routable. -fn is_non_global_v6(v6: std::net::Ipv6Addr) -> bool { - let segs = v6.segments(); - v6.is_loopback() // ::1 - || v6.is_unspecified() // :: - || v6.is_multicast() // ff00::/8 - || (segs[0] & 0xfe00) == 0xfc00 // Unique-local (fc00::/7) - || (segs[0] & 0xffc0) == 0xfe80 // Link-local (fe80::/10) - || (segs[0] == 0x2001 && segs[1] == 0x0db8) // Documentation (2001:db8::/32) - || v6.to_ipv4_mapped().is_some_and(is_non_global_v4) -} - #[cfg(test)] mod tests { use super::*; @@ -425,102 +251,6 @@ mod tests { ) } - #[test] - fn normalize_domain_strips_scheme_path_and_case() { - let got = normalize_domain(" HTTPS://Docs.Example.com/path ").unwrap(); - assert_eq!(got, "docs.example.com"); - } - - #[test] - fn normalize_allowed_domains_deduplicates() { - let got = normalize_allowed_domains(vec![ - "example.com".into(), - "EXAMPLE.COM".into(), - "https://example.com/".into(), - ]); - assert_eq!(got, vec!["example.com".to_string()]); - } - - #[test] - fn validate_accepts_exact_domain() { - let tool = test_tool(vec!["example.com"]); - let got = tool.validate_url("https://example.com/docs").unwrap(); - assert_eq!(got, "https://example.com/docs"); - } - - #[test] - fn validate_accepts_http() { - let tool = test_tool(vec!["example.com"]); - assert!(tool.validate_url("http://example.com").is_ok()); - } - - #[test] - fn validate_accepts_subdomain() { - let tool = test_tool(vec!["example.com"]); - assert!(tool.validate_url("https://api.example.com/v1").is_ok()); - } - - #[test] - fn validate_rejects_allowlist_miss() { - let tool = test_tool(vec!["example.com"]); - let err = tool - .validate_url("https://google.com") - .unwrap_err() - .to_string(); - assert!(err.contains("allowed_domains")); - } - - #[test] - fn validate_rejects_localhost() { - let tool = test_tool(vec!["localhost"]); - let err = tool - .validate_url("https://localhost:8080") - .unwrap_err() - .to_string(); - assert!(err.contains("local/private")); - } - - #[test] - fn validate_rejects_private_ipv4() { - let tool = test_tool(vec!["192.168.1.5"]); - let err = tool - .validate_url("https://192.168.1.5") - .unwrap_err() - .to_string(); - assert!(err.contains("local/private")); - } - - #[test] - fn validate_rejects_whitespace() { - let tool = test_tool(vec!["example.com"]); - let err = tool - .validate_url("https://example.com/hello world") - .unwrap_err() - .to_string(); - assert!(err.contains("whitespace")); - } - - #[test] - fn validate_rejects_userinfo() { - let tool = test_tool(vec!["example.com"]); - let err = tool - .validate_url("https://user@example.com") - .unwrap_err() - .to_string(); - assert!(err.contains("userinfo")); - } - - #[test] - fn validate_requires_allowlist() { - let security = Arc::new(SecurityPolicy::default()); - let tool = HttpRequestTool::new(security, vec![], 1_000_000, 30); - let err = tool - .validate_url("https://example.com") - .unwrap_err() - .to_string(); - assert!(err.contains("allowed_domains")); - } - #[test] fn validate_accepts_valid_methods() { let tool = test_tool(vec!["example.com"]); @@ -540,89 +270,6 @@ mod tests { assert!(err.contains("Unsupported HTTP method")); } - #[test] - fn blocks_multicast_ipv4() { - assert!(is_private_or_local_host("224.0.0.1")); - assert!(is_private_or_local_host("239.255.255.255")); - } - - #[test] - fn blocks_broadcast() { - assert!(is_private_or_local_host("255.255.255.255")); - } - - #[test] - fn blocks_reserved_ipv4() { - assert!(is_private_or_local_host("240.0.0.1")); - assert!(is_private_or_local_host("250.1.2.3")); - } - - #[test] - fn blocks_documentation_ranges() { - assert!(is_private_or_local_host("192.0.2.1")); // TEST-NET-1 - assert!(is_private_or_local_host("198.51.100.1")); // TEST-NET-2 - assert!(is_private_or_local_host("203.0.113.1")); // TEST-NET-3 - } - - #[test] - fn blocks_benchmarking_range() { - assert!(is_private_or_local_host("198.18.0.1")); - assert!(is_private_or_local_host("198.19.255.255")); - } - - #[test] - fn blocks_ipv6_localhost() { - assert!(is_private_or_local_host("::1")); - assert!(is_private_or_local_host("[::1]")); - } - - #[test] - fn blocks_ipv6_multicast() { - assert!(is_private_or_local_host("ff02::1")); - } - - #[test] - fn blocks_ipv6_link_local() { - assert!(is_private_or_local_host("fe80::1")); - } - - #[test] - fn blocks_ipv6_unique_local() { - assert!(is_private_or_local_host("fd00::1")); - } - - #[test] - fn blocks_ipv4_mapped_ipv6() { - assert!(is_private_or_local_host("::ffff:127.0.0.1")); - assert!(is_private_or_local_host("::ffff:192.168.1.1")); - assert!(is_private_or_local_host("::ffff:10.0.0.1")); - } - - #[test] - fn allows_public_ipv4() { - assert!(!is_private_or_local_host("8.8.8.8")); - assert!(!is_private_or_local_host("1.1.1.1")); - assert!(!is_private_or_local_host("93.184.216.34")); - } - - #[test] - fn blocks_ipv6_documentation_range() { - assert!(is_private_or_local_host("2001:db8::1")); - } - - #[test] - fn allows_public_ipv6() { - assert!(!is_private_or_local_host("2607:f8b0:4004:800::200e")); - } - - #[test] - fn blocks_shared_address_space() { - assert!(is_private_or_local_host("100.64.0.1")); - assert!(is_private_or_local_host("100.127.255.255")); - assert!(!is_private_or_local_host("100.63.0.1")); // Just below range - assert!(!is_private_or_local_host("100.128.0.1")); // Just above range - } - #[tokio::test] async fn execute_blocks_readonly_mode() { let security = Arc::new(SecurityPolicy { @@ -670,7 +317,7 @@ mod tests { ); let text = "hello world this is long"; let truncated = tool.truncate_response(text); - assert!(truncated.len() <= 10 + 60); // limit + message + assert!(truncated.len() <= 10 + 60); assert!(truncated.contains("[Response truncated")); } @@ -726,130 +373,9 @@ mod tests { assert_eq!(headers[0].1, "Bearer real-token"); } - // ── SSRF: alternate IP notation bypass defense-in-depth ───────── - // - // Rust's IpAddr::parse() rejects non-standard notations (octal, hex, - // decimal integer, zero-padded). These tests document that property - // so regressions are caught if the parsing strategy ever changes. - - #[test] - fn ssrf_octal_loopback_not_parsed_as_ip() { - // 0177.0.0.1 is octal for 127.0.0.1 in some languages, but - // Rust's IpAddr rejects it — it falls through as a hostname. - assert!(!is_private_or_local_host("0177.0.0.1")); - } - - #[test] - fn ssrf_hex_loopback_not_parsed_as_ip() { - // 0x7f000001 is hex for 127.0.0.1 in some languages. - assert!(!is_private_or_local_host("0x7f000001")); - } - - #[test] - fn ssrf_decimal_loopback_not_parsed_as_ip() { - // 2130706433 is decimal for 127.0.0.1 in some languages. - assert!(!is_private_or_local_host("2130706433")); - } - - #[test] - fn ssrf_zero_padded_loopback_not_parsed_as_ip() { - // 127.000.000.001 uses zero-padded octets. - assert!(!is_private_or_local_host("127.000.000.001")); - } - - #[test] - fn ssrf_alternate_notations_rejected_by_validate_url() { - // Even if is_private_or_local_host doesn't flag these, they - // fail the allowlist because they're treated as hostnames. - let tool = test_tool(vec!["example.com"]); - for notation in [ - "http://0177.0.0.1", - "http://0x7f000001", - "http://2130706433", - "http://127.000.000.001", - ] { - let err = tool.validate_url(notation).unwrap_err().to_string(); - assert!( - err.contains("allowed_domains"), - "Expected allowlist rejection for {notation}, got: {err}" - ); - } - } - #[test] fn redirect_policy_is_none() { - // Structural test: the tool should be buildable with redirect-safe config. - // The actual Policy::none() enforcement is in execute_request's client builder. let tool = test_tool(vec!["example.com"]); assert_eq!(tool.name(), "http_request"); } - - // ── §1.4 DNS rebinding / SSRF defense-in-depth tests ───── - - #[test] - fn ssrf_blocks_loopback_127_range() { - assert!(is_private_or_local_host("127.0.0.1")); - assert!(is_private_or_local_host("127.0.0.2")); - assert!(is_private_or_local_host("127.255.255.255")); - } - - #[test] - fn ssrf_blocks_rfc1918_10_range() { - assert!(is_private_or_local_host("10.0.0.1")); - assert!(is_private_or_local_host("10.255.255.255")); - } - - #[test] - fn ssrf_blocks_rfc1918_172_range() { - assert!(is_private_or_local_host("172.16.0.1")); - assert!(is_private_or_local_host("172.31.255.255")); - } - - #[test] - fn ssrf_blocks_unspecified_address() { - assert!(is_private_or_local_host("0.0.0.0")); - } - - #[test] - fn ssrf_blocks_dot_localhost_subdomain() { - assert!(is_private_or_local_host("evil.localhost")); - assert!(is_private_or_local_host("a.b.localhost")); - } - - #[test] - fn ssrf_blocks_dot_local_tld() { - assert!(is_private_or_local_host("service.local")); - } - - #[test] - fn ssrf_ipv6_unspecified() { - assert!(is_private_or_local_host("::")); - } - - #[test] - fn validate_rejects_ftp_scheme() { - let tool = test_tool(vec!["example.com"]); - let err = tool - .validate_url("ftp://example.com") - .unwrap_err() - .to_string(); - assert!(err.contains("http://") || err.contains("https://")); - } - - #[test] - fn validate_rejects_empty_url() { - let tool = test_tool(vec!["example.com"]); - let err = tool.validate_url("").unwrap_err().to_string(); - assert!(err.contains("empty")); - } - - #[test] - fn validate_rejects_ipv6_host() { - let tool = test_tool(vec!["example.com"]); - let err = tool - .validate_url("http://[::1]:8080/path") - .unwrap_err() - .to_string(); - assert!(err.contains("IPv6")); - } } diff --git a/src/openhuman/tools/impl/network/mod.rs b/src/openhuman/tools/impl/network/mod.rs index d92d07977..23d7fdd1d 100644 --- a/src/openhuman/tools/impl/network/mod.rs +++ b/src/openhuman/tools/impl/network/mod.rs @@ -1,7 +1,12 @@ mod composio; +mod curl; +mod gitbooks; mod http_request; +mod url_guard; mod web_search; pub use composio::{ComposioAction, ComposioTool}; +pub use curl::CurlTool; +pub use gitbooks::{GitbooksGetPageTool, GitbooksSearchTool}; pub use http_request::HttpRequestTool; pub use web_search::WebSearchTool; diff --git a/src/openhuman/tools/impl/network/url_guard.rs b/src/openhuman/tools/impl/network/url_guard.rs new file mode 100644 index 000000000..31dcc15d1 --- /dev/null +++ b/src/openhuman/tools/impl/network/url_guard.rs @@ -0,0 +1,475 @@ +//! Shared URL validation + SSRF guards for outbound network tools. +//! +//! Used by `http_request`, `curl`, and any future tool that takes a +//! user-supplied URL. The contract is intentionally strict: +//! +//! - http(s) only +//! - non-empty allowlist required (callers pass it in) +//! - no whitespace, no userinfo, no IPv6 hosts +//! - blocks loopback / RFC1918 / link-local / multicast / documentation / +//! shared-address / IPv4-mapped IPv6, including `localhost` / +//! `*.localhost` / `*.local` +//! +//! The blocklist deliberately does NOT cover alternate IP notations +//! (octal, hex, decimal) because Rust's `IpAddr::parse` rejects them — +//! they fall through and get rejected by the allowlist instead. See the +//! tests in `http_request.rs` for the documented behaviour. + +/// Validate a URL against the allowlist + SSRF rules. Returns the +/// original URL on success. +pub(super) fn validate_url(raw_url: &str, allowed_domains: &[String]) -> anyhow::Result { + let url = raw_url.trim(); + + if url.is_empty() { + anyhow::bail!("URL cannot be empty"); + } + + if url.chars().any(char::is_whitespace) { + anyhow::bail!("URL cannot contain whitespace"); + } + + if !url.starts_with("http://") && !url.starts_with("https://") { + anyhow::bail!("Only http:// and https:// URLs are allowed"); + } + + if allowed_domains.is_empty() { + anyhow::bail!( + "Network tool is enabled but no allowed_domains are configured. Add [http_request].allowed_domains in config.toml" + ); + } + + let host = extract_host(url)?; + + if is_private_or_local_host(&host) { + anyhow::bail!("Blocked local/private host: {host}"); + } + + if !host_matches_allowlist(&host, allowed_domains) { + anyhow::bail!("Host '{host}' is not in http_request.allowed_domains"); + } + + Ok(url.to_string()) +} + +pub(super) fn normalize_allowed_domains(domains: Vec) -> Vec { + let mut normalized = domains + .into_iter() + .filter_map(|d| normalize_domain(&d)) + .collect::>(); + normalized.sort_unstable(); + normalized.dedup(); + normalized +} + +pub(super) fn normalize_domain(raw: &str) -> Option { + let mut d = raw.trim().to_lowercase(); + if d.is_empty() { + return None; + } + + if let Some(stripped) = d.strip_prefix("https://") { + d = stripped.to_string(); + } else if let Some(stripped) = d.strip_prefix("http://") { + d = stripped.to_string(); + } + + if let Some((host, _)) = d.split_once('/') { + d = host.to_string(); + } + + d = d.trim_start_matches('.').trim_end_matches('.').to_string(); + + if let Some((host, _)) = d.split_once(':') { + d = host.to_string(); + } + + if d.is_empty() || d.chars().any(char::is_whitespace) { + return None; + } + + Some(d) +} + +pub(super) fn extract_host(url: &str) -> anyhow::Result { + let rest = url + .strip_prefix("http://") + .or_else(|| url.strip_prefix("https://")) + .ok_or_else(|| anyhow::anyhow!("Only http:// and https:// URLs are allowed"))?; + + let authority = rest + .split(['/', '?', '#']) + .next() + .ok_or_else(|| anyhow::anyhow!("Invalid URL"))?; + + if authority.is_empty() { + anyhow::bail!("URL must include a host"); + } + + if authority.contains('@') { + anyhow::bail!("URL userinfo is not allowed"); + } + + if authority.starts_with('[') { + anyhow::bail!("IPv6 hosts are not supported in http_request"); + } + + let host = authority + .split(':') + .next() + .unwrap_or_default() + .trim() + .trim_end_matches('.') + .to_lowercase(); + + if host.is_empty() { + anyhow::bail!("URL must include a valid host"); + } + + Ok(host) +} + +pub(super) fn host_matches_allowlist(host: &str, allowed_domains: &[String]) -> bool { + allowed_domains.iter().any(|domain| { + host == domain + || host + .strip_suffix(domain) + .is_some_and(|prefix| prefix.ends_with('.')) + }) +} + +pub(super) fn is_private_or_local_host(host: &str) -> bool { + let bare = host + .strip_prefix('[') + .and_then(|h| h.strip_suffix(']')) + .unwrap_or(host); + + let has_local_tld = bare + .rsplit('.') + .next() + .is_some_and(|label| label == "local"); + + if bare == "localhost" || bare.ends_with(".localhost") || has_local_tld { + return true; + } + + if let Ok(ip) = bare.parse::() { + return match ip { + std::net::IpAddr::V4(v4) => is_non_global_v4(v4), + std::net::IpAddr::V6(v6) => is_non_global_v6(v6), + }; + } + + false +} + +fn is_non_global_v4(v4: std::net::Ipv4Addr) -> bool { + let [a, b, c, _] = v4.octets(); + v4.is_loopback() + || v4.is_private() + || v4.is_link_local() + || v4.is_unspecified() + || v4.is_broadcast() + || v4.is_multicast() + || (a == 100 && (64..=127).contains(&b)) + || a >= 240 + || (a == 192 && b == 0 && (c == 0 || c == 2)) + || (a == 198 && b == 51) + || (a == 203 && b == 0) + || (a == 198 && (18..=19).contains(&b)) +} + +fn is_non_global_v6(v6: std::net::Ipv6Addr) -> bool { + let segs = v6.segments(); + v6.is_loopback() + || v6.is_unspecified() + || v6.is_multicast() + || (segs[0] & 0xfe00) == 0xfc00 + || (segs[0] & 0xffc0) == 0xfe80 + || (segs[0] == 0x2001 && segs[1] == 0x0db8) + || v6.to_ipv4_mapped().is_some_and(is_non_global_v4) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn normalize_domain_strips_scheme_path_and_case() { + let got = normalize_domain(" HTTPS://Docs.Example.com/path ").unwrap(); + assert_eq!(got, "docs.example.com"); + } + + #[test] + fn normalize_allowed_domains_deduplicates() { + let got = normalize_allowed_domains(vec![ + "example.com".into(), + "EXAMPLE.COM".into(), + "https://example.com/".into(), + ]); + assert_eq!(got, vec!["example.com".to_string()]); + } + + #[test] + fn validate_accepts_exact_domain() { + let allow = vec!["example.com".to_string()]; + let got = validate_url("https://example.com/docs", &allow).unwrap(); + assert_eq!(got, "https://example.com/docs"); + } + + #[test] + fn validate_accepts_http() { + let allow = vec!["example.com".to_string()]; + assert!(validate_url("http://example.com", &allow).is_ok()); + } + + #[test] + fn validate_accepts_subdomain() { + let allow = vec!["example.com".to_string()]; + assert!(validate_url("https://api.example.com/v1", &allow).is_ok()); + } + + #[test] + fn validate_rejects_allowlist_miss() { + let allow = vec!["example.com".to_string()]; + let err = validate_url("https://google.com", &allow) + .unwrap_err() + .to_string(); + assert!(err.contains("allowed_domains")); + } + + #[test] + fn validate_rejects_localhost() { + let allow = vec!["localhost".to_string()]; + let err = validate_url("https://localhost:8080", &allow) + .unwrap_err() + .to_string(); + assert!(err.contains("local/private")); + } + + #[test] + fn validate_rejects_private_ipv4() { + let allow = vec!["192.168.1.5".to_string()]; + let err = validate_url("https://192.168.1.5", &allow) + .unwrap_err() + .to_string(); + assert!(err.contains("local/private")); + } + + #[test] + fn validate_rejects_whitespace() { + let allow = vec!["example.com".to_string()]; + let err = validate_url("https://example.com/hello world", &allow) + .unwrap_err() + .to_string(); + assert!(err.contains("whitespace")); + } + + #[test] + fn validate_rejects_userinfo() { + let allow = vec!["example.com".to_string()]; + let err = validate_url("https://user@example.com", &allow) + .unwrap_err() + .to_string(); + assert!(err.contains("userinfo")); + } + + #[test] + fn validate_requires_allowlist() { + let err = validate_url("https://example.com", &[]) + .unwrap_err() + .to_string(); + assert!(err.contains("allowed_domains")); + } + + #[test] + fn validate_rejects_ftp_scheme() { + let allow = vec!["example.com".to_string()]; + let err = validate_url("ftp://example.com", &allow) + .unwrap_err() + .to_string(); + assert!(err.contains("http://") || err.contains("https://")); + } + + #[test] + fn validate_rejects_empty_url() { + let allow = vec!["example.com".to_string()]; + let err = validate_url("", &allow).unwrap_err().to_string(); + assert!(err.contains("empty")); + } + + #[test] + fn validate_rejects_ipv6_host() { + let allow = vec!["example.com".to_string()]; + let err = validate_url("http://[::1]:8080/path", &allow) + .unwrap_err() + .to_string(); + assert!(err.contains("IPv6")); + } + + #[test] + fn blocks_multicast_ipv4() { + assert!(is_private_or_local_host("224.0.0.1")); + assert!(is_private_or_local_host("239.255.255.255")); + } + + #[test] + fn blocks_broadcast() { + assert!(is_private_or_local_host("255.255.255.255")); + } + + #[test] + fn blocks_reserved_ipv4() { + assert!(is_private_or_local_host("240.0.0.1")); + assert!(is_private_or_local_host("250.1.2.3")); + } + + #[test] + fn blocks_documentation_ranges() { + assert!(is_private_or_local_host("192.0.2.1")); + assert!(is_private_or_local_host("198.51.100.1")); + assert!(is_private_or_local_host("203.0.113.1")); + } + + #[test] + fn blocks_benchmarking_range() { + assert!(is_private_or_local_host("198.18.0.1")); + assert!(is_private_or_local_host("198.19.255.255")); + } + + #[test] + fn blocks_ipv6_localhost() { + assert!(is_private_or_local_host("::1")); + assert!(is_private_or_local_host("[::1]")); + } + + #[test] + fn blocks_ipv6_multicast() { + assert!(is_private_or_local_host("ff02::1")); + } + + #[test] + fn blocks_ipv6_link_local() { + assert!(is_private_or_local_host("fe80::1")); + } + + #[test] + fn blocks_ipv6_unique_local() { + assert!(is_private_or_local_host("fd00::1")); + } + + #[test] + fn blocks_ipv4_mapped_ipv6() { + assert!(is_private_or_local_host("::ffff:127.0.0.1")); + assert!(is_private_or_local_host("::ffff:192.168.1.1")); + assert!(is_private_or_local_host("::ffff:10.0.0.1")); + } + + #[test] + fn allows_public_ipv4() { + assert!(!is_private_or_local_host("8.8.8.8")); + assert!(!is_private_or_local_host("1.1.1.1")); + assert!(!is_private_or_local_host("93.184.216.34")); + } + + #[test] + fn blocks_ipv6_documentation_range() { + assert!(is_private_or_local_host("2001:db8::1")); + } + + #[test] + fn allows_public_ipv6() { + assert!(!is_private_or_local_host("2607:f8b0:4004:800::200e")); + } + + #[test] + fn blocks_shared_address_space() { + assert!(is_private_or_local_host("100.64.0.1")); + assert!(is_private_or_local_host("100.127.255.255")); + assert!(!is_private_or_local_host("100.63.0.1")); + assert!(!is_private_or_local_host("100.128.0.1")); + } + + #[test] + fn ssrf_blocks_loopback_127_range() { + assert!(is_private_or_local_host("127.0.0.1")); + assert!(is_private_or_local_host("127.0.0.2")); + assert!(is_private_or_local_host("127.255.255.255")); + } + + #[test] + fn ssrf_blocks_rfc1918_10_range() { + assert!(is_private_or_local_host("10.0.0.1")); + assert!(is_private_or_local_host("10.255.255.255")); + } + + #[test] + fn ssrf_blocks_rfc1918_172_range() { + assert!(is_private_or_local_host("172.16.0.1")); + assert!(is_private_or_local_host("172.31.255.255")); + } + + #[test] + fn ssrf_blocks_unspecified_address() { + assert!(is_private_or_local_host("0.0.0.0")); + } + + #[test] + fn ssrf_blocks_dot_localhost_subdomain() { + assert!(is_private_or_local_host("evil.localhost")); + assert!(is_private_or_local_host("a.b.localhost")); + } + + #[test] + fn ssrf_blocks_dot_local_tld() { + assert!(is_private_or_local_host("service.local")); + } + + #[test] + fn ssrf_ipv6_unspecified() { + assert!(is_private_or_local_host("::")); + } + + // ── Defense-in-depth: alternate IP notations rejected by allowlist + // + // Rust's IpAddr::parse() rejects octal, hex, decimal, and + // zero-padded notations. They fall through as hostnames and get + // rejected by the allowlist instead. These tests pin that + // behaviour so a parser change can't silently re-open SSRF. + + #[test] + fn ssrf_octal_loopback_not_parsed_as_ip() { + assert!(!is_private_or_local_host("0177.0.0.1")); + } + + #[test] + fn ssrf_hex_loopback_not_parsed_as_ip() { + assert!(!is_private_or_local_host("0x7f000001")); + } + + #[test] + fn ssrf_decimal_loopback_not_parsed_as_ip() { + assert!(!is_private_or_local_host("2130706433")); + } + + #[test] + fn ssrf_zero_padded_loopback_not_parsed_as_ip() { + assert!(!is_private_or_local_host("127.000.000.001")); + } + + #[test] + fn ssrf_alternate_notations_rejected_by_validate_url() { + let allow = vec!["example.com".to_string()]; + for notation in [ + "http://0177.0.0.1", + "http://0x7f000001", + "http://2130706433", + "http://127.000.000.001", + ] { + let err = validate_url(notation, &allow).unwrap_err().to_string(); + assert!( + err.contains("allowed_domains"), + "Expected allowlist rejection for {notation}, got: {err}" + ); + } + } +} diff --git a/src/openhuman/tools/ops.rs b/src/openhuman/tools/ops.rs index 6e3d50edf..cd704a269 100644 --- a/src/openhuman/tools/ops.rs +++ b/src/openhuman/tools/ops.rs @@ -169,6 +169,32 @@ pub fn all_tools_with_runtime( http_config.timeout_secs, ))); + // curl — always registered. Shares `http_request.allowed_domains`, + // adds streaming-to-disk with a hard byte ceiling. Writes land + // under `/`. + tools.push(Box::new(CurlTool::new( + security.clone(), + http_config.allowed_domains.clone(), + workspace_dir.to_path_buf(), + root_config.curl.dest_subdir.clone(), + root_config.curl.max_download_bytes, + root_config.curl.timeout_secs, + ))); + + // gitbooks — answers questions about OpenHuman by calling the + // GitBook MCP server. Two tools mirroring the upstream MCP tools. + if root_config.gitbooks.enabled { + tools.push(Box::new(GitbooksSearchTool::new( + root_config.gitbooks.endpoint.clone(), + root_config.gitbooks.timeout_secs, + ))); + tools.push(Box::new(GitbooksGetPageTool::new( + root_config.gitbooks.endpoint.clone(), + root_config.gitbooks.timeout_secs, + ))); + tracing::debug!("[gitbooks] registered gitbooks_search + gitbooks_get_page"); + } + // Web search — always registered. Result/timeout budget // knobs still come from `config.web_search`, but there is no // enable flag: every session needs research as a baseline @@ -372,6 +398,114 @@ mod tests { ); } + #[test] + fn all_tools_always_registers_curl() { + // Regression guard: `curl` is always registered (gated only by + // the shared `http_request.allowed_domains` allowlist at call + // time, like `http_request`). `Write` permission level keeps it + // off agents that aren't allowed to modify the workspace. + let tmp = TempDir::new().unwrap(); + let security = Arc::new(SecurityPolicy::default()); + let mem_cfg = MemoryConfig { + backend: "markdown".into(), + ..MemoryConfig::default() + }; + let mem: Arc = + Arc::from(crate::openhuman::memory::create_memory(&mem_cfg, tmp.path()).unwrap()); + + let browser = BrowserConfig::default(); + let http = crate::openhuman::config::HttpRequestConfig::default(); + let cfg = test_config(&tmp); + + let tools = all_tools( + Arc::new(cfg.clone()), + &security, + mem, + &browser, + &http, + tmp.path(), + &HashMap::new(), + &cfg, + ); + let names: Vec<&str> = tools.iter().map(|t| t.name()).collect(); + assert!( + names.contains(&"curl"), + "curl must always be registered; got: {names:?}" + ); + } + + #[test] + fn all_tools_registers_gitbooks_when_enabled() { + let tmp = TempDir::new().unwrap(); + let security = Arc::new(SecurityPolicy::default()); + let mem_cfg = MemoryConfig { + backend: "markdown".into(), + ..MemoryConfig::default() + }; + let mem: Arc = + Arc::from(crate::openhuman::memory::create_memory(&mem_cfg, tmp.path()).unwrap()); + let browser = BrowserConfig::default(); + let http = crate::openhuman::config::HttpRequestConfig::default(); + let mut cfg = test_config(&tmp); + cfg.gitbooks.enabled = true; + + let tools = all_tools( + Arc::new(cfg.clone()), + &security, + mem, + &browser, + &http, + tmp.path(), + &HashMap::new(), + &cfg, + ); + let names: Vec<&str> = tools.iter().map(|t| t.name()).collect(); + assert!( + names.contains(&"gitbooks_search"), + "gitbooks_search must register when gitbooks.enabled = true; got: {names:?}" + ); + assert!( + names.contains(&"gitbooks_get_page"), + "gitbooks_get_page must register when gitbooks.enabled = true; got: {names:?}" + ); + } + + #[test] + fn all_tools_skips_gitbooks_when_disabled() { + let tmp = TempDir::new().unwrap(); + let security = Arc::new(SecurityPolicy::default()); + let mem_cfg = MemoryConfig { + backend: "markdown".into(), + ..MemoryConfig::default() + }; + let mem: Arc = + Arc::from(crate::openhuman::memory::create_memory(&mem_cfg, tmp.path()).unwrap()); + let browser = BrowserConfig::default(); + let http = crate::openhuman::config::HttpRequestConfig::default(); + let mut cfg = test_config(&tmp); + cfg.gitbooks.enabled = false; + + let tools = all_tools( + Arc::new(cfg.clone()), + &security, + mem, + &browser, + &http, + tmp.path(), + &HashMap::new(), + &cfg, + ); + let names: Vec<&str> = tools.iter().map(|t| t.name()).collect(); + assert!( + !names.contains(&"gitbooks_search"), + "gitbooks_search must NOT register when gitbooks.enabled = false; got: {names:?}" + ); + assert!( + !names.contains(&"gitbooks_get_page"), + "gitbooks_get_page must NOT register when gitbooks.enabled = false; got: {names:?}" + ); + } + #[test] fn all_tools_includes_complete_onboarding() { // Regression guard: the `complete_onboarding` tool must be