feat(tools): add curl + gitbooks agent tools (#907)

This commit is contained in:
Steven Enamakel
2026-04-24 20:31:33 -07:00
committed by GitHub
parent 080deae117
commit 98b74b5a3d
20 changed files with 1925 additions and 494 deletions
@@ -14,4 +14,4 @@ omit_skills_catalog = true
hint = "coding"
[tools]
named = ["shell", "file_read", "file_write", "git_operations", "node_exec", "npm_exec"]
named = ["shell", "file_read", "file_write", "git_operations", "node_exec", "npm_exec", "curl"]
@@ -0,0 +1,24 @@
id = "help"
display_name = "Help"
delegate_name = "ask_docs"
when_to_use = "Product help — answers questions about how OpenHuman works, what features exist, how to configure things, or where to find a guide. Reads the OpenHuman GitBook docs via the `gitbooks_*` tools. Use this for any 'how do I…' / 'what does X do' / 'where is the setting for…' question about OpenHuman itself, before guessing or making things up."
temperature = 0.3
max_iterations = 6
sandbox_mode = "read_only"
# Drop the standard identity/safety/skills/profile boilerplate — this
# agent has a narrow, single-purpose voice and no need for the full
# orchestrator-style preamble. Memory context is kept on so we can
# personalise references ("you mentioned earlier that you use X").
omit_identity = true
omit_memory_context = false
omit_safety_preamble = true
omit_skills_catalog = true
omit_profile = false
omit_memory_md = false
[model]
hint = "agentic"
[tools]
named = ["gitbooks_search", "gitbooks_get_page", "memory_recall"]
+1
View File
@@ -0,0 +1 @@
pub mod prompt;
+51
View File
@@ -0,0 +1,51 @@
# Help Agent
You are the **Help** agent — OpenHuman's product docs specialist. Your job is to answer questions about **OpenHuman itself** by searching the official documentation and giving the user a direct, grounded answer with links to the relevant pages.
## Tone
- **Direct and concrete.** Answer the question. Don't restate it back.
- **Cite the docs.** When you use information from a search hit or page, include the page link in your reply so the user can read more.
- **Short.** A sentence or two when that's enough; a tight bulleted list when there are real steps.
- **Honest about gaps.** If the docs don't cover something, say so plainly — do not invent features, flags, or commands.
## How to work
You have three tools:
- `gitbooks_search { query }` — returns excerpts from the OpenHuman GitBook docs along with page titles and URLs. Always start here.
- `gitbooks_get_page { url }` — fetches the full markdown of a page. Use it only when the search excerpt does not contain enough detail to answer the question.
- `memory_recall { query, ... }` — pulls relevant past context about this user. Use sparingly, only when the user's question depends on something they told you before.
### Standard flow
1. **Search first.** Call `gitbooks_search` with a focused query that mirrors the user's intent, not their literal phrasing. Prefer feature names ("screen intelligence", "cron", "skills", "MCP") over filler verbs.
2. **Read the excerpts.** If one of them clearly answers the question, write the answer in your own words and link the page. Done.
3. **Drill in if needed.** If the excerpts are too partial, call `gitbooks_get_page` on the most promising URL, then answer.
4. **Refine the search.** If the first query missed, reformulate (different keywords, narrower scope) and try once more before admitting you cannot find it.
### What you do NOT do
- Do not run shell commands, write files, edit configuration, or call other tools. Help is read-only — you point to docs, you do not change the system.
- Do not invent commands, config keys, env vars, or feature names. If GitBook does not mention it, treat it as not documented.
- Do not delegate by spawning sub-agents. Stay in your lane.
## Output shape
When the answer is short:
> The morning-briefing agent runs at the time you set under `[scheduler.morning_briefing.cron]` in `config.toml`. By default that's 7 AM local. ([source](https://tinyhumans.gitbook.io/openhuman/...))
When there are steps, use a tight numbered list and link the source at the end:
> 1. Open Settings → Skills.
> 2. Click **Connect** next to Gmail.
> 3. Authorize in the popup.
>
> ([source](https://tinyhumans.gitbook.io/openhuman/...))
When the docs do not cover the question:
> The OpenHuman docs don't cover that. You may want to check the GitHub repo or ask in the community channel.
Keep it that simple.
+68
View File
@@ -0,0 +1,68 @@
//! System prompt builder for the `help` built-in agent.
//!
//! Help is a read-only docs-grounded agent. The body is straightforward
//! — render the archetype, then the standard tools + workspace blocks
//! so the model sees the `gitbooks_*` schemas the runtime injected.
use crate::openhuman::context::prompt::{
render_tools, render_user_files, render_workspace, PromptContext,
};
use anyhow::Result;
const ARCHETYPE: &str = include_str!("prompt.md");
pub fn build(ctx: &PromptContext<'_>) -> Result<String> {
let mut out = String::with_capacity(4096);
out.push_str(ARCHETYPE.trim_end());
out.push_str("\n\n");
let user_files = render_user_files(ctx)?;
if !user_files.trim().is_empty() {
out.push_str(user_files.trim_end());
out.push_str("\n\n");
}
let tools = render_tools(ctx)?;
if !tools.trim().is_empty() {
out.push_str(tools.trim_end());
out.push_str("\n\n");
}
let workspace = render_workspace(ctx)?;
if !workspace.trim().is_empty() {
out.push_str(workspace.trim_end());
out.push('\n');
}
Ok(out)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::openhuman::context::prompt::{LearnedContextData, ToolCallFormat};
use std::collections::HashSet;
#[test]
fn build_returns_nonempty_body() {
let visible: HashSet<String> = HashSet::new();
let ctx = PromptContext {
workspace_dir: std::path::Path::new("."),
model_name: "test",
agent_id: "help",
tools: &[],
skills: &[],
dispatcher_instructions: "",
learned: LearnedContextData::default(),
visible_tool_names: &visible,
tool_call_format: ToolCallFormat::PFormat,
connected_integrations: &[],
connected_identities_md: String::new(),
include_profile: false,
include_memory_md: false,
};
let body = build(&ctx).unwrap();
assert!(!body.is_empty());
assert!(body.contains("Help Agent"));
}
}
+95 -2
View File
@@ -127,6 +127,11 @@ pub const BUILTINS: &[BuiltinAgent] = &[
toml: include_str!("summarizer/agent.toml"),
prompt_fn: super::summarizer::prompt::build,
},
BuiltinAgent {
id: "help",
toml: include_str!("help/agent.toml"),
prompt_fn: super::help::prompt::build,
},
];
/// Parse every entry in [`BUILTINS`] into an [`AgentDefinition`].
@@ -172,7 +177,7 @@ mod tests {
fn all_builtins_parse() {
let defs = load_builtins().expect("built-in TOML must parse");
assert_eq!(defs.len(), BUILTINS.len());
assert_eq!(defs.len(), 14, "expected 14 built-in agents");
assert_eq!(defs.len(), 15, "expected 15 built-in agents");
}
#[test]
@@ -365,13 +370,89 @@ mod tests {
assert_eq!(def.max_iterations, 8);
}
#[test]
fn help_uses_gitbooks_tools_and_is_read_only() {
let def = find("help");
assert_eq!(def.sandbox_mode, SandboxMode::ReadOnly);
match &def.tools {
ToolScope::Named(tools) => {
assert!(
tools.iter().any(|t| t == "gitbooks_search"),
"help needs gitbooks_search"
);
assert!(
tools.iter().any(|t| t == "gitbooks_get_page"),
"help needs gitbooks_get_page"
);
assert!(
tools.iter().any(|t| t == "memory_recall"),
"help needs memory_recall for personalisation"
);
// Help is docs-only — no write/exec tools.
assert!(!tools.iter().any(|t| t == "shell"));
assert!(!tools.iter().any(|t| t == "file_write"));
assert!(!tools.iter().any(|t| t == "curl"));
assert!(!tools.iter().any(|t| t == "spawn_subagent"));
}
ToolScope::Wildcard => panic!("help must have a Named tool scope"),
}
assert!(def.omit_identity);
assert!(def.omit_safety_preamble);
assert!(!def.omit_memory_context);
}
#[test]
fn researcher_has_curl_for_artifact_downloads() {
let def = find("researcher");
match &def.tools {
ToolScope::Named(tools) => {
assert!(
tools.iter().any(|t| t == "curl"),
"researcher needs curl for artifact downloads"
);
assert!(
tools.iter().any(|t| t == "http_request"),
"researcher still needs http_request"
);
}
ToolScope::Wildcard => panic!("researcher must have Named tool scope"),
}
}
#[test]
fn code_executor_has_curl_for_artifact_downloads() {
let def = find("code_executor");
match &def.tools {
ToolScope::Named(tools) => {
assert!(
tools.iter().any(|t| t == "curl"),
"code_executor needs curl for artifact/dataset fetches"
);
}
ToolScope::Wildcard => panic!("code_executor must have Named tool scope"),
}
}
#[test]
fn orchestrator_does_not_get_curl() {
// Per design: curl is a `Write` permission tool that writes
// to the workspace. The orchestrator delegates rather than
// executing — code_executor / researcher own actual downloads.
let def = find("orchestrator");
if let ToolScope::Named(tools) = &def.tools {
assert!(
!tools.iter().any(|t| t == "curl"),
"orchestrator must not have curl — it should delegate"
);
}
}
#[test]
fn welcome_has_onboarding_and_memory_tools() {
let def = find("welcome");
assert_eq!(def.sandbox_mode, SandboxMode::ReadOnly);
match &def.tools {
ToolScope::Named(tools) => {
assert_eq!(tools.len(), 4, "welcome should have exactly four tools");
assert!(
tools.iter().any(|t| t == "check_onboarding_status"),
"welcome needs check_onboarding_status"
@@ -388,6 +469,18 @@ mod tests {
tools.iter().any(|t| t == "composio_authorize"),
"welcome needs composio_authorize"
);
assert!(
tools.iter().any(|t| t == "gitbooks_search"),
"welcome needs gitbooks_search to answer 'how does X work' during onboarding"
);
assert!(
tools.iter().any(|t| t == "gitbooks_get_page"),
"welcome needs gitbooks_get_page for full-page lookups"
);
// Welcome must not gain write/exec power; onboarding stays read-only.
assert!(!tools.iter().any(|t| t == "shell"));
assert!(!tools.iter().any(|t| t == "file_write"));
assert!(!tools.iter().any(|t| t == "curl"));
}
ToolScope::Wildcard => panic!("welcome must have a Named tool scope"),
}
+1
View File
@@ -7,6 +7,7 @@ mod loader;
pub mod archivist;
pub mod code_executor;
pub mod critic;
pub mod help;
pub mod integrations_agent;
pub mod morning_briefing;
pub mod orchestrator;
@@ -14,4 +14,4 @@ omit_skills_catalog = true
hint = "agentic"
[tools]
named = ["http_request", "web_search", "file_read", "memory_recall"]
named = ["http_request", "curl", "web_search", "file_read", "memory_recall"]
@@ -30,9 +30,13 @@ hint = "agentic"
# complete_onboarding: finalize the welcome flow once ready_to_complete is true.
# memory_recall: pull additional user details beyond injected context.
# composio_authorize: start an OAuth flow for a toolkit (e.g. gmail).
# gitbooks_*: answer "how does X work" / "what can OpenHuman do" questions
# during onboarding from the real product docs instead of guessing.
named = [
"check_onboarding_status",
"complete_onboarding",
"memory_recall",
"composio_authorize",
"gitbooks_search",
"gitbooks_get_page",
]
+3 -1
View File
@@ -11,7 +11,9 @@ You are the **Welcome** agent — the first agent a new user talks to in OpenHum
## Tools you must use correctly
You have `check_onboarding_status`, `complete_onboarding`, `memory_recall`, and `composio_authorize`.
You have `check_onboarding_status`, `complete_onboarding`, `memory_recall`, `composio_authorize`, `gitbooks_search`, and `gitbooks_get_page`.
If the user asks a "how does X work" / "what can OpenHuman do" / "where is the setting for…" question during onboarding, call `gitbooks_search` first and ground your answer in the real docs — do **not** guess feature names, flags, or commands. Use `gitbooks_get_page` only when the search excerpt is too partial. Cite the page URL inline so they can read more. Keep doc-grounded replies short and steer the conversation back to setup once they're satisfied — your primary job is still onboarding, not an open-ended Q&A loop.
| Tool | What it does |
|------|--------------|
+10 -9
View File
@@ -26,15 +26,16 @@ pub use schema::{
build_runtime_proxy_client_with_timeouts, runtime_proxy_config, set_runtime_proxy_config,
AgentConfig, AuditConfig, AutocompleteConfig, AutonomyConfig, BrowserComputerUseConfig,
BrowserConfig, ChannelsConfig, ComposioConfig, Config, ContextConfig, CostConfig, CronConfig,
DelegateAgentConfig, DictationActivationMode, DictationConfig, DiscordConfig,
DockerRuntimeConfig, EmbeddingRouteConfig, HeartbeatConfig, HttpRequestConfig, IMessageConfig,
IntegrationToggle, IntegrationsConfig, LarkConfig, LearningConfig, LocalAiConfig, MatrixConfig,
MemoryConfig, ModelRouteConfig, MultimodalConfig, ObservabilityConfig, ProxyConfig, ProxyScope,
ReflectionSource, ReliabilityConfig, ResourceLimitsConfig, RuntimeConfig, SandboxBackend,
SandboxConfig, SchedulerConfig, ScreenIntelligenceConfig, SecretsConfig, SecurityConfig,
SlackConfig, StorageConfig, StorageProviderConfig, StorageProviderSection, StreamMode,
TelegramConfig, UpdateConfig, VoiceActivationMode, VoiceServerConfig, WebSearchConfig,
WebhookConfig, DEFAULT_MODEL, MODEL_AGENTIC_V1, MODEL_CODING_V1, MODEL_REASONING_V1,
CurlConfig, DelegateAgentConfig, DictationActivationMode, DictationConfig, DiscordConfig,
DockerRuntimeConfig, EmbeddingRouteConfig, GitbooksConfig, HeartbeatConfig, HttpRequestConfig,
IMessageConfig, IntegrationToggle, IntegrationsConfig, LarkConfig, LearningConfig,
LocalAiConfig, MatrixConfig, MemoryConfig, ModelRouteConfig, MultimodalConfig,
ObservabilityConfig, ProxyConfig, ProxyScope, ReflectionSource, ReliabilityConfig,
ResourceLimitsConfig, RuntimeConfig, SandboxBackend, SandboxConfig, SchedulerConfig,
ScreenIntelligenceConfig, SecretsConfig, SecurityConfig, SlackConfig, StorageConfig,
StorageProviderConfig, StorageProviderSection, StreamMode, TelegramConfig, UpdateConfig,
VoiceActivationMode, VoiceServerConfig, WebSearchConfig, WebhookConfig, DEFAULT_MODEL,
MODEL_AGENTIC_V1, MODEL_CODING_V1, MODEL_REASONING_V1,
};
pub use schema::{
clear_active_user, default_root_openhuman_dir, pre_login_user_dir, read_active_user_id,
+3 -3
View File
@@ -57,9 +57,9 @@ pub use storage_memory::{
MemoryConfig, MemoryTreeConfig, StorageConfig, StorageProviderConfig, StorageProviderSection,
};
pub use tools::{
BrowserComputerUseConfig, BrowserConfig, ComposioConfig, ComputerControlConfig,
HttpRequestConfig, IntegrationToggle, IntegrationsConfig, MultimodalConfig, SecretsConfig,
WebSearchConfig,
BrowserComputerUseConfig, BrowserConfig, ComposioConfig, ComputerControlConfig, CurlConfig,
GitbooksConfig, HttpRequestConfig, IntegrationToggle, IntegrationsConfig, MultimodalConfig,
SecretsConfig, WebSearchConfig,
};
pub use update::UpdateConfig;
mod voice_server;
+69
View File
@@ -148,6 +148,75 @@ fn default_http_timeout_secs() -> u64 {
30
}
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
pub struct CurlConfig {
/// Subdirectory under `workspace_dir` where downloads land. Inputs
/// are resolved relative to this root; absolute paths and `..`
/// segments are rejected.
#[serde(default = "default_curl_dest_subdir")]
pub dest_subdir: String,
/// Hard byte ceiling per download. Streaming aborts and the
/// partial file is removed if exceeded.
#[serde(default = "default_curl_max_download_bytes")]
pub max_download_bytes: u64,
/// Per-request timeout in seconds.
#[serde(default = "default_curl_timeout_secs")]
pub timeout_secs: u64,
}
fn default_curl_dest_subdir() -> String {
"downloads".into()
}
fn default_curl_max_download_bytes() -> u64 {
50 * 1024 * 1024
}
fn default_curl_timeout_secs() -> u64 {
120
}
impl Default for CurlConfig {
fn default() -> Self {
Self {
dest_subdir: default_curl_dest_subdir(),
max_download_bytes: default_curl_max_download_bytes(),
timeout_secs: default_curl_timeout_secs(),
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
pub struct GitbooksConfig {
/// When `true`, register `gitbooks_search` and `gitbooks_get_page`.
#[serde(default = "defaults::default_true")]
pub enabled: bool,
/// MCP endpoint URL for the OpenHuman GitBook docs.
#[serde(default = "default_gitbooks_endpoint")]
pub endpoint: String,
/// Per-request timeout in seconds.
#[serde(default = "default_gitbooks_timeout_secs")]
pub timeout_secs: u64,
}
fn default_gitbooks_endpoint() -> String {
"https://tinyhumans.gitbook.io/openhuman/~gitbook/mcp".into()
}
fn default_gitbooks_timeout_secs() -> u64 {
30
}
impl Default for GitbooksConfig {
fn default() -> Self {
Self {
enabled: defaults::default_true(),
endpoint: default_gitbooks_endpoint(),
timeout_secs: default_gitbooks_timeout_secs(),
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
pub struct WebSearchConfig {
#[serde(default = "default_web_search_max_results")]
+8
View File
@@ -103,6 +103,12 @@ pub struct Config {
#[serde(default)]
pub http_request: HttpRequestConfig,
#[serde(default)]
pub curl: CurlConfig,
#[serde(default)]
pub gitbooks: GitbooksConfig,
#[serde(default)]
pub multimodal: MultimodalConfig,
@@ -240,6 +246,8 @@ impl Default for Config {
secrets: SecretsConfig::default(),
browser: BrowserConfig::default(),
http_request: HttpRequestConfig::default(),
curl: CurlConfig::default(),
gitbooks: GitbooksConfig::default(),
multimodal: MultimodalConfig::default(),
web_search: WebSearchConfig::default(),
proxy: ProxyConfig::default(),
+544
View File
@@ -0,0 +1,544 @@
//! `curl` — download files from the web to a path under the workspace.
//!
//! Distinct from `http_request`: instead of returning the body inline
//! (size-capped), `curl` streams to disk with a hard byte ceiling. Same
//! SSRF/allowlist guards (shared via `url_guard`), shares
//! `http_request.allowed_domains` so there is one allowlist to reason
//! about.
use super::url_guard::{normalize_allowed_domains, validate_url};
use crate::openhuman::security::SecurityPolicy;
use crate::openhuman::tools::traits::{PermissionLevel, Tool, ToolResult};
use async_trait::async_trait;
use futures_util::StreamExt;
use sha2::{Digest, Sha256};
use std::path::{Component, Path, PathBuf};
use std::sync::Arc;
use std::time::Duration;
use tokio::fs;
use tokio::io::AsyncWriteExt;
pub struct CurlTool {
security: Arc<SecurityPolicy>,
allowed_domains: Vec<String>,
workspace_dir: PathBuf,
dest_subdir: String,
max_download_bytes: u64,
timeout_secs: u64,
}
impl CurlTool {
pub fn new(
security: Arc<SecurityPolicy>,
allowed_domains: Vec<String>,
workspace_dir: PathBuf,
dest_subdir: String,
max_download_bytes: u64,
timeout_secs: u64,
) -> Self {
Self {
security,
allowed_domains: normalize_allowed_domains(allowed_domains),
workspace_dir,
dest_subdir: sanitize_dest_subdir(&dest_subdir),
max_download_bytes,
timeout_secs,
}
}
/// Resolve a user-supplied dest path to an absolute path inside
/// `<workspace>/<dest_subdir>`. Rejects absolute paths, `..`
/// segments, and any other escape attempts.
fn resolve_dest(&self, dest: &str) -> anyhow::Result<PathBuf> {
let trimmed = dest.trim();
if trimmed.is_empty() {
anyhow::bail!("dest_path cannot be empty");
}
let p = Path::new(trimmed);
if p.is_absolute() {
anyhow::bail!("dest_path must be relative — got absolute path");
}
for component in p.components() {
match component {
Component::Normal(_) => {}
Component::CurDir => {}
Component::ParentDir => {
anyhow::bail!("dest_path may not contain '..'");
}
Component::Prefix(_) | Component::RootDir => {
anyhow::bail!("dest_path must be relative");
}
}
}
let root = self.workspace_dir.join(&self.dest_subdir);
let resolved = root.join(p);
// Belt-and-braces: ensure the resolved path still lives under root.
// Lexical check is sufficient because we already rejected `..`.
if !resolved.starts_with(&root) {
anyhow::bail!("dest_path resolves outside the downloads root");
}
Ok(resolved)
}
fn validate_url(&self, raw_url: &str) -> anyhow::Result<String> {
validate_url(raw_url, &self.allowed_domains)
}
fn default_filename_from_url(url: &str) -> String {
let after_scheme = url.split_once("://").map(|(_, rest)| rest).unwrap_or(url);
let path_part = after_scheme.split_once('/').map(|(_, p)| p).unwrap_or("");
let last = path_part
.split('?')
.next()
.unwrap_or("")
.rsplit('/')
.next()
.unwrap_or("");
let cleaned: String = last
.chars()
.filter(|c| c.is_alphanumeric() || matches!(c, '.' | '-' | '_'))
.collect();
if cleaned.is_empty() {
"download.bin".into()
} else {
cleaned
}
}
}
#[async_trait]
impl Tool for CurlTool {
fn name(&self) -> &str {
"curl"
}
fn description(&self) -> &str {
"Download a file from an http(s) URL into the workspace. The body is streamed to disk \
with a hard byte ceiling. Same allowlist as `http_request`. Returns the saved path, \
bytes written, content-type, and SHA-256 of the file."
}
fn parameters_schema(&self) -> serde_json::Value {
serde_json::json!({
"type": "object",
"properties": {
"url": {
"type": "string",
"description": "HTTP or HTTPS URL of the file to download"
},
"dest_path": {
"type": "string",
"description": "Destination path relative to the downloads root inside the workspace. No '..' or absolute paths. If omitted, the filename is inferred from the URL."
},
"headers": {
"type": "object",
"description": "Optional HTTP headers (e.g. {\"Authorization\": \"Bearer …\"})",
"default": {}
}
},
"required": ["url"]
})
}
fn permission_level(&self) -> PermissionLevel {
PermissionLevel::Write
}
async fn execute(&self, args: serde_json::Value) -> anyhow::Result<ToolResult> {
let url = args
.get("url")
.and_then(|v| v.as_str())
.ok_or_else(|| anyhow::anyhow!("Missing 'url' parameter"))?;
let dest_arg = args.get("dest_path").and_then(|v| v.as_str());
let headers_val = args
.get("headers")
.cloned()
.unwrap_or_else(|| serde_json::json!({}));
if !self.security.can_act() {
tracing::debug!(target: "[curl]", url = %url, "blocked: autonomy read-only");
return Ok(ToolResult::error("Action blocked: autonomy is read-only"));
}
if !self.security.record_action() {
tracing::debug!(target: "[curl]", url = %url, "blocked: rate limit");
return Ok(ToolResult::error("Action blocked: rate limit exceeded"));
}
let url = match self.validate_url(url) {
Ok(v) => v,
Err(e) => {
tracing::debug!(target: "[curl]", url = %url, reason = %e, "url validation failed");
return Ok(ToolResult::error(e.to_string()));
}
};
let dest = match dest_arg {
Some(d) => d.to_string(),
None => Self::default_filename_from_url(&url),
};
let dest_path = match self.resolve_dest(&dest) {
Ok(p) => p,
Err(e) => {
tracing::debug!(target: "[curl]", url = %url, dest = %dest, reason = %e, "dest_path rejected");
return Ok(ToolResult::error(e.to_string()));
}
};
if let Some(parent) = dest_path.parent() {
if let Err(e) = fs::create_dir_all(parent).await {
tracing::error!(target: "[curl]", url = %url, dest = %dest_path.display(), reason = %e, "create_dir_all failed");
return Ok(ToolResult::error(format!(
"Failed to create destination directory: {e}"
)));
}
}
let builder = reqwest::Client::builder()
.timeout(Duration::from_secs(self.timeout_secs))
.connect_timeout(Duration::from_secs(10))
.redirect(reqwest::redirect::Policy::none());
let builder =
crate::openhuman::config::apply_runtime_proxy_to_builder(builder, "tool.curl");
let client = match builder.build() {
Ok(c) => c,
Err(e) => {
tracing::error!(target: "[curl]", reason = %e, "HTTP client build failed");
return Ok(ToolResult::error(format!("HTTP client build failed: {e}")));
}
};
let mut request = client.get(&url);
if let Some(obj) = headers_val.as_object() {
for (k, v) in obj {
if let Some(s) = v.as_str() {
request = request.header(k, s);
}
}
}
tracing::debug!(target: "[curl]", url = %url, dest = %dest_path.display(), "starting download");
let response = match request.send().await {
Ok(r) => r,
Err(e) => {
tracing::error!(target: "[curl]", url = %url, reason = %e, "request send failed");
return Ok(ToolResult::error(format!("Request failed: {e}")));
}
};
let status = response.status();
if !status.is_success() {
tracing::debug!(target: "[curl]", url = %url, status = %status.as_u16(), "non-success HTTP status");
return Ok(ToolResult::error(format!(
"HTTP {} {}",
status.as_u16(),
status.canonical_reason().unwrap_or("Unknown")
)));
}
let content_type = response
.headers()
.get(reqwest::header::CONTENT_TYPE)
.and_then(|v| v.to_str().ok())
.unwrap_or("application/octet-stream")
.to_string();
let mut file = match fs::File::create(&dest_path).await {
Ok(f) => f,
Err(e) => {
tracing::error!(target: "[curl]", dest = %dest_path.display(), reason = %e, "fs::File::create failed");
return Ok(ToolResult::error(format!(
"Failed to create destination file: {e}"
)));
}
};
let mut hasher = Sha256::new();
let mut bytes_written: u64 = 0;
let mut stream = response.bytes_stream();
while let Some(chunk) = stream.next().await {
let chunk = match chunk {
Ok(c) => c,
Err(e) => {
drop(file);
if let Err(rm) = fs::remove_file(&dest_path).await {
tracing::debug!(target: "[curl]", dest = %dest_path.display(), reason = %rm, "cleanup remove_file failed");
}
tracing::error!(target: "[curl]", url = %url, bytes_written, reason = %e, "stream error");
return Ok(ToolResult::error(format!("Stream error: {e}")));
}
};
if bytes_written.saturating_add(chunk.len() as u64) > self.max_download_bytes {
let _ = file.flush().await;
drop(file);
if let Err(rm) = fs::remove_file(&dest_path).await {
tracing::debug!(target: "[curl]", dest = %dest_path.display(), reason = %rm, "cleanup remove_file failed");
}
tracing::error!(target: "[curl]", url = %url, bytes_written, max = self.max_download_bytes, "size cap exceeded — download aborted");
return Ok(ToolResult::error(format!(
"Download exceeded max_download_bytes ({} bytes)",
self.max_download_bytes
)));
}
if let Err(e) = file.write_all(&chunk).await {
drop(file);
if let Err(rm) = fs::remove_file(&dest_path).await {
tracing::debug!(target: "[curl]", dest = %dest_path.display(), reason = %rm, "cleanup remove_file failed");
}
tracing::error!(target: "[curl]", dest = %dest_path.display(), bytes_written, reason = %e, "write_all failed");
return Ok(ToolResult::error(format!("Write failed: {e}")));
}
hasher.update(&chunk);
bytes_written += chunk.len() as u64;
}
if let Err(e) = file.flush().await {
drop(file);
if let Err(rm) = fs::remove_file(&dest_path).await {
tracing::debug!(target: "[curl]", dest = %dest_path.display(), reason = %rm, "cleanup remove_file failed");
}
tracing::error!(target: "[curl]", dest = %dest_path.display(), bytes_written, reason = %e, "flush failed");
return Ok(ToolResult::error(format!("Flush failed: {e}")));
}
let sha256 = format!("{:x}", hasher.finalize());
tracing::debug!(
target: "[curl]",
url = %url,
dest = %dest_path.display(),
bytes = bytes_written,
content_type = %content_type,
sha256 = %sha256,
"download complete"
);
let payload = serde_json::json!({
"path": dest_path.display().to_string(),
"bytes_written": bytes_written,
"content_type": content_type,
"sha256": sha256,
});
Ok(ToolResult::success(payload.to_string()))
}
}
/// Sanitize the configured `dest_subdir` so a malicious or misconfigured
/// `[curl].dest_subdir` cannot escape the workspace via absolute paths
/// or `..` segments. Drops disallowed components rather than panicking;
/// falls back to `"downloads"` if everything is filtered out.
fn sanitize_dest_subdir(raw: &str) -> String {
let trimmed = raw.trim();
if trimmed.is_empty() {
return "downloads".into();
}
let p = Path::new(trimmed);
let mut buf = PathBuf::new();
for component in p.components() {
match component {
Component::Normal(c) => buf.push(c),
// Drop everything else: absolute roots, prefixes, parent dirs, cur dirs.
_ => continue,
}
}
if buf.as_os_str().is_empty() {
return "downloads".into();
}
buf.to_string_lossy().into_owned()
}
#[cfg(test)]
mod tests {
use super::*;
use crate::openhuman::security::SecurityPolicy;
use tempfile::TempDir;
fn tool(tmp: &TempDir, allow: Vec<&str>) -> CurlTool {
CurlTool::new(
Arc::new(SecurityPolicy::default()),
allow.into_iter().map(String::from).collect(),
tmp.path().to_path_buf(),
"downloads".into(),
1024 * 1024,
30,
)
}
#[test]
fn sanitize_dest_subdir_strips_absolute_paths() {
assert_eq!(sanitize_dest_subdir("/etc/passwd"), "etc/passwd");
assert_eq!(sanitize_dest_subdir("//foo"), "foo");
}
#[test]
fn sanitize_dest_subdir_strips_parent_segments() {
assert_eq!(sanitize_dest_subdir("../../etc"), "etc");
assert_eq!(sanitize_dest_subdir("a/../b"), "a/b");
}
#[test]
fn sanitize_dest_subdir_falls_back_to_downloads() {
assert_eq!(sanitize_dest_subdir(""), "downloads");
assert_eq!(sanitize_dest_subdir(" "), "downloads");
assert_eq!(sanitize_dest_subdir(".."), "downloads");
assert_eq!(sanitize_dest_subdir("/"), "downloads");
}
#[test]
fn sanitize_dest_subdir_keeps_normal_paths() {
assert_eq!(sanitize_dest_subdir("downloads"), "downloads");
assert_eq!(sanitize_dest_subdir("artifacts/build"), "artifacts/build");
}
#[test]
fn new_sanitizes_malicious_dest_subdir() {
let tmp = TempDir::new().unwrap();
let t = CurlTool::new(
Arc::new(SecurityPolicy::default()),
vec!["example.com".into()],
tmp.path().to_path_buf(),
"../../etc".into(),
1024,
30,
);
let resolved = t.resolve_dest("file.txt").unwrap();
// Sanitizer reduced "../../etc" to "etc"; resolution must stay under workspace.
assert!(resolved.starts_with(tmp.path().join("etc")));
assert!(resolved.starts_with(tmp.path()));
}
#[test]
fn resolve_dest_normal() {
let tmp = TempDir::new().unwrap();
let t = tool(&tmp, vec!["example.com"]);
let p = t.resolve_dest("foo/bar.txt").unwrap();
assert!(p.starts_with(tmp.path().join("downloads")));
assert!(p.ends_with("foo/bar.txt"));
}
#[test]
fn resolve_dest_rejects_absolute() {
let tmp = TempDir::new().unwrap();
let t = tool(&tmp, vec!["example.com"]);
let err = t.resolve_dest("/etc/passwd").unwrap_err().to_string();
assert!(err.contains("relative"));
}
#[test]
fn resolve_dest_rejects_parent_dir() {
let tmp = TempDir::new().unwrap();
let t = tool(&tmp, vec!["example.com"]);
let err = t.resolve_dest("../etc/passwd").unwrap_err().to_string();
assert!(err.contains(".."));
}
#[test]
fn resolve_dest_rejects_nested_parent_dir() {
let tmp = TempDir::new().unwrap();
let t = tool(&tmp, vec!["example.com"]);
let err = t.resolve_dest("a/../../b").unwrap_err().to_string();
assert!(err.contains(".."));
}
#[test]
fn resolve_dest_rejects_empty() {
let tmp = TempDir::new().unwrap();
let t = tool(&tmp, vec!["example.com"]);
assert!(t.resolve_dest("").is_err());
assert!(t.resolve_dest(" ").is_err());
}
#[test]
fn default_filename_from_url_basic() {
assert_eq!(
CurlTool::default_filename_from_url("https://example.com/foo/bar.zip"),
"bar.zip"
);
}
#[test]
fn default_filename_from_url_query_stripped() {
assert_eq!(
CurlTool::default_filename_from_url("https://example.com/file.tar.gz?token=x"),
"file.tar.gz"
);
}
#[test]
fn default_filename_from_url_root_falls_back() {
assert_eq!(
CurlTool::default_filename_from_url("https://example.com/"),
"download.bin"
);
}
#[tokio::test]
async fn execute_blocks_when_rate_limited() {
let tmp = TempDir::new().unwrap();
let security = Arc::new(SecurityPolicy {
max_actions_per_hour: 0,
..SecurityPolicy::default()
});
let t = CurlTool::new(
security,
vec!["example.com".into()],
tmp.path().into(),
"downloads".into(),
1024,
30,
);
let result = t
.execute(serde_json::json!({"url": "https://example.com/x"}))
.await
.unwrap();
assert!(result.is_error);
assert!(result.output().contains("rate limit"));
}
/// Live integration smoke: downloads example.com (a tiny, stable
/// public page). Gated behind `OPENHUMAN_CURL_LIVE_TEST=1` so CI /
/// offline runs don't depend on the network.
#[tokio::test]
async fn live_download_example_com() {
if std::env::var("OPENHUMAN_CURL_LIVE_TEST").ok().as_deref() != Some("1") {
return;
}
let tmp = TempDir::new().unwrap();
let t = tool(&tmp, vec!["example.com"]);
let result = t
.execute(serde_json::json!({
"url": "https://example.com/",
"dest_path": "example.html"
}))
.await
.unwrap();
assert!(!result.is_error, "live curl errored: {}", result.output());
let payload: serde_json::Value = serde_json::from_str(&result.output()).unwrap();
let bytes = payload["bytes_written"].as_u64().unwrap();
assert!(bytes > 100, "unexpectedly small download: {bytes} bytes");
let path = payload["path"].as_str().unwrap();
let content = std::fs::read_to_string(path).unwrap();
assert!(content.to_lowercase().contains("example domain"));
}
#[tokio::test]
async fn execute_rejects_allowlist_miss() {
let tmp = TempDir::new().unwrap();
let t = tool(&tmp, vec!["example.com"]);
let result = t
.execute(serde_json::json!({"url": "https://other.example.org/x"}))
.await
.unwrap();
assert!(result.is_error);
assert!(result.output().contains("allowed_domains"));
}
}
@@ -0,0 +1,425 @@
//! `gitbooks` — answer questions about OpenHuman by talking to the
//! GitBook MCP server.
//!
//! GitBook hosts a stateless Streamable-HTTP MCP server that exposes
//! exactly two tools:
//!
//! - `searchDocumentation { query }` — returns excerpts + page links
//! - `getPage { url }` — returns the full markdown of a page
//!
//! We mirror them as `gitbooks_search` and `gitbooks_get_page`. The
//! server returns each JSON-RPC response in a single
//! `event: message\ndata: {…}` SSE frame, so we do a tiny inline
//! parse — no need to pull in a full SSE/MCP client crate yet.
use crate::openhuman::tools::traits::{PermissionLevel, Tool, ToolResult};
use async_trait::async_trait;
use serde_json::{json, Value};
use std::sync::atomic::{AtomicI64, Ordering};
use std::sync::Arc;
use std::time::Duration;
/// Minimal MCP-over-HTTP client for stateless servers (no session id).
struct McpHttpClient {
endpoint: String,
timeout_secs: u64,
next_id: AtomicI64,
}
impl McpHttpClient {
fn new(endpoint: String, timeout_secs: u64) -> Self {
Self {
endpoint,
timeout_secs,
next_id: AtomicI64::new(1),
}
}
async fn call_tool(&self, name: &str, arguments: Value) -> anyhow::Result<Value> {
let id = self.next_id.fetch_add(1, Ordering::Relaxed);
let body = json!({
"jsonrpc": "2.0",
"id": id,
"method": "tools/call",
"params": { "name": name, "arguments": arguments }
});
let builder = reqwest::Client::builder()
.timeout(Duration::from_secs(self.timeout_secs))
.connect_timeout(Duration::from_secs(10))
.redirect(reqwest::redirect::Policy::none());
let builder =
crate::openhuman::config::apply_runtime_proxy_to_builder(builder, "tool.gitbooks");
let client = builder.build()?;
// Log only the redacted host so query strings / path params /
// any future bearer-token-bearing endpoints don't leak into
// logs aggregated for triage.
tracing::debug!(
target: "[gitbooks]",
endpoint = %redact_endpoint(&self.endpoint),
tool = %name,
"MCP tools/call"
);
let resp = client
.post(&self.endpoint)
.header("Content-Type", "application/json")
.header("Accept", "application/json, text/event-stream")
.body(serde_json::to_vec(&body)?)
.send()
.await?;
let status = resp.status();
let content_type = resp
.headers()
.get(reqwest::header::CONTENT_TYPE)
.and_then(|v| v.to_str().ok())
.unwrap_or("")
.to_string();
let text = resp.text().await?;
if !status.is_success() {
anyhow::bail!("MCP HTTP {} — {}", status.as_u16(), text);
}
let payload: Value = if content_type.starts_with("text/event-stream") {
parse_sse_message(&text)?
} else {
serde_json::from_str(&text).map_err(|e| {
anyhow::anyhow!("Failed to parse MCP JSON response: {e} — body: {text}")
})?
};
if let Some(err) = payload.get("error") {
anyhow::bail!("MCP error: {err}");
}
let result = payload
.get("result")
.ok_or_else(|| anyhow::anyhow!("MCP response missing 'result': {payload}"))?
.clone();
Ok(result)
}
}
/// Reduce a configured endpoint URL to `scheme://host[:port]` for
/// safe logging. Anything that doesn't parse as a recognisable
/// http(s) URL is reported as `<redacted>` rather than echoed.
fn redact_endpoint(raw: &str) -> String {
let trimmed = raw.trim();
let (scheme, rest) = if let Some(r) = trimmed.strip_prefix("https://") {
("https", r)
} else if let Some(r) = trimmed.strip_prefix("http://") {
("http", r)
} else {
return "<redacted>".into();
};
let authority = rest.split(['/', '?', '#']).next().unwrap_or("");
if authority.is_empty() || authority.contains('@') {
return "<redacted>".into();
}
format!("{scheme}://{authority}")
}
/// Parse the first `data: {…}` line from a Streamable-HTTP SSE
/// response. The GitBook server emits exactly one frame per JSON-RPC
/// request, so we do not need a full SSE state machine.
fn parse_sse_message(body: &str) -> anyhow::Result<Value> {
for line in body.lines() {
let line = line.trim_end_matches('\r');
if let Some(data) = line.strip_prefix("data:") {
let data = data.trim_start();
if data.is_empty() {
continue;
}
return serde_json::from_str(data).map_err(|e| {
anyhow::anyhow!("Failed to parse SSE data frame: {e} — line: {data}")
});
}
}
anyhow::bail!("No SSE data frame found in MCP response: {body}")
}
/// Render an MCP `tools/call` result into a single string for the
/// agent. MCP returns `{ content: [{ type, text }, …], isError? }`.
fn render_tool_result(result: &Value) -> ToolResult {
let is_error = result
.get("isError")
.and_then(Value::as_bool)
.unwrap_or(false);
let mut out = String::new();
if let Some(content) = result.get("content").and_then(Value::as_array) {
for block in content {
if let Some(t) = block.get("text").and_then(Value::as_str) {
if !out.is_empty() {
out.push_str("\n\n");
}
out.push_str(t);
}
}
}
if out.is_empty() {
out = result.to_string();
}
if is_error {
ToolResult::error(out)
} else {
ToolResult::success(out)
}
}
// ── Search ─────────────────────────────────────────────────────────
pub struct GitbooksSearchTool {
client: Arc<McpHttpClient>,
}
impl GitbooksSearchTool {
pub fn new(endpoint: String, timeout_secs: u64) -> Self {
Self {
client: Arc::new(McpHttpClient::new(endpoint, timeout_secs)),
}
}
}
#[async_trait]
impl Tool for GitbooksSearchTool {
fn name(&self) -> &str {
"gitbooks_search"
}
fn description(&self) -> &str {
"Search the OpenHuman product documentation. Use this to answer questions about how \
OpenHuman works, find features, look up configuration, or locate guides. Returns \
excerpts with page titles and links — follow up with `gitbooks_get_page` for the \
full markdown of a page."
}
fn parameters_schema(&self) -> Value {
json!({
"type": "object",
"properties": {
"query": {
"type": "string",
"description": "Natural-language question or keyword query about OpenHuman."
}
},
"required": ["query"]
})
}
fn permission_level(&self) -> PermissionLevel {
PermissionLevel::ReadOnly
}
async fn execute(&self, args: Value) -> anyhow::Result<ToolResult> {
let query = args
.get("query")
.and_then(Value::as_str)
.ok_or_else(|| anyhow::anyhow!("Missing 'query' parameter"))?;
if query.trim().is_empty() {
return Ok(ToolResult::error("query cannot be empty"));
}
match self
.client
.call_tool("searchDocumentation", json!({ "query": query }))
.await
{
Ok(result) => Ok(render_tool_result(&result)),
Err(e) => Ok(ToolResult::error(format!("gitbooks_search failed: {e}"))),
}
}
}
// ── Get page ──────────────────────────────────────────────────────
pub struct GitbooksGetPageTool {
client: Arc<McpHttpClient>,
}
impl GitbooksGetPageTool {
pub fn new(endpoint: String, timeout_secs: u64) -> Self {
Self {
client: Arc::new(McpHttpClient::new(endpoint, timeout_secs)),
}
}
}
#[async_trait]
impl Tool for GitbooksGetPageTool {
fn name(&self) -> &str {
"gitbooks_get_page"
}
fn description(&self) -> &str {
"Fetch the full markdown of a specific OpenHuman documentation page by URL. Pair this \
with `gitbooks_search` — search returns partial excerpts; use this to get the \
complete page when more detail is needed."
}
fn parameters_schema(&self) -> Value {
json!({
"type": "object",
"properties": {
"url": {
"type": "string",
"description": "The full URL of the OpenHuman documentation page (e.g. https://tinyhumans.gitbook.io/openhuman/getting-started)."
}
},
"required": ["url"]
})
}
fn permission_level(&self) -> PermissionLevel {
PermissionLevel::ReadOnly
}
async fn execute(&self, args: Value) -> anyhow::Result<ToolResult> {
let url = args
.get("url")
.and_then(Value::as_str)
.ok_or_else(|| anyhow::anyhow!("Missing 'url' parameter"))?;
if url.trim().is_empty() {
return Ok(ToolResult::error("url cannot be empty"));
}
match self
.client
.call_tool("getPage", json!({ "url": url }))
.await
{
Ok(result) => Ok(render_tool_result(&result)),
Err(e) => Ok(ToolResult::error(format!("gitbooks_get_page failed: {e}"))),
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn redact_endpoint_keeps_only_origin() {
assert_eq!(
redact_endpoint("https://tinyhumans.gitbook.io/openhuman/~gitbook/mcp"),
"https://tinyhumans.gitbook.io"
);
assert_eq!(
redact_endpoint("http://example.com:8080/path?token=secret"),
"http://example.com:8080"
);
}
#[test]
fn redact_endpoint_rejects_userinfo_and_unknown_schemes() {
assert_eq!(
redact_endpoint("https://user:pass@example.com/x"),
"<redacted>"
);
assert_eq!(redact_endpoint("ftp://example.com"), "<redacted>");
assert_eq!(redact_endpoint(""), "<redacted>");
}
#[test]
fn parse_sse_extracts_data_frame() {
let body = "event: message\ndata: {\"jsonrpc\":\"2.0\",\"id\":1,\"result\":{\"x\":1}}\n\n";
let v = parse_sse_message(body).unwrap();
assert_eq!(v["result"]["x"], 1);
}
#[test]
fn parse_sse_handles_crlf() {
let body = "event: message\r\ndata: {\"jsonrpc\":\"2.0\",\"id\":1,\"result\":{}}\r\n\r\n";
assert!(parse_sse_message(body).is_ok());
}
#[test]
fn parse_sse_errors_when_no_data_line() {
let body = "event: ping\n\n";
assert!(parse_sse_message(body).is_err());
}
#[test]
fn parse_sse_errors_on_invalid_json() {
let body = "data: not-json\n\n";
assert!(parse_sse_message(body).is_err());
}
#[test]
fn render_tool_result_concatenates_text_blocks() {
let r = json!({
"content": [
{"type": "text", "text": "first"},
{"type": "text", "text": "second"}
]
});
let out = render_tool_result(&r);
assert!(!out.is_error);
assert!(out.output().contains("first"));
assert!(out.output().contains("second"));
}
#[test]
fn render_tool_result_marks_errors() {
let r = json!({
"content": [{"type": "text", "text": "boom"}],
"isError": true
});
let out = render_tool_result(&r);
assert!(out.is_error);
assert!(out.output().contains("boom"));
}
#[test]
fn render_tool_result_falls_back_to_raw_json() {
let r = json!({"weird": "shape"});
let out = render_tool_result(&r);
assert!(out.output().contains("weird"));
}
#[tokio::test]
async fn search_rejects_empty_query() {
let t = GitbooksSearchTool::new("https://example.com/mcp".into(), 5);
let result = t.execute(json!({"query": " "})).await.unwrap();
assert!(result.is_error);
assert!(result.output().contains("empty"));
}
#[tokio::test]
async fn get_page_rejects_empty_url() {
let t = GitbooksGetPageTool::new("https://example.com/mcp".into(), 5);
let result = t.execute(json!({"url": ""})).await.unwrap();
assert!(result.is_error);
}
/// Live integration test against the real GitBook MCP endpoint.
/// Gated behind `OPENHUMAN_GITBOOKS_LIVE_TEST=1` so CI / offline
/// builds don't depend on the public network.
#[tokio::test]
async fn live_search_smoke() {
if std::env::var("OPENHUMAN_GITBOOKS_LIVE_TEST")
.ok()
.as_deref()
!= Some("1")
{
return;
}
let t = GitbooksSearchTool::new(
"https://tinyhumans.gitbook.io/openhuman/~gitbook/mcp".into(),
30,
);
let result = t
.execute(json!({"query": "what is openhuman"}))
.await
.unwrap();
assert!(
!result.is_error,
"live search returned error: {}",
result.output()
);
assert!(!result.output().is_empty());
}
}
@@ -1,3 +1,4 @@
use super::url_guard::{normalize_allowed_domains, validate_url};
use crate::openhuman::security::SecurityPolicy;
use crate::openhuman::tools::traits::{Tool, ToolResult};
use async_trait::async_trait;
@@ -30,37 +31,7 @@ impl HttpRequestTool {
}
fn validate_url(&self, raw_url: &str) -> anyhow::Result<String> {
let url = raw_url.trim();
if url.is_empty() {
anyhow::bail!("URL cannot be empty");
}
if url.chars().any(char::is_whitespace) {
anyhow::bail!("URL cannot contain whitespace");
}
if !url.starts_with("http://") && !url.starts_with("https://") {
anyhow::bail!("Only http:// and https:// URLs are allowed");
}
if self.allowed_domains.is_empty() {
anyhow::bail!(
"HTTP request tool is enabled but no allowed_domains are configured. Add [http_request].allowed_domains in config.toml"
);
}
let host = extract_host(url)?;
if is_private_or_local_host(&host) {
anyhow::bail!("Blocked local/private host: {host}");
}
if !host_matches_allowlist(&host, &self.allowed_domains) {
anyhow::bail!("Host '{host}' is not in http_request.allowed_domains");
}
Ok(url.to_string())
validate_url(raw_url, &self.allowed_domains)
}
fn validate_method(&self, method: &str) -> anyhow::Result<reqwest::Method> {
@@ -225,7 +196,6 @@ impl Tool for HttpRequestTool {
let status = response.status();
let status_code = status.as_u16();
// Get response headers (redact sensitive ones)
let response_headers = response.headers().iter();
let headers_text = response_headers
.map(|(k, _)| {
@@ -239,7 +209,6 @@ impl Tool for HttpRequestTool {
.collect::<Vec<_>>()
.join(", ");
// Get response body with size limit
let response_text = match response.text().await {
Ok(text) => self.truncate_response(&text),
Err(e) => format!("[Failed to read response body: {e}]"),
@@ -264,149 +233,6 @@ impl Tool for HttpRequestTool {
}
}
// Helper functions similar to browser_open.rs
fn normalize_allowed_domains(domains: Vec<String>) -> Vec<String> {
let mut normalized = domains
.into_iter()
.filter_map(|d| normalize_domain(&d))
.collect::<Vec<_>>();
normalized.sort_unstable();
normalized.dedup();
normalized
}
fn normalize_domain(raw: &str) -> Option<String> {
let mut d = raw.trim().to_lowercase();
if d.is_empty() {
return None;
}
if let Some(stripped) = d.strip_prefix("https://") {
d = stripped.to_string();
} else if let Some(stripped) = d.strip_prefix("http://") {
d = stripped.to_string();
}
if let Some((host, _)) = d.split_once('/') {
d = host.to_string();
}
d = d.trim_start_matches('.').trim_end_matches('.').to_string();
if let Some((host, _)) = d.split_once(':') {
d = host.to_string();
}
if d.is_empty() || d.chars().any(char::is_whitespace) {
return None;
}
Some(d)
}
fn extract_host(url: &str) -> anyhow::Result<String> {
let rest = url
.strip_prefix("http://")
.or_else(|| url.strip_prefix("https://"))
.ok_or_else(|| anyhow::anyhow!("Only http:// and https:// URLs are allowed"))?;
let authority = rest
.split(['/', '?', '#'])
.next()
.ok_or_else(|| anyhow::anyhow!("Invalid URL"))?;
if authority.is_empty() {
anyhow::bail!("URL must include a host");
}
if authority.contains('@') {
anyhow::bail!("URL userinfo is not allowed");
}
if authority.starts_with('[') {
anyhow::bail!("IPv6 hosts are not supported in http_request");
}
let host = authority
.split(':')
.next()
.unwrap_or_default()
.trim()
.trim_end_matches('.')
.to_lowercase();
if host.is_empty() {
anyhow::bail!("URL must include a valid host");
}
Ok(host)
}
fn host_matches_allowlist(host: &str, allowed_domains: &[String]) -> bool {
allowed_domains.iter().any(|domain| {
host == domain
|| host
.strip_suffix(domain)
.is_some_and(|prefix| prefix.ends_with('.'))
})
}
fn is_private_or_local_host(host: &str) -> bool {
// Strip brackets from IPv6 addresses like [::1]
let bare = host
.strip_prefix('[')
.and_then(|h| h.strip_suffix(']'))
.unwrap_or(host);
let has_local_tld = bare
.rsplit('.')
.next()
.is_some_and(|label| label == "local");
if bare == "localhost" || bare.ends_with(".localhost") || has_local_tld {
return true;
}
if let Ok(ip) = bare.parse::<std::net::IpAddr>() {
return match ip {
std::net::IpAddr::V4(v4) => is_non_global_v4(v4),
std::net::IpAddr::V6(v6) => is_non_global_v6(v6),
};
}
false
}
/// Returns true if the IPv4 address is not globally routable.
fn is_non_global_v4(v4: std::net::Ipv4Addr) -> bool {
let [a, b, c, _] = v4.octets();
v4.is_loopback() // 127.0.0.0/8
|| v4.is_private() // 10/8, 172.16/12, 192.168/16
|| v4.is_link_local() // 169.254.0.0/16
|| v4.is_unspecified() // 0.0.0.0
|| v4.is_broadcast() // 255.255.255.255
|| v4.is_multicast() // 224.0.0.0/4
|| (a == 100 && (64..=127).contains(&b)) // Shared address space (RFC 6598)
|| a >= 240 // Reserved (240.0.0.0/4, except broadcast)
|| (a == 192 && b == 0 && (c == 0 || c == 2)) // IETF assignments + TEST-NET-1
|| (a == 198 && b == 51) // Documentation (198.51.100.0/24)
|| (a == 203 && b == 0) // Documentation (203.0.113.0/24)
|| (a == 198 && (18..=19).contains(&b)) // Benchmarking (198.18.0.0/15)
}
/// Returns true if the IPv6 address is not globally routable.
fn is_non_global_v6(v6: std::net::Ipv6Addr) -> bool {
let segs = v6.segments();
v6.is_loopback() // ::1
|| v6.is_unspecified() // ::
|| v6.is_multicast() // ff00::/8
|| (segs[0] & 0xfe00) == 0xfc00 // Unique-local (fc00::/7)
|| (segs[0] & 0xffc0) == 0xfe80 // Link-local (fe80::/10)
|| (segs[0] == 0x2001 && segs[1] == 0x0db8) // Documentation (2001:db8::/32)
|| v6.to_ipv4_mapped().is_some_and(is_non_global_v4)
}
#[cfg(test)]
mod tests {
use super::*;
@@ -425,102 +251,6 @@ mod tests {
)
}
#[test]
fn normalize_domain_strips_scheme_path_and_case() {
let got = normalize_domain(" HTTPS://Docs.Example.com/path ").unwrap();
assert_eq!(got, "docs.example.com");
}
#[test]
fn normalize_allowed_domains_deduplicates() {
let got = normalize_allowed_domains(vec![
"example.com".into(),
"EXAMPLE.COM".into(),
"https://example.com/".into(),
]);
assert_eq!(got, vec!["example.com".to_string()]);
}
#[test]
fn validate_accepts_exact_domain() {
let tool = test_tool(vec!["example.com"]);
let got = tool.validate_url("https://example.com/docs").unwrap();
assert_eq!(got, "https://example.com/docs");
}
#[test]
fn validate_accepts_http() {
let tool = test_tool(vec!["example.com"]);
assert!(tool.validate_url("http://example.com").is_ok());
}
#[test]
fn validate_accepts_subdomain() {
let tool = test_tool(vec!["example.com"]);
assert!(tool.validate_url("https://api.example.com/v1").is_ok());
}
#[test]
fn validate_rejects_allowlist_miss() {
let tool = test_tool(vec!["example.com"]);
let err = tool
.validate_url("https://google.com")
.unwrap_err()
.to_string();
assert!(err.contains("allowed_domains"));
}
#[test]
fn validate_rejects_localhost() {
let tool = test_tool(vec!["localhost"]);
let err = tool
.validate_url("https://localhost:8080")
.unwrap_err()
.to_string();
assert!(err.contains("local/private"));
}
#[test]
fn validate_rejects_private_ipv4() {
let tool = test_tool(vec!["192.168.1.5"]);
let err = tool
.validate_url("https://192.168.1.5")
.unwrap_err()
.to_string();
assert!(err.contains("local/private"));
}
#[test]
fn validate_rejects_whitespace() {
let tool = test_tool(vec!["example.com"]);
let err = tool
.validate_url("https://example.com/hello world")
.unwrap_err()
.to_string();
assert!(err.contains("whitespace"));
}
#[test]
fn validate_rejects_userinfo() {
let tool = test_tool(vec!["example.com"]);
let err = tool
.validate_url("https://user@example.com")
.unwrap_err()
.to_string();
assert!(err.contains("userinfo"));
}
#[test]
fn validate_requires_allowlist() {
let security = Arc::new(SecurityPolicy::default());
let tool = HttpRequestTool::new(security, vec![], 1_000_000, 30);
let err = tool
.validate_url("https://example.com")
.unwrap_err()
.to_string();
assert!(err.contains("allowed_domains"));
}
#[test]
fn validate_accepts_valid_methods() {
let tool = test_tool(vec!["example.com"]);
@@ -540,89 +270,6 @@ mod tests {
assert!(err.contains("Unsupported HTTP method"));
}
#[test]
fn blocks_multicast_ipv4() {
assert!(is_private_or_local_host("224.0.0.1"));
assert!(is_private_or_local_host("239.255.255.255"));
}
#[test]
fn blocks_broadcast() {
assert!(is_private_or_local_host("255.255.255.255"));
}
#[test]
fn blocks_reserved_ipv4() {
assert!(is_private_or_local_host("240.0.0.1"));
assert!(is_private_or_local_host("250.1.2.3"));
}
#[test]
fn blocks_documentation_ranges() {
assert!(is_private_or_local_host("192.0.2.1")); // TEST-NET-1
assert!(is_private_or_local_host("198.51.100.1")); // TEST-NET-2
assert!(is_private_or_local_host("203.0.113.1")); // TEST-NET-3
}
#[test]
fn blocks_benchmarking_range() {
assert!(is_private_or_local_host("198.18.0.1"));
assert!(is_private_or_local_host("198.19.255.255"));
}
#[test]
fn blocks_ipv6_localhost() {
assert!(is_private_or_local_host("::1"));
assert!(is_private_or_local_host("[::1]"));
}
#[test]
fn blocks_ipv6_multicast() {
assert!(is_private_or_local_host("ff02::1"));
}
#[test]
fn blocks_ipv6_link_local() {
assert!(is_private_or_local_host("fe80::1"));
}
#[test]
fn blocks_ipv6_unique_local() {
assert!(is_private_or_local_host("fd00::1"));
}
#[test]
fn blocks_ipv4_mapped_ipv6() {
assert!(is_private_or_local_host("::ffff:127.0.0.1"));
assert!(is_private_or_local_host("::ffff:192.168.1.1"));
assert!(is_private_or_local_host("::ffff:10.0.0.1"));
}
#[test]
fn allows_public_ipv4() {
assert!(!is_private_or_local_host("8.8.8.8"));
assert!(!is_private_or_local_host("1.1.1.1"));
assert!(!is_private_or_local_host("93.184.216.34"));
}
#[test]
fn blocks_ipv6_documentation_range() {
assert!(is_private_or_local_host("2001:db8::1"));
}
#[test]
fn allows_public_ipv6() {
assert!(!is_private_or_local_host("2607:f8b0:4004:800::200e"));
}
#[test]
fn blocks_shared_address_space() {
assert!(is_private_or_local_host("100.64.0.1"));
assert!(is_private_or_local_host("100.127.255.255"));
assert!(!is_private_or_local_host("100.63.0.1")); // Just below range
assert!(!is_private_or_local_host("100.128.0.1")); // Just above range
}
#[tokio::test]
async fn execute_blocks_readonly_mode() {
let security = Arc::new(SecurityPolicy {
@@ -670,7 +317,7 @@ mod tests {
);
let text = "hello world this is long";
let truncated = tool.truncate_response(text);
assert!(truncated.len() <= 10 + 60); // limit + message
assert!(truncated.len() <= 10 + 60);
assert!(truncated.contains("[Response truncated"));
}
@@ -726,130 +373,9 @@ mod tests {
assert_eq!(headers[0].1, "Bearer real-token");
}
// ── SSRF: alternate IP notation bypass defense-in-depth ─────────
//
// Rust's IpAddr::parse() rejects non-standard notations (octal, hex,
// decimal integer, zero-padded). These tests document that property
// so regressions are caught if the parsing strategy ever changes.
#[test]
fn ssrf_octal_loopback_not_parsed_as_ip() {
// 0177.0.0.1 is octal for 127.0.0.1 in some languages, but
// Rust's IpAddr rejects it — it falls through as a hostname.
assert!(!is_private_or_local_host("0177.0.0.1"));
}
#[test]
fn ssrf_hex_loopback_not_parsed_as_ip() {
// 0x7f000001 is hex for 127.0.0.1 in some languages.
assert!(!is_private_or_local_host("0x7f000001"));
}
#[test]
fn ssrf_decimal_loopback_not_parsed_as_ip() {
// 2130706433 is decimal for 127.0.0.1 in some languages.
assert!(!is_private_or_local_host("2130706433"));
}
#[test]
fn ssrf_zero_padded_loopback_not_parsed_as_ip() {
// 127.000.000.001 uses zero-padded octets.
assert!(!is_private_or_local_host("127.000.000.001"));
}
#[test]
fn ssrf_alternate_notations_rejected_by_validate_url() {
// Even if is_private_or_local_host doesn't flag these, they
// fail the allowlist because they're treated as hostnames.
let tool = test_tool(vec!["example.com"]);
for notation in [
"http://0177.0.0.1",
"http://0x7f000001",
"http://2130706433",
"http://127.000.000.001",
] {
let err = tool.validate_url(notation).unwrap_err().to_string();
assert!(
err.contains("allowed_domains"),
"Expected allowlist rejection for {notation}, got: {err}"
);
}
}
#[test]
fn redirect_policy_is_none() {
// Structural test: the tool should be buildable with redirect-safe config.
// The actual Policy::none() enforcement is in execute_request's client builder.
let tool = test_tool(vec!["example.com"]);
assert_eq!(tool.name(), "http_request");
}
// ── §1.4 DNS rebinding / SSRF defense-in-depth tests ─────
#[test]
fn ssrf_blocks_loopback_127_range() {
assert!(is_private_or_local_host("127.0.0.1"));
assert!(is_private_or_local_host("127.0.0.2"));
assert!(is_private_or_local_host("127.255.255.255"));
}
#[test]
fn ssrf_blocks_rfc1918_10_range() {
assert!(is_private_or_local_host("10.0.0.1"));
assert!(is_private_or_local_host("10.255.255.255"));
}
#[test]
fn ssrf_blocks_rfc1918_172_range() {
assert!(is_private_or_local_host("172.16.0.1"));
assert!(is_private_or_local_host("172.31.255.255"));
}
#[test]
fn ssrf_blocks_unspecified_address() {
assert!(is_private_or_local_host("0.0.0.0"));
}
#[test]
fn ssrf_blocks_dot_localhost_subdomain() {
assert!(is_private_or_local_host("evil.localhost"));
assert!(is_private_or_local_host("a.b.localhost"));
}
#[test]
fn ssrf_blocks_dot_local_tld() {
assert!(is_private_or_local_host("service.local"));
}
#[test]
fn ssrf_ipv6_unspecified() {
assert!(is_private_or_local_host("::"));
}
#[test]
fn validate_rejects_ftp_scheme() {
let tool = test_tool(vec!["example.com"]);
let err = tool
.validate_url("ftp://example.com")
.unwrap_err()
.to_string();
assert!(err.contains("http://") || err.contains("https://"));
}
#[test]
fn validate_rejects_empty_url() {
let tool = test_tool(vec!["example.com"]);
let err = tool.validate_url("").unwrap_err().to_string();
assert!(err.contains("empty"));
}
#[test]
fn validate_rejects_ipv6_host() {
let tool = test_tool(vec!["example.com"]);
let err = tool
.validate_url("http://[::1]:8080/path")
.unwrap_err()
.to_string();
assert!(err.contains("IPv6"));
}
}
+5
View File
@@ -1,7 +1,12 @@
mod composio;
mod curl;
mod gitbooks;
mod http_request;
mod url_guard;
mod web_search;
pub use composio::{ComposioAction, ComposioTool};
pub use curl::CurlTool;
pub use gitbooks::{GitbooksGetPageTool, GitbooksSearchTool};
pub use http_request::HttpRequestTool;
pub use web_search::WebSearchTool;
@@ -0,0 +1,475 @@
//! Shared URL validation + SSRF guards for outbound network tools.
//!
//! Used by `http_request`, `curl`, and any future tool that takes a
//! user-supplied URL. The contract is intentionally strict:
//!
//! - http(s) only
//! - non-empty allowlist required (callers pass it in)
//! - no whitespace, no userinfo, no IPv6 hosts
//! - blocks loopback / RFC1918 / link-local / multicast / documentation /
//! shared-address / IPv4-mapped IPv6, including `localhost` /
//! `*.localhost` / `*.local`
//!
//! The blocklist deliberately does NOT cover alternate IP notations
//! (octal, hex, decimal) because Rust's `IpAddr::parse` rejects them —
//! they fall through and get rejected by the allowlist instead. See the
//! tests in `http_request.rs` for the documented behaviour.
/// Validate a URL against the allowlist + SSRF rules. Returns the
/// original URL on success.
pub(super) fn validate_url(raw_url: &str, allowed_domains: &[String]) -> anyhow::Result<String> {
let url = raw_url.trim();
if url.is_empty() {
anyhow::bail!("URL cannot be empty");
}
if url.chars().any(char::is_whitespace) {
anyhow::bail!("URL cannot contain whitespace");
}
if !url.starts_with("http://") && !url.starts_with("https://") {
anyhow::bail!("Only http:// and https:// URLs are allowed");
}
if allowed_domains.is_empty() {
anyhow::bail!(
"Network tool is enabled but no allowed_domains are configured. Add [http_request].allowed_domains in config.toml"
);
}
let host = extract_host(url)?;
if is_private_or_local_host(&host) {
anyhow::bail!("Blocked local/private host: {host}");
}
if !host_matches_allowlist(&host, allowed_domains) {
anyhow::bail!("Host '{host}' is not in http_request.allowed_domains");
}
Ok(url.to_string())
}
pub(super) fn normalize_allowed_domains(domains: Vec<String>) -> Vec<String> {
let mut normalized = domains
.into_iter()
.filter_map(|d| normalize_domain(&d))
.collect::<Vec<_>>();
normalized.sort_unstable();
normalized.dedup();
normalized
}
pub(super) fn normalize_domain(raw: &str) -> Option<String> {
let mut d = raw.trim().to_lowercase();
if d.is_empty() {
return None;
}
if let Some(stripped) = d.strip_prefix("https://") {
d = stripped.to_string();
} else if let Some(stripped) = d.strip_prefix("http://") {
d = stripped.to_string();
}
if let Some((host, _)) = d.split_once('/') {
d = host.to_string();
}
d = d.trim_start_matches('.').trim_end_matches('.').to_string();
if let Some((host, _)) = d.split_once(':') {
d = host.to_string();
}
if d.is_empty() || d.chars().any(char::is_whitespace) {
return None;
}
Some(d)
}
pub(super) fn extract_host(url: &str) -> anyhow::Result<String> {
let rest = url
.strip_prefix("http://")
.or_else(|| url.strip_prefix("https://"))
.ok_or_else(|| anyhow::anyhow!("Only http:// and https:// URLs are allowed"))?;
let authority = rest
.split(['/', '?', '#'])
.next()
.ok_or_else(|| anyhow::anyhow!("Invalid URL"))?;
if authority.is_empty() {
anyhow::bail!("URL must include a host");
}
if authority.contains('@') {
anyhow::bail!("URL userinfo is not allowed");
}
if authority.starts_with('[') {
anyhow::bail!("IPv6 hosts are not supported in http_request");
}
let host = authority
.split(':')
.next()
.unwrap_or_default()
.trim()
.trim_end_matches('.')
.to_lowercase();
if host.is_empty() {
anyhow::bail!("URL must include a valid host");
}
Ok(host)
}
pub(super) fn host_matches_allowlist(host: &str, allowed_domains: &[String]) -> bool {
allowed_domains.iter().any(|domain| {
host == domain
|| host
.strip_suffix(domain)
.is_some_and(|prefix| prefix.ends_with('.'))
})
}
pub(super) fn is_private_or_local_host(host: &str) -> bool {
let bare = host
.strip_prefix('[')
.and_then(|h| h.strip_suffix(']'))
.unwrap_or(host);
let has_local_tld = bare
.rsplit('.')
.next()
.is_some_and(|label| label == "local");
if bare == "localhost" || bare.ends_with(".localhost") || has_local_tld {
return true;
}
if let Ok(ip) = bare.parse::<std::net::IpAddr>() {
return match ip {
std::net::IpAddr::V4(v4) => is_non_global_v4(v4),
std::net::IpAddr::V6(v6) => is_non_global_v6(v6),
};
}
false
}
fn is_non_global_v4(v4: std::net::Ipv4Addr) -> bool {
let [a, b, c, _] = v4.octets();
v4.is_loopback()
|| v4.is_private()
|| v4.is_link_local()
|| v4.is_unspecified()
|| v4.is_broadcast()
|| v4.is_multicast()
|| (a == 100 && (64..=127).contains(&b))
|| a >= 240
|| (a == 192 && b == 0 && (c == 0 || c == 2))
|| (a == 198 && b == 51)
|| (a == 203 && b == 0)
|| (a == 198 && (18..=19).contains(&b))
}
fn is_non_global_v6(v6: std::net::Ipv6Addr) -> bool {
let segs = v6.segments();
v6.is_loopback()
|| v6.is_unspecified()
|| v6.is_multicast()
|| (segs[0] & 0xfe00) == 0xfc00
|| (segs[0] & 0xffc0) == 0xfe80
|| (segs[0] == 0x2001 && segs[1] == 0x0db8)
|| v6.to_ipv4_mapped().is_some_and(is_non_global_v4)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn normalize_domain_strips_scheme_path_and_case() {
let got = normalize_domain(" HTTPS://Docs.Example.com/path ").unwrap();
assert_eq!(got, "docs.example.com");
}
#[test]
fn normalize_allowed_domains_deduplicates() {
let got = normalize_allowed_domains(vec![
"example.com".into(),
"EXAMPLE.COM".into(),
"https://example.com/".into(),
]);
assert_eq!(got, vec!["example.com".to_string()]);
}
#[test]
fn validate_accepts_exact_domain() {
let allow = vec!["example.com".to_string()];
let got = validate_url("https://example.com/docs", &allow).unwrap();
assert_eq!(got, "https://example.com/docs");
}
#[test]
fn validate_accepts_http() {
let allow = vec!["example.com".to_string()];
assert!(validate_url("http://example.com", &allow).is_ok());
}
#[test]
fn validate_accepts_subdomain() {
let allow = vec!["example.com".to_string()];
assert!(validate_url("https://api.example.com/v1", &allow).is_ok());
}
#[test]
fn validate_rejects_allowlist_miss() {
let allow = vec!["example.com".to_string()];
let err = validate_url("https://google.com", &allow)
.unwrap_err()
.to_string();
assert!(err.contains("allowed_domains"));
}
#[test]
fn validate_rejects_localhost() {
let allow = vec!["localhost".to_string()];
let err = validate_url("https://localhost:8080", &allow)
.unwrap_err()
.to_string();
assert!(err.contains("local/private"));
}
#[test]
fn validate_rejects_private_ipv4() {
let allow = vec!["192.168.1.5".to_string()];
let err = validate_url("https://192.168.1.5", &allow)
.unwrap_err()
.to_string();
assert!(err.contains("local/private"));
}
#[test]
fn validate_rejects_whitespace() {
let allow = vec!["example.com".to_string()];
let err = validate_url("https://example.com/hello world", &allow)
.unwrap_err()
.to_string();
assert!(err.contains("whitespace"));
}
#[test]
fn validate_rejects_userinfo() {
let allow = vec!["example.com".to_string()];
let err = validate_url("https://user@example.com", &allow)
.unwrap_err()
.to_string();
assert!(err.contains("userinfo"));
}
#[test]
fn validate_requires_allowlist() {
let err = validate_url("https://example.com", &[])
.unwrap_err()
.to_string();
assert!(err.contains("allowed_domains"));
}
#[test]
fn validate_rejects_ftp_scheme() {
let allow = vec!["example.com".to_string()];
let err = validate_url("ftp://example.com", &allow)
.unwrap_err()
.to_string();
assert!(err.contains("http://") || err.contains("https://"));
}
#[test]
fn validate_rejects_empty_url() {
let allow = vec!["example.com".to_string()];
let err = validate_url("", &allow).unwrap_err().to_string();
assert!(err.contains("empty"));
}
#[test]
fn validate_rejects_ipv6_host() {
let allow = vec!["example.com".to_string()];
let err = validate_url("http://[::1]:8080/path", &allow)
.unwrap_err()
.to_string();
assert!(err.contains("IPv6"));
}
#[test]
fn blocks_multicast_ipv4() {
assert!(is_private_or_local_host("224.0.0.1"));
assert!(is_private_or_local_host("239.255.255.255"));
}
#[test]
fn blocks_broadcast() {
assert!(is_private_or_local_host("255.255.255.255"));
}
#[test]
fn blocks_reserved_ipv4() {
assert!(is_private_or_local_host("240.0.0.1"));
assert!(is_private_or_local_host("250.1.2.3"));
}
#[test]
fn blocks_documentation_ranges() {
assert!(is_private_or_local_host("192.0.2.1"));
assert!(is_private_or_local_host("198.51.100.1"));
assert!(is_private_or_local_host("203.0.113.1"));
}
#[test]
fn blocks_benchmarking_range() {
assert!(is_private_or_local_host("198.18.0.1"));
assert!(is_private_or_local_host("198.19.255.255"));
}
#[test]
fn blocks_ipv6_localhost() {
assert!(is_private_or_local_host("::1"));
assert!(is_private_or_local_host("[::1]"));
}
#[test]
fn blocks_ipv6_multicast() {
assert!(is_private_or_local_host("ff02::1"));
}
#[test]
fn blocks_ipv6_link_local() {
assert!(is_private_or_local_host("fe80::1"));
}
#[test]
fn blocks_ipv6_unique_local() {
assert!(is_private_or_local_host("fd00::1"));
}
#[test]
fn blocks_ipv4_mapped_ipv6() {
assert!(is_private_or_local_host("::ffff:127.0.0.1"));
assert!(is_private_or_local_host("::ffff:192.168.1.1"));
assert!(is_private_or_local_host("::ffff:10.0.0.1"));
}
#[test]
fn allows_public_ipv4() {
assert!(!is_private_or_local_host("8.8.8.8"));
assert!(!is_private_or_local_host("1.1.1.1"));
assert!(!is_private_or_local_host("93.184.216.34"));
}
#[test]
fn blocks_ipv6_documentation_range() {
assert!(is_private_or_local_host("2001:db8::1"));
}
#[test]
fn allows_public_ipv6() {
assert!(!is_private_or_local_host("2607:f8b0:4004:800::200e"));
}
#[test]
fn blocks_shared_address_space() {
assert!(is_private_or_local_host("100.64.0.1"));
assert!(is_private_or_local_host("100.127.255.255"));
assert!(!is_private_or_local_host("100.63.0.1"));
assert!(!is_private_or_local_host("100.128.0.1"));
}
#[test]
fn ssrf_blocks_loopback_127_range() {
assert!(is_private_or_local_host("127.0.0.1"));
assert!(is_private_or_local_host("127.0.0.2"));
assert!(is_private_or_local_host("127.255.255.255"));
}
#[test]
fn ssrf_blocks_rfc1918_10_range() {
assert!(is_private_or_local_host("10.0.0.1"));
assert!(is_private_or_local_host("10.255.255.255"));
}
#[test]
fn ssrf_blocks_rfc1918_172_range() {
assert!(is_private_or_local_host("172.16.0.1"));
assert!(is_private_or_local_host("172.31.255.255"));
}
#[test]
fn ssrf_blocks_unspecified_address() {
assert!(is_private_or_local_host("0.0.0.0"));
}
#[test]
fn ssrf_blocks_dot_localhost_subdomain() {
assert!(is_private_or_local_host("evil.localhost"));
assert!(is_private_or_local_host("a.b.localhost"));
}
#[test]
fn ssrf_blocks_dot_local_tld() {
assert!(is_private_or_local_host("service.local"));
}
#[test]
fn ssrf_ipv6_unspecified() {
assert!(is_private_or_local_host("::"));
}
// ── Defense-in-depth: alternate IP notations rejected by allowlist
//
// Rust's IpAddr::parse() rejects octal, hex, decimal, and
// zero-padded notations. They fall through as hostnames and get
// rejected by the allowlist instead. These tests pin that
// behaviour so a parser change can't silently re-open SSRF.
#[test]
fn ssrf_octal_loopback_not_parsed_as_ip() {
assert!(!is_private_or_local_host("0177.0.0.1"));
}
#[test]
fn ssrf_hex_loopback_not_parsed_as_ip() {
assert!(!is_private_or_local_host("0x7f000001"));
}
#[test]
fn ssrf_decimal_loopback_not_parsed_as_ip() {
assert!(!is_private_or_local_host("2130706433"));
}
#[test]
fn ssrf_zero_padded_loopback_not_parsed_as_ip() {
assert!(!is_private_or_local_host("127.000.000.001"));
}
#[test]
fn ssrf_alternate_notations_rejected_by_validate_url() {
let allow = vec!["example.com".to_string()];
for notation in [
"http://0177.0.0.1",
"http://0x7f000001",
"http://2130706433",
"http://127.000.000.001",
] {
let err = validate_url(notation, &allow).unwrap_err().to_string();
assert!(
err.contains("allowed_domains"),
"Expected allowlist rejection for {notation}, got: {err}"
);
}
}
}
+134
View File
@@ -169,6 +169,32 @@ pub fn all_tools_with_runtime(
http_config.timeout_secs,
)));
// curl — always registered. Shares `http_request.allowed_domains`,
// adds streaming-to-disk with a hard byte ceiling. Writes land
// under `<workspace>/<curl.dest_subdir>`.
tools.push(Box::new(CurlTool::new(
security.clone(),
http_config.allowed_domains.clone(),
workspace_dir.to_path_buf(),
root_config.curl.dest_subdir.clone(),
root_config.curl.max_download_bytes,
root_config.curl.timeout_secs,
)));
// gitbooks — answers questions about OpenHuman by calling the
// GitBook MCP server. Two tools mirroring the upstream MCP tools.
if root_config.gitbooks.enabled {
tools.push(Box::new(GitbooksSearchTool::new(
root_config.gitbooks.endpoint.clone(),
root_config.gitbooks.timeout_secs,
)));
tools.push(Box::new(GitbooksGetPageTool::new(
root_config.gitbooks.endpoint.clone(),
root_config.gitbooks.timeout_secs,
)));
tracing::debug!("[gitbooks] registered gitbooks_search + gitbooks_get_page");
}
// Web search — always registered. Result/timeout budget
// knobs still come from `config.web_search`, but there is no
// enable flag: every session needs research as a baseline
@@ -372,6 +398,114 @@ mod tests {
);
}
#[test]
fn all_tools_always_registers_curl() {
// Regression guard: `curl` is always registered (gated only by
// the shared `http_request.allowed_domains` allowlist at call
// time, like `http_request`). `Write` permission level keeps it
// off agents that aren't allowed to modify the workspace.
let tmp = TempDir::new().unwrap();
let security = Arc::new(SecurityPolicy::default());
let mem_cfg = MemoryConfig {
backend: "markdown".into(),
..MemoryConfig::default()
};
let mem: Arc<dyn Memory> =
Arc::from(crate::openhuman::memory::create_memory(&mem_cfg, tmp.path()).unwrap());
let browser = BrowserConfig::default();
let http = crate::openhuman::config::HttpRequestConfig::default();
let cfg = test_config(&tmp);
let tools = all_tools(
Arc::new(cfg.clone()),
&security,
mem,
&browser,
&http,
tmp.path(),
&HashMap::new(),
&cfg,
);
let names: Vec<&str> = tools.iter().map(|t| t.name()).collect();
assert!(
names.contains(&"curl"),
"curl must always be registered; got: {names:?}"
);
}
#[test]
fn all_tools_registers_gitbooks_when_enabled() {
let tmp = TempDir::new().unwrap();
let security = Arc::new(SecurityPolicy::default());
let mem_cfg = MemoryConfig {
backend: "markdown".into(),
..MemoryConfig::default()
};
let mem: Arc<dyn Memory> =
Arc::from(crate::openhuman::memory::create_memory(&mem_cfg, tmp.path()).unwrap());
let browser = BrowserConfig::default();
let http = crate::openhuman::config::HttpRequestConfig::default();
let mut cfg = test_config(&tmp);
cfg.gitbooks.enabled = true;
let tools = all_tools(
Arc::new(cfg.clone()),
&security,
mem,
&browser,
&http,
tmp.path(),
&HashMap::new(),
&cfg,
);
let names: Vec<&str> = tools.iter().map(|t| t.name()).collect();
assert!(
names.contains(&"gitbooks_search"),
"gitbooks_search must register when gitbooks.enabled = true; got: {names:?}"
);
assert!(
names.contains(&"gitbooks_get_page"),
"gitbooks_get_page must register when gitbooks.enabled = true; got: {names:?}"
);
}
#[test]
fn all_tools_skips_gitbooks_when_disabled() {
let tmp = TempDir::new().unwrap();
let security = Arc::new(SecurityPolicy::default());
let mem_cfg = MemoryConfig {
backend: "markdown".into(),
..MemoryConfig::default()
};
let mem: Arc<dyn Memory> =
Arc::from(crate::openhuman::memory::create_memory(&mem_cfg, tmp.path()).unwrap());
let browser = BrowserConfig::default();
let http = crate::openhuman::config::HttpRequestConfig::default();
let mut cfg = test_config(&tmp);
cfg.gitbooks.enabled = false;
let tools = all_tools(
Arc::new(cfg.clone()),
&security,
mem,
&browser,
&http,
tmp.path(),
&HashMap::new(),
&cfg,
);
let names: Vec<&str> = tools.iter().map(|t| t.name()).collect();
assert!(
!names.contains(&"gitbooks_search"),
"gitbooks_search must NOT register when gitbooks.enabled = false; got: {names:?}"
);
assert!(
!names.contains(&"gitbooks_get_page"),
"gitbooks_get_page must NOT register when gitbooks.enabled = false; got: {names:?}"
);
}
#[test]
fn all_tools_includes_complete_onboarding() {
// Regression guard: the `complete_onboarding` tool must be