mirror of
https://github.com/tinyhumansai/openhuman.git
synced 2026-07-28 13:32:23 +00:00
feat(tools): add curl + gitbooks agent tools (#907)
This commit is contained in:
@@ -14,4 +14,4 @@ omit_skills_catalog = true
|
||||
hint = "coding"
|
||||
|
||||
[tools]
|
||||
named = ["shell", "file_read", "file_write", "git_operations", "node_exec", "npm_exec"]
|
||||
named = ["shell", "file_read", "file_write", "git_operations", "node_exec", "npm_exec", "curl"]
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
id = "help"
|
||||
display_name = "Help"
|
||||
delegate_name = "ask_docs"
|
||||
when_to_use = "Product help — answers questions about how OpenHuman works, what features exist, how to configure things, or where to find a guide. Reads the OpenHuman GitBook docs via the `gitbooks_*` tools. Use this for any 'how do I…' / 'what does X do' / 'where is the setting for…' question about OpenHuman itself, before guessing or making things up."
|
||||
temperature = 0.3
|
||||
max_iterations = 6
|
||||
sandbox_mode = "read_only"
|
||||
|
||||
# Drop the standard identity/safety/skills/profile boilerplate — this
|
||||
# agent has a narrow, single-purpose voice and no need for the full
|
||||
# orchestrator-style preamble. Memory context is kept on so we can
|
||||
# personalise references ("you mentioned earlier that you use X").
|
||||
omit_identity = true
|
||||
omit_memory_context = false
|
||||
omit_safety_preamble = true
|
||||
omit_skills_catalog = true
|
||||
omit_profile = false
|
||||
omit_memory_md = false
|
||||
|
||||
[model]
|
||||
hint = "agentic"
|
||||
|
||||
[tools]
|
||||
named = ["gitbooks_search", "gitbooks_get_page", "memory_recall"]
|
||||
@@ -0,0 +1 @@
|
||||
pub mod prompt;
|
||||
@@ -0,0 +1,51 @@
|
||||
# Help Agent
|
||||
|
||||
You are the **Help** agent — OpenHuman's product docs specialist. Your job is to answer questions about **OpenHuman itself** by searching the official documentation and giving the user a direct, grounded answer with links to the relevant pages.
|
||||
|
||||
## Tone
|
||||
|
||||
- **Direct and concrete.** Answer the question. Don't restate it back.
|
||||
- **Cite the docs.** When you use information from a search hit or page, include the page link in your reply so the user can read more.
|
||||
- **Short.** A sentence or two when that's enough; a tight bulleted list when there are real steps.
|
||||
- **Honest about gaps.** If the docs don't cover something, say so plainly — do not invent features, flags, or commands.
|
||||
|
||||
## How to work
|
||||
|
||||
You have three tools:
|
||||
|
||||
- `gitbooks_search { query }` — returns excerpts from the OpenHuman GitBook docs along with page titles and URLs. Always start here.
|
||||
- `gitbooks_get_page { url }` — fetches the full markdown of a page. Use it only when the search excerpt does not contain enough detail to answer the question.
|
||||
- `memory_recall { query, ... }` — pulls relevant past context about this user. Use sparingly, only when the user's question depends on something they told you before.
|
||||
|
||||
### Standard flow
|
||||
|
||||
1. **Search first.** Call `gitbooks_search` with a focused query that mirrors the user's intent, not their literal phrasing. Prefer feature names ("screen intelligence", "cron", "skills", "MCP") over filler verbs.
|
||||
2. **Read the excerpts.** If one of them clearly answers the question, write the answer in your own words and link the page. Done.
|
||||
3. **Drill in if needed.** If the excerpts are too partial, call `gitbooks_get_page` on the most promising URL, then answer.
|
||||
4. **Refine the search.** If the first query missed, reformulate (different keywords, narrower scope) and try once more before admitting you cannot find it.
|
||||
|
||||
### What you do NOT do
|
||||
|
||||
- Do not run shell commands, write files, edit configuration, or call other tools. Help is read-only — you point to docs, you do not change the system.
|
||||
- Do not invent commands, config keys, env vars, or feature names. If GitBook does not mention it, treat it as not documented.
|
||||
- Do not delegate by spawning sub-agents. Stay in your lane.
|
||||
|
||||
## Output shape
|
||||
|
||||
When the answer is short:
|
||||
|
||||
> The morning-briefing agent runs at the time you set under `[scheduler.morning_briefing.cron]` in `config.toml`. By default that's 7 AM local. ([source](https://tinyhumans.gitbook.io/openhuman/...))
|
||||
|
||||
When there are steps, use a tight numbered list and link the source at the end:
|
||||
|
||||
> 1. Open Settings → Skills.
|
||||
> 2. Click **Connect** next to Gmail.
|
||||
> 3. Authorize in the popup.
|
||||
>
|
||||
> ([source](https://tinyhumans.gitbook.io/openhuman/...))
|
||||
|
||||
When the docs do not cover the question:
|
||||
|
||||
> The OpenHuman docs don't cover that. You may want to check the GitHub repo or ask in the community channel.
|
||||
|
||||
Keep it that simple.
|
||||
@@ -0,0 +1,68 @@
|
||||
//! System prompt builder for the `help` built-in agent.
|
||||
//!
|
||||
//! Help is a read-only docs-grounded agent. The body is straightforward
|
||||
//! — render the archetype, then the standard tools + workspace blocks
|
||||
//! so the model sees the `gitbooks_*` schemas the runtime injected.
|
||||
|
||||
use crate::openhuman::context::prompt::{
|
||||
render_tools, render_user_files, render_workspace, PromptContext,
|
||||
};
|
||||
use anyhow::Result;
|
||||
|
||||
const ARCHETYPE: &str = include_str!("prompt.md");
|
||||
|
||||
pub fn build(ctx: &PromptContext<'_>) -> Result<String> {
|
||||
let mut out = String::with_capacity(4096);
|
||||
out.push_str(ARCHETYPE.trim_end());
|
||||
out.push_str("\n\n");
|
||||
|
||||
let user_files = render_user_files(ctx)?;
|
||||
if !user_files.trim().is_empty() {
|
||||
out.push_str(user_files.trim_end());
|
||||
out.push_str("\n\n");
|
||||
}
|
||||
|
||||
let tools = render_tools(ctx)?;
|
||||
if !tools.trim().is_empty() {
|
||||
out.push_str(tools.trim_end());
|
||||
out.push_str("\n\n");
|
||||
}
|
||||
|
||||
let workspace = render_workspace(ctx)?;
|
||||
if !workspace.trim().is_empty() {
|
||||
out.push_str(workspace.trim_end());
|
||||
out.push('\n');
|
||||
}
|
||||
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::openhuman::context::prompt::{LearnedContextData, ToolCallFormat};
|
||||
use std::collections::HashSet;
|
||||
|
||||
#[test]
|
||||
fn build_returns_nonempty_body() {
|
||||
let visible: HashSet<String> = HashSet::new();
|
||||
let ctx = PromptContext {
|
||||
workspace_dir: std::path::Path::new("."),
|
||||
model_name: "test",
|
||||
agent_id: "help",
|
||||
tools: &[],
|
||||
skills: &[],
|
||||
dispatcher_instructions: "",
|
||||
learned: LearnedContextData::default(),
|
||||
visible_tool_names: &visible,
|
||||
tool_call_format: ToolCallFormat::PFormat,
|
||||
connected_integrations: &[],
|
||||
connected_identities_md: String::new(),
|
||||
include_profile: false,
|
||||
include_memory_md: false,
|
||||
};
|
||||
let body = build(&ctx).unwrap();
|
||||
assert!(!body.is_empty());
|
||||
assert!(body.contains("Help Agent"));
|
||||
}
|
||||
}
|
||||
@@ -127,6 +127,11 @@ pub const BUILTINS: &[BuiltinAgent] = &[
|
||||
toml: include_str!("summarizer/agent.toml"),
|
||||
prompt_fn: super::summarizer::prompt::build,
|
||||
},
|
||||
BuiltinAgent {
|
||||
id: "help",
|
||||
toml: include_str!("help/agent.toml"),
|
||||
prompt_fn: super::help::prompt::build,
|
||||
},
|
||||
];
|
||||
|
||||
/// Parse every entry in [`BUILTINS`] into an [`AgentDefinition`].
|
||||
@@ -172,7 +177,7 @@ mod tests {
|
||||
fn all_builtins_parse() {
|
||||
let defs = load_builtins().expect("built-in TOML must parse");
|
||||
assert_eq!(defs.len(), BUILTINS.len());
|
||||
assert_eq!(defs.len(), 14, "expected 14 built-in agents");
|
||||
assert_eq!(defs.len(), 15, "expected 15 built-in agents");
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -365,13 +370,89 @@ mod tests {
|
||||
assert_eq!(def.max_iterations, 8);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn help_uses_gitbooks_tools_and_is_read_only() {
|
||||
let def = find("help");
|
||||
assert_eq!(def.sandbox_mode, SandboxMode::ReadOnly);
|
||||
match &def.tools {
|
||||
ToolScope::Named(tools) => {
|
||||
assert!(
|
||||
tools.iter().any(|t| t == "gitbooks_search"),
|
||||
"help needs gitbooks_search"
|
||||
);
|
||||
assert!(
|
||||
tools.iter().any(|t| t == "gitbooks_get_page"),
|
||||
"help needs gitbooks_get_page"
|
||||
);
|
||||
assert!(
|
||||
tools.iter().any(|t| t == "memory_recall"),
|
||||
"help needs memory_recall for personalisation"
|
||||
);
|
||||
// Help is docs-only — no write/exec tools.
|
||||
assert!(!tools.iter().any(|t| t == "shell"));
|
||||
assert!(!tools.iter().any(|t| t == "file_write"));
|
||||
assert!(!tools.iter().any(|t| t == "curl"));
|
||||
assert!(!tools.iter().any(|t| t == "spawn_subagent"));
|
||||
}
|
||||
ToolScope::Wildcard => panic!("help must have a Named tool scope"),
|
||||
}
|
||||
assert!(def.omit_identity);
|
||||
assert!(def.omit_safety_preamble);
|
||||
assert!(!def.omit_memory_context);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn researcher_has_curl_for_artifact_downloads() {
|
||||
let def = find("researcher");
|
||||
match &def.tools {
|
||||
ToolScope::Named(tools) => {
|
||||
assert!(
|
||||
tools.iter().any(|t| t == "curl"),
|
||||
"researcher needs curl for artifact downloads"
|
||||
);
|
||||
assert!(
|
||||
tools.iter().any(|t| t == "http_request"),
|
||||
"researcher still needs http_request"
|
||||
);
|
||||
}
|
||||
ToolScope::Wildcard => panic!("researcher must have Named tool scope"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn code_executor_has_curl_for_artifact_downloads() {
|
||||
let def = find("code_executor");
|
||||
match &def.tools {
|
||||
ToolScope::Named(tools) => {
|
||||
assert!(
|
||||
tools.iter().any(|t| t == "curl"),
|
||||
"code_executor needs curl for artifact/dataset fetches"
|
||||
);
|
||||
}
|
||||
ToolScope::Wildcard => panic!("code_executor must have Named tool scope"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn orchestrator_does_not_get_curl() {
|
||||
// Per design: curl is a `Write` permission tool that writes
|
||||
// to the workspace. The orchestrator delegates rather than
|
||||
// executing — code_executor / researcher own actual downloads.
|
||||
let def = find("orchestrator");
|
||||
if let ToolScope::Named(tools) = &def.tools {
|
||||
assert!(
|
||||
!tools.iter().any(|t| t == "curl"),
|
||||
"orchestrator must not have curl — it should delegate"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn welcome_has_onboarding_and_memory_tools() {
|
||||
let def = find("welcome");
|
||||
assert_eq!(def.sandbox_mode, SandboxMode::ReadOnly);
|
||||
match &def.tools {
|
||||
ToolScope::Named(tools) => {
|
||||
assert_eq!(tools.len(), 4, "welcome should have exactly four tools");
|
||||
assert!(
|
||||
tools.iter().any(|t| t == "check_onboarding_status"),
|
||||
"welcome needs check_onboarding_status"
|
||||
@@ -388,6 +469,18 @@ mod tests {
|
||||
tools.iter().any(|t| t == "composio_authorize"),
|
||||
"welcome needs composio_authorize"
|
||||
);
|
||||
assert!(
|
||||
tools.iter().any(|t| t == "gitbooks_search"),
|
||||
"welcome needs gitbooks_search to answer 'how does X work' during onboarding"
|
||||
);
|
||||
assert!(
|
||||
tools.iter().any(|t| t == "gitbooks_get_page"),
|
||||
"welcome needs gitbooks_get_page for full-page lookups"
|
||||
);
|
||||
// Welcome must not gain write/exec power; onboarding stays read-only.
|
||||
assert!(!tools.iter().any(|t| t == "shell"));
|
||||
assert!(!tools.iter().any(|t| t == "file_write"));
|
||||
assert!(!tools.iter().any(|t| t == "curl"));
|
||||
}
|
||||
ToolScope::Wildcard => panic!("welcome must have a Named tool scope"),
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ mod loader;
|
||||
pub mod archivist;
|
||||
pub mod code_executor;
|
||||
pub mod critic;
|
||||
pub mod help;
|
||||
pub mod integrations_agent;
|
||||
pub mod morning_briefing;
|
||||
pub mod orchestrator;
|
||||
|
||||
@@ -14,4 +14,4 @@ omit_skills_catalog = true
|
||||
hint = "agentic"
|
||||
|
||||
[tools]
|
||||
named = ["http_request", "web_search", "file_read", "memory_recall"]
|
||||
named = ["http_request", "curl", "web_search", "file_read", "memory_recall"]
|
||||
|
||||
@@ -30,9 +30,13 @@ hint = "agentic"
|
||||
# complete_onboarding: finalize the welcome flow once ready_to_complete is true.
|
||||
# memory_recall: pull additional user details beyond injected context.
|
||||
# composio_authorize: start an OAuth flow for a toolkit (e.g. gmail).
|
||||
# gitbooks_*: answer "how does X work" / "what can OpenHuman do" questions
|
||||
# during onboarding from the real product docs instead of guessing.
|
||||
named = [
|
||||
"check_onboarding_status",
|
||||
"complete_onboarding",
|
||||
"memory_recall",
|
||||
"composio_authorize",
|
||||
"gitbooks_search",
|
||||
"gitbooks_get_page",
|
||||
]
|
||||
|
||||
@@ -11,7 +11,9 @@ You are the **Welcome** agent — the first agent a new user talks to in OpenHum
|
||||
|
||||
## Tools you must use correctly
|
||||
|
||||
You have `check_onboarding_status`, `complete_onboarding`, `memory_recall`, and `composio_authorize`.
|
||||
You have `check_onboarding_status`, `complete_onboarding`, `memory_recall`, `composio_authorize`, `gitbooks_search`, and `gitbooks_get_page`.
|
||||
|
||||
If the user asks a "how does X work" / "what can OpenHuman do" / "where is the setting for…" question during onboarding, call `gitbooks_search` first and ground your answer in the real docs — do **not** guess feature names, flags, or commands. Use `gitbooks_get_page` only when the search excerpt is too partial. Cite the page URL inline so they can read more. Keep doc-grounded replies short and steer the conversation back to setup once they're satisfied — your primary job is still onboarding, not an open-ended Q&A loop.
|
||||
|
||||
| Tool | What it does |
|
||||
|------|--------------|
|
||||
|
||||
@@ -26,15 +26,16 @@ pub use schema::{
|
||||
build_runtime_proxy_client_with_timeouts, runtime_proxy_config, set_runtime_proxy_config,
|
||||
AgentConfig, AuditConfig, AutocompleteConfig, AutonomyConfig, BrowserComputerUseConfig,
|
||||
BrowserConfig, ChannelsConfig, ComposioConfig, Config, ContextConfig, CostConfig, CronConfig,
|
||||
DelegateAgentConfig, DictationActivationMode, DictationConfig, DiscordConfig,
|
||||
DockerRuntimeConfig, EmbeddingRouteConfig, HeartbeatConfig, HttpRequestConfig, IMessageConfig,
|
||||
IntegrationToggle, IntegrationsConfig, LarkConfig, LearningConfig, LocalAiConfig, MatrixConfig,
|
||||
MemoryConfig, ModelRouteConfig, MultimodalConfig, ObservabilityConfig, ProxyConfig, ProxyScope,
|
||||
ReflectionSource, ReliabilityConfig, ResourceLimitsConfig, RuntimeConfig, SandboxBackend,
|
||||
SandboxConfig, SchedulerConfig, ScreenIntelligenceConfig, SecretsConfig, SecurityConfig,
|
||||
SlackConfig, StorageConfig, StorageProviderConfig, StorageProviderSection, StreamMode,
|
||||
TelegramConfig, UpdateConfig, VoiceActivationMode, VoiceServerConfig, WebSearchConfig,
|
||||
WebhookConfig, DEFAULT_MODEL, MODEL_AGENTIC_V1, MODEL_CODING_V1, MODEL_REASONING_V1,
|
||||
CurlConfig, DelegateAgentConfig, DictationActivationMode, DictationConfig, DiscordConfig,
|
||||
DockerRuntimeConfig, EmbeddingRouteConfig, GitbooksConfig, HeartbeatConfig, HttpRequestConfig,
|
||||
IMessageConfig, IntegrationToggle, IntegrationsConfig, LarkConfig, LearningConfig,
|
||||
LocalAiConfig, MatrixConfig, MemoryConfig, ModelRouteConfig, MultimodalConfig,
|
||||
ObservabilityConfig, ProxyConfig, ProxyScope, ReflectionSource, ReliabilityConfig,
|
||||
ResourceLimitsConfig, RuntimeConfig, SandboxBackend, SandboxConfig, SchedulerConfig,
|
||||
ScreenIntelligenceConfig, SecretsConfig, SecurityConfig, SlackConfig, StorageConfig,
|
||||
StorageProviderConfig, StorageProviderSection, StreamMode, TelegramConfig, UpdateConfig,
|
||||
VoiceActivationMode, VoiceServerConfig, WebSearchConfig, WebhookConfig, DEFAULT_MODEL,
|
||||
MODEL_AGENTIC_V1, MODEL_CODING_V1, MODEL_REASONING_V1,
|
||||
};
|
||||
pub use schema::{
|
||||
clear_active_user, default_root_openhuman_dir, pre_login_user_dir, read_active_user_id,
|
||||
|
||||
@@ -57,9 +57,9 @@ pub use storage_memory::{
|
||||
MemoryConfig, MemoryTreeConfig, StorageConfig, StorageProviderConfig, StorageProviderSection,
|
||||
};
|
||||
pub use tools::{
|
||||
BrowserComputerUseConfig, BrowserConfig, ComposioConfig, ComputerControlConfig,
|
||||
HttpRequestConfig, IntegrationToggle, IntegrationsConfig, MultimodalConfig, SecretsConfig,
|
||||
WebSearchConfig,
|
||||
BrowserComputerUseConfig, BrowserConfig, ComposioConfig, ComputerControlConfig, CurlConfig,
|
||||
GitbooksConfig, HttpRequestConfig, IntegrationToggle, IntegrationsConfig, MultimodalConfig,
|
||||
SecretsConfig, WebSearchConfig,
|
||||
};
|
||||
pub use update::UpdateConfig;
|
||||
mod voice_server;
|
||||
|
||||
@@ -148,6 +148,75 @@ fn default_http_timeout_secs() -> u64 {
|
||||
30
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct CurlConfig {
|
||||
/// Subdirectory under `workspace_dir` where downloads land. Inputs
|
||||
/// are resolved relative to this root; absolute paths and `..`
|
||||
/// segments are rejected.
|
||||
#[serde(default = "default_curl_dest_subdir")]
|
||||
pub dest_subdir: String,
|
||||
/// Hard byte ceiling per download. Streaming aborts and the
|
||||
/// partial file is removed if exceeded.
|
||||
#[serde(default = "default_curl_max_download_bytes")]
|
||||
pub max_download_bytes: u64,
|
||||
/// Per-request timeout in seconds.
|
||||
#[serde(default = "default_curl_timeout_secs")]
|
||||
pub timeout_secs: u64,
|
||||
}
|
||||
|
||||
fn default_curl_dest_subdir() -> String {
|
||||
"downloads".into()
|
||||
}
|
||||
|
||||
fn default_curl_max_download_bytes() -> u64 {
|
||||
50 * 1024 * 1024
|
||||
}
|
||||
|
||||
fn default_curl_timeout_secs() -> u64 {
|
||||
120
|
||||
}
|
||||
|
||||
impl Default for CurlConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
dest_subdir: default_curl_dest_subdir(),
|
||||
max_download_bytes: default_curl_max_download_bytes(),
|
||||
timeout_secs: default_curl_timeout_secs(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct GitbooksConfig {
|
||||
/// When `true`, register `gitbooks_search` and `gitbooks_get_page`.
|
||||
#[serde(default = "defaults::default_true")]
|
||||
pub enabled: bool,
|
||||
/// MCP endpoint URL for the OpenHuman GitBook docs.
|
||||
#[serde(default = "default_gitbooks_endpoint")]
|
||||
pub endpoint: String,
|
||||
/// Per-request timeout in seconds.
|
||||
#[serde(default = "default_gitbooks_timeout_secs")]
|
||||
pub timeout_secs: u64,
|
||||
}
|
||||
|
||||
fn default_gitbooks_endpoint() -> String {
|
||||
"https://tinyhumans.gitbook.io/openhuman/~gitbook/mcp".into()
|
||||
}
|
||||
|
||||
fn default_gitbooks_timeout_secs() -> u64 {
|
||||
30
|
||||
}
|
||||
|
||||
impl Default for GitbooksConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
enabled: defaults::default_true(),
|
||||
endpoint: default_gitbooks_endpoint(),
|
||||
timeout_secs: default_gitbooks_timeout_secs(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct WebSearchConfig {
|
||||
#[serde(default = "default_web_search_max_results")]
|
||||
|
||||
@@ -103,6 +103,12 @@ pub struct Config {
|
||||
#[serde(default)]
|
||||
pub http_request: HttpRequestConfig,
|
||||
|
||||
#[serde(default)]
|
||||
pub curl: CurlConfig,
|
||||
|
||||
#[serde(default)]
|
||||
pub gitbooks: GitbooksConfig,
|
||||
|
||||
#[serde(default)]
|
||||
pub multimodal: MultimodalConfig,
|
||||
|
||||
@@ -240,6 +246,8 @@ impl Default for Config {
|
||||
secrets: SecretsConfig::default(),
|
||||
browser: BrowserConfig::default(),
|
||||
http_request: HttpRequestConfig::default(),
|
||||
curl: CurlConfig::default(),
|
||||
gitbooks: GitbooksConfig::default(),
|
||||
multimodal: MultimodalConfig::default(),
|
||||
web_search: WebSearchConfig::default(),
|
||||
proxy: ProxyConfig::default(),
|
||||
|
||||
@@ -0,0 +1,544 @@
|
||||
//! `curl` — download files from the web to a path under the workspace.
|
||||
//!
|
||||
//! Distinct from `http_request`: instead of returning the body inline
|
||||
//! (size-capped), `curl` streams to disk with a hard byte ceiling. Same
|
||||
//! SSRF/allowlist guards (shared via `url_guard`), shares
|
||||
//! `http_request.allowed_domains` so there is one allowlist to reason
|
||||
//! about.
|
||||
|
||||
use super::url_guard::{normalize_allowed_domains, validate_url};
|
||||
use crate::openhuman::security::SecurityPolicy;
|
||||
use crate::openhuman::tools::traits::{PermissionLevel, Tool, ToolResult};
|
||||
use async_trait::async_trait;
|
||||
use futures_util::StreamExt;
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::path::{Component, Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use tokio::fs;
|
||||
use tokio::io::AsyncWriteExt;
|
||||
|
||||
pub struct CurlTool {
|
||||
security: Arc<SecurityPolicy>,
|
||||
allowed_domains: Vec<String>,
|
||||
workspace_dir: PathBuf,
|
||||
dest_subdir: String,
|
||||
max_download_bytes: u64,
|
||||
timeout_secs: u64,
|
||||
}
|
||||
|
||||
impl CurlTool {
|
||||
pub fn new(
|
||||
security: Arc<SecurityPolicy>,
|
||||
allowed_domains: Vec<String>,
|
||||
workspace_dir: PathBuf,
|
||||
dest_subdir: String,
|
||||
max_download_bytes: u64,
|
||||
timeout_secs: u64,
|
||||
) -> Self {
|
||||
Self {
|
||||
security,
|
||||
allowed_domains: normalize_allowed_domains(allowed_domains),
|
||||
workspace_dir,
|
||||
dest_subdir: sanitize_dest_subdir(&dest_subdir),
|
||||
max_download_bytes,
|
||||
timeout_secs,
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolve a user-supplied dest path to an absolute path inside
|
||||
/// `<workspace>/<dest_subdir>`. Rejects absolute paths, `..`
|
||||
/// segments, and any other escape attempts.
|
||||
fn resolve_dest(&self, dest: &str) -> anyhow::Result<PathBuf> {
|
||||
let trimmed = dest.trim();
|
||||
if trimmed.is_empty() {
|
||||
anyhow::bail!("dest_path cannot be empty");
|
||||
}
|
||||
|
||||
let p = Path::new(trimmed);
|
||||
if p.is_absolute() {
|
||||
anyhow::bail!("dest_path must be relative — got absolute path");
|
||||
}
|
||||
|
||||
for component in p.components() {
|
||||
match component {
|
||||
Component::Normal(_) => {}
|
||||
Component::CurDir => {}
|
||||
Component::ParentDir => {
|
||||
anyhow::bail!("dest_path may not contain '..'");
|
||||
}
|
||||
Component::Prefix(_) | Component::RootDir => {
|
||||
anyhow::bail!("dest_path must be relative");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let root = self.workspace_dir.join(&self.dest_subdir);
|
||||
let resolved = root.join(p);
|
||||
|
||||
// Belt-and-braces: ensure the resolved path still lives under root.
|
||||
// Lexical check is sufficient because we already rejected `..`.
|
||||
if !resolved.starts_with(&root) {
|
||||
anyhow::bail!("dest_path resolves outside the downloads root");
|
||||
}
|
||||
|
||||
Ok(resolved)
|
||||
}
|
||||
|
||||
fn validate_url(&self, raw_url: &str) -> anyhow::Result<String> {
|
||||
validate_url(raw_url, &self.allowed_domains)
|
||||
}
|
||||
|
||||
fn default_filename_from_url(url: &str) -> String {
|
||||
let after_scheme = url.split_once("://").map(|(_, rest)| rest).unwrap_or(url);
|
||||
let path_part = after_scheme.split_once('/').map(|(_, p)| p).unwrap_or("");
|
||||
let last = path_part
|
||||
.split('?')
|
||||
.next()
|
||||
.unwrap_or("")
|
||||
.rsplit('/')
|
||||
.next()
|
||||
.unwrap_or("");
|
||||
let cleaned: String = last
|
||||
.chars()
|
||||
.filter(|c| c.is_alphanumeric() || matches!(c, '.' | '-' | '_'))
|
||||
.collect();
|
||||
if cleaned.is_empty() {
|
||||
"download.bin".into()
|
||||
} else {
|
||||
cleaned
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Tool for CurlTool {
|
||||
fn name(&self) -> &str {
|
||||
"curl"
|
||||
}
|
||||
|
||||
fn description(&self) -> &str {
|
||||
"Download a file from an http(s) URL into the workspace. The body is streamed to disk \
|
||||
with a hard byte ceiling. Same allowlist as `http_request`. Returns the saved path, \
|
||||
bytes written, content-type, and SHA-256 of the file."
|
||||
}
|
||||
|
||||
fn parameters_schema(&self) -> serde_json::Value {
|
||||
serde_json::json!({
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"url": {
|
||||
"type": "string",
|
||||
"description": "HTTP or HTTPS URL of the file to download"
|
||||
},
|
||||
"dest_path": {
|
||||
"type": "string",
|
||||
"description": "Destination path relative to the downloads root inside the workspace. No '..' or absolute paths. If omitted, the filename is inferred from the URL."
|
||||
},
|
||||
"headers": {
|
||||
"type": "object",
|
||||
"description": "Optional HTTP headers (e.g. {\"Authorization\": \"Bearer …\"})",
|
||||
"default": {}
|
||||
}
|
||||
},
|
||||
"required": ["url"]
|
||||
})
|
||||
}
|
||||
|
||||
fn permission_level(&self) -> PermissionLevel {
|
||||
PermissionLevel::Write
|
||||
}
|
||||
|
||||
async fn execute(&self, args: serde_json::Value) -> anyhow::Result<ToolResult> {
|
||||
let url = args
|
||||
.get("url")
|
||||
.and_then(|v| v.as_str())
|
||||
.ok_or_else(|| anyhow::anyhow!("Missing 'url' parameter"))?;
|
||||
|
||||
let dest_arg = args.get("dest_path").and_then(|v| v.as_str());
|
||||
let headers_val = args
|
||||
.get("headers")
|
||||
.cloned()
|
||||
.unwrap_or_else(|| serde_json::json!({}));
|
||||
|
||||
if !self.security.can_act() {
|
||||
tracing::debug!(target: "[curl]", url = %url, "blocked: autonomy read-only");
|
||||
return Ok(ToolResult::error("Action blocked: autonomy is read-only"));
|
||||
}
|
||||
if !self.security.record_action() {
|
||||
tracing::debug!(target: "[curl]", url = %url, "blocked: rate limit");
|
||||
return Ok(ToolResult::error("Action blocked: rate limit exceeded"));
|
||||
}
|
||||
|
||||
let url = match self.validate_url(url) {
|
||||
Ok(v) => v,
|
||||
Err(e) => {
|
||||
tracing::debug!(target: "[curl]", url = %url, reason = %e, "url validation failed");
|
||||
return Ok(ToolResult::error(e.to_string()));
|
||||
}
|
||||
};
|
||||
|
||||
let dest = match dest_arg {
|
||||
Some(d) => d.to_string(),
|
||||
None => Self::default_filename_from_url(&url),
|
||||
};
|
||||
let dest_path = match self.resolve_dest(&dest) {
|
||||
Ok(p) => p,
|
||||
Err(e) => {
|
||||
tracing::debug!(target: "[curl]", url = %url, dest = %dest, reason = %e, "dest_path rejected");
|
||||
return Ok(ToolResult::error(e.to_string()));
|
||||
}
|
||||
};
|
||||
|
||||
if let Some(parent) = dest_path.parent() {
|
||||
if let Err(e) = fs::create_dir_all(parent).await {
|
||||
tracing::error!(target: "[curl]", url = %url, dest = %dest_path.display(), reason = %e, "create_dir_all failed");
|
||||
return Ok(ToolResult::error(format!(
|
||||
"Failed to create destination directory: {e}"
|
||||
)));
|
||||
}
|
||||
}
|
||||
|
||||
let builder = reqwest::Client::builder()
|
||||
.timeout(Duration::from_secs(self.timeout_secs))
|
||||
.connect_timeout(Duration::from_secs(10))
|
||||
.redirect(reqwest::redirect::Policy::none());
|
||||
let builder =
|
||||
crate::openhuman::config::apply_runtime_proxy_to_builder(builder, "tool.curl");
|
||||
let client = match builder.build() {
|
||||
Ok(c) => c,
|
||||
Err(e) => {
|
||||
tracing::error!(target: "[curl]", reason = %e, "HTTP client build failed");
|
||||
return Ok(ToolResult::error(format!("HTTP client build failed: {e}")));
|
||||
}
|
||||
};
|
||||
|
||||
let mut request = client.get(&url);
|
||||
if let Some(obj) = headers_val.as_object() {
|
||||
for (k, v) in obj {
|
||||
if let Some(s) = v.as_str() {
|
||||
request = request.header(k, s);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
tracing::debug!(target: "[curl]", url = %url, dest = %dest_path.display(), "starting download");
|
||||
|
||||
let response = match request.send().await {
|
||||
Ok(r) => r,
|
||||
Err(e) => {
|
||||
tracing::error!(target: "[curl]", url = %url, reason = %e, "request send failed");
|
||||
return Ok(ToolResult::error(format!("Request failed: {e}")));
|
||||
}
|
||||
};
|
||||
|
||||
let status = response.status();
|
||||
if !status.is_success() {
|
||||
tracing::debug!(target: "[curl]", url = %url, status = %status.as_u16(), "non-success HTTP status");
|
||||
return Ok(ToolResult::error(format!(
|
||||
"HTTP {} {}",
|
||||
status.as_u16(),
|
||||
status.canonical_reason().unwrap_or("Unknown")
|
||||
)));
|
||||
}
|
||||
|
||||
let content_type = response
|
||||
.headers()
|
||||
.get(reqwest::header::CONTENT_TYPE)
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.unwrap_or("application/octet-stream")
|
||||
.to_string();
|
||||
|
||||
let mut file = match fs::File::create(&dest_path).await {
|
||||
Ok(f) => f,
|
||||
Err(e) => {
|
||||
tracing::error!(target: "[curl]", dest = %dest_path.display(), reason = %e, "fs::File::create failed");
|
||||
return Ok(ToolResult::error(format!(
|
||||
"Failed to create destination file: {e}"
|
||||
)));
|
||||
}
|
||||
};
|
||||
|
||||
let mut hasher = Sha256::new();
|
||||
let mut bytes_written: u64 = 0;
|
||||
let mut stream = response.bytes_stream();
|
||||
|
||||
while let Some(chunk) = stream.next().await {
|
||||
let chunk = match chunk {
|
||||
Ok(c) => c,
|
||||
Err(e) => {
|
||||
drop(file);
|
||||
if let Err(rm) = fs::remove_file(&dest_path).await {
|
||||
tracing::debug!(target: "[curl]", dest = %dest_path.display(), reason = %rm, "cleanup remove_file failed");
|
||||
}
|
||||
tracing::error!(target: "[curl]", url = %url, bytes_written, reason = %e, "stream error");
|
||||
return Ok(ToolResult::error(format!("Stream error: {e}")));
|
||||
}
|
||||
};
|
||||
|
||||
if bytes_written.saturating_add(chunk.len() as u64) > self.max_download_bytes {
|
||||
let _ = file.flush().await;
|
||||
drop(file);
|
||||
if let Err(rm) = fs::remove_file(&dest_path).await {
|
||||
tracing::debug!(target: "[curl]", dest = %dest_path.display(), reason = %rm, "cleanup remove_file failed");
|
||||
}
|
||||
tracing::error!(target: "[curl]", url = %url, bytes_written, max = self.max_download_bytes, "size cap exceeded — download aborted");
|
||||
return Ok(ToolResult::error(format!(
|
||||
"Download exceeded max_download_bytes ({} bytes)",
|
||||
self.max_download_bytes
|
||||
)));
|
||||
}
|
||||
|
||||
if let Err(e) = file.write_all(&chunk).await {
|
||||
drop(file);
|
||||
if let Err(rm) = fs::remove_file(&dest_path).await {
|
||||
tracing::debug!(target: "[curl]", dest = %dest_path.display(), reason = %rm, "cleanup remove_file failed");
|
||||
}
|
||||
tracing::error!(target: "[curl]", dest = %dest_path.display(), bytes_written, reason = %e, "write_all failed");
|
||||
return Ok(ToolResult::error(format!("Write failed: {e}")));
|
||||
}
|
||||
hasher.update(&chunk);
|
||||
bytes_written += chunk.len() as u64;
|
||||
}
|
||||
|
||||
if let Err(e) = file.flush().await {
|
||||
drop(file);
|
||||
if let Err(rm) = fs::remove_file(&dest_path).await {
|
||||
tracing::debug!(target: "[curl]", dest = %dest_path.display(), reason = %rm, "cleanup remove_file failed");
|
||||
}
|
||||
tracing::error!(target: "[curl]", dest = %dest_path.display(), bytes_written, reason = %e, "flush failed");
|
||||
return Ok(ToolResult::error(format!("Flush failed: {e}")));
|
||||
}
|
||||
|
||||
let sha256 = format!("{:x}", hasher.finalize());
|
||||
|
||||
tracing::debug!(
|
||||
target: "[curl]",
|
||||
url = %url,
|
||||
dest = %dest_path.display(),
|
||||
bytes = bytes_written,
|
||||
content_type = %content_type,
|
||||
sha256 = %sha256,
|
||||
"download complete"
|
||||
);
|
||||
|
||||
let payload = serde_json::json!({
|
||||
"path": dest_path.display().to_string(),
|
||||
"bytes_written": bytes_written,
|
||||
"content_type": content_type,
|
||||
"sha256": sha256,
|
||||
});
|
||||
Ok(ToolResult::success(payload.to_string()))
|
||||
}
|
||||
}
|
||||
|
||||
/// Sanitize the configured `dest_subdir` so a malicious or misconfigured
|
||||
/// `[curl].dest_subdir` cannot escape the workspace via absolute paths
|
||||
/// or `..` segments. Drops disallowed components rather than panicking;
|
||||
/// falls back to `"downloads"` if everything is filtered out.
|
||||
fn sanitize_dest_subdir(raw: &str) -> String {
|
||||
let trimmed = raw.trim();
|
||||
if trimmed.is_empty() {
|
||||
return "downloads".into();
|
||||
}
|
||||
let p = Path::new(trimmed);
|
||||
let mut buf = PathBuf::new();
|
||||
for component in p.components() {
|
||||
match component {
|
||||
Component::Normal(c) => buf.push(c),
|
||||
// Drop everything else: absolute roots, prefixes, parent dirs, cur dirs.
|
||||
_ => continue,
|
||||
}
|
||||
}
|
||||
if buf.as_os_str().is_empty() {
|
||||
return "downloads".into();
|
||||
}
|
||||
buf.to_string_lossy().into_owned()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::openhuman::security::SecurityPolicy;
|
||||
use tempfile::TempDir;
|
||||
|
||||
fn tool(tmp: &TempDir, allow: Vec<&str>) -> CurlTool {
|
||||
CurlTool::new(
|
||||
Arc::new(SecurityPolicy::default()),
|
||||
allow.into_iter().map(String::from).collect(),
|
||||
tmp.path().to_path_buf(),
|
||||
"downloads".into(),
|
||||
1024 * 1024,
|
||||
30,
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sanitize_dest_subdir_strips_absolute_paths() {
|
||||
assert_eq!(sanitize_dest_subdir("/etc/passwd"), "etc/passwd");
|
||||
assert_eq!(sanitize_dest_subdir("//foo"), "foo");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sanitize_dest_subdir_strips_parent_segments() {
|
||||
assert_eq!(sanitize_dest_subdir("../../etc"), "etc");
|
||||
assert_eq!(sanitize_dest_subdir("a/../b"), "a/b");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sanitize_dest_subdir_falls_back_to_downloads() {
|
||||
assert_eq!(sanitize_dest_subdir(""), "downloads");
|
||||
assert_eq!(sanitize_dest_subdir(" "), "downloads");
|
||||
assert_eq!(sanitize_dest_subdir(".."), "downloads");
|
||||
assert_eq!(sanitize_dest_subdir("/"), "downloads");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sanitize_dest_subdir_keeps_normal_paths() {
|
||||
assert_eq!(sanitize_dest_subdir("downloads"), "downloads");
|
||||
assert_eq!(sanitize_dest_subdir("artifacts/build"), "artifacts/build");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn new_sanitizes_malicious_dest_subdir() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let t = CurlTool::new(
|
||||
Arc::new(SecurityPolicy::default()),
|
||||
vec!["example.com".into()],
|
||||
tmp.path().to_path_buf(),
|
||||
"../../etc".into(),
|
||||
1024,
|
||||
30,
|
||||
);
|
||||
let resolved = t.resolve_dest("file.txt").unwrap();
|
||||
// Sanitizer reduced "../../etc" to "etc"; resolution must stay under workspace.
|
||||
assert!(resolved.starts_with(tmp.path().join("etc")));
|
||||
assert!(resolved.starts_with(tmp.path()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_dest_normal() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let t = tool(&tmp, vec!["example.com"]);
|
||||
let p = t.resolve_dest("foo/bar.txt").unwrap();
|
||||
assert!(p.starts_with(tmp.path().join("downloads")));
|
||||
assert!(p.ends_with("foo/bar.txt"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_dest_rejects_absolute() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let t = tool(&tmp, vec!["example.com"]);
|
||||
let err = t.resolve_dest("/etc/passwd").unwrap_err().to_string();
|
||||
assert!(err.contains("relative"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_dest_rejects_parent_dir() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let t = tool(&tmp, vec!["example.com"]);
|
||||
let err = t.resolve_dest("../etc/passwd").unwrap_err().to_string();
|
||||
assert!(err.contains(".."));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_dest_rejects_nested_parent_dir() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let t = tool(&tmp, vec!["example.com"]);
|
||||
let err = t.resolve_dest("a/../../b").unwrap_err().to_string();
|
||||
assert!(err.contains(".."));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_dest_rejects_empty() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let t = tool(&tmp, vec!["example.com"]);
|
||||
assert!(t.resolve_dest("").is_err());
|
||||
assert!(t.resolve_dest(" ").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn default_filename_from_url_basic() {
|
||||
assert_eq!(
|
||||
CurlTool::default_filename_from_url("https://example.com/foo/bar.zip"),
|
||||
"bar.zip"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn default_filename_from_url_query_stripped() {
|
||||
assert_eq!(
|
||||
CurlTool::default_filename_from_url("https://example.com/file.tar.gz?token=x"),
|
||||
"file.tar.gz"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn default_filename_from_url_root_falls_back() {
|
||||
assert_eq!(
|
||||
CurlTool::default_filename_from_url("https://example.com/"),
|
||||
"download.bin"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_blocks_when_rate_limited() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let security = Arc::new(SecurityPolicy {
|
||||
max_actions_per_hour: 0,
|
||||
..SecurityPolicy::default()
|
||||
});
|
||||
let t = CurlTool::new(
|
||||
security,
|
||||
vec!["example.com".into()],
|
||||
tmp.path().into(),
|
||||
"downloads".into(),
|
||||
1024,
|
||||
30,
|
||||
);
|
||||
let result = t
|
||||
.execute(serde_json::json!({"url": "https://example.com/x"}))
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(result.is_error);
|
||||
assert!(result.output().contains("rate limit"));
|
||||
}
|
||||
|
||||
/// Live integration smoke: downloads example.com (a tiny, stable
|
||||
/// public page). Gated behind `OPENHUMAN_CURL_LIVE_TEST=1` so CI /
|
||||
/// offline runs don't depend on the network.
|
||||
#[tokio::test]
|
||||
async fn live_download_example_com() {
|
||||
if std::env::var("OPENHUMAN_CURL_LIVE_TEST").ok().as_deref() != Some("1") {
|
||||
return;
|
||||
}
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let t = tool(&tmp, vec!["example.com"]);
|
||||
let result = t
|
||||
.execute(serde_json::json!({
|
||||
"url": "https://example.com/",
|
||||
"dest_path": "example.html"
|
||||
}))
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(!result.is_error, "live curl errored: {}", result.output());
|
||||
let payload: serde_json::Value = serde_json::from_str(&result.output()).unwrap();
|
||||
let bytes = payload["bytes_written"].as_u64().unwrap();
|
||||
assert!(bytes > 100, "unexpectedly small download: {bytes} bytes");
|
||||
let path = payload["path"].as_str().unwrap();
|
||||
let content = std::fs::read_to_string(path).unwrap();
|
||||
assert!(content.to_lowercase().contains("example domain"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_rejects_allowlist_miss() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let t = tool(&tmp, vec!["example.com"]);
|
||||
let result = t
|
||||
.execute(serde_json::json!({"url": "https://other.example.org/x"}))
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(result.is_error);
|
||||
assert!(result.output().contains("allowed_domains"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,425 @@
|
||||
//! `gitbooks` — answer questions about OpenHuman by talking to the
|
||||
//! GitBook MCP server.
|
||||
//!
|
||||
//! GitBook hosts a stateless Streamable-HTTP MCP server that exposes
|
||||
//! exactly two tools:
|
||||
//!
|
||||
//! - `searchDocumentation { query }` — returns excerpts + page links
|
||||
//! - `getPage { url }` — returns the full markdown of a page
|
||||
//!
|
||||
//! We mirror them as `gitbooks_search` and `gitbooks_get_page`. The
|
||||
//! server returns each JSON-RPC response in a single
|
||||
//! `event: message\ndata: {…}` SSE frame, so we do a tiny inline
|
||||
//! parse — no need to pull in a full SSE/MCP client crate yet.
|
||||
|
||||
use crate::openhuman::tools::traits::{PermissionLevel, Tool, ToolResult};
|
||||
use async_trait::async_trait;
|
||||
use serde_json::{json, Value};
|
||||
use std::sync::atomic::{AtomicI64, Ordering};
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
/// Minimal MCP-over-HTTP client for stateless servers (no session id).
|
||||
struct McpHttpClient {
|
||||
endpoint: String,
|
||||
timeout_secs: u64,
|
||||
next_id: AtomicI64,
|
||||
}
|
||||
|
||||
impl McpHttpClient {
|
||||
fn new(endpoint: String, timeout_secs: u64) -> Self {
|
||||
Self {
|
||||
endpoint,
|
||||
timeout_secs,
|
||||
next_id: AtomicI64::new(1),
|
||||
}
|
||||
}
|
||||
|
||||
async fn call_tool(&self, name: &str, arguments: Value) -> anyhow::Result<Value> {
|
||||
let id = self.next_id.fetch_add(1, Ordering::Relaxed);
|
||||
let body = json!({
|
||||
"jsonrpc": "2.0",
|
||||
"id": id,
|
||||
"method": "tools/call",
|
||||
"params": { "name": name, "arguments": arguments }
|
||||
});
|
||||
|
||||
let builder = reqwest::Client::builder()
|
||||
.timeout(Duration::from_secs(self.timeout_secs))
|
||||
.connect_timeout(Duration::from_secs(10))
|
||||
.redirect(reqwest::redirect::Policy::none());
|
||||
let builder =
|
||||
crate::openhuman::config::apply_runtime_proxy_to_builder(builder, "tool.gitbooks");
|
||||
let client = builder.build()?;
|
||||
|
||||
// Log only the redacted host so query strings / path params /
|
||||
// any future bearer-token-bearing endpoints don't leak into
|
||||
// logs aggregated for triage.
|
||||
tracing::debug!(
|
||||
target: "[gitbooks]",
|
||||
endpoint = %redact_endpoint(&self.endpoint),
|
||||
tool = %name,
|
||||
"MCP tools/call"
|
||||
);
|
||||
|
||||
let resp = client
|
||||
.post(&self.endpoint)
|
||||
.header("Content-Type", "application/json")
|
||||
.header("Accept", "application/json, text/event-stream")
|
||||
.body(serde_json::to_vec(&body)?)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let status = resp.status();
|
||||
let content_type = resp
|
||||
.headers()
|
||||
.get(reqwest::header::CONTENT_TYPE)
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
let text = resp.text().await?;
|
||||
|
||||
if !status.is_success() {
|
||||
anyhow::bail!("MCP HTTP {} — {}", status.as_u16(), text);
|
||||
}
|
||||
|
||||
let payload: Value = if content_type.starts_with("text/event-stream") {
|
||||
parse_sse_message(&text)?
|
||||
} else {
|
||||
serde_json::from_str(&text).map_err(|e| {
|
||||
anyhow::anyhow!("Failed to parse MCP JSON response: {e} — body: {text}")
|
||||
})?
|
||||
};
|
||||
|
||||
if let Some(err) = payload.get("error") {
|
||||
anyhow::bail!("MCP error: {err}");
|
||||
}
|
||||
let result = payload
|
||||
.get("result")
|
||||
.ok_or_else(|| anyhow::anyhow!("MCP response missing 'result': {payload}"))?
|
||||
.clone();
|
||||
Ok(result)
|
||||
}
|
||||
}
|
||||
|
||||
/// Reduce a configured endpoint URL to `scheme://host[:port]` for
|
||||
/// safe logging. Anything that doesn't parse as a recognisable
|
||||
/// http(s) URL is reported as `<redacted>` rather than echoed.
|
||||
fn redact_endpoint(raw: &str) -> String {
|
||||
let trimmed = raw.trim();
|
||||
let (scheme, rest) = if let Some(r) = trimmed.strip_prefix("https://") {
|
||||
("https", r)
|
||||
} else if let Some(r) = trimmed.strip_prefix("http://") {
|
||||
("http", r)
|
||||
} else {
|
||||
return "<redacted>".into();
|
||||
};
|
||||
let authority = rest.split(['/', '?', '#']).next().unwrap_or("");
|
||||
if authority.is_empty() || authority.contains('@') {
|
||||
return "<redacted>".into();
|
||||
}
|
||||
format!("{scheme}://{authority}")
|
||||
}
|
||||
|
||||
/// Parse the first `data: {…}` line from a Streamable-HTTP SSE
|
||||
/// response. The GitBook server emits exactly one frame per JSON-RPC
|
||||
/// request, so we do not need a full SSE state machine.
|
||||
fn parse_sse_message(body: &str) -> anyhow::Result<Value> {
|
||||
for line in body.lines() {
|
||||
let line = line.trim_end_matches('\r');
|
||||
if let Some(data) = line.strip_prefix("data:") {
|
||||
let data = data.trim_start();
|
||||
if data.is_empty() {
|
||||
continue;
|
||||
}
|
||||
return serde_json::from_str(data).map_err(|e| {
|
||||
anyhow::anyhow!("Failed to parse SSE data frame: {e} — line: {data}")
|
||||
});
|
||||
}
|
||||
}
|
||||
anyhow::bail!("No SSE data frame found in MCP response: {body}")
|
||||
}
|
||||
|
||||
/// Render an MCP `tools/call` result into a single string for the
|
||||
/// agent. MCP returns `{ content: [{ type, text }, …], isError? }`.
|
||||
fn render_tool_result(result: &Value) -> ToolResult {
|
||||
let is_error = result
|
||||
.get("isError")
|
||||
.and_then(Value::as_bool)
|
||||
.unwrap_or(false);
|
||||
|
||||
let mut out = String::new();
|
||||
if let Some(content) = result.get("content").and_then(Value::as_array) {
|
||||
for block in content {
|
||||
if let Some(t) = block.get("text").and_then(Value::as_str) {
|
||||
if !out.is_empty() {
|
||||
out.push_str("\n\n");
|
||||
}
|
||||
out.push_str(t);
|
||||
}
|
||||
}
|
||||
}
|
||||
if out.is_empty() {
|
||||
out = result.to_string();
|
||||
}
|
||||
|
||||
if is_error {
|
||||
ToolResult::error(out)
|
||||
} else {
|
||||
ToolResult::success(out)
|
||||
}
|
||||
}
|
||||
|
||||
// ── Search ─────────────────────────────────────────────────────────
|
||||
|
||||
pub struct GitbooksSearchTool {
|
||||
client: Arc<McpHttpClient>,
|
||||
}
|
||||
|
||||
impl GitbooksSearchTool {
|
||||
pub fn new(endpoint: String, timeout_secs: u64) -> Self {
|
||||
Self {
|
||||
client: Arc::new(McpHttpClient::new(endpoint, timeout_secs)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Tool for GitbooksSearchTool {
|
||||
fn name(&self) -> &str {
|
||||
"gitbooks_search"
|
||||
}
|
||||
|
||||
fn description(&self) -> &str {
|
||||
"Search the OpenHuman product documentation. Use this to answer questions about how \
|
||||
OpenHuman works, find features, look up configuration, or locate guides. Returns \
|
||||
excerpts with page titles and links — follow up with `gitbooks_get_page` for the \
|
||||
full markdown of a page."
|
||||
}
|
||||
|
||||
fn parameters_schema(&self) -> Value {
|
||||
json!({
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"query": {
|
||||
"type": "string",
|
||||
"description": "Natural-language question or keyword query about OpenHuman."
|
||||
}
|
||||
},
|
||||
"required": ["query"]
|
||||
})
|
||||
}
|
||||
|
||||
fn permission_level(&self) -> PermissionLevel {
|
||||
PermissionLevel::ReadOnly
|
||||
}
|
||||
|
||||
async fn execute(&self, args: Value) -> anyhow::Result<ToolResult> {
|
||||
let query = args
|
||||
.get("query")
|
||||
.and_then(Value::as_str)
|
||||
.ok_or_else(|| anyhow::anyhow!("Missing 'query' parameter"))?;
|
||||
if query.trim().is_empty() {
|
||||
return Ok(ToolResult::error("query cannot be empty"));
|
||||
}
|
||||
|
||||
match self
|
||||
.client
|
||||
.call_tool("searchDocumentation", json!({ "query": query }))
|
||||
.await
|
||||
{
|
||||
Ok(result) => Ok(render_tool_result(&result)),
|
||||
Err(e) => Ok(ToolResult::error(format!("gitbooks_search failed: {e}"))),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── Get page ──────────────────────────────────────────────────────
|
||||
|
||||
pub struct GitbooksGetPageTool {
|
||||
client: Arc<McpHttpClient>,
|
||||
}
|
||||
|
||||
impl GitbooksGetPageTool {
|
||||
pub fn new(endpoint: String, timeout_secs: u64) -> Self {
|
||||
Self {
|
||||
client: Arc::new(McpHttpClient::new(endpoint, timeout_secs)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Tool for GitbooksGetPageTool {
|
||||
fn name(&self) -> &str {
|
||||
"gitbooks_get_page"
|
||||
}
|
||||
|
||||
fn description(&self) -> &str {
|
||||
"Fetch the full markdown of a specific OpenHuman documentation page by URL. Pair this \
|
||||
with `gitbooks_search` — search returns partial excerpts; use this to get the \
|
||||
complete page when more detail is needed."
|
||||
}
|
||||
|
||||
fn parameters_schema(&self) -> Value {
|
||||
json!({
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"url": {
|
||||
"type": "string",
|
||||
"description": "The full URL of the OpenHuman documentation page (e.g. https://tinyhumans.gitbook.io/openhuman/getting-started)."
|
||||
}
|
||||
},
|
||||
"required": ["url"]
|
||||
})
|
||||
}
|
||||
|
||||
fn permission_level(&self) -> PermissionLevel {
|
||||
PermissionLevel::ReadOnly
|
||||
}
|
||||
|
||||
async fn execute(&self, args: Value) -> anyhow::Result<ToolResult> {
|
||||
let url = args
|
||||
.get("url")
|
||||
.and_then(Value::as_str)
|
||||
.ok_or_else(|| anyhow::anyhow!("Missing 'url' parameter"))?;
|
||||
if url.trim().is_empty() {
|
||||
return Ok(ToolResult::error("url cannot be empty"));
|
||||
}
|
||||
|
||||
match self
|
||||
.client
|
||||
.call_tool("getPage", json!({ "url": url }))
|
||||
.await
|
||||
{
|
||||
Ok(result) => Ok(render_tool_result(&result)),
|
||||
Err(e) => Ok(ToolResult::error(format!("gitbooks_get_page failed: {e}"))),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn redact_endpoint_keeps_only_origin() {
|
||||
assert_eq!(
|
||||
redact_endpoint("https://tinyhumans.gitbook.io/openhuman/~gitbook/mcp"),
|
||||
"https://tinyhumans.gitbook.io"
|
||||
);
|
||||
assert_eq!(
|
||||
redact_endpoint("http://example.com:8080/path?token=secret"),
|
||||
"http://example.com:8080"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn redact_endpoint_rejects_userinfo_and_unknown_schemes() {
|
||||
assert_eq!(
|
||||
redact_endpoint("https://user:pass@example.com/x"),
|
||||
"<redacted>"
|
||||
);
|
||||
assert_eq!(redact_endpoint("ftp://example.com"), "<redacted>");
|
||||
assert_eq!(redact_endpoint(""), "<redacted>");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_sse_extracts_data_frame() {
|
||||
let body = "event: message\ndata: {\"jsonrpc\":\"2.0\",\"id\":1,\"result\":{\"x\":1}}\n\n";
|
||||
let v = parse_sse_message(body).unwrap();
|
||||
assert_eq!(v["result"]["x"], 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_sse_handles_crlf() {
|
||||
let body = "event: message\r\ndata: {\"jsonrpc\":\"2.0\",\"id\":1,\"result\":{}}\r\n\r\n";
|
||||
assert!(parse_sse_message(body).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_sse_errors_when_no_data_line() {
|
||||
let body = "event: ping\n\n";
|
||||
assert!(parse_sse_message(body).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_sse_errors_on_invalid_json() {
|
||||
let body = "data: not-json\n\n";
|
||||
assert!(parse_sse_message(body).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn render_tool_result_concatenates_text_blocks() {
|
||||
let r = json!({
|
||||
"content": [
|
||||
{"type": "text", "text": "first"},
|
||||
{"type": "text", "text": "second"}
|
||||
]
|
||||
});
|
||||
let out = render_tool_result(&r);
|
||||
assert!(!out.is_error);
|
||||
assert!(out.output().contains("first"));
|
||||
assert!(out.output().contains("second"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn render_tool_result_marks_errors() {
|
||||
let r = json!({
|
||||
"content": [{"type": "text", "text": "boom"}],
|
||||
"isError": true
|
||||
});
|
||||
let out = render_tool_result(&r);
|
||||
assert!(out.is_error);
|
||||
assert!(out.output().contains("boom"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn render_tool_result_falls_back_to_raw_json() {
|
||||
let r = json!({"weird": "shape"});
|
||||
let out = render_tool_result(&r);
|
||||
assert!(out.output().contains("weird"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn search_rejects_empty_query() {
|
||||
let t = GitbooksSearchTool::new("https://example.com/mcp".into(), 5);
|
||||
let result = t.execute(json!({"query": " "})).await.unwrap();
|
||||
assert!(result.is_error);
|
||||
assert!(result.output().contains("empty"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn get_page_rejects_empty_url() {
|
||||
let t = GitbooksGetPageTool::new("https://example.com/mcp".into(), 5);
|
||||
let result = t.execute(json!({"url": ""})).await.unwrap();
|
||||
assert!(result.is_error);
|
||||
}
|
||||
|
||||
/// Live integration test against the real GitBook MCP endpoint.
|
||||
/// Gated behind `OPENHUMAN_GITBOOKS_LIVE_TEST=1` so CI / offline
|
||||
/// builds don't depend on the public network.
|
||||
#[tokio::test]
|
||||
async fn live_search_smoke() {
|
||||
if std::env::var("OPENHUMAN_GITBOOKS_LIVE_TEST")
|
||||
.ok()
|
||||
.as_deref()
|
||||
!= Some("1")
|
||||
{
|
||||
return;
|
||||
}
|
||||
let t = GitbooksSearchTool::new(
|
||||
"https://tinyhumans.gitbook.io/openhuman/~gitbook/mcp".into(),
|
||||
30,
|
||||
);
|
||||
let result = t
|
||||
.execute(json!({"query": "what is openhuman"}))
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
!result.is_error,
|
||||
"live search returned error: {}",
|
||||
result.output()
|
||||
);
|
||||
assert!(!result.output().is_empty());
|
||||
}
|
||||
}
|
||||
@@ -1,3 +1,4 @@
|
||||
use super::url_guard::{normalize_allowed_domains, validate_url};
|
||||
use crate::openhuman::security::SecurityPolicy;
|
||||
use crate::openhuman::tools::traits::{Tool, ToolResult};
|
||||
use async_trait::async_trait;
|
||||
@@ -30,37 +31,7 @@ impl HttpRequestTool {
|
||||
}
|
||||
|
||||
fn validate_url(&self, raw_url: &str) -> anyhow::Result<String> {
|
||||
let url = raw_url.trim();
|
||||
|
||||
if url.is_empty() {
|
||||
anyhow::bail!("URL cannot be empty");
|
||||
}
|
||||
|
||||
if url.chars().any(char::is_whitespace) {
|
||||
anyhow::bail!("URL cannot contain whitespace");
|
||||
}
|
||||
|
||||
if !url.starts_with("http://") && !url.starts_with("https://") {
|
||||
anyhow::bail!("Only http:// and https:// URLs are allowed");
|
||||
}
|
||||
|
||||
if self.allowed_domains.is_empty() {
|
||||
anyhow::bail!(
|
||||
"HTTP request tool is enabled but no allowed_domains are configured. Add [http_request].allowed_domains in config.toml"
|
||||
);
|
||||
}
|
||||
|
||||
let host = extract_host(url)?;
|
||||
|
||||
if is_private_or_local_host(&host) {
|
||||
anyhow::bail!("Blocked local/private host: {host}");
|
||||
}
|
||||
|
||||
if !host_matches_allowlist(&host, &self.allowed_domains) {
|
||||
anyhow::bail!("Host '{host}' is not in http_request.allowed_domains");
|
||||
}
|
||||
|
||||
Ok(url.to_string())
|
||||
validate_url(raw_url, &self.allowed_domains)
|
||||
}
|
||||
|
||||
fn validate_method(&self, method: &str) -> anyhow::Result<reqwest::Method> {
|
||||
@@ -225,7 +196,6 @@ impl Tool for HttpRequestTool {
|
||||
let status = response.status();
|
||||
let status_code = status.as_u16();
|
||||
|
||||
// Get response headers (redact sensitive ones)
|
||||
let response_headers = response.headers().iter();
|
||||
let headers_text = response_headers
|
||||
.map(|(k, _)| {
|
||||
@@ -239,7 +209,6 @@ impl Tool for HttpRequestTool {
|
||||
.collect::<Vec<_>>()
|
||||
.join(", ");
|
||||
|
||||
// Get response body with size limit
|
||||
let response_text = match response.text().await {
|
||||
Ok(text) => self.truncate_response(&text),
|
||||
Err(e) => format!("[Failed to read response body: {e}]"),
|
||||
@@ -264,149 +233,6 @@ impl Tool for HttpRequestTool {
|
||||
}
|
||||
}
|
||||
|
||||
// Helper functions similar to browser_open.rs
|
||||
|
||||
fn normalize_allowed_domains(domains: Vec<String>) -> Vec<String> {
|
||||
let mut normalized = domains
|
||||
.into_iter()
|
||||
.filter_map(|d| normalize_domain(&d))
|
||||
.collect::<Vec<_>>();
|
||||
normalized.sort_unstable();
|
||||
normalized.dedup();
|
||||
normalized
|
||||
}
|
||||
|
||||
fn normalize_domain(raw: &str) -> Option<String> {
|
||||
let mut d = raw.trim().to_lowercase();
|
||||
if d.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
if let Some(stripped) = d.strip_prefix("https://") {
|
||||
d = stripped.to_string();
|
||||
} else if let Some(stripped) = d.strip_prefix("http://") {
|
||||
d = stripped.to_string();
|
||||
}
|
||||
|
||||
if let Some((host, _)) = d.split_once('/') {
|
||||
d = host.to_string();
|
||||
}
|
||||
|
||||
d = d.trim_start_matches('.').trim_end_matches('.').to_string();
|
||||
|
||||
if let Some((host, _)) = d.split_once(':') {
|
||||
d = host.to_string();
|
||||
}
|
||||
|
||||
if d.is_empty() || d.chars().any(char::is_whitespace) {
|
||||
return None;
|
||||
}
|
||||
|
||||
Some(d)
|
||||
}
|
||||
|
||||
fn extract_host(url: &str) -> anyhow::Result<String> {
|
||||
let rest = url
|
||||
.strip_prefix("http://")
|
||||
.or_else(|| url.strip_prefix("https://"))
|
||||
.ok_or_else(|| anyhow::anyhow!("Only http:// and https:// URLs are allowed"))?;
|
||||
|
||||
let authority = rest
|
||||
.split(['/', '?', '#'])
|
||||
.next()
|
||||
.ok_or_else(|| anyhow::anyhow!("Invalid URL"))?;
|
||||
|
||||
if authority.is_empty() {
|
||||
anyhow::bail!("URL must include a host");
|
||||
}
|
||||
|
||||
if authority.contains('@') {
|
||||
anyhow::bail!("URL userinfo is not allowed");
|
||||
}
|
||||
|
||||
if authority.starts_with('[') {
|
||||
anyhow::bail!("IPv6 hosts are not supported in http_request");
|
||||
}
|
||||
|
||||
let host = authority
|
||||
.split(':')
|
||||
.next()
|
||||
.unwrap_or_default()
|
||||
.trim()
|
||||
.trim_end_matches('.')
|
||||
.to_lowercase();
|
||||
|
||||
if host.is_empty() {
|
||||
anyhow::bail!("URL must include a valid host");
|
||||
}
|
||||
|
||||
Ok(host)
|
||||
}
|
||||
|
||||
fn host_matches_allowlist(host: &str, allowed_domains: &[String]) -> bool {
|
||||
allowed_domains.iter().any(|domain| {
|
||||
host == domain
|
||||
|| host
|
||||
.strip_suffix(domain)
|
||||
.is_some_and(|prefix| prefix.ends_with('.'))
|
||||
})
|
||||
}
|
||||
|
||||
fn is_private_or_local_host(host: &str) -> bool {
|
||||
// Strip brackets from IPv6 addresses like [::1]
|
||||
let bare = host
|
||||
.strip_prefix('[')
|
||||
.and_then(|h| h.strip_suffix(']'))
|
||||
.unwrap_or(host);
|
||||
|
||||
let has_local_tld = bare
|
||||
.rsplit('.')
|
||||
.next()
|
||||
.is_some_and(|label| label == "local");
|
||||
|
||||
if bare == "localhost" || bare.ends_with(".localhost") || has_local_tld {
|
||||
return true;
|
||||
}
|
||||
|
||||
if let Ok(ip) = bare.parse::<std::net::IpAddr>() {
|
||||
return match ip {
|
||||
std::net::IpAddr::V4(v4) => is_non_global_v4(v4),
|
||||
std::net::IpAddr::V6(v6) => is_non_global_v6(v6),
|
||||
};
|
||||
}
|
||||
|
||||
false
|
||||
}
|
||||
|
||||
/// Returns true if the IPv4 address is not globally routable.
|
||||
fn is_non_global_v4(v4: std::net::Ipv4Addr) -> bool {
|
||||
let [a, b, c, _] = v4.octets();
|
||||
v4.is_loopback() // 127.0.0.0/8
|
||||
|| v4.is_private() // 10/8, 172.16/12, 192.168/16
|
||||
|| v4.is_link_local() // 169.254.0.0/16
|
||||
|| v4.is_unspecified() // 0.0.0.0
|
||||
|| v4.is_broadcast() // 255.255.255.255
|
||||
|| v4.is_multicast() // 224.0.0.0/4
|
||||
|| (a == 100 && (64..=127).contains(&b)) // Shared address space (RFC 6598)
|
||||
|| a >= 240 // Reserved (240.0.0.0/4, except broadcast)
|
||||
|| (a == 192 && b == 0 && (c == 0 || c == 2)) // IETF assignments + TEST-NET-1
|
||||
|| (a == 198 && b == 51) // Documentation (198.51.100.0/24)
|
||||
|| (a == 203 && b == 0) // Documentation (203.0.113.0/24)
|
||||
|| (a == 198 && (18..=19).contains(&b)) // Benchmarking (198.18.0.0/15)
|
||||
}
|
||||
|
||||
/// Returns true if the IPv6 address is not globally routable.
|
||||
fn is_non_global_v6(v6: std::net::Ipv6Addr) -> bool {
|
||||
let segs = v6.segments();
|
||||
v6.is_loopback() // ::1
|
||||
|| v6.is_unspecified() // ::
|
||||
|| v6.is_multicast() // ff00::/8
|
||||
|| (segs[0] & 0xfe00) == 0xfc00 // Unique-local (fc00::/7)
|
||||
|| (segs[0] & 0xffc0) == 0xfe80 // Link-local (fe80::/10)
|
||||
|| (segs[0] == 0x2001 && segs[1] == 0x0db8) // Documentation (2001:db8::/32)
|
||||
|| v6.to_ipv4_mapped().is_some_and(is_non_global_v4)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -425,102 +251,6 @@ mod tests {
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normalize_domain_strips_scheme_path_and_case() {
|
||||
let got = normalize_domain(" HTTPS://Docs.Example.com/path ").unwrap();
|
||||
assert_eq!(got, "docs.example.com");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normalize_allowed_domains_deduplicates() {
|
||||
let got = normalize_allowed_domains(vec![
|
||||
"example.com".into(),
|
||||
"EXAMPLE.COM".into(),
|
||||
"https://example.com/".into(),
|
||||
]);
|
||||
assert_eq!(got, vec!["example.com".to_string()]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_accepts_exact_domain() {
|
||||
let tool = test_tool(vec!["example.com"]);
|
||||
let got = tool.validate_url("https://example.com/docs").unwrap();
|
||||
assert_eq!(got, "https://example.com/docs");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_accepts_http() {
|
||||
let tool = test_tool(vec!["example.com"]);
|
||||
assert!(tool.validate_url("http://example.com").is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_accepts_subdomain() {
|
||||
let tool = test_tool(vec!["example.com"]);
|
||||
assert!(tool.validate_url("https://api.example.com/v1").is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_rejects_allowlist_miss() {
|
||||
let tool = test_tool(vec!["example.com"]);
|
||||
let err = tool
|
||||
.validate_url("https://google.com")
|
||||
.unwrap_err()
|
||||
.to_string();
|
||||
assert!(err.contains("allowed_domains"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_rejects_localhost() {
|
||||
let tool = test_tool(vec!["localhost"]);
|
||||
let err = tool
|
||||
.validate_url("https://localhost:8080")
|
||||
.unwrap_err()
|
||||
.to_string();
|
||||
assert!(err.contains("local/private"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_rejects_private_ipv4() {
|
||||
let tool = test_tool(vec!["192.168.1.5"]);
|
||||
let err = tool
|
||||
.validate_url("https://192.168.1.5")
|
||||
.unwrap_err()
|
||||
.to_string();
|
||||
assert!(err.contains("local/private"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_rejects_whitespace() {
|
||||
let tool = test_tool(vec!["example.com"]);
|
||||
let err = tool
|
||||
.validate_url("https://example.com/hello world")
|
||||
.unwrap_err()
|
||||
.to_string();
|
||||
assert!(err.contains("whitespace"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_rejects_userinfo() {
|
||||
let tool = test_tool(vec!["example.com"]);
|
||||
let err = tool
|
||||
.validate_url("https://user@example.com")
|
||||
.unwrap_err()
|
||||
.to_string();
|
||||
assert!(err.contains("userinfo"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_requires_allowlist() {
|
||||
let security = Arc::new(SecurityPolicy::default());
|
||||
let tool = HttpRequestTool::new(security, vec![], 1_000_000, 30);
|
||||
let err = tool
|
||||
.validate_url("https://example.com")
|
||||
.unwrap_err()
|
||||
.to_string();
|
||||
assert!(err.contains("allowed_domains"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_accepts_valid_methods() {
|
||||
let tool = test_tool(vec!["example.com"]);
|
||||
@@ -540,89 +270,6 @@ mod tests {
|
||||
assert!(err.contains("Unsupported HTTP method"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn blocks_multicast_ipv4() {
|
||||
assert!(is_private_or_local_host("224.0.0.1"));
|
||||
assert!(is_private_or_local_host("239.255.255.255"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn blocks_broadcast() {
|
||||
assert!(is_private_or_local_host("255.255.255.255"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn blocks_reserved_ipv4() {
|
||||
assert!(is_private_or_local_host("240.0.0.1"));
|
||||
assert!(is_private_or_local_host("250.1.2.3"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn blocks_documentation_ranges() {
|
||||
assert!(is_private_or_local_host("192.0.2.1")); // TEST-NET-1
|
||||
assert!(is_private_or_local_host("198.51.100.1")); // TEST-NET-2
|
||||
assert!(is_private_or_local_host("203.0.113.1")); // TEST-NET-3
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn blocks_benchmarking_range() {
|
||||
assert!(is_private_or_local_host("198.18.0.1"));
|
||||
assert!(is_private_or_local_host("198.19.255.255"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn blocks_ipv6_localhost() {
|
||||
assert!(is_private_or_local_host("::1"));
|
||||
assert!(is_private_or_local_host("[::1]"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn blocks_ipv6_multicast() {
|
||||
assert!(is_private_or_local_host("ff02::1"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn blocks_ipv6_link_local() {
|
||||
assert!(is_private_or_local_host("fe80::1"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn blocks_ipv6_unique_local() {
|
||||
assert!(is_private_or_local_host("fd00::1"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn blocks_ipv4_mapped_ipv6() {
|
||||
assert!(is_private_or_local_host("::ffff:127.0.0.1"));
|
||||
assert!(is_private_or_local_host("::ffff:192.168.1.1"));
|
||||
assert!(is_private_or_local_host("::ffff:10.0.0.1"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn allows_public_ipv4() {
|
||||
assert!(!is_private_or_local_host("8.8.8.8"));
|
||||
assert!(!is_private_or_local_host("1.1.1.1"));
|
||||
assert!(!is_private_or_local_host("93.184.216.34"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn blocks_ipv6_documentation_range() {
|
||||
assert!(is_private_or_local_host("2001:db8::1"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn allows_public_ipv6() {
|
||||
assert!(!is_private_or_local_host("2607:f8b0:4004:800::200e"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn blocks_shared_address_space() {
|
||||
assert!(is_private_or_local_host("100.64.0.1"));
|
||||
assert!(is_private_or_local_host("100.127.255.255"));
|
||||
assert!(!is_private_or_local_host("100.63.0.1")); // Just below range
|
||||
assert!(!is_private_or_local_host("100.128.0.1")); // Just above range
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_blocks_readonly_mode() {
|
||||
let security = Arc::new(SecurityPolicy {
|
||||
@@ -670,7 +317,7 @@ mod tests {
|
||||
);
|
||||
let text = "hello world this is long";
|
||||
let truncated = tool.truncate_response(text);
|
||||
assert!(truncated.len() <= 10 + 60); // limit + message
|
||||
assert!(truncated.len() <= 10 + 60);
|
||||
assert!(truncated.contains("[Response truncated"));
|
||||
}
|
||||
|
||||
@@ -726,130 +373,9 @@ mod tests {
|
||||
assert_eq!(headers[0].1, "Bearer real-token");
|
||||
}
|
||||
|
||||
// ── SSRF: alternate IP notation bypass defense-in-depth ─────────
|
||||
//
|
||||
// Rust's IpAddr::parse() rejects non-standard notations (octal, hex,
|
||||
// decimal integer, zero-padded). These tests document that property
|
||||
// so regressions are caught if the parsing strategy ever changes.
|
||||
|
||||
#[test]
|
||||
fn ssrf_octal_loopback_not_parsed_as_ip() {
|
||||
// 0177.0.0.1 is octal for 127.0.0.1 in some languages, but
|
||||
// Rust's IpAddr rejects it — it falls through as a hostname.
|
||||
assert!(!is_private_or_local_host("0177.0.0.1"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ssrf_hex_loopback_not_parsed_as_ip() {
|
||||
// 0x7f000001 is hex for 127.0.0.1 in some languages.
|
||||
assert!(!is_private_or_local_host("0x7f000001"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ssrf_decimal_loopback_not_parsed_as_ip() {
|
||||
// 2130706433 is decimal for 127.0.0.1 in some languages.
|
||||
assert!(!is_private_or_local_host("2130706433"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ssrf_zero_padded_loopback_not_parsed_as_ip() {
|
||||
// 127.000.000.001 uses zero-padded octets.
|
||||
assert!(!is_private_or_local_host("127.000.000.001"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ssrf_alternate_notations_rejected_by_validate_url() {
|
||||
// Even if is_private_or_local_host doesn't flag these, they
|
||||
// fail the allowlist because they're treated as hostnames.
|
||||
let tool = test_tool(vec!["example.com"]);
|
||||
for notation in [
|
||||
"http://0177.0.0.1",
|
||||
"http://0x7f000001",
|
||||
"http://2130706433",
|
||||
"http://127.000.000.001",
|
||||
] {
|
||||
let err = tool.validate_url(notation).unwrap_err().to_string();
|
||||
assert!(
|
||||
err.contains("allowed_domains"),
|
||||
"Expected allowlist rejection for {notation}, got: {err}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn redirect_policy_is_none() {
|
||||
// Structural test: the tool should be buildable with redirect-safe config.
|
||||
// The actual Policy::none() enforcement is in execute_request's client builder.
|
||||
let tool = test_tool(vec!["example.com"]);
|
||||
assert_eq!(tool.name(), "http_request");
|
||||
}
|
||||
|
||||
// ── §1.4 DNS rebinding / SSRF defense-in-depth tests ─────
|
||||
|
||||
#[test]
|
||||
fn ssrf_blocks_loopback_127_range() {
|
||||
assert!(is_private_or_local_host("127.0.0.1"));
|
||||
assert!(is_private_or_local_host("127.0.0.2"));
|
||||
assert!(is_private_or_local_host("127.255.255.255"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ssrf_blocks_rfc1918_10_range() {
|
||||
assert!(is_private_or_local_host("10.0.0.1"));
|
||||
assert!(is_private_or_local_host("10.255.255.255"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ssrf_blocks_rfc1918_172_range() {
|
||||
assert!(is_private_or_local_host("172.16.0.1"));
|
||||
assert!(is_private_or_local_host("172.31.255.255"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ssrf_blocks_unspecified_address() {
|
||||
assert!(is_private_or_local_host("0.0.0.0"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ssrf_blocks_dot_localhost_subdomain() {
|
||||
assert!(is_private_or_local_host("evil.localhost"));
|
||||
assert!(is_private_or_local_host("a.b.localhost"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ssrf_blocks_dot_local_tld() {
|
||||
assert!(is_private_or_local_host("service.local"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ssrf_ipv6_unspecified() {
|
||||
assert!(is_private_or_local_host("::"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_rejects_ftp_scheme() {
|
||||
let tool = test_tool(vec!["example.com"]);
|
||||
let err = tool
|
||||
.validate_url("ftp://example.com")
|
||||
.unwrap_err()
|
||||
.to_string();
|
||||
assert!(err.contains("http://") || err.contains("https://"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_rejects_empty_url() {
|
||||
let tool = test_tool(vec!["example.com"]);
|
||||
let err = tool.validate_url("").unwrap_err().to_string();
|
||||
assert!(err.contains("empty"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_rejects_ipv6_host() {
|
||||
let tool = test_tool(vec!["example.com"]);
|
||||
let err = tool
|
||||
.validate_url("http://[::1]:8080/path")
|
||||
.unwrap_err()
|
||||
.to_string();
|
||||
assert!(err.contains("IPv6"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,7 +1,12 @@
|
||||
mod composio;
|
||||
mod curl;
|
||||
mod gitbooks;
|
||||
mod http_request;
|
||||
mod url_guard;
|
||||
mod web_search;
|
||||
|
||||
pub use composio::{ComposioAction, ComposioTool};
|
||||
pub use curl::CurlTool;
|
||||
pub use gitbooks::{GitbooksGetPageTool, GitbooksSearchTool};
|
||||
pub use http_request::HttpRequestTool;
|
||||
pub use web_search::WebSearchTool;
|
||||
|
||||
@@ -0,0 +1,475 @@
|
||||
//! Shared URL validation + SSRF guards for outbound network tools.
|
||||
//!
|
||||
//! Used by `http_request`, `curl`, and any future tool that takes a
|
||||
//! user-supplied URL. The contract is intentionally strict:
|
||||
//!
|
||||
//! - http(s) only
|
||||
//! - non-empty allowlist required (callers pass it in)
|
||||
//! - no whitespace, no userinfo, no IPv6 hosts
|
||||
//! - blocks loopback / RFC1918 / link-local / multicast / documentation /
|
||||
//! shared-address / IPv4-mapped IPv6, including `localhost` /
|
||||
//! `*.localhost` / `*.local`
|
||||
//!
|
||||
//! The blocklist deliberately does NOT cover alternate IP notations
|
||||
//! (octal, hex, decimal) because Rust's `IpAddr::parse` rejects them —
|
||||
//! they fall through and get rejected by the allowlist instead. See the
|
||||
//! tests in `http_request.rs` for the documented behaviour.
|
||||
|
||||
/// Validate a URL against the allowlist + SSRF rules. Returns the
|
||||
/// original URL on success.
|
||||
pub(super) fn validate_url(raw_url: &str, allowed_domains: &[String]) -> anyhow::Result<String> {
|
||||
let url = raw_url.trim();
|
||||
|
||||
if url.is_empty() {
|
||||
anyhow::bail!("URL cannot be empty");
|
||||
}
|
||||
|
||||
if url.chars().any(char::is_whitespace) {
|
||||
anyhow::bail!("URL cannot contain whitespace");
|
||||
}
|
||||
|
||||
if !url.starts_with("http://") && !url.starts_with("https://") {
|
||||
anyhow::bail!("Only http:// and https:// URLs are allowed");
|
||||
}
|
||||
|
||||
if allowed_domains.is_empty() {
|
||||
anyhow::bail!(
|
||||
"Network tool is enabled but no allowed_domains are configured. Add [http_request].allowed_domains in config.toml"
|
||||
);
|
||||
}
|
||||
|
||||
let host = extract_host(url)?;
|
||||
|
||||
if is_private_or_local_host(&host) {
|
||||
anyhow::bail!("Blocked local/private host: {host}");
|
||||
}
|
||||
|
||||
if !host_matches_allowlist(&host, allowed_domains) {
|
||||
anyhow::bail!("Host '{host}' is not in http_request.allowed_domains");
|
||||
}
|
||||
|
||||
Ok(url.to_string())
|
||||
}
|
||||
|
||||
pub(super) fn normalize_allowed_domains(domains: Vec<String>) -> Vec<String> {
|
||||
let mut normalized = domains
|
||||
.into_iter()
|
||||
.filter_map(|d| normalize_domain(&d))
|
||||
.collect::<Vec<_>>();
|
||||
normalized.sort_unstable();
|
||||
normalized.dedup();
|
||||
normalized
|
||||
}
|
||||
|
||||
pub(super) fn normalize_domain(raw: &str) -> Option<String> {
|
||||
let mut d = raw.trim().to_lowercase();
|
||||
if d.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
if let Some(stripped) = d.strip_prefix("https://") {
|
||||
d = stripped.to_string();
|
||||
} else if let Some(stripped) = d.strip_prefix("http://") {
|
||||
d = stripped.to_string();
|
||||
}
|
||||
|
||||
if let Some((host, _)) = d.split_once('/') {
|
||||
d = host.to_string();
|
||||
}
|
||||
|
||||
d = d.trim_start_matches('.').trim_end_matches('.').to_string();
|
||||
|
||||
if let Some((host, _)) = d.split_once(':') {
|
||||
d = host.to_string();
|
||||
}
|
||||
|
||||
if d.is_empty() || d.chars().any(char::is_whitespace) {
|
||||
return None;
|
||||
}
|
||||
|
||||
Some(d)
|
||||
}
|
||||
|
||||
pub(super) fn extract_host(url: &str) -> anyhow::Result<String> {
|
||||
let rest = url
|
||||
.strip_prefix("http://")
|
||||
.or_else(|| url.strip_prefix("https://"))
|
||||
.ok_or_else(|| anyhow::anyhow!("Only http:// and https:// URLs are allowed"))?;
|
||||
|
||||
let authority = rest
|
||||
.split(['/', '?', '#'])
|
||||
.next()
|
||||
.ok_or_else(|| anyhow::anyhow!("Invalid URL"))?;
|
||||
|
||||
if authority.is_empty() {
|
||||
anyhow::bail!("URL must include a host");
|
||||
}
|
||||
|
||||
if authority.contains('@') {
|
||||
anyhow::bail!("URL userinfo is not allowed");
|
||||
}
|
||||
|
||||
if authority.starts_with('[') {
|
||||
anyhow::bail!("IPv6 hosts are not supported in http_request");
|
||||
}
|
||||
|
||||
let host = authority
|
||||
.split(':')
|
||||
.next()
|
||||
.unwrap_or_default()
|
||||
.trim()
|
||||
.trim_end_matches('.')
|
||||
.to_lowercase();
|
||||
|
||||
if host.is_empty() {
|
||||
anyhow::bail!("URL must include a valid host");
|
||||
}
|
||||
|
||||
Ok(host)
|
||||
}
|
||||
|
||||
pub(super) fn host_matches_allowlist(host: &str, allowed_domains: &[String]) -> bool {
|
||||
allowed_domains.iter().any(|domain| {
|
||||
host == domain
|
||||
|| host
|
||||
.strip_suffix(domain)
|
||||
.is_some_and(|prefix| prefix.ends_with('.'))
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn is_private_or_local_host(host: &str) -> bool {
|
||||
let bare = host
|
||||
.strip_prefix('[')
|
||||
.and_then(|h| h.strip_suffix(']'))
|
||||
.unwrap_or(host);
|
||||
|
||||
let has_local_tld = bare
|
||||
.rsplit('.')
|
||||
.next()
|
||||
.is_some_and(|label| label == "local");
|
||||
|
||||
if bare == "localhost" || bare.ends_with(".localhost") || has_local_tld {
|
||||
return true;
|
||||
}
|
||||
|
||||
if let Ok(ip) = bare.parse::<std::net::IpAddr>() {
|
||||
return match ip {
|
||||
std::net::IpAddr::V4(v4) => is_non_global_v4(v4),
|
||||
std::net::IpAddr::V6(v6) => is_non_global_v6(v6),
|
||||
};
|
||||
}
|
||||
|
||||
false
|
||||
}
|
||||
|
||||
fn is_non_global_v4(v4: std::net::Ipv4Addr) -> bool {
|
||||
let [a, b, c, _] = v4.octets();
|
||||
v4.is_loopback()
|
||||
|| v4.is_private()
|
||||
|| v4.is_link_local()
|
||||
|| v4.is_unspecified()
|
||||
|| v4.is_broadcast()
|
||||
|| v4.is_multicast()
|
||||
|| (a == 100 && (64..=127).contains(&b))
|
||||
|| a >= 240
|
||||
|| (a == 192 && b == 0 && (c == 0 || c == 2))
|
||||
|| (a == 198 && b == 51)
|
||||
|| (a == 203 && b == 0)
|
||||
|| (a == 198 && (18..=19).contains(&b))
|
||||
}
|
||||
|
||||
fn is_non_global_v6(v6: std::net::Ipv6Addr) -> bool {
|
||||
let segs = v6.segments();
|
||||
v6.is_loopback()
|
||||
|| v6.is_unspecified()
|
||||
|| v6.is_multicast()
|
||||
|| (segs[0] & 0xfe00) == 0xfc00
|
||||
|| (segs[0] & 0xffc0) == 0xfe80
|
||||
|| (segs[0] == 0x2001 && segs[1] == 0x0db8)
|
||||
|| v6.to_ipv4_mapped().is_some_and(is_non_global_v4)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn normalize_domain_strips_scheme_path_and_case() {
|
||||
let got = normalize_domain(" HTTPS://Docs.Example.com/path ").unwrap();
|
||||
assert_eq!(got, "docs.example.com");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normalize_allowed_domains_deduplicates() {
|
||||
let got = normalize_allowed_domains(vec![
|
||||
"example.com".into(),
|
||||
"EXAMPLE.COM".into(),
|
||||
"https://example.com/".into(),
|
||||
]);
|
||||
assert_eq!(got, vec!["example.com".to_string()]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_accepts_exact_domain() {
|
||||
let allow = vec!["example.com".to_string()];
|
||||
let got = validate_url("https://example.com/docs", &allow).unwrap();
|
||||
assert_eq!(got, "https://example.com/docs");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_accepts_http() {
|
||||
let allow = vec!["example.com".to_string()];
|
||||
assert!(validate_url("http://example.com", &allow).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_accepts_subdomain() {
|
||||
let allow = vec!["example.com".to_string()];
|
||||
assert!(validate_url("https://api.example.com/v1", &allow).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_rejects_allowlist_miss() {
|
||||
let allow = vec!["example.com".to_string()];
|
||||
let err = validate_url("https://google.com", &allow)
|
||||
.unwrap_err()
|
||||
.to_string();
|
||||
assert!(err.contains("allowed_domains"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_rejects_localhost() {
|
||||
let allow = vec!["localhost".to_string()];
|
||||
let err = validate_url("https://localhost:8080", &allow)
|
||||
.unwrap_err()
|
||||
.to_string();
|
||||
assert!(err.contains("local/private"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_rejects_private_ipv4() {
|
||||
let allow = vec!["192.168.1.5".to_string()];
|
||||
let err = validate_url("https://192.168.1.5", &allow)
|
||||
.unwrap_err()
|
||||
.to_string();
|
||||
assert!(err.contains("local/private"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_rejects_whitespace() {
|
||||
let allow = vec!["example.com".to_string()];
|
||||
let err = validate_url("https://example.com/hello world", &allow)
|
||||
.unwrap_err()
|
||||
.to_string();
|
||||
assert!(err.contains("whitespace"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_rejects_userinfo() {
|
||||
let allow = vec!["example.com".to_string()];
|
||||
let err = validate_url("https://user@example.com", &allow)
|
||||
.unwrap_err()
|
||||
.to_string();
|
||||
assert!(err.contains("userinfo"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_requires_allowlist() {
|
||||
let err = validate_url("https://example.com", &[])
|
||||
.unwrap_err()
|
||||
.to_string();
|
||||
assert!(err.contains("allowed_domains"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_rejects_ftp_scheme() {
|
||||
let allow = vec!["example.com".to_string()];
|
||||
let err = validate_url("ftp://example.com", &allow)
|
||||
.unwrap_err()
|
||||
.to_string();
|
||||
assert!(err.contains("http://") || err.contains("https://"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_rejects_empty_url() {
|
||||
let allow = vec!["example.com".to_string()];
|
||||
let err = validate_url("", &allow).unwrap_err().to_string();
|
||||
assert!(err.contains("empty"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_rejects_ipv6_host() {
|
||||
let allow = vec!["example.com".to_string()];
|
||||
let err = validate_url("http://[::1]:8080/path", &allow)
|
||||
.unwrap_err()
|
||||
.to_string();
|
||||
assert!(err.contains("IPv6"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn blocks_multicast_ipv4() {
|
||||
assert!(is_private_or_local_host("224.0.0.1"));
|
||||
assert!(is_private_or_local_host("239.255.255.255"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn blocks_broadcast() {
|
||||
assert!(is_private_or_local_host("255.255.255.255"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn blocks_reserved_ipv4() {
|
||||
assert!(is_private_or_local_host("240.0.0.1"));
|
||||
assert!(is_private_or_local_host("250.1.2.3"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn blocks_documentation_ranges() {
|
||||
assert!(is_private_or_local_host("192.0.2.1"));
|
||||
assert!(is_private_or_local_host("198.51.100.1"));
|
||||
assert!(is_private_or_local_host("203.0.113.1"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn blocks_benchmarking_range() {
|
||||
assert!(is_private_or_local_host("198.18.0.1"));
|
||||
assert!(is_private_or_local_host("198.19.255.255"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn blocks_ipv6_localhost() {
|
||||
assert!(is_private_or_local_host("::1"));
|
||||
assert!(is_private_or_local_host("[::1]"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn blocks_ipv6_multicast() {
|
||||
assert!(is_private_or_local_host("ff02::1"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn blocks_ipv6_link_local() {
|
||||
assert!(is_private_or_local_host("fe80::1"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn blocks_ipv6_unique_local() {
|
||||
assert!(is_private_or_local_host("fd00::1"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn blocks_ipv4_mapped_ipv6() {
|
||||
assert!(is_private_or_local_host("::ffff:127.0.0.1"));
|
||||
assert!(is_private_or_local_host("::ffff:192.168.1.1"));
|
||||
assert!(is_private_or_local_host("::ffff:10.0.0.1"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn allows_public_ipv4() {
|
||||
assert!(!is_private_or_local_host("8.8.8.8"));
|
||||
assert!(!is_private_or_local_host("1.1.1.1"));
|
||||
assert!(!is_private_or_local_host("93.184.216.34"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn blocks_ipv6_documentation_range() {
|
||||
assert!(is_private_or_local_host("2001:db8::1"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn allows_public_ipv6() {
|
||||
assert!(!is_private_or_local_host("2607:f8b0:4004:800::200e"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn blocks_shared_address_space() {
|
||||
assert!(is_private_or_local_host("100.64.0.1"));
|
||||
assert!(is_private_or_local_host("100.127.255.255"));
|
||||
assert!(!is_private_or_local_host("100.63.0.1"));
|
||||
assert!(!is_private_or_local_host("100.128.0.1"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ssrf_blocks_loopback_127_range() {
|
||||
assert!(is_private_or_local_host("127.0.0.1"));
|
||||
assert!(is_private_or_local_host("127.0.0.2"));
|
||||
assert!(is_private_or_local_host("127.255.255.255"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ssrf_blocks_rfc1918_10_range() {
|
||||
assert!(is_private_or_local_host("10.0.0.1"));
|
||||
assert!(is_private_or_local_host("10.255.255.255"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ssrf_blocks_rfc1918_172_range() {
|
||||
assert!(is_private_or_local_host("172.16.0.1"));
|
||||
assert!(is_private_or_local_host("172.31.255.255"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ssrf_blocks_unspecified_address() {
|
||||
assert!(is_private_or_local_host("0.0.0.0"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ssrf_blocks_dot_localhost_subdomain() {
|
||||
assert!(is_private_or_local_host("evil.localhost"));
|
||||
assert!(is_private_or_local_host("a.b.localhost"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ssrf_blocks_dot_local_tld() {
|
||||
assert!(is_private_or_local_host("service.local"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ssrf_ipv6_unspecified() {
|
||||
assert!(is_private_or_local_host("::"));
|
||||
}
|
||||
|
||||
// ── Defense-in-depth: alternate IP notations rejected by allowlist
|
||||
//
|
||||
// Rust's IpAddr::parse() rejects octal, hex, decimal, and
|
||||
// zero-padded notations. They fall through as hostnames and get
|
||||
// rejected by the allowlist instead. These tests pin that
|
||||
// behaviour so a parser change can't silently re-open SSRF.
|
||||
|
||||
#[test]
|
||||
fn ssrf_octal_loopback_not_parsed_as_ip() {
|
||||
assert!(!is_private_or_local_host("0177.0.0.1"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ssrf_hex_loopback_not_parsed_as_ip() {
|
||||
assert!(!is_private_or_local_host("0x7f000001"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ssrf_decimal_loopback_not_parsed_as_ip() {
|
||||
assert!(!is_private_or_local_host("2130706433"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ssrf_zero_padded_loopback_not_parsed_as_ip() {
|
||||
assert!(!is_private_or_local_host("127.000.000.001"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ssrf_alternate_notations_rejected_by_validate_url() {
|
||||
let allow = vec!["example.com".to_string()];
|
||||
for notation in [
|
||||
"http://0177.0.0.1",
|
||||
"http://0x7f000001",
|
||||
"http://2130706433",
|
||||
"http://127.000.000.001",
|
||||
] {
|
||||
let err = validate_url(notation, &allow).unwrap_err().to_string();
|
||||
assert!(
|
||||
err.contains("allowed_domains"),
|
||||
"Expected allowlist rejection for {notation}, got: {err}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -169,6 +169,32 @@ pub fn all_tools_with_runtime(
|
||||
http_config.timeout_secs,
|
||||
)));
|
||||
|
||||
// curl — always registered. Shares `http_request.allowed_domains`,
|
||||
// adds streaming-to-disk with a hard byte ceiling. Writes land
|
||||
// under `<workspace>/<curl.dest_subdir>`.
|
||||
tools.push(Box::new(CurlTool::new(
|
||||
security.clone(),
|
||||
http_config.allowed_domains.clone(),
|
||||
workspace_dir.to_path_buf(),
|
||||
root_config.curl.dest_subdir.clone(),
|
||||
root_config.curl.max_download_bytes,
|
||||
root_config.curl.timeout_secs,
|
||||
)));
|
||||
|
||||
// gitbooks — answers questions about OpenHuman by calling the
|
||||
// GitBook MCP server. Two tools mirroring the upstream MCP tools.
|
||||
if root_config.gitbooks.enabled {
|
||||
tools.push(Box::new(GitbooksSearchTool::new(
|
||||
root_config.gitbooks.endpoint.clone(),
|
||||
root_config.gitbooks.timeout_secs,
|
||||
)));
|
||||
tools.push(Box::new(GitbooksGetPageTool::new(
|
||||
root_config.gitbooks.endpoint.clone(),
|
||||
root_config.gitbooks.timeout_secs,
|
||||
)));
|
||||
tracing::debug!("[gitbooks] registered gitbooks_search + gitbooks_get_page");
|
||||
}
|
||||
|
||||
// Web search — always registered. Result/timeout budget
|
||||
// knobs still come from `config.web_search`, but there is no
|
||||
// enable flag: every session needs research as a baseline
|
||||
@@ -372,6 +398,114 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn all_tools_always_registers_curl() {
|
||||
// Regression guard: `curl` is always registered (gated only by
|
||||
// the shared `http_request.allowed_domains` allowlist at call
|
||||
// time, like `http_request`). `Write` permission level keeps it
|
||||
// off agents that aren't allowed to modify the workspace.
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let security = Arc::new(SecurityPolicy::default());
|
||||
let mem_cfg = MemoryConfig {
|
||||
backend: "markdown".into(),
|
||||
..MemoryConfig::default()
|
||||
};
|
||||
let mem: Arc<dyn Memory> =
|
||||
Arc::from(crate::openhuman::memory::create_memory(&mem_cfg, tmp.path()).unwrap());
|
||||
|
||||
let browser = BrowserConfig::default();
|
||||
let http = crate::openhuman::config::HttpRequestConfig::default();
|
||||
let cfg = test_config(&tmp);
|
||||
|
||||
let tools = all_tools(
|
||||
Arc::new(cfg.clone()),
|
||||
&security,
|
||||
mem,
|
||||
&browser,
|
||||
&http,
|
||||
tmp.path(),
|
||||
&HashMap::new(),
|
||||
&cfg,
|
||||
);
|
||||
let names: Vec<&str> = tools.iter().map(|t| t.name()).collect();
|
||||
assert!(
|
||||
names.contains(&"curl"),
|
||||
"curl must always be registered; got: {names:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn all_tools_registers_gitbooks_when_enabled() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let security = Arc::new(SecurityPolicy::default());
|
||||
let mem_cfg = MemoryConfig {
|
||||
backend: "markdown".into(),
|
||||
..MemoryConfig::default()
|
||||
};
|
||||
let mem: Arc<dyn Memory> =
|
||||
Arc::from(crate::openhuman::memory::create_memory(&mem_cfg, tmp.path()).unwrap());
|
||||
let browser = BrowserConfig::default();
|
||||
let http = crate::openhuman::config::HttpRequestConfig::default();
|
||||
let mut cfg = test_config(&tmp);
|
||||
cfg.gitbooks.enabled = true;
|
||||
|
||||
let tools = all_tools(
|
||||
Arc::new(cfg.clone()),
|
||||
&security,
|
||||
mem,
|
||||
&browser,
|
||||
&http,
|
||||
tmp.path(),
|
||||
&HashMap::new(),
|
||||
&cfg,
|
||||
);
|
||||
let names: Vec<&str> = tools.iter().map(|t| t.name()).collect();
|
||||
assert!(
|
||||
names.contains(&"gitbooks_search"),
|
||||
"gitbooks_search must register when gitbooks.enabled = true; got: {names:?}"
|
||||
);
|
||||
assert!(
|
||||
names.contains(&"gitbooks_get_page"),
|
||||
"gitbooks_get_page must register when gitbooks.enabled = true; got: {names:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn all_tools_skips_gitbooks_when_disabled() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let security = Arc::new(SecurityPolicy::default());
|
||||
let mem_cfg = MemoryConfig {
|
||||
backend: "markdown".into(),
|
||||
..MemoryConfig::default()
|
||||
};
|
||||
let mem: Arc<dyn Memory> =
|
||||
Arc::from(crate::openhuman::memory::create_memory(&mem_cfg, tmp.path()).unwrap());
|
||||
let browser = BrowserConfig::default();
|
||||
let http = crate::openhuman::config::HttpRequestConfig::default();
|
||||
let mut cfg = test_config(&tmp);
|
||||
cfg.gitbooks.enabled = false;
|
||||
|
||||
let tools = all_tools(
|
||||
Arc::new(cfg.clone()),
|
||||
&security,
|
||||
mem,
|
||||
&browser,
|
||||
&http,
|
||||
tmp.path(),
|
||||
&HashMap::new(),
|
||||
&cfg,
|
||||
);
|
||||
let names: Vec<&str> = tools.iter().map(|t| t.name()).collect();
|
||||
assert!(
|
||||
!names.contains(&"gitbooks_search"),
|
||||
"gitbooks_search must NOT register when gitbooks.enabled = false; got: {names:?}"
|
||||
);
|
||||
assert!(
|
||||
!names.contains(&"gitbooks_get_page"),
|
||||
"gitbooks_get_page must NOT register when gitbooks.enabled = false; got: {names:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn all_tools_includes_complete_onboarding() {
|
||||
// Regression guard: the `complete_onboarding` tool must be
|
||||
|
||||
Reference in New Issue
Block a user