diff --git a/crates/openfang-runtime/src/web_search.rs b/crates/openfang-runtime/src/web_search.rs index 66fa2fdf..28e92259 100644 --- a/crates/openfang-runtime/src/web_search.rs +++ b/crates/openfang-runtime/src/web_search.rs @@ -55,6 +55,7 @@ impl WebSearchEngine { SearchProvider::Tavily => self.search_tavily(query, max_results).await, SearchProvider::Perplexity => self.search_perplexity(query).await, SearchProvider::DuckDuckGo => self.search_duckduckgo(query, max_results).await, + SearchProvider::Searxng => self.search_searxng(query, max_results, None, 1).await, SearchProvider::Auto => self.search_auto(query, max_results).await, }; @@ -67,7 +68,7 @@ impl WebSearchEngine { } /// Auto-select provider based on available API keys. - /// Priority: Tavily → Brave → Perplexity → DuckDuckGo + /// Priority: Tavily → Brave → Perplexity → Searxng → DuckDuckGo async fn search_auto(&self, query: &str, max_results: usize) -> Result { // Tavily first (AI-agent-native) if resolve_api_key(&self.config.tavily.api_key_env).is_some() { @@ -96,6 +97,15 @@ impl WebSearchEngine { } } + // Searxng fourth (self-hosted, no API key needed) + if !self.config.searxng.url.is_empty() { + debug!("Auto: trying Searxng"); + match self.search_searxng(query, max_results, None, 1).await { + Ok(result) => return Ok(result), + Err(e) => warn!("Searxng failed, falling back: {e}"), + } + } + // DuckDuckGo always available as zero-config fallback debug!("Auto: falling back to DuckDuckGo"); self.search_duckduckgo(query, max_results).await @@ -313,6 +323,162 @@ impl WebSearchEngine { Ok(output) } + + /// Search via SearXNG self-hosted instance. + async fn search_searxng( + &self, + query: &str, + max_results: usize, + category: Option<&str>, + page: u32, + ) -> Result { + if self.config.searxng.url.is_empty() { + return Err("SearXNG URL is not configured".to_string()); + } + + let category = category.unwrap_or("general"); + + // Validate category against SearXNG instance + match self.list_searxng_categories().await { + Ok(cats) => { + if !cats.iter().any(|c| c == category) { + return Err(format!( + "Invalid SearXNG category '{}'. Available: {}", + category, + cats.join(", ") + )); + } + } + Err(e) => warn!("Could not validate SearXNG category: {e}"), + } + + let limit = max_results; + + debug!(query, "Searching via SearXNG"); + + let resp = self + .client + .get(format!("{}/search", self.config.searxng.url.trim_end_matches('/'))) + .query(&[ + ("q", query), + ("format", "json"), + ("categories", category), + ("page", &page.to_string()), + ]) + .header("User-Agent", "Mozilla/5.0 (compatible; OpenFangAgent/0.1)") + .send() + .await + .map_err(|e| format!("SearXNG request failed: {e}"))?; + + if !resp.status().is_success() { + return Err(format!("SearXNG API returned {}", resp.status())); + } + + #[derive(serde::Deserialize)] + struct SearxngResponse { + results: Vec, + query: String, + } + + #[derive(serde::Deserialize)] + struct SearxngResult { + url: String, + title: String, + content: Option, + #[serde(alias = "pubdate")] + published_date: Option, + } + + let data: SearxngResponse = resp + .json() + .await + .map_err(|e| format!("SearXNG JSON parse failed: {e}"))?; + + if data.results.is_empty() { + return Err(format!("No results found for '{query}' (SearXNG).")); + } + + #[derive(serde::Serialize)] + struct OutputResult<'a> { + title: &'a str, + url: &'a str, + content: &'a str, + #[serde(skip_serializing_if = "Option::is_none")] + published_date: Option<&'a str>, + } + + #[derive(serde::Serialize)] + struct Output<'a> { + query: &'a str, + results: Vec>, + } + + let results: Vec = data + .results + .iter() + .take(limit) + .map(|r| OutputResult { + title: &r.title, + url: &r.url, + content: r.content.as_deref().unwrap_or(""), + published_date: r.published_date.as_deref(), + }) + .collect(); + + let output = Output { + query: &data.query, + results, + }; + + serde_json::to_string(&output) + .map_err(|e| format!("Failed to serialize SearXNG results: {e}")) + } + + /// List available search categories from the SearXNG instance. + /// + /// Fetches the `/config` endpoint and returns the list of categories + /// the instance supports (e.g., "general", "images", "news", "videos"). + /// Returns an error if SearXNG URL is not configured or the request fails. + pub async fn list_searxng_categories(&self) -> Result, String> { + if self.config.searxng.url.is_empty() { + return Err("SearXNG URL is not configured".to_string()); + } + + #[derive(serde::Deserialize)] + struct SearxngConfig { + categories: Vec, + } + + let resp = self + .client + .get(format!("{}/config", self.config.searxng.url.trim_end_matches('/'))) + .header("User-Agent", "Mozilla/5.0 (compatible; OpenFangAgent/0.1)") + .send() + .await + .map_err(|e| format!("SearXNG config request failed: {e}"))?; + + if !resp.status().is_success() { + return Err(format!("SearXNG config API returned {}", resp.status())); + } + + let data: SearxngConfig = resp + .json() + .await + .map_err(|e| format!("SearXNG config JSON parse failed: {e}"))?; + + Ok(data.categories) + } + + /// Return all non-Auto search providers (for provider listing/discovery). + pub fn all_providers() -> Vec { + vec![ + SearchProvider::Brave, + SearchProvider::Tavily, + SearchProvider::Perplexity, + SearchProvider::DuckDuckGo, + SearchProvider::Searxng, + ] + } } // --------------------------------------------------------------------------- diff --git a/crates/openfang-skills/bundled/searxng/SKILL.md b/crates/openfang-skills/bundled/searxng/SKILL.md new file mode 100644 index 00000000..0f5d0602 --- /dev/null +++ b/crates/openfang-skills/bundled/searxng/SKILL.md @@ -0,0 +1,70 @@ +--- +name: searxng +description: Privacy-respecting metasearch specialist using SearXNG instances +--- +# SearXNG Search Specialist + +You are a privacy-respecting web search specialist using SearXNG, a self-hosted metasearch engine that aggregates results from multiple search engines without tracking. + +## Key Principles + +- Prefer SearXNG for privacy-sensitive searches — no API keys, no tracking, no user profiling. +- Always cite sources with URLs so the user can verify information. +- Prefer primary sources (official docs, research papers) over secondary ones (blog posts, forums). +- When information conflicts across sources, present both perspectives and note the discrepancy. +- State the date of information when recency matters. + +## SearXNG Capabilities + +SearXNG supports 30+ search categories. Use the right category for the task: + +| Category | Use Case | +|----------|----------| +| `general` | Default web search | +| `images` | Image search | +| `news` | News articles | +| `videos` | Video results | +| `music` | Music and audio | +| `files` | File search | +| `it` | IT and programming | +| `science` | Scientific content | +| `books` | Book search | +| `maps` | Map and location | +| `q&a` | Q&A sites (Stack Overflow, etc.) | +| `social media` | Social media posts | +| `wikimedia` | Wikipedia and Wikimedia | +| `dictionaries` | Dictionary definitions | +| `currency` | Currency conversion | +| `weather` | Weather information | +| `translate` | Translation results | + +## Search Techniques + +- **Category selection**: Always specify a category when the topic is clear. Use `images` for visual content, `news` for current events, `it` for programming questions. +- **Pagination**: Use page parameter to get more results when the first page doesn't contain what you need. +- **Engine syntax**: SearXNG supports `!engine` syntax to target specific engines (e.g., `!wikipedia rust programming`). +- **Site search**: Use `site:example.com` in queries to search within a specific domain. +- **Exact phrases**: Use quotes for exact phrase matching (e.g., `"rust borrow checker"`). +- **Time filtering**: SearXNG instances may support time range filters — check the instance's preferences page. + +## Query Formulation + +- Start with specific, targeted queries. Use exact phrases for precise matches. +- Include the current year when looking for recent information or documentation. +- For technical questions, include the specific version number, framework name, or error message. +- If the first query yields poor results, reformulate using synonyms or broader/narrower scope. + +## Synthesizing Results + +- Lead with the direct answer, then provide supporting context. +- Organize findings by relevance, not by the order you found them. +- Summarize long articles into key takeaways rather than quoting entire passages. +- When comparing options, use structured comparisons with pros and cons. +- Flag information that may be outdated or from unreliable sources. + +## Pitfalls to Avoid + +- Never present information from a single source as definitive without corroboration. +- Do not include URLs you have not verified — broken links erode trust. +- Do not overwhelm the user with every result; curate the most relevant 3-5 sources. +- Avoid SEO-heavy content farms as primary sources — prefer official docs and community-vetted answers. diff --git a/crates/openfang-skills/src/bundled.rs b/crates/openfang-skills/src/bundled.rs index 3b29f6e6..203c5a41 100644 --- a/crates/openfang-skills/src/bundled.rs +++ b/crates/openfang-skills/src/bundled.rs @@ -13,6 +13,7 @@ pub fn bundled_skills() -> Vec<(&'static str, &'static str)> { ("github", include_str!("../bundled/github/SKILL.md")), ("docker", include_str!("../bundled/docker/SKILL.md")), ("web-search", include_str!("../bundled/web-search/SKILL.md")), + ("searxng", include_str!("../bundled/searxng/SKILL.md")), ( "code-reviewer", include_str!("../bundled/code-reviewer/SKILL.md"), @@ -195,7 +196,7 @@ mod tests { #[test] fn test_bundled_skills_count() { let skills = bundled_skills(); - assert_eq!(skills.len(), 60, "Expected 60 bundled skills"); + assert_eq!(skills.len(), 61, "Expected 61 bundled skills"); } #[test] diff --git a/crates/openfang-types/src/config.rs b/crates/openfang-types/src/config.rs index 6bbe3342..49afb074 100644 --- a/crates/openfang-types/src/config.rs +++ b/crates/openfang-types/src/config.rs @@ -173,7 +173,9 @@ pub enum SearchProvider { Perplexity, /// DuckDuckGo HTML (no API key needed). DuckDuckGo, - /// Auto-select based on available API keys (Tavily → Brave → Perplexity → DuckDuckGo). + /// SearXNG self-hosted search (no API key needed). + Searxng, + /// Auto-select based on available API keys (Tavily → Brave → Perplexity → Searxng → DuckDuckGo). #[default] Auto, } @@ -192,6 +194,8 @@ pub struct WebConfig { pub tavily: TavilySearchConfig, /// Perplexity Search configuration. pub perplexity: PerplexitySearchConfig, + /// SearXNG Search configuration. + pub searxng: SearxngSearchConfig, /// Web fetch configuration. pub fetch: WebFetchConfig, } @@ -204,6 +208,7 @@ impl Default for WebConfig { brave: BraveSearchConfig::default(), tavily: TavilySearchConfig::default(), perplexity: PerplexitySearchConfig::default(), + searxng: SearxngSearchConfig::default(), fetch: WebFetchConfig::default(), } } @@ -281,6 +286,14 @@ impl Default for PerplexitySearchConfig { } } +/// SearXNG self-hosted search configuration. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +#[serde(default)] +pub struct SearxngSearchConfig { + /// Base URL of the SearXNG instance (e.g., "https://search.example.com"). + pub url: String, +} + /// Web fetch configuration. #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(default)] @@ -3598,6 +3611,13 @@ impl KernelConfig { )); } } + SearchProvider::Searxng => { + if self.web.searxng.url.is_empty() { + warnings.push( + "Searxng search selected but searxng.url is not configured".to_string(), + ); + } + } SearchProvider::DuckDuckGo | SearchProvider::Auto => {} } diff --git a/docs/configuration.md b/docs/configuration.md index f0ae8f2d..99e488ff 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -337,10 +337,11 @@ cache_ttl_minutes = 15 | Value | Description | |-------|-------------| -| `auto` | Cascading fallback: tries Tavily, then Brave, then Perplexity, then DuckDuckGo, based on which API keys are available. | +| `auto` | Cascading fallback: tries Tavily, then Brave, then Perplexity, then SearXNG, then DuckDuckGo, based on which API keys/configs are available. | | `brave` | Brave Search API. Requires `BRAVE_API_KEY`. | | `tavily` | Tavily AI-native search. Requires `TAVILY_API_KEY`. | | `perplexity` | Perplexity AI search. Requires `PERPLEXITY_API_KEY`. | +| `searxng` | Self-hosted search engine aggregator. No API key required, just point to your SearXNG instance. | | `duck_duck_go` | DuckDuckGo HTML scraping. No API key needed. | #### `[web.brave]` @@ -392,6 +393,19 @@ model = "sonar" | `api_key_env` | string | `"PERPLEXITY_API_KEY"` | Environment variable name holding the Perplexity API key. | | `model` | string | `"sonar"` | Perplexity model to use for search queries. | +#### `[web.searxng]` + +**SearXNG** — Self-hosted search engine aggregator. No API key required, just point to your SearXNG instance. Supports 30+ search categories (general, images, news, videos, etc.) and pagination. + +```toml +[web.searxng] +url = "https://searxng.example.com" # SearXNG instance URL (required) +``` + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `url` | string | (required) | Full URL of your SearXNG instance (e.g., `https://searxng.example.com`). Must be accessible. | + #### `[web.fetch]` ```toml