Merge pull request #920 from norci/add-searxng-search-provider

feat: add SearXNG search provider with custom URL and JSON output
2026-07-30 15:01:15 +00:00 · 2026-03-31 02:21:36 +03:00
parent 618e83714c 46eac44635
commit 545e710abb
5 changed files with 275 additions and 4 deletions
@@ -55,6 +55,7 @@ impl WebSearchEngine {
            SearchProvider::Tavily => self.search_tavily(query, max_results).await,
            SearchProvider::Perplexity => self.search_perplexity(query).await,
            SearchProvider::DuckDuckGo => self.search_duckduckgo(query, max_results).await,
+            SearchProvider::Searxng => self.search_searxng(query, max_results, None, 1).await,
            SearchProvider::Auto => self.search_auto(query, max_results).await,
        };

@@ -67,7 +68,7 @@ impl WebSearchEngine {
    }

    /// Auto-select provider based on available API keys.
-    /// Priority: Tavily → Brave → Perplexity → DuckDuckGo
+    /// Priority: Tavily → Brave → Perplexity → Searxng → DuckDuckGo
    async fn search_auto(&self, query: &str, max_results: usize) -> Result<String, String> {
        // Tavily first (AI-agent-native)
        if resolve_api_key(&self.config.tavily.api_key_env).is_some() {
@@ -96,6 +97,15 @@ impl WebSearchEngine {
            }
        }

+        // Searxng fourth (self-hosted, no API key needed)
+        if !self.config.searxng.url.is_empty() {
+            debug!("Auto: trying Searxng");
+            match self.search_searxng(query, max_results, None, 1).await {
+                Ok(result) => return Ok(result),
+                Err(e) => warn!("Searxng failed, falling back: {e}"),
+            }
+        }
+
        // DuckDuckGo always available as zero-config fallback
        debug!("Auto: falling back to DuckDuckGo");
        self.search_duckduckgo(query, max_results).await
@@ -313,6 +323,162 @@ impl WebSearchEngine {

        Ok(output)
    }
+
+    /// Search via SearXNG self-hosted instance.
+    async fn search_searxng(
+        &self,
+        query: &str,
+        max_results: usize,
+        category: Option<&str>,
+        page: u32,
+    ) -> Result<String, String> {
+        if self.config.searxng.url.is_empty() {
+            return Err("SearXNG URL is not configured".to_string());
+        }
+
+        let category = category.unwrap_or("general");
+
+        // Validate category against SearXNG instance
+        match self.list_searxng_categories().await {
+            Ok(cats) => {
+                if !cats.iter().any(|c| c == category) {
+                    return Err(format!(
+                        "Invalid SearXNG category '{}'. Available: {}",
+                        category,
+                        cats.join(", ")
+                    ));
+                }
+            }
+            Err(e) => warn!("Could not validate SearXNG category: {e}"),
+        }
+
+        let limit = max_results;
+
+        debug!(query, "Searching via SearXNG");
+
+        let resp = self
+            .client
+            .get(format!("{}/search", self.config.searxng.url.trim_end_matches('/')))
+            .query(&[
+                ("q", query),
+                ("format", "json"),
+                ("categories", category),
+                ("page", &page.to_string()),
+            ])
+            .header("User-Agent", "Mozilla/5.0 (compatible; OpenFangAgent/0.1)")
+            .send()
+            .await
+            .map_err(|e| format!("SearXNG request failed: {e}"))?;
+
+        if !resp.status().is_success() {
+            return Err(format!("SearXNG API returned {}", resp.status()));
+        }
+
+        #[derive(serde::Deserialize)]
+        struct SearxngResponse {
+            results: Vec<SearxngResult>,
+            query: String,
+        }
+
+        #[derive(serde::Deserialize)]
+        struct SearxngResult {
+            url: String,
+            title: String,
+            content: Option<String>,
+            #[serde(alias = "pubdate")]
+            published_date: Option<String>,
+        }
+
+        let data: SearxngResponse = resp
+            .json()
+            .await
+            .map_err(|e| format!("SearXNG JSON parse failed: {e}"))?;
+
+        if data.results.is_empty() {
+            return Err(format!("No results found for '{query}' (SearXNG)."));
+        }
+
+        #[derive(serde::Serialize)]
+        struct OutputResult<'a> {
+            title: &'a str,
+            url: &'a str,
+            content: &'a str,
+            #[serde(skip_serializing_if = "Option::is_none")]
+            published_date: Option<&'a str>,
+        }
+
+        #[derive(serde::Serialize)]
+        struct Output<'a> {
+            query: &'a str,
+            results: Vec<OutputResult<'a>>,
+        }
+
+        let results: Vec<OutputResult> = data
+            .results
+            .iter()
+            .take(limit)
+            .map(|r| OutputResult {
+                title: &r.title,
+                url: &r.url,
+                content: r.content.as_deref().unwrap_or(""),
+                published_date: r.published_date.as_deref(),
+            })
+            .collect();
+
+        let output = Output {
+            query: &data.query,
+            results,
+        };
+
+        serde_json::to_string(&output)
+            .map_err(|e| format!("Failed to serialize SearXNG results: {e}"))
+    }
+
+    /// List available search categories from the SearXNG instance.
+    ///
+    /// Fetches the `/config` endpoint and returns the list of categories
+    /// the instance supports (e.g., "general", "images", "news", "videos").
+    /// Returns an error if SearXNG URL is not configured or the request fails.
+    pub async fn list_searxng_categories(&self) -> Result<Vec<String>, String> {
+        if self.config.searxng.url.is_empty() {
+            return Err("SearXNG URL is not configured".to_string());
+        }
+
+        #[derive(serde::Deserialize)]
+        struct SearxngConfig {
+            categories: Vec<String>,
+        }
+
+        let resp = self
+            .client
+            .get(format!("{}/config", self.config.searxng.url.trim_end_matches('/')))
+            .header("User-Agent", "Mozilla/5.0 (compatible; OpenFangAgent/0.1)")
+            .send()
+            .await
+            .map_err(|e| format!("SearXNG config request failed: {e}"))?;
+
+        if !resp.status().is_success() {
+            return Err(format!("SearXNG config API returned {}", resp.status()));
+        }
+
+        let data: SearxngConfig = resp
+            .json()
+            .await
+            .map_err(|e| format!("SearXNG config JSON parse failed: {e}"))?;
+
+        Ok(data.categories)
+    }
+
+    /// Return all non-Auto search providers (for provider listing/discovery).
+    pub fn all_providers() -> Vec<SearchProvider> {
+        vec![
+            SearchProvider::Brave,
+            SearchProvider::Tavily,
+            SearchProvider::Perplexity,
+            SearchProvider::DuckDuckGo,
+            SearchProvider::Searxng,
+        ]
+    }
 }

 // ---------------------------------------------------------------------------
@@ -0,0 +1,70 @@
+---
+name: searxng
+description: Privacy-respecting metasearch specialist using SearXNG instances
+---
+# SearXNG Search Specialist
+
+You are a privacy-respecting web search specialist using SearXNG, a self-hosted metasearch engine that aggregates results from multiple search engines without tracking.
+
+## Key Principles
+
+- Prefer SearXNG for privacy-sensitive searches — no API keys, no tracking, no user profiling.
+- Always cite sources with URLs so the user can verify information.
+- Prefer primary sources (official docs, research papers) over secondary ones (blog posts, forums).
+- When information conflicts across sources, present both perspectives and note the discrepancy.
+- State the date of information when recency matters.
+
+## SearXNG Capabilities
+
+SearXNG supports 30+ search categories. Use the right category for the task:
+
+| Category | Use Case |
+|----------|----------|
+| `general` | Default web search |
+| `images` | Image search |
+| `news` | News articles |
+| `videos` | Video results |
+| `music` | Music and audio |
+| `files` | File search |
+| `it` | IT and programming |
+| `science` | Scientific content |
+| `books` | Book search |
+| `maps` | Map and location |
+| `q&a` | Q&A sites (Stack Overflow, etc.) |
+| `social media` | Social media posts |
+| `wikimedia` | Wikipedia and Wikimedia |
+| `dictionaries` | Dictionary definitions |
+| `currency` | Currency conversion |
+| `weather` | Weather information |
+| `translate` | Translation results |
+
+## Search Techniques
+
+- **Category selection**: Always specify a category when the topic is clear. Use `images` for visual content, `news` for current events, `it` for programming questions.
+- **Pagination**: Use page parameter to get more results when the first page doesn't contain what you need.
+- **Engine syntax**: SearXNG supports `!engine` syntax to target specific engines (e.g., `!wikipedia rust programming`).
+- **Site search**: Use `site:example.com` in queries to search within a specific domain.
+- **Exact phrases**: Use quotes for exact phrase matching (e.g., `"rust borrow checker"`).
+- **Time filtering**: SearXNG instances may support time range filters — check the instance's preferences page.
+
+## Query Formulation
+
+- Start with specific, targeted queries. Use exact phrases for precise matches.
+- Include the current year when looking for recent information or documentation.
+- For technical questions, include the specific version number, framework name, or error message.
+- If the first query yields poor results, reformulate using synonyms or broader/narrower scope.
+
+## Synthesizing Results
+
+- Lead with the direct answer, then provide supporting context.
+- Organize findings by relevance, not by the order you found them.
+- Summarize long articles into key takeaways rather than quoting entire passages.
+- When comparing options, use structured comparisons with pros and cons.
+- Flag information that may be outdated or from unreliable sources.
+
+## Pitfalls to Avoid
+
+- Never present information from a single source as definitive without corroboration.
+- Do not include URLs you have not verified — broken links erode trust.
+- Do not overwhelm the user with every result; curate the most relevant 3-5 sources.
+- Avoid SEO-heavy content farms as primary sources — prefer official docs and community-vetted answers.
@@ -13,6 +13,7 @@ pub fn bundled_skills() -> Vec<(&'static str, &'static str)> {
        ("github", include_str!("../bundled/github/SKILL.md")),
        ("docker", include_str!("../bundled/docker/SKILL.md")),
        ("web-search", include_str!("../bundled/web-search/SKILL.md")),
+        ("searxng", include_str!("../bundled/searxng/SKILL.md")),
        (
            "code-reviewer",
            include_str!("../bundled/code-reviewer/SKILL.md"),
@@ -195,7 +196,7 @@ mod tests {
    #[test]
    fn test_bundled_skills_count() {
        let skills = bundled_skills();
-        assert_eq!(skills.len(), 60, "Expected 60 bundled skills");
+        assert_eq!(skills.len(), 61, "Expected 61 bundled skills");
    }

    #[test]
@@ -173,7 +173,9 @@ pub enum SearchProvider {
    Perplexity,
    /// DuckDuckGo HTML (no API key needed).
    DuckDuckGo,
-    /// Auto-select based on available API keys (Tavily → Brave → Perplexity → DuckDuckGo).
+    /// SearXNG self-hosted search (no API key needed).
+    Searxng,
+    /// Auto-select based on available API keys (Tavily → Brave → Perplexity → Searxng → DuckDuckGo).
    #[default]
    Auto,
 }
@@ -192,6 +194,8 @@ pub struct WebConfig {
    pub tavily: TavilySearchConfig,
    /// Perplexity Search configuration.
    pub perplexity: PerplexitySearchConfig,
+    /// SearXNG Search configuration.
+    pub searxng: SearxngSearchConfig,
    /// Web fetch configuration.
    pub fetch: WebFetchConfig,
 }
@@ -204,6 +208,7 @@ impl Default for WebConfig {
            brave: BraveSearchConfig::default(),
            tavily: TavilySearchConfig::default(),
            perplexity: PerplexitySearchConfig::default(),
+            searxng: SearxngSearchConfig::default(),
            fetch: WebFetchConfig::default(),
        }
    }
@@ -281,6 +286,14 @@ impl Default for PerplexitySearchConfig {
    }
 }

+/// SearXNG self-hosted search configuration.
+#[derive(Debug, Clone, Default, Serialize, Deserialize)]
+#[serde(default)]
+pub struct SearxngSearchConfig {
+    /// Base URL of the SearXNG instance (e.g., "https://search.example.com").
+    pub url: String,
+}
+
 /// Web fetch configuration.
 #[derive(Debug, Clone, Serialize, Deserialize)]
 #[serde(default)]
@@ -3598,6 +3611,13 @@ impl KernelConfig {
                    ));
                }
            }
+            SearchProvider::Searxng => {
+                if self.web.searxng.url.is_empty() {
+                    warnings.push(
+                        "Searxng search selected but searxng.url is not configured".to_string(),
+                    );
+                }
+            }
            SearchProvider::DuckDuckGo | SearchProvider::Auto => {}
        }

@@ -337,10 +337,11 @@ cache_ttl_minutes = 15

 | Value | Description |
 |-------|-------------|
-| `auto` | Cascading fallback: tries Tavily, then Brave, then Perplexity, then DuckDuckGo, based on which API keys are available. |
+| `auto` | Cascading fallback: tries Tavily, then Brave, then Perplexity, then SearXNG, then DuckDuckGo, based on which API keys/configs are available. |
 | `brave` | Brave Search API. Requires `BRAVE_API_KEY`. |
 | `tavily` | Tavily AI-native search. Requires `TAVILY_API_KEY`. |
 | `perplexity` | Perplexity AI search. Requires `PERPLEXITY_API_KEY`. |
+| `searxng` | Self-hosted search engine aggregator. No API key required, just point to your SearXNG instance. |
 | `duck_duck_go` | DuckDuckGo HTML scraping. No API key needed. |

 #### `[web.brave]`
@@ -392,6 +393,19 @@ model = "sonar"
 | `api_key_env` | string | `"PERPLEXITY_API_KEY"` | Environment variable name holding the Perplexity API key. |
 | `model` | string | `"sonar"` | Perplexity model to use for search queries. |

+#### `[web.searxng]`
+
+**SearXNG** — Self-hosted search engine aggregator. No API key required, just point to your SearXNG instance. Supports 30+ search categories (general, images, news, videos, etc.) and pagination.
+
+```toml
+[web.searxng]
+url = "https://searxng.example.com"    # SearXNG instance URL (required)
+```
+
+| Field | Type | Default | Description |
+|-------|------|---------|-------------|
+| `url` | string | (required) | Full URL of your SearXNG instance (e.g., `https://searxng.example.com`). Must be accessible. |
+
 #### `[web.fetch]`

 ```toml