From 656e2734ce722881ad41cc17b2019fbdb737c1b6 Mon Sep 17 00:00:00 2001 From: beann <1012252+beann@user.noreply.gitee.com> Date: Mon, 30 Mar 2026 18:04:39 +0800 Subject: [PATCH 1/8] feat: add SearXNG search provider with custom URL and JSON output --- crates/openfang-runtime/src/web_search.rs | 105 +++++++++++++++++++++- crates/openfang-types/src/config.rs | 33 ++++++- 2 files changed, 136 insertions(+), 2 deletions(-) diff --git a/crates/openfang-runtime/src/web_search.rs b/crates/openfang-runtime/src/web_search.rs index 66fa2fdf..eb7dd130 100644 --- a/crates/openfang-runtime/src/web_search.rs +++ b/crates/openfang-runtime/src/web_search.rs @@ -55,6 +55,7 @@ impl WebSearchEngine { SearchProvider::Tavily => self.search_tavily(query, max_results).await, SearchProvider::Perplexity => self.search_perplexity(query).await, SearchProvider::DuckDuckGo => self.search_duckduckgo(query, max_results).await, + SearchProvider::Searxng => self.search_searxng(query, max_results).await, SearchProvider::Auto => self.search_auto(query, max_results).await, }; @@ -67,7 +68,7 @@ impl WebSearchEngine { } /// Auto-select provider based on available API keys. - /// Priority: Tavily → Brave → Perplexity → DuckDuckGo + /// Priority: Tavily → Brave → Perplexity → Searxng → DuckDuckGo async fn search_auto(&self, query: &str, max_results: usize) -> Result { // Tavily first (AI-agent-native) if resolve_api_key(&self.config.tavily.api_key_env).is_some() { @@ -96,6 +97,15 @@ impl WebSearchEngine { } } + // Searxng fourth (self-hosted, no API key needed) + if !self.config.searxng.url.is_empty() { + debug!("Auto: trying Searxng"); + match self.search_searxng(query, max_results).await { + Ok(result) => return Ok(result), + Err(e) => warn!("Searxng failed, falling back: {e}"), + } + } + // DuckDuckGo always available as zero-config fallback debug!("Auto: falling back to DuckDuckGo"); self.search_duckduckgo(query, max_results).await @@ -313,6 +323,99 @@ impl WebSearchEngine { Ok(output) } + + /// Search via SearXNG self-hosted instance. + async fn search_searxng(&self, query: &str, max_results: usize) -> Result { + if self.config.searxng.url.is_empty() { + return Err("SearXNG URL is not configured".to_string()); + } + + let limit = max_results.min(self.config.searxng.max_results); + + debug!(query, "Searching via SearXNG"); + + let resp = self + .client + .get(format!("{}/search", self.config.searxng.url.trim_end_matches('/'))) + .query(&[ + ("q", query), + ("format", "json"), + ("engines", "general"), + ("limit", &limit.to_string()), + ]) + .header("User-Agent", "Mozilla/5.0 (compatible; OpenFangAgent/0.1)") + .send() + .await + .map_err(|e| format!("SearXNG request failed: {e}"))?; + + if !resp.status().is_success() { + return Err(format!("SearXNG API returned {}", resp.status())); + } + + #[derive(serde::Deserialize)] + struct SearxngResponse { + results: Vec, + query: String, + } + + #[derive(serde::Deserialize)] + struct SearxngResult { + url: String, + title: String, + content: Option, + } + + let data: SearxngResponse = resp + .json() + .await + .map_err(|e| format!("SearXNG JSON parse failed: {e}"))?; + + if data.results.is_empty() { + return Err(format!("No results found for '{query}' (SearXNG).")); + } + + #[derive(serde::Serialize)] + struct OutputResult<'a> { + title: &'a str, + url: &'a str, + content: &'a str, + } + + #[derive(serde::Serialize)] + struct Output<'a> { + query: &'a str, + results: Vec>, + } + + let results: Vec = data + .results + .iter() + .map(|r| OutputResult { + title: &r.title, + url: &r.url, + content: r.content.as_deref().unwrap_or(""), + }) + .collect(); + + let output = Output { + query: &data.query, + results, + }; + + serde_json::to_string(&output) + .map_err(|e| format!("Failed to serialize SearXNG results: {e}")) + } + + /// Return all non-Auto search providers (for provider listing/discovery). + pub fn all_providers() -> Vec { + vec![ + SearchProvider::Brave, + SearchProvider::Tavily, + SearchProvider::Perplexity, + SearchProvider::DuckDuckGo, + SearchProvider::Searxng, + ] + } } // --------------------------------------------------------------------------- diff --git a/crates/openfang-types/src/config.rs b/crates/openfang-types/src/config.rs index 2c85ae35..2cdd7e59 100644 --- a/crates/openfang-types/src/config.rs +++ b/crates/openfang-types/src/config.rs @@ -173,7 +173,9 @@ pub enum SearchProvider { Perplexity, /// DuckDuckGo HTML (no API key needed). DuckDuckGo, - /// Auto-select based on available API keys (Tavily → Brave → Perplexity → DuckDuckGo). + /// SearXNG self-hosted search (no API key needed). + Searxng, + /// Auto-select based on available API keys (Tavily → Brave → Perplexity → Searxng → DuckDuckGo). #[default] Auto, } @@ -192,6 +194,8 @@ pub struct WebConfig { pub tavily: TavilySearchConfig, /// Perplexity Search configuration. pub perplexity: PerplexitySearchConfig, + /// SearXNG Search configuration. + pub searxng: SearxngSearchConfig, /// Web fetch configuration. pub fetch: WebFetchConfig, } @@ -204,6 +208,7 @@ impl Default for WebConfig { brave: BraveSearchConfig::default(), tavily: TavilySearchConfig::default(), perplexity: PerplexitySearchConfig::default(), + searxng: SearxngSearchConfig::default(), fetch: WebFetchConfig::default(), } } @@ -281,6 +286,25 @@ impl Default for PerplexitySearchConfig { } } +/// SearXNG self-hosted search configuration. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(default)] +pub struct SearxngSearchConfig { + /// Base URL of the SearXNG instance (e.g., "https://search.example.com"). + pub url: String, + /// Maximum results to return. + pub max_results: usize, +} + +impl Default for SearxngSearchConfig { + fn default() -> Self { + Self { + url: String::new(), + max_results: 5, + } + } +} + /// Web fetch configuration. #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(default)] @@ -3588,6 +3612,13 @@ impl KernelConfig { )); } } + SearchProvider::Searxng => { + if self.web.searxng.url.is_empty() { + warnings.push( + "Searxng search selected but searxng.url is not configured".to_string(), + ); + } + } SearchProvider::DuckDuckGo | SearchProvider::Auto => {} } From ce3344a99432ce5873f1e89c9253f22d16239811 Mon Sep 17 00:00:00 2001 From: beann <1012252+beann@user.noreply.gitee.com> Date: Mon, 30 Mar 2026 18:12:16 +0800 Subject: [PATCH 2/8] fix: SearXNG does not support limit param, truncate results client-side --- crates/openfang-runtime/src/web_search.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/openfang-runtime/src/web_search.rs b/crates/openfang-runtime/src/web_search.rs index eb7dd130..567f7a3b 100644 --- a/crates/openfang-runtime/src/web_search.rs +++ b/crates/openfang-runtime/src/web_search.rs @@ -341,7 +341,6 @@ impl WebSearchEngine { ("q", query), ("format", "json"), ("engines", "general"), - ("limit", &limit.to_string()), ]) .header("User-Agent", "Mozilla/5.0 (compatible; OpenFangAgent/0.1)") .send() @@ -390,6 +389,7 @@ impl WebSearchEngine { let results: Vec = data .results .iter() + .take(limit) .map(|r| OutputResult { title: &r.title, url: &r.url, From d75a56a0f6dbcd83e1a941d8f0764487d08a77a1 Mon Sep 17 00:00:00 2001 From: beann <1012252+beann@user.noreply.gitee.com> Date: Mon, 30 Mar 2026 18:15:35 +0800 Subject: [PATCH 3/8] fix: filter SearXNG noise fields, only expose title/url/content/published_date to LLM --- crates/openfang-runtime/src/web_search.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/crates/openfang-runtime/src/web_search.rs b/crates/openfang-runtime/src/web_search.rs index 567f7a3b..544cf8f1 100644 --- a/crates/openfang-runtime/src/web_search.rs +++ b/crates/openfang-runtime/src/web_search.rs @@ -362,6 +362,8 @@ impl WebSearchEngine { url: String, title: String, content: Option, + #[serde(alias = "pubdate")] + published_date: Option, } let data: SearxngResponse = resp @@ -378,6 +380,8 @@ impl WebSearchEngine { title: &'a str, url: &'a str, content: &'a str, + #[serde(skip_serializing_if = "Option::is_none")] + published_date: Option<&'a str>, } #[derive(serde::Serialize)] @@ -394,6 +398,7 @@ impl WebSearchEngine { title: &r.title, url: &r.url, content: r.content.as_deref().unwrap_or(""), + published_date: r.published_date.as_deref(), }) .collect(); From 50c51dd6b73802e41bba2d6eee579d506386d2d7 Mon Sep 17 00:00:00 2001 From: beann <1012252+beann@user.noreply.gitee.com> Date: Mon, 30 Mar 2026 18:26:11 +0800 Subject: [PATCH 4/8] refactor: remove redundant max_results from SearxngSearchConfig --- crates/openfang-runtime/src/web_search.rs | 2 +- crates/openfang-types/src/config.rs | 13 +------------ 2 files changed, 2 insertions(+), 13 deletions(-) diff --git a/crates/openfang-runtime/src/web_search.rs b/crates/openfang-runtime/src/web_search.rs index 544cf8f1..cf4613fb 100644 --- a/crates/openfang-runtime/src/web_search.rs +++ b/crates/openfang-runtime/src/web_search.rs @@ -330,7 +330,7 @@ impl WebSearchEngine { return Err("SearXNG URL is not configured".to_string()); } - let limit = max_results.min(self.config.searxng.max_results); + let limit = max_results; debug!(query, "Searching via SearXNG"); diff --git a/crates/openfang-types/src/config.rs b/crates/openfang-types/src/config.rs index 2cdd7e59..db8536aa 100644 --- a/crates/openfang-types/src/config.rs +++ b/crates/openfang-types/src/config.rs @@ -287,22 +287,11 @@ impl Default for PerplexitySearchConfig { } /// SearXNG self-hosted search configuration. -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Default, Serialize, Deserialize)] #[serde(default)] pub struct SearxngSearchConfig { /// Base URL of the SearXNG instance (e.g., "https://search.example.com"). pub url: String, - /// Maximum results to return. - pub max_results: usize, -} - -impl Default for SearxngSearchConfig { - fn default() -> Self { - Self { - url: String::new(), - max_results: 5, - } - } } /// Web fetch configuration. From 79ca1cda3243c9aba4630c5072ddf5223e3c04da Mon Sep 17 00:00:00 2001 From: beann <1012252+beann@user.noreply.gitee.com> Date: Mon, 30 Mar 2026 18:32:57 +0800 Subject: [PATCH 5/8] feat: SearXNG dynamic category support with validation --- crates/openfang-runtime/src/web_search.rs | 64 +++++++++++++++++++++-- 1 file changed, 60 insertions(+), 4 deletions(-) diff --git a/crates/openfang-runtime/src/web_search.rs b/crates/openfang-runtime/src/web_search.rs index cf4613fb..425104a2 100644 --- a/crates/openfang-runtime/src/web_search.rs +++ b/crates/openfang-runtime/src/web_search.rs @@ -55,7 +55,7 @@ impl WebSearchEngine { SearchProvider::Tavily => self.search_tavily(query, max_results).await, SearchProvider::Perplexity => self.search_perplexity(query).await, SearchProvider::DuckDuckGo => self.search_duckduckgo(query, max_results).await, - SearchProvider::Searxng => self.search_searxng(query, max_results).await, + SearchProvider::Searxng => self.search_searxng(query, max_results, None).await, SearchProvider::Auto => self.search_auto(query, max_results).await, }; @@ -100,7 +100,7 @@ impl WebSearchEngine { // Searxng fourth (self-hosted, no API key needed) if !self.config.searxng.url.is_empty() { debug!("Auto: trying Searxng"); - match self.search_searxng(query, max_results).await { + match self.search_searxng(query, max_results, None).await { Ok(result) => return Ok(result), Err(e) => warn!("Searxng failed, falling back: {e}"), } @@ -325,11 +325,32 @@ impl WebSearchEngine { } /// Search via SearXNG self-hosted instance. - async fn search_searxng(&self, query: &str, max_results: usize) -> Result { + async fn search_searxng( + &self, + query: &str, + max_results: usize, + category: Option<&str>, + ) -> Result { if self.config.searxng.url.is_empty() { return Err("SearXNG URL is not configured".to_string()); } + let category = category.unwrap_or("general"); + + // Validate category against SearXNG instance + match self.list_searxng_categories().await { + Ok(cats) => { + if !cats.iter().any(|c| c == category) { + return Err(format!( + "Invalid SearXNG category '{}'. Available: {}", + category, + cats.join(", ") + )); + } + } + Err(e) => warn!("Could not validate SearXNG category: {e}"), + } + let limit = max_results; debug!(query, "Searching via SearXNG"); @@ -340,7 +361,7 @@ impl WebSearchEngine { .query(&[ ("q", query), ("format", "json"), - ("engines", "general"), + ("categories", category), ]) .header("User-Agent", "Mozilla/5.0 (compatible; OpenFangAgent/0.1)") .send() @@ -411,6 +432,41 @@ impl WebSearchEngine { .map_err(|e| format!("Failed to serialize SearXNG results: {e}")) } + /// List available search categories from the SearXNG instance. + /// + /// Fetches the `/config` endpoint and returns the list of categories + /// the instance supports (e.g., "general", "images", "news", "videos"). + /// Returns an error if SearXNG URL is not configured or the request fails. + pub async fn list_searxng_categories(&self) -> Result, String> { + if self.config.searxng.url.is_empty() { + return Err("SearXNG URL is not configured".to_string()); + } + + #[derive(serde::Deserialize)] + struct SearxngConfig { + categories: Vec, + } + + let resp = self + .client + .get(format!("{}/config", self.config.searxng.url.trim_end_matches('/'))) + .header("User-Agent", "Mozilla/5.0 (compatible; OpenFangAgent/0.1)") + .send() + .await + .map_err(|e| format!("SearXNG config request failed: {e}"))?; + + if !resp.status().is_success() { + return Err(format!("SearXNG config API returned {}", resp.status())); + } + + let data: SearxngConfig = resp + .json() + .await + .map_err(|e| format!("SearXNG config JSON parse failed: {e}"))?; + + Ok(data.categories) + } + /// Return all non-Auto search providers (for provider listing/discovery). pub fn all_providers() -> Vec { vec![ From 9cf37eab22aa37f5451f792a65142fe390afef49 Mon Sep 17 00:00:00 2001 From: beann <1012252+beann@user.noreply.gitee.com> Date: Mon, 30 Mar 2026 18:40:26 +0800 Subject: [PATCH 6/8] feat: SearXNG pagination support --- crates/openfang-runtime/src/web_search.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/crates/openfang-runtime/src/web_search.rs b/crates/openfang-runtime/src/web_search.rs index 425104a2..28e92259 100644 --- a/crates/openfang-runtime/src/web_search.rs +++ b/crates/openfang-runtime/src/web_search.rs @@ -55,7 +55,7 @@ impl WebSearchEngine { SearchProvider::Tavily => self.search_tavily(query, max_results).await, SearchProvider::Perplexity => self.search_perplexity(query).await, SearchProvider::DuckDuckGo => self.search_duckduckgo(query, max_results).await, - SearchProvider::Searxng => self.search_searxng(query, max_results, None).await, + SearchProvider::Searxng => self.search_searxng(query, max_results, None, 1).await, SearchProvider::Auto => self.search_auto(query, max_results).await, }; @@ -100,7 +100,7 @@ impl WebSearchEngine { // Searxng fourth (self-hosted, no API key needed) if !self.config.searxng.url.is_empty() { debug!("Auto: trying Searxng"); - match self.search_searxng(query, max_results, None).await { + match self.search_searxng(query, max_results, None, 1).await { Ok(result) => return Ok(result), Err(e) => warn!("Searxng failed, falling back: {e}"), } @@ -330,6 +330,7 @@ impl WebSearchEngine { query: &str, max_results: usize, category: Option<&str>, + page: u32, ) -> Result { if self.config.searxng.url.is_empty() { return Err("SearXNG URL is not configured".to_string()); @@ -362,6 +363,7 @@ impl WebSearchEngine { ("q", query), ("format", "json"), ("categories", category), + ("page", &page.to_string()), ]) .header("User-Agent", "Mozilla/5.0 (compatible; OpenFangAgent/0.1)") .send() From 9372cc6ff200d440ca4505b6c94d2ffaef2bfc50 Mon Sep 17 00:00:00 2001 From: beann <1012252+beann@user.noreply.gitee.com> Date: Mon, 30 Mar 2026 18:57:06 +0800 Subject: [PATCH 7/8] docs: add SearXNG search provider configuration --- docs/configuration.md | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index f0ae8f2d..99e488ff 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -337,10 +337,11 @@ cache_ttl_minutes = 15 | Value | Description | |-------|-------------| -| `auto` | Cascading fallback: tries Tavily, then Brave, then Perplexity, then DuckDuckGo, based on which API keys are available. | +| `auto` | Cascading fallback: tries Tavily, then Brave, then Perplexity, then SearXNG, then DuckDuckGo, based on which API keys/configs are available. | | `brave` | Brave Search API. Requires `BRAVE_API_KEY`. | | `tavily` | Tavily AI-native search. Requires `TAVILY_API_KEY`. | | `perplexity` | Perplexity AI search. Requires `PERPLEXITY_API_KEY`. | +| `searxng` | Self-hosted search engine aggregator. No API key required, just point to your SearXNG instance. | | `duck_duck_go` | DuckDuckGo HTML scraping. No API key needed. | #### `[web.brave]` @@ -392,6 +393,19 @@ model = "sonar" | `api_key_env` | string | `"PERPLEXITY_API_KEY"` | Environment variable name holding the Perplexity API key. | | `model` | string | `"sonar"` | Perplexity model to use for search queries. | +#### `[web.searxng]` + +**SearXNG** — Self-hosted search engine aggregator. No API key required, just point to your SearXNG instance. Supports 30+ search categories (general, images, news, videos, etc.) and pagination. + +```toml +[web.searxng] +url = "https://searxng.example.com" # SearXNG instance URL (required) +``` + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `url` | string | (required) | Full URL of your SearXNG instance (e.g., `https://searxng.example.com`). Must be accessible. | + #### `[web.fetch]` ```toml From 46eac44635a7ac8455592fee3292ae6b88a0a9fb Mon Sep 17 00:00:00 2001 From: beann <1012252+beann@user.noreply.gitee.com> Date: Mon, 30 Mar 2026 19:35:53 +0800 Subject: [PATCH 8/8] feat: add searxng search specialist skill --- .../openfang-skills/bundled/searxng/SKILL.md | 70 +++++++++++++++++++ crates/openfang-skills/src/bundled.rs | 3 +- 2 files changed, 72 insertions(+), 1 deletion(-) create mode 100644 crates/openfang-skills/bundled/searxng/SKILL.md diff --git a/crates/openfang-skills/bundled/searxng/SKILL.md b/crates/openfang-skills/bundled/searxng/SKILL.md new file mode 100644 index 00000000..0f5d0602 --- /dev/null +++ b/crates/openfang-skills/bundled/searxng/SKILL.md @@ -0,0 +1,70 @@ +--- +name: searxng +description: Privacy-respecting metasearch specialist using SearXNG instances +--- +# SearXNG Search Specialist + +You are a privacy-respecting web search specialist using SearXNG, a self-hosted metasearch engine that aggregates results from multiple search engines without tracking. + +## Key Principles + +- Prefer SearXNG for privacy-sensitive searches — no API keys, no tracking, no user profiling. +- Always cite sources with URLs so the user can verify information. +- Prefer primary sources (official docs, research papers) over secondary ones (blog posts, forums). +- When information conflicts across sources, present both perspectives and note the discrepancy. +- State the date of information when recency matters. + +## SearXNG Capabilities + +SearXNG supports 30+ search categories. Use the right category for the task: + +| Category | Use Case | +|----------|----------| +| `general` | Default web search | +| `images` | Image search | +| `news` | News articles | +| `videos` | Video results | +| `music` | Music and audio | +| `files` | File search | +| `it` | IT and programming | +| `science` | Scientific content | +| `books` | Book search | +| `maps` | Map and location | +| `q&a` | Q&A sites (Stack Overflow, etc.) | +| `social media` | Social media posts | +| `wikimedia` | Wikipedia and Wikimedia | +| `dictionaries` | Dictionary definitions | +| `currency` | Currency conversion | +| `weather` | Weather information | +| `translate` | Translation results | + +## Search Techniques + +- **Category selection**: Always specify a category when the topic is clear. Use `images` for visual content, `news` for current events, `it` for programming questions. +- **Pagination**: Use page parameter to get more results when the first page doesn't contain what you need. +- **Engine syntax**: SearXNG supports `!engine` syntax to target specific engines (e.g., `!wikipedia rust programming`). +- **Site search**: Use `site:example.com` in queries to search within a specific domain. +- **Exact phrases**: Use quotes for exact phrase matching (e.g., `"rust borrow checker"`). +- **Time filtering**: SearXNG instances may support time range filters — check the instance's preferences page. + +## Query Formulation + +- Start with specific, targeted queries. Use exact phrases for precise matches. +- Include the current year when looking for recent information or documentation. +- For technical questions, include the specific version number, framework name, or error message. +- If the first query yields poor results, reformulate using synonyms or broader/narrower scope. + +## Synthesizing Results + +- Lead with the direct answer, then provide supporting context. +- Organize findings by relevance, not by the order you found them. +- Summarize long articles into key takeaways rather than quoting entire passages. +- When comparing options, use structured comparisons with pros and cons. +- Flag information that may be outdated or from unreliable sources. + +## Pitfalls to Avoid + +- Never present information from a single source as definitive without corroboration. +- Do not include URLs you have not verified — broken links erode trust. +- Do not overwhelm the user with every result; curate the most relevant 3-5 sources. +- Avoid SEO-heavy content farms as primary sources — prefer official docs and community-vetted answers. diff --git a/crates/openfang-skills/src/bundled.rs b/crates/openfang-skills/src/bundled.rs index 3b29f6e6..203c5a41 100644 --- a/crates/openfang-skills/src/bundled.rs +++ b/crates/openfang-skills/src/bundled.rs @@ -13,6 +13,7 @@ pub fn bundled_skills() -> Vec<(&'static str, &'static str)> { ("github", include_str!("../bundled/github/SKILL.md")), ("docker", include_str!("../bundled/docker/SKILL.md")), ("web-search", include_str!("../bundled/web-search/SKILL.md")), + ("searxng", include_str!("../bundled/searxng/SKILL.md")), ( "code-reviewer", include_str!("../bundled/code-reviewer/SKILL.md"), @@ -195,7 +196,7 @@ mod tests { #[test] fn test_bundled_skills_count() { let skills = bundled_skills(); - assert_eq!(skills.len(), 60, "Expected 60 bundled skills"); + assert_eq!(skills.len(), 61, "Expected 61 bundled skills"); } #[test]