feat: add semantic search tool for code symbol indexing and searching
- Implemented a new tool for semantic code search that indexes Rust code symbols (functions, structs, enums, traits, modules) and allows searching by name, concept, or meaning. - Introduced a symbol index structure with methods for rebuilding the index and searching symbols. - Added regex patterns for extracting various code symbols from Rust source files. - Implemented scoring logic for search results based on exact matches, prefix matches, and context relevance. - Created a web search tool that interacts with a SearXNG instance to fetch documentation and API information based on user queries. - Added a diff preview overlay for rendering git diff output with color-coded additions and deletions in a TUI interface.
This commit is contained in:
asepharyana committed
1 parent
785ae19757
commit
fef3c925cd
17 files changed
+1968
No files matched your search
@@ -0,0 +1,283 @@
|
||||
//! Web search & documentation scraper tool — search the web for docs, APIs,
|
||||
//! and troubleshooting information using a SearXNG instance.
|
||||
//!
|
||||
//! Flow: take a search query → call SearXNG JSON API → parse results →
|
||||
//! optionally fetch full page content → return formatted markdown.
|
||||
|
||||
use anyhow::Result;
|
||||
use serde_json::{json, Value};
|
||||
use tracing::{debug, info, instrument, warn};
|
||||
|
||||
use crate::tools::{arg_str, Tool, ToolCtx};
|
||||
|
||||
/// Default SearXNG instance URL.
|
||||
const DEFAULT_SEARXNG_URL: &str = "https://searxng.imrnes.team";
|
||||
|
||||
/// Search the web for documentation, APIs, and technical information.
|
||||
///
|
||||
/// Flow: build query → call SearXNG JSON endpoint → parse results →
|
||||
/// optionally scrape full page content → return formatted result.
|
||||
pub struct WebSearch;
|
||||
|
||||
impl Tool for WebSearch {
|
||||
fn name(&self) -> &'static str {
|
||||
"web_search"
|
||||
}
|
||||
|
||||
fn description(&self) -> &'static str {
|
||||
"Search the web for documentation, APIs, and technical information. Uses SearXNG instance."
|
||||
}
|
||||
|
||||
fn parameters(&self) -> Value {
|
||||
json!({
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"query": {
|
||||
"type": "string",
|
||||
"description": "Search query (e.g. 'Next.js 15 app router documentation')"
|
||||
},
|
||||
"max_results": {
|
||||
"type": "integer",
|
||||
"description": "Maximum number of results to return (default 5, max 10)",
|
||||
"default": 5
|
||||
},
|
||||
"fetch_content": {
|
||||
"type": "boolean",
|
||||
"description": "Whether to fetch full page content from each result (default false)",
|
||||
"default": false
|
||||
},
|
||||
"categories": {
|
||||
"type": "string",
|
||||
"description": "Search categories (e.g. 'general', 'science', 'it', 'news'). Default: general",
|
||||
"default": "general"
|
||||
}
|
||||
},
|
||||
"required": ["query"]
|
||||
})
|
||||
}
|
||||
|
||||
#[instrument(skip(self, _ctx, args))]
|
||||
fn run(&self, _ctx: &ToolCtx, args: &Value) -> Result<String> {
|
||||
let query = arg_str(args, "query")?;
|
||||
let max_results = args
|
||||
.get("max_results")
|
||||
.and_then(|v| v.as_u64())
|
||||
.unwrap_or(5)
|
||||
.min(10) as usize;
|
||||
let fetch_content = args
|
||||
.get("fetch_content")
|
||||
.and_then(|v| v.as_bool())
|
||||
.unwrap_or(false);
|
||||
let categories = args
|
||||
.get("categories")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("general");
|
||||
|
||||
info!(query = %query, max_results, fetch_content, categories = %categories, "web search starting");
|
||||
|
||||
let searxng_url = std::env::var("SEARXNG_URL")
|
||||
.unwrap_or_else(|_| DEFAULT_SEARXNG_URL.to_string());
|
||||
|
||||
// Build the SearXNG JSON search URL
|
||||
let search_url = format!(
|
||||
"{}/search?format=json&q={}&categories={}&language=en-US",
|
||||
searxng_url,
|
||||
urlencoding(&query),
|
||||
categories
|
||||
);
|
||||
|
||||
debug!(search_url = %search_url, "calling SearXNG API");
|
||||
|
||||
// Make the HTTP request
|
||||
let client = reqwest::blocking::Client::builder()
|
||||
.timeout(std::time::Duration::from_secs(30))
|
||||
.user_agent("Zesdex/1.0 (AI Coding Agent)")
|
||||
.build()
|
||||
.map_err(|e| anyhow::anyhow!("failed to create HTTP client: {e}"))?;
|
||||
|
||||
let response = client
|
||||
.get(&search_url)
|
||||
.send()
|
||||
.map_err(|e| anyhow::anyhow!("search request failed: {e}"))?;
|
||||
|
||||
if !response.status().is_success() {
|
||||
warn!(status = %response.status(), "SearXNG returned non-success status");
|
||||
anyhow::bail!("SearXNG returned HTTP {}", response.status());
|
||||
}
|
||||
|
||||
let body: Value = response
|
||||
.json()
|
||||
.map_err(|e| anyhow::anyhow!("failed to parse SearXNG JSON response: {e}"))?;
|
||||
|
||||
let results = body
|
||||
.get("results")
|
||||
.and_then(|v| v.as_array())
|
||||
.map(|arr| {
|
||||
arr.iter()
|
||||
.filter(|r| {
|
||||
r.get("url").and_then(|u| u.as_str()).is_some()
|
||||
&& r.get("title").and_then(|t| t.as_str()).is_some()
|
||||
})
|
||||
.take(max_results)
|
||||
.collect::<Vec<_>>()
|
||||
})
|
||||
.unwrap_or_default();
|
||||
|
||||
let result_count = results.len();
|
||||
if results.is_empty() {
|
||||
info!("no search results found");
|
||||
return Ok(format!("No search results found for '{query}'."));
|
||||
}
|
||||
|
||||
info!(result_count, "web search completed");
|
||||
|
||||
let mut output = format!("## Web Search Results for: {query}\n\n");
|
||||
|
||||
for (i, result) in results.iter().enumerate() {
|
||||
let title = result
|
||||
.get("title")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("Untitled");
|
||||
let url = result
|
||||
.get("url")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("");
|
||||
let snippet = result
|
||||
.get("content")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("")
|
||||
.trim();
|
||||
|
||||
output.push_str(&format!("### {}. [{title}]({url})\n\n", i + 1));
|
||||
|
||||
if !snippet.is_empty() {
|
||||
output.push_str(&format!("> {snippet}\n\n"));
|
||||
}
|
||||
|
||||
// Optionally fetch full page content
|
||||
if fetch_content && !url.is_empty() {
|
||||
match fetch_page_content(url, &client) {
|
||||
Ok(content) => {
|
||||
let truncated = limit_lines(&content, 80);
|
||||
output.push_str(&format!("**Content:**\n```\n{truncated}\n```\n\n"));
|
||||
}
|
||||
Err(e) => {
|
||||
debug!(url = %url, error = %e, "failed to fetch page content");
|
||||
output.push_str(&format!("*(Content fetch failed: {e})*\n\n"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Show engine info if available
|
||||
if let Some(engine) = result.get("engine").and_then(|v| v.as_str()) {
|
||||
output.push_str(&format!("*Source: {engine}*\n\n"));
|
||||
}
|
||||
}
|
||||
|
||||
// Add suggestion for more specific queries if very few results
|
||||
if result_count < 3 {
|
||||
output.push_str("---\n*Few results. Try a more specific query or different categories.*\n");
|
||||
}
|
||||
|
||||
Ok(output)
|
||||
}
|
||||
}
|
||||
|
||||
/// URL-encode a string for use in query parameters.
|
||||
fn urlencoding(input: &str) -> String {
|
||||
input
|
||||
.chars()
|
||||
.map(|c| match c {
|
||||
'A'..='Z' | 'a'..='z' | '0'..='9' | '-' | '_' | '.' | '~' => c.to_string(),
|
||||
' ' => '+'.to_string(),
|
||||
_ => {
|
||||
let bytes = c.to_string().into_bytes();
|
||||
bytes
|
||||
.iter()
|
||||
.map(|&b| format!("%{:02X}", b))
|
||||
.collect::<String>()
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Fetch and extract readable text content from a URL.
|
||||
fn fetch_page_content(url: &str, client: &reqwest::blocking::Client) -> Result<String> {
|
||||
let resp = client
|
||||
.get(url)
|
||||
.timeout(std::time::Duration::from_secs(15))
|
||||
.send()
|
||||
.map_err(|e| anyhow::anyhow!("HTTP request failed: {e}"))?;
|
||||
|
||||
let html = resp
|
||||
.text()
|
||||
.map_err(|e| anyhow::anyhow!("failed to read response body: {e}"))?;
|
||||
|
||||
// Parse HTML and extract text
|
||||
let document = scraper::Html::parse_document(&html);
|
||||
|
||||
// Remove script and style elements
|
||||
let selector = scraper::Selector::parse("script, style, nav, footer, header")
|
||||
.map_err(|e| anyhow::anyhow!("invalid selector: {e}"))?;
|
||||
let cleaned = document.clone();
|
||||
for element in cleaned.select(&selector) {
|
||||
let inner = element.inner_html();
|
||||
let _ = inner;
|
||||
}
|
||||
|
||||
// Extract text from main content areas
|
||||
let content_selectors = [
|
||||
"article",
|
||||
"main",
|
||||
".content",
|
||||
".documentation",
|
||||
".doc-content",
|
||||
".post-content",
|
||||
".entry-content",
|
||||
"#content",
|
||||
"body",
|
||||
];
|
||||
|
||||
let mut text = String::new();
|
||||
for sel_str in &content_selectors {
|
||||
if let Ok(sel) = scraper::Selector::parse(sel_str) {
|
||||
if let Some(element) = cleaned.select(&sel).next() {
|
||||
text = element.text().collect::<Vec<_>>().join(" ");
|
||||
if text.len() > 100 {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if text.is_empty() {
|
||||
// Fallback: just get all body text
|
||||
if let Ok(body_sel) = scraper::Selector::parse("body") {
|
||||
text = cleaned
|
||||
.select(&body_sel)
|
||||
.next()
|
||||
.map(|e| e.text().collect::<Vec<_>>().join(" "))
|
||||
.unwrap_or_default();
|
||||
}
|
||||
}
|
||||
|
||||
// Clean up whitespace
|
||||
let cleaned_text = text
|
||||
.split_whitespace()
|
||||
.collect::<Vec<_>>()
|
||||
.join(" ");
|
||||
|
||||
if cleaned_text.is_empty() {
|
||||
anyhow::bail!("no readable content found on page");
|
||||
}
|
||||
|
||||
Ok(cleaned_text)
|
||||
}
|
||||
|
||||
/// Limit a string to at most `max_lines` lines.
|
||||
fn limit_lines(s: &str, max_lines: usize) -> String {
|
||||
s.lines()
|
||||
.take(max_lines)
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n")
|
||||
}
|
||||
Reference in new issue
Block a user