zesdex/apps/infrastructure/src/tools/web_search.rs

//! Web search & documentation scraper tool — search the web for docs, APIs,
//! and troubleshooting information using a SearXNG instance.
//!
//! Flow: take a search query → call SearXNG JSON API → parse results →
//! optionally fetch full page content → return formatted markdown.

use anyhow::Result;
use serde_json::{json, Value};
use tracing::{debug, info, instrument, warn};

use crate::tools::{arg_str, Tool, ToolCtx};

/// Default SearXNG instance URL.
const DEFAULT_SEARXNG_URL: &str = "https://searxng.imrnes.team";

/// Search the web for documentation, APIs, and technical information.
///
/// Flow: build query → call SearXNG JSON endpoint → parse results →
/// optionally scrape full page content → return formatted result.
pub struct WebSearch;

impl Tool for WebSearch {
    fn name(&self) -> &'static str {
        "web_search"
    }

    fn description(&self) -> &'static str {
        "Search the web for documentation, APIs, and technical information. Uses SearXNG instance."
    }

    fn parameters(&self) -> Value {
        json!({
            "type": "object",
            "properties": {
                "query": {
                    "type": "string",
                    "description": "Search query (e.g. 'Next.js 15 app router documentation')"
                },
                "max_results": {
                    "type": "integer",
                    "description": "Maximum number of results to return (default 5, max 10)",
                    "default": 5
                },
                "fetch_content": {
                    "type": "boolean",
                    "description": "Whether to fetch full page content from each result (default false)",
                    "default": false
                },
                "categories": {
                    "type": "string",
                    "description": "Search categories (e.g. 'general', 'science', 'it', 'news'). Default: general",
                    "default": "general"
                }
            },
            "required": ["query"]
        })
    }

    #[instrument(skip(self, _ctx, args))]
    fn run(&self, _ctx: &ToolCtx, args: &Value) -> Result<String> {
        let query = arg_str(args, "query")?;
        let max_results = args
            .get("max_results")
            .and_then(|v| v.as_u64())
            .unwrap_or(5)
            .min(10) as usize;
        let fetch_content = args
            .get("fetch_content")
            .and_then(|v| v.as_bool())
            .unwrap_or(false);
        let categories = args
            .get("categories")
            .and_then(|v| v.as_str())
            .unwrap_or("general");

        info!(query = %query, max_results, fetch_content, categories = %categories, "web search starting");

        let searxng_url =
            std::env::var("SEARXNG_URL").unwrap_or_else(|_| DEFAULT_SEARXNG_URL.to_string());

        // Build the SearXNG JSON search URL
        let search_url = format!(
            "{}/search?format=json&q={}&categories={}&language=en-US",
            searxng_url,
            urlencoding(&query),
            categories
        );

        debug!(search_url = %search_url, "calling SearXNG API");

        // Make the HTTP request
        let client = reqwest::blocking::Client::builder()
            .timeout(std::time::Duration::from_secs(30))
            .user_agent("Zesdex/1.0 (AI Coding Agent)")
            .build()
            .map_err(|e| anyhow::anyhow!("failed to create HTTP client: {e}"))?;

        let response = client
            .get(&search_url)
            .send()
            .map_err(|e| anyhow::anyhow!("search request failed: {e}"))?;

        if !response.status().is_success() {
            warn!(status = %response.status(), "SearXNG returned non-success status");
            anyhow::bail!("SearXNG returned HTTP {}", response.status());
        }

        let body: Value = response
            .json()
            .map_err(|e| anyhow::anyhow!("failed to parse SearXNG JSON response: {e}"))?;

        let results = body
            .get("results")
            .and_then(|v| v.as_array())
            .map(|arr| {
                arr.iter()
                    .filter(|r| {
                        r.get("url").and_then(|u| u.as_str()).is_some()
                            && r.get("title").and_then(|t| t.as_str()).is_some()
                    })
                    .take(max_results)
                    .collect::<Vec<_>>()
            })
            .unwrap_or_default();

        let result_count = results.len();
        if results.is_empty() {
            info!("no search results found");
            return Ok(format!("No search results found for '{query}'."));
        }

        info!(result_count, "web search completed");

        let mut output = format!("## Web Search Results for: {query}\n\n");

        for (i, result) in results.iter().enumerate() {
            let title = result
                .get("title")
                .and_then(|v| v.as_str())
                .unwrap_or("Untitled");
            let url = result.get("url").and_then(|v| v.as_str()).unwrap_or("");
            let snippet = result
                .get("content")
                .and_then(|v| v.as_str())
                .unwrap_or("")
                .trim();

            output.push_str(&format!("### {}. [{title}]({url})\n\n", i + 1));

            if !snippet.is_empty() {
                output.push_str(&format!("> {snippet}\n\n"));
            }

            // Optionally fetch full page content
            if fetch_content && !url.is_empty() {
                match fetch_page_content(url, &client) {
                    Ok(content) => {
                        let truncated = limit_lines(&content, 80);
                        output.push_str(&format!("**Content:**\n```\n{truncated}\n```\n\n"));
                    }
                    Err(e) => {
                        debug!(url = %url, error = %e, "failed to fetch page content");
                        output.push_str(&format!("*(Content fetch failed: {e})*\n\n"));
                    }
                }
            }

            // Show engine info if available
            if let Some(engine) = result.get("engine").and_then(|v| v.as_str()) {
                output.push_str(&format!("*Source: {engine}*\n\n"));
            }
        }

        // Add suggestion for more specific queries if very few results
        if result_count < 3 {
            output.push_str(
                "---\n*Few results. Try a more specific query or different categories.*\n",
            );
        }

        Ok(output)
    }
}

/// URL-encode a string for use in query parameters.
fn urlencoding(input: &str) -> String {
    input
        .chars()
        .map(|c| match c {
            'A'..='Z' | 'a'..='z' | '0'..='9' | '-' | '_' | '.' | '~' => c.to_string(),
            ' ' => '+'.to_string(),
            _ => {
                let bytes = c.to_string().into_bytes();
                bytes
                    .iter()
                    .map(|&b| format!("%{:02X}", b))
                    .collect::<String>()
            }
        })
        .collect()
}

/// Fetch and extract readable text content from a URL.
fn fetch_page_content(url: &str, client: &reqwest::blocking::Client) -> Result<String> {
    let resp = client
        .get(url)
        .timeout(std::time::Duration::from_secs(15))
        .send()
        .map_err(|e| anyhow::anyhow!("HTTP request failed: {e}"))?;

    let html = resp
        .text()
        .map_err(|e| anyhow::anyhow!("failed to read response body: {e}"))?;

    // Parse HTML and extract text
    let document = scraper::Html::parse_document(&html);

    // Remove script and style elements
    let selector = scraper::Selector::parse("script, style, nav, footer, header")
        .map_err(|e| anyhow::anyhow!("invalid selector: {e}"))?;
    let cleaned = document.clone();
    for element in cleaned.select(&selector) {
        let inner = element.inner_html();
        let _ = inner;
    }

    // Extract text from main content areas
    let content_selectors = [
        "article",
        "main",
        ".content",
        ".documentation",
        ".doc-content",
        ".post-content",
        ".entry-content",
        "#content",
        "body",
    ];

    let mut text = String::new();
    for sel_str in &content_selectors {
        if let Ok(sel) = scraper::Selector::parse(sel_str) {
            if let Some(element) = cleaned.select(&sel).next() {
                text = element.text().collect::<Vec<_>>().join(" ");
                if text.len() > 100 {
                    break;
                }
            }
        }
    }

    if text.is_empty() {
        // Fallback: just get all body text
        if let Ok(body_sel) = scraper::Selector::parse("body") {
            text = cleaned
                .select(&body_sel)
                .next()
                .map(|e| e.text().collect::<Vec<_>>().join(" "))
                .unwrap_or_default();
        }
    }

    // Clean up whitespace
    let cleaned_text = text.split_whitespace().collect::<Vec<_>>().join(" ");

    if cleaned_text.is_empty() {
        anyhow::bail!("no readable content found on page");
    }

    Ok(cleaned_text)
}

/// Limit a string to at most `max_lines` lines.
fn limit_lines(s: &str, max_lines: usize) -> String {
    s.lines().take(max_lines).collect::<Vec<_>>().join("\n")
}