import Redis from "ioredis"; import { createChildLogger } from "@/shared/logger/index"; import { createAbortControllerWithTimeout } from "@/shared/utils/index"; import { config } from "../../shared/config/config.js"; const log = createChildLogger("searxng-search"); const SEARXNG_BASE_URL = config.SEARXNG_BASE_URL; const MAX_RESULTS = 3; const TIMEOUT_MS = 8000; const CACHE_TTL = 86400; // 24 hours const CACHE_PREFIX = "searxng:"; let redis: Redis | null = null; /** * Exposes the shared SearXNG Redis connection so other modules (e.g. the * term glossary) reuse the same connection and cache prefix instead of * opening their own. Returns null when Redis is unavailable. */ export function getSearxngRedis(): Redis | null { return redis; } /** Builds a namespaced SearXNG cache key (shared across modules). */ export function makeSearxngCacheKey(namespace: string, key: string): string { return `${CACHE_PREFIX}${namespace}:${key.toLowerCase().trim()}`; } /** Reads a value from the SearXNG Redis cache; null on miss/unavailable. */ export async function searxngCacheGet(key: string): Promise { if (!redis) return null; try { return await redis.get(key); } catch { return null; } } /** Writes a value to the SearXNG Redis cache, fire-and-forget. */ export function searxngCacheSet( key: string, value: string, ttlSeconds: number, ): void { if (!redis) return; redis.setex(key, ttlSeconds, value).catch(() => { // Cache write failed silently }); } /** * Initialize Redis connection for SearXNG cache. * Safe to call multiple times — only creates one connection. */ export function initSearxngCache(redisUrl: string): void { if (redis) return; // Dedicated Redis connection needed because: this connection serves as an // optional cache for SearXNG web search results with graceful degradation // when Redis is unavailable (lazyConnect + null-assignment on failure). // It uses custom retry strategy and must not block or break the main event // pipeline if the cache is down. redis = new Redis(redisUrl, { maxRetriesPerRequest: 3, retryStrategy(times) { const delay = Math.min(times * 200, 2000); return delay; }, lazyConnect: true, enableReadyCheck: false, }); redis.on("error", (err) => { log.warn({ err: err.message }, "SearXNG Redis cache error"); }); redis.connect().catch(() => { log.warn("SearXNG Redis cache unavailable — falling back to no-cache"); redis = null; }); log.info("SearXNG Redis cache initialized"); } export interface SearxngResult { title: string; url: string; snippet: string; } /** * Search SearXNG for a query and return structured results. * Uses Redis cache when available — same query within 24h returns cached results. * * @param engines Optional comma-separated SearXNG engine list to constrain * the search (e.g. "wikipedia"). When set, results are cached under a * separate cache namespace so engine-specific results never collide. */ export async function searchSearxng( query: string, category: "general" | "news" | "science" = "general", engines?: string, timeoutMs: number = TIMEOUT_MS, ): Promise { const engineNs = engines ? `eng:${engines}` : "auto"; const cacheKey = makeSearxngCacheKey(`${category}:${engineNs}`, query); // Try cache first if (redis) { try { const cached = await redis.get(cacheKey); if (cached) { log.debug({ query, category, engines }, "SearXNG cache HIT"); return JSON.parse(cached) as SearxngResult[]; } } catch { // Cache read failed, continue to API } } // Cache miss — hit SearXNG API try { const engineParam = engines ? `&engines=${encodeURIComponent(engines)}` : ""; const url = `${SEARXNG_BASE_URL}/search?q=${encodeURIComponent(query)}&format=json&language=id&categories=${category}${engineParam}`; const { controller, clear } = createAbortControllerWithTimeout(timeoutMs); try { const response = await fetch(url, { signal: controller.signal, headers: { Accept: "application/json", "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", }, }); if (!response.ok) { log.warn({ status: response.status, query }, "SearXNG search failed"); return []; } const data = (await response.json()) as { results?: Array<{ title?: string; url?: string; content?: string }>; }; const results = data.results ?? []; const mapped = results.slice(0, MAX_RESULTS).map((r) => ({ title: r.title ?? "", url: r.url ?? "", snippet: (r.content ?? "").slice(0, 500), })); // Store in cache (fire and forget — don't block on write) if (redis) { redis.setex(cacheKey, CACHE_TTL, JSON.stringify(mapped)).catch(() => { // Cache write failed silently }); } log.debug( { query, category, resultCount: mapped.length }, "SearXNG search OK", ); return mapped; } finally { clear(); } } catch (err) { log.warn( { error: err instanceof Error ? err.message : String(err), query }, "SearXNG search error", ); return []; } } /** * Extract meaningful search queries from message content. * Uses multiple strategies to find terms worth searching. * Returns up to 3 clean queries. */ export function extractSearchQueries(content: string): string[] { const queries = new Set(); // 1. Quoted phrases (explicit user intent) const quotedPhrases = content.match(/"([^"]+)"|'([^']+)'/g); if (quotedPhrases) { for (const phrase of quotedPhrases) { const clean = phrase.replace(/["']/g, "").trim(); if (clean.length >= 3) queries.add(clean); } } // 2. "nonton X" pattern — extract the title const nontonMatch = content.match( /\b(nonton|tonton|rekomen|cari|search|google)\s+(.+?)(?:\s+(?:anime|kartun|film|movie|series|serial))?\s*[!?.]*$/i, ); if (nontonMatch) { const title = nontonMatch[2].trim(); if (title.length >= 2 && title.length <= 80) { queries.add(title); } } // 3. "X anime/film" pattern — title before category const titleBeforeCategory = content.match( /\b(\w[\w\s]{2,40})\s+(?:anime|kartun|film|movie|series|serial)\b/i, ); if (titleBeforeCategory) { const title = titleBeforeCategory[1].trim(); if ( title.length >= 3 && !/^(yang|yang|sama|dari|untuk|ini|itu|ada)$/i.test(title) ) { queries.add(title); } } // 4. Standalone proper nouns (2+ words, capitalized) that look like titles const properNouns = content.match( /\b([A-Z][a-z]+(?:\s+[A-Z][a-z]+){1,4})\b/g, ); if (properNouns) { for (const noun of properNouns) { // Skip common non-title proper nouns const skip = /^(Discord|YouTube|Google|Facebook|Instagram|Twitter|Github|ChatGPT|OpenAI|Claude|Telegram|WhatsApp|TikTok|Netflix|Spotify|Steam|Instagram)$/i; if (!skip.test(noun) && noun.length >= 5) { queries.add(noun); } } } // 5. Terms that suggest research intent const researchTerms = content.match( /\b(apa\s+(?:itu|sih)|what\s+is|siapa\s+itu|who\s+is|arti|meaning|definisi|definition)\s+(.{3,60})/i, ); if (researchTerms) { const term = researchTerms[2].trim().replace(/[?!.]+$/, ""); if (term.length >= 3) queries.add(term); } return Array.from(queries).slice(0, 3); } /** * Format SearXNG results as XML for LLM context. */ export function formatSearchResults(results: SearxngResult[]): string { if (results.length === 0) return ""; const lines = results.map( (r) => ` ${escapeXml(r.snippet)}`, ); return `\n${lines.join("\n")}\n`; } function escapeXml(str: string): string { return str .replace(/&/g, "&") .replace(//g, ">") .replace(/"/g, """); }