diff --git a/services/discord-gateway/src/modules/ai-moderation/termGlossary.ts b/services/discord-gateway/src/modules/ai-moderation/termGlossary.ts index d5485c8b..e5b8bec5 100644 --- a/services/discord-gateway/src/modules/ai-moderation/termGlossary.ts +++ b/services/discord-gateway/src/modules/ai-moderation/termGlossary.ts @@ -49,6 +49,7 @@ import { scoreWord, WORD_RE, } from "./textSignals.js"; +import { isTinyFishEnabled, tinyFishSearchLive } from "./tinyFishSearch.js"; import { wikipediaSummary } from "./wikipediaClient.js"; const log = createChildLogger("term-glossary"); @@ -212,7 +213,17 @@ async function fetchDefinitionLive( // (up to 4 requests per term under stagger+concurrency-2 — pure // Wikipedia pressure for a miss that gets cached 1h anyway). const result = await wikipediaSummary(key, GLOSSARY_SEARCH_TIMEOUT_MS); - const def = result ? buildDefinition(result, term) : null; + let def = result ? buildDefinition(result, term) : null; + + // Wikipedia miss (null result, no throw): one TinyFish web-search + // attempt — its top hit's snippet becomes the definition. Disabled/ + // no-key returns [] instantly. A hit is persisted permanently like a + // Wikipedia definition; a miss keeps the short-TTL miss sentinel. + if (!def && isTinyFishEnabled()) { + const fallback = await tinyFishSearchLive(key); + const top = fallback[0]; + if (top) def = buildDefinition(top, term); + } if (def) { // Persist permanently (definitions rarely change) — best-effort, diff --git a/services/discord-gateway/src/modules/ai-moderation/tinyFishSearch.ts b/services/discord-gateway/src/modules/ai-moderation/tinyFishSearch.ts new file mode 100644 index 00000000..d927bc2b --- /dev/null +++ b/services/discord-gateway/src/modules/ai-moderation/tinyFishSearch.ts @@ -0,0 +1,111 @@ +import { createChildLogger } from "@/shared/logger/index"; +import { createAbortControllerWithTimeout } from "@/shared/utils/index"; +import { config } from "../../shared/config/config.js"; +import type { SearchResult } from "./wikipediaClient.js"; + +const log = createChildLogger("tinyfish-search"); + +/** + * Raw TinyFish search result (subset of fields we use). + * Endpoint: GET {base}?query=..&location=..&language=.. with X-API-Key. + */ +interface TinyFishResult { + title?: string; + url?: string; + snippet?: string; + date?: string; +} + +interface TinyFishResponse { + query?: string; + results?: TinyFishResult[]; +} + +/** + * Fallback detection: true only when the provider is reachable AND returns a + * usable answer. Everything else (missing key, disabled, timeout, non-200, + * malformed body, zero results) returns false/null and the caller falls + * through to empty enrichment — never throws. + */ +export function isTinyFishEnabled(): boolean { + return ( + config.TINYFISH_SEARCH_ENABLED === true && + config.TINYFISH_API_KEY.length > 0 + ); +} + +function mapResult(r: TinyFishResult): SearchResult | null { + const title = (r.title ?? "").trim(); + const url = (r.url ?? "").trim(); + if (!title || !url) return null; + return { + title, + url, + snippet: (r.snippet ?? "").replace(/\s+/g, " ").trim().slice(0, 500), + }; +} + +/** + * Live TinyFish web search for one query. Returns [] on any failure + * (disabled, no key, network error, non-OK, bad JSON, no results). + * Callers treat [] the same as a Wikipedia miss — enrichment is skipped. + */ +export async function tinyFishSearchLive( + query: string, + timeoutMs: number = config.TINYFISH_SEARCH_TIMEOUT_MS, +): Promise { + const q = query.trim(); + if (!q) return []; + if (!isTinyFishEnabled()) return []; + + const params = new URLSearchParams({ + query: q, + location: config.TINYFISH_SEARCH_LOCATION, + language: config.TINYFISH_SEARCH_LANGUAGE, + }); + const { controller, clear } = createAbortControllerWithTimeout(timeoutMs); + try { + const res = await fetch( + `${config.TINYFISH_SEARCH_BASE_URL}?${params.toString()}`, + { + signal: controller.signal, + headers: { + Accept: "application/json", + "X-API-Key": config.TINYFISH_API_KEY, + }, + }, + ); + if (!res.ok) { + log.warn( + { status: res.status, query: q }, + "TinyFish search failed (non-OK) — skipping fallback", + ); + return []; + } + let data: TinyFishResponse; + try { + data = (await res.json()) as TinyFishResponse; + } catch { + log.warn({ query: q }, "TinyFish search returned invalid JSON"); + return []; + } + const results = Array.isArray(data.results) ? data.results : []; + const mapped = results + .slice(0, 3) + .map(mapResult) + .filter((r): r is SearchResult => r !== null); + log.debug( + { query: q, resultCount: mapped.length }, + "TinyFish search fallback OK", + ); + return mapped; + } catch (err) { + log.warn( + { error: err instanceof Error ? err.message : String(err), query: q }, + "TinyFish search error — skipping fallback", + ); + return []; + } finally { + clear(); + } +} diff --git a/services/discord-gateway/src/modules/ai-moderation/wikipediaClient.ts b/services/discord-gateway/src/modules/ai-moderation/wikipediaClient.ts index 41e8f060..1d2a15e5 100644 --- a/services/discord-gateway/src/modules/ai-moderation/wikipediaClient.ts +++ b/services/discord-gateway/src/modules/ai-moderation/wikipediaClient.ts @@ -28,6 +28,7 @@ import { isMostlyStopwords, scoreWord, } from "./textSignals.js"; +import { isTinyFishEnabled, tinyFishSearchLive } from "./tinyFishSearch.js"; const log = createChildLogger("wikipedia-client"); @@ -102,6 +103,19 @@ export async function wikipediaSearch( const mapped = await wikipediaSearchLive(q, timeoutMs); if (mapped.length > 0) { cacheSet(cacheKey, JSON.stringify(mapped), SEARCH_CACHE_TTL_SECONDS); + return mapped; + } + + // Wikipedia miss (empty, not error — errors already returned [] the same + // way): one TinyFish web-search attempt as fallback. Disabled/no-key + // returns [] instantly; a hit is cached under the same key so the next + // batch never pays the fallback latency again. + if (isTinyFishEnabled()) { + const fallback = await tinyFishSearchLive(q); + if (fallback.length > 0) { + cacheSet(cacheKey, JSON.stringify(fallback), SEARCH_CACHE_TTL_SECONDS); + return fallback; + } } return mapped; } diff --git a/services/discord-gateway/src/shared/config/index.ts b/services/discord-gateway/src/shared/config/index.ts index 40dc4a4a..5911104a 100644 --- a/services/discord-gateway/src/shared/config/index.ts +++ b/services/discord-gateway/src/shared/config/index.ts @@ -132,6 +132,22 @@ export const configSchema = z WIKIPEDIA_LANG: z.string().min(1).default("id"), // Per-request timeout (ms) for Wikipedia API calls. WIKIPEDIA_TIMEOUT_MS: z.coerce.number().positive().default(8000), + // ── TinyFish web search (fallback when Wikipedia misses) ───────────── + // GET {base}?query=..&location=..&language=.. with X-API-Key header. + // Empty key = fallback disabled (Wikipedia-only, tests stay offline). + TINYFISH_API_KEY: z.string().optional().default(""), + TINYFISH_SEARCH_ENABLED: z + .string() + .optional() + .transform((v) => v === "true") + .default(true), + TINYFISH_SEARCH_BASE_URL: z + .string() + .url() + .default("https://api.search.tinyfish.ai"), + TINYFISH_SEARCH_TIMEOUT_MS: z.coerce.number().positive().default(10000), + TINYFISH_SEARCH_LOCATION: z.string().min(1).default("US"), + TINYFISH_SEARCH_LANGUAGE: z.string().min(1).default("en"), // ── Voice PCM WebSocket (direct gateway→backend, bypasses Redis) ──── VOICE_PCM_WS_ENABLED: z .string() diff --git a/services/discord-gateway/tests/tinyFishSearch.test.ts b/services/discord-gateway/tests/tinyFishSearch.test.ts new file mode 100644 index 00000000..2cf0db3b --- /dev/null +++ b/services/discord-gateway/tests/tinyFishSearch.test.ts @@ -0,0 +1,150 @@ +// ═══════════════════════════════════════════════════════════════════════════ +// TinyFish web-search fallback — disabled-by-default network guard + mapping. +// No key in env (vitest config) → isTinyFishEnabled() false → all live calls +// return [] without touching the network. Enabled paths use a stubbed fetch. +// ═══════════════════════════════════════════════════════════════════════════ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { + isTinyFishEnabled, + tinyFishSearchLive, +} from "../src/modules/ai-moderation/tinyFishSearch.js"; + +function jsonResponse(body: unknown, ok = true, status = 200): Response { + return { + ok, + status, + json: async () => body, + } as unknown as Response; +} + +const TINYFISH_BODY = { + query: "shirkmaxxing", + results: [ + { + position: 1, + title: "Shirkmaxxing explained", + url: "https://example.com/shirkmaxxing", + snippet: + "Shirkmaxxing is internet slang for performative avoidance of work.", + }, + { + position: 2, + title: "No URL here", + url: "", + snippet: "dropped by the mapper", + }, + ], +}; + +describe("tinyFishSearch fallback", () => { + let prevKey: string; + let prevEnabled: unknown; + + beforeEach(async () => { + vi.restoreAllMocks(); + // Pin the live config object to a known-disabled state: the shell may + // export a real TINYFISH_API_KEY (dev box), which would flip + // isTinyFishEnabled() and let tests hit the network. + const { config } = await import("../../src/shared/config/config.js"); + prevKey = config.TINYFISH_API_KEY; + prevEnabled = config.TINYFISH_SEARCH_ENABLED; + (config as Record).TINYFISH_API_KEY = ""; + (config as Record).TINYFISH_SEARCH_ENABLED = true; + }); + + afterEach(async () => { + vi.restoreAllMocks(); + const { config } = await import("../../src/shared/config/config.js"); + (config as Record).TINYFISH_API_KEY = prevKey; + (config as Record).TINYFISH_SEARCH_ENABLED = prevEnabled; + }); + + async function withTestKey(): Promise<{ + config: Record; + prev: string; + }> { + const { config } = await import("../../src/shared/config/config.js"); + const prev = config.TINYFISH_API_KEY; + (config as Record).TINYFISH_API_KEY = "sk-test-key"; + return { config: config as unknown as Record, prev }; + } + + it("is disabled without an API key — live search returns [] with no fetch", async () => { + expect(isTinyFishEnabled()).toBe(false); + const fetchMock = vi.spyOn(globalThis, "fetch"); + expect(await tinyFishSearchLive("shirkmaxxing")).toEqual([]); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it("returns [] for blank queries without fetching", async () => { + const fetchMock = vi.spyOn(globalThis, "fetch"); + expect(await tinyFishSearchLive(" ")).toEqual([]); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it("maps results to SearchResult shape and drops hits without URL", async () => { + vi.spyOn(globalThis, "fetch").mockResolvedValueOnce( + jsonResponse(TINYFISH_BODY), + ); + const { config, prev: prevKey } = await withTestKey(); + try { + expect(isTinyFishEnabled()).toBe(true); + const out = await tinyFishSearchLive("shirkmaxxing"); + expect(out).toHaveLength(1); + expect(out[0]).toMatchObject({ + title: "Shirkmaxxing explained", + url: "https://example.com/shirkmaxxing", + }); + expect(out[0].snippet).toContain("performative avoidance"); + const calledUrl = String( + (globalThis.fetch as unknown as { mock: { calls: unknown[][] } }).mock + .calls[0][0], + ); + expect(calledUrl).toContain("query=shirkmaxxing"); + expect(calledUrl).toContain("location="); + expect(calledUrl).toContain("language="); + } finally { + (config as Record).TINYFISH_API_KEY = prevKey; + } + }); + + it("returns [] on non-OK status without throwing", async () => { + const { config, prev: prevKey } = await withTestKey(); + try { + vi.spyOn(globalThis, "fetch").mockResolvedValueOnce( + jsonResponse({ error: "boom" }, false, 502), + ); + expect(await tinyFishSearchLive("shirkmaxxing")).toEqual([]); + } finally { + (config as Record).TINYFISH_API_KEY = prevKey; + } + }); + + it("returns [] on network error without throwing", async () => { + const { config, prev: prevKey } = await withTestKey(); + try { + vi.spyOn(globalThis, "fetch").mockRejectedValueOnce( + new Error("network down"), + ); + expect(await tinyFishSearchLive("shirkmaxxing")).toEqual([]); + } finally { + (config as Record).TINYFISH_API_KEY = prevKey; + } + }); + + it("returns [] on invalid JSON without throwing", async () => { + const { config, prev: prevKey } = await withTestKey(); + try { + vi.spyOn(globalThis, "fetch").mockResolvedValueOnce({ + ok: true, + status: 200, + json: async () => { + throw new Error("bad json"); + }, + } as unknown as Response); + expect(await tinyFishSearchLive("shirkmaxxing")).toEqual([]); + } finally { + (config as Record).TINYFISH_API_KEY = prevKey; + } + }); +});