diff --git a/services/discord-gateway/drizzle/migrations/0014_add_term_glossary_cache.sql b/services/discord-gateway/drizzle/migrations/0014_add_term_glossary_cache.sql new file mode 100644 index 0000000..3631ce7 --- /dev/null +++ b/services/discord-gateway/drizzle/migrations/0014_add_term_glossary_cache.sql @@ -0,0 +1,9 @@ +CREATE TABLE IF NOT EXISTS "term_glossary_cache" ( + "term" text PRIMARY KEY NOT NULL, + "definition" text NOT NULL, + "source_url" text DEFAULT '' NOT NULL, + "resolved_at" bigint NOT NULL, + "hit_count" integer DEFAULT 0 NOT NULL +); +--> statement-breakpoint +CREATE INDEX IF NOT EXISTS "idx_term_glossary_cache_resolved_at" ON "term_glossary_cache" USING btree ("resolved_at"); diff --git a/services/discord-gateway/drizzle/migrations/meta/_journal.json b/services/discord-gateway/drizzle/migrations/meta/_journal.json index 2b44f01..7f8a41e 100644 --- a/services/discord-gateway/drizzle/migrations/meta/_journal.json +++ b/services/discord-gateway/drizzle/migrations/meta/_journal.json @@ -99,6 +99,13 @@ "when": 1785551832190, "tag": "0013_rename_mascot_chat_to_chatbot", "breakpoints": true + }, + { + "idx": 14, + "version": "7", + "when": 1785621600000, + "tag": "0014_add_term_glossary_cache", + "breakpoints": true } ] } \ No newline at end of file diff --git a/services/discord-gateway/src/modules/ai-moderation/searxngSearch.ts b/services/discord-gateway/src/modules/ai-moderation/searxngSearch.ts index 27535c1..22f7d98 100644 --- a/services/discord-gateway/src/modules/ai-moderation/searxngSearch.ts +++ b/services/discord-gateway/src/modules/ai-moderation/searxngSearch.ts @@ -1,10 +1,11 @@ import Redis from "ioredis"; import { createChildLogger } from "@/shared/logger/index"; import { createAbortControllerWithTimeout } from "@/shared/utils/index"; +import { config } from "../../shared/config/config.js"; const log = createChildLogger("searxng-search"); -const SEARXNG_BASE_URL = "https://searxng.imrnes.team"; +const SEARXNG_BASE_URL = config.SEARXNG_BASE_URL; const MAX_RESULTS = 3; const TIMEOUT_MS = 8000; const CACHE_TTL = 86400; // 24 hours diff --git a/services/discord-gateway/src/modules/ai-moderation/termGlossary.ts b/services/discord-gateway/src/modules/ai-moderation/termGlossary.ts index 7274cfd..1dd4bbe 100644 --- a/services/discord-gateway/src/modules/ai-moderation/termGlossary.ts +++ b/services/discord-gateway/src/modules/ai-moderation/termGlossary.ts @@ -14,18 +14,26 @@ * the LLM prompt as a `` block so verdicts are based on facts * instead of guesses. * - * Cost control: - * - definitions are cached in an in-memory LRU AND in Redis (shared with the - * SearXNG cache) — a term is searched at most once per TTL across the whole - * service, so repeat lookups are effectively free; + * Cost control & persistence: + * - successfully resolved definitions are PERSISTED PERMANENTLY in Postgres + * (`term_glossary_cache`) — definitions rarely change, so a resolved term + * is never searched again; only misses stay ephemeral (Redis/LRU, 1h); + * - in-memory LRU + Redis (shared with the SearXNG cache) sit in front of + * the DB as fast read caches, so repeat lookups are effectively free; * - lookups per batch are bounded (AI_GLOSSARY_MAX_TERMS); - * - Wikipedia-only search first, generic search as a fallback; - * - everything degrades gracefully: no Redis, no SearXNG, no Wikipedia match + * - live SearXNG calls are rate-limit aware: concurrency 2 + stagger, retry + * once on empty results, and misses cached for only 1h so a limiter/ + * network blip is not treated as a permanent miss; + * - only results that read like actual definitions are accepted (Wikipedia + * preferred; disambiguation/ads/translate-homepages rejected); + * - everything degrades gracefully: no Redis, no SearXNG, no match * → the block is simply omitted and moderation proceeds as before. */ import { LRUCache } from "lru-cache"; +import pLimit from "p-limit"; import { createChildLogger } from "@/shared/logger/index"; +import { delay } from "@/shared/utils/index"; import { config } from "../../shared/config/config.js"; import { escapeXml } from "./moderationBuilders.js"; import { @@ -34,6 +42,10 @@ import { searxngCacheGet, searxngCacheSet, } from "./searxngSearch.js"; +import { + getTermDefinitionFromDb, + setTermDefinitionInDb, +} from "./termGlossaryStore.js"; const log = createChildLogger("term-glossary"); @@ -43,14 +55,29 @@ const log = createChildLogger("term-glossary"); /** Redis TTL for a successfully resolved definition (definitions are stable). */ const DEF_TTL_SECONDS = 7 * 24 * 60 * 60; -/** Redis TTL for a lookup that found nothing — don't re-search every batch. */ -const MISS_TTL_SECONDS = 24 * 60 * 60; +/** + * Redis TTL for a lookup that found nothing. Kept SHORT (1h): SearXNG + * instances silently return empty result sets when rate-limited, so an empty + * response is often a transient failure, not a real miss. A short TTL lets + * the term be retried on a later batch instead of poisoning it for a day. + */ +const MISS_TTL_SECONDS = 60 * 60; +const MISS_TTL_MS = MISS_TTL_SECONDS * 1000; /** Sentinel stored in caches for "term has no resolvable definition". */ const EMPTY_SENTINEL = "__not_found__"; /** Per-search timeout — keep glossary lookups snappy even on a slow SearXNG. */ const GLOSSARY_SEARCH_TIMEOUT_MS = 5000; +/** Delay before retrying a search that returned zero results. */ +const RETRY_DELAY_MS = 350; /** Max definition snippet length kept in the prompt. */ const MAX_DEFINITION_CHARS = 300; +/** + * SearXNG rate-limits aggressive parallel bursts (returns 200 with empty + * results). Never fire all terms at once — cap live searches at 2 concurrent + * and stagger the start times slightly. + */ +const LIVE_SEARCH_CONCURRENCY = 2; +const LIVE_SEARCH_STAGGER_MS = 250; /** In-memory cache: term (lowercase) → definition | NOT_FOUND sentinel. */ const NOT_FOUND: TermDefinition = { @@ -63,6 +90,16 @@ const termLru = new LRUCache({ ttl: 24 * 60 * 60 * 1000, }); +/** Serializes live SearXNG lookups (rate-limit aware) with a small stagger. */ +const liveSearchLimit = pLimit(LIVE_SEARCH_CONCURRENCY); +let lastLiveSearchAt = 0; +async function acquireLiveSlot(): Promise { + const now = Date.now(); + const wait = lastLiveSearchAt + LIVE_SEARCH_STAGGER_MS - now; + if (wait > 0) await delay(wait); + lastLiveSearchAt = Date.now(); +} + // --------------------------------------------------------------------------- // Term extraction // --------------------------------------------------------------------------- @@ -78,7 +115,7 @@ function cleanContent(raw: string): string { .replace(/<@!?\d+>/g, " ") .replace(/<#\d+>/g, " ") .replace(//g, " ") - .replace(/[`*_~|>\[\]]/g, " ") + .replace(/[`*_~|>[\]]/g, " ") .replace(/[\p{Emoji}\p{Extended_Pictographic}]/gu, " ") .replace(/\s+/g, " ") .trim(); @@ -237,15 +274,64 @@ export interface TermDefinition { sourceUrl: string; } -/** Picks the best definition from search results, preferring Wikipedia. */ +/** Definition-like markers for accepting a non-Wikipedia search result. */ +const DEF_MARKERS = + /adalah|merupakan|istilah (?:untuk|yang|yg)|artinya|sebutan|berarti|refers? to|known as|also called|short for|a term (?:for|used)|istilah dalam|kata (?:asing|serapan)? ?untuk/i; + +/** True when the term appears in the result text (or a 4+ char word in the + * result is part of the term). Lenient — "kafircel" matches a "Kafir" + * article via substring, while a Google-Translate homepage snippet does not. */ +function hasTermOverlap(term: string, title: string, snippet: string): boolean { + const termLower = term.toLowerCase(); + const text = `${title} ${snippet}`.toLowerCase(); + if (text.includes(termLower)) return true; + const words = text.match(/[a-z0-9]{4,}/gi) ?? []; + return words.some((w) => termLower.includes(w)); +} + +/** Quality gate: is this result good enough to quote as a definition? */ +function isUsableDefinition( + r: { title: string; url: string; snippet: string }, + term: string, + isWiki: boolean, +): boolean { + const text = `${r.title} ${r.snippet}`; + // Wikipedia disambiguation pages are not definitions + if (/disambiguasi|disambiguation/i.test(text)) return false; + if ((r.snippet ?? "").trim().length < 25) return false; + if (!hasTermOverlap(term, r.title, r.snippet)) return false; + // Wikipedia articles are accepted with just the overlap+length gate; + // everything else must read like an actual definition, not an ad, + // a translate homepage, or a navigation blurb. + if (isWiki) return true; + return DEF_MARKERS.test(r.snippet); +} + +/** Picks the best definition from search results, preferring a genuine + * Wikipedia article; otherwise the first result that reads like a + * definition. Returns null when nothing qualifies. */ function pickDefinition( results: Array<{ title: string; url: string; snippet: string }>, term: string, ): TermDefinition | null { - const best = results.find((r) => /wikipedia/i.test(r.url)) ?? results[0]; - if (!best) return null; + const wiki = results.find((r) => /wikipedia\.org/i.test(r.url)); + const best = wiki && isUsableDefinition(wiki, term, true) ? wiki : null; + if (!best) { + for (const r of results) { + if (isUsableDefinition(r, term, false)) { + return buildDefinition(r, term); + } + } + return null; + } + return buildDefinition(best, term); +} + +function buildDefinition( + best: { title: string; url: string; snippet: string }, + term: string, +): TermDefinition { const snippet = (best.snippet || best.title || "").trim(); - if (snippet.length < 10) return null; const definition = snippet.length > MAX_DEFINITION_CHARS ? `${snippet.slice(0, MAX_DEFINITION_CHARS - 1).trimEnd()}…` @@ -253,94 +339,140 @@ function pickDefinition( return { term, definition, sourceUrl: best.url }; } -async function lookupTermDefinition( +/** Live (network) lookup — runs under the shared SearXNG rate-limit gate. */ +async function fetchDefinitionLive( term: string, + key: string, + cacheKey: string, ): Promise { + return liveSearchLimit(async () => { + await acquireLiveSlot(); + try { + let results = await searchSearxng( + key, + "general", + undefined, + GLOSSARY_SEARCH_TIMEOUT_MS, + ); + let def = pickDefinition(results, term); + // Zero results is usually the limiter kicking in, not a real miss — + // retry once. Results-but-unusable = genuine miss, no retry. + if (!def && results.length === 0) { + await delay(RETRY_DELAY_MS); + results = await searchSearxng( + key, + "general", + undefined, + GLOSSARY_SEARCH_TIMEOUT_MS, + ); + def = pickDefinition(results, term); + } + + if (def) { + // Persist permanently (definitions rarely change) — best-effort, + // then warm the fast caches. + void setTermDefinitionInDb(key, def.definition, def.sourceUrl); + searxngCacheSet( + cacheKey, + JSON.stringify({ + definition: def.definition, + sourceUrl: def.sourceUrl, + }), + DEF_TTL_SECONDS, + ); + termLru.set(key, def); + log.debug({ term: key }, "Term glossary resolved definition"); + return def; + } + } catch (err) { + log.debug( + { term: key, error: err instanceof Error ? err.message : String(err) }, + "Term glossary lookup failed — skipping term", + ); + } + + // No definition — cache the miss with a SHORT TTL so a transient + // limiter/network failure is retried on a later batch. + searxngCacheSet(cacheKey, EMPTY_SENTINEL, MISS_TTL_SECONDS); + termLru.set(key, NOT_FOUND, { ttl: MISS_TTL_MS }); + return null; + }); +} + +/** Resolve one term: LRU → Redis → Postgres (permanent) → live SearXNG + * (rate-limited). The fast caches sit in front of the DB; the DB is the + * source of truth for successfully resolved definitions. */ +async function resolveTerm(term: string): Promise { const key = term.toLowerCase().trim(); // 1. In-memory LRU — same process, instant const lruHit = termLru.get(key); if (lruHit) return lruHit === NOT_FOUND ? null : lruHit; - // 2. Redis — shared across processes/workers + // 2. Redis — shared across processes/workers. A miss sentinel here is NOT + // a definitive answer: it may predate a permanent DB entry written by + // another process, so we keep going and let the DB decide. const cacheKey = makeSearxngCacheKey("def", key); const cached = await searxngCacheGet(cacheKey); + let redisMiss = false; if (cached !== null) { if (cached === EMPTY_SENTINEL) { - termLru.set(key, NOT_FOUND); - return null; - } - try { - const parsed = JSON.parse(cached) as { - definition?: string; - sourceUrl?: string; - }; - if (parsed.definition) { - const def: TermDefinition = { - term, - definition: parsed.definition, - sourceUrl: parsed.sourceUrl ?? "", + redisMiss = true; + } else { + try { + const parsed = JSON.parse(cached) as { + definition?: string; + sourceUrl?: string; }; - termLru.set(key, def); - return def; + if (parsed.definition) { + const def: TermDefinition = { + term, + definition: parsed.definition, + sourceUrl: parsed.sourceUrl ?? "", + }; + termLru.set(key, def); + return def; + } + } catch { + // malformed cache entry — fall through to DB/live } - } catch { - // malformed cache entry — fall through to search } } - // 3. Live search — Wikipedia first, generic search as fallback - try { - let def = pickDefinition( - await searchSearxng( - key, - "general", - "wikipedia", - GLOSSARY_SEARCH_TIMEOUT_MS, - ), + // 3. Postgres — permanent store for resolved definitions. A hit re-warms + // the fast caches so the DB is not hit on every batch. + const dbDef = await getTermDefinitionFromDb(key); + if (dbDef) { + const def: TermDefinition = { term, + definition: dbDef.definition, + sourceUrl: dbDef.sourceUrl, + }; + termLru.set(key, def); + searxngCacheSet( + cacheKey, + JSON.stringify({ definition: def.definition, sourceUrl: def.sourceUrl }), + DEF_TTL_SECONDS, ); - if (!def) { - def = pickDefinition( - await searchSearxng( - `${key} definisi arti`, - "general", - undefined, - GLOSSARY_SEARCH_TIMEOUT_MS, - ), - term, - ); - } - - if (def) { - searxngCacheSet( - cacheKey, - JSON.stringify({ - definition: def.definition, - sourceUrl: def.sourceUrl, - }), - DEF_TTL_SECONDS, - ); - termLru.set(key, def); - log.debug({ term: key }, "Term glossary resolved definition"); - return def; - } - } catch (err) { - log.debug( - { term: key, error: err instanceof Error ? err.message : String(err) }, - "Term glossary lookup failed — skipping term", - ); + log.debug({ term: key }, "Term glossary DB hit"); + return def; } - // No definition — cache the miss so we do not re-search every batch. - searxngCacheSet(cacheKey, EMPTY_SENTINEL, MISS_TTL_SECONDS); - termLru.set(key, NOT_FOUND); - return null; + // 4. Redis already said "miss" recently and the DB has nothing — respect + // that instead of hammering SearXNG again within the miss window. + if (redisMiss) { + termLru.set(key, NOT_FOUND, { ttl: MISS_TTL_MS }); + return null; + } + + // 5. Live search (rate-limited + staggered) + return fetchDefinitionLive(term, key, cacheKey); } /** * Looks up definitions for a batch of terms, in parallel. Returns a map of * term → definition for the terms that resolved. Errors/misses are skipped. + * Live SearXNG calls are throttled internally (concurrency 2 + stagger). */ export async function lookupTermDefinitions( terms: string[], @@ -348,7 +480,7 @@ export async function lookupTermDefinitions( const map = new Map(); if (terms.length === 0) return map; - const results = await Promise.allSettled(terms.map(lookupTermDefinition)); + const results = await Promise.allSettled(terms.map(resolveTerm)); for (let i = 0; i < terms.length; i++) { const r = results[i]; if (r.status === "fulfilled" && r.value) { diff --git a/services/discord-gateway/src/modules/ai-moderation/termGlossaryStore.ts b/services/discord-gateway/src/modules/ai-moderation/termGlossaryStore.ts new file mode 100644 index 0000000..a136567 --- /dev/null +++ b/services/discord-gateway/src/modules/ai-moderation/termGlossaryStore.ts @@ -0,0 +1,86 @@ +/** + * termGlossaryStore.ts + * + * Permanent Postgres layer for the term glossary. Resolved definitions + * (which carry content) are persisted here because they rarely change — + * Redis/LRU only act as fast read caches in front of this table. Terms with + * no definition (misses) are deliberately NOT persisted; they stay ephemeral + * in Redis with a short TTL so transient lookup failures get retried. + * + * All calls are best-effort: any DB error degrades to a cache miss (the + * glossary then falls through to Redis/live search as if the DB layer + * didn't exist). + */ + +import { createChildLogger } from "@/shared/logger/index"; +import { executeAll, executeGet } from "../../shared/database/drizzle.js"; + +const log = createChildLogger("term-glossary-store"); + +export interface StoredTermDefinition { + definition: string; + sourceUrl: string; +} + +/** + * Read a permanently stored definition for a term (lowercase key). + * Returns null when missing or on any DB error (callers fall through). + * A successful read bumps hit_count for observability (fire-and-forget). + */ +export async function getTermDefinitionFromDb( + term: string, +): Promise { + try { + const row = await executeGet( + `SELECT definition, source_url FROM term_glossary_cache WHERE term = $1`, + [term.toLowerCase().trim()], + ); + if (!row) return null; + try { + await executeAll( + `UPDATE term_glossary_cache SET hit_count = hit_count + 1 WHERE term = $1`, + [term.toLowerCase().trim()], + ); + } catch { + // hit_count is observability only — never fail a read for it + } + return { + definition: row.definition as string, + sourceUrl: (row.source_url as string | null) ?? "", + }; + } catch (error) { + log.debug( + { error: error instanceof Error ? error.message : String(error) }, + "getTermDefinitionFromDb failed — falling back to live search", + ); + return null; + } +} + +/** + * Persist a resolved definition permanently (UPSERT by term). + * Only called for successful resolutions — never for misses. + * Best-effort: a DB write failure does not affect the returned definition. + */ +export async function setTermDefinitionInDb( + term: string, + definition: string, + sourceUrl: string, +): Promise { + try { + await executeAll( + `INSERT INTO term_glossary_cache (term, definition, source_url, resolved_at, hit_count) + VALUES ($1, $2, $3, $4, 0) + ON CONFLICT (term) DO UPDATE SET + definition = EXCLUDED.definition, + source_url = EXCLUDED.source_url, + resolved_at = EXCLUDED.resolved_at`, + [term.toLowerCase().trim(), definition, sourceUrl, Date.now()], + ); + } catch (error) { + log.warn( + { error: error instanceof Error ? error.message : String(error) }, + "setTermDefinitionInDb failed — definition stays memory/Redis only", + ); + } +} diff --git a/services/discord-gateway/src/shared/config/index.ts b/services/discord-gateway/src/shared/config/index.ts index 574e3cb..d8567d2 100644 --- a/services/discord-gateway/src/shared/config/index.ts +++ b/services/discord-gateway/src/shared/config/index.ts @@ -92,6 +92,10 @@ export const configSchema = z // ── Redis ──────────────────────────────────────────────────────────── REDIS_URL: z.string().default("redis://localhost:6379"), + // ── SearXNG ─────────────────────────────────────────────────────────── + // Instance for web search + term glossary lookups. Override when the + // default instance is down/rate-limited. + SEARXNG_BASE_URL: z.string().url().default("https://searxng.imrnes.team"), // ── Voice PCM WebSocket (direct gateway→backend, bypasses Redis) ──── VOICE_PCM_WS_ENABLED: z .string() diff --git a/services/discord-gateway/src/shared/database/schema.ts b/services/discord-gateway/src/shared/database/schema.ts index e372464..7715a9d 100644 --- a/services/discord-gateway/src/shared/database/schema.ts +++ b/services/discord-gateway/src/shared/database/schema.ts @@ -435,6 +435,32 @@ export const pgStickerCacheTable = pgTable( export const stickerCacheTable = pgStickerCacheTable; +/** + * Term Glossary Cache Table (PostgreSQL) + * Permanently stores resolved term definitions (Wikipedia/SearXNG lookups). + * Definitions rarely change, so once a term is successfully resolved it is + * persisted here forever — Redis/LRU only act as fast read caches in front. + * Terms with NO definition (misses) are NOT stored here; they stay ephemeral + * in Redis with a short TTL so transient lookup failures get retried. + */ +export const pgTermGlossaryCacheTable = pgTable( + "term_glossary_cache", + { + term: pgText("term").primaryKey(), + definition: pgText("definition").notNull(), + source_url: pgText("source_url").notNull().default(""), + resolved_at: pgBigint("resolved_at", { mode: "number" }).notNull(), + hit_count: pgInteger("hit_count").notNull().default(0), + }, + (table) => ({ + resolvedAtIdx: pgIndex("idx_term_glossary_cache_resolved_at").on( + table.resolved_at, + ), + }), +); + +export const termGlossaryCacheTable = pgTermGlossaryCacheTable; + // ============================================================================= // Meta / System // ============================================================================= @@ -580,6 +606,11 @@ export type TextAnalysisCacheInsert = export type StickerCacheRecord = typeof stickerCacheTable.$inferSelect; export type StickerCacheInsert = typeof stickerCacheTable.$inferInsert; +// Term Glossary Cache +export type TermGlossaryCache = typeof termGlossaryCacheTable.$inferSelect; +export type TermGlossaryCacheInsert = + typeof termGlossaryCacheTable.$inferInsert; + // Muxer Jobs export type MuxerJob = typeof muxerJobsTable.$inferSelect; export type MuxerJobInsert = typeof muxerJobsTable.$inferInsert;