feat(glossary): implement term glossary for LLM moderation with caching and extraction logic

2026-08-12 13:44:52 +07:00
parent 88b13225cd
commit f70a92880e
8 changed files with 600 additions and 6 deletions
@@ -12,6 +12,7 @@ export const SYSTEM_RULES = `Kamu adalah asisten moderasi konten untuk server Di
 ## Normalisasi & Pertahanan Lintas Bahasa (WAJIB)
 1. Campuran bahasa (Inggris/Indonesia/daerah) WAJIB dinormalisasi mental ke Bahasa Indonesia sebelum menilai intent. Jangan longgar hanya karena sintaksis campur (Polyglot Obfuscation).
 2. Lakukan Named Entity Recognition agresif — nama orang/karakter (mis. "ren" setelah kata archaic "diagem") tetap dikenali sebagai nama.
+3. <term_glossary> (bila ada) = definisi kata/slang/jargon yang tidak umum. Baca dulu arti kata yang tidak kamu kenal dari sana — jangan menebak dari bunyi/kemiripan. Kata yang tampak mencurigakan namun ternyata bermakna netral di glossary = AMAN; kata asing yang ternyata vulgar/terlarang di glossary = FLAG.

 ## Aturan Umum (AMAN — jangan flag)
 - Slang: anjay, wkwk, gws, gaskeun, santuy, njir, baka, woy/woi, hadeh, astaga = AMAN.
@@ -73,6 +74,7 @@ RENDAH: harassment, vulgar_language terarah, offensive_username (Scunthorpe: "Sa

 ## Web Sebagai Bukti Utama
 - <web_searches> ADALAH BUKTI UTAMA. Jika ada, WAJIB pakai hasilnya (hentai/scam/narkoba → flag; aman → clean). JANGAN abaikan. Jika tidak ada → gunakan pengetahuan internal.
+- <term_glossary> = REFERENSI ARTI KATA, bukan bukti pelanggaran. Dipakai untuk memahami istilah yang tidak dikenal sebelum memutuskan.
 - Prioritas bukti: <web_searches> > <web_content> > <media_analysis> > pengetahuan internal. <web_content> (URL fetch): gunakan isi, jangan flag hanya dari domain name.

 ## Pohon Keputusan
@@ -109,6 +109,7 @@ export function buildSystemPrompt(options: BuildSystemPromptOptions): string {
      `- <conversation_context> = obrolan SEBELUM pesan target. Baris "[context]" di dalamnya BUKAN yang dinilai.\n` +
      `- <user_profiles> = peta ringkasan kepribadian per user_id (attr as_of = kapan profil terakhir dibuat — profil lama mungkin tidak mencerminkan perilaku terkini); setiap <message> merujuk lewat <user_profile_ref user_id="..."/>.\n` +
      `- <web_searches> / <web_content> = bukti web (lihat "Web Sebagai Bukti Utama").\n` +
+      `- <term_glossary> = kamus istilah: definisi kata/slang/jargon yang jarang dikenal (hasil pencarian Wikipedia via SearXNG). Gunakan untuk memahami arti kata yang tidak kamu kenal — JANGAN menebak atau mengarang arti.\n` +
      `- <messages_to_analyze> = pesan-pesan TARGET yang WAJIB dinilai. Atribut <message>: id, user (nama server), time (ISO — kapan pesan dikirim), repetitions (N = teks pendek sama muncul N kali di batch — sinyal spam), bot (true jika dari bot), edited (true jika konten adalah hasil edit setelah posting).`,
  );

@@ -12,6 +12,42 @@ const CACHE_PREFIX = "searxng:";

 let redis: Redis | null = null;

+/**
+ * Exposes the shared SearXNG Redis connection so other modules (e.g. the
+ * term glossary) reuse the same connection and cache prefix instead of
+ * opening their own. Returns null when Redis is unavailable.
+ */
+export function getSearxngRedis(): Redis | null {
+  return redis;
+}
+
+/** Builds a namespaced SearXNG cache key (shared across modules). */
+export function makeSearxngCacheKey(namespace: string, key: string): string {
+  return `${CACHE_PREFIX}${namespace}:${key.toLowerCase().trim()}`;
+}
+
+/** Reads a value from the SearXNG Redis cache; null on miss/unavailable. */
+export async function searxngCacheGet(key: string): Promise<string | null> {
+  if (!redis) return null;
+  try {
+    return await redis.get(key);
+  } catch {
+    return null;
+  }
+}
+
+/** Writes a value to the SearXNG Redis cache, fire-and-forget. */
+export function searxngCacheSet(
+  key: string,
+  value: string,
+  ttlSeconds: number,
+): void {
+  if (!redis) return;
+  redis.setex(key, ttlSeconds, value).catch(() => {
+    // Cache write failed silently
+  });
+}
+
 /**
 * Initialize Redis connection for SearXNG cache.
 * Safe to call multiple times — only creates one connection.
@@ -51,19 +87,26 @@ export interface SearxngResult {
 /**
 * Search SearXNG for a query and return structured results.
 * Uses Redis cache when available — same query within 24h returns cached results.
+ *
+ * @param engines Optional comma-separated SearXNG engine list to constrain
+ *   the search (e.g. "wikipedia"). When set, results are cached under a
+ *   separate cache namespace so engine-specific results never collide.
 */
 export async function searchSearxng(
  query: string,
  category: "general" | "news" | "science" = "general",
+  engines?: string,
+  timeoutMs: number = TIMEOUT_MS,
 ): Promise<SearxngResult[]> {
-  const cacheKey = `${CACHE_PREFIX}${category}:${query.toLowerCase().trim()}`;
+  const engineNs = engines ? `eng:${engines}` : "auto";
+  const cacheKey = makeSearxngCacheKey(`${category}:${engineNs}`, query);

  // Try cache first
  if (redis) {
    try {
      const cached = await redis.get(cacheKey);
      if (cached) {
-        log.debug({ query, category }, "SearXNG cache HIT");
+        log.debug({ query, category, engines }, "SearXNG cache HIT");
        return JSON.parse(cached) as SearxngResult[];
      }
    } catch {
@@ -73,8 +116,11 @@ export async function searchSearxng(

  // Cache miss — hit SearXNG API
  try {
-    const url = `${SEARXNG_BASE_URL}/search?q=${encodeURIComponent(query)}&format=json&language=id&categories=${category}`;
-    const { controller, clear } = createAbortControllerWithTimeout(TIMEOUT_MS);
+    const engineParam = engines
+      ? `&engines=${encodeURIComponent(engines)}`
+      : "";
+    const url = `${SEARXNG_BASE_URL}/search?q=${encodeURIComponent(query)}&format=json&language=id&categories=${category}${engineParam}`;
+    const { controller, clear } = createAbortControllerWithTimeout(timeoutMs);

    try {
      const response = await fetch(url, {
@@ -0,0 +1,419 @@
+/**
+ * termGlossary.ts
+ *
+ * Per-word "kamus" enrichment for LLM moderation.
+ *
+ * Problem: the moderation LLM often meets words it does not know — regional
+ * slang (Jawa/Sunda), foreign terms, niche anime/game jargon, or obscure
+ * technical vocabulary. When it guesses, it either invents a wrong meaning
+ * (false positive on a safe word) or misses a violation hidden in unfamiliar
+ * wording (false negative on an unknown vulgar/slang term).
+ *
+ * Solution: extract candidate "unknown-looking" words from message content,
+ * look each one up on Wikipedia via SearXNG, and inject the definitions into
+ * the LLM prompt as a `<term_glossary>` block so verdicts are based on facts
+ * instead of guesses.
+ *
+ * Cost control:
+ *  - definitions are cached in an in-memory LRU AND in Redis (shared with the
+ *    SearXNG cache) — a term is searched at most once per TTL across the whole
+ *    service, so repeat lookups are effectively free;
+ *  - lookups per batch are bounded (AI_GLOSSARY_MAX_TERMS);
+ *  - Wikipedia-only search first, generic search as a fallback;
+ *  - everything degrades gracefully: no Redis, no SearXNG, no Wikipedia match
+ *    → the block is simply omitted and moderation proceeds as before.
+ */
+
+import { LRUCache } from "lru-cache";
+import { createChildLogger } from "@/shared/logger/index";
+import { config } from "../../shared/config/config.js";
+import { escapeXml } from "./moderationBuilders.js";
+import {
+  makeSearxngCacheKey,
+  searchSearxng,
+  searxngCacheGet,
+  searxngCacheSet,
+} from "./searxngSearch.js";
+
+const log = createChildLogger("term-glossary");
+
+// ---------------------------------------------------------------------------
+// Constants
+// ---------------------------------------------------------------------------
+
+/** Redis TTL for a successfully resolved definition (definitions are stable). */
+const DEF_TTL_SECONDS = 7 * 24 * 60 * 60;
+/** Redis TTL for a lookup that found nothing — don't re-search every batch. */
+const MISS_TTL_SECONDS = 24 * 60 * 60;
+/** Sentinel stored in caches for "term has no resolvable definition". */
+const EMPTY_SENTINEL = "__not_found__";
+/** Per-search timeout — keep glossary lookups snappy even on a slow SearXNG. */
+const GLOSSARY_SEARCH_TIMEOUT_MS = 5000;
+/** Max definition snippet length kept in the prompt. */
+const MAX_DEFINITION_CHARS = 300;
+
+/** In-memory cache: term (lowercase) → definition | NOT_FOUND sentinel. */
+const NOT_FOUND: TermDefinition = {
+  term: "__not_found__",
+  definition: "",
+  sourceUrl: "",
+};
+const termLru = new LRUCache<string, TermDefinition>({
+  max: 2000,
+  ttl: 24 * 60 * 60 * 1000,
+});
+
+// ---------------------------------------------------------------------------
+// Term extraction
+// ---------------------------------------------------------------------------
+
+/** Word tokenizer — letters/digits plus internal -_'· (handles "well-known",
+ *  "node_modules", diacritics). */
+const WORD_RE = /[\p{L}\p{N}]+(?:[-_'’·][\p{L}\p{N}]+)*/gu;
+
+/** Removes URLs, Discord mentions/custom emoji, code fences, markdown noise. */
+function cleanContent(raw: string): string {
+  return raw
+    .replace(/https?:\/\/\S+/gi, " ")
+    .replace(/<@!?\d+>/g, " ")
+    .replace(/<#\d+>/g, " ")
+    .replace(/<a?:\w+:\d+>/g, " ")
+    .replace(/[`*_~|>\[\]]/g, " ")
+    .replace(/[\p{Emoji}\p{Extended_Pictographic}]/gu, " ")
+    .replace(/\s+/g, " ")
+    .trim();
+}
+
+/** Filters out tokens that are useless as glossary candidates (numbers,
+ *  repeated-char noise, mega-tokens). */
+function isNoiseWord(word: string): boolean {
+  if (word.length > 28) return true;
+  if (/^\d+$/.test(word)) return true;
+  const lower = word.toLowerCase();
+  // "aaaa…", "wwwwww" — single repeated character
+  if (/^(.)\1{2,}$/.test(lower)) return true;
+  // "wkwk", "hehe", "69" alternations — repeated 2–3 char base. "meme" is
+  // the one legit 4-letter word this matches; it is whitelisted below.
+  if (/^([a-z]{2,3})\1{1,}$/.test(lower)) return true;
+  return false;
+}
+
+/** Deterministic bonus for words that look like proper nouns or foreign. */
+function scoreWord(word: string): number {
+  let score = 1;
+  // Capitalized first letter (proper noun / title) but not ALL-CAPS acronyms
+  if (/^[A-Z]/.test(word) && !/^[A-Z]{2,}$/.test(word)) score += 3;
+  // Contains a letter outside basic latin → regional/foreign spelling
+  if (/[\p{L}]/u.test(word.replace(/[A-Za-z]/g, ""))) score += 2;
+  // Contains an internal apostrophe or hyphen → likely a named entity
+  if (/[-_'’·]/.test(word)) score += 2;
+  return score;
+}
+
+const STOPWORDS = new Set(
+  // ── Bahasa Indonesia ────────────────────────────────────────────────
+  (
+    " yang dan di ke dari ini itu dengan untuk pada dalam adalah akan telah sudah bisa dapat harus tidak juga saya kamu kita kami mereka dia aku kau gua lu lo gw gue elu anda kalian nya kah lah pun ya yah kan sih dong deh kok loh toh aja saja gitu gini begitu begini tapi tetapi namun atau karena sebab jika kalau bila maka supaya agar meski meskipun walau walaupun ketika saat setelah sebelum selama antara terhadap tentang mengenai bagi oleh secara sebagai seperti daripada tanpa hingga sampai sejak menuju bahwa padahal sebenarnya sepertinya mungkin memang jadi lalu terus akhirnya misalnya contohnya banyak sedikit semua seluruh setiap tiap beberapa ada bukan jangan boleh mau ingin pengen nggak ngak gak ga kagak ngga ndak nanti kemarin besok hari ini sekarang waktu itu masih sedang belum pernah sering selalu kadang jarang cepat lambat awal akhir baru lama besar kecil tinggi rendah panjang pendek baik buruk benar salah sama beda penting biasanya selamat terima kasih makasih sangat sekali paling cuma cuman hanya lebih kurang sekitar hampir ternyata rupanya begitu gimana bagaimana kenapa mengapa siapa apa mana kapan darimana kemana bilang ngomong omong kata tadi dulu terus lagi tetap pasti seharusnya sebaiknya seakan seolah kayaknya keliatan kelihatan ketahuan disini disitu disana kesini kesana bener pake pakai kayak emang lagian mulu istilah istilahnya banget" +
+    // ── English ───────────────────────────────────────────────────────
+    " the a an and or but if then else for to in on at by with without from of is are was were be been being have has had do does did will would can could should may might must shall this that these those it its i you he she we they them their there here when where why how what which who whom whose only very just about above after before below under over into onto within upon against between among during through across along around behind beyond near off out up down now then so as not no yes ok okay" +
+    // ── Common net slang / acronyms the LLM already knows ──────────────
+    " lol omg wtf idk btw tbh imo aka fyi nsfw smh nvm asap afk brb gg wp ty np mb sry thx kk oke okk ygy frfr"
+  ).split(/\s+/),
+);
+
+/**
+ * Words that are either already defined by the moderation rules, or are so
+ * common (brands, tech vocabulary, project names) that a Wikipedia lookup is
+ * a guaranteed miss/waste. Keeps the glossary focused on genuinely unknown
+ * terms.
+ */
+const KNOWN_SAFE_TERMS = new Set(
+  (
+    "discord youtube google facebook instagram twitter tiktok whatsapp telegram netflix spotify steam github gitlab bitbucket chatgpt openai anthropic claude deepseek gemini llama copilot cursor vscode vscodium jetbrains intellij pycharm webstorm sublime codeblocks" +
+    " docker kubernetes k8s linux ubuntu debian arch fedora manjaro kali windows macos android ios chrome firefox safari edge opera brave" +
+    " react nextjs next vue svelte angular node nodejs deno bun pnpm yarn npm javascript typescript python golang go rust java kotlin swift cplusplus cpp css html json xml yaml toml regex backend frontend database mysql postgres postgresql mongodb redis qdrant sqlite nosql graphql rest websocket webhook" +
+    " bug crash error debug fix issue pr merge commit push pull branch main master dev staging production server client app website web browser" +
+    " stream streaming video audio voice call camera screen share screenshare gameplay gaming game play steam epic xbox playstation nintendo switch console" +
+    " bot discordbot moderation moderator admin member user profile avatar channel server guild message chat dm reply forward embed sticker emoji role permission" +
+    " meme code coding ngoding programmer program developer engineer software hardware cpu gpu ram rom storage disk network internet wifi lan ip dns vpn proxy cloud aws azure gcp vercel netlify heroku railway render vps hosting domain ssl login logout register account password email username" +
+    " anime manga waifu husbando tsundere moe otaku wibu weeb otome isekai shonen seinen josei manga manhwa manhua doujin" +
+    " anjay wkwk wkwkwk gws gaskeun santuy njir baka woy woi hadeh astaga asu anjing bangsat ngehe asal alay lebay caper mabar" +
+    " asus bete imphnen impnhen ngab" +
+    " syahadat sholat shalat solat puasa zakat haji umrah doa tuhan nabi allah yesus muhammad hashem" +
+    " loli shota incest exhibition furry fursuit cosplay costume" +
+    " gaza palestine israel yahudi yahud israel palestina israeli" +
+    " hokkian mandarin arabic jawa sunda betawi minang bugis batak melayu inggris indonesia"
+  ).split(/\s+/),
+);
+
+function isKnownTerm(word: string): boolean {
+  return STOPWORDS.has(word) || KNOWN_SAFE_TERMS.has(word);
+}
+
+/** True when a quoted phrase is mostly filler words (skip it). */
+function isMostlyStopwords(phrase: string): boolean {
+  const words = phrase
+    .toLowerCase()
+    .split(/[^a-zà-öø-ÿ]+/i)
+    .filter(Boolean);
+  if (words.length === 0) return true;
+  const stopCount = words.filter((w) => STOPWORDS.has(w)).length;
+  return stopCount / words.length >= 0.6;
+}
+
+export interface ExtractGlossaryOptions {
+  maxTerms?: number;
+  minWordLength?: number;
+}
+
+/**
+ * Extracts candidate terms that the LLM might not know from message content.
+ * Returns at most `maxTerms` terms (default from config), scored by how
+ * "unknown-looking" they are (proper nouns, foreign spelling, quoted phrases).
+ */
+export function extractGlossaryTerms(
+  contents: string[],
+  options: ExtractGlossaryOptions = {},
+): string[] {
+  const maxTerms = options.maxTerms ?? config.AI_GLOSSARY_MAX_TERMS;
+  const minWordLength =
+    options.minWordLength ?? config.AI_GLOSSARY_MIN_WORD_LENGTH;
+
+  const candidates = new Map<string, { word: string; score: number }>();
+
+  const push = (rawWord: string, score: number): void => {
+    const clean = rawWord
+      .trim()
+      .replace(/^[^\p{L}\p{N}]+|[^\p{L}\p{N}]+$/gu, "");
+    if (clean.length < minWordLength) return;
+    const key = clean.toLowerCase();
+    if (isKnownTerm(key) || isNoiseWord(clean)) return;
+    const existing = candidates.get(key);
+    if (existing) {
+      existing.score += score + 1;
+    } else {
+      candidates.set(key, { word: clean, score });
+    }
+  };
+
+  for (const content of contents) {
+    if (!content) continue;
+    const cleaned = cleanContent(content);
+    if (!cleaned) continue;
+
+    // Quoted phrases — explicit terms the user called out
+    for (const m of cleaned.matchAll(/"([^"]{2,80})"/g)) {
+      const phrase = m[1].trim();
+      const wordCount = phrase.split(/\s+/).length;
+      if (wordCount >= 2 && wordCount <= 6 && !isMostlyStopwords(phrase)) {
+        push(phrase, 10);
+      }
+    }
+
+    // Individual words
+    for (const m of cleaned.matchAll(WORD_RE)) {
+      const w = m[0];
+      if (w.length < minWordLength) continue;
+      if (isNoiseWord(w)) continue;
+      const key = w.toLowerCase();
+      if (isKnownTerm(key)) continue;
+      push(w, scoreWord(w));
+    }
+  }
+
+  return Array.from(candidates.values())
+    .sort((a, b) => b.score - a.score)
+    .slice(0, maxTerms)
+    .map((c) => c.word);
+}
+
+// ---------------------------------------------------------------------------
+// Definition lookup (cached: LRU → Redis → SearXNG/Wikipedia)
+// ---------------------------------------------------------------------------
+
+export interface TermDefinition {
+  term: string;
+  definition: string;
+  sourceUrl: string;
+}
+
+/** Picks the best definition from search results, preferring Wikipedia. */
+function pickDefinition(
+  results: Array<{ title: string; url: string; snippet: string }>,
+  term: string,
+): TermDefinition | null {
+  const best = results.find((r) => /wikipedia/i.test(r.url)) ?? results[0];
+  if (!best) return null;
+  const snippet = (best.snippet || best.title || "").trim();
+  if (snippet.length < 10) return null;
+  const definition =
+    snippet.length > MAX_DEFINITION_CHARS
+      ? `${snippet.slice(0, MAX_DEFINITION_CHARS - 1).trimEnd()}…`
+      : snippet;
+  return { term, definition, sourceUrl: best.url };
+}
+
+async function lookupTermDefinition(
+  term: string,
+): Promise<TermDefinition | null> {
+  const key = term.toLowerCase().trim();
+
+  // 1. In-memory LRU — same process, instant
+  const lruHit = termLru.get(key);
+  if (lruHit) return lruHit === NOT_FOUND ? null : lruHit;
+
+  // 2. Redis — shared across processes/workers
+  const cacheKey = makeSearxngCacheKey("def", key);
+  const cached = await searxngCacheGet(cacheKey);
+  if (cached !== null) {
+    if (cached === EMPTY_SENTINEL) {
+      termLru.set(key, NOT_FOUND);
+      return null;
+    }
+    try {
+      const parsed = JSON.parse(cached) as {
+        definition?: string;
+        sourceUrl?: string;
+      };
+      if (parsed.definition) {
+        const def: TermDefinition = {
+          term,
+          definition: parsed.definition,
+          sourceUrl: parsed.sourceUrl ?? "",
+        };
+        termLru.set(key, def);
+        return def;
+      }
+    } catch {
+      // malformed cache entry — fall through to search
+    }
+  }
+
+  // 3. Live search — Wikipedia first, generic search as fallback
+  try {
+    let def = pickDefinition(
+      await searchSearxng(
+        key,
+        "general",
+        "wikipedia",
+        GLOSSARY_SEARCH_TIMEOUT_MS,
+      ),
+      term,
+    );
+    if (!def) {
+      def = pickDefinition(
+        await searchSearxng(
+          `${key} definisi arti`,
+          "general",
+          undefined,
+          GLOSSARY_SEARCH_TIMEOUT_MS,
+        ),
+        term,
+      );
+    }
+
+    if (def) {
+      searxngCacheSet(
+        cacheKey,
+        JSON.stringify({
+          definition: def.definition,
+          sourceUrl: def.sourceUrl,
+        }),
+        DEF_TTL_SECONDS,
+      );
+      termLru.set(key, def);
+      log.debug({ term: key }, "Term glossary resolved definition");
+      return def;
+    }
+  } catch (err) {
+    log.debug(
+      { term: key, error: err instanceof Error ? err.message : String(err) },
+      "Term glossary lookup failed — skipping term",
+    );
+  }
+
+  // No definition — cache the miss so we do not re-search every batch.
+  searxngCacheSet(cacheKey, EMPTY_SENTINEL, MISS_TTL_SECONDS);
+  termLru.set(key, NOT_FOUND);
+  return null;
+}
+
+/**
+ * Looks up definitions for a batch of terms, in parallel. Returns a map of
+ * term → definition for the terms that resolved. Errors/misses are skipped.
+ */
+export async function lookupTermDefinitions(
+  terms: string[],
+): Promise<Map<string, TermDefinition>> {
+  const map = new Map<string, TermDefinition>();
+  if (terms.length === 0) return map;
+
+  const results = await Promise.allSettled(terms.map(lookupTermDefinition));
+  for (let i = 0; i < terms.length; i++) {
+    const r = results[i];
+    if (r.status === "fulfilled" && r.value) {
+      map.set(r.value.term, r.value);
+    }
+  }
+  return map;
+}
+
+// ---------------------------------------------------------------------------
+// Prompt formatting
+// ---------------------------------------------------------------------------
+
+/**
+ * Formats definitions as a `<term_glossary>` XML block for the LLM prompt:
+ *
+ *   <term_glossary>
+ *     <term word="ngab" source="https://…">definisi…</term>
+ *   </term_glossary>
+ *
+ * Returns "" when there are no definitions (the block is then omitted).
+ */
+export function formatTermGlossary(
+  defs: ReadonlyMap<string, TermDefinition>,
+): string {
+  if (!defs || defs.size === 0) return "";
+  const lines = Array.from(defs.values()).map(
+    (d) =>
+      `  <term word="${escapeXml(d.term)}" source="${escapeXml(d.sourceUrl)}">${escapeXml(d.definition)}</term>`,
+  );
+  return `<term_glossary>\n${lines.join("\n")}\n</term_glossary>`;
+}
+
+// ---------------------------------------------------------------------------
+// Convenience: full pipeline
+// ---------------------------------------------------------------------------
+
+export interface GlossaryBlockOptions extends ExtractGlossaryOptions {
+  enabled?: boolean;
+}
+
+/**
+ * One-shot helper: extract terms from message contents, look up definitions,
+ * and return the formatted `<term_glossary>` block ("" when disabled or no
+ * definitions found). Safe to call on every batch — cached lookups make it
+ * cheap.
+ */
+export async function buildTermGlossaryBlock(
+  contents: string[],
+  options: GlossaryBlockOptions = {},
+): Promise<string> {
+  const enabled = options.enabled ?? config.AI_GLOSSARY_ENABLED;
+  if (!enabled) return "";
+  if (contents.length === 0) return "";
+
+  const terms = extractGlossaryTerms(contents, options);
+  if (terms.length === 0) return "";
+
+  const defs = await lookupTermDefinitions(terms);
+  if (defs.size === 0) return "";
+
+  const block = formatTermGlossary(defs);
+  log.debug(
+    { terms: terms.length, definitions: defs.size },
+    "Term glossary block built",
+  );
+  return block;
+}
@@ -37,6 +37,7 @@ import {
  formatSearchResults,
  searchSearxng,
 } from "./searxngSearch.js";
+import { buildTermGlossaryBlock } from "./termGlossary.js";
 import { getRecentCorrectedModerations } from "./textCacheStore.js";
 import { extractUrlsFromText, fetchUrlSafely } from "./urlFetcher.js";
 import { getUserProfile } from "./userProfileStore.js";
@@ -145,9 +146,17 @@ export async function runTextOnlyBatch(
    return map;
  })();

-  const [urlFetchMaps, searxngResults] = await Promise.all([
+  // Term glossary — per-word Wikipedia lookups for words the LLM may not
+  // know (slang, jargon, regional language). Cached in Redis + in-memory, so
+  // repeat terms resolve instantly and only genuinely new words hit SearXNG.
+  const glossaryPromise = buildTermGlossaryBlock(
+    targets.map((msg) => getAnalysisContent(msg)),
+  ).catch(() => "");
+
+  const [urlFetchMaps, searxngResults, glossaryBlock] = await Promise.all([
    urlFetchPromise,
    searxngPromise,
+    glossaryPromise,
  ]);
  const urlFetchMap = urlFetchMaps.text;

@@ -368,6 +377,7 @@ export async function runTextOnlyBatch(
        userProfilesBlock?.trimEnd() ?? "",
        contextBlock?.trimEnd() ?? "",
        searxngBlock,
+        glossaryBlock,
        `<messages_to_analyze>\n${messagesBlock}\n</messages_to_analyze>`,
      ].filter((b) => b.trim().length > 0);
      return {
@@ -87,6 +87,7 @@ import {
  formatSearchResults,
  searchSearxng,
 } from "./searxngSearch.js";
+import { buildTermGlossaryBlock } from "./termGlossary.js";
 import { extractUrlsFromText } from "./urlFetcher.js";
 import { getUserProfile } from "./userProfileStore.js";
 import {
@@ -413,6 +414,12 @@ export async function prepareMediaMessage(
      searxngXml = `\n<web_searches>\n${parts.join("\n")}\n</web_searches>`;
  }

+  // Term glossary — cached per-word Wikipedia definitions for words the LLM
+  // may not know. Bounded and cached (in-memory + Redis), so this adds no
+  // meaningful latency to the media path either.
+  const glossaryXml = await buildTermGlossaryBlock([content]).catch(() => "");
+  const glossaryCtx = glossaryXml ? `\n${glossaryXml}` : "";
+
  // Build XML block
  const webTexts = webTextMap.get(targetId) ?? [];
  const mediaAnalyses = mediaAnalysisMap.get(targetId) ?? [];
@@ -466,6 +473,6 @@ export async function prepareMediaMessage(

  const isBot = resolveIsBot(target);
  const isEdited = resolveIsEdited(target);
-  const messageBlock = `<message id="${escapeXml(target.id)}" user="${escapeXml(resolveDisplayName(target))}" time="${new Date(target.created_at).toISOString()}"${isBot ? ` bot="true"` : ""}${isEdited ? ` edited="true"` : ""}>\n  ${repXml}${profileRef ? `\n  ${profileRef}` : ""}${refXml ? `\n  ${refXml}` : ""}\n  <content>${escapeXml(truncateForAi(content))}</content>${mediaContext ? ` ${escapeXml(mediaContext)}` : ""}${webContext}${mediaAnalysisContext}${searxngXml}\n</message>`;
+  const messageBlock = `<message id="${escapeXml(target.id)}" user="${escapeXml(resolveDisplayName(target))}" time="${new Date(target.created_at).toISOString()}"${isBot ? ` bot="true"` : ""}${isEdited ? ` edited="true"` : ""}>\n  ${repXml}${profileRef ? `\n  ${profileRef}` : ""}${refXml ? `\n  ${refXml}` : ""}\n  <content>${escapeXml(truncateForAi(content))}</content>${mediaContext ? ` ${escapeXml(mediaContext)}` : ""}${webContext}${mediaAnalysisContext}${searxngXml}${glossaryCtx}\n</message>`;
  return { targetId, messageBlock };
 }
@@ -171,6 +171,24 @@ export const configSchema = z
      .int()
      .positive()
      .default(30000),
+    // Term glossary — per-word Wikipedia lookups (via SearXNG) for words the
+    // LLM may not know (slang, jargon, regional language, foreign terms).
+    // Definitions are cached (in-memory + Redis) so repeat lookups are fast.
+    // Disable to skip glossary lookups entirely and analyze without them.
+    AI_GLOSSARY_ENABLED: z
+      .string()
+      .optional()
+      .transform((v) => v === "true")
+      .default(true),
+    // Max glossary terms looked up per analysis batch (keeps latency bounded).
+    AI_GLOSSARY_MAX_TERMS: z.coerce.number().int().min(1).max(20).default(6),
+    // Min word length for a term to be considered glossary-worthy.
+    AI_GLOSSARY_MIN_WORD_LENGTH: z.coerce
+      .number()
+      .int()
+      .min(2)
+      .max(20)
+      .default(5),

    // ── AI Analysis Timing ──────────────────────────────────────────────
    AI_ANALYSIS_DEBOUNCE_MS: z.coerce.number().positive().default(500),
@@ -0,0 +1,91 @@
+// ═══════════════════════════════════════════════════════════════════════════
+// Term glossary — pure extraction/formatting tests (no DB, Redis, or network)
+// ═══════════════════════════════════════════════════════════════════════════
+import { describe, expect, it } from "vitest";
+import {
+  extractGlossaryTerms,
+  formatTermGlossary,
+} from "../src/modules/ai-moderation/termGlossary.js";
+
+describe("extractGlossaryTerms — filters out words the LLM already knows", () => {
+  it("returns [] for common conversational Indonesian", () => {
+    const terms = extractGlossaryTerms(
+      ["anjay mabar yuk gaskeun gua gas", "iya bener banget sih"],
+      { maxTerms: 6 },
+    );
+    expect(terms).toEqual([]);
+  });
+
+  it("extracts uncommon/foreign-looking words and skips stopwords + brands", () => {
+    const terms = extractGlossaryTerms(
+      [
+        "tadi gua baca soal tempeh di discord",
+        "kayaknya istilahnya shirkmaxxing deh",
+      ],
+      { maxTerms: 6 },
+    );
+    // "tempeh" and "shirkmaxxing" are candidates; "discord"/"istilahnya" are not
+    expect(terms).toContain("tempeh");
+    expect(terms).toContain("shirkmaxxing");
+    expect(terms).not.toContain("discord");
+    expect(terms).not.toContain("istilahnya");
+  });
+
+  it("strips URLs, mentions, and custom emoji before extracting", () => {
+    const terms = extractGlossaryTerms(
+      ["cek https://example.com/foo <@123456> <:hadeh:987> kafircel"],
+      { maxTerms: 6 },
+    );
+    expect(terms).toContain("kafircel");
+    expect(terms.some((t) => /example|hadeh|123/.test(t))).toBe(false);
+  });
+
+  it("extracts quoted phrases as a single term", () => {
+    const terms = extractGlossaryTerms(['dia bilang "kostum hewan" itu aneh'], {
+      maxTerms: 6,
+    });
+    expect(terms).toContain("kostum hewan");
+  });
+
+  it("skips repeated-char noise like wkwkwk and aaaaa", () => {
+    const terms = extractGlossaryTerms(["wkwkwkwk aaaaa xixixi"], {
+      maxTerms: 6,
+    });
+    expect(terms).toEqual([]);
+  });
+
+  it("respects maxTerms and prioritizes proper nouns", () => {
+    const terms = extractGlossaryTerms(
+      ["aku suka Xenogears sama Chrono Cross terus Yakuza"],
+      { maxTerms: 2 },
+    );
+    expect(terms.length).toBeLessThanOrEqual(2);
+    expect(terms[0]).toBe("Xenogears");
+  });
+});
+
+describe("formatTermGlossary — XML block shape", () => {
+  it("returns '' for an empty map", () => {
+    expect(formatTermGlossary(new Map())).toBe("");
+  });
+
+  it("wraps definitions in <term_glossary> with escaped attributes/content", () => {
+    const block = formatTermGlossary(
+      new Map([
+        [
+          "kafircel",
+          {
+            term: "kafircel",
+            definition: "sebutan <memes> untuk & orang",
+            sourceUrl: "https://id.wikipedia.org/wiki/Mem",
+          },
+        ],
+      ]),
+    );
+    expect(block).toContain("<term_glossary>");
+    expect(block).toContain('<term word="kafircel"');
+    expect(block).toContain("&lt;memes&gt;");
+    expect(block).toContain("&amp;");
+    expect(block).toContain("</term_glossary>");
+  });
+});