feat(glossary): persist resolved definitions in Postgres + harden live SearXNG lookups

- Add term_glossary_cache table + migration 0014: resolved definitions are stored permanently (definitions rarely change); misses stay ephemeral in Redis/LRU with 1h TTL so transient failures get retried - Lookup flow: LRU -> Redis -> Postgres (permanent) -> live SearXNG; DB hits re-warm the fast caches; stale Redis miss sentinels no longer shadow DB - Rate-limit-aware live lookups: concurrency 2 + stagger, retry once on empty results, strict definition filter (Wikipedia preferred, rejects disambiguation/ads/translate-homepages) - Make SEARXNG_BASE_URL configurable via env (default unchanged)
2026-08-12 14:22:22 +07:00
parent f1aa08cdf6
commit d9f5592e6e
7 changed files with 347 additions and 77 deletions
@@ -0,0 +1,9 @@
+CREATE TABLE IF NOT EXISTS "term_glossary_cache" (
+	"term" text PRIMARY KEY NOT NULL,
+	"definition" text NOT NULL,
+	"source_url" text DEFAULT '' NOT NULL,
+	"resolved_at" bigint NOT NULL,
+	"hit_count" integer DEFAULT 0 NOT NULL
+);
+--> statement-breakpoint
+CREATE INDEX IF NOT EXISTS "idx_term_glossary_cache_resolved_at" ON "term_glossary_cache" USING btree ("resolved_at");
@@ -99,6 +99,13 @@
      "when": 1785551832190,
      "tag": "0013_rename_mascot_chat_to_chatbot",
      "breakpoints": true
+    },
+    {
+      "idx": 14,
+      "version": "7",
+      "when": 1785621600000,
+      "tag": "0014_add_term_glossary_cache",
+      "breakpoints": true
    }
  ]
 }
@@ -1,10 +1,11 @@
 import Redis from "ioredis";
 import { createChildLogger } from "@/shared/logger/index";
 import { createAbortControllerWithTimeout } from "@/shared/utils/index";
+import { config } from "../../shared/config/config.js";

 const log = createChildLogger("searxng-search");

-const SEARXNG_BASE_URL = "https://searxng.imrnes.team";
+const SEARXNG_BASE_URL = config.SEARXNG_BASE_URL;
 const MAX_RESULTS = 3;
 const TIMEOUT_MS = 8000;
 const CACHE_TTL = 86400; // 24 hours
@@ -14,18 +14,26 @@
 * the LLM prompt as a `<term_glossary>` block so verdicts are based on facts
 * instead of guesses.
 *
- * Cost control:
- *  - definitions are cached in an in-memory LRU AND in Redis (shared with the
- *    SearXNG cache) — a term is searched at most once per TTL across the whole
- *    service, so repeat lookups are effectively free;
+ * Cost control & persistence:
+ *  - successfully resolved definitions are PERSISTED PERMANENTLY in Postgres
+ *    (`term_glossary_cache`) — definitions rarely change, so a resolved term
+ *    is never searched again; only misses stay ephemeral (Redis/LRU, 1h);
+ *  - in-memory LRU + Redis (shared with the SearXNG cache) sit in front of
+ *    the DB as fast read caches, so repeat lookups are effectively free;
 *  - lookups per batch are bounded (AI_GLOSSARY_MAX_TERMS);
- *  - Wikipedia-only search first, generic search as a fallback;
- *  - everything degrades gracefully: no Redis, no SearXNG, no Wikipedia match
+ *  - live SearXNG calls are rate-limit aware: concurrency 2 + stagger, retry
+ *    once on empty results, and misses cached for only 1h so a limiter/
+ *    network blip is not treated as a permanent miss;
+ *  - only results that read like actual definitions are accepted (Wikipedia
+ *    preferred; disambiguation/ads/translate-homepages rejected);
+ *  - everything degrades gracefully: no Redis, no SearXNG, no match
 *    → the block is simply omitted and moderation proceeds as before.
 */

 import { LRUCache } from "lru-cache";
+import pLimit from "p-limit";
 import { createChildLogger } from "@/shared/logger/index";
+import { delay } from "@/shared/utils/index";
 import { config } from "../../shared/config/config.js";
 import { escapeXml } from "./moderationBuilders.js";
 import {
@@ -34,6 +42,10 @@ import {
  searxngCacheGet,
  searxngCacheSet,
 } from "./searxngSearch.js";
+import {
+  getTermDefinitionFromDb,
+  setTermDefinitionInDb,
+} from "./termGlossaryStore.js";

 const log = createChildLogger("term-glossary");

@@ -43,14 +55,29 @@ const log = createChildLogger("term-glossary");

 /** Redis TTL for a successfully resolved definition (definitions are stable). */
 const DEF_TTL_SECONDS = 7 * 24 * 60 * 60;
-/** Redis TTL for a lookup that found nothing — don't re-search every batch. */
-const MISS_TTL_SECONDS = 24 * 60 * 60;
+/**
+ * Redis TTL for a lookup that found nothing. Kept SHORT (1h): SearXNG
+ * instances silently return empty result sets when rate-limited, so an empty
+ * response is often a transient failure, not a real miss. A short TTL lets
+ * the term be retried on a later batch instead of poisoning it for a day.
+ */
+const MISS_TTL_SECONDS = 60 * 60;
+const MISS_TTL_MS = MISS_TTL_SECONDS * 1000;
 /** Sentinel stored in caches for "term has no resolvable definition". */
 const EMPTY_SENTINEL = "__not_found__";
 /** Per-search timeout — keep glossary lookups snappy even on a slow SearXNG. */
 const GLOSSARY_SEARCH_TIMEOUT_MS = 5000;
+/** Delay before retrying a search that returned zero results. */
+const RETRY_DELAY_MS = 350;
 /** Max definition snippet length kept in the prompt. */
 const MAX_DEFINITION_CHARS = 300;
+/**
+ * SearXNG rate-limits aggressive parallel bursts (returns 200 with empty
+ * results). Never fire all terms at once — cap live searches at 2 concurrent
+ * and stagger the start times slightly.
+ */
+const LIVE_SEARCH_CONCURRENCY = 2;
+const LIVE_SEARCH_STAGGER_MS = 250;

 /** In-memory cache: term (lowercase) → definition | NOT_FOUND sentinel. */
 const NOT_FOUND: TermDefinition = {
@@ -63,6 +90,16 @@ const termLru = new LRUCache<string, TermDefinition>({
  ttl: 24 * 60 * 60 * 1000,
 });

+/** Serializes live SearXNG lookups (rate-limit aware) with a small stagger. */
+const liveSearchLimit = pLimit(LIVE_SEARCH_CONCURRENCY);
+let lastLiveSearchAt = 0;
+async function acquireLiveSlot(): Promise<void> {
+  const now = Date.now();
+  const wait = lastLiveSearchAt + LIVE_SEARCH_STAGGER_MS - now;
+  if (wait > 0) await delay(wait);
+  lastLiveSearchAt = Date.now();
+}
+
 // ---------------------------------------------------------------------------
 // Term extraction
 // ---------------------------------------------------------------------------
@@ -78,7 +115,7 @@ function cleanContent(raw: string): string {
    .replace(/<@!?\d+>/g, " ")
    .replace(/<#\d+>/g, " ")
    .replace(/<a?:\w+:\d+>/g, " ")
-    .replace(/[`*_~|>\[\]]/g, " ")
+    .replace(/[`*_~|>[\]]/g, " ")
    .replace(/[\p{Emoji}\p{Extended_Pictographic}]/gu, " ")
    .replace(/\s+/g, " ")
    .trim();
@@ -237,15 +274,64 @@ export interface TermDefinition {
  sourceUrl: string;
 }

-/** Picks the best definition from search results, preferring Wikipedia. */
+/** Definition-like markers for accepting a non-Wikipedia search result. */
+const DEF_MARKERS =
+  /adalah|merupakan|istilah (?:untuk|yang|yg)|artinya|sebutan|berarti|refers? to|known as|also called|short for|a term (?:for|used)|istilah dalam|kata (?:asing|serapan)? ?untuk/i;
+
+/** True when the term appears in the result text (or a 4+ char word in the
+ *  result is part of the term). Lenient — "kafircel" matches a "Kafir"
+ *  article via substring, while a Google-Translate homepage snippet does not. */
+function hasTermOverlap(term: string, title: string, snippet: string): boolean {
+  const termLower = term.toLowerCase();
+  const text = `${title} ${snippet}`.toLowerCase();
+  if (text.includes(termLower)) return true;
+  const words = text.match(/[a-z0-9]{4,}/gi) ?? [];
+  return words.some((w) => termLower.includes(w));
+}
+
+/** Quality gate: is this result good enough to quote as a definition? */
+function isUsableDefinition(
+  r: { title: string; url: string; snippet: string },
+  term: string,
+  isWiki: boolean,
+): boolean {
+  const text = `${r.title} ${r.snippet}`;
+  // Wikipedia disambiguation pages are not definitions
+  if (/disambiguasi|disambiguation/i.test(text)) return false;
+  if ((r.snippet ?? "").trim().length < 25) return false;
+  if (!hasTermOverlap(term, r.title, r.snippet)) return false;
+  // Wikipedia articles are accepted with just the overlap+length gate;
+  // everything else must read like an actual definition, not an ad,
+  // a translate homepage, or a navigation blurb.
+  if (isWiki) return true;
+  return DEF_MARKERS.test(r.snippet);
+}
+
+/** Picks the best definition from search results, preferring a genuine
+ *  Wikipedia article; otherwise the first result that reads like a
+ *  definition. Returns null when nothing qualifies. */
 function pickDefinition(
  results: Array<{ title: string; url: string; snippet: string }>,
  term: string,
 ): TermDefinition | null {
-  const best = results.find((r) => /wikipedia/i.test(r.url)) ?? results[0];
-  if (!best) return null;
+  const wiki = results.find((r) => /wikipedia\.org/i.test(r.url));
+  const best = wiki && isUsableDefinition(wiki, term, true) ? wiki : null;
+  if (!best) {
+    for (const r of results) {
+      if (isUsableDefinition(r, term, false)) {
+        return buildDefinition(r, term);
+      }
+    }
+    return null;
+  }
+  return buildDefinition(best, term);
+}
+
+function buildDefinition(
+  best: { title: string; url: string; snippet: string },
+  term: string,
+): TermDefinition {
  const snippet = (best.snippet || best.title || "").trim();
-  if (snippet.length < 10) return null;
  const definition =
    snippet.length > MAX_DEFINITION_CHARS
      ? `${snippet.slice(0, MAX_DEFINITION_CHARS - 1).trimEnd()}…`
@@ -253,94 +339,140 @@ function pickDefinition(
  return { term, definition, sourceUrl: best.url };
 }

-async function lookupTermDefinition(
+/** Live (network) lookup — runs under the shared SearXNG rate-limit gate. */
+async function fetchDefinitionLive(
  term: string,
+  key: string,
+  cacheKey: string,
 ): Promise<TermDefinition | null> {
+  return liveSearchLimit(async () => {
+    await acquireLiveSlot();
+    try {
+      let results = await searchSearxng(
+        key,
+        "general",
+        undefined,
+        GLOSSARY_SEARCH_TIMEOUT_MS,
+      );
+      let def = pickDefinition(results, term);
+      // Zero results is usually the limiter kicking in, not a real miss —
+      // retry once. Results-but-unusable = genuine miss, no retry.
+      if (!def && results.length === 0) {
+        await delay(RETRY_DELAY_MS);
+        results = await searchSearxng(
+          key,
+          "general",
+          undefined,
+          GLOSSARY_SEARCH_TIMEOUT_MS,
+        );
+        def = pickDefinition(results, term);
+      }
+
+      if (def) {
+        // Persist permanently (definitions rarely change) — best-effort,
+        // then warm the fast caches.
+        void setTermDefinitionInDb(key, def.definition, def.sourceUrl);
+        searxngCacheSet(
+          cacheKey,
+          JSON.stringify({
+            definition: def.definition,
+            sourceUrl: def.sourceUrl,
+          }),
+          DEF_TTL_SECONDS,
+        );
+        termLru.set(key, def);
+        log.debug({ term: key }, "Term glossary resolved definition");
+        return def;
+      }
+    } catch (err) {
+      log.debug(
+        { term: key, error: err instanceof Error ? err.message : String(err) },
+        "Term glossary lookup failed — skipping term",
+      );
+    }
+
+    // No definition — cache the miss with a SHORT TTL so a transient
+    // limiter/network failure is retried on a later batch.
+    searxngCacheSet(cacheKey, EMPTY_SENTINEL, MISS_TTL_SECONDS);
+    termLru.set(key, NOT_FOUND, { ttl: MISS_TTL_MS });
+    return null;
+  });
+}
+
+/** Resolve one term: LRU → Redis → Postgres (permanent) → live SearXNG
+ *  (rate-limited). The fast caches sit in front of the DB; the DB is the
+ *  source of truth for successfully resolved definitions. */
+async function resolveTerm(term: string): Promise<TermDefinition | null> {
  const key = term.toLowerCase().trim();

  // 1. In-memory LRU — same process, instant
  const lruHit = termLru.get(key);
  if (lruHit) return lruHit === NOT_FOUND ? null : lruHit;

-  // 2. Redis — shared across processes/workers
+  // 2. Redis — shared across processes/workers. A miss sentinel here is NOT
+  //    a definitive answer: it may predate a permanent DB entry written by
+  //    another process, so we keep going and let the DB decide.
  const cacheKey = makeSearxngCacheKey("def", key);
  const cached = await searxngCacheGet(cacheKey);
+  let redisMiss = false;
  if (cached !== null) {
    if (cached === EMPTY_SENTINEL) {
-      termLru.set(key, NOT_FOUND);
-      return null;
-    }
-    try {
-      const parsed = JSON.parse(cached) as {
-        definition?: string;
-        sourceUrl?: string;
-      };
-      if (parsed.definition) {
-        const def: TermDefinition = {
-          term,
-          definition: parsed.definition,
-          sourceUrl: parsed.sourceUrl ?? "",
+      redisMiss = true;
+    } else {
+      try {
+        const parsed = JSON.parse(cached) as {
+          definition?: string;
+          sourceUrl?: string;
        };
-        termLru.set(key, def);
-        return def;
+        if (parsed.definition) {
+          const def: TermDefinition = {
+            term,
+            definition: parsed.definition,
+            sourceUrl: parsed.sourceUrl ?? "",
+          };
+          termLru.set(key, def);
+          return def;
+        }
+      } catch {
+        // malformed cache entry — fall through to DB/live
      }
-    } catch {
-      // malformed cache entry — fall through to search
    }
  }

-  // 3. Live search — Wikipedia first, generic search as fallback
-  try {
-    let def = pickDefinition(
-      await searchSearxng(
-        key,
-        "general",
-        "wikipedia",
-        GLOSSARY_SEARCH_TIMEOUT_MS,
-      ),
+  // 3. Postgres — permanent store for resolved definitions. A hit re-warms
+  //    the fast caches so the DB is not hit on every batch.
+  const dbDef = await getTermDefinitionFromDb(key);
+  if (dbDef) {
+    const def: TermDefinition = {
      term,
+      definition: dbDef.definition,
+      sourceUrl: dbDef.sourceUrl,
+    };
+    termLru.set(key, def);
+    searxngCacheSet(
+      cacheKey,
+      JSON.stringify({ definition: def.definition, sourceUrl: def.sourceUrl }),
+      DEF_TTL_SECONDS,
    );
-    if (!def) {
-      def = pickDefinition(
-        await searchSearxng(
-          `${key} definisi arti`,
-          "general",
-          undefined,
-          GLOSSARY_SEARCH_TIMEOUT_MS,
-        ),
-        term,
-      );
-    }
-
-    if (def) {
-      searxngCacheSet(
-        cacheKey,
-        JSON.stringify({
-          definition: def.definition,
-          sourceUrl: def.sourceUrl,
-        }),
-        DEF_TTL_SECONDS,
-      );
-      termLru.set(key, def);
-      log.debug({ term: key }, "Term glossary resolved definition");
-      return def;
-    }
-  } catch (err) {
-    log.debug(
-      { term: key, error: err instanceof Error ? err.message : String(err) },
-      "Term glossary lookup failed — skipping term",
-    );
+    log.debug({ term: key }, "Term glossary DB hit");
+    return def;
  }

-  // No definition — cache the miss so we do not re-search every batch.
-  searxngCacheSet(cacheKey, EMPTY_SENTINEL, MISS_TTL_SECONDS);
-  termLru.set(key, NOT_FOUND);
-  return null;
+  // 4. Redis already said "miss" recently and the DB has nothing — respect
+  //    that instead of hammering SearXNG again within the miss window.
+  if (redisMiss) {
+    termLru.set(key, NOT_FOUND, { ttl: MISS_TTL_MS });
+    return null;
+  }
+
+  // 5. Live search (rate-limited + staggered)
+  return fetchDefinitionLive(term, key, cacheKey);
 }

 /**
 * Looks up definitions for a batch of terms, in parallel. Returns a map of
 * term → definition for the terms that resolved. Errors/misses are skipped.
+ * Live SearXNG calls are throttled internally (concurrency 2 + stagger).
 */
 export async function lookupTermDefinitions(
  terms: string[],
@@ -348,7 +480,7 @@ export async function lookupTermDefinitions(
  const map = new Map<string, TermDefinition>();
  if (terms.length === 0) return map;

-  const results = await Promise.allSettled(terms.map(lookupTermDefinition));
+  const results = await Promise.allSettled(terms.map(resolveTerm));
  for (let i = 0; i < terms.length; i++) {
    const r = results[i];
    if (r.status === "fulfilled" && r.value) {
@@ -0,0 +1,86 @@
+/**
+ * termGlossaryStore.ts
+ *
+ * Permanent Postgres layer for the term glossary. Resolved definitions
+ * (which carry content) are persisted here because they rarely change —
+ * Redis/LRU only act as fast read caches in front of this table. Terms with
+ * no definition (misses) are deliberately NOT persisted; they stay ephemeral
+ * in Redis with a short TTL so transient lookup failures get retried.
+ *
+ * All calls are best-effort: any DB error degrades to a cache miss (the
+ * glossary then falls through to Redis/live search as if the DB layer
+ * didn't exist).
+ */
+
+import { createChildLogger } from "@/shared/logger/index";
+import { executeAll, executeGet } from "../../shared/database/drizzle.js";
+
+const log = createChildLogger("term-glossary-store");
+
+export interface StoredTermDefinition {
+  definition: string;
+  sourceUrl: string;
+}
+
+/**
+ * Read a permanently stored definition for a term (lowercase key).
+ * Returns null when missing or on any DB error (callers fall through).
+ * A successful read bumps hit_count for observability (fire-and-forget).
+ */
+export async function getTermDefinitionFromDb(
+  term: string,
+): Promise<StoredTermDefinition | null> {
+  try {
+    const row = await executeGet(
+      `SELECT definition, source_url FROM term_glossary_cache WHERE term = $1`,
+      [term.toLowerCase().trim()],
+    );
+    if (!row) return null;
+    try {
+      await executeAll(
+        `UPDATE term_glossary_cache SET hit_count = hit_count + 1 WHERE term = $1`,
+        [term.toLowerCase().trim()],
+      );
+    } catch {
+      // hit_count is observability only — never fail a read for it
+    }
+    return {
+      definition: row.definition as string,
+      sourceUrl: (row.source_url as string | null) ?? "",
+    };
+  } catch (error) {
+    log.debug(
+      { error: error instanceof Error ? error.message : String(error) },
+      "getTermDefinitionFromDb failed — falling back to live search",
+    );
+    return null;
+  }
+}
+
+/**
+ * Persist a resolved definition permanently (UPSERT by term).
+ * Only called for successful resolutions — never for misses.
+ * Best-effort: a DB write failure does not affect the returned definition.
+ */
+export async function setTermDefinitionInDb(
+  term: string,
+  definition: string,
+  sourceUrl: string,
+): Promise<void> {
+  try {
+    await executeAll(
+      `INSERT INTO term_glossary_cache (term, definition, source_url, resolved_at, hit_count)
+       VALUES ($1, $2, $3, $4, 0)
+       ON CONFLICT (term) DO UPDATE SET
+         definition = EXCLUDED.definition,
+         source_url = EXCLUDED.source_url,
+         resolved_at = EXCLUDED.resolved_at`,
+      [term.toLowerCase().trim(), definition, sourceUrl, Date.now()],
+    );
+  } catch (error) {
+    log.warn(
+      { error: error instanceof Error ? error.message : String(error) },
+      "setTermDefinitionInDb failed — definition stays memory/Redis only",
+    );
+  }
+}
@@ -92,6 +92,10 @@ export const configSchema = z

    // ── Redis ────────────────────────────────────────────────────────────
    REDIS_URL: z.string().default("redis://localhost:6379"),
+    // ── SearXNG ───────────────────────────────────────────────────────────
+    // Instance for web search + term glossary lookups. Override when the
+    // default instance is down/rate-limited.
+    SEARXNG_BASE_URL: z.string().url().default("https://searxng.imrnes.team"),
    // ── Voice PCM WebSocket (direct gateway→backend, bypasses Redis) ────
    VOICE_PCM_WS_ENABLED: z
      .string()
@@ -435,6 +435,32 @@ export const pgStickerCacheTable = pgTable(

 export const stickerCacheTable = pgStickerCacheTable;

+/**
+ * Term Glossary Cache Table (PostgreSQL)
+ * Permanently stores resolved term definitions (Wikipedia/SearXNG lookups).
+ * Definitions rarely change, so once a term is successfully resolved it is
+ * persisted here forever — Redis/LRU only act as fast read caches in front.
+ * Terms with NO definition (misses) are NOT stored here; they stay ephemeral
+ * in Redis with a short TTL so transient lookup failures get retried.
+ */
+export const pgTermGlossaryCacheTable = pgTable(
+  "term_glossary_cache",
+  {
+    term: pgText("term").primaryKey(),
+    definition: pgText("definition").notNull(),
+    source_url: pgText("source_url").notNull().default(""),
+    resolved_at: pgBigint("resolved_at", { mode: "number" }).notNull(),
+    hit_count: pgInteger("hit_count").notNull().default(0),
+  },
+  (table) => ({
+    resolvedAtIdx: pgIndex("idx_term_glossary_cache_resolved_at").on(
+      table.resolved_at,
+    ),
+  }),
+);
+
+export const termGlossaryCacheTable = pgTermGlossaryCacheTable;
+
 // =============================================================================
 // Meta / System
 // =============================================================================
@@ -580,6 +606,11 @@ export type TextAnalysisCacheInsert =
 export type StickerCacheRecord = typeof stickerCacheTable.$inferSelect;
 export type StickerCacheInsert = typeof stickerCacheTable.$inferInsert;

+// Term Glossary Cache
+export type TermGlossaryCache = typeof termGlossaryCacheTable.$inferSelect;
+export type TermGlossaryCacheInsert =
+  typeof termGlossaryCacheTable.$inferInsert;
+
 // Muxer Jobs
 export type MuxerJob = typeof muxerJobsTable.$inferSelect;
 export type MuxerJobInsert = typeof muxerJobsTable.$inferInsert;