feat(glossary): persist resolved definitions in Postgres + harden live SearXNG lookups

- Add term_glossary_cache table + migration 0014: resolved definitions are
  stored permanently (definitions rarely change); misses stay ephemeral in
  Redis/LRU with 1h TTL so transient failures get retried
- Lookup flow: LRU -> Redis -> Postgres (permanent) -> live SearXNG; DB hits
  re-warm the fast caches; stale Redis miss sentinels no longer shadow DB
- Rate-limit-aware live lookups: concurrency 2 + stagger, retry once on empty
  results, strict definition filter (Wikipedia preferred, rejects
  disambiguation/ads/translate-homepages)
- Make SEARXNG_BASE_URL configurable via env (default unchanged)
This commit is contained in:
asepharyana
2026-08-12 14:22:22 +07:00
parent f1aa08cdf6
commit d9f5592e6e
7 changed files with 347 additions and 77 deletions
@@ -435,6 +435,32 @@ export const pgStickerCacheTable = pgTable(
export const stickerCacheTable = pgStickerCacheTable;
/**
* Term Glossary Cache Table (PostgreSQL)
* Permanently stores resolved term definitions (Wikipedia/SearXNG lookups).
* Definitions rarely change, so once a term is successfully resolved it is
* persisted here forever — Redis/LRU only act as fast read caches in front.
* Terms with NO definition (misses) are NOT stored here; they stay ephemeral
* in Redis with a short TTL so transient lookup failures get retried.
*/
export const pgTermGlossaryCacheTable = pgTable(
"term_glossary_cache",
{
term: pgText("term").primaryKey(),
definition: pgText("definition").notNull(),
source_url: pgText("source_url").notNull().default(""),
resolved_at: pgBigint("resolved_at", { mode: "number" }).notNull(),
hit_count: pgInteger("hit_count").notNull().default(0),
},
(table) => ({
resolvedAtIdx: pgIndex("idx_term_glossary_cache_resolved_at").on(
table.resolved_at,
),
}),
);
export const termGlossaryCacheTable = pgTermGlossaryCacheTable;
// =============================================================================
// Meta / System
// =============================================================================
@@ -580,6 +606,11 @@ export type TextAnalysisCacheInsert =
export type StickerCacheRecord = typeof stickerCacheTable.$inferSelect;
export type StickerCacheInsert = typeof stickerCacheTable.$inferInsert;
// Term Glossary Cache
export type TermGlossaryCache = typeof termGlossaryCacheTable.$inferSelect;
export type TermGlossaryCacheInsert =
typeof termGlossaryCacheTable.$inferInsert;
// Muxer Jobs
export type MuxerJob = typeof muxerJobsTable.$inferSelect;
export type MuxerJobInsert = typeof muxerJobsTable.$inferInsert;