feat(glossary): persist resolved definitions in Postgres + harden live SearXNG lookups

- Add term_glossary_cache table + migration 0014: resolved definitions are
  stored permanently (definitions rarely change); misses stay ephemeral in
  Redis/LRU with 1h TTL so transient failures get retried
- Lookup flow: LRU -> Redis -> Postgres (permanent) -> live SearXNG; DB hits
  re-warm the fast caches; stale Redis miss sentinels no longer shadow DB
- Rate-limit-aware live lookups: concurrency 2 + stagger, retry once on empty
  results, strict definition filter (Wikipedia preferred, rejects
  disambiguation/ads/translate-homepages)
- Make SEARXNG_BASE_URL configurable via env (default unchanged)
This commit is contained in:
asepharyana
2026-08-12 14:22:22 +07:00
parent f1aa08cdf6
commit d9f5592e6e
7 changed files with 347 additions and 77 deletions
@@ -0,0 +1,9 @@
CREATE TABLE IF NOT EXISTS "term_glossary_cache" (
"term" text PRIMARY KEY NOT NULL,
"definition" text NOT NULL,
"source_url" text DEFAULT '' NOT NULL,
"resolved_at" bigint NOT NULL,
"hit_count" integer DEFAULT 0 NOT NULL
);
--> statement-breakpoint
CREATE INDEX IF NOT EXISTS "idx_term_glossary_cache_resolved_at" ON "term_glossary_cache" USING btree ("resolved_at");
@@ -99,6 +99,13 @@
"when": 1785551832190,
"tag": "0013_rename_mascot_chat_to_chatbot",
"breakpoints": true
},
{
"idx": 14,
"version": "7",
"when": 1785621600000,
"tag": "0014_add_term_glossary_cache",
"breakpoints": true
}
]
}