403 lines
14 KiB
TypeScript
403 lines
14 KiB
TypeScript
/**
|
|
* termGlossary.ts
|
|
*
|
|
* Per-word "kamus" enrichment for LLM moderation.
|
|
*
|
|
* Problem: the moderation LLM often meets words it does not know — regional
|
|
* slang (Jawa/Sunda), foreign terms, niche anime/game jargon, or obscure
|
|
* technical vocabulary. When it guesses, it either invents a wrong meaning
|
|
* (false positive on a safe word) or misses a violation hidden in unfamiliar
|
|
* wording (false negative on an unknown vulgar/slang term).
|
|
*
|
|
* Solution: extract candidate "unknown-looking" words from message content,
|
|
* look each one up on Wikipedia via the Wikipedia REST/Action APIs, and inject
|
|
* the definitions into the LLM prompt as a `<term_glossary>` block so verdicts
|
|
* are based on facts instead of guesses.
|
|
*
|
|
* Cost control & persistence:
|
|
* - successfully resolved definitions are PERSISTED PERMANENTLY in Postgres
|
|
* (`term_glossary_cache`) — definitions rarely change, so a resolved term
|
|
* is never searched again; only misses stay ephemeral (Redis/LRU, 1h);
|
|
* - in-memory LRU + Redis (shared cache store) sit in front of
|
|
* the DB as fast read caches, so repeat lookups are effectively free;
|
|
* - lookups per batch are bounded (AI_GLOSSARY_MAX_TERMS);
|
|
* - live Wikipedia calls are rate-limit aware: concurrency 2 + stagger, retry
|
|
* once on empty results, and misses cached for only 1h so a limiter or
|
|
* network blip is not treated as a permanent miss;
|
|
* - only results that read like actual definitions are accepted (Wikipedia
|
|
* preferred; disambiguation/ads/translate-homepages rejected);
|
|
* - everything degrades gracefully: no Redis, no Wikipedia API, no match
|
|
* → the block is simply omitted and moderation proceeds as before.
|
|
*/
|
|
|
|
import { LRUCache } from "lru-cache";
|
|
import pLimit from "p-limit";
|
|
import { createChildLogger } from "@/shared/logger/index";
|
|
import { delay } from "@/shared/utils/index";
|
|
import { config } from "../../shared/config/config.js";
|
|
import { cacheGet, cacheSet, makeCacheKey } from "./cacheStore.js";
|
|
import { escapeXml } from "./moderationBuilders.js";
|
|
import {
|
|
getTermDefinitionFromDb,
|
|
setTermDefinitionInDb,
|
|
} from "./termGlossaryStore.js";
|
|
import {
|
|
cleanContent,
|
|
isKnownTerm,
|
|
isMostlyStopwords,
|
|
isNoiseWord,
|
|
scoreWord,
|
|
WORD_RE,
|
|
} from "./textSignals.js";
|
|
import { wikipediaSummary } from "./wikipediaClient.js";
|
|
|
|
const log = createChildLogger("term-glossary");
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Constants
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/** Redis TTL for a successfully resolved definition (definitions are stable). */
|
|
const DEF_TTL_SECONDS = 7 * 24 * 60 * 60;
|
|
/**
|
|
* Redis TTL for a lookup that found nothing. Kept SHORT (1h): SearXNG
|
|
* instances silently return empty result sets when rate-limited, so an empty
|
|
* response is often a transient failure, not a real miss. A short TTL lets
|
|
* the term be retried on a later batch instead of poisoning it for a day.
|
|
*/
|
|
const MISS_TTL_SECONDS = 60 * 60;
|
|
const MISS_TTL_MS = MISS_TTL_SECONDS * 1000;
|
|
/** Sentinel stored in caches for "term has no resolvable definition". */
|
|
const EMPTY_SENTINEL = "__not_found__";
|
|
/** Delay before retrying a search that returned zero results. */
|
|
const RETRY_DELAY_MS = 350;
|
|
/** Max definition snippet length kept in the prompt. */
|
|
const MAX_DEFINITION_CHARS = 300;
|
|
/**
|
|
* Wikipedia can be flaky under aggressive parallel bursts. Never fire all
|
|
* terms at once — cap live lookups at 2 concurrent and stagger the start times.
|
|
*/
|
|
const LIVE_SEARCH_CONCURRENCY = 2;
|
|
const LIVE_SEARCH_STAGGER_MS = 250;
|
|
|
|
/** In-memory cache: term (lowercase) → definition | NOT_FOUND sentinel. */
|
|
const NOT_FOUND: TermDefinition = {
|
|
term: "__not_found__",
|
|
definition: "",
|
|
sourceUrl: "",
|
|
};
|
|
const termLru = new LRUCache<string, TermDefinition>({
|
|
max: 2000,
|
|
ttl: 24 * 60 * 60 * 1000,
|
|
});
|
|
|
|
/** Serializes live Wikipedia lookups (rate-limit aware) with a small stagger. */
|
|
const liveSearchLimit = pLimit(LIVE_SEARCH_CONCURRENCY);
|
|
let lastLiveSearchAt = 0;
|
|
async function acquireLiveSlot(): Promise<void> {
|
|
const now = Date.now();
|
|
const wait = lastLiveSearchAt + LIVE_SEARCH_STAGGER_MS - now;
|
|
if (wait > 0) await delay(wait);
|
|
lastLiveSearchAt = Date.now();
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Term extraction
|
|
// ---------------------------------------------------------------------------
|
|
// Tokenizer, stopword/known-term filters, and word scoring now live in
|
|
// textSignals.ts (2026-08-31) — shared with wikipediaClient.ts's search-query
|
|
// extractor so both use the exact same "is this word worth looking up"
|
|
// signals instead of two divergent copies.
|
|
|
|
export interface ExtractGlossaryOptions {
|
|
maxTerms?: number;
|
|
minWordLength?: number;
|
|
}
|
|
|
|
/**
|
|
* Extracts candidate terms that the LLM might not know from message content.
|
|
* Returns at most `maxTerms` terms (default from config), scored by how
|
|
* "unknown-looking" they are (proper nouns, foreign spelling, quoted phrases).
|
|
*/
|
|
export function extractGlossaryTerms(
|
|
contents: string[],
|
|
options: ExtractGlossaryOptions = {},
|
|
): string[] {
|
|
const maxTerms = options.maxTerms ?? config.AI_GLOSSARY_MAX_TERMS;
|
|
const minWordLength =
|
|
options.minWordLength ?? config.AI_GLOSSARY_MIN_WORD_LENGTH;
|
|
|
|
const candidates = new Map<string, { word: string; score: number }>();
|
|
|
|
const push = (rawWord: string, score: number): void => {
|
|
const clean = rawWord
|
|
.trim()
|
|
.replace(/^[^\p{L}\p{N}]+|[^\p{L}\p{N}]+$/gu, "");
|
|
if (clean.length < minWordLength) return;
|
|
const key = clean.toLowerCase();
|
|
if (isKnownTerm(key) || isNoiseWord(clean)) return;
|
|
const existing = candidates.get(key);
|
|
if (existing) {
|
|
existing.score += score + 1;
|
|
} else {
|
|
candidates.set(key, { word: clean, score });
|
|
}
|
|
};
|
|
|
|
for (const content of contents) {
|
|
if (!content) continue;
|
|
const cleaned = cleanContent(content);
|
|
if (!cleaned) continue;
|
|
|
|
// Quoted phrases — explicit terms the user called out
|
|
for (const m of cleaned.matchAll(/"([^"]{2,80})"/g)) {
|
|
const phrase = m[1].trim();
|
|
const wordCount = phrase.split(/\s+/).length;
|
|
if (wordCount >= 2 && wordCount <= 6 && !isMostlyStopwords(phrase)) {
|
|
push(phrase, 10);
|
|
}
|
|
}
|
|
|
|
// Individual words
|
|
for (const m of cleaned.matchAll(WORD_RE)) {
|
|
const w = m[0];
|
|
if (w.length < minWordLength) continue;
|
|
if (isNoiseWord(w)) continue;
|
|
const key = w.toLowerCase();
|
|
if (isKnownTerm(key)) continue;
|
|
push(w, scoreWord(w));
|
|
}
|
|
}
|
|
|
|
return Array.from(candidates.values())
|
|
.sort((a, b) => b.score - a.score)
|
|
.slice(0, maxTerms)
|
|
.map((c) => c.word);
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// ─── Definition lookup (cached: LRU → Redis → Wikipedia) ───────────────────
|
|
// ---------------------------------------------------------------------------
|
|
|
|
export interface TermDefinition {
|
|
term: string;
|
|
definition: string;
|
|
sourceUrl: string;
|
|
}
|
|
|
|
/** Per-search timeout — keep glossary lookups snappy even on a slow Wikipedia. */
|
|
const GLOSSARY_SEARCH_TIMEOUT_MS = 5000;
|
|
|
|
function buildDefinition(
|
|
best: { title: string; url: string; snippet: string },
|
|
term: string,
|
|
): TermDefinition {
|
|
const snippet = (best.snippet || best.title || "").trim();
|
|
const definition =
|
|
snippet.length > MAX_DEFINITION_CHARS
|
|
? `${snippet.slice(0, MAX_DEFINITION_CHARS - 1).trimEnd()}…`
|
|
: snippet;
|
|
return { term, definition, sourceUrl: best.url };
|
|
}
|
|
|
|
/** Live (network) lookup — runs under the shared Wikipedia rate-limit gate. */
|
|
async function fetchDefinitionLive(
|
|
term: string,
|
|
key: string,
|
|
cacheKey: string,
|
|
): Promise<TermDefinition | null> {
|
|
return liveSearchLimit(async () => {
|
|
await acquireLiveSlot();
|
|
try {
|
|
let result = await wikipediaSummary(key, GLOSSARY_SEARCH_TIMEOUT_MS);
|
|
let def = result ? buildDefinition(result, term) : null;
|
|
// Zero result is usually the limiter/network blip, not a real miss —
|
|
// retry once. Result-but-unusable = genuine miss, no retry.
|
|
if (!def) {
|
|
await delay(RETRY_DELAY_MS);
|
|
result = await wikipediaSummary(key, GLOSSARY_SEARCH_TIMEOUT_MS);
|
|
def = result ? buildDefinition(result, term) : null;
|
|
}
|
|
|
|
if (def) {
|
|
// Persist permanently (definitions rarely change) — best-effort,
|
|
// then warm the fast caches.
|
|
void setTermDefinitionInDb(key, def.definition, def.sourceUrl);
|
|
cacheSet(
|
|
cacheKey,
|
|
JSON.stringify({
|
|
definition: def.definition,
|
|
sourceUrl: def.sourceUrl,
|
|
}),
|
|
DEF_TTL_SECONDS,
|
|
);
|
|
termLru.set(key, def);
|
|
log.debug({ term: key }, "Term glossary resolved definition");
|
|
return def;
|
|
}
|
|
} catch (err) {
|
|
log.debug(
|
|
{ term: key, error: err instanceof Error ? err.message : String(err) },
|
|
"Term glossary lookup failed — skipping term",
|
|
);
|
|
}
|
|
|
|
// No definition — cache the miss with a SHORT TTL so a transient
|
|
// limiter/network failure is retried on a later batch.
|
|
cacheSet(cacheKey, EMPTY_SENTINEL, MISS_TTL_SECONDS);
|
|
termLru.set(key, NOT_FOUND, { ttl: MISS_TTL_MS });
|
|
return null;
|
|
});
|
|
}
|
|
|
|
/** Resolve one term: LRU → Redis → Postgres (permanent) → live Wikipedia
|
|
* (rate-limited). The fast caches sit in front of the DB; the DB is the
|
|
* source of truth for successfully resolved definitions. */
|
|
async function resolveTerm(term: string): Promise<TermDefinition | null> {
|
|
const key = term.toLowerCase().trim();
|
|
|
|
// 1. In-memory LRU — same process, instant
|
|
const lruHit = termLru.get(key);
|
|
if (lruHit) return lruHit === NOT_FOUND ? null : lruHit;
|
|
|
|
// 2. Redis — shared across processes/workers. A miss sentinel here is NOT
|
|
// a definitive answer: it may predate a permanent DB entry written by
|
|
// another process, so we keep going and let the DB decide.
|
|
const cacheKey = makeCacheKey("def", key);
|
|
const cached = await cacheGet(cacheKey);
|
|
let redisMiss = false;
|
|
if (cached !== null) {
|
|
if (cached === EMPTY_SENTINEL) {
|
|
redisMiss = true;
|
|
} else {
|
|
try {
|
|
const parsed = JSON.parse(cached) as {
|
|
definition?: string;
|
|
sourceUrl?: string;
|
|
};
|
|
if (parsed.definition) {
|
|
const def: TermDefinition = {
|
|
term,
|
|
definition: parsed.definition,
|
|
sourceUrl: parsed.sourceUrl ?? "",
|
|
};
|
|
termLru.set(key, def);
|
|
return def;
|
|
}
|
|
} catch {
|
|
// malformed cache entry — fall through to DB/live
|
|
}
|
|
}
|
|
}
|
|
|
|
// 3. Postgres — permanent store for resolved definitions. A hit re-warms
|
|
// the fast caches so the DB is not hit on every batch.
|
|
const dbDef = await getTermDefinitionFromDb(key);
|
|
if (dbDef) {
|
|
const def: TermDefinition = {
|
|
term,
|
|
definition: dbDef.definition,
|
|
sourceUrl: dbDef.sourceUrl,
|
|
};
|
|
termLru.set(key, def);
|
|
cacheSet(
|
|
cacheKey,
|
|
JSON.stringify({ definition: def.definition, sourceUrl: def.sourceUrl }),
|
|
DEF_TTL_SECONDS,
|
|
);
|
|
log.debug({ term: key }, "Term glossary DB hit");
|
|
return def;
|
|
}
|
|
|
|
// 4. Redis already said "miss" recently and the DB has nothing — respect
|
|
// that instead of hammering SearXNG again within the miss window.
|
|
if (redisMiss) {
|
|
termLru.set(key, NOT_FOUND, { ttl: MISS_TTL_MS });
|
|
return null;
|
|
}
|
|
|
|
// 5. Live search (rate-limited + staggered)
|
|
return fetchDefinitionLive(term, key, cacheKey);
|
|
}
|
|
|
|
/**
|
|
* Looks up definitions for a batch of terms, in parallel. Returns a map of
|
|
* term → definition for the terms that resolved. Errors/misses are skipped.
|
|
* Live SearXNG calls are throttled internally (concurrency 2 + stagger).
|
|
*/
|
|
export async function lookupTermDefinitions(
|
|
terms: string[],
|
|
): Promise<Map<string, TermDefinition>> {
|
|
const map = new Map<string, TermDefinition>();
|
|
if (terms.length === 0) return map;
|
|
|
|
const results = await Promise.allSettled(terms.map(resolveTerm));
|
|
for (let i = 0; i < terms.length; i++) {
|
|
const r = results[i];
|
|
if (r.status === "fulfilled" && r.value) {
|
|
map.set(r.value.term, r.value);
|
|
}
|
|
}
|
|
return map;
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Prompt formatting
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/**
|
|
* Formats definitions as a `<term_glossary>` XML block for the LLM prompt:
|
|
*
|
|
* <term_glossary>
|
|
* <term word="ngab" source="https://…">definisi…</term>
|
|
* </term_glossary>
|
|
*
|
|
* Returns "" when there are no definitions (the block is then omitted).
|
|
*/
|
|
export function formatTermGlossary(
|
|
defs: ReadonlyMap<string, TermDefinition>,
|
|
): string {
|
|
if (!defs || defs.size === 0) return "";
|
|
const lines = Array.from(defs.values()).map(
|
|
(d) =>
|
|
` <term word="${escapeXml(d.term)}" source="${escapeXml(d.sourceUrl)}">${escapeXml(d.definition)}</term>`,
|
|
);
|
|
return `<term_glossary>\n${lines.join("\n")}\n</term_glossary>`;
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Convenience: full pipeline
|
|
// ---------------------------------------------------------------------------
|
|
|
|
export interface GlossaryBlockOptions extends ExtractGlossaryOptions {
|
|
enabled?: boolean;
|
|
}
|
|
|
|
/**
|
|
* One-shot helper: extract terms from message contents, look up definitions,
|
|
* and return the formatted `<term_glossary>` block ("" when disabled or no
|
|
* definitions found). Safe to call on every batch — cached lookups make it
|
|
* cheap.
|
|
*/
|
|
export async function buildTermGlossaryBlock(
|
|
contents: string[],
|
|
options: GlossaryBlockOptions = {},
|
|
): Promise<string> {
|
|
const enabled = options.enabled ?? config.AI_GLOSSARY_ENABLED;
|
|
if (!enabled) return "";
|
|
if (contents.length === 0) return "";
|
|
|
|
const terms = extractGlossaryTerms(contents, options);
|
|
if (terms.length === 0) return "";
|
|
|
|
const defs = await lookupTermDefinitions(terms);
|
|
if (defs.size === 0) return "";
|
|
|
|
const block = formatTermGlossary(defs);
|
|
log.debug(
|
|
{ terms: terms.length, definitions: defs.size },
|
|
"Term glossary block built",
|
|
);
|
|
return block;
|
|
}
|