From c7f53e4f7e49853d60c87a746ba67ee85d3abb4a Mon Sep 17 00:00:00 2001 From: asepharyana Date: Thu, 24 Sep 2026 12:06:36 +0700 Subject: [PATCH] feat(gateway): remove Jev (System One) analyzer, restore LLM-only text moderation Jev (oc/jev-1.13-free via 9router /v1/systemone) added as primary text analyzer was underperforming. Delete the whole feature: - jevAnalyzer.ts + its unit & live-smoke tests - Jev-first branch in textBatchProcessor, restore pure callModerationLLM path - AI_LLM_JEV_* config vars (zod) and .env.example entries - @typesafe-ai/sdk dependency (+ lockfile) Behavior: text moderation is LLM-only again, exactly as before the Jev feature; AGENTS.md invariant 'LLM is the only judge' holds. --- .env.example | 6 - services/discord-gateway/package.json | 1 - services/discord-gateway/pnpm-lock.yaml | 9 - .../src/modules/ai-moderation/jevAnalyzer.ts | 563 ------------------ .../ai-moderation/textBatchProcessor.ts | 99 +-- .../src/shared/config/index.ts | 17 - .../tests/jev-live-smoke.test.ts | 94 --- .../discord-gateway/tests/jevAnalyzer.test.ts | 382 ------------ 8 files changed, 13 insertions(+), 1158 deletions(-) delete mode 100644 services/discord-gateway/src/modules/ai-moderation/jevAnalyzer.ts delete mode 100644 services/discord-gateway/tests/jev-live-smoke.test.ts delete mode 100644 services/discord-gateway/tests/jevAnalyzer.test.ts diff --git a/.env.example b/.env.example index fb65cc35..a88500df 100644 --- a/.env.example +++ b/.env.example @@ -74,12 +74,6 @@ AI_LLM_IMAGE_MAX_DIMENSION=1024 # Max image dimension in pixels before r AI_LLM_TEXT_BATCH_SIZE=20 # Max messages per text-only moderation batch (default: 20) AI_LLM_MEDIA_ANALYSIS_TIMEOUT_MS=60000 # Timeout in ms for media analysis calls (default: 60000) AI_LLM_TEXT_ANALYSIS_TIMEOUT_MS=30000 # Timeout in ms for text-only analysis calls (default: 30000) -AI_LLM_JEV_ENABLED=false # Use TypeSafe Jev (System One) as PRIMARY text analyzer; LLM is fallback -AI_LLM_JEV_API_KEY= # REQUIRED if AI_LLM_JEV_ENABLED=true. 9router/TypeSafe API key for /v1/systemone -AI_LLM_JEV_BASE_URL=http://127.0.0.1:4014 # 9router base URL (default: local 9router; prod: https://9router.asepharyana.my.id) -AI_LLM_JEV_MODEL=oc/jev-1.13-free # Jev model id (default: oc/jev-1.13-free) -AI_LLM_JEV_TIMEOUT_MS=45000 # Timeout in ms for a Jev systemone batch call (default: 45000) -AI_LLM_JEV_MIN_CONFIDENCE=0.9 # Min status-choice confidence to accept a Jev verdict (default: 0.9) # === AI Analysis Tuning === AI_ANALYSIS_DEBOUNCE_MS=500 # Debounce window for batching messages in ms (default: 500) diff --git a/services/discord-gateway/package.json b/services/discord-gateway/package.json index 1b092382..733bc995 100644 --- a/services/discord-gateway/package.json +++ b/services/discord-gateway/package.json @@ -23,7 +23,6 @@ "test:e2e:live": "bash scripts/run-llm-e2e.sh" }, "dependencies": { - "@typesafe-ai/sdk": "^0.6.0", "axios": "^1.20.0", "discord.js-selfbot-v13": "^3.7.1", "dotenv": "^18.0.0", diff --git a/services/discord-gateway/pnpm-lock.yaml b/services/discord-gateway/pnpm-lock.yaml index 148b4224..7d943c26 100644 --- a/services/discord-gateway/pnpm-lock.yaml +++ b/services/discord-gateway/pnpm-lock.yaml @@ -8,9 +8,6 @@ importers: .: dependencies: - '@typesafe-ai/sdk': - specifier: ^0.6.0 - version: 0.6.0 axios: specifier: ^1.20.0 version: 1.20.0(debug@4.4.3(supports-color@7.2.0))(supports-color@7.2.0) @@ -1090,10 +1087,6 @@ packages: '@types/ws@8.18.1': resolution: {integrity: sha512-ThVF6DCVhA8kUGy+aazFQ4kXQ7E1Ty7A3ypFOe0IcJV8O/M511G99AW24irKrW56Wt44yG9+ij8FaqoBGkuBXg==} - '@typesafe-ai/sdk@0.6.0': - resolution: {integrity: sha512-IddX+Q0XM+VagOUZFeP7wZjaO4SHMdvnh2zEBdrZZnXedWI3BNK1lKhMx3ayrkFWvVLbVcUHJy6AVZlY+e6Jaw==} - engines: {node: '>=20'} - '@typescript/typescript-aix-ppc64@7.0.2': resolution: {integrity: sha512-MTKKkWB7p/0E9xi1d1tHtZ5PiLkGEMIq88pK2CubZjOsLtYTLqhgIgi6zepFa+9GHZ6h05NMCkQxGKiPXMxXtQ==} engines: {node: '>=16.20.0'} @@ -2993,8 +2986,6 @@ snapshots: dependencies: '@types/node': 26.4.0 - '@typesafe-ai/sdk@0.6.0': {} - '@typescript/typescript-aix-ppc64@7.0.2': optional: true diff --git a/services/discord-gateway/src/modules/ai-moderation/jevAnalyzer.ts b/services/discord-gateway/src/modules/ai-moderation/jevAnalyzer.ts deleted file mode 100644 index c0d2e6d8..00000000 --- a/services/discord-gateway/src/modules/ai-moderation/jevAnalyzer.ts +++ /dev/null @@ -1,563 +0,0 @@ -/** - * jevAnalyzer.ts - * - * Jev (TypeSafe System One, `oc/jev-1.13-free` via 9router `/v1/systemone`) - * — the PRIMARY analyzer for text-only moderation sub-batches. The existing - * LLM (`llmChat`) stays as the fallback for any message Jev cannot decide - * confidently (see the acceptance gate) and for media batches (Jev is - * decision-only, no image input). - * - * CRITICAL framing rule (verified 2026-09-23, live probes): - * Jev is a System One model — it evaluates typed questions against a STATE. - * Feeding it the chat-optimized `SYSTEM_RULES` verbatim INSIDE chat-style - * XML (``, ``, …) makes it - * pattern-match the structure and return CONFIDENTLY WRONG verdicts - * (flagged clean messages at confidence 0.98 in a probe — would pass any - * naive gate and could auto-delete innocent content). - * - * The state MUST be declarative facts: - * OBJEK PENILAIAN / PESAN: `- Pesan "" dari "": ""` / - * KEBIJAKAN as statements / KONTEKS as statements - * and the questions phrased as "is this true" / "classify this" against - * those facts. With that framing the same 4-message probe returned 4/4 - * correct verdicts at confidence 1.0, including the SARA zero-tolerance - * case and the technical-clean case. - * - * The distilled `JEV_POLICY` below is a compact declarative summary of the - * full chat policy (`prompts/rules.ts` SYSTEM_RULES). It is deliberately - * kept short (~300 tokens) — the LLM keeps the full 12k-char policy; Jev - * triages on the core axes, and anything it can't decide confidently falls - * back to the LLM. Keep this block in sync when SYSTEM_RULES changes. - */ - -import type { Question, Questions } from "@typesafe-ai/sdk"; -import { choice, noul, TypeSafeClient } from "@typesafe-ai/sdk"; -import { createChildLogger } from "@/shared/logger/index"; -import { config } from "../../shared/config/config.js"; -import { incrementCounterBy } from "../gateway-metrics/index.js"; -import type { AnalysisResult } from "../message-capture/types.js"; - -const log = createChildLogger("jev-analyzer"); - -// --------------------------------------------------------------------------- -// Policy + vocab -// --------------------------------------------------------------------------- - -/** - * Distilled declarative policy for Jev. DERIVED from `SYSTEM_RULES` - * (prompts/rules.ts) — update this when the full policy changes. Kept as - * factual statements, NOT instructions (System One evaluates truth). - */ -export const JEV_POLICY = `KEBIJAKAN SERVER (fakta yang berlaku): -- Kata vulgar anatomi (kontol, memek, tit, dick, dll) = pelanggaran berat, tanpa kecuali. -- SARA / penistaan agama / parodi ayat / mockery tokoh agama / provokasi antar-agama = pelanggaran berat. -- Promosi atau diskusi LGBT = pelanggaran berat (zero-tolerance). -- Diskusi Israel/Palestina/Yahudi = pelanggaran berat (zero-tolerance). -- Hinaan terarah ke orang (harassment), seksisme, ageisme, diskriminasi fisik = pelanggaran. -- Konten seksual eksplisit / ajakan seksual / fetish / lolicon-shota = pelanggaran. -- Judi, narkoba, scam, doxxing, ancaman kekerasan, self-harm, child safety, konten ilegal = pelanggaran. -- Teknik evasi (zalgo, leetspeak, regional indicator, simbol acak) yang menyembunyikan kata terlarang = pelanggaran. -- Spam berulang / promosi = pelanggaran ringan. -- Memancing konflik (conflict instigation) = pelanggaran ringan. -- Username ofensif saja (isi pesan bersih) = peringatan ringan, BUKAN hapus pesan. -- Percakapan teknis/normal, slang santai (anjay, wkwk, gaskeun, njir), typo, panggilan akrab (bang, kak, dek), ekspresi religius normal (astaghfirullah, alhamdulillah), istilah anime (waifu, wibu), lirik/kutipan, makian ke benda mati = BUKAN pelanggaran. -- Teks acak (kode, log, stack trace, output API, cuplikan UI) = BUKAN pelanggaran. -- Setiap pesan dinilai dari isinya sendiri; konteks percakapan dapat memengaruhi interpretasi, bukan menggantikan isi.`; - -/** Choice labels must stay in sync with `AIRecommendedAction` (moderation-types). */ -export const JEV_ACTIONS = [ - "none", - "monitor", - "warn", - "review", - "delete", - "escalate", -] as const; - -/** Category choices — the moderation category vocabulary (kept tight). */ -export const JEV_CATEGORIES = [ - "none", - "harassment", - "hate_speech", - "sara", - "sexual_content", - "vulgar_language", - "sexual_deviation", - "self_harm", - "violence", - "illegal_content", - "gambling", - "drugs", - "scam", - "spam", - "conflict_instigation", - "offensive_username", - "other", -] as const; - -export const JEV_STATUSES = ["clean", "warn", "flagged"] as const; -export const JEV_SEVERITIES = [ - "none", - "low", - "medium", - "high", - "critical", -] as const; - -/** Policy version stamped on every Jev verdict (cache/DB provenance). */ -export const JEV_POLICY_VERSION = "jev-systemone-2026-09-23"; - -/** Lazily-built SDK client (config resolves at first use). */ -let client: TypeSafeClient | null = null; -let clientKey = ""; - -function getClient(): TypeSafeClient | null { - if (!config.AI_LLM_JEV_API_KEY) return null; - if (!client || clientKey !== config.AI_LLM_JEV_API_KEY) { - client = new TypeSafeClient({ - apiKey: config.AI_LLM_JEV_API_KEY, - baseURL: config.AI_LLM_JEV_BASE_URL, - defaultModel: config.AI_LLM_JEV_MODEL, - timeout: config.AI_LLM_JEV_TIMEOUT_MS, - retry: { maxRetries: 0 }, // pipeline owns retries/abort - logLevel: "warn", - }); - clientKey = config.AI_LLM_JEV_API_KEY; - log.info( - { - baseURL: config.AI_LLM_JEV_BASE_URL, - model: config.AI_LLM_JEV_MODEL, - }, - "Jev analyzer client initialized", - ); - } - return client; -} - -// --------------------------------------------------------------------------- -// Question + state builders -// --------------------------------------------------------------------------- - -export interface JevTarget { - /** Message id — echoed verbatim into analysis/result. */ - id: string; - /** Display name shown to the model (username). */ - user: string; - /** Content to evaluate (truncated by caller). */ - content: string; -} - -/** - * Build per-message questions, keyed by the message id itself so the - * questions read naturally against the declarative state facts - * ("pesan melanggar kebijakan server"). Five questions per message. - */ -export function buildJevQuestions(targets: JevTarget[]): Questions { - const questions: Record = {}; - for (const t of targets) { - const k = t.id; - questions[`${k}__v`] = noul( - `Pesan ${JSON.stringify(t.id)} dari ${JSON.stringify(t.user)} melanggar kebijakan server`, - ); - questions[`${k}__status`] = choice( - `Klasifikasi pesan ${JSON.stringify(t.id)} dari ${JSON.stringify(t.user)}`, - { - clean: "tidak melanggar kebijakan", - warn: "pelanggaran ringan", - flagged: "melanggar kebijakan", - }, - ); - questions[`${k}__severity`] = choice( - `Tingkat keparahan pelanggaran pesan ${JSON.stringify(t.id)}`, - { - none: "tidak ada pelanggaran", - low: "ringan", - medium: "sedang", - high: "berat", - critical: "kritis/darurat", - }, - ); - questions[`${k}__category`] = choice( - `Kategori utama pelanggaran pesan ${JSON.stringify(t.id)}`, - Object.fromEntries(JEV_CATEGORIES.map((c) => [c, null])), - ); - questions[`${k}__action`] = choice( - `Tindakan moderasi yang tepat untuk pesan ${JSON.stringify(t.id)}`, - { - none: "tidak ada tindakan", - monitor: "pantau", - warn: "beri peringatan", - review: "tinjau manual", - delete: "hapus pesan", - escalate: "eskalasi", - }, - ); - } - return questions as Questions; -} - -export interface JevBatchContext { - /** Context block (location/conversation) as raw XML or prose — stripped to facts. */ - contextBlock: string; - /** `` XML block (may be ""). */ - webSearchBlock: string; - /** `` XML block (may be ""). */ - glossaryBlock: string; - /** Raw channel culture summary (may be undefined). */ - channelCulture?: string; -} - -/** - * Strip XML/HTML tags from a raw block and collapse whitespace so it can be - * restated as plain factual prose in the declarative state. Empty after - * stripping → omitted from the state. - */ -function stripToFacts(block: string, maxLength: number): string | null { - const cleaned = block - .replace(/<[^>]+>/g, " ") - .replace(/\s+/g, " ") - .trim(); - if (!cleaned) return null; - return JSON.stringify(cleaned.slice(0, maxLength)); -} - -/** - * Build the declarative `state` payload. NO chat/XML scaffolding — plain - * factual statements (see the framing rule above; chat-style injection - * makes Jev confidently wrong). - */ -export function buildJevState( - targets: JevTarget[], - ctx: JevBatchContext, - correctedExamples = "", -): string { - const facts = targets.map( - (t) => - `- Pesan ${JSON.stringify(t.id)} dari ${JSON.stringify(t.user)}: ${JSON.stringify(t.content)}`, - ); - const parts = [ - `OBJEK PENILAIAN: ${targets.length} pesan dari server Discord.`, - "PESAN:", - ...facts, - JEV_POLICY, - ]; - - // Conversation/who context as facts (declarative, not instructions). - const extraFacts: string[] = []; - if (ctx.channelCulture) { - extraFacts.push( - `KULTUR CHANNEL (fakta): ${JSON.stringify(ctx.channelCulture.slice(0, 800))}`, - ); - } - const contextFacts = stripToFacts(ctx.contextBlock, 1200); - if (contextFacts) extraFacts.push(`KONTEKS: ${contextFacts}`); - const webSearchFacts = stripToFacts(ctx.webSearchBlock, 1500); - if (webSearchFacts) extraFacts.push(`HASIL PENCARIAN WEB: ${webSearchFacts}`); - const glossaryFacts = stripToFacts(ctx.glossaryBlock, 800); - if (glossaryFacts) extraFacts.push(`GLOSARIUM: ${glossaryFacts}`); - const correctionFacts = stripToFacts(correctedExamples, 800); - if (correctionFacts) - extraFacts.push(`KOREKSI SEBELUMNYA: ${correctionFacts}`); - if (extraFacts.length > 0) parts.push(...extraFacts); - - return parts.join("\n"); -} - -// --------------------------------------------------------------------------- -// Acceptance gate + mapper -// --------------------------------------------------------------------------- - -/** Shape of the raw `answers` map returned by `systemOne`. */ -export type JevAnswers = Record< - string, - | { type: "noul"; noul: number } - | { - type: "choice"; - choice: string; - confidence: number; - probabilities?: Record; - } ->; - -/** Per-message answer subset (nullable until validated — `answersOf`). */ -interface JevMessageAnswers { - v?: { type: "noul"; noul: number }; - status?: { type: "choice"; choice: string; confidence: number }; - severity?: { type: "choice"; choice: string }; - category?: { type: "choice"; choice: string }; - action?: { type: "choice"; choice: string }; -} - -/** Reads the per-message answer subset by id, missing → undefined. */ -function answersOf(answers: JevAnswers, id: string): JevMessageAnswers { - return { - v: answers[`${id}__v`] as JevMessageAnswers["v"], - status: answers[`${id}__status`] as JevMessageAnswers["status"], - severity: answers[`${id}__severity`] as JevMessageAnswers["severity"], - category: answers[`${id}__category`] as JevMessageAnswers["category"], - action: answers[`${id}__action`] as JevMessageAnswers["action"], - }; -} - -/** Optionally-typed accessor for a choice answer's label ("" when missing). */ -function labelOf(a: { type: "choice"; choice: string } | undefined): string { - return a?.type === "choice" ? a.choice : ""; -} - -/** Set form of the vocab arrays for O(1) membership tests. */ -const JEV_STATUS_SET = new Set(JEV_STATUSES); -const JEV_SEVERITY_SET = new Set(JEV_SEVERITIES); -const JEV_CATEGORY_SET = new Set(JEV_CATEGORIES); -const JEV_ACTION_SET = new Set(JEV_ACTIONS); - -/** - * Decide per-message Jev acceptance. Requires ALL five questions present - * with valid labels and CROSS-CONSISTENT semantics: - * - status choice confidence >= threshold - * - status == clean ⟺ noul < 0.5 (flagged/warn need noul ≥ 0.5) - * - severity == none ⟺ status == clean (flagged must have severity) - * - action == none ⟺ status == clean; warn must not delete/escalate; - * clean must never delete/escalate - * - category == none ⟺ status == clean - * Anything else → LLM fallback (fail-open). - */ -export function isJevAccepted( - answers: JevAnswers, - messageId: string, - minConfidence: number, -): boolean { - const a = answersOf(answers, messageId); - if (!a.v || a.v.type !== "noul" || typeof a.v.noul !== "number") return false; - if (!a.status || a.status.type !== "choice" || !a.status.choice) return false; - if (!a.severity || a.severity.type !== "choice" || !a.severity.choice) - return false; - if (!a.category || a.category.type !== "choice" || !a.category.choice) - return false; - if (!a.action || a.action.type !== "choice" || !a.action.choice) return false; - - const { status, severity, category, action } = a; - if ( - typeof status.confidence !== "number" || - status.confidence < minConfidence - ) - return false; - - // Valid label = one of the vocab const arrays. The membership guards - // (Set.has) reject anything unknown, then the labels are narrowed via the - // const-array includes so the downstream comparisons typecheck. - const statusLabel = labelOf(status); - const severityLabel = labelOf(severity); - const categoryLabel = labelOf(category); - const actionLabel = labelOf(action); - if ( - !JEV_STATUS_SET.has(statusLabel) || - !JEV_SEVERITY_SET.has(severityLabel) || - !JEV_CATEGORY_SET.has(categoryLabel) || - !JEV_ACTION_SET.has(actionLabel) - ) - return false; // unknown label — LLM fallback - - const s = statusLabel as (typeof JEV_STATUSES)[number]; - const sev = severityLabel as (typeof JEV_SEVERITIES)[number]; - const cat = categoryLabel as (typeof JEV_CATEGORIES)[number]; - const act = actionLabel as (typeof JEV_ACTIONS)[number]; - - const noulVal = a.v.noul; - - // noul ↔ status consistency - if (s === "clean" && noulVal >= 0.5) return false; - if (s !== "clean" && noulVal < 0.5) return false; - // severity ↔ status: clean must be none; flagged/warn must NOT be none - if (s === "clean" && sev !== "none") return false; - if (s !== "clean" && sev === "none") return false; - // action ↔ status: clean must be none; flagged must NOT be none; - // warn must not delete/escalate; clean must never delete/escalate - if (s === "clean" && act !== "none") return false; - if (s === "flagged" && act === "none") return false; - if (s === "warn" && (act === "delete" || act === "escalate")) return false; - // category ↔ status: clean must be none; flagged must NOT be none - if (s === "clean" && cat !== "none") return false; - if (s === "flagged" && cat === "none") return false; - - return true; -} - -/** - * Map accepted Jev answers for one message into the pipeline's `AnalysisResult`. - * All values are derived from the model's own typed answers — no fabrication. - */ -export function mapJevAnswersToResult( - answers: JevAnswers, - messageId: string, -): AnalysisResult { - const a = answersOf(answers, messageId); - const v = a.v as { type: "noul"; noul: number }; - const st = a.status as { - type: "choice"; - choice: string; - confidence: number; - }; - const sev = labelOf(a.severity); - const cat = labelOf(a.category); - const act = labelOf(a.action); - - const status = st.choice as (typeof JEV_STATUSES)[number]; - // Calibrated score: clean → 0; warn → 0.45; flagged → P(violates) clamped. - const rawNoul = typeof v.noul === "number" ? v.noul : 0; - const score = - status === "clean" - ? 0 - : status === "warn" - ? 0.45 - : Math.min(1, Math.max(0.7, rawNoul)); - const confidence = - typeof st.confidence === "number" - ? st.confidence - : config.AI_LLM_JEV_MIN_CONFIDENCE; - - return { - messageId, - status, - flags: cat === "none" ? [] : [cat], - score, - analysis: - `[Jev] status=${status}, kategori=${cat}, keparahan=${sev}, ` + - `keyakinan=${confidence.toFixed(2)}, tindakan=${act}, p_melanggar=${rawNoul.toFixed(2)}`, - categories: cat === "none" ? [] : [cat], - severity: sev as AnalysisResult["severity"], - confidence, - recommendedAction: act as AnalysisResult["recommendedAction"], - policyVersion: JEV_POLICY_VERSION, - evidence: [], - }; -} - -// --------------------------------------------------------------------------- -// Batch entry point (one systemOne call per sub-batch) -// --------------------------------------------------------------------------- - -export interface JevBatchOutcome { - /** Accepted Jev verdicts (keyed by message id). */ - results: AnalysisResult[]; - /** Answers the gate rejected for ANY reason (keys = message ids). */ - rejectedIds: string[]; - /** Raw `SystemOneResult` (for usage logging / raw passthrough). */ - raw: unknown; - /** Error thrown by the call, if the whole call failed (null = success). */ - error: string | null; -} - -/** True when Jev is configured and enabled (fail-open wrapper). */ -export function isJevEnabled(): boolean { - return ( - config.AI_LLM_JEV_ENABLED === true && Boolean(config.AI_LLM_JEV_API_KEY) - ); -} - -/** - * Analyze one text sub-batch with Jev. NEVER throws for API-level failures — - * returns `{ error }` so the caller falls back to the LLM. Aborts (signal) - * propagate as errors too (the caller's timeout must abort the SDK call and - * fall back, not hang). - */ -export async function analyzeBatchWithJev( - targets: JevTarget[], - ctx: JevBatchContext, - signal?: AbortSignal, - correctedExamples = "", -): Promise { - const outcome: JevBatchOutcome = { - results: [], - rejectedIds: [], - raw: null, - error: null, - }; - if (targets.length === 0) return outcome; - - const jevClient = getClient(); - if (!jevClient) { - outcome.error = "Jev client unavailable (no API key)"; - return outcome; - } - - try { - const questions = buildJevQuestions(targets); - - // CONCURRENCY from the skill: one ownership layer owns retries — the SDK - // gets retry: { maxRetries: 0 } and the pipeline's timeout/abort layer is - // the only retry. The call is wrapped in withLlmConcurrency so Jev down - // can't flood the router. - const { withLlmConcurrency } = await import("./llmClient.js"); - const systemOneResult = await withLlmConcurrency(async () => { - return await jevClient.systemOne( - { - model: config.AI_LLM_JEV_MODEL, - state: buildJevState(targets, ctx, correctedExamples), - questions, - }, - { signal, timeout: config.AI_LLM_JEV_TIMEOUT_MS }, - ); - }); - - outcome.raw = systemOneResult; - const answers = systemOneResult.answers as unknown as JevAnswers; - const minConfidence = config.AI_LLM_JEV_MIN_CONFIDENCE; - - for (const t of targets) { - if (isJevAccepted(answers, t.id, minConfidence)) { - outcome.results.push(mapJevAnswersToResult(answers, t.id)); - incrementCounterBy("moderation_jev_decisions", 1, { type: "jev" }); - } else { - outcome.rejectedIds.push(t.id); - incrementCounterBy("moderation_jev_decisions", 1, { - type: "llm_fallback", - }); - log.debug( - { messageId: t.id, minConfidence }, - "Jev decision rejected — falling back to LLM", - ); - } - } - - // Usage accounting (same counters as the LLM path). - const usage = systemOneResult.usage; - if (usage?.input_tokens || usage?.output_tokens) { - if (usage.input_tokens) { - incrementCounterBy("llm_tokens_total", usage.input_tokens, { - model: config.AI_LLM_JEV_MODEL, - type: "prompt", - label: "jev-batch", - }); - } - if (usage.output_tokens) { - incrementCounterBy("llm_tokens_total", usage.output_tokens, { - model: config.AI_LLM_JEV_MODEL, - type: "completion", - label: "jev-batch", - }); - } - log.info( - { - targetCount: targets.length, - accepted: outcome.results.length, - rejected: outcome.rejectedIds.length, - model: config.AI_LLM_JEV_MODEL, - input_tokens: usage.input_tokens, - output_tokens: usage.output_tokens, - }, - "Jev systemone batch usage", - ); - } - - return outcome; - } catch (err) { - if (err instanceof Error && err.name === "APIUserAbortError") throw err; // real abort — let caller decide - const msg = err instanceof Error ? err.message : String(err); - outcome.error = msg; - log.warn( - { error: msg, targetCount: targets.length }, - "Jev systemone call failed — falling back to LLM for the whole batch", - ); - return outcome; - } -} diff --git a/services/discord-gateway/src/modules/ai-moderation/textBatchProcessor.ts b/services/discord-gateway/src/modules/ai-moderation/textBatchProcessor.ts index 0ba85fac..cbd7cc24 100644 --- a/services/discord-gateway/src/modules/ai-moderation/textBatchProcessor.ts +++ b/services/discord-gateway/src/modules/ai-moderation/textBatchProcessor.ts @@ -15,12 +15,6 @@ import type { } from "../message-capture/types.js"; import { getChannelCulture } from "./channelCultureStore.js"; import { estimateTokens } from "./conversationContext.js"; -import { - analyzeBatchWithJev, - isJevEnabled, - JEV_POLICY_VERSION, - type JevTarget, -} from "./jevAnalyzer.js"; import type { ModerationPromptContent, RetryState } from "./llmCaller.js"; import { callModerationLLM } from "./llmCaller.js"; import { analyzeSingleMediaImage } from "./mediaAnalysisClient.js"; @@ -58,14 +52,14 @@ interface UrlFetchResult { title: Map; } -/** Provider-reported token usage from a raw LLM/Jev payload (may be absent). */ +/** Provider-reported token usage from a raw LLM payload (may be absent). */ interface TokenUsage { prompt_tokens: number; completion_tokens: number; total_tokens: number; } -/** Read provider-reported token usage from either the LLM or Jev raw payload. */ +/** Read provider-reported token usage from the raw LLM payload. */ function extractUsage(raw: unknown): TokenUsage | undefined { return (raw as { usage?: TokenUsage } | null)?.usage ?? undefined; } @@ -419,7 +413,7 @@ export async function runTextOnlyBatch( results: [], raw: null, }; - // Per-sub-batch verdicts before fan-out (Jev + LLM fallback merged). + // Per-sub-batch verdicts before fan-out. let subBatchResults: AnalysisResult[] = []; try { // Output budget scales with the prompt: the JSON verdict block is @@ -440,78 +434,14 @@ export async function runTextOnlyBatch( Math.max(2048, Math.ceil(subBatchPromptEstimate * 1.5)), ); - // ── Jev-first (TypeSafe System One) with LLM fallback ──────────────── - // Jev is the PRIMARY text analyzer: ONE systemOne call per sub-batch - // (5 typed questions × N messages, evaluated in parallel by Jev). - // Verdicts that pass the acceptance gate are used directly; anything - // Jev rejects (low confidence / inconsistent) and any Jev API failure - // falls back to the existing LLM call — fail-open, never dead. - if (isJevEnabled()) { - const jevTargets: JevTarget[] = batch.map((msg) => ({ - id: msg.id, - user: resolveDisplayName(msg), - content: analysisContentOf(msg), - })); - const jevOutcome = await analyzeBatchWithJev( - jevTargets, - { - contextBlock, - webSearchBlock: buildWebSearchBlock(webSearchResults), - glossaryBlock, - channelCulture: channelCultureObj?.culture_summary, - }, - abortController.signal, - correctedExamples, - ); - - subBatchResults.push(...jevOutcome.results); - if (jevOutcome.results.length > 0) { - log.info( - { - subBatch: i + 1, - accepted: jevOutcome.results.length, - rejected: jevOutcome.rejectedIds.length, - }, - "Jev analyzed sub-batch — accepted verdicts kept, rejected go to LLM", - ); - } - - // Which targets still need the LLM? - const coveredIds = new Set(subBatchResults.map((r) => r.messageId)); - const llmTargets = batch.filter((m) => !coveredIds.has(m.id)); - - if (llmTargets.length > 0) { - const llmResult = await callModerationLLM( - (state) => buildContent(state, llmTargets), - llmTargets.map((m) => m.id), - `text-batch-${i + 1}-jev-fallback`, - abortController.signal, - dynamicMaxTokens, - ); - subBatchResults.push(...llmResult.results); - batchResult = llmResult; - logModerationAnalysis( - llmTargets.map((m) => m.id), - config.AI_LLM_MODEL, - llmResult.results, - 0, - extractUsage(llmResult.raw), - ); - } else { - // Jev accepted everything — no LLM usage to attribute. - batchResult = { results: subBatchResults, raw: null }; - } - } else { - // Jev disabled / unconfigured — pure LLM path (unchanged). - batchResult = await callModerationLLM( - buildContent, - targetIds, - `text-batch-${i + 1}`, - abortController.signal, - dynamicMaxTokens, - ); - subBatchResults = batchResult.results; - } + batchResult = await callModerationLLM( + buildContent, + targetIds, + `text-batch-${i + 1}`, + abortController.signal, + dynamicMaxTokens, + ); + subBatchResults = batchResult.results; } catch (err: unknown) { const isAbort = (err instanceof Error && err.name === "AbortError") || @@ -530,8 +460,7 @@ export async function runTextOnlyBatch( clearTimeout(timeoutId); } - // Fan-out results for deduplicated messages (applies to Jev + LLM - // verdicts alike — they only consume AnalysisResult[]). + // Fan-out results for deduplicated messages. const fannedOutResults = groupMapping.size > 0 ? subBatchResults.flatMap((result) => { @@ -548,9 +477,7 @@ export async function runTextOnlyBatch( if (subBatchResults.length > 0) { logModerationAnalysis( targetIds, - subBatchResults.every((r) => r.policyVersion === JEV_POLICY_VERSION) - ? config.AI_LLM_JEV_MODEL - : config.AI_LLM_MODEL, + config.AI_LLM_MODEL, subBatchResults, 0, extractUsage(batchResult.raw), diff --git a/services/discord-gateway/src/shared/config/index.ts b/services/discord-gateway/src/shared/config/index.ts index 2bf32401..9fb59d59 100644 --- a/services/discord-gateway/src/shared/config/index.ts +++ b/services/discord-gateway/src/shared/config/index.ts @@ -158,23 +158,6 @@ export const configSchema = z // (AI_LLM_BASE_URL) but a different model alias. The dedicated NVIDIA // multimodal endpoint was removed. AI_LLM_VISION_MODEL: z.string().default("multimodal"), - // ── Jev (TypeSafe System One) — primary text analyzer ──────────────── - // Jev evaluates typed questions against a state and returns calibrated - // structured answers (no text generation). Runs on 9router's - // /v1/systemone (free model oc/jev-1.13-free). The existing LLM remains - // the fallback for anything Jev cannot decide confidently and for media. - AI_LLM_JEV_ENABLED: z - .string() - .optional() - .default("false") - .transform((v) => v === "true"), - AI_LLM_JEV_API_KEY: z.string().optional().default(""), - AI_LLM_JEV_BASE_URL: z.string().url().default("http://127.0.0.1:4014"), - AI_LLM_JEV_MODEL: z.string().default("oc/jev-1.13-free"), - // Per-message acceptance threshold on the status choice confidence. - // Below this the message falls back to the LLM (fail-open). - AI_LLM_JEV_MIN_CONFIDENCE: z.coerce.number().min(0).max(1).default(0.9), - AI_LLM_JEV_TIMEOUT_MS: z.coerce.number().int().positive().default(45_000), AI_LLM_DISABLE_THINKING: z .string() .default("true") diff --git a/services/discord-gateway/tests/jev-live-smoke.test.ts b/services/discord-gateway/tests/jev-live-smoke.test.ts deleted file mode 100644 index 04e902ee..00000000 --- a/services/discord-gateway/tests/jev-live-smoke.test.ts +++ /dev/null @@ -1,94 +0,0 @@ -/** - * jev-live-smoke.test.ts — LIVE smoke test for the Jev analyzer (real 9router). - * - * Gated behind `AI_LLM_JEV_SMOKE=1` (and the Jev env vars) so plain - * `pnpm test` / CI skips it. Verifies the ACTUAL question set + state - * builder + acceptance gate + mapper end-to-end against the real - * `analyzeBatchWithJev` module (no gateway restart needed). - * - * Run: - * AI_LLM_JEV_SMOKE=1 AI_LLM_JEV_API_KEY=sk-... npx vitest run tests/jev-live-smoke.test.ts - */ -import { describe, expect, it } from "vitest"; -import { analyzeBatchWithJev } from "../src/modules/ai-moderation/jevAnalyzer.js"; -import type { AnalysisResult } from "../src/modules/message-capture/types.js"; - -const SMOKE = process.env.AI_LLM_JEV_SMOKE === "1"; -const SKIP_REASON = - "AI_LLM_JEV_SMOKE != 1 — set it + AI_LLM_JEV_API_KEY to run the live smoke against 9router"; - -describe("Jev live smoke (real 9router /v1/systemone)", () => { - it.runIf(SMOKE)( - "analyzes 4 mixed messages with the production question set", - async () => { - const targets = [ - { - id: "m_vulgar", - user: "alerte", - content: "kontol lu semua pada goblok, dasar tolol", - }, - { - id: "m_clean", - user: "devganteng", - content: "gimana hasil compile nya bang? error terus nih", - }, - { - id: "m_sara", - user: "anon", - content: "agama lu palestina itu sampah, mending dibom habis", - }, - { - id: "m_help", - user: "helper", - content: - "nggak papa bang, nanti gw bantu debug, coba pnpm install dulu", - }, - ]; - - const outcome = await analyzeBatchWithJev( - targets, - { - contextBlock: - "\n" + - "\n[conversation_flow] status=sparse context_msgs=3 dropped=0\n" + - "[context] id='c1' time='2026-09-23T08:00:00Z' user='alice': lagi pada error compile nih\n" + - "", - webSearchBlock: "", - glossaryBlock: "", - channelCulture: "channel santai developer coding", - }, - undefined, - "", - ); - - expect(outcome.error).toBeNull(); - expect(outcome.rejectedIds).toHaveLength(0); - - const byId = Object.fromEntries( - outcome.results.map((r) => [r.messageId, r]), - ); - // Vulgar attack → flagged, vulgar_language - expect(byId.m_vulgar?.status).toBe("flagged"); - expect(byId.m_vulgar?.categories).toContain("vulgar_language"); - // SARA religious slur → flagged (zero-tolerance rule) - expect(byId.m_sara?.status).toBe("flagged"); - // Clean technical messages → clean - expect(byId.m_clean?.status).toBe("clean"); - expect(byId.m_help?.status).toBe("clean"); - - // Verdicts are calibration-honest (typed by the real pipeline shape) - const verdicts = outcome.results as AnalysisResult[]; - for (const r of verdicts) { - expect(r.confidence).toBeGreaterThanOrEqual(0.9); - if (r.status === "clean") expect(r.score).toBe(0); - expect(r.analysis).toContain("[Jev]"); - expect(r.analysis).toContain("p_melanggar="); - } - }, - SMOKE ? 60_000 : 0, - ); - - it.skipIf(!SMOKE)("skipped when AI_LLM_JEV_SMOKE is unset", () => { - expect(SKIP_REASON).toBeTruthy(); - }); -}); diff --git a/services/discord-gateway/tests/jevAnalyzer.test.ts b/services/discord-gateway/tests/jevAnalyzer.test.ts deleted file mode 100644 index dd307e91..00000000 --- a/services/discord-gateway/tests/jevAnalyzer.test.ts +++ /dev/null @@ -1,382 +0,0 @@ -/** - * jevAnalyzer.test.ts — pure unit tests for the Jev analyzer (no network). - * - * Tests the question builder, state builder, acceptance gate, and answer - * mapper against hand-crafted `JevAnswers` objects (the shape `systemOne` - * returns). `analyzeBatchWithJev`'s network path is exercised separately - * by the live smoke harness (no gateway restart). - */ -import { describe, expect, it } from "vitest"; -import { - buildJevQuestions, - buildJevState, - isJevAccepted, - JEV_CATEGORIES, - JEV_POLICY_VERSION, - type JevAnswers, - mapJevAnswersToResult, -} from "../src/modules/ai-moderation/jevAnalyzer.js"; - -const MID = "m1"; - -const cleanAnswers: JevAnswers = { - [`${MID}__v`]: { type: "noul", noul: 0.03 }, - [`${MID}__status`]: { type: "choice", choice: "clean", confidence: 0.99 }, - [`${MID}__severity`]: { type: "choice", choice: "none" }, - [`${MID}__category`]: { type: "choice", choice: "none" }, - [`${MID}__action`]: { type: "choice", choice: "none" }, -}; - -const flaggedAnswers: JevAnswers = { - ...cleanAnswers, - [`${MID}__v`]: { type: "noul", noul: 0.98 }, - [`${MID}__status`]: { type: "choice", choice: "flagged", confidence: 0.99 }, - [`${MID}__severity`]: { type: "choice", choice: "high" }, - [`${MID}__category`]: { type: "choice", choice: "vulgar_language" }, - [`${MID}__action`]: { type: "choice", choice: "delete" }, -}; - -function answersFor(overrides: Partial): JevAnswers { - return { ...cleanAnswers, ...overrides }; -} - -describe("buildJevQuestions", () => { - it("emits 5 id-keyed questions per target", () => { - const q = buildJevQuestions([ - { id: "a", user: "alice", content: "hi" }, - { id: "b", user: "bob", content: "hey" }, - ]); - const keys = Object.keys(q); - expect(keys).toHaveLength(10); - expect(keys.sort()).toEqual( - [ - "a__v", - "a__status", - "a__severity", - "a__category", - "a__action", - "b__v", - "b__status", - "b__severity", - "b__category", - "b__action", - ].sort(), - ); - for (const k of keys) { - if (k.endsWith("__v")) expect(q[k].type).toBe("noul"); - else expect(q[k].type).toBe("choice"); - } - // status question names the message id (not an index) - expect( - (q["a__status"] as { instructions?: string }).instructions, - ).toContain("a"); - }); -}); - -describe("buildJevState", () => { - it("includes the distilled policy + messages + stripped context facts", () => { - const state = buildJevState( - [ - { id: "m1", user: "alice", content: "kontol" }, - { id: "m2", user: "bob", content: "halo bro" }, - ], - { - contextBlock: - "\nLokasi: channel umum ramai.", - webSearchBlock: "\nHasil: tidak ada.", - glossaryBlock: "\nIstilah: none.", - channelCulture: "channel santai", - }, - "## contoh koreksi", - ); - expect(state).toContain("KEBIJAKAN SERVER"); - expect(state).toContain('- Pesan "m1" dari "alice": "kontol"'); - expect(state).toContain("KULTUR CHANNEL"); - expect(state).toContain("KONTEKS:"); - expect(state).toContain("HASIL PENCARIAN WEB:"); - expect(state).toContain("GLOSARIUM:"); - expect(state).toContain("KOREKSI SEBELUMNYA"); - // NO chat-scaffolding artifacts (this is the declarative contract) - expect(state).not.toContain(""); - expect(state).not.toContain(" { - it("accepts a clean verdict with high confidence and noul<0.5", () => { - expect(isJevAccepted(cleanAnswers, MID, 0.9)).toBe(true); - }); - - it("accepts a flagged verdict with noul>=0.5", () => { - expect(isJevAccepted(flaggedAnswers, MID, 0.9)).toBe(true); - }); - - it("rejects low-confidence status", () => { - expect( - isJevAccepted( - answersFor({ - [`${MID}__status`]: { - type: "choice", - choice: "clean", - confidence: 0.5, - }, - }), - MID, - 0.9, - ), - ).toBe(false); - }); - - it("rejects contradictory clean-with-high-noul", () => { - expect( - isJevAccepted( - answersFor({ [`${MID}__v`]: { type: "noul", noul: 0.95 } }), - MID, - 0.9, - ), - ).toBe(false); - }); - - it("rejects flagged-with-low-noul", () => { - expect( - isJevAccepted( - answersFor({ - [`${MID}__v`]: { type: "noul", noul: 0.1 }, - [`${MID}__status`]: { - type: "choice", - choice: "flagged", - confidence: 0.99, - }, - }), - MID, - 0.9, - ), - ).toBe(false); - }); - - it("rejects clean with non-none severity/category/action", () => { - expect( - isJevAccepted( - answersFor({ - [`${MID}__severity`]: { type: "choice", choice: "low" }, - }), - MID, - 0.9, - ), - ).toBe(false); - expect( - isJevAccepted( - answersFor({ - [`${MID}__category`]: { type: "choice", choice: "spam" }, - }), - MID, - 0.9, - ), - ).toBe(false); - expect( - isJevAccepted( - answersFor({ - [`${MID}__action`]: { type: "choice", choice: "monitor" }, - }), - MID, - 0.9, - ), - ).toBe(false); - }); - - it("rejects flagged with none severity/category/action", () => { - expect( - isJevAccepted( - answersFor({ - [`${MID}__v`]: { type: "noul", noul: 0.9 }, - [`${MID}__status`]: { - type: "choice", - choice: "flagged", - confidence: 0.99, - }, - [`${MID}__severity`]: { type: "choice", choice: "none" }, - }), - MID, - 0.9, - ), - ).toBe(false); - expect( - isJevAccepted( - answersFor({ - [`${MID}__v`]: { type: "noul", noul: 0.9 }, - [`${MID}__status`]: { - type: "choice", - choice: "flagged", - confidence: 0.99, - }, - [`${MID}__category`]: { type: "choice", choice: "none" }, - }), - MID, - 0.9, - ), - ).toBe(false); - expect( - isJevAccepted( - answersFor({ - [`${MID}__v`]: { type: "noul", noul: 0.9 }, - [`${MID}__status`]: { - type: "choice", - choice: "flagged", - confidence: 0.99, - }, - [`${MID}__action`]: { type: "choice", choice: "none" }, - }), - MID, - 0.9, - ), - ).toBe(false); - }); - - it("rejects warn with delete/escalate action", () => { - expect( - isJevAccepted( - answersFor({ - [`${MID}__v`]: { type: "noul", noul: 0.6 }, - [`${MID}__status`]: { - type: "choice", - choice: "warn", - confidence: 0.92, - }, - [`${MID}__severity`]: { type: "choice", choice: "low" }, - [`${MID}__category`]: { type: "choice", choice: "spam" }, - [`${MID}__action`]: { type: "choice", choice: "delete" }, - }), - MID, - 0.9, - ), - ).toBe(false); - }); - - it("rejects unknown status/severity/category/action labels", () => { - expect( - isJevAccepted( - answersFor({ - [`${MID}__status`]: { - type: "choice", - choice: "banned", - confidence: 0.99, - }, - }), - MID, - 0.9, - ), - ).toBe(false); - expect( - isJevAccepted( - answersFor({ - [`${MID}__severity`]: { type: "choice", choice: "severe" }, - }), - MID, - 0.9, - ), - ).toBe(false); - expect( - isJevAccepted( - answersFor({ - [`${MID}__category`]: { type: "choice", choice: "doxxing" }, - }), - MID, - 0.9, - ), - ).toBe(false); - expect( - isJevAccepted( - answersFor({ - [`${MID}__action`]: { type: "choice", choice: "destroy" }, - }), - MID, - 0.9, - ), - ).toBe(false); - }); - - it("rejects missing questions", () => { - const { [`${MID}__action`]: _drop, ...partial } = answersFor({}); - expect(isJevAccepted(partial as JevAnswers, MID, 0.9)).toBe(false); - }); -}); - -describe("mapJevAnswersToResult", () => { - it("maps a clean answer to a zero-score analysis result", () => { - const r = mapJevAnswersToResult(cleanAnswers, MID); - expect(r).toMatchObject({ - messageId: MID, - status: "clean", - flags: [], - score: 0, - categories: [], - severity: "none", - recommendedAction: "none", - policyVersion: JEV_POLICY_VERSION, - confidence: 0.99, - }); - expect(r.analysis).toContain("status=clean"); - expect(r.analysis).toContain("[Jev]"); - }); - - it("maps a flagged answer with calibrated score + category flag", () => { - const r = mapJevAnswersToResult(flaggedAnswers, MID); - expect(r.status).toBe("flagged"); - expect(r.flags).toEqual(["vulgar_language"]); - expect(r.categories).toEqual(["vulgar_language"]); - expect(r.severity).toBe("high"); - expect(r.recommendedAction).toBe("delete"); - expect(r.score).toBeGreaterThanOrEqual(0.9); // clamped to >=0.7, noul 0.98 - expect(r.analysis).toContain("p_melanggar=0.98"); - expect(r.evidence).toEqual([]); - }); - - it("warns become score 0.45 with no flags", () => { - const r = mapJevAnswersToResult( - answersFor({ - [`${MID}__status`]: { - type: "choice", - choice: "warn", - confidence: 0.92, - }, - [`${MID}__severity`]: { type: "choice", choice: "low" }, - [`${MID}__category`]: { - type: "choice", - choice: "conflict_instigation", - }, - [`${MID}__action`]: { type: "choice", choice: "warn" }, - [`${MID}__v`]: { type: "noul", noul: 0.6 }, - }), - MID, - ); - expect(r.status).toBe("warn"); - expect(r.score).toBe(0.45); - expect(r.flags).toEqual(["conflict_instigation"]); - expect(r.recommendedAction).toBe("warn"); - }); - - it("every category label in the vocab is accepted (no unknown rejection)", () => { - // A self-consistent FLAGGED base (flagged needs non-none severity/action). - const flaggedBase = { - [`${MID}__v`]: { type: "noul", noul: 0.9 }, - [`${MID}__status`]: { - type: "choice", - choice: "flagged", - confidence: 0.99, - }, - [`${MID}__severity`]: { type: "choice", choice: "medium" }, - [`${MID}__category`]: { type: "choice", choice: "none" }, - [`${MID}__action`]: { type: "choice", choice: "monitor" }, - } satisfies JevAnswers; - for (const c of JEV_CATEGORIES) { - // "none" is only valid with clean status (flagged+none is contradictory - // and MUST be rejected — covered by the cross-consistency tests above). - if (c === "none") continue; - const answers = { - ...flaggedBase, - [`${MID}__category`]: { type: "choice", choice: c }, - } satisfies JevAnswers; - expect(isJevAccepted(answers, MID, 0.9)).toBe(true); - } - }); -});