chore: auto-commit task -

services/discord-gateway/drizzle/migrations/0002_add_sticker_cache.sql

@@ -0,0 +1,12 @@
+-- 0002_add_sticker_cache.sql
+-- Creates the sticker_cache table (defined in schema.ts but missing from migrations)
+
+CREATE TABLE IF NOT EXISTS "sticker_cache" (
+    "name" text PRIMARY KEY NOT NULL,
+    "base64" text NOT NULL,
+    "mime_type" text NOT NULL,
+    "size" integer NOT NULL,
+    "fetched_at" bigint NOT NULL
+);
+
+CREATE INDEX IF NOT EXISTS "idx_sticker_cache_fetched_at" ON "sticker_cache" ("fetched_at");

services/discord-gateway/src/modules/ai-moderation/indonesianTextNormalizer.ts

@@ -1,5 +1,6 @@
 import axios from "axios";
 import OpenAI from "openai";
+import { AbortError } from "p-retry";
 import { config } from "../../shared/config/config.js";
 import { createChildLogger } from "../../shared/logger/logger.js";
 import { retryWithBackoff } from "../../shared/utils/retry.js";
@@ -74,9 +75,12 @@ const BADWORD_CACHE_TTL_MS = 10 * 60 * 1000;
  */
 const DB_CACHE_TTL_MS = 24 * 60 * 60 * 1000;
 
-const NEMOTRON_RATE_LIMIT_COOLDOWN_MS = 0;
+const NEMOTRON_RATE_LIMIT_COOLDOWN_MS = 60_000; // 1 min backoff on 429
-const PRIMARY_AI_RATE_LIMIT_COOLDOWN_MS = 0;
+const PRIMARY_AI_RATE_LIMIT_COOLDOWN_MS = 60_000; // 1 min backoff on 429
-const GROQ_RATE_LIMIT_COOLDOWN_MS = 0;
+const GROQ_RATE_LIMIT_COOLDOWN_MS = 60_000; // 1 min backoff on 429
+
+/** How long to mark a provider unavailable after a transient (5xx/timeout) error. */
+const TRANSIENT_ERROR_COOLDOWN_MS = 30_000; // 30s backoff on 502/timeout
 
 interface BadwordCacheEntry {
   value: string[];
@@ -275,9 +279,9 @@ async function callPrimaryAiModeration(text: string): Promise<string[]> {
       } as OpenAI.Chat.Completions.ChatCompletionCreateParamsNonStreaming);
     },
     {
-      retries: 0,
+      retries: 2,
-      minTimeout: 0,
+      minTimeout: 2000,
-      maxTimeout: 0,
+      maxTimeout: 5000,
       factor: 2,
       logger: log,
     },
@@ -306,19 +310,36 @@ async function callGrokModeration(text: string): Promise<string[]> {
     return [];
   }
 
-  const response = await axios.post(
+  const response = await retryWithBackoff(
-    config.GROQ_MODERATION_BASE_URL,
+    async () => {
-    {
+      const res = await axios.post(
-      model: config.GROQ_MODERATION_MODEL,
+        config.GROQ_MODERATION_BASE_URL,
-      messages: [{ role: "user", content: text }],
+        {
+          model: config.GROQ_MODERATION_MODEL,
+          messages: [{ role: "user", content: text }],
+        },
+        {
+          headers: {
+            Authorization: `Bearer ${apiKey}`,
+            Accept: "application/json",
+            "Content-Type": "application/json",
+          },
+          timeout: 15_000,
+          validateStatus: (status: number) => status < 500,
+        },
+      );
+      // 429 should abort retry immediately — no point hammering a rate limit
+      if (res.status === 429) {
+        throw new AbortError("Groq rate limited");
+      }
+      return res;
     },
     {
-      headers: {
+      retries: 2,
-        Authorization: `Bearer ${apiKey}`,
+      minTimeout: 2000,
-        Accept: "application/json",
+      maxTimeout: 5000,
-        "Content-Type": "application/json",
+      factor: 2,
-      },
+      logger: log,
-      timeout: 10_000,
     },
   );
 
@@ -360,22 +381,39 @@ async function callNemotronContentSafety(text: string): Promise<string[]> {
     return [];
   }
 
-  const response = await axios.post(
+  const response = await retryWithBackoff(
-    config.NVIDIA_NEMOTRON_BASE_URL,
+    async () => {
-    {
+      const res = await axios.post(
-      model: config.NVIDIA_NEMOTRON_MODEL,
+        config.NVIDIA_NEMOTRON_BASE_URL,
-      messages: [{ role: "user", content: text }],
+        {
-      max_tokens: 897,
+          model: config.NVIDIA_NEMOTRON_MODEL,
-      temperature: 0.2,
+          messages: [{ role: "user", content: text }],
-      top_p: 0.7,
+          max_tokens: 897,
-      stream: false,
+          temperature: 0.2,
+          top_p: 0.7,
+          stream: false,
+        },
+        {
+          headers: {
+            Authorization: `Bearer ${apiKey}`,
+            Accept: "application/json",
+          },
+          timeout: 15_000,
+          validateStatus: (status: number) => status < 500,
+        },
+      );
+      // 429 should abort retry immediately — no point hammering a rate limit
+      if (res.status === 429) {
+        throw new AbortError("NVIDIA rate limited");
+      }
+      return res;
     },
     {
-      headers: {
+      retries: 2,
-        Authorization: `Bearer ${apiKey}`,
+      minTimeout: 2000,
-        Accept: "application/json",
+      maxTimeout: 5000,
-      },
+      factor: 2,
-      timeout: 15_000,
+      logger: log,
     },
   );
 
@@ -478,6 +516,9 @@ export async function detectIndonesianBadwords(
         if (status === 429) {
           nemotronUnavailableUntil =
             Date.now() + NEMOTRON_RATE_LIMIT_COOLDOWN_MS;
+        } else {
+          // 502, timeout, or other transient error — cooldown briefly
+          nemotronUnavailableUntil = Date.now() + TRANSIENT_ERROR_COOLDOWN_MS;
         }
         log.warn(
           { error },
@@ -501,6 +542,9 @@ export async function detectIndonesianBadwords(
         if (status === 429) {
           primaryAiUnavailableUntil =
             Date.now() + PRIMARY_AI_RATE_LIMIT_COOLDOWN_MS;
+        } else {
+          // 502, timeout, or other transient error — cooldown briefly
+          primaryAiUnavailableUntil = Date.now() + TRANSIENT_ERROR_COOLDOWN_MS;
         }
         log.warn(
           { error },
@@ -525,6 +569,9 @@ export async function detectIndonesianBadwords(
             : null;
           if (status === 429) {
             groqUnavailableUntil = Date.now() + GROQ_RATE_LIMIT_COOLDOWN_MS;
+          } else {
+            // 502, timeout, or other transient error — cooldown briefly
+            groqUnavailableUntil = Date.now() + TRANSIENT_ERROR_COOLDOWN_MS;
           }
           log.warn({ error }, "Groq Llama Prompt Guard moderation failed");
         }

services/discord-gateway/src/modules/ai-moderation/llmModerationClient.ts

@@ -750,6 +750,7 @@ async function callModerationLLM(
             throw parseError;
           }
         } catch (apiError: any) {
+          // 429/401/403 → abort immediately, never retry
           if (
             apiError?.status === 429 ||
             apiError?.status === 401 ||
@@ -757,63 +758,124 @@ async function callModerationLLM(
           ) {
             throw new AbortError(apiError);
           }
+          // 5xx server errors → retryable transient errors
+          // p-retry will retry these; on final exhaustion the outer catch
+          // will produce synthetic error results for all targets
+          if (
+            apiError?.status >= 500 ||
+            apiError?.code === "ECONNRESET" ||
+            apiError?.code === "ETIMEDOUT" ||
+            apiError?.name === "APIError"
+          ) {
+            // re-throw as-is so p-retry can retry
+            throw apiError;
+          }
           throw apiError;
         }
       },
       {
-        retries: 0,
+        retries: 2,
+        minTimeout: 3000,
+        maxTimeout: 8000,
+        factor: 2,
         logger: log,
       },
     );
     parsed = analysis.parsed;
     result = analysis.result;
-  } catch (parseError) {
+  } catch (err) {
-    if (!state.lastInvalidContent) {
+    const errorMsg = err instanceof Error ? err.message : String(err);
-      throw parseError;
+    const isApiError = !state.lastInvalidContent;
-    }
 
-    const errorMsg =
+    // For API errors (502, timeout, etc.) where retries exhausted, produce
-      parseError instanceof Error ? parseError.message : String(parseError);
+    // synthetic error results so the batch doesn't crash entirely.
+    // For parse errors, we already have lastInvalidContent and the existing
+    // fallback path below handles it.
+    const apiErrorCode = isApiError
+      ? `MOD_${Date.now().toString(36).slice(0, 6)}`
+      : null;
 
-    log.error(
+    if (isApiError) {
-      {
+      log.warn(
-        error: errorMsg,
+        {
-        contentLength: state.lastInvalidContent.length,
+          error: errorMsg,
-        contentPreview: state.lastInvalidContent.substring(0, 500),
+          targetIds,
+          model: config.AI_LLM_MODEL,
+          label,
+        },
+        `LLM API error after retries exhausted (${label}) — marking all targets as analysis errors`,
+      );
+
+      logModerationError(
         targetIds,
-        model: config.AI_LLM_MODEL,
+        config.AI_LLM_MODEL,
-        timestamp: new Date().toISOString(),
+        err instanceof Error ? err : new Error(String(err)),
-      },
+        {
-      `Robust Fallback (${label}): Failed to parse moderation response. Marking all targets as analysis errors.`,
+          phase: "api_call",
-    );
+          label,
+        },
+      );
 
-    // Log error with responseLogger
+      parsed = targetIds.map((id) => ({
-    logModerationError(
+        messageId: id,
-      targetIds,
+        status: "error",
-      config.AI_LLM_MODEL,
+        flags: ["analysis_api_failed"],
-      parseError as Error | string,
+        score: 0,
-      {
+        analysis: `Analisis gagal karena error pada server AI dan memerlukan pemeriksaan manual. Error code: ${apiErrorCode}`,
-        phase: "parse_response",
+        categories: ["analysis_api_failed"],
-        label,
+        severity: "none",
-        contentLength: state.lastInvalidContent.length,
+        confidence: 0,
-      },
+        recommendedAction: "review",
-    );
+        policyVersion: "default-2026-05-30",
+        evidence: [],
+      }));
+    } else {
+      // Parse error fallback — existing path
+      const parseMsg = err instanceof Error ? err.message : String(err);
+      const contentPreview =
+        state.lastInvalidContent?.substring(0, 500) ?? "<empty>";
+      const contentLen = state.lastInvalidContent?.length ?? 0;
 
-    // Sanitized error messages — no internal details exposed (R10)
+      log.error(
-    const errorCode = `MOD_${Date.now().toString(36).slice(0, 6)}`;
+        {
-    parsed = targetIds.map((id) => ({
+          error: parseMsg,
-      messageId: id,
+          contentLength: contentLen,
-      status: "error",
+          contentPreview,
-      flags: ["analysis_parse_failed"],
+          targetIds,
-      score: 0,
+          model: config.AI_LLM_MODEL,
-      analysis: `Analisis gagal dan memerlukan pemeriksaan manual. Error code: ${errorCode}`,
+          timestamp: new Date().toISOString(),
-      categories: ["analysis_parse_failed"],
+        },
-      severity: "none",
+        `Robust Fallback (${label}): Failed to parse moderation response. Marking all targets as analysis errors.`,
-      confidence: 0,
+      );
-      recommendedAction: "review",
+
-      policyVersion: "default-2026-05-30",
+      // Log error with responseLogger
-      evidence: [],
+      logModerationError(
-    }));
+        targetIds,
+        config.AI_LLM_MODEL,
+        err instanceof Error ? err : new Error(String(err)),
+        {
+          phase: "parse_response",
+          label,
+          contentLength: contentLen,
+        },
+      );
+
+      // Sanitized error messages — no internal details exposed (R10)
+      const errorCode = `MOD_${Date.now().toString(36).slice(0, 6)}`;
+      parsed = targetIds.map((id) => ({
+        messageId: id,
+        status: "error",
+        flags: ["analysis_parse_failed"],
+        score: 0,
+        analysis: `Analisis gagal dan memerlukan pemeriksaan manual. Error code: ${errorCode}`,
+        categories: ["analysis_parse_failed"],
+        severity: "none",
+        confidence: 0,
+        recommendedAction: "review",
+        policyVersion: "default-2026-05-30",
+        evidence: [],
+      }));
+    }
   }
 
   return { results: parsed, raw: result };

services/discord-gateway/src/shared/config/config.ts

@@ -131,28 +131,8 @@ const configSchema = z
       .int()
       .positive()
       .default(50),
-    /** NVIDIA Nemotron-3 Content Safety API key for badword detection. */
+    // AI moderation uses the Primary LLM (AI_LLM_*) endpoint only.
-    NVIDIA_NEMOTRON_API_KEY: z.string().optional(),
+    // No NVIDIA or Groq fallback.
-    /** NVIDIA Nemotron model identifier. */
-    NVIDIA_NEMOTRON_MODEL: z
-      .string()
-      .default("nvidia/nemotron-3-content-safety"),
-    /** NVIDIA Nemotron API base URL. */
-    NVIDIA_NEMOTRON_BASE_URL: z
-      .string()
-      .url()
-      .default("https://integrate.api.nvidia.com/v1/chat/completions"),
-    /** Groq API key for Llama Prompt Guard moderation fallback. */
-    GROQ_API_KEY: z.string().optional(),
-    /** Groq moderation model identifier. */
-    GROQ_MODERATION_MODEL: z
-      .string()
-      .default("meta-llama/llama-prompt-guard-2-86m"),
-    /** Groq API base URL. */
-    GROQ_MODERATION_BASE_URL: z
-      .string()
-      .url()
-      .default("https://api.groq.com/openai/v1/chat/completions"),
     AUTO_DELETE_FLAGGED_ENABLED: z
       .string()
       .optional()

src/config.ts

@@ -131,28 +131,8 @@ const configSchema = z
       .int()
       .positive()
       .default(50),
-    /** NVIDIA Nemotron-3 Content Safety API key for badword detection. */
+    // AI moderation uses the Primary LLM (AI_LLM_*) endpoint only.
-    NVIDIA_NEMOTRON_API_KEY: z.string().optional(),
+    // No NVIDIA or Groq fallback.
-    /** NVIDIA Nemotron model identifier. */
-    NVIDIA_NEMOTRON_MODEL: z
-      .string()
-      .default("nvidia/nemotron-3-content-safety"),
-    /** NVIDIA Nemotron API base URL. */
-    NVIDIA_NEMOTRON_BASE_URL: z
-      .string()
-      .url()
-      .default("https://integrate.api.nvidia.com/v1/chat/completions"),
-    /** Groq API key for Llama Prompt Guard moderation fallback. */
-    GROQ_API_KEY: z.string().optional(),
-    /** Groq moderation model identifier. */
-    GROQ_MODERATION_MODEL: z
-      .string()
-      .default("meta-llama/llama-prompt-guard-2-86m"),
-    /** Groq API base URL. */
-    GROQ_MODERATION_BASE_URL: z
-      .string()
-      .url()
-      .default("https://api.groq.com/openai/v1/chat/completions"),
     AUTO_DELETE_FLAGGED_ENABLED: z
       .string()
       .optional()