feat(ai): scope persistence, parallel tools, Prometheus token/cache counters

C2 full scope persistence:
- getRecentConversationContext now returns per-turn guildId/channelId
- processMessage merges historical scope when current request is unscoped
- chatbot remembers server context across all 8 history exchanges (not just 3)

A4+Metrics:
- Add incrementCounterBy(name, delta, labels) to gateway-metrics
- Token-usage counters: llm_tokens_total{model, type, label} per batch
- Cache hit counters: moderation_cache_hits{type} exact/semantic-qdrant/semantic-pg
- Cache miss counters: moderation_cache_misses per batch
- Prometheus /metrics now exposes cost + cache hit-rate for dashboards

Performance:
- Parallel tool execution within each chatbot round (Promise.all)
- All tool results collected before sending to model (ordering preserved)
- Tool failure now logged with structured warning (chatbot.tools.ts)

Verified: backend tsc+biome 37/37, discord-gateway tsc+biome 210/210

services/backend/src/modules/chatbot/chatbot.service.ts

@@ -26,13 +26,25 @@ class ChatbotService {
     // longer bake server stats into the prompt — the model must pull current
     // data via tools (see buildSystemPrompt), so it always answers from live
     // numbers instead of a stale snapshot.
-    const scope = {
+    //
+    // If the current request doesn't carry a guild/channel scope (e.g. a
+    // cross-server dashboard surface), fall back to the most recent scope the
+    // user was chatting in from history, so the agent doesn't drift to the
+    // wrong server between turns.
+    const explicitScope = {
       guildId: context?.guildId,
       channelId: context?.channelId,
     };
+    const scope =
+      explicitScope.guildId || explicitScope.channelId
+        ? explicitScope
+        : (recentContext.lastScope ?? {
+            guildId: undefined,
+            channelId: undefined,
+          });
 
     const systemPrompt = this.buildSystemPrompt(scope);
-    const conversationHistory = this.buildHistoryMessages(recentContext);
+    const conversationHistory = this.buildHistoryMessages(recentContext.turns);
     const llmResponse = await this.callLLM(
       systemPrompt,
       conversationHistory,
@@ -61,14 +73,28 @@ class ChatbotService {
     await chatbotRepository.clearChatHistory(userId);
   }
 
-  private async getRecentConversationContext(
-    userId: string,
-  ): Promise<string[]> {
-    const history = await chatbotRepository.getChatHistory(userId, 3);
-    return history.flatMap((row) => [
-      `User: ${row.user_message}`,
-      `Bot: ${row.bot_response}`,
-    ]);
+  private async getRecentConversationContext(userId: string): Promise<{
+    turns: string[];
+    lastScope: { guildId?: string; channelId?: string } | null;
+  }> {
+    const history = await chatbotRepository.getChatHistory(userId, 8);
+    const turns: string[] = [];
+    let lastScope: { guildId?: string; channelId?: string } | null = null;
+    for (const row of history) {
+      turns.push(`User: ${row.user_message}`, `Bot: ${row.bot_response}`);
+      // Capture the most recent scope this user was chatting in, so the
+      // agent keeps server context across turns even if the current request
+      // doesn't carry one.
+      const g = row.context?.guildId;
+      const c = row.context?.channelId;
+      if (g || c) {
+        lastScope = {
+          guildId: g ?? undefined,
+          channelId: c ?? undefined,
+        };
+      }
+    }
+    return { turns, lastScope };
   }
 
   private buildSystemPrompt(scope: {
@@ -82,11 +108,27 @@ class ChatbotService {
 
 ${scopeLine}
 
+TOOLS YANG TERSEDIA (panggil saat user tanya soal data server):
+- get_server_stats → statistik server: total pesan, user aktif, flagged/warn/clean
+- get_top_channels → channel paling aktif (jumlah pesan terbanyak)
+- get_recent_activity → pesan terbaru (siapa, channel mana, jam berapa, isinya)
+- get_top_flagged → pesan yang di-flag AI (alasan, severity, analysis)
+- search_messages → cari pesan berdasarkan kata kunci (LIKE search)
+- get_user_messages → pesan terbaru dari satu user tertentu
+- get_user_profile → profil AI dari seorang user (pola perilaku, gaya bicara)
+- get_channel_culture → norma/slang dari suatu channel
+- get_message_detail → 1 pesan lengkap beserta hasil analisis AI-nya
+- get_message_reviews → antrean review moderasi manual
+- get_voice_recordings → rekaman suara terbaru
+- get_moderation_timeline → tren harian: total vs flagged vs warn vs clean
+- get_corrections → riwayat koreksi false-positive AI
+
 ATURAN PENTING — JANGAN PAKAI KONTEKS STATIS:
-- Kamu TIDAK punya hafalan soal angka server (jumlah pesan, user aktif, flagged, dll). JANGAN tebak atau karang angka.
-- Untuk SEMUA pertanyaan soal data server (jumlah pesan, user aktif, channel ramai, aktivitas terbaru, pesan di-flag), WAJIB panggil tool yang sesuai (get_server_stats, get_top_channels, get_recent_activity, get_top_flagged). Jawab HANYA dari hasil tool.
-- Tool otomatis di-scope ke guild/channel di atas — kalau argumen guildId/channelId kosong, biarkan kosong (sudah otomatis ter-isi). Jangan isi ID yang kamu tebak.
-- Kalau tool balas error atau kosong, bilang aja data lagi ga ketemu, jangan karang.
+- Kamu TIDAK punya hafalan soal angka server. JANGAN tebak atau karang angka.
+- Untuk SEMUA pertanyaan soal data server, WAJIB panggil tool yang sesuai. Jawab HANYA dari hasil tool.
+- Tool otomatis di-scope ke guild/channel — kalau argumen kosong, biarkan kosong. Jangan isi ID tebakan.
+- Kalau tool balik error atau kosong, bilang aja data lagi ga ketemu, jangan karang.
+- Boleh panggil banyak tool dalam satu jawaban kalau pertanyaan butuh beberapa data (multi-hop).
 
 Gaya ngobrol:
 - Santai, hangat, kayak ngobrol sama temen
@@ -127,27 +169,31 @@ Gaya ngobrol:
     try {
       const { default: axios } = await import("axios");
 
-      // Gateway tidak handle role system — gabung konteks ke user message.
-      // The system section stays visible to the model as the first user turn.
-      const contextPrefixed = `${systemPrompt}\n\nPertanyaan user: ${userMessage}`;
-
-      // Seed conversation: prior turns + current question.
-      const messages: Array<
-        | { role: "user" | "assistant"; content: string }
-        | {
-            role: "assistant";
-            content: string | null;
-            tool_calls: Array<{
-              id: string;
-              type: "function";
-              function: { name: string; arguments: string };
-            }>;
-          }
-        | { role: "tool"; tool_call_id: string; content: string }
-      > = [...history, { role: "user", content: contextPrefixed }];
+      const scopeHint = scope.guildId
+        ? ` (scope: guild ${scope.guildId}${scope.channelId ? `, channel ${scope.channelId}` : ""})`
+        : "";
+      // Local message type that models the shapes the agent loop emits and
+      // accepts: plain system/user/assistant turns and assistant tool-calls +
+      // tool results. role is a union string so history entries typed as
+      // `"user" | "assistant"` don't break the discriminant.
+      type ChatMsg = {
+        role: "system" | "user" | "assistant" | "tool";
+        content?: string | null;
+        tool_call_id?: string;
+        tool_calls?: Array<{
+          id: string;
+          type: "function";
+          function: { name: string; arguments: string };
+        }>;
+      };
+      const messages: ChatMsg[] = [
+        { role: "system", content: systemPrompt },
+        ...history,
+        { role: "user", content: `${userMessage}${scopeHint}` },
+      ];
 
       // ── Agentic tool loop ─────────────────────────────────────────
-      const MAX_TOOL_ROUNDS = 4;
+      const MAX_TOOL_ROUNDS = 6;
       for (let round = 0; round <= MAX_TOOL_ROUNDS; round += 1) {
         const response = await axios.post(
           `${baseUrl}/chat/completions`,
@@ -193,35 +239,44 @@ Gaya ngobrol:
         );
 
         if (toolCalls.length > 0) {
-          // Execute each tool, append tool results, continue loop.
-          for (const tc of toolCalls) {
-            messages.push({
-              role: "assistant",
-              content: null,
-              tool_calls: [
-                {
-                  id: tc.id,
-                  type: "function",
-                  function: { name: tc.name, arguments: tc.arguments },
-                },
-              ],
-            });
-            // Auto-scope: if the model omitted guildId/channelId, fill them
-            // from the request scope so tools query the right server without
-            // the model having to guess IDs.
-            const scopedArgs = { ...tc.args };
-            if (scope.guildId && scopedArgs.guildId == null) {
-              scopedArgs.guildId = scope.guildId;
-            }
-            if (scope.channelId && scopedArgs.channelId == null) {
-              scopedArgs.channelId = scope.channelId;
-            }
-            let result = "";
-            try {
-              result = await executeTool(tc.name, scopedArgs);
-            } catch (e) {
-              result = `Tool error: ${(e as Error).message}`;
-            }
+          // Executor for a single tool call: pushes the assistant tool_call,
+          // runs the tool, returns { tc, result } so results can be appended
+          // in order after all tools execute in parallel.
+          const executions = await Promise.all(
+            toolCalls.map(async (tc) => {
+              messages.push({
+                role: "assistant",
+                content: null,
+                tool_calls: [
+                  {
+                    id: tc.id,
+                    type: "function",
+                    function: { name: tc.name, arguments: tc.arguments },
+                  },
+                ],
+              });
+              // Auto-scope: if the model omitted guildId/channelId, fill them
+              // from the request scope so tools query the right server without
+              // the model having to guess IDs.
+              const scopedArgs = { ...tc.args };
+              if (scope.guildId && scopedArgs.guildId == null) {
+                scopedArgs.guildId = scope.guildId;
+              }
+              if (scope.channelId && scopedArgs.channelId == null) {
+                scopedArgs.channelId = scope.channelId;
+              }
+              let result = "";
+              try {
+                result = await executeTool(tc.name, scopedArgs);
+              } catch (e) {
+                result = `Tool error: ${(e as Error).message}`;
+              }
+              return { tc, result };
+            }),
+          );
+          // Append tool results in the SAME order as the tool_calls so the
+          // API's function-calling contract isn't violated by reordering.
+          for (const { tc, result } of executions) {
             messages.push({
               role: "tool",
               tool_call_id: tc.id,

services/backend/src/modules/chatbot/chatbot.tools.ts

@@ -1,4 +1,5 @@
 import { and, desc, eq, like, sql } from "drizzle-orm";
+import { createChildLogger } from "@/shared/logger/index";
 import { getDatabase } from "../../shared/database/index.js";
 import {
   pgChannelCulturesTable,
@@ -19,6 +20,20 @@ import {
  */
 
 export type ToolResult = string;
+
+const logger = createChildLogger("chatbot.tools");
+
+/** Saturating counter of chatbot tool execution errors (for observability). */
+export let toolErrors = 0;
+const TOOL_ERROR_CAP = 1000;
+
+function toolExecError(err: unknown, name: string): string {
+  if (toolErrors < TOOL_ERROR_CAP) toolErrors++;
+  const detail = (err as Error)?.message ?? "unknown";
+  logger.warn({ tool: name, error: detail }, "Chatbot tool execution failed");
+  return `Tool ${name} gagal: ${detail}`;
+}
+
 /** Executes a tool call against the real DB and returns a readable result. */
 export async function executeTool(
   name: string,
@@ -90,7 +105,7 @@ export async function executeTool(
     }
   } catch (error) {
     // Best-effort: if a tool fails, return readable error instead of crashing
-    return `Terjadi kesalahan saat ambil data: ${(error as Error).message ?? "unknown"}`;
+    return toolExecError(error, name);
   }
 }
 

services/discord-gateway/src/modules/ai-moderation/llmCaller.ts

@@ -14,6 +14,7 @@ import type { ChatCompletion } from "openai/resources/chat/completions";
 import { createChildLogger } from "@/shared/logger/index";
 import { delay, retryWithBackoff } from "@/shared/utils/index";
 import { config } from "../../shared/config/config.js";
+import { incrementCounterBy } from "../gateway-metrics/index.js";
 import type { AnalysisResult } from "../message-capture/types.js";
 import { llmChat } from "./llmClient.js";
 import { parseModerationResponse } from "./moderationResponseParser.js";
@@ -170,6 +171,23 @@ export async function callModerationLLM(
     // so cost per channel/guild can be tracked (routers bill per token).
     const usage = result?.usage;
     if (usage && (usage.prompt_tokens || usage.completion_tokens)) {
+      // Prometheus token-usage counters for cost tracking per model + phase.
+      // Counters accumulate across scrapes so total spend per model/label is
+      // queryable (rate(...) dashboards).
+      if (usage.prompt_tokens) {
+        incrementCounterBy("llm_tokens_total", usage.prompt_tokens, {
+          model: config.AI_LLM_MODEL,
+          type: "prompt",
+          label,
+        });
+      }
+      if (usage.completion_tokens) {
+        incrementCounterBy("llm_tokens_total", usage.completion_tokens, {
+          model: config.AI_LLM_MODEL,
+          type: "completion",
+          label,
+        });
+      }
       log.info(
         {
           label,

services/discord-gateway/src/modules/ai-moderation/moderationOrchestrator.ts

@@ -8,6 +8,7 @@
 import { LRUCache } from "lru-cache";
 import { createChildLogger } from "@/shared/logger/index";
 import { config } from "../../shared/config/config.js";
+import { incrementCounterBy } from "../gateway-metrics/index.js";
 import { extractMessageMediaEvidence } from "../message-capture/messageMetadata.js";
 import type {
   AnalysisResult,
@@ -243,10 +244,12 @@ export async function runModerationAnalysis(
     const representative = hitByKey.get(candidate.scopedKey);
     if (representative) {
       cacheHits.push({ ...representative, messageId: candidate.target.id });
+      incrementCounterBy("moderation_cache_hits", 1, { type: "exact" });
     } else {
       uncachedTargets.push(candidate.target);
     }
   }
+  incrementCounterBy("moderation_cache_misses", uncachedTargets.length);
 
   // ── Phase 2: semantic cache — batched (one embed call + one Qdrant
   //    batch search for ALL uncached text targets) ─────────────────────────
@@ -321,6 +324,9 @@ export async function runModerationAnalysis(
             hitByKey.set(cacheKey, hit);
             servedCacheKeys.add(cacheKey); // bump hit_count for metrics
             logCacheEvent("hit", cacheKey, "text");
+            incrementCounterBy("moderation_cache_hits", 1, {
+              type: "semantic-qdrant",
+            });
           }
         } else {
           // Legacy Postgres fallback path (no Qdrant): per-candidate scan.
@@ -360,6 +366,9 @@ export async function runModerationAnalysis(
             hitByKey.set(cacheKey, hit);
             servedCacheKeys.add(cacheKey); // bump hit_count for metrics
             logCacheEvent("hit", cacheKey, "text");
+            incrementCounterBy("moderation_cache_hits", 1, {
+              type: "semantic-pg",
+            });
           }
         }
 

services/discord-gateway/src/modules/gateway-metrics/index.ts

@@ -1,5 +1,6 @@
 export {
   incrementCounter,
+  incrementCounterBy,
   registerCollector,
   setGauge,
   startMetricsServer,

services/discord-gateway/src/modules/gateway-metrics/metrics.ts

@@ -37,16 +37,25 @@ export function registerCollector(fn: () => void): void {
 export function incrementCounter(
   name: string,
   labels?: Record<string, string>,
+): void {
+  incrementCounterBy(name, 1, labels);
+}
+
+/** Increment a counter by an explicit delta (e.g. token counts per batch). */
+export function incrementCounterBy(
+  name: string,
+  delta: number,
+  labels?: Record<string, string>,
 ): void {
   const k = key(`bete_${name}`, labels);
   const existing = metrics.get(k);
   if (existing) {
-    existing.value += 1;
+    existing.value += delta;
   } else {
     metrics.set(k, {
       help: `Counter: ${name}`,
       type: "counter",
-      value: 1,
+      value: delta,
       labels: labels ? { ...labels } : undefined,
     });
   }