feat(ai): scope persistence, parallel tools, Prometheus token/cache counters

C2 full scope persistence:
- getRecentConversationContext now returns per-turn guildId/channelId
- processMessage merges historical scope when current request is unscoped
- chatbot remembers server context across all 8 history exchanges (not just 3)

A4+Metrics:
- Add incrementCounterBy(name, delta, labels) to gateway-metrics
- Token-usage counters: llm_tokens_total{model, type, label} per batch
- Cache hit counters: moderation_cache_hits{type} exact/semantic-qdrant/semantic-pg
- Cache miss counters: moderation_cache_misses per batch
- Prometheus /metrics now exposes cost + cache hit-rate for dashboards

Performance:
- Parallel tool execution within each chatbot round (Promise.all)
- All tool results collected before sending to model (ordering preserved)
- Tool failure now logged with structured warning (chatbot.tools.ts)

Verified: backend tsc+biome 37/37, discord-gateway tsc+biome 210/210
This commit is contained in:
asepharyana
2026-09-04 15:32:03 +07:00
committed by asepharyana
parent 9682abef82
commit 276062fdd4
5 changed files with 112 additions and 40 deletions
@@ -14,6 +14,7 @@ import type { ChatCompletion } from "openai/resources/chat/completions";
import { createChildLogger } from "@/shared/logger/index";
import { delay, retryWithBackoff } from "@/shared/utils/index";
import { config } from "../../shared/config/config.js";
import { incrementCounterBy } from "../gateway-metrics/index.js";
import type { AnalysisResult } from "../message-capture/types.js";
import { llmChat } from "./llmClient.js";
import { parseModerationResponse } from "./moderationResponseParser.js";
@@ -170,6 +171,23 @@ export async function callModerationLLM(
// so cost per channel/guild can be tracked (routers bill per token).
const usage = result?.usage;
if (usage && (usage.prompt_tokens || usage.completion_tokens)) {
// Prometheus token-usage counters for cost tracking per model + phase.
// Counters accumulate across scrapes so total spend per model/label is
// queryable (rate(...) dashboards).
if (usage.prompt_tokens) {
incrementCounterBy("llm_tokens_total", usage.prompt_tokens, {
model: config.AI_LLM_MODEL,
type: "prompt",
label,
});
}
if (usage.completion_tokens) {
incrementCounterBy("llm_tokens_total", usage.completion_tokens, {
model: config.AI_LLM_MODEL,
type: "completion",
label,
});
}
log.info(
{
label,
@@ -8,6 +8,7 @@
import { LRUCache } from "lru-cache";
import { createChildLogger } from "@/shared/logger/index";
import { config } from "../../shared/config/config.js";
import { incrementCounterBy } from "../gateway-metrics/index.js";
import { extractMessageMediaEvidence } from "../message-capture/messageMetadata.js";
import type {
AnalysisResult,
@@ -243,10 +244,12 @@ export async function runModerationAnalysis(
const representative = hitByKey.get(candidate.scopedKey);
if (representative) {
cacheHits.push({ ...representative, messageId: candidate.target.id });
incrementCounterBy("moderation_cache_hits", 1, { type: "exact" });
} else {
uncachedTargets.push(candidate.target);
}
}
incrementCounterBy("moderation_cache_misses", uncachedTargets.length);
// ── Phase 2: semantic cache — batched (one embed call + one Qdrant
// batch search for ALL uncached text targets) ─────────────────────────
@@ -321,6 +324,9 @@ export async function runModerationAnalysis(
hitByKey.set(cacheKey, hit);
servedCacheKeys.add(cacheKey); // bump hit_count for metrics
logCacheEvent("hit", cacheKey, "text");
incrementCounterBy("moderation_cache_hits", 1, {
type: "semantic-qdrant",
});
}
} else {
// Legacy Postgres fallback path (no Qdrant): per-candidate scan.
@@ -360,6 +366,9 @@ export async function runModerationAnalysis(
hitByKey.set(cacheKey, hit);
servedCacheKeys.add(cacheKey); // bump hit_count for metrics
logCacheEvent("hit", cacheKey, "text");
incrementCounterBy("moderation_cache_hits", 1, {
type: "semantic-pg",
});
}
}
@@ -1,5 +1,6 @@
export {
incrementCounter,
incrementCounterBy,
registerCollector,
setGauge,
startMetricsServer,
@@ -37,16 +37,25 @@ export function registerCollector(fn: () => void): void {
export function incrementCounter(
name: string,
labels?: Record<string, string>,
): void {
incrementCounterBy(name, 1, labels);
}
/** Increment a counter by an explicit delta (e.g. token counts per batch). */
export function incrementCounterBy(
name: string,
delta: number,
labels?: Record<string, string>,
): void {
const k = key(`bete_${name}`, labels);
const existing = metrics.get(k);
if (existing) {
existing.value += 1;
existing.value += delta;
} else {
metrics.set(k, {
help: `Counter: ${name}`,
type: "counter",
value: 1,
value: delta,
labels: labels ? { ...labels } : undefined,
});
}