feat(ai): scope persistence, parallel tools, Prometheus token/cache counters
C2 full scope persistence:
- getRecentConversationContext now returns per-turn guildId/channelId
- processMessage merges historical scope when current request is unscoped
- chatbot remembers server context across all 8 history exchanges (not just 3)
A4+Metrics:
- Add incrementCounterBy(name, delta, labels) to gateway-metrics
- Token-usage counters: llm_tokens_total{model, type, label} per batch
- Cache hit counters: moderation_cache_hits{type} exact/semantic-qdrant/semantic-pg
- Cache miss counters: moderation_cache_misses per batch
- Prometheus /metrics now exposes cost + cache hit-rate for dashboards
Performance:
- Parallel tool execution within each chatbot round (Promise.all)
- All tool results collected before sending to model (ordering preserved)
- Tool failure now logged with structured warning (chatbot.tools.ts)
Verified: backend tsc+biome 37/37, discord-gateway tsc+biome 210/210
This commit is contained in:
@@ -14,6 +14,7 @@ import type { ChatCompletion } from "openai/resources/chat/completions";
|
||||
import { createChildLogger } from "@/shared/logger/index";
|
||||
import { delay, retryWithBackoff } from "@/shared/utils/index";
|
||||
import { config } from "../../shared/config/config.js";
|
||||
import { incrementCounterBy } from "../gateway-metrics/index.js";
|
||||
import type { AnalysisResult } from "../message-capture/types.js";
|
||||
import { llmChat } from "./llmClient.js";
|
||||
import { parseModerationResponse } from "./moderationResponseParser.js";
|
||||
@@ -170,6 +171,23 @@ export async function callModerationLLM(
|
||||
// so cost per channel/guild can be tracked (routers bill per token).
|
||||
const usage = result?.usage;
|
||||
if (usage && (usage.prompt_tokens || usage.completion_tokens)) {
|
||||
// Prometheus token-usage counters for cost tracking per model + phase.
|
||||
// Counters accumulate across scrapes so total spend per model/label is
|
||||
// queryable (rate(...) dashboards).
|
||||
if (usage.prompt_tokens) {
|
||||
incrementCounterBy("llm_tokens_total", usage.prompt_tokens, {
|
||||
model: config.AI_LLM_MODEL,
|
||||
type: "prompt",
|
||||
label,
|
||||
});
|
||||
}
|
||||
if (usage.completion_tokens) {
|
||||
incrementCounterBy("llm_tokens_total", usage.completion_tokens, {
|
||||
model: config.AI_LLM_MODEL,
|
||||
type: "completion",
|
||||
label,
|
||||
});
|
||||
}
|
||||
log.info(
|
||||
{
|
||||
label,
|
||||
|
||||
@@ -8,6 +8,7 @@
|
||||
import { LRUCache } from "lru-cache";
|
||||
import { createChildLogger } from "@/shared/logger/index";
|
||||
import { config } from "../../shared/config/config.js";
|
||||
import { incrementCounterBy } from "../gateway-metrics/index.js";
|
||||
import { extractMessageMediaEvidence } from "../message-capture/messageMetadata.js";
|
||||
import type {
|
||||
AnalysisResult,
|
||||
@@ -243,10 +244,12 @@ export async function runModerationAnalysis(
|
||||
const representative = hitByKey.get(candidate.scopedKey);
|
||||
if (representative) {
|
||||
cacheHits.push({ ...representative, messageId: candidate.target.id });
|
||||
incrementCounterBy("moderation_cache_hits", 1, { type: "exact" });
|
||||
} else {
|
||||
uncachedTargets.push(candidate.target);
|
||||
}
|
||||
}
|
||||
incrementCounterBy("moderation_cache_misses", uncachedTargets.length);
|
||||
|
||||
// ── Phase 2: semantic cache — batched (one embed call + one Qdrant
|
||||
// batch search for ALL uncached text targets) ─────────────────────────
|
||||
@@ -321,6 +324,9 @@ export async function runModerationAnalysis(
|
||||
hitByKey.set(cacheKey, hit);
|
||||
servedCacheKeys.add(cacheKey); // bump hit_count for metrics
|
||||
logCacheEvent("hit", cacheKey, "text");
|
||||
incrementCounterBy("moderation_cache_hits", 1, {
|
||||
type: "semantic-qdrant",
|
||||
});
|
||||
}
|
||||
} else {
|
||||
// Legacy Postgres fallback path (no Qdrant): per-candidate scan.
|
||||
@@ -360,6 +366,9 @@ export async function runModerationAnalysis(
|
||||
hitByKey.set(cacheKey, hit);
|
||||
servedCacheKeys.add(cacheKey); // bump hit_count for metrics
|
||||
logCacheEvent("hit", cacheKey, "text");
|
||||
incrementCounterBy("moderation_cache_hits", 1, {
|
||||
type: "semantic-pg",
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
export {
|
||||
incrementCounter,
|
||||
incrementCounterBy,
|
||||
registerCollector,
|
||||
setGauge,
|
||||
startMetricsServer,
|
||||
|
||||
@@ -37,16 +37,25 @@ export function registerCollector(fn: () => void): void {
|
||||
export function incrementCounter(
|
||||
name: string,
|
||||
labels?: Record<string, string>,
|
||||
): void {
|
||||
incrementCounterBy(name, 1, labels);
|
||||
}
|
||||
|
||||
/** Increment a counter by an explicit delta (e.g. token counts per batch). */
|
||||
export function incrementCounterBy(
|
||||
name: string,
|
||||
delta: number,
|
||||
labels?: Record<string, string>,
|
||||
): void {
|
||||
const k = key(`bete_${name}`, labels);
|
||||
const existing = metrics.get(k);
|
||||
if (existing) {
|
||||
existing.value += 1;
|
||||
existing.value += delta;
|
||||
} else {
|
||||
metrics.set(k, {
|
||||
help: `Counter: ${name}`,
|
||||
type: "counter",
|
||||
value: 1,
|
||||
value: delta,
|
||||
labels: labels ? { ...labels } : undefined,
|
||||
});
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user