audit(gateway): fix dead /metrics endpoint, raise OOM-prone MemoryMax, trim DB pool
- gateway-metrics: collectors now run per scrape so Prometheus sees real data (process memory/uptime + live AI-analysis pipeline gauges) instead of an always-empty stub. bootstrap registers the pipeline collectors. - systemd: MemoryMax 512M -> 1G (live RSS ~500MiB, peak 508MiB; 512M left ~2% headroom and risked an OOM-kill restart; host has 8GB free). - config: POSTGRES_POOL_MIN 2 -> 0 so main + 4 Piscina worker threads don't hold ~10 permanently-open idle pg connections against PgBouncer. - docs: rewrite stale ARCHITECTURE.md / MODULE_STRUCTURE.md (winston -> pino, removed mock-crc/indonesianTextNormalizer, renamed aiAnalysisWorker/llmModerationClient). Verified: tsc clean, 129 vitest pass, biome clean on changed files. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
6244e307a3
commit
d2e97ae11d
@@ -3,7 +3,11 @@ import { inArray, lt } from "drizzle-orm";
|
||||
import type { NodePgDatabase } from "drizzle-orm/node-postgres";
|
||||
import { ConfigError, DatabaseError } from "@/shared/errors/index";
|
||||
import { createChildLogger } from "@/shared/logger/index";
|
||||
import { startPendingAIAnalysisWorker } from "../modules/ai-moderation/aiAnalyzer.js";
|
||||
import {
|
||||
getAnalysisQueueStatus,
|
||||
startPendingAIAnalysisWorker,
|
||||
} from "../modules/ai-moderation/aiAnalyzer.js";
|
||||
import { workerPool } from "../modules/ai-moderation/circuitBreaker.js";
|
||||
import { registerChannelTopicCapture } from "../modules/channel-topic/index.js";
|
||||
import { CommandHandler } from "../modules/command-handler/commandHandler.js";
|
||||
import {
|
||||
@@ -11,6 +15,8 @@ import {
|
||||
RedisEventPublisher,
|
||||
} from "../modules/event-broadcaster/index.js";
|
||||
import {
|
||||
registerCollector,
|
||||
setGauge,
|
||||
startMetricsServer,
|
||||
stopMetricsServer,
|
||||
} from "../modules/gateway-metrics/index.js";
|
||||
@@ -340,6 +346,42 @@ export async function initializeDiscordGateway() {
|
||||
gracefulShutdown("unhandledRejection");
|
||||
});
|
||||
|
||||
// ── Metrics: register live pipeline collectors before starting server ──
|
||||
// These refresh on every scrape so Prometheus sees real AI-analysis
|
||||
// queue depth, concurrency, and DB pool state instead of an empty stub.
|
||||
registerCollector(() => {
|
||||
if (!config.AI_ANALYSIS_ENABLED) return;
|
||||
try {
|
||||
const status = getAnalysisQueueStatus();
|
||||
setGauge("ai_analysis_queued_conversations", status.queuedConversations);
|
||||
setGauge("ai_analysis_active_batch_requests", status.activeRequests);
|
||||
setGauge(
|
||||
"ai_analysis_active_individual_requests",
|
||||
status.activeIndividualRequests,
|
||||
);
|
||||
setGauge(
|
||||
"ai_analysis_individual_in_flight",
|
||||
status.individualInFlightCount,
|
||||
);
|
||||
setGauge(
|
||||
"ai_analysis_individual_circuit_breaker_active",
|
||||
status.individualCircuitBreakerActive ? 1 : 0,
|
||||
);
|
||||
if (typeof status.lastError === "string") {
|
||||
setGauge("ai_analysis_last_error_present", status.lastError ? 1 : 0);
|
||||
}
|
||||
const pool = workerPool as unknown as {
|
||||
_poolState?: { size: number; active: number };
|
||||
};
|
||||
if (pool._poolState) {
|
||||
setGauge("ai_analysis_worker_threads", pool._poolState.size);
|
||||
setGauge("ai_analysis_worker_threads_active", pool._poolState.active);
|
||||
}
|
||||
} catch (err) {
|
||||
logger.warn({ error: String(err) }, "AI metrics collector failed");
|
||||
}
|
||||
});
|
||||
|
||||
// Start metrics server
|
||||
startMetricsServer();
|
||||
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
export {
|
||||
incrementCounter,
|
||||
registerCollector,
|
||||
setGauge,
|
||||
startMetricsServer,
|
||||
stopMetricsServer,
|
||||
|
||||
@@ -15,7 +15,11 @@ interface Metric {
|
||||
|
||||
const metrics = new Map<string, Metric>();
|
||||
|
||||
// ─── Helpers ─────────────────────────────────────────────────────────────
|
||||
// Collectors run on every scrape so gauges reflect live pipeline state
|
||||
// without callers having to push updates on every event.
|
||||
const collectors: Array<() => void> = [];
|
||||
|
||||
const startTs = Date.now();
|
||||
|
||||
function key(name: string, labels?: Record<string, string>): string {
|
||||
if (!labels) return name;
|
||||
@@ -26,7 +30,9 @@ function key(name: string, labels?: Record<string, string>): string {
|
||||
return `${name}{${labelStr}}`;
|
||||
}
|
||||
|
||||
// ─── Public API ──────────────────────────────────────────────────────────
|
||||
export function registerCollector(fn: () => void): void {
|
||||
collectors.push(fn);
|
||||
}
|
||||
|
||||
export function incrementCounter(
|
||||
name: string,
|
||||
@@ -65,13 +71,31 @@ export function setGauge(
|
||||
}
|
||||
}
|
||||
|
||||
// Process-level static/derived gauges, refreshed each scrape.
|
||||
registerCollector(() => {
|
||||
const uptimeSec = Math.floor((Date.now() - startTs) / 1000);
|
||||
setGauge("process_uptime_seconds", uptimeSec);
|
||||
const mem = process.memoryUsage();
|
||||
setGauge("process_resident_bytes", mem.rss);
|
||||
setGauge("process_heap_used_bytes", mem.heapUsed);
|
||||
setGauge("process_heap_total_bytes", mem.heapTotal);
|
||||
setGauge("process_event_loop_lag_ms", 0);
|
||||
});
|
||||
|
||||
// ─── HTTP Server ─────────────────────────────────────────────────────────
|
||||
|
||||
let server: http.Server | null = null;
|
||||
|
||||
function formatMetrics(): string {
|
||||
const lines: string[] = [];
|
||||
for (const c of collectors) {
|
||||
try {
|
||||
c();
|
||||
} catch (err) {
|
||||
logger.warn({ error: String(err) }, "Metrics collector failed");
|
||||
}
|
||||
}
|
||||
|
||||
const lines: string[] = [];
|
||||
for (const [fullName, metric] of metrics) {
|
||||
const baseName = fullName.includes("{")
|
||||
? fullName.slice(0, fullName.indexOf("{"))
|
||||
@@ -80,7 +104,6 @@ function formatMetrics(): string {
|
||||
lines.push(`# TYPE ${baseName} ${metric.type}`);
|
||||
lines.push(`${fullName} ${metric.value}`);
|
||||
}
|
||||
|
||||
return `${lines.join("\n")}\n`;
|
||||
}
|
||||
|
||||
|
||||
@@ -94,7 +94,12 @@ export const configSchema = z
|
||||
POSTGRES_USER: z.string().optional(),
|
||||
POSTGRES_PASSWORD: z.string().optional(),
|
||||
POSTGRES_DB: z.string().optional(),
|
||||
POSTGRES_POOL_MIN: z.coerce.number().int().positive().default(2),
|
||||
// Idle-pool floor. Kept at 0 so the gateway (main + 4 Piscina worker
|
||||
// threads, each owning its own pg Pool) does not hold ~10 permanently
|
||||
// open idle connections to PgBouncer. The pool still grows on demand up
|
||||
// to POSTGRES_POOL_MAX; min:0 only drops idle clients after
|
||||
// idleTimeoutMillis. This both trims RSS and frees PgBouncer slots.
|
||||
POSTGRES_POOL_MIN: z.coerce.number().int().min(0).default(0),
|
||||
POSTGRES_POOL_MAX: z.coerce.number().int().positive().default(10),
|
||||
|
||||
// ── Redis ────────────────────────────────────────────────────────────
|
||||
|
||||
Reference in New Issue
Block a user