From 5520094725d124a524d19b2495df9e4553c06e60 Mon Sep 17 00:00:00 2001 From: MythEclipse Date: Tue, 2 Jun 2026 18:31:32 +0700 Subject: [PATCH] chore: auto-commit task - --- components.json | 20 - postcss.config.js | 5 - src/app/bootstrap.ts | 113 -- src/app/shutdown.ts | 56 - src/audio/ffmpegProcess.ts | 61 - src/audio/pcm.ts | 28 - src/config.ts | 228 --- src/database/drizzle.ts | 129 -- src/database/migrate.ts | 54 - src/database/migrateCli.ts | 14 - .../migrations/001_drop_unused_ai_columns.sql | 10 - src/database/schema.ts | 464 ----- src/database/voiceRecordingRepo.ts | 115 -- src/discordClientOptions.ts | 21 - src/errors.ts | 43 - src/http/app.ts | 148 -- src/http/health.ts | 39 - src/http/server.ts | 114 -- src/index.ts | 13 - src/logger.ts | 132 -- src/loggerSerialization.ts | 110 -- src/media/mediaController.ts | 280 --- src/media/mediaQueue.ts | 64 - src/media/mediaResolver.ts | 126 -- src/media/mediaTypes.ts | 80 - src/media/musicPlayer.ts | 128 -- src/media/playDlResolver.ts | 60 - src/media/screenShareController.ts | 166 -- src/media/ytdlp.ts | 104 - src/metrics.ts | 89 - src/middleware.ts | 36 - src/mock-crc.ts | 48 - src/moderation/actionExecutor.ts | 295 --- src/moderation/aiAnalysisWorker.ts | 145 -- src/moderation/aiAnalyzer.ts | 918 --------- src/moderation/analyticsStore.ts | 929 --------- src/moderation/attachmentUploader.ts | 133 -- src/moderation/autoDeleteManager.ts | 355 ---- src/moderation/backlogSync.ts | 113 -- src/moderation/broadcaster.ts | 82 - src/moderation/concurrencyLimiter.ts | 14 - src/moderation/conversationContext.ts | 77 - src/moderation/imageResizer.ts | 58 - src/moderation/indonesianTextNormalizer.ts | 222 --- src/moderation/llmClient.ts | 276 --- src/moderation/llmModerationClient.ts | 1282 ------------ src/moderation/messageCapture.ts | 296 --- src/moderation/messageMetadata.ts | 375 ---- src/moderation/messageStore.ts | 1239 ------------ src/moderation/moderationPrompt.ts | 220 --- src/moderation/pagination.ts | 21 - src/moderation/retentionManager.ts | 181 -- src/moderation/stickerCache.ts | 162 -- src/moderation/stickerPrompt.ts | 135 -- src/moderation/textCacheStore.ts | 241 --- src/moderation/types.ts | 211 -- src/moderation/urlFetcher.ts | 209 -- src/muxer-aup3.ts | 263 --- src/muxer-queue.ts | 405 ---- src/muxer.ts | 169 -- src/packetFilter.ts | 41 - src/player.ts | 137 -- src/recorder.ts | 326 ---- src/recorder/audioStream.ts | 27 - src/recorder/decoder.ts | 127 -- src/recorder/metadata.ts | 75 - src/recorder/segment.ts | 87 - src/recorder/sessionRecording.ts | 192 -- src/recorder/uploader.ts | 115 -- src/retry.ts | 42 - src/routes/analysisRoutes.ts | 125 -- src/routes/analyticsRoutes.ts | 310 --- src/routes/appConfigRoutes.ts | 15 - src/routes/mediaRoutes.ts | 118 -- src/routes/messageRoutes.ts | 235 --- src/routes/recordingsRoutes.ts | 39 - src/routes/reviewRoutes.ts | 268 --- src/routes/syncRoutes.ts | 111 -- src/routes/uiStateRoutes.ts | 40 - src/routes/voiceRoutes.ts | 185 -- src/state/mediaSettings.ts | 26 - src/state/uiState.ts | 104 - src/streaming/index.ts | 185 -- src/streaming/transcoder.ts | 182 -- src/types.ts | 50 - src/uploader/teleUpload.ts | 85 - src/validation.ts | 22 - src/voiceController.ts | 179 -- src/webserver.ts | 1 - src/ws/broadcastGlobals.ts | 88 - src/ws/server.ts | 78 - src/ws/voiceAudioBridge.ts | 121 -- tailwind.config.js | 27 - tests/audio/ffmpegProcess.test.ts | 102 - tests/audio/pcm.test.ts | 30 - tests/config.test.ts | 125 -- tests/database.test.ts | 93 - tests/database/migrate.test.ts | 82 - tests/decoder.test.ts | 33 - tests/helpers/testDatabase.ts | 108 -- tests/logger.test.ts | 71 - tests/media/mediaController.test.ts | 277 --- tests/media/mediaQueue.test.ts | 93 - tests/media/mediaResolver.test.ts | 146 -- tests/media/musicPlayer.test.ts | 170 -- tests/media/playDlResolver.test.ts | 49 - tests/media/screenShareController.test.ts | 95 - tests/media/ytdlp.test.ts | 107 - tests/moderation/analysisQueue.test.ts | 46 - tests/moderation/attachmentUploader.test.ts | 118 -- tests/moderation/autoDeleteManager.test.ts | 158 -- tests/moderation/broadcaster.test.ts | 163 -- tests/moderation/conversationContext.test.ts | 155 -- .../indonesianTextNormalizer.remote.test.ts | 57 - .../indonesianTextNormalizer.test.ts | 104 - tests/moderation/llmLive.test.ts | 75 - tests/moderation/llmModerationClient.test.ts | 1722 ----------------- tests/moderation/messageCapture.test.ts | 154 -- tests/moderation/messageCaptureFilter.test.ts | 50 - tests/moderation/messageStoreQueries.test.ts | 692 ------- tests/player.test.ts | 82 - tests/recorder.test.ts | 241 --- tests/recorder/metadata.test.ts | 46 - tests/recorder/sessionRecording.test.ts | 103 - tests/routes/mediaRoutes.test.ts | 157 -- tests/routes/syncRoutes.test.ts | 99 - tests/setup.ts | 7 - tests/smoke.test.ts | 5 - tests/state/uiState.test.ts | 46 - tests/streaming/playTranscode.test.ts | 64 - tests/streaming/transcoder.test.ts | 55 - tests/validation.test.ts | 35 - tests/vendor/requestHandlerBackoff.test.ts | 15 - tests/vendor/selfbotClientOptions.test.ts | 22 - tsconfig.json | 21 - vite.backend.config.ts | 35 - vitest.config.ts | 13 - 137 files changed, 22008 deletions(-) delete mode 100644 components.json delete mode 100644 postcss.config.js delete mode 100644 src/app/bootstrap.ts delete mode 100644 src/app/shutdown.ts delete mode 100644 src/audio/ffmpegProcess.ts delete mode 100644 src/audio/pcm.ts delete mode 100644 src/config.ts delete mode 100644 src/database/drizzle.ts delete mode 100644 src/database/migrate.ts delete mode 100644 src/database/migrateCli.ts delete mode 100644 src/database/migrations/001_drop_unused_ai_columns.sql delete mode 100644 src/database/schema.ts delete mode 100644 src/database/voiceRecordingRepo.ts delete mode 100644 src/discordClientOptions.ts delete mode 100644 src/errors.ts delete mode 100644 src/http/app.ts delete mode 100644 src/http/health.ts delete mode 100644 src/http/server.ts delete mode 100644 src/index.ts delete mode 100644 src/logger.ts delete mode 100644 src/loggerSerialization.ts delete mode 100644 src/media/mediaController.ts delete mode 100644 src/media/mediaQueue.ts delete mode 100644 src/media/mediaResolver.ts delete mode 100644 src/media/mediaTypes.ts delete mode 100644 src/media/musicPlayer.ts delete mode 100644 src/media/playDlResolver.ts delete mode 100644 src/media/screenShareController.ts delete mode 100644 src/media/ytdlp.ts delete mode 100644 src/metrics.ts delete mode 100644 src/middleware.ts delete mode 100644 src/mock-crc.ts delete mode 100644 src/moderation/actionExecutor.ts delete mode 100644 src/moderation/aiAnalysisWorker.ts delete mode 100644 src/moderation/aiAnalyzer.ts delete mode 100644 src/moderation/analyticsStore.ts delete mode 100644 src/moderation/attachmentUploader.ts delete mode 100644 src/moderation/autoDeleteManager.ts delete mode 100644 src/moderation/backlogSync.ts delete mode 100644 src/moderation/broadcaster.ts delete mode 100644 src/moderation/concurrencyLimiter.ts delete mode 100644 src/moderation/conversationContext.ts delete mode 100644 src/moderation/imageResizer.ts delete mode 100644 src/moderation/indonesianTextNormalizer.ts delete mode 100644 src/moderation/llmClient.ts delete mode 100644 src/moderation/llmModerationClient.ts delete mode 100644 src/moderation/messageCapture.ts delete mode 100644 src/moderation/messageMetadata.ts delete mode 100644 src/moderation/messageStore.ts delete mode 100644 src/moderation/moderationPrompt.ts delete mode 100644 src/moderation/pagination.ts delete mode 100644 src/moderation/retentionManager.ts delete mode 100644 src/moderation/stickerCache.ts delete mode 100644 src/moderation/stickerPrompt.ts delete mode 100644 src/moderation/textCacheStore.ts delete mode 100644 src/moderation/types.ts delete mode 100644 src/moderation/urlFetcher.ts delete mode 100644 src/muxer-aup3.ts delete mode 100644 src/muxer-queue.ts delete mode 100644 src/muxer.ts delete mode 100644 src/packetFilter.ts delete mode 100644 src/player.ts delete mode 100644 src/recorder.ts delete mode 100644 src/recorder/audioStream.ts delete mode 100644 src/recorder/decoder.ts delete mode 100644 src/recorder/metadata.ts delete mode 100644 src/recorder/segment.ts delete mode 100644 src/recorder/sessionRecording.ts delete mode 100644 src/recorder/uploader.ts delete mode 100644 src/retry.ts delete mode 100644 src/routes/analysisRoutes.ts delete mode 100644 src/routes/analyticsRoutes.ts delete mode 100644 src/routes/appConfigRoutes.ts delete mode 100644 src/routes/mediaRoutes.ts delete mode 100644 src/routes/messageRoutes.ts delete mode 100644 src/routes/recordingsRoutes.ts delete mode 100644 src/routes/reviewRoutes.ts delete mode 100644 src/routes/syncRoutes.ts delete mode 100644 src/routes/uiStateRoutes.ts delete mode 100644 src/routes/voiceRoutes.ts delete mode 100644 src/state/mediaSettings.ts delete mode 100644 src/state/uiState.ts delete mode 100644 src/streaming/index.ts delete mode 100644 src/streaming/transcoder.ts delete mode 100644 src/types.ts delete mode 100644 src/uploader/teleUpload.ts delete mode 100644 src/validation.ts delete mode 100644 src/voiceController.ts delete mode 100644 src/webserver.ts delete mode 100644 src/ws/broadcastGlobals.ts delete mode 100644 src/ws/server.ts delete mode 100644 src/ws/voiceAudioBridge.ts delete mode 100644 tailwind.config.js delete mode 100644 tests/audio/ffmpegProcess.test.ts delete mode 100644 tests/audio/pcm.test.ts delete mode 100644 tests/config.test.ts delete mode 100644 tests/database.test.ts delete mode 100644 tests/database/migrate.test.ts delete mode 100644 tests/decoder.test.ts delete mode 100644 tests/helpers/testDatabase.ts delete mode 100644 tests/logger.test.ts delete mode 100644 tests/media/mediaController.test.ts delete mode 100644 tests/media/mediaQueue.test.ts delete mode 100644 tests/media/mediaResolver.test.ts delete mode 100644 tests/media/musicPlayer.test.ts delete mode 100644 tests/media/playDlResolver.test.ts delete mode 100644 tests/media/screenShareController.test.ts delete mode 100644 tests/media/ytdlp.test.ts delete mode 100644 tests/moderation/analysisQueue.test.ts delete mode 100644 tests/moderation/attachmentUploader.test.ts delete mode 100644 tests/moderation/autoDeleteManager.test.ts delete mode 100644 tests/moderation/broadcaster.test.ts delete mode 100644 tests/moderation/conversationContext.test.ts delete mode 100644 tests/moderation/indonesianTextNormalizer.remote.test.ts delete mode 100644 tests/moderation/indonesianTextNormalizer.test.ts delete mode 100644 tests/moderation/llmLive.test.ts delete mode 100644 tests/moderation/llmModerationClient.test.ts delete mode 100644 tests/moderation/messageCapture.test.ts delete mode 100644 tests/moderation/messageCaptureFilter.test.ts delete mode 100644 tests/moderation/messageStoreQueries.test.ts delete mode 100644 tests/player.test.ts delete mode 100644 tests/recorder.test.ts delete mode 100644 tests/recorder/metadata.test.ts delete mode 100644 tests/recorder/sessionRecording.test.ts delete mode 100644 tests/routes/mediaRoutes.test.ts delete mode 100644 tests/routes/syncRoutes.test.ts delete mode 100644 tests/setup.ts delete mode 100644 tests/smoke.test.ts delete mode 100644 tests/state/uiState.test.ts delete mode 100644 tests/streaming/playTranscode.test.ts delete mode 100644 tests/streaming/transcoder.test.ts delete mode 100644 tests/validation.test.ts delete mode 100644 tests/vendor/requestHandlerBackoff.test.ts delete mode 100644 tests/vendor/selfbotClientOptions.test.ts delete mode 100644 tsconfig.json delete mode 100644 vite.backend.config.ts delete mode 100644 vitest.config.ts diff --git a/components.json b/components.json deleted file mode 100644 index 05bc629..0000000 --- a/components.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "$schema": "https://ui.shadcn.com/schema.json", - "style": "new-york", - "rsc": false, - "tsx": true, - "tailwind": { - "config": "tailwind.config.js", - "css": "frontend/src/styles.css", - "baseColor": "slate", - "cssVariables": true, - "prefix": "" - }, - "aliases": { - "components": "frontend/src/components", - "utils": "frontend/src/lib/utils", - "ui": "frontend/src/components/ui", - "lib": "frontend/src/lib", - "hooks": "frontend/src/hooks" - } -} diff --git a/postcss.config.js b/postcss.config.js deleted file mode 100644 index c2ddf74..0000000 --- a/postcss.config.js +++ /dev/null @@ -1,5 +0,0 @@ -export default { - plugins: { - "@tailwindcss/postcss": {}, - }, -}; diff --git a/src/app/bootstrap.ts b/src/app/bootstrap.ts deleted file mode 100644 index 3872a50..0000000 --- a/src/app/bootstrap.ts +++ /dev/null @@ -1,113 +0,0 @@ -import { Client } from "discord.js-selfbot-v13"; -import { config } from "../config.js"; -import { closeDatabase, initializeDatabase } from "../database/drizzle.js"; -import { runMigrations } from "../database/migrate.js"; -import { createDiscordClientOptions } from "../discordClientOptions.js"; -import { createChildLogger } from "../logger.js"; -import { startPendingAIAnalysisWorker } from "../moderation/aiAnalyzer.js"; -import { syncBacklogMessages } from "../moderation/backlogSync.js"; -import { registerMessageCapture } from "../moderation/messageCapture.js"; -import { discordPlayer } from "../player.js"; -import { VoiceController } from "../voiceController.js"; -import { startWebserver } from "../webserver.js"; -import { createGracefulShutdown } from "./shutdown.js"; - -export async function initializeApp() { - const logger = createChildLogger("bot"); - - if (!config.AI_LLM_API_KEY) { - logger.error( - "AI_LLM_API_KEY is missing from environment. Force closing application as AI environment is required.", - ); - process.exit(1); - } - - const token = config.DISCORD_TOKEN; - logger.info( - { hasToken: token.length > 0, tokenLength: token.length }, - "Config loaded", - ); - - logger.info("Creating Discord client"); - const client = new Client(createDiscordClientOptions()); - const voiceController = new VoiceController(client); - - const gracefulShutdown = createGracefulShutdown({ - logger, - closeDatabase, - voiceController, - discordPlayer, - client, - }); - - try { - if (config.AUTO_MIGRATE_ON_STARTUP) { - logger.info( - "AUTO_MIGRATE_ON_STARTUP enabled; running database migrations", - ); - await runMigrations(); - } - - logger.info("Initializing database"); - await initializeDatabase(); - logger.info("PostgreSQL database initialized"); - } catch (err) { - logger.error({ error: err }, "Failed to initialize database"); - process.exit(1); - } - - client.on("debug", (msg) => { - if ( - msg.includes("[VOICE") || - msg.includes("[ffmpeg") || - msg.toLowerCase().includes("error") || - msg.toLowerCase().includes("stream") - ) { - logger.info({ debugMsg: msg }, "Discord Client Debug"); - } else if (config.VERBOSE) { - logger.debug({ debugMsg: msg }, "Discord Client Debug"); - } - }); - - client.on("ready", async () => { - logger.info({ user: client.user?.tag }, "Bot logged in"); - registerMessageCapture(client); - startPendingAIAnalysisWorker(client); - syncBacklogMessages(client).catch((error) => { - logger.warn({ error }, "Backlog sync failed"); - }); - await startWebserver(config.WEBSERVER_PORT, client, voiceController); - }); - - client.on("error", (err) => { - logger.error({ error: err }, "Client error"); - }); - - process.on("SIGINT", () => { - gracefulShutdown("SIGINT"); - }); - - process.on("SIGTERM", () => { - gracefulShutdown("SIGTERM"); - }); - - process.on("uncaughtException", (err) => { - logger.error({ error: err }, "Uncaught exception"); - gracefulShutdown("uncaughtException"); - }); - - process.on("unhandledRejection", (reason, promise) => { - logger.error({ reason, promise }, "Unhandled rejection"); - gracefulShutdown("unhandledRejection"); - }); - - logger.info("Calling Discord client.login"); - client - .login(token) - .then(() => { - logger.info("Discord client.login resolved"); - }) - .catch((error) => { - logger.error({ error }, "Discord client.login failed"); - }); -} diff --git a/src/app/shutdown.ts b/src/app/shutdown.ts deleted file mode 100644 index 42a7b11..0000000 --- a/src/app/shutdown.ts +++ /dev/null @@ -1,56 +0,0 @@ -import type { Client } from "discord.js-selfbot-v13"; -import type { closeDatabase } from "../database/drizzle.js"; -import type { createChildLogger } from "../logger.js"; -import type { discordPlayer } from "../player.js"; -import type { VoiceController } from "../voiceController.js"; - -type Logger = ReturnType; -type CloseDatabase = typeof closeDatabase; -type DiscordPlayer = typeof discordPlayer; - -export interface GracefulShutdownOptions { - logger: Logger; - closeDatabase: CloseDatabase; - voiceController: VoiceController; - discordPlayer: DiscordPlayer; - client: Client; -} - -export function createGracefulShutdown(options: GracefulShutdownOptions) { - let isShuttingDown = false; - - return async function gracefulShutdown(signal: string) { - if (isShuttingDown) { - options.logger.warn(`Already shutting down, ignoring ${signal}`); - return; - } - - isShuttingDown = true; - options.logger.info({ signal }, "Graceful shutdown initiated"); - - try { - options.logger.info("Closing database..."); - await options.closeDatabase(); - options.logger.info("Database closed"); - - options.logger.info("Stopping voice connection..."); - await options.voiceController.disconnect(); - - options.logger.info("Pausing player..."); - options.discordPlayer.pause(); - - options.logger.info("Destroying Discord client..."); - try { - options.client.destroy(); - } catch (err) { - options.logger.warn({ error: err }, "Error destroying client"); - } - - options.logger.info("Graceful shutdown completed"); - process.exit(0); - } catch (err) { - options.logger.error({ error: err }, "Error during graceful shutdown"); - process.exit(1); - } - }; -} diff --git a/src/audio/ffmpegProcess.ts b/src/audio/ffmpegProcess.ts deleted file mode 100644 index 9fb5675..0000000 --- a/src/audio/ffmpegProcess.ts +++ /dev/null @@ -1,61 +0,0 @@ -import { spawn } from "child_process"; - -export interface MuxFfmpegArgsOptions { - inputs: string[]; - filter: string; - output: string; - codec: string; - audioFrequency?: number; - audioChannels?: number; -} - -/** - * Builds ffmpeg argument array for muxing audio clips. - */ -export function buildMuxFfmpegArgs(options: MuxFfmpegArgsOptions): string[] { - const args: string[] = ["-y"]; - - for (const input of options.inputs) { - args.push("-i", input); - } - - args.push("-filter_complex", options.filter); - args.push("-map", "[out]"); - args.push("-codec:a", options.codec); - - if (options.audioFrequency !== undefined) { - args.push("-ar", String(options.audioFrequency)); - } - - if (options.audioChannels !== undefined) { - args.push("-ac", String(options.audioChannels)); - } - - args.push(options.output); - - return args; -} - -/** - * Runs ffmpeg with the given arguments. - * Resolves on successful (code 0) exit, rejects on error or non-zero exit. - */ -export function runFfmpeg(args: string[]): Promise { - return new Promise((resolve, reject) => { - const proc = spawn("ffmpeg", args, { - stdio: ["ignore", "inherit", "inherit"], - }); - - proc.on("close", (code) => { - if (code === 0) { - resolve(); - } else { - reject(new Error(`ffmpeg exited with code ${code}`)); - } - }); - - proc.on("error", (err) => { - reject(err); - }); - }); -} diff --git a/src/audio/pcm.ts b/src/audio/pcm.ts deleted file mode 100644 index fcf4320..0000000 --- a/src/audio/pcm.ts +++ /dev/null @@ -1,28 +0,0 @@ -export function upsample24kMonoTo48kStereo(mono24k: Buffer): Buffer { - const numSamples = mono24k.length / 2; - const out = Buffer.alloc(numSamples * 8); - - for (let i = 0; i < numSamples; i++) { - const sample = mono24k.readInt16LE(i * 2); - const base = i * 8; - out.writeInt16LE(sample, base); - out.writeInt16LE(sample, base + 2); - out.writeInt16LE(sample, base + 4); - out.writeInt16LE(sample, base + 6); - } - - return out; -} - -export function rmsDb(pcm: Buffer): number { - let sum = 0; - const samples = pcm.length / 2; - - for (let i = 0; i < samples; i++) { - const sample = pcm.readInt16LE(i * 2) / 32768; - sum += sample * sample; - } - - const rms = Math.sqrt(sum / samples); - return 20 * Math.log10(rms); -} diff --git a/src/config.ts b/src/config.ts deleted file mode 100644 index e5972cc..0000000 --- a/src/config.ts +++ /dev/null @@ -1,228 +0,0 @@ -import "dotenv/config"; -import { z } from "zod"; -import { ConfigError } from "./errors.js"; - -const configSchema = z - .object({ - DISCORD_TOKEN: z - .string() - .min(1, "DISCORD_TOKEN is required") - .transform((value) => value.replace(/^("|')|(?:("|'))$/g, "")), - VOICE_CHANNEL_ID: z.string().min(1).optional(), - GUILD_ID: z.string().min(1).optional(), - TEXT_GUILD_ID: z.string().min(1).optional(), - TEXT_CHANNEL_ID: z.string().min(1).optional(), - VOICE_GUILD_ID: z.string().min(1).optional(), - VERBOSE: z - .string() - .optional() - .transform((v) => v === "true") - .default(false), - RECORDINGS_DIR: z.string().default("./recordings"), - RECORDING_SEGMENT_MS: z.coerce.number().positive().default(5000), - DECODER_ROTATE_MS: z.coerce.number().positive().default(5000), - DECODER_COOLDOWN_MS: z.coerce.number().positive().default(0), - WEBSERVER_PORT: z.coerce.number().positive().default(3000), - VOICE_CONNECTION_TIMEOUT_MS: z.coerce.number().positive().default(15000), - RECONNECT_TIMEOUT_MS: z.coerce.number().positive().default(5000), - AUDIO_STREAM_SILENCE_DURATION_MS: z.coerce - .number() - .positive() - .default(3000), - PACKET_FILTER_MIN_SIZE: z.coerce.number().positive().default(8), - OPUS_FRAME_SIZE: z.coerce.number().positive().default(960), - AUDIO_SAMPLE_RATE: z.coerce.number().positive().default(48000), - AUDIO_CHANNELS: z.coerce.number().positive().default(2), - AVATAR_SIZE: z.coerce.number().positive().default(64), - LOG_LEVEL: z - .enum(["error", "warn", "info", "http", "verbose", "debug", "silly"]) - .default("info"), - NODE_ENV: z - .enum(["development", "production", "test"]) - .default("development"), - MONITOR_GUILD_ID: z.string().min(1).optional(), - TELE_UPLOAD_URL: z - .string() - .url() - .default("https://upload.asepharyana.tech/api/upload"), - ATTACHMENT_UPLOAD_TIMEOUT_MS: z.coerce.number().positive().default(30000), - ATTACHMENT_MAX_SIZE_MB: z.coerce.number().positive().default(100), - ATTACHMENT_RETRY_ATTEMPTS: z.coerce.number().positive().default(3), - BACKLOG_SYNC_HOURS: z.coerce.number().positive().default(24), - BACKLOG_SYNC_BATCH_SIZE: z.coerce - .number() - .int() - .positive() - .max(100) - .default(100), - AI_ANALYSIS_ENABLED: z - .string() - .optional() - .transform((v) => v === "true") - .default(false), - OPENAI_MODERATION_API_KEY: z.string().optional(), - OPENAI_MODERATION_BASE_URL: z - .string() - .url() - .default("https://api.openai.com/v1"), - OPENAI_MODERATION_MODEL: z.string().default("omni-moderation-latest"), - AI_LLM_API_KEY: z.string().optional(), - AI_LLM_BASE_URL: z - .string() - .url() - .default("https://9router.asepharyana.my.id/v1"), - /** Model used for text-only moderation (messages, badword analysis). */ - AI_LLM_MODEL: z.string().default("text"), - /** Model used for image/video moderation (vision-capable model). */ - AI_LLM_VISION_MODEL: z.string().optional(), - /** Max concurrent LLM API calls (default: 5). */ - AI_LLM_MAX_CONCURRENT: z.coerce.number().int().positive().default(5), - /** Maximum image dimension in pixels before resize for vision API (default: 1024). */ - AI_LLM_IMAGE_MAX_DIMENSION: z.coerce - .number() - .int() - .positive() - .default(1024), - /** Maximum messages per text-only moderation batch (default: 20). */ - AI_LLM_TEXT_BATCH_SIZE: z.coerce.number().int().positive().default(20), - /** Timeout in ms for individual media analysis calls (default: 60000). */ - AI_LLM_MEDIA_ANALYSIS_TIMEOUT_MS: z.coerce - .number() - .int() - .positive() - .default(60000), - AI_ANALYSIS_DEBOUNCE_MS: z.coerce.number().positive().default(500), - AI_ANALYSIS_RECOVERY_INTERVAL_MS: z.coerce - .number() - .positive() - .default(15000), - AI_ANALYSIS_ERROR_COOLDOWN_MS: z.coerce.number().positive().default(0), - /** Max messages fetched per conversation batch (token budget is the real constraint). */ - AI_ANALYSIS_MAX_BATCH_SIZE: z.coerce.number().int().positive().default(200), - AI_ANALYSIS_MAX_CONTEXT_TOKENS: z.coerce.number().positive().default(8000), - /** Token budget for target messages specifically (separate from context window). */ - AI_ANALYSIS_MAX_TARGET_TOKENS: z.coerce.number().positive().default(4000), - AI_ANALYSIS_CONTEXT_MESSAGE_LIMIT: z.coerce - .number() - .int() - .positive() - .default(20), - /** - * How long a conversation is considered locked while being processed. - * Must exceed (LLM timeout × max retries) + network overhead. - * LLM client timeout=30s, retries=3 → minimum safe value ≈ 100s. - */ - AI_ANALYSIS_PROCESSING_TIMEOUT_MS: z.coerce - .number() - .positive() - .default(120000), - /** Max concurrent individual-fallback LLM calls (effectively unlimited). */ - AI_ANALYSIS_INDIVIDUAL_MAX_CONCURRENT: z.coerce - .number() - .int() - .positive() - .default(1000), - /** - * How many consecutive individual-fallback errors trigger the individual - * circuit breaker (separate from the batch circuit breaker). - */ - AI_ANALYSIS_INDIVIDUAL_CB_THRESHOLD: z.coerce - .number() - .int() - .positive() - .default(50), - // AI moderation uses the Primary LLM (AI_LLM_*) endpoint only. - // No NVIDIA or Groq fallback. - AUTO_DELETE_FLAGGED_ENABLED: z - .string() - .optional() - .transform((v) => v === "true") - .default(true), - AUTO_DELETE_FLAGGED_DELAY_MS: z.coerce.number().min(0).default(0), - AUTO_DELETE_FLAGGED_DRY_RUN: z - .string() - .optional() - .transform((v) => v === "true") - .default(false), - AUTO_DELETE_MIN_CONFIDENCE: z.coerce.number().min(0).max(1).default(0.5), - AUTO_DELETE_ALLOWED_SEVERITIES: z - .string() - .default("critical,high,medium,low"), - AUTO_DELETE_ALLOWED_CATEGORIES: z.string().default(""), - AUTO_DELETE_EXCLUDED_CHANNEL_IDS: z.string().default(""), - AUTO_DELETE_EXCLUDED_USER_IDS: z.string().default(""), - RETENTION_MESSAGES_DAYS: z.coerce.number().int().min(0).default(0), - RETENTION_ATTACHMENTS_DAYS: z.coerce.number().int().min(0).default(0), - RETENTION_VOICE_DAYS: z.coerce.number().int().min(0).default(0), - RETENTION_CLEANUP_INTERVAL_MS: z.coerce - .number() - .positive() - .default(24 * 60 * 60 * 1000), - RETENTION_DRY_RUN: z - .string() - .optional() - .transform((v) => v === "true") - .default(true), - AUTO_MIGRATE_ON_STARTUP: z - .string() - .optional() - .transform((v) => v === "true") - .default(true), - DATABASE_URL: z.string().optional(), - POSTGRES_HOST: z.string().default("localhost"), - POSTGRES_PORT: z.coerce.number().int().positive().default(5432), - POSTGRES_USER: z.string().optional(), - POSTGRES_PASSWORD: z.string().optional(), - POSTGRES_DB: z.string().optional(), - POSTGRES_POOL_MIN: z.coerce.number().int().positive().default(2), - POSTGRES_POOL_MAX: z.coerce.number().int().positive().default(10), - ADMIN_PASSWORD: z.string().default("admin123"), - REDIS_URL: z.string().min(1).default("redis://localhost:6379"), - }) - .superRefine((value, ctx) => { - if (!value.AI_ANALYSIS_ENABLED) { - // Continue to database validationa - } else if (!value.AI_LLM_API_KEY) { - ctx.addIssue({ - code: z.ZodIssueCode.custom, - path: ["AI_LLM_API_KEY"], - message: "AI_LLM_API_KEY is required when AI_ANALYSIS_ENABLED=true", - }); - } - - // Validate PostgreSQL configuration - if (!value.DATABASE_URL && !value.POSTGRES_HOST) { - ctx.addIssue({ - code: z.ZodIssueCode.custom, - path: ["DATABASE_URL"], - message: "Either DATABASE_URL or POSTGRES_HOST must be provided", - }); - } - }); - -export type AppConfig = z.infer & { - EFFECTIVE_TEXT_GUILD_ID?: string; - EFFECTIVE_VOICE_GUILD_ID?: string; -}; - -export function loadConfig(env: NodeJS.ProcessEnv = process.env): AppConfig { - try { - const parsed = configSchema.parse(env); - return { - ...parsed, - // AI text capture and analytics are pinned to the monitor guild. - EFFECTIVE_TEXT_GUILD_ID: parsed.MONITOR_GUILD_ID, - EFFECTIVE_VOICE_GUILD_ID: parsed.VOICE_GUILD_ID ?? parsed.GUILD_ID, - }; - } catch (error) { - if (error instanceof z.ZodError) { - const messages = error.issues - .map((e) => `${e.path.join(".")}: ${e.message}`) - .join("\n"); - throw new ConfigError(`Configuration validation failed:\n${messages}`); - } - throw error; - } -} - -export const config = loadConfig(); diff --git a/src/database/drizzle.ts b/src/database/drizzle.ts deleted file mode 100644 index 079a719..0000000 --- a/src/database/drizzle.ts +++ /dev/null @@ -1,129 +0,0 @@ -import { drizzle as drizzlePostgres } from "drizzle-orm/node-postgres"; -import { Pool } from "pg"; -import type { PoolClient } from "pg"; -import { config } from "../config.js"; -import { createChildLogger } from "../logger.js"; -import * as schema from "./schema.js"; - -const logger = createChildLogger("drizzle"); - -let db: ReturnType | null = null; -let rawPool: Pool | null = null; - -/** - * Initialize the PostgreSQL database connection. - */ -export async function initializeDatabase() { - if (db !== null) { - return db; - } - - let pool: Pool; - - if (config.DATABASE_URL) { - pool = new Pool({ - connectionString: config.DATABASE_URL, - min: config.POSTGRES_POOL_MIN, - max: config.POSTGRES_POOL_MAX, - }); - } else { - pool = new Pool({ - host: config.POSTGRES_HOST, - port: config.POSTGRES_PORT, - user: config.POSTGRES_USER, - password: config.POSTGRES_PASSWORD, - database: config.POSTGRES_DB, - min: config.POSTGRES_POOL_MIN, - max: config.POSTGRES_POOL_MAX, - }); - } - - rawPool = pool; - db = drizzlePostgres(pool, { schema }); - - try { - (db as { run?: (sql: string) => Promise }).run = (sql: string) => - pool.query(sql); - } catch { - // ignore - } - - logger.info("PostgreSQL database initialized"); - return db; -} - -/** - * Get the initialized database instance. - * Throws if database has not been initialized. - */ -export function getDatabase() { - if (db === null) { - throw new Error( - "Database not initialized. Call initializeDatabase() first.", - ); - } - return db; -} - -function convertPlaceholdersForPostgres(sql: string) { - let i = 0; - return sql.replace(/\?/g, () => `$${++i}`); -} - -export async function executeAll(sql: string, params?: unknown[]) { - if (!rawPool) { - throw new Error( - "Database not initialized. Call initializeDatabase() first.", - ); - } - - const query = convertPlaceholdersForPostgres(sql); - const result = await rawPool.query(query, params || []); - return result.rows; -} - -export async function executeGet(sql: string, params?: unknown[]) { - if (!rawPool) { - throw new Error( - "Database not initialized. Call initializeDatabase() first.", - ); - } - - const query = convertPlaceholdersForPostgres(sql); - const result = await rawPool.query(query, params || []); - return result.rows[0] ?? null; -} - -/** - * Run a function with a dedicated PostgreSQL client from the shared pool. - * Use this for session-scoped operations such as advisory locks. - */ -export async function withDatabaseClient( - callback: (client: PoolClient) => Promise, -): Promise { - if (!rawPool) { - throw new Error( - "Database not initialized. Call initializeDatabase() first.", - ); - } - - const client = await rawPool.connect(); - try { - return await callback(client); - } finally { - client.release(); - } -} - -/** - * Close the PostgreSQL connection pool. - */ -export async function closeDatabase() { - if (rawPool !== null) { - await rawPool.end(); - } - - rawPool = null; - db = null; - logger.info("PostgreSQL database closed"); -} diff --git a/src/database/migrate.ts b/src/database/migrate.ts deleted file mode 100644 index 9ed3473..0000000 --- a/src/database/migrate.ts +++ /dev/null @@ -1,54 +0,0 @@ -import "dotenv/config"; -import { drizzle as drizzlePostgres } from "drizzle-orm/node-postgres"; -import { migrate as migratePostgres } from "drizzle-orm/node-postgres/migrator"; -import { config } from "../config.js"; -import { createChildLogger } from "../logger.js"; -import * as schema from "./schema.js"; -import { - closeDatabase, - initializeDatabase, - withDatabaseClient, -} from "./drizzle.js"; - -const logger = createChildLogger("migrate"); -const MIGRATION_LOCK_KEY_1 = 2026; -const MIGRATION_LOCK_KEY_2 = 531; - -export async function runMigrations(): Promise { - try { - logger.info("Starting PostgreSQL migrations"); - await initializeDatabase(); - - try { - await withDatabaseClient(async (client) => { - const db = drizzlePostgres(client, { schema }); - - await client.query("SELECT pg_advisory_lock($1, $2)", [ - MIGRATION_LOCK_KEY_1, - MIGRATION_LOCK_KEY_2, - ]); - - try { - await migratePostgres(db, { - migrationsFolder: "./drizzle/migrations", - }); - } finally { - await client.query("SELECT pg_advisory_unlock($1, $2)", [ - MIGRATION_LOCK_KEY_1, - MIGRATION_LOCK_KEY_2, - ]); - } - }); - } finally { - await closeDatabase(); - } - - logger.info("PostgreSQL migrations completed successfully"); - } catch (error) { - logger.error( - { error: error instanceof Error ? error.message : String(error) }, - "Migration failed", - ); - throw error; - } -} diff --git a/src/database/migrateCli.ts b/src/database/migrateCli.ts deleted file mode 100644 index 65b8aaf..0000000 --- a/src/database/migrateCli.ts +++ /dev/null @@ -1,14 +0,0 @@ -import { createChildLogger } from "../logger.js"; -import { runMigrations } from "./migrate.js"; - -const logger = createChildLogger("migrate-cli"); - -runMigrations() - .then(() => { - logger.info("Migrations completed"); - process.exit(0); - }) - .catch((error) => { - logger.error({ error }, "Migration failed"); - process.exit(1); - }); diff --git a/src/database/migrations/001_drop_unused_ai_columns.sql b/src/database/migrations/001_drop_unused_ai_columns.sql deleted file mode 100644 index a6b0ebe..0000000 --- a/src/database/migrations/001_drop_unused_ai_columns.sql +++ /dev/null @@ -1,10 +0,0 @@ --- Migration: 001_drop_unused_ai_columns.sql --- Date: 2026-05-30 --- Description: Drop columns that are written but never read from messages table --- - ai_moderation_raw: raw LLM response, never consumed --- - ai_policy_version: hardcoded string, never used for decisions --- - ai_evidence: JSON evidence array, never read after write - -ALTER TABLE messages DROP COLUMN IF EXISTS ai_moderation_raw; -ALTER TABLE messages DROP COLUMN IF EXISTS ai_policy_version; -ALTER TABLE messages DROP COLUMN IF EXISTS ai_evidence; diff --git a/src/database/schema.ts b/src/database/schema.ts deleted file mode 100644 index 970580f..0000000 --- a/src/database/schema.ts +++ /dev/null @@ -1,464 +0,0 @@ -import { - bigint as pgBigint, - boolean as pgBoolean, - foreignKey as pgForeignKey, - index as pgIndex, - integer as pgInteger, - real as pgReal, - pgTable, - text as pgText, -} from "drizzle-orm/pg-core"; - -// PostgreSQL Schema -// ================== - -/** - * Muxer Jobs Table (PostgreSQL) - * Tracks audio post-processing jobs with status and retry logic - */ -export const pgMuxerJobsTable = pgTable( - "muxer_jobs", - { - id: pgText("id").primaryKey(), - data: pgText("data").notNull(), - status: pgText("status", { - enum: ["pending", "processing", "completed", "failed"], - }) - .notNull() - .default("pending"), - attempts: pgInteger("attempts").notNull().default(0), - maxAttempts: pgInteger("maxAttempts").notNull().default(3), - createdAt: pgBigint("createdAt", { mode: "number" }).notNull(), - updatedAt: pgBigint("updatedAt", { mode: "number" }).notNull(), - error: pgText("error"), - }, - (table) => ({ - statusIdx: pgIndex("idx_muxer_jobs_status").on(table.status), - createdAtIdx: pgIndex("idx_muxer_jobs_createdAt").on(table.createdAt), - }), -); - -/** - * Messages Table (PostgreSQL) - * Stores text messages with AI moderation analysis - */ -export const pgMessagesTable = pgTable( - "messages", - { - id: pgText("id").primaryKey(), - guild_id: pgText("guild_id").notNull(), - channel_id: pgText("channel_id").notNull(), - thread_id: pgText("thread_id"), - user_id: pgText("user_id").notNull(), - username: pgText("username").notNull(), - avatar_url: pgText("avatar_url"), - content: pgText("content").notNull(), - edited_content: pgText("edited_content"), - created_at: pgBigint("created_at", { mode: "number" }).notNull(), - edited_at: pgBigint("edited_at", { mode: "number" }), - deleted_at: pgBigint("deleted_at", { mode: "number" }), - type: pgText("type", { enum: ["text", "edited", "deleted"] }) - .notNull() - .default("text"), - metadata: pgText("metadata"), - ai_status: pgText("ai_status", { - enum: ["pending", "clean", "warn", "flagged", "error"], - }) - .notNull() - .default("pending"), - ai_moderation_flags: pgText("ai_moderation_flags"), - ai_moderation_score: pgReal("ai_moderation_score"), - ai_analysis: pgText("ai_analysis"), - ai_categories: pgText("ai_categories"), - ai_severity: pgText("ai_severity", { - enum: ["none", "low", "medium", "high", "critical"], - }), - ai_confidence: pgReal("ai_confidence"), - ai_recommended_action: pgText("ai_recommended_action", { - enum: ["none", "monitor", "warn", "review", "delete", "escalate"], - }), - ai_analyzed_at: pgBigint("ai_analyzed_at", { mode: "number" }), - ai_error: pgText("ai_error"), - }, - (table) => ({ - channelIdx: pgIndex("idx_messages_channel").on(table.channel_id), - userIdx: pgIndex("idx_messages_user").on(table.user_id), - createdIdx: pgIndex("idx_messages_created").on(table.created_at), - threadIdx: pgIndex("idx_messages_thread").on(table.thread_id), - channelCreatedIdx: pgIndex("idx_messages_channel_created").on( - table.channel_id, - table.created_at, - table.id, - ), - threadCreatedIdx: pgIndex("idx_messages_thread_created").on( - table.thread_id, - table.created_at, - table.id, - ), - aiStatusCreatedIdx: pgIndex("idx_messages_ai_status_created").on( - table.ai_status, - table.created_at, - table.id, - ), - guildAiStatusCreatedIdx: pgIndex("idx_messages_guild_ai_status_created").on( - table.guild_id, - table.ai_status, - table.created_at, - table.id, - ), - guildCreatedDeletedIdx: pgIndex("idx_messages_guild_created_deleted").on( - table.guild_id, - table.created_at, - table.deleted_at, - table.id, - ), - channelAiStatusCreatedIdx: pgIndex( - "idx_messages_channel_ai_status_created", - ).on(table.channel_id, table.ai_status, table.created_at, table.id), - threadAiStatusCreatedIdx: pgIndex( - "idx_messages_thread_ai_status_created", - ).on(table.thread_id, table.ai_status, table.created_at, table.id), - }), -); - -/** - * Attachments Table (PostgreSQL) - * Stores attachment metadata with upload status tracking - */ -export const pgAttachmentsTable = pgTable( - "attachments", - { - id: pgText("id").primaryKey(), - message_id: pgText("message_id").notNull(), - guild_id: pgText("guild_id").notNull(), - channel_id: pgText("channel_id").notNull(), - thread_id: pgText("thread_id"), - user_id: pgText("user_id").notNull(), - filename: pgText("filename").notNull(), - size: pgInteger("size").notNull(), - type: pgText("type").notNull(), - discord_url: pgText("discord_url").notNull(), - uploaded_url: pgText("uploaded_url"), - upload_status: pgText("upload_status", { - enum: ["pending", "uploaded", "failed"], - }) - .notNull() - .default("pending"), - upload_error: pgText("upload_error"), - created_at: pgBigint("created_at", { mode: "number" }).notNull(), - uploaded_at: pgBigint("uploaded_at", { mode: "number" }), - }, - (table) => ({ - channelIdx: pgIndex("idx_attachments_channel").on(table.channel_id), - messageIdx: pgIndex("idx_attachments_message").on(table.message_id), - statusIdx: pgIndex("idx_attachments_status").on(table.upload_status), - channelCreatedIdx: pgIndex("idx_attachments_channel_created").on( - table.channel_id, - table.created_at, - table.id, - ), - threadCreatedIdx: pgIndex("idx_attachments_thread_created").on( - table.thread_id, - table.created_at, - table.id, - ), - messageFk: pgForeignKey({ - columns: [table.message_id], - foreignColumns: [pgMessagesTable.id], - name: "fk_attachments_message_id", - }).onDelete("cascade"), - }), -); - -/** - * UI State Table (PostgreSQL) - * Stores persistent UI state (e.g., selected channel, filter preferences) - */ -export const pgUIStateTable = pgTable("ui_state", { - key: pgText("key").primaryKey(), - value: pgText("value").notNull(), - updated_at: pgBigint("updated_at", { mode: "number" }).notNull(), -}); - -/** - * AI Analysis Runs Table (PostgreSQL) - * Tracks AI analysis batch runs for conversation-level moderation - */ -export const pgAIAnalysisRunsTable = pgTable( - "ai_analysis_runs", - { - id: pgText("id").primaryKey(), - conversation_key: pgText("conversation_key").notNull(), - target_message_ids: pgText("target_message_ids").notNull(), // JSON array - model: pgText("model").notNull(), - request_tokens_estimate: pgInteger("request_tokens_estimate"), - response_raw: pgText("response_raw"), - status: pgText("status", { - enum: ["pending", "processing", "completed", "failed"], - }) - .notNull() - .default("pending"), - error: pgText("error"), - created_at: pgBigint("created_at", { mode: "number" }).notNull(), - completed_at: pgBigint("completed_at", { mode: "number" }), - }, - (table) => ({ - conversationKeyIdx: pgIndex("idx_ai_analysis_runs_conversation_key").on( - table.conversation_key, - ), - statusIdx: pgIndex("idx_ai_analysis_runs_status").on(table.status), - createdAtIdx: pgIndex("idx_ai_analysis_runs_created_at").on( - table.created_at, - ), - }), -); - -/** - * Voice Recordings Table (PostgreSQL) - * Stores voice recording segment metadata and upload status - */ -export const pgVoiceRecordingsTable = pgTable( - "voice_recordings", - { - id: pgText("id").primaryKey(), - user_id: pgText("user_id").notNull(), - username: pgText("username").notNull(), - avatar_url: pgText("avatar_url"), - guild_id: pgText("guild_id"), - channel_id: pgText("channel_id"), - channel_name: pgText("channel_name"), - filename: pgText("filename").notNull(), - size_bytes: pgInteger("size_bytes").notNull(), - download_url: pgText("download_url"), - upload_status: pgText("upload_status", { - enum: ["pending", "uploaded", "failed"], - }) - .notNull() - .default("pending"), - upload_error: pgText("upload_error"), - created_at: pgBigint("created_at", { mode: "number" }).notNull(), - uploaded_at: pgBigint("uploaded_at", { mode: "number" }), - }, - (table) => ({ - userIdIdx: pgIndex("idx_voice_recordings_user_id").on(table.user_id), - channelIdIdx: pgIndex("idx_voice_recordings_channel_id").on( - table.channel_id, - ), - createdIdx: pgIndex("idx_voice_recordings_created_at").on(table.created_at), - }), -); - -/** - * Message Reviews Table (PostgreSQL) - * Tracks manual reviews of messages flagged by AI moderation - */ -export const pgMessageReviewsTable = pgTable( - "message_reviews", - { - id: pgText("id").primaryKey(), - message_id: pgText("message_id").notNull(), - guild_id: pgText("guild_id").notNull(), - channel_id: pgText("channel_id").notNull(), - reviewer_id: pgText("reviewer_id"), - status: pgText("status", { - enum: ["pending", "approved", "rejected", "escalated"], - }) - .notNull() - .default("pending"), - notes: pgText("notes"), - created_at: pgBigint("created_at", { mode: "number" }).notNull(), - reviewed_at: pgBigint("reviewed_at", { mode: "number" }), - }, - (table) => ({ - messageIdIdx: pgIndex("idx_message_reviews_message_id").on( - table.message_id, - ), - statusIdx: pgIndex("idx_message_reviews_status").on(table.status), - createdAtIdx: pgIndex("idx_message_reviews_created_at").on( - table.created_at, - ), - guildStatusIdx: pgIndex("idx_message_reviews_guild_status").on( - table.guild_id, - table.status, - table.created_at, - ), - }), -); - -/** - * Moderation Actions Table (PostgreSQL) - * Tracks actions taken on messages (delete, mute, etc.) - */ -export const pgModerationActionsTable = pgTable( - "moderation_actions", - { - id: pgText("id").primaryKey(), - message_id: pgText("message_id"), - user_id: pgText("user_id"), - guild_id: pgText("guild_id").notNull(), - action_type: pgText("action_type", { - enum: [ - "delete_message", - "mute_user", - "warn_user", - "kick_user", - "ban_user", - ], - }).notNull(), - reason: pgText("reason"), - executed_by: pgText("executed_by"), - status: pgText("status", { - enum: ["pending", "executed", "failed"], - }) - .notNull() - .default("pending"), - error: pgText("error"), - created_at: pgBigint("created_at", { mode: "number" }).notNull(), - executed_at: pgBigint("executed_at", { mode: "number" }), - }, - (table) => ({ - messageIdIdx: pgIndex("idx_moderation_actions_message_id").on( - table.message_id, - ), - userIdIdx: pgIndex("idx_moderation_actions_user_id").on(table.user_id), - statusIdx: pgIndex("idx_moderation_actions_status").on(table.status), - guildStatusIdx: pgIndex("idx_moderation_actions_guild_status").on( - table.guild_id, - table.status, - table.created_at, - ), - }), -); - -/** - * Retention Policies Table (PostgreSQL) - * Defines data retention rules per guild/channel - */ -export const pgRetentionPoliciesTable = pgTable( - "retention_policies", - { - id: pgText("id").primaryKey(), - guild_id: pgText("guild_id").notNull(), - channel_id: pgText("channel_id"), - retention_days: pgInteger("retention_days").notNull().default(90), - apply_to_media: pgBoolean("apply_to_media").notNull().default(true), - apply_to_voice: pgBoolean("apply_to_voice").notNull().default(true), - enabled: pgBoolean("enabled").notNull().default(true), - created_at: pgBigint("created_at", { mode: "number" }).notNull(), - updated_at: pgBigint("updated_at", { mode: "number" }).notNull(), - }, - (table) => ({ - guildIdIdx: pgIndex("idx_retention_policies_guild_id").on(table.guild_id), - enabledIdx: pgIndex("idx_retention_policies_enabled").on(table.enabled), - }), -); - -/** - * Text Analysis Cache Table (PostgreSQL) - * Caches per-normalized-text moderation analysis results so repeated - * phrases reuse previously computed API / fallback results instead of - * re-calling expensive LLM or external moderation APIs. - * - * Uses the FULL normalized text (not per-word) because context matters: - * "kau" alone is clean, but "awas kau" can be a threat. - */ -export const pgTextAnalysisCacheTable = pgTable( - "text_analysis_cache", - { - /** Normalized text (lowercase, whitespace-collapsed) — primary key. */ - text: pgText("text").primaryKey(), - /** JSON array of moderation flags detected for this text (e.g. ["vulgar_language","harassment"]). */ - flags: pgText("flags").notNull().default("[]"), - /** Which source produced this result: "local" | "primary_ai" | "vision_llm". */ - source: pgText("source", { - enum: ["local", "primary_ai", "vision_llm"], - }) - .notNull() - .default("local"), - /** Epoch millis when the analysis was stored. */ - analyzed_at: pgBigint("analyzed_at", { mode: "number" }).notNull(), - /** Epoch millis when this cache entry expires. */ - expires_at: pgBigint("expires_at", { mode: "number" }).notNull(), - /** How many times this cached text has been reused. */ - hit_count: pgInteger("hit_count").notNull().default(0), - }, - (table) => ({ - expiresAtIdx: pgIndex("idx_text_analysis_cache_expires_at").on( - table.expires_at, - ), - sourceIdx: pgIndex("idx_text_analysis_cache_source").on(table.source), - }), -); - -/** - * Sticker Cache Table (PostgreSQL) - * Stores base64-encoded sticker images for fast retrieval in media moderation. - * Replaces the file-based .dat + index.json cache. - * - * TTL: 7 days (enforced at query time via fetched_at) - * Eviction: LRU by fetched_at, max 100MB total - */ -export const pgStickerCacheTable = pgTable( - "sticker_cache", - { - /** Sanitized sticker name (encodeURIComponent + %→_) — primary key. */ - name: pgText("name").primaryKey(), - /** Base64-encoded image data. */ - base64: pgText("base64").notNull(), - /** MIME type of the image (e.g. "image/png", "image/gif"). */ - mime_type: pgText("mime_type").notNull(), - /** Byte length of the base64 string (for efficient SUM() eviction queries). */ - size: pgInteger("size").notNull(), - /** Epoch millis when this entry was stored. Used for TTL and LRU eviction. */ - fetched_at: pgBigint("fetched_at", { mode: "number" }).notNull(), - }, - (table) => ({ - fetchedAtIdx: pgIndex("idx_sticker_cache_fetched_at").on(table.fetched_at), - }), -); - -// Runtime table exports -// ===================== - -export const muxerJobsTable = pgMuxerJobsTable; -export const messagesTable = pgMessagesTable; -export const attachmentsTable = pgAttachmentsTable; -export const uiStateTable = pgUIStateTable; -export const aiAnalysisRunsTable = pgAIAnalysisRunsTable; -export const voiceRecordingsTable = pgVoiceRecordingsTable; -export const messageReviewsTable = pgMessageReviewsTable; -export const moderationActionsTable = pgModerationActionsTable; -export const retentionPoliciesTable = pgRetentionPoliciesTable; -export const textAnalysisCacheTable = pgTextAnalysisCacheTable; -export const stickerCacheTable = pgStickerCacheTable; - -// Export table types for use in queries -export type MuxerJob = typeof muxerJobsTable.$inferSelect; -export type MuxerJobInsert = typeof muxerJobsTable.$inferInsert; - -export type Message = typeof messagesTable.$inferSelect; -export type MessageInsert = typeof messagesTable.$inferInsert; - -export type Attachment = typeof attachmentsTable.$inferSelect; -export type AttachmentInsert = typeof attachmentsTable.$inferInsert; - -export type UIState = typeof uiStateTable.$inferSelect; -export type UIStateInsert = typeof uiStateTable.$inferInsert; - -export type AIAnalysisRun = typeof aiAnalysisRunsTable.$inferSelect; -export type AIAnalysisRunInsert = typeof aiAnalysisRunsTable.$inferInsert; - -export type VoiceRecording = typeof voiceRecordingsTable.$inferSelect; -export type VoiceRecordingInsert = typeof voiceRecordingsTable.$inferInsert; - -export type MessageReview = typeof messageReviewsTable.$inferSelect; -export type MessageReviewInsert = typeof messageReviewsTable.$inferInsert; - -export type ModerationAction = typeof moderationActionsTable.$inferSelect; -export type ModerationActionInsert = typeof moderationActionsTable.$inferInsert; - -export type RetentionPolicy = typeof retentionPoliciesTable.$inferSelect; -export type RetentionPolicyInsert = typeof retentionPoliciesTable.$inferInsert; - -export type StickerCacheRecord = typeof stickerCacheTable.$inferSelect; -export type StickerCacheInsert = typeof stickerCacheTable.$inferInsert; diff --git a/src/database/voiceRecordingRepo.ts b/src/database/voiceRecordingRepo.ts deleted file mode 100644 index 6496a01..0000000 --- a/src/database/voiceRecordingRepo.ts +++ /dev/null @@ -1,115 +0,0 @@ -import { desc, eq } from "drizzle-orm"; -import { createChildLogger } from "../logger.js"; -import { getDatabase } from "./drizzle.js"; -import { - type VoiceRecording, - type VoiceRecordingInsert, - voiceRecordingsTable, -} from "./schema.js"; - -const logger = createChildLogger("voice-recording-repo"); - -interface QueryBuilder extends PromiseLike { - from(...args: unknown[]): QueryBuilder; - where(...args: unknown[]): QueryBuilder; - orderBy(...args: unknown[]): QueryBuilder; - limit(...args: unknown[]): QueryBuilder; - offset(...args: unknown[]): QueryBuilder; - values(...args: unknown[]): QueryBuilder; - onConflictDoNothing(...args: unknown[]): QueryBuilder; - returning(...args: unknown[]): QueryBuilder; - set(...args: unknown[]): QueryBuilder; -} - -interface RecordingDatabase { - select(...args: unknown[]): QueryBuilder; - insert(...args: unknown[]): QueryBuilder; - update(...args: unknown[]): QueryBuilder; -} - -function db(): RecordingDatabase { - return getDatabase() as unknown as RecordingDatabase; -} - -export async function insertVoiceRecording( - recording: VoiceRecordingInsert, -): Promise { - try { - await db() - .insert(voiceRecordingsTable) - .values(recording) - .onConflictDoNothing(); - } catch (error) { - logger.error( - { - id: recording.id, - error: error instanceof Error ? error.message : String(error), - }, - "Failed to insert voice recording", - ); - throw error; - } -} - -export async function updateVoiceRecordingAsUploaded( - id: string, - downloadUrl: string, - uploadedAt: number, -): Promise { - try { - await db() - .update(voiceRecordingsTable) - .set({ - download_url: downloadUrl, - upload_status: "uploaded", - uploaded_at: uploadedAt, - }) - .where(eq(voiceRecordingsTable.id, id)); - } catch (error) { - logger.error( - { id, error: error instanceof Error ? error.message : String(error) }, - "Failed to update voice recording status to uploaded", - ); - throw error; - } -} - -export async function updateVoiceRecordingAsFailed( - id: string, - error: string, -): Promise { - try { - await db() - .update(voiceRecordingsTable) - .set({ - upload_status: "failed", - upload_error: error, - }) - .where(eq(voiceRecordingsTable.id, id)); - } catch (error) { - logger.error( - { id, error: error instanceof Error ? error.message : String(error) }, - "Failed to update voice recording status to failed", - ); - throw error; - } -} - -export async function listVoiceRecordings( - limit = 100, -): Promise { - try { - const rows = await db() - .select() - .from(voiceRecordingsTable) - .orderBy(desc(voiceRecordingsTable.created_at)) - .limit(limit); - return rows as VoiceRecording[]; - } catch (error) { - logger.error( - { error: error instanceof Error ? error.message : String(error) }, - "Failed to list voice recordings", - ); - throw error; - } -} diff --git a/src/discordClientOptions.ts b/src/discordClientOptions.ts deleted file mode 100644 index 43be8b4..0000000 --- a/src/discordClientOptions.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { type ClientOptions, Options } from "discord.js-selfbot-v13"; - -export function createDiscordClientOptions(): ClientOptions { - return { - makeCache: Options.cacheWithLimits({ - ...Options.defaultMakeCacheSettings, - MessageManager: 25, - ReactionManager: 0, - ReactionUserManager: 0, - PresenceManager: 0, - }), - partials: ["USER", "CHANNEL", "GUILD_MEMBER", "MESSAGE"], - sweepers: { - messages: { interval: 300, lifetime: 600 }, - threads: { interval: 3600, lifetime: 14400 }, - }, - restRequestTimeout: 15_000, - retryLimit: 2, - restGlobalRateLimit: 45, - }; -} diff --git a/src/errors.ts b/src/errors.ts deleted file mode 100644 index 94c1587..0000000 --- a/src/errors.ts +++ /dev/null @@ -1,43 +0,0 @@ -export class AppError extends Error { - public code: string; - public statusCode: number; - - constructor(message: string, code: string, statusCode: number = 500) { - super(message); - this.code = code; - this.statusCode = statusCode; - this.name = "AppError"; - Error.captureStackTrace(this, this.constructor); - } -} - -export class ConfigError extends AppError { - constructor(message: string) { - super(message, "CONFIG_ERROR", 500); - this.name = "ConfigError"; - } -} - -export class AudioError extends AppError { - constructor(message: string) { - super(message, "AUDIO_ERROR", 500); - this.name = "AudioError"; - } -} - -export class VoiceConnectionError extends AppError { - constructor(message: string) { - super(message, "VOICE_CONNECTION_ERROR", 500); - this.name = "VoiceConnectionError"; - } -} - -export class ValidationError extends AppError { - public details?: Record; - - constructor(message: string, details?: Record) { - super(message, "VALIDATION_ERROR", 400); - this.details = details; - this.name = "ValidationError"; - } -} diff --git a/src/http/app.ts b/src/http/app.ts deleted file mode 100644 index 0836ad4..0000000 --- a/src/http/app.ts +++ /dev/null @@ -1,148 +0,0 @@ -import fs from "node:fs"; -import path from "node:path"; -import type { Client } from "discord.js-selfbot-v13"; -import express, { - type NextFunction, - type Request, - type Response, -} from "express"; -import helmet from "helmet"; -import { AppError } from "../errors.js"; -import type { createChildLogger } from "../logger.js"; -import type { MediaController } from "../media/mediaController.js"; -import type { ModerationBroadcaster } from "../moderation/types.js"; -import { createAnalysisRoutes } from "../routes/analysisRoutes.js"; -import { createAppConfigRoutes } from "../routes/appConfigRoutes.js"; -import { createAnalyticsRoutes } from "../routes/analyticsRoutes.js"; -import { createMediaRoutes } from "../routes/mediaRoutes.js"; -import { createMessageRoutes } from "../routes/messageRoutes.js"; -import { createRecordingsRoutes } from "../routes/recordingsRoutes.js"; -import { createReviewRoutes } from "../routes/reviewRoutes.js"; -import { createSyncRoutes } from "../routes/syncRoutes.js"; -import { createUIStateRoutes } from "../routes/uiStateRoutes.js"; -import { createVoiceRoutes } from "../routes/voiceRoutes.js"; -import type { SharedUIStatePatch } from "../state/uiState.js"; -import type { VoiceController } from "../voiceController.js"; -import { createHealthRoutes } from "./health.js"; - -const publicDir = path.resolve(process.cwd(), "public"); -const reactAppDir = path.join(publicDir, "app"); - -type Logger = ReturnType; - -export interface CreateHttpAppOptions { - client: Client; - voiceController: VoiceController; - mediaController: MediaController; - broadcaster: ModerationBroadcaster; - adminPassword: string; - getSharedUIState: () => any; - patchSharedUIState: (patch: SharedUIStatePatch) => any; - activeUserCount: () => number; - wsClientCount: () => number; - logger: Logger; -} - -export function createHttpApp(options: CreateHttpAppOptions) { - const app = express(); - - app.use( - helmet({ - contentSecurityPolicy: false, - }), - ); - - app.use((req: Request, res: Response, next: NextFunction) => { - if (req.path.startsWith("/api/")) { - res.set("Cache-Control", "no-store"); - } - res.on("finish", () => { - if (req.originalUrl.startsWith("/.well-known/appspecific/")) return; - if (req.originalUrl === "/favicon.ico") return; - if (res.statusCode >= 400) { - options.logger.error( - { - method: req.method, - url: req.originalUrl, - statusCode: res.statusCode, - }, - "HTTP request failed", - ); - } - }); - next(); - }); - app.use(express.json()); - - app.use(express.static(publicDir)); - app.use(express.static(reactAppDir)); - - app.get("/", (_req: Request, res: Response) => { - const reactIndex = path.join(reactAppDir, "index.html"); - if (fs.existsSync(reactIndex)) { - res.sendFile(reactIndex); - return; - } - res - .status(503) - .send("React dashboard is not built. Run pnpm run build:web."); - }); - - // Health and auth routes - app.use( - createHealthRoutes({ - adminPassword: options.adminPassword, - activeUserCount: options.activeUserCount, - wsClientCount: options.wsClientCount, - }), - ); - - // Route modules - app.use( - "/api", - createUIStateRoutes({ - getSharedUIState: options.getSharedUIState, - patchSharedUIState: options.patchSharedUIState, - }), - ); - app.use( - "/api", - createVoiceRoutes({ - voiceController: options.voiceController, - patchSharedUIState: options.patchSharedUIState, - broadcaster: options.broadcaster, - adminPassword: options.adminPassword, - }), - ); - app.use("/api", createMessageRoutes()); - app.use("/api", createAnalysisRoutes()); - app.use("/api", createAppConfigRoutes()); - app.use("/api", createReviewRoutes()); - app.use("/api", createAnalyticsRoutes()); - app.use("/api", createSyncRoutes(options.client)); - app.use("/api", createRecordingsRoutes()); - app.use( - "/api", - createMediaRoutes(options.mediaController, { - adminPassword: options.adminPassword, - }), - ); - - app.use((error: Error, _req: Request, res: Response, _next: NextFunction) => { - if (error instanceof AppError) { - res.status(error.statusCode).json({ - error: error.code, - message: error.message, - }); - return; - } - - options.logger.error({ error }, "Unhandled webserver error"); - res.status(500).json({ - error: "INTERNAL_SERVER_ERROR", - message: "Internal server error", - }); - }); - - return app; -} diff --git a/src/http/health.ts b/src/http/health.ts deleted file mode 100644 index 3b6b516..0000000 --- a/src/http/health.ts +++ /dev/null @@ -1,39 +0,0 @@ -import { Router } from "express"; -import { getMetrics, uptimeGauge } from "../metrics.js"; - -export interface HealthRoutesOptions { - adminPassword: string; - activeUserCount: () => number; - wsClientCount: () => number; -} - -export function createHealthRoutes(options: HealthRoutesOptions) { - const router = Router(); - - router.get("/health", (_req, res) => { - res.json({ - status: "ok", - timestamp: new Date().toISOString(), - uptime: process.uptime(), - activeUsers: options.activeUserCount(), - wsClients: options.wsClientCount(), - }); - }); - - router.get("/metrics", async (_req, res) => { - res.set("Content-Type", "text/plain"); - uptimeGauge.set(process.uptime()); - res.send(await getMetrics()); - }); - - router.post("/api/auth/login", (req, res) => { - const { password } = req.body; - if (password === options.adminPassword) { - res.json({ ok: true }); - return; - } - res.status(401).json({ error: "Invalid password" }); - }); - - return router; -} diff --git a/src/http/server.ts b/src/http/server.ts deleted file mode 100644 index 48d1517..0000000 --- a/src/http/server.ts +++ /dev/null @@ -1,114 +0,0 @@ -import http from "node:http"; -import type { Client } from "discord.js-selfbot-v13"; -import { config } from "../config.js"; -import { createChildLogger } from "../logger.js"; -import { MediaController } from "../media/mediaController.js"; -import { createScreenShareController } from "../media/screenShareController.js"; -import { createBroadcaster } from "../moderation/broadcaster.js"; -import { - initializeMediaSettings, - persistMediaSettings, -} from "../state/mediaSettings.js"; -import { createSharedUIStateStore } from "../state/uiState.js"; -import { Streamer } from "../streaming/index.js"; -import type { VoiceController } from "../voiceController.js"; -import { - exposeActiveUserGlobal, - exposeModerationGlobals, - exposePcmBroadcastGlobal, - exposeVideoBroadcastGlobal, -} from "../ws/broadcastGlobals.js"; -import { startWebSocketServer } from "../ws/server.js"; -import { createHttpApp } from "./app.js"; - -const serverLogger = createChildLogger("webserver"); - -const activeUsers = new Map< - string, - { username: string; avatar: string; speaking: boolean } ->(); - -export async function startWebserver( - port: number = 3000, - client: Client, - voiceController: VoiceController, -) { - const { getSharedUIState, patchSharedUIState } = - await createSharedUIStateStore(); - let mediaSettings = await initializeMediaSettings(); - - const wsPath = "/ws"; - - const broadcaster = createBroadcaster(); - exposeModerationGlobals(broadcaster, config.ADMIN_PASSWORD); - - const streamer = new Streamer(client); - const screenController = createScreenShareController({ - getVoiceStatus: () => voiceController.getStatus(), - streamer, - useTranscoder: true, - onBeforeStreamStart: async () => { - await voiceController.disconnect(); - await new Promise((resolve) => setTimeout(resolve, 1500)); - }, - onAfterStreamEnd: async (guildId: string, channelId: string) => { - const current = voiceController.getStatus(); - if (current.connected && current.activeGuildId === guildId) return; - await voiceController.connect(guildId, channelId); - }, - }); - - const mediaController = new MediaController({ - isVoiceConnected: () => voiceController.getStatus().connected, - isBrowserStreaming: () => getSharedUIState().isStreaming, - screenController, - onStateChange: (state) => broadcaster.mediaState(state), - initialMusicVolume: mediaSettings.musicVolume, - onMusicVolumeChange: async (volume) => { - mediaSettings = { ...mediaSettings, musicVolume: volume }; - await persistMediaSettings(mediaSettings); - }, - }); - - const app = createHttpApp({ - client, - voiceController, - mediaController, - broadcaster, - adminPassword: config.ADMIN_PASSWORD, - getSharedUIState, - patchSharedUIState, - activeUserCount: () => activeUsers.size, - wsClientCount: () => broadcaster.clientCount(), - logger: serverLogger, - }); - - const server = http.createServer(app); - - function broadcastUserState() { - const users = Array.from(activeUsers.entries()).map(([id, data]) => ({ - id, - ...data, - })); - broadcaster.userState(users); - } - - exposePcmBroadcastGlobal(broadcaster); - exposeVideoBroadcastGlobal(() => broadcaster.getClients(), serverLogger); - exposeActiveUserGlobal(activeUsers, broadcastUserState); - - startWebSocketServer({ - server, - port, - wsPath, - broadcaster, - activeUsers, - getSharedUIState, - mediaController, - logger: serverLogger, - }); - - server.listen(port, "0.0.0.0", () => { - serverLogger.info({ port }, "Web interface listening"); - }); -} diff --git a/src/index.ts b/src/index.ts deleted file mode 100644 index b8e02a6..0000000 --- a/src/index.ts +++ /dev/null @@ -1,13 +0,0 @@ -import "./mock-crc"; -import "libsodium-wrappers"; -import "@snazzah/davey"; -import "dotenv/config"; -import { initializeApp } from "./app/bootstrap.js"; -import { createChildLogger } from "./logger.js"; - -const logger = createChildLogger("bot"); -// Initialize the app -initializeApp().catch((error) => { - logger.error({ error }, "Failed to initialize app"); - process.exit(1); -}); diff --git a/src/logger.ts b/src/logger.ts deleted file mode 100644 index f7e8733..0000000 --- a/src/logger.ts +++ /dev/null @@ -1,132 +0,0 @@ -import fs from "node:fs"; -import path from "node:path"; -import winston from "winston"; -import { formatLogMetadata, serializeLogValue } from "./loggerSerialization.js"; - -const isDev = process.env.NODE_ENV !== "production"; -const logLevel = process.env.LOG_LEVEL || (isDev ? "debug" : "info"); -const logsDir = path.resolve(process.cwd(), "logs"); - -fs.mkdirSync(logsDir, { recursive: true }); - -const metadataFormat = winston.format((info) => { - const { - level: _level, - message: _message, - timestamp: _timestamp, - ...metadata - } = info; - - for (const key of Object.keys(metadata)) { - delete info[key]; - } - - Object.assign(info, formatLogMetadata(metadata)); - return info; -}); - -const consoleFormat = winston.format.printf((info) => { - const { level, message, timestamp, context, ...metadata } = info; - const contextLabel = context ? ` [${String(context)}]` : ""; - const metadataText = Object.keys(metadata).length - ? ` ${JSON.stringify(formatLogMetadata(metadata))}` - : ""; - - return `${timestamp} ${level}${contextLabel}: ${message}${metadataText}`; -}); - -export interface CustomLogger { - error: (msgOrObj: any, msgOrArgs?: any, ...args: any[]) => void; - warn: (msgOrObj: any, msgOrArgs?: any, ...args: any[]) => void; - info: (msgOrObj: any, msgOrArgs?: any, ...args: any[]) => void; - debug: (msgOrObj: any, msgOrArgs?: any, ...args: any[]) => void; - trace: (msgOrObj: any, msgOrArgs?: any, ...args: any[]) => void; - fatal: (msgOrObj: any, msgOrArgs?: any, ...args: any[]) => void; - silent: (msgOrObj: any, msgOrArgs?: any, ...args: any[]) => void; - child(options: { context: string } & Record): CustomLogger; - [key: string]: any; -} - -const winstonLogger = winston.createLogger({ - level: logLevel, - levels: winston.config.npm.levels, - format: winston.format.combine( - winston.format.timestamp(), - winston.format.errors({ stack: true }), - metadataFormat(), - ), - transports: [ - new winston.transports.Console({ - format: winston.format.combine( - winston.format.colorize(), - winston.format.timestamp(), - metadataFormat(), - consoleFormat, - ), - }), - new winston.transports.File({ - filename: path.join(logsDir, "app.log"), - format: winston.format.json(), - }), - new winston.transports.File({ - filename: path.join(logsDir, "error.log"), - level: "error", - format: winston.format.json(), - }), - ], -}); - -function wrapLogger(wLogger: winston.Logger): CustomLogger { - const logAtLevel = (level: string) => { - return (arg1: any, arg2?: any) => { - if (arg1 instanceof Error) { - wLogger.log(level, arg1.message, { error: arg1 }); - } else if (typeof arg1 === "object" && arg1 !== null) { - const message = typeof arg2 === "string" ? arg2 : ""; - wLogger.log(level, message, { ...arg1 }); - } else { - const message = typeof arg1 === "string" ? arg1 : String(arg1); - const metadata = typeof arg2 === "object" && arg2 !== null ? arg2 : {}; - wLogger.log(level, message, metadata); - } - }; - }; - - const wrapped: CustomLogger = { - error: logAtLevel("error"), - warn: logAtLevel("warn"), - info: logAtLevel("info"), - debug: logAtLevel("debug"), - trace: logAtLevel("debug"), - fatal: logAtLevel("error"), - silent: () => {}, - child: (options: any) => { - const childWinston = wLogger.child(options); - return wrapLogger(childWinston); - }, - }; - - const proxy = new Proxy(wrapped, { - get(target, prop) { - if (prop in target) { - return (target as any)[prop]; - } - const val = (wLogger as any)[prop]; - if (typeof val === "function") { - return val.bind(wLogger); - } - return val; - }, - }); - - return proxy; -} - -export const logger: CustomLogger = wrapLogger(winstonLogger); - -export const createChildLogger = (context: string): CustomLogger => { - return logger.child({ context }); -}; - -export const serializeLogValueForTest = serializeLogValue; -export const formatLogMetadataForTest = formatLogMetadata; diff --git a/src/loggerSerialization.ts b/src/loggerSerialization.ts deleted file mode 100644 index 3df8857..0000000 --- a/src/loggerSerialization.ts +++ /dev/null @@ -1,110 +0,0 @@ -export type LogMetadata = Record; - -type SerializedError = { - name: string; - message: string; - stack?: string; - code?: unknown; - statusCode?: unknown; -} & Record; - -const serializeError = (error: Error): SerializedError => { - const serialized: SerializedError = { - name: error.name, - message: error.message, - }; - - if (error.stack) { - serialized.stack = error.stack; - } - - const errorWithFields = error as Error & { - code?: unknown; - statusCode?: unknown; - [key: string]: unknown; - }; - - if (errorWithFields.code !== undefined) { - serialized.code = errorWithFields.code; - } - - if (errorWithFields.statusCode !== undefined) { - serialized.statusCode = errorWithFields.statusCode; - } - - for (const [key, value] of Object.entries(errorWithFields)) { - if (serialized[key] === undefined) { - serialized[key] = value; - } - } - - return serialized; -}; - -const isPlainObject = (value: unknown): value is Record => { - if (!value || typeof value !== "object") { - return false; - } - - const prototype = Object.getPrototypeOf(value); - return prototype === Object.prototype || prototype === null; -}; - -export const serializeLogValue = ( - value: unknown, - _seen: WeakSet = new WeakSet(), -): unknown => { - if (value === null || value === undefined) return value; - - if (value instanceof Error) { - return serializeError(value); - } - - if (value instanceof Date) { - return value.toISOString(); - } - - if (value instanceof RegExp) { - return value.toString(); - } - - if (typeof value === "object") { - if (_seen.has(value as object)) { - return "[Circular]"; - } - _seen.add(value as object); - } - - if (Array.isArray(value)) { - return value.map((item) => serializeLogValue(item, _seen)); - } - - if (isPlainObject(value)) { - return Object.fromEntries( - Object.entries(value).map(([key, nestedValue]) => [ - key, - serializeLogValue(nestedValue, _seen), - ]), - ); - } - - // Non-plain objects (ClientRequest, IncomingMessage, etc.) — serialize as safe string - if (typeof value === "object") { - try { - return `[Object ${(value as any)?.constructor?.name ?? "unknown"}]`; - } catch { - return "[Object]"; - } - } - - return value; -}; - -export const formatLogMetadata = (metadata: LogMetadata): LogMetadata => { - return Object.fromEntries( - Object.entries(metadata).map(([key, value]) => [ - key, - serializeLogValue(value), - ]), - ); -}; diff --git a/src/media/mediaController.ts b/src/media/mediaController.ts deleted file mode 100644 index f6534f5..0000000 --- a/src/media/mediaController.ts +++ /dev/null @@ -1,280 +0,0 @@ -import { AppError } from "../errors.js"; -import { createChildLogger } from "../logger.js"; -import { discordPlayer } from "../player.js"; -import { MediaQueue } from "./mediaQueue.js"; -import { resolveMediaSource } from "./mediaResolver.js"; -import type { - MediaMode, - MediaState, - MusicPlayback, - MusicPlayer, - QueueMediaOptions, - ResolvedMediaSource, - ScreenShareController, - ScreenSharePlayback, -} from "./mediaTypes.js"; -import { createMusicPlayer } from "./musicPlayer.js"; - -const logger = createChildLogger("mediaController"); - -export interface MediaControllerDependencies { - isVoiceConnected?: () => boolean; - isBrowserStreaming?: () => boolean; - resolveMediaSource?: ( - source: string, - mode?: MediaMode, - ) => Promise; - musicPlayer?: MusicPlayer; - screenController?: ScreenShareController; - onStateChange?: (state: MediaState) => void; - initialMusicVolume?: number; - onMusicVolumeChange?: (volume: number) => void | Promise; - setMusicVolume?: (volume: number) => void; -} - -export class MediaController { - private readonly queueStore = new MediaQueue(); - private readonly musicPlayer: MusicPlayer; - private playback: MusicPlayback | null = null; - private playbackToken = 0; - private skipInProgress = false; - private screenPlayback: ScreenSharePlayback | null = null; - private activeMode: MediaMode | null = null; - private musicVolume: number; - private readonly setPlayerMusicVolume: (volume: number) => void; - - constructor(private readonly dependencies: MediaControllerDependencies = {}) { - this.musicPlayer = dependencies.musicPlayer ?? createMusicPlayer(); - this.setPlayerMusicVolume = - dependencies.setMusicVolume ?? - ((volume) => { - discordPlayer.setMusicVolume(volume); - }); - this.musicVolume = normalizeVolume(dependencies.initialMusicVolume, 1); - this.setPlayerMusicVolume(this.musicVolume); - } - - getState(): MediaState { - const snapshot = this.queueStore.snapshot(); - return { - playing: - this.activeMode === "screen" || snapshot.current?.status === "playing", - activeMode: this.activeMode ?? snapshot.current?.mode ?? null, - musicVolume: this.musicVolume, - ...snapshot, - }; - } - - async setMusicVolume(volume: number): Promise { - const nextVolume = normalizeVolume(volume, this.musicVolume); - if (this.musicVolume === nextVolume) return this.emitState(); - this.musicVolume = nextVolume; - this.setPlayerMusicVolume(nextVolume); - await this.dependencies.onMusicVolumeChange?.(nextVolume); - return this.emitState(); - } - - async queue( - source: string, - options: QueueMediaOptions = {}, - ): Promise { - const mode = options.mode ?? "music"; - - logger.info({ source: source.slice(0, 100), mode }, "Queuing media"); - const resolved = await ( - this.dependencies.resolveMediaSource ?? resolveMediaSource - )(source, mode); - logger.info( - { title: resolved.title, kind: resolved.kind }, - "Media resolved", - ); - - if (mode === "screen") { - // Stop current music if any - this.playbackToken++; - this.playback?.stop(); - this.playback = null; - this.assertCanStartMusic(); - this.queueStore.clear(); - this.queueStore.add(resolved, mode, options.requestedBy); - this.queueStore.startNext(); - logger.info({ title: resolved.title }, "Starting screen share"); - return this.startScreen(resolved.source); - } - - // mode === "music" - // If a screen share is active outside of this controller (browser-owned), - // reject to avoid stealing the shared player. If this controller started - // the screenPlayback, stop it and proceed. - if (this.screenPlayback || this.dependencies.screenController?.isActive()) { - if ( - this.dependencies.screenController?.isActive() && - !this.screenPlayback - ) { - throw new AppError("Another media mode is active", "MEDIA_BUSY", 409); - } - this.screenPlayback?.stop(); - this.screenPlayback = null; - this.activeMode = null; - } - - this.assertCanStartMusic(); - this.queueStore.add(resolved, mode, options.requestedBy); - logger.info( - { - title: resolved.title, - queueSize: this.queueStore.snapshot().queue.length, - }, - "Added to queue", - ); - this.startNextIfIdle(); - return this.emitState(); - } - - async skip(): Promise { - if (this.skipInProgress) { - throw new AppError( - "Skip already in progress", - "MEDIA_SKIP_IN_PROGRESS", - 409, - ); - } - - this.skipInProgress = true; - try { - this.playbackToken++; - this.playback?.stop(); - this.playback = null; - this.queueStore.completeCurrent(); - this.startNextIfIdle(); - return this.emitState(); - } finally { - this.skipInProgress = false; - } - } - - async stop(): Promise { - this.playbackToken++; - this.playback?.stop(); - this.playback = null; - this.screenPlayback?.stop(); - this.screenPlayback = null; - this.activeMode = null; - this.queueStore.clear(); - return this.emitState(); - } - - private assertCanStartMusic(): void { - const isVoiceConnected = - this.dependencies.isVoiceConnected ?? (() => discordPlayer.isConnected()); - if (!isVoiceConnected()) { - throw new AppError( - "Connect to a voice channel before playing media", - "VOICE_NOT_CONNECTED", - 409, - ); - } - - if (this.dependencies.isBrowserStreaming?.()) { - throw new AppError( - "Stop browser microphone streaming before playing media", - "BROWSER_STREAM_ACTIVE", - 409, - ); - } - } - - private async startScreen(source: string): Promise { - const screenController = this.dependencies.screenController; - if (!screenController) { - throw new AppError( - "Screen sharing is unavailable", - "SCREEN_UNAVAILABLE", - 500, - ); - } - - this.activeMode = "screen"; - try { - this.screenPlayback = await screenController.start(source); - } catch (error) { - this.activeMode = null; - this.queueStore.failCurrent(); - throw error; - } - - this.screenPlayback.done.then( - () => this.finishScreen(), - () => this.finishScreen(), - ); - return this.emitState(); - } - - private finishScreen(): void { - if (!this.screenPlayback || this.activeMode !== "screen") return; - this.screenPlayback = null; - this.activeMode = null; - this.queueStore.completeCurrent(); - this.emitState(); - } - - private startNextIfIdle(): void { - if (this.playback) return; - const item = this.queueStore.startNext(); - if (!item) { - logger.debug("Queue empty, no playback started"); - return; - } - - const token = ++this.playbackToken; - logger.info( - { - title: item.title, - token, - queueSize: this.queueStore.snapshot().queue.length, - }, - "Starting playback", - ); - try { - this.playback = this.musicPlayer.play(item); - } catch (err) { - logger.error({ err }, "Failed to start playback"); - this.queueStore.failCurrent(); - this.playback = null; - this.startNextIfIdle(); - this.emitState(); - return; - } - - this.playback.done.then( - () => this.finishCurrent(token, false), - (err) => { - logger.error({ err, token }, "Playback failed"); - this.finishCurrent(token, true); - }, - ); - } - - private finishCurrent(token: number, failed: boolean): void { - if (token !== this.playbackToken) return; - this.playback = null; - if (failed) { - this.queueStore.failCurrent(); - } else { - this.queueStore.completeCurrent(); - } - this.startNextIfIdle(); - this.emitState(); - } - - private emitState(): MediaState { - const state = this.getState(); - this.dependencies.onStateChange?.(state); - return state; - } -} - -function normalizeVolume(value: number | undefined, fallback: number): number { - if (!Number.isFinite(value)) return fallback; - return Math.max(0, Math.min(1, value as number)); -} diff --git a/src/media/mediaQueue.ts b/src/media/mediaQueue.ts deleted file mode 100644 index 55c5fde..0000000 --- a/src/media/mediaQueue.ts +++ /dev/null @@ -1,64 +0,0 @@ -import type { - MediaMode, - MediaQueueItem, - MediaState, - ResolvedMediaSource, -} from "./mediaTypes.js"; - -export class MediaQueue { - private current: MediaQueueItem | null = null; - private readonly items: MediaQueueItem[] = []; - - constructor( - private readonly createId: () => string = () => crypto.randomUUID(), - private readonly now = () => Date.now(), - ) {} - - add( - source: ResolvedMediaSource, - mode: MediaQueueItem["mode"] = "music", - requestedBy = "dashboard", - ): MediaQueueItem { - const item: MediaQueueItem = { - id: this.createId(), - mode, - requestedBy, - addedAt: this.now(), - status: "queued", - ...source, - }; - this.items.push(item); - return { ...item }; - } - - startNext(): MediaQueueItem | null { - if (this.current) return { ...this.current }; - const next = this.items.shift(); - if (!next) return null; - this.current = { ...next, status: "playing" }; - return { ...this.current }; - } - - completeCurrent(): void { - this.current = null; - } - - failCurrent(): MediaQueueItem | null { - if (!this.current) return null; - const failed = { ...this.current, status: "failed" as const }; - this.current = null; - return failed; - } - - clear(): void { - this.current = null; - this.items.length = 0; - } - - snapshot(): Pick { - return { - current: this.current ? { ...this.current } : null, - queue: this.items.map((item) => ({ ...item })), - }; - } -} diff --git a/src/media/mediaResolver.ts b/src/media/mediaResolver.ts deleted file mode 100644 index 40d569e..0000000 --- a/src/media/mediaResolver.ts +++ /dev/null @@ -1,126 +0,0 @@ -import { existsSync, statSync } from "node:fs"; -import path from "node:path"; -import { AppError } from "../errors.js"; -import type { MediaMode, ResolvedMediaSource } from "./mediaTypes.js"; -import { createPlayDlResolver } from "./playDlResolver.js"; -import { createYtDlp, type YtDlpClient } from "./ytdlp.js"; - -type PlayDlResolver = ReturnType; - -export interface MediaResolverDependencies { - ytdlp?: YtDlpClient; - playDlResolver?: PlayDlResolver; -} - -export function createMediaResolver( - dependencies: MediaResolverDependencies = {}, -) { - const ytdlp = dependencies.ytdlp ?? createYtDlp(); - const playDlResolver = dependencies.playDlResolver ?? createPlayDlResolver(); - - return async function resolve( - input: string, - mode: MediaMode = "music", - ): Promise { - const source = input.trim(); - if (!source) { - throw new AppError( - "Media source is required", - "MISSING_MEDIA_SOURCE", - 400, - ); - } - - const url = parseUrl(source); - if (url && isYouTubeUrl(url)) { - const metadata = await ytdlp.getMetadata(source); - const directUrl = - mode === "screen" - ? await ytdlp.getDirectVideoUrl(source) - : await ytdlp.getDirectAudioUrl(source); - return { source: directUrl, title: metadata.title, kind: "youtube" }; - } - - if (url && isSpotifyTrackUrl(url)) { - const result = await playDlResolver.resolveSpotifyTrack(source); - const directUrl = - mode === "screen" - ? await ytdlp.getDirectVideoUrl(result.url) - : await ytdlp.getDirectAudioUrl(result.url); - return { source: directUrl, title: result.title, kind: "spotify" }; - } - - const urlSource = resolveUrlSource(source); - if (urlSource) return urlSource; - - const localPath = path.resolve(source); - if (existsSync(localPath) && statSync(localPath).isFile()) { - return { - source: localPath, - title: path.basename(localPath), - kind: "local", - }; - } - - if (!url && !looksLikeUrl(source)) { - const result = await playDlResolver.searchYouTube(source); - const directUrl = - mode === "screen" - ? await ytdlp.getDirectVideoUrl(result.url) - : await ytdlp.getDirectAudioUrl(result.url); - return { source: directUrl, title: result.title, kind: "search" }; - } - - throw new AppError( - "Media source must be an HTTP(S) URL, YouTube URL, Spotify track URL, search query, or existing local file", - "UNSUPPORTED_MEDIA_SOURCE", - 400, - ); - }; -} - -export const resolveMediaSource = createMediaResolver(); - -function parseUrl(source: string): URL | null { - try { - return new URL(source); - } catch { - return null; - } -} - -function looksLikeUrl(source: string): boolean { - return /^[a-z][a-z\d+.-]*:/i.test(source); -} - -function isYouTubeUrl(url: URL): boolean { - return [ - "youtube.com", - "www.youtube.com", - "m.youtube.com", - "youtu.be", - ].includes(url.hostname); -} - -function isSpotifyTrackUrl(url: URL): boolean { - return ( - url.hostname === "open.spotify.com" && url.pathname.startsWith("/track/") - ); -} - -function resolveUrlSource(source: string): ResolvedMediaSource | null { - const url = parseUrl(source); - if (!url) return null; - if (url.protocol !== "http:" && url.protocol !== "https:") return null; - - return { - source, - title: titleFromUrl(url), - kind: "url", - }; -} - -function titleFromUrl(url: URL): string { - const filename = decodeURIComponent(url.pathname.split("/").pop() || ""); - return path.basename(filename) || url.hostname; -} diff --git a/src/media/mediaTypes.ts b/src/media/mediaTypes.ts deleted file mode 100644 index 0694522..0000000 --- a/src/media/mediaTypes.ts +++ /dev/null @@ -1,80 +0,0 @@ -import type { Readable } from "node:stream"; -import type { StreamType } from "@discordjs/voice"; - -export type MediaMode = "music" | "screen"; -export type MediaSourceKind = - | "url" - | "local" - | "youtube" - | "spotify" - | "search"; -export type MediaQueueItemStatus = "queued" | "playing" | "failed"; - -export interface ResolvedMediaSource { - source: string; - title: string; - kind: MediaSourceKind; -} - -export interface MediaQueueItem extends ResolvedMediaSource { - id: string; - mode: MediaMode; - requestedBy: string; - addedAt: number; - status: MediaQueueItemStatus; -} - -export interface MediaState { - playing: boolean; - activeMode: MediaMode | null; - musicVolume: number; - current: MediaQueueItem | null; - queue: MediaQueueItem[]; -} - -export interface QueueMediaOptions { - mode?: MediaMode; - requestedBy?: string; -} - -export interface MusicPlayback { - done: Promise; - stop(): void; -} - -export interface MusicPlayer { - play(source: ResolvedMediaSource): MusicPlayback; -} - -export interface ScreenSharePlayback { - done: Promise; - stop(): void; -} - -export interface ScreenShareController { - isActive(): boolean; - start(source: string): Promise; -} - -export type DiscordPlayerOwner = "none" | "browser-bridge" | "music" | "screen"; - -export interface DiscordPlayOptions { - inputType?: StreamType; - inlineVolume?: boolean; - volume?: number; -} - -export interface DiscordAudioPlayer { - getOwner(): DiscordPlayerOwner; - isConnected(): boolean; - playStream( - stream: Readable, - owner: DiscordPlayerOwner, - options?: DiscordPlayOptions, - ): void; - pause(owner?: DiscordPlayerOwner): void; - unpause(owner?: DiscordPlayerOwner): boolean; - stop(owner?: DiscordPlayerOwner): void; - getMusicVolume(): number; - setMusicVolume(volume: number): void; -} diff --git a/src/media/musicPlayer.ts b/src/media/musicPlayer.ts deleted file mode 100644 index 55a521e..0000000 --- a/src/media/musicPlayer.ts +++ /dev/null @@ -1,128 +0,0 @@ -import type { ChildProcessWithoutNullStreams } from "node:child_process"; -import { spawn as nodeSpawn } from "node:child_process"; -import { StreamType } from "@discordjs/voice"; -import { discordPlayer } from "../player.js"; -import type { - DiscordAudioPlayer, - MusicPlayback, - MusicPlayer, - ResolvedMediaSource, -} from "./mediaTypes.js"; - -export interface MusicPlayerDependencies { - spawn?: typeof nodeSpawn; - discordPlayer?: DiscordAudioPlayer; -} - -export function createMusicPlayer( - dependencies: MusicPlayerDependencies = {}, -): MusicPlayer { - const spawn = dependencies.spawn ?? nodeSpawn; - const audioPlayer = dependencies.discordPlayer ?? discordPlayer; - - return { - play(source: ResolvedMediaSource): MusicPlayback { - if (!audioPlayer.isConnected()) { - throw new Error("Discord audio player is not connected"); - } - - const proc = spawn("ffmpeg", buildFfmpegArgs(source.source), { - stdio: ["ignore", "pipe", "pipe"], - }) as unknown as ChildProcessWithoutNullStreams; - - let stderrOutput = ""; - proc.stderr.on("data", (chunk) => { - stderrOutput += chunk.toString(); - const line = chunk.toString().trim(); - if (line && !line.includes("frame=")) { - console.log("[musicPlayer] ffmpeg:", line); - } - }); - - audioPlayer.playStream(proc.stdout, "music", { - inputType: StreamType.Raw, - inlineVolume: true, - }); - - let stopped = false; - let released = false; - const release = () => { - if (released) return; - released = true; - audioPlayer.stop("music"); - }; - - const done = new Promise((resolve, reject) => { - proc.on("error", (error) => { - console.error("[musicPlayer] Process error:", error); - release(); - reject(error); - }); - proc.stdout.on("error", (error) => { - console.error("[musicPlayer] Stdout error:", error); - release(); - reject(error); - }); - proc.on("close", (code) => { - release(); - if (code === 0 || stopped) { - resolve(); - return; - } - const errorMsg = `ffmpeg exited with code ${code}`; - console.error("[musicPlayer]", errorMsg); - if (stderrOutput) { - console.error( - "[musicPlayer] ffmpeg stderr:", - stderrOutput.slice(-500), - ); - } - reject(new Error(errorMsg)); - }); - }); - - return { - done, - stop() { - if (stopped) return; - stopped = true; - proc.kill("SIGTERM"); - release(); - }, - }; - }, - }; -} - -export function buildFfmpegArgs(source: string): string[] { - const args = ["-hide_banner", "-loglevel", "warning"]; - - if (source.startsWith("http://") || source.startsWith("https://")) { - args.push( - "-user_agent", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36", - ); - } - - args.push( - "-i", - source, - "-vn", - "-acodec", - "pcm_s16le", - "-ar", - "48000", - "-ac", - "2", - "-f", - "s16le", - "pipe:1", - ); - - console.log( - "[ffmpeg] Command:", - "ffmpeg", - args.join(" ").slice(0, 200) + "...", - ); - return args; -} diff --git a/src/media/playDlResolver.ts b/src/media/playDlResolver.ts deleted file mode 100644 index 719f938..0000000 --- a/src/media/playDlResolver.ts +++ /dev/null @@ -1,60 +0,0 @@ -import play from "play-dl"; - -export interface PlayDlResult { - title: string; - url: string; -} - -interface PlayDlSearchResult { - title?: string; - url?: string; -} - -interface SpotifyTrackLike { - type?: string; - name?: string; - artists?: Array<{ name?: string }>; -} - -type SearchFunction = ( - query: string, - options: { limit: number }, -) => Promise; - -type SpotifyFunction = (url: string) => Promise; - -export interface PlayDlDependencies { - search?: SearchFunction; - spotify?: SpotifyFunction; -} - -export function createPlayDlResolver(dependencies: PlayDlDependencies = {}) { - const search: SearchFunction = dependencies.search ?? play.search; - const spotify: SpotifyFunction = - dependencies.spotify ?? (play.spotify as SpotifyFunction); - - return { - async searchYouTube(query: string): Promise { - const results = await search(query, { limit: 1 }); - const first = results[0]; - if (!first?.url) throw new Error(`No YouTube result found for ${query}`); - return { - title: first.title || query, - url: first.url, - }; - }, - - async resolveSpotifyTrack(url: string): Promise { - const track = await spotify(url); - if (track.type !== "track") { - throw new Error("Only Spotify track URLs are supported"); - } - const artists = (track.artists || []) - .map((artist) => artist.name) - .filter(Boolean) - .join(" "); - const query = `${artists} ${track.name || ""} audio`.trim(); - return this.searchYouTube(query); - }, - }; -} diff --git a/src/media/screenShareController.ts b/src/media/screenShareController.ts deleted file mode 100644 index c399748..0000000 --- a/src/media/screenShareController.ts +++ /dev/null @@ -1,166 +0,0 @@ -import { AppError } from "../errors.js"; -import { createChildLogger } from "../logger.js"; -import { discordPlayer } from "../player.js"; -import { - playPreparedStream, - playTranscodedPreparedStream, - Streamer, -} from "../streaming/index.js"; - -const logger = createChildLogger("screen-share"); - -import type { DiscordPlayerOwner, ScreenSharePlayback } from "./mediaTypes.js"; -import { createYtDlp } from "./ytdlp.js"; - -export interface ScreenShareVoiceStatus { - connected: boolean; - activeGuildId: string | null; - activeChannelId: string | null; -} - -export interface ScreenShareControllerDependencies { - getVoiceStatus: () => ScreenShareVoiceStatus; - getPlayerOwner?: () => DiscordPlayerOwner; - getDirectVideoUrl?: (source: string) => Promise; - streamer: Streamer; - useTranscoder?: boolean; - onBeforeStreamStart?: ( - guildId: string, - channelId: string, - ) => Promise | void; - onAfterStreamEnd?: ( - guildId: string, - channelId: string, - ) => Promise | void; - onStreamStart?: () => void; - onStreamEnd?: () => void; -} - -export function createScreenShareController( - dependencies: ScreenShareControllerDependencies, -) { - let active: ScreenSharePlayback | null = null; - const ytdlp = createYtDlp(); - const getPlayerOwner = - dependencies.getPlayerOwner ?? (() => discordPlayer.getOwner()); - const getDirectVideoUrl = - dependencies.getDirectVideoUrl ?? - ((source) => ytdlp.getDirectVideoUrl(source)); - - return { - isActive(): boolean { - return active !== null; - }, - - async start(source: string): Promise { - const status = dependencies.getVoiceStatus(); - let voiceReleased = false; - let voiceRestored = false; - - const restoreVoice = async () => { - if (voiceRestored || !voiceReleased || !guildId || !channelId) return; - voiceRestored = true; - await dependencies.onAfterStreamEnd?.(guildId, channelId); - }; - - if (active) { - active.stop(); - } - - // Ensure bot is in the voice channel and owns the screen-share stream - if ( - !status.connected || - !status.activeGuildId || - !status.activeChannelId - ) { - throw new AppError( - "Connect to a voice channel before sharing screen", - "VOICE_NOT_CONNECTED", - 409, - ); - } - - const guildId = status.activeGuildId; - const channelId = status.activeChannelId; - - // If another media owner (e.g. music) holds the shared player, reject - const owner = getPlayerOwner(); - if (owner === "music") { - throw new AppError("Another media mode is active", "MEDIA_BUSY", 409); - } - - try { - const directUrl = await getDirectVideoUrl(source); - logger.info( - { - guildId, - channelId, - }, - "Creating screen share session", - ); - await dependencies.onBeforeStreamStart?.(guildId, channelId); - voiceReleased = true; - const session = await dependencies.streamer.createSession( - guildId, - channelId, - ); - - dependencies.onStreamStart?.(); - - let stopped = false; - const playFn = dependencies.useTranscoder - ? playTranscodedPreparedStream - : playPreparedStream; - - const done = playFn(directUrl, session, { - fps: 30, - bitrate: 2500, - includeAudio: true, - presetH26x: "superfast", - }).finally(() => { - active = null; - dependencies.onStreamEnd?.(); - return restoreVoice(); - }); - done.catch(() => undefined); - logger.info( - { - guildId, - channelId, - }, - "Screen share session started", - ); - - active = { - done, - stop() { - if (stopped) return; - stopped = true; - session.stop(); - active = null; - void restoreVoice(); - }, - }; - return active; - } catch (error) { - active = null; - if (voiceReleased) { - await restoreVoice(); - } - logger.error( - { - error, - guildId, - channelId, - }, - "Screen share startup failed", - ); - throw new AppError( - error instanceof Error ? error.message : "Screen stream failed", - "SCREEN_STREAM_FAILED", - 500, - ); - } - }, - }; -} diff --git a/src/media/ytdlp.ts b/src/media/ytdlp.ts deleted file mode 100644 index a25cc7f..0000000 --- a/src/media/ytdlp.ts +++ /dev/null @@ -1,104 +0,0 @@ -import type { ChildProcessWithoutNullStreams } from "node:child_process"; -import { spawn as nodeSpawn } from "node:child_process"; - -export interface YtDlpMetadata { - title: string; - webpageUrl: string; -} - -export interface YtDlpClient { - getMetadata(url: string): Promise; - getDirectAudioUrl(url: string): Promise; - getDirectVideoUrl(url: string): Promise; -} - -export interface YtDlpDependencies { - spawn?: typeof nodeSpawn; -} - -export function createYtDlp(dependencies: YtDlpDependencies = {}): YtDlpClient { - const spawn = dependencies.spawn ?? nodeSpawn; - - return { - async getMetadata(url: string): Promise { - const data = await runYtDlp(spawn, [ - url, - "--dump-single-json", - "--no-playlist", - "--no-warnings", - "--quiet", - ]); - const parsed = JSON.parse(data) as { - title?: string; - webpage_url?: string; - }; - return { - title: parsed.title || url, - webpageUrl: parsed.webpage_url || url, - }; - }, - - async getDirectAudioUrl(url: string): Promise { - const value = await runYtDlp(spawn, [ - url, - "--get-url", - "--format", - "bestaudio[protocol^=http]/bestaudio/best", - "--no-playlist", - "--no-warnings", - "--quiet", - ]); - const directUrl = value.trim().split("\n")[0]; - if (!directUrl) { - console.warn("[ytdlp] No audio URL returned for:", url); - throw new Error(`Failed to resolve audio URL for: ${url}`); - } - console.log( - "[ytdlp] Resolved audio URL:", - directUrl.slice(0, 100) + "...", - ); - return directUrl; - }, - - async getDirectVideoUrl(url: string): Promise { - const value = await runYtDlp(spawn, [ - url, - "--get-url", - "--format", - "best[protocol^=http]/best", - "--no-playlist", - "--no-warnings", - "--quiet", - ]); - return value.trim(); - }, - }; -} - -async function runYtDlp( - spawn: typeof nodeSpawn, - args: string[], -): Promise { - return new Promise((resolve, reject) => { - const proc = spawn("yt-dlp", args, { - stdio: ["ignore", "pipe", "pipe"], - }) as unknown as ChildProcessWithoutNullStreams; - let stdout = ""; - let stderr = ""; - - proc.stdout.on("data", (chunk) => { - stdout += chunk.toString(); - }); - proc.stderr.on("data", (chunk) => { - stderr += chunk.toString(); - }); - proc.on("error", reject); - proc.on("close", (code) => { - if (code === 0) { - resolve(stdout); - return; - } - reject(new Error(`yt-dlp failed with code ${code}: ${stderr.trim()}`)); - }); - }); -} diff --git a/src/metrics.ts b/src/metrics.ts deleted file mode 100644 index 6afa3af..0000000 --- a/src/metrics.ts +++ /dev/null @@ -1,89 +0,0 @@ -import { Counter, Gauge, Histogram, register } from "prom-client"; - -// Audio metrics -export const audioLevelGauge = new Gauge({ - name: "audio_level_db", - help: "Current audio level in dB", - labelNames: ["user_id"], -}); - -export const recordingDurationCounter = new Counter({ - name: "recording_duration_seconds_total", - help: "Total recording duration in seconds", - labelNames: ["user_id"], -}); - -export const activeRecordingsGauge = new Gauge({ - name: "active_recordings", - help: "Number of active recordings", -}); - -export const recordedSegmentsCounter = new Counter({ - name: "recorded_segments_total", - help: "Total number of recorded segments", - labelNames: ["user_id"], -}); - -// Connection metrics -export const voiceConnectionsGauge = new Gauge({ - name: "voice_connections_active", - help: "Number of active voice connections", -}); - -export const connectionErrorsCounter = new Counter({ - name: "connection_errors_total", - help: "Total number of connection errors", - labelNames: ["error_type"], -}); - -export const reconnectAttemptsCounter = new Counter({ - name: "reconnect_attempts_total", - help: "Total number of reconnection attempts", -}); - -// WebSocket metrics -export const wsClientsGauge = new Gauge({ - name: "websocket_clients_connected", - help: "Number of connected WebSocket clients", -}); - -export const wsMessagesCounter = new Counter({ - name: "websocket_messages_total", - help: "Total WebSocket messages sent", - labelNames: ["message_type"], -}); - -// Transcoder metrics -export const transcoderRestartsCounter = new Counter({ - name: "transcoder_restarts_total", - help: "Total number of transcoder restarts", -}); - -export const transcoderRunningGauge = new Gauge({ - name: "transcoder_running", - help: "Whether a transcoder process is currently running (1/0)", -}); - -// HTTP metrics -export const httpRequestDurationHistogram = new Histogram({ - name: "http_request_duration_seconds", - help: "HTTP request duration in seconds", - labelNames: ["method", "route", "status"], - buckets: [0.001, 0.01, 0.1, 0.5, 1, 2, 5], -}); - -export const httpRequestsCounter = new Counter({ - name: "http_requests_total", - help: "Total HTTP requests", - labelNames: ["method", "route", "status"], -}); - -// System metrics -export const uptimeGauge = new Gauge({ - name: "process_uptime_seconds", - help: "Process uptime in seconds", -}); - -export async function getMetrics(): Promise { - return register.metrics(); -} diff --git a/src/middleware.ts b/src/middleware.ts deleted file mode 100644 index 1ab4981..0000000 --- a/src/middleware.ts +++ /dev/null @@ -1,36 +0,0 @@ -import { NextFunction, Request, Response } from "express"; -import { AppError } from "./errors.js"; -import { createChildLogger } from "./logger.js"; - -const logger = createChildLogger("middleware"); - -export function errorHandler( - err: Error, - _req: Request, - res: Response, - _next: NextFunction, -) { - if (err instanceof AppError) { - logger.error( - { code: err.code, statusCode: err.statusCode, message: err.message }, - "Application error", - ); - return res.status(err.statusCode).json({ - error: err.code, - message: err.message, - }); - } - - logger.error({ error: err.message, stack: err.stack }, "Unexpected error"); - res.status(500).json({ - error: "INTERNAL_SERVER_ERROR", - message: "An unexpected error occurred", - }); -} - -export function notFoundHandler(_req: Request, res: Response) { - res.status(404).json({ - error: "NOT_FOUND", - message: "Endpoint not found", - }); -} diff --git a/src/mock-crc.ts b/src/mock-crc.ts deleted file mode 100644 index 4eedcdb..0000000 --- a/src/mock-crc.ts +++ /dev/null @@ -1,48 +0,0 @@ -import { createRequire } from "node:module"; - -const require = createRequire(import.meta.url); - -// Mock node-crc to provide pure JS implementation and bypass native build issues -const CRC_TABLE = new Uint32Array(256); -for (let i = 0; i < 256; i++) { - let r = i << 24; - for (let j = 0; j < 8; j++) { - r = (r & 0x80000000) !== 0 ? (r << 1) ^ 0x04c11db7 : r << 1; - } - CRC_TABLE[i] = r >>> 0; -} - -const Module = require("node:module"); -const originalRequire = Module.prototype.require; -Module.prototype.require = function (id: string) { - if (id === "node-crc") { - return { - crc: function ( - width: number, - reflectIn: boolean, - poly: number, - init: number, - refOut: boolean, - xorOut: number, - unk1: number, - unk2: number, - buffer: Buffer, - ) { - let crc = 0; - for (let i = 0; i < buffer.length; i++) { - crc = - ((crc << 8) >>> 0) ^ CRC_TABLE[((crc >>> 24) ^ buffer[i]) & 0xff]; - crc >>>= 0; - } - const result = Buffer.alloc(4); - result.writeUInt32BE(crc, 0); - return result; - }, - }; - } - return originalRequire.apply(this, arguments); -}; - -console.log("[mock] node-crc has been mocked globally for ESM."); - -export {}; diff --git a/src/moderation/actionExecutor.ts b/src/moderation/actionExecutor.ts deleted file mode 100644 index a6362ef..0000000 --- a/src/moderation/actionExecutor.ts +++ /dev/null @@ -1,295 +0,0 @@ -import type { Client, Guild, User } from "discord.js-selfbot-v13"; -import { createChildLogger } from "../logger.js"; -import { getModerationAction, updateModerationAction } from "./messageStore.js"; -import type { ModerationAction, ModerationActionType } from "./types.js"; - -const logger = createChildLogger("action-executor"); - -interface ActionExecutionContext { - client: Client; - guildId: string; -} - -/** - * Executes a moderation action (delete message, mute user, etc.) - */ -export async function executeModerationAction( - action: ModerationAction, - context: ActionExecutionContext, -): Promise { - try { - const guild = await context.client.guilds.fetch(context.guildId); - if (!guild) { - throw new Error(`Guild ${context.guildId} not found`); - } - - switch (action.action_type) { - case "delete_message": - await executeDeleteMessage(action, guild); - break; - case "mute_user": - await executeMuteUser(action, guild); - break; - case "warn_user": - await executeWarnUser(action, guild); - break; - case "kick_user": - await executeKickUser(action, guild); - break; - case "ban_user": - await executeBanUser(action, guild); - break; - default: - throw new Error(`Unknown action type: ${action.action_type}`); - } - - // Mark action as executed - await updateModerationAction(action.id, { - status: "executed", - executed_at: Date.now(), - error: null, - }); - - logger.info( - { - actionId: action.id, - actionType: action.action_type, - guildId: context.guildId, - }, - "Moderation action executed successfully", - ); - } catch (error) { - const errorMessage = error instanceof Error ? error.message : String(error); - - // Mark action as failed - await updateModerationAction(action.id, { - status: "failed", - error: errorMessage, - }); - - logger.error( - { - actionId: action.id, - actionType: action.action_type, - guildId: context.guildId, - error: errorMessage, - }, - "Failed to execute moderation action", - ); - - throw error; - } -} - -async function executeDeleteMessage( - action: ModerationAction, - guild: Guild, -): Promise { - if (!action.message_id) { - throw new Error("message_id is required for delete_message action"); - } - - // Note: Discord.js selfbot cannot delete messages from other users - // This is a placeholder for the intended behavior - logger.warn( - { messageId: action.message_id }, - "Delete message action requires manual execution or bot permissions", - ); -} - -async function executeMuteUser( - action: ModerationAction, - guild: Guild, -): Promise { - if (!action.user_id) { - throw new Error("user_id is required for mute_user action"); - } - - try { - const member = await guild.members.fetch(action.user_id); - if (!member) { - throw new Error(`Member ${action.user_id} not found in guild`); - } - - // Mute by removing speak permission in all voice channels - const voiceChannels = guild.channels.cache.filter( - (ch) => ch.type === "GUILD_VOICE", - ); - - for (const [, channel] of voiceChannels) { - await channel.permissionOverwrites.create(member, { - SPEAK: false, - }); - } - - logger.info( - { userId: action.user_id, guildId: guild.id }, - "User muted in all voice channels", - ); - } catch (error) { - throw new Error( - `Failed to mute user: ${error instanceof Error ? error.message : String(error)}`, - ); - } -} - -async function executeWarnUser( - action: ModerationAction, - guild: Guild, -): Promise { - if (!action.user_id) { - throw new Error("user_id is required for warn_user action"); - } - - try { - const user = await guild.client.users.fetch(action.user_id); - if (!user) { - throw new Error(`User ${action.user_id} not found`); - } - - const reason = action.reason || "Warned by moderation system"; - await user.send(`You have been warned in ${guild.name}. Reason: ${reason}`); - - logger.info( - { userId: action.user_id, guildId: guild.id }, - "User warned via DM", - ); - } catch (error) { - logger.warn( - { - userId: action.user_id, - guildId: guild.id, - error: error instanceof Error ? error.message : String(error), - }, - "Failed to send warning DM to user", - ); - // Don't throw - warning DM failure is not critical - } -} - -async function executeKickUser( - action: ModerationAction, - guild: Guild, -): Promise { - if (!action.user_id) { - throw new Error("user_id is required for kick_user action"); - } - - try { - const member = await guild.members.fetch(action.user_id); - if (!member) { - throw new Error(`Member ${action.user_id} not found in guild`); - } - - const reason = action.reason || "Kicked by moderation system"; - await member.kick(reason); - - logger.info( - { userId: action.user_id, guildId: guild.id }, - "User kicked from guild", - ); - } catch (error) { - throw new Error( - `Failed to kick user: ${error instanceof Error ? error.message : String(error)}`, - ); - } -} - -async function executeBanUser( - action: ModerationAction, - guild: Guild, -): Promise { - if (!action.user_id) { - throw new Error("user_id is required for ban_user action"); - } - - try { - const reason = action.reason || "Banned by moderation system"; - await guild.bans.create(action.user_id, { reason }); - - logger.info( - { userId: action.user_id, guildId: guild.id }, - "User banned from guild", - ); - } catch (error) { - throw new Error( - `Failed to ban user: ${error instanceof Error ? error.message : String(error)}`, - ); - } -} - -/** - * Processes pending moderation actions for a guild - */ -export async function processPendingActions( - guildId: string, - context: ActionExecutionContext, -): Promise<{ processed: number; failed: number }> { - const result = { processed: 0, failed: 0 }; - - try { - const { listModerationActions } = await import("./messageStore.js"); - - const { data: actions } = await listModerationActions({ - guildId, - status: ["pending"], - limit: 100, - }); - - for (const action of actions) { - try { - await executeModerationAction(action, context); - result.processed++; - } catch (error) { - result.failed++; - logger.error( - { - actionId: action.id, - error: error instanceof Error ? error.message : String(error), - }, - "Failed to process pending action", - ); - } - } - - logger.info({ guildId, ...result }, "Processed pending moderation actions"); - - return result; - } catch (error) { - logger.error( - { - guildId, - error: error instanceof Error ? error.message : String(error), - }, - "Failed to process pending actions", - ); - throw error; - } -} - -/** - * Starts a periodic action processor - */ -export function startActionProcessor( - client: Client, - guildId: string, - intervalMs: number = 60 * 1000, // 1 minute -): NodeJS.Timeout { - logger.info({ guildId, intervalMs }, "Starting action processor"); - - const interval = setInterval(async () => { - try { - await processPendingActions(guildId, { client, guildId }); - } catch (error) { - logger.error( - { - guildId, - error: error instanceof Error ? error.message : String(error), - }, - "Action processor failed", - ); - } - }, intervalMs); - - return interval; -} diff --git a/src/moderation/aiAnalysisWorker.ts b/src/moderation/aiAnalysisWorker.ts deleted file mode 100644 index 7b543ae..0000000 --- a/src/moderation/aiAnalysisWorker.ts +++ /dev/null @@ -1,145 +0,0 @@ -import { config } from "../config.js"; -import { initializeDatabase } from "../database/drizzle.js"; -import { buildConversationContext } from "./conversationContext.js"; -import { runModerationAnalysis } from "./llmModerationClient.js"; -import { - getAttachmentsForMessages, - getConversationContextBefore, - updateMessagesAIAnalysisBulk, -} from "./messageStore.js"; -import type { MessageRecord } from "./types.js"; - -let dbInitialized = false; -let dbInitPromise: Promise | null = null; - -async function ensureDb() { - if (dbInitialized) return; - if (!dbInitPromise) { - dbInitPromise = initializeDatabase().then(() => { - dbInitialized = true; - }); - } - await dbInitPromise; -} - -export interface AnalysisWorkerRequest { - conversationKey: string; - messages: MessageRecord[]; -} - -export type AnalysisWorkerResponse = - | { - ok: true; - conversationKey: string; - rows: MessageRecord[]; - } - | { - ok: false; - conversationKey: string; - rows: MessageRecord[]; - error: string; - }; - -export default async function processAnalysisRequest({ - conversationKey, - messages, -}: AnalysisWorkerRequest): Promise { - if (!config.AI_LLM_API_KEY) { - console.error( - JSON.stringify({ - level: "FATAL", - context: "aiAnalysisWorker", - error: - "AI_LLM_API_KEY is missing from environment. Force closing worker operation.", - timestamp: new Date().toISOString(), - }), - ); - process.exit(1); - } - - try { - try { - await ensureDb(); - } catch (dbError) { - const msg = dbError instanceof Error ? dbError.message : String(dbError); - return { - ok: false, - conversationKey, - rows: [], - error: `Database init failed: ${msg}`, - }; - } - - const firstMessage = messages[0]; - if (!firstMessage) return { ok: true, conversationKey, rows: [] }; - - const contextBefore = await getConversationContextBefore({ - channelId: firstMessage.channel_id, - threadId: firstMessage.thread_id, - beforeCreatedAt: firstMessage.created_at, - limit: config.AI_ANALYSIS_CONTEXT_MESSAGE_LIMIT, - }); - - const contextLines = await buildConversationContext({ - contextBefore, - targets: messages, - maxTokens: config.AI_ANALYSIS_MAX_CONTEXT_TOKENS, - }); - - const targetIds = messages.map((m) => m.id); - const contextIds = contextBefore.map((m) => m.id); - const allMessageIds = [...targetIds, ...contextIds]; - const attachments = await getAttachmentsForMessages(allMessageIds); - - const result = await runModerationAnalysis({ - targets: messages, - contextText: contextLines.join("\n"), - attachments, - }); - - const updates = result.results.map((analysisResult) => ({ - messageId: analysisResult.messageId, - result: { - status: analysisResult.status, - flags: JSON.stringify(analysisResult.flags), - score: analysisResult.score, - analysis: analysisResult.analysis, - categories: analysisResult.categories, - severity: analysisResult.severity, - confidence: analysisResult.confidence, - recommendedAction: analysisResult.recommendedAction, - analyzedAt: Date.now(), - error: null, - }, - })); - - try { - const rows = await updateMessagesAIAnalysisBulk(updates); - return { ok: true, conversationKey, rows }; - } catch (dbErr) { - // If bulk update fails, we log it but don't fail the worker completely - // so it can at least retry later without blowing up the circuit breaker if it was an isolated issue - throw new Error( - `Failed to update DB: ${dbErr instanceof Error ? dbErr.message : String(dbErr)}`, - ); - } - } catch (error) { - const errorMessage = error instanceof Error ? error.message : String(error); - const errorStack = error instanceof Error ? error.stack : undefined; - const rows: MessageRecord[] = []; - - console.error( - JSON.stringify({ - level: "ERROR", - context: "aiAnalysisWorker", - conversationKey, - messageCount: messages.length, - error: errorMessage, - stack: errorStack, - timestamp: new Date().toISOString(), - }), - ); - - return { ok: false, conversationKey, rows, error: errorMessage }; - } -} diff --git a/src/moderation/aiAnalyzer.ts b/src/moderation/aiAnalyzer.ts deleted file mode 100644 index ceae1ab..0000000 --- a/src/moderation/aiAnalyzer.ts +++ /dev/null @@ -1,918 +0,0 @@ -import { existsSync } from "node:fs"; -import { fileURLToPath } from "node:url"; -import type { Client } from "discord.js-selfbot-v13"; -import { AbortError } from "p-retry"; -import { Piscina } from "piscina"; -import { config } from "../config.js"; -import { createChildLogger } from "../logger.js"; -import { retryWithBackoff } from "../retry.js"; -import { attemptAutoDeleteFlaggedMessage } from "./autoDeleteManager.js"; -import { buildConversationContext } from "./conversationContext.js"; -import { runModerationAnalysis } from "./llmModerationClient.js"; -import { isAgeRestrictedMetadata } from "./messageMetadata.js"; -import { - getAttachmentsForMessages, - getConversationContextBefore, - getConversationKeysWithIncompleteAnalysis, - getIncompleteMessagesByConversation, - getMessageById, - getPendingConversationKeys, - getPendingMessagesByConversation, - updateMessageAIAnalysis, - updateMessagesAIAnalysisBulk, -} from "./messageStore.js"; -import { invalidateAnalyticsCache } from "./analyticsStore.js"; -import type { - AnalysisQueueStatus, - MessageRecord, - ModerationBroadcaster, -} from "./types.js"; - -const logger = createChildLogger("ai-analyzer"); - -type ModerationGlobal = typeof globalThis & { - moderationBroadcaster?: ModerationBroadcaster; -}; - -function getModerationBroadcaster(): ModerationBroadcaster | undefined { - return (globalThis as ModerationGlobal).moderationBroadcaster; -} - -function scheduleAutoDelete(row: MessageRecord): void { - if (row.ai_status !== "flagged" && row.ai_status !== "warn") return; - const run = () => { - attemptAutoDeleteFlaggedMessage(moderationClient, row).catch((error) => { - logger.error( - { - messageId: row.id, - error: error instanceof Error ? error.message : String(error), - }, - "Unexpected auto-delete error", - ); - }); - }; - - if (config.AUTO_DELETE_FLAGGED_DELAY_MS > 0) { - setTimeout(run, config.AUTO_DELETE_FLAGGED_DELAY_MS); - return; - } - setImmediate(run); -} - -function isAgeRestrictedMessage(message: MessageRecord): boolean { - return isAgeRestrictedMetadata(message.metadata); -} - -function buildAgeRestrictedSkipResult(): { - status: "clean"; - flags: string | null; - score: number; - analysis: string; - categories: string[]; - severity: "none"; - confidence: number; - recommendedAction: "none"; - analyzedAt: number; - error: null; -} { - return { - status: "clean", - flags: JSON.stringify(["age_restricted"]), - score: 0, - analysis: "Skipped moderation for age-restricted content.", - categories: ["age_restricted"], - severity: "none", - confidence: 1, - recommendedAction: "none", - analyzedAt: Date.now(), - error: null, - }; -} - -async function skipAgeRestrictedMessages( - messages: MessageRecord[], -): Promise { - const ageRestrictedMessages = messages.filter(isAgeRestrictedMessage); - if (ageRestrictedMessages.length === 0) { - return messages; - } - - const skippedRows = await updateMessagesAIAnalysisBulk( - ageRestrictedMessages.map((message) => ({ - messageId: message.id, - result: buildAgeRestrictedSkipResult(), - })), - ); - - for (const row of skippedRows) { - getModerationBroadcaster()?.messageAnalyzed(row); - } - - const skippedIds = new Set( - ageRestrictedMessages.map((message) => message.id), - ); - return messages.filter((message) => !skippedIds.has(message.id)); -} - -// --------------------------------------------------------------------------- -// Batch pipeline state -// --------------------------------------------------------------------------- - -/** Debounce timer handle per conversation key. */ -const conversationDebounceTimers = new Map(); -/** Timestamp of when processing started per conversation key. */ -const conversationProcessing = new Map(); -/** Cooldown expiry timestamp per conversation key after an error. */ -const conversationErrorCooldown = new Map(); - -let activeRequests = 0; -let lastError: string | null = null; -let moderationClient: Client | undefined; - -// Batch circuit breaker -let consecutiveErrors = 0; -const MAX_CONSECUTIVE_ERRORS = 5; -let globalCooldownUntil = 0; - -// --------------------------------------------------------------------------- -// Individual fallback queue — runs PARALLEL to the batch pipeline. -// -// Design guarantees: -// • Concurrency is capped at config.AI_ANALYSIS_INDIVIDUAL_MAX_CONCURRENT. -// • A flat Set de-duplicates so the same message can't be -// in-flight twice (Discord snowflakes are globally unique, but be safe). -// • A Map lets the recovery worker skip conversations -// that already have individual work in progress (#4 fix). -// • A separate circuit breaker prevents a cascade of individual failures -// from hammering a down/rate-limited LLM endpoint (#1+#5 fix). -// --------------------------------------------------------------------------- - -/** IDs currently being processed one-by-one. */ -const individualInFlight = new Set(); - -/** - * Per-conversation count of in-flight individual messages. - * Used by the recovery worker to avoid re-scheduling a conversation that - * already has individual fallback work running for it. - */ -const individualInFlightByConversation = new Map(); - -/** Counter for observability. */ -let activeIndividualRequests = 0; - -// Individual fallback circuit breaker (independent of batch CB) -let individualConsecutiveErrors = 0; -let individualCooldownUntil = 0; -const INDIVIDUAL_COOLDOWN_MS = 30000; - -// --------------------------------------------------------------------------- -// Piscina worker pool (batch path only) -// --------------------------------------------------------------------------- - -function getAnalysisWorkerUrl(): URL { - const candidates = [ - new URL("./aiAnalysisWorker.js", import.meta.url), - new URL("../aiAnalysisWorker.js", import.meta.url), - new URL("./aiAnalysisWorker.ts", import.meta.url), - ]; - - for (const candidate of candidates) { - if (existsSync(fileURLToPath(candidate))) { - return candidate; - } - } - - return candidates[2]; -} - -const workerPool = new Piscina({ - filename: fileURLToPath(getAnalysisWorkerUrl()), - execArgv: process.execArgv, -}); - -interface AnalysisWorkerResponse { - ok: boolean; - conversationKey: string; - rows: MessageRecord[]; - error?: string; -} - -// --------------------------------------------------------------------------- -// Exported helpers -// --------------------------------------------------------------------------- - -/** - * Gets the conversation key for a message (thread_id or channel_id). - */ -export function getConversationKey(message: MessageRecord): string { - return message.thread_id || message.channel_id; -} - -/** - * Picks a batch of messages within a token budget. - * `tokensPerMessage` accounts for JSON structure overhead around each entry. - * Uses a rough character-based token estimate (avoids async formatMessageForPrompt - * since this function runs in a synchronous promise chain). - */ -export function pickBatchWithinBudget( - messages: MessageRecord[], - maxTokens: number, - tokensPerMessage: number, -): MessageRecord[] { - const batch: MessageRecord[] = []; - let usedTokens = 0; - - for (const msg of messages) { - const content = msg.edited_content ?? msg.content; - // Rough token estimate: ~3 chars per token + metadata overhead - const msgTokens = Math.ceil(content.length / 3) + tokensPerMessage; - - if (usedTokens + msgTokens <= maxTokens) { - batch.push(msg); - usedTokens += msgTokens; - } - } - - return batch; -} - -// --------------------------------------------------------------------------- -// Conversation lock helpers -// --------------------------------------------------------------------------- - -function isConversationProcessingLocked(conversationKey: string): boolean { - const startedAt = conversationProcessing.get(conversationKey); - // FIX #7: use configurable timeout that exceeds (LLM timeout × max retries). - // Old hardcoded value was 30 000 ms — shorter than a single LLM call under retries. - return Boolean( - startedAt && - Date.now() - startedAt < config.AI_ANALYSIS_PROCESSING_TIMEOUT_MS, - ); -} - -// --------------------------------------------------------------------------- -// Individual fallback pipeline -// --------------------------------------------------------------------------- - -/** - * Processes a single message directly in the main process (no IPC/worker - * pool overhead). Never called from the batch path. - * - * FIX #1+#5: Increments the individual circuit breaker on failure so a - * sustained outage stops hammering the LLM endpoint. - * - * Infinite-loop prevention: if the LLM consistently drops the single target - * message across all retries (analysis_incomplete), we write a terminal flag - * 'individual_analysis_exhausted' to DB instead of 'analysis_incomplete'. - * The recovery worker only queries for 'analysis_incomplete', so exhausted - * messages are permanently excluded from the reprocessing loop. - * Transient failures (network/parse/DB) are NOT written as exhausted — they - * stay as 'analysis_incomplete' so the circuit-breaker-throttled recovery - * cycle can retry them later. - */ -async function processIndividualFallback( - message: MessageRecord, -): Promise { - const { id: messageId } = message; - const conversationKey = getConversationKey(message); - - activeIndividualRequests++; - // Increment per-conversation counter so the recovery worker can see it. - individualInFlightByConversation.set( - conversationKey, - (individualInFlightByConversation.get(conversationKey) ?? 0) + 1, - ); - - // Track whether all retries were exhausted specifically because the LLM - // consistently returned no result for this message (vs. a transient error). - let exhaustedOnIncomplete = false; - - try { - const contextBefore = await getConversationContextBefore({ - channelId: message.channel_id, - threadId: message.thread_id, - beforeCreatedAt: message.created_at, - limit: config.AI_ANALYSIS_CONTEXT_MESSAGE_LIMIT, - }); - - const contextLines = await buildConversationContext({ - contextBefore, - targets: [message], - maxTokens: config.AI_ANALYSIS_MAX_CONTEXT_TOKENS, - }); - - const contextIds = contextBefore.map((m) => m.id); - const attachments = await getAttachmentsForMessages([ - messageId, - ...contextIds, - ]); - - const analysisResult = await retryWithBackoff( - async () => { - try { - const result = await runModerationAnalysis({ - targets: [message], - contextText: contextLines.join("\n"), - attachments, - }); - - // If the LLM still dropped our only target, convert to a retryable - // throw so backoff kicks in. Track this so the catch block can - // distinguish it from a transient network/parse failure. - const stillIncomplete = result.results.some((r) => - r.flags.includes("analysis_incomplete"), - ); - if (stillIncomplete) { - exhaustedOnIncomplete = true; - throw new Error( - `LLM returned no result for single-target message ${messageId} — will retry with backoff`, - ); - } - - // Got a real result — clear the incomplete flag. - exhaustedOnIncomplete = false; - - return result; - } catch (err: any) { - // Propagate AbortError so outer retry is immediately cancelled on 429. - if (err instanceof AbortError) { - throw err; - } - if ( - err?.status === 429 || - err?.status === 401 || - err?.status === 403 - ) { - throw new AbortError(err); - } - throw err; - } - }, - { - retries: 0, - minTimeout: 0, - maxTimeout: 0, - logger, - }, - ); - - const updates = analysisResult.results.map((r) => ({ - messageId: r.messageId, - result: { - status: r.status, - flags: JSON.stringify(r.flags), - score: r.score, - analysis: r.analysis, - categories: r.categories, - severity: r.severity, - confidence: r.confidence, - recommendedAction: r.recommendedAction, - analyzedAt: Date.now(), - error: null, - }, - })); - - const rows = await updateMessagesAIAnalysisBulk(updates); - for (const row of rows) { - getModerationBroadcaster()?.messageAnalyzed(row); - invalidateAnalyticsCache(row.guild_id); - scheduleAutoDelete(row); - } - - // Reset individual CB on success. - individualConsecutiveErrors = 0; - - logger.info( - { messageId, status: analysisResult.results[0]?.status }, - "Individual fallback analysis complete", - ); - } catch (error) { - // FIX #5: individual failures now feed their own circuit breaker. - individualConsecutiveErrors++; - if ( - individualConsecutiveErrors >= config.AI_ANALYSIS_INDIVIDUAL_CB_THRESHOLD - ) { - individualCooldownUntil = Date.now() + INDIVIDUAL_COOLDOWN_MS; - logger.warn( - { - threshold: config.AI_ANALYSIS_INDIVIDUAL_CB_THRESHOLD, - cooldownUntil: new Date(individualCooldownUntil).toISOString(), - }, - "Individual fallback circuit breaker triggered", - ); - } - - lastError = error instanceof Error ? error.message : String(error); - - // Infinite-loop prevention: if all retries were exhausted because the LLM - // consistently dropped this specific message (not a transient error), - // overwrite the DB entry with a terminal flag that the recovery query - // does NOT match. This permanently removes it from the recovery loop - // while keeping it visible as an error in the dashboard. - if (exhaustedOnIncomplete) { - await updateMessagesAIAnalysisBulk([ - { - messageId, - result: { - status: "error", - flags: JSON.stringify(["individual_analysis_exhausted"]), - score: 0, - analysis: - "Individual fallback exhausted all retries: LLM consistently dropped this message even in single-target mode", - categories: ["individual_analysis_exhausted"], - severity: "none", - confidence: 0, - recommendedAction: "review", - analyzedAt: Date.now(), - error: lastError, - }, - }, - ]).catch((dbErr) => { - logger.error( - { messageId, error: String(dbErr) }, - "Failed to write terminal exhausted status — message may re-enter recovery loop", - ); - }); - logger.warn( - { messageId }, - "Individual fallback exhausted — marked as individual_analysis_exhausted to stop recovery loop", - ); - } else { - // Transient failure (network/parse/DB): do NOT write terminal status. - // Message stays as error/analysis_incomplete in DB and will be retried - // by the recovery worker, subject to the individual circuit breaker. - logger.error( - { - messageId, - error: lastError, - stack: error instanceof Error ? error.stack : undefined, - }, - "Individual fallback analysis failed (transient) — will be retried by recovery worker", - ); - } - } finally { - activeIndividualRequests--; - individualInFlight.delete(messageId); - - // Decrement per-conversation counter; remove key when it hits zero. - const prev = individualInFlightByConversation.get(conversationKey) ?? 1; - if (prev <= 1) { - individualInFlightByConversation.delete(conversationKey); - } else { - individualInFlightByConversation.set(conversationKey, prev - 1); - } - } -} - -/** - * Fans out message records to the individual fallback queue. - * - * FIX #1: Checks concurrency cap before admitting new work. - * FIX #5: Checks individual circuit breaker before admitting new work. - * Messages that cannot be admitted remain as `error/analysis_incomplete` in - * the DB and will be picked up by the recovery worker on the next interval. - */ -function enqueueIndividualFallbacks(messages: MessageRecord[]): void { - // FIX #5: Honour the individual circuit breaker. - if (Date.now() < individualCooldownUntil) { - logger.warn( - { - until: new Date(individualCooldownUntil).toISOString(), - skipped: messages.length, - }, - "Individual fallback circuit breaker active — messages will be recovered later", - ); - return; - } - - const newMessages = messages.filter((m) => !individualInFlight.has(m.id)); - if (newMessages.length === 0) return; - - logger.info( - { - count: newMessages.length, - messageIds: newMessages.map((m) => m.id), - }, - "Enqueueing individual fallback analysis for batch-incomplete messages", - ); - - for (const msg of newMessages) { - individualInFlight.add(msg.id); - // Fire-and-forget: processIndividualFallback handles all errors internally. - processIndividualFallback(msg).catch((err) => { - // Belt-and-suspenders guard — should never reach here. - logger.error( - { messageId: msg.id, error: String(err) }, - "Unexpected uncaught error escaping processIndividualFallback", - ); - individualInFlight.delete(msg.id); - const ck = getConversationKey(msg); - const prev = individualInFlightByConversation.get(ck) ?? 1; - if (prev <= 1) { - individualInFlightByConversation.delete(ck); - } else { - individualInFlightByConversation.set(ck, prev - 1); - } - }); - } -} - -// --------------------------------------------------------------------------- -// Batch pipeline -// --------------------------------------------------------------------------- - -async function processBatch( - conversationKey: string, - messages: MessageRecord[], -): Promise { - if (messages.length === 0) return; - if (Date.now() < globalCooldownUntil) { - return; - } - - activeRequests++; - let shouldScheduleNext = false; - const processingStartedAt = Date.now(); - conversationProcessing.set(conversationKey, processingStartedAt); - try { - const result = (await workerPool.run({ - conversationKey, - messages, - })) as AnalysisWorkerResponse; - - for (const row of result.rows) { - getModerationBroadcaster()?.messageAnalyzed(row); - scheduleAutoDelete(row); - } - - if (!result.ok) { - consecutiveErrors++; - if (consecutiveErrors >= MAX_CONSECUTIVE_ERRORS) { - globalCooldownUntil = Date.now() + 60000; - logger.warn( - "Global circuit breaker triggered due to consecutive errors", - ); - } - - // Batch failed entirely — fall back all messages to individual queue - // so no message is permanently lost behind a cooldown. - logger.warn( - { - conversationKey, - messageCount: messages.length, - error: result.error, - }, - "Batch failed entirely — routing all messages to individual fallback queue", - ); - enqueueIndividualFallbacks(messages); - - lastError = result.error ?? "Analysis worker failed"; - conversationErrorCooldown.set( - conversationKey, - Date.now() + config.AI_ANALYSIS_ERROR_COOLDOWN_MS, - ); - logger.error( - { - conversationKey, - error: lastError, - messageCount: messages.length, - messageIds: messages.map((m) => m.id), - cooldownUntil: new Date( - Date.now() + config.AI_ANALYSIS_ERROR_COOLDOWN_MS, - ).toISOString(), - timestamp: new Date().toISOString(), - }, - "Batch analysis failed, will retry after cooldown", - ); - return; - } - - // Batch succeeded — but check for messages the LLM silently dropped. - // Rows with flag "analysis_incomplete" were produced by parseModerationResponse - // as synthetic errors; they must be re-processed individually. - const incompleteMessages = messages.filter((msg) => { - const row = result.rows.find((r) => r.id === msg.id); - if (!row) { - // The DB update row is missing entirely — treat as incomplete. - return true; - } - const flags: string[] = (() => { - try { - return JSON.parse(row.ai_moderation_flags ?? "[]") as string[]; - } catch { - return []; - } - })(); - return row.ai_status === "error" && flags.includes("analysis_incomplete"); - }); - - if (incompleteMessages.length > 0) { - logger.warn( - { - conversationKey, - incompleteCount: incompleteMessages.length, - incompleteIds: incompleteMessages.map((m) => m.id), - totalBatchSize: messages.length, - }, - "Batch returned incomplete results — fanning out to individual fallback queue", - ); - enqueueIndividualFallbacks(incompleteMessages); - } - - consecutiveErrors = 0; // Reset batch circuit breaker - conversationErrorCooldown.delete(conversationKey); - shouldScheduleNext = true; - } catch (error) { - consecutiveErrors++; - if (consecutiveErrors >= MAX_CONSECUTIVE_ERRORS) { - globalCooldownUntil = Date.now() + 60000; - logger.warn("Global circuit breaker triggered due to consecutive errors"); - } - - // Unhandled exception — route everything to individual fallback. - logger.warn( - { conversationKey, messageCount: messages.length }, - "Batch threw exception — routing all messages to individual fallback queue", - ); - enqueueIndividualFallbacks(messages); - - lastError = error instanceof Error ? error.message : String(error); - const errorStack = error instanceof Error ? error.stack : undefined; - conversationErrorCooldown.set( - conversationKey, - Date.now() + config.AI_ANALYSIS_ERROR_COOLDOWN_MS, - ); - logger.error( - { - conversationKey, - error: lastError, - stack: errorStack, - messageCount: messages.length, - messageIds: messages.map((m) => m.id), - cooldownUntil: new Date( - Date.now() + config.AI_ANALYSIS_ERROR_COOLDOWN_MS, - ).toISOString(), - timestamp: new Date().toISOString(), - }, - "Analysis worker failed, will retry after cooldown", - ); - } finally { - activeRequests--; - if (conversationProcessing.get(conversationKey) === processingStartedAt) { - conversationProcessing.delete(conversationKey); - } - if (shouldScheduleNext) { - setImmediate(() => scheduleConversationAnalysis(conversationKey)); - } - } -} - -// --------------------------------------------------------------------------- -// Scheduling -// --------------------------------------------------------------------------- - -/** - * Schedules a debounced analysis run for a conversation. - * - * FIX #3: The async work inside setTimeout is now wrapped in an explicit - * .catch() so DB errors don't produce unhandled promise rejections. - * FIX #6: Calls pickBatchWithinBudget after fetching messages so token budget - * is respected before handing the batch to the LLM. - */ -function scheduleConversationAnalysis(conversationKey: string): void { - if (isConversationProcessingLocked(conversationKey)) { - return; - } - - const convoCooldown = conversationErrorCooldown.get(conversationKey) || 0; - const activeCooldown = Math.max(convoCooldown, globalCooldownUntil); - - if (activeCooldown && Date.now() < activeCooldown) { - if (!conversationDebounceTimers.has(conversationKey)) { - const remaining = activeCooldown - Date.now(); - const timer = setTimeout(() => { - conversationDebounceTimers.delete(conversationKey); - scheduleConversationAnalysis(conversationKey); - }, remaining + 500); - conversationDebounceTimers.set(conversationKey, timer); - } - return; - } - - const existingTimer = conversationDebounceTimers.get(conversationKey); - if (existingTimer) { - clearTimeout(existingTimer); - } - - const timer = setTimeout(() => { - conversationDebounceTimers.delete(conversationKey); - - // FIX #3: explicit .catch() — no async arrow function to avoid unhandled rejection. - getPendingMessagesByConversation( - conversationKey, - config.AI_ANALYSIS_MAX_BATCH_SIZE, - ) - .then(async (messages) => { - if (messages.length === 0) return; - - const processableMessages = await skipAgeRestrictedMessages(messages); - if (processableMessages.length === 0) return; - - // FIX #6: trim to token budget before sending to LLM. - // 50 tokens overhead accounts for JSON structure + id/username fields. - let trimmed = pickBatchWithinBudget( - processableMessages, - config.AI_ANALYSIS_MAX_TARGET_TOKENS, - 50, - ); - - // FIX #10: if every message individually exceeds the token budget, - // pickBatchWithinBudget returns [] — which would leave them permanently - // stuck as `pending`. Fall back to the first message alone so at - // least one makes progress; the rest will be processed in later ticks. - if (trimmed.length === 0 && processableMessages.length > 0) { - trimmed = processableMessages.slice(0, 1); - logger.warn( - { - conversationKey, - messageId: processableMessages[0]?.id, - tokenBudget: config.AI_ANALYSIS_MAX_TARGET_TOKENS, - }, - "All messages exceed token budget — processing first message alone to avoid stuck-pending deadlock", - ); - } - - return processBatch(conversationKey, trimmed); - }) - .catch((err) => { - logger.error( - { - conversationKey, - error: err instanceof Error ? err.message : String(err), - }, - "Failed to fetch or dispatch pending messages for scheduled analysis", - ); - }); - }, config.AI_ANALYSIS_DEBOUNCE_MS); - - conversationDebounceTimers.set(conversationKey, timer); -} - -// --------------------------------------------------------------------------- -// Public API -// --------------------------------------------------------------------------- - -/** - * Queues a message for analysis (debounced by conversation). - */ -export async function queueMessageAnalysis(messageId: string): Promise { - if (!config.AI_ANALYSIS_ENABLED) return; - - try { - const message = await getMessageById(messageId); - if (!message) { - logger.warn({ messageId }, "Message not found for analysis queue"); - return; - } - - if (isAgeRestrictedMessage(message)) { - const updated = await updateMessageAIAnalysis( - message.id, - buildAgeRestrictedSkipResult(), - ); - if (updated) { - getModerationBroadcaster()?.messageAnalyzed(updated); - } - logger.info( - { messageId }, - "Skipped AI analysis for age-restricted message", - ); - return; - } - - queueConversationAnalysis(getConversationKey(message)); - } catch (error) { - logger.error( - { - messageId, - error: error instanceof Error ? error.message : String(error), - }, - "Failed to queue message for analysis", - ); - } -} - -/** - * Queues a conversation for analysis (debounced). - */ -export function queueConversationAnalysis(conversationKey: string): void { - if (!config.AI_ANALYSIS_ENABLED) return; - scheduleConversationAnalysis(conversationKey); -} - -/** - * Returns current status of both the batch and individual fallback queues. - */ -export function getAnalysisQueueStatus(): AnalysisQueueStatus { - return { - queuedConversations: conversationDebounceTimers.size, - activeRequests, - activeIndividualRequests, - individualInFlightCount: individualInFlight.size, - individualCircuitBreakerActive: Date.now() < individualCooldownUntil, - lastError, - }; -} - -/** - * Starts the periodic recovery worker. - * - * FIX #4: Now also recovers messages stuck in `error/analysis_incomplete` - * state (not just `pending`), and skips conversations that already have - * individual fallback work in progress to avoid DB last-write-wins races. - */ -export function startPendingAIAnalysisWorker(client?: Client): void { - moderationClient = client; - if (!config.AI_ANALYSIS_ENABLED) return; - - setInterval(() => { - // FIX #3 pattern: no async arrow — chain promises explicitly. - Promise.all([ - getPendingConversationKeys(500), - getConversationKeysWithIncompleteAnalysis(200), - ]) - .then(([pendingKeys, incompleteKeys]) => { - const now = Date.now(); - - // FIX #9: Prune stale entries from state maps to prevent unbounded - // memory growth from channels/threads that are no longer active. - for (const [key, expiry] of conversationErrorCooldown) { - if (now >= expiry) conversationErrorCooldown.delete(key); - } - for (const [key, startedAt] of conversationProcessing) { - if (now - startedAt >= config.AI_ANALYSIS_PROCESSING_TIMEOUT_MS) { - conversationProcessing.delete(key); - } - } - - // FIX #8: Build a set of keys already targeted for individual recovery - // so the batch loop below skips them, preventing a race where batch - // scheduling and individual scheduling collide on the same conversation. - const incompleteKeySet = new Set(incompleteKeys); - - // --- Batch recovery for `pending` messages --- - for (const key of pendingKeys) { - if (conversationDebounceTimers.has(key)) continue; - if (isConversationProcessingLocked(key)) continue; - // FIX #4: skip if individual fallback already running for this conversation. - if (individualInFlightByConversation.has(key)) continue; - // FIX #8: skip if this conversation also needs individual recovery - // (batch processing would conflict with in-flight individual work). - if (incompleteKeySet.has(key)) continue; - const cooldownUntil = conversationErrorCooldown.get(key); - if (cooldownUntil && now < cooldownUntil) continue; - scheduleConversationAnalysis(key); - } - - // --- Individual recovery for `error/analysis_incomplete` messages --- - // Circuit breaker check: no point iterating if individual CB is active. - if (now >= individualCooldownUntil) { - const promises: Promise[] = []; - for (const key of incompleteKeys) { - // Skip if individual work is already running for this conversation. - if (individualInFlightByConversation.has(key)) continue; - // Skip if batch processing is running (it will fan-out if it finds more incomplete). - if (isConversationProcessingLocked(key)) continue; - - promises.push( - getIncompleteMessagesByConversation(key, 500) - .then(async (msgs) => { - const processableMessages = - await skipAgeRestrictedMessages(msgs); - return processableMessages; - }) - .then((msgs) => { - if (msgs.length > 0) { - enqueueIndividualFallbacks(msgs); - } - }) - .catch((err) => { - logger.error( - { key, error: String(err) }, - "Failed to fetch incomplete messages for recovery", - ); - }), - ); - } - // Errors are handled per-key; return the combined promise for observability. - return Promise.all(promises); - } - }) - .catch((err) => { - logger.error( - { error: err instanceof Error ? err.message : String(err) }, - "Pending AI analysis recovery worker failed", - ); - }); - }, config.AI_ANALYSIS_RECOVERY_INTERVAL_MS); -} diff --git a/src/moderation/analyticsStore.ts b/src/moderation/analyticsStore.ts deleted file mode 100644 index 20c129e..0000000 --- a/src/moderation/analyticsStore.ts +++ /dev/null @@ -1,929 +0,0 @@ -import { executeAll, executeGet } from "../database/drizzle.js"; -import { createChildLogger } from "../logger.js"; -import type { MessageRecord } from "./types.js"; - -const logger = createChildLogger("analytics-store"); - -// ── Types ────────────────────────────────────────────────────────────── - -export interface HourlyBucket { - hour: string; - count: number; - clean: number; - warned: number; - flagged: number; - error: number; -} - -export interface TopicTrend { - topic: string; - count: number; - score: number; -} - -export interface UserStat { - user_id: string; - username: string; - avatar_url: string | null; - message_count: number; - edited_count: number; - deleted_count: number; - flagged_count: number; - last_active: number; -} - -export interface ModerationBreakdown { - total: number; - clean: number; - warned: number; - flagged: number; - error: number; - pending: number; - average_score: number; -} - -export interface AnalyticsOverview { - period: { start: number; end: number }; - messages: ModerationBreakdown; - hourly: HourlyBucket[]; - topics: TopicTrend[]; - top_users: UserStat[]; - active_users_count: number; - total_channels: number; -} - -// ══════════════════════════════════════════════════════════════════════════ -// GENERIC QUERY CACHE (reduces duplicate DB calls from 5s auto-refresh) -// ══════════════════════════════════════════════════════════════════════════ - -interface CacheEntry { - data: T; - expiresAt: number; -} - -const queryCache = new Map>(); - -/** Default TTL for aggregate queries — 10s is long enough to prevent redundant - * calls from the 5s auto-refresh but short enough to feel real-time. */ -const AGGREGATE_CACHE_TTL_MS = 10_000; - -/** Topic extraction is expensive (JSON parsing). Cache longer. */ -const TOPIC_CACHE_TTL_MS = 120_000; - -function makeCacheKey(prefix: string, params: Record): string { - return `${prefix}:${JSON.stringify(params)}`; -} - -function getCached(key: string): T | undefined { - const entry = queryCache.get(key); - if (entry && entry.expiresAt > Date.now()) return entry.data; - if (entry) queryCache.delete(key); // expired - return undefined; -} - -function setCache(key: string, data: T, ttl: number): void { - queryCache.set(key, { data, expiresAt: Date.now() + ttl }); - // Prune old entries if cache grows too large (>200 entries) - if (queryCache.size > 200) { - const now = Date.now(); - for (const [k, v] of queryCache) { - if (v.expiresAt <= now) queryCache.delete(k); - } - } -} - -// ── Hourly Message Stats ─────────────────────────────────────────────── - -export async function getHourlyStats(input: { - guildId: string; - channelId?: string; - hours?: number; -}): Promise { - const { guildId, channelId, hours = 24 } = input; - const cacheKey = makeCacheKey("hourly", { guildId, channelId, hours }); - const cached = getCached(cacheKey); - if (cached) return cached; - - try { - const since = Date.now() - hours * 3600_000; - const hourExpr = `to_char(to_timestamp((created_at / 3600000) * 3600), 'YYYY-MM-DD HH24:MI:SS') as hour`; - - const rows = await executeAll( - ` - SELECT - ${hourExpr}, - count(*) as count, - count(case when ai_status = 'clean' then 1 end) as clean, - count(case when ai_status = 'warn' then 1 end) as warned, - count(case when ai_status = 'flagged' then 1 end) as flagged, - count(case when ai_status = 'error' then 1 end) as error - FROM messages - WHERE guild_id = ? - AND created_at >= ? - AND deleted_at IS NULL - ${channelId ? `AND (channel_id = ? OR thread_id = ?)` : ""} - GROUP BY (created_at / 3600000) - ORDER BY hour ASC - `, - channelId ? [guildId, since, channelId, channelId] : [guildId, since], - ); - - // Initialize all hour buckets (fill gaps with zeros) - const buckets = new Map< - string, - { - count: number; - clean: number; - warned: number; - flagged: number; - error: number; - } - >(); - - for (let h = 0; h < hours; h++) { - const ts = new Date(since + h * 3600_000); - ts.setMinutes(0, 0, 0); - const key = ts.toISOString().slice(0, 13) + ":00:00Z"; - buckets.set(key, { count: 0, clean: 0, warned: 0, flagged: 0, error: 0 }); - } - - for (const row of rows) { - const d = new Date(row.hour.replace(" ", "T") + "Z"); - const key = d.toISOString().slice(0, 13) + ":00:00Z"; - const bucket = buckets.get(key); - if (!bucket) continue; - bucket.count = row.count; - bucket.clean = row.clean; - bucket.warned = row.warned; - bucket.flagged = row.flagged; - bucket.error = row.error; - } - - const result = Array.from(buckets.entries()) - .sort(([a], [b]) => a.localeCompare(b)) - .map(([hour, data]) => ({ hour, ...data })); - - setCache(cacheKey, result, AGGREGATE_CACHE_TTL_MS); - return result; - } catch (error) { - logger.error( - { error: error instanceof Error ? error.message : String(error) }, - "Failed to get hourly stats", - ); - return []; - } -} - -// ── Topic Trends ─────────────────────────────────────────────────────── - -const STOP_WORDS = new Set([ - "yang", - "dan", - "itu", - "ini", - "dengan", - "akan", - "pada", - "dari", - "di", - "ke", - "untuk", - "tidak", - "ada", - "juga", - "sudah", - "saya", - "kamu", - "dia", - "mereka", - "kami", - "aku", - "lo", - "lu", - "gua", - "gue", - "org", - "orang", - "aja", - "sama", - "kalo", - "kalau", - "bisa", - "karena", - "gak", - "nggak", - "ga", - "tak", - "belum", - "udah", - "dah", - "lah", - "kah", - "pun", - "nih", - "tuh", - "deh", - "dong", - "si", - "nya", - "kan", - "ya", - "yah", - "yuk", - "kok", - "loh", - "nah", - "wow", - "eh", - "the", - "a", - "an", - "is", - "are", - "was", - "were", - "be", - "been", - "being", - "have", - "has", - "had", - "having", - "do", - "does", - "did", - "doing", - "will", - "would", - "could", - "should", - "may", - "might", - "must", - "shall", - "i", - "you", - "he", - "she", - "it", - "we", - "they", - "me", - "him", - "her", - "us", - "them", - "my", - "your", - "his", - "its", - "our", - "their", - "and", - "but", - "or", - "nor", - "not", - "so", - "yet", - "for", - "if", - "to", - "of", - "in", - "on", - "at", - "by", - "as", - "with", - "about", - "just", - "then", - "now", - "here", - "there", - "when", - "where", - "why", - "how", - "all", - "both", - "each", - "few", - "more", - "most", - "other", - "some", - "such", - "only", - "own", - "same", - "too", - "very", - "can", - "go", - "ok", - "okay", - "yeah", - "yes", - "no", -]); - -function extractTopics(messages: MessageRecord[], topN = 15): TopicTrend[] { - const topicScores = new Map(); - const wordFreq = new Map(); - const flaggedWordFreq = new Map(); - - for (const msg of messages) { - if (msg.ai_analysis) { - try { - const analysis = JSON.parse(msg.ai_analysis); - const topics = analysis.topics; - if (topics && Array.isArray(topics)) { - for (const topic of topics) { - const key = - typeof topic === "string" ? topic : topic.name || topic.topic; - if (!key) continue; - const k = key.toLowerCase(); - const score = msg.ai_moderation_score || 0; - const existing = topicScores.get(k); - if (existing) { - existing.count++; - existing.score += score; - } else { - topicScores.set(k, { count: 1, score }); - } - } - } - if (analysis.category) { - const cat = String(analysis.category).toLowerCase(); - const existing = topicScores.get(cat); - if (existing) { - existing.count++; - existing.score += msg.ai_moderation_score || 0; - } else { - topicScores.set(cat, { - count: 1, - score: msg.ai_moderation_score || 0, - }); - } - } - } catch { - /* not valid JSON */ - } - } - - if (msg.content) { - const words = msg.content - .toLowerCase() - .replace(/[^\w\s]/g, " ") - .split(/\s+/) - .filter((w) => w.length > 2 && !STOP_WORDS.has(w)); - - for (const word of words) { - wordFreq.set(word, (wordFreq.get(word) || 0) + 1); - if (msg.ai_status === "flagged" || msg.ai_status === "warn") { - flaggedWordFreq.set(word, (flaggedWordFreq.get(word) || 0) + 1); - } - } - } - } - - const results: TopicTrend[] = []; - for (const [topic, data] of topicScores) { - results.push({ topic, count: data.count, score: data.score }); - } - - const sortedWords = Array.from(wordFreq.entries()) - .sort(([, a], [, b]) => b - a) - .slice(0, topN); - - for (const [word, count] of sortedWords) { - if (!topicScores.has(word)) { - results.push({ - topic: word, - count, - score: flaggedWordFreq.get(word) || 0, - }); - } - } - - return results.sort((a, b) => b.count - a.count).slice(0, topN); -} - -export async function getTopicTrends(input: { - guildId: string; - channelId?: string; - hours?: number; -}): Promise { - const { guildId, channelId, hours = 24 } = input; - const cacheKey = makeCacheKey("topics", { guildId, channelId, hours }); - const cached = getCached(cacheKey); - if (cached) return cached; - - try { - const since = Date.now() - hours * 3600_000; - - // Fetch all analyzed messages within the time window (no hard row cap). - // Messages without ai_analysis are excluded which naturally limits rows. - const rows = (await executeAll( - ` - SELECT - id, content, ai_status, ai_analysis, ai_moderation_score, - ai_moderation_flags, created_at - FROM messages - WHERE guild_id = ? - AND created_at >= ? - AND deleted_at IS NULL - AND ai_analysis IS NOT NULL - ${channelId ? `AND (channel_id = ? OR thread_id = ?)` : ""} - ORDER BY created_at DESC - `, - channelId ? [guildId, since, channelId, channelId] : [guildId, since], - )) as MessageRecord[]; - - const result = extractTopics(rows); - setCache(cacheKey, result, TOPIC_CACHE_TTL_MS); - return result; - } catch (error) { - logger.error( - { error: error instanceof Error ? error.message : String(error) }, - "Failed to get topic trends", - ); - return []; - } -} - -// ── User Leaderboard ──────────────────────────────────────────────────── - -export async function getUserLeaderboard(input: { - guildId: string; - channelId?: string; - hours?: number; - limit?: number; -}): Promise { - const { guildId, channelId, hours = 24, limit = 20 } = input; - const cacheKey = makeCacheKey("leaderboard", { - guildId, - channelId, - hours, - limit, - }); - const cached = getCached(cacheKey); - if (cached) return cached; - - try { - const since = Date.now() - hours * 3600_000; - const rows = await executeAll( - ` - SELECT - user_id, - username, - avatar_url, - count(*) as message_count, - count(case when type = 'edited' then 1 end) as edited_count, - count(case when type = 'deleted' then 1 end) as deleted_count, - count(case when ai_status = 'flagged' then 1 end) as flagged_count, - max(created_at) as last_active - FROM messages - WHERE guild_id = ? - AND created_at >= ? - AND deleted_at IS NULL - ${channelId ? `AND (channel_id = ? OR thread_id = ?)` : ""} - GROUP BY user_id, username, avatar_url - ORDER BY message_count DESC - LIMIT ? - `, - channelId - ? [guildId, since, channelId, channelId, limit] - : [guildId, since, limit], - ); - - const result = rows as UserStat[]; - setCache(cacheKey, result, AGGREGATE_CACHE_TTL_MS); - return result; - } catch (error) { - logger.error( - { error: error instanceof Error ? error.message : String(error) }, - "Failed to get user leaderboard", - ); - return []; - } -} - -// ── Moderation Stats ─────────────────────────────────────────────────── - -export async function getModerationStats(input: { - guildId: string; - channelId?: string; - hours?: number; -}): Promise { - const { guildId, channelId, hours = 24 } = input; - const cacheKey = makeCacheKey("modstats", { guildId, channelId, hours }); - const cached = getCached(cacheKey); - if (cached) return cached; - - try { - const since = Date.now() - hours * 3600_000; - const avgScoreExpr = `round(avg(ai_moderation_score)::numeric, 2)`; - - const row = await executeGet( - ` - SELECT - count(*) as total, - count(case when ai_status = 'clean' then 1 end) as clean, - count(case when ai_status = 'warn' then 1 end) as warned, - count(case when ai_status = 'flagged' then 1 end) as flagged, - count(case when ai_status = 'error' then 1 end) as error, - count(case when ai_status = 'pending' or ai_status IS NULL then 1 end) as pending, - ${avgScoreExpr} as average_score - FROM messages - WHERE guild_id = ? - AND created_at >= ? - AND deleted_at IS NULL - ${channelId ? `AND (channel_id = ? OR thread_id = ?)` : ""} - `, - channelId ? [guildId, since, channelId, channelId] : [guildId, since], - ); - - const result: ModerationBreakdown = row - ? { - total: row.total ?? 0, - clean: row.clean ?? 0, - warned: row.warned ?? 0, - flagged: row.flagged ?? 0, - error: row.error ?? 0, - pending: row.pending ?? 0, - average_score: row.average_score ?? 0, - } - : { - total: 0, - clean: 0, - warned: 0, - flagged: 0, - error: 0, - pending: 0, - average_score: 0, - }; - - setCache(cacheKey, result, AGGREGATE_CACHE_TTL_MS); - return result; - } catch (error) { - logger.error( - { error: error instanceof Error ? error.message : String(error) }, - "Failed to get moderation stats", - ); - return { - total: 0, - clean: 0, - warned: 0, - flagged: 0, - error: 0, - pending: 0, - average_score: 0, - }; - } -} - -// ── Active Channels Count ────────────────────────────────────────────── - -export async function getActiveChannelCount(input: { - guildId: string; - hours?: number; -}): Promise { - const { guildId, hours = 24 } = input; - const cacheKey = makeCacheKey("channels", { guildId, hours }); - const cached = getCached(cacheKey); - if (cached !== undefined) return cached; - - try { - const since = Date.now() - hours * 3600_000; - const row = await executeGet( - ` - SELECT count(DISTINCT channel_id) as cnt - FROM messages - WHERE guild_id = ? - AND created_at >= ? - AND deleted_at IS NULL - `, - [guildId, since], - ); - - const result = row?.cnt ?? 0; - setCache(cacheKey, result, AGGREGATE_CACHE_TTL_MS); - return result; - } catch (error) { - logger.error( - { error: error instanceof Error ? error.message : String(error) }, - "Failed to get active channel count", - ); - return 0; - } -} - -// ── Top Violators ───────────────────────────────────────────────────── - -export interface ViolatorStat { - user_id: string; - username: string; - avatar_url: string | null; - total_messages: number; - flagged_count: number; - warned_count: number; - violation_score: number; - worst_flags: string[]; - last_violation: number; -} - -export async function getTopViolators(input: { - guildId: string; - channelId?: string; - hours?: number; - limit?: number; -}): Promise { - const { guildId, channelId, hours = 24, limit = 20 } = input; - const cacheKey = makeCacheKey("violators", { - guildId, - channelId, - hours, - limit, - }); - const cached = getCached(cacheKey); - if (cached) return cached; - - try { - const since = Date.now() - hours * 3600_000; - const rows = await executeAll( - ` - SELECT - user_id, - username, - avatar_url, - count(*) as total_messages, - count(case when ai_status = 'flagged' then 1 end) as flagged_count, - count(case when ai_status = 'warn' then 1 end) as warned_count, - max(case when ai_status in ('flagged', 'warn') then created_at else 0 end) as last_violation - FROM messages - WHERE guild_id = ? - AND created_at >= ? - AND deleted_at IS NULL - ${channelId ? `AND (channel_id = ? OR thread_id = ?)` : ""} - GROUP BY user_id, username, avatar_url - HAVING count(case when ai_status = 'flagged' then 1 end) > 0 - OR count(case when ai_status = 'warn' then 1 end) > 0 - ORDER BY ( - count(case when ai_status = 'flagged' then 1 end) * 3 - + count(case when ai_status = 'warn' then 1 end) - ) DESC - LIMIT ? - `, - channelId - ? [guildId, since, channelId, channelId, limit] - : [guildId, since, limit], - ); - - const violators: ViolatorStat[] = rows.map((row: any) => { - const flaggedCount = Number(row.flagged_count ?? 0); - const warnedCount = Number(row.warned_count ?? 0); - return { - user_id: row.user_id, - username: row.username, - avatar_url: row.avatar_url, - total_messages: Number(row.total_messages ?? 0), - flagged_count: flaggedCount, - warned_count: warnedCount, - violation_score: flaggedCount * 3 + warnedCount, - worst_flags: [], - last_violation: Number(row.last_violation ?? 0), - }; - }); - - setCache(cacheKey, violators, AGGREGATE_CACHE_TTL_MS); - return violators; - } catch (error) { - logger.error( - { error: error instanceof Error ? error.message : String(error) }, - "Failed to get top violators", - ); - return []; - } -} - -// ── Daily Trend (for multi-day line chart) ──────────────────────────── - -export interface TrendBucket { - date: string; - count: number; - clean: number; - warned: number; - flagged: number; - error: number; -} - -export async function getDailyTrend(input: { - guildId: string; - channelId?: string; - hours?: number; -}): Promise { - const { guildId, channelId, hours = 168 } = input; - const cacheKey = makeCacheKey("daily_trend", { guildId, channelId, hours }); - const cached = getCached(cacheKey); - if (cached) return cached; - - try { - const since = Date.now() - hours * 3600_000; - const dateExpr = `to_char(date_trunc('day', to_timestamp(created_at / 1000)), 'YYYY-MM-DD') as date`; - - const rows = await executeAll( - ` - SELECT - ${dateExpr}, - count(*) as count, - count(case when ai_status = 'clean' then 1 end) as clean, - count(case when ai_status = 'warn' then 1 end) as warned, - count(case when ai_status = 'flagged' then 1 end) as flagged, - count(case when ai_status = 'error' then 1 end) as error - FROM messages - WHERE guild_id = ? - AND created_at >= ? - AND deleted_at IS NULL - ${channelId ? `AND (channel_id = ? OR thread_id = ?)` : ""} - GROUP BY 1 - ORDER BY 1 ASC - `, - channelId ? [guildId, since, channelId, channelId] : [guildId, since], - ); - - // Initialize all day buckets (fill gaps with zeros) - const buckets = new Map< - string, - { - count: number; - clean: number; - warned: number; - flagged: number; - error: number; - } - >(); - const msPerDay = 86400_000; - const startDay = Math.floor(since / msPerDay) * msPerDay; - const endDay = Math.floor(Date.now() / msPerDay) * msPerDay; - - for (let d = startDay; d <= endDay; d += msPerDay) { - const key = new Date(d).toISOString().slice(0, 10); - buckets.set(key, { count: 0, clean: 0, warned: 0, flagged: 0, error: 0 }); - } - - for (const row of rows) { - const bucket = buckets.get(row.date); - if (!bucket) continue; - bucket.count = row.count; - bucket.clean = row.clean; - bucket.warned = row.warned; - bucket.flagged = row.flagged; - bucket.error = row.error; - } - - const result = Array.from(buckets.entries()) - .sort(([a], [b]) => a.localeCompare(b)) - .map(([date, data]) => ({ date, ...data })); - - setCache(cacheKey, result, AGGREGATE_CACHE_TTL_MS); - return result; - } catch (error) { - logger.error( - { error: error instanceof Error ? error.message : String(error) }, - "Failed to get daily trend", - ); - return []; - } -} - -// ── Activity Heatmap (day-of-week × hour-of-day) ────────────────────── - -export interface HeatmapCell { - dayOfWeek: number; // 0=Senin, 6=Minggu - hour: number; // 0-23 - count: number; - clean: number; - warned: number; - flagged: number; -} - -export async function getActivityHeatmap(input: { - guildId: string; - channelId?: string; - hours?: number; -}): Promise { - const { guildId, channelId, hours = 168 } = input; - const cacheKey = makeCacheKey("heatmap", { guildId, channelId, hours }); - const cached = getCached(cacheKey); - if (cached) return cached; - - try { - const since = Date.now() - hours * 3600_000; - const dayExpr = `(extract(isodow from to_timestamp(created_at / 1000)) % 7)::int as day_of_week`; - const hourExpr = `extract(hour from to_timestamp(created_at / 1000))::int as hour`; - - const rows = await executeAll( - ` - SELECT - ${dayExpr}, - ${hourExpr}, - count(*) as count, - count(case when ai_status = 'clean' then 1 end) as clean, - count(case when ai_status = 'warn' then 1 end) as warned, - count(case when ai_status = 'flagged' then 1 end) as flagged - FROM messages - WHERE guild_id = ? - AND created_at >= ? - AND deleted_at IS NULL - ${channelId ? `AND (channel_id = ? OR thread_id = ?)` : ""} - GROUP BY day_of_week, hour - ORDER BY day_of_week, hour - `, - channelId ? [guildId, since, channelId, channelId] : [guildId, since], - ); - - // Initialize all 7×24 cells with zeros - const cells = new Map< - string, - { count: number; clean: number; warned: number; flagged: number } - >(); - for (let d = 0; d < 7; d++) { - for (let h = 0; h < 24; h++) { - cells.set(`${d}-${h}`, { count: 0, clean: 0, warned: 0, flagged: 0 }); - } - } - - for (const row of rows) { - const key = `${row.day_of_week}-${row.hour}`; - const cell = cells.get(key); - if (!cell) continue; - cell.count = row.count; - cell.clean = row.clean; - cell.warned = row.warned; - cell.flagged = row.flagged; - } - - const result = Array.from(cells.entries()) - .map(([key, data]) => { - const [dayOfWeek, hour] = key.split("-").map(Number); - return { dayOfWeek, hour, ...data }; - }) - .sort((a, b) => a.dayOfWeek - b.dayOfWeek || a.hour - b.hour); - - setCache(cacheKey, result, AGGREGATE_CACHE_TTL_MS); - return result; - } catch (error) { - logger.error( - { error: error instanceof Error ? error.message : String(error) }, - "Failed to get activity heatmap", - ); - return []; - } -} - -// ── Cache Invalidation (called when new messages arrive) ─────────────── - -export function invalidateAnalyticsCache(guildId: string): void { - const now = Date.now(); - const needle = `"${guildId}"`; - for (const [key, entry] of queryCache) { - if (key.includes(needle) && entry.expiresAt > now) { - entry.expiresAt = 0; // expire immediately - } - } -} - -// ── Combined Overview ────────────────────────────────────────────────── - -export async function getAnalyticsOverview(input: { - guildId: string; - channelId?: string; - hours?: number; -}): Promise { - const { guildId, hours = 24 } = input; - const now = Date.now(); - const since = now - hours * 3600_000; - - const [messages, hourly, topics, topUsers, totalChannels] = await Promise.all( - [ - getModerationStats(input), - getHourlyStats(input), - getTopicTrends(input), - getUserLeaderboard(input), - getActiveChannelCount({ guildId, hours }), - ], - ); - - return { - period: { start: since, end: now }, - messages, - hourly, - topics, - top_users: topUsers, - active_users_count: topUsers.length, - total_channels: totalChannels, - }; -} diff --git a/src/moderation/attachmentUploader.ts b/src/moderation/attachmentUploader.ts deleted file mode 100644 index db4c603..0000000 --- a/src/moderation/attachmentUploader.ts +++ /dev/null @@ -1,133 +0,0 @@ -import { config } from "../config.js"; -import { createChildLogger } from "../logger.js"; -import { uploadToTele } from "../uploader/teleUpload.js"; -import { - updateAttachmentAsFailedUpload, - updateAttachmentAsUploaded, - updateAttachmentDiscordUrl, -} from "./messageStore.js"; - -const logger = createChildLogger("attachment-uploader"); - -class AttachmentDownloadError extends Error { - constructor( - message: string, - readonly status: number, - ) { - super(message); - this.name = "AttachmentDownloadError"; - } -} - -export type RefreshDiscordAttachmentUrl = () => Promise; - -function toErrorMessage(error: unknown): string { - return error instanceof Error ? error.message : String(error); -} - -function shouldRefreshDiscordUrl(error: unknown): boolean { - return ( - error instanceof AttachmentDownloadError && - (error.status === 403 || error.status === 404) - ); -} - -export async function uploadAttachmentToTele( - fileBuffer: Buffer, - filename: string, - contentType = "application/octet-stream", -): Promise { - try { - const result = await uploadToTele({ - buffer: fileBuffer, - filename, - contentType, - uploadUrl: config.TELE_UPLOAD_URL, - timeoutMs: config.ATTACHMENT_UPLOAD_TIMEOUT_MS, - retries: 0, - logger, - }); - - return result.url; - } catch (error) { - logger.error( - { - filename, - error: toErrorMessage(error), - }, - "Failed to upload attachment", - ); - throw error; - } -} - -export async function downloadDiscordAttachment(url: string): Promise { - try { - const response = await fetch(url, { - signal: AbortSignal.timeout(config.ATTACHMENT_UPLOAD_TIMEOUT_MS), - }); - - if (!response.ok) { - throw new AttachmentDownloadError( - `Download failed with status ${response.status}`, - response.status, - ); - } - - const buffer = await response.arrayBuffer(); - return Buffer.from(buffer); - } catch (error) { - logger.error( - { url, error: toErrorMessage(error) }, - "Failed to download Discord attachment", - ); - throw error; - } -} - -export async function processAttachmentUpload( - attachmentId: string, - discordUrl: string, - filename: string, - options: { - refreshDiscordUrl?: RefreshDiscordAttachmentUrl; - contentType?: string; - } = {}, -): Promise { - try { - let currentDiscordUrl = discordUrl; - let buffer: Buffer; - try { - buffer = await downloadDiscordAttachment(currentDiscordUrl); - } catch (error) { - if (!options.refreshDiscordUrl || !shouldRefreshDiscordUrl(error)) { - throw error; - } - - const freshUrl = await options.refreshDiscordUrl(); - if (!freshUrl) throw error; - currentDiscordUrl = freshUrl; - await updateAttachmentDiscordUrl(attachmentId, freshUrl); - buffer = await downloadDiscordAttachment(currentDiscordUrl); - } - - const sizeMb = buffer.length / (1024 * 1024); - if (sizeMb > config.ATTACHMENT_MAX_SIZE_MB) { - throw new Error( - `File size ${sizeMb.toFixed(2)}MB exceeds limit of ${config.ATTACHMENT_MAX_SIZE_MB}MB`, - ); - } - - const uploadedUrl = await uploadAttachmentToTele( - buffer, - filename, - options.contentType, - ); - - await updateAttachmentAsUploaded(attachmentId, uploadedUrl, Date.now()); - } catch (error) { - const errorMsg = toErrorMessage(error); - await updateAttachmentAsFailedUpload(attachmentId, errorMsg); - logger.error({ attachmentId, error: errorMsg }, "Attachment upload failed"); - } -} diff --git a/src/moderation/autoDeleteManager.ts b/src/moderation/autoDeleteManager.ts deleted file mode 100644 index 3ab37ea..0000000 --- a/src/moderation/autoDeleteManager.ts +++ /dev/null @@ -1,355 +0,0 @@ -import type { Client, PermissionString } from "discord.js-selfbot-v13"; -import { config } from "../config.js"; -import { createChildLogger } from "../logger.js"; -import { createModerationAction } from "./messageStore.js"; -import type { MessageRecord } from "./types.js"; - -const logger = createChildLogger("auto-delete-manager"); - -const parseStringList = (value?: string | null): string[] => { - if (!value) return []; - try { - const parsed = JSON.parse(value) as unknown; - return Array.isArray(parsed) - ? parsed.filter((item): item is string => typeof item === "string") - : []; - } catch { - return value - .split(",") - .map((item) => item.trim()) - .filter(Boolean); - } -}; - -/** Derive severity from legacy messages that lack structured AI fields. */ -function deriveSeverity(msg: MessageRecord): string { - if (msg.ai_severity) return msg.ai_severity; - const score = msg.ai_confidence ?? msg.ai_moderation_score ?? 0; - if (msg.ai_status === "flagged") - return score >= 0.9 ? "critical" : score >= 0.7 ? "high" : "medium"; - if (msg.ai_status === "warn") return score >= 0.6 ? "medium" : "low"; - return "none"; -} - -/** Derive recommended action from legacy messages that lack structured AI fields. */ -function deriveRecommendedAction(msg: MessageRecord): string { - if (msg.ai_recommended_action) return msg.ai_recommended_action; - const severity = deriveSeverity(msg); - if ( - msg.ai_status === "flagged" && - (severity === "critical" || severity === "high") - ) - return "delete"; - if (msg.ai_status === "flagged") return "review"; - if (msg.ai_status === "warn") return "warn"; - return "none"; -} - -function isAutoDeleteEligible(message: MessageRecord): boolean { - if (message.ai_status !== "flagged" && message.ai_status !== "warn") - return false; - - const confidence = message.ai_confidence ?? message.ai_moderation_score ?? 0; - if (confidence < config.AUTO_DELETE_MIN_CONFIDENCE) { - logger.info( - { - messageId: message.id, - confidence, - threshold: config.AUTO_DELETE_MIN_CONFIDENCE, - }, - "Auto-delete skipped: confidence below threshold", - ); - return false; - } - - const severity = deriveSeverity(message); - const allowedSeverities = (config.AUTO_DELETE_ALLOWED_SEVERITIES || "") - .split(",") - .map((s) => s.trim()) - .filter(Boolean); - if (allowedSeverities.length > 0 && !allowedSeverities.includes(severity)) { - logger.info( - { messageId: message.id, severity, allowed: allowedSeverities }, - "Auto-delete skipped: severity not in allowed list", - ); - return false; - } - - const recommendedAction = deriveRecommendedAction(message); - if (recommendedAction !== "delete" && recommendedAction !== "escalate") { - logger.info( - { messageId: message.id, recommendedAction }, - "Auto-delete skipped: recommended action is not delete/escalate", - ); - return false; - } - - const allowedCategories = parseStringList( - config.AUTO_DELETE_ALLOWED_CATEGORIES, - ); - if (allowedCategories.length > 0) { - const messageCategories = parseStringList( - message.ai_categories ?? message.ai_moderation_flags, - ); - const hasAllowedCategory = messageCategories.some((cat) => - allowedCategories.includes(cat), - ); - if (!hasAllowedCategory) { - logger.info( - { - messageId: message.id, - categories: messageCategories, - allowed: allowedCategories, - }, - "Auto-delete skipped: no allowed categories match", - ); - return false; - } - } - - const excludedChannels = parseStringList( - config.AUTO_DELETE_EXCLUDED_CHANNEL_IDS, - ); - if (excludedChannels.length > 0) { - const channelId = message.thread_id ?? message.channel_id; - if (excludedChannels.includes(channelId)) { - logger.info( - { messageId: message.id, channelId }, - "Auto-delete skipped: channel excluded", - ); - return false; - } - } - - const excludedUsers = parseStringList(config.AUTO_DELETE_EXCLUDED_USER_IDS); - if (excludedUsers.length > 0 && excludedUsers.includes(message.user_id)) { - logger.info( - { messageId: message.id, userId: message.user_id }, - "Auto-delete skipped: user excluded", - ); - return false; - } - - return true; -} - -async function logAutoDeleteAttempt( - message: MessageRecord, - result: AutoDeleteResult, -): Promise { - try { - await createModerationAction({ - message_id: message.id, - user_id: message.user_id, - guild_id: message.guild_id, - action_type: "delete_message", - reason: result.reason, - executed_by: "auto-delete-manager", - status: result.deleted - ? "executed" - : result.reason === "dry_run" - ? "executed" - : "failed", - error: result.reason === "error" ? result.reason : null, - executed_at: - result.deleted || result.reason === "dry_run" ? Date.now() : null, - }); - } catch (error) { - logger.warn( - { - messageId: message.id, - error: error instanceof Error ? error.message : String(error), - }, - "Failed to persist auto-delete action log", - ); - } -} - -export interface AutoDeleteResult { - deleted: boolean; - skipped: boolean; - reason: string; -} - -function getErrorCode(error: unknown): number | string | undefined { - if (!error || typeof error !== "object") return undefined; - const maybeCode = (error as { code?: number | string }).code; - const maybeStatus = (error as { status?: number | string }).status; - return maybeCode ?? maybeStatus; -} - -function isAlreadyDeletedError(error: unknown): boolean { - const code = getErrorCode(error); - return code === 10008 || code === 404 || code === "10008" || code === "404"; -} - -function hasChannelMessagesApi(channel: unknown): channel is { - messages: { - fetch: (id: string) => Promise<{ delete: () => Promise }>; - }; -} { - return Boolean( - channel && - typeof channel === "object" && - "messages" in channel && - (channel as { messages?: unknown }).messages && - typeof (channel as { messages: { fetch?: unknown } }).messages.fetch === - "function", - ); -} - -function hasPermissionApi(channel: unknown): channel is { - permissionsFor: ( - member: unknown, - ) => { has: (permission: string) => boolean } | null; -} { - return Boolean( - channel && - typeof channel === "object" && - "permissionsFor" in channel && - typeof (channel as { permissionsFor?: unknown }).permissionsFor === - "function", - ); -} - -export async function attemptAutoDeleteFlaggedMessage( - client: Client | undefined, - message: MessageRecord, -): Promise { - if (!config.AUTO_DELETE_FLAGGED_ENABLED) { - return { deleted: false, skipped: true, reason: "disabled" }; - } - - if (message.ai_status !== "flagged" && message.ai_status !== "warn") { - const result = { - deleted: false, - skipped: true, - reason: "not_flagged_or_warn", - } as AutoDeleteResult; - await logAutoDeleteAttempt(message, result); - return result; - } - - if (!isAutoDeleteEligible(message)) { - const result = { - deleted: false, - skipped: true, - reason: "not_eligible", - } as AutoDeleteResult; - await logAutoDeleteAttempt(message, result); - return result; - } - - if (!client?.user?.id) { - logger.warn( - { messageId: message.id }, - "Auto-delete skipped: client user missing", - ); - return { deleted: false, skipped: true, reason: "client_user_missing" }; - } - - try { - const guild = client.guilds.cache.get(message.guild_id); - if (!guild) { - logger.warn( - { messageId: message.id, guildId: message.guild_id }, - "Auto-delete skipped: guild not found", - ); - return { deleted: false, skipped: true, reason: "guild_not_found" }; - } - - const channelId = message.thread_id ?? message.channel_id; - const channel = guild.channels.cache.get(channelId); - if (!channel) { - logger.warn( - { messageId: message.id, channelId }, - "Auto-delete skipped: channel not found", - ); - return { deleted: false, skipped: true, reason: "channel_not_found" }; - } - - if (!hasPermissionApi(channel) || !hasChannelMessagesApi(channel)) { - logger.warn( - { messageId: message.id, channelId }, - "Auto-delete skipped: channel cannot delete messages", - ); - return { deleted: false, skipped: true, reason: "unsupported_channel" }; - } - - const selfMember = await guild.members.fetch(client.user.id); - const permissions = channel.permissionsFor(selfMember); - const canManageMessages = - permissions?.has("MANAGE_MESSAGES" as PermissionString) ?? false; - - if (!canManageMessages) { - logger.warn( - { messageId: message.id, channelId, userId: client.user.id }, - "Auto-delete skipped: current user lacks Manage Messages", - ); - return { - deleted: false, - skipped: true, - reason: "missing_manage_messages", - }; - } - - if (config.AUTO_DELETE_FLAGGED_DRY_RUN) { - const result = { - deleted: false, - skipped: true, - reason: "dry_run", - } as AutoDeleteResult; - await logAutoDeleteAttempt(message, result); - logger.info( - { messageId: message.id, channelId }, - "Auto-delete dry-run: would delete flagged message", - ); - return result; - } - - const discordMessage = await channel.messages.fetch(message.id); - await discordMessage.delete(); - - const result = { - deleted: true, - skipped: false, - reason: "deleted", - } as AutoDeleteResult; - await logAutoDeleteAttempt(message, result); - logger.info( - { messageId: message.id, channelId }, - "Auto-deleted AI-flagged message", - ); - return result; - } catch (error) { - if (isAlreadyDeletedError(error)) { - const result = { - deleted: true, - skipped: false, - reason: "already_deleted", - } as AutoDeleteResult; - await logAutoDeleteAttempt(message, result); - logger.info( - { messageId: message.id, code: getErrorCode(error) }, - "Auto-delete skipped: message already deleted", - ); - return result; - } - - const result = { - deleted: false, - skipped: true, - reason: "error", - } as AutoDeleteResult; - await logAutoDeleteAttempt(message, result); - logger.error( - { - messageId: message.id, - error: error instanceof Error ? error.message : String(error), - code: getErrorCode(error), - }, - "Auto-delete failed", - ); - return result; - } -} diff --git a/src/moderation/backlogSync.ts b/src/moderation/backlogSync.ts deleted file mode 100644 index ee0d3f5..0000000 --- a/src/moderation/backlogSync.ts +++ /dev/null @@ -1,113 +0,0 @@ -import type { Channel, Client, Message } from "discord.js-selfbot-v13"; -import { config } from "../config.js"; -import { createChildLogger } from "../logger.js"; -import { captureMessage } from "./messageCapture.js"; - -const logger = createChildLogger("backlog-sync"); - -type BacklogChannel = Channel & { - messages: { - fetch(options: { limit: number; before?: string }): Promise<{ - size: number; - values(): IterableIterator; - }>; - }; -}; - -function hasMessageBacklog(channel: Channel): channel is BacklogChannel { - return "messages" in channel; -} - -async function syncChannelMessages( - channel: BacklogChannel, - cutoffTime: number, -): Promise { - let before: string | undefined; - let synced = 0; - let shouldContinue = true; - - while (shouldContinue) { - const batch = await channel.messages.fetch({ - limit: config.BACKLOG_SYNC_BATCH_SIZE, - ...(before ? { before } : {}), - }); - - if (batch.size === 0) break; - - const messages = Array.from(batch.values()) as Message[]; - for (const message of messages) { - if (message.author?.bot) continue; - if (message.createdTimestamp < cutoffTime) { - shouldContinue = false; - continue; - } - - await captureMessage(message, "text", { source: "backlog" }); - synced++; - } - - before = messages[messages.length - 1]?.id; - if (!before || batch.size < config.BACKLOG_SYNC_BATCH_SIZE) break; - } - - return synced; -} - -export async function syncBacklogMessages(client: Client): Promise { - const textGuildId = config.EFFECTIVE_TEXT_GUILD_ID; - if (!textGuildId) { - logger.warn("TEXT_GUILD_ID not configured, skipping backlog sync"); - return; - } - - const guild = client.guilds.cache.get(textGuildId); - if (!guild) { - logger.warn( - { guildId: textGuildId }, - "Text guild not found, skipping backlog sync", - ); - return; - } - - if (config.TEXT_CHANNEL_ID) { - await syncSelectedChannelBacklog(client, guild.id, config.TEXT_CHANNEL_ID); - return; - } -} - -export async function syncSelectedChannelBacklog( - client: Client, - guildId: string, - channelId: string, -): Promise { - const guild = client.guilds.cache.get(guildId); - if (!guild) { - logger.warn({ guildId }, "Guild not found for backlog sync"); - return 0; - } - - const channel = guild.channels.cache.get(channelId); - if (!channel) { - logger.warn({ guildId, channelId }, "Channel not found for backlog sync"); - return 0; - } - if (!hasMessageBacklog(channel)) { - logger.warn({ guildId, channelId }, "Channel cannot fetch message backlog"); - return 0; - } - - const cutoffTime = Date.now() - config.BACKLOG_SYNC_HOURS * 60 * 60 * 1000; - try { - const count = await syncChannelMessages(channel, cutoffTime); - return count; - } catch (error) { - logger.warn( - { - channelId, - error: error instanceof Error ? error.message : String(error), - }, - "Backlog sync failed for selected channel", - ); - return 0; - } -} diff --git a/src/moderation/broadcaster.ts b/src/moderation/broadcaster.ts deleted file mode 100644 index ebb1296..0000000 --- a/src/moderation/broadcaster.ts +++ /dev/null @@ -1,82 +0,0 @@ -import type { WebSocket } from "ws"; -import { createChildLogger } from "../logger.js"; -import type { - AnalysisQueueStatus, - AttachmentRecord, - MediaState, - MessageRecord, - ModerationWsEvent, -} from "./types.js"; - -export type BroadcasterClient = Pick; - -const log = createChildLogger("broadcaster"); - -function sendJson( - clients: Set, - event: ModerationWsEvent, -): void { - const payload = JSON.stringify({ ...event, timestamp: Date.now() }); - for (const client of clients) { - if (client.readyState === 1) { - try { - client.send(payload); - } catch (error) { - log.warn( - { error, eventType: event.type }, - "Failed to send event to client", - ); - } - } - } -} - -export function createBroadcaster() { - const clients = new Set(); - - return { - addClient(client: BroadcasterClient) { - clients.add(client); - log.debug({ clientCount: clients.size }, "Client added"); - }, - removeClient(client: BroadcasterClient) { - clients.delete(client); - log.debug({ clientCount: clients.size }, "Client removed"); - }, - clientCount() { - return clients.size; - }, - getClients() { - return Array.from(clients); - }, - uiState(state: unknown) { - sendJson(clients, { type: "ui_state", state }); - }, - userState(users: unknown[]) { - sendJson(clients, { type: "user_state", users }); - }, - messageCreated(data: MessageRecord) { - sendJson(clients, { type: "message_created", data }); - }, - messageUpdated(data: Partial & { id: string }) { - sendJson(clients, { type: "message_updated", data }); - }, - messageDeleted(data: { id: string; deleted_at: number }) { - sendJson(clients, { type: "message_deleted", data }); - }, - messageAnalyzed(data: MessageRecord) { - sendJson(clients, { type: "message_analyzed", data }); - }, - attachmentCreated(data: AttachmentRecord) { - sendJson(clients, { type: "attachment_created", data }); - }, - analysisQueueStatus(data: AnalysisQueueStatus) { - sendJson(clients, { type: "analysis_queue_status", data }); - }, - mediaState(state: MediaState) { - sendJson(clients, { type: "media_state", state }); - }, - }; -} - -export type ModerationBroadcaster = ReturnType; diff --git a/src/moderation/concurrencyLimiter.ts b/src/moderation/concurrencyLimiter.ts deleted file mode 100644 index 8bf220a..0000000 --- a/src/moderation/concurrencyLimiter.ts +++ /dev/null @@ -1,14 +0,0 @@ -import pLimit from "p-limit"; -import { config } from "../config.js"; - -/** - * Concurrency limiter for LLM API calls. - * - * Prevents rate-limit (429) errors by capping simultaneous requests - * to the configured maximum (default: 5). - */ -const llmSemaphore = pLimit(config.AI_LLM_MAX_CONCURRENT ?? 5); - -export async function withLlmConcurrency(fn: () => Promise): Promise { - return llmSemaphore(fn); -} diff --git a/src/moderation/conversationContext.ts b/src/moderation/conversationContext.ts deleted file mode 100644 index 24b8cab..0000000 --- a/src/moderation/conversationContext.ts +++ /dev/null @@ -1,77 +0,0 @@ -import { formatModerationTextEvidenceForPrompt } from "./indonesianTextNormalizer.js"; -import { formatMediaEvidenceForPrompt } from "./messageMetadata.js"; -import type { MessageRecord } from "./types.js"; - -export interface ConversationContextInput { - contextBefore: MessageRecord[]; - targets: MessageRecord[]; - maxTokens: number; -} - -/** - * Formats a timestamp to ISO 8601 string - */ -function formatTimestamp(ms: number): string { - return new Date(ms).toISOString(); -} - -/** - * Estimates token count for a string (pessimistic approximation for Indonesian slang & JSON overhead) - */ -export function estimateTokens(text: string): number { - return Math.ceil(text.length / 3) + 15; -} - -/** - * Formats a single message for context or target display - */ -export async function formatMessageForPrompt( - msg: MessageRecord, - label: "context" | "target", -): Promise { - const content = msg.edited_content ?? msg.content; - const timestamp = formatTimestamp(msg.created_at); - const textEvidence = await formatModerationTextEvidenceForPrompt(content); - const textSuffix = textEvidence ? ` ${textEvidence}` : ""; - const mediaEvidence = formatMediaEvidenceForPrompt(msg.metadata); - const mediaSuffix = mediaEvidence ? ` ${mediaEvidence}` : ""; - return `[${label}] id=${msg.id} time=${timestamp} user=${msg.username}: ${content}${textSuffix}${mediaSuffix}`; -} - -/** - * Builds conversation historical context without including targets. - * Calculates how much token budget targets use, and fills the rest with context. - */ -export async function buildConversationContext( - input: ConversationContextInput, -): Promise { - const { contextBefore, targets, maxTokens } = input; - - // Calculate tokens used by targets (parallel) - const targetLines = await Promise.all( - targets.map((msg) => formatMessageForPrompt(msg, "target")), - ); - let usedTokens = targetLines.reduce( - (sum, line) => sum + estimateTokens(line), - 0, - ); - - const contextLines = await Promise.all( - contextBefore.map((msg) => formatMessageForPrompt(msg, "context")), - ); - const selectedContextLines: string[] = []; - - // Go backwards through context, taking most recent first - for (let i = contextLines.length - 1; i >= 0; i--) { - const line = contextLines[i]; - const lineTokens = estimateTokens(line); - - if (usedTokens + lineTokens <= maxTokens) { - // Unshift so oldest context is first in the array - selectedContextLines.unshift(line); - usedTokens += lineTokens; - } - } - - return selectedContextLines; -} diff --git a/src/moderation/imageResizer.ts b/src/moderation/imageResizer.ts deleted file mode 100644 index 0c11ecc..0000000 --- a/src/moderation/imageResizer.ts +++ /dev/null @@ -1,58 +0,0 @@ -import sharp from "sharp"; -import { createChildLogger } from "../logger.js"; - -const log = createChildLogger("imageResizer"); - -/** - * Resize an image buffer for optimal vision LLM analysis. - * - * - Resizes to maxDim x maxDim maintaining aspect ratio - * - Converts to JPEG at quality 85 for size reduction - * - Falls back to original buffer if sharp fails - * - * @param buf - Raw image buffer - * @param maxDim - Maximum dimension in pixels (default 1024) - * @returns Resized buffer with detected MIME type - */ -export async function resizeImageForVision( - buf: Buffer, - maxDim = 1024, -): Promise<{ data: Buffer; mimeType: string }> { - try { - const metadata = await sharp(buf).metadata(); - const inputFormat = metadata.format ?? "jpeg"; - - // Skip resize if already smaller than maxDim - if ((metadata.width ?? 0) <= maxDim && (metadata.height ?? 0) <= maxDim) { - return { data: buf, mimeType: `image/${inputFormat}` }; - } - - const resized = await sharp(buf) - .resize(maxDim, maxDim, { - fit: "inside", - withoutEnlargement: true, - }) - .jpeg({ quality: 85 }) - .toBuffer(); - - log.debug( - { - originalSize: buf.length, - resizedSize: resized.length, - reductionPct: Math.round( - ((buf.length - resized.length) / buf.length) * 100, - ), - }, - "Image resized for vision analysis", - ); - - return { data: resized, mimeType: "image/jpeg" }; - } catch (error) { - log.warn( - { error: error instanceof Error ? error.message : String(error) }, - "Image resize failed — using original buffer", - ); - // Fallback: return original buffer with best-effort MIME type - return { data: buf, mimeType: "image/jpeg" }; - } -} diff --git a/src/moderation/indonesianTextNormalizer.ts b/src/moderation/indonesianTextNormalizer.ts deleted file mode 100644 index d24498c..0000000 --- a/src/moderation/indonesianTextNormalizer.ts +++ /dev/null @@ -1,222 +0,0 @@ -import { createChildLogger } from "../logger.js"; -import { getCachedText, upsertCachedText } from "./textCacheStore.js"; -import { llmDetectBadwords } from "./llmClient.js"; - -const log = createChildLogger("indonesianTextNormalizer"); - -const CUSTOM_EMOJI_PATTERN = //g; - -/** - * In-memory cache TTL (10 min) — fastest path for repeated identical texts. - */ -const BADWORD_CACHE_TTL_MS = 10 * 60 * 1000; - -/** - * DB cache TTL (24 hours) — survives restarts, stores full-text results - * so context is preserved (e.g. "kaus" is clean, "kau" alone is clean, - * but "awas kau" is harassment). - */ -const DB_CACHE_TTL_MS = 24 * 60 * 60 * 1000; - -interface BadwordCacheEntry { - value: string[]; - expiresAt: number; -} - -const badwordCache = new Map(); -const inFlightBadwordLookups = new Map>(); - -export interface ModerationTextEvidence { - raw: string; - normalized: string; - notes: string[]; - badwords: string[]; - hasBadwords: boolean; -} - -// --------------------------------------------------------------------------- -// Sync helpers -// --------------------------------------------------------------------------- - -export function normalizeDiscordCustomEmoji(text: string): { - text: string; - emojiNames: string[]; -} { - const emojiNames: string[] = []; - const normalized = text.replace( - CUSTOM_EMOJI_PATTERN, - (_match, name: string) => { - emojiNames.push(name); - return `[emoji:${name}]`; - }, - ); - - return { text: normalized, emojiNames }; -} - -function normalizeBadwordCacheKey(text: string): string { - return text.trim().replace(/\s+/g, " ").toLowerCase(); -} - -function getCachedBadwords(key: string): string[] | null { - const entry = badwordCache.get(key); - if (!entry) return null; - if (entry.expiresAt <= Date.now()) { - badwordCache.delete(key); - return null; - } - return [...entry.value]; -} - -function setCachedBadwords(key: string, value: string[]): void { - badwordCache.set(key, { - value: [...new Set(value)], - expiresAt: Date.now() + BADWORD_CACHE_TTL_MS, - }); - - if (badwordCache.size > 500) { - const now = Date.now(); - for (const [cacheKey, entry] of badwordCache) { - if (entry.expiresAt <= now) { - badwordCache.delete(cacheKey); - } - } - - if (badwordCache.size > 500) { - const oldestKeys = Array.from(badwordCache.entries()) - .sort((a, b) => a[1].expiresAt - b[1].expiresAt) - .slice(0, badwordCache.size - 500) - .map(([cacheKey]) => cacheKey); - for (const cacheKey of oldestKeys) { - badwordCache.delete(cacheKey); - } - } - } -} - -// --------------------------------------------------------------------------- -// Two-tier cache + Primary AI pipeline -// --------------------------------------------------------------------------- - -/** - * Detect badwords in text using a **two-tier cache + Primary AI**: - * - * 1. **In-memory cache** (BADWORD_CACHE_TTL_MS, 10 min) — fastest path, - * keyed by the full normalized text string. - * 2. **DB cache** (DB_CACHE_TTL_MS, 24 h) — same full-text key, persisted - * across restarts. Uses the FULL normalized text (not per-word) because - * context matters: "kau" alone is clean, but "awas kau" can be a threat. - * 3. **Primary AI** (AI_LLM endpoint via llmClient) — only runs when both - * cache layers miss. - * - * No local hardcoded badword list — all detection goes through AI APIs - * to eliminate false positives from substring matching. - */ -export async function detectIndonesianBadwords( - text: string, -): Promise { - const cacheKey = normalizeBadwordCacheKey(text); - - // ── Tier 1: In-memory cache (fastest) ── - const cached = getCachedBadwords(cacheKey); - if (cached) { - return cached; - } - - // De-duplicate concurrent lookups - const inFlight = inFlightBadwordLookups.get(cacheKey); - if (inFlight) { - return inFlight; - } - - const lookupPromise = (async () => { - // ── Tier 2: DB cache (survives restarts, preserves context) ── - const dbEntry = await getCachedText(cacheKey); - if (dbEntry) { - const flags = [...dbEntry.flags]; - setCachedBadwords(cacheKey, flags); // populate in-memory too - return flags; - } - - // ── Tier 3: Primary AI only (via centralized llmClient) ── - let finalHits: string[] = []; - try { - finalHits = await llmDetectBadwords(text); - } catch (error) { - log.warn( - { error: error instanceof Error ? error.message : String(error) }, - "Primary AI badword detection failed", - ); - } - - // Populate all cache tiers so the same text never triggers another API call - // within the TTL window. - setCachedBadwords(cacheKey, finalHits); - await upsertCachedText( - cacheKey, - finalHits, - "primary_ai", - Date.now() + DB_CACHE_TTL_MS, - ); - - return finalHits; - })(); - - inFlightBadwordLookups.set(cacheKey, lookupPromise); - - try { - return await lookupPromise; - } finally { - inFlightBadwordLookups.delete(cacheKey); - } -} - -// --------------------------------------------------------------------------- -// Async evidence builders -// --------------------------------------------------------------------------- - -export async function buildModerationTextEvidence( - text: string, -): Promise { - const emojiNormalized = normalizeDiscordCustomEmoji(text); - const badwordHits = await detectIndonesianBadwords(emojiNormalized.text); - const notes: string[] = []; - - for (const emojiName of emojiNormalized.emojiNames) { - notes.push( - `emoji:${emojiName}=Discord custom emoji/expression; not text offense by default`, - ); - } - - if (badwordHits.length > 0) { - notes.push(`Indonesian badword detected: ${badwordHits.join(", ")}`); - } else { - notes.push("no Indonesian badword detected"); - } - - return { - raw: text, - normalized: emojiNormalized.text, - notes: Array.from(new Set(notes)), - badwords: badwordHits, - hasBadwords: badwordHits.length > 0, - }; -} - -export async function formatModerationTextEvidenceForPrompt( - text: string, -): Promise { - const evidence = await buildModerationTextEvidence(text); - if (evidence.normalized === evidence.raw && evidence.notes.length === 0) { - return ""; - } - - return [ - `[normalized_text: ${evidence.normalized}]`, - evidence.notes.length > 0 - ? `[normalization_notes: ${evidence.notes.join("; ")}]` - : null, - ] - .filter(Boolean) - .join(" "); -} diff --git a/src/moderation/llmClient.ts b/src/moderation/llmClient.ts deleted file mode 100644 index 3b520d6..0000000 --- a/src/moderation/llmClient.ts +++ /dev/null @@ -1,276 +0,0 @@ -/** - * Centralised LLM chat completion helper. - * - * All `openai.chat.completions.create` calls in the moderation subsystem - * go through this module so that model, concurrency, retry, and token - * defaults are maintained in one place. - */ - -import OpenAI from "openai"; -import { config } from "../config.js"; -import { retryWithBackoff } from "../retry.js"; -import { withLlmConcurrency } from "./concurrencyLimiter.js"; -import { createChildLogger } from "../logger.js"; - -const log = createChildLogger("llm-client"); - -// --------------------------------------------------------------------------- -// Lazy singleton — created on first use so that config is always resolved. -// --------------------------------------------------------------------------- - -let openaiClient: OpenAI | null = null; - -function getClient(): OpenAI | null { - if (!config.AI_LLM_API_KEY) return null; - if (!openaiClient) { - openaiClient = new OpenAI({ - apiKey: config.AI_LLM_API_KEY, - baseURL: config.AI_LLM_BASE_URL, - maxRetries: 0, - timeout: 15_000, - }); - } - return openaiClient; -} - -// --------------------------------------------------------------------------- -// Shared defaults -// --------------------------------------------------------------------------- - -const DEFAULT_TEMPERATURE = 0.2; -const DEFAULT_TOP_P = 0.95; -const DEFAULT_RETRIES = 2; - -// --------------------------------------------------------------------------- -// Public API -// --------------------------------------------------------------------------- - -export interface LlmCallOpts { - /** Conversation to send. Either a string (→ single user message) or an array of messages. */ - messages: OpenAI.Chat.Completions.ChatCompletionMessageParam[]; - /** Which model to use (defaults to config.AI_LLM_MODEL). */ - model?: string; - /** Max output tokens (defaults to 8192). */ - max_tokens?: number; - /** Temperature (defaults to 0.2). */ - temperature?: number; - /** Top-p (defaults to 0.95). */ - top_p?: number; - /** Force JSON output. When true, wraps schema in json_schema response_format. */ - jsonResponse?: - | { type: "json_object" } - | { - type: "json_schema"; - name: string; - schema: Record; - strict: boolean; - }; - /** Extra retries beyond DEFAULT_RETRIES (default 2). */ - retries?: number; -} - -/** - * Call the LLM with sensible defaults: concurrency cap, retry, model, tokens. - * - * Returns the raw OpenAI ChatCompletion so callers can inspect - * `choices[0].message.content`, `finish_reason`, `usage`, etc. - */ -export async function llmChat( - opts: LlmCallOpts, -): Promise { - const client = getClient(); - if (!client) return null; - - const { - messages, - model = config.AI_LLM_MODEL, - max_tokens = 8192, - temperature = DEFAULT_TEMPERATURE, - top_p = DEFAULT_TOP_P, - jsonResponse, - retries = DEFAULT_RETRIES, - } = opts; - - const responseFormat: - | { type: "json_object" } - | { - type: "json_schema"; - json_schema: { - name: string; - schema: Record; - strict: boolean; - }; - } - | undefined = jsonResponse - ? jsonResponse.type === "json_schema" - ? { - type: "json_schema", - json_schema: { - name: jsonResponse.name, - schema: jsonResponse.schema, - strict: jsonResponse.strict, - }, - } - : jsonResponse - : undefined; - - const params: OpenAI.Chat.Completions.ChatCompletionCreateParamsNonStreaming = - { - model, - messages, - temperature, - top_p, - max_tokens, - stream: false, - ...(responseFormat ? { response_format: responseFormat } : {}), - } as OpenAI.Chat.Completions.ChatCompletionCreateParamsNonStreaming; - - return retryWithBackoff( - async () => { - return withLlmConcurrency(async () => - client.chat.completions.create(params), - ); - }, - { - retries, - minTimeout: 0, - maxTimeout: 0, - factor: 2, - logger: log, - }, - ); -} - -/** - * Convenience for the legacy text-only badword detection call in - * `indonesianTextNormalizer`. Returns parsed flags or []. - */ -export async function llmDetectBadwords(text: string): Promise { - const completion = await llmChat({ - messages: [ - { - role: "user", - content: - "Deteksi kata kasar / pelanggaran ringan dari teks Indonesia berikut. " + - 'Balas hanya JSON object dengan format {"flags":[...]} dan gunakan hanya flag valid ini: ' + - Array.from(VALID_PRIMARY_AI_FLAGS).join(", ") + - ". Jika tidak ada pelanggaran, flags harus array kosong. Teks: " + - text, - }, - ], - max_tokens: 200, - temperature: 0.1, - top_p: 0.9, - jsonResponse: { type: "json_object" }, - retries: 2, - }); - - if (!completion) return []; - const content = completion.choices[0]?.message?.content?.trim(); - if (!content) return []; - return extractFlagsFromContent(content); -} - -/** - * Convenience for vision (image/sticker/emoji) analysis. - * Returns the raw completion content (trimmed) or null. - */ -export async function llmVision( - promptText: string, - imageUrl: { url: string }, -): Promise { - const completion = await llmChat({ - messages: [ - { - role: "user", - content: [ - { type: "text" as const, text: promptText }, - { type: "image_url" as const, image_url: imageUrl }, - ], - }, - ], - model: config.AI_LLM_VISION_MODEL ?? config.AI_LLM_MODEL, - max_tokens: 500, - temperature: 0.1, - top_p: 0.9, - retries: 2, - }); - - if (!completion) return null; - return completion.choices[0]?.message?.content?.trim() ?? null; -} - -// --------------------------------------------------------------------------- -// Flag extraction (reused from indonesianTextNormalizer) -// --------------------------------------------------------------------------- - -const VALID_PRIMARY_AI_FLAGS = new Set([ - "spam", - "hate_speech", - "sara", - "hoaks", - "harassment", - "vulgar_language", - "sexual_content", - "sexual_deviation", - "violence", - "self_harm", - "doxxing", - "scam", - "misinformation", - "nsfw_image", - "gore_image", - "illegal_content", - "gambling", - "drugs", - "child_safety", - "financial_scam", - "religious_insult", - "self_promo", -]); - -function normalizeFlag(value: string): string | null { - const lower = value.trim().toLowerCase().replace(/[\s-]+/g, "_"); - if (!lower) return null; - if (VALID_PRIMARY_AI_FLAGS.has(lower)) return lower; - return null; -} - -function extractFlagsFromContent(content: string): string[] { - const flags = new Set(); - let parsed: unknown; - try { - parsed = JSON.parse(content); - } catch { - parsed = null; - } - - const addValue = (v: unknown) => { - if (typeof v !== "string") return; - const n = normalizeFlag(v); - if (n) flags.add(n); - }; - - if (Array.isArray(parsed)) { - for (const item of parsed) addValue(item); - } else if (parsed && typeof parsed === "object") { - const obj = parsed as Record; - for (const key of ["flags", "categories", "badwords"]) { - const val = obj[key]; - if (Array.isArray(val)) { - for (const item of val) addValue(item); - } else { - addValue(val); - } - } - } - - if (flags.size > 0) return Array.from(flags); - - const lower = content.toLowerCase(); - for (const flag of VALID_PRIMARY_AI_FLAGS) { - if (lower.includes(flag)) flags.add(flag); - } - - return Array.from(flags); -} diff --git a/src/moderation/llmModerationClient.ts b/src/moderation/llmModerationClient.ts deleted file mode 100644 index 7614d87..0000000 --- a/src/moderation/llmModerationClient.ts +++ /dev/null @@ -1,1282 +0,0 @@ -import type { ChatCompletion } from "openai/resources/chat/completions"; -import { AbortError } from "p-retry"; -import { z } from "zod"; -import { config } from "../config.js"; -import { createChildLogger } from "../logger.js"; -import { retryWithBackoff } from "../retry.js"; -import { resizeImageForVision } from "./imageResizer.js"; -import { formatModerationTextEvidenceForPrompt } from "./indonesianTextNormalizer.js"; -import { llmChat, llmVision } from "./llmClient.js"; -import { extractMessageMediaEvidence } from "./messageMetadata.js"; -import { buildSystemPrompt as buildSystemPromptModular } from "./moderationPrompt.js"; -import { - getStickerFromCache, - initStickerCache, - isStickerCacheReady, - setStickerInCache, -} from "./stickerCache.js"; -import { - buildCustomEmojiVisionPrompt, - buildGeneralImageVisionPrompt, - buildStickerTextOnlyWarning, - buildStickerVisionPrompt, -} from "./stickerPrompt.js"; -import { - getCachedMediaAnalysis, - makeCustomEmojiCacheKey, - makeImageCacheKey, - makeStickerCacheKey, - upsertCachedMediaAnalysis, -} from "./textCacheStore.js"; -import type { - AnalysisResult, - AttachmentRecord, - MessageRecord, -} from "./types.js"; -import { extractUrlsFromText, fetchUrlSafely } from "./urlFetcher.js"; - -const SeveritySchema = z.enum(["none", "low", "medium", "high", "critical"]); -const RecommendedActionSchema = z.enum([ - "none", - "monitor", - "warn", - "review", - "delete", - "escalate", -]); - -const ResultItemSchema = z.object({ - message_id: z.union([z.string(), z.number()]).transform(String), - status: z.enum(["clean", "warn", "flagged"]), - flags: z.array(z.string()).optional(), - score: z.number(), - analysis: z.string().nullable().optional(), - categories: z.array(z.string()).optional(), - severity: SeveritySchema.optional(), - confidence: z.number().optional(), - recommended_action: RecommendedActionSchema.optional(), - policy_version: z.string().optional(), - evidence: z.array(z.string()).optional(), -}); - -const ModerationResponseSchema = z.object({ - results: z.array(ResultItemSchema), -}); - -const log = createChildLogger("llmModerationClient"); - -/** - * Enhanced deferral detection pattern (R9). - * - * Only matches patterns where the model explicitly states it cannot make - * a decision and needs human review. Removed overly broad patterns that - * caused false positives: - * - "admin (perlu|harus|sebaiknya)" → common in regular sentences - * - "bisa (berpotensi|mengandung)" → decisive statements, not deferral - * - "maaf|sorry" → opinions/apologies, not deferral - * - "saya tidak yakin|tahu|paham" → expressing uncertainty, not deferral - */ -const DEFERRAL_ANALYSIS_PATTERN = - /(?:kurang (?:konteks|bukti|informasi|data) (?:untuk (?:menilai|menentukan|memutuskan)|untuk moderasi)|perlu (?:dicek|diperiksa|ditinjau|dikaji|dievaluasi) (?:oleh )?(?:admin|moderator|manusia|human review)|tidak (?:bisa|dapat|mampu) (?:menentukan|menilai|memastikan|menyimpulkan|memberi keputusan|memoderasi).*(?:karena (?:konteks tidak jelas|informasi tidak cukup|bukti kurang|konteks kurang|tidak cukup konteks)|data tidak cukup|informasi tidak lengkap)|cannot determine|insufficient (?:context|evidence|information) (?:to |for )?(?:moderate|judge|evaluate|decide|classify)|(?:sepertinya|tampaknya) (?:perlu|harus) (?:ditinjau|diperiksa|dicek) (?:oleh )?(?:admin|moderator)|tidak cukup (?:bukti|informasi|konteks) (?:untuk (?:memberikan|membuat|menentukan)|memutuskan))/i; - -/** - * Exceptions: patterns that look like deferral but are actually decisive. - * Expanded to catch more variations where the model gives a clear verdict. - */ -const DEFERRAL_EXCEPTION_PATTERN = - /tidak bisa menentukan.*(?:karena|sebab|dengan alasan|sebab tidak ada).*(?:clean|tidak (?:ada|terdapat|menunjukkan).*(?:pelanggaran|masalah|indikasi|konten)|aman|bersih|normal)/i; - -function hasDeferralAnalysis(analysis: string): boolean { - if (DEFERRAL_EXCEPTION_PATTERN.test(analysis)) return false; - return DEFERRAL_ANALYSIS_PATTERN.test(analysis); -} - -function clampScore(value: number | undefined, fallback = 0): number { - return Math.max( - 0, - Math.min(1, Number.isFinite(value) ? (value as number) : fallback), - ); -} - -function deriveSeverity( - status: "clean" | "warn" | "flagged", - score: number, -): z.infer { - if (status === "clean") return "none"; - if (status === "warn") return score >= 0.65 ? "medium" : "low"; - if (score >= 0.9) return "critical"; - return score >= 0.75 ? "high" : "medium"; -} - -function deriveRecommendedAction( - status: "clean" | "warn" | "flagged", - severity: z.infer, -): z.infer { - if (status === "clean") return "none"; - if (status === "warn") return severity === "medium" ? "review" : "warn"; - if (severity === "critical") return "escalate"; - if (severity === "high") return "delete"; - return "review"; -} - -/** - * JSON Schema for OpenAI's response_format: { type: "json_schema" }. - * This enforces the exact structure the LLM must output (R2). - */ -const MODERATION_JSON_SCHEMA = { - type: "object", - properties: { - results: { - type: "array", - items: { - type: "object", - properties: { - message_id: { type: "string" }, - status: { type: "string", enum: ["clean", "warn", "flagged"] }, - flags: { type: "array", items: { type: "string" } }, - score: { type: "number", minimum: 0, maximum: 1 }, - analysis: { type: "string" }, - categories: { type: "array", items: { type: "string" } }, - severity: { - type: "string", - enum: ["none", "low", "medium", "high", "critical"], - }, - confidence: { type: "number", minimum: 0, maximum: 1 }, - recommended_action: { - type: "string", - enum: ["none", "monitor", "warn", "review", "delete", "escalate"], - }, - policy_version: { type: "string" }, - evidence: { type: "array", items: { type: "string" } }, - }, - required: [ - "message_id", - "status", - "flags", - "score", - "severity", - "confidence", - "recommended_action", - "policy_version", - "evidence", - "analysis", - ], - additionalProperties: false, - }, - }, - }, - required: ["results"], - additionalProperties: false, -}; - -/** - * Helper to extract JSON from a potentially conversational or markdown-wrapped string. - */ -export function extractJson(content: string): any { - const codeBlockRegex = /```(?:json)?\s*([\s\S]*?)\s*```/g; - const matches = content.matchAll(codeBlockRegex); - for (const match of matches) { - const codeContent = match[1].trim(); - try { - const parsed = JSON.parse(codeContent); - if (parsed && typeof parsed === "object") { - return parsed; - } - } catch (_) {} - } - - for (let start = 0; start < content.length; start++) { - const firstChar = content[start]; - if (firstChar !== "{" && firstChar !== "[") continue; - - const stack = [firstChar]; - let inString = false; - let escaped = false; - - for (let i = start + 1; i < content.length; i++) { - const char = content[i]; - - if (inString) { - if (escaped) { - escaped = false; - } else if (char === "\\") { - escaped = true; - } else if (char === '"') { - inString = false; - } - continue; - } - - if (char === '"') { - inString = true; - continue; - } - - if (char === "{" || char === "[") { - stack.push(char); - continue; - } - - const last = stack[stack.length - 1]; - if ((char === "}" && last === "{") || (char === "]" && last === "[")) { - stack.pop(); - if (stack.length === 0) { - const candidate = content.slice(start, i + 1); - try { - const parsed = JSON.parse(candidate); - if (parsed && typeof parsed === "object") { - return parsed; - } - } catch (_) {} - break; - } - } - } - } - - throw new Error("No JSON object found in response"); -} - -/** - * Sanitize error messages for client-facing output (R10). - * Internal details are logged but the caller gets a generic message. - */ -function sanitizeErrorMessage(internalMsg: string, messageId: string): string { - // Log the full error for debugging - log.warn( - { messageId, internalError: internalMsg }, - "Internal moderation error (sanitized for client)", - ); - // Return generic message without internal details - return `Analisis gagal dan memerlukan pemeriksaan manual. Error code: MOD_${Date.now().toString(36).slice(0, 6)}`; -} - -export function parseModerationResponse( - content: string, - targetIds: string[], -): AnalysisResult[] { - let parsed: any; - try { - parsed = JSON.parse(content); - } catch (e) { - parsed = extractJson(content); - } - - if (Array.isArray(parsed)) { - parsed = { results: parsed }; - } else if (parsed && typeof parsed === "object" && !("results" in parsed)) { - if ("message_id" in parsed) { - parsed = { results: [parsed] }; - } else { - const arrayKey = Object.keys(parsed).find((key) => { - const val = (parsed as any)[key]; - return ( - Array.isArray(val) && - val.length > 0 && - val.every( - (item: unknown) => - typeof item === "object" && - item !== null && - "message_id" in (item as any), - ) - ); - }); - if (arrayKey) { - parsed.results = (parsed as any)[arrayKey]; - } else { - parsed = { results: [parsed] }; - } - } - } - - const parseResult = ModerationResponseSchema.safeParse(parsed); - if (!parseResult.success) { - throw new Error(`Zod validation failed: ${parseResult.error.message}`); - } - - const response = parseResult.data; - const foundIds = new Set(); - const targetIdSet = new Set(targetIds); - - const results: (AnalysisResult | null)[] = response.results.map((result) => { - const { - message_id, - status, - flags, - score, - analysis, - categories, - severity, - confidence, - recommended_action, - policy_version, - evidence, - } = result; - const finalId = message_id.trim(); - - if (!targetIdSet.has(finalId)) { - return null; - } - - if (foundIds.has(finalId)) { - throw new Error( - `Duplicate message_id in moderation response: ${finalId}`, - ); - } - - foundIds.add(finalId); - - const coalescedAnalysis = analysis ?? ""; - - if (hasDeferralAnalysis(coalescedAnalysis)) { - throw new Error( - `Deferral analysis is not allowed for message ${finalId}; return a direct moderation decision`, - ); - } - - const normalizedScore = clampScore(score); - const normalizedConfidence = clampScore(confidence, normalizedScore); - const normalizedSeverity = - severity ?? deriveSeverity(status, normalizedScore); - - return { - messageId: finalId, - status: status as "clean" | "warn" | "flagged", - flags: flags ?? [], - score: normalizedScore, - analysis: coalescedAnalysis, - categories: categories ?? flags ?? [], - severity: normalizedSeverity, - confidence: normalizedConfidence, - recommendedAction: - recommended_action ?? - deriveRecommendedAction(status, normalizedSeverity), - policyVersion: policy_version ?? "default-2026-05-30", - evidence: evidence ?? [], - }; - }); - - const filteredResults = results.filter( - (r): r is AnalysisResult => r !== null, - ); - - const missingIds = targetIds.filter((id) => !foundIds.has(id)); - if (missingIds.length > 0) { - log.warn( - { missingIds, foundCount: foundIds.size, totalCount: targetIds.length }, - "Some target IDs missing in response - marking as incomplete", - ); - for (const missingId of missingIds) { - filteredResults.push({ - messageId: missingId, - status: "error", - flags: ["analysis_incomplete"], - score: 0, - analysis: sanitizeErrorMessage( - "Analysis incomplete - LLM did not process this message", - missingId, - ), - categories: ["analysis_incomplete"], - severity: "none", - confidence: 0, - recommendedAction: "review", - policyVersion: "default-2026-05-30", - evidence: [], - }); - } - } - - return filteredResults; -} - -interface ModerationInput { - targets: MessageRecord[]; - contextText: string; - attachments?: AttachmentRecord[]; -} - -interface ModerationOutput { - results: AnalysisResult[]; - raw: unknown; -} - -/** - * Sniff the first bytes of a buffer to determine if it is a supported image - * format. Returns the canonical MIME type string on success, or null if the - * bytes are not a recognizable image. - */ -function sniffImageMimeType(buf: Buffer): string | null { - if (buf.length < 12) return null; - - if (buf[0] === 0xff && buf[1] === 0xd8 && buf[2] === 0xff) { - return "image/jpeg"; - } - - if ( - buf[0] === 0x89 && - buf[1] === 0x50 && - buf[2] === 0x4e && - buf[3] === 0x47 && - buf[4] === 0x0d && - buf[5] === 0x0a && - buf[6] === 0x1a && - buf[7] === 0x0a - ) { - return "image/png"; - } - - if ( - buf[0] === 0x47 && - buf[1] === 0x49 && - buf[2] === 0x46 && - buf[3] === 0x38 - ) { - return "image/gif"; - } - - if ( - buf[0] === 0x52 && - buf[1] === 0x49 && - buf[2] === 0x46 && - buf[3] === 0x46 && - buf[8] === 0x57 && - buf[9] === 0x45 && - buf[10] === 0x42 && - buf[11] === 0x50 - ) { - return "image/webp"; - } - - if ( - buf.length >= 12 && - buf[4] === 0x66 && - buf[5] === 0x74 && - buf[6] === 0x79 && - buf[7] === 0x70 - ) { - const brand = buf.subarray(8, 12).toString("ascii"); - if (brand.startsWith("avif") || brand.startsWith("avis")) { - return "image/avif"; - } - if ( - brand.startsWith("mif1") || - brand.startsWith("heic") || - brand.startsWith("heis") - ) { - return "image/heic"; - } - } - - return null; -} - -// --------------------------------------------------------------------------- -// Shared types for image resolution -// --------------------------------------------------------------------------- - -type MessageImagePart = { - type: "image_url"; - image_url: { url: string }; - sourceLabel: string; - stickerName?: string; - customEmojiId?: string; - customEmojiName?: string; -}; - -// --------------------------------------------------------------------------- -// Media detection helper -// --------------------------------------------------------------------------- - -function hasMediaContent( - target: MessageRecord, - attachments?: AttachmentRecord[], -): boolean { - if (target.metadata) { - const evidence = extractMessageMediaEvidence(target.metadata); - if (evidence.stickers.length > 0 || evidence.embeds.length > 0) return true; - } - if (attachments?.some((a) => a.message_id === target.id)) return true; - return false; -} - -// --------------------------------------------------------------------------- -// Single-image vision analysis (reused by both text-only and media paths) -// --------------------------------------------------------------------------- - -const analyzeSingleMediaImage = async ( - messageId: string, - image: MessageImagePart, -): Promise => { - const cacheKey = image.customEmojiId - ? makeCustomEmojiCacheKey(image.customEmojiId) - : image.stickerName - ? makeStickerCacheKey(image.stickerName) - : makeImageCacheKey(image.image_url.url); - - const cached = await getCachedMediaAnalysis(cacheKey); - if (cached) { - log.debug({ cacheKey }, "Media analysis cache HIT"); - return `[Media analysis for message ${messageId}] ${image.sourceLabel}: ${cached}`; - } - - const promptText = image.stickerName - ? buildStickerVisionPrompt(image.stickerName, messageId) - : image.customEmojiName - ? buildCustomEmojiVisionPrompt(image.customEmojiName, messageId) - : buildGeneralImageVisionPrompt(image.sourceLabel, messageId); - - try { - const content = await llmVision(promptText, image.image_url); - if (!content) return null; - - await upsertCachedMediaAnalysis( - cacheKey, - content, - "vision_llm", - Date.now() + 24 * 60 * 60 * 1000, - ); - - return `[Media analysis for message ${messageId}] ${image.sourceLabel}: ${content}`; - } catch (error) { - log.warn( - { - messageId, - error: error instanceof Error ? error.message : String(error), - }, - "Separate media analysis failed", - ); - return `[Media analysis for message ${messageId}] ${image.sourceLabel}: GAGAL DIANALISIS — gambar tidak dapat diunduh atau vision API gagal setelah 3x percobaan. JANGAN mengasumsikan gambar aman hanya karena gagal dianalisis. Gunakan metadata URL/nama file saja sebagai petunjuk.`; - } -}; - -// --------------------------------------------------------------------------- -// Shared LLM call + parse + fallback helper -// --------------------------------------------------------------------------- - -/** - * State object shared between the caller and callModerationLLM so that - * parse-error feedback can be injected into subsequent retry attempts. - * - * The caller creates this object, passes it to callModerationLLM, and - * the internal retry loop mutates it before re-invoking buildContent(). - */ -interface RetryState { - lastParseError: string | null; - lastInvalidContent: string | null; -} - -/** - * Execute a single LLM moderation call (batch or single-message) with retry - * logic, JSON parse, and fallback error markers on failure. - * - * Uses JSON Schema response format (R2) and concurrency limiter (R3). - */ -async function callModerationLLM( - buildContent: (state: RetryState) => Promise, - targetIds: string[], - label: string, -): Promise<{ - results: AnalysisResult[]; - raw: ChatCompletion | null; -}> { - const state: RetryState = { - lastParseError: null, - lastInvalidContent: null, - }; - - let parsed: AnalysisResult[]; - let result: ChatCompletion | null = null; - - try { - const analysis = await retryWithBackoff( - async () => { - try { - const content = await buildContent(state); - - const completion = await llmChat({ - messages: [{ role: "user", content }], - max_tokens: 16384, - jsonResponse: { - type: "json_schema", - name: "moderation_result", - schema: MODERATION_JSON_SCHEMA, - strict: true, - }, - retries: 0, - }); - - if (!completion) { - throw new Error("LLM client unavailable (no API key)"); - } - - if ( - !completion.choices || - !Array.isArray(completion.choices) || - !completion.choices[0] - ) { - throw new Error("Invalid LLM response structure"); - } - - const rawContent = completion.choices[0].message?.content; - if (!rawContent) { - throw new Error("No content in LLM response"); - } - - try { - return { - parsed: parseModerationResponse(rawContent, targetIds), - result: completion, - }; - } catch (parseError) { - state.lastParseError = - parseError instanceof Error - ? parseError.message - : String(parseError); - state.lastInvalidContent = rawContent; - log.warn( - { - error: state.lastParseError, - contentLength: rawContent.length, - contentPreview: rawContent.substring(0, 1000), - targetIds, - model: config.AI_LLM_MODEL, - }, - `Failed to parse moderation response from LLM (${label})`, - ); - throw parseError; - } - } catch (apiError: any) { - if ( - apiError?.status === 429 || - apiError?.status === 401 || - apiError?.status === 403 - ) { - throw new AbortError(apiError); - } - throw apiError; - } - }, - { - retries: 0, - logger: log, - }, - ); - parsed = analysis.parsed; - result = analysis.result; - } catch (parseError) { - if (!state.lastInvalidContent) { - throw parseError; - } - - const errorMsg = - parseError instanceof Error ? parseError.message : String(parseError); - - log.error( - { - error: errorMsg, - contentLength: state.lastInvalidContent.length, - contentPreview: state.lastInvalidContent.substring(0, 500), - targetIds, - model: config.AI_LLM_MODEL, - timestamp: new Date().toISOString(), - }, - `Robust Fallback (${label}): Failed to parse moderation response. Marking all targets as analysis errors.`, - ); - - // Sanitized error messages — no internal details exposed (R10) - const errorCode = `MOD_${Date.now().toString(36).slice(0, 6)}`; - parsed = targetIds.map((id) => ({ - messageId: id, - status: "error", - flags: ["analysis_parse_failed"], - score: 0, - analysis: `Analisis gagal dan memerlukan pemeriksaan manual. Error code: ${errorCode}`, - categories: ["analysis_parse_failed"], - severity: "none", - confidence: 0, - recommendedAction: "review", - policyVersion: "default-2026-05-30", - evidence: [], - })); - } - - return { results: parsed, raw: result }; -} - -// --------------------------------------------------------------------------- -// Text-only fast path — with batch size splitting (R6) -// --------------------------------------------------------------------------- - -/** - * Run a lightweight batch analysis on text-only messages. - * - * If targets exceed AI_LLM_TEXT_BATCH_SIZE, split into sub-batches - * and run sequentially to avoid overwhelming the LLM (R6). - */ -async function runTextOnlyBatch( - targets: MessageRecord[], - contextText: string, -): Promise<{ results: AnalysisResult[]; raw: unknown }> { - if (!targets.length) return { results: [], raw: null }; - - const maxBatchSize = config.AI_LLM_TEXT_BATCH_SIZE ?? 20; - - // Pre-compute text evidence (normalization + badword detection) - const textEvidenceMap = new Map(); - await Promise.all( - targets.map(async (msg) => { - const content = msg.edited_content ?? msg.content; - const evidence = await formatModerationTextEvidenceForPrompt(content); - textEvidenceMap.set(msg.id, evidence); - }), - ); - - // Split into sub-batches if needed (R6) - const subBatches: MessageRecord[][] = []; - for (let i = 0; i < targets.length; i += maxBatchSize) { - subBatches.push(targets.slice(i, i + maxBatchSize)); - } - - if (subBatches.length > 1) { - log.info( - { - totalTargets: targets.length, - subBatchCount: subBatches.length, - maxBatchSize, - }, - "Text targets exceed batch size limit — splitting into sub-batches", - ); - } - - const allResults: AnalysisResult[] = []; - let lastRaw: unknown = null; - - // Run sub-batches sequentially to avoid rate limits - for (let i = 0; i < subBatches.length; i++) { - const batch = subBatches[i]; - const targetIds = batch.map((t) => t.id); - - const buildContent = async (state: RetryState): Promise => { - const correction = state.lastParseError - ? { - error: state.lastParseError, - preview: state.lastInvalidContent?.slice(0, 800) ?? "", - } - : undefined; - - // Use modular system prompt with XML delimiters (R1, R7, R8) - const systemText = buildSystemPromptModular({ - contextText, - includeMediaInstructions: false, - correction, - }); - - const messagesBlock = batch - .map((msg) => { - const content = msg.edited_content ?? msg.content; - const textEvidence = textEvidenceMap.get(msg.id) ?? ""; - const textContext = textEvidence ? `\n${textEvidence}` : ""; - // XML delimiters wrap each message for prompt safety (R1) - return `${content}${textContext}`; - }) - .join("\n"); - - // XML delimiter wraps the entire messages block (R1) - return `${systemText}\n\n\n${messagesBlock}\n`; - }; - - const batchResult = await callModerationLLM( - buildContent, - targetIds, - `text-batch-${i + 1}`, - ); - - allResults.push(...batchResult.results); - if (batchResult.raw) lastRaw = batchResult.raw; - } - - log.info( - { - targetCount: targets.length, - resultCount: allResults.length, - subBatchCount: subBatches.length, - }, - "Text-only batch analysis complete", - ); - - return { results: allResults, raw: lastRaw }; -} - -// --------------------------------------------------------------------------- -// Single media message analysis — one LLM call per message with vision + timeout (R4, R5) -// --------------------------------------------------------------------------- - -/** - * Process a single media-bearing message: - * 1. Download attachment images (resized via sharp — R5) - * 2. Fetch URLs found in the message body - * 3. Download sticker/embed images (resized via sharp — R5) - * 4. Run vision analysis on every image (with DB + sticker cache) - * 5. Build a single-message prompt with XML delimiters (R1) - * 6. One LLM call → single AnalysisResult - * - * Wrapped with overall timeout (R4). - */ -async function runSingleMediaAnalysis( - target: MessageRecord, - contextText: string, - allAttachments: AttachmentRecord[] | undefined, -): Promise<{ results: AnalysisResult[]; raw: unknown }> { - const targetId = target.id; - const targetIds = [targetId]; - - // Timeout wrapper (R4) - const timeoutMs = config.AI_LLM_MEDIA_ANALYSIS_TIMEOUT_MS ?? 60000; - - return Promise.race([ - _runSingleMediaAnalysis( - target, - contextText, - allAttachments, - targetId, - targetIds, - ), - new Promise<{ results: AnalysisResult[]; raw: unknown }>((_, reject) => { - const timeout = setTimeout( - () => - reject( - new Error( - `Media analysis timed out after ${timeoutMs}ms for message ${targetId}`, - ), - ), - timeoutMs, - ); - timeout.unref(); - }), - ]); -} - -async function _runSingleMediaAnalysis( - target: MessageRecord, - contextText: string, - allAttachments: AttachmentRecord[] | undefined, - targetId: string, - targetIds: string[], -): Promise<{ results: AnalysisResult[]; raw: unknown }> { - // Lazy init sticker cache - if (!isStickerCacheReady()) { - await initStickerCache().catch((err) => { - log.warn( - { error: err instanceof Error ? err.message : String(err) }, - "Sticker cache init failed — continuing without cache", - ); - }); - } - - // ── State maps for this single message ── - const imageMap = new Map(); - const webTextMap = new Map(); - const mediaAnalysisMap = new Map(); - - const getAttachmentImageUrl = (att: AttachmentRecord): string | null => - att.uploaded_url ?? null; - - const maxDimension = config.AI_LLM_IMAGE_MAX_DIMENSION ?? 1024; - - // ── 1. Download attachments for this message (with resize — R5) ── - const msgAttachments = (allAttachments ?? []) - .filter( - (att) => - att.message_id === targetId && - getAttachmentImageUrl(att) && - att.type.startsWith("image/"), - ) - .slice(0, 8); - - await Promise.all( - msgAttachments.map(async (att) => { - const urlToUse = getAttachmentImageUrl(att); - if (!urlToUse) return; - - // Check vision cache BEFORE downloading - const attVisionKey = makeImageCacheKey(urlToUse); - const cachedVision = await getCachedMediaAnalysis(attVisionKey); - if (cachedVision) { - log.debug( - { attachmentId: att.id, cacheKey: attVisionKey }, - "Vision cache HIT for attachment — skipped download", - ); - const sourceLabel = `[gambar di atas adalah attachment ${att.filename} dari pesan id=${att.message_id}]`; - const analysisText = `[Media analysis for message ${att.message_id}] ${sourceLabel}: ${cachedVision}`; - const existing = mediaAnalysisMap.get(targetId) ?? []; - existing.push(analysisText); - mediaAnalysisMap.set(targetId, existing); - return; - } - - const controller = new AbortController(); - const timeoutId = setTimeout(() => controller.abort(), 15000); - - try { - const res = await fetch(urlToUse, { signal: controller.signal }); - if (!res.ok || !res.body) return; - - let totalBytes = 0; - const chunks: Uint8Array[] = []; - const reader = res.body.getReader(); - - while (true) { - const { done, value } = await reader.read(); - if (done) break; - if (value) { - totalBytes += value.length; - if (totalBytes > 10 * 1024 * 1024) { - reader.cancel(); - return; - } - chunks.push(value); - } - } - - const imageBytes = Buffer.concat(chunks); - const sniffedMime = sniffImageMimeType(imageBytes); - if (!sniffedMime) { - log.warn( - { attachmentId: att.id }, - "Skipping attachment: not a recognised image format", - ); - return; - } - - // Resize before base64 encoding (R5) - const { data: resizedBuffer, mimeType: resizedMime } = - await resizeImageForVision(imageBytes, maxDimension); - - const dataUrl = `data:${resizedMime};base64,${resizedBuffer.toString("base64")}`; - const part: MessageImagePart = { - type: "image_url", - image_url: { url: dataUrl }, - sourceLabel: `[gambar di atas adalah attachment ${att.filename} dari pesan id=${att.message_id}]`, - }; - const existing = imageMap.get(targetId) ?? []; - existing.push(part); - imageMap.set(targetId, existing); - } catch (err) { - log.warn( - { - attachmentId: att.id, - error: err instanceof Error ? err.message : String(err), - }, - "Error downloading attachment", - ); - } finally { - clearTimeout(timeoutId); - } - }), - ); - - // ── 2. Fetch URLs found in message text ── - const content = target.edited_content ?? target.content; - const urls = extractUrlsFromText(content).slice(0, 3); - - if (urls.length > 0) { - const webTexts: string[] = []; - await Promise.all( - urls.map(async (url) => { - const result = await fetchUrlSafely(url); - if (result.type === "image" && result.data && result.mimeType) { - // Resize fetched images too (R5) - const { data: resizedBuffer, mimeType: resizedMime } = - await resizeImageForVision(result.data, maxDimension); - - const dataUrl = `data:${resizedMime};base64,${resizedBuffer.toString("base64")}`; - const part: MessageImagePart = { - type: "image_url", - image_url: { url: dataUrl }, - sourceLabel: `[gambar di atas berasal dari link ${url} pada pesan id=${targetId}]`, - }; - const existing = imageMap.get(targetId) ?? []; - existing.push(part); - imageMap.set(targetId, existing); - } else if (result.type === "text" && result.textContent) { - webTexts.push(`[Isi Web dari ${url}]: ${result.textContent}`); - } - }), - ); - if (webTexts.length > 0) webTextMap.set(targetId, webTexts); - } - - // ── 3. Sticker / embed / custom emoji images ── - const mediaEvidence = extractMessageMediaEvidence(target.metadata); - const mediaCandidates: Array<{ - messageId: string; - url: string; - label: string; - stickerName?: string; - customEmojiId?: string; - customEmojiName?: string; - }> = [ - ...mediaEvidence.stickers - .filter((s) => s.url) - .map((s) => ({ - messageId: targetId, - url: s.url, - label: `[gambar di atas adalah sticker "${s.name}" dari pesan id=${targetId}]`, - stickerName: s.name, - })), - ...mediaEvidence.embeds.flatMap((embed) => - [ - embed.image - ? { - messageId: targetId, - url: embed.image, - label: `[gambar di atas berasal dari embed image pada pesan id=${targetId}]`, - } - : null, - embed.thumbnail - ? { - messageId: targetId, - url: embed.thumbnail, - label: `[gambar di atas berasal dari embed thumbnail pada pesan id=${targetId}]`, - } - : null, - ].filter( - ( - c, - ): c is { - messageId: string; - url: string; - label: string; - stickerName?: string; - customEmojiId?: string; - customEmojiName?: string; - } => c !== null, - ), - ), - ...mediaEvidence.customEmojis.map((emoji) => ({ - messageId: targetId, - url: emoji.url, - label: `[gambar di atas adalah custom emoji "${emoji.name}" dari pesan id=${targetId}]`, - customEmojiId: emoji.id, - customEmojiName: emoji.name, - })), - ]; - - const remainingSlots = Math.max(0, 8 - (imageMap.get(targetId)?.length ?? 0)); - - await Promise.all( - mediaCandidates.slice(0, remainingSlots).map(async (candidate) => { - // Vision cache check before download - const visionCacheKey = candidate.customEmojiId - ? makeCustomEmojiCacheKey(candidate.customEmojiId) - : candidate.stickerName - ? makeStickerCacheKey(candidate.stickerName) - : makeImageCacheKey(candidate.url); - const cachedVision = await getCachedMediaAnalysis(visionCacheKey); - if (cachedVision) { - log.debug( - { cacheKey: visionCacheKey }, - "Vision cache HIT for media candidate — skipped download", - ); - const analysisText = `[Media analysis for message ${candidate.messageId}] ${candidate.label}: ${cachedVision}`; - const existing = mediaAnalysisMap.get(targetId) ?? []; - existing.push(analysisText); - mediaAnalysisMap.set(targetId, existing); - return; - } - - // Sticker download cache - if (candidate.stickerName && isStickerCacheReady()) { - try { - const cached = await getStickerFromCache(candidate.stickerName); - if (cached) { - const part: MessageImagePart = { - type: "image_url", - image_url: { - url: `data:${cached.mimeType};base64,${cached.base64}`, - }, - sourceLabel: candidate.label, - stickerName: candidate.stickerName, - }; - const existing = imageMap.get(targetId) ?? []; - existing.push(part); - imageMap.set(targetId, existing); - return; - } - } catch { - // Fall through to fetch - } - } - - const result = await fetchUrlSafely(candidate.url); - if (result.type !== "image" || !result.data || !result.mimeType) return; - - // Resize sticker/emoji images too (R5) - const { data: resizedBuffer, mimeType: resizedMime } = - await resizeImageForVision(result.data, maxDimension); - - const base64 = resizedBuffer.toString("base64"); - if (candidate.stickerName) { - setStickerInCache(candidate.stickerName, base64, resizedMime).catch( - () => {}, - ); - } - - const part: MessageImagePart = { - type: "image_url", - image_url: { - url: `data:${resizedMime};base64,${base64}`, - }, - sourceLabel: candidate.label, - stickerName: candidate.stickerName, - customEmojiId: candidate.customEmojiId, - customEmojiName: candidate.customEmojiName, - }; - const existing = imageMap.get(targetId) ?? []; - existing.push(part); - imageMap.set(targetId, existing); - }), - ); - - // ── 4. Vision analysis for every image ── - await Promise.all( - Array.from(imageMap.entries()).flatMap(([msgId, images]) => - images.map(async (image) => { - const summary = await analyzeSingleMediaImage(msgId, image); - if (!summary) return; - const existing = mediaAnalysisMap.get(msgId) ?? []; - existing.push(summary); - mediaAnalysisMap.set(msgId, existing); - }), - ), - ); - - // ── 5. Build single-message prompt with XML delimiters (R1) ── - const textEvidence = await formatModerationTextEvidenceForPrompt(content); - - const webTexts = webTextMap.get(targetId) ?? []; - const mediaAnalyses = mediaAnalysisMap.get(targetId) ?? []; - const webContext = webTexts.length > 0 ? `\n${webTexts.join("\n")}` : ""; - const textContext = textEvidence ? `\n${textEvidence}` : ""; - const mediaAnalysisContext = - mediaAnalyses.length > 0 ? `\n${mediaAnalyses.join("\n")}` : ""; - - const mediaContext = [ - mediaEvidence.stickers.length > 0 - ? mediaEvidence.stickers - .map((s) => buildStickerTextOnlyWarning(s.name, s.url)) - .join(" ") - : null, - mediaEvidence.embeds.length > 0 - ? `[embed evidence: ${mediaEvidence.embeds - .map((e) => - [e.title, e.description, e.url, e.image, e.thumbnail] - .filter(Boolean) - .join(" | "), - ) - .join(" || ")}]` - : null, - ] - .filter(Boolean) - .join(" "); - - // XML delimiters wrap the message content (R1) - const messageBlock = `${content}${mediaContext ? ` ${mediaContext}` : ""}${textContext}${webContext}${mediaAnalysisContext}`; - - // Modular system prompt with XML delimiters (R1, R7, R8) - const systemText = buildSystemPromptModular({ - contextText, - includeMediaInstructions: true, - }); - - const userContent = `${systemText}\n\n\n${messageBlock}\n`; - - // ── 6. LLM call ── - const result = await callModerationLLM( - async (_state: RetryState) => userContent, - targetIds, - `media:${targetId}`, - ); - - return result; -} - -// --------------------------------------------------------------------------- -// Main entry point — splits text-only vs media, runs both paths in parallel -// --------------------------------------------------------------------------- - -/** - * Runs LLM-based moderation analysis on messages. - * - * Architecture: - * - **Text-only messages** → single batch LLM call (fast, no image processing) - * - Split into sub-batches if exceeding AI_LLM_TEXT_BATCH_SIZE (R6) - * - **Media messages** → each gets its own LLM call with vision API (R5: resized images) - * - Both paths execute **in parallel** — text batch does NOT wait for media. - * - All LLM calls go through concurrency limiter (R3). - */ -export async function runModerationAnalysis( - input: ModerationInput, -): Promise { - const { targets, contextText, attachments } = input; - - if (!targets.length) { - throw new Error("No targets provided for analysis"); - } - - // ── Split targets ── - const textOnlyTargets: MessageRecord[] = []; - const mediaTargets: MessageRecord[] = []; - - for (const target of targets) { - if (hasMediaContent(target, attachments)) { - mediaTargets.push(target); - } else { - textOnlyTargets.push(target); - } - } - - log.info( - { - total: targets.length, - textOnly: textOnlyTargets.length, - media: mediaTargets.length, - }, - "Split targets for parallel moderation analysis", - ); - - // ── Run both paths in parallel ── - const [textBatchResult, ...mediaResults] = await Promise.all([ - // Text-only: one fast batch call (or multiple sub-batches) - textOnlyTargets.length > 0 - ? runTextOnlyBatch(textOnlyTargets, contextText) - : Promise.resolve({ results: [] as AnalysisResult[], raw: null }), - - // Media: each message gets its own LLM call (all in parallel, but limited by semaphore — R3) - ...mediaTargets.map((target) => - runSingleMediaAnalysis(target, contextText, attachments), - ), - ]); - - // ── Merge ── - const allResults = [ - ...textBatchResult.results, - ...mediaResults.flatMap((r) => r.results), - ]; - - const raw = - textBatchResult.raw ?? - (mediaResults.length > 0 ? mediaResults[0].raw : null); - - log.info( - { - targetCount: targets.length, - resultCount: allResults.length, - textBatchResults: textBatchResult.results.length, - mediaResults: mediaResults.length, - }, - "Moderation analysis complete", - ); - - return { results: allResults, raw }; -} diff --git a/src/moderation/messageCapture.ts b/src/moderation/messageCapture.ts deleted file mode 100644 index ffecd59..0000000 --- a/src/moderation/messageCapture.ts +++ /dev/null @@ -1,296 +0,0 @@ -import type { Client, Message } from "discord.js-selfbot-v13"; -import { config } from "../config.js"; -import { createChildLogger } from "../logger.js"; -import { getModerationBroadcaster } from "../ws/broadcastGlobals.js"; -import { queueMessageAnalysis } from "./aiAnalyzer.js"; -import { processAttachmentUpload } from "./attachmentUploader.js"; -import { - getDisplayContent, - getMessageLocation, - getMessageMetadata, -} from "./messageMetadata.js"; -import { - getMessageById, - insertAttachment, - updateMessageAsDeleted, - updateMessageAsEdited, - upsertMessageForCapture, -} from "./messageStore.js"; -import { invalidateAnalyticsCache } from "./analyticsStore.js"; -import type { AttachmentRecord, MessageRecord } from "./types.js"; - -const logger = createChildLogger("message-capture"); - -export interface TextCaptureTarget { - guildId?: string; - channelId?: string; -} - -export interface MessageLocationInput { - guildId?: string | null; - channelId?: string | null; -} - -export function shouldCaptureMessageLocation( - message: MessageLocationInput, - target: TextCaptureTarget, -): boolean { - if ( - message.channelId === "1310988070996414494" || - message.channelId === "1265679542144467035" || - message.channelId === "1310867899745046558" - ) - return false; // Skip specific channels - if (!message.guildId || message.guildId !== target.guildId) return false; - if (target.channelId && message.channelId !== target.channelId) return false; - return true; -} - -function getTextCaptureTarget(): TextCaptureTarget { - return { - guildId: config.EFFECTIVE_TEXT_GUILD_ID, - channelId: config.TEXT_CHANNEL_ID, - }; -} - -function buildMessageRecord( - message: Message, - type: "text" | "edited" | "deleted", -): MessageRecord { - const location = getMessageLocation(message); - const metadata = getMessageMetadata(message); - - return { - id: message.id, - guild_id: message.guildId!, - channel_id: location.channelId, - thread_id: location.threadId, - user_id: message.author?.id, - username: message.author?.username, - avatar_url: message.author?.avatarURL() || null, - content: getDisplayContent(message), - edited_content: null, - created_at: message.createdTimestamp, - edited_at: null, - deleted_at: null, - type, - metadata: JSON.stringify(metadata), - }; -} - -function buildAttachmentRecord( - message: Message, - location: ReturnType, - attachment: { - id: string; - name: string | null; - size: number; - contentType: string | null; - url: string; - }, -): AttachmentRecord { - return { - id: attachment.id, - message_id: message.id, - guild_id: message.guildId!, - channel_id: location.channelId, - thread_id: location.threadId, - user_id: message.author?.id, - filename: attachment.name || "unknown", - size: attachment.size, - type: attachment.contentType || "application/octet-stream", - discord_url: attachment.url, - uploaded_url: null, - upload_status: "pending", - upload_error: null, - created_at: Date.now(), - uploaded_at: null, - }; -} - -export async function captureMessage( - message: Message, - type: "text" | "edited" | "deleted", - options: { source?: "live" | "backlog" } = {}, -): Promise { - const location = getMessageLocation(message); - const messageRecord = buildMessageRecord(message, type); - - const inserted = await upsertMessageForCapture(messageRecord); - if (!inserted) { - return; - } - - const isBacklog = options.source === "backlog"; - - const broadcaster = getModerationBroadcaster(); - if (broadcaster && !isBacklog) { - broadcaster.messageCreated(messageRecord); - } - - // Invalidate analytics cache so real-time stats reflect the new message - invalidateAnalyticsCache(messageRecord.guild_id); - - const attachmentUploadTasks: Promise[] = []; - - // Insert attachments before queuing analysis to avoid race condition - if (message.attachments.size > 0) { - for (const [, attachment] of message.attachments) { - const attachmentRecord = buildAttachmentRecord(message, location, { - id: attachment.id, - name: attachment.name, - size: attachment.size, - contentType: attachment.contentType, - url: attachment.url, - }); - - await insertAttachment(attachmentRecord); - - // Initiate async upload (non-blocking, fire-and-forget) - if (!isBacklog) { - attachmentUploadTasks.push( - processAttachmentUpload( - attachment.id, - attachment.url, - attachment.name || "unknown", - { - contentType: attachment.contentType ?? undefined, - refreshDiscordUrl: async () => { - const freshMessage = await message.channel.messages.fetch( - message.id, - ); - const freshAttachment = freshMessage.attachments.get( - attachment.id, - ); - return freshAttachment?.url ?? null; - }, - }, - ).catch((err) => { - logger.error( - { attachmentId: attachment.id, error: err }, - "Failed to initiate attachment upload", - ); - }), - ); - } - - if (broadcaster) { - broadcaster.attachmentCreated(attachmentRecord); - } - } - } - - // Queue analysis after attachment uploads settle so AI uses stable tele URLs. - if (!isBacklog) { - if (attachmentUploadTasks.length > 0) { - let analysisQueued = false; - let fallbackTimer: NodeJS.Timeout | null = null; - const queueAnalysisOnce = () => { - if (analysisQueued) return; - analysisQueued = true; - if (fallbackTimer) { - clearTimeout(fallbackTimer); - fallbackTimer = null; - } - queueMessageAnalysis(message.id); - }; - - fallbackTimer = setTimeout(queueAnalysisOnce, 30000); - Promise.allSettled(attachmentUploadTasks) - .then(queueAnalysisOnce) - .catch((err) => { - logger.error( - { messageId: message.id, error: err }, - "Failed to queue message analysis after attachment upload", - ); - queueAnalysisOnce(); - }); - } else { - queueMessageAnalysis(message.id); - } - } -} - -export function registerMessageCapture(client: Client): void { - client.on("messageCreate", async (message) => { - if (!shouldCaptureMessageLocation(message, getTextCaptureTarget())) return; - if (message.author?.bot) return; - - try { - await captureMessage(message, "text"); - } catch (error) { - logger.error( - { - messageId: message.id, - error: error instanceof Error ? error.message : String(error), - }, - "Failed to capture message", - ); - } - }); - - client.on("messageUpdate", async (_oldMessage, newMessage) => { - if (!shouldCaptureMessageLocation(newMessage, getTextCaptureTarget())) - return; - if (newMessage.author?.bot) return; - - try { - const existing = await getMessageById(newMessage.id); - - if (existing) { - const editedAt = Date.now(); - await updateMessageAsEdited( - newMessage.id, - getDisplayContent(newMessage as Message), - editedAt, - ); - queueMessageAnalysis(newMessage.id); - - const broadcaster = getModerationBroadcaster(); - if (broadcaster) { - broadcaster.messageUpdated({ - id: newMessage.id, - edited_content: getDisplayContent(newMessage as Message), - edited_at: editedAt, - }); - } - } else if (newMessage.author) { - await captureMessage(newMessage as Message, "text"); - } - } catch (error) { - logger.error( - { - messageId: newMessage.id, - error: error instanceof Error ? error.message : String(error), - }, - "Failed to capture message update", - ); - } - }); - - client.on("messageDelete", async (message) => { - if (!shouldCaptureMessageLocation(message, getTextCaptureTarget())) return; - if (!message.author) return; - - try { - const deletedAt = Date.now(); - await updateMessageAsDeleted(message.id, deletedAt); - - const broadcaster = getModerationBroadcaster(); - if (broadcaster) { - broadcaster.messageDeleted({ - id: message.id, - deleted_at: deletedAt, - }); - } - } catch (error) { - logger.error( - { - messageId: message.id, - error: error instanceof Error ? error.message : String(error), - }, - "Failed to capture message deletion", - ); - } - }); -} diff --git a/src/moderation/messageMetadata.ts b/src/moderation/messageMetadata.ts deleted file mode 100644 index 10f466a..0000000 --- a/src/moderation/messageMetadata.ts +++ /dev/null @@ -1,375 +0,0 @@ -import type { - Message, - TextChannel, - ThreadChannel, -} from "discord.js-selfbot-v13"; - -export interface MessageLocation { - channelId: string; - threadId: string | null; - threadName: string | null; - channelName: string | null; - nsfw?: boolean; - nsfwLevel?: string | null; - ageRestricted?: boolean; -} - -export interface StickerEvidence { - id: string; - name: string; - url: string; - format: string | null; -} - -export interface CustomEmojiEvidence { - id: string; - name: string; - animated: boolean; - url: string; -} - -export interface EmbedEvidence { - title: string | null; - description: string | null; - url: string | null; - color: number | null; - image: string | null; - thumbnail: string | null; - author: { - name: string | null; - url: string | null; - iconURL: string | null; - } | null; - footer: { text: string | null; iconURL: string | null } | null; - fields: Array<{ name: string; value: string; inline: boolean }>; -} - -export interface AttachmentEvidence { - id: string; - name: string; - url: string; - contentType: string | null; - size: number; -} - -export interface MessageMediaEvidence { - stickers: StickerEvidence[]; - embeds: EmbedEvidence[]; - attachments: AttachmentEvidence[]; - customEmojis: CustomEmojiEvidence[]; -} - -export interface RichMessageMetadata { - stickers: Array; - embeds: Array; - attachments: Array; - customEmojis: Array; - author: { - id: string; - username: string; - tag: string | null; - avatarURL: string | null; - bot: boolean; - }; - member: { - displayName: string | null; - roles: Array<{ id: string; name: string }>; - joinedTimestamp: number | null; - } | null; - channel: MessageLocation; - reference: { - messageId: string | null; - channelId: string | null; - guildId: string | null; - } | null; -} - -export function getMessageLocation(message: Message): MessageLocation { - const channel = message.channel as TextChannel | ThreadChannel; - const safetyChannel = channel as TextChannel & { - nsfw?: boolean; - nsfwLevel?: string | null; - }; - if (!channel.isThread?.()) { - return { - channelId: message.channelId, - threadId: null, - threadName: null, - channelName: "name" in channel ? channel.name : null, - nsfw: - typeof safetyChannel.nsfw === "boolean" - ? safetyChannel.nsfw - : undefined, - nsfwLevel: - typeof safetyChannel.nsfwLevel === "string" - ? safetyChannel.nsfwLevel - : null, - ageRestricted: - typeof safetyChannel.nsfw === "boolean" - ? safetyChannel.nsfw - : undefined, - }; - } - - return { - channelId: channel.parentId ?? message.channelId, - threadId: channel.id, - threadName: channel.name, - channelName: channel.parent?.name ?? null, - nsfw: - typeof safetyChannel.nsfw === "boolean" ? safetyChannel.nsfw : undefined, - nsfwLevel: - typeof safetyChannel.nsfwLevel === "string" - ? safetyChannel.nsfwLevel - : null, - ageRestricted: - typeof safetyChannel.nsfw === "boolean" ? safetyChannel.nsfw : undefined, - }; -} - -export function getStickerMetadata( - message: Message, -): RichMessageMetadata["stickers"] { - return Array.from(message.stickers.values()).map((sticker) => ({ - id: sticker.id, - name: sticker.name, - url: sticker.url, - format: sticker.format ?? null, - })); -} - -/** - * Extract custom emoji references from message content. - * Builds Discord CDN URLs for each emoji so they can be downloaded - * and sent to the vision model for analysis. - */ -export function getCustomEmojiMetadata( - message: Message, -): RichMessageMetadata["customEmojis"] { - const CUSTOM_EMOJI_PATTERN = /<(a)?:([a-zA-Z0-9_]+):(\d+)>/g; - const emojis: CustomEmojiEvidence[] = []; - let match; - while ((match = CUSTOM_EMOJI_PATTERN.exec(message.content)) !== null) { - const [, animated, name, id] = match; - const ext = animated ? "gif" : "png"; - emojis.push({ - id, - name, - animated: animated === "a", - url: `https://cdn.discordapp.com/emojis/${id}.${ext}?size=128`, - }); - } - return emojis; -} - -export function getAttachmentMetadata( - message: Message, -): RichMessageMetadata["attachments"] { - return Array.from(message.attachments.values()).map((attachment) => ({ - id: attachment.id, - name: attachment.name || "unknown", - url: attachment.url, - contentType: attachment.contentType ?? null, - size: attachment.size, - })); -} - -export function getEmbedMetadata( - message: Message, -): RichMessageMetadata["embeds"] { - return message.embeds.map((embed) => ({ - title: embed.title ?? null, - description: embed.description ?? null, - url: embed.url ?? null, - color: embed.color ?? null, - image: embed.image?.url ?? null, - thumbnail: embed.thumbnail?.url ?? null, - author: embed.author - ? { - name: embed.author.name ?? null, - url: embed.author.url ?? null, - iconURL: embed.author.iconURL ?? null, - } - : null, - footer: embed.footer - ? { - text: embed.footer.text ?? null, - iconURL: embed.footer.iconURL ?? null, - } - : null, - fields: embed.fields.map((field) => ({ - name: field.name, - value: field.value, - inline: Boolean(field.inline), - })), - })); -} - -export function getMessageMetadata(message: Message): RichMessageMetadata { - const member = message.member; - return { - stickers: getStickerMetadata(message), - embeds: getEmbedMetadata(message), - attachments: getAttachmentMetadata(message), - customEmojis: getCustomEmojiMetadata(message), - author: { - id: message.author.id, - username: message.author.username, - tag: "tag" in message.author ? message.author.tag : null, - avatarURL: message.author.avatarURL() ?? null, - bot: Boolean(message.author.bot), - }, - member: member - ? { - displayName: member.displayName ?? null, - roles: member.roles.cache.map((role) => ({ - id: role.id, - name: role.name, - })), - joinedTimestamp: member.joinedTimestamp ?? null, - } - : null, - channel: getMessageLocation(message), - reference: message.reference - ? { - messageId: message.reference.messageId ?? null, - channelId: message.reference.channelId ?? null, - guildId: message.reference.guildId ?? null, - } - : null, - }; -} - -export function parseRichMessageMetadata( - metadata: string | null | undefined, -): RichMessageMetadata | null { - if (!metadata) return null; - - try { - const parsed = JSON.parse(metadata) as Partial; - return { - stickers: Array.isArray(parsed.stickers) ? parsed.stickers : [], - embeds: Array.isArray(parsed.embeds) ? parsed.embeds : [], - attachments: Array.isArray(parsed.attachments) ? parsed.attachments : [], - customEmojis: Array.isArray(parsed.customEmojis) - ? parsed.customEmojis - : [], - author: parsed.author as RichMessageMetadata["author"], - member: (parsed.member ?? null) as RichMessageMetadata["member"], - channel: parsed.channel as RichMessageMetadata["channel"], - reference: (parsed.reference ?? null) as RichMessageMetadata["reference"], - }; - } catch { - return null; - } -} - -export function isAgeRestrictedMetadata( - metadata: string | null | undefined, -): boolean { - const parsed = parseRichMessageMetadata(metadata); - if (!parsed) return false; - - const nsfwLevel = parsed.channel.nsfwLevel?.toUpperCase(); - return Boolean( - parsed.channel.nsfw || - parsed.channel.ageRestricted || - nsfwLevel === "AGE_RESTRICTED", - ); -} - -export function extractMessageMediaEvidence( - metadata: string | null | undefined, -): MessageMediaEvidence { - const parsed = parseRichMessageMetadata(metadata); - return { - stickers: parsed?.stickers ?? [], - embeds: parsed?.embeds ?? [], - attachments: parsed?.attachments ?? [], - customEmojis: parsed?.customEmojis ?? [], - }; -} - -export function formatMediaEvidenceForPrompt( - metadata: string | null | undefined, -): string { - const evidence = extractMessageMediaEvidence(metadata); - const parts: string[] = []; - - if (evidence.stickers.length > 0) { - parts.push( - `[stickers: ${evidence.stickers - .map((sticker) => - [`name=${sticker.name}`, sticker.url ? `url=${sticker.url}` : null] - .filter(Boolean) - .join(", "), - ) - .join(" | ")}]`, - ); - } - - if (evidence.embeds.length > 0) { - parts.push( - `[embeds: ${evidence.embeds - .map((embed) => - [ - embed.title ? `title=${embed.title}` : null, - embed.description ? `description=${embed.description}` : null, - embed.url ? `url=${embed.url}` : null, - embed.image ? `image=${embed.image}` : null, - embed.thumbnail ? `thumbnail=${embed.thumbnail}` : null, - embed.fields.length > 0 - ? `fields=${embed.fields.map((field) => `${field.name}: ${field.value}`).join("; ")}` - : null, - ] - .filter(Boolean) - .join(", "), - ) - .join(" | ")}]`, - ); - } - - if (evidence.attachments.length > 0) { - parts.push( - `[attachments: ${evidence.attachments - .map((attachment) => - [ - `name=${attachment.name}`, - attachment.contentType ? `type=${attachment.contentType}` : null, - `size=${attachment.size}`, - attachment.url ? `url=${attachment.url}` : null, - ] - .filter(Boolean) - .join(", "), - ) - .join(" | ")}]`, - ); - } - - return parts.join(" "); -} - -export function getDisplayContent(message: Message): string { - if (message.content.trim().length > 0) return message.content; - - const stickers = getStickerMetadata(message); - if (stickers.length > 0) { - return stickers.map((sticker) => `[Sticker: ${sticker.name}]`).join(" "); - } - - const attachments = getAttachmentMetadata(message); - if (attachments.length > 0) { - return attachments - .map((attachment) => `[Attachment: ${attachment.name}]`) - .join(" "); - } - - const embeds = getEmbedMetadata(message); - if (embeds.length > 0) { - return embeds - .map((embed) => embed.title || embed.description || "[Embed]") - .join(" "); - } - - return ""; -} diff --git a/src/moderation/messageStore.ts b/src/moderation/messageStore.ts deleted file mode 100644 index a84eca1..0000000 --- a/src/moderation/messageStore.ts +++ /dev/null @@ -1,1239 +0,0 @@ -import { - and, - asc, - desc, - eq, - inArray, - isNull, - or, - type SQL, - sql, -} from "drizzle-orm"; -import { getDatabase } from "../database/drizzle.js"; -import { - attachmentsTable, - messageReviewsTable, - messagesTable, - moderationActionsTable, - retentionPoliciesTable, -} from "../database/schema.js"; -import { createChildLogger } from "../logger.js"; -import { decodeCursor, encodeCursor } from "./pagination.js"; -import type { - AttachmentRecord, - MessageQuery, - MessageRecord, - MessageReview, - ModerationAction, - PageResult, - RetentionPolicy, -} from "./types.js"; - -const logger = createChildLogger("message-store"); - -interface QueryBuilder extends PromiseLike { - from(...args: unknown[]): QueryBuilder; - where(...args: unknown[]): QueryBuilder; - orderBy(...args: unknown[]): QueryBuilder; - limit(...args: unknown[]): QueryBuilder; - offset(...args: unknown[]): QueryBuilder; - values(...args: unknown[]): QueryBuilder; - onConflictDoNothing(...args: unknown[]): QueryBuilder; - returning(...args: unknown[]): QueryBuilder; - set(...args: unknown[]): QueryBuilder; -} - -interface MessageDatabase { - select(...args: unknown[]): QueryBuilder; - selectDistinct(...args: unknown[]): QueryBuilder; - insert(...args: unknown[]): QueryBuilder; - update(...args: unknown[]): QueryBuilder; - transaction(callback: (tx: MessageDatabase) => Promise): Promise; -} - -function db(): MessageDatabase { - return getDatabase() as unknown as MessageDatabase; -} - -function channelOrThreadCondition(channelId: string): SQL { - return or( - eq(messagesTable.channel_id, channelId), - eq(messagesTable.thread_id, channelId), - ) as SQL; -} - -function buildListMessageConditions(query: MessageQuery): SQL[] { - const conditions: SQL[] = []; - - if (query.guildId) { - conditions.push(eq(messagesTable.guild_id, query.guildId)); - } - - if (query.channelId) { - conditions.push(channelOrThreadCondition(query.channelId)); - } - - if (query.threadId) { - conditions.push(eq(messagesTable.thread_id, query.threadId)); - } - - if (query.userId) { - conditions.push(eq(messagesTable.user_id, query.userId)); - } - - if (query.status && query.status.length > 0) { - conditions.push(sql`${messagesTable.ai_status} in ${query.status}`); - } - - if (query.q) { - const pattern = `%${query.q.toLowerCase()}%`; - conditions.push(sql`lower(${messagesTable.content}) like ${pattern}`); - } - - const cursorData = decodeCursor(query.cursor); - if (cursorData) { - conditions.push( - sql`(${messagesTable.created_at} < ${cursorData.created_at} or (${messagesTable.created_at} = ${cursorData.created_at} and ${messagesTable.id} < ${cursorData.id}))`, - ); - } - - return conditions; -} - -function pageRows( - rows: unknown[], - limit: number, -): PageResult { - const hasMore = rows.length > limit; - const data = rows.slice(0, limit) as T[]; - const lastItem = data[data.length - 1]; - const nextCursor = - hasMore && lastItem - ? encodeCursor({ created_at: lastItem.created_at, id: lastItem.id }) - : null; - - return { data, nextCursor }; -} - -function pageMessages( - rows: unknown[], - limit: number, -): PageResult { - return pageRows(rows, limit); -} - -export { decodeCursor, encodeCursor } from "./pagination.js"; - -export async function insertMessage(message: MessageRecord): Promise { - try { - const database = db(); - await database.insert(messagesTable).values(message).onConflictDoNothing(); - } catch (error) { - logger.error( - { - messageId: message.id, - error: error instanceof Error ? error.message : String(error), - }, - "Failed to insert message", - ); - throw error; - } -} - -export async function upsertMessageForCapture( - message: MessageRecord, -): Promise { - try { - const database = db(); - const messageWithAIStatus = { - ...message, - ai_status: "pending" as const, - }; - - const rows = await database - .insert>(messagesTable) - .values(messageWithAIStatus) - .onConflictDoNothing() - .returning({ id: messagesTable.id }); - - return rows.length > 0; - } catch (error) { - logger.error( - { - messageId: message.id, - error: error instanceof Error ? error.message : String(error), - }, - "Failed to upsert message for capture", - ); - throw error; - } -} - -export async function updateMessageAsEdited( - messageId: string, - editedContent: string, - editedAt: number, -): Promise { - try { - const database = db(); - await database - .update(messagesTable) - .set({ - edited_content: editedContent, - edited_at: editedAt, - type: "edited", - ai_status: "pending", - ai_moderation_flags: null, - ai_moderation_score: null, - ai_analysis: null, - ai_categories: null, - ai_severity: null, - ai_confidence: null, - ai_recommended_action: null, - ai_analyzed_at: null, - ai_error: null, - }) - .where(eq(messagesTable.id, messageId)); - } catch (error) { - logger.error( - { - messageId, - error: error instanceof Error ? error.message : String(error), - }, - "Failed to update message as edited", - ); - throw error; - } -} - -export async function updateMessageAsDeleted( - messageId: string, - deletedAt: number, -): Promise { - try { - const database = db(); - await database - .update(messagesTable) - .set({ - deleted_at: deletedAt, - type: "deleted", - }) - .where(eq(messagesTable.id, messageId)); - } catch (error) { - logger.error( - { - messageId, - error: error instanceof Error ? error.message : String(error), - }, - "Failed to update message as deleted", - ); - throw error; - } -} - -export async function getMessagesByChannel( - channelId: string, - limit: number = 50, - offset: number = 0, - guildId?: string, -): Promise { - try { - const database = db(); - const conditions: SQL[] = [ - or( - eq(messagesTable.channel_id, channelId), - eq(messagesTable.thread_id, channelId), - ) as SQL, - ]; - - if (guildId) { - conditions.push(eq(messagesTable.guild_id, guildId)); - } - - const rows = await database - .select() - .from(messagesTable) - .where(and(...conditions)) - // P3: add secondary sort by id for stable pagination - .orderBy(desc(messagesTable.created_at), desc(messagesTable.id)) - .limit(limit) - .offset(offset); - - return rows as MessageRecord[]; - } catch (error) { - logger.error( - { - channelId, - error: error instanceof Error ? error.message : String(error), - }, - "Failed to get messages by channel", - ); - throw error; - } -} - -export async function insertAttachment( - attachment: AttachmentRecord, -): Promise { - try { - const database = db(); - await database - .insert(attachmentsTable) - .values(attachment) - .onConflictDoNothing(); - } catch (error) { - logger.error( - { - attachmentId: attachment.id, - error: error instanceof Error ? error.message : String(error), - }, - "Failed to insert attachment", - ); - throw error; - } -} - -export async function getAttachmentsByChannel( - channelId: string, - limit: number = 50, - offset: number = 0, - guildId?: string, -): Promise { - try { - const database = db(); - const conditions: SQL[] = [ - or( - eq(attachmentsTable.channel_id, channelId), - eq(attachmentsTable.thread_id, channelId), - ) as SQL, - ]; - - if (guildId) { - conditions.push(eq(attachmentsTable.guild_id, guildId)); - } - - const rows = await database - .select() - .from(attachmentsTable) - .where(and(...conditions)) - .orderBy(desc(attachmentsTable.created_at)) - .limit(limit) - .offset(offset); - - return rows as AttachmentRecord[]; - } catch (error) { - logger.error( - { - channelId, - error: error instanceof Error ? error.message : String(error), - }, - "Failed to get attachments by channel", - ); - throw error; - } -} - -export async function updateAttachmentAsUploaded( - attachmentId: string, - uploadedUrl: string, - uploadedAt: number, -): Promise { - try { - const database = db(); - await database - .update(attachmentsTable) - .set({ - uploaded_url: uploadedUrl, - upload_status: "uploaded", - uploaded_at: uploadedAt, - }) - .where(eq(attachmentsTable.id, attachmentId)); - } catch (error) { - logger.error( - { - attachmentId, - error: error instanceof Error ? error.message : String(error), - }, - "Failed to update attachment as uploaded", - ); - throw error; - } -} - -export async function updateAttachmentDiscordUrl( - attachmentId: string, - discordUrl: string, -): Promise { - try { - const database = db(); - await database - .update(attachmentsTable) - .set({ discord_url: discordUrl }) - .where(eq(attachmentsTable.id, attachmentId)); - } catch (error) { - logger.error( - { - attachmentId, - error: error instanceof Error ? error.message : String(error), - }, - "Failed to update attachment Discord URL", - ); - throw error; - } -} - -export async function updateAttachmentAsFailedUpload( - attachmentId: string, - error: string, -): Promise { - try { - const database = db(); - await database - .update(attachmentsTable) - .set({ - upload_status: "failed", - upload_error: error, - }) - .where(eq(attachmentsTable.id, attachmentId)); - } catch (error) { - logger.error( - { - attachmentId, - error: error instanceof Error ? error.message : String(error), - }, - "Failed to update attachment as failed", - ); - throw error; - } -} - -interface AIAnalysisUpdate { - status: "pending" | "clean" | "warn" | "flagged" | "error"; - flags?: string | null; - score?: number | null; - analysis?: string | null; - categories?: string[] | string | null; - severity?: MessageRecord["ai_severity"] | null; - confidence?: number | null; - recommendedAction?: MessageRecord["ai_recommended_action"] | null; - analyzedAt?: number | null; - error?: string | null; -} - -function stringifyAIList( - value: string[] | string | null | undefined, -): string | null { - if (value == null) return null; - return Array.isArray(value) ? JSON.stringify(value) : value; -} - -export async function updateMessageAIAnalysis( - messageId: string, - result: AIAnalysisUpdate, -): Promise { - try { - const database = db(); - await database - .update(messagesTable) - .set({ - ai_status: result.status, - ai_moderation_flags: result.flags ?? null, - ai_moderation_score: result.score ?? null, - ai_analysis: result.analysis ?? null, - ai_categories: stringifyAIList(result.categories), - ai_severity: result.severity ?? null, - ai_confidence: result.confidence ?? result.score ?? null, - ai_recommended_action: result.recommendedAction ?? null, - ai_analyzed_at: result.analyzedAt ?? Date.now(), - ai_error: result.error ?? null, - }) - .where(eq(messagesTable.id, messageId)); - - const rows = await database - .select() - .from(messagesTable) - .where(eq(messagesTable.id, messageId)); - - return (rows[0] as MessageRecord) ?? null; - } catch (error) { - logger.error( - { - messageId, - error: error instanceof Error ? error.message : String(error), - }, - "Failed to update message AI analysis", - ); - throw error; - } -} - -export async function updateMessagesAIAnalysisBulk( - updates: Array<{ messageId: string; result: AIAnalysisUpdate }>, -): Promise { - if (updates.length === 0) return []; - try { - const database = db(); - const now = Date.now(); - - await database.transaction(async (tx) => { - for (const { messageId, result } of updates) { - await tx - .update(messagesTable) - .set({ - ai_status: result.status, - ai_moderation_flags: result.flags ?? null, - ai_moderation_score: result.score ?? null, - ai_analysis: result.analysis ?? null, - ai_categories: stringifyAIList(result.categories), - ai_severity: result.severity ?? null, - ai_confidence: result.confidence ?? result.score ?? null, - ai_recommended_action: result.recommendedAction ?? null, - ai_analyzed_at: result.analyzedAt ?? now, - ai_error: result.error ?? null, - }) - .where(eq(messagesTable.id, messageId)); - } - }); - - // Fetch all updated messages in a single query - const ids = updates.map(({ messageId }) => messageId); - const rows = await database - .select() - .from(messagesTable) - .where(inArray(messagesTable.id, ids)); - - return rows as MessageRecord[]; - } catch (error) { - logger.error( - { - error: error instanceof Error ? error.message : String(error), - }, - "Failed to bulk update messages AI analysis", - ); - throw error; - } -} - -export async function getPendingAIAnalysisMessages( - limit: number = 25, -): Promise { - try { - const database = db(); - const rows = await database - .select() - .from(messagesTable) - .where( - and( - eq(messagesTable.ai_status, "pending"), - isNull(messagesTable.deleted_at), - ), - ) - .orderBy(asc(messagesTable.created_at)) - .limit(limit); - - return rows as MessageRecord[]; - } catch (error) { - logger.error( - { error: error instanceof Error ? error.message : String(error) }, - "Failed to get pending AI analysis messages", - ); - throw error; - } -} - -export async function getMessageById( - messageId: string, -): Promise { - try { - const database = db(); - const rows = await database - .select() - .from(messagesTable) - .where(eq(messagesTable.id, messageId)); - - return (rows[0] as MessageRecord) ?? null; - } catch (error) { - logger.error( - { - messageId, - error: error instanceof Error ? error.message : String(error), - }, - "Failed to get message by id", - ); - throw error; - } -} - -export async function listMessages( - query: MessageQuery, -): Promise> { - try { - const database = db(); - const conditions = buildListMessageConditions(query); - const rows = await database - .select() - .from(messagesTable) - .where(conditions.length > 0 ? and(...conditions) : undefined) - .orderBy(desc(messagesTable.created_at), desc(messagesTable.id)) - .limit(query.limit + 1); - - return pageMessages(rows, query.limit); - } catch (error) { - logger.error( - { - query, - error: error instanceof Error ? error.message : String(error), - }, - "Failed to list messages", - ); - throw error; - } -} - -export async function listReviewMessages( - query: Omit, -): Promise> { - return listMessages({ - ...query, - status: ["warn", "flagged", "error"], - }); -} - -export async function getConversationContextBefore(input: { - channelId: string; - threadId: string | null; - beforeCreatedAt: number; - limit: number; -}): Promise { - try { - const database = db(); - const { channelId, threadId, beforeCreatedAt, limit } = input; - - // Query same thread if threadId exists, otherwise channelId - const locationCondition = threadId - ? eq(messagesTable.thread_id, threadId) - : eq(messagesTable.channel_id, channelId); - - const rows = await database - .select() - .from(messagesTable) - .where( - and( - locationCondition, - sql`${messagesTable.created_at} < ${beforeCreatedAt}`, - isNull(messagesTable.deleted_at), - ), - ) - .orderBy(desc(messagesTable.created_at)) - .limit(limit); - - // Return in chronological order (oldest first) - return (rows as MessageRecord[]).reverse(); - } catch (error) { - logger.error( - { - channelId: input.channelId, - threadId: input.threadId, - error: error instanceof Error ? error.message : String(error), - }, - "Failed to get conversation context before", - ); - throw error; - } -} - -export async function getPendingMessagesByConversation( - conversationKey: string, - limit: number = 200, -): Promise { - try { - const database = db(); - - // conversationKey is either thread_id or channel_id - // Query both to safely handle the key - const rows = await database - .select() - .from(messagesTable) - .where( - and( - or( - eq(messagesTable.thread_id, conversationKey), - eq(messagesTable.channel_id, conversationKey), - ), - eq(messagesTable.ai_status, "pending"), - isNull(messagesTable.deleted_at), - ), - ) - .orderBy(asc(messagesTable.created_at)) - .limit(limit); - - return rows as MessageRecord[]; - } catch (error) { - logger.error( - { - conversationKey, - error: error instanceof Error ? error.message : String(error), - }, - "Failed to get pending messages by conversation", - ); - throw error; - } -} - -export async function getPendingConversationKeys( - limit: number = 500, -): Promise { - try { - const database = db(); - - // Get distinct conversation keys (thread_id or channel_id) for pending messages - const rows = await database - .selectDistinct>({ - thread_id: messagesTable.thread_id, - channel_id: messagesTable.channel_id, - }) - .from(messagesTable) - .where( - and( - eq(messagesTable.ai_status, "pending"), - isNull(messagesTable.deleted_at), - ), - ) - .limit(limit); - - const keys: string[] = []; - for (const row of rows) { - const key = row.thread_id || row.channel_id; - if (key && !keys.includes(key)) { - keys.push(key); - } - } - - return keys; - } catch (error) { - logger.error( - { error: error instanceof Error ? error.message : String(error) }, - "Failed to get pending conversation keys", - ); - throw error; - } -} - -export async function getAttachmentsForMessages( - messageIds: string[], -): Promise { - try { - if (messageIds.length === 0) return []; - const database = db(); - const rows = await database - .select() - .from(attachmentsTable) - .where(inArray(attachmentsTable.message_id, messageIds)); - - return rows as AttachmentRecord[]; - } catch (error) { - logger.error( - { - messageIds, - error: error instanceof Error ? error.message : String(error), - }, - "Failed to get attachments for messages", - ); - throw error; - } -} - -export async function searchMessages(input: { - query: string; - channelId?: string; - guildId?: string; - limit?: number; -}): Promise { - try { - const { query, channelId, guildId, limit = 20 } = input; - const database = db(); - - const searchPattern = `%${query}%`; - const conditions: (SQL | undefined)[] = [isNull(messagesTable.deleted_at)]; - - if (guildId) { - conditions.push(eq(messagesTable.guild_id, guildId)); - } - - if (channelId) { - conditions.push(channelOrThreadCondition(channelId)); - } - - conditions.push( - or( - sql`${messagesTable.content} LIKE ${searchPattern}`, - sql`${messagesTable.edited_content} LIKE ${searchPattern}`, - ), - ); - - const validConditions = conditions.filter((c): c is SQL => c !== undefined); - - const rows = await database - .select() - .from(messagesTable) - .where(and(...validConditions)) - .orderBy(desc(messagesTable.created_at)) - .limit(limit); - - return rows as MessageRecord[]; - } catch (error) { - logger.error( - { - query: input.query, - channelId: input.channelId, - guildId: input.guildId, - error: error instanceof Error ? error.message : String(error), - }, - "Failed to search messages", - ); - throw error; - } -} - -/** - * Returns distinct conversation keys (thread_id or channel_id) that have at - * least one message stuck in `error` status with the `analysis_incomplete` - * flag set. Used by the recovery worker to re-feed those messages through - * the individual-fallback queue. - */ -export async function getConversationKeysWithIncompleteAnalysis( - limit: number = 200, -): Promise { - try { - const database = db(); - const rows = await database - .selectDistinct>({ - thread_id: messagesTable.thread_id, - channel_id: messagesTable.channel_id, - }) - .from(messagesTable) - .where( - and( - eq(messagesTable.ai_status, "error"), - sql`${messagesTable.ai_moderation_flags} LIKE ${"%analysis_incomplete%"}`, - isNull(messagesTable.deleted_at), - ), - ) - .limit(limit); - - const keys: string[] = []; - for (const row of rows) { - const key = row.thread_id || row.channel_id; - if (key && !keys.includes(key)) { - keys.push(key); - } - } - return keys; - } catch (error) { - logger.error( - { error: error instanceof Error ? error.message : String(error) }, - "Failed to get conversation keys with incomplete analysis", - ); - throw error; - } -} - -/** - * Returns MessageRecords for a given conversation key whose AI analysis is - * stuck in `error` + `analysis_incomplete`. Used to feed those records - * directly into the individual-fallback queue without touching their status - * (the individual pipeline will overwrite status on success). - */ -export async function getIncompleteMessagesByConversation( - conversationKey: string, - limit: number = 500, -): Promise { - try { - const database = db(); - const rows = await database - .select() - .from(messagesTable) - .where( - and( - or( - eq(messagesTable.thread_id, conversationKey), - eq(messagesTable.channel_id, conversationKey), - ), - eq(messagesTable.ai_status, "error"), - sql`${messagesTable.ai_moderation_flags} LIKE ${"%analysis_incomplete%"}`, - isNull(messagesTable.deleted_at), - ), - ) - .orderBy(asc(messagesTable.created_at)) - .limit(limit); - - return rows as MessageRecord[]; - } catch (error) { - logger.error( - { - conversationKey, - error: error instanceof Error ? error.message : String(error), - }, - "Failed to get incomplete messages by conversation", - ); - throw error; - } -} - -// Message Reviews CRUD -// ==================== - -export async function createMessageReview( - review: Omit, -): Promise { - try { - const database = db(); - const id = `review-${Date.now()}-${Math.random().toString(36).slice(2, 9)}`; - const created_at = Date.now(); - - const rows = await database - .insert>(messageReviewsTable) - .values({ - ...review, - id, - created_at, - }) - .returning(); - - return rows[0] as MessageReview; - } catch (error) { - logger.error( - { - messageId: review.message_id, - error: error instanceof Error ? error.message : String(error), - }, - "Failed to create message review", - ); - throw error; - } -} - -export async function getMessageReview( - id: string, -): Promise { - try { - const database = db(); - const rows = await database - .select() - .from(messageReviewsTable) - .where(eq(messageReviewsTable.id, id)); - - return (rows[0] as MessageReview) || null; - } catch (error) { - logger.error( - { - reviewId: id, - error: error instanceof Error ? error.message : String(error), - }, - "Failed to get message review", - ); - throw error; - } -} - -export async function listMessageReviews(query: { - guildId?: string; - channelId?: string; - status?: string[]; - cursor?: string; - limit: number; -}): Promise> { - try { - const database = db(); - const limit = Math.max(1, Math.min(query.limit || 50, 100)); - const conditions: SQL[] = []; - - if (query.guildId) { - conditions.push(eq(messageReviewsTable.guild_id, query.guildId)); - } - if (query.channelId) { - conditions.push(eq(messageReviewsTable.channel_id, query.channelId)); - } - if (query.status && query.status.length > 0) { - conditions.push(sql`${messageReviewsTable.status} in ${query.status}`); - } - - const cursorData = decodeCursor(query.cursor); - if (cursorData) { - conditions.push( - sql`(${messageReviewsTable.created_at} < ${cursorData.created_at} or (${messageReviewsTable.created_at} = ${cursorData.created_at} and ${messageReviewsTable.id} < ${cursorData.id}))`, - ); - } - - const rows = await database - .select() - .from(messageReviewsTable) - .where(conditions.length > 0 ? and(...conditions) : undefined) - .orderBy( - desc(messageReviewsTable.created_at), - desc(messageReviewsTable.id), - ) - .limit(limit + 1); - - return pageRows(rows, limit); - } catch (error) { - logger.error( - { error: error instanceof Error ? error.message : String(error) }, - "Failed to list message reviews", - ); - throw error; - } -} - -export async function updateMessageReview( - id: string, - updates: Partial>, -): Promise { - try { - const database = db(); - const rows = (await database - .update(messageReviewsTable) - .set(updates) - .where(eq(messageReviewsTable.id, id)) - .returning()) as MessageReview[]; - - return rows[0] || null; - } catch (error) { - logger.error( - { - reviewId: id, - error: error instanceof Error ? error.message : String(error), - }, - "Failed to update message review", - ); - throw error; - } -} - -// Moderation Actions CRUD -// ======================= - -export async function createModerationAction( - action: Omit, -): Promise { - try { - const database = db(); - const id = `action-${Date.now()}-${Math.random().toString(36).slice(2, 9)}`; - const created_at = Date.now(); - - const rows = await database - .insert>(moderationActionsTable) - .values({ - ...action, - id, - created_at, - }) - .returning(); - - return rows[0] as ModerationAction; - } catch (error) { - logger.error( - { - guildId: action.guild_id, - error: error instanceof Error ? error.message : String(error), - }, - "Failed to create moderation action", - ); - throw error; - } -} - -export async function getModerationAction( - id: string, -): Promise { - try { - const database = db(); - const rows = await database - .select() - .from(moderationActionsTable) - .where(eq(moderationActionsTable.id, id)); - - return (rows[0] as ModerationAction) || null; - } catch (error) { - logger.error( - { - actionId: id, - error: error instanceof Error ? error.message : String(error), - }, - "Failed to get moderation action", - ); - throw error; - } -} - -export async function listModerationActions(query: { - guildId?: string; - status?: string[]; - cursor?: string; - limit: number; -}): Promise> { - try { - const database = db(); - const limit = Math.max(1, Math.min(query.limit || 50, 100)); - const conditions: SQL[] = []; - - if (query.guildId) { - conditions.push(eq(moderationActionsTable.guild_id, query.guildId)); - } - if (query.status && query.status.length > 0) { - conditions.push(sql`${moderationActionsTable.status} in ${query.status}`); - } - - const cursorData = decodeCursor(query.cursor); - if (cursorData) { - conditions.push( - sql`(${moderationActionsTable.created_at} < ${cursorData.created_at} or (${moderationActionsTable.created_at} = ${cursorData.created_at} and ${moderationActionsTable.id} < ${cursorData.id}))`, - ); - } - - const rows = await database - .select() - .from(moderationActionsTable) - .where(conditions.length > 0 ? and(...conditions) : undefined) - .orderBy( - desc(moderationActionsTable.created_at), - desc(moderationActionsTable.id), - ) - .limit(limit + 1); - - return pageRows(rows, limit); - } catch (error) { - logger.error( - { error: error instanceof Error ? error.message : String(error) }, - "Failed to list moderation actions", - ); - throw error; - } -} - -export async function updateModerationAction( - id: string, - updates: Partial>, -): Promise { - try { - const database = db(); - const rows = (await database - .update(moderationActionsTable) - .set(updates) - .where(eq(moderationActionsTable.id, id)) - .returning()) as ModerationAction[]; - - return rows[0] || null; - } catch (error) { - logger.error( - { - actionId: id, - error: error instanceof Error ? error.message : String(error), - }, - "Failed to update moderation action", - ); - throw error; - } -} - -// Retention Policies CRUD -// ======================= - -export async function getRetentionPolicy( - guildId: string, -): Promise { - try { - const database = db(); - const rows = await database - .select() - .from(retentionPoliciesTable) - .where(eq(retentionPoliciesTable.guild_id, guildId)); - - return (rows[0] as RetentionPolicy) || null; - } catch (error) { - logger.error( - { - guildId, - error: error instanceof Error ? error.message : String(error), - }, - "Failed to get retention policy", - ); - throw error; - } -} - -export async function upsertRetentionPolicy( - policy: Omit, -): Promise { - try { - const database = db(); - const now = Date.now(); - const existing = await getRetentionPolicy(policy.guild_id); - - if (existing) { - const rows = (await database - .update(retentionPoliciesTable) - .set({ - ...policy, - updated_at: now, - }) - .where(eq(retentionPoliciesTable.id, existing.id)) - .returning()) as RetentionPolicy[]; - - return rows[0] as RetentionPolicy; - } - - const id = `policy-${Date.now()}-${Math.random().toString(36).slice(2, 9)}`; - const rows = (await database - .insert>(retentionPoliciesTable) - .values({ - ...policy, - id, - created_at: now, - updated_at: now, - }) - .returning()) as RetentionPolicy[]; - - return rows[0] as RetentionPolicy; - } catch (error) { - logger.error( - { - guildId: policy.guild_id, - error: error instanceof Error ? error.message : String(error), - }, - "Failed to upsert retention policy", - ); - throw error; - } -} - -export async function getExpiredMessages( - retentionDays: number, -): Promise { - try { - const database = db(); - const cutoffTime = Date.now() - retentionDays * 24 * 60 * 60 * 1000; - - const rows = await database - .select() - .from(messagesTable) - .where( - and( - sql`${messagesTable.created_at} < ${cutoffTime}`, - isNull(messagesTable.deleted_at), - ), - ) - .limit(1000); - - return rows as MessageRecord[]; - } catch (error) { - logger.error( - { - retentionDays, - error: error instanceof Error ? error.message : String(error), - }, - "Failed to get expired messages", - ); - throw error; - } -} diff --git a/src/moderation/moderationPrompt.ts b/src/moderation/moderationPrompt.ts deleted file mode 100644 index 489011c..0000000 --- a/src/moderation/moderationPrompt.ts +++ /dev/null @@ -1,220 +0,0 @@ -/** - * Modular system prompt builder for LLM moderation. - * - * Split into composable sections: - * - buildSystemRules() — culture/slang/flag definitions (static) - * - buildMediaInstructions() — media/sticker analysis guidance (conditional) - * - buildFewShotExamples() — 3 example outputs (static) - * - buildSystemPrompt() — assembles all sections with XML delimiters - * - * XML delimiters prevent prompt injection by clearly separating - * system instructions from user-supplied data. - */ - -// --------------------------------------------------------------------------- -// Section: System Rules (static — culture, slang, flag definitions) -// --------------------------------------------------------------------------- - -const SYSTEM_RULES = `Kamu adalah asisten moderasi konten untuk server Discord berbahasa Indonesia. -Bahasa utama komunitas ini adalah BAHASA INDONESIA. Bahasa Inggris adalah bahasa sekunder. - -## Aturan Umum -- Bahasa gaul/slang Indonesia: "anjay", "wkwk", "gws", "gaskeun", "santuy", "njir", "baka", "woy", "woi", "hadeh", dll adalah AMAN. -- Singkatan umum: "gw", "lo", "emg", "kyk", "tdk", "krn", "jgn", dll adalah AMAN. -- Makian/kata kasar umum (seperti "anjing", "asu", "bangsat") BUKAN pelanggaran SARA. SARA khusus untuk diskriminasi/hinaan terhadap Suku, Agama, Ras, dan Antargolongan. NAMUN makian/kata kasar TETAP bisa di-flag sebagai "harassment" atau "vulgar_language" HANYA jika: (1) ditujukan langsung ke orang lain sebagai serangan/hinaan, (2) dalam tone agresif/mengancam, atau (3) bagian dari pola harassment berkelanjutan. -- Kata "asus" adalah merk teknologi, jangan pernah dianggap sebagai makian "asu". -- "woy"/"woi" adalah sapaan/interjeksi informal Indonesia dan tidak boleh dianggap SARA, hate speech, atau harassment tanpa target hinaan/ancaman jelas. -- Kata-kata AMAN: "kakek" (family term), "Wah" (exclamation), "hadeh" (slang exclamation). Jangan flag sebagai vulgar_language atau harassment. -- Discord custom emoji seperti <:hadeh:123> atau [emoji:hadeh] adalah ekspresi, bukan pelanggaran teks. -- Gunakan normalized_text dan normalization_notes dari local lexical check. Jika notes hanya berisi slang/emoji aman, jangan flag. Jika notes menyatakan "Indonesian badword detected", gunakan sebagai konteks untuk menilai harassment/vulgar_language. - -## Kategori Pelanggaran & Kriteria Flag -Prioritas tertinggi (ANCAMAN KESELAMATAN): -- child_safety, self_harm, violence, illegal_content — flag jika ada indikasi nyata -- Pornografi/NSFW, ajakan seksual, roleplay seksual → "sexual_content" -- Judi/promosi judi → "gambling" -- Narkoba/promosi → "drugs" - -Prioritas menengah (PERILAKU MERUSAK): -- Ancaman kekerasan, doxxing, scam → flag sesuai kategori -- spam self-promo → "spam" -- Istilah agama/suku/ras: penyebutan netral/edukasi = clean; hinaan/provokasi/diskriminatif = "sara" atau "hate_speech" - -Prioritas rendah (PELANGGARAN RINGAN): -- harassment (targeted insult), vulgar_language (profanity terarah) -- sexual_deviation: jika pesan mempromosikan/mendukung topik seksual/identitas yang dibatasi server sebagai pembahasan utama - -## Pohon Keputusan (Decision Tree) -1. Apakah ada ancaman keselamatan nyata (child_safety, self_harm, violence)? → flagged, critical -2. Apakah ada konten ilegal/explicit (NSFW, drugs, gambling, scam)? → flagged, high -3. Apakah ada harassment terarah/hate speech/sara? → flagged, medium-high -4. Apakah ada spam/promosi borderline? → warn, low-medium -5. Jika tidak ada pelanggaran jelas atau bukti ambigu → clean -Jangan pernah flag hanya berdasarkan kecurigaan atau ketidakjelasan konteks. - -## ATURAN UNTUK GAMBAR — DUA MODE BERBEDA - -### Mode 1: Teks + Gambar (teks adalah bukti utama) -- Jika ada teks percakapan normal ("Aku suka nasgor loh", "Halo guys") → gambar hampir pasti bukan pelanggaran. -- Jika teks clean: OVERRIDE klaim vision tentang judi KECUALI ada bukti spesifik (chip, kartu, odds, logo dikenal). -- Teks lebih penting dari gambar. - -### Mode 2: HANYA GAMBAR (teks kosong/sangat pendek/tidak bermakna) -- **Deskripsi gambar MENJADI bukti utama.** Tidak ada teks untuk dijadikan acuan. -- BACA Media analysis dengan teliti. Deskripsi itulah satu-satunya konteks. -- Jika deskripsi menyebutkan "terminal", "console", "editor kode" → itu BUKAN gambling. Clean. -- Jika deskripsi menyebutkan "aplikasi chat", "screenshot percakapan" → itu BUKAN gambling. Clean. -- Jika deskripsi menyebutkan "foto makanan/pemandangan/selfie/hewan" → Clean. -- **HANYA flag gambling jika deskripsi SECARA EKSPLISIT menyebutkan elemen judi NYATA: chip, kartu remi, meja taruhan, odds, deposit/withdraw, logo situs judi.** -- JANGAN abaikan gambar hanya karena teks kosong. Analisis TETAP harus dilakukan berdasarkan deskripsi gambar. -- **Jika pesan HANYA berisi gambar tanpa teks → WAJIB membaca Media analysis dan membuat keputusan berdasarkan deskripsi tersebut.**`; - -// --------------------------------------------------------------------------- -// Section: Media Instructions (conditional — injected when media present) -// --------------------------------------------------------------------------- - -const MEDIA_INSTRUCTIONS = `## Instruksi Analisis Media -Gambar, sticker, embed image, preview link, dan attachment sudah DIDESKRIPSIKAN oleh vision model sebelum batch utama. -Baris "Media analysis" berisi DESKRIPSI OBJEKTIF tentang apa yang terlihat di gambar, BUKAN keputusan moderasi. -Vision model TIDAK memutuskan apakah gambar melanggar atau tidak — ia hanya mendeskripsikan isi visual. - -## ATURAN KRITIS — Kamu yang Memutuskan, Bukan Vision Model -- **KAMU adalah moderator.** Deskripsi dari vision model adalah SAKSI MATA, bukan hakim. -- Jika deskripsi vision menyebutkan "screenshot terminal", "aplikasi chat", "tampilan website", "foto makanan" → itu BUKAN bukti pelanggaran apapun. -- HANYA flag "gambling" jika KAMU menyimpulkan dari deskripsi bahwa gambar menunjukkan situs judi (chip, kartu remi, meja taruhan, odds, deposit/withdraw). -- **PESAN HANYA GAMBAR (teks kosong/pendek):** WAJIB menganalisis Media analysis. Deskripsi gambar adalah satu-satunya bukti. JANGAN otomatis clean hanya karena teks kosong. Baca deskripsi → putuskan. -- **PESAN DENGAN TEKS:** Bukti teks LEBIH PENTING dari deskripsi gambar. Jika teks pesan adalah percakapan biasa dan tidak mengandung promosi judi, maka gambar tersebut TIDAK MUNGKIN adalah pelanggaran judi. -- **Jika teks clean dan deskripsi gambar biasa → wajib clean.** -- Deskripsi vision yang menyebutkan hal-hal netral (terminal, chat, editor kode, website, grafik, chart) TIDAK BOLEH dijadikan dasar untuk flag gambling. - -## Panduan Khusus Sticker -- Sticker Discord adalah media kartun/meme/ilustrasi, BUKAN foto atau video nyata. -- Sticker sering bersifat humor, satir, atau ekspresi emosi yang dilebih-lebihkan. -- Gambar sticker bisa menampilkan adegan kartun yang terlihat "keras" — itu SENI KARTUN, bukan dokumentasi kekerasan nyata. -- Nama sticker yang terdengar provokatif (mis. "Singa injek pejabat") adalah konteks satir/humor. JANGAN flag berdasarkan nama sticker saja. -- Terapkan standar yang lebih longgar untuk konten kartun/meme dibanding foto/video nyata.`; - -// --------------------------------------------------------------------------- -// Section: Few-Shot Examples -// --------------------------------------------------------------------------- - -const FEW_SHOT_EXAMPLES = `## Contoh Output yang Benak - -Contoh 1 — Pesan bersih dengan slang: -Input: [target] id=12345 user=budi: anjay wkwk gaskeun santuy bro -Output: {"results":[{"message_id":"12345","status":"clean","flags":[],"score":0.0,"categories":[],"severity":"none","confidence":0.95,"recommended_action":"none","policy_version":"default-2026-05-30","evidence":[],"analysis":"Slang Indonesia umum tanpa pelanggaran terdeteksi."}]} - -Contoh 2 — Harassment terarah: -Input: [target] id=67890 user=anon: lu goblok banget sih kontol, mampus aja lo -Output: {"results":[{"message_id":"67890","status":"flagged","flags":["harassment","vulgar_language"],"score":0.85,"categories":["harassment","vulgar_language"],"severity":"high","confidence":0.9,"recommended_action":"delete","policy_version":"default-2026-05-30","evidence":["lu goblok banget sih kontol","mampus aja lo"],"analysis":"Insult langsung dengan kata kasar terarah ke individu."}]} - -Contoh 3 — Sticker kartun dengan nama provokatif: -Input: [target] id=11111 user=citra: <:singa_injek:123456> [sticker: "Singa injek pejabat"] -Output: {"results":[{"message_id":"11111","status":"clean","flags":[],"score":0.1,"categories":[],"severity":"none","confidence":0.8,"recommended_action":"none","policy_version":"default-2026-05-30","evidence":[],"analysis":"Sticker kartun satir dengan nama provokatif namun bukan ancaman nyata."}]} - -Contoh 4 — Pesan biasa dengan gambar (JANGAN flag sebagai judi): -Input: [target] id=22222 user=rina: Aku suka nasgor loh [Media analysis for message 22222] [gambar di atas adalah attachment foto.jpg dari pesan id=22222]: Gambar menampilkan tangkapan layar aplikasi chat dengan teks percakapan biasa. Tidak ada konten melanggar terlihat. Aman. -Output: {"results":[{"message_id":"22222","status":"clean","flags":[],"score":0.0,"categories":[],"severity":"none","confidence":0.95,"recommended_action":"none","policy_version":"default-2026-05-30","evidence":[],"analysis":"Pesan berisi percakapan sehari-hari tentang makanan. Gambar menunjukkan screenshot chat biasa tanpa pelanggaran."}]} - -Contoh 5 — Pesan promosi judi dengan gambar situs judi: -Input: [target] id=33333 user=spammer: MAIN DI SINI GACOR PARAH https://judionline.xyz [Media analysis for message 33333] [gambar di atas adalah attachment slot.jpg dari pesan id=33333]: Gambar menampilkan antarmuka situs judi online dengan mesin slot, chip, dan tombol deposit. Terlihat logo "JudiOnline" dan odds taruhan. -Output: {"results":[{"message_id":"33333","status":"flagged","flags":["gambling"],"score":0.92,"categories":["gambling"],"severity":"high","confidence":0.92,"recommended_action":"delete","policy_version":"default-2026-05-30","evidence":["MAIN DI SINI GACOR PARAH","https://judionline.xyz","Gambar menampilkan antarmuka situs judi online dengan mesin slot, chip, dan tombol deposit"],"analysis":"Promosi situs judi online dengan link, teks promosi, dan gambar antarmuka judi yang jelas."}]} - -Contoh 6 — Pesan HANYA GAMBAR tanpa teks (WAJIB analisis deskripsi): -Input: [target] id=44444 user=dev: [Media analysis for message 44444] [gambar di atas adalah attachment screenshot.png dari pesan id=44444]: Screenshot terminal Linux dengan background hitam dan teks hijau. Terlihat output command 'ls -la' dan 'git status'. Tidak ada teks atau elemen mencurigakan. -Output: {"results":[{"message_id":"44444","status":"clean","flags":[],"score":0.0,"categories":[],"severity":"none","confidence":0.95,"recommended_action":"none","policy_version":"default-2026-05-30","evidence":[],"analysis":"dev mengirim screenshot terminal Linux. Terlihat output command ls -la dan git status dengan teks hijau di background hitam. Aktivitas coding biasa, tidak ada konten melanggar."}]} - -Contoh 7 — Pesan HANYA GAMBAR situs judi (teks kosong, tapi gambar jelas): -Input: [target] id=55555 user=promotor: [Media analysis for message 55555] [gambar di atas adalah attachment promo.jpg dari pesan id=55555]: Screenshot website dengan background merah dan emas. Terlihat teks "DEPOSIT NOW", "BONUS 100%", "SLOT GACOR", chip poker, dan roda roulette. Ada tombol "DAFTAR" dan "LOGIN". -Output: {"results":[{"message_id":"55555","status":"flagged","flags":["gambling"],"score":0.94,"categories":["gambling"],"severity":"high","confidence":0.94,"recommended_action":"delete","policy_version":"default-2026-05-30","evidence":["Gambar menampilkan antarmuka situs judi online dengan chip, roulette, tombol deposit, dan teks promosi judi"],"analysis":"Promosi situs judi melalui gambar dengan elemen judi jelas: chip, roulette, teks deposit dan bonus."}]}`; - -// --------------------------------------------------------------------------- -// Section: Output Schema + XML Delimiter Instructions -// --------------------------------------------------------------------------- - -const OUTPUT_INSTRUCTIONS = `## Format Output -Balas HANYA dengan satu objek JSON valid. Tanpa markdown, tanpa prose, tanpa komentar, tanpa XML. -Struktur wajib: -{ - "results": [ - { - "message_id": "", - "status": "clean" | "warn" | "flagged", - "flags": [""], - "score": 0.0, - "categories": [""], - "severity": "none" | "low" | "medium" | "high" | "critical", - "confidence": 0.0, - "recommended_action": "none" | "monitor" | "warn" | "review" | "delete" | "escalate", - "policy_version": "default-2026-05-30", - "evidence": [""], - "analysis": "" - } - ] -} - -## FORMAT WAJIB — Field "analysis" HARUS deskriptif berdasarkan konten: - -### Jika HANYA TEKS (tidak ada gambar/media): -Tulis: "[user] membahas tentang . . ." -Contoh baik: "budi membahas tentang makan siang dengan teman-teman. Percakapan santai menggunakan slang Indonesia. Tidak ada pelanggaran." -Contoh buruk: "Pesan hanya berisi teks tanpa pelanggaran." - -### Jika HANYA GAMBAR (teks kosong/tidak bermakna): -Tulis: "Gambar berupa . Terlihat . ." -Contoh baik: "Gambar berupa screenshot terminal Linux. Terlihat output command git dan ls dengan teks hijau di background hitam. Tidak ada konten melanggar." -Contoh buruk: "Pesan hanya berisi attachment tanpa pelanggaran." - -### Jika TEKS + GAMBAR: -Tulis: "[user] mengirim sambil membahas tentang . . ." -Contoh baik: "rina mengirim screenshot chat sambil membahas tentang makanan favorit. Gambar dan teks sama-sama tentang percakapan sehari-hari. Tidak ada pelanggaran." -Contoh buruk: "Pesan berisi teks dan gambar tanpa pelanggaran." - -### Jika melanggar: -Tulis: "[user] . . ." -Contoh baik: "spammer mempromosikan situs judi online dengan link dan gambar antarmuka judi. Gambar menunjukkan chip, roulette, dan tombol deposit. Melanggar kebijakan gambling." - -CRITICAL: -- JANGAN PERNAH menulis "Pesan hanya berisi..." atau "Pesan tidak mengandung..." sebagai analysis. -- Selalu sebutkan ISI KONTEN secara spesifik — apa yang dibicarakan, apa yang terlihat di gambar. -- Gunakan informasi dari Media analysis untuk mendeskripsikan gambar. -- Analisis harus MEMBERI KONTEKS, bukan hanya menyatakan status. - -Flag yang valid: spam, hate_speech, sara, hoaks, harassment, vulgar_language, sexual_content, sexual_deviation, violence, self_harm, doxxing, scam, misinformation, nsfw_image, gore_image, illegal_content, gambling, drugs, child_safety, financial_scam, religious_insult, self_promo - -CRITICAL: "message_id" HARUS berupa STRING (dibungkus tanda kutip ganda). Jangan perlakukan ID sebagai angka.`; - -// --------------------------------------------------------------------------- -// Composer: assembles all sections with XML delimiters -// --------------------------------------------------------------------------- - -export interface BuildSystemPromptOptions { - contextText: string; - includeMediaInstructions: boolean; - correction?: { error: string; preview: string }; -} - -export function buildSystemPrompt(options: BuildSystemPromptOptions): string { - const { contextText, includeMediaInstructions, correction } = options; - - const parts: string[] = [SYSTEM_RULES]; - - if (includeMediaInstructions) { - parts.push(MEDIA_INSTRUCTIONS); - } - - parts.push(FEW_SHOT_EXAMPLES); - parts.push(OUTPUT_INSTRUCTIONS); - - // XML-delimited context — prevents prompt injection - const delimitedContext = `\n${contextText}\n`; - parts.push(delimitedContext); - - let base = parts.join("\n\n"); - - if (correction) { - base += `\n\nRESPON SEBELUMNYA GAGAL VALIDASI.\nError: ${correction.error}\nPreview respons tidak valid:\n${correction.preview}\n\nCoba lagi dengan output JSON yang benar sesuai skema di atas.`; - } - - return base; -} diff --git a/src/moderation/pagination.ts b/src/moderation/pagination.ts deleted file mode 100644 index d0949aa..0000000 --- a/src/moderation/pagination.ts +++ /dev/null @@ -1,21 +0,0 @@ -export interface CursorData { - created_at: number; - id: string; -} - -export function encodeCursor(data: CursorData): string { - return Buffer.from(JSON.stringify(data)).toString("base64"); -} - -export function decodeCursor(cursor?: string): CursorData | null { - if (!cursor) return null; - try { - const data = JSON.parse(Buffer.from(cursor, "base64").toString("utf-8")); - if (typeof data.created_at === "number" && typeof data.id === "string") { - return data; - } - return null; - } catch { - return null; - } -} diff --git a/src/moderation/retentionManager.ts b/src/moderation/retentionManager.ts deleted file mode 100644 index c46bc92..0000000 --- a/src/moderation/retentionManager.ts +++ /dev/null @@ -1,181 +0,0 @@ -import { and, eq, isNull, lt } from "drizzle-orm"; -import { getDatabase } from "../database/drizzle.js"; -import { - attachmentsTable, - messagesTable, - retentionPoliciesTable, - voiceRecordingsTable, -} from "../database/schema.js"; -import { createChildLogger } from "../logger.js"; -import { getRetentionPolicy } from "./messageStore.js"; -import type { RetentionPolicy } from "./types.js"; - -const logger = createChildLogger("retention-manager"); - -interface RetentionResult { - messagesDeleted: number; - attachmentsDeleted: number; - voiceRecordingsDeleted: number; - error?: string; -} - -/** - * Executes retention policy for a guild - * Deletes messages, attachments, and voice recordings older than retention_days - */ -export async function executeRetentionPolicy( - guildId: string, -): Promise { - const result: RetentionResult = { - messagesDeleted: 0, - attachmentsDeleted: 0, - voiceRecordingsDeleted: 0, - }; - - try { - const policy = await getRetentionPolicy(guildId); - if (!policy || !policy.enabled) { - logger.debug({ guildId }, "Retention policy not enabled"); - return result; - } - - const db = getDatabase() as any; - const cutoffTime = Date.now() - policy.retention_days * 24 * 60 * 60 * 1000; - - // Delete old messages - const deletedMessages = await db - .delete(messagesTable) - .where( - and( - eq(messagesTable.guild_id, guildId), - lt(messagesTable.created_at, cutoffTime), - isNull(messagesTable.deleted_at), - ), - ); - - result.messagesDeleted = deletedMessages.rowsAffected || 0; - - // Delete old attachments if policy applies - if (policy.apply_to_media) { - const deletedAttachments = await db - .delete(attachmentsTable) - .where( - and( - eq(attachmentsTable.guild_id, guildId), - lt(attachmentsTable.created_at, cutoffTime), - ), - ); - - result.attachmentsDeleted = deletedAttachments.rowsAffected || 0; - } - - // Delete old voice recordings if policy applies - if (policy.apply_to_voice) { - const deletedVoice = await db - .delete(voiceRecordingsTable) - .where( - and( - eq(voiceRecordingsTable.guild_id, guildId), - lt(voiceRecordingsTable.created_at, cutoffTime), - ), - ); - - result.voiceRecordingsDeleted = deletedVoice.rowsAffected || 0; - } - - logger.info( - { - guildId, - retentionDays: policy.retention_days, - ...result, - }, - "Retention policy executed", - ); - - return result; - } catch (error) { - const message = error instanceof Error ? error.message : String(error); - logger.error( - { guildId, error: message }, - "Failed to execute retention policy", - ); - result.error = message; - return result; - } -} - -/** - * Executes retention policies for all enabled guilds - * Returns summary of deletions - */ -export async function executeAllRetentionPolicies(): Promise<{ - policiesExecuted: number; - totalMessagesDeleted: number; - totalAttachmentsDeleted: number; - totalVoiceDeleted: number; - errors: Array<{ guildId: string; error: string }>; -}> { - const summary = { - policiesExecuted: 0, - totalMessagesDeleted: 0, - totalAttachmentsDeleted: 0, - totalVoiceDeleted: 0, - errors: [] as Array<{ guildId: string; error: string }>, - }; - - try { - const db = getDatabase() as any; - const policies = await db - .select() - .from(retentionPoliciesTable) - .where(eq(retentionPoliciesTable.enabled, true)); - - for (const policy of policies as RetentionPolicy[]) { - const result = await executeRetentionPolicy(policy.guild_id); - summary.policiesExecuted++; - summary.totalMessagesDeleted += result.messagesDeleted; - summary.totalAttachmentsDeleted += result.attachmentsDeleted; - summary.totalVoiceDeleted += result.voiceRecordingsDeleted; - - if (result.error) { - summary.errors.push({ - guildId: policy.guild_id, - error: result.error, - }); - } - } - - logger.info(summary, "All retention policies executed"); - return summary; - } catch (error) { - const message = error instanceof Error ? error.message : String(error); - logger.error( - { error: message }, - "Failed to execute all retention policies", - ); - throw error; - } -} - -/** - * Starts a periodic retention policy executor - * Runs every 24 hours by default - */ -export function startRetentionPolicyWorker( - intervalMs: number = 24 * 60 * 60 * 1000, -): NodeJS.Timeout { - logger.info({ intervalMs }, "Starting retention policy worker"); - - const interval = setInterval(async () => { - try { - await executeAllRetentionPolicies(); - } catch (error) { - logger.error( - { error: error instanceof Error ? error.message : String(error) }, - "Retention policy worker failed", - ); - } - }, intervalMs); - - return interval; -} diff --git a/src/moderation/stickerCache.ts b/src/moderation/stickerCache.ts deleted file mode 100644 index 5666568..0000000 --- a/src/moderation/stickerCache.ts +++ /dev/null @@ -1,162 +0,0 @@ -import { executeAll, executeGet } from "../database/drizzle.js"; -import { createChildLogger } from "../logger.js"; - -const logger = createChildLogger("sticker-cache"); - -const TTL_MS = 7 * 24 * 60 * 60 * 1000; // 7 days -const MAX_SIZE_BYTES = 100 * 1024 * 1024; // 100MB hardcoded - -let ready = false; -let statsCache = { entryCount: 0, totalSizeBytes: 0 }; - -export interface StickerCacheEntry { - base64: string; - mimeType: string; - fetchedAt: number; - size: number; -} - -function sanitizeKey(name: string): string { - return encodeURIComponent(name).replace(/%/g, "_"); -} - -/** - * Initialise the sticker cache from PostgreSQL. - * Idempotent — safe to call multiple times. - */ -export async function initStickerCache(): Promise { - if (ready) return; - try { - // Ensure the table exists — belt-and-suspenders in case the migration - // hasn't run yet (e.g. pre-existing DB that Drizzle skips). - await executeAll(` - CREATE TABLE IF NOT EXISTS "sticker_cache" ( - "name" text PRIMARY KEY NOT NULL, - "base64" text NOT NULL, - "mime_type" text NOT NULL, - "size" integer NOT NULL, - "fetched_at" bigint NOT NULL - ) - `); - await executeAll( - `CREATE INDEX IF NOT EXISTS "idx_sticker_cache_fetched_at" ON "sticker_cache" USING btree ("fetched_at")`, - ); - - await executeAll( - "DELETE FROM sticker_cache WHERE fetched_at < $1", - [Date.now() - TTL_MS], - ); - const row = await executeGet( - "SELECT count(*) as cnt, COALESCE(SUM(size), 0) as total FROM sticker_cache", - [], - ); - if (row) { - statsCache = { - entryCount: Number(row.cnt), - totalSizeBytes: Number(row.total), - }; - } - } catch (err) { - logger.warn( - { error: String(err) }, - "Failed to prune expired stickers on init", - ); - } - ready = true; - logger.info(statsCache, "Sticker cache initialized (PostgreSQL)"); -} - -/** - * Look up a sticker image by name. Returns null on miss or TTL expiry. - */ -export async function getStickerFromCache( - stickerName: string, -): Promise { - if (!ready) return null; - const key = sanitizeKey(stickerName); - try { - const row = await executeGet( - "SELECT base64, mime_type, size, fetched_at FROM sticker_cache WHERE name = ? AND fetched_at > ?", - [key, Date.now() - TTL_MS], - ); - if (!row) return null; - return { - base64: row.base64, - mimeType: row.mime_type, - fetchedAt: Number(row.fetched_at), - size: Number(row.size), - }; - } catch (err) { - logger.error( - { error: String(err), stickerName }, - "Failed to get sticker from cache", - ); - return null; - } -} - -/** - * Store a sticker image in the cache. Fires and forgets — never blocks. - */ -export async function setStickerInCache( - stickerName: string, - base64: string, - mimeType: string, -): Promise { - if (!ready) return; - const key = sanitizeKey(stickerName); - const size = Buffer.byteLength(base64, "utf-8"); - const now = Date.now(); - try { - await evictIfNeeded(size); - await executeAll( - `INSERT INTO sticker_cache (name, base64, mime_type, size, fetched_at) - VALUES (?, ?, ?, ?, ?) - ON CONFLICT (name) DO UPDATE SET - base64 = EXCLUDED.base64, - mime_type = EXCLUDED.mime_type, - size = EXCLUDED.size, - fetched_at = EXCLUDED.fetched_at`, - [key, base64, mimeType, size, now], - ); - statsCache.entryCount++; - statsCache.totalSizeBytes += size; - logger.debug({ stickerName, size }, "Sticker cached"); - } catch (err) { - logger.warn( - { stickerName, error: String(err) }, - "Failed to write sticker to cache", - ); - } -} - -async function evictIfNeeded(newSize: number): Promise { - while (statsCache.totalSizeBytes + newSize > MAX_SIZE_BYTES) { - const oldest = await executeGet( - "SELECT name, size FROM sticker_cache ORDER BY fetched_at ASC LIMIT 1", - ); - if (!oldest) break; - await executeAll("DELETE FROM sticker_cache WHERE name = ?", [ - oldest.name, - ]); - statsCache.totalSizeBytes -= Number(oldest.size); - statsCache.entryCount--; - } -} - -/** - * Return current cache stats for observability. - */ -export function getStickerCacheStats(): { - entryCount: number; - totalSizeBytes: number; -} { - return { ...statsCache }; -} - -/** - * Check if cache has been initialized. - */ -export function isStickerCacheReady(): boolean { - return ready; -} diff --git a/src/moderation/stickerPrompt.ts b/src/moderation/stickerPrompt.ts deleted file mode 100644 index 33b859d..0000000 --- a/src/moderation/stickerPrompt.ts +++ /dev/null @@ -1,135 +0,0 @@ -/** - * Sticker-specific prompt templates for AI moderation. - * - * Discord stickers are cartoon/meme artwork — not real photos. - * These prompts give the LLM proper context to avoid false-positive flags - * based solely on sticker names or cartoon imagery. - */ - -/** - * Prompt used when a sticker image was successfully downloaded (from cache - * or network) and is being sent to the vision LLM as a base64 image. - * - * Explains that stickers are cartoon art, not documentation of real events, - * and instructs the model to apply looser standards for cartoon content. - */ -export function buildStickerVisionPrompt( - stickerName: string, - messageId: string, -): string { - return [ - `Analisis sticker Discord berikut sebagai evidence moderasi.`, - `Sticker "${stickerName}" berasal dari pesan id=${messageId}.`, - ``, - `PENTING — Konteks Sticker:`, - `- Sticker Discord adalah gambar KARTUN/MEME/ILUSTRASI, BUKAN foto atau video nyata.`, - `- Sticker sering bersifat humor, satir, atau ekspresi emosi yang dilebih-lebihkan.`, - `- Gambar di sticker bisa menampilkan adegan yang terlihat "keras" (tokoh kartun menginjak sesuatu, ledakan komik, senjata kartun, tokoh berantem) — itu SENI KARTUN, bukan dokumentasi kekerasan atau ancaman nyata.`, - `- Teks di sticker sering berupa lelucon, sindiran, atau ekspresi khas komunitas — bukan ancaman literal.`, - ``, - `Jelaskan isi visual, teks yang terlihat, dan konteks risiko.`, - `Terapkan standar yang lebih longgar untuk konten kartun/meme:`, - `- Adegan kartun yang terlihat "keras" ≠ kekerasan nyata → jangan flag "violence" kecuali jelas menargetkan individu/kelompok nyata dengan ancaman serius.`, - `- Nama sticker yang terdengar provokatif (mis. "Singa injek pejabat") adalah konteks satir/kartun, bukan bukti pelanggaran.`, - `- Humor/satir/politik kartun ≠ SARA atau hate speech.`, - `- Sticker yang menampilkan tokoh kartun dalam pose agresif adalah ekspresi/emosi umum di Discord, bukan harassment.`, - ``, - `Jawab Bahasa Indonesia, maksimal 3 kalimat. Jangan bilang kurang konteks atau perlu admin cek.`, - ].join("\n"); -} - -/** - * Wrapper for text-only evidence when a sticker image failed to download. - * - * Returns a formatted string that explicitly tells the LLM not to flag - * based on the sticker name alone, since names can sound provocative - * while the actual cartoon image is harmless. - */ -export function buildStickerTextOnlyWarning( - stickerName: string, - stickerUrl: string, -): string { - return ( - `[sticker: "${stickerName}" (${stickerUrl}) — GAMBAR GAGAL DIUNDUH. ` + - `"${stickerName}" adalah sticker kartun/meme Discord. ` + - `JANGAN flag berdasarkan nama sticker saja tanpa gambar visual. ` + - `Sticker Discord adalah seni kartun/ekspresi humor, bukan foto nyata. ` + - `Nama yang terdengar provokatif adalah hal umum untuk sticker satir/humor di Discord.]` - ); -} - -/** - * Prompt used when a custom emoji image was successfully downloaded - * and is being sent to the vision LLM as a base64 image. - * - * Custom emojis are small icons — context is similar to stickers. - */ -export function buildCustomEmojiVisionPrompt( - emojiName: string, - messageId: string, -): string { - return [ - `Analisis custom emoji Discord berikut sebagai evidence moderasi.`, - `Emoji "${emojiName}" berasal dari pesan id=${messageId}.`, - ``, - `PENTING — Konteks Custom Emoji:`, - `- Custom emoji Discord adalah ikon kecil/ekspresi, BUKAN foto atau dokumen nyata.`, - `- Emoji sering digunakan untuk ekspresi emosi, reaksi, atau lelucon.`, - `- Jangan flag berdasarkan nama emoji saja — analisis isi visual gambar.`, - `- Emoji yang terlihat lucu/aneh adalah hal umum di Discord, bukan pelanggaran.`, - ``, - `Jelaskan isi visual dan konteks risiko.`, - `Jawab Bahasa Indonesia, maksimal 2 kalimat. Jangan bilang kurang konteks.`, - ].join("\n"); -} - -/** - * Fallback text for when a custom emoji image failed to download. - */ -export function buildCustomEmojiTextOnlyFallback(emojiName: string): string { - return ( - `[custom_emoji: "${emojiName}" — GAMBAR GAGAL DIUNDUH. ` + - `"${emojiName}" adalah custom emoji Discord (ikon kecil). ` + - `JANGAN flag berdasarkan nama emoji saja tanpa gambar visual. ` + - `Custom emoji di Discord adalah ekspresi/emosi umum, bukan konten ofensif.]` - ); -} - -/** - * Prompt for analyzing regular images (attachments, embeds, links). - * - * VISION MODEL ONLY DESCRIBES — it does NOT decide moderation. - * The main text LLM makes all moderation decisions using the description. - */ -export function buildGeneralImageVisionPrompt( - sourceLabel: string, - _messageId: string, -): string { - return [ - `Deskripsikan gambar ini secara objektif dan spesifik.`, - `${sourceLabel}`, - ``, - `Jelaskan HANYA apa yang kamu LIHAT:`, - `- Objek utama apa yang ada di gambar?`, - `- Teks apa yang terlihat? (tulis persis jika bisa dibaca)`, - `- Warna dominan dan layout/tata letak?`, - `- Apakah ini screenshot, foto, meme, kartun, atau dokumen?`, - `- Konteks: apakah terlihat seperti aplikasi chat, terminal/console,`, - ` media sosial, game, website, editor kode, dokumen, atau lainnya?`, - ``, - `PENTING — Deskripsi saja, JANGAN MEMUTUSKAN MODERASI:`, - `- JANGAN sebut "gambling", "judi", "pelanggaran", "melanggar", atau flag apapun.`, - `- JANGAN bilang "harus dihapus", "harus diblokir", atau rekomendasi tindakan.`, - `- Tugasmu HANYA mendeskripsikan isi gambar. BUKAN menilai.`, - `- Screenshot terminal/console/shell/editor kode → deskripsikan sebagai "terminal/console".`, - `- Screenshot aplikasi chat (Discord/WA/Telegram/dll) → deskripsikan sebagai "aplikasi chat".`, - `- Screenshot website dengan grafik/chart → deskripsikan kontennya secara faktual.`, - `- JANGAN PERNAH mengklaim gambar adalah "situs judi" atau "antarmuka perjudian".`, - ` Itu BUKAN tugasmu. Kamu hanya perlu menyebutkan: "tampilan website dengan grafik",`, - ` "screenshot terminal", "aplikasi chat dengan teks percakapan", dll.`, - ``, - `Format jawaban: Deskripsi singkat 2-3 kalimat dalam Bahasa Indonesia.`, - `Mulai dengan menyebutkan JENIS gambar (screenshot/foto/kartun/dokumen).`, - `Akhiri dengan "Deskripsi selesai."`, - ].join("\n"); -} diff --git a/src/moderation/textCacheStore.ts b/src/moderation/textCacheStore.ts deleted file mode 100644 index 7c0b311..0000000 --- a/src/moderation/textCacheStore.ts +++ /dev/null @@ -1,241 +0,0 @@ -import { createHash } from "node:crypto"; -import { executeAll, executeGet } from "../database/drizzle.js"; -import { createChildLogger } from "../logger.js"; - -const logger = createChildLogger("text-cache-store"); - -export interface TextCacheEntry { - text: string; - flags: string[]; - source: "local" | "primary_ai" | "vision_llm"; - analyzed_at: number; - expires_at: number; - hit_count: number; -} - -/** - * Lookup cached analysis result for a normalized text string. - * Returns null if not found or expired. - */ -export async function getCachedText( - text: string, -): Promise { - try { - const row = await executeGet( - `SELECT text, flags, source, analyzed_at, expires_at, hit_count - FROM text_analysis_cache - WHERE text = $1 AND expires_at > $2`, - [text, Date.now()], - ); - - if (!row) return null; - - return { - text: row.text, - flags: JSON.parse(row.flags), - source: row.source, - analyzed_at: row.analyzed_at, - expires_at: row.expires_at, - hit_count: row.hit_count, - }; - } catch (error) { - logger.error( - { error: error instanceof Error ? error.message : String(error) }, - "Failed to get cached text", - ); - return null; - } -} - -/** - * Insert or update a text analysis cache entry. - */ -export async function upsertCachedText( - text: string, - flags: string[], - source: "local" | "primary_ai" | "vision_llm", - expiresAt: number, -): Promise { - const now = Date.now(); - - try { - await executeAll( - `INSERT INTO text_analysis_cache (text, flags, source, analyzed_at, expires_at, hit_count) - VALUES ($1, $2, $3, $4, $5, 0) - ON CONFLICT (text) DO UPDATE SET - flags = EXCLUDED.flags, - source = EXCLUDED.source, - analyzed_at = EXCLUDED.analyzed_at, - expires_at = EXCLUDED.expires_at`, - [text, JSON.stringify(flags), source, now, expiresAt], - ); - } catch (error) { - logger.error( - { error: error instanceof Error ? error.message : String(error) }, - "Failed to upsert cached text", - ); - } -} - -/** - * Increment hit count for a cached text entry (called on cache hit). - */ -export async function incrementTextCacheHit(text: string): Promise { - try { - await executeAll( - `UPDATE text_analysis_cache SET hit_count = hit_count + 1 WHERE text = $1`, - [text], - ); - } catch (error) { - // Silent fail — this is just a counter, not critical - } -} - -/** - * Delete expired cache entries. Run periodically to keep the table clean. - */ -export async function pruneExpiredTexts(): Promise { - try { - const result = await executeAll( - `DELETE FROM text_analysis_cache WHERE expires_at < $1`, - [Date.now()], - ); - return (result as any).rowCount ?? 0; - } catch (error) { - logger.error( - { error: error instanceof Error ? error.message : String(error) }, - "Failed to prune expired texts", - ); - return 0; - } -} - -/** - * Get cache statistics for observability. - */ -export async function getTextCacheStats(): Promise<{ - total: number; - expired: number; - bySource: Record; -}> { - try { - const now = Date.now(); - - const [totalRow, expiredRow, sourceRows] = await Promise.all([ - executeAll(`SELECT count(*) as cnt FROM text_analysis_cache`), - executeAll( - `SELECT count(*) as cnt FROM text_analysis_cache WHERE expires_at < $1`, - [now], - ), - executeAll( - `SELECT source, count(*) as cnt FROM text_analysis_cache GROUP BY source`, - ), - ]); - - const bySource: Record = {}; - for (const row of sourceRows) { - bySource[row.source] = row.cnt; - } - - return { - total: totalRow[0]?.cnt ?? 0, - expired: expiredRow[0]?.cnt ?? 0, - bySource, - }; - } catch (error) { - logger.error( - { error: error instanceof Error ? error.message : String(error) }, - "Failed to get text cache stats", - ); - return { total: 0, expired: 0, bySource: {} }; - } -} - -// --------------------------------------------------------------------------- -// Media / Vision analysis cache helpers (reuses text_analysis_cache table) -// --------------------------------------------------------------------------- - -/** - * Generate a deterministic cache key for a sticker. - * Same sticker name → same key across sessions and servers. - */ -export function makeStickerCacheKey(stickerName: string): string { - return `sticker:${stickerName}`; -} - -/** - * Generate a deterministic cache key for a custom emoji by its Discord ID. - */ -export function makeCustomEmojiCacheKey(emojiId: string): string { - return `emoji:${emojiId}`; -} - -/** - * Generate a deterministic cache key for an image data URL. - * Hashes the first 128 chars of the data URL (enough to identify the image - * without storing the full base64 string as the key). - */ -export function makeImageCacheKey(dataUrl: string): string { - const prefix = dataUrl.slice(0, 128); - const hash = createHash("sha256").update(prefix).digest("hex").slice(0, 16); - return `image:${hash}`; -} - -/** - * Lookup a cached media analysis result. - * Returns the full cached text (the analysis summary string) or null. - */ -export async function getCachedMediaAnalysis( - cacheKey: string, -): Promise { - try { - const row = await executeGet( - `SELECT flags, hit_count - FROM text_analysis_cache - WHERE text = $1 AND expires_at > $2`, - [cacheKey, Date.now()], - ); - - if (!row) return null; - - // flags stores the analysis result for media entries - const result = JSON.parse(row.flags) as string; - return result || null; - } catch (error) { - logger.error( - { error: error instanceof Error ? error.message : String(error) }, - "Failed to get cached media analysis", - ); - return null; - } -} - -/** - * Store a media analysis result in the cache. - */ -export async function upsertCachedMediaAnalysis( - cacheKey: string, - analysisResult: string, - source: "vision_llm", - expiresAt: number, -): Promise { - const now = Date.now(); - - try { - await executeAll( - `INSERT INTO text_analysis_cache (text, flags, source, analyzed_at, expires_at, hit_count) - VALUES ($1, $2, $3, $4, $5, 0) - ON CONFLICT (text) DO UPDATE SET - flags = EXCLUDED.flags, - source = EXCLUDED.source, - analyzed_at = EXCLUDED.analyzed_at, - expires_at = EXCLUDED.expires_at`, - [cacheKey, JSON.stringify(analysisResult), source, now, expiresAt], - ); - } catch (error) { - logger.error( - { error: error instanceof Error ? error.message : String(error) }, - "Failed to upsert cached media analysis", - ); - } -} diff --git a/src/moderation/types.ts b/src/moderation/types.ts deleted file mode 100644 index a5f3c38..0000000 --- a/src/moderation/types.ts +++ /dev/null @@ -1,211 +0,0 @@ -import type { - BroadcasterClient, - ModerationBroadcaster, -} from "./broadcaster.js"; - -export type AIStatus = "pending" | "clean" | "warn" | "flagged" | "error"; -export type AISeverity = "none" | "low" | "medium" | "high" | "critical"; -export type AIRecommendedAction = - | "none" - | "monitor" - | "warn" - | "review" - | "delete" - | "escalate"; - -export type { BroadcasterClient, ModerationBroadcaster }; - -export interface MessageRecord { - id: string; - guild_id: string; - channel_id: string; - thread_id: string | null; - user_id: string; - username: string; - avatar_url: string | null; - content: string; - edited_content: string | null; - created_at: number; - edited_at: number | null; - deleted_at: number | null; - type: "text" | "edited" | "deleted"; - metadata: string | null; - ai_status?: AIStatus | null; - ai_moderation_flags?: string | null; - ai_moderation_score?: number | null; - ai_analysis?: string | null; - ai_categories?: string | null; - ai_severity?: AISeverity | null; - ai_confidence?: number | null; - ai_recommended_action?: AIRecommendedAction | null; - ai_analyzed_at?: number | null; - ai_error?: string | null; -} - -export interface AttachmentRecord { - id: string; - message_id: string; - guild_id: string; - channel_id: string; - thread_id: string | null; - user_id: string; - filename: string; - size: number; - type: string; - discord_url: string; - uploaded_url: string | null; - upload_status: "pending" | "uploaded" | "failed"; - upload_error: string | null; - created_at: number; - uploaded_at: number | null; -} - -export interface VoiceSegmentRecord { - id: string; - user_id: string; - session_id: string; - guild_id: string; - channel_id: string; - filename: string; - duration_ms: number; - created_at: number; -} - -export interface DashboardMessage { - id: string; - channel_id: string; - user_id: string; - username: string; - avatar_url: string | null; - content: string; - created_at: number; - type: "text" | "image" | "voice"; -} - -export interface MessageQuery { - guildId?: string; - channelId?: string; - threadId?: string; - status?: AIStatus[]; - userId?: string; - q?: string; - cursor?: string; - limit: number; -} - -export interface PageResult { - data: T[]; - nextCursor: string | null; -} - -export interface AnalysisResult { - messageId: string; - status: Exclude; - flags: string[]; - score: number; - analysis: string; - categories?: string[]; - severity?: AISeverity; - confidence?: number; - recommendedAction?: AIRecommendedAction; - policyVersion?: string; - evidence?: string[]; -} - -export type MediaMode = "music" | "screen"; -export type MediaSourceKind = - | "url" - | "local" - | "youtube" - | "spotify" - | "search"; -export type MediaQueueItemStatus = "queued" | "playing" | "failed"; - -export interface MediaQueueItem { - id: string; - mode: MediaMode; - source: string; - title: string; - kind: MediaSourceKind; - requestedBy: string; - addedAt: number; - status: MediaQueueItemStatus; -} - -export interface MediaState { - playing: boolean; - musicVolume: number; - current: MediaQueueItem | null; - queue: MediaQueueItem[]; -} - -export type ModerationWsEvent = - | { type: "ui_state"; state: unknown } - | { type: "user_state"; users: unknown[] } - | { type: "message_created"; data: MessageRecord } - | { type: "message_updated"; data: Partial & { id: string } } - | { type: "message_deleted"; data: { id: string; deleted_at: number } } - | { type: "message_analyzed"; data: MessageRecord } - | { type: "attachment_created"; data: AttachmentRecord } - | { type: "analysis_queue_status"; data: AnalysisQueueStatus } - | { type: "media_state"; state: MediaState } - | { type: "voice_recording_uploaded"; data: any }; - -export interface AnalysisQueueStatus { - queuedConversations: number; - activeRequests: number; - /** Number of single-message fallback calls currently awaiting the LLM. */ - activeIndividualRequests: number; - /** Number of message IDs sitting in the dedup set (in-flight or about to start). */ - individualInFlightCount: number; - /** True when the individual-fallback circuit breaker is tripped. */ - individualCircuitBreakerActive: boolean; - lastError: string | null; -} - -export type ReviewStatus = "pending" | "approved" | "rejected" | "escalated"; - -export interface MessageReview { - id: string; - message_id: string; - guild_id: string; - channel_id: string; - reviewer_id: string | null; - status: ReviewStatus; - notes: string | null; - created_at: number; - reviewed_at: number | null; -} - -export type ModerationActionType = - | "delete_message" - | "mute_user" - | "warn_user" - | "kick_user" - | "ban_user"; - -export interface ModerationAction { - id: string; - message_id: string | null; - user_id: string | null; - guild_id: string; - action_type: ModerationActionType; - reason: string | null; - executed_by: string | null; - status: "pending" | "executed" | "failed"; - error: string | null; - created_at: number; - executed_at: number | null; -} - -export interface RetentionPolicy { - id: string; - guild_id: string; - channel_id: string | null; - retention_days: number; - apply_to_media: boolean; - apply_to_voice: boolean; - enabled: boolean; - created_at: number; - updated_at: number; -} diff --git a/src/moderation/urlFetcher.ts b/src/moderation/urlFetcher.ts deleted file mode 100644 index 70ff8e9..0000000 --- a/src/moderation/urlFetcher.ts +++ /dev/null @@ -1,209 +0,0 @@ -import { resolve } from "node:dns/promises"; -import { isIP } from "node:net"; -import { createChildLogger } from "../logger.js"; - -const log = createChildLogger("urlFetcher"); - -export interface FetchedUrlContext { - url: string; - type: "image" | "text" | "error"; - data?: Buffer; - mimeType?: string; - textContent?: string; - error?: string; -} - -const MAX_FETCH_SIZE = 5 * 1024 * 1024; // 5 MB -const FETCH_TIMEOUT_MS = 8000; -const URL_REGEX = /https?:\/\/[^\s<]+[^<.,:;"')\]\s]/gi; - -/** - * Basic SSRF protection. - * Note: A sophisticated attacker could still use DNS rebinding. - */ -async function isSafeUrl(urlStr: string): Promise { - try { - const parsed = new URL(urlStr); - const host = parsed.hostname; - - // Block obvious local IPs/hostnames - if ( - host === "localhost" || - host === "127.0.0.1" || - host === "::1" || - host.startsWith("192.168.") || - host.startsWith("10.") || - /^172\.(1[6-9]|2[0-9]|3[0-1])\./.test(host) - ) { - return false; - } - - // Try resolving to check if it resolves to a local IP - if (!isIP(host)) { - try { - const addresses = await resolve(host); - for (const ip of addresses) { - if ( - ip === "127.0.0.1" || - ip.startsWith("192.168.") || - ip.startsWith("10.") || - /^172\.(1[6-9]|2[0-9]|3[0-1])\./.test(ip) - ) { - return false; - } - } - } catch (err) { - // If DNS fails, we can't fetch it anyway - return false; - } - } - - return true; - } catch (err) { - return false; - } -} - -function extractOgImage(html: string): string | null { - // Look for or - const ogRegex = - /]*(?:property|name)=["'](?:og:image|twitter:image)["'][^>]*content=["']([^"']+)["']/i; - const match = html.match(ogRegex); - if (match && match[1]) { - // Unescape basic HTML entities - return match[1].replace(/&/g, "&").replace(/"/g, '"'); - } - - // Try reversed attribute order: - const ogRegexRev = - /]*content=["']([^"']+)["'][^>]*(?:property|name)=["'](?:og:image|twitter:image)["']/i; - const matchRev = html.match(ogRegexRev); - if (matchRev && matchRev[1]) { - return matchRev[1].replace(/&/g, "&").replace(/"/g, '"'); - } - - return null; -} - -function truncateAndCleanHtml(html: string, maxLen = 1000): string { - // Strip