refactor: split monolith into 3 microservices (frontend, backend, discord-gateway)
- Extract services into services/{frontend,backend,discord-gateway}
- Create packages/shared/ for shared logger, errors, utils, types
- Setup Modular MVC pattern in backend (controller→service→repository)
- Setup event-driven architecture in discord-gateway with Redis pub/sub
- Move Docker files to infra/docker/ with per-service Dockerfiles
- Update docker-compose.yml to use Traefik-only routing (no port exposes)
- Update GitHub Actions deploy workflow for multi-service matrix build
- Fix all import paths and resolve type errors across all services
- All 3 services pass tsc --noEmit clean
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
MythEclipse
co-authored by
Claude Opus 4.8
Claude Opus 4.8
parent
bda8304bb9
commit
c48a0c5e3b
@@ -0,0 +1,275 @@
|
||||
# Backend Service Architecture Map
|
||||
|
||||
## Directory Structure
|
||||
|
||||
```
|
||||
services/backend/
|
||||
├── src/
|
||||
│ ├── shared/ # Shared infrastructure (no business logic)
|
||||
│ │ ├── config/
|
||||
│ │ │ └── index.ts # Zod-validated environment config
|
||||
│ │ ├── database/
|
||||
│ │ │ └── index.ts # Drizzle ORM initialization & connection pool
|
||||
│ │ ├── errors/
|
||||
│ │ │ └── index.ts # Custom error classes (AppError, ValidationError, etc.)
|
||||
│ │ ├── logger/
|
||||
│ │ │ └── index.ts # Pino logger with child context support
|
||||
│ │ ├── middlewares/
|
||||
│ │ │ └── index.ts # Express middleware (errorHandler, asyncHandler, adminAuth)
|
||||
│ │ └── utils/ # Utility functions (placeholder)
|
||||
│ │
|
||||
│ ├── modules/ # Feature modules (Modular MVC pattern)
|
||||
│ │ ├── messages/
|
||||
│ │ │ ├── messages.schema.ts # Zod validation schemas (MessageQuery, MessageCreate, MessageUpdate)
|
||||
│ │ │ ├── messages.repository.ts # Database operations (findMany, findById, create, update, delete)
|
||||
│ │ │ ├── messages.service.ts # Business logic (validation, orchestration)
|
||||
│ │ │ ├── messages.controller.ts # Request handlers (parse → service → response)
|
||||
│ │ │ └── routes/
|
||||
│ │ │ └── index.ts # Express router (GET /api/messages, etc.)
|
||||
│ │ │
|
||||
│ │ ├── analytics/
|
||||
│ │ │ ├── analytics.schema.ts
|
||||
│ │ │ ├── analytics.repository.ts
|
||||
│ │ │ ├── analytics.service.ts
|
||||
│ │ │ ├── analytics.controller.ts
|
||||
│ │ │ └── routes/
|
||||
│ │ │ └── index.ts
|
||||
│ │ │
|
||||
│ │ ├── media/
|
||||
│ │ │ ├── media.service.ts
|
||||
│ │ │ └── routes/
|
||||
│ │ │ └── index.ts
|
||||
│ │ │
|
||||
│ │ ├── voice/
|
||||
│ │ │ ├── voice.service.ts
|
||||
│ │ │ └── routes/
|
||||
│ │ │ └── index.ts
|
||||
│ │ │
|
||||
│ │ └── health/
|
||||
│ │ ├── health.schema.ts
|
||||
│ │ ├── health.repository.ts
|
||||
│ │ ├── health.service.ts
|
||||
│ │ ├── health.controller.ts
|
||||
│ │ └── routes/
|
||||
│ │ └── index.ts
|
||||
│ │
|
||||
│ ├── http/
|
||||
│ │ ├── app.ts # Express app factory (middleware, routes, error handler)
|
||||
│ │ └── server.ts # HTTP server startup (port binding, graceful shutdown)
|
||||
│ │
|
||||
│ ├── ws/ # WebSocket server (placeholder for real-time updates)
|
||||
│ │ └── server.ts # Redis pub/sub listener for Discord Gateway events
|
||||
│ │
|
||||
│ └── index.ts # Entry point (main function, signal handlers)
|
||||
│
|
||||
├── package.json # Backend dependencies
|
||||
├── tsconfig.json # TypeScript configuration
|
||||
└── README.md # Backend-specific documentation
|
||||
```
|
||||
|
||||
## Layer Separation
|
||||
|
||||
### 1. Controller Layer
|
||||
**File:** `modules/*/[module].controller.ts`
|
||||
**Responsibility:** HTTP request handling only
|
||||
- Parse request (query, params, body)
|
||||
- Validate using Zod schemas
|
||||
- Call service methods
|
||||
- Return HTTP response (200, 400, 404, 500)
|
||||
- **No database calls**
|
||||
- **No business logic**
|
||||
|
||||
**Example:**
|
||||
```typescript
|
||||
export function handleListMessages(req: Request, res: Response, next: NextFunction) {
|
||||
return asyncHandler(async (req: Request, res: Response) => {
|
||||
const query = messageQuerySchema.parse(req.query);
|
||||
const result = await messagesService.listMessages(query);
|
||||
res.json(result);
|
||||
})(req, res, next);
|
||||
}
|
||||
```
|
||||
|
||||
### 2. Service Layer
|
||||
**File:** `modules/*/[module].service.ts`
|
||||
**Responsibility:** Business logic and orchestration
|
||||
- Validate input (throw ValidationError if invalid)
|
||||
- Orchestrate repository calls
|
||||
- Apply business rules
|
||||
- Handle cross-cutting concerns (auth, permissions)
|
||||
- **No database calls directly**
|
||||
- **No HTTP request/response handling**
|
||||
|
||||
**Example:**
|
||||
```typescript
|
||||
async listMessages(query: MessageQuery) {
|
||||
if (!query.channelId && !query.guildId) {
|
||||
throw new ValidationError("Either channelId or guildId is required");
|
||||
}
|
||||
return messagesRepository.findMany(query);
|
||||
}
|
||||
```
|
||||
|
||||
### 3. Repository Layer
|
||||
**File:** `modules/*/[module].repository.ts`
|
||||
**Responsibility:** All database operations
|
||||
- Execute Drizzle ORM queries
|
||||
- Handle database errors
|
||||
- Return raw data (no transformation)
|
||||
- **No business logic**
|
||||
- **No HTTP handling**
|
||||
|
||||
**Example:**
|
||||
```typescript
|
||||
async findMany(query: MessageQuery) {
|
||||
const db = getDatabase();
|
||||
return db.select().from(messagesTable).where(...).limit(query.limit);
|
||||
}
|
||||
```
|
||||
|
||||
### 4. Schema Layer
|
||||
**File:** `modules/*/[module].schema.ts`
|
||||
**Responsibility:** Zod validation schemas
|
||||
- Define request/response types
|
||||
- Validate at controller entry point
|
||||
- Export TypeScript types
|
||||
|
||||
**Example:**
|
||||
```typescript
|
||||
export const messageQuerySchema = z.object({
|
||||
channelId: z.string().optional(),
|
||||
limit: z.coerce.number().int().positive().default(50),
|
||||
});
|
||||
```
|
||||
|
||||
## Module Responsibilities
|
||||
|
||||
| Module | Purpose | Routes |
|
||||
|--------|---------|--------|
|
||||
| **messages** | Text message storage & retrieval | GET /api/messages, GET /api/messages/:channelId |
|
||||
| **analytics** | Moderation statistics & trends | GET /api/analytics/overview, /daily-trend, /hourly-stats |
|
||||
| **media** | Media file management | GET /api/media/list, POST /api/media/upload |
|
||||
| **voice** | Voice recording management | GET /api/voice/recordings, POST /api/voice/connect |
|
||||
| **health** | Service health checks | GET /api/health |
|
||||
|
||||
## Data Flow
|
||||
|
||||
### Request Flow (HTTP)
|
||||
```
|
||||
Client Request
|
||||
↓
|
||||
Express Router (routes/index.ts)
|
||||
↓
|
||||
Controller (parse request, validate schema)
|
||||
↓
|
||||
Service (business logic, validation)
|
||||
↓
|
||||
Repository (database query)
|
||||
↓
|
||||
Database (PostgreSQL)
|
||||
↓
|
||||
Repository (return data)
|
||||
↓
|
||||
Service (transform/orchestrate)
|
||||
↓
|
||||
Controller (format response)
|
||||
↓
|
||||
Client Response
|
||||
```
|
||||
|
||||
### Event Flow (WebSocket - Future)
|
||||
```
|
||||
Discord Gateway (publishes event)
|
||||
↓
|
||||
Redis pub/sub
|
||||
↓
|
||||
Backend WebSocket Server (ws/server.ts)
|
||||
↓
|
||||
Broadcast to connected clients
|
||||
↓
|
||||
Frontend (receives real-time update)
|
||||
```
|
||||
|
||||
## Dependency Rules
|
||||
|
||||
### ✅ Allowed
|
||||
- Controller → Service
|
||||
- Service → Repository
|
||||
- Service → Config
|
||||
- Service → Logger
|
||||
- Repository → Database
|
||||
- Any layer → Errors, Logger, Config
|
||||
|
||||
### ❌ Forbidden
|
||||
- Repository → Service (data flows up, not down)
|
||||
- Repository → Controller
|
||||
- Service → HTTP (no req/res in service)
|
||||
- Controller → Database (must go through service)
|
||||
- Cross-module repository imports (each module owns its data)
|
||||
|
||||
## Error Handling
|
||||
|
||||
All errors inherit from `AppError` with `code` and `statusCode`:
|
||||
|
||||
```typescript
|
||||
throw new ValidationError("Invalid input", { field: "error" }); // 400
|
||||
throw new NotFoundError("Message not found"); // 404
|
||||
throw new UnauthorizedError("Invalid password"); // 401
|
||||
throw new ForbiddenError("Access denied"); // 403
|
||||
throw new AppError("Custom error", "CUSTOM_CODE", 500); // 500
|
||||
```
|
||||
|
||||
## Configuration
|
||||
|
||||
All config via environment variables (`.env`), validated with Zod in `shared/config/index.ts`:
|
||||
|
||||
```env
|
||||
# Server
|
||||
WEBSERVER_PORT=3001
|
||||
NODE_ENV=development
|
||||
LOG_LEVEL=info
|
||||
|
||||
# Database
|
||||
DATABASE_URL=postgresql://user:pass@localhost:5432/discord_moderation
|
||||
# OR
|
||||
DATABASE_HOST=localhost
|
||||
DATABASE_PORT=5432
|
||||
DATABASE_NAME=discord_moderation
|
||||
DATABASE_USER=postgres
|
||||
DATABASE_PASSWORD=secret
|
||||
|
||||
# Redis (optional, for pub/sub)
|
||||
REDIS_URL=redis://localhost:6379
|
||||
|
||||
# Discord
|
||||
MONITOR_GUILD_ID=123456789
|
||||
|
||||
# Admin
|
||||
ADMIN_PASSWORD=secret123
|
||||
```
|
||||
|
||||
## Testing Strategy
|
||||
|
||||
Each module should have tests:
|
||||
- `messages.repository.test.ts` — Database query tests
|
||||
- `messages.service.test.ts` — Business logic tests
|
||||
- `messages.controller.test.ts` — HTTP handler tests
|
||||
|
||||
Use Vitest with mocked database and services.
|
||||
|
||||
## Next Steps
|
||||
|
||||
1. **Migrate Drizzle schema** from `src/database/schema.ts` to `services/backend/src/shared/database/schema.ts`
|
||||
2. **Implement repository queries** for each module using Drizzle ORM
|
||||
3. **Add WebSocket server** in `src/ws/server.ts` with Redis pub/sub listener
|
||||
4. **Create Discord Gateway service** in `services/discord-gateway/` (separate microservice)
|
||||
5. **Add Docker & CI/CD** for multi-service deployment
|
||||
6. **Write integration tests** for full request flow
|
||||
|
||||
## Circular Dependency Check
|
||||
|
||||
✅ No circular dependencies detected:
|
||||
- Modules are independent (each owns its data)
|
||||
- Layers flow upward only (Repository → Service → Controller)
|
||||
- Shared infrastructure has no dependencies on modules
|
||||
- Cross-module communication via events (Redis pub/sub), not direct imports
|
||||
@@ -0,0 +1,41 @@
|
||||
{
|
||||
"name": "discord-moderation-backend",
|
||||
"version": "1.0.0",
|
||||
"description": "Backend service for Discord moderation monitoring",
|
||||
"type": "module",
|
||||
"main": "dist/index.js",
|
||||
"scripts": {
|
||||
"dev": "tsx watch src/index.ts",
|
||||
"start": "node dist/index.js",
|
||||
"build": "tsc",
|
||||
"typecheck": "tsc --noEmit",
|
||||
"lint": "biome check --diagnostic-level=error .",
|
||||
"format": "biome format --write .",
|
||||
"test": "vitest run"
|
||||
},
|
||||
"dependencies": {
|
||||
"@discordjs/voice": "^0.19.2",
|
||||
"@types/pg": "^8.20.0",
|
||||
"axios": "^1.16.1",
|
||||
"dotenv": "^17.4.2",
|
||||
"drizzle-orm": "^0.45.2",
|
||||
"express": "^5.2.1",
|
||||
"helmet": "^8.1.0",
|
||||
"ioredis": "^5.11.0",
|
||||
"pg": "^8.21.0",
|
||||
"pino": "^9.6.0",
|
||||
"pino-http": "^10.3.0",
|
||||
"prom-client": "^15.1.3",
|
||||
"ws": "^8.20.1",
|
||||
"zod": "^4.4.3"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@biomejs/biome": "latest",
|
||||
"@types/express": "^5.0.6",
|
||||
"@types/node": "^25.9.0",
|
||||
"@types/ws": "^8.18.1",
|
||||
"tsx": "^4.22.2",
|
||||
"typescript": "^5.9.3",
|
||||
"vitest": "latest"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
import express, {
|
||||
type Express,
|
||||
type NextFunction,
|
||||
type Request,
|
||||
type Response,
|
||||
} from "express";
|
||||
import helmet from "helmet";
|
||||
import { createAnalyticsRouter } from "../modules/analytics/routes/index.js";
|
||||
import { createHealthRouter } from "../modules/health/routes/index.js";
|
||||
import { createMediaRouter } from "../modules/media/routes/index.js";
|
||||
import { createMessagesRouter } from "../modules/messages/routes/index.js";
|
||||
import { createVoiceRouter } from "../modules/voice/routes/index.js";
|
||||
import { createChildLogger } from "../shared/logger/index.js";
|
||||
import { errorHandler } from "../shared/middlewares/index.js";
|
||||
|
||||
const logger = createChildLogger("http.app");
|
||||
|
||||
export function createHttpApp(): Express {
|
||||
const app = express();
|
||||
|
||||
// Security middleware
|
||||
app.use(
|
||||
helmet({
|
||||
contentSecurityPolicy: false,
|
||||
}),
|
||||
);
|
||||
|
||||
// Body parsing
|
||||
app.use(express.json());
|
||||
app.use(express.urlencoded({ extended: true }));
|
||||
|
||||
// Request logging
|
||||
app.use((req: Request, res: Response, next: NextFunction) => {
|
||||
if (req.path.startsWith("/api/")) {
|
||||
res.set("Cache-Control", "no-store");
|
||||
}
|
||||
res.on("finish", () => {
|
||||
if (req.originalUrl.startsWith("/.well-known/")) return;
|
||||
if (req.originalUrl === "/favicon.ico") return;
|
||||
if (res.statusCode >= 400) {
|
||||
logger.warn(
|
||||
{
|
||||
method: req.method,
|
||||
url: req.originalUrl,
|
||||
statusCode: res.statusCode,
|
||||
},
|
||||
"HTTP request failed",
|
||||
);
|
||||
}
|
||||
});
|
||||
next();
|
||||
});
|
||||
|
||||
// Health check (no auth required)
|
||||
app.use("/api", createHealthRouter());
|
||||
|
||||
// API routes
|
||||
app.use("/api", createMessagesRouter());
|
||||
app.use("/api", createAnalyticsRouter());
|
||||
app.use("/api", createMediaRouter());
|
||||
app.use("/api", createVoiceRouter());
|
||||
|
||||
// 404 handler
|
||||
app.use((_req: Request, res: Response) => {
|
||||
res.status(404).json({
|
||||
error: "NOT_FOUND",
|
||||
message: "Endpoint not found",
|
||||
});
|
||||
});
|
||||
|
||||
// Error handler (must be last)
|
||||
app.use(errorHandler);
|
||||
|
||||
return app;
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
import { config } from "../shared/config/index.js";
|
||||
import { initializeDatabase } from "../shared/database/index.js";
|
||||
import { createChildLogger } from "../shared/logger/index.js";
|
||||
import { createHttpApp } from "./app.js";
|
||||
|
||||
const logger = createChildLogger("http.server");
|
||||
|
||||
export async function startHttpServer() {
|
||||
await initializeDatabase();
|
||||
|
||||
const app = createHttpApp();
|
||||
const port = config.WEBSERVER_PORT;
|
||||
|
||||
return new Promise<void>((resolve, reject) => {
|
||||
const server = app.listen(port, () => {
|
||||
logger.info({ port }, "HTTP server started");
|
||||
resolve();
|
||||
});
|
||||
|
||||
server.on("error", (err) => {
|
||||
logger.error({ err }, "HTTP server error");
|
||||
reject(err);
|
||||
});
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
import { startHttpServer } from "./http/server.js";
|
||||
import { createChildLogger } from "./shared/logger/index.js";
|
||||
|
||||
const logger = createChildLogger("backend");
|
||||
|
||||
async function main() {
|
||||
try {
|
||||
logger.info("Starting Discord Moderation Backend Service");
|
||||
await startHttpServer();
|
||||
logger.info("Backend service ready");
|
||||
} catch (err) {
|
||||
logger.error({ err }, "Failed to start backend service");
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
// Graceful shutdown
|
||||
process.on("SIGINT", () => {
|
||||
logger.info("Received SIGINT, shutting down gracefully");
|
||||
process.exit(0);
|
||||
});
|
||||
|
||||
process.on("SIGTERM", () => {
|
||||
logger.info("Received SIGTERM, shutting down gracefully");
|
||||
process.exit(0);
|
||||
});
|
||||
|
||||
process.on("uncaughtException", (err) => {
|
||||
logger.error({ err }, "Uncaught exception");
|
||||
process.exit(1);
|
||||
});
|
||||
|
||||
process.on("unhandledRejection", (reason) => {
|
||||
logger.error({ reason }, "Unhandled rejection");
|
||||
process.exit(1);
|
||||
});
|
||||
|
||||
main();
|
||||
@@ -0,0 +1,96 @@
|
||||
import type { NextFunction, Request, Response } from "express";
|
||||
import { createChildLogger } from "../../shared/logger/index.js";
|
||||
import { asyncHandler } from "../../shared/middlewares/index.js";
|
||||
import { analyticsQuerySchema } from "./analytics.schema.js";
|
||||
import { analyticsService } from "./analytics.service.js";
|
||||
|
||||
const logger = createChildLogger("analytics.controller");
|
||||
|
||||
function requireQueryString(value: unknown, name: string): string {
|
||||
if (typeof value !== "string" || value.length === 0) {
|
||||
throw new Error(`Missing query parameter: ${name}`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
export function handleGetOverview(
|
||||
req: Request,
|
||||
res: Response,
|
||||
next: NextFunction,
|
||||
) {
|
||||
return asyncHandler(async (req: Request, res: Response) => {
|
||||
const query = analyticsQuerySchema.parse(req.query);
|
||||
logger.debug({ query }, "Handling get overview");
|
||||
const result = await analyticsService.getOverview(query);
|
||||
res.json(result);
|
||||
})(req, res, next);
|
||||
}
|
||||
|
||||
export function handleGetDailyTrend(
|
||||
req: Request,
|
||||
res: Response,
|
||||
next: NextFunction,
|
||||
) {
|
||||
return asyncHandler(async (req: Request, res: Response) => {
|
||||
const guildId = requireQueryString(req.query.guildId, "guildId");
|
||||
const hours = req.query.hours ? Number(req.query.hours) : 24;
|
||||
logger.debug({ guildId, hours }, "Handling get daily trend");
|
||||
const result = await analyticsService.getDailyTrend(guildId, hours);
|
||||
res.json(result);
|
||||
})(req, res, next);
|
||||
}
|
||||
|
||||
export function handleGetHourlyStats(
|
||||
req: Request,
|
||||
res: Response,
|
||||
next: NextFunction,
|
||||
) {
|
||||
return asyncHandler(async (req: Request, res: Response) => {
|
||||
const guildId = requireQueryString(req.query.guildId, "guildId");
|
||||
const hours = req.query.hours ? Number(req.query.hours) : 24;
|
||||
logger.debug({ guildId, hours }, "Handling get hourly stats");
|
||||
const result = await analyticsService.getHourlyStats(guildId, hours);
|
||||
res.json(result);
|
||||
})(req, res, next);
|
||||
}
|
||||
|
||||
export function handleGetTopViolators(
|
||||
req: Request,
|
||||
res: Response,
|
||||
next: NextFunction,
|
||||
) {
|
||||
return asyncHandler(async (req: Request, res: Response) => {
|
||||
const guildId = requireQueryString(req.query.guildId, "guildId");
|
||||
const limit = req.query.limit ? Number(req.query.limit) : 10;
|
||||
logger.debug({ guildId, limit }, "Handling get top violators");
|
||||
const result = await analyticsService.getTopViolators(guildId, limit);
|
||||
res.json(result);
|
||||
})(req, res, next);
|
||||
}
|
||||
|
||||
export function handleGetUserLeaderboard(
|
||||
req: Request,
|
||||
res: Response,
|
||||
next: NextFunction,
|
||||
) {
|
||||
return asyncHandler(async (req: Request, res: Response) => {
|
||||
const guildId = requireQueryString(req.query.guildId, "guildId");
|
||||
const limit = req.query.limit ? Number(req.query.limit) : 10;
|
||||
logger.debug({ guildId, limit }, "Handling get user leaderboard");
|
||||
const result = await analyticsService.getUserLeaderboard(guildId, limit);
|
||||
res.json(result);
|
||||
})(req, res, next);
|
||||
}
|
||||
|
||||
export function handleGetModerationStats(
|
||||
req: Request,
|
||||
res: Response,
|
||||
next: NextFunction,
|
||||
) {
|
||||
return asyncHandler(async (req: Request, res: Response) => {
|
||||
const guildId = requireQueryString(req.query.guildId, "guildId");
|
||||
logger.debug({ guildId }, "Handling get moderation stats");
|
||||
const result = await analyticsService.getModerationStats(guildId);
|
||||
res.json(result);
|
||||
})(req, res, next);
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
import { createChildLogger } from "../../shared/logger/index.js";
|
||||
|
||||
const logger = createChildLogger("analytics.repository");
|
||||
|
||||
export class AnalyticsRepository {
|
||||
async getOverview(guildId: string, channelId?: string, hours = 24) {
|
||||
logger.debug({ guildId, channelId, hours }, "Getting analytics overview");
|
||||
// TODO: Implement actual Drizzle ORM queries
|
||||
return {
|
||||
totalMessages: 0,
|
||||
totalUsers: 0,
|
||||
flaggedMessages: 0,
|
||||
averageSeverity: 0,
|
||||
};
|
||||
}
|
||||
|
||||
async getDailyTrend(guildId: string, hours = 24) {
|
||||
logger.debug({ guildId, hours }, "Getting daily trend");
|
||||
// TODO: Implement actual Drizzle ORM queries
|
||||
return [];
|
||||
}
|
||||
|
||||
async getHourlyStats(guildId: string, hours = 24) {
|
||||
logger.debug({ guildId, hours }, "Getting hourly stats");
|
||||
// TODO: Implement actual Drizzle ORM queries
|
||||
return [];
|
||||
}
|
||||
|
||||
async getTopViolators(guildId: string, limit = 10) {
|
||||
logger.debug({ guildId, limit }, "Getting top violators");
|
||||
// TODO: Implement actual Drizzle ORM queries
|
||||
return [];
|
||||
}
|
||||
|
||||
async getUserLeaderboard(guildId: string, limit = 10) {
|
||||
logger.debug({ guildId, limit }, "Getting user leaderboard");
|
||||
// TODO: Implement actual Drizzle ORM queries
|
||||
return [];
|
||||
}
|
||||
|
||||
async getModerationStats(guildId: string) {
|
||||
logger.debug({ guildId }, "Getting moderation stats");
|
||||
// TODO: Implement actual Drizzle ORM queries
|
||||
return {
|
||||
clean: 0,
|
||||
warn: 0,
|
||||
flagged: 0,
|
||||
error: 0,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export const analyticsRepository = new AnalyticsRepository();
|
||||
@@ -0,0 +1,9 @@
|
||||
import { z } from "zod";
|
||||
|
||||
export const analyticsQuerySchema = z.object({
|
||||
guildId: z.string(),
|
||||
channelId: z.string().optional(),
|
||||
hours: z.coerce.number().int().positive().default(24),
|
||||
});
|
||||
|
||||
export type AnalyticsQuery = z.infer<typeof analyticsQuerySchema>;
|
||||
@@ -0,0 +1,61 @@
|
||||
import { config } from "../../shared/config/index.js";
|
||||
import { ForbiddenError, ValidationError } from "../../shared/errors/index.js";
|
||||
import { createChildLogger } from "../../shared/logger/index.js";
|
||||
import { analyticsRepository } from "./analytics.repository.js";
|
||||
import type { AnalyticsQuery } from "./analytics.schema.js";
|
||||
|
||||
const logger = createChildLogger("analytics.service");
|
||||
|
||||
export class AnalyticsService {
|
||||
private assertMonitorGuild(guildId: string) {
|
||||
if (!config.MONITOR_GUILD_ID) {
|
||||
throw new ValidationError("MONITOR_GUILD_ID is not configured");
|
||||
}
|
||||
|
||||
if (guildId !== config.MONITOR_GUILD_ID) {
|
||||
throw new ForbiddenError("Analytics are restricted to the monitor guild");
|
||||
}
|
||||
}
|
||||
|
||||
async getOverview(query: AnalyticsQuery) {
|
||||
this.assertMonitorGuild(query.guildId);
|
||||
logger.debug({ query }, "Getting analytics overview");
|
||||
return analyticsRepository.getOverview(
|
||||
query.guildId,
|
||||
query.channelId,
|
||||
query.hours,
|
||||
);
|
||||
}
|
||||
|
||||
async getDailyTrend(guildId: string, hours = 24) {
|
||||
this.assertMonitorGuild(guildId);
|
||||
logger.debug({ guildId, hours }, "Getting daily trend");
|
||||
return analyticsRepository.getDailyTrend(guildId, hours);
|
||||
}
|
||||
|
||||
async getHourlyStats(guildId: string, hours = 24) {
|
||||
this.assertMonitorGuild(guildId);
|
||||
logger.debug({ guildId, hours }, "Getting hourly stats");
|
||||
return analyticsRepository.getHourlyStats(guildId, hours);
|
||||
}
|
||||
|
||||
async getTopViolators(guildId: string, limit = 10) {
|
||||
this.assertMonitorGuild(guildId);
|
||||
logger.debug({ guildId, limit }, "Getting top violators");
|
||||
return analyticsRepository.getTopViolators(guildId, limit);
|
||||
}
|
||||
|
||||
async getUserLeaderboard(guildId: string, limit = 10) {
|
||||
this.assertMonitorGuild(guildId);
|
||||
logger.debug({ guildId, limit }, "Getting user leaderboard");
|
||||
return analyticsRepository.getUserLeaderboard(guildId, limit);
|
||||
}
|
||||
|
||||
async getModerationStats(guildId: string) {
|
||||
this.assertMonitorGuild(guildId);
|
||||
logger.debug({ guildId }, "Getting moderation stats");
|
||||
return analyticsRepository.getModerationStats(guildId);
|
||||
}
|
||||
}
|
||||
|
||||
export const analyticsService = new AnalyticsService();
|
||||
@@ -0,0 +1,23 @@
|
||||
import type { Router } from "express";
|
||||
import express from "express";
|
||||
import {
|
||||
handleGetDailyTrend,
|
||||
handleGetHourlyStats,
|
||||
handleGetModerationStats,
|
||||
handleGetOverview,
|
||||
handleGetTopViolators,
|
||||
handleGetUserLeaderboard,
|
||||
} from "../analytics.controller.js";
|
||||
|
||||
export function createAnalyticsRouter(): Router {
|
||||
const router = express.Router();
|
||||
|
||||
router.get("/analytics/overview", handleGetOverview);
|
||||
router.get("/analytics/daily-trend", handleGetDailyTrend);
|
||||
router.get("/analytics/hourly-stats", handleGetHourlyStats);
|
||||
router.get("/analytics/top-violators", handleGetTopViolators);
|
||||
router.get("/analytics/user-leaderboard", handleGetUserLeaderboard);
|
||||
router.get("/analytics/moderation-stats", handleGetModerationStats);
|
||||
|
||||
return router;
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
import type { NextFunction, Request, Response } from "express";
|
||||
import { asyncHandler } from "../../shared/middlewares/index.js";
|
||||
import { healthService } from "./health.service.js";
|
||||
|
||||
export function handleHealthCheck(
|
||||
req: Request,
|
||||
res: Response,
|
||||
next: NextFunction,
|
||||
) {
|
||||
return asyncHandler(async (req: Request, res: Response) => {
|
||||
const verbose = req.query.verbose === "true";
|
||||
const result = await healthService.getHealth(verbose);
|
||||
const status = result.status === "healthy" ? 200 : 503;
|
||||
res.status(status).json(result);
|
||||
})(req, res, next);
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
import { createChildLogger } from "../../shared/logger/index.js";
|
||||
|
||||
const logger = createChildLogger("health.repository");
|
||||
|
||||
export class HealthRepository {
|
||||
async checkDatabaseConnection() {
|
||||
try {
|
||||
// TODO: Implement actual health check
|
||||
return { connected: true };
|
||||
} catch (err) {
|
||||
logger.error({ err }, "Database health check failed");
|
||||
return { connected: false };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export const healthRepository = new HealthRepository();
|
||||
@@ -0,0 +1,5 @@
|
||||
import { z } from "zod";
|
||||
|
||||
export const healthCheckSchema = z.object({
|
||||
verbose: z.coerce.boolean().optional().default(false),
|
||||
});
|
||||
@@ -0,0 +1,20 @@
|
||||
import { createChildLogger } from "../../shared/logger/index.js";
|
||||
import { healthRepository } from "./health.repository.js";
|
||||
|
||||
const logger = createChildLogger("health.service");
|
||||
|
||||
export class HealthService {
|
||||
async getHealth(verbose = false) {
|
||||
const dbStatus = await healthRepository.checkDatabaseConnection();
|
||||
|
||||
return {
|
||||
status: dbStatus.connected ? "healthy" : "degraded",
|
||||
timestamp: Date.now(),
|
||||
...(verbose && {
|
||||
database: dbStatus,
|
||||
}),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export const healthService = new HealthService();
|
||||
@@ -0,0 +1,12 @@
|
||||
import type { Router } from "express";
|
||||
import express from "express";
|
||||
import { handleHealthCheck } from "../health.controller.js";
|
||||
|
||||
export function createHealthRouter(): Router {
|
||||
const router = express.Router();
|
||||
|
||||
// GET /api/health
|
||||
router.get("/health", handleHealthCheck);
|
||||
|
||||
return router;
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
import { createChildLogger } from "../../shared/logger/index.js";
|
||||
|
||||
const logger = createChildLogger("media.service");
|
||||
|
||||
export class MediaService {
|
||||
// TODO: Implement media service methods
|
||||
}
|
||||
|
||||
export const mediaService = new MediaService();
|
||||
@@ -0,0 +1,13 @@
|
||||
import type { Router } from "express";
|
||||
import express from "express";
|
||||
|
||||
export function createMediaRouter(): Router {
|
||||
const router = express.Router();
|
||||
|
||||
// TODO: Implement media routes
|
||||
// GET /api/media/list
|
||||
// POST /api/media/upload
|
||||
// GET /api/media/:id
|
||||
|
||||
return router;
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
import type { NextFunction, Request, Response } from "express";
|
||||
import { createChildLogger } from "../../shared/logger/index.js";
|
||||
import { asyncHandler } from "../../shared/middlewares/index.js";
|
||||
import { messageQuerySchema } from "./messages.schema.js";
|
||||
import { messagesService } from "./messages.service.js";
|
||||
|
||||
const logger = createChildLogger("messages.controller");
|
||||
|
||||
function requireRouteParam(
|
||||
value: string | string[] | undefined,
|
||||
name: string,
|
||||
): string {
|
||||
if (typeof value !== "string" || value.length === 0) {
|
||||
throw new Error(`Missing route parameter: ${name}`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
export function handleListMessages(
|
||||
req: Request,
|
||||
res: Response,
|
||||
next: NextFunction,
|
||||
) {
|
||||
return asyncHandler(async (req: Request, res: Response) => {
|
||||
const query = messageQuerySchema.parse(req.query);
|
||||
logger.debug({ query }, "Handling list messages request");
|
||||
const result = await messagesService.listMessages(query);
|
||||
res.json(result);
|
||||
})(req, res, next);
|
||||
}
|
||||
|
||||
export function handleGetMessagesByChannel(
|
||||
req: Request,
|
||||
res: Response,
|
||||
next: NextFunction,
|
||||
) {
|
||||
return asyncHandler(async (req: Request, res: Response) => {
|
||||
const channelId = requireRouteParam(req.params.channelId, "channelId");
|
||||
const query = messageQuerySchema.parse(req.query);
|
||||
logger.debug({ channelId, query }, "Handling get messages by channel");
|
||||
const result = await messagesService.getMessagesByChannel(channelId, query);
|
||||
res.json(result);
|
||||
})(req, res, next);
|
||||
}
|
||||
|
||||
export function handleGetMessageById(
|
||||
req: Request,
|
||||
res: Response,
|
||||
next: NextFunction,
|
||||
) {
|
||||
return asyncHandler(async (req: Request, res: Response) => {
|
||||
const id = requireRouteParam(req.params.id, "id");
|
||||
logger.debug({ id }, "Handling get message by ID");
|
||||
const result = await messagesService.getMessageById(id);
|
||||
res.json(result);
|
||||
})(req, res, next);
|
||||
}
|
||||
|
||||
export function handleGetAttachmentsByChannel(
|
||||
req: Request,
|
||||
res: Response,
|
||||
next: NextFunction,
|
||||
) {
|
||||
return asyncHandler(async (req: Request, res: Response) => {
|
||||
const channelId = requireRouteParam(req.params.channelId, "channelId");
|
||||
const query = messageQuerySchema.parse(req.query);
|
||||
logger.debug({ channelId, query }, "Handling get attachments by channel");
|
||||
const result = await messagesService.getAttachmentsByChannel(
|
||||
channelId,
|
||||
query,
|
||||
);
|
||||
res.json(result);
|
||||
})(req, res, next);
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
import { getDatabase } from "../../shared/database/index.js";
|
||||
import { createChildLogger } from "../../shared/logger/index.js";
|
||||
import type {
|
||||
MessageCreate,
|
||||
MessageQuery,
|
||||
MessageUpdate,
|
||||
} from "./messages.schema.js";
|
||||
|
||||
const logger = createChildLogger("messages.repository");
|
||||
|
||||
export class MessagesRepository {
|
||||
async findMany(query: MessageQuery) {
|
||||
const db = getDatabase();
|
||||
logger.debug({ query }, "Finding messages");
|
||||
|
||||
// TODO: Implement actual Drizzle ORM queries
|
||||
// This is a placeholder that will be filled in when schema is migrated
|
||||
return {
|
||||
messages: [],
|
||||
total: 0,
|
||||
hasMore: false,
|
||||
};
|
||||
}
|
||||
|
||||
async findById(id: string) {
|
||||
const db = getDatabase();
|
||||
logger.debug({ id }, "Finding message by ID");
|
||||
|
||||
// TODO: Implement actual Drizzle ORM query
|
||||
return null;
|
||||
}
|
||||
|
||||
async findByChannel(channelId: string, query: MessageQuery) {
|
||||
const db = getDatabase();
|
||||
logger.debug({ channelId, query }, "Finding messages by channel");
|
||||
|
||||
// TODO: Implement actual Drizzle ORM queries
|
||||
return {
|
||||
messages: [],
|
||||
total: 0,
|
||||
hasMore: false,
|
||||
};
|
||||
}
|
||||
|
||||
async create(data: MessageCreate) {
|
||||
const db = getDatabase();
|
||||
logger.debug({ data }, "Creating message");
|
||||
|
||||
// TODO: Implement actual Drizzle ORM insert
|
||||
return {
|
||||
id: "msg_" + Date.now(),
|
||||
...data,
|
||||
createdAt: Date.now(),
|
||||
};
|
||||
}
|
||||
|
||||
async update(id: string, data: MessageUpdate) {
|
||||
const db = getDatabase();
|
||||
logger.debug({ id, data }, "Updating message");
|
||||
|
||||
// TODO: Implement actual Drizzle ORM update
|
||||
return null;
|
||||
}
|
||||
|
||||
async delete(id: string) {
|
||||
const db = getDatabase();
|
||||
logger.debug({ id }, "Deleting message");
|
||||
|
||||
// TODO: Implement actual Drizzle ORM delete
|
||||
return true;
|
||||
}
|
||||
|
||||
async getAttachmentsByChannel(channelId: string, query: MessageQuery) {
|
||||
const db = getDatabase();
|
||||
logger.debug({ channelId, query }, "Getting attachments by channel");
|
||||
|
||||
// TODO: Implement actual Drizzle ORM queries
|
||||
return {
|
||||
attachments: [],
|
||||
total: 0,
|
||||
hasMore: false,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export const messagesRepository = new MessagesRepository();
|
||||
@@ -0,0 +1,35 @@
|
||||
import { z } from "zod";
|
||||
|
||||
export const messageQuerySchema = z.object({
|
||||
channelId: z.string().optional(),
|
||||
guildId: z.string().optional(),
|
||||
userId: z.string().optional(),
|
||||
status: z.enum(["pending", "clean", "warn", "flagged", "error"]).optional(),
|
||||
limit: z.coerce.number().int().positive().default(50),
|
||||
offset: z.coerce.number().int().nonnegative().default(0),
|
||||
cursor: z.string().optional(),
|
||||
});
|
||||
|
||||
export const messageCreateSchema = z.object({
|
||||
guildId: z.string(),
|
||||
channelId: z.string(),
|
||||
threadId: z.string().optional(),
|
||||
userId: z.string(),
|
||||
username: z.string(),
|
||||
avatarUrl: z.string().optional(),
|
||||
content: z.string(),
|
||||
type: z.enum(["text", "edited", "deleted"]).default("text"),
|
||||
});
|
||||
|
||||
export const messageUpdateSchema = z.object({
|
||||
editedContent: z.string().optional(),
|
||||
aiStatus: z.enum(["pending", "clean", "warn", "flagged", "error"]).optional(),
|
||||
aiAnalysis: z.string().optional(),
|
||||
aiCategories: z.string().optional(),
|
||||
aiSeverity: z.enum(["none", "low", "medium", "high", "critical"]).optional(),
|
||||
aiConfidence: z.number().optional(),
|
||||
});
|
||||
|
||||
export type MessageQuery = z.infer<typeof messageQuerySchema>;
|
||||
export type MessageCreate = z.infer<typeof messageCreateSchema>;
|
||||
export type MessageUpdate = z.infer<typeof messageUpdateSchema>;
|
||||
@@ -0,0 +1,50 @@
|
||||
import { NotFoundError, ValidationError } from "../../shared/errors/index.js";
|
||||
import { createChildLogger } from "../../shared/logger/index.js";
|
||||
import { messagesRepository } from "./messages.repository.js";
|
||||
import type { MessageQuery } from "./messages.schema.js";
|
||||
|
||||
const logger = createChildLogger("messages.service");
|
||||
|
||||
export class MessagesService {
|
||||
async listMessages(query: MessageQuery) {
|
||||
if (!query.channelId && !query.guildId) {
|
||||
throw new ValidationError("Either channelId or guildId is required");
|
||||
}
|
||||
|
||||
logger.debug({ query }, "Listing messages");
|
||||
return messagesRepository.findMany(query);
|
||||
}
|
||||
|
||||
async getMessagesByChannel(channelId: string, query: MessageQuery) {
|
||||
if (!channelId) {
|
||||
throw new ValidationError("channelId is required");
|
||||
}
|
||||
|
||||
logger.debug({ channelId, query }, "Getting messages by channel");
|
||||
return messagesRepository.findByChannel(channelId, query);
|
||||
}
|
||||
|
||||
async getMessageById(id: string) {
|
||||
if (!id) {
|
||||
throw new ValidationError("message ID is required");
|
||||
}
|
||||
|
||||
const message = await messagesRepository.findById(id);
|
||||
if (!message) {
|
||||
throw new NotFoundError(`Message with ID ${id} not found`);
|
||||
}
|
||||
|
||||
return message;
|
||||
}
|
||||
|
||||
async getAttachmentsByChannel(channelId: string, query: MessageQuery) {
|
||||
if (!channelId) {
|
||||
throw new ValidationError("channelId is required");
|
||||
}
|
||||
|
||||
logger.debug({ channelId, query }, "Getting attachments by channel");
|
||||
return messagesRepository.getAttachmentsByChannel(channelId, query);
|
||||
}
|
||||
}
|
||||
|
||||
export const messagesService = new MessagesService();
|
||||
@@ -0,0 +1,26 @@
|
||||
import type { Router } from "express";
|
||||
import express from "express";
|
||||
import {
|
||||
handleGetAttachmentsByChannel,
|
||||
handleGetMessageById,
|
||||
handleGetMessagesByChannel,
|
||||
handleListMessages,
|
||||
} from "../messages.controller.js";
|
||||
|
||||
export function createMessagesRouter(): Router {
|
||||
const router = express.Router();
|
||||
|
||||
// GET /api/messages - List messages
|
||||
router.get("/messages", handleListMessages);
|
||||
|
||||
// GET /api/messages/:channelId - Get messages by channel
|
||||
router.get("/messages/:channelId", handleGetMessagesByChannel);
|
||||
|
||||
// GET /api/messages/:channelId/attachments - Get attachments by channel
|
||||
router.get("/messages/:channelId/attachments", handleGetAttachmentsByChannel);
|
||||
|
||||
// GET /api/messages/:id - Get single message by ID
|
||||
router.get("/messages/:id", handleGetMessageById);
|
||||
|
||||
return router;
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
import type { Router } from "express";
|
||||
import express from "express";
|
||||
|
||||
export function createVoiceRouter(): Router {
|
||||
const router = express.Router();
|
||||
|
||||
// TODO: Implement voice routes
|
||||
// GET /api/voice/recordings
|
||||
// GET /api/voice/recordings/:userId
|
||||
// POST /api/voice/connect
|
||||
// POST /api/voice/disconnect
|
||||
|
||||
return router;
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
import { createChildLogger } from "../../shared/logger/index.js";
|
||||
|
||||
const logger = createChildLogger("voice.service");
|
||||
|
||||
export class VoiceService {
|
||||
// TODO: Implement voice service methods
|
||||
}
|
||||
|
||||
export const voiceService = new VoiceService();
|
||||
@@ -0,0 +1,92 @@
|
||||
import "dotenv/config";
|
||||
import { z } from "zod";
|
||||
|
||||
const configSchema = z
|
||||
.object({
|
||||
// Server
|
||||
WEBSERVER_PORT: z.coerce.number().positive().default(3001),
|
||||
NODE_ENV: z
|
||||
.enum(["development", "production", "test"])
|
||||
.default("development"),
|
||||
LOG_LEVEL: z
|
||||
.enum(["error", "warn", "info", "http", "verbose", "debug", "silly"])
|
||||
.default("info"),
|
||||
VERBOSE: z
|
||||
.string()
|
||||
.optional()
|
||||
.transform((v) => v === "true")
|
||||
.default(false),
|
||||
|
||||
// Database
|
||||
DATABASE_URL: z.string().url().optional(),
|
||||
DATABASE_HOST: z.string().default("localhost"),
|
||||
DATABASE_PORT: z.coerce.number().default(5432),
|
||||
DATABASE_NAME: z.string().default("discord_moderation"),
|
||||
DATABASE_USER: z.string().default("postgres"),
|
||||
DATABASE_PASSWORD: z.string().optional(),
|
||||
|
||||
// Redis (optional, for pub/sub)
|
||||
REDIS_URL: z.string().url().optional(),
|
||||
REDIS_HOST: z.string().default("localhost"),
|
||||
REDIS_PORT: z.coerce.number().default(6379),
|
||||
|
||||
// Discord
|
||||
MONITOR_GUILD_ID: z.string().min(1).optional(),
|
||||
|
||||
// Admin
|
||||
ADMIN_PASSWORD: z.string().optional(),
|
||||
|
||||
// Analytics
|
||||
BACKLOG_SYNC_HOURS: z.coerce.number().positive().default(24),
|
||||
BACKLOG_SYNC_BATCH_SIZE: z.coerce
|
||||
.number()
|
||||
.int()
|
||||
.positive()
|
||||
.max(100)
|
||||
.default(100),
|
||||
|
||||
// AI Moderation
|
||||
AI_ANALYSIS_ENABLED: z
|
||||
.string()
|
||||
.optional()
|
||||
.transform((v) => v === "true")
|
||||
.default(false),
|
||||
OPENAI_MODERATION_API_KEY: z.string().optional(),
|
||||
OPENAI_MODERATION_BASE_URL: z
|
||||
.string()
|
||||
.url()
|
||||
.default("https://api.openai.com/v1"),
|
||||
OPENAI_MODERATION_MODEL: z.string().default("omni-moderation-latest"),
|
||||
AI_LLM_API_KEY: z.string().optional(),
|
||||
AI_LLM_BASE_URL: z
|
||||
.string()
|
||||
.url()
|
||||
.default("https://9router.asepharyana.my.id/v1"),
|
||||
AI_LLM_MODEL: z.string().default("text"),
|
||||
AI_LLM_VISION_MODEL: z.string().optional(),
|
||||
AI_LLM_MAX_CONCURRENT: z.coerce.number().int().positive().default(5),
|
||||
AI_LLM_IMAGE_MAX_DIMENSION: z.coerce
|
||||
.number()
|
||||
.int()
|
||||
.positive()
|
||||
.default(1024),
|
||||
AI_LLM_TEXT_BATCH_SIZE: z.coerce.number().int().positive().default(20),
|
||||
AI_LLM_MEDIA_ANALYSIS_TIMEOUT_MS: z.coerce
|
||||
.number()
|
||||
.int()
|
||||
.positive()
|
||||
.default(60000),
|
||||
|
||||
// Attachments
|
||||
ATTACHMENT_UPLOAD_TIMEOUT_MS: z.coerce.number().positive().default(30000),
|
||||
ATTACHMENT_MAX_SIZE_MB: z.coerce.number().positive().default(100),
|
||||
ATTACHMENT_RETRY_ATTEMPTS: z.coerce.number().positive().default(3),
|
||||
TELE_UPLOAD_URL: z
|
||||
.string()
|
||||
.url()
|
||||
.default("https://upload.asepharyana.tech/api/upload"),
|
||||
})
|
||||
.parse(process.env);
|
||||
|
||||
export const config = configSchema;
|
||||
export type Config = typeof config;
|
||||
@@ -0,0 +1,58 @@
|
||||
import { drizzle } from "drizzle-orm/node-postgres";
|
||||
import { Pool } from "pg";
|
||||
import { config } from "../config/index.js";
|
||||
import { createChildLogger } from "../logger/index.js";
|
||||
|
||||
const logger = createChildLogger("database");
|
||||
|
||||
let pool: Pool | null = null;
|
||||
let db: ReturnType<typeof drizzle> | null = null;
|
||||
|
||||
export async function initializeDatabase() {
|
||||
if (db) {
|
||||
logger.warn("Database already initialized");
|
||||
return db;
|
||||
}
|
||||
|
||||
const databaseUrl =
|
||||
config.DATABASE_URL ||
|
||||
`postgresql://${config.DATABASE_USER}${config.DATABASE_PASSWORD ? `:${config.DATABASE_PASSWORD}` : ""}@${config.DATABASE_HOST}:${config.DATABASE_PORT}/${config.DATABASE_NAME}`;
|
||||
|
||||
pool = new Pool({
|
||||
connectionString: databaseUrl,
|
||||
});
|
||||
|
||||
pool.on("error", (err) => {
|
||||
logger.error({ err }, "Unexpected error on idle client");
|
||||
});
|
||||
|
||||
try {
|
||||
const client = await pool.connect();
|
||||
client.release();
|
||||
logger.info("Database connection successful");
|
||||
} catch (err) {
|
||||
logger.error({ err }, "Failed to connect to database");
|
||||
throw err;
|
||||
}
|
||||
|
||||
db = drizzle(pool);
|
||||
return db;
|
||||
}
|
||||
|
||||
export function getDatabase() {
|
||||
if (!db) {
|
||||
throw new Error(
|
||||
"Database not initialized. Call initializeDatabase() first.",
|
||||
);
|
||||
}
|
||||
return db;
|
||||
}
|
||||
|
||||
export async function closeDatabase() {
|
||||
if (pool) {
|
||||
await pool.end();
|
||||
pool = null;
|
||||
db = null;
|
||||
logger.info("Database connection closed");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
export class AppError extends Error {
|
||||
constructor(
|
||||
message: string,
|
||||
public code: string,
|
||||
public statusCode: number = 500,
|
||||
) {
|
||||
super(message);
|
||||
this.name = "AppError";
|
||||
}
|
||||
}
|
||||
|
||||
export class ValidationError extends AppError {
|
||||
constructor(
|
||||
message: string,
|
||||
public details?: Record<string, unknown>,
|
||||
) {
|
||||
super(message, "VALIDATION_ERROR", 400);
|
||||
this.name = "ValidationError";
|
||||
}
|
||||
}
|
||||
|
||||
export class NotFoundError extends AppError {
|
||||
constructor(message: string) {
|
||||
super(message, "NOT_FOUND", 404);
|
||||
this.name = "NotFoundError";
|
||||
}
|
||||
}
|
||||
|
||||
export class UnauthorizedError extends AppError {
|
||||
constructor(message: string = "Unauthorized") {
|
||||
super(message, "UNAUTHORIZED", 401);
|
||||
this.name = "UnauthorizedError";
|
||||
}
|
||||
}
|
||||
|
||||
export class ForbiddenError extends AppError {
|
||||
constructor(message: string = "Forbidden") {
|
||||
super(message, "FORBIDDEN", 403);
|
||||
this.name = "ForbiddenError";
|
||||
}
|
||||
}
|
||||
|
||||
export class ConflictError extends AppError {
|
||||
constructor(message: string) {
|
||||
super(message, "CONFLICT", 409);
|
||||
this.name = "ConflictError";
|
||||
}
|
||||
}
|
||||
|
||||
export class DatabaseError extends AppError {
|
||||
constructor(
|
||||
message: string,
|
||||
public originalError?: Error,
|
||||
) {
|
||||
super(message, "DATABASE_ERROR", 500);
|
||||
this.name = "DatabaseError";
|
||||
}
|
||||
}
|
||||
|
||||
export class ConfigError extends AppError {
|
||||
constructor(message: string) {
|
||||
super(message, "CONFIG_ERROR", 500);
|
||||
this.name = "ConfigError";
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
import pino from "pino";
|
||||
import { config } from "../config/index.js";
|
||||
|
||||
const isDev = config.NODE_ENV === "development";
|
||||
|
||||
export const logger = pino({
|
||||
level: config.LOG_LEVEL,
|
||||
transport: isDev
|
||||
? {
|
||||
target: "pino-pretty",
|
||||
options: {
|
||||
colorize: true,
|
||||
translateTime: "SYS:standard",
|
||||
ignore: "pid,hostname",
|
||||
},
|
||||
}
|
||||
: undefined,
|
||||
});
|
||||
|
||||
export function createChildLogger(context: string) {
|
||||
return logger.child({ context });
|
||||
}
|
||||
|
||||
export type Logger = ReturnType<typeof createChildLogger>;
|
||||
@@ -0,0 +1,50 @@
|
||||
import type { NextFunction, Request, Response } from "express";
|
||||
import { AppError, UnauthorizedError } from "../errors/index.js";
|
||||
import { createChildLogger } from "../logger/index.js";
|
||||
|
||||
const logger = createChildLogger("middleware");
|
||||
|
||||
export function errorHandler(
|
||||
err: Error,
|
||||
_req: Request,
|
||||
res: Response,
|
||||
_next: NextFunction,
|
||||
) {
|
||||
if (err instanceof AppError) {
|
||||
logger.warn({ code: err.code, statusCode: err.statusCode }, err.message);
|
||||
return res.status(err.statusCode).json({
|
||||
error: err.code,
|
||||
message: err.message,
|
||||
...(err instanceof ValidationError && { details: err.details }),
|
||||
});
|
||||
}
|
||||
|
||||
logger.error({ err }, "Unhandled error");
|
||||
res.status(500).json({
|
||||
error: "INTERNAL_SERVER_ERROR",
|
||||
message: "An unexpected error occurred",
|
||||
});
|
||||
}
|
||||
|
||||
export function adminAuth(adminPassword: string) {
|
||||
return (req: Request, res: Response, next: NextFunction) => {
|
||||
const password = req.headers["x-admin-password"] as string;
|
||||
|
||||
if (!password || password !== adminPassword) {
|
||||
throw new UnauthorizedError("Invalid admin password");
|
||||
}
|
||||
|
||||
next();
|
||||
};
|
||||
}
|
||||
|
||||
export function asyncHandler(
|
||||
fn: (req: Request, res: Response, next: NextFunction) => Promise<void>,
|
||||
) {
|
||||
return (req: Request, res: Response, next: NextFunction) => {
|
||||
Promise.resolve(fn(req, res, next)).catch(next);
|
||||
};
|
||||
}
|
||||
|
||||
// Import ValidationError for type checking
|
||||
import { ValidationError } from "../errors/index.js";
|
||||
@@ -0,0 +1,25 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2020",
|
||||
"module": "ES2020",
|
||||
"lib": ["ES2020"],
|
||||
"moduleResolution": "node",
|
||||
"esModuleInterop": true,
|
||||
"allowSyntheticDefaultImports": true,
|
||||
"strict": true,
|
||||
"skipLibCheck": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"resolveJsonModule": true,
|
||||
"declaration": true,
|
||||
"declarationMap": true,
|
||||
"sourceMap": true,
|
||||
"outDir": "./dist",
|
||||
"rootDir": "./src",
|
||||
"baseUrl": ".",
|
||||
"paths": {
|
||||
"@/*": ["./src/*"]
|
||||
}
|
||||
},
|
||||
"include": ["src/**/*"],
|
||||
"exclude": ["node_modules", "dist", "**/*.test.ts"]
|
||||
}
|
||||
@@ -0,0 +1,172 @@
|
||||
services/discord-gateway/
|
||||
├── src/
|
||||
│ ├── app/
|
||||
│ │ ├── bootstrap.ts # Discord Gateway initialization (no HTTP server)
|
||||
│ │ └── shutdown.ts # Graceful shutdown handler
|
||||
│ ├── shared/
|
||||
│ │ ├── config/
|
||||
│ │ │ └── config.ts # Environment configuration (Zod validated)
|
||||
│ │ ├── database/
|
||||
│ │ │ ├── schema.ts # Drizzle ORM schema
|
||||
│ │ │ ├── drizzle.ts # Database connection
|
||||
│ │ │ ├── migrate.ts # Migration runner
|
||||
│ │ │ └── voiceRecordingRepo.ts
|
||||
│ │ ├── errors/
|
||||
│ │ │ └── errors.ts # Custom error classes
|
||||
│ │ ├── logger/
|
||||
│ │ │ ├── logger.ts # Winston logger wrapper
|
||||
│ │ │ └── serialization.ts # Log value serialization
|
||||
│ │ ├── utils/
|
||||
│ │ │ └── retry.ts # Retry with backoff utility
|
||||
│ │ └── discord/
|
||||
│ │ └── clientOptions.ts # Discord.js client configuration
|
||||
│ ├── modules/
|
||||
│ │ ├── message-capture/ # Modular MVC: Message capture & storage
|
||||
│ │ │ ├── messageCapture.ts # Controller: Discord event listeners
|
||||
│ │ │ ├── messageStore.ts # Repository: Database operations
|
||||
│ │ │ ├── messageMetadata.ts # Service: Message metadata extraction
|
||||
│ │ │ ├── types.ts # Domain types
|
||||
│ │ │ └── index.ts # Module exports
|
||||
│ │ ├── ai-moderation/ # Modular MVC: AI analysis & moderation
|
||||
│ │ │ ├── aiAnalyzer.ts # Controller: Analysis orchestration
|
||||
│ │ │ ├── llmModerationClient.ts # Service: LLM API client
|
||||
│ │ │ ├── aiAnalysisWorker.ts # Service: Worker pool management
|
||||
│ │ │ ├── indonesianTextNormalizer.ts # Service: Text normalization
|
||||
│ │ │ ├── moderationPrompt.ts # Service: Prompt generation
|
||||
│ │ │ └── index.ts # Module exports
|
||||
│ │ ├── voice-recording/ # Modular MVC: Voice recording & streaming
|
||||
│ │ │ ├── voiceController.ts # Controller: Voice connection management
|
||||
│ │ │ ├── recorder.ts # Service: Recording orchestration
|
||||
│ │ │ ├── recorder/
|
||||
│ │ │ │ ├── audioStream.ts # Service: Audio stream subscription
|
||||
│ │ │ │ ├── decoder.ts # Service: Opus decoding
|
||||
│ │ │ │ ├── segment.ts # Service: OGG segment rotation
|
||||
│ │ │ │ ├── metadata.ts # Service: Segment metadata
|
||||
│ │ │ │ ├── sessionRecording.ts # Service: Session management
|
||||
│ │ │ │ └── uploader.ts # Service: Segment upload
|
||||
│ │ │ └── index.ts # Module exports
|
||||
│ │ ├── attachment-upload/ # Modular MVC: Attachment handling
|
||||
│ │ │ ├── attachmentUploader.ts # Service: Upload orchestration
|
||||
│ │ │ ├── imageResizer.ts # Service: Image resizing
|
||||
│ │ │ └── index.ts # Module exports
|
||||
│ │ └── event-broadcaster/ # Event-driven: Redis pub/sub
|
||||
│ │ ├── eventBroadcaster.ts # Service: Event publishing
|
||||
│ │ ├── eventTypes.ts # Domain: Event type definitions
|
||||
│ │ └── index.ts # Module exports
|
||||
│ ├── mock-crc.ts # CRC polyfill for discord.js
|
||||
│ └── index.ts # Service entry point
|
||||
├── package.json # Service dependencies
|
||||
└── tsconfig.json # TypeScript configuration
|
||||
|
||||
## Architecture Patterns
|
||||
|
||||
### Modular MVC Structure
|
||||
Each module follows Controller-Service-Repository pattern:
|
||||
- **Controller**: Discord event listeners (messageCapture, aiAnalyzer, voiceController)
|
||||
- **Service**: Business logic (messageStore, llmModerationClient, recorder)
|
||||
- **Repository**: Data access (messageStore, voiceRecordingRepo)
|
||||
|
||||
### Event-Driven Design
|
||||
- **Redis Pub/Sub**: All events published to Redis channels
|
||||
- **Event Channels**:
|
||||
- `discord:message:created` — New message captured
|
||||
- `discord:message:updated` — Message edited
|
||||
- `discord:message:deleted` — Message deleted
|
||||
- `discord:message:analyzed` — AI analysis complete
|
||||
- `discord:attachment:created` — Attachment detected
|
||||
- `discord:attachment:uploaded` — Attachment uploaded to storage
|
||||
- `discord:voice:started` — Voice recording started
|
||||
- `discord:voice:stopped` — Voice recording stopped
|
||||
- `discord:voice:uploaded` — Voice segment uploaded
|
||||
- `discord:analysis:queue_status` — Analysis queue status update
|
||||
|
||||
### Shared Infrastructure
|
||||
- **Config**: Zod-validated environment variables
|
||||
- **Logger**: Winston logger with context support
|
||||
- **Database**: Drizzle ORM with PostgreSQL
|
||||
- **Errors**: Custom error classes with codes and status codes
|
||||
- **Utils**: Retry logic with exponential backoff
|
||||
|
||||
### No HTTP Server
|
||||
- Discord Gateway service is **event-driven only**
|
||||
- No Express, WebSocket, or HTTP routes
|
||||
- All communication via Redis pub/sub
|
||||
- Backend service consumes events and serves HTTP API
|
||||
|
||||
## Initialization Flow
|
||||
|
||||
1. Load environment config (Zod validation)
|
||||
2. Initialize database connection
|
||||
3. Run pending migrations
|
||||
4. Create Discord client with optimized cache settings
|
||||
5. Initialize Redis event broadcaster
|
||||
6. Register Discord event listeners (messageCapture, aiAnalyzer)
|
||||
7. Login to Discord
|
||||
8. Listen for graceful shutdown signals (SIGINT, SIGTERM)
|
||||
|
||||
## Graceful Shutdown
|
||||
|
||||
On shutdown signal:
|
||||
1. Close database connection
|
||||
2. Disconnect from voice channels
|
||||
3. Close Redis connection
|
||||
4. Destroy Discord client
|
||||
5. Exit process
|
||||
|
||||
## Dependencies
|
||||
|
||||
**Core Discord**:
|
||||
- discord.js-selfbot-v13
|
||||
- @discordjs/voice
|
||||
- @discordjs/opus
|
||||
|
||||
**Audio Processing**:
|
||||
- prism-media (Opus encoding/decoding)
|
||||
- opusscript (Opus fallback)
|
||||
- sharp (Image resizing)
|
||||
|
||||
**Data & Config**:
|
||||
- drizzle-orm (ORM)
|
||||
- pg (PostgreSQL driver)
|
||||
- zod (Config validation)
|
||||
- ioredis (Redis client)
|
||||
|
||||
**Logging & Utilities**:
|
||||
- winston (Structured logging)
|
||||
- p-retry (Retry logic)
|
||||
- p-limit (Concurrency limiting)
|
||||
- piscina (Worker pool)
|
||||
|
||||
## Event Flow Example
|
||||
|
||||
### Message Capture Flow
|
||||
1. Discord emits `messageCreate` event
|
||||
2. `messageCapture.ts` listener receives event
|
||||
3. Extract metadata (user, channel, content, timestamp)
|
||||
4. `messageStore.ts` inserts into database
|
||||
5. `eventBroadcaster.messageCreated()` publishes to Redis
|
||||
6. Backend service subscribes to `discord:message:created` channel
|
||||
7. Backend processes and stores in its own database
|
||||
|
||||
### Voice Recording Flow
|
||||
1. `voiceController.connect()` joins voice channel
|
||||
2. `recorder.ts` subscribes to user audio streams
|
||||
3. For each speaking user:
|
||||
- Create audio stream subscription
|
||||
- Decode Opus packets to PCM
|
||||
- Rotate OGG segments (5s default)
|
||||
- Collect user metadata
|
||||
4. On silence (3s):
|
||||
- Finalize segment
|
||||
- Create metadata JSON
|
||||
- Upload segment to storage
|
||||
- Publish `discord:voice:uploaded` event
|
||||
5. Backend service receives event and indexes recording
|
||||
|
||||
## No Breaking Changes
|
||||
|
||||
- Original `src/` remains untouched for now
|
||||
- Discord Gateway is a **new service** in `services/discord-gateway/`
|
||||
- Can run alongside existing monolith during transition
|
||||
- Backend service will consume Redis events
|
||||
- Frontend continues to use Backend HTTP API
|
||||
@@ -0,0 +1,408 @@
|
||||
# Discord Gateway Service - Module Structure
|
||||
|
||||
## Complete Directory Tree
|
||||
|
||||
```
|
||||
services/discord-gateway/
|
||||
├── src/
|
||||
│ ├── app/
|
||||
│ │ ├── bootstrap.ts
|
||||
│ │ │ └── Initializes Discord client, database, Redis broadcaster
|
||||
│ │ │ Registers event listeners, handles graceful shutdown
|
||||
│ │ └── shutdown.ts
|
||||
│ │ └── Graceful shutdown handler for SIGINT/SIGTERM/exceptions
|
||||
│ │
|
||||
│ ├── shared/
|
||||
│ │ ├── config/
|
||||
│ │ │ └── config.ts
|
||||
│ │ │ └── Zod-validated environment configuration
|
||||
│ │ │ - Discord token, database URL, Redis URL
|
||||
│ │ │ - AI LLM settings, recording parameters
|
||||
│ │ │ - Attachment upload settings, retention policies
|
||||
│ │ │
|
||||
│ │ ├── database/
|
||||
│ │ │ ├── schema.ts
|
||||
│ │ │ │ └── Drizzle ORM schema definitions
|
||||
│ │ │ ├── drizzle.ts
|
||||
│ │ │ │ └── PostgreSQL connection and initialization
|
||||
│ │ │ ├── migrate.ts
|
||||
│ │ │ │ └── Database migration runner
|
||||
│ │ │ ├── migrateCli.ts
|
||||
│ │ │ │ └── CLI for programmatic migrations
|
||||
│ │ │ ├── voiceRecordingRepo.ts
|
||||
│ │ │ │ └── Voice recording repository
|
||||
│ │ │ └── migrations/
|
||||
│ │ │ └── Database migration files
|
||||
│ │ │
|
||||
│ │ ├── errors/
|
||||
│ │ │ └── errors.ts
|
||||
│ │ │ └── Custom error classes
|
||||
│ │ │ - AppError (base)
|
||||
│ │ │ - ConfigError
|
||||
│ │ │ - AudioError
|
||||
│ │ │ - VoiceConnectionError
|
||||
│ │ │ - ValidationError
|
||||
│ │ │
|
||||
│ │ ├── logger/
|
||||
│ │ │ ├── logger.ts
|
||||
│ │ │ │ └── Winston logger wrapper with context support
|
||||
│ │ │ └── serialization.ts
|
||||
│ │ │ └── Log value serialization utilities
|
||||
│ │ │
|
||||
│ │ ├── utils/
|
||||
│ │ │ └── retry.ts
|
||||
│ │ │ └── Retry with exponential backoff utility
|
||||
│ │ │
|
||||
│ │ └── discord/
|
||||
│ │ └── clientOptions.ts
|
||||
│ │ └── Discord.js client configuration
|
||||
│ │
|
||||
│ ├── modules/
|
||||
│ │ │
|
||||
│ │ ├── message-capture/
|
||||
│ │ │ ├── messageCapture.ts
|
||||
│ │ │ │ └── CONTROLLER: Discord event listeners
|
||||
│ │ │ │ - messageCreate, messageUpdate, messageDelete
|
||||
│ │ │ │ - Validates capture target, publishes events
|
||||
│ │ │ │
|
||||
│ │ │ ├── messageStore.ts
|
||||
│ │ │ │ └── REPOSITORY: Database CRUD operations
|
||||
│ │ │ │ - upsertMessageForCapture
|
||||
│ │ │ │ - updateMessageAsEdited
|
||||
│ │ │ │ - updateMessageAsDeleted
|
||||
│ │ │ │ - insertAttachment
|
||||
│ │ │ │ - getMessageById
|
||||
│ │ │ │
|
||||
│ │ │ ├── messageMetadata.ts
|
||||
│ │ │ │ └── SERVICE: Message metadata extraction
|
||||
│ │ │ │ - getMessageMetadata
|
||||
│ │ │ │ - getMessageLocation
|
||||
│ │ │ │ - getDisplayContent
|
||||
│ │ │ │
|
||||
│ │ │ ├── types.ts
|
||||
│ │ │ │ └── Domain types
|
||||
│ │ │ │ - MessageRecord
|
||||
│ │ │ │ - AttachmentRecord
|
||||
│ │ │ │ - VoiceSegmentRecord
|
||||
│ │ │ │ - AIStatus, AISeverity, AIRecommendedAction
|
||||
│ │ │ │
|
||||
│ │ │ └── index.ts
|
||||
│ │ └── Module exports
|
||||
│ │
|
||||
│ │ ├── ai-moderation/
|
||||
│ │ │ ├── aiAnalyzer.ts
|
||||
│ │ │ │ └── CONTROLLER: Analysis orchestration
|
||||
│ │ │ │ - startPendingAIAnalysisWorker
|
||||
│ │ │ │ - queueMessageAnalysis
|
||||
│ │ │ │ - Manages analysis queue and worker pool
|
||||
│ │ │ │
|
||||
│ │ │ ├── llmModerationClient.ts
|
||||
│ │ │ │ └── SERVICE: LLM API integration
|
||||
│ │ │ │ - Calls LLM for text/image moderation
|
||||
│ │ │ │ - Parses responses, handles errors
|
||||
│ │ │ │ - Retry logic with backoff
|
||||
│ │ │ │
|
||||
│ │ │ ├── aiAnalysisWorker.ts
|
||||
│ │ │ │ └── SERVICE: Worker pool management
|
||||
│ │ │ │ - Piscina worker pool for parallel analysis
|
||||
│ │ │ │ - Conversation context batching
|
||||
│ │ │ │
|
||||
│ │ │ ├── indonesianTextNormalizer.ts
|
||||
│ │ │ │ └── SERVICE: Text preprocessing
|
||||
│ │ │ │ - Normalize Indonesian text
|
||||
│ │ │ │ - Handle diacritics, abbreviations
|
||||
│ │ │ │
|
||||
│ │ │ ├── moderationPrompt.ts
|
||||
│ │ │ │ └── SERVICE: Prompt generation
|
||||
│ │ │ │ - Generate LLM prompts for moderation
|
||||
│ │ │ │ - Include context and policy
|
||||
│ │ │ │
|
||||
│ │ │ └── index.ts
|
||||
│ │ └── Module exports
|
||||
│ │
|
||||
│ │ ├── voice-recording/
|
||||
│ │ │ ├── voiceController.ts
|
||||
│ │ │ │ └── CONTROLLER: Voice connection management
|
||||
│ │ │ │ - connect(guildId, channelId)
|
||||
│ │ │ │ - disconnect()
|
||||
│ │ │ │ - listGuilds(), listVoiceChannels()
|
||||
│ │ │ │ - getStatus()
|
||||
│ │ │ │
|
||||
│ │ │ ├── recorder.ts
|
||||
│ │ │ │ └── SERVICE: Recording orchestration
|
||||
│ │ │ │ - startRecording(client, channel)
|
||||
│ │ │ │ - stopRecording(guildId)
|
||||
│ │ │ │ - Manages active recording sessions
|
||||
│ │ │ │
|
||||
│ │ │ ├── recorder/
|
||||
│ │ │ │ ├── audioStream.ts
|
||||
│ │ │ │ │ └── SERVICE: Audio stream subscription
|
||||
│ │ │ │ │ - subscribeToAudioStream
|
||||
│ │ │ │ │ - Opus packet handling
|
||||
│ │ │ │ │
|
||||
│ │ │ │ ├── decoder.ts
|
||||
│ │ │ │ │ └── SERVICE: Opus decoding
|
||||
│ │ │ │ │ - OpusDecoder class
|
||||
│ │ │ │ │ - Decode Opus to PCM
|
||||
│ │ │ │ │ - Rotation and cooldown logic
|
||||
│ │ │ │ │
|
||||
│ │ │ │ ├── segment.ts
|
||||
│ │ │ │ │ └── SERVICE: OGG segment rotation
|
||||
│ │ │ │ │ - SegmentManager class
|
||||
│ │ │ │ │ - Rotate segments (5s default)
|
||||
│ │ │ │ │ - Write OGG files
|
||||
│ │ │ │ │
|
||||
│ │ │ │ ├── metadata.ts
|
||||
│ │ │ │ │ └── SERVICE: Segment metadata
|
||||
│ │ │ │ │ - collectUserMetadata
|
||||
│ │ │ │ │ - createSegmentMetadata
|
||||
│ │ │ │ │ - User info, roles, timestamps
|
||||
│ │ │ │ │
|
||||
│ │ │ │ ├── sessionRecording.ts
|
||||
│ │ │ │ │ └── SERVICE: Session management
|
||||
│ │ │ │ │ - createRecordingSession
|
||||
│ │ │ │ │ - finalizeRecordingSession
|
||||
│ │ │ │ │ - Track active sessions
|
||||
│ │ │ │ │
|
||||
│ │ │ │ └── uploader.ts
|
||||
│ │ │ │ └── SERVICE: Segment upload
|
||||
│ │ │ │ - uploadRecordingSegment
|
||||
│ │ │ │ - Upload to external storage
|
||||
│ │ │ │ - Retry logic
|
||||
│ │ │ │
|
||||
│ │ │ └── index.ts
|
||||
│ │ └── Module exports
|
||||
│ │
|
||||
│ │ ├── attachment-upload/
|
||||
│ │ │ ├── attachmentUploader.ts
|
||||
│ │ │ │ └── SERVICE: Upload orchestration
|
||||
│ │ │ │ - processAttachmentUpload
|
||||
│ │ │ │ - Download from Discord
|
||||
│ │ │ │ - Upload to external storage
|
||||
│ │ │ │ - Retry with backoff
|
||||
│ │ │ │
|
||||
│ │ │ ├── imageResizer.ts
|
||||
│ │ │ │ └── SERVICE: Image processing
|
||||
│ │ │ │ - resizeImage
|
||||
│ │ │ │ - Resize to max dimension
|
||||
│ │ │ │ - Preserve aspect ratio
|
||||
│ │ │ │
|
||||
│ │ │ └── index.ts
|
||||
│ │ └── Module exports
|
||||
│ │
|
||||
│ │ └── event-broadcaster/
|
||||
│ │ ├── eventBroadcaster.ts
|
||||
│ │ │ └── SERVICE: Redis pub/sub publisher
|
||||
│ │ │ - EventBroadcaster class
|
||||
│ │ │ - RedisEventPublisher class
|
||||
│ │ │ - Publish to Redis channels
|
||||
│ │ │ - Methods:
|
||||
│ │ │ - messageCreated()
|
||||
│ │ │ - messageUpdated()
|
||||
│ │ │ - messageDeleted()
|
||||
│ │ │ - messageAnalyzed()
|
||||
│ │ │ - attachmentCreated()
|
||||
│ │ │ - attachmentUploaded()
|
||||
│ │ │ - voiceRecordingStarted()
|
||||
│ │ │ - voiceRecordingStopped()
|
||||
│ │ │ - voiceRecordingUploaded()
|
||||
│ │ │ - analysisQueueStatus()
|
||||
│ │ │
|
||||
│ │ ├── eventTypes.ts
|
||||
│ │ │ └── Domain types
|
||||
│ │ │ - DiscordGatewayEvent interface
|
||||
│ │ │ - EventChannels constants
|
||||
│ │ │ - Event channel names
|
||||
│ │ │
|
||||
│ │ └── index.ts
|
||||
│ └── Module exports
|
||||
│
|
||||
│ ├── mock-crc.ts
|
||||
│ │ └── CRC polyfill for discord.js compatibility
|
||||
│ │
|
||||
│ └── index.ts
|
||||
│ └── Service entry point
|
||||
│ - Initialize Discord Gateway
|
||||
│ - Handle startup errors
|
||||
│
|
||||
├── ARCHITECTURE.md
|
||||
│ └── Detailed architecture documentation
|
||||
│
|
||||
├── README.md
|
||||
│ └── Complete service documentation
|
||||
│
|
||||
├── MODULE_STRUCTURE.md
|
||||
│ └── This file - module structure reference
|
||||
│
|
||||
└── package.json
|
||||
└── Service dependencies and scripts
|
||||
```
|
||||
|
||||
## Module Responsibilities
|
||||
|
||||
### message-capture
|
||||
**Purpose**: Capture Discord messages (create, update, delete)
|
||||
**Pattern**: Controller-Service-Repository
|
||||
- **Controller** (messageCapture.ts): Listens to Discord events
|
||||
- **Service** (messageMetadata.ts): Extracts metadata
|
||||
- **Repository** (messageStore.ts): Database operations
|
||||
- **Events Published**:
|
||||
- `discord:message:created`
|
||||
- `discord:message:updated`
|
||||
- `discord:message:deleted`
|
||||
|
||||
### ai-moderation
|
||||
**Purpose**: Analyze messages with LLM for moderation
|
||||
**Pattern**: Controller-Service-Service-Service
|
||||
- **Controller** (aiAnalyzer.ts): Orchestrates analysis workflow
|
||||
- **Service** (llmModerationClient.ts): LLM API integration
|
||||
- **Service** (aiAnalysisWorker.ts): Worker pool management
|
||||
- **Service** (indonesianTextNormalizer.ts): Text preprocessing
|
||||
- **Service** (moderationPrompt.ts): Prompt generation
|
||||
- **Events Published**:
|
||||
- `discord:message:analyzed`
|
||||
- `discord:analysis:queue_status`
|
||||
|
||||
### voice-recording
|
||||
**Purpose**: Record voice channel audio
|
||||
**Pattern**: Controller-Service-SubServices
|
||||
- **Controller** (voiceController.ts): Voice connection management
|
||||
- **Service** (recorder.ts): Recording orchestration
|
||||
- **Sub-services** (recorder/*): Audio processing pipeline
|
||||
- audioStream.ts: Opus packet subscription
|
||||
- decoder.ts: Opus to PCM decoding
|
||||
- segment.ts: OGG file rotation
|
||||
- metadata.ts: User metadata collection
|
||||
- sessionRecording.ts: Session lifecycle
|
||||
- uploader.ts: Segment upload
|
||||
- **Events Published**:
|
||||
- `discord:voice:started`
|
||||
- `discord:voice:stopped`
|
||||
- `discord:voice:uploaded`
|
||||
|
||||
### attachment-upload
|
||||
**Purpose**: Upload message attachments to external storage
|
||||
**Pattern**: Service-Service
|
||||
- **Service** (attachmentUploader.ts): Upload orchestration
|
||||
- **Service** (imageResizer.ts): Image processing
|
||||
- **Events Published**:
|
||||
- `discord:attachment:created`
|
||||
- `discord:attachment:uploaded`
|
||||
|
||||
### event-broadcaster
|
||||
**Purpose**: Publish events to Redis pub/sub
|
||||
**Pattern**: Service-Domain
|
||||
- **Service** (eventBroadcaster.ts): Redis publisher
|
||||
- **Domain** (eventTypes.ts): Event type definitions
|
||||
- **Channels**:
|
||||
- discord:message:* (message events)
|
||||
- discord:attachment:* (attachment events)
|
||||
- discord:voice:* (voice events)
|
||||
- discord:analysis:* (analysis events)
|
||||
|
||||
## Shared Infrastructure
|
||||
|
||||
### config
|
||||
- Zod-validated environment variables
|
||||
- Type-safe configuration access
|
||||
- Sensible defaults
|
||||
|
||||
### database
|
||||
- Drizzle ORM schema
|
||||
- PostgreSQL connection
|
||||
- Migration management
|
||||
- Voice recording repository
|
||||
|
||||
### logger
|
||||
- Winston logger wrapper
|
||||
- Context-aware logging
|
||||
- Log serialization utilities
|
||||
|
||||
### errors
|
||||
- Custom error classes
|
||||
- Error codes and HTTP status codes
|
||||
- Proper error hierarchy
|
||||
|
||||
### utils
|
||||
- Retry with exponential backoff
|
||||
- Configurable retry parameters
|
||||
|
||||
### discord
|
||||
- Discord.js client configuration
|
||||
- Cache optimization
|
||||
- Partial handling
|
||||
|
||||
## Event Flow
|
||||
|
||||
```
|
||||
Discord Events
|
||||
↓
|
||||
message-capture (Controller)
|
||||
↓
|
||||
messageStore (Repository) → PostgreSQL
|
||||
↓
|
||||
eventBroadcaster (Service)
|
||||
↓
|
||||
Redis Pub/Sub
|
||||
↓
|
||||
Backend Service (Subscriber)
|
||||
↓
|
||||
HTTP API / WebSocket
|
||||
↓
|
||||
Frontend Application
|
||||
```
|
||||
|
||||
## No HTTP Server
|
||||
|
||||
- ✅ No Express
|
||||
- ✅ No WebSocket server
|
||||
- ✅ No HTTP routes
|
||||
- ✅ No middleware
|
||||
- ✅ Pure event-driven service
|
||||
|
||||
## Graceful Shutdown
|
||||
|
||||
1. Close PostgreSQL connection
|
||||
2. Disconnect from voice channels
|
||||
3. Close Redis connection
|
||||
4. Destroy Discord client
|
||||
5. Exit process
|
||||
|
||||
## Dependencies
|
||||
|
||||
**Discord**:
|
||||
- discord.js-selfbot-v13
|
||||
- @discordjs/voice
|
||||
- @discordjs/opus
|
||||
|
||||
**Audio**:
|
||||
- prism-media
|
||||
- opusscript
|
||||
- sharp
|
||||
|
||||
**Data**:
|
||||
- drizzle-orm
|
||||
- pg
|
||||
- zod
|
||||
- ioredis
|
||||
|
||||
**Logging**:
|
||||
- winston
|
||||
- p-retry
|
||||
- p-limit
|
||||
- piscina
|
||||
|
||||
## Summary
|
||||
|
||||
The Discord Gateway service is a **pure event-driven microservice** that:
|
||||
- Captures Discord messages, voice, and attachments
|
||||
- Performs AI moderation analysis
|
||||
- Publishes events to Redis pub/sub
|
||||
- Has no HTTP server or WebSocket
|
||||
- Follows Modular MVC pattern
|
||||
- Maintains clean module boundaries
|
||||
- Provides type-safe configuration
|
||||
- Includes structured logging
|
||||
- Handles graceful shutdown
|
||||
|
||||
The service is designed to run alongside the Backend service, which consumes Redis events and serves the HTTP API to the Frontend.
|
||||
@@ -0,0 +1,363 @@
|
||||
# Discord Gateway Service - Extraction Complete
|
||||
|
||||
## Overview
|
||||
|
||||
Successfully extracted Discord Gateway service with **Modular MVC + Event-Driven Architecture** using Redis pub/sub for inter-service communication.
|
||||
|
||||
## Directory Structure
|
||||
|
||||
```
|
||||
services/discord-gateway/
|
||||
├── src/
|
||||
│ ├── app/
|
||||
│ │ ├── bootstrap.ts # Service initialization (Discord client, DB, Redis)
|
||||
│ │ └── shutdown.ts # Graceful shutdown handler
|
||||
│ ├── shared/ # Shared infrastructure layer
|
||||
│ │ ├── config/
|
||||
│ │ │ └── config.ts # Zod-validated environment config
|
||||
│ │ ├── database/
|
||||
│ │ │ ├── schema.ts # Drizzle ORM schema
|
||||
│ │ │ ├── drizzle.ts # PostgreSQL connection
|
||||
│ │ │ ├── migrate.ts # Migration runner
|
||||
│ │ │ └── voiceRecordingRepo.ts
|
||||
│ │ ├── errors/
|
||||
│ │ │ └── errors.ts # Custom error classes
|
||||
│ │ ├── logger/
|
||||
│ │ │ ├── logger.ts # Winston logger wrapper
|
||||
│ │ │ └── serialization.ts # Log serialization
|
||||
│ │ ├── utils/
|
||||
│ │ │ └── retry.ts # Retry with exponential backoff
|
||||
│ │ └── discord/
|
||||
│ │ └── clientOptions.ts # Discord.js client config
|
||||
│ ├── modules/ # Feature modules (Modular MVC)
|
||||
│ │ ├── message-capture/ # Controller-Service-Repository
|
||||
│ │ │ ├── messageCapture.ts # Controller: Discord event listeners
|
||||
│ │ │ ├── messageStore.ts # Repository: DB operations
|
||||
│ │ │ ├── messageMetadata.ts # Service: Metadata extraction
|
||||
│ │ │ ├── types.ts # Domain types
|
||||
│ │ │ └── index.ts # Module exports
|
||||
│ │ ├── ai-moderation/ # Controller-Service-Repository
|
||||
│ │ │ ├── aiAnalyzer.ts # Controller: Analysis orchestration
|
||||
│ │ │ ├── llmModerationClient.ts # Service: LLM API client
|
||||
│ │ │ ├── aiAnalysisWorker.ts # Service: Worker pool
|
||||
│ │ │ ├── indonesianTextNormalizer.ts # Service: Text normalization
|
||||
│ │ │ ├── moderationPrompt.ts # Service: Prompt generation
|
||||
│ │ │ └── index.ts # Module exports
|
||||
│ │ ├── voice-recording/ # Controller-Service-Repository
|
||||
│ │ │ ├── voiceController.ts # Controller: Voice connection mgmt
|
||||
│ │ │ ├── recorder.ts # Service: Recording orchestration
|
||||
│ │ │ ├── recorder/ # Sub-services
|
||||
│ │ │ │ ├── audioStream.ts # Audio stream subscription
|
||||
│ │ │ │ ├── decoder.ts # Opus decoding
|
||||
│ │ │ │ ├── segment.ts # OGG segment rotation
|
||||
│ │ │ │ ├── metadata.ts # Segment metadata
|
||||
│ │ │ │ ├── sessionRecording.ts # Session management
|
||||
│ │ │ │ └── uploader.ts # Segment upload
|
||||
│ │ │ └── index.ts # Module exports
|
||||
│ │ ├── attachment-upload/ # Controller-Service-Repository
|
||||
│ │ │ ├── attachmentUploader.ts # Service: Upload orchestration
|
||||
│ │ │ ├── imageResizer.ts # Service: Image resizing
|
||||
│ │ │ └── index.ts # Module exports
|
||||
│ │ └── event-broadcaster/ # Event-driven layer
|
||||
│ │ ├── eventBroadcaster.ts # Service: Redis pub/sub publisher
|
||||
│ │ ├── eventTypes.ts # Domain: Event type definitions
|
||||
│ │ └── index.ts # Module exports
|
||||
│ ├── mock-crc.ts # CRC polyfill for discord.js
|
||||
│ └── index.ts # Service entry point
|
||||
├── ARCHITECTURE.md # Detailed architecture documentation
|
||||
├── package.json # Service dependencies
|
||||
└── tsconfig.json # TypeScript configuration (inherited)
|
||||
```
|
||||
|
||||
## Architecture Patterns
|
||||
|
||||
### 1. Modular MVC Structure
|
||||
Each feature module follows **Controller-Service-Repository** pattern:
|
||||
|
||||
**Message Capture Module**:
|
||||
- **Controller** (`messageCapture.ts`): Listens to Discord events (messageCreate, messageUpdate, messageDelete)
|
||||
- **Service** (`messageMetadata.ts`): Extracts and normalizes message metadata
|
||||
- **Repository** (`messageStore.ts`): Database CRUD operations
|
||||
|
||||
**AI Moderation Module**:
|
||||
- **Controller** (`aiAnalyzer.ts`): Orchestrates analysis workflow
|
||||
- **Service** (`llmModerationClient.ts`): LLM API integration
|
||||
- **Service** (`aiAnalysisWorker.ts`): Worker pool management
|
||||
- **Service** (`indonesianTextNormalizer.ts`): Text preprocessing
|
||||
|
||||
**Voice Recording Module**:
|
||||
- **Controller** (`voiceController.ts`): Voice channel connection management
|
||||
- **Service** (`recorder.ts`): Recording orchestration
|
||||
- **Sub-services** (`recorder/*`): Audio stream, decoding, segmentation, upload
|
||||
|
||||
**Attachment Upload Module**:
|
||||
- **Service** (`attachmentUploader.ts`): Upload orchestration
|
||||
- **Service** (`imageResizer.ts`): Image processing
|
||||
|
||||
### 2. Event-Driven Architecture
|
||||
**Redis Pub/Sub** replaces WebSocket broadcaster:
|
||||
|
||||
```
|
||||
Discord Events → Discord Gateway Service → Redis Pub/Sub → Backend Service
|
||||
↓
|
||||
Event Channels:
|
||||
- discord:message:created
|
||||
- discord:message:updated
|
||||
- discord:message:deleted
|
||||
- discord:message:analyzed
|
||||
- discord:attachment:created
|
||||
- discord:attachment:uploaded
|
||||
- discord:voice:started
|
||||
- discord:voice:stopped
|
||||
- discord:voice:uploaded
|
||||
- discord:analysis:queue_status
|
||||
```
|
||||
|
||||
### 3. Shared Infrastructure Layer
|
||||
Centralized, reusable components:
|
||||
- **Config**: Zod-validated environment variables
|
||||
- **Logger**: Winston logger with context support
|
||||
- **Database**: Drizzle ORM with PostgreSQL
|
||||
- **Errors**: Custom error classes with codes and HTTP status codes
|
||||
- **Utils**: Retry logic with exponential backoff
|
||||
- **Discord**: Client configuration and options
|
||||
|
||||
### 4. No HTTP Server
|
||||
- **Event-driven only**: No Express, WebSocket, or HTTP routes
|
||||
- **Redis pub/sub**: All inter-service communication via Redis
|
||||
- **Backend service**: Consumes events and serves HTTP API
|
||||
- **Frontend**: Continues to use Backend HTTP API
|
||||
|
||||
## Key Features
|
||||
|
||||
### Message Capture
|
||||
1. Discord emits `messageCreate`, `messageUpdate`, `messageDelete` events
|
||||
2. `messageCapture.ts` listener receives and validates event
|
||||
3. Extract metadata: user, channel, content, timestamp, attachments
|
||||
4. `messageStore.ts` inserts into PostgreSQL
|
||||
5. `eventBroadcaster.messageCreated()` publishes to Redis
|
||||
6. Backend service subscribes and processes
|
||||
|
||||
### AI Moderation
|
||||
1. `aiAnalyzer.ts` queues messages for analysis
|
||||
2. `llmModerationClient.ts` calls LLM API with context
|
||||
3. `indonesianTextNormalizer.ts` preprocesses text
|
||||
4. Results stored in database
|
||||
5. `eventBroadcaster.messageAnalyzed()` publishes results
|
||||
6. Backend service receives and updates UI
|
||||
|
||||
### Voice Recording
|
||||
1. `voiceController.connect()` joins voice channel
|
||||
2. `recorder.ts` subscribes to user audio streams
|
||||
3. For each speaking user:
|
||||
- `audioStream.ts` subscribes to Opus packets
|
||||
- `decoder.ts` decodes Opus to PCM
|
||||
- `segment.ts` rotates OGG files (5s default)
|
||||
- `metadata.ts` collects user info
|
||||
4. On silence (3s):
|
||||
- `sessionRecording.ts` finalizes segment
|
||||
- `uploader.ts` uploads to storage
|
||||
- `eventBroadcaster.voiceRecordingUploaded()` publishes
|
||||
5. Backend service indexes recording
|
||||
|
||||
### Attachment Upload
|
||||
1. `messageCapture.ts` detects attachments
|
||||
2. `attachmentUploader.ts` downloads from Discord
|
||||
3. `imageResizer.ts` resizes images if needed
|
||||
4. Upload to external storage with retry logic
|
||||
5. `eventBroadcaster.attachmentUploaded()` publishes
|
||||
6. Backend service stores metadata
|
||||
|
||||
## Initialization Flow
|
||||
|
||||
```
|
||||
1. Load environment config (Zod validation)
|
||||
↓
|
||||
2. Initialize PostgreSQL connection
|
||||
↓
|
||||
3. Run pending database migrations
|
||||
↓
|
||||
4. Create Discord client with optimized cache
|
||||
↓
|
||||
5. Initialize Redis event broadcaster
|
||||
↓
|
||||
6. Register Discord event listeners
|
||||
- messageCapture (message events)
|
||||
- aiAnalyzer (analysis worker)
|
||||
↓
|
||||
7. Login to Discord
|
||||
↓
|
||||
8. Listen for graceful shutdown signals
|
||||
```
|
||||
|
||||
## Graceful Shutdown
|
||||
|
||||
On SIGINT/SIGTERM/uncaughtException/unhandledRejection:
|
||||
1. Close PostgreSQL connection
|
||||
2. Disconnect from voice channels
|
||||
3. Close Redis connection
|
||||
4. Destroy Discord client
|
||||
5. Exit process (code 0 for clean, 1 for error)
|
||||
|
||||
## Dependencies
|
||||
|
||||
**Core Discord**:
|
||||
- `discord.js-selfbot-v13` — Discord client (selfbot variant)
|
||||
- `@discordjs/voice` — Voice connection management
|
||||
- `@discordjs/opus` — Native Opus codec
|
||||
|
||||
**Audio Processing**:
|
||||
- `prism-media` — Opus encoding/decoding
|
||||
- `opusscript` — Opus fallback for Node v26+
|
||||
- `sharp` — Image resizing
|
||||
|
||||
**Data & Config**:
|
||||
- `drizzle-orm` — Type-safe ORM
|
||||
- `pg` — PostgreSQL driver
|
||||
- `zod` — Config validation
|
||||
- `ioredis` — Redis client
|
||||
|
||||
**Logging & Utilities**:
|
||||
- `winston` — Structured logging
|
||||
- `p-retry` — Retry with backoff
|
||||
- `p-limit` — Concurrency limiting
|
||||
- `piscina` — Worker pool
|
||||
|
||||
## No Breaking Changes
|
||||
|
||||
- Original `src/` remains untouched
|
||||
- Discord Gateway is a **new service** in `services/discord-gateway/`
|
||||
- Can run alongside existing monolith during transition
|
||||
- Backend service will consume Redis events
|
||||
- Frontend continues to use Backend HTTP API
|
||||
|
||||
## Next Steps
|
||||
|
||||
1. **Create Backend service** (`services/backend/`)
|
||||
- HTTP API endpoints
|
||||
- Redis event subscribers
|
||||
- Database models
|
||||
- WebSocket broadcaster
|
||||
|
||||
2. **Update Frontend** (`frontend/`)
|
||||
- Connect to Backend HTTP API
|
||||
- Subscribe to WebSocket events
|
||||
|
||||
3. **Docker & CI/CD**
|
||||
- Dockerfile for Discord Gateway
|
||||
- Docker Compose for multi-service setup
|
||||
- GitHub Actions for build/deploy
|
||||
|
||||
4. **Documentation**
|
||||
- API documentation
|
||||
- Event schema documentation
|
||||
- Deployment guide
|
||||
|
||||
## Files Created
|
||||
|
||||
**Total: 43 files**
|
||||
|
||||
### Shared Infrastructure (9 files)
|
||||
- `src/shared/config/config.ts`
|
||||
- `src/shared/database/` (5 files)
|
||||
- `src/shared/errors/errors.ts`
|
||||
- `src/shared/logger/logger.ts`
|
||||
- `src/shared/logger/serialization.ts`
|
||||
- `src/shared/utils/retry.ts`
|
||||
- `src/shared/discord/clientOptions.ts`
|
||||
|
||||
### Modules (28 files)
|
||||
- `src/modules/message-capture/` (5 files)
|
||||
- `src/modules/ai-moderation/` (6 files)
|
||||
- `src/modules/voice-recording/` (9 files)
|
||||
- `src/modules/attachment-upload/` (3 files)
|
||||
- `src/modules/event-broadcaster/` (3 files)
|
||||
|
||||
### App & Entry (4 files)
|
||||
- `src/app/bootstrap.ts`
|
||||
- `src/app/shutdown.ts`
|
||||
- `src/index.ts`
|
||||
- `src/mock-crc.ts`
|
||||
|
||||
### Configuration (2 files)
|
||||
- `package.json`
|
||||
- `ARCHITECTURE.md`
|
||||
|
||||
## Verification Checklist
|
||||
|
||||
✅ Directory structure created
|
||||
✅ Shared infrastructure migrated
|
||||
✅ Message capture module migrated
|
||||
✅ AI moderation module migrated
|
||||
✅ Voice recording module migrated
|
||||
✅ Attachment upload module migrated
|
||||
✅ Event broadcaster module created (Redis pub/sub)
|
||||
✅ Bootstrap and entry point created
|
||||
✅ Package.json with dependencies
|
||||
✅ No HTTP server code (Express, WebSocket removed)
|
||||
✅ Event-driven architecture implemented
|
||||
✅ Graceful shutdown handler
|
||||
✅ Module index files for clean exports
|
||||
✅ Architecture documentation
|
||||
|
||||
## Event Flow Diagram
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────────────┐
|
||||
│ Discord Gateway Service │
|
||||
├─────────────────────────────────────────────────────────────────┤
|
||||
│ │
|
||||
│ ┌──────────────────┐ ┌──────────────────┐ ┌──────────────┐ │
|
||||
│ │ Message Capture │ │ AI Moderation │ │ Voice Record │ │
|
||||
│ │ (Controller) │ │ (Controller) │ │ (Controller) │ │
|
||||
│ └────────┬─────────┘ └────────┬─────────┘ └──────┬───────┘ │
|
||||
│ │ │ │ │
|
||||
│ ├─────────────────────┼───────────────────┤ │
|
||||
│ │ │ │ │
|
||||
│ ▼ ▼ ▼ │
|
||||
│ ┌─────────────────────────────────────────────────────────┐ │
|
||||
│ │ Event Broadcaster (Redis Pub/Sub) │ │
|
||||
│ │ - discord:message:created │ │
|
||||
│ │ - discord:message:updated │ │
|
||||
│ │ - discord:message:deleted │ │
|
||||
│ │ - discord:message:analyzed │ │
|
||||
│ │ - discord:attachment:created │ │
|
||||
│ │ - discord:attachment:uploaded │ │
|
||||
│ │ - discord:voice:started │ │
|
||||
│ │ - discord:voice:stopped │ │
|
||||
│ │ - discord:voice:uploaded │ │
|
||||
│ │ - discord:analysis:queue_status │ │
|
||||
│ └─────────────────────────────────────────────────────────┘ │
|
||||
│ │ │
|
||||
└───────────┼────────────────────────────────────────────────────┘
|
||||
│
|
||||
│ Redis Pub/Sub
|
||||
│
|
||||
▼
|
||||
┌─────────────────────────────────────────────────────────────────┐
|
||||
│ Backend Service │
|
||||
│ (Subscribes to events, serves HTTP API, manages WebSocket) │
|
||||
└─────────────────────────────────────────────────────────────────┘
|
||||
│
|
||||
│ HTTP API
|
||||
│
|
||||
▼
|
||||
┌─────────────────────────────────────────────────────────────────┐
|
||||
│ Frontend Application │
|
||||
│ (React SPA, real-time updates via WebSocket) │
|
||||
└─────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## Summary
|
||||
|
||||
The Discord Gateway service has been successfully extracted with:
|
||||
- **Modular MVC architecture** for clean separation of concerns
|
||||
- **Event-driven design** using Redis pub/sub for inter-service communication
|
||||
- **Shared infrastructure layer** for reusable components
|
||||
- **No HTTP server** — pure event-driven service
|
||||
- **Graceful shutdown** handling
|
||||
- **Type-safe configuration** with Zod validation
|
||||
- **Structured logging** with Winston
|
||||
- **PostgreSQL integration** with Drizzle ORM
|
||||
|
||||
The service is ready for integration with the Backend service, which will consume Redis events and serve the HTTP API to the Frontend.
|
||||
@@ -0,0 +1,45 @@
|
||||
{
|
||||
"name": "@bete/discord-gateway",
|
||||
"version": "1.0.0",
|
||||
"description": "Discord Gateway service - handles message capture, voice recording, and AI moderation",
|
||||
"type": "module",
|
||||
"main": "dist/index.js",
|
||||
"packageManager": "pnpm@11.1.3",
|
||||
"scripts": {
|
||||
"dev": "tsx watch src/index.ts",
|
||||
"start": "node dist/index.js",
|
||||
"build": "tsc",
|
||||
"typecheck": "tsc --noEmit",
|
||||
"lint": "biome check --diagnostic-level=error .",
|
||||
"format": "biome format --write .",
|
||||
"test": "vitest run"
|
||||
},
|
||||
"dependencies": {
|
||||
"@discordjs/opus": "^0.10.0",
|
||||
"@discordjs/voice": "^0.19.2",
|
||||
"@snazzah/davey": "^0.1.11",
|
||||
"discord.js-selfbot-v13": "workspace:*",
|
||||
"dotenv": "^17.4.2",
|
||||
"drizzle-orm": "^0.45.2",
|
||||
"ioredis": "^5.11.0",
|
||||
"libsodium-wrappers": "^0.8.4",
|
||||
"openai": "^6.38.0",
|
||||
"opusscript": "^0.0.8",
|
||||
"p-limit": "^7.3.0",
|
||||
"p-retry": "^8.0.0",
|
||||
"pg": "^8.21.0",
|
||||
"piscina": "^5.1.4",
|
||||
"prism-media": "2.0.0-alpha.0",
|
||||
"sharp": "^0.34.5",
|
||||
"winston": "^3.19.0",
|
||||
"zod": "^4.4.3"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@biomejs/biome": "latest",
|
||||
"@types/node": "^25.9.0",
|
||||
"drizzle-kit": "^0.31.10",
|
||||
"tsx": "^4.22.2",
|
||||
"typescript": "^5.9.3",
|
||||
"vitest": "latest"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
import { Client } from "discord.js-selfbot-v13";
|
||||
import { startPendingAIAnalysisWorker } from "../modules/ai-moderation/aiAnalyzer.js";
|
||||
import {
|
||||
EventBroadcaster,
|
||||
RedisEventPublisher,
|
||||
} from "../modules/event-broadcaster/index.js";
|
||||
import {
|
||||
registerMessageCapture,
|
||||
setEventBroadcaster,
|
||||
} from "../modules/message-capture/messageCapture.js";
|
||||
import { VoiceController } from "../modules/voice-recording/voiceController.js";
|
||||
import { config } from "../shared/config/config.js";
|
||||
import {
|
||||
closeDatabase,
|
||||
initializeDatabase,
|
||||
} from "../shared/database/drizzle.js";
|
||||
import { runMigrations } from "../shared/database/migrate.js";
|
||||
import { createDiscordClientOptions } from "../shared/discord/clientOptions.js";
|
||||
import { createChildLogger } from "../shared/logger/logger.js";
|
||||
import { createGracefulShutdown } from "./shutdown.js";
|
||||
|
||||
const logger = createChildLogger("discord-gateway");
|
||||
|
||||
export async function initializeDiscordGateway() {
|
||||
if (!config.AI_LLM_API_KEY) {
|
||||
logger.error(
|
||||
"AI_LLM_API_KEY is missing from environment. Force closing application as AI environment is required.",
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const token = config.DISCORD_TOKEN;
|
||||
logger.info(
|
||||
{ hasToken: token.length > 0, tokenLength: token.length },
|
||||
"Config loaded",
|
||||
);
|
||||
|
||||
logger.info("Creating Discord client");
|
||||
const client = new Client(createDiscordClientOptions());
|
||||
const voiceController = new VoiceController(client);
|
||||
|
||||
// Initialize Redis event broadcaster
|
||||
const redisPublisher = new RedisEventPublisher(config.REDIS_URL, logger);
|
||||
const eventBroadcaster = new EventBroadcaster(redisPublisher, logger);
|
||||
|
||||
const gracefulShutdown = createGracefulShutdown({
|
||||
logger,
|
||||
closeDatabase,
|
||||
voiceController,
|
||||
client,
|
||||
eventBroadcaster,
|
||||
});
|
||||
|
||||
try {
|
||||
if (config.AUTO_MIGRATE_ON_STARTUP) {
|
||||
logger.info(
|
||||
"AUTO_MIGRATE_ON_STARTUP enabled; running database migrations",
|
||||
);
|
||||
await runMigrations();
|
||||
}
|
||||
|
||||
logger.info("Initializing database");
|
||||
await initializeDatabase();
|
||||
logger.info("PostgreSQL database initialized");
|
||||
} catch (err) {
|
||||
logger.error({ error: err }, "Failed to initialize database");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
client.on("debug", (msg) => {
|
||||
if (
|
||||
msg.includes("[VOICE") ||
|
||||
msg.includes("[ffmpeg") ||
|
||||
msg.toLowerCase().includes("error") ||
|
||||
msg.toLowerCase().includes("stream")
|
||||
) {
|
||||
logger.info({ debugMsg: msg }, "Discord Client Debug");
|
||||
} else if (config.VERBOSE) {
|
||||
logger.debug({ debugMsg: msg }, "Discord Client Debug");
|
||||
}
|
||||
});
|
||||
|
||||
client.on("ready", async () => {
|
||||
logger.info({ user: client.user?.tag }, "Bot logged in");
|
||||
setEventBroadcaster(eventBroadcaster);
|
||||
registerMessageCapture(client);
|
||||
startPendingAIAnalysisWorker(client);
|
||||
});
|
||||
|
||||
client.on("error", (err) => {
|
||||
logger.error({ error: err }, "Client error");
|
||||
});
|
||||
|
||||
process.on("SIGINT", () => {
|
||||
gracefulShutdown("SIGINT");
|
||||
});
|
||||
|
||||
process.on("SIGTERM", () => {
|
||||
gracefulShutdown("SIGTERM");
|
||||
});
|
||||
|
||||
process.on("uncaughtException", (err) => {
|
||||
logger.error({ error: err }, "Uncaught exception");
|
||||
gracefulShutdown("uncaughtException");
|
||||
});
|
||||
|
||||
process.on("unhandledRejection", (reason, promise) => {
|
||||
logger.error({ reason, promise }, "Unhandled rejection");
|
||||
gracefulShutdown("unhandledRejection");
|
||||
});
|
||||
|
||||
logger.info("Calling Discord client.login");
|
||||
client
|
||||
.login(token)
|
||||
.then(() => {
|
||||
logger.info("Discord client.login resolved");
|
||||
})
|
||||
.catch((error: unknown) => {
|
||||
logger.error({ error }, "Discord client.login failed");
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
import type { Client } from "discord.js-selfbot-v13";
|
||||
import type { EventBroadcaster } from "../modules/event-broadcaster/index.js";
|
||||
import type { VoiceController } from "../modules/voice-recording/voiceController.js";
|
||||
import type { closeDatabase } from "../shared/database/drizzle.js";
|
||||
import type { createChildLogger } from "../shared/logger/logger.js";
|
||||
|
||||
type Logger = ReturnType<typeof createChildLogger>;
|
||||
type CloseDatabase = typeof closeDatabase;
|
||||
|
||||
export interface GracefulShutdownOptions {
|
||||
logger: Logger;
|
||||
closeDatabase: CloseDatabase;
|
||||
voiceController: VoiceController;
|
||||
client: Client;
|
||||
eventBroadcaster: EventBroadcaster;
|
||||
}
|
||||
|
||||
export function createGracefulShutdown(options: GracefulShutdownOptions) {
|
||||
let isShuttingDown = false;
|
||||
|
||||
return async function gracefulShutdown(signal: string) {
|
||||
if (isShuttingDown) {
|
||||
options.logger.warn(`Already shutting down, ignoring ${signal}`);
|
||||
return;
|
||||
}
|
||||
|
||||
isShuttingDown = true;
|
||||
options.logger.info({ signal }, "Graceful shutdown initiated");
|
||||
|
||||
try {
|
||||
options.logger.info("Closing database...");
|
||||
await options.closeDatabase();
|
||||
options.logger.info("Database closed");
|
||||
|
||||
options.logger.info("Stopping voice connection...");
|
||||
await options.voiceController.disconnect();
|
||||
|
||||
options.logger.info("Closing event broadcaster...");
|
||||
await options.eventBroadcaster.close();
|
||||
|
||||
options.logger.info("Destroying Discord client...");
|
||||
try {
|
||||
options.client.destroy();
|
||||
} catch (err) {
|
||||
options.logger.warn({ error: err }, "Error destroying client");
|
||||
}
|
||||
|
||||
options.logger.info("Graceful shutdown completed");
|
||||
process.exit(0);
|
||||
} catch (err) {
|
||||
options.logger.error({ error: err }, "Error during graceful shutdown");
|
||||
process.exit(1);
|
||||
}
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
import "./mock-crc.js";
|
||||
import "libsodium-wrappers";
|
||||
import "@snazzah/davey";
|
||||
import "dotenv/config";
|
||||
import { initializeDiscordGateway } from "./app/bootstrap.js";
|
||||
import { createChildLogger } from "./shared/logger/logger.js";
|
||||
|
||||
const logger = createChildLogger("discord-gateway");
|
||||
|
||||
// Initialize the Discord Gateway service
|
||||
initializeDiscordGateway().catch((error: unknown) => {
|
||||
logger.error({ error }, "Failed to initialize Discord Gateway");
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,16 @@
|
||||
// Mock CRC for discord.js compatibility
|
||||
export {};
|
||||
|
||||
declare global {
|
||||
var crc32: ((data: Buffer) => number) | undefined;
|
||||
}
|
||||
|
||||
if (!globalThis.crc32) {
|
||||
globalThis.crc32 = (data: Buffer) => {
|
||||
let crc = 0 ^ -1;
|
||||
for (let i = 0; i < data.length; i++) {
|
||||
crc = (crc >>> 8) ^ ((crc ^ data[i]) & 0xff);
|
||||
}
|
||||
return (crc ^ -1) >>> 0;
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,145 @@
|
||||
import { config } from "../../shared/config/config.js";
|
||||
import { initializeDatabase } from "../../shared/database/drizzle.js";
|
||||
import { buildConversationContext } from "./conversationContext.js";
|
||||
import { runModerationAnalysis } from "./llmModerationClient.js";
|
||||
import {
|
||||
getAttachmentsForMessages,
|
||||
getConversationContextBefore,
|
||||
updateMessagesAIAnalysisBulk,
|
||||
} from "../message-capture/messageStore.js";
|
||||
import type { MessageRecord } from "../message-capture/types.js";
|
||||
|
||||
let dbInitialized = false;
|
||||
let dbInitPromise: Promise<any> | null = null;
|
||||
|
||||
async function ensureDb() {
|
||||
if (dbInitialized) return;
|
||||
if (!dbInitPromise) {
|
||||
dbInitPromise = initializeDatabase().then(() => {
|
||||
dbInitialized = true;
|
||||
});
|
||||
}
|
||||
await dbInitPromise;
|
||||
}
|
||||
|
||||
export interface AnalysisWorkerRequest {
|
||||
conversationKey: string;
|
||||
messages: MessageRecord[];
|
||||
}
|
||||
|
||||
export type AnalysisWorkerResponse =
|
||||
| {
|
||||
ok: true;
|
||||
conversationKey: string;
|
||||
rows: MessageRecord[];
|
||||
}
|
||||
| {
|
||||
ok: false;
|
||||
conversationKey: string;
|
||||
rows: MessageRecord[];
|
||||
error: string;
|
||||
};
|
||||
|
||||
export default async function processAnalysisRequest({
|
||||
conversationKey,
|
||||
messages,
|
||||
}: AnalysisWorkerRequest): Promise<AnalysisWorkerResponse> {
|
||||
if (!config.AI_LLM_API_KEY) {
|
||||
console.error(
|
||||
JSON.stringify({
|
||||
level: "FATAL",
|
||||
context: "aiAnalysisWorker",
|
||||
error:
|
||||
"AI_LLM_API_KEY is missing from environment. Force closing worker operation.",
|
||||
timestamp: new Date().toISOString(),
|
||||
}),
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
try {
|
||||
try {
|
||||
await ensureDb();
|
||||
} catch (dbError) {
|
||||
const msg = dbError instanceof Error ? dbError.message : String(dbError);
|
||||
return {
|
||||
ok: false,
|
||||
conversationKey,
|
||||
rows: [],
|
||||
error: `Database init failed: ${msg}`,
|
||||
};
|
||||
}
|
||||
|
||||
const firstMessage = messages[0];
|
||||
if (!firstMessage) return { ok: true, conversationKey, rows: [] };
|
||||
|
||||
const contextBefore = await getConversationContextBefore({
|
||||
channelId: firstMessage.channel_id,
|
||||
threadId: firstMessage.thread_id,
|
||||
beforeCreatedAt: firstMessage.created_at,
|
||||
limit: config.AI_ANALYSIS_CONTEXT_MESSAGE_LIMIT,
|
||||
});
|
||||
|
||||
const contextLines = await buildConversationContext({
|
||||
contextBefore,
|
||||
targets: messages,
|
||||
maxTokens: config.AI_ANALYSIS_MAX_CONTEXT_TOKENS,
|
||||
});
|
||||
|
||||
const targetIds = messages.map((m) => m.id);
|
||||
const contextIds = contextBefore.map((m) => m.id);
|
||||
const allMessageIds = [...targetIds, ...contextIds];
|
||||
const attachments = await getAttachmentsForMessages(allMessageIds);
|
||||
|
||||
const result = await runModerationAnalysis({
|
||||
targets: messages,
|
||||
contextText: contextLines.join("\n"),
|
||||
attachments,
|
||||
});
|
||||
|
||||
const updates = result.results.map((analysisResult) => ({
|
||||
messageId: analysisResult.messageId,
|
||||
result: {
|
||||
status: analysisResult.status,
|
||||
flags: JSON.stringify(analysisResult.flags),
|
||||
score: analysisResult.score,
|
||||
analysis: analysisResult.analysis,
|
||||
categories: analysisResult.categories,
|
||||
severity: analysisResult.severity,
|
||||
confidence: analysisResult.confidence,
|
||||
recommendedAction: analysisResult.recommendedAction,
|
||||
analyzedAt: Date.now(),
|
||||
error: null,
|
||||
},
|
||||
}));
|
||||
|
||||
try {
|
||||
const rows = await updateMessagesAIAnalysisBulk(updates);
|
||||
return { ok: true, conversationKey, rows };
|
||||
} catch (dbErr) {
|
||||
// If bulk update fails, we log it but don't fail the worker completely
|
||||
// so it can at least retry later without blowing up the circuit breaker if it was an isolated issue
|
||||
throw new Error(
|
||||
`Failed to update DB: ${dbErr instanceof Error ? dbErr.message : String(dbErr)}`,
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
const errorMessage = error instanceof Error ? error.message : String(error);
|
||||
const errorStack = error instanceof Error ? error.stack : undefined;
|
||||
const rows: MessageRecord[] = [];
|
||||
|
||||
console.error(
|
||||
JSON.stringify({
|
||||
level: "ERROR",
|
||||
context: "aiAnalysisWorker",
|
||||
conversationKey,
|
||||
messageCount: messages.length,
|
||||
error: errorMessage,
|
||||
stack: errorStack,
|
||||
timestamp: new Date().toISOString(),
|
||||
}),
|
||||
);
|
||||
|
||||
return { ok: false, conversationKey, rows, error: errorMessage };
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,918 @@
|
||||
import { existsSync } from "node:fs";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import type { Client } from "discord.js-selfbot-v13";
|
||||
import { AbortError } from "p-retry";
|
||||
import { Piscina } from "piscina";
|
||||
import { config } from "../../shared/config/config.js";
|
||||
import { createChildLogger } from "../../shared/logger/logger.js";
|
||||
import { retryWithBackoff } from "../../shared/utils/retry.js";
|
||||
import { invalidateAnalyticsCache } from "../message-capture/analyticsStore.js";
|
||||
import { attemptAutoDeleteFlaggedMessage } from "./autoDeleteManager.js";
|
||||
import { buildConversationContext } from "./conversationContext.js";
|
||||
import { runModerationAnalysis } from "./llmModerationClient.js";
|
||||
import { isAgeRestrictedMetadata } from "../message-capture/messageMetadata.js";
|
||||
import {
|
||||
getAttachmentsForMessages,
|
||||
getConversationContextBefore,
|
||||
getConversationKeysWithIncompleteAnalysis,
|
||||
getIncompleteMessagesByConversation,
|
||||
getMessageById,
|
||||
getPendingConversationKeys,
|
||||
getPendingMessagesByConversation,
|
||||
updateMessageAIAnalysis,
|
||||
updateMessagesAIAnalysisBulk,
|
||||
} from "../message-capture/messageStore.js";
|
||||
import type {
|
||||
AnalysisQueueStatus,
|
||||
MessageRecord,
|
||||
ModerationBroadcaster,
|
||||
} from "../message-capture/types.js";
|
||||
|
||||
const logger = createChildLogger("ai-analyzer");
|
||||
|
||||
type ModerationGlobal = typeof globalThis & {
|
||||
moderationBroadcaster?: ModerationBroadcaster;
|
||||
};
|
||||
|
||||
function getModerationBroadcaster(): ModerationBroadcaster | undefined {
|
||||
return (globalThis as ModerationGlobal).moderationBroadcaster;
|
||||
}
|
||||
|
||||
function scheduleAutoDelete(row: MessageRecord): void {
|
||||
if (row.ai_status !== "flagged" && row.ai_status !== "warn") return;
|
||||
const run = () => {
|
||||
attemptAutoDeleteFlaggedMessage(moderationClient, row).catch((error: unknown) => {
|
||||
logger.error(
|
||||
{
|
||||
messageId: row.id,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
},
|
||||
"Unexpected auto-delete error",
|
||||
);
|
||||
});
|
||||
};
|
||||
|
||||
if (config.AUTO_DELETE_FLAGGED_DELAY_MS > 0) {
|
||||
setTimeout(run, config.AUTO_DELETE_FLAGGED_DELAY_MS);
|
||||
return;
|
||||
}
|
||||
setImmediate(run);
|
||||
}
|
||||
|
||||
function isAgeRestrictedMessage(message: MessageRecord): boolean {
|
||||
return isAgeRestrictedMetadata(message.metadata);
|
||||
}
|
||||
|
||||
function buildAgeRestrictedSkipResult(): {
|
||||
status: "clean";
|
||||
flags: string | null;
|
||||
score: number;
|
||||
analysis: string;
|
||||
categories: string[];
|
||||
severity: "none";
|
||||
confidence: number;
|
||||
recommendedAction: "none";
|
||||
analyzedAt: number;
|
||||
error: null;
|
||||
} {
|
||||
return {
|
||||
status: "clean",
|
||||
flags: JSON.stringify(["age_restricted"]),
|
||||
score: 0,
|
||||
analysis: "Skipped moderation for age-restricted content.",
|
||||
categories: ["age_restricted"],
|
||||
severity: "none",
|
||||
confidence: 1,
|
||||
recommendedAction: "none",
|
||||
analyzedAt: Date.now(),
|
||||
error: null,
|
||||
};
|
||||
}
|
||||
|
||||
async function skipAgeRestrictedMessages(
|
||||
messages: MessageRecord[],
|
||||
): Promise<MessageRecord[]> {
|
||||
const ageRestrictedMessages = messages.filter(isAgeRestrictedMessage);
|
||||
if (ageRestrictedMessages.length === 0) {
|
||||
return messages;
|
||||
}
|
||||
|
||||
const skippedRows = await updateMessagesAIAnalysisBulk(
|
||||
ageRestrictedMessages.map((message) => ({
|
||||
messageId: message.id,
|
||||
result: buildAgeRestrictedSkipResult(),
|
||||
})),
|
||||
);
|
||||
|
||||
for (const row of skippedRows) {
|
||||
getModerationBroadcaster()?.messageAnalyzed(row);
|
||||
}
|
||||
|
||||
const skippedIds = new Set(
|
||||
ageRestrictedMessages.map((message) => message.id),
|
||||
);
|
||||
return messages.filter((message) => !skippedIds.has(message.id));
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Batch pipeline state
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** Debounce timer handle per conversation key. */
|
||||
const conversationDebounceTimers = new Map<string, NodeJS.Timeout>();
|
||||
/** Timestamp of when processing started per conversation key. */
|
||||
const conversationProcessing = new Map<string, number>();
|
||||
/** Cooldown expiry timestamp per conversation key after an error. */
|
||||
const conversationErrorCooldown = new Map<string, number>();
|
||||
|
||||
let activeRequests = 0;
|
||||
let lastError: string | null = null;
|
||||
let moderationClient: Client | undefined;
|
||||
|
||||
// Batch circuit breaker
|
||||
let consecutiveErrors = 0;
|
||||
const MAX_CONSECUTIVE_ERRORS = 5;
|
||||
let globalCooldownUntil = 0;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Individual fallback queue — runs PARALLEL to the batch pipeline.
|
||||
//
|
||||
// Design guarantees:
|
||||
// • Concurrency is capped at config.AI_ANALYSIS_INDIVIDUAL_MAX_CONCURRENT.
|
||||
// • A flat Set<messageId> de-duplicates so the same message can't be
|
||||
// in-flight twice (Discord snowflakes are globally unique, but be safe).
|
||||
// • A Map<conversationKey, count> lets the recovery worker skip conversations
|
||||
// that already have individual work in progress (#4 fix).
|
||||
// • A separate circuit breaker prevents a cascade of individual failures
|
||||
// from hammering a down/rate-limited LLM endpoint (#1+#5 fix).
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** IDs currently being processed one-by-one. */
|
||||
const individualInFlight = new Set<string>();
|
||||
|
||||
/**
|
||||
* Per-conversation count of in-flight individual messages.
|
||||
* Used by the recovery worker to avoid re-scheduling a conversation that
|
||||
* already has individual fallback work running for it.
|
||||
*/
|
||||
const individualInFlightByConversation = new Map<string, number>();
|
||||
|
||||
/** Counter for observability. */
|
||||
let activeIndividualRequests = 0;
|
||||
|
||||
// Individual fallback circuit breaker (independent of batch CB)
|
||||
let individualConsecutiveErrors = 0;
|
||||
let individualCooldownUntil = 0;
|
||||
const INDIVIDUAL_COOLDOWN_MS = 30000;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Piscina worker pool (batch path only)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function getAnalysisWorkerUrl(): URL {
|
||||
const candidates = [
|
||||
new URL("./aiAnalysisWorker.js", import.meta.url),
|
||||
new URL("../aiAnalysisWorker.js", import.meta.url),
|
||||
new URL("./aiAnalysisWorker.ts", import.meta.url),
|
||||
];
|
||||
|
||||
for (const candidate of candidates) {
|
||||
if (existsSync(fileURLToPath(candidate))) {
|
||||
return candidate;
|
||||
}
|
||||
}
|
||||
|
||||
return candidates[2];
|
||||
}
|
||||
|
||||
const workerPool = new Piscina({
|
||||
filename: fileURLToPath(getAnalysisWorkerUrl()),
|
||||
execArgv: process.execArgv,
|
||||
});
|
||||
|
||||
interface AnalysisWorkerResponse {
|
||||
ok: boolean;
|
||||
conversationKey: string;
|
||||
rows: MessageRecord[];
|
||||
error?: string;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Exported helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Gets the conversation key for a message (thread_id or channel_id).
|
||||
*/
|
||||
export function getConversationKey(message: MessageRecord): string {
|
||||
return message.thread_id || message.channel_id;
|
||||
}
|
||||
|
||||
/**
|
||||
* Picks a batch of messages within a token budget.
|
||||
* `tokensPerMessage` accounts for JSON structure overhead around each entry.
|
||||
* Uses a rough character-based token estimate (avoids async formatMessageForPrompt
|
||||
* since this function runs in a synchronous promise chain).
|
||||
*/
|
||||
export function pickBatchWithinBudget(
|
||||
messages: MessageRecord[],
|
||||
maxTokens: number,
|
||||
tokensPerMessage: number,
|
||||
): MessageRecord[] {
|
||||
const batch: MessageRecord[] = [];
|
||||
let usedTokens = 0;
|
||||
|
||||
for (const msg of messages) {
|
||||
const content = msg.edited_content ?? msg.content;
|
||||
// Rough token estimate: ~3 chars per token + metadata overhead
|
||||
const msgTokens = Math.ceil(content.length / 3) + tokensPerMessage;
|
||||
|
||||
if (usedTokens + msgTokens <= maxTokens) {
|
||||
batch.push(msg);
|
||||
usedTokens += msgTokens;
|
||||
}
|
||||
}
|
||||
|
||||
return batch;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Conversation lock helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function isConversationProcessingLocked(conversationKey: string): boolean {
|
||||
const startedAt = conversationProcessing.get(conversationKey);
|
||||
// FIX #7: use configurable timeout that exceeds (LLM timeout × max retries).
|
||||
// Old hardcoded value was 30 000 ms — shorter than a single LLM call under retries.
|
||||
return Boolean(
|
||||
startedAt &&
|
||||
Date.now() - startedAt < config.AI_ANALYSIS_PROCESSING_TIMEOUT_MS,
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Individual fallback pipeline
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Processes a single message directly in the main process (no IPC/worker
|
||||
* pool overhead). Never called from the batch path.
|
||||
*
|
||||
* FIX #1+#5: Increments the individual circuit breaker on failure so a
|
||||
* sustained outage stops hammering the LLM endpoint.
|
||||
*
|
||||
* Infinite-loop prevention: if the LLM consistently drops the single target
|
||||
* message across all retries (analysis_incomplete), we write a terminal flag
|
||||
* 'individual_analysis_exhausted' to DB instead of 'analysis_incomplete'.
|
||||
* The recovery worker only queries for 'analysis_incomplete', so exhausted
|
||||
* messages are permanently excluded from the reprocessing loop.
|
||||
* Transient failures (network/parse/DB) are NOT written as exhausted — they
|
||||
* stay as 'analysis_incomplete' so the circuit-breaker-throttled recovery
|
||||
* cycle can retry them later.
|
||||
*/
|
||||
async function processIndividualFallback(
|
||||
message: MessageRecord,
|
||||
): Promise<void> {
|
||||
const { id: messageId } = message;
|
||||
const conversationKey = getConversationKey(message);
|
||||
|
||||
activeIndividualRequests++;
|
||||
// Increment per-conversation counter so the recovery worker can see it.
|
||||
individualInFlightByConversation.set(
|
||||
conversationKey,
|
||||
(individualInFlightByConversation.get(conversationKey) ?? 0) + 1,
|
||||
);
|
||||
|
||||
// Track whether all retries were exhausted specifically because the LLM
|
||||
// consistently returned no result for this message (vs. a transient error).
|
||||
let exhaustedOnIncomplete = false;
|
||||
|
||||
try {
|
||||
const contextBefore = await getConversationContextBefore({
|
||||
channelId: message.channel_id,
|
||||
threadId: message.thread_id,
|
||||
beforeCreatedAt: message.created_at,
|
||||
limit: config.AI_ANALYSIS_CONTEXT_MESSAGE_LIMIT,
|
||||
});
|
||||
|
||||
const contextLines = await buildConversationContext({
|
||||
contextBefore,
|
||||
targets: [message],
|
||||
maxTokens: config.AI_ANALYSIS_MAX_CONTEXT_TOKENS,
|
||||
});
|
||||
|
||||
const contextIds = contextBefore.map((m) => m.id);
|
||||
const attachments = await getAttachmentsForMessages([
|
||||
messageId,
|
||||
...contextIds,
|
||||
]);
|
||||
|
||||
const analysisResult = await retryWithBackoff(
|
||||
async () => {
|
||||
try {
|
||||
const result = await runModerationAnalysis({
|
||||
targets: [message],
|
||||
contextText: contextLines.join("\n"),
|
||||
attachments,
|
||||
});
|
||||
|
||||
// If the LLM still dropped our only target, convert to a retryable
|
||||
// throw so backoff kicks in. Track this so the catch block can
|
||||
// distinguish it from a transient network/parse failure.
|
||||
const stillIncomplete = result.results.some((r) =>
|
||||
r.flags.includes("analysis_incomplete"),
|
||||
);
|
||||
if (stillIncomplete) {
|
||||
exhaustedOnIncomplete = true;
|
||||
throw new Error(
|
||||
`LLM returned no result for single-target message ${messageId} — will retry with backoff`,
|
||||
);
|
||||
}
|
||||
|
||||
// Got a real result — clear the incomplete flag.
|
||||
exhaustedOnIncomplete = false;
|
||||
|
||||
return result;
|
||||
} catch (err: any) {
|
||||
// Propagate AbortError so outer retry is immediately cancelled on 429.
|
||||
if (err instanceof AbortError) {
|
||||
throw err;
|
||||
}
|
||||
if (
|
||||
err?.status === 429 ||
|
||||
err?.status === 401 ||
|
||||
err?.status === 403
|
||||
) {
|
||||
throw new AbortError(err);
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
},
|
||||
{
|
||||
retries: 2,
|
||||
minTimeout: 2000,
|
||||
maxTimeout: 15000,
|
||||
logger,
|
||||
},
|
||||
);
|
||||
|
||||
const updates = analysisResult.results.map((r) => ({
|
||||
messageId: r.messageId,
|
||||
result: {
|
||||
status: r.status,
|
||||
flags: JSON.stringify(r.flags),
|
||||
score: r.score,
|
||||
analysis: r.analysis,
|
||||
categories: r.categories,
|
||||
severity: r.severity,
|
||||
confidence: r.confidence,
|
||||
recommendedAction: r.recommendedAction,
|
||||
analyzedAt: Date.now(),
|
||||
error: null,
|
||||
},
|
||||
}));
|
||||
|
||||
const rows = await updateMessagesAIAnalysisBulk(updates);
|
||||
for (const row of rows) {
|
||||
getModerationBroadcaster()?.messageAnalyzed(row);
|
||||
invalidateAnalyticsCache(row.guild_id);
|
||||
scheduleAutoDelete(row);
|
||||
}
|
||||
|
||||
// Reset individual CB on success.
|
||||
individualConsecutiveErrors = 0;
|
||||
|
||||
logger.info(
|
||||
{ messageId, status: analysisResult.results[0]?.status },
|
||||
"Individual fallback analysis complete",
|
||||
);
|
||||
} catch (error) {
|
||||
// FIX #5: individual failures now feed their own circuit breaker.
|
||||
individualConsecutiveErrors++;
|
||||
if (
|
||||
individualConsecutiveErrors >= config.AI_ANALYSIS_INDIVIDUAL_CB_THRESHOLD
|
||||
) {
|
||||
individualCooldownUntil = Date.now() + INDIVIDUAL_COOLDOWN_MS;
|
||||
logger.warn(
|
||||
{
|
||||
threshold: config.AI_ANALYSIS_INDIVIDUAL_CB_THRESHOLD,
|
||||
cooldownUntil: new Date(individualCooldownUntil).toISOString(),
|
||||
},
|
||||
"Individual fallback circuit breaker triggered",
|
||||
);
|
||||
}
|
||||
|
||||
lastError = error instanceof Error ? error.message : String(error);
|
||||
|
||||
// Infinite-loop prevention: if all retries were exhausted because the LLM
|
||||
// consistently dropped this specific message (not a transient error),
|
||||
// overwrite the DB entry with a terminal flag that the recovery query
|
||||
// does NOT match. This permanently removes it from the recovery loop
|
||||
// while keeping it visible as an error in the dashboard.
|
||||
if (exhaustedOnIncomplete) {
|
||||
await updateMessagesAIAnalysisBulk([
|
||||
{
|
||||
messageId,
|
||||
result: {
|
||||
status: "error",
|
||||
flags: JSON.stringify(["individual_analysis_exhausted"]),
|
||||
score: 0,
|
||||
analysis:
|
||||
"Individual fallback exhausted all retries: LLM consistently dropped this message even in single-target mode",
|
||||
categories: ["individual_analysis_exhausted"],
|
||||
severity: "none",
|
||||
confidence: 0,
|
||||
recommendedAction: "review",
|
||||
analyzedAt: Date.now(),
|
||||
error: lastError,
|
||||
},
|
||||
},
|
||||
]).catch((dbErr: unknown) => {
|
||||
logger.error(
|
||||
{ messageId, error: String(dbErr) },
|
||||
"Failed to write terminal exhausted status — message may re-enter recovery loop",
|
||||
);
|
||||
});
|
||||
logger.warn(
|
||||
{ messageId },
|
||||
"Individual fallback exhausted — marked as individual_analysis_exhausted to stop recovery loop",
|
||||
);
|
||||
} else {
|
||||
// Transient failure (network/parse/DB): do NOT write terminal status.
|
||||
// Message stays as error/analysis_incomplete in DB and will be retried
|
||||
// by the recovery worker, subject to the individual circuit breaker.
|
||||
logger.error(
|
||||
{
|
||||
messageId,
|
||||
error: lastError,
|
||||
stack: error instanceof Error ? error.stack : undefined,
|
||||
},
|
||||
"Individual fallback analysis failed (transient) — will be retried by recovery worker",
|
||||
);
|
||||
}
|
||||
} finally {
|
||||
activeIndividualRequests--;
|
||||
individualInFlight.delete(messageId);
|
||||
|
||||
// Decrement per-conversation counter; remove key when it hits zero.
|
||||
const prev = individualInFlightByConversation.get(conversationKey) ?? 1;
|
||||
if (prev <= 1) {
|
||||
individualInFlightByConversation.delete(conversationKey);
|
||||
} else {
|
||||
individualInFlightByConversation.set(conversationKey, prev - 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Fans out message records to the individual fallback queue.
|
||||
*
|
||||
* FIX #1: Checks concurrency cap before admitting new work.
|
||||
* FIX #5: Checks individual circuit breaker before admitting new work.
|
||||
* Messages that cannot be admitted remain as `error/analysis_incomplete` in
|
||||
* the DB and will be picked up by the recovery worker on the next interval.
|
||||
*/
|
||||
function enqueueIndividualFallbacks(messages: MessageRecord[]): void {
|
||||
// FIX #5: Honour the individual circuit breaker.
|
||||
if (Date.now() < individualCooldownUntil) {
|
||||
logger.warn(
|
||||
{
|
||||
until: new Date(individualCooldownUntil).toISOString(),
|
||||
skipped: messages.length,
|
||||
},
|
||||
"Individual fallback circuit breaker active — messages will be recovered later",
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const newMessages = messages.filter((m) => !individualInFlight.has(m.id));
|
||||
if (newMessages.length === 0) return;
|
||||
|
||||
logger.info(
|
||||
{
|
||||
count: newMessages.length,
|
||||
messageIds: newMessages.map((m) => m.id),
|
||||
},
|
||||
"Enqueueing individual fallback analysis for batch-incomplete messages",
|
||||
);
|
||||
|
||||
for (const msg of newMessages) {
|
||||
individualInFlight.add(msg.id);
|
||||
// Fire-and-forget: processIndividualFallback handles all errors internally.
|
||||
processIndividualFallback(msg).catch((err: unknown) => {
|
||||
// Belt-and-suspenders guard — should never reach here.
|
||||
logger.error(
|
||||
{ messageId: msg.id, error: String(err) },
|
||||
"Unexpected uncaught error escaping processIndividualFallback",
|
||||
);
|
||||
individualInFlight.delete(msg.id);
|
||||
const ck = getConversationKey(msg);
|
||||
const prev = individualInFlightByConversation.get(ck) ?? 1;
|
||||
if (prev <= 1) {
|
||||
individualInFlightByConversation.delete(ck);
|
||||
} else {
|
||||
individualInFlightByConversation.set(ck, prev - 1);
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Batch pipeline
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function processBatch(
|
||||
conversationKey: string,
|
||||
messages: MessageRecord[],
|
||||
): Promise<void> {
|
||||
if (messages.length === 0) return;
|
||||
if (Date.now() < globalCooldownUntil) {
|
||||
return;
|
||||
}
|
||||
|
||||
activeRequests++;
|
||||
let shouldScheduleNext = false;
|
||||
const processingStartedAt = Date.now();
|
||||
conversationProcessing.set(conversationKey, processingStartedAt);
|
||||
try {
|
||||
const result = (await workerPool.run({
|
||||
conversationKey,
|
||||
messages,
|
||||
})) as AnalysisWorkerResponse;
|
||||
|
||||
for (const row of result.rows) {
|
||||
getModerationBroadcaster()?.messageAnalyzed(row);
|
||||
scheduleAutoDelete(row);
|
||||
}
|
||||
|
||||
if (!result.ok) {
|
||||
consecutiveErrors++;
|
||||
if (consecutiveErrors >= MAX_CONSECUTIVE_ERRORS) {
|
||||
globalCooldownUntil = Date.now() + 60000;
|
||||
logger.warn(
|
||||
"Global circuit breaker triggered due to consecutive errors",
|
||||
);
|
||||
}
|
||||
|
||||
// Batch failed entirely — fall back all messages to individual queue
|
||||
// so no message is permanently lost behind a cooldown.
|
||||
logger.warn(
|
||||
{
|
||||
conversationKey,
|
||||
messageCount: messages.length,
|
||||
error: result.error,
|
||||
},
|
||||
"Batch failed entirely — routing all messages to individual fallback queue",
|
||||
);
|
||||
enqueueIndividualFallbacks(messages);
|
||||
|
||||
lastError = result.error ?? "Analysis worker failed";
|
||||
conversationErrorCooldown.set(
|
||||
conversationKey,
|
||||
Date.now() + config.AI_ANALYSIS_ERROR_COOLDOWN_MS,
|
||||
);
|
||||
logger.error(
|
||||
{
|
||||
conversationKey,
|
||||
error: lastError,
|
||||
messageCount: messages.length,
|
||||
messageIds: messages.map((m) => m.id),
|
||||
cooldownUntil: new Date(
|
||||
Date.now() + config.AI_ANALYSIS_ERROR_COOLDOWN_MS,
|
||||
).toISOString(),
|
||||
timestamp: new Date().toISOString(),
|
||||
},
|
||||
"Batch analysis failed, will retry after cooldown",
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
// Batch succeeded — but check for messages the LLM silently dropped.
|
||||
// Rows with flag "analysis_incomplete" were produced by parseModerationResponse
|
||||
// as synthetic errors; they must be re-processed individually.
|
||||
const incompleteMessages = messages.filter((msg) => {
|
||||
const row = result.rows.find((r) => r.id === msg.id);
|
||||
if (!row) {
|
||||
// The DB update row is missing entirely — treat as incomplete.
|
||||
return true;
|
||||
}
|
||||
const flags: string[] = (() => {
|
||||
try {
|
||||
return JSON.parse(row.ai_moderation_flags ?? "[]") as string[];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
})();
|
||||
return row.ai_status === "error" && flags.includes("analysis_incomplete");
|
||||
});
|
||||
|
||||
if (incompleteMessages.length > 0) {
|
||||
logger.warn(
|
||||
{
|
||||
conversationKey,
|
||||
incompleteCount: incompleteMessages.length,
|
||||
incompleteIds: incompleteMessages.map((m) => m.id),
|
||||
totalBatchSize: messages.length,
|
||||
},
|
||||
"Batch returned incomplete results — fanning out to individual fallback queue",
|
||||
);
|
||||
enqueueIndividualFallbacks(incompleteMessages);
|
||||
}
|
||||
|
||||
consecutiveErrors = 0; // Reset batch circuit breaker
|
||||
conversationErrorCooldown.delete(conversationKey);
|
||||
shouldScheduleNext = true;
|
||||
} catch (error) {
|
||||
consecutiveErrors++;
|
||||
if (consecutiveErrors >= MAX_CONSECUTIVE_ERRORS) {
|
||||
globalCooldownUntil = Date.now() + 60000;
|
||||
logger.warn("Global circuit breaker triggered due to consecutive errors");
|
||||
}
|
||||
|
||||
// Unhandled exception — route everything to individual fallback.
|
||||
logger.warn(
|
||||
{ conversationKey, messageCount: messages.length },
|
||||
"Batch threw exception — routing all messages to individual fallback queue",
|
||||
);
|
||||
enqueueIndividualFallbacks(messages);
|
||||
|
||||
lastError = error instanceof Error ? error.message : String(error);
|
||||
const errorStack = error instanceof Error ? error.stack : undefined;
|
||||
conversationErrorCooldown.set(
|
||||
conversationKey,
|
||||
Date.now() + config.AI_ANALYSIS_ERROR_COOLDOWN_MS,
|
||||
);
|
||||
logger.error(
|
||||
{
|
||||
conversationKey,
|
||||
error: lastError,
|
||||
stack: errorStack,
|
||||
messageCount: messages.length,
|
||||
messageIds: messages.map((m) => m.id),
|
||||
cooldownUntil: new Date(
|
||||
Date.now() + config.AI_ANALYSIS_ERROR_COOLDOWN_MS,
|
||||
).toISOString(),
|
||||
timestamp: new Date().toISOString(),
|
||||
},
|
||||
"Analysis worker failed, will retry after cooldown",
|
||||
);
|
||||
} finally {
|
||||
activeRequests--;
|
||||
if (conversationProcessing.get(conversationKey) === processingStartedAt) {
|
||||
conversationProcessing.delete(conversationKey);
|
||||
}
|
||||
if (shouldScheduleNext) {
|
||||
setImmediate(() => scheduleConversationAnalysis(conversationKey));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Scheduling
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Schedules a debounced analysis run for a conversation.
|
||||
*
|
||||
* FIX #3: The async work inside setTimeout is now wrapped in an explicit
|
||||
* .catch() so DB errors don't produce unhandled promise rejections.
|
||||
* FIX #6: Calls pickBatchWithinBudget after fetching messages so token budget
|
||||
* is respected before handing the batch to the LLM.
|
||||
*/
|
||||
function scheduleConversationAnalysis(conversationKey: string): void {
|
||||
if (isConversationProcessingLocked(conversationKey)) {
|
||||
return;
|
||||
}
|
||||
|
||||
const convoCooldown = conversationErrorCooldown.get(conversationKey) || 0;
|
||||
const activeCooldown = Math.max(convoCooldown, globalCooldownUntil);
|
||||
|
||||
if (activeCooldown && Date.now() < activeCooldown) {
|
||||
if (!conversationDebounceTimers.has(conversationKey)) {
|
||||
const remaining = activeCooldown - Date.now();
|
||||
const timer = setTimeout(() => {
|
||||
conversationDebounceTimers.delete(conversationKey);
|
||||
scheduleConversationAnalysis(conversationKey);
|
||||
}, remaining + 500);
|
||||
conversationDebounceTimers.set(conversationKey, timer);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
const existingTimer = conversationDebounceTimers.get(conversationKey);
|
||||
if (existingTimer) {
|
||||
clearTimeout(existingTimer);
|
||||
}
|
||||
|
||||
const timer = setTimeout(() => {
|
||||
conversationDebounceTimers.delete(conversationKey);
|
||||
|
||||
// FIX #3: explicit .catch() — no async arrow function to avoid unhandled rejection.
|
||||
getPendingMessagesByConversation(
|
||||
conversationKey,
|
||||
config.AI_ANALYSIS_MAX_BATCH_SIZE,
|
||||
)
|
||||
.then(async (messages) => {
|
||||
if (messages.length === 0) return;
|
||||
|
||||
const processableMessages = await skipAgeRestrictedMessages(messages);
|
||||
if (processableMessages.length === 0) return;
|
||||
|
||||
// FIX #6: trim to token budget before sending to LLM.
|
||||
// 50 tokens overhead accounts for JSON structure + id/username fields.
|
||||
let trimmed = pickBatchWithinBudget(
|
||||
processableMessages,
|
||||
config.AI_ANALYSIS_MAX_TARGET_TOKENS,
|
||||
50,
|
||||
);
|
||||
|
||||
// FIX #10: if every message individually exceeds the token budget,
|
||||
// pickBatchWithinBudget returns [] — which would leave them permanently
|
||||
// stuck as `pending`. Fall back to the first message alone so at
|
||||
// least one makes progress; the rest will be processed in later ticks.
|
||||
if (trimmed.length === 0 && processableMessages.length > 0) {
|
||||
trimmed = processableMessages.slice(0, 1);
|
||||
logger.warn(
|
||||
{
|
||||
conversationKey,
|
||||
messageId: processableMessages[0]?.id,
|
||||
tokenBudget: config.AI_ANALYSIS_MAX_TARGET_TOKENS,
|
||||
},
|
||||
"All messages exceed token budget — processing first message alone to avoid stuck-pending deadlock",
|
||||
);
|
||||
}
|
||||
|
||||
return processBatch(conversationKey, trimmed);
|
||||
})
|
||||
.catch((err: unknown) => {
|
||||
logger.error(
|
||||
{
|
||||
conversationKey,
|
||||
error: err instanceof Error ? err.message : String(err),
|
||||
},
|
||||
"Failed to fetch or dispatch pending messages for scheduled analysis",
|
||||
);
|
||||
});
|
||||
}, config.AI_ANALYSIS_DEBOUNCE_MS);
|
||||
|
||||
conversationDebounceTimers.set(conversationKey, timer);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Public API
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Queues a message for analysis (debounced by conversation).
|
||||
*/
|
||||
export async function queueMessageAnalysis(messageId: string): Promise<void> {
|
||||
if (!config.AI_ANALYSIS_ENABLED) return;
|
||||
|
||||
try {
|
||||
const message = await getMessageById(messageId);
|
||||
if (!message) {
|
||||
logger.warn({ messageId }, "Message not found for analysis queue");
|
||||
return;
|
||||
}
|
||||
|
||||
if (isAgeRestrictedMessage(message)) {
|
||||
const updated = await updateMessageAIAnalysis(
|
||||
message.id,
|
||||
buildAgeRestrictedSkipResult(),
|
||||
);
|
||||
if (updated) {
|
||||
getModerationBroadcaster()?.messageAnalyzed(updated);
|
||||
}
|
||||
logger.info(
|
||||
{ messageId },
|
||||
"Skipped AI analysis for age-restricted message",
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
queueConversationAnalysis(getConversationKey(message));
|
||||
} catch (error) {
|
||||
logger.error(
|
||||
{
|
||||
messageId,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
},
|
||||
"Failed to queue message for analysis",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Queues a conversation for analysis (debounced).
|
||||
*/
|
||||
export function queueConversationAnalysis(conversationKey: string): void {
|
||||
if (!config.AI_ANALYSIS_ENABLED) return;
|
||||
scheduleConversationAnalysis(conversationKey);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns current status of both the batch and individual fallback queues.
|
||||
*/
|
||||
export function getAnalysisQueueStatus(): AnalysisQueueStatus {
|
||||
return {
|
||||
queuedConversations: conversationDebounceTimers.size,
|
||||
activeRequests,
|
||||
activeIndividualRequests,
|
||||
individualInFlightCount: individualInFlight.size,
|
||||
individualCircuitBreakerActive: Date.now() < individualCooldownUntil,
|
||||
lastError,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Starts the periodic recovery worker.
|
||||
*
|
||||
* FIX #4: Now also recovers messages stuck in `error/analysis_incomplete`
|
||||
* state (not just `pending`), and skips conversations that already have
|
||||
* individual fallback work in progress to avoid DB last-write-wins races.
|
||||
*/
|
||||
export function startPendingAIAnalysisWorker(client?: Client): void {
|
||||
moderationClient = client;
|
||||
if (!config.AI_ANALYSIS_ENABLED) return;
|
||||
|
||||
setInterval(() => {
|
||||
// FIX #3 pattern: no async arrow — chain promises explicitly.
|
||||
Promise.all([
|
||||
getPendingConversationKeys(500),
|
||||
getConversationKeysWithIncompleteAnalysis(200),
|
||||
])
|
||||
.then(([pendingKeys, incompleteKeys]) => {
|
||||
const now = Date.now();
|
||||
|
||||
// FIX #9: Prune stale entries from state maps to prevent unbounded
|
||||
// memory growth from channels/threads that are no longer active.
|
||||
for (const [key, expiry] of conversationErrorCooldown) {
|
||||
if (now >= expiry) conversationErrorCooldown.delete(key);
|
||||
}
|
||||
for (const [key, startedAt] of conversationProcessing) {
|
||||
if (now - startedAt >= config.AI_ANALYSIS_PROCESSING_TIMEOUT_MS) {
|
||||
conversationProcessing.delete(key);
|
||||
}
|
||||
}
|
||||
|
||||
// FIX #8: Build a set of keys already targeted for individual recovery
|
||||
// so the batch loop below skips them, preventing a race where batch
|
||||
// scheduling and individual scheduling collide on the same conversation.
|
||||
const incompleteKeySet = new Set(incompleteKeys);
|
||||
|
||||
// --- Batch recovery for `pending` messages ---
|
||||
for (const key of pendingKeys) {
|
||||
if (conversationDebounceTimers.has(key)) continue;
|
||||
if (isConversationProcessingLocked(key)) continue;
|
||||
// FIX #4: skip if individual fallback already running for this conversation.
|
||||
if (individualInFlightByConversation.has(key)) continue;
|
||||
// FIX #8: skip if this conversation also needs individual recovery
|
||||
// (batch processing would conflict with in-flight individual work).
|
||||
if (incompleteKeySet.has(key)) continue;
|
||||
const cooldownUntil = conversationErrorCooldown.get(key);
|
||||
if (cooldownUntil && now < cooldownUntil) continue;
|
||||
scheduleConversationAnalysis(key);
|
||||
}
|
||||
|
||||
// --- Individual recovery for `error/analysis_incomplete` messages ---
|
||||
// Circuit breaker check: no point iterating if individual CB is active.
|
||||
if (now >= individualCooldownUntil) {
|
||||
const promises: Promise<void>[] = [];
|
||||
for (const key of incompleteKeys) {
|
||||
// Skip if individual work is already running for this conversation.
|
||||
if (individualInFlightByConversation.has(key)) continue;
|
||||
// Skip if batch processing is running (it will fan-out if it finds more incomplete).
|
||||
if (isConversationProcessingLocked(key)) continue;
|
||||
|
||||
promises.push(
|
||||
getIncompleteMessagesByConversation(key, 500)
|
||||
.then(async (msgs) => {
|
||||
const processableMessages =
|
||||
await skipAgeRestrictedMessages(msgs);
|
||||
return processableMessages;
|
||||
})
|
||||
.then((msgs) => {
|
||||
if (msgs.length > 0) {
|
||||
enqueueIndividualFallbacks(msgs);
|
||||
}
|
||||
})
|
||||
.catch((err: unknown) => {
|
||||
logger.error(
|
||||
{ key, error: String(err) },
|
||||
"Failed to fetch incomplete messages for recovery",
|
||||
);
|
||||
}),
|
||||
);
|
||||
}
|
||||
// Errors are handled per-key; return the combined promise for observability.
|
||||
return Promise.all(promises);
|
||||
}
|
||||
})
|
||||
.catch((err: unknown) => {
|
||||
logger.error(
|
||||
{ error: err instanceof Error ? err.message : String(err) },
|
||||
"Pending AI analysis recovery worker failed",
|
||||
);
|
||||
});
|
||||
}, config.AI_ANALYSIS_RECOVERY_INTERVAL_MS);
|
||||
}
|
||||
@@ -0,0 +1,355 @@
|
||||
import type { Client, PermissionString } from "discord.js-selfbot-v13";
|
||||
import { config } from "../../shared/config/config.js";
|
||||
import { createChildLogger } from "../../shared/logger/logger.js";
|
||||
import { createModerationAction } from "../message-capture/messageStore.js";
|
||||
import type { MessageRecord } from "../message-capture/types.js";
|
||||
|
||||
const logger = createChildLogger("auto-delete-manager");
|
||||
|
||||
const parseStringList = (value?: string | null): string[] => {
|
||||
if (!value) return [];
|
||||
try {
|
||||
const parsed = JSON.parse(value) as unknown;
|
||||
return Array.isArray(parsed)
|
||||
? parsed.filter((item): item is string => typeof item === "string")
|
||||
: [];
|
||||
} catch {
|
||||
return value
|
||||
.split(",")
|
||||
.map((item) => item.trim())
|
||||
.filter(Boolean);
|
||||
}
|
||||
};
|
||||
|
||||
/** Derive severity from legacy messages that lack structured AI fields. */
|
||||
function deriveSeverity(msg: MessageRecord): string {
|
||||
if (msg.ai_severity) return msg.ai_severity;
|
||||
const score = msg.ai_confidence ?? msg.ai_moderation_score ?? 0;
|
||||
if (msg.ai_status === "flagged")
|
||||
return score >= 0.9 ? "critical" : score >= 0.7 ? "high" : "medium";
|
||||
if (msg.ai_status === "warn") return score >= 0.6 ? "medium" : "low";
|
||||
return "none";
|
||||
}
|
||||
|
||||
/** Derive recommended action from legacy messages that lack structured AI fields. */
|
||||
function deriveRecommendedAction(msg: MessageRecord): string {
|
||||
if (msg.ai_recommended_action) return msg.ai_recommended_action;
|
||||
const severity = deriveSeverity(msg);
|
||||
if (
|
||||
msg.ai_status === "flagged" &&
|
||||
(severity === "critical" || severity === "high")
|
||||
)
|
||||
return "delete";
|
||||
if (msg.ai_status === "flagged") return "review";
|
||||
if (msg.ai_status === "warn") return "warn";
|
||||
return "none";
|
||||
}
|
||||
|
||||
function isAutoDeleteEligible(message: MessageRecord): boolean {
|
||||
if (message.ai_status !== "flagged" && message.ai_status !== "warn")
|
||||
return false;
|
||||
|
||||
const confidence = message.ai_confidence ?? message.ai_moderation_score ?? 0;
|
||||
if (confidence < config.AUTO_DELETE_MIN_CONFIDENCE) {
|
||||
logger.info(
|
||||
{
|
||||
messageId: message.id,
|
||||
confidence,
|
||||
threshold: config.AUTO_DELETE_MIN_CONFIDENCE,
|
||||
},
|
||||
"Auto-delete skipped: confidence below threshold",
|
||||
);
|
||||
return false;
|
||||
}
|
||||
|
||||
const severity = deriveSeverity(message);
|
||||
const allowedSeverities = (config.AUTO_DELETE_ALLOWED_SEVERITIES || "")
|
||||
.split(",")
|
||||
.map((s) => s.trim())
|
||||
.filter(Boolean);
|
||||
if (allowedSeverities.length > 0 && !allowedSeverities.includes(severity)) {
|
||||
logger.info(
|
||||
{ messageId: message.id, severity, allowed: allowedSeverities },
|
||||
"Auto-delete skipped: severity not in allowed list",
|
||||
);
|
||||
return false;
|
||||
}
|
||||
|
||||
const recommendedAction = deriveRecommendedAction(message);
|
||||
if (recommendedAction !== "delete" && recommendedAction !== "escalate") {
|
||||
logger.info(
|
||||
{ messageId: message.id, recommendedAction },
|
||||
"Auto-delete skipped: recommended action is not delete/escalate",
|
||||
);
|
||||
return false;
|
||||
}
|
||||
|
||||
const allowedCategories = parseStringList(
|
||||
config.AUTO_DELETE_ALLOWED_CATEGORIES,
|
||||
);
|
||||
if (allowedCategories.length > 0) {
|
||||
const messageCategories = parseStringList(
|
||||
message.ai_categories ?? message.ai_moderation_flags,
|
||||
);
|
||||
const hasAllowedCategory = messageCategories.some((cat) =>
|
||||
allowedCategories.includes(cat),
|
||||
);
|
||||
if (!hasAllowedCategory) {
|
||||
logger.info(
|
||||
{
|
||||
messageId: message.id,
|
||||
categories: messageCategories,
|
||||
allowed: allowedCategories,
|
||||
},
|
||||
"Auto-delete skipped: no allowed categories match",
|
||||
);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
const excludedChannels = parseStringList(
|
||||
config.AUTO_DELETE_EXCLUDED_CHANNEL_IDS,
|
||||
);
|
||||
if (excludedChannels.length > 0) {
|
||||
const channelId = message.thread_id ?? message.channel_id;
|
||||
if (excludedChannels.includes(channelId)) {
|
||||
logger.info(
|
||||
{ messageId: message.id, channelId },
|
||||
"Auto-delete skipped: channel excluded",
|
||||
);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
const excludedUsers = parseStringList(config.AUTO_DELETE_EXCLUDED_USER_IDS);
|
||||
if (excludedUsers.length > 0 && excludedUsers.includes(message.user_id)) {
|
||||
logger.info(
|
||||
{ messageId: message.id, userId: message.user_id },
|
||||
"Auto-delete skipped: user excluded",
|
||||
);
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
async function logAutoDeleteAttempt(
|
||||
message: MessageRecord,
|
||||
result: AutoDeleteResult,
|
||||
): Promise<void> {
|
||||
try {
|
||||
await createModerationAction({
|
||||
message_id: message.id,
|
||||
user_id: message.user_id,
|
||||
guild_id: message.guild_id,
|
||||
action_type: "delete_message",
|
||||
reason: result.reason,
|
||||
executed_by: "auto-delete-manager",
|
||||
status: result.deleted
|
||||
? "executed"
|
||||
: result.reason === "dry_run"
|
||||
? "executed"
|
||||
: "failed",
|
||||
error: result.reason === "error" ? result.reason : null,
|
||||
executed_at:
|
||||
result.deleted || result.reason === "dry_run" ? Date.now() : null,
|
||||
});
|
||||
} catch (error) {
|
||||
logger.warn(
|
||||
{
|
||||
messageId: message.id,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
},
|
||||
"Failed to persist auto-delete action log",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export interface AutoDeleteResult {
|
||||
deleted: boolean;
|
||||
skipped: boolean;
|
||||
reason: string;
|
||||
}
|
||||
|
||||
function getErrorCode(error: unknown): number | string | undefined {
|
||||
if (!error || typeof error !== "object") return undefined;
|
||||
const maybeCode = (error as { code?: number | string }).code;
|
||||
const maybeStatus = (error as { status?: number | string }).status;
|
||||
return maybeCode ?? maybeStatus;
|
||||
}
|
||||
|
||||
function isAlreadyDeletedError(error: unknown): boolean {
|
||||
const code = getErrorCode(error);
|
||||
return code === 10008 || code === 404 || code === "10008" || code === "404";
|
||||
}
|
||||
|
||||
function hasChannelMessagesApi(channel: unknown): channel is {
|
||||
messages: {
|
||||
fetch: (id: string) => Promise<{ delete: () => Promise<unknown> }>;
|
||||
};
|
||||
} {
|
||||
return Boolean(
|
||||
channel &&
|
||||
typeof channel === "object" &&
|
||||
"messages" in channel &&
|
||||
(channel as { messages?: unknown }).messages &&
|
||||
typeof (channel as { messages: { fetch?: unknown } }).messages.fetch ===
|
||||
"function",
|
||||
);
|
||||
}
|
||||
|
||||
function hasPermissionApi(channel: unknown): channel is {
|
||||
permissionsFor: (
|
||||
member: unknown,
|
||||
) => { has: (permission: string) => boolean } | null;
|
||||
} {
|
||||
return Boolean(
|
||||
channel &&
|
||||
typeof channel === "object" &&
|
||||
"permissionsFor" in channel &&
|
||||
typeof (channel as { permissionsFor?: unknown }).permissionsFor ===
|
||||
"function",
|
||||
);
|
||||
}
|
||||
|
||||
export async function attemptAutoDeleteFlaggedMessage(
|
||||
client: Client | undefined,
|
||||
message: MessageRecord,
|
||||
): Promise<AutoDeleteResult> {
|
||||
if (!config.AUTO_DELETE_FLAGGED_ENABLED) {
|
||||
return { deleted: false, skipped: true, reason: "disabled" };
|
||||
}
|
||||
|
||||
if (message.ai_status !== "flagged" && message.ai_status !== "warn") {
|
||||
const result = {
|
||||
deleted: false,
|
||||
skipped: true,
|
||||
reason: "not_flagged_or_warn",
|
||||
} as AutoDeleteResult;
|
||||
await logAutoDeleteAttempt(message, result);
|
||||
return result;
|
||||
}
|
||||
|
||||
if (!isAutoDeleteEligible(message)) {
|
||||
const result = {
|
||||
deleted: false,
|
||||
skipped: true,
|
||||
reason: "not_eligible",
|
||||
} as AutoDeleteResult;
|
||||
await logAutoDeleteAttempt(message, result);
|
||||
return result;
|
||||
}
|
||||
|
||||
if (!client?.user?.id) {
|
||||
logger.warn(
|
||||
{ messageId: message.id },
|
||||
"Auto-delete skipped: client user missing",
|
||||
);
|
||||
return { deleted: false, skipped: true, reason: "client_user_missing" };
|
||||
}
|
||||
|
||||
try {
|
||||
const guild = client.guilds.cache.get(message.guild_id);
|
||||
if (!guild) {
|
||||
logger.warn(
|
||||
{ messageId: message.id, guildId: message.guild_id },
|
||||
"Auto-delete skipped: guild not found",
|
||||
);
|
||||
return { deleted: false, skipped: true, reason: "guild_not_found" };
|
||||
}
|
||||
|
||||
const channelId = message.thread_id ?? message.channel_id;
|
||||
const channel = guild.channels.cache.get(channelId);
|
||||
if (!channel) {
|
||||
logger.warn(
|
||||
{ messageId: message.id, channelId },
|
||||
"Auto-delete skipped: channel not found",
|
||||
);
|
||||
return { deleted: false, skipped: true, reason: "channel_not_found" };
|
||||
}
|
||||
|
||||
if (!hasPermissionApi(channel) || !hasChannelMessagesApi(channel)) {
|
||||
logger.warn(
|
||||
{ messageId: message.id, channelId },
|
||||
"Auto-delete skipped: channel cannot delete messages",
|
||||
);
|
||||
return { deleted: false, skipped: true, reason: "unsupported_channel" };
|
||||
}
|
||||
|
||||
const selfMember = await guild.members.fetch(client.user.id);
|
||||
const permissions = channel.permissionsFor(selfMember);
|
||||
const canManageMessages =
|
||||
permissions?.has("MANAGE_MESSAGES" as PermissionString) ?? false;
|
||||
|
||||
if (!canManageMessages) {
|
||||
logger.warn(
|
||||
{ messageId: message.id, channelId, userId: client.user.id },
|
||||
"Auto-delete skipped: current user lacks Manage Messages",
|
||||
);
|
||||
return {
|
||||
deleted: false,
|
||||
skipped: true,
|
||||
reason: "missing_manage_messages",
|
||||
};
|
||||
}
|
||||
|
||||
if (config.AUTO_DELETE_FLAGGED_DRY_RUN) {
|
||||
const result = {
|
||||
deleted: false,
|
||||
skipped: true,
|
||||
reason: "dry_run",
|
||||
} as AutoDeleteResult;
|
||||
await logAutoDeleteAttempt(message, result);
|
||||
logger.info(
|
||||
{ messageId: message.id, channelId },
|
||||
"Auto-delete dry-run: would delete flagged message",
|
||||
);
|
||||
return result;
|
||||
}
|
||||
|
||||
const discordMessage = await channel.messages.fetch(message.id);
|
||||
await discordMessage.delete();
|
||||
|
||||
const result = {
|
||||
deleted: true,
|
||||
skipped: false,
|
||||
reason: "deleted",
|
||||
} as AutoDeleteResult;
|
||||
await logAutoDeleteAttempt(message, result);
|
||||
logger.info(
|
||||
{ messageId: message.id, channelId },
|
||||
"Auto-deleted AI-flagged message",
|
||||
);
|
||||
return result;
|
||||
} catch (error) {
|
||||
if (isAlreadyDeletedError(error)) {
|
||||
const result = {
|
||||
deleted: true,
|
||||
skipped: false,
|
||||
reason: "already_deleted",
|
||||
} as AutoDeleteResult;
|
||||
await logAutoDeleteAttempt(message, result);
|
||||
logger.info(
|
||||
{ messageId: message.id, code: getErrorCode(error) },
|
||||
"Auto-delete skipped: message already deleted",
|
||||
);
|
||||
return result;
|
||||
}
|
||||
|
||||
const result = {
|
||||
deleted: false,
|
||||
skipped: true,
|
||||
reason: "error",
|
||||
} as AutoDeleteResult;
|
||||
await logAutoDeleteAttempt(message, result);
|
||||
logger.error(
|
||||
{
|
||||
messageId: message.id,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
code: getErrorCode(error),
|
||||
},
|
||||
"Auto-delete failed",
|
||||
);
|
||||
return result;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
import pLimit from "p-limit";
|
||||
import { config } from "../../shared/config/config.js";
|
||||
|
||||
/**
|
||||
* Concurrency limiter for LLM API calls.
|
||||
*
|
||||
* Prevents rate-limit (429) errors by capping simultaneous requests
|
||||
* to the configured maximum (default: 5).
|
||||
*/
|
||||
const llmSemaphore = pLimit(config.AI_LLM_MAX_CONCURRENT ?? 5);
|
||||
|
||||
export async function withLlmConcurrency<T>(fn: () => Promise<T>): Promise<T> {
|
||||
return llmSemaphore(fn);
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
import { formatModerationTextEvidenceForPrompt } from "./indonesianTextNormalizer.js";
|
||||
import { formatMediaEvidenceForPrompt } from "../message-capture/messageMetadata.js";
|
||||
import type { MessageRecord } from "../message-capture/types.js";
|
||||
|
||||
export interface ConversationContextInput {
|
||||
contextBefore: MessageRecord[];
|
||||
targets: MessageRecord[];
|
||||
maxTokens: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Formats a timestamp to ISO 8601 string
|
||||
*/
|
||||
function formatTimestamp(ms: number): string {
|
||||
return new Date(ms).toISOString();
|
||||
}
|
||||
|
||||
/**
|
||||
* Estimates token count for a string (pessimistic approximation for Indonesian slang & JSON overhead)
|
||||
*/
|
||||
export function estimateTokens(text: string): number {
|
||||
return Math.ceil(text.length / 3) + 15;
|
||||
}
|
||||
|
||||
/**
|
||||
* Formats a single message for context or target display
|
||||
*/
|
||||
export async function formatMessageForPrompt(
|
||||
msg: MessageRecord,
|
||||
label: "context" | "target",
|
||||
): Promise<string> {
|
||||
const content = msg.edited_content ?? msg.content;
|
||||
const timestamp = formatTimestamp(msg.created_at);
|
||||
const textEvidence = await formatModerationTextEvidenceForPrompt(content);
|
||||
const textSuffix = textEvidence ? ` ${textEvidence}` : "";
|
||||
const mediaEvidence = formatMediaEvidenceForPrompt(msg.metadata);
|
||||
const mediaSuffix = mediaEvidence ? ` ${mediaEvidence}` : "";
|
||||
return `[${label}] id=${msg.id} time=${timestamp} user=${msg.username}: ${content}${textSuffix}${mediaSuffix}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds conversation historical context without including targets.
|
||||
* Calculates how much token budget targets use, and fills the rest with context.
|
||||
*/
|
||||
export async function buildConversationContext(
|
||||
input: ConversationContextInput,
|
||||
): Promise<string[]> {
|
||||
const { contextBefore, targets, maxTokens } = input;
|
||||
|
||||
// Calculate tokens used by targets (parallel)
|
||||
const targetLines = await Promise.all(
|
||||
targets.map((msg) => formatMessageForPrompt(msg, "target")),
|
||||
);
|
||||
let usedTokens = targetLines.reduce(
|
||||
(sum, line) => sum + estimateTokens(line),
|
||||
0,
|
||||
);
|
||||
|
||||
const contextLines = await Promise.all(
|
||||
contextBefore.map((msg) => formatMessageForPrompt(msg, "context")),
|
||||
);
|
||||
const selectedContextLines: string[] = [];
|
||||
|
||||
// Go backwards through context, taking most recent first
|
||||
for (let i = contextLines.length - 1; i >= 0; i--) {
|
||||
const line = contextLines[i];
|
||||
const lineTokens = estimateTokens(line);
|
||||
|
||||
if (usedTokens + lineTokens <= maxTokens) {
|
||||
// Unshift so oldest context is first in the array
|
||||
selectedContextLines.unshift(line);
|
||||
usedTokens += lineTokens;
|
||||
}
|
||||
}
|
||||
|
||||
return selectedContextLines;
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
export { startPendingAIAnalysisWorker } from "./aiAnalyzer.js";
|
||||
export {
|
||||
normalizeDiscordCustomEmoji,
|
||||
detectIndonesianBadwords,
|
||||
buildModerationTextEvidence,
|
||||
} from "./indonesianTextNormalizer.js";
|
||||
export { runModerationAnalysis } from "./llmModerationClient.js";
|
||||
export { buildSystemPrompt } from "./moderationPrompt.js";
|
||||
@@ -0,0 +1,606 @@
|
||||
import axios from "axios";
|
||||
import OpenAI from "openai";
|
||||
import { config } from "../../shared/config/config.js";
|
||||
import { createChildLogger } from "../../shared/logger/logger.js";
|
||||
import { retryWithBackoff } from "../../shared/utils/retry.js";
|
||||
import { getCachedText, upsertCachedText } from "./textCacheStore.js";
|
||||
|
||||
const log = createChildLogger("indonesianTextNormalizer");
|
||||
|
||||
const CUSTOM_EMOJI_PATTERN = /<a?:([a-zA-Z0-9_]+):(\d+)>/g;
|
||||
|
||||
/** NVIDIA content safety categories that map to offensive/badword content. */
|
||||
const NVIDIA_BAD_CATEGORIES = new Set([
|
||||
"hate",
|
||||
"harassment",
|
||||
"sexual",
|
||||
"violence",
|
||||
"self-harm",
|
||||
"illicit",
|
||||
"profanity",
|
||||
"vulgar",
|
||||
"insult",
|
||||
]);
|
||||
|
||||
/**
|
||||
* Map NVIDIA Nemotron category labels to Indonesian badword-style labels.
|
||||
*/
|
||||
const CATEGORY_TO_BADWORD_LABEL: Record<string, string> = {
|
||||
hate: "hate_speech",
|
||||
harassment: "harassment",
|
||||
sexual: "sexual_content",
|
||||
violence: "violence",
|
||||
"self-harm": "self_harm",
|
||||
illicit: "illegal_content",
|
||||
profanity: "vulgar_language",
|
||||
vulgar: "vulgar_language",
|
||||
insult: "harassment",
|
||||
};
|
||||
|
||||
const VALID_PRIMARY_AI_FLAGS = new Set([
|
||||
"spam",
|
||||
"hate_speech",
|
||||
"sara",
|
||||
"hoaks",
|
||||
"harassment",
|
||||
"vulgar_language",
|
||||
"sexual_content",
|
||||
"sexual_deviation",
|
||||
"violence",
|
||||
"self_harm",
|
||||
"doxxing",
|
||||
"scam",
|
||||
"misinformation",
|
||||
"nsfw_image",
|
||||
"gore_image",
|
||||
"illegal_content",
|
||||
"gambling",
|
||||
"drugs",
|
||||
"child_safety",
|
||||
"financial_scam",
|
||||
"religious_insult",
|
||||
"self_promo",
|
||||
]);
|
||||
|
||||
/**
|
||||
* In-memory cache TTL (10 min) — fastest path for repeated identical texts.
|
||||
*/
|
||||
const BADWORD_CACHE_TTL_MS = 10 * 60 * 1000;
|
||||
|
||||
/**
|
||||
* DB cache TTL (24 hours) — survives restarts, stores full-text results
|
||||
* so context is preserved (e.g. "kaus" is clean, "kau" alone is clean,
|
||||
* but "awas kau" is harassment).
|
||||
*/
|
||||
const DB_CACHE_TTL_MS = 24 * 60 * 60 * 1000;
|
||||
|
||||
const NEMOTRON_RATE_LIMIT_COOLDOWN_MS = 60 * 1000;
|
||||
const PRIMARY_AI_RATE_LIMIT_COOLDOWN_MS = 30_000;
|
||||
const GROQ_RATE_LIMIT_COOLDOWN_MS = 60 * 1000;
|
||||
|
||||
interface BadwordCacheEntry {
|
||||
value: string[];
|
||||
expiresAt: number;
|
||||
}
|
||||
|
||||
const badwordCache = new Map<string, BadwordCacheEntry>();
|
||||
const inFlightBadwordLookups = new Map<string, Promise<string[]>>();
|
||||
let nemotronUnavailableUntil = 0;
|
||||
let primaryAiUnavailableUntil = 0;
|
||||
let groqUnavailableUntil = 0;
|
||||
let primaryModerationClient: OpenAI | null = null;
|
||||
|
||||
export interface ModerationTextEvidence {
|
||||
raw: string;
|
||||
normalized: string;
|
||||
notes: string[];
|
||||
badwords: string[];
|
||||
hasBadwords: boolean;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Sync helpers (unchanged)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export function normalizeDiscordCustomEmoji(text: string): {
|
||||
text: string;
|
||||
emojiNames: string[];
|
||||
} {
|
||||
const emojiNames: string[] = [];
|
||||
const normalized = text.replace(
|
||||
CUSTOM_EMOJI_PATTERN,
|
||||
(_match, name: string) => {
|
||||
emojiNames.push(name);
|
||||
return `[emoji:${name}]`;
|
||||
},
|
||||
);
|
||||
|
||||
return { text: normalized, emojiNames };
|
||||
}
|
||||
|
||||
// Local badword detection removed (lines 121-198).
|
||||
// All detection now goes through the API pipeline (NVIDIA → Primary AI → Groq)
|
||||
// to eliminate false positives from substring matching and hardcoded whitelists.
|
||||
|
||||
function normalizeBadwordCacheKey(text: string): string {
|
||||
return text.trim().replace(/\s+/g, " ").toLowerCase();
|
||||
}
|
||||
|
||||
function getCachedBadwords(key: string): string[] | null {
|
||||
const entry = badwordCache.get(key);
|
||||
if (!entry) return null;
|
||||
if (entry.expiresAt <= Date.now()) {
|
||||
badwordCache.delete(key);
|
||||
return null;
|
||||
}
|
||||
return [...entry.value];
|
||||
}
|
||||
|
||||
function setCachedBadwords(key: string, value: string[]): void {
|
||||
badwordCache.set(key, {
|
||||
value: [...new Set(value)],
|
||||
expiresAt: Date.now() + BADWORD_CACHE_TTL_MS,
|
||||
});
|
||||
|
||||
if (badwordCache.size > 500) {
|
||||
const now = Date.now();
|
||||
for (const [cacheKey, entry] of badwordCache) {
|
||||
if (entry.expiresAt <= now) {
|
||||
badwordCache.delete(cacheKey);
|
||||
}
|
||||
}
|
||||
|
||||
if (badwordCache.size > 500) {
|
||||
const oldestKeys = Array.from(badwordCache.entries())
|
||||
.sort((a, b) => a[1].expiresAt - b[1].expiresAt)
|
||||
.slice(0, badwordCache.size - 500)
|
||||
.map(([cacheKey]) => cacheKey);
|
||||
for (const cacheKey of oldestKeys) {
|
||||
badwordCache.delete(cacheKey);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function getPrimaryModerationClient(): OpenAI | null {
|
||||
if (!config.AI_LLM_API_KEY) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (!primaryModerationClient) {
|
||||
primaryModerationClient = new OpenAI({
|
||||
apiKey: config.AI_LLM_API_KEY,
|
||||
baseURL: config.AI_LLM_BASE_URL,
|
||||
maxRetries: 0,
|
||||
timeout: 15000,
|
||||
});
|
||||
}
|
||||
|
||||
return primaryModerationClient;
|
||||
}
|
||||
|
||||
function normalizePrimaryAiFlag(value: string): string | null {
|
||||
const lower = value
|
||||
.trim()
|
||||
.toLowerCase()
|
||||
.replace(/[\s-]+/g, "_");
|
||||
if (!lower) return null;
|
||||
|
||||
if (VALID_PRIMARY_AI_FLAGS.has(lower)) {
|
||||
return lower;
|
||||
}
|
||||
|
||||
return CATEGORY_TO_BADWORD_LABEL[lower] ?? null;
|
||||
}
|
||||
|
||||
function extractFlagsFromPrimaryAiContent(content: string): string[] {
|
||||
const flags = new Set<string>();
|
||||
let parsed: unknown;
|
||||
|
||||
try {
|
||||
parsed = JSON.parse(content);
|
||||
} catch {
|
||||
parsed = null;
|
||||
}
|
||||
|
||||
const addValue = (value: unknown) => {
|
||||
if (typeof value !== "string") return;
|
||||
const normalized = normalizePrimaryAiFlag(value);
|
||||
if (normalized) flags.add(normalized);
|
||||
};
|
||||
|
||||
if (Array.isArray(parsed)) {
|
||||
for (const item of parsed) {
|
||||
addValue(item);
|
||||
}
|
||||
} else if (parsed && typeof parsed === "object") {
|
||||
const candidate = parsed as Record<string, unknown>;
|
||||
for (const key of ["flags", "categories", "badwords"]) {
|
||||
const value = candidate[key];
|
||||
if (Array.isArray(value)) {
|
||||
for (const item of value) addValue(item);
|
||||
} else {
|
||||
addValue(value);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (flags.size > 0) {
|
||||
return Array.from(flags);
|
||||
}
|
||||
|
||||
const lowerContent = content.toLowerCase();
|
||||
for (const flag of VALID_PRIMARY_AI_FLAGS) {
|
||||
if (lowerContent.includes(flag)) {
|
||||
flags.add(flag);
|
||||
}
|
||||
}
|
||||
|
||||
for (const category of Object.keys(CATEGORY_TO_BADWORD_LABEL)) {
|
||||
if (lowerContent.includes(category)) {
|
||||
const mapped = CATEGORY_TO_BADWORD_LABEL[category];
|
||||
if (mapped) flags.add(mapped);
|
||||
}
|
||||
}
|
||||
|
||||
return Array.from(flags);
|
||||
}
|
||||
|
||||
async function callPrimaryAiModeration(text: string): Promise<string[]> {
|
||||
const client = getPrimaryModerationClient();
|
||||
if (!client) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const completion = await retryWithBackoff(
|
||||
async () => {
|
||||
return client.chat.completions.create({
|
||||
model: config.AI_LLM_MODEL,
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
content:
|
||||
"Deteksi kata kasar / pelanggaran ringan dari teks Indonesia berikut. " +
|
||||
'Balas hanya JSON object dengan format {"flags":[...]} dan gunakan hanya flag valid ini: ' +
|
||||
Array.from(VALID_PRIMARY_AI_FLAGS).join(", ") +
|
||||
". Jika tidak ada pelanggaran, flags harus array kosong. Teks: " +
|
||||
text,
|
||||
},
|
||||
],
|
||||
temperature: 0.1,
|
||||
top_p: 0.9,
|
||||
max_tokens: 200,
|
||||
stream: false,
|
||||
response_format: { type: "json_object" },
|
||||
} as OpenAI.Chat.Completions.ChatCompletionCreateParamsNonStreaming);
|
||||
},
|
||||
{
|
||||
retries: 1,
|
||||
minTimeout: 500,
|
||||
maxTimeout: 2000,
|
||||
factor: 2,
|
||||
logger: log,
|
||||
},
|
||||
);
|
||||
|
||||
const content = completion.choices[0]?.message?.content?.trim();
|
||||
if (!content) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return extractFlagsFromPrimaryAiContent(content);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Groq Llama Prompt Guard Moderation API (Fallback)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Call Groq Llama Prompt Guard 2-86M model for moderation scoring.
|
||||
* Returns a probability score as a string (e.g. "0.9988824725151062").
|
||||
* Scores above ~0.5 indicate moderation violations.
|
||||
*/
|
||||
async function callGrokModeration(text: string): Promise<string[]> {
|
||||
const apiKey = config.GROQ_API_KEY;
|
||||
if (!apiKey) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const response = await axios.post(
|
||||
config.GROQ_MODERATION_BASE_URL,
|
||||
{
|
||||
model: config.GROQ_MODERATION_MODEL,
|
||||
messages: [{ role: "user", content: text }],
|
||||
},
|
||||
{
|
||||
headers: {
|
||||
Authorization: `Bearer ${apiKey}`,
|
||||
Accept: "application/json",
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
timeout: 10_000,
|
||||
},
|
||||
);
|
||||
|
||||
const scoreStr = response.data?.choices?.[0]?.message?.content?.trim();
|
||||
if (!scoreStr) {
|
||||
return [];
|
||||
}
|
||||
|
||||
// Parse the score (Llama Prompt Guard returns a single probability score)
|
||||
const score = parseFloat(scoreStr);
|
||||
if (isNaN(score) || score < 0.5) {
|
||||
return [];
|
||||
}
|
||||
|
||||
// Map score to moderation flags based on severity
|
||||
const flags: string[] = [];
|
||||
if (score >= 0.9) {
|
||||
flags.push("vulgar_language", "harassment");
|
||||
} else if (score >= 0.7) {
|
||||
flags.push("vulgar_language");
|
||||
} else {
|
||||
flags.push("spam");
|
||||
}
|
||||
|
||||
return flags;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// NVIDIA Nemotron-3 Content Safety API
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Call NVIDIA Nemotron-3 Content Safety API to detect harmful content.
|
||||
* Returns categories/flags from the API response.
|
||||
*/
|
||||
async function callNemotronContentSafety(text: string): Promise<string[]> {
|
||||
const apiKey = config.NVIDIA_NEMOTRON_API_KEY;
|
||||
if (!apiKey) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const response = await axios.post(
|
||||
config.NVIDIA_NEMOTRON_BASE_URL,
|
||||
{
|
||||
model: config.NVIDIA_NEMOTRON_MODEL,
|
||||
messages: [{ role: "user", content: text }],
|
||||
max_tokens: 897,
|
||||
temperature: 0.2,
|
||||
top_p: 0.7,
|
||||
stream: false,
|
||||
},
|
||||
{
|
||||
headers: {
|
||||
Authorization: `Bearer ${apiKey}`,
|
||||
Accept: "application/json",
|
||||
},
|
||||
timeout: 15_000,
|
||||
},
|
||||
);
|
||||
|
||||
const data = response.data;
|
||||
const categories: string[] = [];
|
||||
|
||||
// Parse the LLM response for category flags
|
||||
const content = data?.choices?.[0]?.message?.content ?? "";
|
||||
if (content) {
|
||||
const lowerContent = content.toLowerCase();
|
||||
for (const category of NVIDIA_BAD_CATEGORIES) {
|
||||
if (lowerContent.includes(category)) {
|
||||
categories.push(CATEGORY_TO_BADWORD_LABEL[category] ?? category);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Also check for structured response fields
|
||||
const choice = data?.choices?.[0];
|
||||
if (choice?.message?.content) {
|
||||
try {
|
||||
const parsed = JSON.parse(choice.message.content);
|
||||
if (parsed.categories && Array.isArray(parsed.categories)) {
|
||||
for (const cat of parsed.categories) {
|
||||
if (NVIDIA_BAD_CATEGORIES.has(cat.name ?? cat)) {
|
||||
categories.push(CATEGORY_TO_BADWORD_LABEL[cat.name ?? cat] ?? cat);
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Not JSON — already handled via text search above
|
||||
}
|
||||
}
|
||||
|
||||
return Array.from(new Set(categories));
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Three-tier cache pipeline
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Detect badwords in text using a **two-tier cache + API pipeline**:
|
||||
*
|
||||
* 1. **In-memory cache** (BADWORD_CACHE_TTL_MS, 10 min) — fastest path,
|
||||
* keyed by the full normalized text string.
|
||||
* 2. **DB cache** (DB_CACHE_TTL_MS, 24 h) — same full-text key, persisted
|
||||
* across restarts. Uses the FULL normalized text (not per-word) because
|
||||
* context matters: "kau" alone is clean, but "awas kau" can be a threat.
|
||||
* 3. **API pipeline** (NVIDIA → Primary AI → Groq)
|
||||
* only runs when both cache layers miss.
|
||||
*
|
||||
* No local hardcoded badword list — all detection goes through AI APIs
|
||||
* to eliminate false positives from substring matching.
|
||||
*/
|
||||
export async function detectIndonesianBadwords(
|
||||
text: string,
|
||||
): Promise<string[]> {
|
||||
const cacheKey = normalizeBadwordCacheKey(text);
|
||||
|
||||
// ── Tier 1: In-memory cache (fastest) ──
|
||||
const cached = getCachedBadwords(cacheKey);
|
||||
if (cached) {
|
||||
return cached;
|
||||
}
|
||||
|
||||
// De-duplicate concurrent lookups
|
||||
const inFlight = inFlightBadwordLookups.get(cacheKey);
|
||||
if (inFlight) {
|
||||
return inFlight;
|
||||
}
|
||||
|
||||
const lookupPromise = (async () => {
|
||||
// ── Tier 2: DB cache (survives restarts, preserves context) ──
|
||||
const dbEntry = await getCachedText(cacheKey);
|
||||
if (dbEntry) {
|
||||
const flags = [...dbEntry.flags];
|
||||
setCachedBadwords(cacheKey, flags); // populate in-memory too
|
||||
return flags;
|
||||
}
|
||||
|
||||
// ── Tier 3: API pipeline ──
|
||||
|
||||
const hits = new Set<string>();
|
||||
let sourceUsed: "nvidia" | "primary_ai" | "groq" = "primary_ai";
|
||||
|
||||
// 3a. Try NVIDIA API if key is configured and not rate limited.
|
||||
const apiKey = config.NVIDIA_NEMOTRON_API_KEY;
|
||||
if (apiKey && Date.now() >= nemotronUnavailableUntil) {
|
||||
try {
|
||||
const apiCategories = await callNemotronContentSafety(text);
|
||||
for (const hit of apiCategories) {
|
||||
hits.add(hit);
|
||||
}
|
||||
if (apiCategories.length > 0) sourceUsed = "nvidia";
|
||||
} catch (error) {
|
||||
const status = axios.isAxiosError(error)
|
||||
? error.response?.status
|
||||
: null;
|
||||
if (status === 429) {
|
||||
nemotronUnavailableUntil =
|
||||
Date.now() + NEMOTRON_RATE_LIMIT_COOLDOWN_MS;
|
||||
}
|
||||
log.warn(
|
||||
{ error },
|
||||
"NVIDIA Nemotron API call failed, falling back to primary AI",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// 3b. Try the main AI model next.
|
||||
if (hits.size === 0 && Date.now() >= primaryAiUnavailableUntil) {
|
||||
try {
|
||||
const primaryHits = await callPrimaryAiModeration(text);
|
||||
for (const hit of primaryHits) {
|
||||
hits.add(hit);
|
||||
}
|
||||
if (primaryHits.length > 0) sourceUsed = "primary_ai";
|
||||
} catch (error) {
|
||||
const status = axios.isAxiosError(error)
|
||||
? error.response?.status
|
||||
: null;
|
||||
if (status === 429) {
|
||||
primaryAiUnavailableUntil =
|
||||
Date.now() + PRIMARY_AI_RATE_LIMIT_COOLDOWN_MS;
|
||||
}
|
||||
log.warn(
|
||||
{ error },
|
||||
"Primary AI badword detection failed, falling back to Groq",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// 3c. Try Groq Llama Prompt Guard as final API fallback.
|
||||
if (hits.size === 0 && Date.now() >= groqUnavailableUntil) {
|
||||
const groqKey = config.GROQ_API_KEY;
|
||||
if (groqKey) {
|
||||
try {
|
||||
const groqHits = await callGrokModeration(text);
|
||||
for (const hit of groqHits) {
|
||||
hits.add(hit);
|
||||
}
|
||||
if (groqHits.length > 0) sourceUsed = "groq";
|
||||
} catch (error) {
|
||||
const status = axios.isAxiosError(error)
|
||||
? error.response?.status
|
||||
: null;
|
||||
if (status === 429) {
|
||||
groqUnavailableUntil = Date.now() + GROQ_RATE_LIMIT_COOLDOWN_MS;
|
||||
}
|
||||
log.warn({ error }, "Groq Llama Prompt Guard moderation failed");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const finalHits = Array.from(hits);
|
||||
|
||||
// Populate all cache tiers so the same text never triggers another API call
|
||||
// within the TTL window.
|
||||
setCachedBadwords(cacheKey, finalHits);
|
||||
await upsertCachedText(
|
||||
cacheKey,
|
||||
finalHits,
|
||||
sourceUsed,
|
||||
Date.now() + DB_CACHE_TTL_MS,
|
||||
);
|
||||
|
||||
return finalHits;
|
||||
})();
|
||||
|
||||
inFlightBadwordLookups.set(cacheKey, lookupPromise);
|
||||
|
||||
try {
|
||||
return await lookupPromise;
|
||||
} finally {
|
||||
inFlightBadwordLookups.delete(cacheKey);
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Async evidence builders
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export async function buildModerationTextEvidence(
|
||||
text: string,
|
||||
): Promise<ModerationTextEvidence> {
|
||||
const emojiNormalized = normalizeDiscordCustomEmoji(text);
|
||||
const badwordHits = await detectIndonesianBadwords(emojiNormalized.text);
|
||||
const notes: string[] = [];
|
||||
|
||||
for (const emojiName of emojiNormalized.emojiNames) {
|
||||
notes.push(
|
||||
`emoji:${emojiName}=Discord custom emoji/expression; not text offense by default`,
|
||||
);
|
||||
}
|
||||
|
||||
if (badwordHits.length > 0) {
|
||||
notes.push(`Indonesian badword detected: ${badwordHits.join(", ")}`);
|
||||
} else {
|
||||
notes.push("no Indonesian badword detected");
|
||||
}
|
||||
|
||||
return {
|
||||
raw: text,
|
||||
normalized: emojiNormalized.text,
|
||||
notes: Array.from(new Set(notes)),
|
||||
badwords: badwordHits,
|
||||
hasBadwords: badwordHits.length > 0,
|
||||
};
|
||||
}
|
||||
|
||||
export async function formatModerationTextEvidenceForPrompt(
|
||||
text: string,
|
||||
): Promise<string> {
|
||||
const evidence = await buildModerationTextEvidence(text);
|
||||
if (evidence.normalized === evidence.raw && evidence.notes.length === 0) {
|
||||
return "";
|
||||
}
|
||||
|
||||
return [
|
||||
`[normalized_text: ${evidence.normalized}]`,
|
||||
evidence.notes.length > 0
|
||||
? `[normalization_notes: ${evidence.notes.join("; ")}]`
|
||||
: null,
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join(" ");
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,161 @@
|
||||
/**
|
||||
* Modular system prompt builder for LLM moderation.
|
||||
*
|
||||
* Split into composable sections:
|
||||
* - buildSystemRules() — culture/slang/flag definitions (static)
|
||||
* - buildMediaInstructions() — media/sticker analysis guidance (conditional)
|
||||
* - buildFewShotExamples() — 3 example outputs (static)
|
||||
* - buildSystemPrompt() — assembles all sections with XML delimiters
|
||||
*
|
||||
* XML delimiters prevent prompt injection by clearly separating
|
||||
* system instructions from user-supplied data.
|
||||
*/
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Section: System Rules (static — culture, slang, flag definitions)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const SYSTEM_RULES = `Kamu adalah asisten moderasi konten untuk server Discord berbahasa Indonesia.
|
||||
Bahasa utama komunitas ini adalah BAHASA INDONESIA. Bahasa Inggris adalah bahasa sekunder.
|
||||
|
||||
## Aturan Umum
|
||||
- Bahasa gaul/slang Indonesia: "anjay", "wkwk", "gws", "gaskeun", "santuy", "njir", "baka", "woy", "woi", "hadeh", dll adalah AMAN.
|
||||
- Singkatan umum: "gw", "lo", "emg", "kyk", "tdk", "krn", "jgn", dll adalah AMAN.
|
||||
- Makian/kata kasar umum (seperti "anjing", "asu", "bangsat") BUKAN pelanggaran SARA. SARA khusus untuk diskriminasi/hinaan terhadap Suku, Agama, Ras, dan Antargolongan. NAMUN makian/kata kasar TETAP bisa di-flag sebagai "harassment" atau "vulgar_language" HANYA jika: (1) ditujukan langsung ke orang lain sebagai serangan/hinaan, (2) dalam tone agresif/mengancam, atau (3) bagian dari pola harassment berkelanjutan.
|
||||
- Kata "asus" adalah merk teknologi, jangan pernah dianggap sebagai makian "asu".
|
||||
- "woy"/"woi" adalah sapaan/interjeksi informal Indonesia dan tidak boleh dianggap SARA, hate speech, atau harassment tanpa target hinaan/ancaman jelas.
|
||||
- Kata-kata AMAN: "kakek" (family term), "Wah" (exclamation), "hadeh" (slang exclamation). Jangan flag sebagai vulgar_language atau harassment.
|
||||
- Discord custom emoji seperti <:hadeh:123> atau [emoji:hadeh] adalah ekspresi, bukan pelanggaran teks.
|
||||
- Gunakan normalized_text dan normalization_notes dari local lexical check. Jika notes hanya berisi slang/emoji aman, jangan flag. Jika notes menyatakan "Indonesian badword detected", gunakan sebagai konteks untuk menilai harassment/vulgar_language.
|
||||
|
||||
## Kategori Pelanggaran & Kriteria Flag
|
||||
Prioritas tertinggi (ANCAMAN KESELAMATAN):
|
||||
- child_safety, self_harm, violence, illegal_content — flag jika ada indikasi nyata
|
||||
- Pornografi/NSFW, ajakan seksual, roleplay seksual → "sexual_content"
|
||||
- Judi/promosi judi → "gambling"
|
||||
- Narkoba/promosi → "drugs"
|
||||
|
||||
Prioritas menengah (PERILAKU MERUSAK):
|
||||
- Ancaman kekerasan, doxxing, scam → flag sesuai kategori
|
||||
- spam self-promo → "spam"
|
||||
- Istilah agama/suku/ras: penyebutan netral/edukasi = clean; hinaan/provokasi/diskriminatif = "sara" atau "hate_speech"
|
||||
|
||||
Prioritas rendah (PELANGGARAN RINGAN):
|
||||
- harassment (targeted insult), vulgar_language (profanity terarah)
|
||||
- sexual_deviation: jika pesan mempromosikan/mendukung topik seksual/identitas yang dibatasi server sebagai pembahasan utama
|
||||
|
||||
## Pohon Keputusan (Decision Tree)
|
||||
1. Apakah ada ancaman keselamatan nyata (child_safety, self_harm, violence)? → flagged, critical
|
||||
2. Apakah ada konten ilegal/explicit (NSFW, drugs, gambling, scam)? → flagged, high
|
||||
3. Apakah ada harassment terarah/hate speech/sara? → flagged, medium-high
|
||||
4. Apakah ada spam/promosi borderline? → warn, low-medium
|
||||
5. Jika tidak ada pelanggaran jelas atau bukti ambigu → clean
|
||||
Jangan pernah flag hanya berdasarkan kecurigaan atau ketidakjelasan konteks.`;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Section: Media Instructions (conditional — injected when media present)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const MEDIA_INSTRUCTIONS = `## Instruksi Analisis Media
|
||||
Gambar, sticker, embed image, preview link, dan attachment sudah dianalisis lewat request media terpisah sebelum batch utama.
|
||||
Gunakan baris "Media analysis" sebagai evidence visual utama dalam keputusan moderasi batch ini.
|
||||
|
||||
## Panduan Khusus Sticker
|
||||
- Sticker Discord adalah media kartun/meme/ilustrasi, BUKAN foto atau video nyata.
|
||||
- Sticker sering bersifat humor, satir, atau ekspresi emosi yang dilebih-lebihkan.
|
||||
- Gambar sticker bisa menampilkan adegan kartun yang terlihat "keras" — itu SENI KARTUN, bukan dokumentasi kekerasan nyata.
|
||||
- Nama sticker yang terdengar provokatif (mis. "Singa injek pejabat") adalah konteks satir/humor. JANGAN flag berdasarkan nama sticker saja.
|
||||
- Terapkan standar yang lebih longgar untuk konten kartun/meme dibanding foto/video nyata.
|
||||
- Sticker yang berhasil diunduh WAJIB diperlakukan sebagai image evidence, bukan sekadar nama sticker.`;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Section: Few-Shot Examples
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const FEW_SHOT_EXAMPLES = `## Contoh Output yang Benak
|
||||
|
||||
Contoh 1 — Pesan bersih dengan slang:
|
||||
Input: [target] id=12345 user=budi: anjay wkwk gaskeun santuy bro
|
||||
Output: {"results":[{"message_id":"12345","status":"clean","flags":[],"score":0.0,"categories":[],"severity":"none","confidence":0.95,"recommended_action":"none","policy_version":"default-2026-05-30","evidence":[],"analysis":"Slang Indonesia umum tanpa pelanggaran terdeteksi."}]}
|
||||
|
||||
Contoh 2 — Harassment terarah:
|
||||
Input: [target] id=67890 user=anon: lu goblok banget sih kontol, mampus aja lo
|
||||
Output: {"results":[{"message_id":"67890","status":"flagged","flags":["harassment","vulgar_language"],"score":0.85,"categories":["harassment","vulgar_language"],"severity":"high","confidence":0.9,"recommended_action":"delete","policy_version":"default-2026-05-30","evidence":["lu goblok banget sih kontol","mampus aja lo"],"analysis":"Insult langsung dengan kata kasar terarah ke individu."}]}
|
||||
|
||||
Contoh 3 — Sticker kartun dengan nama provokatif:
|
||||
Input: [target] id=11111 user=citra: <:singa_injek:123456> [sticker: "Singa injek pejabat"]
|
||||
Output: {"results":[{"message_id":"11111","status":"clean","flags":[],"score":0.1,"categories":[],"severity":"none","confidence":0.8,"recommended_action":"none","policy_version":"default-2026-05-30","evidence":[],"analysis":"Sticker kartun satir dengan nama provokatif namun bukan ancaman nyata."}]}`;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Section: Output Schema + XML Delimiter Instructions
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const OUTPUT_INSTRUCTIONS = `## Format Output
|
||||
Balas HANYA dengan satu objek JSON valid. Tanpa markdown, tanpa prose, tanpa komentar, tanpa XML.
|
||||
Struktur wajib:
|
||||
{
|
||||
"results": [
|
||||
{
|
||||
"message_id": "<ID string PERSIS seperti di input>",
|
||||
"status": "clean" | "warn" | "flagged",
|
||||
"flags": ["<string array, kosong jika clean>"],
|
||||
"score": 0.0,
|
||||
"categories": ["<kategori kebijakan, kosong jika clean>"],
|
||||
"severity": "none" | "low" | "medium" | "high" | "critical",
|
||||
"confidence": 0.0,
|
||||
"recommended_action": "none" | "monitor" | "warn" | "review" | "delete" | "escalate",
|
||||
"policy_version": "default-2026-05-30",
|
||||
"evidence": ["<kutipan/evidence singkat>"],
|
||||
"analysis": "<penjelasan singkat dalam Bahasa Indonesia, maks 2 kalimat>"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
Kriteria status:
|
||||
- "clean": tidak ada pelanggaran terdeteksi, atau kasus ambigu setelah semua evidence dianalisis
|
||||
- "warn": risiko ringan konkret terdeteksi (spam borderline, harassment ringan)
|
||||
- "flagged": pelanggaran jelas terdeteksi
|
||||
|
||||
Larangan output analysis:
|
||||
- Jangan tulis "kurang konteks", "perlu dicek admin", "perlu moderator periksa", "tidak bisa menentukan", atau frasa deferral sejenis.
|
||||
- Jika evidence tidak cukup kuat untuk pelanggaran, status harus "clean" dan analysis menjelaskan alasan langsung.
|
||||
- Jangan pernah menulis analisis yang meminta admin/moderator memeriksa ulang. Berikan kesimpulan langsung.
|
||||
|
||||
Flag yang valid: spam, hate_speech, sara, hoaks, harassment, vulgar_language, sexual_content, sexual_deviation, violence, self_harm, doxxing, scam, misinformation, nsfw_image, gore_image, illegal_content, gambling, drugs, child_safety, financial_scam, religious_insult, self_promo
|
||||
|
||||
CRITICAL: "message_id" HARUS berupa STRING (dibungkus tanda kutip ganda). Jangan perlakukan ID sebagai angka.`;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Composer: assembles all sections with XML delimiters
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export interface BuildSystemPromptOptions {
|
||||
contextText: string;
|
||||
includeMediaInstructions: boolean;
|
||||
correction?: { error: string; preview: string };
|
||||
}
|
||||
|
||||
export function buildSystemPrompt(options: BuildSystemPromptOptions): string {
|
||||
const { contextText, includeMediaInstructions, correction } = options;
|
||||
|
||||
const parts: string[] = [SYSTEM_RULES];
|
||||
|
||||
if (includeMediaInstructions) {
|
||||
parts.push(MEDIA_INSTRUCTIONS);
|
||||
}
|
||||
|
||||
parts.push(FEW_SHOT_EXAMPLES);
|
||||
parts.push(OUTPUT_INSTRUCTIONS);
|
||||
|
||||
// XML-delimited context — prevents prompt injection
|
||||
const delimitedContext = `<conversation_context>\n${contextText}\n</conversation_context>`;
|
||||
parts.push(delimitedContext);
|
||||
|
||||
let base = parts.join("\n\n");
|
||||
|
||||
if (correction) {
|
||||
base += `\n\nRESPON SEBELUMNYA GAGAL VALIDASI.\nError: ${correction.error}\nPreview respons tidak valid:\n${correction.preview}\n\nCoba lagi dengan output JSON yang benar sesuai skema di atas.`;
|
||||
}
|
||||
|
||||
return base;
|
||||
}
|
||||
@@ -0,0 +1,209 @@
|
||||
import { mkdir, readFile, unlink, writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { createChildLogger } from "../../shared/logger/logger.js";
|
||||
|
||||
const logger = createChildLogger("sticker-cache");
|
||||
|
||||
export interface StickerCacheEntry {
|
||||
base64: string;
|
||||
mimeType: string;
|
||||
fetchedAt: number;
|
||||
size: number;
|
||||
}
|
||||
|
||||
interface CacheIndexEntry {
|
||||
file: string;
|
||||
mimeType: string;
|
||||
size: number;
|
||||
fetchedAt: number;
|
||||
}
|
||||
|
||||
interface CacheIndex {
|
||||
entries: Record<string, CacheIndexEntry>;
|
||||
totalSizeBytes: number;
|
||||
}
|
||||
|
||||
export interface StickerCacheOptions {
|
||||
cacheDir: string;
|
||||
maxSizeBytes: number;
|
||||
ttlMs?: number;
|
||||
}
|
||||
|
||||
let cacheDir = "";
|
||||
let maxSizeBytes = 0;
|
||||
let ttlMs = 7 * 24 * 60 * 60 * 1000; // 7 days default
|
||||
let index: CacheIndex = { entries: {}, totalSizeBytes: 0 };
|
||||
let ready = false;
|
||||
|
||||
function sanitizeKey(name: string): string {
|
||||
return encodeURIComponent(name).replace(/%/g, "_");
|
||||
}
|
||||
|
||||
async function loadIndex(): Promise<CacheIndex> {
|
||||
try {
|
||||
const raw = await readFile(join(cacheDir, "index.json"), "utf-8");
|
||||
return JSON.parse(raw) as CacheIndex;
|
||||
} catch {
|
||||
return { entries: {}, totalSizeBytes: 0 };
|
||||
}
|
||||
}
|
||||
|
||||
async function saveIndex(idx: CacheIndex): Promise<void> {
|
||||
await writeFile(
|
||||
join(cacheDir, "index.json"),
|
||||
JSON.stringify(idx, null, 2),
|
||||
"utf-8",
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialise the sticker cache: create directory, load index.
|
||||
* Idempotent — safe to call multiple times.
|
||||
*/
|
||||
export async function initStickerCache(
|
||||
opts: StickerCacheOptions,
|
||||
): Promise<void> {
|
||||
if (ready) return;
|
||||
cacheDir = opts.cacheDir;
|
||||
maxSizeBytes = opts.maxSizeBytes;
|
||||
ttlMs = opts.ttlMs ?? 7 * 24 * 60 * 60 * 1000;
|
||||
|
||||
await mkdir(cacheDir, { recursive: true });
|
||||
index = await loadIndex();
|
||||
|
||||
// Prune expired entries on startup
|
||||
const now = Date.now();
|
||||
let changed = false;
|
||||
for (const [key, meta] of Object.entries(index.entries)) {
|
||||
if (now - meta.fetchedAt > ttlMs) {
|
||||
await unlink(join(cacheDir, meta.file)).catch(() => {});
|
||||
index.totalSizeBytes -= meta.size;
|
||||
delete index.entries[key];
|
||||
changed = true;
|
||||
}
|
||||
}
|
||||
if (changed) await saveIndex(index);
|
||||
|
||||
ready = true;
|
||||
logger.info(
|
||||
{
|
||||
entryCount: Object.keys(index.entries).length,
|
||||
totalSizeBytes: index.totalSizeBytes,
|
||||
},
|
||||
"Sticker cache initialized",
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Look up a sticker image by name. Returns null on miss or TTL expiry.
|
||||
*/
|
||||
export async function getStickerFromCache(
|
||||
stickerName: string,
|
||||
): Promise<StickerCacheEntry | null> {
|
||||
if (!ready) return null;
|
||||
|
||||
const key = sanitizeKey(stickerName);
|
||||
const meta = index.entries[key];
|
||||
if (!meta) return null;
|
||||
|
||||
// TTL check
|
||||
if (Date.now() - meta.fetchedAt > ttlMs) {
|
||||
await unlink(join(cacheDir, meta.file)).catch(() => {});
|
||||
index.totalSizeBytes -= meta.size;
|
||||
delete index.entries[key];
|
||||
await saveIndex(index);
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
const raw = await readFile(join(cacheDir, meta.file), "utf-8");
|
||||
return {
|
||||
base64: raw,
|
||||
mimeType: meta.mimeType,
|
||||
fetchedAt: meta.fetchedAt,
|
||||
size: meta.size,
|
||||
};
|
||||
} catch {
|
||||
// File missing — clean up index entry
|
||||
delete index.entries[key];
|
||||
await saveIndex(index);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Store a sticker image in the cache. Fires and forgets — never blocks.
|
||||
*/
|
||||
export async function setStickerInCache(
|
||||
stickerName: string,
|
||||
base64: string,
|
||||
mimeType: string,
|
||||
): Promise<void> {
|
||||
if (!ready) return;
|
||||
|
||||
const key = sanitizeKey(stickerName);
|
||||
const fileName = `${key}.dat`;
|
||||
const size = Buffer.byteLength(base64, "utf-8");
|
||||
|
||||
// Evict if needed
|
||||
await evictIfNeeded(size);
|
||||
|
||||
try {
|
||||
await writeFile(join(cacheDir, fileName), base64, "utf-8");
|
||||
index.entries[key] = {
|
||||
file: fileName,
|
||||
mimeType,
|
||||
size,
|
||||
fetchedAt: Date.now(),
|
||||
};
|
||||
index.totalSizeBytes += size;
|
||||
await saveIndex(index);
|
||||
logger.debug({ stickerName, size }, "Sticker cached");
|
||||
} catch (err) {
|
||||
logger.warn(
|
||||
{ stickerName, error: err instanceof Error ? err.message : String(err) },
|
||||
"Failed to write sticker to cache",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async function evictIfNeeded(newSize: number): Promise<void> {
|
||||
while (index.totalSizeBytes + newSize > maxSizeBytes) {
|
||||
// Find oldest entry
|
||||
let oldestKey: string | null = null;
|
||||
let oldestTime = Infinity;
|
||||
for (const [key, meta] of Object.entries(index.entries)) {
|
||||
if (meta.fetchedAt < oldestTime) {
|
||||
oldestTime = meta.fetchedAt;
|
||||
oldestKey = key;
|
||||
}
|
||||
}
|
||||
if (!oldestKey) break;
|
||||
|
||||
const meta = index.entries[oldestKey];
|
||||
await unlink(join(cacheDir, meta.file)).catch(() => {});
|
||||
index.totalSizeBytes -= meta.size;
|
||||
delete index.entries[oldestKey];
|
||||
}
|
||||
await saveIndex(index);
|
||||
}
|
||||
|
||||
/**
|
||||
* Return current cache stats for observability.
|
||||
*/
|
||||
export function getStickerCacheStats(): {
|
||||
entryCount: number;
|
||||
totalSizeBytes: number;
|
||||
} {
|
||||
return {
|
||||
entryCount: Object.keys(index.entries).length,
|
||||
totalSizeBytes: index.totalSizeBytes,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if cache has been initialized.
|
||||
*/
|
||||
export function isStickerCacheReady(): boolean {
|
||||
return ready;
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
/**
|
||||
* Sticker-specific prompt templates for AI moderation.
|
||||
*
|
||||
* Discord stickers are cartoon/meme artwork — not real photos.
|
||||
* These prompts give the LLM proper context to avoid false-positive flags
|
||||
* based solely on sticker names or cartoon imagery.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Prompt used when a sticker image was successfully downloaded (from cache
|
||||
* or network) and is being sent to the vision LLM as a base64 image.
|
||||
*
|
||||
* Explains that stickers are cartoon art, not documentation of real events,
|
||||
* and instructs the model to apply looser standards for cartoon content.
|
||||
*/
|
||||
export function buildStickerVisionPrompt(
|
||||
stickerName: string,
|
||||
messageId: string,
|
||||
): string {
|
||||
return [
|
||||
`Analisis sticker Discord berikut sebagai evidence moderasi.`,
|
||||
`Sticker "${stickerName}" berasal dari pesan id=${messageId}.`,
|
||||
``,
|
||||
`PENTING — Konteks Sticker:`,
|
||||
`- Sticker Discord adalah gambar KARTUN/MEME/ILUSTRASI, BUKAN foto atau video nyata.`,
|
||||
`- Sticker sering bersifat humor, satir, atau ekspresi emosi yang dilebih-lebihkan.`,
|
||||
`- Gambar di sticker bisa menampilkan adegan yang terlihat "keras" (tokoh kartun menginjak sesuatu, ledakan komik, senjata kartun, tokoh berantem) — itu SENI KARTUN, bukan dokumentasi kekerasan atau ancaman nyata.`,
|
||||
`- Teks di sticker sering berupa lelucon, sindiran, atau ekspresi khas komunitas — bukan ancaman literal.`,
|
||||
``,
|
||||
`Jelaskan isi visual, teks yang terlihat, dan konteks risiko.`,
|
||||
`Terapkan standar yang lebih longgar untuk konten kartun/meme:`,
|
||||
`- Adegan kartun yang terlihat "keras" ≠ kekerasan nyata → jangan flag "violence" kecuali jelas menargetkan individu/kelompok nyata dengan ancaman serius.`,
|
||||
`- Nama sticker yang terdengar provokatif (mis. "Singa injek pejabat") adalah konteks satir/kartun, bukan bukti pelanggaran.`,
|
||||
`- Humor/satir/politik kartun ≠ SARA atau hate speech.`,
|
||||
`- Sticker yang menampilkan tokoh kartun dalam pose agresif adalah ekspresi/emosi umum di Discord, bukan harassment.`,
|
||||
``,
|
||||
`Jawab Bahasa Indonesia, maksimal 3 kalimat. Jangan bilang kurang konteks atau perlu admin cek.`,
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
/**
|
||||
* Wrapper for text-only evidence when a sticker image failed to download.
|
||||
*
|
||||
* Returns a formatted string that explicitly tells the LLM not to flag
|
||||
* based on the sticker name alone, since names can sound provocative
|
||||
* while the actual cartoon image is harmless.
|
||||
*/
|
||||
export function buildStickerTextOnlyWarning(
|
||||
stickerName: string,
|
||||
stickerUrl: string,
|
||||
): string {
|
||||
return (
|
||||
`[sticker: "${stickerName}" (${stickerUrl}) — GAMBAR GAGAL DIUNDUH. ` +
|
||||
`"${stickerName}" adalah sticker kartun/meme Discord. ` +
|
||||
`JANGAN flag berdasarkan nama sticker saja tanpa gambar visual. ` +
|
||||
`Sticker Discord adalah seni kartun/ekspresi humor, bukan foto nyata. ` +
|
||||
`Nama yang terdengar provokatif adalah hal umum untuk sticker satir/humor di Discord.]`
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Prompt used when a custom emoji image was successfully downloaded
|
||||
* and is being sent to the vision LLM as a base64 image.
|
||||
*
|
||||
* Custom emojis are small icons — context is similar to stickers.
|
||||
*/
|
||||
export function buildCustomEmojiVisionPrompt(
|
||||
emojiName: string,
|
||||
messageId: string,
|
||||
): string {
|
||||
return [
|
||||
`Analisis custom emoji Discord berikut sebagai evidence moderasi.`,
|
||||
`Emoji "${emojiName}" berasal dari pesan id=${messageId}.`,
|
||||
``,
|
||||
`PENTING — Konteks Custom Emoji:`,
|
||||
`- Custom emoji Discord adalah ikon kecil/ekspresi, BUKAN foto atau dokumen nyata.`,
|
||||
`- Emoji sering digunakan untuk ekspresi emosi, reaksi, atau lelucon.`,
|
||||
`- Jangan flag berdasarkan nama emoji saja — analisis isi visual gambar.`,
|
||||
`- Emoji yang terlihat lucu/aneh adalah hal umum di Discord, bukan pelanggaran.`,
|
||||
``,
|
||||
`Jelaskan isi visual dan konteks risiko.`,
|
||||
`Jawab Bahasa Indonesia, maksimal 2 kalimat. Jangan bilang kurang konteks.`,
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
/**
|
||||
* Fallback text for when a custom emoji image failed to download.
|
||||
*/
|
||||
export function buildCustomEmojiTextOnlyFallback(emojiName: string): string {
|
||||
return (
|
||||
`[custom_emoji: "${emojiName}" — GAMBAR GAGAL DIUNDUH. ` +
|
||||
`"${emojiName}" adalah custom emoji Discord (ikon kecil). ` +
|
||||
`JANGAN flag berdasarkan nama emoji saja tanpa gambar visual. ` +
|
||||
`Custom emoji di Discord adalah ekspresi/emosi umum, bukan konten ofensif.]`
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,241 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { executeAll, executeGet } from "../../shared/database/drizzle.js";
|
||||
import { createChildLogger } from "../../shared/logger/logger.js";
|
||||
|
||||
const logger = createChildLogger("text-cache-store");
|
||||
|
||||
export interface TextCacheEntry {
|
||||
text: string;
|
||||
flags: string[];
|
||||
source: "local" | "nvidia" | "primary_ai" | "groq" | "vision_llm";
|
||||
analyzed_at: number;
|
||||
expires_at: number;
|
||||
hit_count: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Lookup cached analysis result for a normalized text string.
|
||||
* Returns null if not found or expired.
|
||||
*/
|
||||
export async function getCachedText(
|
||||
text: string,
|
||||
): Promise<TextCacheEntry | null> {
|
||||
try {
|
||||
const row = await executeGet(
|
||||
`SELECT text, flags, source, analyzed_at, expires_at, hit_count
|
||||
FROM text_analysis_cache
|
||||
WHERE text = $1 AND expires_at > $2`,
|
||||
[text, Date.now()],
|
||||
);
|
||||
|
||||
if (!row) return null;
|
||||
|
||||
return {
|
||||
text: row.text,
|
||||
flags: JSON.parse(row.flags),
|
||||
source: row.source,
|
||||
analyzed_at: row.analyzed_at,
|
||||
expires_at: row.expires_at,
|
||||
hit_count: row.hit_count,
|
||||
};
|
||||
} catch (error) {
|
||||
logger.error(
|
||||
{ error: error instanceof Error ? error.message : String(error) },
|
||||
"Failed to get cached text",
|
||||
);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Insert or update a text analysis cache entry.
|
||||
*/
|
||||
export async function upsertCachedText(
|
||||
text: string,
|
||||
flags: string[],
|
||||
source: "local" | "nvidia" | "primary_ai" | "groq" | "vision_llm",
|
||||
expiresAt: number,
|
||||
): Promise<void> {
|
||||
const now = Date.now();
|
||||
|
||||
try {
|
||||
await executeAll(
|
||||
`INSERT INTO text_analysis_cache (text, flags, source, analyzed_at, expires_at, hit_count)
|
||||
VALUES ($1, $2, $3, $4, $5, 0)
|
||||
ON CONFLICT (text) DO UPDATE SET
|
||||
flags = EXCLUDED.flags,
|
||||
source = EXCLUDED.source,
|
||||
analyzed_at = EXCLUDED.analyzed_at,
|
||||
expires_at = EXCLUDED.expires_at`,
|
||||
[text, JSON.stringify(flags), source, now, expiresAt],
|
||||
);
|
||||
} catch (error) {
|
||||
logger.error(
|
||||
{ error: error instanceof Error ? error.message : String(error) },
|
||||
"Failed to upsert cached text",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Increment hit count for a cached text entry (called on cache hit).
|
||||
*/
|
||||
export async function incrementTextCacheHit(text: string): Promise<void> {
|
||||
try {
|
||||
await executeAll(
|
||||
`UPDATE text_analysis_cache SET hit_count = hit_count + 1 WHERE text = $1`,
|
||||
[text],
|
||||
);
|
||||
} catch (error) {
|
||||
// Silent fail — this is just a counter, not critical
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete expired cache entries. Run periodically to keep the table clean.
|
||||
*/
|
||||
export async function pruneExpiredTexts(): Promise<number> {
|
||||
try {
|
||||
const result = await executeAll(
|
||||
`DELETE FROM text_analysis_cache WHERE expires_at < $1`,
|
||||
[Date.now()],
|
||||
);
|
||||
return (result as any).rowCount ?? 0;
|
||||
} catch (error) {
|
||||
logger.error(
|
||||
{ error: error instanceof Error ? error.message : String(error) },
|
||||
"Failed to prune expired texts",
|
||||
);
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get cache statistics for observability.
|
||||
*/
|
||||
export async function getTextCacheStats(): Promise<{
|
||||
total: number;
|
||||
expired: number;
|
||||
bySource: Record<string, number>;
|
||||
}> {
|
||||
try {
|
||||
const now = Date.now();
|
||||
|
||||
const [totalRow, expiredRow, sourceRows] = await Promise.all([
|
||||
executeAll(`SELECT count(*) as cnt FROM text_analysis_cache`),
|
||||
executeAll(
|
||||
`SELECT count(*) as cnt FROM text_analysis_cache WHERE expires_at < $1`,
|
||||
[now],
|
||||
),
|
||||
executeAll(
|
||||
`SELECT source, count(*) as cnt FROM text_analysis_cache GROUP BY source`,
|
||||
),
|
||||
]);
|
||||
|
||||
const bySource: Record<string, number> = {};
|
||||
for (const row of sourceRows) {
|
||||
bySource[row.source] = row.cnt;
|
||||
}
|
||||
|
||||
return {
|
||||
total: totalRow[0]?.cnt ?? 0,
|
||||
expired: expiredRow[0]?.cnt ?? 0,
|
||||
bySource,
|
||||
};
|
||||
} catch (error) {
|
||||
logger.error(
|
||||
{ error: error instanceof Error ? error.message : String(error) },
|
||||
"Failed to get text cache stats",
|
||||
);
|
||||
return { total: 0, expired: 0, bySource: {} };
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Media / Vision analysis cache helpers (reuses text_analysis_cache table)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Generate a deterministic cache key for a sticker.
|
||||
* Same sticker name → same key across sessions and servers.
|
||||
*/
|
||||
export function makeStickerCacheKey(stickerName: string): string {
|
||||
return `sticker:${stickerName}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a deterministic cache key for a custom emoji by its Discord ID.
|
||||
*/
|
||||
export function makeCustomEmojiCacheKey(emojiId: string): string {
|
||||
return `emoji:${emojiId}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a deterministic cache key for an image data URL.
|
||||
* Hashes the first 128 chars of the data URL (enough to identify the image
|
||||
* without storing the full base64 string as the key).
|
||||
*/
|
||||
export function makeImageCacheKey(dataUrl: string): string {
|
||||
const prefix = dataUrl.slice(0, 128);
|
||||
const hash = createHash("sha256").update(prefix).digest("hex").slice(0, 16);
|
||||
return `image:${hash}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Lookup a cached media analysis result.
|
||||
* Returns the full cached text (the analysis summary string) or null.
|
||||
*/
|
||||
export async function getCachedMediaAnalysis(
|
||||
cacheKey: string,
|
||||
): Promise<string | null> {
|
||||
try {
|
||||
const row = await executeGet(
|
||||
`SELECT flags, hit_count
|
||||
FROM text_analysis_cache
|
||||
WHERE text = $1 AND expires_at > $2`,
|
||||
[cacheKey, Date.now()],
|
||||
);
|
||||
|
||||
if (!row) return null;
|
||||
|
||||
// flags stores the analysis result for media entries
|
||||
const result = JSON.parse(row.flags) as string;
|
||||
return result || null;
|
||||
} catch (error) {
|
||||
logger.error(
|
||||
{ error: error instanceof Error ? error.message : String(error) },
|
||||
"Failed to get cached media analysis",
|
||||
);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Store a media analysis result in the cache.
|
||||
*/
|
||||
export async function upsertCachedMediaAnalysis(
|
||||
cacheKey: string,
|
||||
analysisResult: string,
|
||||
source: "vision_llm",
|
||||
expiresAt: number,
|
||||
): Promise<void> {
|
||||
const now = Date.now();
|
||||
|
||||
try {
|
||||
await executeAll(
|
||||
`INSERT INTO text_analysis_cache (text, flags, source, analyzed_at, expires_at, hit_count)
|
||||
VALUES ($1, $2, $3, $4, $5, 0)
|
||||
ON CONFLICT (text) DO UPDATE SET
|
||||
flags = EXCLUDED.flags,
|
||||
source = EXCLUDED.source,
|
||||
analyzed_at = EXCLUDED.analyzed_at,
|
||||
expires_at = EXCLUDED.expires_at`,
|
||||
[cacheKey, JSON.stringify(analysisResult), source, now, expiresAt],
|
||||
);
|
||||
} catch (error) {
|
||||
logger.error(
|
||||
{ error: error instanceof Error ? error.message : String(error) },
|
||||
"Failed to upsert cached media analysis",
|
||||
);
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user