refactor: split monolith into 3 microservices (frontend, backend, discord-gateway)
- Extract services into services/{frontend,backend,discord-gateway}
- Create packages/shared/ for shared logger, errors, utils, types
- Setup Modular MVC pattern in backend (controller→service→repository)
- Setup event-driven architecture in discord-gateway with Redis pub/sub
- Move Docker files to infra/docker/ with per-service Dockerfiles
- Update docker-compose.yml to use Traefik-only routing (no port exposes)
- Update GitHub Actions deploy workflow for multi-service matrix build
- Fix all import paths and resolve type errors across all services
- All 3 services pass tsc --noEmit clean
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
MythEclipse
co-authored by
Claude Opus 4.8
Claude Opus 4.8
parent
bda8304bb9
commit
c48a0c5e3b
@@ -11,20 +11,21 @@ permissions:
|
|||||||
packages: write
|
packages: write
|
||||||
|
|
||||||
env:
|
env:
|
||||||
IMAGE_NAME: ghcr.io/${{ github.repository_owner }}/bete
|
REGISTRY: ghcr.io
|
||||||
|
OWNER: ${{ github.repository_owner }}
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
deploy:
|
build-and-push:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
|
strategy:
|
||||||
|
matrix:
|
||||||
|
service: [frontend, backend, discord-gateway]
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout repository
|
- name: Checkout repository
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@v4
|
||||||
with:
|
with:
|
||||||
submodules: recursive
|
submodules: recursive
|
||||||
|
|
||||||
- name: Normalize image name
|
|
||||||
run: echo "IMAGE_NAME=${IMAGE_NAME,,}" >> "$GITHUB_ENV"
|
|
||||||
|
|
||||||
- name: Set up Docker Buildx
|
- name: Set up Docker Buildx
|
||||||
uses: docker/setup-buildx-action@v3
|
uses: docker/setup-buildx-action@v3
|
||||||
|
|
||||||
@@ -35,19 +36,28 @@ jobs:
|
|||||||
username: ${{ github.actor }}
|
username: ${{ github.actor }}
|
||||||
password: ${{ secrets.GITHUB_TOKEN }}
|
password: ${{ secrets.GITHUB_TOKEN }}
|
||||||
|
|
||||||
- name: Build and push Docker image
|
- name: Build and push ${{ matrix.service }}
|
||||||
uses: docker/build-push-action@v6
|
uses: docker/build-push-action@v6
|
||||||
with:
|
with:
|
||||||
context: .
|
context: .
|
||||||
|
file: infra/docker/Dockerfile.${{ matrix.service }}
|
||||||
push: true
|
push: true
|
||||||
tags: |
|
tags: |
|
||||||
${{ env.IMAGE_NAME }}:latest
|
${{ env.REGISTRY }}/${{ env.OWNER }}/bete-${{ matrix.service }}:latest
|
||||||
${{ env.IMAGE_NAME }}:${{ github.sha }}
|
${{ env.REGISTRY }}/${{ env.OWNER }}/bete-${{ matrix.service }}:${{ github.sha }}
|
||||||
|
cache-from: type=gha
|
||||||
|
cache-to: type=gha,mode=max
|
||||||
|
|
||||||
|
deploy:
|
||||||
|
needs: build-and-push
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- name: Checkout repository
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
- name: Deploy to VPS
|
- name: Deploy to VPS
|
||||||
uses: appleboy/ssh-action@v1.2.5
|
uses: appleboy/ssh-action@v1.2.5
|
||||||
env:
|
env:
|
||||||
IMAGE_NAME: ${{ env.IMAGE_NAME }}
|
|
||||||
GHCR_USERNAME: ${{ github.actor }}
|
GHCR_USERNAME: ${{ github.actor }}
|
||||||
GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||||
ENV_FILE: ${{ secrets.ENV_FILE }}
|
ENV_FILE: ${{ secrets.ENV_FILE }}
|
||||||
@@ -55,7 +65,7 @@ jobs:
|
|||||||
host: ${{ secrets.VPS_HOST }}
|
host: ${{ secrets.VPS_HOST }}
|
||||||
username: ${{ secrets.VPS_USERNAME }}
|
username: ${{ secrets.VPS_USERNAME }}
|
||||||
key: ${{ secrets.VPS_SSH_KEY }}
|
key: ${{ secrets.VPS_SSH_KEY }}
|
||||||
envs: IMAGE_NAME,GHCR_USERNAME,GHCR_TOKEN,ENV_FILE
|
envs: GHCR_USERNAME,GHCR_TOKEN,ENV_FILE
|
||||||
script: |
|
script: |
|
||||||
set -eu
|
set -eu
|
||||||
|
|
||||||
@@ -63,27 +73,58 @@ jobs:
|
|||||||
mkdir -p "$APP_DIR"
|
mkdir -p "$APP_DIR"
|
||||||
cd "$APP_DIR"
|
cd "$APP_DIR"
|
||||||
|
|
||||||
printf '%s\nIMAGE_NAME=%s:latest\n' "$ENV_FILE" "$IMAGE_NAME" > .env
|
printf '%s\n' "$ENV_FILE" > .env
|
||||||
|
|
||||||
cat > docker-compose.yml <<'EOF'
|
cat > docker-compose.yml <<'COMPOSE_EOF'
|
||||||
services:
|
services:
|
||||||
app:
|
backend:
|
||||||
image: ${IMAGE_NAME}
|
image: ghcr.io/${OWNER:-mytheclipse}/bete-backend:latest
|
||||||
container_name: imphenbot-app
|
container_name: imphenbot-backend
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
env_file:
|
env_file:
|
||||||
- .env
|
- .env
|
||||||
volumes:
|
environment:
|
||||||
- ./recordings:/app/recordings
|
NODE_ENV: production
|
||||||
- ./.muxer-queue.db:/app/.muxer-queue.db
|
WEBSERVER_PORT: 3000
|
||||||
- ./.muxer-queue.db-shm:/app/.muxer-queue.db-shm
|
|
||||||
- ./.muxer-queue.db-wal:/app/.muxer-queue.db-wal
|
|
||||||
labels:
|
labels:
|
||||||
- "traefik.enable=true"
|
- "traefik.enable=true"
|
||||||
- "traefik.http.routers.imphenbot.rule=Host(`imphnen.asepharyana.tech`)"
|
- "traefik.http.routers.imphenbot-backend.rule=Host(`imphnen.asepharyana.my.id`) && PathPrefix(`/api`, `/ws`)"
|
||||||
- "traefik.http.routers.imphenbot.entrypoints=websecure"
|
- "traefik.http.routers.imphenbot-backend.entrypoints=websecure"
|
||||||
- "traefik.http.routers.imphenbot.tls=true"
|
- "traefik.http.routers.imphenbot-backend.tls=true"
|
||||||
- "traefik.http.services.imphenbot.loadbalancer.server.port=3000"
|
- "traefik.http.services.imphenbot-backend.loadbalancer.server.port=3000"
|
||||||
|
depends_on:
|
||||||
|
- discord-gateway
|
||||||
|
networks:
|
||||||
|
- app-shared-net
|
||||||
|
|
||||||
|
discord-gateway:
|
||||||
|
image: ghcr.io/${OWNER:-mytheclipse}/bete-discord-gateway:latest
|
||||||
|
container_name: imphenbot-discord-gateway
|
||||||
|
restart: unless-stopped
|
||||||
|
env_file:
|
||||||
|
- .env
|
||||||
|
environment:
|
||||||
|
NODE_ENV: production
|
||||||
|
volumes:
|
||||||
|
- ./recordings:/app/recordings
|
||||||
|
networks:
|
||||||
|
- app-shared-net
|
||||||
|
|
||||||
|
frontend:
|
||||||
|
image: ghcr.io/${OWNER:-mytheclipse}/bete-frontend:latest
|
||||||
|
container_name: imphenbot-frontend
|
||||||
|
restart: unless-stopped
|
||||||
|
environment:
|
||||||
|
VITE_API_URL: https://imphnen.asepharyana.my.id
|
||||||
|
VITE_WS_URL: wss://imphnen.asepharyana.my.id
|
||||||
|
labels:
|
||||||
|
- "traefik.enable=true"
|
||||||
|
- "traefik.http.routers.imphenbot-frontend.rule=Host(`imphnen.asepharyana.my.id`)"
|
||||||
|
- "traefik.http.routers.imphenbot-frontend.entrypoints=websecure"
|
||||||
|
- "traefik.http.routers.imphenbot-frontend.tls=true"
|
||||||
|
- "traefik.http.services.imphenbot-frontend.loadbalancer.server.port=3000"
|
||||||
|
depends_on:
|
||||||
|
- backend
|
||||||
networks:
|
networks:
|
||||||
- app-shared-net
|
- app-shared-net
|
||||||
|
|
||||||
@@ -91,9 +132,8 @@ jobs:
|
|||||||
app-shared-net:
|
app-shared-net:
|
||||||
name: app-shared-net
|
name: app-shared-net
|
||||||
external: true
|
external: true
|
||||||
EOF
|
COMPOSE_EOF
|
||||||
|
|
||||||
touch .muxer-queue.db .muxer-queue.db-shm .muxer-queue.db-wal
|
|
||||||
mkdir -p recordings
|
mkdir -p recordings
|
||||||
|
|
||||||
echo "$GHCR_TOKEN" | docker login ghcr.io -u "$GHCR_USERNAME" --password-stdin
|
echo "$GHCR_TOKEN" | docker login ghcr.io -u "$GHCR_USERNAME" --password-stdin
|
||||||
|
|||||||
-42
@@ -1,42 +0,0 @@
|
|||||||
FROM nixos/nix:latest
|
|
||||||
|
|
||||||
SHELL ["/bin/sh", "-c"]
|
|
||||||
|
|
||||||
ENV NIX_CONFIG="experimental-features = nix-command flakes"
|
|
||||||
|
|
||||||
# Install all system dependencies in a single nix profile to avoid version conflicts.
|
|
||||||
# The NixOS base image lacks common Unix utilities (sed, coreutils, etc.) that
|
|
||||||
# native Node.js post-install scripts (node-pre-gyp, prebuild-install) require.
|
|
||||||
# We pin nixpkgs to a specific commit for reproducible builds.
|
|
||||||
ARG NIXPKGS_COMMIT=64c08a7ca051951c8eae34e3e3cb1e202fe36786
|
|
||||||
|
|
||||||
RUN nix profile install \
|
|
||||||
"github:NixOS/nixpkgs/${NIXPKGS_COMMIT}#gnused" \
|
|
||||||
"github:NixOS/nixpkgs/${NIXPKGS_COMMIT}#coreutils-full" \
|
|
||||||
"github:NixOS/nixpkgs/${NIXPKGS_COMMIT}#nodejs_22" \
|
|
||||||
"github:NixOS/nixpkgs/${NIXPKGS_COMMIT}#ffmpeg" \
|
|
||||||
"github:NixOS/nixpkgs/${NIXPKGS_COMMIT}#python3" \
|
|
||||||
"github:NixOS/nixpkgs/${NIXPKGS_COMMIT}#gnumake" \
|
|
||||||
"github:NixOS/nixpkgs/${NIXPKGS_COMMIT}#gcc" \
|
|
||||||
"github:NixOS/nixpkgs/${NIXPKGS_COMMIT}#pkg-config" \
|
|
||||||
"github:NixOS/nixpkgs/${NIXPKGS_COMMIT}#vips" \
|
|
||||||
"github:NixOS/nixpkgs/${NIXPKGS_COMMIT}#yt-dlp"
|
|
||||||
|
|
||||||
RUN corepack enable
|
|
||||||
|
|
||||||
WORKDIR /app
|
|
||||||
|
|
||||||
# Install deps from the app-local build context.
|
|
||||||
COPY package.json pnpm-lock.yaml pnpm-workspace.yaml* ./
|
|
||||||
COPY vendor/discord-video-stream/package.json ./vendor/discord-video-stream/
|
|
||||||
COPY vendor/discord.js-selfbot-v13/package.json ./vendor/discord.js-selfbot-v13/
|
|
||||||
RUN pnpm install --no-frozen-lockfile
|
|
||||||
|
|
||||||
COPY . .
|
|
||||||
|
|
||||||
RUN pnpm run prepare:vendor
|
|
||||||
RUN pnpm run build
|
|
||||||
|
|
||||||
ENV NODE_ENV=production
|
|
||||||
|
|
||||||
CMD ["pnpm", "run", "start"]
|
|
||||||
@@ -0,0 +1,380 @@
|
|||||||
|
# Discord Moderation Watcher Bot - Microservices Architecture
|
||||||
|
|
||||||
|
## Quick Start
|
||||||
|
|
||||||
|
### Prerequisites
|
||||||
|
- Docker & Docker Compose
|
||||||
|
- Node.js 20+
|
||||||
|
- pnpm 11+
|
||||||
|
- Discord bot token
|
||||||
|
- OpenAI API key
|
||||||
|
|
||||||
|
### Environment Setup
|
||||||
|
|
||||||
|
Create `.env.local` in the root directory:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Discord Configuration
|
||||||
|
DISCORD_TOKEN=your_discord_token_here
|
||||||
|
MONITOR_GUILD_ID=your_guild_id_here
|
||||||
|
|
||||||
|
# AI Configuration
|
||||||
|
AI_LLM_API_KEY=your_openai_api_key_here
|
||||||
|
|
||||||
|
# Optional: Database URL (defaults to PostgreSQL in Docker)
|
||||||
|
DATABASE_URL=postgresql://postgres:postgres@localhost:5432/bete
|
||||||
|
|
||||||
|
# Optional: Redis URL (defaults to Redis in Docker)
|
||||||
|
REDIS_URL=redis://localhost:6379
|
||||||
|
```
|
||||||
|
|
||||||
|
### Local Development with Docker Compose
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Start all services
|
||||||
|
docker-compose up -d
|
||||||
|
|
||||||
|
# View logs
|
||||||
|
docker-compose logs -f
|
||||||
|
|
||||||
|
# Stop all services
|
||||||
|
docker-compose down
|
||||||
|
|
||||||
|
# Rebuild services
|
||||||
|
docker-compose up -d --build
|
||||||
|
```
|
||||||
|
|
||||||
|
**Services will be available at:**
|
||||||
|
- Frontend: http://localhost:5173
|
||||||
|
- Backend API: http://localhost:3001
|
||||||
|
- Backend WebSocket: ws://localhost:3001
|
||||||
|
- PostgreSQL: localhost:5432
|
||||||
|
- Redis: localhost:6379
|
||||||
|
|
||||||
|
### Local Development without Docker
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Install dependencies
|
||||||
|
pnpm install
|
||||||
|
|
||||||
|
# Run database migrations
|
||||||
|
pnpm run db:migrate
|
||||||
|
|
||||||
|
# Start all services in separate terminals
|
||||||
|
|
||||||
|
# Terminal 1: Backend
|
||||||
|
cd services/backend
|
||||||
|
pnpm run dev
|
||||||
|
|
||||||
|
# Terminal 2: Discord Gateway
|
||||||
|
cd services/discord-gateway
|
||||||
|
pnpm run dev
|
||||||
|
|
||||||
|
# Terminal 3: Frontend
|
||||||
|
cd services/frontend
|
||||||
|
pnpm run dev:web
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Architecture Overview
|
||||||
|
|
||||||
|
### 3 Independent Microservices
|
||||||
|
|
||||||
|
#### 1. Frontend Service (`services/frontend/`)
|
||||||
|
- **Tech:** React 19, Vite, TanStack Query, WebSocket
|
||||||
|
- **Port:** 5173 (dev) / served by Backend (prod)
|
||||||
|
- **Responsibilities:**
|
||||||
|
- Dashboard UI (analytics, messages, voice, media)
|
||||||
|
- Real-time WebSocket connection to Backend
|
||||||
|
- API calls to Backend REST endpoints
|
||||||
|
- State management (React Query)
|
||||||
|
|
||||||
|
#### 2. Backend Service (`services/backend/`)
|
||||||
|
- **Tech:** Express, Drizzle ORM, PostgreSQL, Redis
|
||||||
|
- **Port:** 3001
|
||||||
|
- **Responsibilities:**
|
||||||
|
- REST API endpoints (`/api/*`)
|
||||||
|
- WebSocket server for real-time updates
|
||||||
|
- Database operations (PostgreSQL)
|
||||||
|
- Event orchestration from Discord Gateway
|
||||||
|
- Static file serving (built Frontend)
|
||||||
|
- Admin authentication
|
||||||
|
|
||||||
|
**Modular MVC Structure:**
|
||||||
|
```
|
||||||
|
services/backend/src/
|
||||||
|
├── shared/
|
||||||
|
│ ├── database/ → Drizzle ORM setup
|
||||||
|
│ ├── config/ → Environment config
|
||||||
|
│ ├── errors/ → Custom error classes
|
||||||
|
│ ├── middlewares/ → Express middlewares
|
||||||
|
│ ├── logger/ → Logging utilities
|
||||||
|
│ └── utils/ → Shared utilities
|
||||||
|
├── modules/
|
||||||
|
│ ├── messages/ → Message CRUD
|
||||||
|
│ ├── analytics/ → Analytics queries
|
||||||
|
│ ├── media/ → Media management
|
||||||
|
│ ├── voice/ → Voice recordings
|
||||||
|
│ └── health/ → Health checks
|
||||||
|
└── index.ts
|
||||||
|
```
|
||||||
|
|
||||||
|
#### 3. Discord Gateway Service (`services/discord-gateway/`)
|
||||||
|
- **Tech:** discord.js-selfbot-v13, @discordjs/voice, OpenAI API
|
||||||
|
- **Port:** None (internal service, no HTTP)
|
||||||
|
- **Responsibilities:**
|
||||||
|
- Discord client connection
|
||||||
|
- Message capture (create/edit/delete)
|
||||||
|
- Voice channel recording
|
||||||
|
- AI moderation analysis
|
||||||
|
- Attachment upload
|
||||||
|
- Event publishing to Backend (Redis pub/sub)
|
||||||
|
|
||||||
|
**Modular MVC Structure:**
|
||||||
|
```
|
||||||
|
services/discord-gateway/src/
|
||||||
|
├── shared/
|
||||||
|
│ ├── database/ → Drizzle ORM setup
|
||||||
|
│ ├── config/ → Environment config
|
||||||
|
│ ├── errors/ → Custom error classes
|
||||||
|
│ ├── logger/ → Logging utilities
|
||||||
|
│ └── utils/ → Shared utilities
|
||||||
|
├── modules/
|
||||||
|
│ ├── message-capture/ → Message listeners
|
||||||
|
│ ├── voice-recording/ → Voice recording
|
||||||
|
│ ├── ai-moderation/ → AI analysis
|
||||||
|
│ ├── attachment-upload/ → File uploads
|
||||||
|
│ └── event-broadcaster/ → Redis pub/sub
|
||||||
|
└── index.ts
|
||||||
|
```
|
||||||
|
|
||||||
|
### Shared Package (`packages/shared/`)
|
||||||
|
- **Types:** Common interfaces and data models
|
||||||
|
- **Errors:** Custom error classes
|
||||||
|
- **Logger:** Pino logger setup
|
||||||
|
- **Utils:** Pagination, validation, helpers
|
||||||
|
|
||||||
|
### Communication Patterns
|
||||||
|
|
||||||
|
**Frontend ↔ Backend:**
|
||||||
|
- REST API: `GET/POST /api/*` (HTTP)
|
||||||
|
- WebSocket: Real-time updates (JSON messages)
|
||||||
|
- Auth: Admin password header
|
||||||
|
|
||||||
|
**Backend ↔ Discord Gateway:**
|
||||||
|
- Redis pub/sub (low-latency, decoupled)
|
||||||
|
- Events: `discord:message:created`, `discord:voice:started`, etc.
|
||||||
|
- Backend subscribes and broadcasts to Frontend via WebSocket
|
||||||
|
|
||||||
|
**Shared Resources:**
|
||||||
|
- PostgreSQL: Both Backend and Discord Gateway
|
||||||
|
- Redis: Pub/sub and caching
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Development Workflow
|
||||||
|
|
||||||
|
### Adding a New API Endpoint
|
||||||
|
|
||||||
|
1. **Create module structure** (if new feature):
|
||||||
|
```bash
|
||||||
|
mkdir -p services/backend/src/modules/feature/{routes,controllers,services,repositories,schemas}
|
||||||
|
```
|
||||||
|
|
||||||
|
2. **Define schema** (`feature.schema.ts`):
|
||||||
|
```typescript
|
||||||
|
import { z } from 'zod';
|
||||||
|
|
||||||
|
export const createFeatureSchema = z.object({
|
||||||
|
name: z.string().min(1),
|
||||||
|
description: z.string().optional(),
|
||||||
|
});
|
||||||
|
```
|
||||||
|
|
||||||
|
3. **Create repository** (`feature.repository.ts`):
|
||||||
|
```typescript
|
||||||
|
export async function createFeature(data: CreateFeatureInput) {
|
||||||
|
return db.insert(features).values(data).returning();
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
4. **Create service** (`feature.service.ts`):
|
||||||
|
```typescript
|
||||||
|
export async function createFeatureService(data: CreateFeatureInput) {
|
||||||
|
// Business logic, validation, orchestration
|
||||||
|
return createFeature(data);
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
5. **Create controller** (`feature.controller.ts`):
|
||||||
|
```typescript
|
||||||
|
export async function createFeatureController(req: Request, res: Response) {
|
||||||
|
const data = createFeatureSchema.parse(req.body);
|
||||||
|
const result = await createFeatureService(data);
|
||||||
|
res.json(result);
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
6. **Create route** (`feature.route.ts`):
|
||||||
|
```typescript
|
||||||
|
router.post('/features', createFeatureController);
|
||||||
|
```
|
||||||
|
|
||||||
|
### Adding a New Discord Event
|
||||||
|
|
||||||
|
1. **Create module** in `services/discord-gateway/src/modules/event-name/`
|
||||||
|
|
||||||
|
2. **Register listener** in `index.ts`:
|
||||||
|
```typescript
|
||||||
|
client.on('eventName', async (data) => {
|
||||||
|
await handleEvent(data);
|
||||||
|
publishEvent('discord:event:name', data);
|
||||||
|
});
|
||||||
|
```
|
||||||
|
|
||||||
|
3. **Publish to Redis**:
|
||||||
|
```typescript
|
||||||
|
import { redis } from '../shared/redis';
|
||||||
|
|
||||||
|
redis.publish('discord:event:name', JSON.stringify(data));
|
||||||
|
```
|
||||||
|
|
||||||
|
4. **Subscribe in Backend** (`services/backend/src/ws/server.ts`):
|
||||||
|
```typescript
|
||||||
|
redis.subscribe('discord:event:name', (message) => {
|
||||||
|
broadcastToClients({ type: 'event_name', data: JSON.parse(message) });
|
||||||
|
});
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Testing
|
||||||
|
|
||||||
|
### Run All Tests
|
||||||
|
```bash
|
||||||
|
pnpm run test
|
||||||
|
```
|
||||||
|
|
||||||
|
### Run Tests for Specific Service
|
||||||
|
```bash
|
||||||
|
cd services/backend
|
||||||
|
pnpm run test
|
||||||
|
|
||||||
|
cd services/discord-gateway
|
||||||
|
pnpm run test
|
||||||
|
```
|
||||||
|
|
||||||
|
### Type Checking
|
||||||
|
```bash
|
||||||
|
pnpm run typecheck
|
||||||
|
```
|
||||||
|
|
||||||
|
### Linting
|
||||||
|
```bash
|
||||||
|
pnpm run lint
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Deployment
|
||||||
|
|
||||||
|
### Build Docker Images
|
||||||
|
```bash
|
||||||
|
docker-compose build
|
||||||
|
```
|
||||||
|
|
||||||
|
### Push to Container Registry
|
||||||
|
```bash
|
||||||
|
docker tag bete-backend ghcr.io/username/bete-backend:latest
|
||||||
|
docker push ghcr.io/username/bete-backend:latest
|
||||||
|
```
|
||||||
|
|
||||||
|
### Deploy to Production
|
||||||
|
See `.github/workflows/deploy.yml` for GitHub Actions CI/CD pipeline.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Troubleshooting
|
||||||
|
|
||||||
|
### Backend can't connect to PostgreSQL
|
||||||
|
```bash
|
||||||
|
# Check PostgreSQL is running
|
||||||
|
docker-compose ps postgres
|
||||||
|
|
||||||
|
# Check connection string
|
||||||
|
echo $DATABASE_URL
|
||||||
|
|
||||||
|
# Verify credentials
|
||||||
|
psql -h localhost -U postgres -d bete
|
||||||
|
```
|
||||||
|
|
||||||
|
### Discord Gateway not receiving events
|
||||||
|
```bash
|
||||||
|
# Check Redis connection
|
||||||
|
redis-cli ping
|
||||||
|
|
||||||
|
# Check Discord token
|
||||||
|
echo $DISCORD_TOKEN
|
||||||
|
|
||||||
|
# View logs
|
||||||
|
docker-compose logs discord-gateway
|
||||||
|
```
|
||||||
|
|
||||||
|
### Frontend can't connect to Backend
|
||||||
|
```bash
|
||||||
|
# Check Backend is running
|
||||||
|
curl http://localhost:3001/health
|
||||||
|
|
||||||
|
# Check WebSocket connection
|
||||||
|
# Open browser DevTools → Network → WS
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## API Documentation
|
||||||
|
|
||||||
|
### Health Check
|
||||||
|
```bash
|
||||||
|
GET /health
|
||||||
|
```
|
||||||
|
|
||||||
|
### Messages
|
||||||
|
```bash
|
||||||
|
GET /api/messages?channel=<id>&type=text|image
|
||||||
|
POST /api/messages (admin only)
|
||||||
|
```
|
||||||
|
|
||||||
|
### Analytics
|
||||||
|
```bash
|
||||||
|
GET /api/analytics
|
||||||
|
```
|
||||||
|
|
||||||
|
### Voice Recordings
|
||||||
|
```bash
|
||||||
|
GET /api/recordings
|
||||||
|
```
|
||||||
|
|
||||||
|
### WebSocket Events
|
||||||
|
```
|
||||||
|
message_created
|
||||||
|
message_updated
|
||||||
|
message_deleted
|
||||||
|
attachment_uploaded
|
||||||
|
user_state
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Contributing
|
||||||
|
|
||||||
|
1. Create a feature branch
|
||||||
|
2. Make changes following Modular MVC pattern
|
||||||
|
3. Run tests and linting
|
||||||
|
4. Submit PR with description
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## License
|
||||||
|
|
||||||
|
MIT
|
||||||
@@ -3,6 +3,9 @@
|
|||||||
"includes": [
|
"includes": [
|
||||||
"src/**/*.ts",
|
"src/**/*.ts",
|
||||||
"tests/**/*.ts",
|
"tests/**/*.ts",
|
||||||
|
"services/**/*.ts",
|
||||||
|
"services/**/*.tsx",
|
||||||
|
"packages/**/*.ts",
|
||||||
"*.json",
|
"*.json",
|
||||||
"*.ts",
|
"*.ts",
|
||||||
"!vendor/**",
|
"!vendor/**",
|
||||||
|
|||||||
@@ -1,60 +0,0 @@
|
|||||||
import type { ChildProcess } from "node:child_process";
|
|
||||||
import dotenv from "dotenv";
|
|
||||||
import { createYtDlp } from "./src/media/ytdlp.js";
|
|
||||||
import { prepareStream } from "./src/streaming/index.js";
|
|
||||||
|
|
||||||
dotenv.config();
|
|
||||||
|
|
||||||
async function test() {
|
|
||||||
const ytdlp = createYtDlp();
|
|
||||||
const url = "https://www.youtube.com/watch?v=aqz-KE-bpKQ"; // Small video
|
|
||||||
|
|
||||||
console.log("Getting direct video url...");
|
|
||||||
const directUrl = await ytdlp.getDirectVideoUrl(url);
|
|
||||||
console.log("Direct URL:", directUrl);
|
|
||||||
|
|
||||||
console.log("Preparing stream...");
|
|
||||||
const { command, output } = prepareStream(directUrl, {
|
|
||||||
logLevel: "debug",
|
|
||||||
customInputOptions: [
|
|
||||||
"-headers",
|
|
||||||
"User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.3\r\nConnection: keep-alive\r\n",
|
|
||||||
],
|
|
||||||
});
|
|
||||||
|
|
||||||
const ffmpeg = command as ChildProcess;
|
|
||||||
ffmpeg.stderr?.on("data", (data: Buffer) => {
|
|
||||||
console.log("FFMPEG STDERR:", data.toString());
|
|
||||||
});
|
|
||||||
|
|
||||||
let bytesRead = 0;
|
|
||||||
output.on("data", (chunk: Buffer) => {
|
|
||||||
bytesRead += chunk.length;
|
|
||||||
console.log("Stream bytes:", bytesRead);
|
|
||||||
if (bytesRead > 1024 * 1024) {
|
|
||||||
ffmpeg.kill("SIGTERM");
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
try {
|
|
||||||
await new Promise<void>((resolve, reject) => {
|
|
||||||
ffmpeg.on("exit", (code) => {
|
|
||||||
if (code === 0 || code === null) {
|
|
||||||
resolve();
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
reject(new Error(`ffmpeg exited with code ${code}`));
|
|
||||||
});
|
|
||||||
ffmpeg.on("error", reject);
|
|
||||||
});
|
|
||||||
} catch (error: unknown) {
|
|
||||||
console.error(
|
|
||||||
"Debug stream failed:",
|
|
||||||
error instanceof Error ? error.message : String(error),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
process.exit(0);
|
|
||||||
}
|
|
||||||
|
|
||||||
test();
|
|
||||||
@@ -1,28 +0,0 @@
|
|||||||
services:
|
|
||||||
app:
|
|
||||||
build: .
|
|
||||||
image: ${IMAGE_NAME:-ghcr.io/mytheclipse/gmw:latest}
|
|
||||||
restart: unless-stopped
|
|
||||||
env_file:
|
|
||||||
- .env
|
|
||||||
volumes:
|
|
||||||
- ./recordings:/app/recordings
|
|
||||||
# Mapping SQLite database files if needed, or storing them in a dedicated volume.
|
|
||||||
# Assuming default config uses root directory for DB.
|
|
||||||
- ./.muxer-queue.db:/app/.muxer-queue.db
|
|
||||||
- ./.muxer-queue.db-shm:/app/.muxer-queue.db-shm
|
|
||||||
- ./.muxer-queue.db-wal:/app/.muxer-queue.db-wal
|
|
||||||
labels:
|
|
||||||
- "traefik.enable=true"
|
|
||||||
- "traefik.http.routers.imphenbot.rule=Host(`imphnen.asepharyana.my.id`)"
|
|
||||||
- "traefik.http.routers.imphenbot.entrypoints=websecure"
|
|
||||||
- "traefik.http.routers.imphenbot.tls=true"
|
|
||||||
# Expose port to traefik (adjust if WEBSERVER_PORT differs)
|
|
||||||
- "traefik.http.services.imphenbot.loadbalancer.server.port=3000"
|
|
||||||
networks:
|
|
||||||
- app-shared-net
|
|
||||||
|
|
||||||
networks:
|
|
||||||
app-shared-net:
|
|
||||||
name: app-shared-net
|
|
||||||
external: true
|
|
||||||
@@ -1,137 +0,0 @@
|
|||||||
import { useEffect, useMemo, useState } from "react";
|
|
||||||
import { Component, Suspense, lazy } from "react";
|
|
||||||
import { DashboardLayout } from "./widgets/DashboardLayout";
|
|
||||||
import { MobileTabBar } from "./shared/ui/MobileTabBar";
|
|
||||||
import { AuthOverlay } from "./features/auth";
|
|
||||||
import { LivePanel } from "./features/live";
|
|
||||||
import { MessagesPanel } from "./features/messages";
|
|
||||||
import { useDashboardSocket } from "./shared/ws/socket";
|
|
||||||
import { mergeMessages, useMessages } from "./features/messages/hooks/useMessages";
|
|
||||||
import { useMediaControl } from "./features/live/hooks/useMediaControl";
|
|
||||||
import { useUIState } from "./shared/hooks/useUIState";
|
|
||||||
import { useVoiceControl } from "./features/live/hooks/useVoiceControl";
|
|
||||||
import { useAudioPlayback } from "./shared/hooks/useAudioPlayback";
|
|
||||||
import { useAudioTransmit } from "./shared/hooks/useAudioTransmit";
|
|
||||||
import { getAppConfig, type MessageRecord, type ActiveSpeaker, type MediaState } from "./shared/api/client";
|
|
||||||
import { Skeleton } from "./shared/ui";
|
|
||||||
|
|
||||||
const AnalyticsPanel = lazy(() => import("./features/analytics").then((module) => ({ default: module.AnalyticsPanel })));
|
|
||||||
|
|
||||||
class AnalyticsErrorBoundary extends Component<{ children: React.ReactNode }, { hasError: boolean }> {
|
|
||||||
state = { hasError: false };
|
|
||||||
static getDerivedStateFromError() { return { hasError: true }; }
|
|
||||||
override render() {
|
|
||||||
if (this.state.hasError) {
|
|
||||||
return <div className="rounded-2xl border border-destructive/30 bg-destructive/10 p-6 text-sm text-destructive">Analytics failed to load. The rest of the dashboard is still available.</div>;
|
|
||||||
}
|
|
||||||
return this.props.children;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export default function App() {
|
|
||||||
const { uiState, patchUIState } = useUIState();
|
|
||||||
const voice = useVoiceControl();
|
|
||||||
const media = useMediaControl();
|
|
||||||
const messages = useMessages();
|
|
||||||
const [activeSpeakers, setActiveSpeakers] = useState<ActiveSpeaker[]>([]);
|
|
||||||
const [isAuthenticated, setIsAuthenticated] = useState(!!localStorage.getItem("admin-password"));
|
|
||||||
const [monitorGuildId, setMonitorGuildId] = useState("");
|
|
||||||
|
|
||||||
const audio = useAudioPlayback();
|
|
||||||
const activeTab = uiState.activeTab || "live";
|
|
||||||
const selectedVoiceGuild = uiState.selectedVoiceGuild || uiState.selectedGuild || "";
|
|
||||||
const selectedTextGuild = monitorGuildId || uiState.selectedTextGuild || uiState.selectedGuild || "";
|
|
||||||
const selectedTextChannel = uiState.selectedTextChannel || "";
|
|
||||||
const monitorGuild = useMemo(() => (monitorGuildId ? voice.guilds.find((g) => g.id === monitorGuildId) : undefined), [monitorGuildId, voice.guilds]);
|
|
||||||
|
|
||||||
const socket = useDashboardSocket({
|
|
||||||
onBinary: audio.handleIncomingPcm,
|
|
||||||
onUserState: (users) => setActiveSpeakers(users as ActiveSpeaker[]),
|
|
||||||
onMessageCreated: (m) => messages.setMessages((prev) => mergeMessages(prev, [m as MessageRecord])),
|
|
||||||
onMessageUpdated: (m) => {
|
|
||||||
const d = m as Partial<MessageRecord> & { id: string };
|
|
||||||
messages.setMessages((prev) => prev.map((i) => i.id === d.id ? { ...i, ...d } : i));
|
|
||||||
},
|
|
||||||
onMessageDeleted: (m) => {
|
|
||||||
const d = m as { id: string };
|
|
||||||
messages.setMessages((prev) => prev.map((i) => i.id === d.id ? { ...i, type: "deleted" as const } : i));
|
|
||||||
},
|
|
||||||
onMessageAnalyzed: (m) => messages.setMessages((prev) => mergeMessages(prev, [m as MessageRecord])),
|
|
||||||
onAttachmentUploaded: () => messages.fetchMessages(selectedTextChannel).catch(() => undefined),
|
|
||||||
onMediaState: (state) => media.setMediaState(state as MediaState),
|
|
||||||
onVoiceRecordingUploaded: (d) => window.dispatchEvent(new CustomEvent("voice_recording_uploaded", { detail: d })),
|
|
||||||
});
|
|
||||||
|
|
||||||
const transmit = useAudioTransmit(socket.socketRef);
|
|
||||||
|
|
||||||
useEffect(() => {
|
|
||||||
getAppConfig().then((c) => {
|
|
||||||
if (c.monitorGuildId) {
|
|
||||||
setMonitorGuildId(c.monitorGuildId);
|
|
||||||
patchUIState({ selectedTextGuild: c.monitorGuildId, selectedAnalyticsGuild: c.monitorGuildId, selectedTextChannel: "", selectedAnalyticsChannel: "" });
|
|
||||||
}
|
|
||||||
}).catch(() => undefined);
|
|
||||||
}, [patchUIState]);
|
|
||||||
|
|
||||||
useEffect(() => { if (selectedVoiceGuild) voice.loadVoiceChannels(selectedVoiceGuild).catch(() => undefined); }, [selectedVoiceGuild, voice.loadVoiceChannels]);
|
|
||||||
useEffect(() => { if (monitorGuildId) voice.loadTextTargets(monitorGuildId).catch(() => undefined); }, [monitorGuildId, voice.loadTextTargets]);
|
|
||||||
useEffect(() => { if (selectedTextChannel) messages.fetchMessages(selectedTextChannel).catch(() => undefined); }, [selectedTextChannel, messages.fetchMessages]);
|
|
||||||
|
|
||||||
// Periodic refetch — ensures dashboard stays in sync even if WS events were missed
|
|
||||||
useEffect(() => {
|
|
||||||
if (!selectedTextChannel) return;
|
|
||||||
const interval = setInterval(() => {
|
|
||||||
messages.fetchMessages(selectedTextChannel).catch(() => undefined);
|
|
||||||
}, 15_000); // every 15s (longer than WS, shorter than stale cache)
|
|
||||||
return () => clearInterval(interval);
|
|
||||||
}, [selectedTextChannel, messages.fetchMessages]);
|
|
||||||
|
|
||||||
return (
|
|
||||||
<DashboardLayout activeTab={activeTab} wsStatus={socket.status} voiceStatus={voice.voiceStatus} onTabChange={(tab) => patchUIState({ activeTab: tab })}>
|
|
||||||
{activeTab === "live" ? (
|
|
||||||
!isAuthenticated ? (
|
|
||||||
<AuthOverlay onAuthenticated={() => setIsAuthenticated(true)} />
|
|
||||||
) : (
|
|
||||||
<LivePanel
|
|
||||||
guilds={voice.guilds} voiceChannels={voice.voiceChannels} selectedGuild={selectedVoiceGuild} selectedChannel={uiState.selectedVoiceChannel || ""}
|
|
||||||
status={voice.voiceStatus} voiceLoading={voice.loading} activeSpeakers={activeSpeakers}
|
|
||||||
levels={audio.levels} isListening={audio.isListening} isStreaming={transmit.isStreaming}
|
|
||||||
mediaState={media.mediaState} mediaLoading={media.loading}
|
|
||||||
onGuildChange={(id) => patchUIState({ selectedVoiceGuild: id, selectedVoiceChannel: "" })}
|
|
||||||
onChannelChange={(id) => patchUIState({ selectedVoiceChannel: id })}
|
|
||||||
onJoin={() => voice.joinVoice(selectedVoiceGuild, uiState.selectedVoiceChannel || "")}
|
|
||||||
onDisconnect={() => voice.leaveVoice()}
|
|
||||||
onListenToggle={audio.toggleListening} onStreamingToggle={transmit.toggle}
|
|
||||||
onQueueMusic={(s) => media.enqueue(s, "music")} onStartScreen={(s) => media.enqueue(s, "screen")}
|
|
||||||
onSkip={media.skip} onStop={media.stop} onVolumeChange={media.setVolume}
|
|
||||||
/>
|
|
||||||
)
|
|
||||||
) : activeTab === "messages" ? (
|
|
||||||
<MessagesPanel
|
|
||||||
guilds={monitorGuild ? [monitorGuild] : []} channels={voice.textChannels}
|
|
||||||
selectedGuild={selectedTextGuild} selectedChannel={selectedTextChannel}
|
|
||||||
messages={messages.messages}
|
|
||||||
onGuildChange={(id) => patchUIState({ selectedTextGuild: id, selectedTextChannel: "" })}
|
|
||||||
onChannelChange={(id) => patchUIState({ selectedTextChannel: id })}
|
|
||||||
onReanalyze={messages.reanalyze}
|
|
||||||
onLoadMore={messages.loadMore}
|
|
||||||
hasMore={messages.hasMore}
|
|
||||||
loadingMore={messages.loadingMore}
|
|
||||||
/>
|
|
||||||
) : (
|
|
||||||
<AnalyticsErrorBoundary>
|
|
||||||
<Suspense fallback={<div className="flex flex-col gap-4">{Array.from({ length: 8 }).map((_, i) => <Skeleton key={i} className="h-16 w-full rounded-xl" />)}<Skeleton className="h-64 w-full rounded-xl" /></div>}>
|
|
||||||
<AnalyticsPanel
|
|
||||||
guilds={monitorGuild ? [monitorGuild] : []} channels={voice.textChannels}
|
|
||||||
selectedGuild={uiState.selectedAnalyticsGuild || selectedTextGuild || ""}
|
|
||||||
selectedChannel={uiState.selectedAnalyticsChannel || selectedTextChannel || ""}
|
|
||||||
onGuildChange={(id) => patchUIState({ selectedAnalyticsGuild: id, selectedAnalyticsChannel: "" })}
|
|
||||||
onChannelChange={(id) => patchUIState({ selectedAnalyticsChannel: id })}
|
|
||||||
/>
|
|
||||||
</Suspense>
|
|
||||||
</AnalyticsErrorBoundary>
|
|
||||||
)}
|
|
||||||
<MobileTabBar activeTab={activeTab} onTabChange={(tab) => patchUIState({ activeTab: tab })} />
|
|
||||||
</DashboardLayout>
|
|
||||||
);
|
|
||||||
}
|
|
||||||
@@ -1,54 +0,0 @@
|
|||||||
import type { ModerationBreakdown } from "../../../shared/api/client";
|
|
||||||
import { Card, CardContent, Skeleton } from "../../../shared/ui";
|
|
||||||
import { cn } from "../../../shared/lib/utils";
|
|
||||||
|
|
||||||
interface SummaryCardsProps {
|
|
||||||
messages: ModerationBreakdown | null;
|
|
||||||
activeUsersCount: number;
|
|
||||||
totalChannels: number;
|
|
||||||
loading: boolean;
|
|
||||||
}
|
|
||||||
|
|
||||||
export function SummaryCards({ messages, activeUsersCount, totalChannels, loading }: SummaryCardsProps) {
|
|
||||||
const avgPerHour = messages ? Math.round(messages.total / Math.max(1, 24)) : 0;
|
|
||||||
const cleanPct = messages && messages.total > 0 ? Math.round((messages.clean / messages.total) * 100) : 0;
|
|
||||||
const warnedPct = messages && messages.total > 0 ? Math.round((messages.warned / messages.total) * 100) : 0;
|
|
||||||
const flaggedPct = messages && messages.total > 0 ? Math.round((messages.flagged / messages.total) * 100) : 0;
|
|
||||||
|
|
||||||
const cards = [
|
|
||||||
{ label: "Total Pesan", value: formatNum(messages?.total), accent: "text-foreground" },
|
|
||||||
{ label: "Rata-rata/jam", value: formatNum(avgPerHour), accent: "text-muted-foreground" },
|
|
||||||
{ label: "Clean", value: cleanPct > 0 ? `${cleanPct}%` : "—", accent: "text-emerald-400" },
|
|
||||||
{ label: "Warned", value: warnedPct > 0 ? `${warnedPct}%` : "—", accent: "text-amber-400" },
|
|
||||||
{ label: "Flagged", value: flaggedPct > 0 ? `${flaggedPct}%` : "—", accent: "text-red-400" },
|
|
||||||
{ label: "Pending", value: formatNum(messages?.pending), accent: "text-slate-400" },
|
|
||||||
{ label: "User Aktif", value: formatNum(activeUsersCount), accent: "text-violet-400" },
|
|
||||||
{ label: "Channel", value: formatNum(totalChannels), accent: "text-blue-400" },
|
|
||||||
];
|
|
||||||
|
|
||||||
return (
|
|
||||||
<div className="grid grid-cols-2 gap-2 sm:grid-cols-4 lg:grid-cols-8">
|
|
||||||
{cards.map((card) => (
|
|
||||||
<Card key={card.label} className="overflow-hidden glass border-white/5">
|
|
||||||
<CardContent className="p-3">
|
|
||||||
<div className="text-[10px] font-medium uppercase tracking-wider text-muted-foreground">
|
|
||||||
{card.label}
|
|
||||||
</div>
|
|
||||||
<div className={cn("mt-1 font-mono text-lg font-bold tabular-nums", card.accent)}>
|
|
||||||
{loading ? (
|
|
||||||
<Skeleton className="h-7 w-12 mt-1" />
|
|
||||||
) : (
|
|
||||||
card.value
|
|
||||||
)}
|
|
||||||
</div>
|
|
||||||
</CardContent>
|
|
||||||
</Card>
|
|
||||||
))}
|
|
||||||
</div>
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
function formatNum(v: number | undefined | null): string {
|
|
||||||
if (v == null || v === 0) return "—";
|
|
||||||
return v.toLocaleString("id-ID");
|
|
||||||
}
|
|
||||||
@@ -1,56 +0,0 @@
|
|||||||
import { useState } from "react";
|
|
||||||
import type { Channel, Guild } from "../../shared/api/client";
|
|
||||||
import { useAnalytics } from "./hooks/useAnalytics";
|
|
||||||
import { ControlBar } from "./components/ControlBar";
|
|
||||||
import { SummaryCards } from "./components/SummaryCards";
|
|
||||||
import { ActivityChart } from "./components/ActivityChart";
|
|
||||||
import { TrendChart } from "./components/TrendChart";
|
|
||||||
import { Heatmap } from "./components/Heatmap";
|
|
||||||
import { TopicList } from "./components/TopicList";
|
|
||||||
import { UserTable } from "./components/UserTable";
|
|
||||||
import { ViolatorTable } from "./components/ViolatorTable";
|
|
||||||
|
|
||||||
interface AnalyticsPanelProps {
|
|
||||||
guilds: Guild[];
|
|
||||||
channels: Channel[];
|
|
||||||
selectedGuild: string;
|
|
||||||
selectedChannel: string;
|
|
||||||
onGuildChange: (guildId: string) => void;
|
|
||||||
onChannelChange: (channelId: string) => void;
|
|
||||||
}
|
|
||||||
|
|
||||||
export function AnalyticsPanel({
|
|
||||||
guilds, channels, selectedGuild, selectedChannel,
|
|
||||||
onGuildChange, onChannelChange,
|
|
||||||
}: AnalyticsPanelProps) {
|
|
||||||
const [hours, setHours] = useState(24);
|
|
||||||
const analytics = useAnalytics({ guildId: selectedGuild, channelId: selectedChannel || undefined, hours });
|
|
||||||
|
|
||||||
const { hourly, topics, topUsers, activeUsersCount, totalChannels, violators, trend, heatmap, isLoading, isFetching, error, refresh, refreshViolators, messages: analyticsMessages } = analytics;
|
|
||||||
const loading = isLoading && !isFetching;
|
|
||||||
|
|
||||||
if (error && !analyticsMessages) {
|
|
||||||
return <div className="rounded-lg border border-red-500/30 bg-red-500/5 p-4 text-sm text-red-300">{error}</div>;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!selectedGuild) {
|
|
||||||
return <div className="flex min-h-[300px] flex-col items-center justify-center gap-3 rounded-lg border border-dashed p-8"><p className="text-sm text-muted-foreground">Pilih guild untuk melihat analitik.</p></div>;
|
|
||||||
}
|
|
||||||
|
|
||||||
return (
|
|
||||||
<div className="flex flex-col gap-4">
|
|
||||||
<ControlBar guilds={guilds} channels={channels} selectedGuild={selectedGuild} selectedChannel={selectedChannel} hours={hours} isFetching={isFetching} onGuildChange={onGuildChange} onChannelChange={onChannelChange} onHoursChange={setHours} onRefresh={() => { refresh(); refreshViolators(); }} />
|
|
||||||
<SummaryCards messages={analyticsMessages} activeUsersCount={activeUsersCount} totalChannels={totalChannels} loading={loading} />
|
|
||||||
<div className="grid grid-cols-3 gap-4">
|
|
||||||
<ActivityChart hourly={hourly} loading={loading} />
|
|
||||||
<div className="col-span-1"><TopicList topics={topics} loading={loading} /></div>
|
|
||||||
</div>
|
|
||||||
{hours >= 48 && <TrendChart trend={trend} loading={loading} />}
|
|
||||||
<div className="grid grid-cols-3 gap-4">
|
|
||||||
<Heatmap cells={heatmap} loading={loading} />
|
|
||||||
<div className="col-span-1"><UserTable users={topUsers} loading={loading} /></div>
|
|
||||||
</div>
|
|
||||||
<ViolatorTable users={violators} loading={loading} />
|
|
||||||
</div>
|
|
||||||
);
|
|
||||||
}
|
|
||||||
@@ -1,62 +0,0 @@
|
|||||||
import { Button, Select } from "../../../shared/ui";
|
|
||||||
import type { Channel, Guild, VoiceStatus } from "../../../shared/api/client";
|
|
||||||
import { Radio, Headphones } from "lucide-react";
|
|
||||||
|
|
||||||
interface VoiceConnectionCardProps {
|
|
||||||
guilds: Guild[];
|
|
||||||
voiceChannels: Channel[];
|
|
||||||
selectedGuild: string;
|
|
||||||
selectedChannel: string;
|
|
||||||
status: VoiceStatus;
|
|
||||||
voiceLoading: boolean;
|
|
||||||
isListening: boolean;
|
|
||||||
isStreaming: boolean;
|
|
||||||
onGuildChange: (id: string) => void;
|
|
||||||
onChannelChange: (id: string) => void;
|
|
||||||
onJoin: () => void;
|
|
||||||
onDisconnect: () => void;
|
|
||||||
onListenToggle: () => void;
|
|
||||||
onStreamingToggle: () => void;
|
|
||||||
}
|
|
||||||
|
|
||||||
export function VoiceConnectionCard({
|
|
||||||
guilds, voiceChannels, selectedGuild, selectedChannel,
|
|
||||||
status, voiceLoading, isListening, isStreaming,
|
|
||||||
onGuildChange, onChannelChange, onJoin, onDisconnect,
|
|
||||||
onListenToggle, onStreamingToggle,
|
|
||||||
}: VoiceConnectionCardProps) {
|
|
||||||
return (
|
|
||||||
<div className="rounded-2xl border border-border bg-card shadow-sm">
|
|
||||||
<div className="p-6">
|
|
||||||
<h3 className="flex items-center gap-2 text-lg font-semibold tracking-tight">
|
|
||||||
<Radio className="h-5 w-5" /> Voice Bridge
|
|
||||||
</h3>
|
|
||||||
<p className="mt-1 text-sm text-muted-foreground">Join a Discord voice channel, listen, and transmit audio.</p>
|
|
||||||
|
|
||||||
<div className="mt-4 grid gap-4 md:grid-cols-2">
|
|
||||||
<div className="space-y-2">
|
|
||||||
<label className="text-sm font-medium">Guild</label>
|
|
||||||
<Select value={selectedGuild} onChange={(e) => onGuildChange(e.target.value)} placeholder="Select guild" options={guilds.map((g) => ({ value: g.id, label: g.name }))} />
|
|
||||||
</div>
|
|
||||||
<div className="space-y-2">
|
|
||||||
<label className="text-sm font-medium">Voice Channel</label>
|
|
||||||
<Select value={selectedChannel} onChange={(e) => onChannelChange(e.target.value)} placeholder="Select voice channel" options={voiceChannels.map((c) => ({ value: c.id, label: c.name }))} />
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<div className="mt-4 flex flex-wrap gap-2">
|
|
||||||
<Button disabled={!selectedGuild || !selectedChannel || voiceLoading} onClick={onJoin}>
|
|
||||||
{status.connected ? "Reconnect" : "Join Voice"}
|
|
||||||
</Button>
|
|
||||||
<Button variant="destructive" disabled={!status.connected || voiceLoading} onClick={onDisconnect}>Disconnect</Button>
|
|
||||||
<Button variant={isListening ? "secondary" : "outline"} onClick={onListenToggle}>
|
|
||||||
<Headphones className="mr-1.5 h-4 w-4" /> {isListening ? "Stop Listening" : "Listen"}
|
|
||||||
</Button>
|
|
||||||
<Button variant={isStreaming ? "destructive" : "default"} onClick={onStreamingToggle}>
|
|
||||||
<Radio className="mr-1.5 h-4 w-4" /> {isStreaming ? "Stop Transmit" : "Transmit"}
|
|
||||||
</Button>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
);
|
|
||||||
}
|
|
||||||
@@ -1,149 +0,0 @@
|
|||||||
import { useState, useMemo } from "react";
|
|
||||||
import type { Channel, Guild, MessageRecord } from "../../shared/api/client";
|
|
||||||
import { Card, CardContent, CardDescription, CardHeader, CardTitle, Badge, Button, Input, Select, Tabs, TabsContent, TabsList, TabsTrigger } from "../../shared/ui";
|
|
||||||
import { MessageFeed } from "./components/MessageFeed";
|
|
||||||
import { ImageGrid } from "./components/ImageGrid";
|
|
||||||
import { Search, X, Filter } from "lucide-react";
|
|
||||||
|
|
||||||
interface MessagesPanelProps {
|
|
||||||
guilds: Guild[];
|
|
||||||
channels: Channel[];
|
|
||||||
selectedGuild: string;
|
|
||||||
selectedChannel: string;
|
|
||||||
messages: MessageRecord[];
|
|
||||||
onGuildChange: (guildId: string) => void;
|
|
||||||
onChannelChange: (channelId: string) => void;
|
|
||||||
onReanalyze: (id: string) => Promise<void>;
|
|
||||||
onLoadMore?: () => void;
|
|
||||||
hasMore?: boolean;
|
|
||||||
loadingMore?: boolean;
|
|
||||||
}
|
|
||||||
|
|
||||||
type AiFilter = "all" | "clean" | "warn" | "flagged" | "error" | "pending";
|
|
||||||
|
|
||||||
export function MessagesPanel({
|
|
||||||
guilds, channels, selectedGuild, selectedChannel,
|
|
||||||
messages, onGuildChange, onChannelChange, onReanalyze,
|
|
||||||
onLoadMore, hasMore, loadingMore,
|
|
||||||
}: MessagesPanelProps) {
|
|
||||||
const [searchQuery, setSearchQuery] = useState("");
|
|
||||||
const [searchResults, setSearchResults] = useState<MessageRecord[]>([]);
|
|
||||||
const [isSearching, setIsSearching] = useState(false);
|
|
||||||
const [showSearch, setShowSearch] = useState(false);
|
|
||||||
const [aiFilter, setAiFilter] = useState<AiFilter>("all");
|
|
||||||
const [viewTab, setViewTab] = useState<"all" | "images">("all");
|
|
||||||
|
|
||||||
const handleSearch = async () => {
|
|
||||||
if (!searchQuery.trim()) { setSearchResults([]); setShowSearch(false); return; }
|
|
||||||
setIsSearching(true);
|
|
||||||
try {
|
|
||||||
const params = new URLSearchParams({ q: searchQuery, ...(selectedChannel && { channelId: selectedChannel }), limit: "50" });
|
|
||||||
const response = await fetch(`/api/analysis/search?${params}`);
|
|
||||||
if (!response.ok) throw new Error("Search failed");
|
|
||||||
const data = await response.json();
|
|
||||||
setSearchResults(data.results || []);
|
|
||||||
setShowSearch(true);
|
|
||||||
} catch {
|
|
||||||
setSearchResults([]);
|
|
||||||
} finally {
|
|
||||||
setIsSearching(false);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
const stats = useMemo(() => {
|
|
||||||
const base = showSearch ? searchResults : messages;
|
|
||||||
return {
|
|
||||||
total: base.length,
|
|
||||||
clean: base.filter((m) => m.ai_status === "clean").length,
|
|
||||||
warn: base.filter((m) => m.ai_status === "warn").length,
|
|
||||||
flagged: base.filter((m) => m.ai_status === "flagged").length,
|
|
||||||
error: base.filter((m) => m.ai_status === "error").length,
|
|
||||||
pending: base.filter((m) => m.ai_status === "pending" || !m.ai_status).length,
|
|
||||||
deleted: base.filter((m) => m.deleted_at).length,
|
|
||||||
edited: base.filter((m) => m.edited_at).length,
|
|
||||||
};
|
|
||||||
}, [messages, searchResults, showSearch]);
|
|
||||||
|
|
||||||
const filteredMessages = useMemo(() => {
|
|
||||||
const base = showSearch ? searchResults : messages;
|
|
||||||
if (aiFilter === "all") return base;
|
|
||||||
return base.filter((m) => {
|
|
||||||
const status = m.ai_status ?? "pending";
|
|
||||||
if (aiFilter === "pending") return status === "pending" || status === null || status === undefined;
|
|
||||||
return status === aiFilter;
|
|
||||||
});
|
|
||||||
}, [messages, searchResults, showSearch, aiFilter]);
|
|
||||||
|
|
||||||
return (
|
|
||||||
<div className="grid gap-6">
|
|
||||||
<Card>
|
|
||||||
<CardHeader>
|
|
||||||
<CardTitle>Message Source</CardTitle>
|
|
||||||
<CardDescription>Pick a guild and channel/thread to inspect captures.</CardDescription>
|
|
||||||
</CardHeader>
|
|
||||||
<CardContent className="grid gap-4 md:grid-cols-2">
|
|
||||||
<Select value={selectedGuild} onChange={(e) => onGuildChange(e.target.value)} placeholder="Select text guild" options={guilds.map((g) => ({ value: g.id, label: g.name }))} />
|
|
||||||
<Select value={selectedChannel} onChange={(e) => onChannelChange(e.target.value)} placeholder="Select channel or thread" options={channels.map((c) => ({ value: c.id, label: c.name }))} />
|
|
||||||
</CardContent>
|
|
||||||
</Card>
|
|
||||||
|
|
||||||
{stats.total > 0 && (
|
|
||||||
<div className="flex flex-wrap items-center gap-2">
|
|
||||||
<Badge variant="secondary" className="text-xs">{stats.total} total{hasMore && !showSearch ? "+" : ""}</Badge>
|
|
||||||
<Badge variant="outline" className="text-xs text-green-400 border-green-400/30">{stats.clean} clean</Badge>
|
|
||||||
<Badge variant="outline" className="text-xs text-yellow-400 border-yellow-400/30">{stats.warn} warn</Badge>
|
|
||||||
<Badge variant="outline" className="text-xs text-red-400 border-red-400/30">{stats.flagged} flagged</Badge>
|
|
||||||
<Badge variant="outline" className="text-xs text-orange-400 border-orange-400/30">{stats.error} error</Badge>
|
|
||||||
<Badge variant="outline" className="text-xs">{stats.pending} pending</Badge>
|
|
||||||
{stats.deleted > 0 && <Badge variant="destructive" className="text-xs">{stats.deleted} deleted</Badge>}
|
|
||||||
{stats.edited > 0 && <Badge variant="outline" className="text-xs">{stats.edited} edited</Badge>}
|
|
||||||
</div>
|
|
||||||
)}
|
|
||||||
|
|
||||||
<div className="flex flex-wrap items-center gap-2">
|
|
||||||
<div className="relative flex-1 min-w-[200px]">
|
|
||||||
<Search className="absolute left-3 top-1/2 h-4 w-4 -translate-y-1/2 text-muted-foreground" />
|
|
||||||
<Input className="pl-9" placeholder="Search message content..." value={searchQuery} onChange={(e) => setSearchQuery(e.target.value)} onKeyDown={(e) => e.key === "Enter" && handleSearch()} disabled={isSearching} />
|
|
||||||
</div>
|
|
||||||
<Button onClick={handleSearch} disabled={isSearching || !searchQuery.trim()} size="sm">{isSearching ? "Searching..." : "Search"}</Button>
|
|
||||||
{showSearch && (
|
|
||||||
<Button variant="outline" size="sm" onClick={() => { setShowSearch(false); setSearchResults([]); setSearchQuery(""); }}>
|
|
||||||
<X className="mr-1 h-3 w-3" /> Clear
|
|
||||||
</Button>
|
|
||||||
)}
|
|
||||||
<div className="ml-auto flex items-center gap-1.5">
|
|
||||||
<Filter className="h-4 w-4 text-muted-foreground" />
|
|
||||||
{(["all", "clean", "warn", "flagged", "error", "pending"] as AiFilter[]).map((f) => (
|
|
||||||
<button key={f} onClick={() => setAiFilter(f)} className={`rounded-md px-2 py-1 text-xs font-medium transition-colors ${aiFilter === f ? "bg-primary text-primary-foreground" : "text-muted-foreground hover:text-foreground hover:bg-muted"}`}>
|
|
||||||
{f}
|
|
||||||
</button>
|
|
||||||
))}
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
{showSearch && searchResults.length > 0 && (
|
|
||||||
<div className="text-sm text-muted-foreground">Found {searchResults.length} result{searchResults.length !== 1 ? "s" : ""}</div>
|
|
||||||
)}
|
|
||||||
|
|
||||||
<Tabs value={viewTab} onValueChange={(v) => setViewTab(v as "all" | "images")}>
|
|
||||||
<TabsList>
|
|
||||||
<TabsTrigger value="all">{showSearch ? `Search (${filteredMessages.length})` : `All (${filteredMessages.length})`}</TabsTrigger>
|
|
||||||
<TabsTrigger value="images">Images</TabsTrigger>
|
|
||||||
</TabsList>
|
|
||||||
<TabsContent value="all">
|
|
||||||
<MessageFeed
|
|
||||||
messages={filteredMessages}
|
|
||||||
onReanalyze={onReanalyze}
|
|
||||||
emptyText={showSearch ? "No messages found matching your search." : selectedChannel ? "No captures yet." : "Select a channel to view captures."}
|
|
||||||
onLoadMore={showSearch ? undefined : onLoadMore}
|
|
||||||
hasMore={showSearch ? false : hasMore}
|
|
||||||
loadingMore={loadingMore}
|
|
||||||
/>
|
|
||||||
</TabsContent>
|
|
||||||
<TabsContent value="images">
|
|
||||||
<ImageGrid messages={filteredMessages} />
|
|
||||||
</TabsContent>
|
|
||||||
</Tabs>
|
|
||||||
</div>
|
|
||||||
);
|
|
||||||
}
|
|
||||||
@@ -1,57 +0,0 @@
|
|||||||
// ─── Audio playback hook — receives PCM from WebSocket and plays through Web Audio API ──
|
|
||||||
import { useCallback, useRef, useState } from "react";
|
|
||||||
|
|
||||||
const SAMPLE_RATE = 24000;
|
|
||||||
const CHANNELS = 1;
|
|
||||||
|
|
||||||
export function useAudioPlayback() {
|
|
||||||
const [isListening, setIsListening] = useState(false);
|
|
||||||
const [levels, setLevels] = useState<number[]>(Array.from({ length: 32 }, () => 0.04));
|
|
||||||
const audioContextRef = useRef<AudioContext | null>(null);
|
|
||||||
const userTimelinesRef = useRef(new Map<number, number>());
|
|
||||||
|
|
||||||
const handleIncomingPcm = useCallback((data: ArrayBuffer) => {
|
|
||||||
const headerView = new DataView(data, 0, 4);
|
|
||||||
const userIdHash = headerView.getInt32(0, true);
|
|
||||||
const audioData = data.slice(4);
|
|
||||||
const int16Array = new Int16Array(audioData);
|
|
||||||
let sum = 0;
|
|
||||||
for (const sample of int16Array) sum += Math.abs(sample / 32768);
|
|
||||||
const average = int16Array.length ? sum / int16Array.length : 0;
|
|
||||||
setLevels((prev) =>
|
|
||||||
prev.map((_, index) =>
|
|
||||||
Math.max(0.04, average * (0.5 + Math.sin(index * 0.6 + Date.now() / 140) * 0.35 + 0.65) * 5),
|
|
||||||
),
|
|
||||||
);
|
|
||||||
|
|
||||||
const audioContext = audioContextRef.current;
|
|
||||||
if (!isListening || !audioContext) return;
|
|
||||||
const float32Array = new Float32Array(int16Array.length);
|
|
||||||
for (let i = 0; i < int16Array.length; i++) float32Array[i] = int16Array[i] / 32768;
|
|
||||||
const audioBuffer = audioContext.createBuffer(CHANNELS, float32Array.length / SAMPLE_RATE, SAMPLE_RATE);
|
|
||||||
audioBuffer.getChannelData(0).set(float32Array);
|
|
||||||
const source = audioContext.createBufferSource();
|
|
||||||
source.buffer = audioBuffer;
|
|
||||||
source.connect(audioContext.destination);
|
|
||||||
const currentTime = audioContext.currentTime;
|
|
||||||
let nextStart = userTimelinesRef.current.get(userIdHash) || 0;
|
|
||||||
if (nextStart < currentTime) nextStart = currentTime + 0.05;
|
|
||||||
source.start(nextStart);
|
|
||||||
userTimelinesRef.current.set(userIdHash, nextStart + audioBuffer.duration);
|
|
||||||
}, [isListening]);
|
|
||||||
|
|
||||||
const toggleListening = useCallback(async () => {
|
|
||||||
if (isListening) {
|
|
||||||
await audioContextRef.current?.suspend();
|
|
||||||
userTimelinesRef.current.clear();
|
|
||||||
setIsListening(false);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
const AudioContextCtor = window.AudioContext || (window as unknown as { webkitAudioContext: typeof AudioContext }).webkitAudioContext;
|
|
||||||
audioContextRef.current ??= new AudioContextCtor({ sampleRate: SAMPLE_RATE });
|
|
||||||
await audioContextRef.current.resume();
|
|
||||||
setIsListening(true);
|
|
||||||
}, [isListening]);
|
|
||||||
|
|
||||||
return { isListening, levels, handleIncomingPcm, toggleListening, audioContextRef };
|
|
||||||
}
|
|
||||||
@@ -1,13 +0,0 @@
|
|||||||
import { useCallback } from "react";
|
|
||||||
import type { UIState } from "../../entities/ui/types";
|
|
||||||
import { useLocalStorage, uiStateValidator } from "./useLocalStorage";
|
|
||||||
|
|
||||||
export function useUIState() {
|
|
||||||
const { value: uiState, setValue: setUIState } = useLocalStorage<UIState>("bete-dashboard-ui-state", uiStateValidator());
|
|
||||||
|
|
||||||
const patchUIState = useCallback((patch: Partial<UIState>) => {
|
|
||||||
setUIState((prev) => ({ ...prev, ...patch }));
|
|
||||||
}, [setUIState]);
|
|
||||||
|
|
||||||
return { uiState, setUIState, patchUIState, loading: false, error: null };
|
|
||||||
}
|
|
||||||
@@ -1,26 +0,0 @@
|
|||||||
import type * as React from "react";
|
|
||||||
import { cn } from "../lib/utils";
|
|
||||||
|
|
||||||
export function Card({ className, ...props }: React.HTMLAttributes<HTMLDivElement>) {
|
|
||||||
return <div className={cn("rounded-2xl border border-border bg-card text-card-foreground shadow-sm", className)} {...props} />;
|
|
||||||
}
|
|
||||||
|
|
||||||
export function CardHeader({ className, ...props }: React.HTMLAttributes<HTMLDivElement>) {
|
|
||||||
return <div className={cn("flex flex-col space-y-1.5 p-6", className)} {...props} />;
|
|
||||||
}
|
|
||||||
|
|
||||||
export function CardTitle({ className, ...props }: React.HTMLAttributes<HTMLHeadingElement>) {
|
|
||||||
return <h3 className={cn("font-semibold leading-none tracking-tight", className)} {...props} />;
|
|
||||||
}
|
|
||||||
|
|
||||||
export function CardDescription({ className, ...props }: React.HTMLAttributes<HTMLParagraphElement>) {
|
|
||||||
return <p className={cn("text-sm text-muted-foreground", className)} {...props} />;
|
|
||||||
}
|
|
||||||
|
|
||||||
export function CardContent({ className, ...props }: React.HTMLAttributes<HTMLDivElement>) {
|
|
||||||
return <div className={cn("p-6 pt-0", className)} {...props} />;
|
|
||||||
}
|
|
||||||
|
|
||||||
export function CardFooter({ className, ...props }: React.HTMLAttributes<HTMLDivElement>) {
|
|
||||||
return <div className={cn("flex items-center p-6 pt-0", className)} {...props} />;
|
|
||||||
}
|
|
||||||
@@ -0,0 +1,29 @@
|
|||||||
|
FROM node:20-alpine
|
||||||
|
|
||||||
|
WORKDIR /app
|
||||||
|
|
||||||
|
# Install pnpm
|
||||||
|
RUN npm install -g pnpm
|
||||||
|
|
||||||
|
# Copy workspace files
|
||||||
|
COPY pnpm-workspace.yaml .
|
||||||
|
COPY pnpm-lock.yaml .
|
||||||
|
COPY package.json .
|
||||||
|
|
||||||
|
# Copy shared package
|
||||||
|
COPY packages/shared ./packages/shared
|
||||||
|
|
||||||
|
# Copy backend service
|
||||||
|
COPY services/backend ./services/backend
|
||||||
|
|
||||||
|
# Install dependencies
|
||||||
|
RUN pnpm install --frozen-lockfile
|
||||||
|
|
||||||
|
# Build backend
|
||||||
|
RUN pnpm run build:server
|
||||||
|
|
||||||
|
# Expose port
|
||||||
|
EXPOSE 3001
|
||||||
|
|
||||||
|
# Start backend
|
||||||
|
CMD ["node", "dist/index.js"]
|
||||||
@@ -0,0 +1,29 @@
|
|||||||
|
FROM node:20-alpine
|
||||||
|
|
||||||
|
WORKDIR /app
|
||||||
|
|
||||||
|
# Install pnpm
|
||||||
|
RUN npm install -g pnpm
|
||||||
|
|
||||||
|
# Copy workspace files
|
||||||
|
COPY pnpm-workspace.yaml .
|
||||||
|
COPY pnpm-lock.yaml .
|
||||||
|
COPY package.json .
|
||||||
|
|
||||||
|
# Copy shared package
|
||||||
|
COPY packages/shared ./packages/shared
|
||||||
|
|
||||||
|
# Copy discord gateway service
|
||||||
|
COPY services/discord-gateway ./services/discord-gateway
|
||||||
|
|
||||||
|
# Install dependencies
|
||||||
|
RUN pnpm install --frozen-lockfile
|
||||||
|
|
||||||
|
# Build discord gateway
|
||||||
|
RUN pnpm run build
|
||||||
|
|
||||||
|
# Create recordings directory
|
||||||
|
RUN mkdir -p /app/recordings
|
||||||
|
|
||||||
|
# Start discord gateway
|
||||||
|
CMD ["node", "dist/index.js"]
|
||||||
@@ -0,0 +1,29 @@
|
|||||||
|
FROM node:20-alpine
|
||||||
|
|
||||||
|
WORKDIR /app
|
||||||
|
|
||||||
|
# Install pnpm
|
||||||
|
RUN npm install -g pnpm
|
||||||
|
|
||||||
|
# Copy workspace files
|
||||||
|
COPY pnpm-workspace.yaml .
|
||||||
|
COPY pnpm-lock.yaml .
|
||||||
|
COPY package.json .
|
||||||
|
|
||||||
|
# Copy shared package
|
||||||
|
COPY packages/shared ./packages/shared
|
||||||
|
|
||||||
|
# Copy frontend service
|
||||||
|
COPY services/frontend ./services/frontend
|
||||||
|
|
||||||
|
# Install dependencies
|
||||||
|
RUN pnpm install --frozen-lockfile
|
||||||
|
|
||||||
|
# Build frontend
|
||||||
|
RUN pnpm run build:web
|
||||||
|
|
||||||
|
# Expose port (served by backend)
|
||||||
|
EXPOSE 3000
|
||||||
|
|
||||||
|
# Start development server
|
||||||
|
CMD ["pnpm", "run", "dev:web"]
|
||||||
@@ -0,0 +1,67 @@
|
|||||||
|
version: '3.8'
|
||||||
|
|
||||||
|
services:
|
||||||
|
# Backend Service (REST API + WebSocket via Traefik)
|
||||||
|
backend:
|
||||||
|
build:
|
||||||
|
context: ..
|
||||||
|
dockerfile: infra/docker/Dockerfile.backend
|
||||||
|
container_name: bete-backend
|
||||||
|
restart: unless-stopped
|
||||||
|
env_file:
|
||||||
|
- ../../.env
|
||||||
|
environment:
|
||||||
|
NODE_ENV: production
|
||||||
|
WEBSERVER_PORT: 3000
|
||||||
|
labels:
|
||||||
|
- "traefik.enable=true"
|
||||||
|
- "traefik.http.routers.bete-backend.rule=Host(`imphnen.asepharyana.my.id`) && PathPrefix(`/api`, `/ws`)"
|
||||||
|
- "traefik.http.routers.bete-backend.entrypoints=websecure"
|
||||||
|
- "traefik.http.routers.bete-backend.tls=true"
|
||||||
|
- "traefik.http.services.bete-backend.loadbalancer.server.port=3000"
|
||||||
|
depends_on:
|
||||||
|
- discord-gateway
|
||||||
|
networks:
|
||||||
|
- app-shared-net
|
||||||
|
|
||||||
|
# Discord Gateway Service (Event capture and processing — no HTTP)
|
||||||
|
discord-gateway:
|
||||||
|
build:
|
||||||
|
context: ..
|
||||||
|
dockerfile: infra/docker/Dockerfile.discord-gateway
|
||||||
|
container_name: bete-discord-gateway
|
||||||
|
restart: unless-stopped
|
||||||
|
env_file:
|
||||||
|
- ../../.env
|
||||||
|
environment:
|
||||||
|
NODE_ENV: production
|
||||||
|
volumes:
|
||||||
|
- ../recordings:/app/recordings
|
||||||
|
networks:
|
||||||
|
- app-shared-net
|
||||||
|
|
||||||
|
# Frontend Service (React Dashboard via Traefik)
|
||||||
|
frontend:
|
||||||
|
build:
|
||||||
|
context: ..
|
||||||
|
dockerfile: infra/docker/Dockerfile.frontend
|
||||||
|
container_name: bete-frontend
|
||||||
|
restart: unless-stopped
|
||||||
|
environment:
|
||||||
|
VITE_API_URL: https://imphnen.asepharyana.my.id
|
||||||
|
VITE_WS_URL: wss://imphnen.asepharyana.my.id
|
||||||
|
labels:
|
||||||
|
- "traefik.enable=true"
|
||||||
|
- "traefik.http.routers.bete-frontend.rule=Host(`imphnen.asepharyana.my.id`)"
|
||||||
|
- "traefik.http.routers.bete-frontend.entrypoints=websecure"
|
||||||
|
- "traefik.http.routers.bete-frontend.tls=true"
|
||||||
|
- "traefik.http.services.bete-frontend.loadbalancer.server.port=3000"
|
||||||
|
depends_on:
|
||||||
|
- backend
|
||||||
|
networks:
|
||||||
|
- app-shared-net
|
||||||
|
|
||||||
|
networks:
|
||||||
|
app-shared-net:
|
||||||
|
name: app-shared-net
|
||||||
|
external: true
|
||||||
@@ -0,0 +1,27 @@
|
|||||||
|
{
|
||||||
|
"name": "@bete/shared",
|
||||||
|
"version": "1.0.0",
|
||||||
|
"description": "Shared utilities, types, and errors for Bete microservices",
|
||||||
|
"type": "module",
|
||||||
|
"main": "dist/index.js",
|
||||||
|
"types": "dist/index.d.ts",
|
||||||
|
"exports": {
|
||||||
|
".": "./dist/index.js",
|
||||||
|
"./types": "./dist/types/index.js",
|
||||||
|
"./errors": "./dist/errors/index.js",
|
||||||
|
"./logger": "./dist/logger/index.js",
|
||||||
|
"./utils": "./dist/utils/index.js"
|
||||||
|
},
|
||||||
|
"scripts": {
|
||||||
|
"build": "tsc",
|
||||||
|
"typecheck": "tsc --noEmit"
|
||||||
|
},
|
||||||
|
"dependencies": {
|
||||||
|
"pino": "^9.0.0",
|
||||||
|
"zod": "^4.4.3"
|
||||||
|
},
|
||||||
|
"devDependencies": {
|
||||||
|
"@types/node": "^25.9.0",
|
||||||
|
"typescript": "^5.9.3"
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,86 @@
|
|||||||
|
// Custom error classes for all services
|
||||||
|
|
||||||
|
export class AppError extends Error {
|
||||||
|
constructor(
|
||||||
|
public code: string,
|
||||||
|
public statusCode: number,
|
||||||
|
message: string,
|
||||||
|
public details?: Record<string, unknown>,
|
||||||
|
) {
|
||||||
|
super(message);
|
||||||
|
this.name = "AppError";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export class ValidationError extends AppError {
|
||||||
|
constructor(message: string, details?: Record<string, unknown>) {
|
||||||
|
super("VALIDATION_ERROR", 400, message, details);
|
||||||
|
this.name = "ValidationError";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export class NotFoundError extends AppError {
|
||||||
|
constructor(resource: string, id?: string) {
|
||||||
|
super("NOT_FOUND", 404, `${resource} not found${id ? `: ${id}` : ""}`);
|
||||||
|
this.name = "NotFoundError";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export class UnauthorizedError extends AppError {
|
||||||
|
constructor(message = "Unauthorized") {
|
||||||
|
super("UNAUTHORIZED", 401, message);
|
||||||
|
this.name = "UnauthorizedError";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export class ForbiddenError extends AppError {
|
||||||
|
constructor(message = "Forbidden") {
|
||||||
|
super("FORBIDDEN", 403, message);
|
||||||
|
this.name = "ForbiddenError";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export class ConflictError extends AppError {
|
||||||
|
constructor(message: string) {
|
||||||
|
super("CONFLICT", 409, message);
|
||||||
|
this.name = "ConflictError";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export class InternalServerError extends AppError {
|
||||||
|
constructor(
|
||||||
|
message = "Internal server error",
|
||||||
|
details?: Record<string, unknown>,
|
||||||
|
) {
|
||||||
|
super("INTERNAL_SERVER_ERROR", 500, message, details);
|
||||||
|
this.name = "InternalServerError";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export class DatabaseError extends AppError {
|
||||||
|
constructor(message: string, details?: Record<string, unknown>) {
|
||||||
|
super("DATABASE_ERROR", 500, message, details);
|
||||||
|
this.name = "DatabaseError";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export class ConfigError extends AppError {
|
||||||
|
constructor(message: string) {
|
||||||
|
super("CONFIG_ERROR", 500, message);
|
||||||
|
this.name = "ConfigError";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export class DiscordError extends AppError {
|
||||||
|
constructor(message: string, details?: Record<string, unknown>) {
|
||||||
|
super("DISCORD_ERROR", 500, message, details);
|
||||||
|
this.name = "DiscordError";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export class TimeoutError extends AppError {
|
||||||
|
constructor(operation: string) {
|
||||||
|
super("TIMEOUT", 504, `${operation} timed out`);
|
||||||
|
this.name = "TimeoutError";
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,4 @@
|
|||||||
|
export * from "./errors/index.js";
|
||||||
|
export * from "./logger/index.js";
|
||||||
|
export * from "./types/index.js";
|
||||||
|
export * from "./utils/index.js";
|
||||||
@@ -0,0 +1,25 @@
|
|||||||
|
import pino from "pino";
|
||||||
|
|
||||||
|
export type Logger = ReturnType<typeof createLogger>;
|
||||||
|
|
||||||
|
export function createLogger(context: string) {
|
||||||
|
return pino({
|
||||||
|
name: context,
|
||||||
|
level: process.env.LOG_LEVEL || "info",
|
||||||
|
transport:
|
||||||
|
process.env.NODE_ENV === "development"
|
||||||
|
? {
|
||||||
|
target: "pino-pretty",
|
||||||
|
options: {
|
||||||
|
colorize: true,
|
||||||
|
translateTime: "SYS:standard",
|
||||||
|
ignore: "pid,hostname",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
: undefined,
|
||||||
|
} as pino.LoggerOptions);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function createChildLogger(context: string) {
|
||||||
|
return createLogger(context);
|
||||||
|
}
|
||||||
@@ -0,0 +1,70 @@
|
|||||||
|
// Shared types for all services
|
||||||
|
export interface AppConfig {
|
||||||
|
NODE_ENV: "development" | "production" | "test";
|
||||||
|
LOG_LEVEL: string;
|
||||||
|
VERBOSE: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface DatabaseConfig {
|
||||||
|
DATABASE_URL: string;
|
||||||
|
AUTO_MIGRATE_ON_STARTUP: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface DiscordConfig {
|
||||||
|
DISCORD_TOKEN: string;
|
||||||
|
MONITOR_GUILD_ID: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface AIConfig {
|
||||||
|
AI_LLM_API_KEY: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface RedisConfig {
|
||||||
|
REDIS_URL: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface WebServerConfig {
|
||||||
|
WEBSERVER_PORT: number;
|
||||||
|
ADMIN_PASSWORD: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface MessageRecord {
|
||||||
|
id: string;
|
||||||
|
guildId: string;
|
||||||
|
channelId: string;
|
||||||
|
userId: string;
|
||||||
|
username: string;
|
||||||
|
content: string;
|
||||||
|
createdAt: Date;
|
||||||
|
editedAt?: Date;
|
||||||
|
deletedAt?: Date;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface AttachmentRecord {
|
||||||
|
id: string;
|
||||||
|
messageId: string;
|
||||||
|
filename: string;
|
||||||
|
size: number;
|
||||||
|
mimeType: string;
|
||||||
|
discordUrl: string;
|
||||||
|
uploadedUrl?: string;
|
||||||
|
uploadStatus: "pending" | "uploaded" | "failed";
|
||||||
|
createdAt: Date;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface VoiceSegment {
|
||||||
|
userId: string;
|
||||||
|
sessionStart: number;
|
||||||
|
segmentIndex: number;
|
||||||
|
duration: number;
|
||||||
|
filePath: string;
|
||||||
|
createdAt: Date;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface AnalyticsData {
|
||||||
|
totalMessages: number;
|
||||||
|
totalAttachments: number;
|
||||||
|
totalVoiceSegments: number;
|
||||||
|
activeUsers: number;
|
||||||
|
lastUpdated: Date;
|
||||||
|
}
|
||||||
@@ -0,0 +1,61 @@
|
|||||||
|
// Utility functions shared across services
|
||||||
|
|
||||||
|
export function delay(ms: number): Promise<void> {
|
||||||
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||||
|
}
|
||||||
|
|
||||||
|
export function formatBytes(bytes: number): string {
|
||||||
|
if (bytes === 0) return "0 Bytes";
|
||||||
|
const k = 1024;
|
||||||
|
const sizes = ["Bytes", "KB", "MB", "GB"];
|
||||||
|
const i = Math.floor(Math.log(bytes) / Math.log(k));
|
||||||
|
return Math.round((bytes / Math.pow(k, i)) * 100) / 100 + " " + sizes[i];
|
||||||
|
}
|
||||||
|
|
||||||
|
export function generateId(): string {
|
||||||
|
return `${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isValidUrl(url: string): boolean {
|
||||||
|
try {
|
||||||
|
new URL(url);
|
||||||
|
return true;
|
||||||
|
} catch {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function sanitizeString(str: string): string {
|
||||||
|
return str.replace(/[<>]/g, "").trim().substring(0);
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface PaginationParams {
|
||||||
|
page: number;
|
||||||
|
limit: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface PaginatedResponse<T> {
|
||||||
|
data: T[];
|
||||||
|
total: number;
|
||||||
|
page: number;
|
||||||
|
limit: number;
|
||||||
|
pages: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function calculatePagination(
|
||||||
|
total: number,
|
||||||
|
page: number,
|
||||||
|
limit: number,
|
||||||
|
): PaginatedResponse<never> {
|
||||||
|
return {
|
||||||
|
data: [],
|
||||||
|
total,
|
||||||
|
page,
|
||||||
|
limit,
|
||||||
|
pages: Math.ceil(total / limit),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getOffset(page: number, limit: number): number {
|
||||||
|
return (page - 1) * limit;
|
||||||
|
}
|
||||||
@@ -0,0 +1,19 @@
|
|||||||
|
{
|
||||||
|
"compilerOptions": {
|
||||||
|
"target": "ES2020",
|
||||||
|
"module": "ESNext",
|
||||||
|
"lib": ["ES2020"],
|
||||||
|
"declaration": true,
|
||||||
|
"declarationMap": true,
|
||||||
|
"outDir": "./dist",
|
||||||
|
"rootDir": "./src",
|
||||||
|
"strict": true,
|
||||||
|
"esModuleInterop": true,
|
||||||
|
"skipLibCheck": true,
|
||||||
|
"forceConsistentCasingInFileNames": true,
|
||||||
|
"resolveJsonModule": true,
|
||||||
|
"moduleResolution": "bundler"
|
||||||
|
},
|
||||||
|
"include": ["src"],
|
||||||
|
"exclude": ["node_modules", "dist"]
|
||||||
|
}
|
||||||
Generated
+300
-6
@@ -166,6 +166,226 @@ importers:
|
|||||||
specifier: latest
|
specifier: latest
|
||||||
version: 4.1.7(@opentelemetry/api@1.9.1)(@types/node@25.9.0)(vite@8.0.13(@types/node@25.9.0)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.2)(yaml@2.9.0))
|
version: 4.1.7(@opentelemetry/api@1.9.1)(@types/node@25.9.0)(vite@8.0.13(@types/node@25.9.0)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.2)(yaml@2.9.0))
|
||||||
|
|
||||||
|
packages/shared:
|
||||||
|
dependencies:
|
||||||
|
pino:
|
||||||
|
specifier: ^9.0.0
|
||||||
|
version: 9.14.0
|
||||||
|
zod:
|
||||||
|
specifier: ^4.4.3
|
||||||
|
version: 4.4.3
|
||||||
|
devDependencies:
|
||||||
|
'@types/node':
|
||||||
|
specifier: ^25.9.0
|
||||||
|
version: 25.9.0
|
||||||
|
typescript:
|
||||||
|
specifier: ^5.9.3
|
||||||
|
version: 5.9.3
|
||||||
|
|
||||||
|
services/backend:
|
||||||
|
dependencies:
|
||||||
|
'@discordjs/voice':
|
||||||
|
specifier: ^0.19.2
|
||||||
|
version: 0.19.2(@discordjs/opus@0.10.0)(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)(opusscript@0.0.8)
|
||||||
|
'@types/pg':
|
||||||
|
specifier: ^8.20.0
|
||||||
|
version: 8.20.0
|
||||||
|
axios:
|
||||||
|
specifier: ^1.16.1
|
||||||
|
version: 1.16.1
|
||||||
|
dotenv:
|
||||||
|
specifier: ^17.4.2
|
||||||
|
version: 17.4.2
|
||||||
|
drizzle-orm:
|
||||||
|
specifier: ^0.45.2
|
||||||
|
version: 0.45.2(@opentelemetry/api@1.9.1)(@types/pg@8.20.0)(pg@8.21.0)
|
||||||
|
express:
|
||||||
|
specifier: ^5.2.1
|
||||||
|
version: 5.2.1
|
||||||
|
helmet:
|
||||||
|
specifier: ^8.1.0
|
||||||
|
version: 8.1.0
|
||||||
|
ioredis:
|
||||||
|
specifier: ^5.11.0
|
||||||
|
version: 5.11.0
|
||||||
|
pg:
|
||||||
|
specifier: ^8.21.0
|
||||||
|
version: 8.21.0
|
||||||
|
pino:
|
||||||
|
specifier: ^9.6.0
|
||||||
|
version: 9.14.0
|
||||||
|
pino-http:
|
||||||
|
specifier: ^10.3.0
|
||||||
|
version: 10.5.0
|
||||||
|
prom-client:
|
||||||
|
specifier: ^15.1.3
|
||||||
|
version: 15.1.3
|
||||||
|
ws:
|
||||||
|
specifier: ^8.20.1
|
||||||
|
version: 8.20.1
|
||||||
|
zod:
|
||||||
|
specifier: ^4.4.3
|
||||||
|
version: 4.4.3
|
||||||
|
devDependencies:
|
||||||
|
'@biomejs/biome':
|
||||||
|
specifier: latest
|
||||||
|
version: 2.4.16
|
||||||
|
'@types/express':
|
||||||
|
specifier: ^5.0.6
|
||||||
|
version: 5.0.6
|
||||||
|
'@types/node':
|
||||||
|
specifier: ^25.9.0
|
||||||
|
version: 25.9.0
|
||||||
|
'@types/ws':
|
||||||
|
specifier: ^8.18.1
|
||||||
|
version: 8.18.1
|
||||||
|
tsx:
|
||||||
|
specifier: ^4.22.2
|
||||||
|
version: 4.22.2
|
||||||
|
typescript:
|
||||||
|
specifier: ^5.9.3
|
||||||
|
version: 5.9.3
|
||||||
|
vitest:
|
||||||
|
specifier: latest
|
||||||
|
version: 4.1.7(@opentelemetry/api@1.9.1)(@types/node@25.9.0)(vite@8.0.13(@types/node@25.9.0)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.2)(yaml@2.9.0))
|
||||||
|
|
||||||
|
services/discord-gateway:
|
||||||
|
dependencies:
|
||||||
|
'@discordjs/opus':
|
||||||
|
specifier: ^0.10.0
|
||||||
|
version: 0.10.0
|
||||||
|
'@discordjs/voice':
|
||||||
|
specifier: ^0.19.2
|
||||||
|
version: 0.19.2(@discordjs/opus@0.10.0)(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)(opusscript@0.0.8)
|
||||||
|
'@snazzah/davey':
|
||||||
|
specifier: ^0.1.11
|
||||||
|
version: 0.1.11(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)
|
||||||
|
discord.js-selfbot-v13:
|
||||||
|
specifier: workspace:*
|
||||||
|
version: link:../../vendor/discord.js-selfbot-v13
|
||||||
|
dotenv:
|
||||||
|
specifier: ^17.4.2
|
||||||
|
version: 17.4.2
|
||||||
|
drizzle-orm:
|
||||||
|
specifier: ^0.45.2
|
||||||
|
version: 0.45.2(@opentelemetry/api@1.9.1)(@types/pg@8.20.0)(pg@8.21.0)
|
||||||
|
ioredis:
|
||||||
|
specifier: ^5.11.0
|
||||||
|
version: 5.11.0
|
||||||
|
libsodium-wrappers:
|
||||||
|
specifier: ^0.8.4
|
||||||
|
version: 0.8.4
|
||||||
|
openai:
|
||||||
|
specifier: ^6.38.0
|
||||||
|
version: 6.38.0(ws@8.20.1)(zod@4.4.3)
|
||||||
|
opusscript:
|
||||||
|
specifier: ^0.0.8
|
||||||
|
version: 0.0.8
|
||||||
|
p-limit:
|
||||||
|
specifier: ^7.3.0
|
||||||
|
version: 7.3.0
|
||||||
|
p-retry:
|
||||||
|
specifier: ^8.0.0
|
||||||
|
version: 8.0.0
|
||||||
|
pg:
|
||||||
|
specifier: ^8.21.0
|
||||||
|
version: 8.21.0
|
||||||
|
piscina:
|
||||||
|
specifier: ^5.1.4
|
||||||
|
version: 5.1.4
|
||||||
|
prism-media:
|
||||||
|
specifier: 2.0.0-alpha.0
|
||||||
|
version: 2.0.0-alpha.0
|
||||||
|
sharp:
|
||||||
|
specifier: ^0.34.5
|
||||||
|
version: 0.34.5
|
||||||
|
winston:
|
||||||
|
specifier: ^3.19.0
|
||||||
|
version: 3.19.0
|
||||||
|
zod:
|
||||||
|
specifier: ^4.4.3
|
||||||
|
version: 4.4.3
|
||||||
|
devDependencies:
|
||||||
|
'@biomejs/biome':
|
||||||
|
specifier: latest
|
||||||
|
version: 2.4.16
|
||||||
|
'@types/node':
|
||||||
|
specifier: ^25.9.0
|
||||||
|
version: 25.9.0
|
||||||
|
drizzle-kit:
|
||||||
|
specifier: ^0.31.10
|
||||||
|
version: 0.31.10
|
||||||
|
tsx:
|
||||||
|
specifier: ^4.22.2
|
||||||
|
version: 4.22.2
|
||||||
|
typescript:
|
||||||
|
specifier: ^5.9.3
|
||||||
|
version: 5.9.3
|
||||||
|
vitest:
|
||||||
|
specifier: latest
|
||||||
|
version: 4.1.7(@opentelemetry/api@1.9.1)(@types/node@25.9.0)(vite@8.0.13(@types/node@25.9.0)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.2)(yaml@2.9.0))
|
||||||
|
|
||||||
|
services/frontend:
|
||||||
|
dependencies:
|
||||||
|
'@radix-ui/react-scroll-area':
|
||||||
|
specifier: ^1.2.10
|
||||||
|
version: 1.2.10(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)
|
||||||
|
'@radix-ui/react-slot':
|
||||||
|
specifier: ^1.2.4
|
||||||
|
version: 1.2.4(@types/react@19.2.14)(react@19.2.6)
|
||||||
|
'@radix-ui/react-tabs':
|
||||||
|
specifier: ^1.1.13
|
||||||
|
version: 1.1.13(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)
|
||||||
|
'@tanstack/react-query':
|
||||||
|
specifier: ^5.100.14
|
||||||
|
version: 5.100.14(react@19.2.6)
|
||||||
|
clsx:
|
||||||
|
specifier: ^2.1.1
|
||||||
|
version: 2.1.1
|
||||||
|
lucide-react:
|
||||||
|
specifier: ^1.16.0
|
||||||
|
version: 1.16.0(react@19.2.6)
|
||||||
|
react:
|
||||||
|
specifier: ^19.2.6
|
||||||
|
version: 19.2.6
|
||||||
|
react-dom:
|
||||||
|
specifier: ^19.2.6
|
||||||
|
version: 19.2.6(react@19.2.6)
|
||||||
|
tailwind-merge:
|
||||||
|
specifier: ^3.6.0
|
||||||
|
version: 3.6.0
|
||||||
|
devDependencies:
|
||||||
|
'@biomejs/biome':
|
||||||
|
specifier: latest
|
||||||
|
version: 2.4.16
|
||||||
|
'@tailwindcss/postcss':
|
||||||
|
specifier: ^4.3.0
|
||||||
|
version: 4.3.0
|
||||||
|
'@types/react':
|
||||||
|
specifier: ^19.2.14
|
||||||
|
version: 19.2.14
|
||||||
|
'@types/react-dom':
|
||||||
|
specifier: ^19.2.3
|
||||||
|
version: 19.2.3(@types/react@19.2.14)
|
||||||
|
'@vitejs/plugin-react':
|
||||||
|
specifier: ^6.0.2
|
||||||
|
version: 6.0.2(vite@8.0.13(@types/node@25.9.0)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.2)(yaml@2.9.0))
|
||||||
|
autoprefixer:
|
||||||
|
specifier: ^10.5.0
|
||||||
|
version: 10.5.0(postcss@8.5.14)
|
||||||
|
postcss:
|
||||||
|
specifier: ^8.5.14
|
||||||
|
version: 8.5.14
|
||||||
|
tailwindcss:
|
||||||
|
specifier: ^4.3.0
|
||||||
|
version: 4.3.0
|
||||||
|
typescript:
|
||||||
|
specifier: ^5.9.3
|
||||||
|
version: 5.9.3
|
||||||
|
vite:
|
||||||
|
specifier: ^8.0.13
|
||||||
|
version: 8.0.13(@types/node@25.9.0)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.2)(yaml@2.9.0)
|
||||||
|
|
||||||
vendor/discord-video-stream:
|
vendor/discord-video-stream:
|
||||||
dependencies:
|
dependencies:
|
||||||
'@lng2004/node-datachannel':
|
'@lng2004/node-datachannel':
|
||||||
@@ -1405,6 +1625,9 @@ packages:
|
|||||||
resolution: {integrity: sha512-C2Xj8FZ0uHWeCXXqX5B4/gVFQmtSkiuOolzAgutjTfseNOHT3pUjljDZsTSxXFGgio54bCzVFqmEOUrIVk8RDA==}
|
resolution: {integrity: sha512-C2Xj8FZ0uHWeCXXqX5B4/gVFQmtSkiuOolzAgutjTfseNOHT3pUjljDZsTSxXFGgio54bCzVFqmEOUrIVk8RDA==}
|
||||||
engines: {node: '>=20.0.0'}
|
engines: {node: '>=20.0.0'}
|
||||||
|
|
||||||
|
'@pinojs/redact@0.4.0':
|
||||||
|
resolution: {integrity: sha512-k2ENnmBugE/rzQfEcdWHcCY+/FM3VLzH9cYEsbdsoqrvzAKRhUZeRNhAZvB8OitQJ1TBed3yqWtdjzS6wJKBwg==}
|
||||||
|
|
||||||
'@radix-ui/number@1.1.1':
|
'@radix-ui/number@1.1.1':
|
||||||
resolution: {integrity: sha512-MkKCwxlXTgz6CFoJx3pCwn07GKp36+aZyu/u2Ln2VrA5DcdyCZkASEDBTd8x5whTQQL5CiYf4prXKLcgQdv29g==}
|
resolution: {integrity: sha512-MkKCwxlXTgz6CFoJx3pCwn07GKp36+aZyu/u2Ln2VrA5DcdyCZkASEDBTd8x5whTQQL5CiYf4prXKLcgQdv29g==}
|
||||||
|
|
||||||
@@ -3750,6 +3973,10 @@ packages:
|
|||||||
obug@2.1.1:
|
obug@2.1.1:
|
||||||
resolution: {integrity: sha512-uTqF9MuPraAQ+IsnPf366RG4cP9RtUi7MLO1N3KEc+wb0a6yKpeL0lmk2IB1jY5KHPAlTc6T/JRdC/YqxHNwkQ==}
|
resolution: {integrity: sha512-uTqF9MuPraAQ+IsnPf366RG4cP9RtUi7MLO1N3KEc+wb0a6yKpeL0lmk2IB1jY5KHPAlTc6T/JRdC/YqxHNwkQ==}
|
||||||
|
|
||||||
|
on-exit-leak-free@2.1.2:
|
||||||
|
resolution: {integrity: sha512-0eJJY6hXLGf1udHwfNftBqH+g73EU4B504nZeKpz1sYRKafAghwxEJunB2O7rDZkL4PGfsMVnTXZ2EjibbqcsA==}
|
||||||
|
engines: {node: '>=14.0.0'}
|
||||||
|
|
||||||
on-finished@2.4.1:
|
on-finished@2.4.1:
|
||||||
resolution: {integrity: sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==}
|
resolution: {integrity: sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==}
|
||||||
engines: {node: '>= 0.8'}
|
engines: {node: '>= 0.8'}
|
||||||
@@ -3932,6 +4159,19 @@ packages:
|
|||||||
resolution: {integrity: sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==}
|
resolution: {integrity: sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==}
|
||||||
engines: {node: '>=12'}
|
engines: {node: '>=12'}
|
||||||
|
|
||||||
|
pino-abstract-transport@2.0.0:
|
||||||
|
resolution: {integrity: sha512-F63x5tizV6WCh4R6RHyi2Ml+M70DNRXt/+HANowMflpgGFMAym/VKm6G7ZOQRjqN7XbGxK1Lg9t6ZrtzOaivMw==}
|
||||||
|
|
||||||
|
pino-http@10.5.0:
|
||||||
|
resolution: {integrity: sha512-hD91XjgaKkSsdn8P7LaebrNzhGTdB086W3pyPihX0EzGPjq5uBJBXo4N5guqNaK6mUjg9aubMF7wDViYek9dRA==}
|
||||||
|
|
||||||
|
pino-std-serializers@7.1.0:
|
||||||
|
resolution: {integrity: sha512-BndPH67/JxGExRgiX1dX0w1FvZck5Wa4aal9198SrRhZjH3GxKQUKIBnYJTdj2HDN3UQAS06HlfcSbQj2OHmaw==}
|
||||||
|
|
||||||
|
pino@9.14.0:
|
||||||
|
resolution: {integrity: sha512-8OEwKp5juEvb/MjpIc4hjqfgCNysrS94RIOMXYvpYCdm/jglrKEiAYmiumbmGhCvs+IcInsphYDFwqrjr7398w==}
|
||||||
|
hasBin: true
|
||||||
|
|
||||||
piscina@5.1.4:
|
piscina@5.1.4:
|
||||||
resolution: {integrity: sha512-7uU4ZnKeQq22t9AsmHGD2w4OYQGonwFnTypDypaWi7Qr2EvQIFVtG8J5D/3bE7W123Wdc9+v4CZDu5hJXVCtBg==}
|
resolution: {integrity: sha512-7uU4ZnKeQq22t9AsmHGD2w4OYQGonwFnTypDypaWi7Qr2EvQIFVtG8J5D/3bE7W123Wdc9+v4CZDu5hJXVCtBg==}
|
||||||
engines: {node: '>=20.x'}
|
engines: {node: '>=20.x'}
|
||||||
@@ -4022,6 +4262,9 @@ packages:
|
|||||||
process-nextick-args@2.0.1:
|
process-nextick-args@2.0.1:
|
||||||
resolution: {integrity: sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==}
|
resolution: {integrity: sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==}
|
||||||
|
|
||||||
|
process-warning@5.0.0:
|
||||||
|
resolution: {integrity: sha512-a39t9ApHNx2L4+HBnQKqxxHNs1r7KF+Intd8Q/g1bUh6q0WIp9voPXJ/x0j+ZL45KF1pJd9+q2jLIRMfvEshkA==}
|
||||||
|
|
||||||
progress@2.0.3:
|
progress@2.0.3:
|
||||||
resolution: {integrity: sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==}
|
resolution: {integrity: sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==}
|
||||||
engines: {node: '>=0.4.0'}
|
engines: {node: '>=0.4.0'}
|
||||||
@@ -4072,6 +4315,9 @@ packages:
|
|||||||
queue-microtask@1.2.3:
|
queue-microtask@1.2.3:
|
||||||
resolution: {integrity: sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==}
|
resolution: {integrity: sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==}
|
||||||
|
|
||||||
|
quick-format-unescaped@4.0.4:
|
||||||
|
resolution: {integrity: sha512-tYC1Q1hgyRuHgloV/YXs2w15unPVh8qfu/qCTfhTYamaw7fyhumKa2yGpdSo87vY32rIclj+4fWYQXUMs9EHvg==}
|
||||||
|
|
||||||
quick-lru@4.0.1:
|
quick-lru@4.0.1:
|
||||||
resolution: {integrity: sha512-ARhCpm70fzdcvNQfPoy49IaanKkTlRWF2JMzqhcJbhSFRZv7nPTvZJdcY7301IPmvW+/p0RgIWnQDLJxifsQ7g==}
|
resolution: {integrity: sha512-ARhCpm70fzdcvNQfPoy49IaanKkTlRWF2JMzqhcJbhSFRZv7nPTvZJdcY7301IPmvW+/p0RgIWnQDLJxifsQ7g==}
|
||||||
engines: {node: '>=8'}
|
engines: {node: '>=8'}
|
||||||
@@ -4119,6 +4365,10 @@ packages:
|
|||||||
resolution: {integrity: sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==}
|
resolution: {integrity: sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==}
|
||||||
engines: {node: '>= 6'}
|
engines: {node: '>= 6'}
|
||||||
|
|
||||||
|
real-require@0.2.0:
|
||||||
|
resolution: {integrity: sha512-57frrGM/OCTLqLOAh0mhVA9VBMHd+9U7Zb2THMGdBUoZVOtGbJzjxsYGDJ3A9AYYCP4hn6y1TVbaOfzWtm5GFg==}
|
||||||
|
engines: {node: '>= 12.13.0'}
|
||||||
|
|
||||||
redent@3.0.0:
|
redent@3.0.0:
|
||||||
resolution: {integrity: sha512-6tDA8g98We0zd0GvVeMT9arEOnTw9qM03L9cJXaCjrip1OO764RDBLBfrB4cwzNGDj5OA5ioymC9GkizgWJDUg==}
|
resolution: {integrity: sha512-6tDA8g98We0zd0GvVeMT9arEOnTw9qM03L9cJXaCjrip1OO764RDBLBfrB4cwzNGDj5OA5ioymC9GkizgWJDUg==}
|
||||||
engines: {node: '>=8'}
|
engines: {node: '>=8'}
|
||||||
@@ -4523,6 +4773,9 @@ packages:
|
|||||||
resolution: {integrity: sha512-OEI0IWCe+Dw46019YLl6V10Us5bi574EvlJEOcAkB29IzQ/mYD1A6RyNHLjZPiHCmuodxvgF6U+vZO1L15lxVA==}
|
resolution: {integrity: sha512-OEI0IWCe+Dw46019YLl6V10Us5bi574EvlJEOcAkB29IzQ/mYD1A6RyNHLjZPiHCmuodxvgF6U+vZO1L15lxVA==}
|
||||||
engines: {node: '>=0.2.6'}
|
engines: {node: '>=0.2.6'}
|
||||||
|
|
||||||
|
thread-stream@3.1.0:
|
||||||
|
resolution: {integrity: sha512-OqyPZ9u96VohAyMfJykzmivOrY2wfMSf3C5TtFJVgN+Hm6aj+voFhlK+kZEIv2FBh1X6Xp3DlnCOfEQ3B2J86A==}
|
||||||
|
|
||||||
through@2.3.8:
|
through@2.3.8:
|
||||||
resolution: {integrity: sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==}
|
resolution: {integrity: sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==}
|
||||||
|
|
||||||
@@ -5880,6 +6133,8 @@ snapshots:
|
|||||||
tslib: 2.8.1
|
tslib: 2.8.1
|
||||||
tsyringe: 4.10.0
|
tsyringe: 4.10.0
|
||||||
|
|
||||||
|
'@pinojs/redact@0.4.0': {}
|
||||||
|
|
||||||
'@radix-ui/number@1.1.1': {}
|
'@radix-ui/number@1.1.1': {}
|
||||||
|
|
||||||
'@radix-ui/primitive@1.1.3': {}
|
'@radix-ui/primitive@1.1.3': {}
|
||||||
@@ -6286,7 +6541,7 @@ snapshots:
|
|||||||
'@types/body-parser@1.19.6':
|
'@types/body-parser@1.19.6':
|
||||||
dependencies:
|
dependencies:
|
||||||
'@types/connect': 3.4.38
|
'@types/connect': 3.4.38
|
||||||
'@types/node': 25.8.0
|
'@types/node': 25.9.0
|
||||||
|
|
||||||
'@types/chai@5.2.3':
|
'@types/chai@5.2.3':
|
||||||
dependencies:
|
dependencies:
|
||||||
@@ -6295,7 +6550,7 @@ snapshots:
|
|||||||
|
|
||||||
'@types/connect@3.4.38':
|
'@types/connect@3.4.38':
|
||||||
dependencies:
|
dependencies:
|
||||||
'@types/node': 25.8.0
|
'@types/node': 25.9.0
|
||||||
|
|
||||||
'@types/debug@4.1.13':
|
'@types/debug@4.1.13':
|
||||||
dependencies:
|
dependencies:
|
||||||
@@ -6312,7 +6567,7 @@ snapshots:
|
|||||||
|
|
||||||
'@types/express-serve-static-core@5.1.1':
|
'@types/express-serve-static-core@5.1.1':
|
||||||
dependencies:
|
dependencies:
|
||||||
'@types/node': 25.8.0
|
'@types/node': 25.9.0
|
||||||
'@types/qs': 6.15.1
|
'@types/qs': 6.15.1
|
||||||
'@types/range-parser': 1.2.7
|
'@types/range-parser': 1.2.7
|
||||||
'@types/send': 1.2.1
|
'@types/send': 1.2.1
|
||||||
@@ -6325,7 +6580,7 @@ snapshots:
|
|||||||
|
|
||||||
'@types/fluent-ffmpeg@2.1.28':
|
'@types/fluent-ffmpeg@2.1.28':
|
||||||
dependencies:
|
dependencies:
|
||||||
'@types/node': 25.8.0
|
'@types/node': 25.9.0
|
||||||
|
|
||||||
'@types/http-errors@2.0.5': {}
|
'@types/http-errors@2.0.5': {}
|
||||||
|
|
||||||
@@ -6374,12 +6629,12 @@ snapshots:
|
|||||||
|
|
||||||
'@types/send@1.2.1':
|
'@types/send@1.2.1':
|
||||||
dependencies:
|
dependencies:
|
||||||
'@types/node': 25.8.0
|
'@types/node': 25.9.0
|
||||||
|
|
||||||
'@types/serve-static@2.2.0':
|
'@types/serve-static@2.2.0':
|
||||||
dependencies:
|
dependencies:
|
||||||
'@types/http-errors': 2.0.5
|
'@types/http-errors': 2.0.5
|
||||||
'@types/node': 25.8.0
|
'@types/node': 25.9.0
|
||||||
|
|
||||||
'@types/triple-beam@1.3.5': {}
|
'@types/triple-beam@1.3.5': {}
|
||||||
|
|
||||||
@@ -8079,6 +8334,8 @@ snapshots:
|
|||||||
|
|
||||||
obug@2.1.1: {}
|
obug@2.1.1: {}
|
||||||
|
|
||||||
|
on-exit-leak-free@2.1.2: {}
|
||||||
|
|
||||||
on-finished@2.4.1:
|
on-finished@2.4.1:
|
||||||
dependencies:
|
dependencies:
|
||||||
ee-first: 1.1.1
|
ee-first: 1.1.1
|
||||||
@@ -8245,6 +8502,33 @@ snapshots:
|
|||||||
|
|
||||||
picomatch@4.0.4: {}
|
picomatch@4.0.4: {}
|
||||||
|
|
||||||
|
pino-abstract-transport@2.0.0:
|
||||||
|
dependencies:
|
||||||
|
split2: 4.2.0
|
||||||
|
|
||||||
|
pino-http@10.5.0:
|
||||||
|
dependencies:
|
||||||
|
get-caller-file: 2.0.5
|
||||||
|
pino: 9.14.0
|
||||||
|
pino-std-serializers: 7.1.0
|
||||||
|
process-warning: 5.0.0
|
||||||
|
|
||||||
|
pino-std-serializers@7.1.0: {}
|
||||||
|
|
||||||
|
pino@9.14.0:
|
||||||
|
dependencies:
|
||||||
|
'@pinojs/redact': 0.4.0
|
||||||
|
atomic-sleep: 1.0.0
|
||||||
|
on-exit-leak-free: 2.1.2
|
||||||
|
pino-abstract-transport: 2.0.0
|
||||||
|
pino-std-serializers: 7.1.0
|
||||||
|
process-warning: 5.0.0
|
||||||
|
quick-format-unescaped: 4.0.4
|
||||||
|
real-require: 0.2.0
|
||||||
|
safe-stable-stringify: 2.5.0
|
||||||
|
sonic-boom: 4.2.1
|
||||||
|
thread-stream: 3.1.0
|
||||||
|
|
||||||
piscina@5.1.4:
|
piscina@5.1.4:
|
||||||
optionalDependencies:
|
optionalDependencies:
|
||||||
'@napi-rs/nice': 1.1.1
|
'@napi-rs/nice': 1.1.1
|
||||||
@@ -8334,6 +8618,8 @@ snapshots:
|
|||||||
|
|
||||||
process-nextick-args@2.0.1: {}
|
process-nextick-args@2.0.1: {}
|
||||||
|
|
||||||
|
process-warning@5.0.0: {}
|
||||||
|
|
||||||
progress@2.0.3: {}
|
progress@2.0.3: {}
|
||||||
|
|
||||||
prom-client@15.1.3:
|
prom-client@15.1.3:
|
||||||
@@ -8388,6 +8674,8 @@ snapshots:
|
|||||||
|
|
||||||
queue-microtask@1.2.3: {}
|
queue-microtask@1.2.3: {}
|
||||||
|
|
||||||
|
quick-format-unescaped@4.0.4: {}
|
||||||
|
|
||||||
quick-lru@4.0.1: {}
|
quick-lru@4.0.1: {}
|
||||||
|
|
||||||
quick-lru@7.3.0: {}
|
quick-lru@7.3.0: {}
|
||||||
@@ -8446,6 +8734,8 @@ snapshots:
|
|||||||
string_decoder: 1.3.0
|
string_decoder: 1.3.0
|
||||||
util-deprecate: 1.0.2
|
util-deprecate: 1.0.2
|
||||||
|
|
||||||
|
real-require@0.2.0: {}
|
||||||
|
|
||||||
redent@3.0.0:
|
redent@3.0.0:
|
||||||
dependencies:
|
dependencies:
|
||||||
indent-string: 4.0.0
|
indent-string: 4.0.0
|
||||||
@@ -8887,6 +9177,10 @@ snapshots:
|
|||||||
|
|
||||||
thirty-two@1.0.2: {}
|
thirty-two@1.0.2: {}
|
||||||
|
|
||||||
|
thread-stream@3.1.0:
|
||||||
|
dependencies:
|
||||||
|
real-require: 0.2.0
|
||||||
|
|
||||||
through@2.3.8: {}
|
through@2.3.8: {}
|
||||||
|
|
||||||
thunky@1.1.0: {}
|
thunky@1.1.0: {}
|
||||||
|
|||||||
@@ -1,5 +1,9 @@
|
|||||||
packages:
|
packages:
|
||||||
- .
|
- .
|
||||||
|
- services/frontend
|
||||||
|
- services/backend
|
||||||
|
- services/discord-gateway
|
||||||
|
- packages/shared
|
||||||
- vendor/discord-video-stream
|
- vendor/discord-video-stream
|
||||||
- vendor/discord.js-selfbot-v13
|
- vendor/discord.js-selfbot-v13
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,275 @@
|
|||||||
|
# Backend Service Architecture Map
|
||||||
|
|
||||||
|
## Directory Structure
|
||||||
|
|
||||||
|
```
|
||||||
|
services/backend/
|
||||||
|
├── src/
|
||||||
|
│ ├── shared/ # Shared infrastructure (no business logic)
|
||||||
|
│ │ ├── config/
|
||||||
|
│ │ │ └── index.ts # Zod-validated environment config
|
||||||
|
│ │ ├── database/
|
||||||
|
│ │ │ └── index.ts # Drizzle ORM initialization & connection pool
|
||||||
|
│ │ ├── errors/
|
||||||
|
│ │ │ └── index.ts # Custom error classes (AppError, ValidationError, etc.)
|
||||||
|
│ │ ├── logger/
|
||||||
|
│ │ │ └── index.ts # Pino logger with child context support
|
||||||
|
│ │ ├── middlewares/
|
||||||
|
│ │ │ └── index.ts # Express middleware (errorHandler, asyncHandler, adminAuth)
|
||||||
|
│ │ └── utils/ # Utility functions (placeholder)
|
||||||
|
│ │
|
||||||
|
│ ├── modules/ # Feature modules (Modular MVC pattern)
|
||||||
|
│ │ ├── messages/
|
||||||
|
│ │ │ ├── messages.schema.ts # Zod validation schemas (MessageQuery, MessageCreate, MessageUpdate)
|
||||||
|
│ │ │ ├── messages.repository.ts # Database operations (findMany, findById, create, update, delete)
|
||||||
|
│ │ │ ├── messages.service.ts # Business logic (validation, orchestration)
|
||||||
|
│ │ │ ├── messages.controller.ts # Request handlers (parse → service → response)
|
||||||
|
│ │ │ └── routes/
|
||||||
|
│ │ │ └── index.ts # Express router (GET /api/messages, etc.)
|
||||||
|
│ │ │
|
||||||
|
│ │ ├── analytics/
|
||||||
|
│ │ │ ├── analytics.schema.ts
|
||||||
|
│ │ │ ├── analytics.repository.ts
|
||||||
|
│ │ │ ├── analytics.service.ts
|
||||||
|
│ │ │ ├── analytics.controller.ts
|
||||||
|
│ │ │ └── routes/
|
||||||
|
│ │ │ └── index.ts
|
||||||
|
│ │ │
|
||||||
|
│ │ ├── media/
|
||||||
|
│ │ │ ├── media.service.ts
|
||||||
|
│ │ │ └── routes/
|
||||||
|
│ │ │ └── index.ts
|
||||||
|
│ │ │
|
||||||
|
│ │ ├── voice/
|
||||||
|
│ │ │ ├── voice.service.ts
|
||||||
|
│ │ │ └── routes/
|
||||||
|
│ │ │ └── index.ts
|
||||||
|
│ │ │
|
||||||
|
│ │ └── health/
|
||||||
|
│ │ ├── health.schema.ts
|
||||||
|
│ │ ├── health.repository.ts
|
||||||
|
│ │ ├── health.service.ts
|
||||||
|
│ │ ├── health.controller.ts
|
||||||
|
│ │ └── routes/
|
||||||
|
│ │ └── index.ts
|
||||||
|
│ │
|
||||||
|
│ ├── http/
|
||||||
|
│ │ ├── app.ts # Express app factory (middleware, routes, error handler)
|
||||||
|
│ │ └── server.ts # HTTP server startup (port binding, graceful shutdown)
|
||||||
|
│ │
|
||||||
|
│ ├── ws/ # WebSocket server (placeholder for real-time updates)
|
||||||
|
│ │ └── server.ts # Redis pub/sub listener for Discord Gateway events
|
||||||
|
│ │
|
||||||
|
│ └── index.ts # Entry point (main function, signal handlers)
|
||||||
|
│
|
||||||
|
├── package.json # Backend dependencies
|
||||||
|
├── tsconfig.json # TypeScript configuration
|
||||||
|
└── README.md # Backend-specific documentation
|
||||||
|
```
|
||||||
|
|
||||||
|
## Layer Separation
|
||||||
|
|
||||||
|
### 1. Controller Layer
|
||||||
|
**File:** `modules/*/[module].controller.ts`
|
||||||
|
**Responsibility:** HTTP request handling only
|
||||||
|
- Parse request (query, params, body)
|
||||||
|
- Validate using Zod schemas
|
||||||
|
- Call service methods
|
||||||
|
- Return HTTP response (200, 400, 404, 500)
|
||||||
|
- **No database calls**
|
||||||
|
- **No business logic**
|
||||||
|
|
||||||
|
**Example:**
|
||||||
|
```typescript
|
||||||
|
export function handleListMessages(req: Request, res: Response, next: NextFunction) {
|
||||||
|
return asyncHandler(async (req: Request, res: Response) => {
|
||||||
|
const query = messageQuerySchema.parse(req.query);
|
||||||
|
const result = await messagesService.listMessages(query);
|
||||||
|
res.json(result);
|
||||||
|
})(req, res, next);
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### 2. Service Layer
|
||||||
|
**File:** `modules/*/[module].service.ts`
|
||||||
|
**Responsibility:** Business logic and orchestration
|
||||||
|
- Validate input (throw ValidationError if invalid)
|
||||||
|
- Orchestrate repository calls
|
||||||
|
- Apply business rules
|
||||||
|
- Handle cross-cutting concerns (auth, permissions)
|
||||||
|
- **No database calls directly**
|
||||||
|
- **No HTTP request/response handling**
|
||||||
|
|
||||||
|
**Example:**
|
||||||
|
```typescript
|
||||||
|
async listMessages(query: MessageQuery) {
|
||||||
|
if (!query.channelId && !query.guildId) {
|
||||||
|
throw new ValidationError("Either channelId or guildId is required");
|
||||||
|
}
|
||||||
|
return messagesRepository.findMany(query);
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### 3. Repository Layer
|
||||||
|
**File:** `modules/*/[module].repository.ts`
|
||||||
|
**Responsibility:** All database operations
|
||||||
|
- Execute Drizzle ORM queries
|
||||||
|
- Handle database errors
|
||||||
|
- Return raw data (no transformation)
|
||||||
|
- **No business logic**
|
||||||
|
- **No HTTP handling**
|
||||||
|
|
||||||
|
**Example:**
|
||||||
|
```typescript
|
||||||
|
async findMany(query: MessageQuery) {
|
||||||
|
const db = getDatabase();
|
||||||
|
return db.select().from(messagesTable).where(...).limit(query.limit);
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### 4. Schema Layer
|
||||||
|
**File:** `modules/*/[module].schema.ts`
|
||||||
|
**Responsibility:** Zod validation schemas
|
||||||
|
- Define request/response types
|
||||||
|
- Validate at controller entry point
|
||||||
|
- Export TypeScript types
|
||||||
|
|
||||||
|
**Example:**
|
||||||
|
```typescript
|
||||||
|
export const messageQuerySchema = z.object({
|
||||||
|
channelId: z.string().optional(),
|
||||||
|
limit: z.coerce.number().int().positive().default(50),
|
||||||
|
});
|
||||||
|
```
|
||||||
|
|
||||||
|
## Module Responsibilities
|
||||||
|
|
||||||
|
| Module | Purpose | Routes |
|
||||||
|
|--------|---------|--------|
|
||||||
|
| **messages** | Text message storage & retrieval | GET /api/messages, GET /api/messages/:channelId |
|
||||||
|
| **analytics** | Moderation statistics & trends | GET /api/analytics/overview, /daily-trend, /hourly-stats |
|
||||||
|
| **media** | Media file management | GET /api/media/list, POST /api/media/upload |
|
||||||
|
| **voice** | Voice recording management | GET /api/voice/recordings, POST /api/voice/connect |
|
||||||
|
| **health** | Service health checks | GET /api/health |
|
||||||
|
|
||||||
|
## Data Flow
|
||||||
|
|
||||||
|
### Request Flow (HTTP)
|
||||||
|
```
|
||||||
|
Client Request
|
||||||
|
↓
|
||||||
|
Express Router (routes/index.ts)
|
||||||
|
↓
|
||||||
|
Controller (parse request, validate schema)
|
||||||
|
↓
|
||||||
|
Service (business logic, validation)
|
||||||
|
↓
|
||||||
|
Repository (database query)
|
||||||
|
↓
|
||||||
|
Database (PostgreSQL)
|
||||||
|
↓
|
||||||
|
Repository (return data)
|
||||||
|
↓
|
||||||
|
Service (transform/orchestrate)
|
||||||
|
↓
|
||||||
|
Controller (format response)
|
||||||
|
↓
|
||||||
|
Client Response
|
||||||
|
```
|
||||||
|
|
||||||
|
### Event Flow (WebSocket - Future)
|
||||||
|
```
|
||||||
|
Discord Gateway (publishes event)
|
||||||
|
↓
|
||||||
|
Redis pub/sub
|
||||||
|
↓
|
||||||
|
Backend WebSocket Server (ws/server.ts)
|
||||||
|
↓
|
||||||
|
Broadcast to connected clients
|
||||||
|
↓
|
||||||
|
Frontend (receives real-time update)
|
||||||
|
```
|
||||||
|
|
||||||
|
## Dependency Rules
|
||||||
|
|
||||||
|
### ✅ Allowed
|
||||||
|
- Controller → Service
|
||||||
|
- Service → Repository
|
||||||
|
- Service → Config
|
||||||
|
- Service → Logger
|
||||||
|
- Repository → Database
|
||||||
|
- Any layer → Errors, Logger, Config
|
||||||
|
|
||||||
|
### ❌ Forbidden
|
||||||
|
- Repository → Service (data flows up, not down)
|
||||||
|
- Repository → Controller
|
||||||
|
- Service → HTTP (no req/res in service)
|
||||||
|
- Controller → Database (must go through service)
|
||||||
|
- Cross-module repository imports (each module owns its data)
|
||||||
|
|
||||||
|
## Error Handling
|
||||||
|
|
||||||
|
All errors inherit from `AppError` with `code` and `statusCode`:
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
throw new ValidationError("Invalid input", { field: "error" }); // 400
|
||||||
|
throw new NotFoundError("Message not found"); // 404
|
||||||
|
throw new UnauthorizedError("Invalid password"); // 401
|
||||||
|
throw new ForbiddenError("Access denied"); // 403
|
||||||
|
throw new AppError("Custom error", "CUSTOM_CODE", 500); // 500
|
||||||
|
```
|
||||||
|
|
||||||
|
## Configuration
|
||||||
|
|
||||||
|
All config via environment variables (`.env`), validated with Zod in `shared/config/index.ts`:
|
||||||
|
|
||||||
|
```env
|
||||||
|
# Server
|
||||||
|
WEBSERVER_PORT=3001
|
||||||
|
NODE_ENV=development
|
||||||
|
LOG_LEVEL=info
|
||||||
|
|
||||||
|
# Database
|
||||||
|
DATABASE_URL=postgresql://user:pass@localhost:5432/discord_moderation
|
||||||
|
# OR
|
||||||
|
DATABASE_HOST=localhost
|
||||||
|
DATABASE_PORT=5432
|
||||||
|
DATABASE_NAME=discord_moderation
|
||||||
|
DATABASE_USER=postgres
|
||||||
|
DATABASE_PASSWORD=secret
|
||||||
|
|
||||||
|
# Redis (optional, for pub/sub)
|
||||||
|
REDIS_URL=redis://localhost:6379
|
||||||
|
|
||||||
|
# Discord
|
||||||
|
MONITOR_GUILD_ID=123456789
|
||||||
|
|
||||||
|
# Admin
|
||||||
|
ADMIN_PASSWORD=secret123
|
||||||
|
```
|
||||||
|
|
||||||
|
## Testing Strategy
|
||||||
|
|
||||||
|
Each module should have tests:
|
||||||
|
- `messages.repository.test.ts` — Database query tests
|
||||||
|
- `messages.service.test.ts` — Business logic tests
|
||||||
|
- `messages.controller.test.ts` — HTTP handler tests
|
||||||
|
|
||||||
|
Use Vitest with mocked database and services.
|
||||||
|
|
||||||
|
## Next Steps
|
||||||
|
|
||||||
|
1. **Migrate Drizzle schema** from `src/database/schema.ts` to `services/backend/src/shared/database/schema.ts`
|
||||||
|
2. **Implement repository queries** for each module using Drizzle ORM
|
||||||
|
3. **Add WebSocket server** in `src/ws/server.ts` with Redis pub/sub listener
|
||||||
|
4. **Create Discord Gateway service** in `services/discord-gateway/` (separate microservice)
|
||||||
|
5. **Add Docker & CI/CD** for multi-service deployment
|
||||||
|
6. **Write integration tests** for full request flow
|
||||||
|
|
||||||
|
## Circular Dependency Check
|
||||||
|
|
||||||
|
✅ No circular dependencies detected:
|
||||||
|
- Modules are independent (each owns its data)
|
||||||
|
- Layers flow upward only (Repository → Service → Controller)
|
||||||
|
- Shared infrastructure has no dependencies on modules
|
||||||
|
- Cross-module communication via events (Redis pub/sub), not direct imports
|
||||||
@@ -0,0 +1,41 @@
|
|||||||
|
{
|
||||||
|
"name": "discord-moderation-backend",
|
||||||
|
"version": "1.0.0",
|
||||||
|
"description": "Backend service for Discord moderation monitoring",
|
||||||
|
"type": "module",
|
||||||
|
"main": "dist/index.js",
|
||||||
|
"scripts": {
|
||||||
|
"dev": "tsx watch src/index.ts",
|
||||||
|
"start": "node dist/index.js",
|
||||||
|
"build": "tsc",
|
||||||
|
"typecheck": "tsc --noEmit",
|
||||||
|
"lint": "biome check --diagnostic-level=error .",
|
||||||
|
"format": "biome format --write .",
|
||||||
|
"test": "vitest run"
|
||||||
|
},
|
||||||
|
"dependencies": {
|
||||||
|
"@discordjs/voice": "^0.19.2",
|
||||||
|
"@types/pg": "^8.20.0",
|
||||||
|
"axios": "^1.16.1",
|
||||||
|
"dotenv": "^17.4.2",
|
||||||
|
"drizzle-orm": "^0.45.2",
|
||||||
|
"express": "^5.2.1",
|
||||||
|
"helmet": "^8.1.0",
|
||||||
|
"ioredis": "^5.11.0",
|
||||||
|
"pg": "^8.21.0",
|
||||||
|
"pino": "^9.6.0",
|
||||||
|
"pino-http": "^10.3.0",
|
||||||
|
"prom-client": "^15.1.3",
|
||||||
|
"ws": "^8.20.1",
|
||||||
|
"zod": "^4.4.3"
|
||||||
|
},
|
||||||
|
"devDependencies": {
|
||||||
|
"@biomejs/biome": "latest",
|
||||||
|
"@types/express": "^5.0.6",
|
||||||
|
"@types/node": "^25.9.0",
|
||||||
|
"@types/ws": "^8.18.1",
|
||||||
|
"tsx": "^4.22.2",
|
||||||
|
"typescript": "^5.9.3",
|
||||||
|
"vitest": "latest"
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,75 @@
|
|||||||
|
import express, {
|
||||||
|
type Express,
|
||||||
|
type NextFunction,
|
||||||
|
type Request,
|
||||||
|
type Response,
|
||||||
|
} from "express";
|
||||||
|
import helmet from "helmet";
|
||||||
|
import { createAnalyticsRouter } from "../modules/analytics/routes/index.js";
|
||||||
|
import { createHealthRouter } from "../modules/health/routes/index.js";
|
||||||
|
import { createMediaRouter } from "../modules/media/routes/index.js";
|
||||||
|
import { createMessagesRouter } from "../modules/messages/routes/index.js";
|
||||||
|
import { createVoiceRouter } from "../modules/voice/routes/index.js";
|
||||||
|
import { createChildLogger } from "../shared/logger/index.js";
|
||||||
|
import { errorHandler } from "../shared/middlewares/index.js";
|
||||||
|
|
||||||
|
const logger = createChildLogger("http.app");
|
||||||
|
|
||||||
|
export function createHttpApp(): Express {
|
||||||
|
const app = express();
|
||||||
|
|
||||||
|
// Security middleware
|
||||||
|
app.use(
|
||||||
|
helmet({
|
||||||
|
contentSecurityPolicy: false,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
// Body parsing
|
||||||
|
app.use(express.json());
|
||||||
|
app.use(express.urlencoded({ extended: true }));
|
||||||
|
|
||||||
|
// Request logging
|
||||||
|
app.use((req: Request, res: Response, next: NextFunction) => {
|
||||||
|
if (req.path.startsWith("/api/")) {
|
||||||
|
res.set("Cache-Control", "no-store");
|
||||||
|
}
|
||||||
|
res.on("finish", () => {
|
||||||
|
if (req.originalUrl.startsWith("/.well-known/")) return;
|
||||||
|
if (req.originalUrl === "/favicon.ico") return;
|
||||||
|
if (res.statusCode >= 400) {
|
||||||
|
logger.warn(
|
||||||
|
{
|
||||||
|
method: req.method,
|
||||||
|
url: req.originalUrl,
|
||||||
|
statusCode: res.statusCode,
|
||||||
|
},
|
||||||
|
"HTTP request failed",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
next();
|
||||||
|
});
|
||||||
|
|
||||||
|
// Health check (no auth required)
|
||||||
|
app.use("/api", createHealthRouter());
|
||||||
|
|
||||||
|
// API routes
|
||||||
|
app.use("/api", createMessagesRouter());
|
||||||
|
app.use("/api", createAnalyticsRouter());
|
||||||
|
app.use("/api", createMediaRouter());
|
||||||
|
app.use("/api", createVoiceRouter());
|
||||||
|
|
||||||
|
// 404 handler
|
||||||
|
app.use((_req: Request, res: Response) => {
|
||||||
|
res.status(404).json({
|
||||||
|
error: "NOT_FOUND",
|
||||||
|
message: "Endpoint not found",
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
// Error handler (must be last)
|
||||||
|
app.use(errorHandler);
|
||||||
|
|
||||||
|
return app;
|
||||||
|
}
|
||||||
@@ -0,0 +1,25 @@
|
|||||||
|
import { config } from "../shared/config/index.js";
|
||||||
|
import { initializeDatabase } from "../shared/database/index.js";
|
||||||
|
import { createChildLogger } from "../shared/logger/index.js";
|
||||||
|
import { createHttpApp } from "./app.js";
|
||||||
|
|
||||||
|
const logger = createChildLogger("http.server");
|
||||||
|
|
||||||
|
export async function startHttpServer() {
|
||||||
|
await initializeDatabase();
|
||||||
|
|
||||||
|
const app = createHttpApp();
|
||||||
|
const port = config.WEBSERVER_PORT;
|
||||||
|
|
||||||
|
return new Promise<void>((resolve, reject) => {
|
||||||
|
const server = app.listen(port, () => {
|
||||||
|
logger.info({ port }, "HTTP server started");
|
||||||
|
resolve();
|
||||||
|
});
|
||||||
|
|
||||||
|
server.on("error", (err) => {
|
||||||
|
logger.error({ err }, "HTTP server error");
|
||||||
|
reject(err);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
@@ -0,0 +1,38 @@
|
|||||||
|
import { startHttpServer } from "./http/server.js";
|
||||||
|
import { createChildLogger } from "./shared/logger/index.js";
|
||||||
|
|
||||||
|
const logger = createChildLogger("backend");
|
||||||
|
|
||||||
|
async function main() {
|
||||||
|
try {
|
||||||
|
logger.info("Starting Discord Moderation Backend Service");
|
||||||
|
await startHttpServer();
|
||||||
|
logger.info("Backend service ready");
|
||||||
|
} catch (err) {
|
||||||
|
logger.error({ err }, "Failed to start backend service");
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Graceful shutdown
|
||||||
|
process.on("SIGINT", () => {
|
||||||
|
logger.info("Received SIGINT, shutting down gracefully");
|
||||||
|
process.exit(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
process.on("SIGTERM", () => {
|
||||||
|
logger.info("Received SIGTERM, shutting down gracefully");
|
||||||
|
process.exit(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
process.on("uncaughtException", (err) => {
|
||||||
|
logger.error({ err }, "Uncaught exception");
|
||||||
|
process.exit(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
process.on("unhandledRejection", (reason) => {
|
||||||
|
logger.error({ reason }, "Unhandled rejection");
|
||||||
|
process.exit(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
main();
|
||||||
@@ -0,0 +1,96 @@
|
|||||||
|
import type { NextFunction, Request, Response } from "express";
|
||||||
|
import { createChildLogger } from "../../shared/logger/index.js";
|
||||||
|
import { asyncHandler } from "../../shared/middlewares/index.js";
|
||||||
|
import { analyticsQuerySchema } from "./analytics.schema.js";
|
||||||
|
import { analyticsService } from "./analytics.service.js";
|
||||||
|
|
||||||
|
const logger = createChildLogger("analytics.controller");
|
||||||
|
|
||||||
|
function requireQueryString(value: unknown, name: string): string {
|
||||||
|
if (typeof value !== "string" || value.length === 0) {
|
||||||
|
throw new Error(`Missing query parameter: ${name}`);
|
||||||
|
}
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function handleGetOverview(
|
||||||
|
req: Request,
|
||||||
|
res: Response,
|
||||||
|
next: NextFunction,
|
||||||
|
) {
|
||||||
|
return asyncHandler(async (req: Request, res: Response) => {
|
||||||
|
const query = analyticsQuerySchema.parse(req.query);
|
||||||
|
logger.debug({ query }, "Handling get overview");
|
||||||
|
const result = await analyticsService.getOverview(query);
|
||||||
|
res.json(result);
|
||||||
|
})(req, res, next);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function handleGetDailyTrend(
|
||||||
|
req: Request,
|
||||||
|
res: Response,
|
||||||
|
next: NextFunction,
|
||||||
|
) {
|
||||||
|
return asyncHandler(async (req: Request, res: Response) => {
|
||||||
|
const guildId = requireQueryString(req.query.guildId, "guildId");
|
||||||
|
const hours = req.query.hours ? Number(req.query.hours) : 24;
|
||||||
|
logger.debug({ guildId, hours }, "Handling get daily trend");
|
||||||
|
const result = await analyticsService.getDailyTrend(guildId, hours);
|
||||||
|
res.json(result);
|
||||||
|
})(req, res, next);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function handleGetHourlyStats(
|
||||||
|
req: Request,
|
||||||
|
res: Response,
|
||||||
|
next: NextFunction,
|
||||||
|
) {
|
||||||
|
return asyncHandler(async (req: Request, res: Response) => {
|
||||||
|
const guildId = requireQueryString(req.query.guildId, "guildId");
|
||||||
|
const hours = req.query.hours ? Number(req.query.hours) : 24;
|
||||||
|
logger.debug({ guildId, hours }, "Handling get hourly stats");
|
||||||
|
const result = await analyticsService.getHourlyStats(guildId, hours);
|
||||||
|
res.json(result);
|
||||||
|
})(req, res, next);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function handleGetTopViolators(
|
||||||
|
req: Request,
|
||||||
|
res: Response,
|
||||||
|
next: NextFunction,
|
||||||
|
) {
|
||||||
|
return asyncHandler(async (req: Request, res: Response) => {
|
||||||
|
const guildId = requireQueryString(req.query.guildId, "guildId");
|
||||||
|
const limit = req.query.limit ? Number(req.query.limit) : 10;
|
||||||
|
logger.debug({ guildId, limit }, "Handling get top violators");
|
||||||
|
const result = await analyticsService.getTopViolators(guildId, limit);
|
||||||
|
res.json(result);
|
||||||
|
})(req, res, next);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function handleGetUserLeaderboard(
|
||||||
|
req: Request,
|
||||||
|
res: Response,
|
||||||
|
next: NextFunction,
|
||||||
|
) {
|
||||||
|
return asyncHandler(async (req: Request, res: Response) => {
|
||||||
|
const guildId = requireQueryString(req.query.guildId, "guildId");
|
||||||
|
const limit = req.query.limit ? Number(req.query.limit) : 10;
|
||||||
|
logger.debug({ guildId, limit }, "Handling get user leaderboard");
|
||||||
|
const result = await analyticsService.getUserLeaderboard(guildId, limit);
|
||||||
|
res.json(result);
|
||||||
|
})(req, res, next);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function handleGetModerationStats(
|
||||||
|
req: Request,
|
||||||
|
res: Response,
|
||||||
|
next: NextFunction,
|
||||||
|
) {
|
||||||
|
return asyncHandler(async (req: Request, res: Response) => {
|
||||||
|
const guildId = requireQueryString(req.query.guildId, "guildId");
|
||||||
|
logger.debug({ guildId }, "Handling get moderation stats");
|
||||||
|
const result = await analyticsService.getModerationStats(guildId);
|
||||||
|
res.json(result);
|
||||||
|
})(req, res, next);
|
||||||
|
}
|
||||||
@@ -0,0 +1,53 @@
|
|||||||
|
import { createChildLogger } from "../../shared/logger/index.js";
|
||||||
|
|
||||||
|
const logger = createChildLogger("analytics.repository");
|
||||||
|
|
||||||
|
export class AnalyticsRepository {
|
||||||
|
async getOverview(guildId: string, channelId?: string, hours = 24) {
|
||||||
|
logger.debug({ guildId, channelId, hours }, "Getting analytics overview");
|
||||||
|
// TODO: Implement actual Drizzle ORM queries
|
||||||
|
return {
|
||||||
|
totalMessages: 0,
|
||||||
|
totalUsers: 0,
|
||||||
|
flaggedMessages: 0,
|
||||||
|
averageSeverity: 0,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async getDailyTrend(guildId: string, hours = 24) {
|
||||||
|
logger.debug({ guildId, hours }, "Getting daily trend");
|
||||||
|
// TODO: Implement actual Drizzle ORM queries
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
|
||||||
|
async getHourlyStats(guildId: string, hours = 24) {
|
||||||
|
logger.debug({ guildId, hours }, "Getting hourly stats");
|
||||||
|
// TODO: Implement actual Drizzle ORM queries
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
|
||||||
|
async getTopViolators(guildId: string, limit = 10) {
|
||||||
|
logger.debug({ guildId, limit }, "Getting top violators");
|
||||||
|
// TODO: Implement actual Drizzle ORM queries
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
|
||||||
|
async getUserLeaderboard(guildId: string, limit = 10) {
|
||||||
|
logger.debug({ guildId, limit }, "Getting user leaderboard");
|
||||||
|
// TODO: Implement actual Drizzle ORM queries
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
|
||||||
|
async getModerationStats(guildId: string) {
|
||||||
|
logger.debug({ guildId }, "Getting moderation stats");
|
||||||
|
// TODO: Implement actual Drizzle ORM queries
|
||||||
|
return {
|
||||||
|
clean: 0,
|
||||||
|
warn: 0,
|
||||||
|
flagged: 0,
|
||||||
|
error: 0,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export const analyticsRepository = new AnalyticsRepository();
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
import { z } from "zod";
|
||||||
|
|
||||||
|
export const analyticsQuerySchema = z.object({
|
||||||
|
guildId: z.string(),
|
||||||
|
channelId: z.string().optional(),
|
||||||
|
hours: z.coerce.number().int().positive().default(24),
|
||||||
|
});
|
||||||
|
|
||||||
|
export type AnalyticsQuery = z.infer<typeof analyticsQuerySchema>;
|
||||||
@@ -0,0 +1,61 @@
|
|||||||
|
import { config } from "../../shared/config/index.js";
|
||||||
|
import { ForbiddenError, ValidationError } from "../../shared/errors/index.js";
|
||||||
|
import { createChildLogger } from "../../shared/logger/index.js";
|
||||||
|
import { analyticsRepository } from "./analytics.repository.js";
|
||||||
|
import type { AnalyticsQuery } from "./analytics.schema.js";
|
||||||
|
|
||||||
|
const logger = createChildLogger("analytics.service");
|
||||||
|
|
||||||
|
export class AnalyticsService {
|
||||||
|
private assertMonitorGuild(guildId: string) {
|
||||||
|
if (!config.MONITOR_GUILD_ID) {
|
||||||
|
throw new ValidationError("MONITOR_GUILD_ID is not configured");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (guildId !== config.MONITOR_GUILD_ID) {
|
||||||
|
throw new ForbiddenError("Analytics are restricted to the monitor guild");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async getOverview(query: AnalyticsQuery) {
|
||||||
|
this.assertMonitorGuild(query.guildId);
|
||||||
|
logger.debug({ query }, "Getting analytics overview");
|
||||||
|
return analyticsRepository.getOverview(
|
||||||
|
query.guildId,
|
||||||
|
query.channelId,
|
||||||
|
query.hours,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
async getDailyTrend(guildId: string, hours = 24) {
|
||||||
|
this.assertMonitorGuild(guildId);
|
||||||
|
logger.debug({ guildId, hours }, "Getting daily trend");
|
||||||
|
return analyticsRepository.getDailyTrend(guildId, hours);
|
||||||
|
}
|
||||||
|
|
||||||
|
async getHourlyStats(guildId: string, hours = 24) {
|
||||||
|
this.assertMonitorGuild(guildId);
|
||||||
|
logger.debug({ guildId, hours }, "Getting hourly stats");
|
||||||
|
return analyticsRepository.getHourlyStats(guildId, hours);
|
||||||
|
}
|
||||||
|
|
||||||
|
async getTopViolators(guildId: string, limit = 10) {
|
||||||
|
this.assertMonitorGuild(guildId);
|
||||||
|
logger.debug({ guildId, limit }, "Getting top violators");
|
||||||
|
return analyticsRepository.getTopViolators(guildId, limit);
|
||||||
|
}
|
||||||
|
|
||||||
|
async getUserLeaderboard(guildId: string, limit = 10) {
|
||||||
|
this.assertMonitorGuild(guildId);
|
||||||
|
logger.debug({ guildId, limit }, "Getting user leaderboard");
|
||||||
|
return analyticsRepository.getUserLeaderboard(guildId, limit);
|
||||||
|
}
|
||||||
|
|
||||||
|
async getModerationStats(guildId: string) {
|
||||||
|
this.assertMonitorGuild(guildId);
|
||||||
|
logger.debug({ guildId }, "Getting moderation stats");
|
||||||
|
return analyticsRepository.getModerationStats(guildId);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export const analyticsService = new AnalyticsService();
|
||||||
@@ -0,0 +1,23 @@
|
|||||||
|
import type { Router } from "express";
|
||||||
|
import express from "express";
|
||||||
|
import {
|
||||||
|
handleGetDailyTrend,
|
||||||
|
handleGetHourlyStats,
|
||||||
|
handleGetModerationStats,
|
||||||
|
handleGetOverview,
|
||||||
|
handleGetTopViolators,
|
||||||
|
handleGetUserLeaderboard,
|
||||||
|
} from "../analytics.controller.js";
|
||||||
|
|
||||||
|
export function createAnalyticsRouter(): Router {
|
||||||
|
const router = express.Router();
|
||||||
|
|
||||||
|
router.get("/analytics/overview", handleGetOverview);
|
||||||
|
router.get("/analytics/daily-trend", handleGetDailyTrend);
|
||||||
|
router.get("/analytics/hourly-stats", handleGetHourlyStats);
|
||||||
|
router.get("/analytics/top-violators", handleGetTopViolators);
|
||||||
|
router.get("/analytics/user-leaderboard", handleGetUserLeaderboard);
|
||||||
|
router.get("/analytics/moderation-stats", handleGetModerationStats);
|
||||||
|
|
||||||
|
return router;
|
||||||
|
}
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
import type { NextFunction, Request, Response } from "express";
|
||||||
|
import { asyncHandler } from "../../shared/middlewares/index.js";
|
||||||
|
import { healthService } from "./health.service.js";
|
||||||
|
|
||||||
|
export function handleHealthCheck(
|
||||||
|
req: Request,
|
||||||
|
res: Response,
|
||||||
|
next: NextFunction,
|
||||||
|
) {
|
||||||
|
return asyncHandler(async (req: Request, res: Response) => {
|
||||||
|
const verbose = req.query.verbose === "true";
|
||||||
|
const result = await healthService.getHealth(verbose);
|
||||||
|
const status = result.status === "healthy" ? 200 : 503;
|
||||||
|
res.status(status).json(result);
|
||||||
|
})(req, res, next);
|
||||||
|
}
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
import { createChildLogger } from "../../shared/logger/index.js";
|
||||||
|
|
||||||
|
const logger = createChildLogger("health.repository");
|
||||||
|
|
||||||
|
export class HealthRepository {
|
||||||
|
async checkDatabaseConnection() {
|
||||||
|
try {
|
||||||
|
// TODO: Implement actual health check
|
||||||
|
return { connected: true };
|
||||||
|
} catch (err) {
|
||||||
|
logger.error({ err }, "Database health check failed");
|
||||||
|
return { connected: false };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export const healthRepository = new HealthRepository();
|
||||||
@@ -0,0 +1,5 @@
|
|||||||
|
import { z } from "zod";
|
||||||
|
|
||||||
|
export const healthCheckSchema = z.object({
|
||||||
|
verbose: z.coerce.boolean().optional().default(false),
|
||||||
|
});
|
||||||
@@ -0,0 +1,20 @@
|
|||||||
|
import { createChildLogger } from "../../shared/logger/index.js";
|
||||||
|
import { healthRepository } from "./health.repository.js";
|
||||||
|
|
||||||
|
const logger = createChildLogger("health.service");
|
||||||
|
|
||||||
|
export class HealthService {
|
||||||
|
async getHealth(verbose = false) {
|
||||||
|
const dbStatus = await healthRepository.checkDatabaseConnection();
|
||||||
|
|
||||||
|
return {
|
||||||
|
status: dbStatus.connected ? "healthy" : "degraded",
|
||||||
|
timestamp: Date.now(),
|
||||||
|
...(verbose && {
|
||||||
|
database: dbStatus,
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export const healthService = new HealthService();
|
||||||
@@ -0,0 +1,12 @@
|
|||||||
|
import type { Router } from "express";
|
||||||
|
import express from "express";
|
||||||
|
import { handleHealthCheck } from "../health.controller.js";
|
||||||
|
|
||||||
|
export function createHealthRouter(): Router {
|
||||||
|
const router = express.Router();
|
||||||
|
|
||||||
|
// GET /api/health
|
||||||
|
router.get("/health", handleHealthCheck);
|
||||||
|
|
||||||
|
return router;
|
||||||
|
}
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
import { createChildLogger } from "../../shared/logger/index.js";
|
||||||
|
|
||||||
|
const logger = createChildLogger("media.service");
|
||||||
|
|
||||||
|
export class MediaService {
|
||||||
|
// TODO: Implement media service methods
|
||||||
|
}
|
||||||
|
|
||||||
|
export const mediaService = new MediaService();
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
import type { Router } from "express";
|
||||||
|
import express from "express";
|
||||||
|
|
||||||
|
export function createMediaRouter(): Router {
|
||||||
|
const router = express.Router();
|
||||||
|
|
||||||
|
// TODO: Implement media routes
|
||||||
|
// GET /api/media/list
|
||||||
|
// POST /api/media/upload
|
||||||
|
// GET /api/media/:id
|
||||||
|
|
||||||
|
return router;
|
||||||
|
}
|
||||||
@@ -0,0 +1,74 @@
|
|||||||
|
import type { NextFunction, Request, Response } from "express";
|
||||||
|
import { createChildLogger } from "../../shared/logger/index.js";
|
||||||
|
import { asyncHandler } from "../../shared/middlewares/index.js";
|
||||||
|
import { messageQuerySchema } from "./messages.schema.js";
|
||||||
|
import { messagesService } from "./messages.service.js";
|
||||||
|
|
||||||
|
const logger = createChildLogger("messages.controller");
|
||||||
|
|
||||||
|
function requireRouteParam(
|
||||||
|
value: string | string[] | undefined,
|
||||||
|
name: string,
|
||||||
|
): string {
|
||||||
|
if (typeof value !== "string" || value.length === 0) {
|
||||||
|
throw new Error(`Missing route parameter: ${name}`);
|
||||||
|
}
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function handleListMessages(
|
||||||
|
req: Request,
|
||||||
|
res: Response,
|
||||||
|
next: NextFunction,
|
||||||
|
) {
|
||||||
|
return asyncHandler(async (req: Request, res: Response) => {
|
||||||
|
const query = messageQuerySchema.parse(req.query);
|
||||||
|
logger.debug({ query }, "Handling list messages request");
|
||||||
|
const result = await messagesService.listMessages(query);
|
||||||
|
res.json(result);
|
||||||
|
})(req, res, next);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function handleGetMessagesByChannel(
|
||||||
|
req: Request,
|
||||||
|
res: Response,
|
||||||
|
next: NextFunction,
|
||||||
|
) {
|
||||||
|
return asyncHandler(async (req: Request, res: Response) => {
|
||||||
|
const channelId = requireRouteParam(req.params.channelId, "channelId");
|
||||||
|
const query = messageQuerySchema.parse(req.query);
|
||||||
|
logger.debug({ channelId, query }, "Handling get messages by channel");
|
||||||
|
const result = await messagesService.getMessagesByChannel(channelId, query);
|
||||||
|
res.json(result);
|
||||||
|
})(req, res, next);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function handleGetMessageById(
|
||||||
|
req: Request,
|
||||||
|
res: Response,
|
||||||
|
next: NextFunction,
|
||||||
|
) {
|
||||||
|
return asyncHandler(async (req: Request, res: Response) => {
|
||||||
|
const id = requireRouteParam(req.params.id, "id");
|
||||||
|
logger.debug({ id }, "Handling get message by ID");
|
||||||
|
const result = await messagesService.getMessageById(id);
|
||||||
|
res.json(result);
|
||||||
|
})(req, res, next);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function handleGetAttachmentsByChannel(
|
||||||
|
req: Request,
|
||||||
|
res: Response,
|
||||||
|
next: NextFunction,
|
||||||
|
) {
|
||||||
|
return asyncHandler(async (req: Request, res: Response) => {
|
||||||
|
const channelId = requireRouteParam(req.params.channelId, "channelId");
|
||||||
|
const query = messageQuerySchema.parse(req.query);
|
||||||
|
logger.debug({ channelId, query }, "Handling get attachments by channel");
|
||||||
|
const result = await messagesService.getAttachmentsByChannel(
|
||||||
|
channelId,
|
||||||
|
query,
|
||||||
|
);
|
||||||
|
res.json(result);
|
||||||
|
})(req, res, next);
|
||||||
|
}
|
||||||
@@ -0,0 +1,86 @@
|
|||||||
|
import { getDatabase } from "../../shared/database/index.js";
|
||||||
|
import { createChildLogger } from "../../shared/logger/index.js";
|
||||||
|
import type {
|
||||||
|
MessageCreate,
|
||||||
|
MessageQuery,
|
||||||
|
MessageUpdate,
|
||||||
|
} from "./messages.schema.js";
|
||||||
|
|
||||||
|
const logger = createChildLogger("messages.repository");
|
||||||
|
|
||||||
|
export class MessagesRepository {
|
||||||
|
async findMany(query: MessageQuery) {
|
||||||
|
const db = getDatabase();
|
||||||
|
logger.debug({ query }, "Finding messages");
|
||||||
|
|
||||||
|
// TODO: Implement actual Drizzle ORM queries
|
||||||
|
// This is a placeholder that will be filled in when schema is migrated
|
||||||
|
return {
|
||||||
|
messages: [],
|
||||||
|
total: 0,
|
||||||
|
hasMore: false,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async findById(id: string) {
|
||||||
|
const db = getDatabase();
|
||||||
|
logger.debug({ id }, "Finding message by ID");
|
||||||
|
|
||||||
|
// TODO: Implement actual Drizzle ORM query
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
async findByChannel(channelId: string, query: MessageQuery) {
|
||||||
|
const db = getDatabase();
|
||||||
|
logger.debug({ channelId, query }, "Finding messages by channel");
|
||||||
|
|
||||||
|
// TODO: Implement actual Drizzle ORM queries
|
||||||
|
return {
|
||||||
|
messages: [],
|
||||||
|
total: 0,
|
||||||
|
hasMore: false,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async create(data: MessageCreate) {
|
||||||
|
const db = getDatabase();
|
||||||
|
logger.debug({ data }, "Creating message");
|
||||||
|
|
||||||
|
// TODO: Implement actual Drizzle ORM insert
|
||||||
|
return {
|
||||||
|
id: "msg_" + Date.now(),
|
||||||
|
...data,
|
||||||
|
createdAt: Date.now(),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async update(id: string, data: MessageUpdate) {
|
||||||
|
const db = getDatabase();
|
||||||
|
logger.debug({ id, data }, "Updating message");
|
||||||
|
|
||||||
|
// TODO: Implement actual Drizzle ORM update
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
async delete(id: string) {
|
||||||
|
const db = getDatabase();
|
||||||
|
logger.debug({ id }, "Deleting message");
|
||||||
|
|
||||||
|
// TODO: Implement actual Drizzle ORM delete
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
async getAttachmentsByChannel(channelId: string, query: MessageQuery) {
|
||||||
|
const db = getDatabase();
|
||||||
|
logger.debug({ channelId, query }, "Getting attachments by channel");
|
||||||
|
|
||||||
|
// TODO: Implement actual Drizzle ORM queries
|
||||||
|
return {
|
||||||
|
attachments: [],
|
||||||
|
total: 0,
|
||||||
|
hasMore: false,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export const messagesRepository = new MessagesRepository();
|
||||||
@@ -0,0 +1,35 @@
|
|||||||
|
import { z } from "zod";
|
||||||
|
|
||||||
|
export const messageQuerySchema = z.object({
|
||||||
|
channelId: z.string().optional(),
|
||||||
|
guildId: z.string().optional(),
|
||||||
|
userId: z.string().optional(),
|
||||||
|
status: z.enum(["pending", "clean", "warn", "flagged", "error"]).optional(),
|
||||||
|
limit: z.coerce.number().int().positive().default(50),
|
||||||
|
offset: z.coerce.number().int().nonnegative().default(0),
|
||||||
|
cursor: z.string().optional(),
|
||||||
|
});
|
||||||
|
|
||||||
|
export const messageCreateSchema = z.object({
|
||||||
|
guildId: z.string(),
|
||||||
|
channelId: z.string(),
|
||||||
|
threadId: z.string().optional(),
|
||||||
|
userId: z.string(),
|
||||||
|
username: z.string(),
|
||||||
|
avatarUrl: z.string().optional(),
|
||||||
|
content: z.string(),
|
||||||
|
type: z.enum(["text", "edited", "deleted"]).default("text"),
|
||||||
|
});
|
||||||
|
|
||||||
|
export const messageUpdateSchema = z.object({
|
||||||
|
editedContent: z.string().optional(),
|
||||||
|
aiStatus: z.enum(["pending", "clean", "warn", "flagged", "error"]).optional(),
|
||||||
|
aiAnalysis: z.string().optional(),
|
||||||
|
aiCategories: z.string().optional(),
|
||||||
|
aiSeverity: z.enum(["none", "low", "medium", "high", "critical"]).optional(),
|
||||||
|
aiConfidence: z.number().optional(),
|
||||||
|
});
|
||||||
|
|
||||||
|
export type MessageQuery = z.infer<typeof messageQuerySchema>;
|
||||||
|
export type MessageCreate = z.infer<typeof messageCreateSchema>;
|
||||||
|
export type MessageUpdate = z.infer<typeof messageUpdateSchema>;
|
||||||
@@ -0,0 +1,50 @@
|
|||||||
|
import { NotFoundError, ValidationError } from "../../shared/errors/index.js";
|
||||||
|
import { createChildLogger } from "../../shared/logger/index.js";
|
||||||
|
import { messagesRepository } from "./messages.repository.js";
|
||||||
|
import type { MessageQuery } from "./messages.schema.js";
|
||||||
|
|
||||||
|
const logger = createChildLogger("messages.service");
|
||||||
|
|
||||||
|
export class MessagesService {
|
||||||
|
async listMessages(query: MessageQuery) {
|
||||||
|
if (!query.channelId && !query.guildId) {
|
||||||
|
throw new ValidationError("Either channelId or guildId is required");
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.debug({ query }, "Listing messages");
|
||||||
|
return messagesRepository.findMany(query);
|
||||||
|
}
|
||||||
|
|
||||||
|
async getMessagesByChannel(channelId: string, query: MessageQuery) {
|
||||||
|
if (!channelId) {
|
||||||
|
throw new ValidationError("channelId is required");
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.debug({ channelId, query }, "Getting messages by channel");
|
||||||
|
return messagesRepository.findByChannel(channelId, query);
|
||||||
|
}
|
||||||
|
|
||||||
|
async getMessageById(id: string) {
|
||||||
|
if (!id) {
|
||||||
|
throw new ValidationError("message ID is required");
|
||||||
|
}
|
||||||
|
|
||||||
|
const message = await messagesRepository.findById(id);
|
||||||
|
if (!message) {
|
||||||
|
throw new NotFoundError(`Message with ID ${id} not found`);
|
||||||
|
}
|
||||||
|
|
||||||
|
return message;
|
||||||
|
}
|
||||||
|
|
||||||
|
async getAttachmentsByChannel(channelId: string, query: MessageQuery) {
|
||||||
|
if (!channelId) {
|
||||||
|
throw new ValidationError("channelId is required");
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.debug({ channelId, query }, "Getting attachments by channel");
|
||||||
|
return messagesRepository.getAttachmentsByChannel(channelId, query);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export const messagesService = new MessagesService();
|
||||||
@@ -0,0 +1,26 @@
|
|||||||
|
import type { Router } from "express";
|
||||||
|
import express from "express";
|
||||||
|
import {
|
||||||
|
handleGetAttachmentsByChannel,
|
||||||
|
handleGetMessageById,
|
||||||
|
handleGetMessagesByChannel,
|
||||||
|
handleListMessages,
|
||||||
|
} from "../messages.controller.js";
|
||||||
|
|
||||||
|
export function createMessagesRouter(): Router {
|
||||||
|
const router = express.Router();
|
||||||
|
|
||||||
|
// GET /api/messages - List messages
|
||||||
|
router.get("/messages", handleListMessages);
|
||||||
|
|
||||||
|
// GET /api/messages/:channelId - Get messages by channel
|
||||||
|
router.get("/messages/:channelId", handleGetMessagesByChannel);
|
||||||
|
|
||||||
|
// GET /api/messages/:channelId/attachments - Get attachments by channel
|
||||||
|
router.get("/messages/:channelId/attachments", handleGetAttachmentsByChannel);
|
||||||
|
|
||||||
|
// GET /api/messages/:id - Get single message by ID
|
||||||
|
router.get("/messages/:id", handleGetMessageById);
|
||||||
|
|
||||||
|
return router;
|
||||||
|
}
|
||||||
@@ -0,0 +1,14 @@
|
|||||||
|
import type { Router } from "express";
|
||||||
|
import express from "express";
|
||||||
|
|
||||||
|
export function createVoiceRouter(): Router {
|
||||||
|
const router = express.Router();
|
||||||
|
|
||||||
|
// TODO: Implement voice routes
|
||||||
|
// GET /api/voice/recordings
|
||||||
|
// GET /api/voice/recordings/:userId
|
||||||
|
// POST /api/voice/connect
|
||||||
|
// POST /api/voice/disconnect
|
||||||
|
|
||||||
|
return router;
|
||||||
|
}
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
import { createChildLogger } from "../../shared/logger/index.js";
|
||||||
|
|
||||||
|
const logger = createChildLogger("voice.service");
|
||||||
|
|
||||||
|
export class VoiceService {
|
||||||
|
// TODO: Implement voice service methods
|
||||||
|
}
|
||||||
|
|
||||||
|
export const voiceService = new VoiceService();
|
||||||
@@ -0,0 +1,92 @@
|
|||||||
|
import "dotenv/config";
|
||||||
|
import { z } from "zod";
|
||||||
|
|
||||||
|
const configSchema = z
|
||||||
|
.object({
|
||||||
|
// Server
|
||||||
|
WEBSERVER_PORT: z.coerce.number().positive().default(3001),
|
||||||
|
NODE_ENV: z
|
||||||
|
.enum(["development", "production", "test"])
|
||||||
|
.default("development"),
|
||||||
|
LOG_LEVEL: z
|
||||||
|
.enum(["error", "warn", "info", "http", "verbose", "debug", "silly"])
|
||||||
|
.default("info"),
|
||||||
|
VERBOSE: z
|
||||||
|
.string()
|
||||||
|
.optional()
|
||||||
|
.transform((v) => v === "true")
|
||||||
|
.default(false),
|
||||||
|
|
||||||
|
// Database
|
||||||
|
DATABASE_URL: z.string().url().optional(),
|
||||||
|
DATABASE_HOST: z.string().default("localhost"),
|
||||||
|
DATABASE_PORT: z.coerce.number().default(5432),
|
||||||
|
DATABASE_NAME: z.string().default("discord_moderation"),
|
||||||
|
DATABASE_USER: z.string().default("postgres"),
|
||||||
|
DATABASE_PASSWORD: z.string().optional(),
|
||||||
|
|
||||||
|
// Redis (optional, for pub/sub)
|
||||||
|
REDIS_URL: z.string().url().optional(),
|
||||||
|
REDIS_HOST: z.string().default("localhost"),
|
||||||
|
REDIS_PORT: z.coerce.number().default(6379),
|
||||||
|
|
||||||
|
// Discord
|
||||||
|
MONITOR_GUILD_ID: z.string().min(1).optional(),
|
||||||
|
|
||||||
|
// Admin
|
||||||
|
ADMIN_PASSWORD: z.string().optional(),
|
||||||
|
|
||||||
|
// Analytics
|
||||||
|
BACKLOG_SYNC_HOURS: z.coerce.number().positive().default(24),
|
||||||
|
BACKLOG_SYNC_BATCH_SIZE: z.coerce
|
||||||
|
.number()
|
||||||
|
.int()
|
||||||
|
.positive()
|
||||||
|
.max(100)
|
||||||
|
.default(100),
|
||||||
|
|
||||||
|
// AI Moderation
|
||||||
|
AI_ANALYSIS_ENABLED: z
|
||||||
|
.string()
|
||||||
|
.optional()
|
||||||
|
.transform((v) => v === "true")
|
||||||
|
.default(false),
|
||||||
|
OPENAI_MODERATION_API_KEY: z.string().optional(),
|
||||||
|
OPENAI_MODERATION_BASE_URL: z
|
||||||
|
.string()
|
||||||
|
.url()
|
||||||
|
.default("https://api.openai.com/v1"),
|
||||||
|
OPENAI_MODERATION_MODEL: z.string().default("omni-moderation-latest"),
|
||||||
|
AI_LLM_API_KEY: z.string().optional(),
|
||||||
|
AI_LLM_BASE_URL: z
|
||||||
|
.string()
|
||||||
|
.url()
|
||||||
|
.default("https://9router.asepharyana.my.id/v1"),
|
||||||
|
AI_LLM_MODEL: z.string().default("text"),
|
||||||
|
AI_LLM_VISION_MODEL: z.string().optional(),
|
||||||
|
AI_LLM_MAX_CONCURRENT: z.coerce.number().int().positive().default(5),
|
||||||
|
AI_LLM_IMAGE_MAX_DIMENSION: z.coerce
|
||||||
|
.number()
|
||||||
|
.int()
|
||||||
|
.positive()
|
||||||
|
.default(1024),
|
||||||
|
AI_LLM_TEXT_BATCH_SIZE: z.coerce.number().int().positive().default(20),
|
||||||
|
AI_LLM_MEDIA_ANALYSIS_TIMEOUT_MS: z.coerce
|
||||||
|
.number()
|
||||||
|
.int()
|
||||||
|
.positive()
|
||||||
|
.default(60000),
|
||||||
|
|
||||||
|
// Attachments
|
||||||
|
ATTACHMENT_UPLOAD_TIMEOUT_MS: z.coerce.number().positive().default(30000),
|
||||||
|
ATTACHMENT_MAX_SIZE_MB: z.coerce.number().positive().default(100),
|
||||||
|
ATTACHMENT_RETRY_ATTEMPTS: z.coerce.number().positive().default(3),
|
||||||
|
TELE_UPLOAD_URL: z
|
||||||
|
.string()
|
||||||
|
.url()
|
||||||
|
.default("https://upload.asepharyana.tech/api/upload"),
|
||||||
|
})
|
||||||
|
.parse(process.env);
|
||||||
|
|
||||||
|
export const config = configSchema;
|
||||||
|
export type Config = typeof config;
|
||||||
@@ -0,0 +1,58 @@
|
|||||||
|
import { drizzle } from "drizzle-orm/node-postgres";
|
||||||
|
import { Pool } from "pg";
|
||||||
|
import { config } from "../config/index.js";
|
||||||
|
import { createChildLogger } from "../logger/index.js";
|
||||||
|
|
||||||
|
const logger = createChildLogger("database");
|
||||||
|
|
||||||
|
let pool: Pool | null = null;
|
||||||
|
let db: ReturnType<typeof drizzle> | null = null;
|
||||||
|
|
||||||
|
export async function initializeDatabase() {
|
||||||
|
if (db) {
|
||||||
|
logger.warn("Database already initialized");
|
||||||
|
return db;
|
||||||
|
}
|
||||||
|
|
||||||
|
const databaseUrl =
|
||||||
|
config.DATABASE_URL ||
|
||||||
|
`postgresql://${config.DATABASE_USER}${config.DATABASE_PASSWORD ? `:${config.DATABASE_PASSWORD}` : ""}@${config.DATABASE_HOST}:${config.DATABASE_PORT}/${config.DATABASE_NAME}`;
|
||||||
|
|
||||||
|
pool = new Pool({
|
||||||
|
connectionString: databaseUrl,
|
||||||
|
});
|
||||||
|
|
||||||
|
pool.on("error", (err) => {
|
||||||
|
logger.error({ err }, "Unexpected error on idle client");
|
||||||
|
});
|
||||||
|
|
||||||
|
try {
|
||||||
|
const client = await pool.connect();
|
||||||
|
client.release();
|
||||||
|
logger.info("Database connection successful");
|
||||||
|
} catch (err) {
|
||||||
|
logger.error({ err }, "Failed to connect to database");
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
|
|
||||||
|
db = drizzle(pool);
|
||||||
|
return db;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getDatabase() {
|
||||||
|
if (!db) {
|
||||||
|
throw new Error(
|
||||||
|
"Database not initialized. Call initializeDatabase() first.",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return db;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function closeDatabase() {
|
||||||
|
if (pool) {
|
||||||
|
await pool.end();
|
||||||
|
pool = null;
|
||||||
|
db = null;
|
||||||
|
logger.info("Database connection closed");
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,65 @@
|
|||||||
|
export class AppError extends Error {
|
||||||
|
constructor(
|
||||||
|
message: string,
|
||||||
|
public code: string,
|
||||||
|
public statusCode: number = 500,
|
||||||
|
) {
|
||||||
|
super(message);
|
||||||
|
this.name = "AppError";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export class ValidationError extends AppError {
|
||||||
|
constructor(
|
||||||
|
message: string,
|
||||||
|
public details?: Record<string, unknown>,
|
||||||
|
) {
|
||||||
|
super(message, "VALIDATION_ERROR", 400);
|
||||||
|
this.name = "ValidationError";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export class NotFoundError extends AppError {
|
||||||
|
constructor(message: string) {
|
||||||
|
super(message, "NOT_FOUND", 404);
|
||||||
|
this.name = "NotFoundError";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export class UnauthorizedError extends AppError {
|
||||||
|
constructor(message: string = "Unauthorized") {
|
||||||
|
super(message, "UNAUTHORIZED", 401);
|
||||||
|
this.name = "UnauthorizedError";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export class ForbiddenError extends AppError {
|
||||||
|
constructor(message: string = "Forbidden") {
|
||||||
|
super(message, "FORBIDDEN", 403);
|
||||||
|
this.name = "ForbiddenError";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export class ConflictError extends AppError {
|
||||||
|
constructor(message: string) {
|
||||||
|
super(message, "CONFLICT", 409);
|
||||||
|
this.name = "ConflictError";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export class DatabaseError extends AppError {
|
||||||
|
constructor(
|
||||||
|
message: string,
|
||||||
|
public originalError?: Error,
|
||||||
|
) {
|
||||||
|
super(message, "DATABASE_ERROR", 500);
|
||||||
|
this.name = "DatabaseError";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export class ConfigError extends AppError {
|
||||||
|
constructor(message: string) {
|
||||||
|
super(message, "CONFIG_ERROR", 500);
|
||||||
|
this.name = "ConfigError";
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,24 @@
|
|||||||
|
import pino from "pino";
|
||||||
|
import { config } from "../config/index.js";
|
||||||
|
|
||||||
|
const isDev = config.NODE_ENV === "development";
|
||||||
|
|
||||||
|
export const logger = pino({
|
||||||
|
level: config.LOG_LEVEL,
|
||||||
|
transport: isDev
|
||||||
|
? {
|
||||||
|
target: "pino-pretty",
|
||||||
|
options: {
|
||||||
|
colorize: true,
|
||||||
|
translateTime: "SYS:standard",
|
||||||
|
ignore: "pid,hostname",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
: undefined,
|
||||||
|
});
|
||||||
|
|
||||||
|
export function createChildLogger(context: string) {
|
||||||
|
return logger.child({ context });
|
||||||
|
}
|
||||||
|
|
||||||
|
export type Logger = ReturnType<typeof createChildLogger>;
|
||||||
@@ -0,0 +1,50 @@
|
|||||||
|
import type { NextFunction, Request, Response } from "express";
|
||||||
|
import { AppError, UnauthorizedError } from "../errors/index.js";
|
||||||
|
import { createChildLogger } from "../logger/index.js";
|
||||||
|
|
||||||
|
const logger = createChildLogger("middleware");
|
||||||
|
|
||||||
|
export function errorHandler(
|
||||||
|
err: Error,
|
||||||
|
_req: Request,
|
||||||
|
res: Response,
|
||||||
|
_next: NextFunction,
|
||||||
|
) {
|
||||||
|
if (err instanceof AppError) {
|
||||||
|
logger.warn({ code: err.code, statusCode: err.statusCode }, err.message);
|
||||||
|
return res.status(err.statusCode).json({
|
||||||
|
error: err.code,
|
||||||
|
message: err.message,
|
||||||
|
...(err instanceof ValidationError && { details: err.details }),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.error({ err }, "Unhandled error");
|
||||||
|
res.status(500).json({
|
||||||
|
error: "INTERNAL_SERVER_ERROR",
|
||||||
|
message: "An unexpected error occurred",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export function adminAuth(adminPassword: string) {
|
||||||
|
return (req: Request, res: Response, next: NextFunction) => {
|
||||||
|
const password = req.headers["x-admin-password"] as string;
|
||||||
|
|
||||||
|
if (!password || password !== adminPassword) {
|
||||||
|
throw new UnauthorizedError("Invalid admin password");
|
||||||
|
}
|
||||||
|
|
||||||
|
next();
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function asyncHandler(
|
||||||
|
fn: (req: Request, res: Response, next: NextFunction) => Promise<void>,
|
||||||
|
) {
|
||||||
|
return (req: Request, res: Response, next: NextFunction) => {
|
||||||
|
Promise.resolve(fn(req, res, next)).catch(next);
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// Import ValidationError for type checking
|
||||||
|
import { ValidationError } from "../errors/index.js";
|
||||||
@@ -0,0 +1,25 @@
|
|||||||
|
{
|
||||||
|
"compilerOptions": {
|
||||||
|
"target": "ES2020",
|
||||||
|
"module": "ES2020",
|
||||||
|
"lib": ["ES2020"],
|
||||||
|
"moduleResolution": "node",
|
||||||
|
"esModuleInterop": true,
|
||||||
|
"allowSyntheticDefaultImports": true,
|
||||||
|
"strict": true,
|
||||||
|
"skipLibCheck": true,
|
||||||
|
"forceConsistentCasingInFileNames": true,
|
||||||
|
"resolveJsonModule": true,
|
||||||
|
"declaration": true,
|
||||||
|
"declarationMap": true,
|
||||||
|
"sourceMap": true,
|
||||||
|
"outDir": "./dist",
|
||||||
|
"rootDir": "./src",
|
||||||
|
"baseUrl": ".",
|
||||||
|
"paths": {
|
||||||
|
"@/*": ["./src/*"]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"include": ["src/**/*"],
|
||||||
|
"exclude": ["node_modules", "dist", "**/*.test.ts"]
|
||||||
|
}
|
||||||
@@ -0,0 +1,172 @@
|
|||||||
|
services/discord-gateway/
|
||||||
|
├── src/
|
||||||
|
│ ├── app/
|
||||||
|
│ │ ├── bootstrap.ts # Discord Gateway initialization (no HTTP server)
|
||||||
|
│ │ └── shutdown.ts # Graceful shutdown handler
|
||||||
|
│ ├── shared/
|
||||||
|
│ │ ├── config/
|
||||||
|
│ │ │ └── config.ts # Environment configuration (Zod validated)
|
||||||
|
│ │ ├── database/
|
||||||
|
│ │ │ ├── schema.ts # Drizzle ORM schema
|
||||||
|
│ │ │ ├── drizzle.ts # Database connection
|
||||||
|
│ │ │ ├── migrate.ts # Migration runner
|
||||||
|
│ │ │ └── voiceRecordingRepo.ts
|
||||||
|
│ │ ├── errors/
|
||||||
|
│ │ │ └── errors.ts # Custom error classes
|
||||||
|
│ │ ├── logger/
|
||||||
|
│ │ │ ├── logger.ts # Winston logger wrapper
|
||||||
|
│ │ │ └── serialization.ts # Log value serialization
|
||||||
|
│ │ ├── utils/
|
||||||
|
│ │ │ └── retry.ts # Retry with backoff utility
|
||||||
|
│ │ └── discord/
|
||||||
|
│ │ └── clientOptions.ts # Discord.js client configuration
|
||||||
|
│ ├── modules/
|
||||||
|
│ │ ├── message-capture/ # Modular MVC: Message capture & storage
|
||||||
|
│ │ │ ├── messageCapture.ts # Controller: Discord event listeners
|
||||||
|
│ │ │ ├── messageStore.ts # Repository: Database operations
|
||||||
|
│ │ │ ├── messageMetadata.ts # Service: Message metadata extraction
|
||||||
|
│ │ │ ├── types.ts # Domain types
|
||||||
|
│ │ │ └── index.ts # Module exports
|
||||||
|
│ │ ├── ai-moderation/ # Modular MVC: AI analysis & moderation
|
||||||
|
│ │ │ ├── aiAnalyzer.ts # Controller: Analysis orchestration
|
||||||
|
│ │ │ ├── llmModerationClient.ts # Service: LLM API client
|
||||||
|
│ │ │ ├── aiAnalysisWorker.ts # Service: Worker pool management
|
||||||
|
│ │ │ ├── indonesianTextNormalizer.ts # Service: Text normalization
|
||||||
|
│ │ │ ├── moderationPrompt.ts # Service: Prompt generation
|
||||||
|
│ │ │ └── index.ts # Module exports
|
||||||
|
│ │ ├── voice-recording/ # Modular MVC: Voice recording & streaming
|
||||||
|
│ │ │ ├── voiceController.ts # Controller: Voice connection management
|
||||||
|
│ │ │ ├── recorder.ts # Service: Recording orchestration
|
||||||
|
│ │ │ ├── recorder/
|
||||||
|
│ │ │ │ ├── audioStream.ts # Service: Audio stream subscription
|
||||||
|
│ │ │ │ ├── decoder.ts # Service: Opus decoding
|
||||||
|
│ │ │ │ ├── segment.ts # Service: OGG segment rotation
|
||||||
|
│ │ │ │ ├── metadata.ts # Service: Segment metadata
|
||||||
|
│ │ │ │ ├── sessionRecording.ts # Service: Session management
|
||||||
|
│ │ │ │ └── uploader.ts # Service: Segment upload
|
||||||
|
│ │ │ └── index.ts # Module exports
|
||||||
|
│ │ ├── attachment-upload/ # Modular MVC: Attachment handling
|
||||||
|
│ │ │ ├── attachmentUploader.ts # Service: Upload orchestration
|
||||||
|
│ │ │ ├── imageResizer.ts # Service: Image resizing
|
||||||
|
│ │ │ └── index.ts # Module exports
|
||||||
|
│ │ └── event-broadcaster/ # Event-driven: Redis pub/sub
|
||||||
|
│ │ ├── eventBroadcaster.ts # Service: Event publishing
|
||||||
|
│ │ ├── eventTypes.ts # Domain: Event type definitions
|
||||||
|
│ │ └── index.ts # Module exports
|
||||||
|
│ ├── mock-crc.ts # CRC polyfill for discord.js
|
||||||
|
│ └── index.ts # Service entry point
|
||||||
|
├── package.json # Service dependencies
|
||||||
|
└── tsconfig.json # TypeScript configuration
|
||||||
|
|
||||||
|
## Architecture Patterns
|
||||||
|
|
||||||
|
### Modular MVC Structure
|
||||||
|
Each module follows Controller-Service-Repository pattern:
|
||||||
|
- **Controller**: Discord event listeners (messageCapture, aiAnalyzer, voiceController)
|
||||||
|
- **Service**: Business logic (messageStore, llmModerationClient, recorder)
|
||||||
|
- **Repository**: Data access (messageStore, voiceRecordingRepo)
|
||||||
|
|
||||||
|
### Event-Driven Design
|
||||||
|
- **Redis Pub/Sub**: All events published to Redis channels
|
||||||
|
- **Event Channels**:
|
||||||
|
- `discord:message:created` — New message captured
|
||||||
|
- `discord:message:updated` — Message edited
|
||||||
|
- `discord:message:deleted` — Message deleted
|
||||||
|
- `discord:message:analyzed` — AI analysis complete
|
||||||
|
- `discord:attachment:created` — Attachment detected
|
||||||
|
- `discord:attachment:uploaded` — Attachment uploaded to storage
|
||||||
|
- `discord:voice:started` — Voice recording started
|
||||||
|
- `discord:voice:stopped` — Voice recording stopped
|
||||||
|
- `discord:voice:uploaded` — Voice segment uploaded
|
||||||
|
- `discord:analysis:queue_status` — Analysis queue status update
|
||||||
|
|
||||||
|
### Shared Infrastructure
|
||||||
|
- **Config**: Zod-validated environment variables
|
||||||
|
- **Logger**: Winston logger with context support
|
||||||
|
- **Database**: Drizzle ORM with PostgreSQL
|
||||||
|
- **Errors**: Custom error classes with codes and status codes
|
||||||
|
- **Utils**: Retry logic with exponential backoff
|
||||||
|
|
||||||
|
### No HTTP Server
|
||||||
|
- Discord Gateway service is **event-driven only**
|
||||||
|
- No Express, WebSocket, or HTTP routes
|
||||||
|
- All communication via Redis pub/sub
|
||||||
|
- Backend service consumes events and serves HTTP API
|
||||||
|
|
||||||
|
## Initialization Flow
|
||||||
|
|
||||||
|
1. Load environment config (Zod validation)
|
||||||
|
2. Initialize database connection
|
||||||
|
3. Run pending migrations
|
||||||
|
4. Create Discord client with optimized cache settings
|
||||||
|
5. Initialize Redis event broadcaster
|
||||||
|
6. Register Discord event listeners (messageCapture, aiAnalyzer)
|
||||||
|
7. Login to Discord
|
||||||
|
8. Listen for graceful shutdown signals (SIGINT, SIGTERM)
|
||||||
|
|
||||||
|
## Graceful Shutdown
|
||||||
|
|
||||||
|
On shutdown signal:
|
||||||
|
1. Close database connection
|
||||||
|
2. Disconnect from voice channels
|
||||||
|
3. Close Redis connection
|
||||||
|
4. Destroy Discord client
|
||||||
|
5. Exit process
|
||||||
|
|
||||||
|
## Dependencies
|
||||||
|
|
||||||
|
**Core Discord**:
|
||||||
|
- discord.js-selfbot-v13
|
||||||
|
- @discordjs/voice
|
||||||
|
- @discordjs/opus
|
||||||
|
|
||||||
|
**Audio Processing**:
|
||||||
|
- prism-media (Opus encoding/decoding)
|
||||||
|
- opusscript (Opus fallback)
|
||||||
|
- sharp (Image resizing)
|
||||||
|
|
||||||
|
**Data & Config**:
|
||||||
|
- drizzle-orm (ORM)
|
||||||
|
- pg (PostgreSQL driver)
|
||||||
|
- zod (Config validation)
|
||||||
|
- ioredis (Redis client)
|
||||||
|
|
||||||
|
**Logging & Utilities**:
|
||||||
|
- winston (Structured logging)
|
||||||
|
- p-retry (Retry logic)
|
||||||
|
- p-limit (Concurrency limiting)
|
||||||
|
- piscina (Worker pool)
|
||||||
|
|
||||||
|
## Event Flow Example
|
||||||
|
|
||||||
|
### Message Capture Flow
|
||||||
|
1. Discord emits `messageCreate` event
|
||||||
|
2. `messageCapture.ts` listener receives event
|
||||||
|
3. Extract metadata (user, channel, content, timestamp)
|
||||||
|
4. `messageStore.ts` inserts into database
|
||||||
|
5. `eventBroadcaster.messageCreated()` publishes to Redis
|
||||||
|
6. Backend service subscribes to `discord:message:created` channel
|
||||||
|
7. Backend processes and stores in its own database
|
||||||
|
|
||||||
|
### Voice Recording Flow
|
||||||
|
1. `voiceController.connect()` joins voice channel
|
||||||
|
2. `recorder.ts` subscribes to user audio streams
|
||||||
|
3. For each speaking user:
|
||||||
|
- Create audio stream subscription
|
||||||
|
- Decode Opus packets to PCM
|
||||||
|
- Rotate OGG segments (5s default)
|
||||||
|
- Collect user metadata
|
||||||
|
4. On silence (3s):
|
||||||
|
- Finalize segment
|
||||||
|
- Create metadata JSON
|
||||||
|
- Upload segment to storage
|
||||||
|
- Publish `discord:voice:uploaded` event
|
||||||
|
5. Backend service receives event and indexes recording
|
||||||
|
|
||||||
|
## No Breaking Changes
|
||||||
|
|
||||||
|
- Original `src/` remains untouched for now
|
||||||
|
- Discord Gateway is a **new service** in `services/discord-gateway/`
|
||||||
|
- Can run alongside existing monolith during transition
|
||||||
|
- Backend service will consume Redis events
|
||||||
|
- Frontend continues to use Backend HTTP API
|
||||||
@@ -0,0 +1,408 @@
|
|||||||
|
# Discord Gateway Service - Module Structure
|
||||||
|
|
||||||
|
## Complete Directory Tree
|
||||||
|
|
||||||
|
```
|
||||||
|
services/discord-gateway/
|
||||||
|
├── src/
|
||||||
|
│ ├── app/
|
||||||
|
│ │ ├── bootstrap.ts
|
||||||
|
│ │ │ └── Initializes Discord client, database, Redis broadcaster
|
||||||
|
│ │ │ Registers event listeners, handles graceful shutdown
|
||||||
|
│ │ └── shutdown.ts
|
||||||
|
│ │ └── Graceful shutdown handler for SIGINT/SIGTERM/exceptions
|
||||||
|
│ │
|
||||||
|
│ ├── shared/
|
||||||
|
│ │ ├── config/
|
||||||
|
│ │ │ └── config.ts
|
||||||
|
│ │ │ └── Zod-validated environment configuration
|
||||||
|
│ │ │ - Discord token, database URL, Redis URL
|
||||||
|
│ │ │ - AI LLM settings, recording parameters
|
||||||
|
│ │ │ - Attachment upload settings, retention policies
|
||||||
|
│ │ │
|
||||||
|
│ │ ├── database/
|
||||||
|
│ │ │ ├── schema.ts
|
||||||
|
│ │ │ │ └── Drizzle ORM schema definitions
|
||||||
|
│ │ │ ├── drizzle.ts
|
||||||
|
│ │ │ │ └── PostgreSQL connection and initialization
|
||||||
|
│ │ │ ├── migrate.ts
|
||||||
|
│ │ │ │ └── Database migration runner
|
||||||
|
│ │ │ ├── migrateCli.ts
|
||||||
|
│ │ │ │ └── CLI for programmatic migrations
|
||||||
|
│ │ │ ├── voiceRecordingRepo.ts
|
||||||
|
│ │ │ │ └── Voice recording repository
|
||||||
|
│ │ │ └── migrations/
|
||||||
|
│ │ │ └── Database migration files
|
||||||
|
│ │ │
|
||||||
|
│ │ ├── errors/
|
||||||
|
│ │ │ └── errors.ts
|
||||||
|
│ │ │ └── Custom error classes
|
||||||
|
│ │ │ - AppError (base)
|
||||||
|
│ │ │ - ConfigError
|
||||||
|
│ │ │ - AudioError
|
||||||
|
│ │ │ - VoiceConnectionError
|
||||||
|
│ │ │ - ValidationError
|
||||||
|
│ │ │
|
||||||
|
│ │ ├── logger/
|
||||||
|
│ │ │ ├── logger.ts
|
||||||
|
│ │ │ │ └── Winston logger wrapper with context support
|
||||||
|
│ │ │ └── serialization.ts
|
||||||
|
│ │ │ └── Log value serialization utilities
|
||||||
|
│ │ │
|
||||||
|
│ │ ├── utils/
|
||||||
|
│ │ │ └── retry.ts
|
||||||
|
│ │ │ └── Retry with exponential backoff utility
|
||||||
|
│ │ │
|
||||||
|
│ │ └── discord/
|
||||||
|
│ │ └── clientOptions.ts
|
||||||
|
│ │ └── Discord.js client configuration
|
||||||
|
│ │
|
||||||
|
│ ├── modules/
|
||||||
|
│ │ │
|
||||||
|
│ │ ├── message-capture/
|
||||||
|
│ │ │ ├── messageCapture.ts
|
||||||
|
│ │ │ │ └── CONTROLLER: Discord event listeners
|
||||||
|
│ │ │ │ - messageCreate, messageUpdate, messageDelete
|
||||||
|
│ │ │ │ - Validates capture target, publishes events
|
||||||
|
│ │ │ │
|
||||||
|
│ │ │ ├── messageStore.ts
|
||||||
|
│ │ │ │ └── REPOSITORY: Database CRUD operations
|
||||||
|
│ │ │ │ - upsertMessageForCapture
|
||||||
|
│ │ │ │ - updateMessageAsEdited
|
||||||
|
│ │ │ │ - updateMessageAsDeleted
|
||||||
|
│ │ │ │ - insertAttachment
|
||||||
|
│ │ │ │ - getMessageById
|
||||||
|
│ │ │ │
|
||||||
|
│ │ │ ├── messageMetadata.ts
|
||||||
|
│ │ │ │ └── SERVICE: Message metadata extraction
|
||||||
|
│ │ │ │ - getMessageMetadata
|
||||||
|
│ │ │ │ - getMessageLocation
|
||||||
|
│ │ │ │ - getDisplayContent
|
||||||
|
│ │ │ │
|
||||||
|
│ │ │ ├── types.ts
|
||||||
|
│ │ │ │ └── Domain types
|
||||||
|
│ │ │ │ - MessageRecord
|
||||||
|
│ │ │ │ - AttachmentRecord
|
||||||
|
│ │ │ │ - VoiceSegmentRecord
|
||||||
|
│ │ │ │ - AIStatus, AISeverity, AIRecommendedAction
|
||||||
|
│ │ │ │
|
||||||
|
│ │ │ └── index.ts
|
||||||
|
│ │ └── Module exports
|
||||||
|
│ │
|
||||||
|
│ │ ├── ai-moderation/
|
||||||
|
│ │ │ ├── aiAnalyzer.ts
|
||||||
|
│ │ │ │ └── CONTROLLER: Analysis orchestration
|
||||||
|
│ │ │ │ - startPendingAIAnalysisWorker
|
||||||
|
│ │ │ │ - queueMessageAnalysis
|
||||||
|
│ │ │ │ - Manages analysis queue and worker pool
|
||||||
|
│ │ │ │
|
||||||
|
│ │ │ ├── llmModerationClient.ts
|
||||||
|
│ │ │ │ └── SERVICE: LLM API integration
|
||||||
|
│ │ │ │ - Calls LLM for text/image moderation
|
||||||
|
│ │ │ │ - Parses responses, handles errors
|
||||||
|
│ │ │ │ - Retry logic with backoff
|
||||||
|
│ │ │ │
|
||||||
|
│ │ │ ├── aiAnalysisWorker.ts
|
||||||
|
│ │ │ │ └── SERVICE: Worker pool management
|
||||||
|
│ │ │ │ - Piscina worker pool for parallel analysis
|
||||||
|
│ │ │ │ - Conversation context batching
|
||||||
|
│ │ │ │
|
||||||
|
│ │ │ ├── indonesianTextNormalizer.ts
|
||||||
|
│ │ │ │ └── SERVICE: Text preprocessing
|
||||||
|
│ │ │ │ - Normalize Indonesian text
|
||||||
|
│ │ │ │ - Handle diacritics, abbreviations
|
||||||
|
│ │ │ │
|
||||||
|
│ │ │ ├── moderationPrompt.ts
|
||||||
|
│ │ │ │ └── SERVICE: Prompt generation
|
||||||
|
│ │ │ │ - Generate LLM prompts for moderation
|
||||||
|
│ │ │ │ - Include context and policy
|
||||||
|
│ │ │ │
|
||||||
|
│ │ │ └── index.ts
|
||||||
|
│ │ └── Module exports
|
||||||
|
│ │
|
||||||
|
│ │ ├── voice-recording/
|
||||||
|
│ │ │ ├── voiceController.ts
|
||||||
|
│ │ │ │ └── CONTROLLER: Voice connection management
|
||||||
|
│ │ │ │ - connect(guildId, channelId)
|
||||||
|
│ │ │ │ - disconnect()
|
||||||
|
│ │ │ │ - listGuilds(), listVoiceChannels()
|
||||||
|
│ │ │ │ - getStatus()
|
||||||
|
│ │ │ │
|
||||||
|
│ │ │ ├── recorder.ts
|
||||||
|
│ │ │ │ └── SERVICE: Recording orchestration
|
||||||
|
│ │ │ │ - startRecording(client, channel)
|
||||||
|
│ │ │ │ - stopRecording(guildId)
|
||||||
|
│ │ │ │ - Manages active recording sessions
|
||||||
|
│ │ │ │
|
||||||
|
│ │ │ ├── recorder/
|
||||||
|
│ │ │ │ ├── audioStream.ts
|
||||||
|
│ │ │ │ │ └── SERVICE: Audio stream subscription
|
||||||
|
│ │ │ │ │ - subscribeToAudioStream
|
||||||
|
│ │ │ │ │ - Opus packet handling
|
||||||
|
│ │ │ │ │
|
||||||
|
│ │ │ │ ├── decoder.ts
|
||||||
|
│ │ │ │ │ └── SERVICE: Opus decoding
|
||||||
|
│ │ │ │ │ - OpusDecoder class
|
||||||
|
│ │ │ │ │ - Decode Opus to PCM
|
||||||
|
│ │ │ │ │ - Rotation and cooldown logic
|
||||||
|
│ │ │ │ │
|
||||||
|
│ │ │ │ ├── segment.ts
|
||||||
|
│ │ │ │ │ └── SERVICE: OGG segment rotation
|
||||||
|
│ │ │ │ │ - SegmentManager class
|
||||||
|
│ │ │ │ │ - Rotate segments (5s default)
|
||||||
|
│ │ │ │ │ - Write OGG files
|
||||||
|
│ │ │ │ │
|
||||||
|
│ │ │ │ ├── metadata.ts
|
||||||
|
│ │ │ │ │ └── SERVICE: Segment metadata
|
||||||
|
│ │ │ │ │ - collectUserMetadata
|
||||||
|
│ │ │ │ │ - createSegmentMetadata
|
||||||
|
│ │ │ │ │ - User info, roles, timestamps
|
||||||
|
│ │ │ │ │
|
||||||
|
│ │ │ │ ├── sessionRecording.ts
|
||||||
|
│ │ │ │ │ └── SERVICE: Session management
|
||||||
|
│ │ │ │ │ - createRecordingSession
|
||||||
|
│ │ │ │ │ - finalizeRecordingSession
|
||||||
|
│ │ │ │ │ - Track active sessions
|
||||||
|
│ │ │ │ │
|
||||||
|
│ │ │ │ └── uploader.ts
|
||||||
|
│ │ │ │ └── SERVICE: Segment upload
|
||||||
|
│ │ │ │ - uploadRecordingSegment
|
||||||
|
│ │ │ │ - Upload to external storage
|
||||||
|
│ │ │ │ - Retry logic
|
||||||
|
│ │ │ │
|
||||||
|
│ │ │ └── index.ts
|
||||||
|
│ │ └── Module exports
|
||||||
|
│ │
|
||||||
|
│ │ ├── attachment-upload/
|
||||||
|
│ │ │ ├── attachmentUploader.ts
|
||||||
|
│ │ │ │ └── SERVICE: Upload orchestration
|
||||||
|
│ │ │ │ - processAttachmentUpload
|
||||||
|
│ │ │ │ - Download from Discord
|
||||||
|
│ │ │ │ - Upload to external storage
|
||||||
|
│ │ │ │ - Retry with backoff
|
||||||
|
│ │ │ │
|
||||||
|
│ │ │ ├── imageResizer.ts
|
||||||
|
│ │ │ │ └── SERVICE: Image processing
|
||||||
|
│ │ │ │ - resizeImage
|
||||||
|
│ │ │ │ - Resize to max dimension
|
||||||
|
│ │ │ │ - Preserve aspect ratio
|
||||||
|
│ │ │ │
|
||||||
|
│ │ │ └── index.ts
|
||||||
|
│ │ └── Module exports
|
||||||
|
│ │
|
||||||
|
│ │ └── event-broadcaster/
|
||||||
|
│ │ ├── eventBroadcaster.ts
|
||||||
|
│ │ │ └── SERVICE: Redis pub/sub publisher
|
||||||
|
│ │ │ - EventBroadcaster class
|
||||||
|
│ │ │ - RedisEventPublisher class
|
||||||
|
│ │ │ - Publish to Redis channels
|
||||||
|
│ │ │ - Methods:
|
||||||
|
│ │ │ - messageCreated()
|
||||||
|
│ │ │ - messageUpdated()
|
||||||
|
│ │ │ - messageDeleted()
|
||||||
|
│ │ │ - messageAnalyzed()
|
||||||
|
│ │ │ - attachmentCreated()
|
||||||
|
│ │ │ - attachmentUploaded()
|
||||||
|
│ │ │ - voiceRecordingStarted()
|
||||||
|
│ │ │ - voiceRecordingStopped()
|
||||||
|
│ │ │ - voiceRecordingUploaded()
|
||||||
|
│ │ │ - analysisQueueStatus()
|
||||||
|
│ │ │
|
||||||
|
│ │ ├── eventTypes.ts
|
||||||
|
│ │ │ └── Domain types
|
||||||
|
│ │ │ - DiscordGatewayEvent interface
|
||||||
|
│ │ │ - EventChannels constants
|
||||||
|
│ │ │ - Event channel names
|
||||||
|
│ │ │
|
||||||
|
│ │ └── index.ts
|
||||||
|
│ └── Module exports
|
||||||
|
│
|
||||||
|
│ ├── mock-crc.ts
|
||||||
|
│ │ └── CRC polyfill for discord.js compatibility
|
||||||
|
│ │
|
||||||
|
│ └── index.ts
|
||||||
|
│ └── Service entry point
|
||||||
|
│ - Initialize Discord Gateway
|
||||||
|
│ - Handle startup errors
|
||||||
|
│
|
||||||
|
├── ARCHITECTURE.md
|
||||||
|
│ └── Detailed architecture documentation
|
||||||
|
│
|
||||||
|
├── README.md
|
||||||
|
│ └── Complete service documentation
|
||||||
|
│
|
||||||
|
├── MODULE_STRUCTURE.md
|
||||||
|
│ └── This file - module structure reference
|
||||||
|
│
|
||||||
|
└── package.json
|
||||||
|
└── Service dependencies and scripts
|
||||||
|
```
|
||||||
|
|
||||||
|
## Module Responsibilities
|
||||||
|
|
||||||
|
### message-capture
|
||||||
|
**Purpose**: Capture Discord messages (create, update, delete)
|
||||||
|
**Pattern**: Controller-Service-Repository
|
||||||
|
- **Controller** (messageCapture.ts): Listens to Discord events
|
||||||
|
- **Service** (messageMetadata.ts): Extracts metadata
|
||||||
|
- **Repository** (messageStore.ts): Database operations
|
||||||
|
- **Events Published**:
|
||||||
|
- `discord:message:created`
|
||||||
|
- `discord:message:updated`
|
||||||
|
- `discord:message:deleted`
|
||||||
|
|
||||||
|
### ai-moderation
|
||||||
|
**Purpose**: Analyze messages with LLM for moderation
|
||||||
|
**Pattern**: Controller-Service-Service-Service
|
||||||
|
- **Controller** (aiAnalyzer.ts): Orchestrates analysis workflow
|
||||||
|
- **Service** (llmModerationClient.ts): LLM API integration
|
||||||
|
- **Service** (aiAnalysisWorker.ts): Worker pool management
|
||||||
|
- **Service** (indonesianTextNormalizer.ts): Text preprocessing
|
||||||
|
- **Service** (moderationPrompt.ts): Prompt generation
|
||||||
|
- **Events Published**:
|
||||||
|
- `discord:message:analyzed`
|
||||||
|
- `discord:analysis:queue_status`
|
||||||
|
|
||||||
|
### voice-recording
|
||||||
|
**Purpose**: Record voice channel audio
|
||||||
|
**Pattern**: Controller-Service-SubServices
|
||||||
|
- **Controller** (voiceController.ts): Voice connection management
|
||||||
|
- **Service** (recorder.ts): Recording orchestration
|
||||||
|
- **Sub-services** (recorder/*): Audio processing pipeline
|
||||||
|
- audioStream.ts: Opus packet subscription
|
||||||
|
- decoder.ts: Opus to PCM decoding
|
||||||
|
- segment.ts: OGG file rotation
|
||||||
|
- metadata.ts: User metadata collection
|
||||||
|
- sessionRecording.ts: Session lifecycle
|
||||||
|
- uploader.ts: Segment upload
|
||||||
|
- **Events Published**:
|
||||||
|
- `discord:voice:started`
|
||||||
|
- `discord:voice:stopped`
|
||||||
|
- `discord:voice:uploaded`
|
||||||
|
|
||||||
|
### attachment-upload
|
||||||
|
**Purpose**: Upload message attachments to external storage
|
||||||
|
**Pattern**: Service-Service
|
||||||
|
- **Service** (attachmentUploader.ts): Upload orchestration
|
||||||
|
- **Service** (imageResizer.ts): Image processing
|
||||||
|
- **Events Published**:
|
||||||
|
- `discord:attachment:created`
|
||||||
|
- `discord:attachment:uploaded`
|
||||||
|
|
||||||
|
### event-broadcaster
|
||||||
|
**Purpose**: Publish events to Redis pub/sub
|
||||||
|
**Pattern**: Service-Domain
|
||||||
|
- **Service** (eventBroadcaster.ts): Redis publisher
|
||||||
|
- **Domain** (eventTypes.ts): Event type definitions
|
||||||
|
- **Channels**:
|
||||||
|
- discord:message:* (message events)
|
||||||
|
- discord:attachment:* (attachment events)
|
||||||
|
- discord:voice:* (voice events)
|
||||||
|
- discord:analysis:* (analysis events)
|
||||||
|
|
||||||
|
## Shared Infrastructure
|
||||||
|
|
||||||
|
### config
|
||||||
|
- Zod-validated environment variables
|
||||||
|
- Type-safe configuration access
|
||||||
|
- Sensible defaults
|
||||||
|
|
||||||
|
### database
|
||||||
|
- Drizzle ORM schema
|
||||||
|
- PostgreSQL connection
|
||||||
|
- Migration management
|
||||||
|
- Voice recording repository
|
||||||
|
|
||||||
|
### logger
|
||||||
|
- Winston logger wrapper
|
||||||
|
- Context-aware logging
|
||||||
|
- Log serialization utilities
|
||||||
|
|
||||||
|
### errors
|
||||||
|
- Custom error classes
|
||||||
|
- Error codes and HTTP status codes
|
||||||
|
- Proper error hierarchy
|
||||||
|
|
||||||
|
### utils
|
||||||
|
- Retry with exponential backoff
|
||||||
|
- Configurable retry parameters
|
||||||
|
|
||||||
|
### discord
|
||||||
|
- Discord.js client configuration
|
||||||
|
- Cache optimization
|
||||||
|
- Partial handling
|
||||||
|
|
||||||
|
## Event Flow
|
||||||
|
|
||||||
|
```
|
||||||
|
Discord Events
|
||||||
|
↓
|
||||||
|
message-capture (Controller)
|
||||||
|
↓
|
||||||
|
messageStore (Repository) → PostgreSQL
|
||||||
|
↓
|
||||||
|
eventBroadcaster (Service)
|
||||||
|
↓
|
||||||
|
Redis Pub/Sub
|
||||||
|
↓
|
||||||
|
Backend Service (Subscriber)
|
||||||
|
↓
|
||||||
|
HTTP API / WebSocket
|
||||||
|
↓
|
||||||
|
Frontend Application
|
||||||
|
```
|
||||||
|
|
||||||
|
## No HTTP Server
|
||||||
|
|
||||||
|
- ✅ No Express
|
||||||
|
- ✅ No WebSocket server
|
||||||
|
- ✅ No HTTP routes
|
||||||
|
- ✅ No middleware
|
||||||
|
- ✅ Pure event-driven service
|
||||||
|
|
||||||
|
## Graceful Shutdown
|
||||||
|
|
||||||
|
1. Close PostgreSQL connection
|
||||||
|
2. Disconnect from voice channels
|
||||||
|
3. Close Redis connection
|
||||||
|
4. Destroy Discord client
|
||||||
|
5. Exit process
|
||||||
|
|
||||||
|
## Dependencies
|
||||||
|
|
||||||
|
**Discord**:
|
||||||
|
- discord.js-selfbot-v13
|
||||||
|
- @discordjs/voice
|
||||||
|
- @discordjs/opus
|
||||||
|
|
||||||
|
**Audio**:
|
||||||
|
- prism-media
|
||||||
|
- opusscript
|
||||||
|
- sharp
|
||||||
|
|
||||||
|
**Data**:
|
||||||
|
- drizzle-orm
|
||||||
|
- pg
|
||||||
|
- zod
|
||||||
|
- ioredis
|
||||||
|
|
||||||
|
**Logging**:
|
||||||
|
- winston
|
||||||
|
- p-retry
|
||||||
|
- p-limit
|
||||||
|
- piscina
|
||||||
|
|
||||||
|
## Summary
|
||||||
|
|
||||||
|
The Discord Gateway service is a **pure event-driven microservice** that:
|
||||||
|
- Captures Discord messages, voice, and attachments
|
||||||
|
- Performs AI moderation analysis
|
||||||
|
- Publishes events to Redis pub/sub
|
||||||
|
- Has no HTTP server or WebSocket
|
||||||
|
- Follows Modular MVC pattern
|
||||||
|
- Maintains clean module boundaries
|
||||||
|
- Provides type-safe configuration
|
||||||
|
- Includes structured logging
|
||||||
|
- Handles graceful shutdown
|
||||||
|
|
||||||
|
The service is designed to run alongside the Backend service, which consumes Redis events and serves the HTTP API to the Frontend.
|
||||||
@@ -0,0 +1,363 @@
|
|||||||
|
# Discord Gateway Service - Extraction Complete
|
||||||
|
|
||||||
|
## Overview
|
||||||
|
|
||||||
|
Successfully extracted Discord Gateway service with **Modular MVC + Event-Driven Architecture** using Redis pub/sub for inter-service communication.
|
||||||
|
|
||||||
|
## Directory Structure
|
||||||
|
|
||||||
|
```
|
||||||
|
services/discord-gateway/
|
||||||
|
├── src/
|
||||||
|
│ ├── app/
|
||||||
|
│ │ ├── bootstrap.ts # Service initialization (Discord client, DB, Redis)
|
||||||
|
│ │ └── shutdown.ts # Graceful shutdown handler
|
||||||
|
│ ├── shared/ # Shared infrastructure layer
|
||||||
|
│ │ ├── config/
|
||||||
|
│ │ │ └── config.ts # Zod-validated environment config
|
||||||
|
│ │ ├── database/
|
||||||
|
│ │ │ ├── schema.ts # Drizzle ORM schema
|
||||||
|
│ │ │ ├── drizzle.ts # PostgreSQL connection
|
||||||
|
│ │ │ ├── migrate.ts # Migration runner
|
||||||
|
│ │ │ └── voiceRecordingRepo.ts
|
||||||
|
│ │ ├── errors/
|
||||||
|
│ │ │ └── errors.ts # Custom error classes
|
||||||
|
│ │ ├── logger/
|
||||||
|
│ │ │ ├── logger.ts # Winston logger wrapper
|
||||||
|
│ │ │ └── serialization.ts # Log serialization
|
||||||
|
│ │ ├── utils/
|
||||||
|
│ │ │ └── retry.ts # Retry with exponential backoff
|
||||||
|
│ │ └── discord/
|
||||||
|
│ │ └── clientOptions.ts # Discord.js client config
|
||||||
|
│ ├── modules/ # Feature modules (Modular MVC)
|
||||||
|
│ │ ├── message-capture/ # Controller-Service-Repository
|
||||||
|
│ │ │ ├── messageCapture.ts # Controller: Discord event listeners
|
||||||
|
│ │ │ ├── messageStore.ts # Repository: DB operations
|
||||||
|
│ │ │ ├── messageMetadata.ts # Service: Metadata extraction
|
||||||
|
│ │ │ ├── types.ts # Domain types
|
||||||
|
│ │ │ └── index.ts # Module exports
|
||||||
|
│ │ ├── ai-moderation/ # Controller-Service-Repository
|
||||||
|
│ │ │ ├── aiAnalyzer.ts # Controller: Analysis orchestration
|
||||||
|
│ │ │ ├── llmModerationClient.ts # Service: LLM API client
|
||||||
|
│ │ │ ├── aiAnalysisWorker.ts # Service: Worker pool
|
||||||
|
│ │ │ ├── indonesianTextNormalizer.ts # Service: Text normalization
|
||||||
|
│ │ │ ├── moderationPrompt.ts # Service: Prompt generation
|
||||||
|
│ │ │ └── index.ts # Module exports
|
||||||
|
│ │ ├── voice-recording/ # Controller-Service-Repository
|
||||||
|
│ │ │ ├── voiceController.ts # Controller: Voice connection mgmt
|
||||||
|
│ │ │ ├── recorder.ts # Service: Recording orchestration
|
||||||
|
│ │ │ ├── recorder/ # Sub-services
|
||||||
|
│ │ │ │ ├── audioStream.ts # Audio stream subscription
|
||||||
|
│ │ │ │ ├── decoder.ts # Opus decoding
|
||||||
|
│ │ │ │ ├── segment.ts # OGG segment rotation
|
||||||
|
│ │ │ │ ├── metadata.ts # Segment metadata
|
||||||
|
│ │ │ │ ├── sessionRecording.ts # Session management
|
||||||
|
│ │ │ │ └── uploader.ts # Segment upload
|
||||||
|
│ │ │ └── index.ts # Module exports
|
||||||
|
│ │ ├── attachment-upload/ # Controller-Service-Repository
|
||||||
|
│ │ │ ├── attachmentUploader.ts # Service: Upload orchestration
|
||||||
|
│ │ │ ├── imageResizer.ts # Service: Image resizing
|
||||||
|
│ │ │ └── index.ts # Module exports
|
||||||
|
│ │ └── event-broadcaster/ # Event-driven layer
|
||||||
|
│ │ ├── eventBroadcaster.ts # Service: Redis pub/sub publisher
|
||||||
|
│ │ ├── eventTypes.ts # Domain: Event type definitions
|
||||||
|
│ │ └── index.ts # Module exports
|
||||||
|
│ ├── mock-crc.ts # CRC polyfill for discord.js
|
||||||
|
│ └── index.ts # Service entry point
|
||||||
|
├── ARCHITECTURE.md # Detailed architecture documentation
|
||||||
|
├── package.json # Service dependencies
|
||||||
|
└── tsconfig.json # TypeScript configuration (inherited)
|
||||||
|
```
|
||||||
|
|
||||||
|
## Architecture Patterns
|
||||||
|
|
||||||
|
### 1. Modular MVC Structure
|
||||||
|
Each feature module follows **Controller-Service-Repository** pattern:
|
||||||
|
|
||||||
|
**Message Capture Module**:
|
||||||
|
- **Controller** (`messageCapture.ts`): Listens to Discord events (messageCreate, messageUpdate, messageDelete)
|
||||||
|
- **Service** (`messageMetadata.ts`): Extracts and normalizes message metadata
|
||||||
|
- **Repository** (`messageStore.ts`): Database CRUD operations
|
||||||
|
|
||||||
|
**AI Moderation Module**:
|
||||||
|
- **Controller** (`aiAnalyzer.ts`): Orchestrates analysis workflow
|
||||||
|
- **Service** (`llmModerationClient.ts`): LLM API integration
|
||||||
|
- **Service** (`aiAnalysisWorker.ts`): Worker pool management
|
||||||
|
- **Service** (`indonesianTextNormalizer.ts`): Text preprocessing
|
||||||
|
|
||||||
|
**Voice Recording Module**:
|
||||||
|
- **Controller** (`voiceController.ts`): Voice channel connection management
|
||||||
|
- **Service** (`recorder.ts`): Recording orchestration
|
||||||
|
- **Sub-services** (`recorder/*`): Audio stream, decoding, segmentation, upload
|
||||||
|
|
||||||
|
**Attachment Upload Module**:
|
||||||
|
- **Service** (`attachmentUploader.ts`): Upload orchestration
|
||||||
|
- **Service** (`imageResizer.ts`): Image processing
|
||||||
|
|
||||||
|
### 2. Event-Driven Architecture
|
||||||
|
**Redis Pub/Sub** replaces WebSocket broadcaster:
|
||||||
|
|
||||||
|
```
|
||||||
|
Discord Events → Discord Gateway Service → Redis Pub/Sub → Backend Service
|
||||||
|
↓
|
||||||
|
Event Channels:
|
||||||
|
- discord:message:created
|
||||||
|
- discord:message:updated
|
||||||
|
- discord:message:deleted
|
||||||
|
- discord:message:analyzed
|
||||||
|
- discord:attachment:created
|
||||||
|
- discord:attachment:uploaded
|
||||||
|
- discord:voice:started
|
||||||
|
- discord:voice:stopped
|
||||||
|
- discord:voice:uploaded
|
||||||
|
- discord:analysis:queue_status
|
||||||
|
```
|
||||||
|
|
||||||
|
### 3. Shared Infrastructure Layer
|
||||||
|
Centralized, reusable components:
|
||||||
|
- **Config**: Zod-validated environment variables
|
||||||
|
- **Logger**: Winston logger with context support
|
||||||
|
- **Database**: Drizzle ORM with PostgreSQL
|
||||||
|
- **Errors**: Custom error classes with codes and HTTP status codes
|
||||||
|
- **Utils**: Retry logic with exponential backoff
|
||||||
|
- **Discord**: Client configuration and options
|
||||||
|
|
||||||
|
### 4. No HTTP Server
|
||||||
|
- **Event-driven only**: No Express, WebSocket, or HTTP routes
|
||||||
|
- **Redis pub/sub**: All inter-service communication via Redis
|
||||||
|
- **Backend service**: Consumes events and serves HTTP API
|
||||||
|
- **Frontend**: Continues to use Backend HTTP API
|
||||||
|
|
||||||
|
## Key Features
|
||||||
|
|
||||||
|
### Message Capture
|
||||||
|
1. Discord emits `messageCreate`, `messageUpdate`, `messageDelete` events
|
||||||
|
2. `messageCapture.ts` listener receives and validates event
|
||||||
|
3. Extract metadata: user, channel, content, timestamp, attachments
|
||||||
|
4. `messageStore.ts` inserts into PostgreSQL
|
||||||
|
5. `eventBroadcaster.messageCreated()` publishes to Redis
|
||||||
|
6. Backend service subscribes and processes
|
||||||
|
|
||||||
|
### AI Moderation
|
||||||
|
1. `aiAnalyzer.ts` queues messages for analysis
|
||||||
|
2. `llmModerationClient.ts` calls LLM API with context
|
||||||
|
3. `indonesianTextNormalizer.ts` preprocesses text
|
||||||
|
4. Results stored in database
|
||||||
|
5. `eventBroadcaster.messageAnalyzed()` publishes results
|
||||||
|
6. Backend service receives and updates UI
|
||||||
|
|
||||||
|
### Voice Recording
|
||||||
|
1. `voiceController.connect()` joins voice channel
|
||||||
|
2. `recorder.ts` subscribes to user audio streams
|
||||||
|
3. For each speaking user:
|
||||||
|
- `audioStream.ts` subscribes to Opus packets
|
||||||
|
- `decoder.ts` decodes Opus to PCM
|
||||||
|
- `segment.ts` rotates OGG files (5s default)
|
||||||
|
- `metadata.ts` collects user info
|
||||||
|
4. On silence (3s):
|
||||||
|
- `sessionRecording.ts` finalizes segment
|
||||||
|
- `uploader.ts` uploads to storage
|
||||||
|
- `eventBroadcaster.voiceRecordingUploaded()` publishes
|
||||||
|
5. Backend service indexes recording
|
||||||
|
|
||||||
|
### Attachment Upload
|
||||||
|
1. `messageCapture.ts` detects attachments
|
||||||
|
2. `attachmentUploader.ts` downloads from Discord
|
||||||
|
3. `imageResizer.ts` resizes images if needed
|
||||||
|
4. Upload to external storage with retry logic
|
||||||
|
5. `eventBroadcaster.attachmentUploaded()` publishes
|
||||||
|
6. Backend service stores metadata
|
||||||
|
|
||||||
|
## Initialization Flow
|
||||||
|
|
||||||
|
```
|
||||||
|
1. Load environment config (Zod validation)
|
||||||
|
↓
|
||||||
|
2. Initialize PostgreSQL connection
|
||||||
|
↓
|
||||||
|
3. Run pending database migrations
|
||||||
|
↓
|
||||||
|
4. Create Discord client with optimized cache
|
||||||
|
↓
|
||||||
|
5. Initialize Redis event broadcaster
|
||||||
|
↓
|
||||||
|
6. Register Discord event listeners
|
||||||
|
- messageCapture (message events)
|
||||||
|
- aiAnalyzer (analysis worker)
|
||||||
|
↓
|
||||||
|
7. Login to Discord
|
||||||
|
↓
|
||||||
|
8. Listen for graceful shutdown signals
|
||||||
|
```
|
||||||
|
|
||||||
|
## Graceful Shutdown
|
||||||
|
|
||||||
|
On SIGINT/SIGTERM/uncaughtException/unhandledRejection:
|
||||||
|
1. Close PostgreSQL connection
|
||||||
|
2. Disconnect from voice channels
|
||||||
|
3. Close Redis connection
|
||||||
|
4. Destroy Discord client
|
||||||
|
5. Exit process (code 0 for clean, 1 for error)
|
||||||
|
|
||||||
|
## Dependencies
|
||||||
|
|
||||||
|
**Core Discord**:
|
||||||
|
- `discord.js-selfbot-v13` — Discord client (selfbot variant)
|
||||||
|
- `@discordjs/voice` — Voice connection management
|
||||||
|
- `@discordjs/opus` — Native Opus codec
|
||||||
|
|
||||||
|
**Audio Processing**:
|
||||||
|
- `prism-media` — Opus encoding/decoding
|
||||||
|
- `opusscript` — Opus fallback for Node v26+
|
||||||
|
- `sharp` — Image resizing
|
||||||
|
|
||||||
|
**Data & Config**:
|
||||||
|
- `drizzle-orm` — Type-safe ORM
|
||||||
|
- `pg` — PostgreSQL driver
|
||||||
|
- `zod` — Config validation
|
||||||
|
- `ioredis` — Redis client
|
||||||
|
|
||||||
|
**Logging & Utilities**:
|
||||||
|
- `winston` — Structured logging
|
||||||
|
- `p-retry` — Retry with backoff
|
||||||
|
- `p-limit` — Concurrency limiting
|
||||||
|
- `piscina` — Worker pool
|
||||||
|
|
||||||
|
## No Breaking Changes
|
||||||
|
|
||||||
|
- Original `src/` remains untouched
|
||||||
|
- Discord Gateway is a **new service** in `services/discord-gateway/`
|
||||||
|
- Can run alongside existing monolith during transition
|
||||||
|
- Backend service will consume Redis events
|
||||||
|
- Frontend continues to use Backend HTTP API
|
||||||
|
|
||||||
|
## Next Steps
|
||||||
|
|
||||||
|
1. **Create Backend service** (`services/backend/`)
|
||||||
|
- HTTP API endpoints
|
||||||
|
- Redis event subscribers
|
||||||
|
- Database models
|
||||||
|
- WebSocket broadcaster
|
||||||
|
|
||||||
|
2. **Update Frontend** (`frontend/`)
|
||||||
|
- Connect to Backend HTTP API
|
||||||
|
- Subscribe to WebSocket events
|
||||||
|
|
||||||
|
3. **Docker & CI/CD**
|
||||||
|
- Dockerfile for Discord Gateway
|
||||||
|
- Docker Compose for multi-service setup
|
||||||
|
- GitHub Actions for build/deploy
|
||||||
|
|
||||||
|
4. **Documentation**
|
||||||
|
- API documentation
|
||||||
|
- Event schema documentation
|
||||||
|
- Deployment guide
|
||||||
|
|
||||||
|
## Files Created
|
||||||
|
|
||||||
|
**Total: 43 files**
|
||||||
|
|
||||||
|
### Shared Infrastructure (9 files)
|
||||||
|
- `src/shared/config/config.ts`
|
||||||
|
- `src/shared/database/` (5 files)
|
||||||
|
- `src/shared/errors/errors.ts`
|
||||||
|
- `src/shared/logger/logger.ts`
|
||||||
|
- `src/shared/logger/serialization.ts`
|
||||||
|
- `src/shared/utils/retry.ts`
|
||||||
|
- `src/shared/discord/clientOptions.ts`
|
||||||
|
|
||||||
|
### Modules (28 files)
|
||||||
|
- `src/modules/message-capture/` (5 files)
|
||||||
|
- `src/modules/ai-moderation/` (6 files)
|
||||||
|
- `src/modules/voice-recording/` (9 files)
|
||||||
|
- `src/modules/attachment-upload/` (3 files)
|
||||||
|
- `src/modules/event-broadcaster/` (3 files)
|
||||||
|
|
||||||
|
### App & Entry (4 files)
|
||||||
|
- `src/app/bootstrap.ts`
|
||||||
|
- `src/app/shutdown.ts`
|
||||||
|
- `src/index.ts`
|
||||||
|
- `src/mock-crc.ts`
|
||||||
|
|
||||||
|
### Configuration (2 files)
|
||||||
|
- `package.json`
|
||||||
|
- `ARCHITECTURE.md`
|
||||||
|
|
||||||
|
## Verification Checklist
|
||||||
|
|
||||||
|
✅ Directory structure created
|
||||||
|
✅ Shared infrastructure migrated
|
||||||
|
✅ Message capture module migrated
|
||||||
|
✅ AI moderation module migrated
|
||||||
|
✅ Voice recording module migrated
|
||||||
|
✅ Attachment upload module migrated
|
||||||
|
✅ Event broadcaster module created (Redis pub/sub)
|
||||||
|
✅ Bootstrap and entry point created
|
||||||
|
✅ Package.json with dependencies
|
||||||
|
✅ No HTTP server code (Express, WebSocket removed)
|
||||||
|
✅ Event-driven architecture implemented
|
||||||
|
✅ Graceful shutdown handler
|
||||||
|
✅ Module index files for clean exports
|
||||||
|
✅ Architecture documentation
|
||||||
|
|
||||||
|
## Event Flow Diagram
|
||||||
|
|
||||||
|
```
|
||||||
|
┌─────────────────────────────────────────────────────────────────┐
|
||||||
|
│ Discord Gateway Service │
|
||||||
|
├─────────────────────────────────────────────────────────────────┤
|
||||||
|
│ │
|
||||||
|
│ ┌──────────────────┐ ┌──────────────────┐ ┌──────────────┐ │
|
||||||
|
│ │ Message Capture │ │ AI Moderation │ │ Voice Record │ │
|
||||||
|
│ │ (Controller) │ │ (Controller) │ │ (Controller) │ │
|
||||||
|
│ └────────┬─────────┘ └────────┬─────────┘ └──────┬───────┘ │
|
||||||
|
│ │ │ │ │
|
||||||
|
│ ├─────────────────────┼───────────────────┤ │
|
||||||
|
│ │ │ │ │
|
||||||
|
│ ▼ ▼ ▼ │
|
||||||
|
│ ┌─────────────────────────────────────────────────────────┐ │
|
||||||
|
│ │ Event Broadcaster (Redis Pub/Sub) │ │
|
||||||
|
│ │ - discord:message:created │ │
|
||||||
|
│ │ - discord:message:updated │ │
|
||||||
|
│ │ - discord:message:deleted │ │
|
||||||
|
│ │ - discord:message:analyzed │ │
|
||||||
|
│ │ - discord:attachment:created │ │
|
||||||
|
│ │ - discord:attachment:uploaded │ │
|
||||||
|
│ │ - discord:voice:started │ │
|
||||||
|
│ │ - discord:voice:stopped │ │
|
||||||
|
│ │ - discord:voice:uploaded │ │
|
||||||
|
│ │ - discord:analysis:queue_status │ │
|
||||||
|
│ └─────────────────────────────────────────────────────────┘ │
|
||||||
|
│ │ │
|
||||||
|
└───────────┼────────────────────────────────────────────────────┘
|
||||||
|
│
|
||||||
|
│ Redis Pub/Sub
|
||||||
|
│
|
||||||
|
▼
|
||||||
|
┌─────────────────────────────────────────────────────────────────┐
|
||||||
|
│ Backend Service │
|
||||||
|
│ (Subscribes to events, serves HTTP API, manages WebSocket) │
|
||||||
|
└─────────────────────────────────────────────────────────────────┘
|
||||||
|
│
|
||||||
|
│ HTTP API
|
||||||
|
│
|
||||||
|
▼
|
||||||
|
┌─────────────────────────────────────────────────────────────────┐
|
||||||
|
│ Frontend Application │
|
||||||
|
│ (React SPA, real-time updates via WebSocket) │
|
||||||
|
└─────────────────────────────────────────────────────────────────┘
|
||||||
|
```
|
||||||
|
|
||||||
|
## Summary
|
||||||
|
|
||||||
|
The Discord Gateway service has been successfully extracted with:
|
||||||
|
- **Modular MVC architecture** for clean separation of concerns
|
||||||
|
- **Event-driven design** using Redis pub/sub for inter-service communication
|
||||||
|
- **Shared infrastructure layer** for reusable components
|
||||||
|
- **No HTTP server** — pure event-driven service
|
||||||
|
- **Graceful shutdown** handling
|
||||||
|
- **Type-safe configuration** with Zod validation
|
||||||
|
- **Structured logging** with Winston
|
||||||
|
- **PostgreSQL integration** with Drizzle ORM
|
||||||
|
|
||||||
|
The service is ready for integration with the Backend service, which will consume Redis events and serve the HTTP API to the Frontend.
|
||||||
@@ -0,0 +1,45 @@
|
|||||||
|
{
|
||||||
|
"name": "@bete/discord-gateway",
|
||||||
|
"version": "1.0.0",
|
||||||
|
"description": "Discord Gateway service - handles message capture, voice recording, and AI moderation",
|
||||||
|
"type": "module",
|
||||||
|
"main": "dist/index.js",
|
||||||
|
"packageManager": "pnpm@11.1.3",
|
||||||
|
"scripts": {
|
||||||
|
"dev": "tsx watch src/index.ts",
|
||||||
|
"start": "node dist/index.js",
|
||||||
|
"build": "tsc",
|
||||||
|
"typecheck": "tsc --noEmit",
|
||||||
|
"lint": "biome check --diagnostic-level=error .",
|
||||||
|
"format": "biome format --write .",
|
||||||
|
"test": "vitest run"
|
||||||
|
},
|
||||||
|
"dependencies": {
|
||||||
|
"@discordjs/opus": "^0.10.0",
|
||||||
|
"@discordjs/voice": "^0.19.2",
|
||||||
|
"@snazzah/davey": "^0.1.11",
|
||||||
|
"discord.js-selfbot-v13": "workspace:*",
|
||||||
|
"dotenv": "^17.4.2",
|
||||||
|
"drizzle-orm": "^0.45.2",
|
||||||
|
"ioredis": "^5.11.0",
|
||||||
|
"libsodium-wrappers": "^0.8.4",
|
||||||
|
"openai": "^6.38.0",
|
||||||
|
"opusscript": "^0.0.8",
|
||||||
|
"p-limit": "^7.3.0",
|
||||||
|
"p-retry": "^8.0.0",
|
||||||
|
"pg": "^8.21.0",
|
||||||
|
"piscina": "^5.1.4",
|
||||||
|
"prism-media": "2.0.0-alpha.0",
|
||||||
|
"sharp": "^0.34.5",
|
||||||
|
"winston": "^3.19.0",
|
||||||
|
"zod": "^4.4.3"
|
||||||
|
},
|
||||||
|
"devDependencies": {
|
||||||
|
"@biomejs/biome": "latest",
|
||||||
|
"@types/node": "^25.9.0",
|
||||||
|
"drizzle-kit": "^0.31.10",
|
||||||
|
"tsx": "^4.22.2",
|
||||||
|
"typescript": "^5.9.3",
|
||||||
|
"vitest": "latest"
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,121 @@
|
|||||||
|
import { Client } from "discord.js-selfbot-v13";
|
||||||
|
import { startPendingAIAnalysisWorker } from "../modules/ai-moderation/aiAnalyzer.js";
|
||||||
|
import {
|
||||||
|
EventBroadcaster,
|
||||||
|
RedisEventPublisher,
|
||||||
|
} from "../modules/event-broadcaster/index.js";
|
||||||
|
import {
|
||||||
|
registerMessageCapture,
|
||||||
|
setEventBroadcaster,
|
||||||
|
} from "../modules/message-capture/messageCapture.js";
|
||||||
|
import { VoiceController } from "../modules/voice-recording/voiceController.js";
|
||||||
|
import { config } from "../shared/config/config.js";
|
||||||
|
import {
|
||||||
|
closeDatabase,
|
||||||
|
initializeDatabase,
|
||||||
|
} from "../shared/database/drizzle.js";
|
||||||
|
import { runMigrations } from "../shared/database/migrate.js";
|
||||||
|
import { createDiscordClientOptions } from "../shared/discord/clientOptions.js";
|
||||||
|
import { createChildLogger } from "../shared/logger/logger.js";
|
||||||
|
import { createGracefulShutdown } from "./shutdown.js";
|
||||||
|
|
||||||
|
const logger = createChildLogger("discord-gateway");
|
||||||
|
|
||||||
|
export async function initializeDiscordGateway() {
|
||||||
|
if (!config.AI_LLM_API_KEY) {
|
||||||
|
logger.error(
|
||||||
|
"AI_LLM_API_KEY is missing from environment. Force closing application as AI environment is required.",
|
||||||
|
);
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
const token = config.DISCORD_TOKEN;
|
||||||
|
logger.info(
|
||||||
|
{ hasToken: token.length > 0, tokenLength: token.length },
|
||||||
|
"Config loaded",
|
||||||
|
);
|
||||||
|
|
||||||
|
logger.info("Creating Discord client");
|
||||||
|
const client = new Client(createDiscordClientOptions());
|
||||||
|
const voiceController = new VoiceController(client);
|
||||||
|
|
||||||
|
// Initialize Redis event broadcaster
|
||||||
|
const redisPublisher = new RedisEventPublisher(config.REDIS_URL, logger);
|
||||||
|
const eventBroadcaster = new EventBroadcaster(redisPublisher, logger);
|
||||||
|
|
||||||
|
const gracefulShutdown = createGracefulShutdown({
|
||||||
|
logger,
|
||||||
|
closeDatabase,
|
||||||
|
voiceController,
|
||||||
|
client,
|
||||||
|
eventBroadcaster,
|
||||||
|
});
|
||||||
|
|
||||||
|
try {
|
||||||
|
if (config.AUTO_MIGRATE_ON_STARTUP) {
|
||||||
|
logger.info(
|
||||||
|
"AUTO_MIGRATE_ON_STARTUP enabled; running database migrations",
|
||||||
|
);
|
||||||
|
await runMigrations();
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.info("Initializing database");
|
||||||
|
await initializeDatabase();
|
||||||
|
logger.info("PostgreSQL database initialized");
|
||||||
|
} catch (err) {
|
||||||
|
logger.error({ error: err }, "Failed to initialize database");
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
client.on("debug", (msg) => {
|
||||||
|
if (
|
||||||
|
msg.includes("[VOICE") ||
|
||||||
|
msg.includes("[ffmpeg") ||
|
||||||
|
msg.toLowerCase().includes("error") ||
|
||||||
|
msg.toLowerCase().includes("stream")
|
||||||
|
) {
|
||||||
|
logger.info({ debugMsg: msg }, "Discord Client Debug");
|
||||||
|
} else if (config.VERBOSE) {
|
||||||
|
logger.debug({ debugMsg: msg }, "Discord Client Debug");
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
client.on("ready", async () => {
|
||||||
|
logger.info({ user: client.user?.tag }, "Bot logged in");
|
||||||
|
setEventBroadcaster(eventBroadcaster);
|
||||||
|
registerMessageCapture(client);
|
||||||
|
startPendingAIAnalysisWorker(client);
|
||||||
|
});
|
||||||
|
|
||||||
|
client.on("error", (err) => {
|
||||||
|
logger.error({ error: err }, "Client error");
|
||||||
|
});
|
||||||
|
|
||||||
|
process.on("SIGINT", () => {
|
||||||
|
gracefulShutdown("SIGINT");
|
||||||
|
});
|
||||||
|
|
||||||
|
process.on("SIGTERM", () => {
|
||||||
|
gracefulShutdown("SIGTERM");
|
||||||
|
});
|
||||||
|
|
||||||
|
process.on("uncaughtException", (err) => {
|
||||||
|
logger.error({ error: err }, "Uncaught exception");
|
||||||
|
gracefulShutdown("uncaughtException");
|
||||||
|
});
|
||||||
|
|
||||||
|
process.on("unhandledRejection", (reason, promise) => {
|
||||||
|
logger.error({ reason, promise }, "Unhandled rejection");
|
||||||
|
gracefulShutdown("unhandledRejection");
|
||||||
|
});
|
||||||
|
|
||||||
|
logger.info("Calling Discord client.login");
|
||||||
|
client
|
||||||
|
.login(token)
|
||||||
|
.then(() => {
|
||||||
|
logger.info("Discord client.login resolved");
|
||||||
|
})
|
||||||
|
.catch((error: unknown) => {
|
||||||
|
logger.error({ error }, "Discord client.login failed");
|
||||||
|
});
|
||||||
|
}
|
||||||
@@ -0,0 +1,55 @@
|
|||||||
|
import type { Client } from "discord.js-selfbot-v13";
|
||||||
|
import type { EventBroadcaster } from "../modules/event-broadcaster/index.js";
|
||||||
|
import type { VoiceController } from "../modules/voice-recording/voiceController.js";
|
||||||
|
import type { closeDatabase } from "../shared/database/drizzle.js";
|
||||||
|
import type { createChildLogger } from "../shared/logger/logger.js";
|
||||||
|
|
||||||
|
type Logger = ReturnType<typeof createChildLogger>;
|
||||||
|
type CloseDatabase = typeof closeDatabase;
|
||||||
|
|
||||||
|
export interface GracefulShutdownOptions {
|
||||||
|
logger: Logger;
|
||||||
|
closeDatabase: CloseDatabase;
|
||||||
|
voiceController: VoiceController;
|
||||||
|
client: Client;
|
||||||
|
eventBroadcaster: EventBroadcaster;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function createGracefulShutdown(options: GracefulShutdownOptions) {
|
||||||
|
let isShuttingDown = false;
|
||||||
|
|
||||||
|
return async function gracefulShutdown(signal: string) {
|
||||||
|
if (isShuttingDown) {
|
||||||
|
options.logger.warn(`Already shutting down, ignoring ${signal}`);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
isShuttingDown = true;
|
||||||
|
options.logger.info({ signal }, "Graceful shutdown initiated");
|
||||||
|
|
||||||
|
try {
|
||||||
|
options.logger.info("Closing database...");
|
||||||
|
await options.closeDatabase();
|
||||||
|
options.logger.info("Database closed");
|
||||||
|
|
||||||
|
options.logger.info("Stopping voice connection...");
|
||||||
|
await options.voiceController.disconnect();
|
||||||
|
|
||||||
|
options.logger.info("Closing event broadcaster...");
|
||||||
|
await options.eventBroadcaster.close();
|
||||||
|
|
||||||
|
options.logger.info("Destroying Discord client...");
|
||||||
|
try {
|
||||||
|
options.client.destroy();
|
||||||
|
} catch (err) {
|
||||||
|
options.logger.warn({ error: err }, "Error destroying client");
|
||||||
|
}
|
||||||
|
|
||||||
|
options.logger.info("Graceful shutdown completed");
|
||||||
|
process.exit(0);
|
||||||
|
} catch (err) {
|
||||||
|
options.logger.error({ error: err }, "Error during graceful shutdown");
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -0,0 +1,14 @@
|
|||||||
|
import "./mock-crc.js";
|
||||||
|
import "libsodium-wrappers";
|
||||||
|
import "@snazzah/davey";
|
||||||
|
import "dotenv/config";
|
||||||
|
import { initializeDiscordGateway } from "./app/bootstrap.js";
|
||||||
|
import { createChildLogger } from "./shared/logger/logger.js";
|
||||||
|
|
||||||
|
const logger = createChildLogger("discord-gateway");
|
||||||
|
|
||||||
|
// Initialize the Discord Gateway service
|
||||||
|
initializeDiscordGateway().catch((error: unknown) => {
|
||||||
|
logger.error({ error }, "Failed to initialize Discord Gateway");
|
||||||
|
process.exit(1);
|
||||||
|
});
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
// Mock CRC for discord.js compatibility
|
||||||
|
export {};
|
||||||
|
|
||||||
|
declare global {
|
||||||
|
var crc32: ((data: Buffer) => number) | undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!globalThis.crc32) {
|
||||||
|
globalThis.crc32 = (data: Buffer) => {
|
||||||
|
let crc = 0 ^ -1;
|
||||||
|
for (let i = 0; i < data.length; i++) {
|
||||||
|
crc = (crc >>> 8) ^ ((crc ^ data[i]) & 0xff);
|
||||||
|
}
|
||||||
|
return (crc ^ -1) >>> 0;
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -0,0 +1,145 @@
|
|||||||
|
import { config } from "../../shared/config/config.js";
|
||||||
|
import { initializeDatabase } from "../../shared/database/drizzle.js";
|
||||||
|
import { buildConversationContext } from "./conversationContext.js";
|
||||||
|
import { runModerationAnalysis } from "./llmModerationClient.js";
|
||||||
|
import {
|
||||||
|
getAttachmentsForMessages,
|
||||||
|
getConversationContextBefore,
|
||||||
|
updateMessagesAIAnalysisBulk,
|
||||||
|
} from "../message-capture/messageStore.js";
|
||||||
|
import type { MessageRecord } from "../message-capture/types.js";
|
||||||
|
|
||||||
|
let dbInitialized = false;
|
||||||
|
let dbInitPromise: Promise<any> | null = null;
|
||||||
|
|
||||||
|
async function ensureDb() {
|
||||||
|
if (dbInitialized) return;
|
||||||
|
if (!dbInitPromise) {
|
||||||
|
dbInitPromise = initializeDatabase().then(() => {
|
||||||
|
dbInitialized = true;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
await dbInitPromise;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface AnalysisWorkerRequest {
|
||||||
|
conversationKey: string;
|
||||||
|
messages: MessageRecord[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export type AnalysisWorkerResponse =
|
||||||
|
| {
|
||||||
|
ok: true;
|
||||||
|
conversationKey: string;
|
||||||
|
rows: MessageRecord[];
|
||||||
|
}
|
||||||
|
| {
|
||||||
|
ok: false;
|
||||||
|
conversationKey: string;
|
||||||
|
rows: MessageRecord[];
|
||||||
|
error: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
export default async function processAnalysisRequest({
|
||||||
|
conversationKey,
|
||||||
|
messages,
|
||||||
|
}: AnalysisWorkerRequest): Promise<AnalysisWorkerResponse> {
|
||||||
|
if (!config.AI_LLM_API_KEY) {
|
||||||
|
console.error(
|
||||||
|
JSON.stringify({
|
||||||
|
level: "FATAL",
|
||||||
|
context: "aiAnalysisWorker",
|
||||||
|
error:
|
||||||
|
"AI_LLM_API_KEY is missing from environment. Force closing worker operation.",
|
||||||
|
timestamp: new Date().toISOString(),
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
try {
|
||||||
|
await ensureDb();
|
||||||
|
} catch (dbError) {
|
||||||
|
const msg = dbError instanceof Error ? dbError.message : String(dbError);
|
||||||
|
return {
|
||||||
|
ok: false,
|
||||||
|
conversationKey,
|
||||||
|
rows: [],
|
||||||
|
error: `Database init failed: ${msg}`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const firstMessage = messages[0];
|
||||||
|
if (!firstMessage) return { ok: true, conversationKey, rows: [] };
|
||||||
|
|
||||||
|
const contextBefore = await getConversationContextBefore({
|
||||||
|
channelId: firstMessage.channel_id,
|
||||||
|
threadId: firstMessage.thread_id,
|
||||||
|
beforeCreatedAt: firstMessage.created_at,
|
||||||
|
limit: config.AI_ANALYSIS_CONTEXT_MESSAGE_LIMIT,
|
||||||
|
});
|
||||||
|
|
||||||
|
const contextLines = await buildConversationContext({
|
||||||
|
contextBefore,
|
||||||
|
targets: messages,
|
||||||
|
maxTokens: config.AI_ANALYSIS_MAX_CONTEXT_TOKENS,
|
||||||
|
});
|
||||||
|
|
||||||
|
const targetIds = messages.map((m) => m.id);
|
||||||
|
const contextIds = contextBefore.map((m) => m.id);
|
||||||
|
const allMessageIds = [...targetIds, ...contextIds];
|
||||||
|
const attachments = await getAttachmentsForMessages(allMessageIds);
|
||||||
|
|
||||||
|
const result = await runModerationAnalysis({
|
||||||
|
targets: messages,
|
||||||
|
contextText: contextLines.join("\n"),
|
||||||
|
attachments,
|
||||||
|
});
|
||||||
|
|
||||||
|
const updates = result.results.map((analysisResult) => ({
|
||||||
|
messageId: analysisResult.messageId,
|
||||||
|
result: {
|
||||||
|
status: analysisResult.status,
|
||||||
|
flags: JSON.stringify(analysisResult.flags),
|
||||||
|
score: analysisResult.score,
|
||||||
|
analysis: analysisResult.analysis,
|
||||||
|
categories: analysisResult.categories,
|
||||||
|
severity: analysisResult.severity,
|
||||||
|
confidence: analysisResult.confidence,
|
||||||
|
recommendedAction: analysisResult.recommendedAction,
|
||||||
|
analyzedAt: Date.now(),
|
||||||
|
error: null,
|
||||||
|
},
|
||||||
|
}));
|
||||||
|
|
||||||
|
try {
|
||||||
|
const rows = await updateMessagesAIAnalysisBulk(updates);
|
||||||
|
return { ok: true, conversationKey, rows };
|
||||||
|
} catch (dbErr) {
|
||||||
|
// If bulk update fails, we log it but don't fail the worker completely
|
||||||
|
// so it can at least retry later without blowing up the circuit breaker if it was an isolated issue
|
||||||
|
throw new Error(
|
||||||
|
`Failed to update DB: ${dbErr instanceof Error ? dbErr.message : String(dbErr)}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
const errorMessage = error instanceof Error ? error.message : String(error);
|
||||||
|
const errorStack = error instanceof Error ? error.stack : undefined;
|
||||||
|
const rows: MessageRecord[] = [];
|
||||||
|
|
||||||
|
console.error(
|
||||||
|
JSON.stringify({
|
||||||
|
level: "ERROR",
|
||||||
|
context: "aiAnalysisWorker",
|
||||||
|
conversationKey,
|
||||||
|
messageCount: messages.length,
|
||||||
|
error: errorMessage,
|
||||||
|
stack: errorStack,
|
||||||
|
timestamp: new Date().toISOString(),
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
return { ok: false, conversationKey, rows, error: errorMessage };
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,918 @@
|
|||||||
|
import { existsSync } from "node:fs";
|
||||||
|
import { fileURLToPath } from "node:url";
|
||||||
|
import type { Client } from "discord.js-selfbot-v13";
|
||||||
|
import { AbortError } from "p-retry";
|
||||||
|
import { Piscina } from "piscina";
|
||||||
|
import { config } from "../../shared/config/config.js";
|
||||||
|
import { createChildLogger } from "../../shared/logger/logger.js";
|
||||||
|
import { retryWithBackoff } from "../../shared/utils/retry.js";
|
||||||
|
import { invalidateAnalyticsCache } from "../message-capture/analyticsStore.js";
|
||||||
|
import { attemptAutoDeleteFlaggedMessage } from "./autoDeleteManager.js";
|
||||||
|
import { buildConversationContext } from "./conversationContext.js";
|
||||||
|
import { runModerationAnalysis } from "./llmModerationClient.js";
|
||||||
|
import { isAgeRestrictedMetadata } from "../message-capture/messageMetadata.js";
|
||||||
|
import {
|
||||||
|
getAttachmentsForMessages,
|
||||||
|
getConversationContextBefore,
|
||||||
|
getConversationKeysWithIncompleteAnalysis,
|
||||||
|
getIncompleteMessagesByConversation,
|
||||||
|
getMessageById,
|
||||||
|
getPendingConversationKeys,
|
||||||
|
getPendingMessagesByConversation,
|
||||||
|
updateMessageAIAnalysis,
|
||||||
|
updateMessagesAIAnalysisBulk,
|
||||||
|
} from "../message-capture/messageStore.js";
|
||||||
|
import type {
|
||||||
|
AnalysisQueueStatus,
|
||||||
|
MessageRecord,
|
||||||
|
ModerationBroadcaster,
|
||||||
|
} from "../message-capture/types.js";
|
||||||
|
|
||||||
|
const logger = createChildLogger("ai-analyzer");
|
||||||
|
|
||||||
|
type ModerationGlobal = typeof globalThis & {
|
||||||
|
moderationBroadcaster?: ModerationBroadcaster;
|
||||||
|
};
|
||||||
|
|
||||||
|
function getModerationBroadcaster(): ModerationBroadcaster | undefined {
|
||||||
|
return (globalThis as ModerationGlobal).moderationBroadcaster;
|
||||||
|
}
|
||||||
|
|
||||||
|
function scheduleAutoDelete(row: MessageRecord): void {
|
||||||
|
if (row.ai_status !== "flagged" && row.ai_status !== "warn") return;
|
||||||
|
const run = () => {
|
||||||
|
attemptAutoDeleteFlaggedMessage(moderationClient, row).catch((error: unknown) => {
|
||||||
|
logger.error(
|
||||||
|
{
|
||||||
|
messageId: row.id,
|
||||||
|
error: error instanceof Error ? error.message : String(error),
|
||||||
|
},
|
||||||
|
"Unexpected auto-delete error",
|
||||||
|
);
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
|
if (config.AUTO_DELETE_FLAGGED_DELAY_MS > 0) {
|
||||||
|
setTimeout(run, config.AUTO_DELETE_FLAGGED_DELAY_MS);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
setImmediate(run);
|
||||||
|
}
|
||||||
|
|
||||||
|
function isAgeRestrictedMessage(message: MessageRecord): boolean {
|
||||||
|
return isAgeRestrictedMetadata(message.metadata);
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildAgeRestrictedSkipResult(): {
|
||||||
|
status: "clean";
|
||||||
|
flags: string | null;
|
||||||
|
score: number;
|
||||||
|
analysis: string;
|
||||||
|
categories: string[];
|
||||||
|
severity: "none";
|
||||||
|
confidence: number;
|
||||||
|
recommendedAction: "none";
|
||||||
|
analyzedAt: number;
|
||||||
|
error: null;
|
||||||
|
} {
|
||||||
|
return {
|
||||||
|
status: "clean",
|
||||||
|
flags: JSON.stringify(["age_restricted"]),
|
||||||
|
score: 0,
|
||||||
|
analysis: "Skipped moderation for age-restricted content.",
|
||||||
|
categories: ["age_restricted"],
|
||||||
|
severity: "none",
|
||||||
|
confidence: 1,
|
||||||
|
recommendedAction: "none",
|
||||||
|
analyzedAt: Date.now(),
|
||||||
|
error: null,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async function skipAgeRestrictedMessages(
|
||||||
|
messages: MessageRecord[],
|
||||||
|
): Promise<MessageRecord[]> {
|
||||||
|
const ageRestrictedMessages = messages.filter(isAgeRestrictedMessage);
|
||||||
|
if (ageRestrictedMessages.length === 0) {
|
||||||
|
return messages;
|
||||||
|
}
|
||||||
|
|
||||||
|
const skippedRows = await updateMessagesAIAnalysisBulk(
|
||||||
|
ageRestrictedMessages.map((message) => ({
|
||||||
|
messageId: message.id,
|
||||||
|
result: buildAgeRestrictedSkipResult(),
|
||||||
|
})),
|
||||||
|
);
|
||||||
|
|
||||||
|
for (const row of skippedRows) {
|
||||||
|
getModerationBroadcaster()?.messageAnalyzed(row);
|
||||||
|
}
|
||||||
|
|
||||||
|
const skippedIds = new Set(
|
||||||
|
ageRestrictedMessages.map((message) => message.id),
|
||||||
|
);
|
||||||
|
return messages.filter((message) => !skippedIds.has(message.id));
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Batch pipeline state
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
/** Debounce timer handle per conversation key. */
|
||||||
|
const conversationDebounceTimers = new Map<string, NodeJS.Timeout>();
|
||||||
|
/** Timestamp of when processing started per conversation key. */
|
||||||
|
const conversationProcessing = new Map<string, number>();
|
||||||
|
/** Cooldown expiry timestamp per conversation key after an error. */
|
||||||
|
const conversationErrorCooldown = new Map<string, number>();
|
||||||
|
|
||||||
|
let activeRequests = 0;
|
||||||
|
let lastError: string | null = null;
|
||||||
|
let moderationClient: Client | undefined;
|
||||||
|
|
||||||
|
// Batch circuit breaker
|
||||||
|
let consecutiveErrors = 0;
|
||||||
|
const MAX_CONSECUTIVE_ERRORS = 5;
|
||||||
|
let globalCooldownUntil = 0;
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Individual fallback queue — runs PARALLEL to the batch pipeline.
|
||||||
|
//
|
||||||
|
// Design guarantees:
|
||||||
|
// • Concurrency is capped at config.AI_ANALYSIS_INDIVIDUAL_MAX_CONCURRENT.
|
||||||
|
// • A flat Set<messageId> de-duplicates so the same message can't be
|
||||||
|
// in-flight twice (Discord snowflakes are globally unique, but be safe).
|
||||||
|
// • A Map<conversationKey, count> lets the recovery worker skip conversations
|
||||||
|
// that already have individual work in progress (#4 fix).
|
||||||
|
// • A separate circuit breaker prevents a cascade of individual failures
|
||||||
|
// from hammering a down/rate-limited LLM endpoint (#1+#5 fix).
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
/** IDs currently being processed one-by-one. */
|
||||||
|
const individualInFlight = new Set<string>();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Per-conversation count of in-flight individual messages.
|
||||||
|
* Used by the recovery worker to avoid re-scheduling a conversation that
|
||||||
|
* already has individual fallback work running for it.
|
||||||
|
*/
|
||||||
|
const individualInFlightByConversation = new Map<string, number>();
|
||||||
|
|
||||||
|
/** Counter for observability. */
|
||||||
|
let activeIndividualRequests = 0;
|
||||||
|
|
||||||
|
// Individual fallback circuit breaker (independent of batch CB)
|
||||||
|
let individualConsecutiveErrors = 0;
|
||||||
|
let individualCooldownUntil = 0;
|
||||||
|
const INDIVIDUAL_COOLDOWN_MS = 30000;
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Piscina worker pool (batch path only)
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
function getAnalysisWorkerUrl(): URL {
|
||||||
|
const candidates = [
|
||||||
|
new URL("./aiAnalysisWorker.js", import.meta.url),
|
||||||
|
new URL("../aiAnalysisWorker.js", import.meta.url),
|
||||||
|
new URL("./aiAnalysisWorker.ts", import.meta.url),
|
||||||
|
];
|
||||||
|
|
||||||
|
for (const candidate of candidates) {
|
||||||
|
if (existsSync(fileURLToPath(candidate))) {
|
||||||
|
return candidate;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return candidates[2];
|
||||||
|
}
|
||||||
|
|
||||||
|
const workerPool = new Piscina({
|
||||||
|
filename: fileURLToPath(getAnalysisWorkerUrl()),
|
||||||
|
execArgv: process.execArgv,
|
||||||
|
});
|
||||||
|
|
||||||
|
interface AnalysisWorkerResponse {
|
||||||
|
ok: boolean;
|
||||||
|
conversationKey: string;
|
||||||
|
rows: MessageRecord[];
|
||||||
|
error?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Exported helpers
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Gets the conversation key for a message (thread_id or channel_id).
|
||||||
|
*/
|
||||||
|
export function getConversationKey(message: MessageRecord): string {
|
||||||
|
return message.thread_id || message.channel_id;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Picks a batch of messages within a token budget.
|
||||||
|
* `tokensPerMessage` accounts for JSON structure overhead around each entry.
|
||||||
|
* Uses a rough character-based token estimate (avoids async formatMessageForPrompt
|
||||||
|
* since this function runs in a synchronous promise chain).
|
||||||
|
*/
|
||||||
|
export function pickBatchWithinBudget(
|
||||||
|
messages: MessageRecord[],
|
||||||
|
maxTokens: number,
|
||||||
|
tokensPerMessage: number,
|
||||||
|
): MessageRecord[] {
|
||||||
|
const batch: MessageRecord[] = [];
|
||||||
|
let usedTokens = 0;
|
||||||
|
|
||||||
|
for (const msg of messages) {
|
||||||
|
const content = msg.edited_content ?? msg.content;
|
||||||
|
// Rough token estimate: ~3 chars per token + metadata overhead
|
||||||
|
const msgTokens = Math.ceil(content.length / 3) + tokensPerMessage;
|
||||||
|
|
||||||
|
if (usedTokens + msgTokens <= maxTokens) {
|
||||||
|
batch.push(msg);
|
||||||
|
usedTokens += msgTokens;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return batch;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Conversation lock helpers
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
function isConversationProcessingLocked(conversationKey: string): boolean {
|
||||||
|
const startedAt = conversationProcessing.get(conversationKey);
|
||||||
|
// FIX #7: use configurable timeout that exceeds (LLM timeout × max retries).
|
||||||
|
// Old hardcoded value was 30 000 ms — shorter than a single LLM call under retries.
|
||||||
|
return Boolean(
|
||||||
|
startedAt &&
|
||||||
|
Date.now() - startedAt < config.AI_ANALYSIS_PROCESSING_TIMEOUT_MS,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Individual fallback pipeline
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Processes a single message directly in the main process (no IPC/worker
|
||||||
|
* pool overhead). Never called from the batch path.
|
||||||
|
*
|
||||||
|
* FIX #1+#5: Increments the individual circuit breaker on failure so a
|
||||||
|
* sustained outage stops hammering the LLM endpoint.
|
||||||
|
*
|
||||||
|
* Infinite-loop prevention: if the LLM consistently drops the single target
|
||||||
|
* message across all retries (analysis_incomplete), we write a terminal flag
|
||||||
|
* 'individual_analysis_exhausted' to DB instead of 'analysis_incomplete'.
|
||||||
|
* The recovery worker only queries for 'analysis_incomplete', so exhausted
|
||||||
|
* messages are permanently excluded from the reprocessing loop.
|
||||||
|
* Transient failures (network/parse/DB) are NOT written as exhausted — they
|
||||||
|
* stay as 'analysis_incomplete' so the circuit-breaker-throttled recovery
|
||||||
|
* cycle can retry them later.
|
||||||
|
*/
|
||||||
|
async function processIndividualFallback(
|
||||||
|
message: MessageRecord,
|
||||||
|
): Promise<void> {
|
||||||
|
const { id: messageId } = message;
|
||||||
|
const conversationKey = getConversationKey(message);
|
||||||
|
|
||||||
|
activeIndividualRequests++;
|
||||||
|
// Increment per-conversation counter so the recovery worker can see it.
|
||||||
|
individualInFlightByConversation.set(
|
||||||
|
conversationKey,
|
||||||
|
(individualInFlightByConversation.get(conversationKey) ?? 0) + 1,
|
||||||
|
);
|
||||||
|
|
||||||
|
// Track whether all retries were exhausted specifically because the LLM
|
||||||
|
// consistently returned no result for this message (vs. a transient error).
|
||||||
|
let exhaustedOnIncomplete = false;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const contextBefore = await getConversationContextBefore({
|
||||||
|
channelId: message.channel_id,
|
||||||
|
threadId: message.thread_id,
|
||||||
|
beforeCreatedAt: message.created_at,
|
||||||
|
limit: config.AI_ANALYSIS_CONTEXT_MESSAGE_LIMIT,
|
||||||
|
});
|
||||||
|
|
||||||
|
const contextLines = await buildConversationContext({
|
||||||
|
contextBefore,
|
||||||
|
targets: [message],
|
||||||
|
maxTokens: config.AI_ANALYSIS_MAX_CONTEXT_TOKENS,
|
||||||
|
});
|
||||||
|
|
||||||
|
const contextIds = contextBefore.map((m) => m.id);
|
||||||
|
const attachments = await getAttachmentsForMessages([
|
||||||
|
messageId,
|
||||||
|
...contextIds,
|
||||||
|
]);
|
||||||
|
|
||||||
|
const analysisResult = await retryWithBackoff(
|
||||||
|
async () => {
|
||||||
|
try {
|
||||||
|
const result = await runModerationAnalysis({
|
||||||
|
targets: [message],
|
||||||
|
contextText: contextLines.join("\n"),
|
||||||
|
attachments,
|
||||||
|
});
|
||||||
|
|
||||||
|
// If the LLM still dropped our only target, convert to a retryable
|
||||||
|
// throw so backoff kicks in. Track this so the catch block can
|
||||||
|
// distinguish it from a transient network/parse failure.
|
||||||
|
const stillIncomplete = result.results.some((r) =>
|
||||||
|
r.flags.includes("analysis_incomplete"),
|
||||||
|
);
|
||||||
|
if (stillIncomplete) {
|
||||||
|
exhaustedOnIncomplete = true;
|
||||||
|
throw new Error(
|
||||||
|
`LLM returned no result for single-target message ${messageId} — will retry with backoff`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Got a real result — clear the incomplete flag.
|
||||||
|
exhaustedOnIncomplete = false;
|
||||||
|
|
||||||
|
return result;
|
||||||
|
} catch (err: any) {
|
||||||
|
// Propagate AbortError so outer retry is immediately cancelled on 429.
|
||||||
|
if (err instanceof AbortError) {
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
|
if (
|
||||||
|
err?.status === 429 ||
|
||||||
|
err?.status === 401 ||
|
||||||
|
err?.status === 403
|
||||||
|
) {
|
||||||
|
throw new AbortError(err);
|
||||||
|
}
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
retries: 2,
|
||||||
|
minTimeout: 2000,
|
||||||
|
maxTimeout: 15000,
|
||||||
|
logger,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
const updates = analysisResult.results.map((r) => ({
|
||||||
|
messageId: r.messageId,
|
||||||
|
result: {
|
||||||
|
status: r.status,
|
||||||
|
flags: JSON.stringify(r.flags),
|
||||||
|
score: r.score,
|
||||||
|
analysis: r.analysis,
|
||||||
|
categories: r.categories,
|
||||||
|
severity: r.severity,
|
||||||
|
confidence: r.confidence,
|
||||||
|
recommendedAction: r.recommendedAction,
|
||||||
|
analyzedAt: Date.now(),
|
||||||
|
error: null,
|
||||||
|
},
|
||||||
|
}));
|
||||||
|
|
||||||
|
const rows = await updateMessagesAIAnalysisBulk(updates);
|
||||||
|
for (const row of rows) {
|
||||||
|
getModerationBroadcaster()?.messageAnalyzed(row);
|
||||||
|
invalidateAnalyticsCache(row.guild_id);
|
||||||
|
scheduleAutoDelete(row);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Reset individual CB on success.
|
||||||
|
individualConsecutiveErrors = 0;
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
{ messageId, status: analysisResult.results[0]?.status },
|
||||||
|
"Individual fallback analysis complete",
|
||||||
|
);
|
||||||
|
} catch (error) {
|
||||||
|
// FIX #5: individual failures now feed their own circuit breaker.
|
||||||
|
individualConsecutiveErrors++;
|
||||||
|
if (
|
||||||
|
individualConsecutiveErrors >= config.AI_ANALYSIS_INDIVIDUAL_CB_THRESHOLD
|
||||||
|
) {
|
||||||
|
individualCooldownUntil = Date.now() + INDIVIDUAL_COOLDOWN_MS;
|
||||||
|
logger.warn(
|
||||||
|
{
|
||||||
|
threshold: config.AI_ANALYSIS_INDIVIDUAL_CB_THRESHOLD,
|
||||||
|
cooldownUntil: new Date(individualCooldownUntil).toISOString(),
|
||||||
|
},
|
||||||
|
"Individual fallback circuit breaker triggered",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
lastError = error instanceof Error ? error.message : String(error);
|
||||||
|
|
||||||
|
// Infinite-loop prevention: if all retries were exhausted because the LLM
|
||||||
|
// consistently dropped this specific message (not a transient error),
|
||||||
|
// overwrite the DB entry with a terminal flag that the recovery query
|
||||||
|
// does NOT match. This permanently removes it from the recovery loop
|
||||||
|
// while keeping it visible as an error in the dashboard.
|
||||||
|
if (exhaustedOnIncomplete) {
|
||||||
|
await updateMessagesAIAnalysisBulk([
|
||||||
|
{
|
||||||
|
messageId,
|
||||||
|
result: {
|
||||||
|
status: "error",
|
||||||
|
flags: JSON.stringify(["individual_analysis_exhausted"]),
|
||||||
|
score: 0,
|
||||||
|
analysis:
|
||||||
|
"Individual fallback exhausted all retries: LLM consistently dropped this message even in single-target mode",
|
||||||
|
categories: ["individual_analysis_exhausted"],
|
||||||
|
severity: "none",
|
||||||
|
confidence: 0,
|
||||||
|
recommendedAction: "review",
|
||||||
|
analyzedAt: Date.now(),
|
||||||
|
error: lastError,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
]).catch((dbErr: unknown) => {
|
||||||
|
logger.error(
|
||||||
|
{ messageId, error: String(dbErr) },
|
||||||
|
"Failed to write terminal exhausted status — message may re-enter recovery loop",
|
||||||
|
);
|
||||||
|
});
|
||||||
|
logger.warn(
|
||||||
|
{ messageId },
|
||||||
|
"Individual fallback exhausted — marked as individual_analysis_exhausted to stop recovery loop",
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
// Transient failure (network/parse/DB): do NOT write terminal status.
|
||||||
|
// Message stays as error/analysis_incomplete in DB and will be retried
|
||||||
|
// by the recovery worker, subject to the individual circuit breaker.
|
||||||
|
logger.error(
|
||||||
|
{
|
||||||
|
messageId,
|
||||||
|
error: lastError,
|
||||||
|
stack: error instanceof Error ? error.stack : undefined,
|
||||||
|
},
|
||||||
|
"Individual fallback analysis failed (transient) — will be retried by recovery worker",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
activeIndividualRequests--;
|
||||||
|
individualInFlight.delete(messageId);
|
||||||
|
|
||||||
|
// Decrement per-conversation counter; remove key when it hits zero.
|
||||||
|
const prev = individualInFlightByConversation.get(conversationKey) ?? 1;
|
||||||
|
if (prev <= 1) {
|
||||||
|
individualInFlightByConversation.delete(conversationKey);
|
||||||
|
} else {
|
||||||
|
individualInFlightByConversation.set(conversationKey, prev - 1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Fans out message records to the individual fallback queue.
|
||||||
|
*
|
||||||
|
* FIX #1: Checks concurrency cap before admitting new work.
|
||||||
|
* FIX #5: Checks individual circuit breaker before admitting new work.
|
||||||
|
* Messages that cannot be admitted remain as `error/analysis_incomplete` in
|
||||||
|
* the DB and will be picked up by the recovery worker on the next interval.
|
||||||
|
*/
|
||||||
|
function enqueueIndividualFallbacks(messages: MessageRecord[]): void {
|
||||||
|
// FIX #5: Honour the individual circuit breaker.
|
||||||
|
if (Date.now() < individualCooldownUntil) {
|
||||||
|
logger.warn(
|
||||||
|
{
|
||||||
|
until: new Date(individualCooldownUntil).toISOString(),
|
||||||
|
skipped: messages.length,
|
||||||
|
},
|
||||||
|
"Individual fallback circuit breaker active — messages will be recovered later",
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const newMessages = messages.filter((m) => !individualInFlight.has(m.id));
|
||||||
|
if (newMessages.length === 0) return;
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
{
|
||||||
|
count: newMessages.length,
|
||||||
|
messageIds: newMessages.map((m) => m.id),
|
||||||
|
},
|
||||||
|
"Enqueueing individual fallback analysis for batch-incomplete messages",
|
||||||
|
);
|
||||||
|
|
||||||
|
for (const msg of newMessages) {
|
||||||
|
individualInFlight.add(msg.id);
|
||||||
|
// Fire-and-forget: processIndividualFallback handles all errors internally.
|
||||||
|
processIndividualFallback(msg).catch((err: unknown) => {
|
||||||
|
// Belt-and-suspenders guard — should never reach here.
|
||||||
|
logger.error(
|
||||||
|
{ messageId: msg.id, error: String(err) },
|
||||||
|
"Unexpected uncaught error escaping processIndividualFallback",
|
||||||
|
);
|
||||||
|
individualInFlight.delete(msg.id);
|
||||||
|
const ck = getConversationKey(msg);
|
||||||
|
const prev = individualInFlightByConversation.get(ck) ?? 1;
|
||||||
|
if (prev <= 1) {
|
||||||
|
individualInFlightByConversation.delete(ck);
|
||||||
|
} else {
|
||||||
|
individualInFlightByConversation.set(ck, prev - 1);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Batch pipeline
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
async function processBatch(
|
||||||
|
conversationKey: string,
|
||||||
|
messages: MessageRecord[],
|
||||||
|
): Promise<void> {
|
||||||
|
if (messages.length === 0) return;
|
||||||
|
if (Date.now() < globalCooldownUntil) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
activeRequests++;
|
||||||
|
let shouldScheduleNext = false;
|
||||||
|
const processingStartedAt = Date.now();
|
||||||
|
conversationProcessing.set(conversationKey, processingStartedAt);
|
||||||
|
try {
|
||||||
|
const result = (await workerPool.run({
|
||||||
|
conversationKey,
|
||||||
|
messages,
|
||||||
|
})) as AnalysisWorkerResponse;
|
||||||
|
|
||||||
|
for (const row of result.rows) {
|
||||||
|
getModerationBroadcaster()?.messageAnalyzed(row);
|
||||||
|
scheduleAutoDelete(row);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!result.ok) {
|
||||||
|
consecutiveErrors++;
|
||||||
|
if (consecutiveErrors >= MAX_CONSECUTIVE_ERRORS) {
|
||||||
|
globalCooldownUntil = Date.now() + 60000;
|
||||||
|
logger.warn(
|
||||||
|
"Global circuit breaker triggered due to consecutive errors",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Batch failed entirely — fall back all messages to individual queue
|
||||||
|
// so no message is permanently lost behind a cooldown.
|
||||||
|
logger.warn(
|
||||||
|
{
|
||||||
|
conversationKey,
|
||||||
|
messageCount: messages.length,
|
||||||
|
error: result.error,
|
||||||
|
},
|
||||||
|
"Batch failed entirely — routing all messages to individual fallback queue",
|
||||||
|
);
|
||||||
|
enqueueIndividualFallbacks(messages);
|
||||||
|
|
||||||
|
lastError = result.error ?? "Analysis worker failed";
|
||||||
|
conversationErrorCooldown.set(
|
||||||
|
conversationKey,
|
||||||
|
Date.now() + config.AI_ANALYSIS_ERROR_COOLDOWN_MS,
|
||||||
|
);
|
||||||
|
logger.error(
|
||||||
|
{
|
||||||
|
conversationKey,
|
||||||
|
error: lastError,
|
||||||
|
messageCount: messages.length,
|
||||||
|
messageIds: messages.map((m) => m.id),
|
||||||
|
cooldownUntil: new Date(
|
||||||
|
Date.now() + config.AI_ANALYSIS_ERROR_COOLDOWN_MS,
|
||||||
|
).toISOString(),
|
||||||
|
timestamp: new Date().toISOString(),
|
||||||
|
},
|
||||||
|
"Batch analysis failed, will retry after cooldown",
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Batch succeeded — but check for messages the LLM silently dropped.
|
||||||
|
// Rows with flag "analysis_incomplete" were produced by parseModerationResponse
|
||||||
|
// as synthetic errors; they must be re-processed individually.
|
||||||
|
const incompleteMessages = messages.filter((msg) => {
|
||||||
|
const row = result.rows.find((r) => r.id === msg.id);
|
||||||
|
if (!row) {
|
||||||
|
// The DB update row is missing entirely — treat as incomplete.
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
const flags: string[] = (() => {
|
||||||
|
try {
|
||||||
|
return JSON.parse(row.ai_moderation_flags ?? "[]") as string[];
|
||||||
|
} catch {
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
})();
|
||||||
|
return row.ai_status === "error" && flags.includes("analysis_incomplete");
|
||||||
|
});
|
||||||
|
|
||||||
|
if (incompleteMessages.length > 0) {
|
||||||
|
logger.warn(
|
||||||
|
{
|
||||||
|
conversationKey,
|
||||||
|
incompleteCount: incompleteMessages.length,
|
||||||
|
incompleteIds: incompleteMessages.map((m) => m.id),
|
||||||
|
totalBatchSize: messages.length,
|
||||||
|
},
|
||||||
|
"Batch returned incomplete results — fanning out to individual fallback queue",
|
||||||
|
);
|
||||||
|
enqueueIndividualFallbacks(incompleteMessages);
|
||||||
|
}
|
||||||
|
|
||||||
|
consecutiveErrors = 0; // Reset batch circuit breaker
|
||||||
|
conversationErrorCooldown.delete(conversationKey);
|
||||||
|
shouldScheduleNext = true;
|
||||||
|
} catch (error) {
|
||||||
|
consecutiveErrors++;
|
||||||
|
if (consecutiveErrors >= MAX_CONSECUTIVE_ERRORS) {
|
||||||
|
globalCooldownUntil = Date.now() + 60000;
|
||||||
|
logger.warn("Global circuit breaker triggered due to consecutive errors");
|
||||||
|
}
|
||||||
|
|
||||||
|
// Unhandled exception — route everything to individual fallback.
|
||||||
|
logger.warn(
|
||||||
|
{ conversationKey, messageCount: messages.length },
|
||||||
|
"Batch threw exception — routing all messages to individual fallback queue",
|
||||||
|
);
|
||||||
|
enqueueIndividualFallbacks(messages);
|
||||||
|
|
||||||
|
lastError = error instanceof Error ? error.message : String(error);
|
||||||
|
const errorStack = error instanceof Error ? error.stack : undefined;
|
||||||
|
conversationErrorCooldown.set(
|
||||||
|
conversationKey,
|
||||||
|
Date.now() + config.AI_ANALYSIS_ERROR_COOLDOWN_MS,
|
||||||
|
);
|
||||||
|
logger.error(
|
||||||
|
{
|
||||||
|
conversationKey,
|
||||||
|
error: lastError,
|
||||||
|
stack: errorStack,
|
||||||
|
messageCount: messages.length,
|
||||||
|
messageIds: messages.map((m) => m.id),
|
||||||
|
cooldownUntil: new Date(
|
||||||
|
Date.now() + config.AI_ANALYSIS_ERROR_COOLDOWN_MS,
|
||||||
|
).toISOString(),
|
||||||
|
timestamp: new Date().toISOString(),
|
||||||
|
},
|
||||||
|
"Analysis worker failed, will retry after cooldown",
|
||||||
|
);
|
||||||
|
} finally {
|
||||||
|
activeRequests--;
|
||||||
|
if (conversationProcessing.get(conversationKey) === processingStartedAt) {
|
||||||
|
conversationProcessing.delete(conversationKey);
|
||||||
|
}
|
||||||
|
if (shouldScheduleNext) {
|
||||||
|
setImmediate(() => scheduleConversationAnalysis(conversationKey));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Scheduling
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Schedules a debounced analysis run for a conversation.
|
||||||
|
*
|
||||||
|
* FIX #3: The async work inside setTimeout is now wrapped in an explicit
|
||||||
|
* .catch() so DB errors don't produce unhandled promise rejections.
|
||||||
|
* FIX #6: Calls pickBatchWithinBudget after fetching messages so token budget
|
||||||
|
* is respected before handing the batch to the LLM.
|
||||||
|
*/
|
||||||
|
function scheduleConversationAnalysis(conversationKey: string): void {
|
||||||
|
if (isConversationProcessingLocked(conversationKey)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const convoCooldown = conversationErrorCooldown.get(conversationKey) || 0;
|
||||||
|
const activeCooldown = Math.max(convoCooldown, globalCooldownUntil);
|
||||||
|
|
||||||
|
if (activeCooldown && Date.now() < activeCooldown) {
|
||||||
|
if (!conversationDebounceTimers.has(conversationKey)) {
|
||||||
|
const remaining = activeCooldown - Date.now();
|
||||||
|
const timer = setTimeout(() => {
|
||||||
|
conversationDebounceTimers.delete(conversationKey);
|
||||||
|
scheduleConversationAnalysis(conversationKey);
|
||||||
|
}, remaining + 500);
|
||||||
|
conversationDebounceTimers.set(conversationKey, timer);
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const existingTimer = conversationDebounceTimers.get(conversationKey);
|
||||||
|
if (existingTimer) {
|
||||||
|
clearTimeout(existingTimer);
|
||||||
|
}
|
||||||
|
|
||||||
|
const timer = setTimeout(() => {
|
||||||
|
conversationDebounceTimers.delete(conversationKey);
|
||||||
|
|
||||||
|
// FIX #3: explicit .catch() — no async arrow function to avoid unhandled rejection.
|
||||||
|
getPendingMessagesByConversation(
|
||||||
|
conversationKey,
|
||||||
|
config.AI_ANALYSIS_MAX_BATCH_SIZE,
|
||||||
|
)
|
||||||
|
.then(async (messages) => {
|
||||||
|
if (messages.length === 0) return;
|
||||||
|
|
||||||
|
const processableMessages = await skipAgeRestrictedMessages(messages);
|
||||||
|
if (processableMessages.length === 0) return;
|
||||||
|
|
||||||
|
// FIX #6: trim to token budget before sending to LLM.
|
||||||
|
// 50 tokens overhead accounts for JSON structure + id/username fields.
|
||||||
|
let trimmed = pickBatchWithinBudget(
|
||||||
|
processableMessages,
|
||||||
|
config.AI_ANALYSIS_MAX_TARGET_TOKENS,
|
||||||
|
50,
|
||||||
|
);
|
||||||
|
|
||||||
|
// FIX #10: if every message individually exceeds the token budget,
|
||||||
|
// pickBatchWithinBudget returns [] — which would leave them permanently
|
||||||
|
// stuck as `pending`. Fall back to the first message alone so at
|
||||||
|
// least one makes progress; the rest will be processed in later ticks.
|
||||||
|
if (trimmed.length === 0 && processableMessages.length > 0) {
|
||||||
|
trimmed = processableMessages.slice(0, 1);
|
||||||
|
logger.warn(
|
||||||
|
{
|
||||||
|
conversationKey,
|
||||||
|
messageId: processableMessages[0]?.id,
|
||||||
|
tokenBudget: config.AI_ANALYSIS_MAX_TARGET_TOKENS,
|
||||||
|
},
|
||||||
|
"All messages exceed token budget — processing first message alone to avoid stuck-pending deadlock",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
return processBatch(conversationKey, trimmed);
|
||||||
|
})
|
||||||
|
.catch((err: unknown) => {
|
||||||
|
logger.error(
|
||||||
|
{
|
||||||
|
conversationKey,
|
||||||
|
error: err instanceof Error ? err.message : String(err),
|
||||||
|
},
|
||||||
|
"Failed to fetch or dispatch pending messages for scheduled analysis",
|
||||||
|
);
|
||||||
|
});
|
||||||
|
}, config.AI_ANALYSIS_DEBOUNCE_MS);
|
||||||
|
|
||||||
|
conversationDebounceTimers.set(conversationKey, timer);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Public API
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Queues a message for analysis (debounced by conversation).
|
||||||
|
*/
|
||||||
|
export async function queueMessageAnalysis(messageId: string): Promise<void> {
|
||||||
|
if (!config.AI_ANALYSIS_ENABLED) return;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const message = await getMessageById(messageId);
|
||||||
|
if (!message) {
|
||||||
|
logger.warn({ messageId }, "Message not found for analysis queue");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (isAgeRestrictedMessage(message)) {
|
||||||
|
const updated = await updateMessageAIAnalysis(
|
||||||
|
message.id,
|
||||||
|
buildAgeRestrictedSkipResult(),
|
||||||
|
);
|
||||||
|
if (updated) {
|
||||||
|
getModerationBroadcaster()?.messageAnalyzed(updated);
|
||||||
|
}
|
||||||
|
logger.info(
|
||||||
|
{ messageId },
|
||||||
|
"Skipped AI analysis for age-restricted message",
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
queueConversationAnalysis(getConversationKey(message));
|
||||||
|
} catch (error) {
|
||||||
|
logger.error(
|
||||||
|
{
|
||||||
|
messageId,
|
||||||
|
error: error instanceof Error ? error.message : String(error),
|
||||||
|
},
|
||||||
|
"Failed to queue message for analysis",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Queues a conversation for analysis (debounced).
|
||||||
|
*/
|
||||||
|
export function queueConversationAnalysis(conversationKey: string): void {
|
||||||
|
if (!config.AI_ANALYSIS_ENABLED) return;
|
||||||
|
scheduleConversationAnalysis(conversationKey);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns current status of both the batch and individual fallback queues.
|
||||||
|
*/
|
||||||
|
export function getAnalysisQueueStatus(): AnalysisQueueStatus {
|
||||||
|
return {
|
||||||
|
queuedConversations: conversationDebounceTimers.size,
|
||||||
|
activeRequests,
|
||||||
|
activeIndividualRequests,
|
||||||
|
individualInFlightCount: individualInFlight.size,
|
||||||
|
individualCircuitBreakerActive: Date.now() < individualCooldownUntil,
|
||||||
|
lastError,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Starts the periodic recovery worker.
|
||||||
|
*
|
||||||
|
* FIX #4: Now also recovers messages stuck in `error/analysis_incomplete`
|
||||||
|
* state (not just `pending`), and skips conversations that already have
|
||||||
|
* individual fallback work in progress to avoid DB last-write-wins races.
|
||||||
|
*/
|
||||||
|
export function startPendingAIAnalysisWorker(client?: Client): void {
|
||||||
|
moderationClient = client;
|
||||||
|
if (!config.AI_ANALYSIS_ENABLED) return;
|
||||||
|
|
||||||
|
setInterval(() => {
|
||||||
|
// FIX #3 pattern: no async arrow — chain promises explicitly.
|
||||||
|
Promise.all([
|
||||||
|
getPendingConversationKeys(500),
|
||||||
|
getConversationKeysWithIncompleteAnalysis(200),
|
||||||
|
])
|
||||||
|
.then(([pendingKeys, incompleteKeys]) => {
|
||||||
|
const now = Date.now();
|
||||||
|
|
||||||
|
// FIX #9: Prune stale entries from state maps to prevent unbounded
|
||||||
|
// memory growth from channels/threads that are no longer active.
|
||||||
|
for (const [key, expiry] of conversationErrorCooldown) {
|
||||||
|
if (now >= expiry) conversationErrorCooldown.delete(key);
|
||||||
|
}
|
||||||
|
for (const [key, startedAt] of conversationProcessing) {
|
||||||
|
if (now - startedAt >= config.AI_ANALYSIS_PROCESSING_TIMEOUT_MS) {
|
||||||
|
conversationProcessing.delete(key);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// FIX #8: Build a set of keys already targeted for individual recovery
|
||||||
|
// so the batch loop below skips them, preventing a race where batch
|
||||||
|
// scheduling and individual scheduling collide on the same conversation.
|
||||||
|
const incompleteKeySet = new Set(incompleteKeys);
|
||||||
|
|
||||||
|
// --- Batch recovery for `pending` messages ---
|
||||||
|
for (const key of pendingKeys) {
|
||||||
|
if (conversationDebounceTimers.has(key)) continue;
|
||||||
|
if (isConversationProcessingLocked(key)) continue;
|
||||||
|
// FIX #4: skip if individual fallback already running for this conversation.
|
||||||
|
if (individualInFlightByConversation.has(key)) continue;
|
||||||
|
// FIX #8: skip if this conversation also needs individual recovery
|
||||||
|
// (batch processing would conflict with in-flight individual work).
|
||||||
|
if (incompleteKeySet.has(key)) continue;
|
||||||
|
const cooldownUntil = conversationErrorCooldown.get(key);
|
||||||
|
if (cooldownUntil && now < cooldownUntil) continue;
|
||||||
|
scheduleConversationAnalysis(key);
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- Individual recovery for `error/analysis_incomplete` messages ---
|
||||||
|
// Circuit breaker check: no point iterating if individual CB is active.
|
||||||
|
if (now >= individualCooldownUntil) {
|
||||||
|
const promises: Promise<void>[] = [];
|
||||||
|
for (const key of incompleteKeys) {
|
||||||
|
// Skip if individual work is already running for this conversation.
|
||||||
|
if (individualInFlightByConversation.has(key)) continue;
|
||||||
|
// Skip if batch processing is running (it will fan-out if it finds more incomplete).
|
||||||
|
if (isConversationProcessingLocked(key)) continue;
|
||||||
|
|
||||||
|
promises.push(
|
||||||
|
getIncompleteMessagesByConversation(key, 500)
|
||||||
|
.then(async (msgs) => {
|
||||||
|
const processableMessages =
|
||||||
|
await skipAgeRestrictedMessages(msgs);
|
||||||
|
return processableMessages;
|
||||||
|
})
|
||||||
|
.then((msgs) => {
|
||||||
|
if (msgs.length > 0) {
|
||||||
|
enqueueIndividualFallbacks(msgs);
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.catch((err: unknown) => {
|
||||||
|
logger.error(
|
||||||
|
{ key, error: String(err) },
|
||||||
|
"Failed to fetch incomplete messages for recovery",
|
||||||
|
);
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
// Errors are handled per-key; return the combined promise for observability.
|
||||||
|
return Promise.all(promises);
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.catch((err: unknown) => {
|
||||||
|
logger.error(
|
||||||
|
{ error: err instanceof Error ? err.message : String(err) },
|
||||||
|
"Pending AI analysis recovery worker failed",
|
||||||
|
);
|
||||||
|
});
|
||||||
|
}, config.AI_ANALYSIS_RECOVERY_INTERVAL_MS);
|
||||||
|
}
|
||||||
@@ -0,0 +1,355 @@
|
|||||||
|
import type { Client, PermissionString } from "discord.js-selfbot-v13";
|
||||||
|
import { config } from "../../shared/config/config.js";
|
||||||
|
import { createChildLogger } from "../../shared/logger/logger.js";
|
||||||
|
import { createModerationAction } from "../message-capture/messageStore.js";
|
||||||
|
import type { MessageRecord } from "../message-capture/types.js";
|
||||||
|
|
||||||
|
const logger = createChildLogger("auto-delete-manager");
|
||||||
|
|
||||||
|
const parseStringList = (value?: string | null): string[] => {
|
||||||
|
if (!value) return [];
|
||||||
|
try {
|
||||||
|
const parsed = JSON.parse(value) as unknown;
|
||||||
|
return Array.isArray(parsed)
|
||||||
|
? parsed.filter((item): item is string => typeof item === "string")
|
||||||
|
: [];
|
||||||
|
} catch {
|
||||||
|
return value
|
||||||
|
.split(",")
|
||||||
|
.map((item) => item.trim())
|
||||||
|
.filter(Boolean);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
/** Derive severity from legacy messages that lack structured AI fields. */
|
||||||
|
function deriveSeverity(msg: MessageRecord): string {
|
||||||
|
if (msg.ai_severity) return msg.ai_severity;
|
||||||
|
const score = msg.ai_confidence ?? msg.ai_moderation_score ?? 0;
|
||||||
|
if (msg.ai_status === "flagged")
|
||||||
|
return score >= 0.9 ? "critical" : score >= 0.7 ? "high" : "medium";
|
||||||
|
if (msg.ai_status === "warn") return score >= 0.6 ? "medium" : "low";
|
||||||
|
return "none";
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Derive recommended action from legacy messages that lack structured AI fields. */
|
||||||
|
function deriveRecommendedAction(msg: MessageRecord): string {
|
||||||
|
if (msg.ai_recommended_action) return msg.ai_recommended_action;
|
||||||
|
const severity = deriveSeverity(msg);
|
||||||
|
if (
|
||||||
|
msg.ai_status === "flagged" &&
|
||||||
|
(severity === "critical" || severity === "high")
|
||||||
|
)
|
||||||
|
return "delete";
|
||||||
|
if (msg.ai_status === "flagged") return "review";
|
||||||
|
if (msg.ai_status === "warn") return "warn";
|
||||||
|
return "none";
|
||||||
|
}
|
||||||
|
|
||||||
|
function isAutoDeleteEligible(message: MessageRecord): boolean {
|
||||||
|
if (message.ai_status !== "flagged" && message.ai_status !== "warn")
|
||||||
|
return false;
|
||||||
|
|
||||||
|
const confidence = message.ai_confidence ?? message.ai_moderation_score ?? 0;
|
||||||
|
if (confidence < config.AUTO_DELETE_MIN_CONFIDENCE) {
|
||||||
|
logger.info(
|
||||||
|
{
|
||||||
|
messageId: message.id,
|
||||||
|
confidence,
|
||||||
|
threshold: config.AUTO_DELETE_MIN_CONFIDENCE,
|
||||||
|
},
|
||||||
|
"Auto-delete skipped: confidence below threshold",
|
||||||
|
);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
const severity = deriveSeverity(message);
|
||||||
|
const allowedSeverities = (config.AUTO_DELETE_ALLOWED_SEVERITIES || "")
|
||||||
|
.split(",")
|
||||||
|
.map((s) => s.trim())
|
||||||
|
.filter(Boolean);
|
||||||
|
if (allowedSeverities.length > 0 && !allowedSeverities.includes(severity)) {
|
||||||
|
logger.info(
|
||||||
|
{ messageId: message.id, severity, allowed: allowedSeverities },
|
||||||
|
"Auto-delete skipped: severity not in allowed list",
|
||||||
|
);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
const recommendedAction = deriveRecommendedAction(message);
|
||||||
|
if (recommendedAction !== "delete" && recommendedAction !== "escalate") {
|
||||||
|
logger.info(
|
||||||
|
{ messageId: message.id, recommendedAction },
|
||||||
|
"Auto-delete skipped: recommended action is not delete/escalate",
|
||||||
|
);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
const allowedCategories = parseStringList(
|
||||||
|
config.AUTO_DELETE_ALLOWED_CATEGORIES,
|
||||||
|
);
|
||||||
|
if (allowedCategories.length > 0) {
|
||||||
|
const messageCategories = parseStringList(
|
||||||
|
message.ai_categories ?? message.ai_moderation_flags,
|
||||||
|
);
|
||||||
|
const hasAllowedCategory = messageCategories.some((cat) =>
|
||||||
|
allowedCategories.includes(cat),
|
||||||
|
);
|
||||||
|
if (!hasAllowedCategory) {
|
||||||
|
logger.info(
|
||||||
|
{
|
||||||
|
messageId: message.id,
|
||||||
|
categories: messageCategories,
|
||||||
|
allowed: allowedCategories,
|
||||||
|
},
|
||||||
|
"Auto-delete skipped: no allowed categories match",
|
||||||
|
);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const excludedChannels = parseStringList(
|
||||||
|
config.AUTO_DELETE_EXCLUDED_CHANNEL_IDS,
|
||||||
|
);
|
||||||
|
if (excludedChannels.length > 0) {
|
||||||
|
const channelId = message.thread_id ?? message.channel_id;
|
||||||
|
if (excludedChannels.includes(channelId)) {
|
||||||
|
logger.info(
|
||||||
|
{ messageId: message.id, channelId },
|
||||||
|
"Auto-delete skipped: channel excluded",
|
||||||
|
);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const excludedUsers = parseStringList(config.AUTO_DELETE_EXCLUDED_USER_IDS);
|
||||||
|
if (excludedUsers.length > 0 && excludedUsers.includes(message.user_id)) {
|
||||||
|
logger.info(
|
||||||
|
{ messageId: message.id, userId: message.user_id },
|
||||||
|
"Auto-delete skipped: user excluded",
|
||||||
|
);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function logAutoDeleteAttempt(
|
||||||
|
message: MessageRecord,
|
||||||
|
result: AutoDeleteResult,
|
||||||
|
): Promise<void> {
|
||||||
|
try {
|
||||||
|
await createModerationAction({
|
||||||
|
message_id: message.id,
|
||||||
|
user_id: message.user_id,
|
||||||
|
guild_id: message.guild_id,
|
||||||
|
action_type: "delete_message",
|
||||||
|
reason: result.reason,
|
||||||
|
executed_by: "auto-delete-manager",
|
||||||
|
status: result.deleted
|
||||||
|
? "executed"
|
||||||
|
: result.reason === "dry_run"
|
||||||
|
? "executed"
|
||||||
|
: "failed",
|
||||||
|
error: result.reason === "error" ? result.reason : null,
|
||||||
|
executed_at:
|
||||||
|
result.deleted || result.reason === "dry_run" ? Date.now() : null,
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
logger.warn(
|
||||||
|
{
|
||||||
|
messageId: message.id,
|
||||||
|
error: error instanceof Error ? error.message : String(error),
|
||||||
|
},
|
||||||
|
"Failed to persist auto-delete action log",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface AutoDeleteResult {
|
||||||
|
deleted: boolean;
|
||||||
|
skipped: boolean;
|
||||||
|
reason: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
function getErrorCode(error: unknown): number | string | undefined {
|
||||||
|
if (!error || typeof error !== "object") return undefined;
|
||||||
|
const maybeCode = (error as { code?: number | string }).code;
|
||||||
|
const maybeStatus = (error as { status?: number | string }).status;
|
||||||
|
return maybeCode ?? maybeStatus;
|
||||||
|
}
|
||||||
|
|
||||||
|
function isAlreadyDeletedError(error: unknown): boolean {
|
||||||
|
const code = getErrorCode(error);
|
||||||
|
return code === 10008 || code === 404 || code === "10008" || code === "404";
|
||||||
|
}
|
||||||
|
|
||||||
|
function hasChannelMessagesApi(channel: unknown): channel is {
|
||||||
|
messages: {
|
||||||
|
fetch: (id: string) => Promise<{ delete: () => Promise<unknown> }>;
|
||||||
|
};
|
||||||
|
} {
|
||||||
|
return Boolean(
|
||||||
|
channel &&
|
||||||
|
typeof channel === "object" &&
|
||||||
|
"messages" in channel &&
|
||||||
|
(channel as { messages?: unknown }).messages &&
|
||||||
|
typeof (channel as { messages: { fetch?: unknown } }).messages.fetch ===
|
||||||
|
"function",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function hasPermissionApi(channel: unknown): channel is {
|
||||||
|
permissionsFor: (
|
||||||
|
member: unknown,
|
||||||
|
) => { has: (permission: string) => boolean } | null;
|
||||||
|
} {
|
||||||
|
return Boolean(
|
||||||
|
channel &&
|
||||||
|
typeof channel === "object" &&
|
||||||
|
"permissionsFor" in channel &&
|
||||||
|
typeof (channel as { permissionsFor?: unknown }).permissionsFor ===
|
||||||
|
"function",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function attemptAutoDeleteFlaggedMessage(
|
||||||
|
client: Client | undefined,
|
||||||
|
message: MessageRecord,
|
||||||
|
): Promise<AutoDeleteResult> {
|
||||||
|
if (!config.AUTO_DELETE_FLAGGED_ENABLED) {
|
||||||
|
return { deleted: false, skipped: true, reason: "disabled" };
|
||||||
|
}
|
||||||
|
|
||||||
|
if (message.ai_status !== "flagged" && message.ai_status !== "warn") {
|
||||||
|
const result = {
|
||||||
|
deleted: false,
|
||||||
|
skipped: true,
|
||||||
|
reason: "not_flagged_or_warn",
|
||||||
|
} as AutoDeleteResult;
|
||||||
|
await logAutoDeleteAttempt(message, result);
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!isAutoDeleteEligible(message)) {
|
||||||
|
const result = {
|
||||||
|
deleted: false,
|
||||||
|
skipped: true,
|
||||||
|
reason: "not_eligible",
|
||||||
|
} as AutoDeleteResult;
|
||||||
|
await logAutoDeleteAttempt(message, result);
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!client?.user?.id) {
|
||||||
|
logger.warn(
|
||||||
|
{ messageId: message.id },
|
||||||
|
"Auto-delete skipped: client user missing",
|
||||||
|
);
|
||||||
|
return { deleted: false, skipped: true, reason: "client_user_missing" };
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const guild = client.guilds.cache.get(message.guild_id);
|
||||||
|
if (!guild) {
|
||||||
|
logger.warn(
|
||||||
|
{ messageId: message.id, guildId: message.guild_id },
|
||||||
|
"Auto-delete skipped: guild not found",
|
||||||
|
);
|
||||||
|
return { deleted: false, skipped: true, reason: "guild_not_found" };
|
||||||
|
}
|
||||||
|
|
||||||
|
const channelId = message.thread_id ?? message.channel_id;
|
||||||
|
const channel = guild.channels.cache.get(channelId);
|
||||||
|
if (!channel) {
|
||||||
|
logger.warn(
|
||||||
|
{ messageId: message.id, channelId },
|
||||||
|
"Auto-delete skipped: channel not found",
|
||||||
|
);
|
||||||
|
return { deleted: false, skipped: true, reason: "channel_not_found" };
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!hasPermissionApi(channel) || !hasChannelMessagesApi(channel)) {
|
||||||
|
logger.warn(
|
||||||
|
{ messageId: message.id, channelId },
|
||||||
|
"Auto-delete skipped: channel cannot delete messages",
|
||||||
|
);
|
||||||
|
return { deleted: false, skipped: true, reason: "unsupported_channel" };
|
||||||
|
}
|
||||||
|
|
||||||
|
const selfMember = await guild.members.fetch(client.user.id);
|
||||||
|
const permissions = channel.permissionsFor(selfMember);
|
||||||
|
const canManageMessages =
|
||||||
|
permissions?.has("MANAGE_MESSAGES" as PermissionString) ?? false;
|
||||||
|
|
||||||
|
if (!canManageMessages) {
|
||||||
|
logger.warn(
|
||||||
|
{ messageId: message.id, channelId, userId: client.user.id },
|
||||||
|
"Auto-delete skipped: current user lacks Manage Messages",
|
||||||
|
);
|
||||||
|
return {
|
||||||
|
deleted: false,
|
||||||
|
skipped: true,
|
||||||
|
reason: "missing_manage_messages",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
if (config.AUTO_DELETE_FLAGGED_DRY_RUN) {
|
||||||
|
const result = {
|
||||||
|
deleted: false,
|
||||||
|
skipped: true,
|
||||||
|
reason: "dry_run",
|
||||||
|
} as AutoDeleteResult;
|
||||||
|
await logAutoDeleteAttempt(message, result);
|
||||||
|
logger.info(
|
||||||
|
{ messageId: message.id, channelId },
|
||||||
|
"Auto-delete dry-run: would delete flagged message",
|
||||||
|
);
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
const discordMessage = await channel.messages.fetch(message.id);
|
||||||
|
await discordMessage.delete();
|
||||||
|
|
||||||
|
const result = {
|
||||||
|
deleted: true,
|
||||||
|
skipped: false,
|
||||||
|
reason: "deleted",
|
||||||
|
} as AutoDeleteResult;
|
||||||
|
await logAutoDeleteAttempt(message, result);
|
||||||
|
logger.info(
|
||||||
|
{ messageId: message.id, channelId },
|
||||||
|
"Auto-deleted AI-flagged message",
|
||||||
|
);
|
||||||
|
return result;
|
||||||
|
} catch (error) {
|
||||||
|
if (isAlreadyDeletedError(error)) {
|
||||||
|
const result = {
|
||||||
|
deleted: true,
|
||||||
|
skipped: false,
|
||||||
|
reason: "already_deleted",
|
||||||
|
} as AutoDeleteResult;
|
||||||
|
await logAutoDeleteAttempt(message, result);
|
||||||
|
logger.info(
|
||||||
|
{ messageId: message.id, code: getErrorCode(error) },
|
||||||
|
"Auto-delete skipped: message already deleted",
|
||||||
|
);
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
const result = {
|
||||||
|
deleted: false,
|
||||||
|
skipped: true,
|
||||||
|
reason: "error",
|
||||||
|
} as AutoDeleteResult;
|
||||||
|
await logAutoDeleteAttempt(message, result);
|
||||||
|
logger.error(
|
||||||
|
{
|
||||||
|
messageId: message.id,
|
||||||
|
error: error instanceof Error ? error.message : String(error),
|
||||||
|
code: getErrorCode(error),
|
||||||
|
},
|
||||||
|
"Auto-delete failed",
|
||||||
|
);
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,14 @@
|
|||||||
|
import pLimit from "p-limit";
|
||||||
|
import { config } from "../../shared/config/config.js";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Concurrency limiter for LLM API calls.
|
||||||
|
*
|
||||||
|
* Prevents rate-limit (429) errors by capping simultaneous requests
|
||||||
|
* to the configured maximum (default: 5).
|
||||||
|
*/
|
||||||
|
const llmSemaphore = pLimit(config.AI_LLM_MAX_CONCURRENT ?? 5);
|
||||||
|
|
||||||
|
export async function withLlmConcurrency<T>(fn: () => Promise<T>): Promise<T> {
|
||||||
|
return llmSemaphore(fn);
|
||||||
|
}
|
||||||
@@ -0,0 +1,77 @@
|
|||||||
|
import { formatModerationTextEvidenceForPrompt } from "./indonesianTextNormalizer.js";
|
||||||
|
import { formatMediaEvidenceForPrompt } from "../message-capture/messageMetadata.js";
|
||||||
|
import type { MessageRecord } from "../message-capture/types.js";
|
||||||
|
|
||||||
|
export interface ConversationContextInput {
|
||||||
|
contextBefore: MessageRecord[];
|
||||||
|
targets: MessageRecord[];
|
||||||
|
maxTokens: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Formats a timestamp to ISO 8601 string
|
||||||
|
*/
|
||||||
|
function formatTimestamp(ms: number): string {
|
||||||
|
return new Date(ms).toISOString();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Estimates token count for a string (pessimistic approximation for Indonesian slang & JSON overhead)
|
||||||
|
*/
|
||||||
|
export function estimateTokens(text: string): number {
|
||||||
|
return Math.ceil(text.length / 3) + 15;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Formats a single message for context or target display
|
||||||
|
*/
|
||||||
|
export async function formatMessageForPrompt(
|
||||||
|
msg: MessageRecord,
|
||||||
|
label: "context" | "target",
|
||||||
|
): Promise<string> {
|
||||||
|
const content = msg.edited_content ?? msg.content;
|
||||||
|
const timestamp = formatTimestamp(msg.created_at);
|
||||||
|
const textEvidence = await formatModerationTextEvidenceForPrompt(content);
|
||||||
|
const textSuffix = textEvidence ? ` ${textEvidence}` : "";
|
||||||
|
const mediaEvidence = formatMediaEvidenceForPrompt(msg.metadata);
|
||||||
|
const mediaSuffix = mediaEvidence ? ` ${mediaEvidence}` : "";
|
||||||
|
return `[${label}] id=${msg.id} time=${timestamp} user=${msg.username}: ${content}${textSuffix}${mediaSuffix}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Builds conversation historical context without including targets.
|
||||||
|
* Calculates how much token budget targets use, and fills the rest with context.
|
||||||
|
*/
|
||||||
|
export async function buildConversationContext(
|
||||||
|
input: ConversationContextInput,
|
||||||
|
): Promise<string[]> {
|
||||||
|
const { contextBefore, targets, maxTokens } = input;
|
||||||
|
|
||||||
|
// Calculate tokens used by targets (parallel)
|
||||||
|
const targetLines = await Promise.all(
|
||||||
|
targets.map((msg) => formatMessageForPrompt(msg, "target")),
|
||||||
|
);
|
||||||
|
let usedTokens = targetLines.reduce(
|
||||||
|
(sum, line) => sum + estimateTokens(line),
|
||||||
|
0,
|
||||||
|
);
|
||||||
|
|
||||||
|
const contextLines = await Promise.all(
|
||||||
|
contextBefore.map((msg) => formatMessageForPrompt(msg, "context")),
|
||||||
|
);
|
||||||
|
const selectedContextLines: string[] = [];
|
||||||
|
|
||||||
|
// Go backwards through context, taking most recent first
|
||||||
|
for (let i = contextLines.length - 1; i >= 0; i--) {
|
||||||
|
const line = contextLines[i];
|
||||||
|
const lineTokens = estimateTokens(line);
|
||||||
|
|
||||||
|
if (usedTokens + lineTokens <= maxTokens) {
|
||||||
|
// Unshift so oldest context is first in the array
|
||||||
|
selectedContextLines.unshift(line);
|
||||||
|
usedTokens += lineTokens;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return selectedContextLines;
|
||||||
|
}
|
||||||
@@ -0,0 +1,8 @@
|
|||||||
|
export { startPendingAIAnalysisWorker } from "./aiAnalyzer.js";
|
||||||
|
export {
|
||||||
|
normalizeDiscordCustomEmoji,
|
||||||
|
detectIndonesianBadwords,
|
||||||
|
buildModerationTextEvidence,
|
||||||
|
} from "./indonesianTextNormalizer.js";
|
||||||
|
export { runModerationAnalysis } from "./llmModerationClient.js";
|
||||||
|
export { buildSystemPrompt } from "./moderationPrompt.js";
|
||||||
@@ -0,0 +1,606 @@
|
|||||||
|
import axios from "axios";
|
||||||
|
import OpenAI from "openai";
|
||||||
|
import { config } from "../../shared/config/config.js";
|
||||||
|
import { createChildLogger } from "../../shared/logger/logger.js";
|
||||||
|
import { retryWithBackoff } from "../../shared/utils/retry.js";
|
||||||
|
import { getCachedText, upsertCachedText } from "./textCacheStore.js";
|
||||||
|
|
||||||
|
const log = createChildLogger("indonesianTextNormalizer");
|
||||||
|
|
||||||
|
const CUSTOM_EMOJI_PATTERN = /<a?:([a-zA-Z0-9_]+):(\d+)>/g;
|
||||||
|
|
||||||
|
/** NVIDIA content safety categories that map to offensive/badword content. */
|
||||||
|
const NVIDIA_BAD_CATEGORIES = new Set([
|
||||||
|
"hate",
|
||||||
|
"harassment",
|
||||||
|
"sexual",
|
||||||
|
"violence",
|
||||||
|
"self-harm",
|
||||||
|
"illicit",
|
||||||
|
"profanity",
|
||||||
|
"vulgar",
|
||||||
|
"insult",
|
||||||
|
]);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Map NVIDIA Nemotron category labels to Indonesian badword-style labels.
|
||||||
|
*/
|
||||||
|
const CATEGORY_TO_BADWORD_LABEL: Record<string, string> = {
|
||||||
|
hate: "hate_speech",
|
||||||
|
harassment: "harassment",
|
||||||
|
sexual: "sexual_content",
|
||||||
|
violence: "violence",
|
||||||
|
"self-harm": "self_harm",
|
||||||
|
illicit: "illegal_content",
|
||||||
|
profanity: "vulgar_language",
|
||||||
|
vulgar: "vulgar_language",
|
||||||
|
insult: "harassment",
|
||||||
|
};
|
||||||
|
|
||||||
|
const VALID_PRIMARY_AI_FLAGS = new Set([
|
||||||
|
"spam",
|
||||||
|
"hate_speech",
|
||||||
|
"sara",
|
||||||
|
"hoaks",
|
||||||
|
"harassment",
|
||||||
|
"vulgar_language",
|
||||||
|
"sexual_content",
|
||||||
|
"sexual_deviation",
|
||||||
|
"violence",
|
||||||
|
"self_harm",
|
||||||
|
"doxxing",
|
||||||
|
"scam",
|
||||||
|
"misinformation",
|
||||||
|
"nsfw_image",
|
||||||
|
"gore_image",
|
||||||
|
"illegal_content",
|
||||||
|
"gambling",
|
||||||
|
"drugs",
|
||||||
|
"child_safety",
|
||||||
|
"financial_scam",
|
||||||
|
"religious_insult",
|
||||||
|
"self_promo",
|
||||||
|
]);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* In-memory cache TTL (10 min) — fastest path for repeated identical texts.
|
||||||
|
*/
|
||||||
|
const BADWORD_CACHE_TTL_MS = 10 * 60 * 1000;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* DB cache TTL (24 hours) — survives restarts, stores full-text results
|
||||||
|
* so context is preserved (e.g. "kaus" is clean, "kau" alone is clean,
|
||||||
|
* but "awas kau" is harassment).
|
||||||
|
*/
|
||||||
|
const DB_CACHE_TTL_MS = 24 * 60 * 60 * 1000;
|
||||||
|
|
||||||
|
const NEMOTRON_RATE_LIMIT_COOLDOWN_MS = 60 * 1000;
|
||||||
|
const PRIMARY_AI_RATE_LIMIT_COOLDOWN_MS = 30_000;
|
||||||
|
const GROQ_RATE_LIMIT_COOLDOWN_MS = 60 * 1000;
|
||||||
|
|
||||||
|
interface BadwordCacheEntry {
|
||||||
|
value: string[];
|
||||||
|
expiresAt: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
const badwordCache = new Map<string, BadwordCacheEntry>();
|
||||||
|
const inFlightBadwordLookups = new Map<string, Promise<string[]>>();
|
||||||
|
let nemotronUnavailableUntil = 0;
|
||||||
|
let primaryAiUnavailableUntil = 0;
|
||||||
|
let groqUnavailableUntil = 0;
|
||||||
|
let primaryModerationClient: OpenAI | null = null;
|
||||||
|
|
||||||
|
export interface ModerationTextEvidence {
|
||||||
|
raw: string;
|
||||||
|
normalized: string;
|
||||||
|
notes: string[];
|
||||||
|
badwords: string[];
|
||||||
|
hasBadwords: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Sync helpers (unchanged)
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
export function normalizeDiscordCustomEmoji(text: string): {
|
||||||
|
text: string;
|
||||||
|
emojiNames: string[];
|
||||||
|
} {
|
||||||
|
const emojiNames: string[] = [];
|
||||||
|
const normalized = text.replace(
|
||||||
|
CUSTOM_EMOJI_PATTERN,
|
||||||
|
(_match, name: string) => {
|
||||||
|
emojiNames.push(name);
|
||||||
|
return `[emoji:${name}]`;
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
return { text: normalized, emojiNames };
|
||||||
|
}
|
||||||
|
|
||||||
|
// Local badword detection removed (lines 121-198).
|
||||||
|
// All detection now goes through the API pipeline (NVIDIA → Primary AI → Groq)
|
||||||
|
// to eliminate false positives from substring matching and hardcoded whitelists.
|
||||||
|
|
||||||
|
function normalizeBadwordCacheKey(text: string): string {
|
||||||
|
return text.trim().replace(/\s+/g, " ").toLowerCase();
|
||||||
|
}
|
||||||
|
|
||||||
|
function getCachedBadwords(key: string): string[] | null {
|
||||||
|
const entry = badwordCache.get(key);
|
||||||
|
if (!entry) return null;
|
||||||
|
if (entry.expiresAt <= Date.now()) {
|
||||||
|
badwordCache.delete(key);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return [...entry.value];
|
||||||
|
}
|
||||||
|
|
||||||
|
function setCachedBadwords(key: string, value: string[]): void {
|
||||||
|
badwordCache.set(key, {
|
||||||
|
value: [...new Set(value)],
|
||||||
|
expiresAt: Date.now() + BADWORD_CACHE_TTL_MS,
|
||||||
|
});
|
||||||
|
|
||||||
|
if (badwordCache.size > 500) {
|
||||||
|
const now = Date.now();
|
||||||
|
for (const [cacheKey, entry] of badwordCache) {
|
||||||
|
if (entry.expiresAt <= now) {
|
||||||
|
badwordCache.delete(cacheKey);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (badwordCache.size > 500) {
|
||||||
|
const oldestKeys = Array.from(badwordCache.entries())
|
||||||
|
.sort((a, b) => a[1].expiresAt - b[1].expiresAt)
|
||||||
|
.slice(0, badwordCache.size - 500)
|
||||||
|
.map(([cacheKey]) => cacheKey);
|
||||||
|
for (const cacheKey of oldestKeys) {
|
||||||
|
badwordCache.delete(cacheKey);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function getPrimaryModerationClient(): OpenAI | null {
|
||||||
|
if (!config.AI_LLM_API_KEY) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!primaryModerationClient) {
|
||||||
|
primaryModerationClient = new OpenAI({
|
||||||
|
apiKey: config.AI_LLM_API_KEY,
|
||||||
|
baseURL: config.AI_LLM_BASE_URL,
|
||||||
|
maxRetries: 0,
|
||||||
|
timeout: 15000,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
return primaryModerationClient;
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizePrimaryAiFlag(value: string): string | null {
|
||||||
|
const lower = value
|
||||||
|
.trim()
|
||||||
|
.toLowerCase()
|
||||||
|
.replace(/[\s-]+/g, "_");
|
||||||
|
if (!lower) return null;
|
||||||
|
|
||||||
|
if (VALID_PRIMARY_AI_FLAGS.has(lower)) {
|
||||||
|
return lower;
|
||||||
|
}
|
||||||
|
|
||||||
|
return CATEGORY_TO_BADWORD_LABEL[lower] ?? null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function extractFlagsFromPrimaryAiContent(content: string): string[] {
|
||||||
|
const flags = new Set<string>();
|
||||||
|
let parsed: unknown;
|
||||||
|
|
||||||
|
try {
|
||||||
|
parsed = JSON.parse(content);
|
||||||
|
} catch {
|
||||||
|
parsed = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const addValue = (value: unknown) => {
|
||||||
|
if (typeof value !== "string") return;
|
||||||
|
const normalized = normalizePrimaryAiFlag(value);
|
||||||
|
if (normalized) flags.add(normalized);
|
||||||
|
};
|
||||||
|
|
||||||
|
if (Array.isArray(parsed)) {
|
||||||
|
for (const item of parsed) {
|
||||||
|
addValue(item);
|
||||||
|
}
|
||||||
|
} else if (parsed && typeof parsed === "object") {
|
||||||
|
const candidate = parsed as Record<string, unknown>;
|
||||||
|
for (const key of ["flags", "categories", "badwords"]) {
|
||||||
|
const value = candidate[key];
|
||||||
|
if (Array.isArray(value)) {
|
||||||
|
for (const item of value) addValue(item);
|
||||||
|
} else {
|
||||||
|
addValue(value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (flags.size > 0) {
|
||||||
|
return Array.from(flags);
|
||||||
|
}
|
||||||
|
|
||||||
|
const lowerContent = content.toLowerCase();
|
||||||
|
for (const flag of VALID_PRIMARY_AI_FLAGS) {
|
||||||
|
if (lowerContent.includes(flag)) {
|
||||||
|
flags.add(flag);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const category of Object.keys(CATEGORY_TO_BADWORD_LABEL)) {
|
||||||
|
if (lowerContent.includes(category)) {
|
||||||
|
const mapped = CATEGORY_TO_BADWORD_LABEL[category];
|
||||||
|
if (mapped) flags.add(mapped);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return Array.from(flags);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function callPrimaryAiModeration(text: string): Promise<string[]> {
|
||||||
|
const client = getPrimaryModerationClient();
|
||||||
|
if (!client) {
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
|
||||||
|
const completion = await retryWithBackoff(
|
||||||
|
async () => {
|
||||||
|
return client.chat.completions.create({
|
||||||
|
model: config.AI_LLM_MODEL,
|
||||||
|
messages: [
|
||||||
|
{
|
||||||
|
role: "user",
|
||||||
|
content:
|
||||||
|
"Deteksi kata kasar / pelanggaran ringan dari teks Indonesia berikut. " +
|
||||||
|
'Balas hanya JSON object dengan format {"flags":[...]} dan gunakan hanya flag valid ini: ' +
|
||||||
|
Array.from(VALID_PRIMARY_AI_FLAGS).join(", ") +
|
||||||
|
". Jika tidak ada pelanggaran, flags harus array kosong. Teks: " +
|
||||||
|
text,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
temperature: 0.1,
|
||||||
|
top_p: 0.9,
|
||||||
|
max_tokens: 200,
|
||||||
|
stream: false,
|
||||||
|
response_format: { type: "json_object" },
|
||||||
|
} as OpenAI.Chat.Completions.ChatCompletionCreateParamsNonStreaming);
|
||||||
|
},
|
||||||
|
{
|
||||||
|
retries: 1,
|
||||||
|
minTimeout: 500,
|
||||||
|
maxTimeout: 2000,
|
||||||
|
factor: 2,
|
||||||
|
logger: log,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
const content = completion.choices[0]?.message?.content?.trim();
|
||||||
|
if (!content) {
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
|
||||||
|
return extractFlagsFromPrimaryAiContent(content);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Groq Llama Prompt Guard Moderation API (Fallback)
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Call Groq Llama Prompt Guard 2-86M model for moderation scoring.
|
||||||
|
* Returns a probability score as a string (e.g. "0.9988824725151062").
|
||||||
|
* Scores above ~0.5 indicate moderation violations.
|
||||||
|
*/
|
||||||
|
async function callGrokModeration(text: string): Promise<string[]> {
|
||||||
|
const apiKey = config.GROQ_API_KEY;
|
||||||
|
if (!apiKey) {
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
|
||||||
|
const response = await axios.post(
|
||||||
|
config.GROQ_MODERATION_BASE_URL,
|
||||||
|
{
|
||||||
|
model: config.GROQ_MODERATION_MODEL,
|
||||||
|
messages: [{ role: "user", content: text }],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
headers: {
|
||||||
|
Authorization: `Bearer ${apiKey}`,
|
||||||
|
Accept: "application/json",
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
},
|
||||||
|
timeout: 10_000,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
const scoreStr = response.data?.choices?.[0]?.message?.content?.trim();
|
||||||
|
if (!scoreStr) {
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
|
||||||
|
// Parse the score (Llama Prompt Guard returns a single probability score)
|
||||||
|
const score = parseFloat(scoreStr);
|
||||||
|
if (isNaN(score) || score < 0.5) {
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
|
||||||
|
// Map score to moderation flags based on severity
|
||||||
|
const flags: string[] = [];
|
||||||
|
if (score >= 0.9) {
|
||||||
|
flags.push("vulgar_language", "harassment");
|
||||||
|
} else if (score >= 0.7) {
|
||||||
|
flags.push("vulgar_language");
|
||||||
|
} else {
|
||||||
|
flags.push("spam");
|
||||||
|
}
|
||||||
|
|
||||||
|
return flags;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// NVIDIA Nemotron-3 Content Safety API
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Call NVIDIA Nemotron-3 Content Safety API to detect harmful content.
|
||||||
|
* Returns categories/flags from the API response.
|
||||||
|
*/
|
||||||
|
async function callNemotronContentSafety(text: string): Promise<string[]> {
|
||||||
|
const apiKey = config.NVIDIA_NEMOTRON_API_KEY;
|
||||||
|
if (!apiKey) {
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
|
||||||
|
const response = await axios.post(
|
||||||
|
config.NVIDIA_NEMOTRON_BASE_URL,
|
||||||
|
{
|
||||||
|
model: config.NVIDIA_NEMOTRON_MODEL,
|
||||||
|
messages: [{ role: "user", content: text }],
|
||||||
|
max_tokens: 897,
|
||||||
|
temperature: 0.2,
|
||||||
|
top_p: 0.7,
|
||||||
|
stream: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
headers: {
|
||||||
|
Authorization: `Bearer ${apiKey}`,
|
||||||
|
Accept: "application/json",
|
||||||
|
},
|
||||||
|
timeout: 15_000,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
const data = response.data;
|
||||||
|
const categories: string[] = [];
|
||||||
|
|
||||||
|
// Parse the LLM response for category flags
|
||||||
|
const content = data?.choices?.[0]?.message?.content ?? "";
|
||||||
|
if (content) {
|
||||||
|
const lowerContent = content.toLowerCase();
|
||||||
|
for (const category of NVIDIA_BAD_CATEGORIES) {
|
||||||
|
if (lowerContent.includes(category)) {
|
||||||
|
categories.push(CATEGORY_TO_BADWORD_LABEL[category] ?? category);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Also check for structured response fields
|
||||||
|
const choice = data?.choices?.[0];
|
||||||
|
if (choice?.message?.content) {
|
||||||
|
try {
|
||||||
|
const parsed = JSON.parse(choice.message.content);
|
||||||
|
if (parsed.categories && Array.isArray(parsed.categories)) {
|
||||||
|
for (const cat of parsed.categories) {
|
||||||
|
if (NVIDIA_BAD_CATEGORIES.has(cat.name ?? cat)) {
|
||||||
|
categories.push(CATEGORY_TO_BADWORD_LABEL[cat.name ?? cat] ?? cat);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
// Not JSON — already handled via text search above
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return Array.from(new Set(categories));
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Three-tier cache pipeline
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Detect badwords in text using a **two-tier cache + API pipeline**:
|
||||||
|
*
|
||||||
|
* 1. **In-memory cache** (BADWORD_CACHE_TTL_MS, 10 min) — fastest path,
|
||||||
|
* keyed by the full normalized text string.
|
||||||
|
* 2. **DB cache** (DB_CACHE_TTL_MS, 24 h) — same full-text key, persisted
|
||||||
|
* across restarts. Uses the FULL normalized text (not per-word) because
|
||||||
|
* context matters: "kau" alone is clean, but "awas kau" can be a threat.
|
||||||
|
* 3. **API pipeline** (NVIDIA → Primary AI → Groq)
|
||||||
|
* only runs when both cache layers miss.
|
||||||
|
*
|
||||||
|
* No local hardcoded badword list — all detection goes through AI APIs
|
||||||
|
* to eliminate false positives from substring matching.
|
||||||
|
*/
|
||||||
|
export async function detectIndonesianBadwords(
|
||||||
|
text: string,
|
||||||
|
): Promise<string[]> {
|
||||||
|
const cacheKey = normalizeBadwordCacheKey(text);
|
||||||
|
|
||||||
|
// ── Tier 1: In-memory cache (fastest) ──
|
||||||
|
const cached = getCachedBadwords(cacheKey);
|
||||||
|
if (cached) {
|
||||||
|
return cached;
|
||||||
|
}
|
||||||
|
|
||||||
|
// De-duplicate concurrent lookups
|
||||||
|
const inFlight = inFlightBadwordLookups.get(cacheKey);
|
||||||
|
if (inFlight) {
|
||||||
|
return inFlight;
|
||||||
|
}
|
||||||
|
|
||||||
|
const lookupPromise = (async () => {
|
||||||
|
// ── Tier 2: DB cache (survives restarts, preserves context) ──
|
||||||
|
const dbEntry = await getCachedText(cacheKey);
|
||||||
|
if (dbEntry) {
|
||||||
|
const flags = [...dbEntry.flags];
|
||||||
|
setCachedBadwords(cacheKey, flags); // populate in-memory too
|
||||||
|
return flags;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Tier 3: API pipeline ──
|
||||||
|
|
||||||
|
const hits = new Set<string>();
|
||||||
|
let sourceUsed: "nvidia" | "primary_ai" | "groq" = "primary_ai";
|
||||||
|
|
||||||
|
// 3a. Try NVIDIA API if key is configured and not rate limited.
|
||||||
|
const apiKey = config.NVIDIA_NEMOTRON_API_KEY;
|
||||||
|
if (apiKey && Date.now() >= nemotronUnavailableUntil) {
|
||||||
|
try {
|
||||||
|
const apiCategories = await callNemotronContentSafety(text);
|
||||||
|
for (const hit of apiCategories) {
|
||||||
|
hits.add(hit);
|
||||||
|
}
|
||||||
|
if (apiCategories.length > 0) sourceUsed = "nvidia";
|
||||||
|
} catch (error) {
|
||||||
|
const status = axios.isAxiosError(error)
|
||||||
|
? error.response?.status
|
||||||
|
: null;
|
||||||
|
if (status === 429) {
|
||||||
|
nemotronUnavailableUntil =
|
||||||
|
Date.now() + NEMOTRON_RATE_LIMIT_COOLDOWN_MS;
|
||||||
|
}
|
||||||
|
log.warn(
|
||||||
|
{ error },
|
||||||
|
"NVIDIA Nemotron API call failed, falling back to primary AI",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 3b. Try the main AI model next.
|
||||||
|
if (hits.size === 0 && Date.now() >= primaryAiUnavailableUntil) {
|
||||||
|
try {
|
||||||
|
const primaryHits = await callPrimaryAiModeration(text);
|
||||||
|
for (const hit of primaryHits) {
|
||||||
|
hits.add(hit);
|
||||||
|
}
|
||||||
|
if (primaryHits.length > 0) sourceUsed = "primary_ai";
|
||||||
|
} catch (error) {
|
||||||
|
const status = axios.isAxiosError(error)
|
||||||
|
? error.response?.status
|
||||||
|
: null;
|
||||||
|
if (status === 429) {
|
||||||
|
primaryAiUnavailableUntil =
|
||||||
|
Date.now() + PRIMARY_AI_RATE_LIMIT_COOLDOWN_MS;
|
||||||
|
}
|
||||||
|
log.warn(
|
||||||
|
{ error },
|
||||||
|
"Primary AI badword detection failed, falling back to Groq",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 3c. Try Groq Llama Prompt Guard as final API fallback.
|
||||||
|
if (hits.size === 0 && Date.now() >= groqUnavailableUntil) {
|
||||||
|
const groqKey = config.GROQ_API_KEY;
|
||||||
|
if (groqKey) {
|
||||||
|
try {
|
||||||
|
const groqHits = await callGrokModeration(text);
|
||||||
|
for (const hit of groqHits) {
|
||||||
|
hits.add(hit);
|
||||||
|
}
|
||||||
|
if (groqHits.length > 0) sourceUsed = "groq";
|
||||||
|
} catch (error) {
|
||||||
|
const status = axios.isAxiosError(error)
|
||||||
|
? error.response?.status
|
||||||
|
: null;
|
||||||
|
if (status === 429) {
|
||||||
|
groqUnavailableUntil = Date.now() + GROQ_RATE_LIMIT_COOLDOWN_MS;
|
||||||
|
}
|
||||||
|
log.warn({ error }, "Groq Llama Prompt Guard moderation failed");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const finalHits = Array.from(hits);
|
||||||
|
|
||||||
|
// Populate all cache tiers so the same text never triggers another API call
|
||||||
|
// within the TTL window.
|
||||||
|
setCachedBadwords(cacheKey, finalHits);
|
||||||
|
await upsertCachedText(
|
||||||
|
cacheKey,
|
||||||
|
finalHits,
|
||||||
|
sourceUsed,
|
||||||
|
Date.now() + DB_CACHE_TTL_MS,
|
||||||
|
);
|
||||||
|
|
||||||
|
return finalHits;
|
||||||
|
})();
|
||||||
|
|
||||||
|
inFlightBadwordLookups.set(cacheKey, lookupPromise);
|
||||||
|
|
||||||
|
try {
|
||||||
|
return await lookupPromise;
|
||||||
|
} finally {
|
||||||
|
inFlightBadwordLookups.delete(cacheKey);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Async evidence builders
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
export async function buildModerationTextEvidence(
|
||||||
|
text: string,
|
||||||
|
): Promise<ModerationTextEvidence> {
|
||||||
|
const emojiNormalized = normalizeDiscordCustomEmoji(text);
|
||||||
|
const badwordHits = await detectIndonesianBadwords(emojiNormalized.text);
|
||||||
|
const notes: string[] = [];
|
||||||
|
|
||||||
|
for (const emojiName of emojiNormalized.emojiNames) {
|
||||||
|
notes.push(
|
||||||
|
`emoji:${emojiName}=Discord custom emoji/expression; not text offense by default`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (badwordHits.length > 0) {
|
||||||
|
notes.push(`Indonesian badword detected: ${badwordHits.join(", ")}`);
|
||||||
|
} else {
|
||||||
|
notes.push("no Indonesian badword detected");
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
raw: text,
|
||||||
|
normalized: emojiNormalized.text,
|
||||||
|
notes: Array.from(new Set(notes)),
|
||||||
|
badwords: badwordHits,
|
||||||
|
hasBadwords: badwordHits.length > 0,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function formatModerationTextEvidenceForPrompt(
|
||||||
|
text: string,
|
||||||
|
): Promise<string> {
|
||||||
|
const evidence = await buildModerationTextEvidence(text);
|
||||||
|
if (evidence.normalized === evidence.raw && evidence.notes.length === 0) {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
|
||||||
|
return [
|
||||||
|
`[normalized_text: ${evidence.normalized}]`,
|
||||||
|
evidence.notes.length > 0
|
||||||
|
? `[normalization_notes: ${evidence.notes.join("; ")}]`
|
||||||
|
: null,
|
||||||
|
]
|
||||||
|
.filter(Boolean)
|
||||||
|
.join(" ");
|
||||||
|
}
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,161 @@
|
|||||||
|
/**
|
||||||
|
* Modular system prompt builder for LLM moderation.
|
||||||
|
*
|
||||||
|
* Split into composable sections:
|
||||||
|
* - buildSystemRules() — culture/slang/flag definitions (static)
|
||||||
|
* - buildMediaInstructions() — media/sticker analysis guidance (conditional)
|
||||||
|
* - buildFewShotExamples() — 3 example outputs (static)
|
||||||
|
* - buildSystemPrompt() — assembles all sections with XML delimiters
|
||||||
|
*
|
||||||
|
* XML delimiters prevent prompt injection by clearly separating
|
||||||
|
* system instructions from user-supplied data.
|
||||||
|
*/
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Section: System Rules (static — culture, slang, flag definitions)
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
const SYSTEM_RULES = `Kamu adalah asisten moderasi konten untuk server Discord berbahasa Indonesia.
|
||||||
|
Bahasa utama komunitas ini adalah BAHASA INDONESIA. Bahasa Inggris adalah bahasa sekunder.
|
||||||
|
|
||||||
|
## Aturan Umum
|
||||||
|
- Bahasa gaul/slang Indonesia: "anjay", "wkwk", "gws", "gaskeun", "santuy", "njir", "baka", "woy", "woi", "hadeh", dll adalah AMAN.
|
||||||
|
- Singkatan umum: "gw", "lo", "emg", "kyk", "tdk", "krn", "jgn", dll adalah AMAN.
|
||||||
|
- Makian/kata kasar umum (seperti "anjing", "asu", "bangsat") BUKAN pelanggaran SARA. SARA khusus untuk diskriminasi/hinaan terhadap Suku, Agama, Ras, dan Antargolongan. NAMUN makian/kata kasar TETAP bisa di-flag sebagai "harassment" atau "vulgar_language" HANYA jika: (1) ditujukan langsung ke orang lain sebagai serangan/hinaan, (2) dalam tone agresif/mengancam, atau (3) bagian dari pola harassment berkelanjutan.
|
||||||
|
- Kata "asus" adalah merk teknologi, jangan pernah dianggap sebagai makian "asu".
|
||||||
|
- "woy"/"woi" adalah sapaan/interjeksi informal Indonesia dan tidak boleh dianggap SARA, hate speech, atau harassment tanpa target hinaan/ancaman jelas.
|
||||||
|
- Kata-kata AMAN: "kakek" (family term), "Wah" (exclamation), "hadeh" (slang exclamation). Jangan flag sebagai vulgar_language atau harassment.
|
||||||
|
- Discord custom emoji seperti <:hadeh:123> atau [emoji:hadeh] adalah ekspresi, bukan pelanggaran teks.
|
||||||
|
- Gunakan normalized_text dan normalization_notes dari local lexical check. Jika notes hanya berisi slang/emoji aman, jangan flag. Jika notes menyatakan "Indonesian badword detected", gunakan sebagai konteks untuk menilai harassment/vulgar_language.
|
||||||
|
|
||||||
|
## Kategori Pelanggaran & Kriteria Flag
|
||||||
|
Prioritas tertinggi (ANCAMAN KESELAMATAN):
|
||||||
|
- child_safety, self_harm, violence, illegal_content — flag jika ada indikasi nyata
|
||||||
|
- Pornografi/NSFW, ajakan seksual, roleplay seksual → "sexual_content"
|
||||||
|
- Judi/promosi judi → "gambling"
|
||||||
|
- Narkoba/promosi → "drugs"
|
||||||
|
|
||||||
|
Prioritas menengah (PERILAKU MERUSAK):
|
||||||
|
- Ancaman kekerasan, doxxing, scam → flag sesuai kategori
|
||||||
|
- spam self-promo → "spam"
|
||||||
|
- Istilah agama/suku/ras: penyebutan netral/edukasi = clean; hinaan/provokasi/diskriminatif = "sara" atau "hate_speech"
|
||||||
|
|
||||||
|
Prioritas rendah (PELANGGARAN RINGAN):
|
||||||
|
- harassment (targeted insult), vulgar_language (profanity terarah)
|
||||||
|
- sexual_deviation: jika pesan mempromosikan/mendukung topik seksual/identitas yang dibatasi server sebagai pembahasan utama
|
||||||
|
|
||||||
|
## Pohon Keputusan (Decision Tree)
|
||||||
|
1. Apakah ada ancaman keselamatan nyata (child_safety, self_harm, violence)? → flagged, critical
|
||||||
|
2. Apakah ada konten ilegal/explicit (NSFW, drugs, gambling, scam)? → flagged, high
|
||||||
|
3. Apakah ada harassment terarah/hate speech/sara? → flagged, medium-high
|
||||||
|
4. Apakah ada spam/promosi borderline? → warn, low-medium
|
||||||
|
5. Jika tidak ada pelanggaran jelas atau bukti ambigu → clean
|
||||||
|
Jangan pernah flag hanya berdasarkan kecurigaan atau ketidakjelasan konteks.`;
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Section: Media Instructions (conditional — injected when media present)
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
const MEDIA_INSTRUCTIONS = `## Instruksi Analisis Media
|
||||||
|
Gambar, sticker, embed image, preview link, dan attachment sudah dianalisis lewat request media terpisah sebelum batch utama.
|
||||||
|
Gunakan baris "Media analysis" sebagai evidence visual utama dalam keputusan moderasi batch ini.
|
||||||
|
|
||||||
|
## Panduan Khusus Sticker
|
||||||
|
- Sticker Discord adalah media kartun/meme/ilustrasi, BUKAN foto atau video nyata.
|
||||||
|
- Sticker sering bersifat humor, satir, atau ekspresi emosi yang dilebih-lebihkan.
|
||||||
|
- Gambar sticker bisa menampilkan adegan kartun yang terlihat "keras" — itu SENI KARTUN, bukan dokumentasi kekerasan nyata.
|
||||||
|
- Nama sticker yang terdengar provokatif (mis. "Singa injek pejabat") adalah konteks satir/humor. JANGAN flag berdasarkan nama sticker saja.
|
||||||
|
- Terapkan standar yang lebih longgar untuk konten kartun/meme dibanding foto/video nyata.
|
||||||
|
- Sticker yang berhasil diunduh WAJIB diperlakukan sebagai image evidence, bukan sekadar nama sticker.`;
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Section: Few-Shot Examples
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
const FEW_SHOT_EXAMPLES = `## Contoh Output yang Benak
|
||||||
|
|
||||||
|
Contoh 1 — Pesan bersih dengan slang:
|
||||||
|
Input: [target] id=12345 user=budi: anjay wkwk gaskeun santuy bro
|
||||||
|
Output: {"results":[{"message_id":"12345","status":"clean","flags":[],"score":0.0,"categories":[],"severity":"none","confidence":0.95,"recommended_action":"none","policy_version":"default-2026-05-30","evidence":[],"analysis":"Slang Indonesia umum tanpa pelanggaran terdeteksi."}]}
|
||||||
|
|
||||||
|
Contoh 2 — Harassment terarah:
|
||||||
|
Input: [target] id=67890 user=anon: lu goblok banget sih kontol, mampus aja lo
|
||||||
|
Output: {"results":[{"message_id":"67890","status":"flagged","flags":["harassment","vulgar_language"],"score":0.85,"categories":["harassment","vulgar_language"],"severity":"high","confidence":0.9,"recommended_action":"delete","policy_version":"default-2026-05-30","evidence":["lu goblok banget sih kontol","mampus aja lo"],"analysis":"Insult langsung dengan kata kasar terarah ke individu."}]}
|
||||||
|
|
||||||
|
Contoh 3 — Sticker kartun dengan nama provokatif:
|
||||||
|
Input: [target] id=11111 user=citra: <:singa_injek:123456> [sticker: "Singa injek pejabat"]
|
||||||
|
Output: {"results":[{"message_id":"11111","status":"clean","flags":[],"score":0.1,"categories":[],"severity":"none","confidence":0.8,"recommended_action":"none","policy_version":"default-2026-05-30","evidence":[],"analysis":"Sticker kartun satir dengan nama provokatif namun bukan ancaman nyata."}]}`;
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Section: Output Schema + XML Delimiter Instructions
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
const OUTPUT_INSTRUCTIONS = `## Format Output
|
||||||
|
Balas HANYA dengan satu objek JSON valid. Tanpa markdown, tanpa prose, tanpa komentar, tanpa XML.
|
||||||
|
Struktur wajib:
|
||||||
|
{
|
||||||
|
"results": [
|
||||||
|
{
|
||||||
|
"message_id": "<ID string PERSIS seperti di input>",
|
||||||
|
"status": "clean" | "warn" | "flagged",
|
||||||
|
"flags": ["<string array, kosong jika clean>"],
|
||||||
|
"score": 0.0,
|
||||||
|
"categories": ["<kategori kebijakan, kosong jika clean>"],
|
||||||
|
"severity": "none" | "low" | "medium" | "high" | "critical",
|
||||||
|
"confidence": 0.0,
|
||||||
|
"recommended_action": "none" | "monitor" | "warn" | "review" | "delete" | "escalate",
|
||||||
|
"policy_version": "default-2026-05-30",
|
||||||
|
"evidence": ["<kutipan/evidence singkat>"],
|
||||||
|
"analysis": "<penjelasan singkat dalam Bahasa Indonesia, maks 2 kalimat>"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
Kriteria status:
|
||||||
|
- "clean": tidak ada pelanggaran terdeteksi, atau kasus ambigu setelah semua evidence dianalisis
|
||||||
|
- "warn": risiko ringan konkret terdeteksi (spam borderline, harassment ringan)
|
||||||
|
- "flagged": pelanggaran jelas terdeteksi
|
||||||
|
|
||||||
|
Larangan output analysis:
|
||||||
|
- Jangan tulis "kurang konteks", "perlu dicek admin", "perlu moderator periksa", "tidak bisa menentukan", atau frasa deferral sejenis.
|
||||||
|
- Jika evidence tidak cukup kuat untuk pelanggaran, status harus "clean" dan analysis menjelaskan alasan langsung.
|
||||||
|
- Jangan pernah menulis analisis yang meminta admin/moderator memeriksa ulang. Berikan kesimpulan langsung.
|
||||||
|
|
||||||
|
Flag yang valid: spam, hate_speech, sara, hoaks, harassment, vulgar_language, sexual_content, sexual_deviation, violence, self_harm, doxxing, scam, misinformation, nsfw_image, gore_image, illegal_content, gambling, drugs, child_safety, financial_scam, religious_insult, self_promo
|
||||||
|
|
||||||
|
CRITICAL: "message_id" HARUS berupa STRING (dibungkus tanda kutip ganda). Jangan perlakukan ID sebagai angka.`;
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Composer: assembles all sections with XML delimiters
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
export interface BuildSystemPromptOptions {
|
||||||
|
contextText: string;
|
||||||
|
includeMediaInstructions: boolean;
|
||||||
|
correction?: { error: string; preview: string };
|
||||||
|
}
|
||||||
|
|
||||||
|
export function buildSystemPrompt(options: BuildSystemPromptOptions): string {
|
||||||
|
const { contextText, includeMediaInstructions, correction } = options;
|
||||||
|
|
||||||
|
const parts: string[] = [SYSTEM_RULES];
|
||||||
|
|
||||||
|
if (includeMediaInstructions) {
|
||||||
|
parts.push(MEDIA_INSTRUCTIONS);
|
||||||
|
}
|
||||||
|
|
||||||
|
parts.push(FEW_SHOT_EXAMPLES);
|
||||||
|
parts.push(OUTPUT_INSTRUCTIONS);
|
||||||
|
|
||||||
|
// XML-delimited context — prevents prompt injection
|
||||||
|
const delimitedContext = `<conversation_context>\n${contextText}\n</conversation_context>`;
|
||||||
|
parts.push(delimitedContext);
|
||||||
|
|
||||||
|
let base = parts.join("\n\n");
|
||||||
|
|
||||||
|
if (correction) {
|
||||||
|
base += `\n\nRESPON SEBELUMNYA GAGAL VALIDASI.\nError: ${correction.error}\nPreview respons tidak valid:\n${correction.preview}\n\nCoba lagi dengan output JSON yang benar sesuai skema di atas.`;
|
||||||
|
}
|
||||||
|
|
||||||
|
return base;
|
||||||
|
}
|
||||||
@@ -0,0 +1,209 @@
|
|||||||
|
import { mkdir, readFile, unlink, writeFile } from "node:fs/promises";
|
||||||
|
import { join } from "node:path";
|
||||||
|
import { createChildLogger } from "../../shared/logger/logger.js";
|
||||||
|
|
||||||
|
const logger = createChildLogger("sticker-cache");
|
||||||
|
|
||||||
|
export interface StickerCacheEntry {
|
||||||
|
base64: string;
|
||||||
|
mimeType: string;
|
||||||
|
fetchedAt: number;
|
||||||
|
size: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface CacheIndexEntry {
|
||||||
|
file: string;
|
||||||
|
mimeType: string;
|
||||||
|
size: number;
|
||||||
|
fetchedAt: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface CacheIndex {
|
||||||
|
entries: Record<string, CacheIndexEntry>;
|
||||||
|
totalSizeBytes: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface StickerCacheOptions {
|
||||||
|
cacheDir: string;
|
||||||
|
maxSizeBytes: number;
|
||||||
|
ttlMs?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
let cacheDir = "";
|
||||||
|
let maxSizeBytes = 0;
|
||||||
|
let ttlMs = 7 * 24 * 60 * 60 * 1000; // 7 days default
|
||||||
|
let index: CacheIndex = { entries: {}, totalSizeBytes: 0 };
|
||||||
|
let ready = false;
|
||||||
|
|
||||||
|
function sanitizeKey(name: string): string {
|
||||||
|
return encodeURIComponent(name).replace(/%/g, "_");
|
||||||
|
}
|
||||||
|
|
||||||
|
async function loadIndex(): Promise<CacheIndex> {
|
||||||
|
try {
|
||||||
|
const raw = await readFile(join(cacheDir, "index.json"), "utf-8");
|
||||||
|
return JSON.parse(raw) as CacheIndex;
|
||||||
|
} catch {
|
||||||
|
return { entries: {}, totalSizeBytes: 0 };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function saveIndex(idx: CacheIndex): Promise<void> {
|
||||||
|
await writeFile(
|
||||||
|
join(cacheDir, "index.json"),
|
||||||
|
JSON.stringify(idx, null, 2),
|
||||||
|
"utf-8",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Initialise the sticker cache: create directory, load index.
|
||||||
|
* Idempotent — safe to call multiple times.
|
||||||
|
*/
|
||||||
|
export async function initStickerCache(
|
||||||
|
opts: StickerCacheOptions,
|
||||||
|
): Promise<void> {
|
||||||
|
if (ready) return;
|
||||||
|
cacheDir = opts.cacheDir;
|
||||||
|
maxSizeBytes = opts.maxSizeBytes;
|
||||||
|
ttlMs = opts.ttlMs ?? 7 * 24 * 60 * 60 * 1000;
|
||||||
|
|
||||||
|
await mkdir(cacheDir, { recursive: true });
|
||||||
|
index = await loadIndex();
|
||||||
|
|
||||||
|
// Prune expired entries on startup
|
||||||
|
const now = Date.now();
|
||||||
|
let changed = false;
|
||||||
|
for (const [key, meta] of Object.entries(index.entries)) {
|
||||||
|
if (now - meta.fetchedAt > ttlMs) {
|
||||||
|
await unlink(join(cacheDir, meta.file)).catch(() => {});
|
||||||
|
index.totalSizeBytes -= meta.size;
|
||||||
|
delete index.entries[key];
|
||||||
|
changed = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (changed) await saveIndex(index);
|
||||||
|
|
||||||
|
ready = true;
|
||||||
|
logger.info(
|
||||||
|
{
|
||||||
|
entryCount: Object.keys(index.entries).length,
|
||||||
|
totalSizeBytes: index.totalSizeBytes,
|
||||||
|
},
|
||||||
|
"Sticker cache initialized",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Look up a sticker image by name. Returns null on miss or TTL expiry.
|
||||||
|
*/
|
||||||
|
export async function getStickerFromCache(
|
||||||
|
stickerName: string,
|
||||||
|
): Promise<StickerCacheEntry | null> {
|
||||||
|
if (!ready) return null;
|
||||||
|
|
||||||
|
const key = sanitizeKey(stickerName);
|
||||||
|
const meta = index.entries[key];
|
||||||
|
if (!meta) return null;
|
||||||
|
|
||||||
|
// TTL check
|
||||||
|
if (Date.now() - meta.fetchedAt > ttlMs) {
|
||||||
|
await unlink(join(cacheDir, meta.file)).catch(() => {});
|
||||||
|
index.totalSizeBytes -= meta.size;
|
||||||
|
delete index.entries[key];
|
||||||
|
await saveIndex(index);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const raw = await readFile(join(cacheDir, meta.file), "utf-8");
|
||||||
|
return {
|
||||||
|
base64: raw,
|
||||||
|
mimeType: meta.mimeType,
|
||||||
|
fetchedAt: meta.fetchedAt,
|
||||||
|
size: meta.size,
|
||||||
|
};
|
||||||
|
} catch {
|
||||||
|
// File missing — clean up index entry
|
||||||
|
delete index.entries[key];
|
||||||
|
await saveIndex(index);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Store a sticker image in the cache. Fires and forgets — never blocks.
|
||||||
|
*/
|
||||||
|
export async function setStickerInCache(
|
||||||
|
stickerName: string,
|
||||||
|
base64: string,
|
||||||
|
mimeType: string,
|
||||||
|
): Promise<void> {
|
||||||
|
if (!ready) return;
|
||||||
|
|
||||||
|
const key = sanitizeKey(stickerName);
|
||||||
|
const fileName = `${key}.dat`;
|
||||||
|
const size = Buffer.byteLength(base64, "utf-8");
|
||||||
|
|
||||||
|
// Evict if needed
|
||||||
|
await evictIfNeeded(size);
|
||||||
|
|
||||||
|
try {
|
||||||
|
await writeFile(join(cacheDir, fileName), base64, "utf-8");
|
||||||
|
index.entries[key] = {
|
||||||
|
file: fileName,
|
||||||
|
mimeType,
|
||||||
|
size,
|
||||||
|
fetchedAt: Date.now(),
|
||||||
|
};
|
||||||
|
index.totalSizeBytes += size;
|
||||||
|
await saveIndex(index);
|
||||||
|
logger.debug({ stickerName, size }, "Sticker cached");
|
||||||
|
} catch (err) {
|
||||||
|
logger.warn(
|
||||||
|
{ stickerName, error: err instanceof Error ? err.message : String(err) },
|
||||||
|
"Failed to write sticker to cache",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function evictIfNeeded(newSize: number): Promise<void> {
|
||||||
|
while (index.totalSizeBytes + newSize > maxSizeBytes) {
|
||||||
|
// Find oldest entry
|
||||||
|
let oldestKey: string | null = null;
|
||||||
|
let oldestTime = Infinity;
|
||||||
|
for (const [key, meta] of Object.entries(index.entries)) {
|
||||||
|
if (meta.fetchedAt < oldestTime) {
|
||||||
|
oldestTime = meta.fetchedAt;
|
||||||
|
oldestKey = key;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (!oldestKey) break;
|
||||||
|
|
||||||
|
const meta = index.entries[oldestKey];
|
||||||
|
await unlink(join(cacheDir, meta.file)).catch(() => {});
|
||||||
|
index.totalSizeBytes -= meta.size;
|
||||||
|
delete index.entries[oldestKey];
|
||||||
|
}
|
||||||
|
await saveIndex(index);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Return current cache stats for observability.
|
||||||
|
*/
|
||||||
|
export function getStickerCacheStats(): {
|
||||||
|
entryCount: number;
|
||||||
|
totalSizeBytes: number;
|
||||||
|
} {
|
||||||
|
return {
|
||||||
|
entryCount: Object.keys(index.entries).length,
|
||||||
|
totalSizeBytes: index.totalSizeBytes,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check if cache has been initialized.
|
||||||
|
*/
|
||||||
|
export function isStickerCacheReady(): boolean {
|
||||||
|
return ready;
|
||||||
|
}
|
||||||
@@ -0,0 +1,96 @@
|
|||||||
|
/**
|
||||||
|
* Sticker-specific prompt templates for AI moderation.
|
||||||
|
*
|
||||||
|
* Discord stickers are cartoon/meme artwork — not real photos.
|
||||||
|
* These prompts give the LLM proper context to avoid false-positive flags
|
||||||
|
* based solely on sticker names or cartoon imagery.
|
||||||
|
*/
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Prompt used when a sticker image was successfully downloaded (from cache
|
||||||
|
* or network) and is being sent to the vision LLM as a base64 image.
|
||||||
|
*
|
||||||
|
* Explains that stickers are cartoon art, not documentation of real events,
|
||||||
|
* and instructs the model to apply looser standards for cartoon content.
|
||||||
|
*/
|
||||||
|
export function buildStickerVisionPrompt(
|
||||||
|
stickerName: string,
|
||||||
|
messageId: string,
|
||||||
|
): string {
|
||||||
|
return [
|
||||||
|
`Analisis sticker Discord berikut sebagai evidence moderasi.`,
|
||||||
|
`Sticker "${stickerName}" berasal dari pesan id=${messageId}.`,
|
||||||
|
``,
|
||||||
|
`PENTING — Konteks Sticker:`,
|
||||||
|
`- Sticker Discord adalah gambar KARTUN/MEME/ILUSTRASI, BUKAN foto atau video nyata.`,
|
||||||
|
`- Sticker sering bersifat humor, satir, atau ekspresi emosi yang dilebih-lebihkan.`,
|
||||||
|
`- Gambar di sticker bisa menampilkan adegan yang terlihat "keras" (tokoh kartun menginjak sesuatu, ledakan komik, senjata kartun, tokoh berantem) — itu SENI KARTUN, bukan dokumentasi kekerasan atau ancaman nyata.`,
|
||||||
|
`- Teks di sticker sering berupa lelucon, sindiran, atau ekspresi khas komunitas — bukan ancaman literal.`,
|
||||||
|
``,
|
||||||
|
`Jelaskan isi visual, teks yang terlihat, dan konteks risiko.`,
|
||||||
|
`Terapkan standar yang lebih longgar untuk konten kartun/meme:`,
|
||||||
|
`- Adegan kartun yang terlihat "keras" ≠ kekerasan nyata → jangan flag "violence" kecuali jelas menargetkan individu/kelompok nyata dengan ancaman serius.`,
|
||||||
|
`- Nama sticker yang terdengar provokatif (mis. "Singa injek pejabat") adalah konteks satir/kartun, bukan bukti pelanggaran.`,
|
||||||
|
`- Humor/satir/politik kartun ≠ SARA atau hate speech.`,
|
||||||
|
`- Sticker yang menampilkan tokoh kartun dalam pose agresif adalah ekspresi/emosi umum di Discord, bukan harassment.`,
|
||||||
|
``,
|
||||||
|
`Jawab Bahasa Indonesia, maksimal 3 kalimat. Jangan bilang kurang konteks atau perlu admin cek.`,
|
||||||
|
].join("\n");
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Wrapper for text-only evidence when a sticker image failed to download.
|
||||||
|
*
|
||||||
|
* Returns a formatted string that explicitly tells the LLM not to flag
|
||||||
|
* based on the sticker name alone, since names can sound provocative
|
||||||
|
* while the actual cartoon image is harmless.
|
||||||
|
*/
|
||||||
|
export function buildStickerTextOnlyWarning(
|
||||||
|
stickerName: string,
|
||||||
|
stickerUrl: string,
|
||||||
|
): string {
|
||||||
|
return (
|
||||||
|
`[sticker: "${stickerName}" (${stickerUrl}) — GAMBAR GAGAL DIUNDUH. ` +
|
||||||
|
`"${stickerName}" adalah sticker kartun/meme Discord. ` +
|
||||||
|
`JANGAN flag berdasarkan nama sticker saja tanpa gambar visual. ` +
|
||||||
|
`Sticker Discord adalah seni kartun/ekspresi humor, bukan foto nyata. ` +
|
||||||
|
`Nama yang terdengar provokatif adalah hal umum untuk sticker satir/humor di Discord.]`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Prompt used when a custom emoji image was successfully downloaded
|
||||||
|
* and is being sent to the vision LLM as a base64 image.
|
||||||
|
*
|
||||||
|
* Custom emojis are small icons — context is similar to stickers.
|
||||||
|
*/
|
||||||
|
export function buildCustomEmojiVisionPrompt(
|
||||||
|
emojiName: string,
|
||||||
|
messageId: string,
|
||||||
|
): string {
|
||||||
|
return [
|
||||||
|
`Analisis custom emoji Discord berikut sebagai evidence moderasi.`,
|
||||||
|
`Emoji "${emojiName}" berasal dari pesan id=${messageId}.`,
|
||||||
|
``,
|
||||||
|
`PENTING — Konteks Custom Emoji:`,
|
||||||
|
`- Custom emoji Discord adalah ikon kecil/ekspresi, BUKAN foto atau dokumen nyata.`,
|
||||||
|
`- Emoji sering digunakan untuk ekspresi emosi, reaksi, atau lelucon.`,
|
||||||
|
`- Jangan flag berdasarkan nama emoji saja — analisis isi visual gambar.`,
|
||||||
|
`- Emoji yang terlihat lucu/aneh adalah hal umum di Discord, bukan pelanggaran.`,
|
||||||
|
``,
|
||||||
|
`Jelaskan isi visual dan konteks risiko.`,
|
||||||
|
`Jawab Bahasa Indonesia, maksimal 2 kalimat. Jangan bilang kurang konteks.`,
|
||||||
|
].join("\n");
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Fallback text for when a custom emoji image failed to download.
|
||||||
|
*/
|
||||||
|
export function buildCustomEmojiTextOnlyFallback(emojiName: string): string {
|
||||||
|
return (
|
||||||
|
`[custom_emoji: "${emojiName}" — GAMBAR GAGAL DIUNDUH. ` +
|
||||||
|
`"${emojiName}" adalah custom emoji Discord (ikon kecil). ` +
|
||||||
|
`JANGAN flag berdasarkan nama emoji saja tanpa gambar visual. ` +
|
||||||
|
`Custom emoji di Discord adalah ekspresi/emosi umum, bukan konten ofensif.]`
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -0,0 +1,241 @@
|
|||||||
|
import { createHash } from "node:crypto";
|
||||||
|
import { executeAll, executeGet } from "../../shared/database/drizzle.js";
|
||||||
|
import { createChildLogger } from "../../shared/logger/logger.js";
|
||||||
|
|
||||||
|
const logger = createChildLogger("text-cache-store");
|
||||||
|
|
||||||
|
export interface TextCacheEntry {
|
||||||
|
text: string;
|
||||||
|
flags: string[];
|
||||||
|
source: "local" | "nvidia" | "primary_ai" | "groq" | "vision_llm";
|
||||||
|
analyzed_at: number;
|
||||||
|
expires_at: number;
|
||||||
|
hit_count: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Lookup cached analysis result for a normalized text string.
|
||||||
|
* Returns null if not found or expired.
|
||||||
|
*/
|
||||||
|
export async function getCachedText(
|
||||||
|
text: string,
|
||||||
|
): Promise<TextCacheEntry | null> {
|
||||||
|
try {
|
||||||
|
const row = await executeGet(
|
||||||
|
`SELECT text, flags, source, analyzed_at, expires_at, hit_count
|
||||||
|
FROM text_analysis_cache
|
||||||
|
WHERE text = $1 AND expires_at > $2`,
|
||||||
|
[text, Date.now()],
|
||||||
|
);
|
||||||
|
|
||||||
|
if (!row) return null;
|
||||||
|
|
||||||
|
return {
|
||||||
|
text: row.text,
|
||||||
|
flags: JSON.parse(row.flags),
|
||||||
|
source: row.source,
|
||||||
|
analyzed_at: row.analyzed_at,
|
||||||
|
expires_at: row.expires_at,
|
||||||
|
hit_count: row.hit_count,
|
||||||
|
};
|
||||||
|
} catch (error) {
|
||||||
|
logger.error(
|
||||||
|
{ error: error instanceof Error ? error.message : String(error) },
|
||||||
|
"Failed to get cached text",
|
||||||
|
);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Insert or update a text analysis cache entry.
|
||||||
|
*/
|
||||||
|
export async function upsertCachedText(
|
||||||
|
text: string,
|
||||||
|
flags: string[],
|
||||||
|
source: "local" | "nvidia" | "primary_ai" | "groq" | "vision_llm",
|
||||||
|
expiresAt: number,
|
||||||
|
): Promise<void> {
|
||||||
|
const now = Date.now();
|
||||||
|
|
||||||
|
try {
|
||||||
|
await executeAll(
|
||||||
|
`INSERT INTO text_analysis_cache (text, flags, source, analyzed_at, expires_at, hit_count)
|
||||||
|
VALUES ($1, $2, $3, $4, $5, 0)
|
||||||
|
ON CONFLICT (text) DO UPDATE SET
|
||||||
|
flags = EXCLUDED.flags,
|
||||||
|
source = EXCLUDED.source,
|
||||||
|
analyzed_at = EXCLUDED.analyzed_at,
|
||||||
|
expires_at = EXCLUDED.expires_at`,
|
||||||
|
[text, JSON.stringify(flags), source, now, expiresAt],
|
||||||
|
);
|
||||||
|
} catch (error) {
|
||||||
|
logger.error(
|
||||||
|
{ error: error instanceof Error ? error.message : String(error) },
|
||||||
|
"Failed to upsert cached text",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Increment hit count for a cached text entry (called on cache hit).
|
||||||
|
*/
|
||||||
|
export async function incrementTextCacheHit(text: string): Promise<void> {
|
||||||
|
try {
|
||||||
|
await executeAll(
|
||||||
|
`UPDATE text_analysis_cache SET hit_count = hit_count + 1 WHERE text = $1`,
|
||||||
|
[text],
|
||||||
|
);
|
||||||
|
} catch (error) {
|
||||||
|
// Silent fail — this is just a counter, not critical
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Delete expired cache entries. Run periodically to keep the table clean.
|
||||||
|
*/
|
||||||
|
export async function pruneExpiredTexts(): Promise<number> {
|
||||||
|
try {
|
||||||
|
const result = await executeAll(
|
||||||
|
`DELETE FROM text_analysis_cache WHERE expires_at < $1`,
|
||||||
|
[Date.now()],
|
||||||
|
);
|
||||||
|
return (result as any).rowCount ?? 0;
|
||||||
|
} catch (error) {
|
||||||
|
logger.error(
|
||||||
|
{ error: error instanceof Error ? error.message : String(error) },
|
||||||
|
"Failed to prune expired texts",
|
||||||
|
);
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get cache statistics for observability.
|
||||||
|
*/
|
||||||
|
export async function getTextCacheStats(): Promise<{
|
||||||
|
total: number;
|
||||||
|
expired: number;
|
||||||
|
bySource: Record<string, number>;
|
||||||
|
}> {
|
||||||
|
try {
|
||||||
|
const now = Date.now();
|
||||||
|
|
||||||
|
const [totalRow, expiredRow, sourceRows] = await Promise.all([
|
||||||
|
executeAll(`SELECT count(*) as cnt FROM text_analysis_cache`),
|
||||||
|
executeAll(
|
||||||
|
`SELECT count(*) as cnt FROM text_analysis_cache WHERE expires_at < $1`,
|
||||||
|
[now],
|
||||||
|
),
|
||||||
|
executeAll(
|
||||||
|
`SELECT source, count(*) as cnt FROM text_analysis_cache GROUP BY source`,
|
||||||
|
),
|
||||||
|
]);
|
||||||
|
|
||||||
|
const bySource: Record<string, number> = {};
|
||||||
|
for (const row of sourceRows) {
|
||||||
|
bySource[row.source] = row.cnt;
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
total: totalRow[0]?.cnt ?? 0,
|
||||||
|
expired: expiredRow[0]?.cnt ?? 0,
|
||||||
|
bySource,
|
||||||
|
};
|
||||||
|
} catch (error) {
|
||||||
|
logger.error(
|
||||||
|
{ error: error instanceof Error ? error.message : String(error) },
|
||||||
|
"Failed to get text cache stats",
|
||||||
|
);
|
||||||
|
return { total: 0, expired: 0, bySource: {} };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Media / Vision analysis cache helpers (reuses text_analysis_cache table)
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Generate a deterministic cache key for a sticker.
|
||||||
|
* Same sticker name → same key across sessions and servers.
|
||||||
|
*/
|
||||||
|
export function makeStickerCacheKey(stickerName: string): string {
|
||||||
|
return `sticker:${stickerName}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Generate a deterministic cache key for a custom emoji by its Discord ID.
|
||||||
|
*/
|
||||||
|
export function makeCustomEmojiCacheKey(emojiId: string): string {
|
||||||
|
return `emoji:${emojiId}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Generate a deterministic cache key for an image data URL.
|
||||||
|
* Hashes the first 128 chars of the data URL (enough to identify the image
|
||||||
|
* without storing the full base64 string as the key).
|
||||||
|
*/
|
||||||
|
export function makeImageCacheKey(dataUrl: string): string {
|
||||||
|
const prefix = dataUrl.slice(0, 128);
|
||||||
|
const hash = createHash("sha256").update(prefix).digest("hex").slice(0, 16);
|
||||||
|
return `image:${hash}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Lookup a cached media analysis result.
|
||||||
|
* Returns the full cached text (the analysis summary string) or null.
|
||||||
|
*/
|
||||||
|
export async function getCachedMediaAnalysis(
|
||||||
|
cacheKey: string,
|
||||||
|
): Promise<string | null> {
|
||||||
|
try {
|
||||||
|
const row = await executeGet(
|
||||||
|
`SELECT flags, hit_count
|
||||||
|
FROM text_analysis_cache
|
||||||
|
WHERE text = $1 AND expires_at > $2`,
|
||||||
|
[cacheKey, Date.now()],
|
||||||
|
);
|
||||||
|
|
||||||
|
if (!row) return null;
|
||||||
|
|
||||||
|
// flags stores the analysis result for media entries
|
||||||
|
const result = JSON.parse(row.flags) as string;
|
||||||
|
return result || null;
|
||||||
|
} catch (error) {
|
||||||
|
logger.error(
|
||||||
|
{ error: error instanceof Error ? error.message : String(error) },
|
||||||
|
"Failed to get cached media analysis",
|
||||||
|
);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Store a media analysis result in the cache.
|
||||||
|
*/
|
||||||
|
export async function upsertCachedMediaAnalysis(
|
||||||
|
cacheKey: string,
|
||||||
|
analysisResult: string,
|
||||||
|
source: "vision_llm",
|
||||||
|
expiresAt: number,
|
||||||
|
): Promise<void> {
|
||||||
|
const now = Date.now();
|
||||||
|
|
||||||
|
try {
|
||||||
|
await executeAll(
|
||||||
|
`INSERT INTO text_analysis_cache (text, flags, source, analyzed_at, expires_at, hit_count)
|
||||||
|
VALUES ($1, $2, $3, $4, $5, 0)
|
||||||
|
ON CONFLICT (text) DO UPDATE SET
|
||||||
|
flags = EXCLUDED.flags,
|
||||||
|
source = EXCLUDED.source,
|
||||||
|
analyzed_at = EXCLUDED.analyzed_at,
|
||||||
|
expires_at = EXCLUDED.expires_at`,
|
||||||
|
[cacheKey, JSON.stringify(analysisResult), source, now, expiresAt],
|
||||||
|
);
|
||||||
|
} catch (error) {
|
||||||
|
logger.error(
|
||||||
|
{ error: error instanceof Error ? error.message : String(error) },
|
||||||
|
"Failed to upsert cached media analysis",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,209 @@
|
|||||||
|
import { resolve } from "node:dns/promises";
|
||||||
|
import { isIP } from "node:net";
|
||||||
|
import { createChildLogger } from "../../shared/logger/logger.js";
|
||||||
|
|
||||||
|
const log = createChildLogger("urlFetcher");
|
||||||
|
|
||||||
|
export interface FetchedUrlContext {
|
||||||
|
url: string;
|
||||||
|
type: "image" | "text" | "error";
|
||||||
|
data?: Buffer;
|
||||||
|
mimeType?: string;
|
||||||
|
textContent?: string;
|
||||||
|
error?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
const MAX_FETCH_SIZE = 5 * 1024 * 1024; // 5 MB
|
||||||
|
const FETCH_TIMEOUT_MS = 8000;
|
||||||
|
const URL_REGEX = /https?:\/\/[^\s<]+[^<.,:;"')\]\s]/gi;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Basic SSRF protection.
|
||||||
|
* Note: A sophisticated attacker could still use DNS rebinding.
|
||||||
|
*/
|
||||||
|
async function isSafeUrl(urlStr: string): Promise<boolean> {
|
||||||
|
try {
|
||||||
|
const parsed = new URL(urlStr);
|
||||||
|
const host = parsed.hostname;
|
||||||
|
|
||||||
|
// Block obvious local IPs/hostnames
|
||||||
|
if (
|
||||||
|
host === "localhost" ||
|
||||||
|
host === "127.0.0.1" ||
|
||||||
|
host === "::1" ||
|
||||||
|
host.startsWith("192.168.") ||
|
||||||
|
host.startsWith("10.") ||
|
||||||
|
/^172\.(1[6-9]|2[0-9]|3[0-1])\./.test(host)
|
||||||
|
) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Try resolving to check if it resolves to a local IP
|
||||||
|
if (!isIP(host)) {
|
||||||
|
try {
|
||||||
|
const addresses = await resolve(host);
|
||||||
|
for (const ip of addresses) {
|
||||||
|
if (
|
||||||
|
ip === "127.0.0.1" ||
|
||||||
|
ip.startsWith("192.168.") ||
|
||||||
|
ip.startsWith("10.") ||
|
||||||
|
/^172\.(1[6-9]|2[0-9]|3[0-1])\./.test(ip)
|
||||||
|
) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
// If DNS fails, we can't fetch it anyway
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return true;
|
||||||
|
} catch (err) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function extractOgImage(html: string): string | null {
|
||||||
|
// Look for <meta ... property="og:image" ... content="..."> or <meta ... name="twitter:image" ... content="...">
|
||||||
|
const ogRegex =
|
||||||
|
/<meta[^>]*(?:property|name)=["'](?:og:image|twitter:image)["'][^>]*content=["']([^"']+)["']/i;
|
||||||
|
const match = html.match(ogRegex);
|
||||||
|
if (match && match[1]) {
|
||||||
|
// Unescape basic HTML entities
|
||||||
|
return match[1].replace(/&/g, "&").replace(/"/g, '"');
|
||||||
|
}
|
||||||
|
|
||||||
|
// Try reversed attribute order: <meta ... content="..." ... property="og:image">
|
||||||
|
const ogRegexRev =
|
||||||
|
/<meta[^>]*content=["']([^"']+)["'][^>]*(?:property|name)=["'](?:og:image|twitter:image)["']/i;
|
||||||
|
const matchRev = html.match(ogRegexRev);
|
||||||
|
if (matchRev && matchRev[1]) {
|
||||||
|
return matchRev[1].replace(/&/g, "&").replace(/"/g, '"');
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function truncateAndCleanHtml(html: string, maxLen = 1000): string {
|
||||||
|
// Strip <script> and <style> entirely
|
||||||
|
let text = html.replace(
|
||||||
|
/<script\b[^<]*(?:(?!<\/script>)<[^<]*)*<\/script>/gi,
|
||||||
|
" ",
|
||||||
|
);
|
||||||
|
text = text.replace(/<style\b[^<]*(?:(?!<\/style>)<[^<]*)*<\/style>/gi, " ");
|
||||||
|
// Strip all other HTML tags
|
||||||
|
text = text.replace(/<[^>]+>/g, " ");
|
||||||
|
// Replace multiple spaces/newlines
|
||||||
|
text = text.replace(/\s+/g, " ").trim();
|
||||||
|
|
||||||
|
return text.substring(0, maxLen);
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function fetchUrlSafely(
|
||||||
|
url: string,
|
||||||
|
depth = 0,
|
||||||
|
): Promise<FetchedUrlContext> {
|
||||||
|
if (depth > 1) {
|
||||||
|
return { url, type: "error", error: "Max redirect/meta depth reached" };
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!(await isSafeUrl(url))) {
|
||||||
|
return { url, type: "error", error: "Unsafe URL blocked" };
|
||||||
|
}
|
||||||
|
|
||||||
|
const controller = new AbortController();
|
||||||
|
const timeoutId = setTimeout(() => controller.abort(), FETCH_TIMEOUT_MS);
|
||||||
|
|
||||||
|
try {
|
||||||
|
const response = await fetch(url, {
|
||||||
|
signal: controller.signal,
|
||||||
|
headers: {
|
||||||
|
"User-Agent":
|
||||||
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 DiscordBot/2.0",
|
||||||
|
Accept: "image/webp,image/apng,image/*,*/*;q=0.8",
|
||||||
|
},
|
||||||
|
// Do not follow more than a few redirects natively, fetch handles up to 20 by default
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!response.ok) {
|
||||||
|
return { url, type: "error", error: `HTTP ${response.status}` };
|
||||||
|
}
|
||||||
|
|
||||||
|
const contentType = response.headers.get("content-type") || "";
|
||||||
|
const contentLength = parseInt(
|
||||||
|
response.headers.get("content-length") || "0",
|
||||||
|
10,
|
||||||
|
);
|
||||||
|
|
||||||
|
if (contentLength > MAX_FETCH_SIZE) {
|
||||||
|
return { url, type: "error", error: "Content too large" };
|
||||||
|
}
|
||||||
|
|
||||||
|
const buffer = await response.arrayBuffer();
|
||||||
|
if (buffer.byteLength > MAX_FETCH_SIZE) {
|
||||||
|
return { url, type: "error", error: "Downloaded content too large" };
|
||||||
|
}
|
||||||
|
|
||||||
|
if (contentType.startsWith("image/")) {
|
||||||
|
return {
|
||||||
|
url,
|
||||||
|
type: "image",
|
||||||
|
data: Buffer.from(buffer),
|
||||||
|
mimeType: contentType,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
if (
|
||||||
|
contentType.startsWith("text/html") ||
|
||||||
|
contentType.startsWith("text/plain")
|
||||||
|
) {
|
||||||
|
const text = Buffer.from(buffer).toString("utf-8");
|
||||||
|
|
||||||
|
// If it's HTML, try to find an og:image first (for Tenor/Giphy etc)
|
||||||
|
if (contentType.startsWith("text/html")) {
|
||||||
|
const ogImage = extractOgImage(text);
|
||||||
|
if (ogImage && ogImage.startsWith("http")) {
|
||||||
|
// Fetch the og:image instead
|
||||||
|
return fetchUrlSafely(ogImage, depth + 1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fallback to text content
|
||||||
|
const cleaned = truncateAndCleanHtml(text, 1000);
|
||||||
|
return {
|
||||||
|
url,
|
||||||
|
type: "text",
|
||||||
|
textContent: cleaned,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
url,
|
||||||
|
type: "error",
|
||||||
|
error: `Unsupported content type: ${contentType}`,
|
||||||
|
};
|
||||||
|
} catch (err) {
|
||||||
|
return {
|
||||||
|
url,
|
||||||
|
type: "error",
|
||||||
|
error: err instanceof Error ? err.message : String(err),
|
||||||
|
};
|
||||||
|
} finally {
|
||||||
|
clearTimeout(timeoutId);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function extractUrlsFromText(text: string): string[] {
|
||||||
|
const matches = text.match(URL_REGEX);
|
||||||
|
if (!matches) return [];
|
||||||
|
// Deduplicate and filter out things that obviously aren't valid
|
||||||
|
return Array.from(new Set(matches)).filter((url) => {
|
||||||
|
try {
|
||||||
|
new URL(url);
|
||||||
|
return true;
|
||||||
|
} catch {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
@@ -0,0 +1,133 @@
|
|||||||
|
import { config } from "../../shared/config/config.js";
|
||||||
|
import { createChildLogger } from "../../shared/logger/logger.js";
|
||||||
|
import { uploadToTele } from "./teleUpload.js";
|
||||||
|
import {
|
||||||
|
updateAttachmentAsFailedUpload,
|
||||||
|
updateAttachmentAsUploaded,
|
||||||
|
updateAttachmentDiscordUrl,
|
||||||
|
} from "../message-capture/messageStore.js";
|
||||||
|
|
||||||
|
const logger = createChildLogger("attachment-uploader");
|
||||||
|
|
||||||
|
class AttachmentDownloadError extends Error {
|
||||||
|
constructor(
|
||||||
|
message: string,
|
||||||
|
readonly status: number,
|
||||||
|
) {
|
||||||
|
super(message);
|
||||||
|
this.name = "AttachmentDownloadError";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export type RefreshDiscordAttachmentUrl = () => Promise<string | null>;
|
||||||
|
|
||||||
|
function toErrorMessage(error: unknown): string {
|
||||||
|
return error instanceof Error ? error.message : String(error);
|
||||||
|
}
|
||||||
|
|
||||||
|
function shouldRefreshDiscordUrl(error: unknown): boolean {
|
||||||
|
return (
|
||||||
|
error instanceof AttachmentDownloadError &&
|
||||||
|
(error.status === 403 || error.status === 404)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function uploadAttachmentToTele(
|
||||||
|
fileBuffer: Buffer,
|
||||||
|
filename: string,
|
||||||
|
contentType = "application/octet-stream",
|
||||||
|
): Promise<string> {
|
||||||
|
try {
|
||||||
|
const result = await uploadToTele({
|
||||||
|
buffer: fileBuffer,
|
||||||
|
filename,
|
||||||
|
contentType,
|
||||||
|
uploadUrl: config.TELE_UPLOAD_URL,
|
||||||
|
timeoutMs: config.ATTACHMENT_UPLOAD_TIMEOUT_MS,
|
||||||
|
retries: config.ATTACHMENT_RETRY_ATTEMPTS,
|
||||||
|
logger,
|
||||||
|
});
|
||||||
|
|
||||||
|
return result.url;
|
||||||
|
} catch (error) {
|
||||||
|
logger.error(
|
||||||
|
{
|
||||||
|
filename,
|
||||||
|
error: toErrorMessage(error),
|
||||||
|
},
|
||||||
|
"Failed to upload attachment",
|
||||||
|
);
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function downloadDiscordAttachment(url: string): Promise<Buffer> {
|
||||||
|
try {
|
||||||
|
const response = await fetch(url, {
|
||||||
|
signal: AbortSignal.timeout(config.ATTACHMENT_UPLOAD_TIMEOUT_MS),
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!response.ok) {
|
||||||
|
throw new AttachmentDownloadError(
|
||||||
|
`Download failed with status ${response.status}`,
|
||||||
|
response.status,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const buffer = await response.arrayBuffer();
|
||||||
|
return Buffer.from(buffer);
|
||||||
|
} catch (error) {
|
||||||
|
logger.error(
|
||||||
|
{ url, error: toErrorMessage(error) },
|
||||||
|
"Failed to download Discord attachment",
|
||||||
|
);
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function processAttachmentUpload(
|
||||||
|
attachmentId: string,
|
||||||
|
discordUrl: string,
|
||||||
|
filename: string,
|
||||||
|
options: {
|
||||||
|
refreshDiscordUrl?: RefreshDiscordAttachmentUrl;
|
||||||
|
contentType?: string;
|
||||||
|
} = {},
|
||||||
|
): Promise<void> {
|
||||||
|
try {
|
||||||
|
let currentDiscordUrl = discordUrl;
|
||||||
|
let buffer: Buffer;
|
||||||
|
try {
|
||||||
|
buffer = await downloadDiscordAttachment(currentDiscordUrl);
|
||||||
|
} catch (error) {
|
||||||
|
if (!options.refreshDiscordUrl || !shouldRefreshDiscordUrl(error)) {
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
|
||||||
|
const freshUrl = await options.refreshDiscordUrl();
|
||||||
|
if (!freshUrl) throw error;
|
||||||
|
currentDiscordUrl = freshUrl;
|
||||||
|
await updateAttachmentDiscordUrl(attachmentId, freshUrl);
|
||||||
|
buffer = await downloadDiscordAttachment(currentDiscordUrl);
|
||||||
|
}
|
||||||
|
|
||||||
|
const sizeMb = buffer.length / (1024 * 1024);
|
||||||
|
if (sizeMb > config.ATTACHMENT_MAX_SIZE_MB) {
|
||||||
|
throw new Error(
|
||||||
|
`File size ${sizeMb.toFixed(2)}MB exceeds limit of ${config.ATTACHMENT_MAX_SIZE_MB}MB`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const uploadedUrl = await uploadAttachmentToTele(
|
||||||
|
buffer,
|
||||||
|
filename,
|
||||||
|
options.contentType,
|
||||||
|
);
|
||||||
|
|
||||||
|
await updateAttachmentAsUploaded(attachmentId, uploadedUrl, Date.now());
|
||||||
|
} catch (error) {
|
||||||
|
const errorMsg = toErrorMessage(error);
|
||||||
|
await updateAttachmentAsFailedUpload(attachmentId, errorMsg);
|
||||||
|
logger.error({ attachmentId, error: errorMsg }, "Attachment upload failed");
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,58 @@
|
|||||||
|
import sharp from "sharp";
|
||||||
|
import { createChildLogger } from "../../shared/logger/logger.js";
|
||||||
|
|
||||||
|
const log = createChildLogger("imageResizer");
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Resize an image buffer for optimal vision LLM analysis.
|
||||||
|
*
|
||||||
|
* - Resizes to maxDim x maxDim maintaining aspect ratio
|
||||||
|
* - Converts to JPEG at quality 85 for size reduction
|
||||||
|
* - Falls back to original buffer if sharp fails
|
||||||
|
*
|
||||||
|
* @param buf - Raw image buffer
|
||||||
|
* @param maxDim - Maximum dimension in pixels (default 1024)
|
||||||
|
* @returns Resized buffer with detected MIME type
|
||||||
|
*/
|
||||||
|
export async function resizeImageForVision(
|
||||||
|
buf: Buffer,
|
||||||
|
maxDim = 1024,
|
||||||
|
): Promise<{ data: Buffer; mimeType: string }> {
|
||||||
|
try {
|
||||||
|
const metadata = await sharp(buf).metadata();
|
||||||
|
const inputFormat = metadata.format ?? "jpeg";
|
||||||
|
|
||||||
|
// Skip resize if already smaller than maxDim
|
||||||
|
if ((metadata.width ?? 0) <= maxDim && (metadata.height ?? 0) <= maxDim) {
|
||||||
|
return { data: buf, mimeType: `image/${inputFormat}` };
|
||||||
|
}
|
||||||
|
|
||||||
|
const resized = await sharp(buf)
|
||||||
|
.resize(maxDim, maxDim, {
|
||||||
|
fit: "inside",
|
||||||
|
withoutEnlargement: true,
|
||||||
|
})
|
||||||
|
.jpeg({ quality: 85 })
|
||||||
|
.toBuffer();
|
||||||
|
|
||||||
|
log.debug(
|
||||||
|
{
|
||||||
|
originalSize: buf.length,
|
||||||
|
resizedSize: resized.length,
|
||||||
|
reductionPct: Math.round(
|
||||||
|
((buf.length - resized.length) / buf.length) * 100,
|
||||||
|
),
|
||||||
|
},
|
||||||
|
"Image resized for vision analysis",
|
||||||
|
);
|
||||||
|
|
||||||
|
return { data: resized, mimeType: "image/jpeg" };
|
||||||
|
} catch (error) {
|
||||||
|
log.warn(
|
||||||
|
{ error: error instanceof Error ? error.message : String(error) },
|
||||||
|
"Image resize failed — using original buffer",
|
||||||
|
);
|
||||||
|
// Fallback: return original buffer with best-effort MIME type
|
||||||
|
return { data: buf, mimeType: "image/jpeg" };
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,2 @@
|
|||||||
|
export { processAttachmentUpload } from "./attachmentUploader.js";
|
||||||
|
export { resizeImageForVision as resizeImage } from "./imageResizer.js";
|
||||||
@@ -0,0 +1,85 @@
|
|||||||
|
import type { CustomLogger } from "../../shared/logger/logger.js";
|
||||||
|
import { retryWithBackoff } from "../../shared/utils/retry.js";
|
||||||
|
|
||||||
|
export interface TeleUploadResponse {
|
||||||
|
download_url: string;
|
||||||
|
public_id?: string;
|
||||||
|
file_name?: string;
|
||||||
|
size_bytes?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface TeleUploadResult {
|
||||||
|
url: string;
|
||||||
|
publicId?: string;
|
||||||
|
filename?: string;
|
||||||
|
sizeBytes?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function parseTeleUploadResponse(
|
||||||
|
response: TeleUploadResponse,
|
||||||
|
): TeleUploadResult {
|
||||||
|
if (!response.download_url) {
|
||||||
|
throw new Error("Missing download_url in response");
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
url: response.download_url,
|
||||||
|
publicId: response.public_id,
|
||||||
|
filename: response.file_name,
|
||||||
|
sizeBytes: response.size_bytes,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function uploadToTele(input: {
|
||||||
|
buffer: Buffer;
|
||||||
|
filename: string;
|
||||||
|
contentType: string;
|
||||||
|
uploadUrl: string;
|
||||||
|
timeoutMs?: number;
|
||||||
|
retries: number;
|
||||||
|
logger: CustomLogger;
|
||||||
|
}): Promise<TeleUploadResult> {
|
||||||
|
const {
|
||||||
|
buffer,
|
||||||
|
filename,
|
||||||
|
contentType,
|
||||||
|
uploadUrl,
|
||||||
|
timeoutMs,
|
||||||
|
retries,
|
||||||
|
logger,
|
||||||
|
} = input;
|
||||||
|
|
||||||
|
const response = await retryWithBackoff(
|
||||||
|
async () => {
|
||||||
|
const fileBlob = new Blob([new Uint8Array(buffer)], {
|
||||||
|
type: contentType,
|
||||||
|
});
|
||||||
|
const formData = new FormData();
|
||||||
|
formData.append("file", fileBlob, filename);
|
||||||
|
formData.append("fileName", filename);
|
||||||
|
|
||||||
|
const res = await fetch(uploadUrl, {
|
||||||
|
method: "POST",
|
||||||
|
headers: {
|
||||||
|
accept: "application/json",
|
||||||
|
},
|
||||||
|
body: formData,
|
||||||
|
...(timeoutMs ? { signal: AbortSignal.timeout(timeoutMs) } : {}),
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!res.ok) {
|
||||||
|
throw new Error(`Upload failed: Status ${res.status}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
return (await res.json()) as TeleUploadResponse;
|
||||||
|
},
|
||||||
|
{
|
||||||
|
retries,
|
||||||
|
minTimeout: 1000,
|
||||||
|
maxTimeout: 5000,
|
||||||
|
logger,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
return parseTeleUploadResponse(response);
|
||||||
|
}
|
||||||
@@ -0,0 +1,146 @@
|
|||||||
|
import Redis from "ioredis";
|
||||||
|
import type { CustomLogger } from "../../shared/logger/logger.js";
|
||||||
|
|
||||||
|
export interface DiscordGatewayEvent {
|
||||||
|
type: string;
|
||||||
|
data: unknown;
|
||||||
|
timestamp: number;
|
||||||
|
source: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export class RedisEventPublisher {
|
||||||
|
private redis: Redis;
|
||||||
|
private logger: CustomLogger;
|
||||||
|
|
||||||
|
constructor(redisUrl: string, logger: CustomLogger) {
|
||||||
|
this.redis = new Redis(redisUrl);
|
||||||
|
this.logger = logger;
|
||||||
|
|
||||||
|
this.redis.on("error", (err) => {
|
||||||
|
this.logger.error({ error: err }, "Redis connection error");
|
||||||
|
});
|
||||||
|
|
||||||
|
this.redis.on("connect", () => {
|
||||||
|
this.logger.info("Redis connected");
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async publish(channel: string, event: DiscordGatewayEvent): Promise<void> {
|
||||||
|
try {
|
||||||
|
await this.redis.publish(channel, JSON.stringify(event));
|
||||||
|
} catch (error) {
|
||||||
|
this.logger.error(
|
||||||
|
{ error, channel, eventType: event.type },
|
||||||
|
"Failed to publish event",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async close(): Promise<void> {
|
||||||
|
await this.redis.quit();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export class EventBroadcaster {
|
||||||
|
private publisher: RedisEventPublisher;
|
||||||
|
private logger: CustomLogger;
|
||||||
|
|
||||||
|
constructor(publisher: RedisEventPublisher, logger: CustomLogger) {
|
||||||
|
this.publisher = publisher;
|
||||||
|
this.logger = logger;
|
||||||
|
}
|
||||||
|
|
||||||
|
async messageCreated(data: unknown): Promise<void> {
|
||||||
|
await this.publisher.publish("discord:message:created", {
|
||||||
|
type: "message_created",
|
||||||
|
data,
|
||||||
|
timestamp: Date.now(),
|
||||||
|
source: "discord-gateway",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async messageUpdated(data: unknown): Promise<void> {
|
||||||
|
await this.publisher.publish("discord:message:updated", {
|
||||||
|
type: "message_updated",
|
||||||
|
data,
|
||||||
|
timestamp: Date.now(),
|
||||||
|
source: "discord-gateway",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async messageDeleted(data: unknown): Promise<void> {
|
||||||
|
await this.publisher.publish("discord:message:deleted", {
|
||||||
|
type: "message_deleted",
|
||||||
|
data,
|
||||||
|
timestamp: Date.now(),
|
||||||
|
source: "discord-gateway",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async messageAnalyzed(data: unknown): Promise<void> {
|
||||||
|
await this.publisher.publish("discord:message:analyzed", {
|
||||||
|
type: "message_analyzed",
|
||||||
|
data,
|
||||||
|
timestamp: Date.now(),
|
||||||
|
source: "discord-gateway",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async attachmentCreated(data: unknown): Promise<void> {
|
||||||
|
await this.publisher.publish("discord:attachment:created", {
|
||||||
|
type: "attachment_created",
|
||||||
|
data,
|
||||||
|
timestamp: Date.now(),
|
||||||
|
source: "discord-gateway",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async attachmentUploaded(data: unknown): Promise<void> {
|
||||||
|
await this.publisher.publish("discord:attachment:uploaded", {
|
||||||
|
type: "attachment_uploaded",
|
||||||
|
data,
|
||||||
|
timestamp: Date.now(),
|
||||||
|
source: "discord-gateway",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async voiceRecordingStarted(data: unknown): Promise<void> {
|
||||||
|
await this.publisher.publish("discord:voice:started", {
|
||||||
|
type: "voice_recording_started",
|
||||||
|
data,
|
||||||
|
timestamp: Date.now(),
|
||||||
|
source: "discord-gateway",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async voiceRecordingStopped(data: unknown): Promise<void> {
|
||||||
|
await this.publisher.publish("discord:voice:stopped", {
|
||||||
|
type: "voice_recording_stopped",
|
||||||
|
data,
|
||||||
|
timestamp: Date.now(),
|
||||||
|
source: "discord-gateway",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async voiceRecordingUploaded(data: unknown): Promise<void> {
|
||||||
|
await this.publisher.publish("discord:voice:uploaded", {
|
||||||
|
type: "voice_recording_uploaded",
|
||||||
|
data,
|
||||||
|
timestamp: Date.now(),
|
||||||
|
source: "discord-gateway",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async analysisQueueStatus(data: unknown): Promise<void> {
|
||||||
|
await this.publisher.publish("discord:analysis:queue_status", {
|
||||||
|
type: "analysis_queue_status",
|
||||||
|
data,
|
||||||
|
timestamp: Date.now(),
|
||||||
|
source: "discord-gateway",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async close(): Promise<void> {
|
||||||
|
await this.publisher.close();
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,22 @@
|
|||||||
|
export interface DiscordGatewayEvent {
|
||||||
|
type: string;
|
||||||
|
data: unknown;
|
||||||
|
timestamp: number;
|
||||||
|
source: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export const EventChannels = {
|
||||||
|
MESSAGE_CREATED: "discord:message:created",
|
||||||
|
MESSAGE_UPDATED: "discord:message:updated",
|
||||||
|
MESSAGE_DELETED: "discord:message:deleted",
|
||||||
|
MESSAGE_ANALYZED: "discord:message:analyzed",
|
||||||
|
ATTACHMENT_CREATED: "discord:attachment:created",
|
||||||
|
ATTACHMENT_UPLOADED: "discord:attachment:uploaded",
|
||||||
|
VOICE_STARTED: "discord:voice:started",
|
||||||
|
VOICE_STOPPED: "discord:voice:stopped",
|
||||||
|
VOICE_UPLOADED: "discord:voice:uploaded",
|
||||||
|
ANALYSIS_QUEUE_STATUS: "discord:analysis:queue_status",
|
||||||
|
} as const;
|
||||||
|
|
||||||
|
export type EventChannelType =
|
||||||
|
(typeof EventChannels)[keyof typeof EventChannels];
|
||||||
@@ -0,0 +1,6 @@
|
|||||||
|
export { EventBroadcaster, RedisEventPublisher } from "./eventBroadcaster.js";
|
||||||
|
export {
|
||||||
|
type DiscordGatewayEvent,
|
||||||
|
EventChannels,
|
||||||
|
type EventChannelType,
|
||||||
|
} from "./eventTypes.js";
|
||||||
@@ -0,0 +1,929 @@
|
|||||||
|
import { executeAll, executeGet } from "../../shared/database/drizzle.js";
|
||||||
|
import { createChildLogger } from "../../shared/logger/logger.js";
|
||||||
|
import type { MessageRecord } from "./types.js";
|
||||||
|
|
||||||
|
const logger = createChildLogger("analytics-store");
|
||||||
|
|
||||||
|
// ── Types ──────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
export interface HourlyBucket {
|
||||||
|
hour: string;
|
||||||
|
count: number;
|
||||||
|
clean: number;
|
||||||
|
warned: number;
|
||||||
|
flagged: number;
|
||||||
|
error: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface TopicTrend {
|
||||||
|
topic: string;
|
||||||
|
count: number;
|
||||||
|
score: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface UserStat {
|
||||||
|
user_id: string;
|
||||||
|
username: string;
|
||||||
|
avatar_url: string | null;
|
||||||
|
message_count: number;
|
||||||
|
edited_count: number;
|
||||||
|
deleted_count: number;
|
||||||
|
flagged_count: number;
|
||||||
|
last_active: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ModerationBreakdown {
|
||||||
|
total: number;
|
||||||
|
clean: number;
|
||||||
|
warned: number;
|
||||||
|
flagged: number;
|
||||||
|
error: number;
|
||||||
|
pending: number;
|
||||||
|
average_score: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface AnalyticsOverview {
|
||||||
|
period: { start: number; end: number };
|
||||||
|
messages: ModerationBreakdown;
|
||||||
|
hourly: HourlyBucket[];
|
||||||
|
topics: TopicTrend[];
|
||||||
|
top_users: UserStat[];
|
||||||
|
active_users_count: number;
|
||||||
|
total_channels: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ══════════════════════════════════════════════════════════════════════════
|
||||||
|
// GENERIC QUERY CACHE (reduces duplicate DB calls from 5s auto-refresh)
|
||||||
|
// ══════════════════════════════════════════════════════════════════════════
|
||||||
|
|
||||||
|
interface CacheEntry<T> {
|
||||||
|
data: T;
|
||||||
|
expiresAt: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
const queryCache = new Map<string, CacheEntry<any>>();
|
||||||
|
|
||||||
|
/** Default TTL for aggregate queries — 10s is long enough to prevent redundant
|
||||||
|
* calls from the 5s auto-refresh but short enough to feel real-time. */
|
||||||
|
const AGGREGATE_CACHE_TTL_MS = 10_000;
|
||||||
|
|
||||||
|
/** Topic extraction is expensive (JSON parsing). Cache longer. */
|
||||||
|
const TOPIC_CACHE_TTL_MS = 120_000;
|
||||||
|
|
||||||
|
function makeCacheKey(prefix: string, params: Record<string, any>): string {
|
||||||
|
return `${prefix}:${JSON.stringify(params)}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function getCached<T>(key: string): T | undefined {
|
||||||
|
const entry = queryCache.get(key);
|
||||||
|
if (entry && entry.expiresAt > Date.now()) return entry.data;
|
||||||
|
if (entry) queryCache.delete(key); // expired
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
function setCache<T>(key: string, data: T, ttl: number): void {
|
||||||
|
queryCache.set(key, { data, expiresAt: Date.now() + ttl });
|
||||||
|
// Prune old entries if cache grows too large (>200 entries)
|
||||||
|
if (queryCache.size > 200) {
|
||||||
|
const now = Date.now();
|
||||||
|
for (const [k, v] of queryCache) {
|
||||||
|
if (v.expiresAt <= now) queryCache.delete(k);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Hourly Message Stats ───────────────────────────────────────────────
|
||||||
|
|
||||||
|
export async function getHourlyStats(input: {
|
||||||
|
guildId: string;
|
||||||
|
channelId?: string;
|
||||||
|
hours?: number;
|
||||||
|
}): Promise<HourlyBucket[]> {
|
||||||
|
const { guildId, channelId, hours = 24 } = input;
|
||||||
|
const cacheKey = makeCacheKey("hourly", { guildId, channelId, hours });
|
||||||
|
const cached = getCached<HourlyBucket[]>(cacheKey);
|
||||||
|
if (cached) return cached;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const since = Date.now() - hours * 3600_000;
|
||||||
|
const hourExpr = `to_char(to_timestamp((created_at / 3600000) * 3600), 'YYYY-MM-DD HH24:MI:SS') as hour`;
|
||||||
|
|
||||||
|
const rows = await executeAll(
|
||||||
|
`
|
||||||
|
SELECT
|
||||||
|
${hourExpr},
|
||||||
|
count(*) as count,
|
||||||
|
count(case when ai_status = 'clean' then 1 end) as clean,
|
||||||
|
count(case when ai_status = 'warn' then 1 end) as warned,
|
||||||
|
count(case when ai_status = 'flagged' then 1 end) as flagged,
|
||||||
|
count(case when ai_status = 'error' then 1 end) as error
|
||||||
|
FROM messages
|
||||||
|
WHERE guild_id = ?
|
||||||
|
AND created_at >= ?
|
||||||
|
AND deleted_at IS NULL
|
||||||
|
${channelId ? `AND (channel_id = ? OR thread_id = ?)` : ""}
|
||||||
|
GROUP BY (created_at / 3600000)
|
||||||
|
ORDER BY hour ASC
|
||||||
|
`,
|
||||||
|
channelId ? [guildId, since, channelId, channelId] : [guildId, since],
|
||||||
|
);
|
||||||
|
|
||||||
|
// Initialize all hour buckets (fill gaps with zeros)
|
||||||
|
const buckets = new Map<
|
||||||
|
string,
|
||||||
|
{
|
||||||
|
count: number;
|
||||||
|
clean: number;
|
||||||
|
warned: number;
|
||||||
|
flagged: number;
|
||||||
|
error: number;
|
||||||
|
}
|
||||||
|
>();
|
||||||
|
|
||||||
|
for (let h = 0; h < hours; h++) {
|
||||||
|
const ts = new Date(since + h * 3600_000);
|
||||||
|
ts.setMinutes(0, 0, 0);
|
||||||
|
const key = ts.toISOString().slice(0, 13) + ":00:00Z";
|
||||||
|
buckets.set(key, { count: 0, clean: 0, warned: 0, flagged: 0, error: 0 });
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const row of rows) {
|
||||||
|
const d = new Date(row.hour.replace(" ", "T") + "Z");
|
||||||
|
const key = d.toISOString().slice(0, 13) + ":00:00Z";
|
||||||
|
const bucket = buckets.get(key);
|
||||||
|
if (!bucket) continue;
|
||||||
|
bucket.count = row.count;
|
||||||
|
bucket.clean = row.clean;
|
||||||
|
bucket.warned = row.warned;
|
||||||
|
bucket.flagged = row.flagged;
|
||||||
|
bucket.error = row.error;
|
||||||
|
}
|
||||||
|
|
||||||
|
const result = Array.from(buckets.entries())
|
||||||
|
.sort(([a], [b]) => a.localeCompare(b))
|
||||||
|
.map(([hour, data]) => ({ hour, ...data }));
|
||||||
|
|
||||||
|
setCache(cacheKey, result, AGGREGATE_CACHE_TTL_MS);
|
||||||
|
return result;
|
||||||
|
} catch (error) {
|
||||||
|
logger.error(
|
||||||
|
{ error: error instanceof Error ? error.message : String(error) },
|
||||||
|
"Failed to get hourly stats",
|
||||||
|
);
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Topic Trends ───────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
const STOP_WORDS = new Set([
|
||||||
|
"yang",
|
||||||
|
"dan",
|
||||||
|
"itu",
|
||||||
|
"ini",
|
||||||
|
"dengan",
|
||||||
|
"akan",
|
||||||
|
"pada",
|
||||||
|
"dari",
|
||||||
|
"di",
|
||||||
|
"ke",
|
||||||
|
"untuk",
|
||||||
|
"tidak",
|
||||||
|
"ada",
|
||||||
|
"juga",
|
||||||
|
"sudah",
|
||||||
|
"saya",
|
||||||
|
"kamu",
|
||||||
|
"dia",
|
||||||
|
"mereka",
|
||||||
|
"kami",
|
||||||
|
"aku",
|
||||||
|
"lo",
|
||||||
|
"lu",
|
||||||
|
"gua",
|
||||||
|
"gue",
|
||||||
|
"org",
|
||||||
|
"orang",
|
||||||
|
"aja",
|
||||||
|
"sama",
|
||||||
|
"kalo",
|
||||||
|
"kalau",
|
||||||
|
"bisa",
|
||||||
|
"karena",
|
||||||
|
"gak",
|
||||||
|
"nggak",
|
||||||
|
"ga",
|
||||||
|
"tak",
|
||||||
|
"belum",
|
||||||
|
"udah",
|
||||||
|
"dah",
|
||||||
|
"lah",
|
||||||
|
"kah",
|
||||||
|
"pun",
|
||||||
|
"nih",
|
||||||
|
"tuh",
|
||||||
|
"deh",
|
||||||
|
"dong",
|
||||||
|
"si",
|
||||||
|
"nya",
|
||||||
|
"kan",
|
||||||
|
"ya",
|
||||||
|
"yah",
|
||||||
|
"yuk",
|
||||||
|
"kok",
|
||||||
|
"loh",
|
||||||
|
"nah",
|
||||||
|
"wow",
|
||||||
|
"eh",
|
||||||
|
"the",
|
||||||
|
"a",
|
||||||
|
"an",
|
||||||
|
"is",
|
||||||
|
"are",
|
||||||
|
"was",
|
||||||
|
"were",
|
||||||
|
"be",
|
||||||
|
"been",
|
||||||
|
"being",
|
||||||
|
"have",
|
||||||
|
"has",
|
||||||
|
"had",
|
||||||
|
"having",
|
||||||
|
"do",
|
||||||
|
"does",
|
||||||
|
"did",
|
||||||
|
"doing",
|
||||||
|
"will",
|
||||||
|
"would",
|
||||||
|
"could",
|
||||||
|
"should",
|
||||||
|
"may",
|
||||||
|
"might",
|
||||||
|
"must",
|
||||||
|
"shall",
|
||||||
|
"i",
|
||||||
|
"you",
|
||||||
|
"he",
|
||||||
|
"she",
|
||||||
|
"it",
|
||||||
|
"we",
|
||||||
|
"they",
|
||||||
|
"me",
|
||||||
|
"him",
|
||||||
|
"her",
|
||||||
|
"us",
|
||||||
|
"them",
|
||||||
|
"my",
|
||||||
|
"your",
|
||||||
|
"his",
|
||||||
|
"its",
|
||||||
|
"our",
|
||||||
|
"their",
|
||||||
|
"and",
|
||||||
|
"but",
|
||||||
|
"or",
|
||||||
|
"nor",
|
||||||
|
"not",
|
||||||
|
"so",
|
||||||
|
"yet",
|
||||||
|
"for",
|
||||||
|
"if",
|
||||||
|
"to",
|
||||||
|
"of",
|
||||||
|
"in",
|
||||||
|
"on",
|
||||||
|
"at",
|
||||||
|
"by",
|
||||||
|
"as",
|
||||||
|
"with",
|
||||||
|
"about",
|
||||||
|
"just",
|
||||||
|
"then",
|
||||||
|
"now",
|
||||||
|
"here",
|
||||||
|
"there",
|
||||||
|
"when",
|
||||||
|
"where",
|
||||||
|
"why",
|
||||||
|
"how",
|
||||||
|
"all",
|
||||||
|
"both",
|
||||||
|
"each",
|
||||||
|
"few",
|
||||||
|
"more",
|
||||||
|
"most",
|
||||||
|
"other",
|
||||||
|
"some",
|
||||||
|
"such",
|
||||||
|
"only",
|
||||||
|
"own",
|
||||||
|
"same",
|
||||||
|
"too",
|
||||||
|
"very",
|
||||||
|
"can",
|
||||||
|
"go",
|
||||||
|
"ok",
|
||||||
|
"okay",
|
||||||
|
"yeah",
|
||||||
|
"yes",
|
||||||
|
"no",
|
||||||
|
]);
|
||||||
|
|
||||||
|
function extractTopics(messages: MessageRecord[], topN = 15): TopicTrend[] {
|
||||||
|
const topicScores = new Map<string, { count: number; score: number }>();
|
||||||
|
const wordFreq = new Map<string, number>();
|
||||||
|
const flaggedWordFreq = new Map<string, number>();
|
||||||
|
|
||||||
|
for (const msg of messages) {
|
||||||
|
if (msg.ai_analysis) {
|
||||||
|
try {
|
||||||
|
const analysis = JSON.parse(msg.ai_analysis);
|
||||||
|
const topics = analysis.topics;
|
||||||
|
if (topics && Array.isArray(topics)) {
|
||||||
|
for (const topic of topics) {
|
||||||
|
const key =
|
||||||
|
typeof topic === "string" ? topic : topic.name || topic.topic;
|
||||||
|
if (!key) continue;
|
||||||
|
const k = key.toLowerCase();
|
||||||
|
const score = msg.ai_moderation_score || 0;
|
||||||
|
const existing = topicScores.get(k);
|
||||||
|
if (existing) {
|
||||||
|
existing.count++;
|
||||||
|
existing.score += score;
|
||||||
|
} else {
|
||||||
|
topicScores.set(k, { count: 1, score });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (analysis.category) {
|
||||||
|
const cat = String(analysis.category).toLowerCase();
|
||||||
|
const existing = topicScores.get(cat);
|
||||||
|
if (existing) {
|
||||||
|
existing.count++;
|
||||||
|
existing.score += msg.ai_moderation_score || 0;
|
||||||
|
} else {
|
||||||
|
topicScores.set(cat, {
|
||||||
|
count: 1,
|
||||||
|
score: msg.ai_moderation_score || 0,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
/* not valid JSON */
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (msg.content) {
|
||||||
|
const words = msg.content
|
||||||
|
.toLowerCase()
|
||||||
|
.replace(/[^\w\s]/g, " ")
|
||||||
|
.split(/\s+/)
|
||||||
|
.filter((w) => w.length > 2 && !STOP_WORDS.has(w));
|
||||||
|
|
||||||
|
for (const word of words) {
|
||||||
|
wordFreq.set(word, (wordFreq.get(word) || 0) + 1);
|
||||||
|
if (msg.ai_status === "flagged" || msg.ai_status === "warn") {
|
||||||
|
flaggedWordFreq.set(word, (flaggedWordFreq.get(word) || 0) + 1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const results: TopicTrend[] = [];
|
||||||
|
for (const [topic, data] of topicScores) {
|
||||||
|
results.push({ topic, count: data.count, score: data.score });
|
||||||
|
}
|
||||||
|
|
||||||
|
const sortedWords = Array.from(wordFreq.entries())
|
||||||
|
.sort(([, a], [, b]) => b - a)
|
||||||
|
.slice(0, topN);
|
||||||
|
|
||||||
|
for (const [word, count] of sortedWords) {
|
||||||
|
if (!topicScores.has(word)) {
|
||||||
|
results.push({
|
||||||
|
topic: word,
|
||||||
|
count,
|
||||||
|
score: flaggedWordFreq.get(word) || 0,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return results.sort((a, b) => b.count - a.count).slice(0, topN);
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function getTopicTrends(input: {
|
||||||
|
guildId: string;
|
||||||
|
channelId?: string;
|
||||||
|
hours?: number;
|
||||||
|
}): Promise<TopicTrend[]> {
|
||||||
|
const { guildId, channelId, hours = 24 } = input;
|
||||||
|
const cacheKey = makeCacheKey("topics", { guildId, channelId, hours });
|
||||||
|
const cached = getCached<TopicTrend[]>(cacheKey);
|
||||||
|
if (cached) return cached;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const since = Date.now() - hours * 3600_000;
|
||||||
|
|
||||||
|
// Fetch all analyzed messages within the time window (no hard row cap).
|
||||||
|
// Messages without ai_analysis are excluded which naturally limits rows.
|
||||||
|
const rows = (await executeAll(
|
||||||
|
`
|
||||||
|
SELECT
|
||||||
|
id, content, ai_status, ai_analysis, ai_moderation_score,
|
||||||
|
ai_moderation_flags, created_at
|
||||||
|
FROM messages
|
||||||
|
WHERE guild_id = ?
|
||||||
|
AND created_at >= ?
|
||||||
|
AND deleted_at IS NULL
|
||||||
|
AND ai_analysis IS NOT NULL
|
||||||
|
${channelId ? `AND (channel_id = ? OR thread_id = ?)` : ""}
|
||||||
|
ORDER BY created_at DESC
|
||||||
|
`,
|
||||||
|
channelId ? [guildId, since, channelId, channelId] : [guildId, since],
|
||||||
|
)) as MessageRecord[];
|
||||||
|
|
||||||
|
const result = extractTopics(rows);
|
||||||
|
setCache(cacheKey, result, TOPIC_CACHE_TTL_MS);
|
||||||
|
return result;
|
||||||
|
} catch (error) {
|
||||||
|
logger.error(
|
||||||
|
{ error: error instanceof Error ? error.message : String(error) },
|
||||||
|
"Failed to get topic trends",
|
||||||
|
);
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── User Leaderboard ────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
export async function getUserLeaderboard(input: {
|
||||||
|
guildId: string;
|
||||||
|
channelId?: string;
|
||||||
|
hours?: number;
|
||||||
|
limit?: number;
|
||||||
|
}): Promise<UserStat[]> {
|
||||||
|
const { guildId, channelId, hours = 24, limit = 20 } = input;
|
||||||
|
const cacheKey = makeCacheKey("leaderboard", {
|
||||||
|
guildId,
|
||||||
|
channelId,
|
||||||
|
hours,
|
||||||
|
limit,
|
||||||
|
});
|
||||||
|
const cached = getCached<UserStat[]>(cacheKey);
|
||||||
|
if (cached) return cached;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const since = Date.now() - hours * 3600_000;
|
||||||
|
const rows = await executeAll(
|
||||||
|
`
|
||||||
|
SELECT
|
||||||
|
user_id,
|
||||||
|
username,
|
||||||
|
avatar_url,
|
||||||
|
count(*) as message_count,
|
||||||
|
count(case when type = 'edited' then 1 end) as edited_count,
|
||||||
|
count(case when type = 'deleted' then 1 end) as deleted_count,
|
||||||
|
count(case when ai_status = 'flagged' then 1 end) as flagged_count,
|
||||||
|
max(created_at) as last_active
|
||||||
|
FROM messages
|
||||||
|
WHERE guild_id = ?
|
||||||
|
AND created_at >= ?
|
||||||
|
AND deleted_at IS NULL
|
||||||
|
${channelId ? `AND (channel_id = ? OR thread_id = ?)` : ""}
|
||||||
|
GROUP BY user_id, username, avatar_url
|
||||||
|
ORDER BY message_count DESC
|
||||||
|
LIMIT ?
|
||||||
|
`,
|
||||||
|
channelId
|
||||||
|
? [guildId, since, channelId, channelId, limit]
|
||||||
|
: [guildId, since, limit],
|
||||||
|
);
|
||||||
|
|
||||||
|
const result = rows as UserStat[];
|
||||||
|
setCache(cacheKey, result, AGGREGATE_CACHE_TTL_MS);
|
||||||
|
return result;
|
||||||
|
} catch (error) {
|
||||||
|
logger.error(
|
||||||
|
{ error: error instanceof Error ? error.message : String(error) },
|
||||||
|
"Failed to get user leaderboard",
|
||||||
|
);
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Moderation Stats ───────────────────────────────────────────────────
|
||||||
|
|
||||||
|
export async function getModerationStats(input: {
|
||||||
|
guildId: string;
|
||||||
|
channelId?: string;
|
||||||
|
hours?: number;
|
||||||
|
}): Promise<ModerationBreakdown> {
|
||||||
|
const { guildId, channelId, hours = 24 } = input;
|
||||||
|
const cacheKey = makeCacheKey("modstats", { guildId, channelId, hours });
|
||||||
|
const cached = getCached<ModerationBreakdown>(cacheKey);
|
||||||
|
if (cached) return cached;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const since = Date.now() - hours * 3600_000;
|
||||||
|
const avgScoreExpr = `round(avg(ai_moderation_score)::numeric, 2)`;
|
||||||
|
|
||||||
|
const row = await executeGet(
|
||||||
|
`
|
||||||
|
SELECT
|
||||||
|
count(*) as total,
|
||||||
|
count(case when ai_status = 'clean' then 1 end) as clean,
|
||||||
|
count(case when ai_status = 'warn' then 1 end) as warned,
|
||||||
|
count(case when ai_status = 'flagged' then 1 end) as flagged,
|
||||||
|
count(case when ai_status = 'error' then 1 end) as error,
|
||||||
|
count(case when ai_status = 'pending' or ai_status IS NULL then 1 end) as pending,
|
||||||
|
${avgScoreExpr} as average_score
|
||||||
|
FROM messages
|
||||||
|
WHERE guild_id = ?
|
||||||
|
AND created_at >= ?
|
||||||
|
AND deleted_at IS NULL
|
||||||
|
${channelId ? `AND (channel_id = ? OR thread_id = ?)` : ""}
|
||||||
|
`,
|
||||||
|
channelId ? [guildId, since, channelId, channelId] : [guildId, since],
|
||||||
|
);
|
||||||
|
|
||||||
|
const result: ModerationBreakdown = row
|
||||||
|
? {
|
||||||
|
total: row.total ?? 0,
|
||||||
|
clean: row.clean ?? 0,
|
||||||
|
warned: row.warned ?? 0,
|
||||||
|
flagged: row.flagged ?? 0,
|
||||||
|
error: row.error ?? 0,
|
||||||
|
pending: row.pending ?? 0,
|
||||||
|
average_score: row.average_score ?? 0,
|
||||||
|
}
|
||||||
|
: {
|
||||||
|
total: 0,
|
||||||
|
clean: 0,
|
||||||
|
warned: 0,
|
||||||
|
flagged: 0,
|
||||||
|
error: 0,
|
||||||
|
pending: 0,
|
||||||
|
average_score: 0,
|
||||||
|
};
|
||||||
|
|
||||||
|
setCache(cacheKey, result, AGGREGATE_CACHE_TTL_MS);
|
||||||
|
return result;
|
||||||
|
} catch (error) {
|
||||||
|
logger.error(
|
||||||
|
{ error: error instanceof Error ? error.message : String(error) },
|
||||||
|
"Failed to get moderation stats",
|
||||||
|
);
|
||||||
|
return {
|
||||||
|
total: 0,
|
||||||
|
clean: 0,
|
||||||
|
warned: 0,
|
||||||
|
flagged: 0,
|
||||||
|
error: 0,
|
||||||
|
pending: 0,
|
||||||
|
average_score: 0,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Active Channels Count ──────────────────────────────────────────────
|
||||||
|
|
||||||
|
export async function getActiveChannelCount(input: {
|
||||||
|
guildId: string;
|
||||||
|
hours?: number;
|
||||||
|
}): Promise<number> {
|
||||||
|
const { guildId, hours = 24 } = input;
|
||||||
|
const cacheKey = makeCacheKey("channels", { guildId, hours });
|
||||||
|
const cached = getCached<number>(cacheKey);
|
||||||
|
if (cached !== undefined) return cached;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const since = Date.now() - hours * 3600_000;
|
||||||
|
const row = await executeGet(
|
||||||
|
`
|
||||||
|
SELECT count(DISTINCT channel_id) as cnt
|
||||||
|
FROM messages
|
||||||
|
WHERE guild_id = ?
|
||||||
|
AND created_at >= ?
|
||||||
|
AND deleted_at IS NULL
|
||||||
|
`,
|
||||||
|
[guildId, since],
|
||||||
|
);
|
||||||
|
|
||||||
|
const result = row?.cnt ?? 0;
|
||||||
|
setCache(cacheKey, result, AGGREGATE_CACHE_TTL_MS);
|
||||||
|
return result;
|
||||||
|
} catch (error) {
|
||||||
|
logger.error(
|
||||||
|
{ error: error instanceof Error ? error.message : String(error) },
|
||||||
|
"Failed to get active channel count",
|
||||||
|
);
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Top Violators ─────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
export interface ViolatorStat {
|
||||||
|
user_id: string;
|
||||||
|
username: string;
|
||||||
|
avatar_url: string | null;
|
||||||
|
total_messages: number;
|
||||||
|
flagged_count: number;
|
||||||
|
warned_count: number;
|
||||||
|
violation_score: number;
|
||||||
|
worst_flags: string[];
|
||||||
|
last_violation: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function getTopViolators(input: {
|
||||||
|
guildId: string;
|
||||||
|
channelId?: string;
|
||||||
|
hours?: number;
|
||||||
|
limit?: number;
|
||||||
|
}): Promise<ViolatorStat[]> {
|
||||||
|
const { guildId, channelId, hours = 24, limit = 20 } = input;
|
||||||
|
const cacheKey = makeCacheKey("violators", {
|
||||||
|
guildId,
|
||||||
|
channelId,
|
||||||
|
hours,
|
||||||
|
limit,
|
||||||
|
});
|
||||||
|
const cached = getCached<ViolatorStat[]>(cacheKey);
|
||||||
|
if (cached) return cached;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const since = Date.now() - hours * 3600_000;
|
||||||
|
const rows = await executeAll(
|
||||||
|
`
|
||||||
|
SELECT
|
||||||
|
user_id,
|
||||||
|
username,
|
||||||
|
avatar_url,
|
||||||
|
count(*) as total_messages,
|
||||||
|
count(case when ai_status = 'flagged' then 1 end) as flagged_count,
|
||||||
|
count(case when ai_status = 'warn' then 1 end) as warned_count,
|
||||||
|
max(case when ai_status in ('flagged', 'warn') then created_at else 0 end) as last_violation
|
||||||
|
FROM messages
|
||||||
|
WHERE guild_id = ?
|
||||||
|
AND created_at >= ?
|
||||||
|
AND deleted_at IS NULL
|
||||||
|
${channelId ? `AND (channel_id = ? OR thread_id = ?)` : ""}
|
||||||
|
GROUP BY user_id, username, avatar_url
|
||||||
|
HAVING count(case when ai_status = 'flagged' then 1 end) > 0
|
||||||
|
OR count(case when ai_status = 'warn' then 1 end) > 0
|
||||||
|
ORDER BY (
|
||||||
|
count(case when ai_status = 'flagged' then 1 end) * 3
|
||||||
|
+ count(case when ai_status = 'warn' then 1 end)
|
||||||
|
) DESC
|
||||||
|
LIMIT ?
|
||||||
|
`,
|
||||||
|
channelId
|
||||||
|
? [guildId, since, channelId, channelId, limit]
|
||||||
|
: [guildId, since, limit],
|
||||||
|
);
|
||||||
|
|
||||||
|
const violators: ViolatorStat[] = rows.map((row: any) => {
|
||||||
|
const flaggedCount = Number(row.flagged_count ?? 0);
|
||||||
|
const warnedCount = Number(row.warned_count ?? 0);
|
||||||
|
return {
|
||||||
|
user_id: row.user_id,
|
||||||
|
username: row.username,
|
||||||
|
avatar_url: row.avatar_url,
|
||||||
|
total_messages: Number(row.total_messages ?? 0),
|
||||||
|
flagged_count: flaggedCount,
|
||||||
|
warned_count: warnedCount,
|
||||||
|
violation_score: flaggedCount * 3 + warnedCount,
|
||||||
|
worst_flags: [],
|
||||||
|
last_violation: Number(row.last_violation ?? 0),
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
setCache(cacheKey, violators, AGGREGATE_CACHE_TTL_MS);
|
||||||
|
return violators;
|
||||||
|
} catch (error) {
|
||||||
|
logger.error(
|
||||||
|
{ error: error instanceof Error ? error.message : String(error) },
|
||||||
|
"Failed to get top violators",
|
||||||
|
);
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Daily Trend (for multi-day line chart) ────────────────────────────
|
||||||
|
|
||||||
|
export interface TrendBucket {
|
||||||
|
date: string;
|
||||||
|
count: number;
|
||||||
|
clean: number;
|
||||||
|
warned: number;
|
||||||
|
flagged: number;
|
||||||
|
error: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function getDailyTrend(input: {
|
||||||
|
guildId: string;
|
||||||
|
channelId?: string;
|
||||||
|
hours?: number;
|
||||||
|
}): Promise<TrendBucket[]> {
|
||||||
|
const { guildId, channelId, hours = 168 } = input;
|
||||||
|
const cacheKey = makeCacheKey("daily_trend", { guildId, channelId, hours });
|
||||||
|
const cached = getCached<TrendBucket[]>(cacheKey);
|
||||||
|
if (cached) return cached;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const since = Date.now() - hours * 3600_000;
|
||||||
|
const dateExpr = `to_char(date_trunc('day', to_timestamp(created_at / 1000)), 'YYYY-MM-DD') as date`;
|
||||||
|
|
||||||
|
const rows = await executeAll(
|
||||||
|
`
|
||||||
|
SELECT
|
||||||
|
${dateExpr},
|
||||||
|
count(*) as count,
|
||||||
|
count(case when ai_status = 'clean' then 1 end) as clean,
|
||||||
|
count(case when ai_status = 'warn' then 1 end) as warned,
|
||||||
|
count(case when ai_status = 'flagged' then 1 end) as flagged,
|
||||||
|
count(case when ai_status = 'error' then 1 end) as error
|
||||||
|
FROM messages
|
||||||
|
WHERE guild_id = ?
|
||||||
|
AND created_at >= ?
|
||||||
|
AND deleted_at IS NULL
|
||||||
|
${channelId ? `AND (channel_id = ? OR thread_id = ?)` : ""}
|
||||||
|
GROUP BY 1
|
||||||
|
ORDER BY 1 ASC
|
||||||
|
`,
|
||||||
|
channelId ? [guildId, since, channelId, channelId] : [guildId, since],
|
||||||
|
);
|
||||||
|
|
||||||
|
// Initialize all day buckets (fill gaps with zeros)
|
||||||
|
const buckets = new Map<
|
||||||
|
string,
|
||||||
|
{
|
||||||
|
count: number;
|
||||||
|
clean: number;
|
||||||
|
warned: number;
|
||||||
|
flagged: number;
|
||||||
|
error: number;
|
||||||
|
}
|
||||||
|
>();
|
||||||
|
const msPerDay = 86400_000;
|
||||||
|
const startDay = Math.floor(since / msPerDay) * msPerDay;
|
||||||
|
const endDay = Math.floor(Date.now() / msPerDay) * msPerDay;
|
||||||
|
|
||||||
|
for (let d = startDay; d <= endDay; d += msPerDay) {
|
||||||
|
const key = new Date(d).toISOString().slice(0, 10);
|
||||||
|
buckets.set(key, { count: 0, clean: 0, warned: 0, flagged: 0, error: 0 });
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const row of rows) {
|
||||||
|
const bucket = buckets.get(row.date);
|
||||||
|
if (!bucket) continue;
|
||||||
|
bucket.count = row.count;
|
||||||
|
bucket.clean = row.clean;
|
||||||
|
bucket.warned = row.warned;
|
||||||
|
bucket.flagged = row.flagged;
|
||||||
|
bucket.error = row.error;
|
||||||
|
}
|
||||||
|
|
||||||
|
const result = Array.from(buckets.entries())
|
||||||
|
.sort(([a], [b]) => a.localeCompare(b))
|
||||||
|
.map(([date, data]) => ({ date, ...data }));
|
||||||
|
|
||||||
|
setCache(cacheKey, result, AGGREGATE_CACHE_TTL_MS);
|
||||||
|
return result;
|
||||||
|
} catch (error) {
|
||||||
|
logger.error(
|
||||||
|
{ error: error instanceof Error ? error.message : String(error) },
|
||||||
|
"Failed to get daily trend",
|
||||||
|
);
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Activity Heatmap (day-of-week × hour-of-day) ──────────────────────
|
||||||
|
|
||||||
|
export interface HeatmapCell {
|
||||||
|
dayOfWeek: number; // 0=Senin, 6=Minggu
|
||||||
|
hour: number; // 0-23
|
||||||
|
count: number;
|
||||||
|
clean: number;
|
||||||
|
warned: number;
|
||||||
|
flagged: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function getActivityHeatmap(input: {
|
||||||
|
guildId: string;
|
||||||
|
channelId?: string;
|
||||||
|
hours?: number;
|
||||||
|
}): Promise<HeatmapCell[]> {
|
||||||
|
const { guildId, channelId, hours = 168 } = input;
|
||||||
|
const cacheKey = makeCacheKey("heatmap", { guildId, channelId, hours });
|
||||||
|
const cached = getCached<HeatmapCell[]>(cacheKey);
|
||||||
|
if (cached) return cached;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const since = Date.now() - hours * 3600_000;
|
||||||
|
const dayExpr = `(extract(isodow from to_timestamp(created_at / 1000)) % 7)::int as day_of_week`;
|
||||||
|
const hourExpr = `extract(hour from to_timestamp(created_at / 1000))::int as hour`;
|
||||||
|
|
||||||
|
const rows = await executeAll(
|
||||||
|
`
|
||||||
|
SELECT
|
||||||
|
${dayExpr},
|
||||||
|
${hourExpr},
|
||||||
|
count(*) as count,
|
||||||
|
count(case when ai_status = 'clean' then 1 end) as clean,
|
||||||
|
count(case when ai_status = 'warn' then 1 end) as warned,
|
||||||
|
count(case when ai_status = 'flagged' then 1 end) as flagged
|
||||||
|
FROM messages
|
||||||
|
WHERE guild_id = ?
|
||||||
|
AND created_at >= ?
|
||||||
|
AND deleted_at IS NULL
|
||||||
|
${channelId ? `AND (channel_id = ? OR thread_id = ?)` : ""}
|
||||||
|
GROUP BY day_of_week, hour
|
||||||
|
ORDER BY day_of_week, hour
|
||||||
|
`,
|
||||||
|
channelId ? [guildId, since, channelId, channelId] : [guildId, since],
|
||||||
|
);
|
||||||
|
|
||||||
|
// Initialize all 7×24 cells with zeros
|
||||||
|
const cells = new Map<
|
||||||
|
string,
|
||||||
|
{ count: number; clean: number; warned: number; flagged: number }
|
||||||
|
>();
|
||||||
|
for (let d = 0; d < 7; d++) {
|
||||||
|
for (let h = 0; h < 24; h++) {
|
||||||
|
cells.set(`${d}-${h}`, { count: 0, clean: 0, warned: 0, flagged: 0 });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const row of rows) {
|
||||||
|
const key = `${row.day_of_week}-${row.hour}`;
|
||||||
|
const cell = cells.get(key);
|
||||||
|
if (!cell) continue;
|
||||||
|
cell.count = row.count;
|
||||||
|
cell.clean = row.clean;
|
||||||
|
cell.warned = row.warned;
|
||||||
|
cell.flagged = row.flagged;
|
||||||
|
}
|
||||||
|
|
||||||
|
const result = Array.from(cells.entries())
|
||||||
|
.map(([key, data]) => {
|
||||||
|
const [dayOfWeek, hour] = key.split("-").map(Number);
|
||||||
|
return { dayOfWeek, hour, ...data };
|
||||||
|
})
|
||||||
|
.sort((a, b) => a.dayOfWeek - b.dayOfWeek || a.hour - b.hour);
|
||||||
|
|
||||||
|
setCache(cacheKey, result, AGGREGATE_CACHE_TTL_MS);
|
||||||
|
return result;
|
||||||
|
} catch (error) {
|
||||||
|
logger.error(
|
||||||
|
{ error: error instanceof Error ? error.message : String(error) },
|
||||||
|
"Failed to get activity heatmap",
|
||||||
|
);
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Cache Invalidation (called when new messages arrive) ───────────────
|
||||||
|
|
||||||
|
export function invalidateAnalyticsCache(guildId: string): void {
|
||||||
|
const now = Date.now();
|
||||||
|
const needle = `"${guildId}"`;
|
||||||
|
for (const [key, entry] of queryCache) {
|
||||||
|
if (key.includes(needle) && entry.expiresAt > now) {
|
||||||
|
entry.expiresAt = 0; // expire immediately
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Combined Overview ──────────────────────────────────────────────────
|
||||||
|
|
||||||
|
export async function getAnalyticsOverview(input: {
|
||||||
|
guildId: string;
|
||||||
|
channelId?: string;
|
||||||
|
hours?: number;
|
||||||
|
}): Promise<AnalyticsOverview> {
|
||||||
|
const { guildId, hours = 24 } = input;
|
||||||
|
const now = Date.now();
|
||||||
|
const since = now - hours * 3600_000;
|
||||||
|
|
||||||
|
const [messages, hourly, topics, topUsers, totalChannels] = await Promise.all(
|
||||||
|
[
|
||||||
|
getModerationStats(input),
|
||||||
|
getHourlyStats(input),
|
||||||
|
getTopicTrends(input),
|
||||||
|
getUserLeaderboard(input),
|
||||||
|
getActiveChannelCount({ guildId, hours }),
|
||||||
|
],
|
||||||
|
);
|
||||||
|
|
||||||
|
return {
|
||||||
|
period: { start: since, end: now },
|
||||||
|
messages,
|
||||||
|
hourly,
|
||||||
|
topics,
|
||||||
|
top_users: topUsers,
|
||||||
|
active_users_count: topUsers.length,
|
||||||
|
total_channels: totalChannels,
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -0,0 +1,82 @@
|
|||||||
|
import type { WebSocket } from "ws";
|
||||||
|
import { createChildLogger } from "../../shared/logger/logger.js";
|
||||||
|
import type {
|
||||||
|
AnalysisQueueStatus,
|
||||||
|
AttachmentRecord,
|
||||||
|
MediaState,
|
||||||
|
MessageRecord,
|
||||||
|
ModerationWsEvent,
|
||||||
|
} from "../message-capture/types.js";
|
||||||
|
|
||||||
|
export type BroadcasterClient = Pick<WebSocket, "readyState" | "send">;
|
||||||
|
|
||||||
|
const log = createChildLogger("broadcaster");
|
||||||
|
|
||||||
|
function sendJson(
|
||||||
|
clients: Set<BroadcasterClient>,
|
||||||
|
event: ModerationWsEvent,
|
||||||
|
): void {
|
||||||
|
const payload = JSON.stringify({ ...event, timestamp: Date.now() });
|
||||||
|
for (const client of clients) {
|
||||||
|
if (client.readyState === 1) {
|
||||||
|
try {
|
||||||
|
client.send(payload);
|
||||||
|
} catch (error) {
|
||||||
|
log.warn(
|
||||||
|
{ error, eventType: event.type },
|
||||||
|
"Failed to send event to client",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function createBroadcaster() {
|
||||||
|
const clients = new Set<BroadcasterClient>();
|
||||||
|
|
||||||
|
return {
|
||||||
|
addClient(client: BroadcasterClient) {
|
||||||
|
clients.add(client);
|
||||||
|
log.debug({ clientCount: clients.size }, "Client added");
|
||||||
|
},
|
||||||
|
removeClient(client: BroadcasterClient) {
|
||||||
|
clients.delete(client);
|
||||||
|
log.debug({ clientCount: clients.size }, "Client removed");
|
||||||
|
},
|
||||||
|
clientCount() {
|
||||||
|
return clients.size;
|
||||||
|
},
|
||||||
|
getClients() {
|
||||||
|
return Array.from(clients);
|
||||||
|
},
|
||||||
|
uiState(state: unknown) {
|
||||||
|
sendJson(clients, { type: "ui_state", state });
|
||||||
|
},
|
||||||
|
userState(users: unknown[]) {
|
||||||
|
sendJson(clients, { type: "user_state", users });
|
||||||
|
},
|
||||||
|
messageCreated(data: MessageRecord) {
|
||||||
|
sendJson(clients, { type: "message_created", data });
|
||||||
|
},
|
||||||
|
messageUpdated(data: Partial<MessageRecord> & { id: string }) {
|
||||||
|
sendJson(clients, { type: "message_updated", data });
|
||||||
|
},
|
||||||
|
messageDeleted(data: { id: string; deleted_at: number }) {
|
||||||
|
sendJson(clients, { type: "message_deleted", data });
|
||||||
|
},
|
||||||
|
messageAnalyzed(data: MessageRecord) {
|
||||||
|
sendJson(clients, { type: "message_analyzed", data });
|
||||||
|
},
|
||||||
|
attachmentCreated(data: AttachmentRecord) {
|
||||||
|
sendJson(clients, { type: "attachment_created", data });
|
||||||
|
},
|
||||||
|
analysisQueueStatus(data: AnalysisQueueStatus) {
|
||||||
|
sendJson(clients, { type: "analysis_queue_status", data });
|
||||||
|
},
|
||||||
|
mediaState(state: MediaState) {
|
||||||
|
sendJson(clients, { type: "media_state", state });
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export type ModerationBroadcaster = ReturnType<typeof createBroadcaster>;
|
||||||
@@ -0,0 +1,21 @@
|
|||||||
|
export { registerMessageCapture } from "./messageCapture.js";
|
||||||
|
export {
|
||||||
|
getDisplayContent,
|
||||||
|
getMessageLocation,
|
||||||
|
getMessageMetadata,
|
||||||
|
} from "../message-capture/messageMetadata.js";
|
||||||
|
export {
|
||||||
|
getMessageById,
|
||||||
|
insertAttachment,
|
||||||
|
updateMessageAsDeleted,
|
||||||
|
updateMessageAsEdited,
|
||||||
|
upsertMessageForCapture,
|
||||||
|
} from "../message-capture/messageStore.js";
|
||||||
|
export type {
|
||||||
|
AIRecommendedAction,
|
||||||
|
AISeverity,
|
||||||
|
AIStatus,
|
||||||
|
AttachmentRecord,
|
||||||
|
MessageRecord,
|
||||||
|
VoiceSegmentRecord,
|
||||||
|
} from "../message-capture/types.js";
|
||||||
@@ -0,0 +1,302 @@
|
|||||||
|
import type { Client, Message } from "discord.js-selfbot-v13";
|
||||||
|
import { config } from "../../shared/config/config.js";
|
||||||
|
import { createChildLogger } from "../../shared/logger/logger.js";
|
||||||
|
import { queueMessageAnalysis } from "../ai-moderation/aiAnalyzer.js";
|
||||||
|
import { processAttachmentUpload } from "../attachment-upload/attachmentUploader.js";
|
||||||
|
import type { EventBroadcaster } from "../event-broadcaster/eventBroadcaster.js";
|
||||||
|
import {
|
||||||
|
getDisplayContent,
|
||||||
|
getMessageLocation,
|
||||||
|
getMessageMetadata,
|
||||||
|
} from "../message-capture/messageMetadata.js";
|
||||||
|
import {
|
||||||
|
getMessageById,
|
||||||
|
insertAttachment,
|
||||||
|
updateMessageAsDeleted,
|
||||||
|
updateMessageAsEdited,
|
||||||
|
upsertMessageForCapture,
|
||||||
|
} from "../message-capture/messageStore.js";
|
||||||
|
import type { AttachmentRecord, MessageRecord } from "../message-capture/types.js";
|
||||||
|
|
||||||
|
const logger = createChildLogger("message-capture");
|
||||||
|
|
||||||
|
let _eventBroadcaster: EventBroadcaster | undefined;
|
||||||
|
|
||||||
|
export function setEventBroadcaster(broadcaster: EventBroadcaster | undefined) {
|
||||||
|
_eventBroadcaster = broadcaster;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface TextCaptureTarget {
|
||||||
|
guildId?: string;
|
||||||
|
channelId?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface MessageLocationInput {
|
||||||
|
guildId?: string | null;
|
||||||
|
channelId?: string | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function shouldCaptureMessageLocation(
|
||||||
|
message: MessageLocationInput,
|
||||||
|
target: TextCaptureTarget,
|
||||||
|
): boolean {
|
||||||
|
if (
|
||||||
|
message.channelId === "1310988070996414494" ||
|
||||||
|
message.channelId === "1265679542144467035" ||
|
||||||
|
message.channelId === "1310867899745046558"
|
||||||
|
)
|
||||||
|
return false;
|
||||||
|
if (!message.guildId || message.guildId !== target.guildId) return false;
|
||||||
|
if (target.channelId && message.channelId !== target.channelId) return false;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
function getTextCaptureTarget(): TextCaptureTarget {
|
||||||
|
return {
|
||||||
|
guildId: config.EFFECTIVE_TEXT_GUILD_ID,
|
||||||
|
channelId: config.TEXT_CHANNEL_ID,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function requireMessageGuildId(message: Message): string {
|
||||||
|
if (!message.guildId) {
|
||||||
|
throw new Error(`Message ${message.id} is missing guildId`);
|
||||||
|
}
|
||||||
|
return message.guildId;
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildMessageRecord(
|
||||||
|
message: Message,
|
||||||
|
type: "text" | "edited" | "deleted",
|
||||||
|
): MessageRecord {
|
||||||
|
const location = getMessageLocation(message);
|
||||||
|
const metadata = getMessageMetadata(message);
|
||||||
|
const guildId = requireMessageGuildId(message);
|
||||||
|
|
||||||
|
return {
|
||||||
|
id: message.id,
|
||||||
|
guild_id: guildId,
|
||||||
|
channel_id: location.channelId,
|
||||||
|
thread_id: location.threadId,
|
||||||
|
user_id: message.author?.id,
|
||||||
|
username: message.author?.username,
|
||||||
|
avatar_url: message.author?.avatarURL() || null,
|
||||||
|
content: getDisplayContent(message),
|
||||||
|
edited_content: null,
|
||||||
|
created_at: message.createdTimestamp,
|
||||||
|
edited_at: null,
|
||||||
|
deleted_at: null,
|
||||||
|
type,
|
||||||
|
metadata: JSON.stringify(metadata),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildAttachmentRecord(
|
||||||
|
message: Message,
|
||||||
|
location: ReturnType<typeof getMessageLocation>,
|
||||||
|
attachment: {
|
||||||
|
id: string;
|
||||||
|
name: string | null;
|
||||||
|
size: number;
|
||||||
|
contentType: string | null;
|
||||||
|
url: string;
|
||||||
|
},
|
||||||
|
): AttachmentRecord {
|
||||||
|
const guildId = requireMessageGuildId(message);
|
||||||
|
|
||||||
|
return {
|
||||||
|
id: attachment.id,
|
||||||
|
message_id: message.id,
|
||||||
|
guild_id: guildId,
|
||||||
|
channel_id: location.channelId,
|
||||||
|
thread_id: location.threadId,
|
||||||
|
user_id: message.author?.id,
|
||||||
|
filename: attachment.name || "unknown",
|
||||||
|
size: attachment.size,
|
||||||
|
type: attachment.contentType || "application/octet-stream",
|
||||||
|
discord_url: attachment.url,
|
||||||
|
uploaded_url: null,
|
||||||
|
upload_status: "pending",
|
||||||
|
upload_error: null,
|
||||||
|
created_at: Date.now(),
|
||||||
|
uploaded_at: null,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function captureMessage(
|
||||||
|
message: Message,
|
||||||
|
type: "text" | "edited" | "deleted",
|
||||||
|
options: { source?: "live" | "backlog" } = {},
|
||||||
|
): Promise<void> {
|
||||||
|
const location = getMessageLocation(message);
|
||||||
|
const messageRecord = buildMessageRecord(message, type);
|
||||||
|
|
||||||
|
const inserted = await upsertMessageForCapture(messageRecord);
|
||||||
|
if (!inserted) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const isBacklog = options.source === "backlog";
|
||||||
|
|
||||||
|
if (_eventBroadcaster && !isBacklog) {
|
||||||
|
_eventBroadcaster.messageCreated(messageRecord);
|
||||||
|
}
|
||||||
|
|
||||||
|
const attachmentUploadTasks: Promise<void>[] = [];
|
||||||
|
|
||||||
|
if (message.attachments.size > 0) {
|
||||||
|
for (const [, attachment] of message.attachments) {
|
||||||
|
const attachmentRecord = buildAttachmentRecord(message, location, {
|
||||||
|
id: attachment.id,
|
||||||
|
name: attachment.name,
|
||||||
|
size: attachment.size,
|
||||||
|
contentType: attachment.contentType,
|
||||||
|
url: attachment.url,
|
||||||
|
});
|
||||||
|
|
||||||
|
await insertAttachment(attachmentRecord);
|
||||||
|
|
||||||
|
if (!isBacklog) {
|
||||||
|
attachmentUploadTasks.push(
|
||||||
|
processAttachmentUpload(
|
||||||
|
attachment.id,
|
||||||
|
attachment.url,
|
||||||
|
attachment.name || "unknown",
|
||||||
|
{
|
||||||
|
contentType: attachment.contentType ?? undefined,
|
||||||
|
refreshDiscordUrl: async () => {
|
||||||
|
const freshMessage = await message.channel.messages.fetch(
|
||||||
|
message.id,
|
||||||
|
);
|
||||||
|
const freshAttachment = freshMessage.attachments.get(
|
||||||
|
attachment.id,
|
||||||
|
);
|
||||||
|
return freshAttachment?.url ?? null;
|
||||||
|
},
|
||||||
|
},
|
||||||
|
).catch((err: unknown) => {
|
||||||
|
logger.error(
|
||||||
|
{ attachmentId: attachment.id, error: err },
|
||||||
|
"Failed to initiate attachment upload",
|
||||||
|
);
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (_eventBroadcaster) {
|
||||||
|
_eventBroadcaster.attachmentCreated(attachmentRecord);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!isBacklog) {
|
||||||
|
if (attachmentUploadTasks.length > 0) {
|
||||||
|
let analysisQueued = false;
|
||||||
|
let fallbackTimer: NodeJS.Timeout | null = null;
|
||||||
|
const queueAnalysisOnce = () => {
|
||||||
|
if (analysisQueued) return;
|
||||||
|
analysisQueued = true;
|
||||||
|
if (fallbackTimer) {
|
||||||
|
clearTimeout(fallbackTimer);
|
||||||
|
fallbackTimer = null;
|
||||||
|
}
|
||||||
|
queueMessageAnalysis(message.id);
|
||||||
|
};
|
||||||
|
|
||||||
|
fallbackTimer = setTimeout(queueAnalysisOnce, 30000);
|
||||||
|
Promise.allSettled(attachmentUploadTasks)
|
||||||
|
.then(queueAnalysisOnce)
|
||||||
|
.catch((err: unknown) => {
|
||||||
|
logger.error(
|
||||||
|
{ messageId: message.id, error: err },
|
||||||
|
"Failed to queue message analysis after attachment upload",
|
||||||
|
);
|
||||||
|
queueAnalysisOnce();
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
queueMessageAnalysis(message.id);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function registerMessageCapture(client: Client): void {
|
||||||
|
client.on("messageCreate", async (message) => {
|
||||||
|
if (!shouldCaptureMessageLocation(message, getTextCaptureTarget())) return;
|
||||||
|
if (message.author?.bot) return;
|
||||||
|
|
||||||
|
try {
|
||||||
|
await captureMessage(message, "text");
|
||||||
|
} catch (error) {
|
||||||
|
logger.error(
|
||||||
|
{
|
||||||
|
messageId: message.id,
|
||||||
|
error: error instanceof Error ? error.message : String(error),
|
||||||
|
},
|
||||||
|
"Failed to capture message",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
client.on("messageUpdate", async (_oldMessage, newMessage) => {
|
||||||
|
if (!shouldCaptureMessageLocation(newMessage, getTextCaptureTarget()))
|
||||||
|
return;
|
||||||
|
if (newMessage.author?.bot) return;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const existing = await getMessageById(newMessage.id);
|
||||||
|
|
||||||
|
if (existing) {
|
||||||
|
const editedAt = Date.now();
|
||||||
|
await updateMessageAsEdited(
|
||||||
|
newMessage.id,
|
||||||
|
getDisplayContent(newMessage as Message),
|
||||||
|
editedAt,
|
||||||
|
);
|
||||||
|
queueMessageAnalysis(newMessage.id);
|
||||||
|
|
||||||
|
if (_eventBroadcaster) {
|
||||||
|
_eventBroadcaster.messageUpdated({
|
||||||
|
id: newMessage.id,
|
||||||
|
edited_content: getDisplayContent(newMessage as Message),
|
||||||
|
edited_at: editedAt,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
} else if (newMessage.author) {
|
||||||
|
await captureMessage(newMessage as Message, "text");
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
logger.error(
|
||||||
|
{
|
||||||
|
messageId: newMessage.id,
|
||||||
|
error: error instanceof Error ? error.message : String(error),
|
||||||
|
},
|
||||||
|
"Failed to capture message update",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
client.on("messageDelete", async (message) => {
|
||||||
|
if (!shouldCaptureMessageLocation(message, getTextCaptureTarget())) return;
|
||||||
|
if (!message.author) return;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const deletedAt = Date.now();
|
||||||
|
await updateMessageAsDeleted(message.id, deletedAt);
|
||||||
|
|
||||||
|
if (_eventBroadcaster) {
|
||||||
|
_eventBroadcaster.messageDeleted({
|
||||||
|
id: message.id,
|
||||||
|
deleted_at: deletedAt,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
logger.error(
|
||||||
|
{
|
||||||
|
messageId: message.id,
|
||||||
|
error: error instanceof Error ? error.message : String(error),
|
||||||
|
},
|
||||||
|
"Failed to capture message deletion",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
@@ -0,0 +1,375 @@
|
|||||||
|
import type {
|
||||||
|
Message,
|
||||||
|
TextChannel,
|
||||||
|
ThreadChannel,
|
||||||
|
} from "discord.js-selfbot-v13";
|
||||||
|
|
||||||
|
export interface MessageLocation {
|
||||||
|
channelId: string;
|
||||||
|
threadId: string | null;
|
||||||
|
threadName: string | null;
|
||||||
|
channelName: string | null;
|
||||||
|
nsfw?: boolean;
|
||||||
|
nsfwLevel?: string | null;
|
||||||
|
ageRestricted?: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface StickerEvidence {
|
||||||
|
id: string;
|
||||||
|
name: string;
|
||||||
|
url: string;
|
||||||
|
format: string | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface CustomEmojiEvidence {
|
||||||
|
id: string;
|
||||||
|
name: string;
|
||||||
|
animated: boolean;
|
||||||
|
url: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface EmbedEvidence {
|
||||||
|
title: string | null;
|
||||||
|
description: string | null;
|
||||||
|
url: string | null;
|
||||||
|
color: number | null;
|
||||||
|
image: string | null;
|
||||||
|
thumbnail: string | null;
|
||||||
|
author: {
|
||||||
|
name: string | null;
|
||||||
|
url: string | null;
|
||||||
|
iconURL: string | null;
|
||||||
|
} | null;
|
||||||
|
footer: { text: string | null; iconURL: string | null } | null;
|
||||||
|
fields: Array<{ name: string; value: string; inline: boolean }>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface AttachmentEvidence {
|
||||||
|
id: string;
|
||||||
|
name: string;
|
||||||
|
url: string;
|
||||||
|
contentType: string | null;
|
||||||
|
size: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface MessageMediaEvidence {
|
||||||
|
stickers: StickerEvidence[];
|
||||||
|
embeds: EmbedEvidence[];
|
||||||
|
attachments: AttachmentEvidence[];
|
||||||
|
customEmojis: CustomEmojiEvidence[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface RichMessageMetadata {
|
||||||
|
stickers: Array<StickerEvidence>;
|
||||||
|
embeds: Array<EmbedEvidence>;
|
||||||
|
attachments: Array<AttachmentEvidence>;
|
||||||
|
customEmojis: Array<CustomEmojiEvidence>;
|
||||||
|
author: {
|
||||||
|
id: string;
|
||||||
|
username: string;
|
||||||
|
tag: string | null;
|
||||||
|
avatarURL: string | null;
|
||||||
|
bot: boolean;
|
||||||
|
};
|
||||||
|
member: {
|
||||||
|
displayName: string | null;
|
||||||
|
roles: Array<{ id: string; name: string }>;
|
||||||
|
joinedTimestamp: number | null;
|
||||||
|
} | null;
|
||||||
|
channel: MessageLocation;
|
||||||
|
reference: {
|
||||||
|
messageId: string | null;
|
||||||
|
channelId: string | null;
|
||||||
|
guildId: string | null;
|
||||||
|
} | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getMessageLocation(message: Message): MessageLocation {
|
||||||
|
const channel = message.channel as TextChannel | ThreadChannel;
|
||||||
|
const safetyChannel = channel as TextChannel & {
|
||||||
|
nsfw?: boolean;
|
||||||
|
nsfwLevel?: string | null;
|
||||||
|
};
|
||||||
|
if (!channel.isThread?.()) {
|
||||||
|
return {
|
||||||
|
channelId: message.channelId,
|
||||||
|
threadId: null,
|
||||||
|
threadName: null,
|
||||||
|
channelName: "name" in channel ? channel.name : null,
|
||||||
|
nsfw:
|
||||||
|
typeof safetyChannel.nsfw === "boolean"
|
||||||
|
? safetyChannel.nsfw
|
||||||
|
: undefined,
|
||||||
|
nsfwLevel:
|
||||||
|
typeof safetyChannel.nsfwLevel === "string"
|
||||||
|
? safetyChannel.nsfwLevel
|
||||||
|
: null,
|
||||||
|
ageRestricted:
|
||||||
|
typeof safetyChannel.nsfw === "boolean"
|
||||||
|
? safetyChannel.nsfw
|
||||||
|
: undefined,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
channelId: channel.parentId ?? message.channelId,
|
||||||
|
threadId: channel.id,
|
||||||
|
threadName: channel.name,
|
||||||
|
channelName: channel.parent?.name ?? null,
|
||||||
|
nsfw:
|
||||||
|
typeof safetyChannel.nsfw === "boolean" ? safetyChannel.nsfw : undefined,
|
||||||
|
nsfwLevel:
|
||||||
|
typeof safetyChannel.nsfwLevel === "string"
|
||||||
|
? safetyChannel.nsfwLevel
|
||||||
|
: null,
|
||||||
|
ageRestricted:
|
||||||
|
typeof safetyChannel.nsfw === "boolean" ? safetyChannel.nsfw : undefined,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getStickerMetadata(
|
||||||
|
message: Message,
|
||||||
|
): RichMessageMetadata["stickers"] {
|
||||||
|
return Array.from(message.stickers.values()).map((sticker) => ({
|
||||||
|
id: sticker.id,
|
||||||
|
name: sticker.name,
|
||||||
|
url: sticker.url,
|
||||||
|
format: sticker.format ?? null,
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Extract custom emoji references from message content.
|
||||||
|
* Builds Discord CDN URLs for each emoji so they can be downloaded
|
||||||
|
* and sent to the vision model for analysis.
|
||||||
|
*/
|
||||||
|
export function getCustomEmojiMetadata(
|
||||||
|
message: Message,
|
||||||
|
): RichMessageMetadata["customEmojis"] {
|
||||||
|
const CUSTOM_EMOJI_PATTERN = /<(a)?:([a-zA-Z0-9_]+):(\d+)>/g;
|
||||||
|
const emojis: CustomEmojiEvidence[] = [];
|
||||||
|
let match;
|
||||||
|
while ((match = CUSTOM_EMOJI_PATTERN.exec(message.content)) !== null) {
|
||||||
|
const [, animated, name, id] = match;
|
||||||
|
const ext = animated ? "gif" : "png";
|
||||||
|
emojis.push({
|
||||||
|
id,
|
||||||
|
name,
|
||||||
|
animated: animated === "a",
|
||||||
|
url: `https://cdn.discordapp.com/emojis/${id}.${ext}?size=128`,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return emojis;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getAttachmentMetadata(
|
||||||
|
message: Message,
|
||||||
|
): RichMessageMetadata["attachments"] {
|
||||||
|
return Array.from(message.attachments.values()).map((attachment) => ({
|
||||||
|
id: attachment.id,
|
||||||
|
name: attachment.name || "unknown",
|
||||||
|
url: attachment.url,
|
||||||
|
contentType: attachment.contentType ?? null,
|
||||||
|
size: attachment.size,
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getEmbedMetadata(
|
||||||
|
message: Message,
|
||||||
|
): RichMessageMetadata["embeds"] {
|
||||||
|
return message.embeds.map((embed) => ({
|
||||||
|
title: embed.title ?? null,
|
||||||
|
description: embed.description ?? null,
|
||||||
|
url: embed.url ?? null,
|
||||||
|
color: embed.color ?? null,
|
||||||
|
image: embed.image?.url ?? null,
|
||||||
|
thumbnail: embed.thumbnail?.url ?? null,
|
||||||
|
author: embed.author
|
||||||
|
? {
|
||||||
|
name: embed.author.name ?? null,
|
||||||
|
url: embed.author.url ?? null,
|
||||||
|
iconURL: embed.author.iconURL ?? null,
|
||||||
|
}
|
||||||
|
: null,
|
||||||
|
footer: embed.footer
|
||||||
|
? {
|
||||||
|
text: embed.footer.text ?? null,
|
||||||
|
iconURL: embed.footer.iconURL ?? null,
|
||||||
|
}
|
||||||
|
: null,
|
||||||
|
fields: embed.fields.map((field) => ({
|
||||||
|
name: field.name,
|
||||||
|
value: field.value,
|
||||||
|
inline: Boolean(field.inline),
|
||||||
|
})),
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getMessageMetadata(message: Message): RichMessageMetadata {
|
||||||
|
const member = message.member;
|
||||||
|
return {
|
||||||
|
stickers: getStickerMetadata(message),
|
||||||
|
embeds: getEmbedMetadata(message),
|
||||||
|
attachments: getAttachmentMetadata(message),
|
||||||
|
customEmojis: getCustomEmojiMetadata(message),
|
||||||
|
author: {
|
||||||
|
id: message.author.id,
|
||||||
|
username: message.author.username,
|
||||||
|
tag: "tag" in message.author ? message.author.tag : null,
|
||||||
|
avatarURL: message.author.avatarURL() ?? null,
|
||||||
|
bot: Boolean(message.author.bot),
|
||||||
|
},
|
||||||
|
member: member
|
||||||
|
? {
|
||||||
|
displayName: member.displayName ?? null,
|
||||||
|
roles: member.roles.cache.map((role) => ({
|
||||||
|
id: role.id,
|
||||||
|
name: role.name,
|
||||||
|
})),
|
||||||
|
joinedTimestamp: member.joinedTimestamp ?? null,
|
||||||
|
}
|
||||||
|
: null,
|
||||||
|
channel: getMessageLocation(message),
|
||||||
|
reference: message.reference
|
||||||
|
? {
|
||||||
|
messageId: message.reference.messageId ?? null,
|
||||||
|
channelId: message.reference.channelId ?? null,
|
||||||
|
guildId: message.reference.guildId ?? null,
|
||||||
|
}
|
||||||
|
: null,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function parseRichMessageMetadata(
|
||||||
|
metadata: string | null | undefined,
|
||||||
|
): RichMessageMetadata | null {
|
||||||
|
if (!metadata) return null;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const parsed = JSON.parse(metadata) as Partial<RichMessageMetadata>;
|
||||||
|
return {
|
||||||
|
stickers: Array.isArray(parsed.stickers) ? parsed.stickers : [],
|
||||||
|
embeds: Array.isArray(parsed.embeds) ? parsed.embeds : [],
|
||||||
|
attachments: Array.isArray(parsed.attachments) ? parsed.attachments : [],
|
||||||
|
customEmojis: Array.isArray(parsed.customEmojis)
|
||||||
|
? parsed.customEmojis
|
||||||
|
: [],
|
||||||
|
author: parsed.author as RichMessageMetadata["author"],
|
||||||
|
member: (parsed.member ?? null) as RichMessageMetadata["member"],
|
||||||
|
channel: parsed.channel as RichMessageMetadata["channel"],
|
||||||
|
reference: (parsed.reference ?? null) as RichMessageMetadata["reference"],
|
||||||
|
};
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isAgeRestrictedMetadata(
|
||||||
|
metadata: string | null | undefined,
|
||||||
|
): boolean {
|
||||||
|
const parsed = parseRichMessageMetadata(metadata);
|
||||||
|
if (!parsed) return false;
|
||||||
|
|
||||||
|
const nsfwLevel = parsed.channel.nsfwLevel?.toUpperCase();
|
||||||
|
return Boolean(
|
||||||
|
parsed.channel.nsfw ||
|
||||||
|
parsed.channel.ageRestricted ||
|
||||||
|
nsfwLevel === "AGE_RESTRICTED",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function extractMessageMediaEvidence(
|
||||||
|
metadata: string | null | undefined,
|
||||||
|
): MessageMediaEvidence {
|
||||||
|
const parsed = parseRichMessageMetadata(metadata);
|
||||||
|
return {
|
||||||
|
stickers: parsed?.stickers ?? [],
|
||||||
|
embeds: parsed?.embeds ?? [],
|
||||||
|
attachments: parsed?.attachments ?? [],
|
||||||
|
customEmojis: parsed?.customEmojis ?? [],
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function formatMediaEvidenceForPrompt(
|
||||||
|
metadata: string | null | undefined,
|
||||||
|
): string {
|
||||||
|
const evidence = extractMessageMediaEvidence(metadata);
|
||||||
|
const parts: string[] = [];
|
||||||
|
|
||||||
|
if (evidence.stickers.length > 0) {
|
||||||
|
parts.push(
|
||||||
|
`[stickers: ${evidence.stickers
|
||||||
|
.map((sticker) =>
|
||||||
|
[`name=${sticker.name}`, sticker.url ? `url=${sticker.url}` : null]
|
||||||
|
.filter(Boolean)
|
||||||
|
.join(", "),
|
||||||
|
)
|
||||||
|
.join(" | ")}]`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (evidence.embeds.length > 0) {
|
||||||
|
parts.push(
|
||||||
|
`[embeds: ${evidence.embeds
|
||||||
|
.map((embed) =>
|
||||||
|
[
|
||||||
|
embed.title ? `title=${embed.title}` : null,
|
||||||
|
embed.description ? `description=${embed.description}` : null,
|
||||||
|
embed.url ? `url=${embed.url}` : null,
|
||||||
|
embed.image ? `image=${embed.image}` : null,
|
||||||
|
embed.thumbnail ? `thumbnail=${embed.thumbnail}` : null,
|
||||||
|
embed.fields.length > 0
|
||||||
|
? `fields=${embed.fields.map((field) => `${field.name}: ${field.value}`).join("; ")}`
|
||||||
|
: null,
|
||||||
|
]
|
||||||
|
.filter(Boolean)
|
||||||
|
.join(", "),
|
||||||
|
)
|
||||||
|
.join(" | ")}]`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (evidence.attachments.length > 0) {
|
||||||
|
parts.push(
|
||||||
|
`[attachments: ${evidence.attachments
|
||||||
|
.map((attachment) =>
|
||||||
|
[
|
||||||
|
`name=${attachment.name}`,
|
||||||
|
attachment.contentType ? `type=${attachment.contentType}` : null,
|
||||||
|
`size=${attachment.size}`,
|
||||||
|
attachment.url ? `url=${attachment.url}` : null,
|
||||||
|
]
|
||||||
|
.filter(Boolean)
|
||||||
|
.join(", "),
|
||||||
|
)
|
||||||
|
.join(" | ")}]`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
return parts.join(" ");
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getDisplayContent(message: Message): string {
|
||||||
|
if (message.content.trim().length > 0) return message.content;
|
||||||
|
|
||||||
|
const stickers = getStickerMetadata(message);
|
||||||
|
if (stickers.length > 0) {
|
||||||
|
return stickers.map((sticker) => `[Sticker: ${sticker.name}]`).join(" ");
|
||||||
|
}
|
||||||
|
|
||||||
|
const attachments = getAttachmentMetadata(message);
|
||||||
|
if (attachments.length > 0) {
|
||||||
|
return attachments
|
||||||
|
.map((attachment) => `[Attachment: ${attachment.name}]`)
|
||||||
|
.join(" ");
|
||||||
|
}
|
||||||
|
|
||||||
|
const embeds = getEmbedMetadata(message);
|
||||||
|
if (embeds.length > 0) {
|
||||||
|
return embeds
|
||||||
|
.map((embed) => embed.title || embed.description || "[Embed]")
|
||||||
|
.join(" ");
|
||||||
|
}
|
||||||
|
|
||||||
|
return "";
|
||||||
|
}
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,21 @@
|
|||||||
|
export interface CursorData {
|
||||||
|
created_at: number;
|
||||||
|
id: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function encodeCursor(data: CursorData): string {
|
||||||
|
return Buffer.from(JSON.stringify(data)).toString("base64");
|
||||||
|
}
|
||||||
|
|
||||||
|
export function decodeCursor(cursor?: string): CursorData | null {
|
||||||
|
if (!cursor) return null;
|
||||||
|
try {
|
||||||
|
const data = JSON.parse(Buffer.from(cursor, "base64").toString("utf-8"));
|
||||||
|
if (typeof data.created_at === "number" && typeof data.id === "string") {
|
||||||
|
return data;
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,267 @@
|
|||||||
|
import type fs from "node:fs";
|
||||||
|
import type prism from "prism-media";
|
||||||
|
|
||||||
|
export type AIStatus = "pending" | "clean" | "warn" | "flagged" | "error";
|
||||||
|
export type AISeverity = "none" | "low" | "medium" | "high" | "critical";
|
||||||
|
export type AIRecommendedAction =
|
||||||
|
| "none"
|
||||||
|
| "monitor"
|
||||||
|
| "warn"
|
||||||
|
| "review"
|
||||||
|
| "delete"
|
||||||
|
| "escalate";
|
||||||
|
|
||||||
|
export interface BroadcasterClient {
|
||||||
|
messageCreated: (data: unknown) => void;
|
||||||
|
messageUpdated: (data: unknown) => void;
|
||||||
|
messageDeleted: (data: unknown) => void;
|
||||||
|
messageAnalyzed: (data: unknown) => void;
|
||||||
|
attachmentCreated: (data: unknown) => void;
|
||||||
|
attachmentUploaded: (data: unknown) => void;
|
||||||
|
voiceRecordingStarted: (data: unknown) => void;
|
||||||
|
voiceRecordingStopped: (data: unknown) => void;
|
||||||
|
voiceRecordingUploaded: (data: unknown) => void;
|
||||||
|
analysisQueueStatus: (data: unknown) => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
export type ModerationBroadcaster = BroadcasterClient;
|
||||||
|
|
||||||
|
export interface RoleMetadata {
|
||||||
|
id: string;
|
||||||
|
name: string;
|
||||||
|
position: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface UserMetadata {
|
||||||
|
userId: string;
|
||||||
|
username: string;
|
||||||
|
tag: string;
|
||||||
|
displayName: string;
|
||||||
|
avatarUrl: string;
|
||||||
|
bot: boolean;
|
||||||
|
roles: RoleMetadata[];
|
||||||
|
highestRole: RoleMetadata | null;
|
||||||
|
joinedTimestamp: number | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface SegmentState {
|
||||||
|
index: number;
|
||||||
|
startTime: number;
|
||||||
|
endTime: number | null;
|
||||||
|
filename: string;
|
||||||
|
jsonFilename: string;
|
||||||
|
oggStream: prism.opus.OggLogicalBitstream;
|
||||||
|
out: fs.WriteStream;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface SegmentMetadata extends UserMetadata {
|
||||||
|
recordingSessionId: string;
|
||||||
|
sessionId: string;
|
||||||
|
sessionStartTime: number;
|
||||||
|
segmentIndex: number;
|
||||||
|
segmentMs: number;
|
||||||
|
startTime: number;
|
||||||
|
endTime: number;
|
||||||
|
durationMs: number;
|
||||||
|
filename: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface PcmBroadcaster {
|
||||||
|
broadcastPcmToWeb?: (chunk: Buffer, userId: string) => void;
|
||||||
|
updateActiveUser?: (
|
||||||
|
userId: string,
|
||||||
|
data: { username: string; avatar: string; speaking: boolean },
|
||||||
|
) => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface MessageRecord {
|
||||||
|
id: string;
|
||||||
|
guild_id: string;
|
||||||
|
channel_id: string;
|
||||||
|
thread_id: string | null;
|
||||||
|
user_id: string;
|
||||||
|
username: string;
|
||||||
|
avatar_url: string | null;
|
||||||
|
content: string;
|
||||||
|
edited_content: string | null;
|
||||||
|
created_at: number;
|
||||||
|
edited_at: number | null;
|
||||||
|
deleted_at: number | null;
|
||||||
|
type: "text" | "edited" | "deleted";
|
||||||
|
metadata: string | null;
|
||||||
|
ai_status?: AIStatus | null;
|
||||||
|
ai_moderation_flags?: string | null;
|
||||||
|
ai_moderation_score?: number | null;
|
||||||
|
ai_analysis?: string | null;
|
||||||
|
ai_categories?: string | null;
|
||||||
|
ai_severity?: AISeverity | null;
|
||||||
|
ai_confidence?: number | null;
|
||||||
|
ai_recommended_action?: AIRecommendedAction | null;
|
||||||
|
ai_analyzed_at?: number | null;
|
||||||
|
ai_error?: string | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface AttachmentRecord {
|
||||||
|
id: string;
|
||||||
|
message_id: string;
|
||||||
|
guild_id: string;
|
||||||
|
channel_id: string;
|
||||||
|
thread_id: string | null;
|
||||||
|
user_id: string;
|
||||||
|
filename: string;
|
||||||
|
size: number;
|
||||||
|
type: string;
|
||||||
|
discord_url: string;
|
||||||
|
uploaded_url: string | null;
|
||||||
|
upload_status: "pending" | "uploaded" | "failed";
|
||||||
|
upload_error: string | null;
|
||||||
|
created_at: number;
|
||||||
|
uploaded_at: number | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface VoiceSegmentRecord {
|
||||||
|
id: string;
|
||||||
|
user_id: string;
|
||||||
|
session_id: string;
|
||||||
|
guild_id: string;
|
||||||
|
channel_id: string;
|
||||||
|
filename: string;
|
||||||
|
duration_ms: number;
|
||||||
|
created_at: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface DashboardMessage {
|
||||||
|
id: string;
|
||||||
|
channel_id: string;
|
||||||
|
user_id: string;
|
||||||
|
username: string;
|
||||||
|
avatar_url: string | null;
|
||||||
|
content: string;
|
||||||
|
created_at: number;
|
||||||
|
type: "text" | "image" | "voice";
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface MessageQuery {
|
||||||
|
guildId?: string;
|
||||||
|
channelId?: string;
|
||||||
|
threadId?: string;
|
||||||
|
status?: AIStatus[];
|
||||||
|
userId?: string;
|
||||||
|
q?: string;
|
||||||
|
cursor?: string;
|
||||||
|
limit: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface PageResult<T> {
|
||||||
|
data: T[];
|
||||||
|
nextCursor: string | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface AnalysisResult {
|
||||||
|
messageId: string;
|
||||||
|
status: Exclude<AIStatus, "pending">;
|
||||||
|
flags: string[];
|
||||||
|
score: number;
|
||||||
|
analysis: string;
|
||||||
|
categories?: string[];
|
||||||
|
severity?: AISeverity;
|
||||||
|
confidence?: number;
|
||||||
|
recommendedAction?: AIRecommendedAction;
|
||||||
|
policyVersion?: string;
|
||||||
|
evidence?: string[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export type MediaMode = "music" | "screen";
|
||||||
|
export type MediaSourceKind =
|
||||||
|
| "url"
|
||||||
|
| "local"
|
||||||
|
| "youtube"
|
||||||
|
| "spotify"
|
||||||
|
| "search";
|
||||||
|
export type MediaQueueItemStatus = "queued" | "playing" | "failed";
|
||||||
|
|
||||||
|
export interface MediaQueueItem {
|
||||||
|
id: string;
|
||||||
|
mode: MediaMode;
|
||||||
|
source: string;
|
||||||
|
title: string;
|
||||||
|
kind: MediaSourceKind;
|
||||||
|
requestedBy: string;
|
||||||
|
addedAt: number;
|
||||||
|
status: MediaQueueItemStatus;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface MediaState {
|
||||||
|
playing: boolean;
|
||||||
|
musicVolume: number;
|
||||||
|
current: MediaQueueItem | null;
|
||||||
|
queue: MediaQueueItem[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export type ModerationWsEvent =
|
||||||
|
| { type: "ui_state"; state: unknown }
|
||||||
|
| { type: "user_state"; users: unknown[] }
|
||||||
|
| { type: "message_created"; data: MessageRecord }
|
||||||
|
| { type: "message_updated"; data: Partial<MessageRecord> & { id: string } }
|
||||||
|
| { type: "message_deleted"; data: { id: string; deleted_at: number } }
|
||||||
|
| { type: "message_analyzed"; data: MessageRecord }
|
||||||
|
| { type: "attachment_created"; data: AttachmentRecord }
|
||||||
|
| { type: "analysis_queue_status"; data: AnalysisQueueStatus }
|
||||||
|
| { type: "media_state"; state: MediaState }
|
||||||
|
| { type: "voice_recording_uploaded"; data: any };
|
||||||
|
|
||||||
|
export interface AnalysisQueueStatus {
|
||||||
|
queuedConversations: number;
|
||||||
|
activeRequests: number;
|
||||||
|
activeIndividualRequests: number;
|
||||||
|
individualInFlightCount: number;
|
||||||
|
individualCircuitBreakerActive: boolean;
|
||||||
|
lastError: string | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export type ReviewStatus = "pending" | "approved" | "rejected" | "escalated";
|
||||||
|
|
||||||
|
export interface MessageReview {
|
||||||
|
id: string;
|
||||||
|
message_id: string;
|
||||||
|
guild_id: string;
|
||||||
|
channel_id: string;
|
||||||
|
reviewer_id: string | null;
|
||||||
|
status: ReviewStatus;
|
||||||
|
notes: string | null;
|
||||||
|
created_at: number;
|
||||||
|
reviewed_at: number | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export type ModerationActionType =
|
||||||
|
| "delete_message"
|
||||||
|
| "mute_user"
|
||||||
|
| "warn_user"
|
||||||
|
| "kick_user"
|
||||||
|
| "ban_user";
|
||||||
|
|
||||||
|
export interface ModerationAction {
|
||||||
|
id: string;
|
||||||
|
message_id: string | null;
|
||||||
|
user_id: string | null;
|
||||||
|
guild_id: string;
|
||||||
|
action_type: ModerationActionType;
|
||||||
|
reason: string | null;
|
||||||
|
executed_by: string | null;
|
||||||
|
status: "pending" | "executed" | "failed";
|
||||||
|
error: string | null;
|
||||||
|
created_at: number;
|
||||||
|
executed_at: number | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface RetentionPolicy {
|
||||||
|
id: string;
|
||||||
|
guild_id: string;
|
||||||
|
channel_id: string | null;
|
||||||
|
retention_days: number;
|
||||||
|
apply_to_media: boolean;
|
||||||
|
apply_to_voice: boolean;
|
||||||
|
enabled: boolean;
|
||||||
|
created_at: number;
|
||||||
|
updated_at: number;
|
||||||
|
}
|
||||||
@@ -0,0 +1,61 @@
|
|||||||
|
import { spawn } from "child_process";
|
||||||
|
|
||||||
|
export interface MuxFfmpegArgsOptions {
|
||||||
|
inputs: string[];
|
||||||
|
filter: string;
|
||||||
|
output: string;
|
||||||
|
codec: string;
|
||||||
|
audioFrequency?: number;
|
||||||
|
audioChannels?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Builds ffmpeg argument array for muxing audio clips.
|
||||||
|
*/
|
||||||
|
export function buildMuxFfmpegArgs(options: MuxFfmpegArgsOptions): string[] {
|
||||||
|
const args: string[] = ["-y"];
|
||||||
|
|
||||||
|
for (const input of options.inputs) {
|
||||||
|
args.push("-i", input);
|
||||||
|
}
|
||||||
|
|
||||||
|
args.push("-filter_complex", options.filter);
|
||||||
|
args.push("-map", "[out]");
|
||||||
|
args.push("-codec:a", options.codec);
|
||||||
|
|
||||||
|
if (options.audioFrequency !== undefined) {
|
||||||
|
args.push("-ar", String(options.audioFrequency));
|
||||||
|
}
|
||||||
|
|
||||||
|
if (options.audioChannels !== undefined) {
|
||||||
|
args.push("-ac", String(options.audioChannels));
|
||||||
|
}
|
||||||
|
|
||||||
|
args.push(options.output);
|
||||||
|
|
||||||
|
return args;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Runs ffmpeg with the given arguments.
|
||||||
|
* Resolves on successful (code 0) exit, rejects on error or non-zero exit.
|
||||||
|
*/
|
||||||
|
export function runFfmpeg(args: string[]): Promise<void> {
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
const proc = spawn("ffmpeg", args, {
|
||||||
|
stdio: ["ignore", "inherit", "inherit"],
|
||||||
|
});
|
||||||
|
|
||||||
|
proc.on("close", (code) => {
|
||||||
|
if (code === 0) {
|
||||||
|
resolve();
|
||||||
|
} else {
|
||||||
|
reject(new Error(`ffmpeg exited with code ${code}`));
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
proc.on("error", (err) => {
|
||||||
|
reject(err);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
@@ -0,0 +1,4 @@
|
|||||||
|
export { OpusDecoder } from "./recorder/decoder.js";
|
||||||
|
export { SegmentManager } from "./recorder/segment.js";
|
||||||
|
export { startRecording, stopRecording } from "./recorder.js";
|
||||||
|
export { VoiceController } from "./voiceController.js";
|
||||||
@@ -0,0 +1,80 @@
|
|||||||
|
import type { Readable } from "node:stream";
|
||||||
|
import type { StreamType } from "@discordjs/voice";
|
||||||
|
|
||||||
|
export type MediaMode = "music" | "screen";
|
||||||
|
export type MediaSourceKind =
|
||||||
|
| "url"
|
||||||
|
| "local"
|
||||||
|
| "youtube"
|
||||||
|
| "spotify"
|
||||||
|
| "search";
|
||||||
|
export type MediaQueueItemStatus = "queued" | "playing" | "failed";
|
||||||
|
|
||||||
|
export interface ResolvedMediaSource {
|
||||||
|
source: string;
|
||||||
|
title: string;
|
||||||
|
kind: MediaSourceKind;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface MediaQueueItem extends ResolvedMediaSource {
|
||||||
|
id: string;
|
||||||
|
mode: MediaMode;
|
||||||
|
requestedBy: string;
|
||||||
|
addedAt: number;
|
||||||
|
status: MediaQueueItemStatus;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface MediaState {
|
||||||
|
playing: boolean;
|
||||||
|
activeMode: MediaMode | null;
|
||||||
|
musicVolume: number;
|
||||||
|
current: MediaQueueItem | null;
|
||||||
|
queue: MediaQueueItem[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface QueueMediaOptions {
|
||||||
|
mode?: MediaMode;
|
||||||
|
requestedBy?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface MusicPlayback {
|
||||||
|
done: Promise<void>;
|
||||||
|
stop(): void;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface MusicPlayer {
|
||||||
|
play(source: ResolvedMediaSource): MusicPlayback;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ScreenSharePlayback {
|
||||||
|
done: Promise<void>;
|
||||||
|
stop(): void;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ScreenShareController {
|
||||||
|
isActive(): boolean;
|
||||||
|
start(source: string): Promise<ScreenSharePlayback>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export type DiscordPlayerOwner = "none" | "browser-bridge" | "music" | "screen";
|
||||||
|
|
||||||
|
export interface DiscordPlayOptions {
|
||||||
|
inputType?: StreamType;
|
||||||
|
inlineVolume?: boolean;
|
||||||
|
volume?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface DiscordAudioPlayer {
|
||||||
|
getOwner(): DiscordPlayerOwner;
|
||||||
|
isConnected(): boolean;
|
||||||
|
playStream(
|
||||||
|
stream: Readable,
|
||||||
|
owner: DiscordPlayerOwner,
|
||||||
|
options?: DiscordPlayOptions,
|
||||||
|
): void;
|
||||||
|
pause(owner?: DiscordPlayerOwner): void;
|
||||||
|
unpause(owner?: DiscordPlayerOwner): boolean;
|
||||||
|
stop(owner?: DiscordPlayerOwner): void;
|
||||||
|
getMusicVolume(): number;
|
||||||
|
setMusicVolume(volume: number): void;
|
||||||
|
}
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
export { buildMuxFfmpegArgs, runFfmpeg } from "./ffmpegProcess.js";
|
||||||
@@ -0,0 +1,41 @@
|
|||||||
|
import { Transform, TransformCallback } from "node:stream";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Transform stream untuk memfilter audio packets yang terlalu kecil
|
||||||
|
* Packet yang terlalu kecil kemungkinan gagal didekripsi oleh Discord
|
||||||
|
*/
|
||||||
|
export class PacketFilter extends Transform {
|
||||||
|
private minPacketSize: number;
|
||||||
|
private filteredCount: number = 0;
|
||||||
|
private totalCount: number = 0;
|
||||||
|
|
||||||
|
constructor(minPacketSize: number = 10) {
|
||||||
|
super();
|
||||||
|
this.minPacketSize = minPacketSize;
|
||||||
|
}
|
||||||
|
|
||||||
|
_transform(
|
||||||
|
chunk: Buffer,
|
||||||
|
encoding: string,
|
||||||
|
callback: TransformCallback,
|
||||||
|
): void {
|
||||||
|
this.totalCount++;
|
||||||
|
|
||||||
|
// Filter packet yang terlalu kecil
|
||||||
|
if (chunk.length >= this.minPacketSize) {
|
||||||
|
this.push(chunk);
|
||||||
|
} else {
|
||||||
|
this.filteredCount++;
|
||||||
|
if (this.filteredCount % 10 === 0) {
|
||||||
|
// console.log(`[packet-filter] Filtered ${this.filteredCount} small packets (size < ${this.minPacketSize} bytes)`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
callback();
|
||||||
|
}
|
||||||
|
|
||||||
|
_flush(callback: TransformCallback): void {
|
||||||
|
// console.log(`[packet-filter] Total packets: ${this.totalCount}, filtered: ${this.filteredCount}, passed: ${this.totalCount - this.filteredCount}`);
|
||||||
|
callback();
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,137 @@
|
|||||||
|
import { Readable } from "node:stream";
|
||||||
|
import {
|
||||||
|
AudioPlayer,
|
||||||
|
AudioPlayerStatus,
|
||||||
|
type AudioResource,
|
||||||
|
createAudioPlayer,
|
||||||
|
createAudioResource,
|
||||||
|
StreamType,
|
||||||
|
VoiceConnection,
|
||||||
|
} from "@discordjs/voice";
|
||||||
|
import type {
|
||||||
|
DiscordPlayerOwner,
|
||||||
|
DiscordPlayOptions,
|
||||||
|
} from "./mediaTypes.js";
|
||||||
|
|
||||||
|
export class DiscordPlayer {
|
||||||
|
private player: AudioPlayer;
|
||||||
|
private connection: VoiceConnection | null = null;
|
||||||
|
private owner: DiscordPlayerOwner = "none";
|
||||||
|
private resource: AudioResource | null = null;
|
||||||
|
private musicVolume = 1;
|
||||||
|
|
||||||
|
constructor() {
|
||||||
|
this.player = createAudioPlayer();
|
||||||
|
|
||||||
|
this.player.on(AudioPlayerStatus.Playing, () => {
|
||||||
|
console.log("[player] Audio player is now playing!");
|
||||||
|
});
|
||||||
|
|
||||||
|
this.player.on("error", (error) => {
|
||||||
|
console.error(`[player] Error: ${error.message}`);
|
||||||
|
this.owner = "none";
|
||||||
|
this.resource = null;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
public setConnection(connection: VoiceConnection) {
|
||||||
|
this.connection = connection;
|
||||||
|
this.connection.subscribe(this.player);
|
||||||
|
}
|
||||||
|
|
||||||
|
public getOwner(): DiscordPlayerOwner {
|
||||||
|
return this.owner;
|
||||||
|
}
|
||||||
|
|
||||||
|
public isConnected(): boolean {
|
||||||
|
return this.connection !== null;
|
||||||
|
}
|
||||||
|
|
||||||
|
public playStream(
|
||||||
|
stream: Readable,
|
||||||
|
owner: DiscordPlayerOwner,
|
||||||
|
options: DiscordPlayOptions = {},
|
||||||
|
) {
|
||||||
|
if (owner === "none") {
|
||||||
|
throw new Error("Discord audio player owner is required");
|
||||||
|
}
|
||||||
|
this.assertOwnerAvailable(owner);
|
||||||
|
|
||||||
|
const resource = createAudioResource(stream, {
|
||||||
|
inputType: options.inputType ?? StreamType.OggOpus,
|
||||||
|
inlineVolume: options.inlineVolume ?? false,
|
||||||
|
});
|
||||||
|
|
||||||
|
if (this.owner === owner) {
|
||||||
|
this.player.stop();
|
||||||
|
}
|
||||||
|
this.resource = resource;
|
||||||
|
this.owner = owner;
|
||||||
|
if (owner === "music") {
|
||||||
|
const nextVolume =
|
||||||
|
options.volume !== undefined
|
||||||
|
? this.normalizeVolume(options.volume)
|
||||||
|
: this.musicVolume;
|
||||||
|
this.musicVolume = nextVolume;
|
||||||
|
this.setResourceVolume(nextVolume);
|
||||||
|
}
|
||||||
|
this.player.play(resource);
|
||||||
|
this.unpause(owner);
|
||||||
|
this.connection?.subscribe(this.player);
|
||||||
|
}
|
||||||
|
|
||||||
|
public getStatus(): AudioPlayerStatus {
|
||||||
|
return this.player.state.status;
|
||||||
|
}
|
||||||
|
|
||||||
|
public pause(owner?: DiscordPlayerOwner) {
|
||||||
|
if (!this.canControl(owner)) return;
|
||||||
|
this.player.pause(true);
|
||||||
|
}
|
||||||
|
|
||||||
|
public unpause(owner?: DiscordPlayerOwner): boolean {
|
||||||
|
if (!this.canControl(owner)) return false;
|
||||||
|
return this.player.unpause();
|
||||||
|
}
|
||||||
|
|
||||||
|
public stop(owner?: DiscordPlayerOwner) {
|
||||||
|
if (!this.canControl(owner)) return;
|
||||||
|
this.player.stop();
|
||||||
|
this.owner = "none";
|
||||||
|
this.resource = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
public getMusicVolume(): number {
|
||||||
|
return this.musicVolume;
|
||||||
|
}
|
||||||
|
|
||||||
|
public setMusicVolume(volume: number): void {
|
||||||
|
const nextVolume = this.normalizeVolume(volume);
|
||||||
|
this.musicVolume = nextVolume;
|
||||||
|
if (this.owner === "music") {
|
||||||
|
this.setResourceVolume(nextVolume);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private assertOwnerAvailable(owner: DiscordPlayerOwner): void {
|
||||||
|
if (this.owner !== "none" && this.owner !== owner) {
|
||||||
|
throw new Error(`Discord audio player is owned by ${this.owner}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private canControl(owner?: DiscordPlayerOwner): boolean {
|
||||||
|
return !owner || this.owner === "none" || this.owner === owner;
|
||||||
|
}
|
||||||
|
|
||||||
|
private normalizeVolume(volume: number): number {
|
||||||
|
if (!Number.isFinite(volume)) return this.musicVolume;
|
||||||
|
return Math.max(0, Math.min(1, volume));
|
||||||
|
}
|
||||||
|
|
||||||
|
private setResourceVolume(volume: number): void {
|
||||||
|
if (!this.resource?.volume) return;
|
||||||
|
this.resource.volume.setVolume(volume);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export const discordPlayer = new DiscordPlayer();
|
||||||
@@ -0,0 +1,328 @@
|
|||||||
|
import fs from "node:fs";
|
||||||
|
import path from "node:path";
|
||||||
|
import {
|
||||||
|
type DiscordGatewayAdapterCreator,
|
||||||
|
EndBehaviorType,
|
||||||
|
entersState,
|
||||||
|
getVoiceConnection,
|
||||||
|
joinVoiceChannel,
|
||||||
|
type VoiceConnection,
|
||||||
|
VoiceConnectionStatus,
|
||||||
|
} from "@discordjs/voice";
|
||||||
|
import type { Client, VoiceChannel } from "discord.js-selfbot-v13";
|
||||||
|
import { config } from "../../shared/config/config.js";
|
||||||
|
import { createChildLogger } from "../../shared/logger/logger.js";
|
||||||
|
import { PacketFilter } from "./packetFilter.js";
|
||||||
|
import { subscribeToAudioStream } from "./recorder/audioStream.js";
|
||||||
|
import { OpusDecoder } from "./recorder/decoder.js";
|
||||||
|
import {
|
||||||
|
collectUserMetadata,
|
||||||
|
createSegmentMetadata,
|
||||||
|
} from "./recorder/metadata.js";
|
||||||
|
import { SegmentManager } from "./recorder/segment.js";
|
||||||
|
import {
|
||||||
|
createRecordingSession,
|
||||||
|
finalizeRecordingSession,
|
||||||
|
type RecordingSession,
|
||||||
|
} from "./recorder/sessionRecording.js";
|
||||||
|
import { uploadRecordingSegment } from "./recorder/uploader.js";
|
||||||
|
import { retryWithBackoff } from "../../shared/utils/retry.js";
|
||||||
|
import type { PcmBroadcaster } from "../message-capture/types.js";
|
||||||
|
|
||||||
|
const logger = createChildLogger("recorder");
|
||||||
|
|
||||||
|
const recordingsDir = config.RECORDINGS_DIR;
|
||||||
|
|
||||||
|
// Pastikan folder recordings ada
|
||||||
|
if (!fs.existsSync(recordingsDir)) {
|
||||||
|
fs.mkdirSync(recordingsDir, { recursive: true });
|
||||||
|
}
|
||||||
|
|
||||||
|
const activeSessions = new Map<string, RecordingSession>();
|
||||||
|
|
||||||
|
export function resetActiveSessions(): void {
|
||||||
|
activeSessions.clear();
|
||||||
|
}
|
||||||
|
|
||||||
|
function finalizeActiveRecordingSession(guildId: string): void {
|
||||||
|
const session = activeSessions.get(guildId);
|
||||||
|
if (!session) return;
|
||||||
|
activeSessions.delete(guildId);
|
||||||
|
finalizeRecordingSession(session).catch((error: unknown) => {
|
||||||
|
logger.error({ error }, "Failed to finalize recording session");
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Join ke voice channel dan mulai merekam semua user yang bicara.
|
||||||
|
*/
|
||||||
|
export async function startRecording(
|
||||||
|
client: Client,
|
||||||
|
channel: VoiceChannel,
|
||||||
|
): Promise<VoiceConnection | null> {
|
||||||
|
const connection = joinVoiceChannel({
|
||||||
|
channelId: channel.id,
|
||||||
|
guildId: channel.guild.id,
|
||||||
|
adapterCreator: channel.guild
|
||||||
|
.voiceAdapterCreator as DiscordGatewayAdapterCreator,
|
||||||
|
selfDeaf: false,
|
||||||
|
selfMute: false,
|
||||||
|
debug: true,
|
||||||
|
});
|
||||||
|
|
||||||
|
logger.info({ channelName: channel.name }, "Joining voice channel");
|
||||||
|
|
||||||
|
connection.on("debug", (msg) => {
|
||||||
|
if (config.VERBOSE) {
|
||||||
|
logger.debug({ message: msg }, "Voice debug");
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
connection.on("error", (err) => {
|
||||||
|
logger.error({ error: err }, "Voice connection error");
|
||||||
|
});
|
||||||
|
|
||||||
|
// Tunggu sampai benar-benar terhubung dengan retry logic
|
||||||
|
try {
|
||||||
|
await retryWithBackoff(
|
||||||
|
() =>
|
||||||
|
entersState(
|
||||||
|
connection,
|
||||||
|
VoiceConnectionStatus.Ready,
|
||||||
|
config.VOICE_CONNECTION_TIMEOUT_MS,
|
||||||
|
),
|
||||||
|
{
|
||||||
|
retries: 3,
|
||||||
|
minTimeout: 1000,
|
||||||
|
maxTimeout: 5000,
|
||||||
|
logger,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
logger.info("Connected to voice channel. Recording started");
|
||||||
|
|
||||||
|
// Create recording session after connection is ready
|
||||||
|
const sessionStartTime = Date.now();
|
||||||
|
const session = createRecordingSession({
|
||||||
|
guildId: channel.guild.id,
|
||||||
|
channelId: channel.id,
|
||||||
|
channelName: channel.name,
|
||||||
|
startTime: sessionStartTime,
|
||||||
|
recordingsDir,
|
||||||
|
});
|
||||||
|
activeSessions.set(channel.guild.id, session);
|
||||||
|
} catch (err) {
|
||||||
|
logger.error({ error: err }, "Failed to connect to voice channel");
|
||||||
|
connection.destroy();
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const receiver = connection.receiver;
|
||||||
|
const broadcaster = globalThis as typeof globalThis & PcmBroadcaster;
|
||||||
|
|
||||||
|
// Dengarkan siapapun yang mulai bicara
|
||||||
|
receiver.speaking.on("start", async (userId) => {
|
||||||
|
if (userId === client.user?.id) return;
|
||||||
|
|
||||||
|
const userMetadata = await collectUserMetadata(client, userId, channel);
|
||||||
|
if (userMetadata.bot) return;
|
||||||
|
|
||||||
|
logger.debug(
|
||||||
|
{ userId, username: userMetadata.username },
|
||||||
|
"Voice activity detected",
|
||||||
|
);
|
||||||
|
|
||||||
|
// Notify webserver
|
||||||
|
broadcaster.updateActiveUser?.(userId, {
|
||||||
|
username: userMetadata.username,
|
||||||
|
avatar: userMetadata.avatarUrl,
|
||||||
|
speaking: true,
|
||||||
|
});
|
||||||
|
|
||||||
|
// Jangan record kalau sudah ada stream aktif untuk user ini
|
||||||
|
if (receiver.subscriptions.has(userId)) return;
|
||||||
|
|
||||||
|
const userDir = path.join(recordingsDir, userId);
|
||||||
|
if (!fs.existsSync(userDir)) {
|
||||||
|
fs.mkdirSync(userDir, { recursive: true });
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
// --- OGG file recording with segment rotation ---
|
||||||
|
const packetFilterForOgg = new PacketFilter(
|
||||||
|
config.PACKET_FILTER_MIN_SIZE,
|
||||||
|
);
|
||||||
|
const audioStream = receiver.subscribe(userId, {
|
||||||
|
end: {
|
||||||
|
behavior: EndBehaviorType.AfterSilence,
|
||||||
|
duration: config.AUDIO_STREAM_SILENCE_DURATION_MS,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const oggPacketStream = audioStream.pipe(packetFilterForOgg);
|
||||||
|
const segmentManager = new SegmentManager(
|
||||||
|
userDir,
|
||||||
|
config.RECORDING_SEGMENT_MS,
|
||||||
|
);
|
||||||
|
|
||||||
|
// --- Web broadcast: prism decoder with safe restart and cooldown ---
|
||||||
|
const decoder = new OpusDecoder({
|
||||||
|
cooldownMs: config.DECODER_COOLDOWN_MS,
|
||||||
|
rotateMs: config.DECODER_ROTATE_MS,
|
||||||
|
onData: (pcm) => {
|
||||||
|
if (!broadcaster.broadcastPcmToWeb) return;
|
||||||
|
// Downsample 48kHz stereo → 24kHz mono (left channel, every 2nd sample)
|
||||||
|
const outBuf = Buffer.alloc(pcm.length / 4);
|
||||||
|
for (let i = 0; i < outBuf.length / 2; i++) {
|
||||||
|
outBuf.writeInt16LE(pcm.readInt16LE(i * 8), i * 2);
|
||||||
|
}
|
||||||
|
broadcaster.broadcastPcmToWeb(outBuf, userId);
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
const activeSession = activeSessions.get(channel.guild.id);
|
||||||
|
let currentSegment = segmentManager.open(oggPacketStream);
|
||||||
|
currentSegment.out.on("finish", () => {
|
||||||
|
if (config.VERBOSE) {
|
||||||
|
logger.info({ filename: currentSegment.filename }, "Segment saved");
|
||||||
|
}
|
||||||
|
const endTime = currentSegment.endTime ?? Date.now();
|
||||||
|
if (activeSession) {
|
||||||
|
activeSession.registerSegment({
|
||||||
|
user: userMetadata,
|
||||||
|
oggPath: currentSegment.filename,
|
||||||
|
jsonPath: currentSegment.jsonFilename,
|
||||||
|
startTime: currentSegment.startTime,
|
||||||
|
endTime,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
const metadata = createSegmentMetadata(
|
||||||
|
userMetadata,
|
||||||
|
currentSegment,
|
||||||
|
activeSession?.sessionId ?? `${userId}-0`,
|
||||||
|
activeSession?.sessionId ?? `${channel.guild.id}-${channel.id}-0`,
|
||||||
|
activeSession?.startTime ?? 0,
|
||||||
|
config.RECORDING_SEGMENT_MS,
|
||||||
|
);
|
||||||
|
fs.writeFileSync(
|
||||||
|
currentSegment.jsonFilename,
|
||||||
|
JSON.stringify(metadata, null, 2),
|
||||||
|
);
|
||||||
|
if (config.VERBOSE) {
|
||||||
|
logger.info(
|
||||||
|
{ jsonFile: currentSegment.jsonFilename },
|
||||||
|
"Metadata saved",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Trigger async voice segment upload
|
||||||
|
const segmentId = `${userId}-${currentSegment.startTime}`;
|
||||||
|
uploadRecordingSegment({
|
||||||
|
id: segmentId,
|
||||||
|
oggPath: currentSegment.filename,
|
||||||
|
userId: userMetadata.userId,
|
||||||
|
username: userMetadata.username,
|
||||||
|
avatarUrl: userMetadata.avatarUrl,
|
||||||
|
guildId: channel.guild.id,
|
||||||
|
channelId: channel.id,
|
||||||
|
channelName: channel.name,
|
||||||
|
}).catch((err: unknown) => {
|
||||||
|
const msg = err instanceof Error ? err.message : String(err);
|
||||||
|
logger.error(
|
||||||
|
{ segmentId, error: msg },
|
||||||
|
"Upload segment trigger failed",
|
||||||
|
);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
currentSegment.out.on("error", (err: unknown) => {
|
||||||
|
const msg = err instanceof Error ? err.message : String(err);
|
||||||
|
logger.error({ userId, error: msg }, "File write error");
|
||||||
|
});
|
||||||
|
|
||||||
|
// Feed Opus packets one-by-one
|
||||||
|
subscribeToAudioStream(receiver, userId, {
|
||||||
|
onPacket: (chunk) => {
|
||||||
|
if (chunk.length < 8) return;
|
||||||
|
segmentManager.rotateIfNeeded(oggPacketStream);
|
||||||
|
if (!broadcaster.broadcastPcmToWeb) return;
|
||||||
|
decoder.rotateIfNeeded();
|
||||||
|
decoder.write(chunk);
|
||||||
|
},
|
||||||
|
onEnd: () => {
|
||||||
|
segmentManager.close(oggPacketStream);
|
||||||
|
decoder.destroy();
|
||||||
|
broadcaster.updateActiveUser?.(userId, {
|
||||||
|
username: userMetadata.username,
|
||||||
|
avatar: userMetadata.avatarUrl,
|
||||||
|
speaking: false,
|
||||||
|
});
|
||||||
|
},
|
||||||
|
onError: (error) => {
|
||||||
|
segmentManager.close(oggPacketStream);
|
||||||
|
decoder.destroy();
|
||||||
|
logger.error({ userId, error: error.message }, "Audio stream error");
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
packetFilterForOgg.on("error", (err) => {
|
||||||
|
segmentManager.close(oggPacketStream);
|
||||||
|
logger.error({ userId, error: err.message }, "PacketFilter error");
|
||||||
|
});
|
||||||
|
} catch (e) {
|
||||||
|
logger.error(
|
||||||
|
{ userId, error: e instanceof Error ? e.message : String(e) },
|
||||||
|
"Failed to create stream",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// Handle disconnect yang tidak disengaja
|
||||||
|
connection.on(VoiceConnectionStatus.Disconnected, async () => {
|
||||||
|
if (config.VERBOSE) {
|
||||||
|
logger.warn("Disconnected from voice channel. Reconnecting...");
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
await Promise.race([
|
||||||
|
entersState(
|
||||||
|
connection,
|
||||||
|
VoiceConnectionStatus.Signalling,
|
||||||
|
config.RECONNECT_TIMEOUT_MS,
|
||||||
|
),
|
||||||
|
entersState(
|
||||||
|
connection,
|
||||||
|
VoiceConnectionStatus.Connecting,
|
||||||
|
config.RECONNECT_TIMEOUT_MS,
|
||||||
|
),
|
||||||
|
]);
|
||||||
|
// Berhasil reconnect
|
||||||
|
} catch {
|
||||||
|
logger.error("Could not reconnect. Destroying connection");
|
||||||
|
connection.destroy();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
connection.on(VoiceConnectionStatus.Destroyed, () => {
|
||||||
|
finalizeActiveRecordingSession(channel.guild.id);
|
||||||
|
if (config.VERBOSE) {
|
||||||
|
logger.info("Voice connection destroyed");
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
return connection;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Hentikan recording dan disconnect dari voice channel.
|
||||||
|
*/
|
||||||
|
export function stopRecording(guildId: string): void {
|
||||||
|
const connection = getVoiceConnection(guildId);
|
||||||
|
if (connection) {
|
||||||
|
connection.destroy();
|
||||||
|
if (config.VERBOSE) {
|
||||||
|
logger.info("Recording stopped and disconnected");
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
logger.warn("No active connection to stop");
|
||||||
|
}
|
||||||
|
|
||||||
|
finalizeActiveRecordingSession(guildId);
|
||||||
|
}
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user