refactor: large codebase cleanup - consolidate schemas, migrate to Drizzle ORM, extract frontend components, modernize Docker builds
Build & Deploy / build-and-push (discord-gateway) (push) Failing after 2m22s
Build & Deploy / build-and-push (backend) (push) Failing after 3m22s
Build & Deploy / build-and-push (proxy) (push) Successful in 1m36s
Build & Deploy / deploy (push) Skipped
Build & Deploy / build-and-push (discord-gateway) (push) Failing after 2m22s
Build & Deploy / build-and-push (backend) (push) Failing after 3m22s
Build & Deploy / build-and-push (proxy) (push) Successful in 1m36s
Build & Deploy / deploy (push) Skipped
- Consolidate all DB schema definitions into packages/shared as single source of truth - Migrate backend from raw SQL to Drizzle ORM across all modules - Extract frontend inline UI into separate component files - Refactor discord-gateway circuitBreaker into conversationState + moderationState - Convert messageStore to Proxy singleton pattern - Add validateBody/validateQuery middleware + Zod schemas for API endpoints - Modernize Docker builds with multi-stage + pnpm deploy - Migrate CI/CD from deployment to image-based pipeline - Remove 60+ unused/dead files (~15K lines) - Update color scheme from sky-blue to teal-cyan - Move DB connection management to @bete/shared/database Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
63f21513bd
commit
5802d02e29
@@ -1,10 +1,10 @@
|
||||
/**
|
||||
* E2E API tests — runs against a running backend instance.
|
||||
* Usage: vitest run (or: API_BASE=http://localhost:3001 vitest run)
|
||||
* Usage: API_BASE=http://localhost:3001 vitest run
|
||||
*/
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
const BASE = process.env.API_BASE ?? "https://imphnen.asepharyana.my.id/api";
|
||||
const BASE = process.env.API_BASE ?? "http://localhost:3001/api";
|
||||
|
||||
async function api(path: string, init?: RequestInit) {
|
||||
const res = await fetch(`${BASE}${path}`, {
|
||||
|
||||
@@ -6,17 +6,16 @@ import express, {
|
||||
type Response,
|
||||
} from "express";
|
||||
import helmet from "helmet";
|
||||
import { createAnalysisRouter } from "../modules/analysis/analysis.routes.js";
|
||||
import { createConfigRouter } from "../modules/config/config.routes.js";
|
||||
import { createDashboardRouter } from "../modules/dashboard/dashboard.routes.js";
|
||||
import { createHealthRouter } from "../modules/health/health.routes.js";
|
||||
import { createMascotChatRouter } from "../modules/mascot-chat/mascot-chat.routes.js";
|
||||
import { createMediaRouter } from "../modules/media/media.routes.js";
|
||||
import { createMessagesRouter } from "../modules/messages/messages.routes.js";
|
||||
import { createRecordingsRouter } from "../modules/recordings/recordings.routes.js";
|
||||
import { createUiStateRouter } from "../modules/ui-state/ui-state.routes.js";
|
||||
import { createGuildsRouter } from "../modules/voice/guilds.routes.js";
|
||||
import { createVoiceRouter } from "../modules/voice/voice.routes.js";
|
||||
import { createAnalysisRouter } from "../modules/analysis/index.js";
|
||||
import { createConfigRouter } from "../modules/config/index.js";
|
||||
import { createDashboardRouter } from "../modules/dashboard/index.js";
|
||||
import { createHealthRouter } from "../modules/health/index.js";
|
||||
import { createMascotChatRouter } from "../modules/mascot-chat/index.js";
|
||||
import { createMediaRouter } from "../modules/media/index.js";
|
||||
import { createMessagesRouter } from "../modules/messages/index.js";
|
||||
import { createRecordingsRouter } from "../modules/recordings/index.js";
|
||||
import { createUiStateRouter } from "../modules/ui-state/index.js";
|
||||
import { createVoiceRouter } from "../modules/voice/index.js";
|
||||
import { errorHandler } from "../shared/middlewares/index.js";
|
||||
|
||||
// Auth removed — dashboard is public
|
||||
@@ -68,8 +67,6 @@ export function createHttpApp(): Express {
|
||||
app.use("/api", createMascotChatRouter());
|
||||
app.use("/api", createRecordingsRouter());
|
||||
app.use("/api", createUiStateRouter());
|
||||
app.use("/api/guilds", createGuildsRouter());
|
||||
|
||||
app.use("/api", createMediaRouter());
|
||||
app.use("/api", createVoiceRouter());
|
||||
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
import { pgMessagesTable } from "@bete/shared";
|
||||
import { createChildLogger } from "@bete/shared/logger";
|
||||
import { getPool } from "../../shared/database/index.js";
|
||||
import { and, desc, eq, ilike, type SQL } from "drizzle-orm";
|
||||
import { getDatabase } from "../../shared/database/index.js";
|
||||
import {
|
||||
type MappedMessage,
|
||||
mapMessageRow,
|
||||
@@ -19,44 +21,27 @@ export type AnalysisSearchResult = MappedMessage;
|
||||
|
||||
export class AnalysisRepository {
|
||||
async search(query: AnalysisSearchQuery): Promise<AnalysisSearchResult[]> {
|
||||
const pool = getPool();
|
||||
const db = getDatabase();
|
||||
const { q = "", channelId, guildId, limit = 20 } = query;
|
||||
|
||||
logger.debug({ q, channelId, guildId, limit }, "Searching analysis");
|
||||
|
||||
const searchPattern = `%${q}%`;
|
||||
const clauses: string[] = ["content ILIKE $1"];
|
||||
const params: (string | number)[] = [searchPattern];
|
||||
let p = 2;
|
||||
const conditions: SQL[] = [ilike(pgMessagesTable.content, `%${q}%`)];
|
||||
|
||||
if (guildId) {
|
||||
clauses.push(`guild_id = $${p}`);
|
||||
params.push(guildId);
|
||||
p++;
|
||||
conditions.push(eq(pgMessagesTable.guild_id, guildId));
|
||||
}
|
||||
|
||||
if (channelId) {
|
||||
clauses.push(`channel_id = $${p}`);
|
||||
params.push(channelId);
|
||||
p++;
|
||||
conditions.push(eq(pgMessagesTable.channel_id, channelId));
|
||||
}
|
||||
|
||||
const where = clauses.join(" AND ");
|
||||
const { rows } = await pool.query(
|
||||
`SELECT
|
||||
id, guild_id, channel_id, thread_id,
|
||||
user_id, username, avatar_url,
|
||||
content, edited_content, created_at, edited_at, deleted_at,
|
||||
type, metadata,
|
||||
ai_status, ai_moderation_flags, ai_moderation_score,
|
||||
ai_analysis, ai_categories, ai_severity, ai_confidence,
|
||||
ai_recommended_action, ai_analyzed_at, ai_error
|
||||
FROM messages
|
||||
WHERE ${where}
|
||||
ORDER BY created_at DESC
|
||||
LIMIT $${p}`,
|
||||
[...params, limit],
|
||||
);
|
||||
const rows = await db
|
||||
.select()
|
||||
.from(pgMessagesTable)
|
||||
.where(and(...conditions))
|
||||
.orderBy(desc(pgMessagesTable.created_at))
|
||||
.limit(limit);
|
||||
|
||||
return rows.map((r) => mapMessageRow(r as Record<string, unknown>));
|
||||
}
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
export { createAnalysisRouter } from "./analysis.routes.js";
|
||||
@@ -0,0 +1 @@
|
||||
export { createConfigRouter } from "./config.routes.js";
|
||||
@@ -1,16 +1,23 @@
|
||||
import { createChildLogger } from "@bete/shared/logger";
|
||||
import { getPool } from "../../shared/database/index.js";
|
||||
import {
|
||||
pgChannelCulturesTable,
|
||||
pgMessagesTable,
|
||||
pgUserProfilesTable,
|
||||
pgUserReputationsTable,
|
||||
pgVoiceRecordingsTable,
|
||||
} from "@bete/shared";
|
||||
import type { SQL } from "drizzle-orm";
|
||||
import { sql } from "drizzle-orm";
|
||||
import { getDatabase } from "../../shared/database/index.js";
|
||||
import type { ListUsersQuery } from "./dashboard.service.js";
|
||||
|
||||
const _logger = createChildLogger("dashboard.repository");
|
||||
|
||||
export class DashboardRepository {
|
||||
async getStats() {
|
||||
const pool = getPool();
|
||||
const db = getDatabase();
|
||||
|
||||
const oneDayAgo = Date.now() - 86400000;
|
||||
|
||||
// Total messages and breakdown by ai_status
|
||||
const msgResult = await pool.query(
|
||||
`
|
||||
const msgResult = await db.execute(sql`
|
||||
SELECT
|
||||
COUNT(*)::int AS total_messages,
|
||||
COUNT(*) FILTER (WHERE ai_status = 'flagged')::int AS total_flagged,
|
||||
@@ -20,32 +27,30 @@ export class DashboardRepository {
|
||||
COUNT(*) FILTER (WHERE ai_status = 'pending')::int AS total_pending,
|
||||
COUNT(*) FILTER (WHERE ai_status = 'processing')::int AS total_processing,
|
||||
COUNT(DISTINCT user_id)::int AS total_users,
|
||||
COUNT(*) FILTER (WHERE created_at >= $1)::int AS today_messages,
|
||||
COUNT(*) FILTER (WHERE ai_status = 'flagged' AND created_at >= $1)::int AS today_flagged,
|
||||
COUNT(DISTINCT user_id) FILTER (WHERE created_at >= $2)::int AS active_users_24h
|
||||
FROM messages
|
||||
`,
|
||||
[Date.now() - 86400000, Date.now() - 86400000],
|
||||
);
|
||||
COUNT(*) FILTER (WHERE created_at >= ${oneDayAgo})::int AS today_messages,
|
||||
COUNT(*) FILTER (WHERE ai_status = 'flagged' AND created_at >= ${oneDayAgo})::int AS today_flagged,
|
||||
COUNT(DISTINCT user_id) FILTER (WHERE created_at >= ${oneDayAgo})::int AS active_users_24h
|
||||
FROM ${pgMessagesTable}
|
||||
`);
|
||||
|
||||
const msgRow = msgResult.rows[0];
|
||||
const msgRow = msgResult.rows[0] as Record<string, unknown> | undefined;
|
||||
|
||||
// Total voice recordings
|
||||
const voiceResult = await pool.query(`
|
||||
SELECT COUNT(*)::int AS count FROM voice_recordings
|
||||
const voiceResult = await db.execute(sql`
|
||||
SELECT COUNT(*)::int AS count FROM ${pgVoiceRecordingsTable}
|
||||
`);
|
||||
|
||||
// Total AI user profiles
|
||||
const profileResult = await pool.query(`
|
||||
SELECT COUNT(*)::int AS count FROM user_profiles
|
||||
const profileResult = await db.execute(sql`
|
||||
SELECT COUNT(*)::int AS count FROM ${pgUserProfilesTable}
|
||||
`);
|
||||
|
||||
// Top channels by message count
|
||||
const topChannels = await pool.query(`
|
||||
const topChannels = await db.execute(sql`
|
||||
SELECT channel_id,
|
||||
COALESCE(NULLIF((metadata::jsonb -> 'channel' ->> 'channelName'), ''), channel_id) AS channel_name,
|
||||
COUNT(*)::int AS message_count
|
||||
FROM messages
|
||||
FROM ${pgMessagesTable}
|
||||
WHERE metadata IS NOT NULL AND metadata != ''
|
||||
GROUP BY channel_id, (metadata::jsonb -> 'channel' ->> 'channelName')
|
||||
ORDER BY COUNT(*) DESC
|
||||
@@ -78,31 +83,26 @@ export class DashboardRepository {
|
||||
}
|
||||
|
||||
async listUsers(query: ListUsersQuery) {
|
||||
const pool = getPool();
|
||||
const db = getDatabase();
|
||||
const limit = query.limit ?? 20;
|
||||
const conditions: string[] = [];
|
||||
const params: unknown[] = [];
|
||||
let paramIdx = 1;
|
||||
const conditions: SQL[] = [];
|
||||
|
||||
if (query.search) {
|
||||
conditions.push(
|
||||
`(m.user_id ILIKE $${paramIdx} OR m.username ILIKE $${paramIdx})`,
|
||||
sql`(m.user_id ILIKE ${`%${query.search}%`} OR m.username ILIKE ${`%${query.search}%`})`,
|
||||
);
|
||||
params.push(`%${query.search}%`);
|
||||
paramIdx++;
|
||||
}
|
||||
|
||||
if (query.cursor) {
|
||||
conditions.push(`m.last_message_at < $${paramIdx}`);
|
||||
params.push(Number(query.cursor));
|
||||
paramIdx++;
|
||||
conditions.push(sql`m.last_message_at < ${Number(query.cursor)}`);
|
||||
}
|
||||
|
||||
const whereClause =
|
||||
conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
|
||||
conditions.length > 0
|
||||
? sql`WHERE ${sql.join(conditions, sql` AND `)}`
|
||||
: sql``;
|
||||
|
||||
const { rows } = await pool.query(
|
||||
`
|
||||
const { rows } = await db.execute(sql`
|
||||
SELECT
|
||||
m.user_id,
|
||||
m.username,
|
||||
@@ -120,17 +120,15 @@ export class DashboardRepository {
|
||||
COUNT(*)::int AS total_messages,
|
||||
COUNT(*) FILTER (WHERE ai_status = 'flagged')::int AS flagged_count,
|
||||
MAX(created_at) AS last_message_at
|
||||
FROM messages
|
||||
FROM ${pgMessagesTable}
|
||||
GROUP BY user_id, username, avatar_url
|
||||
) m
|
||||
LEFT JOIN user_profiles p ON p.user_id = m.user_id
|
||||
LEFT JOIN user_reputations r ON r.user_id = m.user_id
|
||||
LEFT JOIN ${pgUserProfilesTable} p ON p.user_id = m.user_id
|
||||
LEFT JOIN ${pgUserReputationsTable} r ON r.user_id = m.user_id
|
||||
${whereClause}
|
||||
ORDER BY m.last_message_at DESC NULLS LAST
|
||||
LIMIT $${paramIdx}
|
||||
`,
|
||||
[...params, limit + 1],
|
||||
);
|
||||
LIMIT ${limit + 1}
|
||||
`);
|
||||
|
||||
const data = (rows as Record<string, unknown>[])
|
||||
.slice(0, limit)
|
||||
@@ -158,31 +156,26 @@ export class DashboardRepository {
|
||||
}
|
||||
|
||||
async listChannels(query: ListUsersQuery & { guildId?: string }) {
|
||||
const pool = getPool();
|
||||
const db = getDatabase();
|
||||
const limit = query.limit ?? 20;
|
||||
const conditions: string[] = [];
|
||||
const params: unknown[] = [];
|
||||
let paramIdx = 1;
|
||||
const conditions: SQL[] = [];
|
||||
|
||||
if (query.search) {
|
||||
conditions.push(
|
||||
`(m.channel_id ILIKE $${paramIdx} OR m.channel_name ILIKE $${paramIdx})`,
|
||||
sql`(m.channel_id ILIKE ${`%${query.search}%`} OR m.channel_name ILIKE ${`%${query.search}%`})`,
|
||||
);
|
||||
params.push(`%${query.search}%`);
|
||||
paramIdx++;
|
||||
}
|
||||
|
||||
if (query.guildId) {
|
||||
conditions.push(`m.guild_id = $${paramIdx}`);
|
||||
params.push(query.guildId);
|
||||
paramIdx++;
|
||||
conditions.push(sql`m.guild_id = ${query.guildId}`);
|
||||
}
|
||||
|
||||
const whereClause =
|
||||
conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
|
||||
conditions.length > 0
|
||||
? sql`WHERE ${sql.join(conditions, sql` AND `)}`
|
||||
: sql``;
|
||||
|
||||
const { rows } = await pool.query(
|
||||
`
|
||||
const { rows } = await db.execute(sql`
|
||||
SELECT
|
||||
m.channel_id,
|
||||
m.channel_name,
|
||||
@@ -200,16 +193,14 @@ export class DashboardRepository {
|
||||
COUNT(*)::int AS total_messages,
|
||||
COUNT(*) FILTER (WHERE ai_status = 'flagged')::int AS flagged_count,
|
||||
MAX(created_at) AS last_message_at
|
||||
FROM messages
|
||||
FROM ${pgMessagesTable}
|
||||
GROUP BY channel_id, guild_id, (metadata::jsonb -> 'channel' ->> 'channelName')
|
||||
) m
|
||||
LEFT JOIN channel_cultures c ON c.channel_id = m.channel_id
|
||||
LEFT JOIN ${pgChannelCulturesTable} c ON c.channel_id = m.channel_id
|
||||
${whereClause}
|
||||
ORDER BY m.total_messages DESC
|
||||
LIMIT $${paramIdx}
|
||||
`,
|
||||
[...params, limit + 1],
|
||||
);
|
||||
LIMIT ${limit + 1}
|
||||
`);
|
||||
|
||||
const data = ((rows as Record<string, unknown>[]) || [])
|
||||
.slice(0, limit)
|
||||
@@ -234,10 +225,9 @@ export class DashboardRepository {
|
||||
}
|
||||
|
||||
async getChannelDetail(channelId: string) {
|
||||
const pool = getPool();
|
||||
const db = getDatabase();
|
||||
|
||||
const channelResult = await pool.query(
|
||||
`
|
||||
const channelResult = await db.execute(sql`
|
||||
SELECT
|
||||
m.channel_id,
|
||||
m.channel_name,
|
||||
@@ -255,28 +245,23 @@ export class DashboardRepository {
|
||||
COUNT(*)::int AS total_messages,
|
||||
COUNT(*) FILTER (WHERE ai_status = 'flagged')::int AS flagged_count,
|
||||
COUNT(*) FILTER (WHERE ai_status = 'clean')::int AS clean_count
|
||||
FROM messages
|
||||
WHERE channel_id = $1
|
||||
FROM ${pgMessagesTable}
|
||||
WHERE channel_id = ${channelId}
|
||||
GROUP BY channel_id, guild_id, (metadata::jsonb -> 'channel' ->> 'channelName')
|
||||
) m
|
||||
LEFT JOIN channel_cultures c ON c.channel_id = m.channel_id
|
||||
`,
|
||||
[channelId],
|
||||
);
|
||||
LEFT JOIN ${pgChannelCulturesTable} c ON c.channel_id = m.channel_id
|
||||
`);
|
||||
|
||||
const row = channelResult.rows[0] as Record<string, unknown> | undefined;
|
||||
if (!row) return null;
|
||||
|
||||
const recent = await pool.query(
|
||||
`
|
||||
const recent = await db.execute(sql`
|
||||
SELECT id, content, channel_id, created_at, ai_status, username
|
||||
FROM messages
|
||||
WHERE channel_id = $1
|
||||
FROM ${pgMessagesTable}
|
||||
WHERE channel_id = ${channelId}
|
||||
ORDER BY created_at DESC
|
||||
LIMIT 20
|
||||
`,
|
||||
[channelId],
|
||||
);
|
||||
`);
|
||||
|
||||
return {
|
||||
channel_id: String(row.channel_id),
|
||||
@@ -301,11 +286,9 @@ export class DashboardRepository {
|
||||
}
|
||||
|
||||
async getUserDetail(userId: string) {
|
||||
const pool = getPool();
|
||||
const db = getDatabase();
|
||||
|
||||
// Basic user info + profile + reputation
|
||||
const userResult = await pool.query(
|
||||
`
|
||||
const userResult = await db.execute(sql`
|
||||
SELECT
|
||||
m.user_id,
|
||||
m.username,
|
||||
@@ -326,32 +309,26 @@ export class DashboardRepository {
|
||||
COUNT(*)::int AS total_messages,
|
||||
COUNT(*) FILTER (WHERE ai_status = 'flagged')::int AS flagged_count,
|
||||
COUNT(*) FILTER (WHERE ai_status = 'clean')::int AS clean_count
|
||||
FROM messages
|
||||
WHERE user_id = $1
|
||||
FROM ${pgMessagesTable}
|
||||
WHERE user_id = ${userId}
|
||||
GROUP BY user_id, username, avatar_url
|
||||
) m
|
||||
LEFT JOIN user_profiles p ON p.user_id = m.user_id
|
||||
LEFT JOIN user_reputations r ON r.user_id = m.user_id
|
||||
`,
|
||||
[userId],
|
||||
);
|
||||
LEFT JOIN ${pgUserProfilesTable} p ON p.user_id = m.user_id
|
||||
LEFT JOIN ${pgUserReputationsTable} r ON r.user_id = m.user_id
|
||||
`);
|
||||
|
||||
const row = userResult.rows[0] as Record<string, unknown> | undefined;
|
||||
if (!row) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Recent messages
|
||||
const recent = await pool.query(
|
||||
`
|
||||
const recent = await db.execute(sql`
|
||||
SELECT id, content, channel_id, created_at, ai_status
|
||||
FROM messages
|
||||
WHERE user_id = $1
|
||||
FROM ${pgMessagesTable}
|
||||
WHERE user_id = ${userId}
|
||||
ORDER BY created_at DESC
|
||||
LIMIT 20
|
||||
`,
|
||||
[userId],
|
||||
);
|
||||
`);
|
||||
|
||||
return {
|
||||
user_id: String(row.user_id),
|
||||
@@ -364,13 +341,13 @@ export class DashboardRepository {
|
||||
last_analyzed_at: row.last_analyzed_at
|
||||
? Number(row.last_analyzed_at)
|
||||
: null,
|
||||
trust_score: row.trust_score !== null ? Number(row.trust_score) : null,
|
||||
trust_score: row.trust_score != null ? Number(row.trust_score) : null,
|
||||
clean_message_streak:
|
||||
row.clean_message_streak !== null
|
||||
row.clean_message_streak != null
|
||||
? Number(row.clean_message_streak)
|
||||
: null,
|
||||
total_infractions:
|
||||
row.total_infractions !== null ? Number(row.total_infractions) : null,
|
||||
row.total_infractions != null ? Number(row.total_infractions) : null,
|
||||
recent_messages: (recent.rows as Record<string, unknown>[]).map((r) => ({
|
||||
id: String(r.id),
|
||||
content: String(r.content),
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
export { createDashboardRouter } from "./dashboard.routes.js";
|
||||
@@ -1,5 +1,6 @@
|
||||
import { createChildLogger } from "@bete/shared/logger";
|
||||
import { getPool } from "../../shared/database/index.js";
|
||||
import { sql } from "drizzle-orm";
|
||||
import { getDatabase } from "../../shared/database/index.js";
|
||||
|
||||
const logger = createChildLogger("health.repository");
|
||||
|
||||
@@ -7,8 +8,8 @@ export class HealthRepository {
|
||||
async checkDatabaseConnection() {
|
||||
try {
|
||||
logger.debug("Running database health check");
|
||||
const pool = getPool();
|
||||
await pool.query("SELECT 1 AS result");
|
||||
const db = getDatabase();
|
||||
await db.execute(sql`SELECT 1 AS result`);
|
||||
logger.debug("Database health check passed");
|
||||
return { connected: true };
|
||||
} catch (err: unknown) {
|
||||
|
||||
@@ -1,8 +1,5 @@
|
||||
import { createChildLogger } from "@bete/shared/logger";
|
||||
import { healthRepository } from "./health.repository.js";
|
||||
|
||||
const _logger = createChildLogger("health.service");
|
||||
|
||||
export class HealthService {
|
||||
async getHealth(verbose = false) {
|
||||
const dbStatus = await healthRepository.checkDatabaseConnection();
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
export { createHealthRouter } from "./health.routes.js";
|
||||
@@ -0,0 +1 @@
|
||||
export { createMascotChatRouter } from "./mascot-chat.routes.js";
|
||||
@@ -11,8 +11,12 @@ interface AuthenticatedRequest extends Request {
|
||||
|
||||
export const handleMascotChat = asyncHandler(
|
||||
async (req: Request, res: Response) => {
|
||||
const { message, context } = req.body;
|
||||
const { message, context } = req.body as {
|
||||
message: string;
|
||||
context?: Record<string, unknown>;
|
||||
};
|
||||
|
||||
// Validate required fields
|
||||
if (!message || typeof message !== "string") {
|
||||
return res.status(400).json({
|
||||
error: "INVALID_INPUT",
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
import { pgMascotChatMessagesTable, pgMessagesTable } from "@bete/shared";
|
||||
import { createChildLogger } from "@bete/shared/logger";
|
||||
import { getPool } from "../../shared/database/index.js";
|
||||
import { and, desc, eq, type SQL, sql } from "drizzle-orm";
|
||||
import { getDatabase } from "../../shared/database/index.js";
|
||||
|
||||
const logger = createChildLogger("mascot-chat.repository");
|
||||
|
||||
@@ -38,22 +40,15 @@ export interface ServerInsights {
|
||||
|
||||
export class MascotChatRepository {
|
||||
async saveConversation(input: SaveConversationInput): Promise<void> {
|
||||
const pool = getPool();
|
||||
const db = getDatabase();
|
||||
|
||||
await pool.query(
|
||||
`
|
||||
INSERT INTO mascot_chat_messages
|
||||
(user_id, user_message, mascot_response, context, created_at)
|
||||
VALUES ($1, $2, $3, $4::jsonb, $5)
|
||||
`,
|
||||
[
|
||||
input.userId,
|
||||
input.userMessage,
|
||||
input.mascotResponse,
|
||||
JSON.stringify(input.context ?? {}),
|
||||
input.timestamp.toISOString(),
|
||||
],
|
||||
);
|
||||
await db.insert(pgMascotChatMessagesTable).values({
|
||||
user_id: input.userId,
|
||||
user_message: input.userMessage,
|
||||
mascot_response: input.mascotResponse,
|
||||
context: (input.context ?? {}) as Record<string, unknown>,
|
||||
created_at: input.timestamp,
|
||||
});
|
||||
|
||||
logger.debug({ userId: input.userId }, "Conversation saved");
|
||||
}
|
||||
@@ -62,69 +57,61 @@ export class MascotChatRepository {
|
||||
userId: string,
|
||||
limit: number,
|
||||
): Promise<MascotChatHistoryRow[]> {
|
||||
const pool = getPool();
|
||||
const db = getDatabase();
|
||||
|
||||
const { rows } = await pool.query<MascotChatHistoryRow>(
|
||||
`
|
||||
SELECT id, user_id, user_message, mascot_response, context, created_at
|
||||
FROM mascot_chat_messages
|
||||
WHERE user_id = $1
|
||||
ORDER BY created_at DESC
|
||||
LIMIT $2
|
||||
`,
|
||||
[userId, limit],
|
||||
);
|
||||
const rows = await db
|
||||
.select()
|
||||
.from(pgMascotChatMessagesTable)
|
||||
.where(eq(pgMascotChatMessagesTable.user_id, userId))
|
||||
.orderBy(desc(pgMascotChatMessagesTable.created_at))
|
||||
.limit(limit);
|
||||
|
||||
logger.debug({ userId, count: rows.length }, "Chat history fetched");
|
||||
return rows.reverse();
|
||||
return rows.reverse() as unknown as MascotChatHistoryRow[];
|
||||
}
|
||||
|
||||
async clearChatHistory(userId: string): Promise<void> {
|
||||
const pool = getPool();
|
||||
const db = getDatabase();
|
||||
|
||||
const { rowCount } = await pool.query(
|
||||
`DELETE FROM mascot_chat_messages WHERE user_id = $1`,
|
||||
[userId],
|
||||
const deleted = await db
|
||||
.delete(pgMascotChatMessagesTable)
|
||||
.where(eq(pgMascotChatMessagesTable.user_id, userId))
|
||||
.returning({ id: pgMascotChatMessagesTable.id });
|
||||
|
||||
logger.info(
|
||||
{ userId, deletedRows: deleted.length },
|
||||
"Chat history cleared",
|
||||
);
|
||||
|
||||
logger.info({ userId, deletedRows: rowCount ?? 0 }, "Chat history cleared");
|
||||
}
|
||||
|
||||
async getServerInsights(
|
||||
guildId?: string,
|
||||
channelId?: string,
|
||||
): Promise<ServerInsights> {
|
||||
const pool = getPool();
|
||||
|
||||
try {
|
||||
const params: string[] = [];
|
||||
const clauses: string[] = [];
|
||||
const db = getDatabase();
|
||||
const conditions: SQL[] = [];
|
||||
|
||||
if (guildId) {
|
||||
params.push(guildId);
|
||||
clauses.push(`guild_id = $${params.length}`);
|
||||
conditions.push(eq(pgMessagesTable.guild_id, guildId));
|
||||
}
|
||||
if (channelId) {
|
||||
params.push(channelId);
|
||||
clauses.push(`channel_id = $${params.length}`);
|
||||
conditions.push(eq(pgMessagesTable.channel_id, channelId));
|
||||
}
|
||||
|
||||
const where = clauses.length ? `WHERE ${clauses.join(" AND ")}` : "";
|
||||
const where = conditions.length > 0 ? and(...conditions) : undefined;
|
||||
|
||||
const { rows } = await pool.query<ServerInsights>(
|
||||
`
|
||||
SELECT
|
||||
COUNT(*)::int AS total_messages,
|
||||
COUNT(DISTINCT user_id)::int AS active_users,
|
||||
COUNT(*) FILTER (WHERE ai_status = 'flagged')::int AS flagged,
|
||||
COUNT(*) FILTER (WHERE ai_status = 'warn')::int AS warned
|
||||
FROM messages
|
||||
${where}
|
||||
`,
|
||||
params,
|
||||
);
|
||||
const [result] = await db
|
||||
.select({
|
||||
total_messages: sql<number>`COUNT(*)::int`,
|
||||
active_users: sql<number>`COUNT(DISTINCT ${pgMessagesTable.user_id})::int`,
|
||||
flagged: sql<number>`COUNT(*) FILTER (WHERE ${pgMessagesTable.ai_status} = 'flagged')::int`,
|
||||
warned: sql<number>`COUNT(*) FILTER (WHERE ${pgMessagesTable.ai_status} = 'warn')::int`,
|
||||
})
|
||||
.from(pgMessagesTable)
|
||||
.where(where);
|
||||
|
||||
const insights = rows[0] ?? {
|
||||
const insights = result ?? {
|
||||
total_messages: 0,
|
||||
active_users: 0,
|
||||
flagged: 0,
|
||||
|
||||
@@ -1,14 +1,20 @@
|
||||
import express, { type Router } from "express";
|
||||
import { validateBody } from "../../shared/middlewares/index.js";
|
||||
import {
|
||||
clearMascotChatHistory,
|
||||
getMascotChatHistory,
|
||||
handleMascotChat,
|
||||
} from "./mascot-chat.controller.js";
|
||||
import { chatRequestSchema } from "./mascot-chat.schema.js";
|
||||
|
||||
export function createMascotChatRouter(): Router {
|
||||
const router = express.Router();
|
||||
|
||||
router.post("/mascot/chat", handleMascotChat);
|
||||
router.post(
|
||||
"/mascot/chat",
|
||||
validateBody(chatRequestSchema),
|
||||
handleMascotChat,
|
||||
);
|
||||
router.get("/mascot/chat/history", getMascotChatHistory);
|
||||
router.delete("/mascot/chat/history", clearMascotChatHistory);
|
||||
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
export { createMediaRouter } from "./media.routes.js";
|
||||
@@ -1,7 +1,8 @@
|
||||
import { createChildLogger } from "@bete/shared/logger";
|
||||
import type { Request, Response, Router } from "express";
|
||||
import express from "express";
|
||||
import { asyncHandler } from "../../shared/middlewares/index.js";
|
||||
import { asyncHandler, validateBody } from "../../shared/middlewares/index.js";
|
||||
import { mediaQueueSchema, mediaVolumeSchema } from "./media.schema.js";
|
||||
import { getStatus, queue, setVolume, skip, stop } from "./media.service.js";
|
||||
|
||||
const logger = createChildLogger("media.routes");
|
||||
@@ -22,16 +23,12 @@ export function createMediaRouter(): Router {
|
||||
// POST /api/media/queue
|
||||
router.post(
|
||||
"/media/queue",
|
||||
validateBody(mediaQueueSchema),
|
||||
asyncHandler(async (req: Request, res: Response) => {
|
||||
const source = req.body?.source as string | undefined;
|
||||
if (!source) {
|
||||
res.status(400).json({
|
||||
error: "VALIDATION_ERROR",
|
||||
message: "source is required",
|
||||
});
|
||||
return;
|
||||
}
|
||||
const mode = (req.body?.mode as "music" | "screen") ?? "music";
|
||||
const { source, mode } = req.body as {
|
||||
source: string;
|
||||
mode: "music" | "screen";
|
||||
};
|
||||
logger.debug({ source, mode }, "Media queue requested");
|
||||
const state = await queue(source, mode);
|
||||
res.json(state);
|
||||
@@ -61,15 +58,9 @@ export function createMediaRouter(): Router {
|
||||
// POST /api/media/volume
|
||||
router.post(
|
||||
"/media/volume",
|
||||
validateBody(mediaVolumeSchema),
|
||||
asyncHandler(async (req: Request, res: Response) => {
|
||||
const volume = Number(req.body?.volume ?? 1.0);
|
||||
if (Number.isNaN(volume) || volume < 0 || volume > 1) {
|
||||
res.status(400).json({
|
||||
error: "VALIDATION_ERROR",
|
||||
message: "volume must be a number between 0 and 1",
|
||||
});
|
||||
return;
|
||||
}
|
||||
const { volume } = req.body as { volume: number };
|
||||
logger.debug({ volume }, "Media volume requested");
|
||||
const state = await setVolume(volume);
|
||||
res.json(state);
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
import { z } from "zod";
|
||||
|
||||
export const mediaQueueSchema = z.object({
|
||||
source: z.string().min(1, "source is required"),
|
||||
mode: z.enum(["music", "screen"]).default("music"),
|
||||
});
|
||||
|
||||
export const mediaVolumeSchema = z.object({
|
||||
volume: z.number().min(0).max(1).default(1.0),
|
||||
});
|
||||
|
||||
export type MediaQueueInput = z.infer<typeof mediaQueueSchema>;
|
||||
export type MediaVolumeInput = z.infer<typeof mediaVolumeSchema>;
|
||||
@@ -0,0 +1 @@
|
||||
export { createMessagesRouter } from "./messages.routes.js";
|
||||
@@ -1,84 +1,68 @@
|
||||
import { createChildLogger } from "@bete/shared/logger";
|
||||
import type { NextFunction, Request, Response } from "express";
|
||||
import { asyncHandler, requireParam } from "../../shared/middlewares/index.js";
|
||||
import type { Request, Response } from "express";
|
||||
import { asyncHandler } from "../../shared/middlewares/index.js";
|
||||
import { messageQuerySchema } from "./messages.schema.js";
|
||||
import { messagesService } from "./messages.service.js";
|
||||
|
||||
const logger = createChildLogger("messages.controller");
|
||||
|
||||
export function handleListMessages(
|
||||
req: Request,
|
||||
res: Response,
|
||||
next: NextFunction,
|
||||
) {
|
||||
return asyncHandler(async (req: Request, res: Response) => {
|
||||
export const handleListMessages = asyncHandler(
|
||||
async (req: Request, res: Response) => {
|
||||
const query = messageQuerySchema.parse(req.query);
|
||||
logger.debug({ query }, "Handling list messages request");
|
||||
const result = await messagesService.listMessages(query);
|
||||
res.json(result);
|
||||
})(req, res, next);
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
export function handleGetMessagesByChannel(
|
||||
req: Request,
|
||||
res: Response,
|
||||
next: NextFunction,
|
||||
) {
|
||||
return asyncHandler(async (req: Request, res: Response) => {
|
||||
const channelId = requireParam(
|
||||
req.params.channelId,
|
||||
"route parameter",
|
||||
"channelId",
|
||||
);
|
||||
export const handleGetMessagesByChannel = asyncHandler(
|
||||
async (req: Request, res: Response) => {
|
||||
if (!req.params.channelId) {
|
||||
res.status(400).json({ error: "Missing route parameter: channelId" });
|
||||
return;
|
||||
}
|
||||
const channelId = req.params.channelId as string;
|
||||
const query = messageQuerySchema.parse(req.query);
|
||||
logger.debug({ channelId, query }, "Handling get messages by channel");
|
||||
const result = await messagesService.getMessagesByChannel(channelId, query);
|
||||
res.json(result);
|
||||
})(req, res, next);
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
export function handleGetMessageById(
|
||||
req: Request,
|
||||
res: Response,
|
||||
next: NextFunction,
|
||||
) {
|
||||
return asyncHandler(async (req: Request, res: Response) => {
|
||||
const id = requireParam(req.params.id, "route parameter", "id");
|
||||
export const handleGetMessageById = asyncHandler(
|
||||
async (req: Request, res: Response) => {
|
||||
if (!req.params.id) {
|
||||
res.status(400).json({ error: "Missing route parameter: id" });
|
||||
return;
|
||||
}
|
||||
const id = req.params.id as string;
|
||||
logger.debug({ id }, "Handling get message by ID");
|
||||
const result = await messagesService.getMessageById(id);
|
||||
res.json(result);
|
||||
})(req, res, next);
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
export function handleGetImageMessages(
|
||||
req: Request,
|
||||
res: Response,
|
||||
next: NextFunction,
|
||||
) {
|
||||
return asyncHandler(async (req: Request, res: Response) => {
|
||||
const guildId = requireParam(
|
||||
req.query.guildId as string,
|
||||
"query parameter",
|
||||
"guildId",
|
||||
);
|
||||
export const handleGetImageMessages = asyncHandler(
|
||||
async (req: Request, res: Response) => {
|
||||
const guildId = req.query.guildId as string | undefined;
|
||||
if (!guildId) {
|
||||
res.status(400).json({ error: "Missing query parameter: guildId" });
|
||||
return;
|
||||
}
|
||||
const limit = Number(req.query.limit) || 50;
|
||||
logger.debug({ guildId, limit }, "Handling get image messages");
|
||||
const result = await messagesService.getImageMessages(guildId, limit);
|
||||
res.json(result);
|
||||
})(req, res, next);
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
export function handleGetAttachmentsByChannel(
|
||||
req: Request,
|
||||
res: Response,
|
||||
next: NextFunction,
|
||||
) {
|
||||
return asyncHandler(async (req: Request, res: Response) => {
|
||||
const channelId = requireParam(
|
||||
req.params.channelId,
|
||||
"route parameter",
|
||||
"channelId",
|
||||
);
|
||||
export const handleGetAttachmentsByChannel = asyncHandler(
|
||||
async (req: Request, res: Response) => {
|
||||
if (!req.params.channelId) {
|
||||
res.status(400).json({ error: "Missing route parameter: channelId" });
|
||||
return;
|
||||
}
|
||||
const channelId = req.params.channelId as string;
|
||||
const query = messageQuerySchema.parse(req.query);
|
||||
logger.debug({ channelId, query }, "Handling get attachments by channel");
|
||||
const result = await messagesService.getAttachmentsByChannel(
|
||||
@@ -86,5 +70,5 @@ export function handleGetAttachmentsByChannel(
|
||||
query,
|
||||
);
|
||||
res.json(result);
|
||||
})(req, res, next);
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
@@ -14,6 +14,7 @@ import {
|
||||
or,
|
||||
type SQL,
|
||||
} from "drizzle-orm";
|
||||
import { config } from "../../shared/config/index.js";
|
||||
import { getDatabase } from "../../shared/database/index.js";
|
||||
import { mapMessageRow } from "../../shared/utils/messageMapper.js";
|
||||
import type {
|
||||
@@ -23,12 +24,12 @@ import type {
|
||||
} from "./messages.schema.js";
|
||||
|
||||
/**
|
||||
* Thread IDs to exclude from all message queries.
|
||||
* Thread/channel IDs to exclude from all message queries.
|
||||
* Messages in these threads (e.g. bot/selfbot spam) are skipped
|
||||
* both at capture time (discord-gateway) and when serving data
|
||||
* (backend API).
|
||||
* (backend API). Configured via EXCLUDED_THREAD_IDS and EXCLUDED_CHANNEL_IDS.
|
||||
*/
|
||||
const EXCLUDED_THREAD_IDS = ["1522077685508083893"];
|
||||
const EXCLUDED_THREAD_IDS = config.EXCLUDED_THREAD_IDS;
|
||||
|
||||
const logger = createChildLogger("messages.repository");
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { createChildLogger } from "@bete/shared/logger";
|
||||
import type { Request, Response, Router } from "express";
|
||||
import express from "express";
|
||||
import { asyncHandler } from "../../shared/middlewares/index.js";
|
||||
import { asyncHandler, validateBody } from "../../shared/middlewares/index.js";
|
||||
import {
|
||||
handleGetAttachmentsByChannel,
|
||||
handleGetImageMessages,
|
||||
@@ -9,6 +9,7 @@ import {
|
||||
handleGetMessagesByChannel,
|
||||
handleListMessages,
|
||||
} from "./messages.controller.js";
|
||||
import { reanalyzeBatchSchema } from "./messages.schema.js";
|
||||
import { messagesService } from "./messages.service.js";
|
||||
|
||||
const logger = createChildLogger("messages.routes");
|
||||
@@ -55,8 +56,9 @@ export function createMessagesRouter(): Router {
|
||||
// is not captured as an :id param.
|
||||
router.post(
|
||||
"/messages/reanalyze-batch",
|
||||
validateBody(reanalyzeBatchSchema),
|
||||
asyncHandler(async (req: Request, res: Response) => {
|
||||
const { guildId, channelId, messageIds } = (req.body ?? {}) as {
|
||||
const { guildId, channelId, messageIds } = req.body as {
|
||||
guildId?: string;
|
||||
channelId?: string;
|
||||
messageIds?: string[];
|
||||
|
||||
@@ -36,6 +36,13 @@ export const messageUpdateSchema = z.object({
|
||||
aiConfidence: z.number().optional(),
|
||||
});
|
||||
|
||||
export const reanalyzeBatchSchema = z.object({
|
||||
guildId: z.string().optional(),
|
||||
channelId: z.string().optional(),
|
||||
messageIds: z.array(z.string()).optional(),
|
||||
});
|
||||
|
||||
export type MessageQuery = z.infer<typeof messageQuerySchema>;
|
||||
export type MessageCreate = z.infer<typeof messageCreateSchema>;
|
||||
export type MessageUpdate = z.infer<typeof messageUpdateSchema>;
|
||||
export type ReanalyzeBatchInput = z.infer<typeof reanalyzeBatchSchema>;
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
export { createRecordingsRouter } from "./recordings.routes.js";
|
||||
@@ -1,5 +1,6 @@
|
||||
import { pgVoiceRecordingsTable } from "@bete/shared";
|
||||
import { createChildLogger } from "@bete/shared/logger";
|
||||
import { sql } from "drizzle-orm";
|
||||
import { and, desc, eq, lt, type SQL } from "drizzle-orm";
|
||||
import { getDatabase } from "../../shared/database/index.js";
|
||||
|
||||
const logger = createChildLogger("recordings.service");
|
||||
@@ -36,37 +37,47 @@ export class RecordingsService {
|
||||
logger.info({ limit }, "getRecent called");
|
||||
const db = getDatabase();
|
||||
|
||||
const conditions: ReturnType<typeof sql>[] = [];
|
||||
const conditions: SQL[] = [];
|
||||
|
||||
if (filters?.cursor) {
|
||||
conditions.push(sql`created_at < ${filters.cursor}::numeric`);
|
||||
conditions.push(
|
||||
lt(pgVoiceRecordingsTable.created_at, Number(filters.cursor)),
|
||||
);
|
||||
}
|
||||
if (filters?.channelId) {
|
||||
conditions.push(sql`channel_id = ${filters.channelId}`);
|
||||
conditions.push(eq(pgVoiceRecordingsTable.channel_id, filters.channelId));
|
||||
}
|
||||
if (filters?.userId) {
|
||||
conditions.push(sql`user_id = ${filters.userId}`);
|
||||
conditions.push(eq(pgVoiceRecordingsTable.user_id, filters.userId));
|
||||
}
|
||||
|
||||
const whereClause =
|
||||
conditions.length > 0
|
||||
? sql`WHERE ${sql.join(conditions, sql` AND `)}`
|
||||
: sql``;
|
||||
const where = conditions.length > 0 ? and(...conditions) : undefined;
|
||||
|
||||
const { rows } = await db.execute(sql`
|
||||
SELECT
|
||||
id, user_id, username, avatar_url, guild_id, channel_id,
|
||||
channel_name, filename, size_bytes, download_url,
|
||||
upload_status, upload_error, created_at, uploaded_at,
|
||||
COALESCE(size_bytes, 0) AS duration_bytes
|
||||
FROM voice_recordings
|
||||
${whereClause}
|
||||
ORDER BY created_at DESC
|
||||
LIMIT ${limit + 1}
|
||||
`);
|
||||
const allRows = await db
|
||||
.select({
|
||||
id: pgVoiceRecordingsTable.id,
|
||||
user_id: pgVoiceRecordingsTable.user_id,
|
||||
username: pgVoiceRecordingsTable.username,
|
||||
avatar_url: pgVoiceRecordingsTable.avatar_url,
|
||||
guild_id: pgVoiceRecordingsTable.guild_id,
|
||||
channel_id: pgVoiceRecordingsTable.channel_id,
|
||||
channel_name: pgVoiceRecordingsTable.channel_name,
|
||||
filename: pgVoiceRecordingsTable.filename,
|
||||
size_bytes: pgVoiceRecordingsTable.size_bytes,
|
||||
download_url: pgVoiceRecordingsTable.download_url,
|
||||
upload_status: pgVoiceRecordingsTable.upload_status,
|
||||
upload_error: pgVoiceRecordingsTable.upload_error,
|
||||
created_at: pgVoiceRecordingsTable.created_at,
|
||||
uploaded_at: pgVoiceRecordingsTable.uploaded_at,
|
||||
duration_bytes: pgVoiceRecordingsTable.size_bytes,
|
||||
})
|
||||
.from(pgVoiceRecordingsTable)
|
||||
.where(where)
|
||||
.orderBy(desc(pgVoiceRecordingsTable.created_at))
|
||||
.limit(limit + 1);
|
||||
|
||||
const items = rows.slice(0, limit) as unknown as RecordingRow[];
|
||||
const hasMore = rows.length > limit;
|
||||
const items = allRows.slice(0, limit) as unknown as RecordingRow[];
|
||||
const hasMore = allRows.length > limit;
|
||||
const nextCursor = hasMore
|
||||
? String(items[items.length - 1]?.created_at)
|
||||
: null;
|
||||
@@ -76,7 +87,9 @@ export class RecordingsService {
|
||||
|
||||
async deleteById(id: string): Promise<void> {
|
||||
const db = getDatabase();
|
||||
await db.execute(sql`DELETE FROM voice_recordings WHERE id = ${id}`);
|
||||
await db
|
||||
.delete(pgVoiceRecordingsTable)
|
||||
.where(eq(pgVoiceRecordingsTable.id, id));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
export { createUiStateRouter } from "./ui-state.routes.js";
|
||||
@@ -1,49 +0,0 @@
|
||||
import { createChildLogger } from "@bete/shared/logger";
|
||||
import type { Request, Response, Router } from "express";
|
||||
import express from "express";
|
||||
import { asyncHandler } from "../../shared/middlewares/index.js";
|
||||
import {
|
||||
getGuilds,
|
||||
getTextChannels,
|
||||
getVoiceChannels,
|
||||
} from "./voice.service.js";
|
||||
|
||||
const logger = createChildLogger("guilds.routes");
|
||||
|
||||
export function createGuildsRouter(): Router {
|
||||
const router = express.Router();
|
||||
|
||||
// GET /api/guilds
|
||||
router.get(
|
||||
"/",
|
||||
asyncHandler(async (_req: Request, res: Response) => {
|
||||
logger.debug("Fetching guilds");
|
||||
const guilds = await getGuilds();
|
||||
res.json(guilds);
|
||||
}),
|
||||
);
|
||||
|
||||
// GET /api/guilds/:guildId/channels
|
||||
router.get(
|
||||
"/:guildId/channels",
|
||||
asyncHandler(async (req: Request, res: Response) => {
|
||||
const guildId = req.params.guildId as string;
|
||||
logger.debug({ guildId }, "Fetching text channels");
|
||||
const channels = await getTextChannels(guildId);
|
||||
res.json(channels);
|
||||
}),
|
||||
);
|
||||
|
||||
// GET /api/guilds/:guildId/voice-channels
|
||||
router.get(
|
||||
"/:guildId/voice-channels",
|
||||
asyncHandler(async (req: Request, res: Response) => {
|
||||
const guildId = req.params.guildId as string;
|
||||
logger.debug({ guildId }, "Fetching voice channels");
|
||||
const channels = await getVoiceChannels(guildId);
|
||||
res.json(channels);
|
||||
}),
|
||||
);
|
||||
|
||||
return router;
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
export { createVoiceRouter } from "./voice.routes.js";
|
||||
@@ -2,6 +2,7 @@ import { createChildLogger } from "@bete/shared/logger";
|
||||
import type { Request, Response } from "express";
|
||||
import { asyncHandler } from "../../shared/middlewares/index.js";
|
||||
import { publishCommandNoReply } from "../../shared/redis/index.js";
|
||||
import type { ConnectVoiceInput, VoiceCommandInput } from "./voice.schema.js";
|
||||
import {
|
||||
connectVoice,
|
||||
disconnectVoice,
|
||||
@@ -17,22 +18,9 @@ export const handleGetVoiceStatus = asyncHandler(
|
||||
},
|
||||
);
|
||||
|
||||
/** Safely extract a string value that may be a single string or string array. */
|
||||
function asString(val: unknown): string {
|
||||
if (Array.isArray(val)) return String(val[0] ?? "");
|
||||
return String(val ?? "");
|
||||
}
|
||||
|
||||
export const handleConnectVoice = asyncHandler(
|
||||
async (req: Request, res: Response) => {
|
||||
const guildId = asString(req.body.guildId);
|
||||
const channelId = asString(req.body.channelId);
|
||||
if (!guildId || !channelId) {
|
||||
return res.status(400).json({
|
||||
error: "VALIDATION_ERROR",
|
||||
message: "guildId and channelId are required",
|
||||
});
|
||||
}
|
||||
const { guildId, channelId } = req.body as ConnectVoiceInput;
|
||||
logger.debug({ guildId, channelId }, "Connecting to voice channel");
|
||||
const status = await connectVoice(guildId, channelId);
|
||||
res.json(status);
|
||||
@@ -49,15 +37,7 @@ export const handleDisconnectVoice = asyncHandler(
|
||||
|
||||
export const handleVoiceCommand = asyncHandler(
|
||||
async (req: Request, res: Response) => {
|
||||
const command = asString(req.body.command);
|
||||
|
||||
if (!command) {
|
||||
return res.status(400).json({
|
||||
error: "VALIDATION_ERROR",
|
||||
message: "command is required",
|
||||
});
|
||||
}
|
||||
|
||||
const { command } = req.body as VoiceCommandInput;
|
||||
logger.debug({ command }, "Publishing voice command");
|
||||
await publishCommandNoReply(command);
|
||||
res.json({ success: true, command });
|
||||
|
||||
@@ -1,26 +1,80 @@
|
||||
import type { Router } from "express";
|
||||
import { createChildLogger } from "@bete/shared/logger";
|
||||
import type { Request, Response, Router } from "express";
|
||||
import express from "express";
|
||||
import { asyncHandler, validateBody } from "../../shared/middlewares/index.js";
|
||||
import {
|
||||
handleConnectVoice,
|
||||
handleDisconnectVoice,
|
||||
handleGetVoiceStatus,
|
||||
handleVoiceCommand,
|
||||
} from "./voice.controller.js";
|
||||
import { connectVoiceSchema, voiceCommandSchema } from "./voice.schema.js";
|
||||
import {
|
||||
getGuilds,
|
||||
getTextChannels,
|
||||
getVoiceChannels,
|
||||
} from "./voice.service.js";
|
||||
|
||||
const logger = createChildLogger("voice.routes");
|
||||
|
||||
export function createVoiceRouter(): Router {
|
||||
const router = express.Router();
|
||||
|
||||
// ── Guilds ──────────────────────────────────────────────────────────────
|
||||
|
||||
// GET /api/guilds
|
||||
router.get(
|
||||
"/guilds",
|
||||
asyncHandler(async (_req: Request, res: Response) => {
|
||||
logger.debug("Fetching guilds");
|
||||
const guilds = await getGuilds();
|
||||
res.json(guilds);
|
||||
}),
|
||||
);
|
||||
|
||||
// GET /api/guilds/:guildId/channels
|
||||
router.get(
|
||||
"/guilds/:guildId/channels",
|
||||
asyncHandler(async (req: Request, res: Response) => {
|
||||
const guildId = req.params.guildId as string;
|
||||
logger.debug({ guildId }, "Fetching text channels");
|
||||
const channels = await getTextChannels(guildId);
|
||||
res.json(channels);
|
||||
}),
|
||||
);
|
||||
|
||||
// GET /api/guilds/:guildId/voice-channels
|
||||
router.get(
|
||||
"/guilds/:guildId/voice-channels",
|
||||
asyncHandler(async (req: Request, res: Response) => {
|
||||
const guildId = req.params.guildId as string;
|
||||
logger.debug({ guildId }, "Fetching voice channels");
|
||||
const channels = await getVoiceChannels(guildId);
|
||||
res.json(channels);
|
||||
}),
|
||||
);
|
||||
|
||||
// ── Voice connection ────────────────────────────────────────────────────
|
||||
|
||||
// GET /api/voice/status
|
||||
router.get("/voice/status", handleGetVoiceStatus);
|
||||
|
||||
// POST /api/voice/connect
|
||||
router.post("/voice/connect", handleConnectVoice);
|
||||
router.post(
|
||||
"/voice/connect",
|
||||
validateBody(connectVoiceSchema),
|
||||
handleConnectVoice,
|
||||
);
|
||||
|
||||
// POST /api/voice/disconnect
|
||||
router.post("/voice/disconnect", handleDisconnectVoice);
|
||||
|
||||
// POST /api/voice/command — send arbitrary voice command (transmit start/stop)
|
||||
router.post("/voice/command", handleVoiceCommand);
|
||||
router.post(
|
||||
"/voice/command",
|
||||
validateBody(voiceCommandSchema),
|
||||
handleVoiceCommand,
|
||||
);
|
||||
|
||||
return router;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
import { z } from "zod";
|
||||
|
||||
export const connectVoiceSchema = z.object({
|
||||
guildId: z.string().min(1, "guildId is required"),
|
||||
channelId: z.string().min(1, "channelId is required"),
|
||||
});
|
||||
|
||||
export const voiceCommandSchema = z.object({
|
||||
command: z.string().min(1, "command is required"),
|
||||
});
|
||||
|
||||
export type ConnectVoiceInput = z.infer<typeof connectVoiceSchema>;
|
||||
export type VoiceCommandInput = z.infer<typeof voiceCommandSchema>;
|
||||
@@ -5,13 +5,15 @@ import {
|
||||
COMMAND_VOICE_CONNECT,
|
||||
COMMAND_VOICE_DISCONNECT,
|
||||
type CommandReply,
|
||||
pgMessagesTable,
|
||||
VOICE_STATUS_KEY,
|
||||
} from "@bete/shared";
|
||||
import { eq } from "drizzle-orm";
|
||||
import {
|
||||
createChildLogger,
|
||||
tryCommandThenFallback,
|
||||
} from "../../shared/commandHelper.js";
|
||||
import { getPool } from "../../shared/database/index.js";
|
||||
import { getDatabase } from "../../shared/database/index.js";
|
||||
import { publishCommand, readRedisStatus } from "../../shared/redis/index.js";
|
||||
|
||||
const logger = createChildLogger("voice.service");
|
||||
@@ -78,11 +80,12 @@ export async function getGuilds(): Promise<Guild[]> {
|
||||
return withFallback(
|
||||
() => publishCommand<Guild[]>(COMMAND_GUILDS_LIST, {}),
|
||||
async () => {
|
||||
const pool = getPool();
|
||||
const { rows } = await pool.query(
|
||||
`SELECT DISTINCT guild_id FROM messages ORDER BY guild_id`,
|
||||
);
|
||||
return rows.map((row: Record<string, unknown>) => ({
|
||||
const db = getDatabase();
|
||||
const rows = await db
|
||||
.selectDistinct({ guild_id: pgMessagesTable.guild_id })
|
||||
.from(pgMessagesTable)
|
||||
.orderBy(pgMessagesTable.guild_id);
|
||||
return rows.map((row) => ({
|
||||
id: String(row.guild_id ?? ""),
|
||||
name: `Guild ${String(row.guild_id).slice(0, 8)}`,
|
||||
icon: null,
|
||||
@@ -101,12 +104,13 @@ export async function getTextChannels(guildId: string): Promise<Channel[]> {
|
||||
return withFallback(
|
||||
() => publishCommand<Channel[]>(COMMAND_GUILDS_TEXT_CHANNELS, { guildId }),
|
||||
async () => {
|
||||
const pool = getPool();
|
||||
const { rows } = await pool.query(
|
||||
`SELECT DISTINCT channel_id FROM messages WHERE guild_id = $1 ORDER BY channel_id`,
|
||||
[guildId],
|
||||
);
|
||||
return rows.map((row: Record<string, unknown>) => ({
|
||||
const db = getDatabase();
|
||||
const rows = await db
|
||||
.selectDistinct({ channel_id: pgMessagesTable.channel_id })
|
||||
.from(pgMessagesTable)
|
||||
.where(eq(pgMessagesTable.guild_id, guildId))
|
||||
.orderBy(pgMessagesTable.channel_id);
|
||||
return rows.map((row) => ({
|
||||
id: String(row.channel_id ?? ""),
|
||||
name: `Channel ${String(row.channel_id).slice(0, 8)}`,
|
||||
type: "text" as const,
|
||||
|
||||
@@ -2,4 +2,3 @@ import "dotenv/config";
|
||||
import { config as sharedConfig } from "@bete/shared/config";
|
||||
|
||||
export const config = sharedConfig;
|
||||
export type Config = typeof config;
|
||||
|
||||
@@ -1,67 +1,39 @@
|
||||
import {
|
||||
closeDatabase as sharedCloseDb,
|
||||
getDatabase as sharedGetDb,
|
||||
getPool as sharedGetPool,
|
||||
initializeDatabase as sharedInit,
|
||||
} from "@bete/shared/database/init";
|
||||
import { createChildLogger } from "@bete/shared/logger";
|
||||
import { drizzle } from "drizzle-orm/node-postgres";
|
||||
import { Pool } from "pg";
|
||||
import { config } from "../config/index.js";
|
||||
|
||||
const logger = createChildLogger("database");
|
||||
|
||||
let pool: Pool | null = null;
|
||||
let db: ReturnType<typeof drizzle> | null = null;
|
||||
const dbConfig = {
|
||||
DATABASE_URL: config.DATABASE_URL,
|
||||
POSTGRES_HOST: config.POSTGRES_HOST as string | undefined,
|
||||
POSTGRES_PORT: config.POSTGRES_PORT,
|
||||
POSTGRES_USER: config.POSTGRES_USER as string | undefined,
|
||||
POSTGRES_PASSWORD: config.POSTGRES_PASSWORD as string | undefined,
|
||||
POSTGRES_DB: config.POSTGRES_DB as string | undefined,
|
||||
POSTGRES_POOL_MIN: config.POSTGRES_POOL_MIN,
|
||||
POSTGRES_POOL_MAX: config.POSTGRES_POOL_MAX,
|
||||
};
|
||||
|
||||
export async function initializeDatabase() {
|
||||
if (db) {
|
||||
logger.warn("Database already initialized");
|
||||
return db;
|
||||
}
|
||||
|
||||
const databaseUrl =
|
||||
config.DATABASE_URL ||
|
||||
`postgresql://${config.POSTGRES_USER}${config.POSTGRES_PASSWORD ? `:${config.POSTGRES_PASSWORD}` : ""}@${config.POSTGRES_HOST}:${config.POSTGRES_PORT}/${config.POSTGRES_DB}`;
|
||||
|
||||
pool = new Pool({
|
||||
connectionString: databaseUrl,
|
||||
});
|
||||
|
||||
pool.on("error", (err) => {
|
||||
logger.error({ err }, "Unexpected error on idle client");
|
||||
});
|
||||
|
||||
try {
|
||||
const client = await pool.connect();
|
||||
client.release();
|
||||
logger.info("Database connection successful");
|
||||
} catch (err) {
|
||||
logger.error({ err }, "Failed to connect to database");
|
||||
throw err;
|
||||
}
|
||||
|
||||
db = drizzle(pool);
|
||||
return db;
|
||||
logger.info("Initializing database");
|
||||
return sharedInit(dbConfig);
|
||||
}
|
||||
|
||||
export function getDatabase() {
|
||||
if (!db) {
|
||||
throw new Error(
|
||||
"Database not initialized. Call initializeDatabase() first.",
|
||||
);
|
||||
}
|
||||
return db;
|
||||
return sharedGetDb();
|
||||
}
|
||||
|
||||
export function getPool() {
|
||||
if (!pool) {
|
||||
throw new Error(
|
||||
"Database not initialized. Call initializeDatabase() first.",
|
||||
);
|
||||
}
|
||||
return pool;
|
||||
return sharedGetPool();
|
||||
}
|
||||
|
||||
export async function closeDatabase() {
|
||||
if (pool) {
|
||||
await pool.end();
|
||||
pool = null;
|
||||
db = null;
|
||||
logger.info("Database connection closed");
|
||||
}
|
||||
logger.info("Closing database");
|
||||
return sharedCloseDb();
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { AppError, ValidationError } from "@bete/shared/errors";
|
||||
import { createChildLogger } from "@bete/shared/logger";
|
||||
import type { NextFunction, Request, Response } from "express";
|
||||
import type { ZodSchema } from "zod";
|
||||
|
||||
const logger = createChildLogger("middleware");
|
||||
|
||||
@@ -90,3 +91,49 @@ export function requireParam(
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
/**
|
||||
* Express middleware that validates `req.body` against a Zod schema.
|
||||
* On success, replaces `req.body` with the parsed (and defaulted) value.
|
||||
* On failure, responds with 400 and the Zod validation errors.
|
||||
*/
|
||||
export function validateBody<T>(schema: ZodSchema<T>) {
|
||||
return (req: Request, res: Response, next: NextFunction) => {
|
||||
const result = schema.safeParse(req.body);
|
||||
if (!result.success) {
|
||||
res.status(400).json({
|
||||
error: "VALIDATION_ERROR",
|
||||
message: "Request body validation failed",
|
||||
details: result.error.flatten().fieldErrors,
|
||||
});
|
||||
return;
|
||||
}
|
||||
req.body = result.data;
|
||||
next();
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Express middleware that validates `req.query` against a Zod schema.
|
||||
* On success, replaces `req.query` with the parsed (and defaulted) value.
|
||||
* On failure, responds with 400 and the Zod validation errors.
|
||||
*/
|
||||
export function validateQuery<T>(schema: ZodSchema<T>) {
|
||||
return (req: Request, res: Response, next: NextFunction) => {
|
||||
const result = schema.safeParse(req.query);
|
||||
if (!result.success) {
|
||||
res.status(400).json({
|
||||
error: "VALIDATION_ERROR",
|
||||
message: "Query parameter validation failed",
|
||||
details: result.error.flatten().fieldErrors,
|
||||
});
|
||||
return;
|
||||
}
|
||||
// Note: Express req.query is typed as ParsedQs — we attach parsed data
|
||||
// alongside it via a custom property. For route handlers that read req.query
|
||||
// directly, the middleware won't change the type; handlers should opt in by
|
||||
// reading from the validated result or by using the schema's output type.
|
||||
(req as Request & { validatedQuery: T }).validatedQuery = result.data;
|
||||
next();
|
||||
};
|
||||
}
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
export {
|
||||
broadcastBinary,
|
||||
broadcastEvent,
|
||||
clearBroadcastFunctions,
|
||||
setBroadcastFunctions,
|
||||
} from "./broadcast.js";
|
||||
export { startRedisBridge, stopRedisBridge } from "./redis-bridge.js";
|
||||
export { closeWebSocketServer, createWebSocketServer } from "./server.js";
|
||||
@@ -1,27 +1,4 @@
|
||||
import {
|
||||
DISCORD_ANALYSIS_QUEUE_STATUS,
|
||||
DISCORD_ATTACHMENT_CREATED,
|
||||
DISCORD_ATTACHMENT_UPLOADED,
|
||||
DISCORD_CHANNEL_TOPIC_UPDATED,
|
||||
DISCORD_GUILD_MEMBER_ADDED,
|
||||
DISCORD_GUILD_MEMBER_REMOVED,
|
||||
DISCORD_MESSAGE_ANALYZED,
|
||||
DISCORD_MESSAGE_CREATED,
|
||||
DISCORD_MESSAGE_DELETED,
|
||||
DISCORD_MESSAGE_UPDATED,
|
||||
DISCORD_PRESENCE_UPDATED,
|
||||
DISCORD_REACTION_ADDED,
|
||||
DISCORD_REACTION_REMOVED,
|
||||
DISCORD_THREAD_CREATED,
|
||||
DISCORD_THREAD_DELETED,
|
||||
DISCORD_THREAD_UPDATED,
|
||||
DISCORD_VOICE_ACTIVE_USER,
|
||||
DISCORD_VOICE_ANALYZED,
|
||||
DISCORD_VOICE_PCM,
|
||||
DISCORD_VOICE_STARTED,
|
||||
DISCORD_VOICE_STOPPED,
|
||||
DISCORD_VOICE_UPLOADED,
|
||||
} from "@bete/shared";
|
||||
import { DISCORD_CHANNEL_TO_WS_EVENT, DISCORD_VOICE_PCM } from "@bete/shared";
|
||||
import { createChildLogger } from "@bete/shared/logger";
|
||||
import Redis from "ioredis";
|
||||
import { config } from "../shared/config/index.js";
|
||||
@@ -29,41 +6,8 @@ import { broadcastBinary, broadcastEvent } from "./broadcast.js";
|
||||
|
||||
const logger = createChildLogger("ws.redis-bridge");
|
||||
|
||||
interface ChannelMapping {
|
||||
channel: string;
|
||||
eventType: string;
|
||||
}
|
||||
|
||||
const SUBSCRIPTIONS: ChannelMapping[] = [
|
||||
{ channel: DISCORD_MESSAGE_CREATED, eventType: "message_created" },
|
||||
{ channel: DISCORD_MESSAGE_UPDATED, eventType: "message_updated" },
|
||||
{ channel: DISCORD_MESSAGE_DELETED, eventType: "message_deleted" },
|
||||
{ channel: DISCORD_MESSAGE_ANALYZED, eventType: "message_analyzed" },
|
||||
{ channel: DISCORD_ATTACHMENT_CREATED, eventType: "attachment_created" },
|
||||
{ channel: DISCORD_ATTACHMENT_UPLOADED, eventType: "attachment_uploaded" },
|
||||
{ channel: DISCORD_VOICE_STARTED, eventType: "voice_recording_started" },
|
||||
{ channel: DISCORD_VOICE_STOPPED, eventType: "voice_recording_stopped" },
|
||||
{ channel: DISCORD_VOICE_UPLOADED, eventType: "voice_recording_uploaded" },
|
||||
{
|
||||
channel: DISCORD_ANALYSIS_QUEUE_STATUS,
|
||||
eventType: "analysis_queue_status",
|
||||
},
|
||||
{ channel: DISCORD_VOICE_ACTIVE_USER, eventType: "voice_active_user" },
|
||||
{ channel: DISCORD_VOICE_PCM, eventType: "voice_pcm_data" },
|
||||
{ channel: DISCORD_VOICE_ANALYZED, eventType: "voice_analyzed" },
|
||||
{ channel: DISCORD_REACTION_ADDED, eventType: "reaction_added" },
|
||||
{ channel: DISCORD_REACTION_REMOVED, eventType: "reaction_removed" },
|
||||
{ channel: DISCORD_THREAD_CREATED, eventType: "thread_created" },
|
||||
{ channel: DISCORD_THREAD_DELETED, eventType: "thread_deleted" },
|
||||
{ channel: DISCORD_THREAD_UPDATED, eventType: "thread_updated" },
|
||||
{
|
||||
channel: DISCORD_CHANNEL_TOPIC_UPDATED,
|
||||
eventType: "channel_topic_updated",
|
||||
},
|
||||
{ channel: DISCORD_PRESENCE_UPDATED, eventType: "presence_updated" },
|
||||
{ channel: DISCORD_GUILD_MEMBER_ADDED, eventType: "guild_member_added" },
|
||||
{ channel: DISCORD_GUILD_MEMBER_REMOVED, eventType: "guild_member_removed" },
|
||||
];
|
||||
/** Channels we subscribe to = all keys in DISCORD_CHANNEL_TO_WS_EVENT */
|
||||
const SUBSCRIPTION_CHANNELS = Object.keys(DISCORD_CHANNEL_TO_WS_EVENT);
|
||||
|
||||
let subscriber: Redis | null = null;
|
||||
|
||||
@@ -72,8 +16,8 @@ function createSubscriber(): Redis {
|
||||
}
|
||||
|
||||
function handleSubscriptionMessage(channel: string, message: string): void {
|
||||
const mapping = SUBSCRIPTIONS.find((m) => m.channel === channel);
|
||||
if (!mapping) {
|
||||
const eventType = DISCORD_CHANNEL_TO_WS_EVENT[channel];
|
||||
if (!eventType) {
|
||||
logger.warn({ channel }, "Received message for unmapped Redis channel");
|
||||
return;
|
||||
}
|
||||
@@ -97,7 +41,7 @@ function handleSubscriptionMessage(channel: string, message: string): void {
|
||||
const data = envelope.data !== undefined ? envelope.data : envelope;
|
||||
|
||||
// Voice PCM: decode base64 → binary broadcast instead of JSON
|
||||
if (mapping.eventType === "voice_pcm_data") {
|
||||
if (channel === DISCORD_VOICE_PCM) {
|
||||
const pcmPayload = data as { userId?: string; pcm?: string };
|
||||
if (pcmPayload?.pcm && pcmPayload?.userId) {
|
||||
try {
|
||||
@@ -115,11 +59,8 @@ function handleSubscriptionMessage(channel: string, message: string): void {
|
||||
}
|
||||
}
|
||||
|
||||
logger.debug(
|
||||
{ channel, eventType: mapping.eventType },
|
||||
"Broadcasting Redis event",
|
||||
);
|
||||
broadcastEvent(mapping.eventType, data);
|
||||
logger.debug({ channel, eventType }, "Broadcasting Redis event");
|
||||
broadcastEvent(eventType, data);
|
||||
}
|
||||
|
||||
/** Simple 32-bit FNV-1a hash for userId → 4-byte identifier */
|
||||
@@ -162,7 +103,7 @@ export async function startRedisBridge(): Promise<void> {
|
||||
await subscriber.ping();
|
||||
logger.info("Redis ping OK");
|
||||
|
||||
const channels = SUBSCRIPTIONS.map((m) => m.channel);
|
||||
const channels = SUBSCRIPTION_CHANNELS;
|
||||
await subscriber.subscribe(...channels);
|
||||
logger.info({ channels }, "Subscribed to Redis channels");
|
||||
|
||||
@@ -184,8 +125,9 @@ export async function stopRedisBridge(): Promise<void> {
|
||||
logger.info("Redis bridge stopped");
|
||||
} catch (err) {
|
||||
logger.error({ err }, "Error stopping Redis bridge");
|
||||
} finally {
|
||||
// Force-close on error
|
||||
subscriber.disconnect();
|
||||
} finally {
|
||||
subscriber = null;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,9 +13,21 @@ interface BroadcastEvent {
|
||||
timestamp: string;
|
||||
}
|
||||
|
||||
interface JsonMessage {
|
||||
type: string;
|
||||
buffer?: string;
|
||||
command?: string;
|
||||
payload?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
// Track the active WebSocket server for lifecycle management
|
||||
let _wss: WebSocketServer | null = null;
|
||||
|
||||
type MessageHandler = (
|
||||
ws: WebSocket,
|
||||
message: JsonMessage,
|
||||
) => Promise<void> | void;
|
||||
|
||||
async function sendInitialStates(ws: WebSocket): Promise<void> {
|
||||
// Send initial user state
|
||||
ws.send(
|
||||
@@ -71,6 +83,35 @@ export function createWebSocketServer(server: Server): WebSocketServer {
|
||||
const wss = new WebSocketServer({ server, path: "/ws" });
|
||||
_wss = wss;
|
||||
|
||||
// Map-based dispatcher for JSON WebSocket message types
|
||||
const jsonHandlers = new Map<string, MessageHandler>();
|
||||
|
||||
jsonHandlers.set("voice_transmit", async (_ws, message) => {
|
||||
if (!message.buffer) return;
|
||||
const { getCommandPublisher } = await import("../shared/redis/index.js");
|
||||
const publisher = getCommandPublisher();
|
||||
await publisher.publish(
|
||||
BACKEND_VOICE_TRANSMIT,
|
||||
JSON.stringify({ type: "pcm", buffer: message.buffer }),
|
||||
);
|
||||
});
|
||||
|
||||
jsonHandlers.set("voice_command", async (_ws, message) => {
|
||||
if (!message.command) return;
|
||||
const { getCommandPublisher } = await import("../shared/redis/index.js");
|
||||
const publisher = getCommandPublisher();
|
||||
const commandId = `cmd-${Date.now()}-${Math.random().toString(36).slice(2, 9)}`;
|
||||
await publisher.publish(
|
||||
BACKEND_COMMAND,
|
||||
JSON.stringify({
|
||||
id: commandId,
|
||||
type: message.command,
|
||||
payload: message.payload ?? {},
|
||||
replyChannel: `reply:${commandId}`,
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
wss.on("connection", (ws: WebSocket, req) => {
|
||||
// Parse auth token from query string
|
||||
const rawUrl = req.url ?? "/";
|
||||
@@ -103,7 +144,7 @@ export function createWebSocketServer(server: Server): WebSocketServer {
|
||||
ws.on("message", (data: Buffer) => {
|
||||
// Gateway PCM forward — broadcast raw binary to frontend clients only
|
||||
if (isGateway && Buffer.isBuffer(data)) {
|
||||
broadcastBinaryToFrontend(data);
|
||||
broadcastBinary(data);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -146,52 +187,11 @@ export function createWebSocketServer(server: Server): WebSocketServer {
|
||||
) {
|
||||
try {
|
||||
const message = JSON.parse(data.toString());
|
||||
|
||||
if (message.type === "voice_transmit" && message.buffer) {
|
||||
// Legacy: Forward PCM data to Redis for discord-gateway
|
||||
import("../shared/redis/index.js").then(
|
||||
({ getCommandPublisher }) => {
|
||||
const publisher = getCommandPublisher();
|
||||
publisher
|
||||
.publish(
|
||||
BACKEND_VOICE_TRANSMIT,
|
||||
JSON.stringify({
|
||||
type: "pcm",
|
||||
buffer: message.buffer,
|
||||
}),
|
||||
)
|
||||
.catch((err: Error) => {
|
||||
logger.error(
|
||||
{ err },
|
||||
"Failed to publish voice transmit to Redis",
|
||||
);
|
||||
});
|
||||
},
|
||||
);
|
||||
} else if (message.type === "voice_command" && message.command) {
|
||||
// Forward voice commands to discord-gateway with payload
|
||||
import("../shared/redis/index.js").then(
|
||||
({ getCommandPublisher }) => {
|
||||
const publisher = getCommandPublisher();
|
||||
const commandId = `cmd-${Date.now()}-${Math.random().toString(36).slice(2, 9)}`;
|
||||
publisher
|
||||
.publish(
|
||||
BACKEND_COMMAND,
|
||||
JSON.stringify({
|
||||
id: commandId,
|
||||
type: message.command,
|
||||
payload: message.payload ?? {},
|
||||
replyChannel: `reply:${commandId}`,
|
||||
}),
|
||||
)
|
||||
.catch((err: Error) => {
|
||||
logger.error(
|
||||
{ err },
|
||||
"Failed to publish voice command to Redis",
|
||||
);
|
||||
});
|
||||
},
|
||||
);
|
||||
const handler = jsonHandlers.get(message.type);
|
||||
if (handler) {
|
||||
Promise.resolve(handler(ws, message)).catch((err: Error) => {
|
||||
logger.error({ err }, "JSON message handler failed");
|
||||
});
|
||||
}
|
||||
} catch (err) {
|
||||
logger.debug({ err }, "Failed to parse WebSocket message as JSON");
|
||||
@@ -234,19 +234,6 @@ export function createWebSocketServer(server: Server): WebSocketServer {
|
||||
// Don't let the interval keep the process alive after wss closes
|
||||
heartbeatInterval.unref();
|
||||
|
||||
// Forward gateway binary to frontend clients (no loopback to gateway)
|
||||
function broadcastBinaryToFrontend(data: Buffer) {
|
||||
for (const client of frontendClients) {
|
||||
if (client.readyState === WebSocket.OPEN) {
|
||||
try {
|
||||
client.send(data);
|
||||
} catch (err) {
|
||||
logger.error({ err }, "Failed to send binary to frontend client");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// JSON event broadcast — frontend clients only
|
||||
function broadcast(event: Omit<BroadcastEvent, "timestamp">) {
|
||||
const payload = JSON.stringify({
|
||||
|
||||
@@ -19,7 +19,6 @@ import {
|
||||
registerMessageCapture,
|
||||
setEventBroadcaster as setMessageCaptureEventBroadcaster,
|
||||
} from "../modules/message-capture/messageCapture.js";
|
||||
import { getExpiredMessages } from "../modules/message-capture/messageStore.js";
|
||||
import { registerReactionCapture } from "../modules/reaction-tracking/index.js";
|
||||
import { registerThreadCapture } from "../modules/thread-tracking/index.js";
|
||||
import { registerPresenceCapture } from "../modules/user-presence/index.js";
|
||||
@@ -50,6 +49,55 @@ const logger = createChildLogger("discord-gateway");
|
||||
|
||||
// ─── Retention Cleanup ─────────────────────────────────────────────────────
|
||||
|
||||
async function deleteExpiredRecords(
|
||||
table: any,
|
||||
timestampField: any,
|
||||
days: number | undefined,
|
||||
dryRun: boolean,
|
||||
label: string,
|
||||
): Promise<void> {
|
||||
if (!days || days <= 0) {
|
||||
logger.debug({ label }, `Retention disabled for ${label}`);
|
||||
return;
|
||||
}
|
||||
|
||||
const cutoff = Date.now() - days * 24 * 60 * 60 * 1000;
|
||||
const db = getDatabase() as unknown as NodePgDatabase<typeof schema>;
|
||||
|
||||
const expired = await db
|
||||
.select({ id: table.id })
|
||||
.from(table)
|
||||
.where(lt(timestampField, cutoff))
|
||||
.limit(1000);
|
||||
|
||||
if (expired.length === 0) {
|
||||
logger.debug({ label }, `No expired ${label} found`);
|
||||
return;
|
||||
}
|
||||
|
||||
logger.info({ count: expired.length, label }, `Found expired ${label}`);
|
||||
|
||||
if (dryRun) {
|
||||
logger.info(
|
||||
{ count: expired.length, label },
|
||||
`[DRY RUN] Would delete ${expired.length} ${label}`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
await db.delete(table).where(
|
||||
inArray(
|
||||
table.id,
|
||||
expired.map((r) => r.id),
|
||||
),
|
||||
);
|
||||
logger.info({ count: expired.length, label }, `Deleted expired ${label}`);
|
||||
} catch (err) {
|
||||
logger.error({ err, label }, `Failed to delete expired ${label}`);
|
||||
}
|
||||
}
|
||||
|
||||
function startRetentionCleanup(): void {
|
||||
const intervalMs = config.RETENTION_CLEANUP_INTERVAL_MS;
|
||||
const dryRun = config.RETENTION_DRY_RUN;
|
||||
@@ -66,113 +114,27 @@ function startRetentionCleanup(): void {
|
||||
);
|
||||
|
||||
async function runCleanupTick(): Promise<void> {
|
||||
const db = getDatabase() as unknown as NodePgDatabase<typeof schema>;
|
||||
|
||||
// ── Expired messages ────────────────────────────────────────────────
|
||||
if (config.RETENTION_MESSAGES_DAYS > 0) {
|
||||
try {
|
||||
const expiredMessages = await getExpiredMessages(
|
||||
config.RETENTION_MESSAGES_DAYS,
|
||||
);
|
||||
|
||||
if (expiredMessages.length > 0) {
|
||||
const ids = expiredMessages.map((m: { id: string }) => m.id);
|
||||
logger.info(
|
||||
{ count: ids.length, dryRun },
|
||||
"Expired messages found for cleanup",
|
||||
);
|
||||
|
||||
if (!dryRun) {
|
||||
await db
|
||||
.delete(messagesTable)
|
||||
.where(inArray(messagesTable.id, ids));
|
||||
logger.info({ count: ids.length }, "Expired messages deleted");
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error(
|
||||
{
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
},
|
||||
"Failed to clean up expired messages",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// ── Expired attachments ─────────────────────────────────────────────
|
||||
if (config.RETENTION_ATTACHMENTS_DAYS > 0) {
|
||||
try {
|
||||
const cutoff =
|
||||
Date.now() - config.RETENTION_ATTACHMENTS_DAYS * 24 * 60 * 60 * 1000;
|
||||
|
||||
const expiredAttachments = await db
|
||||
.select({ id: attachmentsTable.id })
|
||||
.from(attachmentsTable)
|
||||
.where(lt(attachmentsTable.created_at, cutoff))
|
||||
.limit(1000);
|
||||
|
||||
if (expiredAttachments.length > 0) {
|
||||
const ids = expiredAttachments.map((a: { id: string }) => a.id);
|
||||
logger.info(
|
||||
{ count: ids.length, dryRun },
|
||||
"Expired attachments found for cleanup",
|
||||
);
|
||||
|
||||
if (!dryRun) {
|
||||
await db
|
||||
.delete(attachmentsTable)
|
||||
.where(inArray(attachmentsTable.id, ids));
|
||||
logger.info({ count: ids.length }, "Expired attachments deleted");
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error(
|
||||
{
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
},
|
||||
"Failed to clean up expired attachments",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// ── Expired voice recordings ────────────────────────────────────────
|
||||
if (config.RETENTION_VOICE_DAYS > 0) {
|
||||
try {
|
||||
const cutoff =
|
||||
Date.now() - config.RETENTION_VOICE_DAYS * 24 * 60 * 60 * 1000;
|
||||
|
||||
const expiredRecordings = await db
|
||||
.select({ id: voiceRecordingsTable.id })
|
||||
.from(voiceRecordingsTable)
|
||||
.where(lt(voiceRecordingsTable.created_at, cutoff))
|
||||
.limit(1000);
|
||||
|
||||
if (expiredRecordings.length > 0) {
|
||||
const ids = expiredRecordings.map((r: { id: string }) => r.id);
|
||||
logger.info(
|
||||
{ count: ids.length, dryRun },
|
||||
"Expired voice recordings found for cleanup",
|
||||
);
|
||||
|
||||
if (!dryRun) {
|
||||
await db
|
||||
.delete(voiceRecordingsTable)
|
||||
.where(inArray(voiceRecordingsTable.id, ids));
|
||||
logger.info(
|
||||
{ count: ids.length },
|
||||
"Expired voice recordings deleted",
|
||||
);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error(
|
||||
{
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
},
|
||||
"Failed to clean up expired voice recordings",
|
||||
);
|
||||
}
|
||||
}
|
||||
await deleteExpiredRecords(
|
||||
messagesTable,
|
||||
messagesTable.created_at,
|
||||
config.RETENTION_MESSAGES_DAYS,
|
||||
dryRun,
|
||||
"messages",
|
||||
);
|
||||
await deleteExpiredRecords(
|
||||
attachmentsTable,
|
||||
attachmentsTable.created_at,
|
||||
config.RETENTION_ATTACHMENTS_DAYS,
|
||||
dryRun,
|
||||
"attachments",
|
||||
);
|
||||
await deleteExpiredRecords(
|
||||
voiceRecordingsTable,
|
||||
voiceRecordingsTable.created_at,
|
||||
config.RETENTION_VOICE_DAYS,
|
||||
dryRun,
|
||||
"voice recordings",
|
||||
);
|
||||
}
|
||||
|
||||
// Run immediately on start, then schedule
|
||||
@@ -330,12 +292,13 @@ export async function initializeDiscordGateway() {
|
||||
startMetricsServer();
|
||||
|
||||
logger.info("Calling Discord client.login");
|
||||
client
|
||||
.login(token)
|
||||
.then(() => {
|
||||
logger.info("Discord client.login resolved");
|
||||
})
|
||||
.catch((error: unknown) => {
|
||||
logger.error({ error }, "Discord client.login failed");
|
||||
});
|
||||
|
||||
// Fix: use await + try/catch instead of .then().catch()
|
||||
try {
|
||||
await client.login(token);
|
||||
logger.info("Discord client logged in successfully");
|
||||
} catch (err) {
|
||||
logger.fatal({ err }, "Failed to login Discord client");
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,123 @@
|
||||
import { createChildLogger } from "@bete/shared/logger";
|
||||
import { inArray, lt } from "drizzle-orm";
|
||||
import type { NodePgDatabase } from "drizzle-orm/node-postgres";
|
||||
import { config } from "../shared/config/config.js";
|
||||
import { getDatabase } from "../shared/database/drizzle.js";
|
||||
import type * as schema from "../shared/database/schema.js";
|
||||
import {
|
||||
attachmentsTable,
|
||||
messagesTable,
|
||||
voiceRecordingsTable,
|
||||
} from "../shared/database/schema.js";
|
||||
|
||||
const logger = createChildLogger("discord-gateway");
|
||||
|
||||
// ─── Retention Cleanup ─────────────────────────────────────────────────────
|
||||
|
||||
async function deleteExpiredRecords(
|
||||
table: any,
|
||||
timestampField: any,
|
||||
days: number | undefined,
|
||||
dryRun: boolean,
|
||||
label: string,
|
||||
): Promise<void> {
|
||||
if (!days || days <= 0) {
|
||||
logger.debug({ label }, `Retention disabled for ${label}`);
|
||||
return;
|
||||
}
|
||||
|
||||
const cutoff = Date.now() - days * 24 * 60 * 60 * 1000;
|
||||
const db = getDatabase() as unknown as NodePgDatabase<typeof schema>;
|
||||
|
||||
const expired = await db
|
||||
.select({ id: table.id })
|
||||
.from(table)
|
||||
.where(lt(timestampField, cutoff))
|
||||
.limit(1000);
|
||||
|
||||
if (expired.length === 0) {
|
||||
logger.debug({ label }, `No expired ${label} found`);
|
||||
return;
|
||||
}
|
||||
|
||||
logger.info({ count: expired.length, label }, `Found expired ${label}`);
|
||||
|
||||
if (dryRun) {
|
||||
logger.info(
|
||||
{ count: expired.length, label },
|
||||
`[DRY RUN] Would delete ${expired.length} ${label}`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
await db.delete(table).where(
|
||||
inArray(
|
||||
table.id,
|
||||
expired.map((r) => r.id),
|
||||
),
|
||||
);
|
||||
logger.info({ count: expired.length, label }, `Deleted expired ${label}`);
|
||||
} catch (err) {
|
||||
logger.error({ err, label }, `Failed to delete expired ${label}`);
|
||||
}
|
||||
}
|
||||
|
||||
function startRetentionCleanup(): void {
|
||||
const intervalMs = config.RETENTION_CLEANUP_INTERVAL_MS;
|
||||
const dryRun = config.RETENTION_DRY_RUN;
|
||||
|
||||
logger.info(
|
||||
{
|
||||
intervalMs,
|
||||
dryRun,
|
||||
messagesDays: config.RETENTION_MESSAGES_DAYS,
|
||||
attachmentsDays: config.RETENTION_ATTACHMENTS_DAYS,
|
||||
voiceDays: config.RETENTION_VOICE_DAYS,
|
||||
},
|
||||
"Starting retention cleanup scheduler",
|
||||
);
|
||||
|
||||
async function runCleanupTick(): Promise<void> {
|
||||
await deleteExpiredRecords(
|
||||
messagesTable,
|
||||
messagesTable.created_at,
|
||||
config.RETENTION_MESSAGES_DAYS,
|
||||
dryRun,
|
||||
"messages",
|
||||
);
|
||||
await deleteExpiredRecords(
|
||||
attachmentsTable,
|
||||
attachmentsTable.created_at,
|
||||
config.RETENTION_ATTACHMENTS_DAYS,
|
||||
dryRun,
|
||||
"attachments",
|
||||
);
|
||||
await deleteExpiredRecords(
|
||||
voiceRecordingsTable,
|
||||
voiceRecordingsTable.created_at,
|
||||
config.RETENTION_VOICE_DAYS,
|
||||
dryRun,
|
||||
"voice recordings",
|
||||
);
|
||||
}
|
||||
|
||||
// Run immediately on start, then schedule
|
||||
runCleanupTick().catch((error) => {
|
||||
logger.error(
|
||||
{ error: error instanceof Error ? error.message : String(error) },
|
||||
"Initial retention cleanup tick failed",
|
||||
);
|
||||
});
|
||||
|
||||
setInterval(() => {
|
||||
runCleanupTick().catch((error) => {
|
||||
logger.error(
|
||||
{ error: error instanceof Error ? error.message : String(error) },
|
||||
"Retention cleanup tick failed",
|
||||
);
|
||||
});
|
||||
}, intervalMs);
|
||||
}
|
||||
|
||||
export { startRetentionCleanup };
|
||||
@@ -2,20 +2,14 @@ import { createChildLogger } from "@bete/shared/logger";
|
||||
import { config } from "../../shared/config/config.js";
|
||||
import { initializeDatabase } from "../../shared/database/drizzle.js";
|
||||
import { extractMessageMediaEvidence } from "../message-capture/messageMetadata.js";
|
||||
import {
|
||||
getAttachmentsForMessages,
|
||||
getConversationContextBefore,
|
||||
updateMessagesAIAnalysisBulk,
|
||||
} from "../message-capture/messageStore.js";
|
||||
import { messageStore } from "../message-capture/messageStore.js";
|
||||
import type {
|
||||
AnalysisResult,
|
||||
MessageRecord,
|
||||
} from "../message-capture/types.js";
|
||||
import { buildConversationContext } from "./conversationContext.js";
|
||||
import {
|
||||
runModerationAnalysis,
|
||||
runSimpleTextFallback,
|
||||
} from "./llmModerationClient.js";
|
||||
import { runModerationAnalysis } from "./moderationOrchestrator.js";
|
||||
import { runSimpleTextFallback } from "./simpleFallback.js";
|
||||
|
||||
const logger = createChildLogger("aiAnalysisWorker");
|
||||
|
||||
@@ -142,7 +136,7 @@ async function processBatch(job: {
|
||||
const firstMessage = messages[0];
|
||||
if (!firstMessage) return { ok: true, conversationKey, rows: [] };
|
||||
|
||||
const contextBefore = await getConversationContextBefore({
|
||||
const contextBefore = await messageStore.getConversationContextBefore({
|
||||
channelId: firstMessage.channel_id,
|
||||
threadId: firstMessage.thread_id,
|
||||
beforeCreatedAt: firstMessage.created_at,
|
||||
@@ -158,7 +152,8 @@ async function processBatch(job: {
|
||||
const targetIds = messages.map((m) => m.id);
|
||||
const contextIds = contextBefore.map((m) => m.id);
|
||||
const allMessageIds = [...targetIds, ...contextIds];
|
||||
const attachments = await getAttachmentsForMessages(allMessageIds);
|
||||
const attachments =
|
||||
await messageStore.getAttachmentsForMessages(allMessageIds);
|
||||
|
||||
// ── Split: text-only vs media ──────────────────────────────────────
|
||||
// Text-only analysis runs fast (single LLM call, no vision).
|
||||
@@ -223,13 +218,15 @@ async function processBatch(job: {
|
||||
},
|
||||
}));
|
||||
if (updates.length > 0) {
|
||||
return updateMessagesAIAnalysisBulk(updates).then((rows) => {
|
||||
allRows.push(...rows);
|
||||
logger.info(
|
||||
{ count: updates.length, conversationKey },
|
||||
"Text-only batch saved — media analysis still in progress",
|
||||
);
|
||||
});
|
||||
return messageStore
|
||||
.updateMessagesAIAnalysisBulk(updates)
|
||||
.then((rows) => {
|
||||
allRows.push(...rows);
|
||||
logger.info(
|
||||
{ count: updates.length, conversationKey },
|
||||
"Text-only batch saved — media analysis still in progress",
|
||||
);
|
||||
});
|
||||
}
|
||||
})
|
||||
: Promise.resolve();
|
||||
@@ -257,9 +254,11 @@ async function processBatch(job: {
|
||||
},
|
||||
}));
|
||||
if (updates.length > 0) {
|
||||
return updateMessagesAIAnalysisBulk(updates).then((rows) => {
|
||||
allRows.push(...rows);
|
||||
});
|
||||
return messageStore
|
||||
.updateMessagesAIAnalysisBulk(updates)
|
||||
.then((rows) => {
|
||||
allRows.push(...rows);
|
||||
});
|
||||
}
|
||||
})
|
||||
: Promise.resolve();
|
||||
@@ -291,7 +290,7 @@ async function processIndividual(job: {
|
||||
}): Promise<IndividualOkResponse | IndividualErrorResponse> {
|
||||
const { message, skipNormalAnalysis } = job;
|
||||
|
||||
const contextBefore = await getConversationContextBefore({
|
||||
const contextBefore = await messageStore.getConversationContextBefore({
|
||||
channelId: message.channel_id,
|
||||
threadId: message.thread_id,
|
||||
beforeCreatedAt: message.created_at,
|
||||
@@ -305,7 +304,7 @@ async function processIndividual(job: {
|
||||
});
|
||||
|
||||
const contextIds = contextBefore.map((m) => m.id);
|
||||
const attachments = await getAttachmentsForMessages([
|
||||
const attachments = await messageStore.getAttachmentsForMessages([
|
||||
message.id,
|
||||
...contextIds,
|
||||
]);
|
||||
|
||||
@@ -2,14 +2,7 @@ import { createChildLogger } from "@bete/shared/logger";
|
||||
import type { Client } from "discord.js-selfbot-v13";
|
||||
import { config } from "../../shared/config/config.js";
|
||||
import type { EventBroadcaster } from "../event-broadcaster/index.js";
|
||||
import {
|
||||
getConversationKeysWithIncompleteAnalysis,
|
||||
getIncompleteMessagesByConversation,
|
||||
getMessageById,
|
||||
getPendingConversationKeys,
|
||||
revertStuckProcessingMessages,
|
||||
updateMessageAIAnalysis,
|
||||
} from "../message-capture/messageStore.js";
|
||||
import { messageStore } from "../message-capture/messageStore.js";
|
||||
import type { AnalysisQueueStatus } from "../message-capture/types.js";
|
||||
import {
|
||||
activeRequests,
|
||||
@@ -18,18 +11,14 @@ import {
|
||||
skipAgeRestrictedMessages,
|
||||
} from "./batchProcessor.js";
|
||||
import { scheduleConversationAnalysis } from "./batchScheduler.js";
|
||||
import { getConversationKey } from "./circuitBreaker.js";
|
||||
import {
|
||||
broadcastAnalysisCompleted,
|
||||
conversationConsecutiveErrors,
|
||||
conversationDebounceTimers,
|
||||
conversationErrorCooldown,
|
||||
conversationProcessing,
|
||||
getConversationKey,
|
||||
isConversationProcessingLocked,
|
||||
LAST_ERROR,
|
||||
setModerationClient,
|
||||
setSharedEventBroadcaster,
|
||||
} from "./circuitBreaker.js";
|
||||
} from "./conversationState.js";
|
||||
import {
|
||||
activeIndividualRequests,
|
||||
enqueueIndividualFallbacks,
|
||||
@@ -38,6 +27,12 @@ import {
|
||||
individualInFlightByConversation,
|
||||
individualInFlightLastTouched,
|
||||
} from "./individualFallbackProcessor.js";
|
||||
import {
|
||||
broadcastAnalysisCompleted,
|
||||
LAST_ERROR,
|
||||
setModerationClient,
|
||||
setSharedEventBroadcaster,
|
||||
} from "./moderationState.js";
|
||||
|
||||
const logger = createChildLogger("ai-analyzer");
|
||||
|
||||
@@ -46,7 +41,8 @@ const logger = createChildLogger("ai-analyzer");
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export { pickBatchWithinBudget } from "./batchProcessor.js";
|
||||
export { getConversationKey, onCircuitBreakerAlert } from "./circuitBreaker.js";
|
||||
export { getConversationKey } from "./circuitBreaker.js";
|
||||
export { onCircuitBreakerAlert } from "./conversationState.js";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Public API
|
||||
@@ -59,14 +55,14 @@ export async function queueMessageAnalysis(messageId: string): Promise<void> {
|
||||
if (!config.AI_ANALYSIS_ENABLED) return;
|
||||
|
||||
try {
|
||||
const message = await getMessageById(messageId);
|
||||
const message = await messageStore.getMessageById(messageId);
|
||||
if (!message) {
|
||||
logger.warn({ messageId }, "Message not found for analysis queue");
|
||||
return;
|
||||
}
|
||||
|
||||
if (isAgeRestrictedMessage(message)) {
|
||||
const updated = await updateMessageAIAnalysis(
|
||||
const updated = await messageStore.updateMessageAIAnalysis(
|
||||
message.id,
|
||||
buildAgeRestrictedSkipResult(),
|
||||
);
|
||||
@@ -117,7 +113,7 @@ export function getAnalysisQueueStatus(): AnalysisQueueStatus {
|
||||
/**
|
||||
* Starts the periodic recovery worker.
|
||||
*
|
||||
* FIX #4: Now also recovers messages stuck in `error/analysis_incomplete`
|
||||
* Now also recovers messages stuck in `error/analysis_incomplete`
|
||||
* state (not just `pending`), and skips conversations that already have
|
||||
* individual fallback work in progress to avoid DB last-write-wins races.
|
||||
*/
|
||||
@@ -137,23 +133,20 @@ export function startPendingAIAnalysisWorker(
|
||||
.catch(console.error);
|
||||
|
||||
setInterval(() => {
|
||||
revertStuckProcessingMessages(300000).catch((err: unknown) => {
|
||||
messageStore.revertStuckProcessingMessages(300000).catch((err: unknown) => {
|
||||
logger.error(
|
||||
{ error: String(err) },
|
||||
"Failed to run stuck processing recovery",
|
||||
);
|
||||
});
|
||||
|
||||
// FIX #3 pattern: no async arrow -- chain promises explicitly.
|
||||
Promise.all([
|
||||
getPendingConversationKeys(500),
|
||||
getConversationKeysWithIncompleteAnalysis(200),
|
||||
messageStore.getPendingConversationKeys(500),
|
||||
messageStore.getConversationKeysWithIncompleteAnalysis(200),
|
||||
])
|
||||
.then(([pendingKeys, incompleteKeys]) => {
|
||||
const now = Date.now();
|
||||
|
||||
// FIX #9: Prune stale entries from state maps to prevent unbounded
|
||||
// memory growth from channels/threads that are no longer active.
|
||||
for (const [key, expiry] of conversationErrorCooldown) {
|
||||
if (now >= expiry) conversationErrorCooldown.delete(key);
|
||||
}
|
||||
@@ -163,9 +156,6 @@ export function startPendingAIAnalysisWorker(
|
||||
}
|
||||
}
|
||||
|
||||
// FIX #7: Prune stale in-flight counters for conversations that have
|
||||
// been idle longer than the processing timeout -- prevents permanent
|
||||
// blocking if a decrement was missed due to an uncaught exception.
|
||||
const staleThreshold = config.AI_ANALYSIS_PROCESSING_TIMEOUT_MS * 2;
|
||||
for (const [key, lastTouched] of individualInFlightLastTouched) {
|
||||
if (now - lastTouched >= staleThreshold) {
|
||||
@@ -187,17 +177,13 @@ export function startPendingAIAnalysisWorker(
|
||||
}
|
||||
}
|
||||
|
||||
// FIX #8: Build a set of keys already targeted for individual recovery
|
||||
// so the batch loop below skips them.
|
||||
const incompleteKeySet = new Set(incompleteKeys);
|
||||
|
||||
// --- Batch recovery for pending messages ---
|
||||
for (const key of pendingKeys) {
|
||||
if (conversationDebounceTimers.has(key)) continue;
|
||||
if (isConversationProcessingLocked(key)) continue;
|
||||
// FIX #4: skip if individual fallback already running for this conversation.
|
||||
if (individualInFlightByConversation.has(key)) continue;
|
||||
// FIX #8: skip if this conversation also needs individual recovery.
|
||||
if (incompleteKeySet.has(key)) continue;
|
||||
const cooldownUntil = conversationErrorCooldown.get(key);
|
||||
if (cooldownUntil && now < cooldownUntil) continue;
|
||||
@@ -215,7 +201,8 @@ export function startPendingAIAnalysisWorker(
|
||||
if (isConversationProcessingLocked(key)) continue;
|
||||
|
||||
promises.push(
|
||||
getIncompleteMessagesByConversation(key, 500)
|
||||
messageStore
|
||||
.getIncompleteMessagesByConversation(key, 500)
|
||||
.then(async (msgs) => {
|
||||
const processableMessages =
|
||||
await skipAgeRestrictedMessages(msgs);
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { createChildLogger } from "@bete/shared/logger";
|
||||
import type { Client, PermissionString } from "discord.js-selfbot-v13";
|
||||
import { config } from "../../shared/config/config.js";
|
||||
import { createModerationAction } from "../message-capture/messageStore.js";
|
||||
import { messageStore } from "../message-capture/messageStore.js";
|
||||
import type { MessageRecord } from "../message-capture/types.js";
|
||||
import { isEligibleForAutoDelete } from "./autoDeleteEligibility.js";
|
||||
import { logDeletionToChannel } from "./autoDeleteLogger.js";
|
||||
@@ -65,7 +65,7 @@ async function logAutoDeleteAttempt(
|
||||
result: AutoDeleteResult,
|
||||
): Promise<void> {
|
||||
try {
|
||||
await createModerationAction({
|
||||
await messageStore.createModerationAction({
|
||||
message_id: message.id,
|
||||
user_id: message.user_id,
|
||||
guild_id: message.guild_id,
|
||||
|
||||
@@ -1,20 +1,22 @@
|
||||
import { createChildLogger } from "@bete/shared/logger";
|
||||
import { config } from "../../shared/config/config.js";
|
||||
import { isAgeRestrictedMetadata } from "../message-capture/messageMetadata.js";
|
||||
import { updateMessagesAIAnalysisBulk } from "../message-capture/messageStore.js";
|
||||
import { messageStore } from "../message-capture/messageStore.js";
|
||||
import type { MessageRecord } from "../message-capture/types.js";
|
||||
import { workerPool } from "./circuitBreaker.js";
|
||||
import { estimateTokens } from "./conversationContext.js";
|
||||
import {
|
||||
broadcastAnalysisCompleted,
|
||||
conversationErrorCooldown,
|
||||
conversationProcessing,
|
||||
LAST_ERROR,
|
||||
recordConversationBatchFailure,
|
||||
resetConversationBatchFailures,
|
||||
scheduleAutoDelete,
|
||||
workerPool,
|
||||
} from "./circuitBreaker.js";
|
||||
import { estimateTokens } from "./conversationContext.js";
|
||||
} from "./conversationState.js";
|
||||
import { enqueueIndividualFallbacks } from "./individualFallbackProcessor.js";
|
||||
import {
|
||||
broadcastAnalysisCompleted,
|
||||
LAST_ERROR,
|
||||
scheduleAutoDelete,
|
||||
} from "./moderationState.js";
|
||||
|
||||
const logger = createChildLogger("batch-processor");
|
||||
|
||||
@@ -105,7 +107,7 @@ export async function skipAgeRestrictedMessages(
|
||||
return messages;
|
||||
}
|
||||
|
||||
const skippedRows = await updateMessagesAIAnalysisBulk(
|
||||
const skippedRows = await messageStore.updateMessagesAIAnalysisBulk(
|
||||
ageRestrictedMessages.map((message) => ({
|
||||
messageId: message.id,
|
||||
result: buildAgeRestrictedSkipResult(),
|
||||
@@ -300,29 +302,31 @@ export async function processBatch(
|
||||
);
|
||||
|
||||
// Revert to pending so they are picked up again
|
||||
const revertedRows = await updateMessagesAIAnalysisBulk(
|
||||
apiFailedMessages.map((msg) => ({
|
||||
messageId: msg.id,
|
||||
result: {
|
||||
status: "pending",
|
||||
flags: null,
|
||||
score: null,
|
||||
analysis: null,
|
||||
categories: null,
|
||||
severity: null,
|
||||
confidence: null,
|
||||
recommendedAction: null,
|
||||
analyzedAt: null,
|
||||
error: null,
|
||||
},
|
||||
})),
|
||||
).catch((err) => {
|
||||
logger.error(
|
||||
{ error: String(err) },
|
||||
"Failed to revert API failures to pending",
|
||||
);
|
||||
return [];
|
||||
});
|
||||
const revertedRows = await messageStore
|
||||
.updateMessagesAIAnalysisBulk(
|
||||
apiFailedMessages.map((msg) => ({
|
||||
messageId: msg.id,
|
||||
result: {
|
||||
status: "pending",
|
||||
flags: null,
|
||||
score: null,
|
||||
analysis: null,
|
||||
categories: null,
|
||||
severity: null,
|
||||
confidence: null,
|
||||
recommendedAction: null,
|
||||
analyzedAt: null,
|
||||
error: null,
|
||||
},
|
||||
})),
|
||||
)
|
||||
.catch((err) => {
|
||||
logger.error(
|
||||
{ error: String(err) },
|
||||
"Failed to revert API failures to pending",
|
||||
);
|
||||
return [];
|
||||
});
|
||||
|
||||
for (const row of revertedRows) {
|
||||
broadcastAnalysisCompleted(row);
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { createChildLogger } from "@bete/shared/logger";
|
||||
import { config } from "../../shared/config/config.js";
|
||||
import { getPendingMessagesByConversation } from "../message-capture/messageStore.js";
|
||||
import { messageStore } from "../message-capture/messageStore.js";
|
||||
import type { MessageRecord } from "../message-capture/types.js";
|
||||
import {
|
||||
pickBatchWithinBudget,
|
||||
@@ -14,7 +14,7 @@ import {
|
||||
conversationProcessing,
|
||||
isConversationProcessingLocked,
|
||||
MAX_CONSECUTIVE_ERRORS,
|
||||
} from "./circuitBreaker.js";
|
||||
} from "./conversationState.js";
|
||||
|
||||
const logger = createChildLogger("batch-scheduler");
|
||||
|
||||
@@ -25,15 +25,10 @@ const logger = createChildLogger("batch-scheduler");
|
||||
/**
|
||||
* Schedules a debounced analysis run for a conversation.
|
||||
*
|
||||
* FIX #3: The async work inside setTimeout is now wrapped in an explicit
|
||||
* .catch() so DB errors don't produce unhandled promise rejections.
|
||||
* FIX #6: Calls pickBatchWithinBudget after fetching messages so token budget
|
||||
* is respected before handing the batch to the LLM.
|
||||
* FIX #7: Unified single-timer path -- always clear-and-reset one timer per
|
||||
* conversation key regardless of whether a cooldown is active. The delay is
|
||||
* simply max(cooldownRemainder+500, debounce) so the same timer serves both
|
||||
* the "throttled by error cooldown" and "normal debounce" cases, eliminating
|
||||
* the previous two-path logic that could leave both timers live simultaneously.
|
||||
* The async work inside setTimeout is wrapped in an explicit .catch() so
|
||||
* DB errors don't produce unhandled promise rejections. Uses a unified
|
||||
* single-timer path: always clear-and-reset one timer per conversation key
|
||||
* regardless of whether a cooldown is active.
|
||||
*/
|
||||
export function scheduleConversationAnalysis(conversationKey: string): void {
|
||||
if (isConversationProcessingLocked(conversationKey)) {
|
||||
@@ -65,18 +60,17 @@ export function scheduleConversationAnalysis(conversationKey: string): void {
|
||||
const timer = setTimeout(() => {
|
||||
conversationDebounceTimers.delete(conversationKey);
|
||||
|
||||
// FIX TOCTOU: Set lock synchronously BEFORE the async DB fetch starts
|
||||
if (isConversationProcessingLocked(conversationKey)) {
|
||||
return;
|
||||
}
|
||||
const processingStartedAt = Date.now();
|
||||
conversationProcessing.set(conversationKey, processingStartedAt);
|
||||
|
||||
// FIX #3: explicit .catch() -- no async arrow function to avoid unhandled rejection.
|
||||
getPendingMessagesByConversation(
|
||||
conversationKey,
|
||||
config.AI_ANALYSIS_MAX_BATCH_SIZE,
|
||||
)
|
||||
messageStore
|
||||
.getPendingMessagesByConversation(
|
||||
conversationKey,
|
||||
config.AI_ANALYSIS_MAX_BATCH_SIZE,
|
||||
)
|
||||
.then(async (messages: MessageRecord[]) => {
|
||||
if (messages.length === 0) {
|
||||
if (
|
||||
@@ -97,15 +91,14 @@ export function scheduleConversationAnalysis(conversationKey: string): void {
|
||||
return;
|
||||
}
|
||||
|
||||
// FIX #6: trim to token budget before sending to LLM.
|
||||
let trimmed = pickBatchWithinBudget(
|
||||
processableMessages,
|
||||
config.AI_ANALYSIS_MAX_TARGET_TOKENS,
|
||||
50,
|
||||
);
|
||||
|
||||
// FIX #10: if every message individually exceeds the token budget,
|
||||
// fall back to the first message alone.
|
||||
// If every message individually exceeds the token budget,
|
||||
// fall back to the first message alone to avoid stuck-pending deadlock.
|
||||
if (trimmed.length === 0 && processableMessages.length > 0) {
|
||||
trimmed = processableMessages.slice(0, 1);
|
||||
logger.warn(
|
||||
|
||||
@@ -1,16 +1,9 @@
|
||||
import { existsSync } from "node:fs";
|
||||
import { availableParallelism } from "node:os";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { createChildLogger } from "@bete/shared/logger";
|
||||
import type { Client } from "discord.js-selfbot-v13";
|
||||
import { LRUCache } from "lru-cache";
|
||||
import { Piscina } from "piscina";
|
||||
import { config } from "../../shared/config/config.js";
|
||||
import type { EventBroadcaster } from "../event-broadcaster/index.js";
|
||||
import type { MessageRecord } from "../message-capture/types.js";
|
||||
import { attemptAutoDeleteFlaggedMessage } from "./autoDeleteManager.js";
|
||||
|
||||
const logger = createChildLogger("circuit-breaker");
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Piscina worker pool (shared by batch + individual pipelines)
|
||||
@@ -44,192 +37,3 @@ export const workerPool = new Piscina({
|
||||
export function getConversationKey(message: MessageRecord): string {
|
||||
return message.thread_id || message.channel_id;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Shared observable state
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** Redis EventBroadcaster -- set externally so sub-modules can publish events. */
|
||||
export let _redisEventBroadcaster: EventBroadcaster | undefined;
|
||||
|
||||
/** Discord client reference -- needed for auto-delete actions. */
|
||||
export let moderationClient: Client | undefined;
|
||||
|
||||
export function setSharedEventBroadcaster(
|
||||
eb: EventBroadcaster | undefined,
|
||||
): void {
|
||||
_redisEventBroadcaster = eb;
|
||||
}
|
||||
|
||||
export function setModerationClient(mc: Client | undefined): void {
|
||||
moderationClient = mc;
|
||||
}
|
||||
|
||||
/**
|
||||
* Per-message in-flight guard for the auto-delete side-effect.
|
||||
* (LRU-backed to prevent unbounded growth)
|
||||
*/
|
||||
export const autoDeleteInFlight = new LRUCache<string, true>({ max: 10000 });
|
||||
|
||||
/** Last recorded error across all pipelines. */
|
||||
export const LAST_ERROR: { value: string | null } = { value: null };
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Batch circuit breaker state
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export const conversationConsecutiveErrors = new LRUCache<string, number>({
|
||||
max: 10000,
|
||||
});
|
||||
export const MAX_CONSECUTIVE_ERRORS = 5;
|
||||
export const CONVERSATION_CB_COOLDOWN_MS = 60000;
|
||||
export const conversationErrorCooldown = new LRUCache<string, number>({
|
||||
max: 10000,
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Scheduling / timing state (shared so sub-modules can access without cycles)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** Debounce timer handle per conversation key. */
|
||||
export const conversationDebounceTimers = new LRUCache<string, NodeJS.Timeout>({
|
||||
max: 10000,
|
||||
dispose: (value) => {
|
||||
clearTimeout(value);
|
||||
},
|
||||
});
|
||||
|
||||
/** Timestamp of when processing started per conversation key. */
|
||||
export const conversationProcessing = new LRUCache<string, number>({
|
||||
max: 10000,
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Conversation lock helper
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export function isConversationProcessingLocked(
|
||||
conversationKey: string,
|
||||
): boolean {
|
||||
const startedAt = conversationProcessing.get(conversationKey);
|
||||
return Boolean(
|
||||
startedAt &&
|
||||
Date.now() - startedAt < config.AI_ANALYSIS_PROCESSING_TIMEOUT_MS,
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Alert system
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export type CircuitBreakerAlert = {
|
||||
type: "conversation_cb" | "individual_cb" | "sustained_error";
|
||||
conversationKey?: string;
|
||||
consecutiveErrors: number;
|
||||
message: string;
|
||||
lastError?: string | null;
|
||||
};
|
||||
|
||||
const alertHandlers: Array<(alert: CircuitBreakerAlert) => void> = [];
|
||||
|
||||
/**
|
||||
* Register an alert handler (e.g., for webhook integration).
|
||||
*/
|
||||
export function onCircuitBreakerAlert(
|
||||
handler: (alert: CircuitBreakerAlert) => void,
|
||||
): void {
|
||||
alertHandlers.push(handler);
|
||||
}
|
||||
|
||||
export function fireAlert(alert: CircuitBreakerAlert): void {
|
||||
logger.warn(alert, `CB Alert: ${alert.type} -- ${alert.message}`);
|
||||
for (const handler of alertHandlers) {
|
||||
try {
|
||||
handler(alert);
|
||||
} catch {
|
||||
// handler errors are non-critical
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Circuit breaker helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export function recordConversationBatchFailure(conversationKey: string): void {
|
||||
const nextCount =
|
||||
(conversationConsecutiveErrors.get(conversationKey) ?? 0) + 1;
|
||||
conversationConsecutiveErrors.set(conversationKey, nextCount);
|
||||
|
||||
if (nextCount >= MAX_CONSECUTIVE_ERRORS) {
|
||||
conversationErrorCooldown.set(
|
||||
conversationKey,
|
||||
Date.now() + CONVERSATION_CB_COOLDOWN_MS,
|
||||
);
|
||||
fireAlert({
|
||||
type: "conversation_cb",
|
||||
conversationKey,
|
||||
consecutiveErrors: nextCount,
|
||||
message: `Conversation ${conversationKey} circuit breaker triggered after ${nextCount} consecutive errors`,
|
||||
lastError: LAST_ERROR.value,
|
||||
});
|
||||
conversationConsecutiveErrors.set(conversationKey, 0);
|
||||
}
|
||||
}
|
||||
|
||||
export function resetConversationBatchFailures(conversationKey: string): void {
|
||||
conversationConsecutiveErrors.delete(conversationKey);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Broadcast & auto-delete helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export function broadcastAnalysisCompleted(row: MessageRecord): void {
|
||||
if (_redisEventBroadcaster) {
|
||||
_redisEventBroadcaster.messageAnalyzed(row).catch((err: unknown) =>
|
||||
logger.warn(
|
||||
{
|
||||
messageId: row.id,
|
||||
error: err instanceof Error ? err.message : String(err),
|
||||
},
|
||||
"Failed to publish message_analyzed via Redis EventBroadcaster",
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export function scheduleAutoDelete(row: MessageRecord): void {
|
||||
if (row.ai_status !== "flagged" && row.ai_status !== "warn") return;
|
||||
|
||||
if (autoDeleteInFlight.has(row.id)) {
|
||||
logger.debug(
|
||||
{ messageId: row.id },
|
||||
"Auto-delete skipped: already in-flight for this message",
|
||||
);
|
||||
return;
|
||||
}
|
||||
autoDeleteInFlight.set(row.id, true);
|
||||
|
||||
const run = () => {
|
||||
attemptAutoDeleteFlaggedMessage(moderationClient, row)
|
||||
.catch((error: unknown) => {
|
||||
logger.error(
|
||||
{
|
||||
messageId: row.id,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
},
|
||||
"Unexpected auto-delete error",
|
||||
);
|
||||
})
|
||||
.finally(() => {
|
||||
autoDeleteInFlight.delete(row.id);
|
||||
});
|
||||
};
|
||||
|
||||
if (config.AUTO_DELETE_FLAGGED_DELAY_MS > 0) {
|
||||
setTimeout(run, config.AUTO_DELETE_FLAGGED_DELAY_MS);
|
||||
return;
|
||||
}
|
||||
setImmediate(run);
|
||||
}
|
||||
|
||||
@@ -1,53 +0,0 @@
|
||||
import { createChildLogger } from "@bete/shared/logger";
|
||||
import pLimit from "p-limit";
|
||||
import { config } from "../../shared/config/config.js";
|
||||
|
||||
const logger = createChildLogger("concurrencyLimiter");
|
||||
|
||||
/**
|
||||
* Concurrency limiter for LLM API calls.
|
||||
*
|
||||
* Prevents rate-limit (429) errors by capping simultaneous requests
|
||||
* to the configured maximum (default: 5).
|
||||
*/
|
||||
const llmSemaphore = pLimit(config.AI_LLM_MAX_CONCURRENT ?? 5);
|
||||
|
||||
let activeCount = 0;
|
||||
let pendingCount = 0;
|
||||
|
||||
// Track queue state changes for logging
|
||||
function _updateCounts(): void {
|
||||
// p-limit exposes queueSize and activeCount via constructor internals,
|
||||
// but we track via our wrapper to avoid depending on internals.
|
||||
}
|
||||
|
||||
export async function withLlmConcurrency<T>(fn: () => Promise<T>): Promise<T> {
|
||||
const _queuedAt = activeCount + pendingCount;
|
||||
pendingCount++;
|
||||
logger.debug(
|
||||
{ activeCount, pendingCount, maxConcurrent: config.AI_LLM_MAX_CONCURRENT },
|
||||
"Queuing LLM request",
|
||||
);
|
||||
|
||||
return llmSemaphore(async () => {
|
||||
pendingCount--;
|
||||
activeCount++;
|
||||
|
||||
if (activeCount >= (config.AI_LLM_MAX_CONCURRENT ?? 5)) {
|
||||
logger.warn(
|
||||
{ activeCount, maxConcurrent: config.AI_LLM_MAX_CONCURRENT },
|
||||
"LLM concurrency limit reached",
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
return await fn();
|
||||
} finally {
|
||||
activeCount--;
|
||||
logger.debug(
|
||||
{ activeCount, pendingCount },
|
||||
"LLM request completed, concurrency slot released",
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -28,7 +28,7 @@ function formatTimestamp(ms: number): string {
|
||||
}
|
||||
|
||||
/**
|
||||
* Estimates token count for a string (pessimistic approximation for Indonesian slang & JSON overhead)
|
||||
* Estimates token count for a string using tiktoken for accurate counting
|
||||
*/
|
||||
export function estimateTokens(text: string): number {
|
||||
// Use tiktoken for accurate token counting (+15 overhead for JSON structure)
|
||||
|
||||
@@ -0,0 +1,143 @@
|
||||
import { createChildLogger } from "@bete/shared/logger";
|
||||
import { LRUCache } from "lru-cache";
|
||||
import { config } from "../../shared/config/config.js";
|
||||
import { LAST_ERROR } from "./moderationState.js";
|
||||
|
||||
/**
|
||||
* # Boundary: Per-conversation batching, circuit breakers & alerts
|
||||
*
|
||||
* This module owns **per-conversation** state for the AI analysis batching
|
||||
* pipeline: circuit-breaker error tracking, debounce timers, and processing
|
||||
* locks that prevent duplicate concurrent analysis of the same conversation.
|
||||
*
|
||||
* ## What lives here
|
||||
* - `conversationConsecutiveErrors` — circuit-breaker: consecutive error count
|
||||
* per conversation key.
|
||||
* - `conversationErrorCooldown` — circuit-breaker: timestamp at which the
|
||||
* cooldown expires (cooldown = 60s of no batch scheduling after 5 errors).
|
||||
* - `conversationDebounceTimers` — scheduling: active `setTimeout` handles so
|
||||
* pending batches can be cancelled/rescheduled.
|
||||
* - `conversationProcessing` — lock: `Date.now()` when processing started, used
|
||||
* by `isConversationProcessingLocked()` to detect stale processing slots.
|
||||
* - `recordConversationBatchFailure()` / `resetConversationBatchFailures()` —
|
||||
* circuit-breaker mutation helpers.
|
||||
* - Alert system: `CircuitBreakerAlert` type, `fireAlert()`, and
|
||||
* `onCircuitBreakerAlert()` for pluggable handler registration.
|
||||
*
|
||||
* ## Relationship with moderationState.ts
|
||||
* - `moderationState.ts` owns **infrastructure references** (event broadcaster,
|
||||
* Discord client), the auto-delete guard, the `LAST_ERROR` tracker, and
|
||||
* action helpers (`broadcastAnalysisCompleted`, `scheduleAutoDelete`).
|
||||
* - The only cross-module dependency is this file importing `LAST_ERROR` from
|
||||
* `moderationState.ts` to include the latest pipeline error in alerts.
|
||||
* - These are **separate concerns** — do not merge them.
|
||||
*/
|
||||
|
||||
const logger = createChildLogger("conversation-state");
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Batch circuit breaker state
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export const conversationConsecutiveErrors = new LRUCache<string, number>({
|
||||
max: 10000,
|
||||
});
|
||||
export const MAX_CONSECUTIVE_ERRORS = 5;
|
||||
export const CONVERSATION_CB_COOLDOWN_MS = 60000;
|
||||
export const conversationErrorCooldown = new LRUCache<string, number>({
|
||||
max: 10000,
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Scheduling / timing state (shared so sub-modules can access without cycles)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** Debounce timer handle per conversation key. */
|
||||
export const conversationDebounceTimers = new LRUCache<string, NodeJS.Timeout>({
|
||||
max: 10000,
|
||||
dispose: (value) => {
|
||||
clearTimeout(value);
|
||||
},
|
||||
});
|
||||
|
||||
/** Timestamp of when processing started per conversation key. */
|
||||
export const conversationProcessing = new LRUCache<string, number>({
|
||||
max: 10000,
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Conversation lock helper
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export function isConversationProcessingLocked(
|
||||
conversationKey: string,
|
||||
): boolean {
|
||||
const startedAt = conversationProcessing.get(conversationKey);
|
||||
return Boolean(
|
||||
startedAt &&
|
||||
Date.now() - startedAt < config.AI_ANALYSIS_PROCESSING_TIMEOUT_MS,
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Alert system
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export type CircuitBreakerAlert = {
|
||||
type: "conversation_cb" | "individual_cb" | "sustained_error";
|
||||
conversationKey?: string;
|
||||
consecutiveErrors: number;
|
||||
message: string;
|
||||
lastError?: string | null;
|
||||
};
|
||||
|
||||
const alertHandlers: Array<(alert: CircuitBreakerAlert) => void> = [];
|
||||
|
||||
/**
|
||||
* Register an alert handler (e.g., for webhook integration).
|
||||
*/
|
||||
export function onCircuitBreakerAlert(
|
||||
handler: (alert: CircuitBreakerAlert) => void,
|
||||
): void {
|
||||
alertHandlers.push(handler);
|
||||
}
|
||||
|
||||
export function fireAlert(alert: CircuitBreakerAlert): void {
|
||||
logger.warn(alert, `CB Alert: ${alert.type} -- ${alert.message}`);
|
||||
for (const handler of alertHandlers) {
|
||||
try {
|
||||
handler(alert);
|
||||
} catch {
|
||||
// handler errors are non-critical
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Circuit breaker helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export function recordConversationBatchFailure(conversationKey: string): void {
|
||||
const nextCount =
|
||||
(conversationConsecutiveErrors.get(conversationKey) ?? 0) + 1;
|
||||
conversationConsecutiveErrors.set(conversationKey, nextCount);
|
||||
|
||||
if (nextCount >= MAX_CONSECUTIVE_ERRORS) {
|
||||
conversationErrorCooldown.set(
|
||||
conversationKey,
|
||||
Date.now() + CONVERSATION_CB_COOLDOWN_MS,
|
||||
);
|
||||
fireAlert({
|
||||
type: "conversation_cb",
|
||||
conversationKey,
|
||||
consecutiveErrors: nextCount,
|
||||
message: `Conversation ${conversationKey} circuit breaker triggered after ${nextCount} consecutive errors`,
|
||||
lastError: LAST_ERROR.value,
|
||||
});
|
||||
conversationConsecutiveErrors.set(conversationKey, 0);
|
||||
}
|
||||
}
|
||||
|
||||
export function resetConversationBatchFailures(conversationKey: string): void {
|
||||
conversationConsecutiveErrors.delete(conversationKey);
|
||||
}
|
||||
@@ -1,76 +0,0 @@
|
||||
import { createChildLogger } from "@bete/shared/logger";
|
||||
|
||||
const log = createChildLogger("imageMimeSniffer");
|
||||
|
||||
/**
|
||||
* Sniff the first bytes of a buffer to determine if it is a supported image
|
||||
* format. Returns the canonical MIME type string on success, or null if the
|
||||
* bytes are not a recognizable image.
|
||||
*/
|
||||
export function sniffImageMimeType(buf: Buffer): string | null {
|
||||
if (buf.length < 12) return null;
|
||||
|
||||
if (buf[0] === 0xff && buf[1] === 0xd8 && buf[2] === 0xff) {
|
||||
return "image/jpeg";
|
||||
}
|
||||
|
||||
if (
|
||||
buf[0] === 0x89 &&
|
||||
buf[1] === 0x50 &&
|
||||
buf[2] === 0x4e &&
|
||||
buf[3] === 0x47 &&
|
||||
buf[4] === 0x0d &&
|
||||
buf[5] === 0x0a &&
|
||||
buf[6] === 0x1a &&
|
||||
buf[7] === 0x0a
|
||||
) {
|
||||
return "image/png";
|
||||
}
|
||||
|
||||
if (
|
||||
buf[0] === 0x47 &&
|
||||
buf[1] === 0x49 &&
|
||||
buf[2] === 0x46 &&
|
||||
buf[3] === 0x38
|
||||
) {
|
||||
return "image/gif";
|
||||
}
|
||||
|
||||
if (
|
||||
buf[0] === 0x52 &&
|
||||
buf[1] === 0x49 &&
|
||||
buf[2] === 0x46 &&
|
||||
buf[3] === 0x46 &&
|
||||
buf[8] === 0x57 &&
|
||||
buf[9] === 0x45 &&
|
||||
buf[10] === 0x42 &&
|
||||
buf[11] === 0x50
|
||||
) {
|
||||
return "image/webp";
|
||||
}
|
||||
|
||||
if (
|
||||
buf.length >= 12 &&
|
||||
buf[4] === 0x66 &&
|
||||
buf[5] === 0x74 &&
|
||||
buf[6] === 0x79 &&
|
||||
buf[7] === 0x70
|
||||
) {
|
||||
const brand = buf.subarray(8, 12).toString("ascii");
|
||||
if (brand.startsWith("avif") || brand.startsWith("avis")) {
|
||||
return "image/avif";
|
||||
}
|
||||
if (
|
||||
brand.startsWith("mif1") ||
|
||||
brand.startsWith("heic") ||
|
||||
brand.startsWith("heis")
|
||||
) {
|
||||
return "image/heic";
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
// Keep log referenced so TS does not tree-shake the logger init
|
||||
log.debug("imageMimeSniffer loaded");
|
||||
@@ -1,3 +1,4 @@
|
||||
export { startPendingAIAnalysisWorker } from "./aiAnalyzer.js";
|
||||
export { runModerationAnalysis } from "./llmModerationClient.js";
|
||||
export { runModerationAnalysis } from "./moderationOrchestrator.js";
|
||||
export { buildSystemPrompt } from "./moderationPrompt.js";
|
||||
export { runSimpleTextFallback } from "./simpleFallback.js";
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user