MythEclipse/GMW
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor: remove voice, recording, and music/media features (gateway + backend)

Browse Source 
- Gateway: delete voice-recording/, voice-pcm-ws/, voice/video/media handlers,
  vendor/discord-voice-fork/, voice DB repos, 2 voice migrations
- Gateway: cut voice wiring from bootstrap/shutdown/commandHandler/handler-registry,
  eventBroadcaster/eventTypes, redis-channels, moderation-types, message-capture,
  config keys, and deps (@discordjs/voice, opus, libsodium, prism-media, davey)
- Backend: delete voice/, recordings/, media/ modules + @discordjs/voice dep
- Backend: cut voice/media/recordings oRPC routers, WS gateway-PCM auth + voice
  handlers, Redis bridge voice aggregation, redis-channels voice/media constants,
  config keys, moderation-types voice items, e2e voice/recordings suites
- Keep voice DB tables (destructive migration avoided); chatbot voiceRecordings
  tool + dashboard count remain as read-only historical data access
This commit is contained in:
asepharyana
2026-09-23 19:31:06 +07:00
parent b5b370e6eb
commit 33013697e0
79 changed files with 27 additions and 20185 deletions
Download Patch File Download Diff File Expand all files Collapse all files
services/discord-gateway/src/app/bootstrap.ts
+2 -76
View File
@@ -33,18 +33,6 @@ import { startDigestScheduler } from "../modules/monitor/digestScheduler.js";
import { registerReactionCapture } from "../modules/reaction-tracking/index.js";
import { registerThreadCapture } from "../modules/thread-tracking/index.js";
import { registerPresenceCapture } from "../modules/user-presence/index.js";
import { VoicePcmWsClient } from "../modules/voice-pcm-ws/index.js";
import { startMuxerWorker } from "../modules/voice-recording/muxer.js";
import {
registerSelfVoiceStateGuard,
setPcmWsClient,
setEventBroadcaster as setRecorderEventBroadcaster,
} from "../modules/voice-recording/recorder.js";
import {
setVideoRecorderClient,
setVideoRecordingsDir,
} from "../modules/voice-recording/videoRecorder.js";
import { VoiceController } from "../modules/voice-recording/voiceController.js";
import { config } from "../shared/config/config.js";
import {
closeDatabase,
@@ -53,11 +41,7 @@ import {
} from "../shared/database/drizzle.js";
import { runMigrations } from "../shared/database/migrate.js";
import type * as schema from "../shared/database/schema.js";
import {
attachmentsTable,
messagesTable,
voiceRecordingsTable,
} from "../shared/database/schema.js";
import { attachmentsTable, messagesTable } from "../shared/database/schema.js";
import { createDiscordClientOptions } from "../shared/discord/clientOptions.js";
import { createGracefulShutdown } from "./shutdown.js";
@@ -124,7 +108,6 @@ function startRetentionCleanup(): void {
dryRun,
messagesDays: config.RETENTION_MESSAGES_DAYS,
attachmentsDays: config.RETENTION_ATTACHMENTS_DAYS,
voiceDays: config.RETENTION_VOICE_DAYS,
},
"Starting retention cleanup scheduler",
);
@@ -144,13 +127,6 @@ function startRetentionCleanup(): void {
dryRun,
"attachments",
);
await deleteExpiredRecords(
voiceRecordingsTable,
voiceRecordingsTable.created_at,
config.RETENTION_VOICE_DAYS,
dryRun,
"voice recordings",
);
}
// Run immediately on start, then schedule
@@ -188,13 +164,6 @@ export async function initializeDiscordGateway() {
logger.info("Creating Discord client");
const client = new Client(createDiscordClientOptions());
const voiceController = new VoiceController(client);
// Wire the video recorder (others' camera/screen share) to the selfbot's
// native watch/receive stack + its recordings dir. Best-effort: failures are
// logged inside, never fatal.
setVideoRecorderClient(client);
setVideoRecordingsDir(config.RECORDINGS_DIR);
// Initialize Redis event broadcaster
const redisPublisher = new RedisEventPublisher(config.REDIS_URL, logger);
@@ -203,33 +172,13 @@ export async function initializeDiscordGateway() {
// Initialize Redis command handler for backend→gateway commands
const commandHandler = new CommandHandler();
// Initialize Voice PCM WebSocket client (bypasses Redis for real-time audio)
let pcmWsClient: VoicePcmWsClient | undefined;
if (config.VOICE_PCM_WS_ENABLED && config.BACKEND_WS_TOKEN) {
pcmWsClient = new VoicePcmWsClient(
config.BACKEND_WS_URL,
config.BACKEND_WS_TOKEN,
);
pcmWsClient.connect();
setPcmWsClient(pcmWsClient);
logger.info({ url: config.BACKEND_WS_URL }, "Voice PCM WS client enabled");
} else if (config.VOICE_PCM_WS_ENABLED && !config.BACKEND_WS_TOKEN) {
logger.warn(
"VOICE_PCM_WS_ENABLED=true but BACKEND_WS_TOKEN is empty — falling back to Redis for PCM",
);
} else {
logger.info("Voice PCM WS disabled — using Redis for PCM");
}
const gracefulShutdown = createGracefulShutdown({
logger,
closeDatabase,
voiceController,
client,
eventBroadcaster,
commandHandler,
stopMetricsServer,
pcmWsClient,
});
try {
@@ -255,8 +204,6 @@ export async function initializeDiscordGateway() {
client.on("debug", (msg) => {
if (
msg.includes("[VOICE") ||
msg.includes("[ffmpeg") ||
msg.toLowerCase().includes("error") ||
msg.toLowerCase().includes("stream")
) {
@@ -269,7 +216,6 @@ export async function initializeDiscordGateway() {
client.on("ready", async () => {
logger.info({ user: client.user?.tag }, "Bot logged in");
setMessageCaptureEventBroadcaster(eventBroadcaster);
setRecorderEventBroadcaster(eventBroadcaster);
setModerationEventBroadcaster(eventBroadcaster);
registerMessageCapture(client);
startPendingAIAnalysisWorker(client, eventBroadcaster);
@@ -281,30 +227,10 @@ export async function initializeDiscordGateway() {
registerChannelTopicCapture(client, eventBroadcaster);
registerGuildMemberEvents(client, eventBroadcaster);
// Start background workers
startMuxerWorker();
// Start command handler after Discord is ready
commandHandler.start(client, voiceController);
commandHandler.start(client);
logger.info("Command handler started");
// Rejoin persisted voice channels (auto-reconnect on restart/reboot).
// Non-fatal: failures are logged inside autoReconnect.
void voiceController
.autoReconnect()
.catch((err) =>
logger.warn(
{ err: err instanceof Error ? err.message : String(err) },
"Voice auto-reconnect on startup failed",
),
);
// Attach the immediate self-undeafen/self-unmute guard so the bot reacts
// INSTANTLY when an admin server-mutes or server-deafens it (previously
// only re-asserted on video-watch/reconnect, leaving the bot muted for
// minutes).
registerSelfVoiceStateGuard(client);
// Start retention cleanup scheduler
startRetentionCleanup();
// Start weekly moderation digest (public, automated)
services/discord-gateway/src/app/retention.ts
+1 -13
View File
@@ -4,11 +4,7 @@ import { createChildLogger } from "@/shared/logger/index";
import { config } from "../shared/config/config.js";
import { getDatabase } from "../shared/database/drizzle.js";
import type * as schema from "../shared/database/schema.js";
import {
attachmentsTable,
messagesTable,
voiceRecordingsTable,
} from "../shared/database/schema.js";
import { attachmentsTable, messagesTable } from "../shared/database/schema.js";
const logger = createChildLogger("discord-gateway");
@@ -73,7 +69,6 @@ function startRetentionCleanup(): void {
dryRun,
messagesDays: config.RETENTION_MESSAGES_DAYS,
attachmentsDays: config.RETENTION_ATTACHMENTS_DAYS,
voiceDays: config.RETENTION_VOICE_DAYS,
},
"Starting retention cleanup scheduler",
);
@@ -93,13 +88,6 @@ function startRetentionCleanup(): void {
dryRun,
"attachments",
);
await deleteExpiredRecords(
voiceRecordingsTable,
voiceRecordingsTable.created_at,
config.RETENTION_VOICE_DAYS,
dryRun,
"voice recordings",
);
}
// Run immediately on start, then schedule
services/discord-gateway/src/app/shutdown.ts
+2 -18
View File
@@ -3,9 +3,6 @@ import type { createChildLogger } from "@/shared/logger/index";
import type { CommandHandler } from "../modules/command-handler/commandHandler.js";
import type { EventBroadcaster } from "../modules/event-broadcaster/index.js";
import type { stopMetricsServer } from "../modules/gateway-metrics/index.js";
import type { VoicePcmWsClient } from "../modules/voice-pcm-ws/index.js";
import { stopMuxerWorker } from "../modules/voice-recording/muxer.js";
import type { VoiceController } from "../modules/voice-recording/voiceController.js";
import type { closeDatabase } from "../shared/database/drizzle.js";
type Logger = ReturnType<typeof createChildLogger>;
@@ -15,12 +12,10 @@ type StopMetricsServer = typeof stopMetricsServer;
export interface GracefulShutdownOptions {
logger: Logger;
closeDatabase: CloseDatabase;
voiceController: VoiceController;
client: Client;
eventBroadcaster: EventBroadcaster;
commandHandler: CommandHandler;
stopMetricsServer?: StopMetricsServer;
pcmWsClient?: VoicePcmWsClient;
}
export function createGracefulShutdown(options: GracefulShutdownOptions) {
@@ -37,26 +32,15 @@ export function createGracefulShutdown(options: GracefulShutdownOptions) {
try {
options.stopMetricsServer?.();
stopMuxerWorker();
// 1. Voice disconnect (may write final DB records)
options.logger.info("Stopping voice connection...");
await options.voiceController.disconnect();
// 1b. Close PCM WebSocket client (after voice, before Redis)
if (options.pcmWsClient) {
options.logger.info("Closing voice PCM WS client...");
await options.pcmWsClient.close();
}
// 2. Close Redis/pubsub after voice is done
// 1. Close Redis/pubsub
options.logger.info("Closing event broadcaster...");
await options.eventBroadcaster.close();
options.logger.info("Closing command handler...");
await options.commandHandler.close();
// 3. DB pool LAST – voice disconnect may finalize recordings
// 2. DB pool
options.logger.info("Closing database...");
await options.closeDatabase();
options.logger.info("Database closed");
services/discord-gateway/src/index.ts
-2
View File
@@ -1,5 +1,3 @@
import "libsodium-wrappers";
import "@snazzah/davey";
import "dotenv/config";
import { createChildLogger } from "@/shared/logger/index";
import { initializeDiscordGateway } from "./app/bootstrap.js";
services/discord-gateway/src/modules/command-handler/commandHandler.ts
+3 -93
View File
@@ -5,41 +5,17 @@ import {
BACKEND_COMMAND,
type CommandMessage,
type CommandReply,
MEDIA_STATUS_KEY,
VOICE_STATUS_KEY,
} from "../../shared/index.js";
import { createChildLogger } from "../../shared/logger/index.js";
import type { VoiceController } from "../voice-recording/voiceController.js";
import { GuildHandler } from "./guild.handler.js";
import {
type CommandHandlerFn,
createHandlerRegistry,
} from "./handler-registry.js";
import { MediaHandler } from "./media.handler.js";
import { wireMediaStatusWriter } from "./mediaStatusSink.js";
import { ModerationHandler } from "./moderation.handler.js";
import { VideoHandler } from "./video.handler.js";
import { VoiceHandler } from "./voice.handler.js";
const logger = createChildLogger("command-handler");
// ---------------------------------------------------------------------------
// Types
// ---------------------------------------------------------------------------
interface VoiceStatusPayload {
connected: boolean;
activeGuildId: string | null;
activeChannelId: string | null;
activeChannelName: string | null;
connections: Array<{
guildId: string;
channelId: string;
channelName: string;
connectedAt: number;
}>;
}
// ---------------------------------------------------------------------------
// CommandHandler
// ---------------------------------------------------------------------------
@@ -47,13 +23,9 @@ interface VoiceStatusPayload {
export class CommandHandler {
private redisSub: Redis;
private redisPub: Redis;
private voiceController: VoiceController | null = null;
private registry: Map<string, CommandHandlerFn> = new Map();
private voiceHandler!: VoiceHandler;
private mediaHandler!: MediaHandler;
private guildHandler!: GuildHandler;
private moderationHandler!: ModerationHandler;
private videoHandler!: VideoHandler;
constructor() {
// Dedicated Redis connection needed because: Redis requires a dedicated
@@ -77,30 +49,18 @@ export class CommandHandler {
// ---- Lifecycle ----
/**
* Attach the Discord client and VoiceController, then subscribe to the Redis
* command channel. Must be called *after* the Discord client is created.
* Attach the Discord client, then subscribe to the Redis command channel.
* Must be called *after* the Discord client is created.
*/
start(client: Client, voiceController: VoiceController): void {
this.voiceController = voiceController;
start(client: Client): void {
// Create domain-specific handlers with their dependencies
this.voiceHandler = new VoiceHandler(client, voiceController);
this.mediaHandler = new MediaHandler();
this.guildHandler = new GuildHandler(client);
this.moderationHandler = new ModerationHandler(client);
this.videoHandler = new VideoHandler(client, voiceController);
// Wire the media status sink so MediaHandler can persist status on
// queue advances that happen outside a command (natural track end).
wireMediaStatusWriter(this.redisPub);
// Build the command registry
this.registry = createHandlerRegistry(
this.voiceHandler,
this.mediaHandler,
this.guildHandler,
this.moderationHandler,
this.videoHandler,
);
this.redisSub.on("message", (_channel, message) => {
@@ -117,10 +77,6 @@ export class CommandHandler {
logger.info(`Subscribed to Redis channel "${BACKEND_COMMAND}"`);
}
});
// Publish initial status snapshots so the backend knows the starting state.
this.publishVoiceStatus();
this.publishMediaStatus();
}
async close(): Promise<void> {
@@ -175,51 +131,5 @@ export class CommandHandler {
} catch (err) {
logger.error({ err }, "Failed to publish command reply");
}
// Always refresh status keys after every command so the backend has
// the latest snapshot without polling.
this.publishVoiceStatus();
this.publishMediaStatus();
}
// ---- Status publishing ----
private publishVoiceStatus(): void {
const raw = this.voiceController
? this.voiceController.getStatus()
: {
ready: false,
connected: false,
activeGuildId: null,
activeChannelId: null,
activeChannelName: null,
connections: [],
};
const status: VoiceStatusPayload = {
connected: raw.connected,
activeGuildId: raw.activeGuildId,
activeChannelId: raw.activeChannelId,
activeChannelName: raw.activeChannelName,
connections: raw.connections ?? [],
};
this.setKey(VOICE_STATUS_KEY, JSON.stringify(status));
}
private publishMediaStatus(): void {
this.setKey(
MEDIA_STATUS_KEY,
JSON.stringify(this.mediaHandler.getCurrentMediaStatus()),
);
}
/**
* Fire-and-forget SET using the persistent Redis publisher connection.
*/
private setKey(key: string, value: string): void {
this.redisPub.set(key, value).catch((err: unknown) => {
const msg = err instanceof Error ? err.message : String(err);
logger.warn({ key, error: msg }, "Failed to update Redis status key");
});
}
}
services/discord-gateway/src/modules/command-handler/handler-registry.ts
-58
View File
@@ -1,28 +1,12 @@
import {
COMMAND_GUILDS_LIST,
COMMAND_GUILDS_TEXT_CHANNELS,
COMMAND_MEDIA_LOOP,
COMMAND_MEDIA_QUEUE,
COMMAND_MEDIA_SKIP,
COMMAND_MEDIA_STOP,
COMMAND_MEDIA_VOLUME,
COMMAND_MODERATION_ACTION,
COMMAND_VIDEO_UNWATCH,
COMMAND_VIDEO_WATCH,
COMMAND_VOICE_CHANNELS,
COMMAND_VOICE_CONNECT,
COMMAND_VOICE_DISCONNECT,
COMMAND_VOICE_DISCONNECT_GUILD,
COMMAND_VOICE_TRANSMIT_START,
COMMAND_VOICE_TRANSMIT_STOP,
type CommandMessage,
type CommandReply,
} from "../../shared/index.js";
import type { GuildHandler } from "./guild.handler.js";
import type { MediaHandler } from "./media.handler.js";
import type { ModerationHandler } from "./moderation.handler.js";
import type { VideoHandler } from "./video.handler.js";
import type { VoiceHandler } from "./voice.handler.js";
// ---------------------------------------------------------------------------
// Types
@@ -37,53 +21,11 @@ export type CommandHandlerFn = (
// ---------------------------------------------------------------------------
export function createHandlerRegistry(
voiceHandler: VoiceHandler,
mediaHandler: MediaHandler,
guildHandler: GuildHandler,
moderationHandler: ModerationHandler,
videoHandler: VideoHandler,
): Map<string, CommandHandlerFn> {
const registry = new Map<string, CommandHandlerFn>();
// Voice commands
registry.set(COMMAND_VOICE_CONNECT, (cmd) =>
voiceHandler.handleVoiceConnect(cmd),
);
registry.set(COMMAND_VOICE_DISCONNECT, (cmd) =>
voiceHandler.handleVoiceDisconnect(cmd),
);
registry.set(COMMAND_VOICE_DISCONNECT_GUILD, (cmd) =>
voiceHandler.handleVoiceDisconnectGuild(cmd),
);
registry.set(COMMAND_VOICE_CHANNELS, (cmd) =>
voiceHandler.handleVoiceChannels(cmd),
);
registry.set(COMMAND_VOICE_TRANSMIT_START, (cmd) =>
voiceHandler.handleVoiceTransmitStart(cmd),
);
registry.set(COMMAND_VOICE_TRANSMIT_STOP, (cmd) =>
voiceHandler.handleVoiceTransmitStop(cmd),
);
// Video watch commands
registry.set(COMMAND_VIDEO_WATCH, (cmd) =>
videoHandler.handleVideoWatch(cmd),
);
registry.set(COMMAND_VIDEO_UNWATCH, (cmd) =>
videoHandler.handleVideoUnwatch(cmd),
);
// Media commands
registry.set(COMMAND_MEDIA_QUEUE, (cmd) =>
mediaHandler.handleMediaQueue(cmd),
);
registry.set(COMMAND_MEDIA_SKIP, (cmd) => mediaHandler.handleMediaSkip(cmd));
registry.set(COMMAND_MEDIA_STOP, (cmd) => mediaHandler.handleMediaStop(cmd));
registry.set(COMMAND_MEDIA_VOLUME, (cmd) =>
mediaHandler.handleMediaVolume(cmd),
);
registry.set(COMMAND_MEDIA_LOOP, (cmd) => mediaHandler.handleMediaLoop(cmd));
// Guild commands
registry.set(COMMAND_GUILDS_LIST, (cmd) =>
guildHandler.handleListGuilds(cmd),
services/discord-gateway/src/modules/command-handler/media.handler.ts
-342
View File
@@ -1,342 +0,0 @@
import { randomUUID } from "node:crypto";
import { StreamType } from "@discordjs/voice";
import type { CommandMessage, CommandReply } from "../../shared/index.js";
import { createChildLogger } from "../../shared/logger/index.js";
import {
extractMediaInfo,
resolveMediaUrl,
transcodeToHighQualityOgg,
} from "../voice-recording/mediaSource.js";
import type {
MediaMode,
MediaQueueItem,
} from "../voice-recording/mediaTypes.js";
import { discordPlayer } from "../voice-recording/player.js";
import { setMediaStatusKey } from "./mediaStatusSink.js";
// ---------------------------------------------------------------------------
// Types
// ---------------------------------------------------------------------------
interface MediaStatusItem {
id: string;
source: string;
title: string;
mode: MediaMode;
durationMs?: number | null;
thumbnailUrl?: string | null;
}
export interface MediaStatusPayload {
playing: boolean;
activeMode: MediaMode | null;
musicVolume: number;
loop: boolean;
current: MediaStatusItem | null;
queue: MediaStatusItem[];
}
// ---------------------------------------------------------------------------
// Module-level queue
// ---------------------------------------------------------------------------
const mediaQueue: MediaQueueItem[] = [];
let currentTrackItem: MediaQueueItem | null = null;
let loopEnabled = false;
/** Active ffmpeg transcode (killed on stop/skip). */
let currentTranscodeCleanup: (() => void) | null = null;
// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------
function mapToStatusItem(item: MediaQueueItem): MediaStatusItem {
return {
id: item.id,
source: item.source,
title: item.title,
mode: item.mode,
durationMs: item.duration != null ? item.duration * 1000 : null,
thumbnailUrl: item.thumbnailUrl ?? null,
};
}
function buildStatusPayload(): MediaStatusPayload {
return {
playing:
currentTrackItem !== null && discordPlayer.getStatus() === "playing",
activeMode: currentTrackItem?.mode ?? null,
musicVolume: discordPlayer.getMusicVolume(),
loop: loopEnabled,
current: currentTrackItem ? mapToStatusItem(currentTrackItem) : null,
queue: mediaQueue.map(mapToStatusItem),
};
}
// ---------------------------------------------------------------------------
// MediaHandler
// ---------------------------------------------------------------------------
export class MediaHandler {
private logger = createChildLogger("media-handler");
constructor() {
// Register auto-advance on natural track end. advanceQueue mutates the
// module-level currentTrackItem/queue, so we must re-publish the status
// key afterward: otherwise the backend's Redis `media:status` cache (and
// the frontend's 10s polling) stays stuck on the finished track.
discordPlayer.onIdle(() => {
this.advanceQueue()
.then(() => this.publishStatus())
.catch((err) => {
this.logger.error({ err }, "Auto-advance failed");
});
});
}
/**
* Persist the latest media state to Redis so the backend/frontend see queue
* advances that happen outside a command (natural track end). CommandHandler
* owns the Redis status-key writes for command-triggered changes; this
* covers the side-effect-only path.
*/
private publishStatus(): void {
try {
setMediaStatusKey(this.getCurrentMediaStatus());
} catch (err: unknown) {
this.logger.warn(
{ error: err instanceof Error ? err.message : String(err) },
"Failed to publish media status on track end",
);
}
}
getCurrentMediaStatus(): MediaStatusPayload {
return buildStatusPayload();
}
async handleMediaQueue(cmd: CommandMessage): Promise<CommandReply<unknown>> {
// Accept both `url` (canonical) and `source` (legacy FE) for resilience.
const url = String(cmd.payload.url ?? cmd.payload.source ?? "").trim();
const requestedBy = String(cmd.payload.requestedBy ?? "unknown");
if (!url) {
this.logger.warn("media:queue received without a URL");
return {
id: cmd.id,
success: false,
data: null,
error: "url is required",
};
}
if (!discordPlayer.isConnected()) {
this.logger.warn("media:queue attempted without active voice connection");
return {
id: cmd.id,
success: false,
data: null,
error: "Not connected to a voice channel. Connect to voice first.",
};
}
// Lightweight metadata fetch for display — the full resolve happens in playNext
let title: string = url;
let duration: number | undefined;
let thumbnailUrl: string | undefined;
try {
const info = await extractMediaInfo(url);
title = info.title ?? url;
duration = info.duration;
thumbnailUrl = info.thumbnail;
} catch {
this.logger.warn(
{ url },
"Could not pre-resolve media metadata, queueing blind",
);
}
const item: MediaQueueItem = {
id: randomUUID(),
source: url,
title,
kind: "url" as const,
mode: "music",
requestedBy,
addedAt: Date.now(),
status: "queued",
duration,
thumbnailUrl,
};
mediaQueue.push(item);
this.logger.info(
{ url, title, queueLength: mediaQueue.length },
"Media queued",
);
// If nothing is playing, start immediately (next tick so status publishes)
if (currentTrackItem === null) {
setImmediate(() => {
this.playNext().catch((err) => {
this.logger.error({ err }, "playNext after queue failed");
});
});
}
return {
id: cmd.id,
success: true,
data: buildStatusPayload(),
};
}
async handleMediaSkip(cmd: CommandMessage): Promise<CommandReply<unknown>> {
if (currentTrackItem) {
discordPlayer.stop("music");
currentTrackItem = null;
}
await this.playNext();
return {
id: cmd.id,
success: true,
data: buildStatusPayload(),
};
}
async handleMediaStop(cmd: CommandMessage): Promise<CommandReply<unknown>> {
discordPlayer.stop("music");
currentTrackItem = null;
mediaQueue.length = 0; // Clear entire queue
return {
id: cmd.id,
success: true,
data: buildStatusPayload(),
};
}
async handleMediaVolume(cmd: CommandMessage): Promise<CommandReply<unknown>> {
const volume = Number(cmd.payload.volume);
if (!Number.isFinite(volume)) {
return {
id: cmd.id,
success: false,
data: null,
error: "volume must be a number",
};
}
discordPlayer.setMusicVolume(volume);
return {
id: cmd.id,
success: true,
data: buildStatusPayload(),
};
}
async handleMediaLoop(cmd: CommandMessage): Promise<CommandReply<unknown>> {
loopEnabled = Boolean(cmd.payload.loop);
this.logger.info({ loop: loopEnabled }, "Media loop toggled");
this.publishStatus();
return {
id: cmd.id,
success: true,
data: buildStatusPayload(),
};
}
// ---------------------------------------------------------------------------
// Internal
// ---------------------------------------------------------------------------
/**
* Advance to the next item in the queue. Called after natural track end
* (via onIdle) or after manual skip.
*/
private async playNext(): Promise<void> {
// Stop any currently playing track (without triggering idle callback)
if (currentTrackItem) {
discordPlayer.stop("music");
currentTrackItem = null;
}
currentTranscodeCleanup?.();
currentTranscodeCleanup = null;
const next = mediaQueue.shift();
if (!next) {
this.logger.info("Queue empty — nothing to play");
return;
}
currentTrackItem = next;
next.status = "playing";
try {
this.logger.info(
{ url: next.source, title: next.title },
"Playing next from queue",
);
const resolution = await resolveMediaUrl(next.source);
// Update title/duration with actual resolved values
next.title = resolution.title ?? next.title;
next.duration = resolution.duration ?? next.duration;
// Music playback: transcode once to high-quality OggOpus (48kHz stereo,
// 192kbps) with volume baked into the encode. This avoids the double
// lossy encode that inlineVolume would cause and gives Discord the
// cleanest possible stream.
const transcoded = transcodeToHighQualityOgg(
resolution.stream,
discordPlayer.getMusicVolume(),
);
discordPlayer.playStream(transcoded.stream, "music", {
inputType: StreamType.OggOpus,
inlineVolume: false,
});
currentTranscodeCleanup = transcoded.cleanup;
this.logger.info({ title: next.title }, "Playback started");
} catch (err) {
const message = err instanceof Error ? err.message : String(err);
this.logger.error(
{ error: message, url: next.source, title: next.title },
"Failed to play queued item, skipping to next",
);
currentTrackItem = null;
next.status = "failed";
// Try the next item in the queue
setImmediate(() => {
this.playNext().catch((err2) => {
this.logger.error(
{ err: err2 },
"playNext after error recovery failed",
);
});
});
}
}
/**
* Called by the idle callback — delegates to playNext since the player is
* already idle and currentTrackItem is already null. When loop mode is
* enabled and a music track ended naturally, requeue it so it plays again.
*/
private async advanceQueue(): Promise<void> {
const finished = currentTrackItem;
currentTrackItem = null;
if (loopEnabled && finished && finished.mode === "music") {
mediaQueue.unshift(finished);
this.logger.info(
{ title: finished.title },
"Loop enabled — replaying finished track",
);
}
await this.playNext();
}
}
services/discord-gateway/src/modules/command-handler/mediaStatusSink.ts
-41
View File
@@ -1,41 +0,0 @@
import type Redis from "ioredis";
import { createChildLogger } from "@/shared/logger/index";
import { MEDIA_STATUS_KEY } from "../../shared/redis-channels.js";
/**
* Shared sink for writing the media status Redis key.
*
* CommandHandler owns the publisher + status writes for command-triggered
* changes (`publishMediaStatus`). MediaHandler needs to also persist status
* when the queue advances *outside* a command (natural track end / screen-share
* done), so we expose the real publisher here and let CommandHandler wire it
* once at startup.
*/
const logger = createChildLogger("media-status-sink");
let _setMediaStatusKey: ((payload: unknown) => void) | null = null;
export function setMediaStatusWriter(writer: (payload: unknown) => void): void {
_setMediaStatusKey = writer;
}
export function setMediaStatusKey(payload: unknown): void {
if (!_setMediaStatusKey) {
logger.warn("Media status writer not wired — skipping status publish");
return;
}
_setMediaStatusKey(payload);
}
export { MEDIA_STATUS_KEY };
export function wireMediaStatusWriter(redisPub: Redis): void {
setMediaStatusWriter((payload) => {
redisPub
.set(MEDIA_STATUS_KEY, JSON.stringify(payload))
.catch((err: unknown) => {
const msg = err instanceof Error ? err.message : String(err);
logger.warn({ error: msg }, "Failed to update media status key");
});
});
}
services/discord-gateway/src/modules/command-handler/video.handler.ts
-145
View File
@@ -1,145 +0,0 @@
import type { Client, Guild, VoiceChannel } from "discord.js-selfbot-v13";
import type { CommandMessage, CommandReply } from "../../shared/index.js";
import {
startStreamWatch,
stopStreamWatch,
} from "../voice-recording/streamWatchReceiver.js";
import type { VoiceController } from "../voice-recording/voiceController.js";
// ---------------------------------------------------------------------------
// VideoHandler — manual screen-share / camera watch (selfbot-viable capture)
// ---------------------------------------------------------------------------
//
// On a selfbot (user token) there is NO automatic detection of other members'
// video (no VOICE_STATE_UPDATE for them, no member list). The only way to
// capture another member's SCREEN SHARE is an operator-initiated STREAM_WATCH
// (gateway op 20, NOT gated on bot-vs-user) — which this handler exposes as a
// backend→gateway Redis command. `streamWatchReceiver` does the DAVE handshake,
// per-burst `.mp4` segmentation (mirroring voice silence split), DB insert and
// Tele upload exactly like the audio path.
//
// Camera-of-others stays impossible on a selfbot (DAVE decrypt is keyed per real
// userId and a selfbot cannot learn others' ids), so this command targets SCREEN
// SHARE. Watching the operator's OWN camera/share is a separate (parameterized)
// follow-up.
export class VideoHandler {
constructor(
private client: Client | null,
private voiceController: VoiceController | null,
) {}
setClient(client: Client): void {
this.client = client;
}
setVoiceController(vc: VoiceController): void {
this.voiceController = vc;
}
/**
* Resolve the active voice channel for a guild from the voice controller.
* Prefers the controller's live connection; falls back to the client cache.
*/
private async resolveChannel(
guildId: string,
requestedChannelId?: string,
): Promise<{ guild: Guild; channel: VoiceChannel } | null> {
const client = this.client;
if (!client) return null;
const guild =
client.guilds.cache.get(guildId) ??
(await client.guilds.fetch(guildId).catch(() => null));
if (!guild) return null;
// If an explicit channelId was given, use it.
const channelId =
requestedChannelId || this.voiceController?.getStatus()?.activeChannelId;
if (channelId) {
const ch = (guild.channels.cache.get(channelId) ??
(await guild.channels
.fetch(channelId)
.catch(() => null))) as VoiceChannel | null;
if (ch && ch.type === "GUILD_VOICE") return { guild, channel: ch };
}
// Fallback: any voice channel the account is currently in.
const voiceCh = Array.from(guild.channels.cache.values()).find(
(c): c is VoiceChannel =>
c.type === "GUILD_VOICE" && c.members?.has(client.user?.id ?? ""),
);
return voiceCh ? { guild, channel: voiceCh } : null;
}
async handleVideoWatch(cmd: CommandMessage): Promise<CommandReply<unknown>> {
if (!this.client) {
return {
id: cmd.id,
success: false,
data: null,
error: "Gateway not initialized",
};
}
const guildId = String(cmd.payload.guildId ?? "");
const userId = String(cmd.payload.userId ?? "");
const channelId = String(cmd.payload.channelId ?? "");
if (!guildId || !userId) {
return {
id: cmd.id,
success: false,
data: null,
error: "guildId and userId are required",
};
}
const resolved = await this.resolveChannel(guildId, channelId || undefined);
if (!resolved) {
return {
id: cmd.id,
success: false,
data: null,
error: "No active voice channel to watch in this guild",
};
}
if (userId === this.client.user?.id) {
// Self-watch is intentionally not enabled by default (see file header).
return {
id: cmd.id,
success: false,
data: null,
error: "Watching the selfbot's own stream is not enabled",
};
}
await startStreamWatch(resolved.channel, userId);
return {
id: cmd.id,
success: true,
data: {
status: "requested",
guildId,
channelId: resolved.channel.id,
userId,
},
};
}
async handleVideoUnwatch(
cmd: CommandMessage,
): Promise<CommandReply<unknown>> {
const guildId = String(cmd.payload.guildId ?? "");
const userId = String(cmd.payload.userId ?? "");
if (!guildId || !userId) {
return {
id: cmd.id,
success: false,
data: null,
error: "guildId and userId are required",
};
}
stopStreamWatch(guildId, userId);
return {
id: cmd.id,
success: true,
data: { status: "stopped", guildId, userId },
};
}
}
services/discord-gateway/src/modules/command-handler/voice.handler.ts
-229
View File
@@ -1,229 +0,0 @@
import type { Client } from "discord.js-selfbot-v13";
import Redis from "ioredis";
import { config } from "../../shared/config/config.js";
import type { CommandMessage, CommandReply } from "../../shared/index.js";
import { createChildLogger } from "../../shared/logger/index.js";
import { discordPlayer } from "../voice-recording/player.js";
import { voiceTransmitter } from "../voice-recording/transmitter.js";
import type { VoiceController } from "../voice-recording/voiceController.js";
// ---------------------------------------------------------------------------
// VoiceHandler
// ---------------------------------------------------------------------------
export class VoiceHandler {
private logger = createChildLogger("voice-handler");
constructor(
private client: Client | null,
private voiceController: VoiceController | null,
) {}
setClient(client: Client): void {
this.client = client;
}
setVoiceController(voiceController: VoiceController): void {
this.voiceController = voiceController;
}
async handleVoiceConnect(
cmd: CommandMessage,
): Promise<CommandReply<unknown>> {
if (!this.client || !this.voiceController) {
return {
id: cmd.id,
success: false,
data: null,
error: "Gateway not initialized",
};
}
const guildId = String(cmd.payload.guildId ?? "");
const channelId = String(cmd.payload.channelId ?? "");
if (!guildId || !channelId) {
return {
id: cmd.id,
success: false,
data: null,
error: "guildId and channelId are required",
};
}
const status = await this.voiceController.connect(guildId, channelId);
return { id: cmd.id, success: true, data: status };
}
async handleVoiceDisconnect(
cmd: CommandMessage,
): Promise<CommandReply<unknown>> {
if (!this.voiceController) {
return {
id: cmd.id,
success: false,
data: null,
error: "Gateway not initialized",
};
}
const status = await this.voiceController.disconnect({
clearPersisted: true,
intentional: true,
});
return { id: cmd.id, success: true, data: status };
}
async handleVoiceDisconnectGuild(
cmd: CommandMessage,
): Promise<CommandReply<unknown>> {
if (!this.voiceController) {
return {
id: cmd.id,
success: false,
data: null,
error: "Gateway not initialized",
};
}
const guildId = String(cmd.payload.guildId ?? "");
if (!guildId) {
return {
id: cmd.id,
success: false,
data: null,
error: "guildId is required",
};
}
await this.voiceController.disconnectGuild(guildId, {
clearPersisted: true,
intentional: true,
});
const status = this.voiceController.getStatus();
return { id: cmd.id, success: true, data: status };
}
async handleVoiceChannels(
cmd: CommandMessage,
): Promise<CommandReply<unknown>> {
if (!this.client) {
return {
id: cmd.id,
success: false,
data: null,
error: "Gateway not initialized",
};
}
const guildId = String(cmd.payload.guildId ?? "");
if (!guildId) {
return {
id: cmd.id,
success: false,
data: null,
error: "guildId is required",
};
}
try {
const guild = await this.client.guilds.fetch(guildId);
const channels = await guild.channels.fetch();
const voiceChannels = channels
.filter((c) => c?.type === "GUILD_VOICE")
.map((c) => ({
id: c.id,
name: c.name,
type: "voice" as const,
// selfbot exposes joinable (permission check) — let FE filter
// channels the account actually may join.
joinable: (c as { joinable?: boolean }).joinable ?? true,
}));
return { id: cmd.id, success: true, data: voiceChannels };
} catch (err) {
const msg = err instanceof Error ? err.message : String(err);
return { id: cmd.id, success: false, data: null, error: msg };
}
}
async handleVoiceTransmitStart(
cmd: CommandMessage,
): Promise<CommandReply<unknown>> {
if (!discordPlayer.isConnected()) {
return {
id: cmd.id,
success: false,
data: null,
error: "Not connected to voice channel",
};
}
// Double-check: verify the voice controller also reports connected
if (this.voiceController) {
const vcStatus = this.voiceController.getStatus();
if (!vcStatus.connected) {
this.logger.warn(
"Player reports connected but voice controller says disconnected — stale state",
);
return {
id: cmd.id,
success: false,
data: null,
error: "Voice channel connection is stale — reconnect first",
};
}
}
try {
// IMPORTANT: transmitter needs its OWN Redis client because it calls
// .subscribe() which converts the connection to subscriber mode. Reusing
// the publish connection from CommandHandler would corrupt it and break
// every command reply + status update.
const transmitRedis = new Redis(config.REDIS_URL);
await voiceTransmitter.start(transmitRedis);
const status = voiceTransmitter.getStatus();
this.logger.info({ status }, "Voice transmit started");
return {
id: cmd.id,
success: true,
data: status,
};
} catch (err) {
const message = err instanceof Error ? err.message : String(err);
this.logger.error({ error: message }, "Failed to start voice transmit");
return {
id: cmd.id,
success: false,
data: null,
error: message,
};
}
}
async handleVoiceTransmitStop(
cmd: CommandMessage,
): Promise<CommandReply<unknown>> {
try {
await voiceTransmitter.stop();
this.logger.info("Voice transmit stopped");
return {
id: cmd.id,
success: true,
data: { status: "stopped" },
};
} catch (err) {
const message = err instanceof Error ? err.message : String(err);
this.logger.error({ error: message }, "Failed to stop voice transmit");
return {
id: cmd.id,
success: false,
data: null,
error: message,
};
}
}
}
services/discord-gateway/src/modules/event-broadcaster/eventBroadcaster.ts
-91
View File
@@ -112,87 +112,6 @@ export class EventBroadcaster {
});
}
async voiceRecordingStarted(data: Record<string, unknown>): Promise<void> {
this.logger.debug({ data }, "Publishing voice_recording_started");
await this.publisher.publish(EventChannels.VOICE_STARTED, {
type: "voice_recording_started",
data,
timestamp: Date.now(),
source: "discord-gateway",
});
}
async voiceRecordingStopped(data: Record<string, unknown>): Promise<void> {
this.logger.debug({ data }, "Publishing voice_recording_stopped");
await this.publisher.publish(EventChannels.VOICE_STOPPED, {
type: "voice_recording_stopped",
data,
timestamp: Date.now(),
source: "discord-gateway",
});
}
async voiceRecordingUploaded(data: Record<string, unknown>): Promise<void> {
this.logger.debug({ data }, "Publishing voice_recording_uploaded");
await this.publisher.publish(EventChannels.VOICE_UPLOADED, {
type: "voice_recording_uploaded",
data,
timestamp: Date.now(),
source: "discord-gateway",
});
}
/**
* Broadcasts PCM audio data for real-time voice streaming
* @param pcmBuffer - Raw PCM audio buffer
* @param userId - Discord user ID
* @param metadata - Optional metadata about the audio chunk
*/
async voicePcmData(
pcmBuffer: Buffer,
userId: string,
metadata?: Record<string, unknown>,
): Promise<void> {
this.logger.debug(
{ userId, pcmSize: pcmBuffer.length },
"Publishing voice_pcm_data",
);
await this.publisher.publish(EventChannels.VOICE_PCM, {
type: "voice_pcm_data",
data: {
userId,
pcm: pcmBuffer.toString("base64"),
metadata,
},
timestamp: Date.now(),
source: "discord-gateway",
});
}
/**
* Broadcasts voice user activity state changes
* @param userId - Discord user ID
* @param data - User state data including username, avatar, and speaking status
*/
async voiceActiveUser(
userId: string,
data: { username: string; avatar: string; speaking: boolean },
): Promise<void> {
this.logger.debug(
{ userId, speaking: data.speaking },
"Publishing voice_active_user",
);
await this.publisher.publish(EventChannels.VOICE_ACTIVE_USER, {
type: "voice_active_user",
data: {
userId,
...data,
},
timestamp: Date.now(),
source: "discord-gateway",
});
}
async reactionAdded(data: Record<string, unknown>): Promise<void> {
this.logger.debug({ data }, "Publishing reaction_added");
await this.publisher.publish(EventChannels.REACTION_ADDED, {
@@ -283,16 +202,6 @@ export class EventBroadcaster {
});
}
async voiceAnalyzed(data: Record<string, unknown>): Promise<void> {
this.logger.debug({ data }, "Publishing voice_analyzed");
await this.publisher.publish(EventChannels.VOICE_ANALYZED, {
type: "voice_analyzed",
data,
timestamp: Date.now(),
source: "discord-gateway",
});
}
async moderationAction(data: Record<string, unknown>): Promise<void> {
this.logger.debug({ data }, "Publishing moderation_action");
await this.publisher.publish(EventChannels.MODERATION_ACTION, {
services/discord-gateway/src/modules/event-broadcaster/eventTypes.ts
-13
View File
@@ -16,12 +16,6 @@ import {
DISCORD_THREAD_CREATED,
DISCORD_THREAD_DELETED,
DISCORD_THREAD_UPDATED,
DISCORD_VOICE_ACTIVE_USER,
DISCORD_VOICE_ANALYZED,
DISCORD_VOICE_PCM,
DISCORD_VOICE_STARTED,
DISCORD_VOICE_STOPPED,
DISCORD_VOICE_UPLOADED,
type DiscordGatewayEvent,
} from "../../shared/index.js";
@@ -34,12 +28,6 @@ export const EventChannels = {
MESSAGE_ANALYZED: DISCORD_MESSAGE_ANALYZED,
ATTACHMENT_CREATED: DISCORD_ATTACHMENT_CREATED,
ATTACHMENT_UPLOADED: DISCORD_ATTACHMENT_UPLOADED,
VOICE_STARTED: DISCORD_VOICE_STARTED,
VOICE_STOPPED: DISCORD_VOICE_STOPPED,
VOICE_UPLOADED: DISCORD_VOICE_UPLOADED,
// Real-time voice streaming channels
VOICE_ACTIVE_USER: DISCORD_VOICE_ACTIVE_USER, // Active speaker state updates
VOICE_PCM: DISCORD_VOICE_PCM, // Live PCM audio data stream
ANALYSIS_QUEUE_STATUS: DISCORD_ANALYSIS_QUEUE_STATUS,
REACTION_ADDED: DISCORD_REACTION_ADDED,
REACTION_REMOVED: DISCORD_REACTION_REMOVED,
@@ -50,7 +38,6 @@ export const EventChannels = {
PRESENCE_UPDATED: DISCORD_PRESENCE_UPDATED,
GUILD_MEMBER_ADDED: DISCORD_GUILD_MEMBER_ADDED,
GUILD_MEMBER_REMOVED: DISCORD_GUILD_MEMBER_REMOVED,
VOICE_ANALYZED: DISCORD_VOICE_ANALYZED,
MODERATION_ACTION: DISCORD_MODERATION_ACTION,
} as const;
services/discord-gateway/src/modules/message-capture/index.ts
-1
View File
@@ -9,7 +9,6 @@ export type {
AIStatus,
AttachmentRecord,
MessageRecord,
VoiceSegmentRecord,
} from "../message-capture/types.js";
export type { TextCaptureTarget } from "./messageCapture.js";
export {
services/discord-gateway/src/modules/message-capture/types.ts
+1 -39
View File
@@ -1,11 +1,8 @@
import type fs from "node:fs";
import type prism from "prism-media";
import type {
AnalysisQueueStatus,
AttachmentRecord,
MessageRecord,
UserMetadata,
VoiceRecordingUploadData,
} from "../../shared/index.js";
// Re-export all shared types for backward compatibility
@@ -29,41 +26,8 @@ export type {
ReviewStatus,
RoleMetadata,
UserMetadata,
VoiceRecordingUploadData,
VoiceSegmentRecord,
} from "../../shared/index.js";
// Types that are LOCAL ONLY (not in shared) — keep here
export interface SegmentState {
index: number;
startTime: number;
endTime: number | null;
filename: string;
jsonFilename: string;
oggStream: prism.opus.OggLogicalBitstream;
out: fs.WriteStream;
}
export interface SegmentMetadata extends UserMetadata {
recordingSessionId: string;
sessionId: string;
sessionStartTime: number;
segmentIndex: number;
segmentMs: number;
startTime: number;
endTime: number;
durationMs: number;
filename: string;
}
export interface PcmBroadcaster {
broadcastPcmToWeb?: (chunk: Buffer, userId: string) => void;
updateActiveUser?: (
userId: string,
data: { username: string; avatar: string; speaking: boolean },
) => void;
}
// Local-only types (not shared across services)
export type ModerationWsEvent =
| { type: "ui_state"; state: unknown }
@@ -73,6 +37,4 @@ export type ModerationWsEvent =
| { type: "message_deleted"; data: { id: string; deleted_at: number } }
| { type: "message_analyzed"; data: MessageRecord }
| { type: "attachment_created"; data: AttachmentRecord }
| { type: "analysis_queue_status"; data: AnalysisQueueStatus }
| { type: "media_state"; state: unknown }
| { type: "voice_recording_uploaded"; data: VoiceRecordingUploadData };
| { type: "analysis_queue_status"; data: AnalysisQueueStatus };
services/discord-gateway/src/modules/voice-pcm-ws/index.ts
-1
View File
@@ -1 +0,0 @@
export { VoicePcmWsClient } from "./voicePcmWsClient.js";
services/discord-gateway/src/modules/voice-pcm-ws/voicePcmWsClient.ts
-151
View File
@@ -1,151 +0,0 @@
/**
* Direct WebSocket client for streaming voice PCM to the backend.
*
* Bypasses Redis pub/sub entirely — gateway connects as a WS client
* to the backend's existing WebSocket server and sends pre-formatted
* binary frames (4-byte FNV-1a userId hash + raw Int16LE PCM).
* The backend forwards these unchanged to frontend clients.
*/
import WebSocket from "ws";
import { createChildLogger } from "@/shared/logger/index";
const logger = createChildLogger("voice-pcm-ws");
/** 32-bit FNV-1a hash — mirrors backend/ws/redis-bridge.ts and frontend useAudioPlayback.ts */
function hashUserId(userId: string): number {
let hash = 0x811c9dc5;
for (let i = 0; i < userId.length; i++) {
hash ^= userId.charCodeAt(i);
hash = Math.imul(hash, 0x01000193);
}
return hash >>> 0;
}
export class VoicePcmWsClient {
private ws: WebSocket | null = null;
private readonly token: string;
private readonly url: string;
private reconnectTimer: ReturnType<typeof setTimeout> | null = null;
private closed = false;
private reconnectMs = 1000;
constructor(url: string, token: string) {
this.url = url;
this.token = token;
}
/** Open connection and begin automatic reconnect loop. */
connect(): void {
if (this.closed) return;
this.doConnect();
}
/**
* Send a PCM chunk to the backend.
*
* Binary frame format (identical to what frontend expects):
* Byte 0–3: FNV-1a 32-bit userId hash (UInt32LE)
* Byte 4+: PCM audio (24kHz mono Int16LE)
*
* Silently drops if not connected — real-time audio, stale
* chunks from a disconnect gap are useless.
*/
sendPcm(userId: string, pcmBuffer: Buffer): void {
if (!this.ws || this.ws.readyState !== WebSocket.OPEN) return;
const hash = hashUserId(userId);
const frame = Buffer.alloc(4 + pcmBuffer.length);
frame.writeUInt32LE(hash, 0);
pcmBuffer.copy(frame, 4);
try {
this.ws.send(frame);
} catch (err) {
logger.warn({ err }, "Failed to send PCM frame");
}
}
/** Graceful close — send close frame, clear reconnect timer. */
async close(): Promise<void> {
this.closed = true;
this.clearReconnectTimer();
if (!this.ws) return;
// Remove listeners so reconnect doesn't fire during close
this.ws.removeAllListeners();
if (this.ws.readyState === WebSocket.OPEN) {
this.ws.close(1000, "shutdown");
}
this.ws = null;
}
// ── internals ────────────────────────────────────────────────────────
private getUrlWithAuth(): string {
const sep = this.url.includes("?") ? "&" : "?";
return `${this.url}${sep}token=${encodeURIComponent(this.token)}`;
}
private doConnect(): void {
if (this.closed) return;
const authUrl = this.getUrlWithAuth();
logger.info({ url: this.url }, "Connecting to backend WebSocket");
try {
this.ws = new WebSocket(authUrl);
} catch (err) {
logger.error({ err }, "Failed to create WebSocket");
this.scheduleReconnect();
return;
}
this.ws.on("open", () => {
logger.info("Voice PCM WS client connected");
this.reconnectMs = 1000; // reset backoff on successful connect
});
this.ws.on("close", (code, reason) => {
logger.warn(
{ code, reason: reason.toString() },
"Voice PCM WS client disconnected",
);
this.ws = null;
this.scheduleReconnect();
});
this.ws.on("error", (err) => {
logger.error({ err: err.message }, "Voice PCM WS client error");
// on("close") fires after on("error") — reconnect handled there
});
}
private scheduleReconnect(): void {
if (this.closed) return;
this.clearReconnectTimer();
// Exponential backoff with jitter: 1s → 2s → 4s → ... → 30s cap
const delay = this.reconnectMs + Math.random() * 1000;
logger.info({ delayMs: Math.round(delay) }, "Scheduling WS reconnect");
this.reconnectTimer = setTimeout(() => {
this.reconnectTimer = null;
if (!this.closed) {
this.doConnect();
}
}, delay);
this.reconnectMs = Math.min(this.reconnectMs * 2, 30_000);
}
private clearReconnectTimer(): void {
if (this.reconnectTimer !== null) {
clearTimeout(this.reconnectTimer);
this.reconnectTimer = null;
}
}
}
services/discord-gateway/src/modules/voice-recording/ffmpegProcess.ts
-78
View File
@@ -1,78 +0,0 @@
import { spawn } from "node:child_process";
import { createChildLogger } from "@/shared/logger/index";
const logger = createChildLogger("ffmpeg-process");
export interface MuxFfmpegArgsOptions {
inputs: string[];
filter: string;
output: string;
codec: string;
audioFrequency?: number;
audioChannels?: number;
}
/**
* Builds ffmpeg argument array for muxing audio clips.
*/
export function buildMuxFfmpegArgs(options: MuxFfmpegArgsOptions): string[] {
const args: string[] = ["-y"];
for (const input of options.inputs) {
args.push("-i", input);
}
args.push("-filter_complex", options.filter);
args.push("-map", "[out]");
args.push("-codec:a", options.codec);
if (options.audioFrequency !== undefined) {
args.push("-ar", String(options.audioFrequency));
}
if (options.audioChannels !== undefined) {
args.push("-ac", String(options.audioChannels));
}
args.push(options.output);
return args;
}
/**
* Runs ffmpeg with the given arguments.
* Resolves on successful (code 0) exit, rejects on error or non-zero exit.
*/
export function runFfmpeg(args: string[]): Promise<void> {
return new Promise((resolve, reject) => {
logger.debug({ args }, "Starting ffmpeg");
const proc = spawn("ffmpeg", args, {
stdio: ["ignore", "pipe", "pipe"],
});
let stderrBuf = "";
proc.stderr?.on("data", (chunk: Buffer) => {
stderrBuf += chunk.toString("utf8");
});
proc.on("close", (code) => {
if (code === 0) {
logger.debug("ffmpeg completed successfully");
resolve();
} else {
const detail = stderrBuf.trim().slice(0, 2000);
logger.warn(
{ exitCode: code, stderr: detail },
"ffmpeg exited with non-zero code",
);
reject(new Error(`ffmpeg exited with code ${code}: ${detail}`));
}
});
proc.on("error", (err) => {
logger.error({ error: err.message }, "ffmpeg process error");
reject(err);
});
});
}
services/discord-gateway/src/modules/voice-recording/mediaSource.ts
-573
View File
@@ -1,573 +0,0 @@
import { type ChildProcess, spawn } from "node:child_process";
import { chmodSync, existsSync, readFileSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { PassThrough, type Readable } from "node:stream";
import { StreamType } from "@discordjs/voice";
import { createChildLogger } from "@/shared/logger/index";
const logger = createChildLogger("media-source");
// ---------------------------------------------------------------------------
// Types
// ---------------------------------------------------------------------------
export interface MediaInfo {
title: string;
duration: number;
thumbnail?: string;
}
export interface MediaSourceResolution {
stream: Readable;
type: StreamType;
title?: string;
duration?: number;
info: MediaInfo;
}
export interface ResolveOptions {
/** Timeout in milliseconds for the yt-dlp process. */
timeout?: number;
/**
* yt-dlp format string override (e.g. "bestaudio[ext=m4a]").
* Defaults to "bestaudio[ext=m4a]/bestaudio/best".
*/
quality?: string;
}
export interface TranscodeResult {
stream: Readable;
/** Kill the ffmpeg child (used on stop/skip). */
cleanup: () => void;
}
/**
* Re-encode a source stream to high-quality OggOpus (48kHz stereo, 192kbps).
*
* Discord voice downmixes whatever we feed it to the channel's bitrate, so the
* best we can do is hand it a clean 48kHz stereo Opus stream instead of the
* raw source (which may be mono, low-bitrate, or a non-Opus container). The
* volume is baked into the encode with `-af volume=` so the player does not
* need inlineVolume re-encoding (double lossy encode).
*/
export function transcodeToHighQualityOgg(
input: Readable,
volume: number,
): TranscodeResult {
const proc = spawn(
"ffmpeg",
[
"-hide_banner",
"-loglevel",
"error",
"-i",
"pipe:0",
"-vn",
"-ac",
"2",
"-ar",
"48000",
"-c:a",
"libopus",
"-b:a",
"192k",
"-af",
`volume=${volume}`,
"-f",
"ogg",
"pipe:1",
],
{ stdio: ["pipe", "pipe", "ignore"] },
);
input.pipe(proc.stdin);
// ffmpeg teardown closes stdin while the upstream source may still write —
// swallow EPIPE / destroyed-stream errors so they don't crash the gateway.
proc.stdin.on("error", (err: NodeJS.ErrnoException) => {
if (
err.code === "EPIPE" ||
err.code === "ERR_STREAM_DESTROYED" ||
err.code === "ERR_STREAM_WRITE_AFTER_END"
) {
logger.debug(
{ code: err.code },
"Transcode stdin closed during teardown",
);
} else {
logger.error({ error: err.message }, "Transcode stdin error");
}
});
activeProcesses.add(proc);
const cleanup = () => {
activeProcesses.delete(proc);
if (proc.exitCode === null) {
proc.kill("SIGKILL");
}
};
proc.once("exit", () => activeProcesses.delete(proc));
// If ffmpeg fails, surface the error to the consumer stream so the
// AudioPlayer's error handler can advance the queue.
const output = proc.stdout;
output.on("error", () => cleanup());
return { stream: output, cleanup };
}
// ---------------------------------------------------------------------------
// Internal state
// ---------------------------------------------------------------------------
/** Tracks all spawned yt-dlp child processes for shutdown cleanup. */
const activeProcesses: Set<ChildProcess> = new Set();
// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------
function parseSeconds(value: string): number {
const n = Number.parseFloat(value);
return Number.isFinite(n) ? n : 0;
}
/**
* Read the first two newline-delimited lines from a Readable stdout stream.
*
* yt-dlp with `--print before_dl:title --print before_dl:duration` outputs:
* line 1: video title
* line 2: duration in seconds (float)
* rest: raw binary audio data
*
* Returns the parsed header and a new Readable that contains all remaining
* data (the audio stream).
*/
const MAX_HEADER_BUFFER = 65536; // 64KB safety limit for metadata headers
/**
* Read the title + duration header lines from yt-dlp's STDERR.
*
* When yt-dlp streams media to stdout (`-o -`) it redirects its `--print`
* output to STDERR so the media stream on stdout stays clean. The first two
* meaningful lines on stderr are then the title and (before_dl) duration.
*
* Blank lines and `[...]` info prefixes are skipped. An `ERROR:` line is
* reported via `onError` so a failing download surfaces as a resolution error
* instead of a silent empty stream.
*/
function readStderrHeader(
stderr: Readable,
onError: (message: string) => void,
maxBufferSize: number = MAX_HEADER_BUFFER,
): Promise<{ title: string; duration: number }> {
return new Promise((resolve) => {
let buffer = "";
let title = "";
let duration = 0;
let done = false;
const finish = () => {
if (done) return;
done = true;
stderr.removeListener("data", onData);
resolve({ title, duration });
};
const onData = (chunk: Buffer) => {
if (done) return;
buffer += chunk.toString("utf8");
if (buffer.length > maxBufferSize) {
finish();
return;
}
while (!done) {
const nl = buffer.indexOf("\n");
if (nl === -1) break; // need more data
const line = buffer.slice(0, nl).trim();
buffer = buffer.slice(nl + 1);
if (line.length === 0) continue; // blank line
if (line.startsWith("[")) continue; // "[info] ..." — not a header
if (line.startsWith("ERROR")) {
onError(line);
finish();
return;
}
if (!title) {
title = line;
} else {
duration = parseSeconds(line);
finish();
return;
}
}
};
stderr.on("data", onData);
stderr.on("end", finish);
});
}
function buildNotInstalledError(): Error {
return new Error(
"yt-dlp is not installed or not found in PATH. " +
'Run "pnpm run install:yt-dlp" to install it.',
);
}
/**
* Build the yt-dlp --cookies args. YouTube blocks anonymous embeds with a
* "Sign in to confirm you're not a bot" 403 unless yt-dlp is given a logged-
* in account's cookies. The path is configurable via GMW_YT_COOKIES_PATH
* (default: the BWS-provided file the deploy writes to /etc/.../ytcookies.txt).
* If the file doesn't exist we pass nothing and fall back to anon (YouTube
* may 403 — playback will fail gracefully, not crash).
*/
var _cachedCookiePath: string | null = null;
function buildCookieArgs(): string[] {
// Single source of truth: BWS injects the account cookies via env
// (gmw_yt_downloader_cookies → GMW_YT_DOWNLOADER_COOKIES by bws-exec).
// We materialize them to a temp Netscape file because yt-dlp --cookies
// only accepts a file path, not stdin, and multiline env values are not
// reliable to pass directly on the spawn argv. Falls back to the on-disk
// file at GMW_YT_COOKIES_PATH (or /etc/gmw-discord-gateway/ytcookies.txt)
// which the Nix deploy writes from BWS once at start.
if (_cachedCookiePath) return ["--cookies", _cachedCookiePath];
const envCookies = process.env.GMW_YT_DOWNLOADER_COOKIES?.trim();
if (envCookies?.includes("LOGIN_INFO")) {
const fdPath = join(tmpdir(), `gmw-ytcookies.${process.pid}.txt`);
writeFileSync(fdPath, envCookies);
try {
chmodSync(fdPath, 0o600);
} catch {
/* best-effort */
}
_cachedCookiePath = fdPath;
logger.info(
{ cookiePath: fdPath, source: "GMW_YT_DOWNLOADER_COOKIES env" },
"Using YouTube cookies (from BWS env)",
);
return ["--cookies", fdPath];
}
const cookiePath =
process.env.GMW_YT_COOKIES_PATH ?? "/etc/gmw-discord-gateway/ytcookies.txt";
try {
if (cookiePath && existsSync(cookiePath)) {
// Never hand the ORIGINAL system file to yt-dlp: recent yt-dlp rewrites
// the cookie file on close (`--cookies` implies write-back). The system
// file is owned by another user (root/deploy) and the service user
// cannot write it → PermissionError → yt-dlp exits 1 → playback
// fails for every attempt. Copy to a per-run temp file (like the env
// branch above) so write-back lands somewhere we own; if the original
// is not readable we fall back to anonymous (YouTube may 403 → the
// screen-share controller retries via Invidious mirrors without auth).
let cookieContents: string;
try {
cookieContents = readFileSync(cookiePath, "utf8");
} catch {
logger.warn(
{ cookiePath },
"Cookie file not readable; continuing without cookies (anon)",
);
return [];
}
const fdPath = join(tmpdir(), `gmw-ytcookies.${process.pid}.txt`);
writeFileSync(fdPath, cookieContents);
try {
chmodSync(fdPath, 0o600);
} catch {
/* best-effort */
}
_cachedCookiePath = fdPath;
logger.info(
{ cookiePath: fdPath, source: "on-disk file (copied)" },
"Using YouTube cookies for yt-dlp",
);
return ["--cookies", fdPath];
}
} catch {
/* ignore — fallback to anon */
}
logger.warn(
"No YouTube cookies available; yt-dlp will use anonymous (YouTube may 403)",
);
return [];
}
// ---------------------------------------------------------------------------
// Public API
// ---------------------------------------------------------------------------
/**
* Resolve a media URL (YouTube, Spotify, etc.) to a playable audio stream.
*
* Spawns `yt-dlp` with `-o -` so the raw audio bytes stream on stdout. Since
* stdout is the media sink, yt-dlp emits its `--print before_dl:title` /
* `before_dl:duration` header lines on STDERR — the title + duration are read
* from there and the stdout media stream is returned untouched.
*
* The returned stream uses `StreamType.Arbitrary` — suitable for
* `DiscordPlayer.playStream()` with `inputType: StreamType.Arbitrary`.
*
* @throws If yt-dlp is not installed or the process exits with a non-zero code
* before the metadata headers have been parsed.
*/
export function resolveMediaUrl(
url: string,
options?: ResolveOptions,
): Promise<MediaSourceResolution> {
return new Promise<MediaSourceResolution>((resolve, reject) => {
// Fallback chain: YouTube gets progressive m4a audio; direct-file hosts
// (upload mirror / Telegram files) expose a single generic format with an
// arbitrary ID — `bestaudio` alone fails there ("Requested format is not
// available"), so fall back to `bestaudio` then plain `best`.
const format = options?.quality ?? "bestaudio[ext=m4a]/bestaudio/best";
const cookieArgs = buildCookieArgs();
const args = [
"-f",
format,
"-o",
"-",
"--no-progress",
"--no-warnings",
...cookieArgs,
"--print",
"before_dl:title",
"--print",
"before_dl:duration",
url,
];
logger.info({ url }, "Spawning yt-dlp for media resolution");
const proc = spawn("yt-dlp", args, {
stdio: ["ignore", "pipe", "pipe"],
});
activeProcesses.add(proc);
// With `-o -` yt-dlp streams the raw audio on stdout and moves its
// `--print` headers to stderr — pipe stdout immediately so the child
// never blocks on a full pipe while we wait for the headers on stderr.
const mediaStream = new PassThrough();
// Teardown (player stop / ffmpeg exit) destroys this stream while
// yt-dlp may still push bytes — without a listener an EPIPE /
// ERR_STREAM_DESTROYED surfaces as an uncaughtException.
mediaStream.on("error", (err: NodeJS.ErrnoException) => {
if (
err.code === "EPIPE" ||
err.code === "ERR_STREAM_DESTROYED" ||
err.code === "ERR_STREAM_WRITE_AFTER_END"
) {
logger.debug({ code: err.code }, "Media stream closed during teardown");
} else {
logger.error({ error: err.message }, "Media stream error");
}
});
proc.stdout.pipe(mediaStream);
let stderrBuf = "";
let resolved = false;
// -- helpers -----------------------------------------------------------
const failOnce = (err: Error) => {
if (resolved) return;
resolved = true;
activeProcesses.delete(proc);
mediaStream.destroy();
reject(err);
};
const resolveOnce = (info: MediaInfo) => {
if (resolved) return;
resolved = true;
activeProcesses.delete(proc);
resolve({
stream: mediaStream,
type: StreamType.Arbitrary,
title: info.title,
duration: info.duration,
info,
});
};
// -- spawn error (ENOENT etc.) ----------------------------------------
proc.on("error", (err: NodeJS.ErrnoException) => {
if (err.code === "ENOENT") {
failOnce(buildNotInstalledError());
} else {
failOnce(new Error(`yt-dlp failed to start: ${err.message}`));
}
});
// -- stderr: title + duration headers ---------------------------------
// Capture raw stderr too, for the exit-diagnostics in the close handler.
const _MAX_STDERR = 4096;
if (proc.stderr) {
proc.stderr.on("data", (chunk: Buffer) => {
if (stderrBuf.length < _MAX_STDERR) {
stderrBuf += chunk
.toString("utf8")
.slice(0, _MAX_STDERR - stderrBuf.length);
}
});
}
readStderrHeader(proc.stderr, (message) => {
failOnce(new Error(message));
})
.then(({ title, duration }) => {
resolveOnce({ title: title || url, duration });
})
.catch((err: Error) => {
failOnce(err);
});
// -- process exit (non-zero means failure) -----------------------------
proc.on("close", (code, signal) => {
activeProcesses.delete(proc);
if (resolved) return;
if (code !== null && code !== 0) {
const detail = stderrBuf.trim() ? `: ${stderrBuf.trim()}` : "";
failOnce(new Error(`yt-dlp exited with code ${code}${detail}`));
} else if (signal) {
failOnce(new Error(`yt-dlp was killed by signal ${signal}`));
} else {
// Exited cleanly but the header lines never surfaced (e.g. a direct
// file URL with no duration) — keep the media stream alive anyway.
resolveOnce({ title: url, duration: 0 });
}
});
// -- optional timeout --------------------------------------------------
if (options?.timeout && options.timeout > 0) {
const timer = setTimeout(() => {
if (resolved) return;
logger.warn({ url, timeout: options.timeout }, "yt-dlp timed out");
proc.kill("SIGTERM");
failOnce(new Error(`yt-dlp timed out after ${options.timeout}ms`));
}, options.timeout);
proc.once("close", () => clearTimeout(timer));
}
});
}
/**
* Extract metadata (title, duration, thumbnail) from a media URL
* without downloading the audio stream.
*
* Uses `yt-dlp --dump-json` and parses the JSON output.
*
* @throws If yt-dlp is not installed or the process exits with a non-zero
* code.
*/
export async function extractMediaInfo(url: string): Promise<MediaInfo> {
return new Promise<MediaInfo>((resolve, reject) => {
const args = ["--dump-json", "--no-warnings", url];
logger.debug({ url }, "Spawning yt-dlp for metadata extraction");
const proc = spawn("yt-dlp", args, {
stdio: ["pipe", "pipe", "pipe"],
});
activeProcesses.add(proc);
let stdoutBuf = "";
let stderrBuf = "";
const MAX_STDERR = 4096;
const MAX_STDOUT = 1_048_576; // 1MB safety limit
if (proc.stdout) {
proc.stdout.on("data", (chunk: Buffer) => {
if (stdoutBuf.length < MAX_STDOUT) {
stdoutBuf += chunk
.toString("utf8")
.slice(0, MAX_STDOUT - stdoutBuf.length);
}
});
}
if (proc.stderr) {
proc.stderr.on("data", (chunk: Buffer) => {
if (stderrBuf.length < MAX_STDERR) {
stderrBuf += chunk
.toString("utf8")
.slice(0, MAX_STDERR - stderrBuf.length);
}
});
}
proc.on("error", (err: NodeJS.ErrnoException) => {
activeProcesses.delete(proc);
if (err.code === "ENOENT") {
reject(buildNotInstalledError());
} else {
reject(new Error(`yt-dlp failed to start: ${err.message}`));
}
});
proc.on("close", (code) => {
activeProcesses.delete(proc);
if (code !== 0) {
const detail = stderrBuf.trim() ? `: ${stderrBuf.trim()}` : "";
reject(
new Error(
`yt-dlp metadata extraction exited with code ${code}${detail}`,
),
);
return;
}
try {
const raw = JSON.parse(stdoutBuf.trim()) as Record<string, unknown>;
resolve({
title: String(raw.title ?? url),
duration: typeof raw.duration === "number" ? raw.duration : 0,
thumbnail: String(raw.thumbnail ?? "") || undefined,
});
} catch (parseErr) {
reject(
new Error(
`Failed to parse yt-dlp JSON output: ${(parseErr as Error).message}`,
),
);
}
});
});
}
/**
* Kill all active yt-dlp / screen-share merge ffmpeg child processes.
*
* Call during graceful shutdown to ensure no orphan processes remain.
*/
export function cleanup(): void {
if (activeProcesses.size === 0) return;
logger.info(
{ count: activeProcesses.size },
"Killing active yt-dlp processes",
);
for (const proc of activeProcesses) {
try {
proc.kill("SIGTERM");
} catch {
// Process may already be dead — ignore
}
}
activeProcesses.clear();
}
services/discord-gateway/src/modules/voice-recording/mediaTypes.ts
-76
View File
@@ -1,76 +0,0 @@
import type { Readable } from "node:stream";
import type { StreamType } from "@discordjs/voice";
export type MediaMode = "music" | "screenshare";
export type MediaSourceKind =
| "url"
| "local"
| "youtube"
| "spotify"
| "search";
export type MediaQueueItemStatus = "queued" | "playing" | "failed";
export interface ResolvedMediaSource {
source: string;
title: string;
kind: MediaSourceKind;
}
export interface MediaQueueItem extends ResolvedMediaSource {
id: string;
mode: MediaMode;
requestedBy: string;
addedAt: number;
status: MediaQueueItemStatus;
/** Duration in seconds, if known. */
duration?: number;
/** Thumbnail/preview URL, if available. */
thumbnailUrl?: string;
}
export interface MediaState {
playing: boolean;
activeMode: MediaMode | null;
musicVolume: number;
loop: boolean;
current: MediaQueueItem | null;
queue: MediaQueueItem[];
}
export interface QueueMediaOptions {
mode?: MediaMode;
requestedBy?: string;
}
export interface MusicPlayback {
done: Promise<void>;
stop(): void;
}
export interface MusicPlayer {
play(source: ResolvedMediaSource): MusicPlayback;
}
export type DiscordPlayerOwner = "none" | "browser-bridge" | "music";
export interface DiscordPlayOptions {
inputType?: StreamType;
/** Enable volume control via resource.volume (required for setMusicVolume to work). */
inlineVolume?: boolean;
volume?: number;
}
export interface DiscordAudioPlayer {
getOwner(): DiscordPlayerOwner;
isConnected(): boolean;
playStream(
stream: Readable,
owner: DiscordPlayerOwner,
options?: DiscordPlayOptions,
): void;
pause(owner?: DiscordPlayerOwner): void;
unpause(owner?: DiscordPlayerOwner): boolean;
stop(owner?: DiscordPlayerOwner): void;
getMusicVolume(): number;
setMusicVolume(volume: number): void;
}
services/discord-gateway/src/modules/voice-recording/muxer.ts
-193
View File
@@ -1,193 +0,0 @@
import { eq } from "drizzle-orm";
import type { NodePgDatabase } from "drizzle-orm/node-postgres";
import { createChildLogger } from "@/shared/logger/index";
import { getDatabase } from "../../shared/database/drizzle.js";
import type * as schema from "../../shared/database/schema.js";
import { muxerJobsTable } from "../../shared/database/schema.js";
import { buildMuxFfmpegArgs, runFfmpeg } from "./ffmpegProcess.js";
const logger = createChildLogger("muxer");
// ─── Types ─────────────────────────────────────────────────────────────────────────────
export interface MuxerJobData {
inputs: string[];
output: string;
guildId: string;
channelId: string;
sessionId: string;
}
// ─── State ───────────────────────────────────────────────────────────────
let pollTimer: ReturnType<typeof setInterval> | null = null;
// ─── Public API ──────────────────────────────────────────────────────────
/**
* Enqueue a muxer job that merges multiple OGG files into one.
*/
export async function enqueueMuxerJob(data: MuxerJobData): Promise<void> {
const db = getDatabase() as unknown as NodePgDatabase<typeof schema>;
const id = `${data.sessionId}-${Date.now()}`;
try {
await db.insert(muxerJobsTable).values({
id,
data: JSON.stringify(data),
status: "pending",
attempts: 0,
maxAttempts: 3,
createdAt: Date.now(),
updatedAt: Date.now(),
});
logger.info({ id, sessionId: data.sessionId }, "Muxer job enqueued");
} catch (error) {
logger.error(
{ id, error: error instanceof Error ? error.message : String(error) },
"Failed to enqueue muxer job",
);
}
}
/**
* Start the background worker that polls for pending muxer jobs.
*/
export function startMuxerWorker(): void {
if (pollTimer) return;
logger.info("Starting muxer worker (interval: 10s)");
pollTimer = setInterval(() => {
processNextJobs().catch((err: unknown) => {
logger.error({ error: String(err) }, "Muxer worker tick failed");
});
}, 10_000);
}
/**
* Stop the background worker.
*/
export function stopMuxerWorker(): void {
if (!pollTimer) return;
clearInterval(pollTimer);
pollTimer = null;
logger.info("Muxer worker stopped");
}
// ─── Internal ────────────────────────────────────────────────────────────
async function processNextJobs(): Promise<void> {
const db = getDatabase() as unknown as NodePgDatabase<typeof schema>;
try {
const jobs = await db
.select()
.from(muxerJobsTable)
.where(eq(muxerJobsTable.status, "pending"))
.limit(5);
if (jobs.length === 0) return;
const results = await Promise.allSettled(
jobs.map((job) => processJob(db, job)),
);
for (let i = 0; i < results.length; i++) {
const result = results[i];
if (result.status === "rejected") {
logger.error(
{ jobId: jobs[i].id, error: result.reason },
"Muxer job failed",
);
}
}
} catch (error) {
logger.error(
{ error: error instanceof Error ? error.message : String(error) },
"Failed to fetch pending muxer jobs",
);
}
}
async function processJob(
db: NodePgDatabase<typeof schema>,
job: typeof muxerJobsTable.$inferSelect,
): Promise<void> {
// Mark as processing
await db
.update(muxerJobsTable)
.set({ status: "processing", updatedAt: Date.now() })
.where(eq(muxerJobsTable.id, job.id));
try {
const data = JSON.parse(job.data) as MuxerJobData;
if (!data.inputs || data.inputs.length < 2) {
throw new Error(
`Muxer job ${job.id} needs at least 2 inputs, got ${data.inputs?.length ?? 0}`,
);
}
logger.info(
{ jobId: job.id, inputs: data.inputs.length, output: data.output },
"Processing muxer job",
);
// Sequential concat of all OGG segments (amix = simultaneous mix, wrong for this)
const inputLabels = data.inputs.map((_, i) => `[${i}:a:0]`);
const n = data.inputs.length;
const filterComplex = `${inputLabels.join("")}concat=n=${n}:v=0:a=1[out]`;
const args = buildMuxFfmpegArgs({
inputs: data.inputs,
filter: filterComplex,
output: data.output,
codec: "libmp3lame",
audioFrequency: 48000,
audioChannels: 2,
});
await runFfmpeg(args);
// Mark as completed
await db
.update(muxerJobsTable)
.set({ status: "completed", updatedAt: Date.now() })
.where(eq(muxerJobsTable.id, job.id));
logger.info({ jobId: job.id, output: data.output }, "Muxer job completed");
} catch (error) {
const errMsg = error instanceof Error ? error.message : String(error);
const newAttempts = job.attempts + 1;
if (newAttempts >= job.maxAttempts) {
await db
.update(muxerJobsTable)
.set({
status: "failed",
error: errMsg,
attempts: newAttempts,
updatedAt: Date.now(),
})
.where(eq(muxerJobsTable.id, job.id));
logger.error(
{ jobId: job.id, error: errMsg },
"Muxer job failed permanently",
);
} else {
await db
.update(muxerJobsTable)
.set({
status: "pending",
error: errMsg,
attempts: newAttempts,
updatedAt: Date.now(),
})
.where(eq(muxerJobsTable.id, job.id));
logger.warn(
{ jobId: job.id, error: errMsg, attempt: newAttempts },
"Muxer job will be retried",
);
}
}
}
services/discord-gateway/src/modules/voice-recording/packetFilter.ts
-45
View File
@@ -1,45 +0,0 @@
import { Transform, type TransformCallback } from "node:stream";
import { createChildLogger } from "@/shared/logger/index";
const _logger = createChildLogger("packet-filter");
/**
* Transform stream to filter out audio packets that are too small.
* Packets that are too small are likely to fail decryption by Discord.
*/
export class PacketFilter extends Transform {
private minPacketSize: number;
private filteredCount: number = 0;
private totalCount: number = 0;
constructor(minPacketSize: number = 10) {
super();
this.minPacketSize = minPacketSize;
}
_transform(
chunk: Buffer,
_encoding: string,
callback: TransformCallback,
): void {
this.totalCount++;
// Filter out undersized packets
if (chunk.length >= this.minPacketSize) {
this.push(chunk);
} else {
this.filteredCount++;
}
callback();
}
/** Returns the number of packets filtered out and processed. */
getStats(): { filtered: number; total: number } {
return { filtered: this.filteredCount, total: this.totalCount };
}
_flush(callback: TransformCallback): void {
callback();
}
}
services/discord-gateway/src/modules/voice-recording/player.ts
-161
View File
@@ -1,161 +0,0 @@
import type { Readable } from "node:stream";
import {
type AudioPlayer,
AudioPlayerStatus,
type AudioResource,
createAudioPlayer,
createAudioResource,
StreamType,
type VoiceConnection,
} from "@discordjs/voice";
import { createChildLogger } from "@/shared/logger/index";
import type { DiscordPlayerOwner, DiscordPlayOptions } from "./mediaTypes.js";
const logger = createChildLogger("player");
export class DiscordPlayer {
private player: AudioPlayer;
private connection: VoiceConnection | null = null;
private owner: DiscordPlayerOwner = "none";
private resource: AudioResource | null = null;
private musicVolume = 0.3;
private idleCallback: (() => void) | null = null;
/** Set before manual stop() calls to distinguish from natural track end. */
private manualStop = false;
constructor() {
this.player = createAudioPlayer();
this.player.on(AudioPlayerStatus.Playing, () => {
logger.info("Audio player is now playing!");
});
this.player.on(AudioPlayerStatus.Idle, () => {
// manualStop guards against both sync & async idle emission from stop()
if (this.manualStop || this.owner === "none") {
this.manualStop = false;
return;
}
// Natural track end — queue auto-advance
logger.info("Audio player finished — advancing queue");
this.owner = "none";
this.resource = null;
this.idleCallback?.();
});
this.player.on("error", (error) => {
logger.error({ error: error.message }, "Audio player error");
this.owner = "none";
this.resource = null;
});
}
/** Register a callback fired when the player goes idle after a natural track end. */
public onIdle(cb: () => void): void {
this.idleCallback = cb;
}
public setConnection(connection: VoiceConnection) {
this.connection = connection;
this.connection.subscribe(this.player);
}
public getOwner(): DiscordPlayerOwner {
return this.owner;
}
public isConnected(): boolean {
return this.connection !== null;
}
public playStream(
stream: Readable,
owner: DiscordPlayerOwner,
options: DiscordPlayOptions = {},
) {
if (owner === "none") {
throw new Error("Discord audio player owner is required");
}
this.assertOwnerAvailable(owner);
const resource = createAudioResource(stream, {
inputType: options.inputType ?? StreamType.OggOpus,
// Default to true so setMusicVolume/setResourceVolume works
inlineVolume: options.inlineVolume ?? true,
});
if (this.owner === owner) {
this.manualStop = true;
this.player.stop();
}
this.resource = resource;
this.owner = owner;
if (owner === "music") {
const nextVolume =
options.volume !== undefined
? this.normalizeVolume(options.volume)
: this.musicVolume;
this.musicVolume = nextVolume;
this.setResourceVolume(nextVolume);
}
this.player.play(resource);
this.unpause(owner);
this.connection?.subscribe(this.player);
}
public getStatus(): AudioPlayerStatus {
return this.player.state.status;
}
public pause(owner?: DiscordPlayerOwner) {
if (!this.canControl(owner)) return;
this.player.pause(true);
}
public unpause(owner?: DiscordPlayerOwner): boolean {
if (!this.canControl(owner)) return false;
return this.player.unpause();
}
public stop(owner?: DiscordPlayerOwner) {
if (!this.canControl(owner)) return;
this.manualStop = true;
this.player.stop();
this.owner = "none";
this.resource = null;
}
public getMusicVolume(): number {
return this.musicVolume;
}
public setMusicVolume(volume: number): void {
const nextVolume = this.normalizeVolume(volume);
this.musicVolume = nextVolume;
if (this.owner === "music") {
this.setResourceVolume(nextVolume);
}
}
private assertOwnerAvailable(owner: DiscordPlayerOwner): void {
if (this.owner !== "none" && this.owner !== owner) {
throw new Error(`Discord audio player is owned by ${this.owner}`);
}
}
private canControl(owner?: DiscordPlayerOwner): boolean {
return !owner || this.owner === "none" || this.owner === owner;
}
private normalizeVolume(volume: number): number {
if (!Number.isFinite(volume)) return this.musicVolume;
return Math.max(0, Math.min(1, volume));
}
private setResourceVolume(volume: number): void {
if (!this.resource?.volume) return;
this.resource.volume.setVolume(volume);
}
}
export const discordPlayer = new DiscordPlayer();
services/discord-gateway/src/modules/voice-recording/recorder.ts
-377
View File
@@ -1,377 +0,0 @@
import { promises as fsPromises } from "node:fs";
import {
type DiscordGatewayAdapterCreator,
entersState,
getVoiceConnection,
joinVoiceChannel,
type VoiceConnection,
VoiceConnectionStatus,
} from "@discordjs/voice";
import type {
Client,
Guild,
VoiceChannel,
VoiceState,
} from "discord.js-selfbot-v13";
import { createChildLogger } from "@/shared/logger/index";
import { retryWithBackoff } from "@/shared/utils/index";
import { config } from "../../shared/config/config.js";
import type { EventBroadcaster } from "../event-broadcaster/eventBroadcaster.js";
import type { VoicePcmWsClient } from "../voice-pcm-ws/index.js";
import {
createRecordingSession,
type RecordingSession,
} from "./recorder/sessionRecording.js";
import { createSpeakingHandler } from "./recorder/speakingHandler.js";
import { hookScreenShareAudio } from "./screenShareAudio.js";
import { hookVideoReceiver } from "./videoReceiver.js";
import { trackChannel, untrackChannel } from "./videoRecorder.js";
const logger = createChildLogger("recorder");
let _eventBroadcaster: EventBroadcaster | undefined;
/** @internal Export for uploader.ts to broadcast voice_recording_uploaded events */
export { _eventBroadcaster };
export function setEventBroadcaster(broadcaster: EventBroadcaster | undefined) {
_eventBroadcaster = broadcaster;
}
/**
* Force the bot (self member) to be unmuted + undeafened at the SERVER level.
*
* Why: a server-muted / server-deafened bot cannot receive/record other
* members' audio properly (and definitely cannot receive their video/screen
* share). The bot's own voice state may be server-muted/deafened by an admin
* or by a prior state. This issues a REST guild-members PATCH with
* `mute:false, deaf:false`.
*
* Requires the `MUTE_MEMBERS` + `DEAFEN_MEMBERS` permissions on the bot
* (user has granted them). Failures are logged, never fatal — an unmute
* failure must not break the voice join/recording.
*
* Exported so the video-receive path (videoRecorder.ts → streamWatch) can
* re-assert it right before requesting STREAM_WATCH — a server-deafened bot
* is NOT sent the streamer's audiovisual RTP, which is why no video arrives
* even when the DAVE watch reaches Ready. Server-deaf can silently revert on
* reconnect/restart, so this is re-invoked on every video watch attempt.
*/
export async function forceSelfServerUnmuteUndeafen(
client: Client,
guild: Guild,
): Promise<void> {
try {
const selfId = client.user?.id;
if (!selfId) return;
await guild.members.edit(selfId, { mute: false, deaf: false });
// Verify the PATCH actually landed — read back the live member voice state.
const fresh = guild.members.cache.get(selfId);
const stillDeaf = fresh?.voice?.deaf ?? false;
logger.info(
{ guildId: guild.id, stillDeaf },
stillDeaf
? "Forced bot server unmute + undeafen (but guild reports still deaf — will retry on next video)"
: "Forced bot server unmute + undeafen",
);
} catch (err) {
logger.warn(
{
guildId: guild.id,
err: err instanceof Error ? err.message : String(err),
},
"Could not force server unmute/undeafen (permissions not granted?)",
);
}
}
/**
* Attach an IMMEDIATE self-undeafen / self-unmute guard on `voiceStateUpdate`.
*
* Previous behaviour only re-asserted unmute/undeafen on video-watch attempts
* and after voice reconnects — so when an admin server-muted or
* server-deafened the bot the bot stayed muted/deafened until the *next* video
* watch or reconnect cycle (could be minutes or never if no streamer came on).
*
* This listener fires on EVERY guild voice-state change. When it detects that
* the bot's OWN voice state transitioned to serverDeaf||serverMute === true,
* it immediately re-issues `mute:false, deaf:false` so the bot is unmuted/
* undeafened instantly (well within Discord's rate limit window for the bot's
* own member). Idempotent and best-effort — failures are logged, never fatal.
*
* Idempotent: safe to call multiple times; the listener is attached once.
*/
let _selfVoiceGuardAttached = false;
export function registerSelfVoiceStateGuard(client: Client): void {
if (_selfVoiceGuardAttached) return;
_selfVoiceGuardAttached = true;
client.on(
"voiceStateUpdate",
(oldState: VoiceState, newState: VoiceState) => {
void handleSelfVoiceStateUpdate(client, oldState, newState);
},
);
}
async function handleSelfVoiceStateUpdate(
client: Client,
oldState: VoiceState,
newState: VoiceState,
): Promise<void> {
// Only react to the bot's own voice state transitions.
const selfId = client.user?.id;
if (!selfId || newState.id !== selfId) return;
const justServerMuted = !oldState.serverMute && newState.serverMute === true;
const justServerDeafed = !oldState.serverDeaf && newState.serverDeaf === true;
if (!justServerMuted && !justServerDeafed) return;
const guild = newState.guild;
if (!guild) return;
logger.info(
{
guildId: guild.id,
serverMuted: justServerMuted,
serverDeafed: justServerDeafed,
},
"Bot was server-muted/deafened — immediately self-undeafening + unmuting",
);
// Fire-and-forget the corrective edit; forceSelfServerUnmuteUndeafen logs.
void forceSelfServerUnmuteUndeafen(client, guild);
}
let _pcmWsClient: VoicePcmWsClient | undefined;
export function setPcmWsClient(client: VoicePcmWsClient | undefined) {
_pcmWsClient = client;
}
const recordingsDir = config.RECORDINGS_DIR;
// Ensure recordings directory exists
(async () => {
try {
await fsPromises.mkdir(recordingsDir, { recursive: true });
} catch (error) {
// Directory might already exist, that's fine
if ((error as NodeJS.ErrnoException).code !== "EEXIST") {
logger.error({ error }, "Failed to create recordings directory");
}
}
})();
const activeSessions = new Map<string, RecordingSession>();
export function resetActiveSessions(): void {
activeSessions.clear();
}
function finalizeActiveRecordingSession(guildId: string): void {
const session = activeSessions.get(guildId);
if (!session) return;
activeSessions.delete(guildId);
// Per-segment upload is the real flow; session metadata is written alongside
// each segment by finalizeSegment in segment.ts
logger.debug(
{ sessionId: session.sessionId, guildId },
"Active recording session finalized",
);
}
/**
* Join ke voice channel dan mulai merekam semua user yang bicara.
*/
export async function startRecording(
client: Client,
channel: VoiceChannel,
): Promise<VoiceConnection | null> {
const connection = joinVoiceChannel({
channelId: channel.id,
guildId: channel.guild.id,
adapterCreator: channel.guild
.voiceAdapterCreator as DiscordGatewayAdapterCreator,
selfDeaf: false,
selfMute: false,
debug: true,
});
logger.info({ channelName: channel.name }, "Joining voice channel");
connection.on("debug", (msg) => {
if (config.VERBOSE) {
logger.debug({ message: msg }, "Voice debug");
}
});
connection.on("error", (err) => {
logger.error({ error: err }, "Voice connection error");
});
// Wait until fully connected with retry logic
try {
await retryWithBackoff(
() =>
entersState(
connection,
VoiceConnectionStatus.Ready,
config.VOICE_CONNECTION_TIMEOUT_MS,
),
{
retries: 3,
minTimeout: 1000,
maxTimeout: 5000,
},
);
logger.info("Connected to voice channel. Recording started");
// Auto undeafen + unmute the bot itself at the server level (MUTE_MEMBERS /
// DEAFEN_MEMBERS permission granted). Server-muted/deafened bot can't
// receive others' audio/video. Fire-and-forget — never blocks recording.
void forceSelfServerUnmuteUndeafen(client, channel.guild);
// Create recording session after connection is ready
const sessionStartTime = Date.now();
const session = createRecordingSession({
guildId: channel.guild.id,
channelId: channel.id,
channelName: channel.name,
startTime: sessionStartTime,
});
activeSessions.set(channel.guild.id, session);
// Register this channel for video recording (others' camera/screen share).
trackChannel(channel.guild.id, channel);
} catch (err) {
logger.error({ error: err }, "Failed to connect to voice channel");
connection.destroy();
return null;
}
const receiver = connection.receiver;
// Use the extracted speaking handler for voice activity
const speakingHandler = createSpeakingHandler({
client,
channel,
receiver,
eventBroadcaster: _eventBroadcaster,
activeSessions,
recordingsDir,
pcmSender: _pcmWsClient
? (pcm, userId) => _pcmWsClient?.sendPcm(userId, pcm)
: undefined,
});
receiver.speaking.on("start", speakingHandler);
// ── Screen-share audio capture ──────────────────────────────────────
// Discord GoLive sends screen-share audio on a SEPARATE SSRC from the
// user's microphone. `receiver.speaking` only fires for voice (mic) SSRCs,
// so screen-share audio is silently dropped unless we hook the UDP receiver
// to discover and register those SSRCs.
hookScreenShareAudio(receiver, speakingHandler);
// ── Video capture (camera + screen share) ───────────────────────────
// @discordjs/voice only decrypts/forwards audio (opus). Video RTP (H264,
// VP8/VP9/AV1) arrives on the same UDP socket but is dropped by the opus
// gate. hookVideoReceiver wraps onUdpMessage, decrypts video payload types
// with the connection secret key (reusing receiver.parsePacket), then
// depacketizes H264 to AnnexB and writes a raw .h264 stream per user.
// Order matters: video hook must be installed AFTER the screen-share audio
// hook so non-video packets delegate down the existing wrapper chain.
hookVideoReceiver(receiver, client, recordingsDir);
// Handle unexpected disconnection
connection.on(VoiceConnectionStatus.Disconnected, async () => {
if (config.VERBOSE) {
logger.warn("Disconnected from voice channel. Reconnecting...");
}
try {
await Promise.race([
entersState(
connection,
VoiceConnectionStatus.Signalling,
config.RECONNECT_TIMEOUT_MS,
),
entersState(
connection,
VoiceConnectionStatus.Connecting,
config.RECONNECT_TIMEOUT_MS,
),
]);
// Reconnected successfully
// Server-deaf can silently revert across a reconnect — re-assert the bot
// is undeafened + unmuted so video receive keeps working.
void forceSelfServerUnmuteUndeafen(client, channel.guild);
} catch {
logger.error("Could not reconnect. Destroying connection");
connection.destroy();
}
});
connection.on(VoiceConnectionStatus.Destroyed, () => {
finalizeActiveRecordingSession(channel.guild.id);
untrackChannel(channel.guild.id);
if (config.VERBOSE) {
logger.info("Voice connection destroyed");
}
});
return connection;
}
/**
* Stop recording and disconnect from voice channel.
*/
export function stopRecording(guildId: string): void {
const connection = getVoiceConnection(guildId);
if (connection) {
connection.destroy();
if (config.VERBOSE) {
logger.info("Recording stopped and disconnected");
}
} else {
logger.warn("No active connection to stop");
}
untrackChannel(guildId);
const session = activeSessions.get(guildId);
finalizeActiveRecordingSession(guildId);
// Broadcast voice_recording_stopped + finalize session
if (session && _eventBroadcaster) {
const snapshot = session.snapshot(Date.now());
const stoppedAt = Date.now();
_eventBroadcaster
.voiceRecordingStopped({
guild_id: guildId,
session_id: session.sessionId,
duration_ms: snapshot.durationMs,
participants: snapshot.participants.length,
segment_count: snapshot.segments.length,
status: snapshot.status,
stopped_at: stoppedAt,
})
.catch((err) =>
logger.warn({ err }, "Failed to broadcast voice recording stopped"),
);
// Auto-enqueue muxer job if there are multiple segments
const segments = snapshot.segments;
if (segments.length >= 2) {
const outputFile = `${config.RECORDINGS_DIR}/merged/${session.sessionId}.mp3`;
import("./muxer.js")
.then(({ enqueueMuxerJob }) => {
enqueueMuxerJob({
inputs: segments.map((s) => s.oggPath),
output: outputFile,
guildId,
channelId: snapshot.channelId,
sessionId: session.sessionId,
}).catch(() => {});
})
.catch(() => {});
}
}
}
services/discord-gateway/src/modules/voice-recording/recorder/decoder.ts
-126
View File
@@ -1,126 +0,0 @@
import { createRequire } from "node:module";
import * as prism from "prism-media";
import { createChildLogger } from "@/shared/logger/index";
import { config } from "../../../shared/config/config.js";
const require = createRequire(import.meta.url);
const logger = createChildLogger("opus-decoder");
interface OpusDecoderRuntime {
isBun: boolean;
canLoadNativeOpus: boolean;
}
export function shouldEnableDefaultOpusDecoder(
runtime: OpusDecoderRuntime,
): boolean {
return !runtime.isBun || runtime.canLoadNativeOpus;
}
function canLoadNativeOpus(): boolean {
try {
require("@discordjs/opus");
return true;
} catch {
return false;
}
}
const defaultDecoderEnabled = shouldEnableDefaultOpusDecoder({
isBun: Boolean(process.versions.bun),
canLoadNativeOpus: canLoadNativeOpus(),
});
export interface OpusDecoderOptions {
cooldownMs: number;
rotateMs: number;
createDecoder?: () => prism.opus.Decoder;
onData: (pcm: Buffer) => void;
}
export class OpusDecoder {
private decoder: prism.opus.Decoder | null = null;
private disabledUntil = 0;
private createdAt = 0;
private readonly cooldownMs: number;
private readonly rotateMs: number;
private readonly createDecoderFn: () => prism.opus.Decoder;
private readonly onData: (pcm: Buffer) => void;
constructor(options: OpusDecoderOptions) {
this.cooldownMs = options.cooldownMs;
this.rotateMs = options.rotateMs;
this.onData = options.onData;
this.createDecoderFn =
options.createDecoder ??
(() => {
if (!defaultDecoderEnabled) {
throw new Error(
"Native @discordjs/opus is unavailable under Bun; web PCM decode disabled to avoid opusscript aborts",
);
}
return new prism.opus.Decoder({
frameSize: config.OPUS_FRAME_SIZE,
channels: config.AUDIO_CHANNELS as 1 | 2,
rate: config.AUDIO_SAMPLE_RATE as
| 8000
| 12000
| 16000
| 24000
| 48000,
});
});
}
rotateIfNeeded(): void {
if (!this.decoder || this.rotateMs <= 0) return;
if (Date.now() - this.createdAt < this.rotateMs) return;
this.destroy();
this.ensureDecoder();
}
write(chunk: Buffer): void {
const decoder = this.ensureDecoder();
if (!decoder) return;
try {
decoder.write(chunk);
} catch (error) {
logger.warn({ error }, "Opus decoder write failed, cooling down");
this.coolDown();
}
}
destroy(): void {
if (!this.decoder) return;
this.decoder.removeAllListeners();
this.decoder.destroy();
this.decoder = null;
this.createdAt = 0;
}
private ensureDecoder(): prism.opus.Decoder | null {
if (this.decoder) return this.decoder;
if (Date.now() < this.disabledUntil) return null;
try {
const decoder = this.createDecoderFn();
decoder.on("data", this.onData);
decoder.on("error", (error) => {
logger.warn({ error }, "Opus decoder error, cooling down");
this.coolDown();
});
this.decoder = decoder;
this.createdAt = Date.now();
return decoder;
} catch (error) {
logger.warn({ error }, "Opus decoder init failed, cooling down");
this.disabledUntil = Date.now() + this.cooldownMs;
return null;
}
}
private coolDown(): void {
this.disabledUntil = Date.now() + this.cooldownMs;
this.destroy();
}
}
services/discord-gateway/src/modules/voice-recording/recorder/metadata.ts
-107
View File
@@ -1,107 +0,0 @@
import path from "node:path";
import type { Client, VoiceChannel } from "discord.js-selfbot-v13";
import { createChildLogger } from "@/shared/logger/index";
import { config } from "../../../shared/config/config.js";
import type {
SegmentMetadata,
SegmentState,
UserMetadata,
} from "../../message-capture/types.js";
const logger = createChildLogger("voice-metadata");
/** LRU-ish cache: userId -> UserMetadata. Avoids Discord API calls in hotpath. */
const metadataCache = new Map<string, UserMetadata>();
const METADATA_CACHE_MAX = 200;
function cacheMetadata(userId: string, metadata: UserMetadata): void {
if (metadataCache.size >= METADATA_CACHE_MAX) {
// Evict oldest entry via Map iteration (Map preserves insertion order)
const firstKey = metadataCache.keys().next().value;
if (firstKey) metadataCache.delete(firstKey);
}
metadataCache.set(userId, metadata);
}
export async function collectUserMetadata(
client: Client,
userId: string,
channel: VoiceChannel,
): Promise<UserMetadata> {
const cached = metadataCache.get(userId);
if (cached) return cached;
const user =
client.users.cache.get(userId) ||
(await client.users.fetch(userId).catch(() => {
logger.warn({ userId }, "Failed to fetch user");
return null;
}));
const member =
channel.guild.members.cache.get(userId) ||
(await channel.guild.members.fetch(userId).catch(() => {
logger.warn({ userId }, "Failed to fetch guild member");
return null;
}));
const username = user?.username ?? "Unknown User";
const roles =
member?.roles.cache
.filter((role) => role.id !== channel.guild.id)
.sort((a, b) => b.position - a.position)
.map((role) => ({
id: role.id,
name: role.name,
position: role.position,
})) ?? [];
const result: UserMetadata = {
userId,
username,
tag: user?.tag ?? "Unknown#0000",
displayName: member?.displayName ?? username,
avatarUrl:
user?.displayAvatarURL({
format: "png",
size: config.AVATAR_SIZE as
| 16
| 32
| 64
| 128
| 256
| 512
| 1024
| 2048
| 4096,
}) ?? "https://cdn.discordapp.com/embed/avatars/0.png",
bot: user?.bot ?? false,
roles,
highestRole: roles[0] ?? null,
joinedTimestamp: member?.joinedTimestamp ?? null,
};
cacheMetadata(userId, result);
return result;
}
export function createSegmentMetadata(
user: UserMetadata,
segment: SegmentState,
sessionId: string,
recordingSessionId: string,
sessionStartTime: number,
recordingSegmentMs: number,
): SegmentMetadata {
const endTime = segment.endTime ?? Date.now();
return {
...user,
sessionId,
recordingSessionId,
sessionStartTime,
segmentIndex: segment.index,
segmentMs: recordingSegmentMs,
startTime: segment.startTime,
endTime,
durationMs: endTime - segment.startTime,
filename: path.basename(segment.filename),
};
}
services/discord-gateway/src/modules/voice-recording/recorder/oggCrc.ts
-56
View File
@@ -1,56 +0,0 @@
import fs from "node:fs";
/**
* Ogg page CRC-32 (RFC 3533): polynomial 0x04c11db7, init 0, MSB-first
* (no reflection), no final XOR. Table-driven, pure JS — no native deps.
*
* prism-media's OggLogicalBitstream writes pages with `crc: false` (node-crc
* was dropped from the dependency tree), which produces files whose page
* checksums are all zero. Strict players (ffmpeg, iOS) reject those with
* "CRC mismatch! / End of file". This re-computes the checksum of every page
* in place, yielding a spec-compliant Ogg file that any player can open.
*/
const CRC_TABLE = new Int32Array(256);
for (let i = 0; i < 256; i++) {
let c = i << 24;
for (let k = 0; k < 8; k++) {
c = c & 0x80000000 ? (c << 1) ^ 0x04c11db7 : c << 1;
}
CRC_TABLE[i] = c | 0;
}
function oggCrc32(buffer: Buffer): number {
let c = 0;
for (let i = 0; i < buffer.length; i++) {
c = ((c << 8) ^ CRC_TABLE[((c >>> 24) ^ buffer[i]) & 0xff]) | 0;
}
return c >>> 0;
}
/**
* Rewrites the CRC field (bytes 22-25) of every OggS page in `filePath`.
* Returns the number of pages fixed (0 if the file is not a valid Ogg stream).
*/
export function fixOggCrc(filePath: string): number {
const data = fs.readFileSync(filePath);
let pos = 0;
let fixed = 0;
while (pos + 27 <= data.length) {
if (data.toString("latin1", pos, pos + 4) !== "OggS") break;
const nsegs = data[pos + 26];
let bodyLen = 0;
for (let i = 0; i < nsegs; i++) bodyLen += data[pos + 27 + i];
const pageEnd = pos + 27 + nsegs + bodyLen;
if (pageEnd > data.length) break;
// Zero the checksum field, then CRC the whole page (RFC 3533).
data.writeUInt32LE(0, pos + 22);
data.writeUInt32LE(oggCrc32(data.subarray(pos, pageEnd)), pos + 22);
fixed++;
pos = pageEnd;
}
if (fixed > 0) fs.writeFileSync(filePath, data);
return fixed;
}
services/discord-gateway/src/modules/voice-recording/recorder/segment.ts
-131
View File
@@ -1,131 +0,0 @@
import fs from "node:fs";
import path from "node:path";
import * as prism from "prism-media";
import { createChildLogger } from "@/shared/logger/index";
import type { SegmentState } from "../../message-capture/types.js";
const logger = createChildLogger("voice-segment");
// ---------------------------------------------------------------------------
// Path helpers
// ---------------------------------------------------------------------------
export function buildSegmentPaths(
userDir: string,
startTime: number,
): { filename: string; jsonFilename: string } {
return {
filename: path.join(userDir, `${startTime}.ogg`),
jsonFilename: path.join(userDir, `${startTime}.json`),
};
}
export function shouldRotateSegment(
startTime: number,
now: number,
recordingSegmentMs: number,
): boolean {
return recordingSegmentMs > 0 && now - startTime >= recordingSegmentMs;
}
// ---------------------------------------------------------------------------
// SegmentManager
// ---------------------------------------------------------------------------
export class SegmentManager {
private currentSegment: SegmentState | null = null;
private segmentIndex = 0;
constructor(
private readonly userDir: string,
private readonly recordingSegmentMs: number,
) {}
open(oggPacketStream: NodeJS.ReadableStream): SegmentState {
const index = this.segmentIndex++;
const startTime = Date.now();
const { filename, jsonFilename } = buildSegmentPaths(
this.userDir,
startTime,
);
const oggStream = new prism.opus.OggLogicalBitstream({
opusHead: new prism.opus.OpusHead({ channelCount: 2, sampleRate: 48000 }),
pageSizeControl: { maxPackets: 10 },
crc: false,
});
const out = fs.createWriteStream(filename);
oggPacketStream.pipe(oggStream).pipe(out);
this.currentSegment = {
index,
startTime,
endTime: null,
filename,
jsonFilename,
oggStream,
out,
};
logger.debug(
{ index, startTime, filename, userDir: this.userDir },
"Segment opened",
);
return this.currentSegment;
}
close(oggPacketStream: NodeJS.ReadableStream): SegmentState | null {
if (!this.currentSegment) return null;
const segment = this.currentSegment;
segment.endTime = Date.now();
oggPacketStream.unpipe(segment.oggStream);
segment.oggStream.end();
this.currentSegment = null;
// Get file size after closing
let fileSize = 0;
try {
const stat = fs.statSync(segment.filename);
fileSize = stat.size;
} catch {
// File might not exist yet
}
logger.debug(
{
index: segment.index,
filename: segment.filename,
fileSize,
durationMs: (segment.endTime ?? 0) - segment.startTime,
},
"Segment closed",
);
return segment;
}
rotateIfNeeded(oggPacketStream: NodeJS.ReadableStream): SegmentState | null {
if (!this.currentSegment) return null;
if (
!shouldRotateSegment(
this.currentSegment.startTime,
Date.now(),
this.recordingSegmentMs,
)
)
return null;
logger.debug(
{
index: this.currentSegment.index,
filename: this.currentSegment.filename,
durationMs: Date.now() - this.currentSegment.startTime,
},
"Segment rotating",
);
this.close(oggPacketStream);
return this.open(oggPacketStream);
}
getCurrent(): SegmentState | null {
return this.currentSegment;
}
}
services/discord-gateway/src/modules/voice-recording/recorder/segmentFinalizer.ts
-148
View File
@@ -1,148 +0,0 @@
import { promises as fsPromises } from "node:fs";
import { createChildLogger } from "@/shared/logger/index";
import { config } from "../../../shared/config/config.js";
import type {
SegmentState,
UserMetadata,
} from "../../message-capture/types.js";
import { createSegmentMetadata } from "./metadata.js";
import { fixOggCrc } from "./oggCrc.js";
import type { RecordingSession } from "./sessionRecording.js";
import { uploadRecordingSegment } from "./uploader.js";
const logger = createChildLogger("voice-segment-finalizer");
// ---------------------------------------------------------------------------
// SegmentFinalizerInput
// ---------------------------------------------------------------------------
export interface SegmentFinalizerInput {
currentSegment: SegmentState;
userMetadata: UserMetadata;
activeSession: RecordingSession | undefined;
guildId: string;
channelId: string;
channelName: string;
}
// ---------------------------------------------------------------------------
// finalizeSegment
// ---------------------------------------------------------------------------
/**
* Handles the completion of an OGG segment:
* - Logs the saved segment (if VERBOSE)
* - Registers the segment with the active recording session
* - Writes the metadata JSON file alongside the OGG file
* - Triggers async upload of the segment to external storage
*
* This function is fire-and-forget for the metadata write and upload;
* errors are caught and logged without throwing.
*/
export function finalizeSegment(input: SegmentFinalizerInput): void {
const {
currentSegment,
userMetadata,
activeSession,
guildId,
channelId,
channelName,
} = input;
const endTime = currentSegment.endTime ?? Date.now();
const durationMs = endTime - currentSegment.startTime;
// Discard only very short segments — a blip of ambient noise or a mic
// click. At 300ms, brief acknowledgements ("ya", "siap", "ok") are still
// kept while true artifacts are dropped.
const MIN_DURATION_MS = 300;
if (durationMs < MIN_DURATION_MS) {
logger.debug(
{ filename: currentSegment.filename, durationMs },
"Segment too short, discarding",
);
// Clean up the OGG file
fsPromises.unlink(currentSegment.filename).catch(() => {});
fsPromises.unlink(currentSegment.jsonFilename).catch(() => {});
return;
}
if (config.VERBOSE) {
logger.info({ filename: currentSegment.filename }, "Segment saved");
}
// Fix Ogg page CRCs before anything reads the file (upload/merge/transcode).
// prism-media writes pages with crc:false → checksums are zero → strict
// players (ffmpeg, iOS) reject the file. Re-compute in place, pure JS.
try {
const fixed = fixOggCrc(currentSegment.filename);
if (config.VERBOSE) {
logger.info(
{ filename: currentSegment.filename, pages: fixed },
"Ogg page CRCs fixed",
);
}
} catch (err: unknown) {
logger.error(
{
filename: currentSegment.filename,
error: err instanceof Error ? err.message : String(err),
},
"Failed to fix Ogg CRCs — segment may be unplayable",
);
}
// Register segment with the active recording session
if (activeSession) {
activeSession.registerSegment({
user: userMetadata,
oggPath: currentSegment.filename,
jsonPath: currentSegment.jsonFilename,
startTime: currentSegment.startTime,
endTime,
});
}
// Write metadata JSON (async, fire-and-forget)
const metadata = createSegmentMetadata(
userMetadata,
currentSegment,
activeSession?.sessionId ?? `${userMetadata.userId}-0`,
activeSession?.sessionId ?? `${guildId}-${channelId}-0`,
activeSession?.startTime ?? 0,
config.RECORDING_SEGMENT_MS,
);
fsPromises
.writeFile(currentSegment.jsonFilename, JSON.stringify(metadata, null, 2))
.then(() => {
if (config.VERBOSE) {
logger.info(
{ jsonFile: currentSegment.jsonFilename },
"Metadata saved",
);
}
})
.catch((err: unknown) => {
logger.error(
{ error: err instanceof Error ? err.message : String(err) },
"Failed to write segment metadata",
);
});
// Trigger async voice segment upload (fire-and-forget)
const segmentId = `${userMetadata.userId}-${currentSegment.startTime}`;
uploadRecordingSegment({
id: segmentId,
oggPath: currentSegment.filename,
userId: userMetadata.userId,
username: userMetadata.username,
avatarUrl: userMetadata.avatarUrl,
guildId,
channelId,
channelName,
}).catch((err: unknown) => {
const msg = err instanceof Error ? err.message : String(err);
logger.error({ segmentId, error: msg }, "Upload segment trigger failed");
});
}
services/discord-gateway/src/modules/voice-recording/recorder/sessionRecording.ts
-119
View File
@@ -1,119 +0,0 @@
import { createChildLogger } from "@/shared/logger/index";
import type { UserMetadata } from "../../message-capture/types.js";
const logger = createChildLogger("recording-session");
export interface RecordingSessionOptions {
guildId: string;
channelId: string;
channelName: string;
startTime: number;
}
export interface SessionParticipant {
userId: string;
username: string;
tag: string;
displayName: string;
avatarUrl: string;
}
export interface SessionSegmentRef {
userId: string;
oggPath: string;
jsonPath: string;
startTime: number;
endTime: number;
durationMs: number;
offsetMs: number;
}
export interface SessionRecordingMetadata {
sessionId: string;
guildId: string;
channelId: string;
channelName: string;
startTime: number;
endTime: number;
durationMs: number;
status: "pending" | "completed" | "failed" | "empty";
outputFile: string | null;
participants: SessionParticipant[];
segments: SessionSegmentRef[];
error?: string;
}
export interface RecordingSession {
readonly sessionId: string;
readonly startTime: number;
registerSegment(input: {
user: UserMetadata;
oggPath: string;
jsonPath: string;
startTime: number;
endTime: number;
}): void;
snapshot(endTime: number): SessionRecordingMetadata;
}
export function createRecordingSession(
options: RecordingSessionOptions,
): RecordingSession {
const sessionId = `${options.guildId}-${options.channelId}-${options.startTime}`;
const participants = new Map<string, SessionParticipant>();
const segments: SessionSegmentRef[] = [];
logger.info(
{
sessionId,
guildId: options.guildId,
channelId: options.channelId,
channelName: options.channelName,
},
"Recording session created",
);
return {
sessionId,
startTime: options.startTime,
registerSegment(input) {
participants.set(input.user.userId, {
userId: input.user.userId,
username: input.user.username,
tag: input.user.tag,
displayName: input.user.displayName,
avatarUrl: input.user.avatarUrl,
});
segments.push({
userId: input.user.userId,
oggPath: input.oggPath,
jsonPath: input.jsonPath,
startTime: input.startTime,
endTime: input.endTime,
durationMs: input.endTime - input.startTime,
offsetMs: input.startTime - options.startTime,
});
logger.debug(
{ sessionId, userId: input.user.userId, segmentCount: segments.length },
"Segment registered in session",
);
},
snapshot(endTime: number): SessionRecordingMetadata {
return {
sessionId,
guildId: options.guildId,
channelId: options.channelId,
channelName: options.channelName,
startTime: options.startTime,
endTime,
durationMs: endTime - options.startTime,
status: "pending",
outputFile: null,
participants: Array.from(participants.values()),
segments: [...segments],
};
},
};
}
services/discord-gateway/src/modules/voice-recording/recorder/speakingHandler.ts
-234
View File
@@ -1,234 +0,0 @@
import { promises as fsPromises } from "node:fs";
import path from "node:path";
import type { VoiceConnection } from "@discordjs/voice";
import type { Client, VoiceChannel } from "discord.js-selfbot-v13";
import { createChildLogger } from "@/shared/logger/index";
import type { EventBroadcaster } from "../../event-broadcaster/eventBroadcaster.js";
import type {
SegmentState,
UserMetadata,
} from "../../message-capture/types.js";
import { collectUserMetadata } from "./metadata.js";
import { finalizeSegment } from "./segmentFinalizer.js";
import type { RecordingSession } from "./sessionRecording.js";
import { setupUserStream } from "./streamSetup.js";
const logger = createChildLogger("speaking-handler");
export interface SpeakingHandlerContext {
client: Client;
channel: VoiceChannel;
receiver: VoiceConnection["receiver"];
eventBroadcaster: EventBroadcaster | undefined;
activeSessions: Map<string, RecordingSession>;
recordingsDir: string;
/** Direct WS sender for real-time PCM — takes priority over Redis if set. */
pcmSender?: (pcm: Buffer, userId: string) => void;
}
/**
* Creates the event handler for `receiver.speaking.on("start", handler)`.
*
* # Why this is structured the way it is (recording-completeness)
*
* `receiver.speaking"start"` fires from `onUdpMessage` the moment the FIRST
* Opus packet for a user arrives, and the opus bytes are forwarded to the
* subscription stream **only if a subscription already exists** — otherwise
* they are dropped (`subscriptions.get(userId)` is undefined → `return`).
*
* That means the subscription MUST be created synchronously (no `await` before
* it): every frame between "start" and the subscribe call is otherwise silently
* discarded → the START of a burst is MISSING (the user's core complaint).
* The previous code `await`ed `collectUserMetadata` (a Discord REST roundtrip,
* slow on cache miss) BEFORE subscribing — that was the main source of misses.
*
* Metadata is now fetched in the background. If the speaker turns out to be a
* bot, the just-started burst is discarded (unlinked, never uploaded).
*/
export function createSpeakingHandler(
ctx: SpeakingHandlerContext,
): (userId: string) => Promise<void> {
const {
client,
channel,
receiver,
eventBroadcaster,
activeSessions,
recordingsDir,
pcmSender,
} = ctx;
return async (userId: string) => {
// Skip the bot's own audio.
if (userId === client.user?.id) return;
// Synchronous guard BEFORE any await. Two "start" events can race during a
// subscription; guard here so we never create a second subscription.
if (receiver.subscriptions.has(userId)) return;
logger.debug({ userId }, "Voice activity detected");
// Optimistic speaking:true (real metadata arrives async).
eventBroadcaster?.voiceActiveUser(userId, {
username: userId,
avatar: "",
speaking: true,
});
// Subscribe IMMEDIATELY (synchronously — guard is already done above, no
// `await` since then), so the subscription exists before any Opus frame
// arrives. Frames are then buffered by the AudioReceiveStream while we
// finish setup below (mkdir is non-network and fast).
const userDir = path.join(recordingsDir, userId);
const { audioStream, packetFilter, segmentManager, decoder } =
setupUserStream({
userId,
receiver,
userDir,
onPcmData: (pcm) => {
if (pcmSender) {
pcmSender(pcm, userId);
} else {
eventBroadcaster?.voicePcmData(pcm, userId);
}
},
});
// Ensure per-user recording directory (subscription is already live, so
// this await does NOT drop audio — frames buffer in the subscription).
await fsPromises.mkdir(userDir, { recursive: true }).catch(() => {
// Directory already exists, ignore.
});
const activeSession = activeSessions.get(channel.guild.id);
// Mutable per-burst state.
let userMetadata: UserMetadata | null = null;
let accepted = true; // false once we learn it's a bot → discard burst
let finalized = false;
/** Unlink the given segment's files (bot/error discard path). */
const discardSegmentFiles = (seg: SegmentState | null): void => {
if (!seg) return;
fsPromises.unlink(seg.filename).catch(() => {});
fsPromises.unlink(seg.jsonFilename).catch(() => {});
};
const emitSpeakingFalse = (): void => {
eventBroadcaster?.voiceActiveUser(userId, {
username: userMetadata?.username ?? userId,
avatar: userMetadata?.avatarUrl ?? "",
speaking: false,
});
};
/**
* Close the current (per-burst) segment and — once the underlying file has
* finished flushing to disk — finalize it (or discard it if bot/error).
* Safe to call multiple times (guarded by `finalized`).
*/
const finishBurst = (): void => {
if (finalized) return;
finalized = true;
const seg = segmentManager.close(oggPacketStream);
decoder.destroy();
const doFinalize = (): void => {
Promise.resolve(userMetadata).then((meta) => {
if (meta && accepted && seg) {
try {
finalizeSegment({
currentSegment: seg,
userMetadata: meta,
activeSession,
guildId: channel.guild.id,
channelId: channel.id,
channelName: channel.name,
});
} catch (err) {
const msg = err instanceof Error ? err.message : String(err);
logger.error({ userId, error: msg }, "Segment finalize failed");
}
} else {
// Bot audio, metadata failure, or no segment opened — discard.
discardSegmentFiles(seg);
}
});
};
// finalizeSegment reads the OGG file, so wait for the write stream to
// flush before touching it.
if (seg?.out.writableFinished) {
doFinalize();
} else {
seg?.out.once("finish", doFinalize);
}
};
// Attach audioStream handlers BEFORE pipe() (prevents data loss).
audioStream.on("data", (chunk: Buffer) => {
if (chunk.length < 8) return;
// One segment per burst — no time-based rotation here. Only rotate the
// web-PCM broadcast decoder to bound its memory.
decoder.rotateIfNeeded();
decoder.write(chunk);
});
audioStream.on("end", () => {
finishBurst();
emitSpeakingFalse();
});
audioStream.on("error", (error: Error) => {
logger.error({ userId, error: error.message }, "Audio stream error");
finishBurst();
emitSpeakingFalse();
});
// Pipe for OGG recording (handlers already attached).
const oggPacketStream = audioStream.pipe(packetFilter);
// Open the (single, per-burst) segment.
const currentSegment = segmentManager.open(oggPacketStream);
// Handle file-write errors on the underlying write stream.
currentSegment.out.on("error", (err: unknown) => {
const msg = err instanceof Error ? err.message : String(err);
logger.error({ userId, error: msg }, "File write error");
});
// Handle packet-filter errors (rare) by closing the burst.
packetFilter.on("error", (err: Error) => {
logger.error({ userId, error: err.message }, "PacketFilter error");
finishBurst();
emitSpeakingFalse();
});
// Fetch user metadata in the background; reject bots by discarding burst.
collectUserMetadata(client, userId, channel)
.then((meta) => {
if (meta.bot) {
accepted = false;
finishBurst();
emitSpeakingFalse();
return;
}
userMetadata = meta;
eventBroadcaster?.voiceActiveUser(userId, {
username: meta.username,
avatar: meta.avatarUrl,
speaking: true,
});
})
.catch((err: unknown) => {
const msg = err instanceof Error ? err.message : String(err);
logger.warn(
{ userId, error: msg },
"Metadata fetch failed, dropping burst",
);
accepted = false;
finishBurst();
emitSpeakingFalse();
});
};
}
services/discord-gateway/src/modules/voice-recording/recorder/streamSetup.ts
-85
View File
@@ -1,85 +0,0 @@
import type { VoiceConnection } from "@discordjs/voice";
import { EndBehaviorType } from "@discordjs/voice";
import { createChildLogger } from "@/shared/logger/index";
import { config } from "../../../shared/config/config.js";
import { PacketFilter } from "../packetFilter.js";
import { OpusDecoder } from "./decoder.js";
import { SegmentManager } from "./segment.js";
const logger = createChildLogger("stream-setup");
export interface StreamSetupInput {
userId: string;
receiver: VoiceConnection["receiver"];
userDir: string;
onPcmData: (pcm: Buffer) => void;
}
export interface StreamSetupResult {
audioStream: NodeJS.ReadableStream;
packetFilter: PacketFilter;
segmentManager: SegmentManager;
decoder: OpusDecoder;
}
/**
* Creates the audio stream subscription, decoder, packet filter, and segment
* manager for a user who started speaking.
*
* NOTE: This function does NOT pipe the audio stream through the packet filter.
* The caller must attach event handlers to `audioStream` BEFORE calling
* `audioStream.pipe(packetFilter)` to prevent data loss from race conditions.
*/
export function setupUserStream(input: StreamSetupInput): StreamSetupResult {
const { userId, receiver, userDir, onPcmData } = input;
logger.debug({ userId }, "Setting up user audio stream");
// Subscribe to the audio stream from the Discord voice receiver
const audioStream = receiver.subscribe(userId, {
end: {
behavior: EndBehaviorType.AfterSilence,
duration: config.AUDIO_STREAM_SILENCE_DURATION_MS,
},
});
const packetFilter = new PacketFilter(config.PACKET_FILTER_MIN_SIZE);
const segmentManager = new SegmentManager(
userDir,
config.RECORDING_SEGMENT_MS,
);
// Create decoder for web broadcast (PCM downsampling)
const decoder = new OpusDecoder({
cooldownMs: config.DECODER_COOLDOWN_MS,
rotateMs: config.DECODER_ROTATE_MS,
onData: (pcm: Buffer) => {
// Downsample 48kHz stereo -> 24kHz mono (left channel, every 2nd frame)
// Use typed array views for efficient access instead of read/writeInt16LE
const inputView = new Int16Array(
pcm.buffer,
pcm.byteOffset,
pcm.byteLength / 2,
);
const outBuf = Buffer.alloc(inputView.length / 2); // 48k stereo -> 24k mono = 1/4 size
const outputView = new Int16Array(
outBuf.buffer,
outBuf.byteOffset,
outBuf.byteLength / 2,
);
for (let i = 0; i < outputView.length; i++) {
outputView[i] = inputView[i * 4]; // left channel, every 2nd stereo frame
}
onPcmData(outBuf);
},
});
logger.debug({ userId }, "User audio stream setup complete");
return {
audioStream,
packetFilter,
segmentManager,
decoder,
};
}
services/discord-gateway/src/modules/voice-recording/recorder/uploader.ts
-164
View File
@@ -1,164 +0,0 @@
import { execFile } from "node:child_process";
import fs from "node:fs";
import path from "node:path";
import { promisify } from "node:util";
import { createChildLogger } from "@/shared/logger/index";
import { config } from "../../../shared/config/config.js";
import {
insertVoiceRecording,
updateVoiceRecordingAsFailed,
updateVoiceRecordingAsUploaded,
updateVoiceRecordingTranscription,
} from "../../../shared/database/voiceRecordingRepo.js";
import { uploadToTele } from "../../../shared/uploader.js";
import { transcribeRecording } from "../voiceTranscriber.js";
const logger = createChildLogger("recording-uploader");
const execFileAsync = promisify(execFile);
/**
* Transcode an Opus/OGG segment to MP3 so recordings are playable on any
* device (Safari/iPhone cannot play OGG). Uses PATH-resolved ffmpeg from the
* Nix closure. Returns the path to the temporary MP3 file.
*/
async function transcodeToMp3(oggPath: string): Promise<string> {
const mp3Path = oggPath.replace(/\.ogg$/i, ".mp3");
await execFileAsync(
"ffmpeg",
[
"-y",
"-i",
oggPath,
"-codec:a",
"libmp3lame",
"-b:a",
"128k",
"-ar",
"48000",
"-ac",
"2",
mp3Path,
],
{ timeout: 30_000 },
);
return mp3Path;
}
/**
* Uploads a recorded segment OGG file to external server and registers in database
*/
export async function uploadRecordingSegment(input: {
id: string;
oggPath: string;
userId: string;
username: string;
avatarUrl: string | null;
guildId: string | null;
channelId: string | null;
channelName: string | null;
}): Promise<void> {
const {
id,
oggPath,
userId,
username,
avatarUrl,
guildId,
channelId,
channelName,
} = input;
const fileName = path.basename(oggPath);
try {
// 1. Get file size and insert initial pending state to DB
const stats = await fs.promises.stat(oggPath);
await insertVoiceRecording({
id,
user_id: userId,
username,
avatar_url: avatarUrl,
guild_id: guildId,
channel_id: channelId,
channel_name: channelName,
filename: fileName,
size_bytes: stats.size,
upload_status: "pending",
created_at: Date.now(),
});
// 1b. Transcode segment to MP3 (universal playback; Safari can't play OGG)
const mp3Path = await transcodeToMp3(oggPath);
const mp3Name = path.basename(mp3Path);
const mp3Stats = await fs.promises.stat(mp3Path);
// 2. Perform async upload with retry logic
const fileBuffer = await fs.promises.readFile(mp3Path);
const uploadResult = await uploadToTele({
buffer: fileBuffer,
filename: mp3Name,
contentType: "audio/mpeg",
uploadUrl: config.TELE_UPLOAD_URL,
retries: 3,
});
const downloadUrl = uploadResult.url;
// 2b. Clean up the temporary MP3 (source OGG is kept for transcription)
await fs.promises.unlink(mp3Path).catch(() => {});
// 3. Update DB to uploaded state (filename/size reflect the MP3 artifact)
await updateVoiceRecordingAsUploaded(
id,
downloadUrl,
Date.now(),
mp3Name,
mp3Stats.size,
);
logger.info({ id, downloadUrl }, "Recording segment uploaded successfully");
// 4. Broadcast via Redis EventBroadcaster (forwarded to WebSocket clients by backend)
const { _eventBroadcaster } = await import("../recorder.js");
if (_eventBroadcaster) {
_eventBroadcaster
.voiceRecordingUploaded({
id,
user_id: userId,
username,
avatar_url: avatarUrl,
guild_id: guildId,
channel_id: channelId,
channel_name: channelName,
filename: mp3Name,
size_bytes: mp3Stats.size,
download_url: downloadUrl,
upload_status: "uploaded",
created_at: Date.now(),
uploaded_at: Date.now(),
})
.catch((err: unknown) => {
logger.warn(
{ err },
"Failed to broadcast voice recording upload event",
);
});
}
// 5. Fire-and-forget voice transcription
if (config.AI_VOICE_TRANSCRIPTION_ENABLED) {
transcribeRecording(oggPath).then((transcription) => {
if (transcription) {
updateVoiceRecordingTranscription(id, transcription).catch(
(err: unknown) => {
logger.warn({ id, err }, "Failed to persist transcription");
},
);
}
});
}
} catch (error) {
const errorMsg = error instanceof Error ? error.message : String(error);
logger.error({ id, error: errorMsg }, "Failed to upload voice recording");
await updateVoiceRecordingAsFailed(id, errorMsg).catch((err: unknown) => {
logger.error({ id, err }, "Failed to write failure state to DB");
});
}
}
services/discord-gateway/src/modules/voice-recording/screenShareAudio.ts
-222
View File
@@ -1,222 +0,0 @@
import type { VoiceReceiver, VoiceUserData } from "@discordjs/voice";
import { createChildLogger } from "@/shared/logger/index";
const logger = createChildLogger("screen-share-audio");
/**
* Hooks screen-share audio capture into the voice receiver.
*
* ## Background
*
* Discord GoLive (screen share) sends audio on **separate SSRCs** from the
* user's microphone. In `@discordjs/voice` v0.19, `VoiceReceiver.onUdpMessage`
* does:
*
* ```js
* const userData = this.ssrcMap.get(ssrc);
* if (!userData) return; // ← DROPS screen-share audio SSRC
* ```
*
* `ssrcMap` is only populated from `VOICE_STATE_UPDATE` / `VOICE_SERVER_UPDATE`
* WebSocket packets, which carry the **voice audioSSRC** only. When a user
* starts an audio+video screen-share, Discord sends additional RTP packets on
* new SSRCs that are *never* registered in `ssrcMap` → they are silently
* discarded → `receiver.speaking` never fires → screen-share audio is missing.
*
* ## Fix
*
* 1. Wrap `onUdpMessage` to inspect every incoming RTP packet's SSRC.
* 2. If the SSRC isn't in `ssrcMap`, check whether it looks like a screen-share
* audio stream (OPRUS payload type 120, RTP version 2).
* 3. Clone the owning user's VoiceUserData into `ssrcMap` under the new SSRC
* so the *original* (un-patched) `onUdpMessage` picks it up, decrypts it,
* and forwards the Opus packet to the existing subscription stream.
* 4. Emit a synthetic `"start"` speaking event so the existing
* `speakingHandler` sets up the full pipeline (decoder, packet filter,
* segment manager, event handlers) for that userId if not already.
*/
export function hookScreenShareAudio(
receiver: VoiceReceiver,
speakingHandler: (userId: string) => Promise<void>,
): void {
// Cache the original (un-bound) method so we can delegate to it.
const original = receiver.onUdpMessage;
receiver.onUdpMessage = (msg: Buffer) => {
// ── 1. Detect screen-share SSRCs BEFORE the original discards them ──
if (isLikelyScreenShareAudio(msg, receiver)) {
const ssrc = msg.readUInt32BE(8);
const userData = getSsrcMapEntry(receiver, ssrc);
if (!userData) {
// SSRC not registered — try to infer owner and register it
const owner = inferScreenShareOwner(ssrc, receiver);
if (owner) {
registerScreenShareSsrc(receiver, ssrc, owner);
logger.info(
{ userId: owner.userId, ssrc, kind: "screenshare-audio" },
"Registered screen-share audio SSRC in ssrcMap",
);
// Trigger the speaking handler to ensure pipeline is ready
void speakingHandler(owner.userId).catch((err) =>
logger.error(
{ userId: owner.userId, error: err.message },
"Speaking handler for screen-share failed",
),
);
} else {
logger.warn(
{ ssrc },
"Screen-share audio SSRC found but owner unknown",
);
}
}
}
// ── 2. Delegate to the original handler ──
// It will now find the SSRC (we registered it above) and forward the
// decrypted Opus packet to the subscription stream.
original.call(receiver, msg);
};
// ── 3. Listen for dynamic ssrcMap updates ──────────────────────────────
receiver.ssrcMap.on("create", (data: VoiceUserData) => {
if (data.videoSSRC !== undefined) {
logger.info(
{ userId: data.userId, videoSSRC: data.videoSSRC },
"Screen-share video started",
);
void speakingHandler(data.userId).catch((err) =>
logger.error(
{ userId: data.userId, error: err.message },
"Handler for screen-share start failed",
),
);
}
});
receiver.ssrcMap.on(
"update",
(_old: VoiceUserData | undefined, neu: VoiceUserData) => {
if (_old?.videoSSRC !== neu.videoSSRC && neu.videoSSRC !== undefined) {
logger.info(
{ userId: neu.userId, videoSSRC: neu.videoSSRC },
"Screen-share video SSRC appeared",
);
void speakingHandler(neu.userId).catch((err) =>
logger.error(
{ userId: neu.userId, error: err.message },
"Handler for screen-share update failed",
),
);
}
},
);
}
// ─── Helpers ───────────────────────────────────────────────────────────────
/**
* Check if a UDP packet looks like a screen-share audio RTP packet.
*
* Voice packets have: RTP version 2 (top 2 bits of byte 0), and payload type 120 (OPRUS).
* We also require that the SSRC is NOT already in ssrcMap (that's handled
* by the original onUdpMessage).
*/
function isLikelyScreenShareAudio(
msg: Buffer,
receiver: VoiceReceiver,
): boolean {
if (msg.length <= 8) return false;
const ssrc = msg.readUInt32BE(8);
// Already registered as a known voice SSRC?
if (getSsrcMapEntry(receiver, ssrc)) return false;
const rtpVersion = msg[0] >> 6;
const payloadType = msg[1] & 127;
// OPRUS payload type is 120 in Discord voice
return rtpVersion === 2 && payloadType === 120;
}
/**
* Safely read an entry from the SSRCMap via the public `get` API.
*/
function getSsrcMapEntry(
receiver: VoiceReceiver,
ssrc: number,
): VoiceUserData | undefined {
try {
return receiver.ssrcMap.get(ssrc);
} catch {
return undefined;
}
}
/**
* Infer which user owns a screen-share audio SSRC by proximity to their
* known voice audioSSRC (Discord allocates SSRCs in small increments).
*/
function inferScreenShareOwner(
ssrc: number,
receiver: VoiceReceiver,
): { userId: string } | null {
try {
// Iterate known SSRCs via internal _map (the public API only gets by ssrc)
const map = getSsrcInternalMap(receiver.ssrcMap);
if (!map) return null;
for (const [, data] of map.entries()) {
if (data.audioSSRC && Math.abs(data.audioSSRC - ssrc) < 200_000) {
return { userId: data.userId };
}
}
return null;
} catch {
return null;
}
}
/**
* Register a new SSRC in the internal ssrcMap so the original onUdpMessage
* picks it up. We clone the user's existing VoiceUserData (so decryption
* keys, userId mapping, etc. all work) under the new SSRC key.
*/
function registerScreenShareSsrc(
receiver: VoiceReceiver,
ssrc: number,
owner: { userId: string },
): void {
const map = getSsrcInternalMap(receiver.ssrcMap);
if (!map) return;
// Find the owner's existing VoiceUserData and clone it under the new SSRC
for (const [, data] of map.entries()) {
if (data.userId === owner.userId) {
map.set(ssrc, { ...data });
return;
}
}
// Fallback: register with minimal data (userId only)
map.set(ssrc, {
userId: owner.userId,
audioSSRC: ssrc,
});
}
/**
* Access the private `_map` inside an SSRCMap instance.
*
* SSRCMap in @discordjs/voice ≤ v0.19 stores its entries in a private
* `#map` (JS private field) or `_map` depending on the build target.
* We access it defensively for read + write so we can register new SSRCs.
*/
function getSsrcInternalMap(
ssrcMap: VoiceReceiver["ssrcMap"],
): Map<number, VoiceUserData> | undefined {
const asAny = ssrcMap as unknown as Record<string, unknown>;
// v0.19 ESM build uses _map
const m1 = asAny._map;
if (m1 instanceof Map) return m1 as Map<number, VoiceUserData>;
return undefined;
}
services/discord-gateway/src/modules/voice-recording/streamWatchReceiver.ts
-894
View File
@@ -1,894 +0,0 @@
/**
* DAVE-capable stream-watch video receiver for OTHER members (camera + screen
* share). Replaces the dead selfbot-v13 path (which Discord rejects with WS
* close 4017 "E2EE/DAVE protocol required" because that lib's voice stack
* predates DAVE).
*
* HOW IT WORKS (verified architecture, 2026-08-31):
* - Watching a member's stream is a SEPARATE RTC connection, not the guild
* audio socket. We send the gateway signal STREAM_WATCH (op 20) with
* stream_key `guild:<gid>:<chid>:<uid>`; Discord replies with gateway events
* STREAM_CREATE (d.rtc_server_id) + STREAM_SERVER_UPDATE (d.token,
* d.endpoint) that together describe a dedicated voice RTC for the watch.
* - We open a `@discordjs/voice` `Networking` connection pointed at that RTC
* (endpoint/token/serverId = rtc_server_id + the bot's sessionId). `Networking`
* performs the FULL DAVE handshake (identify with max_dave_protocol_version,
* IP discovery, select-protocol, DAVE MLS transitions) — the identical path
* that already works for audio. It is a public export.
* - On Ready, `Networking` exposes `state.udp` (VoiceUDPSocket). We hook its
* `message` event. Video RTP (H264) arrives there; we decrypt via the DAVE
* session with `MediaType.VIDEO` (the wrapper's default is AUDIO), then feed
* the existing `H264Depacketizer` → AnnexB `.h264` → `muxToMp4`.
*
* All failures are best-effort and NEVER break the gateway's audio recording.
*/
import crypto from "node:crypto";
import {
createWriteStream,
promises as fsPromises,
type WriteStream,
} from "node:fs";
import { stat } from "node:fs/promises";
import path from "node:path";
import {
getVoiceConnection,
Networking,
type NetworkingState,
} from "@discordjs/voice";
import type Davey from "@snazzah/davey";
import { MediaType } from "@snazzah/davey";
import type { Client, VoiceChannel } from "discord.js-selfbot-v13";
import { createChildLogger } from "@/shared/logger/index";
import { H264Depacketizer, muxToMp4 } from "./videoReceiver.js";
const logger = createChildLogger("stream-watch");
const _log = createChildLogger("stream-watch-decrypt");
// Faithful ports of @discordjs/voice receive constants (dist/index.mjs).
const AUTH_TAG_LENGTH = 16;
const UNPADDED_NONCE_LENGTH = 4;
const HEADER_EXTENSION_BYTE = Buffer.from([190, 222]); // 0xBE 0xDE
/** Opus RTP payload type (120). Video arrives on non-opus payload types. */
const RTP_OPUS_PAYLOAD_TYPE = 120;
/**
* Video silence threshold — matches voice recording AfterSilence (4000ms).
* When no H264 video RTP packets arrive for this long, the current segment
* is closed and muxed to MP4 (like voice's "end of burst"). A new segment
* opens when packets resume.
*/
const VIDEO_SILENCE_MS = 4000;
/**
* Minimum segment duration — skip registering segments shorter than this
* (avoids creating tiny MP4 files from brief camera flashes).
*/
const VIDEO_MIN_SEGMENT_MS = 1000;
/** A watched user's live DAVE stream connection + file state. */
interface WatchState {
uid: string;
guildId: string;
channelId: string;
net: Networking;
depacketizer: H264Depacketizer;
filePath: string;
out: WriteStream;
bytesWritten: number;
lastPacketAt: number;
startedAt: number;
/** Timestamp of the current segment's first packet (for silence detection). */
segmentStartAt: number;
/** Per-watch segment counter — makes each segment's filename unique. */
segmentSeq: number;
/** True while a segment close is in-flight (rejects new packet writes). */
closing: boolean;
_diag?: { video: number; maxLen: number; pts: Set<number> };
_diagNext?: number;
}
const watches = new Map<string, WatchState>(); // key `${guildId}:${uid}`
const pendingServerId = new Map<string, string>(); // key `${guildId}:${uid}` -> rtc_server_id (from STREAM_CREATE)
let _client: Client | undefined;
let _rawAttached = false;
let _recordingsDir = "/var/lib/gmw/recordings";
export function setStreamWatchRecordingsDir(dir: string): void {
_recordingsDir = dir;
}
/**
* Silence detection interval — runs every 2s and checks each active watch for
* VIDEO_SILENCE_MS of inactivity (no H264 packets). On silence, the current
* segment is closed, muxed to MP4, registered in the DB, and uploaded.
* Matches voice recording's AfterSilence behavior.
*/
const _silenceInterval = setInterval(() => {
const now = Date.now();
for (const [watchKey, watch] of watches) {
if (!watch.out || !watch.segmentStartAt) continue;
const silenceDuration = now - watch.lastPacketAt;
if (silenceDuration < VIDEO_SILENCE_MS) continue;
const segmentDuration = now - watch.segmentStartAt;
logger.info(
{
userId: watch.uid,
watchKey,
segmentDurationMs: segmentDuration,
bytes: watch.bytesWritten,
},
`Video silence detected (${silenceDuration}ms) — closing segment`,
);
// Fire-and-forget: close + finalize the segment.
void closeCurrentSegment(watch);
}
}, 2_000);
// Prevent the interval from keeping the process alive.
if (_silenceInterval.unref) _silenceInterval.unref();
/** Test-only reset. */
export function __resetStreamWatchState(): void {
for (const [, w] of watches) closeWatch(w);
watches.clear();
pendingServerId.clear();
_rawAttached = false;
_client = undefined;
}
/** Bind the selfbot client (needed for STREAM_WATCH + raw). Idempotent. */
export function setStreamWatchClient(client: Client | undefined): void {
_client = client;
if (!client || _rawAttached) return;
_rawAttached = true;
client.on("raw", (packet: unknown) => {
try {
handleRaw(packet);
} catch (err) {
logger.warn(
{ err: err instanceof Error ? err.message : String(err) },
"stream-watch raw handler error (ignoring)",
);
}
});
}
const streamKeyFor = (channelId: string, guildId: string, uid: string) =>
`guild:${guildId}:${channelId}:${uid}`;
/** Parse `guild:<gid>:<chid>:<uid>` back into parts. */
function parseStreamKey(
key: string,
): { guildId: string; channelId: string; uid: string } | null {
const m = /^guild:(\d+):(\d+):(\d+)$/.exec(key);
if (!m) return null;
return { guildId: m[1], channelId: m[2], uid: m[3] };
}
function handleRaw(packet: unknown): void {
const p = packet as { t?: string; d?: Record<string, unknown> } | null;
if (!p?.t || !p?.d) return;
const t = p.t;
const streamKey = p.d.stream_key as string | undefined;
if (!streamKey) return;
const key = parseStreamKey(streamKey);
if (!key) return;
const watchKey = `${key.guildId}:${key.uid}`;
if (t === "STREAM_CREATE") {
const rtc = String(p.d.rtc_server_id ?? "");
pendingServerId.set(watchKey, rtc);
logger.info(
{ guildId: key.guildId, userId: key.uid, rtcServerId: rtc },
"STREAM_CREATE received (watch RTC authorized)",
);
} else if (t === "STREAM_SERVER_UPDATE") {
const endpoint = String(p.d.endpoint ?? "");
const token = String(p.d.token ?? "");
const rtc =
pendingServerId.get(watchKey) ?? String(p.d.rtc_server_id ?? "");
logger.info(
{ guildId: key.guildId, userId: key.uid, endpoint, rtcServerId: rtc },
"STREAM_SERVER_UPDATE received — connecting DAVE watch",
);
connectWatch(key.guildId, key.channelId, key.uid, endpoint, token, rtc);
} else if (t === "STREAM_DELETE") {
logger.info(
{ guildId: key.guildId, userId: key.uid },
"STREAM_DELETE received — closing watch",
);
pendingServerId.delete(watchKey);
closeWatchByKey(watchKey);
} else if (t === "STREAM_UPDATE") {
// paused / resumed — ignore
}
}
/**
* Initiate watching `userId`'s stream in `channel`. Sends STREAM_WATCH and
* waits for STREAM_CREATE / STREAM_SERVER_UPDATE (handled by handleRaw) to
* actually open the DAVE connection. Best-effort.
*/
export async function startStreamWatch(
channel: VoiceChannel,
userId: string,
): Promise<void> {
if (!_client) {
logger.warn("startStreamWatch: no client set");
return;
}
const watchKey = `${channel.guild.id}:${userId}`;
if (watches.has(watchKey)) return; // already watching
// Re-assert the bot is server-undeafened + unmuted BEFORE requesting video.
// A server-deafened bot is NOT sent the streamer's audiovisual RTP by
// Discord, so no video would arrive even though the DAVE watch reaches
// Ready. Server-deaf can silently revert on reconnect/restart, so this is
// re-invoked on every watch attempt. Dynamic import avoids a static module
// cycle (recorder ⇄ videoRecorder ⇄ streamWatchReceiver).
const { forceSelfServerUnmuteUndeafen } = await import("./recorder.js");
void forceSelfServerUnmuteUndeafen(_client, channel.guild);
const sk = streamKeyFor(channel.id, channel.guild.id, userId);
logger.info(
{ userId, guildId: channel.guild.id, channelId: channel.id, streamKey: sk },
"Sending STREAM_WATCH to authorize video receive",
);
try {
(
_client as unknown as { ws: { broadcast: (d: unknown) => unknown } }
).ws.broadcast({
op: 20, // STREAM_WATCH
d: { stream_key: sk },
});
} catch (err) {
logger.warn(
{ err: err instanceof Error ? err.message : String(err) },
"Failed to send STREAM_WATCH",
);
}
}
function connectWatch(
guildId: string,
channelId: string,
uid: string,
endpoint: string,
token: string,
serverId: string,
): void {
const watchKey = `${guildId}:${uid}`;
if (watches.has(watchKey)) return;
const botId = _client?.user?.id;
if (!botId) return;
// The bot's ACTIVE voice-session id, from the guild @discordjs/voice
// connection (the watch RTC identify must carry the SAME session the bot is
// joined in with). The gateway's audio connection is the voice session; the
// selfbot client's voice manager is NOT established anymore.
const guildConn = getVoiceConnection(guildId);
const sessionId =
(
guildConn as unknown as {
state?: {
networking?: {
state?: { connectionOptions?: { sessionId?: string } };
};
connectionOptions?: { sessionId?: string };
};
}
)?.state?.networking?.state?.connectionOptions?.sessionId ??
(
guildConn as unknown as {
state?: { connectionOptions?: { sessionId?: string } };
}
)?.state?.connectionOptions?.sessionId;
logger.info(
{ userId: uid, endpoint, serverId, hasSession: !!sessionId },
"Opening DAVE Networking to watch RTC",
);
// CRITICAL: for a STREAM-WATCH RTC the DAVE MLS group is keyed to
// `rtc_server_id - 1` (see Discord-RE StreamConnection.daveChannelId =
// BigInt(serverId)-1n), NOT the guild voice channel id. Passing the guild
// voice channel id as `connectionOptions.channelId` makes the Davey session
// derive the wrong MLS group → "WrongGroupId" on proposals → never Ready.
// `serverId` stays the rtc_server_id (used in the voice identify server_id).
const daveChannelId = (() => {
try {
return (BigInt(serverId) - 1n).toString();
} catch {
return channelId; // fallback: non-numeric serverId — use channel id
}
})();
const net = new Networking(
{
endpoint,
serverId,
userId: botId,
sessionId: sessionId ?? "none",
token,
channelId: daveChannelId,
},
{ daveEncryption: true, debug: true },
);
const uidProps = uid;
const netAny = net as unknown as {
state: NetworkingState & {
udp?: {
on: (e: string, cb: (m: Buffer) => void) => void;
off: (e: string, cb: (m: Buffer) => void) => void;
};
dave?: { session?: Davey.DAVESession };
connectionData?: {
encryptionMode?: string;
nonceBuffer?: Buffer;
secretKey?: Buffer | Uint8Array;
dave?: { session?: Davey.DAVESession };
};
};
};
const onMessage = (msg: Buffer): void => {
handleUdpMessage(watchKey, netAny, uidProps, msg);
};
net.on("stateChange", (oldState, newState) => {
const s = newState as { code?: number };
const code = s?.code;
const oldCode = (oldState as { code?: number })?.code;
logger.info(
{ userId: uid, code, oldCode, hasUdp: !!netAny.state.udp },
`watch-state ${oldCode}->${code}`,
);
if (code === 4 /* Ready */) {
const udp = netAny.state.udp;
if (udp) {
udp.on("message", onMessage);
logger.info(
{ userId: uid },
"DAVE watch READY — listening for video RTP",
);
} else {
logger.warn({ userId: uid }, "watch ready but no UDP socket");
}
}
});
net.on("error", (err) => {
logger.warn(
{ userId: uid, err: err instanceof Error ? err.message : String(err) },
"watch Networking error",
);
closeWatchByKey(watchKey);
});
net.on("debug", (msg) => {
logger.info({ userId: uid, msg }, "watch-djs-debug");
});
net.on("close", () => {
closeWatchByKey(watchKey);
});
watches.set(watchKey, {
uid,
guildId,
channelId,
net,
depacketizer: new H264Depacketizer(),
filePath: "",
out: undefined as unknown as WriteStream,
bytesWritten: 0,
lastPacketAt: Date.now(),
startedAt: Date.now(),
segmentStartAt: 0,
segmentSeq: 0,
closing: false,
});
}
/**
* Faithful port of @discordjs/voice `VoiceReceiver.decrypt` (audio path), but
* applied to a VIDEO packet. Strips the RTP header (12 + CSRC/extension), then
* legacy-AES-decrypts the wrapped payload, returning the DAVE-encrypted payload
* (still E2EE — Davey removes the NEXT layer).
*/
function legacyDecryptLayer(
buffer: Buffer,
mode: string,
nonce2: Buffer,
secretKey: Buffer | Uint8Array,
): Buffer {
buffer.copy(nonce2, 0, buffer.length - UNPADDED_NONCE_LENGTH);
let headerSize = 12;
const first = buffer.readUint8();
if ((first >> 4) & 1) headerSize += 4;
const header = buffer.subarray(0, headerSize);
const encrypted = buffer.subarray(
headerSize,
buffer.length - AUTH_TAG_LENGTH - UNPADDED_NONCE_LENGTH,
);
const authTag = buffer.subarray(
buffer.length - AUTH_TAG_LENGTH - UNPADDED_NONCE_LENGTH,
buffer.length - UNPADDED_NONCE_LENGTH,
);
switch (mode) {
case "aead_aes256_gcm_rtpsize": {
const decipheriv = crypto.createDecipheriv(
"aes-256-gcm",
secretKey,
nonce2,
);
decipheriv.setAAD(header);
decipheriv.setAuthTag(authTag);
return Buffer.concat([decipheriv.update(encrypted), decipheriv.final()]);
}
case "aead_xchacha20_poly1305_rtpsize": {
// Not statically importable (sodium-wrappers is an optional peer); the
// gateway negotiates aead_aes256_gcm_rtpsize in practice. If a server
// ever selects xchacha20 we log once and skip the burst rather than
// mis-decrypt.
return Buffer.alloc(0);
}
default: {
logger.warn(
{ mode },
"Unsupported video decryption mode (skipping burst)",
);
return Buffer.alloc(0);
}
}
}
/**
* Replicates @discordjs/voice `VoiceReceiver.parsePacket` (audio), parameterized
* for VIDEO: strips header + padding + RTP header-extension, decrypts the legacy
* AES layer, then DAVE-decrypts with `MediaType.VIDEO` (audio's wrapper hardcodes
* AUDIO). Returns the clear H264 RTP payload, or null on any failure.
*/
function decryptVideoPacket(
buffer: Buffer,
connectionData: {
encryptionMode?: string;
nonceBuffer?: Buffer;
secretKey?: Buffer | Uint8Array;
},
daveSession: { session?: Davey.DAVESession } | undefined,
userId: string,
): Buffer | null {
const { encryptionMode, nonceBuffer, secretKey } = connectionData;
if (!encryptionMode || !nonceBuffer || !secretKey || !daveSession?.session)
return null;
if (buffer.length < AUTH_TAG_LENGTH + UNPADDED_NONCE_LENGTH + 12) return null;
let packet: Buffer;
try {
packet = legacyDecryptLayer(buffer, encryptionMode, nonceBuffer, secretKey);
} catch (e) {
_log.debug(
{ userId, err: String(e), msgLen: buffer.length },
"legacyDecryptLayer threw",
);
return null; // per-packet legacy decrypt failures are transient
}
if (!packet || packet.length === 0) {
_log.debug(
{ userId, msgLen: buffer.length },
"legacyDecryptLayer returned empty",
);
return null;
}
// RTP padding (P bit) — strip trailing padding bytes (see parsePacket).
const hasPadding = buffer[0] !== 0 && Boolean(buffer[0] & 32);
if (hasPadding) {
const paddingAmount = packet[packet.length - 1];
if (paddingAmount < packet.length) {
packet = packet.subarray(0, packet.length - paddingAmount);
}
}
// RTP header extension (0xBE 0xDE at bytes 12-13) — skip extension words.
if (buffer.subarray(12, 14).compare(HEADER_EXTENSION_BYTE) === 0) {
const headerExtensionLength = buffer.subarray(14).readUInt16BE();
packet = packet.subarray(4 * headerExtensionLength);
}
// DAVE layer — decrypt as VIDEO (audio wrapper hardcodes MediaType.AUDIO).
// If the DAVE video decryptor is missing (e.g. MLS handshake only set up
// audio decryptors, or the streamer's video stream wasn't part of the
// welcome/proposals), decrypt() throws / returns empty. We fall back:
// 1) MediaType.VIDEO
// 2) MediaType.AUDIO (some Discord streams tag video packets as AUDIO)
// 3) passthrough — return the legacy-decrypted payload as-is (GoLive
// screen-share bursts are sometimes sent unencrypted above the AES
// layer; DAVE passthrough mode permits this).
try {
const decrypted = daveSession.session.decrypt(
userId,
MediaType.VIDEO,
packet,
);
if (decrypted && decrypted.length > 0) return decrypted;
_log.debug(
{ userId, decryptedLen: decrypted?.length, pktLen: packet.length },
"DAVE VIDEO decrypt returned empty/falsy",
);
} catch (e) {
_log.debug({ userId, err: String(e) }, "DAVE VIDEO decrypt threw");
// fall through to AUDIO attempt
}
try {
const decryptedAudio = daveSession.session.decrypt(
userId,
MediaType.AUDIO,
packet,
);
if (decryptedAudio && decryptedAudio.length > 0) return decryptedAudio;
_log.debug(
{ userId, decryptedLen: decryptedAudio?.length, pktLen: packet.length },
"DAVE AUDIO decrypt returned empty/falsy",
);
} catch (e) {
_log.debug({ userId, err: String(e) }, "DAVE AUDIO decrypt threw");
// fall through to passthrough
}
// Passthrough: assume unencrypted above AES layer (or decryptor unavailable).
_log.debug(
{
userId,
passthroughLen: packet.length,
firstBytes: packet.subarray(0, 8).toString("hex"),
},
"VIDEO passthrough (decrypt failed for all modes)",
);
return packet.length > 0 ? packet : null;
}
function handleUdpMessage(
watchKey: string,
net: {
state: {
connectionData?: {
encryptionMode?: string;
nonceBuffer?: Buffer;
secretKey?: Buffer | Uint8Array;
};
dave?: { session?: Davey.DAVESession };
};
},
uid: string,
msg: Buffer,
): void {
if (msg.length <= 12) return;
const payloadType = msg[1] & 127;
// Opus = 120. Anything else on the video/watch socket is video (H264 etc.).
if (payloadType === RTP_OPUS_PAYLOAD_TYPE) return;
const watch = watches.get(watchKey);
if (!watch) return;
const conn = net.state.connectionData ?? {};
// Diagnose: is a DAVE session actually attached, and is it ready to decrypt?
const dave = net.state.dave;
const s = (dave as { session?: { ready?: boolean } } | undefined)?.session;
const daveReady = Boolean(s?.ready);
const haveKey = Boolean(
conn.encryptionMode && conn.nonceBuffer && conn.secretKey,
);
// Aggregate stats — track distinct PTs and max packet size so we can tell
// whether real H264 (large, PT 96-127) arrives vs only small control packets.
const now = Date.now();
const stat =
watch._diag ??
({ video: 0, maxLen: 0, pts: new Set<number>() } as {
video: number;
maxLen: number;
pts: Set<number>;
});
stat.video += 1;
if (msg.length > stat.maxLen) stat.maxLen = msg.length;
stat.pts.add(payloadType);
watch._diag = stat;
const diagCount = stat.video;
const dueAt = watch._diagNext ?? 0;
if (diagCount === 1 || now >= dueAt) {
watch._diagNext = now + 20_000;
const pts = [...stat.pts].join(",");
const ssrc = msg.readUInt32BE(8);
const first = msg[0];
const flags = msg[1];
logger.info(
{
userId: uid,
video: stat.video,
maxLen: stat.maxLen,
pts,
haveKey,
daveAttached: !!dave?.session,
daveReady,
ssrc,
firstByte: first,
flagsByte: flags,
seq: msg.readUInt16BE(2),
hex: msg.subarray(0, 32).toString("hex"),
},
`VIDEO-PKT diag video=${stat.video} maxLen=${stat.maxLen} pts=[${pts}] haveKey=${haveKey} dave=${!!dave?.session} ready=${daveReady} ssrc=${ssrc} first=${first} flags=${flags} hex=${msg.subarray(0, 32).toString("hex")}`,
);
}
const decrypted = decryptVideoPacket(
msg,
conn,
// djs/voice stores the DAVESession at net.state.dave (NOT inside
// connectionData) — look it up at the top-level state so DAVE layer
// decrypt (MediaType.VIDEO) actually runs.
net.state.dave,
uid,
);
if (!decrypted) {
// Log why — no other visibility into per-packet decrypt failures.
_log.debug(
{
videoCount: stat?.video,
daveAttached: !!net.state.dave?.session,
msgLen: msg.length,
},
"decryptVideoPacket returned NULL — packet dropped",
);
return;
}
if (!watch.out) {
// First successful decrypt — open output file.
void openOutput(watchKey, watch, decrypted);
return;
}
// Reject writes while a segment close is in-flight (silence detected).
if (watch.closing) return;
watch.lastPacketAt = Date.now();
let nals: Buffer[];
try {
nals = watch.depacketizer.push(decrypted);
} catch {
return;
}
for (const nal of nals) {
watch.bytesWritten += nal.length;
if (!watch.out.write(nal)) {
watch.out.once("drain", () => {});
}
}
}
async function openOutput(
_watchKey: string,
watch: WatchState,
_first: Buffer,
): Promise<void> {
if (watch.out) return;
const dir = path.join(_recordingsDir, watch.uid);
await fsPromises.mkdir(dir, { recursive: true });
// Filename includes the per-watch segment counter so a new segment after
// silence never collides with the (possibly still in-flight) previous one.
const seq = watch.segmentSeq++;
const filePath = path.join(
dir,
`video-${watch.channelId}-${watch.startedAt}-${seq}.h264`,
);
watch.filePath = filePath;
watch.out = createWriteStream(filePath);
// Swallow EPIPE / ERR_STREAM_WRITE_AFTER_END on the segment file when a
// close races an in-flight UDP write — an unhandled stream 'error' here
// would crash the gateway (same class as the media EPIPE incidents).
watch.out.on("error", () => {});
watch.segmentStartAt = Date.now();
logger.info(
{ userId: watch.uid, path: filePath },
"Video segment opened (DAVE video RTP received)",
);
}
/**
* Close the current segment's write stream, mux the H264 to MP4, and
* register the recording in the DB + upload. After this, the watch is
* ready for a new segment (silence ended → packets resume).
* Mirrors voice recording's finalizeSegment + uploadRecordingSegment flow.
*/
async function closeCurrentSegment(watch: WatchState): Promise<void> {
if (!watch.out || watch.out.destroyed || watch.closing) return;
// Mark closing FIRST (synchronously) so the UDP handler stops writing.
watch.closing = true;
const filePath = watch.filePath;
const segmentStart = watch.segmentStartAt;
const bytes = watch.bytesWritten;
// Flush the write stream to disk.
const flushed = new Promise<void>((resolve) => {
if (!watch.out || watch.out.destroyed) {
resolve();
return;
}
watch.out.once("finish", () => resolve());
watch.out.end();
});
await flushed;
// Reset watch state for the next segment.
watch.out = undefined as unknown as WriteStream;
watch.filePath = "";
watch.bytesWritten = 0;
watch.segmentStartAt = 0;
watch.closing = false;
// Reset the depacketizer so a partial FU-A fragment from this segment
// doesn't bleed into the next segment (it only writes complete NALs).
watch.depacketizer.reset();
if (!filePath) return;
// Discard very short segments (<1s) — avoids creating tiny MP4 files.
const durationMs = Date.now() - segmentStart;
if (durationMs < VIDEO_MIN_SEGMENT_MS) {
logger.debug(
{ userId: watch.uid, durationMs },
"Video segment too short, discarding",
);
fsPromises.unlink(filePath).catch(() => {});
return;
}
// Mux H264 to MP4 and register in DB + upload.
try {
const mp4 = await muxToMp4(filePath);
await finalizeVideoSegment({
mp4Path: mp4,
userId: watch.uid,
guildId: watch.guildId,
channelId: watch.channelId,
startTime: segmentStart,
durationMs,
bytes,
});
logger.info(
{ userId: watch.uid, mp4, durationMs, bytes },
"Video segment finalized + uploaded",
);
} catch (err) {
logger.warn(
{
userId: watch.uid,
err: err instanceof Error ? err.message : String(err),
},
"Video segment finalize failed (raw .h264 kept)",
);
}
}
/**
* Register a completed video MP4 segment in the DB and upload it.
* Reuses the voice_recordings table (filename indicates video).
*/
async function finalizeVideoSegment(input: {
mp4Path: string;
userId: string;
guildId: string;
channelId: string;
startTime: number;
durationMs: number;
bytes: number;
}): Promise<void> {
const { mp4Path, userId, guildId, channelId, startTime, durationMs, bytes } =
input;
const segmentId = `video-${userId}-${startTime}`;
const fileName = path.basename(mp4Path);
try {
// Get file size and register in DB.
const fileStats = await stat(mp4Path);
const { insertVoiceRecording } = await import(
"../../shared/database/voiceRecordingRepo.js"
);
await insertVoiceRecording({
id: segmentId,
user_id: userId,
username: userId, // Will be enriched by frontend from user_profiles
avatar_url: null,
guild_id: guildId,
channel_id: channelId,
channel_name: null,
filename: fileName,
size_bytes: fileStats.size,
upload_status: "pending",
created_at: Date.now(),
});
// Upload MP4 to TeleUploader.
const { uploadToTele } = await import("../../shared/uploader.js");
const { config } = await import("../../shared/config/config.js");
const fileBuffer = await fsPromises.readFile(mp4Path);
const uploadResult = await uploadToTele({
buffer: fileBuffer,
filename: fileName,
contentType: "video/mp4",
uploadUrl: config.TELE_UPLOAD_URL,
retries: 3,
});
// Update DB with upload URL.
const { updateVoiceRecordingAsUploaded } = await import(
"../../shared/database/voiceRecordingRepo.js"
);
await updateVoiceRecordingAsUploaded(
segmentId,
uploadResult.url,
Date.now(),
);
logger.info(
{ segmentId, url: uploadResult.url, durationMs, bytes },
"Video segment uploaded successfully",
);
// Broadcast via EventBroadcaster.
const { _eventBroadcaster } = await import("./recorder.js");
if (_eventBroadcaster) {
_eventBroadcaster
.voiceRecordingUploaded({
id: segmentId,
user_id: userId,
username: userId,
avatar_url: null,
guild_id: guildId,
channel_id: channelId,
channel_name: null,
filename: fileName,
size_bytes: fileStats.size,
download_url: uploadResult.url,
upload_status: "uploaded",
created_at: Date.now(),
uploaded_at: Date.now(),
})
.catch(() => {});
}
} catch (err) {
const errorMsg = err instanceof Error ? err.message : String(err);
logger.error(
{ segmentId, error: errorMsg },
"Failed to upload video segment",
);
// Mark as failed in DB (best-effort).
const { updateVoiceRecordingAsFailed } = await import(
"../../shared/database/voiceRecordingRepo.js"
);
await updateVoiceRecordingAsFailed(segmentId, errorMsg).catch(() => {});
}
}
function closeWatch(watch: WatchState): void {
if (!watch) return;
try {
watch.net.destroy();
} catch {
/* ignore */
}
// Finalize any open segment (mux to MP4 + DB + upload).
if (watch.out && !watch.out.destroyed) {
void closeCurrentSegment(watch);
}
}
export function closeWatchByKey(watchKey: string): void {
const watch = watches.get(watchKey);
if (!watch) return;
watches.delete(watchKey);
pendingServerId.delete(watchKey);
closeWatch(watch);
}
/** Stop watching `userId` in `guildId` (streaming stopped / user left). */
export function stopStreamWatch(guildId: string, userId: string): void {
closeWatchByKey(`${guildId}:${userId}`);
}
/** Stop ALL watches for a guild (voice disconnect / channel untrack). */
export function stopAllStreamWatches(guildId: string): void {
for (const key of [...watches.keys()]) {
if (key.startsWith(`${guildId}:`)) closeWatchByKey(key);
}
}
services/discord-gateway/src/modules/voice-recording/transmitter.ts
-334
View File
@@ -1,334 +0,0 @@
import { spawn } from "node:child_process";
import { PassThrough } from "node:stream";
import { StreamType } from "@discordjs/voice";
import type Redis from "ioredis";
import { BACKEND_VOICE_TRANSMIT } from "../../shared/index.js";
import { createChildLogger } from "../../shared/logger/index.js";
import { discordPlayer } from "./player.js";
const logger = createChildLogger("transmitter");
/**
* Handles real-time PCM audio transmission from browser to Discord voice channel.
*
* Pipeline:
* Browser Mic → base64 PCM (24kHz mono s16le) → Redis →
* FFmpeg (encode to OggOpus) → discordPlayer (StreamType.OggOpus) → Discord Voice
*/
export class VoiceTransmitter {
private redisSub: Redis | null = null;
private pcmStream: PassThrough | null = null;
private ffmpegProcess: ReturnType<typeof spawn> | null = null;
private isActive = false;
private readonly TRANSMIT_CHANNEL = BACKEND_VOICE_TRANSMIT;
/** Queue for PCM chunks when backpressure is active */
private backpressureQueue: Buffer[] = [];
/** Max queued chunks before dropping oldest (prevents unbounded memory growth) */
private static readonly MAX_QUEUE = 500;
/** Serialise start/stop to prevent races between rapid toggle commands */
private gate = Promise.resolve();
/** Set true before sending SIGTERM so exit handler knows it's intentional */
private _expectedExit = false;
/** Auto-stop timer: if no PCM received for this long, stop transmitter */
private _activityTimer: ReturnType<typeof setTimeout> | null = null;
/** How long to wait without PCM before auto-stopping (10s) */
private static readonly ACTIVITY_TIMEOUT_MS = 10_000;
/** Max stderr bytes to retain for error reporting */
private static readonly MAX_STDERR_BYTES = 4096;
/**
* Start listening for PCM audio data from Redis and stream to Discord
*/
async start(redis: Redis): Promise<void> {
logger.info("Transmitter start requested");
// Serialise with stop() so concurrent start+stop don't tear each other down
const prev = this.gate;
let release: () => void = () => {};
this.gate = new Promise<void>((r) => {
release = r;
});
await prev;
try {
if (this.isActive) {
logger.warn("Voice transmitter already active");
return;
}
this.isActive = true;
// Create PCM input stream
this.pcmStream = new PassThrough();
this.pcmStream.setMaxListeners(32); // drain listeners accumulate during backpressure
// Voice teardown (stop / disconnect / ffmpeg exit) destroys this stream
// while Redis PCM messages may still be in flight. Without a listener,
// EPIPE / ERR_STREAM_DESTROYED / ERR_STREAM_WRITE_AFTER_END surface as
// an uncaughtException and crash the whole gateway.
this.pcmStream.on("error", (err: NodeJS.ErrnoException) => {
if (
err.code === "EPIPE" ||
err.code === "ERR_STREAM_DESTROYED" ||
err.code === "ERR_STREAM_WRITE_AFTER_END"
) {
logger.debug(
{ code: err.code },
"PCM stream closed during voice teardown — ignoring",
);
} else {
logger.error({ error: err.message }, "PCM stream error");
}
});
// Spawn FFmpeg to encode 24kHz mono PCM → OggOpus
// Input: 24kHz mono s16le (raw PCM)
// Output: OGG container with Opus audio
this.ffmpegProcess = spawn(
"ffmpeg",
[
"-f",
"s16le", // Input format: signed 16-bit little-endian
"-ar",
"24000", // Input sample rate: 24kHz
"-ac",
"1", // Input channels: mono
"-i",
"pipe:0", // Read from stdin
"-f",
"ogg", // Output format: OGG
"-c:a",
"libopus", // Codec: Opus
"-b:a",
"96k", // Bitrate: 96kbps
"-ar",
"48000", // Output sample rate: 48kHz
"-ac",
"2", // Output channels: stereo
"-application",
"lowdelay", // Low delay mode for real-time
"-frame_duration",
"20", // 20ms frames
"-packet_loss",
"0", // No packet loss expected
"pipe:1", // Write to stdout
],
{
stdio: ["pipe", "pipe", "pipe"],
},
);
// Pipe PCM data to FFmpeg stdin
if (this.ffmpegProcess.stdin) {
this.pcmStream.pipe(this.ffmpegProcess.stdin);
}
// Log FFmpeg stderr for debugging (cap to prevent unbounded growth)
const stderrChunks: Buffer[] = [];
let stderrLen = 0;
this.ffmpegProcess.stderr?.on("data", (chunk: Buffer) => {
stderrLen += chunk.length;
if (stderrLen <= VoiceTransmitter.MAX_STDERR_BYTES) {
stderrChunks.push(chunk);
}
});
this.ffmpegProcess.on("error", (err) => {
const msg =
err.message === "spawn ffmpeg ENOENT"
? "FFmpeg/avconv not found! Install ffmpeg in the container."
: err.message;
logger.error({ error: msg }, "FFmpeg process error");
});
this.ffmpegProcess.on("exit", (code) => {
// SIGTERM from stop() is expected — don't log as error
if (code !== 0 && !this._expectedExit) {
const stderr = Buffer.concat(stderrChunks).toString();
logger.error(
{ code, stderr: stderr.slice(-500) },
"FFmpeg exited with error — auto-stopping transmitter",
);
// FFmpeg crashed — auto-stop to prevent silent audio loss.
// Fire-and-forget; stop() serialises with the gate.
void this.stop();
} else {
logger.debug({ code }, "FFmpeg process exited");
}
});
// Play FFmpeg stdout (OggOpus) to Discord
if (this.ffmpegProcess.stdout) {
discordPlayer.playStream(this.ffmpegProcess.stdout, "browser-bridge", {
inputType: StreamType.OggOpus,
inlineVolume: true,
});
}
logger.info(
"Voice transmitter pipeline ready (PCM → FFmpeg → OggOpus → Discord)",
);
// Subscribe to Redis channel for PCM data
await redis.subscribe(this.TRANSMIT_CHANNEL);
// Assign AFTER subscription succeeds — prevents race with stop()
this.redisSub = redis;
logger.info(
{ channel: this.TRANSMIT_CHANNEL },
"Subscribed to transmit channel",
);
// Start activity timer — will auto-stop if no PCM arrives within timeout
this.resetActivityTimer();
this.redisSub.on("message", (channel, message) => {
if (
!this.isActive ||
channel !== this.TRANSMIT_CHANNEL ||
!this.pcmStream
)
return;
try {
const data = JSON.parse(message);
if (data.type === "pcm" && data.buffer) {
const pcmBuffer = Buffer.from(data.buffer, "base64");
const stream = this.pcmStream;
const canContinue = stream.write(pcmBuffer);
// Reset activity timer — PCM is flowing
this.resetActivityTimer();
// Backpressure: queue until drain (cap to prevent memory leak)
if (!canContinue) {
if (this.backpressureQueue.length >= VoiceTransmitter.MAX_QUEUE) {
// Drop oldest chunks to free memory — real-time audio,
// stale data is useless
const dropCount = Math.floor(VoiceTransmitter.MAX_QUEUE * 0.25);
this.backpressureQueue.splice(0, dropCount);
logger.debug(
{ dropped: dropCount },
"Backpressure overflow — dropping oldest PCM chunks",
);
}
this.backpressureQueue.push(pcmBuffer);
stream.once("drain", () => {
const currentStream = this.pcmStream;
if (!currentStream || !this.isActive) return;
// Flush queued chunks
while (this.backpressureQueue.length > 0) {
const queued = this.backpressureQueue.shift();
if (!queued) break;
try {
if (!currentStream.write(queued)) break;
} catch (err) {
logger.debug(
{
error: err instanceof Error ? err.message : String(err),
},
"PCM flush write failed during teardown — ignoring",
);
break;
}
}
});
}
}
} catch (err) {
logger.error({ error: err }, "Failed to process PCM data");
}
});
logger.info("Voice transmitter started");
} finally {
release?.();
}
}
/**
* Reset the voice activity timer. Called on every PCM chunk received.
* If no chunks arrive for ACTIVITY_TIMEOUT_MS, auto-stop the transmitter
* to prevent dead air and wasted resources.
*/
private resetActivityTimer(): void {
this.clearActivityTimer();
this._activityTimer = setTimeout(() => {
if (!this.isActive) return;
logger.warn(
{ timeoutMs: VoiceTransmitter.ACTIVITY_TIMEOUT_MS },
"Voice activity timeout — no PCM received, auto-stopping transmitter",
);
void this.stop();
}, VoiceTransmitter.ACTIVITY_TIMEOUT_MS);
}
private clearActivityTimer(): void {
if (this._activityTimer !== null) {
clearTimeout(this._activityTimer);
this._activityTimer = null;
}
}
/**
* Stop transmitting and clean up resources
*/
async stop(): Promise<void> {
logger.info("Transmitter stop requested");
// Serialise with start() so concurrent start+stop don't tear each other down
const prev = this.gate;
let release: () => void = () => {};
this.gate = new Promise<void>((r) => {
release = r;
});
await prev;
try {
if (!this.isActive) {
logger.debug("Voice transmitter already stopped");
return;
}
this.isActive = false;
this.clearActivityTimer();
this.backpressureQueue = [];
if (this.pcmStream) {
this.pcmStream.removeAllListeners("drain");
this.pcmStream.end();
this.pcmStream = null;
}
// Flag FFmpeg exit as expected so the exit handler doesn't log it as error
this._expectedExit = true;
if (this.ffmpegProcess) {
this.ffmpegProcess.kill("SIGTERM");
this.ffmpegProcess = null;
}
if (this.redisSub) {
await this.redisSub.unsubscribe(this.TRANSMIT_CHANNEL);
this.redisSub
.quit()
.catch((err) =>
logger.warn({ err }, "Failed to quit Redis subscriber"),
);
this.redisSub = null;
}
discordPlayer.stop("browser-bridge");
logger.info("Voice transmitter stopped");
} finally {
release?.();
}
}
/**
* Check if transmitter is currently active
*/
getStatus(): { active: boolean; channel: string } {
return {
active: this.isActive,
channel: this.TRANSMIT_CHANNEL,
};
}
}
export const voiceTransmitter = new VoiceTransmitter();
services/discord-gateway/src/modules/voice-recording/videoReceiver.ts
-455
View File
@@ -1,455 +0,0 @@
import { spawn } from "node:child_process";
import { createWriteStream, mkdirSync, type WriteStream } from "node:fs";
import { stat, unlink } from "node:fs/promises";
import path from "node:path";
import type { VoiceReceiver } from "@discordjs/voice";
import type { Client } from "discord.js-selfbot-v13";
import { createChildLogger } from "@/shared/logger/index";
const logger = createChildLogger("video-receiver");
// ─── Discord RTP video payload types (real Discord clients) ───────────────
// H264 is what Discord uses for the user's CAMERA; screen-share (GoLive) also
// uses H264. VP8/VP9/AV1 appear for some share types / forced codecs.
export const VIDEO_PAYLOAD_TYPES = new Set<number>([
96, 98, 101, 102, 106, 116, 126, 127,
]);
// H264 NAL unit types (single byte, after the 1-byte NAL header)
const NAL_SPS = 7;
const NAL_PPS = 8;
const NAL_IDR = 5;
// RTP H264 packet types (byte 0 of the payload)
const H264_STAP_A = 24;
const H264_FU_A = 28;
/**
* Pure RTP-H264 depacketizer → AnnexB (prefix NALs with 00 00 00 01 start codes).
*
* Input: RTP payload bytes (after the 12-byte RTP header, after decryption).
* Output: a list of AnnexB NAL buffers (each with start code) ready to append
* to a `.h264` elementary stream file.
*
* Reassembles FU-A fragmented NALs and STAP-A aggregated NALs into standalone
* NALs with start codes. Fragments that don't form a complete NAL are buffered.
*/
export class H264Depacketizer {
/** True after we've seen an SPS/PPS/IDR so we only start writing on a keyframe. */
private sawConfig = false;
/** Buffer for an in-progress FU-A NAL. */
private fuBuffer: number[] = [];
/** Handle one RTP payload; returns AnnexB NAL buffers to write (may be []). */
push(payload: Buffer): Buffer[] {
const out: Buffer[] = [];
if (payload.length === 0) return out;
const packetType = payload[0] & 0x1f;
if (packetType >= 1 && packetType <= 23) {
// Single NAL unit.
const nalType = payload[0] & 0x1f;
this.maybeEnableConfig(nalType);
const annexB = this.toAnnexB(payload);
if (this.sawConfig && annexB) out.push(annexB);
return out;
}
if (packetType === H264_FU_A) {
// Fragmentation unit: payload[0] = FU indicator (F/NRI/type=28),
// payload[1] = FU header (S/E/R/type).
const fuHeader = payload[1];
const start = (fuHeader & 0x80) !== 0;
const end = (fuHeader & 0x40) !== 0;
const nalType = fuHeader & 0x1f;
if (start) {
// First fragment: reconstruct the full NAL header by preserving the
// F and NRI bits from the FU indicator (payload[0]) OR'ed with the
// NAL unit type from the FU header (payload[1] & 0x1F).
const nalHeader = (payload[0] & 0xe0) | nalType;
this.fuBuffer = [nalHeader, ...payload.subarray(2)];
} else {
// Continuation. (Ignore if no buffer — a lost first fragment.)
if (this.fuBuffer.length === 0) {
// We missed the start; the NAL type unknown, skip this fragment.
return out;
}
if (payload.length > 2) {
this.fuBuffer.push(...payload.subarray(2));
}
}
if (end) {
const nal = Buffer.from(this.fuBuffer);
this.fuBuffer = [];
this.maybeEnableConfig(nalType);
if (this.sawConfig && nal.length > 0) {
out.push(this.toAnnexB(nal));
}
}
return out;
}
if (packetType === H264_STAP_A) {
// Aggregation packet: list of (2-byte NAL size, NAL data).
let offset = 1;
while (offset < payload.length) {
if (offset + 2 > payload.length) break;
const naluLength = payload.readUInt16BE(offset);
offset += 2;
if (offset + naluLength > payload.length) break;
const nalu = payload.subarray(offset, offset + naluLength);
offset += naluLength;
const nalType = nalu[0] & 0x1f;
this.maybeEnableConfig(nalType);
const annexB = this.toAnnexB(nalu);
if (this.sawConfig && annexB) out.push(annexB);
}
}
return out;
}
private maybeEnableConfig(nalType: number): void {
if (nalType === NAL_SPS || nalType === NAL_PPS || nalType === NAL_IDR) {
this.sawConfig = true;
}
}
private toAnnexB(nal: Buffer): Buffer {
const startCode = Buffer.from([0x00, 0x00, 0x00, 0x01]);
return Buffer.concat([startCode, nal]);
}
reset(): void {
this.sawConfig = false;
this.fuBuffer = [];
}
}
// ─── Per-user video burst state ────────────────────────────────────────────
interface VideoBurst {
userId: string;
depacketizer: H264Depacketizer;
/** video SSRC this burst is sourced from (may be camera or screenshare). */
ssrc: number;
createdAt: number;
filePath: string;
out: WriteStream;
/** count of AnnexB NAL bytes written (for logging/health). */
bytesWritten: number;
lastPacketAt: number;
}
/**
* Listens for other users' VIDEO RTP (camera + screen share) on the same UDP
* socket @discordjs/voice receives audio from, decrypts each video packet with
* the connection's secret key (via receiver.parsePacket), depacketizes H264 to
* AnnexB, and writes a raw `.h264` elementary stream per user-per-burst.
*
* Wraps `receiver.onUdpMessage` like screenShareAudio.ts. Packets with a video
* payload type are consumed here; ALL other packets (opus mic + screen-share
* audio) are delegated to the current handler (the existing wrapper chain).
*/
export function hookVideoReceiver(
receiver: VoiceReceiver,
client: Client,
recordingsDir: string,
): void {
const original = receiver.onUdpMessage;
const bursts = new Map<string, VideoBurst>();
// The UDP socket delivers video on SSRCs that are NOT the audioSSRC key of
// ssrcMap. Build a videoSSRC→userId index from VoiceUserData as it updates so
// we can attribute packets without SSRC-proximity guessing.
const videoSsrcToUser = new Map<number, string>();
receiver.ssrcMap.on("create", (data) => {
if (data.videoSSRC !== undefined) {
videoSsrcToUser.set(data.videoSSRC, data.userId);
logger.info(
{ userId: data.userId, videoSSRC: data.videoSSRC },
"Video SSRC appeared (camera/screenshare start)",
);
}
});
receiver.ssrcMap.on("update", (_old, neu) => {
if (neu.videoSSRC !== undefined) {
videoSsrcToUser.set(neu.videoSSRC, neu.userId);
}
});
/** Close a burst's file (flush + end) then mux to a playable mp4. */
const closeBurst = (userId: string): void => {
const burst = bursts.get(userId);
if (!burst) return;
bursts.delete(userId);
const { ssrc, bytesWritten, createdAt, filePath } = burst;
// Wait for the write stream to fully flush (file descriptor closed) before
// ffmpeg reads it — otherwise the mux can read a truncated tail.
const flushed = new Promise<void>((resolve) => {
burst.out.once("finish", () => resolve());
burst.out.end();
});
logger.info(
{ userId, ssrc, bytes: bytesWritten, durationMs: Date.now() - createdAt },
"Video burst closed",
);
// Phase B: remux the raw H264 elementary stream into a self-contained,
// playable MP4 (fast `-c copy`, no re-encode) and drop the raw file.
void flushed
.then(() => muxToMp4(filePath))
.then((mp4) => {
logger.info({ userId, ssrc, mp4 }, "Video muxed to mp4");
})
.catch((err) => {
logger.warn(
{
userId,
ssrc,
err: err instanceof Error ? err.message : String(err),
},
"Video mux to mp4 failed (keeping raw .h264)",
);
});
};
const openBurst = (userId: string, ssrc: number): VideoBurst | null => {
try {
const userDir = path.join(recordingsDir, userId);
mkdirSync(userDir, { recursive: true });
const filePath = path.join(userDir, `video-${ssrc}-${Date.now()}.h264`);
const out = createWriteStream(filePath);
const burst: VideoBurst = {
userId,
ssrc,
depacketizer: new H264Depacketizer(),
createdAt: Date.now(),
filePath,
out,
bytesWritten: 0,
lastPacketAt: Date.now(),
};
bursts.set(userId, burst);
logger.info({ userId, ssrc, filePath }, "Video burst opened");
return burst;
} catch (err) {
logger.warn(
{ userId, ssrc, err: err instanceof Error ? err.message : String(err) },
"Failed to open video burst file",
);
return null;
}
};
receiver.onUdpMessage = (msg: Buffer) => {
if (msg.length <= 8) {
original.call(receiver, msg);
return;
}
const payloadType = msg[1] & 127;
// DIAGNOSTIC: log any RTP packet whose payload type isn't Opus (120) so we
// can confirm whether Discord actually delivers video RTP to the bot.
// (Non-opus = video/experimental; if we see these, video IS arriving and the
// attribution/signaling below is the only gap.)
if (payloadType !== 120) {
logger.warn(
{ payloadType, ssrc: msg.readUInt32BE(8) },
"DIAG: non-opus RTP packet on voice socket",
);
}
if (!VIDEO_PAYLOAD_TYPES.has(payloadType)) {
// Not video — let the existing handler (audio + screen-share audio) take it.
original.call(receiver, msg);
return;
}
// ── It's a video RTP packet. Decrypt + depacketize + write. ──
const ssrc = msg.readUInt32BE(8);
// Attribute to a user.
let userId = videoSsrcToUser.get(ssrc);
if (!userId) {
// Fallback: maybe we registered the SSRC under an audio entry via the
// screen-share-audio hook, or proximity. Skip bot's own stream.
const candidate = inferVideoOwner(msg, receiver);
if (candidate && candidate.userId !== client.user?.id) {
userId = candidate.userId;
videoSsrcToUser.set(ssrc, candidate.userId);
}
}
if (!userId || userId === client.user?.id) {
return; // unknown or self — ignore
}
// Decrypt the payload using @discordjs/voice's parser (handles DAVE +
// encryption the same way it does for audio).
let decrypted: Buffer;
try {
const receiverAny = receiver as unknown as {
parsePacket: (
buffer: Buffer,
mode: string,
nonce: Buffer,
secretKey: Uint8Array,
userId: string,
) => Buffer;
connectionData: {
encryptionMode?: string;
nonceBuffer?: Buffer;
secretKey?: Uint8Array;
};
};
const { encryptionMode, nonceBuffer, secretKey } =
receiverAny.connectionData;
if (!encryptionMode || !nonceBuffer || !secretKey) return;
// decrypted payload = RTP header already stripped by parsePacket
decrypted = receiverAny.parsePacket(
msg,
encryptionMode,
nonceBuffer,
secretKey,
userId,
);
} catch {
// Decryption failed — transient; skip this packet.
return;
}
// Depacketize the (video) RTP payload → AnnexB.
let burst = bursts.get(userId);
if (!burst) {
const opened = openBurst(userId, ssrc);
if (!opened) return;
burst = opened;
}
burst.lastPacketAt = Date.now();
let nals: Buffer[];
try {
nals = burst.depacketizer.push(decrypted);
} catch {
return;
}
if (nals.length === 0) return;
for (const nal of nals) {
burst.bytesWritten += nal.length;
if (!burst.out.write(nal)) {
burst.out.once("drain", () => {});
}
}
};
// If a user's video SSRC disappears, close their burst after a short grace so
// the tail flushes.
receiver.ssrcMap.on("delete", (data) => {
// data = the deleted VoiceUserData (userId + audioSSRC). Close any open
// video burst for that user so the mux starts as soon as they stop sharing.
if (data?.userId) {
setTimeout(() => closeBurst(data.userId), 500);
}
});
setInterval(() => {
const now = Date.now();
for (const [userId, burst] of bursts) {
if (now - burst.lastPacketAt > 5000) {
closeBurst(userId);
}
}
}, 2000);
// Prevent the interval from keeping the process alive.
// (The gateway owns the process; a single 2s timer is negligible.)
}
/**
* Infer which user owns a video SSRC by proximity to their known audioSSRC
* (Discord allocates a user's video SSRC near their audio SSRC).
*/
function inferVideoOwner(
msg: Buffer,
receiver: VoiceReceiver,
): { userId: string } | null {
const ssrc = msg.readUInt32BE(8);
try {
const map = getSsrcInternalMap(receiver.ssrcMap);
if (!map) return null;
for (const [, data] of map.entries()) {
if (data.audioSSRC && Math.abs(data.audioSSRC - ssrc) < 400_000) {
return { userId: data.userId };
}
}
} catch {
// ignore
}
return null;
}
/** Access the private `_map` inside SSRCMap (mirrors screenShareAudio.ts). */
function getSsrcInternalMap(
ssrcMap: VoiceReceiver["ssrcMap"],
):
| Map<number, { userId: string; audioSSRC?: number; videoSSRC?: number }>
| undefined {
const asAny = ssrcMap as unknown as Record<string, unknown>;
// @discordjs/voice 0.19.x exposes the map as `map` (public) — but some
// versions used `_map`. Accept both so attribute fallback keeps working.
const m1 = asAny.map ?? asAny._map;
if (m1 instanceof Map) return m1 as Map<number, never>;
// SSRCMap could also be the fork's class — same shape.
const m2 = asAny.map;
if (m2 instanceof Map) return m2 as Map<number, never>;
return undefined;
}
/**
* Remux a raw H264 elementary stream into a playable MP4 with `-c copy` (no
* re-encode, fast). On success the raw `.h264` is removed and the `.mp4` path
* returned. On any failure the raw file is LEFT in place (caller keeps it).
*/
export async function muxToMp4(rawPath: string): Promise<string> {
const mp4Path = rawPath.replace(/\.h264$/, ".mp4");
await new Promise<void>((resolve, reject) => {
const proc = spawn(
"ffmpeg",
[
"-hide_banner",
"-loglevel",
"error",
"-f",
"h264",
"-i",
rawPath,
"-c",
"copy",
"-movflags",
"+faststart",
"-y",
mp4Path,
],
{ stdio: ["ignore", "ignore", "pipe"] },
);
let stderr = "";
proc.stderr?.on("data", (chunk) => {
stderr += chunk.toString();
});
proc.on("error", (err) => reject(err));
proc.on("close", (code) => {
if (code === 0) resolve();
else reject(new Error(`ffmpeg exit ${code ?? "?"}: ${stderr.trim()}`));
});
});
// Sanity check: only delete the raw file if the mp4 is non-empty.
const mp4Stat = await stat(mp4Path).catch(() => null);
if (!mp4Stat || mp4Stat.size === 0) {
throw new Error("mp4 mux produced an empty file");
}
await unlink(rawPath).catch(() => {});
return mp4Path;
}
services/discord-gateway/src/modules/voice-recording/videoRecorder.ts
-385
View File
@@ -1,385 +0,0 @@
/**
* Video recorder (camera + screen share) for OTHER members.
*
* ARCHITECTURE (2026-08-31, Phase D — DAVE-capable stream-watch):
*
* The previous selfbot-v13 path (discord.js-selfbot-v13 `joinStreamConnection`)
* is DEAD because Discord now mandates DAVE (E2EE) on all voice RTC connections.
* The selfbot's voice stack identifies with NO `max_dave_protocol_version` →
* Discord closes the WS with code 4017 ("E2EE/DAVE protocol required") on
* every attempt, making selfbot-based video-receive impossible.
*
* The working path:
* 1. `voiceStateUpdate` detects `streaming` on a member in a tracked channel.
* 2. `streamWatchReceiver.startStreamWatch(channel, userId)` is called.
* - It sends `STREAM_WATCH` (op 20) via the gateway WS.
* - Discord replies with `STREAM_CREATE` + `STREAM_SERVER_UPDATE` containing
* the stream's RTC endpoint/token/serverId.
* - `streamWatchReceiver` builds a `@discordjs/voice` `Networking` connection
* to the stream RTC with DAVE (the same class that handles guild audio).
* - It hooks the UDP `message` event, decrypts H264 video via Davey
* `MediaType.VIDEO`, and writes to `.h264` → muxed to `.mp4`.
* 3. `streamWatchReceiver.stopStreamWatch(guildId, userId)` tears everything down.
*
* This module is the thin orchestration layer: detect → delegate → teardown.
* All failures are best-effort and NEVER break the gateway's audio recording.
*/
import type { Client, VoiceChannel, VoiceState } from "discord.js-selfbot-v13";
import { createChildLogger } from "@/shared/logger/index";
import {
setStreamWatchClient,
setStreamWatchRecordingsDir,
startStreamWatch,
stopAllStreamWatches,
stopStreamWatch,
} from "./streamWatchReceiver.js";
const logger = createChildLogger("video-recorder");
/** Channels currently under audio recording, keyed by guildId. */
const watchedChannels = new Map<string, VoiceChannel>();
/**
* GUILD_CREATE.voice_states captured at gateway WS connect (the selfbot lib
* drops them), keyed by guildId. Consumed by scanExistingStreamers when the
* channel is tracked (voice join happens AFTER GUILD_CREATE, so we must store
* these until trackChannel runs).
*/
const pendingGuildCreateVoiceStates = new Map<
string,
Array<{
user_id?: unknown;
channel_id?: unknown;
self_video?: boolean;
self_stream?: boolean;
}>
>();
let _client: Client | undefined;
let _listenerAttached = false;
// DIAG: collect all raw event types for ~10s after first raw event
const _allRawTypes = new Set<string>();
let _rawDiagLogged = false;
/** Resolved at runtime from config (kept default as fallback). */
let _recordingsDir = "/var/lib/gmw/recordings";
export function setVideoRecordingsDir(dir: string) {
_recordingsDir = dir;
setStreamWatchRecordingsDir(dir);
}
/** Test-only: clear all module-level state. */
export function __resetVideoRecorderState(): void {
watchedChannels.clear();
pendingGuildCreateVoiceStates.clear();
_listenerAttached = false;
_client = undefined;
_allRawTypes.clear();
_rawDiagLogged = false;
}
/**
* Set the selfbot client and register the singleton voiceStateUpdate listener
* (idempotent). Called once at gateway bootstrap. Also passes the client to
* streamWatchReceiver for raw event handling (STREAM_CREATE/SERVER_UPDATE).
*/
export function setVideoRecorderClient(client: Client | undefined) {
_client = client;
setStreamWatchClient(client);
if (!client || _listenerAttached) return;
_listenerAttached = true;
client.on(
"voiceStateUpdate",
(oldState: VoiceState, newState: VoiceState) => {
void handleVoiceStateUpdate(oldState, newState);
},
);
// THE selfbot-v13 GUILD_CREATE handler DROPS `d.voice_states` (it only sends
// GUILD_SUBSCRIPTIONS_BULK, never parses the initial voice-state list), so
// `channel.members` is empty on gateway restart and users who were ALREADY
// in voice (camera/screen on) are never detected. Hook the raw GUILD_CREATE
// payload ourselves and scan the voice_states array — this is the only
// user-token-compatible source of "who is in the channel right now".
client.on("raw", (packet: unknown) => {
const p = packet as {
t?: string;
d?: { voice_states?: unknown; id?: unknown };
} | null;
if (!p?.t) return;
// DIAG: log all event types seen after READY (belt-and-suspenders)
if (_allRawTypes.size < 50) _allRawTypes.add(p.t);
// DIAG: after 10s, dump all raw types seen
if (!_rawDiagLogged) {
_rawDiagLogged = true;
setTimeout(() => {
logger.info(
{ rawEventTypes: [..._allRawTypes].sort() },
"RAW DIAG: all WS raw event types (10s snapshot)",
);
}, 10_000);
}
// Log significant events
if (
p.t === "GUILD_CREATE" ||
p.t === "READY" ||
p.t === "GUILD_MEMBERS_CHUNK" ||
(p.t.startsWith("GUILD") && !p.t.includes("UPDATE"))
) {
const d = p.d as Record<string, unknown>;
const dAny = d as unknown as {
broadcaster_user_ids?: unknown;
sessions?: unknown;
};
logger.info(
{
rawType: p.t,
hasVoiceStates: Array.isArray(d?.voice_states),
voiceStatesCount: Array.isArray(d?.voice_states)
? (d.voice_states as unknown[]).length
: -1,
broadcasterUserIds: dAny.broadcaster_user_ids,
sessionVoice: Array.isArray(dAny.sessions)
? (
dAny.sessions as Array<{
voice?: unknown;
session_id?: unknown;
}>
)
.map((s) => s.voice)
.filter(Boolean)
.slice(0, 5)
: undefined,
dKeys: Object.keys(d ?? {}).slice(0, 10),
},
"RAW diag: GUILD_CREATE/READY/GUILD_MEMBERS_CHUNK",
);
}
if (p.t !== "GUILD_CREATE") return;
const vs = p.d?.voice_states;
handleGuildCreateVoiceStates(p.d as Record<string, unknown>, vs);
});
}
/** Register a channel whose audio recording is active (video should follow). */
export function trackChannel(guildId: string, channel: VoiceChannel) {
watchedChannels.set(guildId, channel);
// When the bot joins a channel that ALREADY has people streaming (screen
// share / camera on BEFORE the bot joined), no voiceStateUpdate with
// streaming:true fires for them. Scan every member now and start watching
// anyone already streaming, so we don't miss a live stream in progress.
// Fire-and-forget: REST fallback may be needed if guild cache is cold.
void scanExistingStreamers(channel);
}
/**
* Scan all members currently in `channel` and start watching anyone whose
* voice state has `streaming` or `self_video` set (screen share / camera
* already on when the bot joined). Fire-and-forget per member — idempotent
* (startVideoRecording is a no-op if a watch already exists for that user).
*
* BUG FIX (2026-09-02 — user report "tidak mendeteksi user yg sudah ada"):
* `channel.members` is empty after a gateway restart because (a) the selfbot
* library's GUILD_CREATE handler DROPS `d.voice_states` (it only sends
* GUILD_SUBSCRIPTIONS_BULK, never parses the initial voice state list), and
* (b) guild member cache isn't populated yet. We now capture voice_states
* raw (Path C, the reliable selfbot-compatible source), fall back to
* guild.members.fetch() (Path D, may 403 for user tokens), then cache.
*/
export async function scanExistingStreamers(
channel: VoiceChannel,
): Promise<void> {
const selfId = _client?.user?.id;
const videoUsers: { userId: string; source: string }[] = [];
// ── Path A: guild cache has members ────────────────────────────────────
const members = channel.members;
if (members && members.size > 0) {
for (const [, member] of members) {
if (member.id === selfId) continue;
const vs = member.voice;
const hasVideo = Boolean(vs?.streaming || vs?.selfVideo);
if (hasVideo) {
videoUsers.push({ userId: member.id, source: "cache" });
}
}
}
// ── Path C: GUILD_CREATE.voice_states (buffered at WS connect) ─────────
// The selfbot lib drops `d.voice_states` from GUILD_CREATE, but we capture
// it in the raw listener. This is the ONLY selfbot-compatible source of
// "who is already in voice with video" — not gated by REST permissions.
if (videoUsers.length === 0) {
const vsList = pendingGuildCreateVoiceStates.get(channel.guild.id);
if (Array.isArray(vsList) && vsList.length > 0) {
for (const vs of vsList) {
const uid = String(vs.user_id ?? "");
if (!uid || uid === selfId) continue;
if (String(vs.channel_id ?? "") !== channel.id) continue;
const hasVideo = Boolean(vs.self_video || vs.self_stream);
if (!hasVideo) continue;
videoUsers.push({ userId: uid, source: "guild-create-voice-states" });
}
}
}
// ── Path D: cache empty → fetch guild members via REST ────────────────
// GET /channels/{id}/voice-states is BOT-ONLY since 2023 — user tokens get
// 404. The user-token-compatible way to discover who is in the voice
// channel: fetch the guild member list (GET /guilds/{id}/members — used
// elsewhere by the selfbot) which populates member.voice states, then scan
// `channel.members` again. NOTE: user tokens may also get 403 on this REST
// path (verified 2026-09-02) — Path C is the reliable one; Path D is a
// best-effort fallback for real bot tokens.
if (videoUsers.length === 0 && members && members.size === 0 && _client) {
try {
await channel.guild.members.fetch();
// Re-read members now that cache is warm.
const warmMembers = channel.members;
if (warmMembers && warmMembers.size > 0) {
for (const [, member] of warmMembers) {
if (member.id === selfId) continue;
const vs = member.voice;
const hasVideo = Boolean(vs?.streaming || vs?.selfVideo);
if (hasVideo) {
videoUsers.push({ userId: member.id, source: "rest-members" });
}
}
}
logger.info(
{
channelId: channel.id,
fetchedMembers: warmMembers?.size ?? 0,
foundVideo: videoUsers.length,
},
"Fetched guild members via REST for pre-existing video detection",
);
} catch (err) {
logger.warn(
{
channelId: channel.id,
err: err instanceof Error ? err.message : String(err),
},
"Failed to fetch guild members via REST — video detection may miss pre-existing users",
);
}
}
// ── Start watching detected video users ────────────────────────────────
let watched = 0;
for (const { userId, source } of videoUsers) {
startVideoRecording(channel, userId);
watched++;
logger.info({ userId, source }, "Detected pre-existing video user on join");
}
if (watched > 0) {
logger.info(
{ guildId: channel.guild.id, watched },
"Scanned pre-existing streamers on join",
);
}
}
/** Unregister a channel (voice stopped). Tear down any video watches. */
export function untrackChannel(guildId: string) {
watchedChannels.delete(guildId);
stopAllStreamWatches(guildId);
}
/**
* Parse `GUILD_CREATE.d.voice_states` and buffer them for when trackChannel
* runs. This is the authoritative, user-token-friendly list of everyone
* already in voice when the gateway connects — but GUILD_CREATE arrives BEFORE
* the voice join, so we can't start watches yet. Store per-guild and consume
* in scanExistingStreamers.
*/
function handleGuildCreateVoiceStates(
data: Record<string, unknown>,
voiceStates: unknown,
): void {
try {
const guildId = String(data.id ?? data.guild_id ?? "");
if (!guildId) return;
if (!Array.isArray(voiceStates)) return;
const list = voiceStates as Array<{
user_id?: unknown;
channel_id?: unknown;
self_video?: boolean;
self_stream?: boolean;
}>;
pendingGuildCreateVoiceStates.set(guildId, list);
logger.info(
{ guildId, voiceStates: list.length },
"Buffered GUILD_CREATE.voice_states for pre-existing video detection",
);
} catch (err) {
logger.warn(
{ err: err instanceof Error ? err.message : String(err) },
"GUILD_CREATE voice_states parse error (ignoring)",
);
}
}
async function handleVoiceStateUpdate(
_oldState: VoiceState,
newState: VoiceState,
): Promise<void> {
try {
const selfId = _client?.user?.id;
if (selfId && newState.id === selfId) return; // never record bot's own video
const guildId = newState.guild?.id;
const channel = guildId ? watchedChannels.get(guildId) : undefined;
if (!channel) return; // not an actively-recorded channel
const streaming = Boolean(newState.streaming);
const cameraOn = Boolean(newState.selfVideo);
const hasVideo = streaming || cameraOn;
const inChannel = newState.channelId === channel.id;
if (hasVideo && inChannel) {
startVideoRecording(channel, newState.id);
} else if (!inChannel || !hasVideo) {
stopVideoRecording(channel.guild.id, newState.id);
}
} catch (err) {
logger.warn(
{ err: err instanceof Error ? err.message : String(err) },
"Video voice-state handler error (ignoring)",
);
}
}
/**
* Begin watching the video (camera / screen share) of `userId` in `channel`.
* Delegates to `streamWatchReceiver` which handles STREAM_WATCH + DAVE connection.
*/
export function startVideoRecording(
channel: VoiceChannel,
userId: string,
): void {
try {
const selfId = _client?.user?.id;
if (selfId && userId === selfId) return;
void startStreamWatch(channel, userId);
} catch (err) {
logger.warn(
{
userId,
guildId: channel.guild.id,
err: err instanceof Error ? err.message : String(err),
},
"startVideoRecording error (best-effort, ignoring)",
);
}
}
/** Stop watching `userId`'s video in `guildId`. */
export function stopVideoRecording(guildId: string, userId: string): void {
stopStreamWatch(guildId, userId);
}
/** Tear down ALL video watches for a guild (e.g. on voice disconnect). */
export function stopAllVideoRecordings(guildId: string): void {
stopAllStreamWatches(guildId);
}
services/discord-gateway/src/modules/voice-recording/voiceController.ts
-392
View File
@@ -1,392 +0,0 @@
import type { VoiceConnection } from "@discordjs/voice";
import { VoiceConnectionStatus } from "@discordjs/voice";
import type { Client, Guild, VoiceChannel } from "discord.js-selfbot-v13";
import {
deleteVoiceAutoReconnect,
listVoiceAutoReconnects,
upsertVoiceAutoReconnect,
} from "@/shared/database/voiceAutoReconnectRepo.js";
import { AppError } from "@/shared/errors/index";
import { createChildLogger } from "@/shared/logger/index";
import { discordPlayer } from "./player.js";
import { startRecording, stopRecording } from "./recorder.js";
const logger = createChildLogger("voice-controller");
// ─── Types ───────────────────────────────────────────────────────────────
export interface GuildVoiceState {
guildId: string;
channelId: string;
channelName: string;
connectedAt: number;
}
export interface VoiceStatus {
ready: boolean;
connected: boolean;
activeGuildId: string | null;
activeChannelId: string | null;
activeChannelName: string | null;
/** Multi-guild: list of all active connections */
connections: GuildVoiceState[];
}
export interface DisconnectOptions {
/** Clear the persisted auto-reconnect row (explicit user leave). */
clearPersisted?: boolean;
/** Mark the disconnect as intentional so the rejoin monitor stops. */
intentional?: boolean;
}
const REJOIN_MAX_ATTEMPTS = 5;
const REJOIN_BASE_DELAY_MS = 2_000;
const REJOIN_MAX_DELAY_MS = 30_000;
// ─── VoiceController ─────────────────────────────────────────────────────
export class VoiceController {
private connections = new Map<string, GuildVoiceState>();
private connecting = new Set<string>();
/** Guilds whose disconnect was intentional (user leave / shutdown). */
private intentionalGuilds = new Set<string>();
/** Last known desired voice state, kept independent of live connection map. */
private rejoinState = new Map<string, GuildVoiceState>();
/** Current live connection per guild — for stale-callback invalidation. */
private liveConnections = new Map<string, VoiceConnection>();
/** Per-guild rejoin attempt counters + timers. */
private rejoinAttempts = new Map<string, number>();
private rejoinTimers = new Map<string, NodeJS.Timeout>();
constructor(private readonly client: Client) {}
getStatus(): VoiceStatus {
logger.debug("getStatus called");
const primaryGuildId = this.connections.keys().next().value ?? null;
const primary = primaryGuildId
? this.connections.get(primaryGuildId)
: undefined;
return {
ready: this.client.isReady(),
connected: this.connections.size > 0,
activeGuildId: primary?.guildId ?? null,
activeChannelId: primary?.channelId ?? null,
activeChannelName: primary?.channelName ?? null,
connections: Array.from(this.connections.values()),
};
}
async connect(guildId: string, channelId: string): Promise<VoiceStatus> {
logger.info({ guildId, channelId }, "connect called");
if (!this.client.isReady()) {
throw new AppError(
"Discord client is not ready",
"CLIENT_NOT_READY",
409,
);
}
if (this.connecting.has(guildId)) {
throw new AppError(
`Voice connection for guild ${guildId} is already in progress`,
"CONNECT_IN_PROGRESS",
409,
);
}
this.connecting.add(guildId);
try {
// Disconnect existing connection for this guild first
if (this.connections.has(guildId)) {
await this.disconnectGuild(guildId, { intentional: true });
}
// Clear any outstanding rejoin work for this guild — we are (re)joining now.
this.clearRejoin(guildId);
this.intentionalGuilds.delete(guildId);
const guild = this.getGuild(guildId);
const channel =
guild.channels.cache.get(channelId) ??
(await guild.channels.fetch(channelId).catch(() => null));
if (!channel) {
// Channel no longer exists — drop the persisted row so we don't retry forever.
await deleteVoiceAutoReconnect(guildId).catch(() => {});
throw new AppError(
"Voice channel not found",
"VOICE_CHANNEL_NOT_FOUND",
404,
);
}
if (channel.type !== "GUILD_VOICE") {
throw new AppError(
"Selected channel is not a voice channel",
"INVALID_CHANNEL_TYPE",
400,
);
}
const connection = await startRecording(
this.client,
channel as VoiceChannel,
);
if (!connection) {
throw new AppError(
"Failed to connect to voice channel",
"VOICE_CONNECT_FAILED",
500,
);
}
// If this is the first connection, set it as the player's connection
if (this.connections.size === 0) {
discordPlayer.setConnection(connection as VoiceConnection);
}
const state: GuildVoiceState = {
guildId,
channelId,
channelName: channel.name,
connectedAt: Date.now(),
};
this.connections.set(guildId, state);
this.rejoinState.set(guildId, state);
this.liveConnections.set(guildId, connection as VoiceConnection);
// Persist desired voice state for auto-reconnect (restart/reboot/kick).
const now = Date.now();
await upsertVoiceAutoReconnect({
guild_id: guildId,
channel_id: channelId,
channel_name: channel.name,
connected_at: now,
updated_at: now,
}).catch((err) => {
logger.warn(
{ err: err instanceof Error ? err.message : String(err) },
"Failed to persist voice auto-reconnect state",
);
});
this.monitorConnection(guildId, connection);
logger.info(
{ guildId, channelId, channelName: channel.name },
"Voice connected",
);
return this.getStatus();
} finally {
this.connecting.delete(guildId);
}
}
/**
* Auto-reconnect on startup/restart: rejoin every persisted voice channel.
* Called from the client `ready` handler. Non-fatal per-guild.
*/
async autoReconnect(): Promise<void> {
let states: Array<{
guild_id: string;
channel_id: string;
channel_name: string | null;
}> = [];
try {
states = await listVoiceAutoReconnects();
} catch (err) {
logger.warn(
{ err: err instanceof Error ? err.message : String(err) },
"Failed to load voice auto-reconnect states",
);
return;
}
if (states.length === 0) {
logger.info("No persisted voice state — skipping auto-reconnect");
return;
}
logger.info(
{ count: states.length },
"Auto-reconnecting to persisted voice channels",
);
// Fire in parallel; each failure is logged but doesn't block the others.
await Promise.allSettled(
states.map(async (s) => {
try {
await this.connect(s.guild_id, s.channel_id);
} catch (err) {
logger.error(
{
guildId: s.guild_id,
channelId: s.channel_id,
channelName: s.channel_name,
err: err instanceof Error ? err.message : String(err),
},
"Auto-reconnect failed for guild",
);
}
}),
);
}
async disconnect(opts?: DisconnectOptions): Promise<VoiceStatus> {
// Default semantics for graceful shutdown: intentional (no rejoin during
// teardown) but keep the persisted row so we rejoin on next boot.
const effective = opts ?? { intentional: true };
logger.info({ opts: effective }, "disconnect called");
const guildIds = Array.from(this.connections.keys());
for (const gid of guildIds) {
await this.disconnectGuild(gid, effective);
}
discordPlayer.stop();
return this.getStatus();
}
async disconnectGuild(
guildId: string,
opts?: DisconnectOptions,
): Promise<void> {
logger.info({ guildId, opts }, "disconnectGuild called");
if (opts?.intentional) {
this.intentionalGuilds.add(guildId);
}
if (opts?.clearPersisted) {
await deleteVoiceAutoReconnect(guildId).catch(() => {});
// Manual leave — forget the desired state entirely.
this.rejoinState.delete(guildId);
this.intentionalGuilds.add(guildId);
}
if (this.connections.has(guildId)) {
stopRecording(guildId);
this.connections.delete(guildId);
}
this.liveConnections.delete(guildId);
this.clearRejoin(guildId);
}
// ─── Rejoin monitor ───────────────────────────────────────────────────
/**
* Watch a live connection. If it drops WITHOUT us marking the guild as
* intentional (user leave / shutdown), schedule a full rejoin with backoff.
*/
private monitorConnection(
guildId: string,
connection: VoiceConnection,
): void {
const onDrop = () => {
// Ignore callbacks from a stale connection that has since been replaced.
if (this.liveConnections.get(guildId) !== connection) {
return;
}
// This connection object is dead.
this.liveConnections.delete(guildId);
if (this.connections.get(guildId)?.guildId === guildId) {
this.connections.delete(guildId);
}
if (this.intentionalGuilds.has(guildId)) {
logger.info({ guildId }, "Intentional disconnect — no auto-rejoin");
return;
}
logger.warn(
{ guildId },
"Voice dropped unexpectedly — scheduling rejoin",
);
this.scheduleRejoin(guildId);
};
connection.on(VoiceConnectionStatus.Disconnected, onDrop);
connection.on(VoiceConnectionStatus.Destroyed, onDrop);
}
private scheduleRejoin(guildId: string): void {
if (this.rejoinTimers.has(guildId)) {
return; // already scheduled
}
if (this.intentionalGuilds.has(guildId)) {
return;
}
const attempt = (this.rejoinAttempts.get(guildId) ?? 0) + 1;
this.rejoinAttempts.set(guildId, attempt);
const persisted = this.rejoinState.get(guildId);
if (!persisted) {
// Desired state was cleared — nothing to rejoin.
this.clearRejoin(guildId);
return;
}
if (attempt > REJOIN_MAX_ATTEMPTS) {
logger.error(
{ guildId, channelId: persisted.channelId, attempt },
"Rejoin attempts exhausted — keeping persisted state for next restart",
);
this.clearRejoin(guildId);
return;
}
const delay = Math.min(
REJOIN_BASE_DELAY_MS * 2 ** (attempt - 1),
REJOIN_MAX_DELAY_MS,
);
logger.info(
{ guildId, channelId: persisted.channelId, attempt, delay },
"Scheduling voice rejoin",
);
const timer = setTimeout(() => {
this.rejoinTimers.delete(guildId);
this.connect(guildId, persisted.channelId)
.then(() => {
this.rejoinAttempts.delete(guildId);
logger.info({ guildId }, "Auto-rejoin succeeded");
})
.catch((err) => {
logger.warn(
{
guildId,
err: err instanceof Error ? err.message : String(err),
attempt,
},
"Auto-rejoin attempt failed",
);
// Schedule the next attempt (the monitor is gone, so drive it here).
this.scheduleRejoin(guildId);
});
}, delay);
this.rejoinTimers.set(guildId, timer);
}
private clearRejoin(guildId: string): void {
const timer = this.rejoinTimers.get(guildId);
if (timer) {
clearTimeout(timer);
this.rejoinTimers.delete(guildId);
}
this.rejoinAttempts.delete(guildId);
}
private getGuild(guildId: string): Guild {
const guild = this.client.guilds.cache.get(guildId);
if (!guild) {
throw new AppError("Guild not found", "GUILD_NOT_FOUND", 404);
}
return guild;
}
}
services/discord-gateway/src/modules/voice-recording/voiceTranscriber.ts
-55
View File
@@ -1,55 +0,0 @@
// ─── Voice Transcription — AI-powered speech-to-text for voice recordings ────
import { createReadStream } from "node:fs";
import OpenAI from "openai";
import { createChildLogger } from "@/shared/logger/index";
import { config } from "../../shared/config/config.js";
const logger = createChildLogger("voice-transcriber");
/**
* Transcribe a voice recording OGG file using OpenAI Whisper API.
* Returns the transcribed text or null on failure.
*/
export async function transcribeRecording(
oggPath: string,
): Promise<string | null> {
if (!config.AI_VOICE_TRANSCRIPTION_ENABLED) return null;
if (!config.AI_LLM_API_KEY) {
logger.warn("AI_LLM_API_KEY not set, skipping transcription");
return null;
}
try {
const openai = new OpenAI({
apiKey: config.AI_LLM_API_KEY,
baseURL: config.AI_LLM_BASE_URL,
maxRetries: 2,
timeout: 120_000,
});
const response = await openai.audio.transcriptions.create({
file: createReadStream(oggPath),
model: config.AI_VOICE_TRANSCRIPTION_MODEL,
// No `language` param → Whisper auto-detects (handles id/en mixed).
// `json` (not `text`): 9router/omniroute proxies only json/verbose_json
// transcription responses; `text` returns 400 through the router.
response_format: "json",
});
// With response_format "json", the SDK returns an object with `text`.
const text =
(typeof response === "string" ? response : response.text)?.trim() || null;
if (!text) {
logger.warn({ oggPath }, "Empty transcription result");
return null;
}
logger.info({ oggPath, length: text.length }, "Transcription completed");
return text;
} catch (error) {
const msg = error instanceof Error ? error.message : String(error);
logger.error({ oggPath, error: msg }, "Transcription failed");
return null;
}
}
services/discord-gateway/src/shared/config/index.ts
-50
View File
@@ -49,32 +49,6 @@ export const configSchema = z
.transform((v) => v.split(",").filter(Boolean))
.describe("User IDs to skip AI analysis for (captured but not analyzed)"),
// ── Legacy voice ─────────────────────────────────────────────────────
VOICE_GUILD_ID: z.string().min(1).optional(),
VOICE_CHANNEL_ID: z.string().min(1).optional(),
// ── Recording ────────────────────────────────────────────────────────
RECORDINGS_DIR: z.string().default("./recordings"),
RECORDING_SEGMENT_MS: z.coerce.number().positive().default(5000),
// ── Decoder ──────────────────────────────────────────────────────────
DECODER_ROTATE_MS: z.coerce.number().positive().default(5000),
DECODER_COOLDOWN_MS: z.coerce.number().positive().default(30000),
// ── Audio ────────────────────────────────────────────────────────────
// AfterSilence: how long a voice burst may stay silent before the receive
// stream ends the segment. Raised 3000→4000 so natural pauses in speech
// (thinking gaps, interruptions) don't split one utterance into multiple
// segments ("terpotong"). Tunable via env; larger = fewer splits but a
// longer silent tail on each recording.
AUDIO_STREAM_SILENCE_DURATION_MS: z.coerce
.number()
.positive()
.default(4000),
PACKET_FILTER_MIN_SIZE: z.coerce.number().positive().default(8),
OPUS_FRAME_SIZE: z.coerce.number().positive().default(960),
AUDIO_SAMPLE_RATE: z.coerce.number().positive().default(48000),
AUDIO_CHANNELS: z.coerce.number().positive().default(2),
AVATAR_SIZE: z.coerce.number().positive().default(64),
// ── Server ───────────────────────────────────────────────────────────
@@ -148,17 +122,8 @@ export const configSchema = z
TINYFISH_SEARCH_TIMEOUT_MS: z.coerce.number().positive().default(10000),
TINYFISH_SEARCH_LOCATION: z.string().min(1).default("US"),
TINYFISH_SEARCH_LANGUAGE: z.string().min(1).default("en"),
// ── Voice PCM WebSocket (direct gateway→backend, bypasses Redis) ────
VOICE_PCM_WS_ENABLED: z
.string()
.optional()
.transform((v) => v === "true")
.default(true),
BACKEND_WS_URL: z.string().default("ws://backend:3000/ws"),
BACKEND_WS_TOKEN: z.string().optional().default(""),
// ── Connection ───────────────────────────────────────────────────────
VOICE_CONNECTION_TIMEOUT_MS: z.coerce.number().positive().default(15000),
RECONNECT_TIMEOUT_MS: z.coerce.number().positive().default(5000),
// ── Attachments ─────────────────────────────────────────────────────
@@ -390,18 +355,6 @@ export const configSchema = z
// tuning one never starves the other.
PISCINA_MEDIA_MAX_THREADS: z.coerce.number().int().positive().default(2),
// ── Voice Transcription ────────────────────────────────────────────────
AI_VOICE_TRANSCRIPTION_ENABLED: z
.string()
.optional()
.transform((v) => v === "true")
.default(false),
// Whisper model routed through the LLM base URL. Through 9router/omniroute
// use a provider-qualified id that has credentials (e.g.
// openrouter/openai/whisper-1) — bare `whisper-1` maps to the `openai`
// provider which the router rejects with "No credentials for provider".
AI_VOICE_TRANSCRIPTION_MODEL: z.string().default("whisper-1"),
// ── Auto Delete ─────────────────────────────────────────────────────
AUTO_DELETE_FLAGGED_ENABLED: z
.string()
@@ -444,7 +397,6 @@ export const configSchema = z
// ── Retention ───────────────────────────────────────────────────────
RETENTION_MESSAGES_DAYS: z.coerce.number().int().min(0).default(0),
RETENTION_ATTACHMENTS_DAYS: z.coerce.number().int().min(0).default(0),
RETENTION_VOICE_DAYS: z.coerce.number().int().min(0).default(0),
RETENTION_CLEANUP_INTERVAL_MS: z.coerce
.number()
.positive()
@@ -483,7 +435,6 @@ export const configSchema = z
export type AppConfig = z.infer<typeof configSchema> & {
EFFECTIVE_TEXT_GUILD_ID?: string;
EFFECTIVE_VOICE_GUILD_ID?: string;
EFFECTIVE_MONITOR_GUILD_IDS: string[];
};
@@ -493,7 +444,6 @@ export function loadConfig(env: NodeJS.ProcessEnv = process.env): AppConfig {
return {
...parsed,
EFFECTIVE_TEXT_GUILD_ID: parsed.TEXT_GUILD_ID ?? parsed.MONITOR_GUILD_ID,
EFFECTIVE_VOICE_GUILD_ID: parsed.VOICE_GUILD_ID,
EFFECTIVE_MONITOR_GUILD_IDS:
parsed.MONITOR_GUILD_IDS.length > 0
? parsed.MONITOR_GUILD_IDS
services/discord-gateway/src/shared/database/schema/voice.ts
-9
View File
@@ -1,9 +0,0 @@
import { pgVoiceRecordingsTable } from "../../../shared/index.js";
// Re-export shared table
export { pgVoiceRecordingsTable };
export const voiceRecordingsTable = pgVoiceRecordingsTable;
// Types
export type VoiceRecording = typeof voiceRecordingsTable.$inferSelect;
export type VoiceRecordingInsert = typeof voiceRecordingsTable.$inferInsert;
services/discord-gateway/src/shared/database/voiceAutoReconnectRepo.ts
-59
View File
@@ -1,59 +0,0 @@
import { eq } from "drizzle-orm";
import { getDatabase } from "./drizzle.js";
import {
type VoiceAutoReconnect,
type VoiceAutoReconnectInsert,
voiceAutoReconnectTable,
} from "./schema.js";
/**
* Upsert a guild's desired voice state.
* Called after a successful connect so the gateway can rejoin after a
* restart/reboot or unexpected drop.
*/
export async function upsertVoiceAutoReconnect(
record: VoiceAutoReconnectInsert,
): Promise<VoiceAutoReconnect | undefined> {
const db = getDatabase();
const [row] = await db
.insert(voiceAutoReconnectTable)
.values(record)
.onConflictDoUpdate({
target: voiceAutoReconnectTable.guild_id,
set: {
channel_id: record.channel_id,
channel_name: record.channel_name ?? null,
connected_at: record.connected_at,
updated_at: record.updated_at,
},
})
.returning();
return row;
}
/** List all persisted desired voice states (for startup auto-reconnect). */
export async function listVoiceAutoReconnects(): Promise<VoiceAutoReconnect[]> {
const db = getDatabase();
return db.select().from(voiceAutoReconnectTable);
}
/** Clear a guild's persisted voice state (explicit manual leave). */
export async function deleteVoiceAutoReconnect(guildId: string): Promise<void> {
const db = getDatabase();
await db
.delete(voiceAutoReconnectTable)
.where(eq(voiceAutoReconnectTable.guild_id, guildId));
}
/** Fetch a single guild's persisted state, if any. */
export async function getVoiceAutoReconnect(
guildId: string,
): Promise<VoiceAutoReconnect | undefined> {
const db = getDatabase();
const rows = await db
.select()
.from(voiceAutoReconnectTable)
.where(eq(voiceAutoReconnectTable.guild_id, guildId))
.limit(1);
return rows[0];
}
services/discord-gateway/src/shared/database/voiceRecordingRepo.ts
-162
View File
@@ -1,162 +0,0 @@
import { desc, eq } from "drizzle-orm";
import { createChildLogger } from "@/shared/logger/index";
import { getDatabase } from "./drizzle.js";
import {
type VoiceRecording,
type VoiceRecordingInsert,
voiceRecordingsTable,
} from "./schema.js";
const logger = createChildLogger("voice-recording-repo");
interface QueryBuilder<T = unknown> extends PromiseLike<T> {
from(...args: unknown[]): QueryBuilder<T>;
where(...args: unknown[]): QueryBuilder<T>;
orderBy(...args: unknown[]): QueryBuilder<T>;
limit(...args: unknown[]): QueryBuilder<T>;
offset(...args: unknown[]): QueryBuilder<T>;
values(...args: unknown[]): QueryBuilder<T>;
onConflictDoNothing(...args: unknown[]): QueryBuilder<T>;
returning(...args: unknown[]): QueryBuilder<T>;
set(...args: unknown[]): QueryBuilder<T>;
}
interface RecordingDatabase {
select<T = unknown[]>(...args: unknown[]): QueryBuilder<T>;
insert<T = unknown>(...args: unknown[]): QueryBuilder<T>;
update(...args: unknown[]): QueryBuilder<unknown>;
}
function db(): RecordingDatabase {
return getDatabase() as unknown as RecordingDatabase;
}
export async function insertVoiceRecording(
recording: VoiceRecordingInsert,
): Promise<void> {
try {
await db()
.insert(voiceRecordingsTable)
.values(recording)
.onConflictDoNothing();
} catch (error) {
// Drizzle wraps PG errors — dig into cause + enumerable props for the real PG error
const err = error as Error & Record<string, unknown>;
const detail: Record<string, unknown> = {
message: err.message,
name: err.name,
};
// node-postgres native error props
for (const k of [
"code",
"detail",
"schema",
"table",
"constraint",
"severity",
]) {
if (err[k] !== undefined) detail[k] = err[k];
}
// drizzle may stash the original in .cause
const cause = err.cause;
if (cause instanceof Error) {
const causeErr = cause as Error & Record<string, unknown>;
detail.cause = {
message: causeErr.message,
name: causeErr.name,
code: causeErr.code,
detail: causeErr.detail,
constraint: causeErr.constraint,
};
}
logger.error(
{ id: recording.id, error: detail },
"Failed to insert voice recording",
);
throw error;
}
}
export async function updateVoiceRecordingAsUploaded(
id: string,
downloadUrl: string,
uploadedAt: number,
filename?: string,
sizeBytes?: number,
): Promise<void> {
try {
await db()
.update(voiceRecordingsTable)
.set({
download_url: downloadUrl,
upload_status: "uploaded",
uploaded_at: uploadedAt,
...(filename !== undefined ? { filename } : {}),
...(sizeBytes !== undefined ? { size_bytes: sizeBytes } : {}),
})
.where(eq(voiceRecordingsTable.id, id));
} catch (error) {
logger.error(
{ id, error: error instanceof Error ? error.message : String(error) },
"Failed to update voice recording status to uploaded",
);
throw error;
}
}
export async function updateVoiceRecordingAsFailed(
id: string,
error: string,
): Promise<void> {
try {
await db()
.update(voiceRecordingsTable)
.set({
upload_status: "failed",
upload_error: error,
})
.where(eq(voiceRecordingsTable.id, id));
} catch (error) {
logger.error(
{ id, error: error instanceof Error ? error.message : String(error) },
"Failed to update voice recording status to failed",
);
throw error;
}
}
export async function updateVoiceRecordingTranscription(
id: string,
transcription: string,
): Promise<void> {
try {
await db()
.update(voiceRecordingsTable)
.set({ transcription })
.where(eq(voiceRecordingsTable.id, id));
} catch (error) {
logger.error(
{ id, error: error instanceof Error ? error.message : String(error) },
"Failed to update voice recording transcription",
);
}
}
export async function listVoiceRecordings(
limit = 100,
): Promise<VoiceRecording[]> {
try {
const rows = await db()
.select()
.from(voiceRecordingsTable)
.orderBy(desc(voiceRecordingsTable.created_at))
.limit(limit);
return rows as VoiceRecording[];
} catch (error) {
logger.error(
{ error: error instanceof Error ? error.message : String(error) },
"Failed to list voice recordings",
);
throw error;
}
}
services/discord-gateway/src/shared/moderation-types.ts
-31
View File
@@ -24,9 +24,6 @@ export interface BroadcasterClient {
messageAnalyzed: (data: unknown) => void;
attachmentCreated: (data: unknown) => void;
attachmentUploaded: (data: unknown) => void;
voiceRecordingStarted: (data: unknown) => void;
voiceRecordingStopped: (data: unknown) => void;
voiceRecordingUploaded: (data: unknown) => void;
analysisQueueStatus: (data: unknown) => void;
}
@@ -101,17 +98,6 @@ export interface AttachmentRecord {
uploaded_at: number | null;
}
export interface VoiceSegmentRecord {
id: string;
user_id: string;
session_id: string;
guild_id: string;
channel_id: string;
filename: string;
duration_ms: number;
created_at: number;
}
export interface DashboardMessage {
id: string;
channel_id: string;
@@ -153,23 +139,6 @@ export interface AnalysisResult {
evidence?: string[];
}
export interface VoiceRecordingUploadData {
id: string;
user_id: string;
username: string;
avatar_url: string | null;
guild_id: string | null;
channel_id: string | null;
channel_name: string | null;
filename: string;
size_bytes: number;
download_url: string;
upload_status: string;
created_at: number;
uploaded_at: number;
transcription?: string | null;
}
export interface AnalysisQueueStatus {
queuedConversations: number;
activeRequests: number;
services/discord-gateway/src/shared/redis-channels.ts
-33
View File
@@ -15,11 +15,6 @@ export const DISCORD_MESSAGE_DELETED = "discord:message:deleted";
export const DISCORD_MESSAGE_ANALYZED = "discord:message:analyzed";
export const DISCORD_ATTACHMENT_CREATED = "discord:attachment:created";
export const DISCORD_ATTACHMENT_UPLOADED = "discord:attachment:uploaded";
export const DISCORD_VOICE_STARTED = "discord:voice:started";
export const DISCORD_VOICE_STOPPED = "discord:voice:stopped";
export const DISCORD_VOICE_UPLOADED = "discord:voice:uploaded";
export const DISCORD_VOICE_ACTIVE_USER = "discord:voice:active_user";
export const DISCORD_VOICE_PCM = "discord:voice:pcm";
export const DISCORD_ANALYSIS_QUEUE_STATUS = "discord:analysis:queue_status";
export const DISCORD_REACTION_ADDED = "discord:reaction:added";
export const DISCORD_REACTION_REMOVED = "discord:reaction:removed";
@@ -37,37 +32,15 @@ export const DISCORD_MODERATION_ACTION = "discord:moderation:action";
// ---------------------------------------------------------------------------
export const BACKEND_COMMAND = "backend:command";
export const BACKEND_VOICE_TRANSMIT = "backend:voice:transmit";
export const BACKEND_COMMAND_REPLY_PREFIX = "backend:command:reply:";
// ---------------------------------------------------------------------------
// Status keys (set by discord-gateway, read by backend via Redis GET)
// ---------------------------------------------------------------------------
export const VOICE_STATUS_KEY = "voice:status";
export const MEDIA_STATUS_KEY = "media:status";
// ---------------------------------------------------------------------------
// Command types (used as the `type` field in CommandMessage envelopes)
// ---------------------------------------------------------------------------
export const COMMAND_VOICE_CONNECT = "voice:connect";
export const COMMAND_VOICE_DISCONNECT = "voice:disconnect";
export const COMMAND_VOICE_DISCONNECT_GUILD = "voice:disconnect:guild";
export const COMMAND_VOICE_CHANNELS = "voice:channels";
export const COMMAND_VOICE_TRANSMIT_START = "voice:transmit:start";
export const COMMAND_VOICE_TRANSMIT_STOP = "voice:transmit:stop";
export const COMMAND_VIDEO_WATCH = "video:watch";
export const COMMAND_VIDEO_UNWATCH = "video:unwatch";
export const COMMAND_GUILDS_LIST = "guilds:list";
export const COMMAND_GUILDS_TEXT_CHANNELS = "guilds:text-channels";
export const COMMAND_MEDIA_QUEUE = "media:queue";
export const COMMAND_MEDIA_SKIP = "media:skip";
export const COMMAND_MEDIA_STOP = "media:stop";
export const COMMAND_MEDIA_VOLUME = "media:volume";
export const COMMAND_MEDIA_LOOP = "media:loop";
export const COMMAND_MODERATION_ACTION = "moderation:action";
export const DISCORD_VOICE_ANALYZED = "discord:voice:analyzed";
// ---------------------------------------------------------------------------
// Event envelope — used by discord-gateway when publishing to Redis
@@ -113,13 +86,7 @@ export const DISCORD_CHANNEL_TO_WS_EVENT: Record<string, string> = {
[DISCORD_MESSAGE_ANALYZED]: "message_analyzed",
[DISCORD_ATTACHMENT_CREATED]: "attachment_created",
[DISCORD_ATTACHMENT_UPLOADED]: "attachment_uploaded",
[DISCORD_VOICE_STARTED]: "voice_recording_started",
[DISCORD_VOICE_STOPPED]: "voice_recording_stopped",
[DISCORD_VOICE_UPLOADED]: "voice_recording_uploaded",
[DISCORD_ANALYSIS_QUEUE_STATUS]: "analysis_queue_status",
[DISCORD_VOICE_ACTIVE_USER]: "voice_active_user",
[DISCORD_VOICE_PCM]: "voice_pcm_data",
[DISCORD_VOICE_ANALYZED]: "voice_analyzed",
[DISCORD_REACTION_ADDED]: "reaction_added",
[DISCORD_REACTION_REMOVED]: "reaction_removed",
[DISCORD_THREAD_CREATED]: "thread_created",