feat(gateway): persistent voice auto-reconnect — rejoin same channel after restart/reboot or unexpected drop

This commit is contained in:
asepharyana
2026-08-30 14:03:12 +07:00
parent 7f45cbff2d
commit 7dfb4035b7
8 changed files with 422 additions and 10 deletions
@@ -0,0 +1,66 @@
# Spec: Voice auto-reconnect (persistent state + rejoin on drop)
## Goal
Setelah `VoiceController.connect()` berhasil, state "sedang merekam di <guild>/<channel>"
disimpan di Postgres. Kalau gateway restart/reboot, atau koneksi voice drop tidak
disengaja (dikeluarkan/moved/server restart), gateway otomatis join ulang ke channel
yang sama.
## Requirement mapping (user's ask)
- "autoreconnect ke channel yg sama jika server restart atau reboot" → reconnect on
startup (ready handler) + keep DB record across graceful shutdown.
- "state nya persistent di db" → `voice_auto_reconnect` table.
- "rejoin jika tidak sengaja dikeluarkan" → watchdog on `Disconnected`/`Destroyed`
(kick / moved / voice server restart) → full rejoin with backoff.
- Manual leave (`/voice disconnect`, dashboard disconnect) MUST NOT rejoin.
## Design decisions
1. **New table** `voice_auto_reconnect` (dedicated, not `ui_state`):
- `guild_id` text PK
- `channel_id` text NOT NULL
- `channel_name` text
- `connected_at` bigint epoch-ms
- `updated_at` bigint epoch-ms
DAO: `voiceAutoReconnectRepo.ts` — `upsert(record)`, `list()`, `delete(guildId)`.
2. **Write on connect**: `VoiceController.connect()` → after `startRecording` success →
`upsert`. IDEMPOTENT (upsert per guild).
3. **Clear on manual leave**: `handleVoiceDisconnect` (all) + `handleVoiceDisconnectGuild`
pass `clearPersisted: true`. Graceful shutdown `disconnect()` keeps the record.
4. **Rejoin on startup**: bootstrap `ready` → `await voiceController.autoReconnect()`
(list persisted → connect each, non-fatal on failure).
5. **Rejoin on unexpected drop**: monitor per-connection; on `Disconnected`/`Destroyed`
with `!intentional` → schedule rejoin `connect(guildId, persistedChannelId)` with
backoff (min 2s, max 30s, max 5 attempts). Track `rejoinAttempts`, reset on success.
6. **Intentional flag**: `disconnectGuild(guildId, { clearPersisted?, intentional? })`.
- shutdown `disconnect()` → `{ intentional: true, clearPersisted: false }`.
- manual `disconnect()` (dashboard) → `{ clearPersisted: true }`, sets intentional.
- manual `disconnectGuild` → `{ clearPersisted: true }`, sets intentional.
The monitor checks `intentional` before rejoin; `clearPersisted` only deletes the row.
## Files touched
- `services/discord-gateway/src/shared/database/schema.ts` — add `pgVoiceAutoReconnectTable`
+ types.
- `services/discord-gateway/src/shared/database/voiceAutoReconnectRepo.ts` (NEW) — DAO.
- `services/discord-gateway/src/modules/voice-recording/voiceController.ts` — upsert on
connect; monitor + rejoin; `autoReconnect()`; `disconnect/disconnectGuild` opts.
- `services/discord-gateway/src/modules/command-handler/voice.handler.ts` — manual
disconnect/disconnectGuild pass `clearPersisted: true`.
- `services/discord-gateway/src/app/bootstrap.ts` — call `voiceController.autoReconnect()`
in `ready`.
- `services/discord-gateway/drizzle/migrations/0020_add_voice_auto_reconnect.sql` +
`meta/_journal.json` entry (apply manually per gmw-ops).
## Edge cases
- Channel deleted / guild lost while persisted → `connect()` throws (channel not found)
→ log + delete persisted row (don't retry forever).
- Rejoin attempts exhausted → keep row (so next restart retries) + log.
- Multiple guilds: per-guild monitor, per-guild persisted row.
- Graceful shutdown order: shutdown sets intentional=true (so no rejoin during teardown)
but keeps row.
## Verification
- `pnpm typecheck && pnpm build && pnpm lint` in `services/discord-gateway`.
- Apply migration `0020` manually; verify table exists.
- CI `Build & Deploy (Nix)` green; gateway deploy lands.
- Manual: connect via dashboard → check `voice_auto_reconnect` row; simulate drop →
confirm rejoin; manual disconnect → row cleared.
@@ -0,0 +1,19 @@
-- Voice auto-reconnect: persist desired voice state per guild.
-- The gateway rejoins the same channel after restart/reboot and after an
-- unexpected voice drop (kick / server move / voice server restart).
-- Idempotent: skips if the table already exists.
DO $$
BEGIN
IF NOT EXISTS (
SELECT 1 FROM information_schema.tables
WHERE table_name = 'voice_auto_reconnect'
) THEN
CREATE TABLE "voice_auto_reconnect" (
"guild_id" text PRIMARY KEY NOT NULL,
"channel_id" text NOT NULL,
"channel_name" text,
"connected_at" bigint NOT NULL,
"updated_at" bigint NOT NULL
);
END IF;
END $$;
@@ -141,6 +141,13 @@
"when": 1787830000000,
"tag": "0019_rename_server_name_to_server_nick",
"breakpoints": true
},
{
"idx": 20,
"version": "7",
"when": 1787860000000,
"tag": "0020_add_voice_auto_reconnect",
"breakpoints": true
}
]
}
@@ -274,6 +274,17 @@ export async function initializeDiscordGateway() {
commandHandler.start(client, voiceController);
logger.info("Command handler started");
// Rejoin persisted voice channels (auto-reconnect on restart/reboot).
// Non-fatal: failures are logged inside autoReconnect.
void voiceController
.autoReconnect()
.catch((err) =>
logger.warn(
{ err: err instanceof Error ? err.message : String(err) },
"Voice auto-reconnect on startup failed",
),
);
// Start retention cleanup scheduler
startRetentionCleanup();
// Start weekly moderation digest (public, automated)
@@ -67,7 +67,10 @@ export class VoiceHandler {
};
}
const status = await this.voiceController.disconnect();
const status = await this.voiceController.disconnect({
clearPersisted: true,
intentional: true,
});
return { id: cmd.id, success: true, data: status };
}
@@ -93,7 +96,10 @@ export class VoiceHandler {
};
}
await this.voiceController.disconnectGuild(guildId);
await this.voiceController.disconnectGuild(guildId, {
clearPersisted: true,
intentional: true,
});
const status = this.voiceController.getStatus();
return { id: cmd.id, success: true, data: status };
}
@@ -1,5 +1,11 @@
import type { VoiceConnection } from "@discordjs/voice";
import { VoiceConnectionStatus } from "@discordjs/voice";
import type { Client, Guild, VoiceChannel } from "discord.js-selfbot-v13";
import {
deleteVoiceAutoReconnect,
listVoiceAutoReconnects,
upsertVoiceAutoReconnect,
} from "@/shared/database/voiceAutoReconnectRepo.js";
import { AppError } from "@/shared/errors/index";
import { createChildLogger } from "@/shared/logger/index";
import { discordPlayer } from "./player.js";
@@ -26,18 +32,37 @@ export interface VoiceStatus {
connections: GuildVoiceState[];
}
export interface DisconnectOptions {
/** Clear the persisted auto-reconnect row (explicit user leave). */
clearPersisted?: boolean;
/** Mark the disconnect as intentional so the rejoin monitor stops. */
intentional?: boolean;
}
const REJOIN_MAX_ATTEMPTS = 5;
const REJOIN_BASE_DELAY_MS = 2_000;
const REJOIN_MAX_DELAY_MS = 30_000;
// ─── VoiceController ─────────────────────────────────────────────────────
export class VoiceController {
private connections = new Map<string, GuildVoiceState>();
private connecting = new Set<string>();
/** Guilds whose disconnect was intentional (user leave / shutdown). */
private intentionalGuilds = new Set<string>();
/** Last known desired voice state, kept independent of live connection map. */
private rejoinState = new Map<string, GuildVoiceState>();
/** Current live connection per guild — for stale-callback invalidation. */
private liveConnections = new Map<string, VoiceConnection>();
/** Per-guild rejoin attempt counters + timers. */
private rejoinAttempts = new Map<string, number>();
private rejoinTimers = new Map<string, NodeJS.Timeout>();
constructor(private readonly client: Client) {}
getStatus(): VoiceStatus {
logger.debug("getStatus called");
// Primary connection (legacy compat — first entry or explicitly set)
const primaryGuildId = this.connections.keys().next().value ?? null;
const primary = primaryGuildId
? this.connections.get(primaryGuildId)
@@ -76,15 +101,21 @@ export class VoiceController {
try {
// Disconnect existing connection for this guild first
if (this.connections.has(guildId)) {
await this.disconnectGuild(guildId);
await this.disconnectGuild(guildId, { intentional: true });
}
// Clear any outstanding rejoin work for this guild — we are (re)joining now.
this.clearRejoin(guildId);
this.intentionalGuilds.delete(guildId);
const guild = this.getGuild(guildId);
const channel =
guild.channels.cache.get(channelId) ??
(await guild.channels.fetch(channelId).catch(() => null));
if (!channel) {
// Channel no longer exists — drop the persisted row so we don't retry forever.
await deleteVoiceAutoReconnect(guildId).catch(() => {});
throw new AppError(
"Voice channel not found",
"VOICE_CHANNEL_NOT_FOUND",
@@ -124,6 +155,25 @@ export class VoiceController {
connectedAt: Date.now(),
};
this.connections.set(guildId, state);
this.rejoinState.set(guildId, state);
this.liveConnections.set(guildId, connection as VoiceConnection);
// Persist desired voice state for auto-reconnect (restart/reboot/kick).
const now = Date.now();
await upsertVoiceAutoReconnect({
guild_id: guildId,
channel_id: channelId,
channel_name: channel.name,
connected_at: now,
updated_at: now,
}).catch((err) => {
logger.warn(
{ err: err instanceof Error ? err.message : String(err) },
"Failed to persist voice auto-reconnect state",
);
});
this.monitorConnection(guildId, connection);
logger.info(
{ guildId, channelId, channelName: channel.name },
@@ -136,25 +186,198 @@ export class VoiceController {
}
}
async disconnect(): Promise<VoiceStatus> {
logger.info("disconnect called");
/**
* Auto-reconnect on startup/restart: rejoin every persisted voice channel.
* Called from the client `ready` handler. Non-fatal per-guild.
*/
async autoReconnect(): Promise<void> {
let states: Array<{
guild_id: string;
channel_id: string;
channel_name: string | null;
}> = [];
try {
states = await listVoiceAutoReconnects();
} catch (err) {
logger.warn(
{ err: err instanceof Error ? err.message : String(err) },
"Failed to load voice auto-reconnect states",
);
return;
}
if (states.length === 0) {
logger.info("No persisted voice state — skipping auto-reconnect");
return;
}
logger.info(
{ count: states.length },
"Auto-reconnecting to persisted voice channels",
);
// Fire in parallel; each failure is logged but doesn't block the others.
await Promise.allSettled(
states.map(async (s) => {
try {
await this.connect(s.guild_id, s.channel_id);
} catch (err) {
logger.error(
{
guildId: s.guild_id,
channelId: s.channel_id,
channelName: s.channel_name,
err: err instanceof Error ? err.message : String(err),
},
"Auto-reconnect failed for guild",
);
}
}),
);
}
async disconnect(opts?: DisconnectOptions): Promise<VoiceStatus> {
// Default semantics for graceful shutdown: intentional (no rejoin during
// teardown) but keep the persisted row so we rejoin on next boot.
const effective = opts ?? { intentional: true };
logger.info({ opts: effective }, "disconnect called");
// Disconnect all guilds
const guildIds = Array.from(this.connections.keys());
for (const gid of guildIds) {
await this.disconnectGuild(gid);
await this.disconnectGuild(gid, effective);
}
discordPlayer.stop();
return this.getStatus();
}
async disconnectGuild(guildId: string): Promise<void> {
logger.info({ guildId }, "disconnectGuild called");
async disconnectGuild(
guildId: string,
opts?: DisconnectOptions,
): Promise<void> {
logger.info({ guildId, opts }, "disconnectGuild called");
if (opts?.intentional) {
this.intentionalGuilds.add(guildId);
}
if (opts?.clearPersisted) {
await deleteVoiceAutoReconnect(guildId).catch(() => {});
// Manual leave — forget the desired state entirely.
this.rejoinState.delete(guildId);
this.intentionalGuilds.add(guildId);
}
if (this.connections.has(guildId)) {
stopRecording(guildId);
this.connections.delete(guildId);
}
this.liveConnections.delete(guildId);
this.clearRejoin(guildId);
}
// ─── Rejoin monitor ───────────────────────────────────────────────────
/**
* Watch a live connection. If it drops WITHOUT us marking the guild as
* intentional (user leave / shutdown), schedule a full rejoin with backoff.
*/
private monitorConnection(
guildId: string,
connection: VoiceConnection,
): void {
const onDrop = () => {
// Ignore callbacks from a stale connection that has since been replaced.
if (this.liveConnections.get(guildId) !== connection) {
return;
}
// This connection object is dead.
this.liveConnections.delete(guildId);
if (this.connections.get(guildId)?.guildId === guildId) {
this.connections.delete(guildId);
}
if (this.intentionalGuilds.has(guildId)) {
logger.info({ guildId }, "Intentional disconnect — no auto-rejoin");
return;
}
logger.warn(
{ guildId },
"Voice dropped unexpectedly — scheduling rejoin",
);
this.scheduleRejoin(guildId);
};
connection.on(VoiceConnectionStatus.Disconnected, onDrop);
connection.on(VoiceConnectionStatus.Destroyed, onDrop);
}
private scheduleRejoin(guildId: string): void {
if (this.rejoinTimers.has(guildId)) {
return; // already scheduled
}
if (this.intentionalGuilds.has(guildId)) {
return;
}
const attempt = (this.rejoinAttempts.get(guildId) ?? 0) + 1;
this.rejoinAttempts.set(guildId, attempt);
const persisted = this.rejoinState.get(guildId);
if (!persisted) {
// Desired state was cleared — nothing to rejoin.
this.clearRejoin(guildId);
return;
}
if (attempt > REJOIN_MAX_ATTEMPTS) {
logger.error(
{ guildId, channelId: persisted.channelId, attempt },
"Rejoin attempts exhausted — keeping persisted state for next restart",
);
this.clearRejoin(guildId);
return;
}
const delay = Math.min(
REJOIN_BASE_DELAY_MS * 2 ** (attempt - 1),
REJOIN_MAX_DELAY_MS,
);
logger.info(
{ guildId, channelId: persisted.channelId, attempt, delay },
"Scheduling voice rejoin",
);
const timer = setTimeout(() => {
this.rejoinTimers.delete(guildId);
this.connect(guildId, persisted.channelId)
.then(() => {
this.rejoinAttempts.delete(guildId);
logger.info({ guildId }, "Auto-rejoin succeeded");
})
.catch((err) => {
logger.warn(
{
guildId,
err: err instanceof Error ? err.message : String(err),
attempt,
},
"Auto-rejoin attempt failed",
);
// Schedule the next attempt (the monitor is gone, so drive it here).
this.scheduleRejoin(guildId);
});
}, delay);
this.rejoinTimers.set(guildId, timer);
}
private clearRejoin(guildId: string): void {
const timer = this.rejoinTimers.get(guildId);
if (timer) {
clearTimeout(timer);
this.rejoinTimers.delete(guildId);
}
this.rejoinAttempts.delete(guildId);
}
private getGuild(guildId: string): Guild {
@@ -277,6 +277,22 @@ export const pgVoiceRecordingsTable = pgTable(
export const voiceRecordingsTable = pgVoiceRecordingsTable;
/**
* Voice Auto-Reconnect Table (PostgreSQL)
* Persists the desired voice state per guild so the gateway can rejoin the
* same channel after a restart/reboot, or after an unexpected drop (kick /
* server move / voice server restart).
*/
export const pgVoiceAutoReconnectTable = pgTable("voice_auto_reconnect", {
guild_id: pgText("guild_id").primaryKey(),
channel_id: pgText("channel_id").notNull(),
channel_name: pgText("channel_name"),
connected_at: pgBigint("connected_at", { mode: "number" }).notNull(),
updated_at: pgBigint("updated_at", { mode: "number" }).notNull(),
});
export const voiceAutoReconnectTable = pgVoiceAutoReconnectTable;
// =============================================================================
// AI Analysis / Analytics
// =============================================================================
@@ -558,6 +574,11 @@ export type CorrectedModerationInsert =
export type VoiceRecording = typeof voiceRecordingsTable.$inferSelect;
export type VoiceRecordingInsert = typeof voiceRecordingsTable.$inferInsert;
// Voice Auto-Reconnect
export type VoiceAutoReconnect = typeof voiceAutoReconnectTable.$inferSelect;
export type VoiceAutoReconnectInsert =
typeof voiceAutoReconnectTable.$inferInsert;
// AI Analysis Runs
export type AIAnalysisRun = typeof aiAnalysisRunsTable.$inferSelect;
export type AIAnalysisRunInsert = typeof aiAnalysisRunsTable.$inferInsert;
@@ -0,0 +1,59 @@
import { eq } from "drizzle-orm";
import { getDatabase } from "./drizzle.js";
import {
type VoiceAutoReconnect,
type VoiceAutoReconnectInsert,
voiceAutoReconnectTable,
} from "./schema.js";
/**
* Upsert a guild's desired voice state.
* Called after a successful connect so the gateway can rejoin after a
* restart/reboot or unexpected drop.
*/
export async function upsertVoiceAutoReconnect(
record: VoiceAutoReconnectInsert,
): Promise<VoiceAutoReconnect | undefined> {
const db = getDatabase();
const [row] = await db
.insert(voiceAutoReconnectTable)
.values(record)
.onConflictDoUpdate({
target: voiceAutoReconnectTable.guild_id,
set: {
channel_id: record.channel_id,
channel_name: record.channel_name ?? null,
connected_at: record.connected_at,
updated_at: record.updated_at,
},
})
.returning();
return row;
}
/** List all persisted desired voice states (for startup auto-reconnect). */
export async function listVoiceAutoReconnects(): Promise<VoiceAutoReconnect[]> {
const db = getDatabase();
return db.select().from(voiceAutoReconnectTable);
}
/** Clear a guild's persisted voice state (explicit manual leave). */
export async function deleteVoiceAutoReconnect(guildId: string): Promise<void> {
const db = getDatabase();
await db
.delete(voiceAutoReconnectTable)
.where(eq(voiceAutoReconnectTable.guild_id, guildId));
}
/** Fetch a single guild's persisted state, if any. */
export async function getVoiceAutoReconnect(
guildId: string,
): Promise<VoiceAutoReconnect | undefined> {
const db = getDatabase();
const rows = await db
.select()
.from(voiceAutoReconnectTable)
.where(eq(voiceAutoReconnectTable.guild_id, guildId))
.limit(1);
return rows[0];
}