voice: deep stability audit — FFmpeg crash recovery, activity timeout, reconnect refresh
Gateway (transmitter.ts): - Auto-stop on FFmpeg crash: non-zero exit triggers stop() to prevent silent audio loss and resource leaks - Voice activity timeout (10s): auto-stops transmitter when no PCM received, preventing dead-air CPU waste on backgrounded tabs - Stderr cap (4KB): prevents unbounded memory growth in long sessions Gateway (voice.handler.ts): - Double-check voiceController.getStatus().connected before starting transmitter — detects stale player state after gateway disconnect Frontend (context.tsx): - Force-refetch voice status on WS reconnect — UI converges in <1s instead of waiting up to 4s for SWR poll interval All: tsc clean, biome clean
This commit is contained in:
@@ -153,6 +153,22 @@ export class VoiceHandler {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Double-check: verify the voice controller also reports connected
|
||||||
|
if (this.voiceController) {
|
||||||
|
const vcStatus = this.voiceController.getStatus();
|
||||||
|
if (!vcStatus.connected) {
|
||||||
|
this.logger.warn(
|
||||||
|
"Player reports connected but voice controller says disconnected — stale state",
|
||||||
|
);
|
||||||
|
return {
|
||||||
|
id: cmd.id,
|
||||||
|
success: false,
|
||||||
|
data: null,
|
||||||
|
error: "Voice channel connection is stale — reconnect first",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
// IMPORTANT: transmitter needs its OWN Redis client because it calls
|
// IMPORTANT: transmitter needs its OWN Redis client because it calls
|
||||||
// .subscribe() which converts the connection to subscriber mode. Reusing
|
// .subscribe() which converts the connection to subscriber mode. Reusing
|
||||||
|
|||||||
@@ -29,6 +29,12 @@ export class VoiceTransmitter {
|
|||||||
private gate = Promise.resolve();
|
private gate = Promise.resolve();
|
||||||
/** Set true before sending SIGTERM so exit handler knows it's intentional */
|
/** Set true before sending SIGTERM so exit handler knows it's intentional */
|
||||||
private _expectedExit = false;
|
private _expectedExit = false;
|
||||||
|
/** Auto-stop timer: if no PCM received for this long, stop transmitter */
|
||||||
|
private _activityTimer: ReturnType<typeof setTimeout> | null = null;
|
||||||
|
/** How long to wait without PCM before auto-stopping (10s) */
|
||||||
|
private static readonly ACTIVITY_TIMEOUT_MS = 10_000;
|
||||||
|
/** Max stderr bytes to retain for error reporting */
|
||||||
|
private static readonly MAX_STDERR_BYTES = 4096;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Start listening for PCM audio data from Redis and stream to Discord
|
* Start listening for PCM audio data from Redis and stream to Discord
|
||||||
@@ -116,10 +122,14 @@ export class VoiceTransmitter {
|
|||||||
this.pcmStream.pipe(this.ffmpegProcess.stdin);
|
this.pcmStream.pipe(this.ffmpegProcess.stdin);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Log FFmpeg stderr for debugging
|
// Log FFmpeg stderr for debugging (cap to prevent unbounded growth)
|
||||||
const stderrChunks: Buffer[] = [];
|
const stderrChunks: Buffer[] = [];
|
||||||
|
let stderrLen = 0;
|
||||||
this.ffmpegProcess.stderr?.on("data", (chunk: Buffer) => {
|
this.ffmpegProcess.stderr?.on("data", (chunk: Buffer) => {
|
||||||
|
stderrLen += chunk.length;
|
||||||
|
if (stderrLen <= VoiceTransmitter.MAX_STDERR_BYTES) {
|
||||||
stderrChunks.push(chunk);
|
stderrChunks.push(chunk);
|
||||||
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
this.ffmpegProcess.on("error", (err) => {
|
this.ffmpegProcess.on("error", (err) => {
|
||||||
@@ -136,8 +146,11 @@ export class VoiceTransmitter {
|
|||||||
const stderr = Buffer.concat(stderrChunks).toString();
|
const stderr = Buffer.concat(stderrChunks).toString();
|
||||||
logger.error(
|
logger.error(
|
||||||
{ code, stderr: stderr.slice(-500) },
|
{ code, stderr: stderr.slice(-500) },
|
||||||
"FFmpeg exited with error",
|
"FFmpeg exited with error — auto-stopping transmitter",
|
||||||
);
|
);
|
||||||
|
// FFmpeg crashed — auto-stop to prevent silent audio loss.
|
||||||
|
// Fire-and-forget; stop() serialises with the gate.
|
||||||
|
void this.stop();
|
||||||
} else {
|
} else {
|
||||||
logger.debug({ code }, "FFmpeg process exited");
|
logger.debug({ code }, "FFmpeg process exited");
|
||||||
}
|
}
|
||||||
@@ -163,6 +176,8 @@ export class VoiceTransmitter {
|
|||||||
{ channel: this.TRANSMIT_CHANNEL },
|
{ channel: this.TRANSMIT_CHANNEL },
|
||||||
"Subscribed to transmit channel",
|
"Subscribed to transmit channel",
|
||||||
);
|
);
|
||||||
|
// Start activity timer — will auto-stop if no PCM arrives within timeout
|
||||||
|
this.resetActivityTimer();
|
||||||
|
|
||||||
this.redisSub.on("message", (channel, message) => {
|
this.redisSub.on("message", (channel, message) => {
|
||||||
if (
|
if (
|
||||||
@@ -178,6 +193,8 @@ export class VoiceTransmitter {
|
|||||||
const pcmBuffer = Buffer.from(data.buffer, "base64");
|
const pcmBuffer = Buffer.from(data.buffer, "base64");
|
||||||
const stream = this.pcmStream;
|
const stream = this.pcmStream;
|
||||||
const canContinue = stream.write(pcmBuffer);
|
const canContinue = stream.write(pcmBuffer);
|
||||||
|
// Reset activity timer — PCM is flowing
|
||||||
|
this.resetActivityTimer();
|
||||||
// Backpressure: queue until drain (cap to prevent memory leak)
|
// Backpressure: queue until drain (cap to prevent memory leak)
|
||||||
if (!canContinue) {
|
if (!canContinue) {
|
||||||
if (this.backpressureQueue.length >= VoiceTransmitter.MAX_QUEUE) {
|
if (this.backpressureQueue.length >= VoiceTransmitter.MAX_QUEUE) {
|
||||||
@@ -224,6 +241,30 @@ export class VoiceTransmitter {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Reset the voice activity timer. Called on every PCM chunk received.
|
||||||
|
* If no chunks arrive for ACTIVITY_TIMEOUT_MS, auto-stop the transmitter
|
||||||
|
* to prevent dead air and wasted resources.
|
||||||
|
*/
|
||||||
|
private resetActivityTimer(): void {
|
||||||
|
this.clearActivityTimer();
|
||||||
|
this._activityTimer = setTimeout(() => {
|
||||||
|
if (!this.isActive) return;
|
||||||
|
logger.warn(
|
||||||
|
{ timeoutMs: VoiceTransmitter.ACTIVITY_TIMEOUT_MS },
|
||||||
|
"Voice activity timeout — no PCM received, auto-stopping transmitter",
|
||||||
|
);
|
||||||
|
void this.stop();
|
||||||
|
}, VoiceTransmitter.ACTIVITY_TIMEOUT_MS);
|
||||||
|
}
|
||||||
|
|
||||||
|
private clearActivityTimer(): void {
|
||||||
|
if (this._activityTimer !== null) {
|
||||||
|
clearTimeout(this._activityTimer);
|
||||||
|
this._activityTimer = null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Stop transmitting and clean up resources
|
* Stop transmitting and clean up resources
|
||||||
*/
|
*/
|
||||||
@@ -246,6 +287,7 @@ export class VoiceTransmitter {
|
|||||||
|
|
||||||
this.isActive = false;
|
this.isActive = false;
|
||||||
|
|
||||||
|
this.clearActivityTimer();
|
||||||
this.backpressureQueue = [];
|
this.backpressureQueue = [];
|
||||||
|
|
||||||
if (this.pcmStream) {
|
if (this.pcmStream) {
|
||||||
|
|||||||
@@ -80,6 +80,7 @@ export class PcmPlayer {
|
|||||||
ring.write++;
|
ring.write++;
|
||||||
}
|
}
|
||||||
// Overflow guard: never let the ring lag more than RING_LEN behind.
|
// Overflow guard: never let the ring lag more than RING_LEN behind.
|
||||||
|
// This handles both normal drift and extreme backpressure scenarios.
|
||||||
const lag = ring.write - ring.readPos;
|
const lag = ring.write - ring.readPos;
|
||||||
if (lag > RING_LEN - 4096) {
|
if (lag > RING_LEN - 4096) {
|
||||||
ring.readPos = ring.write - RING_LEN + 4096;
|
ring.readPos = ring.write - RING_LEN + 4096;
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ import {
|
|||||||
useRef,
|
useRef,
|
||||||
useState,
|
useState,
|
||||||
} from "react";
|
} from "react";
|
||||||
|
import { useSWRConfig } from "swr";
|
||||||
import { toast } from "@/components/primitives";
|
import { toast } from "@/components/primitives";
|
||||||
import { WsConnection } from "./connection";
|
import { WsConnection } from "./connection";
|
||||||
import type { PcmChunk, WsEventHandler, WsEventType, WsStatus } from "./types";
|
import type { PcmChunk, WsEventHandler, WsEventType, WsStatus } from "./types";
|
||||||
@@ -39,6 +40,7 @@ export function WsProvider({
|
|||||||
}) {
|
}) {
|
||||||
const connRef = useRef<WsConnection | null>(null);
|
const connRef = useRef<WsConnection | null>(null);
|
||||||
const [status, setStatus] = useState<WsStatus>("disconnected");
|
const [status, setStatus] = useState<WsStatus>("disconnected");
|
||||||
|
const { mutate } = useSWRConfig();
|
||||||
// Tracks whether we've ever been connected — used to suppress the
|
// Tracks whether we've ever been connected — used to suppress the
|
||||||
// "reconnecting" toast on initial page load.
|
// "reconnecting" toast on initial page load.
|
||||||
const wasConnected = useRef(false);
|
const wasConnected = useRef(false);
|
||||||
@@ -95,6 +97,9 @@ export function WsProvider({
|
|||||||
wasConnected.current = false;
|
wasConnected.current = false;
|
||||||
} else if (s === "connected") {
|
} else if (s === "connected") {
|
||||||
wasConnected.current = true;
|
wasConnected.current = true;
|
||||||
|
// After reconnect, force-refetch voice status immediately so
|
||||||
|
// the UI converges faster instead of waiting up to 4s for SWR poll.
|
||||||
|
void mutate(["voice-status"]);
|
||||||
} else if (s === "error" && !wasConnected.current) {
|
} else if (s === "error" && !wasConnected.current) {
|
||||||
toast({
|
toast({
|
||||||
title: "Connection error",
|
title: "Connection error",
|
||||||
@@ -121,7 +126,7 @@ export function WsProvider({
|
|||||||
unsubEvent();
|
unsubEvent();
|
||||||
};
|
};
|
||||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||||
}, [url, handleBinaryEvent, handleJsonEvent]);
|
}, [url, handleBinaryEvent, handleJsonEvent, mutate]);
|
||||||
|
|
||||||
const subscribe = useCallback(
|
const subscribe = useCallback(
|
||||||
<E extends WsEventType>(_eventType: E, handler: WsEventHandler<E>) => {
|
<E extends WsEventType>(_eventType: E, handler: WsEventHandler<E>) => {
|
||||||
|
|||||||
Reference in New Issue
Block a user