import pDebounce from "p-debounce"; import sharp from "sharp"; import Log from "debug-level"; import { FFmpegCommand } from "fluent-ffmpeg-simplified"; import { type Packet, AV_PKT_FLAG_KEY } from "node-av"; import { PassThrough, type Readable } from "node:stream"; import { demux } from "./LibavDemuxer.js"; import { VideoStream } from "./VideoStream.js"; import { AudioStream } from "./AudioStream.js"; import { isBun, isDeno, isFiniteNonZero } from "../utils.js"; import { AVCodecID } from "./LibavCodecId.js"; import { createDecoder } from "./LibavDecoder.js"; import { Encoders } from "./encoders/index.js"; import type { Request } from "zeromq"; import type { SupportedVideoCodec } from "../utils.js"; import type { Streamer } from "../client/index.js"; import type { EncoderSettingsGetter } from "./encoders/index.js"; import type { VideoStreamInfo } from "./LibavDemuxer.js"; import type { WebRtcConnWrapper } from "../client/voice/WebRtcWrapper.js"; export type PrepareStreamOptions = { /** * Disable video transcoding * If enabled, all video related settings have no effects, and the input * video stream is used as-is. * * You need to ensure that the video stream has the right properties * (keyframe every 1s, B-frames disabled). Failure to do so will result in * a glitchy stream, or degraded performance */ noTranscoding: boolean; /** * Video width */ width: number; /** * Video height */ height: number; /** * Video frame rate */ frameRate?: number; /** * Video codec */ videoCodec: SupportedVideoCodec; /** * Video average bitrate in kbps */ bitrateVideo: number; /** * Video max bitrate in kbps */ bitrateVideoMax: number; /** * Audio bitrate in kbps */ bitrateAudio: number; /** * Enable audio output */ includeAudio: boolean; /** * Functions to get encoder settings * This function will receive the average and max bitrate as the input, and * returns an object containing encoder settings for the supported codecs */ encoder: EncoderSettingsGetter; /** * Enable hardware accelerated decoding */ hardwareAcceleratedDecoding: boolean; /** * Add some options to minimize latency */ minimizeLatency: boolean; /** * Custom headers for HTTP requests */ customHeaders: Record; /** * Custom input options to pass directly to ffmpeg * These will be added to the command before other options */ customInputOptions: string[]; /** * Custom ffmpeg flags/options to pass directly to ffmpeg * These will be added to the command after other options */ customFfmpegFlags: string[]; /** * FFmpeg log level */ logLevel: | "quiet" | "panic" | "fatal" | "error" | "warning" | "info" | "verbose" | "debug" | "trace"; }; export type Controller = { volume: number; setVolume(newVolume: number): Promise; }; export function prepareStream( input: string | Readable, options: Partial = {}, cancelSignal?: AbortSignal, ) { cancelSignal?.throwIfAborted(); const logger = new Log("prepareStream"); const loggerFFmpeg = new Log("prepareStream:ffmpeg"); const defaultOptions = { noTranscoding: false, // negative values = resize by aspect ratio, see https://trac.ffmpeg.org/wiki/Scaling width: -2, height: -2, frameRate: undefined, videoCodec: "H264", bitrateVideo: 5000, bitrateVideoMax: 7000, bitrateAudio: 128, includeAudio: true, encoder: Encoders.software(), hardwareAcceleratedDecoding: false, minimizeLatency: false, customHeaders: { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.3", Connection: "keep-alive", }, customInputOptions: [], customFfmpegFlags: [], logLevel: "verbose", } satisfies PrepareStreamOptions; function mergeOptions(opts: Partial) { return { noTranscoding: opts.noTranscoding ?? defaultOptions.noTranscoding, width: isFiniteNonZero(opts.width) ? Math.round(opts.width) : defaultOptions.width, height: isFiniteNonZero(opts.height) ? Math.round(opts.height) : defaultOptions.height, frameRate: isFiniteNonZero(opts.frameRate) && opts.frameRate > 0 ? opts.frameRate : defaultOptions.frameRate, videoCodec: opts.videoCodec ?? defaultOptions.videoCodec, bitrateVideo: isFiniteNonZero(opts.bitrateVideo) && opts.bitrateVideo > 0 ? Math.round(opts.bitrateVideo) : defaultOptions.bitrateVideo, bitrateVideoMax: isFiniteNonZero(opts.bitrateVideoMax) && opts.bitrateVideoMax > 0 ? Math.round(opts.bitrateVideoMax) : defaultOptions.bitrateVideoMax, bitrateAudio: isFiniteNonZero(opts.bitrateAudio) && opts.bitrateAudio > 0 ? Math.round(opts.bitrateAudio) : defaultOptions.bitrateAudio, encoder: opts.encoder ?? defaultOptions.encoder, includeAudio: opts.includeAudio ?? defaultOptions.includeAudio, hardwareAcceleratedDecoding: opts.hardwareAcceleratedDecoding ?? defaultOptions.hardwareAcceleratedDecoding, minimizeLatency: opts.minimizeLatency ?? defaultOptions.minimizeLatency, customHeaders: { ...defaultOptions.customHeaders, ...opts.customHeaders, }, customInputOptions: opts.customInputOptions ?? defaultOptions.customInputOptions, customFfmpegFlags: opts.customFfmpegFlags ?? defaultOptions.customFfmpegFlags, logLevel: opts.logLevel ?? defaultOptions.logLevel, } satisfies PrepareStreamOptions; } const mergedOptions = mergeOptions(options); let isHttpUrl = false; let isHls = false; let isSrt = false; if (typeof input === "string") { isHttpUrl = input.startsWith("http") || input.startsWith("https"); isHls = input.includes("m3u"); isSrt = input.startsWith("srt://"); } const output = new PassThrough(); // command creation const command = new FFmpegCommand(); command.on("stderr", (line) => { loggerFFmpeg.debug(line); }); command.input(input); command.inputOptions("-y", "-loglevel", mergedOptions.logLevel, "-nostats"); // input options if ( mergedOptions.customInputOptions && mergedOptions.customInputOptions.length > 0 ) { command.inputOptions(mergedOptions.customInputOptions); } const { hardwareAcceleratedDecoding, minimizeLatency, customHeaders } = mergedOptions; if (hardwareAcceleratedDecoding) command.inputOptions("-hwaccel", "auto"); if (minimizeLatency) { command.inputOptions( "-fflags nobuffer", "-flags lowdelay", "-flush_packets 1", "-max_delay 100000", ); } if (isHttpUrl) { const headersString = Object.entries(customHeaders) .map(([k, v]) => `${k}: ${v}`) .join("\r\n"); command.inputOptions(`-headers "${headersString}"`); if (!isHls) { command.inputOptions([ "-reconnect 1", "-reconnect_at_eof 1", "-reconnect_streamed 1", "-reconnect_delay_max 4294", ]); } } if (isSrt) { command.inputOptions("-scan_all_pmts 0"); } // general output options command.output(output).format("nut"); // video setup const { noTranscoding, width, height, frameRate, bitrateVideo, bitrateVideoMax, videoCodec, encoder, } = mergedOptions; command.outputOptions("-map 0:v"); if (noTranscoding) { command.videoCodec("copy"); } else { command.videoFilters(`scale=${width}:${height}`); if (frameRate) command.fps(frameRate); command.outputOptions([ "-b:v", `${bitrateVideo}k`, "-maxrate:v", `${bitrateVideoMax}k`, "-bufsize:v", `${Math.round(bitrateVideo / 2)}k`, "-bf", "0", "-pix_fmt", "yuv420p", "-force_key_frames", "expr:gte(t,n_forced*1)", ]); const encoderSettings = encoder(bitrateVideo, bitrateVideoMax)[videoCodec]; if (!encoderSettings) throw new Error(`Encoder settings not specified for ${videoCodec}`); command .videoCodec(encoderSettings.name) .videoFilters(encoderSettings.outFilters ?? []) .outputOptions(encoderSettings.options) .outputOptions(encoderSettings.globalOptions ?? []); } // audio setup const { includeAudio, bitrateAudio } = mergedOptions; if (includeAudio) command .outputOptions("-map 0:a:0?") .audioChannels(2) /* * I don't have much surround sound material to test this with, * if you do and you have better settings for this, feel free to * contribute! */ .outputOptions("-lfe_mix_level 1") .audioFrequency(48000) .audioCodec("libopus") .audioBitrate(`${bitrateAudio}k`) .audioFilters("volume@internal_lib=1.0"); // Add custom ffmpeg flags if ( mergedOptions.customFfmpegFlags && mergedOptions.customFfmpegFlags.length > 0 ) { command.outputOptions(mergedOptions.customFfmpegFlags); } // realtime control mechanism let currentVolume = 1; let zmqClientPromise: Promise | undefined; if (includeAudio && !isBun() && !isDeno()) { function randomInclusive(start: number, end: number) { return Math.floor(Math.random() * (end - start + 1)) + start; } // Last octet is from 2 to 254 to avoid WSL2 shenanigans const loopbackIp = [ 127, randomInclusive(0, 255), randomInclusive(0, 255), randomInclusive(2, 254), ].join("."); const zmqEndpoint = `tcp://${loopbackIp}:42069`; command.audioFilters(`azmq=b=${zmqEndpoint.replaceAll(":", "\\\\:")}`); zmqClientPromise = import("zeromq").then((zmq) => { const client = new zmq.Request({ sendTimeout: 5000, receiveTimeout: 5000, }); client.connect(zmqEndpoint); promise.catch(() => {}).finally(() => client.disconnect(zmqEndpoint)); return client; }); } command.once("start", (cmdline) => { logger.debug(`Starting ffmpeg: ${cmdline}`); }); const promise = command.run(cancelSignal); return { command, output, promise: promise as Promise, controller: { get volume() { return currentVolume; }, async setVolume(newVolume: number) { if (newVolume < 0) return false; try { if (!zmqClientPromise) return false; const client = await zmqClientPromise; await client.send(`volume@internal_lib volume ${newVolume}`); const [res] = await client.receive(); if (res.toString("utf-8").split(" ")[0] !== "0") return false; currentVolume = newVolume; return true; } catch { return false; } }, } satisfies Controller, }; } export type PlayStreamOptions = { /** * Set stream type as "Go Live" or camera stream */ type: "go-live" | "camera"; /** * Set format of the stream */ format: "matroska" | "nut"; /** * Override video width sent to Discord. * * DO NOT SPECIFY UNLESS YOU KNOW WHAT YOU'RE DOING! */ width: number | ((v: VideoStreamInfo) => number); /** * Override video height sent to Discord. * * DO NOT SPECIFY UNLESS YOU KNOW WHAT YOU'RE DOING! */ height: number | ((v: VideoStreamInfo) => number); /** * Override video frame rate sent to Discord. * * DO NOT SPECIFY UNLESS YOU KNOW WHAT YOU'RE DOING! */ frameRate: number | ((v: VideoStreamInfo) => number); /** * Same as ffmpeg's `readrate_initial_burst` command line flag * * See https://ffmpeg.org/ffmpeg.html#:~:text=%2Dreadrate_initial_burst */ readrateInitialBurst: number | undefined; /** * Enable stream preview from input stream (experimental) */ streamPreview: boolean; }; export async function playStream( input: Readable, streamer: Streamer, options: Partial = {}, cancelSignal?: AbortSignal, ) { const logger = new Log("playStream"); cancelSignal?.throwIfAborted(); if (!streamer.voiceConnection) throw new Error("Bot is not connected to a voice channel"); const defaultOptions = { type: "go-live", format: "nut", width: (video) => video.width, height: (video) => video.height, frameRate: (video) => video.framerate_num / video.framerate_den, readrateInitialBurst: undefined, streamPreview: false, } satisfies PlayStreamOptions; function mergeOptions(opts: Partial) { return { type: opts.type ?? defaultOptions.type, format: opts.format ?? defaultOptions.format, width: typeof opts.width === "function" || (isFiniteNonZero(opts.width) && opts.width > 0) ? opts.width : defaultOptions.width, height: typeof opts.height === "function" || (isFiniteNonZero(opts.height) && opts.height > 0) ? opts.height : defaultOptions.height, frameRate: typeof opts.frameRate === "function" || (isFiniteNonZero(opts.frameRate) && opts.frameRate > 0) ? opts.frameRate : defaultOptions.frameRate, readrateInitialBurst: isFiniteNonZero(opts.readrateInitialBurst) && opts.readrateInitialBurst > 0 ? opts.readrateInitialBurst : defaultOptions.readrateInitialBurst, streamPreview: opts.streamPreview ?? defaultOptions.streamPreview, } satisfies PlayStreamOptions; } const mergedOptions = mergeOptions(options); logger.debug({ options: mergedOptions }, "Merged options"); logger.debug("Initializing demuxer"); const { video, audio } = await demux(input, { format: mergedOptions.format, }); cancelSignal?.throwIfAborted(); if (!video) throw new Error("No video stream in media"); const cleanupFuncs: (() => unknown)[] = []; const videoCodecMap: Record = { [AVCodecID.AV_CODEC_ID_H264]: "H264", [AVCodecID.AV_CODEC_ID_H265]: "H265", [AVCodecID.AV_CODEC_ID_VP8]: "VP8", [AVCodecID.AV_CODEC_ID_VP9]: "VP9", [AVCodecID.AV_CODEC_ID_AV1]: "AV1", }; let conn: WebRtcConnWrapper; let stopStream: () => unknown; if (mergedOptions.type === "go-live") { conn = await streamer.createStream(); stopStream = () => streamer.stopStream(); } else { conn = streamer.voiceConnection.webRtcConn; streamer.signalVideo(true); stopStream = () => streamer.signalVideo(false); } conn.setPacketizer(videoCodecMap[video.codec]); conn.mediaConnection.setSpeaking(true); const { width, height, frameRate } = mergedOptions; conn.mediaConnection.setVideoAttributes(true, { width: Math.round(typeof width === "function" ? width(video) : width), height: Math.round(typeof height === "function" ? height(video) : height), fps: Math.round( typeof frameRate === "function" ? frameRate(video) : frameRate, ), }); const vStream = new VideoStream(conn); video.stream.pipe(vStream); if (audio) { const aStream = new AudioStream(conn); audio.stream.pipe(aStream); vStream.syncStream = aStream; const burstTime = mergedOptions.readrateInitialBurst; if (typeof burstTime === "number") { vStream.sync = false; vStream.noSleep = aStream.noSleep = true; const stopBurst = (pts: number) => { if (pts < burstTime * 1000) return; vStream.sync = true; vStream.noSleep = aStream.noSleep = false; vStream.off("pts", stopBurst); }; vStream.on("pts", stopBurst); } } if (mergedOptions.streamPreview && mergedOptions.type === "go-live") { (async () => { const logger = new Log("playStream:preview"); logger.debug("Initializing decoder for stream preview"); const decoder = await createDecoder(video.avStream); if (!decoder) { logger.warn( "Failed to initialize decoder. Stream preview will be disabled", ); return; } cleanupFuncs.push(() => { logger.debug("Freeing decoder"); decoder.free(); }); const updatePreview = pDebounce.promise(async (packet: Packet) => { if (!(packet.flags !== undefined && packet.flags & AV_PKT_FLAG_KEY)) return; const decodeStart = performance.now(); const frames = await decoder.decode(packet).catch((e) => { logger.error(e, "Failed to decode the frame"); return []; }); if (!frames.length) return; const decodeEnd = performance.now(); logger.debug(`Decoding a frame took ${decodeEnd - decodeStart}ms`); const frame = frames[0]; return sharp(frame.toBuffer(), { raw: { width: frame.width ?? 0, height: frame.height ?? 0, channels: 4, }, }) .resize(1024, 576, { fit: "inside" }) .jpeg() .toBuffer() .then((image) => streamer.setStreamPreview(image)) .catch(() => {}) .finally(() => { frames.forEach((frame) => { frame.free(); }); }); }); video.stream.on("data", updatePreview); cleanupFuncs.push(() => video.stream.off("data", updatePreview)); })(); } const promise = new Promise((resolve, reject) => { cleanupFuncs.push(() => { stopStream(); conn.mediaConnection.setSpeaking(false); conn.mediaConnection.setVideoAttributes(false); }); let cleanedUp = false; const cleanup = () => { if (cleanedUp) return; cleanedUp = true; for (const f of cleanupFuncs) f(); }; cancelSignal?.addEventListener( "abort", () => { cleanup(); reject(cancelSignal.reason); }, { once: true }, ); vStream.once("finish", () => { if (cancelSignal?.aborted) return; cleanup(); resolve(); }); }); promise.catch(() => {}); return promise; }