fix(gateway): deliver audio + per-IDR SPS/PPS in GoLive screen share

Screen share showed a single frozen frame: the GoLive pipeline sent video
only (-"-an", h264 muxer cannot carry audio) so the audio SSRC never
transmitted and Discord kept the stream in thumbnail state.

- prepareStream: mux NUT when includeAudio (h264 muxer drops audio) and
  return the actual container format
- Demuxer: support NUT input with a second output pipe (fd3) carrying
  Ogg Opus; parse OGG pages into opus frames (20ms, 48kHz) emitted as
  GoLiveFrames; fix metadata parsing that dropped the audio stream line
  when it arrived in a later stderr chunk (early parsedMeta return)
- playStream: pipe audio.stream into AudioStream → RTP on the audio SSRC
- Encoders: -x264-params repeat-headers=1 → SPS/PPS inline before EVERY
  IDR (NUT remux drops container extradata; also enables PLI recovery)
- screenShareController: includeAudio true
- tests: demuxerNut.test.ts — OGG parser unit test + real ffmpeg NUT
  integration (video access units + parsed opus frames)
This commit is contained in:
asepharyana
2026-08-12 14:20:29 +07:00
parent f70a92880e
commit f1aa08cdf6
7 changed files with 390 additions and 24 deletions
@@ -12,6 +12,7 @@ import { type ChildProcess, spawn } from "node:child_process";
import { existsSync, readdirSync } from "node:fs";
import { join } from "node:path";
import { PassThrough, type Readable } from "node:stream";
import { AudioStream } from "./AudioStream.js";
import { demux } from "./Demuxer.js";
import { type EncoderSettings, Encoders } from "./Encoders.js";
import { VideoStream } from "./VideoStream.js";
@@ -27,6 +28,8 @@ export interface PrepareStreamResult {
height: number;
frameRate?: number;
includeAudio: boolean;
/** Container the encoder muxes to: "nut" (audio-capable) or "h264" (raw). */
format: "nut" | "h264";
}
function isFiniteNonZero(n: unknown): n is number {
@@ -198,7 +201,11 @@ export function prepareStream(
}
args.push(...mergedOptions.customFfmpegFlags);
args.push("-f", "h264", "pipe:1");
// NUT muxer carries video+audio; the raw h264 muxer cannot ("h264 muxer
// does not support any stream of type audio" → header write fails →
// empty stdout → black tile). Audio delivery requires NUT.
const outFormat = mergedOptions.includeAudio ? "nut" : "h264";
args.push("-f", outFormat, "pipe:1");
const isUrl = typeof input === "string";
const proc: ChildProcess = isUrl
@@ -248,6 +255,7 @@ export function prepareStream(
height: mergedOptions.height,
frameRate: mergedOptions.frameRate,
includeAudio: !!mergedOptions.includeAudio,
format: outFormat,
};
}
@@ -274,13 +282,17 @@ export async function playStream(
const conn = await streamer.createStream();
console.log("[goLive:playStream] createStream resolved");
const { video, close: demuxClose } = await demux(prepared.output, {
format: options.format ?? "nut",
const {
video,
audio,
close: demuxClose,
} = await demux(prepared.output, {
format: options.format ?? prepared.format ?? "nut",
frameRate:
typeof options.frameRate === "number" ? options.frameRate : undefined,
});
console.log(
`[goLive:playStream] demux done codec=${video?.codecName ?? "?"} ${video?.width ?? 0}x${video?.height ?? 0} fps=${video ? video.framerate_num / video.framerate_den || 30 : 30}`,
`[goLive:playStream] demux done codec=${video?.codecName ?? "?"} ${video?.width ?? 0}x${video?.height ?? 0} fps=${video ? video.framerate_num / video.framerate_den || 30 : 30} audio=${audio?.codecName ?? "none"}`,
);
if (!video) throw new Error("No video stream in media");
@@ -314,6 +326,19 @@ export async function playStream(
const vStream = new VideoStream(conn);
video.stream.pipe(vStream);
// Audio: Discord's GoLive pipeline expects RTP on the audio SSRC too —
// a video-only stream (zero audio packets) shows a static tile/thumbnail
// instead of live video. Pipe opus frames from the demuxer (silence is
// injected at the encoder when the source has no audio track).
let aStream: AudioStream | undefined;
if (audio) {
aStream = new AudioStream(conn);
audio.stream.pipe(aStream);
console.log(
`[goLive:playStream] audio stream attached (${audio.codecName})`,
);
}
const cleanup = () => {
try {
prepared.command.kill("SIGTERM");