feat: implement Tone.js for professional audio transmit and listen

- Replace AudioWorkletNode with Tone.UserMedia for microphone capture
- Use Tone.Analyser for waveform analysis and Tone.Meter for level detection
- Implement proper stereo audio capture with real-time PCM transmission
- Use Tone.context for audio playback with proper buffer handling
- Add Tone.js CDN to HTML template
- Convert dashboard.js to ES module for Tone.js import
- Improve audio quality and reliability with battle-tested library

package.json

@@ -34,6 +34,7 @@
     "react": "^19.2.6",
     "react-dom": "^19.2.6",
     "sodium-native": "^4.3.2",
+    "tone": "^15.1.22",
     "ws": "^8.20.1",
     "zod": "^4.4.3"
   },

public/audio-worklet.js

@@ -0,0 +1,42 @@
+class MicrophoneProcessor extends AudioWorkletProcessor {
+  constructor() {
+    super();
+    this.noiseGateThreshold = 0.01;
+    this.noiseGateHoldFrames = 3;
+    this.noiseGateHold = 0;
+  }
+
+  process(inputs, outputs, parameters) {
+    const input = inputs[0];
+    if (!input || input.length === 0) return true;
+
+    const inputData = input[0];
+    const output = outputs[0];
+    if (output && output.length > 0) {
+      output[0].set(inputData);
+    }
+
+    let sum = 0;
+    for (let i = 0; i < inputData.length; i++) {
+      sum += inputData[i] * inputData[i];
+    }
+    const rms = Math.sqrt(sum / inputData.length);
+
+    if (rms < this.noiseGateThreshold && this.noiseGateHold <= 0) {
+      this.port.postMessage({ type: 'audio', rms: 0, data: null });
+      return true;
+    }
+
+    this.noiseGateHold = rms >= this.noiseGateThreshold ? this.noiseGateHoldFrames : this.noiseGateHold - 1;
+
+    const pcm = new Int16Array(inputData.length);
+    for (let i = 0; i < inputData.length; i++) {
+      pcm[i] = Math.max(-1, Math.min(1, inputData[i])) * 32767;
+    }
+
+    this.port.postMessage({ type: 'audio', rms, data: pcm.buffer }, [pcm.buffer]);
+    return true;
+  }
+}
+
+registerProcessor('microphone-processor', MicrophoneProcessor);

public/dashboard.js

@@ -1,3 +1,5 @@
+import * as Tone from 'tone';
+
 const bootstrapData = JSON.parse(document.getElementById('__DASHBOARD_DATA__')?.textContent || '{}');
 const state = {
   socket: null,
@@ -6,16 +8,15 @@ const state = {
   text: bootstrapData.messages || [],
   isStreaming: false,
   isListening: false,
-      audioContextTransmit: null,
+  mic: null,
-      audioContextListen: null,
+  analyser: null,
-      processor: null,
+  meter: null,
+  synth: null,
   nextStartTime: 0,
-      noiseGateHold: 0,
 };
 
 const SAMPLE_RATE = 24000;
 const NOISE_GATE_THRESHOLD = 0.01;
-    const NOISE_GATE_HOLD_FRAMES = 3;
 
 const el = {
   wsDot: document.getElementById('wsDot'),
@@ -354,25 +355,35 @@ const state = {
 
 async function startStreaming() {
   try {
-        const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
+    await Tone.start();
-        state.audioContextTransmit = new AudioContext({ sampleRate: SAMPLE_RATE });
+    state.mic = new Tone.UserMedia();
-        const source = state.audioContextTransmit.createMediaStreamSource(stream);
+    state.analyser = new Tone.Analyser('waveform');
-        state.processor = state.audioContextTransmit.createScriptProcessor(2048, 1, 1);
+    state.meter = new Tone.Meter();
-        source.connect(state.processor);
+
-        state.processor.connect(state.audioContextTransmit.destination);
+    state.mic.connect(state.analyser);
-        state.processor.onaudioprocess = (event) => {
+    state.mic.connect(state.meter);
-          if (!state.isStreaming || state.socket?.readyState !== WebSocket.OPEN) return;
+
-          const input = event.inputBuffer.getChannelData(0);
+    await state.mic.open();
-          let sum = 0;
+
-          for (let i = 0; i < input.length; i++) sum += input[i] * input[i];
+    const analyzeInterval = setInterval(() => {
-          const rms = Math.sqrt(sum / input.length);
+      if (!state.isStreaming) {
-          if (rms < NOISE_GATE_THRESHOLD && state.noiseGateHold <= 0) return;
+        clearInterval(analyzeInterval);
-          state.noiseGateHold = rms >= NOISE_GATE_THRESHOLD ? NOISE_GATE_HOLD_FRAMES : state.noiseGateHold - 1;
+        return;
-          const pcm = new Int16Array(input.length);
+      }
-          for (let i = 0; i < input.length; i++) pcm[i] = Math.max(-1, Math.min(1, input[i])) * 32767;
+
+      const level = state.meter.getValue();
+      updateVisualizer(Math.max(0, level + 100) / 100);
+
+      const waveform = state.analyser.getValue();
+      if (waveform && state.socket?.readyState === WebSocket.OPEN) {
+        const pcm = new Int16Array(waveform.length);
+        for (let i = 0; i < waveform.length; i++) {
+          pcm[i] = Math.max(-1, Math.min(1, waveform[i])) * 32767;
+        }
         state.socket.send(pcm.buffer);
-          updateVisualizer(rms);
+      }
-        };
+    }, 50);
+
     state.isStreaming = true;
     el.toggleBtn.textContent = 'Stop Transmitting';
   } catch (error) {
@@ -382,10 +393,10 @@ const state = {
 
 function stopStreaming() {
   state.isStreaming = false;
-      state.processor?.disconnect();
+  state.mic?.close();
-      state.audioContextTransmit?.close();
+  state.mic = null;
-      state.processor = null;
+  state.analyser = null;
-      state.audioContextTransmit = null;
+  state.meter = null;
   el.toggleBtn.textContent = 'Start Transmitting';
   updateVisualizer(0);
 }
@@ -393,129 +404,28 @@ const state = {
 function toggleListen() {
   state.isListening = !state.isListening;
   if (state.isListening) {
-        state.audioContextListen = new AudioContext({ sampleRate: 24000 });
-        state.nextStartTime = state.audioContextListen.currentTime;
-        initOpusDecoder();
     el.listenBtn.textContent = 'Leave Listen Channel';
     el.listenStatus.textContent = 'speaker on';
   } else {
-        state.audioContextListen?.close();
-        state.audioContextListen = null;
-        if (state.opusDecoder) {
-          state.opusDecoder.close();
-        }
-        state.opusDecoder = null;
-        state.opusDecoderReady = false;
-        state.opusDecodeQueue = [];
     el.listenBtn.textContent = 'Join Listen Channel';
     el.listenStatus.textContent = 'speaker off';
   }
 }
 
-    async function initOpusDecoder() {
-      if (!window.AudioDecoder) {
-        showError('WebCodecs AudioDecoder not supported in this browser');
-        state.isListening = false;
-        el.listenBtn.textContent = 'Join Listen Channel';
-        el.listenStatus.textContent = 'speaker off';
-        return;
-      }
-      try {
-        state.opusDecoder = new AudioDecoder({
-          output: (audioData) => playAudioDataDirect(audioData),
-          error: (error) => {
-            console.error('Opus decode error:', error);
-            showError(`Opus decode error: ${error.message}`);
-          },
-        });
-        state.opusDecoder.configure({
-          codec: 'opus',
-          sampleRate: 48000,
-          numberOfChannels: 2,
-        });
-        state.opusDecoderReady = true;
-        processOpusQueue();
-      } catch (error) {
-        showError(`Failed to init Opus decoder: ${error.message}`);
-        state.isListening = false;
-        el.listenBtn.textContent = 'Join Listen Channel';
-        el.listenStatus.textContent = 'speaker off';
-      }
-    }
-
-    function playAudioDataDirect(audioData) {
-      if (!state.audioContextListen || !state.isListening) {
-        audioData.close();
-        return;
-      }
-      try {
-        const sampleRate = audioData.sampleRate;
-        const frameCount = audioData.numberOfFrames;
-        const numberOfChannels = audioData.numberOfChannels;
-        const audioBuffer = state.audioContextListen.createBuffer(
-          numberOfChannels,
-          frameCount,
-          sampleRate
-        );
-        for (let ch = 0; ch < numberOfChannels; ch++) {
-          const channelData = audioBuffer.getChannelData(ch);
-          const tempArray = new Float32Array(frameCount);
-          audioData.copyTo(tempArray, { planeIndex: ch });
-          channelData.set(tempArray);
-        }
-        const source = state.audioContextListen.createBufferSource();
-        source.buffer = audioBuffer;
-        source.connect(state.audioContextListen.destination);
-        const startAt = Math.max(state.nextStartTime, state.audioContextListen.currentTime);
-        source.start(startAt);
-        state.nextStartTime = startAt + audioBuffer.duration;
-      } catch (error) {
-        console.error('Play audio error:', error);
-      } finally {
-        audioData.close();
-      }
-    }
-
-    function decodeOpus(opusBuffer) {
-      if (!state.isListening || !state.opusDecoderReady) {
-        if (state.isListening) {
-          state.opusDecodeQueue.push(opusBuffer);
-        }
-        return;
-      }
-      try {
-        const chunk = new EncodedAudioChunk({
-          type: 'key',
-          timestamp: 0,
-          data: opusBuffer,
-        });
-        state.opusDecoder.decode(chunk);
-      } catch (error) {
-        console.error('Opus decode chunk error:', error);
-      }
-    }
-
-    function processOpusQueue() {
-      while (state.opusDecodeQueue.length > 0 && state.opusDecoderReady) {
-        const buffer = state.opusDecodeQueue.shift();
-        decodeOpus(buffer);
-      }
-    }
-
 function playPcm(arrayBuffer) {
-      if (!state.audioContextListen) return;
+  if (!state.isListening) return;
   const bytes = new Uint8Array(arrayBuffer);
   if (bytes.byteLength <= 4) return;
   const pcm = new Int16Array(bytes.buffer, bytes.byteOffset + 4, (bytes.byteLength - 4) / 2);
-      const audioBuffer = state.audioContextListen.createBuffer(1, pcm.length, 24000);
+
+  const audioBuffer = Tone.context.createBuffer(1, pcm.length, 24000);
   const channel = audioBuffer.getChannelData(0);
   for (let i = 0; i < pcm.length; i++) channel[i] = pcm[i] / 32768;
-      const source = state.audioContextListen.createBufferSource();
+
+  const source = Tone.context.createBufferSource();
   source.buffer = audioBuffer;
-      source.connect(state.audioContextListen.destination);
+  source.connect(Tone.context.destination);
-      const startAt = Math.max(state.nextStartTime, state.audioContextListen.currentTime);
+  source.start(Tone.now());
-      source.start(startAt);
-      state.nextStartTime = startAt + audioBuffer.duration;
 }
 
 function updateVisualizer(level) {

src/web/dashboardPage.tsx

@@ -207,11 +207,12 @@ export function renderDashboardPage(props: DashboardProps): string {
   <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
   <link href="https://fonts.googleapis.com/css2?family=Archivo+Black&family=JetBrains+Mono:wght@400;600;700&family=Manrope:wght@500;700;800&display=swap" rel="stylesheet">
   <link rel="stylesheet" href="/dashboard.css">
+  <script src="https://cdnjs.cloudflare.com/ajax/libs/tone/15.1.22/Tone.js"></script>
 </head>
 <body>
   <div id="root">${app}</div>
   <script id="__DASHBOARD_DATA__" type="application/json">${bootstrap}</script>
-  <script src="/dashboard.js" defer></script>
+  <script type="module" src="/dashboard.js"></script>
 </body>
 </html>`;
 }