better view for webcam

2026-06-12 15:02:09 -05:00
parent cc0e6f737b
commit 72b22d780f
3 changed files with 328 additions and 57 deletions
@@ -32,6 +32,9 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
  const ctx = canvas.getContext("2d");
  const webrtcMode = document.getElementById("webrtcMode");
  const webrtcVideo = document.getElementById("webrtcVideo");
+  const audioCtrl = document.getElementById("audioCtrl");
+  const audioTransport = document.getElementById("audioTransport");
+  const webrtcAudio = document.getElementById("webrtcAudio");
  let whepClient = null;
  let p2pClient = null;
  function getWebrtcMode() {
@@ -62,6 +65,204 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
  let prefersH264 = typeof VideoDecoder === "function";
  let savedCameraIndex = null;

+  /* ── Remote system audio while viewing webcam ── */
+  const AUDIO_SAMPLE_RATE = 16000;
+  const AUDIO_PLAYBACK_FRAME = 512;
+  const AUDIO_MAX_BUFFER_MS = 120;
+  let audioWs = null;
+  let audioPlayCtx = null;
+  let audioProcessorNode = null;
+  let audioChunks = [];
+  let audioChunkOffset = 0;
+  let audioWhep = null;
+  let audioP2P = null;
+
+  function getAudioTransport() {
+    return audioTransport ? String(audioTransport.value || "off") : "off";
+  }
+
+  function audioResampleInt16ToFloat32(srcInt16, srcRate, dstRate) {
+    if (!srcInt16 || srcInt16.length === 0) return new Float32Array(0);
+    if (srcRate === dstRate) {
+      const out = new Float32Array(srcInt16.length);
+      for (let i = 0; i < srcInt16.length; i++) out[i] = srcInt16[i] / 0x8000;
+      return out;
+    }
+    const outLength = Math.max(1, Math.round((srcInt16.length * dstRate) / srcRate));
+    const out = new Float32Array(outLength);
+    const step = srcRate / dstRate;
+    for (let i = 0; i < outLength; i++) {
+      const srcPos = i * step;
+      const i0 = Math.floor(srcPos);
+      const i1 = Math.min(i0 + 1, srcInt16.length - 1);
+      const frac = srcPos - i0;
+      out[i] = (srcInt16[i0] * (1 - frac) + srcInt16[i1] * frac) / 0x8000;
+    }
+    return out;
+  }
+
+  function initAudioPlayback() {
+    if (!audioPlayCtx) {
+      audioPlayCtx = new AudioContext({ sampleRate: AUDIO_SAMPLE_RATE, latencyHint: "interactive" });
+    }
+    if (!audioProcessorNode) {
+      audioProcessorNode = audioPlayCtx.createScriptProcessor(AUDIO_PLAYBACK_FRAME, 1, 1);
+      audioProcessorNode.onaudioprocess = function (event) {
+        const out = event.outputBuffer.getChannelData(0);
+        out.fill(0);
+        let writeIndex = 0;
+        while (writeIndex < out.length && audioChunks.length > 0) {
+          const head = audioChunks[0];
+          const remaining = head.length - audioChunkOffset;
+          if (remaining <= 0) { audioChunks.shift(); audioChunkOffset = 0; continue; }
+          const take = Math.min(out.length - writeIndex, remaining);
+          out.set(head.subarray(audioChunkOffset, audioChunkOffset + take), writeIndex);
+          writeIndex += take;
+          audioChunkOffset += take;
+          if (audioChunkOffset >= head.length) { audioChunks.shift(); audioChunkOffset = 0; }
+        }
+      };
+      audioProcessorNode.connect(audioPlayCtx.destination);
+    }
+  }
+
+  function appendAudioPcm(binary) {
+    if (!audioPlayCtx) initAudioPlayback();
+    const samples = Math.floor(binary.byteLength / 2);
+    if (samples <= 0) return;
+    const src = new Int16Array(binary.buffer, binary.byteOffset, samples);
+    const chunk = audioResampleInt16ToFloat32(src, AUDIO_SAMPLE_RATE, audioPlayCtx?.sampleRate || AUDIO_SAMPLE_RATE);
+    if (chunk.length === 0) return;
+    audioChunks.push(chunk);
+    let buffered = -audioChunkOffset;
+    for (const c of audioChunks) buffered += c.length;
+    const rate = audioPlayCtx?.sampleRate || AUDIO_SAMPLE_RATE;
+    const max = Math.max(AUDIO_PLAYBACK_FRAME, Math.round(rate * (AUDIO_MAX_BUFFER_MS / 1000)));
+    while (buffered > max && audioChunks.length > 0) {
+      const dropped = audioChunks.shift();
+      buffered -= dropped?.length || 0;
+      audioChunkOffset = 0;
+    }
+  }
+
+  async function stopAudioWebrtc() {
+    const w = audioWhep;
+    audioWhep = null;
+    if (w) { try { await w.stop(); } catch {} }
+    const p = audioP2P;
+    audioP2P = null;
+    if (p) { try { await p.stop(); } catch {} }
+  }
+
+  async function startAudioWhep(whepPath) {
+    await stopAudioWebrtc();
+    audioWhep = new WhepClient({
+      whepPath,
+      audioEl: webrtcAudio,
+      onState: (s) => console.debug("webcam-audio-webrtc[Relayed]: state", s),
+    });
+    try {
+      await audioWhep.start();
+    } catch (err) {
+      console.warn("webcam audio: WHEP start failed", err);
+      audioWhep = null;
+    }
+  }
+
+  function startAudioP2P() {
+    if (!audioWs || audioWs.readyState !== WebSocket.OPEN) return;
+    audioP2P = new P2PClient({
+      audioEl: webrtcAudio,
+      send: (msg) => {
+        if (audioWs && audioWs.readyState === WebSocket.OPEN) {
+          audioWs.send(JSON.stringify(msg));
+        }
+      },
+      onState: (s) => console.debug("webcam-audio-webrtc[P2P]: state", s),
+    });
+    audioP2P.start().catch((err) => {
+      console.warn("webcam audio: P2P start failed", err);
+      audioP2P = null;
+    });
+  }
+
+  function cleanupAudio(uncheckBox) {
+    if (audioProcessorNode) {
+      audioProcessorNode.disconnect();
+      audioProcessorNode.onaudioprocess = null;
+      audioProcessorNode = null;
+    }
+    if (audioPlayCtx) {
+      audioPlayCtx.close().catch(function () {});
+      audioPlayCtx = null;
+    }
+    audioChunks = [];
+    audioChunkOffset = 0;
+    audioWs = null;
+    if (uncheckBox && audioCtrl) audioCtrl.checked = false;
+  }
+
+  function connectAudio() {
+    if (audioWs && audioWs.readyState === WebSocket.OPEN) return;
+    const mode = getAudioTransport();
+    if (mode === "off") {
+      initAudioPlayback();
+      if (audioPlayCtx?.state === "suspended") audioPlayCtx.resume();
+    }
+    const proto = location.protocol === "https:" ? "wss:" : "ws:";
+    const wsAudio = new WebSocket(`${proto}//${location.host}/api/clients/${encodeURIComponent(clientId)}/desktop-audio/ws`);
+    audioWs = wsAudio;
+    wsAudio.binaryType = "arraybuffer";
+    wsAudio.onopen = function () {
+      const start = { type: "start", source: "system" };
+      if (mode === "relayed") start.webrtc = true;
+      try { wsAudio.send(JSON.stringify(start)); } catch (err) {
+        console.warn("webcam audio: send start failed", err);
+        return;
+      }
+      if (mode === "p2p") startAudioP2P();
+    };
+    wsAudio.onmessage = function (ev) {
+      if (typeof ev.data === "string") {
+        let msg;
+        try { msg = JSON.parse(ev.data); } catch { return; }
+        if (!msg || typeof msg.type !== "string") return;
+        if (msg.type === "webrtc_ready" && typeof msg.whepPath === "string") {
+          startAudioWhep(msg.whepPath);
+          return;
+        }
+        if (msg.type === "webrtc_p2p_answer" && typeof msg.sdp === "string") {
+          if (audioP2P) audioP2P.onAnswer(msg.sdp);
+          return;
+        }
+        if (msg.type === "webrtc_p2p_ice") {
+          if (audioP2P) audioP2P.onRemoteCandidate(msg);
+          return;
+        }
+        return;
+      }
+      const bytes = new Uint8Array(ev.data);
+      if (mode === "off" && bytes.byteLength > 1) appendAudioPcm(bytes);
+    };
+    wsAudio.onclose = function () {
+      if (audioWs === wsAudio) {
+        stopAudioWebrtc();
+        cleanupAudio(false);
+      }
+    };
+    wsAudio.onerror = function () {};
+  }
+
+  function disconnectAudio(uncheckBox = true) {
+    if (audioWs) {
+      try { audioWs.send(JSON.stringify({ type: "stop" })); } catch {}
+      try { audioWs.close(); } catch {}
+      audioWs = null;
+    }
+    stopAudioWebrtc();
+    cleanupAudio(uncheckBox);
+  }
+
  function setSelectValue(select, value) {
    if (!select || value === undefined || value === null) return false;
    const normalized = String(value);
@@ -87,6 +288,8 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
    if (fpsInput && settings.fps !== undefined) fpsInput.value = String(settings.fps);
    if (qualitySlider && settings.quality !== undefined) qualitySlider.value = String(settings.quality);
    setSelectValue(webrtcMode, settings.webrtcMode);
+    setSelectValue(audioTransport, settings.audioTransport);
+    if (audioCtrl && typeof settings.audio === "boolean") audioCtrl.checked = settings.audio;
    if (typeof settings.preferH264 === "boolean") {
      prefersH264 = settings.preferH264 && typeof VideoDecoder === "function";
    }
@@ -100,6 +303,8 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
      quality: Number(qualitySlider?.value || 90),
      preferH264: !!prefersH264,
      webrtcMode: getWebrtcMode(),
+      audio: !!audioCtrl?.checked,
+      audioTransport: getAudioTransport(),
    };
  }

@@ -574,6 +779,9 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
    } else {
      send("webcam_start");
    }
+    if (audioCtrl && audioCtrl.checked) {
+      connectAudio();
+    }
    setStreamState("starting", "Starting");
  });

@@ -581,6 +789,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
    desiredStreaming = false;
    send("webcam_stop");
    stopAllWebrtc();
+    disconnectAudio();
    setStreamState("stopping", "Stopping");
    setTimeout(() => {
      if (streamState === "stopping") {
@@ -589,13 +798,16 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
    }, 300);
  });

-  window.addEventListener("beforeunload", () => {
+  function stopOnExit() {
    sharedSettingsSaver.saveNow();
    if (ws && ws.readyState === WebSocket.OPEN) {
      send("webcam_stop");
    }
+    disconnectAudio();
    destroyVideoDecoder();
-  });
+  }
+  window.addEventListener("beforeunload", stopOnExit);
+  window.addEventListener("pagehide", stopOnExit);

  refreshCameras.addEventListener("click", () => {
    requestCameraList();
@@ -652,6 +864,27 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
    });
  }

+  if (audioCtrl) {
+    audioCtrl.addEventListener("change", () => {
+      if (audioCtrl.checked) {
+        connectAudio();
+      } else {
+        disconnectAudio();
+      }
+      sharedSettingsSaver.scheduleSave();
+    });
+  }
+
+  if (audioTransport) {
+    audioTransport.addEventListener("change", () => {
+      if (audioCtrl && audioCtrl.checked) {
+        disconnectAudio(false);
+        connectAudio();
+      }
+      sharedSettingsSaver.scheduleSave();
+    });
+  }
+
  screenshotBtn.addEventListener("click", () => {
    downloadScreenshot();
  });
@@ -22,61 +22,7 @@

    <main class="px-5 py-6 flex-1 flex flex-col">
      <div class="flex-1 flex flex-col gap-4" style="max-width: 100%">
-        <div class="flex flex-wrap gap-3 items-center">
-          <div
-            class="bg-slate-900/70 border border-slate-800 rounded-lg px-3 py-2 flex items-center gap-3"
-          >
-            <label class="text-sm text-slate-300" for="cameraSelect">Camera</label>
-            <select
-              id="cameraSelect"
-              class="bg-slate-800 border border-slate-700 rounded px-2 py-1 text-slate-100 min-w-56"
-            ></select>
-            <button id="refreshCameras" class="button ghost">
-              <i class="fa-solid fa-rotate"></i><span>Refresh</span>
-            </button>
-          </div>
-          <div
-            class="bg-slate-900/70 border border-slate-800 rounded-lg px-3 py-2 flex items-center gap-3 text-sm text-slate-300"
-          >
-            <label class="flex items-center gap-2 text-slate-200" for="fpsInput">
-              <span>FPS</span>
-              <input id="fpsInput" type="number" min="1" max="120" value="30" class="bg-slate-800 border border-slate-700 rounded px-2 py-1 w-20 text-slate-100" />
-            </label>
-            <button id="applyFps" class="button ghost">
-              <i class="fa-solid fa-sliders"></i><span>Apply</span>
-            </button>
-          </div>
-          <div
-            class="bg-slate-900/70 border border-slate-800 rounded-lg px-3 py-2 flex items-center gap-3 text-sm text-slate-300"
-          >
-            <label for="qualitySlider" class="text-slate-200">Quality</label>
-            <input
-              id="qualitySlider"
-              type="range"
-              min="10"
-              max="100"
-              step="5"
-              value="90"
-              class="accent-sky-400"
-            />
-            <span id="qualityValue" class="pill pill-ghost text-slate-100">90%</span>
-          </div>
-          <div
-            class="bg-slate-900/70 border border-slate-800 rounded-lg px-3 py-2 flex items-center gap-3 text-sm text-slate-300"
-          >
-            <label class="flex items-center gap-2 text-slate-200" title="Prefer H.264 when available; disable to force JPEG">
-              <input type="checkbox" id="codecH264" checked /> H.264
-            </label>
-            <span id="codecMode" class="pill pill-ghost text-slate-100">Codec: auto</span>
-            <label class="flex items-center gap-2 text-slate-200" title="Streaming transport. P2P: browser ↔ agent direct (H.264 required). Relayed: through MediaMTX. Canvas: existing WS path.">
-              <span class="text-slate-200">Transport</span>
-              <select id="webrtcMode" class="bg-slate-800 border border-slate-700 rounded px-2 py-1 text-slate-100">
-                <option value="off" selected>Canvas</option>
-                <option value="p2p">WebRTC P2P</option>
-                <option value="relayed">WebRTC Relayed</option>
-              </select>
-            </label>
-          </div>
+        <div class="flex flex-wrap gap-2 items-center">
          <div class="flex items-center gap-2">
            <button id="startBtn" class="button primary">
              <i class="fa-solid fa-play"></i><span>Start</span>
@@ -101,6 +47,95 @@
            <span class="uppercase tracking-wide text-xs text-slate-500">fps</span>
            <span id="viewerFps" class="pill pill-ghost font-mono text-sky-200">--</span>
          </div>
+
+          <div class="relative ml-auto" id="rdSettingsWrap">
+            <button
+              id="rdSettingsBtn"
+              type="button"
+              class="button ghost"
+              aria-haspopup="true"
+              aria-expanded="false"
+            >
+              <i class="fa-solid fa-sliders"></i><span>Settings</span>
+              <i class="fa-solid fa-chevron-down text-[10px] text-slate-400 ml-1"></i>
+            </button>
+
+            <div
+              id="rdSettingsMenu"
+              class="hidden absolute right-0 top-full mt-2 w-[360px] max-h-[75vh] overflow-y-auto rounded-xl border border-slate-700 bg-slate-900/95 backdrop-blur-md shadow-2xl shadow-black/50 z-50"
+              role="menu"
+            >
+              <div class="px-4 py-3 border-b border-slate-800">
+                <div class="text-xs uppercase tracking-wide text-slate-500 mb-2">Camera</div>
+                <div class="flex items-center gap-2">
+                  <label class="text-sm text-slate-300 w-20" for="cameraSelect">Camera</label>
+                  <select
+                    id="cameraSelect"
+                    class="flex-1 bg-slate-800 border border-slate-700 rounded px-2 py-1 text-sm text-slate-100"
+                  ></select>
+                  <button id="refreshCameras" class="button ghost" title="Refresh camera list">
+                    <i class="fa-solid fa-rotate"></i>
+                  </button>
+                </div>
+              </div>
+
+              <div class="px-4 py-3 border-b border-slate-800">
+                <div class="text-xs uppercase tracking-wide text-slate-500 mb-2">Video</div>
+                <div class="flex items-center gap-2 mb-2">
+                  <label class="text-sm text-slate-300 w-20" for="qualitySlider">Quality</label>
+                  <input
+                    id="qualitySlider"
+                    type="range"
+                    min="10"
+                    max="100"
+                    step="5"
+                    value="90"
+                    class="flex-1 accent-sky-400"
+                  />
+                  <span id="qualityValue" class="pill pill-ghost text-slate-100 text-xs min-w-[44px] text-center">90%</span>
+                </div>
+                <div class="flex items-center gap-2 mb-2">
+                  <label class="text-sm text-slate-300 w-20" for="webrtcMode">Transport</label>
+                  <select id="webrtcMode" class="flex-1 bg-slate-800 border border-slate-700 rounded px-2 py-1 text-sm text-slate-100" title="Streaming transport. P2P: browser ↔ agent direct (H.264 required). Relayed: through MediaMTX. Canvas: existing WS path.">
+                    <option value="off" selected>Canvas</option>
+                    <option value="p2p">WebRTC P2P</option>
+                    <option value="relayed">WebRTC Relayed</option>
+                  </select>
+                </div>
+                <div class="flex items-center gap-2 mb-2">
+                  <label class="text-sm text-slate-300 w-20" for="fpsInput">Max FPS</label>
+                  <input id="fpsInput" type="number" min="1" max="120" value="30" class="flex-1 bg-slate-800 border border-slate-700 rounded px-2 py-1 text-sm text-slate-100" />
+                  <button id="applyFps" class="button ghost" title="Apply FPS">
+                    <i class="fa-solid fa-check"></i>
+                  </button>
+                </div>
+                <div class="flex items-center justify-between gap-2">
+                  <label class="flex items-center gap-2 text-sm text-slate-200" title="Prefer H.264 when available; disable to force JPEG">
+                    <input type="checkbox" id="codecH264" checked /> H.264
+                  </label>
+                  <span id="codecMode" class="pill pill-ghost text-slate-100 text-xs">Codec: auto</span>
+                </div>
+              </div>
+
+              <div class="px-4 py-3">
+                <div class="text-xs uppercase tracking-wide text-slate-500 mb-2">Audio</div>
+                <label class="flex items-center gap-2 text-sm text-slate-200 mb-2" title="Stream system audio from the remote machine while viewing webcam">
+                  <input type="checkbox" id="audioCtrl" />
+                  <i class="fa-solid fa-volume-high text-slate-400"></i>
+                  <span>Enable audio</span>
+                </label>
+                <div class="flex items-center gap-2">
+                  <label class="text-sm text-slate-300 w-20" for="audioTransport">Via</label>
+                  <select id="audioTransport" class="flex-1 bg-slate-800 border border-slate-700 rounded px-2 py-1 text-sm text-slate-100" title="Audio transport. WS uses raw PCM; WebRTC modes use an audio track.">
+                    <option value="off" selected>WS</option>
+                    <option value="p2p">WebRTC P2P</option>
+                    <option value="relayed">WebRTC Relayed</option>
+                  </select>
+                </div>
+                <audio id="webrtcAudio" autoplay style="display: none"></audio>
+              </div>
+            </div>
+          </div>
        </div>

        <div
@@ -136,6 +171,7 @@
    </main>

    <script defer src="/vendor/msgpackr/msgpackr.js"></script>
+    <script defer src="/assets/remotedesktop-settings-menu.js"></script>
    <script type="module" src="/assets/nav.js"></script>
    <script type="module" src="/assets/webcam.js"></script>
  </body>
@@ -303,6 +303,8 @@ function sanitizeSharedUiSettings(scope: string, raw: unknown): Record<string, u
  } else if (scope === "webcam") {
    assignIfDefined(out, "camera", pickSteppedNumber(input.camera, 0, 63));
    assignIfDefined(out, "fps", pickSteppedNumber(input.fps, 1, 120));
+    assignIfDefined(out, "audio", pickBoolean(input.audio));
+    assignIfDefined(out, "audioTransport", pickString(input.audioTransport, ["off", "p2p", "relayed"]));
  }

  return out;