vxaboveground/overlord
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

web rtc for webcam, audio, etc.

Browse Source
This commit is contained in:
KingKDot
2026-05-13 20:19:21 -05:00
parent 05fe31f04e
commit bf093f5a7d
20 changed files with 986 additions and 199 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Overlord-Server/public/assets/remotedesktop.js
+121 -20
View File
@@ -183,33 +183,124 @@ import { P2PClient } from "./webrtc-p2p.js";
}
}
function connectAudio() {
if (audioWs && audioWs.readyState === WebSocket.OPEN) return;
// Create AudioContext in click/change handler so the browser trusts the user gesture.
initAudioPlayback();
if (audioPlayCtx?.state === "suspended") audioPlayCtx.resume();
const proto = location.protocol === "https:" ? "wss:" : "ws:";
audioWs = new WebSocket(proto + "//" + location.host + "/api/clients/" + encodeURIComponent(clientId) + "/desktop-audio/ws");
audioWs.binaryType = "arraybuffer";
audioWs.onopen = function () {
audioWs.send(JSON.stringify({ type: "start", source: "system" }));
};
audioWs.onmessage = function (ev) {
if (typeof ev.data === "string") return;
const bytes = new Uint8Array(ev.data);
if (bytes.byteLength > 1) appendAudioPcm(bytes);
};
audioWs.onclose = function () { cleanupAudio(false); };
audioWs.onerror = function () {};
const audioTransport = document.getElementById("audioTransport");
const webrtcAudio = document.getElementById("webrtcAudio");
let audioWhep = null;
let audioP2P = null;
function getAudioTransport() {
return audioTransport ? String(audioTransport.value || "off") : "off";
}
function disconnectAudio() {
async function stopAudioWebrtc() {
const w = audioWhep;
audioWhep = null;
if (w) { try { await w.stop(); } catch {} }
const p = audioP2P;
audioP2P = null;
if (p) { try { await p.stop(); } catch {} }
}
async function startAudioWhep(whepPath) {
await stopAudioWebrtc();
audioWhep = new WhepClient({
whepPath,
audioEl: webrtcAudio,
onState: (s) => console.debug("audio-webrtc[Relayed]: state", s),
});
try {
await audioWhep.start();
} catch (err) {
console.warn("audio: WHEP start failed", err);
audioWhep = null;
}
}
function startAudioP2P() {
if (!audioWs || audioWs.readyState !== WebSocket.OPEN) return;
audioP2P = new P2PClient({
audioEl: webrtcAudio,
send: (msg) => {
if (audioWs && audioWs.readyState === WebSocket.OPEN) {
audioWs.send(JSON.stringify(msg));
}
},
onState: (s) => console.debug("audio-webrtc[P2P]: state", s),
});
audioP2P.start().catch((err) => {
console.warn("audio: P2P start failed", err);
audioP2P = null;
});
}
function connectAudio() {
if (audioWs && audioWs.readyState === WebSocket.OPEN) return;
const mode = getAudioTransport();
// The WS-PCM path requires an AudioContext — only init it when we
// actually need it (WebRTC plays through the <audio> element instead).
if (mode === "off") {
initAudioPlayback();
if (audioPlayCtx?.state === "suspended") audioPlayCtx.resume();
}
const proto = location.protocol === "https:" ? "wss:" : "ws:";
// Each handler captures `ws` locally instead of reading the module-scope
// `audioWs`. When the operator switches transports, the previous socket's
// close event fires asynchronously and would otherwise null out the
// freshly created replacement.
const ws = new WebSocket(proto + "//" + location.host + "/api/clients/" + encodeURIComponent(clientId) + "/desktop-audio/ws");
audioWs = ws;
ws.binaryType = "arraybuffer";
ws.onopen = function () {
const start = { type: "start", source: "system" };
if (mode === "relayed") start.webrtc = true;
try { ws.send(JSON.stringify(start)); } catch (err) {
console.warn("audio: send start failed", err);
return;
}
if (mode === "p2p") startAudioP2P();
};
ws.onmessage = function (ev) {
if (typeof ev.data === "string") {
let msg;
try { msg = JSON.parse(ev.data); } catch { return; }
if (!msg || typeof msg.type !== "string") return;
if (msg.type === "webrtc_ready" && typeof msg.whepPath === "string") {
startAudioWhep(msg.whepPath);
return;
}
if (msg.type === "webrtc_p2p_answer" && typeof msg.sdp === "string") {
if (audioP2P) audioP2P.onAnswer(msg.sdp);
return;
}
if (msg.type === "webrtc_p2p_ice") {
if (audioP2P) audioP2P.onRemoteCandidate(msg);
return;
}
return;
}
const bytes = new Uint8Array(ev.data);
// Only feed PCM into the ScriptProcessor when the WS path is the active
// playback — WebRTC modes ignore the parallel PCM uplink.
if (mode === "off" && bytes.byteLength > 1) appendAudioPcm(bytes);
};
ws.onclose = function () {
// Stale closes from a previous switch must not stomp the current
// socket. Only do the global teardown if this WS is still active.
if (audioWs === ws) {
stopAudioWebrtc();
cleanupAudio(false);
}
};
ws.onerror = function () {};
}
function disconnectAudio(uncheckBox = true) {
if (audioWs) {
try { audioWs.send(JSON.stringify({ type: "stop" })); } catch {}
try { audioWs.close(); } catch {}
audioWs = null;
}
cleanupAudio(true);
stopAudioWebrtc();
cleanupAudio(uncheckBox);
}
function cleanupAudio(uncheckBox) {
@@ -878,6 +969,16 @@ import { P2PClient } from "./webrtc-p2p.js";
}
});
}
if (audioTransport) {
audioTransport.addEventListener("change", function () {
// Reconnect with the new transport if audio is already on. Keep the
// checkbox checked — the user is switching modes, not turning audio off.
if (audioCtrl && audioCtrl.checked) {
disconnectAudio(false);
connectAudio();
}
});
}
if (qualitySlider) {
updateQualityLabel(qualitySlider.value);
Overlord-Server/public/assets/webcam.js
+94 -1
View File
@@ -1,5 +1,7 @@
import { encodeMsgpack, decodeMsgpack } from "./msgpack-helpers.js";
import { checkFeatureAccess } from "./feature-gate.js";
import { WhepClient } from "./whep.js";
import { P2PClient } from "./webrtc-p2p.js";
(async function () {
const clientId = new URLSearchParams(location.search).get("clientId");
@@ -27,6 +29,17 @@ import { checkFeatureAccess } from "./feature-gate.js";
const statusEl = document.getElementById("streamStatus");
const canvas = document.getElementById("frameCanvas");
const ctx = canvas.getContext("2d");
const webrtcMode = document.getElementById("webrtcMode");
const webrtcVideo = document.getElementById("webrtcVideo");
let whepClient = null;
let p2pClient = null;
function getWebrtcMode() {
return webrtcMode ? String(webrtcMode.value || "off") : "off";
}
function setWebrtcViewActive(active) {
if (canvas) canvas.style.display = active ? "none" : "block";
if (webrtcVideo) webrtcVideo.style.display = active ? "block" : "none";
}
clientLabel.textContent = clientId;
@@ -350,8 +363,76 @@ import { checkFeatureAccess } from "./feature-gate.js";
ws.send(encodeMsgpack({ type, ...payload }));
}
async function startWhep(whepPath) {
await stopAllWebrtc();
if (!webrtcVideo) return;
whepClient = new WhepClient({
whepPath,
videoEl: webrtcVideo,
onState: (s) => {
if (s === "connected") setStreamState("streaming", "Streaming (WebRTC Relayed)");
else if (s === "failed" || s === "disconnected") setWebrtcViewActive(false);
},
});
try {
await whepClient.start();
setWebrtcViewActive(true);
} catch (err) {
console.warn("webcam: WHEP start failed, falling back to canvas", err);
setWebrtcViewActive(false);
whepClient = null;
}
}
async function startP2P() {
await stopAllWebrtc();
if (!webrtcVideo) return;
p2pClient = new P2PClient({
videoEl: webrtcVideo,
send: (msg) => {
if (ws && ws.readyState === WebSocket.OPEN) ws.send(encodeMsgpack(msg));
},
onState: (s) => {
if (s === "connected") setStreamState("streaming", "Streaming (WebRTC P2P)");
else if (s === "failed" || s === "disconnected") setWebrtcViewActive(false);
},
});
try {
await p2pClient.start();
setWebrtcViewActive(true);
} catch (err) {
console.warn("webcam: P2P start failed, falling back to canvas", err);
setWebrtcViewActive(false);
const c = p2pClient;
p2pClient = null;
if (c) { try { await c.stop(); } catch {} }
}
}
async function stopAllWebrtc() {
setWebrtcViewActive(false);
const w = whepClient;
whepClient = null;
if (w) { try { await w.stop(); } catch {} }
const p = p2pClient;
p2pClient = null;
if (p) { try { await p.stop(); } catch {} }
}
function handleControlMessage(msg) {
if (!msg || typeof msg !== "object") return;
if (msg.type === "webrtc_ready" && typeof msg.whepPath === "string") {
startWhep(msg.whepPath);
return;
}
if (msg.type === "webrtc_p2p_answer" && typeof msg.sdp === "string") {
if (p2pClient) p2pClient.onAnswer(msg.sdp);
return;
}
if (msg.type === "webrtc_p2p_ice") {
if (p2pClient) p2pClient.onRemoteCandidate(msg);
return;
}
if (msg.type === "webcam_devices") {
renderCameraList(msg.devices, msg.selected);
return;
@@ -404,11 +485,13 @@ import { checkFeatureAccess } from "./feature-gate.js";
};
ws.onclose = () => {
stopAllWebrtc();
setStreamState("disconnected", "Disconnected");
setTimeout(connect, 3000);
};
ws.onerror = () => {
stopAllWebrtc();
setStreamState("error", "Connection error");
};
}
@@ -416,13 +499,23 @@ import { checkFeatureAccess } from "./feature-gate.js";
startBtn.addEventListener("click", () => {
desiredStreaming = true;
applyFpsSettings();
send("webcam_start");
const mode = getWebrtcMode();
if (mode === "relayed") {
// Server replies with webrtc_ready; startWhep happens then.
send("webcam_start", { webrtc: true });
} else if (mode === "p2p") {
send("webcam_start");
startP2P();
} else {
send("webcam_start");
}
setStreamState("starting", "Starting");
});
stopBtn.addEventListener("click", () => {
desiredStreaming = false;
send("webcam_stop");
stopAllWebrtc();
setStreamState("stopping", "Stopping");
setTimeout(() => {
if (streamState === "stopping") {
Overlord-Server/public/assets/webrtc-p2p.js
+15 -7
View File
@@ -16,13 +16,15 @@
export class P2PClient {
/**
* @param {object} opts
* @param {HTMLVideoElement} opts.videoEl
* @param {(msg: object) => void} opts.send Send a JSON-encodable msg over the RD WS.
* @param {HTMLVideoElement} [opts.videoEl]
* @param {HTMLAudioElement} [opts.audioEl]
* @param {(msg: object) => void} opts.send Send a JSON-encodable msg over the viewer's WS.
* @param {(state: string) => void} [opts.onState]
* @param {string[]} [opts.iceServers] STUN URLs. Default: Google public STUN.
*/
constructor(opts) {
this.videoEl = opts.videoEl;
this.videoEl = opts.videoEl || null;
this.audioEl = opts.audioEl || null;
this.send = opts.send;
this.onState = opts.onState || (() => {});
this.iceServers = opts.iceServers || ["stun:stun.l.google.com:19302"];
@@ -38,13 +40,18 @@ export class P2PClient {
});
this.pc = pc;
pc.addTransceiver("video", { direction: "recvonly" });
if (this.videoEl) pc.addTransceiver("video", { direction: "recvonly" });
if (this.audioEl) pc.addTransceiver("audio", { direction: "recvonly" });
pc.ontrack = (ev) => {
if (ev.track.kind !== "video") return;
const stream = ev.streams[0] || new MediaStream([ev.track]);
this.videoEl.srcObject = stream;
this.videoEl.play().catch(() => {});
if (ev.track.kind === "video" && this.videoEl) {
this.videoEl.srcObject = stream;
this.videoEl.play().catch(() => {});
} else if (ev.track.kind === "audio" && this.audioEl) {
this.audioEl.srcObject = stream;
this.audioEl.play().catch(() => {});
}
};
pc.onicecandidate = (ev) => {
// Trickle ICE: empty candidate signals end-of-gathering — we don't
@@ -104,5 +111,6 @@ export class P2PClient {
}
try { this.send({ type: "webrtc_p2p_stop" }); } catch {}
if (this.videoEl) this.videoEl.srcObject = null;
if (this.audioEl) this.audioEl.srcObject = null;
}
}
Overlord-Server/public/assets/whep.js
+17 -10
View File
@@ -11,13 +11,16 @@
export class WhepClient {
/**
* @param {object} opts
* @param {string} opts.whepPath Path under the same origin, e.g. "/api/webrtc/agents/abc/whep"
* @param {HTMLVideoElement} opts.videoEl
* @param {string} opts.whepPath Path under the same origin, e.g. "/api/webrtc/agents/abc/desktop/whep"
* @param {HTMLVideoElement} [opts.videoEl] Attach the video track here.
* @param {HTMLAudioElement} [opts.audioEl] Attach the audio track here.
* Default: hidden <audio> appended to <body>.
* @param {(state: string) => void} [opts.onState]
*/
constructor(opts) {
this.whepPath = opts.whepPath;
this.videoEl = opts.videoEl;
this.videoEl = opts.videoEl || null;
this.audioEl = opts.audioEl || null;
this.onState = opts.onState || (() => {});
this.pc = null;
this.resourceURL = "";
@@ -28,13 +31,18 @@ export class WhepClient {
const pc = new RTCPeerConnection({});
this.pc = pc;
pc.addTransceiver("video", { direction: "recvonly" });
if (this.videoEl) pc.addTransceiver("video", { direction: "recvonly" });
if (this.audioEl) pc.addTransceiver("audio", { direction: "recvonly" });
pc.ontrack = (ev) => {
if (ev.track.kind !== "video") return;
const stream = ev.streams[0] || new MediaStream([ev.track]);
this.videoEl.srcObject = stream;
this.videoEl.play().catch(() => {});
if (ev.track.kind === "video" && this.videoEl) {
this.videoEl.srcObject = stream;
this.videoEl.play().catch(() => {});
} else if (ev.track.kind === "audio" && this.audioEl) {
this.audioEl.srcObject = stream;
this.audioEl.play().catch(() => {});
}
};
pc.onconnectionstatechange = () => {
this.onState(pc.connectionState);
@@ -77,9 +85,8 @@ export class WhepClient {
if (pc) {
try { pc.close(); } catch {}
}
if (this.videoEl) {
this.videoEl.srcObject = null;
}
if (this.videoEl) this.videoEl.srcObject = null;
if (this.audioEl) this.audioEl.srcObject = null;
}
}
Overlord-Server/public/remotedesktop.html
+9
View File
@@ -145,6 +145,15 @@
<label class="flex items-center gap-2 text-sm text-slate-200" title="Stream system audio from the remote machine"
><input type="checkbox" id="audioCtrl" /> <i class="fa-solid fa-volume-high text-slate-400"></i> Audio</label
>
<label class="flex items-center gap-2 text-sm text-slate-200" title="Audio transport. WS: raw PCM over the existing WebSocket (current default). WebRTC modes use G.711 over the WebRTC track instead.">
<span class="text-slate-400">Audio via</span>
<select id="audioTransport" class="bg-slate-800 border border-slate-700 rounded px-2 py-1 text-slate-100">
<option value="off" selected>WS</option>
<option value="p2p">WebRTC P2P</option>
<option value="relayed">WebRTC Relayed</option>
</select>
</label>
<audio id="webrtcAudio" autoplay style="display: none"></audio>
<label class="flex items-center gap-2 text-sm text-slate-200" title="Smooths mouse input for steadier movement">
<span class="text-slate-400">Smoothing</span>
<input id="smoothingSlider" type="range" min="0" max="80" step="5" value="20" class="accent-emerald-400" />
Overlord-Server/public/webcam.html
+16
View File
@@ -68,6 +68,14 @@
<input type="checkbox" id="codecH264" checked /> H.264
</label>
<span id="codecMode" class="pill pill-ghost text-slate-100">Codec: auto</span>
<label class="flex items-center gap-2 text-slate-200" title="Streaming transport. P2P: browser ↔ agent direct (H.264 required). Relayed: through MediaMTX. Canvas: existing WS path.">
<span class="text-slate-200">Transport</span>
<select id="webrtcMode" class="bg-slate-800 border border-slate-700 rounded px-2 py-1 text-slate-100">
<option value="off" selected>Canvas</option>
<option value="p2p">WebRTC P2P</option>
<option value="relayed">WebRTC Relayed</option>
</select>
</label>
</div>
<div class="flex items-center gap-2">
<button id="startBtn" class="button primary">
@@ -114,6 +122,14 @@
height="720"
style="display: block; max-width: 100%; max-height: 100%; width: auto; height: auto;"
></canvas>
<video
id="webrtcVideo"
class="bg-slate-950"
autoplay
playsinline
muted
style="display: none; max-width: 100%; max-height: 100%; width: auto; height: auto;"
></video>
</div>
</div>
</div>
Overlord-Server/src/server/routes/webrtc-routes.ts
+6 -3
View File
@@ -24,15 +24,18 @@ export function issueWebrtcPublishToken(clientId: string): string {
return token;
}
export function webrtcStreamPathFor(clientId: string): string {
return `agents/${clientId}`;
export type WebrtcKind = "desktop" | "webcam" | "audio";
export function webrtcStreamPathFor(clientId: string, kind: WebrtcKind): string {
return `agents/${clientId}/${kind}`;
}
export async function handleWebrtcRoutes(req: Request, url: URL): Promise<Response | null> {
if (!url.pathname.startsWith("/api/webrtc/")) return null;
const rest = url.pathname.slice("/api/webrtc/".length);
const match = rest.match(/^(agents\/[A-Za-z0-9_.\-]+)\/(whip|whep)(\/[A-Za-z0-9_.\-]+)?$/);
// /api/webrtc/agents/<clientId>/<kind>/(whip|whep)[/<sessionId>]
const match = rest.match(/^(agents\/[A-Za-z0-9_.\-]+\/(?:desktop|webcam|audio))\/(whip|whep)(\/[A-Za-z0-9_.\-]+)?$/);
if (!match) return new Response("Not Found", { status: 404 });
const [, streamPath, kind, sessionSuffix] = match;
Overlord-Server/src/server/routes/websocket-lifecycle-routes.ts
+2
View File
@@ -809,6 +809,7 @@ export function handleWebSocketClose(
}
if (role === "webcam_viewer") {
deps.cleanupRdViewerP2P(ws);
let removedClientId = clientId;
for (const [sid, sess] of sessionManager.getAllWebcamSessions().entries()) {
if (sess.viewer === ws) {
@@ -822,6 +823,7 @@ export function handleWebSocketClose(
if (!stillViewing) {
const target = clientManager.getClient(removedClientId);
deps.sendDesktopCommand(target, "webcam_stop", {});
deps.sendDesktopCommand(target, "webrtc_stop", { kind: "webcam" });
deps.webcamStreamingState.delete(removedClientId);
logger.debug(`[webcam] cleaned up state for client ${removedClientId}`);
}
Overlord-Server/src/server/ws-console-rd-hvnc.ts
+87 -14
View File
@@ -214,17 +214,22 @@ export const rdStreamingState = new Map<string, {
const rdInputPending = new Map<string, { clientId: string; sentAt: number; kind: string }>();
const RD_INPUT_TTL_MS = 10_000;
type P2PSession = { viewer: ServerWebSocket<SocketData>; clientId: string };
export type P2PKind = "desktop" | "webcam" | "audio";
type P2PSession = {
viewer: ServerWebSocket<SocketData>;
clientId: string;
kind: P2PKind;
};
const p2pAgentToViewer = new Map<string, P2PSession>();
const p2pViewerToAgent = new WeakMap<ServerWebSocket<SocketData>, string>();
function createP2PSession(ws: ServerWebSocket<SocketData>, clientId: string): string {
export function createP2PSession(ws: ServerWebSocket<SocketData>, clientId: string, kind: P2PKind): string {
const prior = p2pViewerToAgent.get(ws);
if (prior) {
p2pAgentToViewer.delete(prior);
}
const sessionId = uuidv4();
p2pAgentToViewer.set(sessionId, { viewer: ws, clientId });
p2pAgentToViewer.set(sessionId, { viewer: ws, clientId, kind });
p2pViewerToAgent.set(ws, sessionId);
return sessionId;
}
@@ -233,14 +238,20 @@ function lookupP2PSession(sessionId: string): P2PSession | undefined {
return p2pAgentToViewer.get(sessionId);
}
function clearP2PSessionForViewer(ws: ServerWebSocket<SocketData>): { sessionId: string; clientId: string } | null {
export function getP2PSessionIdForViewer(ws: ServerWebSocket<SocketData>): string | undefined {
return p2pViewerToAgent.get(ws);
}
export function clearP2PSessionForViewer(ws: ServerWebSocket<SocketData>):
| { sessionId: string; clientId: string; kind: P2PKind }
| null {
const sessionId = p2pViewerToAgent.get(ws);
if (!sessionId) return null;
const entry = p2pAgentToViewer.get(sessionId);
p2pAgentToViewer.delete(sessionId);
p2pViewerToAgent.delete(ws);
if (!entry) return null;
return { sessionId, clientId: entry.clientId };
return { sessionId, clientId: entry.clientId, kind: entry.kind };
}
function pruneRdInputPending(now = Date.now()) {
@@ -415,13 +426,16 @@ export function handleRemoteDesktopViewerMessage(ws: ServerWebSocket<SocketData>
}
}
if ((payload as any).webrtc === true) {
const streamPath = webrtcStreamPathFor(clientId);
const streamPath = webrtcStreamPathFor(clientId, "desktop");
const token = issueWebrtcPublishToken(clientId);
const whipPath = `/api/webrtc/${streamPath}/whip`;
sendDesktopCommand(target, "webrtc_publish", {
streamPath,
whipPath,
token,
kind: "desktop",
hasVideo: true,
hasAudio: false,
});
safeSendViewer(ws, {
type: "webrtc_ready",
@@ -442,7 +456,7 @@ export function handleRemoteDesktopViewerMessage(ws: ServerWebSocket<SocketData>
.filter(s => s.id !== ws.data.sessionId);
if (otherViewers.length === 0) {
sendDesktopCommand(target, "desktop_stop", {});
sendDesktopCommand(target, "webrtc_stop", {});
sendDesktopCommand(target, "webrtc_stop", { kind: "desktop" });
if (state.isStreaming) {
state.isStreaming = false;
rdStreamingState.set(clientId, state);
@@ -591,8 +605,8 @@ export function handleRemoteDesktopViewerMessage(ws: ServerWebSocket<SocketData>
case "webrtc_p2p_offer": {
const sdp = typeof (payload as any).sdp === "string" ? (payload as any).sdp : "";
if (!sdp) break;
const sessionId = createP2PSession(ws, clientId);
sendDesktopCommand(target, "webrtc_p2p_offer", { sessionId, sdp });
const sessionId = createP2PSession(ws, clientId, "desktop");
sendDesktopCommand(target, "webrtc_p2p_offer", { sessionId, sdp, kind: "desktop", hasVideo: true, hasAudio: false });
break;
}
case "webrtc_p2p_ice": {
@@ -602,6 +616,7 @@ export function handleRemoteDesktopViewerMessage(ws: ServerWebSocket<SocketData>
if (!candidate) break;
sendDesktopCommand(target, "webrtc_p2p_ice", {
sessionId,
kind: "desktop",
candidate,
sdpMid: typeof (payload as any).sdpMid === "string" ? (payload as any).sdpMid : "",
sdpMLineIndex: Number((payload as any).sdpMLineIndex) || 0,
@@ -611,7 +626,7 @@ export function handleRemoteDesktopViewerMessage(ws: ServerWebSocket<SocketData>
case "webrtc_p2p_stop": {
const cleared = clearP2PSessionForViewer(ws);
if (cleared) {
sendDesktopCommand(target, "webrtc_p2p_stop", { sessionId: cleared.sessionId });
sendDesktopCommand(target, "webrtc_p2p_stop", { sessionId: cleared.sessionId, kind: cleared.kind });
}
break;
}
@@ -620,13 +635,23 @@ export function handleRemoteDesktopViewerMessage(ws: ServerWebSocket<SocketData>
}
}
function sendP2PSignalingToViewer(session: P2PSession, msg: Record<string, unknown>) {
if (session.kind === "audio") {
try { session.viewer.send(JSON.stringify(msg)); } catch (err) {
logger.error("[audio] p2p signaling send failed", err);
}
return;
}
safeSendViewer(session.viewer, msg);
}
export function handleWebrtcP2PAnswer(_clientId: string, payload: any) {
const sessionId = typeof payload?.sessionId === "string" ? payload.sessionId : "";
const sdp = typeof payload?.sdp === "string" ? payload.sdp : "";
if (!sessionId || !sdp) return;
const session = lookupP2PSession(sessionId);
if (!session) return;
safeSendViewer(session.viewer, { type: "webrtc_p2p_answer", sdp });
sendP2PSignalingToViewer(session, { type: "webrtc_p2p_answer", sdp });
}
export function handleWebrtcP2PIce(_clientId: string, payload: any) {
@@ -635,7 +660,7 @@ export function handleWebrtcP2PIce(_clientId: string, payload: any) {
if (!sessionId || !candidate) return;
const session = lookupP2PSession(sessionId);
if (!session) return;
safeSendViewer(session.viewer, {
sendP2PSignalingToViewer(session, {
type: "webrtc_p2p_ice",
candidate,
sdpMid: typeof payload?.sdpMid === "string" ? payload.sdpMid : "",
@@ -643,15 +668,17 @@ export function handleWebrtcP2PIce(_clientId: string, payload: any) {
});
}
export function cleanupRdViewerP2P(ws: ServerWebSocket<SocketData>) {
export function cleanupViewerP2P(ws: ServerWebSocket<SocketData>) {
const cleared = clearP2PSessionForViewer(ws);
if (!cleared) return;
const target = clientManager.getClient(cleared.clientId);
if (target) {
sendDesktopCommand(target, "webrtc_p2p_stop", { sessionId: cleared.sessionId });
sendDesktopCommand(target, "webrtc_p2p_stop", { sessionId: cleared.sessionId, kind: cleared.kind });
}
}
export const cleanupRdViewerP2P = cleanupViewerP2P;
function handleRemoteDesktopFrame(payload: any) {
const clientId = payload.clientId as string;
const header = payload.header;
@@ -826,6 +853,23 @@ export function handleWebcamViewerMessage(ws: ServerWebSocket<SocketData>, raw:
if (!state.isStreaming) {
sendDesktopCommand(target, "webcam_set_fps", { fps: state.fps, useMax: state.useMax });
sendDesktopCommand(target, "webcam_set_quality", { quality: state.quality, codec: state.codec });
if ((payload as any).webrtc === true) {
const streamPath = webrtcStreamPathFor(clientId, "webcam");
const token = issueWebrtcPublishToken(clientId);
sendDesktopCommand(target, "webrtc_publish", {
streamPath,
whipPath: `/api/webrtc/${streamPath}/whip`,
token,
kind: "webcam",
hasVideo: true,
hasAudio: false,
});
safeSendViewer(ws, {
type: "webrtc_ready",
streamPath,
whepPath: `/api/webrtc/${streamPath}/whep`,
});
}
sendDesktopCommand(target, "webcam_start", {});
state.isStreaming = true;
webcamStreamingState.set(clientId, state);
@@ -845,6 +889,7 @@ export function handleWebcamViewerMessage(ws: ServerWebSocket<SocketData>, raw:
.filter(s => s.id !== ws.data.sessionId);
if (otherWebcamViewers.length === 0) {
sendDesktopCommand(target, "webcam_stop", {});
sendDesktopCommand(target, "webrtc_stop", { kind: "webcam" });
state.isStreaming = false;
webcamStreamingState.set(clientId, state);
} else {
@@ -852,6 +897,34 @@ export function handleWebcamViewerMessage(ws: ServerWebSocket<SocketData>, raw:
}
break;
}
case "webrtc_p2p_offer": {
const sdp = typeof (payload as any).sdp === "string" ? (payload as any).sdp : "";
if (!sdp) break;
const sessionId = createP2PSession(ws, clientId, "webcam");
sendDesktopCommand(target, "webrtc_p2p_offer", { sessionId, sdp, kind: "webcam", hasVideo: true, hasAudio: false });
break;
}
case "webrtc_p2p_ice": {
const sessionId = p2pViewerToAgent.get(ws);
if (!sessionId) break;
const candidate = typeof (payload as any).candidate === "string" ? (payload as any).candidate : "";
if (!candidate) break;
sendDesktopCommand(target, "webrtc_p2p_ice", {
sessionId,
kind: "webcam",
candidate,
sdpMid: typeof (payload as any).sdpMid === "string" ? (payload as any).sdpMid : "",
sdpMLineIndex: Number((payload as any).sdpMLineIndex) || 0,
});
break;
}
case "webrtc_p2p_stop": {
const cleared = clearP2PSessionForViewer(ws);
if (cleared) {
sendDesktopCommand(target, "webrtc_p2p_stop", { sessionId: cleared.sessionId, kind: cleared.kind });
}
break;
}
default:
break;
}
Overlord-Server/src/server/ws-desktop-audio.ts
+88 -19
View File
@@ -5,10 +5,17 @@ import { encodeMessage } from "../protocol";
import * as sessionManager from "../sessions/sessionManager";
import type { DesktopAudioViewer, SocketData } from "../sessions/types";
import { canUserAccessClient } from "../users";
import { issueWebrtcPublishToken, webrtcStreamPathFor } from "./routes/webrtc-routes";
import {
cleanupViewerP2P,
clearP2PSessionForViewer,
createP2PSession,
getP2PSessionIdForViewer,
} from "./ws-console-rd-hvnc";
function sendDesktopAudioCommand(
clientId: string,
commandType: "desktop_audio_start" | "desktop_audio_stop",
commandType: "desktop_audio_start" | "desktop_audio_stop" | "webrtc_publish" | "webrtc_stop" | "webrtc_p2p_offer" | "webrtc_p2p_ice" | "webrtc_p2p_stop",
payload: Record<string, unknown>,
) {
const target = clientManager.getClient(clientId);
@@ -58,26 +65,86 @@ export function handleDesktopAudioViewerOpen(ws: ServerWebSocket<SocketData>) {
export function handleDesktopAudioViewerMessage(ws: ServerWebSocket<SocketData>, raw: string | ArrayBuffer | Uint8Array) {
const { clientId } = ws.data;
if (typeof raw === "string") {
try {
const message = JSON.parse(raw);
if (message?.type === "start") {
const source = typeof message?.source === "string" ? message.source : "system";
const started = sendDesktopAudioCommand(clientId, "desktop_audio_start", {
source,
sessionId: ws.data.sessionId,
});
safeJson(ws, { type: "status", status: started ? "connected" : "error" });
return;
}
if (message?.type === "stop") {
sendDesktopAudioCommand(clientId, "desktop_audio_stop", {});
safeJson(ws, { type: "status", status: "disconnected" });
}
} catch {
}
if (typeof raw !== "string") return;
let message: any;
try {
message = JSON.parse(raw);
} catch {
return;
}
if (!message || typeof message.type !== "string") return;
switch (message.type) {
case "start": {
const source = typeof message.source === "string" ? message.source : "system";
const started = sendDesktopAudioCommand(clientId, "desktop_audio_start", {
source,
sessionId: ws.data.sessionId,
});
if (message.webrtc === true && started) {
const streamPath = webrtcStreamPathFor(clientId, "audio");
const token = issueWebrtcPublishToken(clientId);
sendDesktopAudioCommand(clientId, "webrtc_publish", {
streamPath,
whipPath: `/api/webrtc/${streamPath}/whip`,
token,
kind: "audio",
hasVideo: false,
hasAudio: true,
});
safeJson(ws, {
type: "webrtc_ready",
streamPath,
whepPath: `/api/webrtc/${streamPath}/whep`,
});
}
safeJson(ws, { type: "status", status: started ? "connected" : "error" });
return;
}
case "stop": {
sendDesktopAudioCommand(clientId, "desktop_audio_stop", {});
sendDesktopAudioCommand(clientId, "webrtc_stop", { kind: "audio" });
safeJson(ws, { type: "status", status: "disconnected" });
return;
}
case "webrtc_p2p_offer": {
const sdp = typeof message.sdp === "string" ? message.sdp : "";
if (!sdp) return;
const sessionId = createP2PSession(ws, clientId, "audio");
sendDesktopAudioCommand(clientId, "webrtc_p2p_offer", {
sessionId,
sdp,
kind: "audio",
hasVideo: false,
hasAudio: true,
});
return;
}
case "webrtc_p2p_ice": {
const sessionId = getP2PSessionIdForViewer(ws);
if (!sessionId) return;
const candidate = typeof message.candidate === "string" ? message.candidate : "";
if (!candidate) return;
sendDesktopAudioCommand(clientId, "webrtc_p2p_ice", {
sessionId,
kind: "audio",
candidate,
sdpMid: typeof message.sdpMid === "string" ? message.sdpMid : "",
sdpMLineIndex: Number(message.sdpMLineIndex) || 0,
});
return;
}
case "webrtc_p2p_stop": {
const cleared = clearP2PSessionForViewer(ws);
if (cleared) {
sendDesktopAudioCommand(clientId, "webrtc_p2p_stop", {
sessionId: cleared.sessionId,
kind: cleared.kind,
});
}
return;
}
}
}
export function handleDesktopAudioUplink(clientId: string, payload: any) {
@@ -101,6 +168,7 @@ export function handleDesktopAudioUplink(clientId: string, payload: any) {
}
export function cleanupDesktopAudioViewer(ws: ServerWebSocket<SocketData>) {
cleanupViewerP2P(ws);
let removedClientId = ws.data.clientId;
for (const [sid, session] of sessionManager.getAllDesktopAudioSessions().entries()) {
if (session.viewer === ws) {
@@ -113,5 +181,6 @@ export function cleanupDesktopAudioViewer(ws: ServerWebSocket<SocketData>) {
const hasViewers = sessionManager.getDesktopAudioSessionsByClient(removedClientId).length > 0;
if (!hasViewers) {
sendDesktopAudioCommand(removedClientId, "desktop_audio_stop", {});
sendDesktopAudioCommand(removedClientId, "webrtc_stop", { kind: "audio" });
}
}