fix some remote desktop stuff

Overlord-Server/public/assets/remotedesktop.js

@@ -152,10 +152,11 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./generated/s
   let frameWidth = 0;
   let frameHeight = 0;
   let latencyAvg = null;
-  let smoothingPct = 20;
+  let smoothingPct = 0;
   let smoothPoint = null;
   let pendingMove = null;
-  let moveFrame = 0;
+  let moveTimer = 0;
+  let lastMoveSentAt = 0;
   let frameDecodeBusy = false;
   let pendingFrame = null;
   let videoDecoder = null;
@@ -185,6 +186,11 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./generated/s
   const H264_MAX_RECOVERY_ATTEMPTS = 1;
   const H264_DECODE_WARN_THROTTLE_MS = 2000;
   const inputBackpressureBytes = 256 * 1024;
+  const inputMoveIntervalMs = 8;
+  const highFrequencyInputTypes = new Set([
+    "mouse_move", "mouse_down", "mouse_up", "mouse_wheel",
+    "key_down", "key_up", "text_input",
+  ]);
   const VIEWER_FRAME_GAP_KEYFRAME_THROTTLE_MS = 1000;
   let lastViewerKeyframeRequestAt = 0;
   const diagnostics = {
@@ -540,7 +546,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./generated/s
       privacy: !!privacyCtrl?.checked,
       audio: !!audioCtrl?.checked,
       audioTransport: getAudioTransport(),
-      smoothing: Number(smoothingSlider?.value || 20),
+      smoothing: Number(smoothingSlider?.value || 0),
       recordMode: recordMode?.value || "normal",
       recordFps: recordFps?.value || "",
     };
@@ -1696,7 +1702,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./generated/s
       return;
     }
     const msg = { type, ...payload };
-    if (type !== "desktop_decode_pressure") {
+    if (type !== "desktop_decode_pressure" && !highFrequencyInputTypes.has(type)) {
       rdDebug("send", {
         msg,
         readyState: ws.readyState,
@@ -2443,7 +2449,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./generated/s
       remoteCursor.hidden = true;
       return;
     }
-    const surfaceRect = surface.getBoundingClientRect();
+    const surfaceRect = getContainedSurfaceRect(surface, sourceWidth, sourceHeight);
     const containerRect = renderSurface.getBoundingClientRect();
     const scaleX = surfaceRect.width / sourceWidth;
     const scaleY = surfaceRect.height / sourceHeight;
@@ -3390,6 +3396,9 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./generated/s
       ? (webrtcVideo.videoHeight || frameHeight)
       : (canvas.height || frameHeight);
     if (!rect.width || !rect.height || !targetW || !targetH) return null;
+    rect = getContainedSurfaceRect(surface, targetW, targetH, rect);
+    if (e.clientX < rect.left || e.clientX > rect.right ||
+        e.clientY < rect.top || e.clientY > rect.bottom) return null;
     let x = ((e.clientX - rect.left) / rect.width) * targetW;
     let y = ((e.clientY - rect.top) / rect.height) * targetH;
     if (!Number.isFinite(x) || !Number.isFinite(y)) return null;
@@ -3398,14 +3407,32 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./generated/s
     return { x, y };
   }
 
-  function scheduleMouseMove() {
+  function getContainedSurfaceRect(surface, sourceWidth, sourceHeight, elementRect = surface.getBoundingClientRect()) {
-    if (!moveFrame) {
+    if (!elementRect.width || !elementRect.height || !sourceWidth || !sourceHeight) return elementRect;
-      moveFrame = requestAnimationFrame(flushMouseMove);
+    const fit = getComputedStyle(surface).objectFit;
+    if (fit !== "contain" && fit !== "scale-down") return elementRect;
+    let scale = Math.min(elementRect.width / sourceWidth, elementRect.height / sourceHeight);
+    if (fit === "scale-down") scale = Math.min(1, scale);
+    const width = sourceWidth * scale;
+    const height = sourceHeight * scale;
+    const left = elementRect.left + (elementRect.width - width) / 2;
+    const top = elementRect.top + (elementRect.height - height) / 2;
+    return {
+      x: left, y: top, left, top, right: left + width, bottom: top + height,
+      width, height, toJSON() { return this; },
+    };
   }
+
+  function scheduleMouseMove() {
+    if (moveTimer) return;
+    const wait = Math.max(0, inputMoveIntervalMs - (performance.now() - lastMoveSentAt));
+    if (wait === 0) flushMouseMove();
+    else moveTimer = window.setTimeout(flushMouseMove, wait);
   }
 
   function flushMouseMove() {
-    moveFrame = 0;
+    if (moveTimer) window.clearTimeout(moveTimer);
+    moveTimer = 0;
     if (!isInputActive() || !pendingMove || !mouseCtrl.checked) {
       smoothPoint = null;
       return;
@@ -3424,6 +3451,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./generated/s
 
     if (ws && ws.bufferedAmount <= inputBackpressureBytes) {
       sendCmd("mouse_move", sendPoint);
+      lastMoveSentAt = performance.now();
     }
 
     if (Math.abs(pendingMove.x - smoothPoint.x) > 0.5 ||
@@ -3447,23 +3475,19 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./generated/s
       canvasContainer.focus({ preventScroll: true });
       if (!isInputActive() || !mouseCtrl.checked) return;
       const pt = getRenderPoint(e);
-      if (pt) {
+      if (!pt) return;
       pendingMove = pt;
       smoothPoint = { x: pt.x, y: pt.y };
-        sendCmd("mouse_move", pt);
+      sendCmd("mouse_down", { button: e.button, ...pt });
-      }
-      sendCmd("mouse_down", { button: e.button, ...(pt || {}) });
       e.preventDefault();
     });
     inputSurface.addEventListener("mouseup", function (e) {
       if (!isInputActive() || !mouseCtrl.checked) return;
       const pt = getRenderPoint(e);
-      if (pt) {
+      if (!pt) return;
       pendingMove = pt;
       smoothPoint = { x: pt.x, y: pt.y };
-        sendCmd("mouse_move", pt);
+      sendCmd("mouse_up", { button: e.button, ...pt });
-      }
-      sendCmd("mouse_up", { button: e.button, ...(pt || {}) });
       e.preventDefault();
     });
     inputSurface.addEventListener("contextmenu", function (e) {

Overlord-Server/public/remotedesktop.html

@@ -272,8 +272,8 @@
                 </div>
                 <div class="flex items-center gap-2" title="Smooths mouse input for steadier movement">
                   <label class="text-sm text-slate-300 w-20" for="smoothingSlider">Smoothing</label>
-                  <input id="smoothingSlider" type="range" min="0" max="80" step="5" value="20" class="flex-1 accent-emerald-400" />
+                  <input id="smoothingSlider" type="range" min="0" max="80" step="5" value="0" class="flex-1 accent-emerald-400" />
-                  <span id="smoothingValue" class="pill pill-ghost text-slate-100 text-xs min-w-[44px] text-center">20%</span>
+                  <span id="smoothingValue" class="pill pill-ghost text-slate-100 text-xs min-w-[44px] text-center">0%</span>
                 </div>
               </div>
 

Overlord-Server/src/server/ws-console-rd-backstage.ts

@@ -237,7 +237,11 @@ function sendDesktopCommandWithId(target: ClientInfo | undefined, commandType: s
     return false;
   }
   try {
+    const isInteractiveInput = commandType.startsWith("desktop_mouse_") ||
+      commandType.startsWith("desktop_key_") || commandType === "desktop_text";
+    if (!isInteractiveInput) {
       logger.debug(`[rd-debug] send command command=${commandType} client=${target.id} id=${commandId} payload=${JSON.stringify(payload || {})}`);
+    }
     target.ws.send(encodeMessage({ type: "command", commandType: commandType as any, payload, id: commandId }));
     metrics.recordCommand(commandType);
     return true;
@@ -648,14 +652,22 @@ export function handleRemoteDesktopViewerMessage(ws: ServerWebSocket<SocketData>
       if (!state.isStreaming) break;
       const commandId = uuidv4();
       recordRdInput(commandId, clientId, "mouse_down");
-      sendDesktopCommandWithId(target, "desktop_mouse_down", { button: Number(payload.button) || 0 }, commandId);
+      const hasPoint = Number.isFinite(Number((payload as any).x)) && Number.isFinite(Number((payload as any).y));
+      sendDesktopCommandWithId(target, "desktop_mouse_down", {
+        button: Number(payload.button) || 0,
+        ...(hasPoint ? { x: Number((payload as any).x), y: Number((payload as any).y) } : {}),
+      }, commandId);
       break;
     }
     case "mouse_up": {
       if (!state.isStreaming) break;
       const commandId = uuidv4();
       recordRdInput(commandId, clientId, "mouse_up");
-      sendDesktopCommandWithId(target, "desktop_mouse_up", { button: Number(payload.button) || 0 }, commandId);
+      const hasPoint = Number.isFinite(Number((payload as any).x)) && Number.isFinite(Number((payload as any).y));
+      sendDesktopCommandWithId(target, "desktop_mouse_up", {
+        button: Number(payload.button) || 0,
+        ...(hasPoint ? { x: Number((payload as any).x), y: Number((payload as any).y) } : {}),
+      }, commandId);
       break;
     }
     case "mouse_wheel": {

Overlord-Server/test/e2e/remotedesktop-input.e2e.ts

@@ -208,6 +208,35 @@ test("WebRTC input stays inactive until start and then forwards mouse and keyboa
     expect.objectContaining({ type: "key_up", key: "Enter", code: "Enter" }),
   ]));
 
+  await page.evaluate(() => {
+    const sent = (window as typeof window & { __rdSent: ArrayBuffer[] }).__rdSent;
+    sent.length = 0;
+    const video = document.querySelector<HTMLVideoElement>("#webrtcVideo")!;
+    video.style.objectFit = "contain";
+    video.getBoundingClientRect = () => ({
+      x: 0, y: 0, left: 0, top: 0, right: 1000, bottom: 1000,
+      width: 1000, height: 1000, toJSON() { return this; },
+    });
+    video.dispatchEvent(new MouseEvent("mousedown", { clientX: 500, clientY: 100, button: 0, bubbles: true }));
+  });
+  await page.waitForTimeout(20);
+  await expect.poll(async () => page.evaluate(async () => {
+    const sent = (window as typeof window & { __rdSent: ArrayBuffer[] }).__rdSent;
+    const { decodeMsgpack } = await import("/assets/msgpack-helpers.js");
+    return sent.map((message) => decodeMsgpack(message))
+      .filter((message) => /^(mouse_|key_|text_input)/.test(message.type));
+  })).toEqual([]);
+
+  await page.evaluate(() => {
+    const video = document.querySelector<HTMLVideoElement>("#webrtcVideo")!;
+    video.dispatchEvent(new MouseEvent("mousedown", { clientX: 250, clientY: 360, button: 0, bubbles: true }));
+  });
+  await expect.poll(async () => page.evaluate(async () => {
+    const sent = (window as typeof window & { __rdSent: ArrayBuffer[] }).__rdSent;
+    const { decodeMsgpack } = await import("/assets/msgpack-helpers.js");
+    return sent.map((message) => decodeMsgpack(message));
+  })).toContainEqual(expect.objectContaining({ type: "mouse_down", button: 0, x: 480, y: 271 }));
+
   await page.evaluate(() => {
     const socket = (window as typeof window & { __rdSocket: EventTarget }).__rdSocket;
     const framePacket = Uint8Array.from([0x46, 0x52, 0x4d, 0, 0, 0, 0, 0]).buffer;