mouse smoothing + fix fuck ass tests

Overlord-Server/bunfig.toml

@@ -1,2 +1,3 @@
 [test]
 preload = ["./test/preload.ts"]
+pathIgnorePatterns = ["test/e2e/**"]

Overlord-Server/playwright.config.ts

@@ -5,6 +5,7 @@ const baseURL = `http://127.0.0.1:${port}`;
 
 export default defineConfig({
   testDir: "./test/e2e",
+  testMatch: "**/*.e2e.ts",
   globalSetup: "./test/e2e/global-setup.ts",
   fullyParallel: false,
   workers: 1,

Overlord-Server/public/assets/backstage.js

@@ -145,12 +145,10 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
   let pendingFrame = null;
   let hasCanvasBase = false;
   let pendingMove = null;
-  let moveTimer = null;
+  let moveFrame = 0;
   let videoDecoder = null;
   let h264TimestampUs = 0;
   let prefersH264 = typeof VideoDecoder === "function";
-  let lastMoveSentAt = 0;
-  const mouseMoveIntervalMs = 33;
   const inputBackpressureBytes = 256 * 1024;
   let h264ErrorCount = 0;
   let h264RetryTimer = null;
@@ -1534,17 +1532,15 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
     return { x, y };
   }
 
-  function flushMouseMove() {
-    moveTimer = null;
-    if (!pendingMove || !mouseCtrl.checked) return;
-    const now = performance.now();
-    if (now - lastMoveSentAt < mouseMoveIntervalMs) {
-      if (!moveTimer) {
-        moveTimer = setTimeout(flushMouseMove, mouseMoveIntervalMs);
-      }
-      return;
+  function scheduleMouseMove() {
+    if (!moveFrame) {
+      moveFrame = requestAnimationFrame(flushMouseMove);
     }
-    lastMoveSentAt = now;
+  }
+
+  function flushMouseMove() {
+    moveFrame = 0;
+    if (!desiredStreaming || !pendingMove || !mouseCtrl.checked) return;
     if (ws.bufferedAmount <= inputBackpressureBytes) {
       sendCmd("backstage_mouse_move", pendingMove);
     }
@@ -1553,17 +1549,15 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
   for (const inputSurface of [canvas, webrtcVideo]) {
     if (!inputSurface) continue;
     inputSurface.addEventListener("mousemove", function (e) {
-      if (!mouseCtrl.checked) return;
+      if (!desiredStreaming || !mouseCtrl.checked) return;
       const pt = getRenderPoint(e);
       if (!pt) return;
       pendingMove = pt;
-      if (!moveTimer) {
-        flushMouseMove();
-      }
+      scheduleMouseMove();
     });
     inputSurface.addEventListener("mousedown", function (e) {
       canvasContainer.focus({ preventScroll: true });
-      if (!mouseCtrl.checked) return;
+      if (!desiredStreaming || !mouseCtrl.checked) return;
       const pt = getRenderPoint(e);
       if (pt) {
         pendingMove = pt;
@@ -1575,7 +1569,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
       e.preventDefault();
     });
     inputSurface.addEventListener("mouseup", function (e) {
-      if (!mouseCtrl.checked) return;
+      if (!desiredStreaming || !mouseCtrl.checked) return;
       const pt = getRenderPoint(e);
       if (pt) {
         pendingMove = pt;
@@ -1587,10 +1581,10 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
       e.preventDefault();
     });
     inputSurface.addEventListener("contextmenu", function (e) {
-      e.preventDefault();
+      if (desiredStreaming && mouseCtrl.checked) e.preventDefault();
     });
     inputSurface.addEventListener("wheel", function (e) {
-      if (!mouseCtrl.checked) return;
+      if (!desiredStreaming || !mouseCtrl.checked) return;
       const pt = getRenderPoint(e);
       if (!pt) return;
       const delta = Math.max(-120, Math.min(120, Math.round(-e.deltaY)));
@@ -1604,8 +1598,12 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
   canvasContainer.setAttribute("tabindex", "0");
   const kbdCapture = createKeyboardCapture({
     container: canvasContainer,
-    sendKeyDown: (e) => sendCmd("backstage_key_down", { key: e.key, code: e.code }),
-    sendKeyUp: (e) => sendCmd("backstage_key_up", { key: e.key, code: e.code }),
+    sendKeyDown: (e) => {
+      if (desiredStreaming) sendCmd("backstage_key_down", { key: e.key, code: e.code });
+    },
+    sendKeyUp: (e) => {
+      if (desiredStreaming) sendCmd("backstage_key_up", { key: e.key, code: e.code });
+    },
   });
   if (kbdCtrl) {
     kbdCtrl.addEventListener("change", function () {

Overlord-Server/public/assets/remotedesktop.js

@@ -117,7 +117,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
   let smoothingPct = 20;
   let smoothPoint = null;
   let pendingMove = null;
-  let moveTimer = null;
+  let moveFrame = 0;
   let frameDecodeBusy = false;
   let pendingFrame = null;
   let videoDecoder = null;
@@ -143,9 +143,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
   const H264_KEYFRAME_ERROR_RESTART_THRESHOLD = 24;
   const H264_MAX_RECOVERY_ATTEMPTS = 1;
   const H264_DECODE_WARN_THROTTLE_MS = 2000;
-  const mouseMoveIntervalMs = 33;
   const inputBackpressureBytes = 256 * 1024;
-  let lastMoveSentAt = 0;
   const diagnostics = {
     agent: null,
     network: null,
@@ -1840,6 +1838,10 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
     }
   }
 
+  function isInputActive() {
+    return desiredStreaming;
+  }
+
   mouseCtrl.addEventListener("change", function () {
     pushInputToggles();
     sharedSettingsSaver.scheduleSave();
@@ -1847,9 +1849,15 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
   canvasContainer.setAttribute("tabindex", "0");
   const kbdCapture = createKeyboardCapture({
     container: canvasContainer,
-    sendKeyDown: (e) => sendCmd("key_down", { key: e.key, code: e.code }),
-    sendKeyUp: (e) => sendCmd("key_up", { key: e.key, code: e.code }),
-    onTextInput: (e) => sendCmd("text_input", { text: e.key }),
+    sendKeyDown: (e) => {
+      if (isInputActive()) sendCmd("key_down", { key: e.key, code: e.code });
+    },
+    sendKeyUp: (e) => {
+      if (isInputActive()) sendCmd("key_up", { key: e.key, code: e.code });
+    },
+    onTextInput: (e) => {
+      if (isInputActive()) sendCmd("text_input", { text: e.key });
+    },
   });
   kbdCtrl.addEventListener("change", function () {
     if (kbdCtrl.checked) kbdCapture.enable();
@@ -2666,50 +2674,54 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
     return { x, y };
   }
 
+  function scheduleMouseMove() {
+    if (!moveFrame) {
+      moveFrame = requestAnimationFrame(flushMouseMove);
+    }
+  }
+
   function flushMouseMove() {
-    moveTimer = null;
-    if (!pendingMove || !mouseCtrl.checked) return;
-    const now = performance.now();
+    moveFrame = 0;
+    if (!isInputActive() || !pendingMove || !mouseCtrl.checked) {
+      smoothPoint = null;
+      return;
+    }
     if (!smoothPoint) {
       smoothPoint = { x: pendingMove.x, y: pendingMove.y };
     }
-    const factor = Math.max(0, Math.min(0.8, smoothingPct / 100));
-    const alpha = 1 - factor;
-    smoothPoint.x += (pendingMove.x - smoothPoint.x) * alpha;
-    smoothPoint.y += (pendingMove.y - smoothPoint.y) * alpha;
 
+    const factor = Math.max(0, Math.min(0.4, smoothingPct / 200));
+    smoothPoint.x += (pendingMove.x - smoothPoint.x) * (1 - factor);
+    smoothPoint.y += (pendingMove.y - smoothPoint.y) * (1 - factor);
     const sendPoint = {
       x: Math.round(smoothPoint.x),
       y: Math.round(smoothPoint.y),
     };
 
-    if (now - lastMoveSentAt < mouseMoveIntervalMs) {
-      if (!moveTimer) {
-        moveTimer = setTimeout(flushMouseMove, mouseMoveIntervalMs);
-      }
-      return;
-    }
-
-    lastMoveSentAt = now;
     if (ws && ws.bufferedAmount <= inputBackpressureBytes) {
       sendCmd("mouse_move", sendPoint);
     }
+
+    if (Math.abs(pendingMove.x - smoothPoint.x) > 0.5 ||
+        Math.abs(pendingMove.y - smoothPoint.y) > 0.5) {
+      scheduleMouseMove();
+    } else {
+      smoothPoint = { x: pendingMove.x, y: pendingMove.y };
+    }
   }
 
   for (const inputSurface of [canvas, webrtcVideo]) {
     if (!inputSurface) continue;
     inputSurface.addEventListener("mousemove", function (e) {
-      if (!mouseCtrl.checked) return;
+      if (!isInputActive() || !mouseCtrl.checked) return;
       const pt = getRenderPoint(e);
       if (!pt) return;
       pendingMove = pt;
-      if (!moveTimer) {
-        flushMouseMove();
-      }
+      scheduleMouseMove();
     });
     inputSurface.addEventListener("mousedown", function (e) {
       canvasContainer.focus({ preventScroll: true });
-      if (!mouseCtrl.checked) return;
+      if (!isInputActive() || !mouseCtrl.checked) return;
       const pt = getRenderPoint(e);
       if (pt) {
         pendingMove = pt;
@@ -2720,7 +2732,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
       e.preventDefault();
     });
     inputSurface.addEventListener("mouseup", function (e) {
-      if (!mouseCtrl.checked) return;
+      if (!isInputActive() || !mouseCtrl.checked) return;
       const pt = getRenderPoint(e);
       if (pt) {
         pendingMove = pt;
@@ -2731,10 +2743,10 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
       e.preventDefault();
     });
     inputSurface.addEventListener("contextmenu", function (e) {
-      e.preventDefault();
+      if (isInputActive() && mouseCtrl.checked) e.preventDefault();
     });
     inputSurface.addEventListener("wheel", function (e) {
-      if (!mouseCtrl.checked) return;
+      if (!isInputActive() || !mouseCtrl.checked) return;
       const pt = getRenderPoint(e);
       if (!pt) return;
       const delta = Math.max(-120, Math.min(120, Math.round(-e.deltaY)));

Overlord-Server/src/paths.test.ts

@@ -8,9 +8,15 @@ test("creates isolated test data when the Bun preload was not discovered", async
   delete childEnv.DATA_DIR;
   delete childEnv.OVERLORD_TEST_DATA_DIR;
   childEnv.NODE_ENV = "test";
+  // VS Code's Bun adapter injects inspector settings into the test process.
+  // A child that inherits them can wait for the same debugger and never exit.
+  delete childEnv.BUN_OPTIONS;
+  for (const key of Object.keys(childEnv)) {
+    if (key.startsWith("BUN_INSPECT")) delete childEnv[key];
+  }
 
   const child = Bun.spawn([
-    "bun",
+    process.execPath,
     "-e",
     'const { resolveDataDir } = await import("./src/paths.ts"); console.log(resolveDataDir());',
   ], {

Overlord-Server/test/e2e/remotedesktop-input.e2e.ts

@@ -1,7 +1,7 @@
 import { expect, test } from "@playwright/test";
 import { login } from "./helpers";
 
-test("WebRTC video forwards mouse and keyboard input", async ({ page }) => {
+test("WebRTC input stays inactive until start and then forwards mouse and keyboard", async ({ page }) => {
   await login(page);
   await page.addInitScript(() => {
     const sent: ArrayBuffer[] = [];
@@ -71,6 +71,22 @@ test("WebRTC video forwards mouse and keyboard input", async ({ page }) => {
       toJSON() { return this; },
     });
 
+    video.dispatchEvent(new MouseEvent("mousedown", { clientX: 480, clientY: 270, button: 0, bubbles: true }));
+    window.dispatchEvent(new KeyboardEvent("keydown", { key: "Enter", code: "Enter", bubbles: true }));
+  });
+
+  await expect.poll(async () => page.evaluate(async () => {
+    const sent = (window as typeof window & { __rdSent: ArrayBuffer[] }).__rdSent;
+    const { decodeMsgpack } = await import("/assets/msgpack-helpers.js");
+    return sent.map((message) => decodeMsgpack(message))
+      .filter((message) => /^(mouse_|key_|text_input)/.test(message.type));
+  })).toEqual([]);
+
+  await page.locator("#startBtn").click();
+  await page.evaluate(() => {
+    const sent = (window as typeof window & { __rdSent: ArrayBuffer[] }).__rdSent;
+    sent.length = 0;
+    const video = document.querySelector<HTMLVideoElement>("#webrtcVideo")!;
     video.dispatchEvent(new MouseEvent("mousemove", { clientX: 480, clientY: 270, bubbles: true }));
     video.dispatchEvent(new MouseEvent("mousedown", { clientX: 480, clientY: 270, button: 0, bubbles: true }));
     video.dispatchEvent(new MouseEvent("mouseup", { clientX: 480, clientY: 270, button: 0, bubbles: true }));