From 12cb3f6b4cae5f8072852fbd12fa9442f009531d Mon Sep 17 00:00:00 2001 From: KingKDot Date: Sat, 18 Jul 2026 19:40:36 -0500 Subject: [PATCH] mouse smoothing + fix fuck ass tests --- Overlord-Server/bunfig.toml | 1 + Overlord-Server/playwright.config.ts | 1 + Overlord-Server/public/assets/backstage.js | 44 ++++++------ .../public/assets/remotedesktop.js | 70 +++++++++++-------- Overlord-Server/src/paths.test.ts | 8 ++- .../e2e/{access.spec.ts => access.e2e.ts} | 0 ...r-api.spec.ts => file-transfer-api.e2e.ts} | 0 ...put.spec.ts => remotedesktop-input.e2e.ts} | 18 ++++- .../test/e2e/{smoke.spec.ts => smoke.e2e.ts} | 0 9 files changed, 88 insertions(+), 54 deletions(-) rename Overlord-Server/test/e2e/{access.spec.ts => access.e2e.ts} (100%) rename Overlord-Server/test/e2e/{file-transfer-api.spec.ts => file-transfer-api.e2e.ts} (100%) rename Overlord-Server/test/e2e/{remotedesktop-input.spec.ts => remotedesktop-input.e2e.ts} (80%) rename Overlord-Server/test/e2e/{smoke.spec.ts => smoke.e2e.ts} (100%) diff --git a/Overlord-Server/bunfig.toml b/Overlord-Server/bunfig.toml index 786a377..07ec35d 100644 --- a/Overlord-Server/bunfig.toml +++ b/Overlord-Server/bunfig.toml @@ -1,2 +1,3 @@ [test] preload = ["./test/preload.ts"] +pathIgnorePatterns = ["test/e2e/**"] diff --git a/Overlord-Server/playwright.config.ts b/Overlord-Server/playwright.config.ts index 4ce3ce7..50fa80d 100644 --- a/Overlord-Server/playwright.config.ts +++ b/Overlord-Server/playwright.config.ts @@ -5,6 +5,7 @@ const baseURL = `http://127.0.0.1:${port}`; export default defineConfig({ testDir: "./test/e2e", + testMatch: "**/*.e2e.ts", globalSetup: "./test/e2e/global-setup.ts", fullyParallel: false, workers: 1, diff --git a/Overlord-Server/public/assets/backstage.js b/Overlord-Server/public/assets/backstage.js index 7b4aab8..5dff681 100644 --- a/Overlord-Server/public/assets/backstage.js +++ b/Overlord-Server/public/assets/backstage.js @@ -145,12 +145,10 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s let pendingFrame = null; let hasCanvasBase = false; let pendingMove = null; - let moveTimer = null; + let moveFrame = 0; let videoDecoder = null; let h264TimestampUs = 0; let prefersH264 = typeof VideoDecoder === "function"; - let lastMoveSentAt = 0; - const mouseMoveIntervalMs = 33; const inputBackpressureBytes = 256 * 1024; let h264ErrorCount = 0; let h264RetryTimer = null; @@ -1534,17 +1532,15 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s return { x, y }; } - function flushMouseMove() { - moveTimer = null; - if (!pendingMove || !mouseCtrl.checked) return; - const now = performance.now(); - if (now - lastMoveSentAt < mouseMoveIntervalMs) { - if (!moveTimer) { - moveTimer = setTimeout(flushMouseMove, mouseMoveIntervalMs); - } - return; + function scheduleMouseMove() { + if (!moveFrame) { + moveFrame = requestAnimationFrame(flushMouseMove); } - lastMoveSentAt = now; + } + + function flushMouseMove() { + moveFrame = 0; + if (!desiredStreaming || !pendingMove || !mouseCtrl.checked) return; if (ws.bufferedAmount <= inputBackpressureBytes) { sendCmd("backstage_mouse_move", pendingMove); } @@ -1553,17 +1549,15 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s for (const inputSurface of [canvas, webrtcVideo]) { if (!inputSurface) continue; inputSurface.addEventListener("mousemove", function (e) { - if (!mouseCtrl.checked) return; + if (!desiredStreaming || !mouseCtrl.checked) return; const pt = getRenderPoint(e); if (!pt) return; pendingMove = pt; - if (!moveTimer) { - flushMouseMove(); - } + scheduleMouseMove(); }); inputSurface.addEventListener("mousedown", function (e) { canvasContainer.focus({ preventScroll: true }); - if (!mouseCtrl.checked) return; + if (!desiredStreaming || !mouseCtrl.checked) return; const pt = getRenderPoint(e); if (pt) { pendingMove = pt; @@ -1575,7 +1569,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s e.preventDefault(); }); inputSurface.addEventListener("mouseup", function (e) { - if (!mouseCtrl.checked) return; + if (!desiredStreaming || !mouseCtrl.checked) return; const pt = getRenderPoint(e); if (pt) { pendingMove = pt; @@ -1587,10 +1581,10 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s e.preventDefault(); }); inputSurface.addEventListener("contextmenu", function (e) { - e.preventDefault(); + if (desiredStreaming && mouseCtrl.checked) e.preventDefault(); }); inputSurface.addEventListener("wheel", function (e) { - if (!mouseCtrl.checked) return; + if (!desiredStreaming || !mouseCtrl.checked) return; const pt = getRenderPoint(e); if (!pt) return; const delta = Math.max(-120, Math.min(120, Math.round(-e.deltaY))); @@ -1604,8 +1598,12 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s canvasContainer.setAttribute("tabindex", "0"); const kbdCapture = createKeyboardCapture({ container: canvasContainer, - sendKeyDown: (e) => sendCmd("backstage_key_down", { key: e.key, code: e.code }), - sendKeyUp: (e) => sendCmd("backstage_key_up", { key: e.key, code: e.code }), + sendKeyDown: (e) => { + if (desiredStreaming) sendCmd("backstage_key_down", { key: e.key, code: e.code }); + }, + sendKeyUp: (e) => { + if (desiredStreaming) sendCmd("backstage_key_up", { key: e.key, code: e.code }); + }, }); if (kbdCtrl) { kbdCtrl.addEventListener("change", function () { diff --git a/Overlord-Server/public/assets/remotedesktop.js b/Overlord-Server/public/assets/remotedesktop.js index 61258a9..b951379 100644 --- a/Overlord-Server/public/assets/remotedesktop.js +++ b/Overlord-Server/public/assets/remotedesktop.js @@ -117,7 +117,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s let smoothingPct = 20; let smoothPoint = null; let pendingMove = null; - let moveTimer = null; + let moveFrame = 0; let frameDecodeBusy = false; let pendingFrame = null; let videoDecoder = null; @@ -143,9 +143,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s const H264_KEYFRAME_ERROR_RESTART_THRESHOLD = 24; const H264_MAX_RECOVERY_ATTEMPTS = 1; const H264_DECODE_WARN_THROTTLE_MS = 2000; - const mouseMoveIntervalMs = 33; const inputBackpressureBytes = 256 * 1024; - let lastMoveSentAt = 0; const diagnostics = { agent: null, network: null, @@ -1840,6 +1838,10 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s } } + function isInputActive() { + return desiredStreaming; + } + mouseCtrl.addEventListener("change", function () { pushInputToggles(); sharedSettingsSaver.scheduleSave(); @@ -1847,9 +1849,15 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s canvasContainer.setAttribute("tabindex", "0"); const kbdCapture = createKeyboardCapture({ container: canvasContainer, - sendKeyDown: (e) => sendCmd("key_down", { key: e.key, code: e.code }), - sendKeyUp: (e) => sendCmd("key_up", { key: e.key, code: e.code }), - onTextInput: (e) => sendCmd("text_input", { text: e.key }), + sendKeyDown: (e) => { + if (isInputActive()) sendCmd("key_down", { key: e.key, code: e.code }); + }, + sendKeyUp: (e) => { + if (isInputActive()) sendCmd("key_up", { key: e.key, code: e.code }); + }, + onTextInput: (e) => { + if (isInputActive()) sendCmd("text_input", { text: e.key }); + }, }); kbdCtrl.addEventListener("change", function () { if (kbdCtrl.checked) kbdCapture.enable(); @@ -2666,50 +2674,54 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s return { x, y }; } + function scheduleMouseMove() { + if (!moveFrame) { + moveFrame = requestAnimationFrame(flushMouseMove); + } + } + function flushMouseMove() { - moveTimer = null; - if (!pendingMove || !mouseCtrl.checked) return; - const now = performance.now(); + moveFrame = 0; + if (!isInputActive() || !pendingMove || !mouseCtrl.checked) { + smoothPoint = null; + return; + } if (!smoothPoint) { smoothPoint = { x: pendingMove.x, y: pendingMove.y }; } - const factor = Math.max(0, Math.min(0.8, smoothingPct / 100)); - const alpha = 1 - factor; - smoothPoint.x += (pendingMove.x - smoothPoint.x) * alpha; - smoothPoint.y += (pendingMove.y - smoothPoint.y) * alpha; + const factor = Math.max(0, Math.min(0.4, smoothingPct / 200)); + smoothPoint.x += (pendingMove.x - smoothPoint.x) * (1 - factor); + smoothPoint.y += (pendingMove.y - smoothPoint.y) * (1 - factor); const sendPoint = { x: Math.round(smoothPoint.x), y: Math.round(smoothPoint.y), }; - if (now - lastMoveSentAt < mouseMoveIntervalMs) { - if (!moveTimer) { - moveTimer = setTimeout(flushMouseMove, mouseMoveIntervalMs); - } - return; - } - - lastMoveSentAt = now; if (ws && ws.bufferedAmount <= inputBackpressureBytes) { sendCmd("mouse_move", sendPoint); } + + if (Math.abs(pendingMove.x - smoothPoint.x) > 0.5 || + Math.abs(pendingMove.y - smoothPoint.y) > 0.5) { + scheduleMouseMove(); + } else { + smoothPoint = { x: pendingMove.x, y: pendingMove.y }; + } } for (const inputSurface of [canvas, webrtcVideo]) { if (!inputSurface) continue; inputSurface.addEventListener("mousemove", function (e) { - if (!mouseCtrl.checked) return; + if (!isInputActive() || !mouseCtrl.checked) return; const pt = getRenderPoint(e); if (!pt) return; pendingMove = pt; - if (!moveTimer) { - flushMouseMove(); - } + scheduleMouseMove(); }); inputSurface.addEventListener("mousedown", function (e) { canvasContainer.focus({ preventScroll: true }); - if (!mouseCtrl.checked) return; + if (!isInputActive() || !mouseCtrl.checked) return; const pt = getRenderPoint(e); if (pt) { pendingMove = pt; @@ -2720,7 +2732,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s e.preventDefault(); }); inputSurface.addEventListener("mouseup", function (e) { - if (!mouseCtrl.checked) return; + if (!isInputActive() || !mouseCtrl.checked) return; const pt = getRenderPoint(e); if (pt) { pendingMove = pt; @@ -2731,10 +2743,10 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s e.preventDefault(); }); inputSurface.addEventListener("contextmenu", function (e) { - e.preventDefault(); + if (isInputActive() && mouseCtrl.checked) e.preventDefault(); }); inputSurface.addEventListener("wheel", function (e) { - if (!mouseCtrl.checked) return; + if (!isInputActive() || !mouseCtrl.checked) return; const pt = getRenderPoint(e); if (!pt) return; const delta = Math.max(-120, Math.min(120, Math.round(-e.deltaY))); diff --git a/Overlord-Server/src/paths.test.ts b/Overlord-Server/src/paths.test.ts index fe16f88..d59b48b 100644 --- a/Overlord-Server/src/paths.test.ts +++ b/Overlord-Server/src/paths.test.ts @@ -8,9 +8,15 @@ test("creates isolated test data when the Bun preload was not discovered", async delete childEnv.DATA_DIR; delete childEnv.OVERLORD_TEST_DATA_DIR; childEnv.NODE_ENV = "test"; + // VS Code's Bun adapter injects inspector settings into the test process. + // A child that inherits them can wait for the same debugger and never exit. + delete childEnv.BUN_OPTIONS; + for (const key of Object.keys(childEnv)) { + if (key.startsWith("BUN_INSPECT")) delete childEnv[key]; + } const child = Bun.spawn([ - "bun", + process.execPath, "-e", 'const { resolveDataDir } = await import("./src/paths.ts"); console.log(resolveDataDir());', ], { diff --git a/Overlord-Server/test/e2e/access.spec.ts b/Overlord-Server/test/e2e/access.e2e.ts similarity index 100% rename from Overlord-Server/test/e2e/access.spec.ts rename to Overlord-Server/test/e2e/access.e2e.ts diff --git a/Overlord-Server/test/e2e/file-transfer-api.spec.ts b/Overlord-Server/test/e2e/file-transfer-api.e2e.ts similarity index 100% rename from Overlord-Server/test/e2e/file-transfer-api.spec.ts rename to Overlord-Server/test/e2e/file-transfer-api.e2e.ts diff --git a/Overlord-Server/test/e2e/remotedesktop-input.spec.ts b/Overlord-Server/test/e2e/remotedesktop-input.e2e.ts similarity index 80% rename from Overlord-Server/test/e2e/remotedesktop-input.spec.ts rename to Overlord-Server/test/e2e/remotedesktop-input.e2e.ts index e88ab3e..b180b60 100644 --- a/Overlord-Server/test/e2e/remotedesktop-input.spec.ts +++ b/Overlord-Server/test/e2e/remotedesktop-input.e2e.ts @@ -1,7 +1,7 @@ import { expect, test } from "@playwright/test"; import { login } from "./helpers"; -test("WebRTC video forwards mouse and keyboard input", async ({ page }) => { +test("WebRTC input stays inactive until start and then forwards mouse and keyboard", async ({ page }) => { await login(page); await page.addInitScript(() => { const sent: ArrayBuffer[] = []; @@ -71,6 +71,22 @@ test("WebRTC video forwards mouse and keyboard input", async ({ page }) => { toJSON() { return this; }, }); + video.dispatchEvent(new MouseEvent("mousedown", { clientX: 480, clientY: 270, button: 0, bubbles: true })); + window.dispatchEvent(new KeyboardEvent("keydown", { key: "Enter", code: "Enter", bubbles: true })); + }); + + await expect.poll(async () => page.evaluate(async () => { + const sent = (window as typeof window & { __rdSent: ArrayBuffer[] }).__rdSent; + const { decodeMsgpack } = await import("/assets/msgpack-helpers.js"); + return sent.map((message) => decodeMsgpack(message)) + .filter((message) => /^(mouse_|key_|text_input)/.test(message.type)); + })).toEqual([]); + + await page.locator("#startBtn").click(); + await page.evaluate(() => { + const sent = (window as typeof window & { __rdSent: ArrayBuffer[] }).__rdSent; + sent.length = 0; + const video = document.querySelector("#webrtcVideo")!; video.dispatchEvent(new MouseEvent("mousemove", { clientX: 480, clientY: 270, bubbles: true })); video.dispatchEvent(new MouseEvent("mousedown", { clientX: 480, clientY: 270, button: 0, bubbles: true })); video.dispatchEvent(new MouseEvent("mouseup", { clientX: 480, clientY: 270, button: 0, bubbles: true })); diff --git a/Overlord-Server/test/e2e/smoke.spec.ts b/Overlord-Server/test/e2e/smoke.e2e.ts similarity index 100% rename from Overlord-Server/test/e2e/smoke.spec.ts rename to Overlord-Server/test/e2e/smoke.e2e.ts