From 6f6e28b8e74037f4eef675248b34e2913b3ce59d Mon Sep 17 00:00:00 2001 From: KingKDot Date: Sat, 18 Jul 2026 19:17:28 -0500 Subject: [PATCH] fix webrtc input --- Overlord-Server/public/assets/backstage.js | 120 ++++++++++-------- .../public/assets/remotedesktop.js | 111 ++++++++-------- .../test/e2e/remotedesktop-input.spec.ts | 94 ++++++++++++++ 3 files changed, 217 insertions(+), 108 deletions(-) create mode 100644 Overlord-Server/test/e2e/remotedesktop-input.spec.ts diff --git a/Overlord-Server/public/assets/backstage.js b/Overlord-Server/public/assets/backstage.js index 8217927..7b4aab8 100644 --- a/Overlord-Server/public/assets/backstage.js +++ b/Overlord-Server/public/assets/backstage.js @@ -1513,17 +1513,24 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s }, 1000); } - function getCanvasPoint(e) { - let rect = canvas.getBoundingClientRect(); + function getRenderPoint(e) { + const surface = e.currentTarget; + let rect = surface.getBoundingClientRect(); if (!rect.width || !rect.height) { rect = canvasContainer?.getBoundingClientRect() || rect; } - if (!rect.width || !rect.height || !canvas.width || !canvas.height) return null; - let x = ((e.clientX - rect.left) / rect.width) * canvas.width; - let y = ((e.clientY - rect.top) / rect.height) * canvas.height; + const targetW = surface === webrtcVideo + ? webrtcVideo.videoWidth + : canvas.width; + const targetH = surface === webrtcVideo + ? webrtcVideo.videoHeight + : canvas.height; + if (!rect.width || !rect.height || !targetW || !targetH) return null; + let x = ((e.clientX - rect.left) / rect.width) * targetW; + let y = ((e.clientY - rect.top) / rect.height) * targetH; if (!Number.isFinite(x) || !Number.isFinite(y)) return null; - x = Math.max(0, Math.min(canvas.width - 1, Math.floor(x))); - y = Math.max(0, Math.min(canvas.height - 1, Math.floor(y))); + x = Math.max(0, Math.min(targetW - 1, Math.floor(x))); + y = Math.max(0, Math.min(targetH - 1, Math.floor(y))); return { x, y }; } @@ -1543,59 +1550,60 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s } } - canvas.addEventListener("mousemove", function (e) { - if (!mouseCtrl.checked) return; - const pt = getCanvasPoint(e); - if (!pt) return; - pendingMove = pt; - if (!moveTimer) { - flushMouseMove(); - } - }); - canvas.addEventListener("mousedown", function (e) { - if (!mouseCtrl.checked) return; - canvas.focus({ preventScroll: true }); - const pt = getCanvasPoint(e); - if (pt) { + for (const inputSurface of [canvas, webrtcVideo]) { + if (!inputSurface) continue; + inputSurface.addEventListener("mousemove", function (e) { + if (!mouseCtrl.checked) return; + const pt = getRenderPoint(e); + if (!pt) return; pendingMove = pt; - if (ws.bufferedAmount <= inputBackpressureBytes) { - sendCmd("backstage_mouse_move", pt); + if (!moveTimer) { + flushMouseMove(); } - } - sendCmd("backstage_mouse_down", { button: e.button, ...(pt || {}) }); - e.preventDefault(); - }); - canvas.addEventListener("mouseup", function (e) { - if (!mouseCtrl.checked) return; - const pt = getCanvasPoint(e); - if (pt) { - pendingMove = pt; - if (ws.bufferedAmount <= inputBackpressureBytes) { - sendCmd("backstage_mouse_move", pt); + }); + inputSurface.addEventListener("mousedown", function (e) { + canvasContainer.focus({ preventScroll: true }); + if (!mouseCtrl.checked) return; + const pt = getRenderPoint(e); + if (pt) { + pendingMove = pt; + if (ws.bufferedAmount <= inputBackpressureBytes) { + sendCmd("backstage_mouse_move", pt); + } } - } - sendCmd("backstage_mouse_up", { button: e.button, ...(pt || {}) }); - e.preventDefault(); - }); - canvas.addEventListener("contextmenu", function (e) { - e.preventDefault(); - }); - - canvas.addEventListener("wheel", function (e) { - if (!mouseCtrl.checked) return; - const pt = getCanvasPoint(e); - if (!pt) return; - const delta = Math.max(-120, Math.min(120, Math.round(-e.deltaY))); - sendCmd("backstage_mouse_wheel", { delta, x: pt.x, y: pt.y }); - e.preventDefault(); - }, { passive: false }); - - canvas.setAttribute("tabindex", "0"); - canvas.addEventListener("click", function () { - canvas.focus({ preventScroll: true }); - }); + sendCmd("backstage_mouse_down", { button: e.button, ...(pt || {}) }); + e.preventDefault(); + }); + inputSurface.addEventListener("mouseup", function (e) { + if (!mouseCtrl.checked) return; + const pt = getRenderPoint(e); + if (pt) { + pendingMove = pt; + if (ws.bufferedAmount <= inputBackpressureBytes) { + sendCmd("backstage_mouse_move", pt); + } + } + sendCmd("backstage_mouse_up", { button: e.button, ...(pt || {}) }); + e.preventDefault(); + }); + inputSurface.addEventListener("contextmenu", function (e) { + e.preventDefault(); + }); + inputSurface.addEventListener("wheel", function (e) { + if (!mouseCtrl.checked) return; + const pt = getRenderPoint(e); + if (!pt) return; + const delta = Math.max(-120, Math.min(120, Math.round(-e.deltaY))); + sendCmd("backstage_mouse_wheel", { delta, x: pt.x, y: pt.y }); + e.preventDefault(); + }, { passive: false }); + inputSurface.addEventListener("click", function () { + canvasContainer.focus({ preventScroll: true }); + }); + } + canvasContainer.setAttribute("tabindex", "0"); const kbdCapture = createKeyboardCapture({ - container: canvas, + container: canvasContainer, sendKeyDown: (e) => sendCmd("backstage_key_down", { key: e.key, code: e.code }), sendKeyUp: (e) => sendCmd("backstage_key_up", { key: e.key, code: e.code }), }); diff --git a/Overlord-Server/public/assets/remotedesktop.js b/Overlord-Server/public/assets/remotedesktop.js index e3264e1..61258a9 100644 --- a/Overlord-Server/public/assets/remotedesktop.js +++ b/Overlord-Server/public/assets/remotedesktop.js @@ -1844,8 +1844,9 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s pushInputToggles(); sharedSettingsSaver.scheduleSave(); }); + canvasContainer.setAttribute("tabindex", "0"); const kbdCapture = createKeyboardCapture({ - container: canvas, + container: canvasContainer, sendKeyDown: (e) => sendCmd("key_down", { key: e.key, code: e.code }), sendKeyUp: (e) => sendCmd("key_up", { key: e.key, code: e.code }), onTextInput: (e) => sendCmd("text_input", { text: e.key }), @@ -2644,13 +2645,18 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s }, 1000); } - function getCanvasPoint(e) { - let rect = canvas.getBoundingClientRect(); + function getRenderPoint(e) { + const surface = e.currentTarget; + let rect = surface.getBoundingClientRect(); if (!rect.width || !rect.height) { rect = canvasContainer?.getBoundingClientRect() || rect; } - const targetW = canvas.width || frameWidth; - const targetH = canvas.height || frameHeight; + const targetW = surface === webrtcVideo + ? (webrtcVideo.videoWidth || frameWidth) + : (canvas.width || frameWidth); + const targetH = surface === webrtcVideo + ? (webrtcVideo.videoHeight || frameHeight) + : (canvas.height || frameHeight); if (!rect.width || !rect.height || !targetW || !targetH) return null; let x = ((e.clientX - rect.left) / rect.width) * targetW; let y = ((e.clientY - rect.top) / rect.height) * targetH; @@ -2690,54 +2696,55 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s } } - canvas.addEventListener("mousemove", function (e) { - if (!mouseCtrl.checked) return; - const pt = getCanvasPoint(e); - if (!pt) return; - pendingMove = pt; - if (!moveTimer) { - flushMouseMove(); - } - }); - canvas.addEventListener("mousedown", function (e) { - if (!mouseCtrl.checked) return; - canvas.focus({ preventScroll: true }); - const pt = getCanvasPoint(e); - if (pt) { + for (const inputSurface of [canvas, webrtcVideo]) { + if (!inputSurface) continue; + inputSurface.addEventListener("mousemove", function (e) { + if (!mouseCtrl.checked) return; + const pt = getRenderPoint(e); + if (!pt) return; pendingMove = pt; - smoothPoint = { x: pt.x, y: pt.y }; - sendCmd("mouse_move", pt); - } - sendCmd("mouse_down", { button: e.button, ...(pt || {}) }); - e.preventDefault(); - }); - canvas.addEventListener("mouseup", function (e) { - if (!mouseCtrl.checked) return; - const pt = getCanvasPoint(e); - if (pt) { - pendingMove = pt; - smoothPoint = { x: pt.x, y: pt.y }; - sendCmd("mouse_move", pt); - } - sendCmd("mouse_up", { button: e.button, ...(pt || {}) }); - e.preventDefault(); - }); - canvas.addEventListener("contextmenu", function (e) { - e.preventDefault(); - }); - canvas.addEventListener("wheel", function (e) { - if (!mouseCtrl.checked) return; - const pt = getCanvasPoint(e); - if (!pt) return; - const delta = Math.max(-120, Math.min(120, Math.round(-e.deltaY))); - sendCmd("mouse_wheel", { delta, x: pt.x, y: pt.y }); - e.preventDefault(); - }, { passive: false }); - - canvas.setAttribute("tabindex", "0"); - canvas.addEventListener("click", function () { - canvas.focus({ preventScroll: true }); - }); + if (!moveTimer) { + flushMouseMove(); + } + }); + inputSurface.addEventListener("mousedown", function (e) { + canvasContainer.focus({ preventScroll: true }); + if (!mouseCtrl.checked) return; + const pt = getRenderPoint(e); + if (pt) { + pendingMove = pt; + smoothPoint = { x: pt.x, y: pt.y }; + sendCmd("mouse_move", pt); + } + sendCmd("mouse_down", { button: e.button, ...(pt || {}) }); + e.preventDefault(); + }); + inputSurface.addEventListener("mouseup", function (e) { + if (!mouseCtrl.checked) return; + const pt = getRenderPoint(e); + if (pt) { + pendingMove = pt; + smoothPoint = { x: pt.x, y: pt.y }; + sendCmd("mouse_move", pt); + } + sendCmd("mouse_up", { button: e.button, ...(pt || {}) }); + e.preventDefault(); + }); + inputSurface.addEventListener("contextmenu", function (e) { + e.preventDefault(); + }); + inputSurface.addEventListener("wheel", function (e) { + if (!mouseCtrl.checked) return; + const pt = getRenderPoint(e); + if (!pt) return; + const delta = Math.max(-120, Math.min(120, Math.round(-e.deltaY))); + sendCmd("mouse_wheel", { delta, x: pt.x, y: pt.y }); + e.preventDefault(); + }, { passive: false }); + inputSurface.addEventListener("click", function () { + canvasContainer.focus({ preventScroll: true }); + }); + } function stopOnExit() { sharedSettingsSaver.saveNow(); diff --git a/Overlord-Server/test/e2e/remotedesktop-input.spec.ts b/Overlord-Server/test/e2e/remotedesktop-input.spec.ts new file mode 100644 index 0000000..e88ab3e --- /dev/null +++ b/Overlord-Server/test/e2e/remotedesktop-input.spec.ts @@ -0,0 +1,94 @@ +import { expect, test } from "@playwright/test"; +import { login } from "./helpers"; + +test("WebRTC video forwards mouse and keyboard input", async ({ page }) => { + await login(page); + await page.addInitScript(() => { + const sent: ArrayBuffer[] = []; + Object.defineProperty(window, "__rdSent", { value: sent, configurable: true }); + + class FakeWebSocket extends EventTarget { + static readonly CONNECTING = 0; + static readonly OPEN = 1; + static readonly CLOSING = 2; + static readonly CLOSED = 3; + readonly CONNECTING = 0; + readonly OPEN = 1; + readonly CLOSING = 2; + readonly CLOSED = 3; + readyState = FakeWebSocket.OPEN; + bufferedAmount = 0; + binaryType: BinaryType = "blob"; + + constructor(_url: string | URL) { + super(); + queueMicrotask(() => this.dispatchEvent(new Event("open"))); + } + + send(data: ArrayBuffer | ArrayBufferView | Blob | string) { + if (data instanceof ArrayBuffer) sent.push(data); + else if (ArrayBuffer.isView(data)) { + sent.push(data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength) as ArrayBuffer); + } + } + + close() { + this.readyState = FakeWebSocket.CLOSED; + this.dispatchEvent(new CloseEvent("close")); + } + } + + Object.defineProperty(window, "WebSocket", { value: FakeWebSocket, configurable: true }); + }); + + await page.goto("/remotedesktop?clientId=webrtc-input-test"); + await expect(page.locator("#canvasContainer")).toHaveAttribute("tabindex", "0"); + + await page.evaluate(() => { + const mouseCtrl = document.querySelector("#mouseCtrl")!; + const kbdCtrl = document.querySelector("#kbdCtrl")!; + mouseCtrl.checked = true; + mouseCtrl.dispatchEvent(new Event("change")); + kbdCtrl.checked = true; + kbdCtrl.dispatchEvent(new Event("change")); + const sent = (window as typeof window & { __rdSent: ArrayBuffer[] }).__rdSent; + sent.length = 0; + const canvas = document.querySelector("#frameCanvas")!; + const video = document.querySelector("#webrtcVideo")!; + canvas.style.display = "none"; + video.style.display = "block"; + Object.defineProperty(video, "videoWidth", { value: 1920, configurable: true }); + Object.defineProperty(video, "videoHeight", { value: 1080, configurable: true }); + video.getBoundingClientRect = () => ({ + x: 0, + y: 0, + left: 0, + top: 0, + right: 960, + bottom: 540, + width: 960, + height: 540, + toJSON() { return this; }, + }); + + video.dispatchEvent(new MouseEvent("mousemove", { clientX: 480, clientY: 270, bubbles: true })); + video.dispatchEvent(new MouseEvent("mousedown", { clientX: 480, clientY: 270, button: 0, bubbles: true })); + video.dispatchEvent(new MouseEvent("mouseup", { clientX: 480, clientY: 270, button: 0, bubbles: true })); + video.dispatchEvent(new WheelEvent("wheel", { clientX: 480, clientY: 270, deltaY: -80, bubbles: true })); + window.dispatchEvent(new KeyboardEvent("keydown", { key: "Enter", code: "Enter", bubbles: true })); + window.dispatchEvent(new KeyboardEvent("keyup", { key: "Enter", code: "Enter", bubbles: true })); + }); + + await expect.poll(async () => page.evaluate(async () => { + const sent = (window as typeof window & { __rdSent: ArrayBuffer[] }).__rdSent; + const { decodeMsgpack } = await import("/assets/msgpack-helpers.js"); + return sent.map((message) => decodeMsgpack(message)); + })).toEqual(expect.arrayContaining([ + expect.objectContaining({ type: "mouse_move", x: 960, y: 540 }), + expect.objectContaining({ type: "mouse_down", button: 0, x: 960, y: 540 }), + expect.objectContaining({ type: "mouse_up", button: 0, x: 960, y: 540 }), + expect.objectContaining({ type: "mouse_wheel", delta: 80, x: 960, y: 540 }), + expect.objectContaining({ type: "key_down", key: "Enter", code: "Enter" }), + expect.objectContaining({ type: "key_up", key: "Enter", code: "Enter" }), + ])); +});