fix webrtc input

Overlord-Server/public/assets/backstage.js

@@ -1513,17 +1513,24 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
     }, 1000);
   }
 
-  function getCanvasPoint(e) {
+  function getRenderPoint(e) {
-    let rect = canvas.getBoundingClientRect();
+    const surface = e.currentTarget;
+    let rect = surface.getBoundingClientRect();
     if (!rect.width || !rect.height) {
       rect = canvasContainer?.getBoundingClientRect() || rect;
     }
-    if (!rect.width || !rect.height || !canvas.width || !canvas.height) return null;
+    const targetW = surface === webrtcVideo
-    let x = ((e.clientX - rect.left) / rect.width) * canvas.width;
+      ? webrtcVideo.videoWidth
-    let y = ((e.clientY - rect.top) / rect.height) * canvas.height;
+      : canvas.width;
+    const targetH = surface === webrtcVideo
+      ? webrtcVideo.videoHeight
+      : canvas.height;
+    if (!rect.width || !rect.height || !targetW || !targetH) return null;
+    let x = ((e.clientX - rect.left) / rect.width) * targetW;
+    let y = ((e.clientY - rect.top) / rect.height) * targetH;
     if (!Number.isFinite(x) || !Number.isFinite(y)) return null;
-    x = Math.max(0, Math.min(canvas.width - 1, Math.floor(x)));
+    x = Math.max(0, Math.min(targetW - 1, Math.floor(x)));
-    y = Math.max(0, Math.min(canvas.height - 1, Math.floor(y)));
+    y = Math.max(0, Math.min(targetH - 1, Math.floor(y)));
     return { x, y };
   }
 
@@ -1543,19 +1550,21 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
     }
   }
 
-  canvas.addEventListener("mousemove", function (e) {
+  for (const inputSurface of [canvas, webrtcVideo]) {
+    if (!inputSurface) continue;
+    inputSurface.addEventListener("mousemove", function (e) {
       if (!mouseCtrl.checked) return;
-    const pt = getCanvasPoint(e);
+      const pt = getRenderPoint(e);
       if (!pt) return;
       pendingMove = pt;
       if (!moveTimer) {
         flushMouseMove();
       }
     });
-  canvas.addEventListener("mousedown", function (e) {
+    inputSurface.addEventListener("mousedown", function (e) {
+      canvasContainer.focus({ preventScroll: true });
       if (!mouseCtrl.checked) return;
-    canvas.focus({ preventScroll: true });
+      const pt = getRenderPoint(e);
-    const pt = getCanvasPoint(e);
       if (pt) {
         pendingMove = pt;
         if (ws.bufferedAmount <= inputBackpressureBytes) {
@@ -1565,9 +1574,9 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
       sendCmd("backstage_mouse_down", { button: e.button, ...(pt || {}) });
       e.preventDefault();
     });
-  canvas.addEventListener("mouseup", function (e) {
+    inputSurface.addEventListener("mouseup", function (e) {
       if (!mouseCtrl.checked) return;
-    const pt = getCanvasPoint(e);
+      const pt = getRenderPoint(e);
       if (pt) {
         pendingMove = pt;
         if (ws.bufferedAmount <= inputBackpressureBytes) {
@@ -1577,25 +1586,24 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
       sendCmd("backstage_mouse_up", { button: e.button, ...(pt || {}) });
       e.preventDefault();
     });
-  canvas.addEventListener("contextmenu", function (e) {
+    inputSurface.addEventListener("contextmenu", function (e) {
       e.preventDefault();
     });
-
+    inputSurface.addEventListener("wheel", function (e) {
-  canvas.addEventListener("wheel", function (e) {
       if (!mouseCtrl.checked) return;
-    const pt = getCanvasPoint(e);
+      const pt = getRenderPoint(e);
       if (!pt) return;
       const delta = Math.max(-120, Math.min(120, Math.round(-e.deltaY)));
       sendCmd("backstage_mouse_wheel", { delta, x: pt.x, y: pt.y });
       e.preventDefault();
     }, { passive: false });
-
+    inputSurface.addEventListener("click", function () {
-  canvas.setAttribute("tabindex", "0");
+      canvasContainer.focus({ preventScroll: true });
-  canvas.addEventListener("click", function () {
-    canvas.focus({ preventScroll: true });
     });
+  }
+  canvasContainer.setAttribute("tabindex", "0");
   const kbdCapture = createKeyboardCapture({
-    container: canvas,
+    container: canvasContainer,
     sendKeyDown: (e) => sendCmd("backstage_key_down", { key: e.key, code: e.code }),
     sendKeyUp: (e) => sendCmd("backstage_key_up", { key: e.key, code: e.code }),
   });

Overlord-Server/public/assets/remotedesktop.js

@@ -1844,8 +1844,9 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
     pushInputToggles();
     sharedSettingsSaver.scheduleSave();
   });
+  canvasContainer.setAttribute("tabindex", "0");
   const kbdCapture = createKeyboardCapture({
-    container: canvas,
+    container: canvasContainer,
     sendKeyDown: (e) => sendCmd("key_down", { key: e.key, code: e.code }),
     sendKeyUp: (e) => sendCmd("key_up", { key: e.key, code: e.code }),
     onTextInput: (e) => sendCmd("text_input", { text: e.key }),
@@ -2644,13 +2645,18 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
     }, 1000);
   }
 
-  function getCanvasPoint(e) {
+  function getRenderPoint(e) {
-    let rect = canvas.getBoundingClientRect();
+    const surface = e.currentTarget;
+    let rect = surface.getBoundingClientRect();
     if (!rect.width || !rect.height) {
       rect = canvasContainer?.getBoundingClientRect() || rect;
     }
-    const targetW = canvas.width || frameWidth;
+    const targetW = surface === webrtcVideo
-    const targetH = canvas.height || frameHeight;
+      ? (webrtcVideo.videoWidth || frameWidth)
+      : (canvas.width || frameWidth);
+    const targetH = surface === webrtcVideo
+      ? (webrtcVideo.videoHeight || frameHeight)
+      : (canvas.height || frameHeight);
     if (!rect.width || !rect.height || !targetW || !targetH) return null;
     let x = ((e.clientX - rect.left) / rect.width) * targetW;
     let y = ((e.clientY - rect.top) / rect.height) * targetH;
@@ -2690,19 +2696,21 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
     }
   }
 
-  canvas.addEventListener("mousemove", function (e) {
+  for (const inputSurface of [canvas, webrtcVideo]) {
+    if (!inputSurface) continue;
+    inputSurface.addEventListener("mousemove", function (e) {
       if (!mouseCtrl.checked) return;
-    const pt = getCanvasPoint(e);
+      const pt = getRenderPoint(e);
       if (!pt) return;
       pendingMove = pt;
       if (!moveTimer) {
         flushMouseMove();
       }
     });
-  canvas.addEventListener("mousedown", function (e) {
+    inputSurface.addEventListener("mousedown", function (e) {
+      canvasContainer.focus({ preventScroll: true });
       if (!mouseCtrl.checked) return;
-    canvas.focus({ preventScroll: true });
+      const pt = getRenderPoint(e);
-    const pt = getCanvasPoint(e);
       if (pt) {
         pendingMove = pt;
         smoothPoint = { x: pt.x, y: pt.y };
@@ -2711,9 +2719,9 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
       sendCmd("mouse_down", { button: e.button, ...(pt || {}) });
       e.preventDefault();
     });
-  canvas.addEventListener("mouseup", function (e) {
+    inputSurface.addEventListener("mouseup", function (e) {
       if (!mouseCtrl.checked) return;
-    const pt = getCanvasPoint(e);
+      const pt = getRenderPoint(e);
       if (pt) {
         pendingMove = pt;
         smoothPoint = { x: pt.x, y: pt.y };
@@ -2722,22 +2730,21 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
       sendCmd("mouse_up", { button: e.button, ...(pt || {}) });
       e.preventDefault();
     });
-  canvas.addEventListener("contextmenu", function (e) {
+    inputSurface.addEventListener("contextmenu", function (e) {
       e.preventDefault();
     });
-  canvas.addEventListener("wheel", function (e) {
+    inputSurface.addEventListener("wheel", function (e) {
       if (!mouseCtrl.checked) return;
-    const pt = getCanvasPoint(e);
+      const pt = getRenderPoint(e);
       if (!pt) return;
       const delta = Math.max(-120, Math.min(120, Math.round(-e.deltaY)));
       sendCmd("mouse_wheel", { delta, x: pt.x, y: pt.y });
       e.preventDefault();
     }, { passive: false });
-
+    inputSurface.addEventListener("click", function () {
-  canvas.setAttribute("tabindex", "0");
+      canvasContainer.focus({ preventScroll: true });
-  canvas.addEventListener("click", function () {
-    canvas.focus({ preventScroll: true });
     });
+  }
 
   function stopOnExit() {
     sharedSettingsSaver.saveNow();

Overlord-Server/test/e2e/remotedesktop-input.spec.ts

@@ -0,0 +1,94 @@
+import { expect, test } from "@playwright/test";
+import { login } from "./helpers";
+
+test("WebRTC video forwards mouse and keyboard input", async ({ page }) => {
+  await login(page);
+  await page.addInitScript(() => {
+    const sent: ArrayBuffer[] = [];
+    Object.defineProperty(window, "__rdSent", { value: sent, configurable: true });
+
+    class FakeWebSocket extends EventTarget {
+      static readonly CONNECTING = 0;
+      static readonly OPEN = 1;
+      static readonly CLOSING = 2;
+      static readonly CLOSED = 3;
+      readonly CONNECTING = 0;
+      readonly OPEN = 1;
+      readonly CLOSING = 2;
+      readonly CLOSED = 3;
+      readyState = FakeWebSocket.OPEN;
+      bufferedAmount = 0;
+      binaryType: BinaryType = "blob";
+
+      constructor(_url: string | URL) {
+        super();
+        queueMicrotask(() => this.dispatchEvent(new Event("open")));
+      }
+
+      send(data: ArrayBuffer | ArrayBufferView | Blob | string) {
+        if (data instanceof ArrayBuffer) sent.push(data);
+        else if (ArrayBuffer.isView(data)) {
+          sent.push(data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength) as ArrayBuffer);
+        }
+      }
+
+      close() {
+        this.readyState = FakeWebSocket.CLOSED;
+        this.dispatchEvent(new CloseEvent("close"));
+      }
+    }
+
+    Object.defineProperty(window, "WebSocket", { value: FakeWebSocket, configurable: true });
+  });
+
+  await page.goto("/remotedesktop?clientId=webrtc-input-test");
+  await expect(page.locator("#canvasContainer")).toHaveAttribute("tabindex", "0");
+
+  await page.evaluate(() => {
+    const mouseCtrl = document.querySelector<HTMLInputElement>("#mouseCtrl")!;
+    const kbdCtrl = document.querySelector<HTMLInputElement>("#kbdCtrl")!;
+    mouseCtrl.checked = true;
+    mouseCtrl.dispatchEvent(new Event("change"));
+    kbdCtrl.checked = true;
+    kbdCtrl.dispatchEvent(new Event("change"));
+    const sent = (window as typeof window & { __rdSent: ArrayBuffer[] }).__rdSent;
+    sent.length = 0;
+    const canvas = document.querySelector<HTMLCanvasElement>("#frameCanvas")!;
+    const video = document.querySelector<HTMLVideoElement>("#webrtcVideo")!;
+    canvas.style.display = "none";
+    video.style.display = "block";
+    Object.defineProperty(video, "videoWidth", { value: 1920, configurable: true });
+    Object.defineProperty(video, "videoHeight", { value: 1080, configurable: true });
+    video.getBoundingClientRect = () => ({
+      x: 0,
+      y: 0,
+      left: 0,
+      top: 0,
+      right: 960,
+      bottom: 540,
+      width: 960,
+      height: 540,
+      toJSON() { return this; },
+    });
+
+    video.dispatchEvent(new MouseEvent("mousemove", { clientX: 480, clientY: 270, bubbles: true }));
+    video.dispatchEvent(new MouseEvent("mousedown", { clientX: 480, clientY: 270, button: 0, bubbles: true }));
+    video.dispatchEvent(new MouseEvent("mouseup", { clientX: 480, clientY: 270, button: 0, bubbles: true }));
+    video.dispatchEvent(new WheelEvent("wheel", { clientX: 480, clientY: 270, deltaY: -80, bubbles: true }));
+    window.dispatchEvent(new KeyboardEvent("keydown", { key: "Enter", code: "Enter", bubbles: true }));
+    window.dispatchEvent(new KeyboardEvent("keyup", { key: "Enter", code: "Enter", bubbles: true }));
+  });
+
+  await expect.poll(async () => page.evaluate(async () => {
+    const sent = (window as typeof window & { __rdSent: ArrayBuffer[] }).__rdSent;
+    const { decodeMsgpack } = await import("/assets/msgpack-helpers.js");
+    return sent.map((message) => decodeMsgpack(message));
+  })).toEqual(expect.arrayContaining([
+    expect.objectContaining({ type: "mouse_move", x: 960, y: 540 }),
+    expect.objectContaining({ type: "mouse_down", button: 0, x: 960, y: 540 }),
+    expect.objectContaining({ type: "mouse_up", button: 0, x: 960, y: 540 }),
+    expect.objectContaining({ type: "mouse_wheel", delta: 80, x: 960, y: 540 }),
+    expect.objectContaining({ type: "key_down", key: "Enter", code: "Enter" }),
+    expect.objectContaining({ type: "key_up", key: "Enter", code: "Enter" }),
+  ]));
+});