vxaboveground/overlord
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

fix webrtc input

Browse Source
This commit is contained in:
KingKDot
2026-07-18 19:17:28 -05:00
parent f4cff71cf3
commit 6f6e28b8e7
3 changed files with 217 additions and 108 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Overlord-Server/public/assets/backstage.js
+31 -23
View File
@@ -1513,17 +1513,24 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
}, 1000);
}
function getCanvasPoint(e) {
let rect = canvas.getBoundingClientRect();
function getRenderPoint(e) {
const surface = e.currentTarget;
let rect = surface.getBoundingClientRect();
if (!rect.width || !rect.height) {
rect = canvasContainer?.getBoundingClientRect() || rect;
}
if (!rect.width || !rect.height || !canvas.width || !canvas.height) return null;
let x = ((e.clientX - rect.left) / rect.width) * canvas.width;
let y = ((e.clientY - rect.top) / rect.height) * canvas.height;
const targetW = surface === webrtcVideo
? webrtcVideo.videoWidth
: canvas.width;
const targetH = surface === webrtcVideo
? webrtcVideo.videoHeight
: canvas.height;
if (!rect.width || !rect.height || !targetW || !targetH) return null;
let x = ((e.clientX - rect.left) / rect.width) * targetW;
let y = ((e.clientY - rect.top) / rect.height) * targetH;
if (!Number.isFinite(x) || !Number.isFinite(y)) return null;
x = Math.max(0, Math.min(canvas.width - 1, Math.floor(x)));
y = Math.max(0, Math.min(canvas.height - 1, Math.floor(y)));
x = Math.max(0, Math.min(targetW - 1, Math.floor(x)));
y = Math.max(0, Math.min(targetH - 1, Math.floor(y)));
return { x, y };
}
@@ -1543,19 +1550,21 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
}
}
canvas.addEventListener("mousemove", function (e) {
for (const inputSurface of [canvas, webrtcVideo]) {
if (!inputSurface) continue;
inputSurface.addEventListener("mousemove", function (e) {
if (!mouseCtrl.checked) return;
const pt = getCanvasPoint(e);
const pt = getRenderPoint(e);
if (!pt) return;
pendingMove = pt;
if (!moveTimer) {
flushMouseMove();
}
});
canvas.addEventListener("mousedown", function (e) {
inputSurface.addEventListener("mousedown", function (e) {
canvasContainer.focus({ preventScroll: true });
if (!mouseCtrl.checked) return;
canvas.focus({ preventScroll: true });
const pt = getCanvasPoint(e);
const pt = getRenderPoint(e);
if (pt) {
pendingMove = pt;
if (ws.bufferedAmount <= inputBackpressureBytes) {
@@ -1565,9 +1574,9 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
sendCmd("backstage_mouse_down", { button: e.button, ...(pt || {}) });
e.preventDefault();
});
canvas.addEventListener("mouseup", function (e) {
inputSurface.addEventListener("mouseup", function (e) {
if (!mouseCtrl.checked) return;
const pt = getCanvasPoint(e);
const pt = getRenderPoint(e);
if (pt) {
pendingMove = pt;
if (ws.bufferedAmount <= inputBackpressureBytes) {
@@ -1577,25 +1586,24 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
sendCmd("backstage_mouse_up", { button: e.button, ...(pt || {}) });
e.preventDefault();
});
canvas.addEventListener("contextmenu", function (e) {
inputSurface.addEventListener("contextmenu", function (e) {
e.preventDefault();
});
canvas.addEventListener("wheel", function (e) {
inputSurface.addEventListener("wheel", function (e) {
if (!mouseCtrl.checked) return;
const pt = getCanvasPoint(e);
const pt = getRenderPoint(e);
if (!pt) return;
const delta = Math.max(-120, Math.min(120, Math.round(-e.deltaY)));
sendCmd("backstage_mouse_wheel", { delta, x: pt.x, y: pt.y });
e.preventDefault();
}, { passive: false });
canvas.setAttribute("tabindex", "0");
canvas.addEventListener("click", function () {
canvas.focus({ preventScroll: true });
inputSurface.addEventListener("click", function () {
canvasContainer.focus({ preventScroll: true });
});
}
canvasContainer.setAttribute("tabindex", "0");
const kbdCapture = createKeyboardCapture({
container: canvas,
container: canvasContainer,
sendKeyDown: (e) => sendCmd("backstage_key_down", { key: e.key, code: e.code }),
sendKeyUp: (e) => sendCmd("backstage_key_up", { key: e.key, code: e.code }),
});
Overlord-Server/public/assets/remotedesktop.js
+26 -19
View File
@@ -1844,8 +1844,9 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
pushInputToggles();
sharedSettingsSaver.scheduleSave();
});
canvasContainer.setAttribute("tabindex", "0");
const kbdCapture = createKeyboardCapture({
container: canvas,
container: canvasContainer,
sendKeyDown: (e) => sendCmd("key_down", { key: e.key, code: e.code }),
sendKeyUp: (e) => sendCmd("key_up", { key: e.key, code: e.code }),
onTextInput: (e) => sendCmd("text_input", { text: e.key }),
@@ -2644,13 +2645,18 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
}, 1000);
}
function getCanvasPoint(e) {
let rect = canvas.getBoundingClientRect();
function getRenderPoint(e) {
const surface = e.currentTarget;
let rect = surface.getBoundingClientRect();
if (!rect.width || !rect.height) {
rect = canvasContainer?.getBoundingClientRect() || rect;
}
const targetW = canvas.width || frameWidth;
const targetH = canvas.height || frameHeight;
const targetW = surface === webrtcVideo
? (webrtcVideo.videoWidth || frameWidth)
: (canvas.width || frameWidth);
const targetH = surface === webrtcVideo
? (webrtcVideo.videoHeight || frameHeight)
: (canvas.height || frameHeight);
if (!rect.width || !rect.height || !targetW || !targetH) return null;
let x = ((e.clientX - rect.left) / rect.width) * targetW;
let y = ((e.clientY - rect.top) / rect.height) * targetH;
@@ -2690,19 +2696,21 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
}
}
canvas.addEventListener("mousemove", function (e) {
for (const inputSurface of [canvas, webrtcVideo]) {
if (!inputSurface) continue;
inputSurface.addEventListener("mousemove", function (e) {
if (!mouseCtrl.checked) return;
const pt = getCanvasPoint(e);
const pt = getRenderPoint(e);
if (!pt) return;
pendingMove = pt;
if (!moveTimer) {
flushMouseMove();
}
});
canvas.addEventListener("mousedown", function (e) {
inputSurface.addEventListener("mousedown", function (e) {
canvasContainer.focus({ preventScroll: true });
if (!mouseCtrl.checked) return;
canvas.focus({ preventScroll: true });
const pt = getCanvasPoint(e);
const pt = getRenderPoint(e);
if (pt) {
pendingMove = pt;
smoothPoint = { x: pt.x, y: pt.y };
@@ -2711,9 +2719,9 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
sendCmd("mouse_down", { button: e.button, ...(pt || {}) });
e.preventDefault();
});
canvas.addEventListener("mouseup", function (e) {
inputSurface.addEventListener("mouseup", function (e) {
if (!mouseCtrl.checked) return;
const pt = getCanvasPoint(e);
const pt = getRenderPoint(e);
if (pt) {
pendingMove = pt;
smoothPoint = { x: pt.x, y: pt.y };
@@ -2722,22 +2730,21 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
sendCmd("mouse_up", { button: e.button, ...(pt || {}) });
e.preventDefault();
});
canvas.addEventListener("contextmenu", function (e) {
inputSurface.addEventListener("contextmenu", function (e) {
e.preventDefault();
});
canvas.addEventListener("wheel", function (e) {
inputSurface.addEventListener("wheel", function (e) {
if (!mouseCtrl.checked) return;
const pt = getCanvasPoint(e);
const pt = getRenderPoint(e);
if (!pt) return;
const delta = Math.max(-120, Math.min(120, Math.round(-e.deltaY)));
sendCmd("mouse_wheel", { delta, x: pt.x, y: pt.y });
e.preventDefault();
}, { passive: false });
canvas.setAttribute("tabindex", "0");
canvas.addEventListener("click", function () {
canvas.focus({ preventScroll: true });
inputSurface.addEventListener("click", function () {
canvasContainer.focus({ preventScroll: true });
});
}
function stopOnExit() {
sharedSettingsSaver.saveNow();
Overlord-Server/test/e2e/remotedesktop-input.spec.ts
+94
View File
@@ -0,0 +1,94 @@
import { expect, test } from "@playwright/test";
import { login } from "./helpers";
test("WebRTC video forwards mouse and keyboard input", async ({ page }) => {
await login(page);
await page.addInitScript(() => {
const sent: ArrayBuffer[] = [];
Object.defineProperty(window, "__rdSent", { value: sent, configurable: true });
class FakeWebSocket extends EventTarget {
static readonly CONNECTING = 0;
static readonly OPEN = 1;
static readonly CLOSING = 2;
static readonly CLOSED = 3;
readonly CONNECTING = 0;
readonly OPEN = 1;
readonly CLOSING = 2;
readonly CLOSED = 3;
readyState = FakeWebSocket.OPEN;
bufferedAmount = 0;
binaryType: BinaryType = "blob";
constructor(_url: string | URL) {
super();
queueMicrotask(() => this.dispatchEvent(new Event("open")));
}
send(data: ArrayBuffer | ArrayBufferView | Blob | string) {
if (data instanceof ArrayBuffer) sent.push(data);
else if (ArrayBuffer.isView(data)) {
sent.push(data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength) as ArrayBuffer);
}
}
close() {
this.readyState = FakeWebSocket.CLOSED;
this.dispatchEvent(new CloseEvent("close"));
}
}
Object.defineProperty(window, "WebSocket", { value: FakeWebSocket, configurable: true });
});
await page.goto("/remotedesktop?clientId=webrtc-input-test");
await expect(page.locator("#canvasContainer")).toHaveAttribute("tabindex", "0");
await page.evaluate(() => {
const mouseCtrl = document.querySelector<HTMLInputElement>("#mouseCtrl")!;
const kbdCtrl = document.querySelector<HTMLInputElement>("#kbdCtrl")!;
mouseCtrl.checked = true;
mouseCtrl.dispatchEvent(new Event("change"));
kbdCtrl.checked = true;
kbdCtrl.dispatchEvent(new Event("change"));
const sent = (window as typeof window & { __rdSent: ArrayBuffer[] }).__rdSent;
sent.length = 0;
const canvas = document.querySelector<HTMLCanvasElement>("#frameCanvas")!;
const video = document.querySelector<HTMLVideoElement>("#webrtcVideo")!;
canvas.style.display = "none";
video.style.display = "block";
Object.defineProperty(video, "videoWidth", { value: 1920, configurable: true });
Object.defineProperty(video, "videoHeight", { value: 1080, configurable: true });
video.getBoundingClientRect = () => ({
x: 0,
y: 0,
left: 0,
top: 0,
right: 960,
bottom: 540,
width: 960,
height: 540,
toJSON() { return this; },
});
video.dispatchEvent(new MouseEvent("mousemove", { clientX: 480, clientY: 270, bubbles: true }));
video.dispatchEvent(new MouseEvent("mousedown", { clientX: 480, clientY: 270, button: 0, bubbles: true }));
video.dispatchEvent(new MouseEvent("mouseup", { clientX: 480, clientY: 270, button: 0, bubbles: true }));
video.dispatchEvent(new WheelEvent("wheel", { clientX: 480, clientY: 270, deltaY: -80, bubbles: true }));
window.dispatchEvent(new KeyboardEvent("keydown", { key: "Enter", code: "Enter", bubbles: true }));
window.dispatchEvent(new KeyboardEvent("keyup", { key: "Enter", code: "Enter", bubbles: true }));
});
await expect.poll(async () => page.evaluate(async () => {
const sent = (window as typeof window & { __rdSent: ArrayBuffer[] }).__rdSent;
const { decodeMsgpack } = await import("/assets/msgpack-helpers.js");
return sent.map((message) => decodeMsgpack(message));
})).toEqual(expect.arrayContaining([
expect.objectContaining({ type: "mouse_move", x: 960, y: 540 }),
expect.objectContaining({ type: "mouse_down", button: 0, x: 960, y: 540 }),
expect.objectContaining({ type: "mouse_up", button: 0, x: 960, y: 540 }),
expect.objectContaining({ type: "mouse_wheel", delta: 80, x: 960, y: 540 }),
expect.objectContaining({ type: "key_down", key: "Enter", code: "Enter" }),
expect.objectContaining({ type: "key_up", key: "Enter", code: "Enter" }),
]));
});