From c3bcc2cdcc2dde1f06e8a0c9fa7444514c5f69c7 Mon Sep 17 00:00:00 2001 From: KingKDot Date: Wed, 5 Aug 2026 11:02:38 -0500 Subject: [PATCH] fix some remote desktop stuff --- .../public/assets/remotedesktop.js | 66 +++++++++++++------ Overlord-Server/public/remotedesktop.html | 4 +- .../src/server/ws-console-rd-backstage.ts | 18 ++++- .../test/e2e/remotedesktop-input.e2e.ts | 29 ++++++++ 4 files changed, 91 insertions(+), 26 deletions(-) diff --git a/Overlord-Server/public/assets/remotedesktop.js b/Overlord-Server/public/assets/remotedesktop.js index 9c4b322..09a3dda 100644 --- a/Overlord-Server/public/assets/remotedesktop.js +++ b/Overlord-Server/public/assets/remotedesktop.js @@ -152,10 +152,11 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./generated/s let frameWidth = 0; let frameHeight = 0; let latencyAvg = null; - let smoothingPct = 20; + let smoothingPct = 0; let smoothPoint = null; let pendingMove = null; - let moveFrame = 0; + let moveTimer = 0; + let lastMoveSentAt = 0; let frameDecodeBusy = false; let pendingFrame = null; let videoDecoder = null; @@ -185,6 +186,11 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./generated/s const H264_MAX_RECOVERY_ATTEMPTS = 1; const H264_DECODE_WARN_THROTTLE_MS = 2000; const inputBackpressureBytes = 256 * 1024; + const inputMoveIntervalMs = 8; + const highFrequencyInputTypes = new Set([ + "mouse_move", "mouse_down", "mouse_up", "mouse_wheel", + "key_down", "key_up", "text_input", + ]); const VIEWER_FRAME_GAP_KEYFRAME_THROTTLE_MS = 1000; let lastViewerKeyframeRequestAt = 0; const diagnostics = { @@ -540,7 +546,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./generated/s privacy: !!privacyCtrl?.checked, audio: !!audioCtrl?.checked, audioTransport: getAudioTransport(), - smoothing: Number(smoothingSlider?.value || 20), + smoothing: Number(smoothingSlider?.value || 0), recordMode: recordMode?.value || "normal", recordFps: recordFps?.value || "", }; @@ -1696,7 +1702,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./generated/s return; } const msg = { type, ...payload }; - if (type !== "desktop_decode_pressure") { + if (type !== "desktop_decode_pressure" && !highFrequencyInputTypes.has(type)) { rdDebug("send", { msg, readyState: ws.readyState, @@ -2443,7 +2449,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./generated/s remoteCursor.hidden = true; return; } - const surfaceRect = surface.getBoundingClientRect(); + const surfaceRect = getContainedSurfaceRect(surface, sourceWidth, sourceHeight); const containerRect = renderSurface.getBoundingClientRect(); const scaleX = surfaceRect.width / sourceWidth; const scaleY = surfaceRect.height / sourceHeight; @@ -3390,6 +3396,9 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./generated/s ? (webrtcVideo.videoHeight || frameHeight) : (canvas.height || frameHeight); if (!rect.width || !rect.height || !targetW || !targetH) return null; + rect = getContainedSurfaceRect(surface, targetW, targetH, rect); + if (e.clientX < rect.left || e.clientX > rect.right || + e.clientY < rect.top || e.clientY > rect.bottom) return null; let x = ((e.clientX - rect.left) / rect.width) * targetW; let y = ((e.clientY - rect.top) / rect.height) * targetH; if (!Number.isFinite(x) || !Number.isFinite(y)) return null; @@ -3398,14 +3407,32 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./generated/s return { x, y }; } + function getContainedSurfaceRect(surface, sourceWidth, sourceHeight, elementRect = surface.getBoundingClientRect()) { + if (!elementRect.width || !elementRect.height || !sourceWidth || !sourceHeight) return elementRect; + const fit = getComputedStyle(surface).objectFit; + if (fit !== "contain" && fit !== "scale-down") return elementRect; + let scale = Math.min(elementRect.width / sourceWidth, elementRect.height / sourceHeight); + if (fit === "scale-down") scale = Math.min(1, scale); + const width = sourceWidth * scale; + const height = sourceHeight * scale; + const left = elementRect.left + (elementRect.width - width) / 2; + const top = elementRect.top + (elementRect.height - height) / 2; + return { + x: left, y: top, left, top, right: left + width, bottom: top + height, + width, height, toJSON() { return this; }, + }; + } + function scheduleMouseMove() { - if (!moveFrame) { - moveFrame = requestAnimationFrame(flushMouseMove); - } + if (moveTimer) return; + const wait = Math.max(0, inputMoveIntervalMs - (performance.now() - lastMoveSentAt)); + if (wait === 0) flushMouseMove(); + else moveTimer = window.setTimeout(flushMouseMove, wait); } function flushMouseMove() { - moveFrame = 0; + if (moveTimer) window.clearTimeout(moveTimer); + moveTimer = 0; if (!isInputActive() || !pendingMove || !mouseCtrl.checked) { smoothPoint = null; return; @@ -3424,6 +3451,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./generated/s if (ws && ws.bufferedAmount <= inputBackpressureBytes) { sendCmd("mouse_move", sendPoint); + lastMoveSentAt = performance.now(); } if (Math.abs(pendingMove.x - smoothPoint.x) > 0.5 || @@ -3447,23 +3475,19 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./generated/s canvasContainer.focus({ preventScroll: true }); if (!isInputActive() || !mouseCtrl.checked) return; const pt = getRenderPoint(e); - if (pt) { - pendingMove = pt; - smoothPoint = { x: pt.x, y: pt.y }; - sendCmd("mouse_move", pt); - } - sendCmd("mouse_down", { button: e.button, ...(pt || {}) }); + if (!pt) return; + pendingMove = pt; + smoothPoint = { x: pt.x, y: pt.y }; + sendCmd("mouse_down", { button: e.button, ...pt }); e.preventDefault(); }); inputSurface.addEventListener("mouseup", function (e) { if (!isInputActive() || !mouseCtrl.checked) return; const pt = getRenderPoint(e); - if (pt) { - pendingMove = pt; - smoothPoint = { x: pt.x, y: pt.y }; - sendCmd("mouse_move", pt); - } - sendCmd("mouse_up", { button: e.button, ...(pt || {}) }); + if (!pt) return; + pendingMove = pt; + smoothPoint = { x: pt.x, y: pt.y }; + sendCmd("mouse_up", { button: e.button, ...pt }); e.preventDefault(); }); inputSurface.addEventListener("contextmenu", function (e) { diff --git a/Overlord-Server/public/remotedesktop.html b/Overlord-Server/public/remotedesktop.html index d27cd76..c05b761 100644 --- a/Overlord-Server/public/remotedesktop.html +++ b/Overlord-Server/public/remotedesktop.html @@ -272,8 +272,8 @@
- - 20% + + 0%
diff --git a/Overlord-Server/src/server/ws-console-rd-backstage.ts b/Overlord-Server/src/server/ws-console-rd-backstage.ts index 92673b2..374f73d 100644 --- a/Overlord-Server/src/server/ws-console-rd-backstage.ts +++ b/Overlord-Server/src/server/ws-console-rd-backstage.ts @@ -237,7 +237,11 @@ function sendDesktopCommandWithId(target: ClientInfo | undefined, commandType: s return false; } try { - logger.debug(`[rd-debug] send command command=${commandType} client=${target.id} id=${commandId} payload=${JSON.stringify(payload || {})}`); + const isInteractiveInput = commandType.startsWith("desktop_mouse_") || + commandType.startsWith("desktop_key_") || commandType === "desktop_text"; + if (!isInteractiveInput) { + logger.debug(`[rd-debug] send command command=${commandType} client=${target.id} id=${commandId} payload=${JSON.stringify(payload || {})}`); + } target.ws.send(encodeMessage({ type: "command", commandType: commandType as any, payload, id: commandId })); metrics.recordCommand(commandType); return true; @@ -648,14 +652,22 @@ export function handleRemoteDesktopViewerMessage(ws: ServerWebSocket if (!state.isStreaming) break; const commandId = uuidv4(); recordRdInput(commandId, clientId, "mouse_down"); - sendDesktopCommandWithId(target, "desktop_mouse_down", { button: Number(payload.button) || 0 }, commandId); + const hasPoint = Number.isFinite(Number((payload as any).x)) && Number.isFinite(Number((payload as any).y)); + sendDesktopCommandWithId(target, "desktop_mouse_down", { + button: Number(payload.button) || 0, + ...(hasPoint ? { x: Number((payload as any).x), y: Number((payload as any).y) } : {}), + }, commandId); break; } case "mouse_up": { if (!state.isStreaming) break; const commandId = uuidv4(); recordRdInput(commandId, clientId, "mouse_up"); - sendDesktopCommandWithId(target, "desktop_mouse_up", { button: Number(payload.button) || 0 }, commandId); + const hasPoint = Number.isFinite(Number((payload as any).x)) && Number.isFinite(Number((payload as any).y)); + sendDesktopCommandWithId(target, "desktop_mouse_up", { + button: Number(payload.button) || 0, + ...(hasPoint ? { x: Number((payload as any).x), y: Number((payload as any).y) } : {}), + }, commandId); break; } case "mouse_wheel": { diff --git a/Overlord-Server/test/e2e/remotedesktop-input.e2e.ts b/Overlord-Server/test/e2e/remotedesktop-input.e2e.ts index 3d69fd9..1ccaae6 100644 --- a/Overlord-Server/test/e2e/remotedesktop-input.e2e.ts +++ b/Overlord-Server/test/e2e/remotedesktop-input.e2e.ts @@ -208,6 +208,35 @@ test("WebRTC input stays inactive until start and then forwards mouse and keyboa expect.objectContaining({ type: "key_up", key: "Enter", code: "Enter" }), ])); + await page.evaluate(() => { + const sent = (window as typeof window & { __rdSent: ArrayBuffer[] }).__rdSent; + sent.length = 0; + const video = document.querySelector("#webrtcVideo")!; + video.style.objectFit = "contain"; + video.getBoundingClientRect = () => ({ + x: 0, y: 0, left: 0, top: 0, right: 1000, bottom: 1000, + width: 1000, height: 1000, toJSON() { return this; }, + }); + video.dispatchEvent(new MouseEvent("mousedown", { clientX: 500, clientY: 100, button: 0, bubbles: true })); + }); + await page.waitForTimeout(20); + await expect.poll(async () => page.evaluate(async () => { + const sent = (window as typeof window & { __rdSent: ArrayBuffer[] }).__rdSent; + const { decodeMsgpack } = await import("/assets/msgpack-helpers.js"); + return sent.map((message) => decodeMsgpack(message)) + .filter((message) => /^(mouse_|key_|text_input)/.test(message.type)); + })).toEqual([]); + + await page.evaluate(() => { + const video = document.querySelector("#webrtcVideo")!; + video.dispatchEvent(new MouseEvent("mousedown", { clientX: 250, clientY: 360, button: 0, bubbles: true })); + }); + await expect.poll(async () => page.evaluate(async () => { + const sent = (window as typeof window & { __rdSent: ArrayBuffer[] }).__rdSent; + const { decodeMsgpack } = await import("/assets/msgpack-helpers.js"); + return sent.map((message) => decodeMsgpack(message)); + })).toContainEqual(expect.objectContaining({ type: "mouse_down", button: 0, x: 480, y: 271 })); + await page.evaluate(() => { const socket = (window as typeof window & { __rdSocket: EventTarget }).__rdSocket; const framePacket = Uint8Array.from([0x46, 0x52, 0x4d, 0, 0, 0, 0, 0]).buffer;