vxaboveground/overlord
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

fix some remote desktop stuff

Browse Source
This commit is contained in:
KingKDot
2026-08-05 11:02:38 -05:00
parent 1d431103bb
commit c3bcc2cdcc
4 changed files with 91 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Overlord-Server/public/assets/remotedesktop.js
+45 -21
View File
@@ -152,10 +152,11 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./generated/s
let frameWidth = 0;
let frameHeight = 0;
let latencyAvg = null;
let smoothingPct = 20;
let smoothingPct = 0;
let smoothPoint = null;
let pendingMove = null;
let moveFrame = 0;
let moveTimer = 0;
let lastMoveSentAt = 0;
let frameDecodeBusy = false;
let pendingFrame = null;
let videoDecoder = null;
@@ -185,6 +186,11 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./generated/s
const H264_MAX_RECOVERY_ATTEMPTS = 1;
const H264_DECODE_WARN_THROTTLE_MS = 2000;
const inputBackpressureBytes = 256 * 1024;
const inputMoveIntervalMs = 8;
const highFrequencyInputTypes = new Set([
"mouse_move", "mouse_down", "mouse_up", "mouse_wheel",
"key_down", "key_up", "text_input",
]);
const VIEWER_FRAME_GAP_KEYFRAME_THROTTLE_MS = 1000;
let lastViewerKeyframeRequestAt = 0;
const diagnostics = {
@@ -540,7 +546,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./generated/s
privacy: !!privacyCtrl?.checked,
audio: !!audioCtrl?.checked,
audioTransport: getAudioTransport(),
smoothing: Number(smoothingSlider?.value || 20),
smoothing: Number(smoothingSlider?.value || 0),
recordMode: recordMode?.value || "normal",
recordFps: recordFps?.value || "",
};
@@ -1696,7 +1702,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./generated/s
return;
}
const msg = { type, ...payload };
if (type !== "desktop_decode_pressure") {
if (type !== "desktop_decode_pressure" && !highFrequencyInputTypes.has(type)) {
rdDebug("send", {
msg,
readyState: ws.readyState,
@@ -2443,7 +2449,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./generated/s
remoteCursor.hidden = true;
return;
}
const surfaceRect = surface.getBoundingClientRect();
const surfaceRect = getContainedSurfaceRect(surface, sourceWidth, sourceHeight);
const containerRect = renderSurface.getBoundingClientRect();
const scaleX = surfaceRect.width / sourceWidth;
const scaleY = surfaceRect.height / sourceHeight;
@@ -3390,6 +3396,9 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./generated/s
? (webrtcVideo.videoHeight || frameHeight)
: (canvas.height || frameHeight);
if (!rect.width || !rect.height || !targetW || !targetH) return null;
rect = getContainedSurfaceRect(surface, targetW, targetH, rect);
if (e.clientX < rect.left || e.clientX > rect.right ||
e.clientY < rect.top || e.clientY > rect.bottom) return null;
let x = ((e.clientX - rect.left) / rect.width) * targetW;
let y = ((e.clientY - rect.top) / rect.height) * targetH;
if (!Number.isFinite(x) || !Number.isFinite(y)) return null;
@@ -3398,14 +3407,32 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./generated/s
return { x, y };
}
function getContainedSurfaceRect(surface, sourceWidth, sourceHeight, elementRect = surface.getBoundingClientRect()) {
if (!elementRect.width || !elementRect.height || !sourceWidth || !sourceHeight) return elementRect;
const fit = getComputedStyle(surface).objectFit;
if (fit !== "contain" && fit !== "scale-down") return elementRect;
let scale = Math.min(elementRect.width / sourceWidth, elementRect.height / sourceHeight);
if (fit === "scale-down") scale = Math.min(1, scale);
const width = sourceWidth * scale;
const height = sourceHeight * scale;
const left = elementRect.left + (elementRect.width - width) / 2;
const top = elementRect.top + (elementRect.height - height) / 2;
return {
x: left, y: top, left, top, right: left + width, bottom: top + height,
width, height, toJSON() { return this; },
};
}
function scheduleMouseMove() {
if (!moveFrame) {
moveFrame = requestAnimationFrame(flushMouseMove);
}
if (moveTimer) return;
const wait = Math.max(0, inputMoveIntervalMs - (performance.now() - lastMoveSentAt));
if (wait === 0) flushMouseMove();
else moveTimer = window.setTimeout(flushMouseMove, wait);
}
function flushMouseMove() {
moveFrame = 0;
if (moveTimer) window.clearTimeout(moveTimer);
moveTimer = 0;
if (!isInputActive() || !pendingMove || !mouseCtrl.checked) {
smoothPoint = null;
return;
@@ -3424,6 +3451,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./generated/s
if (ws && ws.bufferedAmount <= inputBackpressureBytes) {
sendCmd("mouse_move", sendPoint);
lastMoveSentAt = performance.now();
}
if (Math.abs(pendingMove.x - smoothPoint.x) > 0.5 ||
@@ -3447,23 +3475,19 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./generated/s
canvasContainer.focus({ preventScroll: true });
if (!isInputActive() || !mouseCtrl.checked) return;
const pt = getRenderPoint(e);
if (pt) {
pendingMove = pt;
smoothPoint = { x: pt.x, y: pt.y };
sendCmd("mouse_move", pt);
}
sendCmd("mouse_down", { button: e.button, ...(pt || {}) });
if (!pt) return;
pendingMove = pt;
smoothPoint = { x: pt.x, y: pt.y };
sendCmd("mouse_down", { button: e.button, ...pt });
e.preventDefault();
});
inputSurface.addEventListener("mouseup", function (e) {
if (!isInputActive() || !mouseCtrl.checked) return;
const pt = getRenderPoint(e);
if (pt) {
pendingMove = pt;
smoothPoint = { x: pt.x, y: pt.y };
sendCmd("mouse_move", pt);
}
sendCmd("mouse_up", { button: e.button, ...(pt || {}) });
if (!pt) return;
pendingMove = pt;
smoothPoint = { x: pt.x, y: pt.y };
sendCmd("mouse_up", { button: e.button, ...pt });
e.preventDefault();
});
inputSurface.addEventListener("contextmenu", function (e) {
Overlord-Server/public/remotedesktop.html
+2 -2
View File
@@ -272,8 +272,8 @@
</div>
<div class="flex items-center gap-2" title="Smooths mouse input for steadier movement">
<label class="text-sm text-slate-300 w-20" for="smoothingSlider">Smoothing</label>
<input id="smoothingSlider" type="range" min="0" max="80" step="5" value="20" class="flex-1 accent-emerald-400" />
<span id="smoothingValue" class="pill pill-ghost text-slate-100 text-xs min-w-[44px] text-center">20%</span>
<input id="smoothingSlider" type="range" min="0" max="80" step="5" value="0" class="flex-1 accent-emerald-400" />
<span id="smoothingValue" class="pill pill-ghost text-slate-100 text-xs min-w-[44px] text-center">0%</span>
</div>
</div>
Overlord-Server/src/server/ws-console-rd-backstage.ts
+15 -3
View File
@@ -237,7 +237,11 @@ function sendDesktopCommandWithId(target: ClientInfo | undefined, commandType: s
return false;
}
try {
logger.debug(`[rd-debug] send command command=${commandType} client=${target.id} id=${commandId} payload=${JSON.stringify(payload || {})}`);
const isInteractiveInput = commandType.startsWith("desktop_mouse_") ||
commandType.startsWith("desktop_key_") || commandType === "desktop_text";
if (!isInteractiveInput) {
logger.debug(`[rd-debug] send command command=${commandType} client=${target.id} id=${commandId} payload=${JSON.stringify(payload || {})}`);
}
target.ws.send(encodeMessage({ type: "command", commandType: commandType as any, payload, id: commandId }));
metrics.recordCommand(commandType);
return true;
@@ -648,14 +652,22 @@ export function handleRemoteDesktopViewerMessage(ws: ServerWebSocket<SocketData>
if (!state.isStreaming) break;
const commandId = uuidv4();
recordRdInput(commandId, clientId, "mouse_down");
sendDesktopCommandWithId(target, "desktop_mouse_down", { button: Number(payload.button) || 0 }, commandId);
const hasPoint = Number.isFinite(Number((payload as any).x)) && Number.isFinite(Number((payload as any).y));
sendDesktopCommandWithId(target, "desktop_mouse_down", {
button: Number(payload.button) || 0,
...(hasPoint ? { x: Number((payload as any).x), y: Number((payload as any).y) } : {}),
}, commandId);
break;
}
case "mouse_up": {
if (!state.isStreaming) break;
const commandId = uuidv4();
recordRdInput(commandId, clientId, "mouse_up");
sendDesktopCommandWithId(target, "desktop_mouse_up", { button: Number(payload.button) || 0 }, commandId);
const hasPoint = Number.isFinite(Number((payload as any).x)) && Number.isFinite(Number((payload as any).y));
sendDesktopCommandWithId(target, "desktop_mouse_up", {
button: Number(payload.button) || 0,
...(hasPoint ? { x: Number((payload as any).x), y: Number((payload as any).y) } : {}),
}, commandId);
break;
}
case "mouse_wheel": {
Overlord-Server/test/e2e/remotedesktop-input.e2e.ts
+29
View File
@@ -208,6 +208,35 @@ test("WebRTC input stays inactive until start and then forwards mouse and keyboa
expect.objectContaining({ type: "key_up", key: "Enter", code: "Enter" }),
]));
await page.evaluate(() => {
const sent = (window as typeof window & { __rdSent: ArrayBuffer[] }).__rdSent;
sent.length = 0;
const video = document.querySelector<HTMLVideoElement>("#webrtcVideo")!;
video.style.objectFit = "contain";
video.getBoundingClientRect = () => ({
x: 0, y: 0, left: 0, top: 0, right: 1000, bottom: 1000,
width: 1000, height: 1000, toJSON() { return this; },
});
video.dispatchEvent(new MouseEvent("mousedown", { clientX: 500, clientY: 100, button: 0, bubbles: true }));
});
await page.waitForTimeout(20);
await expect.poll(async () => page.evaluate(async () => {
const sent = (window as typeof window & { __rdSent: ArrayBuffer[] }).__rdSent;
const { decodeMsgpack } = await import("/assets/msgpack-helpers.js");
return sent.map((message) => decodeMsgpack(message))
.filter((message) => /^(mouse_|key_|text_input)/.test(message.type));
})).toEqual([]);
await page.evaluate(() => {
const video = document.querySelector<HTMLVideoElement>("#webrtcVideo")!;
video.dispatchEvent(new MouseEvent("mousedown", { clientX: 250, clientY: 360, button: 0, bubbles: true }));
});
await expect.poll(async () => page.evaluate(async () => {
const sent = (window as typeof window & { __rdSent: ArrayBuffer[] }).__rdSent;
const { decodeMsgpack } = await import("/assets/msgpack-helpers.js");
return sent.map((message) => decodeMsgpack(message));
})).toContainEqual(expect.objectContaining({ type: "mouse_down", button: 0, x: 480, y: 271 }));
await page.evaluate(() => {
const socket = (window as typeof window & { __rdSocket: EventTarget }).__rdSocket;
const framePacket = Uint8Array.from([0x46, 0x52, 0x4d, 0, 0, 0, 0, 0]).buffer;