vxaboveground/overlord
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

mouse smoothing + fix fuck ass tests

Browse Source
This commit is contained in:
KingKDot
2026-07-18 19:40:36 -05:00
parent 6f6e28b8e7
commit 12cb3f6b4c
9 changed files with 88 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Overlord-Server/bunfig.toml
+1
View File
@@ -1,2 +1,3 @@
[test]
preload = ["./test/preload.ts"]
pathIgnorePatterns = ["test/e2e/**"]
Overlord-Server/playwright.config.ts
+1
View File
@@ -5,6 +5,7 @@ const baseURL = `http://127.0.0.1:${port}`;
export default defineConfig({
testDir: "./test/e2e",
testMatch: "**/*.e2e.ts",
globalSetup: "./test/e2e/global-setup.ts",
fullyParallel: false,
workers: 1,
Overlord-Server/public/assets/backstage.js
+21 -23
View File
@@ -145,12 +145,10 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
let pendingFrame = null;
let hasCanvasBase = false;
let pendingMove = null;
let moveTimer = null;
let moveFrame = 0;
let videoDecoder = null;
let h264TimestampUs = 0;
let prefersH264 = typeof VideoDecoder === "function";
let lastMoveSentAt = 0;
const mouseMoveIntervalMs = 33;
const inputBackpressureBytes = 256 * 1024;
let h264ErrorCount = 0;
let h264RetryTimer = null;
@@ -1534,17 +1532,15 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
return { x, y };
}
function scheduleMouseMove() {
if (!moveFrame) {
moveFrame = requestAnimationFrame(flushMouseMove);
}
}
function flushMouseMove() {
moveTimer = null;
if (!pendingMove || !mouseCtrl.checked) return;
const now = performance.now();
if (now - lastMoveSentAt < mouseMoveIntervalMs) {
if (!moveTimer) {
moveTimer = setTimeout(flushMouseMove, mouseMoveIntervalMs);
}
return;
}
lastMoveSentAt = now;
moveFrame = 0;
if (!desiredStreaming || !pendingMove || !mouseCtrl.checked) return;
if (ws.bufferedAmount <= inputBackpressureBytes) {
sendCmd("backstage_mouse_move", pendingMove);
}
@@ -1553,17 +1549,15 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
for (const inputSurface of [canvas, webrtcVideo]) {
if (!inputSurface) continue;
inputSurface.addEventListener("mousemove", function (e) {
if (!mouseCtrl.checked) return;
if (!desiredStreaming || !mouseCtrl.checked) return;
const pt = getRenderPoint(e);
if (!pt) return;
pendingMove = pt;
if (!moveTimer) {
flushMouseMove();
}
scheduleMouseMove();
});
inputSurface.addEventListener("mousedown", function (e) {
canvasContainer.focus({ preventScroll: true });
if (!mouseCtrl.checked) return;
if (!desiredStreaming || !mouseCtrl.checked) return;
const pt = getRenderPoint(e);
if (pt) {
pendingMove = pt;
@@ -1575,7 +1569,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
e.preventDefault();
});
inputSurface.addEventListener("mouseup", function (e) {
if (!mouseCtrl.checked) return;
if (!desiredStreaming || !mouseCtrl.checked) return;
const pt = getRenderPoint(e);
if (pt) {
pendingMove = pt;
@@ -1587,10 +1581,10 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
e.preventDefault();
});
inputSurface.addEventListener("contextmenu", function (e) {
e.preventDefault();
if (desiredStreaming && mouseCtrl.checked) e.preventDefault();
});
inputSurface.addEventListener("wheel", function (e) {
if (!mouseCtrl.checked) return;
if (!desiredStreaming || !mouseCtrl.checked) return;
const pt = getRenderPoint(e);
if (!pt) return;
const delta = Math.max(-120, Math.min(120, Math.round(-e.deltaY)));
@@ -1604,8 +1598,12 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
canvasContainer.setAttribute("tabindex", "0");
const kbdCapture = createKeyboardCapture({
container: canvasContainer,
sendKeyDown: (e) => sendCmd("backstage_key_down", { key: e.key, code: e.code }),
sendKeyUp: (e) => sendCmd("backstage_key_up", { key: e.key, code: e.code }),
sendKeyDown: (e) => {
if (desiredStreaming) sendCmd("backstage_key_down", { key: e.key, code: e.code });
},
sendKeyUp: (e) => {
if (desiredStreaming) sendCmd("backstage_key_up", { key: e.key, code: e.code });
},
});
if (kbdCtrl) {
kbdCtrl.addEventListener("change", function () {
Overlord-Server/public/assets/remotedesktop.js
+41 -29
View File
@@ -117,7 +117,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
let smoothingPct = 20;
let smoothPoint = null;
let pendingMove = null;
let moveTimer = null;
let moveFrame = 0;
let frameDecodeBusy = false;
let pendingFrame = null;
let videoDecoder = null;
@@ -143,9 +143,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
const H264_KEYFRAME_ERROR_RESTART_THRESHOLD = 24;
const H264_MAX_RECOVERY_ATTEMPTS = 1;
const H264_DECODE_WARN_THROTTLE_MS = 2000;
const mouseMoveIntervalMs = 33;
const inputBackpressureBytes = 256 * 1024;
let lastMoveSentAt = 0;
const diagnostics = {
agent: null,
network: null,
@@ -1840,6 +1838,10 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
}
}
function isInputActive() {
return desiredStreaming;
}
mouseCtrl.addEventListener("change", function () {
pushInputToggles();
sharedSettingsSaver.scheduleSave();
@@ -1847,9 +1849,15 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
canvasContainer.setAttribute("tabindex", "0");
const kbdCapture = createKeyboardCapture({
container: canvasContainer,
sendKeyDown: (e) => sendCmd("key_down", { key: e.key, code: e.code }),
sendKeyUp: (e) => sendCmd("key_up", { key: e.key, code: e.code }),
onTextInput: (e) => sendCmd("text_input", { text: e.key }),
sendKeyDown: (e) => {
if (isInputActive()) sendCmd("key_down", { key: e.key, code: e.code });
},
sendKeyUp: (e) => {
if (isInputActive()) sendCmd("key_up", { key: e.key, code: e.code });
},
onTextInput: (e) => {
if (isInputActive()) sendCmd("text_input", { text: e.key });
},
});
kbdCtrl.addEventListener("change", function () {
if (kbdCtrl.checked) kbdCapture.enable();
@@ -2666,50 +2674,54 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
return { x, y };
}
function scheduleMouseMove() {
if (!moveFrame) {
moveFrame = requestAnimationFrame(flushMouseMove);
}
}
function flushMouseMove() {
moveTimer = null;
if (!pendingMove || !mouseCtrl.checked) return;
const now = performance.now();
moveFrame = 0;
if (!isInputActive() || !pendingMove || !mouseCtrl.checked) {
smoothPoint = null;
return;
}
if (!smoothPoint) {
smoothPoint = { x: pendingMove.x, y: pendingMove.y };
}
const factor = Math.max(0, Math.min(0.8, smoothingPct / 100));
const alpha = 1 - factor;
smoothPoint.x += (pendingMove.x - smoothPoint.x) * alpha;
smoothPoint.y += (pendingMove.y - smoothPoint.y) * alpha;
const factor = Math.max(0, Math.min(0.4, smoothingPct / 200));
smoothPoint.x += (pendingMove.x - smoothPoint.x) * (1 - factor);
smoothPoint.y += (pendingMove.y - smoothPoint.y) * (1 - factor);
const sendPoint = {
x: Math.round(smoothPoint.x),
y: Math.round(smoothPoint.y),
};
if (now - lastMoveSentAt < mouseMoveIntervalMs) {
if (!moveTimer) {
moveTimer = setTimeout(flushMouseMove, mouseMoveIntervalMs);
}
return;
}
lastMoveSentAt = now;
if (ws && ws.bufferedAmount <= inputBackpressureBytes) {
sendCmd("mouse_move", sendPoint);
}
if (Math.abs(pendingMove.x - smoothPoint.x) > 0.5 ||
Math.abs(pendingMove.y - smoothPoint.y) > 0.5) {
scheduleMouseMove();
} else {
smoothPoint = { x: pendingMove.x, y: pendingMove.y };
}
}
for (const inputSurface of [canvas, webrtcVideo]) {
if (!inputSurface) continue;
inputSurface.addEventListener("mousemove", function (e) {
if (!mouseCtrl.checked) return;
if (!isInputActive() || !mouseCtrl.checked) return;
const pt = getRenderPoint(e);
if (!pt) return;
pendingMove = pt;
if (!moveTimer) {
flushMouseMove();
}
scheduleMouseMove();
});
inputSurface.addEventListener("mousedown", function (e) {
canvasContainer.focus({ preventScroll: true });
if (!mouseCtrl.checked) return;
if (!isInputActive() || !mouseCtrl.checked) return;
const pt = getRenderPoint(e);
if (pt) {
pendingMove = pt;
@@ -2720,7 +2732,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
e.preventDefault();
});
inputSurface.addEventListener("mouseup", function (e) {
if (!mouseCtrl.checked) return;
if (!isInputActive() || !mouseCtrl.checked) return;
const pt = getRenderPoint(e);
if (pt) {
pendingMove = pt;
@@ -2731,10 +2743,10 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
e.preventDefault();
});
inputSurface.addEventListener("contextmenu", function (e) {
e.preventDefault();
if (isInputActive() && mouseCtrl.checked) e.preventDefault();
});
inputSurface.addEventListener("wheel", function (e) {
if (!mouseCtrl.checked) return;
if (!isInputActive() || !mouseCtrl.checked) return;
const pt = getRenderPoint(e);
if (!pt) return;
const delta = Math.max(-120, Math.min(120, Math.round(-e.deltaY)));
Overlord-Server/src/paths.test.ts
+7 -1
View File
@@ -8,9 +8,15 @@ test("creates isolated test data when the Bun preload was not discovered", async
delete childEnv.DATA_DIR;
delete childEnv.OVERLORD_TEST_DATA_DIR;
childEnv.NODE_ENV = "test";
// VS Code's Bun adapter injects inspector settings into the test process.
// A child that inherits them can wait for the same debugger and never exit.
delete childEnv.BUN_OPTIONS;
for (const key of Object.keys(childEnv)) {
if (key.startsWith("BUN_INSPECT")) delete childEnv[key];
}
const child = Bun.spawn([
"bun",
process.execPath,
"-e",
'const { resolveDataDir } = await import("./src/paths.ts"); console.log(resolveDataDir());',
], {
Overlord-Server/test/e2e/access.spec.ts → Overlord-Server/test/e2e/access.e2e.ts
View File
Overlord-Server/test/e2e/file-transfer-api.spec.ts → Overlord-Server/test/e2e/file-transfer-api.e2e.ts
View File
Overlord-Server/test/e2e/remotedesktop-input.spec.ts → Overlord-Server/test/e2e/remotedesktop-input.e2e.ts
+17 -1
View File
@@ -1,7 +1,7 @@
import { expect, test } from "@playwright/test";
import { login } from "./helpers";
test("WebRTC video forwards mouse and keyboard input", async ({ page }) => {
test("WebRTC input stays inactive until start and then forwards mouse and keyboard", async ({ page }) => {
await login(page);
await page.addInitScript(() => {
const sent: ArrayBuffer[] = [];
@@ -71,6 +71,22 @@ test("WebRTC video forwards mouse and keyboard input", async ({ page }) => {
toJSON() { return this; },
});
video.dispatchEvent(new MouseEvent("mousedown", { clientX: 480, clientY: 270, button: 0, bubbles: true }));
window.dispatchEvent(new KeyboardEvent("keydown", { key: "Enter", code: "Enter", bubbles: true }));
});
await expect.poll(async () => page.evaluate(async () => {
const sent = (window as typeof window & { __rdSent: ArrayBuffer[] }).__rdSent;
const { decodeMsgpack } = await import("/assets/msgpack-helpers.js");
return sent.map((message) => decodeMsgpack(message))
.filter((message) => /^(mouse_|key_|text_input)/.test(message.type));
})).toEqual([]);
await page.locator("#startBtn").click();
await page.evaluate(() => {
const sent = (window as typeof window & { __rdSent: ArrayBuffer[] }).__rdSent;
sent.length = 0;
const video = document.querySelector<HTMLVideoElement>("#webrtcVideo")!;
video.dispatchEvent(new MouseEvent("mousemove", { clientX: 480, clientY: 270, bubbles: true }));
video.dispatchEvent(new MouseEvent("mousedown", { clientX: 480, clientY: 270, button: 0, bubbles: true }));
video.dispatchEvent(new MouseEvent("mouseup", { clientX: 480, clientY: 270, button: 0, bubbles: true }));
Overlord-Server/test/e2e/smoke.spec.ts → Overlord-Server/test/e2e/smoke.e2e.ts
View File