major remote desktop improvements

This commit is contained in:
KingKDot
2026-07-30 00:22:08 -05:00
parent a68172538d
commit 095a119bf4
35 changed files with 984 additions and 120 deletions
+7 -7
View File
@@ -26,7 +26,7 @@ export async function showCertBannerIfNeeded(anchor) {
} catch { }
}
const os = detectOS();
const os = getCertificateTrustGuide();
const banner = document.createElement("div");
banner.id = "cert-trust-banner";
@@ -45,7 +45,7 @@ export async function showCertBannerIfNeeded(anchor) {
</p>
<div class="flex flex-wrap items-center gap-2">
<a href="/api/cert/download"
download="overlord-ca.crt"
download="overlord-server.crt"
class="inline-flex items-center gap-1.5 rounded bg-amber-600 hover:bg-amber-500
px-3 py-1 text-xs font-medium text-white transition-colors">
<svg class="w-3.5 h-3.5" fill="none" stroke="currentColor" stroke-width="2"
@@ -80,13 +80,13 @@ export async function showCertBannerIfNeeded(anchor) {
});
}
function detectOS() {
export function getCertificateTrustGuide() {
const ua = navigator.userAgent.toLowerCase();
if (ua.includes("win")) {
return {
label: "Windows",
instructions:
`1. Double-click the downloaded <strong>overlord-ca.crt</strong> file.
`1. Double-click the downloaded <strong>overlord-server.crt</strong> file.
2. Click <strong>Install Certificate...</strong>
3. Choose <strong>Current User</strong> or <strong>Local Machine</strong>.
4. Select <strong>"Place all certificates in the following store"</strong>.
@@ -99,7 +99,7 @@ function detectOS() {
return {
label: "macOS",
instructions:
`1. Double-click the downloaded <strong>overlord-ca.crt</strong> file — it opens in Keychain Access.
`1. Double-click the downloaded <strong>overlord-server.crt</strong> file — it opens in Keychain Access.
2. If prompted, add it to the <strong>login</strong> or <strong>System</strong> keychain.
3. Find the certificate in the list, double-click it.
4. Expand <strong>Trust</strong> and set <strong>"When using this certificate"</strong> to <strong>Always Trust</strong>.
@@ -111,11 +111,11 @@ function detectOS() {
label: "Linux",
instructions:
`<strong>Debian / Ubuntu:</strong>
sudo cp overlord-ca.crt /usr/local/share/ca-certificates/overlord-ca.crt
sudo cp overlord-server.crt /usr/local/share/ca-certificates/overlord-server.crt
sudo update-ca-certificates
<strong>Fedora / RHEL:</strong>
sudo cp overlord-ca.crt /etc/pki/ca-trust/source/anchors/overlord-ca.crt
sudo cp overlord-server.crt /etc/pki/ca-trust/source/anchors/overlord-server.crt
sudo update-ca-trust
Then <strong>restart your browser</strong>.
+93 -32
View File
@@ -153,6 +153,8 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
let pendingFrame = null;
let videoDecoder = null;
let videoDecoderConfigKey = "";
let pendingDecodedVideoFrame = null;
let decodedVideoPresentationFrame = 0;
const disabledDecoderCodecs = new Set();
let h264StreamCodec = "";
let canvasDecodePressure = false;
@@ -167,6 +169,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
let h264KeyframeErrorStreak = 0;
let h264RecoveryAttempts = 0;
let h264LastDecodeWarnAt = 0;
let h264WaitingForKeyframe = true;
const H264_LOW_FPS_THRESHOLD = 6;
const H264_FALLBACK_WARMUP_MS = 10000;
const H264_MIN_FRAMES_BEFORE_FALLBACK = 120;
@@ -175,9 +178,8 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
const H264_MAX_RECOVERY_ATTEMPTS = 1;
const H264_DECODE_WARN_THROTTLE_MS = 2000;
const inputBackpressureBytes = 256 * 1024;
const VIEWER_FRAME_GAP_KEYFRAME_MS = 500;
const VIEWER_FRAME_GAP_KEYFRAME_THROTTLE_MS = 1000;
let lastViewerFrameGapKeyframeAt = 0;
let lastViewerKeyframeRequestAt = 0;
const diagnostics = {
agent: null,
network: null,
@@ -240,6 +242,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
h264FirstFrameAt = 0;
h264FramesSeen = 0;
h264KeyframeErrorStreak = 0;
h264WaitingForKeyframe = true;
}
/* ── Remote Desktop Audio (system audio from client) ── */
@@ -1139,6 +1142,15 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
} else if ((finite(media.lossPercent) ?? 0) > 3 || (finite(network.rttMs) ?? 0) > 150 || (finite(media.jitterMs) ?? 0) > 35) {
summary = "Network conditions are causing delay";
severity = "bad";
} else if (
mode === "off" &&
fps != null &&
viewerRate != null &&
viewerRate < fps * 0.8 &&
diagnostics.coalescedFrames > 0
) {
summary = `Display is presenting ${Math.round(viewerRate)} of ${Math.round(fps)} decoded FPS`;
severity = "warn";
} else if ((queue ?? 0) > 2 || (renderMs != null && renderMs > frameBudget)) {
summary = "Viewer render queue is backing up";
severity = "warn";
@@ -1969,7 +1981,8 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
pushBitrate();
desiredStreaming = true;
lastFrameAt = 0;
lastViewerFrameGapKeyframeAt = 0;
lastViewerKeyframeRequestAt = 0;
diagnostics.coalescedFrames = 0;
firstFrameLogged = false;
resetH264SessionState();
setStreamState("starting", "Starting stream");
@@ -2322,17 +2335,8 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
function markFrameReceived() {
const now = performance.now();
const frameGapMs = lastFrameAt ? now - lastFrameAt : 0;
lastFrameAt = now;
clearOfflineTimer();
if (
desiredStreaming &&
frameGapMs >= VIEWER_FRAME_GAP_KEYFRAME_MS &&
(!lastViewerFrameGapKeyframeAt || now - lastViewerFrameGapKeyframeAt >= VIEWER_FRAME_GAP_KEYFRAME_THROTTLE_MS)
) {
lastViewerFrameGapKeyframeAt = now;
sendCmd("desktop_request_keyframe", { reason: "viewer_frame_gap" });
}
if (!firstFrameLogged) {
firstFrameLogged = true;
rdDebug("first frame received", {
@@ -2394,6 +2398,14 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
function destroyVideoDecoder() {
updateCanvasDecodePressure(true);
if (decodedVideoPresentationFrame) {
cancelAnimationFrame(decodedVideoPresentationFrame);
decodedVideoPresentationFrame = 0;
}
if (pendingDecodedVideoFrame) {
pendingDecodedVideoFrame.frame.close();
pendingDecodedVideoFrame = null;
}
if (videoDecoder) {
try {
videoDecoder.close();
@@ -2406,6 +2418,47 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
resetH264RuntimeState();
}
function presentLatestDecodedVideoFrame() {
decodedVideoPresentationFrame = 0;
const pending = pendingDecodedVideoFrame;
pendingDecodedVideoFrame = null;
if (!pending) return;
const { frame, timing, width, height } = pending;
if (width > 0 && height > 0 && (canvas.width !== width || canvas.height !== height)) {
canvas.width = width;
canvas.height = height;
frameWidth = width;
frameHeight = height;
}
try {
ctx.drawImage(frame, 0, 0, canvas.width, canvas.height);
} finally {
frame.close();
}
const renderedAt = performance.now();
if (timing) {
diagnostics.renderMs = smoothed(diagnostics.renderMs, Math.max(0, renderedAt - timing.receivedAt));
}
diagnostics.width = width;
diagnostics.height = height;
updateFpsDisplay();
}
function queueDecodedVideoFrame(frame, timing) {
const width = frame.displayWidth || frame.codedWidth || frameWidth;
const height = frame.displayHeight || frame.codedHeight || frameHeight;
if (pendingDecodedVideoFrame) {
pendingDecodedVideoFrame.frame.close();
diagnostics.coalescedFrames += 1;
}
pendingDecodedVideoFrame = { frame, timing, width, height };
if (!decodedVideoPresentationFrame) {
decodedVideoPresentationFrame = requestAnimationFrame(presentLatestDecodedVideoFrame);
}
}
function normalizeFallbackReason(reason) {
if (!reason) return "unspecified";
if (typeof reason === "string") return reason;
@@ -2458,6 +2511,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
h264RecoveryAttempts += 1;
h264KeyframeErrorStreak = 0;
destroyVideoDecoder();
console.warn("rd: h264 decode stuck waiting for keyframe; auto-restarting stream once", {
reason,
@@ -2578,34 +2632,24 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
output: (frame) => {
const outputStartedAt = performance.now();
const timing = h264PendingTimings.shift();
const width = frame.displayWidth || frame.codedWidth || frameWidth;
const height = frame.displayHeight || frame.codedHeight || frameHeight;
if (width > 0 && height > 0 && (canvas.width !== width || canvas.height !== height)) {
canvas.width = width;
canvas.height = height;
frameWidth = width;
frameHeight = height;
}
try {
ctx.drawImage(frame, 0, 0, canvas.width, canvas.height);
} finally {
frame.close();
}
const renderedAt = performance.now();
h264KeyframeErrorStreak = 0;
if (timing) {
diagnostics.decodeMs = smoothed(diagnostics.decodeMs, Math.max(0, outputStartedAt - timing.decodeStartedAt));
diagnostics.renderMs = smoothed(diagnostics.renderMs, Math.max(0, renderedAt - timing.receivedAt));
}
diagnostics.decodeQueue = videoDecoder?.decodeQueueSize || h264PendingTimings.length;
diagnostics.width = width;
diagnostics.height = height;
updateFpsDisplay();
queueDecodedVideoFrame(frame, timing);
updateCanvasDecodePressure();
},
error: (err) => {
console.warn(`rd: ${codecName} decoder error`, err);
if (codecName === "h264") h264WaitingForKeyframe = true;
updateCanvasDecodePressure(true);
setTimeout(() => fallbackFromVideoCodec(codecName, err), 0);
setTimeout(() => {
if (codecName === "h264" && tryRecoverH264Stream("h264_async_decoder_error")) {
return;
}
fallbackFromVideoCodec(codecName, err);
}, 0);
},
});
videoDecoder.addEventListener("dequeue", () => {
@@ -2795,6 +2839,23 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
h264FramesSeen += 1;
const isKey = codecName === "hevc" ? isHEVCKeyFrame(videoBytes) : isH264KeyFrame(videoBytes);
if (codecName === "h264" && h264WaitingForKeyframe && !isKey) {
h264KeyframeErrorStreak += 1;
const now = performance.now();
if (!lastViewerKeyframeRequestAt || now - lastViewerKeyframeRequestAt >= VIEWER_FRAME_GAP_KEYFRAME_THROTTLE_MS) {
lastViewerKeyframeRequestAt = now;
sendCmd("desktop_request_keyframe", { reason: "h264_decoder_keyframe_required" });
}
if (h264KeyframeErrorStreak >= H264_KEYFRAME_ERROR_RESTART_THRESHOLD) {
const restarted = tryRecoverH264Stream("h264_keyframe_timeout");
if (!restarted) fallbackToJpegCodec("h264_keyframe_timeout_loop");
}
updateCanvasDecodePressure(true);
return;
}
if (codecName === "h264" && isKey) {
h264WaitingForKeyframe = false;
}
// If software H264 encode on the agent cannot keep up, automatically
// fall back to JPEG blocks for a smoother interactive stream.
@@ -2827,10 +2888,10 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
videoDecoder.decode(chunk);
diagnostics.decodeQueue = videoDecoder.decodeQueueSize;
updateCanvasDecodePressure();
h264KeyframeErrorStreak = 0;
} catch (err) {
h264PendingTimings.pop();
if (isKeyframeRequiredError(err)) {
if (codecName === "h264") h264WaitingForKeyframe = true;
h264KeyframeErrorStreak += 1;
const now = Date.now();
if (now - h264LastDecodeWarnAt >= H264_DECODE_WARN_THROTTLE_MS) {
+45
View File
@@ -6,6 +6,7 @@ import {
requestDesktopNotificationPermission,
} from "./notify-client.js";
import { escapeHtml, formatBytes as formatSharedBytes, formatDate as formatSharedDate } from "./format.js";
import { getCertificateTrustGuide } from "./cert-banner.js";
const PREF_REFRESH_KEY = "overlord_refresh_interval_seconds";
const NAV_MODE_KEY = "sb_mode";
@@ -81,6 +82,10 @@ const tlsCertbotDomainInput = document.getElementById("tls-certbot-domain");
const tlsCertbotCertFileInput = document.getElementById("tls-certbot-cert-file");
const tlsCertbotKeyFileInput = document.getElementById("tls-certbot-key-file");
const tlsCertbotCaFileInput = document.getElementById("tls-certbot-ca-file");
const tlsActiveCertSource = document.getElementById("tls-active-cert-source");
const tlsDownloadCertBtn = document.getElementById("tls-download-cert-btn");
const tlsTrustHelpBtn = document.getElementById("tls-trust-help-btn");
const tlsTrustInstructions = document.getElementById("tls-trust-instructions");
const oidcForm = document.getElementById("oidc-form");
const oidcPermissionNote = document.getElementById("oidc-permission-note");
@@ -457,6 +462,45 @@ function applyTlsForm() {
tlsCertbotCaFileInput.value = certbot.caFileName || "chain.pem";
}
tlsTrustHelpBtn?.addEventListener("click", () => {
tlsTrustInstructions?.classList.toggle("hidden");
});
async function loadActiveTlsCertificate() {
const guide = getCertificateTrustGuide();
if (tlsTrustHelpBtn) {
tlsTrustHelpBtn.innerHTML = `<i class="fa-solid fa-shield-halved"></i>Trust on ${guide.label}`;
}
if (tlsTrustInstructions) {
tlsTrustInstructions.innerHTML = guide.instructions;
}
try {
const res = await fetch("/api/cert/info", { credentials: "include" });
if (!res.ok) throw new Error(`HTTP ${res.status}`);
const data = await res.json();
const source = String(data.source || "unknown");
if (tlsActiveCertSource) {
tlsActiveCertSource.textContent = source === "self-signed"
? "Self-signed"
: source === "certbot"
? "Certbot / Let's Encrypt"
: source === "configured"
? "Configured certificate"
: "Unknown source";
tlsActiveCertSource.classList.toggle("border-amber-600", source === "self-signed");
tlsActiveCertSource.classList.toggle("bg-amber-950", source === "self-signed");
tlsActiveCertSource.classList.toggle("text-amber-200", source === "self-signed");
}
} catch {
if (tlsActiveCertSource) tlsActiveCertSource.textContent = "Unavailable";
if (tlsDownloadCertBtn) {
tlsDownloadCertBtn.setAttribute("aria-disabled", "true");
tlsDownloadCertBtn.classList.add("pointer-events-none", "opacity-50");
}
}
}
async function loadTlsSettings() {
if (!currentUser) return;
@@ -489,6 +533,7 @@ async function loadTlsSettings() {
tlsConfig = data.tls || null;
applyTlsForm();
setTlsFormDisabled(false);
await loadActiveTlsCertificate();
}
function listToCsv(value) {
+40
View File
@@ -788,6 +788,46 @@
Your permissions do not allow TLS setting changes.
</div>
<div class="rounded-lg border border-cyan-800/70 bg-cyan-950/20 px-4 py-4 space-y-3">
<div class="flex flex-col sm:flex-row sm:items-center gap-3">
<div class="flex-1">
<div class="flex flex-wrap items-center gap-2">
<p class="text-sm font-medium text-cyan-100">Active server certificate</p>
<span id="tls-active-cert-source" class="rounded-full border border-slate-600 bg-slate-800 px-2 py-0.5 text-xs text-slate-300">
Checking…
</span>
</div>
<p class="mt-1 text-xs text-slate-400">
Download and trust this certificate when a locally generated certificate causes HTTPS or WebSocket trust errors.
</p>
</div>
<div class="flex flex-wrap gap-2">
<a
id="tls-download-cert-btn"
href="/api/cert/download"
download="overlord-server.crt"
class="inline-flex items-center gap-2 px-3 py-2 rounded-lg bg-cyan-700 hover:bg-cyan-600 transition-colors text-white text-sm font-medium"
>
<i class="fa-solid fa-download"></i>Download Certificate
</a>
<button
type="button"
id="tls-trust-help-btn"
class="inline-flex items-center gap-2 px-3 py-2 rounded-lg border border-slate-600 bg-slate-800 hover:bg-slate-700 transition-colors text-slate-100 text-sm font-medium"
>
<i class="fa-solid fa-shield-halved"></i>Trust Instructions
</button>
</div>
</div>
<div
id="tls-trust-instructions"
class="hidden rounded-lg border border-slate-700 bg-slate-950/80 p-3 text-xs text-slate-300 leading-relaxed whitespace-pre-line"
></div>
<p class="text-xs text-amber-300/90">
Browsers cannot install trusted certificates automatically. Your operating system will require confirmation, and the browser must be restarted afterward.
</p>
</div>
<label class="flex items-center gap-2 text-sm">
<input id="tls-certbot-enabled" type="checkbox" class="h-4 w-4" />
<span>Use certbot certificates in production runtime (NODE_ENV=production)</span>
@@ -0,0 +1,15 @@
import { describe, expect, test } from "bun:test";
import { assetCacheControl } from "./assets-routes";
describe("asset cache policy", () => {
test("revalidates stable JavaScript and stylesheet filenames after a deployment", () => {
expect(assetCacheControl("remotedesktop.js")).toBe("no-cache");
expect(assetCacheControl("main.min.js")).toBe("no-cache");
expect(assetCacheControl("tailwind.css")).toBe("no-cache");
});
test("keeps binary and font assets immutable", () => {
expect(assetCacheControl("logo.png")).toContain("immutable");
expect(assetCacheControl("inter.woff2")).toContain("immutable");
});
});
@@ -22,7 +22,6 @@ function acceptsGzip(req: Request): boolean {
}
const STATIC_ASSET_CACHE = "public, max-age=31536000, immutable";
const MUTABLE_ASSET_CACHE = "public, max-age=86400, stale-while-revalidate=604800";
const NO_CACHE = "no-cache";
const MAX_GZIP_CACHE_ENTRIES = 128;
const MAX_GZIP_CACHE_BYTES = 16 * 1024 * 1024;
@@ -35,16 +34,16 @@ type GzipCacheEntry = {
const gzipAssetCache = new Map<string, GzipCacheEntry>();
let gzipAssetCacheBytes = 0;
function assetCacheControl(relativePath: string): string {
export function assetCacheControl(relativePath: string): string {
if (relativePath === "custom.css") return NO_CACHE;
if (relativePath === "notification-sw.js") return NO_CACHE;
if (relativePath.endsWith(".min.js") || relativePath.endsWith(".min.css")) {
return STATIC_ASSET_CACHE;
}
if (/\.(ico|png|jpg|jpeg|gif|webp|woff2?|ttf|eot|svg)$/.test(relativePath)) {
return STATIC_ASSET_CACHE;
}
return MUTABLE_ASSET_CACHE;
// Script and stylesheet filenames are stable across releases. Revalidate
// them so a rebuilt Docker image cannot leave the previous UI cached for a
// day (or stale-served for a week). ETags keep unchanged requests cheap.
return NO_CACHE;
}
function toExactArrayBuffer(bytes: Uint8Array): ArrayBuffer {
@@ -0,0 +1,70 @@
import { afterAll, beforeAll, describe, expect, test } from "bun:test";
import { mkdtempSync, rmSync, writeFileSync } from "fs";
import { tmpdir } from "os";
import path from "path";
import { generateToken } from "../../auth";
import { createUser, deleteUser, getUserById } from "../../users";
import { handleMiscRoutes } from "./misc-routes";
const password = "Aa1!CertificateDownload_2026";
const tempRoot = mkdtempSync(path.join(tmpdir(), "overlord-cert-download-"));
const certPath = path.join(tempRoot, "server.crt");
const certBody = "-----BEGIN CERTIFICATE-----\ntest\n-----END CERTIFICATE-----\n";
let userId = 0;
let token = "";
const deps = {
CORS_HEADERS: {},
SERVER_VERSION: "test",
PUBLIC_ROOT: ".",
requestIP: () => ({ address: "127.0.0.1" }),
getConsoleSessionCount: () => 0,
getRdSessionCount: () => 0,
getFileBrowserSessionCount: () => 0,
getProcessSessionCount: () => 0,
tlsCertPath: certPath,
tlsSource: "self-signed" as const,
};
beforeAll(async () => {
writeFileSync(certPath, certBody);
const created = await createUser(
`cert_download_${Date.now().toString(36)}`,
password,
"admin",
"test",
);
if (!created.success || !created.userId) throw new Error("failed to create certificate test user");
userId = created.userId;
token = await generateToken(getUserById(userId)!);
});
afterAll(() => {
if (userId) deleteUser(userId);
rmSync(tempRoot, { recursive: true, force: true });
});
describe("active TLS certificate download", () => {
test("requires authentication", async () => {
const url = new URL("https://localhost/api/cert/download");
const response = await handleMiscRoutes(new Request(url), url, deps);
expect(response?.status).toBe(401);
});
test("downloads the active certificate with safe headers", async () => {
const url = new URL("https://localhost/api/cert/download");
const response = await handleMiscRoutes(
new Request(url, { headers: { Cookie: `overlord_token=${token}` } }),
url,
deps,
);
expect(response?.status).toBe(200);
expect(response?.headers.get("content-disposition")).toBe(
'attachment; filename="overlord-server.crt"',
);
expect(response?.headers.get("cache-control")).toBe("no-store");
expect(response?.headers.get("x-content-type-options")).toBe("nosniff");
expect(await response?.text()).toBe(certBody);
});
});
@@ -1380,7 +1380,9 @@ export async function handleMiscRoutes(
return new Response(file, {
headers: {
"Content-Type": "application/x-pem-file",
"Content-Disposition": 'attachment; filename="overlord-ca.crt"',
"Content-Disposition": 'attachment; filename="overlord-server.crt"',
"Cache-Control": "no-store",
"X-Content-Type-Options": "nosniff",
...deps.CORS_HEADERS,
},
});
@@ -146,12 +146,12 @@ describe("remote desktop viewer control", () => {
handleRemoteDesktopViewerMessage(viewerSocket, JSON.stringify({
type: "desktop_request_keyframe",
reason: "viewer_frame_gap",
reason: "h264_decoder_keyframe_required",
}));
const automaticCommand = agentCommands(agentWs)
.filter((msg) => msg.commandType === "desktop_request_keyframe")
.at(-1);
expect(automaticCommand?.payload).toEqual({ reason: "viewer_frame_gap" });
expect(automaticCommand?.payload).toEqual({ reason: "h264_decoder_keyframe_required" });
});
test("requests HEVC recovery after a screenshot only while streaming", () => {
@@ -381,9 +381,39 @@ describe("remote desktop viewer control", () => {
{ format: "h264", fps: 60, width: 2560, height: 1440 },
);
expect(pressuredAck).toBe(false);
for (let frameNumber = 0; frameNumber < 3; frameNumber++) {
const additionalAck = (globalThis as any).__rdBroadcast(
clientId,
new Uint8Array([7 + frameNumber]),
{ format: "h264", fps: 60, width: 2560, height: 1440 },
);
expect(additionalAck).toBe(false);
}
handleRemoteDesktopViewerMessage(viewer as any, JSON.stringify({ type: "desktop_decode_pressure", active: false }));
expect(agentCommands(agentWs).filter((msg) => msg.type === "frame_ack")).toHaveLength(1);
expect(agentCommands(agentWs).filter((msg) => msg.type === "frame_ack")).toHaveLength(4);
});
test("coalesces keyframe requests while a viewer remains backpressured", () => {
const clientId = `rd-backpressure-${Date.now().toString(36)}`;
const { agentWs } = createClient(clientId);
const viewer = createMockWs({ clientId });
viewer.getBufferedAmount = () => 2 * 1024 * 1024;
handleRemoteDesktopViewerOpen(viewer as any);
for (let frame = 0; frame < 3; frame++) {
const shouldAck = (globalThis as any).__rdBroadcast(
clientId,
new Uint8Array([frame]),
{ format: "h264", fps: 60, width: 1920, height: 1080 },
);
expect(shouldAck).toBe(true);
}
const requests = agentCommands(agentWs)
.filter((msg) => msg.commandType === "desktop_request_keyframe");
expect(requests).toHaveLength(1);
expect(requests[0]?.payload?.reason).toBe("viewer_backpressure");
});
});
@@ -138,6 +138,7 @@ type FrameBroadcastResult = {
sent: boolean;
dropped: boolean;
viewers: number;
sentViewers: Set<ServerWebSocket<SocketData>>;
};
function broadcastFrameToViewers(
@@ -148,6 +149,7 @@ function broadcastFrameToViewers(
let sent = false;
let dropped = false;
let viewers = 0;
const sentViewers = new Set<ServerWebSocket<SocketData>>();
const t0 = performance.now();
const byteLen = buf.length;
for (const session of sessions) {
@@ -161,6 +163,7 @@ function broadcastFrameToViewers(
session.viewer.send(buf);
metrics.recordBytesSent(byteLen);
sent = true;
sentViewers.add(session.viewer);
} catch (err) {
logger.error("[rd] viewer frame send failed", err);
}
@@ -172,14 +175,15 @@ function broadcastFrameToViewers(
rdSendStats.sendMs += elapsed;
}
logRdSend(header);
return { sent, dropped, viewers };
return { sent, dropped, viewers, sentViewers };
}
const rdSendStats = { lastLog: 0, frames: 0, sendMs: 0, bytes: 0 };
const rdDebugFrameLogAt = new Map<string, number>();
const rdCanvasFrameAckPending = new Map<string, string>();
const rdCanvasFrameAckPending = new Map<string, { sessionId: string; count: number }>();
const rdBackpressureKeyframeAt = new Map<string, number>();
const RD_BACKPRESSURE_KEYFRAME_INTERVAL_MS = 1_000;
const AUTOMATIC_DESKTOP_KEYFRAME_REASONS: Record<string, true> = {
viewer_frame_gap: true,
h264_decoder_keyframe_required: true,
hevc_decoder_keyframe_required: true,
};
@@ -487,6 +491,7 @@ export function handleRemoteDesktopViewerMessage(ws: ServerWebSocket<SocketData>
.filter(s => s.id !== ws.data.sessionId);
logger.debug(`[rd-debug] desktop_stop requested client=${clientId} session=${ws.data.sessionId || ""} otherViewers=${otherViewers.length} state=${JSON.stringify(state)}`);
if (otherViewers.length === 0) {
rdBackpressureKeyframeAt.delete(clientId);
stopRemoteDesktopRecording(clientId, "desktop stopped");
sendDesktopCommand(target, "desktop_stop", {});
sendDesktopCommand(target, "webrtc_stop", { kind: "desktop" });
@@ -863,23 +868,32 @@ function broadcastRemoteDesktopFrame(clientId: string, bytes: Uint8Array, header
const result = broadcastFrameToViewers(sessions, buf, header);
if (result.sent) {
const controller = sessions.find((session) =>
result.sentViewers.has(session.viewer) &&
(session.viewer.data as any).rdCanvasFlowControl === true &&
(session.viewer.data as any).rdCanvasBackpressure === true
);
if (controller?.viewer.data.sessionId) {
rdCanvasFrameAckPending.set(clientId, controller.viewer.data.sessionId);
const sessionId = controller.viewer.data.sessionId;
const pending = rdCanvasFrameAckPending.get(clientId);
rdCanvasFrameAckPending.set(clientId, {
sessionId,
count: pending?.sessionId === sessionId ? pending.count + 1 : 1,
});
}
}
if (result.dropped) {
const target = clientManager.getClient(clientId);
if (target) {
const now = Date.now();
const lastRequestAt = rdBackpressureKeyframeAt.get(clientId) || 0;
if (target && now - lastRequestAt >= RD_BACKPRESSURE_KEYFRAME_INTERVAL_MS) {
rdBackpressureKeyframeAt.set(clientId, now);
sendDesktopCommand(target, "desktop_request_keyframe", {
reason: "viewer_backpressure",
format: String(header?.format || ""),
});
}
}
return (result.sent && !rdCanvasFrameAckPending.has(clientId)) || result.viewers === 0;
return ((result.sent || result.dropped) && !rdCanvasFrameAckPending.has(clientId)) || result.viewers === 0;
}
(globalThis as any).__rdBroadcast = (clientId: string, bytes: Uint8Array, header?: any): boolean => {
@@ -1588,13 +1602,16 @@ export function handleDesktopEncoderCapabilities(clientId: string, payload: any)
}
function releaseCanvasFrameAck(clientId: string, sessionId?: string): boolean {
const controllerSessionId = rdCanvasFrameAckPending.get(clientId);
if (!controllerSessionId || (sessionId && controllerSessionId !== sessionId)) return false;
const pending = rdCanvasFrameAckPending.get(clientId);
if (!pending || (sessionId && pending.sessionId !== sessionId)) return false;
rdCanvasFrameAckPending.delete(clientId);
const target = clientManager.getClient(clientId);
if (!target) return false;
try {
target.ws.send(encodeMessage({ type: "frame_ack" }));
const ack = encodeMessage({ type: "frame_ack" });
for (let i = 0; i < pending.count; i++) {
target.ws.send(ack);
}
return true;
} catch (err) {
logger.debug(`[rd] canvas frame ack failed client=${clientId}: ${(err as Error).message}`);
@@ -0,0 +1,118 @@
import { expect, test } from "@playwright/test";
import { login } from "./helpers";
test("Canvas presents only the latest decoded video frame per browser refresh", async ({ page }) => {
await login(page);
await page.addInitScript(() => {
const counters = { decoded: 0, drawn: 0, closed: 0 };
Object.defineProperty(window, "__rdPresentationCounters", { value: counters, configurable: true });
class FakeWebSocket extends EventTarget {
static readonly CONNECTING = 0;
static readonly OPEN = 1;
static readonly CLOSING = 2;
static readonly CLOSED = 3;
readonly CONNECTING = 0;
readonly OPEN = 1;
readonly CLOSING = 2;
readonly CLOSED = 3;
readyState = FakeWebSocket.OPEN;
bufferedAmount = 0;
binaryType: BinaryType = "blob";
constructor(_url: string | URL) {
super();
Object.defineProperty(window, "__rdSocket", { value: this, configurable: true });
queueMicrotask(() => this.dispatchEvent(new Event("open")));
}
send() {}
close() {
this.readyState = FakeWebSocket.CLOSED;
this.dispatchEvent(new CloseEvent("close"));
}
}
class FakeVideoDecoder extends EventTarget {
static async isConfigSupported(config: VideoDecoderConfig) {
return { supported: true, config };
}
decodeQueueSize = 0;
private output: (frame: VideoFrame) => void;
constructor(init: VideoDecoderInit) {
super();
this.output = init.output;
}
configure() {}
decode() {
counters.decoded += 1;
const frame = {
displayWidth: 1920,
displayHeight: 1080,
codedWidth: 1920,
codedHeight: 1080,
close() {
counters.closed += 1;
},
} as unknown as VideoFrame;
this.output(frame);
}
close() {}
}
class FakeEncodedVideoChunk {
constructor(_init: EncodedVideoChunkInit) {}
}
const originalGetContext = HTMLCanvasElement.prototype.getContext;
HTMLCanvasElement.prototype.getContext = function (contextId: string, ...args: unknown[]) {
if (this.id === "frameCanvas" && contextId === "2d") {
return {
drawImage() {
counters.drawn += 1;
},
putImageData() {},
} as unknown as CanvasRenderingContext2D;
}
return originalGetContext.call(this, contextId as "2d", ...args as []) as CanvasRenderingContext2D;
} as typeof HTMLCanvasElement.prototype.getContext;
Object.defineProperty(window, "WebSocket", { value: FakeWebSocket, configurable: true });
Object.defineProperty(window, "VideoDecoder", { value: FakeVideoDecoder, configurable: true });
Object.defineProperty(window, "EncodedVideoChunk", { value: FakeEncodedVideoChunk, configurable: true });
});
await page.goto("/remotedesktop?clientId=canvas-presentation-test");
await page.evaluate(async () => {
const packet = new Uint8Array(18);
packet.set([0x46, 0x52, 0x4d, 2, 0, 240, 5, 0]);
new DataView(packet.buffer).setUint16(8, 1920, true);
new DataView(packet.buffer).setUint16(10, 1080, true);
packet.set([0, 0, 0, 1, 38, 1], 12);
const socket = (window as typeof window & { __rdSocket: EventTarget }).__rdSocket;
for (let i = 0; i < 4; i++) {
socket.dispatchEvent(new MessageEvent("message", { data: packet.buffer.slice(0) }));
await Promise.resolve();
await Promise.resolve();
}
});
await expect.poll(() => page.evaluate(() => (
window as typeof window & {
__rdPresentationCounters: { decoded: number; drawn: number; closed: number };
}
).__rdPresentationCounters)).toEqual({
decoded: 4,
drawn: 1,
closed: 4,
});
await expect(page.locator("#diagDropped")).toHaveText("0 / 3");
});
@@ -219,9 +219,7 @@ test("WebRTC input stays inactive until start and then forwards mouse and keyboa
const { decodeMsgpack } = await import("/assets/msgpack-helpers.js");
return sent.map((message) => decodeMsgpack(message))
.filter((message) => message.type === "desktop_request_keyframe");
})).toEqual([
expect.objectContaining({ type: "desktop_request_keyframe", reason: "viewer_frame_gap" }),
]);
})).toEqual([]);
await page.locator("#rdSettingsBtn").click();
await page.locator("#requestKeyframeBtn").click();
await expect.poll(async () => page.evaluate(async () => {