vxaboveground/overlord
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

major remote desktop upgrades + do opus. Hopefully all the tests pass 😭

Browse Source
This commit is contained in:
KingKDot
2026-07-17 04:28:23 -05:00
parent 779f86b259
commit 0219081682
35 changed files with 1304 additions and 91 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Overlord-Server/public/assets/backstage.js
+26
View File
@@ -69,9 +69,29 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
}, reconnectDelay);
}
const latencyEl = document.getElementById("latencyDisplay");
const networkStats = document.getElementById("networkStats");
let lastInputSentAt = 0;
let lastLatencyMs = 0;
function updateNetworkStats(stats) {
if (!networkStats || !stats) return;
const media = stats.video || stats.audio;
const parts = [];
if (Number.isFinite(media?.bitrateMbps)) parts.push(`${media.bitrateMbps.toFixed(1)} Mbps`);
if (Number.isFinite(stats.rttMs)) parts.push(`${Math.round(stats.rttMs)} ms`);
if (Number.isFinite(media?.lossPercent)) parts.push(`${media.lossPercent.toFixed(1)}% loss`);
const route = [stats.protocol, stats.route].filter(Boolean).join("/");
if (route) parts.push(route);
networkStats.textContent = parts.join(" · ") || "Connected";
const details = [];
if (media?.codec) details.push(`Codec: ${media.codec}`);
if (media?.width && media?.height) details.push(`Video: ${media.width}×${media.height}${media.framesPerSecond ? ` @ ${Math.round(media.framesPerSecond)} FPS` : ""}`);
if (Number.isFinite(media?.jitterMs)) details.push(`Jitter: ${media.jitterMs.toFixed(1)} ms`);
if (Number.isFinite(media?.jitterBufferMs)) details.push(`Jitter buffer: ${media.jitterBufferMs.toFixed(1)} ms`);
if (Number.isFinite(media?.framesDropped)) details.push(`Frames dropped: ${media.framesDropped}`);
networkStats.title = details.join("\n");
}
const displaySelect = document.getElementById("displaySelect");
const refreshBtn = document.getElementById("refreshDisplays");
const startBtn = document.getElementById("startBtn");
@@ -771,6 +791,10 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
webrtcActive = !!active;
if (canvas) canvas.style.display = active ? "none" : "block";
if (webrtcVideo) webrtcVideo.style.display = active ? "block" : "none";
if (!active && networkStats) {
networkStats.textContent = "--";
networkStats.title = "";
}
}
function onWebrtcState(label, state) {
@@ -823,6 +847,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
whepPath,
videoEl: webrtcVideo,
onState: (state) => onWebrtcState("WebRTC Relayed", state),
onStats: (stats) => updateNetworkStats({ ...stats, route: "Server" }),
});
try {
await whepClient.start();
@@ -846,6 +871,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
}
},
onState: (state) => onWebrtcState("WebRTC P2P", state),
onStats: updateNetworkStats,
});
try {
await p2pClient.start();
Overlord-Server/public/assets/remotedesktop.js
+360 -35
View File
@@ -65,6 +65,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
const duplicationCtrl = document.getElementById("duplicationCtrl");
const streamProfileSelect = document.getElementById("streamProfileSelect");
const streamProfileDetail = document.getElementById("streamProfileDetail");
const bitrateSelect = document.getElementById("bitrateSelect");
const smoothingSlider = document.getElementById("smoothingSlider");
const smoothingValue = document.getElementById("smoothingValue");
const qualitySlider = document.getElementById("qualitySlider");
@@ -78,6 +79,14 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
const agentFps = document.getElementById("agentFps");
const viewerFps = document.getElementById("viewerFps");
const inputLatency = document.getElementById("inputLatency");
const networkStats = document.getElementById("networkStats");
const diagnosticsBtn = document.getElementById("diagnosticsBtn");
const diagnosticsHud = document.getElementById("diagnosticsHud");
const diagnosticsClose = document.getElementById("diagnosticsClose");
const diagnosticEls = Object.fromEntries([
"Summary", "Transport", "Codec", "Resolution", "Capture", "Encode", "Send", "AgentTotal",
"Bitrate", "Rtt", "LossJitter", "JitterBuffer", "Decode", "Render", "Queue", "Dropped", "Fps", "Input",
].map((name) => [name, document.getElementById(`diag${name}`)]));
const statusEl = document.getElementById("streamStatus");
const clipboardSyncCtrl = document.getElementById("clipboardSyncCtrl");
const privacyCtrl = document.getElementById("privacyCtrl");
@@ -111,6 +120,10 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
let frameDecodeBusy = false;
let pendingFrame = null;
let videoDecoder = null;
let videoDecoderConfigKey = "";
let h264StreamCodec = "";
let canvasDecodePressure = false;
let h264PendingTimings = [];
let h264TimestampUs = 0;
let prefersH264 = typeof VideoDecoder === "function";
let h264LowFpsStreak = 0;
@@ -129,6 +142,22 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
const mouseMoveIntervalMs = 33;
const inputBackpressureBytes = 256 * 1024;
let lastMoveSentAt = 0;
const diagnostics = {
agent: null,
network: null,
decodeMs: null,
renderMs: null,
decodeQueue: 0,
coalescedFrames: 0,
currentAgentFps: null,
currentViewerFps: null,
wsBitrateMbps: null,
wsBytes: 0,
wsWindowStartedAt: performance.now(),
codec: "",
width: 0,
height: 0,
};
let clipboardSyncTimer = null;
let lastClipboardText = "";
@@ -149,6 +178,9 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
function resetH264RuntimeState() {
h264TimestampUs = 0;
h264StreamCodec = "";
h264PendingTimings = [];
diagnostics.decodeQueue = 0;
h264LowFpsStreak = 0;
h264FirstFrameAt = 0;
h264FramesSeen = 0;
@@ -396,6 +428,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
? `${settings.resolution}:${settings.targetFps}`
: savedStreamProfile);
setSelectValue(streamProfileSelect, savedStreamProfile);
setSelectValue(bitrateSelect, settings.bitrateMbps);
setSelectValue(webrtcMode, settings.webrtcMode);
setSelectValue(audioTransport, settings.audioTransport);
setSelectValue(recordMode, settings.recordMode);
@@ -428,6 +461,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
resolution: String(profile.maxHeight),
targetFps: String(profile.fps),
quality: Number(qualitySlider?.value || 90),
bitrateMbps: Number(bitrateSelect?.value || 0),
preferH264: !!prefersH264,
webrtcMode: getWebrtcMode(),
mouse: !!mouseCtrl?.checked,
@@ -469,6 +503,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
function updateFpsDisplay(agentValue) {
if (agentValue !== undefined && agentValue !== null && agentFps) {
agentFps.textContent = String(agentValue);
diagnostics.currentAgentFps = Number(agentValue) || null;
}
const now = performance.now();
renderCount += 1;
@@ -476,6 +511,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
if (elapsed >= 1000 && viewerFps) {
const fps = Math.round((renderCount * 1000) / elapsed);
viewerFps.textContent = String(fps);
diagnostics.currentViewerFps = fps;
renderCount = 0;
renderWindowStart = now;
}
@@ -827,6 +863,142 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
}
}
function finite(value) {
return Number.isFinite(value) ? Number(value) : null;
}
function smoothed(previous, next, weight = 0.25) {
const value = finite(next);
if (value == null) return previous;
return previous == null ? value : previous * (1 - weight) + value * weight;
}
function msText(value) {
const number = finite(value);
return number == null ? "-- ms" : `${number < 10 ? number.toFixed(1) : Math.round(number)} ms`;
}
function setDiagnosticsVisible(visible) {
if (!diagnosticsHud) return;
diagnosticsHud.classList.toggle("hidden", !visible);
diagnosticsBtn?.setAttribute("aria-pressed", visible ? "true" : "false");
try { localStorage.setItem("overlord.rd.statsHud", visible ? "1" : "0"); } catch {}
}
function renderDiagnostics() {
if (!diagnosticsHud) return;
const agent = diagnostics.agent || {};
const network = diagnostics.network || {};
const media = network.video || network.audio || {};
const mode = getWebrtcMode();
const transport = mode === "p2p" ? "WebRTC P2P" : mode === "relayed" ? "WebRTC server" : "Canvas WebSocket";
const bitrate = finite(media.bitrateMbps) ?? finite(diagnostics.wsBitrateMbps);
const decodeMs = finite(media.decodeMs) ?? finite(diagnostics.decodeMs);
const renderMs = finite(media.processingDelayMs) ?? finite(diagnostics.renderMs);
const dropped = finite(media.framesDropped) ?? 0;
const queue = mode === "off" ? diagnostics.decodeQueue : null;
const fps = finite(agent.fps) ?? diagnostics.currentAgentFps;
const viewerRate = finite(media.framesPerSecond) ?? diagnostics.currentViewerFps;
const frameBudget = 1000 / Math.max(1, fps || viewerRate || 30);
let summary = "Pipeline healthy";
let severity = "ok";
if (streamState !== "streaming" && streamState !== "stalled") {
summary = "Stream is not active";
severity = "warn";
} else if (finite(agent.captureMs) > frameBudget * 0.75) {
summary = "Capture is limiting frame rate";
severity = "bad";
} else if (finite(agent.encodeMs) > frameBudget * 0.75) {
summary = "Encoder is limiting frame rate";
severity = "bad";
} else if (decodeMs != null && decodeMs > frameBudget * 0.75) {
const viewerKeepingUp = fps != null && viewerRate != null && viewerRate >= fps * 0.9;
summary = viewerKeepingUp
? `Decoder buffering adds ${Math.round(decodeMs)} ms`
: "Viewer decoder throughput is limiting FPS";
severity = viewerKeepingUp ? "warn" : "bad";
} else if ((finite(media.lossPercent) ?? 0) > 3 || (finite(network.rttMs) ?? 0) > 150 || (finite(media.jitterMs) ?? 0) > 35) {
summary = "Network conditions are causing delay";
severity = "bad";
} else if ((queue ?? 0) > 2 || (renderMs != null && renderMs > frameBudget)) {
summary = "Viewer render queue is backing up";
severity = "warn";
} else if (!diagnostics.agent && !diagnostics.network) {
summary = "Waiting for stream telemetry";
severity = "warn";
}
diagnosticEls.Summary.textContent = summary;
diagnosticEls.Summary.dataset.severity = severity;
diagnosticEls.Transport.textContent = transport;
diagnosticEls.Codec.textContent = String(media.codec || agent.format || diagnostics.codec || "--").toUpperCase();
const width = finite(media.width) ?? diagnostics.width;
const height = finite(media.height) ?? diagnostics.height;
diagnosticEls.Resolution.textContent = width && height ? `${width}×${height}` : "--";
diagnosticEls.Capture.textContent = msText(agent.captureMs);
diagnosticEls.Encode.textContent = msText(agent.encodeMs);
diagnosticEls.Send.textContent = msText(agent.sendMs);
diagnosticEls.AgentTotal.textContent = msText(agent.totalMs);
diagnosticEls.Bitrate.textContent = bitrate == null ? "-- Mbps" : `${bitrate.toFixed(2)} Mbps`;
diagnosticEls.Rtt.textContent = msText(network.rttMs);
diagnosticEls.LossJitter.textContent = `${finite(media.lossPercent)?.toFixed(1) ?? "--"}% / ${msText(media.jitterMs)}`;
diagnosticEls.JitterBuffer.textContent = msText(media.jitterBufferMs);
diagnosticEls.Decode.textContent = msText(decodeMs);
diagnosticEls.Render.textContent = msText(renderMs);
diagnosticEls.Queue.textContent = queue == null ? "managed by browser" : String(queue);
diagnosticEls.Dropped.textContent = `${Math.round(dropped)} / ${diagnostics.coalescedFrames}`;
diagnosticEls.Fps.textContent = `${fps == null ? "--" : Math.round(fps)} → ${viewerRate == null ? "--" : Math.round(viewerRate)}`;
diagnosticEls.Input.textContent = msText(latencyAvg);
}
diagnosticsBtn?.addEventListener("click", () => setDiagnosticsVisible(diagnosticsHud?.classList.contains("hidden")));
diagnosticsClose?.addEventListener("click", () => setDiagnosticsVisible(false));
networkStats?.addEventListener("click", () => setDiagnosticsVisible(true));
try { setDiagnosticsVisible(localStorage.getItem("overlord.rd.statsHud") === "1"); } catch {}
setInterval(renderDiagnostics, 250);
function updateNetworkStats(stats) {
if (!networkStats || !stats) return;
diagnostics.network = stats;
const media = stats.video || stats.audio;
const parts = [];
if (Number.isFinite(media?.bitrateMbps)) parts.push(`${media.bitrateMbps.toFixed(1)} Mbps`);
if (Number.isFinite(stats.rttMs)) parts.push(`${Math.round(stats.rttMs)} ms`);
if (Number.isFinite(media?.lossPercent)) parts.push(`${media.lossPercent.toFixed(1)}% loss`);
const route = [stats.protocol, stats.route].filter(Boolean).join("/");
if (route) parts.push(route);
networkStats.textContent = parts.join(" · ") || "Connected";
const details = [];
if (media?.codec) details.push(`Codec: ${media.codec}`);
if (media?.width && media?.height) details.push(`Video: ${media.width}×${media.height}${media.framesPerSecond ? ` @ ${Math.round(media.framesPerSecond)} FPS` : ""}`);
if (Number.isFinite(media?.jitterMs)) details.push(`Jitter: ${media.jitterMs.toFixed(1)} ms`);
if (Number.isFinite(media?.jitterBufferMs)) details.push(`Jitter buffer: ${media.jitterBufferMs.toFixed(1)} ms`);
if (Number.isFinite(media?.framesDropped)) details.push(`Frames dropped: ${media.framesDropped}`);
if (Number.isFinite(stats.availableIncomingMbps)) details.push(`Available: ${stats.availableIncomingMbps.toFixed(1)} Mbps`);
networkStats.title = details.join("\n");
renderDiagnostics();
}
function handleDesktopStreamStats(stats) {
const previous = diagnostics.agent || {};
diagnostics.agent = {
...stats,
captureMs: smoothed(finite(previous.captureMs), stats.captureMs),
encodeMs: smoothed(finite(previous.encodeMs), stats.encodeMs),
sendMs: smoothed(finite(previous.sendMs), stats.sendMs),
totalMs: smoothed(finite(previous.totalMs), stats.totalMs),
};
diagnostics.currentAgentFps = finite(stats.fps);
diagnostics.codec = String(stats.format || diagnostics.codec || "");
diagnostics.width = finite(stats.width) || diagnostics.width;
diagnostics.height = finite(stats.height) || diagnostics.height;
if (agentFps && diagnostics.currentAgentFps != null) {
agentFps.textContent = String(Math.round(diagnostics.currentAgentFps));
}
renderDiagnostics();
}
function clearOfflineTimer() {
if (offlineTimer) {
clearTimeout(offlineTimer);
@@ -919,10 +1091,10 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
if (mode === "relayed") {
sendCmd("desktop_start", { webrtc: true });
} else if (mode === "p2p") {
sendCmd("desktop_start", {});
sendCmd("desktop_start", { canvasFlowControl: false });
startP2P();
} else {
sendCmd("desktop_start", {});
sendCmd("desktop_start", { canvasFlowControl: true });
}
}
} else {
@@ -1043,17 +1215,19 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
return;
}
const msg = { type, ...payload };
rdDebug("send", {
msg,
readyState: ws.readyState,
streamState,
desiredStreaming,
display: displaySelect?.value ?? "",
quality: qualitySlider?.value ?? "",
streamProfile: streamProfileSelect?.value ?? "",
transport: getWebrtcMode(),
lastFrameAgeMs: lastFrameAt ? Math.round(performance.now() - lastFrameAt) : null,
});
if (type !== "desktop_decode_pressure") {
rdDebug("send", {
msg,
readyState: ws.readyState,
streamState,
desiredStreaming,
display: displaySelect?.value ?? "",
quality: qualitySlider?.value ?? "",
streamProfile: streamProfileSelect?.value ?? "",
transport: getWebrtcMode(),
lastFrameAgeMs: lastFrameAt ? Math.round(performance.now() - lastFrameAt) : null,
});
}
ws.send(encodeMsgpack(msg));
}
@@ -1061,6 +1235,11 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
webrtcActive = !!active;
if (canvas) canvas.style.display = active ? "none" : "block";
if (webrtcVideo) webrtcVideo.style.display = active ? "block" : "none";
if (!active && networkStats) {
networkStats.textContent = "--";
networkStats.title = "";
diagnostics.network = null;
}
}
function onWebrtcState(label, s) {
@@ -1084,15 +1263,22 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
stopWebrtcFrameTicker();
webrtcFpsCount = 0;
webrtcFpsWindowStart = performance.now();
const tick = (now) => {
const tick = (now, metadata = {}) => {
markFrameReceived();
if (Number.isFinite(metadata.processingDuration)) {
diagnostics.decodeMs = smoothed(diagnostics.decodeMs, metadata.processingDuration * 1000);
}
if (Number.isFinite(metadata.receiveTime) && Number.isFinite(metadata.expectedDisplayTime)) {
diagnostics.renderMs = smoothed(diagnostics.renderMs, Math.max(0, metadata.expectedDisplayTime - metadata.receiveTime));
}
diagnostics.width = webrtcVideo.videoWidth || diagnostics.width;
diagnostics.height = webrtcVideo.videoHeight || diagnostics.height;
webrtcFpsCount += 1;
const elapsed = now - webrtcFpsWindowStart;
if (elapsed >= 1000) {
const fps = Math.round((webrtcFpsCount * 1000) / elapsed);
// Agent and viewer FPS are effectively equal in WebRTC mode — the
// browser renders every frame the agent sends, no decode skipping.
updateFpsDisplay(fps);
diagnostics.currentViewerFps = fps;
updateFpsDisplay();
webrtcFpsCount = 0;
webrtcFpsWindowStart = now;
} else {
@@ -1116,6 +1302,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
whepPath,
videoEl: webrtcVideo,
onState: (s) => onWebrtcState("WebRTC Relayed", s),
onStats: (stats) => updateNetworkStats({ ...stats, route: "Server" }),
});
try {
await whepClient.start();
@@ -1139,6 +1326,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
}
},
onState: (s) => onWebrtcState("WebRTC P2P", s),
onStats: updateNetworkStats,
});
try {
await p2pClient.start();
@@ -1306,6 +1494,18 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
sendCmd("desktop_set_profile", profile);
}
function pushBitrate() {
const bitrateMbps = Math.max(0, Math.min(50, Number.parseInt(bitrateSelect?.value || "0", 10) || 0));
sendCmd("desktop_set_bitrate", { bitrateMbps });
}
if (bitrateSelect) {
bitrateSelect.addEventListener("change", function () {
pushBitrate();
sharedSettingsSaver.scheduleSave();
});
}
function requestEncoderCapabilities() {
if (streamProfileDetail) streamProfileDetail.textContent = "Checking hardware encoder profiles…";
sendCmd("desktop_encoder_capabilities", {
@@ -1419,6 +1619,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
ensureDuplicationForH264();
}
pushStreamProfile();
pushBitrate();
desiredStreaming = true;
lastFrameAt = 0;
firstFrameLogged = false;
@@ -1429,10 +1630,10 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
sendCmd("desktop_start", { webrtc: true });
} else if (mode === "p2p") {
// Kick off the P2P offer asynchronously; capture starts in parallel.
sendCmd("desktop_start", {});
sendCmd("desktop_start", { canvasFlowControl: false });
startP2P();
} else {
sendCmd("desktop_start", {});
sendCmd("desktop_start", { canvasFlowControl: true });
}
if (audioCtrl && audioCtrl.checked) {
connectAudio();
@@ -1644,6 +1845,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
}
function destroyVideoDecoder() {
updateCanvasDecodePressure(true);
if (!videoDecoder) return;
try {
videoDecoder.close();
@@ -1651,6 +1853,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
// Ignore close errors when decoder is already shutting down.
}
videoDecoder = null;
videoDecoderConfigKey = "";
resetH264RuntimeState();
}
@@ -1724,6 +1927,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
sendCmd("desktop_start", {
source: "rd_viewer",
reason: "h264_recovery_restart",
canvasFlowControl: getWebrtcMode() === "off",
});
const q = Number(qualitySlider?.value) || 90;
sendCmd("desktop_set_quality", {
@@ -1769,16 +1973,39 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
return false;
}
function ensureVideoDecoder() {
if (videoDecoder) {
function h264CodecFromAnnexB(data) {
for (let i = 0; i + 7 < data.length; i++) {
let startCodeLength = 0;
if (data[i] === 0 && data[i + 1] === 0 && data[i + 2] === 1) startCodeLength = 3;
else if (data[i] === 0 && data[i + 1] === 0 && data[i + 2] === 0 && data[i + 3] === 1) startCodeLength = 4;
if (!startCodeLength) continue;
const nal = i + startCodeLength;
if ((data[nal] & 0x1f) !== 7 || nal + 3 >= data.length) continue;
return `avc1.${[data[nal + 1], data[nal + 2], data[nal + 3]]
.map((value) => value.toString(16).padStart(2, "0"))
.join("")}`;
}
return "";
}
function ensureVideoDecoder(h264Bytes, width, height) {
const detectedCodec = h264CodecFromAnnexB(h264Bytes);
const codec = detectedCodec || h264StreamCodec || "avc1.4d0034";
const codedWidth = Math.max(0, Number(width) || 0);
const codedHeight = Math.max(0, Number(height) || 0);
const configKey = `${codec}:${codedWidth}x${codedHeight}`;
if (videoDecoder && videoDecoderConfigKey === configKey) {
return true;
}
if (typeof VideoDecoder !== "function") {
return false;
}
try {
if (videoDecoder) destroyVideoDecoder();
videoDecoder = new VideoDecoder({
output: (frame) => {
const outputStartedAt = performance.now();
const timing = h264PendingTimings.shift();
const width = frame.displayWidth || frame.codedWidth || frameWidth;
const height = frame.displayHeight || frame.codedHeight || frameHeight;
if (width > 0 && height > 0 && (canvas.width !== width || canvas.height !== height)) {
@@ -1792,12 +2019,34 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
} finally {
frame.close();
}
const renderedAt = performance.now();
if (timing) {
diagnostics.decodeMs = smoothed(diagnostics.decodeMs, Math.max(0, outputStartedAt - timing.decodeStartedAt));
diagnostics.renderMs = smoothed(diagnostics.renderMs, Math.max(0, renderedAt - timing.receivedAt));
}
diagnostics.decodeQueue = videoDecoder?.decodeQueueSize || h264PendingTimings.length;
diagnostics.width = width;
diagnostics.height = height;
updateFpsDisplay();
updateCanvasDecodePressure();
},
error: (err) => {
console.warn("rd: h264 decoder error", err);
updateCanvasDecodePressure(true);
},
});
videoDecoder.configure({ codec: "avc1.42E01E", optimizeForLatency: true });
videoDecoder.addEventListener("dequeue", () => {
diagnostics.decodeQueue = videoDecoder?.decodeQueueSize || 0;
updateCanvasDecodePressure();
});
const config = { codec, optimizeForLatency: true };
if (codedWidth > 0 && codedHeight > 0) {
config.codedWidth = codedWidth;
config.codedHeight = codedHeight;
}
videoDecoder.configure(config);
videoDecoderConfigKey = configKey;
h264StreamCodec = codec;
return true;
} catch (err) {
console.warn("rd: h264 decoder unavailable", err);
@@ -1807,22 +2056,72 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
}
}
async function processFrameBuffer(buf) {
function recordWsFrameBytes(byteLength) {
diagnostics.wsBytes += Math.max(0, Number(byteLength) || 0);
const now = performance.now();
const elapsed = now - diagnostics.wsWindowStartedAt;
if (elapsed >= 1000) {
const rate = (diagnostics.wsBytes * 8) / (elapsed * 1000);
diagnostics.wsBitrateMbps = smoothed(diagnostics.wsBitrateMbps, rate, 0.4);
diagnostics.wsBytes = 0;
diagnostics.wsWindowStartedAt = now;
}
}
function recordCanvasFrameTiming(receivedAt, decodeStartedAt) {
const renderedAt = performance.now();
diagnostics.decodeMs = smoothed(diagnostics.decodeMs, Math.max(0, renderedAt - decodeStartedAt));
diagnostics.renderMs = smoothed(diagnostics.renderMs, Math.max(0, renderedAt - receivedAt));
diagnostics.decodeQueue = videoDecoder?.decodeQueueSize || (pendingFrame ? 1 : 0);
diagnostics.width = frameWidth || canvas.width || diagnostics.width;
diagnostics.height = frameHeight || canvas.height || diagnostics.height;
}
function updateCanvasDecodePressure(forceRelease = false) {
const queueSize = videoDecoder?.decodeQueueSize || 0;
const shouldApply = !forceRelease && getWebrtcMode() === "off" && queueSize > 3;
const shouldRelease = forceRelease || getWebrtcMode() !== "off" || queueSize <= 1;
let next = canvasDecodePressure;
if (!canvasDecodePressure && shouldApply) next = true;
else if (canvasDecodePressure && shouldRelease) next = false;
if (next === canvasDecodePressure) return;
canvasDecodePressure = next;
sendCmd("desktop_decode_pressure", { active: next, queueSize });
}
async function processFrameBuffer(buf, receivedAt = performance.now()) {
const decodeStartedAt = performance.now();
const fps = buf[5];
const format = buf[6];
const headerLength = buf[3] >= 2 && buf.length >= 12 ? 12 : 8;
const encodedWidth = headerLength === 12 ? buf[8] | (buf[9] << 8) : 0;
const encodedHeight = headerLength === 12 ? buf[10] | (buf[11] << 8) : 0;
if (encodedWidth > 0 && encodedHeight > 0) {
frameWidth = encodedWidth;
frameHeight = encodedHeight;
diagnostics.width = encodedWidth;
diagnostics.height = encodedHeight;
}
if (format === 1) {
const jpegBytes = buf.slice(8);
const jpegBytes = buf.slice(headerLength);
setCodecModeLabel("jpeg", "active");
diagnostics.codec = "jpeg";
await drawJpegSlice(jpegBytes, null);
recordCanvasFrameTiming(receivedAt, decodeStartedAt);
updateFpsDisplay(fps);
updateCanvasDecodePressure(true);
return;
}
if (format === 2 || format === 3) {
setCodecModeLabel(format === 3 ? "raw" : "jpeg", format === 3 ? "blocks" : "blocks");
if (buf.length < 16) return;
const dv = new DataView(buf.buffer, 8);
diagnostics.codec = format === 3 ? "raw blocks" : "jpeg blocks";
if (buf.length < headerLength + 8) {
updateCanvasDecodePressure(true);
return;
}
const dv = new DataView(buf.buffer, buf.byteOffset + headerLength, buf.byteLength - headerLength);
let pos = 0;
const width = dv.getUint16(pos, true);
pos += 2;
@@ -1857,7 +2156,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
pos += 2;
const len = dv.getUint32(pos, true);
pos += 4;
const start = 8 + pos;
const start = headerLength + pos;
const end = start + len;
if (end > buf.length) break;
const slice = buf.subarray(start, end);
@@ -1872,15 +2171,22 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
}
updateFpsDisplay(fps);
recordCanvasFrameTiming(receivedAt, decodeStartedAt);
updateCanvasDecodePressure(true);
return;
}
if (format === 4) {
setCodecModeLabel("h264", "active");
const h264Bytes = buf.slice(8);
if (!h264Bytes.length) return;
if (!ensureVideoDecoder()) {
diagnostics.codec = "h264";
const h264Bytes = buf.slice(headerLength);
if (!h264Bytes.length) {
updateCanvasDecodePressure(true);
return;
}
if (!ensureVideoDecoder(h264Bytes, encodedWidth || frameWidth, encodedHeight || frameHeight)) {
fallbackToJpegCodec("WebCodecs decoder unavailable");
updateCanvasDecodePressure(true);
return;
}
@@ -1905,6 +2211,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
h264LowFpsStreak >= H264_LOW_FPS_STREAK_LIMIT
) {
fallbackToJpegCodec(`low h264 fps (${fps})`);
updateCanvasDecodePressure(true);
return;
}
@@ -1916,10 +2223,13 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
});
h264TimestampUs += frameIntervalUs;
try {
h264PendingTimings.push({ receivedAt, decodeStartedAt: performance.now() });
videoDecoder.decode(chunk);
diagnostics.decodeQueue = videoDecoder.decodeQueueSize;
updateCanvasDecodePressure();
h264KeyframeErrorStreak = 0;
updateFpsDisplay(fps);
} catch (err) {
h264PendingTimings.pop();
if (isKeyframeRequiredError(err)) {
h264KeyframeErrorStreak += 1;
const now = Date.now();
@@ -1936,12 +2246,16 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
fallbackToJpegCodec("h264_keyframe_required_loop");
}
}
updateCanvasDecodePressure(true);
return;
}
console.warn("rd: h264 decode failed", err);
fallbackToJpegCodec(err);
updateCanvasDecodePressure(true);
}
return;
}
updateCanvasDecodePressure(true);
}
function flushPendingFrame() {
@@ -1951,7 +2265,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
const next = pendingFrame;
pendingFrame = null;
frameDecodeBusy = true;
processFrameBuffer(next).finally(() => {
processFrameBuffer(next.buf, next.receivedAt).finally(() => {
frameDecodeBusy = false;
if (pendingFrame) {
flushPendingFrame();
@@ -1964,8 +2278,10 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
const buf = new Uint8Array(ev.data);
if (isFramePacket(buf)) {
markFrameReceived();
recordWsFrameBytes(buf.byteLength);
// Coalesce bursty arrivals so the renderer catches up to the newest frame.
pendingFrame = buf;
if (pendingFrame) diagnostics.coalescedFrames += 1;
pendingFrame = { buf, receivedAt: performance.now() };
flushPendingFrame();
return;
}
@@ -1975,6 +2291,10 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
applyEncoderCapabilities(msg);
return;
}
if (msg && msg.type === "desktop_stream_stats") {
handleDesktopStreamStats(msg);
return;
}
if (msg && msg.type === "status" && msg.status) {
handleStatus(msg);
return;
@@ -2014,6 +2334,10 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
applyEncoderCapabilities(msg);
return;
}
if (msg && msg.type === "desktop_stream_stats") {
handleDesktopStreamStats(msg);
return;
}
if (msg && msg.type === "status" && msg.status) {
handleStatus(msg);
return;
@@ -2045,6 +2369,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
if (qualitySlider) {
pushQuality(qualitySlider.value);
}
pushBitrate();
pushInputToggles();
pushCaptureToggles();
clearOfflineTimer();
@@ -2060,10 +2385,10 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
if (mode === "relayed") {
sendCmd("desktop_start", { webrtc: true });
} else if (mode === "p2p") {
sendCmd("desktop_start", {});
sendCmd("desktop_start", { canvasFlowControl: false });
startP2P();
} else {
sendCmd("desktop_start", {});
sendCmd("desktop_start", { canvasFlowControl: true });
}
} else {
setStreamState("idle", "Stopped");
Overlord-Server/public/assets/webrtc-p2p.js
+10
View File
@@ -10,6 +10,8 @@
// The server proxies these to the agent via its existing WS command channel
// (sessionId is server-side state, not visible to the browser).
//
import { WebRTCStatsSampler } from "./webrtc-stats.js";
// Once SDP + ICE settle, the <video> element receives frames directly from
// the agent — MediaMTX is not involved.
@@ -20,6 +22,7 @@ export class P2PClient {
* @param {HTMLAudioElement} [opts.audioEl]
* @param {(msg: object) => void} opts.send Send a JSON-encodable msg over the viewer's WS.
* @param {(state: string) => void} [opts.onState]
* @param {(stats: object) => void} [opts.onStats]
* @param {string[]} [opts.iceServers] STUN URLs. Default: Google public STUN.
*/
constructor(opts) {
@@ -27,9 +30,11 @@ export class P2PClient {
this.audioEl = opts.audioEl || null;
this.send = opts.send;
this.onState = opts.onState || (() => {});
this.onStats = opts.onStats || (() => {});
this.iceServers = opts.iceServers || ["stun:stun.l.google.com:19302"];
this.pc = null;
this.pendingRemoteCandidates = [];
this.statsSampler = null;
}
async start() {
@@ -39,6 +44,7 @@ export class P2PClient {
iceServers: this.iceServers.map((u) => ({ urls: u })),
});
this.pc = pc;
this.statsSampler = new WebRTCStatsSampler(pc, this.onStats);
if (this.videoEl) pc.addTransceiver("video", { direction: "recvonly" });
if (this.audioEl) pc.addTransceiver("audio", { direction: "recvonly" });
@@ -66,6 +72,8 @@ export class P2PClient {
};
pc.onconnectionstatechange = () => {
this.onState(pc.connectionState);
if (pc.connectionState === "connected") this.statsSampler?.start();
if (["failed", "closed", "disconnected"].includes(pc.connectionState)) this.statsSampler?.stop();
};
const offer = await pc.createOffer();
@@ -105,6 +113,8 @@ export class P2PClient {
async stop() {
const pc = this.pc;
this.pc = null;
this.statsSampler?.stop();
this.statsSampler = null;
this.pendingRemoteCandidates = [];
if (pc) {
try { pc.close(); } catch {}
Overlord-Server/public/assets/webrtc-stats.js
+139
View File
@@ -0,0 +1,139 @@
// Shared receiver-side WebRTC telemetry for P2P and WHEP sessions.
// Values are deliberately normalized here so the viewer pages do not need to
// know about browser-specific RTCStatsReport relationships.
export class WebRTCStatsSampler {
constructor(pc, onStats, intervalMs = 1000) {
this.pc = pc;
this.onStats = typeof onStats === "function" ? onStats : () => {};
this.intervalMs = intervalMs;
this.timer = null;
this.running = false;
this.previous = new Map();
}
start() {
if (this.timer || !this.pc) return;
this.sample();
this.timer = setInterval(() => this.sample(), this.intervalMs);
}
stop() {
if (this.timer) clearInterval(this.timer);
this.timer = null;
this.running = false;
this.previous.clear();
}
async sample() {
if (this.running || !this.pc || this.pc.connectionState === "closed") return;
this.running = true;
try {
const report = await this.pc.getStats();
const byId = new Map();
report.forEach((item) => byId.set(item.id, item));
const result = {
timestamp: performance.now(),
rttMs: null,
availableIncomingMbps: null,
protocol: "",
route: "",
video: null,
audio: null,
};
let selectedPair = null;
for (const item of byId.values()) {
if (item.type === "transport" && item.selectedCandidatePairId) {
selectedPair = byId.get(item.selectedCandidatePairId) || selectedPair;
}
if (item.type === "candidate-pair" && item.state === "succeeded" && (item.nominated || item.selected)) {
selectedPair = selectedPair || item;
}
}
if (selectedPair) {
if (Number.isFinite(selectedPair.currentRoundTripTime)) {
result.rttMs = selectedPair.currentRoundTripTime * 1000;
}
if (Number.isFinite(selectedPair.availableIncomingBitrate)) {
result.availableIncomingMbps = selectedPair.availableIncomingBitrate / 1_000_000;
}
const local = byId.get(selectedPair.localCandidateId);
const remote = byId.get(selectedPair.remoteCandidateId);
result.protocol = String(local?.protocol || remote?.protocol || "").toUpperCase();
result.route = summarizeRoute(local?.candidateType, remote?.candidateType);
}
for (const item of byId.values()) {
if (item.type !== "inbound-rtp" || item.isRemote) continue;
const kind = item.kind || item.mediaType;
if (kind !== "video" && kind !== "audio") continue;
const codec = byId.get(item.codecId);
const previous = this.previous.get(item.id);
const timestamp = Number(item.timestamp) || performance.now();
let bitrateMbps = null;
let decodeMs = null;
let processingDelayMs = null;
let framesDroppedDelta = null;
if (previous && timestamp > previous.timestamp && Number(item.bytesReceived) >= previous.bytesReceived) {
bitrateMbps = ((Number(item.bytesReceived) - previous.bytesReceived) * 8) /
((timestamp - previous.timestamp) * 1000);
const decodedDelta = (Number(item.framesDecoded) || 0) - previous.framesDecoded;
if (decodedDelta > 0) {
const decodeDelta = (Number(item.totalDecodeTime) || 0) - previous.totalDecodeTime;
const processingDelta = (Number(item.totalProcessingDelay) || 0) - previous.totalProcessingDelay;
if (decodeDelta >= 0) decodeMs = (decodeDelta * 1000) / decodedDelta;
if (processingDelta >= 0) processingDelayMs = (processingDelta * 1000) / decodedDelta;
}
const droppedDelta = (Number(item.framesDropped) || 0) - previous.framesDropped;
if (droppedDelta >= 0) framesDroppedDelta = droppedDelta;
}
this.previous.set(item.id, {
timestamp,
bytesReceived: Number(item.bytesReceived) || 0,
framesDecoded: Number(item.framesDecoded) || 0,
totalDecodeTime: Number(item.totalDecodeTime) || 0,
totalProcessingDelay: Number(item.totalProcessingDelay) || 0,
framesDropped: Number(item.framesDropped) || 0,
});
const packetsReceived = Number(item.packetsReceived) || 0;
const packetsLost = Math.max(0, Number(item.packetsLost) || 0);
const packetTotal = packetsReceived + packetsLost;
result[kind] = {
bitrateMbps,
packetsLost,
lossPercent: packetTotal > 0 ? (packetsLost * 100) / packetTotal : 0,
jitterMs: Number.isFinite(item.jitter) ? item.jitter * 1000 : null,
jitterBufferMs: averageJitterBufferMs(item),
decodeMs,
processingDelayMs,
framesDropped: Number.isFinite(item.framesDropped) ? item.framesDropped : null,
framesDroppedDelta,
framesPerSecond: Number.isFinite(item.framesPerSecond) ? item.framesPerSecond : null,
width: Number.isFinite(item.frameWidth) ? item.frameWidth : null,
height: Number.isFinite(item.frameHeight) ? item.frameHeight : null,
codec: String(codec?.mimeType || "").replace(/^\w+\//, ""),
};
}
this.onStats(result);
} catch (error) {
if (this.pc?.connectionState !== "closed") console.debug("webrtc stats unavailable", error);
} finally {
this.running = false;
}
}
}
function averageJitterBufferMs(item) {
const delay = Number(item.jitterBufferDelay);
const emitted = Number(item.jitterBufferEmittedCount);
return Number.isFinite(delay) && emitted > 0 ? (delay * 1000) / emitted : null;
}
function summarizeRoute(localType, remoteType) {
const types = [localType, remoteType].filter(Boolean);
if (types.includes("relay")) return "Relay";
if (types.includes("srflx") || types.includes("prflx")) return "P2P";
if (types.includes("host")) return "Direct";
return "";
}
Overlord-Server/public/assets/whep.js
+10
View File
@@ -1,3 +1,5 @@
import { WebRTCStatsSampler } from "./webrtc-stats.js";
// Tiny WHEP (WebRTC-HTTP Egress Protocol) client.
//
// Wires a <video> element to a server-proxied WHEP endpoint:
@@ -16,14 +18,17 @@ export class WhepClient {
* @param {HTMLAudioElement} [opts.audioEl] Attach the audio track here.
* Default: hidden <audio> appended to <body>.
* @param {(state: string) => void} [opts.onState]
* @param {(stats: object) => void} [opts.onStats]
*/
constructor(opts) {
this.whepPath = opts.whepPath;
this.videoEl = opts.videoEl || null;
this.audioEl = opts.audioEl || null;
this.onState = opts.onState || (() => {});
this.onStats = opts.onStats || (() => {});
this.pc = null;
this.resourceURL = "";
this.statsSampler = null;
}
async start() {
@@ -31,6 +36,7 @@ export class WhepClient {
const pc = new RTCPeerConnection({});
this.pc = pc;
this.statsSampler = new WebRTCStatsSampler(pc, this.onStats);
if (this.videoEl) pc.addTransceiver("video", { direction: "recvonly" });
if (this.audioEl) pc.addTransceiver("audio", { direction: "recvonly" });
@@ -46,6 +52,8 @@ export class WhepClient {
};
pc.onconnectionstatechange = () => {
this.onState(pc.connectionState);
if (pc.connectionState === "connected") this.statsSampler?.start();
if (["failed", "closed", "disconnected"].includes(pc.connectionState)) this.statsSampler?.stop();
};
const offer = await pc.createOffer();
@@ -76,6 +84,8 @@ export class WhepClient {
const pc = this.pc;
const url = this.resourceURL;
this.pc = null;
this.statsSampler?.stop();
this.statsSampler = null;
this.resourceURL = "";
if (url) {
try {
Overlord-Server/public/backstage.html
+7
View File
@@ -83,6 +83,13 @@
<span class="uppercase tracking-wide text-xs text-slate-500">lat</span>
<span id="latencyDisplay" class="pill pill-ghost font-mono text-amber-200">--</span>
</div>
<div
class="flex items-center gap-2 bg-slate-900/70 border border-slate-800 rounded-lg px-3 py-2 text-sm text-slate-300"
title="WebRTC receive bitrate, network RTT, packet loss, and selected route"
>
<span class="uppercase tracking-wide text-xs text-slate-500">net</span>
<span id="networkStats" class="pill pill-ghost font-mono text-cyan-200">--</span>
</div>
<div class="relative ml-auto" id="rdSettingsWrap">
<button
Overlord-Server/public/remotedesktop.html
+100 -2
View File
@@ -109,6 +109,17 @@
<span class="uppercase tracking-wide text-xs text-slate-500">input</span>
<span id="inputLatency" class="pill pill-ghost font-mono text-amber-200">-- ms</span>
</div>
<div
class="flex items-center gap-2 bg-slate-900/70 border border-slate-800 rounded-lg px-3 py-2 text-sm text-slate-300"
title="WebRTC receive bitrate, network RTT, packet loss, and selected route"
>
<span class="uppercase tracking-wide text-xs text-slate-500">net</span>
<span id="networkStats" class="pill pill-ghost font-mono text-cyan-200">--</span>
</div>
<button id="diagnosticsBtn" type="button" class="button ghost" aria-pressed="false" title="Show streaming diagnostics HUD">
<i class="fa-solid fa-gauge-high"></i><span>Stats HUD</span>
</button>
<div class="relative ml-auto" id="rdSettingsWrap">
<button
@@ -178,6 +189,19 @@
</select>
</div>
<div id="streamProfileDetail" class="mb-2 text-xs text-slate-500">Checking hardware encoder profiles…</div>
<div class="flex items-center gap-2 mb-2">
<label class="text-sm text-slate-300 w-20" for="bitrateSelect">Bitrate</label>
<select id="bitrateSelect" class="flex-1 bg-slate-800 border border-slate-700 rounded px-2 py-1 text-sm text-slate-100" title="H.264 target bitrate; Auto derives it from resolution and FPS">
<option value="0" selected>Auto</option>
<option value="5">5 Mbps</option>
<option value="10">10 Mbps</option>
<option value="15">15 Mbps</option>
<option value="20">20 Mbps</option>
<option value="30">30 Mbps</option>
<option value="40">40 Mbps</option>
<option value="50">50 Mbps</option>
</select>
</div>
<div class="flex items-center justify-between gap-2">
<label class="flex items-center gap-2 text-sm text-slate-200" title="Prefer H.264 when available; disable to force JPEG">
<input type="checkbox" id="codecH264" checked /> H.264
@@ -201,7 +225,7 @@
</label>
<div class="flex items-center gap-2">
<label class="text-sm text-slate-300 w-20" for="audioTransport">Via</label>
<select id="audioTransport" class="flex-1 bg-slate-800 border border-slate-700 rounded px-2 py-1 text-sm text-slate-100" title="Audio transport. WS: raw PCM over the existing WebSocket (current default). WebRTC modes use G.711 over the WebRTC track instead.">
<select id="audioTransport" class="flex-1 bg-slate-800 border border-slate-700 rounded px-2 py-1 text-sm text-slate-100" title="Audio transport. WS uses raw PCM over the existing WebSocket. WebRTC modes use full-band Opus.">
<option value="off" selected>WS</option>
<option value="p2p">WebRTC P2P</option>
<option value="relayed">WebRTC Relayed</option>
@@ -249,7 +273,7 @@
Client: <span id="clientLabel" class="text-slate-100">-</span>
</div>
<div
class="flex-1 flex items-center justify-center p-4"
class="relative flex-1 flex items-center justify-center p-4"
style="min-height: 0; overflow: hidden"
>
<canvas
@@ -279,6 +303,49 @@
height: auto;
"
></video>
<section id="diagnosticsHud" class="rd-diagnostics-hud hidden" aria-label="Streaming diagnostics">
<div class="rd-diagnostics-header">
<div>
<div class="rd-diagnostics-title">Streaming diagnostics</div>
<div id="diagSummary" class="rd-diagnostics-summary">Waiting for frames</div>
</div>
<button id="diagnosticsClose" type="button" class="rd-diagnostics-close" aria-label="Close diagnostics">×</button>
</div>
<div class="rd-diagnostics-meta">
<span id="diagTransport">Canvas</span>
<span id="diagCodec">--</span>
<span id="diagResolution">--</span>
</div>
<div class="rd-diagnostics-section">
<div class="rd-diagnostics-section-title">Agent pipeline</div>
<div class="rd-diagnostics-grid">
<span>Capture</span><strong id="diagCapture">-- ms</strong>
<span>Encode</span><strong id="diagEncode">-- ms</strong>
<span>Packet/write</span><strong id="diagSend">-- ms</strong>
<span>Agent total</span><strong id="diagAgentTotal">-- ms</strong>
</div>
</div>
<div class="rd-diagnostics-section">
<div class="rd-diagnostics-section-title">Transport</div>
<div class="rd-diagnostics-grid">
<span>Receive rate</span><strong id="diagBitrate">-- Mbps</strong>
<span>Network RTT</span><strong id="diagRtt">-- ms</strong>
<span>Loss / jitter</span><strong id="diagLossJitter">--</strong>
<span>Jitter buffer</span><strong id="diagJitterBuffer">-- ms</strong>
</div>
</div>
<div class="rd-diagnostics-section">
<div class="rd-diagnostics-section-title">Viewer pipeline</div>
<div class="rd-diagnostics-grid">
<span>Decode latency</span><strong id="diagDecode">-- ms</strong>
<span>Receive → render</span><strong id="diagRender">-- ms</strong>
<span>Decode queue</span><strong id="diagQueue">0</strong>
<span>Dropped / coalesced</span><strong id="diagDropped">0 / 0</strong>
<span>FPS agent → viewer</span><strong id="diagFps">-- → --</strong>
<span>Input RTT</span><strong id="diagInput">-- ms</strong>
</div>
</div>
</section>
</div>
</div>
<style>
@@ -311,6 +378,37 @@
object-fit: contain;
margin: auto;
}
.rd-diagnostics-hud {
position: absolute;
top: 1.25rem;
right: 1.25rem;
z-index: 30;
width: min(360px, calc(100% - 2.5rem));
max-height: calc(100% - 2.5rem);
overflow-y: auto;
padding: 14px;
border: 1px solid rgba(71, 85, 105, 0.9);
border-radius: 12px;
color: #e2e8f0;
background: rgba(2, 6, 23, 0.9);
box-shadow: 0 18px 45px rgba(0, 0, 0, 0.5);
backdrop-filter: blur(12px);
font-size: 12px;
}
.rd-diagnostics-hud.hidden { display: none; }
.rd-diagnostics-header, .rd-diagnostics-meta { display: flex; align-items: center; justify-content: space-between; gap: 10px; }
.rd-diagnostics-title { color: #f8fafc; font-size: 14px; font-weight: 700; }
.rd-diagnostics-summary { margin-top: 2px; color: #67e8f9; }
.rd-diagnostics-summary[data-severity="warn"] { color: #fbbf24; }
.rd-diagnostics-summary[data-severity="bad"] { color: #fb7185; }
.rd-diagnostics-close { border: 0; background: transparent; color: #94a3b8; font-size: 22px; cursor: pointer; }
.rd-diagnostics-meta { justify-content: flex-start; flex-wrap: wrap; margin: 10px 0; color: #94a3b8; font-family: ui-monospace, monospace; }
.rd-diagnostics-meta span { padding: 2px 6px; border-radius: 999px; background: rgba(30, 41, 59, 0.8); }
.rd-diagnostics-section { padding-top: 9px; margin-top: 9px; border-top: 1px solid rgba(51, 65, 85, 0.75); }
.rd-diagnostics-section-title { margin-bottom: 6px; color: #64748b; font-size: 10px; font-weight: 700; letter-spacing: .08em; text-transform: uppercase; }
.rd-diagnostics-grid { display: grid; grid-template-columns: 1fr auto; gap: 5px 14px; }
.rd-diagnostics-grid span { color: #94a3b8; }
.rd-diagnostics-grid strong { color: #e2e8f0; font-family: ui-monospace, monospace; font-weight: 600; text-align: right; }
</style>
</div>
</main>
Overlord-Server/src/main-server.ts
+2
View File
@@ -92,6 +92,7 @@ import { createWebSocketRuntime } from "./server/websocket-runtime";
import {
handleConsoleOutput,
handleDesktopEncoderCapabilities,
handleDesktopStreamStats,
handleConsoleViewerMessage,
handleConsoleViewerOpen,
handlebackstageViewerMessage,
@@ -656,6 +657,7 @@ async function startServer() {
handleNotificationScreenshotResult: notificationPluginHandlers.handleNotificationScreenshotResult,
handleConsoleOutput: (clientId: string, payload: any) => handleConsoleOutput(clientId, payload),
handleDesktopEncoderCapabilities: (clientId: string, payload: any) => handleDesktopEncoderCapabilities(clientId, payload),
handleDesktopStreamStats: (clientId: string, payload: any) => handleDesktopStreamStats(clientId, payload),
handleFileBrowserMessage: (clientId: string, payload: any) =>
forwardFileBrowserMessage(clientId, payload, {
pendingHttpDownloads,
Overlord-Server/src/protocol.ts
+18
View File
@@ -8,6 +8,7 @@ export type MessageKind =
| "command"
| "command_result"
| "desktop_encoder_capabilities"
| "desktop_stream_stats"
| "client_logs_result"
| "screenshot_result"
| "frame"
@@ -125,6 +126,7 @@ export type CommandType =
| "desktop_enable_mouse"
| "desktop_enable_keyboard"
| "desktop_set_fps"
| "desktop_set_bitrate"
| "darwin_request_permissions"
| "webrtc_publish"
| "webrtc_stop"
@@ -189,6 +191,8 @@ export type FrameHeader = {
monitor: number;
fps: number;
format: "jpeg" | "webp" | "raw" | "h264";
width?: number;
height?: number;
hash?: string;
backstage?: boolean;
webcam?: boolean;
@@ -196,6 +200,19 @@ export type FrameHeader = {
export type Frame = { type: "frame"; header: FrameHeader; data: Uint8Array };
export type FrameAck = { type: "frame_ack" };
export type DesktopStreamStats = {
type: "desktop_stream_stats";
fps: number;
format: string;
bytes: number;
width: number;
height: number;
captureMs: number;
encodeMs: number;
sendMs: number;
totalMs: number;
transport: "websocket" | "webrtc" | string;
};
export type Status = {
type: "status";
state: "idle" | "streaming" | "error";
@@ -505,6 +522,7 @@ export type WireMessage =
| ScreenshotResult
| Frame
| FrameAck
| DesktopStreamStats
| Status
| ConsoleOutput
| FileListResult
Overlord-Server/src/server/routes/websocket-lifecycle-routes.ts
+4
View File
@@ -222,6 +222,7 @@ type WsLifecycleDeps = {
handleNotificationScreenshotResult: (clientId: string, payload: any) => void;
handleConsoleOutput: (clientId: string, payload: any) => void;
handleDesktopEncoderCapabilities: (clientId: string, payload: any) => void;
handleDesktopStreamStats: (clientId: string, payload: any) => void;
handleFileBrowserMessage: (clientId: string, payload: any) => void;
handleProxyTunnelData: (clientId: string, connectionId: string, data: Uint8Array) => void;
handleProxyTunnelClose: (clientId: string, connectionId: string) => void;
@@ -799,6 +800,9 @@ export async function handleWebSocketMessage(
case "desktop_encoder_capabilities":
deps.handleDesktopEncoderCapabilities(client.id, payload);
break;
case "desktop_stream_stats":
deps.handleDesktopStreamStats(client.id, payload);
break;
case "file_list_result":
case "file_download":
case "file_upload_result":
Overlord-Server/src/server/ws-console-rd-backstage.test.ts
+74
View File
@@ -9,6 +9,7 @@ import {
handlebackstageViewerOpen,
handleRemoteDesktopViewerMessage,
handleRemoteDesktopViewerOpen,
handleDesktopStreamStats,
backstageStreamingState,
rdStreamingState,
} from "./ws-console-rd-backstage";
@@ -143,6 +144,7 @@ describe("remote desktop viewer control", () => {
duplication: true,
maxHeight: 1080,
maxFps: 120,
bitrateMbps: 0,
lastFps: 1,
lastFrameAt: 0,
startedAt: Date.now() - 5000,
@@ -156,6 +158,78 @@ describe("remote desktop viewer control", () => {
expect(commands.filter((msg) => msg.commandType === "desktop_request_keyframe")).toHaveLength(0);
expect(rdStreamingState.get(clientId)?.isStreaming).toBe(true);
});
test("forwards and clamps the desktop bitrate setting", () => {
const clientId = `rd-bitrate-${Date.now().toString(36)}`;
const { agentWs } = createClient(clientId);
const viewer = createMockWs({ clientId });
handleRemoteDesktopViewerOpen(viewer as any);
handleRemoteDesktopViewerMessage(viewer as any, JSON.stringify({
type: "desktop_set_bitrate",
bitrateMbps: 75,
}));
const command = agentCommands(agentWs).find((msg) => msg.commandType === "desktop_set_bitrate");
expect(command?.payload?.bitrateMbps).toBe(50);
expect(rdStreamingState.get(clientId)?.bitrateMbps).toBe(50);
});
test("forwards agent pipeline telemetry to every remote desktop viewer", () => {
const clientId = `rd-stats-${Date.now().toString(36)}`;
createClient(clientId);
const viewer = createMockWs({ clientId });
handleRemoteDesktopViewerOpen(viewer as any);
handleDesktopStreamStats(clientId, {
type: "desktop_stream_stats",
fps: 60,
format: "h264",
captureMs: 2.25,
encodeMs: 4.5,
sendMs: 0.4,
totalMs: 7.3,
transport: "webrtc",
});
const message = decodeMessage(viewer.sent.at(-1) as Uint8Array) as any;
expect(message.type).toBe("desktop_stream_stats");
expect(message.captureMs).toBe(2.25);
expect(message.encodeMs).toBe(4.5);
expect(message.transport).toBe("webrtc");
});
test("holds Canvas acknowledgements only while the browser decoder is under pressure", () => {
const clientId = `rd-flow-${Date.now().toString(36)}`;
const { agentWs } = createClient(clientId);
const viewer = createMockWs({ clientId });
handleRemoteDesktopViewerOpen(viewer as any);
handleRemoteDesktopViewerMessage(viewer as any, JSON.stringify({
type: "desktop_start",
canvasFlowControl: true,
}));
const healthyAck = (globalThis as any).__rdBroadcast(
clientId,
new Uint8Array([1, 2, 3]),
{ format: "h264", fps: 60, width: 2560, height: 1440 },
);
expect(healthyAck).toBe(true);
const frame = viewer.sent.at(-1) as Uint8Array;
expect(frame[3]).toBe(2);
expect(frame.byteLength).toBe(15);
handleRemoteDesktopViewerMessage(viewer as any, JSON.stringify({ type: "desktop_decode_pressure", active: true }));
const pressuredAck = (globalThis as any).__rdBroadcast(
clientId,
new Uint8Array([4, 5, 6]),
{ format: "h264", fps: 60, width: 2560, height: 1440 },
);
expect(pressuredAck).toBe(false);
handleRemoteDesktopViewerMessage(viewer as any, JSON.stringify({ type: "desktop_decode_pressure", active: false }));
expect(agentCommands(agentWs).filter((msg) => msg.type === "frame_ack")).toHaveLength(1);
});
});
describe("backstage viewer control", () => {
Overlord-Server/src/server/ws-console-rd-backstage.ts
+80 -2
View File
@@ -175,6 +175,7 @@ function broadcastFrameToViewers(
const rdSendStats = { lastLog: 0, frames: 0, sendMs: 0, bytes: 0 };
const rdDebugFrameLogAt = new Map<string, number>();
const rdCanvasFrameAckPending = new Map<string, string>();
export const rdStreamingState = new Map<string, {
isStreaming: boolean;
display: number;
@@ -184,6 +185,7 @@ export const rdStreamingState = new Map<string, {
duplication: boolean;
maxHeight: number;
maxFps: number;
bitrateMbps: number;
lastFps: number;
lastFrameAt: number;
startedAt: number;
@@ -209,6 +211,7 @@ function defaultRdStreamingState() {
duplication: false,
maxHeight: 0,
maxFps: 120,
bitrateMbps: 0,
lastFps: 0,
lastFrameAt: 0,
startedAt: 0,
@@ -366,7 +369,9 @@ export function handleRemoteDesktopViewerMessage(ws: ServerWebSocket<SocketData>
const state = rdStreamingState.get(clientId) || defaultRdStreamingState();
logger.debug(`[rd] inbound viewer msg type=${payload.type} client=${clientId}`);
if (payload.type !== "desktop_decode_pressure") {
logger.debug(`[rd] inbound viewer msg type=${payload.type} client=${clientId}`);
}
switch (payload.type) {
case "desktop_encoder_capabilities":
sendDesktopCommand(target, "desktop_encoder_capabilities", {
@@ -374,6 +379,9 @@ export function handleRemoteDesktopViewerMessage(ws: ServerWebSocket<SocketData>
});
break;
case "desktop_start":
(ws.data as any).rdCanvasFlowControl = (payload as any).canvasFlowControl === true && (payload as any).webrtc !== true;
(ws.data as any).rdCanvasBackpressure = false;
releaseCanvasFrameAck(clientId);
logger.debug(`[rd-debug] desktop_start requested client=${clientId} session=${ws.data.sessionId || ""} state=${JSON.stringify(state)} viewers=${sessionManager.getRdSessionsForClient(clientId).length} webrtc=${(payload as any).webrtc === true}`);
if (!state.isStreaming) {
const targetOs = String(target.os || "").toLowerCase();
@@ -412,6 +420,7 @@ export function handleRemoteDesktopViewerMessage(ws: ServerWebSocket<SocketData>
}
safeSendViewer(ws, { type: "status", status: "starting" });
sendDesktopCommand(target, "desktop_set_fps", { fps: clampDesktopFps(state.maxFps) });
sendDesktopCommand(target, "desktop_set_bitrate", { bitrateMbps: state.bitrateMbps || 0 });
sendDesktopCommand(target, "desktop_start", {});
state.isStreaming = true;
state.startedAt = Date.now();
@@ -423,6 +432,7 @@ export function handleRemoteDesktopViewerMessage(ws: ServerWebSocket<SocketData>
if (lastFrameAgeMs > 3000 && startAgeMs > 3000) {
logger.info(`[rd-debug] desktop_start reasserting stale stream client=${clientId} lastFrameAgeMs=${Number.isFinite(lastFrameAgeMs) ? lastFrameAgeMs : -1} state=${JSON.stringify(state)} viewers=${sessionManager.getRdSessionsForClient(clientId).length}`);
sendDesktopCommand(target, "desktop_set_fps", { fps: clampDesktopFps(state.maxFps) });
sendDesktopCommand(target, "desktop_set_bitrate", { bitrateMbps: state.bitrateMbps || 0 });
sendDesktopCommand(target, "desktop_start", {});
safeSendViewer(ws, { type: "status", status: "starting" });
state.isStreaming = true;
@@ -439,6 +449,9 @@ export function handleRemoteDesktopViewerMessage(ws: ServerWebSocket<SocketData>
}
break;
case "desktop_stop": {
(ws.data as any).rdCanvasFlowControl = false;
(ws.data as any).rdCanvasBackpressure = false;
releaseCanvasFrameAck(clientId);
const otherViewers = sessionManager.getRdSessionsForClient(clientId)
.filter(s => s.id !== ws.data.sessionId);
logger.debug(`[rd-debug] desktop_stop requested client=${clientId} session=${ws.data.sessionId || ""} otherViewers=${otherViewers.length} state=${JSON.stringify(state)}`);
@@ -581,6 +594,23 @@ export function handleRemoteDesktopViewerMessage(ws: ServerWebSocket<SocketData>
}
break;
}
case "desktop_decode_pressure": {
if (!(ws.data as any).rdCanvasFlowControl) break;
const active = (payload as any).active === true;
(ws.data as any).rdCanvasBackpressure = active;
if (!active) releaseCanvasFrameAck(clientId, ws.data.sessionId);
break;
}
case "desktop_set_bitrate": {
const newBitrateMbps = Math.max(0, Math.min(50, Math.floor(Number((payload as any).bitrateMbps) || 0)));
if (state.bitrateMbps !== newBitrateMbps) {
sendDesktopCommand(target, "desktop_set_bitrate", { bitrateMbps: newBitrateMbps });
state.bitrateMbps = newBitrateMbps;
rdStreamingState.set(clientId, state);
logger.debug(`[rd] set target bitrate=${newBitrateMbps || "auto"} Mbps`);
}
break;
}
case "desktop_set_duplication": {
const enabled = !!payload.enabled;
if (state.duplication !== enabled) {
@@ -732,6 +762,11 @@ export function handleWebrtcP2PIce(_clientId: string, payload: any) {
}
export function cleanupViewerP2P(ws: ServerWebSocket<SocketData>) {
if (ws.data.role === "rd_viewer") {
(ws.data as any).rdCanvasFlowControl = false;
(ws.data as any).rdCanvasBackpressure = false;
releaseCanvasFrameAck(ws.data.clientId, ws.data.sessionId);
}
const cleared = clearP2PSessionForViewer(ws);
if (!cleared) return;
const target = clientManager.getClient(cleared.clientId);
@@ -776,6 +811,15 @@ function broadcastRemoteDesktopFrame(clientId: string, bytes: Uint8Array, header
const buf = buildViewerFrameBuffer(bytes, header);
const sessions = sessionManager.getRdSessionsForClient(clientId);
const result = broadcastFrameToViewers(sessions, buf, header);
if (result.sent) {
const controller = sessions.find((session) =>
(session.viewer.data as any).rdCanvasFlowControl === true &&
(session.viewer.data as any).rdCanvasBackpressure === true
);
if (controller?.viewer.data.sessionId) {
rdCanvasFrameAckPending.set(clientId, controller.viewer.data.sessionId);
}
}
if (result.dropped) {
const target = clientManager.getClient(clientId);
if (target) {
@@ -785,7 +829,7 @@ function broadcastRemoteDesktopFrame(clientId: string, bytes: Uint8Array, header
});
}
}
return result.sent || result.viewers === 0;
return (result.sent && !rdCanvasFrameAckPending.has(clientId)) || result.viewers === 0;
}
(globalThis as any).__rdBroadcast = (clientId: string, bytes: Uint8Array, header?: any): boolean => {
@@ -1448,6 +1492,40 @@ export function handleDesktopEncoderCapabilities(clientId: string, payload: any)
}
}
function releaseCanvasFrameAck(clientId: string, sessionId?: string): boolean {
const controllerSessionId = rdCanvasFrameAckPending.get(clientId);
if (!controllerSessionId || (sessionId && controllerSessionId !== sessionId)) return false;
rdCanvasFrameAckPending.delete(clientId);
const target = clientManager.getClient(clientId);
if (!target) return false;
try {
target.ws.send(encodeMessage({ type: "frame_ack" }));
return true;
} catch (err) {
logger.debug(`[rd] canvas frame ack failed client=${clientId}: ${(err as Error).message}`);
return false;
}
}
export function handleDesktopStreamStats(clientId: string, payload: any) {
const stats = {
type: "desktop_stream_stats",
fps: Math.max(0, Number(payload?.fps) || 0),
format: String(payload?.format || ""),
bytes: Math.max(0, Number(payload?.bytes) || 0),
width: Math.max(0, Number(payload?.width) || 0),
height: Math.max(0, Number(payload?.height) || 0),
captureMs: Math.max(0, Number(payload?.captureMs) || 0),
encodeMs: Math.max(0, Number(payload?.encodeMs) || 0),
sendMs: Math.max(0, Number(payload?.sendMs) || 0),
totalMs: Math.max(0, Number(payload?.totalMs) || 0),
transport: String(payload?.transport || ""),
};
for (const session of sessionManager.getRdSessionsForClient(clientId)) {
safeSendViewer(session.viewer, stats, "rd-stats");
}
}
(globalThis as any).__backstageBroadcast = (clientId: string, bytes: Uint8Array, header?: any): boolean => {
const state = backstageStreamingState.get(clientId) || defaultbackstageStreamingState();
if (!state.isStreaming) {
Overlord-Server/src/server/ws-viewer-utils.ts
+12 -4
View File
@@ -33,11 +33,14 @@ export function safeSendViewer(
}
export function buildViewerFrameBuffer(bytes: Uint8Array, header?: any): Uint8Array {
const meta = new Uint8Array(8);
const width = Math.max(0, Math.min(65535, Math.floor(Number(header?.width) || 0)));
const height = Math.max(0, Math.min(65535, Math.floor(Number(header?.height) || 0)));
const metaLength = width > 0 && height > 0 ? 12 : 8;
const meta = new Uint8Array(metaLength);
meta[0] = 0x46;
meta[1] = 0x52;
meta[2] = 0x4d;
meta[3] = 1;
meta[3] = metaLength === 12 ? 2 : 1;
meta[4] = (header?.monitor ?? 0) & 0xff;
meta[5] = (header?.fps ?? 0) & 0xff;
const fmt = header?.format === "blocks"
@@ -49,10 +52,15 @@ export function buildViewerFrameBuffer(bytes: Uint8Array, header?: any): Uint8Ar
: 1;
meta[6] = fmt;
meta[7] = 0;
if (metaLength === 12) {
const view = new DataView(meta.buffer);
view.setUint16(8, width, true);
view.setUint16(10, height, true);
}
const buf = new Uint8Array(8 + bytes.length);
const buf = new Uint8Array(metaLength + bytes.length);
buf.set(meta, 0);
buf.set(bytes, 8);
buf.set(bytes, metaLength);
return buf;
}
Overlord-Server/src/wsValidation.ts
+1
View File
@@ -21,6 +21,7 @@ export const ALLOWED_CLIENT_MESSAGE_TYPES = new Set([
"file_hash_result",
"command_result",
"desktop_encoder_capabilities",
"desktop_stream_stats",
"client_logs_result",
"screenshot_result",
"command_progress",