vxaboveground/overlord
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

HEVC (and some other small shit)

Browse Source
This commit is contained in:
KingKDot
2026-07-18 01:27:58 -05:00
parent b619658a95
commit ae74e17d13
34 changed files with 1129 additions and 101 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Overlord-Server/public/assets/remotedesktop.js
+151 -28
View File
@@ -122,11 +122,14 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
let pendingFrame = null;
let videoDecoder = null;
let videoDecoderConfigKey = "";
const disabledDecoderCodecs = new Set();
let h264StreamCodec = "";
let canvasDecodePressure = false;
let h264PendingTimings = [];
let h264TimestampUs = 0;
let prefersH264 = typeof VideoDecoder === "function";
let negotiatedCodec = prefersH264 ? "h264" : "jpeg";
let browserDecoderCodecs = ["jpeg", "raw"];
let h264LowFpsStreak = 0;
let h264FirstFrameAt = 0;
let h264FramesSeen = 0;
@@ -1449,7 +1452,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
function pushQuality(val) {
const q = Number(val) || 90;
const codec = q >= 100 ? "raw" : (prefersH264 ? "h264" : "jpeg");
const codec = q >= 100 ? "raw" : (negotiatedCodec || (prefersH264 ? "h264" : "jpeg"));
const softwareH264 = codec === "h264" && useSoftwareH264();
console.debug("rd: pushQuality val=", val, "q=", q, "codec=", codec, "softwareH264=", softwareH264);
setCodecModeLabel(codec, "requested");
@@ -1475,6 +1478,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
codecH264.addEventListener("change", function () {
resetH264SessionState();
prefersH264 = !!codecH264.checked && typeof VideoDecoder === "function";
negotiatedCodec = prefersH264 ? "h264" : "jpeg";
if (softwareH264Ctrl) {
softwareH264Ctrl.disabled = !prefersH264;
}
@@ -1488,6 +1492,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
if (qualitySlider) {
pushQuality(qualitySlider.value);
}
requestEncoderCapabilities();
sharedSettingsSaver.scheduleSave();
});
}
@@ -1558,15 +1563,80 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
});
}
function requestEncoderCapabilities() {
async function probeBrowserDecoderCodecs(transport) {
const supported = ["jpeg", "raw"];
if (transport === "webrtc") {
try {
const codecs = typeof RTCRtpReceiver !== "undefined" && typeof RTCRtpReceiver.getCapabilities === "function"
? (RTCRtpReceiver.getCapabilities("video")?.codecs || [])
: [];
for (const capability of codecs) {
const mime = String(capability?.mimeType || "").toLowerCase();
if (mime === "video/h264") supported.push("h264");
if (mime === "video/h265" || mime === "video/hevc") supported.push("hevc");
}
} catch (err) {
console.debug("rd: WebRTC codec capability probe failed", err);
}
return [...new Set(supported)];
}
if (typeof VideoDecoder !== "function") return supported;
if (typeof VideoDecoder.isConfigSupported !== "function") {
supported.push("h264");
return supported;
}
const probes = [
["h264", "avc1.42E01E"],
["hevc", "hev1.1.6.L156.B0"],
];
await Promise.all(probes.map(async ([name, codec]) => {
try {
const result = await VideoDecoder.isConfigSupported({ codec, optimizeForLatency: true });
if (result?.supported) supported.push(name);
} catch (err) {
console.debug(`rd: ${name} decoder capability probe failed`, err);
}
}));
return [...new Set(supported)].filter((codec) => !disabledDecoderCodecs.has(codec));
}
async function requestEncoderCapabilities() {
if (streamProfileDetail) streamProfileDetail.textContent = "Checking hardware encoder profiles…";
const mode = getWebrtcMode();
const transport = mode === "off" ? "websocket" : "webrtc";
const decoderCodecs = await probeBrowserDecoderCodecs(transport);
browserDecoderCodecs = decoderCodecs;
if (!ws || ws.readyState !== WebSocket.OPEN) return;
sendCmd("desktop_encoder_capabilities", {
display: Number.parseInt(displaySelect?.value || "0", 10) || 0,
decoderCodecs,
preferredCodecs: prefersH264 ? ["hevc", "h264", "jpeg", "raw"] : ["jpeg", "raw"],
transport,
});
}
function applyEncoderCapabilities(msg) {
if (!msg || !Array.isArray(msg.profiles) || !streamProfileSelect) return;
if (!msg) return;
const selectedCodec = String(msg.selectedCodec || msg.negotiation?.selectedCodec || "").toLowerCase();
if (selectedCodec) {
negotiatedCodec = selectedCodec;
const fallbacks = Array.isArray(msg.fallbackCodecs) ? msg.fallbackCodecs.join(" → ") : selectedCodec;
setCodecModeLabel(selectedCodec, `negotiated; fallback ${fallbacks}`);
if (codecH264) {
const agentHasH264 = Array.isArray(msg.codecs) && msg.codecs.some((entry) => String(entry?.codec || "").toLowerCase() === "h264");
codecH264.disabled = !browserDecoderCodecs.includes("h264") || !agentHasH264;
}
if (desiredStreaming && qualitySlider) pushQuality(qualitySlider.value);
} else if (msg.transport === "webrtc") {
console.warn("rd: no mutually supported WebRTC video codec; falling back to WebSocket canvas");
if (webrtcMode) webrtcMode.value = "off";
negotiatedCodec = browserDecoderCodecs.includes("h264") ? "h264" : "jpeg";
setCodecModeLabel(negotiatedCodec, "WebRTC unavailable; WebSocket fallback");
stopAllWebrtc();
requestEncoderCapabilities();
}
if (!Array.isArray(msg.profiles) || !streamProfileSelect) return;
const selectedDisplay = Number.parseInt(displaySelect?.value || "0", 10) || 0;
if (Number.isFinite(Number(msg.display)) && Number(msg.display) !== selectedDisplay) return;
const previous = streamProfileSelect.value;
@@ -1637,6 +1707,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
if (webrtcMode) {
webrtcMode.addEventListener("change", function () {
updateControls();
requestEncoderCapabilities();
sharedSettingsSaver.scheduleSave();
});
}
@@ -1910,11 +1981,12 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
function destroyVideoDecoder() {
updateCanvasDecodePressure(true);
if (!videoDecoder) return;
try {
videoDecoder.close();
} catch {
// Ignore close errors when decoder is already shutting down.
if (videoDecoder) {
try {
videoDecoder.close();
} catch {
// Ignore close errors when decoder is already shutting down.
}
}
videoDecoder = null;
videoDecoderConfigKey = "";
@@ -1945,6 +2017,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
if (!prefersH264) return;
const reasonText = normalizeFallbackReason(reason);
prefersH264 = false;
negotiatedCodec = "jpeg";
destroyVideoDecoder();
if (codecH264) codecH264.checked = false;
if (softwareH264Ctrl) softwareH264Ctrl.disabled = true;
@@ -2037,6 +2110,26 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
return false;
}
function isHEVCKeyFrame(data) {
for (let i = 0; i + 5 < data.length; i++) {
let startCodeLen = 0;
if (data[i] === 0x00 && data[i + 1] === 0x00 && data[i + 2] === 0x01) {
startCodeLen = 3;
} else if (
data[i] === 0x00 && data[i + 1] === 0x00 && data[i + 2] === 0x00 && data[i + 3] === 0x01
) {
startCodeLen = 4;
}
if (!startCodeLen) continue;
const nalIndex = i + startCodeLen;
if (nalIndex >= data.length) break;
const nalType = (data[nalIndex] >> 1) & 0x3f;
if (nalType >= 16 && nalType <= 21) return true;
i = nalIndex;
}
return false;
}
function h264CodecFromAnnexB(data) {
for (let i = 0; i + 7 < data.length; i++) {
let startCodeLength = 0;
@@ -2052,9 +2145,11 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
return "";
}
function ensureVideoDecoder(h264Bytes, width, height) {
const detectedCodec = h264CodecFromAnnexB(h264Bytes);
const codec = detectedCodec || h264StreamCodec || "avc1.4d0034";
function ensureVideoDecoder(videoBytes, width, height, codecName) {
const detectedCodec = codecName === "h264" ? h264CodecFromAnnexB(videoBytes) : "";
const codec = codecName === "hevc"
? "hev1.1.6.L156.B0"
: (detectedCodec || h264StreamCodec || "avc1.4d0034");
const codedWidth = Math.max(0, Number(width) || 0);
const codedHeight = Math.max(0, Number(height) || 0);
const configKey = `${codec}:${codedWidth}x${codedHeight}`;
@@ -2095,8 +2190,9 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
updateCanvasDecodePressure();
},
error: (err) => {
console.warn("rd: h264 decoder error", err);
console.warn(`rd: ${codecName} decoder error`, err);
updateCanvasDecodePressure(true);
setTimeout(() => fallbackFromVideoCodec(codecName, err), 0);
},
});
videoDecoder.addEventListener("dequeue", () => {
@@ -2110,11 +2206,11 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
}
videoDecoder.configure(config);
videoDecoderConfigKey = configKey;
h264StreamCodec = codec;
if (codecName === "h264") h264StreamCodec = codec;
return true;
} catch (err) {
console.warn("rd: h264 decoder unavailable", err);
fallbackToJpegCodec(err);
console.warn(`rd: ${codecName} decoder unavailable`, err);
fallbackFromVideoCodec(codecName, err);
destroyVideoDecoder();
return false;
}
@@ -2132,6 +2228,32 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
}
}
function fallbackFromVideoCodec(codec, reason) {
if (codec === "hevc" && disabledDecoderCodecs.has("hevc")) return;
const reasonText = normalizeFallbackReason(reason);
if (codec === "hevc" && browserDecoderCodecs.includes("h264")) {
disabledDecoderCodecs.add("hevc");
browserDecoderCodecs = browserDecoderCodecs.filter((name) => name !== "hevc");
negotiatedCodec = "h264";
destroyVideoDecoder();
console.warn("rd: falling back from hevc to h264", reasonText);
setCodecModeLabel("h264", "HEVC fallback");
if (ws.readyState === WebSocket.OPEN) {
const q = Number(qualitySlider?.value) || 90;
sendCmd("desktop_set_quality", {
quality: q,
codec: "h264",
softwareH264: useSoftwareH264(),
source: "viewer_fallback",
reason: reasonText,
});
requestEncoderCapabilities();
}
return;
}
fallbackToJpegCodec(reasonText);
}
function recordCanvasFrameTiming(receivedAt, decodeStartedAt) {
const renderedAt = performance.now();
diagnostics.decodeMs = smoothed(diagnostics.decodeMs, Math.max(0, renderedAt - decodeStartedAt));
@@ -2240,16 +2362,16 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
return;
}
if (format === 4) {
setCodecModeLabel("h264", "active");
diagnostics.codec = "h264";
const h264Bytes = buf.slice(headerLength);
if (!h264Bytes.length) {
if (format === 4 || format === 5) {
const codecName = format === 5 ? "hevc" : "h264";
setCodecModeLabel(codecName, "active");
diagnostics.codec = codecName;
const videoBytes = buf.slice(headerLength);
if (!videoBytes.length) {
updateCanvasDecodePressure(true);
return;
}
if (!ensureVideoDecoder(h264Bytes, encodedWidth || frameWidth, encodedHeight || frameHeight)) {
fallbackToJpegCodec("WebCodecs decoder unavailable");
if (!ensureVideoDecoder(videoBytes, encodedWidth || frameWidth, encodedHeight || frameHeight, codecName)) {
updateCanvasDecodePressure(true);
return;
}
@@ -2259,17 +2381,18 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
}
h264FramesSeen += 1;
const isKey = isH264KeyFrame(h264Bytes);
const isKey = codecName === "hevc" ? isHEVCKeyFrame(videoBytes) : isH264KeyFrame(videoBytes);
// If software H264 encode on the agent cannot keep up, automatically
// fall back to JPEG blocks for a smoother interactive stream.
const h264ElapsedMs = performance.now() - h264FirstFrameAt;
if ((fps || 0) <= H264_LOW_FPS_THRESHOLD) {
if (codecName === "h264" && (fps || 0) <= H264_LOW_FPS_THRESHOLD) {
h264LowFpsStreak += 1;
} else {
h264LowFpsStreak = 0;
}
if (
codecName === "h264" &&
h264ElapsedMs >= H264_FALLBACK_WARMUP_MS &&
h264FramesSeen >= H264_MIN_FRAMES_BEFORE_FALLBACK &&
h264LowFpsStreak >= H264_LOW_FPS_STREAK_LIMIT
@@ -2283,7 +2406,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
const chunk = new EncodedVideoChunk({
type: isKey ? "key" : "delta",
timestamp: h264TimestampUs,
data: h264Bytes,
data: videoBytes,
});
h264TimestampUs += frameIntervalUs;
try {
@@ -2294,7 +2417,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
h264KeyframeErrorStreak = 0;
} catch (err) {
h264PendingTimings.pop();
if (isKeyframeRequiredError(err)) {
if (codecName === "h264" && isKeyframeRequiredError(err)) {
h264KeyframeErrorStreak += 1;
const now = Date.now();
if (now - h264LastDecodeWarnAt >= H264_DECODE_WARN_THROTTLE_MS) {
@@ -2313,8 +2436,8 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
updateCanvasDecodePressure(true);
return;
}
console.warn("rd: h264 decode failed", err);
fallbackToJpegCodec(err);
console.warn(`rd: ${codecName} decode failed`, err);
fallbackFromVideoCodec(codecName, err);
updateCanvasDecodePressure(true);
}
return;
Overlord-Server/public/build.html
+1 -1
View File
@@ -865,7 +865,7 @@
<input type="checkbox" name="enable-nvenc" class="w-4 h-4" checked />
<div class="flex flex-col">
<span><i class="fa-solid fa-microchip text-green-400 mr-2"></i>Include NVIDIA NVENC</span>
<span class="text-xs text-slate-500">Native NVIDIA H.264 acceleration for Windows CGO builds</span>
<span class="text-xs text-slate-500">Native NVIDIA H.264 and HEVC acceleration for Windows CGO builds</span>
</div>
</label>
Overlord-Server/public/remotedesktop.html
+2 -2
View File
@@ -204,8 +204,8 @@
</select>
</div>
<div class="flex items-center justify-between gap-2">
<label class="flex items-center gap-2 text-sm text-slate-200" title="Prefer H.264 when available; disable to force JPEG">
<input type="checkbox" id="codecH264" checked /> H.264
<label class="flex items-center gap-2 text-sm text-slate-200" title="Prefer HEVC or H.264 when available; disable to force JPEG">
<input type="checkbox" id="codecH264" checked /> Efficient video (HEVC/H.264)
</label>
<span id="codecMode" class="pill pill-ghost text-slate-100 text-xs">Codec: auto</span>
</div>
Overlord-Server/src/paths.test.ts
+33
View File
@@ -0,0 +1,33 @@
import { expect, test } from "bun:test";
import { rm } from "node:fs/promises";
import { isAbsolute, resolve } from "node:path";
import { tmpdir } from "node:os";
test("creates isolated test data when the Bun preload was not discovered", async () => {
const childEnv = { ...process.env };
delete childEnv.DATA_DIR;
delete childEnv.OVERLORD_TEST_DATA_DIR;
childEnv.NODE_ENV = "test";
const child = Bun.spawn([
"bun",
"-e",
'const { resolveDataDir } = await import("./src/paths.ts"); console.log(resolveDataDir());',
], {
cwd: resolve(import.meta.dirname, ".."),
env: childEnv,
stdout: "pipe",
stderr: "pipe",
});
const [exitCode, stdout, stderr] = await Promise.all([
child.exited,
new Response(child.stdout).text(),
new Response(child.stderr).text(),
]);
expect(exitCode, stderr).toBe(0);
const dataDir = stdout.trim();
expect(isAbsolute(dataDir)).toBe(true);
expect(dataDir.toLowerCase().startsWith(resolve(tmpdir()).toLowerCase())).toBe(true);
await rm(dataDir, { recursive: true, force: true });
});
Overlord-Server/src/paths.ts
+14 -4
View File
@@ -1,7 +1,19 @@
import path from "path";
import { existsSync, mkdirSync } from "fs";
import { existsSync, mkdirSync, mkdtempSync, writeFileSync } from "fs";
import { tmpdir } from "os";
let fallbackTestDataDir = "";
function createFallbackTestDataDir(): string {
if (fallbackTestDataDir) return fallbackTestDataDir;
const dir = mkdtempSync(path.join(tmpdir(), "overlord-bun-test-"));
writeFileSync(path.join(dir, "config.json"), "{}\n", "utf8");
process.env.DATA_DIR = dir;
process.env.OVERLORD_TEST_DATA_DIR = "1";
fallbackTestDataDir = dir;
return dir;
}
function assertSafeTestDataDir(dir: string): void {
if (String(process.env.NODE_ENV || "").toLowerCase() !== "test") return;
@@ -32,9 +44,7 @@ export function resolveDataDir(): string {
}
if (String(process.env.NODE_ENV || "").toLowerCase() === "test") {
throw new Error(
"Refusing to use the default application data directory while NODE_ENV=test. Run tests through `bun test`.",
);
return createFallbackTestDataDir();
}
if (process.platform === "win32" && process.env.APPDATA) {
Overlord-Server/src/protocol.test.ts
+12
View File
@@ -61,4 +61,16 @@ describe("protocol encode/decode", () => {
expect(decoded.header.fps).toBe(24);
expect(Array.from(decoded.data)).toEqual(Array.from(frame.data));
});
test("round trips HEVC frame metadata", () => {
const frame: Frame = {
type: "frame",
header: { monitor: 0, fps: 60, format: "hevc" },
data: new Uint8Array([0, 0, 0, 1, 0x40, 0x01]),
};
const decoded = decodeMessage(encodeMessage(frame)) as Frame;
expect(decoded.header.format).toBe("hevc");
expect(Array.from(decoded.data)).toEqual(Array.from(frame.data));
});
});
Overlord-Server/src/protocol.ts
+25 -1
View File
@@ -158,6 +158,30 @@ export type CommandResult = {
message?: string;
};
export type DesktopCodecCapability = {
codec: "hevc" | "h264" | "jpeg" | "raw" | string;
encoders?: string[];
transports: Array<"websocket" | "webrtc" | string>;
hardware?: boolean;
};
export type DesktopEncoderCapabilities = {
type: "desktop_encoder_capabilities";
commandId?: string;
probed: boolean;
display: number;
profiles: Array<{
maxHeight: number;
width: number;
height: number;
fps: number;
label: string;
providers: string[];
}>;
codecs: DesktopCodecCapability[];
detail?: string;
};
export type ClientLogEntry = {
seq: number;
at: number;
@@ -190,7 +214,7 @@ export type ScreenshotResult = {
export type FrameHeader = {
monitor: number;
fps: number;
format: "jpeg" | "webp" | "raw" | "h264";
format: "jpeg" | "webp" | "raw" | "h264" | "hevc";
width?: number;
height?: number;
hash?: string;
Overlord-Server/src/server/desktop-codec-negotiation.test.ts
+68
View File
@@ -0,0 +1,68 @@
import { describe, expect, test } from "bun:test";
import { negotiateDesktopCodec } from "./desktop-codec-negotiation";
const encoders = [
{ codec: "hevc", transports: ["websocket"] },
{ codec: "h264", transports: ["websocket", "webrtc"] },
{ codec: "jpeg", transports: ["websocket"] },
];
describe("desktop codec negotiation", () => {
test("selects the first mutually supported preferred codec", () => {
expect(negotiateDesktopCodec({
encoderCodecs: encoders,
decoderCodecs: ["hevc", "h264", "jpeg"],
preferredCodecs: ["hevc", "h264", "jpeg"],
transport: "websocket",
})).toEqual({
selectedCodec: "hevc",
fallbackCodecs: ["hevc", "h264", "jpeg"],
transport: "websocket",
});
});
test("filters codecs that are unavailable on WebRTC", () => {
expect(negotiateDesktopCodec({
encoderCodecs: encoders,
decoderCodecs: ["hevc", "h264", "jpeg"],
preferredCodecs: ["hevc", "h264", "jpeg"],
transport: "p2p",
})).toEqual({
selectedCodec: "h264",
fallbackCodecs: ["h264"],
transport: "webrtc",
});
});
test("falls back to JPEG when H.264 decoding is unavailable", () => {
const result = negotiateDesktopCodec({
encoderCodecs: encoders,
decoderCodecs: ["jpeg"],
preferredCodecs: ["h264", "jpeg"],
transport: "websocket",
});
expect(result.selectedCodec).toBe("jpeg");
expect(result.fallbackCodecs).toEqual(["jpeg"]);
});
test("returns no codec instead of selecting an incompatible transport", () => {
const result = negotiateDesktopCodec({
encoderCodecs: encoders,
decoderCodecs: ["jpeg"],
preferredCodecs: ["jpeg"],
transport: "webrtc",
});
expect(result.selectedCodec).toBe("");
expect(result.fallbackCodecs).toEqual([]);
});
test("does not re-add a codec the viewer explicitly omitted", () => {
const result = negotiateDesktopCodec({
encoderCodecs: encoders,
decoderCodecs: ["h264", "jpeg"],
preferredCodecs: ["jpeg"],
transport: "websocket",
});
expect(result.fallbackCodecs).toEqual(["jpeg"]);
});
});
Overlord-Server/src/server/desktop-codec-negotiation.ts
+88
View File
@@ -0,0 +1,88 @@
export type DesktopCodecTransport = "websocket" | "webrtc";
export type DesktopCodecCapability = {
codec: string;
encoders?: string[];
transports?: string[];
hardware?: boolean;
};
export type DesktopCodecNegotiation = {
selectedCodec: string;
fallbackCodecs: string[];
transport: DesktopCodecTransport;
};
const KNOWN_CODECS = new Set(["hevc", "h264", "jpeg", "raw"]);
const DEFAULT_PREFERENCE = ["h264", "jpeg", "raw"];
function normalizeCodec(value: unknown): string {
const codec = String(value || "").trim().toLowerCase();
if (codec === "h265") return "hevc";
if (codec === "mjpeg" || codec === "jpg") return "jpeg";
return KNOWN_CODECS.has(codec) ? codec : "";
}
function normalizeTransport(value: unknown): DesktopCodecTransport {
const transport = String(value || "").trim().toLowerCase();
return transport === "webrtc" || transport === "relayed" || transport === "p2p"
? "webrtc"
: "websocket";
}
function normalizeDeclaredTransport(value: unknown): DesktopCodecTransport | "" {
const transport = String(value || "").trim().toLowerCase();
if (transport === "websocket") return "websocket";
if (transport === "webrtc" || transport === "relayed" || transport === "p2p") return "webrtc";
return "";
}
function decoderCodecSet(values: unknown): Set<string> {
if (!Array.isArray(values)) return new Set();
return new Set(values.map(normalizeCodec).filter(Boolean));
}
function encoderCodecSet(
values: unknown,
transport: DesktopCodecTransport,
): Set<string> {
if (!Array.isArray(values)) return new Set();
const codecs = new Set<string>();
for (const value of values) {
if (typeof value === "string") {
const codec = normalizeCodec(value);
if (codec) codecs.add(codec);
continue;
}
if (!value || typeof value !== "object") continue;
const capability = value as DesktopCodecCapability;
const codec = normalizeCodec(capability.codec);
if (!codec) continue;
const transports = Array.isArray(capability.transports)
? capability.transports.map(normalizeDeclaredTransport).filter(Boolean)
: ["websocket" as const];
if (transports.includes(transport)) codecs.add(codec);
}
return codecs;
}
export function negotiateDesktopCodec(options: {
encoderCodecs: unknown;
decoderCodecs: unknown;
preferredCodecs?: unknown;
transport?: unknown;
}): DesktopCodecNegotiation {
const transport = normalizeTransport(options.transport);
const encoders = encoderCodecSet(options.encoderCodecs, transport);
const decoders = decoderCodecSet(options.decoderCodecs);
const requested = Array.isArray(options.preferredCodecs)
? options.preferredCodecs.map(normalizeCodec).filter(Boolean)
: [];
const preference = [...new Set(requested.length > 0 ? requested : DEFAULT_PREFERENCE)];
const fallbackCodecs = preference.filter((codec) => encoders.has(codec) && decoders.has(codec));
return {
selectedCodec: fallbackCodecs[0] || "",
fallbackCodecs,
transport,
};
}
Overlord-Server/src/server/routes/webrtc-routes.auth.test.ts
+31 -1
View File
@@ -7,7 +7,11 @@ import {
setUserClientAccessScope,
setUserFeaturePermission,
} from "../../users";
import { handleWebrtcRoutes } from "./webrtc-routes";
import {
handleWebrtcRoutes,
revokeWebrtcViewerSessions,
trackWebrtcViewerSession,
} from "./webrtc-routes";
const PASSWORD = "Aa1!WebrtcAuthTestPass_2026";
const createdUserIds: number[] = [];
@@ -57,4 +61,30 @@ describe("WebRTC viewer authorization", () => {
);
expect(response!.status).toBe(403);
});
test("revokes only the matching user's tracked media sessions", async () => {
const deleted: string[] = [];
trackWebrtcViewerSession({
upstreamUrl: "http://mediamtx/session-a",
userId: 1001,
clientId: "client-a",
kind: "desktop",
createdAt: Date.now(),
});
trackWebrtcViewerSession({
upstreamUrl: "http://mediamtx/session-b",
userId: 1002,
clientId: "client-a",
kind: "desktop",
createdAt: Date.now(),
});
const count = await revokeWebrtcViewerSessions(1001, "client-a", async (url) => {
deleted.push(url);
});
expect(count).toBe(1);
expect(deleted).toEqual(["http://mediamtx/session-a"]);
await revokeWebrtcViewerSessions(1002, "client-a", async () => {});
});
});
Overlord-Server/src/server/routes/webrtc-routes.ts
+58
View File
@@ -12,6 +12,50 @@ const PUBLISH_TOKEN_TTL_MS = 60_000;
type PublishToken = { clientId: string; expiresAt: number };
const publishTokens = new Map<string, PublishToken>();
export type ViewerMediaSession = {
upstreamUrl: string;
userId: number;
clientId: string;
kind: WebrtcKind;
createdAt: number;
};
const viewerMediaSessions = new Map<string, ViewerMediaSession>();
export function trackWebrtcViewerSession(session: ViewerMediaSession): void {
const cutoff = Date.now() - 24 * 60 * 60 * 1000;
for (const [upstreamUrl, existing] of viewerMediaSessions) {
if (existing.createdAt < cutoff) viewerMediaSessions.delete(upstreamUrl);
}
viewerMediaSessions.set(session.upstreamUrl, session);
}
export function forgetWebrtcViewerSession(upstreamUrl: string): void {
viewerMediaSessions.delete(upstreamUrl);
}
export async function revokeWebrtcViewerSessions(
userId: number,
clientId: string,
deleteSession: (upstreamUrl: string) => Promise<unknown> = async (upstreamUrl) => {
await fetch(upstreamUrl, { method: "DELETE" });
},
): Promise<number> {
const revoked: ViewerMediaSession[] = [];
for (const [upstreamUrl, session] of viewerMediaSessions) {
if (session.userId !== userId || session.clientId !== clientId) continue;
viewerMediaSessions.delete(upstreamUrl);
revoked.push(session);
}
await Promise.allSettled(revoked.map(async (session) => {
try {
await deleteSession(session.upstreamUrl);
} catch (error) {
logger.warn(`[webrtc] failed to revoke viewer session client=${clientId}: ${(error as Error).message}`);
}
}));
return revoked.length;
}
function prunePublishTokens(now = Date.now()) {
for (const [t, entry] of publishTokens) {
if (entry.expiresAt < now) publishTokens.delete(t);
@@ -66,6 +110,7 @@ export async function handleWebrtcRoutes(req: Request, url: URL): Promise<Respon
const [, streamPath, kind, sessionSuffix] = match;
const clientId = streamPath.split("/")[1];
let viewerUserId: number | undefined;
if (kind === "whip") {
const authHeader = req.headers.get("authorization") || "";
@@ -87,6 +132,7 @@ export async function handleWebrtcRoutes(req: Request, url: URL): Promise<Respon
requireClientAccess(user, clientId);
const streamKind = streamPath.split("/")[2] as WebrtcKind;
requireFeatureAccess(user, WEBRTC_FEATURES[streamKind]);
viewerUserId = user.userId;
} catch (error) {
if (error instanceof Response) return error;
return new Response("Forbidden", { status: 403 });
@@ -123,9 +169,21 @@ export async function handleWebrtcRoutes(req: Request, url: URL): Promise<Respon
if (loc) {
try {
const abs = new URL(loc, MEDIAMTX_URL);
if (kind === "whep" && req.method === "POST" && upstreamResp.ok && viewerUserId !== undefined) {
trackWebrtcViewerSession({
upstreamUrl: abs.toString(),
userId: viewerUserId,
clientId,
kind: streamPath.split("/")[2] as WebrtcKind,
createdAt: Date.now(),
});
}
respHeaders.set("location", `/api/webrtc${abs.pathname}`);
} catch { }
}
if (kind === "whep" && req.method === "DELETE" && upstreamResp.ok) {
forgetWebrtcViewerSession(upstreamUrl);
}
return new Response(upstreamResp.body, {
status: upstreamResp.status,
Overlord-Server/src/server/viewer-authorization.ts
+4
View File
@@ -8,6 +8,7 @@ import {
getUserById,
type FeatureName,
} from "../users";
import { revokeWebrtcViewerSessions } from "./routes/webrtc-routes";
const VIEWER_FEATURES: Partial<Record<SocketRole, FeatureName>> = {
console_viewer: "console",
@@ -31,6 +32,9 @@ const activeViewerSockets = new Set<ServerWebSocket<SocketData>>();
function deny(ws: ServerWebSocket<SocketData>, reason: string): false {
activeViewerSockets.delete(ws);
if (ws.data.userId !== undefined && ws.data.clientId) {
void revokeWebrtcViewerSessions(ws.data.userId, ws.data.clientId);
}
try {
ws.close(1008, reason);
} catch {}
Overlord-Server/src/server/ws-console-rd-backstage.test.ts
+34
View File
@@ -10,6 +10,7 @@ import {
handleRemoteDesktopViewerMessage,
handleRemoteDesktopViewerOpen,
handleDesktopStreamStats,
handleDesktopEncoderCapabilities,
backstageStreamingState,
rdStreamingState,
} from "./ws-console-rd-backstage";
@@ -217,6 +218,39 @@ describe("remote desktop viewer control", () => {
expect(message.transport).toBe("webrtc");
});
test("selects one mutually compatible codec across all viewer transports", () => {
const clientId = `rd-codecs-${Date.now().toString(36)}`;
createClient(clientId);
const canvasViewer = createMockWs({ clientId });
const webrtcViewer = createMockWs({ clientId });
handleRemoteDesktopViewerOpen(canvasViewer as any);
handleRemoteDesktopViewerOpen(webrtcViewer as any);
canvasViewer.data.rdDecoderCodecs = ["hevc", "h264", "jpeg"];
canvasViewer.data.rdPreferredCodecs = ["hevc", "h264", "jpeg"];
canvasViewer.data.rdCodecTransport = "websocket";
webrtcViewer.data.rdDecoderCodecs = ["hevc", "h264"];
webrtcViewer.data.rdPreferredCodecs = ["hevc", "h264"];
webrtcViewer.data.rdCodecTransport = "webrtc";
handleDesktopEncoderCapabilities(clientId, {
type: "desktop_encoder_capabilities",
profiles: [],
codecs: [
{ codec: "hevc", transports: ["websocket"] },
{ codec: "h264", transports: ["websocket", "webrtc"] },
{ codec: "jpeg", transports: ["websocket"] },
],
});
const canvasMessage = decodeMessage(canvasViewer.sent.at(-1) as Uint8Array) as any;
const webrtcMessage = decodeMessage(webrtcViewer.sent.at(-1) as Uint8Array) as any;
expect(canvasMessage.selectedCodec).toBe("h264");
expect(canvasMessage.fallbackCodecs).toEqual(["h264"]);
expect(webrtcMessage.selectedCodec).toBe("h264");
expect(webrtcMessage.fallbackCodecs).toEqual(["h264"]);
});
test("holds Canvas acknowledgements only while the browser decoder is under pressure", () => {
const clientId = `rd-flow-${Date.now().toString(36)}`;
const { agentWs } = createClient(clientId);
Overlord-Server/src/server/ws-console-rd-backstage.ts
+55 -2
View File
@@ -4,6 +4,7 @@ import { existsSync, readFileSync } from "fs";
import path from "path";
import * as clientManager from "../clientManager";
import { logger } from "../logger";
import { negotiateDesktopCodec } from "./desktop-codec-negotiation";
import { metrics } from "../metrics";
import { encodeMessage } from "../protocol";
import { resolveRuntimeRoot } from "./runtime-paths";
@@ -377,6 +378,15 @@ export function handleRemoteDesktopViewerMessage(ws: ServerWebSocket<SocketData>
}
switch (payload.type) {
case "desktop_encoder_capabilities":
ws.data.rdDecoderCodecs = Array.isArray((payload as any).decoderCodecs)
? (payload as any).decoderCodecs.map((codec: unknown) => String(codec || "").trim().toLowerCase()).filter(Boolean).slice(0, 8)
: ["h264", "jpeg", "raw"];
ws.data.rdPreferredCodecs = Array.isArray((payload as any).preferredCodecs)
? (payload as any).preferredCodecs.map((codec: unknown) => String(codec || "").trim().toLowerCase()).filter(Boolean).slice(0, 8)
: ["h264", "jpeg", "raw"];
ws.data.rdCodecTransport = ["webrtc", "relayed", "p2p"].includes(String((payload as any).transport || "").toLowerCase())
? "webrtc"
: "websocket";
sendDesktopCommand(target, "desktop_encoder_capabilities", {
display: Number((payload as any).display) || 0,
});
@@ -488,6 +498,14 @@ export function handleRemoteDesktopViewerMessage(ws: ServerWebSocket<SocketData>
});
break;
}
if (String(state.codec || "").toLowerCase() === "hevc") {
safeSendViewer(ws, {
type: "recording_status",
recording: null,
error: "HEVC stream recording is not supported yet. Switch the stream codec to H.264 or JPEG first.",
});
break;
}
const inputCodec = String(state.codec || "").toLowerCase() === "h264" ? "h264" : "mjpeg";
const compact = !!(payload as any).compact;
const requestedFps = Number((payload as any).fps) || 0;
@@ -1495,8 +1513,43 @@ export function sendbackstageCommand(target: ClientInfo | undefined, commandType
}
export function handleDesktopEncoderCapabilities(clientId: string, payload: any) {
for (const session of sessionManager.getRdSessionsForClient(clientId)) {
safeSendViewer(session.viewer, payload);
const encoderCodecs = Array.isArray(payload?.codecs) && payload.codecs.length > 0
? payload.codecs
: [
{ codec: "h264", transports: ["websocket", "webrtc"] },
{ codec: "jpeg", transports: ["websocket"] },
{ codec: "raw", transports: ["websocket"] },
];
const sessions = sessionManager.getRdSessionsForClient(clientId);
const negotiations = sessions.map((session) => negotiateDesktopCodec({
encoderCodecs,
decoderCodecs: session.viewer.data.rdDecoderCodecs || ["h264", "jpeg", "raw"],
preferredCodecs: session.viewer.data.rdPreferredCodecs,
transport: session.viewer.data.rdCodecTransport,
}));
const sharedFallbackCodecs = negotiations.length > 0
? negotiations[0].fallbackCodecs.filter((codec) =>
negotiations.every((negotiation) => negotiation.fallbackCodecs.includes(codec)),
)
: [];
const selectedCodec = sharedFallbackCodecs[0] || "";
for (let index = 0; index < sessions.length; index++) {
const session = sessions[index];
const negotiation = {
...negotiations[index],
selectedCodec,
fallbackCodecs: sharedFallbackCodecs,
};
session.viewer.data.rdSelectedCodec = selectedCodec;
safeSendViewer(session.viewer, {
...payload,
codecs: encoderCodecs,
negotiation,
selectedCodec,
fallbackCodecs: sharedFallbackCodecs,
transport: negotiation.transport,
});
}
}
Overlord-Server/src/server/ws-viewer-utils.test.ts
+21
View File
@@ -0,0 +1,21 @@
import { describe, expect, test } from "bun:test";
import { buildViewerFrameBuffer } from "./ws-viewer-utils";
describe("viewer frame protocol", () => {
test("encodes HEVC as compact frame format 5", () => {
const payload = new Uint8Array([0, 0, 0, 1, 0x40]);
const frame = buildViewerFrameBuffer(payload, {
format: "hevc",
monitor: 2,
fps: 60,
width: 1920,
height: 1080,
});
expect(frame[3]).toBe(2);
expect(frame[4]).toBe(2);
expect(frame[5]).toBe(60);
expect(frame[6]).toBe(5);
expect(frame.slice(12)).toEqual(payload);
});
});
Overlord-Server/src/server/ws-viewer-utils.ts
+2
View File
@@ -49,6 +49,8 @@ export function buildViewerFrameBuffer(bytes: Uint8Array, header?: any): Uint8Ar
? 3
: header?.format === "h264"
? 4
: header?.format === "hevc"
? 5
: 1;
meta[6] = fmt;
meta[7] = 0;
Overlord-Server/src/sessions/types.ts
+4
View File
@@ -31,6 +31,10 @@ export type SocketData = {
enrollmentNonce?: string;
disconnectReason?: string;
disconnectDetail?: string;
rdDecoderCodecs?: string[];
rdPreferredCodecs?: string[];
rdCodecTransport?: "websocket" | "webrtc";
rdSelectedCodec?: string;
};
export type ConsoleSession = {