This commit is contained in:
KingKDot
2026-07-17 13:23:12 -05:00
parent 0219081682
commit 9a274811e8
16 changed files with 594 additions and 29 deletions
@@ -0,0 +1,205 @@
const MIN_BITRATE_MBPS = 2;
const MAX_BITRATE_MBPS = 50;
function finite(value) {
const number = Number(value);
return Number.isFinite(number) ? number : null;
}
export function normalizeAdaptiveProfiles(profiles) {
const seen = new Set();
const normalized = [];
for (const raw of Array.isArray(profiles) ? profiles : []) {
const width = Math.max(0, Math.floor(Number(raw?.width) || 0));
const height = Math.max(0, Math.floor(Number(raw?.height) || 0));
const maxHeight = Math.floor(Number(raw?.maxHeight) || 0);
const fps = Math.max(1, Math.min(240, Math.floor(Number(raw?.fps) || 0)));
if (!height || !fps) continue;
const key = `${width}x${height}@${fps}`;
if (seen.has(key)) continue;
seen.add(key);
normalized.push({ ...raw, width, height, maxHeight, fps });
}
normalized.sort((a, b) => {
const pixelDelta = (b.width * b.height) - (a.width * a.height);
if (pixelDelta) return pixelDelta;
if (b.height !== a.height) return b.height - a.height;
return b.fps - a.fps;
});
return normalized;
}
export function automaticBitrateMbps(profile) {
const pixelsPerSecond = Math.max(1, Number(profile?.width) || 0) *
Math.max(1, Number(profile?.height) || 0) *
Math.max(1, Number(profile?.fps) || 60);
return Math.max(MIN_BITRATE_MBPS, Math.min(18, Math.ceil((pixelsPerSecond * 0.08) / 1_000_000)));
}
export class AdaptiveDesktopQuality {
constructor(onTarget, options = {}) {
this.onTarget = typeof onTarget === "function" ? onTarget : () => {};
this.cooldownMs = Number(options.cooldownMs) || 5000;
this.badSamplesRequired = Number(options.badSamplesRequired) || 2;
this.strainedSamplesRequired = Number(options.strainedSamplesRequired) || 3;
this.stableSamplesRequired = Number(options.stableSamplesRequired) || 10;
this.profiles = [];
this.enabled = false;
this.profileIndex = 0;
this.targetBitrateMbps = 0;
this.badSamples = 0;
this.strainedSamples = 0;
this.stableSamples = 0;
this.bitrateReductionsAtProfile = 0;
this.lastChangeAt = Number.NEGATIVE_INFINITY;
this.lastReason = "";
}
setProfiles(profiles) {
this.profiles = normalizeAdaptiveProfiles(profiles);
this.resetState();
return this.profiles;
}
start(now = 0) {
this.enabled = this.profiles.length > 0;
this.resetState();
if (!this.enabled) return null;
this.lastChangeAt = now;
return this.emit("Auto started at best profile");
}
stop() {
this.enabled = false;
this.resetCounters();
}
current() {
const profile = this.profiles[this.profileIndex] || null;
if (!profile) return null;
return {
profile,
profileIndex: this.profileIndex,
profileCount: this.profiles.length,
bitrateMbps: this.targetBitrateMbps || automaticBitrateMbps(profile),
reason: this.lastReason,
};
}
sample(stats, agentStats, now = Date.now()) {
if (!this.enabled || !this.profiles.length) return null;
const media = stats?.video;
if (!media) return null;
const loss = finite(media.lossPercent) ?? 0;
const rtt = finite(stats?.rttMs);
const jitter = finite(media.jitterMs);
const jitterBuffer = finite(media.jitterBufferMs);
const dropped = finite(media.framesDroppedDelta) ?? 0;
const viewerFps = finite(media.framesPerSecond);
const agentFps = finite(agentStats?.fps);
const available = finite(stats?.availableIncomingMbps);
const fpsRatio = viewerFps != null && agentFps != null && agentFps > 0 ? viewerFps / agentFps : null;
const capacityTight = available != null && available > 0 && this.targetBitrateMbps > available * 0.9;
const bad = loss >= 5 || (rtt != null && rtt >= 250) ||
(jitterBuffer != null && jitterBuffer >= 80) || dropped >= 3 ||
(fpsRatio != null && fpsRatio < 0.75) || (capacityTight && available < this.targetBitrateMbps * 0.7);
const strained = !bad && (loss >= 2 || (rtt != null && rtt >= 160) ||
(jitter != null && jitter >= 35) || (jitterBuffer != null && jitterBuffer >= 40) ||
dropped > 0 || (fpsRatio != null && fpsRatio < 0.9) || capacityTight);
const stable = !bad && !strained && loss < 1 && dropped === 0 &&
(rtt == null || rtt < 120) && (jitterBuffer == null || jitterBuffer < 25) &&
(fpsRatio == null || fpsRatio >= 0.95);
this.badSamples = bad ? this.badSamples + 1 : 0;
this.strainedSamples = strained ? this.strainedSamples + 1 : 0;
this.stableSamples = stable ? this.stableSamples + 1 : 0;
if (now - this.lastChangeAt < this.cooldownMs) return null;
if (this.badSamples >= this.badSamplesRequired) {
return this.reduce("Severe WebRTC congestion", now, 0.72);
}
if (this.strainedSamples >= this.strainedSamplesRequired) {
return this.reduce("Sustained WebRTC congestion", now, 0.84);
}
if (this.stableSamples >= this.stableSamplesRequired) {
return this.increase("WebRTC connection stable", now);
}
return null;
}
resetState() {
this.profileIndex = 0;
this.targetBitrateMbps = this.profiles[0] ? automaticBitrateMbps(this.profiles[0]) : 0;
this.bitrateReductionsAtProfile = 0;
this.lastReason = "";
this.resetCounters();
}
resetCounters() {
this.badSamples = 0;
this.strainedSamples = 0;
this.stableSamples = 0;
}
reduce(reason, now, factor) {
const profile = this.profiles[this.profileIndex];
const floor = Math.max(MIN_BITRATE_MBPS, Math.ceil(automaticBitrateMbps(profile) * 0.45));
const reduced = Math.max(floor, Math.floor(this.targetBitrateMbps * factor));
if (reduced < this.targetBitrateMbps && this.bitrateReductionsAtProfile < 1) {
this.targetBitrateMbps = reduced;
this.bitrateReductionsAtProfile += 1;
} else if (this.profileIndex < this.profiles.length - 1) {
this.profileIndex += 1;
this.bitrateReductionsAtProfile = 0;
this.targetBitrateMbps = Math.min(
Math.max(MIN_BITRATE_MBPS, this.targetBitrateMbps),
automaticBitrateMbps(this.profiles[this.profileIndex]),
);
} else if (reduced < this.targetBitrateMbps) {
this.targetBitrateMbps = reduced;
} else {
this.resetCounters();
return null;
}
this.lastChangeAt = now;
this.resetCounters();
return this.emit(reason);
}
increase(reason, now) {
const nominal = automaticBitrateMbps(this.profiles[this.profileIndex]);
if (this.targetBitrateMbps < nominal) {
this.targetBitrateMbps = Math.min(nominal, Math.max(this.targetBitrateMbps + 1, Math.ceil(this.targetBitrateMbps * 1.12)));
} else if (this.profileIndex > 0) {
this.profileIndex -= 1;
this.bitrateReductionsAtProfile = 0;
this.targetBitrateMbps = Math.min(this.targetBitrateMbps, automaticBitrateMbps(this.profiles[this.profileIndex]));
} else {
this.resetCounters();
return null;
}
this.lastChangeAt = now;
this.resetCounters();
return this.emit(reason);
}
emit(reason) {
this.lastReason = reason;
const current = this.current();
if (!current) return null;
const target = {
maxHeight: current.profile.maxHeight,
fps: current.profile.fps,
width: current.profile.width,
height: current.profile.height,
bitrateMbps: Math.max(MIN_BITRATE_MBPS, Math.min(MAX_BITRATE_MBPS, Math.round(current.bitrateMbps))),
profileIndex: current.profileIndex,
profileCount: current.profileCount,
reason,
};
this.onTarget(target);
return target;
}
}
@@ -0,0 +1,52 @@
import { describe, expect, test } from "bun:test";
import { AdaptiveDesktopQuality, automaticBitrateMbps, normalizeAdaptiveProfiles } from "./rd-adaptive-quality.js";
const profiles = [
{ maxHeight: 720, width: 1280, height: 720, fps: 60 },
{ maxHeight: 1080, width: 1920, height: 1080, fps: 60 },
{ maxHeight: -1, width: 2560, height: 1440, fps: 144 },
];
describe("adaptive remote desktop quality", () => {
test("ranks native resolution first and starts at the best profile", () => {
const ranked = normalizeAdaptiveProfiles(profiles);
expect(ranked.map((profile) => profile.height)).toEqual([1440, 1080, 720]);
const targets: any[] = [];
const controller = new AdaptiveDesktopQuality((target) => targets.push(target));
controller.setProfiles(profiles);
controller.start(0);
expect(targets[0].height).toBe(1440);
expect(targets[0].fps).toBe(144);
expect(targets[0].bitrateMbps).toBe(automaticBitrateMbps(ranked[0]));
});
test("requires sustained congestion before reducing bitrate and then profile", () => {
const targets: any[] = [];
const controller = new AdaptiveDesktopQuality((target) => targets.push(target), { cooldownMs: 1 });
controller.setProfiles(profiles);
controller.start(0);
const bad = { rttMs: 300, video: { lossPercent: 7, framesDroppedDelta: 4, framesPerSecond: 40 } };
expect(controller.sample(bad, { fps: 144 }, 10)).toBeNull();
expect(controller.sample(bad, { fps: 144 }, 11)?.height).toBe(1440);
expect(controller.sample(bad, { fps: 144 }, 20)).toBeNull();
expect(controller.sample(bad, { fps: 144 }, 21)?.height).toBe(1080);
});
test("slowly recovers after stable feedback", () => {
const controller = new AdaptiveDesktopQuality(() => {}, {
cooldownMs: 1,
badSamplesRequired: 1,
stableSamplesRequired: 2,
});
controller.setProfiles(profiles);
controller.start(0);
const bad = { rttMs: 300, video: { lossPercent: 8, framesDroppedDelta: 5, framesPerSecond: 20 } };
controller.sample(bad, { fps: 144 }, 2);
controller.sample(bad, { fps: 144 }, 4);
expect(controller.current()?.profile.height).toBe(1080);
const stable = { rttMs: 30, video: { lossPercent: 0, framesDroppedDelta: 0, framesPerSecond: 60, jitterBufferMs: 5 } };
expect(controller.sample(stable, { fps: 60 }, 6)).toBeNull();
const recovered = controller.sample(stable, { fps: 60 }, 7);
expect(recovered?.height).toBe(1440);
});
});
+75 -11
View File
@@ -2,6 +2,7 @@ import { encodeMsgpack, decodeMsgpack } from "./msgpack-helpers.js";
import { checkFeatureAccess } from "./feature-gate.js";
import { WhepClient } from "./whep.js";
import { P2PClient } from "./webrtc-p2p.js";
import { AdaptiveDesktopQuality, normalizeAdaptiveProfiles } from "./rd-adaptive-quality.js";
import { createKeyboardCapture } from "./keyboard-capture.js";
import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-settings.js";
@@ -158,6 +159,25 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
width: 0,
height: 0,
};
let encoderProfiles = [];
let adaptiveProfileRunning = false;
const adaptiveQuality = new AdaptiveDesktopQuality((target) => {
sendCmd("desktop_set_profile", {
maxHeight: target.maxHeight,
fps: target.fps,
source: "webrtc_auto",
reason: target.reason,
});
sendCmd("desktop_set_bitrate", {
bitrateMbps: target.bitrateMbps,
adaptive: true,
source: "webrtc_auto",
reason: target.reason,
});
if (streamProfileDetail) {
streamProfileDetail.textContent = `Auto: ${target.width}×${target.height} @ ${target.fps} FPS, ${target.bitrateMbps} Mbps. ${target.reason}.`;
}
});
let clipboardSyncTimer = null;
let lastClipboardText = "";
@@ -404,7 +424,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
}
let savedDisplay = null;
let savedStreamProfile = null;
let savedStreamProfile = "auto";
function setSelectValue(select, value) {
if (!select || value === undefined || value === null) return false;
@@ -455,7 +475,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
const profile = selectedStreamProfile();
return {
display: Number(displaySelect?.value || 0),
streamProfile: streamProfileSelect?.value || "1080:60",
streamProfile: streamProfileSelect?.value || "auto",
// Keep the legacy fields so backstage and older remote-desktop builds that
// share these preferences continue to receive equivalent settings.
resolution: String(profile.maxHeight),
@@ -977,6 +997,9 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
if (Number.isFinite(media?.framesDropped)) details.push(`Frames dropped: ${media.framesDropped}`);
if (Number.isFinite(stats.availableIncomingMbps)) details.push(`Available: ${stats.availableIncomingMbps.toFixed(1)} Mbps`);
networkStats.title = details.join("\n");
if (adaptiveProfileRunning) {
adaptiveQuality.sample(stats, diagnostics.agent, Date.now());
}
renderDiagnostics();
}
@@ -1479,7 +1502,11 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
}
function selectedStreamProfile() {
const [heightValue, fpsValue] = String(streamProfileSelect?.value || "1080:60").split(":");
if (streamProfileSelect?.value === "auto") {
const best = encoderProfiles[0];
return best ? { maxHeight: best.maxHeight, fps: best.fps } : { maxHeight: -1, fps: 120 };
}
const [heightValue, fpsValue] = String(streamProfileSelect?.value || "auto").split(":");
const maxHeight = Number.parseInt(heightValue, 10);
const fps = Number.parseInt(fpsValue, 10);
return {
@@ -1489,6 +1516,29 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
}
function pushStreamProfile() {
if (streamProfileSelect?.value === "auto") {
const adaptiveWebrtc = getWebrtcMode() !== "off" && Number(bitrateSelect?.value || 0) === 0;
if (adaptiveWebrtc && encoderProfiles.length) {
adaptiveQuality.setProfiles(encoderProfiles);
adaptiveProfileRunning = true;
adaptiveQuality.start(Date.now());
return;
}
adaptiveProfileRunning = false;
adaptiveQuality.stop();
const best = selectedStreamProfile();
console.debug("rd: pushStreamProfile auto best", best);
sendCmd("desktop_set_profile", { ...best, source: "auto_best" });
if (streamProfileDetail) {
const profile = encoderProfiles[0];
streamProfileDetail.textContent = profile
? `Auto selected ${profile.width}×${profile.height} @ ${profile.fps} FPS (best available).`
: "Auto selected native resolution while capabilities are loading.";
}
return;
}
adaptiveProfileRunning = false;
adaptiveQuality.stop();
const profile = selectedStreamProfile();
console.debug("rd: pushStreamProfile", profile);
sendCmd("desktop_set_profile", profile);
@@ -1496,11 +1546,13 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
function pushBitrate() {
const bitrateMbps = Math.max(0, Math.min(50, Number.parseInt(bitrateSelect?.value || "0", 10) || 0));
sendCmd("desktop_set_bitrate", { bitrateMbps });
if (adaptiveProfileRunning && bitrateMbps === 0) return;
sendCmd("desktop_set_bitrate", { bitrateMbps, adaptive: false });
}
if (bitrateSelect) {
bitrateSelect.addEventListener("change", function () {
pushStreamProfile();
pushBitrate();
sharedSettingsSaver.scheduleSave();
});
@@ -1518,8 +1570,14 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
const selectedDisplay = Number.parseInt(displaySelect?.value || "0", 10) || 0;
if (Number.isFinite(Number(msg.display)) && Number(msg.display) !== selectedDisplay) return;
const previous = streamProfileSelect.value;
encoderProfiles = normalizeAdaptiveProfiles(msg.profiles);
streamProfileSelect.innerHTML = "";
for (const profile of msg.profiles) {
const autoOption = document.createElement("option");
autoOption.value = "auto";
autoOption.textContent = "Auto (best)";
autoOption.title = "Starts at the best supported profile and adapts to WebRTC congestion";
streamProfileSelect.appendChild(autoOption);
for (const profile of encoderProfiles) {
const maxHeight = Number(profile.maxHeight);
const fps = Number(profile.fps);
if (!Number.isFinite(maxHeight) || !Number.isFinite(fps)) continue;
@@ -1533,22 +1591,25 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
}
streamProfileSelect.appendChild(option);
}
if (!streamProfileSelect.options.length) {
if (streamProfileSelect.options.length === 1) {
const option = document.createElement("option");
option.value = "1080:60";
option.textContent = "60 FPS - 1080p";
streamProfileSelect.appendChild(option);
}
if (!setSelectValue(streamProfileSelect, savedStreamProfile) &&
if (!setSelectValue(streamProfileSelect, savedStreamProfile || "auto") &&
!setSelectValue(streamProfileSelect, previous) &&
!setSelectValue(streamProfileSelect, "1080:60")) {
streamProfileSelect.selectedIndex = 0;
!setSelectValue(streamProfileSelect, "auto")) {
streamProfileSelect.value = "auto";
}
savedStreamProfile = streamProfileSelect.value;
if (streamProfileDetail) {
const providers = streamProfileSelect.selectedOptions[0]?.dataset?.providers || "";
streamProfileDetail.textContent = providers ? `Available through ${providers}.` :
(msg.detail || (msg.probed ? "Profiles tested on this display adapter." : "Safe fallback profiles shown."));
const best = encoderProfiles[0];
streamProfileDetail.textContent = streamProfileSelect.value === "auto" && best
? `Auto starts at ${best.width}×${best.height} @ ${best.fps} FPS and adapts on WebRTC.`
: (providers ? `Available through ${providers}.` :
(msg.detail || (msg.probed ? "Profiles tested on this display adapter." : "Safe fallback profiles shown.")));
}
if (desiredStreaming) pushStreamProfile();
}
@@ -1559,6 +1620,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
const providers = streamProfileSelect.selectedOptions[0]?.dataset?.providers || "";
if (streamProfileDetail && providers) streamProfileDetail.textContent = `Available through ${providers}.`;
pushStreamProfile();
pushBitrate();
sharedSettingsSaver.scheduleSave();
});
}
@@ -1642,6 +1704,8 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
stopBtn.addEventListener("click", function () {
if (isRecording()) stopRecording();
desiredStreaming = false;
adaptiveProfileRunning = false;
adaptiveQuality.stop();
lastFrameAt = 0;
setStreamState("stopping", "Stopping stream");
disablePrivacyIfActive();
@@ -75,6 +75,7 @@ export class WebRTCStatsSampler {
let decodeMs = null;
let processingDelayMs = null;
let framesDroppedDelta = null;
let intervalLossPercent = null;
if (previous && timestamp > previous.timestamp && Number(item.bytesReceived) >= previous.bytesReceived) {
bitrateMbps = ((Number(item.bytesReceived) - previous.bytesReceived) * 8) /
((timestamp - previous.timestamp) * 1000);
@@ -87,6 +88,10 @@ export class WebRTCStatsSampler {
}
const droppedDelta = (Number(item.framesDropped) || 0) - previous.framesDropped;
if (droppedDelta >= 0) framesDroppedDelta = droppedDelta;
const receivedDelta = (Number(item.packetsReceived) || 0) - previous.packetsReceived;
const lostDelta = (Number(item.packetsLost) || 0) - previous.packetsLost;
const packetDelta = Math.max(0, receivedDelta) + Math.max(0, lostDelta);
if (packetDelta > 0) intervalLossPercent = (Math.max(0, lostDelta) * 100) / packetDelta;
}
this.previous.set(item.id, {
timestamp,
@@ -95,6 +100,8 @@ export class WebRTCStatsSampler {
totalDecodeTime: Number(item.totalDecodeTime) || 0,
totalProcessingDelay: Number(item.totalProcessingDelay) || 0,
framesDropped: Number(item.framesDropped) || 0,
packetsReceived: Number(item.packetsReceived) || 0,
packetsLost: Number(item.packetsLost) || 0,
});
const packetsReceived = Number(item.packetsReceived) || 0;
const packetsLost = Math.max(0, Number(item.packetsLost) || 0);
@@ -102,7 +109,7 @@ export class WebRTCStatsSampler {
result[kind] = {
bitrateMbps,
packetsLost,
lossPercent: packetTotal > 0 ? (packetsLost * 100) / packetTotal : 0,
lossPercent: intervalLossPercent ?? (packetTotal > 0 ? (packetsLost * 100) / packetTotal : 0),
jitterMs: Number.isFinite(item.jitter) ? item.jitter * 1000 : null,
jitterBufferMs: averageJitterBufferMs(item),
decodeMs,
+4 -3
View File
@@ -180,18 +180,19 @@
<select
id="streamProfileSelect"
class="flex-1 bg-slate-800 border border-slate-700 rounded px-2 py-1 text-sm text-slate-100"
title="Resolution and frame-rate combinations supported by this computer"
title="Auto starts with the best supported resolution and adapts to WebRTC network conditions"
>
<option value="auto" selected>Auto (best)</option>
<option value="720:30">30 FPS - 720p</option>
<option value="720:60">60 FPS - 720p</option>
<option value="1080:30">30 FPS - 1080p</option>
<option value="1080:60" selected>60 FPS - 1080p</option>
<option value="1080:60">60 FPS - 1080p</option>
</select>
</div>
<div id="streamProfileDetail" class="mb-2 text-xs text-slate-500">Checking hardware encoder profiles…</div>
<div class="flex items-center gap-2 mb-2">
<label class="text-sm text-slate-300 w-20" for="bitrateSelect">Bitrate</label>
<select id="bitrateSelect" class="flex-1 bg-slate-800 border border-slate-700 rounded px-2 py-1 text-sm text-slate-100" title="H.264 target bitrate; Auto derives it from resolution and FPS">
<select id="bitrateSelect" class="flex-1 bg-slate-800 border border-slate-700 rounded px-2 py-1 text-sm text-slate-100" title="Auto adapts bitrate from receiver feedback when the stream profile is also Auto">
<option value="0" selected>Auto</option>
<option value="5">5 Mbps</option>
<option value="10">10 Mbps</option>