vxaboveground/overlord
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

small optimizations + allow for users to force software H264 since it's not slow as balls

Browse Source
This commit is contained in:
KingKunta
2026-07-06 14:00:34 -05:00
parent e0394ea1fa
commit f7e9da26f8
11 changed files with 171 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Overlord-Client/cmd/agent/capture/capture.go
+18 -4
View File
@@ -494,10 +494,11 @@ var (
statBlockRegions atomic.Int64
statBlockFallbacks atomic.Int64
overrideQuality atomic.Int64
overrideCodec atomic.Value
h264WarnOnce sync.Once
codecLogOnce sync.Once
overrideQuality atomic.Int64
overrideCodec atomic.Value
desktopSoftwareH264 atomic.Bool
h264WarnOnce sync.Once
codecLogOnce sync.Once
prevMu sync.Mutex
prevFrame *prevImage
@@ -627,6 +628,19 @@ func RequestDesktopFullFrame() {
RequestDesktopH264Keyframe()
}
func SetDesktopSoftwareH264(enabled bool) {
if desktopSoftwareH264.Swap(enabled) == enabled {
return
}
resetH264Encoder()
requestFullFrames(2)
log.Printf("capture: desktop software h264 %v", enabled)
}
func useDesktopSoftwareH264() bool {
return desktopSoftwareH264.Load()
}
func RequestHVNCFullFrame() {
hvncPrevMu.Lock()
hvncPrevFrame = nil
Overlord-Client/cmd/agent/capture/h264_direct_windows.go
+1 -1
View File
@@ -10,7 +10,7 @@ import (
)
func tryBuildDirectH264Frame(display int) (wire.Frame, time.Duration, time.Duration, bool, error) {
if blockCodec() != "h264" || !useDesktopDuplication() {
if blockCodec() != "h264" || useDesktopSoftwareH264() || !useDesktopDuplication() {
return wire.Frame{}, 0, 0, false, nil
}
forceKeyframe := webrtcpub.ConsumeKeyframeRequest()
Overlord-Client/cmd/agent/capture/h264_encoder_windows.go
+19
View File
@@ -547,6 +547,10 @@ func closeH264Encoder(slot *h264FrameEncoder) {
}
func newWindowsH264Encoder(stream string, width, height, fps int) (h264FrameEncoder, error) {
if stream == "desktop" && useDesktopSoftwareH264() {
log.Printf("capture: desktop software h264 requested; skipping native NVENC and Media Foundation hardware encoders")
return newMFSoftwareH264Encoder(width, height, fps)
}
if enc, err := newNativeH264Encoder(stream, width, height, fps); err == nil {
return enc, nil
} else {
@@ -592,6 +596,21 @@ func newMFH264Encoder(width, height, fps int) (*mfH264Encoder, error) {
if fallbackErr != nil {
return nil, fallbackErr
}
return newMFH264EncoderFromSoftwareTransform(transform, fallbackHardware, fallbackProvider, width, height, fps)
}
func newMFSoftwareH264Encoder(width, height, fps int) (*mfH264Encoder, error) {
if err := ensureMFStartup(); err != nil {
return nil, err
}
transform, fallbackHardware, fallbackProvider, fallbackErr := createSoftwareH264Transform()
if fallbackErr != nil {
return nil, fallbackErr
}
return newMFH264EncoderFromSoftwareTransform(transform, fallbackHardware, fallbackProvider, width, height, fps)
}
func newMFH264EncoderFromSoftwareTransform(transform *mfTransform, fallbackHardware bool, fallbackProvider string, width, height, fps int) (*mfH264Encoder, error) {
enc := newMFH264EncoderFromTransform(transform, width, height, fps, fps, fallbackHardware, fallbackProvider, MFVideoFormat_NV12, "NV12")
if err := enc.configure(); err != nil {
enc.Close()
Overlord-Client/cmd/agent/capture/rgbapool.go
+53 -9
View File
@@ -2,23 +2,38 @@ package capture
import (
"image"
"sync"
"sync/atomic"
)
var rgbaPool sync.Pool
const (
maxPooledRGBABuffers = 8
maxPooledRGBABytes = 64 << 20
maxTotalRGBABytes = 192 << 20
)
var (
rgbaPool = make(chan []byte, maxPooledRGBABuffers)
pooledRGBABytes atomic.Int64
)
func GetRGBA(w, h int) *image.RGBA {
need := w * h * 4
if need <= 0 {
return image.NewRGBA(image.Rect(0, 0, w, h))
}
if v := rgbaPool.Get(); v != nil {
if buf, ok := v.([]byte); ok && cap(buf) >= need {
return &image.RGBA{
Pix: buf[:need],
Stride: w * 4,
Rect: image.Rect(0, 0, w, h),
for attempts := len(rgbaPool); attempts > 0; attempts-- {
select {
case buf := <-rgbaPool:
pooledRGBABytes.Add(-int64(cap(buf)))
if cap(buf) >= need && cap(buf) <= maxReusableCapacity(need) {
return &image.RGBA{
Pix: buf[:need],
Stride: w * 4,
Rect: image.Rect(0, 0, w, h),
}
}
default:
attempts = 0
}
}
return image.NewRGBA(image.Rect(0, 0, w, h))
@@ -28,5 +43,34 @@ func PutRGBA(img *image.RGBA) {
if img == nil || len(img.Pix) == 0 {
return
}
rgbaPool.Put(img.Pix)
buf := img.Pix
capacity := cap(buf)
if capacity <= 0 || capacity > maxPooledRGBABytes {
return
}
for {
cur := pooledRGBABytes.Load()
if cur+int64(capacity) > maxTotalRGBABytes {
return
}
if pooledRGBABytes.CompareAndSwap(cur, cur+int64(capacity)) {
break
}
}
select {
case rgbaPool <- buf:
default:
pooledRGBABytes.Add(-int64(capacity))
}
}
func maxReusableCapacity(need int) int {
limit := need * 2
if limit < need+(1<<20) {
limit = need + (1 << 20)
}
if limit > maxPooledRGBABytes {
limit = maxPooledRGBABytes
}
return limit
}
Overlord-Client/cmd/agent/capture/win_duplication.go
+29 -3
View File
@@ -687,7 +687,9 @@ func (s *duplicationState) closeLocked() {
s.bounds = image.Rectangle{}
s.cursorBounds = image.Rectangle{}
s.display = -1
PutRGBA(s.lastBase)
s.lastBase = nil
PutRGBA(s.lastFrame)
s.lastFrame = nil
s.lastFrameAt = time.Time{}
s.cursorScratch = nil
@@ -807,11 +809,15 @@ func (s *duplicationState) capture(display int) (*image.RGBA, error) {
}
}
// Cache an independent clone so the timeout path (which serves the cached
// Cache an independent copy so the timeout path (which serves the cached
// frame back to callers) doesn't alias a buffer the caller will return to
// the RGBA pool via PutRGBA.
s.lastBase = cloneRGBA(img)
img = s.composeFrame(img, width, height)
captured := img
s.copyLastBaseLocked(captured)
img = s.composeFrame(captured, width, height)
if img != captured {
PutRGBA(captured)
}
if img == s.cursorScratch {
// cursorScratch is reused across captures; clone before returning so
// the next compose doesn't write into a buffer the caller has pooled.
@@ -823,6 +829,26 @@ func (s *duplicationState) capture(display int) (*image.RGBA, error) {
return img, nil
}
func (s *duplicationState) copyLastBaseLocked(src *image.RGBA) {
if src == nil {
PutRGBA(s.lastBase)
s.lastBase = nil
return
}
width := src.Rect.Dx()
height := src.Rect.Dy()
need := len(src.Pix)
if s.lastBase != nil && cap(s.lastBase.Pix) >= need && cap(s.lastBase.Pix) <= need*2 {
s.lastBase.Pix = s.lastBase.Pix[:need]
s.lastBase.Stride = width * 4
s.lastBase.Rect = image.Rect(0, 0, width, height)
copy(s.lastBase.Pix, src.Pix)
return
}
PutRGBA(s.lastBase)
s.lastBase = cloneRGBA(src)
}
func (s *duplicationState) captureH264(display int, forceKeyframe bool) ([]byte, int, int, time.Duration, time.Duration, bool, error) {
capStart := time.Now()
s.mu.Lock()
Overlord-Client/cmd/agent/handlers/command.go
+8 -2
View File
@@ -774,6 +774,7 @@ func HandleCommand(ctx context.Context, env *runtime.Env, envelope map[string]in
payload, _ := envelope["payload"].(map[string]interface{})
quality := 90
codec := ""
softwareH264 := false
reason := ""
source := ""
if payload != nil {
@@ -783,6 +784,9 @@ func HandleCommand(ctx context.Context, env *runtime.Env, envelope map[string]in
if v, ok := payload["codec"].(string); ok {
codec = v
}
if v, ok := payload["softwareH264"].(bool); ok {
softwareH264 = v
}
if v, ok := payload["reason"].(string); ok {
reason = strings.TrimSpace(v)
}
@@ -790,11 +794,13 @@ func HandleCommand(ctx context.Context, env *runtime.Env, envelope map[string]in
source = strings.TrimSpace(v)
}
}
softwareH264 = softwareH264 && strings.EqualFold(strings.TrimSpace(codec), "h264")
if source != "" || reason != "" {
log.Printf("desktop: set quality=%d codec=%s source=%s reason=%s", quality, codec, source, reason)
log.Printf("desktop: set quality=%d codec=%s software_h264=%v source=%s reason=%s", quality, codec, softwareH264, source, reason)
} else {
log.Printf("desktop: set quality=%d codec=%s", quality, codec)
log.Printf("desktop: set quality=%d codec=%s software_h264=%v", quality, codec, softwareH264)
}
capture.SetDesktopSoftwareH264(softwareH264)
capture.SetQualityAndCodec(quality, codec)
sendCommandResultSafe(env, cmdID, true, "")
return nil
Overlord-Server/public/assets/remotedesktop.js
+30 -5
View File
@@ -70,6 +70,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
const qualitySlider = document.getElementById("qualitySlider");
const qualityValue = document.getElementById("qualityValue");
const codecH264 = document.getElementById("codecH264");
const softwareH264Ctrl = document.getElementById("softwareH264Ctrl");
const codecMode = document.getElementById("codecMode");
const canvas = document.getElementById("frameCanvas");
const canvasContainer = document.getElementById("canvasContainer");
@@ -403,6 +404,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
if (kbdCtrl && typeof settings.keyboard === "boolean") kbdCtrl.checked = settings.keyboard;
if (cursorCtrl && typeof settings.cursor === "boolean") cursorCtrl.checked = settings.cursor;
if (duplicationCtrl && typeof settings.duplication === "boolean") duplicationCtrl.checked = settings.duplication;
if (softwareH264Ctrl && typeof settings.softwareH264 === "boolean") softwareH264Ctrl.checked = settings.softwareH264;
if (clipboardSyncCtrl && typeof settings.clipboardSync === "boolean") clipboardSyncCtrl.checked = settings.clipboardSync;
if (audioCtrl && typeof settings.audio === "boolean") audioCtrl.checked = settings.audio;
if (typeof settings.preferH264 === "boolean") {
@@ -423,6 +425,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
keyboard: !!kbdCtrl?.checked,
cursor: !!cursorCtrl?.checked,
duplication: !!duplicationCtrl?.checked,
softwareH264: !!softwareH264Ctrl?.checked,
clipboardSync: !!clipboardSyncCtrl?.checked,
audio: !!audioCtrl?.checked,
audioTransport: getAudioTransport(),
@@ -439,6 +442,9 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
codecH264.checked = prefersH264;
codecH264.disabled = typeof VideoDecoder !== "function";
}
if (softwareH264Ctrl) {
softwareH264Ctrl.disabled = !prefersH264;
}
function setCodecModeLabel(mode, detail) {
if (!codecMode) return;
@@ -1216,12 +1222,17 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
function pushQuality(val) {
const q = Number(val) || 90;
const codec = q >= 100 ? "raw" : (prefersH264 ? "h264" : "jpeg");
console.debug("rd: pushQuality val=", val, "q=", q, "codec=", codec);
const softwareH264 = codec === "h264" && useSoftwareH264();
console.debug("rd: pushQuality val=", val, "q=", q, "codec=", codec, "softwareH264=", softwareH264);
setCodecModeLabel(codec, "requested");
if (codec === "h264") {
if (codec === "h264" && !softwareH264) {
ensureDuplicationForH264();
}
sendCmd("desktop_set_quality", { quality: q, codec });
sendCmd("desktop_set_quality", { quality: q, codec, softwareH264 });
}
function useSoftwareH264() {
return !!softwareH264Ctrl?.checked && prefersH264;
}
function ensureDuplicationForH264() {
@@ -1236,7 +1247,10 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
codecH264.addEventListener("change", function () {
resetH264SessionState();
prefersH264 = !!codecH264.checked && typeof VideoDecoder === "function";
if (prefersH264) {
if (softwareH264Ctrl) {
softwareH264Ctrl.disabled = !prefersH264;
}
if (prefersH264 && !useSoftwareH264()) {
ensureDuplicationForH264();
}
if (!prefersH264) {
@@ -1249,6 +1263,15 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
sharedSettingsSaver.scheduleSave();
});
}
if (softwareH264Ctrl) {
softwareH264Ctrl.addEventListener("change", function () {
resetH264SessionState();
if (qualitySlider) {
pushQuality(qualitySlider.value);
}
sharedSettingsSaver.scheduleSave();
});
}
function pushResolution() {
if (resolutionSelect) {
@@ -1335,7 +1358,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
if (qualitySlider) {
pushQuality(qualitySlider.value);
}
if (prefersH264) {
if (prefersH264 && !useSoftwareH264()) {
ensureDuplicationForH264();
}
pushTargetFps();
@@ -1588,6 +1611,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
prefersH264 = false;
destroyVideoDecoder();
if (codecH264) codecH264.checked = false;
if (softwareH264Ctrl) softwareH264Ctrl.disabled = true;
sharedSettingsSaver.scheduleSave();
console.warn("rd: falling back to jpeg codec", reasonText);
const q = Number(qualitySlider?.value) || 90;
@@ -1636,6 +1660,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
sendCmd("desktop_set_quality", {
quality: q,
codec: "h264",
softwareH264: useSoftwareH264(),
source: "rd_viewer",
reason: "h264_recovery_quality_push",
});
Overlord-Server/public/remotedesktop.html
+3
View File
@@ -198,6 +198,9 @@
</label>
<span id="codecMode" class="pill pill-ghost text-slate-100 text-xs">Codec: auto</span>
</div>
<label class="mt-2 flex items-center gap-2 text-sm text-slate-200" title="Skip NVENC/direct GPU H.264 and use the software H.264 fallback path">
<input type="checkbox" id="softwareH264Ctrl" /> Software H.264
</label>
<label class="mt-2 flex items-center gap-2 text-sm text-slate-200" title="Use GPU desktop duplication (Windows only)">
<input type="checkbox" id="duplicationCtrl" /> GPU Capture
</label>
Overlord-Server/src/server/ws-console-rd-hvnc.test.ts
+1
View File
@@ -132,6 +132,7 @@ describe("remote desktop viewer control", () => {
display: 0,
quality: 90,
codec: "h264",
softwareH264: false,
duplication: true,
maxHeight: 1080,
maxFps: 120,
Overlord-Server/src/server/ws-console-rd-hvnc.ts
+7 -2
View File
@@ -177,6 +177,7 @@ export const rdStreamingState = new Map<string, {
display: number;
quality: number;
codec: string;
softwareH264: boolean;
duplication: boolean;
maxHeight: number;
maxFps: number;
@@ -201,6 +202,7 @@ function defaultRdStreamingState() {
display: 0,
quality: 90,
codec: "",
softwareH264: false,
duplication: false,
maxHeight: 0,
maxFps: 120,
@@ -507,23 +509,26 @@ export function handleRemoteDesktopViewerMessage(ws: ServerWebSocket<SocketData>
case "desktop_set_quality": {
const newQuality = Number(payload.quality) || 90;
const newCodec = String(payload.codec || "").toLowerCase();
const newSoftwareH264 = newCodec === "h264" && !!(payload as any).softwareH264;
const reason = typeof payload.reason === "string"
? payload.reason.slice(0, 512)
: "";
const source = typeof payload.source === "string"
? payload.source.slice(0, 128)
: "";
if (state.quality !== newQuality || state.codec !== newCodec) {
if (state.quality !== newQuality || state.codec !== newCodec || state.softwareH264 !== newSoftwareH264) {
sendDesktopCommand(target, "desktop_set_quality", {
quality: newQuality,
codec: newCodec,
softwareH264: newSoftwareH264,
...(reason ? { reason } : {}),
...(source ? { source } : {}),
});
state.quality = newQuality;
state.codec = newCodec;
state.softwareH264 = newSoftwareH264;
rdStreamingState.set(clientId, state);
logger.debug(`[rd] set quality=${newQuality} codec=${newCodec || "(default)"}${source ? ` source=${source}` : ""}${reason ? ` reason=${reason}` : ""}`);
logger.debug(`[rd] set quality=${newQuality} codec=${newCodec || "(default)"} software_h264=${newSoftwareH264}${source ? ` source=${source}` : ""}${reason ? ` reason=${reason}` : ""}`);
}
break;
}
scripts/start-dev.bat
+2 -1
View File
@@ -27,10 +27,11 @@ set PORT=5173
set OVERLORD_AGENT_TOKEN=dev-token-insecure-local-only
set OVERLORD_DISABLE_AGENT_AUTH=true
set LOG_LEVEL=debug
set CGO_ENABLED=1
start "Overlord-Server" cmd /k "cd /d ""%ROOT%Overlord-Server"" && set OVERLORD_DISABLE_AGENT_AUTH=true && set OVERLORD_AGENT_TOKEN=dev-token-insecure-local-only && set LOG_LEVEL=debug && set NODE_ENV=development && bun install && bun run start"
timeout /t 3 /nobreak >nul
start "Overlord-Client" cmd /k "cd /d ""%ROOT%Overlord-Client"" && set OVERLORD_SERVER=wss://localhost:5173 && set OVERLORD_AGENT_TOKEN=dev-token-insecure-local-only && set OVERLORD_TLS_INSECURE_SKIP_VERIFY=true && set OVERLORD_MODE=dev && set OVERLORD_CAPTURE_METRICS=true && set GOINSECURE=* && set GOSUMDB=off && set GOPROXY=https://proxy.golang.org,direct && go mod tidy && go run ./cmd/agent"
start "Overlord-Client" cmd /k "cd /d ""%ROOT%Overlord-Client"" && set CGO_ENABLED=1 && set OVERLORD_SERVER=wss://localhost:5173 && set OVERLORD_AGENT_TOKEN=dev-token-insecure-local-only && set OVERLORD_TLS_INSECURE_SKIP_VERIFY=true && set OVERLORD_MODE=dev && set OVERLORD_CAPTURE_METRICS=true && set GOINSECURE=* && set GOSUMDB=off && set GOPROXY=https://proxy.golang.org,direct && go mod tidy && go run ./cmd/agent"
echo Done. Terminals stay open (/k) for logs.
endlocal