From 69b3e7833c2d12ef5cd246d2967f2680b0b55a3c Mon Sep 17 00:00:00 2001 From: KingKDot Date: Fri, 3 Apr 2026 19:43:41 -0500 Subject: [PATCH] linux capturing fix (it fucking sucks) --- Overlord-Client/cmd/agent/capture/capture.go | 17 +- .../cmd/agent/capture/capture_linux.go | 106 ++++++- .../cmd/agent/capture/monitors_linux.go | 13 +- .../cmd/agent/capture/reset_capture_linux.go | 7 + .../cmd/agent/capture/reset_capture_stub.go | 2 +- .../cmd/agent/capture/win_bitblt.go | 11 + .../cmd/agent/capture/x11_capture.go | 279 ++++++++++++++++++ .../cmd/agent/handlers/input_linux.go | 13 + .../cmd/agent/handlers/input_stub.go | 2 +- Overlord-Client/go.mod | 4 +- 10 files changed, 445 insertions(+), 9 deletions(-) create mode 100644 Overlord-Client/cmd/agent/capture/reset_capture_linux.go create mode 100644 Overlord-Client/cmd/agent/capture/x11_capture.go create mode 100644 Overlord-Client/cmd/agent/handlers/input_linux.go diff --git a/Overlord-Client/cmd/agent/capture/capture.go b/Overlord-Client/cmd/agent/capture/capture.go index cf2ccb2..1403898 100644 --- a/Overlord-Client/cmd/agent/capture/capture.go +++ b/Overlord-Client/cmd/agent/capture/capture.go @@ -98,6 +98,14 @@ func CaptureAndSend(ctx context.Context, env *rt.Env) error { log.Printf("capture: requested display %d out of range, defaulting to 0 (monitors=%d)", env.SelectedDisplay, displays) } t0 := time.Now() + + fails := consecutiveCaptureFails.Load() + if fails >= 3 { + ResetDesktopCapture() + ResetMonitorCache() + time.Sleep(50 * time.Millisecond) + } + img, err := safeCaptureDisplay(display) if err != nil { img, err = safeCaptureDisplay(display) @@ -108,13 +116,16 @@ func CaptureAndSend(ctx context.Context, env *rt.Env) error { } } if err != nil { - log.Printf("capture: capture failed: %v (sending black frame)", err) + consecutiveCaptureFails.Add(1) + log.Printf("capture: capture failed: %v (sending black frame, consecutive=%d)", err, consecutiveCaptureFails.Load()) return sendBlackFrame(ctx, env) } if img == nil { - log.Printf("capture: capture returned nil image (sending black frame)") + consecutiveCaptureFails.Add(1) + log.Printf("capture: capture returned nil image (sending black frame, consecutive=%d)", consecutiveCaptureFails.Load()) return sendBlackFrame(ctx, env) } + consecutiveCaptureFails.Store(0) captureDur := time.Since(t0) quality := jpegQuality() @@ -323,6 +334,8 @@ func MonitorCount() int { const frameLogInterval = 5 * time.Second +var consecutiveCaptureFails atomic.Int64 + var ( fpsWindowStart atomic.Int64 fpsCount atomic.Int64 diff --git a/Overlord-Client/cmd/agent/capture/capture_linux.go b/Overlord-Client/cmd/agent/capture/capture_linux.go index f38be42..a4adbec 100644 --- a/Overlord-Client/cmd/agent/capture/capture_linux.go +++ b/Overlord-Client/cmd/agent/capture/capture_linux.go @@ -4,18 +4,122 @@ package capture import ( "image" + "log" + "os" + "sync" + "sync/atomic" "github.com/kbinani/screenshot" ) +var isWaylandSession = os.Getenv("XDG_SESSION_TYPE") == "wayland" || os.Getenv("WAYLAND_DISPLAY") != "" + +var ( + x11Disabled atomic.Bool + x11BlackCount atomic.Int64 + x11CheckOnce sync.Once + x11MaxBlackFrames int64 = 3 +) + +func init() { + if isWaylandSession { + log.Printf("capture: Wayland session detected (XDG_SESSION_TYPE=%s WAYLAND_DISPLAY=%s), using screenshot library (D-Bus portal)", + os.Getenv("XDG_SESSION_TYPE"), os.Getenv("WAYLAND_DISPLAY")) + } +} + var activeDisplays = func() int { + if !isWaylandSession && !x11Disabled.Load() { + if n := x11DisplayCount(); n > 0 { + return n + } + } return screenshot.NumActiveDisplays() } var captureDisplayFn = func(display int) (*image.RGBA, error) { - return screenshot.CaptureDisplay(display) + if isWaylandSession || x11Disabled.Load() { + return captureViaLibrary(display) + } + + img, err := x11CaptureDisplay(display) + if err != nil { + log.Printf("x11 capture: error, falling back to screenshot library: %v", err) + return captureViaLibrary(display) + } + + if isAllBlack(img) { + n := x11BlackCount.Add(1) + if n >= x11MaxBlackFrames { + log.Printf("x11 capture: %d consecutive all-black frames, disabling X11 capture permanently", n) + x11Disabled.Store(true) + x11Reset() + return captureViaLibrary(display) + } + libImg, libErr := captureViaLibrary(display) + if libErr == nil && !isAllBlack(libImg) { + log.Printf("x11 capture: X11 returned black but screenshot library works, disabling X11 capture") + x11Disabled.Store(true) + x11Reset() + return libImg, nil + } + return img, nil + } + x11BlackCount.Store(0) + return img, nil +} + +func captureViaLibrary(display int) (*image.RGBA, error) { + bounds := screenshot.GetDisplayBounds(display) + img, err := screenshot.CaptureRect(bounds) + if err != nil { + return nil, err + } + if img.Rect.Min.X != 0 || img.Rect.Min.Y != 0 { + img.Rect = image.Rect(0, 0, img.Rect.Dx(), img.Rect.Dy()) + } + return img, nil +} + +func isAllBlack(img *image.RGBA) bool { + if img == nil { + return true + } + pix := img.Pix + stride := img.Stride + w := img.Rect.Dx() + h := img.Rect.Dy() + if w == 0 || h == 0 { + return true + } + stepX := w / 4 + stepY := h / 4 + if stepX < 1 { + stepX = 1 + } + if stepY < 1 { + stepY = 1 + } + for y := stepY / 2; y < h; y += stepY { + row := y * stride + for x := stepX / 2; x < w; x += stepX { + off := row + x*4 + if off+2 >= len(pix) { + continue + } + if pix[off] != 0 || pix[off+1] != 0 || pix[off+2] != 0 { + return false + } + } + } + return true } func displayCount() int { + if !isWaylandSession && !x11Disabled.Load() { + if n := x11DisplayCount(); n > 0 { + return n + } + } return screenshot.NumActiveDisplays() } diff --git a/Overlord-Client/cmd/agent/capture/monitors_linux.go b/Overlord-Client/cmd/agent/capture/monitors_linux.go index 9038a8b..7523b3d 100644 --- a/Overlord-Client/cmd/agent/capture/monitors_linux.go +++ b/Overlord-Client/cmd/agent/capture/monitors_linux.go @@ -9,6 +9,15 @@ import ( ) func displayBounds(idx int) image.Rectangle { + if !isWaylandSession && !x11Disabled.Load() { + if b := x11DisplayBounds(idx); b.Dx() > 0 && b.Dy() > 0 { + return b + } + } + n := screenshot.NumActiveDisplays() + if idx < 0 || idx >= n { + return image.Rectangle{} + } return screenshot.GetDisplayBounds(idx) } @@ -17,10 +26,10 @@ func displayScale(idx int) float64 { } func MonitorInfos() []MonitorInfo { - n := screenshot.NumActiveDisplays() + n := displayCount() infos := make([]MonitorInfo, 0, n) for i := 0; i < n; i++ { - b := screenshot.GetDisplayBounds(i) + b := displayBounds(i) infos = append(infos, MonitorInfo{ Width: b.Dx(), Height: b.Dy(), diff --git a/Overlord-Client/cmd/agent/capture/reset_capture_linux.go b/Overlord-Client/cmd/agent/capture/reset_capture_linux.go new file mode 100644 index 0000000..3c0509f --- /dev/null +++ b/Overlord-Client/cmd/agent/capture/reset_capture_linux.go @@ -0,0 +1,7 @@ +//go:build linux + +package capture + +func ResetDesktopCapture() { + x11Reset() +} diff --git a/Overlord-Client/cmd/agent/capture/reset_capture_stub.go b/Overlord-Client/cmd/agent/capture/reset_capture_stub.go index 261d7af..a021462 100644 --- a/Overlord-Client/cmd/agent/capture/reset_capture_stub.go +++ b/Overlord-Client/cmd/agent/capture/reset_capture_stub.go @@ -1,4 +1,4 @@ -//go:build !windows +//go:build !windows && !linux package capture diff --git a/Overlord-Client/cmd/agent/capture/win_bitblt.go b/Overlord-Client/cmd/agent/capture/win_bitblt.go index a522522..6bebe7e 100644 --- a/Overlord-Client/cmd/agent/capture/win_bitblt.go +++ b/Overlord-Client/cmd/agent/capture/win_bitblt.go @@ -166,6 +166,10 @@ var ( lastCapScale atomic.Uint64 state capState captureMu sync.Mutex + + stateCreatedAt time.Time + + stateRefreshInterval = 30 * time.Second ) var captureDisplayFn = func(display int) (*image.RGBA, error) { @@ -324,8 +328,13 @@ func captureDisplayBitBlt(display int) (*image.RGBA, error) { return img, bitDur, dibDur, convDur, nil } + if !stateCreatedAt.IsZero() && time.Since(stateCreatedAt) > stateRefreshInterval { + state.reset() + } + img, bitDur, dibDur, convDur, err := captureWithDC() if err != nil && fromCreateDC { + state.reset() closeDC() hdcScreen = getDC(0) fromCreateDC = false @@ -335,6 +344,7 @@ func captureDisplayBitBlt(display int) (*image.RGBA, error) { img, bitDur, dibDur, convDur, err = captureWithDC() } if err != nil { + state.reset() return nil, err } @@ -456,6 +466,7 @@ func (s *capState) ensure(hdcScreen uintptr, w, h int) (uintptr, uintptr, []byte if s.hdcMem == 0 { return 0, 0, nil, 0, syscall.EINVAL } + stateCreatedAt = time.Now() } if s.w != w || s.h != h || s.hbmp == 0 { diff --git a/Overlord-Client/cmd/agent/capture/x11_capture.go b/Overlord-Client/cmd/agent/capture/x11_capture.go new file mode 100644 index 0000000..1a05c90 --- /dev/null +++ b/Overlord-Client/cmd/agent/capture/x11_capture.go @@ -0,0 +1,279 @@ +//go:build linux + +package capture + +import ( + "fmt" + "image" + "log" + "sync" + + "github.com/gen2brain/shm" + "github.com/jezek/xgb" + xshm "github.com/jezek/xgb/shm" + "github.com/jezek/xgb/xinerama" + "github.com/jezek/xgb/xproto" +) + +type x11State struct { + mu sync.Mutex + + conn *xgb.Conn + screen *xproto.ScreenInfo + root xproto.Window + + screens []xinerama.ScreenInfo + primary xinerama.ScreenInfo + + shmSeg xshm.Seg + shmId int + shmData []byte + shmW int + shmH int + shmReady bool + useShm bool + + imgCache map[int]*image.RGBA +} + +var x11 x11State + +func x11Init() error { + x11.mu.Lock() + defer x11.mu.Unlock() + return x11InitLocked() +} + +func x11InitLocked() error { + if x11.conn != nil { + return nil + } + + c, err := xgb.NewConn() + if err != nil { + return fmt.Errorf("x11 capture: connect: %w", err) + } + + if err := xinerama.Init(c); err != nil { + c.Close() + return fmt.Errorf("x11 capture: xinerama init: %w", err) + } + + reply, err := xinerama.QueryScreens(c).Reply() + if err != nil || reply.Number == 0 { + c.Close() + if err != nil { + return fmt.Errorf("x11 capture: query screens: %w", err) + } + return fmt.Errorf("x11 capture: no screens found") + } + + x11.conn = c + x11.screen = xproto.Setup(c).DefaultScreen(c) + x11.root = x11.screen.Root + x11.screens = reply.ScreenInfo + x11.primary = reply.ScreenInfo[0] + x11.imgCache = make(map[int]*image.RGBA) + + x11.useShm = true + if err := xshm.Init(c); err != nil { + log.Printf("x11 capture: shm extension not available, will use GetImage (slower)") + x11.useShm = false + } + + log.Printf("x11 capture: connected, %d screens, shm=%v, screen=%dx%d", + len(x11.screens), x11.useShm, + x11.screen.WidthInPixels, x11.screen.HeightInPixels) + return nil +} + +func x11ResetLocked() { + if x11.shmReady { + if x11.conn != nil { + xshm.Detach(x11.conn, x11.shmSeg) + } + _ = shm.Dt(x11.shmData) + _ = shm.Rm(x11.shmId) + x11.shmReady = false + x11.shmData = nil + x11.shmW = 0 + x11.shmH = 0 + } + if x11.conn != nil { + x11.conn.Close() + x11.conn = nil + } + x11.screens = nil + x11.imgCache = nil +} + +func x11Reset() { + x11.mu.Lock() + defer x11.mu.Unlock() + x11ResetLocked() +} + +func x11EnsureShm(w, h int) error { + if x11.shmReady && x11.shmW == w && x11.shmH == h { + return nil + } + + if x11.shmReady { + xshm.Detach(x11.conn, x11.shmSeg) + _ = shm.Dt(x11.shmData) + _ = shm.Rm(x11.shmId) + x11.shmReady = false + } + + size := w * h * 4 + shmId, err := shm.Get(shm.IPC_PRIVATE, size, shm.IPC_CREAT|0o777) + if err != nil { + return fmt.Errorf("shm.Get(%d): %w", size, err) + } + + seg, err := xshm.NewSegId(x11.conn) + if err != nil { + _ = shm.Rm(shmId) + return fmt.Errorf("shm.NewSegId: %w", err) + } + + data, err := shm.At(shmId, 0, 0) + if err != nil { + _ = shm.Rm(shmId) + return fmt.Errorf("shm.At: %w", err) + } + + xshm.Attach(x11.conn, seg, uint32(shmId), false) + + x11.shmSeg = seg + x11.shmId = shmId + x11.shmData = data + x11.shmW = w + x11.shmH = h + x11.shmReady = true + return nil +} + +func x11DisplayCount() int { + x11.mu.Lock() + defer x11.mu.Unlock() + + if x11.conn == nil { + if err := x11InitLocked(); err != nil { + return 0 + } + } + return len(x11.screens) +} + +func x11DisplayBounds(idx int) image.Rectangle { + x11.mu.Lock() + defer x11.mu.Unlock() + + if x11.conn == nil { + if err := x11InitLocked(); err != nil { + return image.Rectangle{} + } + } + if idx < 0 || idx >= len(x11.screens) { + return image.Rectangle{} + } + + x0 := int(x11.primary.XOrg) + y0 := int(x11.primary.YOrg) + s := x11.screens[idx] + x := int(s.XOrg) - x0 + y := int(s.YOrg) - y0 + w := int(s.Width) + h := int(s.Height) + return image.Rect(x, y, x+w, y+h) +} + +func x11CaptureDisplay(display int) (*image.RGBA, error) { + x11.mu.Lock() + defer x11.mu.Unlock() + + if x11.conn == nil { + if err := x11InitLocked(); err != nil { + return nil, err + } + } + + if display < 0 || display >= len(x11.screens) { + return nil, fmt.Errorf("x11 capture: display %d out of range (0-%d)", display, len(x11.screens)-1) + } + + s := x11.screens[display] + x := int(s.XOrg) + y := int(s.YOrg) + w := int(s.Width) + h := int(s.Height) + + rootW := int(x11.screen.WidthInPixels) + rootH := int(x11.screen.HeightInPixels) + if x+w > rootW { + w = rootW - x + } + if y+h > rootH { + h = rootH - y + } + if w <= 0 || h <= 0 { + return nil, fmt.Errorf("x11 capture: zero-size region for display %d", display) + } + + var data []byte + + if x11.useShm { + if err := x11EnsureShm(w, h); err != nil { + log.Printf("x11 capture: shm alloc failed, falling back to GetImage: %v", err) + x11.useShm = false + } else { + _, err := xshm.GetImage(x11.conn, xproto.Drawable(x11.root), + int16(x), int16(y), uint16(w), uint16(h), + 0xffffffff, byte(xproto.ImageFormatZPixmap), + x11.shmSeg, 0).Reply() + if err != nil { + log.Printf("x11 capture: shm GetImage failed: %v (resetting)", err) + x11ResetLocked() + return nil, fmt.Errorf("x11 capture: shm GetImage: %w", err) + } + data = x11.shmData[:w*h*4] + } + } + + if data == nil { + xImg, err := xproto.GetImage(x11.conn, xproto.ImageFormatZPixmap, + xproto.Drawable(x11.root), + int16(x), int16(y), uint16(w), uint16(h), + 0xffffffff).Reply() + if err != nil { + log.Printf("x11 capture: GetImage failed: %v (resetting)", err) + x11ResetLocked() + return nil, fmt.Errorf("x11 capture: GetImage: %w", err) + } + data = xImg.Data + } + + img := x11.imgCache[display] + if img == nil || img.Rect.Dx() != w || img.Rect.Dy() != h { + img = image.NewRGBA(image.Rect(0, 0, w, h)) + x11.imgCache[display] = img + } + + pixels := img.Pix + stride := img.Stride + srcOff := 0 + for iy := 0; iy < h; iy++ { + dstRow := iy * stride + for ix := 0; ix < w; ix++ { + dstOff := dstRow + ix*4 + pixels[dstOff+0] = data[srcOff+2] // R + pixels[dstOff+1] = data[srcOff+1] // G + pixels[dstOff+2] = data[srcOff+0] // B + pixels[dstOff+3] = 255 // A + srcOff += 4 + } + } + + return img, nil +} diff --git a/Overlord-Client/cmd/agent/handlers/input_linux.go b/Overlord-Client/cmd/agent/handlers/input_linux.go new file mode 100644 index 0000000..091ec9f --- /dev/null +++ b/Overlord-Client/cmd/agent/handlers/input_linux.go @@ -0,0 +1,13 @@ +//go:build linux + +package handlers + +// Linux input is not implemented — all input functions are no-ops. + +func setCursorPos(x, y int32) {} +func sendMouseDown(button int) {} +func sendMouseUp(button int) {} +func sendKeyDown(vk uint16) {} +func sendKeyUp(vk uint16) {} +func sendTextInput(text string) {} +func keyCodeToVK(code string) uint16 { return 0 } diff --git a/Overlord-Client/cmd/agent/handlers/input_stub.go b/Overlord-Client/cmd/agent/handlers/input_stub.go index 3d6eef5..9781025 100644 --- a/Overlord-Client/cmd/agent/handlers/input_stub.go +++ b/Overlord-Client/cmd/agent/handlers/input_stub.go @@ -1,4 +1,4 @@ -//go:build !windows +//go:build !windows && !linux package handlers diff --git a/Overlord-Client/go.mod b/Overlord-Client/go.mod index 6629270..1e9d4b6 100644 --- a/Overlord-Client/go.mod +++ b/Overlord-Client/go.mod @@ -14,16 +14,16 @@ require ( require ( github.com/Kirizu-Official/windows-camera-go v0.2.0 github.com/gen2brain/malgo v0.11.24 + github.com/gen2brain/shm v0.1.0 + github.com/jezek/xgb v1.1.1 golang.org/x/crypto v0.49.0 golang.org/x/sys v0.42.0 ) require ( - github.com/gen2brain/shm v0.1.0 // indirect github.com/gen2brain/x264-go/x264c v0.0.0-20241022182000-732e1bdb7da2 // indirect github.com/gen2brain/x264-go/yuv v0.0.0-20221204084822-82ee2951dea2 // indirect github.com/godbus/dbus/v5 v5.1.0 // indirect - github.com/jezek/xgb v1.1.1 // indirect github.com/lxn/win v0.0.0-20210218163916-a377121e959e // indirect github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect