vxaboveground/overlord
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

linux capturing fix (it fucking sucks)

Browse Source
This commit is contained in:
KingKDot
2026-04-03 19:43:41 -05:00
parent 37bb3aabd7
commit 69b3e7833c
10 changed files with 445 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Overlord-Client/cmd/agent/capture/capture.go
+15 -2
View File
@@ -98,6 +98,14 @@ func CaptureAndSend(ctx context.Context, env *rt.Env) error {
log.Printf("capture: requested display %d out of range, defaulting to 0 (monitors=%d)", env.SelectedDisplay, displays)
}
t0 := time.Now()
fails := consecutiveCaptureFails.Load()
if fails >= 3 {
ResetDesktopCapture()
ResetMonitorCache()
time.Sleep(50 * time.Millisecond)
}
img, err := safeCaptureDisplay(display)
if err != nil {
img, err = safeCaptureDisplay(display)
@@ -108,13 +116,16 @@ func CaptureAndSend(ctx context.Context, env *rt.Env) error {
}
}
if err != nil {
log.Printf("capture: capture failed: %v (sending black frame)", err)
consecutiveCaptureFails.Add(1)
log.Printf("capture: capture failed: %v (sending black frame, consecutive=%d)", err, consecutiveCaptureFails.Load())
return sendBlackFrame(ctx, env)
}
if img == nil {
log.Printf("capture: capture returned nil image (sending black frame)")
consecutiveCaptureFails.Add(1)
log.Printf("capture: capture returned nil image (sending black frame, consecutive=%d)", consecutiveCaptureFails.Load())
return sendBlackFrame(ctx, env)
}
consecutiveCaptureFails.Store(0)
captureDur := time.Since(t0)
quality := jpegQuality()
@@ -323,6 +334,8 @@ func MonitorCount() int {
const frameLogInterval = 5 * time.Second
var consecutiveCaptureFails atomic.Int64
var (
fpsWindowStart atomic.Int64
fpsCount atomic.Int64
Overlord-Client/cmd/agent/capture/capture_linux.go
+105 -1
View File
@@ -4,18 +4,122 @@ package capture
import (
"image"
"log"
"os"
"sync"
"sync/atomic"
"github.com/kbinani/screenshot"
)
var isWaylandSession = os.Getenv("XDG_SESSION_TYPE") == "wayland" || os.Getenv("WAYLAND_DISPLAY") != ""
var (
x11Disabled atomic.Bool
x11BlackCount atomic.Int64
x11CheckOnce sync.Once
x11MaxBlackFrames int64 = 3
)
func init() {
if isWaylandSession {
log.Printf("capture: Wayland session detected (XDG_SESSION_TYPE=%s WAYLAND_DISPLAY=%s), using screenshot library (D-Bus portal)",
os.Getenv("XDG_SESSION_TYPE"), os.Getenv("WAYLAND_DISPLAY"))
}
}
var activeDisplays = func() int {
if !isWaylandSession && !x11Disabled.Load() {
if n := x11DisplayCount(); n > 0 {
return n
}
}
return screenshot.NumActiveDisplays()
}
var captureDisplayFn = func(display int) (*image.RGBA, error) {
return screenshot.CaptureDisplay(display)
if isWaylandSession || x11Disabled.Load() {
return captureViaLibrary(display)
}
img, err := x11CaptureDisplay(display)
if err != nil {
log.Printf("x11 capture: error, falling back to screenshot library: %v", err)
return captureViaLibrary(display)
}
if isAllBlack(img) {
n := x11BlackCount.Add(1)
if n >= x11MaxBlackFrames {
log.Printf("x11 capture: %d consecutive all-black frames, disabling X11 capture permanently", n)
x11Disabled.Store(true)
x11Reset()
return captureViaLibrary(display)
}
libImg, libErr := captureViaLibrary(display)
if libErr == nil && !isAllBlack(libImg) {
log.Printf("x11 capture: X11 returned black but screenshot library works, disabling X11 capture")
x11Disabled.Store(true)
x11Reset()
return libImg, nil
}
return img, nil
}
x11BlackCount.Store(0)
return img, nil
}
func captureViaLibrary(display int) (*image.RGBA, error) {
bounds := screenshot.GetDisplayBounds(display)
img, err := screenshot.CaptureRect(bounds)
if err != nil {
return nil, err
}
if img.Rect.Min.X != 0 || img.Rect.Min.Y != 0 {
img.Rect = image.Rect(0, 0, img.Rect.Dx(), img.Rect.Dy())
}
return img, nil
}
func isAllBlack(img *image.RGBA) bool {
if img == nil {
return true
}
pix := img.Pix
stride := img.Stride
w := img.Rect.Dx()
h := img.Rect.Dy()
if w == 0 || h == 0 {
return true
}
stepX := w / 4
stepY := h / 4
if stepX < 1 {
stepX = 1
}
if stepY < 1 {
stepY = 1
}
for y := stepY / 2; y < h; y += stepY {
row := y * stride
for x := stepX / 2; x < w; x += stepX {
off := row + x*4
if off+2 >= len(pix) {
continue
}
if pix[off] != 0 || pix[off+1] != 0 || pix[off+2] != 0 {
return false
}
}
}
return true
}
func displayCount() int {
if !isWaylandSession && !x11Disabled.Load() {
if n := x11DisplayCount(); n > 0 {
return n
}
}
return screenshot.NumActiveDisplays()
}
Overlord-Client/cmd/agent/capture/monitors_linux.go
+11 -2
View File
@@ -9,6 +9,15 @@ import (
)
func displayBounds(idx int) image.Rectangle {
if !isWaylandSession && !x11Disabled.Load() {
if b := x11DisplayBounds(idx); b.Dx() > 0 && b.Dy() > 0 {
return b
}
}
n := screenshot.NumActiveDisplays()
if idx < 0 || idx >= n {
return image.Rectangle{}
}
return screenshot.GetDisplayBounds(idx)
}
@@ -17,10 +26,10 @@ func displayScale(idx int) float64 {
}
func MonitorInfos() []MonitorInfo {
n := screenshot.NumActiveDisplays()
n := displayCount()
infos := make([]MonitorInfo, 0, n)
for i := 0; i < n; i++ {
b := screenshot.GetDisplayBounds(i)
b := displayBounds(i)
infos = append(infos, MonitorInfo{
Width: b.Dx(),
Height: b.Dy(),
Overlord-Client/cmd/agent/capture/reset_capture_linux.go
+7
View File
@@ -0,0 +1,7 @@
//go:build linux
package capture
func ResetDesktopCapture() {
x11Reset()
}
Overlord-Client/cmd/agent/capture/reset_capture_stub.go
+1 -1
View File
@@ -1,4 +1,4 @@
//go:build !windows
//go:build !windows && !linux
package capture
Overlord-Client/cmd/agent/capture/win_bitblt.go
+11
View File
@@ -166,6 +166,10 @@ var (
lastCapScale atomic.Uint64
state capState
captureMu sync.Mutex
stateCreatedAt time.Time
stateRefreshInterval = 30 * time.Second
)
var captureDisplayFn = func(display int) (*image.RGBA, error) {
@@ -324,8 +328,13 @@ func captureDisplayBitBlt(display int) (*image.RGBA, error) {
return img, bitDur, dibDur, convDur, nil
}
if !stateCreatedAt.IsZero() && time.Since(stateCreatedAt) > stateRefreshInterval {
state.reset()
}
img, bitDur, dibDur, convDur, err := captureWithDC()
if err != nil && fromCreateDC {
state.reset()
closeDC()
hdcScreen = getDC(0)
fromCreateDC = false
@@ -335,6 +344,7 @@ func captureDisplayBitBlt(display int) (*image.RGBA, error) {
img, bitDur, dibDur, convDur, err = captureWithDC()
}
if err != nil {
state.reset()
return nil, err
}
@@ -456,6 +466,7 @@ func (s *capState) ensure(hdcScreen uintptr, w, h int) (uintptr, uintptr, []byte
if s.hdcMem == 0 {
return 0, 0, nil, 0, syscall.EINVAL
}
stateCreatedAt = time.Now()
}
if s.w != w || s.h != h || s.hbmp == 0 {
Overlord-Client/cmd/agent/capture/x11_capture.go
+279
View File
@@ -0,0 +1,279 @@
//go:build linux
package capture
import (
"fmt"
"image"
"log"
"sync"
"github.com/gen2brain/shm"
"github.com/jezek/xgb"
xshm "github.com/jezek/xgb/shm"
"github.com/jezek/xgb/xinerama"
"github.com/jezek/xgb/xproto"
)
type x11State struct {
mu sync.Mutex
conn *xgb.Conn
screen *xproto.ScreenInfo
root xproto.Window
screens []xinerama.ScreenInfo
primary xinerama.ScreenInfo
shmSeg xshm.Seg
shmId int
shmData []byte
shmW int
shmH int
shmReady bool
useShm bool
imgCache map[int]*image.RGBA
}
var x11 x11State
func x11Init() error {
x11.mu.Lock()
defer x11.mu.Unlock()
return x11InitLocked()
}
func x11InitLocked() error {
if x11.conn != nil {
return nil
}
c, err := xgb.NewConn()
if err != nil {
return fmt.Errorf("x11 capture: connect: %w", err)
}
if err := xinerama.Init(c); err != nil {
c.Close()
return fmt.Errorf("x11 capture: xinerama init: %w", err)
}
reply, err := xinerama.QueryScreens(c).Reply()
if err != nil || reply.Number == 0 {
c.Close()
if err != nil {
return fmt.Errorf("x11 capture: query screens: %w", err)
}
return fmt.Errorf("x11 capture: no screens found")
}
x11.conn = c
x11.screen = xproto.Setup(c).DefaultScreen(c)
x11.root = x11.screen.Root
x11.screens = reply.ScreenInfo
x11.primary = reply.ScreenInfo[0]
x11.imgCache = make(map[int]*image.RGBA)
x11.useShm = true
if err := xshm.Init(c); err != nil {
log.Printf("x11 capture: shm extension not available, will use GetImage (slower)")
x11.useShm = false
}
log.Printf("x11 capture: connected, %d screens, shm=%v, screen=%dx%d",
len(x11.screens), x11.useShm,
x11.screen.WidthInPixels, x11.screen.HeightInPixels)
return nil
}
func x11ResetLocked() {
if x11.shmReady {
if x11.conn != nil {
xshm.Detach(x11.conn, x11.shmSeg)
}
_ = shm.Dt(x11.shmData)
_ = shm.Rm(x11.shmId)
x11.shmReady = false
x11.shmData = nil
x11.shmW = 0
x11.shmH = 0
}
if x11.conn != nil {
x11.conn.Close()
x11.conn = nil
}
x11.screens = nil
x11.imgCache = nil
}
func x11Reset() {
x11.mu.Lock()
defer x11.mu.Unlock()
x11ResetLocked()
}
func x11EnsureShm(w, h int) error {
if x11.shmReady && x11.shmW == w && x11.shmH == h {
return nil
}
if x11.shmReady {
xshm.Detach(x11.conn, x11.shmSeg)
_ = shm.Dt(x11.shmData)
_ = shm.Rm(x11.shmId)
x11.shmReady = false
}
size := w * h * 4
shmId, err := shm.Get(shm.IPC_PRIVATE, size, shm.IPC_CREAT|0o777)
if err != nil {
return fmt.Errorf("shm.Get(%d): %w", size, err)
}
seg, err := xshm.NewSegId(x11.conn)
if err != nil {
_ = shm.Rm(shmId)
return fmt.Errorf("shm.NewSegId: %w", err)
}
data, err := shm.At(shmId, 0, 0)
if err != nil {
_ = shm.Rm(shmId)
return fmt.Errorf("shm.At: %w", err)
}
xshm.Attach(x11.conn, seg, uint32(shmId), false)
x11.shmSeg = seg
x11.shmId = shmId
x11.shmData = data
x11.shmW = w
x11.shmH = h
x11.shmReady = true
return nil
}
func x11DisplayCount() int {
x11.mu.Lock()
defer x11.mu.Unlock()
if x11.conn == nil {
if err := x11InitLocked(); err != nil {
return 0
}
}
return len(x11.screens)
}
func x11DisplayBounds(idx int) image.Rectangle {
x11.mu.Lock()
defer x11.mu.Unlock()
if x11.conn == nil {
if err := x11InitLocked(); err != nil {
return image.Rectangle{}
}
}
if idx < 0 || idx >= len(x11.screens) {
return image.Rectangle{}
}
x0 := int(x11.primary.XOrg)
y0 := int(x11.primary.YOrg)
s := x11.screens[idx]
x := int(s.XOrg) - x0
y := int(s.YOrg) - y0
w := int(s.Width)
h := int(s.Height)
return image.Rect(x, y, x+w, y+h)
}
func x11CaptureDisplay(display int) (*image.RGBA, error) {
x11.mu.Lock()
defer x11.mu.Unlock()
if x11.conn == nil {
if err := x11InitLocked(); err != nil {
return nil, err
}
}
if display < 0 || display >= len(x11.screens) {
return nil, fmt.Errorf("x11 capture: display %d out of range (0-%d)", display, len(x11.screens)-1)
}
s := x11.screens[display]
x := int(s.XOrg)
y := int(s.YOrg)
w := int(s.Width)
h := int(s.Height)
rootW := int(x11.screen.WidthInPixels)
rootH := int(x11.screen.HeightInPixels)
if x+w > rootW {
w = rootW - x
}
if y+h > rootH {
h = rootH - y
}
if w <= 0 || h <= 0 {
return nil, fmt.Errorf("x11 capture: zero-size region for display %d", display)
}
var data []byte
if x11.useShm {
if err := x11EnsureShm(w, h); err != nil {
log.Printf("x11 capture: shm alloc failed, falling back to GetImage: %v", err)
x11.useShm = false
} else {
_, err := xshm.GetImage(x11.conn, xproto.Drawable(x11.root),
int16(x), int16(y), uint16(w), uint16(h),
0xffffffff, byte(xproto.ImageFormatZPixmap),
x11.shmSeg, 0).Reply()
if err != nil {
log.Printf("x11 capture: shm GetImage failed: %v (resetting)", err)
x11ResetLocked()
return nil, fmt.Errorf("x11 capture: shm GetImage: %w", err)
}
data = x11.shmData[:w*h*4]
}
}
if data == nil {
xImg, err := xproto.GetImage(x11.conn, xproto.ImageFormatZPixmap,
xproto.Drawable(x11.root),
int16(x), int16(y), uint16(w), uint16(h),
0xffffffff).Reply()
if err != nil {
log.Printf("x11 capture: GetImage failed: %v (resetting)", err)
x11ResetLocked()
return nil, fmt.Errorf("x11 capture: GetImage: %w", err)
}
data = xImg.Data
}
img := x11.imgCache[display]
if img == nil || img.Rect.Dx() != w || img.Rect.Dy() != h {
img = image.NewRGBA(image.Rect(0, 0, w, h))
x11.imgCache[display] = img
}
pixels := img.Pix
stride := img.Stride
srcOff := 0
for iy := 0; iy < h; iy++ {
dstRow := iy * stride
for ix := 0; ix < w; ix++ {
dstOff := dstRow + ix*4
pixels[dstOff+0] = data[srcOff+2] // R
pixels[dstOff+1] = data[srcOff+1] // G
pixels[dstOff+2] = data[srcOff+0] // B
pixels[dstOff+3] = 255 // A
srcOff += 4
}
}
return img, nil
}
Overlord-Client/cmd/agent/handlers/input_linux.go
+13
View File
@@ -0,0 +1,13 @@
//go:build linux
package handlers
// Linux input is not implemented — all input functions are no-ops.
func setCursorPos(x, y int32) {}
func sendMouseDown(button int) {}
func sendMouseUp(button int) {}
func sendKeyDown(vk uint16) {}
func sendKeyUp(vk uint16) {}
func sendTextInput(text string) {}
func keyCodeToVK(code string) uint16 { return 0 }
Overlord-Client/cmd/agent/handlers/input_stub.go
+1 -1
View File
@@ -1,4 +1,4 @@
//go:build !windows
//go:build !windows && !linux
package handlers
Overlord-Client/go.mod
+2 -2
View File
@@ -14,16 +14,16 @@ require (
require (
github.com/Kirizu-Official/windows-camera-go v0.2.0
github.com/gen2brain/malgo v0.11.24
github.com/gen2brain/shm v0.1.0
github.com/jezek/xgb v1.1.1
golang.org/x/crypto v0.49.0
golang.org/x/sys v0.42.0
)
require (
github.com/gen2brain/shm v0.1.0 // indirect
github.com/gen2brain/x264-go/x264c v0.0.0-20241022182000-732e1bdb7da2 // indirect
github.com/gen2brain/x264-go/yuv v0.0.0-20221204084822-82ee2951dea2 // indirect
github.com/godbus/dbus/v5 v5.1.0 // indirect
github.com/jezek/xgb v1.1.1 // indirect
github.com/lxn/win v0.0.0-20210218163916-a377121e959e // indirect
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect