Files
overlord/Overlord-Client/cmd/agent/capture/capture.go
T

1964 lines
49 KiB
Go
Raw Normal View History

2026-03-05 00:04:26 -06:00
package capture
import (
"bytes"
"context"
2026-03-21 23:09:24 -05:00
"errors"
2026-03-05 00:04:26 -06:00
"image"
"image/draw"
"log"
"os"
goruntime "runtime"
"sort"
"strconv"
"strings"
"sync"
"sync/atomic"
2026-03-21 23:09:24 -05:00
"syscall"
2026-03-05 00:04:26 -06:00
"time"
rt "overlord-client/cmd/agent/runtime"
2026-05-13 13:44:33 -05:00
"overlord-client/cmd/agent/webrtcpub"
2026-03-05 00:04:26 -06:00
"overlord-client/cmd/agent/wire"
)
func Loop(ctx context.Context, env *rt.Env) {
2026-04-20 13:46:59 -05:00
//garble:controlflow block_splits=10 junk_jumps=10 flatten_passes=2
2026-03-05 00:04:26 -06:00
defer recoverAndLog("capture loop", env.Cancel)
logCodecSupport()
2026-03-05 00:04:26 -06:00
if env.Cfg.DisableCapture {
log.Printf("capture: disabled by config, sending black placeholder")
if err := sendBlackFrame(ctx, env); err != nil {
log.Printf("capture: black frame failed: %v", err)
}
} else if supportsCapture() {
done := make(chan struct{})
goSafe("capture initial frame", env.Cancel, func() {
defer close(done)
restore := BypassResolutionCap()
defer restore()
2026-03-05 00:04:26 -06:00
var err error
2026-05-21 09:51:12 -05:00
if goruntime.GOOS == "windows" {
2026-03-05 00:04:26 -06:00
err = captureAllDisplaysAndSend(ctx, env)
} else {
err = CaptureAndSend(ctx, env)
}
if err != nil {
log.Printf("capture: initial frame failed: %v (continuing anyway)", err)
} else {
log.Printf("capture: initial frame sent")
}
})
select {
case <-done:
case <-time.After(3 * time.Second):
log.Printf("capture: initial frame timed out after 3s (continuing anyway)")
}
} else {
log.Printf("capture: no displays detected; skipping initial frame")
}
<-ctx.Done()
}
func Now(ctx context.Context, env *rt.Env) error {
if env.Cfg.DisableCapture {
return sendBlackFrame(ctx, env)
}
if !supportsCapture() {
return nil
}
return CaptureAndSend(ctx, env)
}
func CaptureDisplayRGBA(display int) (*image.RGBA, error) {
if !supportsCapture() {
return nil, nil
}
if display < 0 || display >= safeDisplayCount() {
display = 0
}
return safeCaptureDisplay(display)
}
func CaptureAndSend(ctx context.Context, env *rt.Env) error {
2026-04-20 13:46:59 -05:00
//garble:controlflow block_splits=10 junk_jumps=10 flatten_passes=2
2026-03-05 00:04:26 -06:00
defer recoverAndLog("capture send", env.Cancel)
displays := activeDisplays()
if displays == 0 {
log.Printf("capture: no displays available")
return nil
}
display := env.SelectedDisplay
if display < 0 || display >= displays {
display = 0
log.Printf("capture: requested display %d out of range, defaulting to 0 (monitors=%d)", env.SelectedDisplay, displays)
}
t0 := time.Now()
2026-04-03 19:43:41 -05:00
fails := consecutiveCaptureFails.Load()
if fails >= 3 {
ResetDesktopCapture()
ResetMonitorCache()
time.Sleep(50 * time.Millisecond)
}
2026-06-06 16:08:28 -05:00
if frame, captureDur, encodeDur, used, err := tryBuildDirectH264Frame(display); used {
if err != nil {
consecutiveCaptureFails.Add(1)
2026-07-18 01:27:58 -05:00
log.Printf("capture: direct video capture failed: %v (sending black frame, consecutive=%d)", err, consecutiveCaptureFails.Load())
2026-06-06 16:08:28 -05:00
return sendBlackFrame(ctx, env)
}
if len(frame.Data) == 0 {
consecutiveCaptureFails.Store(0)
return nil
}
consecutiveCaptureFails.Store(0)
return sendCompletedFrame(ctx, env, frame, display, t0, captureDur, encodeDur)
}
2026-03-05 00:04:26 -06:00
img, err := safeCaptureDisplay(display)
2026-03-21 23:09:24 -05:00
if err != nil {
img, err = safeCaptureDisplay(display)
if err != nil && errors.Is(err, syscall.EINVAL) {
ResetDesktopCapture()
ResetMonitorCache()
img, err = safeCaptureDisplay(display)
}
}
2026-03-05 00:04:26 -06:00
if err != nil {
2026-04-03 19:43:41 -05:00
consecutiveCaptureFails.Add(1)
log.Printf("capture: capture failed: %v (sending black frame, consecutive=%d)", err, consecutiveCaptureFails.Load())
2026-03-05 00:04:26 -06:00
return sendBlackFrame(ctx, env)
}
if img == nil {
2026-04-03 19:43:41 -05:00
consecutiveCaptureFails.Add(1)
log.Printf("capture: capture returned nil image (sending black frame, consecutive=%d)", consecutiveCaptureFails.Load())
2026-03-05 00:04:26 -06:00
return sendBlackFrame(ctx, env)
}
2026-04-03 19:43:41 -05:00
consecutiveCaptureFails.Store(0)
2026-03-05 00:04:26 -06:00
captureDur := time.Since(t0)
2026-05-16 00:35:50 -05:00
willSendViaWebRTC := blockCodec() == "h264" && webrtcpub.IsActive(webrtcpub.KindDesktop)
var slotAcquired bool
if !willSendViaWebRTC {
if !AcquireFrameSlot() {
PutRGBA(img)
return nil
}
slotAcquired = true
}
frameWidth := img.Bounds().Dx()
frameHeight := img.Bounds().Dy()
2026-03-05 00:04:26 -06:00
quality := jpegQuality()
frame, encodeDur, err := buildFrame(img, display, quality)
2026-04-08 14:14:09 -05:00
PutRGBA(img)
img = nil
2026-03-05 00:04:26 -06:00
if err != nil {
2026-05-16 00:35:50 -05:00
if slotAcquired {
ReleaseFrameSlot()
}
2026-03-05 00:04:26 -06:00
return err
}
frame.Header.Width = frameWidth
frame.Header.Height = frameHeight
2026-03-05 00:04:26 -06:00
now := time.Now()
fps := frameFPS(now)
if fps <= 0 {
fps = 1
}
frame.Header.FPS = fps
if ctx.Err() != nil {
2026-05-16 00:35:50 -05:00
if slotAcquired {
ReleaseFrameSlot()
}
2026-03-05 00:04:26 -06:00
return nil
}
2026-05-13 20:19:21 -05:00
if frame.Header.Format == "h264" && webrtcpub.IsActive(webrtcpub.KindDesktop) {
2026-05-13 13:44:33 -05:00
dur := time.Second / time.Duration(fps)
if dur <= 0 {
dur = 33 * time.Millisecond
}
sendStart := time.Now()
2026-05-13 20:19:21 -05:00
if werr := webrtcpub.WriteH264(webrtcpub.KindDesktop, frame.Data, dur); werr != nil {
2026-05-13 13:44:33 -05:00
log.Printf("webrtc: write h264 failed: %v", werr)
}
sendDur := time.Since(sendStart)
2026-05-16 00:35:50 -05:00
if slotAcquired {
ReleaseFrameSlot()
}
2026-05-13 13:44:33 -05:00
statFrames.Add(1)
statCapNs.Add(captureDur.Nanoseconds())
statEncNs.Add(encodeDur.Nanoseconds())
totalDur := time.Since(t0)
statTotalNs.Add(totalDur.Nanoseconds())
2026-05-13 13:44:33 -05:00
statBytes.Add(int64(len(frame.Data)))
emitDesktopStreamStats(ctx, env, frame, fps, captureDur, encodeDur, sendDur, totalDur, "webrtc")
2026-05-13 13:44:33 -05:00
return nil
}
2026-05-16 00:35:50 -05:00
if !slotAcquired {
if !AcquireFrameSlot() {
return nil
}
2026-04-08 14:14:09 -05:00
}
2026-03-05 00:04:26 -06:00
sendStart := time.Now()
err = wire.WriteMsg(ctx, env.Conn, frame)
sendDur := time.Since(sendStart)
if shouldLogFrame(now) {
total := time.Since(t0)
frames := statFrames.Load()
capAvg := avgMs(statCapNs.Load(), frames)
encAvg := avgMs(statEncNs.Load(), frames)
sendAvg := avgMs(statSendNs.Load(), frames)
totalAvg := avgMs(statTotalNs.Load(), frames)
bytesAvg := avgBytes(statBytes.Load(), frames)
detectAvg := avgMs(statDetectNs.Load(), frames)
mergeAvg := avgMs(statMergeNs.Load(), frames)
blkJpegAvg := avgMs(statBlkJpegNs.Load(), frames)
prevCopyAvg := avgMs(statPrevCopyNs.Load(), frames)
full := statFullFrames.Load()
blocks := statBlockFrames.Load()
keep := statKeepaliveFrames.Load()
regions := statBlockRegions.Load()
fallbacks := statBlockFallbacks.Load()
avgRegions := float64(0)
if blocks > 0 {
avgRegions = float64(regions) / float64(blocks)
}
log.Printf("capture: stream display=%d fps≈%d format=%s size=%d cap=%s enc=%s send=%s total=%s | avg cap=%.2fms enc=%.2fms send=%.2fms total=%.2fms avgSize=%.0fB frames=%d detect=%.2fms merge=%.2fms blkJpeg=%.2fms prevCopy=%.2fms full=%d blocks=%d keep=%d fallbacks=%d avgRegions=%.2f", display, fps, frame.Header.Format, len(frame.Data), captureDur, encodeDur, sendDur, total, capAvg, encAvg, sendAvg, totalAvg, bytesAvg, frames, detectAvg, mergeAvg, blkJpegAvg, prevCopyAvg, full, blocks, keep, fallbacks, avgRegions)
resetStats()
}
statFrames.Add(1)
statCapNs.Add(captureDur.Nanoseconds())
statEncNs.Add(encodeDur.Nanoseconds())
statSendNs.Add(sendDur.Nanoseconds())
statTotalNs.Add(time.Since(t0).Nanoseconds())
statBytes.Add(int64(len(frame.Data)))
emitDesktopStreamStats(ctx, env, frame, fps, captureDur, encodeDur, sendDur, time.Since(t0), "websocket")
2026-03-05 00:04:26 -06:00
return err
}
2026-06-06 16:08:28 -05:00
func sendCompletedFrame(ctx context.Context, env *rt.Env, frame wire.Frame, display int, t0 time.Time, captureDur, encodeDur time.Duration) error {
now := time.Now()
fps := frameFPS(now)
if fps <= 0 {
fps = 1
}
frame.Header.FPS = fps
if ctx.Err() != nil {
return nil
}
if frame.Header.Format == "h264" && webrtcpub.IsActive(webrtcpub.KindDesktop) {
dur := time.Second / time.Duration(fps)
if dur <= 0 {
dur = 33 * time.Millisecond
}
sendStart := time.Now()
2026-06-06 16:08:28 -05:00
if werr := webrtcpub.WriteH264(webrtcpub.KindDesktop, frame.Data, dur); werr != nil {
log.Printf("webrtc: write h264 failed: %v", werr)
}
sendDur := time.Since(sendStart)
2026-06-06 16:08:28 -05:00
statFrames.Add(1)
statCapNs.Add(captureDur.Nanoseconds())
statEncNs.Add(encodeDur.Nanoseconds())
totalDur := time.Since(t0)
statTotalNs.Add(totalDur.Nanoseconds())
2026-06-06 16:08:28 -05:00
statBytes.Add(int64(len(frame.Data)))
emitDesktopStreamStats(ctx, env, frame, fps, captureDur, encodeDur, sendDur, totalDur, "webrtc")
2026-06-06 16:08:28 -05:00
return nil
}
if !AcquireFrameSlot() {
return nil
}
sendStart := time.Now()
err := wire.WriteMsg(ctx, env.Conn, frame)
sendDur := time.Since(sendStart)
if shouldLogFrame(now) {
total := time.Since(t0)
frames := statFrames.Load()
capAvg := avgMs(statCapNs.Load(), frames)
encAvg := avgMs(statEncNs.Load(), frames)
sendAvg := avgMs(statSendNs.Load(), frames)
totalAvg := avgMs(statTotalNs.Load(), frames)
bytesAvg := avgBytes(statBytes.Load(), frames)
detectAvg := avgMs(statDetectNs.Load(), frames)
mergeAvg := avgMs(statMergeNs.Load(), frames)
blkJpegAvg := avgMs(statBlkJpegNs.Load(), frames)
prevCopyAvg := avgMs(statPrevCopyNs.Load(), frames)
full := statFullFrames.Load()
blocks := statBlockFrames.Load()
keep := statKeepaliveFrames.Load()
regions := statBlockRegions.Load()
fallbacks := statBlockFallbacks.Load()
avgRegions := float64(0)
if blocks > 0 {
avgRegions = float64(regions) / float64(blocks)
}
log.Printf("capture: stream display=%d fps≈%d format=%s size=%d cap=%s enc=%s send=%s total=%s | avg cap=%.2fms enc=%.2fms send=%.2fms total=%.2fms avgSize=%.0fB frames=%d detect=%.2fms merge=%.2fms blkJpeg=%.2fms prevCopy=%.2fms full=%d blocks=%d keep=%d fallbacks=%d avgRegions=%.2f", display, fps, frame.Header.Format, len(frame.Data), captureDur, encodeDur, sendDur, total, capAvg, encAvg, sendAvg, totalAvg, bytesAvg, frames, detectAvg, mergeAvg, blkJpegAvg, prevCopyAvg, full, blocks, keep, fallbacks, avgRegions)
resetStats()
}
if err != nil {
ReleaseFrameSlot()
return err
}
statFrames.Add(1)
statCapNs.Add(captureDur.Nanoseconds())
statEncNs.Add(encodeDur.Nanoseconds())
statSendNs.Add(sendDur.Nanoseconds())
statTotalNs.Add(time.Since(t0).Nanoseconds())
statBytes.Add(int64(len(frame.Data)))
emitDesktopStreamStats(ctx, env, frame, fps, captureDur, encodeDur, sendDur, time.Since(t0), "websocket")
2026-06-06 16:08:28 -05:00
return nil
}
2026-03-05 00:04:26 -06:00
// captureAllDisplaysAndSend stitches all monitors into a single image and sends
// it as the initial frame. Used on Windows so the dashboard thumbnail shows
// every monitor, not just the primary one.
func captureAllDisplaysAndSend(ctx context.Context, env *rt.Env) error {
defer recoverAndLog("capture all displays send", env.Cancel)
n := activeDisplays()
if n <= 0 {
return nil
}
type monCapture struct {
bounds image.Rectangle
img *image.RGBA
}
parts := make([]monCapture, 0, n)
minX, minY := int(1e9), int(1e9)
maxX, maxY := int(-1e9), int(-1e9)
for i := 0; i < n; i++ {
bounds := displayBounds(i)
img, err := safeCaptureDisplay(i)
if err != nil || img == nil {
log.Printf("capture: all-displays: display %d failed: %v, falling back to single", i, err)
return CaptureAndSend(ctx, env)
}
parts = append(parts, monCapture{bounds: bounds, img: img})
if bounds.Min.X < minX {
minX = bounds.Min.X
}
if bounds.Min.Y < minY {
minY = bounds.Min.Y
}
if bounds.Max.X > maxX {
maxX = bounds.Max.X
}
if bounds.Max.Y > maxY {
maxY = bounds.Max.Y
}
}
// Compute per-monitor scale factors to handle resolution-cap or DPI
// mismatches between displayBounds (virtual coords) and captured pixels.
type scaledPart struct {
img *image.RGBA
scaleX float64
scaleY float64
offX int
offY int
}
scaled := make([]scaledPart, len(parts))
for i, part := range parts {
sx, sy := 1.0, 1.0
bw := part.bounds.Dx()
bh := part.bounds.Dy()
iw := part.img.Rect.Dx()
ih := part.img.Rect.Dy()
if bw > 0 && iw > 0 {
sx = float64(iw) / float64(bw)
}
if bh > 0 && ih > 0 {
sy = float64(ih) / float64(bh)
}
scaled[i] = scaledPart{
img: part.img,
scaleX: sx,
scaleY: sy,
offX: int(float64(part.bounds.Min.X-minX) * sx),
offY: int(float64(part.bounds.Min.Y-minY) * sy),
}
}
// Canvas sized from scaled image placements.
cW, cH := 0, 0
for _, sp := range scaled {
rx := sp.offX + sp.img.Rect.Dx()
ry := sp.offY + sp.img.Rect.Dy()
if rx > cW {
cW = rx
}
if ry > cH {
cH = ry
}
}
2026-04-08 14:14:09 -05:00
canvas := GetRGBA(cW, cH)
for _, sp := range scaled {
dst := image.Rect(sp.offX, sp.offY, sp.offX+sp.img.Rect.Dx(), sp.offY+sp.img.Rect.Dy())
draw.Draw(canvas, dst, sp.img, sp.img.Rect.Min, draw.Src)
2026-03-05 00:04:26 -06:00
}
2026-04-08 14:14:09 -05:00
for _, part := range parts {
PutRGBA(part.img)
}
2026-03-05 00:04:26 -06:00
quality := jpegQuality()
frame, _, err := buildFrame(canvas, 0, quality)
canvasW, canvasH := canvas.Rect.Dx(), canvas.Rect.Dy()
2026-04-08 14:14:09 -05:00
PutRGBA(canvas)
canvas = nil
2026-03-05 00:04:26 -06:00
if err != nil {
return err
}
frame.Header.FPS = 1
log.Printf("capture: all-displays initial frame %dx%d (%d monitors)", canvasW, canvasH, n)
2026-03-05 00:04:26 -06:00
return wire.WriteMsg(ctx, env.Conn, frame)
}
func supportsCapture() bool {
2026-03-06 11:48:05 -06:00
return safeActiveDisplays() > 0
}
func safeActiveDisplays() int {
defer func() {
_ = recover()
}()
return activeDisplays()
2026-03-05 00:04:26 -06:00
}
func safeCaptureDisplay(display int) (*image.RGBA, error) {
// Log panics with a stack trace so the actual culprit is captured in the
// crash log instead of being silently swallowed (returns nil image, which
// the caller treats as a transient capture failure).
defer recoverAndLog("safeCaptureDisplay", nil)
2026-03-05 00:04:26 -06:00
img, err := captureDisplayFn(display)
if err != nil {
return nil, err
}
return img, nil
}
func safeDisplayCount() int {
defer func() {
_ = recover()
}()
return displayCount()
}
func sendBlackFrame(ctx context.Context, env *rt.Env) error {
if ctx.Err() != nil {
return nil
}
img := image.NewRGBA(image.Rect(0, 0, 64, 64))
quality := 60
frame, _, err := buildFrame(img, 0, quality)
if err != nil {
return err
}
frame.Header.FPS = 1
return wire.WriteMsg(ctx, env.Conn, frame)
}
func MonitorCount() int {
n := safeDisplayCount()
if n <= 0 {
return 1
}
return n
}
const frameLogInterval = 5 * time.Second
2026-04-03 19:43:41 -05:00
var consecutiveCaptureFails atomic.Int64
2026-03-05 00:04:26 -06:00
var (
fpsWindowStart atomic.Int64
fpsCount atomic.Int64
fpsLatest atomic.Int64
lastFrameLog atomic.Int64
2026-06-06 16:08:28 -05:00
metricsOnce sync.Once
metricsEnabled bool
2026-03-05 00:04:26 -06:00
lastKeyframe atomic.Int64
fullNextFrames atomic.Int64
statFrames atomic.Int64
statCapNs atomic.Int64
statEncNs atomic.Int64
statSendNs atomic.Int64
statTotalNs atomic.Int64
statBytes atomic.Int64
statDetectNs atomic.Int64
statMergeNs atomic.Int64
statBlkJpegNs atomic.Int64
statPrevCopyNs atomic.Int64
statFullFrames atomic.Int64
statBlockFrames atomic.Int64
statKeepaliveFrames atomic.Int64
statBlockRegions atomic.Int64
statBlockFallbacks atomic.Int64
2026-07-06 20:19:38 -05:00
statFrameSlotSkips atomic.Int64
2026-03-05 00:04:26 -06:00
overrideQuality atomic.Int64
overrideCodec atomic.Value
desktopSoftwareH264 atomic.Bool
h264WarnOnce sync.Once
2026-07-18 01:27:58 -05:00
hevcWarnOnce sync.Once
codecLogOnce sync.Once
2026-03-05 00:04:26 -06:00
prevMu sync.Mutex
prevFrame *prevImage
backstagePrevMu sync.Mutex
backstagePrevFrame *prevImage
backstageLastKeyframe atomic.Int64
2026-03-05 00:04:26 -06:00
)
type prevImage struct {
w int
h int
pix []byte
}
func logCodecSupport() {
codecLogOnce.Do(func() {
if h264Available() {
2026-03-06 13:27:51 -06:00
detail := h264AvailabilityDetail()
if detail != "" {
log.Printf("capture: codec support h264=enabled (%s) jpeg=enabled", detail)
return
}
log.Printf("capture: codec support h264=enabled jpeg=enabled")
return
}
2026-03-06 13:27:51 -06:00
detail := h264AvailabilityDetail()
if detail != "" {
log.Printf("capture: codec support h264=disabled (%s), jpeg=enabled", detail)
return
}
log.Printf("capture: codec support h264=disabled, jpeg=enabled")
})
}
2026-03-05 00:04:26 -06:00
func frameFPS(now time.Time) int {
start := fpsWindowStart.Load()
if start == 0 {
if fpsWindowStart.CompareAndSwap(0, now.UnixNano()) {
fpsCount.Store(1)
return int(fpsLatest.Load())
}
start = fpsWindowStart.Load()
}
fpsCount.Add(1)
elapsed := time.Duration(now.UnixNano() - start)
if elapsed >= time.Second {
frames := fpsCount.Swap(0)
if frames > 0 {
fps := int(float64(frames) / elapsed.Seconds())
fpsLatest.Store(int64(fps))
}
fpsWindowStart.Store(now.UnixNano())
}
return int(fpsLatest.Load())
}
func shouldLogFrame(now time.Time) bool {
2026-06-06 16:08:28 -05:00
if !captureMetricsEnabled() {
return false
}
2026-03-05 00:04:26 -06:00
last := time.Unix(0, lastFrameLog.Load())
if now.Sub(last) >= frameLogInterval {
lastFrameLog.Store(now.UnixNano())
return true
}
return false
}
2026-06-06 16:08:28 -05:00
func captureMetricsEnabled() bool {
metricsOnce.Do(func() {
for _, name := range []string{"OVERLORD_CAPTURE_METRICS", "OVERLORD_DEV_METRICS"} {
switch strings.ToLower(strings.TrimSpace(os.Getenv(name))) {
case "1", "true", "yes", "on":
metricsEnabled = true
return
}
}
switch strings.ToLower(strings.TrimSpace(os.Getenv("OVERLORD_MODE"))) {
case "dev", "development":
metricsEnabled = true
}
})
return metricsEnabled
}
2026-03-05 00:04:26 -06:00
func requestFullFrames(count int) {
if count <= 0 {
return
}
fullNextFrames.Store(int64(count))
}
func consumeFullFrame() bool {
for {
val := fullNextFrames.Load()
if val <= 0 {
return false
}
if fullNextFrames.CompareAndSwap(val, val-1) {
return true
}
}
}
func ResetPrev() {
prevMu.Lock()
prevFrame = nil
prevMu.Unlock()
lastKeyframe.Store(0)
requestFullFrames(2)
resetH264Encoder()
2026-03-05 00:04:26 -06:00
}
func ResetPrevbackstage() {
backstagePrevMu.Lock()
backstagePrevFrame = nil
backstagePrevMu.Unlock()
backstageLastKeyframe.Store(0)
resetH264Encoderbackstage()
2026-06-12 09:14:49 -05:00
}
func RequestDesktopFullFrame() {
requestFullFrames(2)
RequestDesktopH264Keyframe()
}
func SetDesktopSoftwareH264(enabled bool) {
if desktopSoftwareH264.Swap(enabled) == enabled {
return
}
resetH264Encoder()
requestFullFrames(2)
log.Printf("capture: desktop software h264 %v", enabled)
}
func useDesktopSoftwareH264() bool {
return desktopSoftwareH264.Load()
}
func RequestbackstageFullFrame() {
backstagePrevMu.Lock()
backstagePrevFrame = nil
backstagePrevMu.Unlock()
backstageLastKeyframe.Store(0)
resetH264Encoderbackstage()
}
2026-03-05 00:04:26 -06:00
func jpegQuality() int {
if q := overrideQuality.Load(); q > 0 {
return int(q)
}
q := int(loadOnceInt(&cachedJPEGQuality, 95))
if q < 20 {
q = 20
}
if q > 100 {
q = 100
}
return q
}
var (
jpegQualityOnce sync.Once
cachedJPEGQuality int64
blockCodecOnce sync.Once
cachedBlockCodec string
)
func loadOnceInt(target *int64, def int) int64 {
jpegQualityOnce.Do(func() {
if env := os.Getenv("OVERLORD_JPEG_QUALITY"); env != "" {
if v, err := strconv.Atoi(env); err == nil {
*target = int64(v)
return
}
}
*target = int64(def)
})
return atomic.LoadInt64(target)
}
func resetStats() {
statFrames.Store(0)
statCapNs.Store(0)
statEncNs.Store(0)
statSendNs.Store(0)
statTotalNs.Store(0)
statBytes.Store(0)
statDetectNs.Store(0)
statMergeNs.Store(0)
statBlkJpegNs.Store(0)
statPrevCopyNs.Store(0)
statFullFrames.Store(0)
statBlockFrames.Store(0)
statKeepaliveFrames.Store(0)
statBlockRegions.Store(0)
statBlockFallbacks.Store(0)
2026-07-06 20:19:38 -05:00
statFrameSlotSkips.Store(0)
2026-03-05 00:04:26 -06:00
}
func avgMs(ns int64, frames int64) float64 {
if frames == 0 {
return 0
}
return float64(ns) / 1e6 / float64(frames)
}
func avgBytes(b int64, frames int64) float64 {
if frames == 0 {
return 0
}
return float64(b) / float64(frames)
}
const (
blockSize = 64
maxBlockRatio = 0.40
keyframeEvery = 5 * time.Second
enableBlocks = true
samplingRate = 3
minBlockSize = 32
changeThresh = 3
blockMargin = 8
blockCodecEnv = "OVERLORD_BLOCK_CODEC"
cursorROIMargin = 32
windowROIMargin = 12
backstageSamplingRate = 1
backstageChangeThresh = 1
2026-03-05 00:04:26 -06:00
)
func buildFrame(img *image.RGBA, display int, quality int) (wire.Frame, time.Duration, error) {
encStart := time.Now()
width := img.Bounds().Dx()
height := img.Bounds().Dy()
full := false
now := time.Now()
codec := blockCodec()
2026-07-18 01:27:58 -05:00
if codec == "hevc" {
if width%2 != 0 || height%2 != 0 {
log.Printf("capture: hevc skipped for odd dimensions (%dx%d), falling back to h264/jpeg", width, height)
codec = "h264"
} else {
if webrtcpub.ConsumeKeyframeRequest() {
resetHEVCEncoder()
}
hevcBytes, err := encodeHEVCFrame(img)
if err == nil && len(hevcBytes) > 0 {
lastKeyframe.Store(now.UnixNano())
statFullFrames.Add(1)
return wire.Frame{Type: "frame", Header: wire.FrameHeader{Monitor: display, FPS: 0, Format: "hevc"}, Data: hevcBytes}, time.Since(encStart), nil
}
hevcWarnOnce.Do(func() {
log.Printf("capture: hevc encode unavailable, falling back to h264/jpeg: %v (%s)", err, hevcAvailabilityDetail())
})
resetHEVCEncoder()
if h264Available() {
codec = "h264"
overrideCodec.Store("h264")
} else {
codec = "jpeg"
overrideCodec.Store("jpeg")
}
}
}
2026-03-05 00:04:26 -06:00
if codec == "h264" {
if width%2 != 0 || height%2 != 0 {
log.Printf("capture: h264 skipped for odd dimensions (%dx%d), falling back to jpeg", width, height)
codec = "jpeg"
} else {
2026-05-13 13:44:33 -05:00
if webrtcpub.ConsumeKeyframeRequest() {
resetH264Encoder()
}
h264Bytes, err := encodeH264Frame(img)
if err == nil && len(h264Bytes) > 0 {
lastKeyframe.Store(now.UnixNano())
statFullFrames.Add(1)
return wire.Frame{Type: "frame", Header: wire.FrameHeader{Monitor: display, FPS: 0, Format: "h264"}, Data: h264Bytes}, time.Since(encStart), nil
2026-03-06 13:27:51 -06:00
}
h264WarnOnce.Do(func() {
detail := h264AvailabilityDetail()
if detail != "" {
log.Printf("capture: h264 encode unavailable, falling back to jpeg: %v (%s)", err, detail)
return
}
log.Printf("capture: h264 encode unavailable, falling back to jpeg: %v", err)
})
codec = "jpeg"
}
}
2026-03-05 00:04:26 -06:00
if !enableBlocks {
jpegBytes, err := encodeJPEG(img, quality)
prevMu.Lock()
copyPrev(img)
prevMu.Unlock()
return wire.Frame{Type: "frame", Header: wire.FrameHeader{Monitor: display, FPS: 0, Format: "jpeg"}, Data: jpegBytes}, time.Since(encStart), err
}
prevMu.Lock()
pf := prevFrame
prevMu.Unlock()
if pf == nil || pf.w != width || pf.h != height {
full = true
}
if consumeFullFrame() {
full = true
}
if !full && keyframeEvery > 0 {
last := time.Unix(0, lastKeyframe.Load())
if now.Sub(last) >= keyframeEvery {
full = true
}
}
if full {
jpegBytes, err := encodeJPEG(img, quality)
prevMu.Lock()
copyPrev(img)
prevMu.Unlock()
lastKeyframe.Store(now.UnixNano())
statFullFrames.Add(1)
return wire.Frame{Type: "frame", Header: wire.FrameHeader{Monitor: display, FPS: 0, Format: "jpeg"}, Data: jpegBytes}, time.Since(encStart), err
}
blocks, blockPayload, encDur, err := encodeBlocks(img, pf, quality, codec, display)
if err != nil {
return wire.Frame{}, encDur, err
}
if blocks == 0 {
// Keepalive frames indicate no block-level changes, so avoid copying the
// full RGBA buffer into prevFrame again.
2026-03-05 00:04:26 -06:00
statKeepaliveFrames.Add(1)
return wire.Frame{Type: "frame", Header: wire.FrameHeader{Monitor: display, FPS: 0, Format: "blocks"}, Data: blockPayload}, encDur, nil
}
totalBlocks := ((width + blockSize - 1) / blockSize) * ((height + blockSize - 1) / blockSize)
changedRatio := float64(blocks) / float64(totalBlocks)
if changedRatio > maxBlockRatio {
jpegBytes, err := encodeJPEG(img, quality)
prevMu.Lock()
copyPrev(img)
prevMu.Unlock()
lastKeyframe.Store(now.UnixNano())
statBlockFallbacks.Add(1)
statFullFrames.Add(1)
return wire.Frame{Type: "frame", Header: wire.FrameHeader{Monitor: display, FPS: 0, Format: "jpeg"}, Data: jpegBytes}, time.Since(encStart), err
}
statBlockFrames.Add(1)
statBlockRegions.Add(int64(blocks))
format := "blocks"
if codec == "raw" {
format = "blocks_raw"
}
return wire.Frame{Type: "frame", Header: wire.FrameHeader{Monitor: display, FPS: 0, Format: format}, Data: blockPayload}, encDur, nil
}
func buildFramebackstage(img *image.RGBA, display int, quality int) (wire.Frame, time.Duration, error) {
2026-03-05 00:04:26 -06:00
encStart := time.Now()
width := img.Bounds().Dx()
height := img.Bounds().Dy()
codec := blockCodec()
now := time.Now()
2026-03-05 18:16:41 -06:00
if codec == "h264" {
if width%2 != 0 || height%2 != 0 {
log.Printf("backstage capture: h264 skipped for odd dimensions (%dx%d), falling back to jpeg", width, height)
codec = "jpeg"
} else {
2026-05-13 13:44:33 -05:00
if webrtcpub.ConsumeKeyframeRequest() {
resetH264Encoderbackstage()
2026-05-13 13:44:33 -05:00
}
h264Bytes, err := encodeH264Framebackstage(img)
if err == nil && len(h264Bytes) > 0 {
backstageLastKeyframe.Store(now.UnixNano())
statFullFrames.Add(1)
frame := wire.Frame{Type: "frame", Header: wire.FrameHeader{Monitor: display, FPS: 0, Format: "h264", Backstage: true}, Data: h264Bytes}
return frame, time.Since(encStart), nil
2026-03-06 13:27:51 -06:00
}
h264WarnOnce.Do(func() {
detail := h264AvailabilityDetail()
if detail != "" {
log.Printf("backstage capture: h264 encode unavailable, falling back to jpeg: %v (%s)", err, detail)
return
}
log.Printf("backstage capture: h264 encode unavailable, falling back to jpeg: %v", err)
})
codec = "jpeg"
}
2026-03-05 18:16:41 -06:00
}
backstagePrevMu.Lock()
pf := backstagePrevFrame
backstagePrevMu.Unlock()
2026-03-05 00:04:26 -06:00
if pf == nil || pf.w != width || pf.h != height || now.Sub(time.Unix(0, backstageLastKeyframe.Load())) > keyframeEvery {
2026-03-05 00:04:26 -06:00
jpegBytes, err := encodeJPEG(img, quality)
backstagePrevMu.Lock()
copyPrevbackstage(img)
backstagePrevMu.Unlock()
backstageLastKeyframe.Store(now.UnixNano())
2026-03-05 00:04:26 -06:00
statFullFrames.Add(1)
frame := wire.Frame{Type: "frame", Header: wire.FrameHeader{Monitor: display, FPS: 0, Format: "jpeg", Backstage: true}, Data: jpegBytes}
2026-03-05 00:04:26 -06:00
return frame, time.Since(encStart), err
}
// use strict block detection for backstage otherwise half the time it just doesn't fucking update pictures
2026-03-05 00:04:26 -06:00
// because of how slow the capture shit is.
blocks, blockPayload, encDur, err := encodeBlocksbackstage(img, pf, quality, codec, display)
2026-03-05 00:04:26 -06:00
if err != nil {
return wire.Frame{}, encDur, err
}
if blocks == 0 {
// Keepalive frames indicate no block-level changes, so avoid copying the
// full RGBA buffer into prevFrame again.
2026-03-05 00:04:26 -06:00
statKeepaliveFrames.Add(1)
frame := wire.Frame{Type: "frame", Header: wire.FrameHeader{Monitor: display, FPS: 0, Format: "blocks", Backstage: true}, Data: blockPayload}
2026-03-05 00:04:26 -06:00
return frame, encDur, nil
}
totalBlocks := ((width + blockSize - 1) / blockSize) * ((height + blockSize - 1) / blockSize)
changedRatio := float64(blocks) / float64(totalBlocks)
if changedRatio > maxBlockRatio {
jpegBytes, err := encodeJPEG(img, quality)
backstagePrevMu.Lock()
copyPrevbackstage(img)
backstagePrevMu.Unlock()
backstageLastKeyframe.Store(now.UnixNano())
2026-03-05 00:04:26 -06:00
statBlockFallbacks.Add(1)
statFullFrames.Add(1)
frame := wire.Frame{Type: "frame", Header: wire.FrameHeader{Monitor: display, FPS: 0, Format: "jpeg", Backstage: true}, Data: jpegBytes}
2026-03-05 00:04:26 -06:00
return frame, time.Since(encStart), err
}
statBlockFrames.Add(1)
statBlockRegions.Add(int64(blocks))
format := "blocks"
if codec == "raw" {
format = "blocks_raw"
}
frame := wire.Frame{Type: "frame", Header: wire.FrameHeader{Monitor: display, FPS: 0, Format: format, Backstage: true}, Data: blockPayload}
2026-03-05 00:04:26 -06:00
return frame, encDur, nil
}
type streamRegion struct{ x, y, w, h int }
2026-05-21 23:28:06 -05:00
type encodeBufs struct {
changedGrid []bool
visited []bool
}
var (
encodeBufsPool = sync.Pool{New: func() interface{} { return &encodeBufs{} }}
frameBufPool = sync.Pool{New: func() interface{} { return bytes.NewBuffer(make([]byte, 0, 512*1024)) }}
)
func writeUint16LE(buf *bytes.Buffer, v uint16) {
buf.WriteByte(byte(v))
buf.WriteByte(byte(v >> 8))
}
func writeUint32LE(buf *bytes.Buffer, v uint32) {
buf.WriteByte(byte(v))
buf.WriteByte(byte(v >> 8))
buf.WriteByte(byte(v >> 16))
buf.WriteByte(byte(v >> 24))
}
2026-03-05 00:04:26 -06:00
type roiHint struct {
rect image.Rectangle
weight int
}
func encodeBlocks(img *image.RGBA, prev *prevImage, quality int, codec string, display int) (int, []byte, time.Duration, error) {
width := img.Bounds().Dx()
height := img.Bounds().Dy()
stride := img.Stride
prevStride := prev.w * 4
rois := collectROIHints(display, width, height)
blocksWide := (width + blockSize - 1) / blockSize
blocksHigh := (height + blockSize - 1) / blockSize
2026-05-21 23:28:06 -05:00
total := blocksWide * blocksHigh
eb := encodeBufsPool.Get().(*encodeBufs)
defer encodeBufsPool.Put(eb)
if cap(eb.changedGrid) < total {
eb.changedGrid = make([]bool, total)
eb.visited = make([]bool, total)
} else {
eb.changedGrid = eb.changedGrid[:total]
eb.visited = eb.visited[:total]
clear(eb.changedGrid)
clear(eb.visited)
}
changedGrid := eb.changedGrid
visited := eb.visited
2026-03-05 00:04:26 -06:00
changedCount := 0
passDetectStart := time.Now()
for by := 0; by < blocksHigh; by++ {
for bx := 0; bx < blocksWide; bx++ {
x := bx * blockSize
y := by * blockSize
ww := blockSize
hh := blockSize
if x+ww > width {
ww = width - x
}
if y+hh > height {
hh = height - y
}
changed := blockChanged(img.Pix, prev.pix, stride, prevStride, x, y, ww, hh)
if !changed && blockIntersectsROI(x, y, ww, hh, rois) {
changed = blockChangedbackstage(img.Pix, prev.pix, stride, prevStride, x, y, ww, hh)
2026-03-05 00:04:26 -06:00
}
if changed {
changedGrid[by*blocksWide+bx] = true
changedCount++
}
}
}
statDetectNs.Add(time.Since(passDetectStart).Nanoseconds())
if changedCount == 0 {
2026-05-21 23:28:06 -05:00
return 0, []byte{byte(width), byte(width >> 8), byte(height), byte(height >> 8), 0, 0, 0, 0}, 0, nil
2026-03-05 00:04:26 -06:00
}
mergeStart := time.Now()
var regions []streamRegion
for by := 0; by < blocksHigh; by++ {
for bx := 0; bx < blocksWide; bx++ {
idx := by*blocksWide + bx
if !changedGrid[idx] || visited[idx] {
continue
}
endX := bx
for endX+1 < blocksWide && changedGrid[by*blocksWide+endX+1] && !visited[by*blocksWide+endX+1] {
endX++
}
endY := by
canExpandY := true
for canExpandY && endY+1 < blocksHigh {
for tx := bx; tx <= endX; tx++ {
if !changedGrid[(endY+1)*blocksWide+tx] || visited[(endY+1)*blocksWide+tx] {
canExpandY = false
break
}
}
if canExpandY {
endY++
}
}
for ry := by; ry <= endY; ry++ {
for rx := bx; rx <= endX; rx++ {
visited[ry*blocksWide+rx] = true
}
}
x := bx * blockSize
y := by * blockSize
w := ((endX + 1) * blockSize)
h := ((endY + 1) * blockSize)
if x >= blockMargin {
x -= blockMargin
w += blockMargin
} else {
w += x
x = 0
}
if y >= blockMargin {
y -= blockMargin
h += blockMargin
} else {
h += y
y = 0
}
w += blockMargin
h += blockMargin
if w > width {
w = width
}
if h > height {
h = height
}
w -= x
h -= y
regions = append(regions, streamRegion{x: x, y: y, w: w, h: h})
}
}
sortRegionsByROI(regions, rois)
statMergeNs.Add(time.Since(mergeStart).Nanoseconds())
2026-05-21 23:28:06 -05:00
buf := frameBufPool.Get().(*bytes.Buffer)
buf.Reset()
defer frameBufPool.Put(buf)
writeUint16LE(buf, uint16(width))
writeUint16LE(buf, uint16(height))
writeUint16LE(buf, uint16(len(regions)))
writeUint16LE(buf, 0)
2026-03-05 00:04:26 -06:00
totalEncDur := time.Duration(0)
for _, r := range regions {
t0 := time.Now()
2026-05-21 23:28:06 -05:00
writeUint16LE(buf, uint16(r.x))
writeUint16LE(buf, uint16(r.y))
writeUint16LE(buf, uint16(r.w))
writeUint16LE(buf, uint16(r.h))
var encErr error
2026-03-05 00:04:26 -06:00
if codec == "raw" {
2026-05-21 23:28:06 -05:00
payload := encodeBlockRaw(img, r.x, r.y, r.w, r.h)
writeUint32LE(buf, uint32(len(payload)))
buf.Write(payload)
2026-03-05 00:04:26 -06:00
} else {
blockQuality := quality + 10
if blockQuality > 100 {
blockQuality = 100
}
2026-05-21 23:28:06 -05:00
lenPos := buf.Len()
writeUint32LE(buf, 0) // placeholder; patched below with actual JPEG size
encErr = encodeJPEGToBuf(buf, img.SubImage(image.Rect(r.x, r.y, r.x+r.w, r.y+r.h)), blockQuality)
if encErr == nil {
jpegLen := uint32(buf.Len() - lenPos - 4)
b := buf.Bytes()
b[lenPos] = byte(jpegLen)
b[lenPos+1] = byte(jpegLen >> 8)
b[lenPos+2] = byte(jpegLen >> 16)
b[lenPos+3] = byte(jpegLen >> 24)
}
2026-03-05 00:04:26 -06:00
}
blockDur := time.Since(t0)
totalEncDur += blockDur
statBlkJpegNs.Add(blockDur.Nanoseconds())
2026-05-21 23:28:06 -05:00
if encErr != nil {
return len(regions), nil, totalEncDur, encErr
2026-03-05 00:04:26 -06:00
}
}
prevCopyStart := time.Now()
prevMu.Lock()
copyPrev(img)
prevMu.Unlock()
statPrevCopyNs.Add(time.Since(prevCopyStart).Nanoseconds())
2026-05-21 23:28:06 -05:00
out := make([]byte, buf.Len())
copy(out, buf.Bytes())
return len(regions), out, totalEncDur, nil
2026-03-05 00:04:26 -06:00
}
func blockChanged(cur, prev []byte, curStride, prevStride int, x, y, w, h int) bool {
changedPixels := 0
sampledPixels := 0
for row := 0; row < h; row += samplingRate {
for col := 0; col < w; col += samplingRate {
sampledPixels++
ci := (y+row)*curStride + (x+col)*4
pi := (y+row)*prevStride + (x+col)*4
if ci+3 >= len(cur) || pi+3 >= len(prev) {
continue
}
dr := int(cur[ci]) - int(prev[pi])
dg := int(cur[ci+1]) - int(prev[pi+1])
db := int(cur[ci+2]) - int(prev[pi+2])
if dr < 0 {
dr = -dr
}
if dg < 0 {
dg = -dg
}
if db < 0 {
db = -db
}
if dr > changeThresh || dg > changeThresh || db > changeThresh {
changedPixels++
if sampledPixels > 20 && changedPixels*33 > sampledPixels {
return true
}
}
}
}
if sampledPixels == 0 {
return false
}
return changedPixels*33 > sampledPixels
}
func collectROIHints(display, width, height int) []roiHint {
frame := image.Rect(0, 0, width, height)
bounds := DisplayBounds(display)
out := make([]roiHint, 0, 2)
add := func(abs image.Rectangle, weight int, margin int) {
if abs.Dx() <= 0 || abs.Dy() <= 0 {
return
}
abs = expandRect(abs, margin)
local := image.Rect(
abs.Min.X-bounds.Min.X,
abs.Min.Y-bounds.Min.Y,
abs.Max.X-bounds.Min.X,
abs.Max.Y-bounds.Min.Y,
).Intersect(frame)
if local.Dx() > 0 && local.Dy() > 0 {
out = append(out, roiHint{rect: local, weight: weight})
}
}
if r, ok := cursorROI(); ok {
add(r, 3, cursorROIMargin)
}
if r, ok := focusWindowROI(); ok {
add(r, 2, windowROIMargin)
}
return out
}
func expandRect(r image.Rectangle, margin int) image.Rectangle {
if margin <= 0 {
return r
}
return image.Rect(r.Min.X-margin, r.Min.Y-margin, r.Max.X+margin, r.Max.Y+margin)
}
func blockIntersectsROI(x, y, w, h int, hints []roiHint) bool {
if len(hints) == 0 {
return false
}
block := image.Rect(x, y, x+w, y+h)
for _, h := range hints {
if block.Overlaps(h.rect) {
return true
}
}
return false
}
func overlapArea(a, b image.Rectangle) int {
r := a.Intersect(b)
if r.Dx() <= 0 || r.Dy() <= 0 {
return 0
}
return r.Dx() * r.Dy()
}
func regionPriority(r streamRegion, hints []roiHint) int {
if len(hints) == 0 {
return 0
}
rr := image.Rect(r.x, r.y, r.x+r.w, r.y+r.h)
score := 0
for _, h := range hints {
score += overlapArea(rr, h.rect) * h.weight
}
return score
}
func sortRegionsByROI(regions []streamRegion, hints []roiHint) {
if len(regions) < 2 || len(hints) == 0 {
return
}
sort.SliceStable(regions, func(i, j int) bool {
si := regionPriority(regions[i], hints)
sj := regionPriority(regions[j], hints)
if si != sj {
return si > sj
}
if regions[i].y != regions[j].y {
return regions[i].y < regions[j].y
}
return regions[i].x < regions[j].x
})
}
func blockChangedbackstage(cur, prev []byte, curStride, prevStride int, x, y, w, h int) bool {
2026-03-05 00:04:26 -06:00
changedPixels := 0
sampledPixels := 0
for row := 0; row < h; row += backstageSamplingRate {
for col := 0; col < w; col += backstageSamplingRate {
2026-03-05 00:04:26 -06:00
sampledPixels++
ci := (y+row)*curStride + (x+col)*4
pi := (y+row)*prevStride + (x+col)*4
if ci+3 >= len(cur) || pi+3 >= len(prev) {
continue
}
dr := int(cur[ci]) - int(prev[pi])
dg := int(cur[ci+1]) - int(prev[pi+1])
db := int(cur[ci+2]) - int(prev[pi+2])
if dr < 0 {
dr = -dr
}
if dg < 0 {
dg = -dg
}
if db < 0 {
db = -db
}
if dr > backstageChangeThresh || dg > backstageChangeThresh || db > backstageChangeThresh {
2026-03-05 00:04:26 -06:00
changedPixels++
if sampledPixels > 20 && changedPixels*100 > sampledPixels {
return true
}
}
}
}
if sampledPixels == 0 {
return false
}
return changedPixels*100 > sampledPixels
}
func encodeBlocksbackstage(img *image.RGBA, prev *prevImage, quality int, codec string, display int) (int, []byte, time.Duration, error) {
2026-03-05 00:04:26 -06:00
width := img.Bounds().Dx()
height := img.Bounds().Dy()
stride := img.Stride
prevStride := prev.w * 4
rois := collectROIHints(display, width, height)
blocksWide := (width + blockSize - 1) / blockSize
blocksHigh := (height + blockSize - 1) / blockSize
2026-05-21 23:28:06 -05:00
total := blocksWide * blocksHigh
eb := encodeBufsPool.Get().(*encodeBufs)
defer encodeBufsPool.Put(eb)
if cap(eb.changedGrid) < total {
eb.changedGrid = make([]bool, total)
eb.visited = make([]bool, total)
} else {
eb.changedGrid = eb.changedGrid[:total]
eb.visited = eb.visited[:total]
clear(eb.changedGrid)
clear(eb.visited)
}
changedGrid := eb.changedGrid
visited := eb.visited
2026-03-05 00:04:26 -06:00
changedCount := 0
passDetectStart := time.Now()
for by := 0; by < blocksHigh; by++ {
for bx := 0; bx < blocksWide; bx++ {
x := bx * blockSize
y := by * blockSize
ww := blockSize
hh := blockSize
if x+ww > width {
ww = width - x
}
if y+hh > height {
hh = height - y
}
if blockChangedbackstage(img.Pix, prev.pix, stride, prevStride, x, y, ww, hh) {
2026-03-05 00:04:26 -06:00
changedGrid[by*blocksWide+bx] = true
changedCount++
}
}
}
statDetectNs.Add(time.Since(passDetectStart).Nanoseconds())
if changedCount == 0 {
2026-05-21 23:28:06 -05:00
return 0, []byte{byte(width), byte(width >> 8), byte(height), byte(height >> 8), 0, 0, 0, 0}, 0, nil
2026-03-05 00:04:26 -06:00
}
mergeStart := time.Now()
var regions []streamRegion
for by := 0; by < blocksHigh; by++ {
for bx := 0; bx < blocksWide; bx++ {
idx := by*blocksWide + bx
if !changedGrid[idx] || visited[idx] {
continue
}
endX := bx
for endX+1 < blocksWide && changedGrid[by*blocksWide+endX+1] && !visited[by*blocksWide+endX+1] {
endX++
}
endY := by
canExpandY := true
for canExpandY && endY+1 < blocksHigh {
for tx := bx; tx <= endX; tx++ {
if !changedGrid[(endY+1)*blocksWide+tx] || visited[(endY+1)*blocksWide+tx] {
canExpandY = false
break
}
}
if canExpandY {
endY++
}
}
for ry := by; ry <= endY; ry++ {
for rx := bx; rx <= endX; rx++ {
visited[ry*blocksWide+rx] = true
}
}
x := bx * blockSize
y := by * blockSize
w := ((endX + 1) * blockSize)
h := ((endY + 1) * blockSize)
if x >= blockMargin {
x -= blockMargin
w += blockMargin
} else {
w += x
x = 0
}
if y >= blockMargin {
y -= blockMargin
h += blockMargin
} else {
h += y
y = 0
}
w += blockMargin
h += blockMargin
if w > width {
w = width
}
if h > height {
h = height
}
w -= x
h -= y
regions = append(regions, streamRegion{x, y, w, h})
}
}
sortRegionsByROI(regions, rois)
statMergeNs.Add(time.Since(mergeStart).Nanoseconds())
2026-05-21 23:28:06 -05:00
buf := frameBufPool.Get().(*bytes.Buffer)
buf.Reset()
defer frameBufPool.Put(buf)
writeUint16LE(buf, uint16(width))
writeUint16LE(buf, uint16(height))
writeUint16LE(buf, uint16(len(regions)))
writeUint16LE(buf, 0)
2026-03-05 00:04:26 -06:00
totalEncDur := time.Duration(0)
for _, r := range regions {
encStart := time.Now()
2026-05-21 23:28:06 -05:00
writeUint16LE(buf, uint16(r.x))
writeUint16LE(buf, uint16(r.y))
writeUint16LE(buf, uint16(r.w))
writeUint16LE(buf, uint16(r.h))
2026-03-05 00:04:26 -06:00
if codec == "raw" {
2026-05-21 23:28:06 -05:00
payload := encodeBlockRaw(img, r.x, r.y, r.w, r.h)
writeUint32LE(buf, uint32(len(payload)))
buf.Write(payload)
2026-03-05 00:04:26 -06:00
} else {
blockQuality := quality + 10
if blockQuality > 100 {
blockQuality = 100
}
2026-05-21 23:28:06 -05:00
lenPos := buf.Len()
writeUint32LE(buf, 0) // placeholder; patched below with actual JPEG size
if err := encodeJPEGToBuf(buf, img.SubImage(image.Rect(r.x, r.y, r.x+r.w, r.y+r.h)), blockQuality); err != nil {
2026-03-05 00:04:26 -06:00
return 0, nil, 0, err
}
2026-05-21 23:28:06 -05:00
jpegLen := uint32(buf.Len() - lenPos - 4)
b := buf.Bytes()
b[lenPos] = byte(jpegLen)
b[lenPos+1] = byte(jpegLen >> 8)
b[lenPos+2] = byte(jpegLen >> 16)
b[lenPos+3] = byte(jpegLen >> 24)
2026-03-05 00:04:26 -06:00
}
totalEncDur += time.Since(encStart)
}
prevCopyStart := time.Now()
prevMu.Lock()
copyPrev(img)
prevMu.Unlock()
statPrevCopyNs.Add(time.Since(prevCopyStart).Nanoseconds())
2026-05-21 23:28:06 -05:00
out := make([]byte, buf.Len())
copy(out, buf.Bytes())
return len(regions), out, totalEncDur, nil
2026-03-05 00:04:26 -06:00
}
func copyPrev(img *image.RGBA) {
width := img.Bounds().Dx()
height := img.Bounds().Dy()
n := len(img.Pix)
if prevFrame != nil && cap(prevFrame.pix) >= n {
prevFrame.w = width
prevFrame.h = height
prevFrame.pix = prevFrame.pix[:n]
copy(prevFrame.pix, img.Pix)
} else {
buf := make([]byte, n)
copy(buf, img.Pix)
prevFrame = &prevImage{w: width, h: height, pix: buf}
}
2026-03-05 00:04:26 -06:00
}
func copyPrevbackstage(img *image.RGBA) {
width := img.Bounds().Dx()
height := img.Bounds().Dy()
n := len(img.Pix)
if backstagePrevFrame != nil && cap(backstagePrevFrame.pix) >= n {
backstagePrevFrame.w = width
backstagePrevFrame.h = height
backstagePrevFrame.pix = backstagePrevFrame.pix[:n]
copy(backstagePrevFrame.pix, img.Pix)
} else {
buf := make([]byte, n)
copy(buf, img.Pix)
backstagePrevFrame = &prevImage{w: width, h: height, pix: buf}
}
}
2026-03-05 00:04:26 -06:00
func encodeBlockRaw(img *image.RGBA, x, y, w, h int) []byte {
stride := img.Stride
buf := make([]byte, w*h*4)
dst := 0
srcBase := y*stride + x*4
for row := 0; row < h; row++ {
src := srcBase + row*stride
copy(buf[dst:dst+w*4], img.Pix[src:src+w*4])
dst += w * 4
}
return buf
}
func blockCodec() string {
if v := overrideCodec.Load(); v != nil {
if s, ok := v.(string); ok && s != "" {
return s
}
}
blockCodecOnce.Do(func() {
codec := strings.ToLower(os.Getenv(blockCodecEnv))
switch codec {
case "h264":
cachedBlockCodec = "h264"
2026-07-18 01:27:58 -05:00
case "hevc", "h265":
cachedBlockCodec = "hevc"
2026-03-05 00:04:26 -06:00
case "raw", "rgba":
cachedBlockCodec = "raw"
case "jpeg", "":
cachedBlockCodec = "jpeg"
default:
cachedBlockCodec = "jpeg"
}
})
return cachedBlockCodec
}
func SetQualityAndCodec(quality int, codec string) {
if quality > 0 {
if quality > 100 {
quality = 100
}
overrideQuality.Store(int64(quality))
}
s := strings.ToLower(strings.TrimSpace(codec))
2026-07-18 01:27:58 -05:00
if s == "h265" {
s = "hevc"
}
if s == "hevc" && !hevcAvailable() {
detail := hevcAvailabilityDetail()
fallback := "jpeg"
if h264Available() {
fallback = "h264"
}
log.Printf("capture: requested codec=hevc but unavailable (%s); forcing codec=%s", detail, fallback)
s = fallback
}
if s == "h264" && !h264Available() {
2026-03-06 13:27:51 -06:00
detail := h264AvailabilityDetail()
if detail != "" {
log.Printf("capture: requested codec=h264 but unavailable (%s); forcing codec=jpeg", detail)
} else {
log.Printf("capture: requested codec=h264 but unavailable; forcing codec=jpeg")
}
s = "jpeg"
}
2026-03-05 00:04:26 -06:00
switch s {
2026-07-18 01:27:58 -05:00
case "raw", "rgba", "jpeg", "h264", "hevc":
2026-03-05 00:04:26 -06:00
overrideCodec.Store(s)
h264WarnOnce = sync.Once{}
2026-07-18 01:27:58 -05:00
hevcWarnOnce = sync.Once{}
if s != "h264" {
resetH264Encoder()
resetH264Encoderbackstage()
}
2026-07-18 01:27:58 -05:00
if s != "hevc" {
resetHEVCEncoder()
}
2026-03-05 00:04:26 -06:00
case "":
default:
overrideCodec.Store("jpeg")
resetH264Encoder()
resetH264Encoderbackstage()
2026-07-18 01:27:58 -05:00
resetHEVCEncoder()
2026-03-05 00:04:26 -06:00
}
}
func Nowbackstage(ctx context.Context, env *rt.Env) error {
2026-03-05 00:04:26 -06:00
if env.Cfg.DisableCapture {
return sendBlackFramebackstage(ctx, env)
2026-03-05 00:04:26 -06:00
}
2026-05-22 20:34:20 -05:00
if !supportsBackstageCapture() {
2026-03-05 00:04:26 -06:00
return nil
}
return captureAndSendbackstage(ctx, env)
2026-03-05 00:04:26 -06:00
}
2026-07-12 14:23:41 -05:00
func NowVirtual(ctx context.Context, env *rt.Env) error {
if env.Cfg.DisableCapture {
return sendBlackFrameVirtual(ctx, env)
}
if VirtualMonitorCount() <= 0 {
return nil
}
return captureAndSendVirtual(ctx, env)
}
func captureAndSendVirtual(ctx context.Context, env *rt.Env) error {
defer func() {
if r := recover(); r != nil {
log.Printf("hidden capture: panic in captureAndSendVirtual: %v", r)
}
}()
t0 := time.Now()
if frame, captureDur, encodeDur, used, err := VirtualTryDirectH264Frame(); used {
if err == nil {
if len(frame.Data) == 0 {
return nil
}
return virtualSendCompletedFrame(ctx, env, frame, t0, captureDur, encodeDur)
}
log.Printf("hidden capture: direct GPU path unavailable: %v", err)
}
// Virtual mode is normal remote desktop restricted to one monitor. Start
// with that exact capture stack; specialized paths are fallbacks only.
img, err := VirtualCaptureNormal()
if err != nil {
img, err = VirtualCaptureDisplay()
if err != nil {
VirtualResetDXGI()
img, err = VirtualCaptureGDI()
if err != nil {
img, err = VirtualCaptureDisplayFallback()
}
}
}
if err != nil {
log.Printf("hidden capture: all capture methods failed: %v (sending black frame)", err)
return sendBlackFrameVirtual(ctx, env)
}
if img == nil {
log.Printf("hidden capture: capture returned nil image (sending black frame)")
return sendBlackFrameVirtual(ctx, env)
}
captureDur := time.Since(t0)
willSendViaWebRTC := blockCodec() == "h264" && webrtcpub.IsActive(webrtcpub.Kindbackstage)
2026-07-12 14:23:41 -05:00
var slotAcquired bool
if !willSendViaWebRTC && !AcquireFrameSlot() {
PutRGBA(img)
return nil
}
slotAcquired = !willSendViaWebRTC
quality := jpegQuality()
frame, encodeDur, err := buildFramebackstage(img, 0, quality)
2026-07-12 14:23:41 -05:00
PutRGBA(img)
img = nil
if err != nil {
if slotAcquired {
ReleaseFrameSlot()
}
return err
}
now := time.Now()
fps := frameFPS(now)
if fps <= 0 {
fps = 1
}
frame.Header.FPS = fps
if ctx.Err() != nil {
if slotAcquired {
ReleaseFrameSlot()
}
return nil
}
if frame.Header.Format == "h264" && webrtcpub.IsActive(webrtcpub.Kindbackstage) {
2026-07-12 14:23:41 -05:00
dur := time.Second / time.Duration(fps)
if dur <= 0 {
dur = 33 * time.Millisecond
}
if werr := webrtcpub.WriteH264(webrtcpub.Kindbackstage, frame.Data, dur); werr != nil {
2026-07-12 14:23:41 -05:00
log.Printf("webrtc: write hidden h264 failed: %v", werr)
}
if slotAcquired {
ReleaseFrameSlot()
}
return nil
}
if !slotAcquired {
if !AcquireFrameSlot() {
return nil
}
slotAcquired = true
}
sendStart := time.Now()
err = wire.WriteMsg(ctx, env.Conn, frame)
sendDur := time.Since(sendStart)
if err != nil {
ReleaseFrameSlot()
}
if shouldLogFrame(now) {
total := time.Since(t0)
log.Printf("hidden capture: stream format=%s size=%d cap=%s enc=%s send=%s total=%s",
frame.Header.Format, len(frame.Data), captureDur, encodeDur, sendDur, total)
}
return err
}
func virtualSendCompletedFrame(ctx context.Context, env *rt.Env, frame wire.Frame, t0 time.Time, captureDur, encodeDur time.Duration) error {
now := time.Now()
fps := frameFPS(now)
if fps <= 0 {
fps = 1
}
frame.Header.FPS = fps
if ctx.Err() != nil {
return nil
}
if frame.Header.Format == "h264" && webrtcpub.IsActive(webrtcpub.Kindbackstage) {
2026-07-12 14:23:41 -05:00
dur := time.Second / time.Duration(fps)
if dur <= 0 {
dur = 33 * time.Millisecond
}
if werr := webrtcpub.WriteH264(webrtcpub.Kindbackstage, frame.Data, dur); werr != nil {
2026-07-12 14:23:41 -05:00
log.Printf("webrtc: write hidden h264 failed: %v", werr)
}
return nil
}
if !AcquireFrameSlot() {
return nil
}
sendStart := time.Now()
err := wire.WriteMsg(ctx, env.Conn, frame)
sendDur := time.Since(sendStart)
ReleaseFrameSlot()
if shouldLogFrame(now) {
total := time.Since(t0)
log.Printf("hidden capture: direct h264 format=%s size=%d cap=%s enc=%s send=%s total=%s",
frame.Header.Format, len(frame.Data), captureDur, encodeDur, sendDur, total)
}
return err
}
func sendBlackFrameVirtual(ctx context.Context, env *rt.Env) error {
if ctx.Err() != nil {
return nil
}
img := image.NewRGBA(image.Rect(0, 0, 64, 64))
quality := 60
frame, _, err := buildFramebackstage(img, 0, quality)
2026-07-12 14:23:41 -05:00
if err != nil {
return err
}
frame.Header.FPS = 1
return wire.WriteMsg(ctx, env.Conn, frame)
}
2026-05-22 20:34:20 -05:00
func supportsBackstageCapture() bool {
count := BackstageMonitorCount()
2026-03-05 00:04:26 -06:00
return count > 0
}
func captureAndSendbackstage(ctx context.Context, env *rt.Env) error {
2026-03-05 00:04:26 -06:00
defer func() {
if r := recover(); r != nil {
log.Printf("backstage capture: panic in captureAndSendbackstage: %v", r)
2026-03-05 00:04:26 -06:00
}
}()
display := env.BackstageSelectedDisplay
if display < 0 || display >= BackstageMonitorCount() {
2026-03-05 00:04:26 -06:00
display = 0
log.Printf("backstage capture: requested display %d out of range, defaulting to 0", display)
2026-03-05 00:04:26 -06:00
}
t0 := time.Now()
2026-05-22 20:34:20 -05:00
img, err := safeBackstageCaptureDisplay(display)
2026-03-05 00:04:26 -06:00
if err != nil {
log.Printf("backstage capture: capture failed: %v (sending black frame)", err)
return sendBlackFramebackstage(ctx, env)
2026-03-05 00:04:26 -06:00
}
if img == nil {
log.Printf("backstage capture: capture returned nil image (sending black frame)")
return sendBlackFramebackstage(ctx, env)
2026-03-05 00:04:26 -06:00
}
captureDur := time.Since(t0)
willSendViaWebRTC := blockCodec() == "h264" && webrtcpub.IsActive(webrtcpub.Kindbackstage)
2026-06-12 09:14:49 -05:00
var slotAcquired bool
if !willSendViaWebRTC && !AcquireFrameSlot() {
PutRGBA(img)
2026-06-12 09:14:49 -05:00
return nil
}
slotAcquired = !willSendViaWebRTC
2026-03-05 00:04:26 -06:00
quality := jpegQuality()
frame, encodeDur, err := buildFramebackstage(img, display, quality)
PutRGBA(img)
img = nil
2026-03-05 00:04:26 -06:00
if err != nil {
2026-06-12 09:14:49 -05:00
if slotAcquired {
ReleaseFrameSlot()
}
2026-03-05 00:04:26 -06:00
return err
}
now := time.Now()
fps := frameFPS(now)
if fps <= 0 {
fps = 1
}
frame.Header.FPS = fps
if ctx.Err() != nil {
2026-06-12 09:14:49 -05:00
if slotAcquired {
ReleaseFrameSlot()
}
2026-03-05 00:04:26 -06:00
return nil
}
if frame.Header.Format == "h264" && webrtcpub.IsActive(webrtcpub.Kindbackstage) {
2026-06-12 09:14:49 -05:00
dur := time.Second / time.Duration(fps)
if dur <= 0 {
dur = 33 * time.Millisecond
}
if werr := webrtcpub.WriteH264(webrtcpub.Kindbackstage, frame.Data, dur); werr != nil {
log.Printf("webrtc: write backstage h264 failed: %v", werr)
2026-06-12 09:14:49 -05:00
}
if slotAcquired {
ReleaseFrameSlot()
}
return nil
}
if !slotAcquired {
if !AcquireFrameSlot() {
return nil
}
slotAcquired = true
}
2026-03-05 00:04:26 -06:00
sendStart := time.Now()
err = wire.WriteMsg(ctx, env.Conn, frame)
sendDur := time.Since(sendStart)
2026-06-12 09:14:49 -05:00
if err != nil {
ReleaseFrameSlot()
}
2026-03-05 00:04:26 -06:00
if shouldLogFrame(now) {
total := time.Since(t0)
log.Printf("backstage capture: stream display=%d fps≈%d format=%s size=%d cap=%s enc=%s send=%s total=%s",
2026-03-05 00:04:26 -06:00
display, fps, frame.Header.Format, len(frame.Data), captureDur, encodeDur, sendDur, total)
}
return err
}
2026-05-22 20:34:20 -05:00
func safeBackstageCaptureDisplay(display int) (*image.RGBA, error) {
2026-03-05 00:04:26 -06:00
defer func() {
_ = recover()
}()
2026-05-22 20:34:20 -05:00
img, err := BackstageCaptureDisplay(display)
2026-03-05 00:04:26 -06:00
if err != nil {
return nil, err
}
return img, nil
}
func sendBlackFramebackstage(ctx context.Context, env *rt.Env) error {
2026-03-05 00:04:26 -06:00
if ctx.Err() != nil {
return nil
}
img := image.NewRGBA(image.Rect(0, 0, 64, 64))
quality := 60
frame, _, err := buildFramebackstage(img, 0, quality)
2026-03-05 00:04:26 -06:00
if err != nil {
return err
}
frame.Header.FPS = 1
return wire.WriteMsg(ctx, env.Conn, frame)
}