HEVC (and some other small shit)

Overlord-Client/cmd/agent/capture/capture.go

@@ -111,7 +111,7 @@ func CaptureAndSend(ctx context.Context, env *rt.Env) error {
 	if frame, captureDur, encodeDur, used, err := tryBuildDirectH264Frame(display); used {
 		if err != nil {
 			consecutiveCaptureFails.Add(1)
-			log.Printf("capture: direct h264 capture failed: %v (sending black frame, consecutive=%d)", err, consecutiveCaptureFails.Load())
+			log.Printf("capture: direct video capture failed: %v (sending black frame, consecutive=%d)", err, consecutiveCaptureFails.Load())
 			return sendBlackFrame(ctx, env)
 		}
 		if len(frame.Data) == 0 {
@@ -513,6 +513,7 @@ var (
 	overrideCodec       atomic.Value
 	desktopSoftwareH264 atomic.Bool
 	h264WarnOnce        sync.Once
+	hevcWarnOnce        sync.Once
 	codecLogOnce        sync.Once
 
 	prevMu    sync.Mutex
@@ -756,6 +757,33 @@ func buildFrame(img *image.RGBA, display int, quality int) (wire.Frame, time.Dur
 	full := false
 	now := time.Now()
 	codec := blockCodec()
+	if codec == "hevc" {
+		if width%2 != 0 || height%2 != 0 {
+			log.Printf("capture: hevc skipped for odd dimensions (%dx%d), falling back to h264/jpeg", width, height)
+			codec = "h264"
+		} else {
+			if webrtcpub.ConsumeKeyframeRequest() {
+				resetHEVCEncoder()
+			}
+			hevcBytes, err := encodeHEVCFrame(img)
+			if err == nil && len(hevcBytes) > 0 {
+				lastKeyframe.Store(now.UnixNano())
+				statFullFrames.Add(1)
+				return wire.Frame{Type: "frame", Header: wire.FrameHeader{Monitor: display, FPS: 0, Format: "hevc"}, Data: hevcBytes}, time.Since(encStart), nil
+			}
+			hevcWarnOnce.Do(func() {
+				log.Printf("capture: hevc encode unavailable, falling back to h264/jpeg: %v (%s)", err, hevcAvailabilityDetail())
+			})
+			resetHEVCEncoder()
+			if h264Available() {
+				codec = "h264"
+				overrideCodec.Store("h264")
+			} else {
+				codec = "jpeg"
+				overrideCodec.Store("jpeg")
+			}
+		}
+	}
 
 	if codec == "h264" {
 		if width%2 != 0 || height%2 != 0 {
@@ -1567,6 +1595,8 @@ func blockCodec() string {
 		switch codec {
 		case "h264":
 			cachedBlockCodec = "h264"
+		case "hevc", "h265":
+			cachedBlockCodec = "hevc"
 		case "raw", "rgba":
 			cachedBlockCodec = "raw"
 		case "jpeg", "":
@@ -1586,6 +1616,18 @@ func SetQualityAndCodec(quality int, codec string) {
 		overrideQuality.Store(int64(quality))
 	}
 	s := strings.ToLower(strings.TrimSpace(codec))
+	if s == "h265" {
+		s = "hevc"
+	}
+	if s == "hevc" && !hevcAvailable() {
+		detail := hevcAvailabilityDetail()
+		fallback := "jpeg"
+		if h264Available() {
+			fallback = "h264"
+		}
+		log.Printf("capture: requested codec=hevc but unavailable (%s); forcing codec=%s", detail, fallback)
+		s = fallback
+	}
 	if s == "h264" && !h264Available() {
 		detail := h264AvailabilityDetail()
 		if detail != "" {
@@ -1596,19 +1638,24 @@ func SetQualityAndCodec(quality int, codec string) {
 		s = "jpeg"
 	}
 	switch s {
-	case "raw", "rgba", "jpeg", "h264":
+	case "raw", "rgba", "jpeg", "h264", "hevc":
 		overrideCodec.Store(s)
 		h264WarnOnce = sync.Once{}
+		hevcWarnOnce = sync.Once{}
 		if s != "h264" {
 			resetH264Encoder()
 			resetH264Encoderbackstage()
 		}
+		if s != "hevc" {
+			resetHEVCEncoder()
+		}
 	case "":
 
 	default:
 		overrideCodec.Store("jpeg")
 		resetH264Encoder()
 		resetH264Encoderbackstage()
+		resetHEVCEncoder()
 	}
 }
 

Overlord-Client/cmd/agent/capture/desktop_encoder_capabilities.go

@@ -9,9 +9,59 @@ type DesktopEncoderProfile struct {
 	Providers []string `msgpack:"providers" json:"providers"`
 }
 
+type DesktopCodecCapability struct {
+	Codec      string   `msgpack:"codec" json:"codec"`
+	Encoders   []string `msgpack:"encoders,omitempty" json:"encoders,omitempty"`
+	Transports []string `msgpack:"transports" json:"transports"`
+	Hardware   bool     `msgpack:"hardware,omitempty" json:"hardware,omitempty"`
+}
+
 type DesktopEncoderCapabilities struct {
-	Probed   bool                    `msgpack:"probed" json:"probed"`
+	Probed   bool                     `msgpack:"probed" json:"probed"`
-	Display  int                     `msgpack:"display" json:"display"`
+	Display  int                      `msgpack:"display" json:"display"`
-	Profiles []DesktopEncoderProfile `msgpack:"profiles" json:"profiles"`
+	Profiles []DesktopEncoderProfile  `msgpack:"profiles" json:"profiles"`
-	Detail   string                  `msgpack:"detail,omitempty" json:"detail,omitempty"`
+	Codecs   []DesktopCodecCapability `msgpack:"codecs" json:"codecs"`
+	Detail   string                   `msgpack:"detail,omitempty" json:"detail,omitempty"`
+}
+
+func completeDesktopEncoderCapabilities(caps DesktopEncoderCapabilities) DesktopEncoderCapabilities {
+	encoders := make([]string, 0)
+	hardware := false
+	for _, profile := range caps.Profiles {
+		for _, provider := range profile.Providers {
+			encoders = appendDesktopUniqueString(encoders, provider)
+			if provider != "" && provider != "Software H.264 / JPEG" {
+				hardware = true
+			}
+		}
+	}
+	caps.Codecs = []DesktopCodecCapability{
+		{Codec: "jpeg", Encoders: []string{"Software JPEG"}, Transports: []string{"websocket"}},
+		{Codec: "raw", Encoders: []string{"Uncompressed"}, Transports: []string{"websocket"}},
+	}
+	if hevcAvailable() {
+		caps.Codecs = append([]DesktopCodecCapability{{
+			Codec: "hevc", Encoders: []string{"NVIDIA NVENC"},
+			Transports: []string{"websocket"}, Hardware: true,
+		}}, caps.Codecs...)
+	}
+	if h264Available() {
+		if len(encoders) == 0 {
+			encoders = []string{"Software H.264"}
+		}
+		caps.Codecs = append([]DesktopCodecCapability{{
+			Codec: "h264", Encoders: encoders,
+			Transports: []string{"websocket", "webrtc"}, Hardware: hardware,
+		}}, caps.Codecs...)
+	}
+	return caps
+}
+
+func appendDesktopUniqueString(values []string, value string) []string {
+	for _, existing := range values {
+		if existing == value {
+			return values
+		}
+	}
+	return append(values, value)
 }

Overlord-Client/cmd/agent/capture/desktop_encoder_capabilities_stub.go

@@ -3,12 +3,12 @@
 package capture
 
 func ProbeDesktopEncoderCapabilities(display int) DesktopEncoderCapabilities {
-	return DesktopEncoderCapabilities{
+	return completeDesktopEncoderCapabilities(DesktopEncoderCapabilities{
 		Display: display,
 		Profiles: []DesktopEncoderProfile{
 			{MaxHeight: 720, Width: 1280, Height: 720, FPS: 60, Label: "60 FPS - 720p", Providers: []string{"Software H.264 / JPEG"}},
 			{MaxHeight: 1080, Width: 1920, Height: 1080, FPS: 60, Label: "60 FPS - 1080p", Providers: []string{"Software H.264 / JPEG"}},
 		},
 		Detail: "Hardware H.264 capability probing is available on Windows; safe software profiles are shown.",
-	}
+	})
 }

Overlord-Client/cmd/agent/capture/desktop_encoder_capabilities_test.go

@@ -0,0 +1,46 @@
+package capture
+
+import "testing"
+
+func TestCompleteDesktopEncoderCapabilitiesAdvertisesUsableCodecs(t *testing.T) {
+	caps := completeDesktopEncoderCapabilities(DesktopEncoderCapabilities{
+		Profiles: []DesktopEncoderProfile{{
+			MaxHeight: 1080,
+			Width:     1920,
+			Height:    1080,
+			FPS:       60,
+			Providers: []string{"Software H.264 / JPEG"},
+		}},
+	})
+
+	byName := make(map[string]DesktopCodecCapability)
+	for _, codec := range caps.Codecs {
+		byName[codec.Codec] = codec
+	}
+	if _, ok := byName["jpeg"]; !ok {
+		t.Fatal("expected JPEG capability")
+	}
+	if _, ok := byName["raw"]; !ok {
+		t.Fatal("expected raw capability")
+	}
+	if h264Available() {
+		h264, ok := byName["h264"]
+		if !ok {
+			t.Fatal("expected H.264 capability when the encoder is available")
+		}
+		if len(h264.Transports) != 2 || h264.Transports[0] != "websocket" || h264.Transports[1] != "webrtc" {
+			t.Fatalf("unexpected H.264 transports: %v", h264.Transports)
+		}
+	}
+	if hevcAvailable() {
+		hevc, ok := byName["hevc"]
+		if !ok {
+			t.Fatal("expected HEVC capability when NVENC HEVC is available")
+		}
+		if len(hevc.Transports) != 1 || hevc.Transports[0] != "websocket" || !hevc.Hardware {
+			t.Fatalf("unexpected HEVC capability: %+v", hevc)
+		}
+	} else if _, ok := byName["hevc"]; ok {
+		t.Fatal("must not advertise HEVC when NVENC HEVC is unavailable")
+	}
+}

Overlord-Client/cmd/agent/capture/desktop_encoder_capabilities_windows.go

@@ -44,6 +44,7 @@ func ProbeDesktopEncoderCapabilities(display int) DesktopEncoderCapabilities {
 		return cached.caps
 	}
 	caps := probeDesktopEncoderCapabilities(display, bounds)
+	caps = completeDesktopEncoderCapabilities(caps)
 	desktopCapabilityCache.entries[cacheKey] = desktopCapabilityCacheEntry{at: time.Now(), caps: caps}
 	for key, entry := range desktopCapabilityCache.entries {
 		if time.Since(entry.at) >= desktopCapabilityCacheTTL {

Overlord-Client/cmd/agent/capture/h264_direct_windows.go

@@ -10,11 +10,21 @@ import (
 )
 
 func tryBuildDirectH264Frame(display int) (wire.Frame, time.Duration, time.Duration, bool, error) {
-	if blockCodec() != "h264" || useDesktopSoftwareH264() || !useDesktopDuplication() {
+	codec := blockCodec()
+	if (codec != "h264" && codec != "hevc") || (codec == "h264" && useDesktopSoftwareH264()) || !useDesktopDuplication() {
 		return wire.Frame{}, 0, 0, false, nil
 	}
 	forceKeyframe := webrtcpub.ConsumeKeyframeRequest()
-	data, width, height, captureDur, encodeDur, used, err := captureDisplayDXGIH264(display, forceKeyframe)
+	var data []byte
+	var width, height int
+	var captureDur, encodeDur time.Duration
+	var used bool
+	var err error
+	if codec == "hevc" {
+		data, width, height, captureDur, encodeDur, used, err = captureDisplayDXGIHEVC(display, forceKeyframe)
+	} else {
+		data, width, height, captureDur, encodeDur, used, err = captureDisplayDXGIH264(display, forceKeyframe)
+	}
 	if !used || err != nil {
 		return wire.Frame{}, captureDur, encodeDur, used, err
 	}
@@ -24,5 +34,5 @@ func tryBuildDirectH264Frame(display int) (wire.Frame, time.Duration, time.Durat
 	now := time.Now()
 	lastKeyframe.Store(now.UnixNano())
 	statFullFrames.Add(1)
-	return wire.Frame{Type: "frame", Header: wire.FrameHeader{Monitor: display, FPS: 0, Format: "h264", Width: width, Height: height}, Data: data}, captureDur, encodeDur, true, nil
+	return wire.Frame{Type: "frame", Header: wire.FrameHeader{Monitor: display, FPS: 0, Format: codec, Width: width, Height: height}, Data: data}, captureDur, encodeDur, true, nil
 }

Overlord-Client/cmd/agent/capture/hevc_codec_test.go

@@ -0,0 +1,86 @@
+package capture
+
+import (
+	"image"
+	"testing"
+)
+
+func TestHEVCEncoderProducesAnnexBHeadersAndKeyframe(t *testing.T) {
+	if !hevcAvailable() {
+		t.Skipf("HEVC encoder unavailable: %s", hevcAvailabilityDetail())
+	}
+	resetHEVCEncoder()
+	t.Cleanup(resetHEVCEncoder)
+
+	frame := image.NewRGBA(image.Rect(0, 0, 1280, 720))
+	encoded, err := encodeHEVCFrame(frame)
+	if err != nil {
+		t.Fatalf("encode HEVC frame: %v", err)
+	}
+	if len(encoded) == 0 {
+		t.Fatal("HEVC encoder returned an empty frame")
+	}
+
+	nalTypes := hevcAnnexBNALTypes(encoded)
+	for _, required := range []byte{32, 33, 34} {
+		if !containsByte(nalTypes, required) {
+			t.Fatalf("HEVC frame missing NAL type %d; got %v", required, nalTypes)
+		}
+	}
+	hasIntra := false
+	for _, nalType := range nalTypes {
+		if nalType >= 16 && nalType <= 21 {
+			hasIntra = true
+			break
+		}
+	}
+	if !hasIntra {
+		t.Fatalf("HEVC first frame is not an IRAP keyframe; got NAL types %v", nalTypes)
+	}
+}
+
+func TestHEVCD3D11TexturePipeline(t *testing.T) {
+	if !hevcAvailable() {
+		t.Skipf("HEVC encoder unavailable: %s", hevcAvailabilityDetail())
+	}
+	result := RunNVENCD3D11Smoke(NVENCD3D11SmokeOptions{
+		Width: 1280, Height: 720, FPS: 60, Frames: 4, Codec: "hevc",
+	})
+	if !result.OK {
+		t.Fatalf("HEVC D3D11 texture pipeline failed: %+v", result)
+	}
+	if result.TotalBytes == 0 {
+		t.Fatal("HEVC D3D11 texture pipeline produced no output")
+	}
+	t.Logf("HEVC D3D11 texture pipeline: first=%.2fms avg=%.2fms bytes=%d", result.FirstMS, result.AvgMS, result.TotalBytes)
+}
+
+func hevcAnnexBNALTypes(data []byte) []byte {
+	types := make([]byte, 0, 8)
+	for i := 0; i+4 < len(data); i++ {
+		startCodeLen := 0
+		if data[i] == 0 && data[i+1] == 0 && data[i+2] == 1 {
+			startCodeLen = 3
+		} else if i+4 < len(data) && data[i] == 0 && data[i+1] == 0 && data[i+2] == 0 && data[i+3] == 1 {
+			startCodeLen = 4
+		}
+		if startCodeLen == 0 {
+			continue
+		}
+		nalIndex := i + startCodeLen
+		if nalIndex < len(data) {
+			types = append(types, (data[nalIndex]>>1)&0x3f)
+			i = nalIndex
+		}
+	}
+	return types
+}
+
+func containsByte(values []byte, wanted byte) bool {
+	for _, value := range values {
+		if value == wanted {
+			return true
+		}
+	}
+	return false
+}

Overlord-Client/cmd/agent/capture/hevc_encoder_stub.go

@@ -0,0 +1,18 @@
+//go:build !windows
+
+package capture
+
+import (
+	"errors"
+	"image"
+)
+
+func encodeHEVCFrame(_ *image.RGBA) ([]byte, error) {
+	return nil, errors.New("HEVC NVENC is only available on Windows")
+}
+
+func hevcAvailable() bool { return false }
+
+func hevcAvailabilityDetail() string { return "HEVC NVENC is only available on Windows" }
+
+func resetHEVCEncoder() {}

Overlord-Client/cmd/agent/capture/hevc_encoder_windows.go

@@ -0,0 +1,68 @@
+//go:build windows
+
+package capture
+
+import (
+	"errors"
+	"fmt"
+	"image"
+	"sync"
+)
+
+var (
+	hevcMu          sync.Mutex
+	hevcEnc         h264FrameEncoder
+	hevcProbeOnce   sync.Once
+	hevcProbeOK     bool
+	hevcProbeDetail string
+)
+
+func encodeHEVCFrame(img *image.RGBA) ([]byte, error) {
+	if img == nil {
+		return nil, errors.New("nil hevc frame")
+	}
+	b := img.Bounds()
+	width, height := b.Dx(), b.Dy()
+	if width <= 0 || height <= 0 || width%2 != 0 || height%2 != 0 {
+		return nil, fmt.Errorf("hevc frame size must be positive and even, got %dx%d", width, height)
+	}
+
+	hevcMu.Lock()
+	defer hevcMu.Unlock()
+	fps := activeH264FPS()
+	if hevcEnc == nil || !hevcEnc.Matches(width, height, fps) {
+		closeH264Encoder(&hevcEnc)
+		enc, err := newNativeHEVCEncoder("desktop", width, height, fps)
+		if err != nil {
+			return nil, err
+		}
+		hevcEnc = enc
+	}
+	return hevcEnc.Encode(img)
+}
+
+func hevcAvailable() bool {
+	hevcProbeOnce.Do(func() {
+		enc, err := newNativeHEVCEncoder("probe", 1280, 720, 30)
+		if err != nil {
+			hevcProbeDetail = err.Error()
+			return
+		}
+		enc.Close()
+		hevcProbeOK = true
+		hevcProbeDetail = "NVIDIA NVENC HEVC Main profile"
+	})
+	return hevcProbeOK
+}
+
+func hevcAvailabilityDetail() string {
+	hevcAvailable()
+	return hevcProbeDetail
+}
+
+func resetHEVCEncoder() {
+	hevcMu.Lock()
+	defer hevcMu.Unlock()
+	closeH264Encoder(&hevcEnc)
+	resetNativeHEVCD3D11TextureEncoder()
+}

Overlord-Client/cmd/agent/capture/native_h264_stub_windows.go

@@ -9,6 +9,10 @@ func newNativeH264Encoder(stream string, width, height, fps int) (h264FrameEncod
 	return nil, fmt.Errorf("native NVENC D3D11 requires cgo for %dx%d@%dfps", width, height, fps)
 }
 
+func newNativeHEVCEncoder(stream string, width, height, fps int) (h264FrameEncoder, error) {
+	return nil, fmt.Errorf("native NVENC HEVC requires cgo for %dx%d@%dfps", width, height, fps)
+}
+
 func nativeH264AvailabilityDetail() string {
 	return ""
 }
@@ -20,3 +24,11 @@ func encodeNativeH264D3D11Texture(device, texture unsafe.Pointer, inputWidth, in
 func requestNativeH264D3D11TextureKeyframe() {}
 
 func resetNativeH264D3D11TextureEncoder() {}
+
+func encodeNativeHEVCD3D11Texture(device, texture unsafe.Pointer, inputWidth, inputHeight, encodeWidth, encodeHeight, fps int, dxgiFormat uint32, forceIDR bool) ([]byte, error) {
+	return nil, fmt.Errorf("native NVENC HEVC D3D11 texture encode requires cgo for input=%dx%d output=%dx%d@%dfps", inputWidth, inputHeight, encodeWidth, encodeHeight, fps)
+}
+
+func requestNativeHEVCD3D11TextureKeyframe() {}
+
+func resetNativeHEVCD3D11TextureEncoder() {}

Overlord-Client/cmd/agent/capture/nvenc_d3d11_smoke_stub.go

@@ -9,11 +9,12 @@ import (
 )
 
 type NVENCD3D11SmokeOptions struct {
-	Width   int `json:"width"`
+	Width   int    `json:"width"`
-	Height  int `json:"height"`
+	Height  int    `json:"height"`
-	FPS     int `json:"fps"`
+	FPS     int    `json:"fps"`
-	Frames  int `json:"frames"`
+	Frames  int    `json:"frames"`
-	Bitrate int `json:"bitrate"`
+	Bitrate int    `json:"bitrate"`
+	Codec   string `json:"codec,omitempty"`
 }
 
 type NVENCD3D11SmokeResult struct {

Overlord-Client/cmd/agent/capture/nvenc_d3d11_smoke_windows.go

@@ -134,6 +134,16 @@ static void nvenc_configure_h264_stream(NV_ENC_CONFIG *cfg, int fps) {
 	cfg->encodeCodecConfig.h264Config.h264VUIParameters.colourMatrix = NV_ENC_VUI_MATRIX_COEFFS_BT709;
 }
 
+static void nvenc_configure_hevc_stream(NV_ENC_CONFIG *cfg, int fps) {
+	(void)fps;
+	cfg->gopLength = NVENC_INFINITE_GOPLENGTH;
+	cfg->frameIntervalP = 1;
+	cfg->encodeCodecConfig.hevcConfig.idrPeriod = NVENC_INFINITE_GOPLENGTH;
+	cfg->encodeCodecConfig.hevcConfig.repeatSPSPPS = 1;
+	cfg->encodeCodecConfig.hevcConfig.level = NV_ENC_LEVEL_HEVC_52;
+	cfg->encodeCodecConfig.hevcConfig.chromaFormatIDC = 1;
+}
+
 static void nvenc_set_video_processor_color_space(nvenc_d3d11_texture_encoder *enc) {
 	D3D11_VIDEO_PROCESSOR_COLOR_SPACE input_space;
 	memset(&input_space, 0, sizeof(input_space));
@@ -287,7 +297,7 @@ static void nvenc_release_texture_encoder(nvenc_d3d11_texture_encoder *enc) {
 	free(enc);
 }
 
-static nvenc_d3d11_texture_create_result nvenc_create_d3d11_texture_encoder(ID3D11Device *device, int input_width, int input_height, int encode_width, int encode_height, int fps, int bitrate, NV_ENC_BUFFER_FORMAT buffer_format, int dxgi_format, int output_mode) {
+static nvenc_d3d11_texture_create_result nvenc_create_d3d11_texture_encoder(ID3D11Device *device, int input_width, int input_height, int encode_width, int encode_height, int fps, int bitrate, NV_ENC_BUFFER_FORMAT buffer_format, int dxgi_format, int output_mode, int codec) {
 	nvenc_d3d11_texture_create_result out;
 	memset(&out, 0, sizeof(out));
 	if (!device) {
@@ -374,7 +384,8 @@ static nvenc_d3d11_texture_create_result nvenc_create_d3d11_texture_encoder(ID3D
 	memset(&preset, 0, sizeof(preset));
 	preset.version = NV_ENC_PRESET_CONFIG_VER;
 	preset.presetCfg.version = NV_ENC_CONFIG_VER;
-	status = enc->api.nvEncGetEncodePresetConfigEx(enc->encoder, NV_ENC_CODEC_H264_GUID, NV_ENC_PRESET_P1_GUID, NV_ENC_TUNING_INFO_ULTRA_LOW_LATENCY, &preset);
+	GUID codec_guid = codec == 1 ? NV_ENC_CODEC_HEVC_GUID : NV_ENC_CODEC_H264_GUID;
+	status = enc->api.nvEncGetEncodePresetConfigEx(enc->encoder, codec_guid, NV_ENC_PRESET_P1_GUID, NV_ENC_TUNING_INFO_ULTRA_LOW_LATENCY, &preset);
 	if (status != NV_ENC_SUCCESS) {
 		out.stage = 9;
 		out.nvstatus = status;
@@ -382,18 +393,22 @@ static nvenc_d3d11_texture_create_result nvenc_create_d3d11_texture_encoder(ID3D
 		nvenc_release_texture_encoder(enc);
 		return out;
 	}
-	preset.presetCfg.profileGUID = NV_ENC_H264_PROFILE_HIGH_GUID;
+	preset.presetCfg.profileGUID = codec == 1 ? NV_ENC_HEVC_PROFILE_MAIN_GUID : NV_ENC_H264_PROFILE_HIGH_GUID;
 	preset.presetCfg.rcParams.rateControlMode = NV_ENC_PARAMS_RC_CBR;
 	preset.presetCfg.rcParams.averageBitRate = (uint32_t)bitrate;
 	preset.presetCfg.rcParams.maxBitRate = (uint32_t)bitrate;
 	preset.presetCfg.rcParams.vbvBufferSize = (uint32_t)(bitrate / fps);
 	preset.presetCfg.rcParams.vbvInitialDelay = (uint32_t)(bitrate / fps);
-	nvenc_configure_h264_stream(&preset.presetCfg, fps);
+	if (codec == 1) {
+		nvenc_configure_hevc_stream(&preset.presetCfg, fps);
+	} else {
+		nvenc_configure_h264_stream(&preset.presetCfg, fps);
+	}
 
 	NV_ENC_INITIALIZE_PARAMS init;
 	memset(&init, 0, sizeof(init));
 	init.version = NV_ENC_INITIALIZE_PARAMS_VER;
-	init.encodeGUID = NV_ENC_CODEC_H264_GUID;
+	init.encodeGUID = codec_guid;
 	init.presetGUID = NV_ENC_PRESET_P1_GUID;
 	init.encodeWidth = (uint32_t)encode_width;
 	init.encodeHeight = (uint32_t)encode_height;
@@ -594,13 +609,13 @@ static nvenc_d3d11_texture_create_result nvenc_create_d3d11_texture_encoder(ID3D
 	return out;
 }
 
-static nvenc_d3d11_create_result nvenc_create_d3d11_encoder(int width, int height, int fps, int bitrate) {
+static nvenc_d3d11_create_result nvenc_create_d3d11_encoder(int width, int height, int fps, int bitrate, int codec) {
 	nvenc_d3d11_create_result out;
 	memset(&out, 0, sizeof(out));
-	if (width <= 0 || height <= 0 || (width & 1) || (height & 1) || fps <= 0) {
+	if (width <= 0 || height <= 0 || (width & 1) || (height & 1) || fps <= 0 || (codec != 0 && codec != 1)) {
 		out.stage = 1;
 		out.nvstatus = NV_ENC_ERR_INVALID_PARAM;
-		strncpy(out.message, "width/height/fps must be positive, and width/height must be even", sizeof(out.message)-1);
+		strncpy(out.message, "invalid width/height/fps or codec", sizeof(out.message)-1);
 		return out;
 	}
 
@@ -679,7 +694,8 @@ static nvenc_d3d11_create_result nvenc_create_d3d11_encoder(int width, int heigh
 	memset(&preset, 0, sizeof(preset));
 	preset.version = NV_ENC_PRESET_CONFIG_VER;
 	preset.presetCfg.version = NV_ENC_CONFIG_VER;
-	status = enc->api.nvEncGetEncodePresetConfigEx(enc->encoder, NV_ENC_CODEC_H264_GUID, NV_ENC_PRESET_P1_GUID, NV_ENC_TUNING_INFO_ULTRA_LOW_LATENCY, &preset);
+	GUID codec_guid = codec == 1 ? NV_ENC_CODEC_HEVC_GUID : NV_ENC_CODEC_H264_GUID;
+	status = enc->api.nvEncGetEncodePresetConfigEx(enc->encoder, codec_guid, NV_ENC_PRESET_P1_GUID, NV_ENC_TUNING_INFO_ULTRA_LOW_LATENCY, &preset);
 	if (status != NV_ENC_SUCCESS) {
 		out.stage = 8;
 		out.nvstatus = status;
@@ -687,18 +703,22 @@ static nvenc_d3d11_create_result nvenc_create_d3d11_encoder(int width, int heigh
 		nvenc_release_encoder(enc);
 		return out;
 	}
-	preset.presetCfg.profileGUID = NV_ENC_H264_PROFILE_HIGH_GUID;
+	preset.presetCfg.profileGUID = codec == 1 ? NV_ENC_HEVC_PROFILE_MAIN_GUID : NV_ENC_H264_PROFILE_HIGH_GUID;
 	preset.presetCfg.rcParams.rateControlMode = NV_ENC_PARAMS_RC_CBR;
 	preset.presetCfg.rcParams.averageBitRate = (uint32_t)bitrate;
 	preset.presetCfg.rcParams.maxBitRate = (uint32_t)bitrate;
 	preset.presetCfg.rcParams.vbvBufferSize = (uint32_t)(bitrate / fps);
 	preset.presetCfg.rcParams.vbvInitialDelay = (uint32_t)(bitrate / fps);
-	nvenc_configure_h264_stream(&preset.presetCfg, fps);
+	if (codec == 1) {
+		nvenc_configure_hevc_stream(&preset.presetCfg, fps);
+	} else {
+		nvenc_configure_h264_stream(&preset.presetCfg, fps);
+	}
 
 	NV_ENC_INITIALIZE_PARAMS init;
 	memset(&init, 0, sizeof(init));
 	init.version = NV_ENC_INITIALIZE_PARAMS_VER;
-	init.encodeGUID = NV_ENC_CODEC_H264_GUID;
+	init.encodeGUID = codec_guid;
 	init.presetGUID = NV_ENC_PRESET_P1_GUID;
 	init.encodeWidth = (uint32_t)width;
 	init.encodeHeight = (uint32_t)height;
@@ -1325,7 +1345,7 @@ static nvenc_d3d11_smoke_result run_nvenc_d3d11_smoke_c(int width, int height, i
 	return out;
 }
 
-static nvenc_d3d11_smoke_result run_nvenc_d3d11_texture_pipeline_smoke_c(int width, int height, int fps, int frames, int bitrate) {
+static nvenc_d3d11_smoke_result run_nvenc_d3d11_texture_pipeline_smoke_c(int width, int height, int fps, int frames, int bitrate, int codec) {
 	nvenc_d3d11_smoke_result out;
 	memset(&out, 0, sizeof(out));
 	out.width = width;
@@ -1391,7 +1411,7 @@ static nvenc_d3d11_smoke_result run_nvenc_d3d11_texture_pipeline_smoke_c(int wid
 		return out;
 	}
 
-	nvenc_d3d11_texture_create_result created = nvenc_create_d3d11_texture_encoder(device, width, height, width, height, fps, bitrate, NV_ENC_BUFFER_FORMAT_ARGB, DXGI_FORMAT_B8G8R8A8_UNORM, 0);
+	nvenc_d3d11_texture_create_result created = nvenc_create_d3d11_texture_encoder(device, width, height, width, height, fps, bitrate, NV_ENC_BUFFER_FORMAT_ARGB, DXGI_FORMAT_B8G8R8A8_UNORM, 0, codec);
 	if (!created.encoder) {
 		out.stage = 5;
 		out.hr = created.hr;
@@ -1467,11 +1487,12 @@ import (
 )
 
 type NVENCD3D11SmokeOptions struct {
-	Width   int `json:"width"`
+	Width   int    `json:"width"`
-	Height  int `json:"height"`
+	Height  int    `json:"height"`
-	FPS     int `json:"fps"`
+	FPS     int    `json:"fps"`
-	Frames  int `json:"frames"`
+	Frames  int    `json:"frames"`
-	Bitrate int `json:"bitrate"`
+	Bitrate int    `json:"bitrate"`
+	Codec   string `json:"codec,omitempty"`
 }
 
 type NVENCD3D11SmokeResult struct {
@@ -1505,8 +1526,15 @@ func RunNVENCD3D11Smoke(opts NVENCD3D11SmokeOptions) NVENCD3D11SmokeResult {
 	}
 	if opts.Bitrate <= 0 {
 		opts.Bitrate = targetH264Bitrate(opts.Width, opts.Height, opts.FPS)
+		if strings.EqualFold(opts.Codec, "hevc") || strings.EqualFold(opts.Codec, "h265") {
+			opts.Bitrate = opts.Bitrate * 7 / 10
+		}
 	}
-	raw := C.run_nvenc_d3d11_texture_pipeline_smoke_c(C.int(opts.Width), C.int(opts.Height), C.int(opts.FPS), C.int(opts.Frames), C.int(opts.Bitrate))
+	codec := 0
+	if strings.EqualFold(opts.Codec, "hevc") || strings.EqualFold(opts.Codec, "h265") {
+		codec = 1
+	}
+	raw := C.run_nvenc_d3d11_texture_pipeline_smoke_c(C.int(opts.Width), C.int(opts.Height), C.int(opts.FPS), C.int(opts.Frames), C.int(opts.Bitrate), C.int(codec))
 	result := NVENCD3D11SmokeResult{
 		OK:         raw.ok != 0,
 		Width:      int(raw.width),
@@ -1545,7 +1573,7 @@ type nativeH264Encoder struct {
 
 func newNativeH264Encoder(stream string, width, height, fps int) (h264FrameEncoder, error) {
 	bitrate := targetH264Bitrate(width, height, fps)
-	raw := C.nvenc_create_d3d11_encoder(C.int(width), C.int(height), C.int(fps), C.int(bitrate))
+	raw := C.nvenc_create_d3d11_encoder(C.int(width), C.int(height), C.int(fps), C.int(bitrate), 0)
 	if raw.encoder == nil {
 		return nil, formatNVENCCreateError(raw)
 	}
@@ -1560,6 +1588,23 @@ func newNativeH264Encoder(stream string, width, height, fps int) (h264FrameEncod
 	return enc, nil
 }
 
+func newNativeHEVCEncoder(stream string, width, height, fps int) (h264FrameEncoder, error) {
+	bitrate := targetH264Bitrate(width, height, fps) * 7 / 10
+	raw := C.nvenc_create_d3d11_encoder(C.int(width), C.int(height), C.int(fps), C.int(bitrate), 1)
+	if raw.encoder == nil {
+		return nil, formatNVENCCreateError(raw)
+	}
+	enc := &nativeH264Encoder{
+		enc:          raw.encoder,
+		width:        width,
+		height:       height,
+		requestedFPS: fps,
+		fps:          fps,
+	}
+	log.Printf("capture: native NVENC D3D11 hevc encoder active stream=%s provider=NVIDIA NVENC size=%dx%d fps=%d input=NV12(upload) bitrate=%d", stream, width, height, fps, bitrate)
+	return enc, nil
+}
+
 func nativeH264AvailabilityDetail() string {
 	probe := RunNVENCSmoke()
 	if probe.Available {
@@ -1645,15 +1690,26 @@ type nativeD3D11TextureH264Encoder struct {
 	outputName   string
 	frame        uint64
 	slowLogs     int
+	codec        string
 }
 
 func newNativeD3D11TextureH264Encoder(device unsafe.Pointer, inputWidth, inputHeight, encodeWidth, encodeHeight, fps int, dxgiFormat uint32) (*nativeD3D11TextureH264Encoder, error) {
+	return newNativeD3D11TextureEncoder(device, inputWidth, inputHeight, encodeWidth, encodeHeight, fps, dxgiFormat, "h264")
+}
+
+func newNativeD3D11TextureEncoder(device unsafe.Pointer, inputWidth, inputHeight, encodeWidth, encodeHeight, fps int, dxgiFormat uint32, codec string) (*nativeD3D11TextureH264Encoder, error) {
 	bufferFormat, name, ok := nvencBufferFormatForDXGI(dxgiFormat)
 	if !ok {
 		return nil, fmt.Errorf("unsupported DXGI texture format %d", dxgiFormat)
 	}
 	bitrate := targetH264Bitrate(encodeWidth, encodeHeight, fps)
 	modes := preferredNVENCD3D11OutputModes()
+	codecID := 0
+	if codec == "hevc" {
+		codecID = 1
+		bitrate = bitrate * 7 / 10
+		modes = []int{0}
+	}
 	var lastErr error
 	var raw C.nvenc_d3d11_texture_create_result
 	outputMode := 0
@@ -1670,6 +1726,7 @@ func newNativeD3D11TextureH264Encoder(device unsafe.Pointer, inputWidth, inputHe
 			C.NV_ENC_BUFFER_FORMAT(bufferFormat),
 			C.int(dxgiFormat),
 			C.int(mode),
+			C.int(codecID),
 		)
 		if raw.encoder != nil {
 			outputMode = mode
@@ -1703,11 +1760,12 @@ func newNativeD3D11TextureH264Encoder(device unsafe.Pointer, inputWidth, inputHe
 		bufferFormat: uint32(bufferFormat),
 		outputMode:   outputMode,
 		outputName:   outputName,
+		codec:        codec,
 	}
 	if inputWidth != encodeWidth || inputHeight != encodeHeight {
-		log.Printf("capture: native NVENC D3D11 desktop texture encoder active provider=NVIDIA NVENC input=%dx%d output=%dx%d fps=%d source_format=%s(dxgi=%d) nvenc_input=%s pipeline=%s bitrate=%d", inputWidth, inputHeight, encodeWidth, encodeHeight, fps, name, dxgiFormat, outputName, pipeline, bitrate)
+		log.Printf("capture: native NVENC D3D11 desktop texture encoder active codec=%s provider=NVIDIA NVENC input=%dx%d output=%dx%d fps=%d source_format=%s(dxgi=%d) nvenc_input=%s pipeline=%s bitrate=%d", codec, inputWidth, inputHeight, encodeWidth, encodeHeight, fps, name, dxgiFormat, outputName, pipeline, bitrate)
 	} else {
-		log.Printf("capture: native NVENC D3D11 desktop texture encoder active provider=NVIDIA NVENC size=%dx%d fps=%d source_format=%s(dxgi=%d) nvenc_input=%s pipeline=%s bitrate=%d", encodeWidth, encodeHeight, fps, name, dxgiFormat, outputName, pipeline, bitrate)
+		log.Printf("capture: native NVENC D3D11 desktop texture encoder active codec=%s provider=NVIDIA NVENC size=%dx%d fps=%d source_format=%s(dxgi=%d) nvenc_input=%s pipeline=%s bitrate=%d", codec, encodeWidth, encodeHeight, fps, name, dxgiFormat, outputName, pipeline, bitrate)
 	}
 	return enc, nil
 }
@@ -1867,6 +1925,9 @@ var (
 	nativeTextureH264Mu       sync.Mutex
 	nativeTextureH264Enc      *nativeD3D11TextureH264Encoder
 	nativeTextureH264ForceIDR atomic.Bool
+	nativeTextureHEVCMu       sync.Mutex
+	nativeTextureHEVCEnc      *nativeD3D11TextureH264Encoder
+	nativeTextureHEVCForceIDR atomic.Bool
 )
 
 func encodeNativeH264D3D11Texture(device, texture unsafe.Pointer, inputWidth, inputHeight, encodeWidth, encodeHeight, fps int, dxgiFormat uint32, forceIDR bool) ([]byte, error) {
@@ -1903,3 +1964,38 @@ func resetNativeH264D3D11TextureEncoder() {
 	}
 	nativeTextureH264ForceIDR.Store(false)
 }
+
+func encodeNativeHEVCD3D11Texture(device, texture unsafe.Pointer, inputWidth, inputHeight, encodeWidth, encodeHeight, fps int, dxgiFormat uint32, forceIDR bool) ([]byte, error) {
+	if device == nil || texture == nil {
+		return nil, fmt.Errorf("nil D3D11 device or texture")
+	}
+	nativeTextureHEVCMu.Lock()
+	defer nativeTextureHEVCMu.Unlock()
+
+	if nativeTextureHEVCEnc == nil || !nativeTextureHEVCEnc.Matches(device, inputWidth, inputHeight, encodeWidth, encodeHeight, fps, dxgiFormat) {
+		if nativeTextureHEVCEnc != nil {
+			nativeTextureHEVCEnc.Close()
+			nativeTextureHEVCEnc = nil
+		}
+		enc, err := newNativeD3D11TextureEncoder(device, inputWidth, inputHeight, encodeWidth, encodeHeight, fps, dxgiFormat, "hevc")
+		if err != nil {
+			return nil, err
+		}
+		nativeTextureHEVCEnc = enc
+	}
+	return nativeTextureHEVCEnc.EncodeTexture(texture, forceIDR || nativeTextureHEVCForceIDR.Swap(false))
+}
+
+func requestNativeHEVCD3D11TextureKeyframe() {
+	nativeTextureHEVCForceIDR.Store(true)
+}
+
+func resetNativeHEVCD3D11TextureEncoder() {
+	nativeTextureHEVCMu.Lock()
+	defer nativeTextureHEVCMu.Unlock()
+	if nativeTextureHEVCEnc != nil {
+		nativeTextureHEVCEnc.Close()
+		nativeTextureHEVCEnc = nil
+	}
+	nativeTextureHEVCForceIDR.Store(false)
+}

Overlord-Client/cmd/agent/capture/win_duplication.go

@@ -661,7 +661,11 @@ func captureDisplayDXGI(display int) (*image.RGBA, error) {
 }
 
 func captureDisplayDXGIH264(display int, forceKeyframe bool) ([]byte, int, int, time.Duration, time.Duration, bool, error) {
-	return dxgiState.captureH264(display, forceKeyframe)
+	return dxgiState.captureVideo(display, "h264", forceKeyframe)
+}
+
+func captureDisplayDXGIHEVC(display int, forceKeyframe bool) ([]byte, int, int, time.Duration, time.Duration, bool, error) {
+	return dxgiState.captureVideo(display, "hevc", forceKeyframe)
 }
 
 func (s *duplicationState) reset() {
@@ -685,6 +689,7 @@ func (s *duplicationState) closeLocked() {
 		s.h264LastDesc = d3d11Texture2DDesc{}
 	}
 	resetH264D3D11TextureEncoder()
+	resetNativeHEVCD3D11TextureEncoder()
 	if s.context != nil {
 		s.context.Release()
 		s.context = nil
@@ -875,13 +880,13 @@ func (s *duplicationState) copyLastBaseLocked(src *image.RGBA) {
 	s.lastBase = cloneRGBA(src)
 }
 
-func (s *duplicationState) captureH264(display int, forceKeyframe bool) ([]byte, int, int, time.Duration, time.Duration, bool, error) {
+func (s *duplicationState) captureVideo(display int, codec string, forceKeyframe bool) ([]byte, int, int, time.Duration, time.Duration, bool, error) {
 	capStart := time.Now()
 	s.mu.Lock()
 	defer s.mu.Unlock()
 
 	if !useDesktopDuplication() || cursorCaptureEnabled.Load() {
-		return nil, 0, 0, 0, 0, false, errors.New("direct h264 requires DXGI duplication and cursor capture disabled")
+		return nil, 0, 0, 0, 0, false, fmt.Errorf("direct %s requires DXGI duplication and cursor capture disabled", codec)
 	}
 	if err := s.ensure(display); err != nil {
 		s.lastFail = time.Now()
@@ -891,10 +896,10 @@ func (s *duplicationState) captureH264(display int, forceKeyframe bool) ([]byte,
 	width := int(s.desc.ModeDesc.Width)
 	height := int(s.desc.ModeDesc.Height)
 	if width <= 0 || height <= 0 || width%2 != 0 || height%2 != 0 {
-		return nil, 0, 0, 0, 0, false, fmt.Errorf("direct h264 requires positive even dimensions, got %dx%d", width, height)
+		return nil, 0, 0, 0, 0, false, fmt.Errorf("direct %s requires positive even dimensions, got %dx%d", codec, width, height)
 	}
 	if s.desc.Rotation != dxgiModeRotationIdentity && s.desc.Rotation != dxgiModeRotationUnspecified {
-		return nil, 0, 0, 0, 0, false, fmt.Errorf("direct h264 requires identity rotation, got %d", s.desc.Rotation)
+		return nil, 0, 0, 0, 0, false, fmt.Errorf("direct %s requires identity rotation, got %d", codec, s.desc.Rotation)
 	}
 	encodeW, encodeH := width, height
 	if scale := effectiveScale(width, height); scale != 1 {
@@ -906,7 +911,7 @@ func (s *duplicationState) captureH264(display int, forceKeyframe bool) ([]byte,
 			return nil, 0, 0, 0, 0, false, fmt.Errorf("direct h264 computed invalid scaled output %dx%d", encodeW, encodeH)
 		}
 		directH264ScaleOnce.Do(func() {
-			log.Printf("capture: direct DXGI hardware h264 scaling enabled scale=%.2f input=%dx%d output=%dx%d", scale, width, height, encodeW, encodeH)
+			log.Printf("capture: direct DXGI hardware %s scaling enabled scale=%.2f input=%dx%d output=%dx%d", codec, scale, width, height, encodeW, encodeH)
 		})
 	}
 
@@ -918,7 +923,7 @@ func (s *duplicationState) captureH264(display int, forceKeyframe bool) ([]byte,
 			captureDur := time.Since(capStart)
 			fps := activeH264FPS()
 			encStart := time.Now()
-			out, _, err := encodeH264D3D11Texture(h264D3D11TextureRequest{
+			out, _, err := encodeDesktopD3D11Texture(codec, h264D3D11TextureRequest{
 				Device: unsafe.Pointer(s.device), Texture: unsafe.Pointer(s.h264LastTex),
 				InputWidth: width, InputHeight: height, EncodeWidth: encodeW, EncodeHeight: encodeH,
 				FPS: fps, DXGIFormat: s.h264LastDesc.Format, ForceIDR: forceKeyframe,
@@ -926,7 +931,7 @@ func (s *duplicationState) captureH264(display int, forceKeyframe bool) ([]byte,
 			encodeDur := time.Since(encStart)
 			if err != nil {
 				directH264WarnOnce.Do(func() {
-					log.Printf("capture: direct DXGI hardware h264 cached-frame encode failed for %dx%d@%dfps format=%d: %v; using readback path", width, height, fps, s.h264LastDesc.Format, err)
+					log.Printf("capture: direct DXGI hardware %s cached-frame encode failed for %dx%d@%dfps format=%d: %v; using readback path", codec, width, height, fps, s.h264LastDesc.Format, err)
 				})
 				return nil, 0, 0, 0, 0, false, err
 			}
@@ -978,13 +983,13 @@ func (s *duplicationState) captureH264(display int, forceKeyframe bool) ([]byte,
 	}
 	if srcDesc.Format != dxgiFormatB8G8R8A8UNorm && srcDesc.Format != dxgiFormatR8G8B8A8UNorm {
 		directH264WarnOnce.Do(func() {
-			log.Printf("capture: direct DXGI hardware h264 unavailable: unsupported DXGI texture format %d; using readback path", srcDesc.Format)
+			log.Printf("capture: direct DXGI hardware %s unavailable: unsupported DXGI texture format %d; using readback path", codec, srcDesc.Format)
 		})
-		return nil, 0, 0, 0, 0, false, fmt.Errorf("direct h264 unsupported DXGI texture format %d", srcDesc.Format)
+		return nil, 0, 0, 0, 0, false, fmt.Errorf("direct %s unsupported DXGI texture format %d", codec, srcDesc.Format)
 	}
 	if err := s.cacheH264TextureLocked(tex, srcDesc); err != nil {
 		directH264WarnOnce.Do(func() {
-			log.Printf("capture: direct DXGI hardware h264 cache setup failed for %dx%d format=%d: %v; using readback path", width, height, srcDesc.Format, err)
+			log.Printf("capture: direct DXGI hardware %s cache setup failed for %dx%d format=%d: %v; using readback path", codec, width, height, srcDesc.Format, err)
 		})
 		return nil, 0, 0, 0, 0, false, err
 	}
@@ -995,7 +1000,7 @@ func (s *duplicationState) captureH264(display int, forceKeyframe bool) ([]byte,
 
 	fps := activeH264FPS()
 	encStart := time.Now()
-	out, provider, err := encodeH264D3D11Texture(h264D3D11TextureRequest{
+	out, provider, err := encodeDesktopD3D11Texture(codec, h264D3D11TextureRequest{
 		Device: unsafe.Pointer(s.device), Texture: unsafe.Pointer(s.h264LastTex),
 		InputWidth: width, InputHeight: height, EncodeWidth: encodeW, EncodeHeight: encodeH,
 		FPS: fps, DXGIFormat: s.h264LastDesc.Format, ForceIDR: forceKeyframe,
@@ -1003,7 +1008,7 @@ func (s *duplicationState) captureH264(display int, forceKeyframe bool) ([]byte,
 	encodeDur := time.Since(encStart)
 	if err != nil {
 		directH264WarnOnce.Do(func() {
-			log.Printf("capture: direct DXGI hardware h264 failed for %dx%d@%dfps format=%d: %v; using readback path", width, height, fps, s.h264LastDesc.Format, err)
+			log.Printf("capture: direct DXGI hardware %s failed for %dx%d@%dfps format=%d: %v; using readback path", codec, width, height, fps, s.h264LastDesc.Format, err)
 		})
 		return nil, 0, 0, 0, 0, false, err
 	}
@@ -1011,11 +1016,20 @@ func (s *duplicationState) captureH264(display int, forceKeyframe bool) ([]byte,
 		return nil, encodeW, encodeH, captureDur, encodeDur, true, nil
 	}
 	directH264ActiveOnce.Do(func() {
-		log.Printf("capture: direct DXGI hardware h264 path active provider=%q input=%dx%d output=%dx%d fps=%d dxgi_format=%d", provider, width, height, encodeW, encodeH, fps, srcDesc.Format)
+		log.Printf("capture: direct DXGI hardware %s path active provider=%q input=%dx%d output=%dx%d fps=%d dxgi_format=%d", codec, provider, width, height, encodeW, encodeH, fps, srcDesc.Format)
 	})
 	return out, encodeW, encodeH, captureDur, encodeDur, true, nil
 }
 
+func encodeDesktopD3D11Texture(codec string, req h264D3D11TextureRequest) ([]byte, string, error) {
+	if codec == "hevc" {
+		out, err := encodeNativeHEVCD3D11Texture(req.Device, req.Texture, req.InputWidth, req.InputHeight,
+			req.EncodeWidth, req.EncodeHeight, req.FPS, req.DXGIFormat, req.ForceIDR)
+		return out, "NVIDIA NVENC", err
+	}
+	return encodeH264D3D11Texture(req)
+}
+
 func (s *duplicationState) cacheH264TextureLocked(src *d3d11Texture2D, srcDesc d3d11Texture2DDesc) error {
 	if src == nil || s.device == nil || s.context == nil {
 		return errors.New("nil D3D11 source/device/context")

Overlord-Client/cmd/agent/handlers/command.go

@@ -889,9 +889,16 @@ func HandleCommand(ctx context.Context, env *runtime.Env, envelope map[string]in
 					FPS: profile.FPS, Label: profile.Label, Providers: profile.Providers,
 				})
 			}
+			codecs := make([]wire.DesktopCodecCapability, 0, len(caps.Codecs))
+			for _, codec := range caps.Codecs {
+				codecs = append(codecs, wire.DesktopCodecCapability{
+					Codec: codec.Codec, Encoders: codec.Encoders,
+					Transports: codec.Transports, Hardware: codec.Hardware,
+				})
+			}
 			if err := wire.WriteMsg(ctx, env.Conn, wire.DesktopEncoderCapabilities{
 				Type: "desktop_encoder_capabilities", CommandID: cmdID, Probed: caps.Probed,
-				Display: caps.Display, Profiles: profiles, Detail: caps.Detail,
+				Display: caps.Display, Profiles: profiles, Codecs: codecs, Detail: caps.Detail,
 			}); err != nil {
 				log.Printf("desktop: encoder capability result send failed: %v", err)
 			}

Overlord-Client/cmd/agent/wire/protocol.go

@@ -78,13 +78,21 @@ type DesktopEncoderProfile struct {
 	Providers []string `msgpack:"providers"`
 }
 
+type DesktopCodecCapability struct {
+	Codec      string   `msgpack:"codec"`
+	Encoders   []string `msgpack:"encoders,omitempty"`
+	Transports []string `msgpack:"transports"`
+	Hardware   bool     `msgpack:"hardware,omitempty"`
+}
+
 type DesktopEncoderCapabilities struct {
-	Type      string                  `msgpack:"type"`
+	Type      string                   `msgpack:"type"`
-	CommandID string                  `msgpack:"commandId,omitempty"`
+	CommandID string                   `msgpack:"commandId,omitempty"`
-	Probed    bool                    `msgpack:"probed"`
+	Probed    bool                     `msgpack:"probed"`
-	Display   int                     `msgpack:"display"`
+	Display   int                      `msgpack:"display"`
-	Profiles  []DesktopEncoderProfile `msgpack:"profiles"`
+	Profiles  []DesktopEncoderProfile  `msgpack:"profiles"`
-	Detail    string                  `msgpack:"detail,omitempty"`
+	Codecs    []DesktopCodecCapability `msgpack:"codecs"`
+	Detail    string                   `msgpack:"detail,omitempty"`
 }
 
 type ClientLogEntry struct {

Overlord-Server/public/assets/remotedesktop.js

@@ -122,11 +122,14 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
   let pendingFrame = null;
   let videoDecoder = null;
   let videoDecoderConfigKey = "";
+  const disabledDecoderCodecs = new Set();
   let h264StreamCodec = "";
   let canvasDecodePressure = false;
   let h264PendingTimings = [];
   let h264TimestampUs = 0;
   let prefersH264 = typeof VideoDecoder === "function";
+  let negotiatedCodec = prefersH264 ? "h264" : "jpeg";
+  let browserDecoderCodecs = ["jpeg", "raw"];
   let h264LowFpsStreak = 0;
   let h264FirstFrameAt = 0;
   let h264FramesSeen = 0;
@@ -1449,7 +1452,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
 
   function pushQuality(val) {
     const q = Number(val) || 90;
-    const codec = q >= 100 ? "raw" : (prefersH264 ? "h264" : "jpeg");
+    const codec = q >= 100 ? "raw" : (negotiatedCodec || (prefersH264 ? "h264" : "jpeg"));
     const softwareH264 = codec === "h264" && useSoftwareH264();
     console.debug("rd: pushQuality val=", val, "q=", q, "codec=", codec, "softwareH264=", softwareH264);
     setCodecModeLabel(codec, "requested");
@@ -1475,6 +1478,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
     codecH264.addEventListener("change", function () {
       resetH264SessionState();
       prefersH264 = !!codecH264.checked && typeof VideoDecoder === "function";
+      negotiatedCodec = prefersH264 ? "h264" : "jpeg";
       if (softwareH264Ctrl) {
         softwareH264Ctrl.disabled = !prefersH264;
       }
@@ -1488,6 +1492,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
       if (qualitySlider) {
         pushQuality(qualitySlider.value);
       }
+      requestEncoderCapabilities();
       sharedSettingsSaver.scheduleSave();
     });
   }
@@ -1558,15 +1563,80 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
     });
   }
 
-  function requestEncoderCapabilities() {
+  async function probeBrowserDecoderCodecs(transport) {
+    const supported = ["jpeg", "raw"];
+    if (transport === "webrtc") {
+      try {
+        const codecs = typeof RTCRtpReceiver !== "undefined" && typeof RTCRtpReceiver.getCapabilities === "function"
+          ? (RTCRtpReceiver.getCapabilities("video")?.codecs || [])
+          : [];
+        for (const capability of codecs) {
+          const mime = String(capability?.mimeType || "").toLowerCase();
+          if (mime === "video/h264") supported.push("h264");
+          if (mime === "video/h265" || mime === "video/hevc") supported.push("hevc");
+        }
+      } catch (err) {
+        console.debug("rd: WebRTC codec capability probe failed", err);
+      }
+      return [...new Set(supported)];
+    }
+
+    if (typeof VideoDecoder !== "function") return supported;
+    if (typeof VideoDecoder.isConfigSupported !== "function") {
+      supported.push("h264");
+      return supported;
+    }
+    const probes = [
+      ["h264", "avc1.42E01E"],
+      ["hevc", "hev1.1.6.L156.B0"],
+    ];
+    await Promise.all(probes.map(async ([name, codec]) => {
+      try {
+        const result = await VideoDecoder.isConfigSupported({ codec, optimizeForLatency: true });
+        if (result?.supported) supported.push(name);
+      } catch (err) {
+        console.debug(`rd: ${name} decoder capability probe failed`, err);
+      }
+    }));
+    return [...new Set(supported)].filter((codec) => !disabledDecoderCodecs.has(codec));
+  }
+
+  async function requestEncoderCapabilities() {
     if (streamProfileDetail) streamProfileDetail.textContent = "Checking hardware encoder profiles…";
+    const mode = getWebrtcMode();
+    const transport = mode === "off" ? "websocket" : "webrtc";
+    const decoderCodecs = await probeBrowserDecoderCodecs(transport);
+    browserDecoderCodecs = decoderCodecs;
+    if (!ws || ws.readyState !== WebSocket.OPEN) return;
     sendCmd("desktop_encoder_capabilities", {
       display: Number.parseInt(displaySelect?.value || "0", 10) || 0,
+      decoderCodecs,
+      preferredCodecs: prefersH264 ? ["hevc", "h264", "jpeg", "raw"] : ["jpeg", "raw"],
+      transport,
     });
   }
 
   function applyEncoderCapabilities(msg) {
-    if (!msg || !Array.isArray(msg.profiles) || !streamProfileSelect) return;
+    if (!msg) return;
+    const selectedCodec = String(msg.selectedCodec || msg.negotiation?.selectedCodec || "").toLowerCase();
+    if (selectedCodec) {
+      negotiatedCodec = selectedCodec;
+      const fallbacks = Array.isArray(msg.fallbackCodecs) ? msg.fallbackCodecs.join(" → ") : selectedCodec;
+      setCodecModeLabel(selectedCodec, `negotiated; fallback ${fallbacks}`);
+      if (codecH264) {
+        const agentHasH264 = Array.isArray(msg.codecs) && msg.codecs.some((entry) => String(entry?.codec || "").toLowerCase() === "h264");
+        codecH264.disabled = !browserDecoderCodecs.includes("h264") || !agentHasH264;
+      }
+      if (desiredStreaming && qualitySlider) pushQuality(qualitySlider.value);
+    } else if (msg.transport === "webrtc") {
+      console.warn("rd: no mutually supported WebRTC video codec; falling back to WebSocket canvas");
+      if (webrtcMode) webrtcMode.value = "off";
+      negotiatedCodec = browserDecoderCodecs.includes("h264") ? "h264" : "jpeg";
+      setCodecModeLabel(negotiatedCodec, "WebRTC unavailable; WebSocket fallback");
+      stopAllWebrtc();
+      requestEncoderCapabilities();
+    }
+    if (!Array.isArray(msg.profiles) || !streamProfileSelect) return;
     const selectedDisplay = Number.parseInt(displaySelect?.value || "0", 10) || 0;
     if (Number.isFinite(Number(msg.display)) && Number(msg.display) !== selectedDisplay) return;
     const previous = streamProfileSelect.value;
@@ -1637,6 +1707,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
   if (webrtcMode) {
     webrtcMode.addEventListener("change", function () {
       updateControls();
+      requestEncoderCapabilities();
       sharedSettingsSaver.scheduleSave();
     });
   }
@@ -1910,11 +1981,12 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
 
   function destroyVideoDecoder() {
     updateCanvasDecodePressure(true);
-    if (!videoDecoder) return;
+    if (videoDecoder) {
-    try {
+      try {
-      videoDecoder.close();
+        videoDecoder.close();
-    } catch {
+      } catch {
-      // Ignore close errors when decoder is already shutting down.
+        // Ignore close errors when decoder is already shutting down.
+      }
     }
     videoDecoder = null;
     videoDecoderConfigKey = "";
@@ -1945,6 +2017,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
     if (!prefersH264) return;
     const reasonText = normalizeFallbackReason(reason);
     prefersH264 = false;
+    negotiatedCodec = "jpeg";
     destroyVideoDecoder();
     if (codecH264) codecH264.checked = false;
     if (softwareH264Ctrl) softwareH264Ctrl.disabled = true;
@@ -2037,6 +2110,26 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
     return false;
   }
 
+  function isHEVCKeyFrame(data) {
+    for (let i = 0; i + 5 < data.length; i++) {
+      let startCodeLen = 0;
+      if (data[i] === 0x00 && data[i + 1] === 0x00 && data[i + 2] === 0x01) {
+        startCodeLen = 3;
+      } else if (
+        data[i] === 0x00 && data[i + 1] === 0x00 && data[i + 2] === 0x00 && data[i + 3] === 0x01
+      ) {
+        startCodeLen = 4;
+      }
+      if (!startCodeLen) continue;
+      const nalIndex = i + startCodeLen;
+      if (nalIndex >= data.length) break;
+      const nalType = (data[nalIndex] >> 1) & 0x3f;
+      if (nalType >= 16 && nalType <= 21) return true;
+      i = nalIndex;
+    }
+    return false;
+  }
+
   function h264CodecFromAnnexB(data) {
     for (let i = 0; i + 7 < data.length; i++) {
       let startCodeLength = 0;
@@ -2052,9 +2145,11 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
     return "";
   }
 
-  function ensureVideoDecoder(h264Bytes, width, height) {
+  function ensureVideoDecoder(videoBytes, width, height, codecName) {
-    const detectedCodec = h264CodecFromAnnexB(h264Bytes);
+    const detectedCodec = codecName === "h264" ? h264CodecFromAnnexB(videoBytes) : "";
-    const codec = detectedCodec || h264StreamCodec || "avc1.4d0034";
+    const codec = codecName === "hevc"
+      ? "hev1.1.6.L156.B0"
+      : (detectedCodec || h264StreamCodec || "avc1.4d0034");
     const codedWidth = Math.max(0, Number(width) || 0);
     const codedHeight = Math.max(0, Number(height) || 0);
     const configKey = `${codec}:${codedWidth}x${codedHeight}`;
@@ -2095,8 +2190,9 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
           updateCanvasDecodePressure();
         },
         error: (err) => {
-          console.warn("rd: h264 decoder error", err);
+          console.warn(`rd: ${codecName} decoder error`, err);
           updateCanvasDecodePressure(true);
+          setTimeout(() => fallbackFromVideoCodec(codecName, err), 0);
         },
       });
       videoDecoder.addEventListener("dequeue", () => {
@@ -2110,11 +2206,11 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
       }
       videoDecoder.configure(config);
       videoDecoderConfigKey = configKey;
-      h264StreamCodec = codec;
+      if (codecName === "h264") h264StreamCodec = codec;
       return true;
     } catch (err) {
-      console.warn("rd: h264 decoder unavailable", err);
+      console.warn(`rd: ${codecName} decoder unavailable`, err);
-      fallbackToJpegCodec(err);
+      fallbackFromVideoCodec(codecName, err);
       destroyVideoDecoder();
       return false;
     }
@@ -2132,6 +2228,32 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
     }
   }
 
+  function fallbackFromVideoCodec(codec, reason) {
+    if (codec === "hevc" && disabledDecoderCodecs.has("hevc")) return;
+    const reasonText = normalizeFallbackReason(reason);
+    if (codec === "hevc" && browserDecoderCodecs.includes("h264")) {
+      disabledDecoderCodecs.add("hevc");
+      browserDecoderCodecs = browserDecoderCodecs.filter((name) => name !== "hevc");
+      negotiatedCodec = "h264";
+      destroyVideoDecoder();
+      console.warn("rd: falling back from hevc to h264", reasonText);
+      setCodecModeLabel("h264", "HEVC fallback");
+      if (ws.readyState === WebSocket.OPEN) {
+        const q = Number(qualitySlider?.value) || 90;
+        sendCmd("desktop_set_quality", {
+          quality: q,
+          codec: "h264",
+          softwareH264: useSoftwareH264(),
+          source: "viewer_fallback",
+          reason: reasonText,
+        });
+        requestEncoderCapabilities();
+      }
+      return;
+    }
+    fallbackToJpegCodec(reasonText);
+  }
+
   function recordCanvasFrameTiming(receivedAt, decodeStartedAt) {
     const renderedAt = performance.now();
     diagnostics.decodeMs = smoothed(diagnostics.decodeMs, Math.max(0, renderedAt - decodeStartedAt));
@@ -2240,16 +2362,16 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
       return;
     }
 
-    if (format === 4) {
+    if (format === 4 || format === 5) {
-      setCodecModeLabel("h264", "active");
+      const codecName = format === 5 ? "hevc" : "h264";
-      diagnostics.codec = "h264";
+      setCodecModeLabel(codecName, "active");
-      const h264Bytes = buf.slice(headerLength);
+      diagnostics.codec = codecName;
-      if (!h264Bytes.length) {
+      const videoBytes = buf.slice(headerLength);
+      if (!videoBytes.length) {
         updateCanvasDecodePressure(true);
         return;
       }
-      if (!ensureVideoDecoder(h264Bytes, encodedWidth || frameWidth, encodedHeight || frameHeight)) {
+      if (!ensureVideoDecoder(videoBytes, encodedWidth || frameWidth, encodedHeight || frameHeight, codecName)) {
-        fallbackToJpegCodec("WebCodecs decoder unavailable");
         updateCanvasDecodePressure(true);
         return;
       }
@@ -2259,17 +2381,18 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
       }
       h264FramesSeen += 1;
 
-      const isKey = isH264KeyFrame(h264Bytes);
+      const isKey = codecName === "hevc" ? isHEVCKeyFrame(videoBytes) : isH264KeyFrame(videoBytes);
 
       // If software H264 encode on the agent cannot keep up, automatically
       // fall back to JPEG blocks for a smoother interactive stream.
       const h264ElapsedMs = performance.now() - h264FirstFrameAt;
-      if ((fps || 0) <= H264_LOW_FPS_THRESHOLD) {
+      if (codecName === "h264" && (fps || 0) <= H264_LOW_FPS_THRESHOLD) {
         h264LowFpsStreak += 1;
       } else {
         h264LowFpsStreak = 0;
       }
       if (
+        codecName === "h264" &&
         h264ElapsedMs >= H264_FALLBACK_WARMUP_MS &&
         h264FramesSeen >= H264_MIN_FRAMES_BEFORE_FALLBACK &&
         h264LowFpsStreak >= H264_LOW_FPS_STREAK_LIMIT
@@ -2283,7 +2406,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
       const chunk = new EncodedVideoChunk({
         type: isKey ? "key" : "delta",
         timestamp: h264TimestampUs,
-        data: h264Bytes,
+        data: videoBytes,
       });
       h264TimestampUs += frameIntervalUs;
       try {
@@ -2294,7 +2417,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
         h264KeyframeErrorStreak = 0;
       } catch (err) {
         h264PendingTimings.pop();
-        if (isKeyframeRequiredError(err)) {
+        if (codecName === "h264" && isKeyframeRequiredError(err)) {
           h264KeyframeErrorStreak += 1;
           const now = Date.now();
           if (now - h264LastDecodeWarnAt >= H264_DECODE_WARN_THROTTLE_MS) {
@@ -2313,8 +2436,8 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
           updateCanvasDecodePressure(true);
           return;
         }
-        console.warn("rd: h264 decode failed", err);
+        console.warn(`rd: ${codecName} decode failed`, err);
-        fallbackToJpegCodec(err);
+        fallbackFromVideoCodec(codecName, err);
         updateCanvasDecodePressure(true);
       }
       return;

Overlord-Server/public/build.html

@@ -865,7 +865,7 @@
                       <input type="checkbox" name="enable-nvenc" class="w-4 h-4" checked />
                       <div class="flex flex-col">
                         <span><i class="fa-solid fa-microchip text-green-400 mr-2"></i>Include NVIDIA NVENC</span>
-                        <span class="text-xs text-slate-500">Native NVIDIA H.264 acceleration for Windows CGO builds</span>
+                        <span class="text-xs text-slate-500">Native NVIDIA H.264 and HEVC acceleration for Windows CGO builds</span>
                       </div>
                     </label>
 

Overlord-Server/public/remotedesktop.html

@@ -204,8 +204,8 @@
                   </select>
                 </div>
                 <div class="flex items-center justify-between gap-2">
-                  <label class="flex items-center gap-2 text-sm text-slate-200" title="Prefer H.264 when available; disable to force JPEG">
+                  <label class="flex items-center gap-2 text-sm text-slate-200" title="Prefer HEVC or H.264 when available; disable to force JPEG">
-                    <input type="checkbox" id="codecH264" checked /> H.264
+                    <input type="checkbox" id="codecH264" checked /> Efficient video (HEVC/H.264)
                   </label>
                   <span id="codecMode" class="pill pill-ghost text-slate-100 text-xs">Codec: auto</span>
                 </div>

Overlord-Server/src/paths.test.ts

@@ -0,0 +1,33 @@
+import { expect, test } from "bun:test";
+import { rm } from "node:fs/promises";
+import { isAbsolute, resolve } from "node:path";
+import { tmpdir } from "node:os";
+
+test("creates isolated test data when the Bun preload was not discovered", async () => {
+  const childEnv = { ...process.env };
+  delete childEnv.DATA_DIR;
+  delete childEnv.OVERLORD_TEST_DATA_DIR;
+  childEnv.NODE_ENV = "test";
+
+  const child = Bun.spawn([
+    "bun",
+    "-e",
+    'const { resolveDataDir } = await import("./src/paths.ts"); console.log(resolveDataDir());',
+  ], {
+    cwd: resolve(import.meta.dirname, ".."),
+    env: childEnv,
+    stdout: "pipe",
+    stderr: "pipe",
+  });
+  const [exitCode, stdout, stderr] = await Promise.all([
+    child.exited,
+    new Response(child.stdout).text(),
+    new Response(child.stderr).text(),
+  ]);
+
+  expect(exitCode, stderr).toBe(0);
+  const dataDir = stdout.trim();
+  expect(isAbsolute(dataDir)).toBe(true);
+  expect(dataDir.toLowerCase().startsWith(resolve(tmpdir()).toLowerCase())).toBe(true);
+  await rm(dataDir, { recursive: true, force: true });
+});

Overlord-Server/src/paths.ts

@@ -1,7 +1,19 @@
 import path from "path";
-import { existsSync, mkdirSync } from "fs";
+import { existsSync, mkdirSync, mkdtempSync, writeFileSync } from "fs";
 import { tmpdir } from "os";
 
+let fallbackTestDataDir = "";
+
+function createFallbackTestDataDir(): string {
+  if (fallbackTestDataDir) return fallbackTestDataDir;
+  const dir = mkdtempSync(path.join(tmpdir(), "overlord-bun-test-"));
+  writeFileSync(path.join(dir, "config.json"), "{}\n", "utf8");
+  process.env.DATA_DIR = dir;
+  process.env.OVERLORD_TEST_DATA_DIR = "1";
+  fallbackTestDataDir = dir;
+  return dir;
+}
+
 function assertSafeTestDataDir(dir: string): void {
   if (String(process.env.NODE_ENV || "").toLowerCase() !== "test") return;
 
@@ -32,9 +44,7 @@ export function resolveDataDir(): string {
   }
 
   if (String(process.env.NODE_ENV || "").toLowerCase() === "test") {
-    throw new Error(
+    return createFallbackTestDataDir();
-      "Refusing to use the default application data directory while NODE_ENV=test. Run tests through `bun test`.",
-    );
   }
 
   if (process.platform === "win32" && process.env.APPDATA) {

Overlord-Server/src/protocol.test.ts

@@ -61,4 +61,16 @@ describe("protocol encode/decode", () => {
     expect(decoded.header.fps).toBe(24);
     expect(Array.from(decoded.data)).toEqual(Array.from(frame.data));
   });
+
+  test("round trips HEVC frame metadata", () => {
+    const frame: Frame = {
+      type: "frame",
+      header: { monitor: 0, fps: 60, format: "hevc" },
+      data: new Uint8Array([0, 0, 0, 1, 0x40, 0x01]),
+    };
+
+    const decoded = decodeMessage(encodeMessage(frame)) as Frame;
+    expect(decoded.header.format).toBe("hevc");
+    expect(Array.from(decoded.data)).toEqual(Array.from(frame.data));
+  });
 });

Overlord-Server/src/protocol.ts

@@ -158,6 +158,30 @@ export type CommandResult = {
   message?: string;
 };
 
+export type DesktopCodecCapability = {
+  codec: "hevc" | "h264" | "jpeg" | "raw" | string;
+  encoders?: string[];
+  transports: Array<"websocket" | "webrtc" | string>;
+  hardware?: boolean;
+};
+
+export type DesktopEncoderCapabilities = {
+  type: "desktop_encoder_capabilities";
+  commandId?: string;
+  probed: boolean;
+  display: number;
+  profiles: Array<{
+    maxHeight: number;
+    width: number;
+    height: number;
+    fps: number;
+    label: string;
+    providers: string[];
+  }>;
+  codecs: DesktopCodecCapability[];
+  detail?: string;
+};
+
 export type ClientLogEntry = {
   seq: number;
   at: number;
@@ -190,7 +214,7 @@ export type ScreenshotResult = {
 export type FrameHeader = {
   monitor: number;
   fps: number;
-  format: "jpeg" | "webp" | "raw" | "h264";
+  format: "jpeg" | "webp" | "raw" | "h264" | "hevc";
   width?: number;
   height?: number;
   hash?: string;

Overlord-Server/src/server/desktop-codec-negotiation.test.ts

@@ -0,0 +1,68 @@
+import { describe, expect, test } from "bun:test";
+import { negotiateDesktopCodec } from "./desktop-codec-negotiation";
+
+const encoders = [
+  { codec: "hevc", transports: ["websocket"] },
+  { codec: "h264", transports: ["websocket", "webrtc"] },
+  { codec: "jpeg", transports: ["websocket"] },
+];
+
+describe("desktop codec negotiation", () => {
+  test("selects the first mutually supported preferred codec", () => {
+    expect(negotiateDesktopCodec({
+      encoderCodecs: encoders,
+      decoderCodecs: ["hevc", "h264", "jpeg"],
+      preferredCodecs: ["hevc", "h264", "jpeg"],
+      transport: "websocket",
+    })).toEqual({
+      selectedCodec: "hevc",
+      fallbackCodecs: ["hevc", "h264", "jpeg"],
+      transport: "websocket",
+    });
+  });
+
+  test("filters codecs that are unavailable on WebRTC", () => {
+    expect(negotiateDesktopCodec({
+      encoderCodecs: encoders,
+      decoderCodecs: ["hevc", "h264", "jpeg"],
+      preferredCodecs: ["hevc", "h264", "jpeg"],
+      transport: "p2p",
+    })).toEqual({
+      selectedCodec: "h264",
+      fallbackCodecs: ["h264"],
+      transport: "webrtc",
+    });
+  });
+
+  test("falls back to JPEG when H.264 decoding is unavailable", () => {
+    const result = negotiateDesktopCodec({
+      encoderCodecs: encoders,
+      decoderCodecs: ["jpeg"],
+      preferredCodecs: ["h264", "jpeg"],
+      transport: "websocket",
+    });
+    expect(result.selectedCodec).toBe("jpeg");
+    expect(result.fallbackCodecs).toEqual(["jpeg"]);
+  });
+
+  test("returns no codec instead of selecting an incompatible transport", () => {
+    const result = negotiateDesktopCodec({
+      encoderCodecs: encoders,
+      decoderCodecs: ["jpeg"],
+      preferredCodecs: ["jpeg"],
+      transport: "webrtc",
+    });
+    expect(result.selectedCodec).toBe("");
+    expect(result.fallbackCodecs).toEqual([]);
+  });
+
+  test("does not re-add a codec the viewer explicitly omitted", () => {
+    const result = negotiateDesktopCodec({
+      encoderCodecs: encoders,
+      decoderCodecs: ["h264", "jpeg"],
+      preferredCodecs: ["jpeg"],
+      transport: "websocket",
+    });
+    expect(result.fallbackCodecs).toEqual(["jpeg"]);
+  });
+});

Overlord-Server/src/server/desktop-codec-negotiation.ts

@@ -0,0 +1,88 @@
+export type DesktopCodecTransport = "websocket" | "webrtc";
+
+export type DesktopCodecCapability = {
+  codec: string;
+  encoders?: string[];
+  transports?: string[];
+  hardware?: boolean;
+};
+
+export type DesktopCodecNegotiation = {
+  selectedCodec: string;
+  fallbackCodecs: string[];
+  transport: DesktopCodecTransport;
+};
+
+const KNOWN_CODECS = new Set(["hevc", "h264", "jpeg", "raw"]);
+const DEFAULT_PREFERENCE = ["h264", "jpeg", "raw"];
+
+function normalizeCodec(value: unknown): string {
+  const codec = String(value || "").trim().toLowerCase();
+  if (codec === "h265") return "hevc";
+  if (codec === "mjpeg" || codec === "jpg") return "jpeg";
+  return KNOWN_CODECS.has(codec) ? codec : "";
+}
+
+function normalizeTransport(value: unknown): DesktopCodecTransport {
+  const transport = String(value || "").trim().toLowerCase();
+  return transport === "webrtc" || transport === "relayed" || transport === "p2p"
+    ? "webrtc"
+    : "websocket";
+}
+
+function normalizeDeclaredTransport(value: unknown): DesktopCodecTransport | "" {
+  const transport = String(value || "").trim().toLowerCase();
+  if (transport === "websocket") return "websocket";
+  if (transport === "webrtc" || transport === "relayed" || transport === "p2p") return "webrtc";
+  return "";
+}
+
+function decoderCodecSet(values: unknown): Set<string> {
+  if (!Array.isArray(values)) return new Set();
+  return new Set(values.map(normalizeCodec).filter(Boolean));
+}
+
+function encoderCodecSet(
+  values: unknown,
+  transport: DesktopCodecTransport,
+): Set<string> {
+  if (!Array.isArray(values)) return new Set();
+  const codecs = new Set<string>();
+  for (const value of values) {
+    if (typeof value === "string") {
+      const codec = normalizeCodec(value);
+      if (codec) codecs.add(codec);
+      continue;
+    }
+    if (!value || typeof value !== "object") continue;
+    const capability = value as DesktopCodecCapability;
+    const codec = normalizeCodec(capability.codec);
+    if (!codec) continue;
+    const transports = Array.isArray(capability.transports)
+      ? capability.transports.map(normalizeDeclaredTransport).filter(Boolean)
+      : ["websocket" as const];
+    if (transports.includes(transport)) codecs.add(codec);
+  }
+  return codecs;
+}
+
+export function negotiateDesktopCodec(options: {
+  encoderCodecs: unknown;
+  decoderCodecs: unknown;
+  preferredCodecs?: unknown;
+  transport?: unknown;
+}): DesktopCodecNegotiation {
+  const transport = normalizeTransport(options.transport);
+  const encoders = encoderCodecSet(options.encoderCodecs, transport);
+  const decoders = decoderCodecSet(options.decoderCodecs);
+  const requested = Array.isArray(options.preferredCodecs)
+    ? options.preferredCodecs.map(normalizeCodec).filter(Boolean)
+    : [];
+  const preference = [...new Set(requested.length > 0 ? requested : DEFAULT_PREFERENCE)];
+  const fallbackCodecs = preference.filter((codec) => encoders.has(codec) && decoders.has(codec));
+  return {
+    selectedCodec: fallbackCodecs[0] || "",
+    fallbackCodecs,
+    transport,
+  };
+}

Overlord-Server/src/server/routes/webrtc-routes.auth.test.ts

@@ -7,7 +7,11 @@ import {
   setUserClientAccessScope,
   setUserFeaturePermission,
 } from "../../users";
-import { handleWebrtcRoutes } from "./webrtc-routes";
+import {
+  handleWebrtcRoutes,
+  revokeWebrtcViewerSessions,
+  trackWebrtcViewerSession,
+} from "./webrtc-routes";
 
 const PASSWORD = "Aa1!WebrtcAuthTestPass_2026";
 const createdUserIds: number[] = [];
@@ -57,4 +61,30 @@ describe("WebRTC viewer authorization", () => {
     );
     expect(response!.status).toBe(403);
   });
+
+  test("revokes only the matching user's tracked media sessions", async () => {
+    const deleted: string[] = [];
+    trackWebrtcViewerSession({
+      upstreamUrl: "http://mediamtx/session-a",
+      userId: 1001,
+      clientId: "client-a",
+      kind: "desktop",
+      createdAt: Date.now(),
+    });
+    trackWebrtcViewerSession({
+      upstreamUrl: "http://mediamtx/session-b",
+      userId: 1002,
+      clientId: "client-a",
+      kind: "desktop",
+      createdAt: Date.now(),
+    });
+
+    const count = await revokeWebrtcViewerSessions(1001, "client-a", async (url) => {
+      deleted.push(url);
+    });
+    expect(count).toBe(1);
+    expect(deleted).toEqual(["http://mediamtx/session-a"]);
+
+    await revokeWebrtcViewerSessions(1002, "client-a", async () => {});
+  });
 });

Overlord-Server/src/server/routes/webrtc-routes.ts

@@ -12,6 +12,50 @@ const PUBLISH_TOKEN_TTL_MS = 60_000;
 type PublishToken = { clientId: string; expiresAt: number };
 const publishTokens = new Map<string, PublishToken>();
 
+export type ViewerMediaSession = {
+  upstreamUrl: string;
+  userId: number;
+  clientId: string;
+  kind: WebrtcKind;
+  createdAt: number;
+};
+const viewerMediaSessions = new Map<string, ViewerMediaSession>();
+
+export function trackWebrtcViewerSession(session: ViewerMediaSession): void {
+  const cutoff = Date.now() - 24 * 60 * 60 * 1000;
+  for (const [upstreamUrl, existing] of viewerMediaSessions) {
+    if (existing.createdAt < cutoff) viewerMediaSessions.delete(upstreamUrl);
+  }
+  viewerMediaSessions.set(session.upstreamUrl, session);
+}
+
+export function forgetWebrtcViewerSession(upstreamUrl: string): void {
+  viewerMediaSessions.delete(upstreamUrl);
+}
+
+export async function revokeWebrtcViewerSessions(
+  userId: number,
+  clientId: string,
+  deleteSession: (upstreamUrl: string) => Promise<unknown> = async (upstreamUrl) => {
+    await fetch(upstreamUrl, { method: "DELETE" });
+  },
+): Promise<number> {
+  const revoked: ViewerMediaSession[] = [];
+  for (const [upstreamUrl, session] of viewerMediaSessions) {
+    if (session.userId !== userId || session.clientId !== clientId) continue;
+    viewerMediaSessions.delete(upstreamUrl);
+    revoked.push(session);
+  }
+  await Promise.allSettled(revoked.map(async (session) => {
+    try {
+      await deleteSession(session.upstreamUrl);
+    } catch (error) {
+      logger.warn(`[webrtc] failed to revoke viewer session client=${clientId}: ${(error as Error).message}`);
+    }
+  }));
+  return revoked.length;
+}
+
 function prunePublishTokens(now = Date.now()) {
   for (const [t, entry] of publishTokens) {
     if (entry.expiresAt < now) publishTokens.delete(t);
@@ -66,6 +110,7 @@ export async function handleWebrtcRoutes(req: Request, url: URL): Promise<Respon
 
   const [, streamPath, kind, sessionSuffix] = match;
   const clientId = streamPath.split("/")[1];
+  let viewerUserId: number | undefined;
 
   if (kind === "whip") {
     const authHeader = req.headers.get("authorization") || "";
@@ -87,6 +132,7 @@ export async function handleWebrtcRoutes(req: Request, url: URL): Promise<Respon
       requireClientAccess(user, clientId);
       const streamKind = streamPath.split("/")[2] as WebrtcKind;
       requireFeatureAccess(user, WEBRTC_FEATURES[streamKind]);
+      viewerUserId = user.userId;
     } catch (error) {
       if (error instanceof Response) return error;
       return new Response("Forbidden", { status: 403 });
@@ -123,9 +169,21 @@ export async function handleWebrtcRoutes(req: Request, url: URL): Promise<Respon
   if (loc) {
     try {
       const abs = new URL(loc, MEDIAMTX_URL);
+      if (kind === "whep" && req.method === "POST" && upstreamResp.ok && viewerUserId !== undefined) {
+        trackWebrtcViewerSession({
+          upstreamUrl: abs.toString(),
+          userId: viewerUserId,
+          clientId,
+          kind: streamPath.split("/")[2] as WebrtcKind,
+          createdAt: Date.now(),
+        });
+      }
       respHeaders.set("location", `/api/webrtc${abs.pathname}`);
     } catch { }
   }
+  if (kind === "whep" && req.method === "DELETE" && upstreamResp.ok) {
+    forgetWebrtcViewerSession(upstreamUrl);
+  }
 
   return new Response(upstreamResp.body, {
     status: upstreamResp.status,

Overlord-Server/src/server/viewer-authorization.ts

@@ -8,6 +8,7 @@ import {
   getUserById,
   type FeatureName,
 } from "../users";
+import { revokeWebrtcViewerSessions } from "./routes/webrtc-routes";
 
 const VIEWER_FEATURES: Partial<Record<SocketRole, FeatureName>> = {
   console_viewer: "console",
@@ -31,6 +32,9 @@ const activeViewerSockets = new Set<ServerWebSocket<SocketData>>();
 
 function deny(ws: ServerWebSocket<SocketData>, reason: string): false {
   activeViewerSockets.delete(ws);
+  if (ws.data.userId !== undefined && ws.data.clientId) {
+    void revokeWebrtcViewerSessions(ws.data.userId, ws.data.clientId);
+  }
   try {
     ws.close(1008, reason);
   } catch {}

Overlord-Server/src/server/ws-console-rd-backstage.test.ts

@@ -10,6 +10,7 @@ import {
   handleRemoteDesktopViewerMessage,
   handleRemoteDesktopViewerOpen,
   handleDesktopStreamStats,
+  handleDesktopEncoderCapabilities,
   backstageStreamingState,
   rdStreamingState,
 } from "./ws-console-rd-backstage";
@@ -217,6 +218,39 @@ describe("remote desktop viewer control", () => {
     expect(message.transport).toBe("webrtc");
   });
 
+  test("selects one mutually compatible codec across all viewer transports", () => {
+    const clientId = `rd-codecs-${Date.now().toString(36)}`;
+    createClient(clientId);
+    const canvasViewer = createMockWs({ clientId });
+    const webrtcViewer = createMockWs({ clientId });
+    handleRemoteDesktopViewerOpen(canvasViewer as any);
+    handleRemoteDesktopViewerOpen(webrtcViewer as any);
+
+    canvasViewer.data.rdDecoderCodecs = ["hevc", "h264", "jpeg"];
+    canvasViewer.data.rdPreferredCodecs = ["hevc", "h264", "jpeg"];
+    canvasViewer.data.rdCodecTransport = "websocket";
+    webrtcViewer.data.rdDecoderCodecs = ["hevc", "h264"];
+    webrtcViewer.data.rdPreferredCodecs = ["hevc", "h264"];
+    webrtcViewer.data.rdCodecTransport = "webrtc";
+
+    handleDesktopEncoderCapabilities(clientId, {
+      type: "desktop_encoder_capabilities",
+      profiles: [],
+      codecs: [
+        { codec: "hevc", transports: ["websocket"] },
+        { codec: "h264", transports: ["websocket", "webrtc"] },
+        { codec: "jpeg", transports: ["websocket"] },
+      ],
+    });
+
+    const canvasMessage = decodeMessage(canvasViewer.sent.at(-1) as Uint8Array) as any;
+    const webrtcMessage = decodeMessage(webrtcViewer.sent.at(-1) as Uint8Array) as any;
+    expect(canvasMessage.selectedCodec).toBe("h264");
+    expect(canvasMessage.fallbackCodecs).toEqual(["h264"]);
+    expect(webrtcMessage.selectedCodec).toBe("h264");
+    expect(webrtcMessage.fallbackCodecs).toEqual(["h264"]);
+  });
+
   test("holds Canvas acknowledgements only while the browser decoder is under pressure", () => {
     const clientId = `rd-flow-${Date.now().toString(36)}`;
     const { agentWs } = createClient(clientId);

Overlord-Server/src/server/ws-console-rd-backstage.ts

@@ -4,6 +4,7 @@ import { existsSync, readFileSync } from "fs";
 import path from "path";
 import * as clientManager from "../clientManager";
 import { logger } from "../logger";
+import { negotiateDesktopCodec } from "./desktop-codec-negotiation";
 import { metrics } from "../metrics";
 import { encodeMessage } from "../protocol";
 import { resolveRuntimeRoot } from "./runtime-paths";
@@ -377,6 +378,15 @@ export function handleRemoteDesktopViewerMessage(ws: ServerWebSocket<SocketData>
   }
   switch (payload.type) {
     case "desktop_encoder_capabilities":
+      ws.data.rdDecoderCodecs = Array.isArray((payload as any).decoderCodecs)
+        ? (payload as any).decoderCodecs.map((codec: unknown) => String(codec || "").trim().toLowerCase()).filter(Boolean).slice(0, 8)
+        : ["h264", "jpeg", "raw"];
+      ws.data.rdPreferredCodecs = Array.isArray((payload as any).preferredCodecs)
+        ? (payload as any).preferredCodecs.map((codec: unknown) => String(codec || "").trim().toLowerCase()).filter(Boolean).slice(0, 8)
+        : ["h264", "jpeg", "raw"];
+      ws.data.rdCodecTransport = ["webrtc", "relayed", "p2p"].includes(String((payload as any).transport || "").toLowerCase())
+        ? "webrtc"
+        : "websocket";
       sendDesktopCommand(target, "desktop_encoder_capabilities", {
         display: Number((payload as any).display) || 0,
       });
@@ -488,6 +498,14 @@ export function handleRemoteDesktopViewerMessage(ws: ServerWebSocket<SocketData>
         });
         break;
       }
+      if (String(state.codec || "").toLowerCase() === "hevc") {
+        safeSendViewer(ws, {
+          type: "recording_status",
+          recording: null,
+          error: "HEVC stream recording is not supported yet. Switch the stream codec to H.264 or JPEG first.",
+        });
+        break;
+      }
       const inputCodec = String(state.codec || "").toLowerCase() === "h264" ? "h264" : "mjpeg";
       const compact = !!(payload as any).compact;
       const requestedFps = Number((payload as any).fps) || 0;
@@ -1495,8 +1513,43 @@ export function sendbackstageCommand(target: ClientInfo | undefined, commandType
 }
 
 export function handleDesktopEncoderCapabilities(clientId: string, payload: any) {
-  for (const session of sessionManager.getRdSessionsForClient(clientId)) {
+  const encoderCodecs = Array.isArray(payload?.codecs) && payload.codecs.length > 0
-    safeSendViewer(session.viewer, payload);
+    ? payload.codecs
+    : [
+        { codec: "h264", transports: ["websocket", "webrtc"] },
+        { codec: "jpeg", transports: ["websocket"] },
+        { codec: "raw", transports: ["websocket"] },
+      ];
+  const sessions = sessionManager.getRdSessionsForClient(clientId);
+  const negotiations = sessions.map((session) => negotiateDesktopCodec({
+      encoderCodecs,
+      decoderCodecs: session.viewer.data.rdDecoderCodecs || ["h264", "jpeg", "raw"],
+      preferredCodecs: session.viewer.data.rdPreferredCodecs,
+      transport: session.viewer.data.rdCodecTransport,
+    }));
+  const sharedFallbackCodecs = negotiations.length > 0
+    ? negotiations[0].fallbackCodecs.filter((codec) =>
+        negotiations.every((negotiation) => negotiation.fallbackCodecs.includes(codec)),
+      )
+    : [];
+  const selectedCodec = sharedFallbackCodecs[0] || "";
+
+  for (let index = 0; index < sessions.length; index++) {
+    const session = sessions[index];
+    const negotiation = {
+      ...negotiations[index],
+      selectedCodec,
+      fallbackCodecs: sharedFallbackCodecs,
+    };
+    session.viewer.data.rdSelectedCodec = selectedCodec;
+    safeSendViewer(session.viewer, {
+      ...payload,
+      codecs: encoderCodecs,
+      negotiation,
+      selectedCodec,
+      fallbackCodecs: sharedFallbackCodecs,
+      transport: negotiation.transport,
+    });
   }
 }
 

Overlord-Server/src/server/ws-viewer-utils.test.ts

@@ -0,0 +1,21 @@
+import { describe, expect, test } from "bun:test";
+import { buildViewerFrameBuffer } from "./ws-viewer-utils";
+
+describe("viewer frame protocol", () => {
+  test("encodes HEVC as compact frame format 5", () => {
+    const payload = new Uint8Array([0, 0, 0, 1, 0x40]);
+    const frame = buildViewerFrameBuffer(payload, {
+      format: "hevc",
+      monitor: 2,
+      fps: 60,
+      width: 1920,
+      height: 1080,
+    });
+
+    expect(frame[3]).toBe(2);
+    expect(frame[4]).toBe(2);
+    expect(frame[5]).toBe(60);
+    expect(frame[6]).toBe(5);
+    expect(frame.slice(12)).toEqual(payload);
+  });
+});

Overlord-Server/src/server/ws-viewer-utils.ts

@@ -49,6 +49,8 @@ export function buildViewerFrameBuffer(bytes: Uint8Array, header?: any): Uint8Ar
     ? 3
     : header?.format === "h264"
     ? 4
+    : header?.format === "hevc"
+    ? 5
     : 1;
   meta[6] = fmt;
   meta[7] = 0;

Overlord-Server/src/sessions/types.ts

@@ -31,6 +31,10 @@ export type SocketData = {
   enrollmentNonce?: string;
   disconnectReason?: string;
   disconnectDetail?: string;
+  rdDecoderCodecs?: string[];
+  rdPreferredCodecs?: string[];
+  rdCodecTransport?: "websocket" | "webrtc";
+  rdSelectedCodec?: string;
 };
 
 export type ConsoleSession = {

docker-compose.windows.yml

@@ -118,7 +118,7 @@ services:
       - no-new-privileges:true
 
   mediamtx:
-    image: bluenviron/mediamtx:latest
+    image: bluenviron/mediamtx:v1.18.2
     container_name: overlord-mediamtx
     restart: unless-stopped
     command: ["/run/overlord-turn/mediamtx.yml"]

docker-compose.yml

@@ -120,7 +120,7 @@ services:
       - no-new-privileges:true
 
   mediamtx:
-    image: bluenviron/mediamtx:latest
+    image: bluenviron/mediamtx:v1.18.2
     container_name: overlord-mediamtx
     # Share the host network so WebRTC ICE candidates use the host's real
     # interfaces directly — no extra config needed for local or LAN browsers.