major remote desktop upgrades + do opus. Hopefully all the tests pass 😭

Overlord-Client/cmd/agent/audio/voice_native_cgo.go

@@ -18,6 +18,8 @@ import (
 const (
 	voiceSampleRate         = 16000
 	voiceChannels           = 1
+	desktopAudioSampleRate  = 48000
+	desktopAudioChannels    = 2
 	voiceBytesPerSample     = 2
 	maxPlaybackBufferMillis = 250
 )
@@ -390,15 +392,15 @@ func StartCaptureOnlySession(parent context.Context, source string, onCapture fu
 	}
 
 	capCfg := malgo.DefaultDeviceConfig(malgo.Capture)
-	capCfg.SampleRate = voiceSampleRate
+	capCfg.SampleRate = desktopAudioSampleRate
 	capCfg.Capture.Format = malgo.FormatS16
-	capCfg.Capture.Channels = voiceChannels
+	capCfg.Capture.Channels = desktopAudioChannels
 	useLoopback := false
 	if strings.EqualFold(source, "system") {
 		capCfg = malgo.DefaultDeviceConfig(malgo.Loopback)
-		capCfg.SampleRate = voiceSampleRate
+		capCfg.SampleRate = desktopAudioSampleRate
 		capCfg.Capture.Format = malgo.FormatS16
-		capCfg.Capture.Channels = voiceChannels
+		capCfg.Capture.Channels = desktopAudioChannels
 		useLoopback = true
 	} else if strings.HasPrefix(strings.ToLower(source), "device:") {
 		deviceName := strings.TrimSpace(source[len("device:"):])
@@ -442,9 +444,9 @@ func StartCaptureOnlySession(parent context.Context, source string, onCapture fu
 	capDev, err := malgo.InitDevice(audioCtx.Context, capCfg, capCallbacks)
 	if err != nil && useLoopback {
 		capCfg = malgo.DefaultDeviceConfig(malgo.Capture)
-		capCfg.SampleRate = voiceSampleRate
+		capCfg.SampleRate = desktopAudioSampleRate
 		capCfg.Capture.Format = malgo.FormatS16
-		capCfg.Capture.Channels = voiceChannels
+		capCfg.Capture.Channels = desktopAudioChannels
 		deviceID, _, ok := findSystemCaptureDevice(audioCtx.Context)
 		if !ok || deviceID == nil {
 			audioCtx.Free()

Overlord-Client/cmd/agent/capture/capture.go

@@ -154,6 +154,8 @@ func CaptureAndSend(ctx context.Context, env *rt.Env) error {
 		slotAcquired = true
 	}
 
+	frameWidth := img.Bounds().Dx()
+	frameHeight := img.Bounds().Dy()
 	quality := jpegQuality()
 	frame, encodeDur, err := buildFrame(img, display, quality)
 	PutRGBA(img)
@@ -164,6 +166,8 @@ func CaptureAndSend(ctx context.Context, env *rt.Env) error {
 		}
 		return err
 	}
+	frame.Header.Width = frameWidth
+	frame.Header.Height = frameHeight
 	now := time.Now()
 	fps := frameFPS(now)
 	if fps <= 0 {
@@ -181,17 +185,21 @@ func CaptureAndSend(ctx context.Context, env *rt.Env) error {
 		if dur <= 0 {
 			dur = 33 * time.Millisecond
 		}
+		sendStart := time.Now()
 		if werr := webrtcpub.WriteH264(webrtcpub.KindDesktop, frame.Data, dur); werr != nil {
 			log.Printf("webrtc: write h264 failed: %v", werr)
 		}
+		sendDur := time.Since(sendStart)
 		if slotAcquired {
 			ReleaseFrameSlot()
 		}
 		statFrames.Add(1)
 		statCapNs.Add(captureDur.Nanoseconds())
 		statEncNs.Add(encodeDur.Nanoseconds())
-		statTotalNs.Add(time.Since(t0).Nanoseconds())
+		totalDur := time.Since(t0)
+		statTotalNs.Add(totalDur.Nanoseconds())
 		statBytes.Add(int64(len(frame.Data)))
+		emitDesktopStreamStats(ctx, env, frame, fps, captureDur, encodeDur, sendDur, totalDur, "webrtc")
 		return nil
 	}
 	if !slotAcquired {
@@ -233,6 +241,7 @@ func CaptureAndSend(ctx context.Context, env *rt.Env) error {
 	statSendNs.Add(sendDur.Nanoseconds())
 	statTotalNs.Add(time.Since(t0).Nanoseconds())
 	statBytes.Add(int64(len(frame.Data)))
+	emitDesktopStreamStats(ctx, env, frame, fps, captureDur, encodeDur, sendDur, time.Since(t0), "websocket")
 	return err
 }
 
@@ -251,14 +260,18 @@ func sendCompletedFrame(ctx context.Context, env *rt.Env, frame wire.Frame, disp
 		if dur <= 0 {
 			dur = 33 * time.Millisecond
 		}
+		sendStart := time.Now()
 		if werr := webrtcpub.WriteH264(webrtcpub.KindDesktop, frame.Data, dur); werr != nil {
 			log.Printf("webrtc: write h264 failed: %v", werr)
 		}
+		sendDur := time.Since(sendStart)
 		statFrames.Add(1)
 		statCapNs.Add(captureDur.Nanoseconds())
 		statEncNs.Add(encodeDur.Nanoseconds())
-		statTotalNs.Add(time.Since(t0).Nanoseconds())
+		totalDur := time.Since(t0)
+		statTotalNs.Add(totalDur.Nanoseconds())
 		statBytes.Add(int64(len(frame.Data)))
+		emitDesktopStreamStats(ctx, env, frame, fps, captureDur, encodeDur, sendDur, totalDur, "webrtc")
 		return nil
 	}
 	if !AcquireFrameSlot() {
@@ -301,6 +314,7 @@ func sendCompletedFrame(ctx context.Context, env *rt.Env, frame wire.Frame, disp
 	statSendNs.Add(sendDur.Nanoseconds())
 	statTotalNs.Add(time.Since(t0).Nanoseconds())
 	statBytes.Add(int64(len(frame.Data)))
+	emitDesktopStreamStats(ctx, env, frame, fps, captureDur, encodeDur, sendDur, time.Since(t0), "websocket")
 	return nil
 }
 

Overlord-Client/cmd/agent/capture/capture_test.go

@@ -44,6 +44,7 @@ func TestCaptureAndSend_NoDisplays(t *testing.T) {
 }
 
 func TestCaptureAndSend_SendsFrame(t *testing.T) {
+	previousStatsAt := lastDesktopStreamStatsMs.Swap(0)
 	originalCount := activeDisplays
 	originalCapture := captureDisplayFn
 	activeDisplays = func() int { return 1 }
@@ -56,6 +57,7 @@ func TestCaptureAndSend_SendsFrame(t *testing.T) {
 	t.Cleanup(func() {
 		activeDisplays = originalCount
 		captureDisplayFn = originalCapture
+		lastDesktopStreamStatsMs.Store(previousStatsAt)
 	})
 
 	writer := &recordingWriter{}
@@ -63,8 +65,8 @@ func TestCaptureAndSend_SendsFrame(t *testing.T) {
 	if err := CaptureAndSend(context.Background(), env); err != nil {
 		t.Fatalf("CaptureAndSend returned error: %v", err)
 	}
-	if len(writer.msgs) != 1 {
+	if len(writer.msgs) != 2 {
-		t.Fatalf("expected one message written, got %d", len(writer.msgs))
+		t.Fatalf("expected frame and telemetry messages, got %d", len(writer.msgs))
 	}
 
 	var frame wire.Frame
@@ -84,6 +86,17 @@ func TestCaptureAndSend_SendsFrame(t *testing.T) {
 	if _, err := jpeg.Decode(bytes.NewReader(frame.Data)); err != nil {
 		t.Fatalf("jpeg payload did not decode: %v", err)
 	}
+
+	var stats wire.DesktopStreamStats
+	if err := msgpack.Unmarshal(writer.msgs[1], &stats); err != nil {
+		t.Fatalf("decode desktop stream stats: %v", err)
+	}
+	if stats.Type != "desktop_stream_stats" || stats.Format != "jpeg" {
+		t.Fatalf("unexpected stream stats: %#v", stats)
+	}
+	if stats.CaptureMs < 0 || stats.EncodeMs < 0 || stats.TotalMs < 0 {
+		t.Fatalf("stream stats contained a negative duration: %#v", stats)
+	}
 }
 
 func TestActualScreenCapture(t *testing.T) {
@@ -306,8 +319,13 @@ func TestCaptureAndSend_DisplayOutOfRange(t *testing.T) {
 				t.Errorf("expected capture from display %d, got %d", tt.expectedDisplay, capturedDisplay)
 			}
 
-			if len(writer.msgs) != 1 {
+			if len(writer.msgs) == 0 {
-				t.Errorf("expected 1 message, got %d", len(writer.msgs))
+				t.Error("expected a frame message")
+			} else {
+				var frame wire.Frame
+				if err := msgpack.Unmarshal(writer.msgs[0], &frame); err != nil || frame.Type != "frame" {
+					t.Errorf("first message was not a valid frame: type=%q err=%v", frame.Type, err)
+				}
 			}
 		})
 	}

Overlord-Client/cmd/agent/capture/h264_bitrate.go

@@ -0,0 +1,66 @@
+package capture
+
+import (
+	"math"
+	"sync/atomic"
+)
+
+const maxH264Bitrate = 50_000_000
+
+var h264TargetBitrate atomic.Int64
+
+func SetH264TargetBitrate(bps int) int {
+	if bps < 0 {
+		bps = 0
+	}
+	if bps > maxH264Bitrate {
+		bps = maxH264Bitrate
+	}
+	previous := h264TargetBitrate.Swap(int64(bps))
+	if previous != int64(bps) {
+		resetH264Encoder()
+		resetH264Encoderbackstage()
+		resetH264TextureEncoderForBitrate()
+		RequestDesktopFullFrame()
+	}
+	return bps
+}
+
+func configuredH264Bitrate() int {
+	return int(h264TargetBitrate.Load())
+}
+
+func automaticH264Bitrate(width, height, fps int) int {
+	pixelsPerSecond := float64(width * height * fps)
+	bitrate := int(pixelsPerSecond * 0.08)
+	if bitrate < 1_500_000 {
+		return 1_500_000
+	}
+	if bitrate > 18_000_000 {
+		return 18_000_000
+	}
+	return bitrate
+}
+
+func targetH264Bitrate(width, height, fps int) int {
+	if configured := configuredH264Bitrate(); configured > 0 {
+		return configured
+	}
+	return automaticH264Bitrate(width, height, fps)
+}
+
+func targetH264CRF(width, height, fps int) float32 {
+	configured := configuredH264Bitrate()
+	if configured <= 0 {
+		return 23
+	}
+	ratio := float64(configured) / float64(automaticH264Bitrate(width, height, fps))
+	crf := 23 - 6*math.Log2(ratio)
+	if crf < 12 {
+		crf = 12
+	}
+	if crf > 35 {
+		crf = 35
+	}
+	return float32(crf)
+}

Overlord-Client/cmd/agent/capture/h264_bitrate_reset_other.go

@@ -0,0 +1,5 @@
+//go:build !windows
+
+package capture
+
+func resetH264TextureEncoderForBitrate() {}

Overlord-Client/cmd/agent/capture/h264_bitrate_reset_windows.go

@@ -0,0 +1,7 @@
+//go:build windows
+
+package capture
+
+func resetH264TextureEncoderForBitrate() {
+	resetH264D3D11TextureEncoder()
+}

Overlord-Client/cmd/agent/capture/h264_bitrate_test.go

@@ -0,0 +1,38 @@
+package capture
+
+import "testing"
+
+func TestH264TargetBitrateAutoAndManual(t *testing.T) {
+	original := configuredH264Bitrate()
+	defer SetH264TargetBitrate(original)
+
+	SetH264TargetBitrate(0)
+	if got := targetH264Bitrate(1920, 1080, 60); got != 9_953_280 {
+		t.Fatalf("automatic 1080p60 bitrate = %d, want 9953280", got)
+	}
+	if got := targetH264Bitrate(3840, 2160, 240); got != 18_000_000 {
+		t.Fatalf("automatic bitrate cap = %d, want 18000000", got)
+	}
+
+	if got := SetH264TargetBitrate(30_000_000); got != 30_000_000 {
+		t.Fatalf("manual bitrate = %d, want 30000000", got)
+	}
+	if got := targetH264Bitrate(1280, 720, 30); got != 30_000_000 {
+		t.Fatalf("manual target = %d, want 30000000", got)
+	}
+	if got := SetH264TargetBitrate(60_000_000); got != maxH264Bitrate {
+		t.Fatalf("manual cap = %d, want %d", got, maxH264Bitrate)
+	}
+}
+
+func TestH264TargetCRFTracksManualRate(t *testing.T) {
+	original := configuredH264Bitrate()
+	defer SetH264TargetBitrate(original)
+	SetH264TargetBitrate(0)
+	auto := targetH264CRF(1920, 1080, 60)
+	SetH264TargetBitrate(20_000_000)
+	higherRate := targetH264CRF(1920, 1080, 60)
+	if higherRate >= auto {
+		t.Fatalf("higher bitrate CRF = %.2f, want below automatic %.2f", higherRate, auto)
+	}
+}

Overlord-Client/cmd/agent/capture/h264_direct_windows.go

@@ -14,7 +14,7 @@ func tryBuildDirectH264Frame(display int) (wire.Frame, time.Duration, time.Durat
 		return wire.Frame{}, 0, 0, false, nil
 	}
 	forceKeyframe := webrtcpub.ConsumeKeyframeRequest()
-	data, _, _, captureDur, encodeDur, used, err := captureDisplayDXGIH264(display, forceKeyframe)
+	data, width, height, captureDur, encodeDur, used, err := captureDisplayDXGIH264(display, forceKeyframe)
 	if !used || err != nil {
 		return wire.Frame{}, captureDur, encodeDur, used, err
 	}
@@ -24,5 +24,5 @@ func tryBuildDirectH264Frame(display int) (wire.Frame, time.Duration, time.Durat
 	now := time.Now()
 	lastKeyframe.Store(now.UnixNano())
 	statFullFrames.Add(1)
-	return wire.Frame{Type: "frame", Header: wire.FrameHeader{Monitor: display, FPS: 0, Format: "h264"}, Data: data}, captureDur, encodeDur, true, nil
+	return wire.Frame{Type: "frame", Header: wire.FrameHeader{Monitor: display, FPS: 0, Format: "h264", Width: width, Height: height}, Data: data}, captureDur, encodeDur, true, nil
 }

Overlord-Client/cmd/agent/capture/h264_encoder_windows.go

@@ -1296,12 +1296,6 @@ func failedHR(hr uintptr) bool {
 	return hr&0x80000000 != 0
 }
 
-func targetH264Bitrate(width, height, fps int) int {
-	pixelsPerSecond := float64(width * height * fps)
-	bitrate := int(pixelsPerSecond * 0.08)
-	return clampInt(bitrate, 1_500_000, 18_000_000)
-}
-
 func h264LevelFor(width, height, fps int) int {
 	mbsPerFrame := ((width + 15) / 16) * ((height + 15) / 16)
 	mbsPerSecond := mbsPerFrame * fps

Overlord-Client/cmd/agent/capture/h264_encoder_x264.go

@@ -104,7 +104,7 @@ func ensureH264EncoderLocked(width, height int) error {
 		Preset:       "veryfast",
 		Profile:      "main",
 		RateControl:  "crf",
-		RateConstant: 23,
+		RateConstant: targetH264CRF(width, height, fps),
 		LogLevel:     x264.LogError,
 	}
 
@@ -201,7 +201,7 @@ func ensurebackstageH264EncoderLocked(width, height int) error {
 		Preset:       "veryfast",
 		Profile:      "main",
 		RateControl:  "crf",
-		RateConstant: 23,
+		RateConstant: targetH264CRF(width, height, fps),
 		LogLevel:     x264.LogError,
 	}
 	enc, err := x264.NewEncoder(&backstageH264Buf, opts)
@@ -280,7 +280,7 @@ func ensureWebcamH264EncoderLocked(width, height int) error {
 		Preset:       "veryfast",
 		Profile:      "main",
 		RateControl:  "crf",
-		RateConstant: 23,
+		RateConstant: targetH264CRF(width, height, fps),
 		LogLevel:     x264.LogError,
 	}
 	enc, err := x264.NewEncoder(&webcamH264Buf, opts)

Overlord-Client/cmd/agent/capture/stream_stats.go

@@ -0,0 +1,37 @@
+package capture
+
+import (
+	"context"
+	"sync/atomic"
+	"time"
+
+	rt "overlord-client/cmd/agent/runtime"
+	"overlord-client/cmd/agent/wire"
+)
+
+var lastDesktopStreamStatsMs atomic.Int64
+
+func emitDesktopStreamStats(ctx context.Context, env *rt.Env, frame wire.Frame, fps int, captureDur, encodeDur, sendDur, totalDur time.Duration, transport string) {
+	now := time.Now().UnixMilli()
+	previous := lastDesktopStreamStatsMs.Load()
+	if now-previous < 500 || !lastDesktopStreamStatsMs.CompareAndSwap(previous, now) {
+		return
+	}
+	_ = wire.WriteMsg(ctx, env.Conn, wire.DesktopStreamStats{
+		Type:      "desktop_stream_stats",
+		FPS:       fps,
+		Format:    frame.Header.Format,
+		Bytes:     len(frame.Data),
+		Width:     frame.Header.Width,
+		Height:    frame.Header.Height,
+		CaptureMs: durationMs(captureDur),
+		EncodeMs:  durationMs(encodeDur),
+		SendMs:    durationMs(sendDur),
+		TotalMs:   durationMs(totalDur),
+		Transport: transport,
+	})
+}
+
+func durationMs(value time.Duration) float64 {
+	return float64(value.Microseconds()) / 1000
+}

Overlord-Client/cmd/agent/handlers/command.go

@@ -503,6 +503,7 @@ func startDesktopAudioSession(ctx context.Context, env *runtime.Env, sessionID s
 	stopDesktopAudioSession()
 
 	vCtx, cancel := context.WithCancel(ctx)
+	legacyConverter := &desktopAudioLegacyConverter{}
 	session, err := audio.StartCaptureOnlySession(vCtx, source, func(chunk []byte) {
 		if len(chunk) == 0 {
 			return
@@ -513,10 +514,14 @@ func startDesktopAudioSession(ctx context.Context, env *runtime.Env, sessionID s
 			samples := pcm16BytesToInt16(chunk)
 			_ = webrtcpub.WriteAudio(webrtcpub.KindAudio, samples)
 		}
+		legacyPCM := legacyConverter.Convert(chunk)
+		if len(legacyPCM) == 0 {
+			return
+		}
 		msg := map[string]interface{}{
 			"type":      "desktop_audio_uplink",
 			"sessionId": sessionID,
-			"data":      chunk,
+			"data":      legacyPCM,
 		}
 		_ = wire.WriteMsg(vCtx, env.Conn, msg)
 	})
@@ -904,6 +909,24 @@ func HandleCommand(ctx context.Context, env *runtime.Env, envelope map[string]in
 		log.Printf("desktop: set target fps=%d", fps)
 		sendCommandResultSafe(env, cmdID, true, "")
 		return nil
+	case "desktop_set_bitrate":
+		payload, _ := envelope["payload"].(map[string]interface{})
+		bitrateMbps := 0
+		if payload != nil {
+			if v, ok := payloadInt(payload, "bitrateMbps"); ok {
+				bitrateMbps = v
+			}
+		}
+		if bitrateMbps < 0 {
+			bitrateMbps = 0
+		}
+		if bitrateMbps > 50 {
+			bitrateMbps = 50
+		}
+		bps := capture.SetH264TargetBitrate(bitrateMbps * 1_000_000)
+		log.Printf("desktop: set target bitrate=%d Mbps (0=auto)", bps/1_000_000)
+		sendCommandResultSafe(env, cmdID, true, "")
+		return nil
 	case "clipboard_sync_start":
 		payload, _ := envelope["payload"].(map[string]interface{})
 		source := "rd"

Overlord-Client/cmd/agent/handlers/webrtc.go

@@ -23,6 +23,43 @@ func pcm16BytesToInt16(chunk []byte) []int16 {
 	return out
 }
 
+// WebRTC desktop audio is captured at 48 kHz stereo. Keep the established raw
+// WebSocket fallback wire format at 16 kHz mono by averaging each stereo frame
+// and decimating groups of three frames. pending preserves sample groups when
+// an audio callback ends between frames or decimation windows.
+type desktopAudioLegacyConverter struct {
+	pending []byte
+}
+
+func (c *desktopAudioLegacyConverter) Convert(chunk []byte) []byte {
+	if len(chunk) == 0 {
+		return nil
+	}
+	data := make([]byte, 0, len(c.pending)+len(chunk))
+	data = append(data, c.pending...)
+	data = append(data, chunk...)
+	const windowBytes = 3 * 2 * 2 // three stereo PCM16 frames
+	windows := len(data) / windowBytes
+	if windows == 0 {
+		c.pending = append(c.pending[:0], data...)
+		return nil
+	}
+	out := make([]byte, 0, windows*2)
+	for window := 0; window < windows; window++ {
+		var sum int64
+		for frame := 0; frame < 3; frame++ {
+			offset := window*windowBytes + frame*4
+			left := int32(int16(binary.LittleEndian.Uint16(data[offset:])))
+			right := int32(int16(binary.LittleEndian.Uint16(data[offset+2:])))
+			sum += int64(left+right) / 2
+		}
+		sample := int16(sum / 3)
+		out = binary.LittleEndian.AppendUint16(out, uint16(sample))
+	}
+	c.pending = append(c.pending[:0], data[windows*windowBytes:]...)
+	return out
+}
+
 func kindFromPayload(payload map[string]interface{}) webrtcpub.Kind {
 	switch s, _ := payload["kind"].(string); s {
 	case "backstage":

Overlord-Client/cmd/agent/handlers/webrtc_audio_test.go

@@ -0,0 +1,27 @@
+package handlers
+
+import (
+	"encoding/binary"
+	"testing"
+)
+
+func TestDesktopAudioLegacyPCMDownmixesAndDecimates(t *testing.T) {
+	// Three 48 kHz stereo frames become one 16 kHz mono sample. Stereo frame
+	// means are 2000, 4000, and 6000; their anti-aliasing mean is 4000.
+	input := make([]byte, 0, 12)
+	for _, pair := range [][2]int16{{1000, 3000}, {3000, 5000}, {5000, 7000}} {
+		input = binary.LittleEndian.AppendUint16(input, uint16(pair[0]))
+		input = binary.LittleEndian.AppendUint16(input, uint16(pair[1]))
+	}
+	converter := &desktopAudioLegacyConverter{}
+	if out := converter.Convert(input[:5]); len(out) != 0 {
+		t.Fatalf("fragment produced %d bytes before a complete window", len(out))
+	}
+	out := converter.Convert(input[5:])
+	if len(out) != 2 {
+		t.Fatalf("legacy PCM length = %d, want 2", len(out))
+	}
+	if got := int16(binary.LittleEndian.Uint16(out)); got != 4000 {
+		t.Fatalf("legacy PCM sample = %d, want 4000", got)
+	}
+}

Overlord-Client/cmd/agent/webrtcpub/audio_opus.go

@@ -0,0 +1,79 @@
+//go:build overlord_webrtc
+
+package webrtcpub
+
+import (
+	"sync"
+	"time"
+
+	"github.com/pion/webrtc/v4"
+	"github.com/pion/webrtc/v4/pkg/media"
+	"github.com/thesyncim/gopus"
+)
+
+const (
+	opusInputRate       = 48000
+	opusChannels        = 2
+	opusPacketMs        = 20
+	opusSamplesPerFrame = opusInputRate * opusPacketMs / 1000
+	opusBitrate         = 96000
+)
+
+type opusAudioWriter struct {
+	t       *webrtc.TrackLocalStaticSample
+	encoder *gopus.Encoder
+	mu      sync.Mutex
+	pcm     []int16
+	packet  []byte
+}
+
+func newOpusAudioWriter(t *webrtc.TrackLocalStaticSample) (*opusAudioWriter, error) {
+	encoder, err := gopus.NewEncoder(gopus.EncoderConfig{
+		SampleRate:  opusInputRate,
+		Channels:    opusChannels,
+		Application: gopus.ApplicationAudio,
+	})
+	if err != nil {
+		return nil, err
+	}
+	if err := encoder.SetBitrate(opusBitrate); err != nil {
+		return nil, err
+	}
+	if err := encoder.SetInBandFEC(gopus.InBandFECEnabled); err != nil {
+		return nil, err
+	}
+	return &opusAudioWriter{
+		t:       t,
+		encoder: encoder,
+		pcm:     make([]int16, 0, opusSamplesPerFrame*4),
+		packet:  make([]byte, 4000),
+	}, nil
+}
+
+func (w *opusAudioWriter) WriteAudio(pcm []int16) error {
+	if len(pcm) == 0 {
+		return nil
+	}
+	w.mu.Lock()
+	defer w.mu.Unlock()
+	w.pcm = append(w.pcm, pcm...)
+	frameValues := opusSamplesPerFrame * opusChannels
+	for len(w.pcm) >= frameValues {
+		n, err := w.encoder.EncodeInt16(w.pcm[:frameValues], w.packet)
+		if err != nil {
+			w.pcm = w.pcm[:0]
+			return err
+		}
+		if n > 0 {
+			if err := w.t.WriteSample(media.Sample{
+				Data:     w.packet[:n],
+				Duration: opusPacketMs * time.Millisecond,
+			}); err != nil {
+				w.pcm = w.pcm[:0]
+				return err
+			}
+		}
+		w.pcm = w.pcm[frameValues:]
+	}
+	return nil
+}

Overlord-Client/cmd/agent/webrtcpub/audio_opus_test.go

@@ -0,0 +1,33 @@
+//go:build overlord_webrtc
+
+package webrtcpub
+
+import (
+	"testing"
+
+	"github.com/pion/webrtc/v4"
+)
+
+func TestOpusAudioWriterAcceptsFragmentedStereoPCM(t *testing.T) {
+	track, err := webrtc.NewTrackLocalStaticSample(
+		webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeOpus, ClockRate: 48000, Channels: 2},
+		"audio", "test",
+	)
+	if err != nil {
+		t.Fatal(err)
+	}
+	writer, err := newOpusAudioWriter(track)
+	if err != nil {
+		t.Fatal(err)
+	}
+	frame := make([]int16, opusSamplesPerFrame*opusChannels)
+	if err := writer.WriteAudio(frame[:333]); err != nil {
+		t.Fatal(err)
+	}
+	if err := writer.WriteAudio(frame[333:]); err != nil {
+		t.Fatal(err)
+	}
+	if len(writer.pcm) != 0 {
+		t.Fatalf("buffered PCM = %d, want 0 after a complete frame", len(writer.pcm))
+	}
+}

Overlord-Client/cmd/agent/webrtcpub/p2p_pion.go

@@ -71,11 +71,12 @@ func StartP2POffer(ctx context.Context, kind Kind, sessionID string, offerSDP st
 	if hasAudio {
 		if err := mediaEngine.RegisterCodec(webrtc.RTPCodecParameters{
 			RTPCodecCapability: webrtc.RTPCodecCapability{
-				MimeType:  webrtc.MimeTypePCMU,
+				MimeType:    webrtc.MimeTypeOpus,
-				ClockRate: 8000,
+				ClockRate:   48000,
-				Channels:  1,
+				Channels:    2,
+				SDPFmtpLine: "minptime=10;useinbandfec=1",
 			},
-			PayloadType: 0,
+			PayloadType: 111,
 		}, webrtc.RTPCodecTypeAudio); err != nil {
 			return "", fmt.Errorf("register audio codec: %w", err)
 		}
@@ -94,6 +95,7 @@ func StartP2POffer(ctx context.Context, kind Kind, sessionID string, offerSDP st
 	var (
 		videoTrack  *webrtc.TrackLocalStaticSample
 		audioTrack  *webrtc.TrackLocalStaticSample
+		audioWriter AudioWriter
 	)
 	if hasVideo {
 		videoTrack, err = webrtc.NewTrackLocalStaticSample(
@@ -117,13 +119,18 @@ func StartP2POffer(ctx context.Context, kind Kind, sessionID string, offerSDP st
 	}
 	if hasAudio {
 		audioTrack, err = webrtc.NewTrackLocalStaticSample(
-			webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypePCMU, ClockRate: 8000, Channels: 1},
+			webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeOpus, ClockRate: 48000, Channels: 2, SDPFmtpLine: "minptime=10;useinbandfec=1"},
 			"overlord-audio-p2p-"+string(kind), "overlord-"+string(kind),
 		)
 		if err != nil {
 			_ = pc.Close()
 			return "", fmt.Errorf("new audio track: %w", err)
 		}
+		audioWriter, err = newOpusAudioWriter(audioTrack)
+		if err != nil {
+			_ = pc.Close()
+			return "", fmt.Errorf("new opus encoder: %w", err)
+		}
 		tx, err := pc.AddTransceiverFromTrack(audioTrack, webrtc.RTPTransceiverInit{
 			Direction: webrtc.RTPTransceiverDirectionSendonly,
 		})
@@ -201,7 +208,7 @@ func StartP2POffer(ctx context.Context, kind Kind, sessionID string, offerSDP st
 		registerVideoWriter(kind, writerID, &h264TrackWriter{t: videoTrack})
 	}
 	if audioTrack != nil {
-		registerAudioWriter(kind, writerID, newPCMUAudioWriter(audioTrack))
+		registerAudioWriter(kind, writerID, audioWriter)
 	}
 	return answer.SDP, nil
 }

Overlord-Client/cmd/agent/webrtcpub/whip_pion.go

@@ -133,11 +133,12 @@ func Start(ctx context.Context, kind Kind, opts Options) (*Publisher, error) {
 	if opts.HasAudio {
 		if err := mediaEngine.RegisterCodec(webrtc.RTPCodecParameters{
 			RTPCodecCapability: webrtc.RTPCodecCapability{
-				MimeType:  webrtc.MimeTypePCMU,
+				MimeType:    webrtc.MimeTypeOpus,
-				ClockRate: 8000,
+				ClockRate:   48000,
-				Channels:  1,
+				Channels:    2,
+				SDPFmtpLine: "minptime=10;useinbandfec=1",
 			},
-			PayloadType: 0,
+			PayloadType: 111,
 		}, webrtc.RTPCodecTypeAudio); err != nil {
 			return nil, fmt.Errorf("register audio codec: %w", err)
 		}
@@ -152,6 +153,7 @@ func Start(ctx context.Context, kind Kind, opts Options) (*Publisher, error) {
 	var (
 		videoTrack  *webrtc.TrackLocalStaticSample
 		audioTrack  *webrtc.TrackLocalStaticSample
+		audioWriter AudioWriter
 	)
 	if opts.HasVideo {
 		videoTrack, err = webrtc.NewTrackLocalStaticSample(
@@ -175,13 +177,18 @@ func Start(ctx context.Context, kind Kind, opts Options) (*Publisher, error) {
 	}
 	if opts.HasAudio {
 		audioTrack, err = webrtc.NewTrackLocalStaticSample(
-			webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypePCMU, ClockRate: 8000, Channels: 1},
+			webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeOpus, ClockRate: 48000, Channels: 2, SDPFmtpLine: "minptime=10;useinbandfec=1"},
 			"overlord-audio-"+string(kind), "overlord-"+string(kind),
 		)
 		if err != nil {
 			_ = pc.Close()
 			return nil, fmt.Errorf("new audio track: %w", err)
 		}
+		audioWriter, err = newOpusAudioWriter(audioTrack)
+		if err != nil {
+			_ = pc.Close()
+			return nil, fmt.Errorf("new opus encoder: %w", err)
+		}
 		tx, err := pc.AddTransceiverFromTrack(audioTrack, webrtc.RTPTransceiverInit{
 			Direction: webrtc.RTPTransceiverDirectionSendonly,
 		})
@@ -256,7 +263,7 @@ func Start(ctx context.Context, kind Kind, opts Options) (*Publisher, error) {
 		registerVideoWriter(kind, writerID, &h264TrackWriter{t: videoTrack})
 	}
 	if audioTrack != nil {
-		registerAudioWriter(kind, writerID, newPCMUAudioWriter(audioTrack))
+		registerAudioWriter(kind, writerID, audioWriter)
 	}
 	log.Printf("webrtcpub: WHIP[%s] session established (resource=%s)", kind, resourceURL)
 	return pub, nil

Overlord-Client/cmd/agent/wire/protocol.go

@@ -110,6 +110,8 @@ type FrameHeader struct {
 	Monitor   int    `msgpack:"monitor"`
 	FPS       int    `msgpack:"fps"`
 	Format    string `msgpack:"format"`
+	Width     int    `msgpack:"width,omitempty"`
+	Height    int    `msgpack:"height,omitempty"`
 	Backstage bool   `msgpack:"backstage,omitempty"`
 	Webcam    bool   `msgpack:"webcam,omitempty"`
 }
@@ -120,6 +122,20 @@ type Frame struct {
 	Data   []byte      `msgpack:"data"`
 }
 
+type DesktopStreamStats struct {
+	Type      string  `msgpack:"type"`
+	FPS       int     `msgpack:"fps"`
+	Format    string  `msgpack:"format"`
+	Bytes     int     `msgpack:"bytes"`
+	Width     int     `msgpack:"width"`
+	Height    int     `msgpack:"height"`
+	CaptureMs float64 `msgpack:"captureMs"`
+	EncodeMs  float64 `msgpack:"encodeMs"`
+	SendMs    float64 `msgpack:"sendMs"`
+	TotalMs   float64 `msgpack:"totalMs"`
+	Transport string  `msgpack:"transport"`
+}
+
 type FrameAck struct {
 	Type string `msgpack:"type"`
 }

Overlord-Client/go.mod

@@ -20,6 +20,7 @@ require (
 	github.com/pion/rtcp v1.2.15
 	github.com/pion/webrtc/v4 v4.0.7
 	github.com/tetratelabs/wazero v1.12.0
+	github.com/thesyncim/gopus v0.1.1
 	golang.org/x/crypto v0.49.0
 	golang.org/x/sys v0.44.0
 )

Overlord-Client/go.sum

@@ -79,6 +79,8 @@ github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOf
 github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/tetratelabs/wazero v1.12.0 h1:DuWcpNu/FzgEXgGBDp8J1Spc+CWOvvtvVyjKlaZopYU=
 github.com/tetratelabs/wazero v1.12.0/go.mod h1:LvKtzl2RqO4gyF27BiXU+nKAjcV8f38U+kP/q2vgxh0=
+github.com/thesyncim/gopus v0.1.1 h1:SM58S7VZGnEZ3ZR0pX5AoC33//LBxJhKkvm15MbBmik=
+github.com/thesyncim/gopus v0.1.1/go.mod h1:orRqwrGs5gqYRRnhqwI0Y3liqQTeDkreUpra+Kv9bQc=
 github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IUPn0Bjt8=
 github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok=
 github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g=

Overlord-Server/public/assets/backstage.js

@@ -69,9 +69,29 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
     }, reconnectDelay);
   }
   const latencyEl = document.getElementById("latencyDisplay");
+  const networkStats = document.getElementById("networkStats");
   let lastInputSentAt = 0;
   let lastLatencyMs = 0;
 
+  function updateNetworkStats(stats) {
+    if (!networkStats || !stats) return;
+    const media = stats.video || stats.audio;
+    const parts = [];
+    if (Number.isFinite(media?.bitrateMbps)) parts.push(`${media.bitrateMbps.toFixed(1)} Mbps`);
+    if (Number.isFinite(stats.rttMs)) parts.push(`${Math.round(stats.rttMs)} ms`);
+    if (Number.isFinite(media?.lossPercent)) parts.push(`${media.lossPercent.toFixed(1)}% loss`);
+    const route = [stats.protocol, stats.route].filter(Boolean).join("/");
+    if (route) parts.push(route);
+    networkStats.textContent = parts.join(" · ") || "Connected";
+    const details = [];
+    if (media?.codec) details.push(`Codec: ${media.codec}`);
+    if (media?.width && media?.height) details.push(`Video: ${media.width}×${media.height}${media.framesPerSecond ? ` @ ${Math.round(media.framesPerSecond)} FPS` : ""}`);
+    if (Number.isFinite(media?.jitterMs)) details.push(`Jitter: ${media.jitterMs.toFixed(1)} ms`);
+    if (Number.isFinite(media?.jitterBufferMs)) details.push(`Jitter buffer: ${media.jitterBufferMs.toFixed(1)} ms`);
+    if (Number.isFinite(media?.framesDropped)) details.push(`Frames dropped: ${media.framesDropped}`);
+    networkStats.title = details.join("\n");
+  }
+
   const displaySelect = document.getElementById("displaySelect");
   const refreshBtn = document.getElementById("refreshDisplays");
   const startBtn = document.getElementById("startBtn");
@@ -771,6 +791,10 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
     webrtcActive = !!active;
     if (canvas) canvas.style.display = active ? "none" : "block";
     if (webrtcVideo) webrtcVideo.style.display = active ? "block" : "none";
+    if (!active && networkStats) {
+      networkStats.textContent = "--";
+      networkStats.title = "";
+    }
   }
 
   function onWebrtcState(label, state) {
@@ -823,6 +847,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
       whepPath,
       videoEl: webrtcVideo,
       onState: (state) => onWebrtcState("WebRTC Relayed", state),
+      onStats: (stats) => updateNetworkStats({ ...stats, route: "Server" }),
     });
     try {
       await whepClient.start();
@@ -846,6 +871,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
         }
       },
       onState: (state) => onWebrtcState("WebRTC P2P", state),
+      onStats: updateNetworkStats,
     });
     try {
       await p2pClient.start();

Overlord-Server/public/assets/remotedesktop.js

@@ -65,6 +65,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
   const duplicationCtrl = document.getElementById("duplicationCtrl");
   const streamProfileSelect = document.getElementById("streamProfileSelect");
   const streamProfileDetail = document.getElementById("streamProfileDetail");
+  const bitrateSelect = document.getElementById("bitrateSelect");
   const smoothingSlider = document.getElementById("smoothingSlider");
   const smoothingValue = document.getElementById("smoothingValue");
   const qualitySlider = document.getElementById("qualitySlider");
@@ -78,6 +79,14 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
   const agentFps = document.getElementById("agentFps");
   const viewerFps = document.getElementById("viewerFps");
   const inputLatency = document.getElementById("inputLatency");
+  const networkStats = document.getElementById("networkStats");
+  const diagnosticsBtn = document.getElementById("diagnosticsBtn");
+  const diagnosticsHud = document.getElementById("diagnosticsHud");
+  const diagnosticsClose = document.getElementById("diagnosticsClose");
+  const diagnosticEls = Object.fromEntries([
+    "Summary", "Transport", "Codec", "Resolution", "Capture", "Encode", "Send", "AgentTotal",
+    "Bitrate", "Rtt", "LossJitter", "JitterBuffer", "Decode", "Render", "Queue", "Dropped", "Fps", "Input",
+  ].map((name) => [name, document.getElementById(`diag${name}`)]));
   const statusEl = document.getElementById("streamStatus");
   const clipboardSyncCtrl = document.getElementById("clipboardSyncCtrl");
   const privacyCtrl = document.getElementById("privacyCtrl");
@@ -111,6 +120,10 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
   let frameDecodeBusy = false;
   let pendingFrame = null;
   let videoDecoder = null;
+  let videoDecoderConfigKey = "";
+  let h264StreamCodec = "";
+  let canvasDecodePressure = false;
+  let h264PendingTimings = [];
   let h264TimestampUs = 0;
   let prefersH264 = typeof VideoDecoder === "function";
   let h264LowFpsStreak = 0;
@@ -129,6 +142,22 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
   const mouseMoveIntervalMs = 33;
   const inputBackpressureBytes = 256 * 1024;
   let lastMoveSentAt = 0;
+  const diagnostics = {
+    agent: null,
+    network: null,
+    decodeMs: null,
+    renderMs: null,
+    decodeQueue: 0,
+    coalescedFrames: 0,
+    currentAgentFps: null,
+    currentViewerFps: null,
+    wsBitrateMbps: null,
+    wsBytes: 0,
+    wsWindowStartedAt: performance.now(),
+    codec: "",
+    width: 0,
+    height: 0,
+  };
 
   let clipboardSyncTimer = null;
   let lastClipboardText = "";
@@ -149,6 +178,9 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
 
   function resetH264RuntimeState() {
     h264TimestampUs = 0;
+    h264StreamCodec = "";
+    h264PendingTimings = [];
+    diagnostics.decodeQueue = 0;
     h264LowFpsStreak = 0;
     h264FirstFrameAt = 0;
     h264FramesSeen = 0;
@@ -396,6 +428,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
         ? `${settings.resolution}:${settings.targetFps}`
         : savedStreamProfile);
     setSelectValue(streamProfileSelect, savedStreamProfile);
+    setSelectValue(bitrateSelect, settings.bitrateMbps);
     setSelectValue(webrtcMode, settings.webrtcMode);
     setSelectValue(audioTransport, settings.audioTransport);
     setSelectValue(recordMode, settings.recordMode);
@@ -428,6 +461,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
       resolution: String(profile.maxHeight),
       targetFps: String(profile.fps),
       quality: Number(qualitySlider?.value || 90),
+      bitrateMbps: Number(bitrateSelect?.value || 0),
       preferH264: !!prefersH264,
       webrtcMode: getWebrtcMode(),
       mouse: !!mouseCtrl?.checked,
@@ -469,6 +503,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
   function updateFpsDisplay(agentValue) {
     if (agentValue !== undefined && agentValue !== null && agentFps) {
       agentFps.textContent = String(agentValue);
+      diagnostics.currentAgentFps = Number(agentValue) || null;
     }
     const now = performance.now();
     renderCount += 1;
@@ -476,6 +511,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
     if (elapsed >= 1000 && viewerFps) {
       const fps = Math.round((renderCount * 1000) / elapsed);
       viewerFps.textContent = String(fps);
+      diagnostics.currentViewerFps = fps;
       renderCount = 0;
       renderWindowStart = now;
     }
@@ -827,6 +863,142 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
     }
   }
 
+  function finite(value) {
+    return Number.isFinite(value) ? Number(value) : null;
+  }
+
+  function smoothed(previous, next, weight = 0.25) {
+    const value = finite(next);
+    if (value == null) return previous;
+    return previous == null ? value : previous * (1 - weight) + value * weight;
+  }
+
+  function msText(value) {
+    const number = finite(value);
+    return number == null ? "-- ms" : `${number < 10 ? number.toFixed(1) : Math.round(number)} ms`;
+  }
+
+  function setDiagnosticsVisible(visible) {
+    if (!diagnosticsHud) return;
+    diagnosticsHud.classList.toggle("hidden", !visible);
+    diagnosticsBtn?.setAttribute("aria-pressed", visible ? "true" : "false");
+    try { localStorage.setItem("overlord.rd.statsHud", visible ? "1" : "0"); } catch {}
+  }
+
+  function renderDiagnostics() {
+    if (!diagnosticsHud) return;
+    const agent = diagnostics.agent || {};
+    const network = diagnostics.network || {};
+    const media = network.video || network.audio || {};
+    const mode = getWebrtcMode();
+    const transport = mode === "p2p" ? "WebRTC P2P" : mode === "relayed" ? "WebRTC server" : "Canvas WebSocket";
+    const bitrate = finite(media.bitrateMbps) ?? finite(diagnostics.wsBitrateMbps);
+    const decodeMs = finite(media.decodeMs) ?? finite(diagnostics.decodeMs);
+    const renderMs = finite(media.processingDelayMs) ?? finite(diagnostics.renderMs);
+    const dropped = finite(media.framesDropped) ?? 0;
+    const queue = mode === "off" ? diagnostics.decodeQueue : null;
+    const fps = finite(agent.fps) ?? diagnostics.currentAgentFps;
+    const viewerRate = finite(media.framesPerSecond) ?? diagnostics.currentViewerFps;
+    const frameBudget = 1000 / Math.max(1, fps || viewerRate || 30);
+
+    let summary = "Pipeline healthy";
+    let severity = "ok";
+    if (streamState !== "streaming" && streamState !== "stalled") {
+      summary = "Stream is not active";
+      severity = "warn";
+    } else if (finite(agent.captureMs) > frameBudget * 0.75) {
+      summary = "Capture is limiting frame rate";
+      severity = "bad";
+    } else if (finite(agent.encodeMs) > frameBudget * 0.75) {
+      summary = "Encoder is limiting frame rate";
+      severity = "bad";
+    } else if (decodeMs != null && decodeMs > frameBudget * 0.75) {
+      const viewerKeepingUp = fps != null && viewerRate != null && viewerRate >= fps * 0.9;
+      summary = viewerKeepingUp
+        ? `Decoder buffering adds ${Math.round(decodeMs)} ms`
+        : "Viewer decoder throughput is limiting FPS";
+      severity = viewerKeepingUp ? "warn" : "bad";
+    } else if ((finite(media.lossPercent) ?? 0) > 3 || (finite(network.rttMs) ?? 0) > 150 || (finite(media.jitterMs) ?? 0) > 35) {
+      summary = "Network conditions are causing delay";
+      severity = "bad";
+    } else if ((queue ?? 0) > 2 || (renderMs != null && renderMs > frameBudget)) {
+      summary = "Viewer render queue is backing up";
+      severity = "warn";
+    } else if (!diagnostics.agent && !diagnostics.network) {
+      summary = "Waiting for stream telemetry";
+      severity = "warn";
+    }
+
+    diagnosticEls.Summary.textContent = summary;
+    diagnosticEls.Summary.dataset.severity = severity;
+    diagnosticEls.Transport.textContent = transport;
+    diagnosticEls.Codec.textContent = String(media.codec || agent.format || diagnostics.codec || "--").toUpperCase();
+    const width = finite(media.width) ?? diagnostics.width;
+    const height = finite(media.height) ?? diagnostics.height;
+    diagnosticEls.Resolution.textContent = width && height ? `${width}×${height}` : "--";
+    diagnosticEls.Capture.textContent = msText(agent.captureMs);
+    diagnosticEls.Encode.textContent = msText(agent.encodeMs);
+    diagnosticEls.Send.textContent = msText(agent.sendMs);
+    diagnosticEls.AgentTotal.textContent = msText(agent.totalMs);
+    diagnosticEls.Bitrate.textContent = bitrate == null ? "-- Mbps" : `${bitrate.toFixed(2)} Mbps`;
+    diagnosticEls.Rtt.textContent = msText(network.rttMs);
+    diagnosticEls.LossJitter.textContent = `${finite(media.lossPercent)?.toFixed(1) ?? "--"}% / ${msText(media.jitterMs)}`;
+    diagnosticEls.JitterBuffer.textContent = msText(media.jitterBufferMs);
+    diagnosticEls.Decode.textContent = msText(decodeMs);
+    diagnosticEls.Render.textContent = msText(renderMs);
+    diagnosticEls.Queue.textContent = queue == null ? "managed by browser" : String(queue);
+    diagnosticEls.Dropped.textContent = `${Math.round(dropped)} / ${diagnostics.coalescedFrames}`;
+    diagnosticEls.Fps.textContent = `${fps == null ? "--" : Math.round(fps)} → ${viewerRate == null ? "--" : Math.round(viewerRate)}`;
+    diagnosticEls.Input.textContent = msText(latencyAvg);
+  }
+
+  diagnosticsBtn?.addEventListener("click", () => setDiagnosticsVisible(diagnosticsHud?.classList.contains("hidden")));
+  diagnosticsClose?.addEventListener("click", () => setDiagnosticsVisible(false));
+  networkStats?.addEventListener("click", () => setDiagnosticsVisible(true));
+  try { setDiagnosticsVisible(localStorage.getItem("overlord.rd.statsHud") === "1"); } catch {}
+  setInterval(renderDiagnostics, 250);
+
+  function updateNetworkStats(stats) {
+    if (!networkStats || !stats) return;
+    diagnostics.network = stats;
+    const media = stats.video || stats.audio;
+    const parts = [];
+    if (Number.isFinite(media?.bitrateMbps)) parts.push(`${media.bitrateMbps.toFixed(1)} Mbps`);
+    if (Number.isFinite(stats.rttMs)) parts.push(`${Math.round(stats.rttMs)} ms`);
+    if (Number.isFinite(media?.lossPercent)) parts.push(`${media.lossPercent.toFixed(1)}% loss`);
+    const route = [stats.protocol, stats.route].filter(Boolean).join("/");
+    if (route) parts.push(route);
+    networkStats.textContent = parts.join(" · ") || "Connected";
+    const details = [];
+    if (media?.codec) details.push(`Codec: ${media.codec}`);
+    if (media?.width && media?.height) details.push(`Video: ${media.width}×${media.height}${media.framesPerSecond ? ` @ ${Math.round(media.framesPerSecond)} FPS` : ""}`);
+    if (Number.isFinite(media?.jitterMs)) details.push(`Jitter: ${media.jitterMs.toFixed(1)} ms`);
+    if (Number.isFinite(media?.jitterBufferMs)) details.push(`Jitter buffer: ${media.jitterBufferMs.toFixed(1)} ms`);
+    if (Number.isFinite(media?.framesDropped)) details.push(`Frames dropped: ${media.framesDropped}`);
+    if (Number.isFinite(stats.availableIncomingMbps)) details.push(`Available: ${stats.availableIncomingMbps.toFixed(1)} Mbps`);
+    networkStats.title = details.join("\n");
+    renderDiagnostics();
+  }
+
+  function handleDesktopStreamStats(stats) {
+    const previous = diagnostics.agent || {};
+    diagnostics.agent = {
+      ...stats,
+      captureMs: smoothed(finite(previous.captureMs), stats.captureMs),
+      encodeMs: smoothed(finite(previous.encodeMs), stats.encodeMs),
+      sendMs: smoothed(finite(previous.sendMs), stats.sendMs),
+      totalMs: smoothed(finite(previous.totalMs), stats.totalMs),
+    };
+    diagnostics.currentAgentFps = finite(stats.fps);
+    diagnostics.codec = String(stats.format || diagnostics.codec || "");
+    diagnostics.width = finite(stats.width) || diagnostics.width;
+    diagnostics.height = finite(stats.height) || diagnostics.height;
+    if (agentFps && diagnostics.currentAgentFps != null) {
+      agentFps.textContent = String(Math.round(diagnostics.currentAgentFps));
+    }
+    renderDiagnostics();
+  }
+
   function clearOfflineTimer() {
     if (offlineTimer) {
       clearTimeout(offlineTimer);
@@ -919,10 +1091,10 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
           if (mode === "relayed") {
             sendCmd("desktop_start", { webrtc: true });
           } else if (mode === "p2p") {
-            sendCmd("desktop_start", {});
+            sendCmd("desktop_start", { canvasFlowControl: false });
             startP2P();
           } else {
-            sendCmd("desktop_start", {});
+            sendCmd("desktop_start", { canvasFlowControl: true });
           }
         }
       } else {
@@ -1043,6 +1215,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
       return;
     }
     const msg = { type, ...payload };
+    if (type !== "desktop_decode_pressure") {
       rdDebug("send", {
         msg,
         readyState: ws.readyState,
@@ -1054,6 +1227,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
         transport: getWebrtcMode(),
         lastFrameAgeMs: lastFrameAt ? Math.round(performance.now() - lastFrameAt) : null,
       });
+    }
     ws.send(encodeMsgpack(msg));
   }
 
@@ -1061,6 +1235,11 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
     webrtcActive = !!active;
     if (canvas) canvas.style.display = active ? "none" : "block";
     if (webrtcVideo) webrtcVideo.style.display = active ? "block" : "none";
+    if (!active && networkStats) {
+      networkStats.textContent = "--";
+      networkStats.title = "";
+      diagnostics.network = null;
+    }
   }
 
   function onWebrtcState(label, s) {
@@ -1084,15 +1263,22 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
     stopWebrtcFrameTicker();
     webrtcFpsCount = 0;
     webrtcFpsWindowStart = performance.now();
-    const tick = (now) => {
+    const tick = (now, metadata = {}) => {
       markFrameReceived();
+      if (Number.isFinite(metadata.processingDuration)) {
+        diagnostics.decodeMs = smoothed(diagnostics.decodeMs, metadata.processingDuration * 1000);
+      }
+      if (Number.isFinite(metadata.receiveTime) && Number.isFinite(metadata.expectedDisplayTime)) {
+        diagnostics.renderMs = smoothed(diagnostics.renderMs, Math.max(0, metadata.expectedDisplayTime - metadata.receiveTime));
+      }
+      diagnostics.width = webrtcVideo.videoWidth || diagnostics.width;
+      diagnostics.height = webrtcVideo.videoHeight || diagnostics.height;
       webrtcFpsCount += 1;
       const elapsed = now - webrtcFpsWindowStart;
       if (elapsed >= 1000) {
         const fps = Math.round((webrtcFpsCount * 1000) / elapsed);
-        // Agent and viewer FPS are effectively equal in WebRTC mode — the
+        diagnostics.currentViewerFps = fps;
-        // browser renders every frame the agent sends, no decode skipping.
+        updateFpsDisplay();
-        updateFpsDisplay(fps);
         webrtcFpsCount = 0;
         webrtcFpsWindowStart = now;
       } else {
@@ -1116,6 +1302,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
       whepPath,
       videoEl: webrtcVideo,
       onState: (s) => onWebrtcState("WebRTC Relayed", s),
+      onStats: (stats) => updateNetworkStats({ ...stats, route: "Server" }),
     });
     try {
       await whepClient.start();
@@ -1139,6 +1326,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
         }
       },
       onState: (s) => onWebrtcState("WebRTC P2P", s),
+      onStats: updateNetworkStats,
     });
     try {
       await p2pClient.start();
@@ -1306,6 +1494,18 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
     sendCmd("desktop_set_profile", profile);
   }
 
+  function pushBitrate() {
+    const bitrateMbps = Math.max(0, Math.min(50, Number.parseInt(bitrateSelect?.value || "0", 10) || 0));
+    sendCmd("desktop_set_bitrate", { bitrateMbps });
+  }
+
+  if (bitrateSelect) {
+    bitrateSelect.addEventListener("change", function () {
+      pushBitrate();
+      sharedSettingsSaver.scheduleSave();
+    });
+  }
+
   function requestEncoderCapabilities() {
     if (streamProfileDetail) streamProfileDetail.textContent = "Checking hardware encoder profiles…";
     sendCmd("desktop_encoder_capabilities", {
@@ -1419,6 +1619,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
       ensureDuplicationForH264();
     }
     pushStreamProfile();
+    pushBitrate();
     desiredStreaming = true;
     lastFrameAt = 0;
     firstFrameLogged = false;
@@ -1429,10 +1630,10 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
       sendCmd("desktop_start", { webrtc: true });
     } else if (mode === "p2p") {
       // Kick off the P2P offer asynchronously; capture starts in parallel.
-      sendCmd("desktop_start", {});
+      sendCmd("desktop_start", { canvasFlowControl: false });
       startP2P();
     } else {
-      sendCmd("desktop_start", {});
+      sendCmd("desktop_start", { canvasFlowControl: true });
     }
     if (audioCtrl && audioCtrl.checked) {
       connectAudio();
@@ -1644,6 +1845,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
   }
 
   function destroyVideoDecoder() {
+    updateCanvasDecodePressure(true);
     if (!videoDecoder) return;
     try {
       videoDecoder.close();
@@ -1651,6 +1853,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
       // Ignore close errors when decoder is already shutting down.
     }
     videoDecoder = null;
+    videoDecoderConfigKey = "";
     resetH264RuntimeState();
   }
 
@@ -1724,6 +1927,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
       sendCmd("desktop_start", {
         source: "rd_viewer",
         reason: "h264_recovery_restart",
+        canvasFlowControl: getWebrtcMode() === "off",
       });
       const q = Number(qualitySlider?.value) || 90;
       sendCmd("desktop_set_quality", {
@@ -1769,16 +1973,39 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
     return false;
   }
 
-  function ensureVideoDecoder() {
+  function h264CodecFromAnnexB(data) {
-    if (videoDecoder) {
+    for (let i = 0; i + 7 < data.length; i++) {
+      let startCodeLength = 0;
+      if (data[i] === 0 && data[i + 1] === 0 && data[i + 2] === 1) startCodeLength = 3;
+      else if (data[i] === 0 && data[i + 1] === 0 && data[i + 2] === 0 && data[i + 3] === 1) startCodeLength = 4;
+      if (!startCodeLength) continue;
+      const nal = i + startCodeLength;
+      if ((data[nal] & 0x1f) !== 7 || nal + 3 >= data.length) continue;
+      return `avc1.${[data[nal + 1], data[nal + 2], data[nal + 3]]
+        .map((value) => value.toString(16).padStart(2, "0"))
+        .join("")}`;
+    }
+    return "";
+  }
+
+  function ensureVideoDecoder(h264Bytes, width, height) {
+    const detectedCodec = h264CodecFromAnnexB(h264Bytes);
+    const codec = detectedCodec || h264StreamCodec || "avc1.4d0034";
+    const codedWidth = Math.max(0, Number(width) || 0);
+    const codedHeight = Math.max(0, Number(height) || 0);
+    const configKey = `${codec}:${codedWidth}x${codedHeight}`;
+    if (videoDecoder && videoDecoderConfigKey === configKey) {
       return true;
     }
     if (typeof VideoDecoder !== "function") {
       return false;
     }
     try {
+      if (videoDecoder) destroyVideoDecoder();
       videoDecoder = new VideoDecoder({
         output: (frame) => {
+          const outputStartedAt = performance.now();
+          const timing = h264PendingTimings.shift();
           const width = frame.displayWidth || frame.codedWidth || frameWidth;
           const height = frame.displayHeight || frame.codedHeight || frameHeight;
           if (width > 0 && height > 0 && (canvas.width !== width || canvas.height !== height)) {
@@ -1792,12 +2019,34 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
           } finally {
             frame.close();
           }
+          const renderedAt = performance.now();
+          if (timing) {
+            diagnostics.decodeMs = smoothed(diagnostics.decodeMs, Math.max(0, outputStartedAt - timing.decodeStartedAt));
+            diagnostics.renderMs = smoothed(diagnostics.renderMs, Math.max(0, renderedAt - timing.receivedAt));
+          }
+          diagnostics.decodeQueue = videoDecoder?.decodeQueueSize || h264PendingTimings.length;
+          diagnostics.width = width;
+          diagnostics.height = height;
+          updateFpsDisplay();
+          updateCanvasDecodePressure();
         },
         error: (err) => {
           console.warn("rd: h264 decoder error", err);
+          updateCanvasDecodePressure(true);
         },
       });
-      videoDecoder.configure({ codec: "avc1.42E01E", optimizeForLatency: true });
+      videoDecoder.addEventListener("dequeue", () => {
+        diagnostics.decodeQueue = videoDecoder?.decodeQueueSize || 0;
+        updateCanvasDecodePressure();
+      });
+      const config = { codec, optimizeForLatency: true };
+      if (codedWidth > 0 && codedHeight > 0) {
+        config.codedWidth = codedWidth;
+        config.codedHeight = codedHeight;
+      }
+      videoDecoder.configure(config);
+      videoDecoderConfigKey = configKey;
+      h264StreamCodec = codec;
       return true;
     } catch (err) {
       console.warn("rd: h264 decoder unavailable", err);
@@ -1807,22 +2056,72 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
     }
   }
 
-  async function processFrameBuffer(buf) {
+  function recordWsFrameBytes(byteLength) {
+    diagnostics.wsBytes += Math.max(0, Number(byteLength) || 0);
+    const now = performance.now();
+    const elapsed = now - diagnostics.wsWindowStartedAt;
+    if (elapsed >= 1000) {
+      const rate = (diagnostics.wsBytes * 8) / (elapsed * 1000);
+      diagnostics.wsBitrateMbps = smoothed(diagnostics.wsBitrateMbps, rate, 0.4);
+      diagnostics.wsBytes = 0;
+      diagnostics.wsWindowStartedAt = now;
+    }
+  }
+
+  function recordCanvasFrameTiming(receivedAt, decodeStartedAt) {
+    const renderedAt = performance.now();
+    diagnostics.decodeMs = smoothed(diagnostics.decodeMs, Math.max(0, renderedAt - decodeStartedAt));
+    diagnostics.renderMs = smoothed(diagnostics.renderMs, Math.max(0, renderedAt - receivedAt));
+    diagnostics.decodeQueue = videoDecoder?.decodeQueueSize || (pendingFrame ? 1 : 0);
+    diagnostics.width = frameWidth || canvas.width || diagnostics.width;
+    diagnostics.height = frameHeight || canvas.height || diagnostics.height;
+  }
+
+  function updateCanvasDecodePressure(forceRelease = false) {
+    const queueSize = videoDecoder?.decodeQueueSize || 0;
+    const shouldApply = !forceRelease && getWebrtcMode() === "off" && queueSize > 3;
+    const shouldRelease = forceRelease || getWebrtcMode() !== "off" || queueSize <= 1;
+    let next = canvasDecodePressure;
+    if (!canvasDecodePressure && shouldApply) next = true;
+    else if (canvasDecodePressure && shouldRelease) next = false;
+    if (next === canvasDecodePressure) return;
+    canvasDecodePressure = next;
+    sendCmd("desktop_decode_pressure", { active: next, queueSize });
+  }
+
+  async function processFrameBuffer(buf, receivedAt = performance.now()) {
+    const decodeStartedAt = performance.now();
     const fps = buf[5];
     const format = buf[6];
+    const headerLength = buf[3] >= 2 && buf.length >= 12 ? 12 : 8;
+    const encodedWidth = headerLength === 12 ? buf[8] | (buf[9] << 8) : 0;
+    const encodedHeight = headerLength === 12 ? buf[10] | (buf[11] << 8) : 0;
+    if (encodedWidth > 0 && encodedHeight > 0) {
+      frameWidth = encodedWidth;
+      frameHeight = encodedHeight;
+      diagnostics.width = encodedWidth;
+      diagnostics.height = encodedHeight;
+    }
 
     if (format === 1) {
-      const jpegBytes = buf.slice(8);
+      const jpegBytes = buf.slice(headerLength);
       setCodecModeLabel("jpeg", "active");
+      diagnostics.codec = "jpeg";
       await drawJpegSlice(jpegBytes, null);
+      recordCanvasFrameTiming(receivedAt, decodeStartedAt);
       updateFpsDisplay(fps);
+      updateCanvasDecodePressure(true);
       return;
     }
 
     if (format === 2 || format === 3) {
       setCodecModeLabel(format === 3 ? "raw" : "jpeg", format === 3 ? "blocks" : "blocks");
-      if (buf.length < 16) return;
+      diagnostics.codec = format === 3 ? "raw blocks" : "jpeg blocks";
-      const dv = new DataView(buf.buffer, 8);
+      if (buf.length < headerLength + 8) {
+        updateCanvasDecodePressure(true);
+        return;
+      }
+      const dv = new DataView(buf.buffer, buf.byteOffset + headerLength, buf.byteLength - headerLength);
       let pos = 0;
       const width = dv.getUint16(pos, true);
       pos += 2;
@@ -1857,7 +2156,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
         pos += 2;
         const len = dv.getUint32(pos, true);
         pos += 4;
-        const start = 8 + pos;
+        const start = headerLength + pos;
         const end = start + len;
         if (end > buf.length) break;
         const slice = buf.subarray(start, end);
@@ -1872,15 +2171,22 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
       }
 
       updateFpsDisplay(fps);
+      recordCanvasFrameTiming(receivedAt, decodeStartedAt);
+      updateCanvasDecodePressure(true);
       return;
     }
 
     if (format === 4) {
       setCodecModeLabel("h264", "active");
-      const h264Bytes = buf.slice(8);
+      diagnostics.codec = "h264";
-      if (!h264Bytes.length) return;
+      const h264Bytes = buf.slice(headerLength);
-      if (!ensureVideoDecoder()) {
+      if (!h264Bytes.length) {
+        updateCanvasDecodePressure(true);
+        return;
+      }
+      if (!ensureVideoDecoder(h264Bytes, encodedWidth || frameWidth, encodedHeight || frameHeight)) {
         fallbackToJpegCodec("WebCodecs decoder unavailable");
+        updateCanvasDecodePressure(true);
         return;
       }
 
@@ -1905,6 +2211,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
         h264LowFpsStreak >= H264_LOW_FPS_STREAK_LIMIT
       ) {
         fallbackToJpegCodec(`low h264 fps (${fps})`);
+        updateCanvasDecodePressure(true);
         return;
       }
 
@@ -1916,10 +2223,13 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
       });
       h264TimestampUs += frameIntervalUs;
       try {
+        h264PendingTimings.push({ receivedAt, decodeStartedAt: performance.now() });
         videoDecoder.decode(chunk);
+        diagnostics.decodeQueue = videoDecoder.decodeQueueSize;
+        updateCanvasDecodePressure();
         h264KeyframeErrorStreak = 0;
-        updateFpsDisplay(fps);
       } catch (err) {
+        h264PendingTimings.pop();
         if (isKeyframeRequiredError(err)) {
           h264KeyframeErrorStreak += 1;
           const now = Date.now();
@@ -1936,12 +2246,16 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
               fallbackToJpegCodec("h264_keyframe_required_loop");
             }
           }
+          updateCanvasDecodePressure(true);
           return;
         }
         console.warn("rd: h264 decode failed", err);
         fallbackToJpegCodec(err);
+        updateCanvasDecodePressure(true);
       }
+      return;
     }
+    updateCanvasDecodePressure(true);
   }
 
   function flushPendingFrame() {
@@ -1951,7 +2265,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
     const next = pendingFrame;
     pendingFrame = null;
     frameDecodeBusy = true;
-    processFrameBuffer(next).finally(() => {
+    processFrameBuffer(next.buf, next.receivedAt).finally(() => {
       frameDecodeBusy = false;
       if (pendingFrame) {
         flushPendingFrame();
@@ -1964,8 +2278,10 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
       const buf = new Uint8Array(ev.data);
       if (isFramePacket(buf)) {
         markFrameReceived();
+        recordWsFrameBytes(buf.byteLength);
         // Coalesce bursty arrivals so the renderer catches up to the newest frame.
-        pendingFrame = buf;
+        if (pendingFrame) diagnostics.coalescedFrames += 1;
+        pendingFrame = { buf, receivedAt: performance.now() };
         flushPendingFrame();
         return;
       }
@@ -1975,6 +2291,10 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
         applyEncoderCapabilities(msg);
         return;
       }
+      if (msg && msg.type === "desktop_stream_stats") {
+        handleDesktopStreamStats(msg);
+        return;
+      }
       if (msg && msg.type === "status" && msg.status) {
         handleStatus(msg);
         return;
@@ -2014,6 +2334,10 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
       applyEncoderCapabilities(msg);
       return;
     }
+    if (msg && msg.type === "desktop_stream_stats") {
+      handleDesktopStreamStats(msg);
+      return;
+    }
     if (msg && msg.type === "status" && msg.status) {
       handleStatus(msg);
       return;
@@ -2045,6 +2369,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
     if (qualitySlider) {
       pushQuality(qualitySlider.value);
     }
+    pushBitrate();
     pushInputToggles();
     pushCaptureToggles();
     clearOfflineTimer();
@@ -2060,10 +2385,10 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
       if (mode === "relayed") {
         sendCmd("desktop_start", { webrtc: true });
       } else if (mode === "p2p") {
-        sendCmd("desktop_start", {});
+        sendCmd("desktop_start", { canvasFlowControl: false });
         startP2P();
       } else {
-        sendCmd("desktop_start", {});
+        sendCmd("desktop_start", { canvasFlowControl: true });
       }
     } else {
       setStreamState("idle", "Stopped");

Overlord-Server/public/assets/webrtc-p2p.js

@@ -10,6 +10,8 @@
 // The server proxies these to the agent via its existing WS command channel
 // (sessionId is server-side state, not visible to the browser).
 //
+import { WebRTCStatsSampler } from "./webrtc-stats.js";
+
 // Once SDP + ICE settle, the <video> element receives frames directly from
 // the agent — MediaMTX is not involved.
 
@@ -20,6 +22,7 @@ export class P2PClient {
    * @param {HTMLAudioElement} [opts.audioEl]
    * @param {(msg: object) => void} opts.send  Send a JSON-encodable msg over the viewer's WS.
    * @param {(state: string) => void} [opts.onState]
+   * @param {(stats: object) => void} [opts.onStats]
    * @param {string[]} [opts.iceServers]  STUN URLs. Default: Google public STUN.
    */
   constructor(opts) {
@@ -27,9 +30,11 @@ export class P2PClient {
     this.audioEl = opts.audioEl || null;
     this.send = opts.send;
     this.onState = opts.onState || (() => {});
+    this.onStats = opts.onStats || (() => {});
     this.iceServers = opts.iceServers || ["stun:stun.l.google.com:19302"];
     this.pc = null;
     this.pendingRemoteCandidates = [];
+    this.statsSampler = null;
   }
 
   async start() {
@@ -39,6 +44,7 @@ export class P2PClient {
       iceServers: this.iceServers.map((u) => ({ urls: u })),
     });
     this.pc = pc;
+    this.statsSampler = new WebRTCStatsSampler(pc, this.onStats);
 
     if (this.videoEl) pc.addTransceiver("video", { direction: "recvonly" });
     if (this.audioEl) pc.addTransceiver("audio", { direction: "recvonly" });
@@ -66,6 +72,8 @@ export class P2PClient {
     };
     pc.onconnectionstatechange = () => {
       this.onState(pc.connectionState);
+      if (pc.connectionState === "connected") this.statsSampler?.start();
+      if (["failed", "closed", "disconnected"].includes(pc.connectionState)) this.statsSampler?.stop();
     };
 
     const offer = await pc.createOffer();
@@ -105,6 +113,8 @@ export class P2PClient {
   async stop() {
     const pc = this.pc;
     this.pc = null;
+    this.statsSampler?.stop();
+    this.statsSampler = null;
     this.pendingRemoteCandidates = [];
     if (pc) {
       try { pc.close(); } catch {}

Overlord-Server/public/assets/webrtc-stats.js

@@ -0,0 +1,139 @@
+// Shared receiver-side WebRTC telemetry for P2P and WHEP sessions.
+// Values are deliberately normalized here so the viewer pages do not need to
+// know about browser-specific RTCStatsReport relationships.
+export class WebRTCStatsSampler {
+  constructor(pc, onStats, intervalMs = 1000) {
+    this.pc = pc;
+    this.onStats = typeof onStats === "function" ? onStats : () => {};
+    this.intervalMs = intervalMs;
+    this.timer = null;
+    this.running = false;
+    this.previous = new Map();
+  }
+
+  start() {
+    if (this.timer || !this.pc) return;
+    this.sample();
+    this.timer = setInterval(() => this.sample(), this.intervalMs);
+  }
+
+  stop() {
+    if (this.timer) clearInterval(this.timer);
+    this.timer = null;
+    this.running = false;
+    this.previous.clear();
+  }
+
+  async sample() {
+    if (this.running || !this.pc || this.pc.connectionState === "closed") return;
+    this.running = true;
+    try {
+      const report = await this.pc.getStats();
+      const byId = new Map();
+      report.forEach((item) => byId.set(item.id, item));
+
+      const result = {
+        timestamp: performance.now(),
+        rttMs: null,
+        availableIncomingMbps: null,
+        protocol: "",
+        route: "",
+        video: null,
+        audio: null,
+      };
+
+      let selectedPair = null;
+      for (const item of byId.values()) {
+        if (item.type === "transport" && item.selectedCandidatePairId) {
+          selectedPair = byId.get(item.selectedCandidatePairId) || selectedPair;
+        }
+        if (item.type === "candidate-pair" && item.state === "succeeded" && (item.nominated || item.selected)) {
+          selectedPair = selectedPair || item;
+        }
+      }
+      if (selectedPair) {
+        if (Number.isFinite(selectedPair.currentRoundTripTime)) {
+          result.rttMs = selectedPair.currentRoundTripTime * 1000;
+        }
+        if (Number.isFinite(selectedPair.availableIncomingBitrate)) {
+          result.availableIncomingMbps = selectedPair.availableIncomingBitrate / 1_000_000;
+        }
+        const local = byId.get(selectedPair.localCandidateId);
+        const remote = byId.get(selectedPair.remoteCandidateId);
+        result.protocol = String(local?.protocol || remote?.protocol || "").toUpperCase();
+        result.route = summarizeRoute(local?.candidateType, remote?.candidateType);
+      }
+
+      for (const item of byId.values()) {
+        if (item.type !== "inbound-rtp" || item.isRemote) continue;
+        const kind = item.kind || item.mediaType;
+        if (kind !== "video" && kind !== "audio") continue;
+        const codec = byId.get(item.codecId);
+        const previous = this.previous.get(item.id);
+        const timestamp = Number(item.timestamp) || performance.now();
+        let bitrateMbps = null;
+        let decodeMs = null;
+        let processingDelayMs = null;
+        let framesDroppedDelta = null;
+        if (previous && timestamp > previous.timestamp && Number(item.bytesReceived) >= previous.bytesReceived) {
+          bitrateMbps = ((Number(item.bytesReceived) - previous.bytesReceived) * 8) /
+            ((timestamp - previous.timestamp) * 1000);
+          const decodedDelta = (Number(item.framesDecoded) || 0) - previous.framesDecoded;
+          if (decodedDelta > 0) {
+            const decodeDelta = (Number(item.totalDecodeTime) || 0) - previous.totalDecodeTime;
+            const processingDelta = (Number(item.totalProcessingDelay) || 0) - previous.totalProcessingDelay;
+            if (decodeDelta >= 0) decodeMs = (decodeDelta * 1000) / decodedDelta;
+            if (processingDelta >= 0) processingDelayMs = (processingDelta * 1000) / decodedDelta;
+          }
+          const droppedDelta = (Number(item.framesDropped) || 0) - previous.framesDropped;
+          if (droppedDelta >= 0) framesDroppedDelta = droppedDelta;
+        }
+        this.previous.set(item.id, {
+          timestamp,
+          bytesReceived: Number(item.bytesReceived) || 0,
+          framesDecoded: Number(item.framesDecoded) || 0,
+          totalDecodeTime: Number(item.totalDecodeTime) || 0,
+          totalProcessingDelay: Number(item.totalProcessingDelay) || 0,
+          framesDropped: Number(item.framesDropped) || 0,
+        });
+        const packetsReceived = Number(item.packetsReceived) || 0;
+        const packetsLost = Math.max(0, Number(item.packetsLost) || 0);
+        const packetTotal = packetsReceived + packetsLost;
+        result[kind] = {
+          bitrateMbps,
+          packetsLost,
+          lossPercent: packetTotal > 0 ? (packetsLost * 100) / packetTotal : 0,
+          jitterMs: Number.isFinite(item.jitter) ? item.jitter * 1000 : null,
+          jitterBufferMs: averageJitterBufferMs(item),
+          decodeMs,
+          processingDelayMs,
+          framesDropped: Number.isFinite(item.framesDropped) ? item.framesDropped : null,
+          framesDroppedDelta,
+          framesPerSecond: Number.isFinite(item.framesPerSecond) ? item.framesPerSecond : null,
+          width: Number.isFinite(item.frameWidth) ? item.frameWidth : null,
+          height: Number.isFinite(item.frameHeight) ? item.frameHeight : null,
+          codec: String(codec?.mimeType || "").replace(/^\w+\//, ""),
+        };
+      }
+      this.onStats(result);
+    } catch (error) {
+      if (this.pc?.connectionState !== "closed") console.debug("webrtc stats unavailable", error);
+    } finally {
+      this.running = false;
+    }
+  }
+}
+
+function averageJitterBufferMs(item) {
+  const delay = Number(item.jitterBufferDelay);
+  const emitted = Number(item.jitterBufferEmittedCount);
+  return Number.isFinite(delay) && emitted > 0 ? (delay * 1000) / emitted : null;
+}
+
+function summarizeRoute(localType, remoteType) {
+  const types = [localType, remoteType].filter(Boolean);
+  if (types.includes("relay")) return "Relay";
+  if (types.includes("srflx") || types.includes("prflx")) return "P2P";
+  if (types.includes("host")) return "Direct";
+  return "";
+}

Overlord-Server/public/assets/whep.js

@@ -1,3 +1,5 @@
+import { WebRTCStatsSampler } from "./webrtc-stats.js";
+
 // Tiny WHEP (WebRTC-HTTP Egress Protocol) client.
 //
 // Wires a <video> element to a server-proxied WHEP endpoint:
@@ -16,14 +18,17 @@ export class WhepClient {
    * @param {HTMLAudioElement} [opts.audioEl]  Attach the audio track here.
    *                                            Default: hidden <audio> appended to <body>.
    * @param {(state: string) => void} [opts.onState]
+   * @param {(stats: object) => void} [opts.onStats]
    */
   constructor(opts) {
     this.whepPath = opts.whepPath;
     this.videoEl = opts.videoEl || null;
     this.audioEl = opts.audioEl || null;
     this.onState = opts.onState || (() => {});
+    this.onStats = opts.onStats || (() => {});
     this.pc = null;
     this.resourceURL = "";
+    this.statsSampler = null;
   }
 
   async start() {
@@ -31,6 +36,7 @@ export class WhepClient {
 
     const pc = new RTCPeerConnection({});
     this.pc = pc;
+    this.statsSampler = new WebRTCStatsSampler(pc, this.onStats);
     if (this.videoEl) pc.addTransceiver("video", { direction: "recvonly" });
     if (this.audioEl) pc.addTransceiver("audio", { direction: "recvonly" });
 
@@ -46,6 +52,8 @@ export class WhepClient {
     };
     pc.onconnectionstatechange = () => {
       this.onState(pc.connectionState);
+      if (pc.connectionState === "connected") this.statsSampler?.start();
+      if (["failed", "closed", "disconnected"].includes(pc.connectionState)) this.statsSampler?.stop();
     };
 
     const offer = await pc.createOffer();
@@ -76,6 +84,8 @@ export class WhepClient {
     const pc = this.pc;
     const url = this.resourceURL;
     this.pc = null;
+    this.statsSampler?.stop();
+    this.statsSampler = null;
     this.resourceURL = "";
     if (url) {
       try {

Overlord-Server/public/backstage.html

@@ -83,6 +83,13 @@
             <span class="uppercase tracking-wide text-xs text-slate-500">lat</span>
             <span id="latencyDisplay" class="pill pill-ghost font-mono text-amber-200">--</span>
           </div>
+          <div
+            class="flex items-center gap-2 bg-slate-900/70 border border-slate-800 rounded-lg px-3 py-2 text-sm text-slate-300"
+            title="WebRTC receive bitrate, network RTT, packet loss, and selected route"
+          >
+            <span class="uppercase tracking-wide text-xs text-slate-500">net</span>
+            <span id="networkStats" class="pill pill-ghost font-mono text-cyan-200">--</span>
+          </div>
 
           <div class="relative ml-auto" id="rdSettingsWrap">
             <button

Overlord-Server/public/remotedesktop.html

@@ -109,6 +109,17 @@
             <span class="uppercase tracking-wide text-xs text-slate-500">input</span>
             <span id="inputLatency" class="pill pill-ghost font-mono text-amber-200">-- ms</span>
           </div>
+          <div
+            class="flex items-center gap-2 bg-slate-900/70 border border-slate-800 rounded-lg px-3 py-2 text-sm text-slate-300"
+            title="WebRTC receive bitrate, network RTT, packet loss, and selected route"
+          >
+            <span class="uppercase tracking-wide text-xs text-slate-500">net</span>
+            <span id="networkStats" class="pill pill-ghost font-mono text-cyan-200">--</span>
+          </div>
+
+          <button id="diagnosticsBtn" type="button" class="button ghost" aria-pressed="false" title="Show streaming diagnostics HUD">
+            <i class="fa-solid fa-gauge-high"></i><span>Stats HUD</span>
+          </button>
 
           <div class="relative ml-auto" id="rdSettingsWrap">
             <button
@@ -178,6 +189,19 @@
                   </select>
                 </div>
                 <div id="streamProfileDetail" class="mb-2 text-xs text-slate-500">Checking hardware encoder profiles…</div>
+                <div class="flex items-center gap-2 mb-2">
+                  <label class="text-sm text-slate-300 w-20" for="bitrateSelect">Bitrate</label>
+                  <select id="bitrateSelect" class="flex-1 bg-slate-800 border border-slate-700 rounded px-2 py-1 text-sm text-slate-100" title="H.264 target bitrate; Auto derives it from resolution and FPS">
+                    <option value="0" selected>Auto</option>
+                    <option value="5">5 Mbps</option>
+                    <option value="10">10 Mbps</option>
+                    <option value="15">15 Mbps</option>
+                    <option value="20">20 Mbps</option>
+                    <option value="30">30 Mbps</option>
+                    <option value="40">40 Mbps</option>
+                    <option value="50">50 Mbps</option>
+                  </select>
+                </div>
                 <div class="flex items-center justify-between gap-2">
                   <label class="flex items-center gap-2 text-sm text-slate-200" title="Prefer H.264 when available; disable to force JPEG">
                     <input type="checkbox" id="codecH264" checked /> H.264
@@ -201,7 +225,7 @@
                 </label>
                 <div class="flex items-center gap-2">
                   <label class="text-sm text-slate-300 w-20" for="audioTransport">Via</label>
-                  <select id="audioTransport" class="flex-1 bg-slate-800 border border-slate-700 rounded px-2 py-1 text-sm text-slate-100" title="Audio transport. WS: raw PCM over the existing WebSocket (current default). WebRTC modes use G.711 over the WebRTC track instead.">
+                  <select id="audioTransport" class="flex-1 bg-slate-800 border border-slate-700 rounded px-2 py-1 text-sm text-slate-100" title="Audio transport. WS uses raw PCM over the existing WebSocket. WebRTC modes use full-band Opus.">
                     <option value="off" selected>WS</option>
                     <option value="p2p">WebRTC P2P</option>
                     <option value="relayed">WebRTC Relayed</option>
@@ -249,7 +273,7 @@
             Client: <span id="clientLabel" class="text-slate-100">-</span>
           </div>
           <div
-            class="flex-1 flex items-center justify-center p-4"
+            class="relative flex-1 flex items-center justify-center p-4"
             style="min-height: 0; overflow: hidden"
           >
             <canvas
@@ -279,6 +303,49 @@
                 height: auto;
               "
             ></video>
+            <section id="diagnosticsHud" class="rd-diagnostics-hud hidden" aria-label="Streaming diagnostics">
+              <div class="rd-diagnostics-header">
+                <div>
+                  <div class="rd-diagnostics-title">Streaming diagnostics</div>
+                  <div id="diagSummary" class="rd-diagnostics-summary">Waiting for frames</div>
+                </div>
+                <button id="diagnosticsClose" type="button" class="rd-diagnostics-close" aria-label="Close diagnostics">×</button>
+              </div>
+              <div class="rd-diagnostics-meta">
+                <span id="diagTransport">Canvas</span>
+                <span id="diagCodec">--</span>
+                <span id="diagResolution">--</span>
+              </div>
+              <div class="rd-diagnostics-section">
+                <div class="rd-diagnostics-section-title">Agent pipeline</div>
+                <div class="rd-diagnostics-grid">
+                  <span>Capture</span><strong id="diagCapture">-- ms</strong>
+                  <span>Encode</span><strong id="diagEncode">-- ms</strong>
+                  <span>Packet/write</span><strong id="diagSend">-- ms</strong>
+                  <span>Agent total</span><strong id="diagAgentTotal">-- ms</strong>
+                </div>
+              </div>
+              <div class="rd-diagnostics-section">
+                <div class="rd-diagnostics-section-title">Transport</div>
+                <div class="rd-diagnostics-grid">
+                  <span>Receive rate</span><strong id="diagBitrate">-- Mbps</strong>
+                  <span>Network RTT</span><strong id="diagRtt">-- ms</strong>
+                  <span>Loss / jitter</span><strong id="diagLossJitter">--</strong>
+                  <span>Jitter buffer</span><strong id="diagJitterBuffer">-- ms</strong>
+                </div>
+              </div>
+              <div class="rd-diagnostics-section">
+                <div class="rd-diagnostics-section-title">Viewer pipeline</div>
+                <div class="rd-diagnostics-grid">
+                  <span>Decode latency</span><strong id="diagDecode">-- ms</strong>
+                  <span>Receive → render</span><strong id="diagRender">-- ms</strong>
+                  <span>Decode queue</span><strong id="diagQueue">0</strong>
+                  <span>Dropped / coalesced</span><strong id="diagDropped">0 / 0</strong>
+                  <span>FPS agent → viewer</span><strong id="diagFps">-- → --</strong>
+                  <span>Input RTT</span><strong id="diagInput">-- ms</strong>
+                </div>
+              </div>
+            </section>
           </div>
         </div>
         <style>
@@ -311,6 +378,37 @@
             object-fit: contain;
             margin: auto;
           }
+          .rd-diagnostics-hud {
+            position: absolute;
+            top: 1.25rem;
+            right: 1.25rem;
+            z-index: 30;
+            width: min(360px, calc(100% - 2.5rem));
+            max-height: calc(100% - 2.5rem);
+            overflow-y: auto;
+            padding: 14px;
+            border: 1px solid rgba(71, 85, 105, 0.9);
+            border-radius: 12px;
+            color: #e2e8f0;
+            background: rgba(2, 6, 23, 0.9);
+            box-shadow: 0 18px 45px rgba(0, 0, 0, 0.5);
+            backdrop-filter: blur(12px);
+            font-size: 12px;
+          }
+          .rd-diagnostics-hud.hidden { display: none; }
+          .rd-diagnostics-header, .rd-diagnostics-meta { display: flex; align-items: center; justify-content: space-between; gap: 10px; }
+          .rd-diagnostics-title { color: #f8fafc; font-size: 14px; font-weight: 700; }
+          .rd-diagnostics-summary { margin-top: 2px; color: #67e8f9; }
+          .rd-diagnostics-summary[data-severity="warn"] { color: #fbbf24; }
+          .rd-diagnostics-summary[data-severity="bad"] { color: #fb7185; }
+          .rd-diagnostics-close { border: 0; background: transparent; color: #94a3b8; font-size: 22px; cursor: pointer; }
+          .rd-diagnostics-meta { justify-content: flex-start; flex-wrap: wrap; margin: 10px 0; color: #94a3b8; font-family: ui-monospace, monospace; }
+          .rd-diagnostics-meta span { padding: 2px 6px; border-radius: 999px; background: rgba(30, 41, 59, 0.8); }
+          .rd-diagnostics-section { padding-top: 9px; margin-top: 9px; border-top: 1px solid rgba(51, 65, 85, 0.75); }
+          .rd-diagnostics-section-title { margin-bottom: 6px; color: #64748b; font-size: 10px; font-weight: 700; letter-spacing: .08em; text-transform: uppercase; }
+          .rd-diagnostics-grid { display: grid; grid-template-columns: 1fr auto; gap: 5px 14px; }
+          .rd-diagnostics-grid span { color: #94a3b8; }
+          .rd-diagnostics-grid strong { color: #e2e8f0; font-family: ui-monospace, monospace; font-weight: 600; text-align: right; }
         </style>
       </div>
     </main>

Overlord-Server/src/main-server.ts

@@ -92,6 +92,7 @@ import { createWebSocketRuntime } from "./server/websocket-runtime";
 import {
   handleConsoleOutput,
   handleDesktopEncoderCapabilities,
+  handleDesktopStreamStats,
   handleConsoleViewerMessage,
   handleConsoleViewerOpen,
   handlebackstageViewerMessage,
@@ -656,6 +657,7 @@ async function startServer() {
     handleNotificationScreenshotResult: notificationPluginHandlers.handleNotificationScreenshotResult,
     handleConsoleOutput: (clientId: string, payload: any) => handleConsoleOutput(clientId, payload),
     handleDesktopEncoderCapabilities: (clientId: string, payload: any) => handleDesktopEncoderCapabilities(clientId, payload),
+    handleDesktopStreamStats: (clientId: string, payload: any) => handleDesktopStreamStats(clientId, payload),
     handleFileBrowserMessage: (clientId: string, payload: any) =>
       forwardFileBrowserMessage(clientId, payload, {
         pendingHttpDownloads,

Overlord-Server/src/protocol.ts

@@ -8,6 +8,7 @@ export type MessageKind =
   | "command"
   | "command_result"
   | "desktop_encoder_capabilities"
+  | "desktop_stream_stats"
   | "client_logs_result"
   | "screenshot_result"
   | "frame"
@@ -125,6 +126,7 @@ export type CommandType =
   | "desktop_enable_mouse"
   | "desktop_enable_keyboard"
   | "desktop_set_fps"
+  | "desktop_set_bitrate"
   | "darwin_request_permissions"
   | "webrtc_publish"
   | "webrtc_stop"
@@ -189,6 +191,8 @@ export type FrameHeader = {
   monitor: number;
   fps: number;
   format: "jpeg" | "webp" | "raw" | "h264";
+  width?: number;
+  height?: number;
   hash?: string;
   backstage?: boolean;
   webcam?: boolean;
@@ -196,6 +200,19 @@ export type FrameHeader = {
 
 export type Frame = { type: "frame"; header: FrameHeader; data: Uint8Array };
 export type FrameAck = { type: "frame_ack" };
+export type DesktopStreamStats = {
+  type: "desktop_stream_stats";
+  fps: number;
+  format: string;
+  bytes: number;
+  width: number;
+  height: number;
+  captureMs: number;
+  encodeMs: number;
+  sendMs: number;
+  totalMs: number;
+  transport: "websocket" | "webrtc" | string;
+};
 export type Status = {
   type: "status";
   state: "idle" | "streaming" | "error";
@@ -505,6 +522,7 @@ export type WireMessage =
   | ScreenshotResult
   | Frame
   | FrameAck
+  | DesktopStreamStats
   | Status
   | ConsoleOutput
   | FileListResult

Overlord-Server/src/server/routes/websocket-lifecycle-routes.ts

@@ -222,6 +222,7 @@ type WsLifecycleDeps = {
   handleNotificationScreenshotResult: (clientId: string, payload: any) => void;
   handleConsoleOutput: (clientId: string, payload: any) => void;
   handleDesktopEncoderCapabilities: (clientId: string, payload: any) => void;
+  handleDesktopStreamStats: (clientId: string, payload: any) => void;
   handleFileBrowserMessage: (clientId: string, payload: any) => void;
   handleProxyTunnelData: (clientId: string, connectionId: string, data: Uint8Array) => void;
   handleProxyTunnelClose: (clientId: string, connectionId: string) => void;
@@ -799,6 +800,9 @@ export async function handleWebSocketMessage(
       case "desktop_encoder_capabilities":
         deps.handleDesktopEncoderCapabilities(client.id, payload);
         break;
+      case "desktop_stream_stats":
+        deps.handleDesktopStreamStats(client.id, payload);
+        break;
       case "file_list_result":
       case "file_download":
       case "file_upload_result":

Overlord-Server/src/server/ws-console-rd-backstage.test.ts

@@ -9,6 +9,7 @@ import {
   handlebackstageViewerOpen,
   handleRemoteDesktopViewerMessage,
   handleRemoteDesktopViewerOpen,
+  handleDesktopStreamStats,
   backstageStreamingState,
   rdStreamingState,
 } from "./ws-console-rd-backstage";
@@ -143,6 +144,7 @@ describe("remote desktop viewer control", () => {
       duplication: true,
       maxHeight: 1080,
       maxFps: 120,
+      bitrateMbps: 0,
       lastFps: 1,
       lastFrameAt: 0,
       startedAt: Date.now() - 5000,
@@ -156,6 +158,78 @@ describe("remote desktop viewer control", () => {
     expect(commands.filter((msg) => msg.commandType === "desktop_request_keyframe")).toHaveLength(0);
     expect(rdStreamingState.get(clientId)?.isStreaming).toBe(true);
   });
+
+  test("forwards and clamps the desktop bitrate setting", () => {
+    const clientId = `rd-bitrate-${Date.now().toString(36)}`;
+    const { agentWs } = createClient(clientId);
+    const viewer = createMockWs({ clientId });
+
+    handleRemoteDesktopViewerOpen(viewer as any);
+    handleRemoteDesktopViewerMessage(viewer as any, JSON.stringify({
+      type: "desktop_set_bitrate",
+      bitrateMbps: 75,
+    }));
+
+    const command = agentCommands(agentWs).find((msg) => msg.commandType === "desktop_set_bitrate");
+    expect(command?.payload?.bitrateMbps).toBe(50);
+    expect(rdStreamingState.get(clientId)?.bitrateMbps).toBe(50);
+  });
+
+  test("forwards agent pipeline telemetry to every remote desktop viewer", () => {
+    const clientId = `rd-stats-${Date.now().toString(36)}`;
+    createClient(clientId);
+    const viewer = createMockWs({ clientId });
+    handleRemoteDesktopViewerOpen(viewer as any);
+
+    handleDesktopStreamStats(clientId, {
+      type: "desktop_stream_stats",
+      fps: 60,
+      format: "h264",
+      captureMs: 2.25,
+      encodeMs: 4.5,
+      sendMs: 0.4,
+      totalMs: 7.3,
+      transport: "webrtc",
+    });
+
+    const message = decodeMessage(viewer.sent.at(-1) as Uint8Array) as any;
+    expect(message.type).toBe("desktop_stream_stats");
+    expect(message.captureMs).toBe(2.25);
+    expect(message.encodeMs).toBe(4.5);
+    expect(message.transport).toBe("webrtc");
+  });
+
+  test("holds Canvas acknowledgements only while the browser decoder is under pressure", () => {
+    const clientId = `rd-flow-${Date.now().toString(36)}`;
+    const { agentWs } = createClient(clientId);
+    const viewer = createMockWs({ clientId });
+    handleRemoteDesktopViewerOpen(viewer as any);
+    handleRemoteDesktopViewerMessage(viewer as any, JSON.stringify({
+      type: "desktop_start",
+      canvasFlowControl: true,
+    }));
+
+    const healthyAck = (globalThis as any).__rdBroadcast(
+      clientId,
+      new Uint8Array([1, 2, 3]),
+      { format: "h264", fps: 60, width: 2560, height: 1440 },
+    );
+    expect(healthyAck).toBe(true);
+    const frame = viewer.sent.at(-1) as Uint8Array;
+    expect(frame[3]).toBe(2);
+    expect(frame.byteLength).toBe(15);
+
+    handleRemoteDesktopViewerMessage(viewer as any, JSON.stringify({ type: "desktop_decode_pressure", active: true }));
+    const pressuredAck = (globalThis as any).__rdBroadcast(
+      clientId,
+      new Uint8Array([4, 5, 6]),
+      { format: "h264", fps: 60, width: 2560, height: 1440 },
+    );
+    expect(pressuredAck).toBe(false);
+
+    handleRemoteDesktopViewerMessage(viewer as any, JSON.stringify({ type: "desktop_decode_pressure", active: false }));
+    expect(agentCommands(agentWs).filter((msg) => msg.type === "frame_ack")).toHaveLength(1);
+  });
 });
 
 describe("backstage viewer control", () => {

Overlord-Server/src/server/ws-console-rd-backstage.ts

@@ -175,6 +175,7 @@ function broadcastFrameToViewers(
 
 const rdSendStats = { lastLog: 0, frames: 0, sendMs: 0, bytes: 0 };
 const rdDebugFrameLogAt = new Map<string, number>();
+const rdCanvasFrameAckPending = new Map<string, string>();
 export const rdStreamingState = new Map<string, {
   isStreaming: boolean;
   display: number;
@@ -184,6 +185,7 @@ export const rdStreamingState = new Map<string, {
   duplication: boolean;
   maxHeight: number;
   maxFps: number;
+  bitrateMbps: number;
   lastFps: number;
   lastFrameAt: number;
   startedAt: number;
@@ -209,6 +211,7 @@ function defaultRdStreamingState() {
     duplication: false,
     maxHeight: 0,
     maxFps: 120,
+    bitrateMbps: 0,
     lastFps: 0,
     lastFrameAt: 0,
     startedAt: 0,
@@ -366,7 +369,9 @@ export function handleRemoteDesktopViewerMessage(ws: ServerWebSocket<SocketData>
 
   const state = rdStreamingState.get(clientId) || defaultRdStreamingState();
 
+  if (payload.type !== "desktop_decode_pressure") {
     logger.debug(`[rd] inbound viewer msg type=${payload.type} client=${clientId}`);
+  }
   switch (payload.type) {
     case "desktop_encoder_capabilities":
       sendDesktopCommand(target, "desktop_encoder_capabilities", {
@@ -374,6 +379,9 @@ export function handleRemoteDesktopViewerMessage(ws: ServerWebSocket<SocketData>
       });
       break;
     case "desktop_start":
+      (ws.data as any).rdCanvasFlowControl = (payload as any).canvasFlowControl === true && (payload as any).webrtc !== true;
+      (ws.data as any).rdCanvasBackpressure = false;
+      releaseCanvasFrameAck(clientId);
       logger.debug(`[rd-debug] desktop_start requested client=${clientId} session=${ws.data.sessionId || ""} state=${JSON.stringify(state)} viewers=${sessionManager.getRdSessionsForClient(clientId).length} webrtc=${(payload as any).webrtc === true}`);
       if (!state.isStreaming) {
         const targetOs = String(target.os || "").toLowerCase();
@@ -412,6 +420,7 @@ export function handleRemoteDesktopViewerMessage(ws: ServerWebSocket<SocketData>
         }
         safeSendViewer(ws, { type: "status", status: "starting" });
         sendDesktopCommand(target, "desktop_set_fps", { fps: clampDesktopFps(state.maxFps) });
+        sendDesktopCommand(target, "desktop_set_bitrate", { bitrateMbps: state.bitrateMbps || 0 });
         sendDesktopCommand(target, "desktop_start", {});
         state.isStreaming = true;
         state.startedAt = Date.now();
@@ -423,6 +432,7 @@ export function handleRemoteDesktopViewerMessage(ws: ServerWebSocket<SocketData>
         if (lastFrameAgeMs > 3000 && startAgeMs > 3000) {
           logger.info(`[rd-debug] desktop_start reasserting stale stream client=${clientId} lastFrameAgeMs=${Number.isFinite(lastFrameAgeMs) ? lastFrameAgeMs : -1} state=${JSON.stringify(state)} viewers=${sessionManager.getRdSessionsForClient(clientId).length}`);
           sendDesktopCommand(target, "desktop_set_fps", { fps: clampDesktopFps(state.maxFps) });
+          sendDesktopCommand(target, "desktop_set_bitrate", { bitrateMbps: state.bitrateMbps || 0 });
           sendDesktopCommand(target, "desktop_start", {});
           safeSendViewer(ws, { type: "status", status: "starting" });
           state.isStreaming = true;
@@ -439,6 +449,9 @@ export function handleRemoteDesktopViewerMessage(ws: ServerWebSocket<SocketData>
       }
       break;
     case "desktop_stop": {
+      (ws.data as any).rdCanvasFlowControl = false;
+      (ws.data as any).rdCanvasBackpressure = false;
+      releaseCanvasFrameAck(clientId);
       const otherViewers = sessionManager.getRdSessionsForClient(clientId)
         .filter(s => s.id !== ws.data.sessionId);
       logger.debug(`[rd-debug] desktop_stop requested client=${clientId} session=${ws.data.sessionId || ""} otherViewers=${otherViewers.length} state=${JSON.stringify(state)}`);
@@ -581,6 +594,23 @@ export function handleRemoteDesktopViewerMessage(ws: ServerWebSocket<SocketData>
       }
       break;
     }
+    case "desktop_decode_pressure": {
+      if (!(ws.data as any).rdCanvasFlowControl) break;
+      const active = (payload as any).active === true;
+      (ws.data as any).rdCanvasBackpressure = active;
+      if (!active) releaseCanvasFrameAck(clientId, ws.data.sessionId);
+      break;
+    }
+    case "desktop_set_bitrate": {
+      const newBitrateMbps = Math.max(0, Math.min(50, Math.floor(Number((payload as any).bitrateMbps) || 0)));
+      if (state.bitrateMbps !== newBitrateMbps) {
+        sendDesktopCommand(target, "desktop_set_bitrate", { bitrateMbps: newBitrateMbps });
+        state.bitrateMbps = newBitrateMbps;
+        rdStreamingState.set(clientId, state);
+        logger.debug(`[rd] set target bitrate=${newBitrateMbps || "auto"} Mbps`);
+      }
+      break;
+    }
     case "desktop_set_duplication": {
       const enabled = !!payload.enabled;
       if (state.duplication !== enabled) {
@@ -732,6 +762,11 @@ export function handleWebrtcP2PIce(_clientId: string, payload: any) {
 }
 
 export function cleanupViewerP2P(ws: ServerWebSocket<SocketData>) {
+  if (ws.data.role === "rd_viewer") {
+    (ws.data as any).rdCanvasFlowControl = false;
+    (ws.data as any).rdCanvasBackpressure = false;
+    releaseCanvasFrameAck(ws.data.clientId, ws.data.sessionId);
+  }
   const cleared = clearP2PSessionForViewer(ws);
   if (!cleared) return;
   const target = clientManager.getClient(cleared.clientId);
@@ -776,6 +811,15 @@ function broadcastRemoteDesktopFrame(clientId: string, bytes: Uint8Array, header
   const buf = buildViewerFrameBuffer(bytes, header);
   const sessions = sessionManager.getRdSessionsForClient(clientId);
   const result = broadcastFrameToViewers(sessions, buf, header);
+  if (result.sent) {
+    const controller = sessions.find((session) =>
+      (session.viewer.data as any).rdCanvasFlowControl === true &&
+      (session.viewer.data as any).rdCanvasBackpressure === true
+    );
+    if (controller?.viewer.data.sessionId) {
+      rdCanvasFrameAckPending.set(clientId, controller.viewer.data.sessionId);
+    }
+  }
   if (result.dropped) {
     const target = clientManager.getClient(clientId);
     if (target) {
@@ -785,7 +829,7 @@ function broadcastRemoteDesktopFrame(clientId: string, bytes: Uint8Array, header
       });
     }
   }
-  return result.sent || result.viewers === 0;
+  return (result.sent && !rdCanvasFrameAckPending.has(clientId)) || result.viewers === 0;
 }
 
 (globalThis as any).__rdBroadcast = (clientId: string, bytes: Uint8Array, header?: any): boolean => {
@@ -1448,6 +1492,40 @@ export function handleDesktopEncoderCapabilities(clientId: string, payload: any)
   }
 }
 
+function releaseCanvasFrameAck(clientId: string, sessionId?: string): boolean {
+  const controllerSessionId = rdCanvasFrameAckPending.get(clientId);
+  if (!controllerSessionId || (sessionId && controllerSessionId !== sessionId)) return false;
+  rdCanvasFrameAckPending.delete(clientId);
+  const target = clientManager.getClient(clientId);
+  if (!target) return false;
+  try {
+    target.ws.send(encodeMessage({ type: "frame_ack" }));
+    return true;
+  } catch (err) {
+    logger.debug(`[rd] canvas frame ack failed client=${clientId}: ${(err as Error).message}`);
+    return false;
+  }
+}
+
+export function handleDesktopStreamStats(clientId: string, payload: any) {
+  const stats = {
+    type: "desktop_stream_stats",
+    fps: Math.max(0, Number(payload?.fps) || 0),
+    format: String(payload?.format || ""),
+    bytes: Math.max(0, Number(payload?.bytes) || 0),
+    width: Math.max(0, Number(payload?.width) || 0),
+    height: Math.max(0, Number(payload?.height) || 0),
+    captureMs: Math.max(0, Number(payload?.captureMs) || 0),
+    encodeMs: Math.max(0, Number(payload?.encodeMs) || 0),
+    sendMs: Math.max(0, Number(payload?.sendMs) || 0),
+    totalMs: Math.max(0, Number(payload?.totalMs) || 0),
+    transport: String(payload?.transport || ""),
+  };
+  for (const session of sessionManager.getRdSessionsForClient(clientId)) {
+    safeSendViewer(session.viewer, stats, "rd-stats");
+  }
+}
+
 (globalThis as any).__backstageBroadcast = (clientId: string, bytes: Uint8Array, header?: any): boolean => {
   const state = backstageStreamingState.get(clientId) || defaultbackstageStreamingState();
   if (!state.isStreaming) {

Overlord-Server/src/server/ws-viewer-utils.ts

@@ -33,11 +33,14 @@ export function safeSendViewer(
 }
 
 export function buildViewerFrameBuffer(bytes: Uint8Array, header?: any): Uint8Array {
-  const meta = new Uint8Array(8);
+  const width = Math.max(0, Math.min(65535, Math.floor(Number(header?.width) || 0)));
+  const height = Math.max(0, Math.min(65535, Math.floor(Number(header?.height) || 0)));
+  const metaLength = width > 0 && height > 0 ? 12 : 8;
+  const meta = new Uint8Array(metaLength);
   meta[0] = 0x46;
   meta[1] = 0x52;
   meta[2] = 0x4d;
-  meta[3] = 1;
+  meta[3] = metaLength === 12 ? 2 : 1;
   meta[4] = (header?.monitor ?? 0) & 0xff;
   meta[5] = (header?.fps ?? 0) & 0xff;
   const fmt = header?.format === "blocks"
@@ -49,10 +52,15 @@ export function buildViewerFrameBuffer(bytes: Uint8Array, header?: any): Uint8Ar
     : 1;
   meta[6] = fmt;
   meta[7] = 0;
+  if (metaLength === 12) {
+    const view = new DataView(meta.buffer);
+    view.setUint16(8, width, true);
+    view.setUint16(10, height, true);
+  }
 
-  const buf = new Uint8Array(8 + bytes.length);
+  const buf = new Uint8Array(metaLength + bytes.length);
   buf.set(meta, 0);
-  buf.set(bytes, 8);
+  buf.set(bytes, metaLength);
   return buf;
 }
 

Overlord-Server/src/wsValidation.ts

@@ -21,6 +21,7 @@ export const ALLOWED_CLIENT_MESSAGE_TYPES = new Set([
   "file_hash_result",
   "command_result",
   "desktop_encoder_capabilities",
+  "desktop_stream_stats",
   "client_logs_result",
   "screenshot_result",
   "command_progress",