major remote desktop upgrades + do opus. Hopefully all the tests pass 😭

Overlord-Client/cmd/agent/audio/voice_native_cgo.go

@@ -18,6 +18,8 @@ import (
 const (
 	voiceSampleRate         = 16000
 	voiceChannels           = 1
+	desktopAudioSampleRate  = 48000
+	desktopAudioChannels    = 2
 	voiceBytesPerSample     = 2
 	maxPlaybackBufferMillis = 250
 )
@@ -390,15 +392,15 @@ func StartCaptureOnlySession(parent context.Context, source string, onCapture fu
 	}
 
 	capCfg := malgo.DefaultDeviceConfig(malgo.Capture)
-	capCfg.SampleRate = voiceSampleRate
+	capCfg.SampleRate = desktopAudioSampleRate
 	capCfg.Capture.Format = malgo.FormatS16
-	capCfg.Capture.Channels = voiceChannels
+	capCfg.Capture.Channels = desktopAudioChannels
 	useLoopback := false
 	if strings.EqualFold(source, "system") {
 		capCfg = malgo.DefaultDeviceConfig(malgo.Loopback)
-		capCfg.SampleRate = voiceSampleRate
+		capCfg.SampleRate = desktopAudioSampleRate
 		capCfg.Capture.Format = malgo.FormatS16
-		capCfg.Capture.Channels = voiceChannels
+		capCfg.Capture.Channels = desktopAudioChannels
 		useLoopback = true
 	} else if strings.HasPrefix(strings.ToLower(source), "device:") {
 		deviceName := strings.TrimSpace(source[len("device:"):])
@@ -442,9 +444,9 @@ func StartCaptureOnlySession(parent context.Context, source string, onCapture fu
 	capDev, err := malgo.InitDevice(audioCtx.Context, capCfg, capCallbacks)
 	if err != nil && useLoopback {
 		capCfg = malgo.DefaultDeviceConfig(malgo.Capture)
-		capCfg.SampleRate = voiceSampleRate
+		capCfg.SampleRate = desktopAudioSampleRate
 		capCfg.Capture.Format = malgo.FormatS16
-		capCfg.Capture.Channels = voiceChannels
+		capCfg.Capture.Channels = desktopAudioChannels
 		deviceID, _, ok := findSystemCaptureDevice(audioCtx.Context)
 		if !ok || deviceID == nil {
 			audioCtx.Free()

Overlord-Client/cmd/agent/capture/capture.go

@@ -154,6 +154,8 @@ func CaptureAndSend(ctx context.Context, env *rt.Env) error {
 		slotAcquired = true
 	}
 
+	frameWidth := img.Bounds().Dx()
+	frameHeight := img.Bounds().Dy()
 	quality := jpegQuality()
 	frame, encodeDur, err := buildFrame(img, display, quality)
 	PutRGBA(img)
@@ -164,6 +166,8 @@ func CaptureAndSend(ctx context.Context, env *rt.Env) error {
 		}
 		return err
 	}
+	frame.Header.Width = frameWidth
+	frame.Header.Height = frameHeight
 	now := time.Now()
 	fps := frameFPS(now)
 	if fps <= 0 {
@@ -181,17 +185,21 @@ func CaptureAndSend(ctx context.Context, env *rt.Env) error {
 		if dur <= 0 {
 			dur = 33 * time.Millisecond
 		}
+		sendStart := time.Now()
 		if werr := webrtcpub.WriteH264(webrtcpub.KindDesktop, frame.Data, dur); werr != nil {
 			log.Printf("webrtc: write h264 failed: %v", werr)
 		}
+		sendDur := time.Since(sendStart)
 		if slotAcquired {
 			ReleaseFrameSlot()
 		}
 		statFrames.Add(1)
 		statCapNs.Add(captureDur.Nanoseconds())
 		statEncNs.Add(encodeDur.Nanoseconds())
-		statTotalNs.Add(time.Since(t0).Nanoseconds())
+		totalDur := time.Since(t0)
+		statTotalNs.Add(totalDur.Nanoseconds())
 		statBytes.Add(int64(len(frame.Data)))
+		emitDesktopStreamStats(ctx, env, frame, fps, captureDur, encodeDur, sendDur, totalDur, "webrtc")
 		return nil
 	}
 	if !slotAcquired {
@@ -233,6 +241,7 @@ func CaptureAndSend(ctx context.Context, env *rt.Env) error {
 	statSendNs.Add(sendDur.Nanoseconds())
 	statTotalNs.Add(time.Since(t0).Nanoseconds())
 	statBytes.Add(int64(len(frame.Data)))
+	emitDesktopStreamStats(ctx, env, frame, fps, captureDur, encodeDur, sendDur, time.Since(t0), "websocket")
 	return err
 }
 
@@ -251,14 +260,18 @@ func sendCompletedFrame(ctx context.Context, env *rt.Env, frame wire.Frame, disp
 		if dur <= 0 {
 			dur = 33 * time.Millisecond
 		}
+		sendStart := time.Now()
 		if werr := webrtcpub.WriteH264(webrtcpub.KindDesktop, frame.Data, dur); werr != nil {
 			log.Printf("webrtc: write h264 failed: %v", werr)
 		}
+		sendDur := time.Since(sendStart)
 		statFrames.Add(1)
 		statCapNs.Add(captureDur.Nanoseconds())
 		statEncNs.Add(encodeDur.Nanoseconds())
-		statTotalNs.Add(time.Since(t0).Nanoseconds())
+		totalDur := time.Since(t0)
+		statTotalNs.Add(totalDur.Nanoseconds())
 		statBytes.Add(int64(len(frame.Data)))
+		emitDesktopStreamStats(ctx, env, frame, fps, captureDur, encodeDur, sendDur, totalDur, "webrtc")
 		return nil
 	}
 	if !AcquireFrameSlot() {
@@ -301,6 +314,7 @@ func sendCompletedFrame(ctx context.Context, env *rt.Env, frame wire.Frame, disp
 	statSendNs.Add(sendDur.Nanoseconds())
 	statTotalNs.Add(time.Since(t0).Nanoseconds())
 	statBytes.Add(int64(len(frame.Data)))
+	emitDesktopStreamStats(ctx, env, frame, fps, captureDur, encodeDur, sendDur, time.Since(t0), "websocket")
 	return nil
 }
 

Overlord-Client/cmd/agent/capture/capture_test.go

@@ -44,6 +44,7 @@ func TestCaptureAndSend_NoDisplays(t *testing.T) {
 }
 
 func TestCaptureAndSend_SendsFrame(t *testing.T) {
+	previousStatsAt := lastDesktopStreamStatsMs.Swap(0)
 	originalCount := activeDisplays
 	originalCapture := captureDisplayFn
 	activeDisplays = func() int { return 1 }
@@ -56,6 +57,7 @@ func TestCaptureAndSend_SendsFrame(t *testing.T) {
 	t.Cleanup(func() {
 		activeDisplays = originalCount
 		captureDisplayFn = originalCapture
+		lastDesktopStreamStatsMs.Store(previousStatsAt)
 	})
 
 	writer := &recordingWriter{}
@@ -63,8 +65,8 @@ func TestCaptureAndSend_SendsFrame(t *testing.T) {
 	if err := CaptureAndSend(context.Background(), env); err != nil {
 		t.Fatalf("CaptureAndSend returned error: %v", err)
 	}
-	if len(writer.msgs) != 1 {
-		t.Fatalf("expected one message written, got %d", len(writer.msgs))
+	if len(writer.msgs) != 2 {
+		t.Fatalf("expected frame and telemetry messages, got %d", len(writer.msgs))
 	}
 
 	var frame wire.Frame
@@ -84,6 +86,17 @@ func TestCaptureAndSend_SendsFrame(t *testing.T) {
 	if _, err := jpeg.Decode(bytes.NewReader(frame.Data)); err != nil {
 		t.Fatalf("jpeg payload did not decode: %v", err)
 	}
+
+	var stats wire.DesktopStreamStats
+	if err := msgpack.Unmarshal(writer.msgs[1], &stats); err != nil {
+		t.Fatalf("decode desktop stream stats: %v", err)
+	}
+	if stats.Type != "desktop_stream_stats" || stats.Format != "jpeg" {
+		t.Fatalf("unexpected stream stats: %#v", stats)
+	}
+	if stats.CaptureMs < 0 || stats.EncodeMs < 0 || stats.TotalMs < 0 {
+		t.Fatalf("stream stats contained a negative duration: %#v", stats)
+	}
 }
 
 func TestActualScreenCapture(t *testing.T) {
@@ -306,8 +319,13 @@ func TestCaptureAndSend_DisplayOutOfRange(t *testing.T) {
 				t.Errorf("expected capture from display %d, got %d", tt.expectedDisplay, capturedDisplay)
 			}
 
-			if len(writer.msgs) != 1 {
-				t.Errorf("expected 1 message, got %d", len(writer.msgs))
+			if len(writer.msgs) == 0 {
+				t.Error("expected a frame message")
+			} else {
+				var frame wire.Frame
+				if err := msgpack.Unmarshal(writer.msgs[0], &frame); err != nil || frame.Type != "frame" {
+					t.Errorf("first message was not a valid frame: type=%q err=%v", frame.Type, err)
+				}
 			}
 		})
 	}

Overlord-Client/cmd/agent/capture/h264_bitrate.go

@@ -0,0 +1,66 @@
+package capture
+
+import (
+	"math"
+	"sync/atomic"
+)
+
+const maxH264Bitrate = 50_000_000
+
+var h264TargetBitrate atomic.Int64
+
+func SetH264TargetBitrate(bps int) int {
+	if bps < 0 {
+		bps = 0
+	}
+	if bps > maxH264Bitrate {
+		bps = maxH264Bitrate
+	}
+	previous := h264TargetBitrate.Swap(int64(bps))
+	if previous != int64(bps) {
+		resetH264Encoder()
+		resetH264Encoderbackstage()
+		resetH264TextureEncoderForBitrate()
+		RequestDesktopFullFrame()
+	}
+	return bps
+}
+
+func configuredH264Bitrate() int {
+	return int(h264TargetBitrate.Load())
+}
+
+func automaticH264Bitrate(width, height, fps int) int {
+	pixelsPerSecond := float64(width * height * fps)
+	bitrate := int(pixelsPerSecond * 0.08)
+	if bitrate < 1_500_000 {
+		return 1_500_000
+	}
+	if bitrate > 18_000_000 {
+		return 18_000_000
+	}
+	return bitrate
+}
+
+func targetH264Bitrate(width, height, fps int) int {
+	if configured := configuredH264Bitrate(); configured > 0 {
+		return configured
+	}
+	return automaticH264Bitrate(width, height, fps)
+}
+
+func targetH264CRF(width, height, fps int) float32 {
+	configured := configuredH264Bitrate()
+	if configured <= 0 {
+		return 23
+	}
+	ratio := float64(configured) / float64(automaticH264Bitrate(width, height, fps))
+	crf := 23 - 6*math.Log2(ratio)
+	if crf < 12 {
+		crf = 12
+	}
+	if crf > 35 {
+		crf = 35
+	}
+	return float32(crf)
+}

Overlord-Client/cmd/agent/capture/h264_bitrate_reset_other.go

@@ -0,0 +1,5 @@
+//go:build !windows
+
+package capture
+
+func resetH264TextureEncoderForBitrate() {}

Overlord-Client/cmd/agent/capture/h264_bitrate_reset_windows.go

@@ -0,0 +1,7 @@
+//go:build windows
+
+package capture
+
+func resetH264TextureEncoderForBitrate() {
+	resetH264D3D11TextureEncoder()
+}

Overlord-Client/cmd/agent/capture/h264_bitrate_test.go

@@ -0,0 +1,38 @@
+package capture
+
+import "testing"
+
+func TestH264TargetBitrateAutoAndManual(t *testing.T) {
+	original := configuredH264Bitrate()
+	defer SetH264TargetBitrate(original)
+
+	SetH264TargetBitrate(0)
+	if got := targetH264Bitrate(1920, 1080, 60); got != 9_953_280 {
+		t.Fatalf("automatic 1080p60 bitrate = %d, want 9953280", got)
+	}
+	if got := targetH264Bitrate(3840, 2160, 240); got != 18_000_000 {
+		t.Fatalf("automatic bitrate cap = %d, want 18000000", got)
+	}
+
+	if got := SetH264TargetBitrate(30_000_000); got != 30_000_000 {
+		t.Fatalf("manual bitrate = %d, want 30000000", got)
+	}
+	if got := targetH264Bitrate(1280, 720, 30); got != 30_000_000 {
+		t.Fatalf("manual target = %d, want 30000000", got)
+	}
+	if got := SetH264TargetBitrate(60_000_000); got != maxH264Bitrate {
+		t.Fatalf("manual cap = %d, want %d", got, maxH264Bitrate)
+	}
+}
+
+func TestH264TargetCRFTracksManualRate(t *testing.T) {
+	original := configuredH264Bitrate()
+	defer SetH264TargetBitrate(original)
+	SetH264TargetBitrate(0)
+	auto := targetH264CRF(1920, 1080, 60)
+	SetH264TargetBitrate(20_000_000)
+	higherRate := targetH264CRF(1920, 1080, 60)
+	if higherRate >= auto {
+		t.Fatalf("higher bitrate CRF = %.2f, want below automatic %.2f", higherRate, auto)
+	}
+}

Overlord-Client/cmd/agent/capture/h264_direct_windows.go

@@ -14,7 +14,7 @@ func tryBuildDirectH264Frame(display int) (wire.Frame, time.Duration, time.Durat
 		return wire.Frame{}, 0, 0, false, nil
 	}
 	forceKeyframe := webrtcpub.ConsumeKeyframeRequest()
-	data, _, _, captureDur, encodeDur, used, err := captureDisplayDXGIH264(display, forceKeyframe)
+	data, width, height, captureDur, encodeDur, used, err := captureDisplayDXGIH264(display, forceKeyframe)
 	if !used || err != nil {
 		return wire.Frame{}, captureDur, encodeDur, used, err
 	}
@@ -24,5 +24,5 @@ func tryBuildDirectH264Frame(display int) (wire.Frame, time.Duration, time.Durat
 	now := time.Now()
 	lastKeyframe.Store(now.UnixNano())
 	statFullFrames.Add(1)
-	return wire.Frame{Type: "frame", Header: wire.FrameHeader{Monitor: display, FPS: 0, Format: "h264"}, Data: data}, captureDur, encodeDur, true, nil
+	return wire.Frame{Type: "frame", Header: wire.FrameHeader{Monitor: display, FPS: 0, Format: "h264", Width: width, Height: height}, Data: data}, captureDur, encodeDur, true, nil
 }

Overlord-Client/cmd/agent/capture/h264_encoder_windows.go

@@ -1296,12 +1296,6 @@ func failedHR(hr uintptr) bool {
 	return hr&0x80000000 != 0
 }
 
-func targetH264Bitrate(width, height, fps int) int {
-	pixelsPerSecond := float64(width * height * fps)
-	bitrate := int(pixelsPerSecond * 0.08)
-	return clampInt(bitrate, 1_500_000, 18_000_000)
-}
-
 func h264LevelFor(width, height, fps int) int {
 	mbsPerFrame := ((width + 15) / 16) * ((height + 15) / 16)
 	mbsPerSecond := mbsPerFrame * fps

Overlord-Client/cmd/agent/capture/h264_encoder_x264.go

@@ -104,7 +104,7 @@ func ensureH264EncoderLocked(width, height int) error {
 		Preset:       "veryfast",
 		Profile:      "main",
 		RateControl:  "crf",
-		RateConstant: 23,
+		RateConstant: targetH264CRF(width, height, fps),
 		LogLevel:     x264.LogError,
 	}
 
@@ -201,7 +201,7 @@ func ensurebackstageH264EncoderLocked(width, height int) error {
 		Preset:       "veryfast",
 		Profile:      "main",
 		RateControl:  "crf",
-		RateConstant: 23,
+		RateConstant: targetH264CRF(width, height, fps),
 		LogLevel:     x264.LogError,
 	}
 	enc, err := x264.NewEncoder(&backstageH264Buf, opts)
@@ -280,7 +280,7 @@ func ensureWebcamH264EncoderLocked(width, height int) error {
 		Preset:       "veryfast",
 		Profile:      "main",
 		RateControl:  "crf",
-		RateConstant: 23,
+		RateConstant: targetH264CRF(width, height, fps),
 		LogLevel:     x264.LogError,
 	}
 	enc, err := x264.NewEncoder(&webcamH264Buf, opts)

Overlord-Client/cmd/agent/capture/stream_stats.go

@@ -0,0 +1,37 @@
+package capture
+
+import (
+	"context"
+	"sync/atomic"
+	"time"
+
+	rt "overlord-client/cmd/agent/runtime"
+	"overlord-client/cmd/agent/wire"
+)
+
+var lastDesktopStreamStatsMs atomic.Int64
+
+func emitDesktopStreamStats(ctx context.Context, env *rt.Env, frame wire.Frame, fps int, captureDur, encodeDur, sendDur, totalDur time.Duration, transport string) {
+	now := time.Now().UnixMilli()
+	previous := lastDesktopStreamStatsMs.Load()
+	if now-previous < 500 || !lastDesktopStreamStatsMs.CompareAndSwap(previous, now) {
+		return
+	}
+	_ = wire.WriteMsg(ctx, env.Conn, wire.DesktopStreamStats{
+		Type:      "desktop_stream_stats",
+		FPS:       fps,
+		Format:    frame.Header.Format,
+		Bytes:     len(frame.Data),
+		Width:     frame.Header.Width,
+		Height:    frame.Header.Height,
+		CaptureMs: durationMs(captureDur),
+		EncodeMs:  durationMs(encodeDur),
+		SendMs:    durationMs(sendDur),
+		TotalMs:   durationMs(totalDur),
+		Transport: transport,
+	})
+}
+
+func durationMs(value time.Duration) float64 {
+	return float64(value.Microseconds()) / 1000
+}

Overlord-Client/cmd/agent/handlers/command.go

@@ -503,6 +503,7 @@ func startDesktopAudioSession(ctx context.Context, env *runtime.Env, sessionID s
 	stopDesktopAudioSession()
 
 	vCtx, cancel := context.WithCancel(ctx)
+	legacyConverter := &desktopAudioLegacyConverter{}
 	session, err := audio.StartCaptureOnlySession(vCtx, source, func(chunk []byte) {
 		if len(chunk) == 0 {
 			return
@@ -513,10 +514,14 @@ func startDesktopAudioSession(ctx context.Context, env *runtime.Env, sessionID s
 			samples := pcm16BytesToInt16(chunk)
 			_ = webrtcpub.WriteAudio(webrtcpub.KindAudio, samples)
 		}
+		legacyPCM := legacyConverter.Convert(chunk)
+		if len(legacyPCM) == 0 {
+			return
+		}
 		msg := map[string]interface{}{
 			"type":      "desktop_audio_uplink",
 			"sessionId": sessionID,
-			"data":      chunk,
+			"data":      legacyPCM,
 		}
 		_ = wire.WriteMsg(vCtx, env.Conn, msg)
 	})
@@ -904,6 +909,24 @@ func HandleCommand(ctx context.Context, env *runtime.Env, envelope map[string]in
 		log.Printf("desktop: set target fps=%d", fps)
 		sendCommandResultSafe(env, cmdID, true, "")
 		return nil
+	case "desktop_set_bitrate":
+		payload, _ := envelope["payload"].(map[string]interface{})
+		bitrateMbps := 0
+		if payload != nil {
+			if v, ok := payloadInt(payload, "bitrateMbps"); ok {
+				bitrateMbps = v
+			}
+		}
+		if bitrateMbps < 0 {
+			bitrateMbps = 0
+		}
+		if bitrateMbps > 50 {
+			bitrateMbps = 50
+		}
+		bps := capture.SetH264TargetBitrate(bitrateMbps * 1_000_000)
+		log.Printf("desktop: set target bitrate=%d Mbps (0=auto)", bps/1_000_000)
+		sendCommandResultSafe(env, cmdID, true, "")
+		return nil
 	case "clipboard_sync_start":
 		payload, _ := envelope["payload"].(map[string]interface{})
 		source := "rd"

Overlord-Client/cmd/agent/handlers/webrtc.go

@@ -23,6 +23,43 @@ func pcm16BytesToInt16(chunk []byte) []int16 {
 	return out
 }
 
+// WebRTC desktop audio is captured at 48 kHz stereo. Keep the established raw
+// WebSocket fallback wire format at 16 kHz mono by averaging each stereo frame
+// and decimating groups of three frames. pending preserves sample groups when
+// an audio callback ends between frames or decimation windows.
+type desktopAudioLegacyConverter struct {
+	pending []byte
+}
+
+func (c *desktopAudioLegacyConverter) Convert(chunk []byte) []byte {
+	if len(chunk) == 0 {
+		return nil
+	}
+	data := make([]byte, 0, len(c.pending)+len(chunk))
+	data = append(data, c.pending...)
+	data = append(data, chunk...)
+	const windowBytes = 3 * 2 * 2 // three stereo PCM16 frames
+	windows := len(data) / windowBytes
+	if windows == 0 {
+		c.pending = append(c.pending[:0], data...)
+		return nil
+	}
+	out := make([]byte, 0, windows*2)
+	for window := 0; window < windows; window++ {
+		var sum int64
+		for frame := 0; frame < 3; frame++ {
+			offset := window*windowBytes + frame*4
+			left := int32(int16(binary.LittleEndian.Uint16(data[offset:])))
+			right := int32(int16(binary.LittleEndian.Uint16(data[offset+2:])))
+			sum += int64(left+right) / 2
+		}
+		sample := int16(sum / 3)
+		out = binary.LittleEndian.AppendUint16(out, uint16(sample))
+	}
+	c.pending = append(c.pending[:0], data[windows*windowBytes:]...)
+	return out
+}
+
 func kindFromPayload(payload map[string]interface{}) webrtcpub.Kind {
 	switch s, _ := payload["kind"].(string); s {
 	case "backstage":

Overlord-Client/cmd/agent/handlers/webrtc_audio_test.go

@@ -0,0 +1,27 @@
+package handlers
+
+import (
+	"encoding/binary"
+	"testing"
+)
+
+func TestDesktopAudioLegacyPCMDownmixesAndDecimates(t *testing.T) {
+	// Three 48 kHz stereo frames become one 16 kHz mono sample. Stereo frame
+	// means are 2000, 4000, and 6000; their anti-aliasing mean is 4000.
+	input := make([]byte, 0, 12)
+	for _, pair := range [][2]int16{{1000, 3000}, {3000, 5000}, {5000, 7000}} {
+		input = binary.LittleEndian.AppendUint16(input, uint16(pair[0]))
+		input = binary.LittleEndian.AppendUint16(input, uint16(pair[1]))
+	}
+	converter := &desktopAudioLegacyConverter{}
+	if out := converter.Convert(input[:5]); len(out) != 0 {
+		t.Fatalf("fragment produced %d bytes before a complete window", len(out))
+	}
+	out := converter.Convert(input[5:])
+	if len(out) != 2 {
+		t.Fatalf("legacy PCM length = %d, want 2", len(out))
+	}
+	if got := int16(binary.LittleEndian.Uint16(out)); got != 4000 {
+		t.Fatalf("legacy PCM sample = %d, want 4000", got)
+	}
+}

Overlord-Client/cmd/agent/webrtcpub/audio_opus.go

@@ -0,0 +1,79 @@
+//go:build overlord_webrtc
+
+package webrtcpub
+
+import (
+	"sync"
+	"time"
+
+	"github.com/pion/webrtc/v4"
+	"github.com/pion/webrtc/v4/pkg/media"
+	"github.com/thesyncim/gopus"
+)
+
+const (
+	opusInputRate       = 48000
+	opusChannels        = 2
+	opusPacketMs        = 20
+	opusSamplesPerFrame = opusInputRate * opusPacketMs / 1000
+	opusBitrate         = 96000
+)
+
+type opusAudioWriter struct {
+	t       *webrtc.TrackLocalStaticSample
+	encoder *gopus.Encoder
+	mu      sync.Mutex
+	pcm     []int16
+	packet  []byte
+}
+
+func newOpusAudioWriter(t *webrtc.TrackLocalStaticSample) (*opusAudioWriter, error) {
+	encoder, err := gopus.NewEncoder(gopus.EncoderConfig{
+		SampleRate:  opusInputRate,
+		Channels:    opusChannels,
+		Application: gopus.ApplicationAudio,
+	})
+	if err != nil {
+		return nil, err
+	}
+	if err := encoder.SetBitrate(opusBitrate); err != nil {
+		return nil, err
+	}
+	if err := encoder.SetInBandFEC(gopus.InBandFECEnabled); err != nil {
+		return nil, err
+	}
+	return &opusAudioWriter{
+		t:       t,
+		encoder: encoder,
+		pcm:     make([]int16, 0, opusSamplesPerFrame*4),
+		packet:  make([]byte, 4000),
+	}, nil
+}
+
+func (w *opusAudioWriter) WriteAudio(pcm []int16) error {
+	if len(pcm) == 0 {
+		return nil
+	}
+	w.mu.Lock()
+	defer w.mu.Unlock()
+	w.pcm = append(w.pcm, pcm...)
+	frameValues := opusSamplesPerFrame * opusChannels
+	for len(w.pcm) >= frameValues {
+		n, err := w.encoder.EncodeInt16(w.pcm[:frameValues], w.packet)
+		if err != nil {
+			w.pcm = w.pcm[:0]
+			return err
+		}
+		if n > 0 {
+			if err := w.t.WriteSample(media.Sample{
+				Data:     w.packet[:n],
+				Duration: opusPacketMs * time.Millisecond,
+			}); err != nil {
+				w.pcm = w.pcm[:0]
+				return err
+			}
+		}
+		w.pcm = w.pcm[frameValues:]
+	}
+	return nil
+}

Overlord-Client/cmd/agent/webrtcpub/audio_opus_test.go

@@ -0,0 +1,33 @@
+//go:build overlord_webrtc
+
+package webrtcpub
+
+import (
+	"testing"
+
+	"github.com/pion/webrtc/v4"
+)
+
+func TestOpusAudioWriterAcceptsFragmentedStereoPCM(t *testing.T) {
+	track, err := webrtc.NewTrackLocalStaticSample(
+		webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeOpus, ClockRate: 48000, Channels: 2},
+		"audio", "test",
+	)
+	if err != nil {
+		t.Fatal(err)
+	}
+	writer, err := newOpusAudioWriter(track)
+	if err != nil {
+		t.Fatal(err)
+	}
+	frame := make([]int16, opusSamplesPerFrame*opusChannels)
+	if err := writer.WriteAudio(frame[:333]); err != nil {
+		t.Fatal(err)
+	}
+	if err := writer.WriteAudio(frame[333:]); err != nil {
+		t.Fatal(err)
+	}
+	if len(writer.pcm) != 0 {
+		t.Fatalf("buffered PCM = %d, want 0 after a complete frame", len(writer.pcm))
+	}
+}

Overlord-Client/cmd/agent/webrtcpub/p2p_pion.go

@@ -71,11 +71,12 @@ func StartP2POffer(ctx context.Context, kind Kind, sessionID string, offerSDP st
 	if hasAudio {
 		if err := mediaEngine.RegisterCodec(webrtc.RTPCodecParameters{
 			RTPCodecCapability: webrtc.RTPCodecCapability{
-				MimeType:  webrtc.MimeTypePCMU,
-				ClockRate: 8000,
-				Channels:  1,
+				MimeType:    webrtc.MimeTypeOpus,
+				ClockRate:   48000,
+				Channels:    2,
+				SDPFmtpLine: "minptime=10;useinbandfec=1",
 			},
-			PayloadType: 0,
+			PayloadType: 111,
 		}, webrtc.RTPCodecTypeAudio); err != nil {
 			return "", fmt.Errorf("register audio codec: %w", err)
 		}
@@ -92,8 +93,9 @@ func StartP2POffer(ctx context.Context, kind Kind, sessionID string, offerSDP st
 	}
 
 	var (
-		videoTrack *webrtc.TrackLocalStaticSample
-		audioTrack *webrtc.TrackLocalStaticSample
+		videoTrack  *webrtc.TrackLocalStaticSample
+		audioTrack  *webrtc.TrackLocalStaticSample
+		audioWriter AudioWriter
 	)
 	if hasVideo {
 		videoTrack, err = webrtc.NewTrackLocalStaticSample(
@@ -117,13 +119,18 @@ func StartP2POffer(ctx context.Context, kind Kind, sessionID string, offerSDP st
 	}
 	if hasAudio {
 		audioTrack, err = webrtc.NewTrackLocalStaticSample(
-			webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypePCMU, ClockRate: 8000, Channels: 1},
+			webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeOpus, ClockRate: 48000, Channels: 2, SDPFmtpLine: "minptime=10;useinbandfec=1"},
 			"overlord-audio-p2p-"+string(kind), "overlord-"+string(kind),
 		)
 		if err != nil {
 			_ = pc.Close()
 			return "", fmt.Errorf("new audio track: %w", err)
 		}
+		audioWriter, err = newOpusAudioWriter(audioTrack)
+		if err != nil {
+			_ = pc.Close()
+			return "", fmt.Errorf("new opus encoder: %w", err)
+		}
 		tx, err := pc.AddTransceiverFromTrack(audioTrack, webrtc.RTPTransceiverInit{
 			Direction: webrtc.RTPTransceiverDirectionSendonly,
 		})
@@ -201,7 +208,7 @@ func StartP2POffer(ctx context.Context, kind Kind, sessionID string, offerSDP st
 		registerVideoWriter(kind, writerID, &h264TrackWriter{t: videoTrack})
 	}
 	if audioTrack != nil {
-		registerAudioWriter(kind, writerID, newPCMUAudioWriter(audioTrack))
+		registerAudioWriter(kind, writerID, audioWriter)
 	}
 	return answer.SDP, nil
 }

Overlord-Client/cmd/agent/webrtcpub/whip_pion.go

@@ -133,11 +133,12 @@ func Start(ctx context.Context, kind Kind, opts Options) (*Publisher, error) {
 	if opts.HasAudio {
 		if err := mediaEngine.RegisterCodec(webrtc.RTPCodecParameters{
 			RTPCodecCapability: webrtc.RTPCodecCapability{
-				MimeType:  webrtc.MimeTypePCMU,
-				ClockRate: 8000,
-				Channels:  1,
+				MimeType:    webrtc.MimeTypeOpus,
+				ClockRate:   48000,
+				Channels:    2,
+				SDPFmtpLine: "minptime=10;useinbandfec=1",
 			},
-			PayloadType: 0,
+			PayloadType: 111,
 		}, webrtc.RTPCodecTypeAudio); err != nil {
 			return nil, fmt.Errorf("register audio codec: %w", err)
 		}
@@ -150,8 +151,9 @@ func Start(ctx context.Context, kind Kind, opts Options) (*Publisher, error) {
 	}
 
 	var (
-		videoTrack *webrtc.TrackLocalStaticSample
-		audioTrack *webrtc.TrackLocalStaticSample
+		videoTrack  *webrtc.TrackLocalStaticSample
+		audioTrack  *webrtc.TrackLocalStaticSample
+		audioWriter AudioWriter
 	)
 	if opts.HasVideo {
 		videoTrack, err = webrtc.NewTrackLocalStaticSample(
@@ -175,13 +177,18 @@ func Start(ctx context.Context, kind Kind, opts Options) (*Publisher, error) {
 	}
 	if opts.HasAudio {
 		audioTrack, err = webrtc.NewTrackLocalStaticSample(
-			webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypePCMU, ClockRate: 8000, Channels: 1},
+			webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeOpus, ClockRate: 48000, Channels: 2, SDPFmtpLine: "minptime=10;useinbandfec=1"},
 			"overlord-audio-"+string(kind), "overlord-"+string(kind),
 		)
 		if err != nil {
 			_ = pc.Close()
 			return nil, fmt.Errorf("new audio track: %w", err)
 		}
+		audioWriter, err = newOpusAudioWriter(audioTrack)
+		if err != nil {
+			_ = pc.Close()
+			return nil, fmt.Errorf("new opus encoder: %w", err)
+		}
 		tx, err := pc.AddTransceiverFromTrack(audioTrack, webrtc.RTPTransceiverInit{
 			Direction: webrtc.RTPTransceiverDirectionSendonly,
 		})
@@ -256,7 +263,7 @@ func Start(ctx context.Context, kind Kind, opts Options) (*Publisher, error) {
 		registerVideoWriter(kind, writerID, &h264TrackWriter{t: videoTrack})
 	}
 	if audioTrack != nil {
-		registerAudioWriter(kind, writerID, newPCMUAudioWriter(audioTrack))
+		registerAudioWriter(kind, writerID, audioWriter)
 	}
 	log.Printf("webrtcpub: WHIP[%s] session established (resource=%s)", kind, resourceURL)
 	return pub, nil

Overlord-Client/cmd/agent/wire/protocol.go

@@ -107,11 +107,13 @@ type ClientLogsResult struct {
 }
 
 type FrameHeader struct {
-	Monitor int    `msgpack:"monitor"`
-	FPS     int    `msgpack:"fps"`
-	Format  string `msgpack:"format"`
-	Backstage    bool   `msgpack:"backstage,omitempty"`
-	Webcam  bool   `msgpack:"webcam,omitempty"`
+	Monitor   int    `msgpack:"monitor"`
+	FPS       int    `msgpack:"fps"`
+	Format    string `msgpack:"format"`
+	Width     int    `msgpack:"width,omitempty"`
+	Height    int    `msgpack:"height,omitempty"`
+	Backstage bool   `msgpack:"backstage,omitempty"`
+	Webcam    bool   `msgpack:"webcam,omitempty"`
 }
 
 type Frame struct {
@@ -120,6 +122,20 @@ type Frame struct {
 	Data   []byte      `msgpack:"data"`
 }
 
+type DesktopStreamStats struct {
+	Type      string  `msgpack:"type"`
+	FPS       int     `msgpack:"fps"`
+	Format    string  `msgpack:"format"`
+	Bytes     int     `msgpack:"bytes"`
+	Width     int     `msgpack:"width"`
+	Height    int     `msgpack:"height"`
+	CaptureMs float64 `msgpack:"captureMs"`
+	EncodeMs  float64 `msgpack:"encodeMs"`
+	SendMs    float64 `msgpack:"sendMs"`
+	TotalMs   float64 `msgpack:"totalMs"`
+	Transport string  `msgpack:"transport"`
+}
+
 type FrameAck struct {
 	Type string `msgpack:"type"`
 }
@@ -381,9 +397,9 @@ type BackstageInstalledApp struct {
 }
 
 type BackstageInstalledAppsResult struct {
-	Type string             `msgpack:"type"`
+	Type string                  `msgpack:"type"`
 	Apps []BackstageInstalledApp `msgpack:"apps"`
-	Done bool               `msgpack:"done"`
+	Done bool                    `msgpack:"done"`
 }
 
 type BackstageBrowserLaunchStatus struct {
@@ -416,7 +432,7 @@ type BackstageMonitorEntry struct {
 }
 
 type BackstageWindowListResult struct {
-	Type     string             `msgpack:"type"`
+	Type     string                  `msgpack:"type"`
 	Windows  []BackstageWindowEntry  `msgpack:"windows"`
 	Monitors []BackstageMonitorEntry `msgpack:"monitors"`
 }