diff --git a/Overlord-Client/cmd/agent/audio/voice_native_cgo.go b/Overlord-Client/cmd/agent/audio/voice_native_cgo.go index 077cf7a..705252a 100644 --- a/Overlord-Client/cmd/agent/audio/voice_native_cgo.go +++ b/Overlord-Client/cmd/agent/audio/voice_native_cgo.go @@ -18,6 +18,8 @@ import ( const ( voiceSampleRate = 16000 voiceChannels = 1 + desktopAudioSampleRate = 48000 + desktopAudioChannels = 2 voiceBytesPerSample = 2 maxPlaybackBufferMillis = 250 ) @@ -390,15 +392,15 @@ func StartCaptureOnlySession(parent context.Context, source string, onCapture fu } capCfg := malgo.DefaultDeviceConfig(malgo.Capture) - capCfg.SampleRate = voiceSampleRate + capCfg.SampleRate = desktopAudioSampleRate capCfg.Capture.Format = malgo.FormatS16 - capCfg.Capture.Channels = voiceChannels + capCfg.Capture.Channels = desktopAudioChannels useLoopback := false if strings.EqualFold(source, "system") { capCfg = malgo.DefaultDeviceConfig(malgo.Loopback) - capCfg.SampleRate = voiceSampleRate + capCfg.SampleRate = desktopAudioSampleRate capCfg.Capture.Format = malgo.FormatS16 - capCfg.Capture.Channels = voiceChannels + capCfg.Capture.Channels = desktopAudioChannels useLoopback = true } else if strings.HasPrefix(strings.ToLower(source), "device:") { deviceName := strings.TrimSpace(source[len("device:"):]) @@ -442,9 +444,9 @@ func StartCaptureOnlySession(parent context.Context, source string, onCapture fu capDev, err := malgo.InitDevice(audioCtx.Context, capCfg, capCallbacks) if err != nil && useLoopback { capCfg = malgo.DefaultDeviceConfig(malgo.Capture) - capCfg.SampleRate = voiceSampleRate + capCfg.SampleRate = desktopAudioSampleRate capCfg.Capture.Format = malgo.FormatS16 - capCfg.Capture.Channels = voiceChannels + capCfg.Capture.Channels = desktopAudioChannels deviceID, _, ok := findSystemCaptureDevice(audioCtx.Context) if !ok || deviceID == nil { audioCtx.Free() diff --git a/Overlord-Client/cmd/agent/capture/capture.go b/Overlord-Client/cmd/agent/capture/capture.go index 9950ca9..6a05497 100644 --- a/Overlord-Client/cmd/agent/capture/capture.go +++ b/Overlord-Client/cmd/agent/capture/capture.go @@ -154,6 +154,8 @@ func CaptureAndSend(ctx context.Context, env *rt.Env) error { slotAcquired = true } + frameWidth := img.Bounds().Dx() + frameHeight := img.Bounds().Dy() quality := jpegQuality() frame, encodeDur, err := buildFrame(img, display, quality) PutRGBA(img) @@ -164,6 +166,8 @@ func CaptureAndSend(ctx context.Context, env *rt.Env) error { } return err } + frame.Header.Width = frameWidth + frame.Header.Height = frameHeight now := time.Now() fps := frameFPS(now) if fps <= 0 { @@ -181,17 +185,21 @@ func CaptureAndSend(ctx context.Context, env *rt.Env) error { if dur <= 0 { dur = 33 * time.Millisecond } + sendStart := time.Now() if werr := webrtcpub.WriteH264(webrtcpub.KindDesktop, frame.Data, dur); werr != nil { log.Printf("webrtc: write h264 failed: %v", werr) } + sendDur := time.Since(sendStart) if slotAcquired { ReleaseFrameSlot() } statFrames.Add(1) statCapNs.Add(captureDur.Nanoseconds()) statEncNs.Add(encodeDur.Nanoseconds()) - statTotalNs.Add(time.Since(t0).Nanoseconds()) + totalDur := time.Since(t0) + statTotalNs.Add(totalDur.Nanoseconds()) statBytes.Add(int64(len(frame.Data))) + emitDesktopStreamStats(ctx, env, frame, fps, captureDur, encodeDur, sendDur, totalDur, "webrtc") return nil } if !slotAcquired { @@ -233,6 +241,7 @@ func CaptureAndSend(ctx context.Context, env *rt.Env) error { statSendNs.Add(sendDur.Nanoseconds()) statTotalNs.Add(time.Since(t0).Nanoseconds()) statBytes.Add(int64(len(frame.Data))) + emitDesktopStreamStats(ctx, env, frame, fps, captureDur, encodeDur, sendDur, time.Since(t0), "websocket") return err } @@ -251,14 +260,18 @@ func sendCompletedFrame(ctx context.Context, env *rt.Env, frame wire.Frame, disp if dur <= 0 { dur = 33 * time.Millisecond } + sendStart := time.Now() if werr := webrtcpub.WriteH264(webrtcpub.KindDesktop, frame.Data, dur); werr != nil { log.Printf("webrtc: write h264 failed: %v", werr) } + sendDur := time.Since(sendStart) statFrames.Add(1) statCapNs.Add(captureDur.Nanoseconds()) statEncNs.Add(encodeDur.Nanoseconds()) - statTotalNs.Add(time.Since(t0).Nanoseconds()) + totalDur := time.Since(t0) + statTotalNs.Add(totalDur.Nanoseconds()) statBytes.Add(int64(len(frame.Data))) + emitDesktopStreamStats(ctx, env, frame, fps, captureDur, encodeDur, sendDur, totalDur, "webrtc") return nil } if !AcquireFrameSlot() { @@ -301,6 +314,7 @@ func sendCompletedFrame(ctx context.Context, env *rt.Env, frame wire.Frame, disp statSendNs.Add(sendDur.Nanoseconds()) statTotalNs.Add(time.Since(t0).Nanoseconds()) statBytes.Add(int64(len(frame.Data))) + emitDesktopStreamStats(ctx, env, frame, fps, captureDur, encodeDur, sendDur, time.Since(t0), "websocket") return nil } diff --git a/Overlord-Client/cmd/agent/capture/capture_test.go b/Overlord-Client/cmd/agent/capture/capture_test.go index f3b20b9..8f6676d 100644 --- a/Overlord-Client/cmd/agent/capture/capture_test.go +++ b/Overlord-Client/cmd/agent/capture/capture_test.go @@ -44,6 +44,7 @@ func TestCaptureAndSend_NoDisplays(t *testing.T) { } func TestCaptureAndSend_SendsFrame(t *testing.T) { + previousStatsAt := lastDesktopStreamStatsMs.Swap(0) originalCount := activeDisplays originalCapture := captureDisplayFn activeDisplays = func() int { return 1 } @@ -56,6 +57,7 @@ func TestCaptureAndSend_SendsFrame(t *testing.T) { t.Cleanup(func() { activeDisplays = originalCount captureDisplayFn = originalCapture + lastDesktopStreamStatsMs.Store(previousStatsAt) }) writer := &recordingWriter{} @@ -63,8 +65,8 @@ func TestCaptureAndSend_SendsFrame(t *testing.T) { if err := CaptureAndSend(context.Background(), env); err != nil { t.Fatalf("CaptureAndSend returned error: %v", err) } - if len(writer.msgs) != 1 { - t.Fatalf("expected one message written, got %d", len(writer.msgs)) + if len(writer.msgs) != 2 { + t.Fatalf("expected frame and telemetry messages, got %d", len(writer.msgs)) } var frame wire.Frame @@ -84,6 +86,17 @@ func TestCaptureAndSend_SendsFrame(t *testing.T) { if _, err := jpeg.Decode(bytes.NewReader(frame.Data)); err != nil { t.Fatalf("jpeg payload did not decode: %v", err) } + + var stats wire.DesktopStreamStats + if err := msgpack.Unmarshal(writer.msgs[1], &stats); err != nil { + t.Fatalf("decode desktop stream stats: %v", err) + } + if stats.Type != "desktop_stream_stats" || stats.Format != "jpeg" { + t.Fatalf("unexpected stream stats: %#v", stats) + } + if stats.CaptureMs < 0 || stats.EncodeMs < 0 || stats.TotalMs < 0 { + t.Fatalf("stream stats contained a negative duration: %#v", stats) + } } func TestActualScreenCapture(t *testing.T) { @@ -306,8 +319,13 @@ func TestCaptureAndSend_DisplayOutOfRange(t *testing.T) { t.Errorf("expected capture from display %d, got %d", tt.expectedDisplay, capturedDisplay) } - if len(writer.msgs) != 1 { - t.Errorf("expected 1 message, got %d", len(writer.msgs)) + if len(writer.msgs) == 0 { + t.Error("expected a frame message") + } else { + var frame wire.Frame + if err := msgpack.Unmarshal(writer.msgs[0], &frame); err != nil || frame.Type != "frame" { + t.Errorf("first message was not a valid frame: type=%q err=%v", frame.Type, err) + } } }) } diff --git a/Overlord-Client/cmd/agent/capture/h264_bitrate.go b/Overlord-Client/cmd/agent/capture/h264_bitrate.go new file mode 100644 index 0000000..f0d3e1c --- /dev/null +++ b/Overlord-Client/cmd/agent/capture/h264_bitrate.go @@ -0,0 +1,66 @@ +package capture + +import ( + "math" + "sync/atomic" +) + +const maxH264Bitrate = 50_000_000 + +var h264TargetBitrate atomic.Int64 + +func SetH264TargetBitrate(bps int) int { + if bps < 0 { + bps = 0 + } + if bps > maxH264Bitrate { + bps = maxH264Bitrate + } + previous := h264TargetBitrate.Swap(int64(bps)) + if previous != int64(bps) { + resetH264Encoder() + resetH264Encoderbackstage() + resetH264TextureEncoderForBitrate() + RequestDesktopFullFrame() + } + return bps +} + +func configuredH264Bitrate() int { + return int(h264TargetBitrate.Load()) +} + +func automaticH264Bitrate(width, height, fps int) int { + pixelsPerSecond := float64(width * height * fps) + bitrate := int(pixelsPerSecond * 0.08) + if bitrate < 1_500_000 { + return 1_500_000 + } + if bitrate > 18_000_000 { + return 18_000_000 + } + return bitrate +} + +func targetH264Bitrate(width, height, fps int) int { + if configured := configuredH264Bitrate(); configured > 0 { + return configured + } + return automaticH264Bitrate(width, height, fps) +} + +func targetH264CRF(width, height, fps int) float32 { + configured := configuredH264Bitrate() + if configured <= 0 { + return 23 + } + ratio := float64(configured) / float64(automaticH264Bitrate(width, height, fps)) + crf := 23 - 6*math.Log2(ratio) + if crf < 12 { + crf = 12 + } + if crf > 35 { + crf = 35 + } + return float32(crf) +} diff --git a/Overlord-Client/cmd/agent/capture/h264_bitrate_reset_other.go b/Overlord-Client/cmd/agent/capture/h264_bitrate_reset_other.go new file mode 100644 index 0000000..731ee54 --- /dev/null +++ b/Overlord-Client/cmd/agent/capture/h264_bitrate_reset_other.go @@ -0,0 +1,5 @@ +//go:build !windows + +package capture + +func resetH264TextureEncoderForBitrate() {} diff --git a/Overlord-Client/cmd/agent/capture/h264_bitrate_reset_windows.go b/Overlord-Client/cmd/agent/capture/h264_bitrate_reset_windows.go new file mode 100644 index 0000000..7126e65 --- /dev/null +++ b/Overlord-Client/cmd/agent/capture/h264_bitrate_reset_windows.go @@ -0,0 +1,7 @@ +//go:build windows + +package capture + +func resetH264TextureEncoderForBitrate() { + resetH264D3D11TextureEncoder() +} diff --git a/Overlord-Client/cmd/agent/capture/h264_bitrate_test.go b/Overlord-Client/cmd/agent/capture/h264_bitrate_test.go new file mode 100644 index 0000000..e5fbf54 --- /dev/null +++ b/Overlord-Client/cmd/agent/capture/h264_bitrate_test.go @@ -0,0 +1,38 @@ +package capture + +import "testing" + +func TestH264TargetBitrateAutoAndManual(t *testing.T) { + original := configuredH264Bitrate() + defer SetH264TargetBitrate(original) + + SetH264TargetBitrate(0) + if got := targetH264Bitrate(1920, 1080, 60); got != 9_953_280 { + t.Fatalf("automatic 1080p60 bitrate = %d, want 9953280", got) + } + if got := targetH264Bitrate(3840, 2160, 240); got != 18_000_000 { + t.Fatalf("automatic bitrate cap = %d, want 18000000", got) + } + + if got := SetH264TargetBitrate(30_000_000); got != 30_000_000 { + t.Fatalf("manual bitrate = %d, want 30000000", got) + } + if got := targetH264Bitrate(1280, 720, 30); got != 30_000_000 { + t.Fatalf("manual target = %d, want 30000000", got) + } + if got := SetH264TargetBitrate(60_000_000); got != maxH264Bitrate { + t.Fatalf("manual cap = %d, want %d", got, maxH264Bitrate) + } +} + +func TestH264TargetCRFTracksManualRate(t *testing.T) { + original := configuredH264Bitrate() + defer SetH264TargetBitrate(original) + SetH264TargetBitrate(0) + auto := targetH264CRF(1920, 1080, 60) + SetH264TargetBitrate(20_000_000) + higherRate := targetH264CRF(1920, 1080, 60) + if higherRate >= auto { + t.Fatalf("higher bitrate CRF = %.2f, want below automatic %.2f", higherRate, auto) + } +} diff --git a/Overlord-Client/cmd/agent/capture/h264_direct_windows.go b/Overlord-Client/cmd/agent/capture/h264_direct_windows.go index e47e643..5fc4f02 100644 --- a/Overlord-Client/cmd/agent/capture/h264_direct_windows.go +++ b/Overlord-Client/cmd/agent/capture/h264_direct_windows.go @@ -14,7 +14,7 @@ func tryBuildDirectH264Frame(display int) (wire.Frame, time.Duration, time.Durat return wire.Frame{}, 0, 0, false, nil } forceKeyframe := webrtcpub.ConsumeKeyframeRequest() - data, _, _, captureDur, encodeDur, used, err := captureDisplayDXGIH264(display, forceKeyframe) + data, width, height, captureDur, encodeDur, used, err := captureDisplayDXGIH264(display, forceKeyframe) if !used || err != nil { return wire.Frame{}, captureDur, encodeDur, used, err } @@ -24,5 +24,5 @@ func tryBuildDirectH264Frame(display int) (wire.Frame, time.Duration, time.Durat now := time.Now() lastKeyframe.Store(now.UnixNano()) statFullFrames.Add(1) - return wire.Frame{Type: "frame", Header: wire.FrameHeader{Monitor: display, FPS: 0, Format: "h264"}, Data: data}, captureDur, encodeDur, true, nil + return wire.Frame{Type: "frame", Header: wire.FrameHeader{Monitor: display, FPS: 0, Format: "h264", Width: width, Height: height}, Data: data}, captureDur, encodeDur, true, nil } diff --git a/Overlord-Client/cmd/agent/capture/h264_encoder_windows.go b/Overlord-Client/cmd/agent/capture/h264_encoder_windows.go index 626b48f..641d398 100644 --- a/Overlord-Client/cmd/agent/capture/h264_encoder_windows.go +++ b/Overlord-Client/cmd/agent/capture/h264_encoder_windows.go @@ -1296,12 +1296,6 @@ func failedHR(hr uintptr) bool { return hr&0x80000000 != 0 } -func targetH264Bitrate(width, height, fps int) int { - pixelsPerSecond := float64(width * height * fps) - bitrate := int(pixelsPerSecond * 0.08) - return clampInt(bitrate, 1_500_000, 18_000_000) -} - func h264LevelFor(width, height, fps int) int { mbsPerFrame := ((width + 15) / 16) * ((height + 15) / 16) mbsPerSecond := mbsPerFrame * fps diff --git a/Overlord-Client/cmd/agent/capture/h264_encoder_x264.go b/Overlord-Client/cmd/agent/capture/h264_encoder_x264.go index c79e913..4e9a9c6 100644 --- a/Overlord-Client/cmd/agent/capture/h264_encoder_x264.go +++ b/Overlord-Client/cmd/agent/capture/h264_encoder_x264.go @@ -104,7 +104,7 @@ func ensureH264EncoderLocked(width, height int) error { Preset: "veryfast", Profile: "main", RateControl: "crf", - RateConstant: 23, + RateConstant: targetH264CRF(width, height, fps), LogLevel: x264.LogError, } @@ -201,7 +201,7 @@ func ensurebackstageH264EncoderLocked(width, height int) error { Preset: "veryfast", Profile: "main", RateControl: "crf", - RateConstant: 23, + RateConstant: targetH264CRF(width, height, fps), LogLevel: x264.LogError, } enc, err := x264.NewEncoder(&backstageH264Buf, opts) @@ -280,7 +280,7 @@ func ensureWebcamH264EncoderLocked(width, height int) error { Preset: "veryfast", Profile: "main", RateControl: "crf", - RateConstant: 23, + RateConstant: targetH264CRF(width, height, fps), LogLevel: x264.LogError, } enc, err := x264.NewEncoder(&webcamH264Buf, opts) diff --git a/Overlord-Client/cmd/agent/capture/stream_stats.go b/Overlord-Client/cmd/agent/capture/stream_stats.go new file mode 100644 index 0000000..6bfcd68 --- /dev/null +++ b/Overlord-Client/cmd/agent/capture/stream_stats.go @@ -0,0 +1,37 @@ +package capture + +import ( + "context" + "sync/atomic" + "time" + + rt "overlord-client/cmd/agent/runtime" + "overlord-client/cmd/agent/wire" +) + +var lastDesktopStreamStatsMs atomic.Int64 + +func emitDesktopStreamStats(ctx context.Context, env *rt.Env, frame wire.Frame, fps int, captureDur, encodeDur, sendDur, totalDur time.Duration, transport string) { + now := time.Now().UnixMilli() + previous := lastDesktopStreamStatsMs.Load() + if now-previous < 500 || !lastDesktopStreamStatsMs.CompareAndSwap(previous, now) { + return + } + _ = wire.WriteMsg(ctx, env.Conn, wire.DesktopStreamStats{ + Type: "desktop_stream_stats", + FPS: fps, + Format: frame.Header.Format, + Bytes: len(frame.Data), + Width: frame.Header.Width, + Height: frame.Header.Height, + CaptureMs: durationMs(captureDur), + EncodeMs: durationMs(encodeDur), + SendMs: durationMs(sendDur), + TotalMs: durationMs(totalDur), + Transport: transport, + }) +} + +func durationMs(value time.Duration) float64 { + return float64(value.Microseconds()) / 1000 +} diff --git a/Overlord-Client/cmd/agent/handlers/command.go b/Overlord-Client/cmd/agent/handlers/command.go index ce045db..a5989e4 100644 --- a/Overlord-Client/cmd/agent/handlers/command.go +++ b/Overlord-Client/cmd/agent/handlers/command.go @@ -503,6 +503,7 @@ func startDesktopAudioSession(ctx context.Context, env *runtime.Env, sessionID s stopDesktopAudioSession() vCtx, cancel := context.WithCancel(ctx) + legacyConverter := &desktopAudioLegacyConverter{} session, err := audio.StartCaptureOnlySession(vCtx, source, func(chunk []byte) { if len(chunk) == 0 { return @@ -513,10 +514,14 @@ func startDesktopAudioSession(ctx context.Context, env *runtime.Env, sessionID s samples := pcm16BytesToInt16(chunk) _ = webrtcpub.WriteAudio(webrtcpub.KindAudio, samples) } + legacyPCM := legacyConverter.Convert(chunk) + if len(legacyPCM) == 0 { + return + } msg := map[string]interface{}{ "type": "desktop_audio_uplink", "sessionId": sessionID, - "data": chunk, + "data": legacyPCM, } _ = wire.WriteMsg(vCtx, env.Conn, msg) }) @@ -904,6 +909,24 @@ func HandleCommand(ctx context.Context, env *runtime.Env, envelope map[string]in log.Printf("desktop: set target fps=%d", fps) sendCommandResultSafe(env, cmdID, true, "") return nil + case "desktop_set_bitrate": + payload, _ := envelope["payload"].(map[string]interface{}) + bitrateMbps := 0 + if payload != nil { + if v, ok := payloadInt(payload, "bitrateMbps"); ok { + bitrateMbps = v + } + } + if bitrateMbps < 0 { + bitrateMbps = 0 + } + if bitrateMbps > 50 { + bitrateMbps = 50 + } + bps := capture.SetH264TargetBitrate(bitrateMbps * 1_000_000) + log.Printf("desktop: set target bitrate=%d Mbps (0=auto)", bps/1_000_000) + sendCommandResultSafe(env, cmdID, true, "") + return nil case "clipboard_sync_start": payload, _ := envelope["payload"].(map[string]interface{}) source := "rd" diff --git a/Overlord-Client/cmd/agent/handlers/webrtc.go b/Overlord-Client/cmd/agent/handlers/webrtc.go index 3fa14cb..5b4fd65 100644 --- a/Overlord-Client/cmd/agent/handlers/webrtc.go +++ b/Overlord-Client/cmd/agent/handlers/webrtc.go @@ -23,6 +23,43 @@ func pcm16BytesToInt16(chunk []byte) []int16 { return out } +// WebRTC desktop audio is captured at 48 kHz stereo. Keep the established raw +// WebSocket fallback wire format at 16 kHz mono by averaging each stereo frame +// and decimating groups of three frames. pending preserves sample groups when +// an audio callback ends between frames or decimation windows. +type desktopAudioLegacyConverter struct { + pending []byte +} + +func (c *desktopAudioLegacyConverter) Convert(chunk []byte) []byte { + if len(chunk) == 0 { + return nil + } + data := make([]byte, 0, len(c.pending)+len(chunk)) + data = append(data, c.pending...) + data = append(data, chunk...) + const windowBytes = 3 * 2 * 2 // three stereo PCM16 frames + windows := len(data) / windowBytes + if windows == 0 { + c.pending = append(c.pending[:0], data...) + return nil + } + out := make([]byte, 0, windows*2) + for window := 0; window < windows; window++ { + var sum int64 + for frame := 0; frame < 3; frame++ { + offset := window*windowBytes + frame*4 + left := int32(int16(binary.LittleEndian.Uint16(data[offset:]))) + right := int32(int16(binary.LittleEndian.Uint16(data[offset+2:]))) + sum += int64(left+right) / 2 + } + sample := int16(sum / 3) + out = binary.LittleEndian.AppendUint16(out, uint16(sample)) + } + c.pending = append(c.pending[:0], data[windows*windowBytes:]...) + return out +} + func kindFromPayload(payload map[string]interface{}) webrtcpub.Kind { switch s, _ := payload["kind"].(string); s { case "backstage": diff --git a/Overlord-Client/cmd/agent/handlers/webrtc_audio_test.go b/Overlord-Client/cmd/agent/handlers/webrtc_audio_test.go new file mode 100644 index 0000000..df12c0c --- /dev/null +++ b/Overlord-Client/cmd/agent/handlers/webrtc_audio_test.go @@ -0,0 +1,27 @@ +package handlers + +import ( + "encoding/binary" + "testing" +) + +func TestDesktopAudioLegacyPCMDownmixesAndDecimates(t *testing.T) { + // Three 48 kHz stereo frames become one 16 kHz mono sample. Stereo frame + // means are 2000, 4000, and 6000; their anti-aliasing mean is 4000. + input := make([]byte, 0, 12) + for _, pair := range [][2]int16{{1000, 3000}, {3000, 5000}, {5000, 7000}} { + input = binary.LittleEndian.AppendUint16(input, uint16(pair[0])) + input = binary.LittleEndian.AppendUint16(input, uint16(pair[1])) + } + converter := &desktopAudioLegacyConverter{} + if out := converter.Convert(input[:5]); len(out) != 0 { + t.Fatalf("fragment produced %d bytes before a complete window", len(out)) + } + out := converter.Convert(input[5:]) + if len(out) != 2 { + t.Fatalf("legacy PCM length = %d, want 2", len(out)) + } + if got := int16(binary.LittleEndian.Uint16(out)); got != 4000 { + t.Fatalf("legacy PCM sample = %d, want 4000", got) + } +} diff --git a/Overlord-Client/cmd/agent/webrtcpub/audio_opus.go b/Overlord-Client/cmd/agent/webrtcpub/audio_opus.go new file mode 100644 index 0000000..09e06f4 --- /dev/null +++ b/Overlord-Client/cmd/agent/webrtcpub/audio_opus.go @@ -0,0 +1,79 @@ +//go:build overlord_webrtc + +package webrtcpub + +import ( + "sync" + "time" + + "github.com/pion/webrtc/v4" + "github.com/pion/webrtc/v4/pkg/media" + "github.com/thesyncim/gopus" +) + +const ( + opusInputRate = 48000 + opusChannels = 2 + opusPacketMs = 20 + opusSamplesPerFrame = opusInputRate * opusPacketMs / 1000 + opusBitrate = 96000 +) + +type opusAudioWriter struct { + t *webrtc.TrackLocalStaticSample + encoder *gopus.Encoder + mu sync.Mutex + pcm []int16 + packet []byte +} + +func newOpusAudioWriter(t *webrtc.TrackLocalStaticSample) (*opusAudioWriter, error) { + encoder, err := gopus.NewEncoder(gopus.EncoderConfig{ + SampleRate: opusInputRate, + Channels: opusChannels, + Application: gopus.ApplicationAudio, + }) + if err != nil { + return nil, err + } + if err := encoder.SetBitrate(opusBitrate); err != nil { + return nil, err + } + if err := encoder.SetInBandFEC(gopus.InBandFECEnabled); err != nil { + return nil, err + } + return &opusAudioWriter{ + t: t, + encoder: encoder, + pcm: make([]int16, 0, opusSamplesPerFrame*4), + packet: make([]byte, 4000), + }, nil +} + +func (w *opusAudioWriter) WriteAudio(pcm []int16) error { + if len(pcm) == 0 { + return nil + } + w.mu.Lock() + defer w.mu.Unlock() + w.pcm = append(w.pcm, pcm...) + frameValues := opusSamplesPerFrame * opusChannels + for len(w.pcm) >= frameValues { + n, err := w.encoder.EncodeInt16(w.pcm[:frameValues], w.packet) + if err != nil { + w.pcm = w.pcm[:0] + return err + } + if n > 0 { + if err := w.t.WriteSample(media.Sample{ + Data: w.packet[:n], + Duration: opusPacketMs * time.Millisecond, + }); err != nil { + w.pcm = w.pcm[:0] + return err + } + } + w.pcm = w.pcm[frameValues:] + } + return nil +} diff --git a/Overlord-Client/cmd/agent/webrtcpub/audio_opus_test.go b/Overlord-Client/cmd/agent/webrtcpub/audio_opus_test.go new file mode 100644 index 0000000..8202afb --- /dev/null +++ b/Overlord-Client/cmd/agent/webrtcpub/audio_opus_test.go @@ -0,0 +1,33 @@ +//go:build overlord_webrtc + +package webrtcpub + +import ( + "testing" + + "github.com/pion/webrtc/v4" +) + +func TestOpusAudioWriterAcceptsFragmentedStereoPCM(t *testing.T) { + track, err := webrtc.NewTrackLocalStaticSample( + webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeOpus, ClockRate: 48000, Channels: 2}, + "audio", "test", + ) + if err != nil { + t.Fatal(err) + } + writer, err := newOpusAudioWriter(track) + if err != nil { + t.Fatal(err) + } + frame := make([]int16, opusSamplesPerFrame*opusChannels) + if err := writer.WriteAudio(frame[:333]); err != nil { + t.Fatal(err) + } + if err := writer.WriteAudio(frame[333:]); err != nil { + t.Fatal(err) + } + if len(writer.pcm) != 0 { + t.Fatalf("buffered PCM = %d, want 0 after a complete frame", len(writer.pcm)) + } +} diff --git a/Overlord-Client/cmd/agent/webrtcpub/p2p_pion.go b/Overlord-Client/cmd/agent/webrtcpub/p2p_pion.go index f3c402e..b500266 100644 --- a/Overlord-Client/cmd/agent/webrtcpub/p2p_pion.go +++ b/Overlord-Client/cmd/agent/webrtcpub/p2p_pion.go @@ -71,11 +71,12 @@ func StartP2POffer(ctx context.Context, kind Kind, sessionID string, offerSDP st if hasAudio { if err := mediaEngine.RegisterCodec(webrtc.RTPCodecParameters{ RTPCodecCapability: webrtc.RTPCodecCapability{ - MimeType: webrtc.MimeTypePCMU, - ClockRate: 8000, - Channels: 1, + MimeType: webrtc.MimeTypeOpus, + ClockRate: 48000, + Channels: 2, + SDPFmtpLine: "minptime=10;useinbandfec=1", }, - PayloadType: 0, + PayloadType: 111, }, webrtc.RTPCodecTypeAudio); err != nil { return "", fmt.Errorf("register audio codec: %w", err) } @@ -92,8 +93,9 @@ func StartP2POffer(ctx context.Context, kind Kind, sessionID string, offerSDP st } var ( - videoTrack *webrtc.TrackLocalStaticSample - audioTrack *webrtc.TrackLocalStaticSample + videoTrack *webrtc.TrackLocalStaticSample + audioTrack *webrtc.TrackLocalStaticSample + audioWriter AudioWriter ) if hasVideo { videoTrack, err = webrtc.NewTrackLocalStaticSample( @@ -117,13 +119,18 @@ func StartP2POffer(ctx context.Context, kind Kind, sessionID string, offerSDP st } if hasAudio { audioTrack, err = webrtc.NewTrackLocalStaticSample( - webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypePCMU, ClockRate: 8000, Channels: 1}, + webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeOpus, ClockRate: 48000, Channels: 2, SDPFmtpLine: "minptime=10;useinbandfec=1"}, "overlord-audio-p2p-"+string(kind), "overlord-"+string(kind), ) if err != nil { _ = pc.Close() return "", fmt.Errorf("new audio track: %w", err) } + audioWriter, err = newOpusAudioWriter(audioTrack) + if err != nil { + _ = pc.Close() + return "", fmt.Errorf("new opus encoder: %w", err) + } tx, err := pc.AddTransceiverFromTrack(audioTrack, webrtc.RTPTransceiverInit{ Direction: webrtc.RTPTransceiverDirectionSendonly, }) @@ -201,7 +208,7 @@ func StartP2POffer(ctx context.Context, kind Kind, sessionID string, offerSDP st registerVideoWriter(kind, writerID, &h264TrackWriter{t: videoTrack}) } if audioTrack != nil { - registerAudioWriter(kind, writerID, newPCMUAudioWriter(audioTrack)) + registerAudioWriter(kind, writerID, audioWriter) } return answer.SDP, nil } diff --git a/Overlord-Client/cmd/agent/webrtcpub/whip_pion.go b/Overlord-Client/cmd/agent/webrtcpub/whip_pion.go index b83acbb..a6aaa42 100644 --- a/Overlord-Client/cmd/agent/webrtcpub/whip_pion.go +++ b/Overlord-Client/cmd/agent/webrtcpub/whip_pion.go @@ -133,11 +133,12 @@ func Start(ctx context.Context, kind Kind, opts Options) (*Publisher, error) { if opts.HasAudio { if err := mediaEngine.RegisterCodec(webrtc.RTPCodecParameters{ RTPCodecCapability: webrtc.RTPCodecCapability{ - MimeType: webrtc.MimeTypePCMU, - ClockRate: 8000, - Channels: 1, + MimeType: webrtc.MimeTypeOpus, + ClockRate: 48000, + Channels: 2, + SDPFmtpLine: "minptime=10;useinbandfec=1", }, - PayloadType: 0, + PayloadType: 111, }, webrtc.RTPCodecTypeAudio); err != nil { return nil, fmt.Errorf("register audio codec: %w", err) } @@ -150,8 +151,9 @@ func Start(ctx context.Context, kind Kind, opts Options) (*Publisher, error) { } var ( - videoTrack *webrtc.TrackLocalStaticSample - audioTrack *webrtc.TrackLocalStaticSample + videoTrack *webrtc.TrackLocalStaticSample + audioTrack *webrtc.TrackLocalStaticSample + audioWriter AudioWriter ) if opts.HasVideo { videoTrack, err = webrtc.NewTrackLocalStaticSample( @@ -175,13 +177,18 @@ func Start(ctx context.Context, kind Kind, opts Options) (*Publisher, error) { } if opts.HasAudio { audioTrack, err = webrtc.NewTrackLocalStaticSample( - webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypePCMU, ClockRate: 8000, Channels: 1}, + webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeOpus, ClockRate: 48000, Channels: 2, SDPFmtpLine: "minptime=10;useinbandfec=1"}, "overlord-audio-"+string(kind), "overlord-"+string(kind), ) if err != nil { _ = pc.Close() return nil, fmt.Errorf("new audio track: %w", err) } + audioWriter, err = newOpusAudioWriter(audioTrack) + if err != nil { + _ = pc.Close() + return nil, fmt.Errorf("new opus encoder: %w", err) + } tx, err := pc.AddTransceiverFromTrack(audioTrack, webrtc.RTPTransceiverInit{ Direction: webrtc.RTPTransceiverDirectionSendonly, }) @@ -256,7 +263,7 @@ func Start(ctx context.Context, kind Kind, opts Options) (*Publisher, error) { registerVideoWriter(kind, writerID, &h264TrackWriter{t: videoTrack}) } if audioTrack != nil { - registerAudioWriter(kind, writerID, newPCMUAudioWriter(audioTrack)) + registerAudioWriter(kind, writerID, audioWriter) } log.Printf("webrtcpub: WHIP[%s] session established (resource=%s)", kind, resourceURL) return pub, nil diff --git a/Overlord-Client/cmd/agent/wire/protocol.go b/Overlord-Client/cmd/agent/wire/protocol.go index bb40868..f06aca9 100644 --- a/Overlord-Client/cmd/agent/wire/protocol.go +++ b/Overlord-Client/cmd/agent/wire/protocol.go @@ -107,11 +107,13 @@ type ClientLogsResult struct { } type FrameHeader struct { - Monitor int `msgpack:"monitor"` - FPS int `msgpack:"fps"` - Format string `msgpack:"format"` - Backstage bool `msgpack:"backstage,omitempty"` - Webcam bool `msgpack:"webcam,omitempty"` + Monitor int `msgpack:"monitor"` + FPS int `msgpack:"fps"` + Format string `msgpack:"format"` + Width int `msgpack:"width,omitempty"` + Height int `msgpack:"height,omitempty"` + Backstage bool `msgpack:"backstage,omitempty"` + Webcam bool `msgpack:"webcam,omitempty"` } type Frame struct { @@ -120,6 +122,20 @@ type Frame struct { Data []byte `msgpack:"data"` } +type DesktopStreamStats struct { + Type string `msgpack:"type"` + FPS int `msgpack:"fps"` + Format string `msgpack:"format"` + Bytes int `msgpack:"bytes"` + Width int `msgpack:"width"` + Height int `msgpack:"height"` + CaptureMs float64 `msgpack:"captureMs"` + EncodeMs float64 `msgpack:"encodeMs"` + SendMs float64 `msgpack:"sendMs"` + TotalMs float64 `msgpack:"totalMs"` + Transport string `msgpack:"transport"` +} + type FrameAck struct { Type string `msgpack:"type"` } @@ -381,9 +397,9 @@ type BackstageInstalledApp struct { } type BackstageInstalledAppsResult struct { - Type string `msgpack:"type"` + Type string `msgpack:"type"` Apps []BackstageInstalledApp `msgpack:"apps"` - Done bool `msgpack:"done"` + Done bool `msgpack:"done"` } type BackstageBrowserLaunchStatus struct { @@ -416,7 +432,7 @@ type BackstageMonitorEntry struct { } type BackstageWindowListResult struct { - Type string `msgpack:"type"` + Type string `msgpack:"type"` Windows []BackstageWindowEntry `msgpack:"windows"` Monitors []BackstageMonitorEntry `msgpack:"monitors"` } diff --git a/Overlord-Client/go.mod b/Overlord-Client/go.mod index 399880e..55228f9 100644 --- a/Overlord-Client/go.mod +++ b/Overlord-Client/go.mod @@ -20,6 +20,7 @@ require ( github.com/pion/rtcp v1.2.15 github.com/pion/webrtc/v4 v4.0.7 github.com/tetratelabs/wazero v1.12.0 + github.com/thesyncim/gopus v0.1.1 golang.org/x/crypto v0.49.0 golang.org/x/sys v0.44.0 ) diff --git a/Overlord-Client/go.sum b/Overlord-Client/go.sum index 0b8cb8f..926f288 100644 --- a/Overlord-Client/go.sum +++ b/Overlord-Client/go.sum @@ -79,6 +79,8 @@ github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOf github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/tetratelabs/wazero v1.12.0 h1:DuWcpNu/FzgEXgGBDp8J1Spc+CWOvvtvVyjKlaZopYU= github.com/tetratelabs/wazero v1.12.0/go.mod h1:LvKtzl2RqO4gyF27BiXU+nKAjcV8f38U+kP/q2vgxh0= +github.com/thesyncim/gopus v0.1.1 h1:SM58S7VZGnEZ3ZR0pX5AoC33//LBxJhKkvm15MbBmik= +github.com/thesyncim/gopus v0.1.1/go.mod h1:orRqwrGs5gqYRRnhqwI0Y3liqQTeDkreUpra+Kv9bQc= github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IUPn0Bjt8= github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok= github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= diff --git a/Overlord-Server/public/assets/backstage.js b/Overlord-Server/public/assets/backstage.js index 60874a4..8217927 100644 --- a/Overlord-Server/public/assets/backstage.js +++ b/Overlord-Server/public/assets/backstage.js @@ -69,9 +69,29 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s }, reconnectDelay); } const latencyEl = document.getElementById("latencyDisplay"); + const networkStats = document.getElementById("networkStats"); let lastInputSentAt = 0; let lastLatencyMs = 0; + function updateNetworkStats(stats) { + if (!networkStats || !stats) return; + const media = stats.video || stats.audio; + const parts = []; + if (Number.isFinite(media?.bitrateMbps)) parts.push(`${media.bitrateMbps.toFixed(1)} Mbps`); + if (Number.isFinite(stats.rttMs)) parts.push(`${Math.round(stats.rttMs)} ms`); + if (Number.isFinite(media?.lossPercent)) parts.push(`${media.lossPercent.toFixed(1)}% loss`); + const route = [stats.protocol, stats.route].filter(Boolean).join("/"); + if (route) parts.push(route); + networkStats.textContent = parts.join(" · ") || "Connected"; + const details = []; + if (media?.codec) details.push(`Codec: ${media.codec}`); + if (media?.width && media?.height) details.push(`Video: ${media.width}×${media.height}${media.framesPerSecond ? ` @ ${Math.round(media.framesPerSecond)} FPS` : ""}`); + if (Number.isFinite(media?.jitterMs)) details.push(`Jitter: ${media.jitterMs.toFixed(1)} ms`); + if (Number.isFinite(media?.jitterBufferMs)) details.push(`Jitter buffer: ${media.jitterBufferMs.toFixed(1)} ms`); + if (Number.isFinite(media?.framesDropped)) details.push(`Frames dropped: ${media.framesDropped}`); + networkStats.title = details.join("\n"); + } + const displaySelect = document.getElementById("displaySelect"); const refreshBtn = document.getElementById("refreshDisplays"); const startBtn = document.getElementById("startBtn"); @@ -771,6 +791,10 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s webrtcActive = !!active; if (canvas) canvas.style.display = active ? "none" : "block"; if (webrtcVideo) webrtcVideo.style.display = active ? "block" : "none"; + if (!active && networkStats) { + networkStats.textContent = "--"; + networkStats.title = ""; + } } function onWebrtcState(label, state) { @@ -823,6 +847,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s whepPath, videoEl: webrtcVideo, onState: (state) => onWebrtcState("WebRTC Relayed", state), + onStats: (stats) => updateNetworkStats({ ...stats, route: "Server" }), }); try { await whepClient.start(); @@ -846,6 +871,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s } }, onState: (state) => onWebrtcState("WebRTC P2P", state), + onStats: updateNetworkStats, }); try { await p2pClient.start(); diff --git a/Overlord-Server/public/assets/remotedesktop.js b/Overlord-Server/public/assets/remotedesktop.js index b49c63e..6bce8a0 100644 --- a/Overlord-Server/public/assets/remotedesktop.js +++ b/Overlord-Server/public/assets/remotedesktop.js @@ -65,6 +65,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s const duplicationCtrl = document.getElementById("duplicationCtrl"); const streamProfileSelect = document.getElementById("streamProfileSelect"); const streamProfileDetail = document.getElementById("streamProfileDetail"); + const bitrateSelect = document.getElementById("bitrateSelect"); const smoothingSlider = document.getElementById("smoothingSlider"); const smoothingValue = document.getElementById("smoothingValue"); const qualitySlider = document.getElementById("qualitySlider"); @@ -78,6 +79,14 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s const agentFps = document.getElementById("agentFps"); const viewerFps = document.getElementById("viewerFps"); const inputLatency = document.getElementById("inputLatency"); + const networkStats = document.getElementById("networkStats"); + const diagnosticsBtn = document.getElementById("diagnosticsBtn"); + const diagnosticsHud = document.getElementById("diagnosticsHud"); + const diagnosticsClose = document.getElementById("diagnosticsClose"); + const diagnosticEls = Object.fromEntries([ + "Summary", "Transport", "Codec", "Resolution", "Capture", "Encode", "Send", "AgentTotal", + "Bitrate", "Rtt", "LossJitter", "JitterBuffer", "Decode", "Render", "Queue", "Dropped", "Fps", "Input", + ].map((name) => [name, document.getElementById(`diag${name}`)])); const statusEl = document.getElementById("streamStatus"); const clipboardSyncCtrl = document.getElementById("clipboardSyncCtrl"); const privacyCtrl = document.getElementById("privacyCtrl"); @@ -111,6 +120,10 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s let frameDecodeBusy = false; let pendingFrame = null; let videoDecoder = null; + let videoDecoderConfigKey = ""; + let h264StreamCodec = ""; + let canvasDecodePressure = false; + let h264PendingTimings = []; let h264TimestampUs = 0; let prefersH264 = typeof VideoDecoder === "function"; let h264LowFpsStreak = 0; @@ -129,6 +142,22 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s const mouseMoveIntervalMs = 33; const inputBackpressureBytes = 256 * 1024; let lastMoveSentAt = 0; + const diagnostics = { + agent: null, + network: null, + decodeMs: null, + renderMs: null, + decodeQueue: 0, + coalescedFrames: 0, + currentAgentFps: null, + currentViewerFps: null, + wsBitrateMbps: null, + wsBytes: 0, + wsWindowStartedAt: performance.now(), + codec: "", + width: 0, + height: 0, + }; let clipboardSyncTimer = null; let lastClipboardText = ""; @@ -149,6 +178,9 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s function resetH264RuntimeState() { h264TimestampUs = 0; + h264StreamCodec = ""; + h264PendingTimings = []; + diagnostics.decodeQueue = 0; h264LowFpsStreak = 0; h264FirstFrameAt = 0; h264FramesSeen = 0; @@ -396,6 +428,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s ? `${settings.resolution}:${settings.targetFps}` : savedStreamProfile); setSelectValue(streamProfileSelect, savedStreamProfile); + setSelectValue(bitrateSelect, settings.bitrateMbps); setSelectValue(webrtcMode, settings.webrtcMode); setSelectValue(audioTransport, settings.audioTransport); setSelectValue(recordMode, settings.recordMode); @@ -428,6 +461,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s resolution: String(profile.maxHeight), targetFps: String(profile.fps), quality: Number(qualitySlider?.value || 90), + bitrateMbps: Number(bitrateSelect?.value || 0), preferH264: !!prefersH264, webrtcMode: getWebrtcMode(), mouse: !!mouseCtrl?.checked, @@ -469,6 +503,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s function updateFpsDisplay(agentValue) { if (agentValue !== undefined && agentValue !== null && agentFps) { agentFps.textContent = String(agentValue); + diagnostics.currentAgentFps = Number(agentValue) || null; } const now = performance.now(); renderCount += 1; @@ -476,6 +511,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s if (elapsed >= 1000 && viewerFps) { const fps = Math.round((renderCount * 1000) / elapsed); viewerFps.textContent = String(fps); + diagnostics.currentViewerFps = fps; renderCount = 0; renderWindowStart = now; } @@ -827,6 +863,142 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s } } + function finite(value) { + return Number.isFinite(value) ? Number(value) : null; + } + + function smoothed(previous, next, weight = 0.25) { + const value = finite(next); + if (value == null) return previous; + return previous == null ? value : previous * (1 - weight) + value * weight; + } + + function msText(value) { + const number = finite(value); + return number == null ? "-- ms" : `${number < 10 ? number.toFixed(1) : Math.round(number)} ms`; + } + + function setDiagnosticsVisible(visible) { + if (!diagnosticsHud) return; + diagnosticsHud.classList.toggle("hidden", !visible); + diagnosticsBtn?.setAttribute("aria-pressed", visible ? "true" : "false"); + try { localStorage.setItem("overlord.rd.statsHud", visible ? "1" : "0"); } catch {} + } + + function renderDiagnostics() { + if (!diagnosticsHud) return; + const agent = diagnostics.agent || {}; + const network = diagnostics.network || {}; + const media = network.video || network.audio || {}; + const mode = getWebrtcMode(); + const transport = mode === "p2p" ? "WebRTC P2P" : mode === "relayed" ? "WebRTC server" : "Canvas WebSocket"; + const bitrate = finite(media.bitrateMbps) ?? finite(diagnostics.wsBitrateMbps); + const decodeMs = finite(media.decodeMs) ?? finite(diagnostics.decodeMs); + const renderMs = finite(media.processingDelayMs) ?? finite(diagnostics.renderMs); + const dropped = finite(media.framesDropped) ?? 0; + const queue = mode === "off" ? diagnostics.decodeQueue : null; + const fps = finite(agent.fps) ?? diagnostics.currentAgentFps; + const viewerRate = finite(media.framesPerSecond) ?? diagnostics.currentViewerFps; + const frameBudget = 1000 / Math.max(1, fps || viewerRate || 30); + + let summary = "Pipeline healthy"; + let severity = "ok"; + if (streamState !== "streaming" && streamState !== "stalled") { + summary = "Stream is not active"; + severity = "warn"; + } else if (finite(agent.captureMs) > frameBudget * 0.75) { + summary = "Capture is limiting frame rate"; + severity = "bad"; + } else if (finite(agent.encodeMs) > frameBudget * 0.75) { + summary = "Encoder is limiting frame rate"; + severity = "bad"; + } else if (decodeMs != null && decodeMs > frameBudget * 0.75) { + const viewerKeepingUp = fps != null && viewerRate != null && viewerRate >= fps * 0.9; + summary = viewerKeepingUp + ? `Decoder buffering adds ${Math.round(decodeMs)} ms` + : "Viewer decoder throughput is limiting FPS"; + severity = viewerKeepingUp ? "warn" : "bad"; + } else if ((finite(media.lossPercent) ?? 0) > 3 || (finite(network.rttMs) ?? 0) > 150 || (finite(media.jitterMs) ?? 0) > 35) { + summary = "Network conditions are causing delay"; + severity = "bad"; + } else if ((queue ?? 0) > 2 || (renderMs != null && renderMs > frameBudget)) { + summary = "Viewer render queue is backing up"; + severity = "warn"; + } else if (!diagnostics.agent && !diagnostics.network) { + summary = "Waiting for stream telemetry"; + severity = "warn"; + } + + diagnosticEls.Summary.textContent = summary; + diagnosticEls.Summary.dataset.severity = severity; + diagnosticEls.Transport.textContent = transport; + diagnosticEls.Codec.textContent = String(media.codec || agent.format || diagnostics.codec || "--").toUpperCase(); + const width = finite(media.width) ?? diagnostics.width; + const height = finite(media.height) ?? diagnostics.height; + diagnosticEls.Resolution.textContent = width && height ? `${width}×${height}` : "--"; + diagnosticEls.Capture.textContent = msText(agent.captureMs); + diagnosticEls.Encode.textContent = msText(agent.encodeMs); + diagnosticEls.Send.textContent = msText(agent.sendMs); + diagnosticEls.AgentTotal.textContent = msText(agent.totalMs); + diagnosticEls.Bitrate.textContent = bitrate == null ? "-- Mbps" : `${bitrate.toFixed(2)} Mbps`; + diagnosticEls.Rtt.textContent = msText(network.rttMs); + diagnosticEls.LossJitter.textContent = `${finite(media.lossPercent)?.toFixed(1) ?? "--"}% / ${msText(media.jitterMs)}`; + diagnosticEls.JitterBuffer.textContent = msText(media.jitterBufferMs); + diagnosticEls.Decode.textContent = msText(decodeMs); + diagnosticEls.Render.textContent = msText(renderMs); + diagnosticEls.Queue.textContent = queue == null ? "managed by browser" : String(queue); + diagnosticEls.Dropped.textContent = `${Math.round(dropped)} / ${diagnostics.coalescedFrames}`; + diagnosticEls.Fps.textContent = `${fps == null ? "--" : Math.round(fps)} → ${viewerRate == null ? "--" : Math.round(viewerRate)}`; + diagnosticEls.Input.textContent = msText(latencyAvg); + } + + diagnosticsBtn?.addEventListener("click", () => setDiagnosticsVisible(diagnosticsHud?.classList.contains("hidden"))); + diagnosticsClose?.addEventListener("click", () => setDiagnosticsVisible(false)); + networkStats?.addEventListener("click", () => setDiagnosticsVisible(true)); + try { setDiagnosticsVisible(localStorage.getItem("overlord.rd.statsHud") === "1"); } catch {} + setInterval(renderDiagnostics, 250); + + function updateNetworkStats(stats) { + if (!networkStats || !stats) return; + diagnostics.network = stats; + const media = stats.video || stats.audio; + const parts = []; + if (Number.isFinite(media?.bitrateMbps)) parts.push(`${media.bitrateMbps.toFixed(1)} Mbps`); + if (Number.isFinite(stats.rttMs)) parts.push(`${Math.round(stats.rttMs)} ms`); + if (Number.isFinite(media?.lossPercent)) parts.push(`${media.lossPercent.toFixed(1)}% loss`); + const route = [stats.protocol, stats.route].filter(Boolean).join("/"); + if (route) parts.push(route); + networkStats.textContent = parts.join(" · ") || "Connected"; + const details = []; + if (media?.codec) details.push(`Codec: ${media.codec}`); + if (media?.width && media?.height) details.push(`Video: ${media.width}×${media.height}${media.framesPerSecond ? ` @ ${Math.round(media.framesPerSecond)} FPS` : ""}`); + if (Number.isFinite(media?.jitterMs)) details.push(`Jitter: ${media.jitterMs.toFixed(1)} ms`); + if (Number.isFinite(media?.jitterBufferMs)) details.push(`Jitter buffer: ${media.jitterBufferMs.toFixed(1)} ms`); + if (Number.isFinite(media?.framesDropped)) details.push(`Frames dropped: ${media.framesDropped}`); + if (Number.isFinite(stats.availableIncomingMbps)) details.push(`Available: ${stats.availableIncomingMbps.toFixed(1)} Mbps`); + networkStats.title = details.join("\n"); + renderDiagnostics(); + } + + function handleDesktopStreamStats(stats) { + const previous = diagnostics.agent || {}; + diagnostics.agent = { + ...stats, + captureMs: smoothed(finite(previous.captureMs), stats.captureMs), + encodeMs: smoothed(finite(previous.encodeMs), stats.encodeMs), + sendMs: smoothed(finite(previous.sendMs), stats.sendMs), + totalMs: smoothed(finite(previous.totalMs), stats.totalMs), + }; + diagnostics.currentAgentFps = finite(stats.fps); + diagnostics.codec = String(stats.format || diagnostics.codec || ""); + diagnostics.width = finite(stats.width) || diagnostics.width; + diagnostics.height = finite(stats.height) || diagnostics.height; + if (agentFps && diagnostics.currentAgentFps != null) { + agentFps.textContent = String(Math.round(diagnostics.currentAgentFps)); + } + renderDiagnostics(); + } + function clearOfflineTimer() { if (offlineTimer) { clearTimeout(offlineTimer); @@ -919,10 +1091,10 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s if (mode === "relayed") { sendCmd("desktop_start", { webrtc: true }); } else if (mode === "p2p") { - sendCmd("desktop_start", {}); + sendCmd("desktop_start", { canvasFlowControl: false }); startP2P(); } else { - sendCmd("desktop_start", {}); + sendCmd("desktop_start", { canvasFlowControl: true }); } } } else { @@ -1043,17 +1215,19 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s return; } const msg = { type, ...payload }; - rdDebug("send", { - msg, - readyState: ws.readyState, - streamState, - desiredStreaming, - display: displaySelect?.value ?? "", - quality: qualitySlider?.value ?? "", - streamProfile: streamProfileSelect?.value ?? "", - transport: getWebrtcMode(), - lastFrameAgeMs: lastFrameAt ? Math.round(performance.now() - lastFrameAt) : null, - }); + if (type !== "desktop_decode_pressure") { + rdDebug("send", { + msg, + readyState: ws.readyState, + streamState, + desiredStreaming, + display: displaySelect?.value ?? "", + quality: qualitySlider?.value ?? "", + streamProfile: streamProfileSelect?.value ?? "", + transport: getWebrtcMode(), + lastFrameAgeMs: lastFrameAt ? Math.round(performance.now() - lastFrameAt) : null, + }); + } ws.send(encodeMsgpack(msg)); } @@ -1061,6 +1235,11 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s webrtcActive = !!active; if (canvas) canvas.style.display = active ? "none" : "block"; if (webrtcVideo) webrtcVideo.style.display = active ? "block" : "none"; + if (!active && networkStats) { + networkStats.textContent = "--"; + networkStats.title = ""; + diagnostics.network = null; + } } function onWebrtcState(label, s) { @@ -1084,15 +1263,22 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s stopWebrtcFrameTicker(); webrtcFpsCount = 0; webrtcFpsWindowStart = performance.now(); - const tick = (now) => { + const tick = (now, metadata = {}) => { markFrameReceived(); + if (Number.isFinite(metadata.processingDuration)) { + diagnostics.decodeMs = smoothed(diagnostics.decodeMs, metadata.processingDuration * 1000); + } + if (Number.isFinite(metadata.receiveTime) && Number.isFinite(metadata.expectedDisplayTime)) { + diagnostics.renderMs = smoothed(diagnostics.renderMs, Math.max(0, metadata.expectedDisplayTime - metadata.receiveTime)); + } + diagnostics.width = webrtcVideo.videoWidth || diagnostics.width; + diagnostics.height = webrtcVideo.videoHeight || diagnostics.height; webrtcFpsCount += 1; const elapsed = now - webrtcFpsWindowStart; if (elapsed >= 1000) { const fps = Math.round((webrtcFpsCount * 1000) / elapsed); - // Agent and viewer FPS are effectively equal in WebRTC mode — the - // browser renders every frame the agent sends, no decode skipping. - updateFpsDisplay(fps); + diagnostics.currentViewerFps = fps; + updateFpsDisplay(); webrtcFpsCount = 0; webrtcFpsWindowStart = now; } else { @@ -1116,6 +1302,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s whepPath, videoEl: webrtcVideo, onState: (s) => onWebrtcState("WebRTC Relayed", s), + onStats: (stats) => updateNetworkStats({ ...stats, route: "Server" }), }); try { await whepClient.start(); @@ -1139,6 +1326,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s } }, onState: (s) => onWebrtcState("WebRTC P2P", s), + onStats: updateNetworkStats, }); try { await p2pClient.start(); @@ -1306,6 +1494,18 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s sendCmd("desktop_set_profile", profile); } + function pushBitrate() { + const bitrateMbps = Math.max(0, Math.min(50, Number.parseInt(bitrateSelect?.value || "0", 10) || 0)); + sendCmd("desktop_set_bitrate", { bitrateMbps }); + } + + if (bitrateSelect) { + bitrateSelect.addEventListener("change", function () { + pushBitrate(); + sharedSettingsSaver.scheduleSave(); + }); + } + function requestEncoderCapabilities() { if (streamProfileDetail) streamProfileDetail.textContent = "Checking hardware encoder profiles…"; sendCmd("desktop_encoder_capabilities", { @@ -1419,6 +1619,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s ensureDuplicationForH264(); } pushStreamProfile(); + pushBitrate(); desiredStreaming = true; lastFrameAt = 0; firstFrameLogged = false; @@ -1429,10 +1630,10 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s sendCmd("desktop_start", { webrtc: true }); } else if (mode === "p2p") { // Kick off the P2P offer asynchronously; capture starts in parallel. - sendCmd("desktop_start", {}); + sendCmd("desktop_start", { canvasFlowControl: false }); startP2P(); } else { - sendCmd("desktop_start", {}); + sendCmd("desktop_start", { canvasFlowControl: true }); } if (audioCtrl && audioCtrl.checked) { connectAudio(); @@ -1644,6 +1845,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s } function destroyVideoDecoder() { + updateCanvasDecodePressure(true); if (!videoDecoder) return; try { videoDecoder.close(); @@ -1651,6 +1853,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s // Ignore close errors when decoder is already shutting down. } videoDecoder = null; + videoDecoderConfigKey = ""; resetH264RuntimeState(); } @@ -1724,6 +1927,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s sendCmd("desktop_start", { source: "rd_viewer", reason: "h264_recovery_restart", + canvasFlowControl: getWebrtcMode() === "off", }); const q = Number(qualitySlider?.value) || 90; sendCmd("desktop_set_quality", { @@ -1769,16 +1973,39 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s return false; } - function ensureVideoDecoder() { - if (videoDecoder) { + function h264CodecFromAnnexB(data) { + for (let i = 0; i + 7 < data.length; i++) { + let startCodeLength = 0; + if (data[i] === 0 && data[i + 1] === 0 && data[i + 2] === 1) startCodeLength = 3; + else if (data[i] === 0 && data[i + 1] === 0 && data[i + 2] === 0 && data[i + 3] === 1) startCodeLength = 4; + if (!startCodeLength) continue; + const nal = i + startCodeLength; + if ((data[nal] & 0x1f) !== 7 || nal + 3 >= data.length) continue; + return `avc1.${[data[nal + 1], data[nal + 2], data[nal + 3]] + .map((value) => value.toString(16).padStart(2, "0")) + .join("")}`; + } + return ""; + } + + function ensureVideoDecoder(h264Bytes, width, height) { + const detectedCodec = h264CodecFromAnnexB(h264Bytes); + const codec = detectedCodec || h264StreamCodec || "avc1.4d0034"; + const codedWidth = Math.max(0, Number(width) || 0); + const codedHeight = Math.max(0, Number(height) || 0); + const configKey = `${codec}:${codedWidth}x${codedHeight}`; + if (videoDecoder && videoDecoderConfigKey === configKey) { return true; } if (typeof VideoDecoder !== "function") { return false; } try { + if (videoDecoder) destroyVideoDecoder(); videoDecoder = new VideoDecoder({ output: (frame) => { + const outputStartedAt = performance.now(); + const timing = h264PendingTimings.shift(); const width = frame.displayWidth || frame.codedWidth || frameWidth; const height = frame.displayHeight || frame.codedHeight || frameHeight; if (width > 0 && height > 0 && (canvas.width !== width || canvas.height !== height)) { @@ -1792,12 +2019,34 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s } finally { frame.close(); } + const renderedAt = performance.now(); + if (timing) { + diagnostics.decodeMs = smoothed(diagnostics.decodeMs, Math.max(0, outputStartedAt - timing.decodeStartedAt)); + diagnostics.renderMs = smoothed(diagnostics.renderMs, Math.max(0, renderedAt - timing.receivedAt)); + } + diagnostics.decodeQueue = videoDecoder?.decodeQueueSize || h264PendingTimings.length; + diagnostics.width = width; + diagnostics.height = height; + updateFpsDisplay(); + updateCanvasDecodePressure(); }, error: (err) => { console.warn("rd: h264 decoder error", err); + updateCanvasDecodePressure(true); }, }); - videoDecoder.configure({ codec: "avc1.42E01E", optimizeForLatency: true }); + videoDecoder.addEventListener("dequeue", () => { + diagnostics.decodeQueue = videoDecoder?.decodeQueueSize || 0; + updateCanvasDecodePressure(); + }); + const config = { codec, optimizeForLatency: true }; + if (codedWidth > 0 && codedHeight > 0) { + config.codedWidth = codedWidth; + config.codedHeight = codedHeight; + } + videoDecoder.configure(config); + videoDecoderConfigKey = configKey; + h264StreamCodec = codec; return true; } catch (err) { console.warn("rd: h264 decoder unavailable", err); @@ -1807,22 +2056,72 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s } } - async function processFrameBuffer(buf) { + function recordWsFrameBytes(byteLength) { + diagnostics.wsBytes += Math.max(0, Number(byteLength) || 0); + const now = performance.now(); + const elapsed = now - diagnostics.wsWindowStartedAt; + if (elapsed >= 1000) { + const rate = (diagnostics.wsBytes * 8) / (elapsed * 1000); + diagnostics.wsBitrateMbps = smoothed(diagnostics.wsBitrateMbps, rate, 0.4); + diagnostics.wsBytes = 0; + diagnostics.wsWindowStartedAt = now; + } + } + + function recordCanvasFrameTiming(receivedAt, decodeStartedAt) { + const renderedAt = performance.now(); + diagnostics.decodeMs = smoothed(diagnostics.decodeMs, Math.max(0, renderedAt - decodeStartedAt)); + diagnostics.renderMs = smoothed(diagnostics.renderMs, Math.max(0, renderedAt - receivedAt)); + diagnostics.decodeQueue = videoDecoder?.decodeQueueSize || (pendingFrame ? 1 : 0); + diagnostics.width = frameWidth || canvas.width || diagnostics.width; + diagnostics.height = frameHeight || canvas.height || diagnostics.height; + } + + function updateCanvasDecodePressure(forceRelease = false) { + const queueSize = videoDecoder?.decodeQueueSize || 0; + const shouldApply = !forceRelease && getWebrtcMode() === "off" && queueSize > 3; + const shouldRelease = forceRelease || getWebrtcMode() !== "off" || queueSize <= 1; + let next = canvasDecodePressure; + if (!canvasDecodePressure && shouldApply) next = true; + else if (canvasDecodePressure && shouldRelease) next = false; + if (next === canvasDecodePressure) return; + canvasDecodePressure = next; + sendCmd("desktop_decode_pressure", { active: next, queueSize }); + } + + async function processFrameBuffer(buf, receivedAt = performance.now()) { + const decodeStartedAt = performance.now(); const fps = buf[5]; const format = buf[6]; + const headerLength = buf[3] >= 2 && buf.length >= 12 ? 12 : 8; + const encodedWidth = headerLength === 12 ? buf[8] | (buf[9] << 8) : 0; + const encodedHeight = headerLength === 12 ? buf[10] | (buf[11] << 8) : 0; + if (encodedWidth > 0 && encodedHeight > 0) { + frameWidth = encodedWidth; + frameHeight = encodedHeight; + diagnostics.width = encodedWidth; + diagnostics.height = encodedHeight; + } if (format === 1) { - const jpegBytes = buf.slice(8); + const jpegBytes = buf.slice(headerLength); setCodecModeLabel("jpeg", "active"); + diagnostics.codec = "jpeg"; await drawJpegSlice(jpegBytes, null); + recordCanvasFrameTiming(receivedAt, decodeStartedAt); updateFpsDisplay(fps); + updateCanvasDecodePressure(true); return; } if (format === 2 || format === 3) { setCodecModeLabel(format === 3 ? "raw" : "jpeg", format === 3 ? "blocks" : "blocks"); - if (buf.length < 16) return; - const dv = new DataView(buf.buffer, 8); + diagnostics.codec = format === 3 ? "raw blocks" : "jpeg blocks"; + if (buf.length < headerLength + 8) { + updateCanvasDecodePressure(true); + return; + } + const dv = new DataView(buf.buffer, buf.byteOffset + headerLength, buf.byteLength - headerLength); let pos = 0; const width = dv.getUint16(pos, true); pos += 2; @@ -1857,7 +2156,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s pos += 2; const len = dv.getUint32(pos, true); pos += 4; - const start = 8 + pos; + const start = headerLength + pos; const end = start + len; if (end > buf.length) break; const slice = buf.subarray(start, end); @@ -1872,15 +2171,22 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s } updateFpsDisplay(fps); + recordCanvasFrameTiming(receivedAt, decodeStartedAt); + updateCanvasDecodePressure(true); return; } if (format === 4) { setCodecModeLabel("h264", "active"); - const h264Bytes = buf.slice(8); - if (!h264Bytes.length) return; - if (!ensureVideoDecoder()) { + diagnostics.codec = "h264"; + const h264Bytes = buf.slice(headerLength); + if (!h264Bytes.length) { + updateCanvasDecodePressure(true); + return; + } + if (!ensureVideoDecoder(h264Bytes, encodedWidth || frameWidth, encodedHeight || frameHeight)) { fallbackToJpegCodec("WebCodecs decoder unavailable"); + updateCanvasDecodePressure(true); return; } @@ -1905,6 +2211,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s h264LowFpsStreak >= H264_LOW_FPS_STREAK_LIMIT ) { fallbackToJpegCodec(`low h264 fps (${fps})`); + updateCanvasDecodePressure(true); return; } @@ -1916,10 +2223,13 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s }); h264TimestampUs += frameIntervalUs; try { + h264PendingTimings.push({ receivedAt, decodeStartedAt: performance.now() }); videoDecoder.decode(chunk); + diagnostics.decodeQueue = videoDecoder.decodeQueueSize; + updateCanvasDecodePressure(); h264KeyframeErrorStreak = 0; - updateFpsDisplay(fps); } catch (err) { + h264PendingTimings.pop(); if (isKeyframeRequiredError(err)) { h264KeyframeErrorStreak += 1; const now = Date.now(); @@ -1936,12 +2246,16 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s fallbackToJpegCodec("h264_keyframe_required_loop"); } } + updateCanvasDecodePressure(true); return; } console.warn("rd: h264 decode failed", err); fallbackToJpegCodec(err); + updateCanvasDecodePressure(true); } + return; } + updateCanvasDecodePressure(true); } function flushPendingFrame() { @@ -1951,7 +2265,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s const next = pendingFrame; pendingFrame = null; frameDecodeBusy = true; - processFrameBuffer(next).finally(() => { + processFrameBuffer(next.buf, next.receivedAt).finally(() => { frameDecodeBusy = false; if (pendingFrame) { flushPendingFrame(); @@ -1964,8 +2278,10 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s const buf = new Uint8Array(ev.data); if (isFramePacket(buf)) { markFrameReceived(); + recordWsFrameBytes(buf.byteLength); // Coalesce bursty arrivals so the renderer catches up to the newest frame. - pendingFrame = buf; + if (pendingFrame) diagnostics.coalescedFrames += 1; + pendingFrame = { buf, receivedAt: performance.now() }; flushPendingFrame(); return; } @@ -1975,6 +2291,10 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s applyEncoderCapabilities(msg); return; } + if (msg && msg.type === "desktop_stream_stats") { + handleDesktopStreamStats(msg); + return; + } if (msg && msg.type === "status" && msg.status) { handleStatus(msg); return; @@ -2014,6 +2334,10 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s applyEncoderCapabilities(msg); return; } + if (msg && msg.type === "desktop_stream_stats") { + handleDesktopStreamStats(msg); + return; + } if (msg && msg.type === "status" && msg.status) { handleStatus(msg); return; @@ -2045,6 +2369,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s if (qualitySlider) { pushQuality(qualitySlider.value); } + pushBitrate(); pushInputToggles(); pushCaptureToggles(); clearOfflineTimer(); @@ -2060,10 +2385,10 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s if (mode === "relayed") { sendCmd("desktop_start", { webrtc: true }); } else if (mode === "p2p") { - sendCmd("desktop_start", {}); + sendCmd("desktop_start", { canvasFlowControl: false }); startP2P(); } else { - sendCmd("desktop_start", {}); + sendCmd("desktop_start", { canvasFlowControl: true }); } } else { setStreamState("idle", "Stopped"); diff --git a/Overlord-Server/public/assets/webrtc-p2p.js b/Overlord-Server/public/assets/webrtc-p2p.js index b97022a..6949f26 100644 --- a/Overlord-Server/public/assets/webrtc-p2p.js +++ b/Overlord-Server/public/assets/webrtc-p2p.js @@ -10,6 +10,8 @@ // The server proxies these to the agent via its existing WS command channel // (sessionId is server-side state, not visible to the browser). // +import { WebRTCStatsSampler } from "./webrtc-stats.js"; + // Once SDP + ICE settle, the