From 095a119bf44e0d55bdbe0c6d218989e8d745393a Mon Sep 17 00:00:00 2001 From: KingKDot Date: Thu, 30 Jul 2026 00:22:08 -0500 Subject: [PATCH] major remote desktop improvements --- .env.example | 5 + .../cmd/agent/capture/amf_d3d11_windows.go | 3 +- Overlord-Client/cmd/agent/capture/capture.go | 103 +++++++++++++-- .../cmd/agent/capture/capture_test.go | 12 +- .../desktop_encoder_capabilities_windows.go | 19 ++- ...sktop_encoder_capabilities_windows_test.go | 21 +++ .../cmd/agent/capture/flowcontrol.go | 4 +- .../cmd/agent/capture/h264_codec_test.go | 26 ++++ .../cmd/agent/capture/h264_direct_windows.go | 13 +- .../capture/nvenc_d3d11_smoke_windows.go | 19 ++- .../cmd/agent/capture/qsv_d3d11_windows.go | 3 +- .../cmd/agent/capture/video_access_unit.go | 73 ++++++++++ .../agent/capture/video_access_unit_test.go | 38 ++++++ .../capture/video_keyframe_interval_test.go | 29 ++++ .../cmd/agent/capture/virtual_windows.go | 14 +- .../cmd/agent/capture/win_duplication_test.go | 11 +- Overlord-Client/cmd/agent/h264util/annexb.go | 45 +++++++ .../cmd/agent/h264util/annexb_test.go | 22 +++ Overlord-Client/cmd/agent/webrtcpub/state.go | 83 +++++++++--- .../cmd/agent/webrtcpub/state_test.go | 30 ++++- .../cmd/agent/webrtcpub/whip_pion.go | 2 +- Overlord-Server/public/assets/cert-banner.js | 14 +- .../public/assets/remotedesktop.js | 125 +++++++++++++----- Overlord-Server/public/assets/settings.js | 45 +++++++ Overlord-Server/public/settings.html | 40 ++++++ .../src/server/routes/assets-routes.test.ts | 15 +++ .../src/server/routes/assets-routes.ts | 11 +- .../src/server/routes/cert-download.test.ts | 70 ++++++++++ .../src/server/routes/misc-routes.ts | 4 +- .../server/ws-console-rd-backstage.test.ts | 36 ++++- .../src/server/ws-console-rd-backstage.ts | 35 +++-- .../remotedesktop-canvas-presentation.e2e.ts | 118 +++++++++++++++++ .../test/e2e/remotedesktop-input.e2e.ts | 4 +- README.md | 5 +- docker-compose.windows.yml | 7 +- 35 files changed, 984 insertions(+), 120 deletions(-) create mode 100644 Overlord-Client/cmd/agent/capture/desktop_encoder_capabilities_windows_test.go create mode 100644 Overlord-Client/cmd/agent/capture/video_access_unit.go create mode 100644 Overlord-Client/cmd/agent/capture/video_access_unit_test.go create mode 100644 Overlord-Client/cmd/agent/capture/video_keyframe_interval_test.go create mode 100644 Overlord-Client/cmd/agent/h264util/annexb.go create mode 100644 Overlord-Client/cmd/agent/h264util/annexb_test.go create mode 100644 Overlord-Server/src/server/routes/assets-routes.test.ts create mode 100644 Overlord-Server/src/server/routes/cert-download.test.ts create mode 100644 Overlord-Server/test/e2e/remotedesktop-canvas-presentation.e2e.ts diff --git a/.env.example b/.env.example index 269c05d..3934ddc 100644 --- a/.env.example +++ b/.env.example @@ -42,6 +42,11 @@ # OVERLORD_TLS_OFFLOAD=true # OVERLORD_HEALTHCHECK_URL=http://127.0.0.1:5173/health +# WebRTC TURN relay ports. Docker Desktop on Windows defaults to 40000-40040 +# because Windows commonly reserves ports in the 49152+ dynamic range. +# OVERLORD_TURN_MIN_PORT=40000 +# OVERLORD_TURN_MAX_PORT=40040 + # ============================================ # ADVANCED: Custom JWT secret # ============================================ diff --git a/Overlord-Client/cmd/agent/capture/amf_d3d11_windows.go b/Overlord-Client/cmd/agent/capture/amf_d3d11_windows.go index a1fa8c5..54bad45 100644 --- a/Overlord-Client/cmd/agent/capture/amf_d3d11_windows.go +++ b/Overlord-Client/cmd/agent/capture/amf_d3d11_windows.go @@ -46,7 +46,8 @@ func (b *amfD3D11TextureBackend) Encode(req h264D3D11TextureRequest) ([]byte, er b.device, b.inputWidth, b.inputHeight = req.Device, req.InputWidth, req.InputHeight b.encodeWidth, b.encodeHeight, b.fps, b.dxgiFormat = req.EncodeWidth, req.EncodeHeight, req.FPS, req.DXGIFormat } - forceIDR := req.ForceIDR || b.forceIDR.Swap(false) + pendingIDR := b.forceIDR.Swap(false) + forceIDR := req.ForceIDR || pendingIDR for attempts := 0; attempts < 2; attempts++ { result, outputSize, err := b.encoder.Encode(req.Texture, forceIDR, b.output) if err != nil { diff --git a/Overlord-Client/cmd/agent/capture/capture.go b/Overlord-Client/cmd/agent/capture/capture.go index c238644..d05fd98 100644 --- a/Overlord-Client/cmd/agent/capture/capture.go +++ b/Overlord-Client/cmd/agent/capture/capture.go @@ -108,19 +108,41 @@ func CaptureAndSend(ctx context.Context, env *rt.Env) error { time.Sleep(50 * time.Millisecond) } + codec := desktopCodec() + directCanvasVideo := (codec == "h264" || codec == "hevc") && + (codec != "h264" || !useDesktopSoftwareH264()) && + useDesktopDuplication() && + !(codec == "h264" && webrtcpub.IsActive(webrtcpub.KindDesktop)) + directSlotAcquired := false + if directCanvasVideo { + if !AcquireFrameSlot() { + return nil + } + directSlotAcquired = true + } + if frame, captureDur, encodeDur, used, err := tryBuildDirectH264Frame(display); used { if err != nil { + if directSlotAcquired { + ReleaseFrameSlot() + directSlotAcquired = false + } suppressDesktopHardwareH264(err) log.Printf("capture: direct video encode failed: %v; falling back to the software H.264 path", err) } else { emitDesktopCursor(ctx, env, display, frame.Header.Width, frame.Header.Height) if len(frame.Data) == 0 { + if directSlotAcquired { + ReleaseFrameSlot() + } consecutiveCaptureFails.Store(0) return nil } consecutiveCaptureFails.Store(0) - return sendCompletedFrame(ctx, env, frame, display, t0, captureDur, encodeDur) + return sendCompletedFrame(ctx, env, frame, display, t0, captureDur, encodeDur, directSlotAcquired) } + } else if directSlotAcquired { + ReleaseFrameSlot() } img, err := safeCaptureDisplay(display) @@ -247,7 +269,7 @@ func CaptureAndSend(ctx context.Context, env *rt.Env) error { return err } -func sendCompletedFrame(ctx context.Context, env *rt.Env, frame wire.Frame, display int, t0 time.Time, captureDur, encodeDur time.Duration) error { +func sendCompletedFrame(ctx context.Context, env *rt.Env, frame wire.Frame, display int, t0 time.Time, captureDur, encodeDur time.Duration, slotAcquired bool) error { now := time.Now() fps := frameFPS(now) if fps <= 0 { @@ -255,6 +277,9 @@ func sendCompletedFrame(ctx context.Context, env *rt.Env, frame wire.Frame, disp } frame.Header.FPS = fps if ctx.Err() != nil { + if slotAcquired { + ReleaseFrameSlot() + } return nil } if frame.Header.Format == "h264" && webrtcpub.IsActive(webrtcpub.KindDesktop) { @@ -274,10 +299,16 @@ func sendCompletedFrame(ctx context.Context, env *rt.Env, frame wire.Frame, disp statTotalNs.Add(totalDur.Nanoseconds()) statBytes.Add(int64(len(frame.Data))) emitDesktopStreamStats(ctx, env, frame, fps, captureDur, encodeDur, sendDur, totalDur, "webrtc") + if slotAcquired { + ReleaseFrameSlot() + } return nil } - if !AcquireFrameSlot() { - return nil + if !slotAcquired { + if !AcquireFrameSlot() { + return nil + } + slotAcquired = true } sendStart := time.Now() err := wire.WriteMsg(ctx, env.Conn, frame) @@ -675,6 +706,8 @@ func SetDesktopSoftwareH264(enabled bool) { } desktopHardwareFallbackUntil.Store(0) resetH264Encoder() + lastKeyframe.Store(0) + webrtcpub.RequestKeyframe(webrtcpub.KindDesktop) requestFullFrames(2) log.Printf("capture: desktop software h264 %v", enabled) } @@ -796,6 +829,25 @@ const ( backstageChangeThresh = 1 ) +var videoKeyframeEvery = configuredVideoKeyframeInterval() + +func configuredVideoKeyframeInterval() time.Duration { + raw := strings.TrimSpace(os.Getenv("OVERLORD_VIDEO_KEYFRAME_INTERVAL")) + if raw == "" || strings.EqualFold(raw, "off") || raw == "0" { + return 0 + } + if interval, err := time.ParseDuration(raw); err == nil { + if interval > 0 { + return interval + } + return 0 + } + if seconds, err := strconv.Atoi(raw); err == nil && seconds > 0 { + return time.Duration(seconds) * time.Second + } + return 0 +} + func buildFrame(img *image.RGBA, display int, quality int) (wire.Frame, time.Duration, error) { encStart := time.Now() width := img.Bounds().Dx() @@ -808,15 +860,22 @@ func buildFrame(img *image.RGBA, display int, quality int) (wire.Frame, time.Dur log.Printf("capture: hevc skipped for odd dimensions (%dx%d), falling back to h264/jpeg", width, height) codec = "h264" } else { - if webrtcpub.ConsumeKeyframeRequest(webrtcpub.KindDesktop) { + requestedKeyframe := webrtcpub.ConsumeKeyframeRequest(webrtcpub.KindDesktop) + if requestedKeyframe || lastKeyframe.Load() == 0 || + (videoKeyframeEvery > 0 && now.Sub(time.Unix(0, lastKeyframe.Load())) >= videoKeyframeEvery) { resetHEVCEncoder() } hevcBytes, err := encodeHEVCFrame(img) if err == nil && len(hevcBytes) > 0 { - lastKeyframe.Store(now.UnixNano()) + if retainKeyframeRequestUntilOutput(webrtcpub.KindDesktop, requestedKeyframe, "hevc", hevcBytes) { + lastKeyframe.Store(now.UnixNano()) + } statFullFrames.Add(1) return wire.Frame{Type: "frame", Header: wire.FrameHeader{Monitor: display, FPS: 0, Format: "hevc"}, Data: hevcBytes}, time.Since(encStart), nil } + if requestedKeyframe { + webrtcpub.RequestKeyframe(webrtcpub.KindDesktop) + } hevcWarnOnce.Do(func() { log.Printf("capture: hevc encode unavailable, falling back to h264/jpeg: %v (%s)", err, hevcAvailabilityDetail()) }) @@ -836,15 +895,22 @@ func buildFrame(img *image.RGBA, display int, quality int) (wire.Frame, time.Dur log.Printf("capture: h264 skipped for odd dimensions (%dx%d), falling back to jpeg", width, height) codec = "jpeg" } else { - if webrtcpub.ConsumeKeyframeRequest(webrtcpub.KindDesktop) { + requestedKeyframe := webrtcpub.ConsumeKeyframeRequest(webrtcpub.KindDesktop) + if requestedKeyframe || lastKeyframe.Load() == 0 || + (videoKeyframeEvery > 0 && now.Sub(time.Unix(0, lastKeyframe.Load())) >= videoKeyframeEvery) { resetH264Encoder() } h264Bytes, err := encodeH264Frame(img) if err == nil && len(h264Bytes) > 0 { - lastKeyframe.Store(now.UnixNano()) + if retainKeyframeRequestUntilOutput(webrtcpub.KindDesktop, requestedKeyframe, "h264", h264Bytes) { + lastKeyframe.Store(now.UnixNano()) + } statFullFrames.Add(1) return wire.Frame{Type: "frame", Header: wire.FrameHeader{Monitor: display, FPS: 0, Format: "h264"}, Data: h264Bytes}, time.Since(encStart), nil } + if requestedKeyframe { + webrtcpub.RequestKeyframe(webrtcpub.KindDesktop) + } h264WarnOnce.Do(func() { detail := h264AvailabilityDetail() if detail != "" { @@ -941,16 +1007,23 @@ func buildFramebackstage(img *image.RGBA, display int, quality int) (wire.Frame, log.Printf("backstage capture: h264 skipped for odd dimensions (%dx%d), falling back to jpeg", width, height) codec = "jpeg" } else { - if webrtcpub.ConsumeKeyframeRequest(webrtcpub.Kindbackstage) { + requestedKeyframe := webrtcpub.ConsumeKeyframeRequest(webrtcpub.Kindbackstage) + if requestedKeyframe || backstageLastKeyframe.Load() == 0 || + (videoKeyframeEvery > 0 && now.Sub(time.Unix(0, backstageLastKeyframe.Load())) >= videoKeyframeEvery) { resetH264Encoderbackstage() } h264Bytes, err := encodeH264Framebackstage(img) if err == nil && len(h264Bytes) > 0 { - backstageLastKeyframe.Store(now.UnixNano()) + if retainKeyframeRequestUntilOutput(webrtcpub.Kindbackstage, requestedKeyframe, "h264", h264Bytes) { + backstageLastKeyframe.Store(now.UnixNano()) + } statFullFrames.Add(1) frame := wire.Frame{Type: "frame", Header: wire.FrameHeader{Monitor: display, FPS: 0, Format: "h264", Backstage: true}, Data: h264Bytes} return frame, time.Since(encStart), nil } + if requestedKeyframe { + webrtcpub.RequestKeyframe(webrtcpub.Kindbackstage) + } backstageH264WarnOnce.Do(func() { detail := h264AvailabilityDetail() if detail != "" { @@ -1671,6 +1744,7 @@ func backstageCodec() string { } func SetDesktopQualityAndCodec(quality int, codec string) { + previousCodec := desktopCodec() if quality > 0 { if quality > 100 { quality = 100 @@ -1706,6 +1780,10 @@ func SetDesktopQualityAndCodec(quality int, codec string) { if s != "hevc" { resetHEVCEncoder() } + if previousCodec != s && (s == "h264" || s == "hevc") { + lastKeyframe.Store(0) + webrtcpub.RequestKeyframe(webrtcpub.KindDesktop) + } case "": default: desktopOverrideCodec.Store("jpeg") @@ -1715,6 +1793,7 @@ func SetDesktopQualityAndCodec(quality int, codec string) { } func SetBackstageQualityAndCodec(quality int, codec string) { + previousCodec := backstageCodec() if quality > 0 { if quality > 100 { quality = 100 @@ -1734,6 +1813,10 @@ func SetBackstageQualityAndCodec(quality int, codec string) { if s != "h264" { resetH264Encoderbackstage() } + if previousCodec != s && s == "h264" { + backstageLastKeyframe.Store(0) + webrtcpub.RequestKeyframe(webrtcpub.Kindbackstage) + } case "": default: backstageOverrideCodec.Store("jpeg") diff --git a/Overlord-Client/cmd/agent/capture/capture_test.go b/Overlord-Client/cmd/agent/capture/capture_test.go index 6b0f722..0693f25 100644 --- a/Overlord-Client/cmd/agent/capture/capture_test.go +++ b/Overlord-Client/cmd/agent/capture/capture_test.go @@ -456,16 +456,16 @@ func TestSetFrameFlowTargetFPSScalesHighFPS(t *testing.T) { t.Setenv("OVERLORD_DESKTOP_IN_FLIGHT_FRAMES", "") ResetFrameSlots() SetFrameFlowTargetFPS(60) - if got := activeFrameSlotLimit(); got != 2 { - t.Fatalf("expected 60 fps slot limit 2, got %d", got) + if got := activeFrameSlotLimit(); got != 4 { + t.Fatalf("expected 60 fps slot limit 4, got %d", got) } SetFrameFlowTargetFPS(120) - if got := activeFrameSlotLimit(); got != 4 { - t.Fatalf("expected 120 fps slot limit 4, got %d", got) + if got := activeFrameSlotLimit(); got != 8 { + t.Fatalf("expected 120 fps slot limit 8, got %d", got) } SetFrameFlowTargetFPS(240) - if got := activeFrameSlotLimit(); got != 8 { - t.Fatalf("expected 240 fps slot limit 8, got %d", got) + if got := activeFrameSlotLimit(); got != 12 { + t.Fatalf("expected 240 fps slot limit 12, got %d", got) } } diff --git a/Overlord-Client/cmd/agent/capture/desktop_encoder_capabilities_windows.go b/Overlord-Client/cmd/agent/capture/desktop_encoder_capabilities_windows.go index 7c6c2b5..278b7e2 100644 --- a/Overlord-Client/cmd/agent/capture/desktop_encoder_capabilities_windows.go +++ b/Overlord-Client/cmd/agent/capture/desktop_encoder_capabilities_windows.go @@ -123,6 +123,9 @@ func probeDesktopEncoderCapabilities(display int, bounds image.Rectangle) Deskto for _, probe := range probes { maxFPS := 0 for _, fps := range fpsOptions { + if fps > desktopProfileFPSCeiling(size.height) { + continue + } req := h264D3D11TextureRequest{ Device: unsafe.Pointer(device), Texture: unsafe.Pointer(texture), InputWidth: nativeWidth, InputHeight: nativeHeight, @@ -140,7 +143,7 @@ func probeDesktopEncoderCapabilities(display int, bounds image.Rectangle) Deskto continue } for _, fps := range fpsOptions { - if fps <= maxFPS { + if fps <= maxFPS && fps <= desktopProfileFPSCeiling(size.height) { key := desktopProfileKey(size.width, size.height, fps) providersByProfile[key] = appendUniqueString(providersByProfile[key], probe.name) } @@ -254,6 +257,20 @@ func safeDesktopProfiles(nativeWidth, nativeHeight int) []DesktopEncoderProfile return profiles } +// The probe repeatedly encodes an unchanged synthetic texture and excludes +// capture, conversion, transport, and browser decode. Bound its advertised +// result to practical end-to-end desktop profiles. +func desktopProfileFPSCeiling(height int) int { + switch { + case height <= 1080: + return 240 + case height <= 1440: + return 120 + default: + return 60 + } +} + func desktopProfileKey(width, height, fps int) string { return fmt.Sprintf("%dx%d@%d", width, height, fps) } diff --git a/Overlord-Client/cmd/agent/capture/desktop_encoder_capabilities_windows_test.go b/Overlord-Client/cmd/agent/capture/desktop_encoder_capabilities_windows_test.go new file mode 100644 index 0000000..7bfbb98 --- /dev/null +++ b/Overlord-Client/cmd/agent/capture/desktop_encoder_capabilities_windows_test.go @@ -0,0 +1,21 @@ +package capture + +import "testing" + +func TestDesktopProfileFPSCeiling(t *testing.T) { + tests := []struct { + height int + want int + }{ + {height: 720, want: 240}, + {height: 1080, want: 240}, + {height: 1440, want: 120}, + {height: 2160, want: 60}, + {height: 4320, want: 60}, + } + for _, test := range tests { + if got := desktopProfileFPSCeiling(test.height); got != test.want { + t.Fatalf("height %d: expected ceiling %d, got %d", test.height, test.want, got) + } + } +} diff --git a/Overlord-Client/cmd/agent/capture/flowcontrol.go b/Overlord-Client/cmd/agent/capture/flowcontrol.go index b9dd351..8ced535 100644 --- a/Overlord-Client/cmd/agent/capture/flowcontrol.go +++ b/Overlord-Client/cmd/agent/capture/flowcontrol.go @@ -57,8 +57,10 @@ func SetFrameFlowTargetFPS(fps int) { limit := defaultMaxInFlightFrames switch { case fps >= 180: - limit = 8 + limit = 12 case fps >= 120: + limit = 8 + case fps >= 60: limit = 4 } if env := strings.TrimSpace(os.Getenv("OVERLORD_DESKTOP_IN_FLIGHT_FRAMES")); env != "" { diff --git a/Overlord-Client/cmd/agent/capture/h264_codec_test.go b/Overlord-Client/cmd/agent/capture/h264_codec_test.go index 329ea5d..e18bffa 100644 --- a/Overlord-Client/cmd/agent/capture/h264_codec_test.go +++ b/Overlord-Client/cmd/agent/capture/h264_codec_test.go @@ -9,6 +9,9 @@ import ( "strings" "sync" "testing" + "time" + + "overlord-client/cmd/agent/webrtcpub" ) func resetCodecSelectionForTest() { @@ -18,6 +21,8 @@ func resetCodecSelectionForTest() { backstageOverrideCodec.Store("") desktopOverrideQuality.Store(0) backstageOverrideQuality.Store(0) + _ = webrtcpub.ConsumeKeyframeRequest(webrtcpub.KindDesktop) + _ = webrtcpub.ConsumeKeyframeRequest(webrtcpub.Kindbackstage) } func TestSetQualityAndCodec_H264FallbackDependsOnAvailability(t *testing.T) { @@ -72,6 +77,27 @@ func TestDesktopAndBackstageCodecSelectionsAreIndependent(t *testing.T) { } } +func TestSwitchingToH264RequestsRecoveryPoint(t *testing.T) { + if !h264Available() { + t.Skip("h264 is unavailable in this build") + } + t.Cleanup(resetCodecSelectionForTest) + resetCodecSelectionForTest() + + SetDesktopQualityAndCodec(80, "jpeg") + _ = webrtcpub.ConsumeKeyframeRequest(webrtcpub.KindDesktop) + lastKeyframe.Store(time.Now().UnixNano()) + + SetDesktopQualityAndCodec(80, "h264") + + if lastKeyframe.Load() != 0 { + t.Fatal("codec transition did not invalidate the previous keyframe timestamp") + } + if !webrtcpub.ConsumeKeyframeRequest(webrtcpub.KindDesktop) { + t.Fatal("codec transition did not request an H.264 recovery point") + } +} + func TestDesktopHardwareFailureDoesNotChangeBackstageCodec(t *testing.T) { t.Cleanup(func() { desktopHardwareFallbackUntil.Store(0) diff --git a/Overlord-Client/cmd/agent/capture/h264_direct_windows.go b/Overlord-Client/cmd/agent/capture/h264_direct_windows.go index d76742d..cf6ed65 100644 --- a/Overlord-Client/cmd/agent/capture/h264_direct_windows.go +++ b/Overlord-Client/cmd/agent/capture/h264_direct_windows.go @@ -13,7 +13,7 @@ func directVideoKeyframeDue(requested bool, nowNs, lastKeyframeNs int64) bool { if requested || lastKeyframeNs == 0 { return true } - return keyframeEvery > 0 && time.Duration(nowNs-lastKeyframeNs) >= keyframeEvery + return videoKeyframeEvery > 0 && time.Duration(nowNs-lastKeyframeNs) >= videoKeyframeEvery } func tryBuildDirectH264Frame(display int) (wire.Frame, time.Duration, time.Duration, bool, error) { @@ -22,8 +22,9 @@ func tryBuildDirectH264Frame(display int) (wire.Frame, time.Duration, time.Durat return wire.Frame{}, 0, 0, false, nil } now := time.Now() + requestedKeyframe := webrtcpub.ConsumeKeyframeRequest(webrtcpub.KindDesktop) forceKeyframe := directVideoKeyframeDue( - webrtcpub.ConsumeKeyframeRequest(webrtcpub.KindDesktop), + requestedKeyframe, now.UnixNano(), lastKeyframe.Load(), ) @@ -38,12 +39,18 @@ func tryBuildDirectH264Frame(display int) (wire.Frame, time.Duration, time.Durat data, width, height, captureDur, encodeDur, used, err = captureDisplayDXGIH264(display, forceKeyframe) } if !used || err != nil { + if requestedKeyframe { + webrtcpub.RequestKeyframe(webrtcpub.KindDesktop) + } return wire.Frame{}, captureDur, encodeDur, used, err } if len(data) == 0 { + if requestedKeyframe { + webrtcpub.RequestKeyframe(webrtcpub.KindDesktop) + } return wire.Frame{}, captureDur, encodeDur, true, nil } - if forceKeyframe { + if retainKeyframeRequestUntilOutput(webrtcpub.KindDesktop, requestedKeyframe, codec, data) { lastKeyframe.Store(now.UnixNano()) } statFullFrames.Add(1) diff --git a/Overlord-Client/cmd/agent/capture/nvenc_d3d11_smoke_windows.go b/Overlord-Client/cmd/agent/capture/nvenc_d3d11_smoke_windows.go index 1549271..5b0933b 100644 --- a/Overlord-Client/cmd/agent/capture/nvenc_d3d11_smoke_windows.go +++ b/Overlord-Client/cmd/agent/capture/nvenc_d3d11_smoke_windows.go @@ -122,9 +122,14 @@ static void nvenc_configure_h264_stream(NV_ENC_CONFIG *cfg, int fps) { (void)fps; cfg->gopLength = NVENC_INFINITE_GOPLENGTH; cfg->frameIntervalP = 1; + // The low-latency presets otherwise constrain an IDR to roughly the same + // bit budget as a P-frame. That makes recovery frames visibly pulse or + // smear under motion. Allow IDRs a larger transient budget while keeping + // the normal one-frame VBV/CBR latency behavior. + cfg->rcParams.lowDelayKeyFrameScale = 4; cfg->encodeCodecConfig.h264Config.idrPeriod = NVENC_INFINITE_GOPLENGTH; cfg->encodeCodecConfig.h264Config.repeatSPSPPS = 1; - cfg->encodeCodecConfig.h264Config.level = NV_ENC_LEVEL_H264_52; + cfg->encodeCodecConfig.h264Config.level = NV_ENC_LEVEL_AUTOSELECT; cfg->encodeCodecConfig.h264Config.h264VUIParameters.videoSignalTypePresentFlag = 1; cfg->encodeCodecConfig.h264Config.h264VUIParameters.videoFormat = NV_ENC_VUI_VIDEO_FORMAT_UNSPECIFIED; cfg->encodeCodecConfig.h264Config.h264VUIParameters.videoFullRangeFlag = 0; @@ -393,7 +398,7 @@ static nvenc_d3d11_texture_create_result nvenc_create_d3d11_texture_encoder(ID3D nvenc_release_texture_encoder(enc); return out; } - preset.presetCfg.profileGUID = codec == 1 ? NV_ENC_HEVC_PROFILE_MAIN_GUID : NV_ENC_H264_PROFILE_HIGH_GUID; + preset.presetCfg.profileGUID = codec == 1 ? NV_ENC_HEVC_PROFILE_MAIN_GUID : NV_ENC_H264_PROFILE_MAIN_GUID; preset.presetCfg.rcParams.rateControlMode = NV_ENC_PARAMS_RC_CBR; preset.presetCfg.rcParams.averageBitRate = (uint32_t)bitrate; preset.presetCfg.rcParams.maxBitRate = (uint32_t)bitrate; @@ -703,7 +708,7 @@ static nvenc_d3d11_create_result nvenc_create_d3d11_encoder(int width, int heigh nvenc_release_encoder(enc); return out; } - preset.presetCfg.profileGUID = codec == 1 ? NV_ENC_HEVC_PROFILE_MAIN_GUID : NV_ENC_H264_PROFILE_HIGH_GUID; + preset.presetCfg.profileGUID = codec == 1 ? NV_ENC_HEVC_PROFILE_MAIN_GUID : NV_ENC_H264_PROFILE_MAIN_GUID; preset.presetCfg.rcParams.rateControlMode = NV_ENC_PARAMS_RC_CBR; preset.presetCfg.rcParams.averageBitRate = (uint32_t)bitrate; preset.presetCfg.rcParams.maxBitRate = (uint32_t)bitrate; @@ -1160,7 +1165,7 @@ static nvenc_d3d11_smoke_result run_nvenc_d3d11_smoke_c(int width, int height, i return out; } - preset.presetCfg.profileGUID = NV_ENC_H264_PROFILE_HIGH_GUID; + preset.presetCfg.profileGUID = NV_ENC_H264_PROFILE_MAIN_GUID; preset.presetCfg.rcParams.rateControlMode = NV_ENC_PARAMS_RC_CBR; preset.presetCfg.rcParams.averageBitRate = (uint32_t)bitrate; preset.presetCfg.rcParams.maxBitRate = (uint32_t)bitrate; @@ -1961,7 +1966,8 @@ func encodeNativeH264D3D11Texture(stream string, device, texture unsafe.Pointer, } state.encoder = enc } - return state.encoder.EncodeTexture(texture, forceIDR || state.forceIDR.Swap(false)) + pendingIDR := state.forceIDR.Swap(false) + return state.encoder.EncodeTexture(texture, forceIDR || pendingIDR) } func requestNativeH264D3D11TextureKeyframe(stream string) { @@ -1997,7 +2003,8 @@ func encodeNativeHEVCD3D11Texture(device, texture unsafe.Pointer, inputWidth, in } nativeTextureHEVCEnc = enc } - return nativeTextureHEVCEnc.EncodeTexture(texture, forceIDR || nativeTextureHEVCForceIDR.Swap(false)) + pendingIDR := nativeTextureHEVCForceIDR.Swap(false) + return nativeTextureHEVCEnc.EncodeTexture(texture, forceIDR || pendingIDR) } func requestNativeHEVCD3D11TextureKeyframe() { diff --git a/Overlord-Client/cmd/agent/capture/qsv_d3d11_windows.go b/Overlord-Client/cmd/agent/capture/qsv_d3d11_windows.go index 5f376b9..9fac416 100644 --- a/Overlord-Client/cmd/agent/capture/qsv_d3d11_windows.go +++ b/Overlord-Client/cmd/agent/capture/qsv_d3d11_windows.go @@ -42,7 +42,8 @@ func (b *qsvD3D11TextureBackend) Encode(req h264D3D11TextureRequest) ([]byte, er b.device, b.inputWidth, b.inputHeight = req.Device, req.InputWidth, req.InputHeight b.encodeWidth, b.encodeHeight, b.fps, b.dxgiFormat = req.EncodeWidth, req.EncodeHeight, req.FPS, req.DXGIFormat } - forceIDR := req.ForceIDR || b.forceIDR.Swap(false) + pendingIDR := b.forceIDR.Swap(false) + forceIDR := req.ForceIDR || pendingIDR for attempts := 0; attempts < 2; attempts++ { result, size, err := b.encoder.Encode(req.Texture, forceIDR, b.output) if err != nil { diff --git a/Overlord-Client/cmd/agent/capture/video_access_unit.go b/Overlord-Client/cmd/agent/capture/video_access_unit.go new file mode 100644 index 0000000..64a30e4 --- /dev/null +++ b/Overlord-Client/cmd/agent/capture/video_access_unit.go @@ -0,0 +1,73 @@ +package capture + +import ( + "overlord-client/cmd/agent/h264util" + "overlord-client/cmd/agent/webrtcpub" +) + +func videoAccessUnitIsKey(codec string, data []byte) bool { + if codec == "hevc" { + return inspectHEVCAccessUnit(data).hasIRAP + } + return h264util.IsIDR(data) +} + +type hevcAccessUnit struct { + hasVPS bool + hasSPS bool + hasPPS bool + hasIRAP bool +} + +func inspectHEVCAccessUnit(data []byte) hevcAccessUnit { + var out hevcAccessUnit + for i := 0; i+4 < len(data); { + startCodeLen := 0 + switch { + case data[i] == 0 && data[i+1] == 0 && data[i+2] == 1: + startCodeLen = 3 + case i+4 < len(data) && data[i] == 0 && data[i+1] == 0 && data[i+2] == 0 && data[i+3] == 1: + startCodeLen = 4 + } + if startCodeLen == 0 { + i++ + continue + } + nal := i + startCodeLen + if nal >= len(data) { + break + } + nalType := (data[nal] >> 1) & 0x3f + switch nalType { + case 32: + out.hasVPS = true + case 33: + out.hasSPS = true + case 34: + out.hasPPS = true + default: + if nalType >= 16 && nalType <= 21 { + out.hasIRAP = true + } + } + i = nal + 1 + } + return out +} + +func videoAccessUnitIsRecoveryPoint(codec string, data []byte) bool { + if codec == "hevc" { + accessUnit := inspectHEVCAccessUnit(data) + return accessUnit.hasVPS && accessUnit.hasSPS && accessUnit.hasPPS && accessUnit.hasIRAP + } + accessUnit := h264util.InspectAnnexB(data) + return accessUnit.HasSPS && accessUnit.HasPPS && accessUnit.HasIDR +} + +func retainKeyframeRequestUntilOutput(kind webrtcpub.Kind, requested bool, codec string, data []byte) bool { + isKey := videoAccessUnitIsKey(codec, data) + if requested && !videoAccessUnitIsRecoveryPoint(codec, data) { + webrtcpub.RequestKeyframe(kind) + } + return isKey +} diff --git a/Overlord-Client/cmd/agent/capture/video_access_unit_test.go b/Overlord-Client/cmd/agent/capture/video_access_unit_test.go new file mode 100644 index 0000000..411a4d7 --- /dev/null +++ b/Overlord-Client/cmd/agent/capture/video_access_unit_test.go @@ -0,0 +1,38 @@ +package capture + +import ( + "testing" + + "overlord-client/cmd/agent/webrtcpub" +) + +func TestRetainKeyframeRequestUntilIDROutput(t *testing.T) { + kind := webrtcpub.KindDesktop + _ = webrtcpub.ConsumeKeyframeRequest(kind) + + if retainKeyframeRequestUntilOutput(kind, true, "h264", []byte{0, 0, 1, 0x41, 1}) { + t.Fatal("delta access unit was classified as a keyframe") + } + if !webrtcpub.ConsumeKeyframeRequest(kind) { + t.Fatal("keyframe request was not retained after delta output") + } + + if !retainKeyframeRequestUntilOutput(kind, true, "h264", []byte{0, 0, 1, 0x65, 1}) { + t.Fatal("IDR access unit was not classified as a keyframe") + } + if !webrtcpub.ConsumeKeyframeRequest(kind) { + t.Fatal("keyframe request was not retained when IDR omitted SPS/PPS") + } + + recoveryPoint := []byte{ + 0, 0, 1, 0x67, 1, + 0, 0, 1, 0x68, 1, + 0, 0, 1, 0x65, 1, + } + if !retainKeyframeRequestUntilOutput(kind, true, "h264", recoveryPoint) { + t.Fatal("complete recovery access unit was not classified as a keyframe") + } + if webrtcpub.ConsumeKeyframeRequest(kind) { + t.Fatal("completed keyframe request was unexpectedly requeued") + } +} diff --git a/Overlord-Client/cmd/agent/capture/video_keyframe_interval_test.go b/Overlord-Client/cmd/agent/capture/video_keyframe_interval_test.go new file mode 100644 index 0000000..7dc8152 --- /dev/null +++ b/Overlord-Client/cmd/agent/capture/video_keyframe_interval_test.go @@ -0,0 +1,29 @@ +package capture + +import ( + "testing" + "time" +) + +func TestConfiguredVideoKeyframeInterval(t *testing.T) { + tests := []struct { + value string + want time.Duration + }{ + {value: "", want: 0}, + {value: "off", want: 0}, + {value: "0", want: 0}, + {value: "30", want: 30 * time.Second}, + {value: "2m", want: 2 * time.Minute}, + {value: "invalid", want: 0}, + } + + for _, test := range tests { + t.Run(test.value, func(t *testing.T) { + t.Setenv("OVERLORD_VIDEO_KEYFRAME_INTERVAL", test.value) + if got := configuredVideoKeyframeInterval(); got != test.want { + t.Fatalf("configuredVideoKeyframeInterval() = %s, want %s", got, test.want) + } + }) + } +} diff --git a/Overlord-Client/cmd/agent/capture/virtual_windows.go b/Overlord-Client/cmd/agent/capture/virtual_windows.go index 1ff5315..8a6b08f 100644 --- a/Overlord-Client/cmd/agent/capture/virtual_windows.go +++ b/Overlord-Client/cmd/agent/capture/virtual_windows.go @@ -281,19 +281,31 @@ func (s *virtualDuplicationState) captureDirectH264() (wire.Frame, time.Duration func (s *virtualDuplicationState) encodeCachedDirectH264(capStart time.Time, width, height int) (wire.Frame, time.Duration, time.Duration, bool, error) { fps := activeH264FPSForStream("backstage") + now := time.Now() + requestedKeyframe := webrtcpub.ConsumeKeyframeRequest(webrtcpub.Kindbackstage) + forceKeyframe := directVideoKeyframeDue(requestedKeyframe, now.UnixNano(), backstageLastKeyframe.Load()) encStart := time.Now() out, _, err := encodeH264D3D11Texture("backstage", h264D3D11TextureRequest{ Device: unsafe.Pointer(s.device), Texture: unsafe.Pointer(s.h264LastTex), InputWidth: width, InputHeight: height, EncodeWidth: width, EncodeHeight: height, - FPS: fps, DXGIFormat: s.h264LastDesc.Format, ForceIDR: webrtcpub.ConsumeKeyframeRequest(webrtcpub.Kindbackstage), + FPS: fps, DXGIFormat: s.h264LastDesc.Format, ForceIDR: forceKeyframe, }) encodeDur := time.Since(encStart) if err != nil { + if requestedKeyframe { + webrtcpub.RequestKeyframe(webrtcpub.Kindbackstage) + } return wire.Frame{}, time.Since(capStart), encodeDur, true, err } if len(out) == 0 { + if requestedKeyframe { + webrtcpub.RequestKeyframe(webrtcpub.Kindbackstage) + } return wire.Frame{}, time.Since(capStart), encodeDur, true, nil } + if retainKeyframeRequestUntilOutput(webrtcpub.Kindbackstage, requestedKeyframe, "h264", out) { + backstageLastKeyframe.Store(now.UnixNano()) + } return wire.Frame{Type: "frame", Header: wire.FrameHeader{Monitor: 0, FPS: 0, Format: "h264", Backstage: true}, Data: out}, time.Since(capStart), encodeDur, true, nil } diff --git a/Overlord-Client/cmd/agent/capture/win_duplication_test.go b/Overlord-Client/cmd/agent/capture/win_duplication_test.go index 7ac579d..d3e6ea7 100644 --- a/Overlord-Client/cmd/agent/capture/win_duplication_test.go +++ b/Overlord-Client/cmd/agent/capture/win_duplication_test.go @@ -93,6 +93,9 @@ func TestCaptureDisplayDXGI(t *testing.T) { func TestDirectVideoKeyframeDue(t *testing.T) { now := (10 * time.Second).Nanoseconds() + previousInterval := videoKeyframeEvery + videoKeyframeEvery = 5 * time.Second + defer func() { videoKeyframeEvery = previousInterval }() tests := []struct { name string @@ -102,7 +105,8 @@ func TestDirectVideoKeyframeDue(t *testing.T) { }{ {name: "explicit request", requested: true, last: now, want: true}, {name: "first frame", last: 0, want: true}, - {name: "periodic keyframes disabled", last: now - (30 * time.Second).Nanoseconds(), want: false}, + {name: "periodic keyframe due", last: now - (30 * time.Second).Nanoseconds(), want: true}, + {name: "periodic keyframe not due", last: now - time.Second.Nanoseconds(), want: false}, } for _, test := range tests { @@ -112,4 +116,9 @@ func TestDirectVideoKeyframeDue(t *testing.T) { } }) } + + videoKeyframeEvery = 0 + if directVideoKeyframeDue(false, now, now-(30*time.Second).Nanoseconds()) { + t.Fatal("periodic keyframe fired while the interval was disabled") + } } diff --git a/Overlord-Client/cmd/agent/h264util/annexb.go b/Overlord-Client/cmd/agent/h264util/annexb.go new file mode 100644 index 0000000..ecb53eb --- /dev/null +++ b/Overlord-Client/cmd/agent/h264util/annexb.go @@ -0,0 +1,45 @@ +package h264util + +type AccessUnit struct { + HasSPS bool + HasPPS bool + HasIDR bool +} + +func InspectAnnexB(data []byte) AccessUnit { + var out AccessUnit + for i := 0; i+3 < len(data); { + startCodeLen := 0 + switch { + case i+3 < len(data) && data[i] == 0 && data[i+1] == 0 && data[i+2] == 1: + startCodeLen = 3 + case i+4 < len(data) && data[i] == 0 && data[i+1] == 0 && data[i+2] == 0 && data[i+3] == 1: + startCodeLen = 4 + } + if startCodeLen == 0 { + i++ + continue + } + nal := i + startCodeLen + if nal >= len(data) { + break + } + switch data[nal] & 0x1f { + case 5: + out.HasIDR = true + case 7: + out.HasSPS = true + case 8: + out.HasPPS = true + } + if out.HasSPS && out.HasPPS && out.HasIDR { + return out + } + i = nal + 1 + } + return out +} + +func IsIDR(data []byte) bool { + return InspectAnnexB(data).HasIDR +} diff --git a/Overlord-Client/cmd/agent/h264util/annexb_test.go b/Overlord-Client/cmd/agent/h264util/annexb_test.go new file mode 100644 index 0000000..b5dd97e --- /dev/null +++ b/Overlord-Client/cmd/agent/h264util/annexb_test.go @@ -0,0 +1,22 @@ +package h264util + +import "testing" + +func TestInspectAnnexB(t *testing.T) { + data := []byte{ + 0, 0, 0, 1, 0x67, 1, 2, 3, + 0, 0, 1, 0x68, 4, 5, + 0, 0, 0, 1, 0x65, 6, 7, + } + got := InspectAnnexB(data) + if !got.HasSPS || !got.HasPPS || !got.HasIDR { + t.Fatalf("InspectAnnexB() = %+v", got) + } +} + +func TestInspectAnnexBRejectsDeltaFrame(t *testing.T) { + got := InspectAnnexB([]byte{0, 0, 1, 0x41, 1, 2, 3}) + if got.HasSPS || got.HasPPS || got.HasIDR { + t.Fatalf("delta frame was misclassified: %+v", got) + } +} diff --git a/Overlord-Client/cmd/agent/webrtcpub/state.go b/Overlord-Client/cmd/agent/webrtcpub/state.go index a4d392c..af97963 100644 --- a/Overlord-Client/cmd/agent/webrtcpub/state.go +++ b/Overlord-Client/cmd/agent/webrtcpub/state.go @@ -4,6 +4,8 @@ import ( "sync" "sync/atomic" "time" + + "overlord-client/cmd/agent/h264util" ) type Kind string @@ -28,30 +30,64 @@ type writerEntry struct { audio AudioWriter } -type latestVideoWriter struct { - writer VideoWriter - mu sync.Mutex - pending []byte - dur time.Duration - closed bool - wake chan struct{} +const maxPendingVideoFrames = 6 + +type queuedVideoFrame struct { + data []byte + dur time.Duration + key bool } -func newLatestVideoWriter(writer VideoWriter) *latestVideoWriter { - queued := &latestVideoWriter{writer: writer, wake: make(chan struct{}, 1)} +type latestVideoWriter struct { + writer VideoWriter + kind Kind + mu sync.Mutex + pending []queuedVideoFrame + needsKeyframe bool + closed bool + wake chan struct{} +} + +func newLatestVideoWriter(kind Kind, writer VideoWriter) *latestVideoWriter { + queued := &latestVideoWriter{writer: writer, kind: kind, wake: make(chan struct{}, 1)} go queued.run() return queued } func (w *latestVideoWriter) enqueue(frame []byte, dur time.Duration) { + isKey := h264util.IsIDR(frame) w.mu.Lock() if w.closed { w.mu.Unlock() return } - w.pending = frame - w.dur = dur + if w.needsKeyframe && !isKey { + w.mu.Unlock() + RequestKeyframe(w.kind) + return + } + + requestKeyframe := false + if len(w.pending) >= maxPendingVideoFrames { + w.pending = nil + w.needsKeyframe = true + requestKeyframe = true + if !isKey { + w.mu.Unlock() + RequestKeyframe(w.kind) + return + } + } + if isKey && w.needsKeyframe { + w.pending = nil + w.needsKeyframe = false + requestKeyframe = false + } + w.pending = append(w.pending, queuedVideoFrame{data: frame, dur: dur, key: isKey}) w.mu.Unlock() + if requestKeyframe { + RequestKeyframe(w.kind) + } select { case w.wake <- struct{}{}: default: @@ -77,13 +113,26 @@ func (w *latestVideoWriter) run() { w.mu.Unlock() return } - frame, dur := w.pending, w.dur - w.pending = nil - w.mu.Unlock() - if len(frame) == 0 { + if len(w.pending) == 0 { + w.mu.Unlock() break } - _ = w.writer.WriteH264(frame, dur) + frame := w.pending[0] + w.pending[0] = queuedVideoFrame{} + w.pending = w.pending[1:] + w.mu.Unlock() + err := w.writer.WriteH264(frame.data, frame.dur) + requestKeyframe := false + w.mu.Lock() + if err != nil { + w.pending = nil + w.needsKeyframe = true + requestKeyframe = true + } + w.mu.Unlock() + if requestKeyframe { + RequestKeyframe(w.kind) + } } } } @@ -107,7 +156,7 @@ func registerVideoWriter(kind Kind, id string, w VideoWriter) { if entry.video != nil { entry.video.close() } - entry.video = newLatestVideoWriter(w) + entry.video = newLatestVideoWriter(kind, w) bucket[id] = entry writersMu.Unlock() RequestKeyframe(kind) diff --git a/Overlord-Client/cmd/agent/webrtcpub/state_test.go b/Overlord-Client/cmd/agent/webrtcpub/state_test.go index 53d02c0..454d457 100644 --- a/Overlord-Client/cmd/agent/webrtcpub/state_test.go +++ b/Overlord-Client/cmd/agent/webrtcpub/state_test.go @@ -4,6 +4,8 @@ import ( "sync" "testing" "time" + + "overlord-client/cmd/agent/h264util" ) type blockingVideoWriter struct { @@ -25,7 +27,15 @@ func (w *blockingVideoWriter) WriteH264(frame []byte, _ time.Duration) error { return nil } -func TestWriteH264DoesNotBlockAndKeepsLatestPendingFrame(t *testing.T) { +func annexBDelta(value byte) []byte { + return []byte{0, 0, 0, 1, 0x41, value} +} + +func annexBKey(value byte) []byte { + return []byte{0, 0, 0, 1, 0x67, 1, 0, 0, 1, 0x68, 1, 0, 0, 1, 0x65, value} +} + +func TestWriteH264DoesNotBlockAndRequestsRecoveryAfterDroppingDelta(t *testing.T) { kind := Kind("queue-test") id := "slow-writer" writer := &blockingVideoWriter{ @@ -34,8 +44,9 @@ func TestWriteH264DoesNotBlockAndKeepsLatestPendingFrame(t *testing.T) { } registerVideoWriter(kind, id, writer) defer unregisterWriter(kind, id) + _ = ConsumeKeyframeRequest(kind) - if err := WriteH264(kind, []byte{1}, time.Millisecond); err != nil { + if err := WriteH264(kind, annexBDelta(1), time.Millisecond); err != nil { t.Fatal(err) } select { @@ -45,11 +56,16 @@ func TestWriteH264DoesNotBlockAndKeepsLatestPendingFrame(t *testing.T) { } started := time.Now() - _ = WriteH264(kind, []byte{2}, time.Millisecond) - _ = WriteH264(kind, []byte{3}, time.Millisecond) + for frame := 2; frame <= maxPendingVideoFrames+2; frame++ { + _ = WriteH264(kind, annexBDelta(byte(frame)), time.Millisecond) + } if elapsed := time.Since(started); elapsed > 100*time.Millisecond { t.Fatalf("capture path blocked for %s", elapsed) } + if !ConsumeKeyframeRequest(kind) { + t.Fatal("dropping a predictive frame did not request a keyframe") + } + _ = WriteH264(kind, annexBKey(9), time.Millisecond) close(writer.release) deadline := time.Now().Add(time.Second) @@ -58,14 +74,14 @@ func TestWriteH264DoesNotBlockAndKeepsLatestPendingFrame(t *testing.T) { frames := append([][]byte(nil), writer.frames...) writer.mu.Unlock() if len(frames) >= 2 { - if frames[0][0] != 1 || frames[1][0] != 3 { - t.Fatalf("expected first and latest frames, got %v", frames) + if frames[0][len(frames[0])-1] != 1 || !h264util.IsIDR(frames[1]) { + t.Fatalf("expected first frame followed by recovery IDR, got %v", frames) } return } time.Sleep(5 * time.Millisecond) } - t.Fatal("latest pending frame was not delivered") + t.Fatal("recovery keyframe was not delivered") } func TestKeyframeRequestsAreIsolatedByStreamKind(t *testing.T) { diff --git a/Overlord-Client/cmd/agent/webrtcpub/whip_pion.go b/Overlord-Client/cmd/agent/webrtcpub/whip_pion.go index 8b7a8fa..323cdca 100644 --- a/Overlord-Client/cmd/agent/webrtcpub/whip_pion.go +++ b/Overlord-Client/cmd/agent/webrtcpub/whip_pion.go @@ -47,7 +47,7 @@ func drainRTCP(sender *webrtc.RTPSender, kind Kind) { } } -const h264SDPFmtpLine = "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=640034" +const h264SDPFmtpLine = "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=4d0034" var rtcpKeyframeLogOnce sync.Once var ( diff --git a/Overlord-Server/public/assets/cert-banner.js b/Overlord-Server/public/assets/cert-banner.js index c50c8d2..37003db 100644 --- a/Overlord-Server/public/assets/cert-banner.js +++ b/Overlord-Server/public/assets/cert-banner.js @@ -26,7 +26,7 @@ export async function showCertBannerIfNeeded(anchor) { } catch { } } - const os = detectOS(); + const os = getCertificateTrustGuide(); const banner = document.createElement("div"); banner.id = "cert-trust-banner"; @@ -45,7 +45,7 @@ export async function showCertBannerIfNeeded(anchor) {

overlord-ca.crt file. +`1. Double-click the downloaded overlord-server.crt file. 2. Click Install Certificate... 3. Choose Current User or Local Machine. 4. Select "Place all certificates in the following store". @@ -99,7 +99,7 @@ function detectOS() { return { label: "macOS", instructions: -`1. Double-click the downloaded overlord-ca.crt file — it opens in Keychain Access. +`1. Double-click the downloaded overlord-server.crt file — it opens in Keychain Access. 2. If prompted, add it to the login or System keychain. 3. Find the certificate in the list, double-click it. 4. Expand Trust and set "When using this certificate" to Always Trust. @@ -111,11 +111,11 @@ function detectOS() { label: "Linux", instructions: `Debian / Ubuntu: - sudo cp overlord-ca.crt /usr/local/share/ca-certificates/overlord-ca.crt + sudo cp overlord-server.crt /usr/local/share/ca-certificates/overlord-server.crt sudo update-ca-certificates Fedora / RHEL: - sudo cp overlord-ca.crt /etc/pki/ca-trust/source/anchors/overlord-ca.crt + sudo cp overlord-server.crt /etc/pki/ca-trust/source/anchors/overlord-server.crt sudo update-ca-trust Then restart your browser. diff --git a/Overlord-Server/public/assets/remotedesktop.js b/Overlord-Server/public/assets/remotedesktop.js index b785e4f..24874ce 100644 --- a/Overlord-Server/public/assets/remotedesktop.js +++ b/Overlord-Server/public/assets/remotedesktop.js @@ -153,6 +153,8 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s let pendingFrame = null; let videoDecoder = null; let videoDecoderConfigKey = ""; + let pendingDecodedVideoFrame = null; + let decodedVideoPresentationFrame = 0; const disabledDecoderCodecs = new Set(); let h264StreamCodec = ""; let canvasDecodePressure = false; @@ -167,6 +169,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s let h264KeyframeErrorStreak = 0; let h264RecoveryAttempts = 0; let h264LastDecodeWarnAt = 0; + let h264WaitingForKeyframe = true; const H264_LOW_FPS_THRESHOLD = 6; const H264_FALLBACK_WARMUP_MS = 10000; const H264_MIN_FRAMES_BEFORE_FALLBACK = 120; @@ -175,9 +178,8 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s const H264_MAX_RECOVERY_ATTEMPTS = 1; const H264_DECODE_WARN_THROTTLE_MS = 2000; const inputBackpressureBytes = 256 * 1024; - const VIEWER_FRAME_GAP_KEYFRAME_MS = 500; const VIEWER_FRAME_GAP_KEYFRAME_THROTTLE_MS = 1000; - let lastViewerFrameGapKeyframeAt = 0; + let lastViewerKeyframeRequestAt = 0; const diagnostics = { agent: null, network: null, @@ -240,6 +242,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s h264FirstFrameAt = 0; h264FramesSeen = 0; h264KeyframeErrorStreak = 0; + h264WaitingForKeyframe = true; } /* ── Remote Desktop Audio (system audio from client) ── */ @@ -1139,6 +1142,15 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s } else if ((finite(media.lossPercent) ?? 0) > 3 || (finite(network.rttMs) ?? 0) > 150 || (finite(media.jitterMs) ?? 0) > 35) { summary = "Network conditions are causing delay"; severity = "bad"; + } else if ( + mode === "off" && + fps != null && + viewerRate != null && + viewerRate < fps * 0.8 && + diagnostics.coalescedFrames > 0 + ) { + summary = `Display is presenting ${Math.round(viewerRate)} of ${Math.round(fps)} decoded FPS`; + severity = "warn"; } else if ((queue ?? 0) > 2 || (renderMs != null && renderMs > frameBudget)) { summary = "Viewer render queue is backing up"; severity = "warn"; @@ -1969,7 +1981,8 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s pushBitrate(); desiredStreaming = true; lastFrameAt = 0; - lastViewerFrameGapKeyframeAt = 0; + lastViewerKeyframeRequestAt = 0; + diagnostics.coalescedFrames = 0; firstFrameLogged = false; resetH264SessionState(); setStreamState("starting", "Starting stream"); @@ -2322,17 +2335,8 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s function markFrameReceived() { const now = performance.now(); - const frameGapMs = lastFrameAt ? now - lastFrameAt : 0; lastFrameAt = now; clearOfflineTimer(); - if ( - desiredStreaming && - frameGapMs >= VIEWER_FRAME_GAP_KEYFRAME_MS && - (!lastViewerFrameGapKeyframeAt || now - lastViewerFrameGapKeyframeAt >= VIEWER_FRAME_GAP_KEYFRAME_THROTTLE_MS) - ) { - lastViewerFrameGapKeyframeAt = now; - sendCmd("desktop_request_keyframe", { reason: "viewer_frame_gap" }); - } if (!firstFrameLogged) { firstFrameLogged = true; rdDebug("first frame received", { @@ -2394,6 +2398,14 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s function destroyVideoDecoder() { updateCanvasDecodePressure(true); + if (decodedVideoPresentationFrame) { + cancelAnimationFrame(decodedVideoPresentationFrame); + decodedVideoPresentationFrame = 0; + } + if (pendingDecodedVideoFrame) { + pendingDecodedVideoFrame.frame.close(); + pendingDecodedVideoFrame = null; + } if (videoDecoder) { try { videoDecoder.close(); @@ -2406,6 +2418,47 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s resetH264RuntimeState(); } + function presentLatestDecodedVideoFrame() { + decodedVideoPresentationFrame = 0; + const pending = pendingDecodedVideoFrame; + pendingDecodedVideoFrame = null; + if (!pending) return; + + const { frame, timing, width, height } = pending; + if (width > 0 && height > 0 && (canvas.width !== width || canvas.height !== height)) { + canvas.width = width; + canvas.height = height; + frameWidth = width; + frameHeight = height; + } + try { + ctx.drawImage(frame, 0, 0, canvas.width, canvas.height); + } finally { + frame.close(); + } + + const renderedAt = performance.now(); + if (timing) { + diagnostics.renderMs = smoothed(diagnostics.renderMs, Math.max(0, renderedAt - timing.receivedAt)); + } + diagnostics.width = width; + diagnostics.height = height; + updateFpsDisplay(); + } + + function queueDecodedVideoFrame(frame, timing) { + const width = frame.displayWidth || frame.codedWidth || frameWidth; + const height = frame.displayHeight || frame.codedHeight || frameHeight; + if (pendingDecodedVideoFrame) { + pendingDecodedVideoFrame.frame.close(); + diagnostics.coalescedFrames += 1; + } + pendingDecodedVideoFrame = { frame, timing, width, height }; + if (!decodedVideoPresentationFrame) { + decodedVideoPresentationFrame = requestAnimationFrame(presentLatestDecodedVideoFrame); + } + } + function normalizeFallbackReason(reason) { if (!reason) return "unspecified"; if (typeof reason === "string") return reason; @@ -2458,6 +2511,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s h264RecoveryAttempts += 1; h264KeyframeErrorStreak = 0; + destroyVideoDecoder(); console.warn("rd: h264 decode stuck waiting for keyframe; auto-restarting stream once", { reason, @@ -2578,34 +2632,24 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s output: (frame) => { const outputStartedAt = performance.now(); const timing = h264PendingTimings.shift(); - const width = frame.displayWidth || frame.codedWidth || frameWidth; - const height = frame.displayHeight || frame.codedHeight || frameHeight; - if (width > 0 && height > 0 && (canvas.width !== width || canvas.height !== height)) { - canvas.width = width; - canvas.height = height; - frameWidth = width; - frameHeight = height; - } - try { - ctx.drawImage(frame, 0, 0, canvas.width, canvas.height); - } finally { - frame.close(); - } - const renderedAt = performance.now(); + h264KeyframeErrorStreak = 0; if (timing) { diagnostics.decodeMs = smoothed(diagnostics.decodeMs, Math.max(0, outputStartedAt - timing.decodeStartedAt)); - diagnostics.renderMs = smoothed(diagnostics.renderMs, Math.max(0, renderedAt - timing.receivedAt)); } diagnostics.decodeQueue = videoDecoder?.decodeQueueSize || h264PendingTimings.length; - diagnostics.width = width; - diagnostics.height = height; - updateFpsDisplay(); + queueDecodedVideoFrame(frame, timing); updateCanvasDecodePressure(); }, error: (err) => { console.warn(`rd: ${codecName} decoder error`, err); + if (codecName === "h264") h264WaitingForKeyframe = true; updateCanvasDecodePressure(true); - setTimeout(() => fallbackFromVideoCodec(codecName, err), 0); + setTimeout(() => { + if (codecName === "h264" && tryRecoverH264Stream("h264_async_decoder_error")) { + return; + } + fallbackFromVideoCodec(codecName, err); + }, 0); }, }); videoDecoder.addEventListener("dequeue", () => { @@ -2795,6 +2839,23 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s h264FramesSeen += 1; const isKey = codecName === "hevc" ? isHEVCKeyFrame(videoBytes) : isH264KeyFrame(videoBytes); + if (codecName === "h264" && h264WaitingForKeyframe && !isKey) { + h264KeyframeErrorStreak += 1; + const now = performance.now(); + if (!lastViewerKeyframeRequestAt || now - lastViewerKeyframeRequestAt >= VIEWER_FRAME_GAP_KEYFRAME_THROTTLE_MS) { + lastViewerKeyframeRequestAt = now; + sendCmd("desktop_request_keyframe", { reason: "h264_decoder_keyframe_required" }); + } + if (h264KeyframeErrorStreak >= H264_KEYFRAME_ERROR_RESTART_THRESHOLD) { + const restarted = tryRecoverH264Stream("h264_keyframe_timeout"); + if (!restarted) fallbackToJpegCodec("h264_keyframe_timeout_loop"); + } + updateCanvasDecodePressure(true); + return; + } + if (codecName === "h264" && isKey) { + h264WaitingForKeyframe = false; + } // If software H264 encode on the agent cannot keep up, automatically // fall back to JPEG blocks for a smoother interactive stream. @@ -2827,10 +2888,10 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s videoDecoder.decode(chunk); diagnostics.decodeQueue = videoDecoder.decodeQueueSize; updateCanvasDecodePressure(); - h264KeyframeErrorStreak = 0; } catch (err) { h264PendingTimings.pop(); if (isKeyframeRequiredError(err)) { + if (codecName === "h264") h264WaitingForKeyframe = true; h264KeyframeErrorStreak += 1; const now = Date.now(); if (now - h264LastDecodeWarnAt >= H264_DECODE_WARN_THROTTLE_MS) { diff --git a/Overlord-Server/public/assets/settings.js b/Overlord-Server/public/assets/settings.js index 6f0eb4b..ff311dd 100644 --- a/Overlord-Server/public/assets/settings.js +++ b/Overlord-Server/public/assets/settings.js @@ -6,6 +6,7 @@ import { requestDesktopNotificationPermission, } from "./notify-client.js"; import { escapeHtml, formatBytes as formatSharedBytes, formatDate as formatSharedDate } from "./format.js"; +import { getCertificateTrustGuide } from "./cert-banner.js"; const PREF_REFRESH_KEY = "overlord_refresh_interval_seconds"; const NAV_MODE_KEY = "sb_mode"; @@ -81,6 +82,10 @@ const tlsCertbotDomainInput = document.getElementById("tls-certbot-domain"); const tlsCertbotCertFileInput = document.getElementById("tls-certbot-cert-file"); const tlsCertbotKeyFileInput = document.getElementById("tls-certbot-key-file"); const tlsCertbotCaFileInput = document.getElementById("tls-certbot-ca-file"); +const tlsActiveCertSource = document.getElementById("tls-active-cert-source"); +const tlsDownloadCertBtn = document.getElementById("tls-download-cert-btn"); +const tlsTrustHelpBtn = document.getElementById("tls-trust-help-btn"); +const tlsTrustInstructions = document.getElementById("tls-trust-instructions"); const oidcForm = document.getElementById("oidc-form"); const oidcPermissionNote = document.getElementById("oidc-permission-note"); @@ -457,6 +462,45 @@ function applyTlsForm() { tlsCertbotCaFileInput.value = certbot.caFileName || "chain.pem"; } +tlsTrustHelpBtn?.addEventListener("click", () => { + tlsTrustInstructions?.classList.toggle("hidden"); +}); + +async function loadActiveTlsCertificate() { + const guide = getCertificateTrustGuide(); + if (tlsTrustHelpBtn) { + tlsTrustHelpBtn.innerHTML = `Trust on ${guide.label}`; + } + if (tlsTrustInstructions) { + tlsTrustInstructions.innerHTML = guide.instructions; + } + + try { + const res = await fetch("/api/cert/info", { credentials: "include" }); + if (!res.ok) throw new Error(`HTTP ${res.status}`); + const data = await res.json(); + const source = String(data.source || "unknown"); + if (tlsActiveCertSource) { + tlsActiveCertSource.textContent = source === "self-signed" + ? "Self-signed" + : source === "certbot" + ? "Certbot / Let's Encrypt" + : source === "configured" + ? "Configured certificate" + : "Unknown source"; + tlsActiveCertSource.classList.toggle("border-amber-600", source === "self-signed"); + tlsActiveCertSource.classList.toggle("bg-amber-950", source === "self-signed"); + tlsActiveCertSource.classList.toggle("text-amber-200", source === "self-signed"); + } + } catch { + if (tlsActiveCertSource) tlsActiveCertSource.textContent = "Unavailable"; + if (tlsDownloadCertBtn) { + tlsDownloadCertBtn.setAttribute("aria-disabled", "true"); + tlsDownloadCertBtn.classList.add("pointer-events-none", "opacity-50"); + } + } +} + async function loadTlsSettings() { if (!currentUser) return; @@ -489,6 +533,7 @@ async function loadTlsSettings() { tlsConfig = data.tls || null; applyTlsForm(); setTlsFormDisabled(false); + await loadActiveTlsCertificate(); } function listToCsv(value) { diff --git a/Overlord-Server/public/settings.html b/Overlord-Server/public/settings.html index 8252b69..07c80f4 100644 --- a/Overlord-Server/public/settings.html +++ b/Overlord-Server/public/settings.html @@ -788,6 +788,46 @@ Your permissions do not allow TLS setting changes.
+
+
+
+
+

Active server certificate

+ + Checking… + +
+

+ Download and trust this certificate when a locally generated certificate causes HTTPS or WebSocket trust errors. +

+
+
+ + Download Certificate + + +
+
+ +

+ Browsers cannot install trusted certificates automatically. Your operating system will require confirmation, and the browser must be restarted afterward. +

+
+