From 6926522cc2f3af537db17aa701377a20f5716656 Mon Sep 17 00:00:00 2001 From: KingKDot Date: Tue, 21 Jul 2026 16:08:45 -0500 Subject: [PATCH] start of fix --- Overlord-Client/cmd/agent/capture/capture.go | 171 ++++++++++++------ .../cmd/agent/capture/capture_smoke.go | 2 +- .../cmd/agent/capture/capture_test.go | 8 +- .../capture/h264_bitrate_reset_windows.go | 2 +- .../cmd/agent/capture/h264_codec_test.go | 113 +++++++++++- .../cmd/agent/capture/h264_direct_windows.go | 4 +- .../cmd/agent/capture/h264_encoder_stub.go | 7 +- .../cmd/agent/capture/h264_encoder_windows.go | 82 ++++++++- .../cmd/agent/capture/h264_encoder_x264.go | 56 ++++-- .../capture/h264_texture_encoder_windows.go | 122 ++++++++----- .../h264_texture_encoder_windows_test.go | 129 ++++++++++++- .../agent/capture/native_h264_stub_windows.go | 6 +- .../capture/nvenc_d3d11_smoke_windows.go | 56 +++--- .../cmd/agent/capture/virtual_windows.go | 8 +- .../cmd/agent/capture/win_duplication.go | 4 +- .../cmd/agent/handlers/backstage.go | 6 +- Overlord-Client/cmd/agent/handlers/command.go | 7 +- Overlord-Client/cmd/agent/handlers/desktop.go | 6 +- Overlord-Client/cmd/agent/handlers/virtual.go | 6 +- Overlord-Client/cmd/agent/handlers/webcam.go | 1 + .../cmd/agent/webrtcpub/p2p_pion.go | 4 +- Overlord-Client/cmd/agent/webrtcpub/state.go | 27 ++- .../cmd/agent/webrtcpub/state_test.go | 18 ++ .../cmd/agent/webrtcpub/whip_pion.go | 35 +++- 24 files changed, 669 insertions(+), 211 deletions(-) diff --git a/Overlord-Client/cmd/agent/capture/capture.go b/Overlord-Client/cmd/agent/capture/capture.go index 5058726..3b7cf4d 100644 --- a/Overlord-Client/cmd/agent/capture/capture.go +++ b/Overlord-Client/cmd/agent/capture/capture.go @@ -110,17 +110,17 @@ func CaptureAndSend(ctx context.Context, env *rt.Env) error { if frame, captureDur, encodeDur, used, err := tryBuildDirectH264Frame(display); used { if err != nil { - consecutiveCaptureFails.Add(1) - log.Printf("capture: direct video capture failed: %v (sending black frame, consecutive=%d)", err, consecutiveCaptureFails.Load()) - return sendBlackFrame(ctx, env) - } - emitDesktopCursor(ctx, env, display, frame.Header.Width, frame.Header.Height) - if len(frame.Data) == 0 { + suppressDesktopHardwareH264(err) + log.Printf("capture: direct video encode failed: %v; falling back to the software H.264 path", err) + } else { + emitDesktopCursor(ctx, env, display, frame.Header.Width, frame.Header.Height) + if len(frame.Data) == 0 { + consecutiveCaptureFails.Store(0) + return nil + } consecutiveCaptureFails.Store(0) - return nil + return sendCompletedFrame(ctx, env, frame, display, t0, captureDur, encodeDur) } - consecutiveCaptureFails.Store(0) - return sendCompletedFrame(ctx, env, frame, display, t0, captureDur, encodeDur) } img, err := safeCaptureDisplay(display) @@ -145,7 +145,7 @@ func CaptureAndSend(ctx context.Context, env *rt.Env) error { consecutiveCaptureFails.Store(0) captureDur := time.Since(t0) - willSendViaWebRTC := blockCodec() == "h264" && webrtcpub.IsActive(webrtcpub.KindDesktop) + willSendViaWebRTC := desktopCodec() == "h264" && webrtcpub.IsActive(webrtcpub.KindDesktop) var slotAcquired bool if !willSendViaWebRTC { if !AcquireFrameSlot() { @@ -157,7 +157,7 @@ func CaptureAndSend(ctx context.Context, env *rt.Env) error { frameWidth := img.Bounds().Dx() frameHeight := img.Bounds().Dy() - quality := jpegQuality() + quality := desktopJPEGQuality() frame, encodeDur, err := buildFrame(img, display, quality) PutRGBA(img) img = nil @@ -413,7 +413,7 @@ func captureAllDisplaysAndSend(ctx context.Context, env *rt.Env) error { PutRGBA(part.img) } - quality := jpegQuality() + quality := desktopJPEGQuality() frame, _, err := buildFrame(canvas, 0, quality) canvasW, canvasH := canvas.Rect.Dx(), canvas.Rect.Dy() PutRGBA(canvas) @@ -482,6 +482,8 @@ func MonitorCount() int { const frameLogInterval = 5 * time.Second +const desktopHardwareH264FallbackCooldown = 30 * time.Second + var consecutiveCaptureFails atomic.Int64 var ( @@ -511,12 +513,16 @@ var ( statBlockFallbacks atomic.Int64 statFrameSlotSkips atomic.Int64 - overrideQuality atomic.Int64 - overrideCodec atomic.Value - desktopSoftwareH264 atomic.Bool - h264WarnOnce sync.Once - hevcWarnOnce sync.Once - codecLogOnce sync.Once + desktopOverrideQuality atomic.Int64 + backstageOverrideQuality atomic.Int64 + desktopOverrideCodec atomic.Value + backstageOverrideCodec atomic.Value + desktopSoftwareH264 atomic.Bool + desktopHardwareFallbackUntil atomic.Int64 + h264WarnOnce sync.Once + backstageH264WarnOnce sync.Once + hevcWarnOnce sync.Once + codecLogOnce sync.Once prevMu sync.Mutex prevFrame *prevImage @@ -645,7 +651,7 @@ func ResetPrevbackstage() { func RequestDesktopFullFrame() { requestFullFrames(2) - webrtcpub.RequestKeyframe() + webrtcpub.RequestKeyframe(webrtcpub.KindDesktop) RequestDesktopH264Keyframe() } @@ -653,13 +659,27 @@ func SetDesktopSoftwareH264(enabled bool) { if desktopSoftwareH264.Swap(enabled) == enabled { return } + desktopHardwareFallbackUntil.Store(0) resetH264Encoder() requestFullFrames(2) log.Printf("capture: desktop software h264 %v", enabled) } func useDesktopSoftwareH264() bool { - return desktopSoftwareH264.Load() + if desktopSoftwareH264.Load() { + return true + } + until := desktopHardwareFallbackUntil.Load() + return until > 0 && time.Now().UnixNano() < until +} + +func suppressDesktopHardwareH264(err error) { + until := time.Now().Add(desktopHardwareH264FallbackCooldown).UnixNano() + previous := desktopHardwareFallbackUntil.Swap(until) + resetH264Encoder() + if previous < time.Now().UnixNano() { + log.Printf("capture: hardware H.264 temporarily unavailable (%v); using software H.264 for %s", err, desktopHardwareH264FallbackCooldown) + } } func RequestbackstageFullFrame() { @@ -667,12 +687,19 @@ func RequestbackstageFullFrame() { backstagePrevFrame = nil backstagePrevMu.Unlock() backstageLastKeyframe.Store(0) - resetH264Encoderbackstage() + RequestBackstageH264Keyframe() } -func jpegQuality() int { +func desktopJPEGQuality() int { + return streamJPEGQuality(&desktopOverrideQuality) +} - if q := overrideQuality.Load(); q > 0 { +func backstageJPEGQuality() int { + return streamJPEGQuality(&backstageOverrideQuality) +} + +func streamJPEGQuality(override *atomic.Int64) int { + if q := override.Load(); q > 0 { return int(q) } q := int(loadOnceInt(&cachedJPEGQuality, 95)) @@ -761,13 +788,13 @@ func buildFrame(img *image.RGBA, display int, quality int) (wire.Frame, time.Dur height := img.Bounds().Dy() full := false now := time.Now() - codec := blockCodec() + codec := desktopCodec() if codec == "hevc" { if width%2 != 0 || height%2 != 0 { log.Printf("capture: hevc skipped for odd dimensions (%dx%d), falling back to h264/jpeg", width, height) codec = "h264" } else { - if webrtcpub.ConsumeKeyframeRequest() { + if webrtcpub.ConsumeKeyframeRequest(webrtcpub.KindDesktop) { resetHEVCEncoder() } hevcBytes, err := encodeHEVCFrame(img) @@ -782,10 +809,10 @@ func buildFrame(img *image.RGBA, display int, quality int) (wire.Frame, time.Dur resetHEVCEncoder() if h264Available() { codec = "h264" - overrideCodec.Store("h264") + desktopOverrideCodec.Store("h264") } else { codec = "jpeg" - overrideCodec.Store("jpeg") + desktopOverrideCodec.Store("jpeg") } } } @@ -795,7 +822,7 @@ func buildFrame(img *image.RGBA, display int, quality int) (wire.Frame, time.Dur log.Printf("capture: h264 skipped for odd dimensions (%dx%d), falling back to jpeg", width, height) codec = "jpeg" } else { - if webrtcpub.ConsumeKeyframeRequest() { + if webrtcpub.ConsumeKeyframeRequest(webrtcpub.KindDesktop) { resetH264Encoder() } h264Bytes, err := encodeH264Frame(img) @@ -892,7 +919,7 @@ func buildFramebackstage(img *image.RGBA, display int, quality int) (wire.Frame, encStart := time.Now() width := img.Bounds().Dx() height := img.Bounds().Dy() - codec := blockCodec() + codec := backstageCodec() now := time.Now() if codec == "h264" { @@ -900,7 +927,7 @@ func buildFramebackstage(img *image.RGBA, display int, quality int) (wire.Frame, log.Printf("backstage capture: h264 skipped for odd dimensions (%dx%d), falling back to jpeg", width, height) codec = "jpeg" } else { - if webrtcpub.ConsumeKeyframeRequest() { + if webrtcpub.ConsumeKeyframeRequest(webrtcpub.Kindbackstage) { resetH264Encoderbackstage() } h264Bytes, err := encodeH264Framebackstage(img) @@ -910,7 +937,7 @@ func buildFramebackstage(img *image.RGBA, display int, quality int) (wire.Frame, frame := wire.Frame{Type: "frame", Header: wire.FrameHeader{Monitor: display, FPS: 0, Format: "h264", Backstage: true}, Data: h264Bytes} return frame, time.Since(encStart), nil } - h264WarnOnce.Do(func() { + backstageH264WarnOnce.Do(func() { detail := h264AvailabilityDetail() if detail != "" { log.Printf("backstage capture: h264 encode unavailable, falling back to jpeg: %v (%s)", err, detail) @@ -1589,12 +1616,7 @@ func encodeBlockRaw(img *image.RGBA, x, y, w, h int) []byte { return buf } -func blockCodec() string { - if v := overrideCodec.Load(); v != nil { - if s, ok := v.(string); ok && s != "" { - return s - } - } +func defaultCodec() string { blockCodecOnce.Do(func() { codec := strings.ToLower(os.Getenv(blockCodecEnv)) switch codec { @@ -1613,12 +1635,33 @@ func blockCodec() string { return cachedBlockCodec } -func SetQualityAndCodec(quality int, codec string) { +func selectedCodec(override *atomic.Value) string { + if v := override.Load(); v != nil { + if s, ok := v.(string); ok && s != "" { + return s + } + } + return defaultCodec() +} + +func desktopCodec() string { + return selectedCodec(&desktopOverrideCodec) +} + +func backstageCodec() string { + codec := selectedCodec(&backstageOverrideCodec) + if codec == "hevc" { + return "jpeg" + } + return codec +} + +func SetDesktopQualityAndCodec(quality int, codec string) { if quality > 0 { if quality > 100 { quality = 100 } - overrideQuality.Store(int64(quality)) + desktopOverrideQuality.Store(int64(quality)) } s := strings.ToLower(strings.TrimSpace(codec)) if s == "h265" { @@ -1630,40 +1673,60 @@ func SetQualityAndCodec(quality int, codec string) { if h264Available() { fallback = "h264" } - log.Printf("capture: requested codec=hevc but unavailable (%s); forcing codec=%s", detail, fallback) + log.Printf("capture: requested desktop codec=hevc but unavailable (%s); forcing codec=%s", detail, fallback) s = fallback } if s == "h264" && !h264Available() { detail := h264AvailabilityDetail() - if detail != "" { - log.Printf("capture: requested codec=h264 but unavailable (%s); forcing codec=jpeg", detail) - } else { - log.Printf("capture: requested codec=h264 but unavailable; forcing codec=jpeg") - } + log.Printf("capture: requested desktop codec=h264 but unavailable (%s); forcing codec=jpeg", detail) s = "jpeg" } switch s { case "raw", "rgba", "jpeg", "h264", "hevc": - overrideCodec.Store(s) + desktopOverrideCodec.Store(s) h264WarnOnce = sync.Once{} hevcWarnOnce = sync.Once{} if s != "h264" { resetH264Encoder() - resetH264Encoderbackstage() } if s != "hevc" { resetHEVCEncoder() } case "": - default: - overrideCodec.Store("jpeg") + desktopOverrideCodec.Store("jpeg") resetH264Encoder() - resetH264Encoderbackstage() resetHEVCEncoder() } } +func SetBackstageQualityAndCodec(quality int, codec string) { + if quality > 0 { + if quality > 100 { + quality = 100 + } + backstageOverrideQuality.Store(int64(quality)) + } + s := strings.ToLower(strings.TrimSpace(codec)) + if s == "h264" && !h264Available() { + detail := h264AvailabilityDetail() + log.Printf("capture: requested backstage codec=h264 but unavailable (%s); forcing codec=jpeg", detail) + s = "jpeg" + } + switch s { + case "raw", "rgba", "jpeg", "h264": + backstageOverrideCodec.Store(s) + backstageH264WarnOnce = sync.Once{} + if s != "h264" { + resetH264Encoderbackstage() + } + case "": + default: + backstageOverrideCodec.Store("jpeg") + resetH264Encoderbackstage() + } +} + func Nowbackstage(ctx context.Context, env *rt.Env) error { if env.Cfg.DisableCapture { return sendBlackFramebackstage(ctx, env) @@ -1725,7 +1788,7 @@ func captureAndSendVirtual(ctx context.Context, env *rt.Env) error { } captureDur := time.Since(t0) - willSendViaWebRTC := blockCodec() == "h264" && webrtcpub.IsActive(webrtcpub.Kindbackstage) + willSendViaWebRTC := backstageCodec() == "h264" && webrtcpub.IsActive(webrtcpub.Kindbackstage) var slotAcquired bool if !willSendViaWebRTC && !AcquireFrameSlot() { PutRGBA(img) @@ -1733,7 +1796,7 @@ func captureAndSendVirtual(ctx context.Context, env *rt.Env) error { } slotAcquired = !willSendViaWebRTC - quality := jpegQuality() + quality := backstageJPEGQuality() frame, encodeDur, err := buildFramebackstage(img, 0, quality) PutRGBA(img) img = nil @@ -1873,7 +1936,7 @@ func captureAndSendbackstage(ctx context.Context, env *rt.Env) error { } captureDur := time.Since(t0) - willSendViaWebRTC := blockCodec() == "h264" && webrtcpub.IsActive(webrtcpub.Kindbackstage) + willSendViaWebRTC := backstageCodec() == "h264" && webrtcpub.IsActive(webrtcpub.Kindbackstage) var slotAcquired bool if !willSendViaWebRTC && !AcquireFrameSlot() { PutRGBA(img) @@ -1881,7 +1944,7 @@ func captureAndSendbackstage(ctx context.Context, env *rt.Env) error { } slotAcquired = !willSendViaWebRTC - quality := jpegQuality() + quality := backstageJPEGQuality() frame, encodeDur, err := buildFramebackstage(img, display, quality) PutRGBA(img) img = nil diff --git a/Overlord-Client/cmd/agent/capture/capture_smoke.go b/Overlord-Client/cmd/agent/capture/capture_smoke.go index 0405cd0..bf93749 100644 --- a/Overlord-Client/cmd/agent/capture/capture_smoke.go +++ b/Overlord-Client/cmd/agent/capture/capture_smoke.go @@ -39,7 +39,7 @@ func RunCaptureSmoke(opts CaptureSmokeOptions) []CaptureSmokeResult { opts.Display = 0 } if opts.FPS > 0 { - SetH264TargetFPS(opts.FPS) + SetDesktopH264TargetFPS(opts.FPS) } if opts.MaxHeight != 0 { SetMaxResolution(opts.MaxHeight) diff --git a/Overlord-Client/cmd/agent/capture/capture_test.go b/Overlord-Client/cmd/agent/capture/capture_test.go index 316b54f..6b0f722 100644 --- a/Overlord-Client/cmd/agent/capture/capture_test.go +++ b/Overlord-Client/cmd/agent/capture/capture_test.go @@ -144,7 +144,7 @@ func TestActualScreenCapture(t *testing.T) { t.Log("WARNING: Captured image appears to be solid color (may indicate capture issue)") } - quality := jpegQuality() + quality := desktopJPEGQuality() frame, encodeDur, err := buildFrame(img, 0, quality) if err != nil { t.Errorf("Failed to build frame from captured image: %v", err) @@ -270,7 +270,7 @@ func TestFrameFPS(t *testing.T) { } func TestJPEGQuality(t *testing.T) { - quality := jpegQuality() + quality := desktopJPEGQuality() if quality < 1 || quality > 100 { t.Errorf("JPEG quality out of range: %d (expected 1-100)", quality) } @@ -479,11 +479,11 @@ func TestSetFrameFlowTargetFPSEnvOverride(t *testing.T) { } func TestRequestDesktopFullFrameRequestsWebRTCKeyframe(t *testing.T) { - _ = webrtcpub.ConsumeKeyframeRequest() + _ = webrtcpub.ConsumeKeyframeRequest(webrtcpub.KindDesktop) RequestDesktopFullFrame() - if !webrtcpub.ConsumeKeyframeRequest() { + if !webrtcpub.ConsumeKeyframeRequest(webrtcpub.KindDesktop) { t.Fatal("expected desktop full-frame request to force the next WebRTC video frame to be a keyframe") } } diff --git a/Overlord-Client/cmd/agent/capture/h264_bitrate_reset_windows.go b/Overlord-Client/cmd/agent/capture/h264_bitrate_reset_windows.go index 7126e65..c674ec2 100644 --- a/Overlord-Client/cmd/agent/capture/h264_bitrate_reset_windows.go +++ b/Overlord-Client/cmd/agent/capture/h264_bitrate_reset_windows.go @@ -3,5 +3,5 @@ package capture func resetH264TextureEncoderForBitrate() { - resetH264D3D11TextureEncoder() + resetAllH264D3D11TextureEncoders() } diff --git a/Overlord-Client/cmd/agent/capture/h264_codec_test.go b/Overlord-Client/cmd/agent/capture/h264_codec_test.go index c404f98..4c87d4a 100644 --- a/Overlord-Client/cmd/agent/capture/h264_codec_test.go +++ b/Overlord-Client/cmd/agent/capture/h264_codec_test.go @@ -1,6 +1,8 @@ package capture import ( + "errors" + "fmt" "image" "image/color" "os" @@ -12,14 +14,17 @@ import ( func resetCodecSelectionForTest() { blockCodecOnce = sync.Once{} cachedBlockCodec = "" - overrideCodec.Store("") + desktopOverrideCodec.Store("") + backstageOverrideCodec.Store("") + desktopOverrideQuality.Store(0) + backstageOverrideQuality.Store(0) } func TestSetQualityAndCodec_H264FallbackDependsOnAvailability(t *testing.T) { t.Cleanup(resetCodecSelectionForTest) - SetQualityAndCodec(80, "h264") - got := blockCodec() + SetDesktopQualityAndCodec(80, "h264") + got := desktopCodec() if h264Available() { if got != "h264" { @@ -36,13 +41,56 @@ func TestSetQualityAndCodec_H264FallbackDependsOnAvailability(t *testing.T) { func TestSetQualityAndCodec_InvalidCodecForcesJpeg(t *testing.T) { t.Cleanup(resetCodecSelectionForTest) - SetQualityAndCodec(75, "invalid-codec") - got := blockCodec() + SetDesktopQualityAndCodec(75, "invalid-codec") + got := desktopCodec() if got != "jpeg" { t.Fatalf("expected invalid codec to force jpeg, got %q", got) } } +func TestDesktopAndBackstageCodecSelectionsAreIndependent(t *testing.T) { + t.Cleanup(resetCodecSelectionForTest) + + SetDesktopQualityAndCodec(80, "h264") + SetBackstageQualityAndCodec(65, "jpeg") + + wantDesktop := "jpeg" + if h264Available() { + wantDesktop = "h264" + } + if got := desktopCodec(); got != wantDesktop { + t.Fatalf("desktop codec = %q, want %q after backstage update", got, wantDesktop) + } + if got := backstageCodec(); got != "jpeg" { + t.Fatalf("backstage codec = %q, want jpeg", got) + } + if got := desktopJPEGQuality(); got != 80 { + t.Fatalf("desktop quality = %d, want 80", got) + } + if got := backstageJPEGQuality(); got != 65 { + t.Fatalf("backstage quality = %d, want 65", got) + } +} + +func TestDesktopHardwareFailureDoesNotChangeBackstageCodec(t *testing.T) { + t.Cleanup(func() { + desktopHardwareFallbackUntil.Store(0) + resetCodecSelectionForTest() + }) + SetDesktopQualityAndCodec(80, "h264") + SetBackstageQualityAndCodec(65, "h264") + before := backstageCodec() + + suppressDesktopHardwareH264(errors.New("NVENC session failure")) + + if !useDesktopSoftwareH264() { + t.Fatal("desktop did not enter temporary software H.264 fallback") + } + if got := backstageCodec(); got != before { + t.Fatalf("backstage codec changed from %q to %q after desktop encoder failure", before, got) + } +} + func TestH264AvailabilityDetail_NotEmpty(t *testing.T) { detail := strings.TrimSpace(h264AvailabilityDetail()) if detail == "" { @@ -93,6 +141,59 @@ func TestEncodeH264Frame_WhenAvailableProducesBytes(t *testing.T) { } } +func TestDesktopAndBackstageH264EncodeConcurrently(t *testing.T) { + if !h264Available() { + t.Skip("h264 is unavailable in this build") + } + SetDesktopH264TargetFPS(60) + SetBackstageH264TargetFPS(30) + t.Cleanup(func() { + resetH264Encoder() + resetH264Encoderbackstage() + }) + + const width, height = 640, 360 + start := make(chan struct{}) + results := make(chan error, 2) + run := func(name string, encode func(*image.RGBA) ([]byte, error), seed uint8) { + <-start + img := image.NewRGBA(image.Rect(0, 0, width, height)) + produced := false + for frame := range 12 { + for y := range height { + for x := range width { + img.SetRGBA(x, y, color.RGBA{ + R: uint8(x*3+frame*7) + seed, + G: uint8(y*3+frame*5) + seed, + B: uint8(frame*11) + seed, + A: 255, + }) + } + } + out, err := encode(img) + if err != nil { + results <- fmt.Errorf("%s encode frame %d: %w", name, frame, err) + return + } + produced = produced || len(out) > 0 + } + if !produced { + results <- fmt.Errorf("%s produced no H.264 output", name) + return + } + results <- nil + } + + go run("desktop", encodeH264Frame, 17) + go run("backstage", encodeH264Framebackstage, 83) + close(start) + for range 2 { + if err := <-results; err != nil { + t.Fatal(err) + } + } +} + func TestBlockCodec_UsesEnvWhenNoOverride(t *testing.T) { prev := os.Getenv(blockCodecEnv) if err := os.Setenv(blockCodecEnv, "raw"); err != nil { @@ -104,7 +205,7 @@ func TestBlockCodec_UsesEnvWhenNoOverride(t *testing.T) { }) resetCodecSelectionForTest() - got := blockCodec() + got := desktopCodec() if got != "raw" { t.Fatalf("expected env codec raw, got %q", got) } diff --git a/Overlord-Client/cmd/agent/capture/h264_direct_windows.go b/Overlord-Client/cmd/agent/capture/h264_direct_windows.go index a13ae2d..d76742d 100644 --- a/Overlord-Client/cmd/agent/capture/h264_direct_windows.go +++ b/Overlord-Client/cmd/agent/capture/h264_direct_windows.go @@ -17,13 +17,13 @@ func directVideoKeyframeDue(requested bool, nowNs, lastKeyframeNs int64) bool { } func tryBuildDirectH264Frame(display int) (wire.Frame, time.Duration, time.Duration, bool, error) { - codec := blockCodec() + codec := desktopCodec() if (codec != "h264" && codec != "hevc") || (codec == "h264" && useDesktopSoftwareH264()) || !useDesktopDuplication() { return wire.Frame{}, 0, 0, false, nil } now := time.Now() forceKeyframe := directVideoKeyframeDue( - webrtcpub.ConsumeKeyframeRequest(), + webrtcpub.ConsumeKeyframeRequest(webrtcpub.KindDesktop), now.UnixNano(), lastKeyframe.Load(), ) diff --git a/Overlord-Client/cmd/agent/capture/h264_encoder_stub.go b/Overlord-Client/cmd/agent/capture/h264_encoder_stub.go index 5d46a4e..1e3e315 100644 --- a/Overlord-Client/cmd/agent/capture/h264_encoder_stub.go +++ b/Overlord-Client/cmd/agent/capture/h264_encoder_stub.go @@ -23,7 +23,8 @@ func resetH264Encoder() {} func RequestDesktopH264Keyframe() {} -func resetH264Encoderbackstage() {} +func resetH264Encoderbackstage() {} +func RequestBackstageH264Keyframe() {} func encodeH264Framebackstage(_ *image.RGBA) ([]byte, error) { return nil, errors.New("h264 support not available (cgo disabled)") @@ -33,4 +34,6 @@ func encodeH264FrameWebcam(_ *image.RGBA) ([]byte, error) { return nil, errors.New("h264 support not available (cgo disabled)") } -func SetH264TargetFPS(_ int) {} +func SetDesktopH264TargetFPS(_ int) {} +func SetBackstageH264TargetFPS(_ int) {} +func SetWebcamH264TargetFPS(_ int) {} diff --git a/Overlord-Client/cmd/agent/capture/h264_encoder_windows.go b/Overlord-Client/cmd/agent/capture/h264_encoder_windows.go index 641d398..9f0a4eb 100644 --- a/Overlord-Client/cmd/agent/capture/h264_encoder_windows.go +++ b/Overlord-Client/cmd/agent/capture/h264_encoder_windows.go @@ -519,7 +519,9 @@ type mfH264Candidate struct { } var ( - h264TargetFPS atomic.Int64 + desktopH264TargetFPS atomic.Int64 + backstageH264TargetFPS atomic.Int64 + webcamH264TargetFPS atomic.Int64 h264Mu sync.Mutex h264Enc h264FrameEncoder @@ -565,7 +567,7 @@ func encodeH264FrameWithEncoder(slot *h264FrameEncoder, stream string, img *imag return nil, fmt.Errorf("h264 frame size must be even, got %dx%d", width, height) } - fps := activeH264FPS() + fps := activeH264FPSForStream(stream) if *slot == nil || !(*slot).Matches(width, height, fps) { if *slot != nil { (*slot).Close() @@ -577,7 +579,39 @@ func encodeH264FrameWithEncoder(slot *h264FrameEncoder, stream string, img *imag } *slot = enc } - return (*slot).Encode(img) + return encodeH264FrameWithRuntimeFallback(slot, stream, img, fps, newSoftwareH264Fallback) +} + +type h264SoftwareFallbackFactory func(width, height, fps int) (h264FrameEncoder, error) + +func newSoftwareH264Fallback(width, height, fps int) (h264FrameEncoder, error) { + return newMFSoftwareH264Encoder(width, height, fps) +} + +func encodeH264FrameWithRuntimeFallback( + slot *h264FrameEncoder, + stream string, + img *image.RGBA, + fps int, + createSoftware h264SoftwareFallbackFactory, +) ([]byte, error) { + out, err := (*slot).Encode(img) + if err == nil { + return out, nil + } + hardwareErr := err + closeH264Encoder(slot) + fallback, fallbackErr := createSoftware(img.Bounds().Dx(), img.Bounds().Dy(), fps) + if fallbackErr != nil { + return nil, fmt.Errorf("%s h264 encoder failed (%v) and software fallback could not start: %w", stream, hardwareErr, fallbackErr) + } + *slot = fallback + log.Printf("capture: %s h264 encoder failed at runtime: %v; switched this stream to Media Foundation software H.264", stream, hardwareErr) + out, err = fallback.Encode(img) + if err != nil { + return nil, fmt.Errorf("%s software h264 fallback encode failed after hardware error %v: %w", stream, hardwareErr, err) + } + return out, nil } func h264Available() bool { @@ -601,18 +635,45 @@ func h264AvailabilityDetail() string { return "Windows Media Foundation H.264 MFT (Microsoft H.264 encoder fallback; hardware unavailable: " + detail + ")" } -func SetH264TargetFPS(fps int) { +func SetDesktopH264TargetFPS(fps int) { + setH264TargetFPS(&desktopH264TargetFPS, fps) +} + +func SetBackstageH264TargetFPS(fps int) { + setH264TargetFPS(&backstageH264TargetFPS, fps) +} + +func SetWebcamH264TargetFPS(fps int) { + setH264TargetFPS(&webcamH264TargetFPS, fps) +} + +func setH264TargetFPS(target *atomic.Int64, fps int) { if fps < 1 { fps = 1 } - h264TargetFPS.Store(int64(fps)) + target.Store(int64(fps)) } func activeH264FPS() int { - if v := int(h264TargetFPS.Load()); v > 0 { + return activeH264FPSValue(&desktopH264TargetFPS, 60) +} + +func activeH264FPSForStream(stream string) int { + switch stream { + case "backstage": + return activeH264FPSValue(&backstageH264TargetFPS, 60) + case "webcam": + return activeH264FPSValue(&webcamH264TargetFPS, 30) + default: + return activeH264FPS() + } +} + +func activeH264FPSValue(target *atomic.Int64, fallback int) int { + if v := int(target.Load()); v > 0 { return v } - return 60 + return fallback } func resetH264Encoder() { @@ -622,7 +683,7 @@ func resetH264Encoder() { } func RequestDesktopH264Keyframe() { - requestH264D3D11TextureKeyframe() + requestH264D3D11TextureKeyframe("desktop") resetH264Encoder() } @@ -632,6 +693,11 @@ func resetH264Encoderbackstage() { closeH264Encoder(&backstageH264Enc) } +func RequestBackstageH264Keyframe() { + requestH264D3D11TextureKeyframe("backstage") + resetH264Encoderbackstage() +} + func closeH264Encoder(slot *h264FrameEncoder) { if slot != nil && *slot != nil { (*slot).Close() diff --git a/Overlord-Client/cmd/agent/capture/h264_encoder_x264.go b/Overlord-Client/cmd/agent/capture/h264_encoder_x264.go index 4e9a9c6..bdf2a67 100644 --- a/Overlord-Client/cmd/agent/capture/h264_encoder_x264.go +++ b/Overlord-Client/cmd/agent/capture/h264_encoder_x264.go @@ -13,15 +13,17 @@ import ( ) var ( - h264Mu sync.Mutex - h264Enc *x264.Encoder - h264Buf bytes.Buffer - h264Width int - h264Height int - h264FPS int - h264LastErr error - h264Scratch []byte - h264TargetFPS atomic.Int64 // set by the stream handler; default 60 + h264Mu sync.Mutex + h264Enc *x264.Encoder + h264Buf bytes.Buffer + h264Width int + h264Height int + h264FPS int + h264LastErr error + h264Scratch []byte + desktopH264TargetFPS atomic.Int64 + backstageH264TargetFPS atomic.Int64 + webcamH264TargetFPS atomic.Int64 ) func encodeH264Frame(img *image.RGBA) ([]byte, error) { @@ -70,22 +72,34 @@ func h264AvailabilityDetail() string { return "cgo build with x264-go" } -// SetH264TargetFPS sets the frame rate hint for the x264 encoder so its -// rate-control matches the actual capture cadence. Call this before (or -// when) streaming starts. The encoder is automatically re-created on the -// next frame if the value differs from the current one. -func SetH264TargetFPS(fps int) { +func SetDesktopH264TargetFPS(fps int) { + setH264TargetFPS(&desktopH264TargetFPS, fps) +} + +func SetBackstageH264TargetFPS(fps int) { + setH264TargetFPS(&backstageH264TargetFPS, fps) +} + +func SetWebcamH264TargetFPS(fps int) { + setH264TargetFPS(&webcamH264TargetFPS, fps) +} + +func setH264TargetFPS(target *atomic.Int64, fps int) { if fps < 1 { fps = 1 } - h264TargetFPS.Store(int64(fps)) + target.Store(int64(fps)) } func activeH264FPS() int { - if v := int(h264TargetFPS.Load()); v > 0 { + return activeH264FPSValue(&desktopH264TargetFPS, 60) +} + +func activeH264FPSValue(target *atomic.Int64, fallback int) int { + if v := int(target.Load()); v > 0 { return v } - return 60 // sensible default for desktop streaming + return fallback } func ensureH264EncoderLocked(width, height int) error { @@ -187,8 +201,12 @@ func resetH264Encoderbackstage() { closebackstageH264EncoderLocked() } +func RequestBackstageH264Keyframe() { + resetH264Encoderbackstage() +} + func ensurebackstageH264EncoderLocked(width, height int) error { - fps := activeH264FPS() + fps := activeH264FPSValue(&backstageH264TargetFPS, 60) if backstageH264Enc != nil && backstageH264Width == width && backstageH264Height == height && backstageH264FPS == fps { return nil } @@ -267,7 +285,7 @@ func encodeH264FrameWebcam(img *image.RGBA) ([]byte, error) { } func ensureWebcamH264EncoderLocked(width, height int) error { - fps := activeH264FPS() + fps := activeH264FPSValue(&webcamH264TargetFPS, 30) if webcamH264Enc != nil && webcamH264Width == width && webcamH264Height == height && webcamH264FPS == fps { return nil } diff --git a/Overlord-Client/cmd/agent/capture/h264_texture_encoder_windows.go b/Overlord-Client/cmd/agent/capture/h264_texture_encoder_windows.go index a65c4c7..e61ec84 100644 --- a/Overlord-Client/cmd/agent/capture/h264_texture_encoder_windows.go +++ b/Overlord-Client/cmd/agent/capture/h264_texture_encoder_windows.go @@ -43,90 +43,107 @@ type h264D3D11TextureBackend interface { Reset() } -type nvencD3D11TextureBackend struct{} +type nvencD3D11TextureBackend struct { + stream string +} -func (nvencD3D11TextureBackend) Name() string { return "NVIDIA NVENC" } +func (*nvencD3D11TextureBackend) Name() string { return "NVIDIA NVENC" } -func (nvencD3D11TextureBackend) Encode(req h264D3D11TextureRequest) ([]byte, error) { - return encodeNativeH264D3D11Texture(req.Device, req.Texture, req.InputWidth, req.InputHeight, +func (b *nvencD3D11TextureBackend) Encode(req h264D3D11TextureRequest) ([]byte, error) { + return encodeNativeH264D3D11Texture(b.stream, req.Device, req.Texture, req.InputWidth, req.InputHeight, req.EncodeWidth, req.EncodeHeight, req.FPS, req.DXGIFormat, req.ForceIDR) } -func (nvencD3D11TextureBackend) RequestKeyframe() { - requestNativeH264D3D11TextureKeyframe() +func (b *nvencD3D11TextureBackend) RequestKeyframe() { + requestNativeH264D3D11TextureKeyframe(b.stream) } -func (nvencD3D11TextureBackend) Reset() { - resetNativeH264D3D11TextureEncoder() +func (b *nvencD3D11TextureBackend) Reset() { + resetNativeH264D3D11TextureEncoder(b.stream) } -var d3d11H264TextureRegistry = struct { +type h264D3D11TextureRegistry struct { sync.Mutex backends []h264D3D11TextureBackend active h264D3D11TextureBackend failures map[d3d11H264FailureKey]d3d11H264Failure -}{ - backends: []h264D3D11TextureBackend{nvencD3D11TextureBackend{}}, - failures: make(map[d3d11H264FailureKey]d3d11H264Failure), } -func init() { +func newH264D3D11TextureRegistry(stream string) *h264D3D11TextureRegistry { + registry := &h264D3D11TextureRegistry{ + backends: []h264D3D11TextureBackend{&nvencD3D11TextureBackend{stream: stream}}, + failures: make(map[d3d11H264FailureKey]d3d11H264Failure), + } if backend := newQSVD3D11TextureBackend(); backend != nil { - d3d11H264TextureRegistry.backends = append(d3d11H264TextureRegistry.backends, backend) + registry.backends = append(registry.backends, backend) } if backend := newAMFD3D11TextureBackend(); backend != nil { - d3d11H264TextureRegistry.backends = append(d3d11H264TextureRegistry.backends, backend) + registry.backends = append(registry.backends, backend) } + return registry } -func encodeH264D3D11Texture(req h264D3D11TextureRequest) ([]byte, string, error) { +var ( + d3d11H264TextureRegistry = newH264D3D11TextureRegistry("desktop") + backstageD3D11H264TextureRegistry = newH264D3D11TextureRegistry("backstage") +) + +func h264D3D11Registry(stream string) *h264D3D11TextureRegistry { + if stream == "backstage" { + return backstageD3D11H264TextureRegistry + } + return d3d11H264TextureRegistry +} + +func encodeH264D3D11Texture(stream string, req h264D3D11TextureRequest) ([]byte, string, error) { if req.Device == nil || req.Texture == nil { return nil, "", fmt.Errorf("nil D3D11 device or texture") } - d3d11H264TextureRegistry.Lock() - defer d3d11H264TextureRegistry.Unlock() + registry := h264D3D11Registry(stream) + registry.Lock() + defer registry.Unlock() failedActiveName := "" - errorsByBackend := make([]string, 0, len(d3d11H264TextureRegistry.backends)) - if active := d3d11H264TextureRegistry.active; active != nil { + errorsByBackend := make([]string, 0, len(registry.backends)) + if active := registry.active; active != nil { key := d3d11H264FailureKeyFor(active.Name(), req) - if failure, failed := activeD3D11H264Failure(key, time.Now()); !failed { + if failure, failed := registry.activeFailure(key, time.Now()); !failed { out, err := active.Encode(req) if err == nil { - delete(d3d11H264TextureRegistry.failures, key) + delete(registry.failures, key) return out, active.Name(), nil } - rememberD3D11H264Failure(key, err) + registry.rememberFailure(key, err) active.Reset() failedActiveName = active.Name() errorsByBackend = append(errorsByBackend, active.Name()+": "+err.Error()) - d3d11H264TextureRegistry.active = nil + registry.active = nil } else { errorsByBackend = append(errorsByBackend, active.Name()+": "+failure.err+" (cooldown)") active.Reset() failedActiveName = active.Name() - d3d11H264TextureRegistry.active = nil + registry.active = nil } } now := time.Now() - for _, backend := range d3d11H264TextureRegistry.backends { + for _, backend := range registry.backends { if backend == nil || (failedActiveName != "" && backend.Name() == failedActiveName) { continue } key := d3d11H264FailureKeyFor(backend.Name(), req) - if failure, failed := activeD3D11H264Failure(key, now); failed { + if failure, failed := registry.activeFailure(key, now); failed { errorsByBackend = append(errorsByBackend, backend.Name()+": "+failure.err+" (cooldown)") continue } out, err := backend.Encode(req) if err == nil { - delete(d3d11H264TextureRegistry.failures, key) - d3d11H264TextureRegistry.active = backend + delete(registry.failures, key) + registry.active = backend return out, backend.Name(), nil } backend.Reset() - rememberD3D11H264Failure(key, err) + registry.rememberFailure(key, err) errorsByBackend = append(errorsByBackend, backend.Name()+": "+err.Error()) } if len(errorsByBackend) == 0 { @@ -142,51 +159,58 @@ func d3d11H264FailureKeyFor(backend string, req h264D3D11TextureRequest) d3d11H2 } } -func activeD3D11H264Failure(key d3d11H264FailureKey, now time.Time) (d3d11H264Failure, bool) { - failure, ok := d3d11H264TextureRegistry.failures[key] +func (r *h264D3D11TextureRegistry) activeFailure(key d3d11H264FailureKey, now time.Time) (d3d11H264Failure, bool) { + failure, ok := r.failures[key] if !ok { return d3d11H264Failure{}, false } if !now.Before(failure.until) { - delete(d3d11H264TextureRegistry.failures, key) + delete(r.failures, key) return d3d11H264Failure{}, false } return failure, true } -func rememberD3D11H264Failure(key d3d11H264FailureKey, err error) { - if len(d3d11H264TextureRegistry.failures) > 128 { +func (r *h264D3D11TextureRegistry) rememberFailure(key d3d11H264FailureKey, err error) { + if len(r.failures) > 128 { now := time.Now() - for existingKey, failure := range d3d11H264TextureRegistry.failures { + for existingKey, failure := range r.failures { if !now.Before(failure.until) { - delete(d3d11H264TextureRegistry.failures, existingKey) + delete(r.failures, existingKey) } } } - d3d11H264TextureRegistry.failures[key] = d3d11H264Failure{until: time.Now().Add(d3d11H264FailureCooldown), err: err.Error()} + r.failures[key] = d3d11H264Failure{until: time.Now().Add(d3d11H264FailureCooldown), err: err.Error()} } -func requestH264D3D11TextureKeyframe() { - d3d11H264TextureRegistry.Lock() - defer d3d11H264TextureRegistry.Unlock() - if d3d11H264TextureRegistry.active != nil { - d3d11H264TextureRegistry.active.RequestKeyframe() +func requestH264D3D11TextureKeyframe(stream string) { + registry := h264D3D11Registry(stream) + registry.Lock() + defer registry.Unlock() + if registry.active != nil { + registry.active.RequestKeyframe() return } - for _, backend := range d3d11H264TextureRegistry.backends { + for _, backend := range registry.backends { if backend != nil { backend.RequestKeyframe() } } } -func resetH264D3D11TextureEncoder() { - d3d11H264TextureRegistry.Lock() - defer d3d11H264TextureRegistry.Unlock() - for _, backend := range d3d11H264TextureRegistry.backends { +func resetH264D3D11TextureEncoder(stream string) { + registry := h264D3D11Registry(stream) + registry.Lock() + defer registry.Unlock() + for _, backend := range registry.backends { if backend != nil { backend.Reset() } } - d3d11H264TextureRegistry.active = nil + registry.active = nil +} + +func resetAllH264D3D11TextureEncoders() { + resetH264D3D11TextureEncoder("desktop") + resetH264D3D11TextureEncoder("backstage") } diff --git a/Overlord-Client/cmd/agent/capture/h264_texture_encoder_windows_test.go b/Overlord-Client/cmd/agent/capture/h264_texture_encoder_windows_test.go index 15b45cd..f334e8e 100644 --- a/Overlord-Client/cmd/agent/capture/h264_texture_encoder_windows_test.go +++ b/Overlord-Client/cmd/agent/capture/h264_texture_encoder_windows_test.go @@ -4,6 +4,7 @@ package capture import ( "errors" + "image" "testing" "unsafe" ) @@ -54,7 +55,7 @@ func TestD3D11TextureBackendSelectionAndFailover(t *testing.T) { device, texture := 1, 2 req := h264D3D11TextureRequest{Device: unsafe.Pointer(&device), Texture: unsafe.Pointer(&texture)} - out, provider, err := encodeH264D3D11Texture(req) + out, provider, err := encodeH264D3D11Texture("desktop", req) if err != nil { t.Fatalf("first encode: %v", err) } @@ -65,7 +66,7 @@ func TestD3D11TextureBackendSelectionAndFailover(t *testing.T) { t.Fatalf("unexpected calls first.reset=%d second.encode=%d", first.resetCalls, second.encodeCalls) } - _, provider, err = encodeH264D3D11Texture(req) + _, provider, err = encodeH264D3D11Texture("desktop", req) if err != nil || provider != "second" { t.Fatalf("active backend reuse provider=%q err=%v", provider, err) } @@ -73,11 +74,11 @@ func TestD3D11TextureBackendSelectionAndFailover(t *testing.T) { t.Fatalf("active backend was not reused: first=%d second=%d", first.encodeCalls, second.encodeCalls) } - requestH264D3D11TextureKeyframe() + requestH264D3D11TextureKeyframe("desktop") if second.keyframeCalls != 1 || first.keyframeCalls != 0 { t.Fatalf("keyframe routed incorrectly: first=%d second=%d", first.keyframeCalls, second.keyframeCalls) } - resetH264D3D11TextureEncoder() + resetH264D3D11TextureEncoder("desktop") if first.resetCalls != 2 || second.resetCalls != 1 { t.Fatalf("reset did not reach every backend: first=%d second=%d", first.resetCalls, second.resetCalls) } @@ -105,13 +106,129 @@ func TestD3D11TextureBackendFailuresUseCooldown(t *testing.T) { device, texture := 1, 2 req := h264D3D11TextureRequest{Device: unsafe.Pointer(&device), Texture: unsafe.Pointer(&texture), EncodeWidth: 2560, EncodeHeight: 1440, FPS: 240} - if _, _, err := encodeH264D3D11Texture(req); err == nil { + if _, _, err := encodeH264D3D11Texture("desktop", req); err == nil { t.Fatal("expected initial profile failure") } - if _, _, err := encodeH264D3D11Texture(req); err == nil { + if _, _, err := encodeH264D3D11Texture("desktop", req); err == nil { t.Fatal("expected cached profile failure") } if first.encodeCalls != 1 || second.encodeCalls != 1 { t.Fatalf("failed profile was retried during cooldown: first=%d second=%d", first.encodeCalls, second.encodeCalls) } } + +type fakeH264FrameEncoder struct { + output []byte + err error + encodeCall int + closed bool +} + +func (e *fakeH264FrameEncoder) Encode(*image.RGBA) ([]byte, error) { + e.encodeCall++ + return e.output, e.err +} + +func (e *fakeH264FrameEncoder) Close() { + e.closed = true +} + +func (e *fakeH264FrameEncoder) Matches(_, _, _ int) bool { + return true +} + +func TestH264RuntimeFailureFallsBackPerStream(t *testing.T) { + primary := &fakeH264FrameEncoder{err: errors.New("NVENC session failed")} + software := &fakeH264FrameEncoder{output: []byte{7, 8, 9}} + var slot h264FrameEncoder = primary + factoryCalls := 0 + factory := func(width, height, fps int) (h264FrameEncoder, error) { + factoryCalls++ + if width != 64 || height != 32 || fps != 47 { + t.Fatalf("fallback config = %dx%d@%d, want 64x32@47", width, height, fps) + } + return software, nil + } + + out, err := encodeH264FrameWithRuntimeFallback( + &slot, + "backstage", + image.NewRGBA(image.Rect(0, 0, 64, 32)), + 47, + factory, + ) + if err != nil { + t.Fatalf("runtime fallback failed: %v", err) + } + if factoryCalls != 1 || primary.encodeCall != 1 || !primary.closed { + t.Fatalf("primary cleanup/fallback calls = factory:%d encode:%d closed:%v", factoryCalls, primary.encodeCall, primary.closed) + } + if slot != software || software.encodeCall != 1 { + t.Fatalf("software encoder was not retained after fallback") + } + if len(out) != 3 || out[0] != 7 { + t.Fatalf("fallback output = %v", out) + } +} + +func TestH264TargetFPSIsIndependentPerStream(t *testing.T) { + SetDesktopH264TargetFPS(120) + SetBackstageH264TargetFPS(37) + SetWebcamH264TargetFPS(24) + + if got := activeH264FPS(); got != 120 { + t.Fatalf("desktop fps = %d, want 120", got) + } + if got := activeH264FPSForStream("backstage"); got != 37 { + t.Fatalf("backstage fps = %d, want 37", got) + } + if got := activeH264FPSForStream("webcam"); got != 24 { + t.Fatalf("webcam fps = %d, want 24", got) + } +} + +func TestD3D11TextureEncodersAreIsolatedByStream(t *testing.T) { + desktopBackend := &fakeD3D11TextureBackend{name: "desktop", outputs: [][]byte{{1}, {2}}} + backstageBackend := &fakeD3D11TextureBackend{name: "backstage", outputs: [][]byte{{3}}} + savedDesktop := d3d11H264TextureRegistry + savedBackstage := backstageD3D11H264TextureRegistry + d3d11H264TextureRegistry = &h264D3D11TextureRegistry{ + backends: []h264D3D11TextureBackend{desktopBackend}, + failures: make(map[d3d11H264FailureKey]d3d11H264Failure), + } + backstageD3D11H264TextureRegistry = &h264D3D11TextureRegistry{ + backends: []h264D3D11TextureBackend{backstageBackend}, + failures: make(map[d3d11H264FailureKey]d3d11H264Failure), + } + t.Cleanup(func() { + d3d11H264TextureRegistry = savedDesktop + backstageD3D11H264TextureRegistry = savedBackstage + }) + + desktopDevice, desktopTexture := 1, 2 + backstageDevice, backstageTexture := 3, 4 + desktopReq := h264D3D11TextureRequest{ + Device: unsafe.Pointer(&desktopDevice), Texture: unsafe.Pointer(&desktopTexture), + EncodeWidth: 1920, EncodeHeight: 1080, FPS: 60, + } + backstageReq := h264D3D11TextureRequest{ + Device: unsafe.Pointer(&backstageDevice), Texture: unsafe.Pointer(&backstageTexture), + EncodeWidth: 1280, EncodeHeight: 720, FPS: 30, + } + + if _, _, err := encodeH264D3D11Texture("desktop", desktopReq); err != nil { + t.Fatalf("desktop encode: %v", err) + } + if _, _, err := encodeH264D3D11Texture("backstage", backstageReq); err != nil { + t.Fatalf("backstage encode: %v", err) + } + if _, _, err := encodeH264D3D11Texture("desktop", desktopReq); err != nil { + t.Fatalf("second desktop encode: %v", err) + } + if desktopBackend.encodeCalls != 2 || backstageBackend.encodeCalls != 1 { + t.Fatalf("stream calls crossed: desktop=%d backstage=%d", desktopBackend.encodeCalls, backstageBackend.encodeCalls) + } + if desktopBackend.resetCalls != 0 || backstageBackend.resetCalls != 0 { + t.Fatalf("one stream reset the other: desktop=%d backstage=%d", desktopBackend.resetCalls, backstageBackend.resetCalls) + } +} diff --git a/Overlord-Client/cmd/agent/capture/native_h264_stub_windows.go b/Overlord-Client/cmd/agent/capture/native_h264_stub_windows.go index 3a8af69..0b0ed2f 100644 --- a/Overlord-Client/cmd/agent/capture/native_h264_stub_windows.go +++ b/Overlord-Client/cmd/agent/capture/native_h264_stub_windows.go @@ -17,13 +17,13 @@ func nativeH264AvailabilityDetail() string { return "" } -func encodeNativeH264D3D11Texture(device, texture unsafe.Pointer, inputWidth, inputHeight, encodeWidth, encodeHeight, fps int, dxgiFormat uint32, forceIDR bool) ([]byte, error) { +func encodeNativeH264D3D11Texture(_ string, device, texture unsafe.Pointer, inputWidth, inputHeight, encodeWidth, encodeHeight, fps int, dxgiFormat uint32, forceIDR bool) ([]byte, error) { return nil, fmt.Errorf("native NVENC D3D11 texture encode requires cgo for input=%dx%d output=%dx%d@%dfps", inputWidth, inputHeight, encodeWidth, encodeHeight, fps) } -func requestNativeH264D3D11TextureKeyframe() {} +func requestNativeH264D3D11TextureKeyframe(_ string) {} -func resetNativeH264D3D11TextureEncoder() {} +func resetNativeH264D3D11TextureEncoder(_ string) {} func encodeNativeHEVCD3D11Texture(device, texture unsafe.Pointer, inputWidth, inputHeight, encodeWidth, encodeHeight, fps int, dxgiFormat uint32, forceIDR bool) ([]byte, error) { return nil, fmt.Errorf("native NVENC HEVC D3D11 texture encode requires cgo for input=%dx%d output=%dx%d@%dfps", inputWidth, inputHeight, encodeWidth, encodeHeight, fps) diff --git a/Overlord-Client/cmd/agent/capture/nvenc_d3d11_smoke_windows.go b/Overlord-Client/cmd/agent/capture/nvenc_d3d11_smoke_windows.go index d845865..1549271 100644 --- a/Overlord-Client/cmd/agent/capture/nvenc_d3d11_smoke_windows.go +++ b/Overlord-Client/cmd/agent/capture/nvenc_d3d11_smoke_windows.go @@ -1921,48 +1921,62 @@ func envBool(name string) bool { } } +type nativeTextureH264StreamState struct { + sync.Mutex + encoder *nativeD3D11TextureH264Encoder + forceIDR atomic.Bool +} + var ( - nativeTextureH264Mu sync.Mutex - nativeTextureH264Enc *nativeD3D11TextureH264Encoder - nativeTextureH264ForceIDR atomic.Bool + nativeTextureH264Streams sync.Map nativeTextureHEVCMu sync.Mutex nativeTextureHEVCEnc *nativeD3D11TextureH264Encoder nativeTextureHEVCForceIDR atomic.Bool ) -func encodeNativeH264D3D11Texture(device, texture unsafe.Pointer, inputWidth, inputHeight, encodeWidth, encodeHeight, fps int, dxgiFormat uint32, forceIDR bool) ([]byte, error) { +func nativeTextureH264State(stream string) *nativeTextureH264StreamState { + if stream == "" { + stream = "desktop" + } + state, _ := nativeTextureH264Streams.LoadOrStore(stream, &nativeTextureH264StreamState{}) + return state.(*nativeTextureH264StreamState) +} + +func encodeNativeH264D3D11Texture(stream string, device, texture unsafe.Pointer, inputWidth, inputHeight, encodeWidth, encodeHeight, fps int, dxgiFormat uint32, forceIDR bool) ([]byte, error) { if device == nil || texture == nil { return nil, fmt.Errorf("nil D3D11 device or texture") } - nativeTextureH264Mu.Lock() - defer nativeTextureH264Mu.Unlock() + state := nativeTextureH264State(stream) + state.Lock() + defer state.Unlock() - if nativeTextureH264Enc == nil || !nativeTextureH264Enc.Matches(device, inputWidth, inputHeight, encodeWidth, encodeHeight, fps, dxgiFormat) { - if nativeTextureH264Enc != nil { - nativeTextureH264Enc.Close() - nativeTextureH264Enc = nil + if state.encoder == nil || !state.encoder.Matches(device, inputWidth, inputHeight, encodeWidth, encodeHeight, fps, dxgiFormat) { + if state.encoder != nil { + state.encoder.Close() + state.encoder = nil } enc, err := newNativeD3D11TextureH264Encoder(device, inputWidth, inputHeight, encodeWidth, encodeHeight, fps, dxgiFormat) if err != nil { return nil, err } - nativeTextureH264Enc = enc + state.encoder = enc } - return nativeTextureH264Enc.EncodeTexture(texture, forceIDR || nativeTextureH264ForceIDR.Swap(false)) + return state.encoder.EncodeTexture(texture, forceIDR || state.forceIDR.Swap(false)) } -func requestNativeH264D3D11TextureKeyframe() { - nativeTextureH264ForceIDR.Store(true) +func requestNativeH264D3D11TextureKeyframe(stream string) { + nativeTextureH264State(stream).forceIDR.Store(true) } -func resetNativeH264D3D11TextureEncoder() { - nativeTextureH264Mu.Lock() - defer nativeTextureH264Mu.Unlock() - if nativeTextureH264Enc != nil { - nativeTextureH264Enc.Close() - nativeTextureH264Enc = nil +func resetNativeH264D3D11TextureEncoder(stream string) { + state := nativeTextureH264State(stream) + state.Lock() + defer state.Unlock() + if state.encoder != nil { + state.encoder.Close() + state.encoder = nil } - nativeTextureH264ForceIDR.Store(false) + state.forceIDR.Store(false) } func encodeNativeHEVCD3D11Texture(device, texture unsafe.Pointer, inputWidth, inputHeight, encodeWidth, encodeHeight, fps int, dxgiFormat uint32, forceIDR bool) ([]byte, error) { diff --git a/Overlord-Client/cmd/agent/capture/virtual_windows.go b/Overlord-Client/cmd/agent/capture/virtual_windows.go index 8904957..1ff5315 100644 --- a/Overlord-Client/cmd/agent/capture/virtual_windows.go +++ b/Overlord-Client/cmd/agent/capture/virtual_windows.go @@ -213,7 +213,7 @@ var ( ) func VirtualTryDirectH264Frame() (wire.Frame, time.Duration, time.Duration, bool, error) { - if blockCodec() != "h264" { + if backstageCodec() != "h264" { return wire.Frame{}, 0, 0, false, nil } return virtualDupState.captureDirectH264() @@ -280,12 +280,12 @@ func (s *virtualDuplicationState) captureDirectH264() (wire.Frame, time.Duration } func (s *virtualDuplicationState) encodeCachedDirectH264(capStart time.Time, width, height int) (wire.Frame, time.Duration, time.Duration, bool, error) { - fps := activeH264FPS() + fps := activeH264FPSForStream("backstage") encStart := time.Now() - out, _, err := encodeH264D3D11Texture(h264D3D11TextureRequest{ + out, _, err := encodeH264D3D11Texture("backstage", h264D3D11TextureRequest{ Device: unsafe.Pointer(s.device), Texture: unsafe.Pointer(s.h264LastTex), InputWidth: width, InputHeight: height, EncodeWidth: width, EncodeHeight: height, - FPS: fps, DXGIFormat: s.h264LastDesc.Format, ForceIDR: webrtcpub.ConsumeKeyframeRequest(), + FPS: fps, DXGIFormat: s.h264LastDesc.Format, ForceIDR: webrtcpub.ConsumeKeyframeRequest(webrtcpub.Kindbackstage), }) encodeDur := time.Since(encStart) if err != nil { diff --git a/Overlord-Client/cmd/agent/capture/win_duplication.go b/Overlord-Client/cmd/agent/capture/win_duplication.go index c32374d..2a01937 100644 --- a/Overlord-Client/cmd/agent/capture/win_duplication.go +++ b/Overlord-Client/cmd/agent/capture/win_duplication.go @@ -688,7 +688,7 @@ func (s *duplicationState) closeLocked() { s.h264LastTex = nil s.h264LastDesc = d3d11Texture2DDesc{} } - resetH264D3D11TextureEncoder() + resetH264D3D11TextureEncoder("desktop") resetNativeHEVCD3D11TextureEncoder() if s.context != nil { s.context.Release() @@ -1027,7 +1027,7 @@ func encodeDesktopD3D11Texture(codec string, req h264D3D11TextureRequest) ([]byt req.EncodeWidth, req.EncodeHeight, req.FPS, req.DXGIFormat, req.ForceIDR) return out, "NVIDIA NVENC", err } - return encodeH264D3D11Texture(req) + return encodeH264D3D11Texture("desktop", req) } func (s *duplicationState) cacheH264TextureLocked(src *d3d11Texture2D, srcDesc d3d11Texture2DDesc) error { diff --git a/Overlord-Client/cmd/agent/handlers/backstage.go b/Overlord-Client/cmd/agent/handlers/backstage.go index f770fd9..a354906 100644 --- a/Overlord-Client/cmd/agent/handlers/backstage.go +++ b/Overlord-Client/cmd/agent/handlers/backstage.go @@ -32,7 +32,7 @@ func backstageStart(ctx context.Context, env *rt.Env, autoStartExplorer bool) er //garble:controlflow block_splits=10 junk_jumps=10 flatten_passes=2 fps := activebackstageTargetFPS() interval := time.Second / time.Duration(fps) - capture.SetH264TargetFPS(fps) + capture.SetBackstageH264TargetFPS(fps) capture.SetFrameFlowTargetFPS(fps) log.Printf("backstage: starting stream (target fps %d)", fps) @@ -62,7 +62,7 @@ func backstageStart(ctx context.Context, env *rt.Env, autoStartExplorer bool) er fps := activebackstageTargetFPS() if fps != currentFPS { currentFPS = fps - capture.SetH264TargetFPS(fps) + capture.SetBackstageH264TargetFPS(fps) capture.SetFrameFlowTargetFPS(fps) ticker.Reset(time.Second / time.Duration(fps)) log.Printf("backstage: target fps changed to %d", fps) @@ -90,7 +90,7 @@ func activebackstageTargetFPS() int { func SetbackstageTargetFPS(fps int) int { fps = clampDesktopTargetFPS(fps) backstageTargetFPS.Store(int64(fps)) - capture.SetH264TargetFPS(fps) + capture.SetBackstageH264TargetFPS(fps) capture.SetFrameFlowTargetFPS(fps) return fps } diff --git a/Overlord-Client/cmd/agent/handlers/command.go b/Overlord-Client/cmd/agent/handlers/command.go index 953457d..abd8388 100644 --- a/Overlord-Client/cmd/agent/handlers/command.go +++ b/Overlord-Client/cmd/agent/handlers/command.go @@ -832,7 +832,7 @@ func HandleCommand(ctx context.Context, env *runtime.Env, envelope map[string]in log.Printf("desktop: set quality=%d codec=%s software_h264=%v", quality, codec, softwareH264) } capture.SetDesktopSoftwareH264(softwareH264) - capture.SetQualityAndCodec(quality, codec) + capture.SetDesktopQualityAndCodec(quality, codec) sendCommandResultSafe(env, cmdID, true, "") return nil case "desktop_request_keyframe": @@ -1335,7 +1335,7 @@ func HandleCommand(ctx context.Context, env *runtime.Env, envelope map[string]in } } log.Printf("backstage: set quality=%d codec=%s", quality, codec) - capture.SetQualityAndCodec(quality, codec) + capture.SetBackstageQualityAndCodec(quality, codec) sendCommandResultSafe(env, cmdID, true, "") return nil case "backstage_request_keyframe": @@ -2209,7 +2209,7 @@ func HandleCommand(ctx context.Context, env *runtime.Env, envelope map[string]in } } log.Printf("hidden: set quality=%d codec=%s", quality, codec) - capture.SetQualityAndCodec(quality, codec) + capture.SetBackstageQualityAndCodec(quality, codec) sendCommandResultSafe(env, cmdID, true, "") return nil case "virtual_request_keyframe": @@ -2447,6 +2447,7 @@ func HandleCommand(ctx context.Context, env *runtime.Env, envelope map[string]in fps = clampedFPS } env.WebcamFPS = fps + capture.SetWebcamH264TargetFPS(fps) env.WebcamUseMaxFPS = useMax sendCommandResultSafe(env, cmdID, true, "") return nil diff --git a/Overlord-Client/cmd/agent/handlers/desktop.go b/Overlord-Client/cmd/agent/handlers/desktop.go index 32a7c70..6e78c56 100644 --- a/Overlord-Client/cmd/agent/handlers/desktop.go +++ b/Overlord-Client/cmd/agent/handlers/desktop.go @@ -32,7 +32,7 @@ func DesktopStart(ctx context.Context, env *rt.Env) error { //garble:controlflow block_splits=10 junk_jumps=10 flatten_passes=2 fps := activeDesktopTargetFPS() interval := time.Second / time.Duration(fps) - capture.SetH264TargetFPS(fps) + capture.SetDesktopH264TargetFPS(fps) capture.SetFrameFlowTargetFPS(fps) log.Printf("desktop: starting stream (target fps %d)", fps) ticker := time.NewTicker(interval) @@ -47,7 +47,7 @@ func DesktopStart(ctx context.Context, env *rt.Env) error { fps := activeDesktopTargetFPS() if fps != currentFPS { currentFPS = fps - capture.SetH264TargetFPS(fps) + capture.SetDesktopH264TargetFPS(fps) capture.SetFrameFlowTargetFPS(fps) ticker.Reset(time.Second / time.Duration(fps)) log.Printf("desktop: target fps changed to %d", fps) @@ -74,7 +74,7 @@ func activeDesktopTargetFPS() int { func SetDesktopTargetFPS(fps int) int { fps = clampDesktopTargetFPS(fps) desktopTargetFPS.Store(int64(fps)) - capture.SetH264TargetFPS(fps) + capture.SetDesktopH264TargetFPS(fps) capture.SetFrameFlowTargetFPS(fps) return fps } diff --git a/Overlord-Client/cmd/agent/handlers/virtual.go b/Overlord-Client/cmd/agent/handlers/virtual.go index 0224fd7..1cb9091 100644 --- a/Overlord-Client/cmd/agent/handlers/virtual.go +++ b/Overlord-Client/cmd/agent/handlers/virtual.go @@ -16,7 +16,7 @@ var ( func VirtualStart(ctx context.Context, env *rt.Env) error { fps := activeVirtualTargetFPS() interval := time.Second / time.Duration(fps) - capture.SetH264TargetFPS(fps) + capture.SetBackstageH264TargetFPS(fps) capture.SetFrameFlowTargetFPS(fps) log.Printf("virtual: starting stream (target fps %d)", fps) @@ -38,7 +38,7 @@ func VirtualStart(ctx context.Context, env *rt.Env) error { fps := activeVirtualTargetFPS() if fps != currentFPS { currentFPS = fps - capture.SetH264TargetFPS(fps) + capture.SetBackstageH264TargetFPS(fps) capture.SetFrameFlowTargetFPS(fps) ticker.Reset(time.Second / time.Duration(fps)) log.Printf("virtual: target fps changed to %d", fps) @@ -65,7 +65,7 @@ func activeVirtualTargetFPS() int { func SetVirtualTargetFPS(fps int) int { fps = clampDesktopTargetFPS(fps) virtualTargetFPS.Store(int64(fps)) - capture.SetH264TargetFPS(fps) + capture.SetBackstageH264TargetFPS(fps) capture.SetFrameFlowTargetFPS(fps) return fps } diff --git a/Overlord-Client/cmd/agent/handlers/webcam.go b/Overlord-Client/cmd/agent/handlers/webcam.go index 145cac5..76aba8f 100644 --- a/Overlord-Client/cmd/agent/handlers/webcam.go +++ b/Overlord-Client/cmd/agent/handlers/webcam.go @@ -21,6 +21,7 @@ func WebcamStart(ctx context.Context, env *rt.Env) error { if fps > 120 { fps = 120 } + capture.SetWebcamH264TargetFPS(fps) interval := time.Second / time.Duration(fps) log.Printf("webcam: starting stream (max fps %d)", fps) defer capture.CleanupWebcam() diff --git a/Overlord-Client/cmd/agent/webrtcpub/p2p_pion.go b/Overlord-Client/cmd/agent/webrtcpub/p2p_pion.go index eadb010..68f9118 100644 --- a/Overlord-Client/cmd/agent/webrtcpub/p2p_pion.go +++ b/Overlord-Client/cmd/agent/webrtcpub/p2p_pion.go @@ -147,7 +147,7 @@ func StartP2POffer(ctx context.Context, kind Kind, sessionID string, offerSDP st return "", fmt.Errorf("add video transceiver: %w", err) } if sender := tx.Sender(); sender != nil { - go drainRTCP(sender) + go drainRTCP(sender, kind) } } if hasAudio { @@ -172,7 +172,7 @@ func StartP2POffer(ctx context.Context, kind Kind, sessionID string, offerSDP st return "", fmt.Errorf("add audio transceiver: %w", err) } if sender := tx.Sender(); sender != nil { - go drainRTCP(sender) + go drainRTCP(sender, "") } } diff --git a/Overlord-Client/cmd/agent/webrtcpub/state.go b/Overlord-Client/cmd/agent/webrtcpub/state.go index 11f11a1..a4d392c 100644 --- a/Overlord-Client/cmd/agent/webrtcpub/state.go +++ b/Overlord-Client/cmd/agent/webrtcpub/state.go @@ -110,7 +110,7 @@ func registerVideoWriter(kind Kind, id string, w VideoWriter) { entry.video = newLatestVideoWriter(w) bucket[id] = entry writersMu.Unlock() - RequestKeyframe() + RequestKeyframe(kind) } func registerAudioWriter(kind Kind, id string, w AudioWriter) { @@ -155,14 +155,29 @@ func IsActive(kind Kind) bool { return len(writers[string(kind)]) > 0 } -var keyframeWanted atomic.Bool +var ( + desktopKeyframeWanted atomic.Bool + backstageKeyframeWanted atomic.Bool + webcamKeyframeWanted atomic.Bool +) -func RequestKeyframe() { - keyframeWanted.Store(true) +func keyframeRequestFlag(kind Kind) *atomic.Bool { + switch kind { + case Kindbackstage: + return &backstageKeyframeWanted + case KindWebcam: + return &webcamKeyframeWanted + default: + return &desktopKeyframeWanted + } } -func ConsumeKeyframeRequest() bool { - return keyframeWanted.Swap(false) +func RequestKeyframe(kind Kind) { + keyframeRequestFlag(kind).Store(true) +} + +func ConsumeKeyframeRequest(kind Kind) bool { + return keyframeRequestFlag(kind).Swap(false) } func WriteH264(kind Kind, nalu []byte, dur time.Duration) error { diff --git a/Overlord-Client/cmd/agent/webrtcpub/state_test.go b/Overlord-Client/cmd/agent/webrtcpub/state_test.go index 1f189a7..53d02c0 100644 --- a/Overlord-Client/cmd/agent/webrtcpub/state_test.go +++ b/Overlord-Client/cmd/agent/webrtcpub/state_test.go @@ -67,3 +67,21 @@ func TestWriteH264DoesNotBlockAndKeepsLatestPendingFrame(t *testing.T) { } t.Fatal("latest pending frame was not delivered") } + +func TestKeyframeRequestsAreIsolatedByStreamKind(t *testing.T) { + for _, kind := range []Kind{KindDesktop, Kindbackstage, KindWebcam} { + _ = ConsumeKeyframeRequest(kind) + } + + RequestKeyframe(Kindbackstage) + + if ConsumeKeyframeRequest(KindDesktop) { + t.Fatal("backstage keyframe request leaked into desktop stream") + } + if ConsumeKeyframeRequest(KindWebcam) { + t.Fatal("backstage keyframe request leaked into webcam stream") + } + if !ConsumeKeyframeRequest(Kindbackstage) { + t.Fatal("backstage keyframe request was not retained for backstage stream") + } +} diff --git a/Overlord-Client/cmd/agent/webrtcpub/whip_pion.go b/Overlord-Client/cmd/agent/webrtcpub/whip_pion.go index 1620292..8b7a8fa 100644 --- a/Overlord-Client/cmd/agent/webrtcpub/whip_pion.go +++ b/Overlord-Client/cmd/agent/webrtcpub/whip_pion.go @@ -25,7 +25,7 @@ import ( "github.com/pion/webrtc/v4/pkg/media" ) -func drainRTCP(sender *webrtc.RTPSender) { +func drainRTCP(sender *webrtc.RTPSender, kind Kind) { buf := make([]byte, 1500) for { n, _, err := sender.Read(buf) @@ -39,7 +39,9 @@ func drainRTCP(sender *webrtc.RTPSender) { for _, p := range packets { switch p.(type) { case *rtcp.PictureLossIndication, *rtcp.FullIntraRequest: - handleRTCPKeyframeFeedback() + if kind != "" { + handleRTCPKeyframeFeedback(kind) + } } } } @@ -48,11 +50,15 @@ func drainRTCP(sender *webrtc.RTPSender) { const h264SDPFmtpLine = "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=640034" var rtcpKeyframeLogOnce sync.Once -var lastRTCPKeyframeRequest atomic.Int64 +var ( + lastDesktopRTCPKeyframeRequest atomic.Int64 + lastBackstageRTCPKeyframeRequest atomic.Int64 + lastWebcamRTCPKeyframeRequest atomic.Int64 +) const rtcpKeyframeMinInterval = 750 * time.Millisecond -func handleRTCPKeyframeFeedback() { +func handleRTCPKeyframeFeedback(kind Kind) { if !honorRTCPKeyframes() { rtcpKeyframeLogOnce.Do(func() { log.Printf("webrtcpub: RTCP video keyframe recovery disabled by OVERLORD_WEBRTC_RTCP_KEYFRAMES") @@ -61,15 +67,26 @@ func handleRTCPKeyframeFeedback() { } now := time.Now().UnixNano() for { - last := lastRTCPKeyframeRequest.Load() + last := rtcpKeyframeTimestamp(kind).Load() if last > 0 && time.Duration(now-last) < rtcpKeyframeMinInterval { return } - if lastRTCPKeyframeRequest.CompareAndSwap(last, now) { + if rtcpKeyframeTimestamp(kind).CompareAndSwap(last, now) { break } } - RequestKeyframe() + RequestKeyframe(kind) +} + +func rtcpKeyframeTimestamp(kind Kind) *atomic.Int64 { + switch kind { + case Kindbackstage: + return &lastBackstageRTCPKeyframeRequest + case KindWebcam: + return &lastWebcamRTCPKeyframeRequest + default: + return &lastDesktopRTCPKeyframeRequest + } } func honorRTCPKeyframes() bool { @@ -188,7 +205,7 @@ func Start(ctx context.Context, kind Kind, opts Options) (*Publisher, error) { return nil, fmt.Errorf("add video transceiver: %w", err) } if sender := tx.Sender(); sender != nil { - go drainRTCP(sender) + go drainRTCP(sender, kind) } } if opts.HasAudio { @@ -213,7 +230,7 @@ func Start(ctx context.Context, kind Kind, opts Options) (*Publisher, error) { return nil, fmt.Errorf("add audio transceiver: %w", err) } if sender := tx.Sender(); sender != nil { - go drainRTCP(sender) + go drainRTCP(sender, "") } }