vxaboveground/overlord
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

major remote desktop improvements

Browse Source
This commit is contained in:
KingKDot
2026-07-30 00:22:08 -05:00
parent a68172538d
commit 095a119bf4
35 changed files with 984 additions and 120 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Overlord-Client/cmd/agent/capture/amf_d3d11_windows.go
+2 -1
View File
@@ -46,7 +46,8 @@ func (b *amfD3D11TextureBackend) Encode(req h264D3D11TextureRequest) ([]byte, er
b.device, b.inputWidth, b.inputHeight = req.Device, req.InputWidth, req.InputHeight
b.encodeWidth, b.encodeHeight, b.fps, b.dxgiFormat = req.EncodeWidth, req.EncodeHeight, req.FPS, req.DXGIFormat
}
forceIDR := req.ForceIDR || b.forceIDR.Swap(false)
pendingIDR := b.forceIDR.Swap(false)
forceIDR := req.ForceIDR || pendingIDR
for attempts := 0; attempts < 2; attempts++ {
result, outputSize, err := b.encoder.Encode(req.Texture, forceIDR, b.output)
if err != nil {
Overlord-Client/cmd/agent/capture/capture.go
+93 -10
View File
@@ -108,19 +108,41 @@ func CaptureAndSend(ctx context.Context, env *rt.Env) error {
time.Sleep(50 * time.Millisecond)
}
codec := desktopCodec()
directCanvasVideo := (codec == "h264" || codec == "hevc") &&
(codec != "h264" || !useDesktopSoftwareH264()) &&
useDesktopDuplication() &&
!(codec == "h264" && webrtcpub.IsActive(webrtcpub.KindDesktop))
directSlotAcquired := false
if directCanvasVideo {
if !AcquireFrameSlot() {
return nil
}
directSlotAcquired = true
}
if frame, captureDur, encodeDur, used, err := tryBuildDirectH264Frame(display); used {
if err != nil {
if directSlotAcquired {
ReleaseFrameSlot()
directSlotAcquired = false
}
suppressDesktopHardwareH264(err)
log.Printf("capture: direct video encode failed: %v; falling back to the software H.264 path", err)
} else {
emitDesktopCursor(ctx, env, display, frame.Header.Width, frame.Header.Height)
if len(frame.Data) == 0 {
if directSlotAcquired {
ReleaseFrameSlot()
}
consecutiveCaptureFails.Store(0)
return nil
}
consecutiveCaptureFails.Store(0)
return sendCompletedFrame(ctx, env, frame, display, t0, captureDur, encodeDur)
return sendCompletedFrame(ctx, env, frame, display, t0, captureDur, encodeDur, directSlotAcquired)
}
} else if directSlotAcquired {
ReleaseFrameSlot()
}
img, err := safeCaptureDisplay(display)
@@ -247,7 +269,7 @@ func CaptureAndSend(ctx context.Context, env *rt.Env) error {
return err
}
func sendCompletedFrame(ctx context.Context, env *rt.Env, frame wire.Frame, display int, t0 time.Time, captureDur, encodeDur time.Duration) error {
func sendCompletedFrame(ctx context.Context, env *rt.Env, frame wire.Frame, display int, t0 time.Time, captureDur, encodeDur time.Duration, slotAcquired bool) error {
now := time.Now()
fps := frameFPS(now)
if fps <= 0 {
@@ -255,6 +277,9 @@ func sendCompletedFrame(ctx context.Context, env *rt.Env, frame wire.Frame, disp
}
frame.Header.FPS = fps
if ctx.Err() != nil {
if slotAcquired {
ReleaseFrameSlot()
}
return nil
}
if frame.Header.Format == "h264" && webrtcpub.IsActive(webrtcpub.KindDesktop) {
@@ -274,10 +299,16 @@ func sendCompletedFrame(ctx context.Context, env *rt.Env, frame wire.Frame, disp
statTotalNs.Add(totalDur.Nanoseconds())
statBytes.Add(int64(len(frame.Data)))
emitDesktopStreamStats(ctx, env, frame, fps, captureDur, encodeDur, sendDur, totalDur, "webrtc")
if slotAcquired {
ReleaseFrameSlot()
}
return nil
}
if !AcquireFrameSlot() {
return nil
if !slotAcquired {
if !AcquireFrameSlot() {
return nil
}
slotAcquired = true
}
sendStart := time.Now()
err := wire.WriteMsg(ctx, env.Conn, frame)
@@ -675,6 +706,8 @@ func SetDesktopSoftwareH264(enabled bool) {
}
desktopHardwareFallbackUntil.Store(0)
resetH264Encoder()
lastKeyframe.Store(0)
webrtcpub.RequestKeyframe(webrtcpub.KindDesktop)
requestFullFrames(2)
log.Printf("capture: desktop software h264 %v", enabled)
}
@@ -796,6 +829,25 @@ const (
backstageChangeThresh = 1
)
var videoKeyframeEvery = configuredVideoKeyframeInterval()
func configuredVideoKeyframeInterval() time.Duration {
raw := strings.TrimSpace(os.Getenv("OVERLORD_VIDEO_KEYFRAME_INTERVAL"))
if raw == "" || strings.EqualFold(raw, "off") || raw == "0" {
return 0
}
if interval, err := time.ParseDuration(raw); err == nil {
if interval > 0 {
return interval
}
return 0
}
if seconds, err := strconv.Atoi(raw); err == nil && seconds > 0 {
return time.Duration(seconds) * time.Second
}
return 0
}
func buildFrame(img *image.RGBA, display int, quality int) (wire.Frame, time.Duration, error) {
encStart := time.Now()
width := img.Bounds().Dx()
@@ -808,15 +860,22 @@ func buildFrame(img *image.RGBA, display int, quality int) (wire.Frame, time.Dur
log.Printf("capture: hevc skipped for odd dimensions (%dx%d), falling back to h264/jpeg", width, height)
codec = "h264"
} else {
if webrtcpub.ConsumeKeyframeRequest(webrtcpub.KindDesktop) {
requestedKeyframe := webrtcpub.ConsumeKeyframeRequest(webrtcpub.KindDesktop)
if requestedKeyframe || lastKeyframe.Load() == 0 ||
(videoKeyframeEvery > 0 && now.Sub(time.Unix(0, lastKeyframe.Load())) >= videoKeyframeEvery) {
resetHEVCEncoder()
}
hevcBytes, err := encodeHEVCFrame(img)
if err == nil && len(hevcBytes) > 0 {
lastKeyframe.Store(now.UnixNano())
if retainKeyframeRequestUntilOutput(webrtcpub.KindDesktop, requestedKeyframe, "hevc", hevcBytes) {
lastKeyframe.Store(now.UnixNano())
}
statFullFrames.Add(1)
return wire.Frame{Type: "frame", Header: wire.FrameHeader{Monitor: display, FPS: 0, Format: "hevc"}, Data: hevcBytes}, time.Since(encStart), nil
}
if requestedKeyframe {
webrtcpub.RequestKeyframe(webrtcpub.KindDesktop)
}
hevcWarnOnce.Do(func() {
log.Printf("capture: hevc encode unavailable, falling back to h264/jpeg: %v (%s)", err, hevcAvailabilityDetail())
})
@@ -836,15 +895,22 @@ func buildFrame(img *image.RGBA, display int, quality int) (wire.Frame, time.Dur
log.Printf("capture: h264 skipped for odd dimensions (%dx%d), falling back to jpeg", width, height)
codec = "jpeg"
} else {
if webrtcpub.ConsumeKeyframeRequest(webrtcpub.KindDesktop) {
requestedKeyframe := webrtcpub.ConsumeKeyframeRequest(webrtcpub.KindDesktop)
if requestedKeyframe || lastKeyframe.Load() == 0 ||
(videoKeyframeEvery > 0 && now.Sub(time.Unix(0, lastKeyframe.Load())) >= videoKeyframeEvery) {
resetH264Encoder()
}
h264Bytes, err := encodeH264Frame(img)
if err == nil && len(h264Bytes) > 0 {
lastKeyframe.Store(now.UnixNano())
if retainKeyframeRequestUntilOutput(webrtcpub.KindDesktop, requestedKeyframe, "h264", h264Bytes) {
lastKeyframe.Store(now.UnixNano())
}
statFullFrames.Add(1)
return wire.Frame{Type: "frame", Header: wire.FrameHeader{Monitor: display, FPS: 0, Format: "h264"}, Data: h264Bytes}, time.Since(encStart), nil
}
if requestedKeyframe {
webrtcpub.RequestKeyframe(webrtcpub.KindDesktop)
}
h264WarnOnce.Do(func() {
detail := h264AvailabilityDetail()
if detail != "" {
@@ -941,16 +1007,23 @@ func buildFramebackstage(img *image.RGBA, display int, quality int) (wire.Frame,
log.Printf("backstage capture: h264 skipped for odd dimensions (%dx%d), falling back to jpeg", width, height)
codec = "jpeg"
} else {
if webrtcpub.ConsumeKeyframeRequest(webrtcpub.Kindbackstage) {
requestedKeyframe := webrtcpub.ConsumeKeyframeRequest(webrtcpub.Kindbackstage)
if requestedKeyframe || backstageLastKeyframe.Load() == 0 ||
(videoKeyframeEvery > 0 && now.Sub(time.Unix(0, backstageLastKeyframe.Load())) >= videoKeyframeEvery) {
resetH264Encoderbackstage()
}
h264Bytes, err := encodeH264Framebackstage(img)
if err == nil && len(h264Bytes) > 0 {
backstageLastKeyframe.Store(now.UnixNano())
if retainKeyframeRequestUntilOutput(webrtcpub.Kindbackstage, requestedKeyframe, "h264", h264Bytes) {
backstageLastKeyframe.Store(now.UnixNano())
}
statFullFrames.Add(1)
frame := wire.Frame{Type: "frame", Header: wire.FrameHeader{Monitor: display, FPS: 0, Format: "h264", Backstage: true}, Data: h264Bytes}
return frame, time.Since(encStart), nil
}
if requestedKeyframe {
webrtcpub.RequestKeyframe(webrtcpub.Kindbackstage)
}
backstageH264WarnOnce.Do(func() {
detail := h264AvailabilityDetail()
if detail != "" {
@@ -1671,6 +1744,7 @@ func backstageCodec() string {
}
func SetDesktopQualityAndCodec(quality int, codec string) {
previousCodec := desktopCodec()
if quality > 0 {
if quality > 100 {
quality = 100
@@ -1706,6 +1780,10 @@ func SetDesktopQualityAndCodec(quality int, codec string) {
if s != "hevc" {
resetHEVCEncoder()
}
if previousCodec != s && (s == "h264" || s == "hevc") {
lastKeyframe.Store(0)
webrtcpub.RequestKeyframe(webrtcpub.KindDesktop)
}
case "":
default:
desktopOverrideCodec.Store("jpeg")
@@ -1715,6 +1793,7 @@ func SetDesktopQualityAndCodec(quality int, codec string) {
}
func SetBackstageQualityAndCodec(quality int, codec string) {
previousCodec := backstageCodec()
if quality > 0 {
if quality > 100 {
quality = 100
@@ -1734,6 +1813,10 @@ func SetBackstageQualityAndCodec(quality int, codec string) {
if s != "h264" {
resetH264Encoderbackstage()
}
if previousCodec != s && s == "h264" {
backstageLastKeyframe.Store(0)
webrtcpub.RequestKeyframe(webrtcpub.Kindbackstage)
}
case "":
default:
backstageOverrideCodec.Store("jpeg")
Overlord-Client/cmd/agent/capture/capture_test.go
+6 -6
View File
@@ -456,16 +456,16 @@ func TestSetFrameFlowTargetFPSScalesHighFPS(t *testing.T) {
t.Setenv("OVERLORD_DESKTOP_IN_FLIGHT_FRAMES", "")
ResetFrameSlots()
SetFrameFlowTargetFPS(60)
if got := activeFrameSlotLimit(); got != 2 {
t.Fatalf("expected 60 fps slot limit 2, got %d", got)
if got := activeFrameSlotLimit(); got != 4 {
t.Fatalf("expected 60 fps slot limit 4, got %d", got)
}
SetFrameFlowTargetFPS(120)
if got := activeFrameSlotLimit(); got != 4 {
t.Fatalf("expected 120 fps slot limit 4, got %d", got)
if got := activeFrameSlotLimit(); got != 8 {
t.Fatalf("expected 120 fps slot limit 8, got %d", got)
}
SetFrameFlowTargetFPS(240)
if got := activeFrameSlotLimit(); got != 8 {
t.Fatalf("expected 240 fps slot limit 8, got %d", got)
if got := activeFrameSlotLimit(); got != 12 {
t.Fatalf("expected 240 fps slot limit 12, got %d", got)
}
}
Overlord-Client/cmd/agent/capture/desktop_encoder_capabilities_windows.go
+18 -1
View File
@@ -123,6 +123,9 @@ func probeDesktopEncoderCapabilities(display int, bounds image.Rectangle) Deskto
for _, probe := range probes {
maxFPS := 0
for _, fps := range fpsOptions {
if fps > desktopProfileFPSCeiling(size.height) {
continue
}
req := h264D3D11TextureRequest{
Device: unsafe.Pointer(device), Texture: unsafe.Pointer(texture),
InputWidth: nativeWidth, InputHeight: nativeHeight,
@@ -140,7 +143,7 @@ func probeDesktopEncoderCapabilities(display int, bounds image.Rectangle) Deskto
continue
}
for _, fps := range fpsOptions {
if fps <= maxFPS {
if fps <= maxFPS && fps <= desktopProfileFPSCeiling(size.height) {
key := desktopProfileKey(size.width, size.height, fps)
providersByProfile[key] = appendUniqueString(providersByProfile[key], probe.name)
}
@@ -254,6 +257,20 @@ func safeDesktopProfiles(nativeWidth, nativeHeight int) []DesktopEncoderProfile
return profiles
}
// The probe repeatedly encodes an unchanged synthetic texture and excludes
// capture, conversion, transport, and browser decode. Bound its advertised
// result to practical end-to-end desktop profiles.
func desktopProfileFPSCeiling(height int) int {
switch {
case height <= 1080:
return 240
case height <= 1440:
return 120
default:
return 60
}
}
func desktopProfileKey(width, height, fps int) string {
return fmt.Sprintf("%dx%d@%d", width, height, fps)
}
Overlord-Client/cmd/agent/capture/desktop_encoder_capabilities_windows_test.go
+21
View File
@@ -0,0 +1,21 @@
package capture
import "testing"
func TestDesktopProfileFPSCeiling(t *testing.T) {
tests := []struct {
height int
want int
}{
{height: 720, want: 240},
{height: 1080, want: 240},
{height: 1440, want: 120},
{height: 2160, want: 60},
{height: 4320, want: 60},
}
for _, test := range tests {
if got := desktopProfileFPSCeiling(test.height); got != test.want {
t.Fatalf("height %d: expected ceiling %d, got %d", test.height, test.want, got)
}
}
}
Overlord-Client/cmd/agent/capture/flowcontrol.go
+3 -1
View File
@@ -57,8 +57,10 @@ func SetFrameFlowTargetFPS(fps int) {
limit := defaultMaxInFlightFrames
switch {
case fps >= 180:
limit = 8
limit = 12
case fps >= 120:
limit = 8
case fps >= 60:
limit = 4
}
if env := strings.TrimSpace(os.Getenv("OVERLORD_DESKTOP_IN_FLIGHT_FRAMES")); env != "" {
Overlord-Client/cmd/agent/capture/h264_codec_test.go
+26
View File
@@ -9,6 +9,9 @@ import (
"strings"
"sync"
"testing"
"time"
"overlord-client/cmd/agent/webrtcpub"
)
func resetCodecSelectionForTest() {
@@ -18,6 +21,8 @@ func resetCodecSelectionForTest() {
backstageOverrideCodec.Store("")
desktopOverrideQuality.Store(0)
backstageOverrideQuality.Store(0)
_ = webrtcpub.ConsumeKeyframeRequest(webrtcpub.KindDesktop)
_ = webrtcpub.ConsumeKeyframeRequest(webrtcpub.Kindbackstage)
}
func TestSetQualityAndCodec_H264FallbackDependsOnAvailability(t *testing.T) {
@@ -72,6 +77,27 @@ func TestDesktopAndBackstageCodecSelectionsAreIndependent(t *testing.T) {
}
}
func TestSwitchingToH264RequestsRecoveryPoint(t *testing.T) {
if !h264Available() {
t.Skip("h264 is unavailable in this build")
}
t.Cleanup(resetCodecSelectionForTest)
resetCodecSelectionForTest()
SetDesktopQualityAndCodec(80, "jpeg")
_ = webrtcpub.ConsumeKeyframeRequest(webrtcpub.KindDesktop)
lastKeyframe.Store(time.Now().UnixNano())
SetDesktopQualityAndCodec(80, "h264")
if lastKeyframe.Load() != 0 {
t.Fatal("codec transition did not invalidate the previous keyframe timestamp")
}
if !webrtcpub.ConsumeKeyframeRequest(webrtcpub.KindDesktop) {
t.Fatal("codec transition did not request an H.264 recovery point")
}
}
func TestDesktopHardwareFailureDoesNotChangeBackstageCodec(t *testing.T) {
t.Cleanup(func() {
desktopHardwareFallbackUntil.Store(0)
Overlord-Client/cmd/agent/capture/h264_direct_windows.go
+10 -3
View File
@@ -13,7 +13,7 @@ func directVideoKeyframeDue(requested bool, nowNs, lastKeyframeNs int64) bool {
if requested || lastKeyframeNs == 0 {
return true
}
return keyframeEvery > 0 && time.Duration(nowNs-lastKeyframeNs) >= keyframeEvery
return videoKeyframeEvery > 0 && time.Duration(nowNs-lastKeyframeNs) >= videoKeyframeEvery
}
func tryBuildDirectH264Frame(display int) (wire.Frame, time.Duration, time.Duration, bool, error) {
@@ -22,8 +22,9 @@ func tryBuildDirectH264Frame(display int) (wire.Frame, time.Duration, time.Durat
return wire.Frame{}, 0, 0, false, nil
}
now := time.Now()
requestedKeyframe := webrtcpub.ConsumeKeyframeRequest(webrtcpub.KindDesktop)
forceKeyframe := directVideoKeyframeDue(
webrtcpub.ConsumeKeyframeRequest(webrtcpub.KindDesktop),
requestedKeyframe,
now.UnixNano(),
lastKeyframe.Load(),
)
@@ -38,12 +39,18 @@ func tryBuildDirectH264Frame(display int) (wire.Frame, time.Duration, time.Durat
data, width, height, captureDur, encodeDur, used, err = captureDisplayDXGIH264(display, forceKeyframe)
}
if !used || err != nil {
if requestedKeyframe {
webrtcpub.RequestKeyframe(webrtcpub.KindDesktop)
}
return wire.Frame{}, captureDur, encodeDur, used, err
}
if len(data) == 0 {
if requestedKeyframe {
webrtcpub.RequestKeyframe(webrtcpub.KindDesktop)
}
return wire.Frame{}, captureDur, encodeDur, true, nil
}
if forceKeyframe {
if retainKeyframeRequestUntilOutput(webrtcpub.KindDesktop, requestedKeyframe, codec, data) {
lastKeyframe.Store(now.UnixNano())
}
statFullFrames.Add(1)
Overlord-Client/cmd/agent/capture/nvenc_d3d11_smoke_windows.go
+13 -6
View File
@@ -122,9 +122,14 @@ static void nvenc_configure_h264_stream(NV_ENC_CONFIG *cfg, int fps) {
(void)fps;
cfg->gopLength = NVENC_INFINITE_GOPLENGTH;
cfg->frameIntervalP = 1;
// The low-latency presets otherwise constrain an IDR to roughly the same
// bit budget as a P-frame. That makes recovery frames visibly pulse or
// smear under motion. Allow IDRs a larger transient budget while keeping
// the normal one-frame VBV/CBR latency behavior.
cfg->rcParams.lowDelayKeyFrameScale = 4;
cfg->encodeCodecConfig.h264Config.idrPeriod = NVENC_INFINITE_GOPLENGTH;
cfg->encodeCodecConfig.h264Config.repeatSPSPPS = 1;
cfg->encodeCodecConfig.h264Config.level = NV_ENC_LEVEL_H264_52;
cfg->encodeCodecConfig.h264Config.level = NV_ENC_LEVEL_AUTOSELECT;
cfg->encodeCodecConfig.h264Config.h264VUIParameters.videoSignalTypePresentFlag = 1;
cfg->encodeCodecConfig.h264Config.h264VUIParameters.videoFormat = NV_ENC_VUI_VIDEO_FORMAT_UNSPECIFIED;
cfg->encodeCodecConfig.h264Config.h264VUIParameters.videoFullRangeFlag = 0;
@@ -393,7 +398,7 @@ static nvenc_d3d11_texture_create_result nvenc_create_d3d11_texture_encoder(ID3D
nvenc_release_texture_encoder(enc);
return out;
}
preset.presetCfg.profileGUID = codec == 1 ? NV_ENC_HEVC_PROFILE_MAIN_GUID : NV_ENC_H264_PROFILE_HIGH_GUID;
preset.presetCfg.profileGUID = codec == 1 ? NV_ENC_HEVC_PROFILE_MAIN_GUID : NV_ENC_H264_PROFILE_MAIN_GUID;
preset.presetCfg.rcParams.rateControlMode = NV_ENC_PARAMS_RC_CBR;
preset.presetCfg.rcParams.averageBitRate = (uint32_t)bitrate;
preset.presetCfg.rcParams.maxBitRate = (uint32_t)bitrate;
@@ -703,7 +708,7 @@ static nvenc_d3d11_create_result nvenc_create_d3d11_encoder(int width, int heigh
nvenc_release_encoder(enc);
return out;
}
preset.presetCfg.profileGUID = codec == 1 ? NV_ENC_HEVC_PROFILE_MAIN_GUID : NV_ENC_H264_PROFILE_HIGH_GUID;
preset.presetCfg.profileGUID = codec == 1 ? NV_ENC_HEVC_PROFILE_MAIN_GUID : NV_ENC_H264_PROFILE_MAIN_GUID;
preset.presetCfg.rcParams.rateControlMode = NV_ENC_PARAMS_RC_CBR;
preset.presetCfg.rcParams.averageBitRate = (uint32_t)bitrate;
preset.presetCfg.rcParams.maxBitRate = (uint32_t)bitrate;
@@ -1160,7 +1165,7 @@ static nvenc_d3d11_smoke_result run_nvenc_d3d11_smoke_c(int width, int height, i
return out;
}
preset.presetCfg.profileGUID = NV_ENC_H264_PROFILE_HIGH_GUID;
preset.presetCfg.profileGUID = NV_ENC_H264_PROFILE_MAIN_GUID;
preset.presetCfg.rcParams.rateControlMode = NV_ENC_PARAMS_RC_CBR;
preset.presetCfg.rcParams.averageBitRate = (uint32_t)bitrate;
preset.presetCfg.rcParams.maxBitRate = (uint32_t)bitrate;
@@ -1961,7 +1966,8 @@ func encodeNativeH264D3D11Texture(stream string, device, texture unsafe.Pointer,
}
state.encoder = enc
}
return state.encoder.EncodeTexture(texture, forceIDR || state.forceIDR.Swap(false))
pendingIDR := state.forceIDR.Swap(false)
return state.encoder.EncodeTexture(texture, forceIDR || pendingIDR)
}
func requestNativeH264D3D11TextureKeyframe(stream string) {
@@ -1997,7 +2003,8 @@ func encodeNativeHEVCD3D11Texture(device, texture unsafe.Pointer, inputWidth, in
}
nativeTextureHEVCEnc = enc
}
return nativeTextureHEVCEnc.EncodeTexture(texture, forceIDR || nativeTextureHEVCForceIDR.Swap(false))
pendingIDR := nativeTextureHEVCForceIDR.Swap(false)
return nativeTextureHEVCEnc.EncodeTexture(texture, forceIDR || pendingIDR)
}
func requestNativeHEVCD3D11TextureKeyframe() {
Overlord-Client/cmd/agent/capture/qsv_d3d11_windows.go
+2 -1
View File
@@ -42,7 +42,8 @@ func (b *qsvD3D11TextureBackend) Encode(req h264D3D11TextureRequest) ([]byte, er
b.device, b.inputWidth, b.inputHeight = req.Device, req.InputWidth, req.InputHeight
b.encodeWidth, b.encodeHeight, b.fps, b.dxgiFormat = req.EncodeWidth, req.EncodeHeight, req.FPS, req.DXGIFormat
}
forceIDR := req.ForceIDR || b.forceIDR.Swap(false)
pendingIDR := b.forceIDR.Swap(false)
forceIDR := req.ForceIDR || pendingIDR
for attempts := 0; attempts < 2; attempts++ {
result, size, err := b.encoder.Encode(req.Texture, forceIDR, b.output)
if err != nil {
Overlord-Client/cmd/agent/capture/video_access_unit.go
+73
View File
@@ -0,0 +1,73 @@
package capture
import (
"overlord-client/cmd/agent/h264util"
"overlord-client/cmd/agent/webrtcpub"
)
func videoAccessUnitIsKey(codec string, data []byte) bool {
if codec == "hevc" {
return inspectHEVCAccessUnit(data).hasIRAP
}
return h264util.IsIDR(data)
}
type hevcAccessUnit struct {
hasVPS bool
hasSPS bool
hasPPS bool
hasIRAP bool
}
func inspectHEVCAccessUnit(data []byte) hevcAccessUnit {
var out hevcAccessUnit
for i := 0; i+4 < len(data); {
startCodeLen := 0
switch {
case data[i] == 0 && data[i+1] == 0 && data[i+2] == 1:
startCodeLen = 3
case i+4 < len(data) && data[i] == 0 && data[i+1] == 0 && data[i+2] == 0 && data[i+3] == 1:
startCodeLen = 4
}
if startCodeLen == 0 {
i++
continue
}
nal := i + startCodeLen
if nal >= len(data) {
break
}
nalType := (data[nal] >> 1) & 0x3f
switch nalType {
case 32:
out.hasVPS = true
case 33:
out.hasSPS = true
case 34:
out.hasPPS = true
default:
if nalType >= 16 && nalType <= 21 {
out.hasIRAP = true
}
}
i = nal + 1
}
return out
}
func videoAccessUnitIsRecoveryPoint(codec string, data []byte) bool {
if codec == "hevc" {
accessUnit := inspectHEVCAccessUnit(data)
return accessUnit.hasVPS && accessUnit.hasSPS && accessUnit.hasPPS && accessUnit.hasIRAP
}
accessUnit := h264util.InspectAnnexB(data)
return accessUnit.HasSPS && accessUnit.HasPPS && accessUnit.HasIDR
}
func retainKeyframeRequestUntilOutput(kind webrtcpub.Kind, requested bool, codec string, data []byte) bool {
isKey := videoAccessUnitIsKey(codec, data)
if requested && !videoAccessUnitIsRecoveryPoint(codec, data) {
webrtcpub.RequestKeyframe(kind)
}
return isKey
}
Overlord-Client/cmd/agent/capture/video_access_unit_test.go
+38
View File
@@ -0,0 +1,38 @@
package capture
import (
"testing"
"overlord-client/cmd/agent/webrtcpub"
)
func TestRetainKeyframeRequestUntilIDROutput(t *testing.T) {
kind := webrtcpub.KindDesktop
_ = webrtcpub.ConsumeKeyframeRequest(kind)
if retainKeyframeRequestUntilOutput(kind, true, "h264", []byte{0, 0, 1, 0x41, 1}) {
t.Fatal("delta access unit was classified as a keyframe")
}
if !webrtcpub.ConsumeKeyframeRequest(kind) {
t.Fatal("keyframe request was not retained after delta output")
}
if !retainKeyframeRequestUntilOutput(kind, true, "h264", []byte{0, 0, 1, 0x65, 1}) {
t.Fatal("IDR access unit was not classified as a keyframe")
}
if !webrtcpub.ConsumeKeyframeRequest(kind) {
t.Fatal("keyframe request was not retained when IDR omitted SPS/PPS")
}
recoveryPoint := []byte{
0, 0, 1, 0x67, 1,
0, 0, 1, 0x68, 1,
0, 0, 1, 0x65, 1,
}
if !retainKeyframeRequestUntilOutput(kind, true, "h264", recoveryPoint) {
t.Fatal("complete recovery access unit was not classified as a keyframe")
}
if webrtcpub.ConsumeKeyframeRequest(kind) {
t.Fatal("completed keyframe request was unexpectedly requeued")
}
}
Overlord-Client/cmd/agent/capture/video_keyframe_interval_test.go
+29
View File
@@ -0,0 +1,29 @@
package capture
import (
"testing"
"time"
)
func TestConfiguredVideoKeyframeInterval(t *testing.T) {
tests := []struct {
value string
want time.Duration
}{
{value: "", want: 0},
{value: "off", want: 0},
{value: "0", want: 0},
{value: "30", want: 30 * time.Second},
{value: "2m", want: 2 * time.Minute},
{value: "invalid", want: 0},
}
for _, test := range tests {
t.Run(test.value, func(t *testing.T) {
t.Setenv("OVERLORD_VIDEO_KEYFRAME_INTERVAL", test.value)
if got := configuredVideoKeyframeInterval(); got != test.want {
t.Fatalf("configuredVideoKeyframeInterval() = %s, want %s", got, test.want)
}
})
}
}
Overlord-Client/cmd/agent/capture/virtual_windows.go
+13 -1
View File
@@ -281,19 +281,31 @@ func (s *virtualDuplicationState) captureDirectH264() (wire.Frame, time.Duration
func (s *virtualDuplicationState) encodeCachedDirectH264(capStart time.Time, width, height int) (wire.Frame, time.Duration, time.Duration, bool, error) {
fps := activeH264FPSForStream("backstage")
now := time.Now()
requestedKeyframe := webrtcpub.ConsumeKeyframeRequest(webrtcpub.Kindbackstage)
forceKeyframe := directVideoKeyframeDue(requestedKeyframe, now.UnixNano(), backstageLastKeyframe.Load())
encStart := time.Now()
out, _, err := encodeH264D3D11Texture("backstage", h264D3D11TextureRequest{
Device: unsafe.Pointer(s.device), Texture: unsafe.Pointer(s.h264LastTex),
InputWidth: width, InputHeight: height, EncodeWidth: width, EncodeHeight: height,
FPS: fps, DXGIFormat: s.h264LastDesc.Format, ForceIDR: webrtcpub.ConsumeKeyframeRequest(webrtcpub.Kindbackstage),
FPS: fps, DXGIFormat: s.h264LastDesc.Format, ForceIDR: forceKeyframe,
})
encodeDur := time.Since(encStart)
if err != nil {
if requestedKeyframe {
webrtcpub.RequestKeyframe(webrtcpub.Kindbackstage)
}
return wire.Frame{}, time.Since(capStart), encodeDur, true, err
}
if len(out) == 0 {
if requestedKeyframe {
webrtcpub.RequestKeyframe(webrtcpub.Kindbackstage)
}
return wire.Frame{}, time.Since(capStart), encodeDur, true, nil
}
if retainKeyframeRequestUntilOutput(webrtcpub.Kindbackstage, requestedKeyframe, "h264", out) {
backstageLastKeyframe.Store(now.UnixNano())
}
return wire.Frame{Type: "frame", Header: wire.FrameHeader{Monitor: 0, FPS: 0, Format: "h264", Backstage: true}, Data: out}, time.Since(capStart), encodeDur, true, nil
}
Overlord-Client/cmd/agent/capture/win_duplication_test.go
+10 -1
View File
@@ -93,6 +93,9 @@ func TestCaptureDisplayDXGI(t *testing.T) {
func TestDirectVideoKeyframeDue(t *testing.T) {
now := (10 * time.Second).Nanoseconds()
previousInterval := videoKeyframeEvery
videoKeyframeEvery = 5 * time.Second
defer func() { videoKeyframeEvery = previousInterval }()
tests := []struct {
name string
@@ -102,7 +105,8 @@ func TestDirectVideoKeyframeDue(t *testing.T) {
}{
{name: "explicit request", requested: true, last: now, want: true},
{name: "first frame", last: 0, want: true},
{name: "periodic keyframes disabled", last: now - (30 * time.Second).Nanoseconds(), want: false},
{name: "periodic keyframe due", last: now - (30 * time.Second).Nanoseconds(), want: true},
{name: "periodic keyframe not due", last: now - time.Second.Nanoseconds(), want: false},
}
for _, test := range tests {
@@ -112,4 +116,9 @@ func TestDirectVideoKeyframeDue(t *testing.T) {
}
})
}
videoKeyframeEvery = 0
if directVideoKeyframeDue(false, now, now-(30*time.Second).Nanoseconds()) {
t.Fatal("periodic keyframe fired while the interval was disabled")
}
}
Overlord-Client/cmd/agent/h264util/annexb.go
+45
View File
@@ -0,0 +1,45 @@
package h264util
type AccessUnit struct {
HasSPS bool
HasPPS bool
HasIDR bool
}
func InspectAnnexB(data []byte) AccessUnit {
var out AccessUnit
for i := 0; i+3 < len(data); {
startCodeLen := 0
switch {
case i+3 < len(data) && data[i] == 0 && data[i+1] == 0 && data[i+2] == 1:
startCodeLen = 3
case i+4 < len(data) && data[i] == 0 && data[i+1] == 0 && data[i+2] == 0 && data[i+3] == 1:
startCodeLen = 4
}
if startCodeLen == 0 {
i++
continue
}
nal := i + startCodeLen
if nal >= len(data) {
break
}
switch data[nal] & 0x1f {
case 5:
out.HasIDR = true
case 7:
out.HasSPS = true
case 8:
out.HasPPS = true
}
if out.HasSPS && out.HasPPS && out.HasIDR {
return out
}
i = nal + 1
}
return out
}
func IsIDR(data []byte) bool {
return InspectAnnexB(data).HasIDR
}
Overlord-Client/cmd/agent/h264util/annexb_test.go
+22
View File
@@ -0,0 +1,22 @@
package h264util
import "testing"
func TestInspectAnnexB(t *testing.T) {
data := []byte{
0, 0, 0, 1, 0x67, 1, 2, 3,
0, 0, 1, 0x68, 4, 5,
0, 0, 0, 1, 0x65, 6, 7,
}
got := InspectAnnexB(data)
if !got.HasSPS || !got.HasPPS || !got.HasIDR {
t.Fatalf("InspectAnnexB() = %+v", got)
}
}
func TestInspectAnnexBRejectsDeltaFrame(t *testing.T) {
got := InspectAnnexB([]byte{0, 0, 1, 0x41, 1, 2, 3})
if got.HasSPS || got.HasPPS || got.HasIDR {
t.Fatalf("delta frame was misclassified: %+v", got)
}
}
Overlord-Client/cmd/agent/webrtcpub/state.go
+66 -17
View File
@@ -4,6 +4,8 @@ import (
"sync"
"sync/atomic"
"time"
"overlord-client/cmd/agent/h264util"
)
type Kind string
@@ -28,30 +30,64 @@ type writerEntry struct {
audio AudioWriter
}
type latestVideoWriter struct {
writer VideoWriter
mu sync.Mutex
pending []byte
dur time.Duration
closed bool
wake chan struct{}
const maxPendingVideoFrames = 6
type queuedVideoFrame struct {
data []byte
dur time.Duration
key bool
}
func newLatestVideoWriter(writer VideoWriter) *latestVideoWriter {
queued := &latestVideoWriter{writer: writer, wake: make(chan struct{}, 1)}
type latestVideoWriter struct {
writer VideoWriter
kind Kind
mu sync.Mutex
pending []queuedVideoFrame
needsKeyframe bool
closed bool
wake chan struct{}
}
func newLatestVideoWriter(kind Kind, writer VideoWriter) *latestVideoWriter {
queued := &latestVideoWriter{writer: writer, kind: kind, wake: make(chan struct{}, 1)}
go queued.run()
return queued
}
func (w *latestVideoWriter) enqueue(frame []byte, dur time.Duration) {
isKey := h264util.IsIDR(frame)
w.mu.Lock()
if w.closed {
w.mu.Unlock()
return
}
w.pending = frame
w.dur = dur
if w.needsKeyframe && !isKey {
w.mu.Unlock()
RequestKeyframe(w.kind)
return
}
requestKeyframe := false
if len(w.pending) >= maxPendingVideoFrames {
w.pending = nil
w.needsKeyframe = true
requestKeyframe = true
if !isKey {
w.mu.Unlock()
RequestKeyframe(w.kind)
return
}
}
if isKey && w.needsKeyframe {
w.pending = nil
w.needsKeyframe = false
requestKeyframe = false
}
w.pending = append(w.pending, queuedVideoFrame{data: frame, dur: dur, key: isKey})
w.mu.Unlock()
if requestKeyframe {
RequestKeyframe(w.kind)
}
select {
case w.wake <- struct{}{}:
default:
@@ -77,13 +113,26 @@ func (w *latestVideoWriter) run() {
w.mu.Unlock()
return
}
frame, dur := w.pending, w.dur
w.pending = nil
w.mu.Unlock()
if len(frame) == 0 {
if len(w.pending) == 0 {
w.mu.Unlock()
break
}
_ = w.writer.WriteH264(frame, dur)
frame := w.pending[0]
w.pending[0] = queuedVideoFrame{}
w.pending = w.pending[1:]
w.mu.Unlock()
err := w.writer.WriteH264(frame.data, frame.dur)
requestKeyframe := false
w.mu.Lock()
if err != nil {
w.pending = nil
w.needsKeyframe = true
requestKeyframe = true
}
w.mu.Unlock()
if requestKeyframe {
RequestKeyframe(w.kind)
}
}
}
}
@@ -107,7 +156,7 @@ func registerVideoWriter(kind Kind, id string, w VideoWriter) {
if entry.video != nil {
entry.video.close()
}
entry.video = newLatestVideoWriter(w)
entry.video = newLatestVideoWriter(kind, w)
bucket[id] = entry
writersMu.Unlock()
RequestKeyframe(kind)
Overlord-Client/cmd/agent/webrtcpub/state_test.go
+23 -7
View File
@@ -4,6 +4,8 @@ import (
"sync"
"testing"
"time"
"overlord-client/cmd/agent/h264util"
)
type blockingVideoWriter struct {
@@ -25,7 +27,15 @@ func (w *blockingVideoWriter) WriteH264(frame []byte, _ time.Duration) error {
return nil
}
func TestWriteH264DoesNotBlockAndKeepsLatestPendingFrame(t *testing.T) {
func annexBDelta(value byte) []byte {
return []byte{0, 0, 0, 1, 0x41, value}
}
func annexBKey(value byte) []byte {
return []byte{0, 0, 0, 1, 0x67, 1, 0, 0, 1, 0x68, 1, 0, 0, 1, 0x65, value}
}
func TestWriteH264DoesNotBlockAndRequestsRecoveryAfterDroppingDelta(t *testing.T) {
kind := Kind("queue-test")
id := "slow-writer"
writer := &blockingVideoWriter{
@@ -34,8 +44,9 @@ func TestWriteH264DoesNotBlockAndKeepsLatestPendingFrame(t *testing.T) {
}
registerVideoWriter(kind, id, writer)
defer unregisterWriter(kind, id)
_ = ConsumeKeyframeRequest(kind)
if err := WriteH264(kind, []byte{1}, time.Millisecond); err != nil {
if err := WriteH264(kind, annexBDelta(1), time.Millisecond); err != nil {
t.Fatal(err)
}
select {
@@ -45,11 +56,16 @@ func TestWriteH264DoesNotBlockAndKeepsLatestPendingFrame(t *testing.T) {
}
started := time.Now()
_ = WriteH264(kind, []byte{2}, time.Millisecond)
_ = WriteH264(kind, []byte{3}, time.Millisecond)
for frame := 2; frame <= maxPendingVideoFrames+2; frame++ {
_ = WriteH264(kind, annexBDelta(byte(frame)), time.Millisecond)
}
if elapsed := time.Since(started); elapsed > 100*time.Millisecond {
t.Fatalf("capture path blocked for %s", elapsed)
}
if !ConsumeKeyframeRequest(kind) {
t.Fatal("dropping a predictive frame did not request a keyframe")
}
_ = WriteH264(kind, annexBKey(9), time.Millisecond)
close(writer.release)
deadline := time.Now().Add(time.Second)
@@ -58,14 +74,14 @@ func TestWriteH264DoesNotBlockAndKeepsLatestPendingFrame(t *testing.T) {
frames := append([][]byte(nil), writer.frames...)
writer.mu.Unlock()
if len(frames) >= 2 {
if frames[0][0] != 1 || frames[1][0] != 3 {
t.Fatalf("expected first and latest frames, got %v", frames)
if frames[0][len(frames[0])-1] != 1 || !h264util.IsIDR(frames[1]) {
t.Fatalf("expected first frame followed by recovery IDR, got %v", frames)
}
return
}
time.Sleep(5 * time.Millisecond)
}
t.Fatal("latest pending frame was not delivered")
t.Fatal("recovery keyframe was not delivered")
}
func TestKeyframeRequestsAreIsolatedByStreamKind(t *testing.T) {
Overlord-Client/cmd/agent/webrtcpub/whip_pion.go
+1 -1
View File
@@ -47,7 +47,7 @@ func drainRTCP(sender *webrtc.RTPSender, kind Kind) {
}
}
const h264SDPFmtpLine = "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=640034"
const h264SDPFmtpLine = "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=4d0034"
var rtcpKeyframeLogOnce sync.Once
var (