force users to pick capable display range.

This commit is contained in:
KingKDot
2026-07-13 09:34:16 -05:00
parent 1bc6364055
commit 447efa5192
16 changed files with 651 additions and 66 deletions
@@ -0,0 +1,17 @@
package capture
type DesktopEncoderProfile struct {
MaxHeight int `msgpack:"maxHeight" json:"maxHeight"`
Width int `msgpack:"width" json:"width"`
Height int `msgpack:"height" json:"height"`
FPS int `msgpack:"fps" json:"fps"`
Label string `msgpack:"label" json:"label"`
Providers []string `msgpack:"providers" json:"providers"`
}
type DesktopEncoderCapabilities struct {
Probed bool `msgpack:"probed" json:"probed"`
Display int `msgpack:"display" json:"display"`
Profiles []DesktopEncoderProfile `msgpack:"profiles" json:"profiles"`
Detail string `msgpack:"detail,omitempty" json:"detail,omitempty"`
}
@@ -0,0 +1,14 @@
//go:build !windows
package capture
func ProbeDesktopEncoderCapabilities(display int) DesktopEncoderCapabilities {
return DesktopEncoderCapabilities{
Display: display,
Profiles: []DesktopEncoderProfile{
{MaxHeight: 720, Width: 1280, Height: 720, FPS: 60, Label: "60 FPS - 720p", Providers: []string{"Software H.264 / JPEG"}},
{MaxHeight: 1080, Width: 1920, Height: 1080, FPS: 60, Label: "60 FPS - 1080p", Providers: []string{"Software H.264 / JPEG"}},
},
Detail: "Hardware H.264 capability probing is available on Windows; safe software profiles are shown.",
}
}
@@ -0,0 +1,277 @@
//go:build windows
package capture
import (
"fmt"
"image"
"sort"
"strings"
"sync"
"time"
"unsafe"
)
const desktopCapabilityCacheTTL = 10 * time.Minute
var desktopCapabilityCache = struct {
sync.Mutex
entries map[string]desktopCapabilityCacheEntry
}{entries: make(map[string]desktopCapabilityCacheEntry)}
type desktopCapabilityCacheEntry struct {
at time.Time
caps DesktopEncoderCapabilities
}
type desktopProbeSize struct {
maxHeight int
width int
height int
}
type desktopProfileProbe struct {
name string
run func(h264D3D11TextureRequest) (time.Duration, error)
}
func ProbeDesktopEncoderCapabilities(display int) DesktopEncoderCapabilities {
bounds := DisplayBounds(display)
cacheKey := fmt.Sprintf("%d:%d:%d:%d:%d:%.3f", display, bounds.Min.X, bounds.Min.Y, bounds.Dx(), bounds.Dy(), captureScale())
desktopCapabilityCache.Lock()
if cached, ok := desktopCapabilityCache.entries[cacheKey]; ok && time.Since(cached.at) < desktopCapabilityCacheTTL {
desktopCapabilityCache.Unlock()
return cached.caps
}
caps := probeDesktopEncoderCapabilities(display, bounds)
desktopCapabilityCache.entries[cacheKey] = desktopCapabilityCacheEntry{at: time.Now(), caps: caps}
for key, entry := range desktopCapabilityCache.entries {
if time.Since(entry.at) >= desktopCapabilityCacheTTL {
delete(desktopCapabilityCache.entries, key)
}
}
desktopCapabilityCache.Unlock()
return caps
}
func probeDesktopEncoderCapabilities(display int, bounds image.Rectangle) DesktopEncoderCapabilities {
caps := DesktopEncoderCapabilities{Display: display}
if bounds.Dx() <= 0 || bounds.Dy() <= 0 {
caps.Profiles = safeDesktopProfiles(1280, 720)
caps.Detail = "Could not determine the selected display size; safe software profiles are shown."
return caps
}
factory, err := createDXGIFactory1()
if err != nil {
caps.Profiles = safeDesktopProfiles(bounds.Dx(), bounds.Dy())
caps.Detail = err.Error()
return caps
}
defer factory.Release()
output, adapter, _, err := findOutput(factory, "", bounds, display)
if err != nil {
caps.Profiles = safeDesktopProfiles(bounds.Dx(), bounds.Dy())
caps.Detail = err.Error()
return caps
}
defer output.Release()
defer adapter.Release()
device, context, err := createD3DDevice(adapter)
if err != nil {
caps.Profiles = safeDesktopProfiles(bounds.Dx(), bounds.Dy())
caps.Detail = err.Error()
return caps
}
defer context.Release()
defer device.Release()
nativeWidth, nativeHeight := even(bounds.Dx()), even(bounds.Dy())
textureDesc := d3d11Texture2DDesc{
Width: uint32(nativeWidth),
Height: uint32(nativeHeight),
MipLevels: 1,
ArraySize: 1,
Format: dxgiFormatB8G8R8A8UNorm,
}
textureDesc.SampleDesc.Count = 1
var texture *d3d11Texture2D
if hr := device.CreateTexture2D(&textureDesc, nil, &texture); hr != S_OK || texture == nil {
caps.Profiles = safeDesktopProfiles(nativeWidth, nativeHeight)
caps.Detail = fmt.Sprintf("D3D11 capability texture creation failed 0x%x", hr)
return caps
}
defer texture.Release()
probes := []desktopProfileProbe{
{name: "NVIDIA NVENC", run: func(req h264D3D11TextureRequest) (time.Duration, error) {
return probeNativeD3D11TextureProfile(req.Device, req.Texture, req.InputWidth, req.InputHeight, req.EncodeWidth, req.EncodeHeight, req.FPS, req.DXGIFormat)
}},
}
if backend := newQSVD3D11TextureBackend(); backend != nil {
probes = append(probes, backendProfileProbe(backend))
}
if backend := newAMFD3D11TextureBackend(); backend != nil {
probes = append(probes, backendProfileProbe(backend))
}
fpsOptions := []int{240, 165, 144, 120, 90, 60, 30}
sizes := desktopProbeSizes(nativeWidth, nativeHeight)
providersByProfile := make(map[string][]string)
for _, size := range sizes {
for _, probe := range probes {
maxFPS := 0
for _, fps := range fpsOptions {
req := h264D3D11TextureRequest{
Device: unsafe.Pointer(device), Texture: unsafe.Pointer(texture),
InputWidth: nativeWidth, InputHeight: nativeHeight,
EncodeWidth: size.width, EncodeHeight: size.height,
FPS: fps, DXGIFormat: dxgiFormatB8G8R8A8UNorm, ForceIDR: true,
}
averageEncode, err := probe.run(req)
frameBudget := time.Second / time.Duration(fps)
if err == nil && averageEncode <= frameBudget+frameBudget/5 {
maxFPS = fps
break
}
}
if maxFPS == 0 {
continue
}
for _, fps := range fpsOptions {
if fps <= maxFPS {
key := desktopProfileKey(size.width, size.height, fps)
providersByProfile[key] = appendUniqueString(providersByProfile[key], probe.name)
}
}
}
}
hardwareProfilesFound := len(providersByProfile) > 0
for _, safe := range safeDesktopProfiles(nativeWidth, nativeHeight) {
key := desktopProfileKey(safe.Width, safe.Height, safe.FPS)
providersByProfile[key] = appendUniqueString(providersByProfile[key], "Software H.264 / JPEG")
}
for _, size := range sizes {
for _, fps := range fpsOptions {
providers := providersByProfile[desktopProfileKey(size.width, size.height, fps)]
if len(providers) == 0 {
continue
}
caps.Profiles = append(caps.Profiles, DesktopEncoderProfile{
MaxHeight: size.maxHeight, Width: size.width, Height: size.height, FPS: fps,
Label: desktopProfileLabel(size.width, size.height, fps), Providers: providers,
})
}
}
sort.SliceStable(caps.Profiles, func(i, j int) bool {
if caps.Profiles[i].Height != caps.Profiles[j].Height {
return caps.Profiles[i].Height < caps.Profiles[j].Height
}
return caps.Profiles[i].FPS > caps.Profiles[j].FPS
})
caps.Probed = true
if len(caps.Profiles) == 0 {
caps.Profiles = safeDesktopProfiles(nativeWidth, nativeHeight)
}
if !hardwareProfilesFound {
caps.Detail = "No hardware H.264 profile initialized successfully; safe software profiles are shown."
} else {
caps.Detail = "Profiles were tested on the selected display adapter."
}
return caps
}
func backendProfileProbe(backend h264D3D11TextureBackend) desktopProfileProbe {
return desktopProfileProbe{name: backend.Name(), run: func(req h264D3D11TextureRequest) (time.Duration, error) {
defer backend.Reset()
producedOutput := false
for frame := 0; frame < 2; frame++ {
req.ForceIDR = frame == 0
out, err := backend.Encode(req)
if err != nil {
return 0, err
}
producedOutput = producedOutput || len(out) > 0
}
started := time.Now()
for frame := 0; frame < 6; frame++ {
req.ForceIDR = false
out, err := backend.Encode(req)
if err != nil {
return 0, err
}
producedOutput = producedOutput || len(out) > 0
}
average := time.Since(started) / 6
if !producedOutput {
return average, fmt.Errorf("encoder produced no output during capability probe")
}
return average, nil
}}
}
func desktopProbeSizes(nativeWidth, nativeHeight int) []desktopProbeSize {
maxHeights := []int{720, 1080, 1440, -1}
seen := make(map[string]bool)
out := make([]desktopProbeSize, 0, len(maxHeights))
for _, maxHeight := range maxHeights {
scale := captureScale()
if maxHeight > 0 && nativeHeight > maxHeight {
if capScale := float64(maxHeight) / float64(nativeHeight); capScale < scale {
scale = capScale
}
}
width, height := even(int(float64(nativeWidth)*scale)), even(int(float64(nativeHeight)*scale))
if width <= 0 || height <= 0 {
continue
}
key := fmt.Sprintf("%dx%d", width, height)
if seen[key] {
continue
}
seen[key] = true
out = append(out, desktopProbeSize{maxHeight: maxHeight, width: width, height: height})
}
return out
}
func safeDesktopProfiles(nativeWidth, nativeHeight int) []DesktopEncoderProfile {
profiles := make([]DesktopEncoderProfile, 0, 4)
for _, size := range desktopProbeSizes(even(nativeWidth), even(nativeHeight)) {
if size.height > 1080 {
continue
}
for _, fps := range []int{60, 30} {
profiles = append(profiles, DesktopEncoderProfile{
MaxHeight: size.maxHeight, Width: size.width, Height: size.height, FPS: fps,
Label: desktopProfileLabel(size.width, size.height, fps), Providers: []string{"Software H.264 / JPEG"},
})
}
}
return profiles
}
func desktopProfileKey(width, height, fps int) string {
return fmt.Sprintf("%dx%d@%d", width, height, fps)
}
func desktopProfileLabel(width, height, fps int) string {
resolution := fmt.Sprintf("%dx%d", width, height)
if height == 720 || height == 1080 || height == 1440 || height == 2160 {
resolution = fmt.Sprintf("%dp", height)
}
return fmt.Sprintf("%d FPS - %s", fps, resolution)
}
func appendUniqueString(values []string, value string) []string {
for _, existing := range values {
if strings.EqualFold(existing, value) {
return values
}
}
return append(values, value)
}
func even(value int) int { return value - value%2 }
@@ -6,9 +6,24 @@ import (
"fmt"
"strings"
"sync"
"time"
"unsafe"
)
const d3d11H264FailureCooldown = 30 * time.Second
type d3d11H264FailureKey struct {
backend string
device uintptr
inputWidth, inputHeight, encodeWidth, encodeHeight, fps int
dxgiFormat uint32
}
type d3d11H264Failure struct {
until time.Time
err string
}
type h264D3D11TextureRequest struct {
Device unsafe.Pointer
Texture unsafe.Pointer
@@ -49,8 +64,10 @@ var d3d11H264TextureRegistry = struct {
sync.Mutex
backends []h264D3D11TextureBackend
active h264D3D11TextureBackend
failures map[d3d11H264FailureKey]d3d11H264Failure
}{
backends: []h264D3D11TextureBackend{nvencD3D11TextureBackend{}},
failures: make(map[d3d11H264FailureKey]d3d11H264Failure),
}
func init() {
@@ -72,26 +89,44 @@ func encodeH264D3D11Texture(req h264D3D11TextureRequest) ([]byte, string, error)
failedActiveName := ""
errorsByBackend := make([]string, 0, len(d3d11H264TextureRegistry.backends))
if active := d3d11H264TextureRegistry.active; active != nil {
out, err := active.Encode(req)
if err == nil {
return out, active.Name(), nil
key := d3d11H264FailureKeyFor(active.Name(), req)
if failure, failed := activeD3D11H264Failure(key, time.Now()); !failed {
out, err := active.Encode(req)
if err == nil {
delete(d3d11H264TextureRegistry.failures, key)
return out, active.Name(), nil
}
rememberD3D11H264Failure(key, err)
active.Reset()
failedActiveName = active.Name()
errorsByBackend = append(errorsByBackend, active.Name()+": "+err.Error())
d3d11H264TextureRegistry.active = nil
} else {
errorsByBackend = append(errorsByBackend, active.Name()+": "+failure.err+" (cooldown)")
active.Reset()
failedActiveName = active.Name()
d3d11H264TextureRegistry.active = nil
}
active.Reset()
failedActiveName = active.Name()
errorsByBackend = append(errorsByBackend, active.Name()+": "+err.Error())
d3d11H264TextureRegistry.active = nil
}
now := time.Now()
for _, backend := range d3d11H264TextureRegistry.backends {
if backend == nil || (failedActiveName != "" && backend.Name() == failedActiveName) {
continue
}
key := d3d11H264FailureKeyFor(backend.Name(), req)
if failure, failed := activeD3D11H264Failure(key, now); failed {
errorsByBackend = append(errorsByBackend, backend.Name()+": "+failure.err+" (cooldown)")
continue
}
out, err := backend.Encode(req)
if err == nil {
delete(d3d11H264TextureRegistry.failures, key)
d3d11H264TextureRegistry.active = backend
return out, backend.Name(), nil
}
backend.Reset()
rememberD3D11H264Failure(key, err)
errorsByBackend = append(errorsByBackend, backend.Name()+": "+err.Error())
}
if len(errorsByBackend) == 0 {
@@ -100,6 +135,37 @@ func encodeH264D3D11Texture(req h264D3D11TextureRequest) ([]byte, string, error)
return nil, "", fmt.Errorf("D3D11 H.264 texture encoders unavailable: %s", strings.Join(errorsByBackend, "; "))
}
func d3d11H264FailureKeyFor(backend string, req h264D3D11TextureRequest) d3d11H264FailureKey {
return d3d11H264FailureKey{
backend: backend, device: uintptr(req.Device), inputWidth: req.InputWidth, inputHeight: req.InputHeight,
encodeWidth: req.EncodeWidth, encodeHeight: req.EncodeHeight, fps: req.FPS, dxgiFormat: req.DXGIFormat,
}
}
func activeD3D11H264Failure(key d3d11H264FailureKey, now time.Time) (d3d11H264Failure, bool) {
failure, ok := d3d11H264TextureRegistry.failures[key]
if !ok {
return d3d11H264Failure{}, false
}
if !now.Before(failure.until) {
delete(d3d11H264TextureRegistry.failures, key)
return d3d11H264Failure{}, false
}
return failure, true
}
func rememberD3D11H264Failure(key d3d11H264FailureKey, err error) {
if len(d3d11H264TextureRegistry.failures) > 128 {
now := time.Now()
for existingKey, failure := range d3d11H264TextureRegistry.failures {
if !now.Before(failure.until) {
delete(d3d11H264TextureRegistry.failures, existingKey)
}
}
}
d3d11H264TextureRegistry.failures[key] = d3d11H264Failure{until: time.Now().Add(d3d11H264FailureCooldown), err: err.Error()}
}
func requestH264D3D11TextureKeyframe() {
d3d11H264TextureRegistry.Lock()
defer d3d11H264TextureRegistry.Unlock()
@@ -39,13 +39,16 @@ func TestD3D11TextureBackendSelectionAndFailover(t *testing.T) {
d3d11H264TextureRegistry.Lock()
savedBackends := d3d11H264TextureRegistry.backends
savedActive := d3d11H264TextureRegistry.active
savedFailures := d3d11H264TextureRegistry.failures
d3d11H264TextureRegistry.backends = []h264D3D11TextureBackend{first, second}
d3d11H264TextureRegistry.active = nil
d3d11H264TextureRegistry.failures = make(map[d3d11H264FailureKey]d3d11H264Failure)
d3d11H264TextureRegistry.Unlock()
t.Cleanup(func() {
d3d11H264TextureRegistry.Lock()
d3d11H264TextureRegistry.backends = savedBackends
d3d11H264TextureRegistry.active = savedActive
d3d11H264TextureRegistry.failures = savedFailures
d3d11H264TextureRegistry.Unlock()
})
@@ -79,3 +82,36 @@ func TestD3D11TextureBackendSelectionAndFailover(t *testing.T) {
t.Fatalf("reset did not reach every backend: first=%d second=%d", first.resetCalls, second.resetCalls)
}
}
func TestD3D11TextureBackendFailuresUseCooldown(t *testing.T) {
first := &fakeD3D11TextureBackend{name: "first", errors: []error{errors.New("unsupported")}}
second := &fakeD3D11TextureBackend{name: "second", errors: []error{errors.New("also unsupported")}}
d3d11H264TextureRegistry.Lock()
savedBackends := d3d11H264TextureRegistry.backends
savedActive := d3d11H264TextureRegistry.active
savedFailures := d3d11H264TextureRegistry.failures
d3d11H264TextureRegistry.backends = []h264D3D11TextureBackend{first, second}
d3d11H264TextureRegistry.active = nil
d3d11H264TextureRegistry.failures = make(map[d3d11H264FailureKey]d3d11H264Failure)
d3d11H264TextureRegistry.Unlock()
t.Cleanup(func() {
d3d11H264TextureRegistry.Lock()
d3d11H264TextureRegistry.backends = savedBackends
d3d11H264TextureRegistry.active = savedActive
d3d11H264TextureRegistry.failures = savedFailures
d3d11H264TextureRegistry.Unlock()
})
device, texture := 1, 2
req := h264D3D11TextureRequest{Device: unsafe.Pointer(&device), Texture: unsafe.Pointer(&texture), EncodeWidth: 2560, EncodeHeight: 1440, FPS: 240}
if _, _, err := encodeH264D3D11Texture(req); err == nil {
t.Fatal("expected initial profile failure")
}
if _, _, err := encodeH264D3D11Texture(req); err == nil {
t.Fatal("expected cached profile failure")
}
if first.encodeCalls != 1 || second.encodeCalls != 1 {
t.Fatalf("failed profile was retried during cooldown: first=%d second=%d", first.encodeCalls, second.encodeCalls)
}
}
@@ -2,6 +2,12 @@
package capture
import (
"fmt"
"time"
"unsafe"
)
type NVENCD3D11SmokeOptions struct {
Width int `json:"width"`
Height int `json:"height"`
@@ -36,3 +42,7 @@ func RunNVENCD3D11Smoke(opts NVENCD3D11SmokeOptions) NVENCD3D11SmokeResult {
Message: "NVENC D3D11 smoke requires Windows with cgo enabled",
}
}
func probeNativeD3D11TextureProfile(_, _ unsafe.Pointer, _, _, _, _, _ int, _ uint32) (time.Duration, error) {
return 0, fmt.Errorf("NVENC D3D11 support is not compiled in")
}
@@ -1462,6 +1462,7 @@ import (
"strings"
"sync"
"sync/atomic"
"time"
"unsafe"
)
@@ -1747,6 +1748,35 @@ func (e *nativeD3D11TextureH264Encoder) EncodeTexture(texture unsafe.Pointer, fo
return C.GoBytes(unsafe.Pointer(raw.data), C.int(raw.size)), nil
}
func probeNativeD3D11TextureProfile(device, texture unsafe.Pointer, inputWidth, inputHeight, encodeWidth, encodeHeight, fps int, dxgiFormat uint32) (time.Duration, error) {
enc, err := newNativeD3D11TextureH264Encoder(device, inputWidth, inputHeight, encodeWidth, encodeHeight, fps, dxgiFormat)
if err != nil {
return 0, err
}
defer enc.Close()
producedOutput := false
for frame := 0; frame < 2; frame++ {
out, err := enc.EncodeTexture(texture, frame == 0)
if err != nil {
return 0, err
}
producedOutput = producedOutput || len(out) > 0
}
started := time.Now()
for frame := 0; frame < 6; frame++ {
out, err := enc.EncodeTexture(texture, false)
if err != nil {
return 0, err
}
producedOutput = producedOutput || len(out) > 0
}
average := time.Since(started) / 6
if !producedOutput {
return average, fmt.Errorf("NVENC produced no output during capability probe")
}
return average, nil
}
func (e *nativeD3D11TextureH264Encoder) maybeLogEncodeDetail(raw C.nvenc_d3d11_encode_result, frame uint64, forceIDR bool) {
total := float64(raw.copy_ms + raw.blt_ms + raw.map_ms + raw.submit_ms + raw.lock_ms)
traceMode := strings.ToLower(strings.TrimSpace(os.Getenv("OVERLORD_NVENC_D3D11_TRACE")))
@@ -846,6 +846,51 @@ func HandleCommand(ctx context.Context, env *runtime.Env, envelope map[string]in
capture.SetMaxResolution(maxH)
sendCommandResultSafe(env, cmdID, true, "")
return nil
case "desktop_set_profile":
payload, _ := envelope["payload"].(map[string]interface{})
maxH, fps := 1080, 60
if payload != nil {
if v, ok := payloadInt(payload, "maxHeight"); ok {
maxH = v
}
if v, ok := payloadInt(payload, "fps"); ok {
fps = v
}
}
SetDesktopTargetFPS(30)
capture.SetMaxResolution(maxH)
fps = SetDesktopTargetFPS(fps)
log.Printf("desktop: set stream profile max_height=%d fps=%d", maxH, fps)
sendCommandResultSafe(env, cmdID, true, "")
return nil
case "desktop_encoder_capabilities":
payload, _ := envelope["payload"].(map[string]interface{})
display := env.SelectedDisplay
if payload != nil {
if v, ok := payloadInt(payload, "display"); ok {
display = v
}
}
if display < 0 || display >= capture.MonitorCount() {
display = 0
}
goSafe("desktop encoder capability probe", env.Cancel, func() {
caps := capture.ProbeDesktopEncoderCapabilities(display)
profiles := make([]wire.DesktopEncoderProfile, 0, len(caps.Profiles))
for _, profile := range caps.Profiles {
profiles = append(profiles, wire.DesktopEncoderProfile{
MaxHeight: profile.MaxHeight, Width: profile.Width, Height: profile.Height,
FPS: profile.FPS, Label: profile.Label, Providers: profile.Providers,
})
}
if err := wire.WriteMsg(ctx, env.Conn, wire.DesktopEncoderCapabilities{
Type: "desktop_encoder_capabilities", CommandID: cmdID, Probed: caps.Probed,
Display: caps.Display, Profiles: profiles, Detail: caps.Detail,
}); err != nil {
log.Printf("desktop: encoder capability result send failed: %v", err)
}
})
return nil
case "desktop_set_fps":
payload, _ := envelope["payload"].(map[string]interface{})
fps := 120
@@ -69,6 +69,24 @@ type CommandResult struct {
Message string `msgpack:"message,omitempty"`
}
type DesktopEncoderProfile struct {
MaxHeight int `msgpack:"maxHeight"`
Width int `msgpack:"width"`
Height int `msgpack:"height"`
FPS int `msgpack:"fps"`
Label string `msgpack:"label"`
Providers []string `msgpack:"providers"`
}
type DesktopEncoderCapabilities struct {
Type string `msgpack:"type"`
CommandID string `msgpack:"commandId,omitempty"`
Probed bool `msgpack:"probed"`
Display int `msgpack:"display"`
Profiles []DesktopEncoderProfile `msgpack:"profiles"`
Detail string `msgpack:"detail,omitempty"`
}
type ClientLogEntry struct {
Seq uint64 `msgpack:"seq" json:"seq"`
At int64 `msgpack:"at" json:"at"`