test3

Overlord-Client/cmd/agent/capture/h264_encoder_windows.go

@@ -37,7 +37,8 @@ const (
 	mftMessageNotifyEndOfStream    = 0x10000002
 	mftMessageNotifyEndStreaming   = 0x10000001
 
-	mftOutputStreamProvidesSamples = 0x00000100
+	mftOutputStreamProvidesSamples   = 0x00000100
+	mftOutputStreamCanProvideSamples = 0x00000200
 
 	mfETransformNeedMoreInput = 0xC00D6D72
 	mfETransformStreamChange  = 0xC00D6D61
@@ -1067,7 +1068,7 @@ func (e *mfH264Encoder) processOutput() ([]byte, error) {
 	}
 
 	var outSample *mfSample
-	if info.flags&mftOutputStreamProvidesSamples == 0 {
+	if !mftCanAllocateOutputSample(info.flags) {
 		size := int(info.cbSize)
 		minSize := e.width * e.height * 3 / 2
 		if size < minSize {
@@ -1098,7 +1099,11 @@ func (e *mfH264Encoder) processOutput() ([]byte, error) {
 		return nil, nil
 	default:
 		if failedHR(hr) {
-			return nil, fmt.Errorf("mf h264: ProcessOutput failed 0x%x", hr)
+			allocation := "transform"
+			if outSample != nil {
+				allocation = "caller"
+			}
+			return nil, fmt.Errorf("mf h264: ProcessOutput failed 0x%x stream_flags=0x%x output_sample=%s", hr, info.flags, allocation)
 		}
 	}
 	if output.sample == nil {
@@ -1110,6 +1115,10 @@ func (e *mfH264Encoder) processOutput() ([]byte, error) {
 	return sampleBytes(output.sample)
 }
 
+func mftCanAllocateOutputSample(flags uint32) bool {
+	return flags&(mftOutputStreamProvidesSamples|mftOutputStreamCanProvideSamples) != 0
+}
+
 func createVideoType(subtype windows.GUID, width, height, fps int) (*mfMediaType, error) {
 	var mt *mfMediaType
 	hr, _, _ := procMFCreateMediaType.Call(uintptr(unsafe.Pointer(&mt)))

Overlord-Client/cmd/agent/capture/h264_encoder_windows_test.go

@@ -0,0 +1,25 @@
+//go:build windows
+
+package capture
+
+import "testing"
+
+func TestMFTCanAllocateOutputSample(t *testing.T) {
+	tests := []struct {
+		name  string
+		flags uint32
+		want  bool
+	}{
+		{name: "caller must allocate", flags: 0, want: false},
+		{name: "transform provides samples", flags: mftOutputStreamProvidesSamples, want: true},
+		{name: "transform can provide samples", flags: mftOutputStreamCanProvideSamples, want: true},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			if got := mftCanAllocateOutputSample(test.flags); got != test.want {
+				t.Fatalf("mftCanAllocateOutputSample(0x%x) = %t, want %t", test.flags, got, test.want)
+			}
+		})
+	}
+}

Overlord-Client/cmd/agent/capture/win_duplication.go

@@ -1365,31 +1365,47 @@ func findOutput(factory *idxgiFactory1, desiredName string, desiredBounds image.
 }
 
 func createD3DDevice(adapter *idxgiAdapter1) (*d3d11Device, *d3d11DeviceContext, error) {
-	var device *d3d11Device
-	var context *d3d11DeviceContext
-	var featureLevel uint32
-	hr, _, _ := procD3D11CreateDevice.Call(
-		uintptr(unsafe.Pointer(adapter)),
-		uintptr(d3dDriverTypeUnknown),
-		0,
-		uintptr(d3d11CreateDeviceBgraSupport|d3d11CreateDeviceVideoSupport),
-		0,
-		0,
-		uintptr(d3d11SdkVersion),
-		uintptr(unsafe.Pointer(&device)),
-		uintptr(unsafe.Pointer(&featureLevel)),
-		uintptr(unsafe.Pointer(&context)),
-	)
-	if hr != S_OK || device == nil || context == nil {
+	flags := d3d11DeviceCreationFlagCandidates()
+	var firstHR uintptr
+	for attempt, creationFlags := range flags {
+		var device *d3d11Device
+		var context *d3d11DeviceContext
+		var featureLevel uint32
+		hr, _, _ := procD3D11CreateDevice.Call(
+			uintptr(unsafe.Pointer(adapter)),
+			uintptr(d3dDriverTypeUnknown),
+			0,
+			uintptr(creationFlags),
+			0,
+			0,
+			uintptr(d3d11SdkVersion),
+			uintptr(unsafe.Pointer(&device)),
+			uintptr(unsafe.Pointer(&featureLevel)),
+			uintptr(unsafe.Pointer(&context)),
+		)
+		if hr == S_OK && device != nil && context != nil {
+			return device, context, nil
+		}
 		if context != nil {
 			context.Release()
 		}
 		if device != nil {
 			device.Release()
 		}
-		return nil, nil, fmt.Errorf("dxgi: D3D11CreateDevice failed 0x%x", hr)
+		if attempt == 0 {
+			firstHR = hr
+			continue
+		}
+		return nil, nil, fmt.Errorf("dxgi: D3D11CreateDevice failed video=0x%x fallback=0x%x", firstHR, hr)
+	}
+	return nil, nil, errors.New("dxgi: D3D11CreateDevice had no creation modes")
+}
+
+func d3d11DeviceCreationFlagCandidates() [2]uint32 {
+	return [2]uint32{
+		d3d11CreateDeviceBgraSupport | d3d11CreateDeviceVideoSupport,
+		d3d11CreateDeviceBgraSupport,
 	}
-	return device, context, nil
 }
 
 func convertBGRA(src []byte, pitch, width, height int, rotation uint32) *image.RGBA {

Overlord-Client/cmd/agent/capture/win_duplication_test.go

@@ -49,6 +49,16 @@ func TestDesktopDuplicationToggleResetsState(t *testing.T) {
 	}
 }
 
+func TestD3D11DeviceCreationFlagsFallBackWithoutVideoSupport(t *testing.T) {
+	flags := d3d11DeviceCreationFlagCandidates()
+	if flags[0] != d3d11CreateDeviceBgraSupport|d3d11CreateDeviceVideoSupport {
+		t.Fatalf("first D3D11 creation flags = 0x%x, want BGRA and video support", flags[0])
+	}
+	if flags[1] != d3d11CreateDeviceBgraSupport {
+		t.Fatalf("fallback D3D11 creation flags = 0x%x, want BGRA only", flags[1])
+	}
+}
+
 func TestCaptureDisplayDXGI(t *testing.T) {
 	if displayCount() == 0 {
 		t.Skip("no displays detected")