slightly better remote desktop

2026-03-17 14:40:41 -05:00
parent 357dc1a69a
commit 9bb5b531b5
8 changed files with 214 additions and 32 deletions
@@ -730,7 +730,7 @@ func hvncCaptureDisplayOnThread(display int) (*image.RGBA, error) {
 		return nil, syscall.EINVAL
 	}

-	userScale := captureScale()
+	userScale := effectiveScale(srcW, srcH)
 	dstW := int(float64(srcW) * userScale)
 	dstH := int(float64(srcH) * userScale)
 	if dstW <= 0 || dstH <= 0 {
@@ -5,6 +5,7 @@ package capture
 import (
 	"image"
 	"log"
+	"math"
 	"os"
 	"strconv"
 	"sync"
@@ -30,6 +31,8 @@ var (
 	procDeleteObject           = gdi32.NewProc("DeleteObject")
 	procSelectObject           = gdi32.NewProc("SelectObject")
 	procBitBlt                 = gdi32.NewProc("BitBlt")
+	procStretchBlt             = gdi32.NewProc("StretchBlt")
+	procSetStretchBltMode      = gdi32.NewProc("SetStretchBltMode")
 	procSetProcessDpiAwareness = shcore.NewProc("SetProcessDpiAwareness")
 )

@@ -45,6 +48,7 @@ const (
 	BI_RGB                        = 0
 	DIB_RGB_COLORS                = 0
 	PROCESS_PER_MONITOR_DPI_AWARE = 2
+	COLORONCOLOR                  = 3
 )

 type bitmapInfoHeader struct {
@@ -122,10 +126,24 @@ func bitBlt(hdcDest uintptr, x, y, cx, cy int32, hdcSrc uintptr, x1, y1 int32, r
 	return r != 0
 }

+func stretchBlt(hdcDest uintptr, xDest, yDest, wDest, hDest int32, hdcSrc uintptr, xSrc, ySrc, wSrc, hSrc int32, rop uint32) bool {
+	r, _, _ := procStretchBlt.Call(hdcDest, uintptr(xDest), uintptr(yDest), uintptr(wDest), uintptr(hDest), hdcSrc, uintptr(xSrc), uintptr(ySrc), uintptr(wSrc), uintptr(hSrc), uintptr(rop))
+	return r != 0
+}
+
+func setStretchBltMode(hdc uintptr, mode int32) {
+	_, _, _ = procSetStretchBltMode.Call(hdc, uintptr(mode))
+}
+
 func bitBltWithFallback(hdcMem, hdcScreen uintptr, bounds image.Rectangle, w, h int) bool {
 	return bitBlt(hdcMem, 0, 0, int32(w), int32(h), hdcScreen, int32(bounds.Min.X), int32(bounds.Min.Y), SRCCOPY)
 }

+func stretchBltCapture(hdcMem, hdcScreen uintptr, bounds image.Rectangle, srcW, srcH, dstW, dstH int) bool {
+	setStretchBltMode(hdcMem, COLORONCOLOR)
+	return stretchBlt(hdcMem, 0, 0, int32(dstW), int32(dstH), hdcScreen, int32(bounds.Min.X), int32(bounds.Min.Y), int32(srcW), int32(srcH), SRCCOPY)
+}
+
 var dpiAwareOnce sync.Once

 func setDPIAware() {
@@ -144,6 +162,8 @@ var (
 	lastCaptureLogNs atomic.Int64
 	scaleOnce        sync.Once
 	cachedScale      float64
+	maxResHeight     atomic.Int64 // 0 = default (1080), -1 = native, >0 = specific max height
+	lastCapScale     atomic.Uint64
 	state            capState
 	captureMu        sync.Mutex
 )
@@ -199,7 +219,7 @@ func captureDisplayBitBlt(display int) (*image.RGBA, error) {
 		}
 	}

-	userScale := captureScale()
+	userScale := effectiveScale(srcW, srcH)
 	dstW := int(float64(srcW) * userScale)
 	dstH := int(float64(srcH) * userScale)
 	if dstW <= 0 || dstH <= 0 {
@@ -207,8 +227,13 @@ func captureDisplayBitBlt(display int) (*image.RGBA, error) {
 		dstH = srcH
 	}

+	useStretch := dstW > 0 && dstH > 0 && (dstW < srcW || dstH < srcH) && !cursorCaptureEnabled.Load()
 	capW := srcW
 	capH := srcH
+	if useStretch {
+		capW = dstW
+		capH = dstH
+	}

 	hdcScreen := createDisplayDC()
 	fromCreateDC := hdcScreen != 0
@@ -242,18 +267,32 @@ func captureDisplayBitBlt(display int) (*image.RGBA, error) {
 		}

 		bitStart := time.Now()
-		if !bitBltWithFallback(hdcMem, hdcScreen, bounds, capW, capH) {
+		var captured bool
+		if useStretch {
+			captured = stretchBltCapture(hdcMem, hdcScreen, bounds, srcW, srcH, capW, capH)
+		} else {
+			captured = bitBltWithFallback(hdcMem, hdcScreen, bounds, capW, capH)
+		}
+		if !captured {
 			state.reset()
 			hdcMem, _, buf, stride, err = state.ensure(hdcScreen, capW, capH)
 			if err != nil {
 				return nil, 0, 0, 0, err
 			}
-			if !bitBltWithFallback(hdcMem, hdcScreen, bounds, capW, capH) {
+			if useStretch {
+				captured = stretchBltCapture(hdcMem, hdcScreen, bounds, srcW, srcH, capW, capH)
+			} else {
+				captured = bitBltWithFallback(hdcMem, hdcScreen, bounds, capW, capH)
+			}
+			if !captured {
 				return nil, 0, 0, 0, syscall.EINVAL
 			}
 		}
 		bitDur := time.Since(bitStart)
-		cursorDrawn := DrawCursorOnDC(hdcMem, bounds)
+		cursorDrawn := false
+		if !useStretch {
+			cursorDrawn = DrawCursorOnDC(hdcMem, bounds)
+		}

 		if len(buf) == 0 {
 			return nil, 0, 0, 0, syscall.EINVAL
@@ -266,7 +305,7 @@ func captureDisplayBitBlt(display int) (*image.RGBA, error) {
 		img := &image.RGBA{Pix: buf, Stride: stride, Rect: image.Rect(0, 0, capW, capH)}
 		convDur := time.Since(convStart)

-		if !cursorDrawn {
+		if !useStretch && !cursorDrawn {
 			DrawCursorOnImage(img, bounds)
 		}
 		if dstW != capW || dstH != capH {
@@ -372,18 +411,22 @@ func resizeNearest(src *image.RGBA, w, h int) *image.RGBA {
 	dst := image.NewRGBA(image.Rect(0, 0, w, h))
 	srcW := src.Bounds().Dx()
 	srcH := src.Bounds().Dy()
-	for y := 0; y < h; y++ {
-		sy := y * srcH / h
-		sp := sy * src.Stride
-		dp := y * dst.Stride
+	if srcW <= 0 || srcH <= 0 || w <= 0 || h <= 0 {
+		return dst
+	}
+	xOff := make([]int, w)
 	for x := 0; x < w; x++ {
-			sx := x * srcW / w
-			s := sp + sx*4
-			d := dp + x*4
-			dst.Pix[d+0] = src.Pix[s+0]
-			dst.Pix[d+1] = src.Pix[s+1]
-			dst.Pix[d+2] = src.Pix[s+2]
-			dst.Pix[d+3] = src.Pix[s+3]
+		xOff[x] = (x * srcW / w) * 4
+	}
+	srcPix := src.Pix
+	dstPix := dst.Pix
+	srcStride := src.Stride
+	dstStride := dst.Stride
+	for y := 0; y < h; y++ {
+		sp := (y * srcH / h) * srcStride
+		dp := y * dstStride
+		for x := 0; x < w; x++ {
+			*(*uint32)(unsafe.Pointer(&dstPix[dp+x*4])) = *(*uint32)(unsafe.Pointer(&srcPix[sp+xOff[x]]))
 		}
 	}
 	return dst
@@ -472,3 +515,39 @@ func captureScale() float64 {
 	})
 	return cachedScale
 }
+
+func effectiveScale(srcW, srcH int) float64 {
+	s := captureScale()
+	maxH := int(maxResHeight.Load())
+	if maxH == 0 {
+		maxH = 1080
+	} else if maxH < 0 {
+		storeLastScale(s)
+		return s
+	}
+	if srcH > maxH {
+		resCap := float64(maxH) / float64(srcH)
+		if resCap < s {
+			s = resCap
+		}
+	}
+	storeLastScale(s)
+	return s
+}
+
+func storeLastScale(s float64) {
+	lastCapScale.Store(math.Float64bits(s))
+}
+
+func SetMaxResolution(maxH int) {
+	maxResHeight.Store(int64(maxH))
+	log.Printf("capture: max resolution set to %d", maxH)
+}
+
+func EffectiveScaleForInput() float64 {
+	bits := lastCapScale.Load()
+	if bits == 0 {
+		return 1.0
+	}
+	return math.Float64frombits(bits)
+}
@@ -696,7 +696,7 @@ func (s *duplicationState) capture(display int) (*image.RGBA, error) {
 			break
 		}
 		if s.lastBase != nil {
-			img := s.composeFrame(s.lastBase)
+			img := s.composeFrame(s.lastBase, int(s.desc.ModeDesc.Width), int(s.desc.ModeDesc.Height))
 			s.lastFrame = img
 			s.lastFrameAt = time.Now()
 			return img, nil
@@ -728,6 +728,12 @@ func (s *duplicationState) capture(display int) (*image.RGBA, error) {
 	}

 	var img *image.RGBA
+	userScale := effectiveScale(width, height)
+	dstW := int(float64(width) * userScale)
+	dstH := int(float64(height) * userScale)
+	withCursor := cursorCaptureEnabled.Load()
+	wantScale := !withCursor && userScale != 1 && dstW > 0 && dstH > 0 && (dstW != width || dstH != height)
+
 	if s.desc.DesktopImageInSystemMemory != 0 {
 		var mapped dxgiMappedRect
 		hr = s.dup.MapDesktopSurface(&mapped)
@@ -743,11 +749,17 @@ func (s *duplicationState) capture(display int) (*image.RGBA, error) {

 		pitch := int(mapped.Pitch)
 		src := unsafe.Slice(mapped.Bits, pitch*height)
+		if wantScale {
+			img = convertBGRAScaled(src, pitch, width, height, dstW, dstH, s.desc.Rotation)
+		}
+		if img == nil {
 			img = convertBGRA(src, pitch, width, height, s.desc.Rotation)
+		}
 		_ = s.dup.UnMapDesktopSurface()
 		_ = s.dup.ReleaseFrame()
 	} else {
-		img, hr = s.readbackFrame(width, height, s.desc.Rotation, resource)
+		img, hr = s.readbackFrame(width, height, s.desc.Rotation, resource,
+			wantScale, dstW, dstH)
 		_ = s.dup.ReleaseFrame()
 		if hr != S_OK || img == nil {
 			return nil, fmt.Errorf("dxgi: staging readback failed 0x%x", hr)
@@ -755,23 +767,26 @@ func (s *duplicationState) capture(display int) (*image.RGBA, error) {
 	}

 	s.lastBase = img
-	img = s.composeFrame(img)
+	img = s.composeFrame(img, width, height)
 	s.lastFrame = img
 	s.lastFrameAt = time.Now()

 	return img, nil
 }

-func (s *duplicationState) composeFrame(base *image.RGBA) *image.RGBA {
+func (s *duplicationState) composeFrame(base *image.RGBA, nativeW, nativeH int) *image.RGBA {
 	if base == nil {
 		return nil
 	}
-	userScale := captureScale()
+	imgW := base.Bounds().Dx()
+	imgH := base.Bounds().Dy()
+	userScale := effectiveScale(nativeW, nativeH)
 	withCursor := cursorCaptureEnabled.Load()

-	// Fast path: no cursor overlay and no scaling means we can reuse the raw
-	// frame without an extra clone/copy.
-	if !withCursor && userScale == 1 {
+	dstW := int(float64(nativeW) * userScale)
+	dstH := int(float64(nativeH) * userScale)
+	alreadyScaled := (imgW == dstW && imgH == dstH) && (imgW != nativeW || imgH != nativeH)
+	if !withCursor && (userScale == 1 || alreadyScaled) {
 		return base
 	}

@@ -788,11 +803,7 @@ func (s *duplicationState) composeFrame(base *image.RGBA) *image.RGBA {
 		drawCursorRotated(img, s.cursorBounds, bounds, s.desc.Rotation)
 	}

-	rotW := img.Bounds().Dx()
-	rotH := img.Bounds().Dy()
-	dstW := int(float64(rotW) * userScale)
-	dstH := int(float64(rotH) * userScale)
-	if dstW > 0 && dstH > 0 && (dstW != rotW || dstH != rotH) {
+	if !alreadyScaled && dstW > 0 && dstH > 0 && (dstW != imgW || dstH != imgH) {
 		img = resizeNearest(img, dstW, dstH)
 	}
 	return img
@@ -896,7 +907,8 @@ func (s *duplicationState) ensure(display int) error {
 	return nil
 }

-func (s *duplicationState) readbackFrame(width, height int, rotation uint32, resource *iunknown) (*image.RGBA, uintptr) {
+func (s *duplicationState) readbackFrame(width, height int, rotation uint32, resource *iunknown,
+	wantScale bool, dstW, dstH int) (*image.RGBA, uintptr) {
 	if resource == nil || s.device == nil || s.context == nil {
 		return nil, uintptr(1)
 	}
@@ -946,7 +958,13 @@ func (s *duplicationState) readbackFrame(width, height int, rotation uint32, res

 	pitch := int(mapped.RowPitch)
 	src := unsafe.Slice((*byte)(mapped.Data), pitch*int(srcDesc.Height))
-	img := convertBGRA(src, pitch, int(srcDesc.Width), int(srcDesc.Height), rotation)
+	var img *image.RGBA
+	if wantScale {
+		img = convertBGRAScaled(src, pitch, int(srcDesc.Width), int(srcDesc.Height), dstW, dstH, rotation)
+	}
+	if img == nil {
+		img = convertBGRA(src, pitch, int(srcDesc.Width), int(srcDesc.Height), rotation)
+	}

 	return img, S_OK
 }
@@ -1125,6 +1143,33 @@ func convertBGRA(src []byte, pitch, width, height int, rotation uint32) *image.R
 	}
 }

+func convertBGRAScaled(src []byte, pitch, srcW, srcH, dstW, dstH int, rotation uint32) *image.RGBA {
+	if rotation != dxgiModeRotationIdentity && rotation != 0 {
+		return nil
+	}
+	dst := image.NewRGBA(image.Rect(0, 0, dstW, dstH))
+	xOff := make([]int, dstW)
+	for x := 0; x < dstW; x++ {
+		xOff[x] = (x * srcW / dstW) * 4
+	}
+	dstPix := dst.Pix
+	dstStride := dst.Stride
+	for y := 0; y < dstH; y++ {
+		sy := y * srcH / dstH
+		sRow := src[sy*pitch:]
+		dp := y * dstStride
+		for x := 0; x < dstW; x++ {
+			si := xOff[x]
+			di := dp + x*4
+			dstPix[di+0] = sRow[si+2]
+			dstPix[di+1] = sRow[si+1]
+			dstPix[di+2] = sRow[si+0]
+			dstPix[di+3] = 255
+		}
+	}
+	return dst
+}
+
 func drawCursorRotated(img *image.RGBA, cursorBounds, captureBounds image.Rectangle, rotation uint32) {
 	if !cursorCaptureEnabled.Load() {
 		return
@@ -684,6 +684,18 @@ func HandleCommand(ctx context.Context, env *runtime.Env, envelope map[string]in
 		capture.SetQualityAndCodec(quality, codec)
 		sendCommandResultSafe(env, cmdID, true, "")
 		return nil
+	case "desktop_set_resolution":
+		payload, _ := envelope["payload"].(map[string]interface{})
+		maxH := 0 // default = 1080p cap
+		if payload != nil {
+			if v, ok := payloadInt(payload, "maxHeight"); ok {
+				maxH = v
+			}
+		}
+		log.Printf("desktop: set max resolution height=%d", maxH)
+		capture.SetMaxResolution(maxH)
+		sendCommandResultSafe(env, cmdID, true, "")
+		return nil
 	case "desktop_mouse_move":
 		if !env.MouseControl {
 			sendCommandResultSafe(env, cmdID, true, "")
@@ -8,6 +8,11 @@ func resolveDesktopPoint(display int, x, y int32) (int32, int32) {
 		return x, y
 	}

+	if scale := capture.EffectiveScaleForInput(); scale > 0 && scale < 1 {
+		x = int32(float64(x) / scale)
+		y = int32(float64(y) / scale)
+	}
+
 	absX := int32(bounds.Min.X) + x
 	absY := int32(bounds.Min.Y) + y

@@ -25,6 +25,7 @@ import { encodeMsgpack, decodeMsgpack } from "./msgpack-helpers.js";
  const kbdCtrl = document.getElementById("kbdCtrl");
  const cursorCtrl = document.getElementById("cursorCtrl");
  const duplicationCtrl = document.getElementById("duplicationCtrl");
+  const resolutionSelect = document.getElementById("resolutionSelect");
  const smoothingSlider = document.getElementById("smoothingSlider");
  const smoothingValue = document.getElementById("smoothingValue");
  const qualitySlider = document.getElementById("qualitySlider");
@@ -348,6 +349,20 @@ import { encodeMsgpack, decodeMsgpack } from "./msgpack-helpers.js";
    });
  }

+  function pushResolution() {
+    if (resolutionSelect) {
+      const maxHeight = parseInt(resolutionSelect.value, 10);
+      console.debug("rd: pushResolution maxHeight=", maxHeight);
+      sendCmd("desktop_set_resolution", { maxHeight: maxHeight });
+    }
+  }
+
+  if (resolutionSelect) {
+    resolutionSelect.addEventListener("change", function () {
+      pushResolution();
+    });
+  }
+
  displaySelect.addEventListener("change", function () {
    console.debug("rd: select display", displaySelect.value);
    sendCmd("desktop_select_display", {
@@ -364,6 +379,7 @@ import { encodeMsgpack, decodeMsgpack } from "./msgpack-helpers.js";
    if (qualitySlider) {
      pushQuality(qualitySlider.value);
    }
+    pushResolution();
    desiredStreaming = true;
    lastFrameAt = 0;
    resetH264SessionState();
@@ -59,6 +59,20 @@
              >95%</span
            >
          </div>
+          <div
+            class="bg-slate-900/70 border border-slate-800 rounded-lg px-3 py-2 flex items-center gap-3 text-sm text-slate-300"
+          >
+            <label for="resolutionSelect" class="text-slate-200">Resolution</label>
+            <select
+              id="resolutionSelect"
+              class="bg-slate-800 border border-slate-700 rounded px-2 py-1 text-slate-100"
+            >
+              <option value="720">720p</option>
+              <option value="1080" selected>1080p</option>
+              <option value="1440">1440p</option>
+              <option value="-1">Native</option>
+            </select>
+          </div>
          <div
            class="bg-slate-900/70 border border-slate-800 rounded-lg px-3 py-2 flex items-center gap-3 text-sm text-slate-300"
          >
@@ -154,6 +154,7 @@ export const rdStreamingState = new Map<string, {
  quality: number;
  codec: string;
  duplication: boolean;
+  maxHeight: number;
 }>();
 const rdInputPending = new Map<string, { clientId: string; sentAt: number; kind: string }>();
 const RD_INPUT_TTL_MS = 10_000;
@@ -370,6 +371,16 @@ export function handleRemoteDesktopViewerMessage(ws: ServerWebSocket<SocketData>
    case "desktop_enable_cursor":
      sendDesktopCommand(target, "desktop_enable_cursor", { enabled: !!payload.enabled });
      break;
+    case "desktop_set_resolution": {
+      const newMaxHeight = Number(payload.maxHeight) || 0;
+      if (state.maxHeight !== newMaxHeight) {
+        sendDesktopCommand(target, "desktop_set_resolution", { maxHeight: newMaxHeight });
+        state.maxHeight = newMaxHeight;
+        rdStreamingState.set(clientId, state);
+        logger.debug(`[rd] set max resolution height=${newMaxHeight}`);
+      }
+      break;
+    }
    case "desktop_set_duplication": {
      const enabled = !!payload.enabled;
      if (state.duplication !== enabled) {