fix jpeg

Overlord-Client/cmd/agent/capture/capture.go

@@ -1648,9 +1648,9 @@ func encodeBlocksbackstage(img *image.RGBA, prev *prevImage, quality int, codec
 	}
 
 	prevCopyStart := time.Now()
-	prevMu.Lock()
+	backstagePrevMu.Lock()
-	copyPrev(img)
+	copyPrevbackstage(img)
-	prevMu.Unlock()
+	backstagePrevMu.Unlock()
 	statPrevCopyNs.Add(time.Since(prevCopyStart).Nanoseconds())
 
 	out := make([]byte, buf.Len())

Overlord-Client/cmd/agent/capture/capture_test.go

@@ -23,6 +23,51 @@ type recordingWriter struct {
 	types []websocket.MessageType
 }
 
+func TestEncodeBlocksBackstageAdvancesOnlyBackstageHistory(t *testing.T) {
+	img := image.NewRGBA(image.Rect(0, 0, 64, 64))
+	for i := range img.Pix {
+		img.Pix[i] = 0xff
+	}
+	previous := &prevImage{w: 64, h: 64, pix: make([]byte, len(img.Pix))}
+	desktopMarker := &prevImage{w: 1, h: 1, pix: []byte{0x42}}
+
+	prevMu.Lock()
+	savedDesktop := prevFrame
+	prevFrame = desktopMarker
+	prevMu.Unlock()
+	backstagePrevMu.Lock()
+	savedBackstage := backstagePrevFrame
+	backstagePrevFrame = nil
+	backstagePrevMu.Unlock()
+	t.Cleanup(func() {
+		prevMu.Lock()
+		prevFrame = savedDesktop
+		prevMu.Unlock()
+		backstagePrevMu.Lock()
+		backstagePrevFrame = savedBackstage
+		backstagePrevMu.Unlock()
+	})
+
+	regions, _, _, err := encodeBlocksbackstage(img, previous, 90, "raw", 0)
+	if err != nil {
+		t.Fatalf("encodeBlocksbackstage: %v", err)
+	}
+	if regions == 0 {
+		t.Fatal("expected changed backstage regions")
+	}
+
+	prevMu.Lock()
+	if prevFrame != desktopMarker || !bytes.Equal(prevFrame.pix, []byte{0x42}) {
+		t.Error("backstage block encode modified desktop history")
+	}
+	prevMu.Unlock()
+	backstagePrevMu.Lock()
+	if backstagePrevFrame == nil || !bytes.Equal(backstagePrevFrame.pix, img.Pix) {
+		t.Error("backstage block encode did not advance backstage history")
+	}
+	backstagePrevMu.Unlock()
+}
+
 func (w *recordingWriter) Write(ctx context.Context, messageType websocket.MessageType, p []byte) error {
 	w.types = append(w.types, messageType)
 	w.msgs = append(w.msgs, append([]byte(nil), p...))

Overlord-Client/cmd/agent/capture/jpeg_quality.go

@@ -0,0 +1,12 @@
+package capture
+
+func jpegEncoderQuality(quality int) int {
+	if quality < 1 {
+		quality = 1
+	}
+	if quality > 100 {
+		quality = 100
+	}
+	deficit := 100 - quality
+	return 100 - (deficit*deficit+99)/100
+}

Overlord-Client/cmd/agent/capture/jpeg_std.go

@@ -1,3 +1,5 @@
+//go:build !turbojpeg
+
 package capture
 
 import (
@@ -17,7 +19,7 @@ var jpegBufPool = sync.Pool{
 func encodeJPEG(img image.Image, quality int) ([]byte, error) {
 	buf := jpegBufPool.Get().(*bytes.Buffer)
 	buf.Reset()
-	err := jpeg.Encode(buf, img, &jpeg.Options{Quality: quality})
+	err := jpeg.Encode(buf, img, &jpeg.Options{Quality: jpegEncoderQuality(quality)})
 	if err != nil {
 		jpegBufPool.Put(buf)
 		return nil, err
@@ -31,7 +33,7 @@ func encodeJPEG(img image.Image, quality int) ([]byte, error) {
 func encodeJPEGToBuf(dst *bytes.Buffer, img image.Image, quality int) error {
 	buf := jpegBufPool.Get().(*bytes.Buffer)
 	buf.Reset()
-	if err := jpeg.Encode(buf, img, &jpeg.Options{Quality: quality}); err != nil {
+	if err := jpeg.Encode(buf, img, &jpeg.Options{Quality: jpegEncoderQuality(quality)}); err != nil {
 		jpegBufPool.Put(buf)
 		return err
 	}

Overlord-Client/cmd/agent/capture/jpeg_test.go

@@ -0,0 +1,25 @@
+package capture
+
+import (
+	"testing"
+)
+
+func TestJPEGEncoderQualityCurve(t *testing.T) {
+	tests := []struct {
+		input int
+		want  int
+	}{
+		{input: -1, want: 1},
+		{input: 20, want: 36},
+		{input: 50, want: 75},
+		{input: 80, want: 96},
+		{input: 90, want: 99},
+		{input: 100, want: 100},
+		{input: 101, want: 100},
+	}
+	for _, tt := range tests {
+		if got := jpegEncoderQuality(tt.input); got != tt.want {
+			t.Errorf("jpegEncoderQuality(%d) = %d, want %d", tt.input, got, tt.want)
+		}
+	}
+}

Overlord-Client/cmd/agent/capture/jpeg_turbo.go

@@ -5,18 +5,19 @@ package capture
 import (
 	"bytes"
 	"image"
+	"image/color"
 
 	ljpeg "github.com/pixiv/go-libjpeg/jpeg"
 )
 
 func encodeJPEG(img image.Image, quality int) ([]byte, error) {
 	opts := &ljpeg.EncoderOptions{
-		Quality:        quality,
+		Quality:        jpegEncoderQuality(quality),
 		OptimizeCoding: true,
 		DCTMethod:      ljpeg.DCTISlow,
 	}
 	buf := bytes.Buffer{}
-	if err := ljpeg.Encode(&buf, img, opts); err != nil {
+	if err := ljpeg.Encode(&buf, jpeg444Image(img), opts); err != nil {
 		return nil, err
 	}
 	return buf.Bytes(), nil
@@ -24,9 +25,49 @@ func encodeJPEG(img image.Image, quality int) ([]byte, error) {
 
 func encodeJPEGToBuf(dst *bytes.Buffer, img image.Image, quality int) error {
 	opts := &ljpeg.EncoderOptions{
-		Quality:        quality,
+		Quality:        jpegEncoderQuality(quality),
 		OptimizeCoding: true,
 		DCTMethod:      ljpeg.DCTISlow,
 	}
-	return ljpeg.Encode(dst, img, opts)
+	return ljpeg.Encode(dst, jpeg444Image(img), opts)
+}
+
+// libjpeg preserves the source YCbCr sampling factors, unlike Go's standard
+// JPEG encoder, so TurboJPEG builds can retain full chroma resolution.
+func jpeg444Image(src image.Image) image.Image {
+	if src == nil {
+		return src
+	}
+	if ycbcr, ok := src.(*image.YCbCr); ok && ycbcr.SubsampleRatio == image.YCbCrSubsampleRatio444 {
+		return src
+	}
+
+	bounds := src.Bounds()
+	dst := image.NewYCbCr(bounds, image.YCbCrSubsampleRatio444)
+	if rgba, ok := src.(*image.RGBA); ok {
+		for y := bounds.Min.Y; y < bounds.Max.Y; y++ {
+			srcOffset := rgba.PixOffset(bounds.Min.X, y)
+			dstOffset := dst.YOffset(bounds.Min.X, y)
+			for x := 0; x < bounds.Dx(); x++ {
+				i := srcOffset + x*4
+				yy, cb, cr := color.RGBToYCbCr(rgba.Pix[i], rgba.Pix[i+1], rgba.Pix[i+2])
+				dst.Y[dstOffset+x] = yy
+				dst.Cb[dstOffset+x] = cb
+				dst.Cr[dstOffset+x] = cr
+			}
+		}
+		return dst
+	}
+
+	for y := bounds.Min.Y; y < bounds.Max.Y; y++ {
+		for x := bounds.Min.X; x < bounds.Max.X; x++ {
+			pixel := color.NRGBAModel.Convert(src.At(x, y)).(color.NRGBA)
+			yy, cb, cr := color.RGBToYCbCr(pixel.R, pixel.G, pixel.B)
+			offset := dst.YOffset(x, y)
+			dst.Y[offset] = yy
+			dst.Cb[offset] = cb
+			dst.Cr[offset] = cr
+		}
+	}
+	return dst
 }

Overlord-Server/public/assets/backstage.js

@@ -149,7 +149,10 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
   let moveFrame = 0;
   let videoDecoder = null;
   let h264TimestampUs = 0;
+  let h264AwaitingKeyframe = true;
+  let h264RecoveryRequestedAt = 0;
   let prefersH264 = typeof VideoDecoder === "function";
+  const maxH264DecodeQueue = 4;
   const inputBackpressureBytes = 256 * 1024;
   let h264ErrorCount = 0;
   let h264RetryTimer = null;
@@ -1114,6 +1117,15 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
     }
     videoDecoder = null;
     h264TimestampUs = 0;
+    h264AwaitingKeyframe = true;
+    h264RecoveryRequestedAt = 0;
+  }
+
+  function requestH264DecoderRecovery(reason) {
+    const now = performance.now();
+    if (now - h264RecoveryRequestedAt < 500) return;
+    h264RecoveryRequestedAt = now;
+    sendCmd("backstage_request_keyframe", { reason });
   }
 
   function fallbackToJpegCodec(reason) {
@@ -1189,6 +1201,8 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
         },
         error: (err) => {
           console.warn("backstage: h264 decoder error", err);
+          h264AwaitingKeyframe = true;
+          requestH264DecoderRecovery("h264_decoder_error");
         },
       });
       videoDecoder.configure({ codec: "avc1.42E01E", optimizeForLatency: true });
@@ -1416,9 +1430,23 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
         fallbackToJpegCodec("WebCodecs decoder unavailable");
         return;
       }
+      const isKeyFrame = isH264KeyFrame(h264Bytes);
+      if (videoDecoder.decodeQueueSize > maxH264DecodeQueue) {
+        videoDecoder.reset();
+        h264TimestampUs = 0;
+        h264AwaitingKeyframe = true;
+        requestH264DecoderRecovery("h264_decoder_backlog");
+      }
+      if (h264AwaitingKeyframe && !isKeyFrame) {
+        requestH264DecoderRecovery("h264_decoder_keyframe_required");
+        return;
+      }
+      if (isKeyFrame) {
+        h264AwaitingKeyframe = false;
+      }
       const frameIntervalUs = Math.floor(1_000_000 / Math.max(1, fps || 25));
       const chunk = new EncodedVideoChunk({
-        type: isH264KeyFrame(h264Bytes) ? "key" : "delta",
+        type: isKeyFrame ? "key" : "delta",
         timestamp: h264TimestampUs,
         data: h264Bytes,
       });