diff --git a/Overlord-Client/cmd/agent/capture/capture.go b/Overlord-Client/cmd/agent/capture/capture.go index e01c924..d717451 100644 --- a/Overlord-Client/cmd/agent/capture/capture.go +++ b/Overlord-Client/cmd/agent/capture/capture.go @@ -1648,9 +1648,9 @@ func encodeBlocksbackstage(img *image.RGBA, prev *prevImage, quality int, codec } prevCopyStart := time.Now() - prevMu.Lock() - copyPrev(img) - prevMu.Unlock() + backstagePrevMu.Lock() + copyPrevbackstage(img) + backstagePrevMu.Unlock() statPrevCopyNs.Add(time.Since(prevCopyStart).Nanoseconds()) out := make([]byte, buf.Len()) diff --git a/Overlord-Client/cmd/agent/capture/capture_test.go b/Overlord-Client/cmd/agent/capture/capture_test.go index 0693f25..6547c6d 100644 --- a/Overlord-Client/cmd/agent/capture/capture_test.go +++ b/Overlord-Client/cmd/agent/capture/capture_test.go @@ -23,6 +23,51 @@ type recordingWriter struct { types []websocket.MessageType } +func TestEncodeBlocksBackstageAdvancesOnlyBackstageHistory(t *testing.T) { + img := image.NewRGBA(image.Rect(0, 0, 64, 64)) + for i := range img.Pix { + img.Pix[i] = 0xff + } + previous := &prevImage{w: 64, h: 64, pix: make([]byte, len(img.Pix))} + desktopMarker := &prevImage{w: 1, h: 1, pix: []byte{0x42}} + + prevMu.Lock() + savedDesktop := prevFrame + prevFrame = desktopMarker + prevMu.Unlock() + backstagePrevMu.Lock() + savedBackstage := backstagePrevFrame + backstagePrevFrame = nil + backstagePrevMu.Unlock() + t.Cleanup(func() { + prevMu.Lock() + prevFrame = savedDesktop + prevMu.Unlock() + backstagePrevMu.Lock() + backstagePrevFrame = savedBackstage + backstagePrevMu.Unlock() + }) + + regions, _, _, err := encodeBlocksbackstage(img, previous, 90, "raw", 0) + if err != nil { + t.Fatalf("encodeBlocksbackstage: %v", err) + } + if regions == 0 { + t.Fatal("expected changed backstage regions") + } + + prevMu.Lock() + if prevFrame != desktopMarker || !bytes.Equal(prevFrame.pix, []byte{0x42}) { + t.Error("backstage block encode modified desktop history") + } + prevMu.Unlock() + backstagePrevMu.Lock() + if backstagePrevFrame == nil || !bytes.Equal(backstagePrevFrame.pix, img.Pix) { + t.Error("backstage block encode did not advance backstage history") + } + backstagePrevMu.Unlock() +} + func (w *recordingWriter) Write(ctx context.Context, messageType websocket.MessageType, p []byte) error { w.types = append(w.types, messageType) w.msgs = append(w.msgs, append([]byte(nil), p...)) diff --git a/Overlord-Client/cmd/agent/capture/jpeg_quality.go b/Overlord-Client/cmd/agent/capture/jpeg_quality.go new file mode 100644 index 0000000..b86d3c0 --- /dev/null +++ b/Overlord-Client/cmd/agent/capture/jpeg_quality.go @@ -0,0 +1,12 @@ +package capture + +func jpegEncoderQuality(quality int) int { + if quality < 1 { + quality = 1 + } + if quality > 100 { + quality = 100 + } + deficit := 100 - quality + return 100 - (deficit*deficit+99)/100 +} diff --git a/Overlord-Client/cmd/agent/capture/jpeg_std.go b/Overlord-Client/cmd/agent/capture/jpeg_std.go index 27fd24c..6f83438 100644 --- a/Overlord-Client/cmd/agent/capture/jpeg_std.go +++ b/Overlord-Client/cmd/agent/capture/jpeg_std.go @@ -1,3 +1,5 @@ +//go:build !turbojpeg + package capture import ( @@ -17,7 +19,7 @@ var jpegBufPool = sync.Pool{ func encodeJPEG(img image.Image, quality int) ([]byte, error) { buf := jpegBufPool.Get().(*bytes.Buffer) buf.Reset() - err := jpeg.Encode(buf, img, &jpeg.Options{Quality: quality}) + err := jpeg.Encode(buf, img, &jpeg.Options{Quality: jpegEncoderQuality(quality)}) if err != nil { jpegBufPool.Put(buf) return nil, err @@ -31,7 +33,7 @@ func encodeJPEG(img image.Image, quality int) ([]byte, error) { func encodeJPEGToBuf(dst *bytes.Buffer, img image.Image, quality int) error { buf := jpegBufPool.Get().(*bytes.Buffer) buf.Reset() - if err := jpeg.Encode(buf, img, &jpeg.Options{Quality: quality}); err != nil { + if err := jpeg.Encode(buf, img, &jpeg.Options{Quality: jpegEncoderQuality(quality)}); err != nil { jpegBufPool.Put(buf) return err } diff --git a/Overlord-Client/cmd/agent/capture/jpeg_test.go b/Overlord-Client/cmd/agent/capture/jpeg_test.go new file mode 100644 index 0000000..bed1e09 --- /dev/null +++ b/Overlord-Client/cmd/agent/capture/jpeg_test.go @@ -0,0 +1,25 @@ +package capture + +import ( + "testing" +) + +func TestJPEGEncoderQualityCurve(t *testing.T) { + tests := []struct { + input int + want int + }{ + {input: -1, want: 1}, + {input: 20, want: 36}, + {input: 50, want: 75}, + {input: 80, want: 96}, + {input: 90, want: 99}, + {input: 100, want: 100}, + {input: 101, want: 100}, + } + for _, tt := range tests { + if got := jpegEncoderQuality(tt.input); got != tt.want { + t.Errorf("jpegEncoderQuality(%d) = %d, want %d", tt.input, got, tt.want) + } + } +} diff --git a/Overlord-Client/cmd/agent/capture/jpeg_turbo.go b/Overlord-Client/cmd/agent/capture/jpeg_turbo.go index f987314..fd6860c 100644 --- a/Overlord-Client/cmd/agent/capture/jpeg_turbo.go +++ b/Overlord-Client/cmd/agent/capture/jpeg_turbo.go @@ -5,18 +5,19 @@ package capture import ( "bytes" "image" + "image/color" ljpeg "github.com/pixiv/go-libjpeg/jpeg" ) func encodeJPEG(img image.Image, quality int) ([]byte, error) { opts := &ljpeg.EncoderOptions{ - Quality: quality, + Quality: jpegEncoderQuality(quality), OptimizeCoding: true, DCTMethod: ljpeg.DCTISlow, } buf := bytes.Buffer{} - if err := ljpeg.Encode(&buf, img, opts); err != nil { + if err := ljpeg.Encode(&buf, jpeg444Image(img), opts); err != nil { return nil, err } return buf.Bytes(), nil @@ -24,9 +25,49 @@ func encodeJPEG(img image.Image, quality int) ([]byte, error) { func encodeJPEGToBuf(dst *bytes.Buffer, img image.Image, quality int) error { opts := &ljpeg.EncoderOptions{ - Quality: quality, + Quality: jpegEncoderQuality(quality), OptimizeCoding: true, DCTMethod: ljpeg.DCTISlow, } - return ljpeg.Encode(dst, img, opts) + return ljpeg.Encode(dst, jpeg444Image(img), opts) +} + +// libjpeg preserves the source YCbCr sampling factors, unlike Go's standard +// JPEG encoder, so TurboJPEG builds can retain full chroma resolution. +func jpeg444Image(src image.Image) image.Image { + if src == nil { + return src + } + if ycbcr, ok := src.(*image.YCbCr); ok && ycbcr.SubsampleRatio == image.YCbCrSubsampleRatio444 { + return src + } + + bounds := src.Bounds() + dst := image.NewYCbCr(bounds, image.YCbCrSubsampleRatio444) + if rgba, ok := src.(*image.RGBA); ok { + for y := bounds.Min.Y; y < bounds.Max.Y; y++ { + srcOffset := rgba.PixOffset(bounds.Min.X, y) + dstOffset := dst.YOffset(bounds.Min.X, y) + for x := 0; x < bounds.Dx(); x++ { + i := srcOffset + x*4 + yy, cb, cr := color.RGBToYCbCr(rgba.Pix[i], rgba.Pix[i+1], rgba.Pix[i+2]) + dst.Y[dstOffset+x] = yy + dst.Cb[dstOffset+x] = cb + dst.Cr[dstOffset+x] = cr + } + } + return dst + } + + for y := bounds.Min.Y; y < bounds.Max.Y; y++ { + for x := bounds.Min.X; x < bounds.Max.X; x++ { + pixel := color.NRGBAModel.Convert(src.At(x, y)).(color.NRGBA) + yy, cb, cr := color.RGBToYCbCr(pixel.R, pixel.G, pixel.B) + offset := dst.YOffset(x, y) + dst.Y[offset] = yy + dst.Cb[offset] = cb + dst.Cr[offset] = cr + } + } + return dst } diff --git a/Overlord-Server/public/assets/backstage.js b/Overlord-Server/public/assets/backstage.js index 3013253..7a6c2ac 100644 --- a/Overlord-Server/public/assets/backstage.js +++ b/Overlord-Server/public/assets/backstage.js @@ -149,7 +149,10 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s let moveFrame = 0; let videoDecoder = null; let h264TimestampUs = 0; + let h264AwaitingKeyframe = true; + let h264RecoveryRequestedAt = 0; let prefersH264 = typeof VideoDecoder === "function"; + const maxH264DecodeQueue = 4; const inputBackpressureBytes = 256 * 1024; let h264ErrorCount = 0; let h264RetryTimer = null; @@ -1114,6 +1117,15 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s } videoDecoder = null; h264TimestampUs = 0; + h264AwaitingKeyframe = true; + h264RecoveryRequestedAt = 0; + } + + function requestH264DecoderRecovery(reason) { + const now = performance.now(); + if (now - h264RecoveryRequestedAt < 500) return; + h264RecoveryRequestedAt = now; + sendCmd("backstage_request_keyframe", { reason }); } function fallbackToJpegCodec(reason) { @@ -1189,6 +1201,8 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s }, error: (err) => { console.warn("backstage: h264 decoder error", err); + h264AwaitingKeyframe = true; + requestH264DecoderRecovery("h264_decoder_error"); }, }); videoDecoder.configure({ codec: "avc1.42E01E", optimizeForLatency: true }); @@ -1416,9 +1430,23 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s fallbackToJpegCodec("WebCodecs decoder unavailable"); return; } + const isKeyFrame = isH264KeyFrame(h264Bytes); + if (videoDecoder.decodeQueueSize > maxH264DecodeQueue) { + videoDecoder.reset(); + h264TimestampUs = 0; + h264AwaitingKeyframe = true; + requestH264DecoderRecovery("h264_decoder_backlog"); + } + if (h264AwaitingKeyframe && !isKeyFrame) { + requestH264DecoderRecovery("h264_decoder_keyframe_required"); + return; + } + if (isKeyFrame) { + h264AwaitingKeyframe = false; + } const frameIntervalUs = Math.floor(1_000_000 / Math.max(1, fps || 25)); const chunk = new EncodedVideoChunk({ - type: isH264KeyFrame(h264Bytes) ? "key" : "delta", + type: isKeyFrame ? "key" : "delta", timestamp: h264TimestampUs, data: h264Bytes, });