vxaboveground/overlord
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

fix jpeg

Browse Source
This commit is contained in:
KingKDot
2026-07-30 14:16:34 -05:00
parent 2669ab1388
commit 9979955a49
7 changed files with 163 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Overlord-Client/cmd/agent/capture/capture.go
+3 -3
View File
@@ -1648,9 +1648,9 @@ func encodeBlocksbackstage(img *image.RGBA, prev *prevImage, quality int, codec
}
prevCopyStart := time.Now()
prevMu.Lock()
copyPrev(img)
prevMu.Unlock()
backstagePrevMu.Lock()
copyPrevbackstage(img)
backstagePrevMu.Unlock()
statPrevCopyNs.Add(time.Since(prevCopyStart).Nanoseconds())
out := make([]byte, buf.Len())
Overlord-Client/cmd/agent/capture/capture_test.go
+45
View File
@@ -23,6 +23,51 @@ type recordingWriter struct {
types []websocket.MessageType
}
func TestEncodeBlocksBackstageAdvancesOnlyBackstageHistory(t *testing.T) {
img := image.NewRGBA(image.Rect(0, 0, 64, 64))
for i := range img.Pix {
img.Pix[i] = 0xff
}
previous := &prevImage{w: 64, h: 64, pix: make([]byte, len(img.Pix))}
desktopMarker := &prevImage{w: 1, h: 1, pix: []byte{0x42}}
prevMu.Lock()
savedDesktop := prevFrame
prevFrame = desktopMarker
prevMu.Unlock()
backstagePrevMu.Lock()
savedBackstage := backstagePrevFrame
backstagePrevFrame = nil
backstagePrevMu.Unlock()
t.Cleanup(func() {
prevMu.Lock()
prevFrame = savedDesktop
prevMu.Unlock()
backstagePrevMu.Lock()
backstagePrevFrame = savedBackstage
backstagePrevMu.Unlock()
})
regions, _, _, err := encodeBlocksbackstage(img, previous, 90, "raw", 0)
if err != nil {
t.Fatalf("encodeBlocksbackstage: %v", err)
}
if regions == 0 {
t.Fatal("expected changed backstage regions")
}
prevMu.Lock()
if prevFrame != desktopMarker || !bytes.Equal(prevFrame.pix, []byte{0x42}) {
t.Error("backstage block encode modified desktop history")
}
prevMu.Unlock()
backstagePrevMu.Lock()
if backstagePrevFrame == nil || !bytes.Equal(backstagePrevFrame.pix, img.Pix) {
t.Error("backstage block encode did not advance backstage history")
}
backstagePrevMu.Unlock()
}
func (w *recordingWriter) Write(ctx context.Context, messageType websocket.MessageType, p []byte) error {
w.types = append(w.types, messageType)
w.msgs = append(w.msgs, append([]byte(nil), p...))
Overlord-Client/cmd/agent/capture/jpeg_quality.go
+12
View File
@@ -0,0 +1,12 @@
package capture
func jpegEncoderQuality(quality int) int {
if quality < 1 {
quality = 1
}
if quality > 100 {
quality = 100
}
deficit := 100 - quality
return 100 - (deficit*deficit+99)/100
}
Overlord-Client/cmd/agent/capture/jpeg_std.go
+4 -2
View File
@@ -1,3 +1,5 @@
//go:build !turbojpeg
package capture
import (
@@ -17,7 +19,7 @@ var jpegBufPool = sync.Pool{
func encodeJPEG(img image.Image, quality int) ([]byte, error) {
buf := jpegBufPool.Get().(*bytes.Buffer)
buf.Reset()
err := jpeg.Encode(buf, img, &jpeg.Options{Quality: quality})
err := jpeg.Encode(buf, img, &jpeg.Options{Quality: jpegEncoderQuality(quality)})
if err != nil {
jpegBufPool.Put(buf)
return nil, err
@@ -31,7 +33,7 @@ func encodeJPEG(img image.Image, quality int) ([]byte, error) {
func encodeJPEGToBuf(dst *bytes.Buffer, img image.Image, quality int) error {
buf := jpegBufPool.Get().(*bytes.Buffer)
buf.Reset()
if err := jpeg.Encode(buf, img, &jpeg.Options{Quality: quality}); err != nil {
if err := jpeg.Encode(buf, img, &jpeg.Options{Quality: jpegEncoderQuality(quality)}); err != nil {
jpegBufPool.Put(buf)
return err
}
Overlord-Client/cmd/agent/capture/jpeg_test.go
+25
View File
@@ -0,0 +1,25 @@
package capture
import (
"testing"
)
func TestJPEGEncoderQualityCurve(t *testing.T) {
tests := []struct {
input int
want int
}{
{input: -1, want: 1},
{input: 20, want: 36},
{input: 50, want: 75},
{input: 80, want: 96},
{input: 90, want: 99},
{input: 100, want: 100},
{input: 101, want: 100},
}
for _, tt := range tests {
if got := jpegEncoderQuality(tt.input); got != tt.want {
t.Errorf("jpegEncoderQuality(%d) = %d, want %d", tt.input, got, tt.want)
}
}
}
Overlord-Client/cmd/agent/capture/jpeg_turbo.go
+45 -4
View File
@@ -5,18 +5,19 @@ package capture
import (
"bytes"
"image"
"image/color"
ljpeg "github.com/pixiv/go-libjpeg/jpeg"
)
func encodeJPEG(img image.Image, quality int) ([]byte, error) {
opts := &ljpeg.EncoderOptions{
Quality: quality,
Quality: jpegEncoderQuality(quality),
OptimizeCoding: true,
DCTMethod: ljpeg.DCTISlow,
}
buf := bytes.Buffer{}
if err := ljpeg.Encode(&buf, img, opts); err != nil {
if err := ljpeg.Encode(&buf, jpeg444Image(img), opts); err != nil {
return nil, err
}
return buf.Bytes(), nil
@@ -24,9 +25,49 @@ func encodeJPEG(img image.Image, quality int) ([]byte, error) {
func encodeJPEGToBuf(dst *bytes.Buffer, img image.Image, quality int) error {
opts := &ljpeg.EncoderOptions{
Quality: quality,
Quality: jpegEncoderQuality(quality),
OptimizeCoding: true,
DCTMethod: ljpeg.DCTISlow,
}
return ljpeg.Encode(dst, img, opts)
return ljpeg.Encode(dst, jpeg444Image(img), opts)
}
// libjpeg preserves the source YCbCr sampling factors, unlike Go's standard
// JPEG encoder, so TurboJPEG builds can retain full chroma resolution.
func jpeg444Image(src image.Image) image.Image {
if src == nil {
return src
}
if ycbcr, ok := src.(*image.YCbCr); ok && ycbcr.SubsampleRatio == image.YCbCrSubsampleRatio444 {
return src
}
bounds := src.Bounds()
dst := image.NewYCbCr(bounds, image.YCbCrSubsampleRatio444)
if rgba, ok := src.(*image.RGBA); ok {
for y := bounds.Min.Y; y < bounds.Max.Y; y++ {
srcOffset := rgba.PixOffset(bounds.Min.X, y)
dstOffset := dst.YOffset(bounds.Min.X, y)
for x := 0; x < bounds.Dx(); x++ {
i := srcOffset + x*4
yy, cb, cr := color.RGBToYCbCr(rgba.Pix[i], rgba.Pix[i+1], rgba.Pix[i+2])
dst.Y[dstOffset+x] = yy
dst.Cb[dstOffset+x] = cb
dst.Cr[dstOffset+x] = cr
}
}
return dst
}
for y := bounds.Min.Y; y < bounds.Max.Y; y++ {
for x := bounds.Min.X; x < bounds.Max.X; x++ {
pixel := color.NRGBAModel.Convert(src.At(x, y)).(color.NRGBA)
yy, cb, cr := color.RGBToYCbCr(pixel.R, pixel.G, pixel.B)
offset := dst.YOffset(x, y)
dst.Y[offset] = yy
dst.Cb[offset] = cb
dst.Cr[offset] = cr
}
}
return dst
}
Overlord-Server/public/assets/backstage.js
+29 -1
View File
@@ -149,7 +149,10 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
let moveFrame = 0;
let videoDecoder = null;
let h264TimestampUs = 0;
let h264AwaitingKeyframe = true;
let h264RecoveryRequestedAt = 0;
let prefersH264 = typeof VideoDecoder === "function";
const maxH264DecodeQueue = 4;
const inputBackpressureBytes = 256 * 1024;
let h264ErrorCount = 0;
let h264RetryTimer = null;
@@ -1114,6 +1117,15 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
}
videoDecoder = null;
h264TimestampUs = 0;
h264AwaitingKeyframe = true;
h264RecoveryRequestedAt = 0;
}
function requestH264DecoderRecovery(reason) {
const now = performance.now();
if (now - h264RecoveryRequestedAt < 500) return;
h264RecoveryRequestedAt = now;
sendCmd("backstage_request_keyframe", { reason });
}
function fallbackToJpegCodec(reason) {
@@ -1189,6 +1201,8 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
},
error: (err) => {
console.warn("backstage: h264 decoder error", err);
h264AwaitingKeyframe = true;
requestH264DecoderRecovery("h264_decoder_error");
},
});
videoDecoder.configure({ codec: "avc1.42E01E", optimizeForLatency: true });
@@ -1416,9 +1430,23 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
fallbackToJpegCodec("WebCodecs decoder unavailable");
return;
}
const isKeyFrame = isH264KeyFrame(h264Bytes);
if (videoDecoder.decodeQueueSize > maxH264DecodeQueue) {
videoDecoder.reset();
h264TimestampUs = 0;
h264AwaitingKeyframe = true;
requestH264DecoderRecovery("h264_decoder_backlog");
}
if (h264AwaitingKeyframe && !isKeyFrame) {
requestH264DecoderRecovery("h264_decoder_keyframe_required");
return;
}
if (isKeyFrame) {
h264AwaitingKeyframe = false;
}
const frameIntervalUs = Math.floor(1_000_000 / Math.max(1, fps || 25));
const chunk = new EncodedVideoChunk({
type: isH264KeyFrame(h264Bytes) ? "key" : "delta",
type: isKeyFrame ? "key" : "delta",
timestamp: h264TimestampUs,
data: h264Bytes,
});