vxaboveground/overlord
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

major remote desktop upgrades + do opus. Hopefully all the tests pass 😭

Browse Source
This commit is contained in:
KingKDot
2026-07-17 04:28:23 -05:00
parent 779f86b259
commit 0219081682
35 changed files with 1304 additions and 91 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Overlord-Client/cmd/agent/webrtcpub/audio_opus.go
+79
View File
@@ -0,0 +1,79 @@
//go:build overlord_webrtc
package webrtcpub
import (
"sync"
"time"
"github.com/pion/webrtc/v4"
"github.com/pion/webrtc/v4/pkg/media"
"github.com/thesyncim/gopus"
)
const (
opusInputRate = 48000
opusChannels = 2
opusPacketMs = 20
opusSamplesPerFrame = opusInputRate * opusPacketMs / 1000
opusBitrate = 96000
)
type opusAudioWriter struct {
t *webrtc.TrackLocalStaticSample
encoder *gopus.Encoder
mu sync.Mutex
pcm []int16
packet []byte
}
func newOpusAudioWriter(t *webrtc.TrackLocalStaticSample) (*opusAudioWriter, error) {
encoder, err := gopus.NewEncoder(gopus.EncoderConfig{
SampleRate: opusInputRate,
Channels: opusChannels,
Application: gopus.ApplicationAudio,
})
if err != nil {
return nil, err
}
if err := encoder.SetBitrate(opusBitrate); err != nil {
return nil, err
}
if err := encoder.SetInBandFEC(gopus.InBandFECEnabled); err != nil {
return nil, err
}
return &opusAudioWriter{
t: t,
encoder: encoder,
pcm: make([]int16, 0, opusSamplesPerFrame*4),
packet: make([]byte, 4000),
}, nil
}
func (w *opusAudioWriter) WriteAudio(pcm []int16) error {
if len(pcm) == 0 {
return nil
}
w.mu.Lock()
defer w.mu.Unlock()
w.pcm = append(w.pcm, pcm...)
frameValues := opusSamplesPerFrame * opusChannels
for len(w.pcm) >= frameValues {
n, err := w.encoder.EncodeInt16(w.pcm[:frameValues], w.packet)
if err != nil {
w.pcm = w.pcm[:0]
return err
}
if n > 0 {
if err := w.t.WriteSample(media.Sample{
Data: w.packet[:n],
Duration: opusPacketMs * time.Millisecond,
}); err != nil {
w.pcm = w.pcm[:0]
return err
}
}
w.pcm = w.pcm[frameValues:]
}
return nil
}
Overlord-Client/cmd/agent/webrtcpub/audio_opus_test.go
+33
View File
@@ -0,0 +1,33 @@
//go:build overlord_webrtc
package webrtcpub
import (
"testing"
"github.com/pion/webrtc/v4"
)
func TestOpusAudioWriterAcceptsFragmentedStereoPCM(t *testing.T) {
track, err := webrtc.NewTrackLocalStaticSample(
webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeOpus, ClockRate: 48000, Channels: 2},
"audio", "test",
)
if err != nil {
t.Fatal(err)
}
writer, err := newOpusAudioWriter(track)
if err != nil {
t.Fatal(err)
}
frame := make([]int16, opusSamplesPerFrame*opusChannels)
if err := writer.WriteAudio(frame[:333]); err != nil {
t.Fatal(err)
}
if err := writer.WriteAudio(frame[333:]); err != nil {
t.Fatal(err)
}
if len(writer.pcm) != 0 {
t.Fatalf("buffered PCM = %d, want 0 after a complete frame", len(writer.pcm))
}
}
Overlord-Client/cmd/agent/webrtcpub/p2p_pion.go
+15 -8
View File
@@ -71,11 +71,12 @@ func StartP2POffer(ctx context.Context, kind Kind, sessionID string, offerSDP st
if hasAudio {
if err := mediaEngine.RegisterCodec(webrtc.RTPCodecParameters{
RTPCodecCapability: webrtc.RTPCodecCapability{
MimeType: webrtc.MimeTypePCMU,
ClockRate: 8000,
Channels: 1,
MimeType: webrtc.MimeTypeOpus,
ClockRate: 48000,
Channels: 2,
SDPFmtpLine: "minptime=10;useinbandfec=1",
},
PayloadType: 0,
PayloadType: 111,
}, webrtc.RTPCodecTypeAudio); err != nil {
return "", fmt.Errorf("register audio codec: %w", err)
}
@@ -92,8 +93,9 @@ func StartP2POffer(ctx context.Context, kind Kind, sessionID string, offerSDP st
}
var (
videoTrack *webrtc.TrackLocalStaticSample
audioTrack *webrtc.TrackLocalStaticSample
videoTrack *webrtc.TrackLocalStaticSample
audioTrack *webrtc.TrackLocalStaticSample
audioWriter AudioWriter
)
if hasVideo {
videoTrack, err = webrtc.NewTrackLocalStaticSample(
@@ -117,13 +119,18 @@ func StartP2POffer(ctx context.Context, kind Kind, sessionID string, offerSDP st
}
if hasAudio {
audioTrack, err = webrtc.NewTrackLocalStaticSample(
webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypePCMU, ClockRate: 8000, Channels: 1},
webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeOpus, ClockRate: 48000, Channels: 2, SDPFmtpLine: "minptime=10;useinbandfec=1"},
"overlord-audio-p2p-"+string(kind), "overlord-"+string(kind),
)
if err != nil {
_ = pc.Close()
return "", fmt.Errorf("new audio track: %w", err)
}
audioWriter, err = newOpusAudioWriter(audioTrack)
if err != nil {
_ = pc.Close()
return "", fmt.Errorf("new opus encoder: %w", err)
}
tx, err := pc.AddTransceiverFromTrack(audioTrack, webrtc.RTPTransceiverInit{
Direction: webrtc.RTPTransceiverDirectionSendonly,
})
@@ -201,7 +208,7 @@ func StartP2POffer(ctx context.Context, kind Kind, sessionID string, offerSDP st
registerVideoWriter(kind, writerID, &h264TrackWriter{t: videoTrack})
}
if audioTrack != nil {
registerAudioWriter(kind, writerID, newPCMUAudioWriter(audioTrack))
registerAudioWriter(kind, writerID, audioWriter)
}
return answer.SDP, nil
}
Overlord-Client/cmd/agent/webrtcpub/whip_pion.go
+15 -8
View File
@@ -133,11 +133,12 @@ func Start(ctx context.Context, kind Kind, opts Options) (*Publisher, error) {
if opts.HasAudio {
if err := mediaEngine.RegisterCodec(webrtc.RTPCodecParameters{
RTPCodecCapability: webrtc.RTPCodecCapability{
MimeType: webrtc.MimeTypePCMU,
ClockRate: 8000,
Channels: 1,
MimeType: webrtc.MimeTypeOpus,
ClockRate: 48000,
Channels: 2,
SDPFmtpLine: "minptime=10;useinbandfec=1",
},
PayloadType: 0,
PayloadType: 111,
}, webrtc.RTPCodecTypeAudio); err != nil {
return nil, fmt.Errorf("register audio codec: %w", err)
}
@@ -150,8 +151,9 @@ func Start(ctx context.Context, kind Kind, opts Options) (*Publisher, error) {
}
var (
videoTrack *webrtc.TrackLocalStaticSample
audioTrack *webrtc.TrackLocalStaticSample
videoTrack *webrtc.TrackLocalStaticSample
audioTrack *webrtc.TrackLocalStaticSample
audioWriter AudioWriter
)
if opts.HasVideo {
videoTrack, err = webrtc.NewTrackLocalStaticSample(
@@ -175,13 +177,18 @@ func Start(ctx context.Context, kind Kind, opts Options) (*Publisher, error) {
}
if opts.HasAudio {
audioTrack, err = webrtc.NewTrackLocalStaticSample(
webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypePCMU, ClockRate: 8000, Channels: 1},
webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeOpus, ClockRate: 48000, Channels: 2, SDPFmtpLine: "minptime=10;useinbandfec=1"},
"overlord-audio-"+string(kind), "overlord-"+string(kind),
)
if err != nil {
_ = pc.Close()
return nil, fmt.Errorf("new audio track: %w", err)
}
audioWriter, err = newOpusAudioWriter(audioTrack)
if err != nil {
_ = pc.Close()
return nil, fmt.Errorf("new opus encoder: %w", err)
}
tx, err := pc.AddTransceiverFromTrack(audioTrack, webrtc.RTPTransceiverInit{
Direction: webrtc.RTPTransceiverDirectionSendonly,
})
@@ -256,7 +263,7 @@ func Start(ctx context.Context, kind Kind, opts Options) (*Publisher, error) {
registerVideoWriter(kind, writerID, &h264TrackWriter{t: videoTrack})
}
if audioTrack != nil {
registerAudioWriter(kind, writerID, newPCMUAudioWriter(audioTrack))
registerAudioWriter(kind, writerID, audioWriter)
}
log.Printf("webrtcpub: WHIP[%s] session established (resource=%s)", kind, resourceURL)
return pub, nil