desktop audio and switches to turn up/down audio

This commit is contained in:
KingKDot
2026-04-16 17:19:56 -05:00
parent df95ea4771
commit 135091f6fa
15 changed files with 818 additions and 12 deletions
@@ -45,6 +45,7 @@ import { checkFeatureAccess } from "./feature-gate.js";
const inputLatency = document.getElementById("inputLatency");
const statusEl = document.getElementById("streamStatus");
const clipboardSyncCtrl = document.getElementById("clipboardSyncCtrl");
const audioCtrl = document.getElementById("audioCtrl");
ws.binaryType = "arraybuffer";
let activeClientId = clientId;
@@ -98,6 +99,125 @@ import { checkFeatureAccess } from "./feature-gate.js";
h264KeyframeErrorStreak = 0;
}
/* ── Remote Desktop Audio (system audio from client) ── */
const AUDIO_SAMPLE_RATE = 16000;
const AUDIO_PLAYBACK_FRAME = 512;
const AUDIO_MAX_BUFFER_MS = 120;
let audioWs = null;
let audioPlayCtx = null;
let audioProcessorNode = null;
let audioChunks = [];
let audioChunkOffset = 0;
function audioResampleInt16ToFloat32(srcInt16, srcRate, dstRate) {
if (!srcInt16 || srcInt16.length === 0) return new Float32Array(0);
if (srcRate === dstRate) {
const out = new Float32Array(srcInt16.length);
for (let i = 0; i < srcInt16.length; i++) out[i] = srcInt16[i] / 0x8000;
return out;
}
const outLength = Math.max(1, Math.round((srcInt16.length * dstRate) / srcRate));
const out = new Float32Array(outLength);
const step = srcRate / dstRate;
for (let i = 0; i < outLength; i++) {
const srcPos = i * step;
const i0 = Math.floor(srcPos);
const i1 = Math.min(i0 + 1, srcInt16.length - 1);
const frac = srcPos - i0;
out[i] = (srcInt16[i0] * (1 - frac) + srcInt16[i1] * frac) / 0x8000;
}
return out;
}
function initAudioPlayback() {
if (!audioPlayCtx) {
audioPlayCtx = new AudioContext({ sampleRate: AUDIO_SAMPLE_RATE, latencyHint: "interactive" });
}
if (!audioProcessorNode) {
audioProcessorNode = audioPlayCtx.createScriptProcessor(AUDIO_PLAYBACK_FRAME, 1, 1);
audioProcessorNode.onaudioprocess = function (event) {
const out = event.outputBuffer.getChannelData(0);
out.fill(0);
let writeIndex = 0;
while (writeIndex < out.length && audioChunks.length > 0) {
const head = audioChunks[0];
const remaining = head.length - audioChunkOffset;
if (remaining <= 0) { audioChunks.shift(); audioChunkOffset = 0; continue; }
const take = Math.min(out.length - writeIndex, remaining);
out.set(head.subarray(audioChunkOffset, audioChunkOffset + take), writeIndex);
writeIndex += take;
audioChunkOffset += take;
if (audioChunkOffset >= head.length) { audioChunks.shift(); audioChunkOffset = 0; }
}
};
audioProcessorNode.connect(audioPlayCtx.destination);
}
}
function appendAudioPcm(binary) {
if (!audioPlayCtx) initAudioPlayback();
const samples = Math.floor(binary.byteLength / 2);
if (samples <= 0) return;
const src = new Int16Array(binary.buffer, binary.byteOffset, samples);
const chunk = audioResampleInt16ToFloat32(src, AUDIO_SAMPLE_RATE, audioPlayCtx?.sampleRate || AUDIO_SAMPLE_RATE);
if (chunk.length === 0) return;
audioChunks.push(chunk);
let buffered = -audioChunkOffset;
for (const c of audioChunks) buffered += c.length;
const rate = audioPlayCtx?.sampleRate || AUDIO_SAMPLE_RATE;
const max = Math.max(AUDIO_PLAYBACK_FRAME, Math.round(rate * (AUDIO_MAX_BUFFER_MS / 1000)));
while (buffered > max && audioChunks.length > 0) {
const dropped = audioChunks.shift();
buffered -= dropped?.length || 0;
audioChunkOffset = 0;
}
}
function connectAudio() {
if (audioWs && audioWs.readyState === WebSocket.OPEN) return;
// Create AudioContext in click/change handler so the browser trusts the user gesture.
initAudioPlayback();
if (audioPlayCtx?.state === "suspended") audioPlayCtx.resume();
const proto = location.protocol === "https:" ? "wss:" : "ws:";
audioWs = new WebSocket(proto + "//" + location.host + "/api/clients/" + encodeURIComponent(clientId) + "/desktop-audio/ws");
audioWs.binaryType = "arraybuffer";
audioWs.onopen = function () {
audioWs.send(JSON.stringify({ type: "start", source: "system" }));
};
audioWs.onmessage = function (ev) {
if (typeof ev.data === "string") return;
const bytes = new Uint8Array(ev.data);
if (bytes.byteLength > 1) appendAudioPcm(bytes);
};
audioWs.onclose = function () { cleanupAudio(false); };
audioWs.onerror = function () {};
}
function disconnectAudio() {
if (audioWs) {
try { audioWs.send(JSON.stringify({ type: "stop" })); } catch {}
try { audioWs.close(); } catch {}
audioWs = null;
}
cleanupAudio(true);
}
function cleanupAudio(uncheckBox) {
if (audioProcessorNode) {
audioProcessorNode.disconnect();
audioProcessorNode.onaudioprocess = null;
audioProcessorNode = null;
}
if (audioPlayCtx) {
audioPlayCtx.close().catch(function () {});
audioPlayCtx = null;
}
audioChunks = [];
audioChunkOffset = 0;
audioWs = null;
if (uncheckBox && audioCtrl) audioCtrl.checked = false;
}
function resetH264SessionState() {
resetH264RuntimeState();
h264RecoveryAttempts = 0;
@@ -565,6 +685,7 @@ import { checkFeatureAccess } from "./feature-gate.js";
desiredStreaming = false;
setStreamState("stopping", "Stopping stream");
sendCmd("desktop_stop", {});
disconnectAudio();
});
fullscreenBtn.addEventListener("click", function () {
if (canvasContainer.requestFullscreen) {
@@ -620,6 +741,15 @@ import { checkFeatureAccess } from "./feature-gate.js";
pushCaptureToggles();
});
}
if (audioCtrl) {
audioCtrl.addEventListener("change", function () {
if (audioCtrl.checked) {
connectAudio();
} else {
disconnectAudio();
}
});
}
if (qualitySlider) {
updateQualityLabel(qualitySlider.value);
@@ -1076,6 +1206,7 @@ import { checkFeatureAccess } from "./feature-gate.js";
ws.addEventListener("close", function () {
desiredStreaming = false;
disconnectAudio();
destroyVideoDecoder();
setStreamState("disconnected", "Disconnected");
});
@@ -1217,6 +1348,7 @@ import { checkFeatureAccess } from "./feature-gate.js";
desiredStreaming = false;
sendCmd("desktop_stop", {});
}
disconnectAudio();
destroyVideoDecoder();
}
+236 -1
View File
@@ -17,6 +17,13 @@ const remoteWaveCanvas = document.getElementById("remote-wave");
const sourceSelect = document.getElementById("source-select");
const muteSelfBtn = document.getElementById("mute-self-btn");
const muteRemoteBtn = document.getElementById("mute-remote-btn");
const micVolumeSlider = document.getElementById("mic-volume");
const micVolumeLabel = document.getElementById("mic-volume-label");
const remoteVolumeSlider = document.getElementById("remote-volume");
const remoteVolumeLabel = document.getElementById("remote-volume-label");
let micGain = 1.0;
let remoteGain = 1.0;
let ws = null;
let mediaStream = null;
@@ -25,6 +32,7 @@ let processorNode = null;
let captureAudioCtx = null;
let playAudioCtx = null;
let playProcessorNode = null;
let playGainNode = null;
let selfMuted = true;
let remoteMuted = true;
let playbackChunks = [];
@@ -236,6 +244,11 @@ function initPlaybackEngine() {
if (!playAudioCtx) {
playAudioCtx = new AudioContext({ sampleRate: 16000, latencyHint: "interactive" });
}
if (!playGainNode) {
playGainNode = playAudioCtx.createGain();
playGainNode.gain.value = remoteGain;
playGainNode.connect(playAudioCtx.destination);
}
if (!playProcessorNode) {
playProcessorNode = playAudioCtx.createScriptProcessor(PLAYBACK_FRAME_SIZE, 1, 1);
playProcessorNode.onaudioprocess = (event) => {
@@ -266,7 +279,7 @@ function initPlaybackEngine() {
}
}
};
playProcessorNode.connect(playAudioCtx.destination);
playProcessorNode.connect(playGainNode);
}
}
@@ -293,6 +306,9 @@ async function startMicCapture() {
const inData = event.inputBuffer.getChannelData(0);
const downsampled = downsampleTo16k(inData, captureAudioCtx.sampleRate);
if (micGain !== 1.0) {
for (let i = 0; i < downsampled.length; i++) downsampled[i] *= micGain;
}
localWaveWrite = pushWaveSamples(localWaveBuffer, localWaveWrite, downsampled);
const pcm16 = floatToInt16(downsampled);
@@ -319,6 +335,10 @@ function stopMicCapture() {
playProcessorNode.onaudioprocess = null;
playProcessorNode = null;
}
if (playGainNode) {
playGainNode.disconnect();
playGainNode = null;
}
if (playAudioCtx) {
playAudioCtx.close().catch(() => {});
playAudioCtx = null;
@@ -437,6 +457,7 @@ async function loadCapabilities() {
}
const sources = Array.isArray(caps.sources) && caps.sources.length > 0 ? caps.sources : ["default"];
if (!sources.includes("system")) sources.push("system");
const defaultSource = caps.defaultSource && sources.includes(caps.defaultSource) ? caps.defaultSource : sources[0];
if (sourceSelect) {
@@ -468,6 +489,16 @@ muteRemoteBtn?.addEventListener("click", () => {
});
window.addEventListener("beforeunload", disconnectVoice);
micVolumeSlider?.addEventListener("input", () => {
micGain = parseInt(micVolumeSlider.value, 10) / 100;
if (micVolumeLabel) micVolumeLabel.textContent = micVolumeSlider.value + "%";
});
remoteVolumeSlider?.addEventListener("input", () => {
remoteGain = parseInt(remoteVolumeSlider.value, 10) / 100;
if (playGainNode) playGainNode.gain.value = remoteGain;
if (remoteVolumeLabel) remoteVolumeLabel.textContent = remoteVolumeSlider.value + "%";
});
updateMuteButtons();
drawWave(localWaveCanvas, localWaveBuffer, localWaveWrite, "rgba(34, 211, 238, 0.95)");
drawWave(remoteWaveCanvas, remoteWaveBuffer, remoteWaveWrite, "rgba(16, 185, 129, 0.95)");
@@ -478,3 +509,207 @@ checkFeatureAccess("voice", clientId).then(ok => {
connectBtn.disabled = true;
}
});
/* =====================================================================
Desktop Audio – independent listen-only stream (system/loopback)
===================================================================== */
const daConnectBtn = document.getElementById("da-connect-btn");
const daDisconnectBtn = document.getElementById("da-disconnect-btn");
const daStatusPill = document.getElementById("da-status-pill");
const daWaveCanvas = document.getElementById("da-wave");
const daVolumeSlider = document.getElementById("da-volume");
const daVolumeLabel = document.getElementById("da-volume-label");
let daWs = null;
let daGain = 1.0;
let daPlayAudioCtx = null;
let daPlayProcessorNode = null;
let daPlayGainNode = null;
let daPlaybackChunks = [];
let daPlaybackChunkReadOffset = 0;
let daLevelLoop = false;
const DA_WAVE_SIZE = 2048;
const daWaveBuffer = new Float32Array(DA_WAVE_SIZE);
let daWaveWrite = 0;
function daSetStatus(text, cls) {
if (!daStatusPill) return;
daStatusPill.className =
"inline-flex items-center gap-2 px-3 py-1.5 rounded-full border " + cls;
daStatusPill.innerHTML = `<i class="fa-solid fa-circle"></i><span>${text}</span>`;
}
function daClearPlayback() {
daPlaybackChunks = [];
daPlaybackChunkReadOffset = 0;
}
function daStartLevelLoop() {
if (daLevelLoop) return;
daLevelLoop = true;
const tick = () => {
if (!daLevelLoop) return;
drawWave(daWaveCanvas, daWaveBuffer, daWaveWrite, "rgba(251, 191, 36, 0.95)");
requestAnimationFrame(tick);
};
requestAnimationFrame(tick);
}
function daStopLevelLoop() {
daLevelLoop = false;
daWaveBuffer.fill(0);
daWaveWrite = 0;
drawWave(daWaveCanvas, daWaveBuffer, daWaveWrite, "rgba(251, 191, 36, 0.95)");
}
function daInitPlaybackEngine() {
if (!daPlayAudioCtx) {
daPlayAudioCtx = new AudioContext({ sampleRate: 16000, latencyHint: "interactive" });
}
if (!daPlayGainNode) {
daPlayGainNode = daPlayAudioCtx.createGain();
daPlayGainNode.gain.value = daGain;
daPlayGainNode.connect(daPlayAudioCtx.destination);
}
if (!daPlayProcessorNode) {
daPlayProcessorNode = daPlayAudioCtx.createScriptProcessor(PLAYBACK_FRAME_SIZE, 1, 1);
daPlayProcessorNode.onaudioprocess = (event) => {
const out = event.outputBuffer.getChannelData(0);
out.fill(0);
let writeIndex = 0;
while (writeIndex < out.length && daPlaybackChunks.length > 0) {
const head = daPlaybackChunks[0];
const remaining = head.length - daPlaybackChunkReadOffset;
if (remaining <= 0) {
daPlaybackChunks.shift();
daPlaybackChunkReadOffset = 0;
continue;
}
const needed = out.length - writeIndex;
const take = Math.min(needed, remaining);
out.set(head.subarray(daPlaybackChunkReadOffset, daPlaybackChunkReadOffset + take), writeIndex);
writeIndex += take;
daPlaybackChunkReadOffset += take;
if (daPlaybackChunkReadOffset >= head.length) {
daPlaybackChunks.shift();
daPlaybackChunkReadOffset = 0;
}
}
};
daPlayProcessorNode.connect(daPlayGainNode);
}
}
function daAppendPcm(binary) {
if (!daPlayAudioCtx) daInitPlaybackEngine();
const samples = Math.floor(binary.byteLength / 2);
if (samples <= 0) return;
const src = new Int16Array(binary.buffer, binary.byteOffset, samples);
const chunk = resampleInt16ToFloat32(src, UPLINK_SAMPLE_RATE, daPlayAudioCtx?.sampleRate || UPLINK_SAMPLE_RATE);
if (chunk.length === 0) return;
daPlaybackChunks.push(chunk);
daWaveWrite = pushWaveSamples(daWaveBuffer, daWaveWrite, chunk);
const sampleRate = daPlayAudioCtx?.sampleRate || UPLINK_SAMPLE_RATE;
const maxSamples = Math.max(PLAYBACK_FRAME_SIZE, Math.round(sampleRate * (MAX_PLAYBACK_BUFFER_MS / 1000)));
let buffered = -daPlaybackChunkReadOffset;
for (const c of daPlaybackChunks) buffered += c.length;
while (buffered > maxSamples && daPlaybackChunks.length > 0) {
const dropped = daPlaybackChunks.shift();
buffered -= dropped?.length || 0;
daPlaybackChunkReadOffset = 0;
}
}
function daCleanup() {
if (daPlayProcessorNode) {
daPlayProcessorNode.disconnect();
daPlayProcessorNode.onaudioprocess = null;
daPlayProcessorNode = null;
}
if (daPlayGainNode) {
daPlayGainNode.disconnect();
daPlayGainNode = null;
}
if (daPlayAudioCtx) {
daPlayAudioCtx.close().catch(() => {});
daPlayAudioCtx = null;
}
daStopLevelLoop();
daClearPlayback();
}
function daConnect() {
if (daWs && daWs.readyState === WebSocket.OPEN) return;
daClearPlayback();
daStartLevelLoop();
// Create AudioContext inside click handler so the browser trusts the user gesture.
daInitPlaybackEngine();
if (daPlayAudioCtx?.state === "suspended") daPlayAudioCtx.resume();
const proto = window.location.protocol === "https:" ? "wss" : "ws";
daWs = new WebSocket(`${proto}://${window.location.host}/api/clients/${encodeURIComponent(clientId)}/desktop-audio/ws`);
daWs.binaryType = "arraybuffer";
daWs.onopen = () => {
daSetStatus("Connecting", "border-blue-700 bg-blue-900/40 text-blue-100");
daConnectBtn.disabled = true;
daDisconnectBtn.disabled = false;
daWs.send(JSON.stringify({ type: "start" }));
};
daWs.onmessage = (ev) => {
if (typeof ev.data === "string") {
try {
const msg = JSON.parse(ev.data);
console.log("[desktop-audio] status message:", msg);
if (msg?.type === "status") {
if (msg.status === "offline") {
daSetStatus("Client Offline", "border-amber-700 bg-amber-900/30 text-amber-200");
} else if (msg.status === "connected") {
daSetStatus("Listening", "border-emerald-700 bg-emerald-900/40 text-emerald-100");
} else if (msg.status === "error") {
daSetStatus("Error", "border-red-700 bg-red-900/40 text-red-100");
}
}
} catch {}
return;
}
const bytes = new Uint8Array(ev.data);
if (bytes.byteLength > 1) daAppendPcm(bytes);
};
daWs.onclose = () => {
daCleanup();
daConnectBtn.disabled = false;
daDisconnectBtn.disabled = true;
daSetStatus("Disconnected", "border-slate-700 bg-slate-800 text-slate-300");
};
daWs.onerror = () => {
daSetStatus("Error", "border-red-700 bg-red-900/40 text-red-100");
};
}
function daDisconnect() {
daCleanup();
if (daWs) {
try { daWs.send(JSON.stringify({ type: "stop" })); } catch {}
try { daWs.close(); } catch {}
daWs = null;
}
daConnectBtn.disabled = false;
daDisconnectBtn.disabled = true;
daSetStatus("Disconnected", "border-slate-700 bg-slate-800 text-slate-300");
}
daConnectBtn?.addEventListener("click", daConnect);
daDisconnectBtn?.addEventListener("click", daDisconnect);
daVolumeSlider?.addEventListener("input", () => {
daGain = parseInt(daVolumeSlider.value, 10) / 100;
if (daPlayGainNode) daPlayGainNode.gain.value = daGain;
if (daVolumeLabel) daVolumeLabel.textContent = daVolumeSlider.value + "%";
});
window.addEventListener("beforeunload", daDisconnect);
drawWave(daWaveCanvas, daWaveBuffer, daWaveWrite, "rgba(251, 191, 36, 0.95)");
@@ -140,6 +140,9 @@
<label class="flex items-center gap-2 text-sm text-slate-200" title="Sync clipboard between your browser and the remote machine"
><input type="checkbox" id="clipboardSyncCtrl" /> Clipboard Sync</label
>
<label class="flex items-center gap-2 text-sm text-slate-200" title="Stream system audio from the remote machine"
><input type="checkbox" id="audioCtrl" /> <i class="fa-solid fa-volume-high text-slate-400"></i> Audio</label
>
<label class="flex items-center gap-2 text-sm text-slate-200" title="Smooths mouse input for steadier movement">
<span class="text-slate-400">Smoothing</span>
<input id="smoothingSlider" type="range" min="0" max="80" step="5" value="20" class="accent-emerald-400" />
+50 -2
View File
@@ -54,16 +54,64 @@
<div class="mt-6 space-y-4">
<div>
<div class="text-sm text-slate-300 mb-1">Your Mic (You -> Client)</div>
<div class="flex items-center gap-3 mb-1">
<span class="text-sm text-slate-300">Your Mic (You -> Client)</span>
<div class="flex items-center gap-2 text-xs text-slate-400 ml-auto">
<i class="fa-solid fa-microphone"></i>
<input id="mic-volume" type="range" min="0" max="200" value="100" step="1" class="w-28 accent-cyan-500 cursor-pointer" />
<span id="mic-volume-label" class="w-10 text-right">100%</span>
</div>
</div>
<canvas id="local-wave" class="mt-2 w-full h-20 rounded-lg border border-slate-700 bg-slate-900/60"></canvas>
</div>
<div>
<div class="text-sm text-slate-300 mb-1">Client Audio (Client -> You)</div>
<div class="flex items-center gap-3 mb-1">
<span class="text-sm text-slate-300">Client Audio (Client -> You)</span>
<div class="flex items-center gap-2 text-xs text-slate-400 ml-auto">
<i class="fa-solid fa-volume-high"></i>
<input id="remote-volume" type="range" min="0" max="200" value="100" step="1" class="w-28 accent-emerald-500 cursor-pointer" />
<span id="remote-volume-label" class="w-10 text-right">100%</span>
</div>
</div>
<canvas id="remote-wave" class="mt-2 w-full h-20 rounded-lg border border-slate-700 bg-slate-900/60"></canvas>
</div>
<div class="text-xs text-slate-500">Uses native Go audio capture/playback on the client agent. If unavailable, rebuild agent with CGO enabled.</div>
</div>
</div>
<div class="max-w-4xl mx-auto mt-6 rounded-2xl border border-slate-800 bg-slate-900/70 p-6 shadow-xl">
<div class="flex items-center justify-between gap-3 flex-wrap">
<div>
<h1 class="text-2xl font-semibold"><i class="fa-solid fa-volume-high text-slate-400"></i> Desktop Audio</h1>
<p class="text-slate-400">Listen to the client's system audio (speakers/loopback). Listen-only, no mic.</p>
</div>
<span id="da-status-pill" class="inline-flex items-center gap-2 px-3 py-1.5 rounded-full border border-slate-700 bg-slate-800 text-slate-300">
<i class="fa-solid fa-plug-circle-xmark"></i><span>Disconnected</span>
</span>
</div>
<div class="mt-6 grid grid-cols-1 sm:grid-cols-2 gap-4">
<button id="da-connect-btn" class="px-4 py-3 rounded-lg border border-emerald-700 bg-emerald-900/40 hover:bg-emerald-800/60 text-emerald-100 font-medium">
<i class="fa-solid fa-play"></i> Start Listening
</button>
<button id="da-disconnect-btn" class="px-4 py-3 rounded-lg border border-red-700 bg-red-900/40 hover:bg-red-800/60 text-red-100 font-medium" disabled>
<i class="fa-solid fa-stop"></i> Stop Listening
</button>
</div>
<div class="mt-4">
<div class="flex items-center gap-3 mb-1">
<span class="text-sm text-slate-300">Desktop Audio (Client Speakers -> You)</span>
<div class="flex items-center gap-2 text-xs text-slate-400 ml-auto">
<i class="fa-solid fa-volume-high"></i>
<input id="da-volume" type="range" min="0" max="200" value="100" step="1" class="w-28 accent-amber-500 cursor-pointer" />
<span id="da-volume-label" class="w-10 text-right">100%</span>
</div>
</div>
<canvas id="da-wave" class="mt-2 w-full h-20 rounded-lg border border-slate-700 bg-slate-900/60"></canvas>
</div>
<div class="mt-2 text-xs text-slate-500">Captures system/loopback audio from the client. Requires a Stereo Mix or loopback capture device enabled on the client machine.</div>
</div>
</main>
<script type="module" src="/assets/nav.js"></script>
<script type="module" src="/assets/voice.js"></script>