import { encodeMsgpack, decodeMsgpack } from "./msgpack-helpers.js"; import { checkFeatureAccess } from "./feature-gate.js"; import { WhepClient } from "./whep.js"; import { P2PClient } from "./webrtc-p2p.js"; (async function () { const clientId = new URLSearchParams(location.search).get("clientId"); if (!clientId) { alert("Missing clientId"); return; } const allowed = await checkFeatureAccess("remote_desktop", clientId); if (!allowed) return; const clientLabel = document.getElementById("clientLabel"); clientLabel.textContent = clientId; const ws = new WebSocket( (location.protocol === "https:" ? "wss://" : "ws://") + location.host + "/api/clients/" + clientId + "/rd/ws", ); const displaySelect = document.getElementById("displaySelect"); const refreshBtn = document.getElementById("refreshDisplays"); const startBtn = document.getElementById("startBtn"); const stopBtn = document.getElementById("stopBtn"); const fullscreenBtn = document.getElementById("fullscreenBtn"); const mouseCtrl = document.getElementById("mouseCtrl"); const kbdCtrl = document.getElementById("kbdCtrl"); const cursorCtrl = document.getElementById("cursorCtrl"); const duplicationCtrl = document.getElementById("duplicationCtrl"); const resolutionSelect = document.getElementById("resolutionSelect"); const smoothingSlider = document.getElementById("smoothingSlider"); const smoothingValue = document.getElementById("smoothingValue"); const qualitySlider = document.getElementById("qualitySlider"); const qualityValue = document.getElementById("qualityValue"); const codecH264 = document.getElementById("codecH264"); const codecMode = document.getElementById("codecMode"); const canvas = document.getElementById("frameCanvas"); const canvasContainer = document.getElementById("canvasContainer"); const ctx = canvas.getContext("2d"); const agentFps = document.getElementById("agentFps"); const viewerFps = document.getElementById("viewerFps"); const inputLatency = document.getElementById("inputLatency"); const statusEl = document.getElementById("streamStatus"); const clipboardSyncCtrl = document.getElementById("clipboardSyncCtrl"); const audioCtrl = document.getElementById("audioCtrl"); const webrtcMode = document.getElementById("webrtcMode"); const webrtcVideo = document.getElementById("webrtcVideo"); let whepClient = null; let p2pClient = null; let webrtcActive = false; function getWebrtcMode() { return webrtcMode ? String(webrtcMode.value || "off") : "off"; } ws.binaryType = "arraybuffer"; let activeClientId = clientId; let renderCount = 0; let renderWindowStart = performance.now(); let lastFrameAt = 0; let desiredStreaming = false; let streamState = "connecting"; let frameWatchTimer = null; let offlineTimer = null; let frameWidth = 0; let frameHeight = 0; let latencyAvg = null; let smoothingPct = 20; let smoothPoint = null; let pendingMove = null; let moveTimer = null; let frameDecodeBusy = false; let pendingFrame = null; let videoDecoder = null; let h264TimestampUs = 0; const codecPrefKey = "rdCodecPreferH264"; let prefersH264 = typeof VideoDecoder === "function"; let h264LowFpsStreak = 0; let h264FirstFrameAt = 0; let h264FramesSeen = 0; let h264KeyframeErrorStreak = 0; let h264RecoveryAttempts = 0; let h264LastDecodeWarnAt = 0; const H264_LOW_FPS_THRESHOLD = 6; const H264_FALLBACK_WARMUP_MS = 10000; const H264_MIN_FRAMES_BEFORE_FALLBACK = 120; const H264_LOW_FPS_STREAK_LIMIT = 120; const H264_KEYFRAME_ERROR_RESTART_THRESHOLD = 24; const H264_MAX_RECOVERY_ATTEMPTS = 1; const H264_DECODE_WARN_THROTTLE_MS = 2000; const mouseMoveIntervalMs = 33; const inputBackpressureBytes = 256 * 1024; let lastMoveSentAt = 0; let clipboardSyncTimer = null; let lastClipboardText = ""; let clipboardSyncActive = false; let elevationPending = false; function resetH264RuntimeState() { h264TimestampUs = 0; h264LowFpsStreak = 0; h264FirstFrameAt = 0; h264FramesSeen = 0; h264KeyframeErrorStreak = 0; } /* ── Remote Desktop Audio (system audio from client) ── */ const AUDIO_SAMPLE_RATE = 16000; const AUDIO_PLAYBACK_FRAME = 512; const AUDIO_MAX_BUFFER_MS = 120; let audioWs = null; let audioPlayCtx = null; let audioProcessorNode = null; let audioChunks = []; let audioChunkOffset = 0; function audioResampleInt16ToFloat32(srcInt16, srcRate, dstRate) { if (!srcInt16 || srcInt16.length === 0) return new Float32Array(0); if (srcRate === dstRate) { const out = new Float32Array(srcInt16.length); for (let i = 0; i < srcInt16.length; i++) out[i] = srcInt16[i] / 0x8000; return out; } const outLength = Math.max(1, Math.round((srcInt16.length * dstRate) / srcRate)); const out = new Float32Array(outLength); const step = srcRate / dstRate; for (let i = 0; i < outLength; i++) { const srcPos = i * step; const i0 = Math.floor(srcPos); const i1 = Math.min(i0 + 1, srcInt16.length - 1); const frac = srcPos - i0; out[i] = (srcInt16[i0] * (1 - frac) + srcInt16[i1] * frac) / 0x8000; } return out; } function initAudioPlayback() { if (!audioPlayCtx) { audioPlayCtx = new AudioContext({ sampleRate: AUDIO_SAMPLE_RATE, latencyHint: "interactive" }); } if (!audioProcessorNode) { audioProcessorNode = audioPlayCtx.createScriptProcessor(AUDIO_PLAYBACK_FRAME, 1, 1); audioProcessorNode.onaudioprocess = function (event) { const out = event.outputBuffer.getChannelData(0); out.fill(0); let writeIndex = 0; while (writeIndex < out.length && audioChunks.length > 0) { const head = audioChunks[0]; const remaining = head.length - audioChunkOffset; if (remaining <= 0) { audioChunks.shift(); audioChunkOffset = 0; continue; } const take = Math.min(out.length - writeIndex, remaining); out.set(head.subarray(audioChunkOffset, audioChunkOffset + take), writeIndex); writeIndex += take; audioChunkOffset += take; if (audioChunkOffset >= head.length) { audioChunks.shift(); audioChunkOffset = 0; } } }; audioProcessorNode.connect(audioPlayCtx.destination); } } function appendAudioPcm(binary) { if (!audioPlayCtx) initAudioPlayback(); const samples = Math.floor(binary.byteLength / 2); if (samples <= 0) return; const src = new Int16Array(binary.buffer, binary.byteOffset, samples); const chunk = audioResampleInt16ToFloat32(src, AUDIO_SAMPLE_RATE, audioPlayCtx?.sampleRate || AUDIO_SAMPLE_RATE); if (chunk.length === 0) return; audioChunks.push(chunk); let buffered = -audioChunkOffset; for (const c of audioChunks) buffered += c.length; const rate = audioPlayCtx?.sampleRate || AUDIO_SAMPLE_RATE; const max = Math.max(AUDIO_PLAYBACK_FRAME, Math.round(rate * (AUDIO_MAX_BUFFER_MS / 1000))); while (buffered > max && audioChunks.length > 0) { const dropped = audioChunks.shift(); buffered -= dropped?.length || 0; audioChunkOffset = 0; } } function connectAudio() { if (audioWs && audioWs.readyState === WebSocket.OPEN) return; // Create AudioContext in click/change handler so the browser trusts the user gesture. initAudioPlayback(); if (audioPlayCtx?.state === "suspended") audioPlayCtx.resume(); const proto = location.protocol === "https:" ? "wss:" : "ws:"; audioWs = new WebSocket(proto + "//" + location.host + "/api/clients/" + encodeURIComponent(clientId) + "/desktop-audio/ws"); audioWs.binaryType = "arraybuffer"; audioWs.onopen = function () { audioWs.send(JSON.stringify({ type: "start", source: "system" })); }; audioWs.onmessage = function (ev) { if (typeof ev.data === "string") return; const bytes = new Uint8Array(ev.data); if (bytes.byteLength > 1) appendAudioPcm(bytes); }; audioWs.onclose = function () { cleanupAudio(false); }; audioWs.onerror = function () {}; } function disconnectAudio() { if (audioWs) { try { audioWs.send(JSON.stringify({ type: "stop" })); } catch {} try { audioWs.close(); } catch {} audioWs = null; } cleanupAudio(true); } function cleanupAudio(uncheckBox) { if (audioProcessorNode) { audioProcessorNode.disconnect(); audioProcessorNode.onaudioprocess = null; audioProcessorNode = null; } if (audioPlayCtx) { audioPlayCtx.close().catch(function () {}); audioPlayCtx = null; } audioChunks = []; audioChunkOffset = 0; audioWs = null; if (uncheckBox && audioCtrl) audioCtrl.checked = false; } function resetH264SessionState() { resetH264RuntimeState(); h264RecoveryAttempts = 0; h264LastDecodeWarnAt = 0; } const storedCodecPref = localStorage.getItem(codecPrefKey); if (storedCodecPref === "0") { prefersH264 = false; } else if (storedCodecPref === "1") { prefersH264 = typeof VideoDecoder === "function"; } if (codecH264) { codecH264.checked = prefersH264; codecH264.disabled = typeof VideoDecoder !== "function"; } function setCodecModeLabel(mode, detail) { if (!codecMode) return; const suffix = detail ? ` (${detail})` : ""; codecMode.textContent = `Codec: ${String(mode || "auto").toUpperCase()}${suffix}`; } setCodecModeLabel(prefersH264 ? "h264" : "jpeg", "preferred"); setStreamState("connecting", "Connecting"); function updateFpsDisplay(agentValue) { if (agentValue !== undefined && agentValue !== null && agentFps) { agentFps.textContent = String(agentValue); } const now = performance.now(); renderCount += 1; const elapsed = now - renderWindowStart; if (elapsed >= 1000 && viewerFps) { const fps = Math.round((renderCount * 1000) / elapsed); viewerFps.textContent = String(fps); renderCount = 0; renderWindowStart = now; } } function setStreamState(state, text) { streamState = state; if (statusEl) { const icons = { connecting: '', starting: '', stopping: '', streaming: '', idle: '', stalled: '', offline: '', disconnected: '', error: '', }; const label = text || (state === "streaming" ? "Streaming" : state === "starting" ? "Starting" : state === "stopping" ? "Stopping" : state === "offline" ? "Client offline" : state === "disconnected" ? "Disconnected" : state === "stalled" ? "No frames" : state === "idle" ? "Stopped" : "Connecting"); statusEl.innerHTML = `${icons[state] || icons.idle} ${label}`; const base = "inline-flex items-center gap-2 px-3 py-2 rounded-full border text-sm"; const styles = { streaming: "bg-emerald-900/40 text-emerald-100 border-emerald-700/70", starting: "bg-sky-900/40 text-sky-100 border-sky-700/70", stopping: "bg-amber-900/40 text-amber-100 border-amber-700/70", stalled: "bg-amber-900/40 text-amber-100 border-amber-700/70", offline: "bg-rose-900/40 text-rose-100 border-rose-700/70", error: "bg-rose-900/40 text-rose-100 border-rose-700/70", disconnected: "bg-slate-800 text-slate-300 border-slate-700", idle: "bg-slate-800 text-slate-300 border-slate-700", connecting: "bg-slate-800 text-slate-300 border-slate-700", }; statusEl.className = `${base} ${styles[state] || styles.idle}`; } if (canvasContainer) { canvasContainer.dataset.streamState = state; } if (state === "idle" || state === "offline" || state === "disconnected" || state === "error") { if (agentFps) agentFps.textContent = "--"; if (viewerFps) viewerFps.textContent = "--"; renderCount = 0; renderWindowStart = performance.now(); } updateControls(); checkClipboardSync(); } function updateControls() { const wsOpen = ws.readyState === WebSocket.OPEN; const isStarting = streamState === "starting"; const isStreaming = streamState === "streaming"; const isStopping = streamState === "stopping"; const isStalled = streamState === "stalled"; const isBlocked = streamState === "offline" || streamState === "disconnected" || streamState === "error"; if (startBtn) { startBtn.disabled = !wsOpen || isStarting || isStreaming || isStopping || isBlocked; } if (stopBtn) { stopBtn.disabled = !wsOpen || (!isStarting && !isStreaming && !isStopping && !isStalled); } } function startClipboardSync() { if (clipboardSyncActive) return; clipboardSyncActive = true; lastClipboardText = ""; sendCmd("clipboard_sync_start", {}); clipboardSyncTimer = setInterval(async () => { if (!clipboardSyncCtrl || !clipboardSyncCtrl.checked || streamState !== "streaming") { stopClipboardSync(); return; } try { const text = await navigator.clipboard.readText(); if (text && text !== lastClipboardText) { lastClipboardText = text; sendCmd("clipboard_sync", { text }); } } catch {} }, 1500); } function stopClipboardSync() { if (!clipboardSyncActive) return; clipboardSyncActive = false; if (clipboardSyncTimer) { clearInterval(clipboardSyncTimer); clipboardSyncTimer = null; } sendCmd("clipboard_sync_stop", {}); } function checkClipboardSync() { const shouldSync = clipboardSyncCtrl && clipboardSyncCtrl.checked && streamState === "streaming"; if (shouldSync && !clipboardSyncActive) { startClipboardSync(); } else if (!shouldSync && clipboardSyncActive) { stopClipboardSync(); } } if (clipboardSyncCtrl) { clipboardSyncCtrl.addEventListener("change", checkClipboardSync); } function updateLatency(ms) { if (ms < 0 || !Number.isFinite(ms)) return; latencyAvg = latencyAvg == null ? ms : latencyAvg * 0.8 + ms * 0.2; if (inputLatency) { inputLatency.textContent = `${Math.round(latencyAvg)} ms`; } } function clearOfflineTimer() { if (offlineTimer) { clearTimeout(offlineTimer); offlineTimer = null; } } function scheduleOffline(reason) { clearOfflineTimer(); setStreamState("connecting", "Reconnecting"); offlineTimer = setTimeout(() => { const now = performance.now(); if (!lastFrameAt || now - lastFrameAt > 3000) { if (elevationPending) return; desiredStreaming = false; setStreamState("offline", reason || "Client offline"); } }, 3000); } function handleStatus(msg) { if (!msg || msg.type !== "status" || !msg.status) return; if (msg.status === "offline") { scheduleOffline(msg.reason); return; } if (msg.status === "permissions_denied") { clearOfflineTimer(); desiredStreaming = false; const missing = Array.isArray(msg.missing) ? msg.missing : []; const labels = { screenRecording: "Screen Recording", accessibility: "Accessibility", fullDiskAccess: "Full Disk Access", }; const list = missing.map(k => labels[k] || k).join(", "); setStreamState("error", `macOS permissions required: ${list}`); showElevateOffer(missing); return; } if (msg.status === "connecting") { clearOfflineTimer(); setStreamState("connecting", "Connecting"); return; } if (msg.status === "online") { clearOfflineTimer(); if (elevationPending) { elevationPending = false; desiredStreaming = true; } if (desiredStreaming) { setStreamState("starting", "Reconnecting"); if (displaySelect && displaySelect.value !== undefined) { sendCmd("desktop_select_display", { display: parseInt(displaySelect.value, 10) || 0, }); } sendCmd("desktop_start", {}); pushInputToggles(); pushCaptureToggles(); if (qualitySlider) pushQuality(qualitySlider.value); pushResolution(); } else { setStreamState("idle", "Stopped"); } } } function showElevateOffer(missing) { // Remove previous elevate banner if any const prev = document.getElementById("rdElevateBanner"); if (prev) prev.remove(); const banner = document.createElement("div"); banner.id = "rdElevateBanner"; banner.className = "flex flex-col items-center gap-3 p-4 rounded-lg border border-amber-700/70 bg-amber-900/30 text-amber-100 text-sm"; const labels = { screenRecording: "Screen Recording", accessibility: "Accessibility", fullDiskAccess: "Full Disk Access", }; const list = missing.map(k => labels[k] || k).join(", "); banner.innerHTML = `
macOS permissions missing: ${list}
The client needs elevated privileges to grant these permissions. Enter the user's password to elevate.
`; // Insert banner above the canvas area if (canvasContainer && canvasContainer.parentNode) { canvasContainer.parentNode.insertBefore(banner, canvasContainer); } const elevateBtn = document.getElementById("rdElevateBtn"); const pwdInput = document.getElementById("rdElevatePwd"); const statusDiv = document.getElementById("rdElevateStatus"); const dismissBtn = document.getElementById("rdElevateDismiss"); if (dismissBtn) { dismissBtn.addEventListener("click", () => banner.remove()); } if (elevateBtn && pwdInput) { elevateBtn.addEventListener("click", async () => { const password = pwdInput.value.trim(); if (!password) { pwdInput.focus(); return; } elevateBtn.disabled = true; elevateBtn.innerHTML = ' Elevating...'; if (statusDiv) { statusDiv.classList.remove("hidden"); statusDiv.textContent = "Sending elevation request..."; } try { const res = await fetch(`/api/clients/${encodeURIComponent(activeClientId)}/command`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ action: "elevate", password }), }); const data = await res.json(); if (data.ok) { if (statusDiv) { statusDiv.textContent = "Elevation successful — client is restarting with elevated permissions. It will reconnect shortly."; statusDiv.className = "text-xs text-emerald-400"; } elevateBtn.textContent = "Done"; elevationPending = true; desiredStreaming = true; } else { if (statusDiv) { statusDiv.textContent = `Elevation failed: ${data.message || "Unknown error"}`; statusDiv.className = "text-xs text-rose-400"; } elevateBtn.disabled = false; elevateBtn.innerHTML = ' Retry'; } } catch (err) { if (statusDiv) { statusDiv.textContent = `Request failed: ${err.message}`; statusDiv.className = "text-xs text-rose-400"; } elevateBtn.disabled = false; elevateBtn.innerHTML = ' Retry'; } }); } } function sendCmd(type, payload) { if (!activeClientId) { console.warn("No active client selected"); return; } if (ws.readyState !== WebSocket.OPEN) { return; } const msg = { type, ...payload }; console.debug("rd: send", msg); ws.send(encodeMsgpack(msg)); } function setWebrtcViewActive(active) { webrtcActive = !!active; if (canvas) canvas.style.display = active ? "none" : "block"; if (webrtcVideo) webrtcVideo.style.display = active ? "block" : "none"; } function onWebrtcState(label, s) { console.debug(`webrtc[${label}]: state`, s); if (s === "connected") { setStreamState("streaming", `Streaming (${label})`); } else if (s === "failed" || s === "disconnected") { setWebrtcViewActive(false); } } // The canvas path counts WS frames via markFrameReceived + updateFpsDisplay. // WebRTC bypasses WS entirely, so we tap the