import { encodeMsgpack, decodeMsgpack } from "./msgpack-helpers.js";
import { checkFeatureAccess } from "./feature-gate.js";
import { WhepClient } from "./whep.js";
import { P2PClient } from "./webrtc-p2p.js";
(async function () {
const clientId = new URLSearchParams(location.search).get("clientId");
if (!clientId) {
alert("Missing clientId");
return;
}
const allowed = await checkFeatureAccess("remote_desktop", clientId);
if (!allowed) return;
const clientLabel = document.getElementById("clientLabel");
clientLabel.textContent = clientId;
const ws = new WebSocket(
(location.protocol === "https:" ? "wss://" : "ws://") +
location.host +
"/api/clients/" +
clientId +
"/rd/ws",
);
const displaySelect = document.getElementById("displaySelect");
const refreshBtn = document.getElementById("refreshDisplays");
const startBtn = document.getElementById("startBtn");
const stopBtn = document.getElementById("stopBtn");
const fullscreenBtn = document.getElementById("fullscreenBtn");
const mouseCtrl = document.getElementById("mouseCtrl");
const kbdCtrl = document.getElementById("kbdCtrl");
const cursorCtrl = document.getElementById("cursorCtrl");
const duplicationCtrl = document.getElementById("duplicationCtrl");
const resolutionSelect = document.getElementById("resolutionSelect");
const smoothingSlider = document.getElementById("smoothingSlider");
const smoothingValue = document.getElementById("smoothingValue");
const qualitySlider = document.getElementById("qualitySlider");
const qualityValue = document.getElementById("qualityValue");
const codecH264 = document.getElementById("codecH264");
const codecMode = document.getElementById("codecMode");
const canvas = document.getElementById("frameCanvas");
const canvasContainer = document.getElementById("canvasContainer");
const ctx = canvas.getContext("2d");
const agentFps = document.getElementById("agentFps");
const viewerFps = document.getElementById("viewerFps");
const inputLatency = document.getElementById("inputLatency");
const statusEl = document.getElementById("streamStatus");
const clipboardSyncCtrl = document.getElementById("clipboardSyncCtrl");
const audioCtrl = document.getElementById("audioCtrl");
const webrtcMode = document.getElementById("webrtcMode");
const webrtcVideo = document.getElementById("webrtcVideo");
let whepClient = null;
let p2pClient = null;
let webrtcActive = false;
function getWebrtcMode() {
return webrtcMode ? String(webrtcMode.value || "off") : "off";
}
ws.binaryType = "arraybuffer";
let activeClientId = clientId;
let renderCount = 0;
let renderWindowStart = performance.now();
let lastFrameAt = 0;
let desiredStreaming = false;
let streamState = "connecting";
let frameWatchTimer = null;
let offlineTimer = null;
let frameWidth = 0;
let frameHeight = 0;
let latencyAvg = null;
let smoothingPct = 20;
let smoothPoint = null;
let pendingMove = null;
let moveTimer = null;
let frameDecodeBusy = false;
let pendingFrame = null;
let videoDecoder = null;
let h264TimestampUs = 0;
const codecPrefKey = "rdCodecPreferH264";
let prefersH264 = typeof VideoDecoder === "function";
let h264LowFpsStreak = 0;
let h264FirstFrameAt = 0;
let h264FramesSeen = 0;
let h264KeyframeErrorStreak = 0;
let h264RecoveryAttempts = 0;
let h264LastDecodeWarnAt = 0;
const H264_LOW_FPS_THRESHOLD = 6;
const H264_FALLBACK_WARMUP_MS = 10000;
const H264_MIN_FRAMES_BEFORE_FALLBACK = 120;
const H264_LOW_FPS_STREAK_LIMIT = 120;
const H264_KEYFRAME_ERROR_RESTART_THRESHOLD = 24;
const H264_MAX_RECOVERY_ATTEMPTS = 1;
const H264_DECODE_WARN_THROTTLE_MS = 2000;
const mouseMoveIntervalMs = 33;
const inputBackpressureBytes = 256 * 1024;
let lastMoveSentAt = 0;
let clipboardSyncTimer = null;
let lastClipboardText = "";
let clipboardSyncActive = false;
let elevationPending = false;
function resetH264RuntimeState() {
h264TimestampUs = 0;
h264LowFpsStreak = 0;
h264FirstFrameAt = 0;
h264FramesSeen = 0;
h264KeyframeErrorStreak = 0;
}
/* ── Remote Desktop Audio (system audio from client) ── */
const AUDIO_SAMPLE_RATE = 16000;
const AUDIO_PLAYBACK_FRAME = 512;
const AUDIO_MAX_BUFFER_MS = 120;
let audioWs = null;
let audioPlayCtx = null;
let audioProcessorNode = null;
let audioChunks = [];
let audioChunkOffset = 0;
function audioResampleInt16ToFloat32(srcInt16, srcRate, dstRate) {
if (!srcInt16 || srcInt16.length === 0) return new Float32Array(0);
if (srcRate === dstRate) {
const out = new Float32Array(srcInt16.length);
for (let i = 0; i < srcInt16.length; i++) out[i] = srcInt16[i] / 0x8000;
return out;
}
const outLength = Math.max(1, Math.round((srcInt16.length * dstRate) / srcRate));
const out = new Float32Array(outLength);
const step = srcRate / dstRate;
for (let i = 0; i < outLength; i++) {
const srcPos = i * step;
const i0 = Math.floor(srcPos);
const i1 = Math.min(i0 + 1, srcInt16.length - 1);
const frac = srcPos - i0;
out[i] = (srcInt16[i0] * (1 - frac) + srcInt16[i1] * frac) / 0x8000;
}
return out;
}
function initAudioPlayback() {
if (!audioPlayCtx) {
audioPlayCtx = new AudioContext({ sampleRate: AUDIO_SAMPLE_RATE, latencyHint: "interactive" });
}
if (!audioProcessorNode) {
audioProcessorNode = audioPlayCtx.createScriptProcessor(AUDIO_PLAYBACK_FRAME, 1, 1);
audioProcessorNode.onaudioprocess = function (event) {
const out = event.outputBuffer.getChannelData(0);
out.fill(0);
let writeIndex = 0;
while (writeIndex < out.length && audioChunks.length > 0) {
const head = audioChunks[0];
const remaining = head.length - audioChunkOffset;
if (remaining <= 0) { audioChunks.shift(); audioChunkOffset = 0; continue; }
const take = Math.min(out.length - writeIndex, remaining);
out.set(head.subarray(audioChunkOffset, audioChunkOffset + take), writeIndex);
writeIndex += take;
audioChunkOffset += take;
if (audioChunkOffset >= head.length) { audioChunks.shift(); audioChunkOffset = 0; }
}
};
audioProcessorNode.connect(audioPlayCtx.destination);
}
}
function appendAudioPcm(binary) {
if (!audioPlayCtx) initAudioPlayback();
const samples = Math.floor(binary.byteLength / 2);
if (samples <= 0) return;
const src = new Int16Array(binary.buffer, binary.byteOffset, samples);
const chunk = audioResampleInt16ToFloat32(src, AUDIO_SAMPLE_RATE, audioPlayCtx?.sampleRate || AUDIO_SAMPLE_RATE);
if (chunk.length === 0) return;
audioChunks.push(chunk);
let buffered = -audioChunkOffset;
for (const c of audioChunks) buffered += c.length;
const rate = audioPlayCtx?.sampleRate || AUDIO_SAMPLE_RATE;
const max = Math.max(AUDIO_PLAYBACK_FRAME, Math.round(rate * (AUDIO_MAX_BUFFER_MS / 1000)));
while (buffered > max && audioChunks.length > 0) {
const dropped = audioChunks.shift();
buffered -= dropped?.length || 0;
audioChunkOffset = 0;
}
}
function connectAudio() {
if (audioWs && audioWs.readyState === WebSocket.OPEN) return;
// Create AudioContext in click/change handler so the browser trusts the user gesture.
initAudioPlayback();
if (audioPlayCtx?.state === "suspended") audioPlayCtx.resume();
const proto = location.protocol === "https:" ? "wss:" : "ws:";
audioWs = new WebSocket(proto + "//" + location.host + "/api/clients/" + encodeURIComponent(clientId) + "/desktop-audio/ws");
audioWs.binaryType = "arraybuffer";
audioWs.onopen = function () {
audioWs.send(JSON.stringify({ type: "start", source: "system" }));
};
audioWs.onmessage = function (ev) {
if (typeof ev.data === "string") return;
const bytes = new Uint8Array(ev.data);
if (bytes.byteLength > 1) appendAudioPcm(bytes);
};
audioWs.onclose = function () { cleanupAudio(false); };
audioWs.onerror = function () {};
}
function disconnectAudio() {
if (audioWs) {
try { audioWs.send(JSON.stringify({ type: "stop" })); } catch {}
try { audioWs.close(); } catch {}
audioWs = null;
}
cleanupAudio(true);
}
function cleanupAudio(uncheckBox) {
if (audioProcessorNode) {
audioProcessorNode.disconnect();
audioProcessorNode.onaudioprocess = null;
audioProcessorNode = null;
}
if (audioPlayCtx) {
audioPlayCtx.close().catch(function () {});
audioPlayCtx = null;
}
audioChunks = [];
audioChunkOffset = 0;
audioWs = null;
if (uncheckBox && audioCtrl) audioCtrl.checked = false;
}
function resetH264SessionState() {
resetH264RuntimeState();
h264RecoveryAttempts = 0;
h264LastDecodeWarnAt = 0;
}
const storedCodecPref = localStorage.getItem(codecPrefKey);
if (storedCodecPref === "0") {
prefersH264 = false;
} else if (storedCodecPref === "1") {
prefersH264 = typeof VideoDecoder === "function";
}
if (codecH264) {
codecH264.checked = prefersH264;
codecH264.disabled = typeof VideoDecoder !== "function";
}
function setCodecModeLabel(mode, detail) {
if (!codecMode) return;
const suffix = detail ? ` (${detail})` : "";
codecMode.textContent = `Codec: ${String(mode || "auto").toUpperCase()}${suffix}`;
}
setCodecModeLabel(prefersH264 ? "h264" : "jpeg", "preferred");
setStreamState("connecting", "Connecting");
function updateFpsDisplay(agentValue) {
if (agentValue !== undefined && agentValue !== null && agentFps) {
agentFps.textContent = String(agentValue);
}
const now = performance.now();
renderCount += 1;
const elapsed = now - renderWindowStart;
if (elapsed >= 1000 && viewerFps) {
const fps = Math.round((renderCount * 1000) / elapsed);
viewerFps.textContent = String(fps);
renderCount = 0;
renderWindowStart = now;
}
}
function setStreamState(state, text) {
streamState = state;
if (statusEl) {
const icons = {
connecting: '',
starting: '',
stopping: '',
streaming: '',
idle: '',
stalled: '',
offline: '',
disconnected: '',
error: '',
};
const label = text ||
(state === "streaming" ? "Streaming" :
state === "starting" ? "Starting" :
state === "stopping" ? "Stopping" :
state === "offline" ? "Client offline" :
state === "disconnected" ? "Disconnected" :
state === "stalled" ? "No frames" :
state === "idle" ? "Stopped" :
"Connecting");
statusEl.innerHTML = `${icons[state] || icons.idle} ${label}`;
const base = "inline-flex items-center gap-2 px-3 py-2 rounded-full border text-sm";
const styles = {
streaming: "bg-emerald-900/40 text-emerald-100 border-emerald-700/70",
starting: "bg-sky-900/40 text-sky-100 border-sky-700/70",
stopping: "bg-amber-900/40 text-amber-100 border-amber-700/70",
stalled: "bg-amber-900/40 text-amber-100 border-amber-700/70",
offline: "bg-rose-900/40 text-rose-100 border-rose-700/70",
error: "bg-rose-900/40 text-rose-100 border-rose-700/70",
disconnected: "bg-slate-800 text-slate-300 border-slate-700",
idle: "bg-slate-800 text-slate-300 border-slate-700",
connecting: "bg-slate-800 text-slate-300 border-slate-700",
};
statusEl.className = `${base} ${styles[state] || styles.idle}`;
}
if (canvasContainer) {
canvasContainer.dataset.streamState = state;
}
if (state === "idle" || state === "offline" || state === "disconnected" || state === "error") {
if (agentFps) agentFps.textContent = "--";
if (viewerFps) viewerFps.textContent = "--";
renderCount = 0;
renderWindowStart = performance.now();
}
updateControls();
checkClipboardSync();
}
function updateControls() {
const wsOpen = ws.readyState === WebSocket.OPEN;
const isStarting = streamState === "starting";
const isStreaming = streamState === "streaming";
const isStopping = streamState === "stopping";
const isStalled = streamState === "stalled";
const isBlocked = streamState === "offline" || streamState === "disconnected" || streamState === "error";
if (startBtn) {
startBtn.disabled = !wsOpen || isStarting || isStreaming || isStopping || isBlocked;
}
if (stopBtn) {
stopBtn.disabled = !wsOpen || (!isStarting && !isStreaming && !isStopping && !isStalled);
}
}
function startClipboardSync() {
if (clipboardSyncActive) return;
clipboardSyncActive = true;
lastClipboardText = "";
sendCmd("clipboard_sync_start", {});
clipboardSyncTimer = setInterval(async () => {
if (!clipboardSyncCtrl || !clipboardSyncCtrl.checked || streamState !== "streaming") {
stopClipboardSync();
return;
}
try {
const text = await navigator.clipboard.readText();
if (text && text !== lastClipboardText) {
lastClipboardText = text;
sendCmd("clipboard_sync", { text });
}
} catch {}
}, 1500);
}
function stopClipboardSync() {
if (!clipboardSyncActive) return;
clipboardSyncActive = false;
if (clipboardSyncTimer) {
clearInterval(clipboardSyncTimer);
clipboardSyncTimer = null;
}
sendCmd("clipboard_sync_stop", {});
}
function checkClipboardSync() {
const shouldSync = clipboardSyncCtrl && clipboardSyncCtrl.checked && streamState === "streaming";
if (shouldSync && !clipboardSyncActive) {
startClipboardSync();
} else if (!shouldSync && clipboardSyncActive) {
stopClipboardSync();
}
}
if (clipboardSyncCtrl) {
clipboardSyncCtrl.addEventListener("change", checkClipboardSync);
}
function updateLatency(ms) {
if (ms < 0 || !Number.isFinite(ms)) return;
latencyAvg = latencyAvg == null ? ms : latencyAvg * 0.8 + ms * 0.2;
if (inputLatency) {
inputLatency.textContent = `${Math.round(latencyAvg)} ms`;
}
}
function clearOfflineTimer() {
if (offlineTimer) {
clearTimeout(offlineTimer);
offlineTimer = null;
}
}
function scheduleOffline(reason) {
clearOfflineTimer();
setStreamState("connecting", "Reconnecting");
offlineTimer = setTimeout(() => {
const now = performance.now();
if (!lastFrameAt || now - lastFrameAt > 3000) {
if (elevationPending) return;
desiredStreaming = false;
setStreamState("offline", reason || "Client offline");
}
}, 3000);
}
function handleStatus(msg) {
if (!msg || msg.type !== "status" || !msg.status) return;
if (msg.status === "offline") {
scheduleOffline(msg.reason);
return;
}
if (msg.status === "permissions_denied") {
clearOfflineTimer();
desiredStreaming = false;
const missing = Array.isArray(msg.missing) ? msg.missing : [];
const labels = {
screenRecording: "Screen Recording",
accessibility: "Accessibility",
fullDiskAccess: "Full Disk Access",
};
const list = missing.map(k => labels[k] || k).join(", ");
setStreamState("error", `macOS permissions required: ${list}`);
showElevateOffer(missing);
return;
}
if (msg.status === "connecting") {
clearOfflineTimer();
setStreamState("connecting", "Connecting");
return;
}
if (msg.status === "online") {
clearOfflineTimer();
if (elevationPending) {
elevationPending = false;
desiredStreaming = true;
}
if (desiredStreaming) {
setStreamState("starting", "Reconnecting");
if (displaySelect && displaySelect.value !== undefined) {
sendCmd("desktop_select_display", {
display: parseInt(displaySelect.value, 10) || 0,
});
}
sendCmd("desktop_start", {});
pushInputToggles();
pushCaptureToggles();
if (qualitySlider) pushQuality(qualitySlider.value);
pushResolution();
} else {
setStreamState("idle", "Stopped");
}
}
}
function showElevateOffer(missing) {
// Remove previous elevate banner if any
const prev = document.getElementById("rdElevateBanner");
if (prev) prev.remove();
const banner = document.createElement("div");
banner.id = "rdElevateBanner";
banner.className = "flex flex-col items-center gap-3 p-4 rounded-lg border border-amber-700/70 bg-amber-900/30 text-amber-100 text-sm";
const labels = {
screenRecording: "Screen Recording",
accessibility: "Accessibility",
fullDiskAccess: "Full Disk Access",
};
const list = missing.map(k => labels[k] || k).join(", ");
banner.innerHTML = `
macOS permissions missing: ${list}
The client needs elevated privileges to grant these permissions. Enter the user's password to elevate.
`;
// Insert banner above the canvas area
if (canvasContainer && canvasContainer.parentNode) {
canvasContainer.parentNode.insertBefore(banner, canvasContainer);
}
const elevateBtn = document.getElementById("rdElevateBtn");
const pwdInput = document.getElementById("rdElevatePwd");
const statusDiv = document.getElementById("rdElevateStatus");
const dismissBtn = document.getElementById("rdElevateDismiss");
if (dismissBtn) {
dismissBtn.addEventListener("click", () => banner.remove());
}
if (elevateBtn && pwdInput) {
elevateBtn.addEventListener("click", async () => {
const password = pwdInput.value.trim();
if (!password) {
pwdInput.focus();
return;
}
elevateBtn.disabled = true;
elevateBtn.innerHTML = ' Elevating...';
if (statusDiv) {
statusDiv.classList.remove("hidden");
statusDiv.textContent = "Sending elevation request...";
}
try {
const res = await fetch(`/api/clients/${encodeURIComponent(activeClientId)}/command`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ action: "elevate", password }),
});
const data = await res.json();
if (data.ok) {
if (statusDiv) {
statusDiv.textContent = "Elevation successful — client is restarting with elevated permissions. It will reconnect shortly.";
statusDiv.className = "text-xs text-emerald-400";
}
elevateBtn.textContent = "Done";
elevationPending = true;
desiredStreaming = true;
} else {
if (statusDiv) {
statusDiv.textContent = `Elevation failed: ${data.message || "Unknown error"}`;
statusDiv.className = "text-xs text-rose-400";
}
elevateBtn.disabled = false;
elevateBtn.innerHTML = ' Retry';
}
} catch (err) {
if (statusDiv) {
statusDiv.textContent = `Request failed: ${err.message}`;
statusDiv.className = "text-xs text-rose-400";
}
elevateBtn.disabled = false;
elevateBtn.innerHTML = ' Retry';
}
});
}
}
function sendCmd(type, payload) {
if (!activeClientId) {
console.warn("No active client selected");
return;
}
if (ws.readyState !== WebSocket.OPEN) {
return;
}
const msg = { type, ...payload };
console.debug("rd: send", msg);
ws.send(encodeMsgpack(msg));
}
function setWebrtcViewActive(active) {
webrtcActive = !!active;
if (canvas) canvas.style.display = active ? "none" : "block";
if (webrtcVideo) webrtcVideo.style.display = active ? "block" : "none";
}
function onWebrtcState(label, s) {
console.debug(`webrtc[${label}]: state`, s);
if (s === "connected") {
setStreamState("streaming", `Streaming (${label})`);
} else if (s === "failed" || s === "disconnected") {
setWebrtcViewActive(false);
}
}
// The canvas path counts WS frames via markFrameReceived + updateFpsDisplay.
// WebRTC bypasses WS entirely, so we tap the