test RTP timestamps

2026-08-05 12:06:13 -05:00
parent c3bcc2cdcc
commit 49604b8b53
14 changed files with 430 additions and 68 deletions
@@ -152,6 +152,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./generated/s
  let frameWidth = 0;
  let frameHeight = 0;
  let latencyAvg = null;
+  let directInputActive = false;
  let smoothingPct = 0;
  let smoothPoint = null;
  let pendingMove = null;
@@ -1173,7 +1174,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./generated/s
    if (ms < 0 || !Number.isFinite(ms)) return;
    latencyAvg = latencyAvg == null ? ms : latencyAvg * 0.8 + ms * 0.2;
    if (inputLatency) {
-      inputLatency.textContent = `${Math.round(latencyAvg)} ms`;
+      inputLatency.textContent = directInputActive ? "P2P direct" : `${Math.round(latencyAvg)} ms`;
    }
  }

@@ -1286,7 +1287,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./generated/s
    if (fsHudFps) {
      fsHudFps.textContent = `${snapshot.fps == null ? "--" : Math.round(snapshot.fps)} → ${snapshot.viewerRate == null ? "--" : Math.round(snapshot.viewerRate)} FPS`;
    }
-    if (fsHudLatency) fsHudLatency.textContent = `${msText(latencyAvg)} input`;
+    if (fsHudLatency) fsHudLatency.textContent = directInputActive ? "P2P direct input" : `${msText(latencyAvg)} input`;
    if (fsHudNetwork) {
      const networkParts = [];
      if (snapshot.bitrate != null) networkParts.push(`${snapshot.bitrate.toFixed(1)} Mbps`);
@@ -1395,14 +1396,15 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./generated/s
    } else if ((finite(media.lossPercent) ?? 0) > 3 || (finite(network.rttMs) ?? 0) > 150 || (finite(media.jitterMs) ?? 0) > 35) {
      summary = "Network conditions are causing delay";
      severity = "bad";
+    } else if ((finite(media.jitterBufferMs) ?? 0) > Math.max(80, frameBudget * 4)) {
+      summary = `Browser playout buffer is adding ${Math.round(media.jitterBufferMs)} ms`;
+      severity = "bad";
    } else if (
-      mode === "off" &&
      fps != null &&
      viewerRate != null &&
-      viewerRate < fps * 0.8 &&
-      diagnostics.coalescedFrames > 0
+      viewerRate < fps * 0.8
    ) {
-      summary = `Display is presenting ${Math.round(viewerRate)} of ${Math.round(fps)} decoded FPS`;
+      summary = `Viewer is presenting ${Math.round(viewerRate)} of ${Math.round(fps)} FPS`;
      severity = "warn";
    } else if ((queue ?? 0) > 2 || (renderMs != null && renderMs > frameBudget)) {
      summary = "Viewer render queue is backing up";
@@ -1432,7 +1434,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./generated/s
    diagnosticEls.Queue.textContent = queue == null ? "managed by browser" : String(queue);
    diagnosticEls.Dropped.textContent = `${Math.round(dropped)} / ${diagnostics.coalescedFrames}`;
    diagnosticEls.Fps.textContent = `${fps == null ? "--" : Math.round(fps)} → ${viewerRate == null ? "--" : Math.round(viewerRate)}`;
-    diagnosticEls.Input.textContent = msText(latencyAvg);
+    diagnosticEls.Input.textContent = directInputActive ? "P2P direct" : msText(latencyAvg);
    renderFullscreenHud();
  }

@@ -1702,6 +1704,9 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./generated/s
      return;
    }
    const msg = { type, ...payload };
+    if (highFrequencyInputTypes.has(type) && getWebrtcMode() === "p2p" && p2pClient?.sendInput(msg)) {
+      return;
+    }
    if (type !== "desktop_decode_pressure" && !highFrequencyInputTypes.has(type)) {
      rdDebug("send", {
        msg,
@@ -1808,6 +1813,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./generated/s
    if (!webrtcVideo) return;
    p2pClient = new P2PClient({
      videoEl: webrtcVideo,
+      enableInput: true,
      send: (msg) => {
        if (ws.readyState === WebSocket.OPEN) {
          ws.send(encodeMsgpack(msg));
@@ -1815,6 +1821,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./generated/s
      },
      onState: (s) => onWebrtcState("WebRTC P2P", s),
      onStats: updateNetworkStats,
+      onInputState: setDirectInputActive,
    });
    try {
      await p2pClient.start();
@@ -1830,6 +1837,7 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./generated/s
  }

  async function stopAllWebrtc() {
+    setDirectInputActive(false);
    stopWebrtcFrameTicker();
    setWebrtcViewActive(false);
    const w = whepClient;
@@ -3407,6 +3415,12 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./generated/s
    return { x, y };
  }

+  function setDirectInputActive(active) {
+    directInputActive = !!active;
+    if (inputLatency) inputLatency.textContent = directInputActive ? "P2P direct" : msText(latencyAvg);
+    renderDiagnostics();
+  }
+
  function getContainedSurfaceRect(surface, sourceWidth, sourceHeight, elementRect = surface.getBoundingClientRect()) {
    if (!elementRect.width || !elementRect.height || !sourceWidth || !sourceHeight) return elementRect;
    const fit = getComputedStyle(surface).objectFit;
@@ -0,0 +1,49 @@
+import { describe, expect, test } from "bun:test";
+import { P2PClient } from "./webrtc-p2p.js";
+import { WebRTCStatsSampler } from "./webrtc-stats.js";
+
+describe("WebRTC low-latency paths", () => {
+  test("routes motion and reliable input over separate data channels", () => {
+    const client = new P2PClient({ send: () => {}, enableInput: true });
+    const motion: string[] = [];
+    const reliable: string[] = [];
+    client.motionInput = { readyState: "open", bufferedAmount: 0, send: (value: string) => motion.push(value) };
+    client.reliableInput = { readyState: "open", bufferedAmount: 0, send: (value: string) => reliable.push(value) };
+
+    client.inputReady = false;
+    expect(client.sendInput({ type: "key_down", code: "KeyA" })).toBe(false);
+    client.inputReady = true;
+    expect(client.sendInput({ type: "mouse_move", x: 10, y: 20 })).toBe(true);
+    expect(client.sendInput({ type: "key_down", code: "KeyA" })).toBe(true);
+    expect(JSON.parse(motion[0])).toEqual({ type: "mouse_move", x: 10, y: 20 });
+    expect(JSON.parse(reliable[0])).toEqual({ type: "key_down", code: "KeyA" });
+  });
+
+  test("reports interval jitter-buffer delay instead of lifetime average", async () => {
+    let reportIndex = 0;
+    const reports = [
+      new Map([["video", {
+        id: "video", type: "inbound-rtp", kind: "video", timestamp: 1000,
+        bytesReceived: 1000, framesDecoded: 10, framesDropped: 0,
+        packetsReceived: 10, packetsLost: 0, totalDecodeTime: 0,
+        totalProcessingDelay: 0, jitterBufferDelay: 3, jitterBufferEmittedCount: 10,
+      }]]),
+      new Map([["video", {
+        id: "video", type: "inbound-rtp", kind: "video", timestamp: 2000,
+        bytesReceived: 2000, framesDecoded: 20, framesDropped: 0,
+        packetsReceived: 20, packetsLost: 0, totalDecodeTime: 0,
+        totalProcessingDelay: 0, jitterBufferDelay: 3.5, jitterBufferEmittedCount: 20,
+      }]]),
+    ];
+    const samples: any[] = [];
+    const sampler = new WebRTCStatsSampler({
+      connectionState: "connected",
+      getStats: async () => reports[Math.min(reportIndex++, reports.length - 1)],
+    }, (sample: any) => samples.push(sample));
+
+    await sampler.sample();
+    await sampler.sample();
+    expect(samples[0].video.jitterBufferMs).toBe(300);
+    expect(samples[1].video.jitterBufferMs).toBe(50);
+  });
+});
@@ -31,8 +31,13 @@ export class P2PClient {
    this.send = opts.send;
    this.onState = opts.onState || (() => {});
    this.onStats = opts.onStats || (() => {});
+    this.onInputState = opts.onInputState || (() => {});
    this.iceServers = Array.isArray(opts.iceServers) ? opts.iceServers : null;
+    this.enableInput = opts.enableInput === true;
    this.pc = null;
+    this.reliableInput = null;
+    this.motionInput = null;
+    this.inputReady = false;
    this.pendingRemoteCandidates = [];
    this.statsSampler = null;
  }
@@ -49,8 +54,26 @@ export class P2PClient {

    if (this.videoEl) pc.addTransceiver("video", { direction: "recvonly" });
    if (this.audioEl) pc.addTransceiver("audio", { direction: "recvonly" });
+    if (this.enableInput) {
+      this.reliableInput = pc.createDataChannel("overlord-input-reliable", { ordered: true });
+      this.motionInput = pc.createDataChannel("overlord-input-motion", { ordered: false, maxRetransmits: 0 });
+      this.reliableInput.onmessage = (event) => {
+        try {
+          const message = JSON.parse(String(event.data || ""));
+          if (message?.type === "input_ready" && message?.version === 1) {
+            this.inputReady = true;
+            this.onInputState(true);
+          }
+        } catch {}
+      };
+      this.reliableInput.onclose = () => {
+        this.inputReady = false;
+        this.onInputState(false);
+      };
+    }

    pc.ontrack = (ev) => {
+      this.applyLowLatencyReceiverHints();
      const stream = ev.streams[0] || new MediaStream([ev.track]);
      if (ev.track.kind === "video" && this.videoEl) {
        this.videoEl.srcObject = stream;
@@ -104,6 +127,7 @@ export class P2PClient {
  async onAnswer(sdp) {
    if (!this.pc || !sdp) return;
    await this.pc.setRemoteDescription({ type: "answer", sdp });
+    this.applyLowLatencyReceiverHints();
    // Drain any ICE candidates that arrived before the answer.
    const drain = this.pendingRemoteCandidates;
    this.pendingRemoteCandidates = [];
@@ -112,6 +136,35 @@ export class P2PClient {
    }
  }

+  applyLowLatencyReceiverHints() {
+    if (!this.pc) return;
+    for (const receiver of this.pc.getReceivers()) {
+      if (receiver.track?.kind !== "video") continue;
+      try {
+        if ("playoutDelayHint" in receiver) receiver.playoutDelayHint = 0;
+        if ("jitterBufferTarget" in receiver) receiver.jitterBufferTarget = 0;
+      } catch (error) {
+        console.debug("p2p: browser rejected low-latency receiver hint", error);
+      }
+    }
+  }
+
+  sendInput(message) {
+    if (!this.enableInput || !this.inputReady || !message || typeof message.type !== "string") return false;
+    const motion = message.type === "mouse_move";
+    const channel = motion ? this.motionInput : this.reliableInput;
+    if (!channel || channel.readyState !== "open") return false;
+    // Never build a motion backlog. Reliable key/button events are tiny and
+    // must stay on one ordered path so a key-up cannot overtake a key-down.
+    if (motion && channel.bufferedAmount > 64 * 1024) return true;
+    try {
+      channel.send(JSON.stringify(message));
+      return true;
+    } catch {
+      return false;
+    }
+  }
+
  /** Add a remote ICE candidate from the agent. */
  async onRemoteCandidate(msg) {
    if (!this.pc) return;
@@ -135,6 +188,10 @@ export class P2PClient {
    this.statsSampler?.stop();
    this.statsSampler = null;
    this.pendingRemoteCandidates = [];
+    this.reliableInput = null;
+    this.motionInput = null;
+    this.inputReady = false;
+    this.onInputState(false);
    if (pc) {
      try { pc.close(); } catch {}
    }
@@ -76,6 +76,7 @@ export class WebRTCStatsSampler {
        let processingDelayMs = null;
        let framesDroppedDelta = null;
        let intervalLossPercent = null;
+        let jitterBufferMs = averageJitterBufferMs(item);
        if (previous && timestamp > previous.timestamp && Number(item.bytesReceived) >= previous.bytesReceived) {
          bitrateMbps = ((Number(item.bytesReceived) - previous.bytesReceived) * 8) /
            ((timestamp - previous.timestamp) * 1000);
@@ -88,6 +89,11 @@ export class WebRTCStatsSampler {
          }
          const droppedDelta = (Number(item.framesDropped) || 0) - previous.framesDropped;
          if (droppedDelta >= 0) framesDroppedDelta = droppedDelta;
+          const jitterDelayDelta = Number(item.jitterBufferDelay) - previous.jitterBufferDelay;
+          const jitterEmittedDelta = Number(item.jitterBufferEmittedCount) - previous.jitterBufferEmittedCount;
+          if (jitterDelayDelta >= 0 && jitterEmittedDelta > 0) {
+            jitterBufferMs = (jitterDelayDelta * 1000) / jitterEmittedDelta;
+          }
          const receivedDelta = (Number(item.packetsReceived) || 0) - previous.packetsReceived;
          const lostDelta = (Number(item.packetsLost) || 0) - previous.packetsLost;
          const packetDelta = Math.max(0, receivedDelta) + Math.max(0, lostDelta);
@@ -102,6 +108,8 @@ export class WebRTCStatsSampler {
          framesDropped: Number(item.framesDropped) || 0,
          packetsReceived: Number(item.packetsReceived) || 0,
          packetsLost: Number(item.packetsLost) || 0,
+          jitterBufferDelay: Number(item.jitterBufferDelay) || 0,
+          jitterBufferEmittedCount: Number(item.jitterBufferEmittedCount) || 0,
        });
        const packetsReceived = Number(item.packetsReceived) || 0;
        const packetsLost = Math.max(0, Number(item.packetsLost) || 0);
@@ -111,7 +119,7 @@ export class WebRTCStatsSampler {
          packetsLost,
          lossPercent: intervalLossPercent ?? (packetTotal > 0 ? (packetsLost * 100) / packetTotal : 0),
          jitterMs: Number.isFinite(item.jitter) ? item.jitter * 1000 : null,
-          jitterBufferMs: averageJitterBufferMs(item),
+          jitterBufferMs,
          decodeMs,
          processingDelayMs,
          framesDropped: Number.isFinite(item.framesDropped) ? item.framesDropped : null,