more tests and better metrics

Overlord-Server/public/assets/metrics.js

@@ -1,5 +1,12 @@
 let clientsChart = null;
 let commandsChart = null;
+let bandwidthChart = null;
+let httpRequestsChart = null;
+let httpChart = null;
+let memoryChart = null;
+let eventLoopChart = null;
+let sessionsChart = null;
+let osChart = null;
 let countryMap = null;
 let countriesLayer = null;
 let countryCounts = {};
@@ -7,6 +14,8 @@ let maxCountryCount = 0;
 let latestByCountry = {};
 
 const GEOJSON_URL = "/vendor/geo-countries/countries.geojson";
+const MAX_CHART_POINTS = 240;
+const METRICS_POLL_INTERVAL_MS = 5000;
 
 if (typeof Chart !== "undefined") {
   Chart.defaults.color = "#cbd5e1";
@@ -14,9 +23,69 @@ if (typeof Chart !== "undefined") {
   Chart.defaults.font.family = "Inter, system-ui, sans-serif";
 }
 
+function chartTooltip() {
+  return {
+    backgroundColor: "#0f172a",
+    titleColor: "#e2e8f0",
+    bodyColor: "#cbd5e1",
+    borderColor: "#334155",
+    borderWidth: 1,
+  };
+}
+
+function lineChartOptions(extra = {}) {
+  const extraPlugins = extra.plugins || {};
+  return {
+    responsive: true,
+    maintainAspectRatio: false,
+    animation: false,
+    interaction: { mode: "index", intersect: false },
+    scales: {
+      y: {
+        beginAtZero: true,
+        ticks: { color: "#94a3b8" },
+        grid: { color: "rgba(100, 116, 139, 0.1)" },
+      },
+      x: {
+        ticks: {
+          color: "#94a3b8",
+          maxTicksLimit: 6,
+          autoSkip: true,
+          maxRotation: 0,
+        },
+        grid: { color: "rgba(100, 116, 139, 0.1)" },
+      },
+      ...(extra.scales || {}),
+    },
+    ...extra,
+    plugins: {
+      legend:
+        extraPlugins.legend ?? { display: true, labels: { boxWidth: 10, boxHeight: 10 } },
+      tooltip: chartTooltip(),
+      ...extraPlugins,
+    },
+  };
+}
+
+function makeLineChart(canvas, datasets, options = {}) {
+  if (!canvas) return null;
+  return new Chart(canvas, {
+    type: "line",
+    data: { labels: [], datasets },
+    options: lineChartOptions(options),
+  });
+}
+
 function initCharts() {
   const clientsCtx = document.getElementById("clients-chart");
   const commandsCtx = document.getElementById("commands-chart");
+  const bandwidthCtx = document.getElementById("bandwidth-chart");
+  const httpRequestsCtx = document.getElementById("http-requests-chart");
+  const httpCtx = document.getElementById("http-chart");
+  const memoryCtx = document.getElementById("memory-chart");
+  const eventLoopCtx = document.getElementById("event-loop-chart");
+  const sessionsCtx = document.getElementById("sessions-chart");
+  const osCtx = document.getElementById("os-chart");
 
   clientsChart = new Chart(clientsCtx, {
     type: "line",
@@ -119,13 +188,283 @@ function initCharts() {
       },
     },
   });
+
+  bandwidthChart = makeLineChart(
+    bandwidthCtx,
+    [
+      {
+        label: "Sent/s",
+        data: [],
+        borderColor: "rgb(251, 146, 60)",
+        backgroundColor: "rgba(251, 146, 60, 0.12)",
+        fill: true,
+        tension: 0.35,
+        pointRadius: 0,
+        borderWidth: 2,
+      },
+      {
+        label: "Received/s",
+        data: [],
+        borderColor: "rgb(56, 189, 248)",
+        backgroundColor: "rgba(56, 189, 248, 0.1)",
+        fill: true,
+        tension: 0.35,
+        pointRadius: 0,
+        borderWidth: 2,
+      },
+    ],
+    {
+      scales: {
+        y: {
+          beginAtZero: true,
+          ticks: {
+            color: "#94a3b8",
+            callback: (value) => formatBytes(Number(value)),
+          },
+          grid: { color: "rgba(100, 116, 139, 0.1)" },
+        },
+      },
+    },
+  );
+
+  httpRequestsChart = makeLineChart(
+    httpRequestsCtx,
+    [
+      {
+        label: "requests/min",
+        data: [],
+        borderColor: "rgb(52, 211, 153)",
+        backgroundColor: "rgba(52, 211, 153, 0.12)",
+        fill: true,
+        tension: 0.35,
+        pointRadius: 0,
+        borderWidth: 2,
+      },
+      {
+        label: "errors/min",
+        data: [],
+        borderColor: "rgb(248, 113, 113)",
+        backgroundColor: "rgba(248, 113, 113, 0.14)",
+        fill: false,
+        tension: 0.25,
+        pointRadius: 0,
+        borderDash: [5, 4],
+        borderWidth: 2,
+      },
+    ],
+    {
+      scales: {
+        y: {
+          beginAtZero: true,
+          ticks: { color: "#94a3b8", precision: 0 },
+          grid: { color: "rgba(100, 116, 139, 0.1)" },
+        },
+      },
+    },
+  );
+
+  httpChart = makeLineChart(
+    httpCtx,
+    [
+      {
+        label: "p99 ms",
+        data: [],
+        borderColor: "rgb(248, 113, 113)",
+        backgroundColor: "rgba(248, 113, 113, 0.08)",
+        fill: false,
+        tension: 0.35,
+        pointRadius: 0,
+        borderDash: [4, 4],
+        borderWidth: 2,
+      },
+      {
+        label: "p95 ms",
+        data: [],
+        borderColor: "rgb(244, 63, 94)",
+        backgroundColor: "rgba(244, 63, 94, 0.1)",
+        fill: true,
+        tension: 0.35,
+        pointRadius: 0,
+        borderWidth: 2,
+      },
+      {
+        label: "avg ms",
+        data: [],
+        borderColor: "rgb(251, 191, 36)",
+        backgroundColor: "rgba(251, 191, 36, 0.08)",
+        fill: false,
+        tension: 0.35,
+        pointRadius: 0,
+        borderWidth: 2,
+      },
+    ],
+    {
+      scales: {
+        y: {
+          beginAtZero: true,
+          ticks: { color: "#94a3b8", callback: (value) => `${Math.round(Number(value))} ms` },
+          grid: { color: "rgba(100, 116, 139, 0.1)" },
+        },
+      },
+    },
+  );
+
+  memoryChart = makeLineChart(
+    memoryCtx,
+    [
+      {
+        label: "Heap",
+        data: [],
+        borderColor: "rgb(34, 211, 238)",
+        backgroundColor: "rgba(34, 211, 238, 0.1)",
+        fill: true,
+        tension: 0.35,
+        pointRadius: 0,
+        yAxisID: "y",
+        borderWidth: 2,
+      },
+      {
+        label: "RSS",
+        data: [],
+        borderColor: "rgb(129, 140, 248)",
+        backgroundColor: "rgba(129, 140, 248, 0.08)",
+        fill: true,
+        tension: 0.35,
+        pointRadius: 0,
+        yAxisID: "y",
+        borderWidth: 2,
+      },
+      {
+        label: "System %",
+        data: [],
+        borderColor: "rgb(52, 211, 153)",
+        backgroundColor: "rgba(52, 211, 153, 0.08)",
+        fill: false,
+        tension: 0.35,
+        pointRadius: 0,
+        yAxisID: "y1",
+        borderWidth: 2,
+      },
+    ],
+    {
+      scales: {
+        y: {
+          beginAtZero: true,
+          position: "left",
+          ticks: { color: "#94a3b8", callback: (value) => formatBytes(Number(value)) },
+          grid: { color: "rgba(100, 116, 139, 0.1)" },
+        },
+        y1: {
+          beginAtZero: true,
+          max: 100,
+          position: "right",
+          ticks: { color: "#86efac", callback: (value) => `${Math.round(Number(value))}%` },
+          grid: { drawOnChartArea: false },
+        },
+      },
+    },
+  );
+
+  eventLoopChart = makeLineChart(
+    eventLoopCtx,
+    [
+      {
+        label: "p95 lag",
+        data: [],
+        borderColor: "rgb(251, 191, 36)",
+        backgroundColor: "rgba(251, 191, 36, 0.12)",
+        fill: true,
+        tension: 0.35,
+        pointRadius: 0,
+        borderWidth: 2,
+      },
+      {
+        label: "avg lag",
+        data: [],
+        borderColor: "rgb(96, 165, 250)",
+        backgroundColor: "rgba(96, 165, 250, 0.08)",
+        fill: false,
+        tension: 0.35,
+        pointRadius: 0,
+        borderWidth: 2,
+      },
+    ],
+    {
+      scales: {
+        y: {
+          beginAtZero: true,
+          ticks: { color: "#94a3b8", callback: (value) => `${Math.round(Number(value))} ms` },
+          grid: { color: "rgba(100, 116, 139, 0.1)" },
+        },
+      },
+    },
+  );
+
+  sessionsChart = sessionsCtx ? new Chart(sessionsCtx, {
+    type: "doughnut",
+    data: {
+      labels: ["Console", "Remote Desktop", "Files", "Processes"],
+      datasets: [{
+        data: [0, 0, 0, 0],
+        backgroundColor: ["#34d399", "#c084fc", "#60a5fa", "#fb923c"],
+        borderColor: "#0f172a",
+        borderWidth: 2,
+      }],
+    },
+    options: {
+      responsive: true,
+      maintainAspectRatio: false,
+      animation: false,
+      cutout: "64%",
+      plugins: {
+        legend: { position: "right", labels: { boxWidth: 10, boxHeight: 10 } },
+        tooltip: chartTooltip(),
+      },
+    },
+  }) : null;
+
+  osChart = osCtx ? new Chart(osCtx, {
+    type: "bar",
+    data: {
+      labels: [],
+      datasets: [{
+        label: "Clients",
+        data: [],
+        backgroundColor: "rgba(56, 189, 248, 0.5)",
+        borderColor: "rgb(56, 189, 248)",
+        borderWidth: 1,
+      }],
+    },
+    options: {
+      indexAxis: "y",
+      responsive: true,
+      maintainAspectRatio: false,
+      animation: false,
+      plugins: {
+        legend: { display: false },
+        tooltip: chartTooltip(),
+      },
+      scales: {
+        x: {
+          beginAtZero: true,
+          ticks: { color: "#94a3b8", precision: 0 },
+          grid: { color: "rgba(100, 116, 139, 0.1)" },
+        },
+        y: {
+          ticks: { color: "#94a3b8" },
+          grid: { display: false },
+        },
+      },
+    },
+  }) : null;
 }
 
 function formatBytes(bytes) {
-  if (bytes === 0) return "0 B";
+  if (!Number.isFinite(bytes) || bytes <= 0) return "0 B";
+  if (bytes < 1) return `${bytes.toFixed(2)} B`;
   const k = 1024;
   const sizes = ["B", "KB", "MB", "GB", "TB"];
-  const i = Math.floor(Math.log(bytes) / Math.log(k));
+  const i = Math.min(sizes.length - 1, Math.floor(Math.log(bytes) / Math.log(k)));
   return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + " " + sizes[i];
 }
 
@@ -269,6 +608,10 @@ function updateMetrics(data, debug) {
   if (httpLatencyP95El) {
     httpLatencyP95El.textContent = formatMs(data.http.latencyP95 || 0);
   }
+  const httpLatencyP99El = document.getElementById("http-latency-p99");
+  if (httpLatencyP99El) {
+    httpLatencyP99El.textContent = formatMs(data.http.latencyP99 || 0);
+  }
   const eventLoopAvgEl = document.getElementById("event-loop-avg");
   if (eventLoopAvgEl) {
     eventLoopAvgEl.textContent = formatMs(data.eventLoop.avg || 0);
@@ -314,6 +657,57 @@ function updateMetrics(data, debug) {
     osList.innerHTML = '<div class="text-slate-500">No clients</div>';
   }
 
+  if (sessionsChart) {
+    sessionsChart.data.datasets[0].data = [
+      data.sessions.console || 0,
+      data.sessions.remoteDesktop || 0,
+      data.sessions.fileBrowser || 0,
+      data.sessions.process || 0,
+    ];
+    sessionsChart.update("none");
+  }
+
+  if (osChart) {
+    const osEntries = Object.entries(data.clients.byOS || {})
+      .sort((a, b) => b[1] - a[1])
+      .slice(0, 10);
+    osChart.data.labels = osEntries.length ? osEntries.map(([label]) => label) : ["No clients"];
+    osChart.data.datasets[0].data = osEntries.length ? osEntries.map(([, count]) => count) : [0];
+    osChart.update("none");
+  }
+
+  const httpRoutesList = document.getElementById("http-routes");
+  if (httpRoutesList) {
+    const routes = Array.isArray(data.http.routes) ? data.http.routes : [];
+    if (routes.length > 0) {
+      httpRoutesList.innerHTML = routes
+        .map((route) => {
+          const errorClass = route.errorsLastMinute > 0 ? "text-red-300" : "text-slate-400";
+          return `
+            <div class="bg-slate-800/50 rounded p-3 min-w-0">
+              <div class="text-xs text-slate-400 mb-2 truncate" title="${escapeHtml(route.route)}">
+                ${escapeHtml(route.route)}
+              </div>
+              <div class="grid grid-cols-2 gap-x-3 gap-y-1 text-xs">
+                <span class="text-slate-500">p95</span>
+                <span class="font-semibold text-right">${formatMs(route.latencyP95 || 0)}</span>
+                <span class="text-slate-500">avg</span>
+                <span class="font-semibold text-right">${formatMs(route.latencyAvg || 0)}</span>
+                <span class="text-slate-500">req/min</span>
+                <span class="font-semibold text-right">${Number(route.countLastMinute || 0).toLocaleString()}</span>
+                <span class="text-slate-500">errors</span>
+                <span class="font-semibold text-right ${errorClass}">${Number(route.errorsLastMinute || 0).toLocaleString()}</span>
+              </div>
+            </div>
+          `;
+        })
+        .join("");
+    } else {
+      httpRoutesList.innerHTML =
+        '<div class="text-slate-500 col-span-full text-center py-4">No HTTP route samples yet</div>';
+    }
+  }
+
   const commandTypesList = document.getElementById("command-types");
   if (Object.keys(data.commands.byType).length > 0) {
     const topCommands = Object.entries(data.commands.byType)
@@ -462,7 +856,11 @@ async function initCountryMap() {
 }
 
 function updateCharts(history, snapshot) {
-  if (history.length === 0) {
+  const points = Array.isArray(history) && history.length > 0
+    ? history.slice(-MAX_CHART_POINTS)
+    : [];
+
+  if (points.length === 0) {
     const now = new Date();
     const label = formatTime(now.getTime());
 
@@ -473,26 +871,100 @@ function updateCharts(history, snapshot) {
     commandsChart.data.labels = [label];
     commandsChart.data.datasets[0].data = [snapshot.commands.lastMinute];
     commandsChart.update("none");
+
+    if (bandwidthChart) {
+      bandwidthChart.data.labels = [label];
+      bandwidthChart.data.datasets[0].data = [snapshot.bandwidth.sentPerSecond || 0];
+      bandwidthChart.data.datasets[1].data = [snapshot.bandwidth.receivedPerSecond || 0];
+      bandwidthChart.update("none");
+    }
+
+    if (httpChart) {
+      httpChart.data.labels = [label];
+      httpChart.data.datasets[0].data = [snapshot.http.latencyP99 || 0];
+      httpChart.data.datasets[1].data = [snapshot.http.latencyP95 || 0];
+      httpChart.data.datasets[2].data = [snapshot.http.latencyAvg || 0];
+      httpChart.update("none");
+    }
+
+    if (httpRequestsChart) {
+      httpRequestsChart.data.labels = [label];
+      httpRequestsChart.data.datasets[0].data = [snapshot.http.lastMinute || 0];
+      httpRequestsChart.data.datasets[1].data = [snapshot.http.lastMinuteErrors || 0];
+      httpRequestsChart.update("none");
+    }
+
+    if (memoryChart) {
+      memoryChart.data.labels = [label];
+      memoryChart.data.datasets[0].data = [snapshot.server.memoryUsage.heapUsed || 0];
+      memoryChart.data.datasets[1].data = [snapshot.server.memoryUsage.rss || 0];
+      memoryChart.data.datasets[2].data = [snapshot.server.systemMemory.usedPercent || 0];
+      memoryChart.update("none");
+    }
+
+    if (eventLoopChart) {
+      eventLoopChart.data.labels = [label];
+      eventLoopChart.data.datasets[0].data = [snapshot.eventLoop.p95 || 0];
+      eventLoopChart.data.datasets[1].data = [snapshot.eventLoop.avg || 0];
+      eventLoopChart.update("none");
+    }
     return;
   }
 
-  const labels = history.map((h) => formatTime(h.timestamp));
+  const labels = points.map((h) => formatTime(h.timestamp));
-  const clientsData = history.map((h) => h.clientsOnline);
+  const clientsData = points.map((h) => h.clientsOnline || 0);
 
   clientsChart.data.labels = labels;
   clientsChart.data.datasets[0].data = clientsData;
   clientsChart.update("none");
 
-  const commandsData = history.map((h) => h.commandsPerMinute);
+  const commandsData = points.map((h) => h.commandsPerMinute || 0);
 
   commandsChart.data.labels = labels;
   commandsChart.data.datasets[0].data = commandsData;
   commandsChart.update("none");
+
+  if (bandwidthChart) {
+    bandwidthChart.data.labels = labels;
+    bandwidthChart.data.datasets[0].data = points.map((h) => h.bandwidthSent || 0);
+    bandwidthChart.data.datasets[1].data = points.map((h) => h.bandwidthReceived || 0);
+    bandwidthChart.update("none");
+  }
+
+  if (httpChart) {
+    httpChart.data.labels = labels;
+    httpChart.data.datasets[0].data = points.map((h) => h.httpLatencyP99 || 0);
+    httpChart.data.datasets[1].data = points.map((h) => h.httpLatencyP95 || 0);
+    httpChart.data.datasets[2].data = points.map((h) => h.httpLatencyAvg || 0);
+    httpChart.update("none");
+  }
+
+  if (httpRequestsChart) {
+    httpRequestsChart.data.labels = labels;
+    httpRequestsChart.data.datasets[0].data = points.map((h) => h.httpRequestsPerMinute || 0);
+    httpRequestsChart.data.datasets[1].data = points.map((h) => h.httpErrorsPerMinute || 0);
+    httpRequestsChart.update("none");
+  }
+
+  if (memoryChart) {
+    memoryChart.data.labels = labels;
+    memoryChart.data.datasets[0].data = points.map((h) => h.heapUsed || 0);
+    memoryChart.data.datasets[1].data = points.map((h) => h.rss || 0);
+    memoryChart.data.datasets[2].data = points.map((h) => h.systemMemoryUsedPercent || 0);
+    memoryChart.update("none");
+  }
+
+  if (eventLoopChart) {
+    eventLoopChart.data.labels = labels;
+    eventLoopChart.data.datasets[0].data = points.map((h) => h.eventLoopP95 || 0);
+    eventLoopChart.data.datasets[1].data = points.map((h) => h.eventLoopAvg || 0);
+    eventLoopChart.update("none");
+  }
 }
 
 async function fetchMetrics() {
   try {
-    const response = await fetch("/api/metrics", {
+    const response = await fetch(`/api/metrics?historyLimit=${MAX_CHART_POINTS}`, {
       credentials: "include",
     });
 
@@ -589,7 +1061,7 @@ document.addEventListener("DOMContentLoaded", async () => {
 
   await fetchMetrics();
 
-  setInterval(fetchMetrics, 2000);
+  setInterval(fetchMetrics, METRICS_POLL_INTERVAL_MS);
 
   const logoutBtn = document.getElementById("logout-btn");
   if (logoutBtn && !logoutBtn.dataset.boundLogout) {

Overlord-Server/public/metrics.html

@@ -145,6 +145,101 @@
           </div>
         </div>
 
+        <!-- Operational Pressure Charts -->
+        <div class="grid grid-cols-1 lg:grid-cols-2 gap-4">
+          <div class="bg-slate-900/70 border border-slate-800 rounded-lg p-4">
+            <h3 class="text-lg font-semibold mb-4 flex items-center gap-2">
+              <i class="fa-solid fa-network-wired text-orange-400"></i>
+              Network Throughput
+            </h3>
+            <div class="h-64">
+              <canvas id="bandwidth-chart"></canvas>
+            </div>
+          </div>
+
+          <div class="bg-slate-900/70 border border-slate-800 rounded-lg p-4">
+            <h3 class="text-lg font-semibold mb-4 flex items-center gap-2">
+              <i class="fa-solid fa-arrow-right-arrow-left text-emerald-400"></i>
+              HTTP Requests
+            </h3>
+            <div class="h-64">
+              <canvas id="http-requests-chart"></canvas>
+            </div>
+          </div>
+
+          <div class="bg-slate-900/70 border border-slate-800 rounded-lg p-4">
+            <h3 class="text-lg font-semibold mb-4 flex items-center gap-2">
+              <i class="fa-solid fa-gauge-high text-rose-400"></i>
+              HTTP Latency
+            </h3>
+            <div class="h-64">
+              <canvas id="http-chart"></canvas>
+            </div>
+          </div>
+
+          <div class="bg-slate-900/70 border border-slate-800 rounded-lg p-4">
+            <h3 class="text-lg font-semibold mb-4 flex items-center gap-2">
+              <i class="fa-solid fa-memory text-cyan-400"></i>
+              Runtime Memory
+            </h3>
+            <div class="h-64">
+              <canvas id="memory-chart"></canvas>
+            </div>
+          </div>
+
+          <div class="bg-slate-900/70 border border-slate-800 rounded-lg p-4">
+            <h3 class="text-lg font-semibold mb-4 flex items-center gap-2">
+              <i class="fa-solid fa-stopwatch text-amber-400"></i>
+              Event Loop Lag
+            </h3>
+            <div class="h-64">
+              <canvas id="event-loop-chart"></canvas>
+            </div>
+          </div>
+        </div>
+
+        <!-- Slow HTTP Routes -->
+        <div class="bg-slate-900/70 border border-slate-800 rounded-lg p-4">
+          <div class="flex items-center justify-between gap-3 flex-wrap mb-3">
+            <h3 class="text-lg font-semibold flex items-center gap-2">
+              <i class="fa-solid fa-route text-rose-400"></i>
+              Slowest HTTP Routes
+            </h3>
+            <div class="text-xs text-slate-500">Sorted by p95 latency, last minute</div>
+          </div>
+          <div
+            id="http-routes"
+            class="grid grid-cols-1 md:grid-cols-2 xl:grid-cols-4 gap-3"
+          >
+            <div class="text-slate-500 col-span-full text-center py-4">
+              No HTTP route samples yet
+            </div>
+          </div>
+        </div>
+
+        <!-- Mix Charts -->
+        <div class="grid grid-cols-1 lg:grid-cols-2 gap-4">
+          <div class="bg-slate-900/70 border border-slate-800 rounded-lg p-4">
+            <h3 class="text-lg font-semibold mb-4 flex items-center gap-2">
+              <i class="fa-solid fa-plug text-emerald-400"></i>
+              Session Mix
+            </h3>
+            <div class="h-64">
+              <canvas id="sessions-chart"></canvas>
+            </div>
+          </div>
+
+          <div class="bg-slate-900/70 border border-slate-800 rounded-lg p-4">
+            <h3 class="text-lg font-semibold mb-4 flex items-center gap-2">
+              <i class="fa-brands fa-windows text-cyan-400"></i>
+              Operating Systems
+            </h3>
+            <div class="h-64">
+              <canvas id="os-chart"></canvas>
+            </div>
+          </div>
+        </div>
+
         <!-- Additional Stats -->
         <div class="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-3 gap-4">
           <!-- Server Stats -->
@@ -181,36 +276,22 @@
             </div>
           </div>
 
-          <!-- Ping Stats -->
+          <!-- Clients by OS -->
           <div class="bg-slate-900/70 border border-slate-800 rounded-lg p-4">
             <h3 class="text-lg font-semibold mb-3 flex items-center gap-2">
-              <i class="fa-solid fa-signal text-blue-400"></i>
+              <i class="fa-brands fa-windows text-cyan-400"></i>
-              Ping Statistics
+              Clients by OS
             </h3>
-            <div class="space-y-2 text-sm">
+            <div id="clients-by-os" class="space-y-2 text-sm">
-              <div class="flex justify-between">
+              <div class="text-slate-500">No data</div>
-                <span class="text-slate-400">Average:</span>
-                <span id="ping-avg">-</span>
-              </div>
-              <div class="flex justify-between">
-                <span class="text-slate-400">Min / Max:</span>
-                <span
-                  ><span id="ping-min">-</span> /
-                  <span id="ping-max">-</span></span
-                >
-              </div>
-              <div class="flex justify-between">
-                <span class="text-slate-400">Samples:</span>
-                <span id="ping-count">0</span>
-              </div>
             </div>
           </div>
 
-          <!-- HTTP & Event Loop -->
+          <!-- HTTP, Ping & Event Loop -->
           <div class="bg-slate-900/70 border border-slate-800 rounded-lg p-4">
             <h3 class="text-lg font-semibold mb-3 flex items-center gap-2">
               <i class="fa-solid fa-wave-square text-amber-400"></i>
-              HTTP & Event Loop
+              HTTP, Ping & Event Loop
             </h3>
             <div class="space-y-2 text-sm">
               <div class="flex justify-between">
@@ -222,10 +303,11 @@
                 <span id="http-errors-minute">0</span>
               </div>
               <div class="flex justify-between">
-                <span class="text-slate-400">Latency avg/p95:</span>
+                <span class="text-slate-400">Latency avg/p95/p99:</span>
                 <span>
                   <span id="http-latency-avg">0</span> /
-                  <span id="http-latency-p95">0</span>
+                  <span id="http-latency-p95">0</span> /
+                  <span id="http-latency-p99">0</span>
                 </span>
               </div>
               <div class="flex justify-between">
@@ -239,17 +321,21 @@
                 <span class="text-slate-400">Loop lag max:</span>
                 <span id="event-loop-max">0</span>
               </div>
-            </div>
+              <div class="flex justify-between">
-          </div>
+                <span class="text-slate-400">Ping avg:</span>
-
+                <span id="ping-avg">-</span>
-          <!-- Client Distribution -->
+              </div>
-          <div class="bg-slate-900/70 border border-slate-800 rounded-lg p-4">
+              <div class="flex justify-between">
-            <h3 class="text-lg font-semibold mb-3 flex items-center gap-2">
+                <span class="text-slate-400">Ping min/max:</span>
-              <i class="fa-brands fa-windows text-cyan-400"></i>
+                <span>
-              Clients by OS
+                  <span id="ping-min">-</span> /
-            </h3>
+                  <span id="ping-max">-</span>
-            <div id="clients-by-os" class="space-y-2 text-sm">
+                </span>
-              <div class="text-slate-500">No data</div>
+              </div>
+              <div class="flex justify-between">
+                <span class="text-slate-400">Ping samples:</span>
+                <span id="ping-count">0</span>
+              </div>
             </div>
           </div>
         </div>

Overlord-Server/src/metrics.test.ts

@@ -125,6 +125,31 @@ describe("MetricsCollector", () => {
       const snap = metrics.getSnapshot();
       expect(snap.http.latencyAvg).toBeCloseTo(50.5, 0);
       expect(snap.http.latencyP95).toBeGreaterThanOrEqual(90);
+      expect(snap.http.latencyP99).toBeGreaterThanOrEqual(99);
+    });
+
+    test("tracks slow HTTP routes", () => {
+      metrics.recordHttpRequest(15, 200, "GET /api/metrics");
+      metrics.recordHttpRequest(120, 200, "GET /api/clients/:id/rd/ws");
+      metrics.recordHttpRequest(200, 500, "GET /api/clients/:id/rd/ws");
+
+      const snap = metrics.getSnapshot();
+      expect(snap.http.routes[0].route).toBe("GET /api/clients/:id/rd/ws");
+      expect(snap.http.routes[0].countLastMinute).toBe(2);
+      expect(snap.http.routes[0].errorsLastMinute).toBe(1);
+      expect(snap.http.routes[0].latencyP95).toBeGreaterThanOrEqual(120);
+    });
+
+    test("does not count the metrics dashboard poll as HTTP pressure", () => {
+      metrics.recordHttpRequest(100, 200, "GET /api/metrics");
+      metrics.recordHttpRequest(20, 200, "GET /api/clients");
+
+      const snap = metrics.getSnapshot();
+      expect(snap.http.total).toBe(1);
+      expect(snap.http.lastMinute).toBe(1);
+      expect(snap.http.routes.map((route) => route.route)).toEqual([
+        "GET /api/clients",
+      ]);
     });
   });
 
@@ -177,6 +202,26 @@ describe("MetricsCollector", () => {
       expect(history[0].timestamp).toBe(snap.timestamp);
     });
 
+    test("history captures pressure metrics for charting", () => {
+      metrics.recordHttpRequest(25, 200);
+      metrics.recordHttpRequest(80, 500);
+      metrics.recordBytesSent(1024);
+      metrics.recordBytesReceived(2048);
+
+      const snap = metrics.getSnapshot();
+      metrics.recordHistoryEntry(snap);
+
+      const [entry] = metrics.getHistory();
+      expect(entry.httpRequestsPerMinute).toBeGreaterThanOrEqual(2);
+      expect(entry.httpErrorsPerMinute).toBeGreaterThanOrEqual(1);
+      expect(entry.httpLatencyAvg).toBeGreaterThan(0);
+      expect(entry.httpLatencyP95).toBeGreaterThan(0);
+      expect(entry.heapUsed).toBeGreaterThan(0);
+      expect(entry.rss).toBeGreaterThan(0);
+      expect(entry.systemMemoryUsedPercent).toBeGreaterThanOrEqual(0);
+      expect(entry.activeSessions).toBe(0);
+    });
+
     test("history is capped at maxHistoryPoints", () => {
       const weekOfFiveSecondSamples = 7 * 24 * 60 * 12;
       for (let i = 0; i < weekOfFiveSecondSamples + 180; i++) {

Overlord-Server/src/metrics.ts

@@ -59,6 +59,8 @@ export interface MetricsSnapshot {
     lastMinuteErrors: number;
     latencyAvg: number;
     latencyP95: number;
+    latencyP99: number;
+    routes: HttpRouteStats[];
   };
   eventLoop: {
     avg: number;
@@ -67,12 +69,41 @@ export interface MetricsSnapshot {
   };
 }
 
+export interface HttpRouteStats {
+  route: string;
+  countLastMinute: number;
+  errorsLastMinute: number;
+  latencyAvg: number;
+  latencyP95: number;
+  latencyP99: number;
+  latencyMax: number;
+  lastDuration: number;
+  lastStatus: number;
+}
+
 export interface MetricsHistory {
   timestamp: number;
   clientsOnline: number;
   commandsPerMinute: number;
   bandwidthSent: number;
   bandwidthReceived: number;
+  httpRequestsPerMinute?: number;
+  httpErrorsPerMinute?: number;
+  httpLatencyAvg?: number;
+  httpLatencyP95?: number;
+  httpLatencyP99?: number;
+  eventLoopAvg?: number;
+  eventLoopP95?: number;
+  heapUsed?: number;
+  rss?: number;
+  systemMemoryUsedPercent?: number;
+  activeSessions?: number;
+}
+
+interface TimedHttpSample {
+  ts: number;
+  duration: number;
+  statusCode: number;
 }
 
 class MetricsCollector {
@@ -103,8 +134,11 @@ class MetricsCollector {
   private httpTotal: number = 0;
   private httpTimestamps: number[] = [];
   private httpErrorTimestamps: number[] = [];
-  private httpLatencies: number[] = [];
+  private httpSamples: TimedHttpSample[] = [];
-  private maxHttpLatencyHistory: number = 1000;
+  private httpRouteSamples: Map<string, TimedHttpSample[]> = new Map();
+  private maxHttpLatencyHistory: number = 2000;
+  private maxHttpRouteSamples: number = 300;
+  private ignoredHttpMetricRoutes: Set<string> = new Set(["GET /api/metrics"]);
 
   private eventLoopDelays: number[] = [];
   private maxEventLoopHistory: number = 300;
@@ -236,7 +270,74 @@ class MetricsCollector {
     }, intervalMs);
   }
 
-  recordHttpRequest(durationMs: number, statusCode: number) {
+  private percentile(samples: number[], percentile: number): number {
+    if (samples.length === 0) return 0;
+    const index = Math.max(0, Math.ceil(samples.length * percentile) - 1);
+    return samples[index] ?? 0;
+  }
+
+  private average(samples: number[]): number {
+    return samples.length
+      ? samples.reduce((sum, value) => sum + value, 0) / samples.length
+      : 0;
+  }
+
+  private pruneHttpSamples(list: TimedHttpSample[], minTs: number): void {
+    let removeCount = 0;
+    while (removeCount < list.length && list[removeCount].ts <= minTs) {
+      removeCount += 1;
+    }
+    if (removeCount > 0) {
+      list.splice(0, removeCount);
+    }
+  }
+
+  private countHttpErrors(samples: TimedHttpSample[]): number {
+    let count = 0;
+    for (const sample of samples) {
+      if (sample.statusCode >= 400) count += 1;
+    }
+    return count;
+  }
+
+  private getTopHttpRoutes(minTs: number): HttpRouteStats[] {
+    const routes: HttpRouteStats[] = [];
+    for (const [route, samples] of this.httpRouteSamples.entries()) {
+      this.pruneHttpSamples(samples, minTs);
+      if (samples.length === 0) {
+        this.httpRouteSamples.delete(route);
+        continue;
+      }
+
+      const durations = samples
+        .map((sample) => sample.duration)
+        .sort((a, b) => a - b);
+      const last = samples[samples.length - 1];
+      routes.push({
+        route,
+        countLastMinute: samples.length,
+        errorsLastMinute: this.countHttpErrors(samples),
+        latencyAvg: this.average(durations),
+        latencyP95: this.percentile(durations, 0.95),
+        latencyP99: this.percentile(durations, 0.99),
+        latencyMax: durations[durations.length - 1] ?? 0,
+        lastDuration: last?.duration ?? 0,
+        lastStatus: last?.statusCode ?? 0,
+      });
+    }
+
+    return routes
+      .sort((a, b) => {
+        if (b.latencyP95 !== a.latencyP95) return b.latencyP95 - a.latencyP95;
+        if (b.latencyAvg !== a.latencyAvg) return b.latencyAvg - a.latencyAvg;
+        return b.countLastMinute - a.countLastMinute;
+      })
+      .slice(0, 8);
+  }
+
+  recordHttpRequest(durationMs: number, statusCode: number, route = "unknown") {
+    if (this.ignoredHttpMetricRoutes.has(route)) return;
+
     this.httpTotal++;
     const now = Date.now();
     this.httpTimestamps.push(now);
@@ -246,14 +347,25 @@ class MetricsCollector {
       this.pruneTimestampWindow(this.httpErrorTimestamps, now - 60000);
     }
     if (Number.isFinite(durationMs)) {
-      this.httpLatencies.push(durationMs);
+      const sample = { ts: now, duration: Math.max(0, durationMs), statusCode };
-      if (this.httpLatencies.length > this.maxHttpLatencyHistory) {
+      this.httpSamples.push(sample);
-        this.httpLatencies.shift();
+      if (this.httpSamples.length > this.maxHttpLatencyHistory) {
+        this.httpSamples.splice(0, this.httpSamples.length - this.maxHttpLatencyHistory);
       }
+
+      const routeSamples = this.httpRouteSamples.get(route) || [];
+      routeSamples.push(sample);
+      if (routeSamples.length > this.maxHttpRouteSamples) {
+        routeSamples.splice(0, routeSamples.length - this.maxHttpRouteSamples);
+      }
+      this.httpRouteSamples.set(route, routeSamples);
     }
   }
 
-  async withHttpMetrics<T extends Response>(handler: () => Promise<T>): Promise<T> {
+  async withHttpMetrics<T extends Response>(
+    handler: () => Promise<T>,
+    route = "unknown",
+  ): Promise<T> {
     const start = typeof performance !== "undefined" && performance.now
       ? performance.now()
       : Date.now();
@@ -262,13 +374,13 @@ class MetricsCollector {
       const end = typeof performance !== "undefined" && performance.now
         ? performance.now()
         : Date.now();
-      this.recordHttpRequest(end - start, response?.status ?? 0);
+      this.recordHttpRequest(end - start, response?.status ?? 0, route);
       return response;
     } catch (err) {
       const end = typeof performance !== "undefined" && performance.now
         ? performance.now()
         : Date.now();
-      this.recordHttpRequest(end - start, 500);
+      this.recordHttpRequest(end - start, 500, route);
       throw err;
     }
   }
@@ -280,6 +392,21 @@ class MetricsCollector {
       commandsPerMinute: snapshot.commands.lastMinute,
       bandwidthSent: this.sentPerSecond,
       bandwidthReceived: this.receivedPerSecond,
+      httpRequestsPerMinute: snapshot.http.lastMinute,
+      httpErrorsPerMinute: snapshot.http.lastMinuteErrors,
+      httpLatencyAvg: snapshot.http.latencyAvg,
+      httpLatencyP95: snapshot.http.latencyP95,
+      httpLatencyP99: snapshot.http.latencyP99,
+      eventLoopAvg: snapshot.eventLoop.avg,
+      eventLoopP95: snapshot.eventLoop.p95,
+      heapUsed: snapshot.server.memoryUsage.heapUsed,
+      rss: snapshot.server.memoryUsage.rss,
+      systemMemoryUsedPercent: snapshot.server.systemMemory.usedPercent,
+      activeSessions:
+        snapshot.sessions.console +
+        snapshot.sessions.remoteDesktop +
+        snapshot.sessions.fileBrowser +
+        snapshot.sessions.process,
     };
 
     this.history.push(historyEntry);
@@ -297,6 +424,7 @@ class MetricsCollector {
     this.pruneTimestampWindow(this.commandTimestamps, oneHourAgo);
     this.pruneTimestampWindow(this.httpTimestamps, oneMinuteAgo);
     this.pruneTimestampWindow(this.httpErrorTimestamps, oneMinuteAgo);
+    this.pruneHttpSamples(this.httpSamples, oneMinuteAgo);
 
     const commandsLastMinute = this.countRecent(this.commandTimestamps, oneMinuteAgo);
     const commandsLastHour = this.commandTimestamps.length;
@@ -309,16 +437,13 @@ class MetricsCollector {
     const httpLastMinute = this.httpTimestamps.length;
     const httpErrorsLastMinute = this.httpErrorTimestamps.length;
 
-    const httpLatencySamples = [...this.httpLatencies].sort((a, b) => a - b);
+    const httpLatencySamples = this.httpSamples
-    const httpLatencyAvg = httpLatencySamples.length
+      .map((sample) => sample.duration)
-      ? httpLatencySamples.reduce((a, b) => a + b, 0) / httpLatencySamples.length
+      .sort((a, b) => a - b);
-      : 0;
+    const httpLatencyAvg = this.average(httpLatencySamples);
-    const httpP95Index = httpLatencySamples.length
+    const httpLatencyP95 = this.percentile(httpLatencySamples, 0.95);
-      ? Math.max(0, Math.floor(httpLatencySamples.length * 0.95) - 1)
+    const httpLatencyP99 = this.percentile(httpLatencySamples, 0.99);
-      : 0;
+    const httpRoutes = this.getTopHttpRoutes(oneMinuteAgo);
-    const httpLatencyP95 = httpLatencySamples.length
-      ? httpLatencySamples[httpP95Index] ?? 0
-      : 0;
 
     const eventLoopSamples = [...this.eventLoopDelays].sort((a, b) => a - b);
     const eventLoopAvg = eventLoopSamples.length
@@ -400,6 +525,8 @@ class MetricsCollector {
         lastMinuteErrors: httpErrorsLastMinute,
         latencyAvg: httpLatencyAvg,
         latencyP95: httpLatencyP95,
+        latencyP99: httpLatencyP99,
+        routes: httpRoutes,
       },
       eventLoop: {
         avg: eventLoopAvg,
@@ -428,7 +555,8 @@ class MetricsCollector {
     this.httpTotal = 0;
     this.httpTimestamps = [];
     this.httpErrorTimestamps = [];
-    this.httpLatencies = [];
+    this.httpSamples = [];
+    this.httpRouteSamples.clear();
     this.eventLoopDelays = [];
   }
 }

Overlord-Server/src/server/http-dispatch.test.ts

@@ -1,6 +1,6 @@
 import { beforeEach, describe, expect, test } from "bun:test";
 import { clearRequestRateLimitsForTests } from "../rateLimit";
-import { createHttpFetchHandler } from "./http-dispatch";
+import { createHttpFetchHandler, normalizeHttpRoute } from "./http-dispatch";
 
 const metrics = {
   withHttpMetrics: (fn: () => Promise<Response>) => fn(),
@@ -35,3 +35,22 @@ describe("createHttpFetchHandler rate limiting", () => {
     expect(limited.headers.get("Retry-After")).toBeTruthy();
   });
 });
+
+describe("normalizeHttpRoute", () => {
+  test("collapses dynamic route segments", () => {
+    const req = new Request(
+      "https://localhost/api/clients/0a44a8d4d61c7840540b43fd187fe3c43e2c46c9600b2aa1a61a6b1ab03c5b92/rd/ws?cache=1",
+      { method: "POST" },
+    );
+
+    expect(normalizeHttpRoute(req, new URL(req.url))).toBe(
+      "POST /api/clients/:id/rd/ws",
+    );
+  });
+
+  test("groups static asset files", () => {
+    const req = new Request("https://localhost/assets/metrics.js");
+
+    expect(normalizeHttpRoute(req, new URL(req.url))).toBe("GET /assets/:file");
+  });
+});

Overlord-Server/src/server/http-dispatch.ts

@@ -3,6 +3,9 @@ import { consumeUnauthorizedRateLimit } from "../rateLimit";
 
 export type RouteHandler = (req: Request, url: URL, server: unknown) => Promise<Response | null>;
 
+const UUID_SEGMENT = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
+const LONG_ID_SEGMENT = /^[0-9a-f]{16,}$/i;
+
 function tooManyRequestsResponse(retryAfter = 60): Response {
   return new Response("Too many requests", {
     status: 429,
@@ -13,17 +16,35 @@ function tooManyRequestsResponse(retryAfter = 60): Response {
   });
 }
 
+function normalizeRouteSegment(segment: string): string {
+  if (/^\d+$/.test(segment)) return ":id";
+  if (UUID_SEGMENT.test(segment) || LONG_ID_SEGMENT.test(segment)) return ":id";
+  if (segment.includes(".") && !segment.startsWith(".")) return ":file";
+  if (segment.length >= 24 && /^[a-z0-9_-]+$/i.test(segment)) return ":id";
+  return segment;
+}
+
+export function normalizeHttpRoute(req: Request, url: URL): string {
+  const path = url.pathname
+    .split("/")
+    .filter(Boolean)
+    .map(normalizeRouteSegment)
+    .join("/");
+  return `${req.method.toUpperCase()} /${path}`;
+}
+
 export function createHttpFetchHandler(deps: {
-  metrics: { withHttpMetrics: (fn: () => Promise<Response>) => Promise<Response> };
+  metrics: { withHttpMetrics: (fn: () => Promise<Response>, route?: string) => Promise<Response> };
   CORS_HEADERS: Record<string, string>;
   routes: RouteHandler[];
 }) {
   return async function fetchHandler(req: Request, server: unknown): Promise<Response> {
+    const url = new URL(req.url);
+    const routeLabel = normalizeHttpRoute(req, url);
     return deps.metrics.withHttpMetrics(async () => {
       if (req.method === "OPTIONS") {
         return new Response("", { headers: deps.CORS_HEADERS });
       }
-      const url = new URL(req.url);
       const wrapped = wrapServerWithClientIp(server as RequestServerLike);
       const ip = wrapped.requestIP(req)?.address || "unknown";
 
@@ -38,6 +59,6 @@ export function createHttpFetchHandler(deps: {
         }
       }
       return new Response("Not found", { status: 404 });
-    });
+    }, routeLabel);
   };
 }

Overlord-Server/src/server/routes/misc-routes.ts

@@ -330,7 +330,11 @@ export async function handleMiscRoutes(
     snapshot.sessions.fileBrowser = deps.getFileBrowserSessionCount();
     snapshot.sessions.process = deps.getProcessSessionCount();
 
-    const history = metrics.getHistory();
+    const requestedHistoryLimit = Number(url.searchParams.get("historyLimit") || 240);
+    const historyLimit = Number.isFinite(requestedHistoryLimit)
+      ? Math.max(1, Math.min(2000, requestedHistoryLimit))
+      : 240;
+    const history = metrics.getHistory().slice(-historyLimit);
 
     return new Response(JSON.stringify({ snapshot, history }), {
       headers: { "Content-Type": "application/json" },