diff --git a/Overlord-Server/public/assets/metrics.js b/Overlord-Server/public/assets/metrics.js index d1573d6..7bb3e6e 100644 --- a/Overlord-Server/public/assets/metrics.js +++ b/Overlord-Server/public/assets/metrics.js @@ -1,5 +1,12 @@ let clientsChart = null; let commandsChart = null; +let bandwidthChart = null; +let httpRequestsChart = null; +let httpChart = null; +let memoryChart = null; +let eventLoopChart = null; +let sessionsChart = null; +let osChart = null; let countryMap = null; let countriesLayer = null; let countryCounts = {}; @@ -7,6 +14,8 @@ let maxCountryCount = 0; let latestByCountry = {}; const GEOJSON_URL = "/vendor/geo-countries/countries.geojson"; +const MAX_CHART_POINTS = 240; +const METRICS_POLL_INTERVAL_MS = 5000; if (typeof Chart !== "undefined") { Chart.defaults.color = "#cbd5e1"; @@ -14,9 +23,69 @@ if (typeof Chart !== "undefined") { Chart.defaults.font.family = "Inter, system-ui, sans-serif"; } +function chartTooltip() { + return { + backgroundColor: "#0f172a", + titleColor: "#e2e8f0", + bodyColor: "#cbd5e1", + borderColor: "#334155", + borderWidth: 1, + }; +} + +function lineChartOptions(extra = {}) { + const extraPlugins = extra.plugins || {}; + return { + responsive: true, + maintainAspectRatio: false, + animation: false, + interaction: { mode: "index", intersect: false }, + scales: { + y: { + beginAtZero: true, + ticks: { color: "#94a3b8" }, + grid: { color: "rgba(100, 116, 139, 0.1)" }, + }, + x: { + ticks: { + color: "#94a3b8", + maxTicksLimit: 6, + autoSkip: true, + maxRotation: 0, + }, + grid: { color: "rgba(100, 116, 139, 0.1)" }, + }, + ...(extra.scales || {}), + }, + ...extra, + plugins: { + legend: + extraPlugins.legend ?? { display: true, labels: { boxWidth: 10, boxHeight: 10 } }, + tooltip: chartTooltip(), + ...extraPlugins, + }, + }; +} + +function makeLineChart(canvas, datasets, options = {}) { + if (!canvas) return null; + return new Chart(canvas, { + type: "line", + data: { labels: [], datasets }, + options: lineChartOptions(options), + }); +} + function initCharts() { const clientsCtx = document.getElementById("clients-chart"); const commandsCtx = document.getElementById("commands-chart"); + const bandwidthCtx = document.getElementById("bandwidth-chart"); + const httpRequestsCtx = document.getElementById("http-requests-chart"); + const httpCtx = document.getElementById("http-chart"); + const memoryCtx = document.getElementById("memory-chart"); + const eventLoopCtx = document.getElementById("event-loop-chart"); + const sessionsCtx = document.getElementById("sessions-chart"); + const osCtx = document.getElementById("os-chart"); clientsChart = new Chart(clientsCtx, { type: "line", @@ -119,13 +188,283 @@ function initCharts() { }, }, }); + + bandwidthChart = makeLineChart( + bandwidthCtx, + [ + { + label: "Sent/s", + data: [], + borderColor: "rgb(251, 146, 60)", + backgroundColor: "rgba(251, 146, 60, 0.12)", + fill: true, + tension: 0.35, + pointRadius: 0, + borderWidth: 2, + }, + { + label: "Received/s", + data: [], + borderColor: "rgb(56, 189, 248)", + backgroundColor: "rgba(56, 189, 248, 0.1)", + fill: true, + tension: 0.35, + pointRadius: 0, + borderWidth: 2, + }, + ], + { + scales: { + y: { + beginAtZero: true, + ticks: { + color: "#94a3b8", + callback: (value) => formatBytes(Number(value)), + }, + grid: { color: "rgba(100, 116, 139, 0.1)" }, + }, + }, + }, + ); + + httpRequestsChart = makeLineChart( + httpRequestsCtx, + [ + { + label: "requests/min", + data: [], + borderColor: "rgb(52, 211, 153)", + backgroundColor: "rgba(52, 211, 153, 0.12)", + fill: true, + tension: 0.35, + pointRadius: 0, + borderWidth: 2, + }, + { + label: "errors/min", + data: [], + borderColor: "rgb(248, 113, 113)", + backgroundColor: "rgba(248, 113, 113, 0.14)", + fill: false, + tension: 0.25, + pointRadius: 0, + borderDash: [5, 4], + borderWidth: 2, + }, + ], + { + scales: { + y: { + beginAtZero: true, + ticks: { color: "#94a3b8", precision: 0 }, + grid: { color: "rgba(100, 116, 139, 0.1)" }, + }, + }, + }, + ); + + httpChart = makeLineChart( + httpCtx, + [ + { + label: "p99 ms", + data: [], + borderColor: "rgb(248, 113, 113)", + backgroundColor: "rgba(248, 113, 113, 0.08)", + fill: false, + tension: 0.35, + pointRadius: 0, + borderDash: [4, 4], + borderWidth: 2, + }, + { + label: "p95 ms", + data: [], + borderColor: "rgb(244, 63, 94)", + backgroundColor: "rgba(244, 63, 94, 0.1)", + fill: true, + tension: 0.35, + pointRadius: 0, + borderWidth: 2, + }, + { + label: "avg ms", + data: [], + borderColor: "rgb(251, 191, 36)", + backgroundColor: "rgba(251, 191, 36, 0.08)", + fill: false, + tension: 0.35, + pointRadius: 0, + borderWidth: 2, + }, + ], + { + scales: { + y: { + beginAtZero: true, + ticks: { color: "#94a3b8", callback: (value) => `${Math.round(Number(value))} ms` }, + grid: { color: "rgba(100, 116, 139, 0.1)" }, + }, + }, + }, + ); + + memoryChart = makeLineChart( + memoryCtx, + [ + { + label: "Heap", + data: [], + borderColor: "rgb(34, 211, 238)", + backgroundColor: "rgba(34, 211, 238, 0.1)", + fill: true, + tension: 0.35, + pointRadius: 0, + yAxisID: "y", + borderWidth: 2, + }, + { + label: "RSS", + data: [], + borderColor: "rgb(129, 140, 248)", + backgroundColor: "rgba(129, 140, 248, 0.08)", + fill: true, + tension: 0.35, + pointRadius: 0, + yAxisID: "y", + borderWidth: 2, + }, + { + label: "System %", + data: [], + borderColor: "rgb(52, 211, 153)", + backgroundColor: "rgba(52, 211, 153, 0.08)", + fill: false, + tension: 0.35, + pointRadius: 0, + yAxisID: "y1", + borderWidth: 2, + }, + ], + { + scales: { + y: { + beginAtZero: true, + position: "left", + ticks: { color: "#94a3b8", callback: (value) => formatBytes(Number(value)) }, + grid: { color: "rgba(100, 116, 139, 0.1)" }, + }, + y1: { + beginAtZero: true, + max: 100, + position: "right", + ticks: { color: "#86efac", callback: (value) => `${Math.round(Number(value))}%` }, + grid: { drawOnChartArea: false }, + }, + }, + }, + ); + + eventLoopChart = makeLineChart( + eventLoopCtx, + [ + { + label: "p95 lag", + data: [], + borderColor: "rgb(251, 191, 36)", + backgroundColor: "rgba(251, 191, 36, 0.12)", + fill: true, + tension: 0.35, + pointRadius: 0, + borderWidth: 2, + }, + { + label: "avg lag", + data: [], + borderColor: "rgb(96, 165, 250)", + backgroundColor: "rgba(96, 165, 250, 0.08)", + fill: false, + tension: 0.35, + pointRadius: 0, + borderWidth: 2, + }, + ], + { + scales: { + y: { + beginAtZero: true, + ticks: { color: "#94a3b8", callback: (value) => `${Math.round(Number(value))} ms` }, + grid: { color: "rgba(100, 116, 139, 0.1)" }, + }, + }, + }, + ); + + sessionsChart = sessionsCtx ? new Chart(sessionsCtx, { + type: "doughnut", + data: { + labels: ["Console", "Remote Desktop", "Files", "Processes"], + datasets: [{ + data: [0, 0, 0, 0], + backgroundColor: ["#34d399", "#c084fc", "#60a5fa", "#fb923c"], + borderColor: "#0f172a", + borderWidth: 2, + }], + }, + options: { + responsive: true, + maintainAspectRatio: false, + animation: false, + cutout: "64%", + plugins: { + legend: { position: "right", labels: { boxWidth: 10, boxHeight: 10 } }, + tooltip: chartTooltip(), + }, + }, + }) : null; + + osChart = osCtx ? new Chart(osCtx, { + type: "bar", + data: { + labels: [], + datasets: [{ + label: "Clients", + data: [], + backgroundColor: "rgba(56, 189, 248, 0.5)", + borderColor: "rgb(56, 189, 248)", + borderWidth: 1, + }], + }, + options: { + indexAxis: "y", + responsive: true, + maintainAspectRatio: false, + animation: false, + plugins: { + legend: { display: false }, + tooltip: chartTooltip(), + }, + scales: { + x: { + beginAtZero: true, + ticks: { color: "#94a3b8", precision: 0 }, + grid: { color: "rgba(100, 116, 139, 0.1)" }, + }, + y: { + ticks: { color: "#94a3b8" }, + grid: { display: false }, + }, + }, + }, + }) : null; } function formatBytes(bytes) { - if (bytes === 0) return "0 B"; + if (!Number.isFinite(bytes) || bytes <= 0) return "0 B"; + if (bytes < 1) return `${bytes.toFixed(2)} B`; const k = 1024; const sizes = ["B", "KB", "MB", "GB", "TB"]; - const i = Math.floor(Math.log(bytes) / Math.log(k)); + const i = Math.min(sizes.length - 1, Math.floor(Math.log(bytes) / Math.log(k))); return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + " " + sizes[i]; } @@ -269,6 +608,10 @@ function updateMetrics(data, debug) { if (httpLatencyP95El) { httpLatencyP95El.textContent = formatMs(data.http.latencyP95 || 0); } + const httpLatencyP99El = document.getElementById("http-latency-p99"); + if (httpLatencyP99El) { + httpLatencyP99El.textContent = formatMs(data.http.latencyP99 || 0); + } const eventLoopAvgEl = document.getElementById("event-loop-avg"); if (eventLoopAvgEl) { eventLoopAvgEl.textContent = formatMs(data.eventLoop.avg || 0); @@ -314,6 +657,57 @@ function updateMetrics(data, debug) { osList.innerHTML = '
No clients
'; } + if (sessionsChart) { + sessionsChart.data.datasets[0].data = [ + data.sessions.console || 0, + data.sessions.remoteDesktop || 0, + data.sessions.fileBrowser || 0, + data.sessions.process || 0, + ]; + sessionsChart.update("none"); + } + + if (osChart) { + const osEntries = Object.entries(data.clients.byOS || {}) + .sort((a, b) => b[1] - a[1]) + .slice(0, 10); + osChart.data.labels = osEntries.length ? osEntries.map(([label]) => label) : ["No clients"]; + osChart.data.datasets[0].data = osEntries.length ? osEntries.map(([, count]) => count) : [0]; + osChart.update("none"); + } + + const httpRoutesList = document.getElementById("http-routes"); + if (httpRoutesList) { + const routes = Array.isArray(data.http.routes) ? data.http.routes : []; + if (routes.length > 0) { + httpRoutesList.innerHTML = routes + .map((route) => { + const errorClass = route.errorsLastMinute > 0 ? "text-red-300" : "text-slate-400"; + return ` +
+
+ ${escapeHtml(route.route)} +
+
+ p95 + ${formatMs(route.latencyP95 || 0)} + avg + ${formatMs(route.latencyAvg || 0)} + req/min + ${Number(route.countLastMinute || 0).toLocaleString()} + errors + ${Number(route.errorsLastMinute || 0).toLocaleString()} +
+
+ `; + }) + .join(""); + } else { + httpRoutesList.innerHTML = + '
No HTTP route samples yet
'; + } + } + const commandTypesList = document.getElementById("command-types"); if (Object.keys(data.commands.byType).length > 0) { const topCommands = Object.entries(data.commands.byType) @@ -462,7 +856,11 @@ async function initCountryMap() { } function updateCharts(history, snapshot) { - if (history.length === 0) { + const points = Array.isArray(history) && history.length > 0 + ? history.slice(-MAX_CHART_POINTS) + : []; + + if (points.length === 0) { const now = new Date(); const label = formatTime(now.getTime()); @@ -473,26 +871,100 @@ function updateCharts(history, snapshot) { commandsChart.data.labels = [label]; commandsChart.data.datasets[0].data = [snapshot.commands.lastMinute]; commandsChart.update("none"); + + if (bandwidthChart) { + bandwidthChart.data.labels = [label]; + bandwidthChart.data.datasets[0].data = [snapshot.bandwidth.sentPerSecond || 0]; + bandwidthChart.data.datasets[1].data = [snapshot.bandwidth.receivedPerSecond || 0]; + bandwidthChart.update("none"); + } + + if (httpChart) { + httpChart.data.labels = [label]; + httpChart.data.datasets[0].data = [snapshot.http.latencyP99 || 0]; + httpChart.data.datasets[1].data = [snapshot.http.latencyP95 || 0]; + httpChart.data.datasets[2].data = [snapshot.http.latencyAvg || 0]; + httpChart.update("none"); + } + + if (httpRequestsChart) { + httpRequestsChart.data.labels = [label]; + httpRequestsChart.data.datasets[0].data = [snapshot.http.lastMinute || 0]; + httpRequestsChart.data.datasets[1].data = [snapshot.http.lastMinuteErrors || 0]; + httpRequestsChart.update("none"); + } + + if (memoryChart) { + memoryChart.data.labels = [label]; + memoryChart.data.datasets[0].data = [snapshot.server.memoryUsage.heapUsed || 0]; + memoryChart.data.datasets[1].data = [snapshot.server.memoryUsage.rss || 0]; + memoryChart.data.datasets[2].data = [snapshot.server.systemMemory.usedPercent || 0]; + memoryChart.update("none"); + } + + if (eventLoopChart) { + eventLoopChart.data.labels = [label]; + eventLoopChart.data.datasets[0].data = [snapshot.eventLoop.p95 || 0]; + eventLoopChart.data.datasets[1].data = [snapshot.eventLoop.avg || 0]; + eventLoopChart.update("none"); + } return; } - const labels = history.map((h) => formatTime(h.timestamp)); - const clientsData = history.map((h) => h.clientsOnline); + const labels = points.map((h) => formatTime(h.timestamp)); + const clientsData = points.map((h) => h.clientsOnline || 0); clientsChart.data.labels = labels; clientsChart.data.datasets[0].data = clientsData; clientsChart.update("none"); - const commandsData = history.map((h) => h.commandsPerMinute); + const commandsData = points.map((h) => h.commandsPerMinute || 0); commandsChart.data.labels = labels; commandsChart.data.datasets[0].data = commandsData; commandsChart.update("none"); + + if (bandwidthChart) { + bandwidthChart.data.labels = labels; + bandwidthChart.data.datasets[0].data = points.map((h) => h.bandwidthSent || 0); + bandwidthChart.data.datasets[1].data = points.map((h) => h.bandwidthReceived || 0); + bandwidthChart.update("none"); + } + + if (httpChart) { + httpChart.data.labels = labels; + httpChart.data.datasets[0].data = points.map((h) => h.httpLatencyP99 || 0); + httpChart.data.datasets[1].data = points.map((h) => h.httpLatencyP95 || 0); + httpChart.data.datasets[2].data = points.map((h) => h.httpLatencyAvg || 0); + httpChart.update("none"); + } + + if (httpRequestsChart) { + httpRequestsChart.data.labels = labels; + httpRequestsChart.data.datasets[0].data = points.map((h) => h.httpRequestsPerMinute || 0); + httpRequestsChart.data.datasets[1].data = points.map((h) => h.httpErrorsPerMinute || 0); + httpRequestsChart.update("none"); + } + + if (memoryChart) { + memoryChart.data.labels = labels; + memoryChart.data.datasets[0].data = points.map((h) => h.heapUsed || 0); + memoryChart.data.datasets[1].data = points.map((h) => h.rss || 0); + memoryChart.data.datasets[2].data = points.map((h) => h.systemMemoryUsedPercent || 0); + memoryChart.update("none"); + } + + if (eventLoopChart) { + eventLoopChart.data.labels = labels; + eventLoopChart.data.datasets[0].data = points.map((h) => h.eventLoopP95 || 0); + eventLoopChart.data.datasets[1].data = points.map((h) => h.eventLoopAvg || 0); + eventLoopChart.update("none"); + } } async function fetchMetrics() { try { - const response = await fetch("/api/metrics", { + const response = await fetch(`/api/metrics?historyLimit=${MAX_CHART_POINTS}`, { credentials: "include", }); @@ -589,7 +1061,7 @@ document.addEventListener("DOMContentLoaded", async () => { await fetchMetrics(); - setInterval(fetchMetrics, 2000); + setInterval(fetchMetrics, METRICS_POLL_INTERVAL_MS); const logoutBtn = document.getElementById("logout-btn"); if (logoutBtn && !logoutBtn.dataset.boundLogout) { diff --git a/Overlord-Server/public/metrics.html b/Overlord-Server/public/metrics.html index 6114fda..1fc48f1 100644 --- a/Overlord-Server/public/metrics.html +++ b/Overlord-Server/public/metrics.html @@ -145,6 +145,101 @@ + +
+
+

+ + Network Throughput +

+
+ +
+
+ +
+

+ + HTTP Requests +

+
+ +
+
+ +
+

+ + HTTP Latency +

+
+ +
+
+ +
+

+ + Runtime Memory +

+
+ +
+
+ +
+

+ + Event Loop Lag +

+
+ +
+
+
+ + +
+
+

+ + Slowest HTTP Routes +

+
Sorted by p95 latency, last minute
+
+
+
+ No HTTP route samples yet +
+
+
+ + +
+
+

+ + Session Mix +

+
+ +
+
+ +
+

+ + Operating Systems +

+
+ +
+
+
+
@@ -181,36 +276,22 @@
- +

- - Ping Statistics + + Clients by OS

-
-
- Average: - - -
-
- Min / Max: - - / - - -
-
- Samples: - 0 -
+
+
No data
- +

- HTTP & Event Loop + HTTP, Ping & Event Loop

@@ -222,10 +303,11 @@ 0
- Latency avg/p95: + Latency avg/p95/p99: 0 / - 0 + 0 / + 0
@@ -239,17 +321,21 @@ Loop lag max: 0
-
-
- - -
-

- - Clients by OS -

-
-
No data
+
+ Ping avg: + - +
+
+ Ping min/max: + + - / + - + +
+
+ Ping samples: + 0 +
diff --git a/Overlord-Server/src/metrics.test.ts b/Overlord-Server/src/metrics.test.ts index f9c059f..2fc6bab 100644 --- a/Overlord-Server/src/metrics.test.ts +++ b/Overlord-Server/src/metrics.test.ts @@ -125,6 +125,31 @@ describe("MetricsCollector", () => { const snap = metrics.getSnapshot(); expect(snap.http.latencyAvg).toBeCloseTo(50.5, 0); expect(snap.http.latencyP95).toBeGreaterThanOrEqual(90); + expect(snap.http.latencyP99).toBeGreaterThanOrEqual(99); + }); + + test("tracks slow HTTP routes", () => { + metrics.recordHttpRequest(15, 200, "GET /api/metrics"); + metrics.recordHttpRequest(120, 200, "GET /api/clients/:id/rd/ws"); + metrics.recordHttpRequest(200, 500, "GET /api/clients/:id/rd/ws"); + + const snap = metrics.getSnapshot(); + expect(snap.http.routes[0].route).toBe("GET /api/clients/:id/rd/ws"); + expect(snap.http.routes[0].countLastMinute).toBe(2); + expect(snap.http.routes[0].errorsLastMinute).toBe(1); + expect(snap.http.routes[0].latencyP95).toBeGreaterThanOrEqual(120); + }); + + test("does not count the metrics dashboard poll as HTTP pressure", () => { + metrics.recordHttpRequest(100, 200, "GET /api/metrics"); + metrics.recordHttpRequest(20, 200, "GET /api/clients"); + + const snap = metrics.getSnapshot(); + expect(snap.http.total).toBe(1); + expect(snap.http.lastMinute).toBe(1); + expect(snap.http.routes.map((route) => route.route)).toEqual([ + "GET /api/clients", + ]); }); }); @@ -177,6 +202,26 @@ describe("MetricsCollector", () => { expect(history[0].timestamp).toBe(snap.timestamp); }); + test("history captures pressure metrics for charting", () => { + metrics.recordHttpRequest(25, 200); + metrics.recordHttpRequest(80, 500); + metrics.recordBytesSent(1024); + metrics.recordBytesReceived(2048); + + const snap = metrics.getSnapshot(); + metrics.recordHistoryEntry(snap); + + const [entry] = metrics.getHistory(); + expect(entry.httpRequestsPerMinute).toBeGreaterThanOrEqual(2); + expect(entry.httpErrorsPerMinute).toBeGreaterThanOrEqual(1); + expect(entry.httpLatencyAvg).toBeGreaterThan(0); + expect(entry.httpLatencyP95).toBeGreaterThan(0); + expect(entry.heapUsed).toBeGreaterThan(0); + expect(entry.rss).toBeGreaterThan(0); + expect(entry.systemMemoryUsedPercent).toBeGreaterThanOrEqual(0); + expect(entry.activeSessions).toBe(0); + }); + test("history is capped at maxHistoryPoints", () => { const weekOfFiveSecondSamples = 7 * 24 * 60 * 12; for (let i = 0; i < weekOfFiveSecondSamples + 180; i++) { diff --git a/Overlord-Server/src/metrics.ts b/Overlord-Server/src/metrics.ts index 4e56101..36a3d48 100644 --- a/Overlord-Server/src/metrics.ts +++ b/Overlord-Server/src/metrics.ts @@ -59,6 +59,8 @@ export interface MetricsSnapshot { lastMinuteErrors: number; latencyAvg: number; latencyP95: number; + latencyP99: number; + routes: HttpRouteStats[]; }; eventLoop: { avg: number; @@ -67,12 +69,41 @@ export interface MetricsSnapshot { }; } +export interface HttpRouteStats { + route: string; + countLastMinute: number; + errorsLastMinute: number; + latencyAvg: number; + latencyP95: number; + latencyP99: number; + latencyMax: number; + lastDuration: number; + lastStatus: number; +} + export interface MetricsHistory { timestamp: number; clientsOnline: number; commandsPerMinute: number; bandwidthSent: number; bandwidthReceived: number; + httpRequestsPerMinute?: number; + httpErrorsPerMinute?: number; + httpLatencyAvg?: number; + httpLatencyP95?: number; + httpLatencyP99?: number; + eventLoopAvg?: number; + eventLoopP95?: number; + heapUsed?: number; + rss?: number; + systemMemoryUsedPercent?: number; + activeSessions?: number; +} + +interface TimedHttpSample { + ts: number; + duration: number; + statusCode: number; } class MetricsCollector { @@ -103,8 +134,11 @@ class MetricsCollector { private httpTotal: number = 0; private httpTimestamps: number[] = []; private httpErrorTimestamps: number[] = []; - private httpLatencies: number[] = []; - private maxHttpLatencyHistory: number = 1000; + private httpSamples: TimedHttpSample[] = []; + private httpRouteSamples: Map = new Map(); + private maxHttpLatencyHistory: number = 2000; + private maxHttpRouteSamples: number = 300; + private ignoredHttpMetricRoutes: Set = new Set(["GET /api/metrics"]); private eventLoopDelays: number[] = []; private maxEventLoopHistory: number = 300; @@ -236,7 +270,74 @@ class MetricsCollector { }, intervalMs); } - recordHttpRequest(durationMs: number, statusCode: number) { + private percentile(samples: number[], percentile: number): number { + if (samples.length === 0) return 0; + const index = Math.max(0, Math.ceil(samples.length * percentile) - 1); + return samples[index] ?? 0; + } + + private average(samples: number[]): number { + return samples.length + ? samples.reduce((sum, value) => sum + value, 0) / samples.length + : 0; + } + + private pruneHttpSamples(list: TimedHttpSample[], minTs: number): void { + let removeCount = 0; + while (removeCount < list.length && list[removeCount].ts <= minTs) { + removeCount += 1; + } + if (removeCount > 0) { + list.splice(0, removeCount); + } + } + + private countHttpErrors(samples: TimedHttpSample[]): number { + let count = 0; + for (const sample of samples) { + if (sample.statusCode >= 400) count += 1; + } + return count; + } + + private getTopHttpRoutes(minTs: number): HttpRouteStats[] { + const routes: HttpRouteStats[] = []; + for (const [route, samples] of this.httpRouteSamples.entries()) { + this.pruneHttpSamples(samples, minTs); + if (samples.length === 0) { + this.httpRouteSamples.delete(route); + continue; + } + + const durations = samples + .map((sample) => sample.duration) + .sort((a, b) => a - b); + const last = samples[samples.length - 1]; + routes.push({ + route, + countLastMinute: samples.length, + errorsLastMinute: this.countHttpErrors(samples), + latencyAvg: this.average(durations), + latencyP95: this.percentile(durations, 0.95), + latencyP99: this.percentile(durations, 0.99), + latencyMax: durations[durations.length - 1] ?? 0, + lastDuration: last?.duration ?? 0, + lastStatus: last?.statusCode ?? 0, + }); + } + + return routes + .sort((a, b) => { + if (b.latencyP95 !== a.latencyP95) return b.latencyP95 - a.latencyP95; + if (b.latencyAvg !== a.latencyAvg) return b.latencyAvg - a.latencyAvg; + return b.countLastMinute - a.countLastMinute; + }) + .slice(0, 8); + } + + recordHttpRequest(durationMs: number, statusCode: number, route = "unknown") { + if (this.ignoredHttpMetricRoutes.has(route)) return; + this.httpTotal++; const now = Date.now(); this.httpTimestamps.push(now); @@ -246,14 +347,25 @@ class MetricsCollector { this.pruneTimestampWindow(this.httpErrorTimestamps, now - 60000); } if (Number.isFinite(durationMs)) { - this.httpLatencies.push(durationMs); - if (this.httpLatencies.length > this.maxHttpLatencyHistory) { - this.httpLatencies.shift(); + const sample = { ts: now, duration: Math.max(0, durationMs), statusCode }; + this.httpSamples.push(sample); + if (this.httpSamples.length > this.maxHttpLatencyHistory) { + this.httpSamples.splice(0, this.httpSamples.length - this.maxHttpLatencyHistory); } + + const routeSamples = this.httpRouteSamples.get(route) || []; + routeSamples.push(sample); + if (routeSamples.length > this.maxHttpRouteSamples) { + routeSamples.splice(0, routeSamples.length - this.maxHttpRouteSamples); + } + this.httpRouteSamples.set(route, routeSamples); } } - async withHttpMetrics(handler: () => Promise): Promise { + async withHttpMetrics( + handler: () => Promise, + route = "unknown", + ): Promise { const start = typeof performance !== "undefined" && performance.now ? performance.now() : Date.now(); @@ -262,13 +374,13 @@ class MetricsCollector { const end = typeof performance !== "undefined" && performance.now ? performance.now() : Date.now(); - this.recordHttpRequest(end - start, response?.status ?? 0); + this.recordHttpRequest(end - start, response?.status ?? 0, route); return response; } catch (err) { const end = typeof performance !== "undefined" && performance.now ? performance.now() : Date.now(); - this.recordHttpRequest(end - start, 500); + this.recordHttpRequest(end - start, 500, route); throw err; } } @@ -280,6 +392,21 @@ class MetricsCollector { commandsPerMinute: snapshot.commands.lastMinute, bandwidthSent: this.sentPerSecond, bandwidthReceived: this.receivedPerSecond, + httpRequestsPerMinute: snapshot.http.lastMinute, + httpErrorsPerMinute: snapshot.http.lastMinuteErrors, + httpLatencyAvg: snapshot.http.latencyAvg, + httpLatencyP95: snapshot.http.latencyP95, + httpLatencyP99: snapshot.http.latencyP99, + eventLoopAvg: snapshot.eventLoop.avg, + eventLoopP95: snapshot.eventLoop.p95, + heapUsed: snapshot.server.memoryUsage.heapUsed, + rss: snapshot.server.memoryUsage.rss, + systemMemoryUsedPercent: snapshot.server.systemMemory.usedPercent, + activeSessions: + snapshot.sessions.console + + snapshot.sessions.remoteDesktop + + snapshot.sessions.fileBrowser + + snapshot.sessions.process, }; this.history.push(historyEntry); @@ -297,6 +424,7 @@ class MetricsCollector { this.pruneTimestampWindow(this.commandTimestamps, oneHourAgo); this.pruneTimestampWindow(this.httpTimestamps, oneMinuteAgo); this.pruneTimestampWindow(this.httpErrorTimestamps, oneMinuteAgo); + this.pruneHttpSamples(this.httpSamples, oneMinuteAgo); const commandsLastMinute = this.countRecent(this.commandTimestamps, oneMinuteAgo); const commandsLastHour = this.commandTimestamps.length; @@ -309,16 +437,13 @@ class MetricsCollector { const httpLastMinute = this.httpTimestamps.length; const httpErrorsLastMinute = this.httpErrorTimestamps.length; - const httpLatencySamples = [...this.httpLatencies].sort((a, b) => a - b); - const httpLatencyAvg = httpLatencySamples.length - ? httpLatencySamples.reduce((a, b) => a + b, 0) / httpLatencySamples.length - : 0; - const httpP95Index = httpLatencySamples.length - ? Math.max(0, Math.floor(httpLatencySamples.length * 0.95) - 1) - : 0; - const httpLatencyP95 = httpLatencySamples.length - ? httpLatencySamples[httpP95Index] ?? 0 - : 0; + const httpLatencySamples = this.httpSamples + .map((sample) => sample.duration) + .sort((a, b) => a - b); + const httpLatencyAvg = this.average(httpLatencySamples); + const httpLatencyP95 = this.percentile(httpLatencySamples, 0.95); + const httpLatencyP99 = this.percentile(httpLatencySamples, 0.99); + const httpRoutes = this.getTopHttpRoutes(oneMinuteAgo); const eventLoopSamples = [...this.eventLoopDelays].sort((a, b) => a - b); const eventLoopAvg = eventLoopSamples.length @@ -400,6 +525,8 @@ class MetricsCollector { lastMinuteErrors: httpErrorsLastMinute, latencyAvg: httpLatencyAvg, latencyP95: httpLatencyP95, + latencyP99: httpLatencyP99, + routes: httpRoutes, }, eventLoop: { avg: eventLoopAvg, @@ -428,7 +555,8 @@ class MetricsCollector { this.httpTotal = 0; this.httpTimestamps = []; this.httpErrorTimestamps = []; - this.httpLatencies = []; + this.httpSamples = []; + this.httpRouteSamples.clear(); this.eventLoopDelays = []; } } diff --git a/Overlord-Server/src/server/http-dispatch.test.ts b/Overlord-Server/src/server/http-dispatch.test.ts index e767f69..dd86673 100644 --- a/Overlord-Server/src/server/http-dispatch.test.ts +++ b/Overlord-Server/src/server/http-dispatch.test.ts @@ -1,6 +1,6 @@ import { beforeEach, describe, expect, test } from "bun:test"; import { clearRequestRateLimitsForTests } from "../rateLimit"; -import { createHttpFetchHandler } from "./http-dispatch"; +import { createHttpFetchHandler, normalizeHttpRoute } from "./http-dispatch"; const metrics = { withHttpMetrics: (fn: () => Promise) => fn(), @@ -35,3 +35,22 @@ describe("createHttpFetchHandler rate limiting", () => { expect(limited.headers.get("Retry-After")).toBeTruthy(); }); }); + +describe("normalizeHttpRoute", () => { + test("collapses dynamic route segments", () => { + const req = new Request( + "https://localhost/api/clients/0a44a8d4d61c7840540b43fd187fe3c43e2c46c9600b2aa1a61a6b1ab03c5b92/rd/ws?cache=1", + { method: "POST" }, + ); + + expect(normalizeHttpRoute(req, new URL(req.url))).toBe( + "POST /api/clients/:id/rd/ws", + ); + }); + + test("groups static asset files", () => { + const req = new Request("https://localhost/assets/metrics.js"); + + expect(normalizeHttpRoute(req, new URL(req.url))).toBe("GET /assets/:file"); + }); +}); diff --git a/Overlord-Server/src/server/http-dispatch.ts b/Overlord-Server/src/server/http-dispatch.ts index d1356f3..bd6cd95 100644 --- a/Overlord-Server/src/server/http-dispatch.ts +++ b/Overlord-Server/src/server/http-dispatch.ts @@ -3,6 +3,9 @@ import { consumeUnauthorizedRateLimit } from "../rateLimit"; export type RouteHandler = (req: Request, url: URL, server: unknown) => Promise; +const UUID_SEGMENT = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; +const LONG_ID_SEGMENT = /^[0-9a-f]{16,}$/i; + function tooManyRequestsResponse(retryAfter = 60): Response { return new Response("Too many requests", { status: 429, @@ -13,17 +16,35 @@ function tooManyRequestsResponse(retryAfter = 60): Response { }); } +function normalizeRouteSegment(segment: string): string { + if (/^\d+$/.test(segment)) return ":id"; + if (UUID_SEGMENT.test(segment) || LONG_ID_SEGMENT.test(segment)) return ":id"; + if (segment.includes(".") && !segment.startsWith(".")) return ":file"; + if (segment.length >= 24 && /^[a-z0-9_-]+$/i.test(segment)) return ":id"; + return segment; +} + +export function normalizeHttpRoute(req: Request, url: URL): string { + const path = url.pathname + .split("/") + .filter(Boolean) + .map(normalizeRouteSegment) + .join("/"); + return `${req.method.toUpperCase()} /${path}`; +} + export function createHttpFetchHandler(deps: { - metrics: { withHttpMetrics: (fn: () => Promise) => Promise }; + metrics: { withHttpMetrics: (fn: () => Promise, route?: string) => Promise }; CORS_HEADERS: Record; routes: RouteHandler[]; }) { return async function fetchHandler(req: Request, server: unknown): Promise { + const url = new URL(req.url); + const routeLabel = normalizeHttpRoute(req, url); return deps.metrics.withHttpMetrics(async () => { if (req.method === "OPTIONS") { return new Response("", { headers: deps.CORS_HEADERS }); } - const url = new URL(req.url); const wrapped = wrapServerWithClientIp(server as RequestServerLike); const ip = wrapped.requestIP(req)?.address || "unknown"; @@ -38,6 +59,6 @@ export function createHttpFetchHandler(deps: { } } return new Response("Not found", { status: 404 }); - }); + }, routeLabel); }; } diff --git a/Overlord-Server/src/server/routes/misc-routes.ts b/Overlord-Server/src/server/routes/misc-routes.ts index 48c186a..c4965c1 100644 --- a/Overlord-Server/src/server/routes/misc-routes.ts +++ b/Overlord-Server/src/server/routes/misc-routes.ts @@ -330,7 +330,11 @@ export async function handleMiscRoutes( snapshot.sessions.fileBrowser = deps.getFileBrowserSessionCount(); snapshot.sessions.process = deps.getProcessSessionCount(); - const history = metrics.getHistory(); + const requestedHistoryLimit = Number(url.searchParams.get("historyLimit") || 240); + const historyLimit = Number.isFinite(requestedHistoryLimit) + ? Math.max(1, Math.min(2000, requestedHistoryLimit)) + : 240; + const history = metrics.getHistory().slice(-historyLimit); return new Response(JSON.stringify({ snapshot, history }), { headers: { "Content-Type": "application/json" },