more tests and better metrics

This commit is contained in:
KingKDot
2026-06-04 16:35:06 -05:00
parent 8240dd61f1
commit d9d9507828
7 changed files with 842 additions and 67 deletions
+480 -8
View File
@@ -1,5 +1,12 @@
let clientsChart = null;
let commandsChart = null;
let bandwidthChart = null;
let httpRequestsChart = null;
let httpChart = null;
let memoryChart = null;
let eventLoopChart = null;
let sessionsChart = null;
let osChart = null;
let countryMap = null;
let countriesLayer = null;
let countryCounts = {};
@@ -7,6 +14,8 @@ let maxCountryCount = 0;
let latestByCountry = {};
const GEOJSON_URL = "/vendor/geo-countries/countries.geojson";
const MAX_CHART_POINTS = 240;
const METRICS_POLL_INTERVAL_MS = 5000;
if (typeof Chart !== "undefined") {
Chart.defaults.color = "#cbd5e1";
@@ -14,9 +23,69 @@ if (typeof Chart !== "undefined") {
Chart.defaults.font.family = "Inter, system-ui, sans-serif";
}
function chartTooltip() {
return {
backgroundColor: "#0f172a",
titleColor: "#e2e8f0",
bodyColor: "#cbd5e1",
borderColor: "#334155",
borderWidth: 1,
};
}
function lineChartOptions(extra = {}) {
const extraPlugins = extra.plugins || {};
return {
responsive: true,
maintainAspectRatio: false,
animation: false,
interaction: { mode: "index", intersect: false },
scales: {
y: {
beginAtZero: true,
ticks: { color: "#94a3b8" },
grid: { color: "rgba(100, 116, 139, 0.1)" },
},
x: {
ticks: {
color: "#94a3b8",
maxTicksLimit: 6,
autoSkip: true,
maxRotation: 0,
},
grid: { color: "rgba(100, 116, 139, 0.1)" },
},
...(extra.scales || {}),
},
...extra,
plugins: {
legend:
extraPlugins.legend ?? { display: true, labels: { boxWidth: 10, boxHeight: 10 } },
tooltip: chartTooltip(),
...extraPlugins,
},
};
}
function makeLineChart(canvas, datasets, options = {}) {
if (!canvas) return null;
return new Chart(canvas, {
type: "line",
data: { labels: [], datasets },
options: lineChartOptions(options),
});
}
function initCharts() {
const clientsCtx = document.getElementById("clients-chart");
const commandsCtx = document.getElementById("commands-chart");
const bandwidthCtx = document.getElementById("bandwidth-chart");
const httpRequestsCtx = document.getElementById("http-requests-chart");
const httpCtx = document.getElementById("http-chart");
const memoryCtx = document.getElementById("memory-chart");
const eventLoopCtx = document.getElementById("event-loop-chart");
const sessionsCtx = document.getElementById("sessions-chart");
const osCtx = document.getElementById("os-chart");
clientsChart = new Chart(clientsCtx, {
type: "line",
@@ -119,13 +188,283 @@ function initCharts() {
},
},
});
bandwidthChart = makeLineChart(
bandwidthCtx,
[
{
label: "Sent/s",
data: [],
borderColor: "rgb(251, 146, 60)",
backgroundColor: "rgba(251, 146, 60, 0.12)",
fill: true,
tension: 0.35,
pointRadius: 0,
borderWidth: 2,
},
{
label: "Received/s",
data: [],
borderColor: "rgb(56, 189, 248)",
backgroundColor: "rgba(56, 189, 248, 0.1)",
fill: true,
tension: 0.35,
pointRadius: 0,
borderWidth: 2,
},
],
{
scales: {
y: {
beginAtZero: true,
ticks: {
color: "#94a3b8",
callback: (value) => formatBytes(Number(value)),
},
grid: { color: "rgba(100, 116, 139, 0.1)" },
},
},
},
);
httpRequestsChart = makeLineChart(
httpRequestsCtx,
[
{
label: "requests/min",
data: [],
borderColor: "rgb(52, 211, 153)",
backgroundColor: "rgba(52, 211, 153, 0.12)",
fill: true,
tension: 0.35,
pointRadius: 0,
borderWidth: 2,
},
{
label: "errors/min",
data: [],
borderColor: "rgb(248, 113, 113)",
backgroundColor: "rgba(248, 113, 113, 0.14)",
fill: false,
tension: 0.25,
pointRadius: 0,
borderDash: [5, 4],
borderWidth: 2,
},
],
{
scales: {
y: {
beginAtZero: true,
ticks: { color: "#94a3b8", precision: 0 },
grid: { color: "rgba(100, 116, 139, 0.1)" },
},
},
},
);
httpChart = makeLineChart(
httpCtx,
[
{
label: "p99 ms",
data: [],
borderColor: "rgb(248, 113, 113)",
backgroundColor: "rgba(248, 113, 113, 0.08)",
fill: false,
tension: 0.35,
pointRadius: 0,
borderDash: [4, 4],
borderWidth: 2,
},
{
label: "p95 ms",
data: [],
borderColor: "rgb(244, 63, 94)",
backgroundColor: "rgba(244, 63, 94, 0.1)",
fill: true,
tension: 0.35,
pointRadius: 0,
borderWidth: 2,
},
{
label: "avg ms",
data: [],
borderColor: "rgb(251, 191, 36)",
backgroundColor: "rgba(251, 191, 36, 0.08)",
fill: false,
tension: 0.35,
pointRadius: 0,
borderWidth: 2,
},
],
{
scales: {
y: {
beginAtZero: true,
ticks: { color: "#94a3b8", callback: (value) => `${Math.round(Number(value))} ms` },
grid: { color: "rgba(100, 116, 139, 0.1)" },
},
},
},
);
memoryChart = makeLineChart(
memoryCtx,
[
{
label: "Heap",
data: [],
borderColor: "rgb(34, 211, 238)",
backgroundColor: "rgba(34, 211, 238, 0.1)",
fill: true,
tension: 0.35,
pointRadius: 0,
yAxisID: "y",
borderWidth: 2,
},
{
label: "RSS",
data: [],
borderColor: "rgb(129, 140, 248)",
backgroundColor: "rgba(129, 140, 248, 0.08)",
fill: true,
tension: 0.35,
pointRadius: 0,
yAxisID: "y",
borderWidth: 2,
},
{
label: "System %",
data: [],
borderColor: "rgb(52, 211, 153)",
backgroundColor: "rgba(52, 211, 153, 0.08)",
fill: false,
tension: 0.35,
pointRadius: 0,
yAxisID: "y1",
borderWidth: 2,
},
],
{
scales: {
y: {
beginAtZero: true,
position: "left",
ticks: { color: "#94a3b8", callback: (value) => formatBytes(Number(value)) },
grid: { color: "rgba(100, 116, 139, 0.1)" },
},
y1: {
beginAtZero: true,
max: 100,
position: "right",
ticks: { color: "#86efac", callback: (value) => `${Math.round(Number(value))}%` },
grid: { drawOnChartArea: false },
},
},
},
);
eventLoopChart = makeLineChart(
eventLoopCtx,
[
{
label: "p95 lag",
data: [],
borderColor: "rgb(251, 191, 36)",
backgroundColor: "rgba(251, 191, 36, 0.12)",
fill: true,
tension: 0.35,
pointRadius: 0,
borderWidth: 2,
},
{
label: "avg lag",
data: [],
borderColor: "rgb(96, 165, 250)",
backgroundColor: "rgba(96, 165, 250, 0.08)",
fill: false,
tension: 0.35,
pointRadius: 0,
borderWidth: 2,
},
],
{
scales: {
y: {
beginAtZero: true,
ticks: { color: "#94a3b8", callback: (value) => `${Math.round(Number(value))} ms` },
grid: { color: "rgba(100, 116, 139, 0.1)" },
},
},
},
);
sessionsChart = sessionsCtx ? new Chart(sessionsCtx, {
type: "doughnut",
data: {
labels: ["Console", "Remote Desktop", "Files", "Processes"],
datasets: [{
data: [0, 0, 0, 0],
backgroundColor: ["#34d399", "#c084fc", "#60a5fa", "#fb923c"],
borderColor: "#0f172a",
borderWidth: 2,
}],
},
options: {
responsive: true,
maintainAspectRatio: false,
animation: false,
cutout: "64%",
plugins: {
legend: { position: "right", labels: { boxWidth: 10, boxHeight: 10 } },
tooltip: chartTooltip(),
},
},
}) : null;
osChart = osCtx ? new Chart(osCtx, {
type: "bar",
data: {
labels: [],
datasets: [{
label: "Clients",
data: [],
backgroundColor: "rgba(56, 189, 248, 0.5)",
borderColor: "rgb(56, 189, 248)",
borderWidth: 1,
}],
},
options: {
indexAxis: "y",
responsive: true,
maintainAspectRatio: false,
animation: false,
plugins: {
legend: { display: false },
tooltip: chartTooltip(),
},
scales: {
x: {
beginAtZero: true,
ticks: { color: "#94a3b8", precision: 0 },
grid: { color: "rgba(100, 116, 139, 0.1)" },
},
y: {
ticks: { color: "#94a3b8" },
grid: { display: false },
},
},
},
}) : null;
}
function formatBytes(bytes) {
if (bytes === 0) return "0 B";
if (!Number.isFinite(bytes) || bytes <= 0) return "0 B";
if (bytes < 1) return `${bytes.toFixed(2)} B`;
const k = 1024;
const sizes = ["B", "KB", "MB", "GB", "TB"];
const i = Math.floor(Math.log(bytes) / Math.log(k));
const i = Math.min(sizes.length - 1, Math.floor(Math.log(bytes) / Math.log(k)));
return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + " " + sizes[i];
}
@@ -269,6 +608,10 @@ function updateMetrics(data, debug) {
if (httpLatencyP95El) {
httpLatencyP95El.textContent = formatMs(data.http.latencyP95 || 0);
}
const httpLatencyP99El = document.getElementById("http-latency-p99");
if (httpLatencyP99El) {
httpLatencyP99El.textContent = formatMs(data.http.latencyP99 || 0);
}
const eventLoopAvgEl = document.getElementById("event-loop-avg");
if (eventLoopAvgEl) {
eventLoopAvgEl.textContent = formatMs(data.eventLoop.avg || 0);
@@ -314,6 +657,57 @@ function updateMetrics(data, debug) {
osList.innerHTML = '<div class="text-slate-500">No clients</div>';
}
if (sessionsChart) {
sessionsChart.data.datasets[0].data = [
data.sessions.console || 0,
data.sessions.remoteDesktop || 0,
data.sessions.fileBrowser || 0,
data.sessions.process || 0,
];
sessionsChart.update("none");
}
if (osChart) {
const osEntries = Object.entries(data.clients.byOS || {})
.sort((a, b) => b[1] - a[1])
.slice(0, 10);
osChart.data.labels = osEntries.length ? osEntries.map(([label]) => label) : ["No clients"];
osChart.data.datasets[0].data = osEntries.length ? osEntries.map(([, count]) => count) : [0];
osChart.update("none");
}
const httpRoutesList = document.getElementById("http-routes");
if (httpRoutesList) {
const routes = Array.isArray(data.http.routes) ? data.http.routes : [];
if (routes.length > 0) {
httpRoutesList.innerHTML = routes
.map((route) => {
const errorClass = route.errorsLastMinute > 0 ? "text-red-300" : "text-slate-400";
return `
<div class="bg-slate-800/50 rounded p-3 min-w-0">
<div class="text-xs text-slate-400 mb-2 truncate" title="${escapeHtml(route.route)}">
${escapeHtml(route.route)}
</div>
<div class="grid grid-cols-2 gap-x-3 gap-y-1 text-xs">
<span class="text-slate-500">p95</span>
<span class="font-semibold text-right">${formatMs(route.latencyP95 || 0)}</span>
<span class="text-slate-500">avg</span>
<span class="font-semibold text-right">${formatMs(route.latencyAvg || 0)}</span>
<span class="text-slate-500">req/min</span>
<span class="font-semibold text-right">${Number(route.countLastMinute || 0).toLocaleString()}</span>
<span class="text-slate-500">errors</span>
<span class="font-semibold text-right ${errorClass}">${Number(route.errorsLastMinute || 0).toLocaleString()}</span>
</div>
</div>
`;
})
.join("");
} else {
httpRoutesList.innerHTML =
'<div class="text-slate-500 col-span-full text-center py-4">No HTTP route samples yet</div>';
}
}
const commandTypesList = document.getElementById("command-types");
if (Object.keys(data.commands.byType).length > 0) {
const topCommands = Object.entries(data.commands.byType)
@@ -462,7 +856,11 @@ async function initCountryMap() {
}
function updateCharts(history, snapshot) {
if (history.length === 0) {
const points = Array.isArray(history) && history.length > 0
? history.slice(-MAX_CHART_POINTS)
: [];
if (points.length === 0) {
const now = new Date();
const label = formatTime(now.getTime());
@@ -473,26 +871,100 @@ function updateCharts(history, snapshot) {
commandsChart.data.labels = [label];
commandsChart.data.datasets[0].data = [snapshot.commands.lastMinute];
commandsChart.update("none");
if (bandwidthChart) {
bandwidthChart.data.labels = [label];
bandwidthChart.data.datasets[0].data = [snapshot.bandwidth.sentPerSecond || 0];
bandwidthChart.data.datasets[1].data = [snapshot.bandwidth.receivedPerSecond || 0];
bandwidthChart.update("none");
}
if (httpChart) {
httpChart.data.labels = [label];
httpChart.data.datasets[0].data = [snapshot.http.latencyP99 || 0];
httpChart.data.datasets[1].data = [snapshot.http.latencyP95 || 0];
httpChart.data.datasets[2].data = [snapshot.http.latencyAvg || 0];
httpChart.update("none");
}
if (httpRequestsChart) {
httpRequestsChart.data.labels = [label];
httpRequestsChart.data.datasets[0].data = [snapshot.http.lastMinute || 0];
httpRequestsChart.data.datasets[1].data = [snapshot.http.lastMinuteErrors || 0];
httpRequestsChart.update("none");
}
if (memoryChart) {
memoryChart.data.labels = [label];
memoryChart.data.datasets[0].data = [snapshot.server.memoryUsage.heapUsed || 0];
memoryChart.data.datasets[1].data = [snapshot.server.memoryUsage.rss || 0];
memoryChart.data.datasets[2].data = [snapshot.server.systemMemory.usedPercent || 0];
memoryChart.update("none");
}
if (eventLoopChart) {
eventLoopChart.data.labels = [label];
eventLoopChart.data.datasets[0].data = [snapshot.eventLoop.p95 || 0];
eventLoopChart.data.datasets[1].data = [snapshot.eventLoop.avg || 0];
eventLoopChart.update("none");
}
return;
}
const labels = history.map((h) => formatTime(h.timestamp));
const clientsData = history.map((h) => h.clientsOnline);
const labels = points.map((h) => formatTime(h.timestamp));
const clientsData = points.map((h) => h.clientsOnline || 0);
clientsChart.data.labels = labels;
clientsChart.data.datasets[0].data = clientsData;
clientsChart.update("none");
const commandsData = history.map((h) => h.commandsPerMinute);
const commandsData = points.map((h) => h.commandsPerMinute || 0);
commandsChart.data.labels = labels;
commandsChart.data.datasets[0].data = commandsData;
commandsChart.update("none");
if (bandwidthChart) {
bandwidthChart.data.labels = labels;
bandwidthChart.data.datasets[0].data = points.map((h) => h.bandwidthSent || 0);
bandwidthChart.data.datasets[1].data = points.map((h) => h.bandwidthReceived || 0);
bandwidthChart.update("none");
}
if (httpChart) {
httpChart.data.labels = labels;
httpChart.data.datasets[0].data = points.map((h) => h.httpLatencyP99 || 0);
httpChart.data.datasets[1].data = points.map((h) => h.httpLatencyP95 || 0);
httpChart.data.datasets[2].data = points.map((h) => h.httpLatencyAvg || 0);
httpChart.update("none");
}
if (httpRequestsChart) {
httpRequestsChart.data.labels = labels;
httpRequestsChart.data.datasets[0].data = points.map((h) => h.httpRequestsPerMinute || 0);
httpRequestsChart.data.datasets[1].data = points.map((h) => h.httpErrorsPerMinute || 0);
httpRequestsChart.update("none");
}
if (memoryChart) {
memoryChart.data.labels = labels;
memoryChart.data.datasets[0].data = points.map((h) => h.heapUsed || 0);
memoryChart.data.datasets[1].data = points.map((h) => h.rss || 0);
memoryChart.data.datasets[2].data = points.map((h) => h.systemMemoryUsedPercent || 0);
memoryChart.update("none");
}
if (eventLoopChart) {
eventLoopChart.data.labels = labels;
eventLoopChart.data.datasets[0].data = points.map((h) => h.eventLoopP95 || 0);
eventLoopChart.data.datasets[1].data = points.map((h) => h.eventLoopAvg || 0);
eventLoopChart.update("none");
}
}
async function fetchMetrics() {
try {
const response = await fetch("/api/metrics", {
const response = await fetch(`/api/metrics?historyLimit=${MAX_CHART_POINTS}`, {
credentials: "include",
});
@@ -589,7 +1061,7 @@ document.addEventListener("DOMContentLoaded", async () => {
await fetchMetrics();
setInterval(fetchMetrics, 2000);
setInterval(fetchMetrics, METRICS_POLL_INTERVAL_MS);
const logoutBtn = document.getElementById("logout-btn");
if (logoutBtn && !logoutBtn.dataset.boundLogout) {
+120 -34
View File
@@ -145,6 +145,101 @@
</div>
</div>
<!-- Operational Pressure Charts -->
<div class="grid grid-cols-1 lg:grid-cols-2 gap-4">
<div class="bg-slate-900/70 border border-slate-800 rounded-lg p-4">
<h3 class="text-lg font-semibold mb-4 flex items-center gap-2">
<i class="fa-solid fa-network-wired text-orange-400"></i>
Network Throughput
</h3>
<div class="h-64">
<canvas id="bandwidth-chart"></canvas>
</div>
</div>
<div class="bg-slate-900/70 border border-slate-800 rounded-lg p-4">
<h3 class="text-lg font-semibold mb-4 flex items-center gap-2">
<i class="fa-solid fa-arrow-right-arrow-left text-emerald-400"></i>
HTTP Requests
</h3>
<div class="h-64">
<canvas id="http-requests-chart"></canvas>
</div>
</div>
<div class="bg-slate-900/70 border border-slate-800 rounded-lg p-4">
<h3 class="text-lg font-semibold mb-4 flex items-center gap-2">
<i class="fa-solid fa-gauge-high text-rose-400"></i>
HTTP Latency
</h3>
<div class="h-64">
<canvas id="http-chart"></canvas>
</div>
</div>
<div class="bg-slate-900/70 border border-slate-800 rounded-lg p-4">
<h3 class="text-lg font-semibold mb-4 flex items-center gap-2">
<i class="fa-solid fa-memory text-cyan-400"></i>
Runtime Memory
</h3>
<div class="h-64">
<canvas id="memory-chart"></canvas>
</div>
</div>
<div class="bg-slate-900/70 border border-slate-800 rounded-lg p-4">
<h3 class="text-lg font-semibold mb-4 flex items-center gap-2">
<i class="fa-solid fa-stopwatch text-amber-400"></i>
Event Loop Lag
</h3>
<div class="h-64">
<canvas id="event-loop-chart"></canvas>
</div>
</div>
</div>
<!-- Slow HTTP Routes -->
<div class="bg-slate-900/70 border border-slate-800 rounded-lg p-4">
<div class="flex items-center justify-between gap-3 flex-wrap mb-3">
<h3 class="text-lg font-semibold flex items-center gap-2">
<i class="fa-solid fa-route text-rose-400"></i>
Slowest HTTP Routes
</h3>
<div class="text-xs text-slate-500">Sorted by p95 latency, last minute</div>
</div>
<div
id="http-routes"
class="grid grid-cols-1 md:grid-cols-2 xl:grid-cols-4 gap-3"
>
<div class="text-slate-500 col-span-full text-center py-4">
No HTTP route samples yet
</div>
</div>
</div>
<!-- Mix Charts -->
<div class="grid grid-cols-1 lg:grid-cols-2 gap-4">
<div class="bg-slate-900/70 border border-slate-800 rounded-lg p-4">
<h3 class="text-lg font-semibold mb-4 flex items-center gap-2">
<i class="fa-solid fa-plug text-emerald-400"></i>
Session Mix
</h3>
<div class="h-64">
<canvas id="sessions-chart"></canvas>
</div>
</div>
<div class="bg-slate-900/70 border border-slate-800 rounded-lg p-4">
<h3 class="text-lg font-semibold mb-4 flex items-center gap-2">
<i class="fa-brands fa-windows text-cyan-400"></i>
Operating Systems
</h3>
<div class="h-64">
<canvas id="os-chart"></canvas>
</div>
</div>
</div>
<!-- Additional Stats -->
<div class="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-3 gap-4">
<!-- Server Stats -->
@@ -181,36 +276,22 @@
</div>
</div>
<!-- Ping Stats -->
<!-- Clients by OS -->
<div class="bg-slate-900/70 border border-slate-800 rounded-lg p-4">
<h3 class="text-lg font-semibold mb-3 flex items-center gap-2">
<i class="fa-solid fa-signal text-blue-400"></i>
Ping Statistics
<i class="fa-brands fa-windows text-cyan-400"></i>
Clients by OS
</h3>
<div class="space-y-2 text-sm">
<div class="flex justify-between">
<span class="text-slate-400">Average:</span>
<span id="ping-avg">-</span>
</div>
<div class="flex justify-between">
<span class="text-slate-400">Min / Max:</span>
<span
><span id="ping-min">-</span> /
<span id="ping-max">-</span></span
>
</div>
<div class="flex justify-between">
<span class="text-slate-400">Samples:</span>
<span id="ping-count">0</span>
</div>
<div id="clients-by-os" class="space-y-2 text-sm">
<div class="text-slate-500">No data</div>
</div>
</div>
<!-- HTTP & Event Loop -->
<!-- HTTP, Ping & Event Loop -->
<div class="bg-slate-900/70 border border-slate-800 rounded-lg p-4">
<h3 class="text-lg font-semibold mb-3 flex items-center gap-2">
<i class="fa-solid fa-wave-square text-amber-400"></i>
HTTP & Event Loop
HTTP, Ping & Event Loop
</h3>
<div class="space-y-2 text-sm">
<div class="flex justify-between">
@@ -222,10 +303,11 @@
<span id="http-errors-minute">0</span>
</div>
<div class="flex justify-between">
<span class="text-slate-400">Latency avg/p95:</span>
<span class="text-slate-400">Latency avg/p95/p99:</span>
<span>
<span id="http-latency-avg">0</span> /
<span id="http-latency-p95">0</span>
<span id="http-latency-p95">0</span> /
<span id="http-latency-p99">0</span>
</span>
</div>
<div class="flex justify-between">
@@ -239,17 +321,21 @@
<span class="text-slate-400">Loop lag max:</span>
<span id="event-loop-max">0</span>
</div>
</div>
</div>
<!-- Client Distribution -->
<div class="bg-slate-900/70 border border-slate-800 rounded-lg p-4">
<h3 class="text-lg font-semibold mb-3 flex items-center gap-2">
<i class="fa-brands fa-windows text-cyan-400"></i>
Clients by OS
</h3>
<div id="clients-by-os" class="space-y-2 text-sm">
<div class="text-slate-500">No data</div>
<div class="flex justify-between">
<span class="text-slate-400">Ping avg:</span>
<span id="ping-avg">-</span>
</div>
<div class="flex justify-between">
<span class="text-slate-400">Ping min/max:</span>
<span>
<span id="ping-min">-</span> /
<span id="ping-max">-</span>
</span>
</div>
<div class="flex justify-between">
<span class="text-slate-400">Ping samples:</span>
<span id="ping-count">0</span>
</div>
</div>
</div>
</div>
+45
View File
@@ -125,6 +125,31 @@ describe("MetricsCollector", () => {
const snap = metrics.getSnapshot();
expect(snap.http.latencyAvg).toBeCloseTo(50.5, 0);
expect(snap.http.latencyP95).toBeGreaterThanOrEqual(90);
expect(snap.http.latencyP99).toBeGreaterThanOrEqual(99);
});
test("tracks slow HTTP routes", () => {
metrics.recordHttpRequest(15, 200, "GET /api/metrics");
metrics.recordHttpRequest(120, 200, "GET /api/clients/:id/rd/ws");
metrics.recordHttpRequest(200, 500, "GET /api/clients/:id/rd/ws");
const snap = metrics.getSnapshot();
expect(snap.http.routes[0].route).toBe("GET /api/clients/:id/rd/ws");
expect(snap.http.routes[0].countLastMinute).toBe(2);
expect(snap.http.routes[0].errorsLastMinute).toBe(1);
expect(snap.http.routes[0].latencyP95).toBeGreaterThanOrEqual(120);
});
test("does not count the metrics dashboard poll as HTTP pressure", () => {
metrics.recordHttpRequest(100, 200, "GET /api/metrics");
metrics.recordHttpRequest(20, 200, "GET /api/clients");
const snap = metrics.getSnapshot();
expect(snap.http.total).toBe(1);
expect(snap.http.lastMinute).toBe(1);
expect(snap.http.routes.map((route) => route.route)).toEqual([
"GET /api/clients",
]);
});
});
@@ -177,6 +202,26 @@ describe("MetricsCollector", () => {
expect(history[0].timestamp).toBe(snap.timestamp);
});
test("history captures pressure metrics for charting", () => {
metrics.recordHttpRequest(25, 200);
metrics.recordHttpRequest(80, 500);
metrics.recordBytesSent(1024);
metrics.recordBytesReceived(2048);
const snap = metrics.getSnapshot();
metrics.recordHistoryEntry(snap);
const [entry] = metrics.getHistory();
expect(entry.httpRequestsPerMinute).toBeGreaterThanOrEqual(2);
expect(entry.httpErrorsPerMinute).toBeGreaterThanOrEqual(1);
expect(entry.httpLatencyAvg).toBeGreaterThan(0);
expect(entry.httpLatencyP95).toBeGreaterThan(0);
expect(entry.heapUsed).toBeGreaterThan(0);
expect(entry.rss).toBeGreaterThan(0);
expect(entry.systemMemoryUsedPercent).toBeGreaterThanOrEqual(0);
expect(entry.activeSessions).toBe(0);
});
test("history is capped at maxHistoryPoints", () => {
const weekOfFiveSecondSamples = 7 * 24 * 60 * 12;
for (let i = 0; i < weekOfFiveSecondSamples + 180; i++) {
+148 -20
View File
@@ -59,6 +59,8 @@ export interface MetricsSnapshot {
lastMinuteErrors: number;
latencyAvg: number;
latencyP95: number;
latencyP99: number;
routes: HttpRouteStats[];
};
eventLoop: {
avg: number;
@@ -67,12 +69,41 @@ export interface MetricsSnapshot {
};
}
export interface HttpRouteStats {
route: string;
countLastMinute: number;
errorsLastMinute: number;
latencyAvg: number;
latencyP95: number;
latencyP99: number;
latencyMax: number;
lastDuration: number;
lastStatus: number;
}
export interface MetricsHistory {
timestamp: number;
clientsOnline: number;
commandsPerMinute: number;
bandwidthSent: number;
bandwidthReceived: number;
httpRequestsPerMinute?: number;
httpErrorsPerMinute?: number;
httpLatencyAvg?: number;
httpLatencyP95?: number;
httpLatencyP99?: number;
eventLoopAvg?: number;
eventLoopP95?: number;
heapUsed?: number;
rss?: number;
systemMemoryUsedPercent?: number;
activeSessions?: number;
}
interface TimedHttpSample {
ts: number;
duration: number;
statusCode: number;
}
class MetricsCollector {
@@ -103,8 +134,11 @@ class MetricsCollector {
private httpTotal: number = 0;
private httpTimestamps: number[] = [];
private httpErrorTimestamps: number[] = [];
private httpLatencies: number[] = [];
private maxHttpLatencyHistory: number = 1000;
private httpSamples: TimedHttpSample[] = [];
private httpRouteSamples: Map<string, TimedHttpSample[]> = new Map();
private maxHttpLatencyHistory: number = 2000;
private maxHttpRouteSamples: number = 300;
private ignoredHttpMetricRoutes: Set<string> = new Set(["GET /api/metrics"]);
private eventLoopDelays: number[] = [];
private maxEventLoopHistory: number = 300;
@@ -236,7 +270,74 @@ class MetricsCollector {
}, intervalMs);
}
recordHttpRequest(durationMs: number, statusCode: number) {
private percentile(samples: number[], percentile: number): number {
if (samples.length === 0) return 0;
const index = Math.max(0, Math.ceil(samples.length * percentile) - 1);
return samples[index] ?? 0;
}
private average(samples: number[]): number {
return samples.length
? samples.reduce((sum, value) => sum + value, 0) / samples.length
: 0;
}
private pruneHttpSamples(list: TimedHttpSample[], minTs: number): void {
let removeCount = 0;
while (removeCount < list.length && list[removeCount].ts <= minTs) {
removeCount += 1;
}
if (removeCount > 0) {
list.splice(0, removeCount);
}
}
private countHttpErrors(samples: TimedHttpSample[]): number {
let count = 0;
for (const sample of samples) {
if (sample.statusCode >= 400) count += 1;
}
return count;
}
private getTopHttpRoutes(minTs: number): HttpRouteStats[] {
const routes: HttpRouteStats[] = [];
for (const [route, samples] of this.httpRouteSamples.entries()) {
this.pruneHttpSamples(samples, minTs);
if (samples.length === 0) {
this.httpRouteSamples.delete(route);
continue;
}
const durations = samples
.map((sample) => sample.duration)
.sort((a, b) => a - b);
const last = samples[samples.length - 1];
routes.push({
route,
countLastMinute: samples.length,
errorsLastMinute: this.countHttpErrors(samples),
latencyAvg: this.average(durations),
latencyP95: this.percentile(durations, 0.95),
latencyP99: this.percentile(durations, 0.99),
latencyMax: durations[durations.length - 1] ?? 0,
lastDuration: last?.duration ?? 0,
lastStatus: last?.statusCode ?? 0,
});
}
return routes
.sort((a, b) => {
if (b.latencyP95 !== a.latencyP95) return b.latencyP95 - a.latencyP95;
if (b.latencyAvg !== a.latencyAvg) return b.latencyAvg - a.latencyAvg;
return b.countLastMinute - a.countLastMinute;
})
.slice(0, 8);
}
recordHttpRequest(durationMs: number, statusCode: number, route = "unknown") {
if (this.ignoredHttpMetricRoutes.has(route)) return;
this.httpTotal++;
const now = Date.now();
this.httpTimestamps.push(now);
@@ -246,14 +347,25 @@ class MetricsCollector {
this.pruneTimestampWindow(this.httpErrorTimestamps, now - 60000);
}
if (Number.isFinite(durationMs)) {
this.httpLatencies.push(durationMs);
if (this.httpLatencies.length > this.maxHttpLatencyHistory) {
this.httpLatencies.shift();
const sample = { ts: now, duration: Math.max(0, durationMs), statusCode };
this.httpSamples.push(sample);
if (this.httpSamples.length > this.maxHttpLatencyHistory) {
this.httpSamples.splice(0, this.httpSamples.length - this.maxHttpLatencyHistory);
}
const routeSamples = this.httpRouteSamples.get(route) || [];
routeSamples.push(sample);
if (routeSamples.length > this.maxHttpRouteSamples) {
routeSamples.splice(0, routeSamples.length - this.maxHttpRouteSamples);
}
this.httpRouteSamples.set(route, routeSamples);
}
}
async withHttpMetrics<T extends Response>(handler: () => Promise<T>): Promise<T> {
async withHttpMetrics<T extends Response>(
handler: () => Promise<T>,
route = "unknown",
): Promise<T> {
const start = typeof performance !== "undefined" && performance.now
? performance.now()
: Date.now();
@@ -262,13 +374,13 @@ class MetricsCollector {
const end = typeof performance !== "undefined" && performance.now
? performance.now()
: Date.now();
this.recordHttpRequest(end - start, response?.status ?? 0);
this.recordHttpRequest(end - start, response?.status ?? 0, route);
return response;
} catch (err) {
const end = typeof performance !== "undefined" && performance.now
? performance.now()
: Date.now();
this.recordHttpRequest(end - start, 500);
this.recordHttpRequest(end - start, 500, route);
throw err;
}
}
@@ -280,6 +392,21 @@ class MetricsCollector {
commandsPerMinute: snapshot.commands.lastMinute,
bandwidthSent: this.sentPerSecond,
bandwidthReceived: this.receivedPerSecond,
httpRequestsPerMinute: snapshot.http.lastMinute,
httpErrorsPerMinute: snapshot.http.lastMinuteErrors,
httpLatencyAvg: snapshot.http.latencyAvg,
httpLatencyP95: snapshot.http.latencyP95,
httpLatencyP99: snapshot.http.latencyP99,
eventLoopAvg: snapshot.eventLoop.avg,
eventLoopP95: snapshot.eventLoop.p95,
heapUsed: snapshot.server.memoryUsage.heapUsed,
rss: snapshot.server.memoryUsage.rss,
systemMemoryUsedPercent: snapshot.server.systemMemory.usedPercent,
activeSessions:
snapshot.sessions.console +
snapshot.sessions.remoteDesktop +
snapshot.sessions.fileBrowser +
snapshot.sessions.process,
};
this.history.push(historyEntry);
@@ -297,6 +424,7 @@ class MetricsCollector {
this.pruneTimestampWindow(this.commandTimestamps, oneHourAgo);
this.pruneTimestampWindow(this.httpTimestamps, oneMinuteAgo);
this.pruneTimestampWindow(this.httpErrorTimestamps, oneMinuteAgo);
this.pruneHttpSamples(this.httpSamples, oneMinuteAgo);
const commandsLastMinute = this.countRecent(this.commandTimestamps, oneMinuteAgo);
const commandsLastHour = this.commandTimestamps.length;
@@ -309,16 +437,13 @@ class MetricsCollector {
const httpLastMinute = this.httpTimestamps.length;
const httpErrorsLastMinute = this.httpErrorTimestamps.length;
const httpLatencySamples = [...this.httpLatencies].sort((a, b) => a - b);
const httpLatencyAvg = httpLatencySamples.length
? httpLatencySamples.reduce((a, b) => a + b, 0) / httpLatencySamples.length
: 0;
const httpP95Index = httpLatencySamples.length
? Math.max(0, Math.floor(httpLatencySamples.length * 0.95) - 1)
: 0;
const httpLatencyP95 = httpLatencySamples.length
? httpLatencySamples[httpP95Index] ?? 0
: 0;
const httpLatencySamples = this.httpSamples
.map((sample) => sample.duration)
.sort((a, b) => a - b);
const httpLatencyAvg = this.average(httpLatencySamples);
const httpLatencyP95 = this.percentile(httpLatencySamples, 0.95);
const httpLatencyP99 = this.percentile(httpLatencySamples, 0.99);
const httpRoutes = this.getTopHttpRoutes(oneMinuteAgo);
const eventLoopSamples = [...this.eventLoopDelays].sort((a, b) => a - b);
const eventLoopAvg = eventLoopSamples.length
@@ -400,6 +525,8 @@ class MetricsCollector {
lastMinuteErrors: httpErrorsLastMinute,
latencyAvg: httpLatencyAvg,
latencyP95: httpLatencyP95,
latencyP99: httpLatencyP99,
routes: httpRoutes,
},
eventLoop: {
avg: eventLoopAvg,
@@ -428,7 +555,8 @@ class MetricsCollector {
this.httpTotal = 0;
this.httpTimestamps = [];
this.httpErrorTimestamps = [];
this.httpLatencies = [];
this.httpSamples = [];
this.httpRouteSamples.clear();
this.eventLoopDelays = [];
}
}
@@ -1,6 +1,6 @@
import { beforeEach, describe, expect, test } from "bun:test";
import { clearRequestRateLimitsForTests } from "../rateLimit";
import { createHttpFetchHandler } from "./http-dispatch";
import { createHttpFetchHandler, normalizeHttpRoute } from "./http-dispatch";
const metrics = {
withHttpMetrics: (fn: () => Promise<Response>) => fn(),
@@ -35,3 +35,22 @@ describe("createHttpFetchHandler rate limiting", () => {
expect(limited.headers.get("Retry-After")).toBeTruthy();
});
});
describe("normalizeHttpRoute", () => {
test("collapses dynamic route segments", () => {
const req = new Request(
"https://localhost/api/clients/0a44a8d4d61c7840540b43fd187fe3c43e2c46c9600b2aa1a61a6b1ab03c5b92/rd/ws?cache=1",
{ method: "POST" },
);
expect(normalizeHttpRoute(req, new URL(req.url))).toBe(
"POST /api/clients/:id/rd/ws",
);
});
test("groups static asset files", () => {
const req = new Request("https://localhost/assets/metrics.js");
expect(normalizeHttpRoute(req, new URL(req.url))).toBe("GET /assets/:file");
});
});
+24 -3
View File
@@ -3,6 +3,9 @@ import { consumeUnauthorizedRateLimit } from "../rateLimit";
export type RouteHandler = (req: Request, url: URL, server: unknown) => Promise<Response | null>;
const UUID_SEGMENT = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
const LONG_ID_SEGMENT = /^[0-9a-f]{16,}$/i;
function tooManyRequestsResponse(retryAfter = 60): Response {
return new Response("Too many requests", {
status: 429,
@@ -13,17 +16,35 @@ function tooManyRequestsResponse(retryAfter = 60): Response {
});
}
function normalizeRouteSegment(segment: string): string {
if (/^\d+$/.test(segment)) return ":id";
if (UUID_SEGMENT.test(segment) || LONG_ID_SEGMENT.test(segment)) return ":id";
if (segment.includes(".") && !segment.startsWith(".")) return ":file";
if (segment.length >= 24 && /^[a-z0-9_-]+$/i.test(segment)) return ":id";
return segment;
}
export function normalizeHttpRoute(req: Request, url: URL): string {
const path = url.pathname
.split("/")
.filter(Boolean)
.map(normalizeRouteSegment)
.join("/");
return `${req.method.toUpperCase()} /${path}`;
}
export function createHttpFetchHandler(deps: {
metrics: { withHttpMetrics: (fn: () => Promise<Response>) => Promise<Response> };
metrics: { withHttpMetrics: (fn: () => Promise<Response>, route?: string) => Promise<Response> };
CORS_HEADERS: Record<string, string>;
routes: RouteHandler[];
}) {
return async function fetchHandler(req: Request, server: unknown): Promise<Response> {
const url = new URL(req.url);
const routeLabel = normalizeHttpRoute(req, url);
return deps.metrics.withHttpMetrics(async () => {
if (req.method === "OPTIONS") {
return new Response("", { headers: deps.CORS_HEADERS });
}
const url = new URL(req.url);
const wrapped = wrapServerWithClientIp(server as RequestServerLike);
const ip = wrapped.requestIP(req)?.address || "unknown";
@@ -38,6 +59,6 @@ export function createHttpFetchHandler(deps: {
}
}
return new Response("Not found", { status: 404 });
});
}, routeLabel);
};
}
@@ -330,7 +330,11 @@ export async function handleMiscRoutes(
snapshot.sessions.fileBrowser = deps.getFileBrowserSessionCount();
snapshot.sessions.process = deps.getProcessSessionCount();
const history = metrics.getHistory();
const requestedHistoryLimit = Number(url.searchParams.get("historyLimit") || 240);
const historyLimit = Number.isFinite(requestedHistoryLimit)
? Math.max(1, Math.min(2000, requestedHistoryLimit))
: 240;
const history = metrics.getHistory().slice(-historyLimit);
return new Response(JSON.stringify({ snapshot, history }), {
headers: { "Content-Type": "application/json" },