more tests and better metrics

Overlord-Server/src/metrics.test.ts

@@ -125,6 +125,31 @@ describe("MetricsCollector", () => {
       const snap = metrics.getSnapshot();
       expect(snap.http.latencyAvg).toBeCloseTo(50.5, 0);
       expect(snap.http.latencyP95).toBeGreaterThanOrEqual(90);
+      expect(snap.http.latencyP99).toBeGreaterThanOrEqual(99);
+    });
+
+    test("tracks slow HTTP routes", () => {
+      metrics.recordHttpRequest(15, 200, "GET /api/metrics");
+      metrics.recordHttpRequest(120, 200, "GET /api/clients/:id/rd/ws");
+      metrics.recordHttpRequest(200, 500, "GET /api/clients/:id/rd/ws");
+
+      const snap = metrics.getSnapshot();
+      expect(snap.http.routes[0].route).toBe("GET /api/clients/:id/rd/ws");
+      expect(snap.http.routes[0].countLastMinute).toBe(2);
+      expect(snap.http.routes[0].errorsLastMinute).toBe(1);
+      expect(snap.http.routes[0].latencyP95).toBeGreaterThanOrEqual(120);
+    });
+
+    test("does not count the metrics dashboard poll as HTTP pressure", () => {
+      metrics.recordHttpRequest(100, 200, "GET /api/metrics");
+      metrics.recordHttpRequest(20, 200, "GET /api/clients");
+
+      const snap = metrics.getSnapshot();
+      expect(snap.http.total).toBe(1);
+      expect(snap.http.lastMinute).toBe(1);
+      expect(snap.http.routes.map((route) => route.route)).toEqual([
+        "GET /api/clients",
+      ]);
     });
   });
 
@@ -177,6 +202,26 @@ describe("MetricsCollector", () => {
       expect(history[0].timestamp).toBe(snap.timestamp);
     });
 
+    test("history captures pressure metrics for charting", () => {
+      metrics.recordHttpRequest(25, 200);
+      metrics.recordHttpRequest(80, 500);
+      metrics.recordBytesSent(1024);
+      metrics.recordBytesReceived(2048);
+
+      const snap = metrics.getSnapshot();
+      metrics.recordHistoryEntry(snap);
+
+      const [entry] = metrics.getHistory();
+      expect(entry.httpRequestsPerMinute).toBeGreaterThanOrEqual(2);
+      expect(entry.httpErrorsPerMinute).toBeGreaterThanOrEqual(1);
+      expect(entry.httpLatencyAvg).toBeGreaterThan(0);
+      expect(entry.httpLatencyP95).toBeGreaterThan(0);
+      expect(entry.heapUsed).toBeGreaterThan(0);
+      expect(entry.rss).toBeGreaterThan(0);
+      expect(entry.systemMemoryUsedPercent).toBeGreaterThanOrEqual(0);
+      expect(entry.activeSessions).toBe(0);
+    });
+
     test("history is capped at maxHistoryPoints", () => {
       const weekOfFiveSecondSamples = 7 * 24 * 60 * 12;
       for (let i = 0; i < weekOfFiveSecondSamples + 180; i++) {

Overlord-Server/src/metrics.ts

@@ -59,6 +59,8 @@ export interface MetricsSnapshot {
     lastMinuteErrors: number;
     latencyAvg: number;
     latencyP95: number;
+    latencyP99: number;
+    routes: HttpRouteStats[];
   };
   eventLoop: {
     avg: number;
@@ -67,12 +69,41 @@ export interface MetricsSnapshot {
   };
 }
 
+export interface HttpRouteStats {
+  route: string;
+  countLastMinute: number;
+  errorsLastMinute: number;
+  latencyAvg: number;
+  latencyP95: number;
+  latencyP99: number;
+  latencyMax: number;
+  lastDuration: number;
+  lastStatus: number;
+}
+
 export interface MetricsHistory {
   timestamp: number;
   clientsOnline: number;
   commandsPerMinute: number;
   bandwidthSent: number;
   bandwidthReceived: number;
+  httpRequestsPerMinute?: number;
+  httpErrorsPerMinute?: number;
+  httpLatencyAvg?: number;
+  httpLatencyP95?: number;
+  httpLatencyP99?: number;
+  eventLoopAvg?: number;
+  eventLoopP95?: number;
+  heapUsed?: number;
+  rss?: number;
+  systemMemoryUsedPercent?: number;
+  activeSessions?: number;
+}
+
+interface TimedHttpSample {
+  ts: number;
+  duration: number;
+  statusCode: number;
 }
 
 class MetricsCollector {
@@ -103,8 +134,11 @@ class MetricsCollector {
   private httpTotal: number = 0;
   private httpTimestamps: number[] = [];
   private httpErrorTimestamps: number[] = [];
-  private httpLatencies: number[] = [];
-  private maxHttpLatencyHistory: number = 1000;
+  private httpSamples: TimedHttpSample[] = [];
+  private httpRouteSamples: Map<string, TimedHttpSample[]> = new Map();
+  private maxHttpLatencyHistory: number = 2000;
+  private maxHttpRouteSamples: number = 300;
+  private ignoredHttpMetricRoutes: Set<string> = new Set(["GET /api/metrics"]);
 
   private eventLoopDelays: number[] = [];
   private maxEventLoopHistory: number = 300;
@@ -236,7 +270,74 @@ class MetricsCollector {
     }, intervalMs);
   }
 
-  recordHttpRequest(durationMs: number, statusCode: number) {
+  private percentile(samples: number[], percentile: number): number {
+    if (samples.length === 0) return 0;
+    const index = Math.max(0, Math.ceil(samples.length * percentile) - 1);
+    return samples[index] ?? 0;
+  }
+
+  private average(samples: number[]): number {
+    return samples.length
+      ? samples.reduce((sum, value) => sum + value, 0) / samples.length
+      : 0;
+  }
+
+  private pruneHttpSamples(list: TimedHttpSample[], minTs: number): void {
+    let removeCount = 0;
+    while (removeCount < list.length && list[removeCount].ts <= minTs) {
+      removeCount += 1;
+    }
+    if (removeCount > 0) {
+      list.splice(0, removeCount);
+    }
+  }
+
+  private countHttpErrors(samples: TimedHttpSample[]): number {
+    let count = 0;
+    for (const sample of samples) {
+      if (sample.statusCode >= 400) count += 1;
+    }
+    return count;
+  }
+
+  private getTopHttpRoutes(minTs: number): HttpRouteStats[] {
+    const routes: HttpRouteStats[] = [];
+    for (const [route, samples] of this.httpRouteSamples.entries()) {
+      this.pruneHttpSamples(samples, minTs);
+      if (samples.length === 0) {
+        this.httpRouteSamples.delete(route);
+        continue;
+      }
+
+      const durations = samples
+        .map((sample) => sample.duration)
+        .sort((a, b) => a - b);
+      const last = samples[samples.length - 1];
+      routes.push({
+        route,
+        countLastMinute: samples.length,
+        errorsLastMinute: this.countHttpErrors(samples),
+        latencyAvg: this.average(durations),
+        latencyP95: this.percentile(durations, 0.95),
+        latencyP99: this.percentile(durations, 0.99),
+        latencyMax: durations[durations.length - 1] ?? 0,
+        lastDuration: last?.duration ?? 0,
+        lastStatus: last?.statusCode ?? 0,
+      });
+    }
+
+    return routes
+      .sort((a, b) => {
+        if (b.latencyP95 !== a.latencyP95) return b.latencyP95 - a.latencyP95;
+        if (b.latencyAvg !== a.latencyAvg) return b.latencyAvg - a.latencyAvg;
+        return b.countLastMinute - a.countLastMinute;
+      })
+      .slice(0, 8);
+  }
+
+  recordHttpRequest(durationMs: number, statusCode: number, route = "unknown") {
+    if (this.ignoredHttpMetricRoutes.has(route)) return;
+
     this.httpTotal++;
     const now = Date.now();
     this.httpTimestamps.push(now);
@@ -246,14 +347,25 @@ class MetricsCollector {
       this.pruneTimestampWindow(this.httpErrorTimestamps, now - 60000);
     }
     if (Number.isFinite(durationMs)) {
-      this.httpLatencies.push(durationMs);
-      if (this.httpLatencies.length > this.maxHttpLatencyHistory) {
-        this.httpLatencies.shift();
+      const sample = { ts: now, duration: Math.max(0, durationMs), statusCode };
+      this.httpSamples.push(sample);
+      if (this.httpSamples.length > this.maxHttpLatencyHistory) {
+        this.httpSamples.splice(0, this.httpSamples.length - this.maxHttpLatencyHistory);
       }
+
+      const routeSamples = this.httpRouteSamples.get(route) || [];
+      routeSamples.push(sample);
+      if (routeSamples.length > this.maxHttpRouteSamples) {
+        routeSamples.splice(0, routeSamples.length - this.maxHttpRouteSamples);
+      }
+      this.httpRouteSamples.set(route, routeSamples);
     }
   }
 
-  async withHttpMetrics<T extends Response>(handler: () => Promise<T>): Promise<T> {
+  async withHttpMetrics<T extends Response>(
+    handler: () => Promise<T>,
+    route = "unknown",
+  ): Promise<T> {
     const start = typeof performance !== "undefined" && performance.now
       ? performance.now()
       : Date.now();
@@ -262,13 +374,13 @@ class MetricsCollector {
       const end = typeof performance !== "undefined" && performance.now
         ? performance.now()
         : Date.now();
-      this.recordHttpRequest(end - start, response?.status ?? 0);
+      this.recordHttpRequest(end - start, response?.status ?? 0, route);
       return response;
     } catch (err) {
       const end = typeof performance !== "undefined" && performance.now
         ? performance.now()
         : Date.now();
-      this.recordHttpRequest(end - start, 500);
+      this.recordHttpRequest(end - start, 500, route);
       throw err;
     }
   }
@@ -280,6 +392,21 @@ class MetricsCollector {
       commandsPerMinute: snapshot.commands.lastMinute,
       bandwidthSent: this.sentPerSecond,
       bandwidthReceived: this.receivedPerSecond,
+      httpRequestsPerMinute: snapshot.http.lastMinute,
+      httpErrorsPerMinute: snapshot.http.lastMinuteErrors,
+      httpLatencyAvg: snapshot.http.latencyAvg,
+      httpLatencyP95: snapshot.http.latencyP95,
+      httpLatencyP99: snapshot.http.latencyP99,
+      eventLoopAvg: snapshot.eventLoop.avg,
+      eventLoopP95: snapshot.eventLoop.p95,
+      heapUsed: snapshot.server.memoryUsage.heapUsed,
+      rss: snapshot.server.memoryUsage.rss,
+      systemMemoryUsedPercent: snapshot.server.systemMemory.usedPercent,
+      activeSessions:
+        snapshot.sessions.console +
+        snapshot.sessions.remoteDesktop +
+        snapshot.sessions.fileBrowser +
+        snapshot.sessions.process,
     };
 
     this.history.push(historyEntry);
@@ -297,6 +424,7 @@ class MetricsCollector {
     this.pruneTimestampWindow(this.commandTimestamps, oneHourAgo);
     this.pruneTimestampWindow(this.httpTimestamps, oneMinuteAgo);
     this.pruneTimestampWindow(this.httpErrorTimestamps, oneMinuteAgo);
+    this.pruneHttpSamples(this.httpSamples, oneMinuteAgo);
 
     const commandsLastMinute = this.countRecent(this.commandTimestamps, oneMinuteAgo);
     const commandsLastHour = this.commandTimestamps.length;
@@ -309,16 +437,13 @@ class MetricsCollector {
     const httpLastMinute = this.httpTimestamps.length;
     const httpErrorsLastMinute = this.httpErrorTimestamps.length;
 
-    const httpLatencySamples = [...this.httpLatencies].sort((a, b) => a - b);
-    const httpLatencyAvg = httpLatencySamples.length
-      ? httpLatencySamples.reduce((a, b) => a + b, 0) / httpLatencySamples.length
-      : 0;
-    const httpP95Index = httpLatencySamples.length
-      ? Math.max(0, Math.floor(httpLatencySamples.length * 0.95) - 1)
-      : 0;
-    const httpLatencyP95 = httpLatencySamples.length
-      ? httpLatencySamples[httpP95Index] ?? 0
-      : 0;
+    const httpLatencySamples = this.httpSamples
+      .map((sample) => sample.duration)
+      .sort((a, b) => a - b);
+    const httpLatencyAvg = this.average(httpLatencySamples);
+    const httpLatencyP95 = this.percentile(httpLatencySamples, 0.95);
+    const httpLatencyP99 = this.percentile(httpLatencySamples, 0.99);
+    const httpRoutes = this.getTopHttpRoutes(oneMinuteAgo);
 
     const eventLoopSamples = [...this.eventLoopDelays].sort((a, b) => a - b);
     const eventLoopAvg = eventLoopSamples.length
@@ -400,6 +525,8 @@ class MetricsCollector {
         lastMinuteErrors: httpErrorsLastMinute,
         latencyAvg: httpLatencyAvg,
         latencyP95: httpLatencyP95,
+        latencyP99: httpLatencyP99,
+        routes: httpRoutes,
       },
       eventLoop: {
         avg: eventLoopAvg,
@@ -428,7 +555,8 @@ class MetricsCollector {
     this.httpTotal = 0;
     this.httpTimestamps = [];
     this.httpErrorTimestamps = [];
-    this.httpLatencies = [];
+    this.httpSamples = [];
+    this.httpRouteSamples.clear();
     this.eventLoopDelays = [];
   }
 }

Overlord-Server/src/server/http-dispatch.test.ts

@@ -1,6 +1,6 @@
 import { beforeEach, describe, expect, test } from "bun:test";
 import { clearRequestRateLimitsForTests } from "../rateLimit";
-import { createHttpFetchHandler } from "./http-dispatch";
+import { createHttpFetchHandler, normalizeHttpRoute } from "./http-dispatch";
 
 const metrics = {
   withHttpMetrics: (fn: () => Promise<Response>) => fn(),
@@ -35,3 +35,22 @@ describe("createHttpFetchHandler rate limiting", () => {
     expect(limited.headers.get("Retry-After")).toBeTruthy();
   });
 });
+
+describe("normalizeHttpRoute", () => {
+  test("collapses dynamic route segments", () => {
+    const req = new Request(
+      "https://localhost/api/clients/0a44a8d4d61c7840540b43fd187fe3c43e2c46c9600b2aa1a61a6b1ab03c5b92/rd/ws?cache=1",
+      { method: "POST" },
+    );
+
+    expect(normalizeHttpRoute(req, new URL(req.url))).toBe(
+      "POST /api/clients/:id/rd/ws",
+    );
+  });
+
+  test("groups static asset files", () => {
+    const req = new Request("https://localhost/assets/metrics.js");
+
+    expect(normalizeHttpRoute(req, new URL(req.url))).toBe("GET /assets/:file");
+  });
+});

Overlord-Server/src/server/http-dispatch.ts

@@ -3,6 +3,9 @@ import { consumeUnauthorizedRateLimit } from "../rateLimit";
 
 export type RouteHandler = (req: Request, url: URL, server: unknown) => Promise<Response | null>;
 
+const UUID_SEGMENT = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
+const LONG_ID_SEGMENT = /^[0-9a-f]{16,}$/i;
+
 function tooManyRequestsResponse(retryAfter = 60): Response {
   return new Response("Too many requests", {
     status: 429,
@@ -13,17 +16,35 @@ function tooManyRequestsResponse(retryAfter = 60): Response {
   });
 }
 
+function normalizeRouteSegment(segment: string): string {
+  if (/^\d+$/.test(segment)) return ":id";
+  if (UUID_SEGMENT.test(segment) || LONG_ID_SEGMENT.test(segment)) return ":id";
+  if (segment.includes(".") && !segment.startsWith(".")) return ":file";
+  if (segment.length >= 24 && /^[a-z0-9_-]+$/i.test(segment)) return ":id";
+  return segment;
+}
+
+export function normalizeHttpRoute(req: Request, url: URL): string {
+  const path = url.pathname
+    .split("/")
+    .filter(Boolean)
+    .map(normalizeRouteSegment)
+    .join("/");
+  return `${req.method.toUpperCase()} /${path}`;
+}
+
 export function createHttpFetchHandler(deps: {
-  metrics: { withHttpMetrics: (fn: () => Promise<Response>) => Promise<Response> };
+  metrics: { withHttpMetrics: (fn: () => Promise<Response>, route?: string) => Promise<Response> };
   CORS_HEADERS: Record<string, string>;
   routes: RouteHandler[];
 }) {
   return async function fetchHandler(req: Request, server: unknown): Promise<Response> {
+    const url = new URL(req.url);
+    const routeLabel = normalizeHttpRoute(req, url);
     return deps.metrics.withHttpMetrics(async () => {
       if (req.method === "OPTIONS") {
         return new Response("", { headers: deps.CORS_HEADERS });
       }
-      const url = new URL(req.url);
       const wrapped = wrapServerWithClientIp(server as RequestServerLike);
       const ip = wrapped.requestIP(req)?.address || "unknown";
 
@@ -38,6 +59,6 @@ export function createHttpFetchHandler(deps: {
         }
       }
       return new Response("Not found", { status: 404 });
-    });
+    }, routeLabel);
   };
 }

Overlord-Server/src/server/routes/misc-routes.ts

@@ -330,7 +330,11 @@ export async function handleMiscRoutes(
     snapshot.sessions.fileBrowser = deps.getFileBrowserSessionCount();
     snapshot.sessions.process = deps.getProcessSessionCount();
 
-    const history = metrics.getHistory();
+    const requestedHistoryLimit = Number(url.searchParams.get("historyLimit") || 240);
+    const historyLimit = Number.isFinite(requestedHistoryLimit)
+      ? Math.max(1, Math.min(2000, requestedHistoryLimit))
+      : 240;
+    const history = metrics.getHistory().slice(-historyLimit);
 
     return new Response(JSON.stringify({ snapshot, history }), {
       headers: { "Content-Type": "application/json" },