import os from "node:os"; export interface MetricsSnapshot { timestamp: number; clients: { total: number; online: number; offline: number; byOS: Record; byCountry: Record; }; connections: { totalConnections: number; totalDisconnections: number; activeConnections: number; }; commands: { total: number; lastMinute: number; lastHour: number; byType: Record; }; sessions: { console: number; remoteDesktop: number; fileBrowser: number; process: number; }; bandwidth: { sent: number; received: number; sentPerSecond: number; receivedPerSecond: number; }; server: { uptime: number; startTime: number; memoryUsage: NodeJS.MemoryUsage; systemMemory: { total: number; free: number; used: number; usedPercent: number; }; cpu: { cores: number; loadAvg: [number, number, number]; }; }; ping: { min: number | null; max: number | null; avg: number | null; count: number; }; http: { total: number; lastMinute: number; lastMinuteErrors: number; latencyAvg: number; latencyP95: number; latencyP99: number; routes: HttpRouteStats[]; }; eventLoop: { avg: number; max: number; p95: number; sampleMs: number; samples: number; }; internal: { tasks: InternalTaskStats[]; }; diagnostics?: { retained: Record; }; } export interface HttpRouteStats { route: string; countLastMinute: number; errorsLastMinute: number; latencyAvg: number; latencyP95: number; latencyP99: number; latencyMax: number; lastDuration: number; lastStatus: number; } export interface InternalTaskStats { task: string; countLastMinute: number; durationAvg: number; durationP95: number; durationP99: number; durationMax: number; lastDuration: number; lastAt: number; } export interface MetricsHistory { timestamp: number; clientsOnline: number; commandsPerMinute: number; bandwidthSent: number; bandwidthReceived: number; httpRequestsPerMinute?: number; httpErrorsPerMinute?: number; httpLatencyAvg?: number; httpLatencyP95?: number; httpLatencyP99?: number; eventLoopAvg?: number; eventLoopP95?: number; heapUsed?: number; rss?: number; systemMemoryUsedPercent?: number; activeSessions?: number; } interface TimedHttpSample { ts: number; duration: number; statusCode: number; } interface TimedInternalTaskSample { ts: number; duration: number; } class MetricsCollector { private startTime: number = Date.now(); private totalConnections: number = 0; private totalDisconnections: number = 0; private commandCount: number = 0; private commandTypeCount: Map = new Map(); private commandTimestamps: number[] = []; private bytesSent: number = 0; private bytesReceived: number = 0; private lastBandwidthCheck: number = Date.now(); private lastBytesSent: number = 0; private lastBytesReceived: number = 0; private sentPerSecond: number = 0; private receivedPerSecond: number = 0; private history: MetricsHistory[] = []; private maxHistoryPoints: number = 7 * 24 * 60 * 12; private snapshotEnricher: ((snapshot: MetricsSnapshot) => void) | null = null; private pingValues: number[] = []; private maxPingHistory: number = 1000; private httpTotal: number = 0; private httpTimestamps: number[] = []; private httpErrorTimestamps: number[] = []; private httpSamples: TimedHttpSample[] = []; private httpRouteSamples: Map = new Map(); private maxHttpLatencyHistory: number = 2000; private maxHttpRouteSamples: number = 300; private ignoredHttpMetricRoutes: Set = new Set(["GET /api/metrics"]); private eventLoopDelays: number[] = []; private eventLoopSampleMs: number = Math.max( 20, Number(process.env.OVERLORD_EVENT_LOOP_SAMPLE_MS || 100), ); private maxEventLoopHistory: number = Math.max( 10, Math.ceil((Number(process.env.OVERLORD_EVENT_LOOP_HISTORY_SECONDS || 60) * 1000) / this.eventLoopSampleMs), ); private internalTaskSamples: Map = new Map(); private maxInternalTaskSamples: number = 300; private pruneTimestampWindow(list: number[], minTs: number): void { let removeCount = 0; while (removeCount < list.length && list[removeCount] <= minTs) { removeCount += 1; } if (removeCount > 0) { list.splice(0, removeCount); } } private countRecent(list: number[], minTs: number): number { let count = 0; for (let index = list.length - 1; index >= 0; index -= 1) { if (list[index] <= minTs) { break; } count += 1; } return count; } constructor() { setInterval(() => this.updateBandwidthRates(), 1000); setInterval(() => this.recordHistory(), 5000); this.trackEventLoopDelay(); } recordConnection() { this.totalConnections++; } recordDisconnection() { this.totalDisconnections++; } recordCommand(type: string) { this.commandCount++; const now = Date.now(); this.commandTimestamps.push(now); const count = this.commandTypeCount.get(type) || 0; this.commandTypeCount.set(type, count + 1); this.pruneTimestampWindow(this.commandTimestamps, now - 3600000); } recordBytesSent(bytes: number) { this.bytesSent += bytes; } recordBytesReceived(bytes: number) { this.bytesReceived += bytes; } private updateBandwidthRates() { const now = Date.now(); const elapsed = (now - this.lastBandwidthCheck) / 1000; if (elapsed > 0) { this.sentPerSecond = (this.bytesSent - this.lastBytesSent) / elapsed; this.receivedPerSecond = (this.bytesReceived - this.lastBytesReceived) / elapsed; this.lastBytesSent = this.bytesSent; this.lastBytesReceived = this.bytesReceived; this.lastBandwidthCheck = now; } } recordPing(pingMs: number) { this.pingValues.push(pingMs); if (this.pingValues.length > this.maxPingHistory) { this.pingValues.shift(); } } private getPingStats() { if (this.pingValues.length === 0) { return { min: null, max: null, avg: null, count: 0 }; } let min = Number.POSITIVE_INFINITY; let max = Number.NEGATIVE_INFINITY; let sum = 0; for (const ping of this.pingValues) { if (ping < min) min = ping; if (ping > max) max = ping; sum += ping; } const avg = sum / this.pingValues.length; return { min, max, avg, count: this.pingValues.length }; } setSnapshotEnricher(fn: ((snapshot: MetricsSnapshot) => void) | null): void { this.snapshotEnricher = fn; } private recordHistory() { const snapshot = this.getSnapshot(); if (this.snapshotEnricher) { try { this.snapshotEnricher(snapshot); } catch (err) { console.error("metrics snapshot enricher failed:", err); } } this.recordHistoryEntry(snapshot); } private trackEventLoopDelay() { const intervalMs = this.eventLoopSampleMs; let last = Date.now(); setInterval(() => { const now = Date.now(); const delay = Math.max(0, now - last - intervalMs); this.eventLoopDelays.push(delay); if (this.eventLoopDelays.length > this.maxEventLoopHistory) { this.eventLoopDelays.shift(); } last = now; }, intervalMs); } private percentile(samples: number[], percentile: number): number { if (samples.length === 0) return 0; const index = Math.max(0, Math.ceil(samples.length * percentile) - 1); return samples[index] ?? 0; } private average(samples: number[]): number { return samples.length ? samples.reduce((sum, value) => sum + value, 0) / samples.length : 0; } private pruneTimedSamples(list: Array<{ ts: number }>, minTs: number): void { let removeCount = 0; while (removeCount < list.length && list[removeCount].ts <= minTs) { removeCount += 1; } if (removeCount > 0) { list.splice(0, removeCount); } } private countHttpErrors(samples: TimedHttpSample[]): number { let count = 0; for (const sample of samples) { if (sample.statusCode >= 400) count += 1; } return count; } private getTopHttpRoutes(minTs: number): HttpRouteStats[] { const routes: HttpRouteStats[] = []; for (const [route, samples] of this.httpRouteSamples.entries()) { this.pruneTimedSamples(samples, minTs); if (samples.length === 0) { this.httpRouteSamples.delete(route); continue; } const durations = samples .map((sample) => sample.duration) .sort((a, b) => a - b); const last = samples[samples.length - 1]; routes.push({ route, countLastMinute: samples.length, errorsLastMinute: this.countHttpErrors(samples), latencyAvg: this.average(durations), latencyP95: this.percentile(durations, 0.95), latencyP99: this.percentile(durations, 0.99), latencyMax: durations[durations.length - 1] ?? 0, lastDuration: last?.duration ?? 0, lastStatus: last?.statusCode ?? 0, }); } return routes .sort((a, b) => { if (b.latencyP95 !== a.latencyP95) return b.latencyP95 - a.latencyP95; if (b.latencyAvg !== a.latencyAvg) return b.latencyAvg - a.latencyAvg; return b.countLastMinute - a.countLastMinute; }) .slice(0, 8); } private getTopInternalTasks(minTs: number): InternalTaskStats[] { const tasks: InternalTaskStats[] = []; for (const [task, samples] of this.internalTaskSamples.entries()) { this.pruneTimedSamples(samples, minTs); if (samples.length === 0) { this.internalTaskSamples.delete(task); continue; } const durations = samples .map((sample) => sample.duration) .sort((a, b) => a - b); const last = samples[samples.length - 1]; tasks.push({ task, countLastMinute: samples.length, durationAvg: this.average(durations), durationP95: this.percentile(durations, 0.95), durationP99: this.percentile(durations, 0.99), durationMax: durations[durations.length - 1] ?? 0, lastDuration: last?.duration ?? 0, lastAt: last?.ts ?? 0, }); } return tasks .sort((a, b) => { if (b.durationP95 !== a.durationP95) return b.durationP95 - a.durationP95; if (b.durationAvg !== a.durationAvg) return b.durationAvg - a.durationAvg; return b.countLastMinute - a.countLastMinute; }) .slice(0, 8); } recordHttpRequest(durationMs: number, statusCode: number, route = "unknown") { if (this.ignoredHttpMetricRoutes.has(route)) return; this.httpTotal++; const now = Date.now(); this.httpTimestamps.push(now); this.pruneTimestampWindow(this.httpTimestamps, now - 60000); if (statusCode >= 400) { this.httpErrorTimestamps.push(now); this.pruneTimestampWindow(this.httpErrorTimestamps, now - 60000); } if (Number.isFinite(durationMs)) { const sample = { ts: now, duration: Math.max(0, durationMs), statusCode }; this.httpSamples.push(sample); if (this.httpSamples.length > this.maxHttpLatencyHistory) { this.httpSamples.splice(0, this.httpSamples.length - this.maxHttpLatencyHistory); } const routeSamples = this.httpRouteSamples.get(route) || []; routeSamples.push(sample); if (routeSamples.length > this.maxHttpRouteSamples) { routeSamples.splice(0, routeSamples.length - this.maxHttpRouteSamples); } this.httpRouteSamples.set(route, routeSamples); } } recordInternalTask(task: string, durationMs: number) { if (!task || !Number.isFinite(durationMs)) return; const now = Date.now(); const sample = { ts: now, duration: Math.max(0, durationMs) }; const samples = this.internalTaskSamples.get(task) || []; samples.push(sample); if (samples.length > this.maxInternalTaskSamples) { samples.splice(0, samples.length - this.maxInternalTaskSamples); } this.internalTaskSamples.set(task, samples); } async withHttpMetrics( handler: () => Promise, route = "unknown", ): Promise { const start = typeof performance !== "undefined" && performance.now ? performance.now() : Date.now(); try { const response = await handler(); const end = typeof performance !== "undefined" && performance.now ? performance.now() : Date.now(); this.recordHttpRequest(end - start, response?.status ?? 0, route); return response; } catch (err) { const end = typeof performance !== "undefined" && performance.now ? performance.now() : Date.now(); this.recordHttpRequest(end - start, 500, route); throw err; } } recordHistoryEntry(snapshot: MetricsSnapshot) { const historyEntry: MetricsHistory = { timestamp: snapshot.timestamp, clientsOnline: snapshot.clients.online, commandsPerMinute: snapshot.commands.lastMinute, bandwidthSent: this.sentPerSecond, bandwidthReceived: this.receivedPerSecond, httpRequestsPerMinute: snapshot.http.lastMinute, httpErrorsPerMinute: snapshot.http.lastMinuteErrors, httpLatencyAvg: snapshot.http.latencyAvg, httpLatencyP95: snapshot.http.latencyP95, httpLatencyP99: snapshot.http.latencyP99, eventLoopAvg: snapshot.eventLoop.avg, eventLoopP95: snapshot.eventLoop.p95, heapUsed: snapshot.server.memoryUsage.heapUsed, rss: snapshot.server.memoryUsage.rss, systemMemoryUsedPercent: snapshot.server.systemMemory.usedPercent, activeSessions: snapshot.sessions.console + snapshot.sessions.remoteDesktop + snapshot.sessions.fileBrowser + snapshot.sessions.process, }; this.history.push(historyEntry); if (this.history.length > this.maxHistoryPoints) { this.history.shift(); } } getSnapshot(): MetricsSnapshot { const now = Date.now(); const oneMinuteAgo = now - 60000; const oneHourAgo = now - 3600000; this.pruneTimestampWindow(this.commandTimestamps, oneHourAgo); this.pruneTimestampWindow(this.httpTimestamps, oneMinuteAgo); this.pruneTimestampWindow(this.httpErrorTimestamps, oneMinuteAgo); this.pruneTimedSamples(this.httpSamples, oneMinuteAgo); const commandsLastMinute = this.countRecent(this.commandTimestamps, oneMinuteAgo); const commandsLastHour = this.commandTimestamps.length; const commandsByType: Record = {}; for (const [type, count] of this.commandTypeCount.entries()) { commandsByType[type] = count; } const httpLastMinute = this.httpTimestamps.length; const httpErrorsLastMinute = this.httpErrorTimestamps.length; const httpLatencySamples = this.httpSamples .map((sample) => sample.duration) .sort((a, b) => a - b); const httpLatencyAvg = this.average(httpLatencySamples); const httpLatencyP95 = this.percentile(httpLatencySamples, 0.95); const httpLatencyP99 = this.percentile(httpLatencySamples, 0.99); const httpRoutes = this.getTopHttpRoutes(oneMinuteAgo); const internalTasks = this.getTopInternalTasks(oneMinuteAgo); const eventLoopSamples = [...this.eventLoopDelays].sort((a, b) => a - b); const eventLoopAvg = eventLoopSamples.length ? eventLoopSamples.reduce((a, b) => a + b, 0) / eventLoopSamples.length : 0; const eventLoopMax = eventLoopSamples.length ? eventLoopSamples[eventLoopSamples.length - 1] : 0; const eventLoopP95Index = eventLoopSamples.length ? Math.max(0, Math.floor(eventLoopSamples.length * 0.95) - 1) : 0; const eventLoopP95 = eventLoopSamples.length ? eventLoopSamples[eventLoopP95Index] ?? 0 : 0; const memoryUsage = process.memoryUsage(); const totalMem = os.totalmem(); const freeMem = os.freemem(); const usedMem = Math.max(0, totalMem - freeMem); const usedPercent = totalMem > 0 ? (usedMem / totalMem) * 100 : 0; const loadAvg = os.loadavg(); const loadTuple: [number, number, number] = [ loadAvg[0] ?? 0, loadAvg[1] ?? 0, loadAvg[2] ?? 0, ]; return { timestamp: now, clients: { total: 0, online: 0, offline: 0, byOS: {}, byCountry: {}, }, connections: { totalConnections: this.totalConnections, totalDisconnections: this.totalDisconnections, activeConnections: this.totalConnections - this.totalDisconnections, }, commands: { total: this.commandCount, lastMinute: commandsLastMinute, lastHour: commandsLastHour, byType: commandsByType, }, sessions: { console: 0, remoteDesktop: 0, fileBrowser: 0, process: 0, }, bandwidth: { sent: this.bytesSent, received: this.bytesReceived, sentPerSecond: this.sentPerSecond, receivedPerSecond: this.receivedPerSecond, }, server: { uptime: now - this.startTime, startTime: this.startTime, memoryUsage, systemMemory: { total: totalMem, free: freeMem, used: usedMem, usedPercent, }, cpu: { cores: os.cpus().length || 0, loadAvg: loadTuple, }, }, ping: this.getPingStats(), http: { total: this.httpTotal, lastMinute: httpLastMinute, lastMinuteErrors: httpErrorsLastMinute, latencyAvg: httpLatencyAvg, latencyP95: httpLatencyP95, latencyP99: httpLatencyP99, routes: httpRoutes, }, eventLoop: { avg: eventLoopAvg, max: eventLoopMax, p95: eventLoopP95, sampleMs: this.eventLoopSampleMs, samples: eventLoopSamples.length, }, internal: { tasks: internalTasks, }, }; } getHistory(): MetricsHistory[] { return [...this.history]; } reset() { this.commandCount = 0; this.commandTypeCount.clear(); this.commandTimestamps = []; this.bytesSent = 0; this.bytesReceived = 0; this.lastBytesSent = 0; this.lastBytesReceived = 0; this.sentPerSecond = 0; this.receivedPerSecond = 0; this.pingValues = []; this.history = []; this.httpTotal = 0; this.httpTimestamps = []; this.httpErrorTimestamps = []; this.httpSamples = []; this.httpRouteSamples.clear(); this.eventLoopDelays = []; this.internalTaskSamples.clear(); } } export const metrics = new MetricsCollector();