From d6ad2cadea5111e652cb38478d6256bba5e7c7ab Mon Sep 17 00:00:00 2001 From: KingKDot Date: Mon, 8 Jun 2026 22:33:34 -0500 Subject: [PATCH] huge optimizations --- .gitignore | 3 + Overlord-Server/public/assets/data.js | 51 +++++++- Overlord-Server/public/assets/metrics.js | 61 +++++++++ .../public/assets/thumbnail-loader.js | 1 + Overlord-Server/public/metrics.html | 38 ++++++ Overlord-Server/src/auditLog.ts | 7 + Overlord-Server/src/client-db-sync.ts | 43 +++++- Overlord-Server/src/config.ts | 18 ++- Overlord-Server/src/db/repositories.ts | 61 ++++++++- Overlord-Server/src/main-server.ts | 36 +++++- Overlord-Server/src/metrics.ts | 97 +++++++++++++- Overlord-Server/src/server/agent-auth.ts | 25 ++-- .../src/server/maintenance-loops.test.ts | 16 +++ .../src/server/maintenance-loops.ts | 45 ++++++- .../src/server/routes/client-routes.ts | 7 + .../routes/websocket-lifecycle-routes.ts | 122 ++++++++++++++---- .../src/server/routes/ws-upgrade-routes.ts | 13 +- .../src/server/runtime-constants.ts | 14 +- .../src/sessions/sessionManager.ts | 34 ++++- Overlord-Server/src/wsHandlers.test.ts | 25 +++- Overlord-Server/src/wsHandlers.ts | 4 +- 21 files changed, 649 insertions(+), 72 deletions(-) create mode 100644 Overlord-Server/src/server/maintenance-loops.test.ts diff --git a/.gitignore b/.gitignore index 87a17a1..94a0c50 100644 --- a/.gitignore +++ b/.gitignore @@ -66,6 +66,7 @@ plugins/build-sysprofile.sh plugins/build-ranreg.bat plugins/build-toxi.bat Overlord-Server/plugins/* +plugins/toxi/ plugins/toxi/* plugins/toxi copy/* plugins/extension/* @@ -79,6 +80,8 @@ Overlord-Server/data/ release/ stress/ +Overlord-Client/cmd/loadgen/ +start-prod-tuned.bat ## Ignore Visual Studio temporary files, build results, and ## files generated by popular Visual Studio add-ons. diff --git a/Overlord-Server/public/assets/data.js b/Overlord-Server/public/assets/data.js index 7efedd9..d131fc8 100644 --- a/Overlord-Server/public/assets/data.js +++ b/Overlord-Server/public/assets/data.js @@ -4,6 +4,8 @@ import { digestData } from "./utils.js"; const POLL_INTERVAL_MS = 5000; const FALLBACK_POLL_MS = 30000; const PREF_REFRESH_KEY = "overlord_refresh_interval_seconds"; +const REALTIME_MIN_LOAD_MS = 1500; +const REALTIME_CHURN_WINDOW_MS = 5000; let pollTimer = null; let render = () => {}; let lastClientData = null; @@ -12,6 +14,12 @@ const PAGE_CACHE_LIMIT = 8; let dashboardWs = null; let dashboardWsConnected = false; let wsReconnectTimer = null; +let realtimeLoadTimer = null; +let lastRealtimeLoadAt = 0; +let lastRealtimeEventAt = 0; +let realtimeEventCount = 0; +let realtimeForce = false; +let realtimeReorder = false; const manuallyDisconnecting = new Set(); function stopDashboardRealtime() { @@ -19,6 +27,8 @@ function stopDashboardRealtime() { pollTimer = null; clearTimeout(wsReconnectTimer); wsReconnectTimer = null; + clearTimeout(realtimeLoadTimer); + realtimeLoadTimer = null; if (dashboardWs) { try { dashboardWs.onopen = null; @@ -32,6 +42,38 @@ function stopDashboardRealtime() { dashboardWsConnected = false; } +function noteRealtimeEvent() { + const now = Date.now(); + if (now - lastRealtimeEventAt > REALTIME_CHURN_WINDOW_MS) { + realtimeEventCount = 0; + } + lastRealtimeEventAt = now; + realtimeEventCount += 1; +} + +function isRealtimeChurnActive() { + return Date.now() - lastRealtimeEventAt < REALTIME_CHURN_WINDOW_MS && realtimeEventCount > 5; +} + +function scheduleRealtimeLoad(options = {}) { + noteRealtimeEvent(); + realtimeForce = realtimeForce || !!options.force; + realtimeReorder = realtimeReorder || options.reorder !== false; + if (realtimeLoadTimer) return; + + const elapsed = Date.now() - lastRealtimeLoadAt; + const delay = Math.max(0, REALTIME_MIN_LOAD_MS - elapsed); + realtimeLoadTimer = setTimeout(() => { + realtimeLoadTimer = null; + lastRealtimeLoadAt = Date.now(); + const force = realtimeForce; + const reorder = realtimeReorder; + realtimeForce = false; + realtimeReorder = false; + loadWithOptions({ force, reorder }); + }, delay); +} + function bindDashboardPagehideCleanup() { window.removeEventListener("pagehide", stopDashboardRealtime); window.addEventListener("pagehide", stopDashboardRealtime); @@ -101,6 +143,7 @@ async function prefetchClientPage(page) { } function prefetchAdjacentPages(data) { + if (isRealtimeChurnActive()) return; const totalPages = Math.max(1, Math.ceil((Number(data.total) || 0) / (Number(data.pageSize) || state.pageSize || 1))); const current = Number(data.page) || state.page; if (current < totalPages) prefetchClientPage(current + 1); @@ -219,16 +262,16 @@ function connectDashboardWs() { try { const msg = typeof event.data === "string" ? JSON.parse(event.data) : null; if (msg && msg.type === "clients_changed") { - loadWithOptions({ force: false, reorder: true }); + scheduleRealtimeLoad({ force: false, reorder: true }); return; } if (msg && msg.type === "client_event") { if (msg.event === "client_offline" && manuallyDisconnecting.has(msg.clientId)) { manuallyDisconnecting.delete(msg.clientId); - loadWithOptions({ force: true }); + scheduleRealtimeLoad({ force: true, reorder: true }); } else { - moveClientCardImmediately(msg); - loadWithOptions({ force: false, reorder: true }); + if (!isRealtimeChurnActive()) moveClientCardImmediately(msg); + scheduleRealtimeLoad({ force: false, reorder: true }); } } } catch {} diff --git a/Overlord-Server/public/assets/metrics.js b/Overlord-Server/public/assets/metrics.js index f86ff84..1f9c182 100644 --- a/Overlord-Server/public/assets/metrics.js +++ b/Overlord-Server/public/assets/metrics.js @@ -780,6 +780,67 @@ function updateMetrics(data, debug) { } } + const internalTasksList = document.getElementById("internal-tasks"); + if (internalTasksList) { + const tasks = Array.isArray(data.internal?.tasks) ? data.internal.tasks : []; + if (tasks.length > 0) { + internalTasksList.innerHTML = tasks + .map((task) => ` +
+
+ ${escapeHtml(task.task)} +
+
+ p95 + ${formatMs(task.durationP95 || 0)} + avg + ${formatMs(task.durationAvg || 0)} + max + ${formatMs(task.durationMax || 0)} + runs/min + ${Number(task.countLastMinute || 0).toLocaleString()} +
+
+ `) + .join(""); + } else { + internalTasksList.innerHTML = + '
No internal job samples yet
'; + } + } + + const retainedStateList = document.getElementById("retained-state"); + if (retainedStateList) { + const retained = data.diagnostics?.retained || {}; + const entries = Object.entries(retained) + .filter(([, value]) => typeof value === "number" || typeof value === "boolean") + .sort(([a], [b]) => a.localeCompare(b)); + if (entries.length > 0) { + retainedStateList.innerHTML = entries + .map(([key, value]) => { + const displayValue = key.toLowerCase().includes("bytes") + ? formatBytes(Number(value || 0)) + : typeof value === "boolean" + ? value ? "yes" : "no" + : Number(value || 0).toLocaleString(); + return ` +
+
+ ${escapeHtml(key)} +
+
+ ${escapeHtml(displayValue)} +
+
+ `; + }) + .join(""); + } else { + retainedStateList.innerHTML = + '
No retained state counters yet
'; + } + } + const commandTypesList = document.getElementById("command-types"); if (Object.keys(data.commands.byType).length > 0) { const topCommands = Object.entries(data.commands.byType) diff --git a/Overlord-Server/public/assets/thumbnail-loader.js b/Overlord-Server/public/assets/thumbnail-loader.js index 367adfd..fd93f6b 100644 --- a/Overlord-Server/public/assets/thumbnail-loader.js +++ b/Overlord-Server/public/assets/thumbnail-loader.js @@ -36,6 +36,7 @@ async function triggerAgentThumbnail(clientId) { const res = await fetch(`/api/clients/${encodeURIComponent(clientId)}/thumbnail`, { method: "POST", credentials: "include", + headers: { "X-Overlord-Thumbnail-Source": "dashboard" }, }); if (!res.ok) return null; return await res.json().catch(() => null); diff --git a/Overlord-Server/public/metrics.html b/Overlord-Server/public/metrics.html index 8c2d1f0..10d81b1 100644 --- a/Overlord-Server/public/metrics.html +++ b/Overlord-Server/public/metrics.html @@ -258,6 +258,44 @@ + +
+
+

+ + Slowest Internal Jobs +

+
Sorted by p95 duration, last minute
+
+
+
+ No internal job samples yet +
+
+
+ + +
+
+

+ + Retained Runtime State +

+
Long-lived maps, queues, and caches
+
+
+
+ No retained state counters yet +
+
+
+
diff --git a/Overlord-Server/src/auditLog.ts b/Overlord-Server/src/auditLog.ts index 3d43d33..af7b271 100644 --- a/Overlord-Server/src/auditLog.ts +++ b/Overlord-Server/src/auditLog.ts @@ -1,6 +1,7 @@ import Database from "bun:sqlite"; import { resolve } from "path"; import { ensureDataDir } from "./paths"; +import { metrics } from "./metrics"; const dataDir = ensureDataDir(); const dbPath = resolve(dataDir, "overlord.db"); @@ -145,6 +146,7 @@ function flushAuditLogs(): void { if (auditQueue.length === 0) return; const logsToWrite = auditQueue.splice(0, auditQueue.length); + const startedAt = Date.now(); try { const insert = db.prepare( @@ -168,6 +170,7 @@ function flushAuditLogs(): void { }); transaction(logsToWrite); + metrics.recordInternalTask("audit-flush", Date.now() - startedAt); } catch (error) { console.error( `[audit] Failed to flush ${logsToWrite.length} audit entries:`, @@ -180,6 +183,10 @@ export function flushAuditLogsSync(): void { flushAuditLogs(); } +export function getAuditQueueStats(): { queued: number } { + return { queued: auditQueue.length }; +} + export function getAuditLogs(filters: AuditLogFilters = {}): { logs: AuditLogEntry[]; total: number; diff --git a/Overlord-Server/src/client-db-sync.ts b/Overlord-Server/src/client-db-sync.ts index 446222f..ffbda52 100644 --- a/Overlord-Server/src/client-db-sync.ts +++ b/Overlord-Server/src/client-db-sync.ts @@ -1,9 +1,15 @@ -import { upsertClientRow, type ClientDbRow } from "./db"; +import { upsertClientRows, type ClientDbRow } from "./db"; +import { metrics } from "./metrics"; export const CLIENT_DB_SYNC_INTERVAL_MS = Number(process.env.OVERLORD_CLIENT_DB_SYNC_MS || 5000); +export const CLIENT_DB_SYNC_BATCH_SIZE = Math.max( + 1, + Number(process.env.OVERLORD_CLIENT_DB_SYNC_BATCH_SIZE || 100), +); const lastClientDbSync = new Map(); const pendingClientDbUpdates = new Map(); +let flushSoonTimer: ReturnType | null = null; export function queueClientDbUpdate(partial: ClientDbRow): void { const existing = pendingClientDbUpdates.get(partial.id); @@ -23,10 +29,27 @@ export function queueClientDbUpdate(partial: ClientDbRow): void { export function flushQueuedClientDbUpdates(): void { if (pendingClientDbUpdates.size === 0) return; - for (const update of pendingClientDbUpdates.values()) { - upsertClientRow(update); + + const startedAt = Date.now(); + let processed = 0; + const updates: ClientDbRow[] = []; + for (const [clientId, update] of pendingClientDbUpdates.entries()) { + updates.push(update); + pendingClientDbUpdates.delete(clientId); + processed += 1; + if (processed >= CLIENT_DB_SYNC_BATCH_SIZE) { + break; + } } - pendingClientDbUpdates.clear(); + upsertClientRows(updates); + + if (pendingClientDbUpdates.size > 0 && !flushSoonTimer) { + flushSoonTimer = setTimeout(() => { + flushSoonTimer = null; + flushQueuedClientDbUpdates(); + }, 25); + } + metrics.recordInternalTask("client-db-flush", Date.now() - startedAt); } setInterval(flushQueuedClientDbUpdates, CLIENT_DB_SYNC_INTERVAL_MS); @@ -46,3 +69,15 @@ export function clearClientSyncState(clientId: string): void { lastClientDbSync.delete(clientId); pendingClientDbUpdates.delete(clientId); } + +export function getClientDbSyncStats(): { + trackedClients: number; + pendingUpdates: number; + flushScheduled: boolean; +} { + return { + trackedClients: lastClientDbSync.size, + pendingUpdates: pendingClientDbUpdates.size, + flushScheduled: flushSoonTimer !== null, + }; +} diff --git a/Overlord-Server/src/config.ts b/Overlord-Server/src/config.ts index 569df3e..84a760e 100644 --- a/Overlord-Server/src/config.ts +++ b/Overlord-Server/src/config.ts @@ -178,6 +178,14 @@ const DEFAULT_CONFIG: Config = { }, }; +function envBoolOverride(name: string): boolean | undefined { + const raw = String(process.env[name] || "").trim().toLowerCase(); + if (!raw) return undefined; + if (["1", "true", "yes", "on"].includes(raw)) return true; + if (["0", "false", "no", "off"].includes(raw)) return false; + return undefined; +} + type SaveSecrets = { auth?: { jwtSecret?: string; @@ -591,13 +599,15 @@ export function loadConfig(): Config { }, thumbnails: { dashboardEnabled: - fileConfig.thumbnails?.dashboardEnabled !== undefined + envBoolOverride("OVERLORD_THUMBNAILS_DASHBOARD_ENABLED") ?? + (fileConfig.thumbnails?.dashboardEnabled !== undefined ? Boolean(fileConfig.thumbnails.dashboardEnabled) - : DEFAULT_CONFIG.thumbnails.dashboardEnabled, + : DEFAULT_CONFIG.thumbnails.dashboardEnabled), wallEnabled: - fileConfig.thumbnails?.wallEnabled !== undefined + envBoolOverride("OVERLORD_THUMBNAILS_WALL_ENABLED") ?? + (fileConfig.thumbnails?.wallEnabled !== undefined ? Boolean(fileConfig.thumbnails.wallEnabled) - : DEFAULT_CONFIG.thumbnails.wallEnabled, + : DEFAULT_CONFIG.thumbnails.wallEnabled), }, }; diff --git a/Overlord-Server/src/db/repositories.ts b/Overlord-Server/src/db/repositories.ts index f51f116..ba8ad33 100644 --- a/Overlord-Server/src/db/repositories.ts +++ b/Overlord-Server/src/db/repositories.ts @@ -215,12 +215,13 @@ export type ClientDbRow = Omit, "online"> & { online?: number; }; -export function upsertClientRow( - partial: ClientDbRow, -) { - const now = partial.lastSeen ?? Date.now(); - db.run( - `INSERT INTO clients (id, hwid, role, ip, host, os, arch, version, user, nickname, custom_tag, custom_tag_note, monitors, country, last_seen, online, ping_ms, build_tag, built_by_user_id, enrollment_status, public_key, key_fingerprint, cpu, gpu, ram, battery_percent, battery_charging, is_admin, elevation, permissions) +export type OfflineStateUpdate = { + id: string; + disconnectReason?: string; + disconnectDetail?: string; +}; + +const UPSERT_CLIENT_ROW_SQL = `INSERT INTO clients (id, hwid, role, ip, host, os, arch, version, user, nickname, custom_tag, custom_tag_note, monitors, country, last_seen, online, ping_ms, build_tag, built_by_user_id, enrollment_status, public_key, key_fingerprint, cpu, gpu, ram, battery_percent, battery_charging, is_admin, elevation, permissions) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, COALESCE(?, 0), ?, ?) ON CONFLICT(id) DO UPDATE SET hwid=COALESCE(excluded.hwid, clients.hwid), @@ -252,7 +253,13 @@ export function upsertClientRow( is_admin=COALESCE(excluded.is_admin, clients.is_admin), elevation=COALESCE(excluded.elevation, clients.elevation), permissions=COALESCE(excluded.permissions, clients.permissions) - `, + `; + +const upsertClientRowStmt = db.prepare(UPSERT_CLIENT_ROW_SQL); + +function upsertClientRowInternal(partial: ClientDbRow): void { + const now = partial.lastSeen ?? Date.now(); + upsertClientRowStmt.run( partial.id, partial.hwid ?? partial.id, partial.role ?? null, @@ -292,6 +299,24 @@ export function upsertClientRow( partial.id, ); } +} + +const upsertClientRowsTx = db.transaction((rows: ClientDbRow[]) => { + for (const row of rows) { + upsertClientRowInternal(row); + } +}); + +export function upsertClientRow( + partial: ClientDbRow, +) { + upsertClientRowInternal(partial); + invalidateClientMetricsSummaryCache(); +} + +export function upsertClientRows(rows: ClientDbRow[]): void { + if (rows.length === 0) return; + upsertClientRowsTx(rows); invalidateClientMetricsSummaryCache(); } @@ -314,6 +339,28 @@ export function setOnlineState(id: string, online: boolean, disconnectReason?: s invalidateClientMetricsSummaryCache(); } +const setOfflineStateStmt = db.prepare( + `UPDATE clients SET online=0, last_seen=?, disconnect_reason=?, disconnect_detail=? WHERE id=?`, +); + +const setOfflineStatesTx = db.transaction((updates: OfflineStateUpdate[]) => { + const now = Date.now(); + for (const update of updates) { + setOfflineStateStmt.run( + now, + update.disconnectReason ?? null, + update.disconnectDetail ?? null, + update.id, + ); + } +}); + +export function setOfflineStates(updates: OfflineStateUpdate[]): void { + if (updates.length === 0) return; + setOfflineStatesTx(updates); + invalidateClientMetricsSummaryCache(); +} + export function deleteClientRow(id: string) { db.run(`DELETE FROM clients WHERE id=?`, id); invalidateClientMetricsSummaryCache(); diff --git a/Overlord-Server/src/main-server.ts b/Overlord-Server/src/main-server.ts index b941640..82ddc01 100644 --- a/Overlord-Server/src/main-server.ts +++ b/Overlord-Server/src/main-server.ts @@ -12,10 +12,11 @@ import { ClientInfo, ClientRole } from "./types"; import { v4 as uuidv4 } from "uuid"; import { authenticateRequest } from "./auth"; import { loadConfig, getConfig } from "./config"; -import { flushAuditLogsSync } from "./auditLog"; +import { flushAuditLogsSync, getAuditQueueStats } from "./auditLog"; import { getUserById, getUsersForNotificationDelivery, getUsersForNotificationDeliveryByClientOwnership, isClientOwnedByUser, canUserAccessClient } from "./users"; import { requireAuth, requirePermission } from "./rbac"; import { metrics } from "./metrics"; +import { getClientDbSyncStats } from "./client-db-sync"; import { ensureDataDir } from "./paths"; import { handleAuthRoutes } from "./server/routes/auth-routes"; import { handleAutoScriptsRoutes } from "./server/routes/auto-scripts-routes"; @@ -146,6 +147,7 @@ import { } from "./server/ws-desktop-audio"; import { createNotificationPluginHandlers } from "./server/ws-notifications-plugin"; import { loadOrGenerateVapidKeys } from "./server/web-push"; +import { getThumbnailStats } from "./thumbnails"; import * as clientManager from "./clientManager"; import * as sessionManager from "./sessions/sessionManager"; import type { SocketData } from "./sessions/types"; @@ -156,6 +158,9 @@ import { handleBuildTagConnection } from "./server/build-tag"; metrics.setSnapshotEnricher((snapshot) => { const summary = getClientMetricsSummary(); + const dbSync = getClientDbSyncStats(); + const audit = getAuditQueueStats(); + const thumbnails = getThumbnailStats(); snapshot.clients.total = summary.total; snapshot.clients.online = summary.online; snapshot.clients.offline = summary.total - summary.online; @@ -165,6 +170,35 @@ metrics.setSnapshotEnricher((snapshot) => { snapshot.sessions.remoteDesktop = sessionManager.getRdSessionCount(); snapshot.sessions.fileBrowser = sessionManager.getFileBrowserSessionCount(); snapshot.sessions.process = sessionManager.getProcessSessionCount(); + snapshot.diagnostics = { + retained: { + clientsMap: clientManager.getClientCount(), + clientDbSyncTracked: dbSync.trackedClients, + clientDbSyncPending: dbSync.pendingUpdates, + clientDbSyncFlushScheduled: dbSync.flushScheduled, + auditQueued: audit.queued, + thumbnailsCached: thumbnails.cachedCount, + thumbnailBytes: thumbnails.cachedBytes, + thumbnailPendingFrames: thumbnails.pendingFrames, + thumbnailGenQueued: thumbnails.genQueued, + thumbnailGenState: thumbnails.genStateTracked, + dashboardSessions: sessionManager.getDashboardSessionCount(), + notificationSessions: sessionManager.getNotificationSessionCount(), + chatSessions: sessionManager.getChatSessionCount(), + pluginLoadedClients: pluginLoadedByClient.size, + pluginLoadingClients: pluginLoadingByClient.size, + pendingPluginEvents: pendingPluginEvents.size, + pendingHttpDownloads: pendingHttpDownloads.size, + downloadIntents: downloadIntents.size, + deployUploads: deployUploads.size, + winreUploads: winreUploads.size, + notificationRateClients: notificationRate.size, + pendingNotificationScreenshots: pendingNotificationScreenshots.size, + rdStreamingClients: rdStreamingState.size, + hvncStreamingClients: hvncStreamingState.size, + webcamStreamingClients: webcamStreamingState.size, + }, + }; }); const config = loadConfig(); diff --git a/Overlord-Server/src/metrics.ts b/Overlord-Server/src/metrics.ts index 36a3d48..d73ba00 100644 --- a/Overlord-Server/src/metrics.ts +++ b/Overlord-Server/src/metrics.ts @@ -66,6 +66,14 @@ export interface MetricsSnapshot { avg: number; max: number; p95: number; + sampleMs: number; + samples: number; + }; + internal: { + tasks: InternalTaskStats[]; + }; + diagnostics?: { + retained: Record; }; } @@ -81,6 +89,17 @@ export interface HttpRouteStats { lastStatus: number; } +export interface InternalTaskStats { + task: string; + countLastMinute: number; + durationAvg: number; + durationP95: number; + durationP99: number; + durationMax: number; + lastDuration: number; + lastAt: number; +} + export interface MetricsHistory { timestamp: number; clientsOnline: number; @@ -106,6 +125,11 @@ interface TimedHttpSample { statusCode: number; } +interface TimedInternalTaskSample { + ts: number; + duration: number; +} + class MetricsCollector { private startTime: number = Date.now(); @@ -141,7 +165,17 @@ class MetricsCollector { private ignoredHttpMetricRoutes: Set = new Set(["GET /api/metrics"]); private eventLoopDelays: number[] = []; - private maxEventLoopHistory: number = 300; + private eventLoopSampleMs: number = Math.max( + 20, + Number(process.env.OVERLORD_EVENT_LOOP_SAMPLE_MS || 100), + ); + private maxEventLoopHistory: number = Math.max( + 10, + Math.ceil((Number(process.env.OVERLORD_EVENT_LOOP_HISTORY_SECONDS || 60) * 1000) / this.eventLoopSampleMs), + ); + + private internalTaskSamples: Map = new Map(); + private maxInternalTaskSamples: number = 300; private pruneTimestampWindow(list: number[], minTs: number): void { let removeCount = 0; @@ -257,7 +291,7 @@ class MetricsCollector { } private trackEventLoopDelay() { - const intervalMs = 1000; + const intervalMs = this.eventLoopSampleMs; let last = Date.now(); setInterval(() => { const now = Date.now(); @@ -282,7 +316,7 @@ class MetricsCollector { : 0; } - private pruneHttpSamples(list: TimedHttpSample[], minTs: number): void { + private pruneTimedSamples(list: Array<{ ts: number }>, minTs: number): void { let removeCount = 0; while (removeCount < list.length && list[removeCount].ts <= minTs) { removeCount += 1; @@ -303,7 +337,7 @@ class MetricsCollector { private getTopHttpRoutes(minTs: number): HttpRouteStats[] { const routes: HttpRouteStats[] = []; for (const [route, samples] of this.httpRouteSamples.entries()) { - this.pruneHttpSamples(samples, minTs); + this.pruneTimedSamples(samples, minTs); if (samples.length === 0) { this.httpRouteSamples.delete(route); continue; @@ -335,6 +369,40 @@ class MetricsCollector { .slice(0, 8); } + private getTopInternalTasks(minTs: number): InternalTaskStats[] { + const tasks: InternalTaskStats[] = []; + for (const [task, samples] of this.internalTaskSamples.entries()) { + this.pruneTimedSamples(samples, minTs); + if (samples.length === 0) { + this.internalTaskSamples.delete(task); + continue; + } + + const durations = samples + .map((sample) => sample.duration) + .sort((a, b) => a - b); + const last = samples[samples.length - 1]; + tasks.push({ + task, + countLastMinute: samples.length, + durationAvg: this.average(durations), + durationP95: this.percentile(durations, 0.95), + durationP99: this.percentile(durations, 0.99), + durationMax: durations[durations.length - 1] ?? 0, + lastDuration: last?.duration ?? 0, + lastAt: last?.ts ?? 0, + }); + } + + return tasks + .sort((a, b) => { + if (b.durationP95 !== a.durationP95) return b.durationP95 - a.durationP95; + if (b.durationAvg !== a.durationAvg) return b.durationAvg - a.durationAvg; + return b.countLastMinute - a.countLastMinute; + }) + .slice(0, 8); + } + recordHttpRequest(durationMs: number, statusCode: number, route = "unknown") { if (this.ignoredHttpMetricRoutes.has(route)) return; @@ -362,6 +430,18 @@ class MetricsCollector { } } + recordInternalTask(task: string, durationMs: number) { + if (!task || !Number.isFinite(durationMs)) return; + const now = Date.now(); + const sample = { ts: now, duration: Math.max(0, durationMs) }; + const samples = this.internalTaskSamples.get(task) || []; + samples.push(sample); + if (samples.length > this.maxInternalTaskSamples) { + samples.splice(0, samples.length - this.maxInternalTaskSamples); + } + this.internalTaskSamples.set(task, samples); + } + async withHttpMetrics( handler: () => Promise, route = "unknown", @@ -424,7 +504,7 @@ class MetricsCollector { this.pruneTimestampWindow(this.commandTimestamps, oneHourAgo); this.pruneTimestampWindow(this.httpTimestamps, oneMinuteAgo); this.pruneTimestampWindow(this.httpErrorTimestamps, oneMinuteAgo); - this.pruneHttpSamples(this.httpSamples, oneMinuteAgo); + this.pruneTimedSamples(this.httpSamples, oneMinuteAgo); const commandsLastMinute = this.countRecent(this.commandTimestamps, oneMinuteAgo); const commandsLastHour = this.commandTimestamps.length; @@ -444,6 +524,7 @@ class MetricsCollector { const httpLatencyP95 = this.percentile(httpLatencySamples, 0.95); const httpLatencyP99 = this.percentile(httpLatencySamples, 0.99); const httpRoutes = this.getTopHttpRoutes(oneMinuteAgo); + const internalTasks = this.getTopInternalTasks(oneMinuteAgo); const eventLoopSamples = [...this.eventLoopDelays].sort((a, b) => a - b); const eventLoopAvg = eventLoopSamples.length @@ -532,6 +613,11 @@ class MetricsCollector { avg: eventLoopAvg, max: eventLoopMax, p95: eventLoopP95, + sampleMs: this.eventLoopSampleMs, + samples: eventLoopSamples.length, + }, + internal: { + tasks: internalTasks, }, }; } @@ -558,6 +644,7 @@ class MetricsCollector { this.httpSamples = []; this.httpRouteSamples.clear(); this.eventLoopDelays = []; + this.internalTaskSamples.clear(); } } diff --git a/Overlord-Server/src/server/agent-auth.ts b/Overlord-Server/src/server/agent-auth.ts index de07418..d6df259 100644 --- a/Overlord-Server/src/server/agent-auth.ts +++ b/Overlord-Server/src/server/agent-auth.ts @@ -1,6 +1,10 @@ import { logger } from "../logger"; import { timingSafeEqual } from "crypto"; +let warnedDisableAuthIgnored = false; +let loggedAuthDisabledByEnv = false; +let loggedAuthDisabledNoToken = false; + function safeCompare(a: string, b: string): boolean { const bufA = Buffer.from(a); const bufB = Buffer.from(b); @@ -19,16 +23,25 @@ export function isAuthorizedAgentRequest( if (disableAuth) { const nodeEnv = String(process.env.NODE_ENV || "development").toLowerCase(); if (nodeEnv === "production") { - logger.warn("[auth] OVERLORD_DISABLE_AGENT_AUTH is ignored in production mode"); + if (!warnedDisableAuthIgnored) { + warnedDisableAuthIgnored = true; + logger.warn("[auth] OVERLORD_DISABLE_AGENT_AUTH is ignored in production mode"); + } } else { - logger.info("[auth] Agent auth explicitly disabled by OVERLORD_DISABLE_AGENT_AUTH=true (non-production mode)"); + if (!loggedAuthDisabledByEnv) { + loggedAuthDisabledByEnv = true; + logger.info("[auth] Agent auth explicitly disabled by OVERLORD_DISABLE_AGENT_AUTH=true (non-production mode)"); + } return true; } } const token = agentToken?.trim(); if (!token) { - logger.info("[auth] Agent auth disabled"); + if (!loggedAuthDisabledNoToken) { + loggedAuthDisabledNoToken = true; + logger.info("[auth] Agent auth disabled"); + } return true; } @@ -38,11 +51,5 @@ export function isAuthorizedAgentRequest( (headerToken !== null && safeCompare(headerToken, token)) || (queryToken !== null && safeCompare(queryToken, token)); - if (!isAuthed) { - logger.info("[auth] Agent auth failed"); - } else { - logger.info("[auth] Agent authenticated successfully"); - } - return isAuthed; } diff --git a/Overlord-Server/src/server/maintenance-loops.test.ts b/Overlord-Server/src/server/maintenance-loops.test.ts new file mode 100644 index 0000000..97615cc --- /dev/null +++ b/Overlord-Server/src/server/maintenance-loops.test.ts @@ -0,0 +1,16 @@ +import { describe, expect, test } from "bun:test"; +import { getHeartbeatBatchSize } from "./maintenance-loops"; + +describe("maintenance heartbeat batching", () => { + test("spreads large fleets across the heartbeat interval", () => { + expect(getHeartbeatBatchSize(1000, 30_000)).toBe(34); + }); + + test("keeps small fleets responsive", () => { + expect(getHeartbeatBatchSize(10, 30_000)).toBe(1); + }); + + test("handles empty fleets", () => { + expect(getHeartbeatBatchSize(0, 30_000)).toBe(0); + }); +}); diff --git a/Overlord-Server/src/server/maintenance-loops.ts b/Overlord-Server/src/server/maintenance-loops.ts index 9330cc5..6b20b00 100644 --- a/Overlord-Server/src/server/maintenance-loops.ts +++ b/Overlord-Server/src/server/maintenance-loops.ts @@ -1,4 +1,5 @@ import { logger } from "../logger"; +import { metrics } from "../metrics"; import { sendPingRequest } from "../wsHandlers"; import type { ClientInfo } from "../types"; import { pruneStaleClients } from "./stale-prune"; @@ -13,8 +14,30 @@ type StartMaintenanceParams = { disconnectTimeoutMs: number; }; +function nonNegativeIntEnv(name: string, fallback: number): number { + const raw = String(process.env[name] || "").trim(); + if (!raw) return fallback; + const parsed = Number(raw); + if (!Number.isFinite(parsed) || parsed < 0) return fallback; + return Math.floor(parsed); +} + +const HEARTBEAT_SWEEP_TICK_MS = Math.max( + 250, + nonNegativeIntEnv("OVERLORD_HEARTBEAT_SWEEP_TICK_MS", 1_000), +); +const HEARTBEAT_BATCH_SIZE = nonNegativeIntEnv("OVERLORD_HEARTBEAT_BATCH_SIZE", 0); + +export function getHeartbeatBatchSize(totalClients: number, heartbeatIntervalMs: number): number { + if (totalClients <= 0) return 0; + if (HEARTBEAT_BATCH_SIZE > 0) return Math.min(totalClients, HEARTBEAT_BATCH_SIZE); + const ticksPerSweep = Math.max(1, Math.floor(heartbeatIntervalMs / HEARTBEAT_SWEEP_TICK_MS)); + return Math.max(1, Math.ceil(totalClients / ticksPerSweep)); +} + export function startMaintenanceLoops(params: StartMaintenanceParams): void { setInterval(() => { + const startedAt = Date.now(); pruneStaleClients({ clients: params.getClients(), staleMs: params.staleMs, @@ -22,6 +45,7 @@ export function startMaintenanceLoops(params: StartMaintenanceParams): void { setOnlineState: params.setOnlineState, deleteClient: params.deleteClient, }); + metrics.recordInternalTask("stale-prune", Date.now() - startedAt); }, 5000); const livenessTimeoutMs = Math.max( @@ -29,10 +53,23 @@ export function startMaintenanceLoops(params: StartMaintenanceParams): void { 60_000, ); + let heartbeatCursor = 0; setInterval(() => { + const startedAt = Date.now(); const now = Date.now(); - for (const [id, info] of params.getClients().entries()) { - if (info.role !== "client") continue; + const clients = Array.from(params.getClients().entries()).filter( + ([, info]) => info.role === "client", + ); + if (clients.length === 0) { + heartbeatCursor = 0; + metrics.recordInternalTask("heartbeat-sweep", Date.now() - startedAt); + return; + } + if (heartbeatCursor >= clients.length) heartbeatCursor = 0; + + const batchSize = getHeartbeatBatchSize(clients.length, params.heartbeatIntervalMs); + for (let i = 0; i < batchSize; i++) { + const [id, info] = clients[(heartbeatCursor + i) % clients.length]; const lastActivity = info.lastSeen || 0; if (lastActivity && now - lastActivity > livenessTimeoutMs) { logger.warn( @@ -51,5 +88,7 @@ export function startMaintenanceLoops(params: StartMaintenanceParams): void { logger.debug(`[ping] heartbeat failed for ${id}`, err); } } - }, params.heartbeatIntervalMs); + heartbeatCursor = (heartbeatCursor + batchSize) % clients.length; + metrics.recordInternalTask("heartbeat-sweep", Date.now() - startedAt); + }, HEARTBEAT_SWEEP_TICK_MS); } diff --git a/Overlord-Server/src/server/routes/client-routes.ts b/Overlord-Server/src/server/routes/client-routes.ts index b834ad2..297c303 100644 --- a/Overlord-Server/src/server/routes/client-routes.ts +++ b/Overlord-Server/src/server/routes/client-routes.ts @@ -23,6 +23,7 @@ import { } from "../../db"; import { metrics } from "../../metrics"; import { encodeMessage } from "../../protocol"; +import { getConfig } from "../../config"; import { requireClientAccess, requireFeatureAccess, requirePermission } from "../../rbac"; import { canUserAccessClient, @@ -176,6 +177,12 @@ export async function handleClientRoutes( if (error instanceof Response) return error; return new Response("Forbidden", { status: 403 }); } + if ( + req.headers.get("x-overlord-thumbnail-source") === "dashboard" && + getConfig().thumbnails.dashboardEnabled === false + ) { + return Response.json({ ok: false, disabled: true }, { headers: deps.CORS_HEADERS }); + } const { requestThumbnailRegen, markThumbnailRequested, getThumbnailVersion, waitForThumbnail, clearThumbnailRequest } = await import("../../thumbnails"); markThumbnailRequested(clientId); const target = clientManager.getClient(clientId); diff --git a/Overlord-Server/src/server/routes/websocket-lifecycle-routes.ts b/Overlord-Server/src/server/routes/websocket-lifecycle-routes.ts index 7417cb8..743caaf 100644 --- a/Overlord-Server/src/server/routes/websocket-lifecycle-routes.ts +++ b/Overlord-Server/src/server/routes/websocket-lifecycle-routes.ts @@ -9,7 +9,7 @@ async function getGeoip() { } import { logAudit, AuditAction } from "../../auditLog"; import * as clientManager from "../../clientManager"; -import { clientExists, setOnlineState, upsertClientRow, getClientEnrollmentStatus, setClientEnrollmentStatus, lookupClientByPublicKey, getClientPublicKeyById, getBuildByTag, computeClientSuspiciousFlags } from "../../db"; +import { clientExists, setOnlineState, setOfflineStates, upsertClientRow, getClientEnrollmentStatus, setClientEnrollmentStatus, lookupClientByPublicKey, getClientPublicKeyById, getBuildByTag, computeClientSuspiciousFlags, type OfflineStateUpdate } from "../../db"; import { getConfig } from "../../config"; import { logger } from "../../logger"; import { metrics } from "../../metrics"; @@ -18,6 +18,7 @@ import * as sessionManager from "../../sessions/sessionManager"; import type { SocketData } from "../../sessions/types"; import type { ClientInfo } from "../../types"; import { clearClientSyncState, handleFrame, handleHello, handlePing, handlePong } from "../../wsHandlers"; +import { queueClientDbUpdate } from "../../client-db-sync"; import { getMaxPayloadLimit, getMessageByteLength, isAllowedClientMessageType } from "../../wsValidation"; import { stopAllProxiesForClient } from "../socks5-proxy-manager"; import { verifyBuildToken, isBuildBanned } from "../build-signing"; @@ -42,19 +43,96 @@ type ClientLifecyclePayload = { }; type PendingOffline = { - timer: ReturnType; + dueAt: number; + payload: ClientLifecyclePayload; + disconnectReason: string | undefined; + disconnectDetail: string | undefined; + deps: WsLifecycleDeps; }; const pendingOffline = new Map(); +let pendingOfflineFlushTimer: ReturnType | null = null; +const OFFLINE_BATCH_SIZE = Math.max( + 1, + Number(process.env.OVERLORD_OFFLINE_BATCH_SIZE || 250), +); +const OFFLINE_BATCH_DELAY_MS = Math.max( + 1, + Number(process.env.OVERLORD_OFFLINE_BATCH_DELAY_MS || 25), +); function cancelPendingOffline(clientId: string): boolean { const pending = pendingOffline.get(clientId); if (!pending) return false; - clearTimeout(pending.timer); pendingOffline.delete(clientId); return true; } +function applyPendingOffline( + clientId: string, + pending: PendingOffline, + offlineUpdates: OfflineStateUpdate[], +): void { + if (clientManager.hasClient(clientId)) return; + pending.deps.notifyDashboardClientEvent("client_offline", pending.payload); + pending.deps.broadcastClientEvent("client_offline", pending.payload); + pending.deps.notifyRemoteDesktopStatus(clientId, "offline", "Client disconnected"); + offlineUpdates.push({ + id: clientId, + disconnectReason: pending.disconnectReason, + disconnectDetail: pending.disconnectDetail, + }); + pending.deps.notifyDashboard(); +} + +function schedulePendingOfflineFlush(delayMs: number): void { + if (pendingOfflineFlushTimer) return; + pendingOfflineFlushTimer = setTimeout(() => { + pendingOfflineFlushTimer = null; + flushPendingOffline(); + }, delayMs); +} + +function flushPendingOffline(): void { + if (pendingOffline.size === 0) return; + + const startedAt = Date.now(); + const now = Date.now(); + let processed = 0; + let nextDueAt = Number.POSITIVE_INFINITY; + const offlineUpdates: OfflineStateUpdate[] = []; + + for (const [clientId, pending] of pendingOffline) { + if (pending.dueAt > now) { + if (pending.dueAt < nextDueAt) nextDueAt = pending.dueAt; + continue; + } + + pendingOffline.delete(clientId); + applyPendingOffline(clientId, pending, offlineUpdates); + processed += 1; + + if (processed >= OFFLINE_BATCH_SIZE) { + break; + } + } + + if (offlineUpdates.length > 0) { + const dbStartedAt = Date.now(); + setOfflineStates(offlineUpdates); + metrics.recordInternalTask("offline-db-flush", Date.now() - dbStartedAt); + } + metrics.recordInternalTask("offline-flush", Date.now() - startedAt); + if (pendingOffline.size === 0) return; + if (processed >= OFFLINE_BATCH_SIZE) { + schedulePendingOfflineFlush(OFFLINE_BATCH_DELAY_MS); + return; + } + if (Number.isFinite(nextDueAt)) { + schedulePendingOfflineFlush(Math.max(1, nextDueAt - Date.now())); + } +} + function schedulePendingOffline( clientId: string, payload: ClientLifecyclePayload, @@ -65,24 +143,25 @@ function schedulePendingOffline( cancelPendingOffline(clientId); if (OFFLINE_GRACE_MS <= 0) { - deps.notifyDashboardClientEvent("client_offline", payload); - deps.broadcastClientEvent("client_offline", payload); - setOnlineState(clientId, false, disconnectReason, disconnectDetail); - deps.notifyRemoteDesktopStatus(clientId, "offline", "Client disconnected"); - deps.notifyDashboard(); + pendingOffline.set(clientId, { + dueAt: Date.now(), + payload, + disconnectReason, + disconnectDetail, + deps, + }); + schedulePendingOfflineFlush(1); return; } - const timer = setTimeout(() => { - pendingOffline.delete(clientId); - deps.notifyDashboardClientEvent("client_offline", payload); - deps.broadcastClientEvent("client_offline", payload); - setOnlineState(clientId, false, disconnectReason, disconnectDetail); - deps.notifyRemoteDesktopStatus(clientId, "offline", "Client disconnected"); - deps.notifyDashboard(); - }, OFFLINE_GRACE_MS); - - pendingOffline.set(clientId, { timer }); + pendingOffline.set(clientId, { + dueAt: Date.now() + OFFLINE_GRACE_MS, + payload, + disconnectReason, + disconnectDetail, + deps, + }); + schedulePendingOfflineFlush(OFFLINE_GRACE_MS); } type PendingScript = { @@ -263,8 +342,6 @@ export function handleWebSocketOpen(ws: ServerWebSocket, deps: WsLif } catch {} }, ENROLLMENT_TIMEOUT_MS); enrollmentTimeouts.set(id, timeout); - - logger.info(`[open] ${id} role=${role} — purgatory challenge sent`); } export async function handleWebSocketMessage( @@ -560,7 +637,7 @@ export async function handleWebSocketMessage( }; clientManager.addClient(resolvedId, infoObj); - upsertClientRow({ + queueClientDbUpdate({ id: resolvedId, publicKey, keyFingerprint, @@ -644,7 +721,6 @@ export async function handleWebSocketMessage( break; case "pong": handlePong(client, payload); - deps.notifyDashboard(); break; case "frame": if ((payload as any)?.header?.fps === 0) { @@ -963,7 +1039,6 @@ export function handleWebSocketClose( const currentClient = clientManager.getClient(clientId); if (currentClient && currentClient.ws !== ws) { - logger.info(`[close] ${clientId} code=${code} (superseded socket, skipping cleanup)`); return; } @@ -994,7 +1069,6 @@ export function handleWebSocketClose( deps.hvncStreamingState.delete(clientId); deps.webcamStreamingState.delete(clientId); stopRemoteDesktopRecording(clientId, "client disconnected"); - logger.info(`[close] ${clientId} code=${code} reason=${reason} disconnect_reason=${storedDisconnectReason || "unknown"}`); if (role === "client" && currentClient) { schedulePendingOffline( diff --git a/Overlord-Server/src/server/routes/ws-upgrade-routes.ts b/Overlord-Server/src/server/routes/ws-upgrade-routes.ts index a5aea22..3e35340 100644 --- a/Overlord-Server/src/server/routes/ws-upgrade-routes.ts +++ b/Overlord-Server/src/server/routes/ws-upgrade-routes.ts @@ -14,8 +14,16 @@ type WsUpgradeDeps = { isAuthorizedAgentRequest: (req: Request, url: URL) => boolean; }; -const WS_RATE_WINDOW_MS = 10_000; -const WS_RATE_MAX = 30; +function positiveIntEnv(name: string, fallback: number): number { + const raw = String(process.env[name] || "").trim(); + if (!raw) return fallback; + const parsed = Number(raw); + if (!Number.isFinite(parsed) || parsed <= 0) return fallback; + return Math.floor(parsed); +} + +const WS_RATE_WINDOW_MS = positiveIntEnv("OVERLORD_WS_UPGRADE_RATE_WINDOW_MS", 10_000); +const WS_RATE_MAX = positiveIntEnv("OVERLORD_WS_UPGRADE_RATE_MAX", 30); const wsRateMap = new Map(); setInterval(() => { @@ -238,7 +246,6 @@ export async function handleWsUpgradeRoutes( const wsMatch = url.pathname.match(/^\/api\/clients\/(.+)\/stream\/ws$/); if (wsMatch) { - logger.info(`[auth] Checking agent authorization for client connection`); if (!deps.isAuthorizedAgentRequest(req, url)) { return new Response("Unauthorized", { status: 401 }); } diff --git a/Overlord-Server/src/server/runtime-constants.ts b/Overlord-Server/src/server/runtime-constants.ts index b36c5e4..c326220 100644 --- a/Overlord-Server/src/server/runtime-constants.ts +++ b/Overlord-Server/src/server/runtime-constants.ts @@ -1,6 +1,14 @@ -export const HEARTBEAT_INTERVAL_MS = 15_000; -export const STALE_MS = 90_000; -export const DISCONNECT_TIMEOUT_MS = 10_000; +function positiveIntEnv(name: string, fallback: number): number { + const raw = String(process.env[name] || "").trim(); + if (!raw) return fallback; + const parsed = Number(raw); + if (!Number.isFinite(parsed) || parsed <= 0) return fallback; + return Math.floor(parsed); +} + +export const HEARTBEAT_INTERVAL_MS = positiveIntEnv("OVERLORD_HEARTBEAT_INTERVAL_MS", 15_000); +export const STALE_MS = positiveIntEnv("OVERLORD_STALE_MS", 90_000); +export const DISCONNECT_TIMEOUT_MS = positiveIntEnv("OVERLORD_DISCONNECT_TIMEOUT_MS", 10_000); export const PRUNE_BATCH = Number(process.env.PRUNE_BATCH || 500); export const MAX_WS_MESSAGE_BYTES_VIEWER = Number( diff --git a/Overlord-Server/src/sessions/sessionManager.ts b/Overlord-Server/src/sessions/sessionManager.ts index 8259c67..8c4ff3e 100644 --- a/Overlord-Server/src/sessions/sessionManager.ts +++ b/Overlord-Server/src/sessions/sessionManager.ts @@ -488,12 +488,37 @@ export function getDashboardSessionCount(): number { } let dashboardBroadcastTimer: ReturnType | null = null; -const DASHBOARD_DEBOUNCE_MS = 150; +let dashboardClientEventTimer: ReturnType | null = null; +let dashboardClientEventCount = 0; +const DASHBOARD_DEBOUNCE_MS = Math.max( + 150, + Number(process.env.OVERLORD_DASHBOARD_DEBOUNCE_MS || 1000), +); +const DASHBOARD_CLIENT_EVENT_FLUSH_MS = Math.max( + 250, + Number(process.env.OVERLORD_DASHBOARD_EVENT_FLUSH_MS || 1000), +); +const DASHBOARD_CLIENT_EVENT_INLINE_LIMIT = Math.max( + 0, + Number(process.env.OVERLORD_DASHBOARD_EVENT_INLINE_LIMIT || 20), +); export function notifyDashboardClientEvent( event: "client_online" | "client_offline" | "client_purgatory", info: { id: string; host?: string; user?: string; os?: string; ip?: string; country?: string }, ): void { + dashboardClientEventCount += 1; + if (dashboardClientEventCount > DASHBOARD_CLIENT_EVENT_INLINE_LIMIT) { + if (!dashboardClientEventTimer) { + dashboardClientEventTimer = setTimeout(() => { + dashboardClientEventTimer = null; + dashboardClientEventCount = 0; + notifyDashboardViewers(); + }, DASHBOARD_CLIENT_EVENT_FLUSH_MS); + } + return; + } + const msg = JSON.stringify({ type: "client_event", event, @@ -512,6 +537,13 @@ export function notifyDashboardClientEvent( dashboardSessions.delete(id); } } + + if (!dashboardClientEventTimer) { + dashboardClientEventTimer = setTimeout(() => { + dashboardClientEventTimer = null; + dashboardClientEventCount = 0; + }, DASHBOARD_CLIENT_EVENT_FLUSH_MS); + } } export function notifyDashboardViewers(): void { diff --git a/Overlord-Server/src/wsHandlers.test.ts b/Overlord-Server/src/wsHandlers.test.ts index 03de8b0..3838f60 100644 --- a/Overlord-Server/src/wsHandlers.test.ts +++ b/Overlord-Server/src/wsHandlers.test.ts @@ -1,7 +1,7 @@ import { describe, expect, test } from "bun:test"; import { decodeMessage } from "./protocol"; import { metrics } from "./metrics"; -import { handlePong, sendPingRequest } from "./wsHandlers"; +import { handlePing, handlePong, sendPingRequest } from "./wsHandlers"; type MockWs = { sent: Uint8Array[]; send: (msg: Uint8Array) => void }; @@ -30,6 +30,29 @@ describe("wsHandlers ping/pong", () => { expect(ws.sent.length).toBe(1); }); + test("handlePing responds with pong without starting another server ping", () => { + const ws: MockWs = { + sent: [], + send(msg) { + this.sent.push(msg); + }, + }; + const info = { + id: "client-ping", + role: "client", + ws, + lastSeen: Date.now(), + } as any; + + handlePing(info, { type: "ping", ts: 9876 } as any, ws); + + expect(ws.sent.length).toBe(1); + const payload = decodeMessage(ws.sent[0]) as any; + expect(payload.type).toBe("pong"); + expect(payload.ts).toBe(9876); + expect(info.lastPingNonce).toBeUndefined(); + }); + test("handlePong clears nonce and records ping", () => { metrics.reset(); const ws: MockWs = { diff --git a/Overlord-Server/src/wsHandlers.ts b/Overlord-Server/src/wsHandlers.ts index 18d7ed8..a038f3f 100644 --- a/Overlord-Server/src/wsHandlers.ts +++ b/Overlord-Server/src/wsHandlers.ts @@ -15,7 +15,6 @@ import { notifyThumbnailGenerated, requestThumbnailRegen, } from "./thumbnails"; -import { upsertClientRow } from "./db"; import { metrics } from "./metrics"; import { markClientDbSynced, queueClientDbUpdate, shouldSyncClientToDb } from "./client-db-sync"; @@ -124,7 +123,7 @@ export async function handleHello( info.lastSeen = Date.now(); info.online = true; - upsertClientRow({ + queueClientDbUpdate({ id: info.id, hwid: info.hwid, role: info.role, @@ -166,7 +165,6 @@ export function handlePing(info: ClientInfo, payload: Ping, ws: any) { }); } ws.send(encodeMessage({ type: "pong", ts: payload.ts || Date.now() })); - sendPingRequest(info, ws, "client_ping"); } export function sendPingRequest(info: ClientInfo, ws: any, reason: string) {