stability fixes

Overlord-Server/src/server/maintenance-loops.ts

@@ -24,16 +24,20 @@ export function startMaintenanceLoops(params: StartMaintenanceParams): void {
     });
   }, 5000);
 
+  const livenessTimeoutMs = Math.max(
+    params.heartbeatIntervalMs * 4 + params.disconnectTimeoutMs,
+    60_000,
+  );
+
   setInterval(() => {
     const now = Date.now();
     for (const [id, info] of params.getClients().entries()) {
       if (info.role !== "client") continue;
-      if (
-        info.lastPingNonce !== undefined &&
-        info.lastPingSent &&
-        now - info.lastPingSent > params.heartbeatIntervalMs + params.disconnectTimeoutMs
-      ) {
-        logger.warn(`[ping] no pong from ${id} within timeout; closing socket`);
+      const lastActivity = info.lastSeen || 0;
+      if (lastActivity && now - lastActivity > livenessTimeoutMs) {
+        logger.warn(
+          `[ping] no activity from ${id} for ${now - lastActivity}ms; closing socket`,
+        );
         try {
           info.ws.close(4001, "ping timeout");
         } catch (err) {

Overlord-Server/src/server/routes/websocket-lifecycle-routes.ts

@@ -20,6 +20,67 @@ import { clearClientSyncState, handleFrame, handleHello, handlePing, handlePong
 import { getMaxPayloadLimit, getMessageByteLength, isAllowedClientMessageType } from "../../wsValidation";
 import { stopAllProxiesForClient } from "../socks5-proxy-manager";
 
+const OFFLINE_GRACE_MS = (() => {
+  const raw = process.env.OVERLORD_OFFLINE_GRACE_MS;
+  if (raw === undefined || raw === "") return 7_000;
+  const n = Number(raw);
+  if (!Number.isFinite(n) || n < 0) return 7_000;
+  return Math.floor(n);
+})();
+
+type ClientLifecyclePayload = {
+  id: string;
+  host?: string;
+  user?: string;
+  os?: string;
+  ip?: string;
+  country?: string;
+};
+
+type PendingOffline = {
+  timer: ReturnType<typeof setTimeout>;
+};
+
+const pendingOffline = new Map<string, PendingOffline>();
+
+function cancelPendingOffline(clientId: string): boolean {
+  const pending = pendingOffline.get(clientId);
+  if (!pending) return false;
+  clearTimeout(pending.timer);
+  pendingOffline.delete(clientId);
+  return true;
+}
+
+function schedulePendingOffline(
+  clientId: string,
+  payload: ClientLifecyclePayload,
+  disconnectReason: string | undefined,
+  disconnectDetail: string | undefined,
+  deps: WsLifecycleDeps,
+): void {
+  cancelPendingOffline(clientId);
+
+  if (OFFLINE_GRACE_MS <= 0) {
+    deps.notifyDashboardClientEvent("client_offline", payload);
+    deps.broadcastClientEvent("client_offline", payload);
+    setOnlineState(clientId, false, disconnectReason, disconnectDetail);
+    deps.notifyRemoteDesktopStatus(clientId, "offline", "Client disconnected");
+    deps.notifyDashboard();
+    return;
+  }
+
+  const timer = setTimeout(() => {
+    pendingOffline.delete(clientId);
+    deps.notifyDashboardClientEvent("client_offline", payload);
+    deps.broadcastClientEvent("client_offline", payload);
+    setOnlineState(clientId, false, disconnectReason, disconnectDetail);
+    deps.notifyRemoteDesktopStatus(clientId, "offline", "Client disconnected");
+    deps.notifyDashboard();
+  }, OFFLINE_GRACE_MS);
+
+  pendingOffline.set(clientId, { timer });
+}
+
 type PendingScript = {
   timeout: ReturnType<typeof setTimeout>;
   resolve: (value: { ok?: boolean; result?: string; error?: string }) => void;
@@ -457,26 +518,30 @@ export async function handleWebSocketMessage(
         await handleHello(infoObj, payload, ws, ip);
         clientManager.addClient(infoObj.id, infoObj);
 
+        const reconnectedWithinGrace = cancelPendingOffline(infoObj.id);
+
         deps.dispatchAutoScriptsForConnection(infoObj, ws);
         deps.dispatchAutoDeploysForConnection(infoObj, ws);
         deps.dispatchAutoLoadPlugins(infoObj);
         deps.notifyDashboard();
-        deps.notifyDashboardClientEvent("client_online", {
-            id: infoObj.id,
-            host: infoObj.host,
-            user: infoObj.user,
-            os: infoObj.os,
-            ip: infoObj.ip,
-            country: infoObj.country,
-          });
-        deps.broadcastClientEvent("client_online", {
-            id: infoObj.id,
-            host: infoObj.host,
-            user: infoObj.user,
-            os: infoObj.os,
-            ip: infoObj.ip,
-            country: infoObj.country,
-          });
+        if (!reconnectedWithinGrace) {
+          deps.notifyDashboardClientEvent("client_online", {
+              id: infoObj.id,
+              host: infoObj.host,
+              user: infoObj.user,
+              os: infoObj.os,
+              ip: infoObj.ip,
+              country: infoObj.country,
+            });
+          deps.broadcastClientEvent("client_online", {
+              id: infoObj.id,
+              host: infoObj.host,
+              user: infoObj.user,
+              os: infoObj.os,
+              ip: infoObj.ip,
+              country: infoObj.country,
+            });
+        }
         if (infoObj.role === "client") {
           deps.notifyRemoteDesktopStatus(resolvedId, "online");
           metrics.recordConnection();
@@ -802,25 +867,6 @@ export function handleWebSocketClose(
     return;
   }
 
-  if (role === "client" && currentClient) {
-    deps.notifyDashboardClientEvent("client_offline", {
-      id: clientId,
-      host: currentClient.host,
-      user: currentClient.user,
-      os: currentClient.os,
-      ip: currentClient.ip,
-      country: currentClient.country,
-    });
-    deps.broadcastClientEvent("client_offline", {
-      id: clientId,
-      host: currentClient.host,
-      user: currentClient.user,
-      os: currentClient.os,
-      ip: currentClient.ip,
-      country: currentClient.country,
-    });
-  }
-
   const storedDisconnectReason = ws.data.disconnectReason;
   const storedDisconnectDetail = ws.data.disconnectDetail;
 
@@ -828,7 +874,6 @@ export function handleWebSocketClose(
   stopAllProxiesForClient(clientId);
   clearClientSyncState(clientId);
   deps.notifyConsoleClosed(clientId, "Client disconnected");
-  setOnlineState(clientId, false, storedDisconnectReason, storedDisconnectDetail);
   deps.clearPendingNotificationScreenshots(clientId);
   deps.clearClientPluginState(clientId);
   for (const [cmdId, pending] of deps.pendingScripts) {
@@ -848,11 +893,26 @@ export function handleWebSocketClose(
   deps.rdStreamingState.delete(clientId);
   deps.hvncStreamingState.delete(clientId);
   deps.webcamStreamingState.delete(clientId);
-  deps.notifyDashboard();
   logger.info(`[close] ${clientId} code=${code} reason=${reason} disconnect_reason=${storedDisconnectReason || "unknown"}`);
 
+  if (role === "client" && currentClient) {
+    schedulePendingOffline(
+      clientId,
+      {
+        id: clientId,
+        host: currentClient.host,
+        user: currentClient.user,
+        os: currentClient.os,
+        ip: currentClient.ip,
+        country: currentClient.country,
+      },
+      storedDisconnectReason,
+      storedDisconnectDetail,
+      deps,
+    );
+  }
+
   if (role === "client") {
-    deps.notifyRemoteDesktopStatus(clientId, "offline", "Client disconnected");
     metrics.recordDisconnection();
     logAudit({
       timestamp: Date.now(),

Overlord-Server/src/server/runtime-constants.ts

@@ -1,5 +1,5 @@
 export const HEARTBEAT_INTERVAL_MS = 15_000;
-export const STALE_MS = 5 * 60_000;
+export const STALE_MS = 90_000;
 export const DISCONNECT_TIMEOUT_MS = 10_000;
 
 export const PRUNE_BATCH = Number(process.env.PRUNE_BATCH || 500);

Overlord-Server/src/types.ts

@@ -9,6 +9,7 @@ export type ClientInfo = {
   ws: any;
   lastPingSent?: number;
   lastPingNonce?: number;
+  lastPongAt?: number;
   online?: boolean;
   hwid?: string;
   ip?: string;

Overlord-Server/src/wsHandlers.ts

@@ -207,11 +207,7 @@ export function handlePing(info: ClientInfo, payload: WireMessage, ws: any) {
 
 export function sendPingRequest(info: ClientInfo, ws: any, reason: string) {
   const now = Date.now();
-  if (
-    info.lastPingNonce !== undefined &&
-    info.lastPingSent &&
-    now-info.lastPingSent < MAX_PING_RTT_MS
-  ) {
+  if (info.lastPingSent && now - info.lastPingSent < 1_000) {
     return;
   }
   const nonce = now + Math.floor(Math.random() * 1000);
@@ -245,6 +241,7 @@ export function handlePong(info: ClientInfo, payload: WireMessage) {
   const nowTs = Date.now();
 
   info.lastSeen = nowTs;
+  info.lastPongAt = nowTs;
   info.online = true;
   info.lastPingNonce = undefined;