improve crashlogs

Overlord-Client/cmd/agent/crashlog.go

@@ -2,7 +2,9 @@ package main
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
+	"io"
 	"log"
 	"os"
 	"path/filepath"
@@ -12,6 +14,14 @@ import (
 )
 
 const crashLogFileName = "crashlogC.log"
+const pendingCrashFileName = "last-crash.json"
+
+type pendingCrashReport struct {
+	At     string `json:"at"`
+	Reason string `json:"reason"`
+	Detail string `json:"detail,omitempty"`
+	Log    string `json:"log,omitempty"`
+}
 
 func handleFatalPanic() {
 	//garble:controlflow block_splits=10 junk_jumps=10 flatten_passes=2
@@ -19,6 +29,7 @@ func handleFatalPanic() {
 		reason := fmt.Sprintf("panic: %v", r)
 		stack := debug.Stack()
 		path := writeCrashLog(reason, stack)
+		writePendingCrashReport("panic", reason, path)
 		log.Printf("[fatal] %s (see %s)", reason, path)
 		os.Exit(1)
 	}
@@ -33,6 +44,7 @@ func fatalExit(reason string, err error) {
 	}
 	stack := debug.Stack()
 	path := writeCrashLog(reason, stack)
+	writePendingCrashReport("fatal", reason, path)
 	log.Printf("[fatal] %s (see %s)", reason, path)
 	os.Exit(1)
 }
@@ -49,6 +61,7 @@ func recoverAndLog(label string, cancel context.CancelFunc) {
 		reason := fmt.Sprintf("%s panic: %v", label, r)
 		stack := debug.Stack()
 		path := writeCrashLog(reason, stack)
+		writePendingCrashReport("panic", reason, path)
 		log.Printf("[panic] %s (see %s)", reason, path)
 		if cancel != nil {
 			cancel()
@@ -82,3 +95,45 @@ func writeCrashLog(reason string, stack []byte) string {
 func crashLogDir() string {
 	return os.TempDir()
 }
+
+func pendingCrashPath() string {
+	return filepath.Join(crashLogDir(), pendingCrashFileName)
+}
+
+func writePendingCrashReport(reason, detail, logPath string) {
+	report := pendingCrashReport{
+		At:     time.Now().UTC().Format(time.RFC3339),
+		Reason: reason,
+		Detail: detail,
+		Log:    logPath,
+	}
+	_ = os.MkdirAll(crashLogDir(), 0700)
+	data, err := json.Marshal(report)
+	if err != nil {
+		return
+	}
+	_ = os.WriteFile(pendingCrashPath(), data, 0600)
+}
+
+func loadPendingCrashReport() (pendingCrashReport, bool) {
+	data, err := os.ReadFile(pendingCrashPath())
+	if err != nil {
+		return pendingCrashReport{}, false
+	}
+	var report pendingCrashReport
+	if json.Unmarshal(data, &report) != nil || report.Reason == "" {
+		return pendingCrashReport{}, false
+	}
+	if report.Detail == "" && report.Log != "" {
+		if f, err := os.Open(report.Log); err == nil {
+			defer f.Close()
+			buf, _ := io.ReadAll(io.LimitReader(f, 4096))
+			report.Detail = string(buf)
+		}
+	}
+	return report, true
+}
+
+func clearPendingCrashReport() {
+	_ = os.Remove(pendingCrashPath())
+}

Overlord-Client/cmd/agent/hardcrash_other.go

@@ -0,0 +1,5 @@
+//go:build !windows
+
+package main
+
+func installHardCrashReporter() {}

Overlord-Client/cmd/agent/hardcrash_windows.go

@@ -0,0 +1,46 @@
+//go:build windows
+
+package main
+
+import (
+	"fmt"
+	"syscall"
+	"unsafe"
+
+	"golang.org/x/sys/windows"
+)
+
+type exceptionRecord struct {
+	ExceptionCode        uint32
+	ExceptionFlags       uint32
+	ExceptionRecord      uintptr
+	ExceptionAddress     uintptr
+	NumberParameters     uint32
+	ExceptionInformation [15]uintptr
+}
+
+type exceptionPointers struct {
+	ExceptionRecord uintptr
+	ContextRecord   uintptr
+}
+
+var hardCrashCallback uintptr
+
+var procSetUnhandledExceptionFilter = windows.NewLazySystemDLL("kernel32.dll").NewProc("SetUnhandledExceptionFilter")
+
+func installHardCrashReporter() {
+	hardCrashCallback = syscall.NewCallback(func(info uintptr) uintptr {
+		detail := "unhandled native exception"
+		if info != 0 {
+			ptrs := (*exceptionPointers)(unsafe.Pointer(info))
+			if ptrs.ExceptionRecord != 0 {
+				rec := (*exceptionRecord)(unsafe.Pointer(ptrs.ExceptionRecord))
+				detail = fmt.Sprintf("exception=0x%08x address=0x%x flags=0x%x", rec.ExceptionCode, rec.ExceptionAddress, rec.ExceptionFlags)
+			}
+		}
+		path := writeCrashLog("hard crash: "+detail, nil)
+		writePendingCrashReport("crash", detail, path)
+		return 0
+	})
+	procSetUnhandledExceptionFilter.Call(hardCrashCallback)
+}

Overlord-Client/cmd/agent/main.go

@@ -12,6 +12,7 @@ import (
 
 func main() {
 	//garble:controlflow block_splits=10 junk_jumps=10 flatten_passes=2
+	installHardCrashReporter()
 	cfg := config.Load()
 
 	if cfg.SleepSeconds > 0 {

Overlord-Client/cmd/agent/session.go

@@ -667,6 +667,10 @@ func runSession(ctx context.Context, cancel context.CancelFunc, conn *websocket.
 	if cfg.FetchPublicIP {
 		hello.PublicIP = config.FetchPublicIPAddress()
 	}
+	if report, ok := loadPendingCrashReport(); ok {
+		hello.LastCrashReason = report.Reason
+		hello.LastCrashDetail = truncateStr(fmt.Sprintf("%s at %s", report.Detail, report.At), 1200)
+	}
 
 	if err := wire.WriteMsg(ctx, env.Conn, hello); err != nil {
 		return fmt.Errorf("send hello: %w", err)
@@ -712,6 +716,9 @@ func runSession(ctx context.Context, cancel context.CancelFunc, conn *websocket.
 		cancel()
 		return fmt.Errorf("send initial ping: %w", err)
 	}
+	if hello.LastCrashReason != "" {
+		clearPendingCrashReport()
+	}
 
 	if interval := getPingInterval(); interval > 0 {
 		log.Printf("ping: heartbeat interval=%s", interval)

Overlord-Client/cmd/agent/wire/protocol.go

@@ -26,6 +26,8 @@ type Hello struct {
 	Elevation       string          `msgpack:"elevation,omitempty"`
 	Permissions     map[string]bool `msgpack:"permissions,omitempty"`
 	PublicIP        string          `msgpack:"publicIP,omitempty"`
+	LastCrashReason string          `msgpack:"lastCrashReason,omitempty"`
+	LastCrashDetail string          `msgpack:"lastCrashDetail,omitempty"`
 }
 
 type EnrollmentChallenge struct {

Overlord-Server/src/db/repositories.ts

@@ -343,6 +343,16 @@ export function setOnlineState(id: string, online: boolean, disconnectReason?: s
   invalidateClientMetricsSummaryCache();
 }
 
+export function setClientDisconnectInfo(id: string, disconnectReason: string, disconnectDetail?: string) {
+  db.run(
+    `UPDATE clients SET disconnect_reason=?, disconnect_detail=? WHERE id=?`,
+    disconnectReason || null,
+    disconnectDetail || null,
+    id,
+  );
+  invalidateClientMetricsSummaryCache();
+}
+
 const setOfflineStateStmt = db.prepare(
   `UPDATE clients SET online=0, last_seen=?, disconnect_reason=?, disconnect_detail=? WHERE id=?`,
 );

Overlord-Server/src/protocol.ts

@@ -43,6 +43,8 @@ export type Hello = {
   publicKey?: string;
   signature?: string;
   publicIP?: string;
+  lastCrashReason?: string;
+  lastCrashDetail?: string;
 };
 
 export type EnrollmentChallenge = {

Overlord-Server/src/wsHandlers.ts

@@ -16,7 +16,8 @@ import {
   requestThumbnailRegen,
 } from "./thumbnails";
 import { metrics } from "./metrics";
-import { markClientDbSynced, queueClientDbUpdate, shouldSyncClientToDb } from "./client-db-sync";
+import { flushQueuedClientDbUpdates, markClientDbSynced, queueClientDbUpdate, shouldSyncClientToDb } from "./client-db-sync";
+import { setClientDisconnectInfo } from "./db";
 
 export { clearClientSyncState } from "./client-db-sync";
 
@@ -122,6 +123,8 @@ export async function handleHello(
   info.country = country;
   info.lastSeen = Date.now();
   info.online = true;
+  const lastCrashReason = sanitizeInfoString((payload as any).lastCrashReason, 64);
+  const lastCrashDetail = sanitizeInfoString((payload as any).lastCrashDetail, 1200);
 
   queueClientDbUpdate({
     id: info.id,
@@ -146,6 +149,10 @@ export async function handleHello(
     lastSeen: info.lastSeen,
     online: 1,
   });
+  if (lastCrashReason) {
+    flushQueuedClientDbUpdates();
+    setClientDisconnectInfo(info.id, lastCrashReason, lastCrashDetail);
+  }
 
   sendPingRequest(info, ws, "hello");