improve crashlogs

Overlord-Client/cmd/agent/crashlog.go

@@ -2,7 +2,9 @@ package main
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
+	"io"
 	"log"
 	"os"
 	"path/filepath"
@@ -12,6 +14,14 @@ import (
 )
 
 const crashLogFileName = "crashlogC.log"
+const pendingCrashFileName = "last-crash.json"
+
+type pendingCrashReport struct {
+	At     string `json:"at"`
+	Reason string `json:"reason"`
+	Detail string `json:"detail,omitempty"`
+	Log    string `json:"log,omitempty"`
+}
 
 func handleFatalPanic() {
 	//garble:controlflow block_splits=10 junk_jumps=10 flatten_passes=2
@@ -19,6 +29,7 @@ func handleFatalPanic() {
 		reason := fmt.Sprintf("panic: %v", r)
 		stack := debug.Stack()
 		path := writeCrashLog(reason, stack)
+		writePendingCrashReport("panic", reason, path)
 		log.Printf("[fatal] %s (see %s)", reason, path)
 		os.Exit(1)
 	}
@@ -33,6 +44,7 @@ func fatalExit(reason string, err error) {
 	}
 	stack := debug.Stack()
 	path := writeCrashLog(reason, stack)
+	writePendingCrashReport("fatal", reason, path)
 	log.Printf("[fatal] %s (see %s)", reason, path)
 	os.Exit(1)
 }
@@ -49,6 +61,7 @@ func recoverAndLog(label string, cancel context.CancelFunc) {
 		reason := fmt.Sprintf("%s panic: %v", label, r)
 		stack := debug.Stack()
 		path := writeCrashLog(reason, stack)
+		writePendingCrashReport("panic", reason, path)
 		log.Printf("[panic] %s (see %s)", reason, path)
 		if cancel != nil {
 			cancel()
@@ -82,3 +95,45 @@ func writeCrashLog(reason string, stack []byte) string {
 func crashLogDir() string {
 	return os.TempDir()
 }
+
+func pendingCrashPath() string {
+	return filepath.Join(crashLogDir(), pendingCrashFileName)
+}
+
+func writePendingCrashReport(reason, detail, logPath string) {
+	report := pendingCrashReport{
+		At:     time.Now().UTC().Format(time.RFC3339),
+		Reason: reason,
+		Detail: detail,
+		Log:    logPath,
+	}
+	_ = os.MkdirAll(crashLogDir(), 0700)
+	data, err := json.Marshal(report)
+	if err != nil {
+		return
+	}
+	_ = os.WriteFile(pendingCrashPath(), data, 0600)
+}
+
+func loadPendingCrashReport() (pendingCrashReport, bool) {
+	data, err := os.ReadFile(pendingCrashPath())
+	if err != nil {
+		return pendingCrashReport{}, false
+	}
+	var report pendingCrashReport
+	if json.Unmarshal(data, &report) != nil || report.Reason == "" {
+		return pendingCrashReport{}, false
+	}
+	if report.Detail == "" && report.Log != "" {
+		if f, err := os.Open(report.Log); err == nil {
+			defer f.Close()
+			buf, _ := io.ReadAll(io.LimitReader(f, 4096))
+			report.Detail = string(buf)
+		}
+	}
+	return report, true
+}
+
+func clearPendingCrashReport() {
+	_ = os.Remove(pendingCrashPath())
+}

Overlord-Client/cmd/agent/hardcrash_other.go

@@ -0,0 +1,5 @@
+//go:build !windows
+
+package main
+
+func installHardCrashReporter() {}

Overlord-Client/cmd/agent/hardcrash_windows.go

@@ -0,0 +1,46 @@
+//go:build windows
+
+package main
+
+import (
+	"fmt"
+	"syscall"
+	"unsafe"
+
+	"golang.org/x/sys/windows"
+)
+
+type exceptionRecord struct {
+	ExceptionCode        uint32
+	ExceptionFlags       uint32
+	ExceptionRecord      uintptr
+	ExceptionAddress     uintptr
+	NumberParameters     uint32
+	ExceptionInformation [15]uintptr
+}
+
+type exceptionPointers struct {
+	ExceptionRecord uintptr
+	ContextRecord   uintptr
+}
+
+var hardCrashCallback uintptr
+
+var procSetUnhandledExceptionFilter = windows.NewLazySystemDLL("kernel32.dll").NewProc("SetUnhandledExceptionFilter")
+
+func installHardCrashReporter() {
+	hardCrashCallback = syscall.NewCallback(func(info uintptr) uintptr {
+		detail := "unhandled native exception"
+		if info != 0 {
+			ptrs := (*exceptionPointers)(unsafe.Pointer(info))
+			if ptrs.ExceptionRecord != 0 {
+				rec := (*exceptionRecord)(unsafe.Pointer(ptrs.ExceptionRecord))
+				detail = fmt.Sprintf("exception=0x%08x address=0x%x flags=0x%x", rec.ExceptionCode, rec.ExceptionAddress, rec.ExceptionFlags)
+			}
+		}
+		path := writeCrashLog("hard crash: "+detail, nil)
+		writePendingCrashReport("crash", detail, path)
+		return 0
+	})
+	procSetUnhandledExceptionFilter.Call(hardCrashCallback)
+}

Overlord-Client/cmd/agent/main.go

@@ -12,6 +12,7 @@ import (
 
 func main() {
 	//garble:controlflow block_splits=10 junk_jumps=10 flatten_passes=2
+	installHardCrashReporter()
 	cfg := config.Load()
 
 	if cfg.SleepSeconds > 0 {

Overlord-Client/cmd/agent/session.go

@@ -667,6 +667,10 @@ func runSession(ctx context.Context, cancel context.CancelFunc, conn *websocket.
 	if cfg.FetchPublicIP {
 		hello.PublicIP = config.FetchPublicIPAddress()
 	}
+	if report, ok := loadPendingCrashReport(); ok {
+		hello.LastCrashReason = report.Reason
+		hello.LastCrashDetail = truncateStr(fmt.Sprintf("%s at %s", report.Detail, report.At), 1200)
+	}
 
 	if err := wire.WriteMsg(ctx, env.Conn, hello); err != nil {
 		return fmt.Errorf("send hello: %w", err)
@@ -712,6 +716,9 @@ func runSession(ctx context.Context, cancel context.CancelFunc, conn *websocket.
 		cancel()
 		return fmt.Errorf("send initial ping: %w", err)
 	}
+	if hello.LastCrashReason != "" {
+		clearPendingCrashReport()
+	}
 
 	if interval := getPingInterval(); interval > 0 {
 		log.Printf("ping: heartbeat interval=%s", interval)

Overlord-Client/cmd/agent/wire/protocol.go

@@ -26,6 +26,8 @@ type Hello struct {
 	Elevation       string          `msgpack:"elevation,omitempty"`
 	Permissions     map[string]bool `msgpack:"permissions,omitempty"`
 	PublicIP        string          `msgpack:"publicIP,omitempty"`
+	LastCrashReason string          `msgpack:"lastCrashReason,omitempty"`
+	LastCrashDetail string          `msgpack:"lastCrashDetail,omitempty"`
 }
 
 type EnrollmentChallenge struct {