pinning + new version of backups

This commit is contained in:
KingKDot
2026-07-30 14:03:16 -05:00
parent 477b9a5353
commit 2669ab1388
32 changed files with 1527 additions and 133 deletions
+6
View File
@@ -1,9 +1,15 @@
declare module "adm-zip" {
class AdmZip {
constructor(input?: string | Buffer);
addFile(entryName: string, content: Buffer, comment?: string, attr?: number): void;
toBuffer(): Buffer;
getEntries(): Array<{
isDirectory: boolean;
entryName: string;
header: {
size: number;
compressedSize: number;
};
getData(): Buffer;
}>;
}
+7 -1
View File
@@ -1 +1,7 @@
import "./main-server";
import { applyPendingServerRestore } from "./server/server-backup";
if (applyPendingServerRestore()) {
console.log("[restore] Portable server backup applied successfully");
}
await import("./main-server");
+4
View File
@@ -561,7 +561,11 @@ async function startServer() {
getRdSessionCount: sessionManager.getRdSessionCount,
getFileBrowserSessionCount: sessionManager.getFileBrowserSessionCount,
getProcessSessionCount: sessionManager.getProcessSessionCount,
DATA_DIR,
PLUGIN_ROOT,
tlsCertPath: tls?.certPathUsed,
tlsKeyPath: TLS_KEY_PATH,
tlsCaPath: TLS_CA_PATH,
tlsSource: tls?.source,
},
assets: {
@@ -0,0 +1,17 @@
import { describe, expect, test } from "bun:test";
import { createAgentTlsPinsLdflag } from "./build-process";
describe("agent build TLS pin embedding", () => {
test("creates the linker flag consumed by the Go agent config", () => {
const currentPin = Buffer.alloc(32, 0x11).toString("base64");
const nextPin = Buffer.alloc(32, 0x22).toString("base64");
expect(createAgentTlsPinsLdflag([currentPin, nextPin])).toBe(
`-X overlord-client/cmd/agent/config.DefaultTLSSPKIPins=${currentPin},${nextPin}`,
);
});
test("does not create a linker flag without pins", () => {
expect(createAgentTlsPinsLdflag([])).toBe("");
});
});
@@ -24,6 +24,12 @@ import { runSgn } from "./sgn-manager";
import { resolveContainedPath } from "./upload-security";
import { createIsolatedBuildEnv } from "./build-environment";
import { cleanupMacosSdkUpload, extractAndValidateMacosSdk } from "./macos-sdk-manager";
import { getActiveTlsSpkiPins } from "./tls-bootstrap";
export function createAgentTlsPinsLdflag(pins: string[]): string {
if (pins.length === 0) return "";
return `-X overlord-client/cmd/agent/config.DefaultTLSSPKIPins=${pins.join(",")}`;
}
function isClientModuleDir(dir: string): boolean {
return (
@@ -628,6 +634,16 @@ export async function startBuildProcess(
const serverConfig = getConfig();
const buildAgentToken = (serverConfig.auth.agentToken || "").trim();
const buildTlsSpkiPins = getActiveTlsSpkiPins();
if (
String(process.env.OVERLORD_TLS_OFFLOAD || "").toLowerCase() === "true" &&
buildTlsSpkiPins.length === 0
) {
throw new Error(
"TLS offload requires OVERLORD_TLS_SPKI_PINS before building agents; " +
"pin the external HTTPS/WSS certificate public key",
);
}
sendToStream({ type: "status", text: "Preparing build environment..." });
@@ -1205,6 +1221,22 @@ func runBoundFiles() {
sendToStream({ type: "output", text: `Server URL: ${config.serverUrl}\n`, level: "info" });
}
if (buildTlsSpkiPins.length > 0) {
const tlsPinsFlag = createAgentTlsPinsLdflag(buildTlsSpkiPins);
ldflags = `${ldflags} ${tlsPinsFlag}`;
sendToStream({
type: "output",
text: `TLS identity pinning: enabled (${buildTlsSpkiPins.length} trusted key${buildTlsSpkiPins.length === 1 ? "" : "s"})\n`,
level: "info",
});
} else {
sendToStream({
type: "output",
text: "WARNING: No TLS SPKI pin is available. Configure OVERLORD_TLS_SPKI_PINS when TLS is terminated externally.\n",
level: "warn",
});
}
if (config.rawServerList) {
const rawServerFlag = "-X overlord-client/cmd/agent/config.DefaultServerURLIsRaw=true";
ldflags = ldflags ? `${ldflags} ${rawServerFlag}` : rawServerFlag;
@@ -1,7 +1,7 @@
import { authenticateRequest } from "../../auth";
import { AuditAction, getAuditLogs, logAudit } from "../../auditLog";
import { logger } from "../../logger";
import { getConfig, updateSecurityConfig, updateTlsConfig, updateOidcConfig, updateAppearanceConfig, updateChatConfig, getExportableConfig, importFullConfig, updateRegistrationConfig, updateBuildRateLimitConfig, updateThumbnailsConfig, updateInputArchiveConfig, updateFileTransfersConfig } from "../../config";
import { getConfig, updateSecurityConfig, updateTlsConfig, updateOidcConfig, updateAppearanceConfig, updateChatConfig, importFullConfig, updateRegistrationConfig, updateBuildRateLimitConfig, updateThumbnailsConfig, updateInputArchiveConfig, updateFileTransfersConfig } from "../../config";
import {
saveBrandingImage,
getClientMetricsSummary,
@@ -23,6 +23,13 @@ import {
startProxy,
stopProxy,
} from "../socks5-proxy-manager";
import { db } from "../../db/connection";
import {
createDatabaseSnapshot,
createPortableServerBackup,
resolveServerBackupPaths,
stagePortableServerRestore,
} from "../server-backup";
let activeServerProfile: Promise<any> | null = null;
let jscRuntimePromise: Promise<any> | null | undefined;
@@ -38,7 +45,11 @@ type MiscRouteDeps = {
getFileBrowserSessionCount: () => number;
getProcessSessionCount: () => number;
tlsCertPath?: string;
tlsKeyPath?: string;
tlsCaPath?: string;
tlsSource?: "certbot" | "configured" | "self-signed";
DATA_DIR?: string;
PLUGIN_ROOT?: string;
};
const BRAND_IMAGE_MAX_BYTES = 5 * 1024 * 1024;
@@ -1070,7 +1081,21 @@ export async function handleMiscRoutes(
return new Response("Forbidden", { status: 403 });
}
const exportData = getExportableConfig(deps.SERVER_VERSION);
const backupPaths = resolveServerBackupPaths({
dataDir: deps.DATA_DIR,
pluginRoot: deps.PLUGIN_ROOT,
tlsCertPath: deps.tlsCertPath,
tlsKeyPath: deps.tlsKeyPath,
tlsCaPath: deps.tlsCaPath,
});
const databaseSnapshot = createDatabaseSnapshot(db);
const configSnapshot = Buffer.from(JSON.stringify(getConfig(), null, 2));
const exportData = createPortableServerBackup(
backupPaths,
deps.SERVER_VERSION,
databaseSnapshot,
configSnapshot,
);
logAudit({
timestamp: Date.now(),
@@ -1082,10 +1107,11 @@ export async function handleMiscRoutes(
});
const dateStr = new Date().toISOString().slice(0, 10);
return new Response(JSON.stringify(exportData, null, 2), {
return new Response(Uint8Array.from(exportData), {
headers: {
"Content-Type": "application/json",
"Content-Disposition": `attachment; filename="overlord-settings-${dateStr}.json"`,
"Content-Type": "application/zip",
"Content-Disposition": `attachment; filename="overlord-server-backup-${dateStr}.zip"`,
"Cache-Control": "no-store",
...deps.CORS_HEADERS,
},
});
@@ -1103,6 +1129,49 @@ export async function handleMiscRoutes(
return new Response("Forbidden", { status: 403 });
}
const contentType = String(req.headers.get("content-type") || "").toLowerCase();
if (contentType.includes("application/zip") || contentType.includes("application/octet-stream")) {
try {
const archive = Buffer.from(await req.arrayBuffer());
const result = stagePortableServerRestore(
archive,
resolveServerBackupPaths({
dataDir: deps.DATA_DIR,
pluginRoot: deps.PLUGIN_ROOT,
tlsCertPath: deps.tlsCertPath,
tlsKeyPath: deps.tlsKeyPath,
tlsCaPath: deps.tlsCaPath,
}),
deps.SERVER_VERSION,
);
logAudit({
timestamp: Date.now(),
username: user.username,
ip: deps.requestIP?.(req)?.address || "unknown",
action: AuditAction.COMMAND,
details: `Staged portable server restore (${result.files} files); restart required`,
success: true,
});
return Response.json({ ok: true, applied: ["portable server restore staged"], ...result }, {
headers: deps.CORS_HEADERS,
});
} catch (error: any) {
logAudit({
timestamp: Date.now(),
username: user.username,
ip: deps.requestIP?.(req)?.address || "unknown",
action: AuditAction.COMMAND,
details: "Portable server restore validation failed",
success: false,
errorMessage: String(error?.message || error),
});
return Response.json({ ok: false, error: String(error?.message || "Restore failed") }, {
status: 400,
headers: deps.CORS_HEADERS,
});
}
}
let body: any = {};
try {
body = await req.json();
@@ -0,0 +1,99 @@
import { afterAll, beforeAll, describe, expect, test } from "bun:test";
import AdmZip from "adm-zip";
import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "fs";
import { tmpdir } from "os";
import path from "path";
import { generateToken } from "../../auth";
import { generateSelfSignedCert } from "../../certGenerator";
import { createUser, deleteUser, getUserById } from "../../users";
import { handleMiscRoutes } from "./misc-routes";
const root = mkdtempSync(path.join(tmpdir(), "overlord-settings-backup-route-"));
const dataDir = path.join(root, "data");
const pluginRoot = path.join(root, "plugins");
const certPath = path.join(root, "certs", "server.crt");
const keyPath = path.join(root, "certs", "server.key");
let userId = 0;
let token = "";
const deps = {
CORS_HEADERS: {},
SERVER_VERSION: "3.0.0",
PUBLIC_ROOT: ".",
DATA_DIR: dataDir,
PLUGIN_ROOT: pluginRoot,
tlsCertPath: certPath,
tlsKeyPath: keyPath,
tlsSource: "self-signed" as const,
requestIP: () => ({ address: "127.0.0.1" }),
getConsoleSessionCount: () => 0,
getRdSessionCount: () => 0,
getFileBrowserSessionCount: () => 0,
getProcessSessionCount: () => 0,
};
beforeAll(async () => {
mkdirSync(dataDir, { recursive: true });
mkdirSync(pluginRoot, { recursive: true });
writeFileSync(path.join(dataDir, "save.json"), JSON.stringify({
auth: { jwtSecret: "route-jwt", agentToken: "route-agent" },
}));
await generateSelfSignedCert({ certPath, keyPath, commonName: "localhost" });
const created = await createUser(
`backup_route_${Date.now().toString(36)}`,
"Aa1!PortableBackupRoute_2026",
"admin",
"test",
);
if (!created.success || !created.userId) throw new Error("failed to create backup route test user");
userId = created.userId;
token = await generateToken(getUserById(userId)!);
});
afterAll(() => {
if (userId) deleteUser(userId);
rmSync(root, { recursive: true, force: true });
});
describe("portable settings backup routes", () => {
test("exports a ZIP and accepts it as a staged restore", async () => {
const exportUrl = new URL("https://localhost/api/settings/export");
const exportResponse = await handleMiscRoutes(
new Request(exportUrl, { headers: { Cookie: `overlord_token=${token}` } }),
exportUrl,
deps,
);
expect(exportResponse?.status).toBe(200);
expect(exportResponse?.headers.get("content-type")).toBe("application/zip");
expect(exportResponse?.headers.get("content-disposition")).toContain("overlord-server-backup-");
expect(exportResponse?.headers.get("cache-control")).toBe("no-store");
const archive = Buffer.from(await exportResponse!.arrayBuffer());
const zip = new AdmZip(archive);
const names = new Set(zip.getEntries().map((entry) => entry.entryName));
expect(names.has("manifest.json")).toBe(true);
expect(names.has("data/overlord.db")).toBe(true);
expect(names.has("data/save.json")).toBe(true);
expect(names.has("tls/server.crt")).toBe(true);
expect(names.has("tls/server.key")).toBe(true);
const importUrl = new URL("https://localhost/api/settings/import");
const importResponse = await handleMiscRoutes(
new Request(importUrl, {
method: "POST",
headers: {
Cookie: `overlord_token=${token}`,
"Content-Type": "application/zip",
},
body: Uint8Array.from(archive),
}),
importUrl,
deps,
);
expect(importResponse?.status).toBe(200);
const result = await importResponse!.json() as any;
expect(result.restartRequired).toBe(true);
expect(result.applied).toContain("portable server restore staged");
expect(existsSync(path.join(dataDir, ".pending-server-restore.json"))).toBe(true);
});
});
@@ -0,0 +1,174 @@
import { afterAll, describe, expect, test } from "bun:test";
import AdmZip from "adm-zip";
import Database from "bun:sqlite";
import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync, mkdirSync } from "fs";
import { tmpdir } from "os";
import path from "path";
import { generateSelfSignedCert } from "../certGenerator";
import {
applyPendingServerRestore,
createDatabaseSnapshot,
createPortableServerBackup,
stagePortableServerRestore,
type ServerBackupPaths,
} from "./server-backup";
const roots: string[] = [];
function tempRoot(label: string): string {
const root = mkdtempSync(path.join(tmpdir(), label));
roots.push(root);
return root;
}
function pathsAt(root: string): ServerBackupPaths {
return {
dataDir: path.join(root, "data"),
pluginRoot: path.join(root, "plugins"),
tlsCertPath: path.join(root, "certs", "server.crt"),
tlsKeyPath: path.join(root, "certs", "server.key"),
tlsCaPath: path.join(root, "certs", "ca.crt"),
};
}
function createSourceDatabase(databasePath: string): Database {
mkdirSync(path.dirname(databasePath), { recursive: true });
const database = new Database(databasePath);
database.exec(`
CREATE TABLE users (id INTEGER PRIMARY KEY, username TEXT NOT NULL);
CREATE TABLE clients (id TEXT PRIMARY KEY, nickname TEXT);
INSERT INTO users (id, username) VALUES (7, 'portable-admin');
INSERT INTO clients (id, nickname) VALUES ('agent-1', 'portable-agent');
`);
return database;
}
afterAll(() => {
for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true });
});
describe("portable server backup", () => {
test("round-trips database, identity, secrets, plugins, and persistent files", async () => {
const source = pathsAt(tempRoot("overlord-backup-source-"));
mkdirSync(source.dataDir, { recursive: true });
mkdirSync(path.join(source.dataDir, "file-share"), { recursive: true });
mkdirSync(path.join(source.pluginRoot, "example", "data"), { recursive: true });
writeFileSync(path.join(source.dataDir, "save.json"), JSON.stringify({
auth: { jwtSecret: "jwt-secret", agentToken: "agent-token" },
buildSigning: { privateKey: "build-private", publicKey: "build-public" },
clientLogs: { privateKey: "log-private", publicKey: "log-public" },
}));
writeFileSync(path.join(source.dataDir, "vapid-keys.json"), '{"publicKey":"vapid-public","privateKey":"vapid-private"}');
writeFileSync(path.join(source.dataDir, "file-share", "payload.txt"), "portable file");
writeFileSync(path.join(source.pluginRoot, ".plugin-state.json"), '{"example":{"enabled":true}}');
writeFileSync(path.join(source.pluginRoot, "example", "data", "state.json"), '{"value":42}');
await generateSelfSignedCert({
certPath: source.tlsCertPath,
keyPath: source.tlsKeyPath,
commonName: "localhost",
daysValid: 3650,
});
const sourceDatabase = createSourceDatabase(path.join(source.dataDir, "overlord.db"));
const snapshot = createDatabaseSnapshot(sourceDatabase);
sourceDatabase.close();
const configSnapshot = Buffer.from(JSON.stringify({
tls: {
certPath: "/old-machine/certs/server.crt",
keyPath: "/old-machine/certs/server.key",
certbot: { enabled: true },
},
appearance: { customCSS: "body{}" },
}));
const archive = createPortableServerBackup(source, "3.0.0", snapshot, configSnapshot);
const destination = pathsAt(tempRoot("overlord-backup-destination-"));
mkdirSync(destination.dataDir, { recursive: true });
writeFileSync(path.join(destination.dataDir, "save.json"), '{"old":true}');
const staged = stagePortableServerRestore(archive, destination, "3.0.0");
expect(staged.restartRequired).toBe(true);
expect(staged.serverVersion).toBe("3.0.0");
expect(existsSync(path.join(destination.dataDir, ".pending-server-restore.json"))).toBe(true);
expect(applyPendingServerRestore(destination)).toBe(true);
expect(readFileSync(destination.tlsCertPath)).toEqual(readFileSync(source.tlsCertPath));
expect(readFileSync(destination.tlsKeyPath)).toEqual(readFileSync(source.tlsKeyPath));
expect(readFileSync(path.join(destination.dataDir, "file-share", "payload.txt"), "utf8")).toBe("portable file");
expect(readFileSync(path.join(destination.pluginRoot, "example", "data", "state.json"), "utf8")).toBe('{"value":42}');
const restoredConfig = JSON.parse(readFileSync(path.join(destination.dataDir, "config.json"), "utf8"));
expect(restoredConfig.tls.certPath).toBe(destination.tlsCertPath);
expect(restoredConfig.tls.keyPath).toBe(destination.tlsKeyPath);
expect(restoredConfig.tls.certbot.enabled).toBe(false);
const restoredDatabase = new Database(path.join(destination.dataDir, "overlord.db"), { readonly: true });
expect(restoredDatabase.query("SELECT username FROM users WHERE id = 7").get()).toEqual({
username: "portable-admin",
});
expect(restoredDatabase.query("SELECT nickname FROM clients WHERE id = 'agent-1'").get()).toEqual({
nickname: "portable-agent",
});
restoredDatabase.close();
expect(applyPendingServerRestore(destination)).toBe(false);
});
test("rejects a modified archive before staging any restore", async () => {
const source = pathsAt(tempRoot("overlord-backup-tamper-source-"));
mkdirSync(source.dataDir, { recursive: true });
writeFileSync(path.join(source.dataDir, "save.json"), '{"auth":{"jwtSecret":"a","agentToken":"b"}}');
await generateSelfSignedCert({
certPath: source.tlsCertPath,
keyPath: source.tlsKeyPath,
commonName: "localhost",
});
const database = createSourceDatabase(path.join(source.dataDir, "overlord.db"));
const archive = createPortableServerBackup(
source,
"3.0.0",
createDatabaseSnapshot(database),
Buffer.from('{"tls":{}}'),
);
database.close();
const originalZip = new AdmZip(archive);
const tamperedZip = new AdmZip();
for (const entry of originalZip.getEntries()) {
if (entry.isDirectory) continue;
tamperedZip.addFile(
entry.entryName,
entry.entryName === "data/save.json" ? Buffer.from('{"tampered":true}') : entry.getData(),
);
}
const destination = pathsAt(tempRoot("overlord-backup-tamper-destination-"));
mkdirSync(destination.dataDir, { recursive: true });
writeFileSync(path.join(destination.dataDir, "save.json"), '{"keep":"current"}');
expect(() => stagePortableServerRestore(tamperedZip.toBuffer(), destination)).toThrow(
"checksum validation failed",
);
expect(existsSync(path.join(destination.dataDir, ".pending-server-restore.json"))).toBe(false);
expect(readFileSync(path.join(destination.dataDir, "save.json"), "utf8")).toBe('{"keep":"current"}');
const newerZip = new AdmZip();
for (const entry of originalZip.getEntries()) {
if (entry.isDirectory) continue;
if (entry.entryName === "manifest.json") {
const manifest = JSON.parse(entry.getData().toString("utf8"));
manifest.serverVersion = "4.0.0";
newerZip.addFile(entry.entryName, Buffer.from(JSON.stringify(manifest)));
} else {
newerZip.addFile(entry.entryName, entry.getData());
}
}
expect(() => stagePortableServerRestore(newerZip.toBuffer(), destination, "3.0.0")).toThrow(
"created by newer server 4.0.0",
);
expect(existsSync(path.join(destination.dataDir, ".pending-server-restore.json"))).toBe(false);
const unsafeZip = new AdmZip(archive);
unsafeZip.addFile("../outside.txt", Buffer.from("must not escape"));
expect(() => stagePortableServerRestore(unsafeZip.toBuffer(), destination, "3.0.0")).toThrow();
expect(existsSync(path.join(path.dirname(destination.dataDir), "outside.txt"))).toBe(false);
expect(existsSync(path.join(destination.dataDir, ".pending-server-restore.json"))).toBe(false);
});
});
+571
View File
@@ -0,0 +1,571 @@
import AdmZip from "adm-zip";
import Database from "bun:sqlite";
import { createHash, createPrivateKey, createPublicKey, X509Certificate } from "crypto";
import {
chmodSync,
copyFileSync,
cpSync,
existsSync,
lstatSync,
mkdirSync,
mkdtempSync,
readFileSync,
readdirSync,
renameSync,
rmSync,
writeFileSync,
} from "fs";
import { tmpdir } from "os";
import path from "path";
import { ensureDataDir } from "../paths";
import { resolveRuntimeRoot } from "./runtime-paths";
const BACKUP_SCHEMA = "overlord-server-backup";
const BACKUP_FORMAT_VERSION = 1;
const MAX_ARCHIVE_BYTES = 1024 * 1024 * 1024;
const MAX_ENTRY_COUNT = 20_000;
const MAX_ENTRY_BYTES = 512 * 1024 * 1024;
const MAX_TOTAL_BYTES = 2 * 1024 * 1024 * 1024;
const PENDING_MARKER = ".pending-server-restore.json";
type BackupEntry = {
path: string;
size: number;
sha256: string;
required: boolean;
};
type BackupManifest = {
schema: typeof BACKUP_SCHEMA;
formatVersion: typeof BACKUP_FORMAT_VERSION;
serverVersion: string;
exportedAt: string;
entries: BackupEntry[];
};
export type ServerBackupPaths = {
dataDir: string;
pluginRoot: string;
tlsCertPath: string;
tlsKeyPath: string;
tlsCaPath?: string;
};
const DATA_FILES = ["config.json", "save.json", "vapid-keys.json"] as const;
const DATA_DIRECTORIES = ["file-share", "deploy", "winre", "rd-recordings", "macos-sdk-uploads"] as const;
function sha256(bytes: Buffer): string {
return createHash("sha256").update(bytes).digest("hex");
}
function normalizeArchivePath(value: string): string {
const normalized = value.replace(/\\/g, "/").replace(/^\/+/, "");
if (
!normalized ||
normalized.includes("\0") ||
normalized.split("/").some((part) => part === "" || part === "." || part === "..")
) {
throw new Error(`Unsafe backup entry path: ${value}`);
}
return normalized;
}
function isAllowedArchivePath(entryPath: string): boolean {
if (entryPath === "data/overlord.db") return true;
if (DATA_FILES.some((name) => entryPath === `data/${name}`)) return true;
if (entryPath === "tls/server.crt" || entryPath === "tls/server.key" || entryPath === "tls/ca.crt") return true;
if (entryPath.startsWith("plugins/")) return true;
return DATA_DIRECTORIES.some((name) => entryPath.startsWith(`data/${name}/`));
}
function addArchiveFile(
zip: AdmZip,
manifestEntries: BackupEntry[],
entryPath: string,
bytes: Buffer,
required: boolean,
): void {
const safePath = normalizeArchivePath(entryPath);
if (!isAllowedArchivePath(safePath)) {
throw new Error(`Backup entry is outside the portable server allowlist: ${safePath}`);
}
if (bytes.length > MAX_ENTRY_BYTES) {
throw new Error(`Backup entry exceeds the ${MAX_ENTRY_BYTES} byte limit: ${safePath}`);
}
zip.addFile(safePath, bytes);
manifestEntries.push({
path: safePath,
size: bytes.length,
sha256: sha256(bytes),
required,
});
}
function addDirectory(
zip: AdmZip,
manifestEntries: BackupEntry[],
sourceRoot: string,
archiveRoot: string,
): void {
if (!existsSync(sourceRoot)) return;
const walk = (current: string, relative: string): void => {
for (const item of readdirSync(current, { withFileTypes: true })) {
const sourcePath = path.join(current, item.name);
const relativePath = relative ? `${relative}/${item.name}` : item.name;
const archivePath = `${archiveRoot}/${relativePath}`;
const stat = lstatSync(sourcePath);
if (stat.isSymbolicLink()) {
throw new Error(`Refusing to back up symbolic link: ${sourcePath}`);
}
if (stat.isDirectory()) {
walk(sourcePath, relativePath);
} else if (stat.isFile()) {
addArchiveFile(zip, manifestEntries, archivePath, readFileSync(sourcePath), false);
}
}
};
walk(sourceRoot, "");
}
function validateTlsIdentity(certPem: Buffer, keyPem: Buffer): void {
const certificate = new X509Certificate(certPem);
const certificateSpki = certificate.publicKey.export({ type: "spki", format: "der" });
const keySpki = createPublicKey(createPrivateKey(keyPem)).export({ type: "spki", format: "der" });
if (!Buffer.from(certificateSpki).equals(Buffer.from(keySpki))) {
throw new Error("TLS certificate and private key do not match");
}
}
export function resolveServerBackupPaths(overrides: Partial<ServerBackupPaths> = {}): ServerBackupPaths {
const dataDir = path.resolve(overrides.dataDir || ensureDataDir());
const runtimeRoot = resolveRuntimeRoot();
return {
dataDir,
pluginRoot: path.resolve(
overrides.pluginRoot ||
process.env.OVERLORD_PLUGIN_ROOT?.trim() ||
path.join(runtimeRoot, "plugins"),
),
tlsCertPath: path.resolve(
overrides.tlsCertPath ||
process.env.OVERLORD_TLS_CERT?.trim() ||
"./certs/server.crt",
),
tlsKeyPath: path.resolve(
overrides.tlsKeyPath ||
process.env.OVERLORD_TLS_KEY?.trim() ||
"./certs/server.key",
),
tlsCaPath: path.resolve(
overrides.tlsCaPath ||
process.env.OVERLORD_TLS_CA?.trim() ||
"./certs/ca.crt",
),
};
}
export function createDatabaseSnapshot(database: { exec: (sql: string) => unknown }): Buffer {
const root = mkdtempSync(path.join(tmpdir(), "overlord-db-backup-"));
const snapshotPath = path.join(root, "overlord.db");
try {
database.exec(`VACUUM INTO '${snapshotPath.replace(/'/g, "''")}'`);
return readFileSync(snapshotPath);
} finally {
rmSync(root, { recursive: true, force: true });
}
}
export function createPortableServerBackup(
paths: ServerBackupPaths,
serverVersion: string,
databaseSnapshot: Buffer,
configSnapshot?: Buffer,
): Buffer {
if (!existsSync(paths.tlsCertPath) || !existsSync(paths.tlsKeyPath)) {
throw new Error("TLS certificate and private key are required for a portable server backup");
}
const cert = readFileSync(paths.tlsCertPath);
const key = readFileSync(paths.tlsKeyPath);
validateTlsIdentity(cert, key);
const zip = new AdmZip();
const entries: BackupEntry[] = [];
addArchiveFile(zip, entries, "data/overlord.db", databaseSnapshot, true);
addArchiveFile(zip, entries, "tls/server.crt", cert, true);
addArchiveFile(zip, entries, "tls/server.key", key, true);
if (paths.tlsCaPath && existsSync(paths.tlsCaPath)) {
addArchiveFile(zip, entries, "tls/ca.crt", readFileSync(paths.tlsCaPath), false);
}
for (const name of DATA_FILES) {
const filePath = path.join(paths.dataDir, name);
if (name === "config.json" && configSnapshot) {
JSON.parse(configSnapshot.toString("utf8"));
addArchiveFile(zip, entries, "data/config.json", configSnapshot, true);
} else if (existsSync(filePath)) {
addArchiveFile(zip, entries, `data/${name}`, readFileSync(filePath), name !== "vapid-keys.json");
} else if (name !== "vapid-keys.json") {
throw new Error(`Required server state file is missing: ${filePath}`);
}
}
for (const name of DATA_DIRECTORIES) {
addDirectory(zip, entries, path.join(paths.dataDir, name), `data/${name}`);
}
addDirectory(zip, entries, paths.pluginRoot, "plugins");
if (entries.length > MAX_ENTRY_COUNT) {
throw new Error(`Backup contains too many files (${entries.length}; max ${MAX_ENTRY_COUNT})`);
}
const totalBytes = entries.reduce((sum, entry) => sum + entry.size, 0);
if (totalBytes > MAX_TOTAL_BYTES) {
throw new Error(`Backup contents exceed the ${MAX_TOTAL_BYTES} byte limit`);
}
const manifest: BackupManifest = {
schema: BACKUP_SCHEMA,
formatVersion: BACKUP_FORMAT_VERSION,
serverVersion,
exportedAt: new Date().toISOString(),
entries,
};
zip.addFile("manifest.json", Buffer.from(JSON.stringify(manifest, null, 2)));
const archive = zip.toBuffer();
if (archive.length > MAX_ARCHIVE_BYTES) {
throw new Error(`Compressed backup exceeds the ${MAX_ARCHIVE_BYTES} byte limit`);
}
return archive;
}
function parseAndValidateArchive(archive: Buffer): {
manifest: BackupManifest;
files: Map<string, Buffer>;
} {
if (archive.length === 0 || archive.length > MAX_ARCHIVE_BYTES) {
throw new Error("Backup archive is empty or exceeds the maximum size");
}
let zip: AdmZip;
try {
zip = new AdmZip(archive);
} catch {
throw new Error("Backup is not a valid ZIP archive");
}
const zipEntries = zip.getEntries().filter((entry) => !entry.isDirectory);
if (zipEntries.length > MAX_ENTRY_COUNT + 1) {
throw new Error("Backup contains too many entries");
}
let declaredUncompressedBytes = 0;
for (const entry of zipEntries) {
const declaredSize = Number(entry.header?.size);
if (!Number.isSafeInteger(declaredSize) || declaredSize < 0 || declaredSize > MAX_ENTRY_BYTES) {
throw new Error(`Backup entry has an invalid or excessive declared size: ${entry.entryName}`);
}
declaredUncompressedBytes += declaredSize;
if (declaredUncompressedBytes > MAX_TOTAL_BYTES + 1024 * 1024) {
throw new Error("Backup declares excessive uncompressed content");
}
}
const entryMap = new Map<string, Buffer>();
for (const entry of zipEntries) {
const safePath = normalizeArchivePath(entry.entryName);
if (entryMap.has(safePath)) throw new Error(`Duplicate backup entry: ${safePath}`);
const bytes = entry.getData();
if (bytes.length > MAX_ENTRY_BYTES) throw new Error(`Backup entry is too large: ${safePath}`);
entryMap.set(safePath, bytes);
}
const manifestBytes = entryMap.get("manifest.json");
if (!manifestBytes || manifestBytes.length > 1024 * 1024) {
throw new Error("Backup manifest is missing or too large");
}
let manifest: BackupManifest;
try {
manifest = JSON.parse(manifestBytes.toString("utf8"));
} catch {
throw new Error("Backup manifest is invalid JSON");
}
if (manifest.schema !== BACKUP_SCHEMA || manifest.formatVersion !== BACKUP_FORMAT_VERSION) {
throw new Error("Unsupported server backup format");
}
if (!Array.isArray(manifest.entries) || manifest.entries.length > MAX_ENTRY_COUNT) {
throw new Error("Backup manifest entries are invalid");
}
const files = new Map<string, Buffer>();
let totalBytes = 0;
const declared = new Set<string>();
for (const item of manifest.entries) {
if (!item || typeof item !== "object") throw new Error("Invalid backup manifest entry");
const safePath = normalizeArchivePath(String(item.path || ""));
if (!isAllowedArchivePath(safePath) || declared.has(safePath)) {
throw new Error(`Invalid or duplicate manifest path: ${safePath}`);
}
declared.add(safePath);
const bytes = entryMap.get(safePath);
if (!bytes) throw new Error(`Backup entry is missing: ${safePath}`);
if (bytes.length !== item.size || sha256(bytes) !== item.sha256) {
throw new Error(`Backup checksum validation failed: ${safePath}`);
}
totalBytes += bytes.length;
if (totalBytes > MAX_TOTAL_BYTES) throw new Error("Uncompressed backup exceeds the maximum size");
files.set(safePath, bytes);
}
for (const entryPath of entryMap.keys()) {
if (entryPath !== "manifest.json" && !declared.has(entryPath)) {
throw new Error(`Undeclared backup entry: ${entryPath}`);
}
}
for (const required of ["data/overlord.db", "data/config.json", "data/save.json", "tls/server.crt", "tls/server.key"]) {
if (!files.has(required)) throw new Error(`Required backup entry is missing: ${required}`);
}
validateTlsIdentity(files.get("tls/server.crt")!, files.get("tls/server.key")!);
JSON.parse(files.get("data/config.json")!.toString("utf8"));
JSON.parse(files.get("data/save.json")!.toString("utf8"));
return { manifest, files };
}
function validateDatabase(bytes: Buffer): void {
const root = mkdtempSync(path.join(tmpdir(), "overlord-db-validate-"));
const databasePath = path.join(root, "overlord.db");
try {
writeFileSync(databasePath, bytes, { mode: 0o600 });
const database = new Database(databasePath, { readonly: true });
try {
const result = database.query("PRAGMA quick_check").get() as Record<string, unknown> | null;
if (!result || !Object.values(result).includes("ok")) {
throw new Error("SQLite integrity check did not return ok");
}
const tables = database
.query("SELECT name FROM sqlite_master WHERE type = 'table'")
.all() as Array<{ name: string }>;
const names = new Set(tables.map((row) => row.name));
if (!names.has("users") || !names.has("clients")) {
throw new Error("Backup database does not contain the required Overlord schema");
}
} finally {
database.close();
}
} catch (error) {
throw new Error(`Backup database validation failed: ${(error as Error).message}`);
} finally {
rmSync(root, { recursive: true, force: true });
}
}
function writeStagedFiles(stageRoot: string, files: Map<string, Buffer>): void {
for (const [entryPath, bytes] of files) {
const destination = path.join(stageRoot, ...entryPath.split("/"));
mkdirSync(path.dirname(destination), { recursive: true });
writeFileSync(destination, bytes, { mode: entryPath.endsWith(".key") || entryPath === "data/save.json" ? 0o600 : 0o644 });
}
}
export function stagePortableServerRestore(
archive: Buffer,
paths: ServerBackupPaths,
currentServerVersion?: string,
): { restartRequired: true; serverVersion: string; files: number; warnings: string[] } {
const { manifest, files } = parseAndValidateArchive(archive);
if (currentServerVersion && compareVersions(manifest.serverVersion, currentServerVersion) > 0) {
throw new Error(
`Backup was created by newer server ${manifest.serverVersion}; upgrade this server (${currentServerVersion}) before restoring`,
);
}
validateDatabase(files.get("data/overlord.db")!);
const restoredConfig = JSON.parse(files.get("data/config.json")!.toString("utf8"));
restoredConfig.tls = {
...(restoredConfig.tls && typeof restoredConfig.tls === "object" ? restoredConfig.tls : {}),
certPath: paths.tlsCertPath,
keyPath: paths.tlsKeyPath,
caPath: files.has("tls/ca.crt") ? paths.tlsCaPath : "",
certbot: {
...(restoredConfig.tls?.certbot && typeof restoredConfig.tls.certbot === "object"
? restoredConfig.tls.certbot
: {}),
enabled: false,
},
};
const portableConfig = Buffer.from(JSON.stringify(restoredConfig, null, 2));
files.set("data/config.json", portableConfig);
const configEntry = manifest.entries.find((entry) => entry.path === "data/config.json")!;
configEntry.size = portableConfig.length;
configEntry.sha256 = sha256(portableConfig);
mkdirSync(paths.dataDir, { recursive: true });
const markerPath = path.join(paths.dataDir, PENDING_MARKER);
if (existsSync(markerPath)) {
throw new Error("A server restore is already pending; restart the server before importing another backup");
}
const stageRoot = mkdtempSync(path.join(paths.dataDir, ".server-restore-stage-"));
try {
writeStagedFiles(stageRoot, files);
writeFileSync(path.join(stageRoot, "manifest.json"), JSON.stringify(manifest, null, 2), { mode: 0o600 });
const markerTemp = `${markerPath}.tmp`;
writeFileSync(markerTemp, JSON.stringify({
schema: BACKUP_SCHEMA,
formatVersion: BACKUP_FORMAT_VERSION,
stageDirectory: path.basename(stageRoot),
stagedAt: new Date().toISOString(),
}), { mode: 0o600 });
renameSync(markerTemp, markerPath);
} catch (error) {
rmSync(stageRoot, { recursive: true, force: true });
throw error;
}
const warnings: string[] = [];
if (String(process.env.JWT_SECRET || "").trim() || String(process.env.OVERLORD_AGENT_TOKEN || "").trim()) {
warnings.push("JWT_SECRET or OVERLORD_AGENT_TOKEN environment overrides will take priority over restored identity secrets.");
}
return {
restartRequired: true,
serverVersion: manifest.serverVersion,
files: files.size,
warnings,
};
}
function replaceFile(source: string, destination: string, rollbackRoot: string, label: string): void {
mkdirSync(path.dirname(destination), { recursive: true });
const rollback = path.join(rollbackRoot, label);
mkdirSync(path.dirname(rollback), { recursive: true });
const incoming = `${destination}.restore-incoming`;
const hadDestination = existsSync(destination);
if (hadDestination) renameSync(destination, rollback);
try {
copyFileSync(source, incoming);
chmodSync(incoming, destination.endsWith(".key") || destination.endsWith("save.json") ? 0o600 : 0o644);
renameSync(incoming, destination);
} catch (error) {
rmSync(incoming, { force: true });
if (hadDestination && existsSync(rollback) && !existsSync(destination)) {
renameSync(rollback, destination);
}
throw error;
}
}
function replaceDirectory(source: string, destination: string, rollbackRoot: string, label: string): void {
const rollback = path.join(rollbackRoot, label);
mkdirSync(path.dirname(destination), { recursive: true });
mkdirSync(path.dirname(rollback), { recursive: true });
const incoming = `${destination}.restore-incoming`;
const hadDestination = existsSync(destination);
if (hadDestination) renameSync(destination, rollback);
try {
if (existsSync(source)) {
cpSync(source, incoming, { recursive: true, errorOnExist: true });
renameSync(incoming, destination);
} else {
mkdirSync(destination, { recursive: true });
}
} catch (error) {
rmSync(incoming, { recursive: true, force: true });
if (hadDestination && existsSync(rollback) && !existsSync(destination)) {
renameSync(rollback, destination);
}
throw error;
}
}
function compareVersions(left: string, right: string): number {
const parse = (value: string): number[] | null => {
const match = String(value || "").trim().match(/^v?(\d+)\.(\d+)\.(\d+)(?:[-+].*)?$/);
return match ? [Number(match[1]), Number(match[2]), Number(match[3])] : null;
};
const a = parse(left);
const b = parse(right);
if (!a || !b) return 0;
for (let index = 0; index < 3; index++) {
if (a[index] !== b[index]) return a[index] > b[index] ? 1 : -1;
}
return 0;
}
export function applyPendingServerRestore(overrides: Partial<ServerBackupPaths> = {}): boolean {
const paths = resolveServerBackupPaths(overrides);
const markerPath = path.join(paths.dataDir, PENDING_MARKER);
if (!existsSync(markerPath)) return false;
const marker = JSON.parse(readFileSync(markerPath, "utf8")) as {
schema?: string;
formatVersion?: number;
stageDirectory?: string;
};
if (
marker.schema !== BACKUP_SCHEMA ||
marker.formatVersion !== BACKUP_FORMAT_VERSION ||
!marker.stageDirectory ||
path.basename(marker.stageDirectory) !== marker.stageDirectory ||
!marker.stageDirectory.startsWith(".server-restore-stage-")
) {
throw new Error("Pending server restore marker is invalid");
}
const stageRoot = path.join(paths.dataDir, marker.stageDirectory);
const manifestPath = path.join(stageRoot, "manifest.json");
if (!existsSync(manifestPath)) throw new Error("Pending server restore staging directory is incomplete");
const manifest = JSON.parse(readFileSync(manifestPath, "utf8")) as BackupManifest;
const stagedZip = new AdmZip();
for (const entry of manifest.entries) {
const source = path.join(stageRoot, ...normalizeArchivePath(entry.path).split("/"));
if (!existsSync(source)) throw new Error(`Pending restore entry is missing: ${entry.path}`);
const bytes = readFileSync(source);
if (bytes.length !== entry.size || sha256(bytes) !== entry.sha256) {
throw new Error(`Pending restore checksum failed: ${entry.path}`);
}
stagedZip.addFile(entry.path, bytes);
}
stagedZip.addFile("manifest.json", Buffer.from(JSON.stringify(manifest)));
parseAndValidateArchive(stagedZip.toBuffer());
const rollbackRoot = mkdtempSync(path.join(paths.dataDir, ".server-restore-rollback-"));
const completed: Array<{ destination: string; rollback: string; directory: boolean }> = [];
const replaceTracked = (source: string, destination: string, label: string, directory: boolean) => {
const rollback = path.join(rollbackRoot, label);
if (directory) replaceDirectory(source, destination, rollbackRoot, label);
else replaceFile(source, destination, rollbackRoot, label);
completed.push({ destination, rollback, directory });
};
try {
replaceTracked(path.join(stageRoot, "data", "overlord.db"), path.join(paths.dataDir, "overlord.db"), "data/overlord.db", false);
for (const name of DATA_FILES) {
const source = path.join(stageRoot, "data", name);
if (existsSync(source)) replaceTracked(source, path.join(paths.dataDir, name), `data/${name}`, false);
}
for (const name of DATA_DIRECTORIES) {
replaceTracked(path.join(stageRoot, "data", name), path.join(paths.dataDir, name), `data/${name}`, true);
}
replaceTracked(path.join(stageRoot, "plugins"), paths.pluginRoot, "plugins", true);
replaceTracked(path.join(stageRoot, "tls", "server.crt"), paths.tlsCertPath, "tls/server.crt", false);
replaceTracked(path.join(stageRoot, "tls", "server.key"), paths.tlsKeyPath, "tls/server.key", false);
const stagedCa = path.join(stageRoot, "tls", "ca.crt");
if (paths.tlsCaPath && existsSync(stagedCa)) {
replaceTracked(stagedCa, paths.tlsCaPath, "tls/ca.crt", false);
}
for (const suffix of ["-wal", "-shm"]) {
const sidecar = path.join(paths.dataDir, `overlord.db${suffix}`);
if (existsSync(sidecar)) rmSync(sidecar, { force: true });
}
rmSync(markerPath, { force: true });
rmSync(stageRoot, { recursive: true, force: true });
rmSync(rollbackRoot, { recursive: true, force: true });
return true;
} catch (error) {
for (const item of completed.reverse()) {
try {
rmSync(item.destination, { recursive: item.directory, force: true });
if (existsSync(item.rollback)) {
mkdirSync(path.dirname(item.destination), { recursive: true });
renameSync(item.rollback, item.destination);
}
} catch {}
}
throw new Error(`Server restore failed and was rolled back: ${(error as Error).message}`);
}
}
@@ -0,0 +1,45 @@
import { afterEach, describe, expect, test } from "bun:test";
import { createHash, X509Certificate } from "crypto";
import { mkdtempSync, readFileSync, rmSync } from "fs";
import { tmpdir } from "os";
import path from "path";
import {
computeCertificateSpkiPin,
getActiveTlsSpkiPins,
prepareTlsOptions,
} from "./tls-bootstrap";
const originalPins = process.env.OVERLORD_TLS_SPKI_PINS;
const tempRoots: string[] = [];
afterEach(() => {
if (originalPins === undefined) delete process.env.OVERLORD_TLS_SPKI_PINS;
else process.env.OVERLORD_TLS_SPKI_PINS = originalPins;
for (const root of tempRoots.splice(0)) {
rmSync(root, { recursive: true, force: true });
}
});
describe("TLS identity pin bootstrap", () => {
test("creates a self-signed certificate and publishes its SPKI pin", async () => {
const root = mkdtempSync(path.join(tmpdir(), "overlord-tls-pin-"));
tempRoots.push(root);
const certPath = path.join(root, "server.crt");
const keyPath = path.join(root, "server.key");
const rotationPin = Buffer.alloc(32, 0x42).toString("base64");
process.env.OVERLORD_TLS_SPKI_PINS =
`sha256/${rotationPin},invalid,${rotationPin}`;
const result = await prepareTlsOptions({ certPath, keyPath });
const certificatePem = readFileSync(certPath, "utf8");
const certificate = new X509Certificate(certificatePem);
const expectedPin = createHash("sha256")
.update(certificate.publicKey.export({ type: "spki", format: "der" }))
.digest("base64");
expect(result.source).toBe("self-signed");
expect(result.tlsOptions.cert).toBe(certificatePem);
expect(computeCertificateSpkiPin(certificatePem)).toBe(expectedPin);
expect(getActiveTlsSpkiPins()).toEqual([expectedPin, rotationPin]);
});
});
+36 -1
View File
@@ -1,6 +1,6 @@
import { certificatesExist, generateSelfSignedCert, getLocalIPs, isOpenSSLAvailable } from "../certGenerator";
import { logger } from "../logger";
import { X509Certificate } from "crypto";
import { X509Certificate, createHash } from "crypto";
import path from "path";
type TlsBootstrapParams = {
@@ -23,6 +23,39 @@ type TlsBootstrapResult = {
source: "certbot" | "configured" | "self-signed";
};
let activeTlsSpkiPins: string[] = [];
function normalizeSpkiPins(value: string | undefined): string[] {
if (!value) return [];
return [...new Set(value.split(",").map((pin) =>
pin.trim().replace(/^sha256\//i, "")
).filter((pin) =>
/^[A-Za-z0-9+/]{43}=$/.test(pin)
))];
}
export function computeCertificateSpkiPin(certificatePem: string): string {
const certificate = new X509Certificate(certificatePem);
const spki = certificate.publicKey.export({ type: "spki", format: "der" });
return createHash("sha256").update(spki).digest("base64");
}
function publishActiveTlsPins(certificatePem: string): void {
activeTlsSpkiPins = [
computeCertificateSpkiPin(certificatePem),
...normalizeSpkiPins(process.env.OVERLORD_TLS_SPKI_PINS),
].filter((pin, index, pins) => pins.indexOf(pin) === index);
logger.info(`[TLS] Active SPKI pin: sha256/${activeTlsSpkiPins[0]}`);
if (activeTlsSpkiPins.length > 1) {
logger.info(`[TLS] ${activeTlsSpkiPins.length - 1} additional SPKI rotation pin(s) configured`);
}
}
export function getActiveTlsSpkiPins(): string[] {
if (activeTlsSpkiPins.length > 0) return [...activeTlsSpkiPins];
return normalizeSpkiPins(process.env.OVERLORD_TLS_SPKI_PINS);
}
export async function prepareTlsOptions(params: TlsBootstrapParams): Promise<TlsBootstrapResult> {
logger.info("[TLS] TLS/HTTPS is always enabled for security");
@@ -57,6 +90,7 @@ export async function prepareTlsOptions(params: TlsBootstrapParams): Promise<Tls
if (await caFile.exists()) {
tlsOptions.ca = await caFile.text();
}
publishActiveTlsPins(tlsOptions.cert!);
return {
tlsOptions,
@@ -128,6 +162,7 @@ export async function prepareTlsOptions(params: TlsBootstrapParams): Promise<Tls
}
} catch { }
}
publishActiveTlsPins(tlsOptions.cert!);
return {
tlsOptions,