327 lines
9.5 KiB
TypeScript
327 lines
9.5 KiB
TypeScript
import path from "path";
|
|
import fs from "fs";
|
|
import { $ } from "bun";
|
|
import { ensureDataDir } from "../paths";
|
|
import { logger } from "../logger";
|
|
|
|
type SendToStream = (data: any) => void;
|
|
|
|
export type ToolchainKey =
|
|
| "mingw-w64-x64"
|
|
| "mingw-w64-x86"
|
|
| "linux-musl-x64"
|
|
| "linux-musl-arm64"
|
|
| "linux-musl-armv7"
|
|
| "android-ndk"
|
|
| "ldid"
|
|
| "upx";
|
|
|
|
export type EnsuredToolchain = {
|
|
key: ToolchainKey;
|
|
binDir: string;
|
|
ccPath?: string;
|
|
cxxPath?: string;
|
|
arPath?: string;
|
|
rootDir: string;
|
|
};
|
|
|
|
type ArchiveFormat = "tar.gz" | "tar.xz" | "zip" | "binary";
|
|
|
|
type Manifest = {
|
|
displayName: string;
|
|
url: string;
|
|
approxSizeMB: number;
|
|
archive: ArchiveFormat;
|
|
binSubdir: string;
|
|
ccBasename?: string;
|
|
cxxBasename?: string;
|
|
arBasename?: string;
|
|
binaryFilename?: string;
|
|
};
|
|
|
|
const ANDROID_NDK_VERSION = "r27c";
|
|
|
|
function ndkHostTriple(): string {
|
|
return process.arch === "arm64" ? "linux-aarch64" : "linux-x86_64";
|
|
}
|
|
|
|
function ldidArch(): string {
|
|
return process.arch === "arm64" ? "aarch64" : "x86_64";
|
|
}
|
|
|
|
const MANIFESTS: Record<ToolchainKey, Manifest> = {
|
|
"mingw-w64-x64": {
|
|
displayName: "mingw-w64 x86_64 cross-compiler",
|
|
url: "https://musl.cc/x86_64-w64-mingw32-cross.tgz",
|
|
approxSizeMB: 70,
|
|
archive: "tar.gz",
|
|
binSubdir: "x86_64-w64-mingw32-cross/bin",
|
|
ccBasename: "x86_64-w64-mingw32-gcc",
|
|
cxxBasename: "x86_64-w64-mingw32-g++",
|
|
arBasename: "x86_64-w64-mingw32-ar",
|
|
},
|
|
"mingw-w64-x86": {
|
|
displayName: "mingw-w64 i686 cross-compiler",
|
|
url: "https://musl.cc/i686-w64-mingw32-cross.tgz",
|
|
approxSizeMB: 65,
|
|
archive: "tar.gz",
|
|
binSubdir: "i686-w64-mingw32-cross/bin",
|
|
ccBasename: "i686-w64-mingw32-gcc",
|
|
cxxBasename: "i686-w64-mingw32-g++",
|
|
arBasename: "i686-w64-mingw32-ar",
|
|
},
|
|
"linux-musl-x64": {
|
|
displayName: "musl x86_64 Linux cross-compiler",
|
|
url: "https://musl.cc/x86_64-linux-musl-cross.tgz",
|
|
approxSizeMB: 60,
|
|
archive: "tar.gz",
|
|
binSubdir: "x86_64-linux-musl-cross/bin",
|
|
ccBasename: "x86_64-linux-musl-gcc",
|
|
cxxBasename: "x86_64-linux-musl-g++",
|
|
arBasename: "x86_64-linux-musl-ar",
|
|
},
|
|
"linux-musl-arm64": {
|
|
displayName: "musl aarch64 Linux cross-compiler",
|
|
url: "https://musl.cc/aarch64-linux-musl-cross.tgz",
|
|
approxSizeMB: 60,
|
|
archive: "tar.gz",
|
|
binSubdir: "aarch64-linux-musl-cross/bin",
|
|
ccBasename: "aarch64-linux-musl-gcc",
|
|
cxxBasename: "aarch64-linux-musl-g++",
|
|
arBasename: "aarch64-linux-musl-ar",
|
|
},
|
|
"linux-musl-armv7": {
|
|
displayName: "musl armv7l Linux cross-compiler",
|
|
url: "https://musl.cc/armv7l-linux-musleabihf-cross.tgz",
|
|
approxSizeMB: 60,
|
|
archive: "tar.gz",
|
|
binSubdir: "armv7l-linux-musleabihf-cross/bin",
|
|
ccBasename: "armv7l-linux-musleabihf-gcc",
|
|
cxxBasename: "armv7l-linux-musleabihf-g++",
|
|
arBasename: "armv7l-linux-musleabihf-ar",
|
|
},
|
|
"android-ndk": {
|
|
displayName: `Android NDK ${ANDROID_NDK_VERSION}`,
|
|
url: `https://dl.google.com/android/repository/android-ndk-${ANDROID_NDK_VERSION}-linux.zip`,
|
|
approxSizeMB: 1500,
|
|
archive: "zip",
|
|
binSubdir: `android-ndk-${ANDROID_NDK_VERSION}/toolchains/llvm/prebuilt/${ndkHostTriple()}/bin`,
|
|
arBasename: "llvm-ar",
|
|
},
|
|
ldid: {
|
|
displayName: "ldid (iOS pseudo-signing)",
|
|
url: `https://github.com/ProcursusTeam/ldid/releases/download/v2.1.5-procursus7/ldid_linux_${ldidArch()}`,
|
|
approxSizeMB: 2,
|
|
archive: "binary",
|
|
binSubdir: "bin",
|
|
binaryFilename: "ldid",
|
|
},
|
|
upx: {
|
|
displayName: "UPX 4.2.4",
|
|
url: "https://github.com/upx/upx/releases/download/v4.2.4/upx-4.2.4-amd64_linux.tar.xz",
|
|
approxSizeMB: 1,
|
|
archive: "tar.xz",
|
|
binSubdir: "upx-4.2.4-amd64_linux",
|
|
binaryFilename: "upx",
|
|
},
|
|
};
|
|
|
|
const inFlight = new Map<ToolchainKey, Promise<EnsuredToolchain>>();
|
|
|
|
export function getToolchainRoot(): string {
|
|
const explicit = process.env.OVERLORD_TOOLCHAIN_DIR?.trim();
|
|
if (explicit) return path.resolve(explicit);
|
|
return path.join(ensureDataDir(), "toolchains");
|
|
}
|
|
|
|
function toolchainDir(key: ToolchainKey): string {
|
|
return path.join(getToolchainRoot(), key);
|
|
}
|
|
|
|
function sentinelPath(key: ToolchainKey): string {
|
|
return path.join(toolchainDir(key), ".installed");
|
|
}
|
|
|
|
function isInstalled(key: ToolchainKey): boolean {
|
|
try {
|
|
return fs.existsSync(sentinelPath(key));
|
|
} catch {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
function resolveEnsured(key: ToolchainKey): EnsuredToolchain {
|
|
const m = MANIFESTS[key];
|
|
const rootDir = toolchainDir(key);
|
|
const binDir = path.join(rootDir, m.binSubdir);
|
|
return {
|
|
key,
|
|
rootDir,
|
|
binDir,
|
|
ccPath: m.ccBasename ? path.join(binDir, m.ccBasename) : undefined,
|
|
cxxPath: m.cxxBasename ? path.join(binDir, m.cxxBasename) : undefined,
|
|
arPath: m.arBasename ? path.join(binDir, m.arBasename) : undefined,
|
|
};
|
|
}
|
|
|
|
async function downloadAndExtract(
|
|
key: ToolchainKey,
|
|
send: SendToStream,
|
|
): Promise<void> {
|
|
const m = MANIFESTS[key];
|
|
const rootDir = toolchainDir(key);
|
|
const tmpDir = `${rootDir}.tmp-${process.pid}-${Date.now()}`;
|
|
|
|
fs.mkdirSync(getToolchainRoot(), { recursive: true });
|
|
fs.mkdirSync(tmpDir, { recursive: true });
|
|
|
|
const archiveName =
|
|
m.archive === "binary"
|
|
? m.binaryFilename || "download"
|
|
: path.basename(new URL(m.url).pathname);
|
|
const archivePath = path.join(tmpDir, archiveName);
|
|
|
|
send({
|
|
type: "output",
|
|
text: `[toolchain] Downloading ${m.displayName} (~${m.approxSizeMB}MB) from ${m.url}\n`,
|
|
level: "info",
|
|
});
|
|
logger.info(`[toolchain] downloading ${key} from ${m.url}`);
|
|
|
|
const dl = await $`wget --tries=2 --timeout=60 -q -O ${archivePath} ${m.url}`
|
|
.nothrow();
|
|
if (dl.exitCode !== 0) {
|
|
const stderr = dl.stderr.toString().trim();
|
|
throw new Error(
|
|
`Failed to download ${m.displayName} (wget exit ${dl.exitCode}): ${stderr || "no stderr"}`,
|
|
);
|
|
}
|
|
|
|
send({
|
|
type: "output",
|
|
text: `[toolchain] Extracting ${m.displayName}...\n`,
|
|
level: "info",
|
|
});
|
|
|
|
switch (m.archive) {
|
|
case "tar.gz": {
|
|
const r = await $`tar -xzf ${archivePath} -C ${tmpDir}`.nothrow();
|
|
if (r.exitCode !== 0) {
|
|
throw new Error(`tar -xzf failed for ${key}: ${r.stderr.toString()}`);
|
|
}
|
|
fs.unlinkSync(archivePath);
|
|
break;
|
|
}
|
|
case "tar.xz": {
|
|
const r = await $`tar -xJf ${archivePath} -C ${tmpDir}`.nothrow();
|
|
if (r.exitCode !== 0) {
|
|
throw new Error(`tar -xJf failed for ${key}: ${r.stderr.toString()}`);
|
|
}
|
|
fs.unlinkSync(archivePath);
|
|
break;
|
|
}
|
|
case "zip": {
|
|
const r = await $`unzip -q ${archivePath} -d ${tmpDir}`.nothrow();
|
|
if (r.exitCode !== 0) {
|
|
throw new Error(`unzip failed for ${key}: ${r.stderr.toString()}`);
|
|
}
|
|
fs.unlinkSync(archivePath);
|
|
break;
|
|
}
|
|
case "binary": {
|
|
// Move single-binary download into a bin/ subdir so binSubdir resolves.
|
|
const binDir = path.join(tmpDir, m.binSubdir);
|
|
fs.mkdirSync(binDir, { recursive: true });
|
|
const finalBin = path.join(binDir, m.binaryFilename || "download");
|
|
fs.renameSync(archivePath, finalBin);
|
|
fs.chmodSync(finalBin, 0o755);
|
|
break;
|
|
}
|
|
}
|
|
|
|
// Atomic publish: rename tmpDir → rootDir. If another process already
|
|
// created rootDir (race), fall back to checking the sentinel.
|
|
try {
|
|
fs.renameSync(tmpDir, rootDir);
|
|
} catch (err: any) {
|
|
// If another worker won the race and rootDir already exists, accept it.
|
|
if (err.code === "ENOTEMPTY" || err.code === "EEXIST" || err.code === "EPERM") {
|
|
try {
|
|
fs.rmSync(tmpDir, { recursive: true, force: true });
|
|
} catch {}
|
|
if (isInstalled(key)) return;
|
|
}
|
|
throw err;
|
|
}
|
|
|
|
fs.writeFileSync(sentinelPath(key), `${m.url}\n${new Date().toISOString()}\n`);
|
|
send({
|
|
type: "output",
|
|
text: `[toolchain] ${m.displayName} ready at ${rootDir}\n`,
|
|
level: "info",
|
|
});
|
|
logger.info(`[toolchain] installed ${key} at ${rootDir}`);
|
|
}
|
|
|
|
function tryResolveFromPath(key: ToolchainKey): EnsuredToolchain | null {
|
|
const m = MANIFESTS[key];
|
|
const primaryBin = m.ccBasename || m.binaryFilename;
|
|
if (!primaryBin) return null;
|
|
|
|
const found = Bun.which(primaryBin);
|
|
if (!found) return null;
|
|
|
|
const binDir = path.dirname(found);
|
|
return {
|
|
key,
|
|
rootDir: path.dirname(binDir),
|
|
binDir,
|
|
ccPath: m.ccBasename ? found : undefined,
|
|
cxxPath: m.cxxBasename ? (Bun.which(m.cxxBasename) ?? path.join(binDir, m.cxxBasename)) : undefined,
|
|
arPath: m.arBasename ? (Bun.which(m.arBasename) ?? path.join(binDir, m.arBasename)) : undefined,
|
|
};
|
|
}
|
|
|
|
export async function ensureToolchain(
|
|
key: ToolchainKey,
|
|
send: SendToStream,
|
|
): Promise<EnsuredToolchain> {
|
|
const fromPath = tryResolveFromPath(key);
|
|
if (fromPath) {
|
|
logger.info(`[toolchain] using ${key} from PATH: ${fromPath.binDir}`);
|
|
send({ type: "output", text: `[toolchain] Using system ${MANIFESTS[key].displayName} from PATH\n`, level: "info" });
|
|
return fromPath;
|
|
}
|
|
|
|
if (isInstalled(key)) {
|
|
return resolveEnsured(key);
|
|
}
|
|
|
|
const existing = inFlight.get(key);
|
|
if (existing) return existing;
|
|
|
|
const promise = (async () => {
|
|
try {
|
|
await downloadAndExtract(key, send);
|
|
return resolveEnsured(key);
|
|
} finally {
|
|
inFlight.delete(key);
|
|
}
|
|
})();
|
|
|
|
inFlight.set(key, promise);
|
|
return promise;
|
|
}
|
|
|
|
const TARGET_TO_TOOLCHAIN: Record<string, ToolchainKey> = {
|
|
"linux/amd64": "linux-musl-x64",
|
|
"linux/arm64": "linux-musl-arm64",
|
|
"linux/arm/v7": "linux-musl-armv7",
|
|
"windows/amd64": "mingw-w64-x64",
|
|
"windows/386": "mingw-w64-x86",
|
|
};
|
|
|
|
export function toolchainKeyForTarget(targetKey: string): ToolchainKey | null {
|
|
return TARGET_TO_TOOLCHAIN[targetKey] || null;
|
|
}
|