massive shrink on docker
This commit is contained in:
KingKDot
@@ -0,0 +1,300 @@
|
||||
import path from "path";
|
||||
import fs from "fs";
|
||||
import { $ } from "bun";
|
||||
import { ensureDataDir } from "../paths";
|
||||
import { logger } from "../logger";
|
||||
|
||||
type SendToStream = (data: any) => void;
|
||||
|
||||
export type ToolchainKey =
|
||||
| "mingw-w64-x64"
|
||||
| "mingw-w64-x86"
|
||||
| "linux-musl-x64"
|
||||
| "linux-musl-arm64"
|
||||
| "linux-musl-armv7"
|
||||
| "android-ndk"
|
||||
| "ldid"
|
||||
| "upx";
|
||||
|
||||
export type EnsuredToolchain = {
|
||||
key: ToolchainKey;
|
||||
binDir: string;
|
||||
ccPath?: string;
|
||||
cxxPath?: string;
|
||||
arPath?: string;
|
||||
rootDir: string;
|
||||
};
|
||||
|
||||
type ArchiveFormat = "tar.gz" | "tar.xz" | "zip" | "binary";
|
||||
|
||||
type Manifest = {
|
||||
displayName: string;
|
||||
url: string;
|
||||
approxSizeMB: number;
|
||||
archive: ArchiveFormat;
|
||||
binSubdir: string;
|
||||
ccBasename?: string;
|
||||
cxxBasename?: string;
|
||||
arBasename?: string;
|
||||
binaryFilename?: string;
|
||||
};
|
||||
|
||||
const ANDROID_NDK_VERSION = "r27c";
|
||||
|
||||
function ndkHostTriple(): string {
|
||||
return process.arch === "arm64" ? "linux-aarch64" : "linux-x86_64";
|
||||
}
|
||||
|
||||
function ldidArch(): string {
|
||||
return process.arch === "arm64" ? "aarch64" : "x86_64";
|
||||
}
|
||||
|
||||
const MANIFESTS: Record<ToolchainKey, Manifest> = {
|
||||
"mingw-w64-x64": {
|
||||
displayName: "mingw-w64 x86_64 cross-compiler",
|
||||
url: "https://musl.cc/x86_64-w64-mingw32-cross.tgz",
|
||||
approxSizeMB: 70,
|
||||
archive: "tar.gz",
|
||||
binSubdir: "x86_64-w64-mingw32-cross/bin",
|
||||
ccBasename: "x86_64-w64-mingw32-gcc",
|
||||
cxxBasename: "x86_64-w64-mingw32-g++",
|
||||
arBasename: "x86_64-w64-mingw32-ar",
|
||||
},
|
||||
"mingw-w64-x86": {
|
||||
displayName: "mingw-w64 i686 cross-compiler",
|
||||
url: "https://musl.cc/i686-w64-mingw32-cross.tgz",
|
||||
approxSizeMB: 65,
|
||||
archive: "tar.gz",
|
||||
binSubdir: "i686-w64-mingw32-cross/bin",
|
||||
ccBasename: "i686-w64-mingw32-gcc",
|
||||
cxxBasename: "i686-w64-mingw32-g++",
|
||||
arBasename: "i686-w64-mingw32-ar",
|
||||
},
|
||||
"linux-musl-x64": {
|
||||
displayName: "musl x86_64 Linux cross-compiler",
|
||||
url: "https://musl.cc/x86_64-linux-musl-cross.tgz",
|
||||
approxSizeMB: 60,
|
||||
archive: "tar.gz",
|
||||
binSubdir: "x86_64-linux-musl-cross/bin",
|
||||
ccBasename: "x86_64-linux-musl-gcc",
|
||||
cxxBasename: "x86_64-linux-musl-g++",
|
||||
arBasename: "x86_64-linux-musl-ar",
|
||||
},
|
||||
"linux-musl-arm64": {
|
||||
displayName: "musl aarch64 Linux cross-compiler",
|
||||
url: "https://musl.cc/aarch64-linux-musl-cross.tgz",
|
||||
approxSizeMB: 60,
|
||||
archive: "tar.gz",
|
||||
binSubdir: "aarch64-linux-musl-cross/bin",
|
||||
ccBasename: "aarch64-linux-musl-gcc",
|
||||
cxxBasename: "aarch64-linux-musl-g++",
|
||||
arBasename: "aarch64-linux-musl-ar",
|
||||
},
|
||||
"linux-musl-armv7": {
|
||||
displayName: "musl armv7l Linux cross-compiler",
|
||||
url: "https://musl.cc/armv7l-linux-musleabihf-cross.tgz",
|
||||
approxSizeMB: 60,
|
||||
archive: "tar.gz",
|
||||
binSubdir: "armv7l-linux-musleabihf-cross/bin",
|
||||
ccBasename: "armv7l-linux-musleabihf-gcc",
|
||||
cxxBasename: "armv7l-linux-musleabihf-g++",
|
||||
arBasename: "armv7l-linux-musleabihf-ar",
|
||||
},
|
||||
"android-ndk": {
|
||||
displayName: `Android NDK ${ANDROID_NDK_VERSION}`,
|
||||
url: `https://dl.google.com/android/repository/android-ndk-${ANDROID_NDK_VERSION}-linux.zip`,
|
||||
approxSizeMB: 1500,
|
||||
archive: "zip",
|
||||
binSubdir: `android-ndk-${ANDROID_NDK_VERSION}/toolchains/llvm/prebuilt/${ndkHostTriple()}/bin`,
|
||||
arBasename: "llvm-ar",
|
||||
},
|
||||
ldid: {
|
||||
displayName: "ldid (iOS pseudo-signing)",
|
||||
url: `https://github.com/ProcursusTeam/ldid/releases/download/v2.1.5-procursus7/ldid_linux_${ldidArch()}`,
|
||||
approxSizeMB: 2,
|
||||
archive: "binary",
|
||||
binSubdir: "bin",
|
||||
binaryFilename: "ldid",
|
||||
},
|
||||
upx: {
|
||||
displayName: "UPX 4.2.4",
|
||||
url: "https://github.com/upx/upx/releases/download/v4.2.4/upx-4.2.4-amd64_linux.tar.xz",
|
||||
approxSizeMB: 1,
|
||||
archive: "tar.xz",
|
||||
binSubdir: "upx-4.2.4-amd64_linux",
|
||||
binaryFilename: "upx",
|
||||
},
|
||||
};
|
||||
|
||||
const inFlight = new Map<ToolchainKey, Promise<EnsuredToolchain>>();
|
||||
|
||||
export function getToolchainRoot(): string {
|
||||
const explicit = process.env.OVERLORD_TOOLCHAIN_DIR?.trim();
|
||||
if (explicit) return path.resolve(explicit);
|
||||
return path.join(ensureDataDir(), "toolchains");
|
||||
}
|
||||
|
||||
function toolchainDir(key: ToolchainKey): string {
|
||||
return path.join(getToolchainRoot(), key);
|
||||
}
|
||||
|
||||
function sentinelPath(key: ToolchainKey): string {
|
||||
return path.join(toolchainDir(key), ".installed");
|
||||
}
|
||||
|
||||
function isInstalled(key: ToolchainKey): boolean {
|
||||
try {
|
||||
return fs.existsSync(sentinelPath(key));
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function resolveEnsured(key: ToolchainKey): EnsuredToolchain {
|
||||
const m = MANIFESTS[key];
|
||||
const rootDir = toolchainDir(key);
|
||||
const binDir = path.join(rootDir, m.binSubdir);
|
||||
return {
|
||||
key,
|
||||
rootDir,
|
||||
binDir,
|
||||
ccPath: m.ccBasename ? path.join(binDir, m.ccBasename) : undefined,
|
||||
cxxPath: m.cxxBasename ? path.join(binDir, m.cxxBasename) : undefined,
|
||||
arPath: m.arBasename ? path.join(binDir, m.arBasename) : undefined,
|
||||
};
|
||||
}
|
||||
|
||||
async function downloadAndExtract(
|
||||
key: ToolchainKey,
|
||||
send: SendToStream,
|
||||
): Promise<void> {
|
||||
const m = MANIFESTS[key];
|
||||
const rootDir = toolchainDir(key);
|
||||
const tmpDir = `${rootDir}.tmp-${process.pid}-${Date.now()}`;
|
||||
|
||||
fs.mkdirSync(getToolchainRoot(), { recursive: true });
|
||||
fs.mkdirSync(tmpDir, { recursive: true });
|
||||
|
||||
const archiveName =
|
||||
m.archive === "binary"
|
||||
? m.binaryFilename || "download"
|
||||
: path.basename(new URL(m.url).pathname);
|
||||
const archivePath = path.join(tmpDir, archiveName);
|
||||
|
||||
send({
|
||||
type: "output",
|
||||
text: `[toolchain] Downloading ${m.displayName} (~${m.approxSizeMB}MB) from ${m.url}\n`,
|
||||
level: "info",
|
||||
});
|
||||
logger.info(`[toolchain] downloading ${key} from ${m.url}`);
|
||||
|
||||
const dl = await $`wget --tries=2 --timeout=60 -q -O ${archivePath} ${m.url}`
|
||||
.nothrow();
|
||||
if (dl.exitCode !== 0) {
|
||||
const stderr = dl.stderr.toString().trim();
|
||||
throw new Error(
|
||||
`Failed to download ${m.displayName} (wget exit ${dl.exitCode}): ${stderr || "no stderr"}`,
|
||||
);
|
||||
}
|
||||
|
||||
send({
|
||||
type: "output",
|
||||
text: `[toolchain] Extracting ${m.displayName}...\n`,
|
||||
level: "info",
|
||||
});
|
||||
|
||||
switch (m.archive) {
|
||||
case "tar.gz": {
|
||||
const r = await $`tar -xzf ${archivePath} -C ${tmpDir}`.nothrow();
|
||||
if (r.exitCode !== 0) {
|
||||
throw new Error(`tar -xzf failed for ${key}: ${r.stderr.toString()}`);
|
||||
}
|
||||
fs.unlinkSync(archivePath);
|
||||
break;
|
||||
}
|
||||
case "tar.xz": {
|
||||
const r = await $`tar -xJf ${archivePath} -C ${tmpDir}`.nothrow();
|
||||
if (r.exitCode !== 0) {
|
||||
throw new Error(`tar -xJf failed for ${key}: ${r.stderr.toString()}`);
|
||||
}
|
||||
fs.unlinkSync(archivePath);
|
||||
break;
|
||||
}
|
||||
case "zip": {
|
||||
const r = await $`unzip -q ${archivePath} -d ${tmpDir}`.nothrow();
|
||||
if (r.exitCode !== 0) {
|
||||
throw new Error(`unzip failed for ${key}: ${r.stderr.toString()}`);
|
||||
}
|
||||
fs.unlinkSync(archivePath);
|
||||
break;
|
||||
}
|
||||
case "binary": {
|
||||
// Move single-binary download into a bin/ subdir so binSubdir resolves.
|
||||
const binDir = path.join(tmpDir, m.binSubdir);
|
||||
fs.mkdirSync(binDir, { recursive: true });
|
||||
const finalBin = path.join(binDir, m.binaryFilename || "download");
|
||||
fs.renameSync(archivePath, finalBin);
|
||||
fs.chmodSync(finalBin, 0o755);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Atomic publish: rename tmpDir → rootDir. If another process already
|
||||
// created rootDir (race), fall back to checking the sentinel.
|
||||
try {
|
||||
fs.renameSync(tmpDir, rootDir);
|
||||
} catch (err: any) {
|
||||
// If another worker won the race and rootDir already exists, accept it.
|
||||
if (err.code === "ENOTEMPTY" || err.code === "EEXIST" || err.code === "EPERM") {
|
||||
try {
|
||||
fs.rmSync(tmpDir, { recursive: true, force: true });
|
||||
} catch {}
|
||||
if (isInstalled(key)) return;
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
|
||||
fs.writeFileSync(sentinelPath(key), `${m.url}\n${new Date().toISOString()}\n`);
|
||||
send({
|
||||
type: "output",
|
||||
text: `[toolchain] ${m.displayName} ready at ${rootDir}\n`,
|
||||
level: "info",
|
||||
});
|
||||
logger.info(`[toolchain] installed ${key} at ${rootDir}`);
|
||||
}
|
||||
|
||||
export async function ensureToolchain(
|
||||
key: ToolchainKey,
|
||||
send: SendToStream,
|
||||
): Promise<EnsuredToolchain> {
|
||||
if (isInstalled(key)) {
|
||||
return resolveEnsured(key);
|
||||
}
|
||||
|
||||
const existing = inFlight.get(key);
|
||||
if (existing) return existing;
|
||||
|
||||
const promise = (async () => {
|
||||
try {
|
||||
await downloadAndExtract(key, send);
|
||||
return resolveEnsured(key);
|
||||
} finally {
|
||||
inFlight.delete(key);
|
||||
}
|
||||
})();
|
||||
|
||||
inFlight.set(key, promise);
|
||||
return promise;
|
||||
}
|
||||
|
||||
const TARGET_TO_TOOLCHAIN: Record<string, ToolchainKey> = {
|
||||
"linux/amd64": "linux-musl-x64",
|
||||
"linux/arm64": "linux-musl-arm64",
|
||||
"linux/arm/v7": "linux-musl-armv7",
|
||||
"windows/amd64": "mingw-w64-x64",
|
||||
"windows/386": "mingw-w64-x86",
|
||||
};
|
||||
|
||||
export function toolchainKeyForTarget(targetKey: string): ToolchainKey | null {
|
||||
return TARGET_TO_TOOLCHAIN[targetKey] || null;
|
||||
}
|
||||
Reference in New Issue
Block a user