massive shrink on docker
This commit is contained in:
KingKDot
@@ -10,6 +10,11 @@ import * as buildManager from "../build/buildManager";
|
||||
import type { BuildStream } from "../build/types";
|
||||
import { ALLOWED_PLATFORMS } from "./validation-constants";
|
||||
import { resolveRuntimeRoot } from "./runtime-paths";
|
||||
import {
|
||||
ensureToolchain,
|
||||
toolchainKeyForTarget,
|
||||
type EnsuredToolchain,
|
||||
} from "./toolchain-manager";
|
||||
|
||||
function isClientModuleDir(dir: string): boolean {
|
||||
return (
|
||||
@@ -45,12 +50,11 @@ function resolveClientBuildCacheRoot(): string {
|
||||
return path.resolve(ensureDataDir(), "client-build-cache");
|
||||
}
|
||||
|
||||
function resolveAndroidNdkToolchainBin(): string | null {
|
||||
function resolveExplicitAndroidNdkToolchainBin(): string | null {
|
||||
const explicit = process.env.ANDROID_NDK_HOME?.trim();
|
||||
const ndkHome = explicit || (process.platform === "win32" ? "" : "/opt/android-ndk");
|
||||
if (!ndkHome) return null;
|
||||
if (!explicit) return null;
|
||||
const hostArch = process.arch === "arm64" ? "linux-aarch64" : "linux-x86_64";
|
||||
const toolchainBin = path.join(ndkHome, "toolchains", "llvm", "prebuilt", hostArch, "bin");
|
||||
const toolchainBin = path.join(explicit, "toolchains", "llvm", "prebuilt", hostArch, "bin");
|
||||
try {
|
||||
return fs.existsSync(toolchainBin) ? toolchainBin : null;
|
||||
} catch {
|
||||
@@ -297,13 +301,21 @@ export async function startBuildProcess(
|
||||
});
|
||||
}
|
||||
|
||||
const ndkBin = hasAndroidTargets ? resolveAndroidNdkToolchainBin() : null;
|
||||
if (hasAndroidTargets && !ndkBin) {
|
||||
sendToStream({
|
||||
type: "output",
|
||||
text: "Warning: Android NDK not found. Android builds require the NDK. Install it to /opt/android-ndk or set the ANDROID_NDK_HOME environment variable.\n",
|
||||
level: "warn",
|
||||
});
|
||||
let ndkBin: string | null = null;
|
||||
if (hasAndroidTargets) {
|
||||
ndkBin = resolveExplicitAndroidNdkToolchainBin();
|
||||
if (!ndkBin) {
|
||||
try {
|
||||
const ndk = await ensureToolchain("android-ndk", sendToStream);
|
||||
ndkBin = ndk.binDir;
|
||||
} catch (err: any) {
|
||||
sendToStream({
|
||||
type: "output",
|
||||
text: `Warning: Android NDK download failed (${err.message || err}). Android builds require the NDK. Set ANDROID_NDK_HOME to use a pre-installed NDK.\n`,
|
||||
level: "warn",
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let buildMutex = "";
|
||||
@@ -323,16 +335,21 @@ export async function startBuildProcess(
|
||||
|
||||
let upxBin: string | null = null;
|
||||
if (config.enableUpx) {
|
||||
const upxFound = await checkUpxAvailable(sendToStream);
|
||||
if (!upxFound) {
|
||||
sendToStream({
|
||||
type: "output",
|
||||
text: "ERROR: UPX is not installed or not found in PATH. Please install UPX (https://upx.github.io) and ensure it is available on PATH, then retry.\n",
|
||||
level: "error",
|
||||
});
|
||||
throw new Error("UPX not found");
|
||||
if (await checkUpxAvailable(sendToStream)) {
|
||||
upxBin = "upx";
|
||||
} else {
|
||||
try {
|
||||
const upxTool = await ensureToolchain("upx", sendToStream);
|
||||
upxBin = path.join(upxTool.binDir, "upx");
|
||||
} catch (err: any) {
|
||||
sendToStream({
|
||||
type: "output",
|
||||
text: `ERROR: UPX is not installed and on-demand download failed (${err.message || err}).\n`,
|
||||
level: "error",
|
||||
});
|
||||
throw new Error("UPX not found");
|
||||
}
|
||||
}
|
||||
upxBin = "upx";
|
||||
}
|
||||
|
||||
const hasAssemblyData = !!(config.assemblyTitle || config.assemblyProduct || config.assemblyCompany || config.assemblyVersion || config.assemblyCopyright || config.iconBase64 || config.requireAdmin);
|
||||
@@ -668,33 +685,43 @@ func runBoundFiles() {
|
||||
}
|
||||
|
||||
if (env.CGO_ENABLED === "1") {
|
||||
const cCompilerByTarget: Record<string, string> = {
|
||||
"linux/amd64": "musl-gcc",
|
||||
"linux/arm64": "aarch64-linux-gnu-gcc",
|
||||
"linux/arm/v7": "arm-linux-gnueabihf-gcc",
|
||||
"windows/amd64": "x86_64-w64-mingw32-gcc",
|
||||
"windows/386": "i686-w64-mingw32-gcc",
|
||||
...(ndkBin ? {
|
||||
"android/amd64": path.join(ndkBin, "x86_64-linux-android21-clang"),
|
||||
"android/arm64": path.join(ndkBin, "aarch64-linux-android21-clang"),
|
||||
"android/arm/v7": path.join(ndkBin, "armv7a-linux-androideabi21-clang"),
|
||||
} : {}),
|
||||
};
|
||||
const cxxCompilerByTarget: Record<string, string> = {
|
||||
"linux/amd64": "g++",
|
||||
"linux/arm64": "aarch64-linux-gnu-g++",
|
||||
"linux/arm/v7": "arm-linux-gnueabihf-g++",
|
||||
"windows/amd64": "x86_64-w64-mingw32-g++",
|
||||
"windows/386": "i686-w64-mingw32-g++",
|
||||
...(ndkBin ? {
|
||||
"android/amd64": path.join(ndkBin, "x86_64-linux-android21-clang++"),
|
||||
"android/arm64": path.join(ndkBin, "aarch64-linux-android21-clang++"),
|
||||
"android/arm/v7": path.join(ndkBin, "armv7a-linux-androideabi21-clang++"),
|
||||
} : {}),
|
||||
};
|
||||
let cc: string | undefined;
|
||||
let cxx: string | undefined;
|
||||
let extraBinDir: string | undefined;
|
||||
|
||||
if (os === "android" && ndkBin) {
|
||||
const ndkCcByTarget: Record<string, string> = {
|
||||
"android/amd64": "x86_64-linux-android21-clang",
|
||||
"android/arm64": "aarch64-linux-android21-clang",
|
||||
"android/arm/v7": "armv7a-linux-androideabi21-clang",
|
||||
};
|
||||
const basename = ndkCcByTarget[targetKey];
|
||||
if (basename) {
|
||||
cc = path.join(ndkBin, basename);
|
||||
cxx = path.join(ndkBin, `${basename}++`);
|
||||
extraBinDir = ndkBin;
|
||||
env.AR = path.join(ndkBin, "llvm-ar");
|
||||
}
|
||||
} else {
|
||||
const tcKey = toolchainKeyForTarget(targetKey);
|
||||
if (tcKey) {
|
||||
try {
|
||||
const tc: EnsuredToolchain = await ensureToolchain(tcKey, sendToStream);
|
||||
cc = tc.ccPath;
|
||||
cxx = tc.cxxPath;
|
||||
extraBinDir = tc.binDir;
|
||||
if (tc.arPath) env.AR = tc.arPath;
|
||||
} catch (err: any) {
|
||||
sendToStream({
|
||||
type: "output",
|
||||
text: `ERROR: Failed to provision cross-compile toolchain for ${targetKey}: ${err.message || err}\n`,
|
||||
level: "error",
|
||||
});
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const cc = cCompilerByTarget[targetKey];
|
||||
const cxx = cxxCompilerByTarget[targetKey];
|
||||
if (cc) {
|
||||
env.CC = cc;
|
||||
sendToStream({ type: "output", text: `CGO compiler: ${cc}\n`, level: "info" });
|
||||
@@ -708,8 +735,9 @@ func runBoundFiles() {
|
||||
if (cxx) {
|
||||
env.CXX = cxx;
|
||||
}
|
||||
if (os === "android" && ndkBin) {
|
||||
env.AR = path.join(ndkBin, "llvm-ar");
|
||||
if (extraBinDir) {
|
||||
const sep = process.platform === "win32" ? ";" : ":";
|
||||
env.PATH = `${extraBinDir}${sep}${env.PATH || process.env.PATH || ""}`;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1017,18 +1045,31 @@ func runBoundFiles() {
|
||||
</plist>`;
|
||||
fs.writeFileSync(path.join(payloadAppDir, "Info.plist"), infoPlist, "utf8");
|
||||
|
||||
// Attempt pseudo-signing with ldid if available
|
||||
// Attempt pseudo-signing with ldid (system-installed or downloaded on demand).
|
||||
try {
|
||||
const ldidResult = await $`which ldid`.nothrow().quiet();
|
||||
if (ldidResult.exitCode === 0) {
|
||||
const ldidSign = await $`ldid -S ${path.join(payloadAppDir, appName)}`.nothrow().quiet();
|
||||
let ldidBin: string | null = null;
|
||||
const sysLdid = await $`which ldid`.nothrow().quiet();
|
||||
if (sysLdid.exitCode === 0) {
|
||||
ldidBin = "ldid";
|
||||
} else {
|
||||
try {
|
||||
const tc = await ensureToolchain("ldid", sendToStream);
|
||||
ldidBin = path.join(tc.binDir, "ldid");
|
||||
} catch (dlErr: any) {
|
||||
sendToStream({
|
||||
type: "output",
|
||||
text: `ldid not found and download failed (${dlErr.message || dlErr}); skipping pseudo-signing.\n`,
|
||||
level: "warn",
|
||||
});
|
||||
}
|
||||
}
|
||||
if (ldidBin) {
|
||||
const ldidSign = await $`${ldidBin} -S ${path.join(payloadAppDir, appName)}`.nothrow().quiet();
|
||||
if (ldidSign.exitCode === 0) {
|
||||
sendToStream({ type: "output", text: `Pseudo-signed with ldid\n`, level: "info" });
|
||||
} else {
|
||||
sendToStream({ type: "output", text: `WARNING: ldid signing failed (non-fatal)\n`, level: "warn" });
|
||||
}
|
||||
} else {
|
||||
sendToStream({ type: "output", text: `ldid not found; skipping pseudo-signing (install ldid for TrollStore/sideload compatibility)\n`, level: "warn" });
|
||||
}
|
||||
} catch { /* ldid is optional */ }
|
||||
|
||||
|
||||
@@ -0,0 +1,300 @@
|
||||
import path from "path";
|
||||
import fs from "fs";
|
||||
import { $ } from "bun";
|
||||
import { ensureDataDir } from "../paths";
|
||||
import { logger } from "../logger";
|
||||
|
||||
type SendToStream = (data: any) => void;
|
||||
|
||||
export type ToolchainKey =
|
||||
| "mingw-w64-x64"
|
||||
| "mingw-w64-x86"
|
||||
| "linux-musl-x64"
|
||||
| "linux-musl-arm64"
|
||||
| "linux-musl-armv7"
|
||||
| "android-ndk"
|
||||
| "ldid"
|
||||
| "upx";
|
||||
|
||||
export type EnsuredToolchain = {
|
||||
key: ToolchainKey;
|
||||
binDir: string;
|
||||
ccPath?: string;
|
||||
cxxPath?: string;
|
||||
arPath?: string;
|
||||
rootDir: string;
|
||||
};
|
||||
|
||||
type ArchiveFormat = "tar.gz" | "tar.xz" | "zip" | "binary";
|
||||
|
||||
type Manifest = {
|
||||
displayName: string;
|
||||
url: string;
|
||||
approxSizeMB: number;
|
||||
archive: ArchiveFormat;
|
||||
binSubdir: string;
|
||||
ccBasename?: string;
|
||||
cxxBasename?: string;
|
||||
arBasename?: string;
|
||||
binaryFilename?: string;
|
||||
};
|
||||
|
||||
const ANDROID_NDK_VERSION = "r27c";
|
||||
|
||||
function ndkHostTriple(): string {
|
||||
return process.arch === "arm64" ? "linux-aarch64" : "linux-x86_64";
|
||||
}
|
||||
|
||||
function ldidArch(): string {
|
||||
return process.arch === "arm64" ? "aarch64" : "x86_64";
|
||||
}
|
||||
|
||||
const MANIFESTS: Record<ToolchainKey, Manifest> = {
|
||||
"mingw-w64-x64": {
|
||||
displayName: "mingw-w64 x86_64 cross-compiler",
|
||||
url: "https://musl.cc/x86_64-w64-mingw32-cross.tgz",
|
||||
approxSizeMB: 70,
|
||||
archive: "tar.gz",
|
||||
binSubdir: "x86_64-w64-mingw32-cross/bin",
|
||||
ccBasename: "x86_64-w64-mingw32-gcc",
|
||||
cxxBasename: "x86_64-w64-mingw32-g++",
|
||||
arBasename: "x86_64-w64-mingw32-ar",
|
||||
},
|
||||
"mingw-w64-x86": {
|
||||
displayName: "mingw-w64 i686 cross-compiler",
|
||||
url: "https://musl.cc/i686-w64-mingw32-cross.tgz",
|
||||
approxSizeMB: 65,
|
||||
archive: "tar.gz",
|
||||
binSubdir: "i686-w64-mingw32-cross/bin",
|
||||
ccBasename: "i686-w64-mingw32-gcc",
|
||||
cxxBasename: "i686-w64-mingw32-g++",
|
||||
arBasename: "i686-w64-mingw32-ar",
|
||||
},
|
||||
"linux-musl-x64": {
|
||||
displayName: "musl x86_64 Linux cross-compiler",
|
||||
url: "https://musl.cc/x86_64-linux-musl-cross.tgz",
|
||||
approxSizeMB: 60,
|
||||
archive: "tar.gz",
|
||||
binSubdir: "x86_64-linux-musl-cross/bin",
|
||||
ccBasename: "x86_64-linux-musl-gcc",
|
||||
cxxBasename: "x86_64-linux-musl-g++",
|
||||
arBasename: "x86_64-linux-musl-ar",
|
||||
},
|
||||
"linux-musl-arm64": {
|
||||
displayName: "musl aarch64 Linux cross-compiler",
|
||||
url: "https://musl.cc/aarch64-linux-musl-cross.tgz",
|
||||
approxSizeMB: 60,
|
||||
archive: "tar.gz",
|
||||
binSubdir: "aarch64-linux-musl-cross/bin",
|
||||
ccBasename: "aarch64-linux-musl-gcc",
|
||||
cxxBasename: "aarch64-linux-musl-g++",
|
||||
arBasename: "aarch64-linux-musl-ar",
|
||||
},
|
||||
"linux-musl-armv7": {
|
||||
displayName: "musl armv7l Linux cross-compiler",
|
||||
url: "https://musl.cc/armv7l-linux-musleabihf-cross.tgz",
|
||||
approxSizeMB: 60,
|
||||
archive: "tar.gz",
|
||||
binSubdir: "armv7l-linux-musleabihf-cross/bin",
|
||||
ccBasename: "armv7l-linux-musleabihf-gcc",
|
||||
cxxBasename: "armv7l-linux-musleabihf-g++",
|
||||
arBasename: "armv7l-linux-musleabihf-ar",
|
||||
},
|
||||
"android-ndk": {
|
||||
displayName: `Android NDK ${ANDROID_NDK_VERSION}`,
|
||||
url: `https://dl.google.com/android/repository/android-ndk-${ANDROID_NDK_VERSION}-linux.zip`,
|
||||
approxSizeMB: 1500,
|
||||
archive: "zip",
|
||||
binSubdir: `android-ndk-${ANDROID_NDK_VERSION}/toolchains/llvm/prebuilt/${ndkHostTriple()}/bin`,
|
||||
arBasename: "llvm-ar",
|
||||
},
|
||||
ldid: {
|
||||
displayName: "ldid (iOS pseudo-signing)",
|
||||
url: `https://github.com/ProcursusTeam/ldid/releases/download/v2.1.5-procursus7/ldid_linux_${ldidArch()}`,
|
||||
approxSizeMB: 2,
|
||||
archive: "binary",
|
||||
binSubdir: "bin",
|
||||
binaryFilename: "ldid",
|
||||
},
|
||||
upx: {
|
||||
displayName: "UPX 4.2.4",
|
||||
url: "https://github.com/upx/upx/releases/download/v4.2.4/upx-4.2.4-amd64_linux.tar.xz",
|
||||
approxSizeMB: 1,
|
||||
archive: "tar.xz",
|
||||
binSubdir: "upx-4.2.4-amd64_linux",
|
||||
binaryFilename: "upx",
|
||||
},
|
||||
};
|
||||
|
||||
const inFlight = new Map<ToolchainKey, Promise<EnsuredToolchain>>();
|
||||
|
||||
export function getToolchainRoot(): string {
|
||||
const explicit = process.env.OVERLORD_TOOLCHAIN_DIR?.trim();
|
||||
if (explicit) return path.resolve(explicit);
|
||||
return path.join(ensureDataDir(), "toolchains");
|
||||
}
|
||||
|
||||
function toolchainDir(key: ToolchainKey): string {
|
||||
return path.join(getToolchainRoot(), key);
|
||||
}
|
||||
|
||||
function sentinelPath(key: ToolchainKey): string {
|
||||
return path.join(toolchainDir(key), ".installed");
|
||||
}
|
||||
|
||||
function isInstalled(key: ToolchainKey): boolean {
|
||||
try {
|
||||
return fs.existsSync(sentinelPath(key));
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function resolveEnsured(key: ToolchainKey): EnsuredToolchain {
|
||||
const m = MANIFESTS[key];
|
||||
const rootDir = toolchainDir(key);
|
||||
const binDir = path.join(rootDir, m.binSubdir);
|
||||
return {
|
||||
key,
|
||||
rootDir,
|
||||
binDir,
|
||||
ccPath: m.ccBasename ? path.join(binDir, m.ccBasename) : undefined,
|
||||
cxxPath: m.cxxBasename ? path.join(binDir, m.cxxBasename) : undefined,
|
||||
arPath: m.arBasename ? path.join(binDir, m.arBasename) : undefined,
|
||||
};
|
||||
}
|
||||
|
||||
async function downloadAndExtract(
|
||||
key: ToolchainKey,
|
||||
send: SendToStream,
|
||||
): Promise<void> {
|
||||
const m = MANIFESTS[key];
|
||||
const rootDir = toolchainDir(key);
|
||||
const tmpDir = `${rootDir}.tmp-${process.pid}-${Date.now()}`;
|
||||
|
||||
fs.mkdirSync(getToolchainRoot(), { recursive: true });
|
||||
fs.mkdirSync(tmpDir, { recursive: true });
|
||||
|
||||
const archiveName =
|
||||
m.archive === "binary"
|
||||
? m.binaryFilename || "download"
|
||||
: path.basename(new URL(m.url).pathname);
|
||||
const archivePath = path.join(tmpDir, archiveName);
|
||||
|
||||
send({
|
||||
type: "output",
|
||||
text: `[toolchain] Downloading ${m.displayName} (~${m.approxSizeMB}MB) from ${m.url}\n`,
|
||||
level: "info",
|
||||
});
|
||||
logger.info(`[toolchain] downloading ${key} from ${m.url}`);
|
||||
|
||||
const dl = await $`wget --tries=2 --timeout=60 -q -O ${archivePath} ${m.url}`
|
||||
.nothrow();
|
||||
if (dl.exitCode !== 0) {
|
||||
const stderr = dl.stderr.toString().trim();
|
||||
throw new Error(
|
||||
`Failed to download ${m.displayName} (wget exit ${dl.exitCode}): ${stderr || "no stderr"}`,
|
||||
);
|
||||
}
|
||||
|
||||
send({
|
||||
type: "output",
|
||||
text: `[toolchain] Extracting ${m.displayName}...\n`,
|
||||
level: "info",
|
||||
});
|
||||
|
||||
switch (m.archive) {
|
||||
case "tar.gz": {
|
||||
const r = await $`tar -xzf ${archivePath} -C ${tmpDir}`.nothrow();
|
||||
if (r.exitCode !== 0) {
|
||||
throw new Error(`tar -xzf failed for ${key}: ${r.stderr.toString()}`);
|
||||
}
|
||||
fs.unlinkSync(archivePath);
|
||||
break;
|
||||
}
|
||||
case "tar.xz": {
|
||||
const r = await $`tar -xJf ${archivePath} -C ${tmpDir}`.nothrow();
|
||||
if (r.exitCode !== 0) {
|
||||
throw new Error(`tar -xJf failed for ${key}: ${r.stderr.toString()}`);
|
||||
}
|
||||
fs.unlinkSync(archivePath);
|
||||
break;
|
||||
}
|
||||
case "zip": {
|
||||
const r = await $`unzip -q ${archivePath} -d ${tmpDir}`.nothrow();
|
||||
if (r.exitCode !== 0) {
|
||||
throw new Error(`unzip failed for ${key}: ${r.stderr.toString()}`);
|
||||
}
|
||||
fs.unlinkSync(archivePath);
|
||||
break;
|
||||
}
|
||||
case "binary": {
|
||||
// Move single-binary download into a bin/ subdir so binSubdir resolves.
|
||||
const binDir = path.join(tmpDir, m.binSubdir);
|
||||
fs.mkdirSync(binDir, { recursive: true });
|
||||
const finalBin = path.join(binDir, m.binaryFilename || "download");
|
||||
fs.renameSync(archivePath, finalBin);
|
||||
fs.chmodSync(finalBin, 0o755);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Atomic publish: rename tmpDir → rootDir. If another process already
|
||||
// created rootDir (race), fall back to checking the sentinel.
|
||||
try {
|
||||
fs.renameSync(tmpDir, rootDir);
|
||||
} catch (err: any) {
|
||||
// If another worker won the race and rootDir already exists, accept it.
|
||||
if (err.code === "ENOTEMPTY" || err.code === "EEXIST" || err.code === "EPERM") {
|
||||
try {
|
||||
fs.rmSync(tmpDir, { recursive: true, force: true });
|
||||
} catch {}
|
||||
if (isInstalled(key)) return;
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
|
||||
fs.writeFileSync(sentinelPath(key), `${m.url}\n${new Date().toISOString()}\n`);
|
||||
send({
|
||||
type: "output",
|
||||
text: `[toolchain] ${m.displayName} ready at ${rootDir}\n`,
|
||||
level: "info",
|
||||
});
|
||||
logger.info(`[toolchain] installed ${key} at ${rootDir}`);
|
||||
}
|
||||
|
||||
export async function ensureToolchain(
|
||||
key: ToolchainKey,
|
||||
send: SendToStream,
|
||||
): Promise<EnsuredToolchain> {
|
||||
if (isInstalled(key)) {
|
||||
return resolveEnsured(key);
|
||||
}
|
||||
|
||||
const existing = inFlight.get(key);
|
||||
if (existing) return existing;
|
||||
|
||||
const promise = (async () => {
|
||||
try {
|
||||
await downloadAndExtract(key, send);
|
||||
return resolveEnsured(key);
|
||||
} finally {
|
||||
inFlight.delete(key);
|
||||
}
|
||||
})();
|
||||
|
||||
inFlight.set(key, promise);
|
||||
return promise;
|
||||
}
|
||||
|
||||
const TARGET_TO_TOOLCHAIN: Record<string, ToolchainKey> = {
|
||||
"linux/amd64": "linux-musl-x64",
|
||||
"linux/arm64": "linux-musl-arm64",
|
||||
"linux/arm/v7": "linux-musl-armv7",
|
||||
"windows/amd64": "mingw-w64-x64",
|
||||
"windows/386": "mingw-w64-x86",
|
||||
};
|
||||
|
||||
export function toolchainKeyForTarget(targetKey: string): ToolchainKey | null {
|
||||
return TARGET_TO_TOOLCHAIN[targetKey] || null;
|
||||
}
|
||||
Reference in New Issue
Block a user