From d82986452789a2ffadd89f63d8f5c1e6e1d85ca0 Mon Sep 17 00:00:00 2001 From: KingKDot Date: Thu, 30 Jul 2026 16:09:24 -0500 Subject: [PATCH] hopefully fix build times --- .github/workflows/docker-publish.yml | 108 ++++++++++++++++++++--- Dockerfile | 12 ++- Overlord-Server/scripts/minify-public.ts | 54 ++++++++---- Overlord-Server/scripts/vendor.ts | 5 ++ 4 files changed, 150 insertions(+), 29 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index a9f2d33..cad34c2 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -59,7 +59,19 @@ jobs: build-and-push: needs: build-backstage-dll - runs-on: ubuntu-latest + name: Build and push (${{ matrix.platform }}) + runs-on: ${{ matrix.runner }} + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + include: + - platform: linux/amd64 + runner: ubuntu-latest + artifact: amd64 + - platform: linux/arm64 + runner: ubuntu-24.04-arm + artifact: arm64 permissions: contents: read packages: write @@ -89,7 +101,7 @@ jobs: exit 1 fi - if [[ "$client_version" != "$server_version" || "$client_version" != "$package_version" ]]; then + if ! { [[ "$client_version" == "$server_version" ]] && [[ "$client_version" == "$package_version" ]]; }; then echo "Version mismatch detected:" >&2 echo " client(config.go): $client_version" >&2 echo " server(version.ts): $server_version" >&2 @@ -101,9 +113,11 @@ jobs: rest="${client_version#*.}" minor="${rest%%.*}" - echo "version=$client_version" >> "$GITHUB_OUTPUT" - echo "major=$major" >> "$GITHUB_OUTPUT" - echo "minor=$minor" >> "$GITHUB_OUTPUT" + { + echo "version=$client_version" + echo "major=$major" + echo "minor=$minor" + } >> "$GITHUB_OUTPUT" - name: Validate Release Version Matches Source Version shell: bash @@ -144,13 +158,87 @@ jobs: type=raw,value=latest - name: Build and push server image + id: build uses: docker/build-push-action@ca052bb54ab0790a636c9b5f226502c73d547a25 # v5 with: context: . file: ./Dockerfile - push: true - tags: ${{ steps.meta.outputs.tags }} + platforms: ${{ matrix.platform }} labels: ${{ steps.meta.outputs.labels }} - cache-from: type=gha - cache-to: type=gha,mode=max - platforms: linux/amd64,linux/arm64 + cache-from: type=gha,scope=server-${{ matrix.artifact }} + cache-to: type=gha,mode=max,scope=server-${{ matrix.artifact }} + outputs: type=image,name=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }},push-by-digest=true,name-canonical=true,push=true + + - name: Export image digest + shell: bash + run: | + set -euo pipefail + mkdir -p /tmp/digests + digest="${{ steps.build.outputs.digest }}" + test -n "$digest" + touch "/tmp/digests/${digest#sha256:}" + + - name: Upload image digest + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: image-digest-${{ matrix.artifact }} + path: /tmp/digests/* + if-no-files-found: error + retention-days: 1 + + merge-and-publish: + name: Merge and publish multi-architecture image + needs: build-and-push + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + packages: write + + steps: + - name: Normalize image name + run: printf 'IMAGE_NAME=%s\n' "${GITHUB_REPOSITORY,,}" >> "$GITHUB_ENV" + + - name: Download image digests + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + pattern: image-digest-* + path: /tmp/digests + merge-multiple: true + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract release tags + id: meta + uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=semver,pattern={{version}},value=${{ env.RELEASE_VERSION }} + type=semver,pattern={{major}}.{{minor}},value=${{ env.RELEASE_VERSION }} + type=semver,pattern={{major}},value=${{ env.RELEASE_VERSION }} + type=raw,value=latest + + - name: Create and push multi-architecture manifest + working-directory: /tmp/digests + shell: bash + run: | + set -euo pipefail + test "$(find . -maxdepth 1 -type f | wc -l)" -eq 2 + mapfile -t tag_args < <(jq -r '.tags[] | "-t", .' <<< "$DOCKER_METADATA_OUTPUT_JSON") + digest_args=() + for digest_file in *; do + digest_args+=("${REGISTRY}/${IMAGE_NAME}@sha256:${digest_file}") + done + docker buildx imagetools create "${tag_args[@]}" "${digest_args[@]}" + + - name: Verify published manifest + run: docker buildx imagetools inspect "${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${RELEASE_VERSION}" diff --git a/Dockerfile b/Dockerfile index 9c28405..c105fe8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -110,10 +110,14 @@ RUN if [ -f dist-clients/BackstageCapture.x64.dll ]; then \ echo "WARNING: BackstageCapture DLL not available (build with MSVC on Windows)"; \ fi -# Tailwind CSS, vendored frontend assets, minified public assets, and bundled Bun server runtime. -RUN bun run build:public:prod \ - && bun run build:bundle \ - && test "$(wc -l < ./public/index.html)" -lt 20 \ +# Keep production build phases separate so BuildKit reports the exact slow or +# failing phase and can cache each completed phase independently. +RUN bun run build:css +RUN bun run vendor +RUN MINIFY_CONCURRENCY=4 bun run minify +RUN bun run build:bundle + +RUN test "$(wc -l < ./public/index.html)" -lt 20 \ && test "$(wc -l < ./public/assets/main.js)" -lt 50 \ && test -s ./public/assets/tailwind.css \ && test -d ./public/vendor/fontawesome \ diff --git a/Overlord-Server/scripts/minify-public.ts b/Overlord-Server/scripts/minify-public.ts index 7032a1b..3db107c 100644 --- a/Overlord-Server/scripts/minify-public.ts +++ b/Overlord-Server/scripts/minify-public.ts @@ -34,6 +34,10 @@ const htmlOpts = { type Stats = { js: number; css: number; html: number; savedBytes: number }; const stats: Stats = { js: 0, css: 0, html: 0, savedBytes: 0 }; let failures = 0; +const requestedConcurrency = Number.parseInt(process.env.MINIFY_CONCURRENCY || "", 10); +const minifyConcurrency = Number.isFinite(requestedConcurrency) && requestedConcurrency > 0 + ? requestedConcurrency + : 4; async function collectFiles(dir: string, exts: Set): Promise { const results: string[] = []; @@ -86,28 +90,48 @@ async function minifyHTML(filePath: string) { stats.html++; } +async function runPool( + items: T[], + concurrency: number, + worker: (item: T) => Promise, +): Promise { + let nextIndex = 0; + const workers = Array.from( + { length: Math.min(concurrency, Math.max(1, items.length)) }, + async () => { + while (nextIndex < items.length) { + const item = items[nextIndex++]; + await worker(item); + } + }, + ); + await Promise.all(workers); +} + // Collect all files const jsFiles = await collectFiles(path.join(publicDir, "assets"), new Set([".js"])); const cssFiles = await collectFiles(path.join(publicDir, "assets"), new Set([".css"])); const htmlFiles = await collectFiles(publicDir, new Set([".html"])); -console.log(`Minifying ${jsFiles.length} JS, ${cssFiles.length} CSS, ${htmlFiles.length} HTML files...`); +console.log( + `Minifying ${jsFiles.length} JS, ${cssFiles.length} CSS, ${htmlFiles.length} HTML files ` + + `(concurrency=${minifyConcurrency})...`, +); -// Process in parallel batches -await Promise.all([ - ...jsFiles.map((f) => minifyJS(f).catch((e) => { +const jobs = [ + ...jsFiles.map((file) => ({ kind: "JS", file, minify: minifyJS })), + ...cssFiles.map((file) => ({ kind: "CSS", file, minify: minifyCSS })), + ...htmlFiles.map((file) => ({ kind: "HTML", file, minify: minifyHTML })), +]; +await runPool(jobs, minifyConcurrency, async ({ kind, file, minify }) => { + try { + await minify(file); + } catch (e) { failures++; - console.error(`JS error ${f}: ${e.message}`); - })), - ...cssFiles.map((f) => minifyCSS(f).catch((e) => { - failures++; - console.error(`CSS error ${f}: ${e.message}`); - })), - ...htmlFiles.map((f) => minifyHTML(f).catch((e) => { - failures++; - console.error(`HTML error ${f}: ${e.message}`); - })), -]); + const message = e instanceof Error ? e.message : String(e); + console.error(`${kind} error ${file}: ${message}`); + } +}); console.log( `Done: ${stats.js} JS, ${stats.css} CSS, ${stats.html} HTML — saved ${(stats.savedBytes / 1024).toFixed(1)} KB`, diff --git a/Overlord-Server/scripts/vendor.ts b/Overlord-Server/scripts/vendor.ts index 9195c17..7930d05 100644 --- a/Overlord-Server/scripts/vendor.ts +++ b/Overlord-Server/scripts/vendor.ts @@ -361,15 +361,20 @@ rmSync(ansiTmp, { force: true }); console.log("Downloading countries GeoJSON ..."); const geojsonDest = path.join(VENDOR, "geo-countries", "countries.geojson"); ensureDir(path.dirname(geojsonDest)); +const geojsonController = new AbortController(); +const geojsonTimeout = setTimeout(() => geojsonController.abort(), 15_000); try { const resp = await fetch( "https://cdn.jsdelivr.net/gh/datasets/geo-countries@master/data/countries.geojson", + { signal: geojsonController.signal }, ); if (!resp.ok) throw new Error(`HTTP ${resp.status}`); await Bun.write(geojsonDest, resp); console.log(" GeoJSON saved (" + ((await Bun.file(geojsonDest).size) / 1024 / 1024).toFixed(1) + " MB)"); } catch (err) { console.warn(" WARNING: Could not download GeoJSON. Map features may not work offline.", err); +} finally { + clearTimeout(geojsonTimeout); } /* ── done ─────────────────────────────────────────────────────────── */