diff --git a/.gitignore b/.gitignore index 31a2f0b..864308c 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,7 @@ config.json # Docker .env +.docker-cache/ # Go bin/ diff --git a/Dockerfile b/Dockerfile index d8f250f..6d32b3a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,3 +1,4 @@ +# syntax=docker/dockerfile:1.7 # Overlord Server Dockerfile FROM oven/bun:1 AS base WORKDIR /app @@ -30,9 +31,13 @@ RUN case "${TARGETARCH}" in \ ENV PATH="/usr/local/go/bin:${PATH}" ENV GOPATH="/go" ENV PATH="${GOPATH}/bin:${PATH}" +ENV GOCACHE=/root/.cache/go-build +ENV GOMODCACHE=/go/pkg/mod # Install garble for obfuscated agent builds (requires Go 1.25+) -RUN go install mvdan.cc/garble@latest +RUN --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg/mod \ + go install mvdan.cc/garble@latest # Copy package files and lockfile COPY Overlord-Server/package.json Overlord-Server/bun.lock* ./ diff --git a/Overlord-Server/public/assets/filebrowser.js b/Overlord-Server/public/assets/filebrowser.js index 593dd8f..af23cc0 100644 --- a/Overlord-Server/public/assets/filebrowser.js +++ b/Overlord-Server/public/assets/filebrowser.js @@ -1630,21 +1630,24 @@ async function uploadFileViaHttpPull(file, path, transfer) { console.debug("[filebrowser] upload stage url", { uploadUrl }); + const uploadRequestOptions = { + headers: { "Content-Type": "application/octet-stream" }, + credentials: "include", + body: file, + signal: transfer.abortController?.signal, + }; + let uploadRes; try { uploadRes = await fetch(uploadUrl, { - method: "PUT", - headers: { "Content-Type": "application/octet-stream" }, - credentials: "include", - body: file, + method: "POST", + ...uploadRequestOptions, }); } catch (err) { - console.warn("[filebrowser] upload stage PUT failed, retrying as POST", err); + console.warn("[filebrowser] upload stage POST failed, retrying as PUT", err); uploadRes = await fetch(uploadUrl, { - method: "POST", - headers: { "Content-Type": "application/octet-stream" }, - credentials: "include", - body: file, + method: "PUT", + ...uploadRequestOptions, }); } @@ -1721,6 +1724,7 @@ async function uploadFile(file) { ackedOffsets: new Set(), transferId, completed: false, + abortController: new AbortController(), }; fileUploads.set(path, transfer); diff --git a/Overlord-Server/src/server/build-process.ts b/Overlord-Server/src/server/build-process.ts index 105bc30..da99410 100644 --- a/Overlord-Server/src/server/build-process.ts +++ b/Overlord-Server/src/server/build-process.ts @@ -4,6 +4,7 @@ import fs from "fs"; import { saveBuild } from "../db"; import { logger } from "../logger"; import { getConfig } from "../config"; +import { ensureDataDir } from "../paths"; import * as buildManager from "../build/buildManager"; import type { BuildStream } from "../build/types"; import { ALLOWED_PLATFORMS } from "./validation-constants"; @@ -33,6 +34,16 @@ function resolveClientModuleDir(rootDir: string): string | null { return null; } +function resolveClientBuildCacheRoot(): string { + const explicit = process.env.OVERLORD_CLIENT_BUILD_CACHE_DIR?.trim(); + if (explicit) { + return path.resolve(explicit); + } + + // Keep UI build caches under persistent app data by default. + return path.resolve(ensureDataDir(), "client-build-cache"); +} + type BuildProcessConfig = { platforms: string[]; serverUrl?: string; @@ -133,10 +144,16 @@ export async function startBuildProcess( ); } const outDir = path.join(rootDir, "dist-clients"); + const cacheRoot = resolveClientBuildCacheRoot(); + const goBuildCacheDir = path.join(cacheRoot, "go-build"); + const goModCacheDir = path.join(cacheRoot, "go-mod"); await Bun.$`mkdir -p ${outDir}`.quiet(); + fs.mkdirSync(goBuildCacheDir, { recursive: true }); + fs.mkdirSync(goModCacheDir, { recursive: true }); sendToStream({ type: "output", text: `Build directory: ${outDir}\n`, level: "info" }); sendToStream({ type: "output", text: `Client source: ${clientDir}\n`, level: "info" }); + sendToStream({ type: "output", text: `Client build cache: ${cacheRoot}\n`, level: "info" }); const platformsToBuild = (config.platforms || []).filter((p) => ALLOWED_PLATFORMS.has(p)); if (platformsToBuild.length !== (config.platforms || []).length) { @@ -168,6 +185,8 @@ export async function startBuildProcess( GOARCH: actualArch, CGO_ENABLED: config.disableCgo === true ? "0" : "1", GOWORK: "off", + GOCACHE: goBuildCacheDir, + GOMODCACHE: goModCacheDir, ...(goarm ? { GOARM: goarm } : {}), }; diff --git a/Overlord-Server/src/server/routes/file-download-routes.ts b/Overlord-Server/src/server/routes/file-download-routes.ts index 1f610d0..f6a10b8 100644 --- a/Overlord-Server/src/server/routes/file-download-routes.ts +++ b/Overlord-Server/src/server/routes/file-download-routes.ts @@ -118,6 +118,24 @@ const UUID_TOKEN_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{ const uploadIntents = new Map(); const uploadPulls = new Map(); +function parsePositiveIntEnv(name: string, fallback: number): number { + const raw = process.env[name]; + if (!raw) return fallback; + const parsed = Number(raw); + if (!Number.isFinite(parsed) || parsed <= 0) return fallback; + return Math.floor(parsed); +} + +// WAN uploads can be significantly slower than LAN transfers. +const FILE_UPLOAD_INTENT_TTL_MS = parsePositiveIntEnv( + "OVERLORD_FILE_UPLOAD_INTENT_TTL_MS", + 30 * 60_000, +); +const FILE_UPLOAD_PULL_TTL_MS = parsePositiveIntEnv( + "OVERLORD_FILE_UPLOAD_PULL_TTL_MS", + 30 * 60_000, +); + function isSafeRemotePath(value: string): boolean { if (!value || value.length > 4096) return false; return !/[\x00-\x1F\x7F]/.test(value); @@ -323,10 +341,10 @@ export async function handleFileDownloadRoutes( } const uploadId = uuidv4(); - const expiresAt = Date.now() + 2 * 60_000; + const expiresAt = Date.now() + FILE_UPLOAD_INTENT_TTL_MS; const timeout = setTimeout(() => { uploadIntents.delete(uploadId); - }, 2 * 60_000); + }, FILE_UPLOAD_INTENT_TTL_MS); uploadIntents.set(uploadId, { id: uploadId, @@ -423,14 +441,14 @@ export async function handleFileDownloadRoutes( clearTimeout(intent.timeout); const pullId = uuidv4(); - const pullExpiresAt = Date.now() + 10 * 60_000; + const pullExpiresAt = Date.now() + FILE_UPLOAD_PULL_TTL_MS; const pullTimeout = setTimeout(() => { const pull = uploadPulls.get(pullId); uploadPulls.delete(pullId); if (pull) { void fs.unlink(pull.tmpPath).catch(() => {}); } - }, 10 * 60_000); + }, FILE_UPLOAD_PULL_TTL_MS); uploadPulls.set(pullId, { id: pullId, diff --git a/README.md b/README.md index da818ca..5a13b06 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,34 @@ to update run docker compose pull ``` +### Faster local rebuilds (BuildKit cache) + +`docker-compose.yml` now includes local BuildKit cache settings under `build.cache_from` / `build.cache_to`. +Use these commands when changing source code and rebuilding locally: + +```sh +docker compose build +docker compose up -d +``` + +or in one step: + +```sh +docker compose up --build -d +``` + +Build cache is stored in `.docker-cache/buildx` and reused across builds. + +### UI client build cache (runtime) + +Client builds triggered from the Overlord UI now use a dedicated persistent cache volume: + +- volume: `overlord-client-build-cache` +- mount path: `/app/client-build-cache` +- env var: `OVERLORD_CLIENT_BUILD_CACHE_DIR` (default `/app/client-build-cache`) + +This cache is used only for runtime client builds (`go build` / `garble build` from the Build page) and stays warm across container restarts and updates. + ## Docker TLS with certbot If you want to avoid self-signed certificates in Docker/production, enable certbot TLS settings: diff --git a/docker-compose.quickstart.yml b/docker-compose.quickstart.yml index 0c84487..88d5a42 100644 --- a/docker-compose.quickstart.yml +++ b/docker-compose.quickstart.yml @@ -1,6 +1,13 @@ services: overlord-server: image: ghcr.io/vxaboveground/overlord:latest + build: + context: . + dockerfile: Dockerfile + cache_from: + - type=local,src=.docker-cache/buildx + cache_to: + - type=local,dest=.docker-cache/buildx,mode=max container_name: overlord-server ports: - "5173:5173" @@ -28,9 +35,13 @@ services: - OVERLORD_TLS_CERTBOT_CERT_FILE=fullchain.pem - OVERLORD_TLS_CERTBOT_KEY_FILE=privkey.pem - OVERLORD_TLS_CERTBOT_CA_FILE=chain.pem + - OVERLORD_CLIENT_BUILD_CACHE_DIR=/app/client-build-cache + - OVERLORD_FILE_UPLOAD_INTENT_TTL_MS=1800000 + - OVERLORD_FILE_UPLOAD_PULL_TTL_MS=1800000 volumes: - overlord-data:/app/data - overlord-certs:/app/certs + - overlord-client-build-cache:/app/client-build-cache # Uncomment to mount host certbot certs read-only # - /etc/letsencrypt:/etc/letsencrypt:ro # Uncomment to use a static config file instead of env vars @@ -52,3 +63,4 @@ networks: volumes: overlord-data: overlord-certs: + overlord-client-build-cache: diff --git a/docker-compose.yml b/docker-compose.yml index ccbff62..a701b83 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,13 @@ services: overlord-server: image: ${DOCKER_IMAGE:-ghcr.io/vxaboveground/overlord:latest} + build: + context: . + dockerfile: Dockerfile + cache_from: + - type=local,src=.docker-cache/buildx + cache_to: + - type=local,dest=.docker-cache/buildx,mode=max container_name: overlord-server ports: - "${PORT:-5173}:5173" @@ -28,9 +35,13 @@ services: OVERLORD_TLS_CERTBOT_CERT_FILE: ${OVERLORD_TLS_CERTBOT_CERT_FILE:-fullchain.pem} OVERLORD_TLS_CERTBOT_KEY_FILE: ${OVERLORD_TLS_CERTBOT_KEY_FILE:-privkey.pem} OVERLORD_TLS_CERTBOT_CA_FILE: ${OVERLORD_TLS_CERTBOT_CA_FILE:-chain.pem} + OVERLORD_CLIENT_BUILD_CACHE_DIR: ${OVERLORD_CLIENT_BUILD_CACHE_DIR:-/app/client-build-cache} + OVERLORD_FILE_UPLOAD_INTENT_TTL_MS: ${OVERLORD_FILE_UPLOAD_INTENT_TTL_MS:-1800000} + OVERLORD_FILE_UPLOAD_PULL_TTL_MS: ${OVERLORD_FILE_UPLOAD_PULL_TTL_MS:-1800000} volumes: - overlord-data:/app/data - overlord-certs:/app/certs + - overlord-client-build-cache:/app/client-build-cache # Uncomment to mount host certbot certs read-only # - /etc/letsencrypt:/etc/letsencrypt:ro # Uncomment to use a static config file instead of env vars @@ -55,3 +66,4 @@ networks: volumes: overlord-data: overlord-certs: + overlord-client-build-cache: