hopefully all ts works (it's getting pushed to prod fuck you)

This commit is contained in:
KingKDot
2026-03-08 03:07:23 -05:00
parent d4284456f9
commit 82e4a5daaf
8 changed files with 113 additions and 14 deletions
+1
View File
@@ -15,6 +15,7 @@ config.json
# Docker # Docker
.env .env
.docker-cache/
# Go # Go
bin/ bin/
+6 -1
View File
@@ -1,3 +1,4 @@
# syntax=docker/dockerfile:1.7
# Overlord Server Dockerfile # Overlord Server Dockerfile
FROM oven/bun:1 AS base FROM oven/bun:1 AS base
WORKDIR /app WORKDIR /app
@@ -30,9 +31,13 @@ RUN case "${TARGETARCH}" in \
ENV PATH="/usr/local/go/bin:${PATH}" ENV PATH="/usr/local/go/bin:${PATH}"
ENV GOPATH="/go" ENV GOPATH="/go"
ENV PATH="${GOPATH}/bin:${PATH}" ENV PATH="${GOPATH}/bin:${PATH}"
ENV GOCACHE=/root/.cache/go-build
ENV GOMODCACHE=/go/pkg/mod
# Install garble for obfuscated agent builds (requires Go 1.25+) # Install garble for obfuscated agent builds (requires Go 1.25+)
RUN go install mvdan.cc/garble@latest RUN --mount=type=cache,target=/root/.cache/go-build \
--mount=type=cache,target=/go/pkg/mod \
go install mvdan.cc/garble@latest
# Copy package files and lockfile # Copy package files and lockfile
COPY Overlord-Server/package.json Overlord-Server/bun.lock* ./ COPY Overlord-Server/package.json Overlord-Server/bun.lock* ./
+13 -9
View File
@@ -1630,21 +1630,24 @@ async function uploadFileViaHttpPull(file, path, transfer) {
console.debug("[filebrowser] upload stage url", { uploadUrl }); console.debug("[filebrowser] upload stage url", { uploadUrl });
const uploadRequestOptions = {
headers: { "Content-Type": "application/octet-stream" },
credentials: "include",
body: file,
signal: transfer.abortController?.signal,
};
let uploadRes; let uploadRes;
try { try {
uploadRes = await fetch(uploadUrl, { uploadRes = await fetch(uploadUrl, {
method: "PUT", method: "POST",
headers: { "Content-Type": "application/octet-stream" }, ...uploadRequestOptions,
credentials: "include",
body: file,
}); });
} catch (err) { } catch (err) {
console.warn("[filebrowser] upload stage PUT failed, retrying as POST", err); console.warn("[filebrowser] upload stage POST failed, retrying as PUT", err);
uploadRes = await fetch(uploadUrl, { uploadRes = await fetch(uploadUrl, {
method: "POST", method: "PUT",
headers: { "Content-Type": "application/octet-stream" }, ...uploadRequestOptions,
credentials: "include",
body: file,
}); });
} }
@@ -1721,6 +1724,7 @@ async function uploadFile(file) {
ackedOffsets: new Set(), ackedOffsets: new Set(),
transferId, transferId,
completed: false, completed: false,
abortController: new AbortController(),
}; };
fileUploads.set(path, transfer); fileUploads.set(path, transfer);
@@ -4,6 +4,7 @@ import fs from "fs";
import { saveBuild } from "../db"; import { saveBuild } from "../db";
import { logger } from "../logger"; import { logger } from "../logger";
import { getConfig } from "../config"; import { getConfig } from "../config";
import { ensureDataDir } from "../paths";
import * as buildManager from "../build/buildManager"; import * as buildManager from "../build/buildManager";
import type { BuildStream } from "../build/types"; import type { BuildStream } from "../build/types";
import { ALLOWED_PLATFORMS } from "./validation-constants"; import { ALLOWED_PLATFORMS } from "./validation-constants";
@@ -33,6 +34,16 @@ function resolveClientModuleDir(rootDir: string): string | null {
return null; return null;
} }
function resolveClientBuildCacheRoot(): string {
const explicit = process.env.OVERLORD_CLIENT_BUILD_CACHE_DIR?.trim();
if (explicit) {
return path.resolve(explicit);
}
// Keep UI build caches under persistent app data by default.
return path.resolve(ensureDataDir(), "client-build-cache");
}
type BuildProcessConfig = { type BuildProcessConfig = {
platforms: string[]; platforms: string[];
serverUrl?: string; serverUrl?: string;
@@ -133,10 +144,16 @@ export async function startBuildProcess(
); );
} }
const outDir = path.join(rootDir, "dist-clients"); const outDir = path.join(rootDir, "dist-clients");
const cacheRoot = resolveClientBuildCacheRoot();
const goBuildCacheDir = path.join(cacheRoot, "go-build");
const goModCacheDir = path.join(cacheRoot, "go-mod");
await Bun.$`mkdir -p ${outDir}`.quiet(); await Bun.$`mkdir -p ${outDir}`.quiet();
fs.mkdirSync(goBuildCacheDir, { recursive: true });
fs.mkdirSync(goModCacheDir, { recursive: true });
sendToStream({ type: "output", text: `Build directory: ${outDir}\n`, level: "info" }); sendToStream({ type: "output", text: `Build directory: ${outDir}\n`, level: "info" });
sendToStream({ type: "output", text: `Client source: ${clientDir}\n`, level: "info" }); sendToStream({ type: "output", text: `Client source: ${clientDir}\n`, level: "info" });
sendToStream({ type: "output", text: `Client build cache: ${cacheRoot}\n`, level: "info" });
const platformsToBuild = (config.platforms || []).filter((p) => ALLOWED_PLATFORMS.has(p)); const platformsToBuild = (config.platforms || []).filter((p) => ALLOWED_PLATFORMS.has(p));
if (platformsToBuild.length !== (config.platforms || []).length) { if (platformsToBuild.length !== (config.platforms || []).length) {
@@ -168,6 +185,8 @@ export async function startBuildProcess(
GOARCH: actualArch, GOARCH: actualArch,
CGO_ENABLED: config.disableCgo === true ? "0" : "1", CGO_ENABLED: config.disableCgo === true ? "0" : "1",
GOWORK: "off", GOWORK: "off",
GOCACHE: goBuildCacheDir,
GOMODCACHE: goModCacheDir,
...(goarm ? { GOARM: goarm } : {}), ...(goarm ? { GOARM: goarm } : {}),
}; };
@@ -118,6 +118,24 @@ const UUID_TOKEN_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{
const uploadIntents = new Map<string, UploadIntent>(); const uploadIntents = new Map<string, UploadIntent>();
const uploadPulls = new Map<string, UploadPull>(); const uploadPulls = new Map<string, UploadPull>();
function parsePositiveIntEnv(name: string, fallback: number): number {
const raw = process.env[name];
if (!raw) return fallback;
const parsed = Number(raw);
if (!Number.isFinite(parsed) || parsed <= 0) return fallback;
return Math.floor(parsed);
}
// WAN uploads can be significantly slower than LAN transfers.
const FILE_UPLOAD_INTENT_TTL_MS = parsePositiveIntEnv(
"OVERLORD_FILE_UPLOAD_INTENT_TTL_MS",
30 * 60_000,
);
const FILE_UPLOAD_PULL_TTL_MS = parsePositiveIntEnv(
"OVERLORD_FILE_UPLOAD_PULL_TTL_MS",
30 * 60_000,
);
function isSafeRemotePath(value: string): boolean { function isSafeRemotePath(value: string): boolean {
if (!value || value.length > 4096) return false; if (!value || value.length > 4096) return false;
return !/[\x00-\x1F\x7F]/.test(value); return !/[\x00-\x1F\x7F]/.test(value);
@@ -323,10 +341,10 @@ export async function handleFileDownloadRoutes(
} }
const uploadId = uuidv4(); const uploadId = uuidv4();
const expiresAt = Date.now() + 2 * 60_000; const expiresAt = Date.now() + FILE_UPLOAD_INTENT_TTL_MS;
const timeout = setTimeout(() => { const timeout = setTimeout(() => {
uploadIntents.delete(uploadId); uploadIntents.delete(uploadId);
}, 2 * 60_000); }, FILE_UPLOAD_INTENT_TTL_MS);
uploadIntents.set(uploadId, { uploadIntents.set(uploadId, {
id: uploadId, id: uploadId,
@@ -423,14 +441,14 @@ export async function handleFileDownloadRoutes(
clearTimeout(intent.timeout); clearTimeout(intent.timeout);
const pullId = uuidv4(); const pullId = uuidv4();
const pullExpiresAt = Date.now() + 10 * 60_000; const pullExpiresAt = Date.now() + FILE_UPLOAD_PULL_TTL_MS;
const pullTimeout = setTimeout(() => { const pullTimeout = setTimeout(() => {
const pull = uploadPulls.get(pullId); const pull = uploadPulls.get(pullId);
uploadPulls.delete(pullId); uploadPulls.delete(pullId);
if (pull) { if (pull) {
void fs.unlink(pull.tmpPath).catch(() => {}); void fs.unlink(pull.tmpPath).catch(() => {});
} }
}, 10 * 60_000); }, FILE_UPLOAD_PULL_TTL_MS);
uploadPulls.set(pullId, { uploadPulls.set(pullId, {
id: pullId, id: pullId,
+28
View File
@@ -48,6 +48,34 @@ to update run
docker compose pull docker compose pull
``` ```
### Faster local rebuilds (BuildKit cache)
`docker-compose.yml` now includes local BuildKit cache settings under `build.cache_from` / `build.cache_to`.
Use these commands when changing source code and rebuilding locally:
```sh
docker compose build
docker compose up -d
```
or in one step:
```sh
docker compose up --build -d
```
Build cache is stored in `.docker-cache/buildx` and reused across builds.
### UI client build cache (runtime)
Client builds triggered from the Overlord UI now use a dedicated persistent cache volume:
- volume: `overlord-client-build-cache`
- mount path: `/app/client-build-cache`
- env var: `OVERLORD_CLIENT_BUILD_CACHE_DIR` (default `/app/client-build-cache`)
This cache is used only for runtime client builds (`go build` / `garble build` from the Build page) and stays warm across container restarts and updates.
## Docker TLS with certbot ## Docker TLS with certbot
If you want to avoid self-signed certificates in Docker/production, enable certbot TLS settings: If you want to avoid self-signed certificates in Docker/production, enable certbot TLS settings:
+12
View File
@@ -1,6 +1,13 @@
services: services:
overlord-server: overlord-server:
image: ghcr.io/vxaboveground/overlord:latest image: ghcr.io/vxaboveground/overlord:latest
build:
context: .
dockerfile: Dockerfile
cache_from:
- type=local,src=.docker-cache/buildx
cache_to:
- type=local,dest=.docker-cache/buildx,mode=max
container_name: overlord-server container_name: overlord-server
ports: ports:
- "5173:5173" - "5173:5173"
@@ -28,9 +35,13 @@ services:
- OVERLORD_TLS_CERTBOT_CERT_FILE=fullchain.pem - OVERLORD_TLS_CERTBOT_CERT_FILE=fullchain.pem
- OVERLORD_TLS_CERTBOT_KEY_FILE=privkey.pem - OVERLORD_TLS_CERTBOT_KEY_FILE=privkey.pem
- OVERLORD_TLS_CERTBOT_CA_FILE=chain.pem - OVERLORD_TLS_CERTBOT_CA_FILE=chain.pem
- OVERLORD_CLIENT_BUILD_CACHE_DIR=/app/client-build-cache
- OVERLORD_FILE_UPLOAD_INTENT_TTL_MS=1800000
- OVERLORD_FILE_UPLOAD_PULL_TTL_MS=1800000
volumes: volumes:
- overlord-data:/app/data - overlord-data:/app/data
- overlord-certs:/app/certs - overlord-certs:/app/certs
- overlord-client-build-cache:/app/client-build-cache
# Uncomment to mount host certbot certs read-only # Uncomment to mount host certbot certs read-only
# - /etc/letsencrypt:/etc/letsencrypt:ro # - /etc/letsencrypt:/etc/letsencrypt:ro
# Uncomment to use a static config file instead of env vars # Uncomment to use a static config file instead of env vars
@@ -52,3 +63,4 @@ networks:
volumes: volumes:
overlord-data: overlord-data:
overlord-certs: overlord-certs:
overlord-client-build-cache:
+12
View File
@@ -1,6 +1,13 @@
services: services:
overlord-server: overlord-server:
image: ${DOCKER_IMAGE:-ghcr.io/vxaboveground/overlord:latest} image: ${DOCKER_IMAGE:-ghcr.io/vxaboveground/overlord:latest}
build:
context: .
dockerfile: Dockerfile
cache_from:
- type=local,src=.docker-cache/buildx
cache_to:
- type=local,dest=.docker-cache/buildx,mode=max
container_name: overlord-server container_name: overlord-server
ports: ports:
- "${PORT:-5173}:5173" - "${PORT:-5173}:5173"
@@ -28,9 +35,13 @@ services:
OVERLORD_TLS_CERTBOT_CERT_FILE: ${OVERLORD_TLS_CERTBOT_CERT_FILE:-fullchain.pem} OVERLORD_TLS_CERTBOT_CERT_FILE: ${OVERLORD_TLS_CERTBOT_CERT_FILE:-fullchain.pem}
OVERLORD_TLS_CERTBOT_KEY_FILE: ${OVERLORD_TLS_CERTBOT_KEY_FILE:-privkey.pem} OVERLORD_TLS_CERTBOT_KEY_FILE: ${OVERLORD_TLS_CERTBOT_KEY_FILE:-privkey.pem}
OVERLORD_TLS_CERTBOT_CA_FILE: ${OVERLORD_TLS_CERTBOT_CA_FILE:-chain.pem} OVERLORD_TLS_CERTBOT_CA_FILE: ${OVERLORD_TLS_CERTBOT_CA_FILE:-chain.pem}
OVERLORD_CLIENT_BUILD_CACHE_DIR: ${OVERLORD_CLIENT_BUILD_CACHE_DIR:-/app/client-build-cache}
OVERLORD_FILE_UPLOAD_INTENT_TTL_MS: ${OVERLORD_FILE_UPLOAD_INTENT_TTL_MS:-1800000}
OVERLORD_FILE_UPLOAD_PULL_TTL_MS: ${OVERLORD_FILE_UPLOAD_PULL_TTL_MS:-1800000}
volumes: volumes:
- overlord-data:/app/data - overlord-data:/app/data
- overlord-certs:/app/certs - overlord-certs:/app/certs
- overlord-client-build-cache:/app/client-build-cache
# Uncomment to mount host certbot certs read-only # Uncomment to mount host certbot certs read-only
# - /etc/letsencrypt:/etc/letsencrypt:ro # - /etc/letsencrypt:/etc/letsencrypt:ro
# Uncomment to use a static config file instead of env vars # Uncomment to use a static config file instead of env vars
@@ -55,3 +66,4 @@ networks:
volumes: volumes:
overlord-data: overlord-data:
overlord-certs: overlord-certs:
overlord-client-build-cache: