diff --git a/Overlord-Server/src/config.ts b/Overlord-Server/src/config.ts
index 53c5e36..b3adfee 100644
--- a/Overlord-Server/src/config.ts
+++ b/Overlord-Server/src/config.ts
@@ -134,6 +134,14 @@ export interface Config {
maxFileBytes: number;
pollIntervalSeconds: number;
};
+ fileTransfers: {
+ maxFileBytes: number;
+ maxStagedBytes: number;
+ maxActiveGlobal: number;
+ maxActivePerUser: number;
+ uploadIntentTtlMs: number;
+ uploadPullTtlMs: number;
+ };
}
const DEFAULT_CONFIG: Config = {
@@ -266,6 +274,14 @@ const DEFAULT_CONFIG: Config = {
maxFileBytes: 5 * 1024 * 1024,
pollIntervalSeconds: 300,
},
+ fileTransfers: {
+ maxFileBytes: 1024 * 1024 * 1024,
+ maxStagedBytes: 4 * 1024 * 1024 * 1024,
+ maxActiveGlobal: 16,
+ maxActivePerUser: 4,
+ uploadIntentTtlMs: 30 * 60_000,
+ uploadPullTtlMs: 30 * 60_000,
+ },
};
function envBoolOverride(name: string): boolean | undefined {
@@ -959,6 +975,68 @@ export function loadConfig(): Config {
),
),
},
+ fileTransfers: (() => {
+ const maxFileBytes = Math.min(
+ 1024 * 1024 * 1024,
+ Math.max(
+ 1024 * 1024,
+ Number(process.env.OVERLORD_FILE_UPLOAD_MAX_BYTES)
+ || Number(fileConfig.fileTransfers?.maxFileBytes)
+ || DEFAULT_CONFIG.fileTransfers.maxFileBytes,
+ ),
+ );
+ const maxStagedBytes = Math.min(
+ 500 * 1024 * 1024 * 1024,
+ Math.max(
+ maxFileBytes,
+ Number(process.env.OVERLORD_FILE_UPLOAD_MAX_STAGED_BYTES)
+ || Number(fileConfig.fileTransfers?.maxStagedBytes)
+ || DEFAULT_CONFIG.fileTransfers.maxStagedBytes,
+ ),
+ );
+ const maxActiveGlobal = Math.min(
+ 1000,
+ Math.max(
+ 1,
+ Number(process.env.OVERLORD_FILE_UPLOAD_MAX_ACTIVE_GLOBAL)
+ || Number(fileConfig.fileTransfers?.maxActiveGlobal)
+ || DEFAULT_CONFIG.fileTransfers.maxActiveGlobal,
+ ),
+ );
+ const maxActivePerUser = Math.min(
+ maxActiveGlobal,
+ Math.max(
+ 1,
+ Number(process.env.OVERLORD_FILE_UPLOAD_MAX_ACTIVE_PER_USER)
+ || Number(fileConfig.fileTransfers?.maxActivePerUser)
+ || DEFAULT_CONFIG.fileTransfers.maxActivePerUser,
+ ),
+ );
+ return {
+ maxFileBytes: Math.floor(maxFileBytes),
+ maxStagedBytes: Math.floor(maxStagedBytes),
+ maxActiveGlobal: Math.floor(maxActiveGlobal),
+ maxActivePerUser: Math.floor(maxActivePerUser),
+ uploadIntentTtlMs: Math.min(
+ 24 * 60 * 60_000,
+ Math.max(
+ 60_000,
+ Number(process.env.OVERLORD_FILE_UPLOAD_INTENT_TTL_MS)
+ || Number(fileConfig.fileTransfers?.uploadIntentTtlMs)
+ || DEFAULT_CONFIG.fileTransfers.uploadIntentTtlMs,
+ ),
+ ),
+ uploadPullTtlMs: Math.min(
+ 24 * 60 * 60_000,
+ Math.max(
+ 60_000,
+ Number(process.env.OVERLORD_FILE_UPLOAD_PULL_TTL_MS)
+ || Number(fileConfig.fileTransfers?.uploadPullTtlMs)
+ || DEFAULT_CONFIG.fileTransfers.uploadPullTtlMs,
+ ),
+ ),
+ };
+ })(),
};
if (saveChanged) {
@@ -1303,6 +1381,51 @@ export async function updateInputArchiveConfig(
return next;
}
+export async function updateFileTransfersConfig(
+ updates: Partial,
+): Promise {
+ const current = getConfig();
+ const numberOr = (value: unknown, fallback: number) => {
+ const parsed = Number(value);
+ return Number.isFinite(parsed) ? parsed : fallback;
+ };
+ const next: Config["fileTransfers"] = {
+ ...current.fileTransfers,
+ ...updates,
+ };
+
+ next.maxFileBytes = Math.floor(Math.min(
+ 1024 * 1024 * 1024,
+ Math.max(1024 * 1024, numberOr(next.maxFileBytes, DEFAULT_CONFIG.fileTransfers.maxFileBytes)),
+ ));
+ next.maxStagedBytes = Math.floor(Math.min(
+ 500 * 1024 * 1024 * 1024,
+ Math.max(next.maxFileBytes, numberOr(next.maxStagedBytes, DEFAULT_CONFIG.fileTransfers.maxStagedBytes)),
+ ));
+ next.maxActiveGlobal = Math.floor(Math.min(
+ 1000,
+ Math.max(1, numberOr(next.maxActiveGlobal, DEFAULT_CONFIG.fileTransfers.maxActiveGlobal)),
+ ));
+ next.maxActivePerUser = Math.floor(Math.min(
+ next.maxActiveGlobal,
+ Math.max(1, numberOr(next.maxActivePerUser, DEFAULT_CONFIG.fileTransfers.maxActivePerUser)),
+ ));
+ next.uploadIntentTtlMs = Math.floor(Math.min(
+ 24 * 60 * 60_000,
+ Math.max(60_000, numberOr(next.uploadIntentTtlMs, DEFAULT_CONFIG.fileTransfers.uploadIntentTtlMs)),
+ ));
+ next.uploadPullTtlMs = Math.floor(Math.min(
+ 24 * 60 * 60_000,
+ Math.max(60_000, numberOr(next.uploadPullTtlMs, DEFAULT_CONFIG.fileTransfers.uploadPullTtlMs)),
+ ));
+
+ configCache = { ...current, fileTransfers: next };
+ const fileConfig = readFileConfigForUpdate();
+ fileConfig.fileTransfers = next;
+ await writePersistentFileConfig(fileConfig);
+ return next;
+}
+
export async function updateRegistrationConfig(
updates: Partial,
): Promise {
@@ -1489,6 +1612,7 @@ export function getExportableConfig(serverVersion: string): Record): Promise<{ app
applied.push("inputArchive");
}
+ if (data.fileTransfers && typeof data.fileTransfers === "object") {
+ await updateFileTransfersConfig(data.fileTransfers);
+ applied.push("fileTransfers");
+ }
+
if (data.auth && typeof data.auth === "object") {
const dataDir = ensureDataDir();
const savePath = resolve(dataDir, "save.json");
diff --git a/Overlord-Server/src/file-transfer-settings.test.ts b/Overlord-Server/src/file-transfer-settings.test.ts
new file mode 100644
index 0000000..9a1f191
--- /dev/null
+++ b/Overlord-Server/src/file-transfer-settings.test.ts
@@ -0,0 +1,51 @@
+import { afterAll, beforeAll, describe, expect, test } from "bun:test";
+import { getConfig, updateFileTransfersConfig, type Config } from "./config";
+import { getFileTransferLimits } from "./server/file-transfer-state";
+
+let original: Config["fileTransfers"];
+
+beforeAll(() => {
+ original = { ...getConfig().fileTransfers };
+});
+
+afterAll(async () => {
+ await updateFileTransfersConfig(original);
+});
+
+describe("file transfer settings", () => {
+ test("saved limits are visible to transfer handlers immediately", async () => {
+ await updateFileTransfersConfig({
+ maxFileBytes: 25 * 1024 * 1024,
+ maxStagedBytes: 100 * 1024 * 1024,
+ maxActiveGlobal: 8,
+ maxActivePerUser: 2,
+ uploadIntentTtlMs: 5 * 60_000,
+ uploadPullTtlMs: 10 * 60_000,
+ });
+
+ expect(getFileTransferLimits()).toEqual({
+ maxFileBytes: 25 * 1024 * 1024,
+ maxStagedBytes: 100 * 1024 * 1024,
+ maxActiveGlobal: 8,
+ maxActivePerUser: 2,
+ uploadIntentTtlMs: 5 * 60_000,
+ uploadPullTtlMs: 10 * 60_000,
+ });
+ });
+
+ test("dependent limits are clamped to safe relationships", async () => {
+ const updated = await updateFileTransfersConfig({
+ maxFileBytes: 20 * 1024 * 1024,
+ maxStagedBytes: 10 * 1024 * 1024,
+ maxActiveGlobal: 3,
+ maxActivePerUser: 20,
+ uploadIntentTtlMs: 1,
+ uploadPullTtlMs: 1,
+ });
+
+ expect(updated.maxStagedBytes).toBe(updated.maxFileBytes);
+ expect(updated.maxActivePerUser).toBe(updated.maxActiveGlobal);
+ expect(updated.uploadIntentTtlMs).toBe(60_000);
+ expect(updated.uploadPullTtlMs).toBe(60_000);
+ });
+});
diff --git a/Overlord-Server/src/server/file-transfer-state.ts b/Overlord-Server/src/server/file-transfer-state.ts
index 2e78da8..6a651a2 100644
--- a/Overlord-Server/src/server/file-transfer-state.ts
+++ b/Overlord-Server/src/server/file-transfer-state.ts
@@ -2,6 +2,7 @@ import fs from "fs/promises";
import type { FileHandle } from "fs/promises";
import path from "path";
import { v4 as uuidv4 } from "uuid";
+import { getConfig } from "../config";
export type PendingHttpDownload = {
commandId: string;
@@ -71,27 +72,14 @@ export type UploadPull = {
deleteFile: boolean;
state?: StreamingPullState;
expectedTotal?: number;
+ userId?: number;
};
-function parsePositiveIntEnv(name: string, fallback: number): number {
- const raw = process.env[name];
- if (!raw) return fallback;
- const parsed = Number(raw);
- if (!Number.isFinite(parsed) || parsed <= 0) return fallback;
- return Math.floor(parsed);
-}
-
export const UUID_TOKEN_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
-// WAN uploads can be significantly slower than LAN transfers.
-export const FILE_UPLOAD_INTENT_TTL_MS = parsePositiveIntEnv(
- "OVERLORD_FILE_UPLOAD_INTENT_TTL_MS",
- 30 * 60_000,
-);
-export const FILE_UPLOAD_PULL_TTL_MS = parsePositiveIntEnv(
- "OVERLORD_FILE_UPLOAD_PULL_TTL_MS",
- 30 * 60_000,
-);
+export function getFileTransferLimits() {
+ return getConfig().fileTransfers;
+}
export const uploadIntents = new Map();
export const uploadPulls = new Map();
@@ -260,7 +248,7 @@ export function createUploadPull(opts: {
ttlMs?: number;
}): string {
const pullId = uuidv4();
- const ttl = opts.ttlMs ?? FILE_UPLOAD_PULL_TTL_MS;
+ const ttl = opts.ttlMs ?? getFileTransferLimits().uploadPullTtlMs;
const expiresAt = Date.now() + ttl;
const pullTimeout = setTimeout(() => {
uploadPulls.delete(pullId);
diff --git a/Overlord-Server/src/server/http-download-consumer.test.ts b/Overlord-Server/src/server/http-download-consumer.test.ts
index 2ce7af1..fa89ec0 100644
--- a/Overlord-Server/src/server/http-download-consumer.test.ts
+++ b/Overlord-Server/src/server/http-download-consumer.test.ts
@@ -2,6 +2,7 @@ import { describe, expect, test } from "bun:test";
import {
consumeHttpDownloadPayload,
STREAM_MAX_CHUNK_BYTES,
+ STREAM_OUTPUT_QUEUE_MAX_CHUNKS,
type PendingHttpDownload,
} from "./http-download-consumer";
@@ -55,4 +56,32 @@ describe("HTTP download stream limits", () => {
expect(entries.has("cmd")).toBeFalse();
expect(fixture.state().streamError).toContain("requested byte limit");
});
+
+ test("bounds queued output when the HTTP receiver stops consuming", async () => {
+ const fixture = pendingDownload({
+ streamController: {
+ desiredSize: -STREAM_OUTPUT_QUEUE_MAX_CHUNKS,
+ enqueue() {},
+ close() {},
+ error(error: Error) { fixtureError = error.message; },
+ } as any,
+ });
+ let fixtureError = "";
+ const entries = new Map([[fixture.pending.commandId, fixture.pending]]);
+ await consumeHttpDownloadPayload({
+ commandId: "cmd", offset: 0, total: 4, data: new Uint8Array(4),
+ }, entries);
+ clearTimeout(fixture.pending.timeout);
+ expect(entries.has("cmd")).toBeFalse();
+ expect(fixtureError).toContain("not consuming");
+ });
+
+ test("rejects malformed encoded chunks without throwing", async () => {
+ const fixture = pendingDownload();
+ const entries = new Map([[fixture.pending.commandId, fixture.pending]]);
+ await consumeHttpDownloadPayload({ commandId: "cmd", data: "%%%not-base64%%%" }, entries);
+ clearTimeout(fixture.pending.timeout);
+ expect(entries.has("cmd")).toBeFalse();
+ expect(fixture.state().streamError).toContain("encoding");
+ });
});
diff --git a/Overlord-Server/src/server/http-download-consumer.ts b/Overlord-Server/src/server/http-download-consumer.ts
index 62d47d8..9df8d20 100644
--- a/Overlord-Server/src/server/http-download-consumer.ts
+++ b/Overlord-Server/src/server/http-download-consumer.ts
@@ -5,6 +5,7 @@ import { logger } from "../logger";
export const STREAM_REORDER_BUFFER_LIMIT = 16;
export const STREAM_REORDER_BUFFER_MAX_BYTES = 32 * 1024 * 1024;
export const STREAM_MAX_CHUNK_BYTES = 4 * 1024 * 1024;
+export const STREAM_OUTPUT_QUEUE_MAX_CHUNKS = 16;
export type PendingHttpDownload = {
commandId: string;
@@ -54,7 +55,14 @@ async function failStream(pending: PendingHttpDownload, err: Error): Promise {
+ const desiredSize = pending.streamController?.desiredSize;
+ if (typeof desiredSize !== "number" || desiredSize > -STREAM_OUTPUT_QUEUE_MAX_CHUNKS) return false;
+ void failStream(pending, new Error("download receiver is not consuming data fast enough"));
+ return true;
+ };
if (offset === (pending.nextExpectedOffset ?? 0)) {
+ if (outputQueueFull()) return false;
try {
pending.streamController.enqueue(data);
} catch (err) {
@@ -67,6 +75,7 @@ function enqueueOrdered(pending: PendingHttpDownload, offset: number, data: Uint
const next = pending.reorderBuffer.get(pending.nextExpectedOffset ?? 0);
if (!next) break;
pending.reorderBuffer.delete(pending.nextExpectedOffset ?? 0);
+ if (outputQueueFull()) return false;
try {
pending.streamController.enqueue(next);
} catch (err) {
@@ -120,7 +129,7 @@ export async function consumeHttpDownloadPayload(
if (!pending) return;
if (payload?.error) {
- const err = new Error(String(payload.error));
+ const err = new Error(String(payload.error).slice(0, 1024));
if (isStreamMode(pending)) {
await failStream(pending, err);
clearTimeout(pending.timeout);
@@ -134,7 +143,7 @@ export async function consumeHttpDownloadPayload(
}
const toNumber = (value: unknown): number | null => {
- if (typeof value === "number" && Number.isFinite(value)) return value;
+ if (typeof value === "number" && Number.isSafeInteger(value)) return value;
if (typeof value === "bigint") {
const asNumber = Number(value);
if (Number.isSafeInteger(asNumber)) return asNumber;
@@ -168,12 +177,20 @@ export async function consumeHttpDownloadPayload(
if (payload?.data) {
let data = payload.data;
- if (data instanceof ArrayBuffer) {
- data = new Uint8Array(data);
- } else if (typeof data === "string") {
- data = Uint8Array.from(atob(data), (c) => c.charCodeAt(0));
- } else if (ArrayBuffer.isView(data)) {
- data = new Uint8Array(data.buffer, data.byteOffset, data.byteLength);
+ try {
+ if (data instanceof ArrayBuffer) {
+ data = new Uint8Array(data);
+ } else if (typeof data === "string") {
+ data = Uint8Array.from(atob(data), (c) => c.charCodeAt(0));
+ } else if (ArrayBuffer.isView(data)) {
+ data = new Uint8Array(data.buffer, data.byteOffset, data.byteLength);
+ }
+ } catch {
+ await failStream(pending, new Error("invalid download chunk encoding"));
+ clearTimeout(pending.timeout);
+ pendingHttpDownloads.delete(commandId);
+ pending.resolve(pending);
+ return;
}
if (data instanceof Uint8Array) {
diff --git a/Overlord-Server/src/server/routes/file-download-routes.test.ts b/Overlord-Server/src/server/routes/file-download-routes.test.ts
index a7abddb..8cd5776 100644
--- a/Overlord-Server/src/server/routes/file-download-routes.test.ts
+++ b/Overlord-Server/src/server/routes/file-download-routes.test.ts
@@ -12,6 +12,7 @@ import {
setUserFeaturePermission,
} from "../../users";
import { handleFileDownloadRoutes } from "./file-download-routes";
+import { getFileTransferLimits } from "../file-transfer-state";
const PASSWORD = "Aa1!RouteUploadTestPass123";
@@ -58,6 +59,114 @@ async function removeTempDataDir(dir: string): Promise {
}
describe("file upload route flow", () => {
+ test("consumes an upload intent atomically when staging requests race", async () => {
+ const auth = await createAdminToken();
+ const clientId = `client-${Date.now().toString(36)}`;
+ const dataDir = await createTempDataDir();
+ const deps = makeRouteDeps(dataDir);
+ clientManager.addClient(clientId, {
+ id: clientId, lastSeen: Date.now(), role: "client", ws: { send: () => {} },
+ });
+ const previousDisableAgentAuth = process.env.OVERLORD_DISABLE_AGENT_AUTH;
+ process.env.OVERLORD_DISABLE_AGENT_AUTH = "true";
+
+ try {
+ const requestUrl = new URL("https://localhost/api/file/upload/request");
+ const requestRes = await handleFileDownloadRoutes(
+ new Request(requestUrl, {
+ method: "POST",
+ headers: { "Content-Type": "application/json", Authorization: `Bearer ${auth.token}` },
+ body: JSON.stringify({ clientId, path: "C:\\Games\\race.bin", fileName: "race.bin" }),
+ }),
+ requestUrl,
+ mockServer,
+ deps,
+ );
+ const requestJson = await requestRes!.json() as any;
+ const stageUrl = new URL(`https://localhost${requestJson.uploadUrl}`);
+ const makeStageRequest = () => new Request(stageUrl, {
+ method: "PUT",
+ headers: { Authorization: `Bearer ${auth.token}` },
+ body: new TextEncoder().encode("race payload"),
+ });
+ const responses = await Promise.all([
+ handleFileDownloadRoutes(makeStageRequest(), stageUrl, mockServer, deps),
+ handleFileDownloadRoutes(makeStageRequest(), stageUrl, mockServer, deps),
+ ]);
+
+ expect(responses.map((response) => response!.status).sort()).toEqual([200, 404]);
+ const successful = responses.find((response) => response!.status === 200)!;
+ const staged = await successful!.json() as any;
+ const pullUrl = new URL(staged.pullUrl, "https://localhost");
+ const pullResponse = await handleFileDownloadRoutes(
+ new Request(pullUrl, { headers: { "x-overlord-client-id": clientId } }),
+ pullUrl,
+ mockServer,
+ deps,
+ );
+ expect(await pullResponse!.text()).toBe("race payload");
+ } finally {
+ if (previousDisableAgentAuth === undefined) delete process.env.OVERLORD_DISABLE_AGENT_AUTH;
+ else process.env.OVERLORD_DISABLE_AGENT_AUTH = previousDisableAgentAuth;
+ clientManager.deleteClient(clientId);
+ deleteUser(auth.userId);
+ await removeTempDataDir(dataDir);
+ }
+ });
+
+ test("rejects an oversized staged upload before writing a payload file", async () => {
+ const auth = await createAdminToken();
+ const clientId = `client-${Date.now().toString(36)}`;
+ const dataDir = await createTempDataDir();
+ const deps = makeRouteDeps(dataDir);
+
+ clientManager.addClient(clientId, {
+ id: clientId,
+ lastSeen: Date.now(),
+ role: "client",
+ ws: { send: () => {} },
+ });
+
+ try {
+ const requestUrl = new URL("https://localhost/api/file/upload/request");
+ const requestRes = await handleFileDownloadRoutes(
+ new Request(requestUrl, {
+ method: "POST",
+ headers: {
+ "Content-Type": "application/json",
+ Authorization: `Bearer ${auth.token}`,
+ },
+ body: JSON.stringify({ clientId, path: "C:\\Games\\too-big.iso", fileName: "too-big.iso" }),
+ }),
+ requestUrl,
+ mockServer,
+ deps,
+ );
+ const requestJson = await requestRes!.json() as any;
+ const stageUrl = new URL(`https://localhost${requestJson.uploadUrl}`);
+ const stageRes = await handleFileDownloadRoutes(
+ new Request(stageUrl, {
+ method: "PUT",
+ headers: {
+ Authorization: `Bearer ${auth.token}`,
+ "Content-Length": String(getFileTransferLimits().maxFileBytes + 1),
+ },
+ }),
+ stageUrl,
+ mockServer,
+ deps,
+ );
+
+ expect(stageRes!.status).toBe(413);
+ const uploadEntries = await readdir(join(dataDir, "uploads")).catch(() => [] as string[]);
+ expect(uploadEntries).toHaveLength(0);
+ } finally {
+ clientManager.deleteClient(clientId);
+ deleteUser(auth.userId);
+ await removeTempDataDir(dataDir);
+ }
+ });
+
test("stages and serves one-time upload payload for matching client", async () => {
const auth = await createAdminToken();
const clientId = `client-${Date.now().toString(36)}`;
@@ -586,6 +695,52 @@ describe("file route feature_browser enforcement", () => {
}
});
+ test("upload stage PUT rechecks client access after intent creation", async () => {
+ const auth = await createOperatorToken({ fileBrowserAllowed: true });
+ const clientId = `client-${Date.now().toString(36)}`;
+ const dataDir = await createTempDataDir();
+ const deps = makeRouteDeps(dataDir);
+ clientManager.addClient(clientId, {
+ id: clientId, lastSeen: Date.now(), role: "client", ws: { send: () => {} },
+ });
+
+ try {
+ const requestUrl = new URL("https://localhost/api/file/upload/request");
+ const requestRes = await handleFileDownloadRoutes(
+ new Request(requestUrl, {
+ method: "POST",
+ headers: { "Content-Type": "application/json", Authorization: `Bearer ${auth.token}` },
+ body: JSON.stringify({ clientId, path: "C:\\Games\\blocked.bin", fileName: "blocked.bin" }),
+ }),
+ requestUrl,
+ mockServer,
+ deps,
+ );
+ const requestJson = await requestRes!.json() as any;
+ setUserClientAccessScope(auth.userId, "none");
+
+ const stageUrl = new URL(`https://localhost${requestJson.uploadUrl}`);
+ const stageRes = await handleFileDownloadRoutes(
+ new Request(stageUrl, {
+ method: "PUT",
+ headers: { Authorization: `Bearer ${auth.token}` },
+ body: new TextEncoder().encode("must not be staged"),
+ }),
+ stageUrl,
+ mockServer,
+ deps,
+ );
+
+ expect(stageRes!.status).toBe(403);
+ const uploadEntries = await readdir(join(dataDir, "uploads")).catch(() => [] as string[]);
+ expect(uploadEntries).toHaveLength(0);
+ } finally {
+ clientManager.deleteClient(clientId);
+ deleteUser(auth.userId);
+ await removeTempDataDir(dataDir);
+ }
+ });
+
test("download serve returns 403 when file_browser is revoked after intent creation", async () => {
const auth = await createOperatorToken({ fileBrowserAllowed: true });
const clientId = `client-${Date.now().toString(36)}`;
diff --git a/Overlord-Server/src/server/routes/file-download-routes.ts b/Overlord-Server/src/server/routes/file-download-routes.ts
index 46ea543..ef88bb0 100644
--- a/Overlord-Server/src/server/routes/file-download-routes.ts
+++ b/Overlord-Server/src/server/routes/file-download-routes.ts
@@ -10,9 +10,8 @@ import { getConfig } from "../../config";
import { isAuthorizedAgentRequest } from "../agent-auth";
import { requireClientAccess, requireFeatureAccess, requirePermission } from "../../rbac";
import {
- FILE_UPLOAD_INTENT_TTL_MS,
- FILE_UPLOAD_PULL_TTL_MS,
UUID_TOKEN_RE,
+ getFileTransferLimits,
isSafeRemotePath,
makeIncrementalPullStream,
notifyPullWaiters,
@@ -37,6 +36,26 @@ type FileDownloadRouteDeps = {
downloadIntents: Map;
};
+class UploadStagingLimitError extends Error {
+ constructor(message: string, readonly status: number) {
+ super(message);
+ }
+}
+
+function activeStagedPulls() {
+ const now = Date.now();
+ return [...uploadPulls.values()].filter((pull) =>
+ pull.deleteFile && pull.expiresAt >= now,
+ );
+}
+
+function reservedStagedBytes(excludePullId = ""): number {
+ return activeStagedPulls().reduce((total, pull) => {
+ if (pull.id === excludePullId) return total;
+ return total + Math.max(0, pull.expectedTotal || 0, pull.state?.size || 0, pull.size || 0);
+ }, 0);
+}
+
async function serveDownloadById(
req: Request,
downloadId: string,
@@ -155,6 +174,7 @@ async function serveDownloadById(
}
firstChunkReject(new Error("Download timed out"));
reject(new Error("Download timed out"));
+ try { target.ws.send(encodeMessage({ type: "command_abort", commandId } as any)); } catch {}
}, 5 * 60_000);
deps.pendingHttpDownloads.set(commandId, {
@@ -309,11 +329,19 @@ export async function handleFileDownloadRoutes(
return new Response("Client offline", { status: 404 });
}
+ const limits = getFileTransferLimits();
+ const activeForUser = [...uploadIntents.values()]
+ .filter((intent) => intent.userId === user.userId && intent.expiresAt >= Date.now()).length
+ + activeStagedPulls().filter((pull) => pull.userId === user.userId).length;
+ if (activeForUser >= limits.maxActivePerUser) {
+ return new Response("Too many pending uploads", { status: 429, headers: { "Retry-After": "10" } });
+ }
+
const uploadId = uuidv4();
- const expiresAt = Date.now() + FILE_UPLOAD_INTENT_TTL_MS;
+ const expiresAt = Date.now() + limits.uploadIntentTtlMs;
const timeout = setTimeout(() => {
uploadIntents.delete(uploadId);
- }, FILE_UPLOAD_INTENT_TTL_MS);
+ }, limits.uploadIntentTtlMs);
uploadIntents.set(uploadId, {
id: uploadId,
@@ -372,25 +400,42 @@ export async function handleFileDownloadRoutes(
if (!intent || intent.userId !== user.userId || intent.expiresAt < Date.now()) {
return new Response("Not found", { status: 404 });
}
+ try {
+ requireClientAccess(user, intent.clientId);
+ } catch (error) {
+ if (error instanceof Response) return error;
+ return new Response("Forbidden", { status: 403 });
+ }
- const uploadDir = path.join(deps.DATA_DIR, "uploads");
- await fs.mkdir(uploadDir, { recursive: true });
-
- const pullId = uuidv4();
- const tmpPath = path.join(uploadDir, `${pullId}.bin`);
+ const limits = getFileTransferLimits();
+ const activePulls = activeStagedPulls();
+ if (activePulls.length >= limits.maxActiveGlobal
+ || activePulls.filter((pull) => pull.userId === user.userId).length >= limits.maxActivePerUser) {
+ return new Response("Too many active staged uploads", { status: 429, headers: { "Retry-After": "10" } });
+ }
const contentLength = Number(req.headers.get("content-length") || 0);
const expectedTotal = Number.isFinite(contentLength) && contentLength >= 0 ? contentLength : 0;
+ if (expectedTotal > limits.maxFileBytes) {
+ return new Response("Upload exceeds the server staging limit", { status: 413 });
+ }
+ if (reservedStagedBytes() + expectedTotal > limits.maxStagedBytes) {
+ return new Response("Server upload staging quota exceeded", { status: 429, headers: { "Retry-After": "30" } });
+ }
+
+ const uploadDir = path.join(deps.DATA_DIR, "uploads");
+ const pullId = uuidv4();
+ const tmpPath = path.join(uploadDir, `${pullId}.bin`);
const canStream = !!req.body && expectedTotal > 0;
- const pullExpiresAt = Date.now() + FILE_UPLOAD_PULL_TTL_MS;
+ const pullExpiresAt = Date.now() + limits.uploadPullTtlMs;
const pullTimeout = setTimeout(() => {
const stale = uploadPulls.get(pullId);
uploadPulls.delete(pullId);
if (stale && stale.deleteFile) {
void fs.unlink(stale.tmpPath).catch(() => {});
}
- }, FILE_UPLOAD_PULL_TTL_MS);
+ }, limits.uploadPullTtlMs);
const state: StreamingPullState | undefined = canStream
? { size: 0, done: false, error: null, waiters: [] }
@@ -408,6 +453,7 @@ export async function handleFileDownloadRoutes(
deleteFile: true,
state,
expectedTotal,
+ userId: user.userId,
});
uploadIntents.delete(uploadId);
@@ -416,14 +462,27 @@ export async function handleFileDownloadRoutes(
const pullUrl = `/api/file/upload/pull/${encodeURIComponent(pullId)}`;
let stagedSize = 0;
- const fileHandle = await fs.open(tmpPath, "w");
+ let fileHandle: Awaited> | null = null;
try {
+ await fs.mkdir(uploadDir, { recursive: true });
+ fileHandle = await fs.open(tmpPath, "w");
if (req.body) {
const reader = req.body.getReader();
while (true) {
const { done, value } = await reader.read();
if (done) break;
if (!value || value.byteLength === 0) continue;
+ const currentPull = uploadPulls.get(pullId);
+ if (!currentPull || currentPull.expiresAt < Date.now()) {
+ throw new UploadStagingLimitError("Upload staging expired", 408);
+ }
+ if (stagedSize + value.byteLength > limits.maxFileBytes) {
+ throw new UploadStagingLimitError("Upload exceeds the server staging limit", 413);
+ }
+ if (reservedStagedBytes(pullId) + stagedSize + value.byteLength > limits.maxStagedBytes) {
+ throw new UploadStagingLimitError("Server upload staging quota exceeded", 429);
+ }
+ currentPull.size = Math.max(currentPull.size, stagedSize + value.byteLength);
await fileHandle.write(value, 0, value.byteLength, stagedSize);
stagedSize += value.byteLength;
if (state) {
@@ -434,6 +493,17 @@ export async function handleFileDownloadRoutes(
} else {
const bytes = new Uint8Array(await req.arrayBuffer());
if (bytes.byteLength > 0) {
+ if (bytes.byteLength > limits.maxFileBytes) {
+ throw new UploadStagingLimitError("Upload exceeds the server staging limit", 413);
+ }
+ if (reservedStagedBytes(pullId) + bytes.byteLength > limits.maxStagedBytes) {
+ throw new UploadStagingLimitError("Server upload staging quota exceeded", 429);
+ }
+ const currentPull = uploadPulls.get(pullId);
+ if (!currentPull || currentPull.expiresAt < Date.now()) {
+ throw new UploadStagingLimitError("Upload staging expired", 408);
+ }
+ currentPull.size = Math.max(currentPull.size, bytes.byteLength);
await fileHandle.write(bytes, 0, bytes.byteLength, 0);
stagedSize = bytes.byteLength;
if (state) {
@@ -452,10 +522,16 @@ export async function handleFileDownloadRoutes(
state.done = true;
notifyPullWaiters(state);
}
- await fileHandle.close();
+ if (fileHandle) await fileHandle.close().catch(() => {});
await fs.unlink(tmpPath).catch(() => {});
uploadPulls.delete(pullId);
clearTimeout(pullTimeout);
+ if (err instanceof UploadStagingLimitError) {
+ return new Response(err.message, {
+ status: err.status,
+ ...(err.status === 429 ? { headers: { "Retry-After": "30" } } : {}),
+ });
+ }
return new Response("Upload staging failed", { status: 500 });
}
await fileHandle.close();
@@ -467,6 +543,9 @@ export async function handleFileDownloadRoutes(
const finalPull = uploadPulls.get(pullId);
if (finalPull) {
finalPull.size = stagedSize;
+ } else {
+ await fs.unlink(tmpPath).catch(() => {});
+ return new Response("Upload staging expired", { status: 408 });
}
logger.debug("[filebrowser] http upload staged bytes", {
diff --git a/Overlord-Server/src/server/routes/misc-routes.ts b/Overlord-Server/src/server/routes/misc-routes.ts
index b65a6cd..4996d40 100644
--- a/Overlord-Server/src/server/routes/misc-routes.ts
+++ b/Overlord-Server/src/server/routes/misc-routes.ts
@@ -1,7 +1,7 @@
import { authenticateRequest } from "../../auth";
import { AuditAction, getAuditLogs, logAudit } from "../../auditLog";
import { logger } from "../../logger";
-import { getConfig, updateSecurityConfig, updateTlsConfig, updateOidcConfig, updateAppearanceConfig, updateChatConfig, getExportableConfig, importFullConfig, updateRegistrationConfig, updateBuildRateLimitConfig, updateThumbnailsConfig, updateInputArchiveConfig } from "../../config";
+import { getConfig, updateSecurityConfig, updateTlsConfig, updateOidcConfig, updateAppearanceConfig, updateChatConfig, getExportableConfig, importFullConfig, updateRegistrationConfig, updateBuildRateLimitConfig, updateThumbnailsConfig, updateInputArchiveConfig, updateFileTransfersConfig } from "../../config";
import {
saveBrandingImage,
getClientMetricsSummary,
@@ -670,6 +670,40 @@ export async function handleMiscRoutes(
}
}
+ if (url.pathname === "/api/settings/file-transfers") {
+ const user = await authenticateRequest(req);
+ if (!user) return new Response("Unauthorized", { status: 401 });
+ try { requirePermission(user, "system:security"); } catch (error) { if (error instanceof Response) return error; return new Response("Forbidden", { status: 403 }); }
+
+ if (req.method === "GET") {
+ return Response.json({ fileTransfers: getConfig().fileTransfers }, { headers: deps.CORS_HEADERS });
+ }
+
+ if (req.method === "PUT") {
+ let body: any = {};
+ try { body = await req.json(); } catch {
+ return Response.json({ error: "Invalid JSON" }, { status: 400 });
+ }
+ const updated = await updateFileTransfersConfig({
+ maxFileBytes: body?.maxFileBytes !== undefined ? Number(body.maxFileBytes) : undefined,
+ maxStagedBytes: body?.maxStagedBytes !== undefined ? Number(body.maxStagedBytes) : undefined,
+ maxActiveGlobal: body?.maxActiveGlobal !== undefined ? Number(body.maxActiveGlobal) : undefined,
+ maxActivePerUser: body?.maxActivePerUser !== undefined ? Number(body.maxActivePerUser) : undefined,
+ uploadIntentTtlMs: body?.uploadIntentTtlMs !== undefined ? Number(body.uploadIntentTtlMs) : undefined,
+ uploadPullTtlMs: body?.uploadPullTtlMs !== undefined ? Number(body.uploadPullTtlMs) : undefined,
+ });
+ logAudit({
+ timestamp: Date.now(),
+ username: user.username,
+ ip: deps.requestIP?.(req)?.address || "unknown",
+ action: AuditAction.COMMAND,
+ details: "Updated file transfer limits",
+ success: true,
+ });
+ return Response.json({ ok: true, fileTransfers: updated }, { headers: deps.CORS_HEADERS });
+ }
+ }
+
if (url.pathname === "/api/settings/tls") {
const user = await authenticateRequest(req);
if (!user) {
diff --git a/Overlord-Server/src/server/ws-file-process-proxy-keylogger.test.ts b/Overlord-Server/src/server/ws-file-process-proxy-keylogger.test.ts
new file mode 100644
index 0000000..b3f217a
--- /dev/null
+++ b/Overlord-Server/src/server/ws-file-process-proxy-keylogger.test.ts
@@ -0,0 +1,32 @@
+import { describe, expect, test } from "bun:test";
+import { handleFileBrowserMessage } from "./ws-file-process-proxy-keylogger";
+
+describe("file browser agent download routing", () => {
+ test("only consumes pending HTTP downloads for the matching client", () => {
+ let consumed = 0;
+ const payload = { type: "file_download", commandId: "download-1", data: new Uint8Array([1]) };
+ const deps = {
+ pendingHttpDownloads: new Map([["download-1", { clientId: "client-a" }]]),
+ consumeHttpDownloadPayload: () => { consumed += 1; },
+ };
+
+ handleFileBrowserMessage("client-b", payload, deps);
+ expect(consumed).toBe(0);
+
+ handleFileBrowserMessage("client-a", payload, deps);
+ expect(consumed).toBe(1);
+ });
+
+ test("drops unsolicited download payloads", () => {
+ let consumed = 0;
+ handleFileBrowserMessage(
+ "client-a",
+ { type: "file_download", commandId: "unknown", data: new Uint8Array([1]) },
+ {
+ pendingHttpDownloads: new Map(),
+ consumeHttpDownloadPayload: () => { consumed += 1; },
+ },
+ );
+ expect(consumed).toBe(0);
+ });
+});
diff --git a/Overlord-Server/src/server/ws-file-process-proxy-keylogger.ts b/Overlord-Server/src/server/ws-file-process-proxy-keylogger.ts
index 8d62784..0792d60 100644
--- a/Overlord-Server/src/server/ws-file-process-proxy-keylogger.ts
+++ b/Overlord-Server/src/server/ws-file-process-proxy-keylogger.ts
@@ -45,7 +45,7 @@ type ProcessViewer = {
};
type WsViewerClusterDeps = {
- pendingHttpDownloads: Map;
+ pendingHttpDownloads: ReadonlyMap;
consumeHttpDownloadPayload: (payload: any) => Promise | void;
};
@@ -397,17 +397,23 @@ export function handleFileBrowserViewerMessage(ws: ServerWebSocket,
export function handleFileBrowserMessage(clientId: string, payload: any, deps: WsViewerClusterDeps) {
const type = payload?.type as string | undefined;
+ const payloadCommandId = typeof payload?.commandId === "string" ? payload.commandId : undefined;
+ const pendingDownload = payloadCommandId
+ ? deps.pendingHttpDownloads.get(payloadCommandId)
+ : undefined;
const isHttpDownload =
type === "file_download" &&
- typeof payload?.commandId === "string" &&
- deps.pendingHttpDownloads.has(payload.commandId);
+ pendingDownload?.clientId === clientId;
- if (type === "file_download" && typeof payload?.commandId === "string") {
+ if (isHttpDownload) {
void deps.consumeHttpDownloadPayload(payload);
}
- const payloadCommandId = typeof payload?.commandId === "string" ? payload.commandId : undefined;
const commandOwner = payloadCommandId ? fileBrowserCommandSessions.get(payloadCommandId) : undefined;
+ if (type === "file_download" && !isHttpDownload && !commandOwner) {
+ logger.debug(`[filebrowser] dropped unsolicited download payload client=${clientId}`);
+ return;
+ }
const ownerSessionId = commandOwner?.sessionId;
let hasSession = false;
diff --git a/Overlord-Server/test/e2e/file-transfer-api.spec.ts b/Overlord-Server/test/e2e/file-transfer-api.spec.ts
new file mode 100644
index 0000000..91968fc
--- /dev/null
+++ b/Overlord-Server/test/e2e/file-transfer-api.spec.ts
@@ -0,0 +1,218 @@
+import { webcrypto } from "node:crypto";
+import {
+ expect,
+ request as requestFactory,
+ test,
+ type APIRequestContext,
+} from "@playwright/test";
+import { decodeMessage, encodeMessage } from "../../src/protocol";
+import { ADMIN } from "./credentials";
+
+const BASE_URL = "http://127.0.0.1:5193";
+const AGENT_TOKEN = "e2e-agent-token-that-is-only-used-for-browser-tests";
+const CLIENT_ID = "e2e-file-transfer-agent";
+
+type AgentCommand = {
+ type: "command";
+ commandType: string;
+ id?: string;
+ payload?: { path?: string };
+};
+
+class TransferTestAgent {
+ private socket: WebSocket | null = null;
+ private downloads = new Map();
+
+ setDownload(path: string, bytes: Uint8Array): void {
+ this.downloads.set(path, bytes);
+ }
+
+ async connect(): Promise {
+ const keyPair = await webcrypto.subtle.generateKey(
+ { name: "Ed25519" },
+ true,
+ ["sign", "verify"],
+ );
+ const publicKey = new Uint8Array(
+ await webcrypto.subtle.exportKey("raw", keyPair.publicKey),
+ );
+ const socket = new WebSocket(
+ `ws://127.0.0.1:5193/api/clients/${CLIENT_ID}/stream/ws?token=${encodeURIComponent(AGENT_TOKEN)}`,
+ );
+ socket.binaryType = "arraybuffer";
+ this.socket = socket;
+
+ await new Promise((resolve, reject) => {
+ const timeout = setTimeout(() => reject(new Error("Agent enrollment timed out")), 10_000);
+
+ socket.addEventListener("error", () => {
+ clearTimeout(timeout);
+ reject(new Error("Agent WebSocket failed"));
+ });
+ socket.addEventListener("close", (event) => {
+ if (socket.readyState !== WebSocket.OPEN) {
+ clearTimeout(timeout);
+ reject(new Error(`Agent WebSocket closed during enrollment (${event.code}: ${event.reason})`));
+ }
+ });
+ socket.addEventListener("message", async (event) => {
+ const message = decodeMessage(event.data as ArrayBuffer) as any;
+ if (message.type === "enrollment_challenge") {
+ const nonce = Buffer.from(message.nonce, "base64");
+ const signature = await webcrypto.subtle.sign("Ed25519", keyPair.privateKey, nonce);
+ socket.send(encodeMessage({
+ type: "hello",
+ id: CLIENT_ID,
+ host: "e2e-transfer-host",
+ os: "E2E",
+ arch: "x64",
+ version: "e2e",
+ user: "playwright",
+ monitors: 1,
+ hwid: CLIENT_ID,
+ publicKey: Buffer.from(publicKey).toString("base64"),
+ signature: Buffer.from(signature).toString("base64"),
+ }));
+ return;
+ }
+ if (message.type === "hello_ack") {
+ clearTimeout(timeout);
+ resolve();
+ return;
+ }
+ if (message.type === "command") {
+ this.handleCommand(message as AgentCommand);
+ }
+ });
+ });
+ }
+
+ close(): void {
+ this.socket?.close(1000, "test_complete");
+ this.socket = null;
+ }
+
+ private handleCommand(command: AgentCommand): void {
+ if (command.commandType !== "file_download" || !command.id) return;
+ const path = command.payload?.path || "";
+ const bytes = this.downloads.get(path);
+ if (!bytes) {
+ this.socket?.send(encodeMessage({
+ type: "file_download",
+ commandId: command.id,
+ path,
+ data: new Uint8Array(),
+ offset: 0,
+ total: 0,
+ error: "E2E fixture not found",
+ }));
+ return;
+ }
+ this.socket?.send(encodeMessage({
+ type: "file_download",
+ commandId: command.id,
+ path,
+ data: bytes,
+ offset: 0,
+ total: bytes.byteLength,
+ chunkIndex: 0,
+ chunksTotal: 1,
+ }));
+ }
+}
+
+test.describe("file transfer HTTP API runtime", () => {
+ let api: APIRequestContext;
+ let agent: TransferTestAgent;
+
+ test.beforeAll(async () => {
+ const loginApi = await requestFactory.newContext({ baseURL: BASE_URL });
+ const login = await loginApi.post("/api/login", {
+ data: { user: ADMIN.username, pass: ADMIN.password },
+ });
+ expect(login.ok()).toBeTruthy();
+ const { token } = await login.json();
+ await loginApi.dispose();
+ api = await requestFactory.newContext({
+ baseURL: BASE_URL,
+ extraHTTPHeaders: { Authorization: `Bearer ${token}` },
+ });
+
+ const enrollment = await api.post("/api/enrollment/settings", {
+ data: { requireApproval: false, autoApproveUnlessSuspicious: false },
+ });
+ expect(enrollment.ok()).toBeTruthy();
+
+ agent = new TransferTestAgent();
+ await agent.connect();
+ });
+
+ test.afterAll(async () => {
+ agent?.close();
+ if (api) {
+ await api.post("/api/enrollment/settings", {
+ data: { requireApproval: true, autoApproveUnlessSuspicious: false },
+ });
+ await api.dispose();
+ }
+ });
+
+ test("uploads bytes for a one-time authenticated agent pull", async () => {
+ const remotePath = "C:\\Temp\\overlord-e2e-upload.bin";
+ const uploadBytes = Buffer.from("playwright upload payload\n\u0000with binary bytes", "utf8");
+
+ const requestUpload = await api.post("/api/file/upload/request", {
+ data: {
+ clientId: CLIENT_ID,
+ path: remotePath,
+ fileName: "overlord-e2e-upload.bin",
+ },
+ });
+ expect(requestUpload.ok()).toBeTruthy();
+ const { uploadUrl } = await requestUpload.json();
+
+ const stageUpload = await api.put(uploadUrl, {
+ data: uploadBytes,
+ headers: { "Content-Type": "application/octet-stream" },
+ });
+ expect(stageUpload.ok()).toBeTruthy();
+ const staged = await stageUpload.json();
+ expect(staged).toMatchObject({
+ ok: true,
+ size: uploadBytes.byteLength,
+ path: remotePath,
+ });
+
+ const pullHeaders = {
+ "x-agent-token": AGENT_TOKEN,
+ "x-overlord-client-id": CLIENT_ID,
+ };
+ const pull = await api.get(staged.pullUrl, { headers: pullHeaders });
+ expect(pull.ok()).toBeTruthy();
+ expect(Buffer.from(await pull.body())).toEqual(uploadBytes);
+
+ const replay = await api.get(staged.pullUrl, { headers: pullHeaders });
+ expect(replay.status()).toBe(404);
+ });
+
+ test("streams a requested agent file through the download API", async () => {
+ const remotePath = "C:\\Temp\\overlord-e2e-download.bin";
+ const downloadBytes = new Uint8Array([
+ 0x00, 0x01, 0x02, 0x7f, 0x80, 0xfe, 0xff,
+ ...Buffer.from("playwright download payload", "utf8"),
+ ]);
+ agent.setDownload(remotePath, downloadBytes);
+
+ const requestDownload = await api.post("/api/file/download/request", {
+ data: { clientId: CLIENT_ID, path: remotePath },
+ });
+ expect(requestDownload.ok()).toBeTruthy();
+ const { downloadUrl } = await requestDownload.json();
+
+ const download = await api.get(downloadUrl);
+ expect(download.ok()).toBeTruthy();
+ expect(download.headers()["content-type"]).toContain("application/octet-stream");
+ expect(download.headers()["content-disposition"]).toContain("overlord-e2e-download.bin");
+ expect(Buffer.from(await download.body())).toEqual(Buffer.from(downloadBytes));
+ });
+});