vxaboveground/overlord
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

add restrictions on file handler

Browse Source
This commit is contained in:
KingKDot
2026-07-16 23:51:20 -05:00
parent 755626a315
commit 10ec62cf95
13 changed files with 919 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Overlord-Server/src/config.ts
+129
View File
@@ -134,6 +134,14 @@ export interface Config {
maxFileBytes: number;
pollIntervalSeconds: number;
};
fileTransfers: {
maxFileBytes: number;
maxStagedBytes: number;
maxActiveGlobal: number;
maxActivePerUser: number;
uploadIntentTtlMs: number;
uploadPullTtlMs: number;
};
}
const DEFAULT_CONFIG: Config = {
@@ -266,6 +274,14 @@ const DEFAULT_CONFIG: Config = {
maxFileBytes: 5 * 1024 * 1024,
pollIntervalSeconds: 300,
},
fileTransfers: {
maxFileBytes: 1024 * 1024 * 1024,
maxStagedBytes: 4 * 1024 * 1024 * 1024,
maxActiveGlobal: 16,
maxActivePerUser: 4,
uploadIntentTtlMs: 30 * 60_000,
uploadPullTtlMs: 30 * 60_000,
},
};
function envBoolOverride(name: string): boolean | undefined {
@@ -959,6 +975,68 @@ export function loadConfig(): Config {
),
),
},
fileTransfers: (() => {
const maxFileBytes = Math.min(
1024 * 1024 * 1024,
Math.max(
1024 * 1024,
Number(process.env.OVERLORD_FILE_UPLOAD_MAX_BYTES)
|| Number(fileConfig.fileTransfers?.maxFileBytes)
|| DEFAULT_CONFIG.fileTransfers.maxFileBytes,
),
);
const maxStagedBytes = Math.min(
500 * 1024 * 1024 * 1024,
Math.max(
maxFileBytes,
Number(process.env.OVERLORD_FILE_UPLOAD_MAX_STAGED_BYTES)
|| Number(fileConfig.fileTransfers?.maxStagedBytes)
|| DEFAULT_CONFIG.fileTransfers.maxStagedBytes,
),
);
const maxActiveGlobal = Math.min(
1000,
Math.max(
1,
Number(process.env.OVERLORD_FILE_UPLOAD_MAX_ACTIVE_GLOBAL)
|| Number(fileConfig.fileTransfers?.maxActiveGlobal)
|| DEFAULT_CONFIG.fileTransfers.maxActiveGlobal,
),
);
const maxActivePerUser = Math.min(
maxActiveGlobal,
Math.max(
1,
Number(process.env.OVERLORD_FILE_UPLOAD_MAX_ACTIVE_PER_USER)
|| Number(fileConfig.fileTransfers?.maxActivePerUser)
|| DEFAULT_CONFIG.fileTransfers.maxActivePerUser,
),
);
return {
maxFileBytes: Math.floor(maxFileBytes),
maxStagedBytes: Math.floor(maxStagedBytes),
maxActiveGlobal: Math.floor(maxActiveGlobal),
maxActivePerUser: Math.floor(maxActivePerUser),
uploadIntentTtlMs: Math.min(
24 * 60 * 60_000,
Math.max(
60_000,
Number(process.env.OVERLORD_FILE_UPLOAD_INTENT_TTL_MS)
|| Number(fileConfig.fileTransfers?.uploadIntentTtlMs)
|| DEFAULT_CONFIG.fileTransfers.uploadIntentTtlMs,
),
),
uploadPullTtlMs: Math.min(
24 * 60 * 60_000,
Math.max(
60_000,
Number(process.env.OVERLORD_FILE_UPLOAD_PULL_TTL_MS)
|| Number(fileConfig.fileTransfers?.uploadPullTtlMs)
|| DEFAULT_CONFIG.fileTransfers.uploadPullTtlMs,
),
),
};
})(),
};
if (saveChanged) {
@@ -1303,6 +1381,51 @@ export async function updateInputArchiveConfig(
return next;
}
export async function updateFileTransfersConfig(
updates: Partial<Config["fileTransfers"]>,
): Promise<Config["fileTransfers"]> {
const current = getConfig();
const numberOr = (value: unknown, fallback: number) => {
const parsed = Number(value);
return Number.isFinite(parsed) ? parsed : fallback;
};
const next: Config["fileTransfers"] = {
...current.fileTransfers,
...updates,
};
next.maxFileBytes = Math.floor(Math.min(
1024 * 1024 * 1024,
Math.max(1024 * 1024, numberOr(next.maxFileBytes, DEFAULT_CONFIG.fileTransfers.maxFileBytes)),
));
next.maxStagedBytes = Math.floor(Math.min(
500 * 1024 * 1024 * 1024,
Math.max(next.maxFileBytes, numberOr(next.maxStagedBytes, DEFAULT_CONFIG.fileTransfers.maxStagedBytes)),
));
next.maxActiveGlobal = Math.floor(Math.min(
1000,
Math.max(1, numberOr(next.maxActiveGlobal, DEFAULT_CONFIG.fileTransfers.maxActiveGlobal)),
));
next.maxActivePerUser = Math.floor(Math.min(
next.maxActiveGlobal,
Math.max(1, numberOr(next.maxActivePerUser, DEFAULT_CONFIG.fileTransfers.maxActivePerUser)),
));
next.uploadIntentTtlMs = Math.floor(Math.min(
24 * 60 * 60_000,
Math.max(60_000, numberOr(next.uploadIntentTtlMs, DEFAULT_CONFIG.fileTransfers.uploadIntentTtlMs)),
));
next.uploadPullTtlMs = Math.floor(Math.min(
24 * 60 * 60_000,
Math.max(60_000, numberOr(next.uploadPullTtlMs, DEFAULT_CONFIG.fileTransfers.uploadPullTtlMs)),
));
configCache = { ...current, fileTransfers: next };
const fileConfig = readFileConfigForUpdate();
fileConfig.fileTransfers = next;
await writePersistentFileConfig(fileConfig);
return next;
}
export async function updateRegistrationConfig(
updates: Partial<Config["registration"]>,
): Promise<Config["registration"]> {
@@ -1489,6 +1612,7 @@ export function getExportableConfig(serverVersion: string): Record<string, unkno
},
thumbnails: config.thumbnails,
inputArchive: config.inputArchive,
fileTransfers: config.fileTransfers,
};
}
@@ -1619,6 +1743,11 @@ export async function importFullConfig(data: Record<string, any>): Promise<{ app
applied.push("inputArchive");
}
if (data.fileTransfers && typeof data.fileTransfers === "object") {
await updateFileTransfersConfig(data.fileTransfers);
applied.push("fileTransfers");
}
if (data.auth && typeof data.auth === "object") {
const dataDir = ensureDataDir();
const savePath = resolve(dataDir, "save.json");
Overlord-Server/src/file-transfer-settings.test.ts
+51
View File
@@ -0,0 +1,51 @@
import { afterAll, beforeAll, describe, expect, test } from "bun:test";
import { getConfig, updateFileTransfersConfig, type Config } from "./config";
import { getFileTransferLimits } from "./server/file-transfer-state";
let original: Config["fileTransfers"];
beforeAll(() => {
original = { ...getConfig().fileTransfers };
});
afterAll(async () => {
await updateFileTransfersConfig(original);
});
describe("file transfer settings", () => {
test("saved limits are visible to transfer handlers immediately", async () => {
await updateFileTransfersConfig({
maxFileBytes: 25 * 1024 * 1024,
maxStagedBytes: 100 * 1024 * 1024,
maxActiveGlobal: 8,
maxActivePerUser: 2,
uploadIntentTtlMs: 5 * 60_000,
uploadPullTtlMs: 10 * 60_000,
});
expect(getFileTransferLimits()).toEqual({
maxFileBytes: 25 * 1024 * 1024,
maxStagedBytes: 100 * 1024 * 1024,
maxActiveGlobal: 8,
maxActivePerUser: 2,
uploadIntentTtlMs: 5 * 60_000,
uploadPullTtlMs: 10 * 60_000,
});
});
test("dependent limits are clamped to safe relationships", async () => {
const updated = await updateFileTransfersConfig({
maxFileBytes: 20 * 1024 * 1024,
maxStagedBytes: 10 * 1024 * 1024,
maxActiveGlobal: 3,
maxActivePerUser: 20,
uploadIntentTtlMs: 1,
uploadPullTtlMs: 1,
});
expect(updated.maxStagedBytes).toBe(updated.maxFileBytes);
expect(updated.maxActivePerUser).toBe(updated.maxActiveGlobal);
expect(updated.uploadIntentTtlMs).toBe(60_000);
expect(updated.uploadPullTtlMs).toBe(60_000);
});
});
Overlord-Server/src/server/file-transfer-state.ts
+6 -18
View File
@@ -2,6 +2,7 @@ import fs from "fs/promises";
import type { FileHandle } from "fs/promises";
import path from "path";
import { v4 as uuidv4 } from "uuid";
import { getConfig } from "../config";
export type PendingHttpDownload = {
commandId: string;
@@ -71,27 +72,14 @@ export type UploadPull = {
deleteFile: boolean;
state?: StreamingPullState;
expectedTotal?: number;
userId?: number;
};
function parsePositiveIntEnv(name: string, fallback: number): number {
const raw = process.env[name];
if (!raw) return fallback;
const parsed = Number(raw);
if (!Number.isFinite(parsed) || parsed <= 0) return fallback;
return Math.floor(parsed);
}
export const UUID_TOKEN_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
// WAN uploads can be significantly slower than LAN transfers.
export const FILE_UPLOAD_INTENT_TTL_MS = parsePositiveIntEnv(
"OVERLORD_FILE_UPLOAD_INTENT_TTL_MS",
30 * 60_000,
);
export const FILE_UPLOAD_PULL_TTL_MS = parsePositiveIntEnv(
"OVERLORD_FILE_UPLOAD_PULL_TTL_MS",
30 * 60_000,
);
export function getFileTransferLimits() {
return getConfig().fileTransfers;
}
export const uploadIntents = new Map<string, UploadIntent>();
export const uploadPulls = new Map<string, UploadPull>();
@@ -260,7 +248,7 @@ export function createUploadPull(opts: {
ttlMs?: number;
}): string {
const pullId = uuidv4();
const ttl = opts.ttlMs ?? FILE_UPLOAD_PULL_TTL_MS;
const ttl = opts.ttlMs ?? getFileTransferLimits().uploadPullTtlMs;
const expiresAt = Date.now() + ttl;
const pullTimeout = setTimeout(() => {
uploadPulls.delete(pullId);
Overlord-Server/src/server/http-download-consumer.test.ts
+29
View File
@@ -2,6 +2,7 @@ import { describe, expect, test } from "bun:test";
import {
consumeHttpDownloadPayload,
STREAM_MAX_CHUNK_BYTES,
STREAM_OUTPUT_QUEUE_MAX_CHUNKS,
type PendingHttpDownload,
} from "./http-download-consumer";
@@ -55,4 +56,32 @@ describe("HTTP download stream limits", () => {
expect(entries.has("cmd")).toBeFalse();
expect(fixture.state().streamError).toContain("requested byte limit");
});
test("bounds queued output when the HTTP receiver stops consuming", async () => {
const fixture = pendingDownload({
streamController: {
desiredSize: -STREAM_OUTPUT_QUEUE_MAX_CHUNKS,
enqueue() {},
close() {},
error(error: Error) { fixtureError = error.message; },
} as any,
});
let fixtureError = "";
const entries = new Map([[fixture.pending.commandId, fixture.pending]]);
await consumeHttpDownloadPayload({
commandId: "cmd", offset: 0, total: 4, data: new Uint8Array(4),
}, entries);
clearTimeout(fixture.pending.timeout);
expect(entries.has("cmd")).toBeFalse();
expect(fixtureError).toContain("not consuming");
});
test("rejects malformed encoded chunks without throwing", async () => {
const fixture = pendingDownload();
const entries = new Map([[fixture.pending.commandId, fixture.pending]]);
await consumeHttpDownloadPayload({ commandId: "cmd", data: "%%%not-base64%%%" }, entries);
clearTimeout(fixture.pending.timeout);
expect(entries.has("cmd")).toBeFalse();
expect(fixture.state().streamError).toContain("encoding");
});
});
Overlord-Server/src/server/http-download-consumer.ts
+25 -8
View File
@@ -5,6 +5,7 @@ import { logger } from "../logger";
export const STREAM_REORDER_BUFFER_LIMIT = 16;
export const STREAM_REORDER_BUFFER_MAX_BYTES = 32 * 1024 * 1024;
export const STREAM_MAX_CHUNK_BYTES = 4 * 1024 * 1024;
export const STREAM_OUTPUT_QUEUE_MAX_CHUNKS = 16;
export type PendingHttpDownload = {
commandId: string;
@@ -54,7 +55,14 @@ async function failStream(pending: PendingHttpDownload, err: Error): Promise<voi
function enqueueOrdered(pending: PendingHttpDownload, offset: number, data: Uint8Array): boolean {
if (!pending.streamController || pending.streamErrored) return false;
const outputQueueFull = () => {
const desiredSize = pending.streamController?.desiredSize;
if (typeof desiredSize !== "number" || desiredSize > -STREAM_OUTPUT_QUEUE_MAX_CHUNKS) return false;
void failStream(pending, new Error("download receiver is not consuming data fast enough"));
return true;
};
if (offset === (pending.nextExpectedOffset ?? 0)) {
if (outputQueueFull()) return false;
try {
pending.streamController.enqueue(data);
} catch (err) {
@@ -67,6 +75,7 @@ function enqueueOrdered(pending: PendingHttpDownload, offset: number, data: Uint
const next = pending.reorderBuffer.get(pending.nextExpectedOffset ?? 0);
if (!next) break;
pending.reorderBuffer.delete(pending.nextExpectedOffset ?? 0);
if (outputQueueFull()) return false;
try {
pending.streamController.enqueue(next);
} catch (err) {
@@ -120,7 +129,7 @@ export async function consumeHttpDownloadPayload(
if (!pending) return;
if (payload?.error) {
const err = new Error(String(payload.error));
const err = new Error(String(payload.error).slice(0, 1024));
if (isStreamMode(pending)) {
await failStream(pending, err);
clearTimeout(pending.timeout);
@@ -134,7 +143,7 @@ export async function consumeHttpDownloadPayload(
}
const toNumber = (value: unknown): number | null => {
if (typeof value === "number" && Number.isFinite(value)) return value;
if (typeof value === "number" && Number.isSafeInteger(value)) return value;
if (typeof value === "bigint") {
const asNumber = Number(value);
if (Number.isSafeInteger(asNumber)) return asNumber;
@@ -168,12 +177,20 @@ export async function consumeHttpDownloadPayload(
if (payload?.data) {
let data = payload.data;
if (data instanceof ArrayBuffer) {
data = new Uint8Array(data);
} else if (typeof data === "string") {
data = Uint8Array.from(atob(data), (c) => c.charCodeAt(0));
} else if (ArrayBuffer.isView(data)) {
data = new Uint8Array(data.buffer, data.byteOffset, data.byteLength);
try {
if (data instanceof ArrayBuffer) {
data = new Uint8Array(data);
} else if (typeof data === "string") {
data = Uint8Array.from(atob(data), (c) => c.charCodeAt(0));
} else if (ArrayBuffer.isView(data)) {
data = new Uint8Array(data.buffer, data.byteOffset, data.byteLength);
}
} catch {
await failStream(pending, new Error("invalid download chunk encoding"));
clearTimeout(pending.timeout);
pendingHttpDownloads.delete(commandId);
pending.resolve(pending);
return;
}
if (data instanceof Uint8Array) {
Overlord-Server/src/server/routes/file-download-routes.test.ts
+155
View File
@@ -12,6 +12,7 @@ import {
setUserFeaturePermission,
} from "../../users";
import { handleFileDownloadRoutes } from "./file-download-routes";
import { getFileTransferLimits } from "../file-transfer-state";
const PASSWORD = "Aa1!RouteUploadTestPass123";
@@ -58,6 +59,114 @@ async function removeTempDataDir(dir: string): Promise<void> {
}
describe("file upload route flow", () => {
test("consumes an upload intent atomically when staging requests race", async () => {
const auth = await createAdminToken();
const clientId = `client-${Date.now().toString(36)}`;
const dataDir = await createTempDataDir();
const deps = makeRouteDeps(dataDir);
clientManager.addClient(clientId, {
id: clientId, lastSeen: Date.now(), role: "client", ws: { send: () => {} },
});
const previousDisableAgentAuth = process.env.OVERLORD_DISABLE_AGENT_AUTH;
process.env.OVERLORD_DISABLE_AGENT_AUTH = "true";
try {
const requestUrl = new URL("https://localhost/api/file/upload/request");
const requestRes = await handleFileDownloadRoutes(
new Request(requestUrl, {
method: "POST",
headers: { "Content-Type": "application/json", Authorization: `Bearer ${auth.token}` },
body: JSON.stringify({ clientId, path: "C:\\Games\\race.bin", fileName: "race.bin" }),
}),
requestUrl,
mockServer,
deps,
);
const requestJson = await requestRes!.json() as any;
const stageUrl = new URL(`https://localhost${requestJson.uploadUrl}`);
const makeStageRequest = () => new Request(stageUrl, {
method: "PUT",
headers: { Authorization: `Bearer ${auth.token}` },
body: new TextEncoder().encode("race payload"),
});
const responses = await Promise.all([
handleFileDownloadRoutes(makeStageRequest(), stageUrl, mockServer, deps),
handleFileDownloadRoutes(makeStageRequest(), stageUrl, mockServer, deps),
]);
expect(responses.map((response) => response!.status).sort()).toEqual([200, 404]);
const successful = responses.find((response) => response!.status === 200)!;
const staged = await successful!.json() as any;
const pullUrl = new URL(staged.pullUrl, "https://localhost");
const pullResponse = await handleFileDownloadRoutes(
new Request(pullUrl, { headers: { "x-overlord-client-id": clientId } }),
pullUrl,
mockServer,
deps,
);
expect(await pullResponse!.text()).toBe("race payload");
} finally {
if (previousDisableAgentAuth === undefined) delete process.env.OVERLORD_DISABLE_AGENT_AUTH;
else process.env.OVERLORD_DISABLE_AGENT_AUTH = previousDisableAgentAuth;
clientManager.deleteClient(clientId);
deleteUser(auth.userId);
await removeTempDataDir(dataDir);
}
});
test("rejects an oversized staged upload before writing a payload file", async () => {
const auth = await createAdminToken();
const clientId = `client-${Date.now().toString(36)}`;
const dataDir = await createTempDataDir();
const deps = makeRouteDeps(dataDir);
clientManager.addClient(clientId, {
id: clientId,
lastSeen: Date.now(),
role: "client",
ws: { send: () => {} },
});
try {
const requestUrl = new URL("https://localhost/api/file/upload/request");
const requestRes = await handleFileDownloadRoutes(
new Request(requestUrl, {
method: "POST",
headers: {
"Content-Type": "application/json",
Authorization: `Bearer ${auth.token}`,
},
body: JSON.stringify({ clientId, path: "C:\\Games\\too-big.iso", fileName: "too-big.iso" }),
}),
requestUrl,
mockServer,
deps,
);
const requestJson = await requestRes!.json() as any;
const stageUrl = new URL(`https://localhost${requestJson.uploadUrl}`);
const stageRes = await handleFileDownloadRoutes(
new Request(stageUrl, {
method: "PUT",
headers: {
Authorization: `Bearer ${auth.token}`,
"Content-Length": String(getFileTransferLimits().maxFileBytes + 1),
},
}),
stageUrl,
mockServer,
deps,
);
expect(stageRes!.status).toBe(413);
const uploadEntries = await readdir(join(dataDir, "uploads")).catch(() => [] as string[]);
expect(uploadEntries).toHaveLength(0);
} finally {
clientManager.deleteClient(clientId);
deleteUser(auth.userId);
await removeTempDataDir(dataDir);
}
});
test("stages and serves one-time upload payload for matching client", async () => {
const auth = await createAdminToken();
const clientId = `client-${Date.now().toString(36)}`;
@@ -586,6 +695,52 @@ describe("file route feature_browser enforcement", () => {
}
});
test("upload stage PUT rechecks client access after intent creation", async () => {
const auth = await createOperatorToken({ fileBrowserAllowed: true });
const clientId = `client-${Date.now().toString(36)}`;
const dataDir = await createTempDataDir();
const deps = makeRouteDeps(dataDir);
clientManager.addClient(clientId, {
id: clientId, lastSeen: Date.now(), role: "client", ws: { send: () => {} },
});
try {
const requestUrl = new URL("https://localhost/api/file/upload/request");
const requestRes = await handleFileDownloadRoutes(
new Request(requestUrl, {
method: "POST",
headers: { "Content-Type": "application/json", Authorization: `Bearer ${auth.token}` },
body: JSON.stringify({ clientId, path: "C:\\Games\\blocked.bin", fileName: "blocked.bin" }),
}),
requestUrl,
mockServer,
deps,
);
const requestJson = await requestRes!.json() as any;
setUserClientAccessScope(auth.userId, "none");
const stageUrl = new URL(`https://localhost${requestJson.uploadUrl}`);
const stageRes = await handleFileDownloadRoutes(
new Request(stageUrl, {
method: "PUT",
headers: { Authorization: `Bearer ${auth.token}` },
body: new TextEncoder().encode("must not be staged"),
}),
stageUrl,
mockServer,
deps,
);
expect(stageRes!.status).toBe(403);
const uploadEntries = await readdir(join(dataDir, "uploads")).catch(() => [] as string[]);
expect(uploadEntries).toHaveLength(0);
} finally {
clientManager.deleteClient(clientId);
deleteUser(auth.userId);
await removeTempDataDir(dataDir);
}
});
test("download serve returns 403 when file_browser is revoked after intent creation", async () => {
const auth = await createOperatorToken({ fileBrowserAllowed: true });
const clientId = `client-${Date.now().toString(36)}`;
Overlord-Server/src/server/routes/file-download-routes.ts
+92 -13
View File
@@ -10,9 +10,8 @@ import { getConfig } from "../../config";
import { isAuthorizedAgentRequest } from "../agent-auth";
import { requireClientAccess, requireFeatureAccess, requirePermission } from "../../rbac";
import {
FILE_UPLOAD_INTENT_TTL_MS,
FILE_UPLOAD_PULL_TTL_MS,
UUID_TOKEN_RE,
getFileTransferLimits,
isSafeRemotePath,
makeIncrementalPullStream,
notifyPullWaiters,
@@ -37,6 +36,26 @@ type FileDownloadRouteDeps = {
downloadIntents: Map<string, DownloadIntent>;
};
class UploadStagingLimitError extends Error {
constructor(message: string, readonly status: number) {
super(message);
}
}
function activeStagedPulls() {
const now = Date.now();
return [...uploadPulls.values()].filter((pull) =>
pull.deleteFile && pull.expiresAt >= now,
);
}
function reservedStagedBytes(excludePullId = ""): number {
return activeStagedPulls().reduce((total, pull) => {
if (pull.id === excludePullId) return total;
return total + Math.max(0, pull.expectedTotal || 0, pull.state?.size || 0, pull.size || 0);
}, 0);
}
async function serveDownloadById(
req: Request,
downloadId: string,
@@ -155,6 +174,7 @@ async function serveDownloadById(
}
firstChunkReject(new Error("Download timed out"));
reject(new Error("Download timed out"));
try { target.ws.send(encodeMessage({ type: "command_abort", commandId } as any)); } catch {}
}, 5 * 60_000);
deps.pendingHttpDownloads.set(commandId, {
@@ -309,11 +329,19 @@ export async function handleFileDownloadRoutes(
return new Response("Client offline", { status: 404 });
}
const limits = getFileTransferLimits();
const activeForUser = [...uploadIntents.values()]
.filter((intent) => intent.userId === user.userId && intent.expiresAt >= Date.now()).length
+ activeStagedPulls().filter((pull) => pull.userId === user.userId).length;
if (activeForUser >= limits.maxActivePerUser) {
return new Response("Too many pending uploads", { status: 429, headers: { "Retry-After": "10" } });
}
const uploadId = uuidv4();
const expiresAt = Date.now() + FILE_UPLOAD_INTENT_TTL_MS;
const expiresAt = Date.now() + limits.uploadIntentTtlMs;
const timeout = setTimeout(() => {
uploadIntents.delete(uploadId);
}, FILE_UPLOAD_INTENT_TTL_MS);
}, limits.uploadIntentTtlMs);
uploadIntents.set(uploadId, {
id: uploadId,
@@ -372,25 +400,42 @@ export async function handleFileDownloadRoutes(
if (!intent || intent.userId !== user.userId || intent.expiresAt < Date.now()) {
return new Response("Not found", { status: 404 });
}
try {
requireClientAccess(user, intent.clientId);
} catch (error) {
if (error instanceof Response) return error;
return new Response("Forbidden", { status: 403 });
}
const uploadDir = path.join(deps.DATA_DIR, "uploads");
await fs.mkdir(uploadDir, { recursive: true });
const pullId = uuidv4();
const tmpPath = path.join(uploadDir, `${pullId}.bin`);
const limits = getFileTransferLimits();
const activePulls = activeStagedPulls();
if (activePulls.length >= limits.maxActiveGlobal
|| activePulls.filter((pull) => pull.userId === user.userId).length >= limits.maxActivePerUser) {
return new Response("Too many active staged uploads", { status: 429, headers: { "Retry-After": "10" } });
}
const contentLength = Number(req.headers.get("content-length") || 0);
const expectedTotal = Number.isFinite(contentLength) && contentLength >= 0 ? contentLength : 0;
if (expectedTotal > limits.maxFileBytes) {
return new Response("Upload exceeds the server staging limit", { status: 413 });
}
if (reservedStagedBytes() + expectedTotal > limits.maxStagedBytes) {
return new Response("Server upload staging quota exceeded", { status: 429, headers: { "Retry-After": "30" } });
}
const uploadDir = path.join(deps.DATA_DIR, "uploads");
const pullId = uuidv4();
const tmpPath = path.join(uploadDir, `${pullId}.bin`);
const canStream = !!req.body && expectedTotal > 0;
const pullExpiresAt = Date.now() + FILE_UPLOAD_PULL_TTL_MS;
const pullExpiresAt = Date.now() + limits.uploadPullTtlMs;
const pullTimeout = setTimeout(() => {
const stale = uploadPulls.get(pullId);
uploadPulls.delete(pullId);
if (stale && stale.deleteFile) {
void fs.unlink(stale.tmpPath).catch(() => {});
}
}, FILE_UPLOAD_PULL_TTL_MS);
}, limits.uploadPullTtlMs);
const state: StreamingPullState | undefined = canStream
? { size: 0, done: false, error: null, waiters: [] }
@@ -408,6 +453,7 @@ export async function handleFileDownloadRoutes(
deleteFile: true,
state,
expectedTotal,
userId: user.userId,
});
uploadIntents.delete(uploadId);
@@ -416,14 +462,27 @@ export async function handleFileDownloadRoutes(
const pullUrl = `/api/file/upload/pull/${encodeURIComponent(pullId)}`;
let stagedSize = 0;
const fileHandle = await fs.open(tmpPath, "w");
let fileHandle: Awaited<ReturnType<typeof fs.open>> | null = null;
try {
await fs.mkdir(uploadDir, { recursive: true });
fileHandle = await fs.open(tmpPath, "w");
if (req.body) {
const reader = req.body.getReader();
while (true) {
const { done, value } = await reader.read();
if (done) break;
if (!value || value.byteLength === 0) continue;
const currentPull = uploadPulls.get(pullId);
if (!currentPull || currentPull.expiresAt < Date.now()) {
throw new UploadStagingLimitError("Upload staging expired", 408);
}
if (stagedSize + value.byteLength > limits.maxFileBytes) {
throw new UploadStagingLimitError("Upload exceeds the server staging limit", 413);
}
if (reservedStagedBytes(pullId) + stagedSize + value.byteLength > limits.maxStagedBytes) {
throw new UploadStagingLimitError("Server upload staging quota exceeded", 429);
}
currentPull.size = Math.max(currentPull.size, stagedSize + value.byteLength);
await fileHandle.write(value, 0, value.byteLength, stagedSize);
stagedSize += value.byteLength;
if (state) {
@@ -434,6 +493,17 @@ export async function handleFileDownloadRoutes(
} else {
const bytes = new Uint8Array(await req.arrayBuffer());
if (bytes.byteLength > 0) {
if (bytes.byteLength > limits.maxFileBytes) {
throw new UploadStagingLimitError("Upload exceeds the server staging limit", 413);
}
if (reservedStagedBytes(pullId) + bytes.byteLength > limits.maxStagedBytes) {
throw new UploadStagingLimitError("Server upload staging quota exceeded", 429);
}
const currentPull = uploadPulls.get(pullId);
if (!currentPull || currentPull.expiresAt < Date.now()) {
throw new UploadStagingLimitError("Upload staging expired", 408);
}
currentPull.size = Math.max(currentPull.size, bytes.byteLength);
await fileHandle.write(bytes, 0, bytes.byteLength, 0);
stagedSize = bytes.byteLength;
if (state) {
@@ -452,10 +522,16 @@ export async function handleFileDownloadRoutes(
state.done = true;
notifyPullWaiters(state);
}
await fileHandle.close();
if (fileHandle) await fileHandle.close().catch(() => {});
await fs.unlink(tmpPath).catch(() => {});
uploadPulls.delete(pullId);
clearTimeout(pullTimeout);
if (err instanceof UploadStagingLimitError) {
return new Response(err.message, {
status: err.status,
...(err.status === 429 ? { headers: { "Retry-After": "30" } } : {}),
});
}
return new Response("Upload staging failed", { status: 500 });
}
await fileHandle.close();
@@ -467,6 +543,9 @@ export async function handleFileDownloadRoutes(
const finalPull = uploadPulls.get(pullId);
if (finalPull) {
finalPull.size = stagedSize;
} else {
await fs.unlink(tmpPath).catch(() => {});
return new Response("Upload staging expired", { status: 408 });
}
logger.debug("[filebrowser] http upload staged bytes", {
Overlord-Server/src/server/routes/misc-routes.ts
+35 -1
View File
@@ -1,7 +1,7 @@
import { authenticateRequest } from "../../auth";
import { AuditAction, getAuditLogs, logAudit } from "../../auditLog";
import { logger } from "../../logger";
import { getConfig, updateSecurityConfig, updateTlsConfig, updateOidcConfig, updateAppearanceConfig, updateChatConfig, getExportableConfig, importFullConfig, updateRegistrationConfig, updateBuildRateLimitConfig, updateThumbnailsConfig, updateInputArchiveConfig } from "../../config";
import { getConfig, updateSecurityConfig, updateTlsConfig, updateOidcConfig, updateAppearanceConfig, updateChatConfig, getExportableConfig, importFullConfig, updateRegistrationConfig, updateBuildRateLimitConfig, updateThumbnailsConfig, updateInputArchiveConfig, updateFileTransfersConfig } from "../../config";
import {
saveBrandingImage,
getClientMetricsSummary,
@@ -670,6 +670,40 @@ export async function handleMiscRoutes(
}
}
if (url.pathname === "/api/settings/file-transfers") {
const user = await authenticateRequest(req);
if (!user) return new Response("Unauthorized", { status: 401 });
try { requirePermission(user, "system:security"); } catch (error) { if (error instanceof Response) return error; return new Response("Forbidden", { status: 403 }); }
if (req.method === "GET") {
return Response.json({ fileTransfers: getConfig().fileTransfers }, { headers: deps.CORS_HEADERS });
}
if (req.method === "PUT") {
let body: any = {};
try { body = await req.json(); } catch {
return Response.json({ error: "Invalid JSON" }, { status: 400 });
}
const updated = await updateFileTransfersConfig({
maxFileBytes: body?.maxFileBytes !== undefined ? Number(body.maxFileBytes) : undefined,
maxStagedBytes: body?.maxStagedBytes !== undefined ? Number(body.maxStagedBytes) : undefined,
maxActiveGlobal: body?.maxActiveGlobal !== undefined ? Number(body.maxActiveGlobal) : undefined,
maxActivePerUser: body?.maxActivePerUser !== undefined ? Number(body.maxActivePerUser) : undefined,
uploadIntentTtlMs: body?.uploadIntentTtlMs !== undefined ? Number(body.uploadIntentTtlMs) : undefined,
uploadPullTtlMs: body?.uploadPullTtlMs !== undefined ? Number(body.uploadPullTtlMs) : undefined,
});
logAudit({
timestamp: Date.now(),
username: user.username,
ip: deps.requestIP?.(req)?.address || "unknown",
action: AuditAction.COMMAND,
details: "Updated file transfer limits",
success: true,
});
return Response.json({ ok: true, fileTransfers: updated }, { headers: deps.CORS_HEADERS });
}
}
if (url.pathname === "/api/settings/tls") {
const user = await authenticateRequest(req);
if (!user) {
Overlord-Server/src/server/ws-file-process-proxy-keylogger.test.ts
+32
View File
@@ -0,0 +1,32 @@
import { describe, expect, test } from "bun:test";
import { handleFileBrowserMessage } from "./ws-file-process-proxy-keylogger";
describe("file browser agent download routing", () => {
test("only consumes pending HTTP downloads for the matching client", () => {
let consumed = 0;
const payload = { type: "file_download", commandId: "download-1", data: new Uint8Array([1]) };
const deps = {
pendingHttpDownloads: new Map([["download-1", { clientId: "client-a" }]]),
consumeHttpDownloadPayload: () => { consumed += 1; },
};
handleFileBrowserMessage("client-b", payload, deps);
expect(consumed).toBe(0);
handleFileBrowserMessage("client-a", payload, deps);
expect(consumed).toBe(1);
});
test("drops unsolicited download payloads", () => {
let consumed = 0;
handleFileBrowserMessage(
"client-a",
{ type: "file_download", commandId: "unknown", data: new Uint8Array([1]) },
{
pendingHttpDownloads: new Map(),
consumeHttpDownloadPayload: () => { consumed += 1; },
},
);
expect(consumed).toBe(0);
});
});
Overlord-Server/src/server/ws-file-process-proxy-keylogger.ts
+11 -5
View File
@@ -45,7 +45,7 @@ type ProcessViewer = {
};
type WsViewerClusterDeps = {
pendingHttpDownloads: Map<string, unknown>;
pendingHttpDownloads: ReadonlyMap<string, { clientId?: string }>;
consumeHttpDownloadPayload: (payload: any) => Promise<void> | void;
};
@@ -397,17 +397,23 @@ export function handleFileBrowserViewerMessage(ws: ServerWebSocket<SocketData>,
export function handleFileBrowserMessage(clientId: string, payload: any, deps: WsViewerClusterDeps) {
const type = payload?.type as string | undefined;
const payloadCommandId = typeof payload?.commandId === "string" ? payload.commandId : undefined;
const pendingDownload = payloadCommandId
? deps.pendingHttpDownloads.get(payloadCommandId)
: undefined;
const isHttpDownload =
type === "file_download" &&
typeof payload?.commandId === "string" &&
deps.pendingHttpDownloads.has(payload.commandId);
pendingDownload?.clientId === clientId;
if (type === "file_download" && typeof payload?.commandId === "string") {
if (isHttpDownload) {
void deps.consumeHttpDownloadPayload(payload);
}
const payloadCommandId = typeof payload?.commandId === "string" ? payload.commandId : undefined;
const commandOwner = payloadCommandId ? fileBrowserCommandSessions.get(payloadCommandId) : undefined;
if (type === "file_download" && !isHttpDownload && !commandOwner) {
logger.debug(`[filebrowser] dropped unsolicited download payload client=${clientId}`);
return;
}
const ownerSessionId = commandOwner?.sessionId;
let hasSession = false;