add restrictions on file handler
This commit is contained in:
KingKDot
@@ -2118,6 +2118,88 @@ function initRegistrationHandlers() {
|
||||
});
|
||||
}
|
||||
|
||||
// ── File Transfer Limit Settings ──────────────────────────────────────────
|
||||
|
||||
async function loadFileTransferLimits() {
|
||||
if (!userHas("system:security")) return;
|
||||
try {
|
||||
const res = await fetch("/api/settings/file-transfers", { credentials: "include" });
|
||||
if (!res.ok) return;
|
||||
const data = await res.json();
|
||||
const limits = data.fileTransfers || {};
|
||||
const mb = 1024 * 1024;
|
||||
document.getElementById("ft-max-file-mb").value = Math.round((limits.maxFileBytes || 1024 * mb) / mb);
|
||||
document.getElementById("ft-max-staged-mb").value = Math.round((limits.maxStagedBytes || 4096 * mb) / mb);
|
||||
document.getElementById("ft-max-active-global").value = limits.maxActiveGlobal ?? 16;
|
||||
document.getElementById("ft-max-active-user").value = limits.maxActivePerUser ?? 4;
|
||||
document.getElementById("ft-intent-ttl-minutes").value = Math.round((limits.uploadIntentTtlMs || 30 * 60_000) / 60_000);
|
||||
document.getElementById("ft-pull-ttl-minutes").value = Math.round((limits.uploadPullTtlMs || 30 * 60_000) / 60_000);
|
||||
} catch (error) {
|
||||
console.error("Failed to load file transfer limits", error);
|
||||
}
|
||||
}
|
||||
|
||||
function showFileTransferLimitsMsg(text, type) {
|
||||
const el = document.getElementById("file-transfer-limits-msg");
|
||||
if (!el) return;
|
||||
el.textContent = text;
|
||||
el.className = `text-sm rounded-lg px-3 py-2 border ${type === "error" ? "border-rose-800 bg-rose-900/20 text-rose-200" : "border-emerald-800 bg-emerald-900/20 text-emerald-200"}`;
|
||||
el.classList.remove("hidden");
|
||||
setTimeout(() => el.classList.add("hidden"), 5000);
|
||||
}
|
||||
|
||||
async function saveFileTransferLimits(event) {
|
||||
event.preventDefault();
|
||||
if (!requireUiPermission("system:security", "Security settings permission required.")) return;
|
||||
const mb = 1024 * 1024;
|
||||
const maxFileMb = Number(document.getElementById("ft-max-file-mb")?.value);
|
||||
const maxStagedMb = Number(document.getElementById("ft-max-staged-mb")?.value);
|
||||
const maxActiveGlobal = Number(document.getElementById("ft-max-active-global")?.value);
|
||||
const maxActivePerUser = Number(document.getElementById("ft-max-active-user")?.value);
|
||||
const intentMinutes = Number(document.getElementById("ft-intent-ttl-minutes")?.value);
|
||||
const pullMinutes = Number(document.getElementById("ft-pull-ttl-minutes")?.value);
|
||||
if (![maxFileMb, maxStagedMb, maxActiveGlobal, maxActivePerUser, intentMinutes, pullMinutes].every(Number.isFinite)) {
|
||||
showFileTransferLimitsMsg("All limits must be valid numbers.", "error");
|
||||
return;
|
||||
}
|
||||
if (maxStagedMb < maxFileMb) {
|
||||
showFileTransferLimitsMsg("Total staging quota must be at least the maximum file size.", "error");
|
||||
return;
|
||||
}
|
||||
if (maxActivePerUser > maxActiveGlobal) {
|
||||
showFileTransferLimitsMsg("Per-user active transfers cannot exceed the global limit.", "error");
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const res = await fetch("/api/settings/file-transfers", {
|
||||
method: "PUT",
|
||||
credentials: "include",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
maxFileBytes: Math.round(maxFileMb * mb),
|
||||
maxStagedBytes: Math.round(maxStagedMb * mb),
|
||||
maxActiveGlobal: Math.round(maxActiveGlobal),
|
||||
maxActivePerUser: Math.round(maxActivePerUser),
|
||||
uploadIntentTtlMs: Math.round(intentMinutes * 60_000),
|
||||
uploadPullTtlMs: Math.round(pullMinutes * 60_000),
|
||||
}),
|
||||
});
|
||||
const data = await res.json().catch(() => ({}));
|
||||
if (!res.ok) {
|
||||
showFileTransferLimitsMsg(data.error || "Failed to save transfer limits.", "error");
|
||||
return;
|
||||
}
|
||||
await loadFileTransferLimits();
|
||||
showFileTransferLimitsMsg("File transfer limits saved and active.", "success");
|
||||
} catch {
|
||||
showFileTransferLimitsMsg("Network error.", "error");
|
||||
}
|
||||
}
|
||||
|
||||
function initFileTransferLimitHandlers() {
|
||||
document.getElementById("file-transfer-limits-form")?.addEventListener("submit", saveFileTransferLimits);
|
||||
}
|
||||
|
||||
// ── Build Rate Limit Settings ──────────────────────────────────────────────
|
||||
|
||||
async function loadBuildRateLimitSettings() {
|
||||
@@ -2599,6 +2681,10 @@ async function init() {
|
||||
await loadBuildRateLimitSettings();
|
||||
initBuildRateLimitHandlers();
|
||||
}
|
||||
if (userHas("system:security")) {
|
||||
await loadFileTransferLimits();
|
||||
initFileTransferLimitHandlers();
|
||||
}
|
||||
if (userHas("system:thumbnails")) {
|
||||
await loadThumbnailSettings();
|
||||
initThumbnailHandlers();
|
||||
|
||||
@@ -120,6 +120,7 @@
|
||||
<span class="settings-nav-group-label">Admin</span>
|
||||
<a data-target="section-permissions-overview" class="settings-nav-link" data-permission="users:manage" style="--accent: 167 139 250"><i class="fa-solid fa-user-shield w-4 text-violet-400"></i><span>Access Overview</span></a>
|
||||
<a data-target="section-security" class="settings-nav-link" data-permission="system:security" style="--accent: 129 140 248"><i class="fa-solid fa-shield-halved w-4 text-indigo-400"></i><span>Security Policy</span></a>
|
||||
<a data-target="file-transfer-limits-section" class="settings-nav-link" data-permission="system:security" style="--accent: 45 212 191"><i class="fa-solid fa-hard-drive w-4 text-teal-400"></i><span>File Transfer Limits</span></a>
|
||||
<a data-target="section-tls" class="settings-nav-link" data-permission="system:tls" style="--accent: 34 211 238"><i class="fa-solid fa-certificate w-4 text-cyan-400"></i><span>TLS Certificates</span></a>
|
||||
<a data-target="section-oidc" class="settings-nav-link" data-permission="system:oidc" style="--accent: 20 184 166"><i class="fa-solid fa-right-to-bracket w-4 text-teal-400"></i><span>OIDC / SSO</span></a>
|
||||
<a data-target="chat-settings-section" class="settings-nav-link" data-permission="system:chat" style="--accent: 56 189 248"><i class="fa-solid fa-comments w-4 text-sky-400"></i><span>Team Chat</span></a>
|
||||
@@ -721,6 +722,55 @@
|
||||
</form>
|
||||
</section>
|
||||
|
||||
<section id="file-transfer-limits-section" data-permission="system:security" class="hidden bg-slate-900/60 border border-slate-800 rounded-xl p-5 space-y-4 settings-section">
|
||||
<div class="flex items-center gap-3">
|
||||
<div class="flex items-center justify-center w-9 h-9 rounded-lg bg-teal-500/10 border border-teal-500/30">
|
||||
<i class="fa-solid fa-hard-drive text-teal-400"></i>
|
||||
</div>
|
||||
<div>
|
||||
<h2 class="text-lg font-semibold">File Transfer Limits</h2>
|
||||
<p class="text-sm text-slate-400">Bound temporary server disk usage for operator-to-agent uploads.</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="text-xs text-slate-400 bg-slate-800/40 border border-slate-700/60 rounded-lg p-3">
|
||||
Agent-to-operator downloads are streamed and are not staged on server disk. These limits apply to temporary files under the server uploads directory and take effect immediately for new transfers.
|
||||
</div>
|
||||
|
||||
<form id="file-transfer-limits-form" class="space-y-4">
|
||||
<div class="grid grid-cols-1 sm:grid-cols-2 lg:grid-cols-3 gap-3">
|
||||
<label class="block">
|
||||
<span class="text-sm text-slate-300">Maximum file size (MB)</span>
|
||||
<input id="ft-max-file-mb" type="number" min="1" max="1024" step="1" class="mt-1 w-full px-3 py-2 rounded-lg bg-slate-900 border border-slate-700 focus:outline-none focus:ring-2 focus:ring-teal-500" />
|
||||
</label>
|
||||
<label class="block">
|
||||
<span class="text-sm text-slate-300">Total staging quota (MB)</span>
|
||||
<input id="ft-max-staged-mb" type="number" min="1" max="512000" step="1" class="mt-1 w-full px-3 py-2 rounded-lg bg-slate-900 border border-slate-700 focus:outline-none focus:ring-2 focus:ring-teal-500" />
|
||||
</label>
|
||||
<label class="block">
|
||||
<span class="text-sm text-slate-300">Maximum active globally</span>
|
||||
<input id="ft-max-active-global" type="number" min="1" max="1000" step="1" class="mt-1 w-full px-3 py-2 rounded-lg bg-slate-900 border border-slate-700 focus:outline-none focus:ring-2 focus:ring-teal-500" />
|
||||
</label>
|
||||
<label class="block">
|
||||
<span class="text-sm text-slate-300">Maximum active per user</span>
|
||||
<input id="ft-max-active-user" type="number" min="1" max="1000" step="1" class="mt-1 w-full px-3 py-2 rounded-lg bg-slate-900 border border-slate-700 focus:outline-none focus:ring-2 focus:ring-teal-500" />
|
||||
</label>
|
||||
<label class="block">
|
||||
<span class="text-sm text-slate-300">Upload intent expiry (minutes)</span>
|
||||
<input id="ft-intent-ttl-minutes" type="number" min="1" max="1440" step="1" class="mt-1 w-full px-3 py-2 rounded-lg bg-slate-900 border border-slate-700 focus:outline-none focus:ring-2 focus:ring-teal-500" />
|
||||
</label>
|
||||
<label class="block">
|
||||
<span class="text-sm text-slate-300">Staged file expiry (minutes)</span>
|
||||
<input id="ft-pull-ttl-minutes" type="number" min="1" max="1440" step="1" class="mt-1 w-full px-3 py-2 rounded-lg bg-slate-900 border border-slate-700 focus:outline-none focus:ring-2 focus:ring-teal-500" />
|
||||
</label>
|
||||
</div>
|
||||
<button type="submit" class="inline-flex items-center gap-2 px-4 py-2 rounded-lg bg-teal-700 hover:bg-teal-600 transition-colors text-white text-sm font-medium">
|
||||
<i class="fa-solid fa-floppy-disk"></i>Save Transfer Limits
|
||||
</button>
|
||||
</form>
|
||||
<p id="file-transfer-limits-msg" class="hidden text-sm rounded-lg px-3 py-2 border"></p>
|
||||
</section>
|
||||
|
||||
<!-- TLS Certificates -->
|
||||
<section id="section-tls" data-permission="system:tls" class="hidden bg-slate-900/60 border border-slate-800 rounded-xl p-5 space-y-4 settings-section">
|
||||
<div class="flex items-center gap-3">
|
||||
|
||||
@@ -134,6 +134,14 @@ export interface Config {
|
||||
maxFileBytes: number;
|
||||
pollIntervalSeconds: number;
|
||||
};
|
||||
fileTransfers: {
|
||||
maxFileBytes: number;
|
||||
maxStagedBytes: number;
|
||||
maxActiveGlobal: number;
|
||||
maxActivePerUser: number;
|
||||
uploadIntentTtlMs: number;
|
||||
uploadPullTtlMs: number;
|
||||
};
|
||||
}
|
||||
|
||||
const DEFAULT_CONFIG: Config = {
|
||||
@@ -266,6 +274,14 @@ const DEFAULT_CONFIG: Config = {
|
||||
maxFileBytes: 5 * 1024 * 1024,
|
||||
pollIntervalSeconds: 300,
|
||||
},
|
||||
fileTransfers: {
|
||||
maxFileBytes: 1024 * 1024 * 1024,
|
||||
maxStagedBytes: 4 * 1024 * 1024 * 1024,
|
||||
maxActiveGlobal: 16,
|
||||
maxActivePerUser: 4,
|
||||
uploadIntentTtlMs: 30 * 60_000,
|
||||
uploadPullTtlMs: 30 * 60_000,
|
||||
},
|
||||
};
|
||||
|
||||
function envBoolOverride(name: string): boolean | undefined {
|
||||
@@ -959,6 +975,68 @@ export function loadConfig(): Config {
|
||||
),
|
||||
),
|
||||
},
|
||||
fileTransfers: (() => {
|
||||
const maxFileBytes = Math.min(
|
||||
1024 * 1024 * 1024,
|
||||
Math.max(
|
||||
1024 * 1024,
|
||||
Number(process.env.OVERLORD_FILE_UPLOAD_MAX_BYTES)
|
||||
|| Number(fileConfig.fileTransfers?.maxFileBytes)
|
||||
|| DEFAULT_CONFIG.fileTransfers.maxFileBytes,
|
||||
),
|
||||
);
|
||||
const maxStagedBytes = Math.min(
|
||||
500 * 1024 * 1024 * 1024,
|
||||
Math.max(
|
||||
maxFileBytes,
|
||||
Number(process.env.OVERLORD_FILE_UPLOAD_MAX_STAGED_BYTES)
|
||||
|| Number(fileConfig.fileTransfers?.maxStagedBytes)
|
||||
|| DEFAULT_CONFIG.fileTransfers.maxStagedBytes,
|
||||
),
|
||||
);
|
||||
const maxActiveGlobal = Math.min(
|
||||
1000,
|
||||
Math.max(
|
||||
1,
|
||||
Number(process.env.OVERLORD_FILE_UPLOAD_MAX_ACTIVE_GLOBAL)
|
||||
|| Number(fileConfig.fileTransfers?.maxActiveGlobal)
|
||||
|| DEFAULT_CONFIG.fileTransfers.maxActiveGlobal,
|
||||
),
|
||||
);
|
||||
const maxActivePerUser = Math.min(
|
||||
maxActiveGlobal,
|
||||
Math.max(
|
||||
1,
|
||||
Number(process.env.OVERLORD_FILE_UPLOAD_MAX_ACTIVE_PER_USER)
|
||||
|| Number(fileConfig.fileTransfers?.maxActivePerUser)
|
||||
|| DEFAULT_CONFIG.fileTransfers.maxActivePerUser,
|
||||
),
|
||||
);
|
||||
return {
|
||||
maxFileBytes: Math.floor(maxFileBytes),
|
||||
maxStagedBytes: Math.floor(maxStagedBytes),
|
||||
maxActiveGlobal: Math.floor(maxActiveGlobal),
|
||||
maxActivePerUser: Math.floor(maxActivePerUser),
|
||||
uploadIntentTtlMs: Math.min(
|
||||
24 * 60 * 60_000,
|
||||
Math.max(
|
||||
60_000,
|
||||
Number(process.env.OVERLORD_FILE_UPLOAD_INTENT_TTL_MS)
|
||||
|| Number(fileConfig.fileTransfers?.uploadIntentTtlMs)
|
||||
|| DEFAULT_CONFIG.fileTransfers.uploadIntentTtlMs,
|
||||
),
|
||||
),
|
||||
uploadPullTtlMs: Math.min(
|
||||
24 * 60 * 60_000,
|
||||
Math.max(
|
||||
60_000,
|
||||
Number(process.env.OVERLORD_FILE_UPLOAD_PULL_TTL_MS)
|
||||
|| Number(fileConfig.fileTransfers?.uploadPullTtlMs)
|
||||
|| DEFAULT_CONFIG.fileTransfers.uploadPullTtlMs,
|
||||
),
|
||||
),
|
||||
};
|
||||
})(),
|
||||
};
|
||||
|
||||
if (saveChanged) {
|
||||
@@ -1303,6 +1381,51 @@ export async function updateInputArchiveConfig(
|
||||
return next;
|
||||
}
|
||||
|
||||
export async function updateFileTransfersConfig(
|
||||
updates: Partial<Config["fileTransfers"]>,
|
||||
): Promise<Config["fileTransfers"]> {
|
||||
const current = getConfig();
|
||||
const numberOr = (value: unknown, fallback: number) => {
|
||||
const parsed = Number(value);
|
||||
return Number.isFinite(parsed) ? parsed : fallback;
|
||||
};
|
||||
const next: Config["fileTransfers"] = {
|
||||
...current.fileTransfers,
|
||||
...updates,
|
||||
};
|
||||
|
||||
next.maxFileBytes = Math.floor(Math.min(
|
||||
1024 * 1024 * 1024,
|
||||
Math.max(1024 * 1024, numberOr(next.maxFileBytes, DEFAULT_CONFIG.fileTransfers.maxFileBytes)),
|
||||
));
|
||||
next.maxStagedBytes = Math.floor(Math.min(
|
||||
500 * 1024 * 1024 * 1024,
|
||||
Math.max(next.maxFileBytes, numberOr(next.maxStagedBytes, DEFAULT_CONFIG.fileTransfers.maxStagedBytes)),
|
||||
));
|
||||
next.maxActiveGlobal = Math.floor(Math.min(
|
||||
1000,
|
||||
Math.max(1, numberOr(next.maxActiveGlobal, DEFAULT_CONFIG.fileTransfers.maxActiveGlobal)),
|
||||
));
|
||||
next.maxActivePerUser = Math.floor(Math.min(
|
||||
next.maxActiveGlobal,
|
||||
Math.max(1, numberOr(next.maxActivePerUser, DEFAULT_CONFIG.fileTransfers.maxActivePerUser)),
|
||||
));
|
||||
next.uploadIntentTtlMs = Math.floor(Math.min(
|
||||
24 * 60 * 60_000,
|
||||
Math.max(60_000, numberOr(next.uploadIntentTtlMs, DEFAULT_CONFIG.fileTransfers.uploadIntentTtlMs)),
|
||||
));
|
||||
next.uploadPullTtlMs = Math.floor(Math.min(
|
||||
24 * 60 * 60_000,
|
||||
Math.max(60_000, numberOr(next.uploadPullTtlMs, DEFAULT_CONFIG.fileTransfers.uploadPullTtlMs)),
|
||||
));
|
||||
|
||||
configCache = { ...current, fileTransfers: next };
|
||||
const fileConfig = readFileConfigForUpdate();
|
||||
fileConfig.fileTransfers = next;
|
||||
await writePersistentFileConfig(fileConfig);
|
||||
return next;
|
||||
}
|
||||
|
||||
export async function updateRegistrationConfig(
|
||||
updates: Partial<Config["registration"]>,
|
||||
): Promise<Config["registration"]> {
|
||||
@@ -1489,6 +1612,7 @@ export function getExportableConfig(serverVersion: string): Record<string, unkno
|
||||
},
|
||||
thumbnails: config.thumbnails,
|
||||
inputArchive: config.inputArchive,
|
||||
fileTransfers: config.fileTransfers,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1619,6 +1743,11 @@ export async function importFullConfig(data: Record<string, any>): Promise<{ app
|
||||
applied.push("inputArchive");
|
||||
}
|
||||
|
||||
if (data.fileTransfers && typeof data.fileTransfers === "object") {
|
||||
await updateFileTransfersConfig(data.fileTransfers);
|
||||
applied.push("fileTransfers");
|
||||
}
|
||||
|
||||
if (data.auth && typeof data.auth === "object") {
|
||||
const dataDir = ensureDataDir();
|
||||
const savePath = resolve(dataDir, "save.json");
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
import { afterAll, beforeAll, describe, expect, test } from "bun:test";
|
||||
import { getConfig, updateFileTransfersConfig, type Config } from "./config";
|
||||
import { getFileTransferLimits } from "./server/file-transfer-state";
|
||||
|
||||
let original: Config["fileTransfers"];
|
||||
|
||||
beforeAll(() => {
|
||||
original = { ...getConfig().fileTransfers };
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await updateFileTransfersConfig(original);
|
||||
});
|
||||
|
||||
describe("file transfer settings", () => {
|
||||
test("saved limits are visible to transfer handlers immediately", async () => {
|
||||
await updateFileTransfersConfig({
|
||||
maxFileBytes: 25 * 1024 * 1024,
|
||||
maxStagedBytes: 100 * 1024 * 1024,
|
||||
maxActiveGlobal: 8,
|
||||
maxActivePerUser: 2,
|
||||
uploadIntentTtlMs: 5 * 60_000,
|
||||
uploadPullTtlMs: 10 * 60_000,
|
||||
});
|
||||
|
||||
expect(getFileTransferLimits()).toEqual({
|
||||
maxFileBytes: 25 * 1024 * 1024,
|
||||
maxStagedBytes: 100 * 1024 * 1024,
|
||||
maxActiveGlobal: 8,
|
||||
maxActivePerUser: 2,
|
||||
uploadIntentTtlMs: 5 * 60_000,
|
||||
uploadPullTtlMs: 10 * 60_000,
|
||||
});
|
||||
});
|
||||
|
||||
test("dependent limits are clamped to safe relationships", async () => {
|
||||
const updated = await updateFileTransfersConfig({
|
||||
maxFileBytes: 20 * 1024 * 1024,
|
||||
maxStagedBytes: 10 * 1024 * 1024,
|
||||
maxActiveGlobal: 3,
|
||||
maxActivePerUser: 20,
|
||||
uploadIntentTtlMs: 1,
|
||||
uploadPullTtlMs: 1,
|
||||
});
|
||||
|
||||
expect(updated.maxStagedBytes).toBe(updated.maxFileBytes);
|
||||
expect(updated.maxActivePerUser).toBe(updated.maxActiveGlobal);
|
||||
expect(updated.uploadIntentTtlMs).toBe(60_000);
|
||||
expect(updated.uploadPullTtlMs).toBe(60_000);
|
||||
});
|
||||
});
|
||||
@@ -2,6 +2,7 @@ import fs from "fs/promises";
|
||||
import type { FileHandle } from "fs/promises";
|
||||
import path from "path";
|
||||
import { v4 as uuidv4 } from "uuid";
|
||||
import { getConfig } from "../config";
|
||||
|
||||
export type PendingHttpDownload = {
|
||||
commandId: string;
|
||||
@@ -71,27 +72,14 @@ export type UploadPull = {
|
||||
deleteFile: boolean;
|
||||
state?: StreamingPullState;
|
||||
expectedTotal?: number;
|
||||
userId?: number;
|
||||
};
|
||||
|
||||
function parsePositiveIntEnv(name: string, fallback: number): number {
|
||||
const raw = process.env[name];
|
||||
if (!raw) return fallback;
|
||||
const parsed = Number(raw);
|
||||
if (!Number.isFinite(parsed) || parsed <= 0) return fallback;
|
||||
return Math.floor(parsed);
|
||||
}
|
||||
|
||||
export const UUID_TOKEN_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
|
||||
|
||||
// WAN uploads can be significantly slower than LAN transfers.
|
||||
export const FILE_UPLOAD_INTENT_TTL_MS = parsePositiveIntEnv(
|
||||
"OVERLORD_FILE_UPLOAD_INTENT_TTL_MS",
|
||||
30 * 60_000,
|
||||
);
|
||||
export const FILE_UPLOAD_PULL_TTL_MS = parsePositiveIntEnv(
|
||||
"OVERLORD_FILE_UPLOAD_PULL_TTL_MS",
|
||||
30 * 60_000,
|
||||
);
|
||||
export function getFileTransferLimits() {
|
||||
return getConfig().fileTransfers;
|
||||
}
|
||||
|
||||
export const uploadIntents = new Map<string, UploadIntent>();
|
||||
export const uploadPulls = new Map<string, UploadPull>();
|
||||
@@ -260,7 +248,7 @@ export function createUploadPull(opts: {
|
||||
ttlMs?: number;
|
||||
}): string {
|
||||
const pullId = uuidv4();
|
||||
const ttl = opts.ttlMs ?? FILE_UPLOAD_PULL_TTL_MS;
|
||||
const ttl = opts.ttlMs ?? getFileTransferLimits().uploadPullTtlMs;
|
||||
const expiresAt = Date.now() + ttl;
|
||||
const pullTimeout = setTimeout(() => {
|
||||
uploadPulls.delete(pullId);
|
||||
|
||||
@@ -2,6 +2,7 @@ import { describe, expect, test } from "bun:test";
|
||||
import {
|
||||
consumeHttpDownloadPayload,
|
||||
STREAM_MAX_CHUNK_BYTES,
|
||||
STREAM_OUTPUT_QUEUE_MAX_CHUNKS,
|
||||
type PendingHttpDownload,
|
||||
} from "./http-download-consumer";
|
||||
|
||||
@@ -55,4 +56,32 @@ describe("HTTP download stream limits", () => {
|
||||
expect(entries.has("cmd")).toBeFalse();
|
||||
expect(fixture.state().streamError).toContain("requested byte limit");
|
||||
});
|
||||
|
||||
test("bounds queued output when the HTTP receiver stops consuming", async () => {
|
||||
const fixture = pendingDownload({
|
||||
streamController: {
|
||||
desiredSize: -STREAM_OUTPUT_QUEUE_MAX_CHUNKS,
|
||||
enqueue() {},
|
||||
close() {},
|
||||
error(error: Error) { fixtureError = error.message; },
|
||||
} as any,
|
||||
});
|
||||
let fixtureError = "";
|
||||
const entries = new Map([[fixture.pending.commandId, fixture.pending]]);
|
||||
await consumeHttpDownloadPayload({
|
||||
commandId: "cmd", offset: 0, total: 4, data: new Uint8Array(4),
|
||||
}, entries);
|
||||
clearTimeout(fixture.pending.timeout);
|
||||
expect(entries.has("cmd")).toBeFalse();
|
||||
expect(fixtureError).toContain("not consuming");
|
||||
});
|
||||
|
||||
test("rejects malformed encoded chunks without throwing", async () => {
|
||||
const fixture = pendingDownload();
|
||||
const entries = new Map([[fixture.pending.commandId, fixture.pending]]);
|
||||
await consumeHttpDownloadPayload({ commandId: "cmd", data: "%%%not-base64%%%" }, entries);
|
||||
clearTimeout(fixture.pending.timeout);
|
||||
expect(entries.has("cmd")).toBeFalse();
|
||||
expect(fixture.state().streamError).toContain("encoding");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -5,6 +5,7 @@ import { logger } from "../logger";
|
||||
export const STREAM_REORDER_BUFFER_LIMIT = 16;
|
||||
export const STREAM_REORDER_BUFFER_MAX_BYTES = 32 * 1024 * 1024;
|
||||
export const STREAM_MAX_CHUNK_BYTES = 4 * 1024 * 1024;
|
||||
export const STREAM_OUTPUT_QUEUE_MAX_CHUNKS = 16;
|
||||
|
||||
export type PendingHttpDownload = {
|
||||
commandId: string;
|
||||
@@ -54,7 +55,14 @@ async function failStream(pending: PendingHttpDownload, err: Error): Promise<voi
|
||||
|
||||
function enqueueOrdered(pending: PendingHttpDownload, offset: number, data: Uint8Array): boolean {
|
||||
if (!pending.streamController || pending.streamErrored) return false;
|
||||
const outputQueueFull = () => {
|
||||
const desiredSize = pending.streamController?.desiredSize;
|
||||
if (typeof desiredSize !== "number" || desiredSize > -STREAM_OUTPUT_QUEUE_MAX_CHUNKS) return false;
|
||||
void failStream(pending, new Error("download receiver is not consuming data fast enough"));
|
||||
return true;
|
||||
};
|
||||
if (offset === (pending.nextExpectedOffset ?? 0)) {
|
||||
if (outputQueueFull()) return false;
|
||||
try {
|
||||
pending.streamController.enqueue(data);
|
||||
} catch (err) {
|
||||
@@ -67,6 +75,7 @@ function enqueueOrdered(pending: PendingHttpDownload, offset: number, data: Uint
|
||||
const next = pending.reorderBuffer.get(pending.nextExpectedOffset ?? 0);
|
||||
if (!next) break;
|
||||
pending.reorderBuffer.delete(pending.nextExpectedOffset ?? 0);
|
||||
if (outputQueueFull()) return false;
|
||||
try {
|
||||
pending.streamController.enqueue(next);
|
||||
} catch (err) {
|
||||
@@ -120,7 +129,7 @@ export async function consumeHttpDownloadPayload(
|
||||
if (!pending) return;
|
||||
|
||||
if (payload?.error) {
|
||||
const err = new Error(String(payload.error));
|
||||
const err = new Error(String(payload.error).slice(0, 1024));
|
||||
if (isStreamMode(pending)) {
|
||||
await failStream(pending, err);
|
||||
clearTimeout(pending.timeout);
|
||||
@@ -134,7 +143,7 @@ export async function consumeHttpDownloadPayload(
|
||||
}
|
||||
|
||||
const toNumber = (value: unknown): number | null => {
|
||||
if (typeof value === "number" && Number.isFinite(value)) return value;
|
||||
if (typeof value === "number" && Number.isSafeInteger(value)) return value;
|
||||
if (typeof value === "bigint") {
|
||||
const asNumber = Number(value);
|
||||
if (Number.isSafeInteger(asNumber)) return asNumber;
|
||||
@@ -168,12 +177,20 @@ export async function consumeHttpDownloadPayload(
|
||||
|
||||
if (payload?.data) {
|
||||
let data = payload.data;
|
||||
if (data instanceof ArrayBuffer) {
|
||||
data = new Uint8Array(data);
|
||||
} else if (typeof data === "string") {
|
||||
data = Uint8Array.from(atob(data), (c) => c.charCodeAt(0));
|
||||
} else if (ArrayBuffer.isView(data)) {
|
||||
data = new Uint8Array(data.buffer, data.byteOffset, data.byteLength);
|
||||
try {
|
||||
if (data instanceof ArrayBuffer) {
|
||||
data = new Uint8Array(data);
|
||||
} else if (typeof data === "string") {
|
||||
data = Uint8Array.from(atob(data), (c) => c.charCodeAt(0));
|
||||
} else if (ArrayBuffer.isView(data)) {
|
||||
data = new Uint8Array(data.buffer, data.byteOffset, data.byteLength);
|
||||
}
|
||||
} catch {
|
||||
await failStream(pending, new Error("invalid download chunk encoding"));
|
||||
clearTimeout(pending.timeout);
|
||||
pendingHttpDownloads.delete(commandId);
|
||||
pending.resolve(pending);
|
||||
return;
|
||||
}
|
||||
|
||||
if (data instanceof Uint8Array) {
|
||||
|
||||
@@ -12,6 +12,7 @@ import {
|
||||
setUserFeaturePermission,
|
||||
} from "../../users";
|
||||
import { handleFileDownloadRoutes } from "./file-download-routes";
|
||||
import { getFileTransferLimits } from "../file-transfer-state";
|
||||
|
||||
const PASSWORD = "Aa1!RouteUploadTestPass123";
|
||||
|
||||
@@ -58,6 +59,114 @@ async function removeTempDataDir(dir: string): Promise<void> {
|
||||
}
|
||||
|
||||
describe("file upload route flow", () => {
|
||||
test("consumes an upload intent atomically when staging requests race", async () => {
|
||||
const auth = await createAdminToken();
|
||||
const clientId = `client-${Date.now().toString(36)}`;
|
||||
const dataDir = await createTempDataDir();
|
||||
const deps = makeRouteDeps(dataDir);
|
||||
clientManager.addClient(clientId, {
|
||||
id: clientId, lastSeen: Date.now(), role: "client", ws: { send: () => {} },
|
||||
});
|
||||
const previousDisableAgentAuth = process.env.OVERLORD_DISABLE_AGENT_AUTH;
|
||||
process.env.OVERLORD_DISABLE_AGENT_AUTH = "true";
|
||||
|
||||
try {
|
||||
const requestUrl = new URL("https://localhost/api/file/upload/request");
|
||||
const requestRes = await handleFileDownloadRoutes(
|
||||
new Request(requestUrl, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json", Authorization: `Bearer ${auth.token}` },
|
||||
body: JSON.stringify({ clientId, path: "C:\\Games\\race.bin", fileName: "race.bin" }),
|
||||
}),
|
||||
requestUrl,
|
||||
mockServer,
|
||||
deps,
|
||||
);
|
||||
const requestJson = await requestRes!.json() as any;
|
||||
const stageUrl = new URL(`https://localhost${requestJson.uploadUrl}`);
|
||||
const makeStageRequest = () => new Request(stageUrl, {
|
||||
method: "PUT",
|
||||
headers: { Authorization: `Bearer ${auth.token}` },
|
||||
body: new TextEncoder().encode("race payload"),
|
||||
});
|
||||
const responses = await Promise.all([
|
||||
handleFileDownloadRoutes(makeStageRequest(), stageUrl, mockServer, deps),
|
||||
handleFileDownloadRoutes(makeStageRequest(), stageUrl, mockServer, deps),
|
||||
]);
|
||||
|
||||
expect(responses.map((response) => response!.status).sort()).toEqual([200, 404]);
|
||||
const successful = responses.find((response) => response!.status === 200)!;
|
||||
const staged = await successful!.json() as any;
|
||||
const pullUrl = new URL(staged.pullUrl, "https://localhost");
|
||||
const pullResponse = await handleFileDownloadRoutes(
|
||||
new Request(pullUrl, { headers: { "x-overlord-client-id": clientId } }),
|
||||
pullUrl,
|
||||
mockServer,
|
||||
deps,
|
||||
);
|
||||
expect(await pullResponse!.text()).toBe("race payload");
|
||||
} finally {
|
||||
if (previousDisableAgentAuth === undefined) delete process.env.OVERLORD_DISABLE_AGENT_AUTH;
|
||||
else process.env.OVERLORD_DISABLE_AGENT_AUTH = previousDisableAgentAuth;
|
||||
clientManager.deleteClient(clientId);
|
||||
deleteUser(auth.userId);
|
||||
await removeTempDataDir(dataDir);
|
||||
}
|
||||
});
|
||||
|
||||
test("rejects an oversized staged upload before writing a payload file", async () => {
|
||||
const auth = await createAdminToken();
|
||||
const clientId = `client-${Date.now().toString(36)}`;
|
||||
const dataDir = await createTempDataDir();
|
||||
const deps = makeRouteDeps(dataDir);
|
||||
|
||||
clientManager.addClient(clientId, {
|
||||
id: clientId,
|
||||
lastSeen: Date.now(),
|
||||
role: "client",
|
||||
ws: { send: () => {} },
|
||||
});
|
||||
|
||||
try {
|
||||
const requestUrl = new URL("https://localhost/api/file/upload/request");
|
||||
const requestRes = await handleFileDownloadRoutes(
|
||||
new Request(requestUrl, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
Authorization: `Bearer ${auth.token}`,
|
||||
},
|
||||
body: JSON.stringify({ clientId, path: "C:\\Games\\too-big.iso", fileName: "too-big.iso" }),
|
||||
}),
|
||||
requestUrl,
|
||||
mockServer,
|
||||
deps,
|
||||
);
|
||||
const requestJson = await requestRes!.json() as any;
|
||||
const stageUrl = new URL(`https://localhost${requestJson.uploadUrl}`);
|
||||
const stageRes = await handleFileDownloadRoutes(
|
||||
new Request(stageUrl, {
|
||||
method: "PUT",
|
||||
headers: {
|
||||
Authorization: `Bearer ${auth.token}`,
|
||||
"Content-Length": String(getFileTransferLimits().maxFileBytes + 1),
|
||||
},
|
||||
}),
|
||||
stageUrl,
|
||||
mockServer,
|
||||
deps,
|
||||
);
|
||||
|
||||
expect(stageRes!.status).toBe(413);
|
||||
const uploadEntries = await readdir(join(dataDir, "uploads")).catch(() => [] as string[]);
|
||||
expect(uploadEntries).toHaveLength(0);
|
||||
} finally {
|
||||
clientManager.deleteClient(clientId);
|
||||
deleteUser(auth.userId);
|
||||
await removeTempDataDir(dataDir);
|
||||
}
|
||||
});
|
||||
|
||||
test("stages and serves one-time upload payload for matching client", async () => {
|
||||
const auth = await createAdminToken();
|
||||
const clientId = `client-${Date.now().toString(36)}`;
|
||||
@@ -586,6 +695,52 @@ describe("file route feature_browser enforcement", () => {
|
||||
}
|
||||
});
|
||||
|
||||
test("upload stage PUT rechecks client access after intent creation", async () => {
|
||||
const auth = await createOperatorToken({ fileBrowserAllowed: true });
|
||||
const clientId = `client-${Date.now().toString(36)}`;
|
||||
const dataDir = await createTempDataDir();
|
||||
const deps = makeRouteDeps(dataDir);
|
||||
clientManager.addClient(clientId, {
|
||||
id: clientId, lastSeen: Date.now(), role: "client", ws: { send: () => {} },
|
||||
});
|
||||
|
||||
try {
|
||||
const requestUrl = new URL("https://localhost/api/file/upload/request");
|
||||
const requestRes = await handleFileDownloadRoutes(
|
||||
new Request(requestUrl, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json", Authorization: `Bearer ${auth.token}` },
|
||||
body: JSON.stringify({ clientId, path: "C:\\Games\\blocked.bin", fileName: "blocked.bin" }),
|
||||
}),
|
||||
requestUrl,
|
||||
mockServer,
|
||||
deps,
|
||||
);
|
||||
const requestJson = await requestRes!.json() as any;
|
||||
setUserClientAccessScope(auth.userId, "none");
|
||||
|
||||
const stageUrl = new URL(`https://localhost${requestJson.uploadUrl}`);
|
||||
const stageRes = await handleFileDownloadRoutes(
|
||||
new Request(stageUrl, {
|
||||
method: "PUT",
|
||||
headers: { Authorization: `Bearer ${auth.token}` },
|
||||
body: new TextEncoder().encode("must not be staged"),
|
||||
}),
|
||||
stageUrl,
|
||||
mockServer,
|
||||
deps,
|
||||
);
|
||||
|
||||
expect(stageRes!.status).toBe(403);
|
||||
const uploadEntries = await readdir(join(dataDir, "uploads")).catch(() => [] as string[]);
|
||||
expect(uploadEntries).toHaveLength(0);
|
||||
} finally {
|
||||
clientManager.deleteClient(clientId);
|
||||
deleteUser(auth.userId);
|
||||
await removeTempDataDir(dataDir);
|
||||
}
|
||||
});
|
||||
|
||||
test("download serve returns 403 when file_browser is revoked after intent creation", async () => {
|
||||
const auth = await createOperatorToken({ fileBrowserAllowed: true });
|
||||
const clientId = `client-${Date.now().toString(36)}`;
|
||||
|
||||
@@ -10,9 +10,8 @@ import { getConfig } from "../../config";
|
||||
import { isAuthorizedAgentRequest } from "../agent-auth";
|
||||
import { requireClientAccess, requireFeatureAccess, requirePermission } from "../../rbac";
|
||||
import {
|
||||
FILE_UPLOAD_INTENT_TTL_MS,
|
||||
FILE_UPLOAD_PULL_TTL_MS,
|
||||
UUID_TOKEN_RE,
|
||||
getFileTransferLimits,
|
||||
isSafeRemotePath,
|
||||
makeIncrementalPullStream,
|
||||
notifyPullWaiters,
|
||||
@@ -37,6 +36,26 @@ type FileDownloadRouteDeps = {
|
||||
downloadIntents: Map<string, DownloadIntent>;
|
||||
};
|
||||
|
||||
class UploadStagingLimitError extends Error {
|
||||
constructor(message: string, readonly status: number) {
|
||||
super(message);
|
||||
}
|
||||
}
|
||||
|
||||
function activeStagedPulls() {
|
||||
const now = Date.now();
|
||||
return [...uploadPulls.values()].filter((pull) =>
|
||||
pull.deleteFile && pull.expiresAt >= now,
|
||||
);
|
||||
}
|
||||
|
||||
function reservedStagedBytes(excludePullId = ""): number {
|
||||
return activeStagedPulls().reduce((total, pull) => {
|
||||
if (pull.id === excludePullId) return total;
|
||||
return total + Math.max(0, pull.expectedTotal || 0, pull.state?.size || 0, pull.size || 0);
|
||||
}, 0);
|
||||
}
|
||||
|
||||
async function serveDownloadById(
|
||||
req: Request,
|
||||
downloadId: string,
|
||||
@@ -155,6 +174,7 @@ async function serveDownloadById(
|
||||
}
|
||||
firstChunkReject(new Error("Download timed out"));
|
||||
reject(new Error("Download timed out"));
|
||||
try { target.ws.send(encodeMessage({ type: "command_abort", commandId } as any)); } catch {}
|
||||
}, 5 * 60_000);
|
||||
|
||||
deps.pendingHttpDownloads.set(commandId, {
|
||||
@@ -309,11 +329,19 @@ export async function handleFileDownloadRoutes(
|
||||
return new Response("Client offline", { status: 404 });
|
||||
}
|
||||
|
||||
const limits = getFileTransferLimits();
|
||||
const activeForUser = [...uploadIntents.values()]
|
||||
.filter((intent) => intent.userId === user.userId && intent.expiresAt >= Date.now()).length
|
||||
+ activeStagedPulls().filter((pull) => pull.userId === user.userId).length;
|
||||
if (activeForUser >= limits.maxActivePerUser) {
|
||||
return new Response("Too many pending uploads", { status: 429, headers: { "Retry-After": "10" } });
|
||||
}
|
||||
|
||||
const uploadId = uuidv4();
|
||||
const expiresAt = Date.now() + FILE_UPLOAD_INTENT_TTL_MS;
|
||||
const expiresAt = Date.now() + limits.uploadIntentTtlMs;
|
||||
const timeout = setTimeout(() => {
|
||||
uploadIntents.delete(uploadId);
|
||||
}, FILE_UPLOAD_INTENT_TTL_MS);
|
||||
}, limits.uploadIntentTtlMs);
|
||||
|
||||
uploadIntents.set(uploadId, {
|
||||
id: uploadId,
|
||||
@@ -372,25 +400,42 @@ export async function handleFileDownloadRoutes(
|
||||
if (!intent || intent.userId !== user.userId || intent.expiresAt < Date.now()) {
|
||||
return new Response("Not found", { status: 404 });
|
||||
}
|
||||
try {
|
||||
requireClientAccess(user, intent.clientId);
|
||||
} catch (error) {
|
||||
if (error instanceof Response) return error;
|
||||
return new Response("Forbidden", { status: 403 });
|
||||
}
|
||||
|
||||
const uploadDir = path.join(deps.DATA_DIR, "uploads");
|
||||
await fs.mkdir(uploadDir, { recursive: true });
|
||||
|
||||
const pullId = uuidv4();
|
||||
const tmpPath = path.join(uploadDir, `${pullId}.bin`);
|
||||
const limits = getFileTransferLimits();
|
||||
const activePulls = activeStagedPulls();
|
||||
if (activePulls.length >= limits.maxActiveGlobal
|
||||
|| activePulls.filter((pull) => pull.userId === user.userId).length >= limits.maxActivePerUser) {
|
||||
return new Response("Too many active staged uploads", { status: 429, headers: { "Retry-After": "10" } });
|
||||
}
|
||||
|
||||
const contentLength = Number(req.headers.get("content-length") || 0);
|
||||
const expectedTotal = Number.isFinite(contentLength) && contentLength >= 0 ? contentLength : 0;
|
||||
if (expectedTotal > limits.maxFileBytes) {
|
||||
return new Response("Upload exceeds the server staging limit", { status: 413 });
|
||||
}
|
||||
if (reservedStagedBytes() + expectedTotal > limits.maxStagedBytes) {
|
||||
return new Response("Server upload staging quota exceeded", { status: 429, headers: { "Retry-After": "30" } });
|
||||
}
|
||||
|
||||
const uploadDir = path.join(deps.DATA_DIR, "uploads");
|
||||
const pullId = uuidv4();
|
||||
const tmpPath = path.join(uploadDir, `${pullId}.bin`);
|
||||
const canStream = !!req.body && expectedTotal > 0;
|
||||
|
||||
const pullExpiresAt = Date.now() + FILE_UPLOAD_PULL_TTL_MS;
|
||||
const pullExpiresAt = Date.now() + limits.uploadPullTtlMs;
|
||||
const pullTimeout = setTimeout(() => {
|
||||
const stale = uploadPulls.get(pullId);
|
||||
uploadPulls.delete(pullId);
|
||||
if (stale && stale.deleteFile) {
|
||||
void fs.unlink(stale.tmpPath).catch(() => {});
|
||||
}
|
||||
}, FILE_UPLOAD_PULL_TTL_MS);
|
||||
}, limits.uploadPullTtlMs);
|
||||
|
||||
const state: StreamingPullState | undefined = canStream
|
||||
? { size: 0, done: false, error: null, waiters: [] }
|
||||
@@ -408,6 +453,7 @@ export async function handleFileDownloadRoutes(
|
||||
deleteFile: true,
|
||||
state,
|
||||
expectedTotal,
|
||||
userId: user.userId,
|
||||
});
|
||||
|
||||
uploadIntents.delete(uploadId);
|
||||
@@ -416,14 +462,27 @@ export async function handleFileDownloadRoutes(
|
||||
const pullUrl = `/api/file/upload/pull/${encodeURIComponent(pullId)}`;
|
||||
|
||||
let stagedSize = 0;
|
||||
const fileHandle = await fs.open(tmpPath, "w");
|
||||
let fileHandle: Awaited<ReturnType<typeof fs.open>> | null = null;
|
||||
try {
|
||||
await fs.mkdir(uploadDir, { recursive: true });
|
||||
fileHandle = await fs.open(tmpPath, "w");
|
||||
if (req.body) {
|
||||
const reader = req.body.getReader();
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
if (!value || value.byteLength === 0) continue;
|
||||
const currentPull = uploadPulls.get(pullId);
|
||||
if (!currentPull || currentPull.expiresAt < Date.now()) {
|
||||
throw new UploadStagingLimitError("Upload staging expired", 408);
|
||||
}
|
||||
if (stagedSize + value.byteLength > limits.maxFileBytes) {
|
||||
throw new UploadStagingLimitError("Upload exceeds the server staging limit", 413);
|
||||
}
|
||||
if (reservedStagedBytes(pullId) + stagedSize + value.byteLength > limits.maxStagedBytes) {
|
||||
throw new UploadStagingLimitError("Server upload staging quota exceeded", 429);
|
||||
}
|
||||
currentPull.size = Math.max(currentPull.size, stagedSize + value.byteLength);
|
||||
await fileHandle.write(value, 0, value.byteLength, stagedSize);
|
||||
stagedSize += value.byteLength;
|
||||
if (state) {
|
||||
@@ -434,6 +493,17 @@ export async function handleFileDownloadRoutes(
|
||||
} else {
|
||||
const bytes = new Uint8Array(await req.arrayBuffer());
|
||||
if (bytes.byteLength > 0) {
|
||||
if (bytes.byteLength > limits.maxFileBytes) {
|
||||
throw new UploadStagingLimitError("Upload exceeds the server staging limit", 413);
|
||||
}
|
||||
if (reservedStagedBytes(pullId) + bytes.byteLength > limits.maxStagedBytes) {
|
||||
throw new UploadStagingLimitError("Server upload staging quota exceeded", 429);
|
||||
}
|
||||
const currentPull = uploadPulls.get(pullId);
|
||||
if (!currentPull || currentPull.expiresAt < Date.now()) {
|
||||
throw new UploadStagingLimitError("Upload staging expired", 408);
|
||||
}
|
||||
currentPull.size = Math.max(currentPull.size, bytes.byteLength);
|
||||
await fileHandle.write(bytes, 0, bytes.byteLength, 0);
|
||||
stagedSize = bytes.byteLength;
|
||||
if (state) {
|
||||
@@ -452,10 +522,16 @@ export async function handleFileDownloadRoutes(
|
||||
state.done = true;
|
||||
notifyPullWaiters(state);
|
||||
}
|
||||
await fileHandle.close();
|
||||
if (fileHandle) await fileHandle.close().catch(() => {});
|
||||
await fs.unlink(tmpPath).catch(() => {});
|
||||
uploadPulls.delete(pullId);
|
||||
clearTimeout(pullTimeout);
|
||||
if (err instanceof UploadStagingLimitError) {
|
||||
return new Response(err.message, {
|
||||
status: err.status,
|
||||
...(err.status === 429 ? { headers: { "Retry-After": "30" } } : {}),
|
||||
});
|
||||
}
|
||||
return new Response("Upload staging failed", { status: 500 });
|
||||
}
|
||||
await fileHandle.close();
|
||||
@@ -467,6 +543,9 @@ export async function handleFileDownloadRoutes(
|
||||
const finalPull = uploadPulls.get(pullId);
|
||||
if (finalPull) {
|
||||
finalPull.size = stagedSize;
|
||||
} else {
|
||||
await fs.unlink(tmpPath).catch(() => {});
|
||||
return new Response("Upload staging expired", { status: 408 });
|
||||
}
|
||||
|
||||
logger.debug("[filebrowser] http upload staged bytes", {
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { authenticateRequest } from "../../auth";
|
||||
import { AuditAction, getAuditLogs, logAudit } from "../../auditLog";
|
||||
import { logger } from "../../logger";
|
||||
import { getConfig, updateSecurityConfig, updateTlsConfig, updateOidcConfig, updateAppearanceConfig, updateChatConfig, getExportableConfig, importFullConfig, updateRegistrationConfig, updateBuildRateLimitConfig, updateThumbnailsConfig, updateInputArchiveConfig } from "../../config";
|
||||
import { getConfig, updateSecurityConfig, updateTlsConfig, updateOidcConfig, updateAppearanceConfig, updateChatConfig, getExportableConfig, importFullConfig, updateRegistrationConfig, updateBuildRateLimitConfig, updateThumbnailsConfig, updateInputArchiveConfig, updateFileTransfersConfig } from "../../config";
|
||||
import {
|
||||
saveBrandingImage,
|
||||
getClientMetricsSummary,
|
||||
@@ -670,6 +670,40 @@ export async function handleMiscRoutes(
|
||||
}
|
||||
}
|
||||
|
||||
if (url.pathname === "/api/settings/file-transfers") {
|
||||
const user = await authenticateRequest(req);
|
||||
if (!user) return new Response("Unauthorized", { status: 401 });
|
||||
try { requirePermission(user, "system:security"); } catch (error) { if (error instanceof Response) return error; return new Response("Forbidden", { status: 403 }); }
|
||||
|
||||
if (req.method === "GET") {
|
||||
return Response.json({ fileTransfers: getConfig().fileTransfers }, { headers: deps.CORS_HEADERS });
|
||||
}
|
||||
|
||||
if (req.method === "PUT") {
|
||||
let body: any = {};
|
||||
try { body = await req.json(); } catch {
|
||||
return Response.json({ error: "Invalid JSON" }, { status: 400 });
|
||||
}
|
||||
const updated = await updateFileTransfersConfig({
|
||||
maxFileBytes: body?.maxFileBytes !== undefined ? Number(body.maxFileBytes) : undefined,
|
||||
maxStagedBytes: body?.maxStagedBytes !== undefined ? Number(body.maxStagedBytes) : undefined,
|
||||
maxActiveGlobal: body?.maxActiveGlobal !== undefined ? Number(body.maxActiveGlobal) : undefined,
|
||||
maxActivePerUser: body?.maxActivePerUser !== undefined ? Number(body.maxActivePerUser) : undefined,
|
||||
uploadIntentTtlMs: body?.uploadIntentTtlMs !== undefined ? Number(body.uploadIntentTtlMs) : undefined,
|
||||
uploadPullTtlMs: body?.uploadPullTtlMs !== undefined ? Number(body.uploadPullTtlMs) : undefined,
|
||||
});
|
||||
logAudit({
|
||||
timestamp: Date.now(),
|
||||
username: user.username,
|
||||
ip: deps.requestIP?.(req)?.address || "unknown",
|
||||
action: AuditAction.COMMAND,
|
||||
details: "Updated file transfer limits",
|
||||
success: true,
|
||||
});
|
||||
return Response.json({ ok: true, fileTransfers: updated }, { headers: deps.CORS_HEADERS });
|
||||
}
|
||||
}
|
||||
|
||||
if (url.pathname === "/api/settings/tls") {
|
||||
const user = await authenticateRequest(req);
|
||||
if (!user) {
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import { handleFileBrowserMessage } from "./ws-file-process-proxy-keylogger";
|
||||
|
||||
describe("file browser agent download routing", () => {
|
||||
test("only consumes pending HTTP downloads for the matching client", () => {
|
||||
let consumed = 0;
|
||||
const payload = { type: "file_download", commandId: "download-1", data: new Uint8Array([1]) };
|
||||
const deps = {
|
||||
pendingHttpDownloads: new Map([["download-1", { clientId: "client-a" }]]),
|
||||
consumeHttpDownloadPayload: () => { consumed += 1; },
|
||||
};
|
||||
|
||||
handleFileBrowserMessage("client-b", payload, deps);
|
||||
expect(consumed).toBe(0);
|
||||
|
||||
handleFileBrowserMessage("client-a", payload, deps);
|
||||
expect(consumed).toBe(1);
|
||||
});
|
||||
|
||||
test("drops unsolicited download payloads", () => {
|
||||
let consumed = 0;
|
||||
handleFileBrowserMessage(
|
||||
"client-a",
|
||||
{ type: "file_download", commandId: "unknown", data: new Uint8Array([1]) },
|
||||
{
|
||||
pendingHttpDownloads: new Map(),
|
||||
consumeHttpDownloadPayload: () => { consumed += 1; },
|
||||
},
|
||||
);
|
||||
expect(consumed).toBe(0);
|
||||
});
|
||||
});
|
||||
@@ -45,7 +45,7 @@ type ProcessViewer = {
|
||||
};
|
||||
|
||||
type WsViewerClusterDeps = {
|
||||
pendingHttpDownloads: Map<string, unknown>;
|
||||
pendingHttpDownloads: ReadonlyMap<string, { clientId?: string }>;
|
||||
consumeHttpDownloadPayload: (payload: any) => Promise<void> | void;
|
||||
};
|
||||
|
||||
@@ -397,17 +397,23 @@ export function handleFileBrowserViewerMessage(ws: ServerWebSocket<SocketData>,
|
||||
|
||||
export function handleFileBrowserMessage(clientId: string, payload: any, deps: WsViewerClusterDeps) {
|
||||
const type = payload?.type as string | undefined;
|
||||
const payloadCommandId = typeof payload?.commandId === "string" ? payload.commandId : undefined;
|
||||
const pendingDownload = payloadCommandId
|
||||
? deps.pendingHttpDownloads.get(payloadCommandId)
|
||||
: undefined;
|
||||
const isHttpDownload =
|
||||
type === "file_download" &&
|
||||
typeof payload?.commandId === "string" &&
|
||||
deps.pendingHttpDownloads.has(payload.commandId);
|
||||
pendingDownload?.clientId === clientId;
|
||||
|
||||
if (type === "file_download" && typeof payload?.commandId === "string") {
|
||||
if (isHttpDownload) {
|
||||
void deps.consumeHttpDownloadPayload(payload);
|
||||
}
|
||||
|
||||
const payloadCommandId = typeof payload?.commandId === "string" ? payload.commandId : undefined;
|
||||
const commandOwner = payloadCommandId ? fileBrowserCommandSessions.get(payloadCommandId) : undefined;
|
||||
if (type === "file_download" && !isHttpDownload && !commandOwner) {
|
||||
logger.debug(`[filebrowser] dropped unsolicited download payload client=${clientId}`);
|
||||
return;
|
||||
}
|
||||
const ownerSessionId = commandOwner?.sessionId;
|
||||
|
||||
let hasSession = false;
|
||||
|
||||
@@ -0,0 +1,218 @@
|
||||
import { webcrypto } from "node:crypto";
|
||||
import {
|
||||
expect,
|
||||
request as requestFactory,
|
||||
test,
|
||||
type APIRequestContext,
|
||||
} from "@playwright/test";
|
||||
import { decodeMessage, encodeMessage } from "../../src/protocol";
|
||||
import { ADMIN } from "./credentials";
|
||||
|
||||
const BASE_URL = "http://127.0.0.1:5193";
|
||||
const AGENT_TOKEN = "e2e-agent-token-that-is-only-used-for-browser-tests";
|
||||
const CLIENT_ID = "e2e-file-transfer-agent";
|
||||
|
||||
type AgentCommand = {
|
||||
type: "command";
|
||||
commandType: string;
|
||||
id?: string;
|
||||
payload?: { path?: string };
|
||||
};
|
||||
|
||||
class TransferTestAgent {
|
||||
private socket: WebSocket | null = null;
|
||||
private downloads = new Map<string, Uint8Array>();
|
||||
|
||||
setDownload(path: string, bytes: Uint8Array): void {
|
||||
this.downloads.set(path, bytes);
|
||||
}
|
||||
|
||||
async connect(): Promise<void> {
|
||||
const keyPair = await webcrypto.subtle.generateKey(
|
||||
{ name: "Ed25519" },
|
||||
true,
|
||||
["sign", "verify"],
|
||||
);
|
||||
const publicKey = new Uint8Array(
|
||||
await webcrypto.subtle.exportKey("raw", keyPair.publicKey),
|
||||
);
|
||||
const socket = new WebSocket(
|
||||
`ws://127.0.0.1:5193/api/clients/${CLIENT_ID}/stream/ws?token=${encodeURIComponent(AGENT_TOKEN)}`,
|
||||
);
|
||||
socket.binaryType = "arraybuffer";
|
||||
this.socket = socket;
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const timeout = setTimeout(() => reject(new Error("Agent enrollment timed out")), 10_000);
|
||||
|
||||
socket.addEventListener("error", () => {
|
||||
clearTimeout(timeout);
|
||||
reject(new Error("Agent WebSocket failed"));
|
||||
});
|
||||
socket.addEventListener("close", (event) => {
|
||||
if (socket.readyState !== WebSocket.OPEN) {
|
||||
clearTimeout(timeout);
|
||||
reject(new Error(`Agent WebSocket closed during enrollment (${event.code}: ${event.reason})`));
|
||||
}
|
||||
});
|
||||
socket.addEventListener("message", async (event) => {
|
||||
const message = decodeMessage(event.data as ArrayBuffer) as any;
|
||||
if (message.type === "enrollment_challenge") {
|
||||
const nonce = Buffer.from(message.nonce, "base64");
|
||||
const signature = await webcrypto.subtle.sign("Ed25519", keyPair.privateKey, nonce);
|
||||
socket.send(encodeMessage({
|
||||
type: "hello",
|
||||
id: CLIENT_ID,
|
||||
host: "e2e-transfer-host",
|
||||
os: "E2E",
|
||||
arch: "x64",
|
||||
version: "e2e",
|
||||
user: "playwright",
|
||||
monitors: 1,
|
||||
hwid: CLIENT_ID,
|
||||
publicKey: Buffer.from(publicKey).toString("base64"),
|
||||
signature: Buffer.from(signature).toString("base64"),
|
||||
}));
|
||||
return;
|
||||
}
|
||||
if (message.type === "hello_ack") {
|
||||
clearTimeout(timeout);
|
||||
resolve();
|
||||
return;
|
||||
}
|
||||
if (message.type === "command") {
|
||||
this.handleCommand(message as AgentCommand);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
close(): void {
|
||||
this.socket?.close(1000, "test_complete");
|
||||
this.socket = null;
|
||||
}
|
||||
|
||||
private handleCommand(command: AgentCommand): void {
|
||||
if (command.commandType !== "file_download" || !command.id) return;
|
||||
const path = command.payload?.path || "";
|
||||
const bytes = this.downloads.get(path);
|
||||
if (!bytes) {
|
||||
this.socket?.send(encodeMessage({
|
||||
type: "file_download",
|
||||
commandId: command.id,
|
||||
path,
|
||||
data: new Uint8Array(),
|
||||
offset: 0,
|
||||
total: 0,
|
||||
error: "E2E fixture not found",
|
||||
}));
|
||||
return;
|
||||
}
|
||||
this.socket?.send(encodeMessage({
|
||||
type: "file_download",
|
||||
commandId: command.id,
|
||||
path,
|
||||
data: bytes,
|
||||
offset: 0,
|
||||
total: bytes.byteLength,
|
||||
chunkIndex: 0,
|
||||
chunksTotal: 1,
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
test.describe("file transfer HTTP API runtime", () => {
|
||||
let api: APIRequestContext;
|
||||
let agent: TransferTestAgent;
|
||||
|
||||
test.beforeAll(async () => {
|
||||
const loginApi = await requestFactory.newContext({ baseURL: BASE_URL });
|
||||
const login = await loginApi.post("/api/login", {
|
||||
data: { user: ADMIN.username, pass: ADMIN.password },
|
||||
});
|
||||
expect(login.ok()).toBeTruthy();
|
||||
const { token } = await login.json();
|
||||
await loginApi.dispose();
|
||||
api = await requestFactory.newContext({
|
||||
baseURL: BASE_URL,
|
||||
extraHTTPHeaders: { Authorization: `Bearer ${token}` },
|
||||
});
|
||||
|
||||
const enrollment = await api.post("/api/enrollment/settings", {
|
||||
data: { requireApproval: false, autoApproveUnlessSuspicious: false },
|
||||
});
|
||||
expect(enrollment.ok()).toBeTruthy();
|
||||
|
||||
agent = new TransferTestAgent();
|
||||
await agent.connect();
|
||||
});
|
||||
|
||||
test.afterAll(async () => {
|
||||
agent?.close();
|
||||
if (api) {
|
||||
await api.post("/api/enrollment/settings", {
|
||||
data: { requireApproval: true, autoApproveUnlessSuspicious: false },
|
||||
});
|
||||
await api.dispose();
|
||||
}
|
||||
});
|
||||
|
||||
test("uploads bytes for a one-time authenticated agent pull", async () => {
|
||||
const remotePath = "C:\\Temp\\overlord-e2e-upload.bin";
|
||||
const uploadBytes = Buffer.from("playwright upload payload\n\u0000with binary bytes", "utf8");
|
||||
|
||||
const requestUpload = await api.post("/api/file/upload/request", {
|
||||
data: {
|
||||
clientId: CLIENT_ID,
|
||||
path: remotePath,
|
||||
fileName: "overlord-e2e-upload.bin",
|
||||
},
|
||||
});
|
||||
expect(requestUpload.ok()).toBeTruthy();
|
||||
const { uploadUrl } = await requestUpload.json();
|
||||
|
||||
const stageUpload = await api.put(uploadUrl, {
|
||||
data: uploadBytes,
|
||||
headers: { "Content-Type": "application/octet-stream" },
|
||||
});
|
||||
expect(stageUpload.ok()).toBeTruthy();
|
||||
const staged = await stageUpload.json();
|
||||
expect(staged).toMatchObject({
|
||||
ok: true,
|
||||
size: uploadBytes.byteLength,
|
||||
path: remotePath,
|
||||
});
|
||||
|
||||
const pullHeaders = {
|
||||
"x-agent-token": AGENT_TOKEN,
|
||||
"x-overlord-client-id": CLIENT_ID,
|
||||
};
|
||||
const pull = await api.get(staged.pullUrl, { headers: pullHeaders });
|
||||
expect(pull.ok()).toBeTruthy();
|
||||
expect(Buffer.from(await pull.body())).toEqual(uploadBytes);
|
||||
|
||||
const replay = await api.get(staged.pullUrl, { headers: pullHeaders });
|
||||
expect(replay.status()).toBe(404);
|
||||
});
|
||||
|
||||
test("streams a requested agent file through the download API", async () => {
|
||||
const remotePath = "C:\\Temp\\overlord-e2e-download.bin";
|
||||
const downloadBytes = new Uint8Array([
|
||||
0x00, 0x01, 0x02, 0x7f, 0x80, 0xfe, 0xff,
|
||||
...Buffer.from("playwright download payload", "utf8"),
|
||||
]);
|
||||
agent.setDownload(remotePath, downloadBytes);
|
||||
|
||||
const requestDownload = await api.post("/api/file/download/request", {
|
||||
data: { clientId: CLIENT_ID, path: remotePath },
|
||||
});
|
||||
expect(requestDownload.ok()).toBeTruthy();
|
||||
const { downloadUrl } = await requestDownload.json();
|
||||
|
||||
const download = await api.get(downloadUrl);
|
||||
expect(download.ok()).toBeTruthy();
|
||||
expect(download.headers()["content-type"]).toContain("application/octet-stream");
|
||||
expect(download.headers()["content-disposition"]).toContain("overlord-e2e-download.bin");
|
||||
expect(Buffer.from(await download.body())).toEqual(Buffer.from(downloadBytes));
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user