new upload system

This commit is contained in:
KingKDot
2026-03-08 00:53:22 -06:00
parent 8662f4310f
commit 4b48f462da
8 changed files with 1075 additions and 285 deletions
@@ -1345,6 +1345,12 @@ func HandleCommand(ctx context.Context, env *runtime.Env, envelope map[string]in
data = d data = d
} }
return HandleFileUpload(ctx, env, cmdID, path, data, offset, total, transferID) return HandleFileUpload(ctx, env, cmdID, path, data, offset, total, transferID)
case "file_upload_http":
payload, _ := envelope["payload"].(map[string]interface{})
path, _ := payload["path"].(string)
sourceURL, _ := payload["url"].(string)
total := payloadNumberToInt64(payload["total"])
return HandleFileUploadHTTP(ctx, env, cmdID, path, sourceURL, total)
case "file_delete": case "file_delete":
path, _ := envelopePayloadString(envelope, "path") path, _ := envelopePayloadString(envelope, "path")
return HandleFileDelete(ctx, env, cmdID, path) return HandleFileDelete(ctx, env, cmdID, path)
@@ -5,9 +5,13 @@ import (
"bufio" "bufio"
"bytes" "bytes"
"context" "context"
"crypto/tls"
"crypto/x509"
"fmt" "fmt"
"io" "io"
"log" "log"
"net/http"
"net/url"
"os" "os"
"path/filepath" "path/filepath"
"runtime" "runtime"
@@ -462,6 +466,88 @@ func HandleFileUpload(ctx context.Context, env *agentRuntime.Env, cmdID string,
return wire.WriteMsg(ctx, env.Conn, result) return wire.WriteMsg(ctx, env.Conn, result)
} }
func HandleFileUploadHTTP(ctx context.Context, env *agentRuntime.Env, cmdID string, destPath string, sourceURL string, expectedSize int64) error {
parsed, err := url.Parse(strings.TrimSpace(sourceURL))
if err != nil || parsed == nil || parsed.Host == "" {
return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: false, Message: "invalid upload url"})
}
if parsed.Scheme != "https" && parsed.Scheme != "http" {
return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: false, Message: "unsupported upload url scheme"})
}
tlsConfig := &tls.Config{InsecureSkipVerify: env.Cfg.TLSInsecureSkipVerify, MinVersion: tls.VersionTLS12}
if caPath := strings.TrimSpace(env.Cfg.TLSCAPath); caPath != "" {
caBytes, err := os.ReadFile(caPath)
if err != nil {
return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: false, Message: fmt.Sprintf("failed to read TLS CA: %v", err)})
}
pool := x509.NewCertPool()
if !pool.AppendCertsFromPEM(caBytes) {
return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: false, Message: "failed to parse TLS CA"})
}
tlsConfig.RootCAs = pool
}
client := &http.Client{Transport: &http.Transport{TLSClientConfig: tlsConfig}}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, parsed.String(), nil)
if err != nil {
return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: false, Message: err.Error()})
}
if token := strings.TrimSpace(env.Cfg.AgentToken); token != "" {
req.Header.Set("x-agent-token", token)
}
if id := strings.TrimSpace(env.Cfg.ID); id != "" {
req.Header.Set("x-overlord-client-id", id)
}
resp, err := client.Do(req)
if err != nil {
return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: false, Message: err.Error()})
}
defer resp.Body.Close()
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: false, Message: fmt.Sprintf("upload fetch failed: status %d", resp.StatusCode)})
}
dir := filepath.Dir(destPath)
if dir != "." {
if err := os.MkdirAll(dir, 0755); err != nil {
return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: false, Message: err.Error()})
}
}
tmpPath := destPath + ".httpuploading"
f, err := os.OpenFile(tmpPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644)
if err != nil {
return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: false, Message: err.Error()})
}
written, copyErr := io.Copy(f, resp.Body)
closeErr := f.Close()
if copyErr != nil {
_ = os.Remove(tmpPath)
return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: false, Message: copyErr.Error()})
}
if closeErr != nil {
_ = os.Remove(tmpPath)
return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: false, Message: closeErr.Error()})
}
if expectedSize > 0 && written != expectedSize {
_ = os.Remove(tmpPath)
return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: false, Message: "upload size mismatch"})
}
_ = os.Remove(destPath)
if err := os.Rename(tmpPath, destPath); err != nil {
_ = os.Remove(tmpPath)
return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: false, Message: err.Error()})
}
return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: true})
}
func HandleFileDelete(ctx context.Context, env *agentRuntime.Env, cmdID string, path string) error { func HandleFileDelete(ctx context.Context, env *agentRuntime.Env, cmdID string, path string) error {
log.Printf("file_delete: %s", path) log.Printf("file_delete: %s", path)
@@ -3,6 +3,8 @@ package handlers
import ( import (
"bytes" "bytes"
"context" "context"
"net/http"
"net/http/httptest"
"os" "os"
"path/filepath" "path/filepath"
"testing" "testing"
@@ -467,3 +469,87 @@ func TestHandleFileZip(t *testing.T) {
t.Error("Expected either command_result or file_download message") t.Error("Expected either command_result or file_download message")
} }
} }
func TestHandleFileUploadHTTP(t *testing.T) {
data := []byte("http-upload-payload")
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if got := r.Header.Get("x-overlord-client-id"); got != "client-abc" {
t.Fatalf("expected x-overlord-client-id client-abc, got %q", got)
}
if got := r.Header.Get("x-agent-token"); got != "agent-token" {
t.Fatalf("expected x-agent-token agent-token, got %q", got)
}
_, _ = w.Write(data)
}))
defer ts.Close()
tmpDir := t.TempDir()
destPath := filepath.Join(tmpDir, "uploaded-http.bin")
writer := &testWriter{}
env := &rt.Env{
Conn: writer,
Cfg: config.Config{
ID: "client-abc",
AgentToken: "agent-token",
TLSInsecureSkipVerify: true,
},
}
if err := HandleFileUploadHTTP(context.Background(), env, "cmd-http-upload", destPath, ts.URL+"/file", int64(len(data))); err != nil {
t.Fatalf("HandleFileUploadHTTP failed: %v", err)
}
content, err := os.ReadFile(destPath)
if err != nil {
t.Fatalf("failed reading uploaded file: %v", err)
}
if !bytes.Equal(content, data) {
t.Fatalf("uploaded content mismatch")
}
if len(writer.msgs) != 1 {
t.Fatalf("expected 1 message, got %d", len(writer.msgs))
}
var result wire.CommandResult
if err := msgpack.Unmarshal(writer.msgs[0], &result); err != nil {
t.Fatalf("failed to unmarshal command result: %v", err)
}
if !result.OK {
t.Fatalf("expected OK=true, got false: %s", result.Message)
}
}
func TestHandleFileUploadHTTP_SizeMismatch(t *testing.T) {
data := []byte("small")
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = w.Write(data)
}))
defer ts.Close()
tmpDir := t.TempDir()
destPath := filepath.Join(tmpDir, "uploaded-http-mismatch.bin")
writer := &testWriter{}
env := &rt.Env{
Conn: writer,
Cfg: config.Config{TLSInsecureSkipVerify: true},
}
if err := HandleFileUploadHTTP(context.Background(), env, "cmd-http-upload-mismatch", destPath, ts.URL+"/file", int64(len(data)+1)); err != nil {
t.Fatalf("HandleFileUploadHTTP failed: %v", err)
}
if len(writer.msgs) != 1 {
t.Fatalf("expected 1 message, got %d", len(writer.msgs))
}
var result wire.CommandResult
if err := msgpack.Unmarshal(writer.msgs[0], &result); err != nil {
t.Fatalf("failed to unmarshal command result: %v", err)
}
if result.OK {
t.Fatalf("expected OK=false for size mismatch")
}
}
+170 -134
View File
@@ -14,6 +14,7 @@ let lastSuccessfulResponse = 0;
let pendingToast = null; let pendingToast = null;
const recentToasts = new Map(); const recentToasts = new Map();
const pendingCommandResults = new Map(); const pendingCommandResults = new Map();
const pendingCommandWaiters = new Map();
const VIRTUALIZATION_THRESHOLD = 400; const VIRTUALIZATION_THRESHOLD = 400;
const VIRTUAL_ROW_HEIGHT = 58; const VIRTUAL_ROW_HEIGHT = 58;
const VIRTUAL_OVERSCAN = 8; const VIRTUAL_OVERSCAN = 8;
@@ -192,6 +193,25 @@ function trackCommandResult(commandId, options = {}) {
}); });
} }
function waitForCommandResult(commandId, timeoutMs = 10 * 60 * 1000) {
return new Promise((resolve, reject) => {
if (!commandId) {
reject(new Error("missing command id"));
return;
}
const existing = pendingCommandWaiters.get(commandId);
if (existing) {
clearTimeout(existing.timeoutId);
existing.reject(new Error("superseded command waiter"));
}
const timeoutId = setTimeout(() => {
pendingCommandWaiters.delete(commandId);
reject(new Error("command timed out"));
}, timeoutMs);
pendingCommandWaiters.set(commandId, { resolve, reject, timeoutId });
});
}
document.addEventListener("visibilitychange", () => { document.addEventListener("visibilitychange", () => {
if (document.visibilityState !== "visible") return; if (document.visibilityState !== "visible") return;
if (!pendingToast || typeof window.showToast !== "function") return; if (!pendingToast || typeof window.showToast !== "function") return;
@@ -838,7 +858,13 @@ function downloadFile(path) {
} }
const requestData = await requestRes.json(); const requestData = await requestRes.json();
if (!requestData?.downloadId) { const downloadUrl = typeof requestData?.downloadUrl === "string"
? requestData.downloadUrl
: (requestData?.downloadId
? `/api/file/download/${encodeURIComponent(requestData.downloadId)}`
: "");
if (!downloadUrl) {
notifyToast("Download failed", "error", 5000); notifyToast("Download failed", "error", 5000);
removeTransfer(transferId); removeTransfer(transferId);
fileDownloads.delete(path); fileDownloads.delete(path);
@@ -847,19 +873,17 @@ function downloadFile(path) {
} }
console.debug("[filebrowser] download request accepted", { console.debug("[filebrowser] download request accepted", {
downloadId: requestData.downloadId, downloadUrl,
}); });
console.debug("[filebrowser] download stream start", { console.debug("[filebrowser] download stream start", {
downloadId: requestData.downloadId, downloadUrl,
}); });
const res = await fetch("/api/file/download", { const res = await fetch(downloadUrl, {
method: "POST", method: "GET",
headers: { "Content-Type": "application/json" },
credentials: "include", credentials: "include",
signal: abortController.signal, signal: abortController.signal,
body: JSON.stringify({ downloadId: requestData.downloadId }),
}); });
console.debug("[filebrowser] download response", { console.debug("[filebrowser] download response", {
@@ -1309,6 +1333,24 @@ function handleFileUploadResult(msg) {
} }
const offset = toNumber(msg.offset); const offset = toNumber(msg.offset);
if (offset !== null) {
const firstAckForOffset = !transfer.ackedOffsets.has(offset);
if (firstAckForOffset) {
transfer.ackedOffsets.add(offset);
transfer.receivedChunks += 1;
}
if (Number.isFinite(msg.received)) {
transfer.receivedBytes = Math.min(Number(msg.received), transfer.total);
transfer.sent = transfer.receivedBytes;
}
if (transfer.total > 0) {
transfer.progress = Math.round((transfer.sent / transfer.total) * 100);
updateTransferProgress(transfer.id, transfer.progress, transfer.sent, transfer.total);
}
}
const pendingOffset = offset !== null const pendingOffset = offset !== null
? offset ? offset
: (transfer.pendingAcks.has(0) ? 0 : null); : (transfer.pendingAcks.has(0) ? 0 : null);
@@ -1352,6 +1394,20 @@ function handleCommandResult(msg) {
const tracked = msg.commandId const tracked = msg.commandId
? pendingCommandResults.get(msg.commandId) ? pendingCommandResults.get(msg.commandId)
: null; : null;
const waiter = msg.commandId
? pendingCommandWaiters.get(msg.commandId)
: null;
if (waiter) {
clearTimeout(waiter.timeoutId);
pendingCommandWaiters.delete(msg.commandId);
if (msg.ok) {
waiter.resolve(msg);
} else {
waiter.reject(new Error(msg.message || "operation failed"));
}
}
if (!tracked) { if (!tracked) {
return; return;
} }
@@ -1528,57 +1584,6 @@ if (sortOrderBtn) {
}); });
} }
function waitForUploadAck(transfer, offset, timeoutMs) {
return new Promise((resolve, reject) => {
const existing = transfer.pendingAcks.get(offset);
if (existing) {
clearTimeout(existing.timeoutId);
existing.reject(new Error("Superseded upload ack"));
}
const timeoutId = setTimeout(() => {
transfer.pendingAcks.delete(offset);
reject(new Error("Upload chunk timed out"));
}, timeoutMs);
transfer.pendingAcks.set(offset, { resolve, reject, timeoutId });
});
}
function clearUploadPendingAcks(transfer, reason) {
transfer.pendingAcks.forEach((pending, _offset) => {
clearTimeout(pending.timeoutId);
pending.reject(new Error(reason));
});
transfer.pendingAcks.clear();
}
async function sendChunkWithRetry(transfer, payload, offset) {
const maxRetries = 5;
const ackTimeoutMs = 20000;
const backoffMs = 300;
for (let attempt = 1; attempt <= maxRetries; attempt += 1) {
if (transfer.cancelled) {
throw new Error("Upload cancelled");
}
const ackPromise = waitForUploadAck(transfer, offset, ackTimeoutMs);
send(payload);
try {
return await ackPromise;
} catch (err) {
if (attempt === maxRetries) {
throw err;
}
await new Promise((resolve) => setTimeout(resolve, backoffMs * attempt));
}
}
throw new Error("Upload failed after retries");
}
function finishUpload(transfer) { function finishUpload(transfer) {
console.log("Upload complete:", transfer.path); console.log("Upload complete:", transfer.path);
updateStatus("connected", "Connected"); updateStatus("connected", "Connected");
@@ -1589,6 +1594,111 @@ function finishUpload(transfer) {
listFiles(currentPath); listFiles(currentPath);
} }
async function uploadFileViaHttpPull(file, path, transfer) {
console.debug("[filebrowser] upload request start", {
clientId,
path,
fileName: file.name,
size: file.size,
});
const requestRes = await fetch("/api/file/upload/request", {
method: "POST",
headers: { "Content-Type": "application/json" },
credentials: "include",
body: JSON.stringify({
clientId,
path,
fileName: file.name,
}),
});
if (!requestRes.ok) {
const text = await requestRes.text();
throw new Error(text || "upload request failed");
}
const requestData = await requestRes.json();
const uploadUrl = typeof requestData?.uploadUrl === "string"
? requestData.uploadUrl
: (requestData?.uploadId
? `/api/file/upload/${encodeURIComponent(requestData.uploadId)}`
: "");
if (!uploadUrl) {
throw new Error("upload request failed");
}
console.debug("[filebrowser] upload stage url", { uploadUrl });
let uploadRes;
try {
uploadRes = await fetch(uploadUrl, {
method: "PUT",
headers: { "Content-Type": "application/octet-stream" },
credentials: "include",
body: file,
});
} catch (err) {
console.warn("[filebrowser] upload stage PUT failed, retrying as POST", err);
uploadRes = await fetch(uploadUrl, {
method: "POST",
headers: { "Content-Type": "application/octet-stream" },
credentials: "include",
body: file,
});
}
if (!uploadRes.ok) {
const text = await uploadRes.text();
console.debug("[filebrowser] upload stage failed", {
status: uploadRes.status,
body: text,
});
throw new Error(text || "upload staging failed");
}
transfer.receivedBytes = Math.round(file.size * 0.5);
transfer.sent = transfer.receivedBytes;
transfer.progress = 50;
updateTransferProgress(transfer.id, transfer.progress, transfer.sent, transfer.total);
const uploadData = await uploadRes.json();
if (!uploadData?.pullUrl) {
throw new Error("upload staging failed");
}
console.debug("[filebrowser] upload staged", {
pullUrl: uploadData.pullUrl,
size: uploadData.size,
});
const commandId = `upload-http-${Date.now()}-${Math.random()}`;
const waitResult = waitForCommandResult(commandId, 12 * 60 * 1000);
send({
type: "command",
commandType: "file_upload_http",
id: commandId,
payload: {
path,
url: uploadData.pullUrl,
total: file.size,
},
});
await waitResult;
console.debug("[filebrowser] upload command completed", {
path,
size: file.size,
});
transfer.receivedBytes = file.size;
transfer.sent = file.size;
transfer.progress = 100;
transfer.receivedChunks = transfer.expectedChunks;
updateTransferProgress(transfer.id, transfer.progress, transfer.sent, transfer.total);
}
async function uploadFile(file) { async function uploadFile(file) {
const path = currentPath ? `${currentPath}/${file.name}` : file.name; const path = currentPath ? `${currentPath}/${file.name}` : file.name;
const transferId = `upload-${Date.now()}-${Math.random()}`; const transferId = `upload-${Date.now()}-${Math.random()}`;
@@ -1618,84 +1728,10 @@ async function uploadFile(file) {
activeTransfers.set(transferId, transfer); activeTransfers.set(transferId, transfer);
addTransferToUI(transfer); addTransferToUI(transfer);
const chunkSize = 900 * 1024;
const maxInFlightChunks = 6;
transfer.expectedChunks = Math.max(1, Math.ceil(file.size / chunkSize));
let nextOffset = 0;
const inFlight = new Set();
const processChunk = async (offset) => {
const chunk = file.slice(offset, offset + chunkSize);
const arrayBuffer = await chunk.arrayBuffer();
const uint8Array = new Uint8Array(arrayBuffer);
const payload = {
type: "file_upload",
path,
data: uint8Array,
offset,
transferId,
total: file.size,
};
const ack = await sendChunkWithRetry(transfer, payload, offset);
if (!transfer.ackedOffsets.has(offset)) {
transfer.ackedOffsets.add(offset);
transfer.receivedChunks += 1;
}
if (Number.isFinite(ack?.received)) {
transfer.receivedBytes = Math.min(Number(ack.received), transfer.total);
transfer.sent = transfer.receivedBytes;
} else {
transfer.receivedBytes = Math.min(transfer.receivedBytes + chunk.size, transfer.total);
transfer.sent = transfer.receivedBytes;
}
transfer.progress = Math.round((transfer.sent / file.size) * 100);
updateTransferProgress(transferId, transfer.progress, transfer.sent, file.size);
};
const fillUploadWindow = () => {
while (
!transfer.cancelled &&
nextOffset < file.size &&
inFlight.size < maxInFlightChunks
) {
const offset = nextOffset;
nextOffset += chunkSize;
const task = processChunk(offset).finally(() => {
inFlight.delete(task);
});
inFlight.add(task);
}
};
try { try {
fillUploadWindow(); await uploadFileViaHttpPull(file, path, transfer);
finishUpload(transfer);
while (inFlight.size > 0) {
await Promise.race(inFlight);
if (transfer.cancelled) {
clearUploadPendingAcks(transfer, "Upload cancelled");
console.log("Upload cancelled:", path);
removeTransfer(transferId);
fileUploads.delete(path);
fileUploadsById.delete(transferId);
return;
}
fillUploadWindow();
}
transfer.completed = true;
if (transfer.receivedChunks >= transfer.expectedChunks) {
finishUpload(transfer);
}
} catch (err) { } catch (err) {
clearUploadPendingAcks(transfer, "Upload aborted");
console.error("Upload error:", err); console.error("Upload error:", err);
removeTransfer(transferId); removeTransfer(transferId);
fileUploads.delete(path); fileUploads.delete(path);
+21 -1
View File
@@ -20,7 +20,7 @@ import { handleAutoScriptsRoutes } from "./server/routes/auto-scripts-routes";
import { handleBuildRoutes } from "./server/routes/build-routes"; import { handleBuildRoutes } from "./server/routes/build-routes";
import { handleAssetsRoutes } from "./server/routes/assets-routes"; import { handleAssetsRoutes } from "./server/routes/assets-routes";
import { handleDeployRoutes } from "./server/routes/deploy-routes"; import { handleDeployRoutes } from "./server/routes/deploy-routes";
import { handleFileDownloadRoutes } from "./server/routes/file-download-routes"; import { cleanupFileTransferTempFiles, handleFileDownloadRoutes } from "./server/routes/file-download-routes";
import { handleClientRoutes } from "./server/routes/client-routes"; import { handleClientRoutes } from "./server/routes/client-routes";
import { handleMiscRoutes } from "./server/routes/misc-routes"; import { handleMiscRoutes } from "./server/routes/misc-routes";
import { handleNotificationsConfigRoutes } from "./server/routes/notifications-config-routes"; import { handleNotificationsConfigRoutes } from "./server/routes/notifications-config-routes";
@@ -141,6 +141,21 @@ function envFlagEnabled(name: string): boolean {
const TLS_OFFLOAD = envFlagEnabled("OVERLORD_TLS_OFFLOAD"); const TLS_OFFLOAD = envFlagEnabled("OVERLORD_TLS_OFFLOAD");
function parseMaxHttpBodyBytes(): number {
const raw = String(process.env.OVERLORD_MAX_HTTP_BODY_BYTES || "").trim();
if (raw !== "") {
const parsed = Number(raw);
if (Number.isFinite(parsed) && parsed > 0) {
return Math.floor(parsed);
}
logger.warn(`[HTTP] Invalid OVERLORD_MAX_HTTP_BODY_BYTES=${raw}; using default`);
}
// Default to 2 GiB to allow large ISO uploads in file browser staging.
return 2 * 1024 * 1024 * 1024;
}
const MAX_HTTP_BODY_BYTES = parseMaxHttpBodyBytes();
const pluginLoadedByClient = new Map<string, Set<string>>(); const pluginLoadedByClient = new Map<string, Set<string>>();
const pendingPluginEvents = new Map<string, Array<{ event: string; payload: any }>>(); const pendingPluginEvents = new Map<string, Array<{ event: string; payload: any }>>();
const pluginLoadingByClient = new Map<string, Set<string>>(); const pluginLoadingByClient = new Map<string, Set<string>>();
@@ -241,6 +256,8 @@ const pendingCommandReplies = new Map<string, PendingCommandReply>();
async function startServer() { async function startServer() {
await loadPluginState(); await loadPluginState();
await cleanupFileTransferTempFiles(DATA_DIR);
logger.info("[filebrowser] cleaned stale transfer temp files on startup");
let tls: let tls:
| { | {
tlsOptions: { cert?: string; key?: string; ca?: string }; tlsOptions: { cert?: string; key?: string; ca?: string };
@@ -392,6 +409,7 @@ async function startServer() {
hostname: HOST, hostname: HOST,
...(tls ? { tls: tls.tlsOptions } : {}), ...(tls ? { tls: tls.tlsOptions } : {}),
idleTimeout: 255, idleTimeout: 255,
maxRequestBodySize: MAX_HTTP_BODY_BYTES,
fetch: createHttpFetchHandler({ fetch: createHttpFetchHandler({
metrics, metrics,
CORS_HEADERS, CORS_HEADERS,
@@ -440,6 +458,7 @@ async function startServer() {
if (tls) { if (tls) {
logServerStartup(server, tls.certPathUsed, tls.source); logServerStartup(server, tls.certPathUsed, tls.source);
logger.info(`[HTTP] maxRequestBodySize=${MAX_HTTP_BODY_BYTES} bytes`);
} else { } else {
const hostname = server.hostname || "0.0.0.0"; const hostname = server.hostname || "0.0.0.0";
const port = server.port ?? 0; const port = server.port ?? 0;
@@ -451,6 +470,7 @@ async function startServer() {
logger.info(""); logger.info("");
logger.info("External access should be HTTPS/WSS via your reverse proxy platform."); logger.info("External access should be HTTPS/WSS via your reverse proxy platform.");
logger.info("Set this mode only when TLS is terminated by the platform (for example Render). "); logger.info("Set this mode only when TLS is terminated by the platform (for example Render). ");
logger.info(`[HTTP] maxRequestBodySize=${MAX_HTTP_BODY_BYTES} bytes`);
logger.info("========================================"); logger.info("========================================");
} }
} }
@@ -0,0 +1,251 @@
import { describe, expect, test } from "bun:test";
import { mkdtemp, readdir, rm } from "fs/promises";
import { join } from "path";
import { tmpdir } from "os";
import { generateToken } from "../../auth";
import * as clientManager from "../../clientManager";
import { createUser, deleteUser, getUserById } from "../../users";
import { handleFileDownloadRoutes } from "./file-download-routes";
const PASSWORD = "Aa1!RouteUploadTestPass123";
function makeRouteDeps(dataDir: string) {
return {
DATA_DIR: dataDir,
secureHeaders: (_contentType?: string) => ({}),
sanitizeOutputName: (name: string) => {
const cleaned = (name || "").replace(/[^A-Za-z0-9._-]/g, "");
if (!cleaned) throw new Error("invalid filename");
return cleaned;
},
pendingHttpDownloads: new Map<string, any>(),
downloadIntents: new Map<string, any>(),
};
}
const mockServer = {
requestIP: () => ({ address: "127.0.0.1" }),
};
async function createAdminToken() {
const username = `u_${Date.now().toString(36)}_${Math.floor(Math.random() * 1e6).toString(36)}`;
const created = await createUser(username, PASSWORD, "admin", "test");
expect(created.success).toBe(true);
expect(typeof created.userId).toBe("number");
const user = getUserById(created.userId!);
expect(user).not.toBeNull();
const token = await generateToken(user!);
return {
userId: created.userId!,
token,
};
}
async function createTempDataDir(): Promise<string> {
return mkdtemp(join(tmpdir(), "overlord-upload-route-test-"));
}
async function removeTempDataDir(dir: string): Promise<void> {
await rm(dir, { recursive: true, force: true }).catch(() => {});
}
describe("file upload route flow", () => {
test("stages and serves one-time upload payload for matching client", async () => {
const auth = await createAdminToken();
const clientId = `client-${Date.now().toString(36)}`;
const dataDir = await createTempDataDir();
const deps = makeRouteDeps(dataDir);
const payload = new Uint8Array(3 * 1024 * 1024 + 123);
crypto.getRandomValues(payload.subarray(0, Math.min(payload.length, 65536)));
clientManager.addClient(clientId, {
id: clientId,
lastSeen: Date.now(),
role: "client",
ws: { send: () => {} },
});
const prevDisableAgentAuth = process.env.OVERLORD_DISABLE_AGENT_AUTH;
process.env.OVERLORD_DISABLE_AGENT_AUTH = "true";
try {
const requestUrl = new URL("https://localhost/api/file/upload/request");
const requestRes = await handleFileDownloadRoutes(
new Request(requestUrl, {
method: "POST",
headers: {
"Content-Type": "application/json",
Authorization: `Bearer ${auth.token}`,
},
body: JSON.stringify({
clientId,
path: "C:\\Games\\big.iso",
fileName: "big.iso",
}),
}),
requestUrl,
mockServer,
deps,
);
expect(requestRes).not.toBeNull();
expect(requestRes!.status).toBe(200);
const requestJson = await requestRes!.json() as any;
expect(requestJson.ok).toBe(true);
expect(typeof requestJson.uploadUrl).toBe("string");
const stageUrl = new URL(`https://localhost${requestJson.uploadUrl}`);
const stageRes = await handleFileDownloadRoutes(
new Request(stageUrl, {
method: "POST",
headers: {
"Content-Type": "application/octet-stream",
Authorization: `Bearer ${auth.token}`,
},
body: payload,
}),
stageUrl,
mockServer,
deps,
);
expect(stageRes).not.toBeNull();
expect(stageRes!.status).toBe(200);
const stageJson = await stageRes!.json() as any;
expect(stageJson.ok).toBe(true);
expect(stageJson.size).toBe(payload.length);
expect(typeof stageJson.pullUrl).toBe("string");
const pullUrl = new URL(stageJson.pullUrl);
const pullRes = await handleFileDownloadRoutes(
new Request(pullUrl, {
method: "GET",
headers: {
"x-overlord-client-id": clientId,
},
}),
pullUrl,
mockServer,
deps,
);
expect(pullRes).not.toBeNull();
expect(pullRes!.status).toBe(200);
const pulled = new Uint8Array(await pullRes!.arrayBuffer());
expect(pulled.length).toBe(payload.length);
expect(pulled[0]).toBe(payload[0]);
expect(pulled[pulled.length - 1]).toBe(payload[payload.length - 1]);
const uploadDir = join(dataDir, "uploads");
const uploadEntries = await readdir(uploadDir).catch(() => [] as string[]);
expect(uploadEntries.filter((name) => name.endsWith(".bin")).length).toBe(0);
const pullAgainRes = await handleFileDownloadRoutes(
new Request(pullUrl, {
method: "GET",
headers: {
"x-overlord-client-id": clientId,
},
}),
pullUrl,
mockServer,
deps,
);
expect(pullAgainRes).not.toBeNull();
expect(pullAgainRes!.status).toBe(404);
} finally {
if (prevDisableAgentAuth === undefined) {
delete process.env.OVERLORD_DISABLE_AGENT_AUTH;
} else {
process.env.OVERLORD_DISABLE_AGENT_AUTH = prevDisableAgentAuth;
}
clientManager.deleteClient(clientId);
const deleted = deleteUser(auth.userId);
expect(deleted.success).toBe(true);
await removeTempDataDir(dataDir);
}
});
test("rejects pull when client id does not match", async () => {
const auth = await createAdminToken();
const clientId = `client-${Date.now().toString(36)}`;
const dataDir = await createTempDataDir();
const deps = makeRouteDeps(dataDir);
const payload = new TextEncoder().encode("hello");
clientManager.addClient(clientId, {
id: clientId,
lastSeen: Date.now(),
role: "client",
ws: { send: () => {} },
});
const prevDisableAgentAuth = process.env.OVERLORD_DISABLE_AGENT_AUTH;
process.env.OVERLORD_DISABLE_AGENT_AUTH = "true";
try {
const requestUrl = new URL("https://localhost/api/file/upload/request");
const requestRes = await handleFileDownloadRoutes(
new Request(requestUrl, {
method: "POST",
headers: {
"Content-Type": "application/json",
Authorization: `Bearer ${auth.token}`,
},
body: JSON.stringify({
clientId,
path: "C:\\Games\\small.bin",
fileName: "small.bin",
}),
}),
requestUrl,
mockServer,
deps,
);
const requestJson = await requestRes!.json() as any;
const stageUrl = new URL(`https://localhost${requestJson.uploadUrl}`);
const stageRes = await handleFileDownloadRoutes(
new Request(stageUrl, {
method: "PUT",
headers: {
Authorization: `Bearer ${auth.token}`,
},
body: payload,
}),
stageUrl,
mockServer,
deps,
);
const stageJson = await stageRes!.json() as any;
const pullUrl = new URL(stageJson.pullUrl);
const badPullRes = await handleFileDownloadRoutes(
new Request(pullUrl, {
method: "GET",
headers: {
"x-overlord-client-id": "different-client",
},
}),
pullUrl,
mockServer,
deps,
);
expect(badPullRes).not.toBeNull();
expect(badPullRes!.status).toBe(403);
} finally {
if (prevDisableAgentAuth === undefined) {
delete process.env.OVERLORD_DISABLE_AGENT_AUTH;
} else {
process.env.OVERLORD_DISABLE_AGENT_AUTH = prevDisableAgentAuth;
}
clientManager.deleteClient(clientId);
deleteUser(auth.userId);
await removeTempDataDir(dataDir);
}
});
});
@@ -6,6 +6,8 @@ import { AuditAction, logAudit } from "../../auditLog";
import * as clientManager from "../../clientManager"; import * as clientManager from "../../clientManager";
import { logger } from "../../logger"; import { logger } from "../../logger";
import { encodeMessage } from "../../protocol"; import { encodeMessage } from "../../protocol";
import { getConfig } from "../../config";
import { isAuthorizedAgentRequest } from "../agent-auth";
type RequestIpProvider = { type RequestIpProvider = {
requestIP: (req: Request) => { address?: string } | null | undefined; requestIP: (req: Request) => { address?: string } | null | undefined;
@@ -40,6 +42,27 @@ type DownloadIntent = {
timeout: ReturnType<typeof setTimeout>; timeout: ReturnType<typeof setTimeout>;
}; };
type UploadIntent = {
id: string;
userId: string;
clientId: string;
path: string;
fileName: string;
expiresAt: number;
timeout: ReturnType<typeof setTimeout>;
};
type UploadPull = {
id: string;
clientId: string;
path: string;
fileName: string;
tmpPath: string;
size: number;
expiresAt: number;
timeout: ReturnType<typeof setTimeout>;
};
type FileDownloadRouteDeps = { type FileDownloadRouteDeps = {
DATA_DIR: string; DATA_DIR: string;
secureHeaders: (contentType?: string) => Record<string, string>; secureHeaders: (contentType?: string) => Record<string, string>;
@@ -48,17 +71,222 @@ type FileDownloadRouteDeps = {
downloadIntents: Map<string, DownloadIntent>; downloadIntents: Map<string, DownloadIntent>;
}; };
export async function cleanupFileTransferTempFiles(dataDir: string): Promise<void> {
const uploadsDir = path.join(dataDir, "uploads");
const downloadsDir = path.join(dataDir, "downloads");
await fs.rm(uploadsDir, { recursive: true, force: true }).catch(() => {});
await fs.rm(downloadsDir, { recursive: true, force: true }).catch(() => {});
}
function streamFileAndDelete(tmpPath: string): ReadableStream<Uint8Array> {
const reader = Bun.file(tmpPath).stream().getReader();
let cleaned = false;
const cleanup = async () => {
if (cleaned) return;
cleaned = true;
await fs.unlink(tmpPath).catch(() => {});
};
return new ReadableStream<Uint8Array>({
async pull(controller) {
try {
const { done, value } = await reader.read();
if (done) {
controller.close();
await cleanup();
return;
}
if (value) {
controller.enqueue(value);
}
} catch (err) {
controller.error(err);
await cleanup();
}
},
async cancel() {
try {
await reader.cancel();
} catch {}
await cleanup();
},
});
}
const UUID_TOKEN_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
const uploadIntents = new Map<string, UploadIntent>();
const uploadPulls = new Map<string, UploadPull>();
function isSafeRemotePath(value: string): boolean {
if (!value || value.length > 4096) return false;
return !/[\x00-\x1F\x7F]/.test(value);
}
async function serveDownloadById(
req: Request,
downloadId: string,
server: RequestIpProvider,
deps: FileDownloadRouteDeps,
): Promise<Response> {
const user = await authenticateRequest(req);
if (!user) {
return new Response("Unauthorized", { status: 401 });
}
if (user.role !== "admin" && user.role !== "operator") {
return new Response("Forbidden: Admin or operator access required", { status: 403 });
}
logger.debug("[filebrowser] http download request", {
downloadId,
userId: user.userId,
ip: server.requestIP(req)?.address || "unknown",
});
if (!UUID_TOKEN_RE.test(downloadId)) {
return new Response("Bad request", { status: 400 });
}
const intent = deps.downloadIntents.get(downloadId);
if (!intent || intent.userId !== user.userId || intent.expiresAt < Date.now()) {
logger.debug("[filebrowser] http download intent missing", {
downloadId,
userId: user.userId,
intentUserId: intent?.userId,
expiresAt: intent?.expiresAt,
});
return new Response("Not found", { status: 404 });
}
deps.downloadIntents.delete(downloadId);
clearTimeout(intent.timeout);
const clientId = intent.clientId;
const downloadPath = intent.path;
const target = clientManager.getClient(clientId);
if (!target) {
logger.debug("[filebrowser] http download target offline", {
downloadId,
clientId,
});
return new Response("Client offline", { status: 404 });
}
const commandId = uuidv4();
const downloadDir = path.join(deps.DATA_DIR, "downloads");
await fs.mkdir(downloadDir, { recursive: true });
const tmpPath = path.join(downloadDir, `${commandId}.bin`);
let fileName = path.basename(downloadPath) || "download.bin";
try {
fileName = deps.sanitizeOutputName(fileName);
} catch {
fileName = "download.bin";
}
const fileHandle = await fs.open(tmpPath, "w+");
logger.debug("[filebrowser] http download start", {
commandId,
clientId,
path: downloadPath,
tmpPath,
});
const downloadPromise = new Promise<PendingHttpDownload>((resolve, reject) => {
const timeout = setTimeout(() => {
deps.pendingHttpDownloads.delete(commandId);
void fileHandle.close();
void fs.unlink(tmpPath).catch(() => {});
reject(new Error("Download timed out"));
}, 5 * 60_000);
deps.pendingHttpDownloads.set(commandId, {
commandId,
clientId,
path: downloadPath,
fileName,
total: 0,
receivedBytes: 0,
receivedOffsets: new Set(),
receivedChunks: new Set(),
chunkSize: 0,
expectedChunks: 0,
loggedTotal: false,
loggedFirstChunk: false,
tmpPath,
fileHandle,
resolve,
reject,
timeout,
});
});
target.ws.send(
encodeMessage({
type: "command",
commandType: "file_download",
id: commandId,
payload: { path: downloadPath },
}),
);
logger.debug("[filebrowser] http download command sent", {
commandId,
clientId,
path: downloadPath,
});
logAudit({
timestamp: Date.now(),
username: user.username,
ip: server.requestIP(req)?.address || "unknown",
action: AuditAction.FILE_DOWNLOAD,
targetClientId: clientId,
details: JSON.stringify({ path: downloadPath, via: "http" }),
success: true,
});
let completed: PendingHttpDownload;
try {
completed = await downloadPromise;
} catch (err) {
logger.debug("[filebrowser] http download failed", {
commandId,
clientId,
path: downloadPath,
error: (err as Error)?.message || String(err),
});
return new Response((err as Error).message || "Download failed", { status: 500 });
}
logger.debug("[filebrowser] http download complete", {
commandId,
clientId,
path: downloadPath,
total: completed.total,
receivedBytes: completed.receivedBytes,
expectedChunks: completed.expectedChunks,
receivedChunks: completed.receivedChunks.size,
});
const headers = {
...deps.secureHeaders("application/octet-stream"),
"Content-Disposition": `attachment; filename="${completed.fileName}"`,
"Cache-Control": "no-store, private",
};
return new Response(streamFileAndDelete(completed.tmpPath), { headers });
}
export async function handleFileDownloadRoutes( export async function handleFileDownloadRoutes(
req: Request, req: Request,
url: URL, url: URL,
server: RequestIpProvider, server: RequestIpProvider,
deps: FileDownloadRouteDeps, deps: FileDownloadRouteDeps,
): Promise<Response | null> { ): Promise<Response | null> {
if (!url.pathname.startsWith("/api/file/download")) { if (req.method === "POST" && url.pathname === "/api/file/upload/request") {
return null;
}
if (req.method === "POST" && url.pathname === "/api/file/download") {
const user = await authenticateRequest(req); const user = await authenticateRequest(req);
if (!user) { if (!user) {
return new Response("Unauthorized", { status: 401 }); return new Response("Unauthorized", { status: 401 });
@@ -74,151 +302,225 @@ export async function handleFileDownloadRoutes(
return new Response("Bad request", { status: 400 }); return new Response("Bad request", { status: 400 });
} }
const downloadId = typeof body?.downloadId === "string" ? body.downloadId : ""; const clientId = typeof body?.clientId === "string" ? body.clientId.trim() : "";
logger.debug("[filebrowser] http download request", { const uploadPath = typeof body?.path === "string" ? body.path.trim() : "";
downloadId, const fileName = typeof body?.fileName === "string" ? body.fileName.trim() : "upload.bin";
userId: user.userId, if (!clientId || !uploadPath || !isSafeRemotePath(uploadPath)) {
ip: server.requestIP(req)?.address || "unknown",
});
if (!downloadId) {
return new Response("Bad request", { status: 400 }); return new Response("Bad request", { status: 400 });
} }
const intent = deps.downloadIntents.get(downloadId); logger.debug("[filebrowser] http upload request", {
if (!intent || intent.userId !== user.userId || intent.expiresAt < Date.now()) { userId: user.userId,
logger.debug("[filebrowser] http download intent missing", { clientId,
downloadId, path: uploadPath,
userId: user.userId, fileName,
intentUserId: intent?.userId, ip: server.requestIP(req)?.address || "unknown",
expiresAt: intent?.expiresAt, });
});
return new Response("Not found", { status: 404 });
}
deps.downloadIntents.delete(downloadId);
clearTimeout(intent.timeout);
const clientId = intent.clientId;
const downloadPath = intent.path;
const target = clientManager.getClient(clientId); const target = clientManager.getClient(clientId);
if (!target) { if (!target) {
logger.debug("[filebrowser] http download target offline", {
downloadId,
clientId,
});
return new Response("Client offline", { status: 404 }); return new Response("Client offline", { status: 404 });
} }
const commandId = uuidv4(); const uploadId = uuidv4();
const downloadDir = path.join(deps.DATA_DIR, "downloads"); const expiresAt = Date.now() + 2 * 60_000;
await fs.mkdir(downloadDir, { recursive: true }); const timeout = setTimeout(() => {
const tmpPath = path.join(downloadDir, `${commandId}.bin`); uploadIntents.delete(uploadId);
}, 2 * 60_000);
let fileName = path.basename(downloadPath) || "download.bin"; uploadIntents.set(uploadId, {
try { id: uploadId,
fileName = deps.sanitizeOutputName(fileName); userId: user.userId,
} catch { clientId,
fileName = "download.bin"; path: uploadPath,
fileName,
expiresAt,
timeout,
});
return Response.json({
ok: true,
uploadId,
uploadUrl: `/api/file/upload/${encodeURIComponent(uploadId)}`,
});
}
if ((req.method === "PUT" || req.method === "POST") && url.pathname.startsWith("/api/file/upload/")) {
const user = await authenticateRequest(req);
if (!user) {
return new Response("Unauthorized", { status: 401 });
}
if (user.role !== "admin" && user.role !== "operator") {
return new Response("Forbidden: Admin or operator access required", { status: 403 });
} }
const fileHandle = await fs.open(tmpPath, "w+"); let uploadId = "";
logger.debug("[filebrowser] http download start", {
commandId,
clientId,
path: downloadPath,
tmpPath,
});
const downloadPromise = new Promise<PendingHttpDownload>((resolve, reject) => {
const timeout = setTimeout(() => {
deps.pendingHttpDownloads.delete(commandId);
void fileHandle.close();
void fs.unlink(tmpPath).catch(() => {});
reject(new Error("Download timed out"));
}, 5 * 60_000);
deps.pendingHttpDownloads.set(commandId, {
commandId,
clientId,
path: downloadPath,
fileName,
total: 0,
receivedBytes: 0,
receivedOffsets: new Set(),
receivedChunks: new Set(),
chunkSize: 0,
expectedChunks: 0,
loggedTotal: false,
loggedFirstChunk: false,
tmpPath,
fileHandle,
resolve,
reject,
timeout,
});
});
target.ws.send(
encodeMessage({
type: "command",
commandType: "file_download",
id: commandId,
payload: { path: downloadPath },
}),
);
logger.debug("[filebrowser] http download command sent", {
commandId,
clientId,
path: downloadPath,
});
logAudit({
timestamp: Date.now(),
username: user.username,
ip: server.requestIP(req)?.address || "unknown",
action: AuditAction.FILE_DOWNLOAD,
targetClientId: clientId,
details: JSON.stringify({ path: downloadPath, via: "http" }),
success: true,
});
let completed: PendingHttpDownload;
try { try {
completed = await downloadPromise; uploadId = decodeURIComponent(url.pathname.slice("/api/file/upload/".length));
} catch {
return new Response("Bad request", { status: 400 });
}
if (!UUID_TOKEN_RE.test(uploadId)) {
return new Response("Bad request", { status: 400 });
}
logger.debug("[filebrowser] http upload stage", {
method: req.method,
uploadId,
userId: user.userId,
ip: server.requestIP(req)?.address || "unknown",
});
const intent = uploadIntents.get(uploadId);
if (!intent || intent.userId !== user.userId || intent.expiresAt < Date.now()) {
return new Response("Not found", { status: 404 });
}
const uploadDir = path.join(deps.DATA_DIR, "uploads");
await fs.mkdir(uploadDir, { recursive: true });
const tmpPath = path.join(uploadDir, `${uploadId}.bin`);
let stagedSize = 0;
const fileHandle = await fs.open(tmpPath, "w");
try {
if (req.body) {
const reader = req.body.getReader();
let position = 0;
while (true) {
const { done, value } = await reader.read();
if (done) break;
if (!value || value.byteLength === 0) continue;
await fileHandle.write(value, 0, value.byteLength, position);
position += value.byteLength;
stagedSize += value.byteLength;
}
} else {
const bytes = new Uint8Array(await req.arrayBuffer());
if (bytes.byteLength > 0) {
await fileHandle.write(bytes, 0, bytes.byteLength, 0);
stagedSize = bytes.byteLength;
}
}
} catch (err) { } catch (err) {
logger.debug("[filebrowser] http download failed", { logger.debug("[filebrowser] http upload stage error", {
commandId, uploadId,
clientId,
path: downloadPath,
error: (err as Error)?.message || String(err), error: (err as Error)?.message || String(err),
}); });
return new Response((err as Error).message || "Download failed", { status: 500 }); await fileHandle.close();
await fs.unlink(tmpPath).catch(() => {});
return new Response("Upload staging failed", { status: 500 });
}
await fileHandle.close();
logger.debug("[filebrowser] http upload staged bytes", {
uploadId,
bytes: stagedSize,
clientId: intent.clientId,
path: intent.path,
});
uploadIntents.delete(uploadId);
clearTimeout(intent.timeout);
const pullId = uuidv4();
const pullExpiresAt = Date.now() + 10 * 60_000;
const pullTimeout = setTimeout(() => {
const pull = uploadPulls.get(pullId);
uploadPulls.delete(pullId);
if (pull) {
void fs.unlink(pull.tmpPath).catch(() => {});
}
}, 10 * 60_000);
uploadPulls.set(pullId, {
id: pullId,
clientId: intent.clientId,
path: intent.path,
fileName: intent.fileName,
tmpPath,
size: stagedSize,
expiresAt: pullExpiresAt,
timeout: pullTimeout,
});
return Response.json({
ok: true,
size: stagedSize,
path: intent.path,
pullUrl: `${url.origin}/api/file/upload/pull/${encodeURIComponent(pullId)}`,
});
}
if (req.method === "GET" && url.pathname.startsWith("/api/file/upload/pull/")) {
const agentToken = getConfig().auth.agentToken;
if (!isAuthorizedAgentRequest(req, url, agentToken)) {
return new Response("Unauthorized", { status: 401 });
} }
logger.debug("[filebrowser] http download complete", { let pullId = "";
commandId, try {
clientId, pullId = decodeURIComponent(url.pathname.slice("/api/file/upload/pull/".length));
path: downloadPath, } catch {
total: completed.total, return new Response("Bad request", { status: 400 });
receivedBytes: completed.receivedBytes, }
expectedChunks: completed.expectedChunks, if (!UUID_TOKEN_RE.test(pullId)) {
receivedChunks: completed.receivedChunks.size, return new Response("Bad request", { status: 400 });
}
const pull = uploadPulls.get(pullId);
if (!pull || pull.expiresAt < Date.now()) {
return new Response("Not found", { status: 404 });
}
const requesterClientId = req.headers.get("x-overlord-client-id") || "";
if (!requesterClientId || requesterClientId !== pull.clientId) {
return new Response("Forbidden", { status: 403 });
}
logger.debug("[filebrowser] http upload pull", {
pullId,
clientId: pull.clientId,
path: pull.path,
bytes: pull.size,
ip: server.requestIP(req)?.address || "unknown",
}); });
uploadPulls.delete(pullId);
clearTimeout(pull.timeout);
const headers = { const headers = {
...deps.secureHeaders("application/octet-stream"), ...deps.secureHeaders("application/octet-stream"),
"Content-Disposition": `attachment; filename="${completed.fileName}"`, "Content-Disposition": `attachment; filename="${deps.sanitizeOutputName(path.basename(pull.fileName) || "upload.bin")}"`,
"Cache-Control": "no-store", "Cache-Control": "no-store, private",
"Content-Length": String(pull.size),
}; };
setTimeout(() => { return new Response(streamFileAndDelete(pull.tmpPath), { headers });
void fs.unlink(completed.tmpPath).catch(() => {}); }
}, 5 * 60_000);
return new Response(Bun.file(completed.tmpPath), { headers }); if (!url.pathname.startsWith("/api/file/download")) {
return null;
}
if (req.method === "GET" && url.pathname.startsWith("/api/file/download/")) {
let downloadId = "";
try {
downloadId = decodeURIComponent(url.pathname.slice("/api/file/download/".length));
} catch {
return new Response("Bad request", { status: 400 });
}
return serveDownloadById(req, downloadId, server, deps);
}
if (req.method === "POST" && url.pathname === "/api/file/download") {
let body: any = {};
try {
body = await req.json();
} catch {
return new Response("Bad request", { status: 400 });
}
const downloadId = typeof body?.downloadId === "string" ? body.downloadId : "";
return serveDownloadById(req, downloadId, server, deps);
} }
if (req.method === "POST" && url.pathname === "/api/file/download/request") { if (req.method === "POST" && url.pathname === "/api/file/download/request") {
@@ -237,9 +539,9 @@ export async function handleFileDownloadRoutes(
return new Response("Bad request", { status: 400 }); return new Response("Bad request", { status: 400 });
} }
const clientId = typeof body?.clientId === "string" ? body.clientId : ""; const clientId = typeof body?.clientId === "string" ? body.clientId.trim() : "";
const downloadPath = typeof body?.path === "string" ? body.path : ""; const downloadPath = typeof body?.path === "string" ? body.path.trim() : "";
if (!clientId || !downloadPath) { if (!clientId || !downloadPath || !isSafeRemotePath(downloadPath)) {
return new Response("Bad request", { status: 400 }); return new Response("Bad request", { status: 400 });
} }
@@ -263,7 +565,11 @@ export async function handleFileDownloadRoutes(
timeout, timeout,
}); });
return Response.json({ ok: true, downloadId }); return Response.json({
ok: true,
downloadId,
downloadUrl: `/api/file/download/${encodeURIComponent(downloadId)}`,
});
} }
return null; return null;
@@ -116,6 +116,19 @@ export function handleFileBrowserViewerMessage(ws: ServerWebSocket<SocketData>,
target.ws.send(encodeMessage({ type: "command", commandType: "file_execute", id: payload.id || commandId, payload: actualPayload } as any)); target.ws.send(encodeMessage({ type: "command", commandType: "file_execute", id: payload.id || commandId, payload: actualPayload } as any));
metrics.recordCommand("file_execute"); metrics.recordCommand("file_execute");
break; break;
case "file_upload_http":
target.ws.send(encodeMessage({ type: "command", commandType: "file_upload_http", id: payload.id || commandId, payload: actualPayload } as any));
metrics.recordCommand("file_upload");
logAudit({
timestamp: Date.now(),
username: (ws.data as any).username || "unknown",
ip: ws.data.ip || "unknown",
action: AuditAction.FILE_UPLOAD,
targetClientId: clientId,
details: JSON.stringify({ path: actualPayload.path || "", mode: "http_pull" }),
success: true,
});
break;
default: default:
break; break;
} }
@@ -152,27 +165,13 @@ export function handleFileBrowserViewerMessage(ws: ServerWebSocket<SocketData>,
case "file_upload": { case "file_upload": {
const upload = normalizeFileUploadPayload(payload); const upload = normalizeFileUploadPayload(payload);
if (!upload) return; if (!upload) return;
target.ws.send(encodeMessage({ safeSendViewer(ws, {
type: "command", type: "file_upload_result",
commandType: "file_upload", commandId,
id: commandId, transferId: upload.transferId,
payload: { path: upload.path,
path: upload.path, ok: false,
data: upload.data, error: "chunked uploads are disabled; refresh and retry",
offset: upload.offset,
total: upload.total,
transferId: upload.transferId,
},
} as any));
metrics.recordCommand("file_upload");
logAudit({
timestamp: Date.now(),
username: (ws.data as any).username || "unknown",
ip: ws.data.ip || "unknown",
action: AuditAction.FILE_UPLOAD,
targetClientId: clientId,
details: JSON.stringify({ path: upload.path, size: upload.data.length, offset: upload.offset }),
success: true,
}); });
break; break;
} }