new upload system
This commit is contained in:
@@ -1345,6 +1345,12 @@ func HandleCommand(ctx context.Context, env *runtime.Env, envelope map[string]in
|
|||||||
data = d
|
data = d
|
||||||
}
|
}
|
||||||
return HandleFileUpload(ctx, env, cmdID, path, data, offset, total, transferID)
|
return HandleFileUpload(ctx, env, cmdID, path, data, offset, total, transferID)
|
||||||
|
case "file_upload_http":
|
||||||
|
payload, _ := envelope["payload"].(map[string]interface{})
|
||||||
|
path, _ := payload["path"].(string)
|
||||||
|
sourceURL, _ := payload["url"].(string)
|
||||||
|
total := payloadNumberToInt64(payload["total"])
|
||||||
|
return HandleFileUploadHTTP(ctx, env, cmdID, path, sourceURL, total)
|
||||||
case "file_delete":
|
case "file_delete":
|
||||||
path, _ := envelopePayloadString(envelope, "path")
|
path, _ := envelopePayloadString(envelope, "path")
|
||||||
return HandleFileDelete(ctx, env, cmdID, path)
|
return HandleFileDelete(ctx, env, cmdID, path)
|
||||||
|
|||||||
@@ -5,9 +5,13 @@ import (
|
|||||||
"bufio"
|
"bufio"
|
||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
|
"crypto/tls"
|
||||||
|
"crypto/x509"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"log"
|
"log"
|
||||||
|
"net/http"
|
||||||
|
"net/url"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"runtime"
|
"runtime"
|
||||||
@@ -462,6 +466,88 @@ func HandleFileUpload(ctx context.Context, env *agentRuntime.Env, cmdID string,
|
|||||||
return wire.WriteMsg(ctx, env.Conn, result)
|
return wire.WriteMsg(ctx, env.Conn, result)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func HandleFileUploadHTTP(ctx context.Context, env *agentRuntime.Env, cmdID string, destPath string, sourceURL string, expectedSize int64) error {
|
||||||
|
parsed, err := url.Parse(strings.TrimSpace(sourceURL))
|
||||||
|
if err != nil || parsed == nil || parsed.Host == "" {
|
||||||
|
return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: false, Message: "invalid upload url"})
|
||||||
|
}
|
||||||
|
if parsed.Scheme != "https" && parsed.Scheme != "http" {
|
||||||
|
return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: false, Message: "unsupported upload url scheme"})
|
||||||
|
}
|
||||||
|
|
||||||
|
tlsConfig := &tls.Config{InsecureSkipVerify: env.Cfg.TLSInsecureSkipVerify, MinVersion: tls.VersionTLS12}
|
||||||
|
if caPath := strings.TrimSpace(env.Cfg.TLSCAPath); caPath != "" {
|
||||||
|
caBytes, err := os.ReadFile(caPath)
|
||||||
|
if err != nil {
|
||||||
|
return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: false, Message: fmt.Sprintf("failed to read TLS CA: %v", err)})
|
||||||
|
}
|
||||||
|
pool := x509.NewCertPool()
|
||||||
|
if !pool.AppendCertsFromPEM(caBytes) {
|
||||||
|
return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: false, Message: "failed to parse TLS CA"})
|
||||||
|
}
|
||||||
|
tlsConfig.RootCAs = pool
|
||||||
|
}
|
||||||
|
|
||||||
|
client := &http.Client{Transport: &http.Transport{TLSClientConfig: tlsConfig}}
|
||||||
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, parsed.String(), nil)
|
||||||
|
if err != nil {
|
||||||
|
return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: false, Message: err.Error()})
|
||||||
|
}
|
||||||
|
if token := strings.TrimSpace(env.Cfg.AgentToken); token != "" {
|
||||||
|
req.Header.Set("x-agent-token", token)
|
||||||
|
}
|
||||||
|
if id := strings.TrimSpace(env.Cfg.ID); id != "" {
|
||||||
|
req.Header.Set("x-overlord-client-id", id)
|
||||||
|
}
|
||||||
|
|
||||||
|
resp, err := client.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: false, Message: err.Error()})
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
|
||||||
|
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||||
|
return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: false, Message: fmt.Sprintf("upload fetch failed: status %d", resp.StatusCode)})
|
||||||
|
}
|
||||||
|
|
||||||
|
dir := filepath.Dir(destPath)
|
||||||
|
if dir != "." {
|
||||||
|
if err := os.MkdirAll(dir, 0755); err != nil {
|
||||||
|
return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: false, Message: err.Error()})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
tmpPath := destPath + ".httpuploading"
|
||||||
|
f, err := os.OpenFile(tmpPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644)
|
||||||
|
if err != nil {
|
||||||
|
return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: false, Message: err.Error()})
|
||||||
|
}
|
||||||
|
|
||||||
|
written, copyErr := io.Copy(f, resp.Body)
|
||||||
|
closeErr := f.Close()
|
||||||
|
if copyErr != nil {
|
||||||
|
_ = os.Remove(tmpPath)
|
||||||
|
return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: false, Message: copyErr.Error()})
|
||||||
|
}
|
||||||
|
if closeErr != nil {
|
||||||
|
_ = os.Remove(tmpPath)
|
||||||
|
return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: false, Message: closeErr.Error()})
|
||||||
|
}
|
||||||
|
|
||||||
|
if expectedSize > 0 && written != expectedSize {
|
||||||
|
_ = os.Remove(tmpPath)
|
||||||
|
return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: false, Message: "upload size mismatch"})
|
||||||
|
}
|
||||||
|
|
||||||
|
_ = os.Remove(destPath)
|
||||||
|
if err := os.Rename(tmpPath, destPath); err != nil {
|
||||||
|
_ = os.Remove(tmpPath)
|
||||||
|
return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: false, Message: err.Error()})
|
||||||
|
}
|
||||||
|
|
||||||
|
return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: true})
|
||||||
|
}
|
||||||
|
|
||||||
func HandleFileDelete(ctx context.Context, env *agentRuntime.Env, cmdID string, path string) error {
|
func HandleFileDelete(ctx context.Context, env *agentRuntime.Env, cmdID string, path string) error {
|
||||||
log.Printf("file_delete: %s", path)
|
log.Printf("file_delete: %s", path)
|
||||||
|
|
||||||
|
|||||||
@@ -3,6 +3,8 @@ package handlers
|
|||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"testing"
|
"testing"
|
||||||
@@ -467,3 +469,87 @@ func TestHandleFileZip(t *testing.T) {
|
|||||||
t.Error("Expected either command_result or file_download message")
|
t.Error("Expected either command_result or file_download message")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestHandleFileUploadHTTP(t *testing.T) {
|
||||||
|
data := []byte("http-upload-payload")
|
||||||
|
|
||||||
|
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if got := r.Header.Get("x-overlord-client-id"); got != "client-abc" {
|
||||||
|
t.Fatalf("expected x-overlord-client-id client-abc, got %q", got)
|
||||||
|
}
|
||||||
|
if got := r.Header.Get("x-agent-token"); got != "agent-token" {
|
||||||
|
t.Fatalf("expected x-agent-token agent-token, got %q", got)
|
||||||
|
}
|
||||||
|
_, _ = w.Write(data)
|
||||||
|
}))
|
||||||
|
defer ts.Close()
|
||||||
|
|
||||||
|
tmpDir := t.TempDir()
|
||||||
|
destPath := filepath.Join(tmpDir, "uploaded-http.bin")
|
||||||
|
|
||||||
|
writer := &testWriter{}
|
||||||
|
env := &rt.Env{
|
||||||
|
Conn: writer,
|
||||||
|
Cfg: config.Config{
|
||||||
|
ID: "client-abc",
|
||||||
|
AgentToken: "agent-token",
|
||||||
|
TLSInsecureSkipVerify: true,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := HandleFileUploadHTTP(context.Background(), env, "cmd-http-upload", destPath, ts.URL+"/file", int64(len(data))); err != nil {
|
||||||
|
t.Fatalf("HandleFileUploadHTTP failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
content, err := os.ReadFile(destPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed reading uploaded file: %v", err)
|
||||||
|
}
|
||||||
|
if !bytes.Equal(content, data) {
|
||||||
|
t.Fatalf("uploaded content mismatch")
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(writer.msgs) != 1 {
|
||||||
|
t.Fatalf("expected 1 message, got %d", len(writer.msgs))
|
||||||
|
}
|
||||||
|
var result wire.CommandResult
|
||||||
|
if err := msgpack.Unmarshal(writer.msgs[0], &result); err != nil {
|
||||||
|
t.Fatalf("failed to unmarshal command result: %v", err)
|
||||||
|
}
|
||||||
|
if !result.OK {
|
||||||
|
t.Fatalf("expected OK=true, got false: %s", result.Message)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandleFileUploadHTTP_SizeMismatch(t *testing.T) {
|
||||||
|
data := []byte("small")
|
||||||
|
|
||||||
|
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
_, _ = w.Write(data)
|
||||||
|
}))
|
||||||
|
defer ts.Close()
|
||||||
|
|
||||||
|
tmpDir := t.TempDir()
|
||||||
|
destPath := filepath.Join(tmpDir, "uploaded-http-mismatch.bin")
|
||||||
|
|
||||||
|
writer := &testWriter{}
|
||||||
|
env := &rt.Env{
|
||||||
|
Conn: writer,
|
||||||
|
Cfg: config.Config{TLSInsecureSkipVerify: true},
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := HandleFileUploadHTTP(context.Background(), env, "cmd-http-upload-mismatch", destPath, ts.URL+"/file", int64(len(data)+1)); err != nil {
|
||||||
|
t.Fatalf("HandleFileUploadHTTP failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(writer.msgs) != 1 {
|
||||||
|
t.Fatalf("expected 1 message, got %d", len(writer.msgs))
|
||||||
|
}
|
||||||
|
var result wire.CommandResult
|
||||||
|
if err := msgpack.Unmarshal(writer.msgs[0], &result); err != nil {
|
||||||
|
t.Fatalf("failed to unmarshal command result: %v", err)
|
||||||
|
}
|
||||||
|
if result.OK {
|
||||||
|
t.Fatalf("expected OK=false for size mismatch")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ let lastSuccessfulResponse = 0;
|
|||||||
let pendingToast = null;
|
let pendingToast = null;
|
||||||
const recentToasts = new Map();
|
const recentToasts = new Map();
|
||||||
const pendingCommandResults = new Map();
|
const pendingCommandResults = new Map();
|
||||||
|
const pendingCommandWaiters = new Map();
|
||||||
const VIRTUALIZATION_THRESHOLD = 400;
|
const VIRTUALIZATION_THRESHOLD = 400;
|
||||||
const VIRTUAL_ROW_HEIGHT = 58;
|
const VIRTUAL_ROW_HEIGHT = 58;
|
||||||
const VIRTUAL_OVERSCAN = 8;
|
const VIRTUAL_OVERSCAN = 8;
|
||||||
@@ -192,6 +193,25 @@ function trackCommandResult(commandId, options = {}) {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function waitForCommandResult(commandId, timeoutMs = 10 * 60 * 1000) {
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
if (!commandId) {
|
||||||
|
reject(new Error("missing command id"));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const existing = pendingCommandWaiters.get(commandId);
|
||||||
|
if (existing) {
|
||||||
|
clearTimeout(existing.timeoutId);
|
||||||
|
existing.reject(new Error("superseded command waiter"));
|
||||||
|
}
|
||||||
|
const timeoutId = setTimeout(() => {
|
||||||
|
pendingCommandWaiters.delete(commandId);
|
||||||
|
reject(new Error("command timed out"));
|
||||||
|
}, timeoutMs);
|
||||||
|
pendingCommandWaiters.set(commandId, { resolve, reject, timeoutId });
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
document.addEventListener("visibilitychange", () => {
|
document.addEventListener("visibilitychange", () => {
|
||||||
if (document.visibilityState !== "visible") return;
|
if (document.visibilityState !== "visible") return;
|
||||||
if (!pendingToast || typeof window.showToast !== "function") return;
|
if (!pendingToast || typeof window.showToast !== "function") return;
|
||||||
@@ -838,7 +858,13 @@ function downloadFile(path) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
const requestData = await requestRes.json();
|
const requestData = await requestRes.json();
|
||||||
if (!requestData?.downloadId) {
|
const downloadUrl = typeof requestData?.downloadUrl === "string"
|
||||||
|
? requestData.downloadUrl
|
||||||
|
: (requestData?.downloadId
|
||||||
|
? `/api/file/download/${encodeURIComponent(requestData.downloadId)}`
|
||||||
|
: "");
|
||||||
|
|
||||||
|
if (!downloadUrl) {
|
||||||
notifyToast("Download failed", "error", 5000);
|
notifyToast("Download failed", "error", 5000);
|
||||||
removeTransfer(transferId);
|
removeTransfer(transferId);
|
||||||
fileDownloads.delete(path);
|
fileDownloads.delete(path);
|
||||||
@@ -847,19 +873,17 @@ function downloadFile(path) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
console.debug("[filebrowser] download request accepted", {
|
console.debug("[filebrowser] download request accepted", {
|
||||||
downloadId: requestData.downloadId,
|
downloadUrl,
|
||||||
});
|
});
|
||||||
|
|
||||||
console.debug("[filebrowser] download stream start", {
|
console.debug("[filebrowser] download stream start", {
|
||||||
downloadId: requestData.downloadId,
|
downloadUrl,
|
||||||
});
|
});
|
||||||
|
|
||||||
const res = await fetch("/api/file/download", {
|
const res = await fetch(downloadUrl, {
|
||||||
method: "POST",
|
method: "GET",
|
||||||
headers: { "Content-Type": "application/json" },
|
|
||||||
credentials: "include",
|
credentials: "include",
|
||||||
signal: abortController.signal,
|
signal: abortController.signal,
|
||||||
body: JSON.stringify({ downloadId: requestData.downloadId }),
|
|
||||||
});
|
});
|
||||||
|
|
||||||
console.debug("[filebrowser] download response", {
|
console.debug("[filebrowser] download response", {
|
||||||
@@ -1309,6 +1333,24 @@ function handleFileUploadResult(msg) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
const offset = toNumber(msg.offset);
|
const offset = toNumber(msg.offset);
|
||||||
|
if (offset !== null) {
|
||||||
|
const firstAckForOffset = !transfer.ackedOffsets.has(offset);
|
||||||
|
if (firstAckForOffset) {
|
||||||
|
transfer.ackedOffsets.add(offset);
|
||||||
|
transfer.receivedChunks += 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (Number.isFinite(msg.received)) {
|
||||||
|
transfer.receivedBytes = Math.min(Number(msg.received), transfer.total);
|
||||||
|
transfer.sent = transfer.receivedBytes;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (transfer.total > 0) {
|
||||||
|
transfer.progress = Math.round((transfer.sent / transfer.total) * 100);
|
||||||
|
updateTransferProgress(transfer.id, transfer.progress, transfer.sent, transfer.total);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
const pendingOffset = offset !== null
|
const pendingOffset = offset !== null
|
||||||
? offset
|
? offset
|
||||||
: (transfer.pendingAcks.has(0) ? 0 : null);
|
: (transfer.pendingAcks.has(0) ? 0 : null);
|
||||||
@@ -1352,6 +1394,20 @@ function handleCommandResult(msg) {
|
|||||||
const tracked = msg.commandId
|
const tracked = msg.commandId
|
||||||
? pendingCommandResults.get(msg.commandId)
|
? pendingCommandResults.get(msg.commandId)
|
||||||
: null;
|
: null;
|
||||||
|
|
||||||
|
const waiter = msg.commandId
|
||||||
|
? pendingCommandWaiters.get(msg.commandId)
|
||||||
|
: null;
|
||||||
|
if (waiter) {
|
||||||
|
clearTimeout(waiter.timeoutId);
|
||||||
|
pendingCommandWaiters.delete(msg.commandId);
|
||||||
|
if (msg.ok) {
|
||||||
|
waiter.resolve(msg);
|
||||||
|
} else {
|
||||||
|
waiter.reject(new Error(msg.message || "operation failed"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if (!tracked) {
|
if (!tracked) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@@ -1528,57 +1584,6 @@ if (sortOrderBtn) {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
function waitForUploadAck(transfer, offset, timeoutMs) {
|
|
||||||
return new Promise((resolve, reject) => {
|
|
||||||
const existing = transfer.pendingAcks.get(offset);
|
|
||||||
if (existing) {
|
|
||||||
clearTimeout(existing.timeoutId);
|
|
||||||
existing.reject(new Error("Superseded upload ack"));
|
|
||||||
}
|
|
||||||
|
|
||||||
const timeoutId = setTimeout(() => {
|
|
||||||
transfer.pendingAcks.delete(offset);
|
|
||||||
reject(new Error("Upload chunk timed out"));
|
|
||||||
}, timeoutMs);
|
|
||||||
|
|
||||||
transfer.pendingAcks.set(offset, { resolve, reject, timeoutId });
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
function clearUploadPendingAcks(transfer, reason) {
|
|
||||||
transfer.pendingAcks.forEach((pending, _offset) => {
|
|
||||||
clearTimeout(pending.timeoutId);
|
|
||||||
pending.reject(new Error(reason));
|
|
||||||
});
|
|
||||||
transfer.pendingAcks.clear();
|
|
||||||
}
|
|
||||||
|
|
||||||
async function sendChunkWithRetry(transfer, payload, offset) {
|
|
||||||
const maxRetries = 5;
|
|
||||||
const ackTimeoutMs = 20000;
|
|
||||||
const backoffMs = 300;
|
|
||||||
|
|
||||||
for (let attempt = 1; attempt <= maxRetries; attempt += 1) {
|
|
||||||
if (transfer.cancelled) {
|
|
||||||
throw new Error("Upload cancelled");
|
|
||||||
}
|
|
||||||
|
|
||||||
const ackPromise = waitForUploadAck(transfer, offset, ackTimeoutMs);
|
|
||||||
send(payload);
|
|
||||||
|
|
||||||
try {
|
|
||||||
return await ackPromise;
|
|
||||||
} catch (err) {
|
|
||||||
if (attempt === maxRetries) {
|
|
||||||
throw err;
|
|
||||||
}
|
|
||||||
await new Promise((resolve) => setTimeout(resolve, backoffMs * attempt));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
throw new Error("Upload failed after retries");
|
|
||||||
}
|
|
||||||
|
|
||||||
function finishUpload(transfer) {
|
function finishUpload(transfer) {
|
||||||
console.log("Upload complete:", transfer.path);
|
console.log("Upload complete:", transfer.path);
|
||||||
updateStatus("connected", "Connected");
|
updateStatus("connected", "Connected");
|
||||||
@@ -1589,6 +1594,111 @@ function finishUpload(transfer) {
|
|||||||
listFiles(currentPath);
|
listFiles(currentPath);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function uploadFileViaHttpPull(file, path, transfer) {
|
||||||
|
console.debug("[filebrowser] upload request start", {
|
||||||
|
clientId,
|
||||||
|
path,
|
||||||
|
fileName: file.name,
|
||||||
|
size: file.size,
|
||||||
|
});
|
||||||
|
|
||||||
|
const requestRes = await fetch("/api/file/upload/request", {
|
||||||
|
method: "POST",
|
||||||
|
headers: { "Content-Type": "application/json" },
|
||||||
|
credentials: "include",
|
||||||
|
body: JSON.stringify({
|
||||||
|
clientId,
|
||||||
|
path,
|
||||||
|
fileName: file.name,
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!requestRes.ok) {
|
||||||
|
const text = await requestRes.text();
|
||||||
|
throw new Error(text || "upload request failed");
|
||||||
|
}
|
||||||
|
|
||||||
|
const requestData = await requestRes.json();
|
||||||
|
const uploadUrl = typeof requestData?.uploadUrl === "string"
|
||||||
|
? requestData.uploadUrl
|
||||||
|
: (requestData?.uploadId
|
||||||
|
? `/api/file/upload/${encodeURIComponent(requestData.uploadId)}`
|
||||||
|
: "");
|
||||||
|
if (!uploadUrl) {
|
||||||
|
throw new Error("upload request failed");
|
||||||
|
}
|
||||||
|
|
||||||
|
console.debug("[filebrowser] upload stage url", { uploadUrl });
|
||||||
|
|
||||||
|
let uploadRes;
|
||||||
|
try {
|
||||||
|
uploadRes = await fetch(uploadUrl, {
|
||||||
|
method: "PUT",
|
||||||
|
headers: { "Content-Type": "application/octet-stream" },
|
||||||
|
credentials: "include",
|
||||||
|
body: file,
|
||||||
|
});
|
||||||
|
} catch (err) {
|
||||||
|
console.warn("[filebrowser] upload stage PUT failed, retrying as POST", err);
|
||||||
|
uploadRes = await fetch(uploadUrl, {
|
||||||
|
method: "POST",
|
||||||
|
headers: { "Content-Type": "application/octet-stream" },
|
||||||
|
credentials: "include",
|
||||||
|
body: file,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!uploadRes.ok) {
|
||||||
|
const text = await uploadRes.text();
|
||||||
|
console.debug("[filebrowser] upload stage failed", {
|
||||||
|
status: uploadRes.status,
|
||||||
|
body: text,
|
||||||
|
});
|
||||||
|
throw new Error(text || "upload staging failed");
|
||||||
|
}
|
||||||
|
|
||||||
|
transfer.receivedBytes = Math.round(file.size * 0.5);
|
||||||
|
transfer.sent = transfer.receivedBytes;
|
||||||
|
transfer.progress = 50;
|
||||||
|
updateTransferProgress(transfer.id, transfer.progress, transfer.sent, transfer.total);
|
||||||
|
|
||||||
|
const uploadData = await uploadRes.json();
|
||||||
|
if (!uploadData?.pullUrl) {
|
||||||
|
throw new Error("upload staging failed");
|
||||||
|
}
|
||||||
|
|
||||||
|
console.debug("[filebrowser] upload staged", {
|
||||||
|
pullUrl: uploadData.pullUrl,
|
||||||
|
size: uploadData.size,
|
||||||
|
});
|
||||||
|
|
||||||
|
const commandId = `upload-http-${Date.now()}-${Math.random()}`;
|
||||||
|
const waitResult = waitForCommandResult(commandId, 12 * 60 * 1000);
|
||||||
|
send({
|
||||||
|
type: "command",
|
||||||
|
commandType: "file_upload_http",
|
||||||
|
id: commandId,
|
||||||
|
payload: {
|
||||||
|
path,
|
||||||
|
url: uploadData.pullUrl,
|
||||||
|
total: file.size,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
await waitResult;
|
||||||
|
|
||||||
|
console.debug("[filebrowser] upload command completed", {
|
||||||
|
path,
|
||||||
|
size: file.size,
|
||||||
|
});
|
||||||
|
|
||||||
|
transfer.receivedBytes = file.size;
|
||||||
|
transfer.sent = file.size;
|
||||||
|
transfer.progress = 100;
|
||||||
|
transfer.receivedChunks = transfer.expectedChunks;
|
||||||
|
updateTransferProgress(transfer.id, transfer.progress, transfer.sent, transfer.total);
|
||||||
|
}
|
||||||
|
|
||||||
async function uploadFile(file) {
|
async function uploadFile(file) {
|
||||||
const path = currentPath ? `${currentPath}/${file.name}` : file.name;
|
const path = currentPath ? `${currentPath}/${file.name}` : file.name;
|
||||||
const transferId = `upload-${Date.now()}-${Math.random()}`;
|
const transferId = `upload-${Date.now()}-${Math.random()}`;
|
||||||
@@ -1618,84 +1728,10 @@ async function uploadFile(file) {
|
|||||||
activeTransfers.set(transferId, transfer);
|
activeTransfers.set(transferId, transfer);
|
||||||
addTransferToUI(transfer);
|
addTransferToUI(transfer);
|
||||||
|
|
||||||
const chunkSize = 900 * 1024;
|
|
||||||
const maxInFlightChunks = 6;
|
|
||||||
transfer.expectedChunks = Math.max(1, Math.ceil(file.size / chunkSize));
|
|
||||||
let nextOffset = 0;
|
|
||||||
const inFlight = new Set();
|
|
||||||
|
|
||||||
const processChunk = async (offset) => {
|
|
||||||
const chunk = file.slice(offset, offset + chunkSize);
|
|
||||||
const arrayBuffer = await chunk.arrayBuffer();
|
|
||||||
const uint8Array = new Uint8Array(arrayBuffer);
|
|
||||||
|
|
||||||
const payload = {
|
|
||||||
type: "file_upload",
|
|
||||||
path,
|
|
||||||
data: uint8Array,
|
|
||||||
offset,
|
|
||||||
transferId,
|
|
||||||
total: file.size,
|
|
||||||
};
|
|
||||||
const ack = await sendChunkWithRetry(transfer, payload, offset);
|
|
||||||
|
|
||||||
if (!transfer.ackedOffsets.has(offset)) {
|
|
||||||
transfer.ackedOffsets.add(offset);
|
|
||||||
transfer.receivedChunks += 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (Number.isFinite(ack?.received)) {
|
|
||||||
transfer.receivedBytes = Math.min(Number(ack.received), transfer.total);
|
|
||||||
transfer.sent = transfer.receivedBytes;
|
|
||||||
} else {
|
|
||||||
transfer.receivedBytes = Math.min(transfer.receivedBytes + chunk.size, transfer.total);
|
|
||||||
transfer.sent = transfer.receivedBytes;
|
|
||||||
}
|
|
||||||
|
|
||||||
transfer.progress = Math.round((transfer.sent / file.size) * 100);
|
|
||||||
updateTransferProgress(transferId, transfer.progress, transfer.sent, file.size);
|
|
||||||
};
|
|
||||||
|
|
||||||
const fillUploadWindow = () => {
|
|
||||||
while (
|
|
||||||
!transfer.cancelled &&
|
|
||||||
nextOffset < file.size &&
|
|
||||||
inFlight.size < maxInFlightChunks
|
|
||||||
) {
|
|
||||||
const offset = nextOffset;
|
|
||||||
nextOffset += chunkSize;
|
|
||||||
|
|
||||||
const task = processChunk(offset).finally(() => {
|
|
||||||
inFlight.delete(task);
|
|
||||||
});
|
|
||||||
inFlight.add(task);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
try {
|
try {
|
||||||
fillUploadWindow();
|
await uploadFileViaHttpPull(file, path, transfer);
|
||||||
|
finishUpload(transfer);
|
||||||
while (inFlight.size > 0) {
|
|
||||||
await Promise.race(inFlight);
|
|
||||||
|
|
||||||
if (transfer.cancelled) {
|
|
||||||
clearUploadPendingAcks(transfer, "Upload cancelled");
|
|
||||||
console.log("Upload cancelled:", path);
|
|
||||||
removeTransfer(transferId);
|
|
||||||
fileUploads.delete(path);
|
|
||||||
fileUploadsById.delete(transferId);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
fillUploadWindow();
|
|
||||||
}
|
|
||||||
|
|
||||||
transfer.completed = true;
|
|
||||||
if (transfer.receivedChunks >= transfer.expectedChunks) {
|
|
||||||
finishUpload(transfer);
|
|
||||||
}
|
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
clearUploadPendingAcks(transfer, "Upload aborted");
|
|
||||||
console.error("Upload error:", err);
|
console.error("Upload error:", err);
|
||||||
removeTransfer(transferId);
|
removeTransfer(transferId);
|
||||||
fileUploads.delete(path);
|
fileUploads.delete(path);
|
||||||
|
|||||||
@@ -20,7 +20,7 @@ import { handleAutoScriptsRoutes } from "./server/routes/auto-scripts-routes";
|
|||||||
import { handleBuildRoutes } from "./server/routes/build-routes";
|
import { handleBuildRoutes } from "./server/routes/build-routes";
|
||||||
import { handleAssetsRoutes } from "./server/routes/assets-routes";
|
import { handleAssetsRoutes } from "./server/routes/assets-routes";
|
||||||
import { handleDeployRoutes } from "./server/routes/deploy-routes";
|
import { handleDeployRoutes } from "./server/routes/deploy-routes";
|
||||||
import { handleFileDownloadRoutes } from "./server/routes/file-download-routes";
|
import { cleanupFileTransferTempFiles, handleFileDownloadRoutes } from "./server/routes/file-download-routes";
|
||||||
import { handleClientRoutes } from "./server/routes/client-routes";
|
import { handleClientRoutes } from "./server/routes/client-routes";
|
||||||
import { handleMiscRoutes } from "./server/routes/misc-routes";
|
import { handleMiscRoutes } from "./server/routes/misc-routes";
|
||||||
import { handleNotificationsConfigRoutes } from "./server/routes/notifications-config-routes";
|
import { handleNotificationsConfigRoutes } from "./server/routes/notifications-config-routes";
|
||||||
@@ -141,6 +141,21 @@ function envFlagEnabled(name: string): boolean {
|
|||||||
|
|
||||||
const TLS_OFFLOAD = envFlagEnabled("OVERLORD_TLS_OFFLOAD");
|
const TLS_OFFLOAD = envFlagEnabled("OVERLORD_TLS_OFFLOAD");
|
||||||
|
|
||||||
|
function parseMaxHttpBodyBytes(): number {
|
||||||
|
const raw = String(process.env.OVERLORD_MAX_HTTP_BODY_BYTES || "").trim();
|
||||||
|
if (raw !== "") {
|
||||||
|
const parsed = Number(raw);
|
||||||
|
if (Number.isFinite(parsed) && parsed > 0) {
|
||||||
|
return Math.floor(parsed);
|
||||||
|
}
|
||||||
|
logger.warn(`[HTTP] Invalid OVERLORD_MAX_HTTP_BODY_BYTES=${raw}; using default`);
|
||||||
|
}
|
||||||
|
// Default to 2 GiB to allow large ISO uploads in file browser staging.
|
||||||
|
return 2 * 1024 * 1024 * 1024;
|
||||||
|
}
|
||||||
|
|
||||||
|
const MAX_HTTP_BODY_BYTES = parseMaxHttpBodyBytes();
|
||||||
|
|
||||||
const pluginLoadedByClient = new Map<string, Set<string>>();
|
const pluginLoadedByClient = new Map<string, Set<string>>();
|
||||||
const pendingPluginEvents = new Map<string, Array<{ event: string; payload: any }>>();
|
const pendingPluginEvents = new Map<string, Array<{ event: string; payload: any }>>();
|
||||||
const pluginLoadingByClient = new Map<string, Set<string>>();
|
const pluginLoadingByClient = new Map<string, Set<string>>();
|
||||||
@@ -241,6 +256,8 @@ const pendingCommandReplies = new Map<string, PendingCommandReply>();
|
|||||||
|
|
||||||
async function startServer() {
|
async function startServer() {
|
||||||
await loadPluginState();
|
await loadPluginState();
|
||||||
|
await cleanupFileTransferTempFiles(DATA_DIR);
|
||||||
|
logger.info("[filebrowser] cleaned stale transfer temp files on startup");
|
||||||
let tls:
|
let tls:
|
||||||
| {
|
| {
|
||||||
tlsOptions: { cert?: string; key?: string; ca?: string };
|
tlsOptions: { cert?: string; key?: string; ca?: string };
|
||||||
@@ -392,6 +409,7 @@ async function startServer() {
|
|||||||
hostname: HOST,
|
hostname: HOST,
|
||||||
...(tls ? { tls: tls.tlsOptions } : {}),
|
...(tls ? { tls: tls.tlsOptions } : {}),
|
||||||
idleTimeout: 255,
|
idleTimeout: 255,
|
||||||
|
maxRequestBodySize: MAX_HTTP_BODY_BYTES,
|
||||||
fetch: createHttpFetchHandler({
|
fetch: createHttpFetchHandler({
|
||||||
metrics,
|
metrics,
|
||||||
CORS_HEADERS,
|
CORS_HEADERS,
|
||||||
@@ -440,6 +458,7 @@ async function startServer() {
|
|||||||
|
|
||||||
if (tls) {
|
if (tls) {
|
||||||
logServerStartup(server, tls.certPathUsed, tls.source);
|
logServerStartup(server, tls.certPathUsed, tls.source);
|
||||||
|
logger.info(`[HTTP] maxRequestBodySize=${MAX_HTTP_BODY_BYTES} bytes`);
|
||||||
} else {
|
} else {
|
||||||
const hostname = server.hostname || "0.0.0.0";
|
const hostname = server.hostname || "0.0.0.0";
|
||||||
const port = server.port ?? 0;
|
const port = server.port ?? 0;
|
||||||
@@ -451,6 +470,7 @@ async function startServer() {
|
|||||||
logger.info("");
|
logger.info("");
|
||||||
logger.info("External access should be HTTPS/WSS via your reverse proxy platform.");
|
logger.info("External access should be HTTPS/WSS via your reverse proxy platform.");
|
||||||
logger.info("Set this mode only when TLS is terminated by the platform (for example Render). ");
|
logger.info("Set this mode only when TLS is terminated by the platform (for example Render). ");
|
||||||
|
logger.info(`[HTTP] maxRequestBodySize=${MAX_HTTP_BODY_BYTES} bytes`);
|
||||||
logger.info("========================================");
|
logger.info("========================================");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,251 @@
|
|||||||
|
import { describe, expect, test } from "bun:test";
|
||||||
|
import { mkdtemp, readdir, rm } from "fs/promises";
|
||||||
|
import { join } from "path";
|
||||||
|
import { tmpdir } from "os";
|
||||||
|
import { generateToken } from "../../auth";
|
||||||
|
import * as clientManager from "../../clientManager";
|
||||||
|
import { createUser, deleteUser, getUserById } from "../../users";
|
||||||
|
import { handleFileDownloadRoutes } from "./file-download-routes";
|
||||||
|
|
||||||
|
const PASSWORD = "Aa1!RouteUploadTestPass123";
|
||||||
|
|
||||||
|
function makeRouteDeps(dataDir: string) {
|
||||||
|
return {
|
||||||
|
DATA_DIR: dataDir,
|
||||||
|
secureHeaders: (_contentType?: string) => ({}),
|
||||||
|
sanitizeOutputName: (name: string) => {
|
||||||
|
const cleaned = (name || "").replace(/[^A-Za-z0-9._-]/g, "");
|
||||||
|
if (!cleaned) throw new Error("invalid filename");
|
||||||
|
return cleaned;
|
||||||
|
},
|
||||||
|
pendingHttpDownloads: new Map<string, any>(),
|
||||||
|
downloadIntents: new Map<string, any>(),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const mockServer = {
|
||||||
|
requestIP: () => ({ address: "127.0.0.1" }),
|
||||||
|
};
|
||||||
|
|
||||||
|
async function createAdminToken() {
|
||||||
|
const username = `u_${Date.now().toString(36)}_${Math.floor(Math.random() * 1e6).toString(36)}`;
|
||||||
|
const created = await createUser(username, PASSWORD, "admin", "test");
|
||||||
|
expect(created.success).toBe(true);
|
||||||
|
expect(typeof created.userId).toBe("number");
|
||||||
|
|
||||||
|
const user = getUserById(created.userId!);
|
||||||
|
expect(user).not.toBeNull();
|
||||||
|
|
||||||
|
const token = await generateToken(user!);
|
||||||
|
return {
|
||||||
|
userId: created.userId!,
|
||||||
|
token,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async function createTempDataDir(): Promise<string> {
|
||||||
|
return mkdtemp(join(tmpdir(), "overlord-upload-route-test-"));
|
||||||
|
}
|
||||||
|
|
||||||
|
async function removeTempDataDir(dir: string): Promise<void> {
|
||||||
|
await rm(dir, { recursive: true, force: true }).catch(() => {});
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("file upload route flow", () => {
|
||||||
|
test("stages and serves one-time upload payload for matching client", async () => {
|
||||||
|
const auth = await createAdminToken();
|
||||||
|
const clientId = `client-${Date.now().toString(36)}`;
|
||||||
|
const dataDir = await createTempDataDir();
|
||||||
|
|
||||||
|
const deps = makeRouteDeps(dataDir);
|
||||||
|
const payload = new Uint8Array(3 * 1024 * 1024 + 123);
|
||||||
|
crypto.getRandomValues(payload.subarray(0, Math.min(payload.length, 65536)));
|
||||||
|
|
||||||
|
clientManager.addClient(clientId, {
|
||||||
|
id: clientId,
|
||||||
|
lastSeen: Date.now(),
|
||||||
|
role: "client",
|
||||||
|
ws: { send: () => {} },
|
||||||
|
});
|
||||||
|
|
||||||
|
const prevDisableAgentAuth = process.env.OVERLORD_DISABLE_AGENT_AUTH;
|
||||||
|
process.env.OVERLORD_DISABLE_AGENT_AUTH = "true";
|
||||||
|
|
||||||
|
try {
|
||||||
|
const requestUrl = new URL("https://localhost/api/file/upload/request");
|
||||||
|
const requestRes = await handleFileDownloadRoutes(
|
||||||
|
new Request(requestUrl, {
|
||||||
|
method: "POST",
|
||||||
|
headers: {
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
Authorization: `Bearer ${auth.token}`,
|
||||||
|
},
|
||||||
|
body: JSON.stringify({
|
||||||
|
clientId,
|
||||||
|
path: "C:\\Games\\big.iso",
|
||||||
|
fileName: "big.iso",
|
||||||
|
}),
|
||||||
|
}),
|
||||||
|
requestUrl,
|
||||||
|
mockServer,
|
||||||
|
deps,
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(requestRes).not.toBeNull();
|
||||||
|
expect(requestRes!.status).toBe(200);
|
||||||
|
const requestJson = await requestRes!.json() as any;
|
||||||
|
expect(requestJson.ok).toBe(true);
|
||||||
|
expect(typeof requestJson.uploadUrl).toBe("string");
|
||||||
|
|
||||||
|
const stageUrl = new URL(`https://localhost${requestJson.uploadUrl}`);
|
||||||
|
const stageRes = await handleFileDownloadRoutes(
|
||||||
|
new Request(stageUrl, {
|
||||||
|
method: "POST",
|
||||||
|
headers: {
|
||||||
|
"Content-Type": "application/octet-stream",
|
||||||
|
Authorization: `Bearer ${auth.token}`,
|
||||||
|
},
|
||||||
|
body: payload,
|
||||||
|
}),
|
||||||
|
stageUrl,
|
||||||
|
mockServer,
|
||||||
|
deps,
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(stageRes).not.toBeNull();
|
||||||
|
expect(stageRes!.status).toBe(200);
|
||||||
|
const stageJson = await stageRes!.json() as any;
|
||||||
|
expect(stageJson.ok).toBe(true);
|
||||||
|
expect(stageJson.size).toBe(payload.length);
|
||||||
|
expect(typeof stageJson.pullUrl).toBe("string");
|
||||||
|
|
||||||
|
const pullUrl = new URL(stageJson.pullUrl);
|
||||||
|
const pullRes = await handleFileDownloadRoutes(
|
||||||
|
new Request(pullUrl, {
|
||||||
|
method: "GET",
|
||||||
|
headers: {
|
||||||
|
"x-overlord-client-id": clientId,
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
pullUrl,
|
||||||
|
mockServer,
|
||||||
|
deps,
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(pullRes).not.toBeNull();
|
||||||
|
expect(pullRes!.status).toBe(200);
|
||||||
|
const pulled = new Uint8Array(await pullRes!.arrayBuffer());
|
||||||
|
expect(pulled.length).toBe(payload.length);
|
||||||
|
expect(pulled[0]).toBe(payload[0]);
|
||||||
|
expect(pulled[pulled.length - 1]).toBe(payload[payload.length - 1]);
|
||||||
|
|
||||||
|
const uploadDir = join(dataDir, "uploads");
|
||||||
|
const uploadEntries = await readdir(uploadDir).catch(() => [] as string[]);
|
||||||
|
expect(uploadEntries.filter((name) => name.endsWith(".bin")).length).toBe(0);
|
||||||
|
|
||||||
|
const pullAgainRes = await handleFileDownloadRoutes(
|
||||||
|
new Request(pullUrl, {
|
||||||
|
method: "GET",
|
||||||
|
headers: {
|
||||||
|
"x-overlord-client-id": clientId,
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
pullUrl,
|
||||||
|
mockServer,
|
||||||
|
deps,
|
||||||
|
);
|
||||||
|
expect(pullAgainRes).not.toBeNull();
|
||||||
|
expect(pullAgainRes!.status).toBe(404);
|
||||||
|
} finally {
|
||||||
|
if (prevDisableAgentAuth === undefined) {
|
||||||
|
delete process.env.OVERLORD_DISABLE_AGENT_AUTH;
|
||||||
|
} else {
|
||||||
|
process.env.OVERLORD_DISABLE_AGENT_AUTH = prevDisableAgentAuth;
|
||||||
|
}
|
||||||
|
clientManager.deleteClient(clientId);
|
||||||
|
const deleted = deleteUser(auth.userId);
|
||||||
|
expect(deleted.success).toBe(true);
|
||||||
|
await removeTempDataDir(dataDir);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test("rejects pull when client id does not match", async () => {
|
||||||
|
const auth = await createAdminToken();
|
||||||
|
const clientId = `client-${Date.now().toString(36)}`;
|
||||||
|
const dataDir = await createTempDataDir();
|
||||||
|
const deps = makeRouteDeps(dataDir);
|
||||||
|
const payload = new TextEncoder().encode("hello");
|
||||||
|
|
||||||
|
clientManager.addClient(clientId, {
|
||||||
|
id: clientId,
|
||||||
|
lastSeen: Date.now(),
|
||||||
|
role: "client",
|
||||||
|
ws: { send: () => {} },
|
||||||
|
});
|
||||||
|
|
||||||
|
const prevDisableAgentAuth = process.env.OVERLORD_DISABLE_AGENT_AUTH;
|
||||||
|
process.env.OVERLORD_DISABLE_AGENT_AUTH = "true";
|
||||||
|
|
||||||
|
try {
|
||||||
|
const requestUrl = new URL("https://localhost/api/file/upload/request");
|
||||||
|
const requestRes = await handleFileDownloadRoutes(
|
||||||
|
new Request(requestUrl, {
|
||||||
|
method: "POST",
|
||||||
|
headers: {
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
Authorization: `Bearer ${auth.token}`,
|
||||||
|
},
|
||||||
|
body: JSON.stringify({
|
||||||
|
clientId,
|
||||||
|
path: "C:\\Games\\small.bin",
|
||||||
|
fileName: "small.bin",
|
||||||
|
}),
|
||||||
|
}),
|
||||||
|
requestUrl,
|
||||||
|
mockServer,
|
||||||
|
deps,
|
||||||
|
);
|
||||||
|
const requestJson = await requestRes!.json() as any;
|
||||||
|
|
||||||
|
const stageUrl = new URL(`https://localhost${requestJson.uploadUrl}`);
|
||||||
|
const stageRes = await handleFileDownloadRoutes(
|
||||||
|
new Request(stageUrl, {
|
||||||
|
method: "PUT",
|
||||||
|
headers: {
|
||||||
|
Authorization: `Bearer ${auth.token}`,
|
||||||
|
},
|
||||||
|
body: payload,
|
||||||
|
}),
|
||||||
|
stageUrl,
|
||||||
|
mockServer,
|
||||||
|
deps,
|
||||||
|
);
|
||||||
|
const stageJson = await stageRes!.json() as any;
|
||||||
|
|
||||||
|
const pullUrl = new URL(stageJson.pullUrl);
|
||||||
|
const badPullRes = await handleFileDownloadRoutes(
|
||||||
|
new Request(pullUrl, {
|
||||||
|
method: "GET",
|
||||||
|
headers: {
|
||||||
|
"x-overlord-client-id": "different-client",
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
pullUrl,
|
||||||
|
mockServer,
|
||||||
|
deps,
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(badPullRes).not.toBeNull();
|
||||||
|
expect(badPullRes!.status).toBe(403);
|
||||||
|
} finally {
|
||||||
|
if (prevDisableAgentAuth === undefined) {
|
||||||
|
delete process.env.OVERLORD_DISABLE_AGENT_AUTH;
|
||||||
|
} else {
|
||||||
|
process.env.OVERLORD_DISABLE_AGENT_AUTH = prevDisableAgentAuth;
|
||||||
|
}
|
||||||
|
clientManager.deleteClient(clientId);
|
||||||
|
deleteUser(auth.userId);
|
||||||
|
await removeTempDataDir(dataDir);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -6,6 +6,8 @@ import { AuditAction, logAudit } from "../../auditLog";
|
|||||||
import * as clientManager from "../../clientManager";
|
import * as clientManager from "../../clientManager";
|
||||||
import { logger } from "../../logger";
|
import { logger } from "../../logger";
|
||||||
import { encodeMessage } from "../../protocol";
|
import { encodeMessage } from "../../protocol";
|
||||||
|
import { getConfig } from "../../config";
|
||||||
|
import { isAuthorizedAgentRequest } from "../agent-auth";
|
||||||
|
|
||||||
type RequestIpProvider = {
|
type RequestIpProvider = {
|
||||||
requestIP: (req: Request) => { address?: string } | null | undefined;
|
requestIP: (req: Request) => { address?: string } | null | undefined;
|
||||||
@@ -40,6 +42,27 @@ type DownloadIntent = {
|
|||||||
timeout: ReturnType<typeof setTimeout>;
|
timeout: ReturnType<typeof setTimeout>;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
type UploadIntent = {
|
||||||
|
id: string;
|
||||||
|
userId: string;
|
||||||
|
clientId: string;
|
||||||
|
path: string;
|
||||||
|
fileName: string;
|
||||||
|
expiresAt: number;
|
||||||
|
timeout: ReturnType<typeof setTimeout>;
|
||||||
|
};
|
||||||
|
|
||||||
|
type UploadPull = {
|
||||||
|
id: string;
|
||||||
|
clientId: string;
|
||||||
|
path: string;
|
||||||
|
fileName: string;
|
||||||
|
tmpPath: string;
|
||||||
|
size: number;
|
||||||
|
expiresAt: number;
|
||||||
|
timeout: ReturnType<typeof setTimeout>;
|
||||||
|
};
|
||||||
|
|
||||||
type FileDownloadRouteDeps = {
|
type FileDownloadRouteDeps = {
|
||||||
DATA_DIR: string;
|
DATA_DIR: string;
|
||||||
secureHeaders: (contentType?: string) => Record<string, string>;
|
secureHeaders: (contentType?: string) => Record<string, string>;
|
||||||
@@ -48,17 +71,222 @@ type FileDownloadRouteDeps = {
|
|||||||
downloadIntents: Map<string, DownloadIntent>;
|
downloadIntents: Map<string, DownloadIntent>;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
export async function cleanupFileTransferTempFiles(dataDir: string): Promise<void> {
|
||||||
|
const uploadsDir = path.join(dataDir, "uploads");
|
||||||
|
const downloadsDir = path.join(dataDir, "downloads");
|
||||||
|
await fs.rm(uploadsDir, { recursive: true, force: true }).catch(() => {});
|
||||||
|
await fs.rm(downloadsDir, { recursive: true, force: true }).catch(() => {});
|
||||||
|
}
|
||||||
|
|
||||||
|
function streamFileAndDelete(tmpPath: string): ReadableStream<Uint8Array> {
|
||||||
|
const reader = Bun.file(tmpPath).stream().getReader();
|
||||||
|
let cleaned = false;
|
||||||
|
|
||||||
|
const cleanup = async () => {
|
||||||
|
if (cleaned) return;
|
||||||
|
cleaned = true;
|
||||||
|
await fs.unlink(tmpPath).catch(() => {});
|
||||||
|
};
|
||||||
|
|
||||||
|
return new ReadableStream<Uint8Array>({
|
||||||
|
async pull(controller) {
|
||||||
|
try {
|
||||||
|
const { done, value } = await reader.read();
|
||||||
|
if (done) {
|
||||||
|
controller.close();
|
||||||
|
await cleanup();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (value) {
|
||||||
|
controller.enqueue(value);
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
controller.error(err);
|
||||||
|
await cleanup();
|
||||||
|
}
|
||||||
|
},
|
||||||
|
async cancel() {
|
||||||
|
try {
|
||||||
|
await reader.cancel();
|
||||||
|
} catch {}
|
||||||
|
await cleanup();
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
const UUID_TOKEN_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
|
||||||
|
const uploadIntents = new Map<string, UploadIntent>();
|
||||||
|
const uploadPulls = new Map<string, UploadPull>();
|
||||||
|
|
||||||
|
function isSafeRemotePath(value: string): boolean {
|
||||||
|
if (!value || value.length > 4096) return false;
|
||||||
|
return !/[\x00-\x1F\x7F]/.test(value);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function serveDownloadById(
|
||||||
|
req: Request,
|
||||||
|
downloadId: string,
|
||||||
|
server: RequestIpProvider,
|
||||||
|
deps: FileDownloadRouteDeps,
|
||||||
|
): Promise<Response> {
|
||||||
|
const user = await authenticateRequest(req);
|
||||||
|
if (!user) {
|
||||||
|
return new Response("Unauthorized", { status: 401 });
|
||||||
|
}
|
||||||
|
if (user.role !== "admin" && user.role !== "operator") {
|
||||||
|
return new Response("Forbidden: Admin or operator access required", { status: 403 });
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.debug("[filebrowser] http download request", {
|
||||||
|
downloadId,
|
||||||
|
userId: user.userId,
|
||||||
|
ip: server.requestIP(req)?.address || "unknown",
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!UUID_TOKEN_RE.test(downloadId)) {
|
||||||
|
return new Response("Bad request", { status: 400 });
|
||||||
|
}
|
||||||
|
|
||||||
|
const intent = deps.downloadIntents.get(downloadId);
|
||||||
|
if (!intent || intent.userId !== user.userId || intent.expiresAt < Date.now()) {
|
||||||
|
logger.debug("[filebrowser] http download intent missing", {
|
||||||
|
downloadId,
|
||||||
|
userId: user.userId,
|
||||||
|
intentUserId: intent?.userId,
|
||||||
|
expiresAt: intent?.expiresAt,
|
||||||
|
});
|
||||||
|
return new Response("Not found", { status: 404 });
|
||||||
|
}
|
||||||
|
|
||||||
|
deps.downloadIntents.delete(downloadId);
|
||||||
|
clearTimeout(intent.timeout);
|
||||||
|
|
||||||
|
const clientId = intent.clientId;
|
||||||
|
const downloadPath = intent.path;
|
||||||
|
|
||||||
|
const target = clientManager.getClient(clientId);
|
||||||
|
if (!target) {
|
||||||
|
logger.debug("[filebrowser] http download target offline", {
|
||||||
|
downloadId,
|
||||||
|
clientId,
|
||||||
|
});
|
||||||
|
return new Response("Client offline", { status: 404 });
|
||||||
|
}
|
||||||
|
|
||||||
|
const commandId = uuidv4();
|
||||||
|
const downloadDir = path.join(deps.DATA_DIR, "downloads");
|
||||||
|
await fs.mkdir(downloadDir, { recursive: true });
|
||||||
|
const tmpPath = path.join(downloadDir, `${commandId}.bin`);
|
||||||
|
|
||||||
|
let fileName = path.basename(downloadPath) || "download.bin";
|
||||||
|
try {
|
||||||
|
fileName = deps.sanitizeOutputName(fileName);
|
||||||
|
} catch {
|
||||||
|
fileName = "download.bin";
|
||||||
|
}
|
||||||
|
|
||||||
|
const fileHandle = await fs.open(tmpPath, "w+");
|
||||||
|
|
||||||
|
logger.debug("[filebrowser] http download start", {
|
||||||
|
commandId,
|
||||||
|
clientId,
|
||||||
|
path: downloadPath,
|
||||||
|
tmpPath,
|
||||||
|
});
|
||||||
|
|
||||||
|
const downloadPromise = new Promise<PendingHttpDownload>((resolve, reject) => {
|
||||||
|
const timeout = setTimeout(() => {
|
||||||
|
deps.pendingHttpDownloads.delete(commandId);
|
||||||
|
void fileHandle.close();
|
||||||
|
void fs.unlink(tmpPath).catch(() => {});
|
||||||
|
reject(new Error("Download timed out"));
|
||||||
|
}, 5 * 60_000);
|
||||||
|
|
||||||
|
deps.pendingHttpDownloads.set(commandId, {
|
||||||
|
commandId,
|
||||||
|
clientId,
|
||||||
|
path: downloadPath,
|
||||||
|
fileName,
|
||||||
|
total: 0,
|
||||||
|
receivedBytes: 0,
|
||||||
|
receivedOffsets: new Set(),
|
||||||
|
receivedChunks: new Set(),
|
||||||
|
chunkSize: 0,
|
||||||
|
expectedChunks: 0,
|
||||||
|
loggedTotal: false,
|
||||||
|
loggedFirstChunk: false,
|
||||||
|
tmpPath,
|
||||||
|
fileHandle,
|
||||||
|
resolve,
|
||||||
|
reject,
|
||||||
|
timeout,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
target.ws.send(
|
||||||
|
encodeMessage({
|
||||||
|
type: "command",
|
||||||
|
commandType: "file_download",
|
||||||
|
id: commandId,
|
||||||
|
payload: { path: downloadPath },
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
logger.debug("[filebrowser] http download command sent", {
|
||||||
|
commandId,
|
||||||
|
clientId,
|
||||||
|
path: downloadPath,
|
||||||
|
});
|
||||||
|
|
||||||
|
logAudit({
|
||||||
|
timestamp: Date.now(),
|
||||||
|
username: user.username,
|
||||||
|
ip: server.requestIP(req)?.address || "unknown",
|
||||||
|
action: AuditAction.FILE_DOWNLOAD,
|
||||||
|
targetClientId: clientId,
|
||||||
|
details: JSON.stringify({ path: downloadPath, via: "http" }),
|
||||||
|
success: true,
|
||||||
|
});
|
||||||
|
|
||||||
|
let completed: PendingHttpDownload;
|
||||||
|
try {
|
||||||
|
completed = await downloadPromise;
|
||||||
|
} catch (err) {
|
||||||
|
logger.debug("[filebrowser] http download failed", {
|
||||||
|
commandId,
|
||||||
|
clientId,
|
||||||
|
path: downloadPath,
|
||||||
|
error: (err as Error)?.message || String(err),
|
||||||
|
});
|
||||||
|
return new Response((err as Error).message || "Download failed", { status: 500 });
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.debug("[filebrowser] http download complete", {
|
||||||
|
commandId,
|
||||||
|
clientId,
|
||||||
|
path: downloadPath,
|
||||||
|
total: completed.total,
|
||||||
|
receivedBytes: completed.receivedBytes,
|
||||||
|
expectedChunks: completed.expectedChunks,
|
||||||
|
receivedChunks: completed.receivedChunks.size,
|
||||||
|
});
|
||||||
|
|
||||||
|
const headers = {
|
||||||
|
...deps.secureHeaders("application/octet-stream"),
|
||||||
|
"Content-Disposition": `attachment; filename="${completed.fileName}"`,
|
||||||
|
"Cache-Control": "no-store, private",
|
||||||
|
};
|
||||||
|
|
||||||
|
return new Response(streamFileAndDelete(completed.tmpPath), { headers });
|
||||||
|
}
|
||||||
|
|
||||||
export async function handleFileDownloadRoutes(
|
export async function handleFileDownloadRoutes(
|
||||||
req: Request,
|
req: Request,
|
||||||
url: URL,
|
url: URL,
|
||||||
server: RequestIpProvider,
|
server: RequestIpProvider,
|
||||||
deps: FileDownloadRouteDeps,
|
deps: FileDownloadRouteDeps,
|
||||||
): Promise<Response | null> {
|
): Promise<Response | null> {
|
||||||
if (!url.pathname.startsWith("/api/file/download")) {
|
if (req.method === "POST" && url.pathname === "/api/file/upload/request") {
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (req.method === "POST" && url.pathname === "/api/file/download") {
|
|
||||||
const user = await authenticateRequest(req);
|
const user = await authenticateRequest(req);
|
||||||
if (!user) {
|
if (!user) {
|
||||||
return new Response("Unauthorized", { status: 401 });
|
return new Response("Unauthorized", { status: 401 });
|
||||||
@@ -74,151 +302,225 @@ export async function handleFileDownloadRoutes(
|
|||||||
return new Response("Bad request", { status: 400 });
|
return new Response("Bad request", { status: 400 });
|
||||||
}
|
}
|
||||||
|
|
||||||
const downloadId = typeof body?.downloadId === "string" ? body.downloadId : "";
|
const clientId = typeof body?.clientId === "string" ? body.clientId.trim() : "";
|
||||||
logger.debug("[filebrowser] http download request", {
|
const uploadPath = typeof body?.path === "string" ? body.path.trim() : "";
|
||||||
downloadId,
|
const fileName = typeof body?.fileName === "string" ? body.fileName.trim() : "upload.bin";
|
||||||
userId: user.userId,
|
if (!clientId || !uploadPath || !isSafeRemotePath(uploadPath)) {
|
||||||
ip: server.requestIP(req)?.address || "unknown",
|
|
||||||
});
|
|
||||||
if (!downloadId) {
|
|
||||||
return new Response("Bad request", { status: 400 });
|
return new Response("Bad request", { status: 400 });
|
||||||
}
|
}
|
||||||
|
|
||||||
const intent = deps.downloadIntents.get(downloadId);
|
logger.debug("[filebrowser] http upload request", {
|
||||||
if (!intent || intent.userId !== user.userId || intent.expiresAt < Date.now()) {
|
userId: user.userId,
|
||||||
logger.debug("[filebrowser] http download intent missing", {
|
clientId,
|
||||||
downloadId,
|
path: uploadPath,
|
||||||
userId: user.userId,
|
fileName,
|
||||||
intentUserId: intent?.userId,
|
ip: server.requestIP(req)?.address || "unknown",
|
||||||
expiresAt: intent?.expiresAt,
|
});
|
||||||
});
|
|
||||||
return new Response("Not found", { status: 404 });
|
|
||||||
}
|
|
||||||
|
|
||||||
deps.downloadIntents.delete(downloadId);
|
|
||||||
clearTimeout(intent.timeout);
|
|
||||||
|
|
||||||
const clientId = intent.clientId;
|
|
||||||
const downloadPath = intent.path;
|
|
||||||
|
|
||||||
const target = clientManager.getClient(clientId);
|
const target = clientManager.getClient(clientId);
|
||||||
if (!target) {
|
if (!target) {
|
||||||
logger.debug("[filebrowser] http download target offline", {
|
|
||||||
downloadId,
|
|
||||||
clientId,
|
|
||||||
});
|
|
||||||
return new Response("Client offline", { status: 404 });
|
return new Response("Client offline", { status: 404 });
|
||||||
}
|
}
|
||||||
|
|
||||||
const commandId = uuidv4();
|
const uploadId = uuidv4();
|
||||||
const downloadDir = path.join(deps.DATA_DIR, "downloads");
|
const expiresAt = Date.now() + 2 * 60_000;
|
||||||
await fs.mkdir(downloadDir, { recursive: true });
|
const timeout = setTimeout(() => {
|
||||||
const tmpPath = path.join(downloadDir, `${commandId}.bin`);
|
uploadIntents.delete(uploadId);
|
||||||
|
}, 2 * 60_000);
|
||||||
|
|
||||||
let fileName = path.basename(downloadPath) || "download.bin";
|
uploadIntents.set(uploadId, {
|
||||||
try {
|
id: uploadId,
|
||||||
fileName = deps.sanitizeOutputName(fileName);
|
userId: user.userId,
|
||||||
} catch {
|
clientId,
|
||||||
fileName = "download.bin";
|
path: uploadPath,
|
||||||
|
fileName,
|
||||||
|
expiresAt,
|
||||||
|
timeout,
|
||||||
|
});
|
||||||
|
|
||||||
|
return Response.json({
|
||||||
|
ok: true,
|
||||||
|
uploadId,
|
||||||
|
uploadUrl: `/api/file/upload/${encodeURIComponent(uploadId)}`,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
if ((req.method === "PUT" || req.method === "POST") && url.pathname.startsWith("/api/file/upload/")) {
|
||||||
|
const user = await authenticateRequest(req);
|
||||||
|
if (!user) {
|
||||||
|
return new Response("Unauthorized", { status: 401 });
|
||||||
|
}
|
||||||
|
if (user.role !== "admin" && user.role !== "operator") {
|
||||||
|
return new Response("Forbidden: Admin or operator access required", { status: 403 });
|
||||||
}
|
}
|
||||||
|
|
||||||
const fileHandle = await fs.open(tmpPath, "w+");
|
let uploadId = "";
|
||||||
|
|
||||||
logger.debug("[filebrowser] http download start", {
|
|
||||||
commandId,
|
|
||||||
clientId,
|
|
||||||
path: downloadPath,
|
|
||||||
tmpPath,
|
|
||||||
});
|
|
||||||
|
|
||||||
const downloadPromise = new Promise<PendingHttpDownload>((resolve, reject) => {
|
|
||||||
const timeout = setTimeout(() => {
|
|
||||||
deps.pendingHttpDownloads.delete(commandId);
|
|
||||||
void fileHandle.close();
|
|
||||||
void fs.unlink(tmpPath).catch(() => {});
|
|
||||||
reject(new Error("Download timed out"));
|
|
||||||
}, 5 * 60_000);
|
|
||||||
|
|
||||||
deps.pendingHttpDownloads.set(commandId, {
|
|
||||||
commandId,
|
|
||||||
clientId,
|
|
||||||
path: downloadPath,
|
|
||||||
fileName,
|
|
||||||
total: 0,
|
|
||||||
receivedBytes: 0,
|
|
||||||
receivedOffsets: new Set(),
|
|
||||||
receivedChunks: new Set(),
|
|
||||||
chunkSize: 0,
|
|
||||||
expectedChunks: 0,
|
|
||||||
loggedTotal: false,
|
|
||||||
loggedFirstChunk: false,
|
|
||||||
tmpPath,
|
|
||||||
fileHandle,
|
|
||||||
resolve,
|
|
||||||
reject,
|
|
||||||
timeout,
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
target.ws.send(
|
|
||||||
encodeMessage({
|
|
||||||
type: "command",
|
|
||||||
commandType: "file_download",
|
|
||||||
id: commandId,
|
|
||||||
payload: { path: downloadPath },
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
|
|
||||||
logger.debug("[filebrowser] http download command sent", {
|
|
||||||
commandId,
|
|
||||||
clientId,
|
|
||||||
path: downloadPath,
|
|
||||||
});
|
|
||||||
|
|
||||||
logAudit({
|
|
||||||
timestamp: Date.now(),
|
|
||||||
username: user.username,
|
|
||||||
ip: server.requestIP(req)?.address || "unknown",
|
|
||||||
action: AuditAction.FILE_DOWNLOAD,
|
|
||||||
targetClientId: clientId,
|
|
||||||
details: JSON.stringify({ path: downloadPath, via: "http" }),
|
|
||||||
success: true,
|
|
||||||
});
|
|
||||||
|
|
||||||
let completed: PendingHttpDownload;
|
|
||||||
try {
|
try {
|
||||||
completed = await downloadPromise;
|
uploadId = decodeURIComponent(url.pathname.slice("/api/file/upload/".length));
|
||||||
|
} catch {
|
||||||
|
return new Response("Bad request", { status: 400 });
|
||||||
|
}
|
||||||
|
if (!UUID_TOKEN_RE.test(uploadId)) {
|
||||||
|
return new Response("Bad request", { status: 400 });
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.debug("[filebrowser] http upload stage", {
|
||||||
|
method: req.method,
|
||||||
|
uploadId,
|
||||||
|
userId: user.userId,
|
||||||
|
ip: server.requestIP(req)?.address || "unknown",
|
||||||
|
});
|
||||||
|
|
||||||
|
const intent = uploadIntents.get(uploadId);
|
||||||
|
if (!intent || intent.userId !== user.userId || intent.expiresAt < Date.now()) {
|
||||||
|
return new Response("Not found", { status: 404 });
|
||||||
|
}
|
||||||
|
|
||||||
|
const uploadDir = path.join(deps.DATA_DIR, "uploads");
|
||||||
|
await fs.mkdir(uploadDir, { recursive: true });
|
||||||
|
const tmpPath = path.join(uploadDir, `${uploadId}.bin`);
|
||||||
|
|
||||||
|
let stagedSize = 0;
|
||||||
|
const fileHandle = await fs.open(tmpPath, "w");
|
||||||
|
try {
|
||||||
|
if (req.body) {
|
||||||
|
const reader = req.body.getReader();
|
||||||
|
let position = 0;
|
||||||
|
while (true) {
|
||||||
|
const { done, value } = await reader.read();
|
||||||
|
if (done) break;
|
||||||
|
if (!value || value.byteLength === 0) continue;
|
||||||
|
await fileHandle.write(value, 0, value.byteLength, position);
|
||||||
|
position += value.byteLength;
|
||||||
|
stagedSize += value.byteLength;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
const bytes = new Uint8Array(await req.arrayBuffer());
|
||||||
|
if (bytes.byteLength > 0) {
|
||||||
|
await fileHandle.write(bytes, 0, bytes.byteLength, 0);
|
||||||
|
stagedSize = bytes.byteLength;
|
||||||
|
}
|
||||||
|
}
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
logger.debug("[filebrowser] http download failed", {
|
logger.debug("[filebrowser] http upload stage error", {
|
||||||
commandId,
|
uploadId,
|
||||||
clientId,
|
|
||||||
path: downloadPath,
|
|
||||||
error: (err as Error)?.message || String(err),
|
error: (err as Error)?.message || String(err),
|
||||||
});
|
});
|
||||||
return new Response((err as Error).message || "Download failed", { status: 500 });
|
await fileHandle.close();
|
||||||
|
await fs.unlink(tmpPath).catch(() => {});
|
||||||
|
return new Response("Upload staging failed", { status: 500 });
|
||||||
|
}
|
||||||
|
await fileHandle.close();
|
||||||
|
|
||||||
|
logger.debug("[filebrowser] http upload staged bytes", {
|
||||||
|
uploadId,
|
||||||
|
bytes: stagedSize,
|
||||||
|
clientId: intent.clientId,
|
||||||
|
path: intent.path,
|
||||||
|
});
|
||||||
|
|
||||||
|
uploadIntents.delete(uploadId);
|
||||||
|
clearTimeout(intent.timeout);
|
||||||
|
|
||||||
|
const pullId = uuidv4();
|
||||||
|
const pullExpiresAt = Date.now() + 10 * 60_000;
|
||||||
|
const pullTimeout = setTimeout(() => {
|
||||||
|
const pull = uploadPulls.get(pullId);
|
||||||
|
uploadPulls.delete(pullId);
|
||||||
|
if (pull) {
|
||||||
|
void fs.unlink(pull.tmpPath).catch(() => {});
|
||||||
|
}
|
||||||
|
}, 10 * 60_000);
|
||||||
|
|
||||||
|
uploadPulls.set(pullId, {
|
||||||
|
id: pullId,
|
||||||
|
clientId: intent.clientId,
|
||||||
|
path: intent.path,
|
||||||
|
fileName: intent.fileName,
|
||||||
|
tmpPath,
|
||||||
|
size: stagedSize,
|
||||||
|
expiresAt: pullExpiresAt,
|
||||||
|
timeout: pullTimeout,
|
||||||
|
});
|
||||||
|
|
||||||
|
return Response.json({
|
||||||
|
ok: true,
|
||||||
|
size: stagedSize,
|
||||||
|
path: intent.path,
|
||||||
|
pullUrl: `${url.origin}/api/file/upload/pull/${encodeURIComponent(pullId)}`,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
if (req.method === "GET" && url.pathname.startsWith("/api/file/upload/pull/")) {
|
||||||
|
const agentToken = getConfig().auth.agentToken;
|
||||||
|
if (!isAuthorizedAgentRequest(req, url, agentToken)) {
|
||||||
|
return new Response("Unauthorized", { status: 401 });
|
||||||
}
|
}
|
||||||
|
|
||||||
logger.debug("[filebrowser] http download complete", {
|
let pullId = "";
|
||||||
commandId,
|
try {
|
||||||
clientId,
|
pullId = decodeURIComponent(url.pathname.slice("/api/file/upload/pull/".length));
|
||||||
path: downloadPath,
|
} catch {
|
||||||
total: completed.total,
|
return new Response("Bad request", { status: 400 });
|
||||||
receivedBytes: completed.receivedBytes,
|
}
|
||||||
expectedChunks: completed.expectedChunks,
|
if (!UUID_TOKEN_RE.test(pullId)) {
|
||||||
receivedChunks: completed.receivedChunks.size,
|
return new Response("Bad request", { status: 400 });
|
||||||
|
}
|
||||||
|
|
||||||
|
const pull = uploadPulls.get(pullId);
|
||||||
|
if (!pull || pull.expiresAt < Date.now()) {
|
||||||
|
return new Response("Not found", { status: 404 });
|
||||||
|
}
|
||||||
|
|
||||||
|
const requesterClientId = req.headers.get("x-overlord-client-id") || "";
|
||||||
|
if (!requesterClientId || requesterClientId !== pull.clientId) {
|
||||||
|
return new Response("Forbidden", { status: 403 });
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.debug("[filebrowser] http upload pull", {
|
||||||
|
pullId,
|
||||||
|
clientId: pull.clientId,
|
||||||
|
path: pull.path,
|
||||||
|
bytes: pull.size,
|
||||||
|
ip: server.requestIP(req)?.address || "unknown",
|
||||||
});
|
});
|
||||||
|
|
||||||
|
uploadPulls.delete(pullId);
|
||||||
|
clearTimeout(pull.timeout);
|
||||||
|
|
||||||
const headers = {
|
const headers = {
|
||||||
...deps.secureHeaders("application/octet-stream"),
|
...deps.secureHeaders("application/octet-stream"),
|
||||||
"Content-Disposition": `attachment; filename="${completed.fileName}"`,
|
"Content-Disposition": `attachment; filename="${deps.sanitizeOutputName(path.basename(pull.fileName) || "upload.bin")}"`,
|
||||||
"Cache-Control": "no-store",
|
"Cache-Control": "no-store, private",
|
||||||
|
"Content-Length": String(pull.size),
|
||||||
};
|
};
|
||||||
|
|
||||||
setTimeout(() => {
|
return new Response(streamFileAndDelete(pull.tmpPath), { headers });
|
||||||
void fs.unlink(completed.tmpPath).catch(() => {});
|
}
|
||||||
}, 5 * 60_000);
|
|
||||||
|
|
||||||
return new Response(Bun.file(completed.tmpPath), { headers });
|
if (!url.pathname.startsWith("/api/file/download")) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (req.method === "GET" && url.pathname.startsWith("/api/file/download/")) {
|
||||||
|
let downloadId = "";
|
||||||
|
try {
|
||||||
|
downloadId = decodeURIComponent(url.pathname.slice("/api/file/download/".length));
|
||||||
|
} catch {
|
||||||
|
return new Response("Bad request", { status: 400 });
|
||||||
|
}
|
||||||
|
return serveDownloadById(req, downloadId, server, deps);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (req.method === "POST" && url.pathname === "/api/file/download") {
|
||||||
|
let body: any = {};
|
||||||
|
try {
|
||||||
|
body = await req.json();
|
||||||
|
} catch {
|
||||||
|
return new Response("Bad request", { status: 400 });
|
||||||
|
}
|
||||||
|
const downloadId = typeof body?.downloadId === "string" ? body.downloadId : "";
|
||||||
|
return serveDownloadById(req, downloadId, server, deps);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (req.method === "POST" && url.pathname === "/api/file/download/request") {
|
if (req.method === "POST" && url.pathname === "/api/file/download/request") {
|
||||||
@@ -237,9 +539,9 @@ export async function handleFileDownloadRoutes(
|
|||||||
return new Response("Bad request", { status: 400 });
|
return new Response("Bad request", { status: 400 });
|
||||||
}
|
}
|
||||||
|
|
||||||
const clientId = typeof body?.clientId === "string" ? body.clientId : "";
|
const clientId = typeof body?.clientId === "string" ? body.clientId.trim() : "";
|
||||||
const downloadPath = typeof body?.path === "string" ? body.path : "";
|
const downloadPath = typeof body?.path === "string" ? body.path.trim() : "";
|
||||||
if (!clientId || !downloadPath) {
|
if (!clientId || !downloadPath || !isSafeRemotePath(downloadPath)) {
|
||||||
return new Response("Bad request", { status: 400 });
|
return new Response("Bad request", { status: 400 });
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -263,7 +565,11 @@ export async function handleFileDownloadRoutes(
|
|||||||
timeout,
|
timeout,
|
||||||
});
|
});
|
||||||
|
|
||||||
return Response.json({ ok: true, downloadId });
|
return Response.json({
|
||||||
|
ok: true,
|
||||||
|
downloadId,
|
||||||
|
downloadUrl: `/api/file/download/${encodeURIComponent(downloadId)}`,
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
return null;
|
return null;
|
||||||
|
|||||||
@@ -116,6 +116,19 @@ export function handleFileBrowserViewerMessage(ws: ServerWebSocket<SocketData>,
|
|||||||
target.ws.send(encodeMessage({ type: "command", commandType: "file_execute", id: payload.id || commandId, payload: actualPayload } as any));
|
target.ws.send(encodeMessage({ type: "command", commandType: "file_execute", id: payload.id || commandId, payload: actualPayload } as any));
|
||||||
metrics.recordCommand("file_execute");
|
metrics.recordCommand("file_execute");
|
||||||
break;
|
break;
|
||||||
|
case "file_upload_http":
|
||||||
|
target.ws.send(encodeMessage({ type: "command", commandType: "file_upload_http", id: payload.id || commandId, payload: actualPayload } as any));
|
||||||
|
metrics.recordCommand("file_upload");
|
||||||
|
logAudit({
|
||||||
|
timestamp: Date.now(),
|
||||||
|
username: (ws.data as any).username || "unknown",
|
||||||
|
ip: ws.data.ip || "unknown",
|
||||||
|
action: AuditAction.FILE_UPLOAD,
|
||||||
|
targetClientId: clientId,
|
||||||
|
details: JSON.stringify({ path: actualPayload.path || "", mode: "http_pull" }),
|
||||||
|
success: true,
|
||||||
|
});
|
||||||
|
break;
|
||||||
default:
|
default:
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
@@ -152,27 +165,13 @@ export function handleFileBrowserViewerMessage(ws: ServerWebSocket<SocketData>,
|
|||||||
case "file_upload": {
|
case "file_upload": {
|
||||||
const upload = normalizeFileUploadPayload(payload);
|
const upload = normalizeFileUploadPayload(payload);
|
||||||
if (!upload) return;
|
if (!upload) return;
|
||||||
target.ws.send(encodeMessage({
|
safeSendViewer(ws, {
|
||||||
type: "command",
|
type: "file_upload_result",
|
||||||
commandType: "file_upload",
|
commandId,
|
||||||
id: commandId,
|
transferId: upload.transferId,
|
||||||
payload: {
|
path: upload.path,
|
||||||
path: upload.path,
|
ok: false,
|
||||||
data: upload.data,
|
error: "chunked uploads are disabled; refresh and retry",
|
||||||
offset: upload.offset,
|
|
||||||
total: upload.total,
|
|
||||||
transferId: upload.transferId,
|
|
||||||
},
|
|
||||||
} as any));
|
|
||||||
metrics.recordCommand("file_upload");
|
|
||||||
logAudit({
|
|
||||||
timestamp: Date.now(),
|
|
||||||
username: (ws.data as any).username || "unknown",
|
|
||||||
ip: ws.data.ip || "unknown",
|
|
||||||
action: AuditAction.FILE_UPLOAD,
|
|
||||||
targetClientId: clientId,
|
|
||||||
details: JSON.stringify({ path: upload.path, size: upload.data.length, offset: upload.offset }),
|
|
||||||
success: true,
|
|
||||||
});
|
});
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user