more modularization

2026-05-29 20:14:31 -05:00
parent b975907b20
commit 3102a00e8c
13 changed files with 497 additions and 453 deletions
@@ -0,0 +1,99 @@
+import type { AuthenticatedUser } from "../../auth";
+import {
+  deleteBuildProfileForUser,
+  listBuildProfilesForUser,
+  saveBuildProfileForUser,
+} from "../../db";
+import { requirePermission } from "../../rbac";
+
+export async function handleBuildProfileRoutes(
+  req: Request,
+  url: URL,
+  user: AuthenticatedUser,
+): Promise<Response | null> {
+  if (req.method === "GET" && url.pathname === "/api/build/profiles") {
+    requirePermission(user, "clients:build");
+
+    const profiles = listBuildProfilesForUser(user.userId).map((p) => {
+      let parsedConfig: any = {};
+      try {
+        parsedConfig = JSON.parse(p.profileJson);
+      } catch {
+        parsedConfig = {};
+      }
+      return {
+        name: p.name,
+        config: parsedConfig,
+        createdAt: p.createdAt,
+        updatedAt: p.updatedAt,
+      };
+    });
+    return Response.json({ profiles });
+  }
+
+  if (req.method === "POST" && url.pathname === "/api/build/profiles") {
+    requirePermission(user, "clients:build");
+
+    const body = await req.json();
+    const name = typeof body?.name === "string" ? body.name.trim() : "";
+    const config = body?.config;
+
+    if (!name || name.length > 64 || !/^[A-Za-z0-9 _.-]+$/.test(name)) {
+      return Response.json(
+        { error: "Invalid profile name (1-64 chars, letters/numbers/space/._-)" },
+        { status: 400 },
+      );
+    }
+
+    if (!config || typeof config !== "object" || Array.isArray(config)) {
+      return Response.json({ error: "Invalid profile config" }, { status: 400 });
+    }
+
+    const MAX_PROFILES_PER_USER = 50;
+    const existingProfiles = listBuildProfilesForUser(user.userId);
+    const hasExistingWithSameName = existingProfiles.some((p) => p.name === name);
+    if (!hasExistingWithSameName && existingProfiles.length >= MAX_PROFILES_PER_USER) {
+      return Response.json(
+        { error: `Profile limit reached (${MAX_PROFILES_PER_USER} max)` },
+        { status: 400 },
+      );
+    }
+
+    let profileJson = "{}";
+    try {
+      profileJson = JSON.stringify(config);
+    } catch {
+      return Response.json({ error: "Profile config must be JSON serializable" }, { status: 400 });
+    }
+
+    if (profileJson.length > 512 * 1024) {
+      return Response.json({ error: "Profile config too large (max 512KB)" }, { status: 400 });
+    }
+
+    saveBuildProfileForUser(user.userId, name, profileJson);
+    return Response.json({ success: true });
+  }
+
+  if (req.method === "DELETE" && url.pathname.match(/^\/api\/build\/profiles\/.+$/)) {
+    requirePermission(user, "clients:build");
+
+    const encodedName = url.pathname.split("/api/build/profiles/")[1] || "";
+    let name = "";
+    try {
+      name = decodeURIComponent(encodedName).trim();
+    } catch {
+      return Response.json({ error: "Invalid profile name" }, { status: 400 });
+    }
+    if (!name) {
+      return Response.json({ error: "Invalid profile name" }, { status: 400 });
+    }
+
+    const deleted = deleteBuildProfileForUser(user.userId, name);
+    if (!deleted) {
+      return Response.json({ error: "Profile not found" }, { status: 404 });
+    }
+    return Response.json({ success: true });
+  }
+
+  return null;
+}
@@ -10,9 +10,6 @@ import {
  getBuild,
  countBuildsForUser,
  getOldestBuildForUser,
-  listBuildProfilesForUser,
-  saveBuildProfileForUser,
-  deleteBuildProfileForUser,
  setBuildBlocked,
  listClientIdsByBuildTag,
  countBuildClaimsBatch,
@@ -29,7 +26,8 @@ import { addBuildToBanlist, removeBuildFromBanlist } from "../build-signing";
 import path from "path";
 import fs from "fs";
 import { resolveRuntimeRoot } from "../runtime-paths";
-import { createUploadPull } from "./file-download-routes";
+import { createUploadPull } from "../file-transfer-state";
+import { handleBuildProfileRoutes } from "./build-profile-routes";

 type RequestIpProvider = {
  requestIP: (req: Request) => { address?: string } | null | undefined;
@@ -426,89 +424,8 @@ export async function handleBuildRoutes(
      return Response.json({ builds: enriched });
    }

-    if (req.method === "GET" && url.pathname === "/api/build/profiles") {
-      requirePermission(user, "clients:build");
-
-      const profiles = listBuildProfilesForUser(user.userId).map((p) => {
-        let parsedConfig: any = {};
-        try {
-          parsedConfig = JSON.parse(p.profileJson);
-        } catch {
-          parsedConfig = {};
-        }
-        return {
-          name: p.name,
-          config: parsedConfig,
-          createdAt: p.createdAt,
-          updatedAt: p.updatedAt,
-        };
-      });
-      return Response.json({ profiles });
-    }
-
-    if (req.method === "POST" && url.pathname === "/api/build/profiles") {
-      requirePermission(user, "clients:build");
-
-      const body = await req.json();
-      const name = typeof body?.name === "string" ? body.name.trim() : "";
-      const config = body?.config;
-
-      if (!name || name.length > 64 || !/^[A-Za-z0-9 _.-]+$/.test(name)) {
-        return Response.json(
-          { error: "Invalid profile name (1-64 chars, letters/numbers/space/._-)" },
-          { status: 400 },
-        );
-      }
-
-      if (!config || typeof config !== "object" || Array.isArray(config)) {
-        return Response.json({ error: "Invalid profile config" }, { status: 400 });
-      }
-
-      const MAX_PROFILES_PER_USER = 50;
-      const existingProfiles = listBuildProfilesForUser(user.userId);
-      const hasExistingWithSameName = existingProfiles.some((p) => p.name === name);
-      if (!hasExistingWithSameName && existingProfiles.length >= MAX_PROFILES_PER_USER) {
-        return Response.json(
-          { error: `Profile limit reached (${MAX_PROFILES_PER_USER} max)` },
-          { status: 400 },
-        );
-      }
-
-      let profileJson = "{}";
-      try {
-        profileJson = JSON.stringify(config);
-      } catch {
-        return Response.json({ error: "Profile config must be JSON serializable" }, { status: 400 });
-      }
-
-      if (profileJson.length > 512 * 1024) {
-        return Response.json({ error: "Profile config too large (max 512KB)" }, { status: 400 });
-      }
-
-      saveBuildProfileForUser(user.userId, name, profileJson);
-      return Response.json({ success: true });
-    }
-
-    if (req.method === "DELETE" && url.pathname.match(/^\/api\/build\/profiles\/.+$/)) {
-      requirePermission(user, "clients:build");
-
-      const encodedName = url.pathname.split("/api/build/profiles/")[1] || "";
-      let name = "";
-      try {
-        name = decodeURIComponent(encodedName).trim();
-      } catch {
-        return Response.json({ error: "Invalid profile name" }, { status: 400 });
-      }
-      if (!name) {
-        return Response.json({ error: "Invalid profile name" }, { status: 400 });
-      }
-
-      const deleted = deleteBuildProfileForUser(user.userId, name);
-      if (!deleted) {
-        return Response.json({ error: "Profile not found" }, { status: 404 });
-      }
-      return Response.json({ success: true });
-    }
+    const profileResult = await handleBuildProfileRoutes(req, url, user);
+    if (profileResult) return profileResult;

    if (req.method === "DELETE" && url.pathname.match(/^\/api\/build\/(.+)\/delete$/)) {
      requirePermission(user, "clients:build");
@@ -8,7 +8,7 @@ import * as clientManager from "../../clientManager";
 import { metrics } from "../../metrics";
 import { encodeMessage } from "../../protocol";
 import { requirePermission } from "../../rbac";
-import { createUploadPull } from "./file-download-routes";
+import { createUploadPull } from "../file-transfer-state";

 type RequestIpProvider = {
  requestIP: (req: Request) => { address?: string } | null | undefined;
@@ -1,5 +1,4 @@
 import fs from "fs/promises";
-import type { FileHandle } from "fs/promises";
 import path from "path";
 import { v4 as uuidv4 } from "uuid";
 import { authenticateRequest } from "../../auth";
@@ -10,143 +9,26 @@ import { encodeMessage } from "../../protocol";
 import { getConfig } from "../../config";
 import { isAuthorizedAgentRequest } from "../agent-auth";
 import { requireClientAccess, requireFeatureAccess, requirePermission } from "../../rbac";
+import {
+  FILE_UPLOAD_INTENT_TTL_MS,
+  FILE_UPLOAD_PULL_TTL_MS,
+  UUID_TOKEN_RE,
+  isSafeRemotePath,
+  makeIncrementalPullStream,
+  notifyPullWaiters,
+  streamFileAndDelete,
+  uploadIntents,
+  uploadPulls,
+  waitForPullProgress,
+  type DownloadIntent,
+  type PendingHttpDownload,
+  type StreamingPullState,
+} from "../file-transfer-state";

 type RequestIpProvider = {
  requestIP: (req: Request) => { address?: string } | null | undefined;
 };

-type PendingHttpDownload = {
-  commandId: string;
-  clientId: string;
-  path: string;
-  fileName: string;
-  total: number;
-  receivedBytes: number;
-  receivedOffsets: Set<number>;
-  receivedChunks: Set<number>;
-  chunkSize: number;
-  expectedChunks: number;
-  loggedTotal?: boolean;
-  loggedFirstChunk?: boolean;
-  tmpPath: string;
-  fileHandle: any;
-  resolve: (entry: PendingHttpDownload) => void;
-  reject: (error: Error) => void;
-  timeout: ReturnType<typeof setTimeout>;
-  streamController?: ReadableStreamDefaultController<Uint8Array>;
-  reorderBuffer?: Map<number, Uint8Array>;
-  nextExpectedOffset?: number;
-  streamErrored?: boolean;
-  onFirstChunk?: () => void;
-  firstChunkSignaled?: boolean;
-};
-
-type DownloadIntent = {
-  id: string;
-  userId: number;
-  clientId: string;
-  path: string;
-  expiresAt: number;
-  timeout: ReturnType<typeof setTimeout>;
-};
-
-type UploadIntent = {
-  id: string;
-  userId: number;
-  clientId: string;
-  path: string;
-  fileName: string;
-  expiresAt: number;
-  timeout: ReturnType<typeof setTimeout>;
-};
-
-type StreamingPullState = {
-  size: number;
-  done: boolean;
-  error: Error | null;
-  waiters: Array<() => void>;
-};
-
-type UploadPull = {
-  id: string;
-  clientId: string;
-  path: string;
-  fileName: string;
-  tmpPath: string;
-  size: number;
-  expiresAt: number;
-  timeout: ReturnType<typeof setTimeout>;
-  deleteFile: boolean;
-  state?: StreamingPullState;
-  expectedTotal?: number;
-};
-
-function notifyPullWaiters(state: StreamingPullState) {
-  const list = state.waiters.splice(0);
-  for (const w of list) {
-    try { w(); } catch {}
-  }
-}
-
-function waitForPullProgress(state: StreamingPullState): Promise<void> {
-  if (state.done) return Promise.resolve();
-  return new Promise<void>((resolve) => state.waiters.push(resolve));
-}
-
-function makeIncrementalPullStream(pull: UploadPull): ReadableStream<Uint8Array> {
-  let cancelled = false;
-  return new ReadableStream<Uint8Array>({
-    async start(controller) {
-      const state = pull.state;
-      if (!state) {
-        controller.close();
-        return;
-      }
-      let fh: FileHandle | null = null;
-      try {
-        fh = await fs.open(pull.tmpPath, "r");
-        let pos = 0;
-        const READ_CHUNK = 256 * 1024;
-        while (!cancelled) {
-          while (pos < state.size && !cancelled) {
-            const remaining = state.size - pos;
-            const buf = new Uint8Array(Math.min(remaining, READ_CHUNK));
-            const { bytesRead } = await fh.read(buf, 0, buf.length, pos);
-            if (bytesRead === 0) break;
-            controller.enqueue(bytesRead === buf.length ? buf : buf.subarray(0, bytesRead));
-            pos += bytesRead;
-          }
-          if (cancelled) return;
-          if (state.error) {
-            controller.error(state.error);
-            return;
-          }
-          if (state.done) {
-            controller.close();
-            return;
-          }
-          await Promise.race([
-            waitForPullProgress(state),
-            new Promise<void>((res) => setTimeout(res, 5000)),
-          ]);
-        }
-      } catch (err) {
-        try { controller.error(err); } catch {}
-      } finally {
-        if (fh) {
-          try { await fh.close(); } catch {}
-        }
-      }
-    },
-    cancel() {
-      cancelled = true;
-      if (pull.state) {
-        notifyPullWaiters(pull.state);
-      }
-    },
-  });
-}
-
 type FileDownloadRouteDeps = {
  DATA_DIR: string;
  secureHeaders: (contentType?: string) => Record<string, string>;
@@ -155,79 +37,6 @@ type FileDownloadRouteDeps = {
  downloadIntents: Map<string, DownloadIntent>;
 };

-export async function cleanupFileTransferTempFiles(dataDir: string): Promise<void> {
-  const uploadsDir = path.join(dataDir, "uploads");
-  const downloadsDir = path.join(dataDir, "downloads");
-  await fs.rm(uploadsDir, { recursive: true, force: true }).catch(() => {});
-  await fs.rm(downloadsDir, { recursive: true, force: true }).catch(() => {});
-}
-
-function streamFileAndDelete(tmpPath: string): ReadableStream<Uint8Array> {
-  const reader = Bun.file(tmpPath).stream().getReader();
-  let cleaned = false;
-
-  const cleanup = async () => {
-    if (cleaned) return;
-    cleaned = true;
-    try { await reader.cancel(); } catch {}
-    await fs.unlink(tmpPath).catch(() => {});
-  };
-
-  return new ReadableStream<Uint8Array>({
-    async pull(controller) {
-      try {
-        const { done, value } = await reader.read();
-        if (done) {
-          await cleanup();
-          controller.close();
-          return;
-        }
-        if (value) {
-          controller.enqueue(value);
-        }
-      } catch (err) {
-        controller.error(err);
-        await cleanup();
-      }
-    },
-    async cancel() {
-      try {
-        await reader.cancel();
-      } catch {}
-      await cleanup();
-    },
-  });
-}
-
-const UUID_TOKEN_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
-const uploadIntents = new Map<string, UploadIntent>();
-const uploadPulls = new Map<string, UploadPull>();
-
-function parsePositiveIntEnv(name: string, fallback: number): number {
-  const raw = process.env[name];
-  if (!raw) return fallback;
-  const parsed = Number(raw);
-  if (!Number.isFinite(parsed) || parsed <= 0) return fallback;
-  return Math.floor(parsed);
-}
-
-// WAN uploads can be significantly slower than LAN transfers.
-const FILE_UPLOAD_INTENT_TTL_MS = parsePositiveIntEnv(
-  "OVERLORD_FILE_UPLOAD_INTENT_TTL_MS",
-  30 * 60_000,
-);
-const FILE_UPLOAD_PULL_TTL_MS = parsePositiveIntEnv(
-  "OVERLORD_FILE_UPLOAD_PULL_TTL_MS",
-  30 * 60_000,
-);
-
-function isSafeRemotePath(value: string): boolean {
-  if (!value || value.length > 4096) return false;
-  if (/[\x00-\x1F\x7F]/.test(value)) return false;
-  if (/(\.\.([\/\\]|$))/.test(value)) return false;
-  return true;
-}
-
 async function serveDownloadById(
  req: Request,
  downloadId: string,
@@ -437,33 +246,6 @@ async function serveDownloadById(
  return new Response(stream, { headers });
 }

-export function createUploadPull(opts: {
-  clientId: string;
-  filePath: string;
-  fileName: string;
-  size: number;
-  ttlMs?: number;
-}): string {
-  const pullId = uuidv4();
-  const ttl = opts.ttlMs ?? FILE_UPLOAD_PULL_TTL_MS;
-  const expiresAt = Date.now() + ttl;
-  const pullTimeout = setTimeout(() => {
-    uploadPulls.delete(pullId);
-  }, ttl);
-  uploadPulls.set(pullId, {
-    id: pullId,
-    clientId: opts.clientId,
-    path: opts.filePath,
-    fileName: opts.fileName,
-    tmpPath: opts.filePath,
-    size: opts.size,
-    expiresAt,
-    timeout: pullTimeout,
-    deleteFile: false,
-  });
-  return pullId;
-}
-
 export async function handleFileDownloadRoutes(
  req: Request,
  url: URL,
@@ -7,7 +7,7 @@ import * as clientManager from "../../clientManager";
 import { metrics } from "../../metrics";
 import { encodeMessage } from "../../protocol";
 import { requirePermission } from "../../rbac";
-import { createUploadPull } from "./file-download-routes";
+import { createUploadPull } from "../file-transfer-state";

 type RequestIpProvider = {
  requestIP: (req: Request) => { address?: string } | null | undefined;