let plugins read files on disk (and write and execute and update docs)

Overlord-Server/src/server/routes/plugin-routes.ts

@@ -231,6 +231,155 @@ export async function handlePluginRoutes(
     });
   }
 
+  function resolveDataPath(pluginId: string, relPath: string): string | null {
+    const dataDir = path.join(deps.PLUGIN_ROOT, pluginId, "data");
+    const target = path.resolve(dataDir, relPath);
+    const prefix = dataDir.endsWith(path.sep) ? dataDir : `${dataDir}${path.sep}`;
+    if (target !== dataDir && !target.startsWith(prefix)) return null;
+    return target;
+  }
+
+  const pluginDataListMatch = url.pathname.match(/^\/api\/plugins\/([^/]+)\/data$/);
+  if (req.method === "GET" && pluginDataListMatch) {
+    const user = await authenticateRequest(req);
+    if (!user) return new Response("Unauthorized", { status: 401 });
+    try { requirePermission(user, "clients:control"); } catch (e) { if (e instanceof Response) return e; return new Response("Forbidden", { status: 403 }); }
+    let pluginId = "";
+    try { pluginId = deps.sanitizePluginId(pluginDataListMatch[1]); } catch { return new Response("Invalid plugin id", { status: 400 }); }
+    const dataDir = path.join(deps.PLUGIN_ROOT, pluginId, "data");
+    await fs.mkdir(dataDir, { recursive: true });
+    async function walkDir(dir: string, base: string): Promise<{ path: string; size: number; isDir: boolean }[]> {
+      const entries = await fs.readdir(dir, { withFileTypes: true });
+      const results: { path: string; size: number; isDir: boolean }[] = [];
+      for (const ent of entries) {
+        const rel = base ? `${base}/${ent.name}` : ent.name;
+        if (ent.isDirectory()) {
+          results.push({ path: rel, size: 0, isDir: true });
+          const sub = await walkDir(path.join(dir, ent.name), rel);
+          results.push(...sub);
+        } else {
+          const st = await fs.stat(path.join(dir, ent.name)).catch(() => null);
+          results.push({ path: rel, size: st?.size ?? 0, isDir: false });
+        }
+      }
+      return results;
+    }
+    const files = await walkDir(dataDir, "");
+    return Response.json({ ok: true, files });
+  }
+
+  const pluginDataReadMatch = url.pathname.match(/^\/api\/plugins\/([^/]+)\/data\/(.+)$/);
+  if (req.method === "GET" && pluginDataReadMatch) {
+    const user = await authenticateRequest(req);
+    if (!user) return new Response("Unauthorized", { status: 401 });
+    try { requirePermission(user, "clients:control"); } catch (e) { if (e instanceof Response) return e; return new Response("Forbidden", { status: 403 }); }
+    let pluginId = "";
+    try { pluginId = deps.sanitizePluginId(pluginDataReadMatch[1]); } catch { return new Response("Invalid plugin id", { status: 400 }); }
+    let relPath = pluginDataReadMatch[2];
+    try { relPath = decodeURIComponent(relPath); } catch { return new Response("Bad request", { status: 400 }); }
+    if (relPath.includes("\u0000")) return new Response("Bad request", { status: 400 });
+    const target = resolveDataPath(pluginId, relPath);
+    if (!target) return new Response("Forbidden", { status: 403 });
+    const file = Bun.file(target);
+    if (!(await file.exists())) return new Response("Not found", { status: 404 });
+    const st = await fs.stat(target);
+    if (st.isDirectory()) return new Response("Is a directory", { status: 400 });
+    return new Response(file, { headers: deps.secureHeaders(deps.mimeType(relPath)) });
+  }
+
+  const pluginDataWriteMatch = url.pathname.match(/^\/api\/plugins\/([^/]+)\/data\/(.+)$/);
+  if (req.method === "PUT" && pluginDataWriteMatch) {
+    const user = await authenticateRequest(req);
+    if (!user) return new Response("Unauthorized", { status: 401 });
+    try { requirePermission(user, "clients:control"); } catch (e) { if (e instanceof Response) return e; return new Response("Forbidden", { status: 403 }); }
+    let pluginId = "";
+    try { pluginId = deps.sanitizePluginId(pluginDataWriteMatch[1]); } catch { return new Response("Invalid plugin id", { status: 400 }); }
+    let relPath = pluginDataWriteMatch[2];
+    try { relPath = decodeURIComponent(relPath); } catch { return new Response("Bad request", { status: 400 }); }
+    if (relPath.includes("\u0000") || relPath.endsWith("/") || relPath.endsWith(path.sep)) {
+      return new Response("Bad request", { status: 400 });
+    }
+    const target = resolveDataPath(pluginId, relPath);
+    if (!target) return new Response("Forbidden", { status: 403 });
+    await fs.mkdir(path.dirname(target), { recursive: true });
+    const body = await req.arrayBuffer();
+    await fs.writeFile(target, new Uint8Array(body));
+    return Response.json({ ok: true, path: relPath, size: body.byteLength });
+  }
+
+  const pluginDataDeleteMatch = url.pathname.match(/^\/api\/plugins\/([^/]+)\/data\/(.+)$/);
+  if (req.method === "DELETE" && pluginDataDeleteMatch) {
+    const user = await authenticateRequest(req);
+    if (!user) return new Response("Unauthorized", { status: 401 });
+    try { requirePermission(user, "clients:control"); } catch (e) { if (e instanceof Response) return e; return new Response("Forbidden", { status: 403 }); }
+    let pluginId = "";
+    try { pluginId = deps.sanitizePluginId(pluginDataDeleteMatch[1]); } catch { return new Response("Invalid plugin id", { status: 400 }); }
+    let relPath = pluginDataDeleteMatch[2];
+    try { relPath = decodeURIComponent(relPath); } catch { return new Response("Bad request", { status: 400 }); }
+    if (relPath.includes("\u0000")) return new Response("Bad request", { status: 400 });
+    const target = resolveDataPath(pluginId, relPath);
+    if (!target) return new Response("Forbidden", { status: 403 });
+    try {
+      const st = await fs.stat(target);
+      if (st.isDirectory()) {
+        await fs.rm(target, { recursive: true, force: true });
+      } else {
+        await fs.unlink(target);
+      }
+    } catch {
+      return new Response("Not found", { status: 404 });
+    }
+    return Response.json({ ok: true, path: relPath });
+  }
+
+  // Execute a file stored in the plugin's data directory
+  const pluginExecMatch = url.pathname.match(/^\/api\/plugins\/([^/]+)\/exec$/);
+  if (req.method === "POST" && pluginExecMatch) {
+    const user = await authenticateRequest(req);
+    if (!user) return new Response("Unauthorized", { status: 401 });
+    if (user.role !== "admin" && user.role !== "operator") {
+      return new Response("Forbidden: Admin or operator access required", { status: 403 });
+    }
+    let pluginId = "";
+    try { pluginId = deps.sanitizePluginId(pluginExecMatch[1]); } catch { return new Response("Invalid plugin id", { status: 400 }); }
+    let body: any = {};
+    try { body = await req.json(); } catch { return new Response("Bad request", { status: 400 }); }
+    const filePath = typeof body.file === "string" ? body.file : "";
+    if (!filePath) return new Response("Missing file", { status: 400 });
+    let decodedFile = filePath;
+    try { decodedFile = decodeURIComponent(filePath); } catch { return new Response("Bad request", { status: 400 }); }
+    if (decodedFile.includes("\u0000")) return new Response("Bad request", { status: 400 });
+    const target = resolveDataPath(pluginId, decodedFile);
+    if (!target) return new Response("Forbidden", { status: 403 });
+    try {
+      const st = await fs.stat(target);
+      if (st.isDirectory()) return new Response("Is a directory", { status: 400 });
+    } catch {
+      return new Response("Not found", { status: 404 });
+    }
+    const args: string[] = Array.isArray(body.args) ? body.args.filter((a: any) => typeof a === "string") : [];
+    const stdinData: string = typeof body.stdin === "string" ? body.stdin : "";
+    const timeoutMs: number = typeof body.timeoutMs === "number" && body.timeoutMs > 0 ? Math.min(body.timeoutMs, 60_000) : 30_000;
+    // Ensure the binary is executable
+    try { await fs.chmod(target, 0o755); } catch {}
+    const proc = Bun.spawn([target, ...args], {
+      cwd: path.dirname(target),
+      stdin: stdinData ? Buffer.from(stdinData) : "ignore",
+      stdout: "pipe",
+      stderr: "pipe",
+    });
+    const killTimer = setTimeout(() => { try { proc.kill(); } catch {} }, timeoutMs);
+    let exitCode = 0;
+    try {
+      exitCode = await proc.exited;
+    } finally {
+      clearTimeout(killTimer);
+    }
+    const stdout = await new Response(proc.stdout).text();
+    const stderr = await new Response(proc.stderr).text();
+    return Response.json({ ok: true, exitCode, stdout, stderr });
+  }
+
   const pluginDeleteMatch = url.pathname.match(/^\/api\/plugins\/(.+)$/);
   if (req.method === "DELETE" && pluginDeleteMatch) {
     const user = await authenticateRequest(req);
@@ -255,9 +404,19 @@ export async function handlePluginRoutes(
       await fs.rm(zipPath, { force: true });
     } catch {}
 
-    try {
+    // Remove everything except the data/ subdirectory so plugin-stored files survive reinstalls.
-      await fs.rm(pluginDir, { recursive: true, force: true });
+    if (pluginDir) {
-    } catch {}
+      try {
+        const entries = await fs.readdir(pluginDir, { withFileTypes: true });
+        for (const ent of entries) {
+          if (ent.name === "data") continue; // preserve plugin data directory
+          await fs.rm(path.join(pluginDir, ent.name), { recursive: true, force: true });
+        }
+        // Remove the directory itself only if it is now empty
+        const remaining = await fs.readdir(pluginDir);
+        if (remaining.length === 0) await fs.rmdir(pluginDir);
+      } catch {}
+    }
 
     // Unload from all clients that have it loaded
     for (const [cid, loadedSet] of deps.pluginLoadedByClient) {

PLUGINS.md

@@ -254,7 +254,17 @@ Plugin UIs run in a **sandboxed iframe** with a fetch bridge.
 - `POST /api/plugins/upload` — upload zip
 - `POST /api/plugins/<id>/enable` — enable/disable
 - `POST /api/plugins/<id>/autoload` — configure auto-load on client connect
-- `DELETE /api/plugins/<id>` — remove
+- `DELETE /api/plugins/<id>` — remove (preserves `data/` directory)
+
+### Plugin data directory
+
+- `GET /api/plugins/<id>/data` — list files in the plugin's persistent data directory
+- `GET /api/plugins/<id>/data/<path>` — read a file
+- `PUT /api/plugins/<id>/data/<path>` — write a file
+- `DELETE /api/plugins/<id>/data/<path>` — delete a file or directory
+- `POST /api/plugins/<id>/exec` — execute a stored binary (admin/operator only)
+
+See [section 8](#8-server-side-plugin-data-directory) for full details.
 
 ### Per-client plugin runtime
 
@@ -358,4 +368,117 @@ Content-Type: application/json
 - The server selects the correct binary for each client's OS/architecture automatically
 - If a plugin binary isn't available for a client's platform, the auto-load silently skips that client
 
+## 8) Server-side plugin data directory
+
+Each plugin has a **persistent data directory** on the server:
+
+```
+Overlord-Server/plugins/<pluginId>/data/
+```
+
+This directory is **never deleted** when a plugin is removed or reinstalled. It survives the entire plugin lifecycle and gives server-side plugins a dedicated place to store files — SQLite databases, config files, cached data, executables, etc.
+
+### Read/write files
+
+From your plugin UI JS (or any authenticated API consumer):
+
+**List all files**
+```
+GET /api/plugins/<pluginId>/data
+```
+```json
+{
+  "ok": true,
+  "files": [
+    { "path": "config.json", "size": 128, "isDir": false },
+    { "path": "cache", "size": 0, "isDir": true },
+    { "path": "cache/data.db", "size": 40960, "isDir": false }
+  ]
+}
+```
+
+**Read a file**
+```
+GET /api/plugins/<pluginId>/data/<path>
+```
+Returns the raw file bytes with an appropriate `Content-Type`.
+
+**Write a file**
+```
+PUT /api/plugins/<pluginId>/data/<path>
+Content-Type: application/octet-stream
+<raw bytes>
+```
+Parent directories are created automatically.
+```json
+{ "ok": true, "path": "config.json", "size": 128 }
+```
+
+**Delete a file or directory**
+```
+DELETE /api/plugins/<pluginId>/data/<path>
+```
+Deleting a directory removes it recursively.
+```json
+{ "ok": true, "path": "cache" }
+```
+
+### Execute a stored binary
+
+Plugins can store executables in their data directory and run them on the server. This endpoint is restricted to **admin and operator** roles.
+
+```
+POST /api/plugins/<pluginId>/exec
+Content-Type: application/json
+
+{
+  "file": "mytool",
+  "args": ["--flag", "value"],
+  "stdin": "optional stdin string",
+  "timeoutMs": 10000
+}
+```
+
+| Field | Type | Description |
+|-------|------|-------------|
+| `file` | string | Path to the binary, relative to the plugin's `data/` directory |
+| `args` | string[] | Command-line arguments (optional) |
+| `stdin` | string | Text to pipe to stdin (optional) |
+| `timeoutMs` | number | Max run time in ms, max 60000, default 30000 |
+
+```json
+{
+  "ok": true,
+  "exitCode": 0,
+  "stdout": "tool output here",
+  "stderr": ""
+}
+```
+
+The binary is run from the `data/` directory as the working directory. The server sets the executable bit automatically before running on Unix.
+
+### Example: storing a SQLite database from plugin JS
+
+```js
+// Write a new database seed file
+await fetch(`/api/plugins/myplugin/data/app.db`, {
+  method: "PUT",
+  body: dbBytes, // ArrayBuffer
+});
+
+// Read it back
+const res = await fetch(`/api/plugins/myplugin/data/app.db`);
+const db = await res.arrayBuffer();
+
+// List all stored files
+const { files } = await (await fetch(`/api/plugins/myplugin/data`)).json();
+```
+
+### Notes
+
+- Path traversal is blocked — paths may not escape the `data/` directory
+- Null bytes in paths are rejected
+- The data directory is created automatically on first use (no pre-setup needed)
+- Deleting and reinstalling a plugin leaves `data/` untouched
+