change how plugins are sent to clients (https is better than whatever tf I can do)

Overlord-Client/cmd/agent/handlers/command.go

@@ -575,45 +575,19 @@ func HandleCommand(ctx context.Context, env *runtime.Env, envelope map[string]in
 		}
 		_ = wire.WriteMsg(ctx, env.Conn, wire.PluginEvent{Type: "plugin_event", PluginID: manifest.ID, Event: "loaded"})
 		return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: true})
-	case "plugin_load_init":
+	case "plugin_load_http":
 		payload, _ := envelope["payload"].(map[string]interface{})
-		if payload == nil || env.Plugins == nil {
+		if payload == nil {
-			return nil
+			return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: false, Message: "missing payload"})
 		}
 		manifestRaw, _ := payload["manifest"].(map[string]interface{})
-		totalSize := toInt(payload["size"])
-		totalChunks := toInt(payload["chunks"])
 		manifest, err := plugins.ManifestFromMap(manifestRaw)
 		if err != nil {
-			return nil
+			return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: false, Message: err.Error()})
 		}
-		_ = env.Plugins.StartBundle(manifest, totalSize, totalChunks)
+		pullURL, _ := payload["url"].(string)
-		return nil
+		expectedSize := int64(toInt(payload["size"]))
-	case "plugin_load_chunk":
+		return HandlePluginLoadHTTP(ctx, env, cmdID, manifest, pullURL, expectedSize)
-		payload, _ := envelope["payload"].(map[string]interface{})
-		if payload == nil || env.Plugins == nil {
-			return nil
-		}
-		pluginId, _ := payload["pluginId"].(string)
-		index := toInt(payload["index"])
-		data, _ := payload["data"].([]byte)
-		_ = env.Plugins.AddChunk(pluginId, index, data)
-		return nil
-	case "plugin_load_finish":
-		payload, _ := envelope["payload"].(map[string]interface{})
-		if payload == nil || env.Plugins == nil {
-			return nil
-		}
-		pluginId, _ := payload["pluginId"].(string)
-		if pluginId == "" {
-			return nil
-		}
-		if err := env.Plugins.FinalizeBundle(ctx, pluginId); err != nil {
-			_ = wire.WriteMsg(ctx, env.Conn, wire.PluginEvent{Type: "plugin_event", PluginID: pluginId, Event: "error", Error: err.Error()})
-			return nil
-		}
-		_ = wire.WriteMsg(ctx, env.Conn, wire.PluginEvent{Type: "plugin_event", PluginID: pluginId, Event: "loaded"})
-		return nil
 	case "plugin_unload":
 		payload, _ := envelope["payload"].(map[string]interface{})
 		if payload == nil || env.Plugins == nil {

Overlord-Client/cmd/agent/handlers/plugin_load.go

@@ -0,0 +1,142 @@
+package handlers
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"strings"
+
+	"overlord-client/cmd/agent/plugins"
+	agentRuntime "overlord-client/cmd/agent/runtime"
+	"overlord-client/cmd/agent/wire"
+)
+
+const maxPluginBinaryBytes = 200 * 1024 * 1024
+
+func HandlePluginLoadHTTP(ctx context.Context, env *agentRuntime.Env, cmdID string, manifest plugins.PluginManifest, pullURL string, expectedSize int64) error {
+	if env.Plugins == nil {
+		return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: false, Message: "plugin manager not ready"})
+	}
+
+	resolved, err := resolvePluginPullURL(env, pullURL)
+	if err != nil {
+		_ = wire.WriteMsg(ctx, env.Conn, wire.PluginEvent{Type: "plugin_event", PluginID: manifest.ID, Event: "error", Error: err.Error()})
+		return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: false, Message: err.Error()})
+	}
+
+	tlsConfig := &tls.Config{InsecureSkipVerify: env.Cfg.TLSInsecureSkipVerify, MinVersion: tls.VersionTLS12}
+	if caPath := strings.TrimSpace(env.Cfg.TLSCAPath); caPath != "" {
+		caBytes, err := os.ReadFile(caPath)
+		if err != nil {
+			return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: false, Message: fmt.Sprintf("failed to read TLS CA: %v", err)})
+		}
+		pool := x509.NewCertPool()
+		if !pool.AppendCertsFromPEM(caBytes) {
+			return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: false, Message: "failed to parse TLS CA"})
+		}
+		tlsConfig.RootCAs = pool
+	}
+
+	client := &http.Client{Transport: &http.Transport{TLSClientConfig: tlsConfig}}
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, resolved, nil)
+	if err != nil {
+		return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: false, Message: err.Error()})
+	}
+	if token := strings.TrimSpace(env.Cfg.AgentToken); token != "" {
+		req.Header.Set("x-agent-token", token)
+	}
+	if id := strings.TrimSpace(env.Cfg.ID); id != "" {
+		req.Header.Set("x-overlord-client-id", id)
+	}
+
+	resp, err := client.Do(req)
+	if err != nil {
+		_ = wire.WriteMsg(ctx, env.Conn, wire.PluginEvent{Type: "plugin_event", PluginID: manifest.ID, Event: "error", Error: err.Error()})
+		return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: false, Message: err.Error()})
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+		msg := fmt.Sprintf("plugin pull failed: status %d", resp.StatusCode)
+		_ = wire.WriteMsg(ctx, env.Conn, wire.PluginEvent{Type: "plugin_event", PluginID: manifest.ID, Event: "error", Error: msg})
+		return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: false, Message: msg})
+	}
+
+	limit := int64(maxPluginBinaryBytes)
+	if expectedSize > 0 && expectedSize < limit {
+		limit = expectedSize
+	}
+	binary, err := io.ReadAll(io.LimitReader(resp.Body, limit+1))
+	if err != nil {
+		_ = wire.WriteMsg(ctx, env.Conn, wire.PluginEvent{Type: "plugin_event", PluginID: manifest.ID, Event: "error", Error: err.Error()})
+		return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: false, Message: err.Error()})
+	}
+	if int64(len(binary)) > limit {
+		msg := "plugin pull exceeded size limit"
+		_ = wire.WriteMsg(ctx, env.Conn, wire.PluginEvent{Type: "plugin_event", PluginID: manifest.ID, Event: "error", Error: msg})
+		return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: false, Message: msg})
+	}
+	if expectedSize > 0 && int64(len(binary)) != expectedSize {
+		msg := "plugin pull size mismatch"
+		_ = wire.WriteMsg(ctx, env.Conn, wire.PluginEvent{Type: "plugin_event", PluginID: manifest.ID, Event: "error", Error: msg})
+		return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: false, Message: msg})
+	}
+
+	if err := env.Plugins.Load(ctx, manifest, binary); err != nil {
+		_ = wire.WriteMsg(ctx, env.Conn, wire.PluginEvent{Type: "plugin_event", PluginID: manifest.ID, Event: "error", Error: err.Error()})
+		return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: false, Message: err.Error()})
+	}
+	_ = wire.WriteMsg(ctx, env.Conn, wire.PluginEvent{Type: "plugin_event", PluginID: manifest.ID, Event: "loaded"})
+	return wire.WriteMsg(ctx, env.Conn, wire.CommandResult{Type: "command_result", CommandID: cmdID, OK: true})
+}
+
+func resolvePluginPullURL(env *agentRuntime.Env, raw string) (string, error) {
+	trimmed := strings.TrimSpace(raw)
+	if trimmed == "" {
+		return "", errors.New("missing plugin pull url")
+	}
+
+	parsed, err := url.Parse(trimmed)
+	if err != nil {
+		return "", fmt.Errorf("invalid plugin pull url: %w", err)
+	}
+
+	if parsed.IsAbs() {
+		scheme := strings.ToLower(parsed.Scheme)
+		if scheme != "http" && scheme != "https" {
+			return "", fmt.Errorf("unsupported plugin pull scheme: %s", parsed.Scheme)
+		}
+		return parsed.String(), nil
+	}
+
+	if len(env.Cfg.ServerURLs) == 0 {
+		return "", errors.New("no server url configured for plugin pull")
+	}
+	idx := env.Cfg.ServerIndex
+	if idx < 0 || idx >= len(env.Cfg.ServerURLs) {
+		idx = 0
+	}
+	server, err := url.Parse(env.Cfg.ServerURLs[idx])
+	if err != nil {
+		return "", fmt.Errorf("invalid agent server url: %w", err)
+	}
+	switch strings.ToLower(server.Scheme) {
+	case "wss":
+		server.Scheme = "https"
+	case "ws":
+		server.Scheme = "http"
+	case "https", "http":
+	default:
+		return "", fmt.Errorf("unsupported agent server scheme: %s", server.Scheme)
+	}
+	server.Path = parsed.Path
+	server.RawQuery = parsed.RawQuery
+	server.Fragment = ""
+	return server.String(), nil
+}

Overlord-Client/cmd/agent/plugins/manager.go

@@ -13,7 +13,6 @@ import (
 type Manager struct {
 	mu      sync.Mutex
 	plugins map[string]*pluginInstance
-	pending map[string]*pendingBundle
 	writer  wire.Writer
 	host    HostInfo
 }
@@ -24,103 +23,14 @@ type pluginInstance struct {
 	native   NativePlugin
 }
 
-type pendingBundle struct {
-	manifest    PluginManifest
-	totalSize   int
-	totalChunks int
-	chunks      map[int][]byte
-	received    int
-	receivedSz  int
-}
-
 func NewManager(writer wire.Writer, host HostInfo) *Manager {
 	return &Manager{
 		plugins: make(map[string]*pluginInstance),
-		pending: make(map[string]*pendingBundle),
 		writer:  writer,
 		host:    host,
 	}
 }
 
-func (m *Manager) StartBundle(manifest PluginManifest, totalSize int, totalChunks int) error {
-	if manifest.ID == "" {
-		return errors.New("missing plugin id")
-	}
-	if totalChunks <= 0 {
-		return errors.New("invalid total chunks")
-	}
-	if totalSize <= 0 {
-		return errors.New("invalid total size")
-	}
-	if totalChunks > 10000 {
-		return errors.New("too many chunks")
-	}
-
-	m.mu.Lock()
-	m.pending[manifest.ID] = &pendingBundle{
-		manifest:    manifest,
-		totalSize:   totalSize,
-		totalChunks: totalChunks,
-		chunks:      make(map[int][]byte),
-	}
-	m.mu.Unlock()
-	return nil
-}
-
-func (m *Manager) AddChunk(pluginId string, index int, data []byte) error {
-	if pluginId == "" {
-		return errors.New("missing plugin id")
-	}
-	if index < 0 {
-		return errors.New("invalid chunk index")
-	}
-
-	m.mu.Lock()
-	b := m.pending[pluginId]
-	if b == nil {
-		m.mu.Unlock()
-		return errors.New("bundle not initialized")
-	}
-	if _, exists := b.chunks[index]; !exists {
-		b.chunks[index] = data
-		b.received++
-		b.receivedSz += len(data)
-	}
-	m.mu.Unlock()
-	return nil
-}
-
-func (m *Manager) FinalizeBundle(ctx context.Context, pluginId string) error {
-	m.mu.Lock()
-	b := m.pending[pluginId]
-	if b == nil {
-		m.mu.Unlock()
-		return errors.New("bundle not initialized")
-	}
-	if b.received < b.totalChunks {
-		m.mu.Unlock()
-		return errors.New("bundle incomplete")
-	}
-
-	chunks := make([][]byte, b.totalChunks)
-	for i := 0; i < b.totalChunks; i++ {
-		chunks[i] = b.chunks[i]
-		if chunks[i] == nil {
-			m.mu.Unlock()
-			return errors.New("missing chunk")
-		}
-	}
-	manifest := b.manifest
-	delete(m.pending, pluginId)
-	m.mu.Unlock()
-
-	combined := make([]byte, 0, b.totalSize)
-	for _, part := range chunks {
-		combined = append(combined, part...)
-	}
-	return m.Load(ctx, manifest, combined)
-}
-
 func (m *Manager) Load(ctx context.Context, manifest PluginManifest, binary []byte) error {
 	//garble:controlflow block_splits=max junk_jumps=max flatten_passes=max
 	if len(binary) == 0 {

Overlord-Server/src/protocol.ts

@@ -95,9 +95,7 @@ export type CommandType =
   | "process_list"
   | "process_kill"
   | "plugin_load"
-  | "plugin_load_init"
+  | "plugin_load_http"
-  | "plugin_load_chunk"
-  | "plugin_load_finish"
   | "plugin_unload"
   | "agent_update"
   | "clipboard_set"

Overlord-Server/src/server/http-security.ts

@@ -5,7 +5,7 @@ export const CORS_HEADERS = {
 };
 
 export const SECURITY_HEADERS = {
-  "Content-Security-Policy": "default-src 'self'; script-src 'self'; worker-src 'self' blob:; style-src 'self' 'unsafe-inline'; img-src 'self' data: blob: https://a.basemaps.cartocdn.com https://b.basemaps.cartocdn.com https://c.basemaps.cartocdn.com https://d.basemaps.cartocdn.com; font-src 'self'; connect-src 'self'; object-src blob:; frame-ancestors 'none'",
+  "Content-Security-Policy": "default-src 'self'; script-src 'self'; worker-src 'self' blob:; style-src 'self' 'unsafe-inline'; img-src 'self' data: blob: https://cdn.discordapp.com https://a.basemaps.cartocdn.com https://b.basemaps.cartocdn.com https://c.basemaps.cartocdn.com https://d.basemaps.cartocdn.com; font-src 'self'; connect-src 'self'; object-src blob:; frame-ancestors 'none'",
   "X-Content-Type-Options": "nosniff",
   "X-Frame-Options": "DENY",
   "X-XSS-Protection": "1; mode=block",

Overlord-Server/src/server/plugin-state-bundle.ts

@@ -233,13 +233,19 @@ export async function listPluginManifests(
   }
 }
 
+export type PluginBundle = {
+  manifest: PluginManifest;
+  binaryPath: string | null;
+  size: number;
+};
+
 export async function loadPluginBundle(
   pluginRoot: string,
   pluginId: string,
   ensureExtracted: (pluginId: string) => Promise<void>,
   clientOS?: string,
   clientArch?: string,
-): Promise<{ manifest: PluginManifest; binary: Uint8Array | null }> {
+): Promise<PluginBundle> {
   await ensureExtracted(pluginId);
   const dir = path.join(pluginRoot, pluginId);
   const manifestPath = path.join(dir, "manifest.json");
@@ -250,7 +256,7 @@ export async function loadPluginBundle(
 
   const hasBinaries = manifest.binaries && Object.keys(manifest.binaries).length > 0;
   if (!hasBinaries) {
-    return { manifest, binary: null };
+    return { manifest, binaryPath: null, size: 0 };
   }
 
   let binaryPath: string | null = null;
@@ -291,52 +297,66 @@ export async function loadPluginBundle(
     binaryPath = path.join(dir, found);
   }
 
-  const binary = new Uint8Array(await fs.readFile(binaryPath));
+  const stat = await fs.stat(binaryPath);
-  return { manifest, binary };
+  return { manifest, binaryPath, size: stat.size };
+}
+
+export type PluginPull = {
+  id: string;
+  clientId: string;
+  pluginId: string;
+  binaryPath: string;
+  size: number;
+  expiresAt: number;
+  timeout: ReturnType<typeof setTimeout>;
+};
+
+const PLUGIN_PULL_TTL_MS = 5 * 60_000;
+const pluginPulls = new Map<string, PluginPull>();
+
+export function getPluginPull(id: string): PluginPull | undefined {
+  return pluginPulls.get(id);
+}
+
+export function deletePluginPull(id: string): void {
+  const pull = pluginPulls.get(id);
+  if (!pull) return;
+  pluginPulls.delete(id);
+  clearTimeout(pull.timeout);
 }
 
 export function sendPluginBundle(
   target: ClientInfo,
-  bundle: { manifest: PluginManifest; binary: Uint8Array | null },
+  bundle: PluginBundle,
 ): void {
-  if (!bundle.binary) return;
+  if (!bundle.binaryPath) return;
-  const chunkSize = 16 * 1024;
-  const data = bundle.binary;
-  const totalChunks = Math.ceil(data.length / chunkSize);
-  const initPayload = {
-    manifest: bundle.manifest,
-    size: data.length,
-    chunks: totalChunks,
-  };
-  target.ws.send(
-    encodeMessage({
-      type: "command",
-      commandType: "plugin_load_init",
-      id: uuidv4(),
-      payload: initPayload,
-    }),
-  );
 
-  for (let i = 0; i < totalChunks; i++) {
+  const pullId = uuidv4();
-    const start = i * chunkSize;
+  const expiresAt = Date.now() + PLUGIN_PULL_TTL_MS;
-    const end = Math.min(start + chunkSize, data.length);
+  const timeout = setTimeout(() => {
-    const chunk = data.slice(start, end);
+    pluginPulls.delete(pullId);
-    target.ws.send(
+  }, PLUGIN_PULL_TTL_MS);
-      encodeMessage({
+
-        type: "command",
+  pluginPulls.set(pullId, {
-        commandType: "plugin_load_chunk",
+    id: pullId,
-        id: uuidv4(),
+    clientId: target.id,
-        payload: { pluginId: bundle.manifest.id, index: i, data: chunk },
+    pluginId: bundle.manifest.id,
-      }),
+    binaryPath: bundle.binaryPath,
-    );
+    size: bundle.size,
-  }
+    expiresAt,
+    timeout,
+  });
 
   target.ws.send(
     encodeMessage({
       type: "command",
-      commandType: "plugin_load_finish",
+      commandType: "plugin_load_http",
       id: uuidv4(),
-      payload: { pluginId: bundle.manifest.id },
+      payload: {
+        manifest: bundle.manifest,
+        size: bundle.size,
+        url: `/api/plugins/pull/${encodeURIComponent(pullId)}`,
+      },
     }),
   );
 }
@@ -348,7 +368,7 @@ export async function dispatchAutoLoadPlugins(
   isPluginLoading: (clientId: string, pluginId: string) => boolean,
   markPluginLoading: (clientId: string, pluginId: string) => void,
   enqueuePluginEvent: (clientId: string, pluginId: string, event: string, payload: any) => void,
-  loadBundle: (pluginId: string, clientOS?: string, clientArch?: string) => Promise<{ manifest: PluginManifest; binary: Uint8Array | null }>,
+  loadBundle: (pluginId: string, clientOS?: string, clientArch?: string) => Promise<PluginBundle>,
 ): Promise<void> {
   const autoLoadIds = Object.entries(pluginState.autoLoad)
     .filter(([id, enabled]) => enabled && pluginState.enabled[id] !== false)

Overlord-Server/src/server/routes/plugin-routes.ts

@@ -12,6 +12,9 @@ import { encodeMessage, type PluginSignatureInfo } from "../../protocol";
 import { getConfig, updatePluginsConfig } from "../../config";
 import { getOrVerifySignature, BUILTIN_TRUSTED_KEYS } from "../plugin-signature";
 import type { PluginRuntime } from "../plugin-runtime/runtime";
+import { getPluginPull, deletePluginPull } from "../plugin-state-bundle";
+import { isAuthorizedAgentRequest } from "../agent-auth";
+import { logger } from "../../logger";
 
 type PluginManifest = {
   id: string;
@@ -22,7 +25,8 @@ type PluginManifest = {
 
 type PluginBundle = {
   manifest: PluginManifest;
-  binary: Uint8Array | null;
+  binaryPath: string | null;
+  size: number;
 };
 
 type PluginState = {
@@ -68,6 +72,48 @@ export async function handlePluginRoutes(
     return null;
   }
 
+  if (req.method === "GET" && url.pathname.startsWith("/api/plugins/pull/")) {
+    const agentToken = getConfig().auth.agentToken;
+    if (!isAuthorizedAgentRequest(req, url, agentToken)) {
+      return new Response("Unauthorized", { status: 401 });
+    }
+
+    let pullId = "";
+    try {
+      pullId = decodeURIComponent(url.pathname.slice("/api/plugins/pull/".length));
+    } catch {
+      return new Response("Bad request", { status: 400 });
+    }
+    if (!/^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(pullId)) {
+      return new Response("Bad request", { status: 400 });
+    }
+
+    const pull = getPluginPull(pullId);
+    if (!pull || pull.expiresAt < Date.now()) {
+      return new Response("Not found", { status: 404 });
+    }
+    const requesterClientId = req.headers.get("x-overlord-client-id") || "";
+    if (!requesterClientId || requesterClientId !== pull.clientId) {
+      return new Response("Forbidden", { status: 403 });
+    }
+
+    deletePluginPull(pullId);
+
+    logger.debug("[plugin] http pull", {
+      pullId,
+      clientId: pull.clientId,
+      pluginId: pull.pluginId,
+      bytes: pull.size,
+    });
+
+    const headers = {
+      ...deps.secureHeaders("application/octet-stream"),
+      "Cache-Control": "no-store, private",
+      "Content-Length": String(pull.size),
+    };
+    return new Response(Bun.file(pull.binaryPath).stream(), { headers });
+  }
+
   async function serveLoginOrUnauthorized(): Promise<Response> {
     const loginFile = Bun.file(`${deps.PUBLIC_ROOT}/login.html`);
     if (await loginFile.exists()) {