add profiler

This commit is contained in:
KingKDot
2026-06-03 22:31:48 -05:00
parent 4f82a3618a
commit 8c0996ef01
3 changed files with 579 additions and 0 deletions
+189
View File
@@ -1855,6 +1855,8 @@ function formatUptime(totalSeconds) {
return `${secs}s`;
}
let latestServerCpuProfile = null;
async function loadHealthStats() {
const loading = document.getElementById("health-loading");
const content = document.getElementById("health-content");
@@ -1927,6 +1929,189 @@ async function loadHealthStats() {
}
}
function formatMs(ms) {
const n = Number(ms) || 0;
if (n >= 1000) return `${(n / 1000).toFixed(1)}s`;
return `${Math.round(n)}ms`;
}
function formatPercent(value) {
const n = Number(value);
if (!Number.isFinite(n)) return "-";
return `${n.toFixed(n >= 10 ? 1 : 2)}%`;
}
function formatSignedBytes(value) {
const n = Number(value) || 0;
if (n === 0) return "0 B";
return `${n > 0 ? "+" : "-"}${formatBytes(Math.abs(n))}`;
}
function setProfileMessage(text, type = "ok") {
const el = document.getElementById("server-profile-message");
if (!el) return;
el.textContent = text;
el.classList.remove(
"hidden",
"text-emerald-200",
"border-emerald-700",
"bg-emerald-900/30",
"text-cyan-200",
"border-cyan-700",
"bg-cyan-900/30",
"text-rose-200",
"border-rose-700",
"bg-rose-900/30",
);
if (type === "error") {
el.classList.add("text-rose-200", "border-rose-700", "bg-rose-900/30");
} else if (type === "busy") {
el.classList.add("text-cyan-200", "border-cyan-700", "bg-cyan-900/30");
} else {
el.classList.add("text-emerald-200", "border-emerald-700", "bg-emerald-900/30");
}
}
function renderProfileRows(tbodyId, rows, renderRow, emptyText) {
const tbody = document.getElementById(tbodyId);
if (!tbody) return;
const list = Array.isArray(rows) ? rows : [];
if (!list.length) {
tbody.innerHTML = `<tr><td colspan="3" class="px-3 py-3 text-slate-500 text-center">${escapeHtml(emptyText)}</td></tr>`;
return;
}
tbody.innerHTML = list.map(renderRow).join("");
}
function renderServerProfileResults(data) {
const results = document.getElementById("server-profile-results");
const downloadBtn = document.getElementById("server-profile-download-btn");
const set = (id, value) => {
const el = document.getElementById(id);
if (el) el.textContent = value;
};
const cpu = data?.cpu || {};
const summary = cpu.summary || {};
const beforeMem = data?.memory?.before?.process || {};
const afterMem = data?.memory?.after?.process || {};
const rssDelta = (Number(afterMem.rss) || 0) - (Number(beforeMem.rss) || 0);
latestServerCpuProfile = cpu.rawProfile || null;
if (downloadBtn) downloadBtn.classList.toggle("hidden", !latestServerCpuProfile);
set("profile-duration", formatMs(data?.durationMs || 0));
set("profile-cpu-percent", formatPercent(cpu.processPercent));
set("profile-samples", summary.totalSamples ?? 0);
set("profile-rss-delta", formatSignedBytes(rssDelta));
const totalSamples = Number(summary.totalSamples) || 0;
renderProfileRows(
"profile-functions-body",
(summary.topFunctions || []).slice(0, 12),
(fn) => {
const pct = Number.isFinite(Number(fn.percent)) && Number(fn.percent) > 0
? Number(fn.percent)
: (totalSamples > 0 ? (Number(fn.samples || 0) / totalSamples) * 100 : 0);
return `<tr class="bg-slate-900/70">
<td class="px-3 py-2 min-w-[15rem]">
<div class="font-mono text-slate-100 truncate" title="${escapeHtml(fn.name || "")}">${escapeHtml(fn.name || "(anonymous)")}</div>
<div class="font-mono text-[11px] text-slate-500 truncate" title="${escapeHtml(fn.location || "")}">${escapeHtml(fn.location || "(runtime)")}</div>
</td>
<td class="px-3 py-2 text-right font-mono text-cyan-300 whitespace-nowrap">${formatPercent(pct)}</td>
<td class="px-3 py-2 text-right font-mono text-slate-300">${Number(fn.samples || 0)}</td>
</tr>`;
},
"No CPU samples were captured.",
);
const topObjectTypes = data?.memory?.after?.jscHeap?.topObjectTypes || [];
renderProfileRows(
"profile-objects-body",
topObjectTypes.slice(0, 12),
(item) => `<tr class="bg-slate-900/70">
<td class="px-3 py-2 font-mono text-slate-100 truncate" title="${escapeHtml(item.type || "")}">${escapeHtml(item.type || "-")}</td>
<td class="px-3 py-2 text-right font-mono text-slate-300">${Number(item.count || 0).toLocaleString()}</td>
</tr>`,
"Heap object details are unavailable.",
);
const stacksEl = document.getElementById("profile-stacks-list");
if (stacksEl) {
const stacks = (summary.topStacks || []).slice(0, 5);
stacksEl.innerHTML = stacks.length
? stacks.map((entry) => {
const pct = Number.isFinite(Number(entry.percent)) && Number(entry.percent) > 0
? Number(entry.percent)
: (totalSamples > 0 ? (Number(entry.samples || 0) / totalSamples) * 100 : 0);
const stack = Array.isArray(entry.stack) ? entry.stack : [];
return `<div class="rounded-lg bg-slate-950 border border-slate-800 px-3 py-2">
<div class="flex items-center justify-between gap-3 text-xs">
<span class="font-mono text-cyan-300">${formatPercent(pct)}</span>
<span class="font-mono text-slate-500">${Number(entry.samples || 0)} samples</span>
</div>
<div class="mt-1 font-mono text-[11px] text-slate-300 leading-relaxed break-all">${escapeHtml(stack.join(" <- ") || "(runtime)")}</div>
</div>`;
}).join("")
: '<div class="rounded-lg bg-slate-950 border border-slate-800 px-3 py-3 text-xs text-slate-500 text-center">No stack samples were captured.</div>';
}
if (results) results.classList.remove("hidden");
}
async function runServerProfile() {
const btn = document.getElementById("server-profile-btn");
const durationEl = document.getElementById("server-profile-duration");
const durationMs = Number(durationEl?.value || 5000);
if (btn) {
btn.disabled = true;
btn.innerHTML = '<i class="fa-solid fa-spinner fa-spin"></i>Capturing...';
}
setProfileMessage(`Capturing server CPU and memory for ${formatMs(durationMs)}. Keep using the server while this runs.`, "busy");
try {
const res = await fetch("/api/settings/profile", {
method: "POST",
credentials: "include",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ durationMs }),
});
const data = await res.json().catch(() => ({}));
if (!res.ok) {
throw new Error(data.error || res.statusText || "Profile failed");
}
renderServerProfileResults(data);
const profilerError = data?.cpu?.profilerError;
setProfileMessage(
profilerError
? `Memory snapshot captured. CPU profiler was unavailable: ${profilerError}`
: "Profile complete. Hot functions and memory object types are listed below.",
profilerError ? "error" : "ok",
);
} catch (e) {
setProfileMessage(`Profile failed: ${e.message}`, "error");
} finally {
if (btn) {
btn.disabled = false;
btn.innerHTML = '<i class="fa-solid fa-gauge-high"></i>Run Profile';
}
}
}
function downloadServerCpuProfile() {
if (!latestServerCpuProfile) return;
const blob = new Blob([JSON.stringify(latestServerCpuProfile, null, 2)], { type: "application/json" });
const url = URL.createObjectURL(blob);
const a = document.createElement("a");
const stamp = new Date().toISOString().replace(/[:.]/g, "-");
a.href = url;
a.download = `overlord-server-${stamp}.cpuprofile`;
document.body.appendChild(a);
a.click();
a.remove();
URL.revokeObjectURL(url);
}
async function runGC() {
const gcBtn = document.getElementById("health-gc-btn");
const msgEl = document.getElementById("health-gc-message");
@@ -1952,8 +2137,12 @@ async function runGC() {
function initHealthHandlers() {
const refreshBtn = document.getElementById("health-refresh-btn");
const gcBtn = document.getElementById("health-gc-btn");
const profileBtn = document.getElementById("server-profile-btn");
const downloadProfileBtn = document.getElementById("server-profile-download-btn");
if (refreshBtn) refreshBtn.addEventListener("click", loadHealthStats);
if (gcBtn) gcBtn.addEventListener("click", runGC);
if (profileBtn) profileBtn.addEventListener("click", runServerProfile);
if (downloadProfileBtn) downloadProfileBtn.addEventListener("click", downloadServerCpuProfile);
}
async function init() {
+88
View File
@@ -1193,6 +1193,94 @@
</div>
</div>
<!-- Server profiler -->
<div class="rounded-lg bg-slate-900 border border-slate-800 p-4 space-y-4">
<div class="flex items-center justify-between gap-3 flex-wrap">
<div>
<p class="text-xs font-semibold uppercase tracking-wider text-slate-500 mb-1">Server Profiler</p>
<p class="text-sm text-slate-400">Capture a short CPU sample and memory snapshot to find hot functions and heavy allocations.</p>
</div>
<div class="flex items-center gap-2 flex-wrap">
<select id="server-profile-duration"
class="px-3 py-2 rounded-lg bg-slate-950 border border-slate-700 text-sm text-slate-200 focus:outline-none focus:ring-2 focus:ring-cyan-500">
<option value="3000">3 seconds</option>
<option value="5000" selected>5 seconds</option>
<option value="10000">10 seconds</option>
<option value="30000">30 seconds</option>
</select>
<button id="server-profile-btn" type="button"
class="inline-flex items-center gap-2 px-3 py-2 rounded-lg bg-cyan-700 hover:bg-cyan-600 border border-cyan-600 text-sm transition-colors text-white">
<i class="fa-solid fa-gauge-high"></i>Run Profile
</button>
<button id="server-profile-download-btn" type="button"
class="hidden inline-flex items-center gap-2 px-3 py-2 rounded-lg bg-slate-800 hover:bg-slate-700 border border-slate-700 text-sm transition-colors text-slate-300">
<i class="fa-solid fa-download"></i>CPU Profile
</button>
</div>
</div>
<p id="server-profile-message" class="hidden text-xs rounded-lg px-3 py-2 border"></p>
<div id="server-profile-results" class="hidden space-y-4">
<div class="grid grid-cols-2 sm:grid-cols-4 gap-3 text-sm">
<div class="rounded-lg bg-slate-950 border border-slate-800 px-3 py-2">
<div class="text-slate-400 text-xs mb-0.5">Captured</div>
<div id="profile-duration" class="font-medium font-mono">-</div>
</div>
<div class="rounded-lg bg-slate-950 border border-slate-800 px-3 py-2">
<div class="text-slate-400 text-xs mb-0.5">Process CPU</div>
<div id="profile-cpu-percent" class="font-medium font-mono">-</div>
</div>
<div class="rounded-lg bg-slate-950 border border-slate-800 px-3 py-2">
<div class="text-slate-400 text-xs mb-0.5">Samples</div>
<div id="profile-samples" class="font-medium font-mono">-</div>
</div>
<div class="rounded-lg bg-slate-950 border border-slate-800 px-3 py-2">
<div class="text-slate-400 text-xs mb-0.5">RSS Delta</div>
<div id="profile-rss-delta" class="font-medium font-mono">-</div>
</div>
</div>
<div class="grid grid-cols-1 xl:grid-cols-2 gap-4">
<div class="space-y-2">
<p class="text-xs font-semibold uppercase tracking-wider text-slate-500">Hot Functions</p>
<div class="overflow-x-auto rounded-lg border border-slate-800">
<table class="min-w-full text-xs">
<thead class="bg-slate-950 text-slate-400">
<tr>
<th class="text-left px-3 py-2 font-medium">Function</th>
<th class="text-right px-3 py-2 font-medium">Self</th>
<th class="text-right px-3 py-2 font-medium">Samples</th>
</tr>
</thead>
<tbody id="profile-functions-body" class="divide-y divide-slate-800"></tbody>
</table>
</div>
</div>
<div class="space-y-2">
<p class="text-xs font-semibold uppercase tracking-wider text-slate-500">Memory Object Types</p>
<div class="overflow-x-auto rounded-lg border border-slate-800">
<table class="min-w-full text-xs">
<thead class="bg-slate-950 text-slate-400">
<tr>
<th class="text-left px-3 py-2 font-medium">Type</th>
<th class="text-right px-3 py-2 font-medium">Count</th>
</tr>
</thead>
<tbody id="profile-objects-body" class="divide-y divide-slate-800"></tbody>
</table>
</div>
</div>
</div>
<div class="space-y-2">
<p class="text-xs font-semibold uppercase tracking-wider text-slate-500">Hottest Stacks</p>
<div id="profile-stacks-list" class="space-y-2"></div>
</div>
</div>
</div>
<p id="health-gc-message" class="hidden text-xs rounded-lg px-3 py-2 border border-emerald-700 bg-emerald-900/20 text-emerald-300"></p>
</div>
</section>
@@ -15,6 +15,10 @@ import {
stopProxy,
} from "../socks5-proxy-manager";
let activeServerProfile: Promise<any> | null = null;
let jscRuntimePromise: Promise<any> | null | undefined;
let inspectorRuntimePromise: Promise<any> | null | undefined;
type MiscRouteDeps = {
CORS_HEADERS: Record<string, string>;
SERVER_VERSION: string;
@@ -27,6 +31,274 @@ type MiscRouteDeps = {
tlsSource?: "certbot" | "configured" | "self-signed";
};
function clampProfileDuration(value: unknown): number {
const parsed = Number(value);
if (!Number.isFinite(parsed)) return 5000;
return Math.max(1000, Math.min(30000, Math.floor(parsed)));
}
function sleep(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}
function readProcessCpuUsage(previous?: NodeJS.CpuUsage): NodeJS.CpuUsage | null {
const cpuUsage = (process as any).cpuUsage;
if (typeof cpuUsage !== "function") return null;
return previous ? cpuUsage(previous) : cpuUsage();
}
function readResourceUsage(): any | null {
const resourceUsage = (process as any).resourceUsage;
if (typeof resourceUsage !== "function") return null;
try {
return resourceUsage();
} catch {
return null;
}
}
async function loadJscRuntime(): Promise<any | null> {
if (jscRuntimePromise === undefined) {
try {
const runtimeImport = new Function("specifier", "return import(specifier)") as (specifier: string) => Promise<any>;
jscRuntimePromise = runtimeImport("bun:jsc").catch(() => null);
} catch {
jscRuntimePromise = null;
}
}
return jscRuntimePromise;
}
async function loadInspectorRuntime(): Promise<any | null> {
if (inspectorRuntimePromise === undefined) {
try {
const runtimeImport = new Function("specifier", "return import(specifier)") as (specifier: string) => Promise<any>;
inspectorRuntimePromise = runtimeImport("node:inspector").catch(() => null);
} catch {
inspectorRuntimePromise = null;
}
}
return inspectorRuntimePromise;
}
async function summarizeRuntimeMemory() {
const mem = process.memoryUsage();
let jscHeap: any = null;
let jscMemory: any = null;
try {
const jsc = await loadJscRuntime();
if (typeof jsc?.heapStats === "function") jscHeap = jsc.heapStats();
if (typeof jsc?.memoryUsage === "function") jscMemory = jsc.memoryUsage();
} catch { }
const objectTypeCounts = jscHeap?.objectTypeCounts && typeof jscHeap.objectTypeCounts === "object"
? Object.entries(jscHeap.objectTypeCounts)
.map(([type, count]) => ({ type, count: Number(count) || 0 }))
.filter((entry) => entry.count > 0)
.sort((a, b) => b.count - a.count)
.slice(0, 20)
: [];
const activeResourcesInfo = typeof (process as any).getActiveResourcesInfo === "function"
? (process as any).getActiveResourcesInfo()
: [];
const activeResources = Array.isArray(activeResourcesInfo)
? Object.entries(activeResourcesInfo.reduce((acc: Record<string, number>, name: string) => {
acc[name] = (acc[name] || 0) + 1;
return acc;
}, {})).map(([type, count]) => ({ type, count }))
: [];
return {
process: {
rss: mem.rss,
heapTotal: mem.heapTotal,
heapUsed: mem.heapUsed,
external: mem.external,
arrayBuffers: (mem as any).arrayBuffers ?? 0,
},
jscHeap: jscHeap ? {
heapSize: jscHeap.heapSize,
heapCapacity: jscHeap.heapCapacity,
extraMemorySize: jscHeap.extraMemorySize,
objectCount: jscHeap.objectCount,
protectedObjectCount: jscHeap.protectedObjectCount,
globalObjectCount: jscHeap.globalObjectCount,
topObjectTypes: objectTypeCounts,
} : null,
jscMemory,
resources: activeResources.sort((a: any, b: any) => b.count - a.count).slice(0, 20),
components: {
thumbnails: getThumbnailStats(),
clients: {
inMemory: getClientCount(),
online: getOnlineClients().length,
},
database: {
fileSizeBytes: getDatabaseFileSizeBytes(),
},
},
};
}
function normalizeProfileUrl(url: string): string {
if (!url) return "(runtime)";
const clean = url.replace(/^file:\/\//, "").replaceAll("\\", "/");
const parts = clean.split("/");
return parts.slice(-3).join("/");
}
function summarizeCpuProfile(profile: any, durationMs: number) {
const nodes = Array.isArray(profile?.nodes) ? profile.nodes : [];
const samples = Array.isArray(profile?.samples) ? profile.samples : [];
const timeDeltas = Array.isArray(profile?.timeDeltas) ? profile.timeDeltas : [];
const nodeById = new Map<number, any>();
const parentById = new Map<number, number>();
for (const node of nodes) {
if (typeof node?.id !== "number") continue;
nodeById.set(node.id, node);
if (Array.isArray(node.children)) {
for (const childId of node.children) {
if (typeof childId === "number") parentById.set(childId, node.id);
}
}
}
const functionTotals = new Map<string, any>();
const moduleTotals = new Map<string, any>();
const stackTotals = new Map<string, any>();
let totalUs = 0;
samples.forEach((nodeId: number, index: number) => {
const node = nodeById.get(nodeId);
if (!node) return;
const deltaUs = Number(timeDeltas[index]) || 0;
totalUs += deltaUs;
const frame = node.callFrame || {};
const name = frame.functionName || "(anonymous)";
const moduleName = normalizeProfileUrl(frame.url || "");
const location = frame.url
? `${moduleName}:${(Number(frame.lineNumber) || 0) + 1}`
: moduleName;
const key = `${name}\n${location}`;
const current = functionTotals.get(key) || { name, location, samples: 0, selfTimeUs: 0 };
current.samples += 1;
current.selfTimeUs += deltaUs;
functionTotals.set(key, current);
const moduleCurrent = moduleTotals.get(moduleName) || { module: moduleName, samples: 0, selfTimeUs: 0 };
moduleCurrent.samples += 1;
moduleCurrent.selfTimeUs += deltaUs;
moduleTotals.set(moduleName, moduleCurrent);
const stackFrames: string[] = [];
let cursor: number | undefined = nodeId;
while (typeof cursor === "number" && stackFrames.length < 8) {
const stackNode = nodeById.get(cursor);
if (!stackNode) break;
const stackFrame = stackNode.callFrame || {};
const stackName = stackFrame.functionName || "(anonymous)";
stackFrames.push(`${stackName} @ ${normalizeProfileUrl(stackFrame.url || "")}`);
cursor = parentById.get(cursor);
}
const stackKey = stackFrames.join(" <- ");
const stackCurrent = stackTotals.get(stackKey) || { stack: stackFrames, samples: 0, selfTimeUs: 0 };
stackCurrent.samples += 1;
stackCurrent.selfTimeUs += deltaUs;
stackTotals.set(stackKey, stackCurrent);
});
if (!totalUs && samples.length > 0) totalUs = durationMs * 1000;
const decorate = (item: any) => ({
...item,
selfTimeMs: item.selfTimeUs / 1000,
percent: totalUs > 0 ? (item.selfTimeUs / totalUs) * 100 : 0,
});
return {
totalSamples: samples.length,
totalTimeMs: totalUs / 1000,
topFunctions: Array.from(functionTotals.values()).map(decorate).sort((a, b) => b.samples - a.samples).slice(0, 25),
topModules: Array.from(moduleTotals.values()).map(decorate).sort((a, b) => b.samples - a.samples).slice(0, 12),
topStacks: Array.from(stackTotals.values()).map(decorate).sort((a, b) => b.samples - a.samples).slice(0, 10),
};
}
async function runInspectorCpuProfile(durationMs: number) {
const inspector = await loadInspectorRuntime();
const Session = (inspector as any).Session;
if (typeof Session !== "function") {
throw new Error("Runtime inspector profiler is unavailable");
}
const session = new Session();
const post = (method: string, params?: Record<string, unknown>) => new Promise<any>((resolve, reject) => {
session.post(method, params || {}, (error: Error | null, result: any) => {
if (error) reject(error);
else resolve(result);
});
});
session.connect();
try {
await post("Profiler.enable");
await post("Profiler.setSamplingInterval", { interval: 1000 });
await post("Profiler.start");
await sleep(durationMs);
const result = await post("Profiler.stop");
await post("Profiler.disable").catch(() => undefined);
return result?.profile || null;
} finally {
session.disconnect();
}
}
async function collectServerProfile(durationMs: number) {
const startedAt = Date.now();
const beforeCpu = readProcessCpuUsage();
const beforeResource = readResourceUsage();
const beforeMemory = await summarizeRuntimeMemory();
let cpuProfile: any = null;
let profilerError: string | null = null;
try {
cpuProfile = await runInspectorCpuProfile(durationMs);
} catch (error: any) {
profilerError = String(error?.message || error || "CPU profiler failed");
await sleep(durationMs);
}
const elapsedMs = Date.now() - startedAt;
const cpuDelta = beforeCpu ? readProcessCpuUsage(beforeCpu) : null;
const afterResource = readResourceUsage();
const afterMemory = await summarizeRuntimeMemory();
const cpuTotalUs = cpuDelta ? (cpuDelta.user + cpuDelta.system) : 0;
return {
ok: true,
startedAt,
durationMs: elapsedMs,
requestedDurationMs: durationMs,
cpu: {
userUs: cpuDelta?.user ?? null,
systemUs: cpuDelta?.system ?? null,
totalUs: cpuTotalUs || null,
processPercent: elapsedMs > 0 && cpuTotalUs > 0 ? (cpuTotalUs / (elapsedMs * 1000)) * 100 : null,
resourceBefore: beforeResource,
resourceAfter: afterResource,
profilerError,
summary: cpuProfile ? summarizeCpuProfile(cpuProfile, elapsedMs) : null,
rawProfile: cpuProfile,
},
memory: {
before: beforeMemory,
after: afterMemory,
},
};
}
export async function handleMiscRoutes(
req: Request,
url: URL,
@@ -807,5 +1079,35 @@ export async function handleMiscRoutes(
return Response.json({ ok: true, freedBytes: Math.max(0, before - after) });
}
// POST /api/settings/profile
if (req.method === "POST" && url.pathname === "/api/settings/profile") {
const user = await authenticateRequest(req);
if (!user) return new Response("Unauthorized", { status: 401 });
if (user.role !== "admin") return new Response("Forbidden", { status: 403 });
if (activeServerProfile) {
return Response.json({ error: "A server profile capture is already running." }, { status: 409 });
}
let body: any = {};
try { body = await req.json(); } catch { body = {}; }
const durationMs = clampProfileDuration(body?.durationMs);
activeServerProfile = collectServerProfile(durationMs);
try {
const result = await activeServerProfile;
logAudit({
timestamp: Date.now(),
username: user.username,
ip: deps.requestIP?.(req)?.address || "unknown",
action: AuditAction.COMMAND,
details: `Captured server profile (${durationMs}ms)`,
success: true,
});
return Response.json(result, { headers: deps.CORS_HEADERS });
} finally {
activeServerProfile = null;
}
}
return null;
}