Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

CPU logs: who calls each route, every query per route, the profile per route, and call counts (#2388)

0xken1516f74c7ccd91
diff --git a/backend/src/lib/queryStats.test.ts b/backend/src/lib/queryStats.test.ts
index ad5d350a1..f8497cfb1 100644
--- a/backend/src/lib/queryStats.test.ts
+++ b/backend/src/lib/queryStats.test.ts
@@ -14,6 +14,22 @@ function stats(options: Partial<QueryStatsOptions> = {}) {
type Row = { op: string; shape: string; route: string; n: number; ms: number; docs: number; bytes: number; errors: number; maxMs: number }
const rows = (logged: Logged) => logged.context.top as Row[]
+it('totals every query of each route, however small, beside the top shapes', () => {
+ const { s, infos } = stats({ sampleEvery: 1, top: { bytes: 1, count: 1, ms: 1 } })
+ runWork('http GET /api/a', () => {
+ s.record('Big', 'findMany', { where: { id: 'x' } }, 50, [{ text: 'x'.repeat(500) }])
+ for (let i = 0; i < 5; i++) s.record('Small', 'findFirst', { where: { k: 'y' } }, 1, [])
+ })
+ runWork('http GET /api/b', () => s.record('Small', 'findFirst', { where: { k: 'z' } }, 2, []))
+ s.flush()
+ // The top lists keep one shape; the route totals keep every query.
+ expect((infos[0]!.context.top as Row[]).length).toBeLessThan(3)
+ expect(infos[0]!.context.routes).toEqual({
+ 'http GET /api/a': { n: 6, ms: 55, docs: 1, bytes: JSON.stringify([{ text: 'x'.repeat(500) }]).length + 5 * 2, ops: { 'Small.findFirst': 5, 'Big.findMany': 1 } },
+ 'http GET /api/b': { n: 1, ms: 2, docs: 0, bytes: 2, ops: { 'Small.findFirst': 1 } },
+ })
+})
+
describe('filterShape', () => {
it('keeps the keys of a Prisma filter and replaces every value with ?', () => {
const shape = filterShape('findMany', { where: { tabId: 'tab-secret-1', OR: [{ status: { in: ['open', 'held'] } }, { owner: { email: '[email protected]' } }] },
diff --git a/backend/src/lib/queryStats.ts b/backend/src/lib/queryStats.ts
index 381392c12..0382ed815 100644
--- a/backend/src/lib/queryStats.ts
+++ b/backend/src/lib/queryStats.ts
@@ -274,9 +274,19 @@ export function createQueryStats({ pod = `${hostname()}:${process.pid}`, slowMs
const ranked = (by: (r: typeof rows[number]) => number, n: number) => [...rows].sort((a, b) => by(b) - by(a)).slice(0, n)
const picked = new Set([...ranked(r => r.bytes, top.bytes), ...ranked(r => r.n, top.count), ...ranked(r => r.ms, top.ms)])
const sum = (by: (r: typeof rows[number]) => number) => rows.reduce((total, r) => total + by(r), 0)
+ // Every query of each route, small ones too: the top lists drop a cheap shape however often it runs (an empty poll's
+ // read, 2026-10-31), so a route's own count would read short.
+ const routes = new Map<string, { n: number; ms: number; docs: number; bytes: number; ops: Record<string, number> }>()
+ for (const r of rows) {
+ let total = routes.get(r.route)
+ if (!total) routes.set(r.route, total = { n: 0, ms: 0, docs: 0, bytes: 0, ops: {} })
+ total.n += r.n; total.ms += r.ms; total.docs += r.docs; total.bytes += r.bytes; total.ops[r.op] = (total.ops[r.op] ?? 0) + r.n
+ }
+ const byRoute = Object.fromEntries([...routes].sort((a, b) => b[1].ms - a[1].ms).slice(0, 40).map(([route, t]) =>
+ [route, { ...t, ops: Object.fromEntries(Object.entries(t.ops).sort((a, b) => b[1] - a[1]).slice(0, 12)) }]))
log('db by shape', { pod, windowMs: Math.round(performance.now() - started), queries: sum(r => r.n), ms: sum(r => r.ms), docs: sum(r => r.docs),
bytes: sum(r => r.bytes), errors: sum(r => r.errors), keys: rows.length, overflowQueries, bytesCapped, slowQueries, slowMs: Math.round(slowTotalMs), slowSuppressed,
- top: [...picked].sort((a, b) => b.bytes - a.bytes || b.n - a.n) })
+ top: [...picked].sort((a, b) => b.bytes - a.bytes || b.n - a.n), routes: byRoute })
entries = new Map(); overflowQueries = 0; slowLogged = 0; slowSuppressed = 0; slowQueries = 0; slowTotalMs = 0; bytesCapped = 0; started = performance.now()
}
diff --git a/backend/src/lib/routeLatency.test.ts b/backend/src/lib/routeLatency.test.ts
index 380ad535c..cad93b4f9 100644
--- a/backend/src/lib/routeLatency.test.ts
+++ b/backend/src/lib/routeLatency.test.ts
@@ -62,3 +62,22 @@ it('names each request by its route pattern, never its URL', async () => {
expect(JSON.stringify(logs)).not.toMatch(/alice|bob|hunter2|nowhere/)
await app.close()
})
+
+it('counts each route by kind of client, from its user agent and whether it reads as an agent, never the agent itself', async () => {
+ const app = Fastify()
+ const logs: Logged[] = []
+ const l = registerRouteLatency(app, { pod: 'pod-1', log: (message, context) => logs.push({ message, context }) })
+ app.get('/api/items', async () => ({ ok: true }))
+ const as = (ua: string, url = '/api/items') => app.inject({ method: 'GET', url, headers: { 'user-agent': ua } })
+ await as('Mozilla/5.0 (Macintosh) Chrome/130')
+ await as('Dart/3.5 (dart:io)')
+ await as('Dart/3.5 (dart:io)')
+ await as('node')
+ await as('undici', '/api/items?asAgentId=agent-secret-7')
+ await as('curl/8.4')
+ l.flush()
+ expect((logs[0]!.context.routes as Record<string, { clients: Record<string, number> }>)['GET /api/items']!.clients)
+ .toEqual({ web: 1, app: 2, cli: 1, agent: 1, other: 1 })
+ expect(JSON.stringify(logs)).not.toMatch(/agent-secret|curl|Chrome/)
+ await app.close()
+})
diff --git a/backend/src/lib/routeLatency.ts b/backend/src/lib/routeLatency.ts
index 5445d1a68..def7155b7 100644
--- a/backend/src/lib/routeLatency.ts
+++ b/backend/src/lib/routeLatency.ts
@@ -10,7 +10,20 @@ import { logger } from '../utils/logger.js'
* percentiles; its count and max cover every request.
*/
-type Window = { n: number; max: number; samples: number[] }
+type Window = { n: number; max: number; samples: number[]; clients: Record<string, number> }
+
+/** Who sent a request, from what every client sends anyway (2026-10-31: who reads Swarm messages so often was unknown):
+ * the web build's browser, the Desktop and phone apps' Dart client, the CLI daemon's Node client, and a read as an agent
+ * (`asAgentId`), which only the CLI sends. Never the user agent itself, nor the agent. */
+export type ClientKind = 'web' | 'app' | 'cli' | 'agent' | 'other'
+export function clientKind(userAgent: string | undefined, asAgent: boolean): ClientKind {
+ if (asAgent) return 'agent'
+ if (!userAgent) return 'other'
+ if (/^Mozilla\//.test(userAgent)) return 'web'
+ if (/^Dart\//.test(userAgent)) return 'app'
+ if (/^(node|undici)\b/.test(userAgent)) return 'cli'
+ return 'other'
+}
export type RouteLatencyOptions = {
pod?: string
@@ -24,7 +37,7 @@ export type RouteLatencyOptions = {
log?: (message: string, context: Record<string, unknown>) => void
}
-export type RouteLatency = { record(route: string, ms: number): void; flush(): void }
+export type RouteLatency = { record(route: string, ms: number, client?: ClientKind): void; flush(): void }
const OTHER = 'other'
@@ -36,14 +49,15 @@ export function createRouteLatency({ pod = `${hostname()}:${process.pid}`, maxSa
log = (m, c) => logger.info(m, c) }: RouteLatencyOptions = {}): RouteLatency {
let routes = new Map<string, Window>(), started = performance.now()
- const record = (route: string, ms: number) => {
+ const record = (route: string, ms: number, client?: ClientKind) => {
let window = routes.get(route)
if (!window) {
const key = routes.size >= maxRoutes - 1 ? OTHER : route
window = routes.get(key)
- if (!window) routes.set(key, window = { n: 0, max: 0, samples: [] })
+ if (!window) routes.set(key, window = { n: 0, max: 0, samples: [], clients: {} })
}
window.n++
+ if (client) window.clients[client] = (window.clients[client] ?? 0) + 1
if (ms > window.max) window.max = ms
if (window.samples.length < maxSamples) window.samples.push(ms)
else {
@@ -56,7 +70,8 @@ export function createRouteLatency({ pod = `${hostname()}:${process.pid}`, maxSa
const rows = [...routes].map(([route, w]) => {
const sorted = w.samples.sort((a, b) => a - b)
const mean = sorted.reduce((total, ms) => total + ms, 0) / (sorted.length || 1)
- return { route, totalMs: mean * w.n, stats: { n: w.n, p50: round(percentile(sorted, 50)), p95: round(percentile(sorted, 95)), max: round(w.max) } }
+ return { route, totalMs: mean * w.n, stats: { n: w.n, p50: round(percentile(sorted, 50)), p95: round(percentile(sorted, 95)), max: round(w.max),
+ ...(Object.keys(w.clients).length ? { clients: w.clients } : {}) } }
}).sort((a, b) => b.totalMs - a.totalMs).slice(0, top)
log('route latency', { pod, windowMs: Math.round(performance.now() - started), requests: [...routes.values()].reduce((total, w) => total + w.n, 0),
routes: Object.fromEntries(rows.map(r => [r.route, r.stats])) })
@@ -70,7 +85,8 @@ export function createRouteLatency({ pod = `${hostname()}:${process.pid}`, maxSa
export function registerRouteLatency(app: FastifyInstance, options: RouteLatencyOptions = {}): RouteLatency {
const latency = createRouteLatency(options)
app.addHook('onResponse', (request, reply, done) => {
- latency.record(`${request.method} ${request.routeOptions.url ?? 'unrouted'}`, reply.elapsedTime)
+ const asAgent = typeof (request.query as Record<string, unknown> | undefined)?.asAgentId === 'string'
+ latency.record(`${request.method} ${request.routeOptions.url ?? 'unrouted'}`, reply.elapsedTime, clientKind(request.headers['user-agent'], asAgent))
done()
})
return latency
diff --git a/backend/src/lib/selfProfile.test.ts b/backend/src/lib/selfProfile.test.ts
index 5c32ef7f5..51c58dbb7 100644
--- a/backend/src/lib/selfProfile.test.ts
+++ b/backend/src/lib/selfProfile.test.ts
@@ -1,6 +1,6 @@
import Fastify from 'fastify'
import { afterEach, expect, it } from 'vitest'
-import { frameKind, markCpuKind, selfProfileFromEnv, startSelfProfile, topSelfTime } from './selfProfile.js'
+import { frameKind, markCpuKind, selfProfileFromEnv, startSelfProfile, topCalls, topSelfTime } from './selfProfile.js'
import { enterWork, workKind } from './queryStats.js'
let stop = () => {}
@@ -111,3 +111,31 @@ it('names the work for the query stats too, with the profile off', async () => {
expect(seen).toBe('web-ws swarm_changes')
enterWork(undefined)
})
+
+it('splits the profile by the route that ran each sample, from the work kinds marked while it ran', () => {
+ const app = (path: string) => `file://${process.cwd()}/${path}`
+ const at = (id: number, functionName: string, url: string, children: number[] = [], lineNumber = 0) => ({ id, callFrame: { functionName, url, lineNumber }, children })
+ const top = topSelfTime({
+ startTime: 0,
+ nodes: [at(1, '(root)', '', [2, 4]), at(2, 'changes', app('dist/read/service.js'), [3], 9), at(3, 'structuredClone', 'node:internal/worker/js_transferable', [], 111),
+ at(4, 'observeMachine', app('dist/superTabs/store.js'), [5], 99), at(5, 'structuredClone', 'node:internal/worker/js_transferable', [], 111)],
+ // Samples at 1, 2, 3, 4 and 5 ms, each 1 ms long.
+ samples: [3, 3, 5, 5, 5], timeDeltas: [1000, 1000, 1000, 1000, 1000, 1000],
+ }, { start: 0, marks: [[0, 'http GET /changes'], [2.5, 'http POST /machines']] })
+ expect(top.byRoute).toEqual({
+ 'http POST /machines': { sampledMs: 3, inclusive: { 'observeMachine dist/superTabs/store.js:100': 3 },
+ callers: { 'structuredClone node:internal/worker/js_transferable:112 ← observeMachine dist/superTabs/store.js:100': 3 } },
+ 'http GET /changes': { sampledMs: 2, inclusive: { 'changes dist/read/service.js:10': 2 },
+ callers: { 'structuredClone node:internal/worker/js_transferable:112 ← changes dist/read/service.js:10': 2 } },
+ })
+})
+
+it('counts calls of our functions from precise coverage, an anonymous one by its place, and nothing outside our code', () => {
+ const app = (path: string) => `file://${process.cwd()}/${path}`
+ const calls = topCalls([
+ { url: app('dist/superTabs/model.js'), functions: [{ functionName: 'stableId', ranges: [{ startOffset: 10, count: 900 }] },
+ { functionName: '', ranges: [{ startOffset: 4021, count: 50 }] }, { functionName: 'idle', ranges: [{ startOffset: 90, count: 0 }] }] },
+ { url: 'file:///app/node_modules/@prisma/client/runtime/library.js', functions: [{ functionName: 'parse', ranges: [{ startOffset: 1, count: 99999 }] }] },
+ ])
+ expect(calls).toEqual({ 'stableId dist/superTabs/model.js': 900, '(anonymous) dist/superTabs/model.js@4021': 50 })
+})
diff --git a/backend/src/lib/selfProfile.ts b/backend/src/lib/selfProfile.ts
index ce1bab12e..fe27013e9 100644
--- a/backend/src/lib/selfProfile.ts
+++ b/backend/src/lib/selfProfile.ts
@@ -31,7 +31,16 @@ export function markCpuKind(kind: string): void {
if (!enabled) return
const resource = executionAsyncResource() as Tagged
if (resource) resource[KIND] = kind
- account(); current = kind
+ account(); switchTo(kind)
+}
+
+/** While a profile runs, when each kind of work began, so its samples can be split by route (`byRoute`). Bounded: past
+ * MAX_MARKS a window stops marking and its later samples count under the last kind. */
+let marks: Array<[number, string]> | null = null
+const MAX_MARKS = 1_000_000
+function switchTo(kind: string | undefined): void {
+ current = kind
+ if (marks && marks.length < MAX_MARKS) marks.push([performance.now(), kind ?? '(idle)'])
}
function account(): void {
@@ -74,8 +83,8 @@ export function startSelfProfile({ everyMs = 15 * 60_000, profileMs = 30_000, re
const kind = inherited ?? (type === 'Timeout' ? `job ${timer || 'anonymous'}` : undefined)
if (kind) (resource as Tagged)[KIND] = kind
},
- before() { account(); current = (executionAsyncResource() as Tagged)?.[KIND] ?? 'untagged' },
- after() { account(); current = undefined },
+ before() { account(); switchTo((executionAsyncResource() as Tagged)?.[KIND] ?? 'untagged') },
+ after() { account(); switchTo(undefined) },
}).enable()
let cpu = process.cpuUsage(), wall = performance.now()
@@ -88,18 +97,33 @@ export function startSelfProfile({ everyMs = 15 * 60_000, profileMs = 30_000, re
}, reportMs)
report.unref()
+ // One profile at a time: a second would replace the first one's marks (and V8 runs one coverage session per process).
+ let profiling = false
const profile = async () => {
+ if (profiling) return
+ profiling = true
const session = new Session()
session.connect()
try {
await session.post('Profiler.enable')
+ // Call counts for the same window: ~2.6% CPU while on (messages + changes load on a copy of prod's busiest Swarm).
+ await session.post('Profiler.startPreciseCoverage', { callCount: true, detailed: false })
+ const start = performance.now()
+ const recorded: Array<[number, string]> = [[start, current ?? '(idle)']]
+ marks = recorded
await session.post('Profiler.start')
await new Promise(resolve => setTimeout(resolve, profileMs).unref())
const { profile: p } = await session.post('Profiler.stop')
- log('cpu profile', { profileMs, ...topSelfTime(p) })
+ marks = null
+ const kinds = { start, marks: recorded }
+ const { result: coverage } = await session.post('Profiler.takePreciseCoverage')
+ await session.post('Profiler.stopPreciseCoverage')
+ log('cpu profile', { profileMs, ...topSelfTime(p, kinds), calls: topCalls(coverage) })
} catch (error) {
logger.warn('cpu profile failed', { error: error instanceof Error ? error.message : String(error) })
} finally {
+ marks = null
+ profiling = false
session.disconnect()
}
}
@@ -109,7 +133,10 @@ export function startSelfProfile({ everyMs = 15 * 60_000, profileMs = 30_000, re
}
type Frame = { functionName: string; url: string; lineNumber: number }
-type Profile = { nodes: { id: number; callFrame: Frame; children?: number[] }[]; samples?: number[]; timeDeltas?: number[] }
+type Profile = { nodes: { id: number; callFrame: Frame; children?: number[] }[]; samples?: number[]; timeDeltas?: number[]; startTime?: number }
+/** When each kind of work began during the profile, in performance.now() ms, from `start` (the profile's own start). */
+type Kinds = { start: number; marks: ReadonlyArray<readonly [number, string]> }
+type RouteProfile = { sampledMs: number; inclusive: Record<string, number>; callers: Record<string, number> }
/**
* The profile by self time (`files`, `functions`), and what the self time alone cannot tell, the stacks being dropped:
@@ -117,11 +144,13 @@ type Profile = { nodes: { id: number; callFrame: Frame; children?: number[] }[];
* above it, as `<function> ← <our function>`;
* - `inclusive`: each function of ours by the time spent in it and everything it called, counted once per sample however
* deep it recurses.
+ * - `byRoute` (with `kinds`): the same two for each kind of work that ran a sample (an HTTP route, a frame, a job), so a
+ * built-in's cost is known per route, not only per caller.
* Ours is what runs from `dist/` or `src/`; V8's own entries (idle, program, garbage collector) have no stack and count in
- * neither.
+ * none of them.
*/
-export function topSelfTime(profile: Profile): { sampledMs: number; files: Record<string, number>; functions: Record<string, number>
- callers: Record<string, number>; inclusive: Record<string, number> } {
+export function topSelfTime(profile: Profile, kinds?: Kinds): { sampledMs: number; files: Record<string, number>; functions: Record<string, number>
+ callers: Record<string, number>; inclusive: Record<string, number>; byRoute?: Record<string, RouteProfile> } {
const byId = new Map(profile.nodes.map(n => [n.id, n.callFrame]))
const parent = new Map<number, number>()
for (const node of profile.nodes) for (const child of node.children ?? []) parent.set(child, node.id)
@@ -129,10 +158,12 @@ export function topSelfTime(profile: Profile): { sampledMs: number; files: Recor
const ourName = new Map(profile.nodes.filter(n => isOurs(n.callFrame.url)).map(n => [n.id, name(n.callFrame)]))
const files = new Map<string, number>(), functions = new Map<string, number>(), callers = new Map<string, number>(), inclusive = new Map<string, number>()
const add = (m: Map<string, number>, key: string, us: number) => m.set(key, (m.get(key) ?? 0) + us)
- let total = 0
+ let total = 0, at = 0, mark = 0
+ const routes = new Map<string, { us: number; callers: Map<string, number>; inclusive: Map<string, number> }>()
const samples = profile.samples ?? [], deltas = profile.timeDeltas ?? []
for (let i = 0; i < samples.length; i++) {
const frame = byId.get(samples[i]!), us = deltas[i + 1] ?? 0
+ at += deltas[i] ?? 0
if (!frame) continue
total += us
const file = shortFile(frame.url) || `(${frame.functionName || 'native'})`
@@ -150,10 +181,43 @@ export function topSelfTime(profile: Profile): { sampledMs: number; files: Recor
}
if (nearest && !ourName.has(samples[i]!)) add(callers, `${fn} ← ${nearest}`, us)
for (const key of ours) add(inclusive, key, us)
+ if (!kinds?.marks.length) continue
+ // Samples come in time order, so the kind that ran each one is found by walking the marks forward once.
+ const when = kinds.start + at / 1000
+ while (mark + 1 < kinds.marks.length && kinds.marks[mark + 1]![0] <= when) mark++
+ const kind = kinds.marks[mark]![1]
+ let route = routes.get(kind)
+ if (!route) routes.set(kind, route = { us: 0, callers: new Map(), inclusive: new Map() })
+ route.us += us
+ if (nearest && !ourName.has(samples[i]!)) add(route.callers, `${fn} ← ${nearest}`, us)
+ for (const key of ours) add(route.inclusive, key, us)
}
const top = (m: Map<string, number>, n: number) =>
Object.fromEntries([...m].sort((a, b) => b[1] - a[1]).slice(0, n).map(([k, us]) => [k, Math.round(us / 1000)]))
- return { sampledMs: Math.round(total / 1000), files: top(files, 20), functions: top(functions, 30), callers: top(callers, 30), inclusive: top(inclusive, 40) }
+ const byRoute = kinds ? Object.fromEntries([...routes].sort((a, b) => b[1].us - a[1].us).slice(0, 15).map(([kind, r]) =>
+ [kind, { sampledMs: Math.round(r.us / 1000), inclusive: top(r.inclusive, 10), callers: top(r.callers, 10) }])) : undefined
+ return { sampledMs: Math.round(total / 1000), files: top(files, 20), functions: top(functions, 30), callers: top(callers, 30), inclusive: top(inclusive, 40),
+ ...(byRoute ? { byRoute } : {}) }
+}
+
+type Coverage = Array<{ url: string; functions: Array<{ functionName: string; ranges: Array<{ startOffset: number; count: number }> }> }>
+
+/** How many times each function of ours ran in the window (V8 precise coverage), most first. A sampling profile tells time,
+ * never counts; a function called millions of times for little each (a filter callback, a hash) shows here. An anonymous
+ * function is named by its file and character offset. */
+export function topCalls(coverage: Coverage, n = 30): Record<string, number> {
+ const calls = new Map<string, number>()
+ for (const script of coverage) {
+ if (!isOurs(script.url)) continue
+ const file = shortFile(script.url)
+ for (const fn of script.functions) {
+ const range = fn.ranges[0]
+ if (!range?.count) continue
+ const key = fn.functionName ? `${fn.functionName} ${file}` : `(anonymous) ${file}@${range.startOffset}`
+ calls.set(key, (calls.get(key) ?? 0) + range.count)
+ }
+ }
+ return Object.fromEntries([...calls].sort((a, b) => b[1] - a[1]).slice(0, n))
}
const name = (frame: Frame) => {