The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
CPU logs: who calls each route, every query per route, the profile per route, and call counts (#2388)
0xken1516f74c7ccd91
diff --git a/backend/src/lib/queryStats.test.ts b/backend/src/lib/queryStats.test.tsindex ad5d350a1..f8497cfb1 100644--- a/backend/src/lib/queryStats.test.ts+++ b/backend/src/lib/queryStats.test.ts@@ -14,6 +14,22 @@ function stats(options: Partial<QueryStatsOptions> = {}) {type Row = { op: string; shape: string; route: string; n: number; ms: number; docs: number; bytes: number; errors: number; maxMs: number }const rows = (logged: Logged) => logged.context.top as Row[]+it('totals every query of each route, however small, beside the top shapes', () => {+ const { s, infos } = stats({ sampleEvery: 1, top: { bytes: 1, count: 1, ms: 1 } })+ runWork('http GET /api/a', () => {+ s.record('Big', 'findMany', { where: { id: 'x' } }, 50, [{ text: 'x'.repeat(500) }])+ for (let i = 0; i < 5; i++) s.record('Small', 'findFirst', { where: { k: 'y' } }, 1, [])+ })+ runWork('http GET /api/b', () => s.record('Small', 'findFirst', { where: { k: 'z' } }, 2, []))+ s.flush()+ // The top lists keep one shape; the route totals keep every query.+ expect((infos[0]!.context.top as Row[]).length).toBeLessThan(3)+ expect(infos[0]!.context.routes).toEqual({+ 'http GET /api/a': { n: 6, ms: 55, docs: 1, bytes: JSON.stringify([{ text: 'x'.repeat(500) }]).length + 5 * 2, ops: { 'Small.findFirst': 5, 'Big.findMany': 1 } },+ 'http GET /api/b': { n: 1, ms: 2, docs: 0, bytes: 2, ops: { 'Small.findFirst': 1 } },+ })+})+describe('filterShape', () => {it('keeps the keys of a Prisma filter and replaces every value with ?', () => {const shape = filterShape('findMany', { where: { tabId: 'tab-secret-1', OR: [{ status: { in: ['open', 'held'] } }, { owner: { email: '[email protected]' } }] },diff --git a/backend/src/lib/queryStats.ts b/backend/src/lib/queryStats.tsindex 381392c12..0382ed815 100644--- a/backend/src/lib/queryStats.ts+++ b/backend/src/lib/queryStats.ts@@ -274,9 +274,19 @@ export function createQueryStats({ pod = `${hostname()}:${process.pid}`, slowMsconst ranked = (by: (r: typeof rows[number]) => number, n: number) => [...rows].sort((a, b) => by(b) - by(a)).slice(0, n)const picked = new Set([...ranked(r => r.bytes, top.bytes), ...ranked(r => r.n, top.count), ...ranked(r => r.ms, top.ms)])const sum = (by: (r: typeof rows[number]) => number) => rows.reduce((total, r) => total + by(r), 0)+ // Every query of each route, small ones too: the top lists drop a cheap shape however often it runs (an empty poll's+ // read, 2026-10-31), so a route's own count would read short.+ const routes = new Map<string, { n: number; ms: number; docs: number; bytes: number; ops: Record<string, number> }>()+ for (const r of rows) {+ let total = routes.get(r.route)+ if (!total) routes.set(r.route, total = { n: 0, ms: 0, docs: 0, bytes: 0, ops: {} })+ total.n += r.n; total.ms += r.ms; total.docs += r.docs; total.bytes += r.bytes; total.ops[r.op] = (total.ops[r.op] ?? 0) + r.n+ }+ const byRoute = Object.fromEntries([...routes].sort((a, b) => b[1].ms - a[1].ms).slice(0, 40).map(([route, t]) =>+ [route, { ...t, ops: Object.fromEntries(Object.entries(t.ops).sort((a, b) => b[1] - a[1]).slice(0, 12)) }]))log('db by shape', { pod, windowMs: Math.round(performance.now() - started), queries: sum(r => r.n), ms: sum(r => r.ms), docs: sum(r => r.docs),bytes: sum(r => r.bytes), errors: sum(r => r.errors), keys: rows.length, overflowQueries, bytesCapped, slowQueries, slowMs: Math.round(slowTotalMs), slowSuppressed,- top: [...picked].sort((a, b) => b.bytes - a.bytes || b.n - a.n) })+ top: [...picked].sort((a, b) => b.bytes - a.bytes || b.n - a.n), routes: byRoute })entries = new Map(); overflowQueries = 0; slowLogged = 0; slowSuppressed = 0; slowQueries = 0; slowTotalMs = 0; bytesCapped = 0; started = performance.now()}diff --git a/backend/src/lib/routeLatency.test.ts b/backend/src/lib/routeLatency.test.tsindex 380ad535c..cad93b4f9 100644--- a/backend/src/lib/routeLatency.test.ts+++ b/backend/src/lib/routeLatency.test.ts@@ -62,3 +62,22 @@ it('names each request by its route pattern, never its URL', async () => {expect(JSON.stringify(logs)).not.toMatch(/alice|bob|hunter2|nowhere/)await app.close()})++it('counts each route by kind of client, from its user agent and whether it reads as an agent, never the agent itself', async () => {+ const app = Fastify()+ const logs: Logged[] = []+ const l = registerRouteLatency(app, { pod: 'pod-1', log: (message, context) => logs.push({ message, context }) })+ app.get('/api/items', async () => ({ ok: true }))+ const as = (ua: string, url = '/api/items') => app.inject({ method: 'GET', url, headers: { 'user-agent': ua } })+ await as('Mozilla/5.0 (Macintosh) Chrome/130')+ await as('Dart/3.5 (dart:io)')+ await as('Dart/3.5 (dart:io)')+ await as('node')+ await as('undici', '/api/items?asAgentId=agent-secret-7')+ await as('curl/8.4')+ l.flush()+ expect((logs[0]!.context.routes as Record<string, { clients: Record<string, number> }>)['GET /api/items']!.clients)+ .toEqual({ web: 1, app: 2, cli: 1, agent: 1, other: 1 })+ expect(JSON.stringify(logs)).not.toMatch(/agent-secret|curl|Chrome/)+ await app.close()+})diff --git a/backend/src/lib/routeLatency.ts b/backend/src/lib/routeLatency.tsindex 5445d1a68..def7155b7 100644--- a/backend/src/lib/routeLatency.ts+++ b/backend/src/lib/routeLatency.ts@@ -10,7 +10,20 @@ import { logger } from '../utils/logger.js'* percentiles; its count and max cover every request.*/-type Window = { n: number; max: number; samples: number[] }+type Window = { n: number; max: number; samples: number[]; clients: Record<string, number> }++/** Who sent a request, from what every client sends anyway (2026-10-31: who reads Swarm messages so often was unknown):+ * the web build's browser, the Desktop and phone apps' Dart client, the CLI daemon's Node client, and a read as an agent+ * (`asAgentId`), which only the CLI sends. Never the user agent itself, nor the agent. */+export type ClientKind = 'web' | 'app' | 'cli' | 'agent' | 'other'+export function clientKind(userAgent: string | undefined, asAgent: boolean): ClientKind {+ if (asAgent) return 'agent'+ if (!userAgent) return 'other'+ if (/^Mozilla\//.test(userAgent)) return 'web'+ if (/^Dart\//.test(userAgent)) return 'app'+ if (/^(node|undici)\b/.test(userAgent)) return 'cli'+ return 'other'+}export type RouteLatencyOptions = {pod?: string@@ -24,7 +37,7 @@ export type RouteLatencyOptions = {log?: (message: string, context: Record<string, unknown>) => void}-export type RouteLatency = { record(route: string, ms: number): void; flush(): void }+export type RouteLatency = { record(route: string, ms: number, client?: ClientKind): void; flush(): void }const OTHER = 'other'@@ -36,14 +49,15 @@ export function createRouteLatency({ pod = `${hostname()}:${process.pid}`, maxSalog = (m, c) => logger.info(m, c) }: RouteLatencyOptions = {}): RouteLatency {let routes = new Map<string, Window>(), started = performance.now()- const record = (route: string, ms: number) => {+ const record = (route: string, ms: number, client?: ClientKind) => {let window = routes.get(route)if (!window) {const key = routes.size >= maxRoutes - 1 ? OTHER : routewindow = routes.get(key)- if (!window) routes.set(key, window = { n: 0, max: 0, samples: [] })+ if (!window) routes.set(key, window = { n: 0, max: 0, samples: [], clients: {} })}window.n+++ if (client) window.clients[client] = (window.clients[client] ?? 0) + 1if (ms > window.max) window.max = msif (window.samples.length < maxSamples) window.samples.push(ms)else {@@ -56,7 +70,8 @@ export function createRouteLatency({ pod = `${hostname()}:${process.pid}`, maxSaconst rows = [...routes].map(([route, w]) => {const sorted = w.samples.sort((a, b) => a - b)const mean = sorted.reduce((total, ms) => total + ms, 0) / (sorted.length || 1)- return { route, totalMs: mean * w.n, stats: { n: w.n, p50: round(percentile(sorted, 50)), p95: round(percentile(sorted, 95)), max: round(w.max) } }+ return { route, totalMs: mean * w.n, stats: { n: w.n, p50: round(percentile(sorted, 50)), p95: round(percentile(sorted, 95)), max: round(w.max),+ ...(Object.keys(w.clients).length ? { clients: w.clients } : {}) } }}).sort((a, b) => b.totalMs - a.totalMs).slice(0, top)log('route latency', { pod, windowMs: Math.round(performance.now() - started), requests: [...routes.values()].reduce((total, w) => total + w.n, 0),routes: Object.fromEntries(rows.map(r => [r.route, r.stats])) })@@ -70,7 +85,8 @@ export function createRouteLatency({ pod = `${hostname()}:${process.pid}`, maxSaexport function registerRouteLatency(app: FastifyInstance, options: RouteLatencyOptions = {}): RouteLatency {const latency = createRouteLatency(options)app.addHook('onResponse', (request, reply, done) => {- latency.record(`${request.method} ${request.routeOptions.url ?? 'unrouted'}`, reply.elapsedTime)+ const asAgent = typeof (request.query as Record<string, unknown> | undefined)?.asAgentId === 'string'+ latency.record(`${request.method} ${request.routeOptions.url ?? 'unrouted'}`, reply.elapsedTime, clientKind(request.headers['user-agent'], asAgent))done()})return latencydiff --git a/backend/src/lib/selfProfile.test.ts b/backend/src/lib/selfProfile.test.tsindex 5c32ef7f5..51c58dbb7 100644--- a/backend/src/lib/selfProfile.test.ts+++ b/backend/src/lib/selfProfile.test.ts@@ -1,6 +1,6 @@import Fastify from 'fastify'import { afterEach, expect, it } from 'vitest'-import { frameKind, markCpuKind, selfProfileFromEnv, startSelfProfile, topSelfTime } from './selfProfile.js'+import { frameKind, markCpuKind, selfProfileFromEnv, startSelfProfile, topCalls, topSelfTime } from './selfProfile.js'import { enterWork, workKind } from './queryStats.js'let stop = () => {}@@ -111,3 +111,31 @@ it('names the work for the query stats too, with the profile off', async () => {expect(seen).toBe('web-ws swarm_changes')enterWork(undefined)})++it('splits the profile by the route that ran each sample, from the work kinds marked while it ran', () => {+ const app = (path: string) => `file://${process.cwd()}/${path}`+ const at = (id: number, functionName: string, url: string, children: number[] = [], lineNumber = 0) => ({ id, callFrame: { functionName, url, lineNumber }, children })+ const top = topSelfTime({+ startTime: 0,+ nodes: [at(1, '(root)', '', [2, 4]), at(2, 'changes', app('dist/read/service.js'), [3], 9), at(3, 'structuredClone', 'node:internal/worker/js_transferable', [], 111),+ at(4, 'observeMachine', app('dist/superTabs/store.js'), [5], 99), at(5, 'structuredClone', 'node:internal/worker/js_transferable', [], 111)],+ // Samples at 1, 2, 3, 4 and 5 ms, each 1 ms long.+ samples: [3, 3, 5, 5, 5], timeDeltas: [1000, 1000, 1000, 1000, 1000, 1000],+ }, { start: 0, marks: [[0, 'http GET /changes'], [2.5, 'http POST /machines']] })+ expect(top.byRoute).toEqual({+ 'http POST /machines': { sampledMs: 3, inclusive: { 'observeMachine dist/superTabs/store.js:100': 3 },+ callers: { 'structuredClone node:internal/worker/js_transferable:112 ← observeMachine dist/superTabs/store.js:100': 3 } },+ 'http GET /changes': { sampledMs: 2, inclusive: { 'changes dist/read/service.js:10': 2 },+ callers: { 'structuredClone node:internal/worker/js_transferable:112 ← changes dist/read/service.js:10': 2 } },+ })+})++it('counts calls of our functions from precise coverage, an anonymous one by its place, and nothing outside our code', () => {+ const app = (path: string) => `file://${process.cwd()}/${path}`+ const calls = topCalls([+ { url: app('dist/superTabs/model.js'), functions: [{ functionName: 'stableId', ranges: [{ startOffset: 10, count: 900 }] },+ { functionName: '', ranges: [{ startOffset: 4021, count: 50 }] }, { functionName: 'idle', ranges: [{ startOffset: 90, count: 0 }] }] },+ { url: 'file:///app/node_modules/@prisma/client/runtime/library.js', functions: [{ functionName: 'parse', ranges: [{ startOffset: 1, count: 99999 }] }] },+ ])+ expect(calls).toEqual({ 'stableId dist/superTabs/model.js': 900, '(anonymous) dist/superTabs/model.js@4021': 50 })+})diff --git a/backend/src/lib/selfProfile.ts b/backend/src/lib/selfProfile.tsindex ce1bab12e..fe27013e9 100644--- a/backend/src/lib/selfProfile.ts+++ b/backend/src/lib/selfProfile.ts@@ -31,7 +31,16 @@ export function markCpuKind(kind: string): void {if (!enabled) returnconst resource = executionAsyncResource() as Taggedif (resource) resource[KIND] = kind- account(); current = kind+ account(); switchTo(kind)+}++/** While a profile runs, when each kind of work began, so its samples can be split by route (`byRoute`). Bounded: past+ * MAX_MARKS a window stops marking and its later samples count under the last kind. */+let marks: Array<[number, string]> | null = null+const MAX_MARKS = 1_000_000+function switchTo(kind: string | undefined): void {+ current = kind+ if (marks && marks.length < MAX_MARKS) marks.push([performance.now(), kind ?? '(idle)'])}function account(): void {@@ -74,8 +83,8 @@ export function startSelfProfile({ everyMs = 15 * 60_000, profileMs = 30_000, reconst kind = inherited ?? (type === 'Timeout' ? `job ${timer || 'anonymous'}` : undefined)if (kind) (resource as Tagged)[KIND] = kind},- before() { account(); current = (executionAsyncResource() as Tagged)?.[KIND] ?? 'untagged' },- after() { account(); current = undefined },+ before() { account(); switchTo((executionAsyncResource() as Tagged)?.[KIND] ?? 'untagged') },+ after() { account(); switchTo(undefined) },}).enable()let cpu = process.cpuUsage(), wall = performance.now()@@ -88,18 +97,33 @@ export function startSelfProfile({ everyMs = 15 * 60_000, profileMs = 30_000, re}, reportMs)report.unref()+ // One profile at a time: a second would replace the first one's marks (and V8 runs one coverage session per process).+ let profiling = falseconst profile = async () => {+ if (profiling) return+ profiling = trueconst session = new Session()session.connect()try {await session.post('Profiler.enable')+ // Call counts for the same window: ~2.6% CPU while on (messages + changes load on a copy of prod's busiest Swarm).+ await session.post('Profiler.startPreciseCoverage', { callCount: true, detailed: false })+ const start = performance.now()+ const recorded: Array<[number, string]> = [[start, current ?? '(idle)']]+ marks = recordedawait session.post('Profiler.start')await new Promise(resolve => setTimeout(resolve, profileMs).unref())const { profile: p } = await session.post('Profiler.stop')- log('cpu profile', { profileMs, ...topSelfTime(p) })+ marks = null+ const kinds = { start, marks: recorded }+ const { result: coverage } = await session.post('Profiler.takePreciseCoverage')+ await session.post('Profiler.stopPreciseCoverage')+ log('cpu profile', { profileMs, ...topSelfTime(p, kinds), calls: topCalls(coverage) })} catch (error) {logger.warn('cpu profile failed', { error: error instanceof Error ? error.message : String(error) })} finally {+ marks = null+ profiling = falsesession.disconnect()}}@@ -109,7 +133,10 @@ export function startSelfProfile({ everyMs = 15 * 60_000, profileMs = 30_000, re}type Frame = { functionName: string; url: string; lineNumber: number }-type Profile = { nodes: { id: number; callFrame: Frame; children?: number[] }[]; samples?: number[]; timeDeltas?: number[] }+type Profile = { nodes: { id: number; callFrame: Frame; children?: number[] }[]; samples?: number[]; timeDeltas?: number[]; startTime?: number }+/** When each kind of work began during the profile, in performance.now() ms, from `start` (the profile's own start). */+type Kinds = { start: number; marks: ReadonlyArray<readonly [number, string]> }+type RouteProfile = { sampledMs: number; inclusive: Record<string, number>; callers: Record<string, number> }/*** The profile by self time (`files`, `functions`), and what the self time alone cannot tell, the stacks being dropped:@@ -117,11 +144,13 @@ type Profile = { nodes: { id: number; callFrame: Frame; children?: number[] }[];* above it, as `<function> ← <our function>`;* - `inclusive`: each function of ours by the time spent in it and everything it called, counted once per sample however* deep it recurses.+ * - `byRoute` (with `kinds`): the same two for each kind of work that ran a sample (an HTTP route, a frame, a job), so a+ * built-in's cost is known per route, not only per caller.* Ours is what runs from `dist/` or `src/`; V8's own entries (idle, program, garbage collector) have no stack and count in- * neither.+ * none of them.*/-export function topSelfTime(profile: Profile): { sampledMs: number; files: Record<string, number>; functions: Record<string, number>- callers: Record<string, number>; inclusive: Record<string, number> } {+export function topSelfTime(profile: Profile, kinds?: Kinds): { sampledMs: number; files: Record<string, number>; functions: Record<string, number>+ callers: Record<string, number>; inclusive: Record<string, number>; byRoute?: Record<string, RouteProfile> } {const byId = new Map(profile.nodes.map(n => [n.id, n.callFrame]))const parent = new Map<number, number>()for (const node of profile.nodes) for (const child of node.children ?? []) parent.set(child, node.id)@@ -129,10 +158,12 @@ export function topSelfTime(profile: Profile): { sampledMs: number; files: Recorconst ourName = new Map(profile.nodes.filter(n => isOurs(n.callFrame.url)).map(n => [n.id, name(n.callFrame)]))const files = new Map<string, number>(), functions = new Map<string, number>(), callers = new Map<string, number>(), inclusive = new Map<string, number>()const add = (m: Map<string, number>, key: string, us: number) => m.set(key, (m.get(key) ?? 0) + us)- let total = 0+ let total = 0, at = 0, mark = 0+ const routes = new Map<string, { us: number; callers: Map<string, number>; inclusive: Map<string, number> }>()const samples = profile.samples ?? [], deltas = profile.timeDeltas ?? []for (let i = 0; i < samples.length; i++) {const frame = byId.get(samples[i]!), us = deltas[i + 1] ?? 0+ at += deltas[i] ?? 0if (!frame) continuetotal += usconst file = shortFile(frame.url) || `(${frame.functionName || 'native'})`@@ -150,10 +181,43 @@ export function topSelfTime(profile: Profile): { sampledMs: number; files: Recor}if (nearest && !ourName.has(samples[i]!)) add(callers, `${fn} ← ${nearest}`, us)for (const key of ours) add(inclusive, key, us)+ if (!kinds?.marks.length) continue+ // Samples come in time order, so the kind that ran each one is found by walking the marks forward once.+ const when = kinds.start + at / 1000+ while (mark + 1 < kinds.marks.length && kinds.marks[mark + 1]![0] <= when) mark+++ const kind = kinds.marks[mark]![1]+ let route = routes.get(kind)+ if (!route) routes.set(kind, route = { us: 0, callers: new Map(), inclusive: new Map() })+ route.us += us+ if (nearest && !ourName.has(samples[i]!)) add(route.callers, `${fn} ← ${nearest}`, us)+ for (const key of ours) add(route.inclusive, key, us)}const top = (m: Map<string, number>, n: number) =>Object.fromEntries([...m].sort((a, b) => b[1] - a[1]).slice(0, n).map(([k, us]) => [k, Math.round(us / 1000)]))- return { sampledMs: Math.round(total / 1000), files: top(files, 20), functions: top(functions, 30), callers: top(callers, 30), inclusive: top(inclusive, 40) }+ const byRoute = kinds ? Object.fromEntries([...routes].sort((a, b) => b[1].us - a[1].us).slice(0, 15).map(([kind, r]) =>+ [kind, { sampledMs: Math.round(r.us / 1000), inclusive: top(r.inclusive, 10), callers: top(r.callers, 10) }])) : undefined+ return { sampledMs: Math.round(total / 1000), files: top(files, 20), functions: top(functions, 30), callers: top(callers, 30), inclusive: top(inclusive, 40),+ ...(byRoute ? { byRoute } : {}) }+}++type Coverage = Array<{ url: string; functions: Array<{ functionName: string; ranges: Array<{ startOffset: number; count: number }> }> }>++/** How many times each function of ours ran in the window (V8 precise coverage), most first. A sampling profile tells time,+ * never counts; a function called millions of times for little each (a filter callback, a hash) shows here. An anonymous+ * function is named by its file and character offset. */+export function topCalls(coverage: Coverage, n = 30): Record<string, number> {+ const calls = new Map<string, number>()+ for (const script of coverage) {+ if (!isOurs(script.url)) continue+ const file = shortFile(script.url)+ for (const fn of script.functions) {+ const range = fn.ranges[0]+ if (!range?.count) continue+ const key = fn.functionName ? `${fn.functionName} ${file}` : `(anonymous) ${file}@${range.startOffset}`+ calls.set(key, (calls.get(key) ?? 0) + range.count)+ }+ }+ return Object.fromEntries([...calls].sort((a, b) => b[1] - a[1]).slice(0, n))}const name = (frame: Frame) => {