Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

9.5 KB
Raw Blame
import { readdirSync, readFileSync } from 'node:fs'
import { constants, performance, PerformanceObserver } from 'node:perf_hooks'
import type { FastifyInstance } from 'fastify'
import { logger } from '../utils/logger.js'

/**
 * Where this process's CPU goes, by thread (Watch Production gap 2, #2801). On 11-04 a pod-minute had 43.9 s of process CPU
 * and `cpu by kind` (lib/selfProfile.ts) named 15.3 s of it: that log sees the main thread only, and ~40% ran off it.
 * Linux names each thread, so one read of `/proc/self/task/<tid>/stat` per thread a minute splits it with nothing per
 * request. Measured on Node 22.23 with Prisma 5.22 (library engine, in-process), arm64 Ubuntu:
 *
 * - the main thread's tid is the pid;
 * - `tokio-runtime-w` (and its tracing exporter `opentelemetry-e`) are the Prisma query engine's: it runs each query,
 *   encodes and decodes BSON and builds the JSON string the client then parses;
 * - `libuv-worker` is the libuv pool: zlib, fs, crypto, dns lookups;
 * - `node` threads other than the main one, and `DelayedTaskSche`, are V8's platform workers: concurrent GC and JIT.
 *
 * `cpu by thread`, every minute, always on: process CPU, each bucket's CPU, `exitedMs` (threads that ended inside the
 * window, known only by the process total) and `namedPct`, the four named buckets' share. On the main thread, two
 * counters say what part of it is GC pauses (a `gc` PerformanceObserver) and reply JSON (registerReplyJsonTiming); the
 * rest of it is split by route in `cpu by kind`. Off Linux the thread split says `unavailable` and the rest still logs.
 */

export type ThreadBucket = 'main' | 'prismaEngine' | 'libuvPool' | 'v8Workers' | 'other'
export type ThreadCpu = { tid: number; name: string; cpuMs: number }

/** USER_HZ: the unit of /proc's CPU times, 100 on every Linux architecture the pods run on. */
const TICKS_PER_SECOND = 100

/** A thread's name and its user + system CPU in ms, from its `/proc/<pid>/task/<tid>/stat` line. The name sits between the
 *  first `(` and the last `)` and may hold either; utime and stime are the 14th and 15th fields. */
export function parseThreadStat(text: string): { name: string; cpuMs: number } {
  const open = text.indexOf('('), close = text.lastIndexOf(')')
  if (open < 0 || close < open) throw new Error('not a stat line')
  // Fields after the name start at the 3rd (state): utime is the 14th, stime the 15th.
  const fields = text.slice(close + 2).split(' ')
  const ticks = Number(fields[11]) + Number(fields[12])
  if (!Number.isFinite(ticks)) throw new Error('stat line without CPU times')
  return { name: text.slice(open + 1, close), cpuMs: ticks * 1000 / TICKS_PER_SECOND }
}

export function threadBucket(name: string, tid: number, pid: number): ThreadBucket {
  if (tid === pid) return 'main'
  if (name.startsWith('tokio-runtime') || name.startsWith('opentelemetry')) return 'prismaEngine'
  if (name === 'libuv-worker') return 'libuvPool'
  if (name === 'node' || name.startsWith('DelayedTaskSche') || name.startsWith('V8')) return 'v8Workers'
  return 'other'
}

/** Every thread of this process with its CPU so far. A thread that exits between the listing and its read is skipped; an
 *  unreadable folder throws (not Linux, or /proc not mounted). */
export function readThreads(root = '/proc/self/task'): ThreadCpu[] {
  const threads: ThreadCpu[] = []
  for (const entry of readdirSync(root)) {
    let text: string
    try { text = readFileSync(`${root}/${entry}/stat`, 'utf8') } catch { continue }
    threads.push({ tid: Number(entry), ...parseThreadStat(text) })
  }
  return threads
}

export type CpuStats = {
  /** One reply's JSON: how long `JSON.stringify` took and the bytes it made. */
  replyJson(ms: number, bytes: number): void
  /** Logs the window now and starts the next one (the timer calls it every minute). */
  flush(): void
  stop(): void
}

export type CpuStatsOptions = {
  everyMs?: number
  log?: (message: string, context: Record<string, unknown>) => void
  pid?: number
  threads?: () => ThreadCpu[]
  cpuUsage?: () => NodeJS.CpuUsage
  /** The main thread's event-loop utilization since the last call, 0 to 1. */
  loopUtilization?: () => number
  /** Calls onGc for every main-thread GC pause; returns its stop. */
  observeGc?: (onGc: (durationMs: number, major: boolean) => void) => () => void
}

const MAX_NAMES = 10

function loopSinceLast(): () => number {
  let last = performance.eventLoopUtilization()
  return () => {
    const now = performance.eventLoopUtilization()
    const since = performance.eventLoopUtilization(now, last)
    last = now
    return since.utilization
  }
}

function gcObserver(onGc: (durationMs: number, major: boolean) => void): () => void {
  const observer = new PerformanceObserver(list => {
    for (const entry of list.getEntries()) {
      const kind = (entry as { detail?: { kind?: number } }).detail?.kind
      onGc(entry.duration, kind === constants.NODE_PERFORMANCE_GC_MAJOR)
    }
  })
  observer.observe({ entryTypes: ['gc'] })
  return () => observer.disconnect()
}

export function startCpuStats({ everyMs = 60_000, log = (message, context) => logger.info(message, context), pid = process.pid,
  threads = () => readThreads(), cpuUsage = () => process.cpuUsage(), loopUtilization = loopSinceLast(),
  observeGc = gcObserver }: CpuStatsOptions = {}): CpuStats {
  const toMs = (usage: NodeJS.CpuUsage) => (usage.user + usage.system) / 1000
  const read = (): { threads: ThreadCpu[] } | { error: string } => {
    try { return { threads: threads() } } catch (error) { return { error: error instanceof Error ? error.message : String(error) } }
  }
  const byTid = (snapshot: ReturnType<typeof read>) => new Map('threads' in snapshot ? snapshot.threads.map(t => [t.tid, t.cpuMs]) : [])
  let processMs = toMs(cpuUsage()), started = performance.now(), before = byTid(read())
  let gc = { pauseMs: 0, count: 0, majorMs: 0 }, json = { ms: 0, count: 0, bytes: 0 }
  let stopGc: (() => void) | undefined = observeGc((ms, major) => {
    if (!stopGc) return
    gc.pauseMs += ms; gc.count++
    if (major) gc.majorMs += ms
  })

  const flush = () => {
    const nowMs = toMs(cpuUsage()), processCpuMs = nowMs - processMs
    const context: Record<string, unknown> = { windowMs: Math.round(performance.now() - started), processCpuMs: Math.round(processCpuMs) }
    const current = read()
    if ('threads' in current) {
      const sums: Record<ThreadBucket, number> = { main: 0, prismaEngine: 0, libuvPool: 0, v8Workers: 0, other: 0 }
      const names = new Map<string, number>()
      let counted = 0
      for (const thread of current.threads) {
        // A thread new in the window has all its CPU in it.
        const delta = Math.max(0, thread.cpuMs - (before.get(thread.tid) ?? 0))
        const bucket = threadBucket(thread.name, thread.tid, pid)
        sums[bucket] += delta; counted += delta
        if (bucket === 'other' && delta > 0) names.set(thread.name, (names.get(thread.name) ?? 0) + delta)
      }
      const named = sums.main + sums.prismaEngine + sums.libuvPool + sums.v8Workers
      Object.assign(context, {
        mainMs: Math.round(sums.main), prismaEngineMs: Math.round(sums.prismaEngine), libuvPoolMs: Math.round(sums.libuvPool),
        v8WorkersMs: Math.round(sums.v8Workers), otherThreadsMs: Math.round(sums.other),
        // Threads that ended inside the window: in the process total, gone from /proc.
        exitedMs: Math.round(Math.max(0, processCpuMs - counted)),
        namedPct: processCpuMs > 0 ? Math.round(Math.min(named, processCpuMs) / processCpuMs * 100) : 0,
        otherNames: Object.fromEntries([...names].sort((a, b) => b[1] - a[1]).slice(0, MAX_NAMES).map(([name, ms]) => [name, Math.round(ms)])),
      })
    } else Object.assign(context, { threads: 'unavailable', reason: current.error })
    Object.assign(context, {
      gc: { pauseMs: Math.round(gc.pauseMs), count: gc.count, majorMs: Math.round(gc.majorMs) },
      replyJson: { ms: Math.round(json.ms), count: json.count, kb: Math.round(json.bytes / 1024) },
      loopBusyPct: Math.round(loopUtilization() * 100),
    })
    log('cpu by thread', context)
    processMs = nowMs; started = performance.now(); before = byTid(current)
    gc = { pauseMs: 0, count: 0, majorMs: 0 }; json = { ms: 0, count: 0, bytes: 0 }
  }

  const timer = setInterval(flush, everyMs)
  timer.unref()
  return {
    replyJson(ms, bytes) { json.ms += ms; json.count++; json.bytes += bytes },
    flush,
    stop() { clearInterval(timer); const stop = stopGc; stopGc = undefined; stop?.() },
  }
}

/**
 * Every object reply's JSON, timed: what `JSON.stringify` of big replies costs the main thread. Fastify's own serializer
 * for a route without a response schema is that same `JSON.stringify` (fastify/lib/reply.js serialize), so the bytes do
 * not change. A response schema would be bypassed by this serializer, so a route declaring one is refused at startup;
 * none has one. Replies sent as strings or buffers are not serialized and not counted. Size is in characters, not bytes,
 * to keep the count free.
 */
export function registerReplyJsonTiming(app: FastifyInstance, stats: Pick<CpuStats, 'replyJson'>): void {
  app.addHook('onRoute', route => {
    if (route.schema?.response) {
      throw new Error(`${String(route.method)} ${route.url}: a response schema would be bypassed by the timed reply serializer ` +
        '(lib/cpuStats.ts registerReplyJsonTiming); serialize in the handler instead')
    }
  })
  app.setReplySerializer(payload => {
    const start = performance.now()
    const text = JSON.stringify(payload)
    stats.replyJson(performance.now() - start, text.length)
    return text
  })
}