Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

19.3 KB
Raw Blame
import Fastify from 'fastify'
import { afterEach, describe, expect, it } from 'vitest'
import type { PrismaClient } from '@prisma/client'
import type { EngineCpu } from './engineCpu.js'
import { createQueryStats, enterWork, estimateBytes, filterShape, registerWorkContext, runWork, withQueryStats, workKind, type QueryStatsOptions } from './queryStats.js'

type Logged = { message: string; context: Record<string, unknown> }

function stats(options: Partial<QueryStatsOptions> = {}) {
  const infos: Logged[] = [], warns: Logged[] = []
  const s = createQueryStats({ pod: 'pod-1', log: (message, context) => infos.push({ message, context }),
    warn: (message, context) => warns.push({ message, context }), ...options })
  return { s, infos, warns }
}

type Row = { op: string; shape: string; route: string; n: number; ms: number; docs: number; bytes: number; errors: number; maxMs: number }
const rows = (logged: Logged) => logged.context.top as Row[]

it('totals every query of each route, however small, beside the top shapes', () => {
  const { s, infos } = stats({ sampleEvery: 1, top: { bytes: 1, count: 1, ms: 1 } })
  runWork('http GET /api/a', () => {
    s.record('Big', 'findMany', { where: { id: 'x' } }, 50, [{ text: 'x'.repeat(500) }])
    for (let i = 0; i < 5; i++) s.record('Small', 'findFirst', { where: { k: 'y' } }, 1, [])
  })
  runWork('http GET /api/b', () => s.record('Small', 'findFirst', { where: { k: 'z' } }, 2, []))
  s.flush()
  // The top lists keep one shape; the route totals keep every query.
  expect((infos[0]!.context.top as Row[]).length).toBeLessThan(3)
  expect(infos[0]!.context.routes).toEqual({
    'http GET /api/a': { n: 6, ms: 55, docs: 1, bytes: JSON.stringify([{ text: 'x'.repeat(500) }]).length + 5 * 2, ops: { 'Small.findFirst': 5, 'Big.findMany': 1 } },
    'http GET /api/b': { n: 1, ms: 2, docs: 0, bytes: 2, ops: { 'Small.findFirst': 1 } },
  })
})

describe('filterShape', () => {
  it('keeps the keys of a Prisma filter and replaces every value with ?', () => {
    const shape = filterShape('findMany', { where: { tabId: 'tab-secret-1', OR: [{ status: { in: ['open', 'held'] } }, { owner: { email: '[email protected]' } }] },
      select: { id: true }, take: 5 })
    expect(shape).toBe('{"OR":[{"status":{"in":"?"}},{"owner":{"email":"?"}}],"tabId":"?"}')
    expect(shape).not.toContain('tab-secret-1')
    expect(shape).not.toContain('alice')
  })

  it('never shows a value, even an object under a value operator or a string that looks like a field path', () => {
    const shape = filterShape('findFirst', { where: { meta: { equals: { private: 'note' } }, label: { not: '$secret' }, at: { gt: new Date(0) } } })
    expect(shape).toBe('{"at":{"gt":"?"},"label":{"not":"?"},"meta":{"equals":"?"}}')
  })

  it('hides keys that carry ids, emails or anything else that is not a plain name', () => {
    const shape = filterShape('$runCommandRaw', { update: 'SwarmGroup', updates: [{ q: { 'parts.a5e7c91b0462f5fa1a4a7f1b': 1, '[email protected]': 2, 'state.ok': 3, 'agent 7': 4 } }] })
    expect(shape).toBe('{"*":"?","parts.*":"?","state.ok":"?"}')
  })

  it('names the stages of a raw pipeline and the keys of its $match', () => {
    expect(filterShape('aggregateRaw', { pipeline: [{ $match: { tabId: 't1', archived: false } }, { $sort: { at: -1 } }, { $limit: 5 }] }))
      .toBe('[{"$match":{"archived":"?","tabId":"?"}},"$sort","$limit"]')
    expect(filterShape('findRaw', { filter: { machineIds: 'm1' } })).toBe('{"machineIds":"?"}')
    expect(filterShape('$runCommandRaw', { find: 'users', filter: { email: '[email protected]' } })).toBe('{"email":"?"}')
    expect(filterShape('$runCommandRaw', { aggregate: 'SwarmGroup', pipeline: [{ $match: { tabId: 'x' } }], cursor: {} })).toBe('[{"$match":{"tabId":"?"}}]')
    expect(filterShape('$runCommandRaw', { ping: 1 })).toBe('')
  })

  it('has an empty shape for writes without a filter', () => {
    expect(filterShape('create', { data: { email: '[email protected]' } })).toBe('')
    expect(filterShape('findMany', undefined)).toBe('')
  })

  it('stays bounded for deep or wide filters', () => {
    let deep: Record<string, unknown> = { leaf: 1 }
    for (let i = 0; i < 20; i++) deep = { AND: deep }
    expect(filterShape('findMany', { where: deep })).toContain('…')
    const wide = Object.fromEntries(Array.from({ length: 200 }, (_, i) => [`field${String.fromCharCode(97 + (i % 26))}${'x'.repeat(i % 7)}`, i]))
    expect(filterShape('findMany', { where: wide }).length).toBeLessThanOrEqual(301)
  })
})

describe('per-minute totals', () => {
  it('records count, ms, documents and bytes per model.action, shape and route, then starts a new window', () => {
    const { s, infos } = stats({ sampleEvery: 1 })
    runWork('http GET /api/users/:id', () => {
      s.record('User', 'findMany', { where: { email: '[email protected]' } }, 3, [{ id: '1', email: '[email protected]' }])
      s.record('User', 'findMany', { where: { email: '[email protected]' } }, 5, [{ id: '2', email: '[email protected]' }, { id: '3', email: '[email protected]' }])
    })
    s.record('User', 'findMany', { where: { email: '[email protected]' } }, 7, [])
    s.flush()
    expect(infos).toHaveLength(1)
    expect(infos[0]!.message).toBe('db by shape')
    const all = JSON.stringify(infos[0]!.context)
    expect(all).not.toContain('[email protected]')
    const byRoute = Object.fromEntries(rows(infos[0]!).map(r => [r.route, r]))
    expect(byRoute['http GET /api/users/:id']).toEqual({ op: 'User.findMany', shape: '{"email":"?"}', route: 'http GET /api/users/:id', n: 2, ms: 8, docs: 3,
      bytes: JSON.stringify([{ id: '1', email: '[email protected]' }]).length + JSON.stringify([{ id: '2', email: '[email protected]' }, { id: '3', email: '[email protected]' }]).length, errors: 0, maxMs: 5 })
    expect(byRoute.background).toMatchObject({ n: 1, ms: 7, docs: 0, bytes: 2 })
    expect(infos[0]!.context).toMatchObject({ pod: 'pod-1', queries: 3, ms: 15, docs: 3, keys: 2, overflowQueries: 0, slowSuppressed: 0 })
    s.flush()
    expect(infos[1]!.context).toMatchObject({ queries: 0, keys: 0, top: [] })
  })

  it('measures the bytes of every Nth result per key and scales them to the count', () => {
    const { s, infos } = stats({ sampleEvery: 4 })
    const big = [{ blob: 'x'.repeat(100) }], small = [{ blob: '' }]
    // Results 1 and 5 are measured (two bigs), the rest are not even serialized: 2 × big over 2 samples, times 8 queries.
    const results = [big, small, small, small, big, small, small, small]
    for (const r of results) s.record('SwarmGroup', 'findMany', { where: { tabId: 't' } }, 1, r)
    s.flush()
    expect(rows(infos[0]!)[0]!.bytes).toBe(JSON.stringify(big).length * 8)
  })

  it('counts a failed query as an error with its time and no bytes', () => {
    const { s, infos } = stats({ sampleEvery: 1 })
    s.record('Machine', 'update', { where: { machineId: 'm' } }, 4, undefined, true)
    s.flush()
    expect(rows(infos[0]!)[0]).toMatchObject({ op: 'Machine.update', n: 1, ms: 4, errors: 1, bytes: 0, docs: 0 })
  })

  it('counts a single-document result as one document and a raw cursor reply by its batch', () => {
    const { s, infos } = stats({ sampleEvery: 1 })
    s.record('User', 'findUnique', { where: { id: '1' } }, 1, { id: '1' })
    s.record('User', 'findUnique', { where: { id: '2' } }, 1, null)
    s.record(undefined, '$runCommandRaw', { aggregate: 'SwarmGroup', pipeline: [] }, 1, { cursor: { firstBatch: [{}, {}, {}] }, ok: 1 })
    s.flush()
    const byOp = Object.fromEntries(rows(infos[0]!).map(r => [r.op, r]))
    expect(byOp['User.findUnique']!.docs).toBe(1)
    expect(byOp['SwarmGroup.$runCommandRaw aggregate']!.docs).toBe(3)
  })

  it('keeps a bounded number of keys per window and counts the rest under other', () => {
    const { s, infos } = stats({ maxKeys: 3 })
    for (const model of ['A', 'B', 'C', 'D', 'E']) s.record(model, 'findMany', { where: { id: 'x' } }, 1, [])
    s.flush()
    expect(infos[0]!.context).toMatchObject({ queries: 5, keys: 3, overflowQueries: 3 })
    expect(rows(infos[0]!).map(r => r.op).sort()).toEqual(['A.findMany', 'B.findMany', 'other'])
  })

  it('logs the keys a window ranks highest by bytes, by count and by time', () => {
    const { s, infos } = stats({ sampleEvery: 1, top: { bytes: 1, count: 1, ms: 1 } })
    s.record('Big', 'findMany', {}, 1, [{ x: 'x'.repeat(500) }])
    s.record('Often', 'count', {}, 1, 1); s.record('Often', 'count', {}, 1, 1); s.record('Often', 'count', {}, 1, 1)
    s.record('Slow', 'findFirst', {}, 150, null)
    s.record('Quiet', 'findFirst', {}, 1, null)
    s.flush()
    // Ranked by bytes: Big (500+), Slow ("null", 4), Often ("1" three times, 3); Quiet makes no list.
    expect(rows(infos[0]!).map(r => r.op)).toEqual(['Big.findMany', 'Slow.findFirst', 'Often.count'])
  })
})

describe('slow queries', () => {
  it('warns about a query at or over the threshold with its shape and route, never its values', () => {
    const { s, warns } = stats({ slowMs: 200 })
    runWork('web-ws swarm_changes', () => s.record('SwarmGroup', 'findMany', { where: { tabId: 'tab-secret', title: { contains: 'payroll' } } }, 250, [{ title: 'payroll 2026' }]))
    s.record('SwarmGroup', 'findMany', { where: { tabId: 'quick' } }, 199, [])
    expect(warns).toHaveLength(1)
    expect(warns[0]).toEqual({ message: 'slow query', context: { pod: 'pod-1', op: 'SwarmGroup.findMany', shape: '{"tabId":"?","title":{"contains":"?"}}',
      route: 'web-ws swarm_changes', ms: 250, docs: 1, bytes: JSON.stringify([{ title: 'payroll 2026' }]).length, error: false } })
    expect(JSON.stringify(warns)).not.toMatch(/tab-secret|payroll/)
  })

  it('warns at most slowPerWindow times a window and reports how many it held back', () => {
    const { s, warns, infos } = stats({ slowMs: 10, slowPerWindow: 2 })
    for (let i = 0; i < 5; i++) s.record('A', 'findMany', {}, 20, [])
    s.record('A', 'findMany', {}, 9, [])
    s.flush()
    expect(warns).toHaveLength(2)
    // Every slow query's time counts, logged or held back: what an alert on slow seconds a minute sums across pods.
    expect(infos[0]!.context).toMatchObject({ slowSuppressed: 3, slowQueries: 5, slowMs: 100 })
    s.record('A', 'findMany', {}, 20, [])
    expect(warns).toHaveLength(3)
  })
})

describe('work context', () => {
  afterEach(() => enterWork(undefined))

  it('follows awaited work, and is background outside any', async () => {
    expect(workKind()).toBe('background')
    const seen = await runWork('http POST /api/x', async () => {
      await new Promise(resolve => setTimeout(resolve, 1))
      return workKind()
    })
    expect(seen).toBe('http POST /api/x')
    expect(workKind()).toBe('background')
  })

  it('can name the work running now and what it schedules', async () => {
    await new Promise<void>(resolve => setImmediate(() => {
      enterWork('manager-ws up')
      void Promise.resolve().then(() => { expect(workKind()).toBe('manager-ws up'); resolve() })
    }))
  })
})

describe('HTTP work context', () => {
  it('names a request by its route pattern in later hooks and in the handler, a JSON body included', async () => {
    const app = Fastify()
    registerWorkContext(app)
    const seen: string[] = []
    // Like the rate limit and sign-in: hooks added after it, which read the database too.
    app.addHook('onRequest', async () => { await new Promise(resolve => setTimeout(resolve, 1)); seen.push(`hook ${workKind()}`) })
    app.get('/api/items/:id', async () => { await new Promise(resolve => setImmediate(resolve)); seen.push(workKind()); return {} })
    app.post('/api/items', async request => { await Promise.resolve(); seen.push(`${workKind()} ${(request.body as { n: number }).n}`); return {} })
    await app.inject({ method: 'GET', url: '/api/items/alice-1' })
    // A body big enough to arrive in more than one chunk.
    await app.inject({ method: 'POST', url: '/api/items', payload: { n: 2, pad: 'x'.repeat(256 * 1024) } })
    expect(seen).toEqual(['hook http GET /api/items/:id', 'http GET /api/items/:id', 'hook http POST /api/items', 'http POST /api/items 2'])
    // A real socket, where the body's chunks arrive on the connection's own callbacks after the hooks ran.
    seen.length = 0
    const url = await app.listen({ port: 0, host: '127.0.0.1' })
    const sent = await fetch(`${url}/api/items`, { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ n: 3, pad: 'x'.repeat(256 * 1024) }) })
    expect(sent.status).toBe(200)
    expect(seen).toEqual(['hook http POST /api/items', 'http POST /api/items 3'])
    expect(workKind()).toBe('background')
    await app.close()
  })
})

describe('estimateBytes', () => {
  it('is the JSON length of an ordinary result', () => {
    const result = [{ id: 'a', n: 1.5, ok: true, no: false, at: new Date(0), none: null, skipped: undefined, tags: ['x', 'y'], nested: { deep: { v: -3 } } }, null]
    expect(estimateBytes(result)).toEqual({ bytes: JSON.stringify(result).length, capped: false })
    expect(estimateBytes(null)).toEqual({ bytes: 4, capped: false })
    expect(estimateBytes(undefined)).toEqual({ bytes: 0, capped: false })
  })

  it('counts a big page up to its budget only, and scales up from the documents it counted', () => {
    const doc = (i: number) => ({ id: `doc-${String(i).padStart(5, '0')}`, body: 'x'.repeat(1000) })
    const page = Array.from({ length: 10_000 }, (_, i) => doc(i))
    const real = JSON.stringify(page).length
    // Past the first 1,000 documents (~1 MB), reading a document throws: the estimate must stop before them.
    let touched = 0
    for (let i = 1000; i < page.length; i++) Object.defineProperty(page, i, { get: () => { touched++; throw new Error('read past the budget') } })
    const estimate = estimateBytes(page, 256 * 1024)
    expect(touched).toBe(0)
    expect(estimate.capped).toBe(true)
    expect(Math.abs(estimate.bytes - real) / real).toBeLessThan(0.01)
  })

  it('stops inside one huge document and gives at least the budget', () => {
    const huge: Record<string, unknown> = {}
    for (let i = 0; i < 5_000; i++) huge[`k${i}`] = 'x'.repeat(1000)
    let touched = 0
    for (let i = 1000; i < 5_000; i++) Object.defineProperty(huge, `k${i}`, { enumerable: true, get: () => { touched++; return 'x'.repeat(1000) } })
    const estimate = estimateBytes(huge, 256 * 1024)
    expect(touched).toBe(0)
    expect(estimate).toMatchObject({ capped: true })
    expect(estimate.bytes).toBeGreaterThanOrEqual(256 * 1024)
  })

  it('reports how many results a window estimated past the budget', () => {
    const { s, infos } = stats({ sampleEvery: 1, byteBudget: 1024 })
    s.record('SwarmGroup', 'findMany', {}, 1, Array.from({ length: 100 }, () => ({ body: 'x'.repeat(100) })))
    s.record('SwarmGroup', 'findMany', {}, 1, [])
    s.flush()
    expect(infos[0]!.context).toMatchObject({ bytesCapped: 1 })
  })
})

describe('the query hook', () => {
  type Call = { model?: string; operation: string; args: unknown; query: (args: unknown) => Promise<unknown> }
  /** A client whose only part is `$extends`: it keeps the hook so a test can call it as Prisma would. */
  function hooked(observe: (model: string, operation: string, seconds: number) => void) {
    let hook: ((call: Call) => Promise<unknown>) | undefined
    const client = { $extends: (extension: { query: { $allOperations: (call: Call) => Promise<unknown> } }) => { hook = extension.query.$allOperations; return client } }
    withQueryStats(client as unknown as PrismaClient, () => undefined, () => {}, observe)
    return (call: Call) => hook!(call)
  }

  it('times every query once into the duration histogram, a failed one and a raw command too (#1880)', async () => {
    const seen: Array<[string, string, number]> = []
    const run = hooked((model, operation, seconds) => { seen.push([model, operation, seconds]) })
    await expect(run({ model: 'User', operation: 'findFirst', args: { where: { email: '[email protected]' } }, query: async () => ({ id: 'u1' }) })).resolves.toEqual({ id: 'u1' })
    const failure = new Error('write conflict')
    await expect(run({ model: 'User', operation: 'update', args: {}, query: async () => { throw failure } })).rejects.toBe(failure)
    await run({ operation: '$runCommandRaw', args: { ping: 1 }, query: async () => ({ ok: 1 }) })
    expect(seen.map(([model, operation]) => [model, operation])).toEqual([['User', 'findFirst'], ['User', 'update'], ['raw', '$runCommandRaw']])
    for (const [, , seconds] of seen) expect(seconds).toBeGreaterThanOrEqual(0)
    expect(JSON.stringify(seen)).not.toContain('[email protected]')
  })

  it('keeps a query\'s result when the histogram throws', async () => {
    const run = hooked(() => { throw new Error('registry broke') })
    await expect(run({ model: 'User', operation: 'count', args: {}, query: async () => 3 })).resolves.toBe(3)
  })
})

describe('the query hook feeds the engine CPU sampler (#3041)', () => {
  type Call = { model?: string; operation: string; args: unknown; query: (args: unknown) => Promise<unknown> }
  function hookedEngine(engine: Partial<EngineCpu>) {
    let hook: ((call: Call) => Promise<unknown>) | undefined
    const client = { $extends: (extension: { query: { $allOperations: (call: Call) => Promise<unknown> } }) => { hook = extension.query.$allOperations; return client } }
    withQueryStats(client as unknown as PrismaClient, () => undefined, () => {}, undefined, undefined, () => engine as EngineCpu)
    return (call: Call) => hook!(call)
  }

  it('starts and ends one slot for every query, a failed one too, with what its start answered', async () => {
    const calls: unknown[][] = []
    const sample = { ns: 5, seq: 1 }
    const run = hookedEngine({ start: () => { calls.push(['start']); return sample }, end: (...args) => { calls.push(['end', ...args]) } })
    await run({ model: 'User', operation: 'findFirst', args: {}, query: async () => ({ id: 'u1' }) })
    const failure = new Error('write conflict')
    await expect(run({ model: 'User', operation: 'update', args: {}, query: async () => { throw failure } })).rejects.toBe(failure)
    expect(calls).toEqual([
      ['start'], ['end', sample, 'User', 'findFirst', {}, { id: 'u1' }],
      ['start'], ['end', sample, 'User', 'update', {}, undefined],
    ])
  })

  it('keeps a query\'s result when the sampler throws at its start or its end', async () => {
    const run = hookedEngine({ start: () => { throw new Error('start broke') }, end: () => { throw new Error('end broke') } })
    await expect(run({ model: 'User', operation: 'count', args: {}, query: async () => 3 })).resolves.toBe(3)
  })
})

describe('the minute handed on to the counters (#2957)', () => {
  it('hands every route of the minute to onWindow, not only the 40 the line lists', () => {
    const windows: Array<Map<string, { n: number; ms: number; bytes: number }>> = []
    const { s, infos } = stats({ sampleEvery: 1, onWindow: routes => { windows.push(new Map(routes)) } })
    for (let i = 0; i < 45; i++) runWork(`http GET /api/r${i}`, () => s.record('User', 'findFirst', { where: { k: 'y' } }, 1, [{ a: 1 }]))
    s.flush()
    expect(Object.keys(infos[0]!.context.routes as object)).toHaveLength(40)
    expect(windows[0]!.size).toBe(45)
    expect(windows[0]!.get('http GET /api/r44')).toMatchObject({ n: 1, ms: 1, bytes: JSON.stringify([{ a: 1 }]).length })
  })

  it('still logs its line, and warns, when onWindow throws', () => {
    const { s, infos, warns } = stats({ onWindow: () => { throw new Error('counter broke') } })
    runWork('http GET /api/a', () => s.record('User', 'count', {}, 1, 3))
    expect(() => s.flush()).not.toThrow()
    expect(infos).toHaveLength(1)
    expect(warns.map(w => w.message)).toContain('query stats window failed')
  })
})