Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
10.2 KB
Raw Blame
import { beforeEach, describe, expect, it, vi } from 'vitest'
import type * as k8s from '@kubernetes/client-node'
import { readFileSync, readdirSync } from 'node:fs'
import { join } from 'node:path'
import ts from 'typescript'
import { CREATE_KINDS, createsTotal, registry } from './metrics'
import { applyNetworkPolicy, observeCreate } from './utils'

async function count(kind: string, outcome: string): Promise<number> {
  const snapshot = await createsTotal.get()
  const sample = snapshot.values.find(v => v.labels.kind === kind && v.labels.outcome === outcome)
  expect(sample, `Missing ${kind}/${outcome} sample`).toBeDefined()
  return sample!.value
}

it('initializes create samples in the real registry before any test seeding', async () => {
  expect(registry.getSingleMetric('clerum_hcc_creates_total')).toBe(createsTotal)
  const snapshot = await createsTotal.get()
  expect(snapshot.values).toHaveLength(CREATE_KINDS.length * 4)
  expect(new Set(snapshot.values.map(sample => sample.labels.outcome))).toEqual(
    new Set(['created', 'conflict', 'error', 'skipped'])
  )
  for (const kind of CREATE_KINDS) {
    for (const outcome of ['created', 'conflict', 'error', 'skipped']) {
      expect(await count(kind, outcome)).toBe(0)
    }
  }
})

describe('Kubernetes create instrumentation', () => {
  beforeEach(() => {
    createsTotal.reset()
    for (const kind of CREATE_KINDS) {
      for (const outcome of ['created', 'conflict', 'error', 'skipped']) {
        createsTotal.inc({ kind, outcome }, 0)
      }
    }
  })

  it('instruments the complete production create inventory with the matching kind', () => {
    const kinds: string[] = []
    const root = __dirname
    for (const relative of readdirSync(root, { recursive: true }) as string[]) {
      if (
        !relative.endsWith('.ts') ||
        relative.endsWith('.test.ts') ||
        relative.includes('__tests__')
      )
        continue
      const file = ts.createSourceFile(
        relative,
        readFileSync(join(root, relative), 'utf8'),
        ts.ScriptTarget.Latest,
        true
      )
      const visit = (node: ts.Node): void => {
        if (ts.isCallExpression(node) && ts.isPropertyAccessExpression(node.expression)) {
          const match = /^create(?:Namespaced|Cluster)(.+)$/.exec(node.expression.name.text)
          if (match) {
            kinds.push(match[1])
            const arrow = node.parent
            expect(ts.isArrowFunction(arrow), `${relative}: ${node.expression.name.text}`).toBe(
              true
            )
            const wrapper = arrow.parent
            expect(ts.isCallExpression(wrapper)).toBe(true)
            if (ts.isCallExpression(wrapper)) {
              expect(wrapper.expression.getText(file)).toBe('observeCreate')
              expect(wrapper.arguments[0].getText(file)).toBe(`'${match[1]}'`)
            }
          }
        }
        ts.forEachChild(node, visit)
      }
      visit(file)
    }
    expect(kinds).toHaveLength(24)
    expect([...new Set(kinds)].sort()).toEqual([...CREATE_KINDS].sort())
  })

  it('preserves a synchronous client failure and counts the attempt', async () => {
    const error = new Error('client request construction failed')
    const create = vi.fn(() => {
      throw error
    })
    await expect(observeCreate('Service', create)).rejects.toBe(error)
    expect(create).toHaveBeenCalledTimes(1)
    expect(await count('Service', 'error')).toBe(1)
    expect(await count('Service', 'created')).toBe(0)
    expect(await count('Service', 'conflict')).toBe(0)
    expect(await count('Service', 'skipped')).toBe(0)
  })

  it('preserves real registry samples and nonzero counts across re-import', async () => {
    // Existing observations must survive module initialization without being reset.
    const create = vi.fn().mockResolvedValue({})
    await observeCreate('Service', create)
    expect(create).toHaveBeenCalledTimes(1)
    vi.resetModules()
    const reloaded = await import('./metrics')
    expect(reloaded.registry).toBe(registry)
    expect(reloaded.createsTotal).toBe(createsTotal)
    const exposition = await registry.metrics()
    for (const kind of CREATE_KINDS) {
      for (const outcome of ['created', 'conflict', 'error', 'skipped']) {
        expect(exposition).toContain(`clerum_hcc_creates_total{kind="${kind}",outcome="${outcome}"`)
        expect(await count(kind, outcome)).toBe(kind === 'Service' && outcome === 'created' ? 1 : 0)
      }
    }
  })

  it('counts a resolved create once and returns the original server response', async () => {
    const response = { metadata: { uid: 'created-policy', resourceVersion: '17' } }
    const create = vi.fn(async () => {
      expect(await count('NetworkPolicy', 'created')).toBe(0)
      return response
    })
    expect(await observeCreate('NetworkPolicy', create)).toBe(response)
    expect(create).toHaveBeenCalledTimes(1)
    expect(await count('NetworkPolicy', 'created')).toBe(1)
    expect(await count('NetworkPolicy', 'conflict')).toBe(0)
    expect(await count('NetworkPolicy', 'error')).toBe(0)
    expect(await count('NetworkPolicy', 'skipped')).toBe(0)
  })

  it('partitions settled callbacks without counting operations still in flight', async () => {
    const response = { metadata: { uid: 'settled-create' } }
    const conflict = { code: 409 }
    const forbidden = { code: 403 }
    let resolveCreate!: (value: typeof response) => void
    let rejectConflict!: (reason: unknown) => void
    let rejectForbidden!: (reason: unknown) => void
    const success = new Promise<typeof response>(resolve => {
      resolveCreate = resolve
    })
    const conflicting = new Promise<never>((_resolve, reject) => {
      rejectConflict = reject
    })
    const denied = new Promise<never>((_resolve, reject) => {
      rejectForbidden = reject
    })
    const create = vi
      .fn()
      .mockReturnValueOnce(success)
      .mockReturnValueOnce(conflicting)
      .mockReturnValueOnce(denied)
    const observed = [
      observeCreate('Service', create),
      observeCreate('Service', create),
      observeCreate('Service', create),
    ]
    // Attach rejection handlers immediately, including if a mutant resolves early.
    const settled = Promise.allSettled(observed)
    const values = () =>
      Promise.all(
        ['created', 'conflict', 'error', 'skipped'].map(outcome => count('Service', outcome))
      )
    expect(create).toHaveBeenCalledTimes(3)
    expect(await values()).toEqual([0, 0, 0, 0])

    resolveCreate(response)
    expect(await observed[0]).toBe(response)
    expect(await values()).toEqual([1, 0, 0, 0])

    const conflictCheck = expect(observed[1]).rejects.toBe(conflict)
    rejectConflict(conflict)
    await conflictCheck
    expect(await values()).toEqual([1, 1, 0, 0])

    const forbiddenCheck = expect(observed[2]).rejects.toBe(forbidden)
    rejectForbidden(forbidden)
    await forbiddenCheck
    expect(await values()).toEqual([1, 1, 1, 0])
    expect((await settled).map(result => result.status)).toEqual([
      'fulfilled',
      'rejected',
      'rejected',
    ])
    const completed = await Promise.all(
      ['created', 'conflict', 'error'].map(outcome => count('Service', outcome))
    )
    expect(completed.reduce((sum, value) => sum + value, 0)).toBe(create.mock.calls.length)
  })

  // Minimal fixtures isolate getErrorCode's supported layouts: ApiException.code
  // from the installed client and response.statusCode for compatibility.
  it.each([{ code: 409 }, { response: { statusCode: 409 } }])(
    'counts only conflict and preserves the original rejection: %j',
    async error => {
      const create = vi.fn().mockRejectedValue(error)
      await expect(observeCreate('Service', create)).rejects.toBe(error)
      expect(create).toHaveBeenCalledTimes(1)
      expect(await count('Service', 'created')).toBe(0)
      expect(await count('Service', 'error')).toBe(0)
      expect(await count('Service', 'conflict')).toBe(1)
      expect(await count('Service', 'skipped')).toBe(0)
    }
  )

  it.each([
    { code: 403 },
    { response: { statusCode: 403 } },
    { code: 500 },
    { code: 'ECONNRESET' },
    null,
    undefined,
  ])('propagates non-conflict errors without retrying or counting a conflict: %j', async error => {
    const create = vi.fn().mockRejectedValue(error)
    await expect(observeCreate('Deployment', create)).rejects.toBe(error)
    expect(create).toHaveBeenCalledTimes(1)
    expect(await count('Deployment', 'error')).toBe(1)
    expect(await count('Deployment', 'created')).toBe(0)
    expect(await count('Deployment', 'conflict')).toBe(0)
    expect(await count('Deployment', 'skipped')).toBe(0)
  })

  it('counts a presence skip without POST or replacing an unchanged policy', async () => {
    const policy: k8s.V1NetworkPolicy = {
      kind: 'NetworkPolicy',
      metadata: { name: 'metrics-policy' },
      spec: { podSelector: {}, policyTypes: ['Ingress'], ingress: [] },
    }
    const api = {
      createNamespacedNetworkPolicy: vi.fn().mockRejectedValue({ code: 409 }),
      readNamespacedNetworkPolicy: vi.fn().mockResolvedValue(policy),
      replaceNamespacedNetworkPolicy: vi.fn(),
    }
    await applyNetworkPolicy(
      api as unknown as k8s.NetworkingV1Api,
      'metrics-policy',
      'test',
      policy
    )
    expect(api.createNamespacedNetworkPolicy).not.toHaveBeenCalled()
    expect(api.readNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
    expect(api.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
    expect(await count('NetworkPolicy', 'created')).toBe(0)
    expect(await count('NetworkPolicy', 'error')).toBe(0)
    expect(await count('NetworkPolicy', 'conflict')).toBe(0)
    expect(await count('NetworkPolicy', 'skipped')).toBe(1)
  })

  it('does not count a create when its mutation fence has already expired', async () => {
    const allowed = vi.fn(() => false)
    const create = vi.fn()
    await applyNetworkPolicy(
      { createNamespacedNetworkPolicy: create } as unknown as k8s.NetworkingV1Api,
      'metrics-policy',
      'test',
      {},
      '[test]',
      allowed
    )
    expect(allowed).toHaveBeenCalledTimes(1)
    expect(create).not.toHaveBeenCalled()
    expect(await count('NetworkPolicy', 'created')).toBe(0)
    expect(await count('NetworkPolicy', 'conflict')).toBe(0)
    expect(await count('NetworkPolicy', 'error')).toBe(0)
    expect(await count('NetworkPolicy', 'skipped')).toBe(0)
  })
})