Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
17.6 KB
Raw Blame
import { describe, expect, it, vi } from 'vitest'
import { ApiException } from '@kubernetes/client-node'
import type * as k8s from '@kubernetes/client-node'
import { registry } from '../metrics'
import {
  preserveServiceAssignedFields,
  replaceWithConflictRetry,
  serviceMatchesDesired,
} from '../utils'

async function readLabeledCounter(name: string, kind: string): Promise<number> {
  const metric = registry.getSingleMetric(name)
  if (!metric) throw new Error(`${name} is not registered`)
  const snapshot = await metric.get()
  return snapshot.values.find(entry => entry.labels.kind === kind)?.value ?? 0
}

async function readWritesTotal(kind: string): Promise<number> {
  return readLabeledCounter('clerum_hcc_writes_total', kind)
}

async function readWriteSkipsTotal(kind: string): Promise<number> {
  return readLabeledCounter('clerum_hcc_write_skips_total', kind)
}

function apiException(code: number): ApiException<unknown> {
  return new ApiException(code, 'test', {}, {})
}

const desired: k8s.V1Service = {
  apiVersion: 'v1',
  kind: 'Service',
  metadata: { name: 'svc', namespace: 'ns', labels: { app: 'svc' } },
  spec: { selector: { app: 'svc' }, ports: [{ name: 'http', port: 8080 }] },
}

const existing: k8s.V1Service = {
  apiVersion: 'v1',
  kind: 'Service',
  metadata: {
    name: 'svc',
    namespace: 'ns',
    labels: { app: 'svc' },
    resourceVersion: '9',
    uid: 'uid-svc',
  },
  spec: {
    type: 'ClusterIP',
    sessionAffinity: 'None',
    clusterIP: '10.96.14.7',
    selector: { app: 'svc' },
    ports: [{ name: 'http', port: 8080, protocol: 'TCP' }],
  },
  status: { loadBalancer: {} },
}

describe('replaceWithConflictRetry order and retry', () => {
  it('ORDER-1: validateExisting throws before isUpToDate is consulted', async () => {
    const replace = vi.fn()
    const isUpToDate = vi.fn(() => true)
    const validateExisting = vi.fn(() => {
      throw new Error('unsafe to replace')
    })

    await expect(
      replaceWithConflictRetry({
        description: 'Service "svc"',
        logPrefix: '[Test]',
        body: desired,
        mergeExisting: preserveServiceAssignedFields,
        isUpToDate,
        validateExisting,
        read: async () => existing,
        replace,
      })
    ).rejects.toThrow('unsafe to replace')

    expect(validateExisting).toHaveBeenCalledOnce()
    expect(isUpToDate).not.toHaveBeenCalled()
    expect(replace).not.toHaveBeenCalled()
  })

  it('RETRY-SVC-1: 409 then success threads the fresh resourceVersion', async () => {
    const firstRead: k8s.V1Service = {
      ...existing,
      metadata: { ...existing.metadata, resourceVersion: '9' },
    }
    const secondRead: k8s.V1Service = {
      ...existing,
      metadata: { ...existing.metadata, resourceVersion: '10' },
    }
    const read = vi.fn().mockResolvedValueOnce(firstRead).mockResolvedValueOnce(secondRead)
    const replace = vi.fn().mockRejectedValueOnce({ code: 409 }).mockResolvedValueOnce({})
    const log = vi.spyOn(console, 'log')
    try {
      await replaceWithConflictRetry({
        description: 'Service "svc"',
        logPrefix: '[Test]',
        body: desired,
        mergeExisting: preserveServiceAssignedFields,
        isUpToDate: () => false,
        read,
        replace,
      })

      expect(replace).toHaveBeenCalledTimes(2)
      expect(replace.mock.calls[0][0].metadata?.resourceVersion).toBe('9')
      expect(replace.mock.calls[1][0].metadata?.resourceVersion).toBe('10')
      const updated = log.mock.calls
        .map((call: unknown[]) => JSON.parse(String(call[0])))
        .filter(entry => entry.msg === 'Kubernetes resource updated')
      expect(updated).toEqual([
        expect.objectContaining({
          level: 'info',
          scope: '[Test]',
          description: 'Service "svc"',
          attempt: 2,
        }),
      ])
    } finally {
      log.mockRestore()
    }
  })

  it.each([
    { status: 403, form: 'ApiException', error: apiException(403) },
    { status: 403, form: 'statusCode', error: { response: { statusCode: 403 } } },
    { status: 500, form: 'ApiException', error: apiException(500) },
    { status: 500, form: 'statusCode', error: { response: { statusCode: 500 } } },
  ])('TOCTOU-NP-2: helper read $status ($form) still throws', async ({ error }) => {
    const replace = vi.fn()
    const validateExisting = vi.fn()
    const read = vi.fn().mockRejectedValue(error)

    await expect(
      replaceWithConflictRetry({
        description: 'Service "svc"',
        logPrefix: '[Test]',
        body: desired,
        mergeExisting: preserveServiceAssignedFields,
        isUpToDate: serviceMatchesDesired,
        validateExisting,
        read,
        replace,
      })
    ).rejects.toBe(error)

    expect(read).toHaveBeenCalledOnce()
    expect(validateExisting).not.toHaveBeenCalled()
    expect(replace).not.toHaveBeenCalled()
  })

  it.each([
    { form: 'ApiException', error: apiException(404) },
    { form: 'statusCode', error: { response: { statusCode: 404 } } },
  ])(
    'TOCTOU-NP-3: replace 409 then second read 404 ($form) returns without a third replace',
    async ({ error }) => {
      const read = vi.fn().mockResolvedValueOnce(existing).mockRejectedValueOnce(error)
      const replace = vi.fn().mockRejectedValueOnce({ code: 409 })
      const validateExisting = vi.fn()

      await expect(
        replaceWithConflictRetry({
          description: 'Service "svc"',
          logPrefix: '[Test]',
          body: desired,
          mergeExisting: preserveServiceAssignedFields,
          isUpToDate: () => false,
          validateExisting,
          read,
          replace,
        })
      ).resolves.toBe('missing')

      expect(read).toHaveBeenCalledTimes(2)
      expect(validateExisting).toHaveBeenCalledOnce()
      expect(replace).toHaveBeenCalledOnce()
      expect(replace.mock.calls[0][0].metadata?.resourceVersion).toBe('9')
    }
  )

  it('TOCTOU-NP-4: network Error without code is rethrown', async () => {
    const networkErr = new Error('socket hang up')
    const replace = vi.fn()
    const validateExisting = vi.fn()
    const read = vi.fn().mockRejectedValue(networkErr)

    await expect(
      replaceWithConflictRetry({
        description: 'Service "svc"',
        logPrefix: '[Test]',
        body: desired,
        mergeExisting: preserveServiceAssignedFields,
        isUpToDate: serviceMatchesDesired,
        validateExisting,
        read,
        replace,
      })
    ).rejects.toBe(networkErr)

    expect(read).toHaveBeenCalledOnce()
    expect(validateExisting).not.toHaveBeenCalled()
    expect(replace).not.toHaveBeenCalled()
  })

  it.each([
    { form: 'ApiException', error: apiException(404) },
    { form: 'statusCode', error: { response: { statusCode: 404 } } },
  ])('TOCTOU-NP-5: helper replace 404 ($form) still throws', async ({ error }) => {
    const read = vi.fn().mockResolvedValue(existing)
    const replace = vi.fn().mockRejectedValue(error)

    await expect(
      replaceWithConflictRetry({
        description: 'Service "svc"',
        logPrefix: '[Test]',
        body: desired,
        mergeExisting: preserveServiceAssignedFields,
        isUpToDate: () => false,
        read,
        replace,
      })
    ).rejects.toBe(error)

    expect(read).toHaveBeenCalledOnce()
    expect(replace).toHaveBeenCalledOnce()
  })

  it.each([
    { status: 403, form: 'ApiException', error: apiException(403) },
    { status: 403, form: 'statusCode', error: { response: { statusCode: 403 } } },
    { status: 500, form: 'ApiException', error: apiException(500) },
    { status: 500, form: 'statusCode', error: { response: { statusCode: 500 } } },
  ])(
    'TOCTOU-NP-6: replace 409 then second read $status ($form) still throws',
    async ({ error }) => {
      const read = vi.fn().mockResolvedValueOnce(existing).mockRejectedValueOnce(error)
      const replace = vi.fn().mockRejectedValueOnce({ code: 409 })
      const validateExisting = vi.fn()

      await expect(
        replaceWithConflictRetry({
          description: 'Service "svc"',
          logPrefix: '[Test]',
          body: desired,
          mergeExisting: preserveServiceAssignedFields,
          isUpToDate: () => false,
          validateExisting,
          read,
          replace,
        })
      ).rejects.toBe(error)

      expect(read).toHaveBeenCalledTimes(2)
      expect(validateExisting).toHaveBeenCalledOnce()
      expect(replace).toHaveBeenCalledOnce()
      expect(replace.mock.calls[0][0].metadata?.resourceVersion).toBe('9')
    }
  )

  it('TOCTOU-NP-7: replace 409 then second-read network Error is rethrown', async () => {
    const networkErr = new Error('socket hang up')
    const read = vi.fn().mockResolvedValueOnce(existing).mockRejectedValueOnce(networkErr)
    const replace = vi.fn().mockRejectedValueOnce({ code: 409 })
    const validateExisting = vi.fn()

    await expect(
      replaceWithConflictRetry({
        description: 'Service "svc"',
        logPrefix: '[Test]',
        body: desired,
        mergeExisting: preserveServiceAssignedFields,
        isUpToDate: () => false,
        validateExisting,
        read,
        replace,
      })
    ).rejects.toBe(networkErr)

    expect(read).toHaveBeenCalledTimes(2)
    expect(validateExisting).toHaveBeenCalledOnce()
    expect(replace).toHaveBeenCalledOnce()
  })

  it('RETRY-SVC-2: post-409 re-read that matches skips the second replace', async () => {
    const drifted: k8s.V1Service = {
      ...existing,
      metadata: { ...existing.metadata, resourceVersion: '9' },
      spec: {
        ...existing.spec,
        ports: [{ name: 'http', port: 9090, protocol: 'TCP' }],
      },
    }
    const converged: k8s.V1Service = {
      ...existing,
      metadata: { ...existing.metadata, resourceVersion: '10' },
    }
    const read = vi.fn().mockResolvedValueOnce(drifted).mockResolvedValueOnce(converged)
    const replace = vi.fn().mockRejectedValueOnce({ code: 409 })

    await replaceWithConflictRetry({
      description: 'Service "svc"',
      logPrefix: '[Test]',
      body: desired,
      mergeExisting: preserveServiceAssignedFields,
      isUpToDate: serviceMatchesDesired,
      read,
      replace,
    })

    expect(replace).toHaveBeenCalledOnce()
    expect(replace.mock.calls[0][0].metadata?.resourceVersion).toBe('9')
    expect(read).toHaveBeenCalledTimes(2)
  })

  it('WRITE-1: successful replace increments clerum_hcc_writes_total{kind=Service}', async () => {
    const replace = vi.fn().mockResolvedValue({})
    const before = await readWritesTotal('Service')
    const log = vi.spyOn(console, 'log').mockImplementation(() => {})

    await replaceWithConflictRetry({
      description: 'Service "svc"',
      logPrefix: '[Test]',
      body: desired,
      mergeExisting: preserveServiceAssignedFields,
      isUpToDate: () => false,
      read: async () => existing,
      replace,
    })

    expect(replace).toHaveBeenCalledOnce()
    expect(await readWritesTotal('Service')).toBe(before + 1)
    expect(await registry.metrics()).toContain('clerum_hcc_writes_total')
    log.mockRestore()
  })

  it('WRITE-2: isUpToDate skip and read-404 do not increment', async () => {
    const replace = vi.fn()
    const before = await readWritesTotal('Service')
    const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})

    await replaceWithConflictRetry({
      description: 'Service "svc"',
      logPrefix: '[Test]',
      body: desired,
      mergeExisting: preserveServiceAssignedFields,
      isUpToDate: () => true,
      read: async () => existing,
      replace,
    })

    await replaceWithConflictRetry({
      description: 'Service "svc"',
      logPrefix: '[Test]',
      body: desired,
      read: async () => {
        throw { code: 404 }
      },
      replace,
    })

    expect(replace).not.toHaveBeenCalled()
    expect(await readWritesTotal('Service')).toBe(before)
    expect(warn).toHaveBeenCalledOnce()
    expect(JSON.parse(String(warn.mock.calls[0][0]))).toMatchObject({
      level: 'warn',
      msg: 'Kubernetes resource disappeared; skipping replace',
      scope: '[Test]',
      description: 'Service "svc"',
    })
    warn.mockRestore()
  })

  it('SKIP-3: isUpToDate skip with mutationAllowed false does not increment writes or skips', async () => {
    const replace = vi.fn()
    const beforeWrites = await readWritesTotal('Service')
    const beforeSkips = await readWriteSkipsTotal('Service')

    await replaceWithConflictRetry({
      description: 'Service "svc"',
      logPrefix: '[Test]',
      body: desired,
      mergeExisting: preserveServiceAssignedFields,
      isUpToDate: () => true,
      mutationAllowed: () => false,
      read: async () => existing,
      replace,
    })

    expect(replace).not.toHaveBeenCalled()
    expect(await readWritesTotal('Service')).toBe(beforeWrites)
    expect(await readWriteSkipsTotal('Service')).toBe(beforeSkips)
  })

  it('SKIP-1: isUpToDate skip increments clerum_hcc_write_skips_total{kind=Service}', async () => {
    const replace = vi.fn()
    const beforeWrites = await readWritesTotal('Service')
    const beforeSkips = await readWriteSkipsTotal('Service')

    await replaceWithConflictRetry({
      description: 'Service "svc"',
      logPrefix: '[Test]',
      body: desired,
      mergeExisting: preserveServiceAssignedFields,
      isUpToDate: () => true,
      read: async () => existing,
      replace,
    })

    expect(replace).not.toHaveBeenCalled()
    expect(await readWritesTotal('Service')).toBe(beforeWrites)
    expect(await readWriteSkipsTotal('Service')).toBe(beforeSkips + 1)
    expect(await registry.metrics()).toContain('clerum_hcc_write_skips_total')
  })

  it('SKIP-2: successful replace and read-404 do not increment write skips', async () => {
    const replace = vi.fn().mockResolvedValue({})
    const beforeSkips = await readWriteSkipsTotal('Service')
    const log = vi.spyOn(console, 'log').mockImplementation(() => {})

    await replaceWithConflictRetry({
      description: 'Service "svc"',
      logPrefix: '[Test]',
      body: desired,
      mergeExisting: preserveServiceAssignedFields,
      isUpToDate: () => false,
      read: async () => existing,
      replace,
    })

    await replaceWithConflictRetry({
      description: 'Service "svc"',
      logPrefix: '[Test]',
      body: desired,
      read: async () => {
        throw { code: 404 }
      },
      replace,
    })

    expect(await readWriteSkipsTotal('Service')).toBe(beforeSkips)
    log.mockRestore()
  })

  it('WRITE-3: 409 then successful replace increments once, not per attempt', async () => {
    const firstRead: k8s.V1Service = {
      ...existing,
      metadata: { ...existing.metadata, resourceVersion: '9' },
    }
    const secondRead: k8s.V1Service = {
      ...existing,
      metadata: { ...existing.metadata, resourceVersion: '10' },
    }
    const read = vi.fn().mockResolvedValueOnce(firstRead).mockResolvedValueOnce(secondRead)
    const replace = vi.fn().mockRejectedValueOnce({ code: 409 }).mockResolvedValueOnce({})
    const before = await readWritesTotal('Service')
    const log = vi.spyOn(console, 'log').mockImplementation(() => {})

    await replaceWithConflictRetry({
      description: 'Service "svc"',
      logPrefix: '[Test]',
      body: desired,
      mergeExisting: preserveServiceAssignedFields,
      isUpToDate: () => false,
      read,
      replace,
    })

    expect(replace).toHaveBeenCalledTimes(2)
    expect(await readWritesTotal('Service')).toBe(before + 1)
    log.mockRestore()
  })

  it('WRITE-4: successful replace without kind increments {kind=unknown}', async () => {
    const body: {
      metadata?: { resourceVersion?: string; name?: string; namespace?: string }
      kind?: string
    } = {
      metadata: { name: 'svc', namespace: 'ns' },
    }
    const live: typeof body = {
      metadata: { name: 'svc', namespace: 'ns', resourceVersion: '9' },
    }
    const before = await readWritesTotal('unknown')
    const log = vi.spyOn(console, 'log').mockImplementation(() => {})

    await replaceWithConflictRetry({
      description: 'object "svc"',
      logPrefix: '[Test]',
      body,
      isUpToDate: () => false,
      read: async () => live,
      replace: async () => ({}),
    })

    expect(await readWritesTotal('unknown')).toBe(before + 1)
    log.mockRestore()
  })

  it('WRITE-5: 409 then isUpToDate skip does not increment', async () => {
    const drifted: k8s.V1Service = {
      ...existing,
      metadata: { ...existing.metadata, resourceVersion: '9' },
      spec: {
        ...existing.spec,
        ports: [{ name: 'http', port: 9090, protocol: 'TCP' }],
      },
    }
    const converged: k8s.V1Service = {
      ...existing,
      metadata: { ...existing.metadata, resourceVersion: '10' },
    }
    const read = vi.fn().mockResolvedValueOnce(drifted).mockResolvedValueOnce(converged)
    const replace = vi.fn().mockRejectedValueOnce({ code: 409 })
    const before = await readWritesTotal('Service')

    await replaceWithConflictRetry({
      description: 'Service "svc"',
      logPrefix: '[Test]',
      body: desired,
      mergeExisting: preserveServiceAssignedFields,
      isUpToDate: serviceMatchesDesired,
      read,
      replace,
    })

    expect(replace).toHaveBeenCalledOnce()
    expect(await readWritesTotal('Service')).toBe(before)
  })

  it('WRITE-6: failed replace does not increment', async () => {
    const replace = vi.fn().mockRejectedValue(apiException(404))
    const before = await readWritesTotal('Service')

    await expect(
      replaceWithConflictRetry({
        description: 'Service "svc"',
        logPrefix: '[Test]',
        body: desired,
        mergeExisting: preserveServiceAssignedFields,
        isUpToDate: () => false,
        read: async () => existing,
        replace,
      })
    ).rejects.toBeInstanceOf(ApiException)

    expect(replace).toHaveBeenCalledOnce()
    expect(await readWritesTotal('Service')).toBe(before)
  })
})