Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
5.5 KB
Raw Blame
import { describe, expect, it, vi } from 'vitest'
import {
  evaluateCompletedRuntimePodRecovery,
  evaluateCrashRecovery,
  getPodPresence,
  inspectPodReadiness,
  waitForPodDeletion,
} from './crashRecovery'

describe('crash recovery pod readiness classification', () => {
  it('treats scheduling Pending without a container waiting reason as recoverable wait', () => {
    expect(evaluateCrashRecovery('Pending', { phase: 'initializing' })).toEqual({
      action: 'none',
      message: 'Pod is healthy',
    })
  })

  it('keeps image pull Pending failures bounded by replacement attempts', () => {
    expect(
      evaluateCrashRecovery('Pending', { phase: 'recovering', attempt: 2 }, 'ImagePullBackOff')
    ).toMatchObject({
      action: 'replace',
      newPhase: 'recovering',
      newAttempt: 3,
    })
    expect(
      evaluateCrashRecovery('Pending', { phase: 'recovering', attempt: 3 }, 'ImagePullBackOff')
    ).toMatchObject({
      action: 'fail',
      newPhase: 'failed',
    })
  })

  it('keeps Running CrashLoopBackOff pods bounded by replacement attempts', () => {
    expect(
      evaluateCrashRecovery('Running', { phase: 'recovering', attempt: 2 }, 'CrashLoopBackOff')
    ).toMatchObject({
      action: 'replace',
      newPhase: 'recovering',
      newAttempt: 3,
    })
    expect(
      evaluateCrashRecovery('Running', { phase: 'recovering', attempt: 3 }, 'CrashLoopBackOff')
    ).toMatchObject({
      action: 'fail',
      newPhase: 'failed',
    })
  })

  it('keeps completed runtime pods bounded by replacement attempts', () => {
    expect(
      evaluateCompletedRuntimePodRecovery({ phase: 'recovering', attempt: 2 }, 'mcp_host')
    ).toMatchObject({
      action: 'replace',
      newPhase: 'recovering',
      newAttempt: 3,
    })
    expect(
      evaluateCompletedRuntimePodRecovery({ phase: 'recovering', attempt: 3 }, 'mcp_host')
    ).toMatchObject({
      action: 'fail',
      newPhase: 'failed',
    })
  })

  it('extracts readiness and scheduling details from a Pending pod', () => {
    expect(
      inspectPodReadiness({
        status: {
          phase: 'Pending',
          conditions: [
            {
              type: 'PodScheduled',
              status: 'False',
              reason: 'Unschedulable',
              message: '0/3 nodes are available',
            },
          ],
        },
      })
    ).toEqual({
      phase: 'Pending',
      ready: false,
      schedulingReason: 'Unschedulable',
    })
  })

  it('requires the Ready condition, not only Running phase', () => {
    expect(
      inspectPodReadiness({
        status: {
          phase: 'Running',
          conditions: [{ type: 'Ready', status: 'False' }],
        },
      })
    ).toEqual({
      phase: 'Running',
      ready: false,
    })

    expect(
      inspectPodReadiness({
        status: {
          phase: 'Running',
          conditions: [{ type: 'Ready', status: 'True' }],
        },
      })
    ).toEqual({
      phase: 'Running',
      ready: true,
    })
  })
})

describe('getPodPresence', () => {
  it('returns absent on GET 404 and present on GET 200 with empty status', async () => {
    const readNamespacedPod = vi
      .fn()
      .mockRejectedValueOnce({ code: 404 })
      .mockResolvedValueOnce({ metadata: { name: 'recipe-coordinator' }, status: {} })
    const coreApi = { readNamespacedPod } as unknown as Parameters<typeof getPodPresence>[0]

    await expect(getPodPresence(coreApi, 'recipe-coordinator', 'sandbox-recipes')).resolves.toEqual(
      {
        kind: 'absent',
      }
    )
    await expect(getPodPresence(coreApi, 'recipe-coordinator', 'sandbox-recipes')).resolves.toEqual(
      {
        kind: 'present',
        phase: undefined,
      }
    )
  })

  it.each([403, 500])('rethrows a GET %i instead of reporting the pod absent', async code => {
    const readNamespacedPod = vi.fn().mockRejectedValueOnce({ code })
    const coreApi = { readNamespacedPod } as unknown as Parameters<typeof getPodPresence>[0]

    await expect(
      getPodPresence(coreApi, 'recipe-coordinator', 'sandbox-recipes')
    ).rejects.toMatchObject({ code })
    expect(readNamespacedPod).toHaveBeenCalledWith({
      name: 'recipe-coordinator',
      namespace: 'sandbox-recipes',
    })
  })
})

describe('waitForPodDeletion', () => {
  it('returns true once getPodPresence reports the pod is gone', async () => {
    const readNamespacedPod = vi
      .fn()
      .mockResolvedValueOnce({ status: { phase: 'Running' } })
      .mockRejectedValueOnce({ response: { statusCode: 404 } })
    const coreApi = {
      readNamespacedPod,
    } as unknown as Parameters<typeof waitForPodDeletion>[0]

    await expect(
      waitForPodDeletion(coreApi, 'recipe-mcp-host', 'sandbox-recipes', {
        timeoutMs: 1_000,
        pollIntervalMs: 1,
      })
    ).resolves.toBe(true)
    expect(readNamespacedPod).toHaveBeenCalledTimes(2)
  })

  it('does not treat a GET 200 with empty status.phase as deleted', async () => {
    const readNamespacedPod = vi
      .fn()
      .mockResolvedValue({ metadata: { name: 'recipe-mcp-host' }, status: {} })
    const coreApi = {
      readNamespacedPod,
    } as unknown as Parameters<typeof waitForPodDeletion>[0]

    await expect(
      waitForPodDeletion(coreApi, 'recipe-mcp-host', 'sandbox-recipes', {
        timeoutMs: 20,
        pollIntervalMs: 5,
      })
    ).resolves.toBe(false)
    // Liveness witness: `false` also results if the wait never polled. Several GETs
    // that each answered 200 show it kept looking until the timeout.
    expect(readNamespacedPod.mock.calls.length).toBeGreaterThan(1)
  })
})