Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
12.0 KB
Raw Blame
/**
 * HeartbeatPoller unit tests — HCC consumes heartbeats from control-api's
 * InternalControl feed instead of verifying Host JWTs itself. Fake timers +
 * injected fetch: no real network, no real clock.
 */
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import {
  HeartbeatPoller,
  type HeartbeatPollerFetch,
  type HeartbeatPollerOptions,
  INITIAL_HEARTBEAT_LOOKBACK_MS,
} from '../src/heartbeatPoller'
import { HeartbeatPayload, HeartbeatVerdict } from '../src/statelessLifecycleTracker'
import { HostCRD } from '../src/types'

const POLL_MS = 10_000

interface RecordedCall {
  url: string
  init: { method: string; headers: Record<string, string> }
}

type FakeResponse = { ok: boolean; status: number; json(): Promise<unknown> }

function jsonResponse(status: number, body: unknown): FakeResponse {
  return { ok: status >= 200 && status < 300, status, json: async () => body }
}

function makeHost(name = 'stateless-host'): HostCRD {
  return {
    name,
    namespace: 'mcp-host',
    spec: {
      host: name,
      contextRef: 'context-a',
      secretRef: 'host-secret',
      lifecycle: { stateless: true },
    },
    status: { lifecycle: { state: 'active', wakeHandledGeneration: 0 } },
  }
}

function feedRow(overrides: Partial<HeartbeatPayload & { receivedAtMs: number }> = {}) {
  return {
    hostRef: 'stateless-host',
    podUid: 'pod-a',
    activeWork: false,
    conditions: {
      activeTask: false,
      awaitingApproval: false,
      pendingResults: false,
      activeCronSchedules: false,
    },
    lastActivityTs: 1_700_000_000_000,
    state: 'active',
    receivedAtMs: 1_700_000_030_000,
    ...overrides,
  }
}

function makeHarness(overrides: Partial<HeartbeatPollerOptions> = {}) {
  const calls: RecordedCall[] = []
  let responder: () => Promise<FakeResponse> = async () => jsonResponse(200, { heartbeats: [] })
  const fetchImpl: HeartbeatPollerFetch = async (url, init) => {
    calls.push({ url, init })
    return responder()
  }
  const verdicts: HeartbeatVerdict[] = []
  const tracker = {
    handleHeartbeat: vi.fn(async (_payload: HeartbeatPayload): Promise<HeartbeatVerdict> => {
      return verdicts.shift() ?? { drain: false }
    }),
    noteSuccessfulPoll: vi.fn(),
    stop: vi.fn(),
  }
  const host = makeHost()
  const getHost = vi.fn((hostRef: string) => (hostRef === host.name ? host : undefined))
  const markHostDraining = vi.fn(async () => {})
  const poller = new HeartbeatPoller({
    pollIntervalMs: POLL_MS,
    controlApiBaseUrl: 'http://control-api.control-plane.svc.cluster.local:8090',
    tracker,
    getHost,
    markHostDraining,
    signInternalControlJwt: () => 'internal-control-jwt',
    fetchImpl,
    ...overrides,
  })
  return {
    poller,
    calls,
    tracker,
    host,
    getHost,
    markHostDraining,
    setResponder(next: () => Promise<FakeResponse>): void {
      responder = next
    },
    queueVerdicts(...next: HeartbeatVerdict[]): void {
      verdicts.push(...next)
    },
    sinceOf(index: number): number {
      const url = new URL(calls[index].url)
      return Number(url.searchParams.get('since'))
    },
  }
}

describe('HeartbeatPoller', () => {
  beforeEach(() => {
    vi.useFakeTimers()
    vi.spyOn(console, 'error').mockImplementation(() => {})
    vi.spyOn(console, 'warn').mockImplementation(() => {})
  })
  afterEach(() => {
    vi.useRealTimers()
    vi.restoreAllMocks()
  })

  it('fails loud at construction on a bad interval or an empty base URL', () => {
    expect(() => makeHarness({ pollIntervalMs: 0 })).toThrow(/positive integer/)
    expect(() => makeHarness({ pollIntervalMs: 1.5 })).toThrow(/positive integer/)
    expect(() => makeHarness({ controlApiBaseUrl: '  ' })).toThrow(/CONTROL_API_BASE_URL/)
  })

  it('GETs the InternalControl feed with the signed JWT and a 5-minute initial lookback', async () => {
    const h = makeHarness()
    const t0 = Date.now()
    h.poller.start()

    await vi.advanceTimersByTimeAsync(POLL_MS)

    expect(h.calls).toHaveLength(1)
    expect(h.calls[0].url).toBe(
      `http://control-api.control-plane.svc.cluster.local:8090/api/v1/auth/mcp-host/heartbeats?since=${
        t0 - INITIAL_HEARTBEAT_LOOKBACK_MS
      }`
    )
    expect(h.calls[0].init.method).toBe('GET')
    expect(h.calls[0].init.headers.Authorization).toBe('Bearer internal-control-jwt')

    h.poller.stop()
  })

  it('advances the since cursor to the poll start time ONLY on success', async () => {
    const h = makeHarness()
    const t0 = Date.now()
    h.poller.start()

    await vi.advanceTimersByTimeAsync(POLL_MS) // poll 1 (success)
    await vi.advanceTimersByTimeAsync(POLL_MS) // poll 2
    expect(h.sinceOf(1)).toBe(t0 + POLL_MS)

    // Poll 3 fails → poll 4 repeats poll 3's cursor.
    h.setResponder(async () => jsonResponse(500, {}))
    await vi.advanceTimersByTimeAsync(POLL_MS) // poll 3 (fails)
    h.setResponder(async () => jsonResponse(200, { heartbeats: [] }))
    await vi.advanceTimersByTimeAsync(POLL_MS) // poll 4
    expect(h.sinceOf(2)).toBe(t0 + 2 * POLL_MS)
    expect(h.sinceOf(3)).toBe(t0 + 2 * POLL_MS)

    h.poller.stop()
  })

  it('feeds rows into the tracker oldest-first regardless of feed order (podUid ordering intact)', async () => {
    const h = makeHarness()
    h.setResponder(async () =>
      jsonResponse(200, {
        heartbeats: [
          feedRow({ podUid: 'pod-b', receivedAtMs: 2_000 }),
          feedRow({ podUid: 'pod-a', receivedAtMs: 1_000 }),
        ],
      })
    )
    h.poller.start()
    await vi.advanceTimersByTimeAsync(POLL_MS)

    expect(h.tracker.handleHeartbeat).toHaveBeenCalledTimes(2)
    expect(h.tracker.handleHeartbeat.mock.calls[0][0]).toMatchObject({ podUid: 'pod-a' })
    expect(h.tracker.handleHeartbeat.mock.calls[1][0]).toMatchObject({ podUid: 'pod-b' })
    // The tracker payload is reconstructed with schemaVersion 1.
    expect(h.tracker.handleHeartbeat.mock.calls[0][0]).toMatchObject({ schemaVersion: 1 })

    h.poller.stop()
  })

  it('persists a drain:true decision via the draining status write; drain:false writes nothing', async () => {
    const h = makeHarness()
    h.setResponder(async () =>
      jsonResponse(200, {
        heartbeats: [feedRow({ receivedAtMs: 1_000 }), feedRow({ receivedAtMs: 2_000 })],
      })
    )
    h.queueVerdicts({ drain: false }, { drain: true, entryWakeHandledGeneration: 4 })
    h.poller.start()
    await vi.advanceTimersByTimeAsync(POLL_MS)

    expect(h.markHostDraining).toHaveBeenCalledTimes(1)
    // The AP-1 epoch the tracker decided with rides through unchanged.
    expect(h.markHostDraining).toHaveBeenCalledWith(h.host, 4)

    h.poller.stop()
  })

  it('a drain for an unknown Host skips the status write loudly and keeps processing', async () => {
    const h = makeHarness()
    h.setResponder(async () =>
      jsonResponse(200, { heartbeats: [feedRow({ hostRef: 'ghost-host' })] })
    )
    h.queueVerdicts({ drain: true, entryWakeHandledGeneration: 0 })
    h.poller.start()
    await vi.advanceTimersByTimeAsync(POLL_MS)

    expect(h.markHostDraining).not.toHaveBeenCalled()
    expect(console.warn).toHaveBeenCalledWith(expect.stringContaining('ghost-host'))

    // The poll still counts as a success: the next poll advances the cursor.
    const firstSince = h.sinceOf(0)
    await vi.advanceTimersByTimeAsync(POLL_MS)
    expect(h.sinceOf(1)).toBeGreaterThan(firstSince)

    h.poller.stop()
  })

  it('a failing draining status write is logged loudly and does not stop the poller', async () => {
    const h = makeHarness()
    h.setResponder(async () => jsonResponse(200, { heartbeats: [feedRow()] }))
    h.queueVerdicts(
      { drain: true, entryWakeHandledGeneration: 0 },
      { drain: true, entryWakeHandledGeneration: 0 }
    )
    h.markHostDraining.mockRejectedValueOnce(new Error('status write conflict'))
    h.poller.start()

    await vi.advanceTimersByTimeAsync(POLL_MS)
    expect(console.error).toHaveBeenCalledWith(
      expect.stringContaining('draining status write failed'),
      expect.any(Error)
    )

    // Next poll retries the write.
    await vi.advanceTimersByTimeAsync(POLL_MS)
    expect(h.markHostDraining).toHaveBeenCalledTimes(2)

    h.poller.stop()
  })

  it('reports feed health to the tracker ONLY on a fully-successful poll (FIX 4 seam)', async () => {
    const h = makeHarness()
    h.poller.start()
    // Successful (empty) poll → the tracker learns the feed is flowing.
    await vi.advanceTimersByTimeAsync(POLL_MS)
    expect(h.tracker.noteSuccessfulPoll).toHaveBeenCalledTimes(1)
    expect(h.tracker.noteSuccessfulPoll).toHaveBeenCalledWith(POLL_MS)
    // Failing poll → NOT reported: a dead feed must never look healthy.
    h.setResponder(async () => jsonResponse(500, {}))
    await vi.advanceTimersByTimeAsync(POLL_MS)
    expect(h.tracker.noteSuccessfulPoll).toHaveBeenCalledTimes(1)
    // Malformed row fails the WHOLE poll → NOT reported either.
    h.setResponder(async () => jsonResponse(200, { heartbeats: [{ bogus: true }] }))
    await vi.advanceTimersByTimeAsync(POLL_MS)
    expect(h.tracker.noteSuccessfulPoll).toHaveBeenCalledTimes(1)

    h.poller.stop()
  })

  it('poll failures (HTTP error / rejecting fetch) log rate-limited and never stop the loop', async () => {
    const h = makeHarness()
    h.setResponder(async () => jsonResponse(503, {}))
    h.poller.start()

    // 5 failing polls inside the 60s suppression window → exactly one log.
    await vi.advanceTimersByTimeAsync(5 * POLL_MS)
    expect(h.calls).toHaveLength(5)
    expect(console.error).toHaveBeenCalledTimes(1)

    // Past the window the failure is re-logged; the poller keeps going.
    await vi.advanceTimersByTimeAsync(2 * POLL_MS)
    expect(h.calls).toHaveLength(7)
    expect(console.error).toHaveBeenCalledTimes(2)

    h.setResponder(async () => {
      throw new Error('ECONNREFUSED')
    })
    await vi.advanceTimersByTimeAsync(POLL_MS)
    expect(h.calls).toHaveLength(8)

    h.poller.stop()
  })

  it('a malformed feed row fails the WHOLE poll loudly: no partial apply, cursor not advanced', async () => {
    const h = makeHarness()
    h.setResponder(async () =>
      jsonResponse(200, {
        heartbeats: [feedRow(), feedRow({ state: 'sleeping' as never })],
      })
    )
    h.poller.start()

    const t0 = Date.now()
    await vi.advanceTimersByTimeAsync(POLL_MS)
    expect(h.tracker.handleHeartbeat).not.toHaveBeenCalled()
    expect(console.error).toHaveBeenCalledTimes(1)

    // Cursor stayed put.
    await vi.advanceTimersByTimeAsync(POLL_MS)
    expect(h.sinceOf(1)).toBe(h.sinceOf(0))
    expect(t0).toBeGreaterThan(0)

    h.poller.stop()
  })

  it('a response without a heartbeats array is a contract breach, not an empty poll', async () => {
    const h = makeHarness()
    h.setResponder(async () => jsonResponse(200, { rows: [] }))
    h.poller.start()

    await vi.advanceTimersByTimeAsync(POLL_MS)
    expect(console.error).toHaveBeenCalledTimes(1)
    await vi.advanceTimersByTimeAsync(POLL_MS)
    expect(h.sinceOf(1)).toBe(h.sinceOf(0))

    h.poller.stop()
  })

  it('stop() cancels the pending poll', async () => {
    const h = makeHarness()
    h.poller.start()
    await vi.advanceTimersByTimeAsync(POLL_MS)
    expect(h.calls).toHaveLength(1)

    h.poller.stop()
    await vi.advanceTimersByTimeAsync(10 * POLL_MS)
    expect(h.calls).toHaveLength(1)
  })
})

describe('parseHeartbeatPollMs (config load validation)', () => {
  it('returns the 10s default when unset or empty', async () => {
    const { parseHeartbeatPollMs } = await import('../src/config')
    expect(parseHeartbeatPollMs(undefined)).toBe(10_000)
    expect(parseHeartbeatPollMs('')).toBe(10_000)
    expect(parseHeartbeatPollMs('   ')).toBe(10_000)
  })

  it('parses an explicitly-set positive integer', async () => {
    const { parseHeartbeatPollMs } = await import('../src/config')
    expect(parseHeartbeatPollMs('5000')).toBe(5_000)
    expect(parseHeartbeatPollMs('1')).toBe(1)
  })

  it('fails config load loudly on garbage values', async () => {
    const { parseHeartbeatPollMs } = await import('../src/config')
    for (const bad of ['banana', '0', '-5', '1.5', '10s', 'NaN', 'Infinity']) {
      expect(() => parseHeartbeatPollMs(bad)).toThrow(/positive integer/)
    }
  })
})