Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
67.1 KB
Raw Blame
import { Mock, afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import {
  HeartbeatPayload,
  StatelessLifecycleReconcilerPort,
  StatelessLifecycleTracker,
  parseHeartbeatPayload,
} from '../src/statelessLifecycleTracker'
import { HostCRD, HostCrdStatus } from '../src/types'

const NOW = Date.parse('2026-07-03T12:00:00.000Z')
const MIN = 60_000
const HOUR = 3_600_000

function makeHost(status?: HostCrdStatus): HostCRD {
  return {
    name: 'stateless-host',
    namespace: 'mcp-host',
    spec: {
      host: 'stateless-host',
      contextRef: 'context-a',
      secretRef: 'host-secret',
      lifecycle: { stateless: true },
    },
    status: status ?? { lifecycle: { state: 'active', wakeHandledGeneration: 0 } },
  }
}

type MockPort = {
  [K in keyof StatelessLifecycleReconcilerPort]: Mock
}

function makePort(): MockPort {
  return {
    getEffectiveLifecycle: vi.fn().mockReturnValue({ stateless: true, state: 'active' }),
    getWakeRequestedGeneration: vi.fn().mockReturnValue(0),
    // Fresh read defaults to echoing the caller's host -- cache and server
    // agree unless a test overrides it (cache-stale resolution suite).
    readFreshHost: vi.fn().mockImplementation(async (host: HostCRD) => host),
    suspendHostFromHeartbeat: vi.fn().mockResolvedValue(undefined),
    markHostActiveFromHeartbeat: vi.fn().mockResolvedValue(undefined),
    publishSuspendBlockedReason: vi.fn().mockResolvedValue(undefined),
    // Default: a young pod (1 minute old) — max-uptime never fires.
    findPodCreationTimestamp: vi.fn().mockResolvedValue(new Date(NOW - MIN)),
  }
}

function makeTracker(options: {
  port: MockPort
  host?: HostCRD | undefined
  idleMinutes?: number
  idleFloorMinutes?: number
  drainGraceMs?: number
  maxUptimeHours?: number
}): StatelessLifecycleTracker {
  return new StatelessLifecycleTracker({
    idleMinutes: options.idleMinutes ?? 30,
    idleFloorMinutes: options.idleFloorMinutes ?? 15,
    drainGraceMs: options.drainGraceMs ?? 60_000,
    maxUptimeHours: options.maxUptimeHours ?? 72,
    reconciler: options.port as unknown as StatelessLifecycleReconcilerPort,
    getHost: () => options.host ?? makeHost(),
    now: () => NOW,
  })
}

function payload(overrides: Partial<HeartbeatPayload> = {}): HeartbeatPayload {
  return {
    schemaVersion: 1,
    hostRef: 'stateless-host',
    podUid: 'pod-a',
    activeWork: false,
    conditions: {
      activeTask: false,
      awaitingApproval: false,
      pendingResults: false,
      activeCronSchedules: false,
    },
    lastActivityTs: NOW - 5 * MIN,
    state: 'active',
    ...overrides,
  }
}

describe('parseHeartbeatPayload — strict shape', () => {
  it('accepts the canonical emitter payload', () => {
    const result = parseHeartbeatPayload(payload())
    expect(result).toEqual({ ok: true, payload: payload() })
  })

  it('rejects a wrong schemaVersion', () => {
    const result = parseHeartbeatPayload({ ...payload(), schemaVersion: 2 })
    expect(result).toEqual({ ok: false, error: 'schemaVersion must be 1' })
  })

  it('rejects non-boolean fields', () => {
    expect(parseHeartbeatPayload({ ...payload(), activeWork: 'yes' }).ok).toBe(false)
    expect(
      parseHeartbeatPayload({
        ...payload(),
        conditions: { activeTask: 1, awaitingApproval: false, pendingResults: false },
      }).ok
    ).toBe(false)
  })

  it('rejects a non-finite lastActivityTs and unknown states', () => {
    expect(parseHeartbeatPayload({ ...payload(), lastActivityTs: Number.NaN }).ok).toBe(false)
    expect(parseHeartbeatPayload({ ...payload(), lastActivityTs: -1 }).ok).toBe(false)
    expect(parseHeartbeatPayload({ ...payload(), state: 'paused' }).ok).toBe(false)
  })

  it('rejects non-object bodies', () => {
    expect(parseHeartbeatPayload(null).ok).toBe(false)
    expect(parseHeartbeatPayload([]).ok).toBe(false)
    expect(parseHeartbeatPayload('{}').ok).toBe(false)
  })
})

describe('StatelessLifecycleTracker — D8 idle rule', () => {
  it('answers drain:false while activeWork=true regardless of lastActivityTs', async () => {
    const port = makePort()
    const tracker = makeTracker({ port })
    const verdict = await tracker.handleHeartbeat(
      payload({
        activeWork: true,
        conditions: {
          activeTask: true,
          awaitingApproval: false,
          pendingResults: false,
          activeCronSchedules: false,
        },
        lastActivityTs: NOW - 10 * HOUR,
      })
    )
    expect(verdict).toEqual({ drain: false })
    expect(port.suspendHostFromHeartbeat).not.toHaveBeenCalled()
  })

  it('answers drain:true once idle exceeds T_idle', async () => {
    const port = makePort()
    const tracker = makeTracker({ port })
    const verdict = await tracker.handleHeartbeat(payload({ lastActivityTs: NOW - 31 * MIN }))
    expect(verdict).toEqual({ drain: true, entryWakeHandledGeneration: 0 })
    // Decision only — the tracker never scales; suspension waits for the
    // drained report (or grace expiry).
    expect(port.suspendHostFromHeartbeat).not.toHaveBeenCalled()
  })

  it('enforces the idle floor when idleMinutes is configured below it', async () => {
    const port = makePort()
    const tracker = makeTracker({ port, idleMinutes: 1, idleFloorMinutes: 15 })
    expect(await tracker.handleHeartbeat(payload({ lastActivityTs: NOW - 10 * MIN }))).toEqual({
      drain: false,
    })
    expect(await tracker.handleHeartbeat(payload({ lastActivityTs: NOW - 16 * MIN }))).toEqual({
      drain: true,
      entryWakeHandledGeneration: 0,
    })
  })

  it('answers drain:false while lastActivityTs is fresh', async () => {
    const port = makePort()
    const tracker = makeTracker({ port })
    expect(await tracker.handleHeartbeat(payload({ lastActivityTs: NOW - 1 * MIN }))).toEqual({
      drain: false,
    })
  })

  it('never drains a host whose assessed lifecycle is not stateless', async () => {
    const port = makePort()
    port.getEffectiveLifecycle.mockReturnValue({ stateless: false, state: 'active' })
    const tracker = makeTracker({ port })
    expect(await tracker.handleHeartbeat(payload({ lastActivityTs: NOW - 10 * HOUR }))).toEqual({
      drain: false,
    })
  })

  it('does not drain or suspend a stateless Host while channel authority is unavailable', async () => {
    const port = makePort()
    port.getEffectiveLifecycle.mockReturnValue({
      stateless: true,
      state: 'active',
      suspensionBlocked: true,
    })
    const tracker = makeTracker({ port })
    for (let cycle = 0; cycle < 2; cycle++) {
      expect(await tracker.handleHeartbeat(payload({ lastActivityTs: NOW - 10 * HOUR }))).toEqual({
        drain: false,
      })
    }
    expect(port.suspendHostFromHeartbeat).not.toHaveBeenCalled()
  })

  it('reverts a durable draining Host through a fresh read when a cached Host is stale', async () => {
    const port = makePort()
    const cachedHost = makeHost({ lifecycle: { state: 'active', wakeHandledGeneration: 3 } })
    port.getEffectiveLifecycle.mockReturnValue({
      stateless: true,
      state: 'active',
      suspensionBlocked: true,
    })
    port.readFreshHost.mockResolvedValue(
      makeHost({ lifecycle: { state: 'draining', wakeHandledGeneration: 3 } })
    )
    const tracker = makeTracker({ port, host: cachedHost })

    await expect(tracker.handleHeartbeat(payload({ state: 'draining' }))).resolves.toEqual({
      drain: false,
    })
    expect(port.markHostActiveFromHeartbeat).toHaveBeenCalledWith(cachedHost)
    expect(port.suspendHostFromHeartbeat).not.toHaveBeenCalled()
  })

  it('fresh-checks a suspension-blocked Host once per channel authority loss epoch', async () => {
    const port = makePort()
    const blocked = { stateless: true, state: 'active', suspensionBlocked: true }
    port.getEffectiveLifecycle.mockReturnValue(blocked)
    const tracker = makeTracker({ port, host: makeHost() })

    await expect(tracker.handleHeartbeat(payload())).resolves.toEqual({ drain: false })
    expect(port.getEffectiveLifecycle).toHaveBeenCalledTimes(1)
    expect(port.markHostActiveFromHeartbeat).toHaveBeenCalledTimes(1)

    await expect(tracker.handleHeartbeat(payload())).resolves.toEqual({ drain: false })
    expect(port.getEffectiveLifecycle).toHaveBeenCalledTimes(2)
    expect(port.markHostActiveFromHeartbeat).toHaveBeenCalledTimes(1)

    port.getEffectiveLifecycle.mockReturnValue({ stateless: true, state: 'active' })
    await expect(tracker.handleHeartbeat(payload())).resolves.toEqual({ drain: false })
    expect(port.getEffectiveLifecycle).toHaveBeenCalledTimes(3)
    expect(port.markHostActiveFromHeartbeat).toHaveBeenCalledTimes(1)

    port.getEffectiveLifecycle.mockReturnValue(blocked)
    await expect(tracker.handleHeartbeat(payload())).resolves.toEqual({ drain: false })
    expect(port.getEffectiveLifecycle).toHaveBeenCalledTimes(4)
    expect(port.markHostActiveFromHeartbeat).toHaveBeenCalledTimes(2)
    expect(port.suspendHostFromHeartbeat).not.toHaveBeenCalled()
  })

  it('repeats the suspension-blocked fresh check after the tracker is stopped', async () => {
    const port = makePort()
    port.getEffectiveLifecycle.mockReturnValue({
      stateless: true,
      state: 'active',
      suspensionBlocked: true,
    })
    const tracker = makeTracker({ port, host: makeHost() })

    await expect(tracker.handleHeartbeat(payload())).resolves.toEqual({ drain: false })
    await expect(tracker.handleHeartbeat(payload())).resolves.toEqual({ drain: false })
    // The epoch's single fresh check ran, and the second blocked beat skipped it.
    expect(port.getEffectiveLifecycle).toHaveBeenCalledTimes(2)
    expect(port.markHostActiveFromHeartbeat).toHaveBeenCalledTimes(1)

    tracker.stop()
    await expect(tracker.handleHeartbeat(payload())).resolves.toEqual({ drain: false })
    expect(port.getEffectiveLifecycle).toHaveBeenCalledTimes(3)
    expect(port.markHostActiveFromHeartbeat).toHaveBeenCalledTimes(2)
    expect(port.suspendHostFromHeartbeat).not.toHaveBeenCalled()
  })

  it('reports a cancel-drain with nothing to revert as settled', async () => {
    const port = makePort()
    const tracker = makeTracker({ port })
    const cancelDrain = (
      tracker as unknown as {
        cancelDrainOnEvidence(hostRef: string, host: HostCRD, bypass?: boolean): Promise<boolean>
      }
    ).cancelDrainOnEvidence.bind(tracker)
    const active = makeHost()
    const suspended = makeHost({ lifecycle: { state: 'suspended', wakeHandledGeneration: 0 } })
    const draining = makeHost({ lifecycle: { state: 'draining', wakeHandledGeneration: 0 } })

    // Cached active or suspended evidence needs no revert: settled, no write.
    await expect(cancelDrain(active.name, active)).resolves.toBe(true)
    await expect(cancelDrain(suspended.name, suspended)).resolves.toBe(true)
    expect(port.markHostActiveFromHeartbeat).not.toHaveBeenCalled()

    // Liveness witness: cached draining evidence does reach the revert write.
    await expect(cancelDrain(draining.name, draining)).resolves.toBe(true)
    expect(port.markHostActiveFromHeartbeat).toHaveBeenCalledOnce()
    expect(port.markHostActiveFromHeartbeat).toHaveBeenCalledWith(draining)
  })

  it('retries a failed suspension-blocked fresh check and still reverts cached draining evidence', async () => {
    const port = makePort()
    port.getEffectiveLifecycle.mockReturnValue({
      stateless: true,
      state: 'active',
      suspensionBlocked: true,
    })
    port.markHostActiveFromHeartbeat.mockRejectedValueOnce(new Error('api down'))
    let cachedHost = makeHost()
    const tracker = new StatelessLifecycleTracker({
      idleMinutes: 30,
      idleFloorMinutes: 15,
      drainGraceMs: 60_000,
      maxUptimeHours: 72,
      reconciler: port as unknown as StatelessLifecycleReconcilerPort,
      getHost: () => cachedHost,
      now: () => NOW,
    })
    const errors = vi.spyOn(console, 'error').mockImplementation(() => undefined)
    try {
      await tracker.handleHeartbeat(payload())
      await tracker.handleHeartbeat(payload())
      expect(port.markHostActiveFromHeartbeat).toHaveBeenCalledTimes(2)
      expect(errors).toHaveBeenCalledOnce()

      await tracker.handleHeartbeat(payload())
      expect(port.markHostActiveFromHeartbeat).toHaveBeenCalledTimes(2)

      cachedHost = makeHost({ lifecycle: { state: 'draining', wakeHandledGeneration: 0 } })
      await tracker.handleHeartbeat(payload())
      expect(port.markHostActiveFromHeartbeat).toHaveBeenCalledTimes(3)
      expect(port.markHostActiveFromHeartbeat).toHaveBeenLastCalledWith(cachedHost)
    } finally {
      errors.mockRestore()
    }
  })

  it('answers drain:false for an unknown host', async () => {
    const port = makePort()
    const tracker = new StatelessLifecycleTracker({
      idleMinutes: 30,
      idleFloorMinutes: 15,
      drainGraceMs: 60_000,
      maxUptimeHours: 72,
      reconciler: port as unknown as StatelessLifecycleReconcilerPort,
      getHost: () => undefined,
      now: () => NOW,
    })
    expect(await tracker.handleHeartbeat(payload({ lastActivityTs: NOW - 10 * HOUR }))).toEqual({
      drain: false,
    })
    expect(port.suspendHostFromHeartbeat).not.toHaveBeenCalled()
  })
})

describe('StatelessLifecycleTracker — pending wake wins', () => {
  it('answers drain:false when wakeRequested > wakeHandledGeneration even while idle', async () => {
    const port = makePort()
    port.getWakeRequestedGeneration.mockReturnValue(5)
    const tracker = makeTracker({
      port,
      host: makeHost({ lifecycle: { state: 'active', wakeHandledGeneration: 3 } }),
    })
    expect(await tracker.handleHeartbeat(payload({ lastActivityTs: NOW - 10 * HOUR }))).toEqual({
      drain: false,
    })
  })

  it('cancels a drained report when a wake is pending (no suspension)', async () => {
    const port = makePort()
    port.getWakeRequestedGeneration.mockReturnValue(2)
    const tracker = makeTracker({ port })
    const verdict = await tracker.handleHeartbeat(
      payload({ state: 'drained', lastActivityTs: NOW - 10 * HOUR })
    )
    expect(verdict).toEqual({ drain: false })
    expect(port.suspendHostFromHeartbeat).not.toHaveBeenCalled()
  })

  it('holds idle drain after a newly handled wake generation reaches the tracker', async () => {
    const port = makePort()
    const host = makeHost({ lifecycle: { state: 'active', wakeHandledGeneration: 1 } })
    const tracker = makeTracker({ port, host })

    expect(await tracker.handleHeartbeat(payload({ lastActivityTs: NOW - 10 * HOUR }))).toEqual({
      drain: true,
      entryWakeHandledGeneration: 1,
    })

    host.status = { lifecycle: { state: 'active', wakeHandledGeneration: 2 } }
    expect(
      await tracker.handleHeartbeat(payload({ podUid: 'pod-b', lastActivityTs: NOW - 10 * HOUR }))
    ).toEqual({ drain: false })
    expect(
      await tracker.handleHeartbeat(
        payload({ podUid: 'pod-b', state: 'drained', lastActivityTs: NOW - 10 * HOUR })
      )
    ).toEqual({ drain: false })
    expect(port.suspendHostFromHeartbeat).not.toHaveBeenCalled()
  })
})

describe('StatelessLifecycleTracker — cache-stale wakePending resolution', () => {
  it('proceeds with the drain when the fresh read shows the wake generation handled', async () => {
    const port = makePort()
    port.getWakeRequestedGeneration.mockReturnValue(5)
    port.readFreshHost.mockResolvedValue(
      makeHost({ lifecycle: { state: 'active', wakeHandledGeneration: 5 } })
    )
    const tracker = makeTracker({
      port,
      host: makeHost({ lifecycle: { state: 'active', wakeHandledGeneration: 3 } }),
    })
    expect(await tracker.handleHeartbeat(payload({ lastActivityTs: NOW - 10 * HOUR }))).toEqual({
      drain: true,
      entryWakeHandledGeneration: 3,
    })
    expect(port.readFreshHost).toHaveBeenCalledTimes(1)
  })

  it('keeps drain:false when the fresh read still shows the wake pending', async () => {
    const port = makePort()
    port.getWakeRequestedGeneration.mockReturnValue(5)
    port.readFreshHost.mockResolvedValue(
      makeHost({ lifecycle: { state: 'active', wakeHandledGeneration: 3 } })
    )
    const tracker = makeTracker({
      port,
      host: makeHost({ lifecycle: { state: 'active', wakeHandledGeneration: 3 } }),
    })
    expect(await tracker.handleHeartbeat(payload({ lastActivityTs: NOW - 10 * HOUR }))).toEqual({
      drain: false,
    })
    expect(port.readFreshHost).toHaveBeenCalledTimes(1)
  })

  it('keeps the conservative drain:false when the fresh read fails (retried next beat)', async () => {
    const port = makePort()
    port.getWakeRequestedGeneration.mockReturnValue(5)
    port.readFreshHost.mockRejectedValue(new Error('apiserver unavailable'))
    const tracker = makeTracker({
      port,
      host: makeHost({ lifecycle: { state: 'active', wakeHandledGeneration: 3 } }),
    })
    expect(await tracker.handleHeartbeat(payload({ lastActivityTs: NOW - 10 * HOUR }))).toEqual({
      drain: false,
    })
  })

  it('does not fresh-read at all when the cached view has no pending wake', async () => {
    const port = makePort()
    port.getWakeRequestedGeneration.mockReturnValue(2)
    const tracker = makeTracker({
      port,
      host: makeHost({ lifecycle: { state: 'active', wakeHandledGeneration: 2 } }),
    })
    expect(await tracker.handleHeartbeat(payload({ lastActivityTs: NOW - 10 * HOUR }))).toEqual({
      drain: true,
      entryWakeHandledGeneration: 2,
    })
    expect(port.readFreshHost).not.toHaveBeenCalled()
  })
})

describe('StatelessLifecycleTracker — tracker-side cancel-drain', () => {
  it('reverts a draining Host to active on activity evidence (drain:false)', async () => {
    const port = makePort()
    const host = makeHost({ lifecycle: { state: 'draining', wakeHandledGeneration: 0 } })
    const tracker = makeTracker({ port, host })
    const verdict = await tracker.handleHeartbeat(
      payload({
        activeWork: true,
        conditions: {
          activeTask: true,
          awaitingApproval: false,
          pendingResults: false,
          activeCronSchedules: false,
        },
        lastActivityTs: NOW - 1000,
      })
    )
    expect(verdict).toEqual({ drain: false })
    expect(port.markHostActiveFromHeartbeat).toHaveBeenCalledTimes(1)
    expect(port.markHostActiveFromHeartbeat).toHaveBeenCalledWith(host)
  })

  it('does not revert when the Host is already active', async () => {
    const port = makePort()
    const host = makeHost({ lifecycle: { state: 'active', wakeHandledGeneration: 0 } })
    const tracker = makeTracker({ port, host })
    const verdict = await tracker.handleHeartbeat(
      payload({
        activeWork: true,
        conditions: {
          activeTask: true,
          awaitingApproval: false,
          pendingResults: false,
          activeCronSchedules: false,
        },
        lastActivityTs: NOW - 1000,
      })
    )
    expect(verdict).toEqual({ drain: false })
    expect(port.markHostActiveFromHeartbeat).not.toHaveBeenCalled()
  })

  it('never revives a suspended Host (that is the wake fast-path job)', async () => {
    const port = makePort()
    const host = makeHost({
      lifecycle: { state: 'suspended', wakeHandledGeneration: 1, reason: 'idle' },
    })
    const tracker = makeTracker({ port, host })
    const verdict = await tracker.handleHeartbeat(
      payload({
        activeWork: true,
        conditions: {
          activeTask: true,
          awaitingApproval: false,
          pendingResults: false,
          activeCronSchedules: false,
        },
        lastActivityTs: NOW - 1000,
      })
    )
    expect(verdict).toEqual({ drain: false })
    expect(port.markHostActiveFromHeartbeat).not.toHaveBeenCalled()
  })

  it('reverts a draining Host when a wake is pending even before the fast-path lands', async () => {
    const port = makePort()
    port.getWakeRequestedGeneration.mockReturnValue(1)
    const host = makeHost({ lifecycle: { state: 'draining', wakeHandledGeneration: 0 } })
    const tracker = makeTracker({ port, host })
    // Idle payload → the standalone wakePending branch decides drain:false.
    const verdict = await tracker.handleHeartbeat(payload({ lastActivityTs: NOW - 31 * MIN }))
    expect(verdict).toEqual({ drain: false })
    expect(port.markHostActiveFromHeartbeat).toHaveBeenCalledTimes(1)
    expect(port.markHostActiveFromHeartbeat).toHaveBeenCalledWith(host)
  })

  it('keeps drain:false, logs loudly, and never throws when the revert write fails', async () => {
    const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
    try {
      const port = makePort()
      port.markHostActiveFromHeartbeat.mockRejectedValue(new Error('api down'))
      const host = makeHost({ lifecycle: { state: 'draining', wakeHandledGeneration: 0 } })
      const tracker = makeTracker({ port, host })
      const verdict = await tracker.handleHeartbeat(
        payload({
          activeWork: true,
          conditions: {
            activeTask: true,
            awaitingApproval: false,
            pendingResults: false,
            activeCronSchedules: false,
          },
          lastActivityTs: NOW - 1000,
        })
      )
      expect(verdict).toEqual({ drain: false })
      expect(
        errorSpy.mock.calls.filter(args => String(args[0]).includes('Cancel-drain revert failed'))
      ).toHaveLength(1)
    } finally {
      errorSpy.mockRestore()
    }
  })
})

describe('StatelessLifecycleTracker — podUid ordering', () => {
  it('discards heartbeats from a retired pod and logs the discard once', async () => {
    const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})
    try {
      const port = makePort()
      const tracker = makeTracker({ port })
      // pod-a is current and idle → drain:true
      expect(
        await tracker.handleHeartbeat(payload({ podUid: 'pod-a', lastActivityTs: NOW - 31 * MIN }))
      ).toEqual({ drain: true, entryWakeHandledGeneration: 0 })
      // pod-b (never seen) is a newer pod after a wake → accepted
      expect(
        await tracker.handleHeartbeat(payload({ podUid: 'pod-b', lastActivityTs: NOW - 1 * MIN }))
      ).toEqual({ drain: false })
      // pod-a is now retired: its (idle) heartbeat is discarded → drain:false
      const stale = await tracker.handleHeartbeat(
        payload({ podUid: 'pod-a', lastActivityTs: NOW - 31 * MIN })
      )
      expect(stale).toEqual({ drain: false })
      const staleLogs = warnSpy.mock.calls.filter(args =>
        String(args[0]).includes('stale pod "pod-a"')
      )
      expect(staleLogs).toHaveLength(1)
      // Repeated stale heartbeats stay discarded but are not re-logged.
      await tracker.handleHeartbeat(payload({ podUid: 'pod-a', lastActivityTs: NOW - 31 * MIN }))
      expect(
        warnSpy.mock.calls.filter(args => String(args[0]).includes('stale pod "pod-a"'))
      ).toHaveLength(1)
    } finally {
      warnSpy.mockRestore()
    }
  })
})

describe('StatelessLifecycleTracker — drained report and drain grace', () => {
  beforeEach(() => {
    vi.useFakeTimers()
  })
  afterEach(() => {
    vi.useRealTimers()
  })

  it("suspends through the reconciler on a 'drained' report (reason 'idle')", async () => {
    const port = makePort()
    const host = makeHost()
    const tracker = makeTracker({ port, host })
    const verdict = await tracker.handleHeartbeat(
      payload({ state: 'drained', lastActivityTs: NOW - 31 * MIN })
    )
    expect(verdict).toEqual({ drain: true, entryWakeHandledGeneration: 0 })
    expect(port.suspendHostFromHeartbeat).toHaveBeenCalledTimes(1)
    expect(port.suspendHostFromHeartbeat).toHaveBeenCalledWith(host, 'idle', 0)
    // Stale lastActivityTs → no fresh-activity supersede, no revert.
    expect(port.markHostActiveFromHeartbeat).not.toHaveBeenCalled()
  })

  it("passes the entry snapshot's handled generation as the AP-1 suspend epoch", async () => {
    const port = makePort()
    port.getWakeRequestedGeneration.mockReturnValue(4)
    const host = makeHost({ lifecycle: { state: 'draining', wakeHandledGeneration: 4 } })
    const tracker = makeTracker({ port, host })
    const verdict = await tracker.handleHeartbeat(
      payload({ state: 'drained', lastActivityTs: NOW - 31 * MIN })
    )
    expect(verdict).toEqual({ drain: true, entryWakeHandledGeneration: 4 })
    expect(port.suspendHostFromHeartbeat).toHaveBeenCalledTimes(1)
    // The epoch is the generation the DECISION saw (4), never a hardcoded 0 —
    // the executor's commit guard compares fresh against exactly this value.
    expect(port.suspendHostFromHeartbeat).toHaveBeenCalledWith(host, 'idle', 4)
  })

  it('suspends when the drain grace expires after an acked drain without a drained report', async () => {
    const port = makePort()
    const host = makeHost()
    const tracker = makeTracker({ port, host, drainGraceMs: 60_000 })
    expect(await tracker.handleHeartbeat(payload({ lastActivityTs: NOW - 31 * MIN }))).toEqual({
      drain: true,
      entryWakeHandledGeneration: 0,
    })
    // The emitter ACKS the drain on its next beat (state 'draining') but the
    // 'drained' report never lands → the grace expiry force-suspends.
    expect(
      await tracker.handleHeartbeat(payload({ state: 'draining', lastActivityTs: NOW - 31 * MIN }))
    ).toEqual({ drain: true, entryWakeHandledGeneration: 0 })
    expect(port.suspendHostFromHeartbeat).not.toHaveBeenCalled()
    await vi.advanceTimersByTimeAsync(60_000)
    expect(port.suspendHostFromHeartbeat).toHaveBeenCalledTimes(1)
    expect(port.suspendHostFromHeartbeat).toHaveBeenCalledWith(host, 'idle', 0)
  })

  it('cancels the drain grace when activity resumes', async () => {
    const port = makePort()
    const tracker = makeTracker({ port, drainGraceMs: 60_000 })
    expect(await tracker.handleHeartbeat(payload({ lastActivityTs: NOW - 31 * MIN }))).toEqual({
      drain: true,
      entryWakeHandledGeneration: 0,
    })
    // Activity resumed before the grace expired → verdict flips, grace disarmed.
    expect(
      await tracker.handleHeartbeat(
        payload({
          activeWork: true,
          conditions: {
            activeTask: true,
            awaitingApproval: false,
            pendingResults: false,
            activeCronSchedules: false,
          },
          lastActivityTs: NOW - 1000,
        })
      )
    ).toEqual({ drain: false })
    await vi.advanceTimersByTimeAsync(120_000)
    expect(port.suspendHostFromHeartbeat).not.toHaveBeenCalled()
  })

  it('skips the grace-expiry suspension when a wake arrived meanwhile', async () => {
    const port = makePort()
    const tracker = makeTracker({ port, drainGraceMs: 60_000 })
    expect(await tracker.handleHeartbeat(payload({ lastActivityTs: NOW - 31 * MIN }))).toEqual({
      drain: true,
      entryWakeHandledGeneration: 0,
    })
    port.getWakeRequestedGeneration.mockReturnValue(9)
    await vi.advanceTimersByTimeAsync(60_000)
    expect(port.suspendHostFromHeartbeat).not.toHaveBeenCalled()
  })
})

describe('StatelessLifecycleTracker — grace-expiry wakePending fresh-read guard (KZ-R1)', () => {
  beforeEach(() => {
    vi.useFakeTimers()
  })
  afterEach(() => {
    vi.useRealTimers()
  })

  // Drive the tracker to a force-suspend grace expiry: a drain verdict arms the
  // grace, then an acked 'draining' beat (with no 'drained' report) makes the
  // expiry force-suspend UNLESS a wake is pending. Each case controls the cached
  // vs fresh wake generation to exercise the new guard.
  async function armForceSuspendGrace(tracker: StatelessLifecycleTracker): Promise<void> {
    // Hosts differ per test (wakeHandledGeneration 0 vs 5), so the shared
    // helper matches the drain flag only; the verdict-epoch value is pinned
    // by the dedicated epoch tests.
    expect(
      await tracker.handleHeartbeat(payload({ lastActivityTs: NOW - 31 * MIN }))
    ).toMatchObject({ drain: true })
    expect(
      await tracker.handleHeartbeat(payload({ state: 'draining', lastActivityTs: NOW - 31 * MIN }))
    ).toMatchObject({ drain: true })
  }

  it('does NOT suspend when the cache says not-pending but a FRESH read shows a wake pending', async () => {
    const port = makePort()
    // Cached host: handled==requested (5) → cache reads NOT pending.
    const host = makeHost({ lifecycle: { state: 'active', wakeHandledGeneration: 5 } })
    port.getWakeRequestedGeneration.mockReturnValue(5)
    // Fresh read: handled=3 < requested 5 → the wake is actually still pending.
    port.readFreshHost.mockResolvedValue(
      makeHost({ lifecycle: { state: 'active', wakeHandledGeneration: 3 } })
    )
    const tracker = makeTracker({ port, host, drainGraceMs: 60_000 })
    await armForceSuspendGrace(tracker)

    await vi.advanceTimersByTimeAsync(60_000)
    expect(port.readFreshHost).toHaveBeenCalled()
    expect(port.suspendHostFromHeartbeat).not.toHaveBeenCalled()
  })

  it('does NOT suspend when the cache ALREADY shows a wake pending (short-circuits the fresh read)', async () => {
    const port = makePort()
    // Arm the grace with a NON-pending host (default gen 0 == requested 0), then
    // make the wake pending in-cache only at expiry — mirrors the existing
    // "wake arrived meanwhile" test so handleHeartbeat still answers drain:true.
    const tracker = makeTracker({ port, drainGraceMs: 60_000 })
    await armForceSuspendGrace(tracker)
    // Cache now reports a wake pending (requested 9 > handled 0).
    port.getWakeRequestedGeneration.mockReturnValue(9)
    port.readFreshHost.mockClear()

    await vi.advanceTimersByTimeAsync(60_000)
    // Cache pending is trusted directly — no fresh read is issued for the gate.
    expect(port.readFreshHost).not.toHaveBeenCalled()
    expect(port.suspendHostFromHeartbeat).not.toHaveBeenCalled()
  })

  it('suspends when the cache AND the fresh read agree the wake is handled (not pending)', async () => {
    const port = makePort()
    // Cached host: handled==requested (5) → not pending.
    const host = makeHost({ lifecycle: { state: 'active', wakeHandledGeneration: 5 } })
    port.getWakeRequestedGeneration.mockReturnValue(5)
    // Fresh read agrees: handled=5 → still not pending → suspend proceeds.
    port.readFreshHost.mockResolvedValue(
      makeHost({ lifecycle: { state: 'active', wakeHandledGeneration: 5 } })
    )
    const tracker = makeTracker({ port, host, drainGraceMs: 60_000 })
    await armForceSuspendGrace(tracker)

    await vi.advanceTimersByTimeAsync(60_000)
    expect(port.readFreshHost).toHaveBeenCalled()
    expect(port.suspendHostFromHeartbeat).toHaveBeenCalledTimes(1)
    expect(port.suspendHostFromHeartbeat).toHaveBeenCalledWith(host, 'idle', 5)
  })

  it('fail-conservative: a fresh-read error at grace expiry skips the suspend for this cycle', async () => {
    const port = makePort()
    // Cache not pending, but the confirming fresh read throws → never suspend
    // over a possibly-pending wake.
    const host = makeHost({ lifecycle: { state: 'active', wakeHandledGeneration: 5 } })
    port.getWakeRequestedGeneration.mockReturnValue(5)
    port.readFreshHost.mockRejectedValue(new Error('apiserver unavailable'))
    const tracker = makeTracker({ port, host, drainGraceMs: 60_000 })
    await armForceSuspendGrace(tracker)

    await vi.advanceTimersByTimeAsync(60_000)
    expect(port.readFreshHost).toHaveBeenCalled()
    expect(port.suspendHostFromHeartbeat).not.toHaveBeenCalled()
  })
})

describe('StatelessLifecycleTracker — drained report superseded by fresh activity', () => {
  it('cancels the drain instead of suspending when lastActivityTs is fresher than T_idle', async () => {
    const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {})
    try {
      const port = makePort()
      const host = makeHost({ lifecycle: { state: 'draining', wakeHandledGeneration: 0 } })
      const tracker = makeTracker({ port, host })
      // A turn was accepted and served ~5s before the emitter's final flush
      // reported 'drained' — the drain decision is stale.
      const verdict = await tracker.handleHeartbeat(
        payload({
          state: 'drained',
          lastActivityTs: NOW - 5000,
          activeWork: true,
          conditions: {
            activeTask: true,
            awaitingApproval: false,
            pendingResults: false,
            activeCronSchedules: false,
          },
        })
      )
      expect(verdict).toEqual({ drain: false })
      expect(port.suspendHostFromHeartbeat).not.toHaveBeenCalled()
      // The CR (still 'draining') reverts to active through the seam so the
      // emitter un-fences on its next answered beat.
      expect(port.markHostActiveFromHeartbeat).toHaveBeenCalledTimes(1)
      expect(port.markHostActiveFromHeartbeat).toHaveBeenCalledWith(host)
      expect(port.publishSuspendBlockedReason).toHaveBeenCalledWith(
        host,
        'SuspendBlocked: activeTask'
      )
      const supersededLines = logSpy.mock.calls
        .map(args => String(args[0]))
        .filter(line => line.includes('phase=drained_report_superseded'))
      expect(supersededLines).toEqual([
        `[StatelessSuspend] host=stateless-host phase=drained_report_superseded reason=fresh_activity ts=${NOW}`,
      ])
      tracker.stop()
    } finally {
      logSpy.mockRestore()
    }
  })

  it('suspends anyway when fresh activity coincides with an exceeded max-uptime ceiling', async () => {
    const port = makePort()
    port.findPodCreationTimestamp.mockResolvedValue(new Date(NOW - 73 * HOUR))
    const host = makeHost({ lifecycle: { state: 'draining', wakeHandledGeneration: 0 } })
    const tracker = makeTracker({ port, host, maxUptimeHours: 72 })
    const verdict = await tracker.handleHeartbeat(
      payload({ state: 'drained', lastActivityTs: NOW - 5000 })
    )
    // An uptime-exceeded drain is not superseded by activity: suspend.
    expect(verdict).toEqual({ drain: true, entryWakeHandledGeneration: 0 })
    expect(port.suspendHostFromHeartbeat).toHaveBeenCalledTimes(1)
    expect(port.suspendHostFromHeartbeat).toHaveBeenCalledWith(host, 'idle', 0)
    expect(port.markHostActiveFromHeartbeat).not.toHaveBeenCalled()
  })

  it('answers drain:false without a revert when the CR is already active', async () => {
    const port = makePort()
    const host = makeHost({ lifecycle: { state: 'active', wakeHandledGeneration: 0 } })
    const tracker = makeTracker({ port, host })
    const verdict = await tracker.handleHeartbeat(
      payload({ state: 'drained', lastActivityTs: NOW - 5000 })
    )
    expect(verdict).toEqual({ drain: false })
    expect(port.suspendHostFromHeartbeat).not.toHaveBeenCalled()
    // cancelDrainOnEvidence no-ops on a non-draining CR — no revert needed.
    expect(port.markHostActiveFromHeartbeat).not.toHaveBeenCalled()
    expect(port.publishSuspendBlockedReason).toHaveBeenCalledWith(
      host,
      'SuspendBlocked: recentActivity'
    )
  })
})

describe('StatelessLifecycleTracker — suspend-blocked reason', () => {
  it('publishes the blocking D8 condition, priority-ordered', async () => {
    const port = makePort()
    const host = makeHost()
    const tracker = makeTracker({ port, host })
    await tracker.handleHeartbeat(
      payload({
        activeWork: true,
        conditions: {
          activeTask: true,
          awaitingApproval: true,
          pendingResults: false,
          activeCronSchedules: false,
        },
      })
    )
    expect(port.publishSuspendBlockedReason).toHaveBeenLastCalledWith(
      host,
      'SuspendBlocked: awaitingApproval'
    )
    await tracker.handleHeartbeat(
      payload({
        activeWork: true,
        conditions: {
          activeTask: false,
          awaitingApproval: false,
          pendingResults: true,
          activeCronSchedules: false,
        },
      })
    )
    expect(port.publishSuspendBlockedReason).toHaveBeenLastCalledWith(
      host,
      'SuspendBlocked: pendingResults'
    )
    await tracker.handleHeartbeat(payload({ lastActivityTs: NOW - 1 * MIN }))
    expect(port.publishSuspendBlockedReason).toHaveBeenLastCalledWith(
      host,
      'SuspendBlocked: recentActivity'
    )
  })

  it('publishes awaitingApproval as the blocking D8 condition (MEDIUM-5)', async () => {
    // The hardest refusal to induce live: a heartbeat whose ONLY active-work
    // signal is conditions.awaitingApproval. With activeTask false it must be
    // the winning (priority-ordered) block reason, so the tracker publishes
    // 'SuspendBlocked: awaitingApproval' instead of suspending.
    const port = makePort()
    const host = makeHost()
    const tracker = makeTracker({ port, host })
    await tracker.handleHeartbeat(
      payload({
        activeWork: true,
        conditions: {
          activeTask: false,
          awaitingApproval: true,
          pendingResults: false,
          activeCronSchedules: false,
        },
      })
    )
    expect(port.publishSuspendBlockedReason).toHaveBeenLastCalledWith(
      host,
      'SuspendBlocked: awaitingApproval'
    )
    expect(port.suspendHostFromHeartbeat).not.toHaveBeenCalled()
  })

  it('does not publish a reason when the verdict is drain:true', async () => {
    const port = makePort()
    const tracker = makeTracker({ port })
    await tracker.handleHeartbeat(payload({ lastActivityTs: NOW - 31 * MIN }))
    expect(port.publishSuspendBlockedReason).not.toHaveBeenCalled()
  })
})

describe('StatelessLifecycleTracker — max-uptime ceiling', () => {
  it('forces drain:true past the ceiling regardless of activity (decision only)', async () => {
    const port = makePort()
    port.findPodCreationTimestamp.mockResolvedValue(new Date(NOW - 73 * HOUR))
    const tracker = makeTracker({ port, maxUptimeHours: 72 })
    const verdict = await tracker.handleHeartbeat(
      payload({
        activeWork: true,
        conditions: {
          activeTask: true,
          awaitingApproval: false,
          pendingResults: false,
          activeCronSchedules: false,
        },
        lastActivityTs: NOW - 1000,
      })
    )
    expect(verdict).toEqual({ drain: true, entryWakeHandledGeneration: 0 })
    // The ceiling only changes the VERDICT — no direct scale/suspend call.
    expect(port.suspendHostFromHeartbeat).not.toHaveBeenCalled()
  })

  it('caches the pod creationTimestamp per podUid (one lookup)', async () => {
    const port = makePort()
    const tracker = makeTracker({ port })
    await tracker.handleHeartbeat(payload())
    await tracker.handleHeartbeat(payload())
    expect(port.findPodCreationTimestamp).toHaveBeenCalledTimes(1)
  })

  it('leaves the ceiling unapplied (drain:false) when the pod age lookup fails', async () => {
    const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
    try {
      const port = makePort()
      port.findPodCreationTimestamp.mockRejectedValue(new Error('api blip'))
      const tracker = makeTracker({ port })
      expect(await tracker.handleHeartbeat(payload({ lastActivityTs: NOW - 1 * MIN }))).toEqual({
        drain: false,
      })
      expect(errorSpy).toHaveBeenCalled()
    } finally {
      errorSpy.mockRestore()
    }
  })
})

describe('StatelessLifecycleTracker — HCC restart amnesia', () => {
  it('a FRESH tracker answers drain:true from the payload alone when idle', async () => {
    const port = makePort()
    const tracker = makeTracker({ port })
    // First heartbeat this tracker instance has ever seen — the decision
    // derives from payload.lastActivityTs, not from HCC-side memory.
    expect(await tracker.handleHeartbeat(payload({ lastActivityTs: NOW - 2 * HOUR }))).toEqual({
      drain: true,
      entryWakeHandledGeneration: 0,
    })
  })

  it('a FRESH tracker never suspends immediately after restart on recent activity', async () => {
    const port = makePort()
    const tracker = makeTracker({ port })
    expect(await tracker.handleHeartbeat(payload({ lastActivityTs: NOW - 2 * MIN }))).toEqual({
      drain: false,
    })
    expect(port.suspendHostFromHeartbeat).not.toHaveBeenCalled()
  })
})

describe('StatelessLifecycleTracker — construction (fail-loud)', () => {
  it('rejects non-positive configuration', () => {
    const port = makePort()
    const base = {
      idleMinutes: 30,
      idleFloorMinutes: 15,
      drainGraceMs: 60_000,
      maxUptimeHours: 72,
      reconciler: port as unknown as StatelessLifecycleReconcilerPort,
      getHost: () => undefined,
    }
    expect(() => new StatelessLifecycleTracker({ ...base, idleMinutes: 0 })).toThrow(/idleMinutes/)
    expect(() => new StatelessLifecycleTracker({ ...base, idleFloorMinutes: -1 })).toThrow(
      /idleFloorMinutes/
    )
    expect(() => new StatelessLifecycleTracker({ ...base, drainGraceMs: 0 })).toThrow(
      /drainGraceMs/
    )
    expect(() => new StatelessLifecycleTracker({ ...base, maxUptimeHours: Number.NaN })).toThrow(
      /maxUptimeHours/
    )
  })
})

describe('StatelessLifecycleTracker — structured suspend-phase timestamps (Stage 6, W2)', () => {
  it('emits drain_answered when the idle gate first answers drain:true', async () => {
    const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {})
    try {
      const port = makePort()
      const tracker = makeTracker({ port })
      const verdict = await tracker.handleHeartbeat(payload({ lastActivityTs: NOW - 31 * MIN }))
      expect(verdict).toEqual({ drain: true, entryWakeHandledGeneration: 0 })
      const suspendLines = logSpy.mock.calls
        .map(args => String(args[0]))
        .filter(line => line.startsWith('[StatelessSuspend]'))
      expect(suspendLines).toEqual([
        `[StatelessSuspend] host=stateless-host phase=drain_answered ts=${NOW}`,
      ])
      tracker.stop()
    } finally {
      logSpy.mockRestore()
    }
  })

  it('emits drained_reported when the pod reports state=drained', async () => {
    const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {})
    try {
      const port = makePort()
      const tracker = makeTracker({ port })
      const verdict = await tracker.handleHeartbeat(
        payload({ state: 'drained', lastActivityTs: NOW - 31 * MIN })
      )
      expect(verdict).toEqual({ drain: true, entryWakeHandledGeneration: 0 })
      expect(port.suspendHostFromHeartbeat).toHaveBeenCalledTimes(1)
      const suspendLines = logSpy.mock.calls
        .map(args => String(args[0]))
        .filter(line => line.startsWith('[StatelessSuspend]'))
      expect(suspendLines).toEqual([
        `[StatelessSuspend] host=stateless-host phase=drained_reported ts=${NOW}`,
      ])
      tracker.stop()
    } finally {
      logSpy.mockRestore()
    }
  })
})

describe('StatelessLifecycleTracker — ack-aware drain-grace expiry', () => {
  beforeEach(() => {
    vi.useFakeTimers()
  })
  afterEach(() => {
    vi.useRealTimers()
  })

  /**
   * Mutable clock: `this.now()` advances in lockstep with the fake timers so
   * armedAtMs / lastBeat.atMs comparisons reflect real elapsed time across
   * grace windows (the shared makeTracker pins now() to a constant).
   */
  function makeClockTracker(options: { port: MockPort; host: HostCRD; drainGraceMs?: number }): {
    tracker: StatelessLifecycleTracker
    tick: (ms: number) => Promise<void>
  } {
    const clock = { ms: NOW }
    const tracker = new StatelessLifecycleTracker({
      idleMinutes: 30,
      idleFloorMinutes: 15,
      drainGraceMs: options.drainGraceMs ?? 60_000,
      maxUptimeHours: 72,
      reconciler: options.port as unknown as StatelessLifecycleReconcilerPort,
      getHost: () => options.host,
      now: () => clock.ms,
    })
    const tick = async (ms: number): Promise<void> => {
      clock.ms += ms
      await vi.advanceTimersByTimeAsync(ms)
    }
    return { tracker, tick }
  }

  it('re-arms instead of suspending when the emitter never acked the drain (last beat active)', async () => {
    const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {})
    try {
      const port = makePort()
      const host = makeHost()
      const { tracker, tick } = makeClockTracker({ port, host })
      expect(await tracker.handleHeartbeat(payload({ lastActivityTs: NOW - 31 * MIN }))).toEqual({
        drain: true,
        entryWakeHandledGeneration: 0,
      })
      // Grace expires while the last beat (same pod, at arming) still says
      // state:'active' — the emitter provably never heard the verdict.
      await tick(60_000)
      expect(port.suspendHostFromHeartbeat).not.toHaveBeenCalled()
      const rearmLines = logSpy.mock.calls
        .map(args => String(args[0]))
        .filter(line => line.includes('phase=grace_rearmed'))
      expect(rearmLines).toEqual([
        `[StatelessSuspend] host=stateless-host phase=grace_rearmed reason=drain_not_acked ts=${NOW + 60_000}`,
      ])
      // The emitter acks on its next beat (state 'draining') but never
      // reports drained → the re-armed grace expiry now force-suspends.
      expect(
        await tracker.handleHeartbeat(
          payload({ state: 'draining', lastActivityTs: NOW - 31 * MIN })
        )
      ).toEqual({ drain: true, entryWakeHandledGeneration: 0 })
      await tick(60_000)
      expect(port.suspendHostFromHeartbeat).toHaveBeenCalledTimes(1)
      expect(port.suspendHostFromHeartbeat).toHaveBeenCalledWith(host, 'idle', 0)
      tracker.stop()
    } finally {
      logSpy.mockRestore()
    }
  })

  it('suspends when the emitter goes silent (no beat since the re-armed window)', async () => {
    const port = makePort()
    const host = makeHost()
    const { tracker, tick } = makeClockTracker({ port, host })
    expect(await tracker.handleHeartbeat(payload({ lastActivityTs: NOW - 31 * MIN }))).toEqual({
      drain: true,
      entryWakeHandledGeneration: 0,
    })
    // First expiry: the arming beat itself (state 'active', atMs equals
    // armedAtMs) counts as "beats flowed since arming" → one re-arm. This is
    // load-bearing: with a grace shorter than the beat interval the first
    // window only ever contains the arming beat.
    await tick(60_000)
    expect(port.suspendHostFromHeartbeat).not.toHaveBeenCalled()
    // Second expiry: NO beat since the re-arm — dead/wedged emitter, nothing
    // live to kill. The FEED kept flowing (a successful poll after the
    // re-arm, recent at expiry), so the silence is attributable to the
    // EMITTER → suspend (dead-emitter path preserved).
    await tick(30_000)
    tracker.noteSuccessfulPoll(30_000)
    await tick(30_000)
    expect(port.suspendHostFromHeartbeat).toHaveBeenCalledTimes(1)
    expect(port.suspendHostFromHeartbeat).toHaveBeenCalledWith(host, 'idle', 0)
  })

  it("a retired pod's beat does not count as the current pod's ack/liveness", async () => {
    const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})
    try {
      const port = makePort()
      const host = makeHost()
      const { tracker, tick } = makeClockTracker({ port, host })
      // pod-a beats (recent activity), then pod-b replaces it → pod-a retired.
      expect(
        await tracker.handleHeartbeat(payload({ podUid: 'pod-a', lastActivityTs: NOW - 1 * MIN }))
      ).toEqual({ drain: false })
      expect(
        await tracker.handleHeartbeat(payload({ podUid: 'pod-b', lastActivityTs: NOW - 31 * MIN }))
      ).toEqual({ drain: true, entryWakeHandledGeneration: 0 })
      // First expiry re-arms off pod-b's own (active) arming beat.
      await tick(60_000)
      expect(port.suspendHostFromHeartbeat).not.toHaveBeenCalled()
      // A STALE pod-a beat (state 'active') arrives inside the re-armed
      // window: discarded — it must NOT register as pod-b's ack/liveness.
      expect(
        await tracker.handleHeartbeat(payload({ podUid: 'pod-a', lastActivityTs: NOW - 1 * MIN }))
      ).toEqual({ drain: false })
      // Had the stale beat counted, this expiry would re-arm again; instead
      // pod-b has been silent since the re-arm while the FEED kept flowing
      // (successful poll after the re-arm) → suspend.
      await tick(30_000)
      tracker.noteSuccessfulPoll(30_000)
      await tick(30_000)
      expect(port.suspendHostFromHeartbeat).toHaveBeenCalledTimes(1)
      expect(port.suspendHostFromHeartbeat).toHaveBeenCalledWith(host, 'idle', 0)
    } finally {
      warnSpy.mockRestore()
    }
  })

  it('re-arm followed by an activity beat cancels the drain — no suspend ever', async () => {
    const port = makePort()
    const host = makeHost({ lifecycle: { state: 'draining', wakeHandledGeneration: 0 } })
    const { tracker, tick } = makeClockTracker({ port, host })
    expect(await tracker.handleHeartbeat(payload({ lastActivityTs: NOW - 31 * MIN }))).toEqual({
      drain: true,
      entryWakeHandledGeneration: 0,
    })
    // Grace expires unacked → re-arm, no suspend.
    await tick(60_000)
    expect(port.suspendHostFromHeartbeat).not.toHaveBeenCalled()
    // The next beat carries activity → drain cancelled via the existing
    // paths: grace cleared + CR reverted to active through the seam.
    expect(
      await tracker.handleHeartbeat(
        payload({
          activeWork: true,
          conditions: {
            activeTask: true,
            awaitingApproval: false,
            pendingResults: false,
            activeCronSchedules: false,
          },
          lastActivityTs: NOW + 60_000 - 1000,
        })
      )
    ).toEqual({ drain: false })
    expect(port.markHostActiveFromHeartbeat).toHaveBeenCalledTimes(1)
    await tick(300_000)
    expect(port.suspendHostFromHeartbeat).not.toHaveBeenCalled()
  })

  it('FIX 4: a feed outage across the grace re-arms (reason=feed_outage) instead of suspending; the recovered feed suspends', async () => {
    const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {})
    try {
      const port = makePort()
      const host = makeHost()
      const { tracker, tick } = makeClockTracker({ port, host })
      expect(await tracker.handleHeartbeat(payload({ lastActivityTs: NOW - 31 * MIN }))).toEqual({
        drain: true,
        entryWakeHandledGeneration: 0,
      })
      // First expiry: arming beat counts as flowing → re-arm (drain_not_acked).
      await tick(60_000)
      expect(port.suspendHostFromHeartbeat).not.toHaveBeenCalled()
      // Second expiry: NO beat since the re-arm, and NO successful poll was
      // ever reported — the silence is unattributable (the feed may be out,
      // the emitter may be alive and beating into the void) → re-arm, never
      // force-suspend on feed silence.
      await tick(60_000)
      expect(port.suspendHostFromHeartbeat).not.toHaveBeenCalled()
      const feedOutageLines = logSpy.mock.calls
        .map(args => String(args[0]))
        .filter(line => line.includes('reason=feed_outage'))
      expect(feedOutageLines).toEqual([
        `[StatelessSuspend] host=stateless-host phase=grace_rearmed reason=feed_outage ts=${NOW + 120_000}`,
      ])
      // Feed recovers: a successful poll after the latest re-arm, recent at
      // the next expiry — the emitter is still silent → NOW it suspends.
      await tick(30_000)
      tracker.noteSuccessfulPoll(30_000)
      await tick(30_000)
      expect(port.suspendHostFromHeartbeat).toHaveBeenCalledTimes(1)
      expect(port.suspendHostFromHeartbeat).toHaveBeenCalledWith(host, 'idle', 0)
      tracker.stop()
    } finally {
      logSpy.mockRestore()
    }
  })
})

describe('StatelessLifecycleTracker — grace-expiry ack-aware re-arm over live pending work (C2)', () => {
  beforeEach(() => {
    vi.useFakeTimers()
  })
  afterEach(() => {
    vi.useRealTimers()
  })

  // A genuine pending intake keeps the emitter fenced in state:'draining' while
  // still reporting activeWork:true. onDrainGraceExpired consults the cached
  // last beat; before C2 it re-armed ONLY on state==='active', so a
  // draining+activeWork:true last beat would force-suspend over live work.
  // We keep the grace armed for the activeWork beat by making the pod exceed
  // max-uptime (so the non-idle beat still arms rather than cancelling), which
  // isolates the activeWork re-arm decision from the wakePending/idle paths.
  it('re-arms (does NOT suspend) when the last beat is draining + activeWork:true from the same pod after arming', async () => {
    const port = makePort()
    const host = makeHost()
    // Pod is old → max-uptime exceeded, so a non-idle (activeWork:true) beat
    // still answers drain:true and keeps the grace armed instead of cancelling.
    port.findPodCreationTimestamp.mockResolvedValue(new Date(NOW - 100 * HOUR))
    const tracker = makeTracker({ port, host, drainGraceMs: 60_000, maxUptimeHours: 72 })

    // Beat 1: arm the grace (uptime-exceeded → drain:true).
    expect(await tracker.handleHeartbeat(payload({ lastActivityTs: NOW - 1000 }))).toEqual({
      drain: true,
      entryWakeHandledGeneration: 0,
    })
    // Beat 2: the emitter acked (state 'draining') but a pending intake keeps
    // activeWork:true. Uptime-exceeded keeps the grace armed and records the
    // activeWork:true beat as the last beat.
    expect(
      await tracker.handleHeartbeat(
        payload({
          state: 'draining',
          activeWork: true,
          conditions: {
            activeTask: true,
            awaitingApproval: false,
            pendingResults: false,
            activeCronSchedules: false,
          },
          lastActivityTs: NOW - 1000,
        })
      )
    ).toEqual({ drain: true, entryWakeHandledGeneration: 0 })

    await vi.advanceTimersByTimeAsync(60_000)

    // C2: the grace expiry re-armed over live pending work — NO suspend.
    expect(port.suspendHostFromHeartbeat).not.toHaveBeenCalled()
  })

  it('still suspends when the last beat is draining + activeWork:false (acked, no live work) — existing behavior preserved', async () => {
    const port = makePort()
    const host = makeHost()
    const tracker = makeTracker({ port, host, drainGraceMs: 60_000 })

    // Beat 1: idle → arm the grace.
    expect(await tracker.handleHeartbeat(payload({ lastActivityTs: NOW - 31 * MIN }))).toEqual({
      drain: true,
      entryWakeHandledGeneration: 0,
    })
    // Beat 2: acked drain (state 'draining') with NO active work and stale
    // activity → the emitter is idle+acked, the drained report just never
    // landed. This is the legitimate force-suspend case.
    expect(
      await tracker.handleHeartbeat(payload({ state: 'draining', lastActivityTs: NOW - 31 * MIN }))
    ).toEqual({ drain: true, entryWakeHandledGeneration: 0 })

    await vi.advanceTimersByTimeAsync(60_000)

    expect(port.suspendHostFromHeartbeat).toHaveBeenCalledTimes(1)
    expect(port.suspendHostFromHeartbeat).toHaveBeenCalledWith(host, 'idle', 0)
  })
})

describe('StatelessLifecycleTracker — stale suspend commit answers drain:false (FIX 2)', () => {
  it("answers drain:false and logs when the suspend outcome is 'skipped_stale'", async () => {
    const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {})
    try {
      const port = makePort()
      // The executor's commit-point guards refused the drained evidence as
      // aged (wake handled past the epoch / wake pending / drain overturned).
      port.suspendHostFromHeartbeat.mockResolvedValue('skipped_stale')
      const host = makeHost({ lifecycle: { state: 'draining', wakeHandledGeneration: 0 } })
      const tracker = makeTracker({ port, host })
      const verdict = await tracker.handleHeartbeat(
        payload({ state: 'drained', lastActivityTs: NOW - 31 * MIN })
      )
      // drain:false → the pod UN-FENCES; answering drain:true here would
      // re-fence a just-woken pod on evidence the commit itself rejected.
      expect(verdict).toEqual({ drain: false })
      expect(port.suspendHostFromHeartbeat).toHaveBeenCalledTimes(1)
      const skippedLines = logSpy.mock.calls
        .map(args => String(args[0]))
        .filter(line => line.includes('phase=drained_suspend_skipped_stale'))
      expect(skippedLines).toEqual([
        `[StatelessSuspend] host=stateless-host phase=drained_suspend_skipped_stale ts=${NOW}`,
      ])
      tracker.stop()
    } finally {
      logSpy.mockRestore()
    }
  })

  it("keeps drain:true for the idempotent already-suspended retry ('already_suspended')", async () => {
    const port = makePort()
    port.suspendHostFromHeartbeat.mockResolvedValue('already_suspended')
    const host = makeHost({ lifecycle: { state: 'suspended', wakeHandledGeneration: 2 } })
    const tracker = makeTracker({ port, host })
    const verdict = await tracker.handleHeartbeat(
      payload({ state: 'drained', lastActivityTs: NOW - 31 * MIN })
    )
    expect(verdict).toEqual({ drain: true, entryWakeHandledGeneration: 2 })
    tracker.stop()
  })

  it("keeps drain:true when the commit lands ('suspended')", async () => {
    const port = makePort()
    port.suspendHostFromHeartbeat.mockResolvedValue('suspended')
    const host = makeHost({ lifecycle: { state: 'draining', wakeHandledGeneration: 0 } })
    const tracker = makeTracker({ port, host })
    const verdict = await tracker.handleHeartbeat(
      payload({ state: 'drained', lastActivityTs: NOW - 31 * MIN })
    )
    expect(verdict).toEqual({ drain: true, entryWakeHandledGeneration: 0 })
    tracker.stop()
  })
})

describe('StatelessLifecycleTracker — grace-expiry ownership re-validation (FIX 3)', () => {
  beforeEach(() => {
    vi.useFakeTimers()
  })
  afterEach(() => {
    vi.useRealTimers()
  })

  it('aborts the force-suspend when the pod is replaced during the fresh-read await', async () => {
    const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})
    try {
      const port = makePort()
      const host = makeHost()
      let resolveFresh: ((h: HostCRD) => void) | undefined
      const tracker = makeTracker({ port, host, drainGraceMs: 60_000 })
      // Arm + ack so the expiry would legitimately force-suspend pod-a.
      expect(
        await tracker.handleHeartbeat(payload({ podUid: 'pod-a', lastActivityTs: NOW - 31 * MIN }))
      ).toEqual({ drain: true, entryWakeHandledGeneration: 0 })
      expect(
        await tracker.handleHeartbeat(
          payload({ podUid: 'pod-a', state: 'draining', lastActivityTs: NOW - 31 * MIN })
        )
      ).toEqual({ drain: true, entryWakeHandledGeneration: 0 })
      // The expiry's fresh read HANGS — a pod replacement lands meanwhile.
      port.readFreshHost.mockImplementation(
        () =>
          new Promise<HostCRD>(res => {
            resolveFresh = res
          })
      )
      await vi.advanceTimersByTimeAsync(60_000)
      expect(resolveFresh).toBeDefined()
      // pod-b (a fresh roll, same creationTimestamp granularity) replaces
      // pod-a while the expiry is parked on its await.
      expect(
        await tracker.handleHeartbeat(payload({ podUid: 'pod-b', lastActivityTs: NOW - 1 * MIN }))
      ).toEqual({ drain: false })
      resolveFresh?.(makeHost())
      await vi.advanceTimersByTimeAsync(0)
      // The aged expiry MUST abort: pod-b owns the decision now. Suspending
      // would kill the live replacement pod on misattributed silence.
      expect(port.suspendHostFromHeartbeat).not.toHaveBeenCalled()
      const abortLines = warnSpy.mock.calls
        .map(args => String(args[0]))
        .filter(line => line.includes('aborted after the fresh read'))
      expect(abortLines).toHaveLength(1)
      tracker.stop()
    } finally {
      warnSpy.mockRestore()
    }
  })

  it('aborts the force-suspend when a new grace was re-armed during the fresh-read await', async () => {
    const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})
    try {
      const port = makePort()
      const host = makeHost()
      let resolveFresh: ((h: HostCRD) => void) | undefined
      const tracker = makeTracker({ port, host, drainGraceMs: 60_000 })
      expect(
        await tracker.handleHeartbeat(payload({ podUid: 'pod-a', lastActivityTs: NOW - 31 * MIN }))
      ).toEqual({ drain: true, entryWakeHandledGeneration: 0 })
      expect(
        await tracker.handleHeartbeat(
          payload({ podUid: 'pod-a', state: 'draining', lastActivityTs: NOW - 31 * MIN })
        )
      ).toEqual({ drain: true, entryWakeHandledGeneration: 0 })
      port.readFreshHost.mockImplementation(
        () =>
          new Promise<HostCRD>(res => {
            resolveFresh = res
          })
      )
      await vi.advanceTimersByTimeAsync(60_000)
      expect(resolveFresh).toBeDefined()
      // A NEW drain:true beat re-arms a fresh grace while the aged expiry is
      // parked — the new arming owns the decision (its own window must run).
      expect(
        await tracker.handleHeartbeat(payload({ podUid: 'pod-a', lastActivityTs: NOW - 31 * MIN }))
      ).toEqual({ drain: true, entryWakeHandledGeneration: 0 })
      resolveFresh?.(makeHost())
      await vi.advanceTimersByTimeAsync(0)
      expect(port.suspendHostFromHeartbeat).not.toHaveBeenCalled()
      const abortLines = warnSpy.mock.calls
        .map(args => String(args[0]))
        .filter(line => line.includes('aborted after the fresh read'))
      expect(abortLines).toHaveLength(1)
      tracker.stop()
    } finally {
      warnSpy.mockRestore()
    }
  })
})

describe('StatelessLifecycleTracker — out-of-order pod replay tie-break (FIX 5)', () => {
  it('a straggler beat of an OLDER pod never retires the live current pod', async () => {
    const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})
    try {
      const port = makePort()
      port.findPodCreationTimestamp.mockImplementation(async (_host: HostCRD, uid: string) =>
        uid === 'pod-old' ? new Date(NOW - 10 * MIN) : new Date(NOW - 1 * MIN)
      )
      const tracker = makeTracker({ port })
      // The live pod beats first (post-HCC-restart, fresh tracker).
      expect(
        await tracker.handleHeartbeat(payload({ podUid: 'pod-new', lastActivityTs: NOW - 1 * MIN }))
      ).toEqual({ drain: false })
      // An out-of-order replay of the OLD pod arrives: never seen, but
      // provably older — it must NOT become current.
      expect(
        await tracker.handleHeartbeat(
          payload({ podUid: 'pod-old', lastActivityTs: NOW - 31 * MIN })
        )
      ).toEqual({ drain: false })
      expect(
        warnSpy.mock.calls.filter(args =>
          String(args[0]).includes('OLDER than the current pod "pod-new"')
        )
      ).toHaveLength(1)
      // The live pod still steers the lifecycle (not retired, not wedged).
      expect(
        await tracker.handleHeartbeat(
          payload({ podUid: 'pod-new', lastActivityTs: NOW - 31 * MIN })
        )
      ).toEqual({ drain: true, entryWakeHandledGeneration: 0 })
      // The straggler itself was retired: its next beat hits the retired
      // discard path (logged once).
      await tracker.handleHeartbeat(payload({ podUid: 'pod-old', lastActivityTs: NOW - 31 * MIN }))
      expect(
        warnSpy.mock.calls.filter(args => String(args[0]).includes('stale pod "pod-old"'))
      ).toHaveLength(1)
      tracker.stop()
    } finally {
      warnSpy.mockRestore()
    }
  })

  it('a genuinely newer pod is adopted as current (wake/roll behavior preserved)', async () => {
    const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})
    try {
      const port = makePort()
      port.findPodCreationTimestamp.mockImplementation(async (_host: HostCRD, uid: string) =>
        uid === 'pod-b' ? new Date(NOW - 1 * MIN) : new Date(NOW - 10 * MIN)
      )
      const tracker = makeTracker({ port })
      expect(
        await tracker.handleHeartbeat(payload({ podUid: 'pod-a', lastActivityTs: NOW - 1 * MIN }))
      ).toEqual({ drain: false })
      // pod-b is provably newer → adopted; pod-a retired.
      expect(
        await tracker.handleHeartbeat(payload({ podUid: 'pod-b', lastActivityTs: NOW - 1 * MIN }))
      ).toEqual({ drain: false })
      await tracker.handleHeartbeat(payload({ podUid: 'pod-a', lastActivityTs: NOW - 1 * MIN }))
      expect(
        warnSpy.mock.calls.filter(args => String(args[0]).includes('stale pod "pod-a"'))
      ).toHaveLength(1)
      tracker.stop()
    } finally {
      warnSpy.mockRestore()
    }
  })

  it('an ordering lookup failure keeps the current pod, retires nobody, and retries next beat', async () => {
    const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})
    const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
    try {
      const port = makePort()
      const tracker = makeTracker({ port })
      expect(
        await tracker.handleHeartbeat(payload({ podUid: 'pod-a', lastActivityTs: NOW - 1 * MIN }))
      ).toEqual({ drain: false })
      // Lookups fail while the never-seen pod-b beats: conservative verdict.
      port.findPodCreationTimestamp.mockRejectedValue(new Error('api blip'))
      expect(
        await tracker.handleHeartbeat(payload({ podUid: 'pod-b', lastActivityTs: NOW - 1 * MIN }))
      ).toEqual({ drain: false })
      expect(
        warnSpy.mock.calls.filter(args =>
          String(args[0]).includes('Could not order never-seen pod')
        )
      ).toHaveLength(1)
      // pod-a is STILL current (not retired): its beat is processed normally.
      expect(
        await tracker.handleHeartbeat(payload({ podUid: 'pod-a', lastActivityTs: NOW - 31 * MIN }))
      ).toEqual({ drain: true, entryWakeHandledGeneration: 0 })
      // Lookup recovers with pod-b provably newer → the retry adopts it.
      port.findPodCreationTimestamp.mockImplementation(async (_host: HostCRD, uid: string) =>
        uid === 'pod-b' ? new Date(NOW - 1 * MIN) : new Date(NOW - 10 * MIN)
      )
      expect(
        await tracker.handleHeartbeat(payload({ podUid: 'pod-b', lastActivityTs: NOW - 1 * MIN }))
      ).toEqual({ drain: false })
      await tracker.handleHeartbeat(payload({ podUid: 'pod-a', lastActivityTs: NOW - 1 * MIN }))
      expect(
        warnSpy.mock.calls.filter(args => String(args[0]).includes('stale pod "pod-a"'))
      ).toHaveLength(1)
      tracker.stop()
    } finally {
      warnSpy.mockRestore()
      errorSpy.mockRestore()
    }
  })
})

describe('parseHeartbeatPayload — activeCronSchedules (cron×stateless, additive)', () => {
  it('accepts an explicit boolean and preserves it', () => {
    const result = parseHeartbeatPayload(
      payload({
        conditions: {
          activeTask: false,
          awaitingApproval: false,
          pendingResults: false,
          activeCronSchedules: true,
        },
      })
    )
    expect(result.ok).toBe(true)
    if (result.ok) {
      expect(result.payload.conditions.activeCronSchedules).toBe(true)
    }
  })

  it('defaults to false when absent so older emitters keep validating', () => {
    const legacy = {
      ...payload(),
      conditions: { activeTask: false, awaitingApproval: false, pendingResults: false },
    }
    const result = parseHeartbeatPayload(legacy)
    expect(result.ok).toBe(true)
    if (result.ok) {
      expect(result.payload.conditions.activeCronSchedules).toBe(false)
    }
  })

  it('rejects a present-but-non-boolean value loudly', () => {
    const bad = {
      ...payload(),
      conditions: {
        activeTask: false,
        awaitingApproval: false,
        pendingResults: false,
        activeCronSchedules: 'yes',
      },
    }
    expect(parseHeartbeatPayload(bad)).toEqual({
      ok: false,
      error: 'conditions.activeCronSchedules must be a boolean when present',
    })
  })
})

describe('StatelessLifecycleTracker — activeCronSchedules blocks suspension (cron×stateless)', () => {
  it('publishes SuspendBlocked: activeCronSchedules while a schedule pins activeWork', async () => {
    const port = makePort()
    const tracker = makeTracker({ port })
    const verdict = await tracker.handleHeartbeat(
      payload({
        activeWork: true,
        conditions: {
          activeTask: false,
          awaitingApproval: false,
          pendingResults: false,
          activeCronSchedules: true,
        },
        // Long idle — ONLY the schedule blocks the drain decision.
        lastActivityTs: NOW - 10 * HOUR,
      })
    )
    expect(verdict).toEqual({ drain: false })
    expect(port.suspendHostFromHeartbeat).not.toHaveBeenCalled()
    expect(port.publishSuspendBlockedReason).toHaveBeenCalledWith(
      expect.objectContaining({ name: 'stateless-host' }),
      'SuspendBlocked: activeCronSchedules'
    )
  })
})