Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
45.4 KB
Raw Blame
/**
 * Stage 3 (stateless-agents) — heartbeat lifecycle tracker + D8 idle gate.
 *
 * Consumes authenticated heartbeats from stateless mcp-host pods
 * (`POST /api/v1/hosts/:hostRef/heartbeat`) and DECIDES the drain verdict;
 * EXECUTION (status writes, replicas derivation) stays in HostReconciler
 * behind the `StatelessLifecycleReconcilerPort` seam.
 *
 * Decision rules:
 *   - Idle (D8): `drain = statelessEnabled && !activeWork &&
 *     (now - payload.lastActivityTs >= T_idle)` with
 *     `T_idle = max(idleMinutes, idleFloorMinutes)`. The decision derives
 *     from the PAYLOAD's lastActivityTs, so it survives HCC restarts — a
 *     fresh tracker answers correctly on the very first heartbeat.
 *   - Max-uptime ceiling: a pod older than `maxUptimeHours` (from the pod's
 *     creationTimestamp, resolved once per podUid through the reconciler)
 *     drains regardless of activity.
 *   - Pending wake wins: when the Host's wake-requested generation exceeds
 *     status.lifecycle.wakeHandledGeneration the answer is ALWAYS
 *     `drain: false` (cancel-drain) and no suspension is executed.
 *   - Tracker-side cancel-drain: a `drain: false` verdict backed by host
 *     evidence — activity (activeWork / recent lastActivityTs) or a pending
 *     wake — while the CR still says `draining` REVERTS
 *     status.lifecycle.state to `active` through the reconciler seam, so
 *     control-api stops answering {drain:true} and the emitter's fence
 *     lifts even if the rpc-proxy wake call never lands. The wake fast-path
 *     remains the only reviver of a `suspended` Host.
 *   - `state: 'drained'` asks the reconciler to suspend
 *     (status.lifecycle.state='suspended', reason 'idle') and reconcile —
 *     UNLESS the report's own lastActivityTs is fresher than T_idle (a turn
 *     was accepted and served after the drain verdict but before the emitter
 *     fenced, so the idle decision is stale): the drained report is
 *     SUPERSEDED — the drain is cancelled (CR reverts to `active`, the
 *     emitter un-fences) and the verdict is drain:false. Max-uptime drains
 *     are exempt: an over-age pod suspends regardless of activity.
 *   - Drain-grace expiry is ACK-AWARE: after a `drain: true` verdict the
 *     grace only force-suspends when the emitter ACKED the drain (its last
 *     beat reported 'draining'/'drained' but the drained report never
 *     landed) or went silent (no beat since arming). If beats kept flowing
 *     with state 'active' — the emitter provably never heard the verdict —
 *     the grace RE-ARMS instead of killing in-flight work. Silence is only
 *     attributed to the EMITTER when the heartbeat FEED provably flowed
 *     across the window (AP-3): the poller reports each fully-successful
 *     poll via noteSuccessfulPoll, and an expiry that observes no such poll
 *     strictly after arming (and recent enough) re-arms instead — a
 *     control-api/poller outage must never make a live emitter look silent.
 *
 * podUid ordering: heartbeats are keyed by (hostRef, podUid). A heartbeat
 * from a podUid that was RETIRED (seen before the current pod) is discarded
 * and logged once — a stale pod that outlives its replacement must never
 * steer the lifecycle. A never-seen podUid while a current pod exists is
 * NOT automatically a newer pod: after an HCC restart a straggler beat of
 * the OLD pod can replay after the new pod's first beat, so adoption is
 * tie-broken on the pods' immutable creationTimestamps (a strictly-older
 * newcomer is itself retired; an unresolvable order keeps the current pod).
 */
import { LIFECYCLE_REASON_SUSPENDED_IDLE, SUSPEND_BLOCKED_REASON_PREFIX } from './constants'
import { EffectiveHostLifecycle, SuspendFromHeartbeatOutcome } from './statelessLifecycle.types'
import { HostCRD } from './types'

export type HeartbeatState = 'active' | 'draining' | 'drained'

/** D8 activity conditions — mirrors mcp-host's heartbeat emitter payload. */
export interface HeartbeatConditions {
  activeTask: boolean
  awaitingApproval: boolean
  pendingResults: boolean
  /**
   * Cron×stateless: the emitter's CronScheduler holds >=1 ENABLED schedule.
   * Additive (schemaVersion stays 1) — absent on older emitters parses as
   * false. Surfaces as `SuspendBlocked: activeCronSchedules`.
   */
  activeCronSchedules: boolean
}

export interface HeartbeatPayload {
  schemaVersion: 1
  hostRef: string
  podUid: string
  activeWork: boolean
  conditions: HeartbeatConditions
  /** Epoch ms of the last inbound runtime message / turn activity. */
  lastActivityTs: number
  state: HeartbeatState
}

export type HeartbeatPayloadParseResult =
  | { ok: true; payload: HeartbeatPayload }
  | { ok: false; error: string }

const HEARTBEAT_STATES: ReadonlySet<string> = new Set(['active', 'draining', 'drained'])

/**
 * Strict shape validation for the heartbeat body. Runs BEFORE any auth
 * crypto so malformed requests are rejected at 400 without paying for a
 * signature verification. Unknown extra fields are tolerated (forward
 * compatibility); every REQUIRED field is type-checked exactly.
 */
export function parseHeartbeatPayload(raw: unknown): HeartbeatPayloadParseResult {
  if (typeof raw !== 'object' || raw === null || Array.isArray(raw)) {
    return { ok: false, error: 'body must be a JSON object' }
  }
  const body = raw as Record<string, unknown>
  if (body.schemaVersion !== 1) {
    return { ok: false, error: 'schemaVersion must be 1' }
  }
  if (typeof body.hostRef !== 'string' || !body.hostRef.trim()) {
    return { ok: false, error: 'hostRef must be a non-empty string' }
  }
  if (typeof body.podUid !== 'string' || !body.podUid.trim()) {
    return { ok: false, error: 'podUid must be a non-empty string' }
  }
  if (typeof body.activeWork !== 'boolean') {
    return { ok: false, error: 'activeWork must be a boolean' }
  }
  const conditions = body.conditions
  if (typeof conditions !== 'object' || conditions === null || Array.isArray(conditions)) {
    return { ok: false, error: 'conditions must be an object' }
  }
  const cond = conditions as Record<string, unknown>
  for (const key of ['activeTask', 'awaitingApproval', 'pendingResults'] as const) {
    if (typeof cond[key] !== 'boolean') {
      return { ok: false, error: `conditions.${key} must be a boolean` }
    }
  }
  // Cron×stateless (additive, schemaVersion stays 1): older emitters omit the
  // field — absent parses as false so their payloads keep validating.
  // Present-but-wrong-type is still rejected loudly.
  if ('activeCronSchedules' in cond && typeof cond.activeCronSchedules !== 'boolean') {
    return { ok: false, error: 'conditions.activeCronSchedules must be a boolean when present' }
  }
  if (
    typeof body.lastActivityTs !== 'number' ||
    !Number.isFinite(body.lastActivityTs) ||
    body.lastActivityTs < 0
  ) {
    return { ok: false, error: 'lastActivityTs must be a non-negative finite number (epoch ms)' }
  }
  if (typeof body.state !== 'string' || !HEARTBEAT_STATES.has(body.state)) {
    return { ok: false, error: "state must be one of 'active' | 'draining' | 'drained'" }
  }
  return {
    ok: true,
    payload: {
      schemaVersion: 1,
      hostRef: body.hostRef,
      podUid: body.podUid,
      activeWork: body.activeWork,
      conditions: {
        activeTask: cond.activeTask as boolean,
        awaitingApproval: cond.awaitingApproval as boolean,
        pendingResults: cond.pendingResults as boolean,
        activeCronSchedules: cond.activeCronSchedules === true,
      },
      lastActivityTs: body.lastActivityTs,
      state: body.state as HeartbeatState,
    },
  }
}

export type HeartbeatVerdict =
  | { drain: false }
  | {
      drain: true
      /**
       * AP-1 generation epoch: the `wakeHandledGeneration` observed on the
       * tracker's Host snapshot when this drain verdict was DECIDED. The
       * poller threads it into the durable draining write
       * (markHostDrainingFromHeartbeat), whose commit no-ops when the fresh
       * generation has advanced past it — an aged drain:true verdict must
       * never re-fence a pod whose wake was handled after the decision.
       */
      entryWakeHandledGeneration: number
    }

/**
 * Minimal view the HTTP server needs. `ContextMapperServer` depends on this
 * interface, not on the concrete tracker.
 */
export interface HeartbeatLifecycleTracker {
  handleHeartbeat(payload: HeartbeatPayload): Promise<HeartbeatVerdict>
  /**
   * Feed-health seam (AP-3): the heartbeat poller reports every FULLY
   * successful poll cycle (rows fetched, parsed and applied end to end).
   * The ack-aware drain-grace expiry consults it to distinguish emitter
   * silence (no beat while the feed flowed) from feed silence (poller /
   * control-api outage — nothing could have been observed): only the former
   * may force-suspend. Optional so minimal fakes stay valid.
   */
  noteSuccessfulPoll?(pollIntervalMs: number): void
  stop(): void
}

/**
 * Execution seam into HostReconciler: the tracker DECIDES, the reconciler
 * EXECUTES (status subresource writes + full reconcile → replicas
 * derivation). HostReconciler implements this structurally.
 */
export interface StatelessLifecycleReconcilerPort {
  getEffectiveLifecycle(host: HostCRD): EffectiveHostLifecycle
  getWakeRequestedGeneration(host: HostCRD): number
  /**
   * Fresh (non-cache) read of the Host CR from the API server. The tracker
   * uses it to resolve a cached-pending wake verdict against server-side
   * truth (see the wakePending resolution in handleHeartbeat): the informer
   * cache can miss the wake fast-path's status flip, and a stale pending
   * verdict blocks every drain until the periodic resync.
   */
  readFreshHost(host: HostCRD): Promise<HostCRD>
  /**
   * Suspend after a drained report or drain-grace expiry.
   * `entryWakeHandledGeneration` is the `wakeHandledGeneration` observed on
   * the tracker's Host snapshot when the suspend was DECIDED (AP-1 epoch):
   * the executor no-ops the commit when the fresh generation has advanced
   * past it — a wake/cancel-drain was handled after the decision, so the
   * drained evidence is aged. The returned outcome tells the caller whether
   * the commit landed ('suspended'), was an idempotent retry over an
   * already-suspended CR ('already_suspended'), or was refused as stale
   * ('skipped_stale') — in the stale case the tracker answers drain:false
   * so the pod un-fences instead of staying fenced behind aged evidence.
   */
  suspendHostFromHeartbeat(
    host: HostCRD,
    reason: string,
    entryWakeHandledGeneration: number
  ): Promise<SuspendFromHeartbeatOutcome>
  markHostActiveFromHeartbeat(host: HostCRD): Promise<void>
  publishSuspendBlockedReason(host: HostCRD, reason: string): Promise<void>
  findPodCreationTimestamp(host: HostCRD, podUid: string): Promise<Date | null>
}

export interface StatelessLifecycleTrackerOptions {
  /** CONTEXT_MAPPER_STATELESS_IDLE_MINUTES (default 30). */
  idleMinutes: number
  /** CONTEXT_MAPPER_STATELESS_IDLE_FLOOR_MINUTES (default 15). */
  idleFloorMinutes: number
  /** CONTEXT_MAPPER_STATELESS_DRAIN_GRACE_MS (default 60000). */
  drainGraceMs: number
  /** CONTEXT_MAPPER_STATELESS_MAX_UPTIME_HOURS (default 72). */
  maxUptimeHours: number
  reconciler: StatelessLifecycleReconcilerPort
  /** Host CRD lookup from the informer cache (McpServerWatcher.getHost). */
  getHost: (hostRef: string) => HostCRD | undefined
  /** Injectable clock for tests (epoch ms). */
  now?: () => number
}

/** Cap on remembered retired podUids per host (FIFO on Set insertion order). */
const MAX_RETIRED_POD_UIDS_PER_HOST = 16

export class StatelessLifecycleTracker implements HeartbeatLifecycleTracker {
  private readonly idleMs: number
  private readonly drainGraceMs: number
  private readonly maxUptimeMs: number
  private readonly reconciler: StatelessLifecycleReconcilerPort
  private readonly getHost: (hostRef: string) => HostCRD | undefined
  private readonly now: () => number

  /** Current (most recently accepted) podUid per host. */
  private readonly currentPodByHost = new Map<string, string>()
  /** Retired podUids per host — heartbeats from these are discarded. */
  private readonly retiredPodUids = new Map<string, Set<string>>()
  /** `${hostRef}/${podUid}` keys whose stale-discard has been logged once. */
  private readonly stalePodLogged = new Set<string>()
  /** Pod creationTimestamp cache (immutable per podUid), key `${hostRef}/${podUid}`. */
  private readonly podCreationTsByUid = new Map<string, number>()
  /** Armed drain-grace timers per host. */
  private readonly drainGrace = new Map<
    string,
    { podUid: string; timer: ReturnType<typeof setTimeout>; armedAtMs: number }
  >()
  /**
   * Last accepted (non-retired) beat per host: emitter-reported state + time,
   * plus the activity signals the ack-aware drain-grace expiry consults.
   * `activeWork`/`lastActivityTs` are recorded (C2) so onDrainGraceExpired can
   * re-arm over LIVE PENDING WORK even when the emitter is still fenced in
   * state:'draining' -- a genuine pending intake keeps activeWork:true while
   * state stays 'draining', which the state==='active' re-arm test alone would
   * miss and suspend over.
   */
  private readonly lastBeatByHost = new Map<
    string,
    {
      podUid: string
      state: HeartbeatState
      atMs: number
      activeWork: boolean
      lastActivityTs: number
    }
  >()
  /** Last handled wake generation observed by this tracker process per host. */
  private readonly lastWakeHandledGenerationByHost = new Map<string, number>()
  /**
   * Wake startup grace per host. A newly handled wake starts a short active
   * window so the first heartbeat from the replacement pod cannot drain only
   * because its persisted lastActivityTs predates the wake.
   */
  private readonly wakeStartupGraceUntilByHost = new Map<string, number>()
  /**
   * Hosts whose durable state was already checked with a fresh read during the
   * current suspension-blocked (CommunicationChannel authority loss) epoch. The
   * cached Host can be stale about a `draining` write that landed just before
   * the loss, so the first blocked beat bypasses the cached state once; later
   * beats in the same epoch revert only on cached `draining` evidence instead of
   * issuing a fresh GET and a skip log on every heartbeat of the outage. The
   * entry is dropped on the first beat that is no longer blocked.
   */
  private readonly suspensionBlockedFreshCheckDone = new Set<string>()
  /**
   * Last FULLY successful heartbeat poll reported by the poller (AP-3 feed
   * evidence). Undefined until the first report — the silent-emitter expiry
   * branch treats an unreported feed as NOT demonstrably healthy and re-arms.
   */
  private lastSuccessfulPoll: { atMs: number; intervalMs: number } | undefined

  constructor(options: StatelessLifecycleTrackerOptions) {
    if (!Number.isFinite(options.idleMinutes) || options.idleMinutes <= 0) {
      throw new Error(
        `[StatelessLifecycleTracker] idleMinutes must be a positive number, got '${options.idleMinutes}'`
      )
    }
    if (!Number.isFinite(options.idleFloorMinutes) || options.idleFloorMinutes <= 0) {
      throw new Error(
        `[StatelessLifecycleTracker] idleFloorMinutes must be a positive number, got '${options.idleFloorMinutes}'`
      )
    }
    if (!Number.isFinite(options.drainGraceMs) || options.drainGraceMs <= 0) {
      throw new Error(
        `[StatelessLifecycleTracker] drainGraceMs must be a positive number, got '${options.drainGraceMs}'`
      )
    }
    if (!Number.isFinite(options.maxUptimeHours) || options.maxUptimeHours <= 0) {
      throw new Error(
        `[StatelessLifecycleTracker] maxUptimeHours must be a positive number, got '${options.maxUptimeHours}'`
      )
    }
    // Effective T_idle = max(idle, floor): the floor is a platform guardrail
    // against an operator configuring an aggressively short idle window.
    this.idleMs = Math.max(options.idleMinutes, options.idleFloorMinutes) * 60_000
    this.drainGraceMs = options.drainGraceMs
    this.maxUptimeMs = options.maxUptimeHours * 3_600_000
    this.reconciler = options.reconciler
    this.getHost = options.getHost
    this.now = options.now ?? (() => Date.now())
  }

  /** Clear every armed drain-grace timer (server shutdown). */
  stop(): void {
    for (const entry of this.drainGrace.values()) {
      clearTimeout(entry.timer)
    }
    this.drainGrace.clear()
    this.lastBeatByHost.clear()
    this.lastWakeHandledGenerationByHost.clear()
    this.wakeStartupGraceUntilByHost.clear()
    this.suspensionBlockedFreshCheckDone.clear()
  }

  /**
   * Feed-health seam (see HeartbeatLifecycleTracker.noteSuccessfulPoll):
   * called by the poller after every fully-successful poll cycle. The
   * ack-aware grace expiry only attributes silence to the emitter when a
   * successful poll landed strictly after the grace was armed AND recently
   * enough (within pollIntervalMs × 2 of the expiry moment).
   */
  noteSuccessfulPoll(pollIntervalMs: number): void {
    this.lastSuccessfulPoll = { atMs: this.now(), intervalMs: pollIntervalMs }
  }

  async handleHeartbeat(payload: HeartbeatPayload): Promise<HeartbeatVerdict> {
    const hostRef = payload.hostRef

    // ── podUid ordering ──
    if (this.retiredPodUids.get(hostRef)?.has(payload.podUid)) {
      const logKey = `${hostRef}/${payload.podUid}`
      if (!this.stalePodLogged.has(logKey)) {
        this.stalePodLogged.add(logKey)
        console.warn(
          `[StatelessLifecycleTracker] Discarding heartbeat from stale pod "${payload.podUid}" of host "${hostRef}" (current pod is "${this.currentPodByHost.get(hostRef)}")`
        )
      }
      return { drain: false }
    }
    const currentPodUid = this.currentPodByHost.get(hostRef)
    if (currentPodUid !== undefined && currentPodUid !== payload.podUid) {
      // Never-seen podUid while a current pod exists. 'Never seen' alone is
      // NOT proof of a newer pod: after an HCC restart a straggler beat of
      // the OLD pod can replay AFTER the new pod's first beat — adopting it
      // would retire the LIVE pod and wedge the lifecycle (every live beat
      // discarded until the straggler dies). Tie-break on the pods'
      // immutable creationTimestamps before adopting.
      const ordering = await this.resolvePodOrdering(hostRef, currentPodUid, payload.podUid)
      if (ordering === 'newcomer-stale') {
        // The newcomer is strictly older than the current pod: an
        // out-of-order replay of a pod that was already replaced. Retire the
        // newcomer itself — the current (live) pod keeps steering.
        this.retirePod(hostRef, payload.podUid)
        console.warn(
          `[StatelessLifecycleTracker] Heartbeat from never-seen pod "${payload.podUid}" of host "${hostRef}" is OLDER than the current pod "${currentPodUid}" (out-of-order replay) — retiring the newcomer, keeping the current pod`
        )
        return { drain: false }
      }
      if (ordering === 'unresolved') {
        // Conservative: keep the current pod, retire NOBODY. The next beat
        // (either pod's) retries the ordering with fresh evidence.
        console.warn(
          `[StatelessLifecycleTracker] Could not order never-seen pod "${payload.podUid}" against current pod "${currentPodUid}" of host "${hostRef}" — keeping the current pod, retiring nobody (retried next beat)`
        )
        return { drain: false }
      }
      // 'adopt-newcomer': a genuinely newer pod (first heartbeat after a
      // wake/roll). Retire the previous pod and drop its grace timer +
      // cached age.
      this.retirePod(hostRef, currentPodUid)
      this.clearDrainGrace(hostRef)
      this.lastBeatByHost.delete(hostRef)
      console.log(
        `[StatelessLifecycleTracker] Host "${hostRef}" pod changed: "${currentPodUid}" → "${payload.podUid}"`
      )
    }
    this.currentPodByHost.set(hostRef, payload.podUid)
    // Liveness/ack bookkeeping for the ack-aware drain-grace expiry. Placed
    // AFTER the retired-pod discard + podUid ordering so a stale pod's beat
    // can never masquerade as the current pod's ack or liveness.
    this.lastBeatByHost.set(hostRef, {
      podUid: payload.podUid,
      state: payload.state,
      atMs: this.now(),
      // C2: record the activity signal so the ack-aware grace expiry can
      // re-arm over a fenced-but-pending intake (state:'draining' +
      // activeWork:true), which the state==='active' test alone would not
      // catch. lastActivityTs is kept for future TTL/liveness use.
      activeWork: payload.activeWork,
      lastActivityTs: payload.lastActivityTs,
    })

    // ── host + assessed statelessness (D8 condition 5 lives in assessLifecycle) ──
    const host = this.getHost(hostRef)
    if (!host) {
      console.warn(
        `[StatelessLifecycleTracker] Heartbeat for unknown Host "${hostRef}" — answering drain:false`
      )
      return { drain: false }
    }
    const effective = this.reconciler.getEffectiveLifecycle(host)
    if (!effective.suspensionBlocked) {
      this.suspensionBlockedFreshCheckDone.delete(hostRef)
    }
    if (!effective.stateless) {
      // Rejected/disabled stateless lifecycle: never drain. The reconciler's
      // StatelessEnableRejected condition already names the reason.
      this.clearDrainGrace(hostRef)
      return { drain: false }
    }
    if (effective.suspensionBlocked) {
      this.clearDrainGrace(hostRef)
      if (this.suspensionBlockedFreshCheckDone.has(hostRef)) {
        await this.cancelDrainOnEvidence(hostRef, host)
      } else if (await this.cancelDrainOnEvidence(hostRef, host, true)) {
        // A failed fresh check is retried on the next blocked beat.
        this.suspensionBlockedFreshCheckDone.add(hostRef)
      }
      return { drain: false }
    }
    this.noteWakeHandledGeneration(hostRef, host)

    let wakePending =
      this.reconciler.getWakeRequestedGeneration(host) >
      (host.status?.lifecycle?.wakeHandledGeneration ?? 0)
    if (wakePending) {
      // The informer-cached Host can be STALE right here: the wake
      // fast-path's status flip (wakeHandledGeneration = requested) races
      // the watch stream, and a missed MODIFIED event leaves the cache
      // pending until the periodic Host resync (default 5 min). A pending
      // verdict blocks EVERY drain ("cancel-drain wins"), so a stale cached
      // read starves suspension for the whole staleness window -- measured
      // live 2026-07-05: supra-threshold idle gaps never suspended within
      // 360s after a cold wake. Resolve a cached-pending verdict against a
      // FRESH read -- the same discipline the status writers already follow
      // (suspendHostFromHeartbeatCore / markHostActiveFromHeartbeatCore).
      // A fresh-read failure keeps the conservative pending verdict for
      // THIS beat and retries on the next polled heartbeat (same precedent
      // as isMaxUptimeExceeded leaving the ceiling unapplied on a failed
      // lookup): never drain over a possibly-pending wake.
      try {
        const fresh = await this.reconciler.readFreshHost(host)
        const freshPending =
          this.reconciler.getWakeRequestedGeneration(fresh) >
          (fresh.status?.lifecycle?.wakeHandledGeneration ?? 0)
        if (!freshPending) {
          console.log(
            `[StatelessLifecycleTracker] Cached wakePending for "${hostRef}" was stale -- fresh read shows the wake generation handled; drain gate proceeds`
          )
          wakePending = false
        }
      } catch (err) {
        console.error(
          `[StatelessLifecycleTracker] Fresh wakePending resolution failed for "${hostRef}" -- keeping the conservative pending verdict for this beat:`,
          err
        )
      }
    }

    // ── drained report → suspend ──
    if (payload.state === 'drained') {
      const now = this.now()
      const uptimeExceeded = await this.isMaxUptimeExceeded(host, payload.podUid, now)
      const wakeStartupGraceActive = this.isWakeStartupGraceActive(hostRef, now)
      // Stage 6 (W2): machine-parseable suspend-phase timestamp.
      console.log(`[StatelessSuspend] host=${hostRef} phase=drained_reported ts=${this.now()}`)
      this.clearDrainGrace(hostRef)
      // AP-1 epoch: capture the handled wake generation from THIS handler's
      // entry snapshot. It rides to the suspend writer, whose commit-point
      // guard no-ops when the fresh generation has advanced past it — a
      // wake/cancel-drain handled between this drained report and the commit
      // means the report is aged evidence (the pod was un-fenced and may
      // have served turns after it flushed).
      const entryWakeHandledGeneration = host.status?.lifecycle?.wakeHandledGeneration ?? 0
      // Fresh-activity supersede: re-check idle eligibility against the
      // report's OWN payload (same rule as the main gate) before honoring
      // the drained state. A turn accepted and served after the drain
      // verdict — but before the emitter fenced — refreshed lastActivityTs,
      // so the drain decision was made on evidence that turn invalidated.
      // The drained report itself refreshes nothing: payload.lastActivityTs
      // is the emitter's own account of the last served intake, the same
      // trust level as `activeWork`. Max-uptime drains are exempt — an
      // uptime-exceeded drain must still suspend regardless of activity.
      if (now - payload.lastActivityTs < this.idleMs || wakeStartupGraceActive) {
        if (!uptimeExceeded) {
          console.log(
            `[StatelessSuspend] host=${hostRef} phase=drained_report_superseded reason=fresh_activity ts=${this.now()}`
          )
          // Revert the CR to active so control-api answers drain:false and
          // the emitter un-fences and resumes.
          await this.cancelDrainOnEvidence(hostRef, host)
          const reason = SUSPEND_BLOCKED_REASON_PREFIX + blockingCondition(payload)
          try {
            await this.reconciler.publishSuspendBlockedReason(host, reason)
          } catch (err) {
            console.error(
              `[StatelessLifecycleTracker] Failed to publish suspend-blocked reason for "${hostRef}":`,
              err
            )
          }
          return { drain: false }
        }
      }
      if (wakePending) {
        // Cancel-drain wins even over a drained report: the emitter lifts
        // the fence on drain:false and the wake fast-path keeps replicas=1.
        return { drain: false }
      }
      try {
        const outcome = await this.reconciler.suspendHostFromHeartbeat(
          host,
          LIFECYCLE_REASON_SUSPENDED_IDLE,
          entryWakeHandledGeneration
        )
        if (outcome === 'skipped_stale') {
          // The commit refused the drained evidence as aged/overturned (a
          // wake was handled past the entry epoch, a wake is pending, or the
          // drain was cancelled underneath). Answer drain:false so the pod
          // UN-FENCES and resumes — answering drain:true here would re-fence
          // a just-woken pod on evidence the commit itself rejected, with
          // requested==handled leaving nothing to revive it.
          console.log(
            `[StatelessSuspend] host=${hostRef} phase=drained_suspend_skipped_stale ts=${this.now()}`
          )
          return { drain: false }
        }
      } catch (err) {
        // Loud, not fatal: the emitter keeps reporting 'drained' at the
        // accelerated cadence, so the suspension retries on the next beat.
        console.error(
          `[StatelessLifecycleTracker] Suspend after drained report failed for "${hostRef}":`,
          err
        )
      }
      return { drain: true, entryWakeHandledGeneration }
    }

    // ── D8 idle gate + max-uptime ceiling ──
    const now = this.now()
    const uptimeExceeded = await this.isMaxUptimeExceeded(host, payload.podUid, now)
    const wakeStartupGraceActive = this.isWakeStartupGraceActive(hostRef, now)
    const idleEligible =
      !payload.activeWork && !wakeStartupGraceActive && now - payload.lastActivityTs >= this.idleMs

    if (!idleEligible && !uptimeExceeded) {
      this.clearDrainGrace(hostRef)
      // Real activity evidence cancels an in-flight drain: revert the CR so
      // control-api stops answering {drain:true} and the emitter's fence
      // lifts without requiring the rpc-proxy wake round-trip.
      await this.cancelDrainOnEvidence(hostRef, host)
      const reason = SUSPEND_BLOCKED_REASON_PREFIX + blockingCondition(payload)
      try {
        await this.reconciler.publishSuspendBlockedReason(host, reason)
      } catch (err) {
        console.error(
          `[StatelessLifecycleTracker] Failed to publish suspend-blocked reason for "${hostRef}":`,
          err
        )
      }
      return { drain: false }
    }

    if (wakePending) {
      // Cancel-drain wins: never answer drain:true over a pending wake.
      this.clearDrainGrace(hostRef)
      // A pending wake cancels the drain even before the wake fast-path
      // lands: revert the CR so the fence lifts on the next answered beat.
      await this.cancelDrainOnEvidence(hostRef, host)
      return { drain: false }
    }

    // AP-1 epoch for the durable draining write: the handled wake generation
    // on THIS handler's entry snapshot. The poller threads it into
    // markHostDrainingFromHeartbeat, whose commit no-ops when the fresh
    // generation advanced past it — an aged drain verdict must never
    // re-fence a pod whose wake was handled after this decision.
    const entryWakeHandledGeneration = host.status?.lifecycle?.wakeHandledGeneration ?? 0
    this.armDrainGrace(hostRef, payload.podUid)
    // Stage 6 (W2): machine-parseable suspend-phase timestamp — the moment
    // HCC first answers drain:true to this pod's heartbeat.
    console.log(`[StatelessSuspend] host=${hostRef} phase=drain_answered ts=${this.now()}`)
    return { drain: true, entryWakeHandledGeneration }
  }

  /**
   * Tracker-side cancel-drain (wedged-fence fix): a drain:false verdict
   * backed by HOST evidence (activity or a pending wake) while the CR still
   * says `draining` reverts status.lifecycle.state to `active` through the
   * reconciler seam. Never touches `active` (no-op) or `suspended` (reviving
   * a suspended Host is exclusively the wake fast-path's job). Loud, not
   * fatal on failure — the emitter keeps beating while fenced, so the
   * revert retries on the next polled heartbeat.
   *
   * When CommunicationChannel authority is unknown, the cached Host may still
   * say `active` while the durable CR already says `draining`. The caller then
   * bypasses the cache precheck so this fresh-read writer can safely revert the
   * durable `draining` state (and no-op for active/suspended).
   */
  private async cancelDrainOnEvidence(
    hostRef: string,
    host: HostCRD,
    bypassCachedState = false
  ): Promise<boolean> {
    if (!bypassCachedState && (host.status?.lifecycle?.state ?? 'active') !== 'draining') {
      return true
    }
    try {
      await this.reconciler.markHostActiveFromHeartbeat(host)
      return true
    } catch (err) {
      console.error(`[StatelessLifecycleTracker] Cancel-drain revert failed for "${hostRef}":`, err)
      return false
    }
  }

  private retirePod(hostRef: string, podUid: string): void {
    let retired = this.retiredPodUids.get(hostRef)
    if (!retired) {
      retired = new Set()
      this.retiredPodUids.set(hostRef, retired)
    }
    retired.add(podUid)
    if (retired.size > MAX_RETIRED_POD_UIDS_PER_HOST) {
      const oldest = retired.values().next()
      if (!oldest.done) {
        retired.delete(oldest.value)
        this.stalePodLogged.delete(`${hostRef}/${oldest.value}`)
      }
    }
    this.podCreationTsByUid.delete(`${hostRef}/${podUid}`)
  }

  /**
   * FIX 5 (AP-2, pod identity): order a never-seen podUid against the
   * current pod by immutable creationTimestamp before adoption.
   *   - 'adopt-newcomer': the newcomer's creation is newer-or-equal (K8s
   *     creationTimestamp has 1-second granularity, so a genuine same-second
   *     roll must not wedge behind a strict > test), or the current pod no
   *     longer exists in the pod list.
   *   - 'newcomer-stale': the newcomer is strictly older — an out-of-order
   *     replay; the caller retires the newcomer.
   *   - 'unresolved': host unknown, a lookup failed, or the newcomer is not
   *     visible in the pod list while the current pod still exists — keep
   *     the current pod, retire nobody (retried on the next beat).
   */
  private async resolvePodOrdering(
    hostRef: string,
    currentPodUid: string,
    newcomerPodUid: string
  ): Promise<'adopt-newcomer' | 'newcomer-stale' | 'unresolved'> {
    const host = this.getHost(hostRef)
    if (!host) {
      return 'unresolved'
    }
    let newcomerCreated: Date | null
    let currentCreated: Date | null
    try {
      newcomerCreated = await this.reconciler.findPodCreationTimestamp(host, newcomerPodUid)
      currentCreated = await this.reconciler.findPodCreationTimestamp(host, currentPodUid)
    } catch (err) {
      console.error(
        `[StatelessLifecycleTracker] Pod ordering lookup failed for host "${hostRef}" (current "${currentPodUid}", newcomer "${newcomerPodUid}") — keeping the current pod:`,
        err
      )
      return 'unresolved'
    }
    if (currentCreated === null) {
      // The current pod no longer exists — the newcomer is the live pod.
      return 'adopt-newcomer'
    }
    if (newcomerCreated === null) {
      // Never adopt a pod we cannot even see over a still-existing current
      // pod.
      return 'unresolved'
    }
    const newcomerMs = newcomerCreated.getTime()
    // Cache the newcomer's immutable creation time for the max-uptime
    // ceiling (same key shape as isMaxUptimeExceeded).
    this.podCreationTsByUid.set(`${host.name}/${newcomerPodUid}`, newcomerMs)
    return newcomerMs >= currentCreated.getTime() ? 'adopt-newcomer' : 'newcomer-stale'
  }

  /**
   * Max-uptime ceiling from the pod's creationTimestamp. The timestamp is
   * immutable, so it is resolved ONCE per (hostRef, podUid) via the
   * reconciler's pod list and cached. A failed or empty lookup logs loudly
   * and leaves the ceiling UNAPPLIED for this beat (retried next beat) —
   * an API blip must not force-drain an active host.
   */
  private async isMaxUptimeExceeded(host: HostCRD, podUid: string, now: number): Promise<boolean> {
    const cacheKey = `${host.name}/${podUid}`
    let createdMs = this.podCreationTsByUid.get(cacheKey)
    if (createdMs === undefined) {
      let created: Date | null
      try {
        created = await this.reconciler.findPodCreationTimestamp(host, podUid)
      } catch (err) {
        console.error(
          `[StatelessLifecycleTracker] Pod age lookup failed for host "${host.name}" pod "${podUid}" — max-uptime ceiling not applied this beat:`,
          err
        )
        return false
      }
      if (created === null) {
        console.warn(
          `[StatelessLifecycleTracker] Pod "${podUid}" of host "${host.name}" not found in the pod list — max-uptime ceiling not applied this beat`
        )
        return false
      }
      createdMs = created.getTime()
      this.podCreationTsByUid.set(cacheKey, createdMs)
    }
    return now - createdMs >= this.maxUptimeMs
  }

  /**
   * Arm the drain-grace timer on the FIRST drain:true answer for this pod.
   * If the pod never reports 'drained' within the grace window, the tracker
   * suspends anyway (the emitter may be wedged; the reconciler's
   * drained-pre-scale wake guard remains the last line of defense).
   */
  private armDrainGrace(hostRef: string, podUid: string): void {
    const existing = this.drainGrace.get(hostRef)
    if (existing) {
      if (existing.podUid === podUid) {
        return // grace runs from the first drain answer — do not extend
      }
      clearTimeout(existing.timer)
    }
    const timer = setTimeout(() => {
      void this.onDrainGraceExpired(hostRef, podUid)
    }, this.drainGraceMs)
    ;(timer as { unref?: () => void }).unref?.()
    this.drainGrace.set(hostRef, { podUid, timer, armedAtMs: this.now() })
  }

  private clearDrainGrace(hostRef: string): void {
    const entry = this.drainGrace.get(hostRef)
    if (entry) {
      clearTimeout(entry.timer)
      this.drainGrace.delete(hostRef)
    }
  }

  private noteWakeHandledGeneration(hostRef: string, host: HostCRD): void {
    const handled = host.status?.lifecycle?.wakeHandledGeneration ?? 0
    const prior = this.lastWakeHandledGenerationByHost.get(hostRef)
    this.lastWakeHandledGenerationByHost.set(hostRef, handled)
    if (prior === undefined || handled <= prior) {
      return
    }
    const graceUntil = this.now() + this.idleMs
    this.wakeStartupGraceUntilByHost.set(hostRef, graceUntil)
    console.log(
      `[StatelessLifecycleTracker] Host "${hostRef}" wake generation advanced ${prior} -> ${handled}; suppressing idle drain until ${new Date(graceUntil).toISOString()}`
    )
  }

  private isWakeStartupGraceActive(hostRef: string, now: number): boolean {
    const graceUntil = this.wakeStartupGraceUntilByHost.get(hostRef)
    if (graceUntil === undefined) {
      return false
    }
    if (now < graceUntil) {
      return true
    }
    this.wakeStartupGraceUntilByHost.delete(hostRef)
    return false
  }

  private async onDrainGraceExpired(hostRef: string, podUid: string): Promise<void> {
    const entry = this.drainGrace.get(hostRef)
    if (!entry || entry.podUid !== podUid) {
      return
    }
    this.drainGrace.delete(hostRef)
    if (this.currentPodByHost.get(hostRef) !== podUid) {
      return // pod rolled while the grace was pending
    }
    const host = this.getHost(hostRef)
    if (!host) {
      console.warn(
        `[StatelessLifecycleTracker] Drain grace expired for unknown Host "${hostRef}" — skipping suspension`
      )
      return
    }
    const effective = this.reconciler.getEffectiveLifecycle(host)
    if (!effective.stateless || effective.suspensionBlocked) {
      return
    }
    // KZ-R1: the informer-cached Host can be STALE here exactly as it is on the
    // heartbeat path -- the wake fast-path's status flip
    // (wakeHandledGeneration set to requested) races the watch stream, and a
    // missed MODIFIED event leaves the cache pending until the periodic resync.
    // The heartbeat path (handleHeartbeat) was already hardened to resolve a
    // cached-pending verdict against a FRESH read; the grace-expiry path was
    // left cache-only, so a stale cached NOT-pending read could suspend a host
    // whose wake already landed. Apply the same discipline: read the cached
    // verdict, and if it is pending trust it, but if it is NOT pending confirm
    // against a fresh read before suspending. Fail-conservative: on a fresh-read
    // error, skip the suspend this cycle and let the next beat/grace decide --
    // never suspend over a possibly-pending wake (mirrors handleHeartbeat).
    // AP-1 epoch (see suspendHostFromHeartbeat): the handled wake generation
    // from this expiry's entry snapshot — the suspend commit no-ops when a
    // wake was handled after this point.
    const entryWakeHandledGeneration = host.status?.lifecycle?.wakeHandledGeneration ?? 0
    let wakePending = this.reconciler.getWakeRequestedGeneration(host) > entryWakeHandledGeneration
    if (!wakePending) {
      try {
        const fresh = await this.reconciler.readFreshHost(host)
        wakePending =
          this.reconciler.getWakeRequestedGeneration(fresh) >
          (fresh.status?.lifecycle?.wakeHandledGeneration ?? 0)
        if (wakePending) {
          console.log(
            `[StatelessLifecycleTracker] Cached not-pending for "${hostRef}" was stale -- fresh read shows a wake pending; grace-expiry suspension cancelled`
          )
        }
      } catch (err) {
        console.error(
          `[StatelessLifecycleTracker] Fresh wakePending resolution failed for "${hostRef}" at grace expiry -- skipping suspension for this cycle:`,
          err
        )
        return
      }
    }
    if (wakePending) {
      console.log(
        `[StatelessLifecycleTracker] Drain grace expired for "${hostRef}" but a wake is pending — suspension cancelled`
      )
      return
    }
    // FIX 3 (AP-2, pod identity): re-validate decision ownership AFTER the
    // awaits above. The entry checks (grace entry, currentPodByHost) ran
    // synchronously BEFORE the fresh read; a pod replacement or a re-arm/
    // clear landing DURING the await would otherwise let this expiry
    // force-suspend over the live replacement pod (whose lastBeat bookkeeping
    // was reset, making it look "silent"). If the current pod changed, or a
    // NEW grace was armed for this host since this expiry deleted its own
    // entry, this callback's evidence is aged — the new pod's beats / the
    // new arming own the decision now.
    if (this.currentPodByHost.get(hostRef) !== podUid || this.drainGrace.has(hostRef)) {
      console.warn(
        `[StatelessLifecycleTracker] Drain-grace expiry for "${hostRef}" aborted after the fresh read — pod identity or grace arming changed underneath (expiry pod "${podUid}", current pod "${this.currentPodByHost.get(hostRef)}")`
      )
      return
    }
    // ── ack-aware expiry ──
    // Distinguish "emitter acked the drain but the drained report never
    // landed" (legitimate force-suspend) from two re-arm cases (suspending
    // would kill live work; the emitter's beat cadence is NOT controllable by
    // HCC and may exceed the configured grace):
    //   (a) the emitter never even heard the drain while its beats keep
    //       flowing (last beat still reports state==='active'); or
    //   (b) C2: a genuine PENDING intake is fenced — the emitter reports
    //       state:'draining' (it DID ack) but the beat still carries
    //       activeWork:true. The `state==='active'` test alone misses this and
    //       would suspend over live pending work. A fresh activeWork:true from
    //       the same pod after arming is therefore an EQUAL re-arm trigger.
    //
    // pendingIntake note (C2-devops, KNOWN RESIDUAL — do NOT implement here):
    // activeWork:true is the emitter's projection of a fenced pending intake.
    // If the rpc-proxy->control-api wake for that intake NEVER lands, the
    // emitter keeps beating draining+activeWork:true forever, so this re-arm
    // loops indefinitely and the host never suspends OR un-drains. The durable
    // fix is a bounded TTL on the pending-intake wake (or a tracker-side wake
    // re-attempt) — Phase 2, out of scope here. Documented so it is
    // discoverable at the site that consults the pending-intake signal.
    const last = this.lastBeatByHost.get(hostRef)
    const beatIsFreshForThisPod =
      last !== undefined && last.podUid === podUid && last.atMs >= entry.armedAtMs
    const emitterStillActive = beatIsFreshForThisPod && last.state === 'active'
    const pendingWorkFenced = beatIsFreshForThisPod && last.activeWork === true
    if (emitterStillActive || pendingWorkFenced) {
      // Re-arm a fresh full grace window instead of suspending. No cap on
      // re-arms is needed BY DESIGN for case (a): each subsequent beat is
      // answered from the CR (`draining` → drain:true), so a live emitter
      // converges to acking within one beat interval; if instead its beats
      // carry activity evidence the tracker cancels the drain — either way
      // re-arming cannot loop forever with a live, idle emitter. Case (b)
      // (activeWork:true) carries the C2-devops residual noted above.
      const reason = emitterStillActive ? 'drain_not_acked' : 'pending_work'
      console.log(
        `[StatelessSuspend] host=${hostRef} phase=grace_rearmed reason=${reason} ts=${this.now()}`
      )
      this.armDrainGrace(hostRef, podUid)
      return
    }
    // FIX 4 (AP-3): 'no beat since arming' is read from lastBeatByHost,
    // which only successful polls feed — a control-api/poller outage
    // spanning the grace makes a LIVE active emitter look silent. Silence is
    // evidence against the EMITTER only when the feed was demonstrably
    // healthy across the window: at least one fully-successful poll landed
    // strictly after this grace was armed AND recently enough (within
    // poll-interval × 2 of now). Otherwise the silence is unattributable —
    // re-arm and let the recovered feed decide on real beats.
    const emitterSilent = !beatIsFreshForThisPod
    if (emitterSilent) {
      const poll = this.lastSuccessfulPoll
      const feedHealthy =
        poll !== undefined &&
        poll.atMs > entry.armedAtMs &&
        this.now() - poll.atMs <= poll.intervalMs * 2
      if (!feedHealthy) {
        console.log(
          `[StatelessSuspend] host=${hostRef} phase=grace_rearmed reason=feed_outage ts=${this.now()}`
        )
        this.armDrainGrace(hostRef, podUid)
        return
      }
    }
    // Suspend cases: emitter silent with a provably-healthy feed (pod dead
    // or wedged; nothing live to kill), or the last beat reported
    // 'draining'/'drained' (acked, but the drained report never landed).
    console.warn(
      `[StatelessLifecycleTracker] Drain grace (${this.drainGraceMs}ms) expired for "${hostRef}" without a drained report — suspending`
    )
    try {
      await this.reconciler.suspendHostFromHeartbeat(
        host,
        LIFECYCLE_REASON_SUSPENDED_IDLE,
        entryWakeHandledGeneration
      )
    } catch (err) {
      console.error(
        `[StatelessLifecycleTracker] Grace-expiry suspension failed for "${hostRef}":`,
        err
      )
    }
  }
}

/** Name the D8 condition blocking suspension (priority-ordered). */
function blockingCondition(payload: HeartbeatPayload): string {
  if (payload.conditions.awaitingApproval) return 'awaitingApproval'
  if (payload.conditions.activeTask) return 'activeTask'
  if (payload.conditions.pendingResults) return 'pendingResults'
  if (payload.conditions.activeCronSchedules) return 'activeCronSchedules'
  if (payload.activeWork) return 'activeWork'
  return 'recentActivity'
}