Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
86.8 KB
Raw Blame
/**
 * Stateless Host lifecycle execution (Stages 2–6), extracted from
 * HostReconciler: reconcile-time assessment + durable status writes, the
 * heartbeat-driven suspend/drain/cancel-drain executors backing the
 * tracker's StatelessLifecycleReconcilerPort, the wake fast-path, and the
 * per-host reconcile serialization chain. The tracker DECIDES; this class
 * EXECUTES. HostReconciler holds one instance and delegates.
 */
import * as k8s from '@kubernetes/client-node'
import { config } from './config'
import { isHeartbeatManagedLifecycleReason } from './constants'
import type { ResolvedSfsMount } from './hostReconciler'
import { SFS_LABEL, SFS_NAMESPACE_LABEL, WFC_APP_LABEL } from './k8s/sharedFileSystemFactory'
import { hccLogger } from './logger'
import {
  STATELESS_PULL_POLICY_REJECTED_CONDITION_TYPE,
  pullPolicyNotApplicableCondition,
  statelessPullPolicyCondition,
} from './statelessDeployment'
import {
  EffectiveHostLifecycle,
  HostLifecycleAssessment,
  SuspendFromHeartbeatOutcome,
} from './statelessLifecycle.types'
import {
  HostCRD,
  HostCondition,
  HostCrdStatus,
  HostLifecycleState,
  HostLifecycleStatus,
} from './types'
import { getErrorCode } from './utils'

// ─── Stateless lifecycle (Stage 2) ─────────────────────────────────────────
const log = hccLogger.child({ module: 'stateless-lifecycle' })
const GROUP = 'clerum.io'
const VERSION = 'v1alpha1'
const PLURAL_HOSTS = 'hosts'
const STATELESS_REJECTED_CONDITION_TYPE = 'StatelessEnableRejected'
const COMMUNICATION_CHANNEL_CACHE_UNSYNCED_REASON = 'CommunicationChannelCacheUnsynced'
const COMMUNICATION_CHANNEL_CACHE_UNSYNCED_MESSAGE =
  'CommunicationChannel cache is not synchronized; stateless lifecycle transitions are held'
const ACTIVE_COMMUNICATION_CHANNELS_REASON = 'ActiveCommunicationChannels'
/**
 * Host CR annotation carrying the monotonic wake generation (Stage 4.3).
 * control-api bumps a Postgres-backed generation on every wake request —
 * for suspended AND draining Hosts (a bump while draining means "cancel the
 * drain") — and projects it here. HCC compares it against
 * status.lifecycle.wakeHandledGeneration to detect pending wakes.
 */
const WAKE_REQUESTED_ANNOTATION = 'clerum.io/wake-requested'

/**
 * H2: reflect a committed lifecycle outcome onto the CURRENT canonical cache
 * entry for `name` — but only if that entry still has the admitted `uid`.
 * The receiver (McpServerWatcher) owns the lookup and the uid guard; the
 * executor never touches the cache directly. A same-name recreation with a
 * different uid MUST be skipped silently (the mutation already committed
 * server-side under the old uid; corrupting the new object would be worse
 * than staleness, which the watch/resync repairs).
 */
export type ReflectHostOutcomeFn = (
  name: string,
  uid: string | undefined,
  apply: (target: HostCRD) => void
) => void

/** Constructor-injected seams from HostReconciler (late-bound where needed). */
export interface StatelessLifecycleExecutorDeps {
  appsApi: k8s.AppsV1Api
  coreApi: k8s.CoreV1Api
  /**
   * Lazily resolves the CustomObjects client: HostReconciler constructs it
   * on first use (existing tests construct the reconciler with a bare
   * KubeConfig stub that cannot makeApiClient), so the executor must never
   * capture it eagerly.
   */
  getCustomApi: () => k8s.CustomObjectsApi
  /** Injectable clock so tests get deterministic condition lastTransitionTime. */
  now: () => Date
  /**
   * Count CommunicationChannels referencing this Host. Late-bound through
   * the reconciler (McpServerWatcher wires the counter after construction).
   */
  countCommunicationChannels: (hostName: string) => number
  /**
   * True only after the CommunicationChannel initial list has completed.
   * An unsynchronized cache cannot safely prove that a Host has no channels.
   */
  isCommunicationChannelCacheSynced: () => boolean
  /**
   * HostReconciler.reconcileCore — the UN-serialized reconcile body.
   * Callers inside this executor already run inside the host's
   * serialization slot, so re-entering the per-host chain would
   * self-deadlock.
   */
  reconcileCore: (host: HostCRD, revalidate?: () => HostCRD) => Promise<void>
  /**
   * Capture Host inventory authority at heartbeat dispatch time and return an
   * admission closure that revalidates that authority and resolves the current
   * same-identity Host inside this executor's per-Host serializer. The two-step
   * shape is intentional: capturing only after the queue drains cannot detect
   * an authority generation change that happened while the work was waiting.
   */
  prepareHostMutationAdmission?: (action: string, host: HostCRD) => () => HostCRD
  reflectHostOutcome?: ReflectHostOutcomeFn
  onLifecycleStatusCommitted?: (host: HostCRD, lifecycle: HostLifecycleStatus) => void
}

export class StatelessLifecycleExecutor {
  private readonly appsApi: k8s.AppsV1Api
  private readonly coreApi: k8s.CoreV1Api
  private readonly getCustomApiFn: () => k8s.CustomObjectsApi
  private readonly now: () => Date
  private readonly countCommunicationChannels: (hostName: string) => number
  private readonly isCommunicationChannelCacheSynced: () => boolean
  private readonly reconcileCore: (host: HostCRD, revalidate?: () => HostCRD) => Promise<void>
  private readonly prepareHostMutationAdmission: (action: string, host: HostCRD) => () => HostCRD
  private readonly reflectHostOutcome: ReflectHostOutcomeFn
  private readonly onLifecycleStatusCommitted: (
    host: HostCRD,
    lifecycle: HostLifecycleStatus
  ) => void
  /**
   * Serialized lifecycle status last written to the Host /status subresource,
   * keyed by host name. Mirrors sharedFileSystemReconciler's dirty check: a
   * no-op /status patch bumps resourceVersion → MODIFIED watch event →
   * re-reconcile → write again, so steady-state writes must be skipped.
   */
  private readonly lastWrittenLifecycleStatus: Map<string, string> = new Map()
  /**
   * Last malformed clerum.io/wake-requested annotation value logged per host.
   * A broken writer must stay operator-visible, but the periodic resync
   * would otherwise repeat the identical log line forever — log loudly once
   * per distinct value instead.
   */
  private readonly malformedWakeAnnotationLogged: Map<string, string> = new Map()
  /**
   * In-memory scale-transition counter per host (Stage 6 metric). Resets on
   * HCC restart — acceptable: the wake-budget script correlates transitions
   * within a single HCC run.
   */
  private readonly scaleTransitionsByHost: Map<string, number> = new Map()
  private readonly suspendedAppliedLoggedByHost: Set<string> = new Set()
  private readonly suspendedScaleMetricRecordedByHost: Set<string> = new Set()
  /**
   * L2: per-host serializing promise chain. @kubernetes/client-node's Watch
   * does NOT await the prior callback, and the tracker (heartbeat poller)
   * drives suspend/drain reconciles on a separate timer. So a MODIFIED-wake
   * reconcile and a tracker-driven suspend for the SAME host could otherwise
   * interleave and clobber each other's status/replicas writes. Every public
   * mutation entry point for a host funnels through serializeByHost(), which
   * chains that host's work onto its previous promise — reconciles for the
   * same host run strictly one-at-a-time, while different hosts still run
   * concurrently. Internal self-calls (e.g. suspendHostFromHeartbeat →
   * reconcileCore) deliberately bypass the chain to avoid self-deadlock, since
   * they already run inside the caller's held slot.
   */
  private readonly reconcileChainByHost: Map<string, Promise<void>> = new Map()

  constructor(deps: StatelessLifecycleExecutorDeps) {
    this.appsApi = deps.appsApi
    this.coreApi = deps.coreApi
    this.getCustomApiFn = deps.getCustomApi
    this.now = deps.now
    this.countCommunicationChannels = deps.countCommunicationChannels
    this.isCommunicationChannelCacheSynced = deps.isCommunicationChannelCacheSynced
    this.reconcileCore = deps.reconcileCore
    this.prepareHostMutationAdmission =
      deps.prepareHostMutationAdmission ?? ((_action, host) => () => host)
    this.reflectHostOutcome = deps.reflectHostOutcome ?? (() => {})
    this.onLifecycleStatusCommitted = deps.onLifecycleStatusCommitted ?? (() => {})
    StatelessLifecycleExecutor.warnSingleReplicaInvariantOnce()
  }

  /**
   * SINGLE-REPLICA INVARIANT (undocumented until now, load-bearing): the
   * stateless lifecycle status writers are made safe by TWO mechanisms working
   * together — the per-host serializeByHost chain (orders writers WITHIN one
   * process) and the D3 resourceVersion precondition + 409-retry (closes the
   * cross-writer window serialization cannot). Neither provides leader
   * election. If HCC ever runs with replicas > 1, two independent
   * reconcile loops drive suspend/drain/wake for the same Host with no
   * cross-process ordering; the resourceVersion precondition then degrades
   * from "correct" to "one writer wins each round, the loser retries" — which
   * is safe against corruption but can livelock two racing controllers. HCC is
   * therefore deployed replicas:1 without leader election. This warn makes the
   * assumption operator-visible once per process instead of silent.
   */
  private static singleReplicaInvariantWarned = false
  private static warnSingleReplicaInvariantOnce(): void {
    if (StatelessLifecycleExecutor.singleReplicaInvariantWarned) {
      return
    }
    StatelessLifecycleExecutor.singleReplicaInvariantWarned = true
    log.warn(
      '[HostReconciler] Stateless lifecycle status writes assume HCC runs replicas:1 WITHOUT leader election. serializeByHost orders writers within this process; the resourceVersion precondition + 409-retry closes the cross-writer window — but neither elects a leader. Running >1 replica risks two controllers racing (safe against corruption, but can livelock).'
    )
  }

  /** CustomObjects API client for Host /status writes (lazily constructed). */
  private get customApi(): k8s.CustomObjectsApi {
    return this.getCustomApiFn()
  }

  /**
   * Drop this host's lifecycle bookkeeping (called from
   * HostReconciler.clearStatus on Host deletion).
   */
  clearHost(name: string): void {
    this.lastWrittenLifecycleStatus.delete(name)
    this.malformedWakeAnnotationLogged.delete(name)
  }

  // ─── Stateless lifecycle (Stage 2) ──────────────────────────────────

  /**
   * Synchronous effective-lifecycle derivation for direct buildDeployment
   * callers. reconcile() passes the full async assessment (which also runs
   * the SharedFileSystem co-location check); this fallback covers the
   * synchronously checkable rejections so a direct build never suspends a
   * Host that has a desktop or CommunicationChannels.
   */
  effectiveLifecycleFromCache(host: HostCRD): EffectiveHostLifecycle {
    if (host.spec.lifecycle?.stateless !== true) {
      return { stateless: false, state: 'active' }
    }
    const hasCommunicationChannels =
      this.isCommunicationChannelCacheSynced() && this.countCommunicationChannels(host.name) > 0
    if (host.spec.desktop !== undefined || hasCommunicationChannels) {
      return { stateless: false, state: 'active' }
    }
    if (!this.isCommunicationChannelCacheSynced()) {
      return {
        stateless: true,
        state: host.status?.lifecycle?.state ?? 'active',
        suspensionBlocked: true,
      }
    }
    return { stateless: true, state: host.status?.lifecycle?.state ?? 'active' }
  }

  private communicationChannelPolicyRejection(hostName: string): {
    reasons: string[]
    messages: string[]
  } {
    const reasons: string[] = []
    const messages: string[] = []
    // A retained positive entry is no more authoritative than an empty cache.
    const ccCount = this.isCommunicationChannelCacheSynced()
      ? this.countCommunicationChannels(hostName)
      : 0
    if (ccCount > 0) {
      // Addendum 6 (operator visibility): the rejection message names both the
      // associated channel count AND the recovery action, because control-ui
      // renders `condition.message` verbatim in the stateless-rejection banner.
      // The rejection is state-derived, so disassociating the channels
      // re-enables the requested stateless lifecycle on the next reconcile.
      reasons.push(ACTIVE_COMMUNICATION_CHANNELS_REASON)
      messages.push(
        `${ccCount} CommunicationChannel(s) reference this Host; disassociate them to enable the requested stateless lifecycle`
      )
    }
    return { reasons, messages }
  }

  /**
   * Assess whether the Host may run the stateless lifecycle and compute the
   * durable status to persist. Rejections (each also acts as an automatic
   * kill-switch when it appears AFTER enable):
   *   - active CommunicationChannels referencing the Host
   *   - spec.desktop present
   *   - provably unsatisfiable SharedFileSystem co-location (2+ SFS whose
   *     wfc pods are pinned to different nodes)
   * A rejected or disabled Host is forced to state=active (replicas 1) so it
   * never sits unsuspendable without an operator-visible signal.
   */
  async assessLifecycle(
    host: HostCRD,
    mounts: ResolvedSfsMount[]
  ): Promise<HostLifecycleAssessment> {
    const wakeHandledGeneration = host.status?.lifecycle?.wakeHandledGeneration ?? 0

    if (host.spec.lifecycle?.stateless !== true) {
      // Kill-switch: stateless disabled or the field was removed — back to
      // active + replicas 1 on this reconcile.
      return {
        effective: { stateless: false, state: 'active' },
        lifecycle: { state: 'active', wakeHandledGeneration },
        condition: {
          type: STATELESS_REJECTED_CONDITION_TYPE,
          status: 'False',
          reason: 'StatelessDisabled',
          message: 'spec.lifecycle.stateless is not enabled',
        },
        pullPolicyCondition: pullPolicyNotApplicableCondition(),
      }
    }

    const reasons: string[] = []
    const messages: string[] = []

    const channelRejection = this.communicationChannelPolicyRejection(host.name)
    reasons.push(...channelRejection.reasons)
    messages.push(...channelRejection.messages)
    if (host.spec.desktop !== undefined) {
      reasons.push('DesktopEnabled')
      messages.push('spec.desktop is present')
    }
    if (mounts.length >= 2) {
      const conflictingNodes = await this.findSfsColocationConflict(host, mounts)
      if (conflictingNodes) {
        reasons.push('SfsColocationUnsatisfiable')
        messages.push(
          `SharedFileSystem co-location is unsatisfiable: wfc pods are pinned to different nodes (${conflictingNodes.join(', ')})`
        )
      }
    }

    if (reasons.length > 0) {
      const message = messages.join('; ')
      return {
        effective: { stateless: false, state: 'active' },
        lifecycle: { state: 'active', wakeHandledGeneration, reason: message },
        condition: {
          type: STATELESS_REJECTED_CONDITION_TYPE,
          status: 'True',
          reason: reasons.join(','),
          message,
        },
        pullPolicyCondition: pullPolicyNotApplicableCondition(),
      }
    }

    if (!this.isCommunicationChannelCacheSynced()) {
      return this.holdDurableStateDuringChannelCacheRecovery(
        wakeHandledGeneration,
        host.status?.lifecycle?.state ?? 'active'
      )
    }

    // Accepted: preserve the durable state from the CRD status — a suspended
    // Host must survive HCC restarts and periodic resyncs.
    const state = host.status?.lifecycle?.state ?? 'active'
    // Preserve heartbeat-tracker-owned reasons ('idle' suspension, D8
    // suspend-blocked): the accepted path otherwise clears `reason`, which
    // would erase the tracker's status one watch event after every publish.
    const observedReason = host.status?.lifecycle?.reason
    const preservedReason = isHeartbeatManagedLifecycleReason(observedReason)
      ? observedReason
      : undefined
    return {
      effective: { stateless: true, state },
      lifecycle: {
        state,
        wakeHandledGeneration,
        ...(preservedReason !== undefined ? { reason: preservedReason } : {}),
      },
      condition: {
        type: STATELESS_REJECTED_CONDITION_TYPE,
        status: 'False',
        reason: 'StatelessEnabled',
        message: 'stateless lifecycle is enabled',
      },
      // Stateless is never desktop (spec.desktop rejects above), so the pod
      // image is always config.hostImage here — same input buildDeployment
      // resolves against.
      pullPolicyCondition: statelessPullPolicyCondition(config.hostImage),
    }
  }

  /** Unknown channel inventory requires preserving the applied Deployment. */
  private holdDurableStateDuringChannelCacheRecovery(
    wakeHandledGeneration: number,
    state: HostLifecycleState = 'active'
  ): HostLifecycleAssessment {
    return {
      effective: { stateless: true, state, suspensionBlocked: true },
      lifecycle: {
        state,
        wakeHandledGeneration,
        reason: COMMUNICATION_CHANNEL_CACHE_UNSYNCED_MESSAGE,
      },
      condition: {
        type: STATELESS_REJECTED_CONDITION_TYPE,
        status: 'False',
        reason: COMMUNICATION_CHANNEL_CACHE_UNSYNCED_REASON,
        message: COMMUNICATION_CHANNEL_CACHE_UNSYNCED_MESSAGE,
      },
      pullPolicyCondition: statelessPullPolicyCondition(config.hostImage),
    }
  }

  /**
   * Close the interval between the initial lifecycle assessment and the
   * Deployment write. Reconciliation performs several Kubernetes operations
   * in between; if the CommunicationChannel watch ends, retain the applied
   * template, whose mode may be stateful. A confirmed channel still applies the
   * established incompatible-mode policy.
   *
   * This synchronous cache check adds no Kubernetes API calls to steady state.
   */
  enforceCommunicationChannelPolicyBeforeDeployment(
    hostName: string,
    assessment: HostLifecycleAssessment
  ): HostLifecycleAssessment {
    if (!assessment.effective.stateless) {
      // Independent desktop/SFS incompatibilities retain their policy. A
      // channel-only rejection cannot choose a runtime after authority is lost.
      if (
        !this.isCommunicationChannelCacheSynced() &&
        assessment.condition.reason === ACTIVE_COMMUNICATION_CHANNELS_REASON
      ) {
        return this.holdDurableStateDuringChannelCacheRecovery(
          assessment.lifecycle.wakeHandledGeneration,
          assessment.effective.state
        )
      }
      return assessment
    }

    const { reasons, messages } = this.communicationChannelPolicyRejection(hostName)
    if (reasons.length === 0) {
      if (!this.isCommunicationChannelCacheSynced() && !assessment.effective.suspensionBlocked) {
        return this.holdDurableStateDuringChannelCacheRecovery(
          assessment.lifecycle.wakeHandledGeneration,
          assessment.effective.state
        )
      }
      return assessment
    }
    const message = messages.join('; ')

    return {
      effective: { stateless: false, state: 'active' },
      lifecycle: {
        state: 'active',
        wakeHandledGeneration: assessment.lifecycle.wakeHandledGeneration,
        reason: message,
      },
      condition: {
        type: STATELESS_REJECTED_CONDITION_TYPE,
        status: 'True',
        reason: reasons.join(','),
        message,
      },
      pullPolicyCondition: pullPolicyNotApplicableCondition(),
    }
  }

  /**
   * Returns the sorted distinct node names hosting the wfc pods of the given
   * SFS mounts when they PROVABLY conflict (2+ distinct nodes), else null.
   *
   * Every SFS PVC is RWO and its wfc pod is the RW writer, so a Host mounting
   * 2+ SFS is schedulable only when all wfc pods share one node. When node
   * placement cannot be resolved (list error, pods not scheduled yet) this
   * returns null: only the provably-unsatisfiable case rejects — a transient
   * lookup failure must not flap the lifecycle.
   */
  private async findSfsColocationConflict(
    host: HostCRD,
    mounts: ResolvedSfsMount[]
  ): Promise<string[] | null> {
    const nodes = new Set<string>()
    for (const m of mounts) {
      try {
        const pods = await this.coreApi.listNamespacedPod({
          namespace: m.namespace,
          labelSelector: `app=${WFC_APP_LABEL},${SFS_LABEL}=${m.name},${SFS_NAMESPACE_LABEL}=${m.namespace}`,
        })
        for (const pod of pods.items ?? []) {
          if (pod.spec?.nodeName) {
            nodes.add(pod.spec.nodeName)
          }
        }
      } catch (err) {
        log.warn(
          `[HostReconciler] Could not resolve wfc node placement for SFS "${m.namespace}/${m.name}" (host "${host.name}"); skipping the co-location rejection check:`,
          { err }
        )
        return null
      }
    }
    if (nodes.size <= 1) {
      return null
    }
    return [...nodes].sort()
  }

  /**
   * Write status.lifecycle + the StatelessEnableRejected condition to the
   * Host /status subresource. Follows the sharedFileSystemReconciler status
   * pattern: idempotent-by-comparison (skip when nothing changed, preserve
   * lastTransitionTime across unchanged conditions) and best-effort (errors
   * are logged, not thrown — status writes must not block reconciliation).
   *
   * Hosts that never opted into the lifecycle (no spec.lifecycle AND no
   * previously-written status.lifecycle) are skipped entirely so legacy
   * Hosts keep an untouched status.
   */
  async writeLifecycleStatusToCluster(
    host: HostCRD,
    assessment: HostLifecycleAssessment
  ): Promise<boolean> {
    if (host.spec.lifecycle === undefined && host.status?.lifecycle === undefined) {
      return false
    }

    const desired = JSON.stringify({
      lifecycle: assessment.lifecycle,
      condition: assessment.condition,
      pullPolicyCondition: assessment.pullPolicyCondition,
    })
    if (this.lastWrittenLifecycleStatus.get(host.name) === desired) {
      return false
    }

    const observedLifecycle = host.status?.lifecycle
    const observedCondition = host.status?.conditions?.find(
      c => c.type === STATELESS_REJECTED_CONDITION_TYPE
    )
    const observedPullPolicyCondition = host.status?.conditions?.find(
      c => c.type === STATELESS_PULL_POLICY_REJECTED_CONDITION_TYPE
    )
    const observedMatches =
      observedLifecycle !== undefined &&
      observedLifecycle.state === assessment.lifecycle.state &&
      observedLifecycle.wakeHandledGeneration === assessment.lifecycle.wakeHandledGeneration &&
      observedLifecycle.reason === assessment.lifecycle.reason &&
      observedCondition !== undefined &&
      observedCondition.status === assessment.condition.status &&
      observedCondition.reason === assessment.condition.reason &&
      observedCondition.message === assessment.condition.message &&
      observedPullPolicyCondition !== undefined &&
      observedPullPolicyCondition.status === assessment.pullPolicyCondition.status &&
      observedPullPolicyCondition.reason === assessment.pullPolicyCondition.reason &&
      observedPullPolicyCondition.message === assessment.pullPolicyCondition.message
    if (observedMatches) {
      this.lastWrittenLifecycleStatus.set(host.name, desired)
      return false
    }

    // Kubernetes convention: lastTransitionTime tracks a STATUS change, not
    // every re-assessment — keeping it stable also keeps a converged status
    // byte-identical for the dirty checks above.
    const stampCondition = (
      observed: HostCondition | undefined,
      desiredCondition: Omit<HostCondition, 'lastTransitionTime'>
    ): HostCondition => {
      const priorTransitionTime =
        observed !== undefined &&
        observed.status === desiredCondition.status &&
        observed.reason === desiredCondition.reason
          ? observed.lastTransitionTime
          : undefined
      return {
        ...desiredCondition,
        lastTransitionTime: priorTransitionTime ?? this.now().toISOString(),
      }
    }
    try {
      // D2/AP-1: TARGETED sub-object patch under the D3 resourceVersion
      // precondition. The previous implementation spread the CACHED
      // `host.status` into a whole-`/status` write, which could clobber a
      // fresher conditions[]/reason/wakeHandledGeneration that a heartbeat
      // core wrote between the cache snapshot and this write (this is the
      // highest-traffic lifecycle writer). Instead we compute the two owned
      // sub-objects against a FRESH read inside the build callback and emit
      // ONLY `/status/lifecycle` + `/status/conditions` ops, so fields this
      // method did not compute are never overwritten. The build callback
      // stamps conditions off the FRESH observed values so lastTransitionTime
      // stays stable, and it re-runs on a 409 so a racing writer's newer
      // resourceVersion wins the round and this write retries against it.
      //
      // AP-1 (the 8th costume): the D3 precondition proves nothing changed
      // AFTER the fresh read, but `assessment.lifecycle` was derived by
      // assessLifecycle from the CACHED `host` snapshot captured BEFORE the
      // fresh read. A heartbeat suspend that landed between the snapshot and
      // this fresh read passes the precondition trivially (the serialized
      // per-host chain means the fresh GET always matches the current
      // resourceVersion) yet would be silently overwritten if we wrote
      // `assessment.lifecycle` verbatim — resurrecting a `suspended` Host to
      // replicas:1 with NO wake, and/or regressing wakeHandledGeneration.
      // So we re-source the ECHOED `state` from FRESH inside the callback
      // (mirroring the heartbeat cores, which recompute both the guard AND the
      // written value from the fresh object) while preserving the assessment's
      // INTENDED state overrides.
      //
      // The clean override-vs-echo discriminator is whether the assessment's
      // `state` DIFFERS from the cached snapshot's state (`host.status`):
      //   - EQUAL → the assessment did not intend to change state; the value
      //     is a pass-through ECHO of the snapshot (assessLifecycle accepted
      //     path echoes `host.status.lifecycle.state`; a steady kill-switch on
      //     an already-active Host also echoes). Prefer FRESH so a heartbeat
      //     suspend/wake that landed under this write is preserved (never move
      //     suspended→active without a wake).
      //   - DIFFERS → the assessment INTENDED a transition and that override
      //     MUST win: kill-switch/rejection forcing active over a suspended
      //     snapshot, OR the resolveWakeBeforeScaleDown wake transition
      //     (stateless + active, derived from its OWN fresh wake read). Using
      //     `effective.stateless` alone would misclassify that wake transition
      //     as an echo and regress it back to suspended.
      // `reason` follows `state`: on the echo branch we re-source the
      // heartbeat-managed reason from fresh (mirroring the accepted-path
      // preservedReason logic); on the override branch the assessment's reason
      // (rejection message, or cleared) wins with its intended state.
      // `wakeHandledGeneration` is an ECHO in EVERY branch, never an intended
      // override, so it is kept monotonic against fresh and never regressed.
      let committedLifecycle: HostLifecycleStatus | undefined
      const result = await this.patchStatusWithPrecondition(host, fresh => {
        const freshStatus = fresh.status ?? {}
        const freshRejected = freshStatus.conditions?.find(
          c => c.type === STATELESS_REJECTED_CONDITION_TYPE
        )
        const freshPullPolicy = freshStatus.conditions?.find(
          c => c.type === STATELESS_PULL_POLICY_REJECTED_CONDITION_TYPE
        )
        const stamped = stampCondition(freshRejected, assessment.condition)
        const stampedPullPolicy = stampCondition(freshPullPolicy, assessment.pullPolicyCondition)
        const otherConditions = (freshStatus.conditions ?? []).filter(
          c =>
            c.type !== STATELESS_REJECTED_CONDITION_TYPE &&
            c.type !== STATELESS_PULL_POLICY_REJECTED_CONDITION_TYPE
        )
        const conditions = [...otherConditions, stamped, stampedPullPolicy]
        // Re-source the echoed lifecycle from fresh (see AP-1 note above).
        const freshLifecycle = freshStatus.lifecycle
        const wakeHandledGeneration = Math.max(
          assessment.lifecycle.wakeHandledGeneration,
          freshLifecycle?.wakeHandledGeneration ?? 0
        )
        const cachedState = host.status?.lifecycle?.state ?? 'active'
        // Confirmed rejection requires active even when the cached state was
        // already active. Cache uncertainty is different: preserve the fresh
        // durable state and block scale/template transitions until authority
        // returns.
        const assessmentForcesActive = assessment.condition.status === 'True'
        const assessmentIntendsStateOverride = assessment.lifecycle.state !== cachedState
        let lifecycle: HostLifecycleStatus
        if (assessmentIntendsStateOverride || assessmentForcesActive) {
          // The assessment requires a transition or forces active by policy.
          // Only wakeHandledGeneration is sourced monotonically from fresh.
          lifecycle = { ...assessment.lifecycle, wakeHandledGeneration }
        } else {
          // The assessment's state was a pass-through ECHO of the cached
          // snapshot — prefer the FRESH durable state and the heartbeat-managed
          // reason so a suspend/wake that landed since the snapshot is
          // preserved (never move suspended→active without a wake).
          const freshReason = isHeartbeatManagedLifecycleReason(freshLifecycle?.reason)
            ? freshLifecycle.reason
            : undefined
          lifecycle = {
            state: freshLifecycle?.state ?? 'active',
            wakeHandledGeneration,
            ...(freshReason !== undefined ? { reason: freshReason } : {}),
          }
        }
        if (freshStatus.lifecycle === undefined && freshStatus.conditions === undefined) {
          // A fresh Host has no /status yet: `add /status/lifecycle` would
          // fail on a missing parent, so seed the whole subresource once. All
          // subsequent writes take the targeted-op path below.
          committedLifecycle = lifecycle
          return this.fullStatusOps({ ...freshStatus, lifecycle, conditions })
        }
        // Targeted ops: never touch /status members this method did not
        // compute. The lifecycle written here is re-sourced from fresh for the
        // echoed fields (AP-1), so a heartbeat suspend/wake that landed under a
        // conditions-transition write is preserved instead of clobbered.
        committedLifecycle = lifecycle
        return [
          { op: 'add', path: '/status/lifecycle', value: lifecycle },
          { op: 'add', path: '/status/conditions', value: conditions },
        ]
      })
      if (!('skipped' in result)) {
        this.lastWrittenLifecycleStatus.set(host.name, desired)
        this.onLifecycleStatusCommitted(host, committedLifecycle ?? assessment.lifecycle)
        return true
      }
      return false
    } catch (err) {
      // Best-effort: a status-write failure (including 409-retry exhaustion)
      // is logged, not thrown — status writes must not block reconciliation.
      log.error(
        `[HostReconciler] Failed to write lifecycle status for "${host.name}" (${getErrorCode(err) ?? 'no code'}):`,
        { err }
      )
      return false
    }
  }

  // ─── Stateless heartbeat execution (Stage 3) ────────────────────────
  //
  // The StatelessLifecycleTracker DECIDES (drain verdicts, D8 idle gate,
  // podUid ordering); these methods EXECUTE. HostReconciler implements the
  // tracker's `StatelessLifecycleReconcilerPort` structurally.

  /** Public view of the synchronous effective-lifecycle derivation. */
  getEffectiveLifecycle(host: HostCRD): EffectiveHostLifecycle {
    return this.effectiveLifecycleFromCache(host)
  }

  private canSuspendFromCache(host: HostCRD): boolean {
    const lifecycle = this.effectiveLifecycleFromCache(host)
    return lifecycle.stateless && !lifecycle.suspensionBlocked
  }

  /** Public view of the clerum.io/wake-requested generation parser. */
  getWakeRequestedGeneration(host: HostCRD): number {
    return this.wakeRequestedGeneration(host)
  }

  /**
   * FRESH GET (not the watch cache) of a Host, mirroring the
   * drained-pre-scale guard's read inside reconcileCore. The watch callback
   * builds a NEW HostCRD instance per ADDED/MODIFIED event, so two
   * independently-triggered heartbeat-path callers can hold DIFFERENT
   * instances for the same host and the in-memory `host.status` reflection
   * never propagates cross-instance. Heartbeat-path status writers therefore
   * guard against the CURRENT server-side lifecycle, never the snapshot they
   * were handed. Throws on GET failure — each caller logs loudly and the
   * next polled heartbeat retries.
   */
  async readFreshHost(host: HostCRD): Promise<HostCRD> {
    const response = await this.customApi.getNamespacedCustomObject({
      group: GROUP,
      version: VERSION,
      namespace: host.namespace,
      plural: PLURAL_HOSTS,
      name: host.name,
    })
    const obj = response as {
      metadata: {
        name: string
        namespace?: string
        uid?: string
        generation?: number
        resourceVersion?: string
        annotations?: Record<string, string>
      }
      spec: HostCRD['spec']
      status?: HostCRD['status']
    }
    const fresh: HostCRD = {
      name: obj.metadata.name,
      namespace: obj.metadata.namespace ?? host.namespace,
      uid: obj.metadata.uid,
      generation: obj.metadata.generation,
      // Preserve resourceVersion so heartbeat-path status writers can pass it
      // back as an optimistic-concurrency precondition (D3). Dropping it here
      // would force every write to be an unconditioned last-write-wins patch.
      resourceVersion: obj.metadata.resourceVersion,
      annotations: obj.metadata.annotations,
      spec: obj.spec,
      status: obj.status,
    }
    this.requireSameHostIdentity(host, fresh)
    return fresh
  }

  /**
   * Suspend a stateless Host after the drained report (or drain-grace
   * expiry): durable status flip FIRST (same crash-safety precedent as the
   * wake fast-path), then a full reconcile so replicas derive to 0. The core
   * re-reads the Host FRESH from the API and proceeds only when the CURRENT
   * server-side state is `draining` — a stale watch-cache snapshot can never
   * suspend a Host whose drain decision was overturned. The fresh read and
   * the status write THROW on failure — the tracker logs and the next
   * drained heartbeat retries. The wake fast-path + drained-pre-scale guard
   * inside reconcile() remain the last line of defense against a racing wake.
   *
   * `entryWakeHandledGeneration` (AP-1 generation epoch): the
   * `wakeHandledGeneration` the tracker observed on its Host snapshot when
   * it DECIDED this suspend. Fresh `draining` alone cannot prove the drain
   * decision is still current — an aged drain:true verdict can re-persist
   * `draining` (heartbeat poller) after a wake already cancelled the drain
   * and was handled, and that handled wake leaves requested==handled so
   * NOTHING would revive the suspended Host. The commit no-ops when the
   * fresh generation advanced past the epoch, or when a wake is pending in
   * fresh (see the guards in the build callback).
   */
  async suspendHostFromHeartbeat(
    host: HostCRD,
    reason: string,
    entryWakeHandledGeneration: number
  ): Promise<SuspendFromHeartbeatOutcome> {
    return this.serializeHeartbeatMutation(
      'heartbeat suspension',
      host,
      (admittedHost, revalidate) =>
        this.suspendHostFromHeartbeatCore(
          admittedHost,
          reason,
          entryWakeHandledGeneration,
          revalidate
        )
    )
  }

  private async suspendHostFromHeartbeatCore(
    host: HostCRD,
    reason: string,
    entryWakeHandledGeneration: number,
    revalidate?: () => HostCRD
  ): Promise<SuspendFromHeartbeatOutcome> {
    // FRESH GET (see readFreshHost): guard against the CURRENT server-side
    // lifecycle, not the caller's watch-cache snapshot. The guard is re-run
    // inside the D3 precondition helper's build callback so a 409-retry
    // re-reads and re-checks it against the newest server state.
    const initialFresh = await this.readFreshHost(host)
    let writtenStatus: HostCrdStatus | undefined
    // Outcome carried out of the build callback: every skip is 'skipped_stale'
    // (evidence aged/overturned — the tracker answers drain:false) except the
    // idempotent already-suspended retry. Re-evaluated on each 409 re-build.
    let skipOutcome: SuspendFromHeartbeatOutcome = 'skipped_stale'
    const result = await this.patchStatusWithPrecondition(
      host,
      fresh => {
        // UID proves object identity, not decision freshness. A same-identity
        // Host may have advanced from stateless to stateful while the watch
        // cache trails the API. Re-evaluate the complete effective stateless
        // policy from every fresh read/retry before honoring old heartbeat
        // evidence.
        if (!this.sameHostSpecRevision(host, fresh) || !this.canSuspendFromCache(fresh)) {
          return { skip: true }
        }
        const freshLifecycle = fresh.status?.lifecycle
        if (freshLifecycle?.state !== 'draining') {
          // Every legitimate suspend follows a durable `draining` write (the
          // poller persists drain:true before the emitter can fence/flush, and
          // drain-grace is armed by the same verdict). A fresh CR that is NOT
          // draining means the drain decision has been overturned (a
          // cancel-drain or wake landed in between) — suspending would clobber
          // newer evidence. A fresh `suspended` returns as a SILENT idempotent
          // no-op: the drained-report retry path relies on exactly that.
          if (freshLifecycle?.state !== 'suspended') {
            log.info(
              `[HostReconciler] Suspend of stateless Host "${host.name}" skipped — fresh state "${freshLifecycle?.state ?? 'active'}" is not draining (drain decision overturned)`
            )
          }
          skipOutcome =
            freshLifecycle?.state === 'suspended' ? 'already_suspended' : 'skipped_stale'
          return { skip: true }
        }
        // AP-1 generation-epoch guard (aged drained evidence): a wake handled
        // AFTER the suspend decision bumps wakeHandledGeneration (wake
        // fast-path) — the pod was un-fenced and may have served turns, and
        // with requested==handled nothing would ever revive the Host after a
        // scale-down. A fresh handled generation past the caller's entry
        // epoch therefore no-ops the suspend; the emitter's next drained
        // report retries on current evidence. Do NOT key this on "fresh
        // state is active": every normal suspension commits over `draining`,
        // and an aged drain:true verdict can legitimately re-persist
        // `draining` after the wake — the generation is the only safe
        // discriminator. This applies to max-uptime drains too: skipping
        // strands nothing (the over-age pod is recycled on its next idle
        // drain cycle), while suspending would strand a just-demanded Host
        // with no reviver.
        const freshHandled = freshLifecycle.wakeHandledGeneration ?? 0
        if (freshHandled > entryWakeHandledGeneration) {
          log.info(
            `[StatelessSuspend] host=${host.name} phase=drained_report_stale reason=wake_handled_since entryGeneration=${entryWakeHandledGeneration} freshGeneration=${freshHandled} ts=${this.now().getTime()}`
          )
          return { skip: true }
        }
        // Pending-wake mirror (the wake-annotation-not-yet-processed flavor
        // of the same race): a fresh requested generation above handled is a
        // wake in flight right now — resolveWakeBeforeScaleDown would abort
        // the scale-down later, but skipping here avoids committing a
        // suspended state that must immediately be overturned.
        const freshRequested = this.wakeRequestedGeneration(fresh)
        if (freshRequested > freshHandled) {
          log.info(
            `[StatelessSuspend] host=${host.name} phase=drained_report_stale reason=wake_pending requestedGeneration=${freshRequested} handledGeneration=${freshHandled} ts=${this.now().getTime()}`
          )
          return { skip: true }
        }
        // Both the write base and the wake generation come from the FRESH
        // object, so a stale snapshot can never regress wakeHandledGeneration.
        const lifecycle: HostLifecycleStatus = {
          state: 'suspended',
          wakeHandledGeneration: freshLifecycle.wakeHandledGeneration ?? 0,
          reason,
        }
        writtenStatus = { ...(fresh.status ?? {}), lifecycle }
        return this.fullStatusOps(writtenStatus)
      },
      initialFresh,
      revalidate
    )
    if ('skipped' in result || writtenStatus === undefined) {
      return skipOutcome
    }
    // Revalidate authority/UID immediately before follow-on effects, then read
    // once more from the API. The successful PATCH attempt's fresh object is
    // the exact spec on which the status write committed, but a wake/spec
    // update can land immediately AFTER that commit and BEFORE runtime
    // reconciliation. A linearizable follow-on GET observes both our durable
    // suspended write and any such newer same-UID update; the watch cache can
    // legitimately trail either one.
    const currentAtFollowOn = revalidate?.()
    if (currentAtFollowOn) this.requireSameHostIdentity(host, currentAtFollowOn)
    this.reflectOutcome(host, target => {
      target.status = writtenStatus
    })
    const reconcileHost = await this.readFreshHost(result.fresh)
    if (!this.sameHostSpecRevision(host, reconcileHost)) {
      const error = new Error(
        `Host spec generation changed before heartbeat suspension follow-on for "${host.name}"`
      )
      error.name = 'HostMutationSpecRevisionChangedError'
      throw error
    }
    const currentAfterFollowOnRead = revalidate?.()
    if (currentAfterFollowOnRead) {
      this.requireSameHostIdentity(host, currentAfterFollowOnRead)
    }
    this.lastWrittenLifecycleStatus.delete(host.name)
    this.logSuspendedApplied(host.name)
    log.info(
      `[HostReconciler] Suspending stateless Host "${host.name}" (reason: ${reason}) — reconciling to replicas=0`
    )
    // L2: call reconcileCore (not the serialized reconcile) — this method
    // already runs inside this host's serialization slot, so re-entering the
    // per-host chain here would self-deadlock.
    await this.reconcileCore(reconcileHost, revalidate)
    // Preserve the historical caller-visible reflection contract even when
    // follow-on work correctly used a fresher API object. The tracker may hold
    // the originally admitted instance until the watch delivers the next
    // MODIFIED event; reflect the final authoritative outcome immediately so
    // it cannot make a second decision from the pre-follow-on status.
    this.reflectOutcome(host, target => {
      target.uid = reconcileHost.uid
      target.generation = reconcileHost.generation
      target.resourceVersion = reconcileHost.resourceVersion
      target.annotations = reconcileHost.annotations
      target.spec = reconcileHost.spec
      target.status = reconcileHost.status
    })
    // The wake fast-path / drained-pre-scale guard inside reconcile() may
    // have flipped the host back to active (a wake raced the suspension) —
    // in that case no scale-down happened, so no phase/metric is emitted.
    const wakePending =
      this.wakeRequestedGeneration(reconcileHost) >
      (reconcileHost.status?.lifecycle?.wakeHandledGeneration ?? 0)
    if (reconcileHost.status?.lifecycle?.state !== 'suspended' || wakePending) {
      // The durable suspended write DID commit (a racing wake revived the
      // host afterwards), so the drained evidence was honored: 'suspended'.
      return 'suspended'
    }
    this.recordSuspendedApplied(reconcileHost.name)
    return 'suspended'
  }

  /**
   * Publish the D8 condition blocking suspension into
   * status.lifecycle.reason. Change-only: an unchanged reason is a no-op
   * (heartbeats arrive every ~30s; steady state must not churn the CRD).
   * Throws on API failure — the tracker logs it loudly.
   */
  async publishSuspendBlockedReason(host: HostCRD, reason: string): Promise<void> {
    return this.serializeHeartbeatMutation(
      'heartbeat suspend-blocked reason publication',
      host,
      (admittedHost, revalidate) =>
        this.publishSuspendBlockedReasonCore(admittedHost, reason, revalidate)
    )
  }

  private async publishSuspendBlockedReasonCore(
    host: HostCRD,
    reason: string,
    revalidate?: () => HostCRD
  ): Promise<void> {
    // FRESH GET (see readFreshHost): this is a suspend-BLOCKED reason
    // ANNOTATOR, not a state writer — it must NEVER change state, only stamp
    // the reason on top of the CURRENT server-side state. AP-1 (the 9th
    // costume): the earlier implementation read state from the CACHED
    // `host.status` snapshot and spread the whole `/status` with NO
    // resourceVersion precondition. The tracker's `!idleEligible` branch calls
    // cancelDrainOnEvidence(hostRef, host) then publishSuspendBlockedReason(host)
    // against the SAME host ref; if a suspend landed on the serializeByHost
    // chain first, cancelDrainOnEvidence's fresh guard skips (fresh is no
    // longer draining) leaving host.status stale `draining`, and this writer
    // would then re-source that stale `draining` and write state back —
    // resurrecting a just-suspended Host (replicas derive from state) with no
    // wake and no precondition to catch it. So we re-source state +
    // wakeHandledGeneration from FRESH inside the D3 precondition callback,
    // GUARD against fresh states where a suspend-blocked reason is no longer
    // meaningful (fresh `suspended`/`draining` — the Host is no longer being
    // kept active/blocked), and emit ONLY a targeted /status/lifecycle op.
    const initialFresh = await this.readFreshHost(host)
    let writtenLifecycle: HostLifecycleStatus | undefined
    const result = await this.patchStatusWithPrecondition(
      host,
      fresh => {
        if (
          !this.sameHostSpecRevision(host, fresh) ||
          !this.effectiveLifecycleFromCache(fresh).stateless
        ) {
          return { skip: true }
        }
        const freshLifecycle = fresh.status?.lifecycle
        const freshState = freshLifecycle?.state ?? 'active'
        if (freshState !== 'active') {
          // A suspend-blocked reason only annotates a Host being KEPT active.
          // Fresh `suspended`/`draining` means the state changed underneath
          // (a suspend/drain landed first) — writing the reason would either
          // resurrect a suspended Host or misannotate a draining one. No-op.
          return { skip: true }
        }
        // Re-evaluate the change-only idempotency short-circuit against fresh:
        // an unchanged reason on the fresh active state is a no-op (heartbeats
        // arrive ~30s; steady state must not churn the CRD).
        if (freshLifecycle?.reason === reason) {
          return { skip: true }
        }
        const lifecycle: HostLifecycleStatus = {
          state: freshState,
          wakeHandledGeneration: freshLifecycle?.wakeHandledGeneration ?? 0,
          reason,
        }
        writtenLifecycle = lifecycle
        return [{ op: 'add', path: '/status/lifecycle', value: lifecycle }]
      },
      initialFresh,
      revalidate
    )
    if ('skipped' in result || writtenLifecycle === undefined) {
      return
    }
    const reflectedStatus = { ...(host.status ?? {}), lifecycle: writtenLifecycle }
    this.reflectOutcome(host, target => {
      target.status = reflectedStatus
    })
    this.lastWrittenLifecycleStatus.delete(host.name)
  }

  /**
   * Persist the tracker's drain decision durably. With heartbeats ingested
   * by control-api (which answers the emitter's `{drain}` verdict from
   * `status.lifecycle.state`), this status write is the ONLY channel through
   * which a drain decision reaches the pod. Idempotent by state: only an
   * `active` Host flips to `draining` — an already-draining Host is a no-op
   * and a `suspended` Host (drained-report suspension in the same poll pass)
   * is never clobbered back. The state guard runs against a FRESH API read
   * of the Host, never the caller's watch-cache snapshot (a stale `active`
   * snapshot over an actually-suspended CR must not write a running state
   * over it). Fresh `active` alone is NOT sufficient (AP-1): `active` is
   * exactly the state a just-handled wake produces, so an AGED drain verdict
   * queued behind that wake would re-fence the woken pod — and its consumed
   * wake (requested==handled) revives nothing. The commit therefore also
   * carries the tracker's `entryWakeHandledGeneration` epoch from VERDICT
   * time and no-ops when the fresh handled generation advanced past it, or
   * when a wake is pending in fresh. The stale suspend-blocked reason is dropped on
   * the flip (the Host is draining, not blocked). Throws on API failure —
   * the poller logs it loudly and the next polled beat retries. Cancel-drain
   * (`draining` → `active`) is executed by the tracker via
   * markHostActiveFromHeartbeat on activity/pending-wake evidence; the wake
   * fast-path also flips it back and remains the only reviver of a
   * `suspended` Host.
   */
  async markHostDrainingFromHeartbeat(
    host: HostCRD,
    entryWakeHandledGeneration: number
  ): Promise<void> {
    return this.serializeHeartbeatMutation(
      'heartbeat draining transition',
      host,
      (admittedHost, revalidate) =>
        this.markHostDrainingFromHeartbeatCore(admittedHost, entryWakeHandledGeneration, revalidate)
    )
  }

  private async markHostDrainingFromHeartbeatCore(
    host: HostCRD,
    entryWakeHandledGeneration: number,
    revalidate?: () => HostCRD
  ): Promise<void> {
    // FRESH GET (see readFreshHost): a stale `active` snapshot over an
    // actually-`suspended` CR would write `draining` (a running state) and
    // the next reconcile would resurrect the pod with NO wake. Proceed only
    // when the CURRENT server-side state is active. The guard runs inside the
    // D3 precondition helper's build callback so a 409-retry re-checks it.
    const initialFresh = await this.readFreshHost(host)
    let writtenStatus: HostCrdStatus | undefined
    const result = await this.patchStatusWithPrecondition(
      host,
      fresh => {
        if (!this.sameHostSpecRevision(host, fresh) || !this.canSuspendFromCache(fresh)) {
          return { skip: true }
        }
        const freshLifecycle = fresh.status?.lifecycle
        if ((freshLifecycle?.state ?? 'active') !== 'active') {
          return { skip: true }
        }
        // AP-1 generation-epoch guard (aged drain verdict): a wake handled
        // AFTER this verdict was decided bumps wakeHandledGeneration and
        // leaves the CR `active` — the exact state this writer commits over.
        // Re-writing `draining` from the aged verdict would re-fence the
        // just-woken pod with requested==handled (nothing revives it). A
        // fresh handled generation past the verdict's entry epoch therefore
        // no-ops the write; the next polled beat re-decides on fresh
        // evidence.
        const freshHandled = freshLifecycle?.wakeHandledGeneration ?? 0
        if (freshHandled > entryWakeHandledGeneration) {
          log.info(
            `[StatelessSuspend] host=${host.name} phase=draining_write_stale reason=wake_handled_since entryGeneration=${entryWakeHandledGeneration} freshGeneration=${freshHandled} ts=${this.now().getTime()}`
          )
          return { skip: true }
        }
        // Pending-wake mirror: a wake in flight right now cancels the drain
        // by definition ("pending wake wins") — never fence over it.
        const freshRequested = this.wakeRequestedGeneration(fresh)
        if (freshRequested > freshHandled) {
          log.info(
            `[StatelessSuspend] host=${host.name} phase=draining_write_stale reason=wake_pending requestedGeneration=${freshRequested} handledGeneration=${freshHandled} ts=${this.now().getTime()}`
          )
          return { skip: true }
        }
        const lifecycle: HostLifecycleStatus = {
          state: 'draining',
          wakeHandledGeneration: freshHandled,
        }
        writtenStatus = { ...(fresh.status ?? {}), lifecycle }
        return this.fullStatusOps(writtenStatus)
      },
      initialFresh,
      revalidate
    )
    if ('skipped' in result || writtenStatus === undefined) {
      return
    }
    this.reflectOutcome(host, target => {
      target.status = writtenStatus
    })
    this.lastWrittenLifecycleStatus.delete(host.name)
    log.info(
      `[HostReconciler] Marked stateless Host "${host.name}" draining — control-api now answers {drain:true} from Host.status`
    )
  }

  /**
   * Tracker-side cancel-drain: revert an in-flight drain when the tracker
   * sees HOST evidence (activity or a pending wake) on a polled heartbeat
   * while the CR still says `draining`. Without this write the emitter is
   * wedged behind the fence whenever the rpc-proxy wake call never lands —
   * control-api keeps answering {drain:true} from Host.status while the
   * tracker keeps deciding drain:false. Guarded by state against a FRESH
   * API read of the Host (never the caller's watch-cache snapshot): ONLY a
   * currently-`draining` Host reverts — `active` is a no-op and `suspended`
   * is NEVER touched
   * (reviving a suspended Host is exclusively the wake fast-path's job).
   * Throws on API failure — the tracker logs it loudly and the next polled
   * heartbeat retries.
   */
  async markHostActiveFromHeartbeat(host: HostCRD): Promise<void> {
    return this.serializeHeartbeatMutation(
      'heartbeat cancel-drain transition',
      host,
      (admittedHost, revalidate) => this.markHostActiveFromHeartbeatCore(admittedHost, revalidate)
    )
  }

  private async markHostActiveFromHeartbeatCore(
    host: HostCRD,
    revalidate?: () => HostCRD
  ): Promise<void> {
    // FRESH GET (see readFreshHost): a stale `draining` snapshot over an
    // actually-`suspended` CR must never resurrect it (and would regress
    // wakeHandledGeneration). Proceed only when the CURRENT server-side
    // state is draining; the wake generation comes from the fresh lifecycle.
    // The guard runs inside the D3 precondition helper's build callback so a
    // 409-retry re-checks it against the newest server state.
    const initialFresh = await this.readFreshHost(host)
    let skippedNotDraining: string | undefined
    let writtenStatus: HostCrdStatus | undefined
    const result = await this.patchStatusWithPrecondition(
      host,
      fresh => {
        if (
          !this.sameHostSpecRevision(host, fresh) ||
          !this.effectiveLifecycleFromCache(fresh).stateless
        ) {
          return { skip: true }
        }
        const freshLifecycle = fresh.status?.lifecycle
        if (freshLifecycle?.state !== 'draining') {
          skippedNotDraining = freshLifecycle?.state ?? 'active'
          return { skip: true }
        }
        const lifecycle: HostLifecycleStatus = {
          state: 'active',
          wakeHandledGeneration: freshLifecycle.wakeHandledGeneration ?? 0,
        }
        writtenStatus = { ...(fresh.status ?? {}), lifecycle }
        return this.fullStatusOps(writtenStatus)
      },
      initialFresh,
      revalidate
    )
    if ('skipped' in result || writtenStatus === undefined) {
      log.info(
        `[StatelessLifecycle] host=${host.name} cancel-drain skipped — fresh state "${skippedNotDraining ?? 'active'}" is not draining`
      )
      return
    }
    this.reflectOutcome(host, target => {
      target.status = writtenStatus
    })
    this.lastWrittenLifecycleStatus.delete(host.name)
    log.info(
      `[StatelessLifecycle] host=${host.name} cancel-drain: activity evidence while draining — reverting status to active`
    )
  }

  /**
   * Resolve a Host pod's creationTimestamp by pod UID (max-uptime ceiling).
   * The mcp-host Deployment selector is `app=<host.name>` (see
   * buildDeployment). Returns null when no pod with that UID is visible.
   */
  async findPodCreationTimestamp(host: HostCRD, podUid: string): Promise<Date | null> {
    const pods = await this.coreApi.listNamespacedPod({
      namespace: host.namespace,
      labelSelector: `app=${host.name}`,
    })
    for (const pod of pods.items ?? []) {
      if (pod.metadata?.uid === podUid) {
        return pod.metadata.creationTimestamp ? new Date(pod.metadata.creationTimestamp) : null
      }
    }
    return null
  }

  // ─── Wake fast-path (Stage 4.3) ─────────────────────────────────────

  /**
   * Parse the clerum.io/wake-requested annotation into a monotonic wake
   * generation. Absent means no wake intent (0). A malformed (non-integer)
   * value is also treated as no wake intent and logged loudly once per
   * distinct value, so a broken writer is operator-visible without flooding
   * the periodic-resync log. Never throws.
   */
  private wakeRequestedGeneration(host: HostCRD): number {
    const raw = host.annotations?.[WAKE_REQUESTED_ANNOTATION]
    if (raw === undefined) {
      return 0
    }
    const parsed = Number.parseInt(raw, 10)
    if (Number.isNaN(parsed)) {
      if (this.malformedWakeAnnotationLogged.get(host.name) !== raw) {
        this.malformedWakeAnnotationLogged.set(host.name, raw)
        log.error(
          `[HostReconciler] Malformed ${WAKE_REQUESTED_ANNOTATION} annotation on Host "${host.name}": ${JSON.stringify(
            raw
          )} is not an integer; treating it as no wake intent`
        )
      }
      return 0
    }
    this.malformedWakeAnnotationLogged.delete(host.name)
    return parsed
  }

  /**
   * Wake fast-path: runs at the very start of reconcile() for stateless
   * Hosts — the watch-event callback AND the periodic resync both funnel
   * through reconcile(), so a wake event dropped on a watch disconnect is
   * recovered at the next resync. A wake generation above
   * status.lifecycle.wakeHandledGeneration is a pending wake, executed by
   * CURRENT durable state:
   *   - suspended → flip to active + minimal replicas=1 scale patch on the
   *     existing Deployment (a missing Deployment falls through to the full
   *     reconcile, which builds it with replicas=1)
   *   - draining  → cancel-drain: flip to active; replicas were never
   *     dropped, so no scale call is issued
   *   - active    → record wakeHandledGeneration only (no-op wake; keeps
   *     the comparison monotonic)
   * Repeated events carrying an already-handled generation are no-ops. The
   * fast-path NEVER awaits pod readiness: the watch callback is serial for
   * the whole fleet, so it is bounded to one fresh GET + one status write +
   * one scale patch (wakes are user-triggered and rare). AP-1: the pending
   * wake and the transition are re-decided from a FRESH read at the commit
   * point and written as a TARGETED /status/lifecycle op under the D3
   * resourceVersion precondition — see the notes inside.
   */
  async handleWakeFastPath(host: HostCRD): Promise<boolean> {
    if (host.spec.lifecycle?.stateless !== true) {
      return false
    }
    const wakeRequested = this.wakeRequestedGeneration(host)
    const wakeHandled = host.status?.lifecycle?.wakeHandledGeneration ?? 0
    if (wakeRequested <= wakeHandled) {
      return false
    }

    // AP-1 (the 10th costume): the cached gate above only DISCOVERS a
    // potentially-pending wake — the commit is re-decided from a FRESH read
    // below. The previous implementation decided requested/handled from the
    // CACHED snapshot and spread the whole cached `/status` with no
    // resourceVersion precondition, so a snapshot captured before a hardened
    // sibling's write on the same serialized chain silently resurrected that
    // sibling's fresher conditions/reason. A fresh-read failure is logged
    // and returns without a write — the wake is NOT lost: the annotation is
    // durable, the drained-pre-scale guard re-checks it before any
    // replicas:0 apply, and the periodic resync re-enters this fast-path.
    let initialFresh: HostCRD
    try {
      initialFresh = await this.readFreshHost(host)
    } catch (err) {
      log.error(
        `[HostReconciler] Wake fast-path fresh Host read failed for "${host.name}" (${getErrorCode(err) ?? 'no code'}):`,
        { err }
      )
      return false
    }
    const freshWakeRequested = this.wakeRequestedGeneration(initialFresh)
    const freshWakeHandled = initialFresh.status?.lifecycle?.wakeHandledGeneration ?? 0
    if (freshWakeRequested <= freshWakeHandled) {
      // A sibling writer (or a prior fast-path pass over another snapshot)
      // already handled this wake: no write, and the wake phases are NOT
      // logged a second time (they were emitted by whoever handled it).
      // Reflect the fresh truth on the in-memory object so the heavy
      // reconcile body derives replicas from the current durable state
      // instead of the stale event payload.
      this.reflectOutcome(host, target => {
        target.annotations = initialFresh.annotations
        target.status = initialFresh.status
      })
      return false
    }

    const currentState = initialFresh.status?.lifecycle?.state ?? 'active'
    log.info(
      `[HostReconciler] Wake fast-path for "${host.name}": generation ${freshWakeRequested} > handled ${freshWakeHandled} while ${currentState}`
    )
    // Stage 6 (W2): machine-parseable wake-phase timestamps for the
    // wake-budget script. One line per phase, correlated by generation.
    log.info(
      `[StatelessWake] host=${host.name} generation=${freshWakeRequested} phase=wake_observed ts=${this.now().getTime()}`
    )

    // Durable flip FIRST: the woken state must survive an HCC crash between
    // this write and the scale patch (the resync re-derives replicas from
    // the CRD status, not from HCC memory). The write goes through the D3
    // precondition helper: the build callback re-decides requested/handled
    // from FRESH (a 409-retry re-reads and respects a racing sibling's newer
    // write, returning skip) and emits ONLY a targeted /status/lifecycle op,
    // so fresh conditions and a fresher heartbeat-managed reason are never
    // spread over. Every wake transition lands on active; the stale cached
    // reason is dropped, mirroring markHostActiveFromHeartbeatCore and
    // resolveWakeBeforeScaleDown (the change-only suspend-blocked publisher
    // re-stamps a still-valid reason on the next heartbeat).
    let latestFresh = initialFresh
    let writtenLifecycle: HostLifecycleStatus | undefined
    let commitState = currentState
    try {
      const result = await this.patchStatusWithPrecondition(
        host,
        fresh => {
          latestFresh = fresh
          const freshLifecycle = fresh.status?.lifecycle
          const buildRequested = this.wakeRequestedGeneration(fresh)
          const buildHandled = freshLifecycle?.wakeHandledGeneration ?? 0
          if (buildRequested <= buildHandled) {
            // A racing writer handled the wake between the fresh read and
            // this (re)build — skip: no write, no duplicate wake phases.
            return { skip: true }
          }
          commitState = freshLifecycle?.state ?? 'active'
          const lifecycle: HostLifecycleStatus = {
            state: 'active',
            wakeHandledGeneration: buildRequested,
          }
          writtenLifecycle = lifecycle
          if (fresh.status?.lifecycle === undefined && fresh.status?.conditions === undefined) {
            // A fresh Host has no /status yet: `add /status/lifecycle` would
            // fail on a missing parent, so seed the whole subresource once
            // (there are no fresher conditions to spread over when /status
            // does not exist) — same seed as writeLifecycleStatusToCluster.
            return this.fullStatusOps({ ...(fresh.status ?? {}), lifecycle })
          }
          return [{ op: 'add', path: '/status/lifecycle', value: lifecycle }]
        },
        initialFresh
      )
      if ('skipped' in result || writtenLifecycle === undefined) {
        this.reflectOutcome(host, target => {
          target.annotations = latestFresh.annotations
          target.status = latestFresh.status
        })
        return false
      }
    } catch (err) {
      // Same precedent as writeLifecycleStatusToCluster: a status-write
      // failure is logged and retried on the next pass. The wake is NOT
      // lost — the annotation is durable and the drained-pre-scale guard in
      // reconcile() re-checks it before any replicas:0 apply, so a pending
      // wake can never be raced away by the suspension.
      log.error(
        `[HostReconciler] Wake fast-path status write failed for "${host.name}" (${getErrorCode(err) ?? 'no code'}):`,
        { err }
      )
      return false
    }
    // Reflect the flip on the in-memory object so the heavy reconcile body
    // (assessLifecycle reads host.status) derives replicas from the woken
    // state instead of re-suspending from the stale event payload. The base
    // is the FRESH status the write committed over, never the cached spread.
    const reflectedStatus = { ...(latestFresh.status ?? {}), lifecycle: writtenLifecycle }
    this.reflectOutcome(host, target => {
      target.annotations = latestFresh.annotations
      target.status = reflectedStatus
    })
    log.info(
      `[StatelessWake] host=${host.name} generation=${writtenLifecycle.wakeHandledGeneration} phase=status_flipped ts=${this.now().getTime()}`
    )

    if (commitState !== 'suspended') {
      // draining (cancel-drain) or active (record-only): replicas were
      // never dropped — no scale call.
      return false
    }

    // Cache recovery uses the full reconcile's owned, resource-version-fenced
    // replica-only update. Do not send the unfenced wake scale patch here.
    if (!this.isCommunicationChannelCacheSynced()) return true

    // Minimal scale patch — NOT the full buildDeployment replace. It also
    // starts the pod even when the heavy body aborts early (e.g. on a
    // transient Secret read failure).
    try {
      await this.appsApi.patchNamespacedDeployment(
        { name: host.name, namespace: host.namespace, body: { spec: { replicas: 1 } } },
        {
          middleware: [
            k8s.setHeaderMiddleware('Content-Type', 'application/strategic-merge-patch+json'),
          ],
        }
      )
      log.info(`[HostReconciler] Wake fast-path scaled Deployment "${host.name}" to replicas=1`)
      log.info(
        `[StatelessWake] host=${host.name} generation=${writtenLifecycle.wakeHandledGeneration} phase=replicas_patched ts=${this.now().getTime()}`
      )
      this.markHostNotSuspended(host.name)
      this.recordScaleTransition(host.name, 'up')
      return true
    } catch (err) {
      if (getErrorCode(err) === 404) {
        // Deployment missing (e.g. deleted while suspended): fall through —
        // the full reconcile below builds it with replicas=1.
        log.info(
          `[HostReconciler] Wake fast-path: Deployment "${host.name}" not found; the full reconcile will create it`
        )
        return true
      }
      if (getErrorCode(err) === 403) {
        // Compatibility with deployed RBAC that grants update but not patch on
        // mcp-host Deployments. Keep the fast-path bounded to replicas only by
        // reusing the current object and preserving its resourceVersion.
        try {
          const deployment = await this.appsApi.readNamespacedDeployment({
            name: host.name,
            namespace: host.namespace,
          })
          if (!deployment.spec) {
            throw new Error('deployment has no spec')
          }
          deployment.spec.replicas = 1
          await this.appsApi.replaceNamespacedDeployment({
            name: host.name,
            namespace: host.namespace,
            body: deployment,
          })
          log.warn(
            `[HostReconciler] Wake fast-path scale patch forbidden for "${host.name}"; used deployment update fallback`
          )
          log.info(`[HostReconciler] Wake fast-path scaled Deployment "${host.name}" to replicas=1`)
          log.info(
            `[StatelessWake] host=${host.name} generation=${writtenLifecycle.wakeHandledGeneration} phase=replicas_patched ts=${this.now().getTime()}`
          )
          this.markHostNotSuspended(host.name)
          this.recordScaleTransition(host.name, 'up')
          return true
        } catch (fallbackErr) {
          log.error(
            `[HostReconciler] Wake fast-path deployment update fallback failed for "${host.name}":`,
            { err: fallbackErr }
          )
          return true
        }
      }
      // Surfaced loudly; ensureDeployment in this same reconcile pass
      // replaces the Deployment with replicas=1, so nothing is masked.
      log.error(`[HostReconciler] Wake fast-path scale patch failed for "${host.name}":`, { err })
      return true
    }
  }

  /**
   * Stage 6 scale-transition metric: one counter line per replicas 0→1 (up)
   * or 1→0 (down) transition driven by the wake fast-path or the heartbeat
   * suspension. Totals live in HCC memory only and reset on restart —
   * acceptable: the wake-budget script correlates within one HCC run.
   */
  private recordScaleTransition(hostName: string, direction: 'up' | 'down'): void {
    const total = (this.scaleTransitionsByHost.get(hostName) ?? 0) + 1
    this.scaleTransitionsByHost.set(hostName, total)
    log.info(
      `[StatelessMetric] scale_transition host=${hostName} direction=${direction} total=${total}`
    )
  }

  recordSuspendedApplied(hostName: string): void {
    this.logSuspendedApplied(hostName)
    if (this.suspendedScaleMetricRecordedByHost.has(hostName)) {
      return
    }
    this.suspendedScaleMetricRecordedByHost.add(hostName)
    this.recordScaleTransition(hostName, 'down')
  }

  private logSuspendedApplied(hostName: string): void {
    if (this.suspendedAppliedLoggedByHost.has(hostName)) {
      return
    }
    this.suspendedAppliedLoggedByHost.add(hostName)
    log.info(
      `[StatelessSuspend] host=${hostName} phase=suspended_applied ts=${this.now().getTime()}`
    )
  }

  markHostNotSuspended(hostName: string): void {
    this.suspendedAppliedLoggedByHost.delete(hostName)
    this.suspendedScaleMetricRecordedByHost.delete(hostName)
  }

  /**
   * RFC 6902 JSON Patch op. `add` replaces an existing member and creates a
   * missing one; `test` is used only for structural preconditions we do not
   * currently need (resourceVersion is enforced via an `add` op on
   * /metadata/resourceVersion, honored by the apiserver as an
   * optimistic-concurrency precondition).
   */
  private prependResourceVersionOp(
    resourceVersion: string | undefined,
    ops: Array<{ op: 'add'; path: string; value: unknown }>
  ): Array<{ op: 'add'; path: string; value: unknown }> {
    if (resourceVersion === undefined) {
      // A watch-cache snapshot with no resourceVersion falls back to an
      // unconditioned write — but every heartbeat-path caller reads fresh
      // first, so that fallback never fires in practice.
      return ops
    }
    // metadata.resourceVersion carried on the object being written is honored
    // by the apiserver as an optimistic-concurrency precondition: the patch is
    // rejected with 409 if the CR changed since the read that produced this
    // resourceVersion (Kubernetes API concepts, "Updates to existing
    // resources").
    return [{ op: 'add', path: '/metadata/resourceVersion', value: resourceVersion }, ...ops]
  }

  /**
   * D3: write a Host /status subresource under an optimistic-concurrency
   * precondition with a bounded 409-retry, mirroring control-api's
   * patchAnnotationMonotonic. `build` runs against a FRESH read of the Host
   * and returns either the JSON Patch ops to apply or `{ skip: true }` when the
   * guard no longer holds (e.g. the drain decision was overturned between the
   * caller's read and this write). On a 409 the loop re-reads fresh, re-runs
   * `build` (so the guard is re-checked against the newest server state) and
   * re-patches; exhausting the attempts throws loudly so the caller logs it
   * and the next heartbeat retries. `initialFresh` is the fresh read the
   * caller already performed on attempt 1 (avoids a redundant GET); pass
   * undefined to have the helper read fresh itself. The helper prepends the
   * resourceVersion precondition op from the fresh read to whatever ops
   * `build` returns.
   *
   * SINGLE-REPLICA COUPLING: this precondition defends against concurrent
   * writers, but HCC's safety model still assumes replicas:1 without leader
   * election (see warnSingleReplicaInvariantOnce) — the serializeByHost chain
   * only orders writers WITHIN one process. The 409-retry closes the residual
   * cross-writer window that serialization alone cannot.
   */
  private async patchStatusWithPrecondition(
    host: HostCRD,
    build: (fresh: HostCRD) => Array<{ op: 'add'; path: string; value: unknown }> | { skip: true },
    initialFresh?: HostCRD,
    revalidate?: () => HostCRD
  ): Promise<{ fresh: HostCRD } | { skipped: true }> {
    const maxAttempts = 5
    let fresh = initialFresh
    for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
      const currentAtAttempt = revalidate?.()
      if (currentAtAttempt) this.requireSameHostIdentity(host, currentAtAttempt)
      if (fresh === undefined) {
        fresh = await this.readFreshHost(host)
      }
      if (host.uid !== undefined && fresh.resourceVersion === undefined) {
        const error = new Error(
          `Host status mutation for "${host.name}" has no fresh resourceVersion; refusing an unconditioned write`
        )
        error.name = 'HostMutationPreconditionUnavailableError'
        throw error
      }
      const built = build(fresh)
      if ('skip' in built) {
        return { skipped: true }
      }
      try {
        const currentBeforePatch = revalidate?.()
        if (currentBeforePatch) this.requireSameHostIdentity(host, currentBeforePatch)
        await this.customApi.patchNamespacedCustomObjectStatus({
          group: GROUP,
          version: VERSION,
          namespace: host.namespace,
          plural: PLURAL_HOSTS,
          name: host.name,
          body: this.prependResourceVersionOp(fresh.resourceVersion, built),
        })
        return { fresh }
      } catch (err) {
        if (getErrorCode(err) === 409) {
          if (attempt < maxAttempts) {
            // The CR changed under us (a concurrent lifecycle writer or an
            // unrelated metadata write). Re-read fresh and re-evaluate the
            // guard so a decision the racing writer just made is respected.
            log.warn(
              `[HostReconciler] Status write for "${host.name}" hit 409 (stale resourceVersion); re-reading fresh and retrying (attempt ${attempt}/${maxAttempts})`
            )
            fresh = undefined
            continue
          }
          // Bounded retries exhausted on a persistent conflict: fail LOUD with
          // a descriptive message (never swallow, never last-write-wins).
          throw new Error(
            `[HostReconciler] Status write for "${host.name}" failed after ${maxAttempts} attempts (persistent 409 conflict)`
          )
        }
        throw err
      }
    }
    // Unreachable: the loop either returns, retries, or throws above.
    throw new Error(
      `[HostReconciler] Status write for "${host.name}" exited its retry loop unexpectedly`
    )
  }

  /** Full-status `/status` write ops (used by the heartbeat-path cores). */
  private fullStatusOps(status: HostCrdStatus): Array<{ op: 'add'; path: string; value: unknown }> {
    // `add` (not `replace`): add replaces an existing member and creates a
    // missing one, and a fresh Host has no /status yet.
    return [{ op: 'add', path: '/status', value: status }]
  }

  /**
   * L2: serialize `work` for a single host onto that host's promise chain so
   * concurrent callers (watch MODIFIED events, periodic resync, SFS/CC/Context
   * fan-out, and the heartbeat tracker) never interleave writes for the same
   * host. Work for DIFFERENT hosts runs concurrently. Errors propagate to the
   * caller; a rejected slot must NOT poison the chain, so the stored tail is
   * the settled (caught) promise while the caller still sees the real result.
   */
  serializeByHost<T>(hostName: string, work: () => Promise<T>): Promise<T> {
    const prior = this.reconcileChainByHost.get(hostName) ?? Promise.resolve()
    const result = prior.then(work)
    // Tail swallows rejection so the next queued call still runs; drop the
    // entry once it is the current tail to avoid unbounded map growth.
    const tail = result.then(
      () => undefined,
      () => undefined
    )
    this.reconcileChainByHost.set(hostName, tail)
    void tail.then(() => {
      if (this.reconcileChainByHost.get(hostName) === tail) {
        this.reconcileChainByHost.delete(hostName)
      }
    })
    return result
  }

  /**
   * Capture admission before enqueueing, then resolve the current Host only
   * after this mutation reaches the front of its own serializer. Keeping the
   * check here avoids wrapping a lifecycle call in HostReconciler's serializer,
   * which would recursively acquire the same Host chain and deadlock.
   */
  private serializeHeartbeatMutation<T>(
    action: string,
    host: HostCRD,
    work: (admittedHost: HostCRD, revalidate: () => HostCRD) => Promise<T>
  ): Promise<T> {
    const revalidate = this.prepareHostMutationAdmission(action, host)
    return this.serializeByHost(host.name, () => work(revalidate(), revalidate))
  }

  /**
   * A resourceVersion conflict can be caused by ordinary writes or by a
   * delete/recreate. Every fresh read, including every bounded 409 retry, must
   * remain on the Kubernetes UID that was admitted. Name equality is not an
   * identity fence. Production watch snapshots always carry UID; if only one
   * side carries it, fail closed rather than guessing.
   */
  /**
   * H2: apply a committed outcome to BOTH visible surfaces:
   *  1. the local working object (`host`) — with a resolver wired this is the
   *     admitted CLONE (serializeHeartbeatMutation), so this keeps same-pass
   *     logic (assessLifecycle, drained-pre-scale guard, follow-on reconcile)
   *     reading the outcome exactly as before; standalone callers (no resolver)
   *     get the legacy caller-visible reflection unchanged;
   *  2. the CURRENT cache entry, via the uid-guarded reflector (no-op unless
   *     McpServerWatcher wired it).
   * `host.uid` is the admitted identity (admission proved requested.uid ===
   * current.uid; readFreshHost re-proves it), so passing it after mutate() ran
   * is equivalent to passing it before.
   *
   * `mutate` closures MUST only ASSIGN precomputed values — never derive from
   * `target` — so both surfaces receive the identical outcome.
   */
  private reflectOutcome(host: HostCRD, mutate: (target: HostCRD) => void): void {
    mutate(host)
    this.reflectHostOutcome(host.name, host.uid, mutate)
  }

  private requireSameHostIdentity(expected: HostCRD, current: HostCRD): void {
    if (expected.uid === undefined && current.uid === undefined) return
    if (expected.uid === undefined || current.uid === undefined || expected.uid !== current.uid) {
      const error = new Error(
        `Host identity changed while mutating "${expected.name}"; refusing to cross a same-name recreation`
      )
      error.name = 'HostMutationIdentityChangedError'
      throw error
    }
  }

  /**
   * Heartbeat evidence belongs to one Host spec epoch. UID equality allows
   * status-only updates and retries, but a generation change can replace the
   * runtime inputs (context, Secret, image, lifecycle policy, etc.) while
   * remaining stateless. Never apply an older runtime's heartbeat decision to
   * that new spec. UID-less standalone fixtures retain their legacy behavior;
   * production objects (which carry UID) fail closed on a missing generation.
   */
  private sameHostSpecRevision(expected: HostCRD, current: HostCRD): boolean {
    if (expected.generation === undefined && current.generation === undefined) {
      return expected.uid === undefined && current.uid === undefined
    }
    return (
      expected.generation !== undefined &&
      current.generation !== undefined &&
      expected.generation === current.generation
    )
  }

  /**
   * Drained-pre-scale guard (Stage 4.3): a pending wake must abort the
   * suspension IMMEDIATELY before replicas:0 derives from the assessment.
   * Normally the wake fast-path already flipped the state, but when its
   * status write failed the assessment still observes suspended — run the
   * wake transition here instead of scaling down over it.
   *
   * M2: `host` came from the informer/watch CACHE. A wake PATCH that landed
   * AFTER this cache entry but BEFORE its MODIFIED event is processed is
   * invisible to the cached annotations, so the cached guard could pass and
   * strand the wake behind replicas:0 until the next resync. Before
   * committing the scale-down we therefore re-check the guard against a
   * FRESH GET (not the cache) of the Host metadata+status. If the fresh
   * read sees a pending wake, the scale-down is aborted and the wake
   * transition runs instead (the returned assessment derives replicas=1).
   * A GET failure returns null — fail loud and skip the scale this pass
   * (the periodic resync retries) rather than scaling on stale data.
   *
   * AP-1/AP-5 (FIX 1): the wake check alone is NOT enough. assessLifecycle
   * echoes the reconcile payload's CACHED `host.status`, and ensureDeployment
   * derives replicas from that assessment — so a stale cached lifecycle
   * STATE (not just a missed wake) could commit the wrong replicas:
   *   - cached `suspended` over a fresh `active` (wake fully handled,
   *     requested==handled) → replicas:0 would KILL the live serving pod
   *     with nothing left to revive it until resync;
   *   - cached `active` over a fresh `suspended` → replicas:1 would
   *     resurrect a suspended Host with NO wake.
   * So this guard runs for EVERY stateless assessment and re-derives the
   * scale-relevant state from the SAME fresh read: when fresh disagrees with
   * the cached assessment across the suspended/not-suspended boundary, the
   * assessment is rebuilt from FRESH (mirroring how the wake fast-path's
   * epoch guard reflects fresh onto the in-memory host). The replicas value
   * handed to buildDeployment is therefore always a function of FRESH state
   * whenever fresh and cache disagree. Draining↔active disagreements are
   * left to the status writers — both derive replicas=1, so they are not
   * scale-relevant and rebuilding would only churn the heartbeat writers.
   */
  async resolveWakeBeforeScaleDown(
    host: HostCRD,
    assessment: HostLifecycleAssessment
  ): Promise<HostLifecycleAssessment | null> {
    if (assessment.effective.suspensionBlocked) {
      // Cache authority loss blocks lifecycle-driven scale-down; preserve the
      // durable state for accurate reporting without mutating the Deployment.
      return assessment
    }
    if (!assessment.effective.stateless) {
      // Kill-switch/rejection assessments INTEND active+replicas:1 regardless
      // of the durable state — never second-guess them from fresh.
      return assessment
    }
    const cachedState = assessment.effective.state
    let fresh: HostCRD
    try {
      fresh = await this.readFreshHost(host)
    } catch (err) {
      if (cachedState === 'suspended') {
        // Fail loud and skip the scale-down this pass: a suspend derived from
        // a possibly-stale cache must NOT strand a racing wake at replicas:0.
        // The periodic resync re-runs this reconcile with a fresh cache.
        log.error(
          `[HostReconciler] Drained-pre-scale guard for "${host.name}": fresh Host read failed; skipping replicas=0 this pass (resync will retry):`,
          { err }
        )
      } else {
        // Same posture in the mirror direction: never commit replicas from a
        // possibly-stale cached state when the fresh read is unavailable.
        log.error(
          `[HostReconciler] Stateless replicas guard for "${host.name}": fresh Host read failed; skipping the Deployment scale this pass (resync will retry):`,
          { err }
        )
      }
      return null
    }
    const freshWakeRequested = this.wakeRequestedGeneration(fresh)
    const freshWakeHandled = fresh.status?.lifecycle?.wakeHandledGeneration ?? 0
    const freshState = fresh.status?.lifecycle?.state ?? 'active'
    if (freshWakeRequested > freshWakeHandled) {
      if (cachedState === 'suspended') {
        log.info(
          `[HostReconciler] Drained-pre-scale guard for "${host.name}": aborting replicas=0 — fresh wake generation ${freshWakeRequested} > handled ${freshWakeHandled}`
        )
        // A wake is pending in the fresh read — run the wake transition
        // instead of the scale-down. Reflect the fresh wake generation on the
        // in-memory object so downstream writes preserve it.
        this.reflectOutcome(host, target => {
          target.annotations = fresh.annotations
          target.status = fresh.status
        })
        return {
          effective: { stateless: true, state: 'active' },
          lifecycle: { state: 'active', wakeHandledGeneration: freshWakeRequested },
          condition: assessment.condition,
          pullPolicyCondition: assessment.pullPolicyCondition,
        }
      }
      // Cached active/draining with a wake pending in fresh: replicas already
      // derive to 1, which is what the pending wake wants — keep the
      // assessment (the wake fast-path/resync records the generation).
      return assessment
    }
    // FIX 1: no pending wake — re-derive the scale-relevant state from the
    // SAME fresh read. Only the suspended/not-suspended boundary changes
    // replicas, so only that disagreement rebuilds the assessment.
    const freshSuspended = freshState === 'suspended'
    if ((cachedState === 'suspended') === freshSuspended) {
      return assessment
    }
    log.info(
      `[HostReconciler] Stateless replicas guard for "${host.name}": cached lifecycle state "${cachedState}" disagrees with fresh "${freshState}" — deriving replicas from FRESH state`
    )
    // Reflect fresh onto the in-memory host (mirroring the wake fast-path's
    // epoch guard) so every downstream write derives from the current
    // server-side truth instead of the stale event payload.
    this.reflectOutcome(host, target => {
      target.annotations = fresh.annotations
      target.status = fresh.status
    })
    const freshReason = fresh.status?.lifecycle?.reason
    const preservedReason = isHeartbeatManagedLifecycleReason(freshReason) ? freshReason : undefined
    return {
      effective: { stateless: true, state: freshState },
      lifecycle: {
        state: freshState,
        wakeHandledGeneration: freshWakeHandled,
        ...(preservedReason !== undefined ? { reason: preservedReason } : {}),
      },
      condition: assessment.condition,
      pullPolicyCondition: assessment.pullPolicyCondition,
    }
  }
}