Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
33.6 KB
Raw Blame
import type { Response as ExpressResponse } from 'express'
import { config } from '../config.js'
import type { ResolvedServerConnection, RpcAccessClaims } from '../types.js'
import { type HostWakeApiResponse, requestHostWakeFromControlApi } from './controlApiRestService.js'
import { forwardHostHealth } from './mcpHostRestService.js'

/** Strip control characters and newlines from user-derived hostRef before log interpolation. */
export function sanitizeHostRefForLog(hostRef: string): string {
  return hostRef.replace(/[\r\n\t\x00-\x1f\x7f]/g, '')
}

/** Keeps an error name or code to the characters such identifiers use. */
function logToken(value: string): string {
  return value.replace(/[^A-Za-z0-9_.:-]/g, '').slice(0, 64)
}

/**
 * Identifies a failed upstream call for a log line WITHOUT its message. An
 * UpstreamHostError message embeds up to 300 characters of the Host response
 * body, and a Host error body can carry user content, so no Host-facing log
 * line may print `error.message`. What tells failures apart survives: the
 * error name, the upstream HTTP status and the network error code, e.g.
 * `UpstreamHostError status=500` or `TypeError code=ECONNREFUSED`.
 */
export function describeErrorForLog(error: unknown): string {
  if (!(error instanceof Error)) return `non-error type=${typeof error}`
  const { status, code, cause } = error as Error & {
    status?: unknown
    code?: unknown
    cause?: unknown
  }
  const parts = [logToken(error.name)]
  if (typeof status === 'number' && Number.isInteger(status)) parts.push(`status=${status}`)
  const causeCode =
    cause !== null && typeof cause === 'object' ? (cause as { code?: unknown }).code : undefined
  const errorCode = typeof code === 'string' ? code : causeCode
  if (typeof errorCode === 'string') parts.push(`code=${logToken(errorCode)}`)
  return parts.join(' ')
}

/**
 * Stateless wake-and-hold (Stage 5, Issue #791 §11).
 *
 * When a request targets a stateless Host whose pod is suspended (network
 * failure) or draining (upstream 503 {code:'host_draining'}), rpc-proxy calls
 * the control-api wake endpoint and — for a 202 — parks the request while the
 * pod comes up, instead of surfacing the generic 502.
 *
 * Coordination is keyed by PRINCIPAL + Host (§11.1), not by Host alone. Only a
 * same-principal wake-capable authorization (unexpired, carrying
 * `host:wake:write` AND the target hostRef) may occupy or replace the wake
 * slot. A non-wake token can ride an existing wake but can never trigger one
 * nor displace the wake authorization, and different principals never share a
 * group — so a non-wake or foreign token can no longer displace the only
 * wake-capable authorization and drive an unauthorized 403 that collapses
 * every waiter on the Host.
 *
 * Fleet note: rpc-proxy runs with replicas:2, so the in-memory dedup map here
 * is per-instance best effort. The fleet-level authority for wake dedup and
 * rate limiting is control-api (Postgres-backed generation + token bucket).
 */

const TOKEN_EXP_SAFETY_MARGIN_MS = 2_000
const ENTRY_TTL_MARGIN_MS = 5_000
/**
 * Capacity of the coordination map, counted in COORDINATIONS, not hosts. The
 * map is keyed by `wakeCoordinationKey(claims, hostRef)` = principal × Host, so
 * N distinct principals waking ONE Host consume N slots: a fleet of H hosts
 * with P concurrently-holding principals per host consumes up to H×P slots
 * (review F2). Named for what it bounds — a "hosts" name would understate the
 * real limit by a factor of P. Oldest-first eviction settles the victim
 * retryable (503 host_waking), so overflow degrades politely.
 */
const MAX_TRACKED_WAKE_COORDINATIONS = 1_000
const WAKE_SCOPE = 'host:wake:write' as const
const ADMISSION_RETRY_DELAYS_MS = [250, 500, 1_000]
/**
 * An admission retry with less budget than this cannot outlast the upstream's
 * own admission latency: it would time out AFTER the upstream applied the
 * request, turning an applied-once operation into a 504. Below it the request
 * answers a retryable 503 host_waking instead. Bounded by `upstreamTimeoutMs`
 * so a deliberately short upstream timeout still gets a retry.
 */
const MIN_ADMISSION_RETRY_TIMEOUT_MS = 1_000

export type WakeHoldOutcome =
  /** Host is (or just became) reachable — re-issue the upstream request now. */
  | { kind: 'proceed'; lastKnownState: string }
  /** Host is not stateless / wake plane failed / not wake-capable — keep today's error behavior. */
  | { kind: 'legacy'; reason: string }
  /** Host CR is gone — 404 to the caller. */
  | { kind: 'not-found' }
  /** Still waking (or hold aborted) — structured retryable to the caller. */
  | { kind: 'waking'; retryAfterMs: number; reason: string; lastKnownState: string }

type Waiter = {
  id: number
  resolve: (outcome: WakeHoldOutcome) => void
  timer: NodeJS.Timeout
  /** Per-waiter one-shot latch: the deadline timer and a group settle can race. */
  settled: boolean
}

/**
 * The single same-principal wake-capable authorization used to drive the wake
 * plane. Only replaced by a later-expiry same-principal wake-capable token;
 * never by a non-wake token.
 */
type WakeAuthorization = {
  token: string
  tokenExpMs: number
}

type HostWakeEntry = {
  /** Principal+Host coordination key (never logged — see §11.1). */
  key: string
  /** Sanitized logical Host reference — the only identity allowed in logs. */
  hostRef: string
  createdAt: number
  /** Last state reported by the wake endpoint — surfaced as diagnostics. */
  lastKnownState: string
  waiters: Map<number, Waiter>
  /** Same-principal wake-capable authorization used for the wake plane. */
  wakeAuthorization: WakeAuthorization
  /** Current resolved Host connection used by the readiness probe. */
  connection: ResolvedServerConnection
  pollTimer: NodeJS.Timeout | null
  retriggerTimer: NodeJS.Timeout | null
  ttlTimer: NodeJS.Timeout | null
  settled: boolean
}

export type WakeCoordinatorDeps = {
  requestWake: (hostRef: string, rpcAccessToken: string) => Promise<HostWakeApiResponse>
  probeReady: (host: ResolvedServerConnection) => Promise<boolean>
  maxHoldMs: number
  pollMs: number
  retriggerMs: number
  /**
   * Capacity in (principal, host) coordination entries — NOT in distinct hosts.
   * See MAX_TRACKED_WAKE_COORDINATIONS.
   */
  maxTrackedCoordinations: number
  now?: () => number
}

export type WakeHoldParams = {
  hostRef: string
  host: ResolvedServerConnection
  /** Parsed claims of the caller — used for principal keying + wake capability. */
  claims: RpcAccessClaims
  /** Raw bearer forwarded to the wake plane when this caller is wake-capable. */
  rpcAccessToken: string
  /** Optional request deadline, shared with upstream admission retries. */
  deadlineMs?: number
}

/**
 * Stable in-memory coordination key from existing claims + hostRef (§11.1):
 * `{ typ, sub, accessScope, teamId ?? '', hostRef }`. Excludes jti/exp/role/
 * scopes (they rotate per token and would fragment one principal), and never
 * contains bearer material.
 */
export function wakeCoordinationKey(claims: RpcAccessClaims, hostRef: string): string {
  // Delimiter-safe encoding (adversarial review F4): sub/teamId are
  // server-assigned UUIDs today, but principal isolation must not silently
  // depend on ID format — JSON framing makes boundary-shifting collisions
  // impossible regardless of future identifier shapes.
  return JSON.stringify([claims.typ, claims.sub, claims.accessScope, claims.teamId ?? '', hostRef])
}

/**
 * "Wake-capable" (§11.2) = unexpired claims that contain `host:wake:write` AND
 * the target hostRef in `hostRefs`. Expiry uses the same safety margin as the
 * hold budget so a token that cannot survive the wake call is not treated as
 * usable for it.
 */
function isWakeCapable(claims: RpcAccessClaims, hostRef: string, now: number): boolean {
  const tokenExpMs = claims.exp * 1000
  if (tokenExpMs - TOKEN_EXP_SAFETY_MARGIN_MS <= now) return false
  if (!claims.scopes.includes(WAKE_SCOPE)) return false
  if (!claims.hostRefs.includes(hostRef)) return false
  return true
}

export class WakeAndHoldCoordinator {
  private readonly entries = new Map<string, HostWakeEntry>()
  private waiterSeq = 0
  /** Set once by drain(); permanently fences new holds during shutdown. */
  private draining = false

  constructor(private readonly deps: WakeCoordinatorDeps) {}

  /**
   * Live (principal, host) coordination entries — NOT a count of distinct
   * hosts. Exposed for tests and observability.
   */
  trackedCoordinationCount(): number {
    return this.entries.size
  }

  async hold(params: WakeHoldParams): Promise<WakeHoldOutcome> {
    if (this.draining) {
      // Shutdown fence (review F1): drain() already settled the map; a hold
      // arriving from an in-flight request after SIGTERM must not park past
      // server.close(). The client retries against a live replica.
      return {
        kind: 'waking',
        retryAfterMs: this.deps.pollMs,
        reason: 'shutting-down',
        lastKnownState: 'unknown',
      }
    }
    const now = this.now()
    const tokenExpMs = params.claims.exp * 1000
    const tokenBudgetMs = tokenExpMs - TOKEN_EXP_SAFETY_MARGIN_MS - now
    if (tokenBudgetMs <= 0) {
      // Never park a request that could only be forwarded with an expired
      // token. The client re-issues with a fresh token and retries.
      return {
        kind: 'waking',
        retryAfterMs: this.deps.pollMs,
        reason: 'token-expiring',
        lastKnownState: 'unknown',
      }
    }
    const holdBudgetMs = Math.min(
      this.deps.maxHoldMs,
      tokenBudgetMs,
      params.deadlineMs === undefined ? Infinity : Math.max(0, params.deadlineMs - now)
    )
    const deadlineReason =
      tokenBudgetMs <= Math.min(this.deps.maxHoldMs, (params.deadlineMs ?? Infinity) - now)
        ? 'token-expiring'
        : 'max-hold-exceeded'
    if (holdBudgetMs <= 0) {
      return {
        kind: 'waking',
        retryAfterMs: this.deps.pollMs,
        reason: deadlineReason,
        lastKnownState: 'unknown',
      }
    }

    const key = wakeCoordinationKey(params.claims, params.hostRef)
    const wakeCapable = isWakeCapable(params.claims, params.hostRef, now)

    let entry = this.entries.get(key)
    if (!entry) {
      if (!wakeCapable) {
        // No wake-capable authorization exists for this principal+Host, so
        // calling the wake plane would only earn a deterministic 403. Return
        // today's pre-wake availability result instead (§11.2, §13.4). No
        // fallback grant is minted.
        return { kind: 'legacy', reason: 'no-wake-authorization' }
      }
      entry = this.createEntry(key, params, tokenExpMs)
    } else if (wakeCapable) {
      if (tokenExpMs > entry.wakeAuthorization.tokenExpMs) {
        // Same principal, wake-capable, later valid expiry → replace the wake
        // authorization and the connection the probe uses.
        entry.wakeAuthorization = { token: params.rpcAccessToken, tokenExpMs }
        entry.connection = params.host
      }
      // Same principal, wake-capable, earlier/equal expiry → retain current.
    } else if (entry.wakeAuthorization.tokenExpMs - TOKEN_EXP_SAFETY_MARGIN_MS <= now) {
      // Not wake-capable and the group's wake authorization has itself expired:
      // there is no valid authorization to ride, so no wake is possible.
      return { kind: 'legacy', reason: 'wake-authorization-expired' }
    }
    // Otherwise (not wake-capable, valid wake authorization present) the caller
    // rides the existing wake as a waiter only — never replacing the auth.

    const activeEntry = entry
    return await new Promise<WakeHoldOutcome>(resolve => {
      const id = ++this.waiterSeq
      const waiter: Waiter = {
        id,
        resolve,
        timer: null as unknown as NodeJS.Timeout,
        settled: false,
      }
      waiter.timer = setTimeout(() => {
        if (waiter.settled) return
        waiter.settled = true
        activeEntry.waiters.delete(id)
        resolve({
          kind: 'waking',
          retryAfterMs: this.deps.pollMs,
          reason: deadlineReason,
          lastKnownState: activeEntry.lastKnownState,
        })
        this.releaseIfEmpty(activeEntry)
      }, holdBudgetMs)
      activeEntry.waiters.set(id, waiter)
    })
  }

  /**
   * Deterministically settles every parked waiter AND permanently fences new
   * holds (terminal `waking`/`shutting-down`) — used by the service's graceful
   * shutdown path so no held request hangs across `server.close()`.
   */
  drain(reason = 'shutdown-drain'): void {
    this.draining = true
    for (const entry of [...this.entries.values()]) {
      this.settle(entry, {
        kind: 'waking',
        retryAfterMs: this.deps.pollMs,
        reason,
        lastKnownState: entry.lastKnownState,
      })
    }
  }

  private now(): number {
    return this.deps.now ? this.deps.now() : Date.now()
  }

  private createEntry(key: string, params: WakeHoldParams, tokenExpMs: number): HostWakeEntry {
    this.enforceCapacity()
    const entry: HostWakeEntry = {
      key,
      hostRef: params.hostRef,
      createdAt: this.now(),
      lastKnownState: 'unknown',
      waiters: new Map(),
      wakeAuthorization: { token: params.rpcAccessToken, tokenExpMs },
      connection: params.host,
      pollTimer: null,
      retriggerTimer: null,
      ttlTimer: null,
      settled: false,
    }
    this.entries.set(key, entry)
    // Safety net: even if a waiter bookkeeping bug ever left the entry behind,
    // it self-destructs loudly instead of leaking timers forever.
    entry.ttlTimer = setTimeout(() => {
      console.warn(
        `[RPC_PROXY] wake-hold entry TTL exceeded (principal-scoped entry) host=${entry.hostRef} waiters=${entry.waiters.size} — evicting`
      )
      this.settle(entry, {
        kind: 'waking',
        retryAfterMs: this.deps.pollMs,
        reason: 'entry-ttl-exceeded',
        lastKnownState: entry.lastKnownState,
      })
    }, this.deps.maxHoldMs + ENTRY_TTL_MARGIN_MS)
    void this.issueWake(entry, { initial: true })
    return entry
  }

  private enforceCapacity(): void {
    if (this.entries.size < this.deps.maxTrackedCoordinations) return
    let oldest: HostWakeEntry | null = null
    for (const candidate of this.entries.values()) {
      if (!oldest || candidate.createdAt < oldest.createdAt) oldest = candidate
    }
    if (!oldest) return
    console.warn(
      `[RPC_PROXY] wake-hold capacity exceeded (max=${this.deps.maxTrackedCoordinations} principal-scoped entries) — evicting oldest entry host=${oldest.hostRef} waiters=${oldest.waiters.size}`
    )
    this.settle(oldest, {
      kind: 'waking',
      retryAfterMs: this.deps.pollMs,
      reason: 'capacity-evicted',
      lastKnownState: oldest.lastKnownState,
    })
  }

  private async issueWake(entry: HostWakeEntry, options: { initial: boolean }): Promise<void> {
    let response: HostWakeApiResponse
    try {
      response = await this.deps.requestWake(entry.hostRef, entry.wakeAuthorization.token)
    } catch (error) {
      console.warn(
        `[RPC_PROXY] wake call failed host=${entry.hostRef} initial=${options.initial} error=${
          error instanceof Error ? error.message : String(error)
        }`
      )
      if (options.initial) {
        // No wake was recorded anywhere — surface today's error path rather
        // than holding a request nothing is going to release.
        this.settle(entry, { kind: 'legacy', reason: 'wake-endpoint-unreachable' })
      }
      // Retrigger failures keep holding: the next retrigger tick tries again,
      // bounded by the per-waiter deadlines and the entry TTL.
      return
    }
    await this.processWakeResponse(entry, response, options)
  }

  private async processWakeResponse(
    entry: HostWakeEntry,
    response: HostWakeApiResponse,
    options: { initial: boolean }
  ): Promise<void> {
    if (entry.settled) {
      console.debug(
        `[RPC_PROXY] late wake-hold artifact ignored (already resolved) host=${entry.hostRef} artifact=wake-response kind=${response.kind}`
      )
      return
    }
    switch (response.kind) {
      case 'active':
        entry.lastKnownState = 'active'
        this.settle(entry, { kind: 'proceed', lastKnownState: 'active' })
        return
      case 'wake-requested':
        entry.lastKnownState = 'wake-requested'
        if (options.initial) this.startHoldLoops(entry)
        return
      case 'not-stateless':
        // Kill-switch (possibly flipped mid-hold): behave exactly like today.
        this.settle(entry, { kind: 'legacy', reason: 'not-stateless' })
        return
      case 'unknown':
        this.settle(entry, { kind: 'not-found' })
        return
      case 'rate-limited':
        if (options.initial) {
          this.settle(entry, {
            kind: 'waking',
            retryAfterMs: Math.ceil(response.retryAfterSeconds * 1000),
            reason: 'wake-rate-limited',
            lastKnownState: entry.lastKnownState,
          })
        }
        // A rate-limited retrigger usually means the wake intent is already
        // recorded fleet-wide, but the bucket may also have been consumed by
        // concurrent callers (prewarm bursts, other holds). Either way the
        // next retrigger tick retries, so keep holding for readiness.
        return
      case 'auth':
        // The wake plane rejected the wake-capable authorization. Distinguish
        // initial from retrigger deliberately (§11.2): an INITIAL rejection is
        // terminal for this principal group, but a mid-hold RETRIGGER rejection
        // (token rotated/revoked after the wake was already recorded) must NOT
        // collapse the still-valid waiters — keep holding, bounded by the
        // per-waiter deadlines and the entry TTL. Because coordination is
        // principal-scoped, this only ever affects this principal's group.
        console.warn(
          `[RPC_PROXY] wake call rejected host=${entry.hostRef} status=${response.status} initial=${options.initial}`
        )
        if (options.initial) {
          this.settle(entry, { kind: 'legacy', reason: `wake-auth-${response.status}` })
        }
        return
    }
  }

  private startHoldLoops(entry: HostWakeEntry): void {
    const schedulePoll = () => {
      entry.pollTimer = setTimeout(async () => {
        let ready = false
        try {
          ready = await this.deps.probeReady(entry.connection)
        } catch (error) {
          // A failed probe is "not ready yet" — the hold stays bounded by the
          // per-waiter deadlines, so a broken probe source cannot hang callers.
          console.warn(
            `[RPC_PROXY] wake-hold readiness probe failed host=${entry.hostRef} error=${
              error instanceof Error ? error.message : String(error)
            }`
          )
        }
        if (entry.settled) {
          console.debug(
            `[RPC_PROXY] late wake-hold artifact ignored (already resolved) host=${entry.hostRef} artifact=readiness-probe`
          )
          return
        }
        if (ready) {
          this.settle(entry, { kind: 'proceed', lastKnownState: entry.lastKnownState })
          return
        }
        schedulePoll()
      }, this.deps.pollMs)
    }
    const scheduleRetrigger = () => {
      entry.retriggerTimer = setTimeout(async () => {
        // Hygiene: a retrigger whose wake authorization has already expired is
        // doomed — control-api would reject it — so skip the call rather than
        // spend it. We deliberately KEEP HOLDING: the readiness poll runs on its
        // own timer and the entry still has TTL left, so a host that wakes via
        // any other path still resolves this waiter. Rescheduling matters
        // because a later waiter can adopt this entry and upgrade
        // `wakeAuthorization` to a longer-lived token, after which retriggers
        // resume. Same predicate as the adopt path above.
        if (entry.wakeAuthorization.tokenExpMs - TOKEN_EXP_SAFETY_MARGIN_MS <= this.now()) {
          console.debug(
            `[RPC_PROXY] wake retrigger skipped (wake authorization expired) host=${entry.hostRef}`
          )
          if (!entry.settled) scheduleRetrigger()
          return
        }
        await this.issueWake(entry, { initial: false })
        if (entry.settled) {
          console.debug(
            `[RPC_PROXY] late wake-hold artifact ignored (already resolved) host=${entry.hostRef} artifact=retrigger`
          )
          return
        }
        scheduleRetrigger()
      }, this.deps.retriggerMs)
    }
    schedulePoll()
    scheduleRetrigger()
  }

  private clearEntryTimers(entry: HostWakeEntry): void {
    if (entry.pollTimer) clearTimeout(entry.pollTimer)
    if (entry.retriggerTimer) clearTimeout(entry.retriggerTimer)
    if (entry.ttlTimer) clearTimeout(entry.ttlTimer)
    entry.pollTimer = null
    entry.retriggerTimer = null
    entry.ttlTimer = null
  }

  private settle(entry: HostWakeEntry, outcome: WakeHoldOutcome): void {
    if (entry.settled) {
      console.debug(
        `[RPC_PROXY] late wake-hold artifact ignored (already resolved) host=${entry.hostRef} artifact=settle outcome=${outcome.kind}`
      )
      return
    }
    entry.settled = true
    this.clearEntryTimers(entry)
    this.entries.delete(entry.key)
    for (const waiter of entry.waiters.values()) {
      if (waiter.settled) continue
      waiter.settled = true
      clearTimeout(waiter.timer)
      waiter.resolve(outcome)
    }
    entry.waiters.clear()
  }

  private releaseIfEmpty(entry: HostWakeEntry): void {
    if (entry.settled || entry.waiters.size > 0) return
    entry.settled = true
    this.clearEntryTimers(entry)
    this.entries.delete(entry.key)
  }
}

/**
 * True for an upstream 503 carrying mcp-host's DRAINING fence
 * ({code:'host_draining'}). Matched structurally (name + status) instead of
 * `instanceof` so mocked UpstreamHostError classes in sibling tests behave
 * identically to the real one.
 */
export function isHostDrainingError(error: unknown): boolean {
  if (!(error instanceof Error) || error.name !== 'UpstreamHostError') return false
  const upstream = error as Error & { status?: unknown; bodySnippet?: unknown }
  return upstream.status === 503 && String(upstream.bodySnippet || '').includes('host_draining')
}

/**
 * True for an upstream 413 from mcp-host: the Host refused the body as too
 * large, which is the client's error to fix, not a sign the Host is
 * unavailable. Matched structurally (name + status) like isHostDrainingError.
 */
export function isHostPayloadTooLargeError(error: unknown): boolean {
  if (!(error instanceof Error) || error.name !== 'UpstreamHostError') return false
  return (error as Error & { status?: unknown }).status === 413
}

/**
 * undici's own header/body timers (300 s defaults). They fire on a connection
 * the host accepted, so they prove a slow host, not a down one: a timeout, and
 * never a reason to wake and re-issue the request.
 */
const UNDICI_TIMEOUT_CODES: ReadonlySet<unknown> = new Set([
  'UND_ERR_HEADERS_TIMEOUT',
  'UND_ERR_BODY_TIMEOUT',
])

/** Fetch uses AbortError for controller aborts and TimeoutError for
 * AbortSignal.timeout(); undici's own header/body timers reject with their
 * code on `cause`. All represent the same sanitized 504 boundary. An
 * UpstreamBodyReadError is a timeout when the body read it wraps was one. */
export function isUpstreamTimeoutError(error: unknown): boolean {
  if (!(error instanceof Error)) return false
  if (error.name === 'AbortError' || error.name === 'TimeoutError') return true
  if (error.name === 'UpstreamBodyReadError') return isUpstreamTimeoutError(error.cause)
  return UNDICI_TIMEOUT_CODES.has((error as Error & { cause?: { code?: unknown } }).cause?.code)
}

/**
 * True for a network-level fetch failure against the upstream host (no HTTP
 * response at all — suspended pod, no endpoints). Excludes timeouts (today's
 * 504 path), UpstreamHostError (the host answered) and UpstreamBodyReadError
 * (the headers arrived, so the host received the request).
 */
export function isHostDownNetworkError(error: unknown): boolean {
  if (!(error instanceof Error)) return false
  if (
    isUpstreamTimeoutError(error) ||
    error.name === 'UpstreamHostError' ||
    error.name === 'UpstreamBodyReadError'
  ) {
    return false
  }
  const cause = (error as Error & { cause?: { code?: unknown } }).cause
  const details = `${error.message} ${String(cause?.code || '')}`.toLowerCase()
  return (
    details.includes('fetch failed') ||
    /econnrefused|econnreset|enotfound|ehostunreach|epipe|socket hang up|und_err/.test(details)
  )
}

export function isWakeEligibleHostError(error: unknown): boolean {
  return isHostDownNetworkError(error) || isHostDrainingError(error)
}

async function sleep(ms: number): Promise<void> {
  await new Promise(resolve => setTimeout(resolve, ms))
}

function respondHostWaking(
  res: ExpressResponse,
  details: { hostRef: string; retryAfterMs: number; lastKnownState: string }
): void {
  if (res.headersSent) return
  res.setHeader('Retry-After', String(Math.max(1, Math.ceil(details.retryAfterMs / 1000))))
  res.status(503).json({
    code: 'host_waking',
    hostRef: details.hostRef,
    retryAfterMs: details.retryAfterMs,
    message: 'Host is waking up',
    lastKnownState: details.lastKnownState,
  })
}

/**
 * Readiness probe (§11.3): mcp-host's UNAUTHENTICATED `/v1/runtime/health`.
 * It comes up before MCP background init and is the same signal the Pod
 * readiness probe uses. `forwardHostHealth` throws on non-2xx / network
 * failure, so a resolved value means the pod is answering runtime traffic —
 * the retried operation still enforces its own authorization/business
 * contract. Bounded by `config.upstreamTimeoutMs` (AbortController inside the
 * REST helper); on shutdown `drain()` stops the poll loop and any late result
 * is ignored via the entry `settled` latch.
 *
 * Rollout-skew note (review F5): an old mcp-host image without
 * `/v1/runtime/health` makes this probe return false until the hold deadline;
 * holds still settle because the retrigger wake's `active` response — derived
 * by control-api from replica/endpoint state, not from this endpoint — settles
 * `proceed`. Readiness latency degrades from ~pollMs to ~retriggerMs in that
 * skew window.
 */
export async function probeHostReadyViaHealth(host: ResolvedServerConnection): Promise<boolean> {
  try {
    await forwardHostHealth(host)
    return true
  } catch {
    // Unreachable/erroring health endpoint == not ready yet. Bounded by the
    // hold deadline — never a silent success.
    return false
  }
}

/** Singleton wired to production dependencies. Tests construct their own. */
export const hostWakeCoordinator = new WakeAndHoldCoordinator({
  requestWake: requestHostWakeFromControlApi,
  probeReady: probeHostReadyViaHealth,
  maxHoldMs: config.wakeMaxHoldMs,
  pollMs: config.wakePollMs,
  retriggerMs: config.wakeRetriggerMs,
  maxTrackedCoordinations: MAX_TRACKED_WAKE_COORDINATIONS,
})

export type RespondWithWakeAndHoldOptions = {
  res: ExpressResponse
  hostRef: string
  host: ResolvedServerConnection
  /** Parsed claims of the caller — principal keying + wake capability (§11.1). */
  claims: RpcAccessClaims
  /** Raw bearer forwarded to the wake plane when the caller is wake-capable. */
  rpcAccessToken: string
  /**
   * Re-issues the original upstream request with a deadline-bounded timeout.
   * The route's FIRST attempt runs without a client-side timeout for mutating
   * calls; `timeoutMs` only ever applies to these hold retries. `deadlineMs`
   * is the absolute request deadline, for phases after the headers (a body
   * transfer) that `timeoutMs` alone does not bound.
   */
  attemptUpstream: (timeoutMs: number, deadlineMs: number) => Promise<void>
  /**
   * Writes the route's pre-wake error response (502/504, plus any route-owned
   * mapping such as the messages route's Host 413). Also receives the error of
   * a post-wake retry the Host answered with a non-availability failure.
   */
  respondLegacy: (error: unknown) => void
  coordinator?: WakeAndHoldCoordinator
  /** Absolute deadline captured before a route's initial availability probe. */
  deadlineMs?: number
  /**
   * Keep re-issuing the request (with backoff) until it is accepted or the
   * deadline expires. Only safe when a re-issued request cannot duplicate a
   * side effect: the idempotent GET reads (sessions, transcript, context
   * breakdown, models, task result, artifacts list and download, which commits
   * only at `res.send`), and `/messages`, whose per-request `messageId`
   * mcp-host's admission sink dedupes. Approve/deny/model/cancel have no
   * idempotency key, so they re-issue at most ONCE after the hold: each extra
   * POST risks a duplicate side effect. Defaults to false.
   */
  retryUntilDeadline?: boolean
}

/**
 * Drives one held request through the wake flow and always writes a response:
 * - proceed  → retry availability failures until upstream accepts or the
 *              single request deadline expires.
 * - legacy   → today's 502 path (409 not-stateless, wake plane failure, or a
 *              non-wake-capable caller with no wake authorization to ride).
 * - not-found→ 404.
 * - waking   → structured 503 {code:'host_waking'} + Retry-After.
 */
export async function respondWithWakeAndHold(
  options: RespondWithWakeAndHoldOptions
): Promise<void> {
  const coordinator = options.coordinator ?? hostWakeCoordinator
  const safeHostRef = sanitizeHostRefForLog(options.hostRef)
  const deadlineMs = options.deadlineMs ?? Date.now() + config.wakeMaxHoldMs
  if (options.res.headersSent) {
    // A response is already committed for this request: parking it could only
    // ever produce a duplicate upstream delivery. Refuse loudly.
    console.warn(`[RPC_PROXY] wake-hold refused: response already committed host=${safeHostRef}`)
    return
  }
  const outcome = await coordinator.hold({
    hostRef: options.hostRef,
    host: options.host,
    claims: options.claims,
    rpcAccessToken: options.rpcAccessToken,
    deadlineMs,
  })

  if (options.res.headersSent) {
    console.debug(
      `[RPC_PROXY] late wake-hold artifact ignored (already resolved) host=${safeHostRef} artifact=hold-outcome`
    )
    return
  }

  switch (outcome.kind) {
    case 'proceed': {
      let availabilityFailures = 0
      let attempts = 0
      const maxAttempts = options.retryUntilDeadline ? Infinity : 1
      const minAttemptTimeoutMs = Math.min(MIN_ADMISSION_RETRY_TIMEOUT_MS, config.upstreamTimeoutMs)
      while (Date.now() < deadlineMs) {
        if (options.res.headersSent) {
          // Resolution latch: the response was committed while this retry
          // schedule was pending. Re-forwarding now would duplicate a message
          // the upstream already accepted.
          console.debug(
            `[RPC_PROXY] late wake-hold artifact ignored (already resolved) host=${safeHostRef} artifact=proceed-retry`
          )
          return
        }
        try {
          const remainingMs = deadlineMs - Date.now()
          if (remainingMs < minAttemptTimeoutMs) break
          attempts++
          await options.attemptUpstream(Math.min(config.upstreamTimeoutMs, remainingMs), deadlineMs)
          return
        } catch (error) {
          if (options.res.headersSent) {
            // The attempt failed AFTER committing the response (e.g. the
            // success write threw): the request is resolved. A retry here is
            // the duplicate-delivery bug — never re-forward, fail loudly.
            console.warn(
              `[RPC_PROXY] wake-hold post-response failure suppressed (already resolved) host=${safeHostRef} error=${describeErrorForLog(
                error
              )}`
            )
            return
          }
          if (isWakeEligibleHostError(error)) {
            // Single-retry routes stop here: the host is still unreachable
            // after the wake, so the caller gets the retryable 503 below.
            if (attempts >= maxAttempts) break
            const remainingMs = deadlineMs - Date.now()
            const delayMs =
              ADMISSION_RETRY_DELAYS_MS[
                Math.min(availabilityFailures++, ADMISSION_RETRY_DELAYS_MS.length - 1)
              ]
            if (remainingMs > 0) await sleep(Math.min(delayMs, remainingMs))
            continue
          }
          // The host answered with a non-availability failure — exactly
          // today's behavior for an up-but-erroring host.
          console.warn(
            `[RPC_PROXY] wake-hold upstream retry failed host=${safeHostRef} error=${describeErrorForLog(
              error
            )}`
          )
          options.respondLegacy(error)
          return
        }
      }
      if (options.res.headersSent) {
        console.debug(
          `[RPC_PROXY] late wake-hold artifact ignored (already resolved) host=${safeHostRef} artifact=host-waking-response`
        )
        return
      }
      console.warn(
        `[RPC_PROXY] wake-hold admission ended without an accepted attempt host=${safeHostRef} attempts=${attempts} lastKnownState=${outcome.lastKnownState}`
      )
      respondHostWaking(options.res, {
        hostRef: options.hostRef,
        retryAfterMs: config.wakePollMs,
        lastKnownState: outcome.lastKnownState,
      })
      return
    }
    case 'legacy':
      console.warn(
        `[RPC_PROXY] wake-hold falling back to legacy error path host=${safeHostRef} reason=${outcome.reason}`
      )
      options.respondLegacy(new Error(`Upstream host unavailable (${outcome.reason})`))
      return
    case 'not-found':
      res404(options.res)
      return
    case 'waking':
      console.info(
        `[RPC_PROXY] wake-hold responding host_waking host=${safeHostRef} reason=${outcome.reason} lastKnownState=${outcome.lastKnownState}`
      )
      respondHostWaking(options.res, {
        hostRef: options.hostRef,
        retryAfterMs: outcome.retryAfterMs,
        lastKnownState: outcome.lastKnownState,
      })
      return
  }
}

function res404(res: ExpressResponse): void {
  if (res.headersSent) return
  res.status(404).json({ error: 'Host not found or not accessible' })
}