Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
19.1 KB
Raw Blame
import type { Request, RequestHandler, Response } from 'express'
import { type Logger, rootLogger } from '../observability/logger.js'
import type { McpHostAccessClaims } from '../utils/auth/mcpHostJwtToken.js'

export type AuthorizeBodyAdmissionPolicy = Readonly<{
  maxInFlight: number
  maxQueued: number
  maxPerPrincipal: number
  queueWaitMs: number
  readDeadlineMs: number
  workDeadlineMs: number
  closeGraceMs: number
  /** `Retry-After` on a capacity refusal; the Host retries once after it. */
  retryAfterSeconds: number
  /** Most bytes a capacity refusal reads and discards before it answers. */
  maxDiscardBodyBytes: number
}>

// The route supplies these claims only after requireMcpHostJwt succeeds.
// Neither body fields nor unverified headers supply admission identity.
export type AuthorizeBodyAdmissionPrincipal = Readonly<
  Pick<McpHostAccessClaims, 'sub' | 'hostRefs'>
>

type AdmissionWaiter = { grant: () => void }
type AdmissionRefusal = 'principal_share' | 'queue_full' | 'queue_wait'

export class AuthorizeWorkInterrupted extends Error {
  constructor(readonly code: 'authorize_timeout' | 'authorize_aborted') {
    super(code)
    this.name = 'AuthorizeWorkInterrupted'
  }
}

type ParserError = {
  type?: unknown
  status?: unknown
  body?: unknown
}

const MAX_TIMER_DELAY_MS = 2_147_483_647
const LOG_MODULE = 'llm-provider-attempt-body-admission'
const log = rootLogger.child({ module: LOG_MODULE })

/**
 * Owns one full permitted authorize body from before parsing until both the
 * parser callback and the signal-aware authorize work have unwound.
 *
 * The route decides which requests are charged (`run`) and which are not
 * (`runUncharged`). Every charged request costs the same full retained-body
 * unit whatever its Content-Length, so chunked bodies stay accepted and
 * framing stays with Node/body-parser. Both paths keep the read deadline, the
 * work deadline and disconnect cancellation.
 */
export class AuthorizeBodyAdmission {
  private inFlight = 0
  private readonly waiters: AdmissionWaiter[] = []
  private readonly principalUse = new Map<string, number>()
  private draining = false
  private readonly policy: AuthorizeBodyAdmissionPolicy

  constructor(policy: AuthorizeBodyAdmissionPolicy) {
    if (!Number.isSafeInteger(policy.maxInFlight) || policy.maxInFlight <= 0) {
      throw new TypeError('maxInFlight must be a positive safe integer')
    }
    if (!Number.isSafeInteger(policy.maxQueued) || policy.maxQueued < 0) {
      throw new TypeError('maxQueued must be a non-negative safe integer')
    }
    if (!Number.isSafeInteger(policy.maxPerPrincipal) || policy.maxPerPrincipal <= 0) {
      throw new TypeError('maxPerPrincipal must be a positive safe integer')
    }
    for (const value of [
      policy.queueWaitMs,
      policy.readDeadlineMs,
      policy.workDeadlineMs,
      policy.closeGraceMs,
    ]) {
      if (!Number.isSafeInteger(value) || value <= 0 || value > MAX_TIMER_DELAY_MS) {
        throw new TypeError('authorize timers must be positive signed-32-bit integers')
      }
    }
    // The Host parses 1-3600 s (mcp-host `retryAfter.ts`) but only retries
    // when the wait is at most RATE_LIMIT_RETRY_MAX_WAIT_MS (30 s,
    // `rateLimitRetry.ts`). A longer value would silently turn the one
    // same-provider retry off, so it is refused here.
    if (
      !Number.isSafeInteger(policy.retryAfterSeconds) ||
      policy.retryAfterSeconds <= 0 ||
      policy.retryAfterSeconds > 30
    ) {
      throw new TypeError('retryAfterSeconds must be an integer from 1 to 30')
    }
    if (!Number.isSafeInteger(policy.maxDiscardBodyBytes) || policy.maxDiscardBodyBytes <= 0) {
      throw new TypeError('maxDiscardBodyBytes must be a positive safe integer')
    }
    this.policy = Object.freeze({
      maxInFlight: policy.maxInFlight,
      maxQueued: policy.maxQueued,
      maxPerPrincipal: policy.maxPerPrincipal,
      queueWaitMs: policy.queueWaitMs,
      readDeadlineMs: policy.readDeadlineMs,
      workDeadlineMs: policy.workDeadlineMs,
      closeGraceMs: policy.closeGraceMs,
      retryAfterSeconds: policy.retryAfterSeconds,
      maxDiscardBodyBytes: policy.maxDiscardBodyBytes,
    })
  }

  tryAcquire(principal: AuthorizeBodyAdmissionPrincipal): (() => void) | null {
    const key = principalKey(principal)
    // Reserve released units for existing waiters synchronously. An arrival
    // cannot jump the FIFO while a granted waiter is awaiting its continuation.
    if (
      this.waiters.length > 0 ||
      this.inFlight >= this.policy.maxInFlight ||
      (this.principalUse.get(key) ?? 0) >= this.policy.maxPerPrincipal
    )
      return null
    this.retainPrincipal(key)
    this.inFlight += 1
    return this.releaseOwner(key)
  }

  private releaseOwner(key: string): () => void {
    let held = true
    return () => {
      if (!held) return
      held = false
      this.inFlight -= 1
      this.releasePrincipal(key)
      this.drainWaiters()
    }
  }

  snapshot(): Readonly<{ inFlight: number; queued: number; principals: number }> {
    return {
      inFlight: this.inFlight,
      queued: this.waiters.length,
      principals: this.principalUse.size,
    }
  }

  async run(
    req: Request,
    res: Response,
    parser: RequestHandler,
    work: (signal: AbortSignal) => Promise<void>,
    principal: AuthorizeBodyAdmissionPrincipal
  ): Promise<void> {
    const release = await this.waitForBodyOwner(req, res, principal)
    if (!release) return
    await this.runOwned(req, res, parser, work, release)
  }

  // For bodies the route has bounded below the retained-body unit. They take
  // no unit and never queue; the parser's own limit bounds what they retain.
  async runUncharged(
    req: Request,
    res: Response,
    parser: RequestHandler,
    work: (signal: AbortSignal) => Promise<void>
  ): Promise<void> {
    if (cannotStartBodyRead(req, res)) return
    await this.runOwned(req, res, parser, work, () => {})
  }

  private async runOwned(
    req: Request,
    res: Response,
    parser: RequestHandler,
    work: (signal: AbortSignal) => Promise<void>,
    release: () => void
  ): Promise<void> {
    try {
      // A waiter may disconnect after a grant but before this continuation.
      // It must unwind the granted unit without ever installing a body reader.
      if (cannotStartBodyRead(req, res)) {
        if (!req.destroyed) req.destroy()
        return
      }
      await parseToCompletion(req, res, parser, this.policy)
      if (req.aborted || res.destroyed || res.writableEnded) return

      const cancellation = new AbortController()
      const abortDisconnected = (): void => {
        if (!res.writableFinished) {
          cancellation.abort(new AuthorizeWorkInterrupted('authorize_aborted'))
        }
      }
      res.once('close', abortDisconnected)
      const workTimer = setTimeout(() => {
        cancellation.abort(new AuthorizeWorkInterrupted('authorize_timeout'))
      }, this.policy.workDeadlineMs)

      try {
        // Work must wire this signal to real dependency termination. Its
        // promise includes transaction unwind and producer disposal.
        await work(cancellation.signal)
        cancellation.signal.throwIfAborted()
      } catch (err) {
        if (cancellation.signal.aborted) throw cancellation.signal.reason
        throw err
      } finally {
        clearTimeout(workTimer)
        res.off('close', abortDisconnected)
      }
    } finally {
      req.body = undefined
      release()
    }
  }

  private async waitForBodyOwner(
    req: Request,
    res: Response,
    principal: AuthorizeBodyAdmissionPrincipal
  ): Promise<(() => void) | null> {
    if (cannotStartBodyRead(req, res)) return null
    const release = this.tryAcquire(principal)
    if (release) return release

    const key = principalKey(principal)
    if ((this.principalUse.get(key) ?? 0) >= this.policy.maxPerPrincipal) {
      this.refuseBeforeRead(req, res, 'principal_share')
      return null
    }
    if (this.waiters.length >= this.policy.maxQueued) {
      this.refuseBeforeRead(req, res, 'queue_full')
      return null
    }

    req.pause()
    this.retainPrincipal(key)
    const deadline = performance.now() + this.policy.queueWaitMs
    return new Promise((resolve, reject) => {
      let pending = true
      let waitTimer: ReturnType<typeof setTimeout> | undefined
      const finishWaiting = (): boolean => {
        if (!pending) return false
        pending = false
        if (waitTimer !== undefined) clearTimeout(waitTimer)
        req.off('aborted', abortWaiting)
        res.off('close', closeWaiting)
        const index = this.waiters.indexOf(waiter)
        if (index !== -1) this.waiters.splice(index, 1)
        return true
      }
      const cancelWaiting = (reason: 'aborted' | 'queue_wait'): void => {
        if (!finishWaiting()) return
        this.releasePrincipal(key)
        try {
          if (reason === 'queue_wait') this.refuseBeforeRead(req, res, reason)
          else if (!req.destroyed) req.destroy()
          resolve(null)
        } catch (err) {
          reject(err)
        } finally {
          this.drainWaiters()
        }
      }
      const abortWaiting = (): void => cancelWaiting('aborted')
      const closeWaiting = (): void => {
        if (!res.writableFinished) abortWaiting()
      }
      const waiter: AdmissionWaiter = {
        grant: () => {
          if (!pending) return
          if (cannotStartBodyRead(req, res)) {
            abortWaiting()
            return
          }
          // Timers can run late under load. A release cannot revive an
          // already expired waiter before its timer callback is delivered.
          if (performance.now() >= deadline) {
            cancelWaiting('queue_wait')
            return
          }
          if (!finishWaiting()) return
          this.inFlight += 1
          // The principal already owns this queued share: transfer it to
          // running rather than charging twice or briefly returning it.
          resolve(this.releaseOwner(key))
        },
      }
      this.waiters.push(waiter)
      req.once('aborted', abortWaiting)
      res.once('close', closeWaiting)
      waitTimer = setTimeout(() => cancelWaiting('queue_wait'), this.policy.queueWaitMs)
      if (cannotStartBodyRead(req, res)) abortWaiting()
    })
  }

  private retainPrincipal(key: string): void {
    this.principalUse.set(key, (this.principalUse.get(key) ?? 0) + 1)
  }

  private releasePrincipal(key: string): void {
    const remaining = (this.principalUse.get(key) ?? 0) - 1
    if (remaining === 0) this.principalUse.delete(key)
    else this.principalUse.set(key, remaining)
  }

  private drainWaiters(): void {
    if (this.draining) return
    this.draining = true
    try {
      while (this.inFlight < this.policy.maxInFlight && this.waiters.length > 0) {
        this.waiters[0].grant()
      }
    } finally {
      this.draining = false
    }
  }

  // The refusal reads and discards the body before it answers. Closing a
  // socket with unread request bytes sends a TCP reset, and the gateway
  // (nginx, which buffers the whole body and is still writing it) can then
  // fail its write before it reads the queued 503. It answers 502 instead,
  // mapped to control_plane_unavailable, and the Host never sees the
  // retryable code. Discarding retains nothing, so the refusal still takes no
  // admission unit. A body that does not end within the read deadline, or
  // that declares or sends more than maxDiscardBodyBytes, is answered at once
  // and closed, as before.
  private refuseBeforeRead(req: Request, res: Response, refusal: AdmissionRefusal): void {
    const inFlight = this.inFlight
    req.pause()
    res.setHeader('Connection', 'close')
    // Nothing was read or recorded, so the Host may retry the same attempt
    // once after this delay (same provider, no failover).
    res.setHeader('Retry-After', String(this.policy.retryAfterSeconds))
    requestLog(req).warn(
      {
        event: 'llm_provider_attempt_admission_refused',
        reason: 'authorize_capacity_exceeded',
        refusal,
        inFlight,
        maxInFlight: this.policy.maxInFlight,
        queued: this.waiters.length,
        maxQueued: this.policy.maxQueued,
        maxPerPrincipal: this.policy.maxPerPrincipal,
        queueWaitMs: this.policy.queueWaitMs,
      },
      'authorize admission refused before body read'
    )
    this.discardThenAnswer(req, res)
  }

  private discardThenAnswer(req: Request, res: Response): void {
    let settled = false
    let discarded = 0
    let readTimer: ReturnType<typeof setTimeout> | undefined
    let closeTimer: ReturnType<typeof setTimeout> | undefined
    const { closeGraceMs, maxDiscardBodyBytes, readDeadlineMs } = this.policy

    const detachReader = (): void => {
      if (readTimer !== undefined) clearTimeout(readTimer)
      req.off('data', discard)
      req.off('end', answerDiscarded)
      req.off('aborted', abandon)
      req.off('error', abandon)
      res.off('close', abandonUnfinished)
    }
    const clearClose = (): void => {
      if (closeTimer !== undefined) clearTimeout(closeTimer)
      res.off('finish', stopSocket)
      res.off('close', stopSocket)
      res.off('finish', clearClose)
      res.off('close', clearClose)
    }
    const stopSocket = (): void => {
      clearClose()
      if (!req.destroyed) req.destroy()
    }
    // Inside an event callback a throw would be uncaught, so it is logged and
    // the socket closed; on the synchronous path it propagates to the caller.
    const send = (rethrow: boolean): void => {
      try {
        res.status(503).json({ error: 'authorize_capacity_exceeded' })
      } catch (err) {
        stopSocket()
        if (rethrow) throw err
        requestLog(req).error(
          { event: 'llm_provider_attempt_admission_refusal_write_failed', err },
          'authorize capacity refusal could not be written'
        )
      }
    }
    // Unread bytes remain: answer now and close the socket. A blocked or
    // pipelined peer can leave the 503 unable to reach finish/close, so the
    // close is also bounded by closeGraceMs.
    const answerAndClose = (
      reason: 'declared_length' | 'discard_limit' | 'read_deadline',
      rethrow: boolean
    ): void => {
      if (settled) return
      settled = true
      detachReader()
      req.pause()
      requestLog(req).warn(
        {
          event: 'llm_provider_attempt_admission_discard_stopped',
          reason,
          discardedBytes: discarded,
          maxDiscardBodyBytes,
        },
        'authorize capacity refusal closed before the body ended'
      )
      closeTimer = setTimeout(stopSocket, closeGraceMs)
      res.once('finish', stopSocket)
      res.once('close', stopSocket)
      send(rethrow)
    }
    // The body was read to its end, so Node's Connection: close ends the
    // socket without a reset. closeGraceMs still bounds a peer that never
    // reads the 503.
    const answerDiscarded = (): void => {
      if (settled) return
      settled = true
      detachReader()
      closeTimer = setTimeout(stopSocket, closeGraceMs)
      res.once('finish', clearClose)
      res.once('close', clearClose)
      send(false)
    }
    const abandon = (): void => {
      if (settled) return
      settled = true
      detachReader()
      if (!req.destroyed) req.destroy()
    }
    const abandonUnfinished = (): void => {
      if (!res.writableFinished) abandon()
    }
    const discard = (chunk: Buffer): void => {
      discarded += chunk.length
      if (discarded > maxDiscardBodyBytes) answerAndClose('discard_limit', false)
    }

    const declared = req.headers?.['content-length']
    if (
      typeof declared === 'string' &&
      /^\d+$/.test(declared) &&
      Number(declared) > maxDiscardBodyBytes
    ) {
      answerAndClose('declared_length', true)
      return
    }
    readTimer = setTimeout(() => answerAndClose('read_deadline', false), readDeadlineMs)
    req.on('data', discard)
    req.once('end', answerDiscarded)
    req.once('aborted', abandon)
    req.once('error', abandon)
    res.once('close', abandonUnfinished)
    req.resume()
  }
}

function principalKey(principal: AuthorizeBodyAdmissionPrincipal): string {
  return JSON.stringify([principal.sub, [...new Set(principal.hostRefs)].sort()])
}

// A request stream auto-destroys once its body has been read to the end, so
// `destroyed` means a disconnect only while the body is still unread. An
// already-read body goes to the parser, which skips it.
function cannotStartBodyRead(req: Request, res: Response): boolean {
  return req.aborted || (req.destroyed && !req.readableEnded) || res.destroyed || res.writableEnded
}

function removeParserBody(err: unknown): void {
  if (err && typeof err === 'object' && 'body' in err) {
    delete (err as ParserError).body
  }
}

function parserErrorRequiresConnectionClose(err: unknown): boolean {
  const typed = err as ParserError
  return typed?.type === 'entity.too.large' || typed?.status === 413
}

function requestLog(req: Request): Logger {
  return req.log?.child({ module: LOG_MODULE }) ?? log
}

function parseToCompletion(
  req: Request,
  res: Response,
  parser: RequestHandler,
  policy: AuthorizeBodyAdmissionPolicy
): Promise<void> {
  return new Promise((resolve, reject) => {
    let timedOut = false
    let settled = false
    let readStopped = false
    let closeBackstop: ReturnType<typeof setTimeout> | undefined

    const stopRead = (): void => {
      if (readStopped) return
      readStopped = true
      req.destroy()
    }
    const stopDisconnected = (): void => {
      if (!res.writableFinished) stopRead()
    }
    res.once('close', stopDisconnected)

    const timer = setTimeout(() => {
      timedOut = true
      requestLog(req).warn(
        {
          event: 'llm_provider_attempt_body_refused',
          reason: 'read_deadline',
          closeGraceMs: policy.closeGraceMs,
        },
        'authorize body read deadline exceeded'
      )
      // This backstop is independent of response finish. A peer that never
      // reads the 408 must not leave the body reader charged indefinitely.
      closeBackstop = setTimeout(stopRead, policy.closeGraceMs)
      res.once('finish', stopRead)
      if (res.destroyed || res.writableEnded || res.headersSent) {
        stopRead()
        return
      }
      try {
        res.setHeader('Connection', 'close')
        res.status(408).json({ error: 'request_timeout' })
      } catch {
        stopRead()
      }
    }, policy.readDeadlineMs)

    const finish = (err?: unknown): void => {
      removeParserBody(err)
      if (settled) return
      settled = true
      clearTimeout(timer)
      if (closeBackstop !== undefined) clearTimeout(closeBackstop)
      res.off('finish', stopRead)
      res.off('close', stopDisconnected)
      if (!timedOut && err && parserErrorRequiresConnectionClose(err) && !res.headersSent) {
        res.setHeader('Connection', 'close')
      }
      // Capacity remains charged until this callback proves parser unwind.
      if (timedOut) resolve()
      else if (err) reject(err)
      else resolve()
    }

    try {
      parser(req, res, finish)
      // A queued IncomingMessage was explicitly paused. Resume only after
      // the admitted parser has installed its body reader, never on grant.
      if (!settled && !readStopped) req.resume()
    } catch (err) {
      finish(err)
    }
  })
}