Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
19.6 KB
Raw Blame
import {
  ENVELOPE_ALLOWANCE_BYTES as CONTRACT_ENVELOPE_ALLOWANCE_BYTES,
  LIMITS as CONTRACT_LIMITS,
} from '@clerum/llm-provider-attempt-contract'

/**
 * #731 — room for the runtime envelope around the contract-capped `request`.
 * The contract owns the value, shared with control-api and mcp-host.
 */
export const ENVELOPE_ALLOWANCE_BYTES = CONTRACT_ENVELOPE_ALLOWANCE_BYTES

/**
 * The default body limit, derived from the contract so the proxy never refuses,
 * as a 413, a request the contract accepts.
 */
export const DEFAULT_MAX_BODY_BYTES = CONTRACT_LIMITS.maxRequestBodyBytes + ENVELOPE_ALLOWANCE_BYTES

/**
 * #731 R3-2 — how many maximum-size bodies may be read and parsed at once.
 * Several copies of a body are alive while it is parsed, hashed and forwarded
 * (the raw buffer, the decoded string, the parsed object, the contract copy and
 * more than one serialization of it). Without this bound the stream gate would
 * let 24 bodies in.
 * A body whose Content-Length exceeds the ordinary cap is read under
 * `visualStreamGate`. If it parses to a size within the ordinary cap
 * (demotion), it takes a reservation for that size before releasing the
 * visual slot, so the declared bodies a pod can hold at once are still 3 x 8
 * MiB here plus 2 x 24 MiB there. Measured with the
 * full load the gates admit (eight 8 MiB streams, two 24 MiB visual streams
 * and three queued 8 MiB bodies, #739 D5), the process peaked at 790 MiB of
 * RSS with `--max-old-space-size=384` and at 886-1009 MiB with an uncapped
 * heap, which is why the deployment sets a heap cap. The budget counts bytes,
 * not structure: bodies at the worst structure the contract admits (262 144
 * containers, 262 144 members, 1 048 576 elements) are 2.8 MiB each when
 * compact, so it admits about eight of them parsing at once, or about five at
 * 4.7 MiB, each a ~44 MiB tree. That aborted the process at a 384 MiB heap
 * cap; the deployment uses 768 MiB and a 2048Mi memory limit, the maximum
 * peak of that load (1327-1460 MiB of RSS, macOS, not cgroup memory) plus 25 %
 * (#806 Q1). Charging structure as well as bytes is deferred code work.
 *
 * Bytes alone do not bound the parse: JSON.parse allocates one heap
 * object per container, so the contracts cap containers at
 * `maxRequestContainers` (262 144) and every parser scans the raw body for it
 * before JSON.parse. Measured in this proxy (tsc build,
 * `--max-old-space-size=384`, upstream held open) with three ordinary bodies
 * at the 8 404 992-byte cap and two visual bodies in flight, each exactly
 * 262 144 containers: every run exited 0 and admitted all five, the heap kept
 * 154.4-163.4 MiB after a full GC and peaked at 205.6-238.4 MiB sampled. At
 * 524 288 containers one run peaked at 319.6 MiB, and at 1 048 576 the
 * process aborted. The bound for a new value: every run exits 0 and admits
 * five of five, at most 200 MiB after GC and 280 MiB sampled peak. These are
 * lower bounds: the serializations made while forwarding are not included.
 */
export const IN_FLIGHT_BODY_BUDGET_BODIES = 3

/**
 * R9-M-B — how long a body that holds a budget reservation may take to be
 * read and parsed. The clock starts when the reservation is granted, so time
 * spent queued does not count, and stops when the parser finishes. Measured on
 * loopback, an 8 MiB body is read in 7.6-22.6 ms alone and in under 60 ms with
 * three at once, so 10 s only cuts a body that has stalled. Without it a
 * stalled body keeps its reservation until Node's 300 s `requestTimeout`.
 * The wait before the grant is bounded separately, by the request's admission
 * clock (`STREAM_LIMITS.maxQueueWaitMs` from arrival, #739 D1).
 * Not env-tunable.
 */
export const BODY_READ_DEADLINE_MS = 10_000

/**
 * Bounds the interval between a visual read timeout/client response close and
 * forced request destruction. The parser's callback—not this timer—releases the
 * visual slot and principal share.
 */
export const VISUAL_READ_CLOSE_GRACE_MS = 100

export const STREAM_LIMITS = {
  maxConcurrentStreams: 8,
  maxQueuedRequests: 16,
  maxStreamDurationMs: 1_800_000,
  // Longest total time a request may spend queued in this proxy: one
  // admission clock from arrival bounds the body budget, the visual gate and
  // the stream gate together (#739 D1), so a visual request that waits at both
  // gates still waits at most this long in total. Bounded so that queue wait +
  // redeem + the first keepalive stays below the Host HTTP client's 300 s
  // header timeout.
  maxQueueWaitMs: 60_000,
  // Longest silence tolerated while waiting on the upstream (response headers
  // or the next SSE chunk). Same value as the Codex CLI default,
  // `DEFAULT_STREAM_IDLE_TIMEOUT_MS` (300 s) in openai/codex
  // codex-rs/model-provider-info/src/lib.rs:63 (commit 6824dabe0, read
  // 2026-09-23).
  upstreamIdleTimeoutMs: 300_000,
} as const

/**
 * Admission for a body whose Content-Length exceeds the ordinary cap.
 * The pod cannot hold the ordinary 8-stream gate across a 24 MiB image, so
 * those requests are a tighter sibling and a V2 request keeps the slot until
 * the stream ends. Small bodies, including every valid V1, must not enter this
 * gate. The 2048Mi limit is sized for these two slots plus the ordinary body
 * budget; widening either one needs a new memory measurement first.
 */
export const VISUAL_STREAM_LIMITS = {
  maxConcurrentStreams: 2,
  maxQueuedRequests: 8,
} as const

/**
 * Fair share of the visual gate's entries — running plus queued — that one
 * platform principal (`sub` plus sorted `hostRefs`) may hold at once. Above the
 * share, further visual requests from that principal are refused 503
 * `visual_host_share`, so one principal
 * cannot fill a gate that every other host still needs. The gate widths above
 * are unchanged; the share only bounds how much of them one principal occupies.
 * Its refusal is a local capacity outcome, not an upstream outage.
 */
export const VISUAL_PER_HOST_MAX_ADMITTED = 4

// Largest delay setTimeout honors; Node fires anything above it after 1 ms.
const MAX_TIMER_DELAY_MS = 2_147_483_647

/**
 * Why a wait was refused. `deadline` means the caller's own deadline ended
 * the wait, so a caller that passed the ticket's expiry as that deadline can
 * tell a dead ticket from a full queue without reading the clock again: the
 * deadline timer can fire a millisecond before `Date.now()` reaches it.
 */
export type RequestLimitKind =
  | 'aborted'
  | 'deadline'
  | 'invalid'
  | 'queue_full'
  | 'queue_wait'
  | 'ticket_life'

export type RequestLimitCode = 'provider_unavailable' | 'proxy_capacity_exceeded' | 'visual_gate'

export class RequestLimitError extends Error {
  readonly code: RequestLimitCode
  constructor(
    message: string,
    readonly kind: RequestLimitKind,
    code: RequestLimitCode = 'provider_unavailable'
  ) {
    super(message)
    this.name = 'RequestLimitError'
    this.code = code
  }
}

/**
 * #739 D1-bis — the execution ticket expired while its request still waited
 * for a stream slot. Redeeming it could only end in `ticket_expired`, so the
 * request is refused as a capacity limit instead, before any redeem. It is a
 * RequestLimitError on the wire (503 `provider_unavailable`); the attempt
 * failure metric counts it as `provider_unavailable`, not `request_limit`.
 */
export class TicketLifeError extends RequestLimitError {
  constructor() {
    super('execution ticket expired while queued', 'ticket_life')
    this.name = 'TicketLifeError'
  }
}

export function assertBoundedDeadline(
  deadlineMs: number | undefined,
  maxDeadlineMs: number
): number {
  // Without a valid maximum, Math.min below would return NaN for a valid
  // request deadline instead of refusing it.
  if (!Number.isInteger(maxDeadlineMs) || maxDeadlineMs <= 0) {
    throw new RequestLimitError('deadline is invalid', 'invalid')
  }
  const requested = deadlineMs ?? Math.min(maxDeadlineMs, STREAM_LIMITS.maxStreamDurationMs)
  if (!Number.isFinite(requested) || !Number.isInteger(requested) || requested <= 0) {
    throw new RequestLimitError('deadline is invalid', 'invalid')
  }
  return Math.min(
    requested,
    maxDeadlineMs,
    STREAM_LIMITS.maxStreamDurationMs,
    CONTRACT_LIMITS.maxDeadlineMs
  )
}

export function assertBoundedIdleTimeout(idleTimeoutMs: number | undefined): number {
  const requested = idleTimeoutMs ?? STREAM_LIMITS.upstreamIdleTimeoutMs
  if (!Number.isInteger(requested) || requested <= 0) {
    throw new RequestLimitError('upstream idle timeout is invalid', 'invalid')
  }
  return Math.min(requested, STREAM_LIMITS.upstreamIdleTimeoutMs)
}

/**
 * A queued stream-gate caller. `grant` hands it the slot a release passes on
 * and returns false, after refusing it, when its wait already reached its
 * bound.
 */
type StreamWaiter = { grant: (release: () => void) => boolean }

export class StreamGate {
  private running = 0
  private readonly waiters: StreamWaiter[] = []

  constructor(
    private readonly maxConcurrent: number = STREAM_LIMITS.maxConcurrentStreams,
    private readonly maxQueued: number = STREAM_LIMITS.maxQueuedRequests,
    private readonly maxQueueWaitMs: number = STREAM_LIMITS.maxQueueWaitMs,
    private readonly queueCapacityCode: RequestLimitCode = 'proxy_capacity_exceeded'
  ) {
    // A NaN or fractional size would let every caller through or queue
    // without bound, because the comparisons below would never hold.
    if (!Number.isInteger(maxConcurrent) || maxConcurrent < 1) {
      throw new RangeError('maxConcurrent must be a positive integer')
    }
    if (!Number.isInteger(maxQueued) || maxQueued < 0) {
      throw new RangeError('maxQueued must be a non-negative integer')
    }
    // setTimeout turns NaN, 0, a negative delay or one above 2^31 - 1 into a
    // 1 ms timer, which would refuse every queued waiter at once instead of
    // after the configured wait. Fractions are refused so the bound stays a
    // whole number of milliseconds.
    if (
      !Number.isInteger(maxQueueWaitMs) ||
      maxQueueWaitMs <= 0 ||
      maxQueueWaitMs > MAX_TIMER_DELAY_MS
    ) {
      throw new RangeError(`maxQueueWaitMs must be an integer in 1..${MAX_TIMER_DELAY_MS}`)
    }
  }

  /**
   * Take a stream slot. Waiters are granted in arrival order: a release hands
   * its slot straight to the oldest waiter, and a new caller queues behind the
   * waiters instead of taking a slot ahead of them. The returned release is
   * idempotent. When `signal` aborts (client disconnected) while the caller is
   * still queued, the waiter is rejected and removed, so a dropped client never
   * proceeds to redeem an attempt. A waiter still queued after
   * `maxQueueWaitMs`, or at `deadlineAt` (epoch ms) when that comes first, is
   * rejected and removed the same way (#739 D1). The bound is fixed when the
   * caller queues, as the smaller of the two, and measured on the monotonic
   * clock from then on. A deadline timer settles the waiter at the bound, and
   * a release that reaches a waiter after the bound checks the elapsed wait
   * first and passes the slot on to the next waiter, so a slot that frees after
   * the bound does not admit it even when the event loop stalled across the
   * bound. A free slot with nobody queued is granted at once whatever the
   * deadline; the caller checks a deadline already past.
   *
   * The refusal names which bound ran out. The gate's own bound is
   * `maxQueueWaitMs` after the enqueue, or after `queueWaitStartedAt` (epoch
   * ms) when the caller passes it: the visual gate's own wait is the request's
   * admission clock, so its caller passes the arrival instant. Without
   * `queueWaitStartedAt`, a `deadlineAt` at or before the own bound governs
   * (`deadline`, `provider_unavailable`). With it, the deadline and the own
   * bound are measured from the same instant, so a `deadlineAt` equal to the
   * own bound is that bound restated and only a strictly earlier one governs.
   * Every other expiry is the gate's queue wait (`queue_wait`,
   * `queueCapacityCode`). The decision does not depend on how many
   * milliseconds passed between the caller's stamp and the enqueue.
   */
  async acquire(
    signal?: AbortSignal,
    deadlineAt?: number,
    queueWaitStartedAt?: number
  ): Promise<() => void> {
    if (deadlineAt !== undefined && !Number.isFinite(deadlineAt)) {
      throw new RangeError(`a stream gate deadline must be a finite epoch time, got ${deadlineAt}`)
    }
    if (queueWaitStartedAt !== undefined && !Number.isFinite(queueWaitStartedAt)) {
      throw new RangeError(
        `a stream gate queue wait start must be a finite epoch time, got ${queueWaitStartedAt}`
      )
    }
    if (signal?.aborted) throw new RequestLimitError('stream request was aborted', 'aborted')
    if (this.waiters.length === 0 && this.running < this.maxConcurrent) {
      this.running += 1
      return this.slotRelease()
    }
    if (this.waiters.length >= this.maxQueued)
      throw new RequestLimitError('stream queue is full', 'queue_full', this.queueCapacityCode)
    return new Promise<() => void>((resolve, reject) => {
      const leave = () => {
        const index = this.waiters.indexOf(waiter)
        if (index !== -1) this.waiters.splice(index, 1)
        clearTimeout(deadline)
        signal?.removeEventListener('abort', onAbort)
      }
      const onAbort = () => {
        leave()
        reject(new RequestLimitError('stream request was aborted', 'aborted'))
      }
      const queuedAt = performance.now()
      const enqueuedAt = Date.now()
      const ownBoundAt = (queueWaitStartedAt ?? enqueuedAt) + this.maxQueueWaitMs
      const deadlineGoverns =
        deadlineAt !== undefined &&
        (queueWaitStartedAt === undefined ? deadlineAt <= ownBoundAt : deadlineAt < ownBoundAt)
      // No waiter stays queued longer than maxQueueWaitMs after the enqueue.
      const boundAt = deadlineAt !== undefined && deadlineGoverns ? deadlineAt : ownBoundAt
      const waitBoundMs = Math.min(this.maxQueueWaitMs, boundAt - enqueuedAt)
      const expire = () => {
        leave()
        reject(
          new RequestLimitError(
            'stream queue wait exceeded',
            deadlineGoverns ? 'deadline' : 'queue_wait',
            deadlineGoverns ? 'provider_unavailable' : this.queueCapacityCode
          )
        )
      }
      const waiter: StreamWaiter = {
        grant: release => {
          // After the event loop stalls, a release can run while the deadline
          // timer is overdue but has not fired yet.
          if (performance.now() - queuedAt >= waitBoundMs) {
            expire()
            return false
          }
          leave()
          resolve(release)
          return true
        },
      }
      const deadline = setTimeout(expire, Math.max(0, waitBoundMs))
      signal?.addEventListener('abort', onAbort, { once: true })
      this.waiters.push(waiter)
      if (waitBoundMs <= 0) expire()
    })
  }

  private slotRelease(): () => void {
    let released = false
    return () => {
      if (released) return
      released = true
      // The slot passes to the oldest waiter still inside its bound, so
      // `running` stays the same; it is freed only when nobody can take it.
      while (this.waiters.length > 0) {
        if (this.waiters[0]!.grant(this.slotRelease())) return
      }
      this.running -= 1
    }
  }

  /** Observable occupancy for tests. Production callers must not branch on this. */
  snapshot(): { running: number; queued: number } {
    return { running: this.running, queued: this.waiters.length }
  }
}

export const streamGate = new StreamGate()

export const visualStreamGate = new StreamGate(
  VISUAL_STREAM_LIMITS.maxConcurrentStreams,
  VISUAL_STREAM_LIMITS.maxQueuedRequests,
  STREAM_LIMITS.maxQueueWaitMs,
  'visual_gate'
)

type BodyWaiter = { bytes: number; grant: (release: () => void) => void }

/**
 * #731 R3-2 — a byte budget for request bodies, taken from the declared
 * `Content-Length` before the body is read. Bodies that do not fit wait in a
 * bounded FIFO queue; a full queue is refused with the same RequestLimitError
 * the stream gate raises, so the client sees the existing overload response.
 */
export class BodyBudget {
  private inFlight = 0
  private readonly waiters: BodyWaiter[] = []

  constructor(
    private readonly capacityBytes: number,
    private readonly maxQueued: number = STREAM_LIMITS.maxQueuedRequests
  ) {
    // The same standard as StreamGate: a NaN capacity would queue every body
    // until the queue is full, and an infinite one would admit every body.
    if (!Number.isSafeInteger(capacityBytes) || capacityBytes < 1) {
      throw new RangeError('capacityBytes must be a positive integer')
    }
    if (!Number.isInteger(maxQueued) || maxQueued < 0) {
      throw new RangeError('maxQueued must be a non-negative integer')
    }
  }

  get inFlightBytes(): number {
    return this.inFlight
  }

  get queued(): number {
    return this.waiters.length
  }

  /**
   * Reserve `bytes` of the budget. The returned release is idempotent. When
   * `signal` aborts while the caller is queued, the waiter is rejected and
   * removed, and the waiters behind it are reconsidered. A waiter still queued
   * at `deadlineAt` (epoch ms, the request's admission deadline, #739 D1) is
   * rejected and removed the same way, so a large body at the head of the
   * queue cannot hold back the smaller ones behind it past that instant.
   */
  async acquire(bytes: number, signal?: AbortSignal, deadlineAt?: number): Promise<() => void> {
    if (!Number.isInteger(bytes) || bytes < 0 || bytes > this.capacityBytes) {
      throw new RangeError(`a body of ${bytes} bytes cannot fit a budget of ${this.capacityBytes}`)
    }
    if (deadlineAt !== undefined && !Number.isFinite(deadlineAt)) {
      throw new RangeError(
        `a body admission deadline must be a finite epoch time, got ${deadlineAt}`
      )
    }
    if (signal?.aborted) throw new RequestLimitError('body admission was aborted', 'aborted')
    if (this.waiters.length === 0 && this.inFlight + bytes <= this.capacityBytes) {
      return this.take(bytes)
    }
    if (this.waiters.length >= this.maxQueued) {
      throw new RequestLimitError(
        'body admission queue is full',
        'queue_full',
        'proxy_capacity_exceeded'
      )
    }
    return new Promise<() => void>((resolve, reject) => {
      let deadline: ReturnType<typeof setTimeout> | undefined
      const leave = (reason: string, kind: RequestLimitKind) => {
        const index = this.waiters.indexOf(waiter)
        if (index === -1) return
        this.waiters.splice(index, 1)
        clearTimeout(deadline)
        signal?.removeEventListener('abort', onAbort)
        reject(new RequestLimitError(reason, kind))
        this.drain()
      }
      const onAbort = () => leave('body admission was aborted', 'aborted')
      const waiter: BodyWaiter = {
        bytes,
        grant: release => {
          clearTimeout(deadline)
          signal?.removeEventListener('abort', onAbort)
          resolve(release)
        },
      }
      signal?.addEventListener('abort', onAbort, { once: true })
      this.waiters.push(waiter)
      if (deadlineAt !== undefined) {
        deadline = setTimeout(
          () => leave('body admission wait exceeded', 'deadline'),
          Math.max(0, deadlineAt - Date.now())
        )
      }
    })
  }

  private take(bytes: number): () => void {
    this.inFlight += bytes
    let released = false
    return () => {
      if (released) return
      released = true
      this.inFlight -= bytes
      this.drain()
    }
  }

  /** Grant queued bodies in arrival order while the head of the queue fits. */
  private drain(): void {
    while (
      this.waiters.length > 0 &&
      this.inFlight + this.waiters[0]!.bytes <= this.capacityBytes
    ) {
      const waiter = this.waiters.shift()!
      waiter.grant(this.take(waiter.bytes))
    }
  }
}