Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
19.2 KB
Raw Blame
import {
  ENVELOPE_ALLOWANCE_BYTES as CONTRACT_ENVELOPE_ALLOWANCE_BYTES,
  LIMITS as CONTRACT_LIMITS,
} from '@clerum/grok-provider-attempt-contract'

/**
 * #731 — room for the runtime envelope around the contract-capped `request`.
 * The contract owns the value, shared with control-api and mcp-host.
 */
export const ENVELOPE_ALLOWANCE_BYTES = CONTRACT_ENVELOPE_ALLOWANCE_BYTES

/**
 * The default body limit, derived from the contract so the proxy never refuses,
 * as a 413, a request the contract accepts.
 */
export const DEFAULT_MAX_BODY_BYTES = CONTRACT_LIMITS.maxRequestBodyBytes + ENVELOPE_ALLOWANCE_BYTES

/**
 * #731 R3-2 — how many maximum-size bodies may be read and parsed at once.
 * Several copies of a body are alive while it is parsed and hashed (the raw
 * buffer, the decoded string, the parsed object, the contract copy and the
 * canonical serialization). Without this bound the stream gate would let 24
 * bodies in.
 * A body whose Content-Length exceeds the ordinary cap is read under
 * `visualStreamGate`. If it parses to a size within the ordinary cap
 * (demotion), it takes a reservation for that size before releasing the
 * visual slot. With eight 8 MiB streams and three queued 8 MiB bodies (#739
 * D5) the process peaked at 511 MiB of RSS with `--max-old-space-size=384`
 * (#739 measured 509.8) and at 480-511 MiB with an uncapped heap, which is
 * why the deployment sets a heap cap. The full load the gates admit adds
 * one visual stream; `VISUAL_STREAM_LIMITS` records that peak and the memory
 * limit it sets. The budget counts bytes, not structure: bodies at the worst
 * structure the contract admits (262 144 containers, 262 144 members,
 * 1 048 576 elements) are 2.8 MiB each when compact, so it admits about eight
 * of them parsing at once, or about five at 4.7 MiB, each a ~44 MiB tree. That
 * aborted the process at a 384 MiB heap cap; the deployment uses 768 MiB
 * (#806 Q1). Charging structure as well as bytes is deferred code work.
 */
export const IN_FLIGHT_BODY_BUDGET_BODIES = 3

/**
 * R9-M-B — how long a body that holds a budget reservation may take to be
 * read and parsed. The clock starts when the reservation is granted, so time
 * spent queued does not count, and stops when the parser finishes. Measured on
 * the Codex twin over loopback, an 8 MiB body is read in 7.6-22.6 ms alone and
 * in under 60 ms with three at once, so 10 s only cuts a body that has
 * stalled. Without it a stalled body keeps its reservation until Node's 300 s
 * `requestTimeout`. The wait before the grant is bounded separately, by the
 * request's admission clock (`STREAM_LIMITS.maxQueueWaitMs` from arrival,
 * #739 D1). Not env-tunable.
 */
export const BODY_READ_DEADLINE_MS = 10_000

/**
 * Bounds the interval between a visual read timeout/client response close and
 * forced request destruction. The parser's callback—not this timer—releases the
 * visual slot and principal share.
 */
export const VISUAL_READ_CLOSE_GRACE_MS = 100

export const STREAM_LIMITS = {
  maxConcurrentStreams: 8,
  maxQueuedRequests: 16,
  maxStreamDurationMs: 1_800_000,
  // Longest total time a request may spend queued in this proxy: one
  // admission clock from arrival bounds the body budget, the visual gate and
  // the stream gate together (#739 D1), so a visual request that waits at both
  // gates still waits at most this long in total. Bounded so that queue wait +
  // redeem + the first keepalive stays below the Host HTTP client's 300 s
  // header timeout.
  maxQueueWaitMs: 60_000,
  // Longest silence tolerated while waiting on the upstream (response headers
  // or the next SSE chunk). Same value as the Grok Build CLI default,
  // `inference_idle_timeout_secs = 600` in the default config embedded in the
  // grok-build 1.0.41 macOS binary (read with `strings`, 2026-09-23). The CLI
  // source is not public, so no file and line can be cited.
  upstreamIdleTimeoutMs: 600_000,
} as const

/**
 * Admission for a body whose Content-Length exceeds the ordinary cap. A V2
 * request keeps its image bytes resident until the upstream stream ends, and
 * a 35 MiB envelope is more than four ordinary bodies, so those requests take
 * a 1-wide sibling of the 8-wide stream gate and keep the slot for the
 * stream. Small bodies, including every valid V1, must not enter this gate.
 * Measured with the full load the gates admit (the D5 load above plus one
 * ~36 MB V2 stream: a 20 MiB PNG and 8 MiB of text; tsc build, one process,
 * `--max-old-space-size=384`, upstream request through undici), the process
 * peaked at 775 MiB of RSS, against 511 MiB for D5 alone. With 8 held streams,
 * 8 queued bodies at the worst structure and a 768 MiB heap cap, the same load
 * peaked at 1212-1482 MiB (macOS RSS, not cgroup memory). The deployment's
 * 2048Mi memory limit is the maximum plus 25 %, rounded up; widening this gate
 * or the ordinary body budget needs a new memory measurement first.
 */
export const VISUAL_STREAM_LIMITS = {
  maxConcurrentStreams: 1,
  maxQueuedRequests: 4,
} as const

/**
 * Fair share of the visual gate's entries — running plus queued — that one
 * platform principal (`sub` plus sorted `hostRefs`) may hold at once. Above the
 * share, further visual requests from that principal are refused 503
 * `visual_host_share`, so one principal
 * cannot fill a gate that every other host still needs. The gate widths above
 * are unchanged; the share only bounds how much of them one principal occupies.
 * Its refusal is a local capacity outcome, not an upstream outage.
 */
export const VISUAL_PER_HOST_MAX_ADMITTED = 2

// Largest delay setTimeout honors; Node fires anything above it after 1 ms.
const MAX_TIMER_DELAY_MS = 2_147_483_647

/**
 * Why a wait was refused. `deadline` means the caller's own deadline ended
 * the wait, so a caller that passed the ticket's expiry as that deadline can
 * tell a dead ticket from a full queue without reading the clock again: the
 * deadline timer can fire a millisecond before `Date.now()` reaches it.
 */
export type RequestLimitKind =
  | 'aborted'
  | 'deadline'
  | 'invalid'
  | 'queue_full'
  | 'queue_wait'
  | 'ticket_life'

export type RequestLimitCode = 'provider_unavailable' | 'proxy_capacity_exceeded' | 'visual_gate'

export class RequestLimitError extends Error {
  readonly code: RequestLimitCode
  constructor(
    message: string,
    readonly kind: RequestLimitKind,
    code: RequestLimitCode = 'provider_unavailable'
  ) {
    super(message)
    this.name = 'RequestLimitError'
    this.code = code
  }
}

/**
 * #739 D1-bis — the execution ticket expired while its request still waited
 * for a stream slot. Redeeming it could only end in `ticket_expired`, so the
 * request is refused as a capacity limit instead, before any redeem. It is a
 * RequestLimitError on the wire (503 `provider_unavailable`); the attempt
 * failure metric counts it as `provider_unavailable`, not `request_limit`.
 */
export class TicketLifeError extends RequestLimitError {
  constructor() {
    super('execution ticket expired while queued', 'ticket_life')
    this.name = 'TicketLifeError'
  }
}

export function assertBoundedDeadline(
  deadlineMs: number | undefined,
  maxDeadlineMs: number
): number {
  // Without a valid maximum, Math.min below would return NaN for a valid
  // request deadline instead of refusing it.
  if (!Number.isInteger(maxDeadlineMs) || maxDeadlineMs <= 0) {
    throw new RequestLimitError('deadline is invalid', 'invalid')
  }
  const requested = deadlineMs ?? Math.min(maxDeadlineMs, STREAM_LIMITS.maxStreamDurationMs)
  if (!Number.isFinite(requested) || !Number.isInteger(requested) || requested <= 0) {
    throw new RequestLimitError('deadline is invalid', 'invalid')
  }
  return Math.min(
    requested,
    maxDeadlineMs,
    STREAM_LIMITS.maxStreamDurationMs,
    CONTRACT_LIMITS.maxDeadlineMs
  )
}

export function assertBoundedIdleTimeout(idleTimeoutMs: number | undefined): number {
  const requested = idleTimeoutMs ?? STREAM_LIMITS.upstreamIdleTimeoutMs
  if (!Number.isInteger(requested) || requested <= 0) {
    throw new RequestLimitError('upstream idle timeout is invalid', 'invalid')
  }
  return Math.min(requested, STREAM_LIMITS.upstreamIdleTimeoutMs)
}

/**
 * A queued stream-gate caller. `grant` hands it the slot a release passes on
 * and returns false, after refusing it, when its wait already reached its
 * bound.
 */
type StreamWaiter = { grant: (release: () => void) => boolean }

export class StreamGate {
  private running = 0
  private readonly waiters: StreamWaiter[] = []

  constructor(
    private readonly maxConcurrent: number = STREAM_LIMITS.maxConcurrentStreams,
    private readonly maxQueued: number = STREAM_LIMITS.maxQueuedRequests,
    private readonly maxQueueWaitMs: number = STREAM_LIMITS.maxQueueWaitMs,
    private readonly queueCapacityCode: RequestLimitCode = 'proxy_capacity_exceeded'
  ) {
    // A NaN or fractional size would let every caller through or queue
    // without bound, because the comparisons below would never hold.
    if (!Number.isInteger(maxConcurrent) || maxConcurrent < 1) {
      throw new RangeError('maxConcurrent must be a positive integer')
    }
    if (!Number.isInteger(maxQueued) || maxQueued < 0) {
      throw new RangeError('maxQueued must be a non-negative integer')
    }
    // setTimeout turns NaN, 0, a negative delay or one above 2^31 - 1 into a
    // 1 ms timer, which would refuse every queued waiter at once instead of
    // after the configured wait. Fractions are refused so the bound stays a
    // whole number of milliseconds.
    if (
      !Number.isInteger(maxQueueWaitMs) ||
      maxQueueWaitMs <= 0 ||
      maxQueueWaitMs > MAX_TIMER_DELAY_MS
    ) {
      throw new RangeError(`maxQueueWaitMs must be an integer in 1..${MAX_TIMER_DELAY_MS}`)
    }
  }

  /**
   * Take a stream slot. Waiters are granted in arrival order: a release hands
   * its slot straight to the oldest waiter, and a new caller queues behind the
   * waiters instead of taking a slot ahead of them. The returned release is
   * idempotent. When `signal` aborts (client disconnected) while the caller is
   * still queued, the waiter is rejected and removed, so a dropped client never
   * proceeds to redeem an attempt. A waiter still queued after
   * `maxQueueWaitMs`, or at `deadlineAt` (epoch ms) when that comes first, is
   * rejected and removed the same way (#739 D1). The bound is fixed when the
   * caller queues, as the smaller of the two, and measured on the monotonic
   * clock from then on. A deadline timer settles the waiter at the bound, and
   * a release that reaches a waiter after the bound checks the elapsed wait
   * first and passes the slot on to the next waiter, so a slot that frees after
   * the bound does not admit it even when the event loop stalled across the
   * bound. A free slot with nobody queued is granted at once whatever the
   * deadline; the caller checks a deadline already past.
   *
   * The refusal names which bound ran out. The gate's own bound is
   * `maxQueueWaitMs` after the enqueue, or after `queueWaitStartedAt` (epoch
   * ms) when the caller passes it: the visual gate's own wait is the request's
   * admission clock, so its caller passes the arrival instant. Without
   * `queueWaitStartedAt`, a `deadlineAt` at or before the own bound governs
   * (`deadline`, `provider_unavailable`). With it, the deadline and the own
   * bound are measured from the same instant, so a `deadlineAt` equal to the
   * own bound is that bound restated and only a strictly earlier one governs.
   * Every other expiry is the gate's queue wait (`queue_wait`,
   * `queueCapacityCode`). The decision does not depend on how many
   * milliseconds passed between the caller's stamp and the enqueue.
   */
  async acquire(
    signal?: AbortSignal,
    deadlineAt?: number,
    queueWaitStartedAt?: number
  ): Promise<() => void> {
    if (deadlineAt !== undefined && !Number.isFinite(deadlineAt)) {
      throw new RangeError(`a stream gate deadline must be a finite epoch time, got ${deadlineAt}`)
    }
    if (queueWaitStartedAt !== undefined && !Number.isFinite(queueWaitStartedAt)) {
      throw new RangeError(
        `a stream gate queue wait start must be a finite epoch time, got ${queueWaitStartedAt}`
      )
    }
    if (signal?.aborted) throw new RequestLimitError('stream request was aborted', 'aborted')
    if (this.waiters.length === 0 && this.running < this.maxConcurrent) {
      this.running += 1
      return this.slotRelease()
    }
    if (this.waiters.length >= this.maxQueued)
      throw new RequestLimitError('stream queue is full', 'queue_full', this.queueCapacityCode)
    return new Promise<() => void>((resolve, reject) => {
      const leave = () => {
        const index = this.waiters.indexOf(waiter)
        if (index !== -1) this.waiters.splice(index, 1)
        clearTimeout(deadline)
        signal?.removeEventListener('abort', onAbort)
      }
      const onAbort = () => {
        leave()
        reject(new RequestLimitError('stream request was aborted', 'aborted'))
      }
      const queuedAt = performance.now()
      const enqueuedAt = Date.now()
      const ownBoundAt = (queueWaitStartedAt ?? enqueuedAt) + this.maxQueueWaitMs
      const deadlineGoverns =
        deadlineAt !== undefined &&
        (queueWaitStartedAt === undefined ? deadlineAt <= ownBoundAt : deadlineAt < ownBoundAt)
      // No waiter stays queued longer than maxQueueWaitMs after the enqueue.
      const boundAt = deadlineAt !== undefined && deadlineGoverns ? deadlineAt : ownBoundAt
      const waitBoundMs = Math.min(this.maxQueueWaitMs, boundAt - enqueuedAt)
      const expire = () => {
        leave()
        reject(
          new RequestLimitError(
            'stream queue wait exceeded',
            deadlineGoverns ? 'deadline' : 'queue_wait',
            deadlineGoverns ? 'provider_unavailable' : this.queueCapacityCode
          )
        )
      }
      const waiter: StreamWaiter = {
        grant: release => {
          // After the event loop stalls, a release can run while the deadline
          // timer is overdue but has not fired yet.
          if (performance.now() - queuedAt >= waitBoundMs) {
            expire()
            return false
          }
          leave()
          resolve(release)
          return true
        },
      }
      const deadline = setTimeout(expire, Math.max(0, waitBoundMs))
      signal?.addEventListener('abort', onAbort, { once: true })
      this.waiters.push(waiter)
      if (waitBoundMs <= 0) expire()
    })
  }

  private slotRelease(): () => void {
    let released = false
    return () => {
      if (released) return
      released = true
      // The slot passes to the oldest waiter still inside its bound, so
      // `running` stays the same; it is freed only when nobody can take it.
      while (this.waiters.length > 0) {
        if (this.waiters[0]!.grant(this.slotRelease())) return
      }
      this.running -= 1
    }
  }

  /** Observable occupancy for tests. Production callers must not branch on this. */
  snapshot(): { running: number; queued: number } {
    return { running: this.running, queued: this.waiters.length }
  }
}

export const streamGate = new StreamGate()

export const visualStreamGate = new StreamGate(
  VISUAL_STREAM_LIMITS.maxConcurrentStreams,
  VISUAL_STREAM_LIMITS.maxQueuedRequests,
  STREAM_LIMITS.maxQueueWaitMs,
  'visual_gate'
)

type BodyWaiter = { bytes: number; grant: (release: () => void) => void }

/**
 * #731 R3-2 — a byte budget for request bodies, taken from the declared
 * `Content-Length` before the body is read. Bodies that do not fit wait in a
 * bounded FIFO queue; a full queue is refused with the same RequestLimitError
 * the stream gate raises, so the client sees the existing overload response.
 */
export class BodyBudget {
  private inFlight = 0
  private readonly waiters: BodyWaiter[] = []

  constructor(
    private readonly capacityBytes: number,
    private readonly maxQueued: number = STREAM_LIMITS.maxQueuedRequests
  ) {
    // The same standard as StreamGate: a NaN capacity would queue every body
    // until the queue is full, and an infinite one would admit every body.
    if (!Number.isSafeInteger(capacityBytes) || capacityBytes < 1) {
      throw new RangeError('capacityBytes must be a positive integer')
    }
    if (!Number.isInteger(maxQueued) || maxQueued < 0) {
      throw new RangeError('maxQueued must be a non-negative integer')
    }
  }

  get inFlightBytes(): number {
    return this.inFlight
  }

  get queued(): number {
    return this.waiters.length
  }

  /**
   * Reserve `bytes` of the budget. The returned release is idempotent. When
   * `signal` aborts while the caller is queued, the waiter is rejected and
   * removed, and the waiters behind it are reconsidered. A waiter still queued
   * at `deadlineAt` (epoch ms, the request's admission deadline, #739 D1) is
   * rejected and removed the same way, so a large body at the head of the
   * queue cannot hold back the smaller ones behind it past that instant.
   */
  async acquire(bytes: number, signal?: AbortSignal, deadlineAt?: number): Promise<() => void> {
    if (!Number.isInteger(bytes) || bytes < 0 || bytes > this.capacityBytes) {
      throw new RangeError(`a body of ${bytes} bytes cannot fit a budget of ${this.capacityBytes}`)
    }
    if (deadlineAt !== undefined && !Number.isFinite(deadlineAt)) {
      throw new RangeError(
        `a body admission deadline must be a finite epoch time, got ${deadlineAt}`
      )
    }
    if (signal?.aborted) throw new RequestLimitError('body admission was aborted', 'aborted')
    if (this.waiters.length === 0 && this.inFlight + bytes <= this.capacityBytes) {
      return this.take(bytes)
    }
    if (this.waiters.length >= this.maxQueued) {
      throw new RequestLimitError(
        'body admission queue is full',
        'queue_full',
        'proxy_capacity_exceeded'
      )
    }
    return new Promise<() => void>((resolve, reject) => {
      let deadline: ReturnType<typeof setTimeout> | undefined
      const leave = (reason: string, kind: RequestLimitKind) => {
        const index = this.waiters.indexOf(waiter)
        if (index === -1) return
        this.waiters.splice(index, 1)
        clearTimeout(deadline)
        signal?.removeEventListener('abort', onAbort)
        reject(new RequestLimitError(reason, kind))
        this.drain()
      }
      const onAbort = () => leave('body admission was aborted', 'aborted')
      const waiter: BodyWaiter = {
        bytes,
        grant: release => {
          clearTimeout(deadline)
          signal?.removeEventListener('abort', onAbort)
          resolve(release)
        },
      }
      signal?.addEventListener('abort', onAbort, { once: true })
      this.waiters.push(waiter)
      if (deadlineAt !== undefined) {
        deadline = setTimeout(
          () => leave('body admission wait exceeded', 'deadline'),
          Math.max(0, deadlineAt - Date.now())
        )
      }
    })
  }

  private take(bytes: number): () => void {
    this.inFlight += bytes
    let released = false
    return () => {
      if (released) return
      released = true
      this.inFlight -= bytes
      this.drain()
    }
  }

  /** Grant queued bodies in arrival order while the head of the queue fits. */
  private drain(): void {
    while (
      this.waiters.length > 0 &&
      this.inFlight + this.waiters[0]!.bytes <= this.capacityBytes
    ) {
      const waiter = this.waiters.shift()!
      waiter.grant(this.take(waiter.bytes))
    }
  }
}