Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
45.1 KB
Raw Blame
import express, {
  type Express,
  type NextFunction,
  type Request,
  type RequestHandler,
  type Response,
} from 'express'
import { rateLimit } from 'express-rate-limit'
import { type Server, createServer } from 'node:http'
import { Registry, collectDefaultMetrics } from 'prom-client'
import { z } from 'zod'
import {
  BODY_STRUCTURE_LIMITS,
  LIMITS,
  SCHEMA_VERSION_V2,
  createBodyStructureVerify,
  measureNonImageCompletionBytes,
  requestBodyLimitBytes,
} from '@clerum/grok-provider-attempt-contract'
import { verifyAdminPermit } from './auth/adminPermitVerifier.js'
import { isExpiredExecutionTicket, verifyExecutionTicket } from './auth/executionTicketVerifier.js'
import { type PlatformJwtClaims, verifyPlatformJwt } from './auth/platformJwtVerifier.js'
import type { GrokLlmProxyConfig } from './config.js'
import { ControlApiClient, ControlApiClientError, fetchCauseCode } from './controlApiClient.js'
import {
  GrokTransportError,
  type StreamGrokCompletionInput,
  UpstreamTimeoutError,
  listGrokModels,
  streamGrokCompletion,
  testGrokConnection,
} from './grokTransport.js'
import { logger } from './logger.js'
import { createProxyMetrics } from './metrics.js'
import {
  OriginDeniedError,
  type OriginPolicyOptions,
  defaultAddressLookup,
} from './originPolicy.js'
import {
  BODY_READ_DEADLINE_MS,
  BodyBudget,
  ENVELOPE_ALLOWANCE_BYTES,
  IN_FLIGHT_BODY_BUDGET_BODIES,
  RequestLimitError,
  STREAM_LIMITS,
  TicketLifeError,
  VISUAL_PER_HOST_MAX_ADMITTED,
  VISUAL_READ_CLOSE_GRACE_MS,
  streamGate,
  visualStreamGate,
} from './requestLimits.js'
import { startSseHeartbeat } from './sseHeartbeat.js'

type AdmittedRequest = Request & {
  /**
   * #739 D1 — the request's one admission clock, stamped by body admission:
   * arrival + `STREAM_LIMITS.maxQueueWaitMs`, in epoch ms. Every wait the
   * request performs (body budget, visual gate, stream gate) ends by then.
   */
  grokAdmissionDeadlineAt?: number
  /**
   * The arrival instant the admission clock was stamped from, in epoch ms.
   * The visual gate measures its own queue wait from it, so its refusal
   * identity does not depend on when the request reached the gate.
   */
  grokAdmissionArrivedAt?: number
  /**
   * #739 D2 — the body's budget reservation, set once it was granted. The
   * handler releases it when the upstream accepted the request; the
   * response's `close` event releases it on every other path. Idempotent.
   */
  grokBodyRelease?: () => void
}

type GatedRequest = AdmittedRequest & {
  /** Set by the platform gate before body admission (R9-M-B). */
  grokPlatform?: PlatformJwtClaims
  /**
   * The visual gate slot a body above the ordinary cap was read under. The
   * handler keeps it for the stream or releases it; every refusal releases it.
   */
  grokStreamRelease?: () => void
}

const COMPLETION_PATH = '/internal/runtime/v1/grok/completions'
const COMPLETION_KEYS = new Set(['executionTicket', 'requestHash', 'request', 'deadlineMs'])
const ADMIN_KEYS = new Set(['accessToken'])

const completionBodySchema = z
  .object({
    // A signed JWT of a few hundred bytes; the bound keeps an oversized
    // string from reaching jwt.verify.
    executionTicket: z.string().min(1).max(ENVELOPE_ALLOWANCE_BYTES),
    requestHash: z.string().regex(/^[a-f0-9]{64}$/),
    request: z.object({}).passthrough(),
    deadlineMs: z.number().int().positive().optional(),
  })
  .strict()

const adminBodySchema = z.object({ accessToken: z.string().min(1) }).strict()

function bearer(req: Request): string {
  const raw = String(req.header('authorization') || '')
  return raw.replace(/^bearer\s+/i, '').trim()
}

function contentLengthBytes(req: Request): number | null {
  const header = req.headers['content-length']
  if (typeof header !== 'string' || !/^[0-9]+$/.test(header)) return null
  const value = Number(header)
  return Number.isSafeInteger(value) ? value : null
}

function isRecord(value: unknown): value is Record<string, unknown> {
  return value !== null && typeof value === 'object' && !Array.isArray(value)
}

/**
 * `type` is the fixed error type of a refused body (body-parser's or the
 * structure scan's), never a message or any part of the body.
 */
function reject(res: Response, status: number, code: string, type?: string): void {
  if (res.headersSent) return
  logger.warn(
    type === undefined
      ? { event: 'grok_proxy_denied', code }
      : { event: 'grok_proxy_denied', code, type },
    'request denied'
  )
  res.status(status).json({ error: code })
}

// Every JSON parser scans the raw body before JSON.parse. JSON.parse
// allocates one heap object per container, so a body within the byte limit
// can still exhaust the heap; the scan refuses a body denser, more nested or
// with more containers than any request the contract accepts. Its depth bound
// (LIMITS.maxNestingDepth + 6) also keeps JSON.stringify and the contract's
// byte measurement from throwing RangeError on a deep body after parse.
const verifyBodyStructure = createBodyStructureVerify(BODY_STRUCTURE_LIMITS)

function boundedErrorHandler(err: unknown, _req: Request, res: Response, _next: () => void): void {
  const typed = err as { type?: string; status?: number; body?: unknown }
  // The refusals below share one response code per status, so the logged type
  // is what tells a structural 413 from a byte-limit 413.
  const type = typeof typed?.type === 'string' ? typed.type : undefined
  if (typed?.type === 'entity.too.large' || typed?.status === 413) {
    reject(res, 413, 'payload_too_large', type)
    return
  }
  // R9-1: the parsers run with `inflate: false`, so body-parser refuses an
  // encoded body before reading it.
  if (typed?.type === 'encoding.unsupported' || typed?.type === 'charset.unsupported') {
    reject(res, 415, 'unsupported_media_type', type)
    return
  }
  if (err instanceof SyntaxError || typed?.type === 'body.structure.too.deep') {
    reject(res, 400, 'invalid_request', type)
    return
  }
  // The client closed the connection before it sent the whole body. Nobody
  // reads a response, so none is written; the budget or visual slot is
  // released by the response's `close` hook.
  if (typed?.type === 'request.aborted') {
    logger.info({ event: 'grok_proxy_request_aborted', type }, 'client aborted the request body')
    return
  }
  // body-parser attaches the raw body to an error thrown by `verify` or by
  // JSON.parse. Such an error is logged by its type and status only.
  if (typed?.body !== undefined) {
    logger.error(
      { event: 'grok_proxy_error', type: typed.type, status: typed.status },
      'unhandled request error'
    )
  } else {
    logger.error({ event: 'grok_proxy_error', err }, 'unhandled request error')
  }
  reject(res, 500, 'internal_error')
}

/**
 * #731 R3-2 — memory-bounded admission around the body parser `parse`. It runs
 * after the caller's token was checked (R9-M-B), so an anonymous caller never
 * takes budget. A body is read only once its declared `Content-Length` fits
 * the shared byte budget, so the bodies in memory are bounded by bytes rather
 * than by the stream gate's request count. A `Transfer-Encoding` body is
 * refused with 411 instead of being read unbounded; a non-numeric
 * `Content-Length` is refused with 400. A request with neither header has no
 * body under HTTP/1.1 (RFC 9112 §6.3): `parse` reads nothing, and bytes sent
 * after its headers are parsed as the next request, which Node answers 400.
 * Bodies over the limit go to `parse` without a reservation: on the admin app
 * express.json answers 413 from the header without buffering them, and on the
 * runtime app `selectTransportBudget` either answers 413 or admits them
 * through the visual gate. A granted body must be read and parsed within
 * `readDeadlineMs` of the grant, or it is answered 408 `request_timeout`, its
 * reservation released and its connection closed. Otherwise the reservation is
 * held until the upstream accepted the completion (#739 D2: the handler
 * releases `req.grokBodyRelease` once the upstream fetch resolved, when the
 * body has been written), or until the response closes on every path that
 * never reaches the upstream. A queued waiter is dropped if the client
 * leaves first; a full queue gets the stream gate's overload response. #739
 * D1: the request's admission clock is stamped here, at arrival, and a waiter
 * still queued when it runs out gets the same overload response.
 */
function bodyAdmission(
  budget: BodyBudget,
  maxBodyBytes: number,
  readDeadlineMs: number,
  parse: RequestHandler
) {
  return (req: AdmittedRequest, res: Response, next: NextFunction): void => {
    const arrivedAt = Date.now()
    const admissionDeadlineAt = arrivedAt + STREAM_LIMITS.maxQueueWaitMs
    req.grokAdmissionArrivedAt = arrivedAt
    req.grokAdmissionDeadlineAt = admissionDeadlineAt
    if (req.headers['transfer-encoding'] !== undefined) {
      reject(res, 411, 'length_required')
      return
    }
    const declared = req.headers['content-length']
    if (declared === undefined) {
      parse(req, res, next)
      return
    }
    if (!/^\d+$/.test(declared)) {
      reject(res, 400, 'invalid_request')
      return
    }
    const bytes = Number(declared)
    if (bytes === 0 || bytes > maxBodyBytes) {
      parse(req, res, next)
      return
    }
    const abort = new AbortController()
    let release: (() => void) | undefined
    let readDeadline: ReturnType<typeof setTimeout> | undefined
    res.once('close', () => {
      clearTimeout(readDeadline)
      if (release) release()
      else abort.abort()
    })
    budget.acquire(bytes, abort.signal, admissionDeadlineAt).then(
      granted => {
        if (abort.signal.aborted) {
          granted()
          return
        }
        release = granted
        req.grokBodyRelease = granted
        let expired = false
        readDeadline = setTimeout(() => {
          expired = true
          granted()
          if (res.headersSent) return
          // The rest of the body is never read, so the connection cannot be reused.
          res.setHeader('connection', 'close')
          reject(res, 408, 'request_timeout')
        }, readDeadlineMs)
        parse(req, res, err => {
          clearTimeout(readDeadline)
          // The 408 already answered this request; the parser's late error
          // (the body aborted by the closed connection) has no one to reach.
          if (expired) return
          next(err)
        })
      },
      (err: unknown) => {
        if (!(err instanceof RequestLimitError)) {
          next(err)
          return
        }
        if (abort.signal.aborted) return
        // RequestLimitError messages are fixed strings with no request data.
        logger.warn(
          {
            event: 'grok_proxy_admission_refused',
            reason: 'body_budget',
            code: err.code,
            kind: err.kind,
            detail: err.message,
          },
          'admission refused'
        )
        reject(res, 503, err.code)
      }
    )
  }
}

export type ProxyRuntimeDeps = {
  controlApiClient?: ControlApiClient
  fetchFn?: typeof fetch
  /**
   * Test seam for the DNS half of the origin policy. Production always uses
   * `defaultAddressLookup`; hermetic e2e injects a resolver so the frozen
   * cli-chat-proxy.grok.com origin check runs without live DNS. The URL freeze in
   * `assertAllowedUpstreamUrl` is unaffected by this seam.
   */
  lookup?: OriginPolicyOptions['lookup']
  /** Test seam for the body-read deadline. Production uses `BODY_READ_DEADLINE_MS`. */
  bodyReadDeadlineMs?: number
  /** Test seam for the bounded visual-parser close backstop. */
  visualReadCloseGraceMs?: number
  /** Test seam: hang or observe a stream without contacting Grok. */
  streamCompletion?: typeof streamGrokCompletion
}

export type ProxyServers = {
  runtime: Server
  admin: Server
  probe: Server
  runtimeApp: Express
  adminApp: Express
  probeApp: Express
  close: () => Promise<void>
}

export function createProxyApps(
  config: GrokLlmProxyConfig,
  deps: ProxyRuntimeDeps = {}
): ProxyServers {
  const metricsRegistry = new Registry()
  collectDefaultMetrics({ register: metricsRegistry })
  const metrics = createProxyMetrics(metricsRegistry)
  const client =
    deps.controlApiClient ??
    new ControlApiClient({
      baseUrl: config.controlApiBaseUrl,
      serviceName: config.controlApiServiceName,
      serviceToken: config.controlApiServiceToken,
    })
  const fetchFn = deps.fetchFn ?? fetch
  const lookup = deps.lookup ?? defaultAddressLookup
  const runtimeRateLimit = rateLimit({
    windowMs: 60_000,
    limit: 60,
    standardHeaders: 'draft-7',
    legacyHeaders: false,
    // G1-2 (#720): the Host reads a JSON error code. The library sets
    // Retry-After and the draft-7 headers before it calls the handler.
    handler: (_req, res) => reject(res, 429, 'rate_limited'),
  })
  const adminRateLimit = rateLimit({
    windowMs: 60_000,
    limit: 30,
    standardHeaders: 'draft-7',
    legacyHeaders: false,
  })

  // One budget for both apps: every body this process reads counts against it.
  const bodyBudget = new BodyBudget(IN_FLIGHT_BODY_BUDGET_BODIES * config.maxBodyBytes)
  const bodyReadDeadlineMs = deps.bodyReadDeadlineMs ?? BODY_READ_DEADLINE_MS
  const visualReadCloseGraceMs = deps.visualReadCloseGraceMs ?? VISUAL_READ_CLOSE_GRACE_MS
  for (const [name, value] of Object.entries({ bodyReadDeadlineMs, visualReadCloseGraceMs })) {
    if (!Number.isSafeInteger(value) || value <= 0 || value > 2_147_483_647) {
      throw new RangeError(`${name} must be a positive signed-32-bit integer`)
    }
  }
  // Per-principal occupancy of the visual gate (running plus queued entries).
  // Keyed on the verified platform principal: sub plus sorted hostRefs.
  const visualPrincipalAdmissions = new Map<string, number>()
  const visualRequestOwners = new WeakSet<GatedRequest>()

  const runtimeApp = express()
  // R9-M-B: the token is checked from the header before any budget is taken or
  // any body byte is read, so an anonymous caller cannot hold a reservation.
  // It runs after the rate limiter, so it cannot be forced ahead of the limit.
  const platformGate = (req: GatedRequest, res: Response, next: NextFunction): void => {
    const token = bearer(req)
    if (verifyAdminPermit(token, config)) {
      reject(res, 403, 'insufficient_scope')
      return
    }
    const platform = verifyPlatformJwt(token, config)
    if (!platform) {
      reject(res, 401, 'Unauthorized')
      return
    }
    req.grokPlatform = platform
    next()
  }
  // R9-1: `inflate: false` on every parser. The budgets count the declared wire
  // length, so an encoded body is refused (415) instead of inflated past it.
  const ordinaryJson = express.json({
    limit: config.maxBodyBytes,
    inflate: false,
    verify: verifyBodyStructure,
  })
  const visualJson = express.json({
    limit: config.maxVisualBodyBytes,
    inflate: false,
    verify: verifyBodyStructure,
  })
  // The parser behind body admission. Every request here carries a verified
  // platform JWT.
  //
  // schemaVersion is not known until the body is parsed, so the visual gate
  // must not be taken for every platform JWT. Only a declared Content-Length
  // above the ordinary cap can be a visual envelope. Those bodies take the
  // 1-wide gate (the image bytes stay resident for the Grok stream). Every
  // smaller body, including every valid V1, stays on the ordinary parser and
  // later the 8-wide stream gate. A missing length cannot be upgraded: body
  // admission already refused a chunked body with 411, and a request with
  // neither header has no body, so only a declared length takes a visual slot.
  //
  // A granted slot is released exactly once. The handler hands it to the
  // stream; parser errors and timeouts release it only after body-parser's
  // callback proves that work has unwound. A response close starts a bounded
  // request-destruction backstop rather than releasing live parser ownership.
  // Like a budgeted body, a visual body must be read and parsed within
  // `bodyReadDeadlineMs` of the grant, or it is answered 408 `request_timeout`
  // with `connection: close`.
  const selectTransportBudget = (req: GatedRequest, res: Response, next: NextFunction): void => {
    const declared = contentLengthBytes(req)
    if (declared !== null && declared > config.maxVisualBodyBytes) {
      reject(res, 413, 'payload_too_large')
      return
    }
    if (declared === null || declared <= config.maxBodyBytes) {
      ordinaryJson(req, res, next)
      return
    }
    // One principal gets a fair share of the visual gate, so a
    // single host cannot fill the entries every other host also needs. The
    // share is taken before the gate wait, so queued entries count too.
    const platform = req.grokPlatform
    if (!platform) {
      // Fail closed: admission without verified claims must never fall back to
      // a shared per-principal key. platformGate runs before this middleware.
      reject(res, 401, 'Unauthorized')
      return
    }
    // bodyAdmission stamps both before it hands the request here.
    const arrivedAt = req.grokAdmissionArrivedAt
    const admissionDeadlineAt = req.grokAdmissionDeadlineAt
    if (arrivedAt === undefined || admissionDeadlineAt === undefined) {
      next(new Error('visual admission was reached without body admission'))
      return
    }
    const visualPrincipal = JSON.stringify([platform.sub, ...[...platform.hostRefs].sort()])
    const admittedByPrincipal = visualPrincipalAdmissions.get(visualPrincipal) ?? 0
    if (admittedByPrincipal >= VISUAL_PER_HOST_MAX_ADMITTED) {
      logger.warn(
        {
          event: 'grok_proxy_admission_refused',
          reason: 'visual_host_share',
          limit: VISUAL_PER_HOST_MAX_ADMITTED,
          sub: platform.sub,
          hostRefs: [...platform.hostRefs].sort(),
        },
        'admission refused'
      )
      reject(res, 503, 'visual_host_share')
      return
    }
    visualPrincipalAdmissions.set(visualPrincipal, admittedByPrincipal + 1)
    void (async () => {
      let release: (() => void) | undefined
      let principalShareHeld = true
      const releasePrincipalShare = (): void => {
        if (!principalShareHeld) return
        principalShareHeld = false
        const held = visualPrincipalAdmissions.get(visualPrincipal)
        if (held === undefined) {
          // A held share always has an entry, so a missing one is a counting
          // defect. This runs from `close` handlers, where a throw has no
          // caller to reach, so it is logged and the release completes.
          logger.error(
            { event: 'grok_proxy_error', reason: 'principal_share_missing', sub: platform.sub },
            'visual principal share missing at release'
          )
          visualPrincipalAdmissions.delete(visualPrincipal)
          return
        }
        const remaining = held - 1
        if (remaining > 0) visualPrincipalAdmissions.set(visualPrincipal, remaining)
        else visualPrincipalAdmissions.delete(visualPrincipal)
      }
      const parseAbort = new AbortController()
      const abortParse = (): void => parseAbort.abort()
      req.once('aborted', abortParse)
      try {
        // The visual gate's own wait is the admission clock: it is measured
        // from arrival, so a waiter that waits it out is refused as
        // `visual_gate` however many ms passed before it reached the gate.
        release = await visualStreamGate.acquire(parseAbort.signal, admissionDeadlineAt, arrivedAt)
      } catch (err) {
        req.off('aborted', abortParse)
        releasePrincipalShare()
        // The client left while queued: nobody reads a refusal, and a departed
        // client is not gate saturation, so it is not logged as `visual_gate`.
        // Same rule as the ordinary path's `!abort.signal.aborted`.
        if (err instanceof RequestLimitError && err.kind === 'aborted') return
        if (err instanceof RequestLimitError) {
          logger.warn(
            {
              event: 'grok_proxy_admission_refused',
              reason: 'visual_gate',
              code: err.code,
              detail: err.message,
            },
            'admission refused'
          )
          reject(res, 503, err.code)
          return
        }
        next(err)
        return
      }
      req.off('aborted', abortParse)
      const releaseVisual = (): void => {
        releasePrincipalShare()
        release?.()
        release = undefined
        req.grokStreamRelease = undefined
      }
      req.grokStreamRelease = releaseVisual
      let cancelled = false
      let parserSettled = false
      let closeGrace: ReturnType<typeof setTimeout> | undefined
      const cancelParser = (): void => {
        if (parserSettled || closeGrace !== undefined) return
        closeGrace = setTimeout(() => {
          if (!parserSettled) req.destroy()
        }, visualReadCloseGraceMs)
      }
      const readDeadline = setTimeout(() => {
        cancelled = true
        cancelParser()
        if (res.headersSent) return
        // The rest of the body is never read, so the connection cannot be reused.
        res.setHeader('connection', 'close')
        reject(res, 408, 'request_timeout')
      }, bodyReadDeadlineMs)
      const onResponseClose = (): void => {
        cancelled = true
        clearTimeout(readDeadline)
        cancelParser()
      }
      res.once('close', onResponseClose)
      visualJson(req, res, err => {
        parserSettled = true
        clearTimeout(readDeadline)
        if (closeGrace !== undefined) clearTimeout(closeGrace)
        res.off('close', onResponseClose)
        if (cancelled || err) {
          req.body = undefined
          if (err !== null && typeof err === 'object' && 'body' in err) {
            delete (err as { body?: unknown }).body
          }
        }
        // Capacity remains charged until body-parser has actually unwound. On
        // success, `req.grokStreamRelease` transfers ownership to the handler,
        // which releases it in its outer `finally`; refusals release here.
        if (cancelled || err) releaseVisual()
        if (cancelled) return
        if (err) {
          next(err)
          return
        }
        res.once('close', () => {
          // A synchronous handler throw can happen before its async owner
          // installs cancellation and a finally block. Once that owner has
          // taken over, response close must not release its live work.
          if (!visualRequestOwners.has(req)) releaseVisual()
        })
        next()
      })
    })()
  }
  // Order: rate limit, token, body admission around the transport budget,
  // handler.
  const runtimeAdmission = bodyAdmission(
    bodyBudget,
    config.maxBodyBytes,
    bodyReadDeadlineMs,
    selectTransportBudget
  )
  runtimeApp.post(COMPLETION_PATH, runtimeRateLimit, platformGate, runtimeAdmission, (req, res) => {
    const gated = req as GatedRequest
    const releaseAdmission = (): void => {
      gated.grokStreamRelease?.()
      gated.grokStreamRelease = undefined
    }
    const platform = gated.grokPlatform
    if (!platform) {
      releaseAdmission()
      throw new Error('the completion route was reached without the platform gate')
    }
    const admissionDeadlineAt = gated.grokAdmissionDeadlineAt
    if (admissionDeadlineAt === undefined) {
      releaseAdmission()
      throw new Error('the completion route was reached without body admission')
    }
    if (!req.is('application/json')) {
      releaseAdmission()
      reject(res, 415, 'unsupported_media_type')
      return
    }
    const request = isRecord(req.body) ? req.body.request : undefined
    const visualDeclared = isRecord(request) && request.schemaVersion === SCHEMA_VERSION_V2
    const configuredLimit = visualDeclared ? config.maxVisualBodyBytes : config.maxBodyBytes
    const wholeBodyBytes = Buffer.byteLength(JSON.stringify(req.body ?? {}), 'utf8')
    // A V1 envelope carries the ticket, the hash and the deadline beside a
    // request that may itself sit at the contract cap (#731), so its limit adds
    // the envelope allowance. A V2 envelope stays on the contract's visual
    // ceiling, the same one buildGrokProxyEnvelope enforces.
    const envelopeLimit = visualDeclared
      ? requestBodyLimitBytes(request)
      : requestBodyLimitBytes(request) + ENVELOPE_ALLOWANCE_BYTES
    // Declaring V2 raises only the image budget. Text, tools and wrapper
    // fields stay on the contract's maxRequestBodyBytes non-image ceiling, plus
    // the same envelope allowance control-api's authorizer grants its wrapper.
    // The min stays: `createProxyApps` accepts a config that did not pass
    // `loadConfig` (bodyAdmission.test.ts builds one with a 16 KiB body limit).
    if (
      wholeBodyBytes > Math.min(configuredLimit, envelopeLimit) ||
      measureNonImageCompletionBytes(req.body) >
        LIMITS.maxRequestBodyBytes + ENVELOPE_ALLOWANCE_BYTES
    ) {
      releaseAdmission()
      reject(res, 413, 'payload_too_large')
      return
    }
    const extra = Object.keys(req.body ?? {}).find(key => !COMPLETION_KEYS.has(key))
    if (extra) {
      releaseAdmission()
      reject(res, 400, 'unknown_field')
      return
    }
    const parsed = completionBodySchema.safeParse(req.body)
    if (!parsed.success) {
      releaseAdmission()
      reject(res, 400, 'invalid_request')
      return
    }
    if (parsed.data.deadlineMs !== undefined && parsed.data.deadlineMs > config.maxDeadlineMs) {
      releaseAdmission()
      reject(res, 400, 'invalid_request')
      return
    }
    const ticket = verifyExecutionTicket(parsed.data.executionTicket, config)
    if (!ticket) {
      releaseAdmission()
      // R17-2: body admission can wait up to maxQueueWaitMs, the ticket TTL,
      // before this read, so an honest ticket can arrive expired. The Host
      // retries ticket_expired with a fresh authorization.
      if (isExpiredExecutionTicket(parsed.data.executionTicket, config)) {
        reject(res, 403, 'ticket_expired')
      } else {
        reject(res, 403, 'ticket_invalid')
      }
      return
    }
    if (platform.hostRefs.includes('*') || !platform.hostRefs.includes(ticket.hostRef)) {
      releaseAdmission()
      reject(res, 403, 'host_binding_mismatch')
      return
    }
    if (!config.executionEnabled) {
      releaseAdmission()
      reject(res, 404, 'disabled')
      return
    }

    void (async () => {
      let release: (() => void) | undefined
      const abort = new AbortController()
      // After express.json() the incoming request is already complete. Listening
      // to req 'close' aborts the Grok hop on every call (3–12ms canceled).
      // Abort only when the client drops the response before we finish writing.
      abortWhenClientDisconnects(req, res, abort)
      visualRequestOwners.add(gated)
      // One `grok_proxy_attempt_finished` line per attempt. Identifiers and
      // counts only: never the body, ticket, frames, tool names or arguments.
      const attempt = {
        providerAttemptId: ticket.providerAttemptId,
        hostRef: ticket.hostRef,
        model: ticket.model,
        requestHash: ticket.requestHash,
      }
      const attemptStarted = Date.now()
      let toolCalls = 0
      let textChunks = 0
      let heartbeats = 0
      // Set once the redeem succeeded. Every exit path below calls it before
      // it ends the response: a tick after `res.end()` would write after end.
      let stopHeartbeat: (() => void) | undefined
      try {
        // A visual slot exists only when Content-Length exceeded the ordinary
        // cap. Keep it for the Grok stream only when the parsed V2 body still
        // exceeds that cap (image bytes stay resident). Padding, V1, and small
        // V2 release it and take the 8-wide gate.
        // #739 D1-bis: every admission wait also ends when the ticket dies. A
        // request still queued at `exp` could only be redeemed into
        // ticket_expired, so it is refused here, before any redeem. The margin
        // is zero: a ticket still alive when a slot frees is served as before.
        const deadlineAt = Math.min(admissionDeadlineAt, ticket.expiresAtMs)
        const refuseTicketLife = (): never => {
          logger.warn(
            {
              event: 'grok_proxy_admission_refused',
              reason: 'ticket_life',
              providerAttemptId: ticket.providerAttemptId,
              hostRef: ticket.hostRef,
            },
            'admission refused'
          )
          throw new TicketLifeError()
        }
        const waitWithinTicketLife = async (
          acquire: () => Promise<() => void>
        ): Promise<() => void> => {
          if (deadlineAt <= Date.now()) {
            if (ticket.expiresAtMs <= admissionDeadlineAt) refuseTicketLife()
            throw new RequestLimitError('admission deadline exceeded', 'deadline')
          }
          try {
            return await acquire()
          } catch (err) {
            // The wait ended on its own deadline, and that deadline was the
            // ticket's expiry: comparing clocks here misses a timer that fired
            // a millisecond early.
            if (
              err instanceof RequestLimitError &&
              err.kind === 'deadline' &&
              !abort.signal.aborted &&
              ticket.expiresAtMs <= admissionDeadlineAt
            ) {
              refuseTicketLife()
            }
            throw err
          }
        }
        if (visualDeclared && wholeBodyBytes > config.maxBodyBytes) {
          release = gated.grokStreamRelease
          gated.grokStreamRelease = undefined
        } else if (gated.grokStreamRelease) {
          // A demoted body was read through the visual gate, so it holds
          // no body-budget reservation. It takes one for its compact size
          // before it gives up the visual slot, so the 8-wide stream gate
          // never holds more resident bodies than the budget admits. Every
          // request takes its locks in the same order (visual, budget,
          // stream), so no two requests wait on each other.
          try {
            gated.grokBodyRelease = await waitWithinTicketLife(() =>
              bodyBudget.acquire(wholeBodyBytes, abort.signal, deadlineAt)
            )
          } finally {
            releaseAdmission()
          }
        } else {
          releaseAdmission()
        }
        if (!release) {
          release = await waitWithinTicketLife(() => streamGate.acquire(abort.signal, deadlineAt))
        }
        res.status(200)
        res.setHeader('content-type', 'text/event-stream')
        res.setHeader('cache-control', 'no-cache')
        const started = Date.now()
        const stream = deps.streamCompletion ?? streamGrokCompletion
        const streamInput: StreamGrokCompletionInput = {
          executionTicket: parsed.data.executionTicket,
          requestHash: parsed.data.requestHash,
          request: parsed.data.request,
          deadlineMs: parsed.data.deadlineMs,
          maxDeadlineMs: Math.min(config.maxDeadlineMs, config.maxStreamDurationMs),
          upstreamIdleTimeoutMs: config.upstreamIdleTimeoutMs,
          ticket: {
            jti: ticket.jti,
            hostRef: ticket.hostRef,
            model: ticket.model,
            requestHash: ticket.requestHash,
            providerAttemptId: ticket.providerAttemptId,
          },
          signal: abort.signal,
          redeem: input => client.redeem(input),
          onRedeemed: () => {
            stopHeartbeat = startSseHeartbeat(res, abort.signal, config.heartbeatIntervalMs, () => {
              heartbeats += 1
            })
            abort.signal.addEventListener('abort', stopHeartbeat, { once: true })
          },
          // #739 D2: the body has been written upstream, so its budget
          // reservation ends here rather than with the stream. Both the raw
          // body and the parsed copy are dropped below (#806 Q1).
          onUpstreamAccepted: () => {
            gated.grokBodyRelease?.()
            gated.grokBodyRelease = undefined
            // Every reference to the parsed tree ends with the reservation: a
            // live stream keeps this frame, `parsed` and `streamInput` for as
            // long as the upstream answers, and an 8-wide gate of dense trees
            // would otherwise outlast the budget that admitted them.
            req.body = undefined
            parsed.data.request = {}
            streamInput.request = undefined
          },
          finalize: input => client.finalize(input),
          fetchFn,
          lookup,
          onFrame: frame => {
            if (frame.type === 'tool_call') toolCalls += 1
            else textChunks += 1
            return writeSseChunk(res, `data: ${JSON.stringify(frame)}\n\n`, abort.signal)
          },
        }
        const result = await stream(streamInput)
        stopHeartbeat?.()
        res.write(
          `data: ${JSON.stringify({ type: 'done', outcome: result.outcome, ...(result.usage ? { usage: result.usage } : {}) })}\n\n`
        )
        metrics.observeAttempt(result.outcome, 'completion_stream')
        metrics.observeStream(Date.now() - started)
        const finished = {
          event: 'grok_proxy_attempt_finished',
          ...attempt,
          outcome: result.outcome,
          deliveredAs: 'sse_done',
          toolCalls,
          textChunks,
          heartbeats,
          durationMs: Date.now() - attemptStarted,
          ...(result.usage ? { usage: result.usage } : {}),
        }
        if (result.outcome === 'success') logger.info(finished, 'grok attempt finished')
        else logger.warn(finished, 'grok attempt finished')
        res.end()
      } catch (err) {
        stopHeartbeat?.()
        const mapped = mapError(err)
        metrics.observeAttempt('error', 'completion_stream')
        // Queue capacity has a terminal local wire code; the request_limit
        // metric also includes abort/deadline kinds that keep their existing
        // wire semantics. Ticket expiry remains provider_unavailable.
        metrics.observeAttemptFailure(
          err instanceof RequestLimitError && !(err instanceof TicketLifeError)
            ? 'request_limit'
            : failureLabel(mapped.code)
        )
        if (err instanceof UpstreamTimeoutError) metrics.observeUpstreamTimeout(err.kind)
        const deliveredAs = res.headersSent ? 'sse_error' : 'http_status'
        const causeCode = err instanceof ControlApiClientError ? err.causeCode : fetchCauseCode(err)
        logger.warn(
          {
            event: 'grok_proxy_attempt_finished',
            ...attempt,
            outcome: 'failed',
            code: mapped.code,
            // An invalid_request message is the contract parser's, which
            // names caller-supplied fields; the code alone is logged for it.
            ...(err instanceof GrokTransportError && err.code !== 'invalid_request'
              ? { reason: err.message, ...(err.details ? { details: err.details } : {}) }
              : {}),
            // RequestLimitError messages are fixed strings with no request data.
            ...(err instanceof RequestLimitError ? { reason: err.message } : {}),
            // An error no class above maps is a handler defect answered 503:
            // the error itself is logged, or nothing would show it.
            ...(isMappedError(err) ? {} : { err }),
            // G1-4: a failed fetch's cause code only, never its message or URL.
            ...(causeCode ? { causeCode } : {}),
            deliveredAs,
            ...(deliveredAs === 'http_status' ? { httpStatus: mapped.status } : {}),
            toolCalls,
            textChunks,
            heartbeats,
            durationMs: Date.now() - attemptStarted,
          },
          'grok attempt finished'
        )
        // R1-H2: the upstream status of an upstream_rejected travels on both
        // paths, so the Host can tell an entitlement refusal from a 404.
        const upstreamStatus =
          err instanceof GrokTransportError && err.code === 'upstream_rejected'
            ? err.details?.upstreamStatus
            : undefined
        const rejectedBy = typeof upstreamStatus === 'number' ? { upstreamStatus } : {}
        if (deliveredAs === 'sse_error') {
          res.write(
            `data: ${JSON.stringify({ type: 'error', code: mapped.code, ...rejectedBy })}\n\n`
          )
          res.end()
          return
        }
        // Nothing was written yet, so the staged SSE headers can still be
        // replaced; res.json() keeps an existing content-type.
        res.removeHeader('cache-control')
        res.setHeader('content-type', 'application/json; charset=utf-8')
        // G1-1: an upstream Retry-After reaches the Host only on this path; an
        // SSE error frame carries the code and, for upstream_rejected, the
        // upstream status, never Retry-After.
        const retryAfter =
          err instanceof GrokTransportError ? err.details?.retryAfterSeconds : undefined
        if (typeof retryAfter === 'number') res.setHeader('retry-after', String(retryAfter))
        res.status(mapped.status).json({ error: mapped.code, ...rejectedBy })
      } finally {
        stopHeartbeat?.()
        release?.()
        // Defence in depth for an early throw after this async owner took over
        // but before the visual release was moved into `release`.
        gated.grokStreamRelease?.()
        gated.grokStreamRelease = undefined
        // A demoted body's reservation, when the stream ended before the body
        // was written upstream. Budgeted bodies are also released on close.
        gated.grokBodyRelease?.()
        gated.grokBodyRelease = undefined
      }
    })()
  })
  runtimeApp.use((_req, res) => reject(res, 404, 'not_found'))
  runtimeApp.use(boundedErrorHandler)

  const adminApp = express()
  // R9-M-B: the admin permit is checked before any budget is taken, exactly as
  // the platform JWT is on the runtime app.
  const adminGate =
    (operation: 'catalog_list' | 'connection_test') =>
    (req: Request, res: Response, next: NextFunction): void => {
      const token = bearer(req)
      if (verifyExecutionTicket(token, config)) {
        reject(res, 403, 'insufficient_scope')
        return
      }
      if (!verifyAdminPermit(token, config, operation)) {
        reject(res, 401, 'Unauthorized')
        return
      }
      next()
    }
  const adminAdmission = bodyAdmission(
    bodyBudget,
    config.maxBodyBytes,
    bodyReadDeadlineMs,
    express.json({ limit: config.maxBodyBytes, inflate: false, verify: verifyBodyStructure })
  )
  const adminHandler = (kind: 'models' | 'test') => (req: Request, res: Response) => {
    if (!req.is('application/json')) {
      reject(res, 415, 'unsupported_media_type')
      return
    }
    if (!config.executionEnabled) {
      reject(res, 404, 'disabled')
      return
    }
    const extra = Object.keys(req.body ?? {}).find(key => !ADMIN_KEYS.has(key))
    if (extra) {
      reject(res, 400, 'unknown_field')
      return
    }
    const parsed = adminBodySchema.safeParse(req.body)
    if (!parsed.success) {
      reject(res, 400, 'invalid_request')
      return
    }
    void (async () => {
      try {
        if (kind === 'models') {
          const listed = await listGrokModels({
            accessToken: parsed.data.accessToken,
            fetchFn,
            lookup,
          })
          res.status(200).json(listed)
          return
        }
        const tested = await testGrokConnection({
          accessToken: parsed.data.accessToken,
          fetchFn,
          lookup,
        })
        res.status(200).json(tested)
      } catch (err) {
        const mapped = mapError(err)
        reject(res, mapped.status, mapped.code)
      }
    })()
  }
  adminApp.post(
    '/internal/admin/v1/grok/models',
    adminRateLimit,
    adminGate('catalog_list'),
    adminAdmission,
    adminHandler('models')
  )
  adminApp.post(
    '/internal/admin/v1/grok/test',
    adminRateLimit,
    adminGate('connection_test'),
    adminAdmission,
    adminHandler('test')
  )
  adminApp.use((_req, res) => reject(res, 404, 'not_found'))
  adminApp.use(boundedErrorHandler)

  const probeApp = express()
  probeApp.get('/healthz', (_req, res) => res.status(200).json({ ok: true }))
  probeApp.get('/readyz', (_req, res) => res.status(200).json({ ok: true }))
  probeApp.get('/metrics', async (_req, res) => {
    res.set('content-type', metricsRegistry.contentType)
    res.status(200).send(await metricsRegistry.metrics())
  })
  probeApp.use((_req, res) => reject(res, 404, 'not_found'))

  const runtime = createServer(runtimeApp)
  const admin = createServer(adminApp)
  const probe = createServer(probeApp)

  return {
    runtime,
    admin,
    probe,
    runtimeApp,
    adminApp,
    probeApp,
    close: async () => {
      await Promise.all([runtime, admin, probe].map(server => closeServer(server)))
    },
  }
}

export function startProxy(config: GrokLlmProxyConfig): ProxyServers {
  const servers = createProxyApps(config)
  servers.runtime.listen(config.runtimePort)
  servers.admin.listen(config.adminPort)
  servers.probe.listen(config.probePort)
  logger.info(
    {
      event: 'grok_proxy_listen',
      runtimePort: config.runtimePort,
      adminPort: config.adminPort,
      probePort: config.probePort,
    },
    'grok-llm-proxy listeners ready'
  )
  return servers
}

// Every code the proxy or control-api is known to send. The failure metric
// uses this as its label allowlist because a control-api error body is not
// bounded by the proxy.
const ATTEMPT_ERROR_STATUS: Record<string, number> = {
  invalid_request: 400,
  // The upstream's refusal of a prompt over the model's context window, at the
  // upstream's own status class (R10).
  context_length_exceeded: 400,
  // A contract `size` refusal before redeem (a byte budget, or the container,
  // member or element bound), or a body over the proxy's own caps.
  payload_too_large: 413,
  Unauthorized: 401,
  origin_denied: 403,
  request_hash_mismatch: 403,
  ticket_invalid: 403,
  ticket_expired: 403,
  no_grant: 403,
  host_binding_mismatch: 403,
  model_not_allowed: 403,
  insufficient_scope: 403,
  disabled: 404,
  // A reserved body that was not read within BODY_READ_DEADLINE_MS (R9-M-B).
  request_timeout: 408,
  ticket_replayed: 409,
  // An upstream 429 (G1-1, #720); its Retry-After travels as a header.
  rate_limited: 429,
  tool_call_limit_exceeded: 422,
  // An upstream 4xx the same request would get again (G1-3, #720). Not 502:
  // the gateways answer 502 when nothing behind them answered (R1-H2).
  upstream_rejected: 422,
  // A redeem no control-plane process answered (G1-4, #720); control-api may
  // still have received it when the gateway's 502 mapping produced the code.
  control_plane_unavailable: 503,
  tool_call_arguments_exceeded: 422,
  invalid_tool_arguments: 422,
  client_upgrade_required: 426,
  // The attempt ran for its whole stream budget. Retrying the same request
  // would spend the same budget again, so it is a gateway timeout, not 503.
  stream_duration_exceeded: 504,
  connection_unavailable: 503,
  visual_host_share: 503,
  visual_gate: 503,
  proxy_capacity_exceeded: 503,
  provider_unavailable: 503,
  sse_buffer_exceeded: 503,
  invalid_receipt: 503,
  conflict: 503,
}

// A control-api code is an unbounded string, so both readers of the table must
// ignore inherited names: `ATTEMPT_ERROR_STATUS['constructor']` is a function,
// not undefined, and `??` would pass it straight to `res.status()`.
function attemptErrorStatus(code: string): number {
  return Object.hasOwn(ATTEMPT_ERROR_STATUS, code) ? ATTEMPT_ERROR_STATUS[code]! : 503
}

function failureLabel(code: string): string {
  return Object.hasOwn(ATTEMPT_ERROR_STATUS, code) ? code : 'other'
}

type MappedError =
  | OriginDeniedError
  | RequestLimitError
  | GrokTransportError
  | ControlApiClientError

/**
 * True for the errors `mapError` answers with a status of their own. Anything
 * else is a defect in the handler; it is answered 503 like an outage, so the
 * caller logs the error itself to keep it visible.
 */
function isMappedError(err: unknown): err is MappedError {
  return (
    err instanceof OriginDeniedError ||
    err instanceof RequestLimitError ||
    err instanceof GrokTransportError ||
    err instanceof ControlApiClientError
  )
}

function mapError(err: unknown): { status: number; code: string } {
  if (!isMappedError(err)) return { status: 503, code: 'provider_unavailable' }
  if (err instanceof OriginDeniedError) return { status: 403, code: 'origin_denied' }
  return { status: attemptErrorStatus(err.code), code: err.code }
}

/**
 * Write one SSE chunk and honor socket backpressure. Returns undefined when
 * the chunk was accepted (hot path stays synchronous); otherwise a promise that
 * settles on 'drain', or when the client closes / the stream aborts so a
 * departed consumer never pins the stream slot.
 */
export function writeSseChunk(
  res: Response,
  chunk: string,
  signal: AbortSignal
): Promise<void> | undefined {
  if (res.write(chunk)) return undefined
  return new Promise<void>(resolve => {
    const done = () => {
      res.off('drain', done)
      res.off('close', done)
      signal.removeEventListener('abort', done)
      resolve()
    }
    res.on('drain', done)
    res.on('close', done)
    signal.addEventListener('abort', done, { once: true })
    if (signal.aborted || res.destroyed) done()
  })
}

export function abortWhenClientDisconnects(
  req: Request,
  res: Response,
  abort: AbortController
): void {
  const cancel = () => {
    if (!res.writableEnded) abort.abort()
  }
  res.on('close', cancel)
  req.on('aborted', cancel)
}

function closeServer(server: Server): Promise<void> {
  return new Promise(resolve => {
    server.close(() => resolve())
  })
}