Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
44.4 KB
Raw Blame
import { Router } from 'express'
import type { NextFunction, Request, Response } from 'express'
import { PROVIDER_AUTH_MODE } from '@clerum/llm-providers'
import { config } from '../../config.js'
import { pool } from '../../db.js'
import { asyncHandler } from '../../http/asyncHandler.js'
import type { K8sGateway } from '../../k8s.js'
import { deriveOAuthEncryptionKey } from '../../oauth/encryption.js'
import { rootLogger } from '../../observability/logger.js'
import { llmAllowlistConfigMapWriteFailuresTotal } from '../../observability/metrics.js'
import {
  type CodexCatalogTransport,
  createCodexCatalogTransportFromEnv,
  listCodexCatalogModels,
  listOfferedCodexModelsForAssignment,
  pickCodexGrantModel,
  setCodexCatalogModelEnabled,
} from '../../services/codexSubscriptionCatalog.js'
import {
  CODEX_UNASSIGNED_CONNECTION_KEY,
  CodexSubscriptionInvalidConnectionKeyError,
  assertCodexConnectionKey,
  createNamedCodexSubscriptionConnection,
  generateCodexConnectionKey,
  getSafeCodexSubscriptionConnection,
  listSafeCodexSubscriptionConnections,
  normalizeCodexConnectionKey,
  updateCodexSubscriptionConnectionMetadata,
} from '../../services/codexSubscriptionConnection.js'
import {
  type CodexOAuthDeps,
  CodexSubscriptionOAuthError,
  getCodexSubscriptionConnection,
  isCodexOAuthErrorCode,
  pollCodexDevice,
  refreshCodexSubscriptionConnection,
  revokeCodexSubscription,
  runCodexCatalogSync,
  startCodexBrowserConnect,
  startCodexDeviceConnect,
} from '../../services/codexSubscriptionOAuth.js'
import type { CodexSubscriptionOAuthIntent } from '../../services/codexSubscriptionOAuthState.js'
import {
  buildCodexBrowserRedirectUri,
  isPublicCodexCliClient,
  resolveCodexControlUiBaseUrl,
} from '../../services/codexSubscriptionRedirectUri.js'
import {
  type GrokCatalogTransport,
  createGrokCatalogTransportFromEnv,
  listGrokCatalogModels,
  listOfferedGrokModelsForAssignment,
  pickGrokGrantModel,
  setGrokCatalogModelEnabled,
} from '../../services/grokSubscriptionCatalog.js'
import {
  GROK_UNASSIGNED_CONNECTION_KEY,
  GrokSubscriptionConnectionKeyConflictError,
  GrokSubscriptionFingerprintConflictError,
  GrokSubscriptionInvalidConnectionKeyError,
  GrokSubscriptionStaleRevisionError,
  assertGrokConnectionKey,
  createNamedGrokSubscriptionConnection,
  generateGrokConnectionKey,
  getSafeGrokSubscriptionConnection,
  listSafeGrokSubscriptionConnections,
  updateGrokSubscriptionConnectionMetadata,
} from '../../services/grokSubscriptionConnection.js'
import {
  type GrokOAuthDeps,
  GrokSubscriptionOAuthError,
  getGrokSubscriptionConnection,
  isGrokOAuthErrorCode,
  pollGrokDevice,
  refreshGrokSubscriptionConnection,
  revokeGrokSubscription,
  runGrokCatalogSync,
  startGrokDeviceConnect,
} from '../../services/grokSubscriptionOAuth.js'
import {
  publishAllowedModelsConfigMapAfterGrantChange,
  syncOutcomeChangedTheRow,
} from '../../services/llmAllowedModelsConfigMap.js'
import { K8sConflictError } from '../../services/resourceService.js'
import {
  adminSubscriptionReadRateLimits,
  adminSubscriptionWriteRateLimits,
} from '../workflows/shared/rateLimit.js'
import { createHostValidationDeps, validateHostSpec } from './hostSpecValidation.js'
import {
  type HostSpecIncoherenceToleratedEvent,
  emitHostSpecIncoherenceTolerated,
} from './hostWriteGateAudit.js'
import type { StaleModelWarning } from './staleModelWarning.js'

const log = rootLogger.child({ module: 'admin-codex-subscription' })
export const SUBSCRIPTION_ADMIN_BASE = '/admin/llm/providers/:providerId'
const BASE = SUBSCRIPTION_ADMIN_BASE

type SubscriptionBrokerId = 'codex-subscription' | 'grok-subscription'

const subscriptionBrokerByReq = new WeakMap<object, SubscriptionBrokerId>()

function bindKnownSubscriptionBroker(req: object, providerId: string): SubscriptionBrokerId | null {
  if (providerId === 'codex-subscription') {
    subscriptionBrokerByReq.set(req, 'codex-subscription')
    return 'codex-subscription'
  }
  if (providerId === 'grok-subscription') {
    subscriptionBrokerByReq.set(req, 'grok-subscription')
    return 'grok-subscription'
  }
  return null
}

function isGrokReq(req: object): boolean {
  return subscriptionBrokerByReq.get(req) === 'grok-subscription'
}

export function mountSubscriptionAdminRoutes(
  router: Router,
  options: { providerIds: readonly string[]; enabled: (providerId: string) => boolean }
): void {
  router.use(SUBSCRIPTION_ADMIN_BASE, (req: Request, res: Response, next: NextFunction) => {
    const raw = typeof req.params.providerId === 'string' ? req.params.providerId : ''
    const broker = bindKnownSubscriptionBroker(req, raw)
    if (!broker || PROVIDER_AUTH_MODE[broker] !== 'oauth-broker') {
      res.status(404).json({ error: 'not_found' })
      return
    }
    if (!options.providerIds.includes(broker)) {
      res.status(404).json({ error: 'not_found' })
      return
    }
    if (!options.enabled(broker)) {
      res.status(404).json({ error: 'disabled' })
      return
    }
    next()
  })
}

function dbClient() {
  return { query: (text: string, values?: unknown[]) => pool.query(text, values) }
}

function resolveBrowserRedirectUri(req: { get: (h: string) => string | undefined }): string {
  const controlUiOrigin = resolveCodexControlUiBaseUrl(config.controlUiBaseUrl, req.get('origin'))
  return buildCodexBrowserRedirectUri(controlUiOrigin)
}

function oauthDeps(
  req: {
    get: (h: string) => string | undefined
  },
  redirectUri = resolveBrowserRedirectUri(req),
  connectionKey?: string
): CodexOAuthDeps {
  return {
    db: dbClient(),
    encryptionKey: deriveOAuthEncryptionKey(config.oauthEncryptionKey),
    fetchFn: fetch,
    clientId: config.codexOAuthClientId,
    redirectUri,
    enabled: config.codexSubscriptionEnabled,
    connectionKey: normalizeCodexConnectionKey(connectionKey),
  }
}

function keyFromReq(req: { params?: { key?: string } }): string {
  const raw = typeof req.params?.key === 'string' ? req.params.key.trim() : ''
  if (!raw) return normalizeCodexConnectionKey(raw)
  return assertCodexConnectionKey(raw)
}

function parseIntent(value: unknown): CodexSubscriptionOAuthIntent {
  return value === 'reconnect' || value === 'replace' ? value : 'connect'
}

function sendOAuthError(
  res: { status: (n: number) => { json: (body: unknown) => void } },
  err: unknown
) {
  if (err instanceof CodexSubscriptionInvalidConnectionKeyError) {
    res.status(400).json({ error: 'invalid_connection_key' })
    return
  }
  if (err instanceof GrokSubscriptionInvalidConnectionKeyError) {
    res.status(400).json({ error: 'invalid_connection_key' })
    return
  }
  // Grok connection-layer write races must never surface as unmapped 500s.
  if (err instanceof GrokSubscriptionConnectionKeyConflictError) {
    log.warn({ event: 'grok_oauth_admin_denied', code: err.code }, 'Grok connection key conflict')
    res.status(409).json({ error: 'connection_key_taken' })
    return
  }
  if (err instanceof GrokSubscriptionFingerprintConflictError) {
    log.warn({ event: 'grok_oauth_admin_denied', code: err.code }, 'Grok fingerprint conflict')
    res.status(409).json({ error: 'fingerprint_in_use' })
    return
  }
  if (err instanceof GrokSubscriptionStaleRevisionError) {
    log.warn({ event: 'grok_oauth_admin_denied', code: err.code }, 'Grok stale revision')
    res.status(409).json({ error: 'stale_revision' })
    return
  }
  if (err instanceof GrokSubscriptionOAuthError) {
    const status =
      err.code === 'disabled'
        ? 404
        : err.code === 'replacement_required' ||
            err.code === 'fingerprint_in_use' ||
            err.code === 'connection_mismatch'
          ? 409
          : err.code === 'refresh_in_flight' || err.code === 'stale_revision'
            ? 409
            : err.code === 'not_connected' || err.code === 'no_grant'
              ? 404
              : 400
    log.warn({ event: 'grok_oauth_admin_denied', code: err.code }, 'Grok subscription OAuth denied')
    res.status(status).json({ error: err.code })
    return
  }
  if (err instanceof CodexSubscriptionOAuthError) {
    const status =
      err.code === 'disabled'
        ? 404
        : err.code === 'replacement_required' ||
            err.code === 'fingerprint_in_use' ||
            err.code === 'connection_mismatch'
          ? 409
          : err.code === 'refresh_in_flight' || err.code === 'stale_revision'
            ? 409
            : err.code === 'not_connected' || err.code === 'no_grant'
              ? 404
              : 400
    log.warn(
      { event: 'codex_oauth_admin_denied', code: err.code },
      'Codex subscription OAuth denied'
    )
    res.status(status).json({ error: err.code })
    return
  }
  throw err
}

type HostRecord = {
  metadata?: { name?: string; resourceVersion?: string }
  spec?: unknown
}

type CodexAssignableHostRow = {
  name: string
  connectionRef: string
  displayName: string
  provider?: string
  model?: string
}

function hostModel(
  spec: unknown
): { provider?: string; connectionRef?: string; name?: string } | null {
  if (!spec || typeof spec !== 'object') return null
  const model = (spec as { model?: { provider?: string; connectionRef?: string; name?: string } })
    .model
  if (!model || typeof model !== 'object') return null
  return model
}

function hostModelName(spec: unknown): string {
  const name = hostModel(spec)?.name
  return typeof name === 'string' ? name.trim() : ''
}

function hostConnectionRef(spec: unknown, provider = 'codex-subscription'): string | null {
  const model = hostModel(spec)
  if (!model || model.provider !== provider) return null
  const raw = typeof model.connectionRef === 'string' ? model.connectionRef.trim() : ''
  if (provider === 'grok-subscription') return raw || GROK_UNASSIGNED_CONNECTION_KEY
  return raw || CODEX_UNASSIGNED_CONNECTION_KEY
}

function assignableHostFromRecord(host: HostRecord): CodexAssignableHostRow | null {
  const name = String(host.metadata?.name ?? '').trim()
  if (!name) return null
  const model = hostModel(host.spec)
  const provider = typeof model?.provider === 'string' ? model.provider.trim() : ''
  const modelName = hostModelName(host.spec)
  const connectionRef =
    provider === 'codex-subscription' || provider === 'grok-subscription'
      ? hostConnectionRef(host.spec, provider) ||
        (provider === 'grok-subscription'
          ? GROK_UNASSIGNED_CONNECTION_KEY
          : CODEX_UNASSIGNED_CONNECTION_KEY)
      : CODEX_UNASSIGNED_CONNECTION_KEY
  return {
    name,
    connectionRef,
    displayName: name,
    ...(provider ? { provider } : {}),
    ...(modelName ? { model: modelName } : {}),
  }
}

function storedCodexConnectionRef(spec: unknown): string | null {
  const model = hostModel(spec)
  if (!model || model.provider !== 'codex-subscription') return null
  return typeof model.connectionRef === 'string' ? model.connectionRef.trim() : ''
}

function storedGrokConnectionRef(spec: unknown): string | null {
  const model = hostModel(spec)
  if (!model || model.provider !== 'grok-subscription') return null
  return typeof model.connectionRef === 'string' ? model.connectionRef.trim() : ''
}

export function createAdminCodexSubscriptionRouter(
  catalogTransport: CodexCatalogTransport = createCodexCatalogTransportFromEnv(),
  gateway?: K8sGateway
): Router {
  const router = Router()
  mountSubscriptionAdminRoutes(router, {
    providerIds: ['codex-subscription', 'grok-subscription'],
    enabled: providerId =>
      providerId === 'codex-subscription'
        ? config.codexSubscriptionEnabled
        : providerId === 'grok-subscription'
          ? config.grokSubscriptionEnabled
          : false,
  })

  const grokCatalogTransport: GrokCatalogTransport = createGrokCatalogTransportFromEnv()

  function rejectGrokUnkeyed(req: Request, res: Response, next: NextFunction): void {
    if (isGrokReq(req)) {
      res.status(404).json({ error: 'not_found' })
      return
    }
    next()
  }

  function grokOauthDeps(connectionKey: string): GrokOAuthDeps {
    return {
      db: dbClient(),
      encryptionKey: deriveOAuthEncryptionKey(config.oauthEncryptionKey),
      fetchFn: fetch,
      clientId: config.grokOAuthClientId,
      enabled: config.grokSubscriptionEnabled,
      connectionKey,
    }
  }

  /**
   * Migration 0113_grok_subscription_terminal_connection_key archives
   * superseded tombstones as `<key>~revoked~<id>`.
   * Those keys sit outside the key grammar, so no keyed route can address
   * them; listing them would only surface unactionable duplicates. The key's
   * own terminal tombstone (status `revoked`, `revokedAt` set) stays listed.
   */
  function isAddressableGrokConnection(row: { connectionKey: string }): boolean {
    try {
      assertGrokConnectionKey(row.connectionKey)
      return true
    } catch {
      return false
    }
  }

  function grokKeyFromReq(req: { params?: { key?: string } }): string {
    const raw = typeof req.params?.key === 'string' ? req.params.key.trim() : ''
    return assertGrokConnectionKey(raw)
  }

  async function listHostsOrUnavailable(): Promise<HostRecord[] | null> {
    if (!gateway) return null
    try {
      return (await gateway.listResource('hosts', config.hostsNamespace)) as HostRecord[]
    } catch (err) {
      log.warn({ event: 'codex_assigned_hosts_list_failed', err }, 'failed to list assigned hosts')
      return null
    }
  }

  function hostsForConnection(
    hosts: HostRecord[],
    connectionKey: string,
    provider = 'codex-subscription'
  ): Array<{ name: string }> {
    return hosts
      .filter(host => hostConnectionRef(host.spec, provider) === connectionKey)
      .map(host => ({ name: String(host.metadata?.name ?? '') }))
      .filter(host => host.name)
  }

  function collectAssignableHosts(hosts: HostRecord[]): CodexAssignableHostRow[] {
    return hosts
      .map(assignableHostFromRecord)
      .filter((row): row is CodexAssignableHostRow => Boolean(row))
  }

  async function loadHost(
    hostRef: string
  ): Promise<{ ok: true; host: HostRecord } | { ok: false; status: number; error: string }> {
    if (!gateway) return { ok: false, status: 503, error: 'hosts_unavailable' }
    try {
      const host = (await gateway.getResource(
        'hosts',
        hostRef,
        config.hostsNamespace
      )) as HostRecord
      return { ok: true, host }
    } catch (err) {
      const name = err instanceof Error ? err.name : ''
      const status = (err as { httpStatus?: number; code?: number } | null)?.httpStatus
      if (name === 'K8sNotFoundError' || status === 404) {
        return { ok: false, status: 404, error: 'host_not_found' }
      }
      log.warn(
        { event: 'codex_host_read_failed', err, hostRef },
        'failed to read host for bind/unbind'
      )
      return { ok: false, status: 503, error: 'hosts_unavailable' }
    }
  }

  async function writeHostConnectionRef(
    hostRef: string,
    host: HostRecord,
    nextConnectionRef: string,
    provider: 'codex-subscription' | 'grok-subscription' = 'codex-subscription'
  ): Promise<
    | { ok: true; warnings: StaleModelWarning[]; model: string }
    | { ok: false; status: number; error: string; message?: string; reason?: string }
  > {
    if (!gateway) return { ok: false, status: 503, error: 'hosts_unavailable' }
    const spec =
      host.spec && typeof host.spec === 'object'
        ? { ...(host.spec as Record<string, unknown>) }
        : {}
    const model =
      spec.model && typeof spec.model === 'object'
        ? { ...(spec.model as Record<string, unknown>) }
        : {}
    const grok = provider === 'grok-subscription'
    model.provider = grok ? 'grok-subscription' : 'codex-subscription'
    model.connectionRef = nextConnectionRef
    let resolvedModel = typeof model.name === 'string' ? model.name.trim() : ''
    const unassigned = grok ? GROK_UNASSIGNED_CONNECTION_KEY : CODEX_UNASSIGNED_CONNECTION_KEY
    if (nextConnectionRef !== unassigned) {
      const offered = grok
        ? await listOfferedGrokModelsForAssignment(dbClient(), nextConnectionRef)
        : await listOfferedCodexModelsForAssignment(dbClient(), nextConnectionRef)
      if (offered.length === 0) {
        return {
          ok: false,
          status: 422,
          error: 'catalog_not_ready',
          message:
            'This subscription has no offered models yet. Sign in and sync the catalog before assigning agents.',
        }
      }
      if (grok) {
        const grant = await getSafeGrokSubscriptionConnection(dbClient(), nextConnectionRef)
        resolvedModel = pickGrokGrantModel(resolvedModel, offered, grant?.defaultModel)
      } else {
        const grant = await getSafeCodexSubscriptionConnection(dbClient(), nextConnectionRef)
        resolvedModel = pickCodexGrantModel(resolvedModel, offered, grant?.defaultModel)
      }
      model.name = resolvedModel
    }
    spec.model = model
    const stored =
      host.spec && typeof host.spec === 'object'
        ? (host.spec as Record<string, unknown>)
        : undefined
    const hostRefId = { namespace: config.hostsNamespace, name: hostRef }
    const hostTolerations: HostSpecIncoherenceToleratedEvent[] = []
    const hostWarnings: StaleModelWarning[] = []
    const issue = await validateHostSpec(spec, createHostValidationDeps(pool), {
      stored,
      hostRef: hostRefId,
      ...(stored !== undefined ? { tolerations: hostTolerations, warnings: hostWarnings } : {}),
    })
    if (issue) {
      return {
        ok: false,
        status: 422,
        error: 'invalid_host_spec',
        message: issue.errors[0]?.message,
      }
    }
    try {
      await gateway.updateResource(
        'hosts',
        hostRef,
        {
          metadata: host.metadata?.resourceVersion
            ? { resourceVersion: host.metadata.resourceVersion }
            : undefined,
          spec,
        },
        config.hostsNamespace
      )
      for (const event of hostTolerations) emitHostSpecIncoherenceTolerated(event)
      return { ok: true, warnings: hostWarnings, model: resolvedModel }
    } catch (err) {
      if (err instanceof K8sConflictError) {
        return { ok: false, status: 409, error: 'conflict', reason: 'resource_changed' }
      }
      log.warn(
        { event: 'codex_host_write_failed', err, hostRef },
        'failed to write host connectionRef'
      )
      return { ok: false, status: 503, error: 'hosts_unavailable' }
    }
  }

  async function publishRuntimeAllowlist(): Promise<void> {
    await publishAllowedModelsConfigMapAfterGrantChange(gateway?.llmAllowedModelsConfigMap())
  }

  async function publishRuntimeAllowlistOrFail(res: {
    status: (n: number) => { json: (body: unknown) => void }
  }): Promise<boolean> {
    try {
      await publishRuntimeAllowlist()
      return false
    } catch (err) {
      llmAllowlistConfigMapWriteFailuresTotal.inc({ phase: 'mutation' })
      log.error(
        { err, event: 'codex_allowed_models_cm_write_failed' },
        'Codex grant change saved but the runtime ConfigMap was not updated'
      )
      res.status(503).json({
        error: 'configmap_write_failed',
        message:
          'the grant change was saved but the ConfigMap could not be updated; runtime hosts still see the previous snapshot — retry or it will reconcile on the next change/boot',
      })
      return true
    }
  }

  async function withAssignedHosts<T extends { connectionKey: string }>(
    connection: T,
    hosts?: HostRecord[] | null,
    provider = 'codex-subscription'
  ) {
    const resolved = hosts === undefined ? await listHostsOrUnavailable() : hosts
    if (resolved === null) {
      return { ...connection, assignedHostsUnavailable: true as const }
    }
    return {
      ...connection,
      assignedHosts: hostsForConnection(resolved, connection.connectionKey, provider),
    }
  }

  const getConnectionHandler = asyncHandler(async (req, res) => {
    try {
      if (isGrokReq(req)) {
        const connection = await getGrokSubscriptionConnection(grokOauthDeps(grokKeyFromReq(req)))
        res.status(200).json(await withAssignedHosts(connection, undefined, 'grok-subscription'))
        return
      }
      const connection = await getCodexSubscriptionConnection(
        oauthDeps(req, resolveBrowserRedirectUri(req), keyFromReq(req))
      )
      res.status(200).json(await withAssignedHosts(connection))
    } catch (err) {
      sendOAuthError(res, err)
    }
  })

  const browserStartHandler = asyncHandler(async (req, res) => {
    try {
      const redirectUri = resolveBrowserRedirectUri(req)
      if (isPublicCodexCliClient(config.codexOAuthClientId)) {
        throw new CodexSubscriptionOAuthError(
          'browser_oauth_unregistered',
          'browser OAuth requires a deployment-registered OpenAI client'
        )
      }
      const started = await startCodexBrowserConnect(
        oauthDeps(req, redirectUri, keyFromReq(req)),
        parseIntent(req.body?.intent)
      )
      res.status(200).json(started)
    } catch (err) {
      sendOAuthError(res, err)
    }
  })

  const deviceStartHandler = asyncHandler(async (req, res) => {
    try {
      if (isGrokReq(req)) {
        const started = await startGrokDeviceConnect(
          grokOauthDeps(grokKeyFromReq(req)),
          parseIntent(req.body?.intent)
        )
        res.status(200).json(started)
        return
      }
      const started = await startCodexDeviceConnect(
        oauthDeps(req, resolveBrowserRedirectUri(req), keyFromReq(req)),
        parseIntent(req.body?.intent)
      )
      res.status(200).json(started)
    } catch (err) {
      sendOAuthError(res, err)
    }
  })

  const devicePollHandler = asyncHandler(async (req, res) => {
    try {
      const state = typeof req.query.state === 'string' ? req.query.state : ''
      if (isGrokReq(req)) {
        const connectionKey = grokKeyFromReq(req)
        const deps = grokOauthDeps(connectionKey)
        let result = await pollGrokDevice(deps, state)
        if (result.status === 'connected') {
          const sync = await runGrokCatalogSync(
            deps,
            result.connection.connectionKey,
            grokCatalogTransport
          )
          if (sync.ok) {
            const latest = await getGrokSubscriptionConnection(deps)
            if ('id' in latest) result = { ...result, connection: latest }
          } else if ('connection' in result) {
            result = {
              ...result,
              connection: { ...result.connection, catalogStatus: sync.catalogStatus },
            }
          }
          if (await publishRuntimeAllowlistOrFail(res)) return
        }
        res.status(200).json(result)
        return
      }
      const connectionKey = keyFromReq(req)
      const deps = oauthDeps(req, resolveBrowserRedirectUri(req), connectionKey)
      let result = await pollCodexDevice(deps, state)
      if (result.status === 'connected') {
        const sync = await runCodexCatalogSync(
          deps,
          result.connection.connectionKey,
          catalogTransport
        )
        if (sync.connection) {
          result = { ...result, connection: sync.connection }
        } else if (!sync.ok) {
          result = {
            ...result,
            connection: { ...result.connection, catalogStatus: sync.catalogStatus },
          }
        }
        if (await publishRuntimeAllowlistOrFail(res)) return
      }
      res.status(200).json(result)
    } catch (err) {
      sendOAuthError(res, err)
    }
  })

  const refreshHandler = asyncHandler(async (req, res) => {
    try {
      if (isGrokReq(req)) {
        const connection = await refreshGrokSubscriptionConnection(
          grokOauthDeps(grokKeyFromReq(req))
        )
        if (await publishRuntimeAllowlistOrFail(res)) return
        res.status(200).json(connection)
        return
      }
      const connection = await refreshCodexSubscriptionConnection(
        oauthDeps(req, resolveBrowserRedirectUri(req), keyFromReq(req))
      )
      if (await publishRuntimeAllowlistOrFail(res)) return
      res.status(200).json(connection)
    } catch (err) {
      // A rejected refresh token moves the row to `reauth_required` and only
      // then throws. mcp-host and HCC read the ConfigMap, not Postgres, so the
      // publish comes before the error response, as on the catalog-sync route.
      if (
        (err instanceof CodexSubscriptionOAuthError || err instanceof GrokSubscriptionOAuthError) &&
        err.persistedConnectionStatus
      ) {
        if (await publishRuntimeAllowlistOrFail(res)) return
      }
      sendOAuthError(res, err)
    }
  })

  const catalogSyncHandler = asyncHandler(async (req, res) => {
    try {
      if (isGrokReq(req)) {
        if (!config.grokSubscriptionEnabled) {
          res.status(404).json({ error: 'disabled' })
          return
        }
        const connectionKey = grokKeyFromReq(req)
        const synced = await runGrokCatalogSync(
          grokOauthDeps(connectionKey),
          connectionKey,
          grokCatalogTransport
        )
        if (synced.ok) {
          if (await publishRuntimeAllowlistOrFail(res)) return
          res.status(200).json({
            outcome: synced.catalogStatus,
            connection: synced.connection,
          })
          return
        }
        // The publish comes BEFORE the error responses, because a failed sync
        // can still have written the connection row — a rejected refresh token
        // moves it to `reauth_required` and only then throws, which surfaces
        // here as `never_synced` plus `persisted`. mcp-host and HCC never read
        // Postgres, so an unpublished write leaves the runtime serving a grant
        // the control plane already knows is broken, whatever status code this
        // handler picks afterwards. `syncOutcomeChangedTheRow` is the same rule
        // the reconciliation cron applies.
        if (syncOutcomeChangedTheRow(synced)) {
          if (await publishRuntimeAllowlistOrFail(res)) return
        }
        if (synced.reason === 'no_grant' || synced.reason === 'disabled') {
          res.status(404).json({ error: synced.reason })
          return
        }
        if (synced.reason === 'stale_revision') {
          res.status(409).json({ error: 'stale_revision' })
          return
        }
        if (isGrokOAuthErrorCode(synced.reason ?? '')) {
          sendOAuthError(
            res,
            new GrokSubscriptionOAuthError(synced.reason as never, synced.reason ?? '')
          )
          return
        }
        res.status(503).json({ error: 'catalog_sync_failed', outcome: synced.catalogStatus })
        return
      }
      if (!config.codexSubscriptionEnabled) {
        res.status(404).json({ error: 'disabled' })
        return
      }
      const connectionKey = keyFromReq(req)
      const synced = await runCodexCatalogSync(
        oauthDeps(req, resolveBrowserRedirectUri(req), connectionKey),
        connectionKey,
        catalogTransport
      )
      if (synced.ok) {
        if (await publishRuntimeAllowlistOrFail(res)) return
        res.status(200).json({
          outcome: synced.catalogStatus,
          added: synced.added,
          refreshed: synced.refreshed,
          staled: synced.staled,
          connection: synced.connection,
        })
        return
      }
      // Same rule and same order as the Grok branch above: a rejected refresh
      // token moves the row to `reauth_required` before the sync fails, so the
      // publish comes before any error response.
      if (syncOutcomeChangedTheRow(synced)) {
        if (await publishRuntimeAllowlistOrFail(res)) return
      }
      if (synced.reason === 'no_grant' || synced.reason === 'disabled') {
        res.status(404).json({ error: synced.reason })
        return
      }
      if (synced.reason === 'stale_revision') {
        res.status(409).json({ error: 'stale_revision' })
        return
      }
      if (isCodexOAuthErrorCode(synced.reason)) {
        sendOAuthError(res, new CodexSubscriptionOAuthError(synced.reason, synced.reason))
        return
      }
      res.status(503).json({
        error: 'catalog_sync_failed',
        // Report what the sync reported. Rewriting `never_synced` as
        // `unavailable` maps "nothing was persisted" onto a status that means
        // "we reached the vendor and it was down", and the Hub renders this
        // string to the operator verbatim.
        outcome: synced.catalogStatus,
        added: synced.added ?? 0,
        refreshed: synced.refreshed ?? 0,
        staled: synced.staled ?? 0,
        connection: synced.connection ?? null,
      })
    } catch (err) {
      sendOAuthError(res, err)
    }
  })

  const revokeHandler = asyncHandler(async (req, res) => {
    try {
      if (isGrokReq(req)) {
        const connection = await revokeGrokSubscription(grokOauthDeps(grokKeyFromReq(req)))
        if (await publishRuntimeAllowlistOrFail(res)) return
        res.status(200).json(await withAssignedHosts(connection, undefined, 'grok-subscription'))
        return
      }
      const connection = await revokeCodexSubscription(
        oauthDeps(req, resolveBrowserRedirectUri(req), keyFromReq(req))
      )
      if (await publishRuntimeAllowlistOrFail(res)) return
      res.status(200).json(await withAssignedHosts(connection))
    } catch (err) {
      sendOAuthError(res, err)
    }
  })

  router.get(
    `${BASE}/connection`,
    rejectGrokUnkeyed,
    ...adminSubscriptionReadRateLimits(),
    getConnectionHandler
  )
  router.post(
    `${BASE}/browser/start`,
    rejectGrokUnkeyed,
    ...adminSubscriptionWriteRateLimits(),
    browserStartHandler
  )
  router.post(
    `${BASE}/device/start`,
    rejectGrokUnkeyed,
    ...adminSubscriptionWriteRateLimits(),
    deviceStartHandler
  )
  router.get(
    `${BASE}/device/poll`,
    rejectGrokUnkeyed,
    ...adminSubscriptionReadRateLimits(),
    devicePollHandler
  )
  router.post(
    `${BASE}/refresh`,
    rejectGrokUnkeyed,
    ...adminSubscriptionWriteRateLimits(),
    refreshHandler
  )
  router.post(
    `${BASE}/catalog/sync`,
    rejectGrokUnkeyed,
    ...adminSubscriptionWriteRateLimits(),
    catalogSyncHandler
  )
  router.post(
    `${BASE}/revoke`,
    rejectGrokUnkeyed,
    ...adminSubscriptionWriteRateLimits(),
    revokeHandler
  )
  router.get(`${BASE}/connections/:key`, ...adminSubscriptionReadRateLimits(), getConnectionHandler)
  router.post(
    `${BASE}/connections/:key/browser/start`,
    rejectGrokUnkeyed,
    ...adminSubscriptionWriteRateLimits(),
    browserStartHandler
  )
  router.post(
    `${BASE}/connections/:key/device/start`,
    ...adminSubscriptionWriteRateLimits(),
    deviceStartHandler
  )
  router.get(
    `${BASE}/connections/:key/device/poll`,
    ...adminSubscriptionReadRateLimits(),
    devicePollHandler
  )
  router.post(
    `${BASE}/connections/:key/refresh`,
    ...adminSubscriptionWriteRateLimits(),
    refreshHandler
  )
  router.post(
    `${BASE}/connections/:key/catalog/sync`,
    ...adminSubscriptionWriteRateLimits(),
    catalogSyncHandler
  )
  router.post(
    `${BASE}/connections/:key/revoke`,
    ...adminSubscriptionWriteRateLimits(),
    revokeHandler
  )

  router.get(
    `${BASE}/connections`,
    ...adminSubscriptionReadRateLimits(),
    asyncHandler(async (req, res) => {
      if (isGrokReq(req)) {
        if (!config.grokSubscriptionEnabled) {
          res.status(404).json({ error: 'disabled' })
          return
        }
        const rows = (await listSafeGrokSubscriptionConnections(dbClient())).filter(
          isAddressableGrokConnection
        )
        const hosts = await listHostsOrUnavailable()
        const connections = []
        for (const row of rows) {
          connections.push(await withAssignedHosts(row, hosts, 'grok-subscription'))
        }
        if (hosts === null) {
          res.status(200).json({ connections, assignableHostsUnavailable: true })
          return
        }
        res.status(200).json({
          connections,
          assignableHosts: collectAssignableHosts(hosts),
        })
        return
      }
      if (!config.codexSubscriptionEnabled) {
        res.status(404).json({ error: 'disabled' })
        return
      }
      const rows = await listSafeCodexSubscriptionConnections(dbClient())
      const hosts = await listHostsOrUnavailable()
      const connections = []
      for (const row of rows) {
        connections.push(await withAssignedHosts(row, hosts))
      }
      if (hosts === null) {
        res.status(200).json({ connections, assignableHostsUnavailable: true })
        return
      }
      res.status(200).json({
        connections,
        assignableHosts: collectAssignableHosts(hosts),
      })
    })
  )

  router.post(
    `${BASE}/connections`,
    ...adminSubscriptionWriteRateLimits(),
    asyncHandler(async (req, res) => {
      if (isGrokReq(req)) {
        if (!config.grokSubscriptionEnabled) {
          res.status(404).json({ error: 'disabled' })
          return
        }
        try {
          const displayName =
            typeof req.body?.displayName === 'string' && req.body.displayName.trim()
              ? req.body.displayName.trim()
              : 'Grok subscription'
          if (displayName.length > 64) {
            res.status(400).json({ error: 'display_name_too_long' })
            return
          }
          const requestedKey =
            typeof req.body?.connectionKey === 'string' && req.body.connectionKey.trim()
              ? assertGrokConnectionKey(req.body.connectionKey.trim())
              : generateGrokConnectionKey()
          const created = await createNamedGrokSubscriptionConnection(dbClient(), {
            connectionKey: requestedKey,
            displayName,
          })
          res.status(201).json(await withAssignedHosts(created, undefined, 'grok-subscription'))
        } catch (err) {
          if (err instanceof GrokSubscriptionInvalidConnectionKeyError) {
            res.status(400).json({ error: 'invalid_connection_key' })
            return
          }
          const code = (err as { code?: string } | null)?.code
          // A revoked key is a terminal tombstone (0113): recreating it conflicts.
          if (code === '23505') {
            res.status(409).json({ error: 'connection_key_taken' })
            return
          }
          throw err
        }
        return
      }
      if (!config.codexSubscriptionEnabled) {
        res.status(404).json({ error: 'disabled' })
        return
      }
      try {
        const displayName =
          typeof req.body?.displayName === 'string' && req.body.displayName.trim()
            ? req.body.displayName.trim()
            : 'Codex subscription'
        if (displayName.length > 64) {
          res.status(400).json({ error: 'display_name_too_long' })
          return
        }
        const requestedKey =
          typeof req.body?.connectionKey === 'string' && req.body.connectionKey.trim()
            ? assertCodexConnectionKey(req.body.connectionKey.trim())
            : generateCodexConnectionKey()
        const created = await createNamedCodexSubscriptionConnection(dbClient(), {
          connectionKey: requestedKey,
          displayName,
        })
        res.status(201).json(await withAssignedHosts(created))
      } catch (err) {
        if (err instanceof CodexSubscriptionInvalidConnectionKeyError) {
          res.status(400).json({ error: 'invalid_connection_key' })
          return
        }
        const code = (err as { code?: string } | null)?.code
        if (code === '23505') {
          res.status(409).json({ error: 'connection_key_taken' })
          return
        }
        throw err
      }
    })
  )

  router.get(
    `${BASE}/connections/:key/models`,
    ...adminSubscriptionReadRateLimits(),
    asyncHandler(async (req, res) => {
      if (isGrokReq(req)) {
        if (!config.grokSubscriptionEnabled) {
          res.status(404).json({ error: 'disabled' })
          return
        }
        const connection = await getGrokSubscriptionConnection(grokOauthDeps(grokKeyFromReq(req)))
        if (!('id' in connection)) {
          res.status(409).json({ error: 'not_connected' })
          return
        }
        const models = await listGrokCatalogModels(dbClient(), connection.id)
        res.status(200).json({ models })
        return
      }
      if (!config.codexSubscriptionEnabled) {
        res.status(404).json({ error: 'disabled' })
        return
      }
      const connection = await getCodexSubscriptionConnection(
        oauthDeps(req, resolveBrowserRedirectUri(req), keyFromReq(req))
      )
      if (!('id' in connection)) {
        res.status(409).json({ error: 'not_connected' })
        return
      }
      const models = await listCodexCatalogModels(dbClient(), connection.id)
      res.status(200).json({ models })
    })
  )

  router.patch(
    `${BASE}/connections/:key`,
    ...adminSubscriptionWriteRateLimits(),
    asyncHandler(async (req, res) => {
      const grok = isGrokReq(req)
      if (grok ? !config.grokSubscriptionEnabled : !config.codexSubscriptionEnabled) {
        res.status(404).json({ error: 'disabled' })
        return
      }
      let connectionKey: string
      try {
        connectionKey = grok ? grokKeyFromReq(req) : keyFromReq(req)
      } catch (err) {
        sendOAuthError(res, err)
        return
      }
      const hasDisplayName = typeof req.body?.displayName === 'string'
      const hasDefaultModel = Object.prototype.hasOwnProperty.call(req.body ?? {}, 'defaultModel')
      if (!hasDisplayName && !hasDefaultModel) {
        res.status(400).json({ error: 'empty_patch' })
        return
      }
      if (hasDisplayName && req.body.displayName.trim().length > 64) {
        res.status(400).json({ error: 'display_name_too_long' })
        return
      }
      const defaultModel = hasDefaultModel
        ? typeof req.body.defaultModel === 'string'
          ? req.body.defaultModel.trim()
          : ''
        : undefined
      if (defaultModel) {
        const offered = grok
          ? await listOfferedGrokModelsForAssignment(dbClient(), connectionKey)
          : await listOfferedCodexModelsForAssignment(dbClient(), connectionKey)
        if (!offered.includes(defaultModel)) {
          res.status(422).json({ error: 'default_model_not_offered' })
          return
        }
      }
      const updated = grok
        ? await updateGrokSubscriptionConnectionMetadata(dbClient(), connectionKey, {
            ...(hasDisplayName ? { displayName: req.body.displayName } : {}),
            ...(hasDefaultModel ? { defaultModel: defaultModel || null } : {}),
          })
        : await updateCodexSubscriptionConnectionMetadata(dbClient(), connectionKey, {
            ...(hasDisplayName ? { displayName: req.body.displayName } : {}),
            ...(hasDefaultModel ? { defaultModel: defaultModel || null } : {}),
          })
      if (!updated) {
        res.status(404).json({ error: 'no_grant' })
        return
      }
      res
        .status(200)
        .json(
          await withAssignedHosts(
            updated,
            undefined,
            grok ? 'grok-subscription' : 'codex-subscription'
          )
        )
    })
  )

  router.patch(
    `${BASE}/connections/:key/models/:model`,
    ...adminSubscriptionWriteRateLimits(),
    asyncHandler(async (req, res) => {
      const grok = isGrokReq(req)
      if (grok ? !config.grokSubscriptionEnabled : !config.codexSubscriptionEnabled) {
        res.status(404).json({ error: 'disabled' })
        return
      }
      let connectionKey: string
      try {
        connectionKey = grok ? grokKeyFromReq(req) : keyFromReq(req)
      } catch (err) {
        sendOAuthError(res, err)
        return
      }
      if (typeof req.body?.enabled !== 'boolean') {
        res.status(400).json({ error: 'invalid_enabled' })
        return
      }
      const connection = grok
        ? await getSafeGrokSubscriptionConnection(dbClient(), connectionKey)
        : await getSafeCodexSubscriptionConnection(dbClient(), connectionKey)
      if (!connection) {
        res.status(404).json({ error: 'no_grant' })
        return
      }
      const model = typeof req.params.model === 'string' ? req.params.model.trim() : ''
      if (!model) {
        res.status(404).json({ error: 'model_not_found' })
        return
      }
      const models = grok
        ? await setGrokCatalogModelEnabled(dbClient(), connection.id, model, req.body.enabled)
        : await setCodexCatalogModelEnabled(dbClient(), connection.id, model, req.body.enabled)
      if (!models) {
        res.status(404).json({ error: 'model_not_found' })
        return
      }
      if (await publishRuntimeAllowlistOrFail(res)) return
      res.status(200).json({ models })
    })
  )

  router.get(
    `${BASE}/assignable-hosts`,
    ...adminSubscriptionReadRateLimits(),
    asyncHandler(async (req, res) => {
      if (isGrokReq(req) ? !config.grokSubscriptionEnabled : !config.codexSubscriptionEnabled) {
        res.status(404).json({ error: 'disabled' })
        return
      }
      const hosts = await listHostsOrUnavailable()
      if (hosts === null) {
        res.status(503).json({ error: 'hosts_unavailable' })
        return
      }
      res.status(200).json({ hosts: collectAssignableHosts(hosts) })
    })
  )

  const bindUnbindHandler = (action: 'bind' | 'unbind') =>
    asyncHandler(async (req, res) => {
      const grok = isGrokReq(req)
      if (grok ? !config.grokSubscriptionEnabled : !config.codexSubscriptionEnabled) {
        res.status(404).json({ error: 'disabled' })
        return
      }
      let connectionKey: string
      try {
        connectionKey = grok
          ? assertGrokConnectionKey(typeof req.params.key === 'string' ? req.params.key : '')
          : assertCodexConnectionKey(typeof req.params.key === 'string' ? req.params.key : '')
      } catch (err) {
        sendOAuthError(res, err)
        return
      }
      const hostRef = typeof req.params.hostRef === 'string' ? req.params.hostRef.trim() : ''
      if (!hostRef) {
        res.status(400).json({ error: 'invalid_host_ref' })
        return
      }

      const loaded = await loadHost(hostRef)
      if (!loaded.ok) {
        res.status(loaded.status).json({ error: loaded.error })
        return
      }
      const storedRef = grok
        ? storedGrokConnectionRef(loaded.host.spec)
        : storedCodexConnectionRef(loaded.host.spec)
      if (storedRef === null && action === 'unbind') {
        res.status(409).json({ error: grok ? 'not_grok_host' : 'not_codex_host' })
        return
      }
      const unassigned = grok ? GROK_UNASSIGNED_CONNECTION_KEY : CODEX_UNASSIGNED_CONNECTION_KEY
      const provider = grok ? ('grok-subscription' as const) : ('codex-subscription' as const)
      // Empty/missing connectionRef is unassigned, never the reserved default grant.
      // Non-broker hosts bind as a conversion: the hub is the ops center that
      // sets provider, grant, and a seeded default model in one write.
      const currentRef = storedRef || unassigned
      const currentModel = hostModelName(loaded.host.spec)

      if (action === 'unbind') {
        if (currentRef === unassigned) {
          res.status(200).json({
            host: hostRef,
            connectionRef: unassigned,
            ...(currentModel ? { model: currentModel } : {}),
          })
          return
        }
        if (currentRef !== connectionKey) {
          res.status(409).json({ error: 'connection_mismatch' })
          return
        }
        const written = await writeHostConnectionRef(hostRef, loaded.host, unassigned, provider)
        if (!written.ok) {
          res.status(written.status).json({
            error: written.error,
            ...(written.message ? { message: written.message } : {}),
            ...(written.reason ? { reason: written.reason } : {}),
          })
          return
        }
        res.status(200).json({
          host: hostRef,
          connectionRef: unassigned,
          ...(written.model ? { model: written.model } : {}),
          ...(written.warnings.length > 0 ? { warnings: written.warnings } : {}),
        })
        return
      }

      if (currentRef === connectionKey && currentModel) {
        res.status(200).json({
          host: hostRef,
          connectionRef: connectionKey,
          model: currentModel,
        })
        return
      }
      const written = await writeHostConnectionRef(hostRef, loaded.host, connectionKey, provider)
      if (!written.ok) {
        res.status(written.status).json({
          error: written.error,
          ...(written.message ? { message: written.message } : {}),
          ...(written.reason ? { reason: written.reason } : {}),
        })
        return
      }
      res.status(200).json({
        host: hostRef,
        connectionRef: connectionKey,
        ...(written.model ? { model: written.model } : {}),
        ...(written.warnings.length > 0 ? { warnings: written.warnings } : {}),
      })
    })

  router.post(
    `${BASE}/connections/:key/hosts/:hostRef/unbind`,
    ...adminSubscriptionWriteRateLimits(),
    bindUnbindHandler('unbind')
  )
  router.post(
    `${BASE}/connections/:key/hosts/:hostRef/bind`,
    ...adminSubscriptionWriteRateLimits(),
    bindUnbindHandler('bind')
  )

  return router
}