Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
11.4 KB
Raw Blame
import type { NextFunction, Request, Response } from 'express'
import { ApiException } from '@kubernetes/client-node'
import { STATUS_CODES } from 'node:http'
import { clearRateLimitHeaders } from '../middleware/rateLimitMiddleware.js'
import { rootLogger } from '../observability/logger.js'
import {
  PasswordMutationInternalError,
  passwordMutationCauseKind,
} from '../services/auth/passwordAdmissionError.js'
import { memberRegistrationErrorResponse } from '../services/memberRegistrationErrors.js'

// Only errors we construct with these codes are ever forwarded verbatim by the
// global handler. Keeps the blast radius tight: every other status-less or
// non-allowlisted error still collapses to a 500.
export const FORWARDABLE_INTEGRATION_CODES = new Set([
  'registry_unavailable',
  'registry_integration_error',
])

// Stable machine-readable codes added to a forwarded 4xx body. Internal
// reporters (HCC) use them to tell a deterministic rejection, which must not be
// retried, from a transient one. Only codes set by our own error constructors
// belong here; every other 4xx keeps the `{ error, correlationId }` shape.
export const FORWARDABLE_CLIENT_ERROR_CODES = new Set([
  'tracing_idempotency_conflict',
  'unsafe_tracing_input',
  'invalid_tracing_input',
  // Deterministic: the annotated generation is already superseded, so no
  // future report can bind. `tracing_binding_unavailable` stays OUT on
  // purpose — that 403 must remain code-less and therefore retryable, which
  // is what lets a control-api/HCC deploy overlap heal itself.
  'administrative_intent_generation_drift',
])

/**
 * Extract an HTTP status from an error the same way `extractK8sStatus`
 * (resourceServiceHelpers) does, so a raw @kubernetes/client-node error — which
 * carries its status on `.code` / `.statusCode` / `.response.*`, NOT a top-level
 * `.status` — is forwarded with its real 4xx instead of collapsing to a 500.
 *
 * Precedence mirrors extractK8sStatus, with `.status` FIRST so the legacy
 * service-layer path (errors that set `.status` deliberately) is preserved.
 */
function extractErrorStatus(err: unknown): number | undefined {
  if (!err || typeof err !== 'object') return undefined
  const maybe = err as {
    status?: unknown
    statusCode?: unknown
    code?: unknown
    httpStatus?: unknown
    response?: { statusCode?: unknown; status?: unknown }
  }
  if (typeof maybe.status === 'number') return maybe.status
  if (typeof maybe.statusCode === 'number') return maybe.statusCode
  if (typeof maybe.code === 'number') return maybe.code
  // `.httpStatus` mirrors extractK8sError's chain (http/k8sError.ts): the
  // service-layer K8sNotFoundError/K8sConflictError expose their status there,
  // so read it after code/statusCode or their 404/409 collapses to 500.
  if (typeof maybe.httpStatus === 'number') return maybe.httpStatus
  if (maybe.response && typeof maybe.response.statusCode === 'number') {
    return maybe.response.statusCode
  }
  if (maybe.response && typeof maybe.response.status === 'number') return maybe.response.status
  return undefined
}

/**
 * True when `err` is a raw @kubernetes/client-node ApiException, or an object
 * carrying its leaking shape. ApiException stores the metav1.Status on `.body`
 * and EVERY apiserver response header (Audit-Id, x-kubernetes-pf-*) on
 * `.headers`, and its `.message` is built as
 * `"HTTP-Code: …\nMessage: …\nBody: <JSON status>\nHeaders: <JSON headers>"`.
 * Forwarding that message verbatim to the client leaks internals — hence the
 * shape fallback, so a copy created against a duplicated module instance (where
 * `instanceof` would miss) is still caught.
 */
function isK8sApiException(err: unknown): boolean {
  if (err instanceof ApiException) return true
  return (
    err instanceof Error &&
    'body' in err &&
    'headers' in err &&
    typeof (err as { headers?: unknown }).headers === 'object' &&
    (err as { headers?: unknown }).headers !== null
  )
}

/**
 * Client-safe message for a forwarded 4xx.
 *
 * A raw K8s ApiException `.message` embeds the full metav1.Status body AND every
 * apiserver response header — never forward it. Collapse K8s errors to the
 * standard status text, surfacing at most the metav1.Status `body.message`
 * (the apiserver's human explanation, e.g. `... already exists`), which is the
 * same class of info the service layer already returns and carries no headers.
 * Non-K8s Errors keep their own (author-controlled) message; non-Errors get
 * 'Bad Request'.
 */
function clientErrorMessage(err: unknown, status: number): string {
  const fallback = STATUS_CODES[status] ?? 'Bad Request'
  if (isK8sApiException(err)) {
    const body = (err as { body?: unknown }).body
    if (body && typeof body === 'object') {
      const bodyMessage = (body as { message?: unknown }).message
      if (typeof bodyMessage === 'string' && bodyMessage.length > 0) return bodyMessage
    } else if (typeof body === 'string' && body.length > 0) {
      try {
        const parsed = JSON.parse(body) as { message?: unknown }
        if (typeof parsed.message === 'string' && parsed.message.length > 0) return parsed.message
      } catch {
        // body is not JSON — never forward the raw string, fall through to the
        // generic status text.
      }
    }
    return fallback
  }
  return err instanceof Error ? err.message : 'Bad Request'
}

/**
 * Log-safe rendering of an error's message. A raw @kubernetes/client-node
 * ApiException `.message` embeds the metav1.Status body AND every apiserver
 * response header (Audit-Id, x-kubernetes-pf-*) — writing it to a log puts
 * infrastructure PID/PII into internal logs (S1). Collapse K8s exceptions to the
 * same sanitized text the client sees (metav1.Status `body.message` or the
 * status text, never headers); non-K8s errors keep their author-controlled
 * detail. Used for EVERY branch that logs an error message, not just the 4xx one.
 */
function safeErrorLogMessage(err: unknown): string {
  if (isK8sApiException(err)) return clientErrorMessage(err, extractErrorStatus(err) ?? 500)
  return err instanceof Error ? err.message : String(err)
}

/**
 * Log-safe stack: a K8s ApiException's `.stack` begins with its `.message`, so it
 * leaks the same headers — drop it for K8s exceptions and keep it otherwise.
 */
function safeErrorLogStack(err: unknown): string | undefined {
  if (isK8sApiException(err)) return undefined
  return err instanceof Error ? err.stack : undefined
}

/**
 * Global Express error handler for control-api.
 *
 * Exported (rather than an inline closure in createApp) so it can be unit-tested
 * in isolation. Behavior:
 *   - typed member-registration failures → their mapped status (spec §8.6);
 *   - any error carrying a 4xx status (via `.status` / `.statusCode` / `.code` /
 *     `.response.*`) is FORWARDED with a sanitized message — this is FIX-B: raw
 *     K8s 400/403/404/409/422 no longer collapse to 500;
 *   - allowlisted registry-integration 502/503 codes are forwarded verbatim;
 *   - everything else (5xx, status-less, non-Error throws) → 500, unchanged.
 */
export function clerumErrorHandler(
  err: unknown,
  req: Request,
  res: Response,
  _next: NextFunction
): void {
  // Use the request-scoped correlation id if middleware ran, otherwise
  // synthesize a short tag for legacy paths.
  const correlationId = req.correlationId ?? Math.random().toString(36).slice(2, 10)
  const log = req.log ?? rootLogger

  if (err instanceof PasswordMutationInternalError) {
    log.error(
      {
        event: 'password_mutation_callback_failed',
        stage: 'mutation_callback',
        causeKind: passwordMutationCauseKind(err.cause),
        correlationId,
      },
      'password mutation callback failed'
    )
    clearRateLimitHeaders(res)
    res.removeHeader('Retry-After')
    res.setHeader('Cache-Control', 'no-store')
    res.status(500).json({ error: err.code, correlationId })
    return
  }

  // Typed member-registration failures map to 503 (spec §8.6) — scoped
  // instanceof check; the generic 5xx-collapse below stays intact.
  const memberRegistration = memberRegistrationErrorResponse(err)
  if (memberRegistration) {
    log.warn(
      {
        event: memberRegistration.error,
        correlationId,
        err: safeErrorLogMessage(err),
      },
      'member registration unavailable'
    )
    res.status(memberRegistration.status).json({ error: memberRegistration.error, correlationId })
    return
  }

  // Errors thrown by service-layer code (and raw K8s client errors) carry a
  // status that signals the upstream response should be forwarded to the API
  // client rather than collapsed into a 500. Honor 4xx values; 5xx still surface
  // as Internal Server Error so backend failures don't leak through as
  // caller-visible errors.
  const errStatus = extractErrorStatus(err)
  const isClientError = errStatus !== undefined && errStatus >= 400 && errStatus < 500

  if (isClientError) {
    const clientCode = (err as { code?: unknown }).code
    const forwardedCode =
      typeof clientCode === 'string' && FORWARDABLE_CLIENT_ERROR_CODES.has(clientCode)
        ? clientCode
        : undefined
    log.warn(
      {
        event: 'forwarded_client_error',
        correlationId,
        status: errStatus,
        ...(forwardedCode ? { code: forwardedCode } : {}),
        err: safeErrorLogMessage(err),
      },
      'forwarded client error from upstream'
    )
    res.status(errStatus as number).json({
      error: clientErrorMessage(err, errStatus as number),
      ...(forwardedCode ? { code: forwardedCode } : {}),
      correlationId,
    })
    return
  }

  // Allowlisted registry-integration errors carry a safe code + message we set
  // ourselves (RegistryUnavailableError → 503; the 401 remap → 502). Forward
  // them so the marketplace shows a clear message instead of a raw 500. The
  // message is safe because only our own constructors set these codes.
  const integrationCode = (err as { code?: unknown }).code
  const integrationStatus = (err as { status?: unknown }).status
  if (
    typeof integrationStatus === 'number' &&
    (integrationStatus === 502 || integrationStatus === 503) &&
    typeof integrationCode === 'string' &&
    FORWARDABLE_INTEGRATION_CODES.has(integrationCode)
  ) {
    log.warn(
      {
        event: 'forwarded_registry_integration_error',
        correlationId,
        status: integrationStatus,
        code: integrationCode,
        err: safeErrorLogMessage(err),
      },
      'forwarded registry integration error'
    )
    res.status(integrationStatus).json({
      error: integrationCode,
      message: err instanceof Error ? err.message : 'Registry integration error',
      correlationId,
    })
    return
  }

  log.error(
    {
      event: 'unhandled_error',
      correlationId,
      err: safeErrorLogMessage(err),
      stack: safeErrorLogStack(err),
    },
    'unhandled error'
  )
  if (process.env.NODE_ENV !== 'production') {
    res.status(500).json({
      error: 'Internal Server Error',
      // Sanitize the client-facing debug message the same way the 4xx branch
      // does: a raw K8s ApiException `.message` embeds the metav1.Status body AND
      // every apiserver header (Audit-Id, x-kubernetes-pf-*). A 5xx K8s error
      // (etcd/apiserver down) is still an Error, so forwarding `err.message`
      // verbatim here leaks those headers in dev/test.
      message: clientErrorMessage(err, 500),
      correlationId,
    })
  } else {
    res.status(500).json({
      error: 'Internal Server Error',
      correlationId,
    })
  }
}