Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
50.6 KB
Raw Blame
import { type NextFunction, type Request, type Response, Router } from 'express'
import { config } from '../../config.js'
import {
  type DbClient,
  advisoryLockModelNames,
  boundCarrierTransactionIdleTimeout,
  pool,
  withTransaction,
} from '../../db.js'
import { asyncHandler } from '../../http/asyncHandler.js'
import { enforceNamespace } from '../../http/namespaceAudit.js'
import { validateCommunicationChannelSpec } from '../../http/validateCommunicationChannelSpec.js'
import { validateMcpServerSpecPreflight } from '../../http/validateMcpServerSpec.js'
import { K8sGateway } from '../../k8s.js'
import { deriveOAuthEncryptionKey } from '../../oauth/encryption.js'
import { uninstallMcpServer } from '../../oauth/mcpServerUninstall.js'
import { rootLogger } from '../../observability/logger.js'
import { stripHookRefFromHosts } from '../../services/hostGuardrailRefs.js'
import {
  K8sConflictError,
  K8sNotFoundError,
  type MutableResourceSnapshot,
} from '../../services/resourceService.js'
import { captureSecretForCleanup } from '../../services/secretCleanup.js'
import { secretKeyNames } from '../../services/secretKeyNames.js'
import {
  ClerumResourceType,
  type ResourcePreconditions,
  type SecretPreconditions,
} from '../../types.js'
import {
  registerCommunicationChannelCredentialsRoutes,
  validateCommunicationChannelCredentials,
} from './communicationChannelCredentials.js'
import {
  ccCredentialsSecretName,
  extractK8sStatusCode,
  preserveCommunicationChannelCredentialsSecretRef,
} from './communicationChannelSpecHelpers.js'
import { enumerateHostModelReferences } from './hostModelReferences.js'
import { validateHostSecretRef } from './hostSecrets.js'
import { createHostValidationDeps, validateHostSpec } from './hostSpecValidation.js'
import {
  type HostSpecIncoherenceToleratedEvent,
  emitHostSpecIncoherenceTolerated,
} from './hostWriteGateAudit.js'
import { collectResourceSpecFieldIssues, validateResourceName } from './resourceFieldValidation.js'
import type { StaleModelWarning } from './staleModelWarning.js'

export const adminResourcesLogger = rootLogger.child({ module: 'admin-resources' })
const log = adminResourcesLogger

const PROVIDER_SETTINGS_FIELDS: Readonly<Record<string, readonly string[]>> = {
  telegramSettings: ['botHandle', 'replyOnlyWhenMentioned'],
  slackSettings: ['workspaceId', 'botHandle', 'replyOnlyWhenMentioned', 'replyInThreads'],
  teamsSettings: ['appName', 'appId', 'tenantId', 'replyOnlyWhenMentioned'],
}

type CommunicationChannelSpecSnapshot = {
  spec: Record<string, unknown>
  uid?: string
  resourceVersion?: string
}

function recordValue(value: unknown): Record<string, unknown> | null {
  return value && typeof value === 'object' && !Array.isArray(value)
    ? (value as Record<string, unknown>)
    : null
}

function resourcePreconditionsFromResource(resource: unknown): ResourcePreconditions | undefined {
  const metadata = recordValue((resource as { metadata?: unknown } | null)?.metadata)
  const uid = metadata?.uid
  const resourceVersion = metadata?.resourceVersion
  if (typeof uid !== 'string' || !uid || typeof resourceVersion !== 'string' || !resourceVersion) {
    return undefined
  }
  return { uid, resourceVersion }
}

function credentialPreconditionsFromObject(object: unknown): SecretPreconditions | undefined {
  const metadata = recordValue((object as { metadata?: unknown } | null)?.metadata)
  const flat = recordValue(object)
  const uid = typeof flat?.uid === 'string' ? flat.uid : metadata?.uid
  const resourceVersion =
    typeof flat?.resourceVersion === 'string' ? flat.resourceVersion : metadata?.resourceVersion
  if (typeof uid !== 'string' || !uid || typeof resourceVersion !== 'string' || !resourceVersion) {
    return undefined
  }
  return { uid, resourceVersion }
}

function missingPersistedProviderSetting(
  requestedSpec: Record<string, unknown>,
  persisted: unknown
): string | null {
  const persistedSpec = recordValue((persisted as { spec?: unknown } | null)?.spec) || {}

  for (const [settingsKey, fields] of Object.entries(PROVIDER_SETTINGS_FIELDS)) {
    const requestedSettings = recordValue(requestedSpec[settingsKey])
    if (!requestedSettings) continue
    const persistedSettings = recordValue(persistedSpec[settingsKey])

    for (const field of fields) {
      if (requestedSettings[field] === undefined) continue
      if (persistedSettings?.[field] === undefined) return `spec.${settingsKey}.${field}`
    }
  }

  return null
}

function sendPrunedProviderSettingError(res: Response, field: string): void {
  log.warn({ field }, 'CommunicationChannel provider setting was pruned')
  res.status(409).json({
    code: 'communication_channel_crd_outdated',
    error:
      `CommunicationChannel ${field} was not persisted by Kubernetes. ` +
      'Apply the latest clerum-crds before creating or updating this provider.',
  })
}

async function rollbackPrunedCommunicationChannelCreate(
  gateway: K8sGateway,
  name: string,
  namespace: string,
  credentialsSecretName?: string,
  resourcePreconditions?: ResourcePreconditions,
  credentialPreconditions?: SecretPreconditions
): Promise<void> {
  if (resourcePreconditions) {
    try {
      await gateway.deleteResource('communicationchannels', name, namespace, resourcePreconditions)
    } catch (err) {
      log.warn({ name, err }, 'CommunicationChannel pruned-setting rollback failed')
    }
  } else {
    log.warn({ name }, 'CommunicationChannel pruned-setting rollback skipped: identity unavailable')
  }
  if (!credentialsSecretName) return
  if (credentialPreconditions) {
    try {
      await gateway.deleteSecret(credentialsSecretName, namespace, credentialPreconditions)
    } catch (err) {
      log.warn(
        { secretName: credentialsSecretName, err },
        'CommunicationChannel credentials rollback failed'
      )
    }
  } else {
    log.warn(
      { secretName: credentialsSecretName },
      'CommunicationChannel credentials rollback skipped: identity unavailable'
    )
  }
}

async function loadCommunicationChannelSpecSnapshot(
  gateway: K8sGateway,
  name: string,
  namespace: string
): Promise<CommunicationChannelSpecSnapshot | null> {
  try {
    const existing = await gateway.getResource('communicationchannels', name, namespace)
    const spec = recordValue((existing as { spec?: unknown } | null)?.spec)
    const identity = resourcePreconditionsFromResource(existing)
    return {
      spec: spec ? { ...spec } : {},
      ...(identity ?? {}),
    }
  } catch (err) {
    if (extractK8sStatusCode(err) === 404) return null
    throw err
  }
}

async function rollbackPrunedCommunicationChannelUpdate(
  gateway: K8sGateway,
  name: string,
  namespace: string,
  snapshot: CommunicationChannelSpecSnapshot | null,
  resourcePreconditions?: ResourcePreconditions
): Promise<void> {
  if (!snapshot) return
  if (!resourcePreconditions?.uid || !resourcePreconditions.resourceVersion) {
    log.warn({ name }, 'CommunicationChannel rollback skipped: identity unavailable')
    return
  }
  try {
    const restored = await gateway.updateResource(
      'communicationchannels',
      name,
      {
        metadata: resourcePreconditions,
        spec: snapshot.spec,
      },
      namespace
    )
    const missingField = missingPersistedProviderSetting(snapshot.spec, restored)
    if (missingField) {
      log.warn({ name, field: missingField }, 'CommunicationChannel rollback did not restore field')
    }
  } catch (err) {
    log.warn({ name, err }, 'CommunicationChannel pruned-setting rollback failed')
  }
}

/**
 * Deploy-order guard for the additive context `spec.displayName` field (PR #304).
 *
 * `spec.displayName` is a new, optional, additive field on the context CRD. If a
 * create/update carries a non-empty `spec.displayName` but the CRD applied to
 * the cluster does NOT yet declare the field (deploy-order skew: control-ui
 * writes the field before the CRD update lands), the apiserver PRUNES the
 * unknown field SILENTLY and returns success — the display name is lost with no
 * error surfaced to the operator.
 *
 * This is a pure read-after-write check: it compares the value the caller sent
 * against the object the apiserver actually persisted. A create/replace returns
 * the persisted object AFTER pruning, so this reuses that response with NO extra
 * apiserver read (preserving the N1 single-read-per-PUT optimization).
 *
 * STRICTLY SCOPED to `contexts` + `spec.displayName` — the only additive field
 * at pruning risk in this PR. Do NOT generalize to other fields or resources:
 * that would be a decision module (which fields, precedence) requiring its own
 * mini-spec, not a one-field hardening net.
 */
function contextDisplayNameNotPersisted(
  requestedSpec: Record<string, unknown>,
  persisted: unknown
): boolean {
  const requested = requestedSpec.displayName
  // Only fire when the caller actually asked for a non-empty display name.
  if (typeof requested !== 'string' || requested.trim() === '') return false
  const persistedSpec = recordValue((persisted as { spec?: unknown } | null)?.spec) || {}
  // Missing (pruned) OR differs from what was sent — either way it did not
  // round-trip. displayName is stored verbatim (free text, no server-side
  // normalization), so a mismatch here means loss, not transformation.
  return persistedSpec.displayName !== requested
}

function sendPrunedDisplayNameError(res: Response): void {
  log.warn('Context spec.displayName was pruned by the apiserver (CRD outdated)')
  res.status(409).json({
    code: 'context_crd_outdated',
    error:
      'Context spec.displayName was not persisted by Kubernetes; the context CRD does not ' +
      'support spec.displayName. Apply the latest clerum-crds (which add spec.displayName) ' +
      'before setting a display name.',
  })
}

/**
 * Best-effort rollback of a context whose create silently dropped
 * `spec.displayName`. Unlike the update path, a POST is NOT idempotent: leaving
 * the pruned context in place would make the operator's retry (after applying
 * the CRD) collide with a 409 AlreadyExists. Deleting it restores a clean retry
 * path. Mirrors rollbackPrunedCommunicationChannelCreate.
 */
async function rollbackPrunedContextCreate(
  gateway: K8sGateway,
  name: string,
  namespace: string,
  resourcePreconditions?: ResourcePreconditions
): Promise<void> {
  if (!resourcePreconditions) {
    log.warn({ name }, 'Context pruned-displayName rollback skipped: identity unavailable')
    return
  }
  try {
    await gateway.deleteResource('contexts', name, namespace, resourcePreconditions)
  } catch (err) {
    log.warn({ name, err }, 'Context pruned-displayName rollback failed')
  }
}

/**
 * Return true if the CC spec has at least one non-empty provider array
 * (telegram, slack, teams, or email). These are the providers that require a
 * credentials Secret to function; a CC without any provider is valid
 * (e.g. a bare hostRef-only CC for future configuration).
 */
function ccSpecHasProvider(spec: Record<string, unknown>): boolean {
  const telegramSettings = recordValue(spec.telegramSettings)
  const slackSettings = recordValue(spec.slackSettings)
  const teamsSettings = recordValue(spec.teamsSettings)
  return (
    (Array.isArray(spec.telegram) && spec.telegram.length > 0) ||
    (Array.isArray(spec.slack) && spec.slack.length > 0) ||
    (Array.isArray(spec.teams) && spec.teams.length > 0) ||
    (Array.isArray(spec.email) && spec.email.length > 0) ||
    !!telegramSettings?.botHandle ||
    !!slackSettings?.workspaceId ||
    !!slackSettings?.botHandle ||
    !!teamsSettings?.appName ||
    !!teamsSettings?.appId ||
    !!teamsSettings?.tenantId
  )
}

function ccSpecHasTelegramProvider(spec: Record<string, unknown>): boolean {
  const telegramSettings = recordValue(spec.telegramSettings)
  return (Array.isArray(spec.telegram) && spec.telegram.length > 0) || !!telegramSettings?.botHandle
}

function ccSpecHasSlackProvider(spec: Record<string, unknown>): boolean {
  const slackSettings = recordValue(spec.slackSettings)
  return (
    (Array.isArray(spec.slack) && spec.slack.length > 0) ||
    !!slackSettings?.workspaceId ||
    !!slackSettings?.botHandle
  )
}

function ccSpecHasTeamsProvider(spec: Record<string, unknown>): boolean {
  const teamsSettings = recordValue(spec.teamsSettings)
  return (
    (Array.isArray(spec.teams) && spec.teams.length > 0) ||
    !!teamsSettings?.appName ||
    !!teamsSettings?.appId ||
    !!teamsSettings?.tenantId
  )
}

function missingCreateCredentialKey(
  spec: Record<string, unknown>,
  credentials: Record<string, string>
): string | null {
  if (ccSpecHasTelegramProvider(spec) && !credentials['telegram-bot-token']) {
    return 'telegram-bot-token'
  }
  if (ccSpecHasSlackProvider(spec)) {
    for (const key of ['slack-signing-secret', 'slack-bot-token']) {
      if (!credentials[key]) return key
    }
  }
  if (ccSpecHasTeamsProvider(spec) && !credentials['teams-app-password']) {
    return 'teams-app-password'
  }
  return null
}

const PROVIDER_REQUIRED_KEYS: Record<string, string[]> = {
  telegram: ['telegram-bot-token'],
  slack: ['slack-bot-token', 'slack-signing-secret'],
  teams: ['teams-app-password'],
  email: ['email-username', 'email-password'],
}

function providersEnabled(spec: Record<string, unknown> | null): Set<string> {
  const enabled = new Set<string>()
  if (!spec) return enabled
  if (ccSpecHasTelegramProvider(spec)) enabled.add('telegram')
  if (ccSpecHasSlackProvider(spec)) enabled.add('slack')
  if (ccSpecHasTeamsProvider(spec)) enabled.add('teams')
  if (Array.isArray(spec.email) && spec.email.length > 0) enabled.add('email')
  return enabled
}

/**
 * Validate providers that go absent -> present. Returns an error message, or null.
 *
 * Deliberately scoped to the transition: a channel already missing keys in a
 * cluster stays editable, and enabling a NEW provider without its credentials is
 * refused. Presence of `credentialsSecretRef` proves nothing, which is why the
 * pre-existing POST guard missed this entirely — the channel that motivated this
 * (#312) already had a ref, holding only its Telegram token, and still accepted a
 * Slack App Name.
 *
 * Fails CLOSED: an unreadable Secret rejects the write. Refusing a write on a
 * read failure is safe; assuming the keys are there is how a channel ends up
 * advertising a provider it cannot serve.
 */
async function providerTransitionError(
  gateway: K8sGateway,
  nextSpec: Record<string, unknown>,
  previousSpec: Record<string, unknown> | null,
  credentials: Record<string, string> | undefined,
  namespace: string
): Promise<string | null> {
  const previouslyEnabled = providersEnabled(previousSpec)
  const added = [...providersEnabled(nextSpec)].filter(p => !previouslyEnabled.has(p))
  if (added.length === 0) return null

  const secretName = (nextSpec.credentialsSecretRef as { name?: string } | undefined)?.name?.trim()
  let existingKeys: string[] = []
  if (secretName) {
    try {
      existingKeys = await secretKeyNames(gateway, secretName, namespace)
    } catch {
      return `Cannot read the credentials Secret "${secretName}" to validate the new provider. Fix access to that Secret, or supply the credentials with this request.`
    }
  }

  for (const provider of added) {
    for (const key of PROVIDER_REQUIRED_KEYS[provider] || []) {
      // Trimmed, because this reads the RAW envelope while the Secret is written
      // from the CLEANED values: `validateCommunicationChannelCredentials` drops
      // whitespace-only entries, so a truthy "   " here would satisfy the guard
      // and then never reach the Secret. A present-but-non-string value (e.g. a
      // number) is NOT "missing" though: it's `validateCommunicationChannelCredentials`
      // below that owns rejecting the wrong type with its own 400, so this only
      // calls `.trim()` once `typeof` has confirmed there's a string to trim.
      const rawValue = credentials?.[key]
      const isMissing = rawValue === undefined || (typeof rawValue === 'string' && !rawValue.trim())
      if (isMissing && !existingKeys.includes(key)) {
        return `credentials["${key}"] is required to enable the ${provider} provider on this CommunicationChannel`
      }
    }
  }
  return null
}

// workflowrecipes is handled by recipes.ts (canonical sandbox-recipes route).
// workflowrecipepolicies is read inline by the recipes policy-invariant
// helper and does not need generic admin CRUD routing.
// sharedfilesystems is handled by sharedFilesystems.ts (admin-only writes
// + token mint + reverse proxy to the per-SFS wfc).
type AdminResourceType = Exclude<
  ClerumResourceType,
  'workflowrecipes' | 'workflowrecipepolicies' | 'sharedfilesystems'
>

const RESOURCE_MAP: Record<string, AdminResourceType> = {
  hosts: 'hosts',
  contexts: 'contexts',
  communicationchannels: 'communicationchannels',
  'communication-channels': 'communicationchannels',
  mcpservers: 'mcpservers',
  'mcp-servers': 'mcpservers',
  llmhooks: 'llmhooks',
  'llm-hooks': 'llmhooks',
}

function resourceNamespace(plural: AdminResourceType): string {
  switch (plural) {
    case 'hosts':
      return config.hostsNamespace
    case 'contexts':
      return config.contextsNamespace
    case 'communicationchannels':
      return config.communicationChannelsNamespace
    case 'mcpservers':
      return config.mcpServersNamespace
    case 'llmhooks':
      return config.llmHooksNamespace
  }
}

function communicationChannelMatchesConfirmedUser(item: unknown, userId: string): boolean {
  const spec = (item as { spec?: Record<string, unknown> } | null)?.spec
  if (!spec) return false
  const groups = [
    ...(Array.isArray(spec.telegram) ? spec.telegram : []),
    ...(Array.isArray(spec.slack) ? spec.slack : []),
    ...(Array.isArray(spec.teams) ? spec.teams : []),
    ...(Array.isArray(spec.email) ? spec.email : []),
  ]
  return groups.some(group => {
    if (!group || typeof group !== 'object') return false
    return String((group as { confirmedByUserId?: unknown }).confirmedByUserId || '') === userId
  })
}

const RESOURCE_PATTERN =
  '/admin/:resource(hosts|contexts|communication-channels|communicationchannels|mcp-servers|mcpservers)'

// LlmHook guardrail CRs get READ + DELETE only through the generic router.
// Creation/update goes through the org-scoped registry `install-hook` saga
// (registry.ts) so the install-time trust_level + image-allowlist preflight
// cannot be bypassed by a raw POST/PUT — the same reason recipes are not in the
// generic CRUD pattern. This lane powers the control-ui cluster-wide list, the
// per-hook status detail, and uninstall.
const LLMHOOK_PATTERN = '/admin/:resource(llmhooks|llm-hooks)'

/** Resolve the plural type and canonical namespace for a resource route param. */
function resolveResource(param: string): { plural: AdminResourceType; ns: string } | null {
  const plural = RESOURCE_MAP[param]
  if (!plural) return null
  return { plural, ns: resourceNamespace(plural) }
}

// R1-H3 fase 1: a Host write (create/update) validates its model references and
// writes the CR INSIDE a carrier transaction that holds the per-model-name
// advisory lock across the K8s write, so a concurrent llm-model disable/delete
// cannot strand a reference (INV-1). `validateHostSpec` returning an issue must
// abort the whole thing WITHOUT persisting: it throws this so `withTransaction`
// rolls back (releasing the lock) and the caller answers 422 with the issue.
class HostSpecInvalidError extends Error {
  constructor(readonly issue: { errors: Array<{ field: string; message: string }> }) {
    super('host_spec_invalid')
    this.name = 'HostSpecInvalidError'
  }
}

// Every distinct model NAME a Host spec references (primary + allowedModels +
// fallbacks). Reuses the single Host-reference enumeration (regla D4); the raw
// model name is what the advisory lock keys on (adenda A1). Dedup + total order
// are handled by `advisoryLockModelNames`.
function hostModelLockNames(spec: unknown): string[] {
  return enumerateHostModelReferences(spec).map(ref => ref.model)
}

// Bind the Host-spec validation gates to the carrier TRANSACTION client so the
// allowlist reads happen under the advisory lock, on the same connection. BOTH
// `isModelAllowed` (the fail-closed gate) AND `getModelAllowlistState` (the
// stale-warning lookup) are bound — leaving the latter defaulted would let
// `maybeWarnStale` escape to the global pool under the lock (adenda A3).
function hostValidationDeps(db: DbClient) {
  return createHostValidationDeps(db)
}

export function createAdminResourcesRouter(gateway: K8sGateway): Router {
  const router = Router()
  // Uses the module-level `log` (= exported `adminResourcesLogger`). A local
  // child here would shadow it with a different instance, so handler logs would
  // bypass any spy/redaction attached to the exported logger.
  // Reliable local revocation of a remote server's DCR client on uninstall; the
  // AS-side RFC 7592 delete is courtesy (real pinned transport in production).
  const oauthEncryptionKey = deriveOAuthEncryptionKey(config.oauthEncryptionKey)
  const dcrDb: DbClient = { query: (text, values) => pool.query(text, values) }

  // Middleware: enforce namespace per resource type and audit any injection attempt.
  // Uses enforceNamespace() consistently with all other admin routers.
  router.use(RESOURCE_PATTERN, (req: Request, _res: Response, next: NextFunction) => {
    const resolved = resolveResource(req.params.resource)
    if (resolved) {
      enforceNamespace(resolved.ns)(req, _res, next)
    } else {
      next()
    }
  })

  router.get(
    RESOURCE_PATTERN,
    asyncHandler(async (req, res) => {
      const { plural, ns } = resolveResource(req.params.resource)!
      let items = await gateway.listResource(plural, ns)
      const confirmedByUserId =
        typeof req.query.confirmedByUserId === 'string' ? req.query.confirmedByUserId.trim() : ''
      if (plural === 'communicationchannels' && confirmedByUserId) {
        items = items.filter(item =>
          communicationChannelMatchesConfirmedUser(item, confirmedByUserId)
        )
      }
      res.status(200).json({ items })
    })
  )

  router.post(
    RESOURCE_PATTERN,
    asyncHandler(async (req, res) => {
      const { plural, ns } = resolveResource(req.params.resource)!
      const body = req.body as {
        metadata: { name: string }
        spec: Record<string, unknown> & {
          envSecret?: { name?: unknown }
        }
        credentials?: Record<string, string>
      }
      // Fase 6 soft-quarantine warnings granted during Host validation, returned
      // in the 201 body only after the CR persists (below). Empty for every
      // non-host create.
      const hostWarnings: StaleModelWarning[] = []

      // FIX-A1: validate metadata.name (RFC1123) for ALL plurals BEFORE any spec
      // validation or side effect (e.g. CC credentials Secret creation). An
      // invalid name would otherwise reach K8s and its 422 would collapse to 500.
      const nameIssue = validateResourceName(body.metadata?.name)
      if (nameIssue) {
        res.status(422).json(nameIssue)
        return
      }

      // F0.3: identifier/display field validation for hosts + contexts (create =
      // no ratchet; every present field is validated).
      if ((plural === 'hosts' || plural === 'contexts') && body.spec) {
        const fieldIssues = collectResourceSpecFieldIssues(plural, body.spec, null)
        if (fieldIssues.length > 0) {
          res.status(422).json({ errors: fieldIssues })
          return
        }
      }

      if (plural === 'communicationchannels' && body.spec) {
        const errors = validateCommunicationChannelSpec(body.spec)
        if (errors.length > 0) {
          res.status(422).json({ errors })
          return
        }

        // B4: a CC with a provider (telegram/slack/teams/email) must not be created
        // without credentials. Require EITHER a `credentials` envelope on the
        // request OR a `credentialsSecretRef` already present in the spec.
        if (ccSpecHasProvider(body.spec) && !body.credentials && !body.spec.credentialsSecretRef) {
          res.status(400).json({
            error:
              'A CommunicationChannel with a provider (telegram, slack, teams, or email) requires ' +
              'credentials. Supply a "credentials" envelope on the request, or set ' +
              '"spec.credentialsSecretRef" to an existing Secret.',
          })
          return
        }

        // Every declared provider is new on create, so the whole spec is the
        // transition. A credentialsSecretRef satisfies the guard above without
        // proving the Secret behind it holds this provider's keys.
        const transitionError = await providerTransitionError(
          gateway,
          body.spec,
          null,
          body.credentials,
          ns
        )
        if (transitionError) {
          res.status(400).json({ error: transitionError })
          return
        }
      }

      if (plural === 'communicationchannels' && body.credentials) {
        const credentialValidation = validateCommunicationChannelCredentials(body.credentials)
        if (!credentialValidation.ok) {
          res.status(400).json({ error: credentialValidation.error })
          return
        }
        const missingKey = missingCreateCredentialKey(body.spec, credentialValidation.values)
        if (missingKey) {
          res.status(400).json({
            error: `credentials["${missingKey}"] is required for this CommunicationChannel provider`,
          })
          return
        }
        const ccName = body.metadata?.name
        if (!ccName) {
          res.status(400).json({ error: 'metadata.name is required' })
          return
        }
        const secretName = ccCredentialsSecretName(ccName)
        let createdCredentialPreconditions: SecretPreconditions | undefined
        const createdCredential = await gateway.createSecret({
          name: secretName,
          namespace: ns,
          type: 'Opaque',
          stringData: credentialValidation.values,
        })
        createdCredentialPreconditions = credentialPreconditionsFromObject(createdCredential)
        // Inject credentialsSecretRef into the CC spec; strip the
        // non-CRD `credentials` envelope so it does not leak into the
        // CustomObjects API call.
        const { credentials: _credentials, ...rest } = body
        const ccBody = {
          ...rest,
          spec: {
            ...rest.spec,
            credentialsSecretRef: { name: secretName },
          },
        }
        try {
          const created = await gateway.createResource(plural, ccBody, ns)
          const missingField = missingPersistedProviderSetting(ccBody.spec, created)
          if (missingField) {
            await rollbackPrunedCommunicationChannelCreate(
              gateway,
              ccName,
              ns,
              secretName,
              resourcePreconditionsFromResource(created),
              createdCredentialPreconditions
            )
            sendPrunedProviderSettingError(res, missingField)
            return
          }
          res.status(201).json(created)
        } catch (err) {
          // Best-effort rollback. 404 from a stale Secret read or a
          // missing-secret race is tolerable — log and bubble the
          // original CRD error up so the caller sees the failure cause.
          if (createdCredentialPreconditions) {
            try {
              await gateway.deleteSecret(secretName, ns, createdCredentialPreconditions)
            } catch (rollbackErr) {
              log.warn(
                { secretName, err: rollbackErr },
                'CommunicationChannel credentials rollback failed'
              )
            }
          } else {
            log.warn(
              { secretName },
              'CommunicationChannel credentials rollback skipped: identity unavailable'
            )
          }
          throw err
        }
        return
      }

      // Host create (R1-H3 fase 1): validate + CREATE the CR INSIDE a carrier
      // transaction that HOLDS the per-model-name advisory lock across the K8s
      // write, so a concurrent llm-model disable/delete cannot strand the new
      // reference (INV-1). The impact gate on the reductor side takes the SAME
      // lock and LISTs Hosts live, so whichever side commits first wins and the
      // loser re-reads under the lock and aborts.
      if (plural === 'hosts' && body.spec) {
        const hostSpec = body.spec
        let created: unknown
        try {
          created = await withTransaction(async db => {
            // Bound idle-in-transaction tenancy first (the lock is held across the
            // live K8s write), then acquire the model-name locks in total order.
            await boundCarrierTransactionIdleTimeout(db)
            await advisoryLockModelNames(db, hostModelLockNames(hostSpec))
            // Create: no stored CR, so the no-worsening tolerance (Pieza D) can
            // never apply — a fresh Host may not introduce a disabled model. The
            // allowlist gates read UNDER the lock (deps bound to `db`).
            const issue = await validateHostSpec(hostSpec, hostValidationDeps(db), {
              hostRef: { namespace: ns, name: body.metadata.name },
              // Fase 6: a fresh Host has no stored CR, so EVERY assignment is new —
              // a stale enabled model warns (never blocks). Emitted after the CR
              // lands (below), so a rejected write surfaces no warning.
              warnings: hostWarnings,
            })
            if (issue) throw new HostSpecInvalidError(issue)
            // Anti-spoofing: an existing secretRef target must be an LLM host
            // Secret. Kept before the write and inside the lock so precedence
            // (spec issue → secretRef issue) is byte-identical to before.
            const secretRefIssue = await validateHostSecretRef(gateway, hostSpec)
            if (secretRefIssue) throw new HostSpecInvalidError(secretRefIssue)
            // K8s write INSIDE the held lock: the COMMIT that releases the lock
            // happens only after the CR is durable.
            return gateway.createResource(plural, body, ns)
          })
        } catch (err) {
          if (err instanceof HostSpecInvalidError) {
            res.status(422).json(err.issue)
            return
          }
          throw err
        }
        res
          .status(201)
          .json(
            hostWarnings.length > 0
              ? { ...(created as Record<string, unknown>), warnings: hostWarnings }
              : created
          )
        return
      }

      // Early validation for McpServer specs — UX safety net + defense-in-depth.
      // HCC reconciler also sanitizes as a backstop.
      if (plural === 'mcpservers' && body.spec) {
        const errors = await validateMcpServerSpecPreflight(body.spec, {
          allowedImagePrefixes: config.allowedPluginImagePrefixes,
          enforceImageAllowlist: config.enforcePluginImageAllowlist,
        })
        if (errors.length > 0) {
          res.status(422).json({ errors })
          return
        }

        // Missing envSecret refs are allowed at creation time so Connector
        // Secrets can be completed from the Secrets UI after the McpServer CRD
        // exists. HCC/Kubernetes still fail closed until the Secret is added.
      }

      const created = await gateway.createResource(plural, body, ns)
      if (plural === 'communicationchannels' && body.spec) {
        const missingField = missingPersistedProviderSetting(body.spec, created)
        if (missingField) {
          await rollbackPrunedCommunicationChannelCreate(
            gateway,
            body.metadata.name,
            ns,
            undefined,
            resourcePreconditionsFromResource(created)
          )
          sendPrunedProviderSettingError(res, missingField)
          return
        }
      }
      if (
        plural === 'contexts' &&
        body.spec &&
        contextDisplayNameNotPersisted(body.spec, created)
      ) {
        await rollbackPrunedContextCreate(
          gateway,
          body.metadata.name,
          ns,
          resourcePreconditionsFromResource(created)
        )
        sendPrunedDisplayNameError(res)
        return
      }
      // Host CR persisted: attach any Fase 6 warnings additively (older clients
      // ignore the field). Absent when there is nothing to warn about.
      res
        .status(201)
        .json(
          hostWarnings.length > 0
            ? { ...(created as Record<string, unknown>), warnings: hostWarnings }
            : created
        )
    })
  )

  router.get(
    `${RESOURCE_PATTERN}/:name`,
    asyncHandler(async (req, res) => {
      const { plural, ns } = resolveResource(req.params.resource)!
      const resource = await gateway.getResource(plural, req.params.name, ns)
      res.status(200).json(resource)
    })
  )

  router.put(
    `${RESOURCE_PATTERN}/:name`,
    asyncHandler(async (req, res) => {
      const { plural, ns } = resolveResource(req.params.resource)!
      const body = req.body as {
        metadata?: {
          labels?: Record<string, string>
          annotations?: Record<string, string>
          resourceVersion?: string
        }
        spec: Record<string, unknown>
      }
      // Pieza D tolerations granted during Host validation, emitted only after
      // the CR persists (below). Empty for every non-host / non-tolerated write.
      const hostTolerations: HostSpecIncoherenceToleratedEvent[] = []
      // Fase 6 soft-quarantine warnings, returned in the 200 body only after the
      // CR persists (below). Empty for every non-host / non-stale write.
      const hostWarnings: StaleModelWarning[] = []

      // N1 — the CR read by the ratchet below is the single pre-write read for
      // hosts + contexts. For CONTEXTS it is handed to updateResource
      // (preReadCurrent, see below) so the PUT reads the apiserver ONCE, not
      // twice. For HOSTS it also feeds the Pieza D tolerance (storedHostSpec,
      // below) — but the host write goes through the carrier transaction, whose
      // updateResource deliberately reads afresh UNDER the advisory lock (INV-1:
      // a pre-lock snapshot would reopen the availability race the lock closes),
      // so preReadCurrent is not used there. Undefined for non hosts/contexts
      // plurals and on the 404 path, where updateResource reads afresh and
      // surfaces the real 404.
      let ratchetCurrent: MutableResourceSnapshot | undefined

      // F0.3 RATCHET: for hosts + contexts, validate an identifier/display field
      // ONLY IF its value changed vs the current CR, so a legacy resource whose
      // spec.host / contextId is out of norm is never blocked for other edits.
      if ((plural === 'hosts' || plural === 'contexts') && body.spec) {
        let currentSpec: Record<string, unknown> | null = null
        try {
          const current = (await gateway.getResource(plural, req.params.name, ns)) as
            | (MutableResourceSnapshot & { spec?: Record<string, unknown> })
            | null
          currentSpec = recordValue(current?.spec)
          ratchetCurrent = current ?? undefined
        } catch (err) {
          // Missing resource: fall through with currentSpec=null (validate every
          // present field). The subsequent updateResource surfaces the real 404.
          // getResource wraps a namespaced 404 as K8sNotFoundError (httpStatus),
          // which extractK8sStatusCode does not read, so check both.
          if (!(err instanceof K8sNotFoundError) && extractK8sStatusCode(err) !== 404) throw err
        }
        const fieldIssues = collectResourceSpecFieldIssues(plural, body.spec, currentSpec)
        if (fieldIssues.length > 0) {
          res.status(422).json({ errors: fieldIssues })
          return
        }
      }

      if (plural === 'communicationchannels' && body.spec) {
        const errors = validateCommunicationChannelSpec(body.spec)
        if (errors.length > 0) {
          res.status(422).json({ errors })
          return
        }
      }

      // Same early validation for McpServer updates.
      if (plural === 'mcpservers' && body.spec) {
        const errors = await validateMcpServerSpecPreflight(body.spec, {
          allowedImagePrefixes: config.allowedPluginImagePrefixes,
          enforceImageAllowlist: config.enforcePluginImageAllowlist,
        })
        if (errors.length > 0) {
          res.status(422).json({ errors })
          return
        }
      }

      // Host update (R1-H3 fase 1): validate + REPLACE the CR INSIDE a carrier
      // transaction holding the per-model-name advisory lock across the K8s write,
      // exactly like create. The revalidation under the lock reuses the SAME
      // `validateHostSpec` — which already encapsulates the no-worsening tolerance
      // (Pieza D) — so the tolerance is preserved, not tightened (mini-spec §7).
      if (plural === 'hosts' && body.spec) {
        const hostSpec = body.spec
        // The stored CR lets the write gate tolerate a pre-existing
        // model_not_allowed incoherence this write does not worsen (Pieza D — the
        // editability trap). Reuse the snapshot the ratchet already read above
        // (ratchetCurrent) instead of a second getResource: for hosts the ratchet
        // block always ran (its guard is a superset of this one), so ratchetCurrent
        // is set, or undefined on the 404 path. If it is absent, tolerance is
        // skipped (fail-closed: the gate behaves exactly as before this feature).
        // This snapshot feeds tolerance, not the availability race — the write gate
        // re-reads fresh under the lock — and the K8s resourceVersion precondition
        // guards CR-level concurrency.
        const storedHostSpec: Record<string, unknown> | undefined =
          recordValue(ratchetCurrent?.spec) ?? undefined
        let updated: unknown
        try {
          updated = await withTransaction(async db => {
            await boundCarrierTransactionIdleTimeout(db)
            // Lock the models the INCOMING spec references — a newly added
            // reference is the only one this write can strand.
            await advisoryLockModelNames(db, hostModelLockNames(hostSpec))
            const issue = await validateHostSpec(hostSpec, hostValidationDeps(db), {
              stored: storedHostSpec,
              hostRef: { namespace: ns, name: req.params.name },
              // Sink for tolerations granted by validation; emitted below ONLY
              // after the CR persists (mini-spec 01), so a secretRef 422 or a K8s
              // conflict leaves no audit record.
              tolerations: hostTolerations,
              // Fase 6 warnings: a NEW assignment of an enabled-but-stale model
              // warns (never blocks). A live reference already in the stored CR is
              // not revalidated. Returned in the body only after the CR persists.
              warnings: hostWarnings,
            })
            if (issue) throw new HostSpecInvalidError(issue)
            const secretRefIssue = await validateHostSecretRef(gateway, hostSpec)
            if (secretRefIssue) throw new HostSpecInvalidError(secretRefIssue)
            return gateway.updateResource(plural, req.params.name, body, ns)
          })
        } catch (err) {
          if (err instanceof HostSpecInvalidError) {
            res.status(422).json(err.issue)
            return
          }
          // AP-6: the caller sent the resourceVersion it read and the resource
          // changed underneath it. Machine-readable so the UI can tell the operator
          // to reload — never retried server-side with the stale body.
          if (err instanceof K8sConflictError) {
            res.status(409).json({ error: 'conflict', reason: 'resource_changed' })
            return
          }
          throw err
        }
        // Host CR persisted: NOW emit any Pieza D tolerations (never before the
        // write lands, so a K8s conflict/error leaves no audit record).
        for (const event of hostTolerations) emitHostSpecIncoherenceTolerated(event)
        res
          .status(200)
          .json(
            hostWarnings.length > 0
              ? { ...(updated as Record<string, unknown>), warnings: hostWarnings }
              : updated
          )
        return
      }

      const isCommunicationChannelUpdate = plural === 'communicationchannels' && body.spec
      const previousCommunicationChannelSpec = isCommunicationChannelUpdate
        ? await loadCommunicationChannelSpecSnapshot(gateway, req.params.name, ns)
        : null
      const updateBody = isCommunicationChannelUpdate
        ? {
            ...body,
            metadata: {
              ...body.metadata,
              ...(previousCommunicationChannelSpec?.uid
                ? { uid: previousCommunicationChannelSpec.uid }
                : {}),
              ...(body.metadata?.resourceVersion ||
              !previousCommunicationChannelSpec?.resourceVersion
                ? {}
                : { resourceVersion: previousCommunicationChannelSpec.resourceVersion }),
            },
            spec: await preserveCommunicationChannelCredentialsSecretRef(
              gateway,
              req.params.name,
              ns,
              body.spec,
              previousCommunicationChannelSpec?.spec ?? null
            ),
          }
        : body
      // A null snapshot means the channel does not exist (loadCommunicationChannelSpecSnapshot
      // returns null only on 404). Skipping the guard here is what makes the response
      // honest: with no previous spec every provider reads as newly added, so the guard
      // would answer a PUT to a missing channel with `credentials[...] is required`
      // instead of the 404 updateResource is about to produce.
      if (isCommunicationChannelUpdate && previousCommunicationChannelSpec) {
        // updateBody.spec, not body.spec: the ref the write will actually carry
        // is the preserved one, and that is the Secret whose keys decide whether
        // a newly enabled provider can work. Reuses the snapshot already loaded
        // above — no second fetch.
        const transitionError = await providerTransitionError(
          gateway,
          updateBody.spec,
          previousCommunicationChannelSpec?.spec ?? null,
          undefined,
          ns
        )
        if (transitionError) {
          res.status(400).json({ error: transitionError })
          return
        }
      }
      try {
        // Only the ratchet path (hosts/contexts) has a pre-read to hand off; the
        // CC/mcpservers paths call with the original 4-arg arity untouched.
        const updated = ratchetCurrent
          ? await gateway.updateResource(plural, req.params.name, updateBody, ns, {
              preReadCurrent: ratchetCurrent,
            })
          : await gateway.updateResource(plural, req.params.name, updateBody, ns)
        // Host CR persisted: NOW emit any Pieza D tolerations (never before the
        // write lands, so a K8s conflict/error leaves no audit record).
        for (const event of hostTolerations) emitHostSpecIncoherenceTolerated(event)
        if (plural === 'communicationchannels' && body.spec) {
          const missingField = missingPersistedProviderSetting(updateBody.spec, updated)
          if (missingField) {
            await rollbackPrunedCommunicationChannelUpdate(
              gateway,
              req.params.name,
              ns,
              previousCommunicationChannelSpec,
              resourcePreconditionsFromResource(updated)
            )
            sendPrunedProviderSettingError(res, missingField)
            return
          }
        }
        if (
          plural === 'contexts' &&
          body.spec &&
          contextDisplayNameNotPersisted(body.spec, updated)
        ) {
          // Deliberately NO spec restore (unlike the CC update rollback): the
          // context's other spec fields persisted legitimately, and a PUT is
          // idempotent — once the operator applies the CRD and replays the same
          // request, spec.displayName round-trips and the whole spec converges.
          // Restoring the prior spec would instead discard those legitimate
          // edits. The loud 409 is the recovery signal.
          sendPrunedDisplayNameError(res)
          return
        }
        res
          .status(200)
          .json(
            hostWarnings.length > 0
              ? { ...(updated as Record<string, unknown>), warnings: hostWarnings }
              : updated
          )
      } catch (err) {
        // AP-6: the caller sent the resourceVersion it read and the resource
        // changed underneath it. Machine-readable so the UI can tell the
        // operator to reload — never retried server-side with the stale body.
        if (err instanceof K8sConflictError) {
          res.status(409).json({ error: 'conflict', reason: 'resource_changed' })
          return
        }
        throw err
      }
    })
  )

  router.delete(
    `${RESOURCE_PATTERN}/:name`,
    asyncHandler(async (req, res) => {
      const { plural, ns } = resolveResource(req.params.resource)!
      const name = req.params.name

      if (plural === 'mcpservers') {
        const outcome = await uninstallMcpServer(
          {
            gateway,
            db: dcrDb,
            encryptionKey: oauthEncryptionKey,
            dcrDeps: { logger: log },
            contextsNamespace: resourceNamespace('contexts'),
            logger: log,
          },
          { name, namespace: ns }
        )
        if (outcome.status === 'completed') {
          res.status(200).json(outcome.crDeleteResponse)
          return
        }
        if (outcome.status === 'incomplete') {
          res.status(503).json({
            error: 'mcp_server_uninstall_incomplete',
            outcome: 'repair_required',
            pending: outcome.pending,
            deleted: outcome.deleted,
          })
          return
        }
        if (outcome.status === 'not_found') {
          // Every cleanup runs before the CR delete, so a missing CR means "already clean"
          // or "deleted out of band". Cleaning by name here would have no uid to fence
          // by: it could delete a CR a reinstall recreated after this read, or the Secret
          // a pre-registered reinstall creates before its CR.
          throw new K8sNotFoundError(`mcpservers/${name} not found`)
        }
        const unhandled: never = outcome
        throw new Error(`unhandled mcp-server uninstall outcome: ${JSON.stringify(unhandled)}`)
      }

      // CommunicationChannel: read the credentialsSecretRef name FIRST (we
      // can't read the CC after it's deleted), then delete the CC, then
      // delete the Secret. The CC-first order means HCC's SecretInformer
      // observes the Secret DELETED event when no CC references it anymore,
      // so reconcileChannelReaderRevision short-circuits cleanly with zero
      // CCs — no transient "patch with sha256 of empty data" annotation
      // roll. Tradeoff: if Secret delete fails after CC delete, an orphan
      // Secret survives. That's harmless (Secrets without referencing CCs
      // do nothing) and easier to clean up than an orphan CC.
      let ccSecretRefName: string | undefined
      if (plural === 'communicationchannels') {
        try {
          const cc = (await gateway.getResource(plural, name, ns)) as {
            spec?: { credentialsSecretRef?: { name?: string } }
          }
          ccSecretRefName = cc.spec?.credentialsSecretRef?.name
        } catch (err) {
          if (extractK8sStatusCode(err) !== 404) {
            throw err
          }
        }
      }

      // Capture the credential Secret identity BEFORE deleting the parent CR: a
      // same-name Secret created in the gap belongs to a different owner, and a
      // name-addressed delete afterwards would remove it.
      const ccSecretCleanup =
        plural === 'communicationchannels' && ccSecretRefName
          ? await captureSecretForCleanup(gateway, ccSecretRefName, ns, log)
          : null

      const deleted = await gateway.deleteResource(plural, name, ns)

      if (plural === 'communicationchannels' && ccSecretRefName && ccSecretCleanup) {
        if (ccSecretCleanup.status !== 'ready') {
          log.warn(
            { secretName: ccSecretRefName, namespace: ns, reason: ccSecretCleanup.status },
            'Skipped CommunicationChannel credentials Secret cleanup'
          )
        } else {
          try {
            await gateway.deleteSecret(ccSecretRefName, ns, ccSecretCleanup.precondition)
            log.info(
              { secretName: ccSecretRefName, namespace: ns },
              'Deleted CommunicationChannel credentials Secret'
            )
          } catch (err) {
            if (extractK8sStatusCode(err) === 404) {
              log.info(
                { secretName: ccSecretRefName, namespace: ns },
                'CommunicationChannel credentials Secret already gone'
              )
            } else {
              // CC is already gone; log and swallow so the operator gets a 200
              // and can clean the orphan Secret manually if needed.
              log.error(
                { secretName: ccSecretRefName, namespace: ns, err },
                'CommunicationChannel delete succeeded but credentials cleanup failed'
              )
            }
          }
        }
      }

      res.status(200).json(deleted)
    })
  )

  registerCommunicationChannelCredentialsRoutes(router, gateway)

  // ── LlmHook guardrail CRs: read + delete only (see LLMHOOK_PATTERN) ──────────
  router.use(LLMHOOK_PATTERN, (req: Request, _res: Response, next: NextFunction) => {
    const resolved = resolveResource(req.params.resource)
    if (resolved) {
      enforceNamespace(resolved.ns)(req, _res, next)
    } else {
      next()
    }
  })

  // Cluster-wide list (control-ui hooks dashboard).
  router.get(
    LLMHOOK_PATTERN,
    asyncHandler(async (req, res) => {
      const { plural, ns } = resolveResource(req.params.resource)!
      const items = await gateway.listResource(plural, ns)
      res.status(200).json({ items })
    })
  )

  // Single hook (status detail / digest drift).
  router.get(
    `${LLMHOOK_PATTERN}/:name`,
    asyncHandler(async (req, res) => {
      const { plural, ns } = resolveResource(req.params.resource)!
      const resource = await gateway.getResource(plural, req.params.name, ns)
      res.status(200).json(resource)
    })
  )

  // Uninstall. Referential integrity (§8.2): strip the `{id}` reference from every
  // referencing Host's guardrails.hooks FIRST, so no dangling ref is ever visible
  // (mcp-host can't fail-closed on a vanished CR — N11), THEN delete the LlmHook
  // CR; host-context-controller garbage-collects the shared pod once the last
  // referencing hook is gone.
  router.delete(
    `${LLMHOOK_PATTERN}/:name`,
    asyncHandler(async (req, res) => {
      const { plural, ns } = resolveResource(req.params.resource)!
      const unlinkedHosts = await stripHookRefFromHosts(
        gateway,
        req.params.name,
        config.hostsNamespace
      )
      const deleted = await gateway.deleteResource(plural, req.params.name, ns)
      res.status(200).json({ ...(deleted as object), unlinkedHosts })
    })
  )

  return router
}