Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
41.4 KB
Raw Blame
import { z } from 'zod'
import {
  type LlmProviderId,
  PROVIDER_AUTH_MODE,
  PROVIDER_CREDENTIAL_SLOTS,
  isLlmProviderId,
} from '@clerum/llm-providers'
import type { DbClient } from '../../db.js'
import { rootLogger } from '../../observability/logger.js'
import { isCodexAssignmentAllowed } from '../../services/codexSubscriptionCatalog.js'
import {
  CODEX_UNASSIGNED_CONNECTION_KEY,
  isCodexUnassignedConnectionKey,
  readHostCodexConnectionRef,
} from '../../services/codexSubscriptionConnection.js'
import { isGrokAssignmentAllowed } from '../../services/grokSubscriptionCatalog.js'
import {
  GROK_UNASSIGNED_CONNECTION_KEY,
  isGrokUnassignedConnectionKey,
  readHostGrokConnectionRef,
} from '../../services/grokSubscriptionConnection.js'
import {
  getModelAllowlistState as getModelAllowlistStateDefault,
  isModelAllowed as isModelAllowedDefault,
} from '../../services/llmAllowedModels.js'
import { isPlainObject } from '../../utils/isPlainObject.js'
import { enumerateHostModelReferences } from './hostModelReferences.js'
import {
  type HostSpecIncoherenceToleratedEvent,
  type IncoherenceToleranceGate,
} from './hostWriteGateAudit.js'
import {
  type CoverageSet,
  isNonWorseningToleration,
  offeredKey,
} from './modelAllowlistTolerance.js'
import { STALE_MODEL_ASSIGNED, type StaleModelWarning } from './staleModelWarning.js'

const logger = rootLogger.child({ module: 'admin-host-spec-validation' })

// A Kubernetes Secret/ConfigMap data key: `[-._a-zA-Z0-9]`, max 253 chars.
// A fallback `credentialSlot` names a key inside the chatllm-api-keys Secret, so
// it must satisfy that format. We validate the FORMAT only — the key's existence
// in the Secret is resolved by mcp-host at runtime (a cheap DB-free write gate
// cannot see the Secret's contents; spec §3-R5.3). Same pattern as recipes.ts.
const SECRET_KEY_RE = /^[-._a-zA-Z0-9]+$/

/**
 * Whether a provider may carry a per-fallback `credentialSlot` (an extra
 * single-line Secret key that overrides its one API key). Only providers whose
 * SOLE credential slot is a single-line API key qualify. A multi-slot provider
 * (Bedrock: access-key-id + secret-access-key) or a single-slot provider whose
 * key is the multiline service-account JSON (Vertex: `vertex-service-account-json`
 * → `VERTEX_SERVICE_ACCOUNT_JSON`) cannot be expressed by one extra key, so those
 * fallbacks must reuse the primary's credentials. Derived entirely from the
 * shared registry so adding a provider needs no change here.
 */
function providerSupportsFallbackCredentialSlot(provider: LlmProviderId): boolean {
  const slots = PROVIDER_CREDENTIAL_SLOTS[provider]
  // A per-fallback credentialSlot is ONE single-line API key. A multi-slot
  // provider (Bedrock's pair) or a multi-line slot (Vertex's service-account
  // JSON) can't be expressed as one extra key. `multiline` is the explicit
  // registry flag (replaces the old `-json`/`_JSON` name heuristic).
  return slots.length === 1 && !slots[0].multiline
}

function isCodexSubscriptionEnabled(): boolean {
  return process.env.CONTROL_API_CODEX_SUBSCRIPTION_ENABLED === 'true'
}

function isGrokSubscriptionEnabled(): boolean {
  return process.env.CONTROL_API_GROK_SUBSCRIPTION_ENABLED === 'true'
}

function oauthBrokerFlagEnabled(provider: string): boolean {
  if (provider === 'codex-subscription') return isCodexSubscriptionEnabled()
  if (provider === 'grok-subscription') return isGrokSubscriptionEnabled()
  return true
}

function isOauthBrokerProviderId(provider: string): provider is LlmProviderId {
  return isLlmProviderId(provider) && PROVIDER_AUTH_MODE[provider] === 'oauth-broker'
}

function readHostBrokerConnectionRef(provider: LlmProviderId, value: unknown): string {
  const raw = typeof value === 'string' ? value : null
  return provider === 'grok-subscription'
    ? readHostGrokConnectionRef(raw)
    : readHostCodexConnectionRef(raw)
}

function isUnassignedBrokerConnectionRef(provider: LlmProviderId, connectionRef: string): boolean {
  return provider === 'grok-subscription'
    ? isGrokUnassignedConnectionKey(connectionRef)
    : isCodexUnassignedConnectionKey(connectionRef)
}

/**
 * The Host's single oauth-broker grant, resolved ONCE per write and shared by the
 * primary, per-host subset (allowedModels) and fallback gates (RP-001). Admission
 * (`validateCodexBrokerAdmission`) already guarantees at most one broker id per
 * Host, so the provider is the primary's when it is a broker, else the one broker
 * named by a fallback/subset entry. The grant key always lives in
 * `spec.model.connectionRef` (empty/missing reads as `unassigned`). `null` means
 * the Host references no oauth-broker target.
 */
type HostBrokerContext = { provider: LlmProviderId; connectionRef: string } | null

function resolveHostBrokerContext(spec: Record<string, unknown>): HostBrokerContext {
  const model = isPlainObject(spec.model) ? spec.model : undefined
  const primary = typeof model?.provider === 'string' ? model.provider.trim() : ''
  const provider = isOauthBrokerProviderId(primary)
    ? primary
    : collectHostLlmTargets(spec)
        .map(target => target.provider)
        .find(isOauthBrokerProviderId)
  if (!provider) return null
  return { provider, connectionRef: readHostBrokerConnectionRef(provider, model?.connectionRef) }
}

/**
 * Allowlist/grant gate for one `(provider, model)` entry. A broker entry is
 * checked against the Host's grant key (skipped when the Host is `unassigned`:
 * an unassigned Host cannot spend, so there is nothing to validate). A static
 * entry never receives a grant key.
 */
async function isHostEntryAllowed(
  deps: HostSpecValidationDeps,
  broker: HostBrokerContext,
  provider: string,
  model: string
): Promise<boolean> {
  if (!isOauthBrokerProviderId(provider)) return deps.isModelAllowed(provider, model, undefined)
  // Admission guarantees broker?.provider === provider here; a mismatch fails
  // closed by letting the deps resolve a missing key as unassigned.
  const connectionRef = broker?.provider === provider ? broker.connectionRef : undefined
  if (connectionRef !== undefined && isUnassignedBrokerConnectionRef(provider, connectionRef)) {
    return true
  }
  return deps.isModelAllowed(provider, model, connectionRef)
}

type HostLlmTarget = {
  provider: string
  field: string
  credentialSlot?: unknown
}

function collectHostLlmTargets(spec: Record<string, unknown>): HostLlmTarget[] {
  const targets: HostLlmTarget[] = []
  if (isPlainObject(spec.model) && typeof spec.model.provider === 'string') {
    targets.push({
      provider: spec.model.provider.trim(),
      field: 'spec.model.provider',
    })
  }
  if (Array.isArray(spec.allowedModels)) {
    spec.allowedModels.forEach((entry, i) => {
      if (isPlainObject(entry) && typeof entry.provider === 'string') {
        targets.push({
          provider: entry.provider.trim(),
          field: `spec.allowedModels[${i}].provider`,
        })
      }
    })
  }
  if (isPlainObject(spec.llmPolicy) && Array.isArray(spec.llmPolicy.fallbacks)) {
    spec.llmPolicy.fallbacks.forEach((entry, i) => {
      if (!isPlainObject(entry) || typeof entry.provider !== 'string') return
      targets.push({
        provider: entry.provider.trim(),
        field: `spec.llmPolicy.fallbacks[${i}].provider`,
        credentialSlot: entry.credentialSlot,
      })
    })
  }
  return targets
}

function validateCodexBrokerAdmission(
  spec: Record<string, unknown>
): { errors: Array<{ field: string; message: string }> } | null {
  const targets = collectHostLlmTargets(spec)
  for (const target of targets) {
    if (
      !isLlmProviderId(target.provider) ||
      PROVIDER_AUTH_MODE[target.provider] !== 'oauth-broker'
    ) {
      continue
    }
    if (!oauthBrokerFlagEnabled(target.provider)) {
      const flag =
        target.provider === 'grok-subscription'
          ? 'CONTROL_API_GROK_SUBSCRIPTION_ENABLED'
          : 'CONTROL_API_CODEX_SUBSCRIPTION_ENABLED'
      return {
        errors: [
          {
            field: target.field,
            message: `provider "${target.provider}" is disabled (set ${flag}=true)`,
          },
        ],
      }
    }
    if (target.credentialSlot !== undefined) {
      return {
        errors: [
          {
            field: target.field.replace('.provider', '.credentialSlot'),
            message:
              'credentialSlot is not supported for oauth-broker providers; drop credentialSlot',
          },
        ],
      }
    }
  }

  const brokerIds = [
    ...new Set(
      targets
        .map(target => target.provider)
        .filter(
          (provider): provider is string =>
            isLlmProviderId(provider) && PROVIDER_AUTH_MODE[provider] === 'oauth-broker'
        )
    ),
  ]
  if (brokerIds.length > 1) {
    return {
      errors: [
        {
          field: 'spec.model.provider',
          message:
            'a Host may declare at most one oauth-broker provider across model, fallbacks, and allowedModels',
        },
      ],
    }
  }
  const hasBroker = brokerIds.length > 0
  const hasStatic = targets.some(
    target =>
      isLlmProviderId(target.provider) &&
      PROVIDER_AUTH_MODE[target.provider] === 'static-credentials'
  )
  const secretRef = typeof spec.secretRef === 'string' ? spec.secretRef.trim() : ''
  if (hasBroker && hasStatic && !secretRef) {
    return {
      errors: [
        {
          field: 'spec.secretRef',
          message: 'spec.secretRef is required when any static-credentials LLM target is present',
        },
      ],
    }
  }
  return null
}

/**
 * Array caps mirrored from charts/clerum-crds/crds/host.yaml `maxItems`
 * (`spec.allowedModels` and `spec.llmPolicy.fallbacks`; parity pinned by
 * test/hostSpecValidation.limits.test.ts). control-api answers a field-level 422
 * instead of surfacing the apiserver's schema rejection.
 */
export const HOST_ALLOWED_MODELS_MAX_ITEMS = 32
export const HOST_LLM_POLICY_FALLBACKS_MAX_ITEMS = 8

function hostLlmPolicyFallbacks(spec: Record<string, unknown> | undefined): unknown {
  return isPlainObject(spec) && isPlainObject(spec.llmPolicy) ? spec.llmPolicy.fallbacks : undefined
}

/**
 * Reject an array over its CRD cap only when this write GREW it versus the
 * stored Host: a Host persisted before the cap existed may still be saved
 * unchanged or shrunk (no-worsening, like the allowlist tolerance). On create
 * there is no stored array, so any over-cap array is rejected.
 */
function validateHostArrayCaps(
  spec: Record<string, unknown>,
  stored: Record<string, unknown> | undefined
): { errors: Array<{ field: string; message: string }> } | null {
  const caps: Array<{ field: string; max: number; incoming: unknown; before: unknown }> = [
    {
      field: 'spec.allowedModels',
      max: HOST_ALLOWED_MODELS_MAX_ITEMS,
      incoming: spec.allowedModels,
      before: isPlainObject(stored) ? stored.allowedModels : undefined,
    },
    {
      field: 'spec.llmPolicy.fallbacks',
      max: HOST_LLM_POLICY_FALLBACKS_MAX_ITEMS,
      incoming: hostLlmPolicyFallbacks(spec),
      before: hostLlmPolicyFallbacks(stored),
    },
  ]
  for (const cap of caps) {
    if (!Array.isArray(cap.incoming) || cap.incoming.length <= cap.max) continue
    const storedLength = Array.isArray(cap.before) ? cap.before.length : 0
    if (cap.incoming.length <= storedLength) continue
    return {
      errors: [{ field: cap.field, message: `${cap.field} must contain at most ${cap.max} items` }],
    }
  }
  return null
}

const HostApprovalSchema = z
  .object({
    defaultPolicy: z.string().optional(),
    channels: z.any().optional(),
    tools: z.record(z.string(), z.boolean()).optional(),
  })
  .passthrough()
  .optional()

export interface HostSpecValidationDeps {
  isModelAllowed: (provider: string, model: string, connectionRef?: string) => Promise<boolean>
  /**
   * Full `{ enabled, stale }` allowlist state for a pair (Fase 6). OPTIONAL and
   * defaulted so existing callers that inject only `isModelAllowed` keep working;
   * it is consulted ONLY when `context.warnings` is supplied (the stale-warning
   * sink), so a caller not requesting warnings never triggers this lookup.
   */
  getModelAllowlistState?: (
    provider: string,
    model: string
  ) => Promise<{ enabled: boolean; stale: boolean } | null>
}

/**
 * Shared Host-write deps: Codex assignments check the grant catalog, not the
 * global allowlist. Bind/unbind and Host PATCH must use this factory so a
 * second writer cannot drop `connectionRef` or treat it as a DbClient.
 */
export function createHostValidationDeps(db: DbClient): HostSpecValidationDeps {
  return {
    isModelAllowed: (provider, model, connectionRef) =>
      provider === 'codex-subscription'
        ? isCodexAssignmentAllowed(db, readHostCodexConnectionRef(connectionRef), model)
        : provider === 'grok-subscription'
          ? isGrokAssignmentAllowed(db, readHostGrokConnectionRef(connectionRef), model)
          : isModelAllowedDefault(provider, model, db),
    getModelAllowlistState: (provider, model) => getModelAllowlistStateDefault(provider, model, db),
  }
}

/**
 * Per-write context for the no-worsening tolerance (Pieza D). ABSENT on create
 * (there is no stored CR, so tolerance can never apply — a fresh spec may not
 * introduce a disabled model). Supplied on update by the admin facade, which
 * reads the stored Host CR before the write.
 */
export interface HostSpecValidationContext {
  /** The stored Host CR spec read before this write; absent on create. */
  stored?: Record<string, unknown>
  /** Host identity for the audit event payload. */
  hostRef?: { namespace: string; name: string }
  /**
   * OUTPUT sink. When the write is ACCEPTED, the validator appends here the
   * audit events for each tolerated gate. It does NOT emit them: the caller
   * emits only AFTER the write persists (mini-spec 01), so a write that a later
   * step (secretRef check, K8s conflict) rejects leaves no audit record.
   */
  tolerations?: HostSpecIncoherenceToleratedEvent[]
  /**
   * OUTPUT sink for Fase 6 soft-quarantine warnings. When present, the validator
   * appends a warning for every NEW assignment of an `enabled` but `stale` model
   * (a live reference in `stored` is never revalidated). Same emit-after-persist
   * discipline as `tolerations`: the validator only fills the sink on ACCEPT; the
   * caller returns it in the success body only after the write persists — a write
   * rejected by a later gate carries no warning. Independent of `stored`: warnings
   * apply on both create (all assignments new) and update.
   */
  warnings?: StaleModelWarning[]
}

// spec.lifecycle follows the facade's contract for optional spec objects,
// exactly like spec.desktop: the admin facade forwards body.spec to the K8s
// client WHOLESALE (resourceService.updateResource is a full replace via
// replaceNamespacedCustomObject), so an update payload WITHOUT lifecycle
// strips it from the CR — absent means disabled. Control UI must always echo
// the full spec it read (get -> edit -> update); the round-trip regression in
// test/routes.resources.test.ts pins that guarantee. When present, lifecycle
// must be an object carrying a boolean `stateless`; unknown extra keys pass
// through, matching the HostApprovalSchema idiom above.
//
// AP-6 (docs/architecture/stateless-invariants.md): the echo alone cannot
// protect against a STALE echo — a form saved after the CR changed would
// replace the fresh spec with what the form read earlier. Control UI
// therefore also sends metadata.resourceVersion of the read the edit form
// was built from; resourceService.updateResource uses it as the replace
// precondition and surfaces 409 {error:'conflict', reason:'resource_changed'}
// instead of retrying with the stale payload.
const HostLifecycleSchema = z
  .object({
    stateless: z.boolean(),
  })
  .passthrough()
  .optional()

/**
 * Validate a Host spec before it reaches the K8s API.
 *
 * - `spec.approval`: defensive zod shape check (unchanged behavior).
 * - `spec.lifecycle` (stateless agents): optional object carrying a boolean
 *   `stateless`; absent means disabled (the admin facade full-replaces the CR,
 *   so Control UI must echo the full spec it read).
 * - `spec.model` (R3): when a model name is present it must exist AND be enabled
 *   in the operator allowlist under its declared provider — fail-closed for new
 *   selections. A spec without `spec.model.name` is left untouched (deployed
 *   Hosts are never disrupted; enforcement applies to create/edit via this API).
 * - `spec.llmPolicy` (R5): opt-in provider-fallback policy; every fallback entry
 *   must reference a known provider and an allowlisted model.
 * - `spec.allowedModels` (Topic 3a): optional/additive per-host SUBSET of the
 *   global allowlist this host offers. Absent/empty = the host offers the FULL
 *   global allowlist (back-compat). When present, every entry must be a global
 *   allowlist pair, and the primary (`spec.model.name`) plus every fallback must
 *   fall WITHIN the offered subset (coherence — a host cannot serve what it does
 *   not offer).
 *
 * `isModelAllowed` is injectable so this stays unit-testable without a DB.
 */
export async function validateHostSpec(
  spec: Record<string, unknown>,
  deps: HostSpecValidationDeps = {
    isModelAllowed: (provider, model) => isModelAllowedDefault(provider, model),
    getModelAllowlistState: getModelAllowlistStateDefault,
  },
  context: HostSpecValidationContext = {}
): Promise<{ errors: Array<{ field: string; message: string }> } | null> {
  // Contract guard (mini-spec 01): passing `stored` enables tolerance, so the
  // caller MUST also pass the `tolerations` sink — otherwise a tolerated pair
  // would be accepted while its audit event is silently dropped ("never silent"
  // violated). Couple them: a caller that supplies `stored` without a sink is a
  // programming error, not a silent degrade.
  if (context.stored !== undefined && context.tolerations === undefined) {
    throw new Error(
      'validateHostSpec: context.tolerations sink is required when context.stored is provided (a tolerated pair must never be emitted silently)'
    )
  }

  // Structural (sync, zod) checks first: accumulate approval + lifecycle shape
  // errors so a payload bad in both surfaces both — dev's stateless contract
  // pinned by test/routes.resources.test.ts.
  const structuralErrors: Array<{ field: string; message: string }> = []
  if (spec.approval !== undefined) {
    const result = HostApprovalSchema.safeParse(spec.approval)
    if (!result.success) {
      for (const issue of result.error.issues) {
        structuralErrors.push({
          field: ['spec', 'approval', ...issue.path].join('.'),
          message: issue.message,
        })
      }
    }
  }
  if (spec.lifecycle !== undefined) {
    const result = HostLifecycleSchema.safeParse(spec.lifecycle)
    if (!result.success) {
      for (const issue of result.error.issues) {
        structuralErrors.push({
          field: ['spec', 'lifecycle', ...issue.path].join('.'),
          message: issue.message,
        })
      }
    }
  }
  if (structuralErrors.length > 0) return { errors: structuralErrors }

  const capErrors = validateHostArrayCaps(spec, context.stored)
  if (capErrors) return capErrors

  const brokerErrors = validateCodexBrokerAdmission(spec)
  if (brokerErrors) return brokerErrors
  const broker = resolveHostBrokerContext(spec)

  // No-worsening tolerance context (Pieza D), computed once and shared by the 3
  // global-allowlist gates below. On create `context.stored` is absent, so
  // `storedPairs` is empty and no gate can tolerate — a fresh spec may never
  // introduce a disabled model.
  const tol: HostToleranceBundle = {
    context,
    deps,
    storedRoles: storedRoleSets(context.stored),
    incomingCov: hostCoverage(spec),
    storedCov: hostCoverage(context.stored),
    pending: [],
    warnings: [],
    warnedKeys: new Set(),
  }

  // R3 model allowlist. Enforcement is keyed on the PRESENCE of `spec.model.name`
  // (not just a truthy string) so a non-string/garbage name cannot slip past the
  // gate — that is strictly fail-closed. A spec with no `name` key at all is left
  // untouched (deployed Hosts are never disrupted).
  const model = spec.model
  if (isPlainObject(model) && Object.prototype.hasOwnProperty.call(model, 'name')) {
    const name = typeof model.name === 'string' ? model.name.trim() : ''
    if (!name) {
      return {
        errors: [
          { field: 'spec.model.name', message: 'spec.model.name must be a non-empty string' },
        ],
      }
    }
    const provider = typeof model.provider === 'string' ? model.provider.trim() : ''
    if (!provider) {
      // The allowlist is keyed by (provider, model); a bare model name cannot be
      // validated. Ask for the provider explicitly instead of routing an empty
      // provider through the lookup (which would reject with a confusing message).
      return {
        errors: [
          {
            field: 'spec.model.provider',
            message: 'spec.model.provider is required when spec.model.name is set',
          },
        ],
      }
    }
    const primaryIsBroker = isOauthBrokerProviderId(provider)
    const connectionRef =
      primaryIsBroker && broker?.provider === provider ? broker.connectionRef : undefined
    if (
      primaryIsBroker &&
      connectionRef !== undefined &&
      isUnassignedBrokerConnectionRef(provider, connectionRef)
    ) {
      // Persist the sentinel so empty and unassigned are not two Host states.
      model.connectionRef =
        provider === 'grok-subscription'
          ? GROK_UNASSIGNED_CONNECTION_KEY
          : CODEX_UNASSIGNED_CONNECTION_KEY
    }
    const allowed = await isHostEntryAllowed(deps, broker, provider, name)
    // Switching connectionRef is a new assignment even when (provider, model)
    // already lived on the Host. A revoked/unknown grant must not ride the
    // identity/channels full-replace tolerance. An unchanged revoked ref stays
    // tolerable so operators can still save unrelated fields.
    const storedConnectionRef = storedBrokerConnectionRef(context.stored)
    const connectionAssignmentChanged =
      primaryIsBroker && storedConnectionRef !== undefined && storedConnectionRef !== connectionRef
    if (
      !allowed &&
      (connectionAssignmentChanged || !toleratePair(provider, name, 'primary', tol))
    ) {
      return {
        errors: [
          {
            field: 'spec.model.name',
            message: `model_not_allowed: "${name}" is not enabled in the allowlist for provider "${provider}"`,
          },
        ],
      }
    }
    // Fase 6: an ENABLED pair that passed the gate may still be `stale`. Warn (no
    // block) if this is a NEW assignment.
    if (allowed) await maybeWarnStale(provider, name, 'spec.model.name', tol)
  }

  // R5 provider-fallback policy. Opt-in and additive: a spec with no `llmPolicy`
  // is untouched. When present, EVERY fallback entry must reference a real
  // provider and a model enabled in the operator allowlist under that provider
  // (same fail-closed gate as spec.model.name), so a broken fallback is caught
  // on write instead of surfacing only during the incident it was meant to
  // absorb (spec V16). `credentialSlot`, when present, is format-checked only.
  const policyErrors = await validateLlmPolicy(spec.llmPolicy, deps, tol, broker)
  if (policyErrors) return policyErrors

  // Topic 3a per-host allowlist. Runs AFTER the global-allowlist gates above, so
  // by the time coherence is checked the primary + fallbacks are already known
  // to be valid GLOBAL pairs; this narrows them to the host's offered subset.
  const allowedModelsErrors = await validateAllowedModels(spec, deps, tol, broker)
  if (allowedModelsErrors) return allowedModelsErrors

  // The write PASSED validation: hand the queued tolerations to the caller via
  // the context sink. The caller emits them only AFTER the write persists — never
  // here, so a later rejection (secretRef check, K8s conflict) leaves no audit
  // record (mini-spec 01, persist-ordering fix).
  if (context.tolerations) context.tolerations.push(...tol.pending)

  // Fase 6: hand the queued soft-quarantine warnings to the caller via the sink,
  // on ACCEPT only — same persist-ordering as tolerations, so a write a later gate
  // rejects (secretRef check, K8s conflict) surfaces no warning.
  if (context.warnings) context.warnings.push(...tol.warnings)

  return null
}

/**
 * The stored `(provider, model)` pairs a Host spec references, ranked by ROLE
 * ACTIVENESS (mini-spec 01 v2 — role-scoped tolerance):
 *
 *   - `primary` — the ACTIVE slot (`spec.model`), the default mcp-host routes.
 *   - `any`     — every referenced pair, across primary + fallbacks + subset.
 *
 * Tolerance at the ACTIVE `primary` gate is strict (`P ∈ primary`), so a disabled
 * pair that was only a fallback/subset entry cannot be PROMOTED to the active
 * default. The non-active gates (fallback, subset) accept `P ∈ any`, so DEMOTING
 * a disabled pair from primary to a less-active role — pure non-worsening — is
 * tolerated. This mirrors the grant seam (`storedDefaultTarget` strict for the
 * default slot, `storedAnyTarget` for the rest), keeping Host and grants
 * symmetric (D4). Both sets are empty when there is no stored spec (create), so
 * tolerance never applies to a fresh Host.
 */
interface StoredRoleSets {
  /** The active slot: `spec.model`. */
  primary: ReadonlySet<string>
  /** Union of every referenced pair (primary ∪ fallbacks ∪ allowedModels). */
  any: ReadonlySet<string>
}

function storedRoleSets(stored: Record<string, unknown> | undefined): StoredRoleSets {
  const primary = new Set<string>()
  const any = new Set<string>()
  // Location enumeration is shared verbatim with the live impact gate
  // (`enumerateHostModelReferences`, regla D4). This caller's own concern —
  // layered on top — is bucketing every referenced key into `any` and only the
  // `primary`-role keys into `primary`, so the active default slot stays strict
  // while non-active roles (fallback/subset) tolerate demotion.
  for (const ref of enumerateHostModelReferences(stored)) {
    any.add(ref.key)
    if (ref.role === 'primary') primary.add(ref.key)
  }
  return { primary, any }
}

/**
 * Coverage a Host spec offers, for condition (b). `spec.allowedModels` absent,
 * non-array, or EMPTY means "offers the full global allowlist" (Topic 3a
 * back-compat), represented by the `'UNIVERSAL'` sentinel; a non-empty array is
 * the finite set of offered `(provider, model)` keys.
 */
function hostCoverage(spec: Record<string, unknown> | undefined): CoverageSet {
  if (!isPlainObject(spec)) return 'UNIVERSAL'
  const allowedModels = spec.allowedModels
  if (!Array.isArray(allowedModels) || allowedModels.length === 0) return 'UNIVERSAL'
  const keys = new Set<string>()
  for (const entry of allowedModels) {
    if (!isPlainObject(entry)) continue
    const provider = typeof entry.provider === 'string' ? entry.provider.trim() : ''
    const model = typeof entry.model === 'string' ? entry.model.trim() : ''
    if (provider && model) keys.add(offeredKey(provider, model))
  }
  return keys
}

/**
 * Shared no-worsening decision for a Host write gate. Returns `true` (and emits
 * the audit event) when the disallowed `(provider, model)` pair may be tolerated
 * because the write does not worsen a pre-existing incoherence.
 */

/** The precomputed no-worsening tolerance inputs, shared by the 3 Host gates. */
interface HostToleranceBundle {
  context: HostSpecValidationContext
  deps: HostSpecValidationDeps
  storedRoles: StoredRoleSets
  incomingCov: CoverageSet
  storedCov: CoverageSet
  /**
   * Audit events for tolerated gates, ACCUMULATED here and copied into
   * `context.tolerations` only once `validateHostSpec` confirms the write is
   * ACCEPTED (returns null). Never emitted by the validator — the caller emits
   * after the write persists. A write that a later gate hard-rejects (422)
   * leaves no audit record. One event per tolerated gate decision.
   */
  pending: HostSpecIncoherenceToleratedEvent[]
  /**
   * Fase 6 soft-quarantine warnings, ACCUMULATED here and copied into
   * `context.warnings` only on ACCEPT (same persist-ordering as `pending`).
   */
  warnings: StaleModelWarning[]
  /** Pair keys already warned, so a model in two roles warns at most once. */
  warnedKeys: Set<string>
}

/**
 * Fase 6 soft quarantine. Queue a NON-BLOCKING warning when an `enabled` but
 * `stale` model is assigned to something NEW. NO-OP unless `context.warnings` is
 * provided (opt-in sink), so callers that do not request warnings pay no lookup.
 * A pair already referenced by the STORED record is a live reference and is never
 * revalidated (spec §3.3) → no warning. Called ONLY on the `enabled` path (after
 * the gate accepted the pair), so a disabled/tolerated pair (Fase 2) never routes
 * here.
 */
async function maybeWarnStale(
  provider: string,
  model: string,
  field: string,
  tol: HostToleranceBundle
): Promise<void> {
  if (!tol.context.warnings) return
  const key = offeredKey(provider, model)
  if (tol.storedRoles.any.has(key)) return
  if (tol.warnedKeys.has(key)) return
  const getState = tol.deps.getModelAllowlistState ?? getModelAllowlistStateDefault
  // Best-effort: this is an EXTRA lookup (separate from the isModelAllowed gate
  // query) that ONLY feeds the additive stale warning. The PR invariant is that
  // the soft quarantine is additive and NEVER blocks a write (never 422/409/500).
  // A blip on this query (connection reset) must not turn a valid Host
  // create/update into an HTTP 500 — swallow it, proceed WITHOUT a warning, and
  // let the write persist. The correctness gate (isModelAllowed) is untouched: it
  // still propagates its failures and still hard-rejects a disallowed model.
  let state: { enabled: boolean; stale: boolean } | null
  try {
    state = await getState(provider, model)
  } catch (err) {
    // Structured (Pino) logging JSON-escapes field values — CR/LF included — so a
    // request-derived provider/model cannot forge or split a log record. Do NOT
    // interpolate these into a line-oriented sink (console.*). Best-effort:
    // swallow and proceed without a warning; behavior is unchanged.
    logger.warn(
      { provider, model, err: err instanceof Error ? err.message : String(err) },
      'stale-model warning lookup failed; proceeding without a warning'
    )
    return
  }
  if (state?.enabled && state.stale) {
    tol.warnedKeys.add(key)
    tol.warnings.push({ code: STALE_MODEL_ASSIGNED, provider, model, field })
  }
}

function storedBrokerConnectionRef(
  stored: Record<string, unknown> | undefined
): string | undefined {
  if (!isPlainObject(stored) || !isPlainObject(stored.model)) return undefined
  const provider = typeof stored.model.provider === 'string' ? stored.model.provider.trim() : ''
  if (!isOauthBrokerProviderId(provider)) return undefined
  return readHostBrokerConnectionRef(provider, stored.model.connectionRef)
}

/**
 * Decide whether a disallowed `(provider, model)` may be tolerated at `gate`
 * because the write does not worsen a pre-existing incoherence AND the pair does
 * not GAIN activeness (role-scoped, mini-spec 01 v2). On tolerance, QUEUES the
 * audit event (not emitted until the whole write persists) and returns true so
 * the gate skips its rejection.
 */
function toleratePair(
  provider: string,
  model: string,
  gate: 'primary' | 'fallback' | 'subset',
  tol: HostToleranceBundle
): boolean {
  // Role-scoped membership. The ACTIVE `primary` slot is strict: the pair must
  // have been the stored primary (blocks promotion). Non-active gates accept any
  // stored role (allows demotion — pure non-worsening).
  const storedRoleSet = gate === 'primary' ? tol.storedRoles.primary : tol.storedRoles.any
  const tolerated = isNonWorseningToleration({
    pairKey: offeredKey(provider, model),
    storedReferencedPairKeys: storedRoleSet,
    incomingCoverage: tol.incomingCov,
    storedCoverage: tol.storedCov,
  })
  if (!tolerated) return false
  tol.pending.push({
    resourceKind: 'host',
    namespace: tol.context.hostRef?.namespace ?? '',
    name: tol.context.hostRef?.name ?? '',
    provider,
    model,
    gate,
    offeredBefore: tol.storedCov,
    offeredAfter: tol.incomingCov,
  })
  return true
}

/**
 * Validate `spec.allowedModels[]` on write (Topic 3a). OPTIONAL/ADDITIVE: absent
 * returns null (host offers the full global allowlist). When present it is the
 * per-host SUBSET of the global allowlist this host offers:
 *
 *  1. every entry `(provider, model)` must be a known provider AND a pair enabled
 *     in the GLOBAL allowlist (same fail-closed gate as spec.model / R5); an
 *     offending entry answers 422 with a `spec.allowedModels[i]` field path;
 *  2. COHERENCE (only when the offered subset is non-empty): the primary
 *     `spec.model.name` and every `spec.llmPolicy.fallbacks[i].model` must fall
 *     WITHIN that offered subset - a host cannot serve a model it does not offer.
 *
 * An EMPTY array is treated exactly like absent (full global allowlist, coherence
 * skipped) so existing hosts are never disrupted.
 */
async function validateAllowedModels(
  spec: Record<string, unknown>,
  deps: HostSpecValidationDeps,
  tol: HostToleranceBundle,
  broker: HostBrokerContext
): Promise<{ errors: Array<{ field: string; message: string }> } | null> {
  const allowedModels = spec.allowedModels
  if (allowedModels === undefined) return null
  if (!Array.isArray(allowedModels)) {
    return {
      errors: [{ field: 'spec.allowedModels', message: 'spec.allowedModels must be an array' }],
    }
  }

  const offered = new Set<string>()
  for (let i = 0; i < allowedModels.length; i++) {
    const entry = allowedModels[i]
    const base = `spec.allowedModels[${i}]`
    if (!isPlainObject(entry)) {
      return { errors: [{ field: base, message: `${base} must be an object` }] }
    }

    const provider = typeof entry.provider === 'string' ? entry.provider.trim() : ''
    if (!provider) {
      return { errors: [{ field: `${base}.provider`, message: `${base}.provider is required` }] }
    }
    if (!isLlmProviderId(provider)) {
      return {
        errors: [
          {
            field: `${base}.provider`,
            message: `${base}.provider "${provider}" is not a known provider`,
          },
        ],
      }
    }

    const model = typeof entry.model === 'string' ? entry.model.trim() : ''
    if (!model) {
      return { errors: [{ field: `${base}.model`, message: `${base}.model is required` }] }
    }

    // Global-allowlist gate LAST (only async/DB check): a pair not in the global
    // catalog can never be offered by a host — unless tolerating a pre-existing
    // incoherence this write does not worsen (Pieza D). A tolerated pair is still
    // added to `offered` so the coherence gate below sees the host as offering it.
    const allowed = await isHostEntryAllowed(deps, broker, provider, model)
    if (!allowed && !toleratePair(provider, model, 'subset', tol)) {
      return {
        errors: [
          {
            field: `${base}.model`,
            message: `model_not_allowed: "${model}" is not enabled in the allowlist for provider "${provider}"`,
          },
        ],
      }
    }
    // Fase 6 soft quarantine on the per-host offered subset.
    if (allowed) await maybeWarnStale(provider, model, `${base}.model`, tol)

    offered.add(offeredKey(provider, model))
  }

  // Empty offered set (a literal `[]`) -> additive back-compat: full global
  // allowlist, no subset to violate, so the coherence gate is skipped.
  if (offered.size === 0) return null

  // COHERENCE - primary. Only enforced when `spec.model.name` is actually present
  // (a Host without a declared primary is left untouched, mirroring the R3 gate).
  // The `provider` truthiness guard is defensive only: the R3 gate above already
  // 422s a present `spec.model.name` that lacks a provider, so by here a present
  // name guarantees a present provider.
  const model = spec.model
  if (isPlainObject(model) && typeof model.name === 'string') {
    const name = model.name.trim()
    const provider = typeof model.provider === 'string' ? model.provider.trim() : ''
    if (name && provider && !offered.has(offeredKey(provider, name))) {
      return {
        errors: [
          {
            field: 'spec.model.name',
            message: `model_not_offered: primary model "${name}" (provider "${provider}") is not in spec.allowedModels`,
          },
        ],
      }
    }
  }

  // COHERENCE - fallbacks. Entries here are already known-good GLOBAL pairs
  // (validateLlmPolicy ran first); this narrows them to the offered subset.
  const llmPolicy = spec.llmPolicy
  if (isPlainObject(llmPolicy) && Array.isArray(llmPolicy.fallbacks)) {
    const { fallbacks } = llmPolicy
    for (let i = 0; i < fallbacks.length; i++) {
      const entry = fallbacks[i]
      if (!isPlainObject(entry)) continue
      const provider = typeof entry.provider === 'string' ? entry.provider.trim() : ''
      const fbModel = typeof entry.model === 'string' ? entry.model.trim() : ''
      if (provider && fbModel && !offered.has(offeredKey(provider, fbModel))) {
        return {
          errors: [
            {
              field: `spec.llmPolicy.fallbacks[${i}].model`,
              message: `model_not_offered: fallback "${fbModel}" (provider "${provider}") is not in spec.allowedModels`,
            },
          ],
        }
      }
    }
  }

  return null
}

/**
 * Validate `spec.llmPolicy.fallbacks[]` on write (spec §3-R5.3). Returns the
 * FIRST offending entry's error (mirroring the short-circuit style above) so the
 * caller can answer 422 with a field path like `spec.llmPolicy.fallbacks[1].model`.
 * Absent `llmPolicy` returns null (no failover configured). The CRD schema is the
 * structural authority; this is the semantic gate (allowlist + slot format) that
 * the apiserver cannot perform.
 */
async function validateLlmPolicy(
  llmPolicy: unknown,
  deps: HostSpecValidationDeps,
  tol: HostToleranceBundle,
  broker: HostBrokerContext
): Promise<{ errors: Array<{ field: string; message: string }> } | null> {
  if (llmPolicy === undefined) return null
  if (!isPlainObject(llmPolicy)) {
    return { errors: [{ field: 'spec.llmPolicy', message: 'spec.llmPolicy must be an object' }] }
  }

  const { fallbacks } = llmPolicy
  // `fallbacks` absent is left to the CRD schema (it is required there). We only
  // validate its entries when it is actually an array with content.
  if (fallbacks === undefined) return null
  if (!Array.isArray(fallbacks)) {
    return {
      errors: [
        { field: 'spec.llmPolicy.fallbacks', message: 'spec.llmPolicy.fallbacks must be an array' },
      ],
    }
  }

  for (let i = 0; i < fallbacks.length; i++) {
    const entry = fallbacks[i]
    const base = `spec.llmPolicy.fallbacks[${i}]`
    if (!isPlainObject(entry)) {
      return { errors: [{ field: base, message: `${base} must be an object` }] }
    }

    const provider = typeof entry.provider === 'string' ? entry.provider.trim() : ''
    if (!provider) {
      return { errors: [{ field: `${base}.provider`, message: `${base}.provider is required` }] }
    }
    if (!isLlmProviderId(provider)) {
      return {
        errors: [
          {
            field: `${base}.provider`,
            message: `${base}.provider "${provider}" is not a known provider`,
          },
        ],
      }
    }

    const model = typeof entry.model === 'string' ? entry.model.trim() : ''
    if (!model) {
      return { errors: [{ field: `${base}.model`, message: `${base}.model is required` }] }
    }

    if (entry.credentialSlot !== undefined) {
      const slot = typeof entry.credentialSlot === 'string' ? entry.credentialSlot : ''
      if (!slot || slot.length > 253 || !SECRET_KEY_RE.test(slot)) {
        return {
          errors: [
            {
              field: `${base}.credentialSlot`,
              message: `${base}.credentialSlot must be a valid Secret data key (matching ${SECRET_KEY_RE}, max 253 chars)`,
            },
          ],
        }
      }
      // Capability gate (residual 1b, decision 3): a fallback `credentialSlot`
      // names ONE single-line Secret key that overrides the provider's single API
      // key. It cannot express a multi-credential provider (Bedrock = access-key +
      // secret pair) or a multiline service-account JSON (Vertex) — those MUST
      // reuse the primary's credentials. `provider` is already narrowed to a known
      // LlmProviderId above, so the registry lookup is total.
      if (PROVIDER_AUTH_MODE[provider] === 'oauth-broker') {
        return {
          errors: [
            {
              field: `${base}.credentialSlot`,
              message: `${base}.credentialSlot is not supported for oauth-broker providers; drop credentialSlot`,
            },
          ],
        }
      }
      if (!providerSupportsFallbackCredentialSlot(provider)) {
        return {
          errors: [
            {
              field: `${base}.credentialSlot`,
              message: `${base}.credentialSlot is not supported for provider "${provider}": it uses multiple or JSON credentials and must reuse the primary's credentials (drop credentialSlot)`,
            },
          ],
        }
      }
    }

    // Allowlist gate LAST: it is the only async (DB) check, so cheap structural
    // rejections above avoid a needless query. Tolerated (Pieza D) when the pair
    // is a pre-existing incoherence this write does not worsen.
    const allowed = await isHostEntryAllowed(deps, broker, provider, model)
    if (!allowed && !toleratePair(provider, model, 'fallback', tol)) {
      return {
        errors: [
          {
            field: `${base}.model`,
            message: `model_not_allowed: "${model}" is not enabled in the allowlist for provider "${provider}"`,
          },
        ],
      }
    }
    // Fase 6 soft quarantine on a fallback assignment.
    if (allowed) await maybeWarnStale(provider, model, `${base}.model`, tol)
  }

  return null
}