Skip to content

Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active

63.3 KB
Raw Blame
import { logger } from '@librechat/data-schemas';
import { ErrorCode, McpError } from '@modelcontextprotocol/sdk/types.js';
import type { MCPClientCapabilityProfile } from './capabilities';
import type * as t from './types';
import {
  applyRequestHeaders,
  canUseAppConnection,
  canBackfillSharedServerInstructions,
  getMissingRuntimeBodyPlaceholderFields,
  hasRuntimeUrlPlaceholders,
  isUserSourced,
  requiresEphemeralUserConnection,
  requiresOAuthMachinery,
} from './utils';
import {
  cancelMCPToolsChanged,
  getMCPAppToolsPublicationGeneration,
  getMCPToolsChangedGeneration,
  notifyMCPToolsChanged,
  renewMCPToolsChangedGeneration,
} from '~/mcp/toolsChanged';
import {
  getMCPConnectionPoolKey,
  parseMCPConnectionPoolKey,
  getMCPUserConnectionPoolKey,
  MCP_APPS_CAPABILITY_PROFILE,
  STANDARD_MCP_CAPABILITY_PROFILE,
} from './capabilities';
import {
  resolveScheduledMCPBearerConfig,
  requiresScheduledMCPBearerConnection,
  isScheduledMCPBearer,
} from '~/schedules/bearer';
import { MCPAuthenticationRejectedError, createScheduledMCPTransportError } from '~/mcp/errors';
import { resolveDirectOpenIDBearerConfig, usesDirectOpenIDBearerRecovery } from '~/mcp/openid';
import { MCPServersRegistry } from '~/mcp/registry/MCPServersRegistry';
import { ConnectionsRepository } from '~/mcp/ConnectionsRepository';
import { MCPConnectionFactory } from '~/mcp/MCPConnectionFactory';
import { processMCPEnv, isPluginSourced } from '~/utils/env';
import { OAuthLifecycleRelay } from '~/mcp/oauth/pending';
import { preProcessGraphTokens } from '~/utils/graph';
import { detectOAuthRequirement } from '~/mcp/oauth';
import { awaitOboOperation } from '~/mcp/oauth/obo';
import { isMCPDomainAllowed } from '~/auth/domain';
import { getMCPRequestSignal } from './request';
import { MCPConnection } from './connection';
import { mcpConfig } from './mcpConfig';
import { isEnabled } from '~/utils';

type PendingConnection = {
  promise: Promise<MCPConnection>;
  oauth: OAuthLifecycleRelay;
  directBearerRecoveryState: t.DirectBearerRecoveryState;
  configGeneration: string;
  signal?: AbortSignal;
};

type ResolvedUserMCPConnectionOptions = Omit<
  t.UserMCPConnectionOptions,
  'connectionTarget' | 'serverConfig'
> & {
  connectionTarget: t.MCPConnectionTarget;
};

/**
 * Why a connection is being torn down. A `mutation` teardown follows a committed change to the
 * server config or its credentials, so whatever an in-flight creation resolved before it may be
 * stale and that creation cannot be re-established from the same inputs. A `lifecycle` teardown
 * — an idle sweep, a forced replacement, a dead connection cleaned up — leaves those inputs
 * exactly as valid as they were, and only the connection object itself goes away.
 */
type ConnectionTeardownReason = 'mutation' | 'lifecycle';

type ConnectionTeardownOptions = {
  /** The creation that requested the teardown; it is fenced by its own replacement. */
  preservedCreation?: ConnectionCreationGuard;
  reason?: ConnectionTeardownReason;
  /** Limits an internal lifecycle teardown to one negotiated session. */
  capabilityProfile?: MCPClientCapabilityProfile;
};

type ConnectionCreationGuard = { cancelledBy: ConnectionTeardownReason | null };

type ConnectionMutationFence = {
  assertCurrent: () => void;
  release: () => void;
};

/** Signals that a teardown fenced an in-flight creation before it could finish. */
class ConnectionCreationCancelledError extends Error {
  constructor(
    message: string,
    public readonly reason: ConnectionTeardownReason,
  ) {
    super(message);
  }
}

/** Bounds re-establishment so back-to-back teardowns cannot spin a caller forever. */
const MAX_TEARDOWN_RESTARTS = 3;

const MCP_CAPABILITY_PROFILES: readonly MCPClientCapabilityProfile[] = [
  STANDARD_MCP_CAPABILITY_PROFILE,
  MCP_APPS_CAPABILITY_PROFILE,
];

function getUserConnectionKey(
  userId: string,
  serverName: string,
  capabilityProfile: MCPClientCapabilityProfile,
): string {
  return getMCPUserConnectionPoolKey(userId, serverName, capabilityProfile);
}

function isUserConnectionKeyForUser(key: string, userId: string): boolean {
  try {
    const parsed = JSON.parse(key) as unknown;
    return (
      Array.isArray(parsed) &&
      parsed.length === 3 &&
      parsed[0] === userId &&
      typeof parsed[1] === 'string' &&
      (parsed[2] === STANDARD_MCP_CAPABILITY_PROFILE || parsed[2] === MCP_APPS_CAPABILITY_PROFILE)
    );
  } catch {
    return false;
  }
}

/**
 * Abstract base class for managing user-specific MCP connections with lifecycle management.
 * Only meant to be extended by MCPManager.
 * Much of the logic was move here from the old MCPManager to make it more manageable.
 * User connections will soon be ephemeral and not cached anymore:
 * https://github.com/LibreChat-AI/LibreChat/discussions/8790
 */
export abstract class UserConnectionManager {
  // Connections shared by all users.
  public appConnections: ConnectionsRepository | null = null;
  // Connections per userId -> serverName/capability profile -> connection
  protected userConnections: Map<string, Map<string, MCPConnection>> = new Map();
  /** Last activity timestamp for users (not per server) */
  protected userLastActivity: Map<string, number> = new Map();
  /** In-flight connection promises keyed by user, server, and capability profile. */
  protected pendingConnections: Map<string, PendingConnection> = new Map();
  /** Recovery state owned by request-scoped promises, which the public store intentionally keeps opaque. */
  private readonly requestPendingDirectBearerRecoveryStates = new WeakMap<
    Promise<unknown>,
    t.DirectBearerRecoveryState
  >();

  private readonly connectionBorrowers = new WeakMap<MCPConnection, number>();
  private readonly connectionBorrowerDrainWaiters = new WeakMap<MCPConnection, Set<() => void>>();
  private readonly deferredConnectionDisposalHolds = new WeakMap<MCPConnection, number>();
  private readonly deferredConnectionDisposals = new WeakMap<MCPConnection, string>();
  /** All durable creations, including forced replacements, visible to mutation teardown. */
  private readonly activeConnectionCreations: Map<string, Set<ConnectionCreationGuard>> = new Map();
  /** Serializes explicit durable replacements without coalescing their callers. */
  private readonly forceNewConnectionQueues: Map<string, Promise<void>> = new Map();
  /** Fences durable connections whose credentials were invalidated on another replica. */
  protected readonly toolPublicationGenerations: WeakMap<MCPConnection, string> = new WeakMap();
  /** Binds a durable connection to the stored config that created it, independently of Redis. */
  protected readonly toolConfigGenerations: WeakMap<MCPConnection, string> = new WeakMap();
  /** Limits Redis lease refreshes while ensuring active connections cannot outlive their lease. */
  private readonly toolPublicationLeaseRefreshes: WeakMap<MCPConnection, number> = new WeakMap();
  /** Coalesces concurrent activity updates for the same durable connection. */
  private readonly toolPublicationLeaseRenewals: WeakMap<MCPConnection, Promise<void>> =
    new WeakMap();

  /** Records connections whose distributed publication authority was replaced during renewal. */
  private readonly lostToolPublicationLeases: WeakSet<MCPConnection> = new WeakSet();

  /** Returns the cache-publication generation captured for a durable connection. */
  public getToolPublicationGeneration(connection: MCPConnection): string | undefined {
    return this.toolPublicationGenerations.get(connection);
  }

  /** Returns the config identity captured when a durable connection was created. */
  public getToolConfigGeneration(connection: MCPConnection): string | undefined {
    return this.toolConfigGenerations.get(connection);
  }

  protected getSuppliedConnectionTarget({
    connectionTarget,
    serverConfig,
  }: {
    connectionTarget?: t.MCPConnectionTarget;
    serverConfig?: t.ParsedServerConfig;
  }): t.MCPConnectionTarget | undefined {
    if (connectionTarget) {
      return connectionTarget;
    }
    if (serverConfig) {
      return {
        serverConfig,
        connectionOwner: 'principal',
      };
    }
    return undefined;
  }

  /**
   * A principal-scoped registry lookup for the acting user. The role and `idOnTheSource` the
   * request already loaded go along, so deciding which servers the user's config hides needs no
   * user read.
   */
  protected getPrincipalServerConfig(
    serverName: string,
    user?: t.MCPLookupUser,
    configServers?: Record<string, t.ParsedServerConfig>,
  ): Promise<t.ParsedServerConfig | undefined> {
    return MCPServersRegistry.getInstance().getServerConfig(
      serverName,
      user?.id,
      configServers,
      user?.role,
      { idOnTheSource: user?.idOnTheSource },
    );
  }

  protected async resolveConnectionTarget({
    serverName,
    user,
  }: {
    serverName: string;
    user?: t.MCPLookupUser;
  }): Promise<t.MCPConnectionTarget | undefined> {
    const config = await this.getPrincipalServerConfig(serverName, user);
    if (!config) {
      return undefined;
    }
    return {
      serverConfig: config,
      connectionOwner: 'principal',
    };
  }

  private runWithForceNewConnectionQueue<T>(key: string, operation: () => Promise<T>): Promise<T> {
    const previous = this.forceNewConnectionQueues.get(key) ?? Promise.resolve();
    const result = previous.then(operation, operation);
    const tail = result.then(
      () => undefined,
      () => undefined,
    );
    this.forceNewConnectionQueues.set(key, tail);
    void tail.then(() => {
      if (this.forceNewConnectionQueues.get(key) === tail) {
        this.forceNewConnectionQueues.delete(key);
      }
    });
    return result;
  }

  private registerConnectionCreation(key: string, guard: ConnectionCreationGuard): void {
    const guards = this.activeConnectionCreations.get(key) ?? new Set<ConnectionCreationGuard>();
    guards.add(guard);
    this.activeConnectionCreations.set(key, guards);
  }

  private unregisterConnectionCreation(key: string, guard: ConnectionCreationGuard): void {
    const guards = this.activeConnectionCreations.get(key);
    guards?.delete(guard);
    if (guards?.size === 0) {
      this.activeConnectionCreations.delete(key);
    }
  }

  private assertCreationNotCancelled(
    guard: ConnectionCreationGuard | undefined,
    userId: string,
    serverName: string,
  ): void {
    if (!guard?.cancelledBy) {
      return;
    }
    throw new ConnectionCreationCancelledError(
      `[MCP][User: ${userId}][${serverName}] Connection creation was cancelled during teardown`,
      guard.cancelledBy,
    );
  }

  /** Registers work that resolves a replacement before the ordinary creation queue begins. */
  protected createConnectionMutationFence(
    userId: string,
    serverName: string,
    capabilityProfile: MCPClientCapabilityProfile = STANDARD_MCP_CAPABILITY_PROFILE,
  ): ConnectionMutationFence {
    const key = getUserConnectionKey(userId, serverName, capabilityProfile);
    const guard: ConnectionCreationGuard = { cancelledBy: null };
    this.registerConnectionCreation(key, guard);
    return {
      assertCurrent: () => {
        /** Lifecycle churn does not invalidate the captured config; a committed mutation does. */
        if (guard.cancelledBy === 'lifecycle') {
          guard.cancelledBy = null;
        }
        this.assertCreationNotCancelled(guard, userId, serverName);
      },
      release: () => this.unregisterConnectionCreation(key, guard),
    };
  }

  /** A mutation fence outranks a lifecycle one: the stale inputs stay stale either way. */
  private cancelConnectionCreations(
    key: string,
    reason: ConnectionTeardownReason,
    preserved?: ConnectionCreationGuard,
  ): void {
    for (const guard of this.activeConnectionCreations.get(key) ?? []) {
      if (guard !== preserved && guard.cancelledBy !== 'mutation') {
        guard.cancelledBy = reason;
      }
    }
  }

  private async renewUserToolPublicationLeases(userId: string, now: number): Promise<void> {
    const userConnections = this.userConnections.get(userId);
    if (!userConnections) {
      return;
    }
    const configuredIdleTimeout = Number(mcpConfig.USER_CONNECTION_IDLE_TIMEOUT);
    const refreshInterval =
      Number.isFinite(configuredIdleTimeout) && configuredIdleTimeout > 0
        ? Math.max(1_000, configuredIdleTimeout / 2)
        : 15 * 60 * 1000;
    const renewals: Promise<void>[] = [];
    for (const [poolKey, connection] of userConnections) {
      const { serverName } = parseMCPConnectionPoolKey(poolKey);
      const publicationGeneration = this.toolPublicationGenerations.get(connection);
      if (!publicationGeneration) {
        continue;
      }
      const lastRefresh = this.toolPublicationLeaseRefreshes.get(connection) ?? 0;
      if (now - lastRefresh < refreshInterval) {
        continue;
      }
      const pendingRenewal = this.toolPublicationLeaseRenewals.get(connection);
      if (pendingRenewal) {
        renewals.push(pendingRenewal);
        continue;
      }
      const renewal = renewMCPToolsChangedGeneration({
        userId,
        serverName,
        publicationGeneration,
      })
        .then((renewed) => {
          if (renewed === true) {
            this.toolPublicationLeaseRefreshes.set(connection, now);
          } else if (renewed === false) {
            this.lostToolPublicationLeases.add(connection);
            logger.info(
              `[MCP][User: ${userId}][${serverName}] Publication lease is no longer current`,
            );
          }
        })
        .catch((error) => {
          logger.warn(
            `[MCP][User: ${userId}][${serverName}] Failed to renew tool publication lease`,
            error,
          );
        });
      this.toolPublicationLeaseRenewals.set(connection, renewal);
      void renewal.finally(() => {
        if (this.toolPublicationLeaseRenewals.get(connection) === renewal) {
          this.toolPublicationLeaseRenewals.delete(connection);
        }
      });
      renewals.push(renewal);
    }
    await Promise.all(renewals);
  }

  /** Updates activity and keeps every durable connection retained by that user leased. */
  protected async updateUserLastActivity(userId: string): Promise<void> {
    const now = Date.now();
    this.userLastActivity.set(userId, now);
    logger.debug(
      `[MCP][User: ${userId}] Updated last activity timestamp: ${new Date(now).toISOString()}`,
    );
    await this.renewUserToolPublicationLeases(userId, now);
  }

  private async assertToolPublicationLeaseCurrent(
    connection: MCPConnection,
    userId: string,
    serverName: string,
    creationGuard?: ConnectionCreationGuard,
  ): Promise<void> {
    if (!this.lostToolPublicationLeases.has(connection)) {
      return;
    }
    await this.disconnectUserConnection(userId, serverName, {
      preservedCreation: creationGuard,
      capabilityProfile: connection.capabilityProfile,
    });
    throw new Error(`[MCP][User: ${userId}][${serverName}] Publication lease is no longer current`);
  }

  /**
   * A teardown fences every creation in flight for the same user and server, since one of them
   * may install the connection it is removing, or hand back the one it just disposed. What the
   * fenced attempt deserves depends on why the teardown ran. After a `lifecycle` teardown it is
   * next in line rather than stale — nothing its caller resolved changed — so the attempt is
   * discarded and re-established. After a `mutation` teardown the config and credentials that
   * caller resolved may be the ones that were just replaced, and only the caller can resolve
   * them again, so the fence stays fatal. A caller that has since aborted gets neither: there
   * is nobody left to hand the connection to.
   *
   * Re-establishment happens inside the queue slot the attempt already holds, so a replacement
   * queued behind another one keeps its position instead of moving to the tail and overwriting
   * a newer connection later.
   */
  private async createUserConnectionWithLifecycleRestarts(
    options: ResolvedUserMCPConnectionOptions,
    userId: string,
    clearCooldown: boolean,
    creationGuard: ConnectionCreationGuard,
  ): Promise<MCPConnection> {
    for (let attempt = 0; ; attempt++) {
      /** A lifecycle fence raised while this attempt waited its turn has nothing left to fence;
       *  a mutation fence is kept, so the attempt below fails on it immediately. */
      if (creationGuard.cancelledBy === 'lifecycle') {
        creationGuard.cancelledBy = null;
      }
      try {
        return await this.createUserConnectionInternal(
          options,
          userId,
          clearCooldown,
          creationGuard,
        );
      } catch (error) {
        if (
          !(error instanceof ConnectionCreationCancelledError) ||
          error.reason !== 'lifecycle' ||
          attempt >= MAX_TEARDOWN_RESTARTS ||
          options.signal?.aborted === true
        ) {
          throw error;
        }
        logger.info(
          `[MCP][User: ${userId}][${options.serverName}] Connection creation raced a teardown; re-establishing`,
        );
      }
    }
  }

  /** Gets or creates a connection for a specific user, coalescing concurrent attempts */
  public getUserConnection(inputOpts: t.UserMCPConnectionOptions): Promise<MCPConnection> {
    const userId = inputOpts.user?.id;
    if (!userId) {
      return Promise.reject(
        new McpError(ErrorCode.InvalidRequest, `[MCP] User object missing id property`),
      );
    }
    const connectionTarget = this.getSuppliedConnectionTarget(inputOpts);
    if (connectionTarget) {
      return this.getUserConnectionForTarget(inputOpts, connectionTarget, userId);
    }
    return this.resolveConnectionTarget(inputOpts).then((resolvedTarget) => {
      inputOpts.signal?.throwIfAborted();
      if (!resolvedTarget) {
        throw new McpError(
          ErrorCode.InvalidRequest,
          `[MCP] Configuration for server "${inputOpts.serverName}" not found.`,
        );
      }
      return this.getUserConnectionForTarget(inputOpts, resolvedTarget, userId);
    });
  }

  private async getUserConnectionForTarget(
    inputOpts: t.UserMCPConnectionOptions,
    connectionTarget: t.MCPConnectionTarget,
    userId: string,
  ): Promise<MCPConnection> {
    const { serverConfig: _serverConfig, ...targetOptions } = inputOpts;
    const opts = {
      ...targetOptions,
      connectionTarget,
    } as ResolvedUserMCPConnectionOptions;
    const { serverName, forceNew } = opts;
    const capabilityProfile = opts.capabilityProfile ?? STANDARD_MCP_CAPABILITY_PROFILE;
    opts.capabilityProfile = capabilityProfile;
    const directBearerRecoveryState = opts.directBearerRecoveryState ?? { attempted: false };

    const config = connectionTarget.serverConfig;
    const configGeneration = getMCPAppToolsPublicationGeneration(config);
    const missingBodyFields = config
      ? getMissingRuntimeBodyPlaceholderFields(config, opts.requestBody)
      : [];
    if (missingBodyFields.length > 0) {
      throw new McpError(
        ErrorCode.InvalidRequest,
        `[MCP][User: ${userId}][${serverName}] Request body field(s) required to resolve runtime MCP placeholders: ${missingBodyFields.join(', ')}.`,
      );
    }
    const ephemeralConnection =
      opts.ephemeralConnection === true ||
      requiresScheduledMCPBearerConnection(opts.requestScopedConnections, config) ||
      (config ? requiresEphemeralUserConnection(config) : false);
    const requestScopedConnections = ephemeralConnection
      ? opts.requestScopedConnections
      : undefined;
    if (requestScopedConnections) {
      if (requestScopedConnections.cleanupStarted) {
        throw new Error(`[MCP][User: ${userId}] Request-scoped connection context is closed`);
      }
      this.bindRequestScopedConnectionStore(requestScopedConnections);
      const sharedCreation = requiresScheduledMCPBearerConnection(requestScopedConnections, config);
      const creationSignal = sharedCreation
        ? getMCPRequestSignal(requestScopedConnections)
        : opts.signal;
      opts.signal?.throwIfAborted();
      const requestConnectionKey = getUserConnectionKey(userId, serverName, capabilityProfile);
      const existing = requestScopedConnections.connections.get(requestConnectionKey) as
        | MCPConnection
        | undefined;
      if (existing) {
        if (!config || (config.updatedAt && existing.isStale(config.updatedAt))) {
          requestScopedConnections.connections.delete(requestConnectionKey);
          await this.disposeEvictedConnection(existing, `[MCP][User: ${userId}]`);
        } else {
          const activeRecovery = this.getActiveConnectionRecovery(existing);
          this.propagateDirectBearerRecoveryState(existing, opts.directBearerRecoveryState);
          let awaitedRecovery = activeRecovery;
          if (activeRecovery) {
            await this.waitForConnectionRecovery(activeRecovery, opts.signal);
          }
          let connected = await existing.isConnected(opts.signal);
          opts.signal?.throwIfAborted();
          let recovery = this.getActiveConnectionRecovery(existing);
          this.propagateDirectBearerRecoveryState(existing, opts.directBearerRecoveryState);
          while (recovery && recovery !== awaitedRecovery) {
            awaitedRecovery = recovery;
            await this.waitForConnectionRecovery(recovery, opts.signal);
            connected = await existing.isConnected(opts.signal);
            opts.signal?.throwIfAborted();
            recovery = this.getActiveConnectionRecovery(existing);
            this.propagateDirectBearerRecoveryState(existing, opts.directBearerRecoveryState);
          }
          if (requestScopedConnections.connections.get(requestConnectionKey) !== existing) {
            return this.getUserConnection({ ...opts, directBearerRecoveryState });
          }
          if (connected) {
            logger.debug(`[MCP][User: ${userId}] Reusing request-scoped connection`);
            return existing;
          }
          requestScopedConnections.connections.delete(requestConnectionKey);
          await this.disposeEvictedConnection(existing, `[MCP][User: ${userId}]`);
        }
      }

      const pending = requestScopedConnections.pending.get(requestConnectionKey) as
        | Promise<MCPConnection>
        | undefined;
      if (pending) {
        logger.debug(`[MCP][User: ${userId}] Joining in-flight request-scoped connection attempt`);
        const connection = sharedCreation
          ? await awaitOboOperation(pending, opts.signal)
          : await pending;
        if (this.requestPendingDirectBearerRecoveryStates.get(pending)?.attempted) {
          directBearerRecoveryState.attempted = true;
        }
        directBearerRecoveryState.resolvedConfig =
          this.requestPendingDirectBearerRecoveryStates.get(pending)?.resolvedConfig;
        return connection;
      }

      const pendingOAuth = new OAuthLifecycleRelay({
        oauthStart: opts.oauthStart,
        oauthEnd: opts.oauthEnd,
        logPrefix: `[MCP][User: ${userId}][${serverName}]`,
      });
      const creationGuard: ConnectionCreationGuard = { cancelledBy: null };
      this.registerConnectionCreation(requestConnectionKey, creationGuard);
      const connectionPromise = this.createUserConnectionWithLifecycleRestarts(
        {
          ...opts,
          signal: creationSignal,
          forceNew: true,
          ephemeralConnection: true,
          connectionTarget,
          directBearerRecoveryState,
          oauthStart: pendingOAuth.start,
          oauthEnd: pendingOAuth.end,
        },
        userId,
        forceNew === true,
        creationGuard,
      )
        .then(async (connection) => {
          try {
            creationSignal?.throwIfAborted();
            this.assertCreationNotCancelled(creationGuard, userId, serverName);
            if (requestScopedConnections.cleanupStarted) {
              throw new Error(`[MCP][User: ${userId}] Request-scoped connection context is closed`);
            }
          } catch (error) {
            await this.disposeEvictedConnection(
              connection,
              `[MCP][Request-scoped: ${requestConnectionKey}] Invalidated during connection creation`,
            );
            throw error;
          }
          requestScopedConnections.connections.set(requestConnectionKey, connection);
          return connection;
        })
        .finally(() => {
          this.unregisterConnectionCreation(requestConnectionKey, creationGuard);
          if (requestScopedConnections.pending.get(requestConnectionKey) === connectionPromise)
            requestScopedConnections.pending.delete(requestConnectionKey);
        });

      requestScopedConnections.pending.set(
        requestConnectionKey,
        connectionPromise as Promise<unknown>,
      );
      this.requestPendingDirectBearerRecoveryStates.set(
        connectionPromise,
        directBearerRecoveryState,
      );

      // The occurrence owns the shared attempt; callers leave only their own waiters.
      return sharedCreation ? awaitOboOperation(connectionPromise, opts.signal) : connectionPromise;
    }

    const forceNewConnection = forceNew || ephemeralConnection;
    const clearCooldown = forceNew === true;

    const lockKey = getUserConnectionKey(userId, serverName, capabilityProfile);

    if (!forceNewConnection) {
      const pending = this.pendingConnections.get(lockKey);
      if (pending) {
        if (pending.configGeneration !== configGeneration) {
          await this.waitForConnectionRecovery(
            pending.promise.then(
              () => undefined,
              () => undefined,
            ),
            opts.signal,
          );
          opts.signal?.throwIfAborted();
          return this.getUserConnection(opts);
        }
        logger.debug(`[MCP][User: ${userId}] Joining in-flight connection attempt`);
        const mutationFence = this.createConnectionMutationFence(
          userId,
          serverName,
          capabilityProfile,
        );
        try {
          await pending.oauth.add({
            oauthStart: opts.oauthStart,
            oauthEnd: opts.oauthEnd,
            flowManager: opts.flowManager,
            userId,
            serverName,
          });
          opts.signal?.throwIfAborted();
          await this.waitForConnectionRecovery(
            pending.promise.then(() => undefined),
            opts.signal,
          );
          const connection = await pending.promise;
          opts.signal?.throwIfAborted();
          mutationFence?.assertCurrent();
          if (pending.directBearerRecoveryState.attempted) {
            directBearerRecoveryState.attempted = true;
          }
          directBearerRecoveryState.resolvedConfig =
            pending.directBearerRecoveryState.resolvedConfig;
          return connection;
        } catch (error) {
          opts.signal?.throwIfAborted();
          mutationFence?.assertCurrent();
          if (!mutationFence || !pending.signal?.aborted || error !== pending.signal.reason) {
            throw error;
          }
          /** The stopped leader owns its aborted attempt, not this still-active request.
           * Continue with an independently owned creation, preserving any spent auth budget. */
          directBearerRecoveryState.attempted ||= pending.directBearerRecoveryState.attempted;
          if (this.pendingConnections.get(lockKey) === pending) {
            this.pendingConnections.delete(lockKey);
          }
        } finally {
          mutationFence?.release();
        }
      }
    }

    const pendingOAuth = new OAuthLifecycleRelay({
      oauthStart: opts.oauthStart,
      oauthEnd: opts.oauthEnd,
      logPrefix: `[MCP][User: ${userId}][${serverName}]`,
    });
    const creationGuard: ConnectionCreationGuard = { cancelledBy: null };
    this.registerConnectionCreation(lockKey, creationGuard);
    const createConnection = () =>
      this.createUserConnectionWithLifecycleRestarts(
        {
          ...opts,
          forceNew: forceNewConnection,
          ephemeralConnection,
          connectionTarget,
          directBearerRecoveryState,
          oauthStart: pendingOAuth.start,
          oauthEnd: pendingOAuth.end,
        },
        userId,
        clearCooldown,
        creationGuard,
      );
    const connectionPromise = !ephemeralConnection
      ? this.runWithForceNewConnectionQueue(lockKey, createConnection)
      : createConnection();

    if (!forceNewConnection) {
      this.pendingConnections.set(lockKey, {
        promise: connectionPromise,
        oauth: pendingOAuth,
        directBearerRecoveryState,
        configGeneration,
        signal: opts.signal,
      });
    }

    try {
      return await connectionPromise;
    } finally {
      this.unregisterConnectionCreation(lockKey, creationGuard);
      if (
        !forceNewConnection &&
        this.pendingConnections.get(lockKey)?.promise === connectionPromise
      ) {
        this.pendingConnections.delete(lockKey);
      }
    }
  }

  private async createUserConnectionInternal(
    {
      serverName,
      forceNew,
      user,
      flowManager,
      customUserVars,
      requestBody,
      tokenMethods,
      oauthStart,
      oauthEnd,
      oboTokenResolver,
      oboTrustChecker,
      upstreamTokenProvider,
      requestScopedConnections,
      upstreamTokenProviderResolver,
      oboIdentityContext,
      onOAuthCredentialsChanged,
      onOAuthCredentialsChanging,
      signal,
      returnOnOAuth = false,
      connectionTimeout,
      graphTokenResolver,
      ephemeralConnection = false,
      capabilityProfile = STANDARD_MCP_CAPABILITY_PROFILE,
      connectionTarget,
      directBearerRecoveryState = { attempted: false },
      directBearerResolvedConfig,
    }: ResolvedUserMCPConnectionOptions,
    userId: string,
    clearCooldown: boolean,
    creationGuard?: ConnectionCreationGuard,
  ): Promise<MCPConnection> {
    signal?.throwIfAborted();
    this.assertCreationNotCancelled(creationGuard, userId, serverName);
    if (
      connectionTarget.connectionOwner === 'operator' &&
      canUseAppConnection(connectionTarget.serverConfig) &&
      capabilityProfile !== MCP_APPS_CAPABILITY_PROFILE
    ) {
      throw new McpError(
        ErrorCode.InvalidRequest,
        `[MCP][User: ${userId}] Trying to create user-specific connection for app-level server "${serverName}"`,
      );
    }

    const declaredConfig = connectionTarget.serverConfig;
    /** Resolution uses effective headers; identity and persistence keep the declaration. */
    const config = applyRequestHeaders(declaredConfig);
    const poolKey = getMCPConnectionPoolKey(serverName, capabilityProfile);

    /** Capture before resolving credentials/creating the connection. If another replica rotates
     *  the generation while creation is in flight, this connection's publications are fenced. */
    const publicationGeneration = ephemeralConnection
      ? undefined
      : await getMCPToolsChangedGeneration({ userId, serverName });

    const userServerMap = this.userConnections.get(userId);
    let connection = userServerMap?.get(poolKey);
    if (forceNew && connection && !ephemeralConnection) {
      logger.info(
        `[MCP][User: ${userId}][${serverName}] Disposing existing connection before forced replacement`,
      );
      await this.disconnectUserConnection(userId, serverName, {
        preservedCreation: creationGuard,
        reason: 'lifecycle',
        capabilityProfile,
      });
      connection = undefined;
    } else if (forceNew) {
      connection = undefined;
    }
    if (clearCooldown) {
      MCPConnection.clearCooldown(serverName);
    }
    const now = Date.now();

    const existingPublicationGeneration = connection
      ? this.toolPublicationGenerations.get(connection)
      : undefined;
    const configGeneration = getMCPAppToolsPublicationGeneration(declaredConfig);
    const existingConfigGeneration = connection
      ? this.toolConfigGenerations.get(connection)
      : undefined;
    if (connection && existingConfigGeneration && configGeneration !== existingConfigGeneration) {
      logger.info(
        `[MCP][User: ${userId}][${serverName}] Config identity changed, disconnecting stale connection`,
      );
      await this.disconnectUserConnection(userId, serverName, {
        preservedCreation: creationGuard,
        capabilityProfile,
      });
      connection = undefined;
    }
    if (
      connection &&
      publicationGeneration &&
      existingPublicationGeneration &&
      publicationGeneration !== existingPublicationGeneration
    ) {
      logger.info(
        `[MCP][User: ${userId}][${serverName}] Cache generation changed, disconnecting stale connection`,
      );
      await this.disconnectUserConnection(userId, serverName, {
        preservedCreation: creationGuard,
        capabilityProfile,
      });
      connection = undefined;
    }

    // Check if user is idle
    const lastActivity = this.userLastActivity.get(userId);
    if (lastActivity && now - lastActivity > mcpConfig.USER_CONNECTION_IDLE_TIMEOUT) {
      logger.info(`[MCP][User: ${userId}] User idle for too long. Disconnecting all connections.`);
      // Disconnect all user connections
      try {
        await this.disconnectUserConnections(userId, {
          preservedCreation: creationGuard,
          reason: 'lifecycle',
        });
      } catch {
        logger.error(`[MCP][User: ${userId}] Error disconnecting idle connections`);
      }
      connection = undefined; // Force creation of a new connection
    } else if (connection) {
      if (declaredConfig.updatedAt && connection.isStale(declaredConfig.updatedAt)) {
        logger.info(
          `[MCP][User: ${userId}] Server configuration updated; disconnecting stale connection`,
        );
        await this.disconnectUserConnection(userId, serverName, {
          preservedCreation: creationGuard,
          capabilityProfile,
        });
        connection = undefined;
      } else {
        const activeRecovery = this.getActiveConnectionRecovery(connection);
        this.propagateDirectBearerRecoveryState(connection, directBearerRecoveryState);
        let awaitedRecovery = activeRecovery;
        if (activeRecovery) {
          await this.waitForConnectionRecovery(activeRecovery, signal);
        }
        let connected = await connection.isConnected();
        let recovery = this.getActiveConnectionRecovery(connection);
        this.propagateDirectBearerRecoveryState(connection, directBearerRecoveryState);
        while (recovery && recovery !== awaitedRecovery) {
          awaitedRecovery = recovery;
          await this.waitForConnectionRecovery(recovery, signal);
          connected = await connection.isConnected();
          recovery = this.getActiveConnectionRecovery(connection);
          this.propagateDirectBearerRecoveryState(connection, directBearerRecoveryState);
        }
        this.assertCreationNotCancelled(creationGuard, userId, serverName);
        if (connected) {
          logger.debug(`[MCP][User: ${userId}] Reusing active connection`);
          await this.updateUserLastActivity(userId);
          await this.assertToolPublicationLeaseCurrent(
            connection,
            userId,
            serverName,
            creationGuard,
          );
          this.assertCreationNotCancelled(creationGuard, userId, serverName);
          return connection;
        }
        logger.warn(`[MCP][User: ${userId}] Found disconnected connection object; cleaning up`);
        await this.disconnectUserConnection(userId, serverName, {
          preservedCreation: creationGuard,
          reason: 'lifecycle',
          capabilityProfile,
        });
        connection = undefined;
      }
    }

    // If no valid connection exists, create a new one
    logger.info(`[MCP][User: ${userId}] Establishing new connection`);

    try {
      signal?.throwIfAborted();
      const scheduledConfig = await resolveScheduledMCPBearerConfig({
        user,
        serverName,
        config: declaredConfig,
        context: requestScopedConnections,
        signal,
      });
      const bearerConfig =
        (!isScheduledMCPBearer(requestScopedConnections) && directBearerResolvedConfig) ||
        (await resolveDirectOpenIDBearerConfig({
          config: isScheduledMCPBearer(requestScopedConnections)
            ? applyRequestHeaders(scheduledConfig)
            : config,
          upstreamTokenProvider,
          signal,
        }));
      signal?.throwIfAborted();
      if (usesDirectOpenIDBearerRecovery(config)) {
        directBearerRecoveryState.resolvedConfig = bearerConfig;
      }
      const runtimeConfig = await this.applyRuntimeOAuthDetection({
        config: bearerConfig,
        user,
        customUserVars,
        requestBody,
        graphTokenResolver,
      });
      const registry = MCPServersRegistry.getInstance();
      const { allowedDomains, allowedAddresses, useSSRFProtection, mcpApps } =
        await registry.resolveAllowlists({ userId: user?.id, role: user?.role });
      await this.assertResolvedRuntimeConfigAllowed({
        config: runtimeConfig,
        user,
        customUserVars,
        requestBody,
        graphTokenResolver,
        allowedDomains,
        allowedAddresses,
        logPrefix: `[MCP][User: ${userId}]`,
      });
      const basic: t.BasicConnectionOptions = {
        serverConfig: runtimeConfig,
        /** Runtime OAuth detection enriches the connection config. Keep the durable definition
         * separate so callback liveness compares against the config that actually owns it. */
        serverDefinition: declaredConfig,
        ...(usesDirectOpenIDBearerRecovery(config) && { directBearerSourceConfig: config }),
        directBearerRecoveryState,
        serverName: serverName,
        dbSourced: isUserSourced(runtimeConfig),
        useSSRFProtection,
        allowedDomains,
        allowedAddresses,
        ephemeralConnection,
        capabilityProfile,
        ...(capabilityProfile === MCP_APPS_CAPABILITY_PROFILE && {
          operationLimits: mcpApps.operationLimits,
        }),
      };

      /**
       * Credentials resolved after the capture above were published under a generation that
       * capture predates, so the build would otherwise fence itself: a refresh it performs, an
       * authorization it waits on, or a re-read after a change invalidated its token cache each
       * hand it the newest credentials, which it would then publish under a retired generation.
       * Recording the generation those credentials carry, while it is still the stored one, keeps
       * the fence pointing at rotations that follow them; a carried generation the store has
       * already retired proves nothing about the current one, so the capture stands for it.
       */
      let credentialGeneration: string | undefined;
      const trackPublishedGeneration: t.UserConnectionContext['onOAuthCredentialsChanging'] =
        onOAuthCredentialsChanging == null
          ? undefined
          : async (scope) => {
              const publish = await onOAuthCredentialsChanging(scope);
              return async () => {
                const published = await publish();
                if (published) {
                  credentialGeneration = published;
                }
                return published;
              };
            };
      const adoptCredentialGeneration: t.UserConnectionContext['onOAuthCredentialsAdopted'] =
        ephemeralConnection
          ? undefined
          : async (published) => {
              if ((await getMCPToolsChangedGeneration({ userId, serverName })) === published) {
                credentialGeneration = published;
              }
            };
      const recaptureCredentialGeneration: t.UserConnectionContext['onOAuthCredentialsInvalidated'] =
        async () => {
          credentialGeneration = await getMCPToolsChangedGeneration({ userId, serverName });
          return credentialGeneration;
        };

      const useOAuth = usesDirectOpenIDBearerRecovery(config)
        ? false
        : requiresOAuthMachinery(runtimeConfig);
      let connectionOptions: t.OAuthConnectionOptions | t.UserConnectionContext;
      if (useOAuth) {
        if (!flowManager) {
          throw new McpError(
            ErrorCode.InvalidRequest,
            `[MCP][User: ${userId}] OAuth server "${serverName}" requires a flowManager`,
          );
        }

        connectionOptions = {
          requestScopedConnections,
          useOAuth: true,
          user: user,
          customUserVars: customUserVars,
          flowManager: flowManager,
          tokenMethods: tokenMethods,
          signal: signal,
          oauthStart: oauthStart,
          oauthEnd: oauthEnd,
          oboTokenResolver: oboTokenResolver,
          oboTrustChecker: oboTrustChecker,
          upstreamTokenProvider: upstreamTokenProvider,
          upstreamTokenProviderResolver,
          oboIdentityContext,
          graphTokenResolver,
          returnOnOAuth: returnOnOAuth,
          requestBody: requestBody,
          connectionTimeout: connectionTimeout,
          onOAuthCredentialsChanged,
          onOAuthCredentialsChanging: trackPublishedGeneration,
          onOAuthCredentialsAdopted: adoptCredentialGeneration,
          onOAuthCredentialsInvalidated: recaptureCredentialGeneration,
        };
      } else {
        connectionOptions = {
          requestScopedConnections,
          user,
          customUserVars,
          requestBody,
          graphTokenResolver,
          upstreamTokenProvider,
          upstreamTokenProviderResolver,
          connectionTimeout,
          signal,
        };
      }

      connection = await MCPConnectionFactory.create(basic, connectionOptions);

      this.assertCreationNotCancelled(creationGuard, userId, serverName);

      const effectiveGeneration = ephemeralConnection
        ? undefined
        : (credentialGeneration ?? publicationGeneration);

      if (effectiveGeneration) {
        this.toolPublicationGenerations.set(connection, effectiveGeneration);
      }
      if (configGeneration) {
        this.toolConfigGenerations.set(connection, configGeneration);
      }

      connection.on('toolsChanged', (tools: t.MCPTool[], publicationRevision?: string) => {
        void notifyMCPToolsChanged({
          tools,
          userId,
          serverName,
          serverConfig: declaredConfig,
          ...(effectiveGeneration && { publicationGeneration: effectiveGeneration }),
          ...(publicationRevision && { publicationRevision }),
          capabilityProfile,
        });
      });

      if (!(await connection?.isConnected())) {
        throw new Error('Failed to establish connection after initialization attempt.');
      }

      const directBearerRecovery = usesDirectOpenIDBearerRecovery(config);
      if (!ephemeralConnection || directBearerRecovery) {
        const toolListSnapshot = await connection.refreshToolList(signal);
        signal?.throwIfAborted();
        const toolListAuthenticationError = toolListSnapshot?.authenticationError;
        if (toolListAuthenticationError && directBearerRecovery && user) {
          if (isScheduledMCPBearer(requestScopedConnections))
            throw createScheduledMCPTransportError(toolListAuthenticationError, serverName);
          if (directBearerRecoveryState.attempted) {
            throw new MCPAuthenticationRejectedError(
              serverName,
              false,
              toolListAuthenticationError,
            );
          }
          directBearerRecoveryState.attempted = true;
          signal?.throwIfAborted();
          const refreshedConfig = await resolveDirectOpenIDBearerConfig({
            config,
            upstreamTokenProvider,
            forceRefresh: true,
            signal,
          });
          signal?.throwIfAborted();
          connection.removeAllListeners('toolsChanged');
          await connection.dispose();
          return this.createUserConnectionInternal(
            {
              serverName,
              forceNew: true,
              user,
              flowManager,
              customUserVars,
              requestBody,
              tokenMethods,
              oauthStart,
              oauthEnd,
              oboTokenResolver,
              oboTrustChecker,
              upstreamTokenProvider,
              upstreamTokenProviderResolver,
              oboIdentityContext,
              signal,
              returnOnOAuth,
              connectionTimeout,
              graphTokenResolver,
              ephemeralConnection,
              capabilityProfile,
              connectionTarget,
              directBearerRecoveryState,
              directBearerResolvedConfig: refreshedConfig,
            },
            userId,
            clearCooldown,
            creationGuard,
          );
        }
      }
      if (!ephemeralConnection) {
        signal?.throwIfAborted();
        this.assertCreationNotCancelled(creationGuard, userId, serverName);
        if (!this.userConnections.has(userId)) {
          this.userConnections.set(userId, new Map());
        }
        this.userConnections.get(userId)?.set(poolKey, connection);
      }

      logger.info(`[MCP][User: ${userId}][${serverName}] Connection successfully established`);
      if (
        capabilityProfile === STANDARD_MCP_CAPABILITY_PROFILE ||
        connectionTarget.connectionOwner !== 'operator'
      ) {
        await this.backfillResolvedInstructions(serverName, declaredConfig, connection, userId);
      }
      signal?.throwIfAborted();
      if (!ephemeralConnection) {
        await this.updateUserLastActivity(userId);
        await this.assertToolPublicationLeaseCurrent(connection, userId, serverName, creationGuard);
      }
      this.assertCreationNotCancelled(creationGuard, userId, serverName);
      signal?.throwIfAborted();
      return connection;
    } catch (error) {
      logger.error(`[MCP][User: ${userId}] Failed to establish connection`);
      // Ensure partial connection state is cleaned up if initialization fails
      connection?.removeAllListeners?.('toolsChanged');
      await connection?.dispose().catch(() => {
        logger.error(`[MCP][User: ${userId}] Error during cleanup after failed connection`);
      });
      // Ensure cleanup even if connection attempt fails
      if (connection && this.userConnections.get(userId)?.get(poolKey) === connection) {
        this.removeUserConnection(userId, serverName, capabilityProfile);
      }
      throw error; // Re-throw the error to the caller
    }
  }

  /**
   * An explicitly startup-deferred, context-independent YAML server has no
   * fetched text until its first live connection. Persist that static text so
   * subsequent context builds include it. OAuth/OBO, user credentials, custom
   * variables, and runtime placeholders stay connection-scoped: their
   * instructions can vary by identity or request and must never enter the
   * shared registry. Best-effort: a failure here must never break connection
   * creation.
   */
  protected async backfillResolvedInstructions(
    serverName: string,
    config: t.ParsedServerConfig | undefined,
    connection: MCPConnection,
    userId: string,
  ): Promise<void> {
    try {
      if (!config || !isEnabled(config.serverInstructions)) {
        return;
      }
      /** Only the YAML tier is writable here, and a config-overlay, user, or plugin
       *  server would otherwise spend a cache round-trip per connection to rediscover
       *  that. An unset source is left to proceed: it predates per-tier stamping and
       *  the registry still resolves it by name. A config-tier override shadowing a
       *  YAML base keeps the base's 'yaml' tag (`overlaySource`), so `config` is also
       *  passed through for the registry's field-level identity check. */
      if (
        isUserSourced(config) ||
        isPluginSourced(config) ||
        config.source === 'config' ||
        !canBackfillSharedServerInstructions(config)
      ) {
        return;
      }
      const instructions = connection.client.getInstructions();
      if (!instructions) {
        return;
      }
      if (config.resolvedInstructions != null) {
        return;
      }
      const updated = await MCPServersRegistry.getInstance().setResolvedInstructions(
        serverName,
        instructions,
        userId,
        config,
      );
      if (updated) {
        logger.info(
          `[MCP][User: ${userId}][${serverName}] Stored server instructions from live connection`,
        );
      }
    } catch (error) {
      logger.warn(
        `[MCP][User: ${userId}][${serverName}] Failed to store server instructions from live connection`,
        error,
      );
    }
  }

  /**
   * Mirrors the resolution MCPConnectionFactory performs internally
   * (preProcessGraphTokens + processMCPEnv). Both must stay in sync so the
   * config validated here matches the one the factory actually connects with.
   */
  protected async resolveRuntimeConfig({
    config,
    user,
    customUserVars,
    requestBody,
    graphTokenResolver,
  }: {
    config: t.ParsedServerConfig;
    user?: t.UserMCPConnectionOptions['user'];
    customUserVars?: Record<string, string>;
    requestBody?: t.UserMCPConnectionOptions['requestBody'];
    graphTokenResolver?: t.UserMCPConnectionOptions['graphTokenResolver'];
  }): Promise<t.ParsedServerConfig> {
    /** Mirrors the factory's entry-point normalization; without it this
     *  validation pass would inspect a different header map than the one the
     *  connection ends up sending. */
    const runtimeConfig = applyRequestHeaders(config);
    const dbSourced = isUserSourced(runtimeConfig);
    /** Plugin-authored placeholders must never resolve against the user's Graph token. */
    const graphProcessedConfig =
      dbSourced || isPluginSourced(runtimeConfig)
        ? runtimeConfig
        : await preProcessGraphTokens(runtimeConfig, {
            user,
            graphTokenResolver,
            scopes: process.env.GRAPH_API_SCOPES,
          });

    return processMCPEnv({
      user,
      body: requestBody,
      dbSourced,
      options: graphProcessedConfig,
      customUserVars,
    }) as t.ParsedServerConfig;
  }

  protected async assertResolvedRuntimeConfigAllowed({
    config,
    user,
    customUserVars,
    requestBody,
    graphTokenResolver,
    allowedDomains,
    allowedAddresses,
    logPrefix,
  }: {
    config: t.ParsedServerConfig;
    user?: t.UserMCPConnectionOptions['user'];
    customUserVars?: Record<string, string>;
    requestBody?: t.UserMCPConnectionOptions['requestBody'];
    graphTokenResolver?: t.UserMCPConnectionOptions['graphTokenResolver'];
    allowedDomains?: string[] | null;
    allowedAddresses?: string[] | null;
    logPrefix: string;
  }): Promise<t.ParsedServerConfig> {
    const resolvedConfig = await this.resolveRuntimeConfig({
      config,
      user,
      customUserVars,
      requestBody,
      graphTokenResolver,
    });

    if (!resolvedConfig.url) {
      return resolvedConfig;
    }

    if (hasRuntimeUrlPlaceholders(resolvedConfig)) {
      throw new McpError(
        ErrorCode.InvalidRequest,
        `${logPrefix} Runtime URL still contains unresolved MCP placeholders after resolution.`,
      );
    }

    const allowed = await isMCPDomainAllowed(resolvedConfig, allowedDomains, allowedAddresses);
    if (!allowed) {
      throw new McpError(
        ErrorCode.InvalidRequest,
        `${logPrefix} Resolved MCP server URL is not allowed by the configured domain policy.`,
      );
    }

    return resolvedConfig;
  }

  private async applyRuntimeOAuthDetection({
    config,
    user,
    customUserVars,
    requestBody,
    graphTokenResolver,
  }: {
    config: t.ParsedServerConfig;
    user?: t.UserMCPConnectionOptions['user'];
    customUserVars?: Record<string, string>;
    requestBody?: t.UserMCPConnectionOptions['requestBody'];
    graphTokenResolver?: t.UserMCPConnectionOptions['graphTokenResolver'];
  }): Promise<t.ParsedServerConfig> {
    if (
      config.requiresOAuth != null ||
      (config.apiKey != null && config.oauth == null) ||
      !hasRuntimeUrlPlaceholders(config)
    ) {
      return config;
    }

    const resolvedConfig = await this.resolveRuntimeConfig({
      config,
      user,
      customUserVars,
      requestBody,
      graphTokenResolver,
    });

    if (!resolvedConfig.url || hasRuntimeUrlPlaceholders(resolvedConfig)) {
      logger.warn(
        `[MCP][User: ${user?.id}] Runtime URL still contains placeholders after resolution; skipping OAuth detection`,
      );
      return config;
    }

    const registry = MCPServersRegistry.getInstance();
    const { allowedDomains, allowedAddresses } = await registry.resolveAllowlists({
      userId: user?.id,
      role: user?.role,
    });
    const allowed = await isMCPDomainAllowed(resolvedConfig, allowedDomains, allowedAddresses);
    if (!allowed) {
      throw new McpError(
        ErrorCode.InvalidRequest,
        `[MCP][User: ${user?.id}][${config.url}] Resolved MCP server URL is not allowed by the configured domain policy.`,
      );
    }

    const result = await detectOAuthRequirement(
      resolvedConfig.url,
      allowedDomains,
      allowedAddresses,
    );

    return {
      ...config,
      requiresOAuth: result.requiresOAuth,
      oauthMetadata: result.metadata,
    };
  }

  /** Returns the logical server map for one immutable capability profile. */
  public getUserConnections(
    userId: string,
    capabilityProfile: MCPClientCapabilityProfile = STANDARD_MCP_CAPABILITY_PROFILE,
  ): Map<string, MCPConnection> | undefined {
    const pooledConnections = this.userConnections.get(userId);
    if (!pooledConnections) {
      return undefined;
    }
    const connections = new Map<string, MCPConnection>();
    for (const [poolKey, connection] of pooledConnections) {
      const identity = parseMCPConnectionPoolKey(poolKey);
      if (identity.capabilityProfile === capabilityProfile) {
        connections.set(identity.serverName, connection);
      }
    }
    return connections;
  }

  /** Removes a specific user connection entry */
  protected removeUserConnection(
    userId: string,
    serverName: string,
    capabilityProfile: MCPClientCapabilityProfile,
  ): void {
    const userMap = this.userConnections.get(userId);
    if (userMap) {
      userMap.delete(getMCPConnectionPoolKey(serverName, capabilityProfile));
      if (userMap.size === 0) {
        this.userConnections.delete(userId);
        // Only remove user activity timestamp if all connections are gone
        this.userLastActivity.delete(userId);
      }
    }

    logger.debug(`[MCP][User: ${userId}] Removed connection entry`);
  }

  protected retainConnection(connection: MCPConnection): void {
    const borrowers = this.connectionBorrowers.get(connection) ?? 0;
    this.connectionBorrowers.set(connection, borrowers + 1);
  }

  protected getActiveConnectionRecovery(_connection: MCPConnection): Promise<void> | undefined {
    return undefined;
  }

  protected propagateDirectBearerRecoveryState(
    _connection: MCPConnection,
    _state?: t.DirectBearerRecoveryState,
  ): void {}

  protected waitForConnectionRecovery(
    recovery: Promise<void>,
    _signal?: AbortSignal,
  ): Promise<void> {
    return recovery;
  }

  protected holdDeferredConnectionDisposal(connection: MCPConnection): void {
    const holds = this.deferredConnectionDisposalHolds.get(connection) ?? 0;
    this.deferredConnectionDisposalHolds.set(connection, holds + 1);
  }

  protected async releaseDeferredConnectionDisposal(connection: MCPConnection): Promise<void> {
    const holds = this.deferredConnectionDisposalHolds.get(connection) ?? 0;
    if (holds > 1) {
      this.deferredConnectionDisposalHolds.set(connection, holds - 1);
      return;
    }
    this.deferredConnectionDisposalHolds.delete(connection);
    await this.finalizeDeferredConnectionDisposal(connection);
  }

  protected async releaseConnection(connection: MCPConnection): Promise<void> {
    const borrowers = this.connectionBorrowers.get(connection) ?? 0;
    if (borrowers > 1) {
      this.connectionBorrowers.set(connection, borrowers - 1);
      return;
    }

    this.connectionBorrowers.delete(connection);
    await this.finalizeDeferredConnectionDisposal(connection);

    const drainWaiters = this.connectionBorrowerDrainWaiters.get(connection);
    if (drainWaiters) {
      this.connectionBorrowerDrainWaiters.delete(connection);
      for (const resolve of drainWaiters) {
        resolve();
      }
    }
  }

  protected hasConnectionBorrowers(connection: MCPConnection): boolean {
    return (this.connectionBorrowers.get(connection) ?? 0) > 0;
  }

  protected waitForConnectionBorrowersToDrain(connection: MCPConnection): Promise<void> {
    if (!this.hasConnectionBorrowers(connection)) {
      return Promise.resolve();
    }

    return new Promise<void>((resolve) => {
      const drainWaiters = this.connectionBorrowerDrainWaiters.get(connection) ?? new Set();
      drainWaiters.add(resolve);
      this.connectionBorrowerDrainWaiters.set(connection, drainWaiters);
    });
  }

  protected bindRequestScopedConnectionStore(
    requestScopedConnections?: t.RequestScopedMCPConnectionStore,
  ): void {
    if (!requestScopedConnections || requestScopedConnections.disposeConnection) {
      return;
    }

    requestScopedConnections.disposeConnection = async (connectionKey, connection) => {
      await this.disposeEvictedConnection(
        connection as MCPConnection,
        `[MCP][Request-scoped: ${connectionKey}]`,
      );
    };
  }

  protected async disposeEvictedConnection(
    connection: MCPConnection,
    logPrefix: string,
  ): Promise<void> {
    this.deferredConnectionDisposals.set(connection, logPrefix);
    await this.finalizeDeferredConnectionDisposal(connection);
  }

  private async finalizeDeferredConnectionDisposal(connection: MCPConnection): Promise<void> {
    if (
      (this.connectionBorrowers.get(connection) ?? 0) > 0 ||
      (this.deferredConnectionDisposalHolds.get(connection) ?? 0) > 0
    ) {
      return;
    }

    const logPrefix = this.deferredConnectionDisposals.get(connection);
    if (!logPrefix) {
      return;
    }
    this.deferredConnectionDisposals.delete(connection);
    await this.disposeConnection(connection, logPrefix);
  }

  private async disposeConnection(connection: MCPConnection, logPrefix: string): Promise<void> {
    try {
      await connection.dispose();
    } catch (error) {
      logger.warn(`${logPrefix} Failed to dispose evicted connection`, error);
    }
  }

  /**
   * Disconnects and removes a specific user connection. Teardowns default to `mutation`, the
   * conservative reading for external callers, which all tear down after committing a change.
   */
  public async disconnectUserConnection(
    userId: string,
    serverName: string,
    { preservedCreation, reason = 'mutation', capabilityProfile }: ConnectionTeardownOptions = {},
  ): Promise<void> {
    if (capabilityProfile == null) {
      await Promise.all(
        MCP_CAPABILITY_PROFILES.map((profile) =>
          this.disconnectUserConnection(userId, serverName, {
            preservedCreation,
            reason,
            capabilityProfile: profile,
          }),
        ),
      );
      return;
    }
    const pendingKey = getUserConnectionKey(userId, serverName, capabilityProfile);
    const pending = this.pendingConnections.get(pendingKey);
    this.cancelConnectionCreations(pendingKey, reason, preservedCreation);
    if (pending && preservedCreation == null) {
      this.pendingConnections.delete(pendingKey);
    }
    const userMap = this.userConnections.get(userId);
    const connection = userMap?.get(getMCPConnectionPoolKey(serverName, capabilityProfile));
    const logPrefix = `[MCP][User: ${userId}]`;
    if (connection) {
      logger.info(`${logPrefix} Disconnecting server connection`);
      connection.removeAllListeners?.('toolsChanged');
      this.removeUserConnection(userId, serverName, capabilityProfile);
    }
    /**
     * Started in the same synchronous stretch as the fence: `cancelMCPToolsChanged` drops the
     * pending publication before it awaits, so a creation re-established while this teardown
     * is still disposing cannot have its own publication and retry timer cancelled by it.
     */
    const publicationCancelled = cancelMCPToolsChanged({
      userId,
      serverName,
      capabilityProfile,
    });
    /** Observed here so a rejection cannot surface as unhandled while disposal is awaited. */
    publicationCancelled.catch(() => undefined);
    try {
      if (connection) {
        await this.disposeEvictedConnection(connection, logPrefix);
      }
    } finally {
      await publicationCancelled;
    }
  }

  /** Disconnects and removes all connections for a specific user */
  public async disconnectUserConnections(
    userId: string,
    { preservedCreation, reason = 'mutation' }: ConnectionTeardownOptions = {},
  ): Promise<void> {
    for (const key of this.activeConnectionCreations.keys()) {
      if (isUserConnectionKeyForUser(key, userId)) {
        this.cancelConnectionCreations(key, reason, preservedCreation);
      }
    }
    if (preservedCreation == null) {
      for (const key of this.pendingConnections.keys()) {
        if (isUserConnectionKeyForUser(key, userId)) {
          this.pendingConnections.delete(key);
        }
      }
    }
    const userMap = this.userConnections.get(userId);
    const disconnectPromises: Promise<void>[] = [];
    if (userMap) {
      logger.info(`[MCP][User: ${userId}] Disconnecting all servers...`);
      const userServers = Array.from(
        new Set(Array.from(userMap.keys(), (key) => parseMCPConnectionPoolKey(key).serverName)),
      );
      for (const serverName of userServers) {
        disconnectPromises.push(
          this.disconnectUserConnection(userId, serverName, { preservedCreation, reason }).catch(
            () => {
              logger.error(`[MCP][User: ${userId}] Error during server disconnection`);
            },
          ),
        );
      }
      await Promise.allSettled(disconnectPromises);
      logger.info(`[MCP][User: ${userId}] All connections processed for disconnection.`);
    }
    /**
     * Clear the activity timestamp whenever nothing is left to track, including when userMap
     * was missing. `updateUserLastActivity` can be called before a connection is established
     * (e.g. in MCPManager.callTool prior to getConnection); if that connection attempt fails,
     * the activity entry would otherwise leak and trigger the idle check repeatedly for the
     * same userId. A connection installed while this teardown was disposing keeps its own
     * timestamp: `checkIdleConnections` walks activity rather than connections, so clearing it
     * would hide that connection from every later idle sweep.
     */
    if ((this.userConnections.get(userId)?.size ?? 0) === 0) {
      this.userLastActivity.delete(userId);
    }
  }

  /** Check for and disconnect idle connections */
  protected checkIdleConnections(currentUserId?: string): void {
    const now = Date.now();

    // Iterate through all users to check for idle ones
    for (const [userId, lastActivity] of this.userLastActivity.entries()) {
      if (currentUserId && currentUserId === userId) {
        continue;
      }
      if (now - lastActivity > mcpConfig.USER_CONNECTION_IDLE_TIMEOUT) {
        logger.info(
          `[MCP][User: ${userId}] User idle for too long. Disconnecting all connections...`,
        );
        // Disconnect all user connections asynchronously (fire and forget)
        this.disconnectUserConnections(userId, { reason: 'lifecycle' }).catch(() =>
          logger.error(`[MCP][User: ${userId}] Error disconnecting idle connections`),
        );
      }
    }
  }

  /** Returns counts of tracked users and connections for diagnostics */
  public getConnectionStats(): {
    trackedUsers: number;
    totalConnections: number;
    activityEntries: number;
    appConnectionCount: number;
  } {
    let totalConnections = 0;
    for (const serverMap of this.userConnections.values()) {
      totalConnections += serverMap.size;
    }
    return {
      trackedUsers: this.userConnections.size,
      totalConnections,
      activityEntries: this.userLastActivity.size,
      appConnectionCount: this.appConnections?.getConnectionCount() ?? 0,
    };
  }
}