Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active
TypeScript aianthropicartifactsawsazurechatgptchatgpt-cloneclaudeclonedeepseekgeminigooglegpt-5librechatmcpo1openairesponses-apivisionwebui
🪂 fix: Release Stalled MCP Catalog Recovery Without Dropping Refreshed Tokens (#15871)
Danny Avila2e1ccbb0fd4a
diff --git a/api/server/services/initializeMCPs.js b/api/server/services/initializeMCPs.jsindex 4796f38e95..cd18896f89 100644--- a/api/server/services/initializeMCPs.js+++ b/api/server/services/initializeMCPs.js@@ -116,6 +116,8 @@ async function initializeMCPs() {try {const mcpManager = await createMCPManager(mcpServers || {}, {catalogRecoveryMaxStateEntries: appConfig?.mcpSettings?.catalogRecovery?.maxStateEntries,+ catalogRecoveryMaxDetachedDiscoveries:+ appConfig?.mcpSettings?.catalogRecovery?.maxDetachedDiscoveries,});startMCPAuthorizationFenceRetryWorker(invalidateCachedTools, {intervalMs: appConfig?.mcpSettings?.catalogRecovery?.authorizationFenceRetryIntervalMs,diff --git a/api/server/services/initializeMCPs.spec.js b/api/server/services/initializeMCPs.spec.jsindex d1b17bff2d..0f1f9250e2 100644--- a/api/server/services/initializeMCPs.spec.js+++ b/api/server/services/initializeMCPs.spec.js@@ -239,6 +239,7 @@ describe('initializeMCPs', () => {{},{catalogRecoveryMaxStateEntries: undefined,+ catalogRecoveryMaxDetachedDiscoveries: undefined,},);});@@ -254,6 +255,7 @@ describe('initializeMCPs', () => {expect(mockCreateMCPManager).toHaveBeenCalledWith(mcpServers, {catalogRecoveryMaxStateEntries: undefined,+ catalogRecoveryMaxDetachedDiscoveries: undefined,});});@@ -263,6 +265,7 @@ describe('initializeMCPs', () => {mcpSettings: {catalogRecovery: {maxStateEntries: 2500,+ maxDetachedDiscoveries: 8,authorizationFenceRetryIntervalMs: 15_000,authorizationFenceRetryBatchSize: 250,authorizationFenceTimeoutMs: 750,@@ -276,6 +279,7 @@ describe('initializeMCPs', () => {{},{catalogRecoveryMaxStateEntries: 2500,+ catalogRecoveryMaxDetachedDiscoveries: 8,},);expect(mockStartMCPAuthorizationFenceRetryWorker).toHaveBeenCalledWith(@@ -465,6 +469,7 @@ describe('initializeMCPs', () => {{},{catalogRecoveryMaxStateEntries: undefined,+ catalogRecoveryMaxDetachedDiscoveries: undefined,},);});diff --git a/packages/api/src/mcp/MCPConnectionFactory.ts b/packages/api/src/mcp/MCPConnectionFactory.tsindex 8d470d1d87..81d7f7617b 100644--- a/packages/api/src/mcp/MCPConnectionFactory.ts+++ b/packages/api/src/mcp/MCPConnectionFactory.ts@@ -36,7 +36,12 @@ import {isMCPTransportAuthenticationError,MCPAuthenticationRejectedError,} from './errors';-import { createDeadlineAbortSignal, isClientRejectionMessage, isOAuthServer } from './utils';+import {+ isOAuthServer,+ waitUntilDeadline,+ isClientRejectionMessage,+ createDeadlineAbortSignal,+} from './utils';import { PENDING_STALE_MS, normalizeExpiresAt } from '~/flow/manager';import { preProcessGraphTokens } from '~/utils/graph';import { MCPConnection } from './connection';@@ -92,6 +97,7 @@ export class MCPConnectionFactory {protected readonly deadlineMs?: number;protected readonly onOAuthCredentialsChanged?: t.UserConnectionContext['onOAuthCredentialsChanged'];protected readonly onOAuthCredentialsChanging?: t.UserConnectionContext['onOAuthCredentialsChanging'];+ protected readonly onDiscoveryDetached?: t.UserConnectionContext['onDiscoveryDetached'];protected readonly oboTokenResolver?: OboTokenResolver;protected readonly oboTrustChecker?: OboTrustChecker;protected upstreamTokenProvider?: UpstreamTokenProvider;@@ -353,7 +359,20 @@ export class MCPConnectionFactory {);}} else if (this.useOAuth) {- oauthTokens = await this.getOAuthTokens();+ /** The token flow is shared, and a refresh inside it may already be redeemed at the provider+ * while its rotation persists. Stop waiting when the budget ends rather than cancelling that+ * work, so the flow still stores the new tokens for the next caller, and hand the caller the+ * work that keeps running so it can account for it. */+ const tokenLoad = this.getOAuthTokens();+ const loaded = await waitUntilDeadline(tokenLoad, this.deadlineMs, this.signal);+ if (!loaded.settled) {+ this.onDiscoveryDetached?.(tokenLoad);+ logger.debug(+ `${this.logPrefix} [Discovery] Cancelled or out of budget while loading OAuth tokens; leaving the token flow to finish`,+ );+ return { tools: null, connection: null, oauthRequired: false, oauthUrl: null };+ }+ oauthTokens = loaded.value;}let connection: MCPConnection | null = null;@@ -581,6 +600,7 @@ export class MCPConnectionFactory {this.deadlineMs = options?.deadlineMs;this.onOAuthCredentialsChanged = options?.onOAuthCredentialsChanged;this.onOAuthCredentialsChanging = options?.onOAuthCredentialsChanging;+ this.onDiscoveryDetached = options?.onDiscoveryDetached;this.signal = options?.signal;this.tenantContext = tenantStorage?.getStore?.();this.tenantId = this.tenantContext?.tenantId ?? getTenantId();diff --git a/packages/api/src/mcp/MCPManager.ts b/packages/api/src/mcp/MCPManager.tsindex ae33c8a8b7..7e7d273aa1 100644--- a/packages/api/src/mcp/MCPManager.ts+++ b/packages/api/src/mcp/MCPManager.ts@@ -110,20 +110,30 @@ export class MCPManager extends UserConnectionManager {}>();- constructor(catalogRecoveryMaxStateEntries?: number) {+ constructor(+ catalogRecoveryMaxStateEntries?: number,+ catalogRecoveryMaxDetachedDiscoveries?: number,+ ) {super();this.catalogRecoveryTracker = new MCPServerCatalogRecoveryTracker(catalogRecoveryMaxStateEntries,+ catalogRecoveryMaxDetachedDiscoveries,);}/** Creates and initializes the singleton MCPManager instance */public static async createInstance(configs: t.MCPServers,- options?: { catalogRecoveryMaxStateEntries?: number },+ options?: {+ catalogRecoveryMaxStateEntries?: number;+ catalogRecoveryMaxDetachedDiscoveries?: number;+ },): Promise<MCPManager> {if (MCPManager.instance) throw new Error('MCPManager has already been initialized.');- MCPManager.instance = new MCPManager(options?.catalogRecoveryMaxStateEntries);+ MCPManager.instance = new MCPManager(+ options?.catalogRecoveryMaxStateEntries,+ options?.catalogRecoveryMaxDetachedDiscoveries,+ );await MCPManager.instance.initialize(configs);return MCPManager.instance;}@@ -521,6 +531,7 @@ export class MCPManager extends UserConnectionManager {deadlineMs: args.deadlineMs,onOAuthCredentialsChanged: args.onOAuthCredentialsChanged,onOAuthCredentialsChanging: args.onOAuthCredentialsChanging,+ onDiscoveryDetached: args.onDiscoveryDetached,oboTokenResolver: args.oboTokenResolver,oboTrustChecker: args.oboTrustChecker,upstreamTokenProvider: args.upstreamTokenProvider,diff --git a/packages/api/src/mcp/__tests__/MCPConnectionFactory.test.ts b/packages/api/src/mcp/__tests__/MCPConnectionFactory.test.tsindex eb00688506..56d595980b 100644--- a/packages/api/src/mcp/__tests__/MCPConnectionFactory.test.ts+++ b/packages/api/src/mcp/__tests__/MCPConnectionFactory.test.ts@@ -4701,6 +4701,156 @@ describe('MCPConnectionFactory', () => {expect(result.tools).toBeNull();expect(Date.now() - start).toBeLessThan(2000);});++ describe('OAuth token loading', () => {+ const oauthDiscoveryOptions = (bounds: {+ deadlineMs?: number;+ signal?: AbortSignal;+ onDiscoveryDetached?: t.UserConnectionContext['onDiscoveryDetached'];+ }) => ({+ useOAuth: true as const,+ user: mockUser,+ flowManager: mockFlowManager,+ tokenMethods: {+ findToken: jest.fn(),+ createToken: jest.fn(),+ updateToken: jest.fn(),+ deleteTokens: jest.fn(),+ },+ ...bounds,+ });+ const cancelledDiscovery = {+ tools: null,+ connection: null,+ oauthRequired: false,+ oauthUrl: null,+ };+ const refreshUnavailable = () => {+ const error = new Error(+ 'OAuth token refresh is temporarily unavailable for "test-server"',+ );+ error.name = 'MCPTokenRefreshUnavailableError';+ return error;+ };+ const deferTokenFlow = () => {+ let resolve!: (tokens: MCPOAuthTokens | null) => void;+ let reject!: (error: Error) => void;+ const promise = new Promise<MCPOAuthTokens | null>((resolveFlow, rejectFlow) => {+ resolve = resolveFlow;+ reject = rejectFlow;+ });+ return { promise, resolve, reject };+ };+ type DeferredTokenFlow = ReturnType<typeof deferTokenFlow>;++ it.each<+ [string, (flow: DeferredTokenFlow) => void, PromiseSettledResult<unknown>['status']]+ >([+ [+ 'stores its tokens',+ (flow) =>+ flow.resolve({+ access_token: 'late-access',+ token_type: 'Bearer',+ obtained_at: Date.now(),+ credential_set_id: 'persisted-generation',+ }),+ 'fulfilled',+ ],+ ['fails', (flow) => flow.reject(refreshUnavailable()), 'rejected'],+ ])(+ 'stops waiting at the deadline for a stalled token flow that later %s, handing it off',+ async (_outcome, finish, settlement) => {+ const flow = deferTokenFlow();+ const onDiscoveryDetached = jest.fn();+ mockFlowManager.createFlowWithHandler.mockReturnValue(flow.promise);++ const start = Date.now();+ const result = await MCPConnectionFactory.discoverTools(+ { serverName: 'test-server', serverConfig: mockServerConfig },+ oauthDiscoveryOptions({ deadlineMs: Date.now() + 50, onDiscoveryDetached }),+ );++ expect(result).toEqual(cancelledDiscovery);+ expect(Date.now() - start).toBeLessThan(2000);+ expect(mockFlowManager.createFlowWithHandler).toHaveBeenCalledWith(+ expect.any(String),+ 'mcp_get_tokens',+ expect.any(Function),+ undefined,+ );+ expect(mockMCPConnection).not.toHaveBeenCalled();+ expect(onDiscoveryDetached).toHaveBeenCalledTimes(1);+ const [detachedWork] = onDiscoveryDetached.mock.calls[0] as [Promise<unknown>];++ finish(flow);+ await expect(Promise.allSettled([detachedWork])).resolves.toEqual([+ expect.objectContaining({ status: settlement }),+ ]);+ expect(mockFlowManager.deleteFlow).not.toHaveBeenCalled();+ expect(mockMCPConnection).not.toHaveBeenCalled();+ },+ );++ it('stops waiting for the token flow when the caller aborts, handing it off', async () => {+ const controller = new AbortController();+ const onDiscoveryDetached = jest.fn();+ mockFlowManager.createFlowWithHandler.mockImplementation(() => {+ setTimeout(() => controller.abort(), 20);+ return new Promise<MCPOAuthTokens | null>(() => undefined);+ });++ const start = Date.now();+ const result = await MCPConnectionFactory.discoverTools(+ { serverName: 'test-server', serverConfig: mockServerConfig },+ oauthDiscoveryOptions({ signal: controller.signal, onDiscoveryDetached }),+ );++ expect(result).toEqual(cancelledDiscovery);+ expect(Date.now() - start).toBeLessThan(2000);+ expect(mockMCPConnection).not.toHaveBeenCalled();+ expect(onDiscoveryDetached).toHaveBeenCalledWith(expect.any(Promise));+ });++ it('still surfaces a token failure that lands inside the budget', async () => {+ mockFlowManager.createFlowWithHandler.mockRejectedValue(refreshUnavailable());++ await expect(+ MCPConnectionFactory.discoverTools(+ { serverName: 'test-server', serverConfig: mockServerConfig },+ oauthDiscoveryOptions({ deadlineMs: Date.now() + 5000 }),+ ),+ ).rejects.toThrow('OAuth token refresh is temporarily unavailable');+ expect(mockMCPConnection).not.toHaveBeenCalled();+ });++ it('connects with tokens that load inside the budget', async () => {+ const tokens: MCPOAuthTokens = {+ access_token: 'fresh-access',+ token_type: 'Bearer',+ obtained_at: Date.now(),+ credential_set_id: 'persisted-generation',+ };+ mockFlowManager.createFlowWithHandler.mockResolvedValue(tokens);+ mockConnectionInstance.connect.mockResolvedValue(undefined);+ mockConnectionInstance.isConnected.mockResolvedValue(true);+ mockConnectionInstance.fetchOrderedToolsSnapshot = jest+ .fn()+ .mockResolvedValue({ tools: mockTools, complete: true });++ const onDiscoveryDetached = jest.fn();+ const result = await MCPConnectionFactory.discoverTools(+ { serverName: 'test-server', serverConfig: mockServerConfig },+ oauthDiscoveryOptions({ deadlineMs: Date.now() + 5000, onDiscoveryDetached }),+ );++ expect(result.tools).toEqual(mockTools);+ expect(mockMCPConnection).toHaveBeenCalledWith(+ expect.objectContaining({ oauthTokens: tokens }),+ );+ expect(onDiscoveryDetached).not.toHaveBeenCalled();+ });+ });});});diff --git a/packages/api/src/mcp/__tests__/MCPManager.test.ts b/packages/api/src/mcp/__tests__/MCPManager.test.tsindex a6411617b8..385262174b 100644--- a/packages/api/src/mcp/__tests__/MCPManager.test.ts+++ b/packages/api/src/mcp/__tests__/MCPManager.test.ts@@ -4303,12 +4303,14 @@ describe('MCPManager', () => {oauthUrl: 'https://auth.example.com/authorize',});+ const onDiscoveryDetached = jest.fn();const manager = await MCPManager.createInstance(newMCPServersConfig());const result = await manager.discoverServerTools({serverName,user: mockUser,flowManager: mockFlowManager as unknown as t.ToolDiscoveryOptions['flowManager'],graphTokenResolver: jest.fn(),+ onDiscoveryDetached,});expect(result.tools).toEqual(mockTools);@@ -4321,6 +4323,7 @@ describe('MCPManager', () => {user: mockUser,useOAuth: true,graphTokenResolver: expect.any(Function),+ onDiscoveryDetached,}),);});diff --git a/packages/api/src/mcp/__tests__/MCPOAuthTokenExpiry.test.ts b/packages/api/src/mcp/__tests__/MCPOAuthTokenExpiry.test.tsindex 340d11dee8..672ffe9611 100644--- a/packages/api/src/mcp/__tests__/MCPOAuthTokenExpiry.test.ts+++ b/packages/api/src/mcp/__tests__/MCPOAuthTokenExpiry.test.ts@@ -13,11 +13,14 @@import { Keyv } from 'keyv';import { logger } from '@librechat/data-schemas';+import type { IUser } from '@librechat/data-schemas';import type { OAuthTestServer } from './helpers/oauthTestServer';import type { MCPOAuthTokens } from '~/mcp/oauth';import { MockKeyv, InMemoryTokenStore, createOAuthMCPServer } from './helpers/oauthTestServer';-import { MCPTokenStorage, ReauthenticationRequiredError } from '~/mcp/oauth';+import { MCPTokenStorage, MCPOAuthHandler, ReauthenticationRequiredError } from '~/mcp/oauth';+import { prepareMCPAuthorizationMutation } from '~/mcp/authorization';import { FlowStateManager, PENDING_STALE_MS } from '~/flow/manager';+import { MCPConnectionFactory } from '~/mcp/MCPConnectionFactory';jest.mock('@librechat/data-schemas', () => ({logger: {@@ -45,6 +48,19 @@ function bindingMetadata(serverUrl: string) {};}+async function waitForCondition(+ predicate: () => boolean | Promise<boolean>,+ timeoutMs = 5000,+): Promise<void> {+ const deadline = Date.now() + timeoutMs;+ while (!(await predicate())) {+ if (Date.now() > deadline) {+ throw new Error('Timed out waiting for condition');+ }+ await new Promise((resolve) => setTimeout(resolve, 10));+ }+}+async function seedStoredClient(tokenStore: InMemoryTokenStore, serverUrl: string) {await tokenStore.createToken({userId: 'u1',@@ -167,6 +183,166 @@ describe('MCP OAuth Token Expiry Scenarios', () => {});});+ describe('Discovery budget ends while refreshed tokens wait on the authorization fence', () => {+ let server: OAuthTestServer;+ let tokenStore: InMemoryTokenStore;++ beforeEach(async () => {+ server = await createOAuthMCPServer({+ tokenTTLMs: 60000,+ issueRefreshTokens: true,+ rotateRefreshTokens: true,+ });+ tokenStore = new InMemoryTokenStore();+ });++ afterEach(async () => {+ await server.close();+ });++ it('returns within its budget and still stores the rotated tokens once the fence write lands', async () => {+ const code = await server.getAuthCode();+ const tokenRes = await fetch(`${server.url}token`, {+ method: 'POST',+ headers: { 'Content-Type': 'application/x-www-form-urlencoded' },+ body: `grant_type=authorization_code&code=${code}`,+ });+ const initial = (await tokenRes.json()) as { access_token: string; refresh_token: string };+ await tokenStore.createToken({+ userId: 'u1',+ type: 'mcp_oauth',+ identifier: 'mcp:test-srv',+ token: `enc:${initial.access_token}`,+ expiresIn: -1,+ metadata: tokenMetadata,+ });+ await tokenStore.createToken({+ userId: 'u1',+ type: 'mcp_oauth_refresh',+ identifier: 'mcp:test-srv:refresh',+ token: `enc:${initial.refresh_token}`,+ expiresIn: 86400,+ metadata: tokenMetadata,+ });+ await seedStoredClient(tokenStore, server.url);++ const tokenEndpoint = `${server.url}token`;+ /** Redeems at the test provider directly; provider discovery is outside this scenario. */+ class ProviderRefreshFactory extends MCPConnectionFactory {+ protected createRefreshTokensFunction() {+ return async (refreshToken: string): Promise<MCPOAuthTokens> => {+ const res = await fetch(tokenEndpoint, {+ method: 'POST',+ headers: { 'Content-Type': 'application/x-www-form-urlencoded' },+ body: `grant_type=refresh_token&refresh_token=${refreshToken}`,+ });+ const data = (await res.json()) as MCPOAuthTokens & { expires_in: number };+ return {+ ...data,+ obtained_at: Date.now(),+ expires_at: Date.now() + data.expires_in * 1000,+ };+ };+ }+ }++ let releaseFenceWrite: (() => void) | undefined;+ const fenceWrite = new Promise<void>((resolve) => {+ releaseFenceWrite = resolve;+ });+ const persistPublicationRetry = jest.fn(async () => {+ await fenceWrite;+ return 'retry-v1';+ });+ const invalidateRecoveryGeneration = jest.fn().mockResolvedValue('generation-2');+ const clearLocalRecovery = jest.fn();+ const onDiscoveryDetached = jest.fn();+ const flowManager = new FlowStateManager<MCPOAuthTokens | null>(+ new MockKeyv<MCPOAuthTokens | null>() as unknown as Keyv,+ { ttl: 30000, ci: true },+ );+ const refreshGrants = () =>+ server.tokenRequests.filter(({ grantType }) => grantType === 'refresh_token');+ const storedToken = async (type: string, identifier: string) =>+ (await tokenStore.findToken({ userId: 'u1', type, identifier }))?.token.replace(+ /^enc:/,+ '',+ );++ const budgetMs = 2_000;+ const startedAt = Date.now();+ const discovery = ProviderRefreshFactory.discoverTools(+ { serverName: 'test-srv', serverConfig: { type: 'streamable-http', url: server.url } },+ {+ useOAuth: true,+ user: { id: 'u1' } as IUser,+ flowManager,+ tokenMethods: {+ findToken: tokenStore.findToken,+ createToken: tokenStore.createToken,+ updateToken: tokenStore.updateToken,+ deleteTokens: tokenStore.deleteTokens,+ },+ deadlineMs: startedAt + budgetMs,+ onDiscoveryDetached,+ onOAuthCredentialsChanging: (scope) =>+ prepareMCPAuthorizationMutation(scope, {+ invalidateRecoveryGeneration,+ clearLocalRecovery,+ persistPublicationRetry,+ retryDelaysMs: [0],+ }),+ },+ );++ await waitForCondition(() => persistPublicationRetry.mock.calls.length === 1);+ expect(refreshGrants()).toHaveLength(1);+ expect(server.issuedRefreshTokens.has(initial.refresh_token)).toBe(false);++ await expect(discovery).resolves.toEqual({+ tools: null,+ connection: null,+ oauthRequired: false,+ oauthUrl: null,+ });+ expect(Date.now() - startedAt).toBeLessThan(budgetMs + 1_000);+ expect(await storedToken('mcp_oauth_refresh', 'mcp:test-srv:refresh')).toBe(+ initial.refresh_token,+ );+ expect(onDiscoveryDetached).toHaveBeenCalledTimes(1);+ const [tokenWork] = onDiscoveryDetached.mock.calls[0] as [Promise<unknown>];++ releaseFenceWrite?.();+ await Promise.allSettled([tokenWork]);+ const tokenFlowId = MCPOAuthHandler.generateTokenFlowId('u1', 'test-srv');+ expect((await flowManager.getFlowState(tokenFlowId, 'mcp_get_tokens'))?.status).toBe(+ 'COMPLETED',+ );++ const rotatedRefreshToken = await storedToken('mcp_oauth_refresh', 'mcp:test-srv:refresh');+ const refreshedAccessToken = await storedToken('mcp_oauth', 'mcp:test-srv');+ expect(server.issuedRefreshTokens.has(rotatedRefreshToken ?? '')).toBe(true);+ expect(server.issuedTokens.has(refreshedAccessToken ?? '')).toBe(true);+ expect(invalidateRecoveryGeneration).toHaveBeenCalledWith({+ userId: 'u1',+ serverName: 'test-srv',+ });+ expect(clearLocalRecovery).toHaveBeenCalledWith('u1', 'test-srv', 'generation-2');++ await expect(+ MCPTokenStorage.getTokens({+ userId: 'u1',+ serverName: 'test-srv',+ findToken: tokenStore.findToken,+ }),+ ).resolves.toMatchObject({+ access_token: refreshedAccessToken,+ refresh_token: rotatedRefreshToken,+ });+ expect(refreshGrants()).toHaveLength(1);+ });+ });+describe('Access token expired + refresh token rejected by server', () => {let tokenStore: InMemoryTokenStore;diff --git a/packages/api/src/mcp/__tests__/utils.test.ts b/packages/api/src/mcp/__tests__/utils.test.tsindex 4a8229c9d4..8b533241ad 100644--- a/packages/api/src/mcp/__tests__/utils.test.ts+++ b/packages/api/src/mcp/__tests__/utils.test.ts@@ -23,6 +23,7 @@ import {requiresOAuthMachinery,isChatSelectableMCPServer,filterChatSelectableMCPServers,+ waitUntilDeadline,} from '~/mcp/utils';describe('normalizeServerName', () => {@@ -1268,3 +1269,59 @@ describe('filterChatSelectableMCPServers', () => {await expect(filterChatSelectableMCPServers(null, opts)).resolves.toEqual([]);});});++describe('waitUntilDeadline', () => {+ const deferred = () => {+ let resolve!: (value: string) => void;+ let reject!: (error: Error) => void;+ const promise = new Promise<string>((resolveWork, rejectWork) => {+ resolve = resolveWork;+ reject = rejectWork;+ });+ return { promise, resolve, reject };+ };++ it('returns the value of work that settles before the deadline', async () => {+ await expect(waitUntilDeadline(Promise.resolve('tokens'), Date.now() + 1000)).resolves.toEqual({+ settled: true,+ value: 'tokens',+ });+ });++ it('rethrows a rejection that lands before the deadline', async () => {+ await expect(+ waitUntilDeadline(Promise.reject(new Error('storage unavailable')), Date.now() + 1000),+ ).rejects.toThrow('storage unavailable');+ });++ it('stops waiting at the deadline and keeps observing the abandoned work', async () => {+ const work = deferred();++ await expect(waitUntilDeadline(work.promise, Date.now() + 10)).resolves.toEqual({+ settled: false,+ });++ /** Jest fails the test if this late rejection goes unhandled. */+ work.reject(new Error('late failure'));+ await new Promise((resolve) => setImmediate(resolve));+ });++ it('stops waiting when the signal aborts, including one aborted before the wait', async () => {+ const aborting = new AbortController();+ const pending = deferred();+ const waiting = waitUntilDeadline(pending.promise, undefined, aborting.signal);+ aborting.abort();+ await expect(waiting).resolves.toEqual({ settled: false });++ const aborted = new AbortController();+ aborted.abort();+ const abandoned = deferred();+ await expect(waitUntilDeadline(abandoned.promise, undefined, aborted.signal)).resolves.toEqual({+ settled: false,+ });++ pending.reject(new Error('late failure'));+ abandoned.reject(new Error('late failure'));+ await new Promise((resolve) => setImmediate(resolve));+ });+});diff --git a/packages/api/src/mcp/catalog/recovery.spec.ts b/packages/api/src/mcp/catalog/recovery.spec.tsindex d3393a8642..f5206ee836 100644--- a/packages/api/src/mcp/catalog/recovery.spec.ts+++ b/packages/api/src/mcp/catalog/recovery.spec.ts@@ -2,8 +2,12 @@ import { logger } from '@librechat/data-schemas';import { Constants } from 'librechat-data-provider';import { ErrorCode, McpError, type Tool } from '@modelcontextprotocol/sdk/types.js';import type { IUser } from '@librechat/data-schemas';+import type {+ MCPRecoveryGenerationScope,+ MCPServerCatalogLoaderDeps,+ MCPServerCatalogRecoveryPolicy,+} from './recovery';import type { LCAvailableTools, ParsedServerConfig, ToolDiscoveryOptions } from '../types';-import type { MCPRecoveryGenerationScope, MCPServerCatalogLoaderDeps } from './recovery';import {MCPCatalogCapacityError,MCPServerCatalogRecoveryTracker,@@ -171,8 +175,10 @@ describe('MCPServerCatalogRecoveryTracker capacity', () => {const policy = {discoveryBackoffMs: [60_000],discoveryTimeoutMs: 3_000,+ discoverySettleGraceMs: 10_000,reauthRetryMs: 60_000,maxStateEntries: 2,+ maxDetachedDiscoveries: 3,generationReadTimeoutMs: 500,authorizationFenceRetryMs: [0],authorizationFenceTimeoutMs: 1_000,@@ -228,8 +234,10 @@ describe('MCPServerCatalogRecoveryTracker own credential publications', () => {const policy = {discoveryBackoffMs: [60_000],discoveryTimeoutMs: 3_000,+ discoverySettleGraceMs: 10_000,reauthRetryMs: 60_000,maxStateEntries: 10,+ maxDetachedDiscoveries: 3,generationReadTimeoutMs: 500,authorizationFenceRetryMs: [0],authorizationFenceTimeoutMs: 1_000,@@ -1085,6 +1093,7 @@ describe('loadMCPServerCatalogs — credential refresh during discovery', () =>let generation = 'generation-1';let readsFail = false;let publicationHold: Promise<void> | undefined;+ let retryIntentHold: Promise<void> | undefined;let retryCleanup: Promise<void> = Promise.resolve();const recoveryTracker = new MCPServerCatalogRecoveryTracker();const rotate = () => {@@ -1100,7 +1109,12 @@ describe('loadMCPServerCatalogs — credential refresh during discovery', () =>const onOAuthCredentialsChanging = (changed: MCPRecoveryGenerationScope) =>prepareMCPAuthorizationMutation(changed, {invalidateRecoveryGeneration,- persistPublicationRetry: async () => 'retry-v1',+ persistPublicationRetry: async () => {+ if (retryIntentHold != null) {+ await retryIntentHold;+ }+ return 'retry-v1';+ },clearPublicationRetry: () => retryCleanup,clearLocalRecovery: (userId, changedServerName, published) =>recoveryTracker.clear(userId, changedServerName, published),@@ -1114,6 +1128,17 @@ describe('loadMCPServerCatalogs — credential refresh during discovery', () =>stallRetryCleanup: () => {retryCleanup = new Promise<void>(() => undefined);},+ /** Holds the durable retry-intent write that precedes every token-row write. */+ holdRetryIntent: () => {+ let release: () => void = () => undefined;+ retryIntentHold = new Promise<void>((resolve) => {+ release = resolve;+ });+ return () => {+ retryIntentHold = undefined;+ release();+ };+ },getRecoveryGeneration: async () => {if (readsFail) {throw new Error('tool cache unavailable');@@ -1149,9 +1174,10 @@ describe('loadMCPServerCatalogs — credential refresh during discovery', () =>const loadCatalogs = (fence: ReturnType<typeof createFence>,discoverServerTools: MCPServerCatalogLoaderDeps['discoverServerTools'],+ recoveryPolicy?: Partial<MCPServerCatalogRecoveryPolicy>,) =>loadMCPServerCatalogs(- { user, servers },+ { user, servers, recoveryPolicy },{getCachedServerTools: jest.fn().mockResolvedValue(null),getServerToolFunctionsSnapshot: jest.fn().mockResolvedValue({ tools: null }),@@ -1264,6 +1290,36 @@ describe('loadMCPServerCatalogs — credential refresh during discovery', () =>expect(discoverServerTools).toHaveBeenCalledTimes(1);});+ it('rediscovers once a refresh released with its stalled flight publishes the rotation', async () => {+ const fence = createFence();+ const releaseRetryIntent = fence.holdRetryIntent();+ const recoveryPolicy = { discoveryTimeoutMs: 100, discoverySettleGraceMs: 20 };+ let refresh: Promise<void> | undefined;+ const discoverServerTools = jest.fn(async (options: ToolDiscoveryOptions) => {+ if (refresh == null) {+ refresh = refreshAccessToken(options);+ await refresh;+ }+ return { tools: listedTools };+ });++ const released = await loadCatalogs(fence, discoverServerTools, recoveryPolicy);+ const suppressed = await loadCatalogs(fence, discoverServerTools, recoveryPolicy);+ expect(released.serversWithoutTools).toEqual([serverName]);+ expect(suppressed.serversWithoutTools).toEqual([serverName]);+ expect(discoverServerTools).toHaveBeenCalledTimes(1);+ await expect(fence.getRecoveryGeneration()).resolves.toBe('generation-1');++ releaseRetryIntent();+ await refresh;+ await flush();+ await expect(fence.getRecoveryGeneration()).resolves.toBe('generation-2');++ const recovered = await loadCatalogs(fence, discoverServerTools, recoveryPolicy);+ expect(recovered.serverTools).toEqual(new Map([[serverName, recoveredTools]]));+ expect(discoverServerTools).toHaveBeenCalledTimes(2);+ });+it('discards a shared catalog that the shared generation cannot confirm for a request that saw a newer one', async () => {const fence = createFence();const releasePublication = fence.holdPublication();@@ -1741,3 +1797,334 @@ describe('recoverMCPServerCatalogs — bounded, skippable discovery', () => {);});});++describe('recoverMCPServerCatalogs — discovery that outlives its budget', () => {+ /** Stands in for discovery parked on an await that ignores its deadline, such as an OAuth refresh+ * whose redeemed tokens are waiting on a stalled authorization-fence write. */+ const stalledDiscovery = () => {+ const pending: Array<{+ resolve: (result: { tools: Tool[] | null }) => void;+ reject: (error: Error) => void;+ }> = [];+ const discoverServerTools = jest.fn(+ (_options: ToolDiscoveryOptions) =>+ new Promise<{ tools: Tool[] | null }>((resolve, reject) => {+ pending.push({ resolve, reject });+ }),+ );+ return { discoverServerTools, pending };+ };+ const recoveryPolicy = { discoveryTimeoutMs: 200, discoverySettleGraceMs: 50 };+ const stalledServers = [{ serverName: 'stalled', serverConfig: serverConfig('stalled') }];++ it('releases a stalled shared flight, the requests that joined it, and their catalog slots', async () => {+ const { discoverServerTools, pending } = stalledDiscovery();+ const deps = {+ loadUserMCPAuthMap: jest.fn().mockResolvedValue({}),+ discoverServerTools,+ formatServerTools: jest.fn().mockReturnValue(availableTools('late-tool')),+ recoveryTracker,+ };+ const healthyServers = [{ serverName: 'healthy', serverConfig: serverConfig('healthy') }];+ const healthyDeps = {+ ...deps,+ discoverServerTools: jest.fn().mockResolvedValue({ tools: [] }),+ formatServerTools: jest.fn().mockReturnValue(availableTools('healthy-tool')),+ };++ try {+ const joined = Array.from({ length: 3 }, () =>+ recoverMCPServerCatalogs({ user, servers: stalledServers, recoveryPolicy }, deps),+ );+ await new Promise((resolve) => setImmediate(resolve));+ expect(discoverServerTools).toHaveBeenCalledTimes(1);+ await expect(+ recoverMCPServerCatalogs({ user, servers: healthyServers, recoveryPolicy }, healthyDeps),+ ).rejects.toBeInstanceOf(MCPCatalogCapacityError);++ await expect(Promise.all(joined)).resolves.toEqual([new Map(), new Map(), new Map()]);+ expect(logger.warn).toHaveBeenCalledWith(+ expect.stringContaining('Discovery for stalled is still running 50ms past its budget'),+ );+ await expect(+ recoverMCPServerCatalogs({ user, servers: healthyServers, recoveryPolicy }, healthyDeps),+ ).resolves.toEqual(new Map([['healthy', availableTools('healthy-tool')]]));++ pending[0].resolve({ tools: [] });+ await new Promise((resolve) => setImmediate(resolve));+ await expect(+ recoverMCPServerCatalogs({ user, servers: stalledServers, recoveryPolicy }, deps),+ ).resolves.toEqual(new Map());+ expect(discoverServerTools).toHaveBeenCalledTimes(1);+ } finally {+ pending.forEach(({ resolve }) => resolve({ tools: null }));+ }+ });++ it('keeps observing a released discovery that fails afterwards', async () => {+ const { discoverServerTools, pending } = stalledDiscovery();++ const result = await recoverMCPServerCatalogs(+ { user, servers: stalledServers, recoveryPolicy },+ {+ loadUserMCPAuthMap: jest.fn().mockResolvedValue({}),+ discoverServerTools,+ formatServerTools: jest.fn(),+ recoveryTracker,+ },+ );+ expect(result).toEqual(new Map());++ /** Jest fails the test if this late rejection goes unhandled. */+ pending[0].reject(new Error('authorization fence write failed'));+ await new Promise((resolve) => setTimeout(resolve, 10));+ expect(discoverServerTools).toHaveBeenCalledTimes(1);+ });++ it('starts no second discovery for a server until its released discovery settles', async () => {+ const { discoverServerTools, pending } = stalledDiscovery();+ const deps = {+ loadUserMCPAuthMap: jest.fn().mockResolvedValue({}),+ discoverServerTools,+ formatServerTools: jest.fn().mockReturnValue(availableTools('late-tool')),+ recoveryTracker,+ };+ const servers = [{ serverName: 'held', serverConfig: serverConfig('held') }];+ const policy = { ...recoveryPolicy, discoveryBackoffMs: [10] };++ try {+ await expect(+ recoverMCPServerCatalogs({ user, servers, recoveryPolicy: policy }, deps),+ ).resolves.toEqual(new Map());+ await new Promise((resolve) => setTimeout(resolve, 20));+ await expect(+ recoverMCPServerCatalogs({ user, servers, recoveryPolicy: policy }, deps),+ ).resolves.toEqual(new Map());+ recoveryTracker.clear(user.id, 'held');+ await expect(+ recoverMCPServerCatalogs({ user, servers, recoveryPolicy: policy }, deps),+ ).resolves.toEqual(new Map());+ expect(discoverServerTools).toHaveBeenCalledTimes(1);++ pending[0].resolve({ tools: null });+ await new Promise((resolve) => setImmediate(resolve));+ discoverServerTools.mockResolvedValueOnce({ tools: [] });+ await expect(+ recoverMCPServerCatalogs({ user, servers, recoveryPolicy: policy }, deps),+ ).resolves.toEqual(new Map([['held', availableTools('late-tool')]]));+ expect(discoverServerTools).toHaveBeenCalledTimes(2);+ } finally {+ pending.forEach(({ resolve }) => resolve({ tools: null }));+ }+ });++ it('holds a request-bound server until its released discovery settles', async () => {+ const { discoverServerTools, pending } = stalledDiscovery();+ const deps = {+ loadUserMCPAuthMap: jest.fn().mockResolvedValue({}),+ discoverServerTools,+ formatServerTools: jest.fn().mockReturnValue(availableTools('obo-tool')),+ recoveryTracker,+ };+ const servers = [+ {+ serverName: 'obo',+ serverConfig: {+ ...serverConfig('obo'),+ obo: { scopes: 'api://mcp/.default' },+ } as ParsedServerConfig,+ },+ ];++ try {+ await recoverMCPServerCatalogs({ user, servers, recoveryPolicy }, deps);+ await expect(+ recoverMCPServerCatalogs({ user, servers, recoveryPolicy }, deps),+ ).resolves.toEqual(new Map());+ expect(discoverServerTools).toHaveBeenCalledTimes(1);++ pending[0].resolve({ tools: null });+ await new Promise((resolve) => setImmediate(resolve));+ discoverServerTools.mockResolvedValueOnce({ tools: [] });+ await expect(+ recoverMCPServerCatalogs({ user, servers, recoveryPolicy }, deps),+ ).resolves.toEqual(new Map([['obo', availableTools('obo-tool')]]));+ expect(discoverServerTools).toHaveBeenCalledTimes(2);+ } finally {+ pending.forEach(({ resolve }) => resolve({ tools: null }));+ }+ });++ it.each([+ [+ 'configuration',+ {+ serverConfig: { ...serverConfig('changed'), url: 'https://changed-v2.example.com/mcp' },+ generation: 'generation-1',+ },+ ],+ [+ 'credential generation',+ { serverConfig: serverConfig('changed'), generation: 'generation-2' },+ ],+ ])('lets a changed %s discover beside a released discovery', async (_change, next) => {+ let generation = 'generation-1';+ const { discoverServerTools, pending } = stalledDiscovery();+ const deps = {+ loadUserMCPAuthMap: jest.fn().mockResolvedValue({}),+ discoverServerTools,+ formatServerTools: jest.fn().mockReturnValue(availableTools('changed-tool')),+ recoveryTracker,+ getRecoveryGeneration: jest.fn(async () => generation),+ };+ const released = [{ serverName: 'changed', serverConfig: serverConfig('changed') }];++ try {+ await expect(+ recoverMCPServerCatalogs({ user, servers: released, recoveryPolicy }, deps),+ ).resolves.toEqual(new Map());+ generation = next.generation;+ discoverServerTools.mockResolvedValueOnce({ tools: [] });+ await expect(+ recoverMCPServerCatalogs(+ {+ user,+ servers: [+ { serverName: 'changed', serverConfig: next.serverConfig as ParsedServerConfig },+ ],+ recoveryPolicy,+ },+ deps,+ ),+ ).resolves.toEqual(new Map([['changed', availableTools('changed-tool')]]));+ expect(discoverServerTools).toHaveBeenCalledTimes(2);+ } finally {+ pending.forEach(({ resolve }) => resolve({ tools: null }));+ }+ });++ it('starts no new discovery while released discoveries fill the process-wide limit', async () => {+ const tracker = new MCPServerCatalogRecoveryTracker(undefined, 1);+ const { discoverServerTools, pending } = stalledDiscovery();+ const deps = {+ loadUserMCPAuthMap: jest.fn().mockResolvedValue({}),+ discoverServerTools,+ formatServerTools: jest.fn().mockReturnValue(availableTools('fresh-tool')),+ recoveryTracker: tracker,+ };+ const named = (serverName: string, overrides: Partial<ParsedServerConfig> = {}) => [+ {+ serverName,+ serverConfig: { ...serverConfig(serverName), ...overrides } as ParsedServerConfig,+ },+ ];++ try {+ await expect(+ recoverMCPServerCatalogs({ user, servers: named('stuck'), recoveryPolicy }, deps),+ ).resolves.toEqual(new Map());+ expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('reached the limit of 1'));+ await expect(+ recoverMCPServerCatalogs({ user, servers: named('other'), recoveryPolicy }, deps),+ ).resolves.toEqual(new Map());+ await expect(+ recoverMCPServerCatalogs(+ {+ user,+ servers: named('request-bound', { obo: { scopes: 'api://mcp/.default' } }),+ recoveryPolicy,+ },+ deps,+ ),+ ).resolves.toEqual(new Map());+ expect(discoverServerTools).toHaveBeenCalledTimes(1);++ pending[0].resolve({ tools: null });+ await new Promise((resolve) => setImmediate(resolve));+ discoverServerTools.mockResolvedValueOnce({ tools: [] });+ await expect(+ recoverMCPServerCatalogs({ user, servers: named('other'), recoveryPolicy }, deps),+ ).resolves.toEqual(new Map([['other', availableTools('fresh-tool')]]));+ expect(discoverServerTools).toHaveBeenCalledTimes(2);+ } finally {+ pending.forEach(({ resolve }) => resolve({ tools: null }));+ }+ });++ it('counts token work a discovery hands off at its deadline toward the hold and the limit', async () => {+ const tracker = new MCPServerCatalogRecoveryTracker(undefined, 1);+ let settleTokenWork: () => void = () => undefined;+ const tokenWork = new Promise<void>((resolve) => {+ settleTokenWork = resolve;+ });+ const discoverServerTools = jest.fn(async (options: ToolDiscoveryOptions) => {+ if (options.serverName !== 'refreshing') {+ return { tools: [] };+ }+ options.onDiscoveryDetached?.(tokenWork);+ return { tools: null };+ });+ const deps = {+ loadUserMCPAuthMap: jest.fn().mockResolvedValue({}),+ discoverServerTools,+ formatServerTools: jest.fn().mockReturnValue(availableTools('fresh-tool')),+ recoveryTracker: tracker,+ };+ const policy = { ...recoveryPolicy, discoveryBackoffMs: [10] };+ const named = (serverName: string) => [{ serverName, serverConfig: serverConfig(serverName) }];++ try {+ await expect(+ recoverMCPServerCatalogs(+ { user, servers: named('refreshing'), recoveryPolicy: policy },+ deps,+ ),+ ).resolves.toEqual(new Map());+ await new Promise((resolve) => setTimeout(resolve, 20));+ await expect(+ recoverMCPServerCatalogs(+ { user, servers: named('refreshing'), recoveryPolicy: policy },+ deps,+ ),+ ).resolves.toEqual(new Map());+ await expect(+ recoverMCPServerCatalogs({ user, servers: named('other'), recoveryPolicy: policy }, deps),+ ).resolves.toEqual(new Map());+ expect(discoverServerTools).toHaveBeenCalledTimes(1);++ settleTokenWork();+ await new Promise((resolve) => setImmediate(resolve));+ await expect(+ recoverMCPServerCatalogs({ user, servers: named('other'), recoveryPolicy: policy }, deps),+ ).resolves.toEqual(new Map([['other', availableTools('fresh-tool')]]));+ expect(discoverServerTools).toHaveBeenCalledTimes(2);+ } finally {+ settleTokenWork();+ }+ });++ it('lets a discovery that settles just past its budget keep its slot and its catalog', async () => {+ const discoverServerTools = jest.fn(async ({ deadlineMs }: ToolDiscoveryOptions) => {+ const remainingMs = Math.max(0, (deadlineMs ?? Date.now()) - Date.now());+ await new Promise((resolve) => setTimeout(resolve, remainingMs + 30));+ return { tools: [] };+ });++ const result = await recoverMCPServerCatalogs(+ {+ user,+ servers: [{ serverName: 'closing', serverConfig: serverConfig('closing') }],+ recoveryPolicy: { discoveryTimeoutMs: 20 },+ },+ {+ loadUserMCPAuthMap: jest.fn().mockResolvedValue({}),+ discoverServerTools,+ formatServerTools: jest.fn().mockReturnValue(availableTools('closing-tool')),+ recoveryTracker,+ },+ );++ expect(result).toEqual(new Map([['closing', availableTools('closing-tool')]]));+ expect(logger.warn).not.toHaveBeenCalledWith(expect.stringContaining('Discovery for closing'));+ });+});diff --git a/packages/api/src/mcp/catalog/recovery.ts b/packages/api/src/mcp/catalog/recovery.tsindex 59ffe7adc8..a1668bceb5 100644--- a/packages/api/src/mcp/catalog/recovery.ts+++ b/packages/api/src/mcp/catalog/recovery.ts@@ -3,7 +3,7 @@ import { ErrorCode, McpError } from '@modelcontextprotocol/sdk/types.js';import type { Tool } from '@modelcontextprotocol/sdk/types.js';import type { IUser } from '@librechat/data-schemas';import type { LCAvailableTools, ParsedServerConfig, ToolDiscoveryOptions } from '../types';-import { hasCustomUserVars, getMissingCustomUserVars } from '../utils';+import { hasCustomUserVars, waitUntilDeadline, getMissingCustomUserVars } from '../utils';import { usesDirectOpenIDBearerRecovery } from '../openid';import { getServerCustomUserVars } from '../auth';import { mcpConfig } from '../mcpConfig';@@ -15,15 +15,20 @@ const pendingCatalogLanes: CatalogWorkLane[] = [];let activeCatalogWork = 0;/*** Bounds one server's discovery end to end — connect, `tools/list` pagination, and the- * unauthenticated fallback all draw down this single budget, so a slot is held for at most this- * long regardless of where the server stalls. Recovery targets a server that is reachable and- * authorized but whose catalog cache expired, and such a server answers well inside this window.+ * unauthenticated fallback all draw down this single budget. Recovery targets a server that is+ * reachable and authorized but whose catalog cache expired, and such a server answers well inside+ * this window. The settle grace lets a discovery that honored the budget finish closing its+ * connections; past it, recovery releases the slot even if some dependency is still stalled. By+ * default no more released discoveries may still run than there are catalog slots, the most work a+ * stalled dependency could hold before discoveries were released.*/const DEFAULT_RECOVERY_POLICY: MCPServerCatalogRecoveryPolicy = {discoveryBackoffMs: [5 * 60_000, 10 * 60_000, 20 * 60_000, 30 * 60_000],discoveryTimeoutMs: 3_000,+ discoverySettleGraceMs: 10_000,reauthRetryMs: 30 * 60_000,maxStateEntries: 10_000,+ maxDetachedDiscoveries: CATALOG_FANOUT_CONCURRENCY,generationReadTimeoutMs: 500,authorizationFenceRetryMs: [0, 50, 200],authorizationFenceTimeoutMs: 1_000,@@ -149,8 +154,12 @@ export async function publishMCPAuthorizationMutation(export interface MCPServerCatalogRecoveryPolicy {discoveryBackoffMs: readonly number[];discoveryTimeoutMs: number;+ /** How long past its budget a discovery may keep its catalog slot and coalesced requests. */+ discoverySettleGraceMs: number;reauthRetryMs: number;maxStateEntries: number;+ /** How many released discoveries may still run before no new discovery starts. */+ maxDetachedDiscoveries: number;generationReadTimeoutMs: number;authorizationFenceRetryMs: readonly number[];authorizationFenceTimeoutMs: number;@@ -239,6 +248,15 @@ type PublicationTracker = (publish: () => Promise<string | undefined>,) => Promise<string | undefined>;+/** Hands the tracker a discovery released past its settle grace that is still running. */+type DiscoveryDetacher = (discovery: Promise<unknown>) => void;++/** The observed state a released discovery ran under; it holds only an attempt that matches it. */+interface DetachedDiscovery {+ configFingerprint: string;+ recoveryGeneration?: string;+}+interface CatalogWorkLane {tasks: ReadonlyArray<() => Promise<void>>;resolve: (admitted: boolean) => void;@@ -337,8 +355,96 @@ function getRecoveryFingerprint(export class MCPServerCatalogRecoveryTracker {private readonly states = new Map<string, RecoveryStateEntry>();+ /** Discoveries released past their settle grace that are still running, by recovery key. */+ private readonly detachedDiscoveries = new Map<string, DetachedDiscovery[]>();+ private detachedDiscoveryCount = 0;++ constructor(+ private readonly maxStateEntries: number = DEFAULT_RECOVERY_POLICY.maxStateEntries,+ private readonly maxDetachedDiscoveries: number = DEFAULT_RECOVERY_POLICY.maxDetachedDiscoveries,+ ) {}- constructor(private readonly maxStateEntries: number = DEFAULT_RECOVERY_POLICY.maxStateEntries) {}+ /** Whether a request-bound discovery, which never joins a flight, may start. */+ public admitsDiscovery(+ user: IUser,+ candidate: RecoveryCandidate,+ policy: MCPServerCatalogRecoveryPolicy,+ ): boolean {+ return !this.isDiscoveryHeld(+ getRecoveryKey(user.id, candidate.serverName),+ getRecoveryFingerprint(candidate.serverConfig, policy),+ undefined,+ );+ }++ /** Counts a request-bound discovery released past its settle grace until it settles. */+ public detachDiscovery(+ user: IUser,+ candidate: RecoveryCandidate,+ policy: MCPServerCatalogRecoveryPolicy,+ discovery: Promise<unknown>,+ ): void {+ this.detach(+ getRecoveryKey(user.id, candidate.serverName),+ { configFingerprint: getRecoveryFingerprint(candidate.serverConfig, policy) },+ discovery,+ );+ }++ /**+ * A released discovery keeps running, so another attempt under the same observed state would stack+ * on the same stuck dependency. Released discoveries also share one process-wide limit: once that+ * many still run, no new discovery starts until one settles. A changed configuration or credential+ * generation is new state and may discover within that limit. Only settling lifts a hold; clears+ * do not. Discoveries already running when the limit fills can still be released, so the count+ * can exceed it by at most the catalog concurrency.+ */+ private isDiscoveryHeld(+ key: string,+ configFingerprint: string,+ recoveryGeneration: string | undefined,+ ): boolean {+ if (this.detachedDiscoveryCount >= this.maxDetachedDiscoveries) {+ return true;+ }+ return (+ this.detachedDiscoveries+ .get(key)+ ?.some(+ (detached) =>+ detached.configFingerprint === configFingerprint &&+ (recoveryGeneration == null || detached.recoveryGeneration === recoveryGeneration),+ ) === true+ );+ }++ private detach(key: string, detached: DetachedDiscovery, discovery: Promise<unknown>): void {+ const held = this.detachedDiscoveries.get(key);+ if (held == null) {+ this.detachedDiscoveries.set(key, [detached]);+ } else {+ held.push(detached);+ }+ this.detachedDiscoveryCount += 1;+ if (this.detachedDiscoveryCount === this.maxDetachedDiscoveries) {+ logger.warn(+ `[MCP catalog recovery] Released discoveries still running reached the limit of ${this.maxDetachedDiscoveries}; starting no new discovery until one settles`,+ );+ }+ const release = () => {+ const current = this.detachedDiscoveries.get(key);+ const index = current?.indexOf(detached) ?? -1;+ if (current == null || index < 0) {+ return;+ }+ current.splice(index, 1);+ if (current.length === 0) {+ this.detachedDiscoveries.delete(key);+ }+ this.detachedDiscoveryCount -= 1;+ };+ discovery.then(release, release);+ }private touch(key: string, entry: RecoveryStateEntry): void {if (this.states.get(key) !== entry) {@@ -393,12 +499,16 @@ export class MCPServerCatalogRecoveryTracker {candidate: RecoveryCandidate,policy: MCPServerCatalogRecoveryPolicy,recoveryGeneration: string | undefined,- discover: (trackPublication: PublicationTracker) => Promise<RecoveryOutcome>,+ discover: (+ trackPublication: PublicationTracker,+ detach: DiscoveryDetacher,+ ) => Promise<RecoveryOutcome>,): Promise<RecoveryOutcome> {const key = getRecoveryKey(user.id, candidate.serverName);const configFingerprint = getRecoveryFingerprint(candidate.serverConfig, policy);const now = Date.now();const existing = this.states.get(key);+ const held = this.isDiscoveryHeld(key, configFingerprint, recoveryGeneration);const sameObservedState =existing?.configFingerprint === configFingerprint &&(recoveryGeneration == null ||@@ -410,10 +520,13 @@ export class MCPServerCatalogRecoveryTracker {if (existing.inFlight != null) {return existing.inFlight;}- if (existing.outcome != null && existing.nextRetryAt > now) {+ if (existing.outcome != null && (held || existing.nextRetryAt > now)) {return Promise.resolve(existing.outcome);}}+ if (held) {+ return Promise.resolve({ serverName: candidate.serverName, tools: null });+ }const entry: RecoveryStateEntry = {configFingerprint,@@ -449,8 +562,14 @@ export class MCPServerCatalogRecoveryTracker {entry.publishing = false;}};+ const detach: DiscoveryDetacher = (discovery) =>+ this.detach(+ key,+ { configFingerprint, recoveryGeneration: entry.recoveryGeneration },+ discovery,+ );const flight = Promise.resolve()- .then(() => discover(trackPublication))+ .then(() => discover(trackPublication, detach)).finally(() => {settled = true;})@@ -543,21 +662,41 @@ async function discoverCandidate({signal,trackPublication,- }: { signal?: AbortSignal; trackPublication?: PublicationTracker } = {},+ onDetached,+ }: {+ signal?: AbortSignal;+ trackPublication?: PublicationTracker;+ onDetached?: DiscoveryDetacher;+ } = {},): Promise<RecoveryOutcome> {+ const deadlineMs = Date.now() + resolveBudget(serverConfig, policy);try {- const result = await deps.discoverServerTools({+ const pending = deps.discoverServerTools({user,serverName,configServers: { [serverName]: serverConfig },customUserVars,- deadlineMs: Date.now() + resolveBudget(serverConfig, policy),+ deadlineMs,signal,onOAuthCredentialsChanging: trackPublications(deps.onOAuthCredentialsChanging,trackPublication,),+ onDiscoveryDetached: onDetached,});+ /** Discovery can await work that ignores its budget — a token refresh persisting behind a+ * stalled write — and a shared flight has no request signal to end that wait, so it would hold+ * every coalesced request and its slot indefinitely. Stop waiting instead of cancelling: that+ * work still commits what it redeemed. */+ const discovery = await waitUntilDeadline(pending, deadlineMs + policy.discoverySettleGraceMs);+ if (!discovery.settled) {+ onDetached?.(pending);+ logger.warn(+ `[MCP catalog recovery] Discovery for ${serverName} is still running ${policy.discoverySettleGraceMs}ms past its budget; releasing its catalog slot while it finishes in the background`,+ );+ return { serverName, tools: null, ...(!signal?.aborted && { state: 'backoff' as const }) };+ }+ const result = discovery.value;const tools = result.tools == null ? null : deps.formatServerTools(serverName, result.tools);if (signal?.aborted) {return { serverName, tools: null };@@ -766,7 +905,13 @@ async function recoverMCPServerCatalogsWithState(candidate.serverConfig.obo != null ||usesDirectOpenIDBearerRecovery(candidate.serverConfig);if (usesRequestCredential) {- results[index] = await discoverCandidate(user, candidate, deps, policy, { signal });+ results[index] = tracker.admitsDiscovery(user, candidate, policy)+ ? await discoverCandidate(user, candidate, deps, policy, {+ signal,+ onDetached: (discovery) =>+ tracker.detachDiscovery(user, candidate, policy, discovery),+ })+ : { serverName: candidate.serverName, tools: null };return;}const observedGeneration = await readMCPRecoveryGeneration(@@ -792,8 +937,11 @@ async function recoverMCPServerCatalogsWithState(candidate,policy,recoveryGeneration,- (trackPublication) =>- discoverCandidate(user, candidate, deps, policy, { trackPublication }),+ (trackPublication, detach) =>+ discoverCandidate(user, candidate, deps, policy, {+ trackPublication,+ onDetached: detach,+ }),);const finalGeneration = await readMCPRecoveryGeneration({ userId: user.id, serverName: candidate.serverName },diff --git a/packages/api/src/mcp/types/index.ts b/packages/api/src/mcp/types/index.tsindex 8fac212b02..576f55f57d 100644--- a/packages/api/src/mcp/types/index.ts+++ b/packages/api/src/mcp/types/index.ts@@ -272,6 +272,11 @@ export interface UserConnectionContext {userId: string;serverName: string;}) => Promise<() => Promise<string | undefined>>;+ /**+ * Receives discovery work the caller stopped waiting for at its deadline or abort, such as an+ * OAuth token flow still persisting a refresh. The work keeps running; the promise settles with it.+ */+ onDiscoveryDetached?: (work: Promise<unknown>) => void;}export interface RequestScopedMCPConnectionStore {@@ -337,6 +342,7 @@ export interface ToolDiscoveryOptions {deadlineMs?: number;onOAuthCredentialsChanged?: (scope: { userId: string; serverName: string }) => Promise<void>;onOAuthCredentialsChanging?: UserConnectionContext['onOAuthCredentialsChanging'];+ onDiscoveryDetached?: UserConnectionContext['onDiscoveryDetached'];/** Pre-resolved config-source servers for tenant-scoped lookup */configServers?: Record<string, ParsedServerConfig>;oboTokenResolver?: OboTokenResolver;diff --git a/packages/api/src/mcp/utils.ts b/packages/api/src/mcp/utils.tsindex aa8b3af0c4..12edf4aaac 100644--- a/packages/api/src/mcp/utils.ts+++ b/packages/api/src/mcp/utils.ts@@ -761,6 +761,49 @@ export function createDeadlineAbortSignal(return budget ?? callerSignal;}+export type DeadlineWaitResult<T> = { settled: true; value: T } | { settled: false };++/**+ * Waits for `promise` only until `deadlineMs` passes or `signal` aborts, and never cancels it:+ * shared work such as an OAuth token refresh keeps running for whoever depends on its outcome. A+ * rejection that lands first is rethrown; one that lands after the wait ended stays observed, so+ * abandoned work never surfaces as an unhandled rejection.+ */+export async function waitUntilDeadline<T>(+ promise: Promise<T>,+ deadlineMs?: number,+ signal?: AbortSignal,+): Promise<DeadlineWaitResult<T>> {+ if (deadlineMs == null && signal == null) {+ return { settled: true, value: await promise };+ }+ const settlement = promise.then((value): DeadlineWaitResult<T> => ({ settled: true, value }));+ if (signal?.aborted === true) {+ settlement.catch(() => undefined);+ return { settled: false };+ }+ let timer: NodeJS.Timeout | undefined;+ let onAbort: (() => void) | undefined;+ const interrupted = new Promise<DeadlineWaitResult<T>>((resolve) => {+ if (deadlineMs != null) {+ timer = setTimeout(() => resolve({ settled: false }), Math.max(0, deadlineMs - Date.now()));+ timer.unref?.();+ }+ if (signal != null) {+ onAbort = () => resolve({ settled: false });+ signal.addEventListener('abort', onAbort, { once: true });+ }+ });+ try {+ return await Promise.race([settlement, interrupted]);+ } finally {+ clearTimeout(timer);+ if (onAbort != null) {+ signal?.removeEventListener('abort', onAbort);+ }+ }+}+export function generateServerNameFromTitle(title: string): string {const slug = title.toLowerCase()diff --git a/packages/data-provider/src/config.spec.ts b/packages/data-provider/src/config.spec.tsindex 71cf1e0d7b..dfa3f34196 100644--- a/packages/data-provider/src/config.spec.ts+++ b/packages/data-provider/src/config.spec.ts@@ -988,8 +988,10 @@ describe('allowedAddressesSchema', () => {expect(defaults.mcpSettings?.catalogRecovery).toEqual({discoveryBackoffMs: [300_000, 600_000, 1_200_000, 1_800_000],discoveryTimeoutMs: 3_000,+ discoverySettleGraceMs: 10_000,reauthRetryMs: 1_800_000,maxStateEntries: 10_000,+ maxDetachedDiscoveries: 3,generationReadTimeoutMs: 500,authorizationFenceRetryMs: [0, 50, 200],authorizationFenceTimeoutMs: 1_000,@@ -1009,6 +1011,24 @@ describe('allowedAddressesSchema', () => {mcpSettings: { catalogRecovery: { discoveryTimeoutMs: 0 } },}).success,).toBe(false);+ expect(+ configSchema.safeParse({+ version: '1.0',+ mcpSettings: { catalogRecovery: { discoverySettleGraceMs: -1 } },+ }).success,+ ).toBe(false);+ expect(+ configSchema.safeParse({+ version: '1.0',+ mcpSettings: { catalogRecovery: { maxDetachedDiscoveries: 0 } },+ }).success,+ ).toBe(false);+ expect(+ configSchema.parse({+ version: '1.0',+ mcpSettings: { catalogRecovery: { discoverySettleGraceMs: 0 } },+ }).mcpSettings?.catalogRecovery.discoverySettleGraceMs,+ ).toBe(0);expect(configSchema.safeParse({version: '1.0',diff --git a/packages/data-provider/src/config.ts b/packages/data-provider/src/config.tsindex 1799e71653..965fb2ee4e 100644--- a/packages/data-provider/src/config.ts+++ b/packages/data-provider/src/config.ts@@ -2730,6 +2730,15 @@ export const configSchema = z.object({.positive().max(5 * 60_000).default(3_000),+ /** How long past `discoveryTimeoutMs` a stalled discovery may hold its catalog slot and+ * coalesced requests. It is never cancelled, so OAuth tokens it redeemed still persist,+ * and no other discovery for the same server state starts until it settles. */+ discoverySettleGraceMs: z+ .number()+ .int()+ .nonnegative()+ .max(5 * 60_000)+ .default(10_000),reauthRetryMs: z.number().int()@@ -2737,6 +2746,10 @@ export const configSchema = z.object({.max(24 * 60 * 60_000).default(30 * 60_000),maxStateEntries: z.number().int().positive().max(1_000_000).default(10_000),+ /** Process-wide: how many discoveries released past `discoverySettleGraceMs` may still be+ * running before recovery starts no new discovery until one settles. The default matches+ * the three catalog slots a stalled dependency could hold before discoveries were released. */+ maxDetachedDiscoveries: z.number().int().positive().max(1_000).default(3),generationReadTimeoutMs: z.number().int().positive().max(10_000).default(500),authorizationFenceRetryMs: z.array(z.number().int().nonnegative().max(60_000))