Skip to content

Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active

🪂 fix: Release Stalled MCP Catalog Recovery Without Dropping Refreshed Tokens (#15871)

Danny Avila2e1ccbb0fd4a
diff --git a/api/server/services/initializeMCPs.js b/api/server/services/initializeMCPs.js
index 4796f38e95..cd18896f89 100644
--- a/api/server/services/initializeMCPs.js
+++ b/api/server/services/initializeMCPs.js
@@ -116,6 +116,8 @@ async function initializeMCPs() {
try {
const mcpManager = await createMCPManager(mcpServers || {}, {
catalogRecoveryMaxStateEntries: appConfig?.mcpSettings?.catalogRecovery?.maxStateEntries,
+ catalogRecoveryMaxDetachedDiscoveries:
+ appConfig?.mcpSettings?.catalogRecovery?.maxDetachedDiscoveries,
});
startMCPAuthorizationFenceRetryWorker(invalidateCachedTools, {
intervalMs: appConfig?.mcpSettings?.catalogRecovery?.authorizationFenceRetryIntervalMs,
diff --git a/api/server/services/initializeMCPs.spec.js b/api/server/services/initializeMCPs.spec.js
index d1b17bff2d..0f1f9250e2 100644
--- a/api/server/services/initializeMCPs.spec.js
+++ b/api/server/services/initializeMCPs.spec.js
@@ -239,6 +239,7 @@ describe('initializeMCPs', () => {
{},
{
catalogRecoveryMaxStateEntries: undefined,
+ catalogRecoveryMaxDetachedDiscoveries: undefined,
},
);
});
@@ -254,6 +255,7 @@ describe('initializeMCPs', () => {
expect(mockCreateMCPManager).toHaveBeenCalledWith(mcpServers, {
catalogRecoveryMaxStateEntries: undefined,
+ catalogRecoveryMaxDetachedDiscoveries: undefined,
});
});
@@ -263,6 +265,7 @@ describe('initializeMCPs', () => {
mcpSettings: {
catalogRecovery: {
maxStateEntries: 2500,
+ maxDetachedDiscoveries: 8,
authorizationFenceRetryIntervalMs: 15_000,
authorizationFenceRetryBatchSize: 250,
authorizationFenceTimeoutMs: 750,
@@ -276,6 +279,7 @@ describe('initializeMCPs', () => {
{},
{
catalogRecoveryMaxStateEntries: 2500,
+ catalogRecoveryMaxDetachedDiscoveries: 8,
},
);
expect(mockStartMCPAuthorizationFenceRetryWorker).toHaveBeenCalledWith(
@@ -465,6 +469,7 @@ describe('initializeMCPs', () => {
{},
{
catalogRecoveryMaxStateEntries: undefined,
+ catalogRecoveryMaxDetachedDiscoveries: undefined,
},
);
});
diff --git a/packages/api/src/mcp/MCPConnectionFactory.ts b/packages/api/src/mcp/MCPConnectionFactory.ts
index 8d470d1d87..81d7f7617b 100644
--- a/packages/api/src/mcp/MCPConnectionFactory.ts
+++ b/packages/api/src/mcp/MCPConnectionFactory.ts
@@ -36,7 +36,12 @@ import {
isMCPTransportAuthenticationError,
MCPAuthenticationRejectedError,
} from './errors';
-import { createDeadlineAbortSignal, isClientRejectionMessage, isOAuthServer } from './utils';
+import {
+ isOAuthServer,
+ waitUntilDeadline,
+ isClientRejectionMessage,
+ createDeadlineAbortSignal,
+} from './utils';
import { PENDING_STALE_MS, normalizeExpiresAt } from '~/flow/manager';
import { preProcessGraphTokens } from '~/utils/graph';
import { MCPConnection } from './connection';
@@ -92,6 +97,7 @@ export class MCPConnectionFactory {
protected readonly deadlineMs?: number;
protected readonly onOAuthCredentialsChanged?: t.UserConnectionContext['onOAuthCredentialsChanged'];
protected readonly onOAuthCredentialsChanging?: t.UserConnectionContext['onOAuthCredentialsChanging'];
+ protected readonly onDiscoveryDetached?: t.UserConnectionContext['onDiscoveryDetached'];
protected readonly oboTokenResolver?: OboTokenResolver;
protected readonly oboTrustChecker?: OboTrustChecker;
protected upstreamTokenProvider?: UpstreamTokenProvider;
@@ -353,7 +359,20 @@ export class MCPConnectionFactory {
);
}
} else if (this.useOAuth) {
- oauthTokens = await this.getOAuthTokens();
+ /** The token flow is shared, and a refresh inside it may already be redeemed at the provider
+ * while its rotation persists. Stop waiting when the budget ends rather than cancelling that
+ * work, so the flow still stores the new tokens for the next caller, and hand the caller the
+ * work that keeps running so it can account for it. */
+ const tokenLoad = this.getOAuthTokens();
+ const loaded = await waitUntilDeadline(tokenLoad, this.deadlineMs, this.signal);
+ if (!loaded.settled) {
+ this.onDiscoveryDetached?.(tokenLoad);
+ logger.debug(
+ `${this.logPrefix} [Discovery] Cancelled or out of budget while loading OAuth tokens; leaving the token flow to finish`,
+ );
+ return { tools: null, connection: null, oauthRequired: false, oauthUrl: null };
+ }
+ oauthTokens = loaded.value;
}
let connection: MCPConnection | null = null;
@@ -581,6 +600,7 @@ export class MCPConnectionFactory {
this.deadlineMs = options?.deadlineMs;
this.onOAuthCredentialsChanged = options?.onOAuthCredentialsChanged;
this.onOAuthCredentialsChanging = options?.onOAuthCredentialsChanging;
+ this.onDiscoveryDetached = options?.onDiscoveryDetached;
this.signal = options?.signal;
this.tenantContext = tenantStorage?.getStore?.();
this.tenantId = this.tenantContext?.tenantId ?? getTenantId();
diff --git a/packages/api/src/mcp/MCPManager.ts b/packages/api/src/mcp/MCPManager.ts
index ae33c8a8b7..7e7d273aa1 100644
--- a/packages/api/src/mcp/MCPManager.ts
+++ b/packages/api/src/mcp/MCPManager.ts
@@ -110,20 +110,30 @@ export class MCPManager extends UserConnectionManager {
}
>();
- constructor(catalogRecoveryMaxStateEntries?: number) {
+ constructor(
+ catalogRecoveryMaxStateEntries?: number,
+ catalogRecoveryMaxDetachedDiscoveries?: number,
+ ) {
super();
this.catalogRecoveryTracker = new MCPServerCatalogRecoveryTracker(
catalogRecoveryMaxStateEntries,
+ catalogRecoveryMaxDetachedDiscoveries,
);
}
/** Creates and initializes the singleton MCPManager instance */
public static async createInstance(
configs: t.MCPServers,
- options?: { catalogRecoveryMaxStateEntries?: number },
+ options?: {
+ catalogRecoveryMaxStateEntries?: number;
+ catalogRecoveryMaxDetachedDiscoveries?: number;
+ },
): Promise<MCPManager> {
if (MCPManager.instance) throw new Error('MCPManager has already been initialized.');
- MCPManager.instance = new MCPManager(options?.catalogRecoveryMaxStateEntries);
+ MCPManager.instance = new MCPManager(
+ options?.catalogRecoveryMaxStateEntries,
+ options?.catalogRecoveryMaxDetachedDiscoveries,
+ );
await MCPManager.instance.initialize(configs);
return MCPManager.instance;
}
@@ -521,6 +531,7 @@ export class MCPManager extends UserConnectionManager {
deadlineMs: args.deadlineMs,
onOAuthCredentialsChanged: args.onOAuthCredentialsChanged,
onOAuthCredentialsChanging: args.onOAuthCredentialsChanging,
+ onDiscoveryDetached: args.onDiscoveryDetached,
oboTokenResolver: args.oboTokenResolver,
oboTrustChecker: args.oboTrustChecker,
upstreamTokenProvider: args.upstreamTokenProvider,
diff --git a/packages/api/src/mcp/__tests__/MCPConnectionFactory.test.ts b/packages/api/src/mcp/__tests__/MCPConnectionFactory.test.ts
index eb00688506..56d595980b 100644
--- a/packages/api/src/mcp/__tests__/MCPConnectionFactory.test.ts
+++ b/packages/api/src/mcp/__tests__/MCPConnectionFactory.test.ts
@@ -4701,6 +4701,156 @@ describe('MCPConnectionFactory', () => {
expect(result.tools).toBeNull();
expect(Date.now() - start).toBeLessThan(2000);
});
+
+ describe('OAuth token loading', () => {
+ const oauthDiscoveryOptions = (bounds: {
+ deadlineMs?: number;
+ signal?: AbortSignal;
+ onDiscoveryDetached?: t.UserConnectionContext['onDiscoveryDetached'];
+ }) => ({
+ useOAuth: true as const,
+ user: mockUser,
+ flowManager: mockFlowManager,
+ tokenMethods: {
+ findToken: jest.fn(),
+ createToken: jest.fn(),
+ updateToken: jest.fn(),
+ deleteTokens: jest.fn(),
+ },
+ ...bounds,
+ });
+ const cancelledDiscovery = {
+ tools: null,
+ connection: null,
+ oauthRequired: false,
+ oauthUrl: null,
+ };
+ const refreshUnavailable = () => {
+ const error = new Error(
+ 'OAuth token refresh is temporarily unavailable for "test-server"',
+ );
+ error.name = 'MCPTokenRefreshUnavailableError';
+ return error;
+ };
+ const deferTokenFlow = () => {
+ let resolve!: (tokens: MCPOAuthTokens | null) => void;
+ let reject!: (error: Error) => void;
+ const promise = new Promise<MCPOAuthTokens | null>((resolveFlow, rejectFlow) => {
+ resolve = resolveFlow;
+ reject = rejectFlow;
+ });
+ return { promise, resolve, reject };
+ };
+ type DeferredTokenFlow = ReturnType<typeof deferTokenFlow>;
+
+ it.each<
+ [string, (flow: DeferredTokenFlow) => void, PromiseSettledResult<unknown>['status']]
+ >([
+ [
+ 'stores its tokens',
+ (flow) =>
+ flow.resolve({
+ access_token: 'late-access',
+ token_type: 'Bearer',
+ obtained_at: Date.now(),
+ credential_set_id: 'persisted-generation',
+ }),
+ 'fulfilled',
+ ],
+ ['fails', (flow) => flow.reject(refreshUnavailable()), 'rejected'],
+ ])(
+ 'stops waiting at the deadline for a stalled token flow that later %s, handing it off',
+ async (_outcome, finish, settlement) => {
+ const flow = deferTokenFlow();
+ const onDiscoveryDetached = jest.fn();
+ mockFlowManager.createFlowWithHandler.mockReturnValue(flow.promise);
+
+ const start = Date.now();
+ const result = await MCPConnectionFactory.discoverTools(
+ { serverName: 'test-server', serverConfig: mockServerConfig },
+ oauthDiscoveryOptions({ deadlineMs: Date.now() + 50, onDiscoveryDetached }),
+ );
+
+ expect(result).toEqual(cancelledDiscovery);
+ expect(Date.now() - start).toBeLessThan(2000);
+ expect(mockFlowManager.createFlowWithHandler).toHaveBeenCalledWith(
+ expect.any(String),
+ 'mcp_get_tokens',
+ expect.any(Function),
+ undefined,
+ );
+ expect(mockMCPConnection).not.toHaveBeenCalled();
+ expect(onDiscoveryDetached).toHaveBeenCalledTimes(1);
+ const [detachedWork] = onDiscoveryDetached.mock.calls[0] as [Promise<unknown>];
+
+ finish(flow);
+ await expect(Promise.allSettled([detachedWork])).resolves.toEqual([
+ expect.objectContaining({ status: settlement }),
+ ]);
+ expect(mockFlowManager.deleteFlow).not.toHaveBeenCalled();
+ expect(mockMCPConnection).not.toHaveBeenCalled();
+ },
+ );
+
+ it('stops waiting for the token flow when the caller aborts, handing it off', async () => {
+ const controller = new AbortController();
+ const onDiscoveryDetached = jest.fn();
+ mockFlowManager.createFlowWithHandler.mockImplementation(() => {
+ setTimeout(() => controller.abort(), 20);
+ return new Promise<MCPOAuthTokens | null>(() => undefined);
+ });
+
+ const start = Date.now();
+ const result = await MCPConnectionFactory.discoverTools(
+ { serverName: 'test-server', serverConfig: mockServerConfig },
+ oauthDiscoveryOptions({ signal: controller.signal, onDiscoveryDetached }),
+ );
+
+ expect(result).toEqual(cancelledDiscovery);
+ expect(Date.now() - start).toBeLessThan(2000);
+ expect(mockMCPConnection).not.toHaveBeenCalled();
+ expect(onDiscoveryDetached).toHaveBeenCalledWith(expect.any(Promise));
+ });
+
+ it('still surfaces a token failure that lands inside the budget', async () => {
+ mockFlowManager.createFlowWithHandler.mockRejectedValue(refreshUnavailable());
+
+ await expect(
+ MCPConnectionFactory.discoverTools(
+ { serverName: 'test-server', serverConfig: mockServerConfig },
+ oauthDiscoveryOptions({ deadlineMs: Date.now() + 5000 }),
+ ),
+ ).rejects.toThrow('OAuth token refresh is temporarily unavailable');
+ expect(mockMCPConnection).not.toHaveBeenCalled();
+ });
+
+ it('connects with tokens that load inside the budget', async () => {
+ const tokens: MCPOAuthTokens = {
+ access_token: 'fresh-access',
+ token_type: 'Bearer',
+ obtained_at: Date.now(),
+ credential_set_id: 'persisted-generation',
+ };
+ mockFlowManager.createFlowWithHandler.mockResolvedValue(tokens);
+ mockConnectionInstance.connect.mockResolvedValue(undefined);
+ mockConnectionInstance.isConnected.mockResolvedValue(true);
+ mockConnectionInstance.fetchOrderedToolsSnapshot = jest
+ .fn()
+ .mockResolvedValue({ tools: mockTools, complete: true });
+
+ const onDiscoveryDetached = jest.fn();
+ const result = await MCPConnectionFactory.discoverTools(
+ { serverName: 'test-server', serverConfig: mockServerConfig },
+ oauthDiscoveryOptions({ deadlineMs: Date.now() + 5000, onDiscoveryDetached }),
+ );
+
+ expect(result.tools).toEqual(mockTools);
+ expect(mockMCPConnection).toHaveBeenCalledWith(
+ expect.objectContaining({ oauthTokens: tokens }),
+ );
+ expect(onDiscoveryDetached).not.toHaveBeenCalled();
+ });
+ });
});
});
diff --git a/packages/api/src/mcp/__tests__/MCPManager.test.ts b/packages/api/src/mcp/__tests__/MCPManager.test.ts
index a6411617b8..385262174b 100644
--- a/packages/api/src/mcp/__tests__/MCPManager.test.ts
+++ b/packages/api/src/mcp/__tests__/MCPManager.test.ts
@@ -4303,12 +4303,14 @@ describe('MCPManager', () => {
oauthUrl: 'https://auth.example.com/authorize',
});
+ const onDiscoveryDetached = jest.fn();
const manager = await MCPManager.createInstance(newMCPServersConfig());
const result = await manager.discoverServerTools({
serverName,
user: mockUser,
flowManager: mockFlowManager as unknown as t.ToolDiscoveryOptions['flowManager'],
graphTokenResolver: jest.fn(),
+ onDiscoveryDetached,
});
expect(result.tools).toEqual(mockTools);
@@ -4321,6 +4323,7 @@ describe('MCPManager', () => {
user: mockUser,
useOAuth: true,
graphTokenResolver: expect.any(Function),
+ onDiscoveryDetached,
}),
);
});
diff --git a/packages/api/src/mcp/__tests__/MCPOAuthTokenExpiry.test.ts b/packages/api/src/mcp/__tests__/MCPOAuthTokenExpiry.test.ts
index 340d11dee8..672ffe9611 100644
--- a/packages/api/src/mcp/__tests__/MCPOAuthTokenExpiry.test.ts
+++ b/packages/api/src/mcp/__tests__/MCPOAuthTokenExpiry.test.ts
@@ -13,11 +13,14 @@
import { Keyv } from 'keyv';
import { logger } from '@librechat/data-schemas';
+import type { IUser } from '@librechat/data-schemas';
import type { OAuthTestServer } from './helpers/oauthTestServer';
import type { MCPOAuthTokens } from '~/mcp/oauth';
import { MockKeyv, InMemoryTokenStore, createOAuthMCPServer } from './helpers/oauthTestServer';
-import { MCPTokenStorage, ReauthenticationRequiredError } from '~/mcp/oauth';
+import { MCPTokenStorage, MCPOAuthHandler, ReauthenticationRequiredError } from '~/mcp/oauth';
+import { prepareMCPAuthorizationMutation } from '~/mcp/authorization';
import { FlowStateManager, PENDING_STALE_MS } from '~/flow/manager';
+import { MCPConnectionFactory } from '~/mcp/MCPConnectionFactory';
jest.mock('@librechat/data-schemas', () => ({
logger: {
@@ -45,6 +48,19 @@ function bindingMetadata(serverUrl: string) {
};
}
+async function waitForCondition(
+ predicate: () => boolean | Promise<boolean>,
+ timeoutMs = 5000,
+): Promise<void> {
+ const deadline = Date.now() + timeoutMs;
+ while (!(await predicate())) {
+ if (Date.now() > deadline) {
+ throw new Error('Timed out waiting for condition');
+ }
+ await new Promise((resolve) => setTimeout(resolve, 10));
+ }
+}
+
async function seedStoredClient(tokenStore: InMemoryTokenStore, serverUrl: string) {
await tokenStore.createToken({
userId: 'u1',
@@ -167,6 +183,166 @@ describe('MCP OAuth Token Expiry Scenarios', () => {
});
});
+ describe('Discovery budget ends while refreshed tokens wait on the authorization fence', () => {
+ let server: OAuthTestServer;
+ let tokenStore: InMemoryTokenStore;
+
+ beforeEach(async () => {
+ server = await createOAuthMCPServer({
+ tokenTTLMs: 60000,
+ issueRefreshTokens: true,
+ rotateRefreshTokens: true,
+ });
+ tokenStore = new InMemoryTokenStore();
+ });
+
+ afterEach(async () => {
+ await server.close();
+ });
+
+ it('returns within its budget and still stores the rotated tokens once the fence write lands', async () => {
+ const code = await server.getAuthCode();
+ const tokenRes = await fetch(`${server.url}token`, {
+ method: 'POST',
+ headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
+ body: `grant_type=authorization_code&code=${code}`,
+ });
+ const initial = (await tokenRes.json()) as { access_token: string; refresh_token: string };
+ await tokenStore.createToken({
+ userId: 'u1',
+ type: 'mcp_oauth',
+ identifier: 'mcp:test-srv',
+ token: `enc:${initial.access_token}`,
+ expiresIn: -1,
+ metadata: tokenMetadata,
+ });
+ await tokenStore.createToken({
+ userId: 'u1',
+ type: 'mcp_oauth_refresh',
+ identifier: 'mcp:test-srv:refresh',
+ token: `enc:${initial.refresh_token}`,
+ expiresIn: 86400,
+ metadata: tokenMetadata,
+ });
+ await seedStoredClient(tokenStore, server.url);
+
+ const tokenEndpoint = `${server.url}token`;
+ /** Redeems at the test provider directly; provider discovery is outside this scenario. */
+ class ProviderRefreshFactory extends MCPConnectionFactory {
+ protected createRefreshTokensFunction() {
+ return async (refreshToken: string): Promise<MCPOAuthTokens> => {
+ const res = await fetch(tokenEndpoint, {
+ method: 'POST',
+ headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
+ body: `grant_type=refresh_token&refresh_token=${refreshToken}`,
+ });
+ const data = (await res.json()) as MCPOAuthTokens & { expires_in: number };
+ return {
+ ...data,
+ obtained_at: Date.now(),
+ expires_at: Date.now() + data.expires_in * 1000,
+ };
+ };
+ }
+ }
+
+ let releaseFenceWrite: (() => void) | undefined;
+ const fenceWrite = new Promise<void>((resolve) => {
+ releaseFenceWrite = resolve;
+ });
+ const persistPublicationRetry = jest.fn(async () => {
+ await fenceWrite;
+ return 'retry-v1';
+ });
+ const invalidateRecoveryGeneration = jest.fn().mockResolvedValue('generation-2');
+ const clearLocalRecovery = jest.fn();
+ const onDiscoveryDetached = jest.fn();
+ const flowManager = new FlowStateManager<MCPOAuthTokens | null>(
+ new MockKeyv<MCPOAuthTokens | null>() as unknown as Keyv,
+ { ttl: 30000, ci: true },
+ );
+ const refreshGrants = () =>
+ server.tokenRequests.filter(({ grantType }) => grantType === 'refresh_token');
+ const storedToken = async (type: string, identifier: string) =>
+ (await tokenStore.findToken({ userId: 'u1', type, identifier }))?.token.replace(
+ /^enc:/,
+ '',
+ );
+
+ const budgetMs = 2_000;
+ const startedAt = Date.now();
+ const discovery = ProviderRefreshFactory.discoverTools(
+ { serverName: 'test-srv', serverConfig: { type: 'streamable-http', url: server.url } },
+ {
+ useOAuth: true,
+ user: { id: 'u1' } as IUser,
+ flowManager,
+ tokenMethods: {
+ findToken: tokenStore.findToken,
+ createToken: tokenStore.createToken,
+ updateToken: tokenStore.updateToken,
+ deleteTokens: tokenStore.deleteTokens,
+ },
+ deadlineMs: startedAt + budgetMs,
+ onDiscoveryDetached,
+ onOAuthCredentialsChanging: (scope) =>
+ prepareMCPAuthorizationMutation(scope, {
+ invalidateRecoveryGeneration,
+ clearLocalRecovery,
+ persistPublicationRetry,
+ retryDelaysMs: [0],
+ }),
+ },
+ );
+
+ await waitForCondition(() => persistPublicationRetry.mock.calls.length === 1);
+ expect(refreshGrants()).toHaveLength(1);
+ expect(server.issuedRefreshTokens.has(initial.refresh_token)).toBe(false);
+
+ await expect(discovery).resolves.toEqual({
+ tools: null,
+ connection: null,
+ oauthRequired: false,
+ oauthUrl: null,
+ });
+ expect(Date.now() - startedAt).toBeLessThan(budgetMs + 1_000);
+ expect(await storedToken('mcp_oauth_refresh', 'mcp:test-srv:refresh')).toBe(
+ initial.refresh_token,
+ );
+ expect(onDiscoveryDetached).toHaveBeenCalledTimes(1);
+ const [tokenWork] = onDiscoveryDetached.mock.calls[0] as [Promise<unknown>];
+
+ releaseFenceWrite?.();
+ await Promise.allSettled([tokenWork]);
+ const tokenFlowId = MCPOAuthHandler.generateTokenFlowId('u1', 'test-srv');
+ expect((await flowManager.getFlowState(tokenFlowId, 'mcp_get_tokens'))?.status).toBe(
+ 'COMPLETED',
+ );
+
+ const rotatedRefreshToken = await storedToken('mcp_oauth_refresh', 'mcp:test-srv:refresh');
+ const refreshedAccessToken = await storedToken('mcp_oauth', 'mcp:test-srv');
+ expect(server.issuedRefreshTokens.has(rotatedRefreshToken ?? '')).toBe(true);
+ expect(server.issuedTokens.has(refreshedAccessToken ?? '')).toBe(true);
+ expect(invalidateRecoveryGeneration).toHaveBeenCalledWith({
+ userId: 'u1',
+ serverName: 'test-srv',
+ });
+ expect(clearLocalRecovery).toHaveBeenCalledWith('u1', 'test-srv', 'generation-2');
+
+ await expect(
+ MCPTokenStorage.getTokens({
+ userId: 'u1',
+ serverName: 'test-srv',
+ findToken: tokenStore.findToken,
+ }),
+ ).resolves.toMatchObject({
+ access_token: refreshedAccessToken,
+ refresh_token: rotatedRefreshToken,
+ });
+ expect(refreshGrants()).toHaveLength(1);
+ });
+ });
+
describe('Access token expired + refresh token rejected by server', () => {
let tokenStore: InMemoryTokenStore;
diff --git a/packages/api/src/mcp/__tests__/utils.test.ts b/packages/api/src/mcp/__tests__/utils.test.ts
index 4a8229c9d4..8b533241ad 100644
--- a/packages/api/src/mcp/__tests__/utils.test.ts
+++ b/packages/api/src/mcp/__tests__/utils.test.ts
@@ -23,6 +23,7 @@ import {
requiresOAuthMachinery,
isChatSelectableMCPServer,
filterChatSelectableMCPServers,
+ waitUntilDeadline,
} from '~/mcp/utils';
describe('normalizeServerName', () => {
@@ -1268,3 +1269,59 @@ describe('filterChatSelectableMCPServers', () => {
await expect(filterChatSelectableMCPServers(null, opts)).resolves.toEqual([]);
});
});
+
+describe('waitUntilDeadline', () => {
+ const deferred = () => {
+ let resolve!: (value: string) => void;
+ let reject!: (error: Error) => void;
+ const promise = new Promise<string>((resolveWork, rejectWork) => {
+ resolve = resolveWork;
+ reject = rejectWork;
+ });
+ return { promise, resolve, reject };
+ };
+
+ it('returns the value of work that settles before the deadline', async () => {
+ await expect(waitUntilDeadline(Promise.resolve('tokens'), Date.now() + 1000)).resolves.toEqual({
+ settled: true,
+ value: 'tokens',
+ });
+ });
+
+ it('rethrows a rejection that lands before the deadline', async () => {
+ await expect(
+ waitUntilDeadline(Promise.reject(new Error('storage unavailable')), Date.now() + 1000),
+ ).rejects.toThrow('storage unavailable');
+ });
+
+ it('stops waiting at the deadline and keeps observing the abandoned work', async () => {
+ const work = deferred();
+
+ await expect(waitUntilDeadline(work.promise, Date.now() + 10)).resolves.toEqual({
+ settled: false,
+ });
+
+ /** Jest fails the test if this late rejection goes unhandled. */
+ work.reject(new Error('late failure'));
+ await new Promise((resolve) => setImmediate(resolve));
+ });
+
+ it('stops waiting when the signal aborts, including one aborted before the wait', async () => {
+ const aborting = new AbortController();
+ const pending = deferred();
+ const waiting = waitUntilDeadline(pending.promise, undefined, aborting.signal);
+ aborting.abort();
+ await expect(waiting).resolves.toEqual({ settled: false });
+
+ const aborted = new AbortController();
+ aborted.abort();
+ const abandoned = deferred();
+ await expect(waitUntilDeadline(abandoned.promise, undefined, aborted.signal)).resolves.toEqual({
+ settled: false,
+ });
+
+ pending.reject(new Error('late failure'));
+ abandoned.reject(new Error('late failure'));
+ await new Promise((resolve) => setImmediate(resolve));
+ });
+});
diff --git a/packages/api/src/mcp/catalog/recovery.spec.ts b/packages/api/src/mcp/catalog/recovery.spec.ts
index d3393a8642..f5206ee836 100644
--- a/packages/api/src/mcp/catalog/recovery.spec.ts
+++ b/packages/api/src/mcp/catalog/recovery.spec.ts
@@ -2,8 +2,12 @@ import { logger } from '@librechat/data-schemas';
import { Constants } from 'librechat-data-provider';
import { ErrorCode, McpError, type Tool } from '@modelcontextprotocol/sdk/types.js';
import type { IUser } from '@librechat/data-schemas';
+import type {
+ MCPRecoveryGenerationScope,
+ MCPServerCatalogLoaderDeps,
+ MCPServerCatalogRecoveryPolicy,
+} from './recovery';
import type { LCAvailableTools, ParsedServerConfig, ToolDiscoveryOptions } from '../types';
-import type { MCPRecoveryGenerationScope, MCPServerCatalogLoaderDeps } from './recovery';
import {
MCPCatalogCapacityError,
MCPServerCatalogRecoveryTracker,
@@ -171,8 +175,10 @@ describe('MCPServerCatalogRecoveryTracker capacity', () => {
const policy = {
discoveryBackoffMs: [60_000],
discoveryTimeoutMs: 3_000,
+ discoverySettleGraceMs: 10_000,
reauthRetryMs: 60_000,
maxStateEntries: 2,
+ maxDetachedDiscoveries: 3,
generationReadTimeoutMs: 500,
authorizationFenceRetryMs: [0],
authorizationFenceTimeoutMs: 1_000,
@@ -228,8 +234,10 @@ describe('MCPServerCatalogRecoveryTracker own credential publications', () => {
const policy = {
discoveryBackoffMs: [60_000],
discoveryTimeoutMs: 3_000,
+ discoverySettleGraceMs: 10_000,
reauthRetryMs: 60_000,
maxStateEntries: 10,
+ maxDetachedDiscoveries: 3,
generationReadTimeoutMs: 500,
authorizationFenceRetryMs: [0],
authorizationFenceTimeoutMs: 1_000,
@@ -1085,6 +1093,7 @@ describe('loadMCPServerCatalogs — credential refresh during discovery', () =>
let generation = 'generation-1';
let readsFail = false;
let publicationHold: Promise<void> | undefined;
+ let retryIntentHold: Promise<void> | undefined;
let retryCleanup: Promise<void> = Promise.resolve();
const recoveryTracker = new MCPServerCatalogRecoveryTracker();
const rotate = () => {
@@ -1100,7 +1109,12 @@ describe('loadMCPServerCatalogs — credential refresh during discovery', () =>
const onOAuthCredentialsChanging = (changed: MCPRecoveryGenerationScope) =>
prepareMCPAuthorizationMutation(changed, {
invalidateRecoveryGeneration,
- persistPublicationRetry: async () => 'retry-v1',
+ persistPublicationRetry: async () => {
+ if (retryIntentHold != null) {
+ await retryIntentHold;
+ }
+ return 'retry-v1';
+ },
clearPublicationRetry: () => retryCleanup,
clearLocalRecovery: (userId, changedServerName, published) =>
recoveryTracker.clear(userId, changedServerName, published),
@@ -1114,6 +1128,17 @@ describe('loadMCPServerCatalogs — credential refresh during discovery', () =>
stallRetryCleanup: () => {
retryCleanup = new Promise<void>(() => undefined);
},
+ /** Holds the durable retry-intent write that precedes every token-row write. */
+ holdRetryIntent: () => {
+ let release: () => void = () => undefined;
+ retryIntentHold = new Promise<void>((resolve) => {
+ release = resolve;
+ });
+ return () => {
+ retryIntentHold = undefined;
+ release();
+ };
+ },
getRecoveryGeneration: async () => {
if (readsFail) {
throw new Error('tool cache unavailable');
@@ -1149,9 +1174,10 @@ describe('loadMCPServerCatalogs — credential refresh during discovery', () =>
const loadCatalogs = (
fence: ReturnType<typeof createFence>,
discoverServerTools: MCPServerCatalogLoaderDeps['discoverServerTools'],
+ recoveryPolicy?: Partial<MCPServerCatalogRecoveryPolicy>,
) =>
loadMCPServerCatalogs(
- { user, servers },
+ { user, servers, recoveryPolicy },
{
getCachedServerTools: jest.fn().mockResolvedValue(null),
getServerToolFunctionsSnapshot: jest.fn().mockResolvedValue({ tools: null }),
@@ -1264,6 +1290,36 @@ describe('loadMCPServerCatalogs — credential refresh during discovery', () =>
expect(discoverServerTools).toHaveBeenCalledTimes(1);
});
+ it('rediscovers once a refresh released with its stalled flight publishes the rotation', async () => {
+ const fence = createFence();
+ const releaseRetryIntent = fence.holdRetryIntent();
+ const recoveryPolicy = { discoveryTimeoutMs: 100, discoverySettleGraceMs: 20 };
+ let refresh: Promise<void> | undefined;
+ const discoverServerTools = jest.fn(async (options: ToolDiscoveryOptions) => {
+ if (refresh == null) {
+ refresh = refreshAccessToken(options);
+ await refresh;
+ }
+ return { tools: listedTools };
+ });
+
+ const released = await loadCatalogs(fence, discoverServerTools, recoveryPolicy);
+ const suppressed = await loadCatalogs(fence, discoverServerTools, recoveryPolicy);
+ expect(released.serversWithoutTools).toEqual([serverName]);
+ expect(suppressed.serversWithoutTools).toEqual([serverName]);
+ expect(discoverServerTools).toHaveBeenCalledTimes(1);
+ await expect(fence.getRecoveryGeneration()).resolves.toBe('generation-1');
+
+ releaseRetryIntent();
+ await refresh;
+ await flush();
+ await expect(fence.getRecoveryGeneration()).resolves.toBe('generation-2');
+
+ const recovered = await loadCatalogs(fence, discoverServerTools, recoveryPolicy);
+ expect(recovered.serverTools).toEqual(new Map([[serverName, recoveredTools]]));
+ expect(discoverServerTools).toHaveBeenCalledTimes(2);
+ });
+
it('discards a shared catalog that the shared generation cannot confirm for a request that saw a newer one', async () => {
const fence = createFence();
const releasePublication = fence.holdPublication();
@@ -1741,3 +1797,334 @@ describe('recoverMCPServerCatalogs — bounded, skippable discovery', () => {
);
});
});
+
+describe('recoverMCPServerCatalogs — discovery that outlives its budget', () => {
+ /** Stands in for discovery parked on an await that ignores its deadline, such as an OAuth refresh
+ * whose redeemed tokens are waiting on a stalled authorization-fence write. */
+ const stalledDiscovery = () => {
+ const pending: Array<{
+ resolve: (result: { tools: Tool[] | null }) => void;
+ reject: (error: Error) => void;
+ }> = [];
+ const discoverServerTools = jest.fn(
+ (_options: ToolDiscoveryOptions) =>
+ new Promise<{ tools: Tool[] | null }>((resolve, reject) => {
+ pending.push({ resolve, reject });
+ }),
+ );
+ return { discoverServerTools, pending };
+ };
+ const recoveryPolicy = { discoveryTimeoutMs: 200, discoverySettleGraceMs: 50 };
+ const stalledServers = [{ serverName: 'stalled', serverConfig: serverConfig('stalled') }];
+
+ it('releases a stalled shared flight, the requests that joined it, and their catalog slots', async () => {
+ const { discoverServerTools, pending } = stalledDiscovery();
+ const deps = {
+ loadUserMCPAuthMap: jest.fn().mockResolvedValue({}),
+ discoverServerTools,
+ formatServerTools: jest.fn().mockReturnValue(availableTools('late-tool')),
+ recoveryTracker,
+ };
+ const healthyServers = [{ serverName: 'healthy', serverConfig: serverConfig('healthy') }];
+ const healthyDeps = {
+ ...deps,
+ discoverServerTools: jest.fn().mockResolvedValue({ tools: [] }),
+ formatServerTools: jest.fn().mockReturnValue(availableTools('healthy-tool')),
+ };
+
+ try {
+ const joined = Array.from({ length: 3 }, () =>
+ recoverMCPServerCatalogs({ user, servers: stalledServers, recoveryPolicy }, deps),
+ );
+ await new Promise((resolve) => setImmediate(resolve));
+ expect(discoverServerTools).toHaveBeenCalledTimes(1);
+ await expect(
+ recoverMCPServerCatalogs({ user, servers: healthyServers, recoveryPolicy }, healthyDeps),
+ ).rejects.toBeInstanceOf(MCPCatalogCapacityError);
+
+ await expect(Promise.all(joined)).resolves.toEqual([new Map(), new Map(), new Map()]);
+ expect(logger.warn).toHaveBeenCalledWith(
+ expect.stringContaining('Discovery for stalled is still running 50ms past its budget'),
+ );
+ await expect(
+ recoverMCPServerCatalogs({ user, servers: healthyServers, recoveryPolicy }, healthyDeps),
+ ).resolves.toEqual(new Map([['healthy', availableTools('healthy-tool')]]));
+
+ pending[0].resolve({ tools: [] });
+ await new Promise((resolve) => setImmediate(resolve));
+ await expect(
+ recoverMCPServerCatalogs({ user, servers: stalledServers, recoveryPolicy }, deps),
+ ).resolves.toEqual(new Map());
+ expect(discoverServerTools).toHaveBeenCalledTimes(1);
+ } finally {
+ pending.forEach(({ resolve }) => resolve({ tools: null }));
+ }
+ });
+
+ it('keeps observing a released discovery that fails afterwards', async () => {
+ const { discoverServerTools, pending } = stalledDiscovery();
+
+ const result = await recoverMCPServerCatalogs(
+ { user, servers: stalledServers, recoveryPolicy },
+ {
+ loadUserMCPAuthMap: jest.fn().mockResolvedValue({}),
+ discoverServerTools,
+ formatServerTools: jest.fn(),
+ recoveryTracker,
+ },
+ );
+ expect(result).toEqual(new Map());
+
+ /** Jest fails the test if this late rejection goes unhandled. */
+ pending[0].reject(new Error('authorization fence write failed'));
+ await new Promise((resolve) => setTimeout(resolve, 10));
+ expect(discoverServerTools).toHaveBeenCalledTimes(1);
+ });
+
+ it('starts no second discovery for a server until its released discovery settles', async () => {
+ const { discoverServerTools, pending } = stalledDiscovery();
+ const deps = {
+ loadUserMCPAuthMap: jest.fn().mockResolvedValue({}),
+ discoverServerTools,
+ formatServerTools: jest.fn().mockReturnValue(availableTools('late-tool')),
+ recoveryTracker,
+ };
+ const servers = [{ serverName: 'held', serverConfig: serverConfig('held') }];
+ const policy = { ...recoveryPolicy, discoveryBackoffMs: [10] };
+
+ try {
+ await expect(
+ recoverMCPServerCatalogs({ user, servers, recoveryPolicy: policy }, deps),
+ ).resolves.toEqual(new Map());
+ await new Promise((resolve) => setTimeout(resolve, 20));
+ await expect(
+ recoverMCPServerCatalogs({ user, servers, recoveryPolicy: policy }, deps),
+ ).resolves.toEqual(new Map());
+ recoveryTracker.clear(user.id, 'held');
+ await expect(
+ recoverMCPServerCatalogs({ user, servers, recoveryPolicy: policy }, deps),
+ ).resolves.toEqual(new Map());
+ expect(discoverServerTools).toHaveBeenCalledTimes(1);
+
+ pending[0].resolve({ tools: null });
+ await new Promise((resolve) => setImmediate(resolve));
+ discoverServerTools.mockResolvedValueOnce({ tools: [] });
+ await expect(
+ recoverMCPServerCatalogs({ user, servers, recoveryPolicy: policy }, deps),
+ ).resolves.toEqual(new Map([['held', availableTools('late-tool')]]));
+ expect(discoverServerTools).toHaveBeenCalledTimes(2);
+ } finally {
+ pending.forEach(({ resolve }) => resolve({ tools: null }));
+ }
+ });
+
+ it('holds a request-bound server until its released discovery settles', async () => {
+ const { discoverServerTools, pending } = stalledDiscovery();
+ const deps = {
+ loadUserMCPAuthMap: jest.fn().mockResolvedValue({}),
+ discoverServerTools,
+ formatServerTools: jest.fn().mockReturnValue(availableTools('obo-tool')),
+ recoveryTracker,
+ };
+ const servers = [
+ {
+ serverName: 'obo',
+ serverConfig: {
+ ...serverConfig('obo'),
+ obo: { scopes: 'api://mcp/.default' },
+ } as ParsedServerConfig,
+ },
+ ];
+
+ try {
+ await recoverMCPServerCatalogs({ user, servers, recoveryPolicy }, deps);
+ await expect(
+ recoverMCPServerCatalogs({ user, servers, recoveryPolicy }, deps),
+ ).resolves.toEqual(new Map());
+ expect(discoverServerTools).toHaveBeenCalledTimes(1);
+
+ pending[0].resolve({ tools: null });
+ await new Promise((resolve) => setImmediate(resolve));
+ discoverServerTools.mockResolvedValueOnce({ tools: [] });
+ await expect(
+ recoverMCPServerCatalogs({ user, servers, recoveryPolicy }, deps),
+ ).resolves.toEqual(new Map([['obo', availableTools('obo-tool')]]));
+ expect(discoverServerTools).toHaveBeenCalledTimes(2);
+ } finally {
+ pending.forEach(({ resolve }) => resolve({ tools: null }));
+ }
+ });
+
+ it.each([
+ [
+ 'configuration',
+ {
+ serverConfig: { ...serverConfig('changed'), url: 'https://changed-v2.example.com/mcp' },
+ generation: 'generation-1',
+ },
+ ],
+ [
+ 'credential generation',
+ { serverConfig: serverConfig('changed'), generation: 'generation-2' },
+ ],
+ ])('lets a changed %s discover beside a released discovery', async (_change, next) => {
+ let generation = 'generation-1';
+ const { discoverServerTools, pending } = stalledDiscovery();
+ const deps = {
+ loadUserMCPAuthMap: jest.fn().mockResolvedValue({}),
+ discoverServerTools,
+ formatServerTools: jest.fn().mockReturnValue(availableTools('changed-tool')),
+ recoveryTracker,
+ getRecoveryGeneration: jest.fn(async () => generation),
+ };
+ const released = [{ serverName: 'changed', serverConfig: serverConfig('changed') }];
+
+ try {
+ await expect(
+ recoverMCPServerCatalogs({ user, servers: released, recoveryPolicy }, deps),
+ ).resolves.toEqual(new Map());
+ generation = next.generation;
+ discoverServerTools.mockResolvedValueOnce({ tools: [] });
+ await expect(
+ recoverMCPServerCatalogs(
+ {
+ user,
+ servers: [
+ { serverName: 'changed', serverConfig: next.serverConfig as ParsedServerConfig },
+ ],
+ recoveryPolicy,
+ },
+ deps,
+ ),
+ ).resolves.toEqual(new Map([['changed', availableTools('changed-tool')]]));
+ expect(discoverServerTools).toHaveBeenCalledTimes(2);
+ } finally {
+ pending.forEach(({ resolve }) => resolve({ tools: null }));
+ }
+ });
+
+ it('starts no new discovery while released discoveries fill the process-wide limit', async () => {
+ const tracker = new MCPServerCatalogRecoveryTracker(undefined, 1);
+ const { discoverServerTools, pending } = stalledDiscovery();
+ const deps = {
+ loadUserMCPAuthMap: jest.fn().mockResolvedValue({}),
+ discoverServerTools,
+ formatServerTools: jest.fn().mockReturnValue(availableTools('fresh-tool')),
+ recoveryTracker: tracker,
+ };
+ const named = (serverName: string, overrides: Partial<ParsedServerConfig> = {}) => [
+ {
+ serverName,
+ serverConfig: { ...serverConfig(serverName), ...overrides } as ParsedServerConfig,
+ },
+ ];
+
+ try {
+ await expect(
+ recoverMCPServerCatalogs({ user, servers: named('stuck'), recoveryPolicy }, deps),
+ ).resolves.toEqual(new Map());
+ expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('reached the limit of 1'));
+ await expect(
+ recoverMCPServerCatalogs({ user, servers: named('other'), recoveryPolicy }, deps),
+ ).resolves.toEqual(new Map());
+ await expect(
+ recoverMCPServerCatalogs(
+ {
+ user,
+ servers: named('request-bound', { obo: { scopes: 'api://mcp/.default' } }),
+ recoveryPolicy,
+ },
+ deps,
+ ),
+ ).resolves.toEqual(new Map());
+ expect(discoverServerTools).toHaveBeenCalledTimes(1);
+
+ pending[0].resolve({ tools: null });
+ await new Promise((resolve) => setImmediate(resolve));
+ discoverServerTools.mockResolvedValueOnce({ tools: [] });
+ await expect(
+ recoverMCPServerCatalogs({ user, servers: named('other'), recoveryPolicy }, deps),
+ ).resolves.toEqual(new Map([['other', availableTools('fresh-tool')]]));
+ expect(discoverServerTools).toHaveBeenCalledTimes(2);
+ } finally {
+ pending.forEach(({ resolve }) => resolve({ tools: null }));
+ }
+ });
+
+ it('counts token work a discovery hands off at its deadline toward the hold and the limit', async () => {
+ const tracker = new MCPServerCatalogRecoveryTracker(undefined, 1);
+ let settleTokenWork: () => void = () => undefined;
+ const tokenWork = new Promise<void>((resolve) => {
+ settleTokenWork = resolve;
+ });
+ const discoverServerTools = jest.fn(async (options: ToolDiscoveryOptions) => {
+ if (options.serverName !== 'refreshing') {
+ return { tools: [] };
+ }
+ options.onDiscoveryDetached?.(tokenWork);
+ return { tools: null };
+ });
+ const deps = {
+ loadUserMCPAuthMap: jest.fn().mockResolvedValue({}),
+ discoverServerTools,
+ formatServerTools: jest.fn().mockReturnValue(availableTools('fresh-tool')),
+ recoveryTracker: tracker,
+ };
+ const policy = { ...recoveryPolicy, discoveryBackoffMs: [10] };
+ const named = (serverName: string) => [{ serverName, serverConfig: serverConfig(serverName) }];
+
+ try {
+ await expect(
+ recoverMCPServerCatalogs(
+ { user, servers: named('refreshing'), recoveryPolicy: policy },
+ deps,
+ ),
+ ).resolves.toEqual(new Map());
+ await new Promise((resolve) => setTimeout(resolve, 20));
+ await expect(
+ recoverMCPServerCatalogs(
+ { user, servers: named('refreshing'), recoveryPolicy: policy },
+ deps,
+ ),
+ ).resolves.toEqual(new Map());
+ await expect(
+ recoverMCPServerCatalogs({ user, servers: named('other'), recoveryPolicy: policy }, deps),
+ ).resolves.toEqual(new Map());
+ expect(discoverServerTools).toHaveBeenCalledTimes(1);
+
+ settleTokenWork();
+ await new Promise((resolve) => setImmediate(resolve));
+ await expect(
+ recoverMCPServerCatalogs({ user, servers: named('other'), recoveryPolicy: policy }, deps),
+ ).resolves.toEqual(new Map([['other', availableTools('fresh-tool')]]));
+ expect(discoverServerTools).toHaveBeenCalledTimes(2);
+ } finally {
+ settleTokenWork();
+ }
+ });
+
+ it('lets a discovery that settles just past its budget keep its slot and its catalog', async () => {
+ const discoverServerTools = jest.fn(async ({ deadlineMs }: ToolDiscoveryOptions) => {
+ const remainingMs = Math.max(0, (deadlineMs ?? Date.now()) - Date.now());
+ await new Promise((resolve) => setTimeout(resolve, remainingMs + 30));
+ return { tools: [] };
+ });
+
+ const result = await recoverMCPServerCatalogs(
+ {
+ user,
+ servers: [{ serverName: 'closing', serverConfig: serverConfig('closing') }],
+ recoveryPolicy: { discoveryTimeoutMs: 20 },
+ },
+ {
+ loadUserMCPAuthMap: jest.fn().mockResolvedValue({}),
+ discoverServerTools,
+ formatServerTools: jest.fn().mockReturnValue(availableTools('closing-tool')),
+ recoveryTracker,
+ },
+ );
+
+ expect(result).toEqual(new Map([['closing', availableTools('closing-tool')]]));
+ expect(logger.warn).not.toHaveBeenCalledWith(expect.stringContaining('Discovery for closing'));
+ });
+});
diff --git a/packages/api/src/mcp/catalog/recovery.ts b/packages/api/src/mcp/catalog/recovery.ts
index 59ffe7adc8..a1668bceb5 100644
--- a/packages/api/src/mcp/catalog/recovery.ts
+++ b/packages/api/src/mcp/catalog/recovery.ts
@@ -3,7 +3,7 @@ import { ErrorCode, McpError } from '@modelcontextprotocol/sdk/types.js';
import type { Tool } from '@modelcontextprotocol/sdk/types.js';
import type { IUser } from '@librechat/data-schemas';
import type { LCAvailableTools, ParsedServerConfig, ToolDiscoveryOptions } from '../types';
-import { hasCustomUserVars, getMissingCustomUserVars } from '../utils';
+import { hasCustomUserVars, waitUntilDeadline, getMissingCustomUserVars } from '../utils';
import { usesDirectOpenIDBearerRecovery } from '../openid';
import { getServerCustomUserVars } from '../auth';
import { mcpConfig } from '../mcpConfig';
@@ -15,15 +15,20 @@ const pendingCatalogLanes: CatalogWorkLane[] = [];
let activeCatalogWork = 0;
/**
* Bounds one server's discovery end to end — connect, `tools/list` pagination, and the
- * unauthenticated fallback all draw down this single budget, so a slot is held for at most this
- * long regardless of where the server stalls. Recovery targets a server that is reachable and
- * authorized but whose catalog cache expired, and such a server answers well inside this window.
+ * unauthenticated fallback all draw down this single budget. Recovery targets a server that is
+ * reachable and authorized but whose catalog cache expired, and such a server answers well inside
+ * this window. The settle grace lets a discovery that honored the budget finish closing its
+ * connections; past it, recovery releases the slot even if some dependency is still stalled. By
+ * default no more released discoveries may still run than there are catalog slots, the most work a
+ * stalled dependency could hold before discoveries were released.
*/
const DEFAULT_RECOVERY_POLICY: MCPServerCatalogRecoveryPolicy = {
discoveryBackoffMs: [5 * 60_000, 10 * 60_000, 20 * 60_000, 30 * 60_000],
discoveryTimeoutMs: 3_000,
+ discoverySettleGraceMs: 10_000,
reauthRetryMs: 30 * 60_000,
maxStateEntries: 10_000,
+ maxDetachedDiscoveries: CATALOG_FANOUT_CONCURRENCY,
generationReadTimeoutMs: 500,
authorizationFenceRetryMs: [0, 50, 200],
authorizationFenceTimeoutMs: 1_000,
@@ -149,8 +154,12 @@ export async function publishMCPAuthorizationMutation(
export interface MCPServerCatalogRecoveryPolicy {
discoveryBackoffMs: readonly number[];
discoveryTimeoutMs: number;
+ /** How long past its budget a discovery may keep its catalog slot and coalesced requests. */
+ discoverySettleGraceMs: number;
reauthRetryMs: number;
maxStateEntries: number;
+ /** How many released discoveries may still run before no new discovery starts. */
+ maxDetachedDiscoveries: number;
generationReadTimeoutMs: number;
authorizationFenceRetryMs: readonly number[];
authorizationFenceTimeoutMs: number;
@@ -239,6 +248,15 @@ type PublicationTracker = (
publish: () => Promise<string | undefined>,
) => Promise<string | undefined>;
+/** Hands the tracker a discovery released past its settle grace that is still running. */
+type DiscoveryDetacher = (discovery: Promise<unknown>) => void;
+
+/** The observed state a released discovery ran under; it holds only an attempt that matches it. */
+interface DetachedDiscovery {
+ configFingerprint: string;
+ recoveryGeneration?: string;
+}
+
interface CatalogWorkLane {
tasks: ReadonlyArray<() => Promise<void>>;
resolve: (admitted: boolean) => void;
@@ -337,8 +355,96 @@ function getRecoveryFingerprint(
export class MCPServerCatalogRecoveryTracker {
private readonly states = new Map<string, RecoveryStateEntry>();
+ /** Discoveries released past their settle grace that are still running, by recovery key. */
+ private readonly detachedDiscoveries = new Map<string, DetachedDiscovery[]>();
+ private detachedDiscoveryCount = 0;
+
+ constructor(
+ private readonly maxStateEntries: number = DEFAULT_RECOVERY_POLICY.maxStateEntries,
+ private readonly maxDetachedDiscoveries: number = DEFAULT_RECOVERY_POLICY.maxDetachedDiscoveries,
+ ) {}
- constructor(private readonly maxStateEntries: number = DEFAULT_RECOVERY_POLICY.maxStateEntries) {}
+ /** Whether a request-bound discovery, which never joins a flight, may start. */
+ public admitsDiscovery(
+ user: IUser,
+ candidate: RecoveryCandidate,
+ policy: MCPServerCatalogRecoveryPolicy,
+ ): boolean {
+ return !this.isDiscoveryHeld(
+ getRecoveryKey(user.id, candidate.serverName),
+ getRecoveryFingerprint(candidate.serverConfig, policy),
+ undefined,
+ );
+ }
+
+ /** Counts a request-bound discovery released past its settle grace until it settles. */
+ public detachDiscovery(
+ user: IUser,
+ candidate: RecoveryCandidate,
+ policy: MCPServerCatalogRecoveryPolicy,
+ discovery: Promise<unknown>,
+ ): void {
+ this.detach(
+ getRecoveryKey(user.id, candidate.serverName),
+ { configFingerprint: getRecoveryFingerprint(candidate.serverConfig, policy) },
+ discovery,
+ );
+ }
+
+ /**
+ * A released discovery keeps running, so another attempt under the same observed state would stack
+ * on the same stuck dependency. Released discoveries also share one process-wide limit: once that
+ * many still run, no new discovery starts until one settles. A changed configuration or credential
+ * generation is new state and may discover within that limit. Only settling lifts a hold; clears
+ * do not. Discoveries already running when the limit fills can still be released, so the count
+ * can exceed it by at most the catalog concurrency.
+ */
+ private isDiscoveryHeld(
+ key: string,
+ configFingerprint: string,
+ recoveryGeneration: string | undefined,
+ ): boolean {
+ if (this.detachedDiscoveryCount >= this.maxDetachedDiscoveries) {
+ return true;
+ }
+ return (
+ this.detachedDiscoveries
+ .get(key)
+ ?.some(
+ (detached) =>
+ detached.configFingerprint === configFingerprint &&
+ (recoveryGeneration == null || detached.recoveryGeneration === recoveryGeneration),
+ ) === true
+ );
+ }
+
+ private detach(key: string, detached: DetachedDiscovery, discovery: Promise<unknown>): void {
+ const held = this.detachedDiscoveries.get(key);
+ if (held == null) {
+ this.detachedDiscoveries.set(key, [detached]);
+ } else {
+ held.push(detached);
+ }
+ this.detachedDiscoveryCount += 1;
+ if (this.detachedDiscoveryCount === this.maxDetachedDiscoveries) {
+ logger.warn(
+ `[MCP catalog recovery] Released discoveries still running reached the limit of ${this.maxDetachedDiscoveries}; starting no new discovery until one settles`,
+ );
+ }
+ const release = () => {
+ const current = this.detachedDiscoveries.get(key);
+ const index = current?.indexOf(detached) ?? -1;
+ if (current == null || index < 0) {
+ return;
+ }
+ current.splice(index, 1);
+ if (current.length === 0) {
+ this.detachedDiscoveries.delete(key);
+ }
+ this.detachedDiscoveryCount -= 1;
+ };
+ discovery.then(release, release);
+ }
private touch(key: string, entry: RecoveryStateEntry): void {
if (this.states.get(key) !== entry) {
@@ -393,12 +499,16 @@ export class MCPServerCatalogRecoveryTracker {
candidate: RecoveryCandidate,
policy: MCPServerCatalogRecoveryPolicy,
recoveryGeneration: string | undefined,
- discover: (trackPublication: PublicationTracker) => Promise<RecoveryOutcome>,
+ discover: (
+ trackPublication: PublicationTracker,
+ detach: DiscoveryDetacher,
+ ) => Promise<RecoveryOutcome>,
): Promise<RecoveryOutcome> {
const key = getRecoveryKey(user.id, candidate.serverName);
const configFingerprint = getRecoveryFingerprint(candidate.serverConfig, policy);
const now = Date.now();
const existing = this.states.get(key);
+ const held = this.isDiscoveryHeld(key, configFingerprint, recoveryGeneration);
const sameObservedState =
existing?.configFingerprint === configFingerprint &&
(recoveryGeneration == null ||
@@ -410,10 +520,13 @@ export class MCPServerCatalogRecoveryTracker {
if (existing.inFlight != null) {
return existing.inFlight;
}
- if (existing.outcome != null && existing.nextRetryAt > now) {
+ if (existing.outcome != null && (held || existing.nextRetryAt > now)) {
return Promise.resolve(existing.outcome);
}
}
+ if (held) {
+ return Promise.resolve({ serverName: candidate.serverName, tools: null });
+ }
const entry: RecoveryStateEntry = {
configFingerprint,
@@ -449,8 +562,14 @@ export class MCPServerCatalogRecoveryTracker {
entry.publishing = false;
}
};
+ const detach: DiscoveryDetacher = (discovery) =>
+ this.detach(
+ key,
+ { configFingerprint, recoveryGeneration: entry.recoveryGeneration },
+ discovery,
+ );
const flight = Promise.resolve()
- .then(() => discover(trackPublication))
+ .then(() => discover(trackPublication, detach))
.finally(() => {
settled = true;
})
@@ -543,21 +662,41 @@ async function discoverCandidate(
{
signal,
trackPublication,
- }: { signal?: AbortSignal; trackPublication?: PublicationTracker } = {},
+ onDetached,
+ }: {
+ signal?: AbortSignal;
+ trackPublication?: PublicationTracker;
+ onDetached?: DiscoveryDetacher;
+ } = {},
): Promise<RecoveryOutcome> {
+ const deadlineMs = Date.now() + resolveBudget(serverConfig, policy);
try {
- const result = await deps.discoverServerTools({
+ const pending = deps.discoverServerTools({
user,
serverName,
configServers: { [serverName]: serverConfig },
customUserVars,
- deadlineMs: Date.now() + resolveBudget(serverConfig, policy),
+ deadlineMs,
signal,
onOAuthCredentialsChanging: trackPublications(
deps.onOAuthCredentialsChanging,
trackPublication,
),
+ onDiscoveryDetached: onDetached,
});
+ /** Discovery can await work that ignores its budget — a token refresh persisting behind a
+ * stalled write — and a shared flight has no request signal to end that wait, so it would hold
+ * every coalesced request and its slot indefinitely. Stop waiting instead of cancelling: that
+ * work still commits what it redeemed. */
+ const discovery = await waitUntilDeadline(pending, deadlineMs + policy.discoverySettleGraceMs);
+ if (!discovery.settled) {
+ onDetached?.(pending);
+ logger.warn(
+ `[MCP catalog recovery] Discovery for ${serverName} is still running ${policy.discoverySettleGraceMs}ms past its budget; releasing its catalog slot while it finishes in the background`,
+ );
+ return { serverName, tools: null, ...(!signal?.aborted && { state: 'backoff' as const }) };
+ }
+ const result = discovery.value;
const tools = result.tools == null ? null : deps.formatServerTools(serverName, result.tools);
if (signal?.aborted) {
return { serverName, tools: null };
@@ -766,7 +905,13 @@ async function recoverMCPServerCatalogsWithState(
candidate.serverConfig.obo != null ||
usesDirectOpenIDBearerRecovery(candidate.serverConfig);
if (usesRequestCredential) {
- results[index] = await discoverCandidate(user, candidate, deps, policy, { signal });
+ results[index] = tracker.admitsDiscovery(user, candidate, policy)
+ ? await discoverCandidate(user, candidate, deps, policy, {
+ signal,
+ onDetached: (discovery) =>
+ tracker.detachDiscovery(user, candidate, policy, discovery),
+ })
+ : { serverName: candidate.serverName, tools: null };
return;
}
const observedGeneration = await readMCPRecoveryGeneration(
@@ -792,8 +937,11 @@ async function recoverMCPServerCatalogsWithState(
candidate,
policy,
recoveryGeneration,
- (trackPublication) =>
- discoverCandidate(user, candidate, deps, policy, { trackPublication }),
+ (trackPublication, detach) =>
+ discoverCandidate(user, candidate, deps, policy, {
+ trackPublication,
+ onDetached: detach,
+ }),
);
const finalGeneration = await readMCPRecoveryGeneration(
{ userId: user.id, serverName: candidate.serverName },
diff --git a/packages/api/src/mcp/types/index.ts b/packages/api/src/mcp/types/index.ts
index 8fac212b02..576f55f57d 100644
--- a/packages/api/src/mcp/types/index.ts
+++ b/packages/api/src/mcp/types/index.ts
@@ -272,6 +272,11 @@ export interface UserConnectionContext {
userId: string;
serverName: string;
}) => Promise<() => Promise<string | undefined>>;
+ /**
+ * Receives discovery work the caller stopped waiting for at its deadline or abort, such as an
+ * OAuth token flow still persisting a refresh. The work keeps running; the promise settles with it.
+ */
+ onDiscoveryDetached?: (work: Promise<unknown>) => void;
}
export interface RequestScopedMCPConnectionStore {
@@ -337,6 +342,7 @@ export interface ToolDiscoveryOptions {
deadlineMs?: number;
onOAuthCredentialsChanged?: (scope: { userId: string; serverName: string }) => Promise<void>;
onOAuthCredentialsChanging?: UserConnectionContext['onOAuthCredentialsChanging'];
+ onDiscoveryDetached?: UserConnectionContext['onDiscoveryDetached'];
/** Pre-resolved config-source servers for tenant-scoped lookup */
configServers?: Record<string, ParsedServerConfig>;
oboTokenResolver?: OboTokenResolver;
diff --git a/packages/api/src/mcp/utils.ts b/packages/api/src/mcp/utils.ts
index aa8b3af0c4..12edf4aaac 100644
--- a/packages/api/src/mcp/utils.ts
+++ b/packages/api/src/mcp/utils.ts
@@ -761,6 +761,49 @@ export function createDeadlineAbortSignal(
return budget ?? callerSignal;
}
+export type DeadlineWaitResult<T> = { settled: true; value: T } | { settled: false };
+
+/**
+ * Waits for `promise` only until `deadlineMs` passes or `signal` aborts, and never cancels it:
+ * shared work such as an OAuth token refresh keeps running for whoever depends on its outcome. A
+ * rejection that lands first is rethrown; one that lands after the wait ended stays observed, so
+ * abandoned work never surfaces as an unhandled rejection.
+ */
+export async function waitUntilDeadline<T>(
+ promise: Promise<T>,
+ deadlineMs?: number,
+ signal?: AbortSignal,
+): Promise<DeadlineWaitResult<T>> {
+ if (deadlineMs == null && signal == null) {
+ return { settled: true, value: await promise };
+ }
+ const settlement = promise.then((value): DeadlineWaitResult<T> => ({ settled: true, value }));
+ if (signal?.aborted === true) {
+ settlement.catch(() => undefined);
+ return { settled: false };
+ }
+ let timer: NodeJS.Timeout | undefined;
+ let onAbort: (() => void) | undefined;
+ const interrupted = new Promise<DeadlineWaitResult<T>>((resolve) => {
+ if (deadlineMs != null) {
+ timer = setTimeout(() => resolve({ settled: false }), Math.max(0, deadlineMs - Date.now()));
+ timer.unref?.();
+ }
+ if (signal != null) {
+ onAbort = () => resolve({ settled: false });
+ signal.addEventListener('abort', onAbort, { once: true });
+ }
+ });
+ try {
+ return await Promise.race([settlement, interrupted]);
+ } finally {
+ clearTimeout(timer);
+ if (onAbort != null) {
+ signal?.removeEventListener('abort', onAbort);
+ }
+ }
+}
+
export function generateServerNameFromTitle(title: string): string {
const slug = title
.toLowerCase()
diff --git a/packages/data-provider/src/config.spec.ts b/packages/data-provider/src/config.spec.ts
index 71cf1e0d7b..dfa3f34196 100644
--- a/packages/data-provider/src/config.spec.ts
+++ b/packages/data-provider/src/config.spec.ts
@@ -988,8 +988,10 @@ describe('allowedAddressesSchema', () => {
expect(defaults.mcpSettings?.catalogRecovery).toEqual({
discoveryBackoffMs: [300_000, 600_000, 1_200_000, 1_800_000],
discoveryTimeoutMs: 3_000,
+ discoverySettleGraceMs: 10_000,
reauthRetryMs: 1_800_000,
maxStateEntries: 10_000,
+ maxDetachedDiscoveries: 3,
generationReadTimeoutMs: 500,
authorizationFenceRetryMs: [0, 50, 200],
authorizationFenceTimeoutMs: 1_000,
@@ -1009,6 +1011,24 @@ describe('allowedAddressesSchema', () => {
mcpSettings: { catalogRecovery: { discoveryTimeoutMs: 0 } },
}).success,
).toBe(false);
+ expect(
+ configSchema.safeParse({
+ version: '1.0',
+ mcpSettings: { catalogRecovery: { discoverySettleGraceMs: -1 } },
+ }).success,
+ ).toBe(false);
+ expect(
+ configSchema.safeParse({
+ version: '1.0',
+ mcpSettings: { catalogRecovery: { maxDetachedDiscoveries: 0 } },
+ }).success,
+ ).toBe(false);
+ expect(
+ configSchema.parse({
+ version: '1.0',
+ mcpSettings: { catalogRecovery: { discoverySettleGraceMs: 0 } },
+ }).mcpSettings?.catalogRecovery.discoverySettleGraceMs,
+ ).toBe(0);
expect(
configSchema.safeParse({
version: '1.0',
diff --git a/packages/data-provider/src/config.ts b/packages/data-provider/src/config.ts
index 1799e71653..965fb2ee4e 100644
--- a/packages/data-provider/src/config.ts
+++ b/packages/data-provider/src/config.ts
@@ -2730,6 +2730,15 @@ export const configSchema = z.object({
.positive()
.max(5 * 60_000)
.default(3_000),
+ /** How long past `discoveryTimeoutMs` a stalled discovery may hold its catalog slot and
+ * coalesced requests. It is never cancelled, so OAuth tokens it redeemed still persist,
+ * and no other discovery for the same server state starts until it settles. */
+ discoverySettleGraceMs: z
+ .number()
+ .int()
+ .nonnegative()
+ .max(5 * 60_000)
+ .default(10_000),
reauthRetryMs: z
.number()
.int()
@@ -2737,6 +2746,10 @@ export const configSchema = z.object({
.max(24 * 60 * 60_000)
.default(30 * 60_000),
maxStateEntries: z.number().int().positive().max(1_000_000).default(10_000),
+ /** Process-wide: how many discoveries released past `discoverySettleGraceMs` may still be
+ * running before recovery starts no new discovery until one settles. The default matches
+ * the three catalog slots a stalled dependency could hold before discoveries were released. */
+ maxDetachedDiscoveries: z.number().int().positive().max(1_000).default(3),
generationReadTimeoutMs: z.number().int().positive().max(10_000).default(500),
authorizationFenceRetryMs: z
.array(z.number().int().nonnegative().max(60_000))