Skip to content

Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active

🤺 fix: Prefix MCP Tool Cache Write Fences (#17458)

lia-by-librechat[bot]8e68cab31b77
diff --git a/api/server/services/Config/__tests__/getCachedTools.lock.spec.js b/api/server/services/Config/__tests__/getCachedTools.lock.spec.js
index 50e99ecfd0..3ee2aa59b4 100644
--- a/api/server/services/Config/__tests__/getCachedTools.lock.spec.js
+++ b/api/server/services/Config/__tests__/getCachedTools.lock.spec.js
@@ -34,6 +34,7 @@ const getLogStores = require('~/cache/getLogStores');
const mockCache = { get: jest.fn(), set: jest.fn(), delete: jest.fn() };
getLogStores.mockReturnValue(mockCache);
+const { createMCPCatalogStore } = require('@librechat/api');
const {
getCachedTools,
updateCachedGlobalTools,
@@ -62,6 +63,100 @@ describe('global tool cache write lock', () => {
mockCache.delete.mockResolvedValue(true);
});
+ describe.each([
+ { REDIS_KEY_PREFIX: undefined },
+ { REDIS_KEY_PREFIX: '' },
+ { REDIS_KEY_PREFIX: 'myprefix_' },
+ { REDIS_KEY_PREFIX: 'myprefix_', GLOBAL_PREFIX_SEPARATOR: ':' },
+ { REDIS_KEY_PREFIX: 'myprefix_{tenant}', GLOBAL_PREFIX_SEPARATOR: '' },
+ { REDIS_KEY_PREFIX: 'myprefix_{' },
+ { REDIS_KEY_PREFIX: 'myprefix_}' },
+ { REDIS_KEY_PREFIX: 'myprefix_{{tenant}' },
+ { REDIS_KEY_PREFIX: 'myprefix_', GLOBAL_PREFIX_SEPARATOR: '{' },
+ ])('fence key namespace %j', (cacheConfig) => {
+ it.each([
+ ['global', false],
+ ['user', false],
+ ['global', true],
+ ['user', true],
+ ])(
+ 'namespaces the %s fence without changing its slot (restricted ACL: %s)',
+ async (scope, restricted) => {
+ mockCache.get.mockResolvedValue(null);
+ const store = createMCPCatalogStore({
+ cacheConfig,
+ getCache: () => mockCache,
+ ioredisClient: mockRedisClient,
+ keyvRedisClient: mockKeyvRedisClient,
+ });
+ const prefix = cacheConfig.REDIS_KEY_PREFIX
+ ? `${cacheConfig.REDIS_KEY_PREFIX}${cacheConfig.GLOBAL_PREFIX_SEPARATOR ?? '::'}`
+ : '';
+ const namespace = `${prefix}${CacheKeys.TOOL_CACHE}:`;
+ const guardedKey =
+ scope === 'global'
+ ? `${namespace}tools:global`
+ : `${namespace}tools:metadata:mcp:user-generation:{user-1:server-1}`;
+
+ const opening = guardedKey.indexOf('{');
+ const closing = opening < 0 ? -1 : guardedKey.indexOf('}', opening + 1);
+ const hashTag = closing > opening + 1 ? guardedKey.slice(opening + 1, closing) : guardedKey;
+ const legacyFenceKey = `tools:${scope === 'global' ? 'global' : 'mcp'}:write-fence:{${hashTag}}`;
+ mockKeyvRedisClient.eval.mockImplementation(async (_script, { keys }) => {
+ if (keys.some((key) => calculateSlot(key) !== calculateSlot(keys[0]))) {
+ throw new Error('Redis catalog script keys must share one cluster slot');
+ }
+ if (restricted && keys.includes(legacyFenceKey)) {
+ throw new Error('NOPERM no permissions to access one of the keys used as arguments');
+ }
+ return 1;
+ });
+
+ if (scope === 'global') {
+ await expect(store.setCachedTools({ builtin: {} })).resolves.toBe(true);
+ } else {
+ await expect(
+ store.getMCPToolsCacheGeneration({ userId: 'user-1', serverName: 'server-1' }),
+ ).resolves.toEqual(expect.any(String));
+ }
+
+ const fenceKey = mockKeyvRedisClient.eval.mock.calls[0][1].keys[0];
+ expect(fenceKey.startsWith(namespace)).toBe(true);
+ expect(fenceKey).toContain('write-fence');
+ expect(fenceKey).not.toBe(guardedKey);
+ const fallback = restricted || calculateSlot(legacyFenceKey) !== calculateSlot(guardedKey);
+ const claimKeys = fallback ? [fenceKey] : [fenceKey, legacyFenceKey];
+ const calls = [[fenceKey, legacyFenceKey]];
+ if (fallback) calls.push(claimKeys);
+ calls.push(
+ fallback ? [fenceKey, guardedKey] : [fenceKey, guardedKey, legacyFenceKey],
+ claimKeys,
+ );
+ expect(mockKeyvRedisClient.eval.mock.calls.map(([, options]) => options.keys)).toEqual(
+ calls,
+ );
+ expect(calculateSlot(fenceKey)).toBe(calculateSlot(guardedKey));
+ },
+ );
+ });
+
+ it.each([
+ 'READONLY replica is read-only',
+ 'ERR Error running script: NOPERM inside script',
+ 'connection lost',
+ 'CROSSSLOT Keys in request do not hash to the same slot',
+ 'Redis catalog script keys must share one cluster slot',
+ ])('does not retry an ambiguous fence claim failure: %s', async (message) => {
+ mockKeyvRedisClient.eval.mockRejectedValue(new Error(message));
+ const operation = jest.fn();
+
+ await expect(runWithGlobalCacheLock(operation)).rejects.toThrow(message);
+
+ expect(operation).not.toHaveBeenCalled();
+ expect(mockKeyvRedisClient.eval).toHaveBeenCalledTimes(1);
+ expect(mockRedisClient.eval).toHaveBeenCalledTimes(1);
+ });
+
it('waits for the shared Redis client before accessing the catalog', async () => {
mockCache.get.mockResolvedValue(null);
let resolveRedisReady;
@@ -147,8 +242,9 @@ describe('global tool cache write lock', () => {
expect.stringContaining("redis.call('GET', KEYS[1])"),
expect.objectContaining({
keys: [
- `tools:global:write-fence:{${CacheKeys.TOOL_CACHE}:tools:global}`,
+ `${CacheKeys.TOOL_CACHE}:tools:global:write-fence:{${CacheKeys.TOOL_CACHE}:tools:global}`,
`${CacheKeys.TOOL_CACHE}:tools:global`,
+ `tools:global:write-fence:{${CacheKeys.TOOL_CACHE}:tools:global}`,
],
}),
);
@@ -168,8 +264,9 @@ describe('global tool cache write lock', () => {
expect.stringContaining("redis.call('PSETEX', KEYS[2]"),
expect.objectContaining({
keys: [
- `tools:global:write-fence:{${CacheKeys.TOOL_CACHE}:tools:global}`,
+ `${CacheKeys.TOOL_CACHE}:tools:global:write-fence:{${CacheKeys.TOOL_CACHE}:tools:global}`,
`${CacheKeys.TOOL_CACHE}:tools:global`,
+ `tools:global:write-fence:{${CacheKeys.TOOL_CACHE}:tools:global}`,
],
arguments: [
expect.any(String),
@@ -214,6 +311,7 @@ describe('global tool cache write lock', () => {
expect(operation).not.toHaveBeenCalled();
const [claimScript, claimOptions] = mockKeyvRedisClient.eval.mock.calls[0];
expect(claimScript).toContain('current ~= ARGV[1]');
+ expect(claimOptions.keys).toHaveLength(2);
expect(claimScript).toContain("redis.call('TIME')");
expect(claimOptions.arguments).toEqual([
mockRedisClient.set.mock.calls[0][1],
@@ -339,10 +437,11 @@ describe('global tool cache write lock', () => {
expect.stringContaining("redis.call('EXISTS', KEYS[2])"),
expect.objectContaining({
keys: [
- `tools:mcp:write-fence:{user-1:server-1}`,
+ `${CacheKeys.TOOL_CACHE}:tools:mcp:write-fence:{user-1:server-1}`,
`${CacheKeys.TOOL_CACHE}:tools:metadata:mcp:user-legacy-fence:{user-1:server-1}`,
`${CacheKeys.TOOL_CACHE}:tools:metadata:mcp:user-generation:{user-1:server-1}`,
`${CacheKeys.TOOL_CACHE}:tools:mcp:user:{user-1:server-1}:v2:config-current`,
+ `tools:mcp:write-fence:{user-1:server-1}`,
],
}),
);
diff --git a/packages/api/src/mcp/catalog/store.cache_integration.spec.ts b/packages/api/src/mcp/catalog/store.cache_integration.spec.ts
new file mode 100644
index 0000000000..c7b9475259
--- /dev/null
+++ b/packages/api/src/mcp/catalog/store.cache_integration.spec.ts
@@ -0,0 +1,442 @@
+import { randomUUID } from 'node:crypto';
+import { CacheKeys } from 'librechat-data-provider';
+import type { RedisTestClient } from '../../stream/__tests__/helpers/redis';
+import type { CatalogCache, CatalogStoreDeps } from './store';
+import { createRedisTestClient } from '../../stream/__tests__/helpers/redis';
+import { createMCPCatalogStore, ToolCacheKeys } from './store';
+
+const describeRedis = process.env.USE_REDIS === 'true' ? describe : describe.skip;
+const tools = (name: string) => ({ [name]: { type: 'function' as const, function: { name } } });
+const RELEASE_SCRIPT = `
+if redis.call('GET', KEYS[1]) == ARGV[1] then
+ return redis.call('DEL', KEYS[1])
+end
+return 0
+`;
+const CLAIM_SCRIPT = `
+local current = redis.call('GET', KEYS[1])
+if current and current ~= ARGV[1] then return 0 end
+local redisTime = redis.call('TIME')
+local now = (tonumber(redisTime[1]) * 1000) + math.floor(tonumber(redisTime[2]) / 1000)
+local ttl = tonumber(ARGV[2]) - now - tonumber(ARGV[3])
+if ttl <= 0 then return 0 end
+redis.call('PSETEX', KEYS[1], ttl, ARGV[1])
+return 1
+`;
+
+function barrier() {
+ let release!: () => void;
+ const ready = new Promise<void>((resolve) => {
+ release = resolve;
+ });
+ return { ready, release };
+}
+
+/** Frozen claim/release scripts model the replicas that predate the prefix fix. */
+describeRedis('MCP catalog fence compatibility (real Redis)', () => {
+ const prefix = `CatalogFence-${randomUUID()}`;
+ const globalKey = `${prefix}::${CacheKeys.TOOL_CACHE}:tools:global`;
+ const globalLock = `${prefix}::${CacheKeys.TOOL_CACHE}:tools:global:write-lock`;
+ const legacyGlobalFence = `tools:global:write-fence:{${globalKey}}`;
+ const namespacedGlobalFence = `${prefix}::${CacheKeys.TOOL_CACHE}:tools:global:write-fence:{${globalKey}}`;
+ const userId = `user-${randomUUID()}`;
+ const serverName = 'server';
+ const generationKey = `${prefix}::${CacheKeys.TOOL_CACHE}:${ToolCacheKeys.MCP_SERVER_GENERATION(userId, serverName)}`;
+ const userLock = `${prefix}::${CacheKeys.TOOL_CACHE}:tools:mcp-write-lock:${userId}:${serverName}`;
+ const legacyUserFence = `tools:mcp:write-fence:{${userId}:${serverName}}`;
+ const namespacedUserFence = `${prefix}::${CacheKeys.TOOL_CACHE}:tools:mcp:write-fence:{${userId}:${serverName}}`;
+ const keys = new Set([
+ globalKey,
+ globalLock,
+ legacyGlobalFence,
+ namespacedGlobalFence,
+ generationKey,
+ userLock,
+ legacyUserFence,
+ namespacedUserFence,
+ ]);
+ let redis: RedisTestClient;
+
+ beforeAll(async () => {
+ redis = createRedisTestClient('');
+ await redis.connect();
+ });
+
+ afterEach(async () => {
+ await Promise.all([...keys].map((key) => redis.del(key)));
+ });
+
+ afterAll(() => redis.disconnect());
+
+ function createStore(legacy: boolean, get?: CatalogCache['get'], deployment = prefix) {
+ const cache: CatalogCache = {
+ get:
+ get ??
+ (async (key: string) => {
+ const raw = await redis.get(`${deployment}::${CacheKeys.TOOL_CACHE}:${key}`);
+ return raw == null ? null : JSON.parse(raw).value;
+ }),
+ set: async () => {
+ throw new Error('Expected an atomic Redis write');
+ },
+ delete: async () => {
+ throw new Error('Expected an atomic Redis delete');
+ },
+ };
+ const client: NonNullable<CatalogStoreDeps['keyvRedisClient']> = {
+ eval: async (script, options) => {
+ options.keys.forEach((key) => keys.add(key));
+ const scriptKeys = [
+ ...new Set(
+ options.keys.map((key) => {
+ if (!legacy) return key;
+ if (key === namespacedGlobalFence) return legacyGlobalFence;
+ if (key === namespacedUserFence) return legacyUserFence;
+ return key;
+ }),
+ ),
+ ];
+ if (legacy && script.includes("local current = redis.call('GET', KEYS[1])")) {
+ return redis.eval(
+ CLAIM_SCRIPT,
+ 1,
+ scriptKeys[scriptKeys.length - 1],
+ ...options.arguments,
+ );
+ }
+ if (legacy && script.includes('return released')) {
+ return redis.eval(
+ RELEASE_SCRIPT,
+ 1,
+ scriptKeys[scriptKeys.length - 1],
+ ...options.arguments,
+ );
+ }
+ return redis.eval(script, scriptKeys.length, ...scriptKeys, ...options.arguments);
+ },
+ };
+ const store = createMCPCatalogStore({
+ cacheConfig: { REDIS_KEY_PREFIX: deployment },
+ getCache: () => cache,
+ ioredisClient: {
+ set: (key, value, mode, ttl, condition) => {
+ keys.add(`${deployment}::${key}`);
+ return redis.set(`${deployment}::${key}`, value, mode, ttl, condition);
+ },
+ eval: (script, count, key, token) =>
+ redis.eval(legacy ? RELEASE_SCRIPT : script, count, `${deployment}::${key}`, token),
+ },
+ keyvRedisClient: client,
+ });
+ return store;
+ }
+
+ it.each([false, true])(
+ 'keeps a surviving global fence when the lock is lost (old owner: %s)',
+ async (oldOwner) => {
+ const store = createStore(oldOwner);
+ const contender = createStore(!oldOwner);
+ const entered = barrier();
+ const resume = barrier();
+ const write = store.runWithGlobalCacheLock(async () => {
+ entered.release();
+ await resume.ready;
+ return store.setCachedToolsWithinGlobalLock(tools('owner'));
+ });
+ await entered.ready;
+ try {
+ await redis.del(globalLock);
+ await expect(contender.setCachedTools(tools('stale'))).rejects.toThrow('superseded');
+ expect(await redis.get(globalKey)).toBeNull();
+ } finally {
+ resume.release();
+ await expect(write).resolves.toBe(true);
+ }
+ await expect(store.getCachedTools()).resolves.toEqual(tools('owner'));
+ expect(await redis.get(legacyGlobalFence)).toBeNull();
+ expect(await redis.get(namespacedGlobalFence)).toBeNull();
+ },
+ );
+
+ it.each([false, true])(
+ 'keeps a surviving user fence when the lock is lost (old owner: %s)',
+ async (oldOwner) => {
+ const entered = barrier();
+ const resume = barrier();
+ let reads = 0;
+ const store = createStore(oldOwner, async () => {
+ if (++reads === 2) {
+ entered.release();
+ await resume.ready;
+ }
+ return null;
+ });
+ const contender = createStore(!oldOwner);
+ const generation = store.getMCPToolsCacheGeneration({ userId, serverName });
+ await entered.ready;
+ try {
+ await redis.del(userLock);
+ await expect(contender.getMCPToolsCacheGeneration({ userId, serverName })).rejects.toThrow(
+ 'superseded',
+ );
+ expect(await redis.get(generationKey)).toBeNull();
+ } finally {
+ resume.release();
+ await expect(generation).resolves.toEqual(expect.any(String));
+ }
+ expect(await redis.get(legacyUserFence)).toBeNull();
+ expect(await redis.get(namespacedUserFence)).toBeNull();
+ },
+ );
+
+ it('isolates deployments sharing user/server IDs and releases only its own barrier entry', async () => {
+ const otherPrefix = `${prefix}-other`;
+ const entered = [barrier(), barrier()];
+ const resume = [barrier(), barrier()];
+ const stores = [prefix, otherPrefix].map((deployment, index) => {
+ let reads = 0;
+ return createStore(
+ false,
+ async () => {
+ if (++reads === 2) {
+ entered[index].release();
+ await resume[index].ready;
+ }
+ return null;
+ },
+ deployment,
+ );
+ });
+ const first = stores[0].getMCPToolsCacheGeneration({ userId, serverName });
+ await entered[0].ready;
+ const second = stores[1].getMCPToolsCacheGeneration({ userId, serverName });
+ const secondSettled = second.catch((error: Error) => error);
+ const enteredOrRejected = Promise.race([
+ entered[1].ready.then(() => 'entered'),
+ secondSettled.then(() => 'settled'),
+ ]);
+ try {
+ await expect(enteredOrRejected).resolves.toBe('entered');
+ const barrierBefore = JSON.parse((await redis.get(legacyUserFence))!);
+ expect(Object.keys(barrierBefore.owners)).toHaveLength(2);
+ const firstToken = await redis.get(namespacedUserFence);
+ const otherFence = `${otherPrefix}::${CacheKeys.TOOL_CACHE}:tools:mcp:write-fence:{${userId}:${serverName}}`;
+ const secondToken = await redis.get(otherFence);
+ expect(firstToken).not.toBe(secondToken);
+ expect(await redis.pttl(legacyUserFence)).toBeGreaterThan(0);
+ // Losing one deployment's lock must still reject its own competing writer.
+ await redis.del(userLock);
+ await expect(
+ createStore(false).getMCPToolsCacheGeneration({ userId, serverName }),
+ ).rejects.toThrow('superseded');
+ resume[0].release();
+ await expect(first).resolves.toEqual(expect.any(String));
+ const barrierAfter = JSON.parse((await redis.get(legacyUserFence))!);
+ expect(Object.keys(barrierAfter.owners)).toEqual([secondToken]);
+ expect(await redis.get(otherFence)).toBe(secondToken);
+ // The remaining deployment still excludes old writers after its peer exits.
+ expect(
+ Number(
+ await redis.eval(
+ CLAIM_SCRIPT,
+ 1,
+ legacyUserFence,
+ 'old-replica',
+ String(Date.now() + 30_000),
+ '1000',
+ ),
+ ),
+ ).toBe(0);
+ resume[1].release();
+ await expect(secondSettled).resolves.toEqual(expect.any(String));
+ expect(await redis.get(legacyUserFence)).toBeNull();
+ const otherGeneration = `${otherPrefix}::${CacheKeys.TOOL_CACHE}:${ToolCacheKeys.MCP_SERVER_GENERATION(userId, serverName)}`;
+ expect(await redis.get(generationKey)).not.toBeNull();
+ expect(await redis.get(otherGeneration)).not.toBeNull();
+ expect(await redis.get(otherFence)).toBeNull();
+ } finally {
+ resume.forEach((pause) => pause.release());
+ await Promise.allSettled([first, second]);
+ }
+ });
+
+ it('preserves longer peer leases and prunes expired compatibility owners', async () => {
+ const owner = 'peer';
+ const expired = 'expired';
+ const deadline = Date.now() + 60_000;
+ const value = {
+ protocol: 'librechat:mcp-catalog-fence:v1',
+ owners: { [owner]: deadline, [expired]: 1 },
+ };
+ await redis.set(legacyUserFence, JSON.stringify(value), 'PX', 60_000);
+ await expect(
+ createStore(false).getMCPToolsCacheGeneration({ userId, serverName }),
+ ).resolves.toEqual(expect.any(String));
+ expect(JSON.parse((await redis.get(legacyUserFence))!).owners).toEqual({ [owner]: deadline });
+ expect(await redis.pttl(legacyUserFence)).toBeGreaterThan(50_000);
+ });
+
+ it.each(['generation', 'migration'] as const)(
+ 'rejects a delayed user %s after its compatibility lease is replaced',
+ async (operation) => {
+ const entered = barrier();
+ const resume = barrier();
+ const rawKey = (key: string) => `${prefix}::${CacheKeys.TOOL_CACHE}:${key}`;
+ let reads = 0;
+ const store = createStore(false, async (key) => {
+ if (++reads === 2) {
+ entered.release();
+ await resume.ready;
+ }
+ const raw = await redis.get(rawKey(key));
+ return raw == null ? null : JSON.parse(raw).value;
+ });
+ const legacyTools = rawKey(ToolCacheKeys.MCP_SERVER(userId, serverName));
+ const migratedTools = rawKey(ToolCacheKeys.MCP_SERVER(userId, serverName, 'config'));
+ keys.add(legacyTools);
+ keys.add(migratedTools);
+ await redis.set(
+ legacyTools,
+ JSON.stringify({ value: tools('legacy'), expires: Date.now() + 60_000 }),
+ );
+ const write =
+ operation === 'generation'
+ ? store.getMCPToolsCacheGeneration({ userId, serverName })
+ : store.getCachedTools({ userId, serverName, configGeneration: 'config' });
+ const settled = write.catch((error: Error) => error);
+ await entered.ready;
+ try {
+ await redis.set(legacyUserFence, 'old-replica', 'PX', 30_000);
+ } finally {
+ resume.release();
+ await expect(settled).resolves.toMatchObject({
+ message: expect.stringContaining('ownership was lost'),
+ });
+ }
+ expect(await redis.get(generationKey)).toBeNull();
+ expect(await redis.get(migratedTools)).toBeNull();
+ expect(await redis.get(legacyUserFence)).toBe('old-replica');
+ },
+ );
+
+ it.each(['', '{tenant}', '{', '}'])(
+ 'works with a prefix-only ACL and prefix suffix %j',
+ async (suffix) => {
+ const aclPrefix = `${prefix}-${randomUUID()}${suffix}`;
+ const username = `catalog-${randomUUID()}`;
+ const password = randomUUID();
+ const nodes = 'nodes' in redis ? redis.nodes('master') : [redis];
+ const touched = new Set<string>();
+ const rawKey = (key: string) => `${aclPrefix}::${CacheKeys.TOOL_CACHE}:${key}`;
+ await Promise.all(
+ nodes.map((node) =>
+ node.acl('SETUSER', username, 'on', `>${password}`, `~${aclPrefix}*`, '+@all'),
+ ),
+ );
+ const restricted =
+ 'nodes' in redis
+ ? redis.duplicate([], {
+ lazyConnect: true,
+ redisOptions: { ...redis.options.redisOptions, username, password, keyPrefix: '' },
+ })
+ : redis.duplicate({ lazyConnect: true, username, password, keyPrefix: '' });
+ try {
+ await restricted.connect();
+ await expect(restricted.get(legacyGlobalFence)).rejects.toThrow('NOPERM');
+ const createStoreForClient = (client: RedisTestClient) =>
+ createMCPCatalogStore({
+ cacheConfig: { REDIS_KEY_PREFIX: aclPrefix },
+ getCache: () => ({
+ get: async (key) => {
+ const value = await client.get(rawKey(key));
+ return value == null ? null : JSON.parse(value).value;
+ },
+ set: async () => {
+ throw new Error('Expected an atomic Redis write');
+ },
+ delete: async () => {
+ throw new Error('Expected an atomic Redis delete');
+ },
+ }),
+ ioredisClient: {
+ set: (key, value, mode, ttl, condition) => {
+ const physical = `${aclPrefix}::${key}`;
+ touched.add(physical);
+ return client.set(physical, value, mode, ttl, condition);
+ },
+ eval: (script, count, key, token) =>
+ client.eval(script, count, `${aclPrefix}::${key}`, token),
+ },
+ keyvRedisClient: {
+ eval: (script, options) => {
+ options.keys.forEach((key) => touched.add(key));
+ return client.eval(
+ script,
+ options.keys.length,
+ ...options.keys,
+ ...options.arguments,
+ );
+ },
+ },
+ });
+ for (const client of [restricted, redis]) {
+ const store = createStoreForClient(client);
+ await expect(store.setCachedTools(tools('current'))).resolves.toBe(true);
+ await expect(store.getCachedTools()).resolves.toEqual(tools('current'));
+ await expect(store.getMCPToolsCacheGeneration({ userId, serverName })).resolves.toEqual(
+ expect.any(String),
+ );
+ await expect(
+ store.invalidateCachedTools({ invalidateGlobal: true }),
+ ).resolves.toBeUndefined();
+ expect(await client.get(rawKey(ToolCacheKeys.GLOBAL))).toBeNull();
+ await redis.del(rawKey(ToolCacheKeys.MCP_SERVER_GENERATION(userId, serverName)));
+ }
+ } finally {
+ restricted.disconnect();
+ await Promise.all([...touched].map((key) => redis.del(key)));
+ await Promise.all(nodes.map((node) => node.acl('DELUSER', username)));
+ }
+ },
+ );
+
+ it('rejects a delayed new write after an old replica replaces its legacy fence', async () => {
+ const store = createStore(false);
+ const entered = barrier();
+ const resume = barrier();
+ const write = store.runWithGlobalCacheLock(async () => {
+ entered.release();
+ await resume.ready;
+ return store.setCachedToolsWithinGlobalLock(tools('stale'));
+ });
+ const settled = write.catch((error: Error) => error);
+ await entered.ready;
+ try {
+ await redis.del(globalLock);
+ await redis.del(legacyGlobalFence);
+ expect(
+ Number(
+ await redis.eval(
+ CLAIM_SCRIPT,
+ 1,
+ legacyGlobalFence,
+ 'old-replica',
+ String(Date.now() + 30_000),
+ '1000',
+ ),
+ ),
+ ).toBe(1);
+ await redis.set(
+ globalKey,
+ JSON.stringify({ value: tools('replacement'), expires: Date.now() + 60_000 }),
+ );
+ } finally {
+ resume.release();
+ await expect(settled).resolves.toMatchObject({
+ message: expect.stringContaining('ownership was lost'),
+ });
+ }
+ await expect(store.getCachedTools()).resolves.toEqual(tools('replacement'));
+ expect(await redis.get(legacyGlobalFence)).toBe('old-replica');
+ });
+});
diff --git a/packages/api/src/mcp/catalog/store.ts b/packages/api/src/mcp/catalog/store.ts
index 80492245ff..f3950f8096 100644
--- a/packages/api/src/mcp/catalog/store.ts
+++ b/packages/api/src/mcp/catalog/store.ts
@@ -1,4 +1,5 @@
import { randomUUID } from 'crypto';
+import calculateSlot from 'cluster-key-slot';
import { logger } from '@librechat/data-schemas';
import { CacheKeys, Time } from 'librechat-data-provider';
import type { LCAvailableTools } from '../types';
@@ -7,28 +8,84 @@ const GLOBAL_LOCK_TTL_MS = 30_000;
const LOCK_FENCE_SAFETY_MS = 1_000;
const LOCK_RETRY_MS = 25;
const CACHE_ENTRY_VERSION = 1;
+const LEGACY_FENCE_BARRIER = 'librechat:mcp-catalog-fence:v1';
const GLOBAL_LOCK_KEY = `${CacheKeys.TOOL_CACHE}:tools:global:write-lock`;
const RELEASE_LOCK_SCRIPT = `
-if redis.call('GET', KEYS[1]) == ARGV[1] then
- return redis.call('DEL', KEYS[1])
+local released = 0
+for _, key in ipairs(KEYS) do
+ if redis.call('GET', key) == ARGV[1] then
+ released = released + redis.call('DEL', key)
+ end
end
-return 0
+return released
`;
const CLAIM_OWNERSHIP_FENCE_SCRIPT = `
local current = redis.call('GET', KEYS[1])
if current and current ~= ARGV[1] then return 0 end
+local owners = {}
+if KEYS[2] then
+ local legacy = redis.call('GET', KEYS[2])
+ if legacy then
+ local decoded, barrier = pcall(cjson.decode, legacy)
+ if not decoded or type(barrier) ~= 'table' or barrier['protocol'] ~= '${LEGACY_FENCE_BARRIER}'
+ or type(barrier['owners']) ~= 'table' then return 0 end
+ owners = barrier['owners']
+ end
+end
local redisTime = redis.call('TIME')
local now = (tonumber(redisTime[1]) * 1000) + math.floor(tonumber(redisTime[2]) / 1000)
-local ttl = tonumber(ARGV[2]) - now - tonumber(ARGV[3])
+local deadline = tonumber(ARGV[2]) - tonumber(ARGV[3])
+local ttl = deadline - now
if ttl <= 0 then return 0 end
+local barrierTtl = ttl
+for owner, expires in pairs(owners) do
+ if expires <= now then owners[owner] = nil
+ else barrierTtl = math.max(barrierTtl, expires - now) end
+end
+owners[ARGV[1]] = deadline
redis.call('PSETEX', KEYS[1], ttl, ARGV[1])
+if KEYS[2] then
+ redis.call('PSETEX', KEYS[2], barrierTtl, cjson.encode({protocol='${LEGACY_FENCE_BARRIER}', owners=owners}))
+end
return 1
`;
+const RELEASE_OWNERSHIP_FENCE_SCRIPT = `
+local released = 0
+if redis.call('GET', KEYS[1]) == ARGV[1] then
+ released = redis.call('DEL', KEYS[1])
+end
+if not KEYS[2] then return released end
+local decoded, barrier = pcall(cjson.decode, redis.call('GET', KEYS[2]) or '')
+if not decoded or type(barrier) ~= 'table' or barrier['protocol'] ~= '${LEGACY_FENCE_BARRIER}'
+ or type(barrier['owners']) ~= 'table' or not barrier['owners'][ARGV[1]] then return released end
+barrier['owners'][ARGV[1]] = nil
+local redisTime = redis.call('TIME')
+local now = (tonumber(redisTime[1]) * 1000) + math.floor(tonumber(redisTime[2]) / 1000)
+local ttl = 0
+for owner, expires in pairs(barrier['owners']) do
+ if expires <= now then barrier['owners'][owner] = nil
+ else ttl = math.max(ttl, expires - now) end
+end
+if ttl == 0 then redis.call('DEL', KEYS[2])
+else redis.call('PSETEX', KEYS[2], ttl, cjson.encode(barrier)) end
+return released
+`;
+
+function legacyOwnershipCheck(index: number, rejected: number): string {
+ return `
+if KEYS[${index}] then
+ local decoded, barrier = pcall(cjson.decode, redis.call('GET', KEYS[${index}]) or '')
+ if not decoded or type(barrier) ~= 'table' or barrier['protocol'] ~= '${LEGACY_FENCE_BARRIER}'
+ or type(barrier['owners']) ~= 'table' or not barrier['owners'][ARGV[1]] then return ${rejected} end
+end`;
+}
+
const CREATE_GENERATION_IF_OWNER_SCRIPT = `
if redis.call('GET', KEYS[1]) ~= ARGV[1] then return -1 end
+${legacyOwnershipCheck(3, -1)}
if redis.call('EXISTS', KEYS[2]) == 1 then return 0 end
redis.call('PSETEX', KEYS[2], ARGV[4], cjson.encode({value=ARGV[2], expires=tonumber(ARGV[3])}))
return 1
@@ -36,6 +93,7 @@ return 1
const MIGRATE_USER_TOOLS_IF_OWNER_SCRIPT = `
if redis.call('GET', KEYS[1]) ~= ARGV[1] then return -1 end
+${legacyOwnershipCheck(5, -1)}
if redis.call('EXISTS', KEYS[2]) == 1 or redis.call('EXISTS', KEYS[4]) == 1 then return 0 end
local rawGeneration = redis.call('GET', KEYS[3])
if rawGeneration then
@@ -75,6 +133,7 @@ return 1
const WRITE_GLOBAL_IF_OWNER_SCRIPT = `
if redis.call('GET', KEYS[1]) ~= ARGV[1] then return 0 end
+${legacyOwnershipCheck(3, 0)}
local decoded, tools = pcall(cjson.decode, ARGV[2])
if not decoded then return redis.error_reply('Invalid global tools catalog') end
redis.call('PSETEX', KEYS[2], ARGV[4], cjson.encode({value=tools, expires=tonumber(ARGV[3])}))
@@ -112,6 +171,7 @@ return 1
const DELETE_GLOBAL_IF_OWNER_SCRIPT = `
if redis.call('GET', KEYS[1]) ~= ARGV[1] then return 0 end
+${legacyOwnershipCheck(3, 0)}
redis.call('DEL', KEYS[2])
return 1
`;
@@ -197,6 +257,13 @@ interface AppToolsEntry {
interface OwnershipFence {
key: string;
token: string;
+ legacyKey?: string;
+}
+
+function ownershipKeys(fence: OwnershipFence, ...keys: string[]): string[] {
+ const result = [fence.key, ...keys];
+ if (fence.legacyKey) result.push(fence.legacyKey);
+ return result;
}
export interface MCPCatalogStore {
@@ -293,6 +360,8 @@ export function createMCPCatalogStore(deps: CatalogStoreDeps): MCPCatalogStore {
const appWriteQueues = new Map<string, Promise<void>>();
let globalQueue = Promise.resolve();
let globalLockToken: string | undefined;
+ let globalLockFenceKey: string | undefined;
+ let globalLockLegacyFenceKey: string | undefined;
const sharedRedis = (): boolean =>
deps.ioredisClient != null &&
@@ -348,9 +417,25 @@ export function createMCPCatalogStore(deps: CatalogStoreDeps): MCPCatalogStore {
const closingBrace = openingBrace >= 0 ? key.indexOf('}', openingBrace + 1) : -1;
return closingBrace > openingBrace + 1 ? key.slice(openingBrace + 1, closingBrace) : key;
};
+ const prefixHasBraces = /[{}]/.test(rawKey(''));
+ const ownershipFenceKey = (name: string, guardedKey: string): string => {
+ let key = rawKey(`${name}:{${redisHashTag(guardedKey)}}`);
+ if (!prefixHasBraces) return key;
+ const slot = calculateSlot(guardedKey);
+ if (calculateSlot(key) === slot) return key;
+ // Literal prefix braces can override the synthetic tag. Reuse the guarded tag
+ // when present; otherwise find a same-slot suffix without adding braces.
+ let suffix = 0;
+ key = `${guardedKey}:${name}`;
+ while (calculateSlot(key) !== slot) {
+ key = `${guardedKey}:${name}:${suffix++}`;
+ }
+ return key;
+ };
const globalCacheRawKey = rawKey(ToolCacheKeys.GLOBAL);
/** Hashes to the same Redis Cluster slot as the unchanged legacy global catalog key. */
- const globalFenceKey = `tools:global:write-fence:{${redisHashTag(globalCacheRawKey)}}`;
+ const globalFenceKey = ownershipFenceKey('tools:global:write-fence', globalCacheRawKey);
+ const legacyGlobalFenceKey = `tools:global:write-fence:{${redisHashTag(globalCacheRawKey)}}`;
async function withRedisLock<T>(
lockKey: string,
@@ -389,6 +474,7 @@ export function createMCPCatalogStore(deps: CatalogStoreDeps): MCPCatalogStore {
async function withOwnershipFence<T>(
fenceKey: string,
+ legacyFenceKey: string,
token: string | undefined,
leaseExpiresAt: number | undefined,
operation: (fence?: OwnershipFence) => Promise<T>,
@@ -400,19 +486,36 @@ export function createMCPCatalogStore(deps: CatalogStoreDeps): MCPCatalogStore {
if (fenceTtl <= 0) {
throw new Error('Tool cache lock expired before ownership could be fenced');
}
- const claimed = await deps.keyvRedisClient!.eval(CLAIM_OWNERSHIP_FENCE_SCRIPT, {
- keys: [fenceKey],
- arguments: [token, String(leaseExpiresAt), String(LOCK_FENCE_SAFETY_MS)],
- });
+ const redis = deps.keyvRedisClient!;
+ const args = [token, String(leaseExpiresAt), String(LOCK_FENCE_SAFETY_MS)];
+ let keys = [fenceKey, legacyFenceKey];
+ let claimed: unknown;
+ try {
+ claimed = await redis.eval(CLAIM_OWNERSHIP_FENCE_SCRIPT, { keys, arguments: args });
+ } catch (error) {
+ if (!(error instanceof Error)) throw error;
+ // Both rejections precede execution. Legacy writers cannot publish under
+ // the same key ACL, or when their fence cannot share the catalog's slot.
+ const denied = error.message.startsWith('NOPERM ');
+ const crossSlot =
+ calculateSlot(fenceKey) !== calculateSlot(legacyFenceKey) &&
+ (error.message === 'Redis catalog script keys must share one cluster slot' ||
+ error.message.startsWith('CROSSSLOT '));
+ if (!denied && !crossSlot) throw error;
+ keys = [fenceKey];
+ claimed = await redis.eval(CLAIM_OWNERSHIP_FENCE_SCRIPT, { keys, arguments: args });
+ }
if (Number(claimed) !== 1) {
throw new Error('Tool cache lock expired or was superseded before ownership could be fenced');
}
try {
- return await operation({ key: fenceKey, token });
+ // Ownership stays deployment-scoped. The shared barrier excludes old writers
+ // without making current deployments contend with each other.
+ return await operation({ key: fenceKey, token, legacyKey: keys[1] });
} finally {
try {
- await deps.keyvRedisClient!.eval(RELEASE_LOCK_SCRIPT, {
- keys: [fenceKey],
+ await redis.eval(RELEASE_OWNERSHIP_FENCE_SCRIPT, {
+ keys,
arguments: [token],
});
} catch (error) {
@@ -431,11 +534,12 @@ export function createMCPCatalogStore(deps: CatalogStoreDeps): MCPCatalogStore {
const previous = userQueues.get(scope) ?? Promise.resolve();
const lockKey = `${CacheKeys.TOOL_CACHE}:tools:mcp-write-lock:${encodeURIComponent(userId)}:${encodeURIComponent(serverName)}`;
const generationRawKey = rawKey(ToolCacheKeys.MCP_SERVER_GENERATION(userId, serverName));
- const fenceKey = `tools:mcp:write-fence:{${redisHashTag(generationRawKey)}}`;
+ const fenceKey = ownershipFenceKey('tools:mcp:write-fence', generationRawKey);
+ const legacyFenceKey = `tools:mcp:write-fence:{${redisHashTag(generationRawKey)}}`;
const locked = () =>
withRedisLock(lockKey, GLOBAL_LOCK_TTL_MS, (token, leaseExpiresAt) =>
requireOwnershipFence
- ? withOwnershipFence(fenceKey, token, leaseExpiresAt, operation)
+ ? withOwnershipFence(fenceKey, legacyFenceKey, token, leaseExpiresAt, operation)
: operation(),
);
const result = previous.then(locked, locked);
@@ -481,11 +585,14 @@ export function createMCPCatalogStore(deps: CatalogStoreDeps): MCPCatalogStore {
if (!sharedRedis()) {
return (await cache.set(ToolCacheKeys.GLOBAL, tools, ttl)) !== false;
}
- if (!globalLockToken) {
+ if (!globalLockToken || !globalLockFenceKey) {
throw new Error('Global tool cache write requires lock ownership');
}
const written = await deps.keyvRedisClient!.eval(WRITE_GLOBAL_IF_OWNER_SCRIPT, {
- keys: [globalFenceKey, globalCacheRawKey],
+ keys: ownershipKeys(
+ { key: globalLockFenceKey, token: globalLockToken, legacyKey: globalLockLegacyFenceKey },
+ globalCacheRawKey,
+ ),
arguments: [globalLockToken, JSON.stringify(tools), String(Date.now() + ttl), String(ttl)],
});
if (Number(written) !== 1) {
@@ -499,11 +606,14 @@ export function createMCPCatalogStore(deps: CatalogStoreDeps): MCPCatalogStore {
await cache.delete(ToolCacheKeys.GLOBAL);
return;
}
- if (!globalLockToken) {
+ if (!globalLockToken || !globalLockFenceKey) {
throw new Error('Global tool cache invalidation requires lock ownership');
}
const deleted = await deps.keyvRedisClient!.eval(DELETE_GLOBAL_IF_OWNER_SCRIPT, {
- keys: [globalFenceKey, globalCacheRawKey],
+ keys: ownershipKeys(
+ { key: globalLockFenceKey, token: globalLockToken, legacyKey: globalLockLegacyFenceKey },
+ globalCacheRawKey,
+ ),
arguments: [globalLockToken],
});
if (Number(deleted) !== 1) {
@@ -514,14 +624,24 @@ export function createMCPCatalogStore(deps: CatalogStoreDeps): MCPCatalogStore {
function runWithGlobalCacheLock<T>(operation: () => Promise<T>): Promise<T> {
const locked = () =>
withRedisLock(GLOBAL_LOCK_KEY, GLOBAL_LOCK_TTL_MS, async (token, leaseExpiresAt) => {
- return withOwnershipFence(globalFenceKey, token, leaseExpiresAt, async (fence) => {
- globalLockToken = fence?.token;
- try {
- return await operation();
- } finally {
- globalLockToken = undefined;
- }
- });
+ return withOwnershipFence(
+ globalFenceKey,
+ legacyGlobalFenceKey,
+ token,
+ leaseExpiresAt,
+ async (fence) => {
+ globalLockToken = fence?.token;
+ globalLockFenceKey = fence?.key;
+ globalLockLegacyFenceKey = fence?.legacyKey;
+ try {
+ return await operation();
+ } finally {
+ globalLockToken = undefined;
+ globalLockFenceKey = undefined;
+ globalLockLegacyFenceKey = undefined;
+ }
+ },
+ );
});
const result = globalQueue.then(locked, locked);
globalQueue = result.then(
@@ -582,7 +702,12 @@ export function createMCPCatalogStore(deps: CatalogStoreDeps): MCPCatalogStore {
}
const toolsTtl = Time.TWELVE_HOURS;
const written = await deps.keyvRedisClient!.eval(MIGRATE_USER_TOOLS_IF_OWNER_SCRIPT, {
- keys: [fence.key, rawKey(legacyFenceKey), rawKey(generationKey), rawKey(toolsKey)],
+ keys: ownershipKeys(
+ fence,
+ rawKey(legacyFenceKey),
+ rawKey(generationKey),
+ rawKey(toolsKey),
+ ),
arguments: [
fence.token,
publicationGeneration,
@@ -680,7 +805,7 @@ export function createMCPCatalogStore(deps: CatalogStoreDeps): MCPCatalogStore {
throw new Error('Tool publication generation creation requires lock ownership');
}
const created = await deps.keyvRedisClient!.eval(CREATE_GENERATION_IF_OWNER_SCRIPT, {
- keys: [fence.key, rawKey(key)],
+ keys: ownershipKeys(fence, rawKey(key)),
arguments: [
fence.token,
generation,