Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active
TypeScript aianthropicartifactsawsazurechatgptchatgpt-cloneclaudeclonedeepseekgeminigooglegpt-5librechatmcpo1openairesponses-apivisionwebui
🤺 fix: Prefix MCP Tool Cache Write Fences (#17458)
lia-by-librechat[bot]8e68cab31b77
diff --git a/api/server/services/Config/__tests__/getCachedTools.lock.spec.js b/api/server/services/Config/__tests__/getCachedTools.lock.spec.jsindex 50e99ecfd0..3ee2aa59b4 100644--- a/api/server/services/Config/__tests__/getCachedTools.lock.spec.js+++ b/api/server/services/Config/__tests__/getCachedTools.lock.spec.js@@ -34,6 +34,7 @@ const getLogStores = require('~/cache/getLogStores');const mockCache = { get: jest.fn(), set: jest.fn(), delete: jest.fn() };getLogStores.mockReturnValue(mockCache);+const { createMCPCatalogStore } = require('@librechat/api');const {getCachedTools,updateCachedGlobalTools,@@ -62,6 +63,100 @@ describe('global tool cache write lock', () => {mockCache.delete.mockResolvedValue(true);});+ describe.each([+ { REDIS_KEY_PREFIX: undefined },+ { REDIS_KEY_PREFIX: '' },+ { REDIS_KEY_PREFIX: 'myprefix_' },+ { REDIS_KEY_PREFIX: 'myprefix_', GLOBAL_PREFIX_SEPARATOR: ':' },+ { REDIS_KEY_PREFIX: 'myprefix_{tenant}', GLOBAL_PREFIX_SEPARATOR: '' },+ { REDIS_KEY_PREFIX: 'myprefix_{' },+ { REDIS_KEY_PREFIX: 'myprefix_}' },+ { REDIS_KEY_PREFIX: 'myprefix_{{tenant}' },+ { REDIS_KEY_PREFIX: 'myprefix_', GLOBAL_PREFIX_SEPARATOR: '{' },+ ])('fence key namespace %j', (cacheConfig) => {+ it.each([+ ['global', false],+ ['user', false],+ ['global', true],+ ['user', true],+ ])(+ 'namespaces the %s fence without changing its slot (restricted ACL: %s)',+ async (scope, restricted) => {+ mockCache.get.mockResolvedValue(null);+ const store = createMCPCatalogStore({+ cacheConfig,+ getCache: () => mockCache,+ ioredisClient: mockRedisClient,+ keyvRedisClient: mockKeyvRedisClient,+ });+ const prefix = cacheConfig.REDIS_KEY_PREFIX+ ? `${cacheConfig.REDIS_KEY_PREFIX}${cacheConfig.GLOBAL_PREFIX_SEPARATOR ?? '::'}`+ : '';+ const namespace = `${prefix}${CacheKeys.TOOL_CACHE}:`;+ const guardedKey =+ scope === 'global'+ ? `${namespace}tools:global`+ : `${namespace}tools:metadata:mcp:user-generation:{user-1:server-1}`;++ const opening = guardedKey.indexOf('{');+ const closing = opening < 0 ? -1 : guardedKey.indexOf('}', opening + 1);+ const hashTag = closing > opening + 1 ? guardedKey.slice(opening + 1, closing) : guardedKey;+ const legacyFenceKey = `tools:${scope === 'global' ? 'global' : 'mcp'}:write-fence:{${hashTag}}`;+ mockKeyvRedisClient.eval.mockImplementation(async (_script, { keys }) => {+ if (keys.some((key) => calculateSlot(key) !== calculateSlot(keys[0]))) {+ throw new Error('Redis catalog script keys must share one cluster slot');+ }+ if (restricted && keys.includes(legacyFenceKey)) {+ throw new Error('NOPERM no permissions to access one of the keys used as arguments');+ }+ return 1;+ });++ if (scope === 'global') {+ await expect(store.setCachedTools({ builtin: {} })).resolves.toBe(true);+ } else {+ await expect(+ store.getMCPToolsCacheGeneration({ userId: 'user-1', serverName: 'server-1' }),+ ).resolves.toEqual(expect.any(String));+ }++ const fenceKey = mockKeyvRedisClient.eval.mock.calls[0][1].keys[0];+ expect(fenceKey.startsWith(namespace)).toBe(true);+ expect(fenceKey).toContain('write-fence');+ expect(fenceKey).not.toBe(guardedKey);+ const fallback = restricted || calculateSlot(legacyFenceKey) !== calculateSlot(guardedKey);+ const claimKeys = fallback ? [fenceKey] : [fenceKey, legacyFenceKey];+ const calls = [[fenceKey, legacyFenceKey]];+ if (fallback) calls.push(claimKeys);+ calls.push(+ fallback ? [fenceKey, guardedKey] : [fenceKey, guardedKey, legacyFenceKey],+ claimKeys,+ );+ expect(mockKeyvRedisClient.eval.mock.calls.map(([, options]) => options.keys)).toEqual(+ calls,+ );+ expect(calculateSlot(fenceKey)).toBe(calculateSlot(guardedKey));+ },+ );+ });++ it.each([+ 'READONLY replica is read-only',+ 'ERR Error running script: NOPERM inside script',+ 'connection lost',+ 'CROSSSLOT Keys in request do not hash to the same slot',+ 'Redis catalog script keys must share one cluster slot',+ ])('does not retry an ambiguous fence claim failure: %s', async (message) => {+ mockKeyvRedisClient.eval.mockRejectedValue(new Error(message));+ const operation = jest.fn();++ await expect(runWithGlobalCacheLock(operation)).rejects.toThrow(message);++ expect(operation).not.toHaveBeenCalled();+ expect(mockKeyvRedisClient.eval).toHaveBeenCalledTimes(1);+ expect(mockRedisClient.eval).toHaveBeenCalledTimes(1);+ });+it('waits for the shared Redis client before accessing the catalog', async () => {mockCache.get.mockResolvedValue(null);let resolveRedisReady;@@ -147,8 +242,9 @@ describe('global tool cache write lock', () => {expect.stringContaining("redis.call('GET', KEYS[1])"),expect.objectContaining({keys: [- `tools:global:write-fence:{${CacheKeys.TOOL_CACHE}:tools:global}`,+ `${CacheKeys.TOOL_CACHE}:tools:global:write-fence:{${CacheKeys.TOOL_CACHE}:tools:global}`,`${CacheKeys.TOOL_CACHE}:tools:global`,+ `tools:global:write-fence:{${CacheKeys.TOOL_CACHE}:tools:global}`,],}),);@@ -168,8 +264,9 @@ describe('global tool cache write lock', () => {expect.stringContaining("redis.call('PSETEX', KEYS[2]"),expect.objectContaining({keys: [- `tools:global:write-fence:{${CacheKeys.TOOL_CACHE}:tools:global}`,+ `${CacheKeys.TOOL_CACHE}:tools:global:write-fence:{${CacheKeys.TOOL_CACHE}:tools:global}`,`${CacheKeys.TOOL_CACHE}:tools:global`,+ `tools:global:write-fence:{${CacheKeys.TOOL_CACHE}:tools:global}`,],arguments: [expect.any(String),@@ -214,6 +311,7 @@ describe('global tool cache write lock', () => {expect(operation).not.toHaveBeenCalled();const [claimScript, claimOptions] = mockKeyvRedisClient.eval.mock.calls[0];expect(claimScript).toContain('current ~= ARGV[1]');+ expect(claimOptions.keys).toHaveLength(2);expect(claimScript).toContain("redis.call('TIME')");expect(claimOptions.arguments).toEqual([mockRedisClient.set.mock.calls[0][1],@@ -339,10 +437,11 @@ describe('global tool cache write lock', () => {expect.stringContaining("redis.call('EXISTS', KEYS[2])"),expect.objectContaining({keys: [- `tools:mcp:write-fence:{user-1:server-1}`,+ `${CacheKeys.TOOL_CACHE}:tools:mcp:write-fence:{user-1:server-1}`,`${CacheKeys.TOOL_CACHE}:tools:metadata:mcp:user-legacy-fence:{user-1:server-1}`,`${CacheKeys.TOOL_CACHE}:tools:metadata:mcp:user-generation:{user-1:server-1}`,`${CacheKeys.TOOL_CACHE}:tools:mcp:user:{user-1:server-1}:v2:config-current`,+ `tools:mcp:write-fence:{user-1:server-1}`,],}),);diff --git a/packages/api/src/mcp/catalog/store.cache_integration.spec.ts b/packages/api/src/mcp/catalog/store.cache_integration.spec.tsnew file mode 100644index 0000000000..c7b9475259--- /dev/null+++ b/packages/api/src/mcp/catalog/store.cache_integration.spec.ts@@ -0,0 +1,442 @@+import { randomUUID } from 'node:crypto';+import { CacheKeys } from 'librechat-data-provider';+import type { RedisTestClient } from '../../stream/__tests__/helpers/redis';+import type { CatalogCache, CatalogStoreDeps } from './store';+import { createRedisTestClient } from '../../stream/__tests__/helpers/redis';+import { createMCPCatalogStore, ToolCacheKeys } from './store';++const describeRedis = process.env.USE_REDIS === 'true' ? describe : describe.skip;+const tools = (name: string) => ({ [name]: { type: 'function' as const, function: { name } } });+const RELEASE_SCRIPT = `+if redis.call('GET', KEYS[1]) == ARGV[1] then+ return redis.call('DEL', KEYS[1])+end+return 0+`;+const CLAIM_SCRIPT = `+local current = redis.call('GET', KEYS[1])+if current and current ~= ARGV[1] then return 0 end+local redisTime = redis.call('TIME')+local now = (tonumber(redisTime[1]) * 1000) + math.floor(tonumber(redisTime[2]) / 1000)+local ttl = tonumber(ARGV[2]) - now - tonumber(ARGV[3])+if ttl <= 0 then return 0 end+redis.call('PSETEX', KEYS[1], ttl, ARGV[1])+return 1+`;++function barrier() {+ let release!: () => void;+ const ready = new Promise<void>((resolve) => {+ release = resolve;+ });+ return { ready, release };+}++/** Frozen claim/release scripts model the replicas that predate the prefix fix. */+describeRedis('MCP catalog fence compatibility (real Redis)', () => {+ const prefix = `CatalogFence-${randomUUID()}`;+ const globalKey = `${prefix}::${CacheKeys.TOOL_CACHE}:tools:global`;+ const globalLock = `${prefix}::${CacheKeys.TOOL_CACHE}:tools:global:write-lock`;+ const legacyGlobalFence = `tools:global:write-fence:{${globalKey}}`;+ const namespacedGlobalFence = `${prefix}::${CacheKeys.TOOL_CACHE}:tools:global:write-fence:{${globalKey}}`;+ const userId = `user-${randomUUID()}`;+ const serverName = 'server';+ const generationKey = `${prefix}::${CacheKeys.TOOL_CACHE}:${ToolCacheKeys.MCP_SERVER_GENERATION(userId, serverName)}`;+ const userLock = `${prefix}::${CacheKeys.TOOL_CACHE}:tools:mcp-write-lock:${userId}:${serverName}`;+ const legacyUserFence = `tools:mcp:write-fence:{${userId}:${serverName}}`;+ const namespacedUserFence = `${prefix}::${CacheKeys.TOOL_CACHE}:tools:mcp:write-fence:{${userId}:${serverName}}`;+ const keys = new Set([+ globalKey,+ globalLock,+ legacyGlobalFence,+ namespacedGlobalFence,+ generationKey,+ userLock,+ legacyUserFence,+ namespacedUserFence,+ ]);+ let redis: RedisTestClient;++ beforeAll(async () => {+ redis = createRedisTestClient('');+ await redis.connect();+ });++ afterEach(async () => {+ await Promise.all([...keys].map((key) => redis.del(key)));+ });++ afterAll(() => redis.disconnect());++ function createStore(legacy: boolean, get?: CatalogCache['get'], deployment = prefix) {+ const cache: CatalogCache = {+ get:+ get ??+ (async (key: string) => {+ const raw = await redis.get(`${deployment}::${CacheKeys.TOOL_CACHE}:${key}`);+ return raw == null ? null : JSON.parse(raw).value;+ }),+ set: async () => {+ throw new Error('Expected an atomic Redis write');+ },+ delete: async () => {+ throw new Error('Expected an atomic Redis delete');+ },+ };+ const client: NonNullable<CatalogStoreDeps['keyvRedisClient']> = {+ eval: async (script, options) => {+ options.keys.forEach((key) => keys.add(key));+ const scriptKeys = [+ ...new Set(+ options.keys.map((key) => {+ if (!legacy) return key;+ if (key === namespacedGlobalFence) return legacyGlobalFence;+ if (key === namespacedUserFence) return legacyUserFence;+ return key;+ }),+ ),+ ];+ if (legacy && script.includes("local current = redis.call('GET', KEYS[1])")) {+ return redis.eval(+ CLAIM_SCRIPT,+ 1,+ scriptKeys[scriptKeys.length - 1],+ ...options.arguments,+ );+ }+ if (legacy && script.includes('return released')) {+ return redis.eval(+ RELEASE_SCRIPT,+ 1,+ scriptKeys[scriptKeys.length - 1],+ ...options.arguments,+ );+ }+ return redis.eval(script, scriptKeys.length, ...scriptKeys, ...options.arguments);+ },+ };+ const store = createMCPCatalogStore({+ cacheConfig: { REDIS_KEY_PREFIX: deployment },+ getCache: () => cache,+ ioredisClient: {+ set: (key, value, mode, ttl, condition) => {+ keys.add(`${deployment}::${key}`);+ return redis.set(`${deployment}::${key}`, value, mode, ttl, condition);+ },+ eval: (script, count, key, token) =>+ redis.eval(legacy ? RELEASE_SCRIPT : script, count, `${deployment}::${key}`, token),+ },+ keyvRedisClient: client,+ });+ return store;+ }++ it.each([false, true])(+ 'keeps a surviving global fence when the lock is lost (old owner: %s)',+ async (oldOwner) => {+ const store = createStore(oldOwner);+ const contender = createStore(!oldOwner);+ const entered = barrier();+ const resume = barrier();+ const write = store.runWithGlobalCacheLock(async () => {+ entered.release();+ await resume.ready;+ return store.setCachedToolsWithinGlobalLock(tools('owner'));+ });+ await entered.ready;+ try {+ await redis.del(globalLock);+ await expect(contender.setCachedTools(tools('stale'))).rejects.toThrow('superseded');+ expect(await redis.get(globalKey)).toBeNull();+ } finally {+ resume.release();+ await expect(write).resolves.toBe(true);+ }+ await expect(store.getCachedTools()).resolves.toEqual(tools('owner'));+ expect(await redis.get(legacyGlobalFence)).toBeNull();+ expect(await redis.get(namespacedGlobalFence)).toBeNull();+ },+ );++ it.each([false, true])(+ 'keeps a surviving user fence when the lock is lost (old owner: %s)',+ async (oldOwner) => {+ const entered = barrier();+ const resume = barrier();+ let reads = 0;+ const store = createStore(oldOwner, async () => {+ if (++reads === 2) {+ entered.release();+ await resume.ready;+ }+ return null;+ });+ const contender = createStore(!oldOwner);+ const generation = store.getMCPToolsCacheGeneration({ userId, serverName });+ await entered.ready;+ try {+ await redis.del(userLock);+ await expect(contender.getMCPToolsCacheGeneration({ userId, serverName })).rejects.toThrow(+ 'superseded',+ );+ expect(await redis.get(generationKey)).toBeNull();+ } finally {+ resume.release();+ await expect(generation).resolves.toEqual(expect.any(String));+ }+ expect(await redis.get(legacyUserFence)).toBeNull();+ expect(await redis.get(namespacedUserFence)).toBeNull();+ },+ );++ it('isolates deployments sharing user/server IDs and releases only its own barrier entry', async () => {+ const otherPrefix = `${prefix}-other`;+ const entered = [barrier(), barrier()];+ const resume = [barrier(), barrier()];+ const stores = [prefix, otherPrefix].map((deployment, index) => {+ let reads = 0;+ return createStore(+ false,+ async () => {+ if (++reads === 2) {+ entered[index].release();+ await resume[index].ready;+ }+ return null;+ },+ deployment,+ );+ });+ const first = stores[0].getMCPToolsCacheGeneration({ userId, serverName });+ await entered[0].ready;+ const second = stores[1].getMCPToolsCacheGeneration({ userId, serverName });+ const secondSettled = second.catch((error: Error) => error);+ const enteredOrRejected = Promise.race([+ entered[1].ready.then(() => 'entered'),+ secondSettled.then(() => 'settled'),+ ]);+ try {+ await expect(enteredOrRejected).resolves.toBe('entered');+ const barrierBefore = JSON.parse((await redis.get(legacyUserFence))!);+ expect(Object.keys(barrierBefore.owners)).toHaveLength(2);+ const firstToken = await redis.get(namespacedUserFence);+ const otherFence = `${otherPrefix}::${CacheKeys.TOOL_CACHE}:tools:mcp:write-fence:{${userId}:${serverName}}`;+ const secondToken = await redis.get(otherFence);+ expect(firstToken).not.toBe(secondToken);+ expect(await redis.pttl(legacyUserFence)).toBeGreaterThan(0);+ // Losing one deployment's lock must still reject its own competing writer.+ await redis.del(userLock);+ await expect(+ createStore(false).getMCPToolsCacheGeneration({ userId, serverName }),+ ).rejects.toThrow('superseded');+ resume[0].release();+ await expect(first).resolves.toEqual(expect.any(String));+ const barrierAfter = JSON.parse((await redis.get(legacyUserFence))!);+ expect(Object.keys(barrierAfter.owners)).toEqual([secondToken]);+ expect(await redis.get(otherFence)).toBe(secondToken);+ // The remaining deployment still excludes old writers after its peer exits.+ expect(+ Number(+ await redis.eval(+ CLAIM_SCRIPT,+ 1,+ legacyUserFence,+ 'old-replica',+ String(Date.now() + 30_000),+ '1000',+ ),+ ),+ ).toBe(0);+ resume[1].release();+ await expect(secondSettled).resolves.toEqual(expect.any(String));+ expect(await redis.get(legacyUserFence)).toBeNull();+ const otherGeneration = `${otherPrefix}::${CacheKeys.TOOL_CACHE}:${ToolCacheKeys.MCP_SERVER_GENERATION(userId, serverName)}`;+ expect(await redis.get(generationKey)).not.toBeNull();+ expect(await redis.get(otherGeneration)).not.toBeNull();+ expect(await redis.get(otherFence)).toBeNull();+ } finally {+ resume.forEach((pause) => pause.release());+ await Promise.allSettled([first, second]);+ }+ });++ it('preserves longer peer leases and prunes expired compatibility owners', async () => {+ const owner = 'peer';+ const expired = 'expired';+ const deadline = Date.now() + 60_000;+ const value = {+ protocol: 'librechat:mcp-catalog-fence:v1',+ owners: { [owner]: deadline, [expired]: 1 },+ };+ await redis.set(legacyUserFence, JSON.stringify(value), 'PX', 60_000);+ await expect(+ createStore(false).getMCPToolsCacheGeneration({ userId, serverName }),+ ).resolves.toEqual(expect.any(String));+ expect(JSON.parse((await redis.get(legacyUserFence))!).owners).toEqual({ [owner]: deadline });+ expect(await redis.pttl(legacyUserFence)).toBeGreaterThan(50_000);+ });++ it.each(['generation', 'migration'] as const)(+ 'rejects a delayed user %s after its compatibility lease is replaced',+ async (operation) => {+ const entered = barrier();+ const resume = barrier();+ const rawKey = (key: string) => `${prefix}::${CacheKeys.TOOL_CACHE}:${key}`;+ let reads = 0;+ const store = createStore(false, async (key) => {+ if (++reads === 2) {+ entered.release();+ await resume.ready;+ }+ const raw = await redis.get(rawKey(key));+ return raw == null ? null : JSON.parse(raw).value;+ });+ const legacyTools = rawKey(ToolCacheKeys.MCP_SERVER(userId, serverName));+ const migratedTools = rawKey(ToolCacheKeys.MCP_SERVER(userId, serverName, 'config'));+ keys.add(legacyTools);+ keys.add(migratedTools);+ await redis.set(+ legacyTools,+ JSON.stringify({ value: tools('legacy'), expires: Date.now() + 60_000 }),+ );+ const write =+ operation === 'generation'+ ? store.getMCPToolsCacheGeneration({ userId, serverName })+ : store.getCachedTools({ userId, serverName, configGeneration: 'config' });+ const settled = write.catch((error: Error) => error);+ await entered.ready;+ try {+ await redis.set(legacyUserFence, 'old-replica', 'PX', 30_000);+ } finally {+ resume.release();+ await expect(settled).resolves.toMatchObject({+ message: expect.stringContaining('ownership was lost'),+ });+ }+ expect(await redis.get(generationKey)).toBeNull();+ expect(await redis.get(migratedTools)).toBeNull();+ expect(await redis.get(legacyUserFence)).toBe('old-replica');+ },+ );++ it.each(['', '{tenant}', '{', '}'])(+ 'works with a prefix-only ACL and prefix suffix %j',+ async (suffix) => {+ const aclPrefix = `${prefix}-${randomUUID()}${suffix}`;+ const username = `catalog-${randomUUID()}`;+ const password = randomUUID();+ const nodes = 'nodes' in redis ? redis.nodes('master') : [redis];+ const touched = new Set<string>();+ const rawKey = (key: string) => `${aclPrefix}::${CacheKeys.TOOL_CACHE}:${key}`;+ await Promise.all(+ nodes.map((node) =>+ node.acl('SETUSER', username, 'on', `>${password}`, `~${aclPrefix}*`, '+@all'),+ ),+ );+ const restricted =+ 'nodes' in redis+ ? redis.duplicate([], {+ lazyConnect: true,+ redisOptions: { ...redis.options.redisOptions, username, password, keyPrefix: '' },+ })+ : redis.duplicate({ lazyConnect: true, username, password, keyPrefix: '' });+ try {+ await restricted.connect();+ await expect(restricted.get(legacyGlobalFence)).rejects.toThrow('NOPERM');+ const createStoreForClient = (client: RedisTestClient) =>+ createMCPCatalogStore({+ cacheConfig: { REDIS_KEY_PREFIX: aclPrefix },+ getCache: () => ({+ get: async (key) => {+ const value = await client.get(rawKey(key));+ return value == null ? null : JSON.parse(value).value;+ },+ set: async () => {+ throw new Error('Expected an atomic Redis write');+ },+ delete: async () => {+ throw new Error('Expected an atomic Redis delete');+ },+ }),+ ioredisClient: {+ set: (key, value, mode, ttl, condition) => {+ const physical = `${aclPrefix}::${key}`;+ touched.add(physical);+ return client.set(physical, value, mode, ttl, condition);+ },+ eval: (script, count, key, token) =>+ client.eval(script, count, `${aclPrefix}::${key}`, token),+ },+ keyvRedisClient: {+ eval: (script, options) => {+ options.keys.forEach((key) => touched.add(key));+ return client.eval(+ script,+ options.keys.length,+ ...options.keys,+ ...options.arguments,+ );+ },+ },+ });+ for (const client of [restricted, redis]) {+ const store = createStoreForClient(client);+ await expect(store.setCachedTools(tools('current'))).resolves.toBe(true);+ await expect(store.getCachedTools()).resolves.toEqual(tools('current'));+ await expect(store.getMCPToolsCacheGeneration({ userId, serverName })).resolves.toEqual(+ expect.any(String),+ );+ await expect(+ store.invalidateCachedTools({ invalidateGlobal: true }),+ ).resolves.toBeUndefined();+ expect(await client.get(rawKey(ToolCacheKeys.GLOBAL))).toBeNull();+ await redis.del(rawKey(ToolCacheKeys.MCP_SERVER_GENERATION(userId, serverName)));+ }+ } finally {+ restricted.disconnect();+ await Promise.all([...touched].map((key) => redis.del(key)));+ await Promise.all(nodes.map((node) => node.acl('DELUSER', username)));+ }+ },+ );++ it('rejects a delayed new write after an old replica replaces its legacy fence', async () => {+ const store = createStore(false);+ const entered = barrier();+ const resume = barrier();+ const write = store.runWithGlobalCacheLock(async () => {+ entered.release();+ await resume.ready;+ return store.setCachedToolsWithinGlobalLock(tools('stale'));+ });+ const settled = write.catch((error: Error) => error);+ await entered.ready;+ try {+ await redis.del(globalLock);+ await redis.del(legacyGlobalFence);+ expect(+ Number(+ await redis.eval(+ CLAIM_SCRIPT,+ 1,+ legacyGlobalFence,+ 'old-replica',+ String(Date.now() + 30_000),+ '1000',+ ),+ ),+ ).toBe(1);+ await redis.set(+ globalKey,+ JSON.stringify({ value: tools('replacement'), expires: Date.now() + 60_000 }),+ );+ } finally {+ resume.release();+ await expect(settled).resolves.toMatchObject({+ message: expect.stringContaining('ownership was lost'),+ });+ }+ await expect(store.getCachedTools()).resolves.toEqual(tools('replacement'));+ expect(await redis.get(legacyGlobalFence)).toBe('old-replica');+ });+});diff --git a/packages/api/src/mcp/catalog/store.ts b/packages/api/src/mcp/catalog/store.tsindex 80492245ff..f3950f8096 100644--- a/packages/api/src/mcp/catalog/store.ts+++ b/packages/api/src/mcp/catalog/store.ts@@ -1,4 +1,5 @@import { randomUUID } from 'crypto';+import calculateSlot from 'cluster-key-slot';import { logger } from '@librechat/data-schemas';import { CacheKeys, Time } from 'librechat-data-provider';import type { LCAvailableTools } from '../types';@@ -7,28 +8,84 @@ const GLOBAL_LOCK_TTL_MS = 30_000;const LOCK_FENCE_SAFETY_MS = 1_000;const LOCK_RETRY_MS = 25;const CACHE_ENTRY_VERSION = 1;+const LEGACY_FENCE_BARRIER = 'librechat:mcp-catalog-fence:v1';const GLOBAL_LOCK_KEY = `${CacheKeys.TOOL_CACHE}:tools:global:write-lock`;const RELEASE_LOCK_SCRIPT = `-if redis.call('GET', KEYS[1]) == ARGV[1] then- return redis.call('DEL', KEYS[1])+local released = 0+for _, key in ipairs(KEYS) do+ if redis.call('GET', key) == ARGV[1] then+ released = released + redis.call('DEL', key)+ endend-return 0+return released`;const CLAIM_OWNERSHIP_FENCE_SCRIPT = `local current = redis.call('GET', KEYS[1])if current and current ~= ARGV[1] then return 0 end+local owners = {}+if KEYS[2] then+ local legacy = redis.call('GET', KEYS[2])+ if legacy then+ local decoded, barrier = pcall(cjson.decode, legacy)+ if not decoded or type(barrier) ~= 'table' or barrier['protocol'] ~= '${LEGACY_FENCE_BARRIER}'+ or type(barrier['owners']) ~= 'table' then return 0 end+ owners = barrier['owners']+ end+endlocal redisTime = redis.call('TIME')local now = (tonumber(redisTime[1]) * 1000) + math.floor(tonumber(redisTime[2]) / 1000)-local ttl = tonumber(ARGV[2]) - now - tonumber(ARGV[3])+local deadline = tonumber(ARGV[2]) - tonumber(ARGV[3])+local ttl = deadline - nowif ttl <= 0 then return 0 end+local barrierTtl = ttl+for owner, expires in pairs(owners) do+ if expires <= now then owners[owner] = nil+ else barrierTtl = math.max(barrierTtl, expires - now) end+end+owners[ARGV[1]] = deadlineredis.call('PSETEX', KEYS[1], ttl, ARGV[1])+if KEYS[2] then+ redis.call('PSETEX', KEYS[2], barrierTtl, cjson.encode({protocol='${LEGACY_FENCE_BARRIER}', owners=owners}))+endreturn 1`;+const RELEASE_OWNERSHIP_FENCE_SCRIPT = `+local released = 0+if redis.call('GET', KEYS[1]) == ARGV[1] then+ released = redis.call('DEL', KEYS[1])+end+if not KEYS[2] then return released end+local decoded, barrier = pcall(cjson.decode, redis.call('GET', KEYS[2]) or '')+if not decoded or type(barrier) ~= 'table' or barrier['protocol'] ~= '${LEGACY_FENCE_BARRIER}'+ or type(barrier['owners']) ~= 'table' or not barrier['owners'][ARGV[1]] then return released end+barrier['owners'][ARGV[1]] = nil+local redisTime = redis.call('TIME')+local now = (tonumber(redisTime[1]) * 1000) + math.floor(tonumber(redisTime[2]) / 1000)+local ttl = 0+for owner, expires in pairs(barrier['owners']) do+ if expires <= now then barrier['owners'][owner] = nil+ else ttl = math.max(ttl, expires - now) end+end+if ttl == 0 then redis.call('DEL', KEYS[2])+else redis.call('PSETEX', KEYS[2], ttl, cjson.encode(barrier)) end+return released+`;++function legacyOwnershipCheck(index: number, rejected: number): string {+ return `+if KEYS[${index}] then+ local decoded, barrier = pcall(cjson.decode, redis.call('GET', KEYS[${index}]) or '')+ if not decoded or type(barrier) ~= 'table' or barrier['protocol'] ~= '${LEGACY_FENCE_BARRIER}'+ or type(barrier['owners']) ~= 'table' or not barrier['owners'][ARGV[1]] then return ${rejected} end+end`;+}+const CREATE_GENERATION_IF_OWNER_SCRIPT = `if redis.call('GET', KEYS[1]) ~= ARGV[1] then return -1 end+${legacyOwnershipCheck(3, -1)}if redis.call('EXISTS', KEYS[2]) == 1 then return 0 endredis.call('PSETEX', KEYS[2], ARGV[4], cjson.encode({value=ARGV[2], expires=tonumber(ARGV[3])}))return 1@@ -36,6 +93,7 @@ return 1const MIGRATE_USER_TOOLS_IF_OWNER_SCRIPT = `if redis.call('GET', KEYS[1]) ~= ARGV[1] then return -1 end+${legacyOwnershipCheck(5, -1)}if redis.call('EXISTS', KEYS[2]) == 1 or redis.call('EXISTS', KEYS[4]) == 1 then return 0 endlocal rawGeneration = redis.call('GET', KEYS[3])if rawGeneration then@@ -75,6 +133,7 @@ return 1const WRITE_GLOBAL_IF_OWNER_SCRIPT = `if redis.call('GET', KEYS[1]) ~= ARGV[1] then return 0 end+${legacyOwnershipCheck(3, 0)}local decoded, tools = pcall(cjson.decode, ARGV[2])if not decoded then return redis.error_reply('Invalid global tools catalog') endredis.call('PSETEX', KEYS[2], ARGV[4], cjson.encode({value=tools, expires=tonumber(ARGV[3])}))@@ -112,6 +171,7 @@ return 1const DELETE_GLOBAL_IF_OWNER_SCRIPT = `if redis.call('GET', KEYS[1]) ~= ARGV[1] then return 0 end+${legacyOwnershipCheck(3, 0)}redis.call('DEL', KEYS[2])return 1`;@@ -197,6 +257,13 @@ interface AppToolsEntry {interface OwnershipFence {key: string;token: string;+ legacyKey?: string;+}++function ownershipKeys(fence: OwnershipFence, ...keys: string[]): string[] {+ const result = [fence.key, ...keys];+ if (fence.legacyKey) result.push(fence.legacyKey);+ return result;}export interface MCPCatalogStore {@@ -293,6 +360,8 @@ export function createMCPCatalogStore(deps: CatalogStoreDeps): MCPCatalogStore {const appWriteQueues = new Map<string, Promise<void>>();let globalQueue = Promise.resolve();let globalLockToken: string | undefined;+ let globalLockFenceKey: string | undefined;+ let globalLockLegacyFenceKey: string | undefined;const sharedRedis = (): boolean =>deps.ioredisClient != null &&@@ -348,9 +417,25 @@ export function createMCPCatalogStore(deps: CatalogStoreDeps): MCPCatalogStore {const closingBrace = openingBrace >= 0 ? key.indexOf('}', openingBrace + 1) : -1;return closingBrace > openingBrace + 1 ? key.slice(openingBrace + 1, closingBrace) : key;};+ const prefixHasBraces = /[{}]/.test(rawKey(''));+ const ownershipFenceKey = (name: string, guardedKey: string): string => {+ let key = rawKey(`${name}:{${redisHashTag(guardedKey)}}`);+ if (!prefixHasBraces) return key;+ const slot = calculateSlot(guardedKey);+ if (calculateSlot(key) === slot) return key;+ // Literal prefix braces can override the synthetic tag. Reuse the guarded tag+ // when present; otherwise find a same-slot suffix without adding braces.+ let suffix = 0;+ key = `${guardedKey}:${name}`;+ while (calculateSlot(key) !== slot) {+ key = `${guardedKey}:${name}:${suffix++}`;+ }+ return key;+ };const globalCacheRawKey = rawKey(ToolCacheKeys.GLOBAL);/** Hashes to the same Redis Cluster slot as the unchanged legacy global catalog key. */- const globalFenceKey = `tools:global:write-fence:{${redisHashTag(globalCacheRawKey)}}`;+ const globalFenceKey = ownershipFenceKey('tools:global:write-fence', globalCacheRawKey);+ const legacyGlobalFenceKey = `tools:global:write-fence:{${redisHashTag(globalCacheRawKey)}}`;async function withRedisLock<T>(lockKey: string,@@ -389,6 +474,7 @@ export function createMCPCatalogStore(deps: CatalogStoreDeps): MCPCatalogStore {async function withOwnershipFence<T>(fenceKey: string,+ legacyFenceKey: string,token: string | undefined,leaseExpiresAt: number | undefined,operation: (fence?: OwnershipFence) => Promise<T>,@@ -400,19 +486,36 @@ export function createMCPCatalogStore(deps: CatalogStoreDeps): MCPCatalogStore {if (fenceTtl <= 0) {throw new Error('Tool cache lock expired before ownership could be fenced');}- const claimed = await deps.keyvRedisClient!.eval(CLAIM_OWNERSHIP_FENCE_SCRIPT, {- keys: [fenceKey],- arguments: [token, String(leaseExpiresAt), String(LOCK_FENCE_SAFETY_MS)],- });+ const redis = deps.keyvRedisClient!;+ const args = [token, String(leaseExpiresAt), String(LOCK_FENCE_SAFETY_MS)];+ let keys = [fenceKey, legacyFenceKey];+ let claimed: unknown;+ try {+ claimed = await redis.eval(CLAIM_OWNERSHIP_FENCE_SCRIPT, { keys, arguments: args });+ } catch (error) {+ if (!(error instanceof Error)) throw error;+ // Both rejections precede execution. Legacy writers cannot publish under+ // the same key ACL, or when their fence cannot share the catalog's slot.+ const denied = error.message.startsWith('NOPERM ');+ const crossSlot =+ calculateSlot(fenceKey) !== calculateSlot(legacyFenceKey) &&+ (error.message === 'Redis catalog script keys must share one cluster slot' ||+ error.message.startsWith('CROSSSLOT '));+ if (!denied && !crossSlot) throw error;+ keys = [fenceKey];+ claimed = await redis.eval(CLAIM_OWNERSHIP_FENCE_SCRIPT, { keys, arguments: args });+ }if (Number(claimed) !== 1) {throw new Error('Tool cache lock expired or was superseded before ownership could be fenced');}try {- return await operation({ key: fenceKey, token });+ // Ownership stays deployment-scoped. The shared barrier excludes old writers+ // without making current deployments contend with each other.+ return await operation({ key: fenceKey, token, legacyKey: keys[1] });} finally {try {- await deps.keyvRedisClient!.eval(RELEASE_LOCK_SCRIPT, {- keys: [fenceKey],+ await redis.eval(RELEASE_OWNERSHIP_FENCE_SCRIPT, {+ keys,arguments: [token],});} catch (error) {@@ -431,11 +534,12 @@ export function createMCPCatalogStore(deps: CatalogStoreDeps): MCPCatalogStore {const previous = userQueues.get(scope) ?? Promise.resolve();const lockKey = `${CacheKeys.TOOL_CACHE}:tools:mcp-write-lock:${encodeURIComponent(userId)}:${encodeURIComponent(serverName)}`;const generationRawKey = rawKey(ToolCacheKeys.MCP_SERVER_GENERATION(userId, serverName));- const fenceKey = `tools:mcp:write-fence:{${redisHashTag(generationRawKey)}}`;+ const fenceKey = ownershipFenceKey('tools:mcp:write-fence', generationRawKey);+ const legacyFenceKey = `tools:mcp:write-fence:{${redisHashTag(generationRawKey)}}`;const locked = () =>withRedisLock(lockKey, GLOBAL_LOCK_TTL_MS, (token, leaseExpiresAt) =>requireOwnershipFence- ? withOwnershipFence(fenceKey, token, leaseExpiresAt, operation)+ ? withOwnershipFence(fenceKey, legacyFenceKey, token, leaseExpiresAt, operation): operation(),);const result = previous.then(locked, locked);@@ -481,11 +585,14 @@ export function createMCPCatalogStore(deps: CatalogStoreDeps): MCPCatalogStore {if (!sharedRedis()) {return (await cache.set(ToolCacheKeys.GLOBAL, tools, ttl)) !== false;}- if (!globalLockToken) {+ if (!globalLockToken || !globalLockFenceKey) {throw new Error('Global tool cache write requires lock ownership');}const written = await deps.keyvRedisClient!.eval(WRITE_GLOBAL_IF_OWNER_SCRIPT, {- keys: [globalFenceKey, globalCacheRawKey],+ keys: ownershipKeys(+ { key: globalLockFenceKey, token: globalLockToken, legacyKey: globalLockLegacyFenceKey },+ globalCacheRawKey,+ ),arguments: [globalLockToken, JSON.stringify(tools), String(Date.now() + ttl), String(ttl)],});if (Number(written) !== 1) {@@ -499,11 +606,14 @@ export function createMCPCatalogStore(deps: CatalogStoreDeps): MCPCatalogStore {await cache.delete(ToolCacheKeys.GLOBAL);return;}- if (!globalLockToken) {+ if (!globalLockToken || !globalLockFenceKey) {throw new Error('Global tool cache invalidation requires lock ownership');}const deleted = await deps.keyvRedisClient!.eval(DELETE_GLOBAL_IF_OWNER_SCRIPT, {- keys: [globalFenceKey, globalCacheRawKey],+ keys: ownershipKeys(+ { key: globalLockFenceKey, token: globalLockToken, legacyKey: globalLockLegacyFenceKey },+ globalCacheRawKey,+ ),arguments: [globalLockToken],});if (Number(deleted) !== 1) {@@ -514,14 +624,24 @@ export function createMCPCatalogStore(deps: CatalogStoreDeps): MCPCatalogStore {function runWithGlobalCacheLock<T>(operation: () => Promise<T>): Promise<T> {const locked = () =>withRedisLock(GLOBAL_LOCK_KEY, GLOBAL_LOCK_TTL_MS, async (token, leaseExpiresAt) => {- return withOwnershipFence(globalFenceKey, token, leaseExpiresAt, async (fence) => {- globalLockToken = fence?.token;- try {- return await operation();- } finally {- globalLockToken = undefined;- }- });+ return withOwnershipFence(+ globalFenceKey,+ legacyGlobalFenceKey,+ token,+ leaseExpiresAt,+ async (fence) => {+ globalLockToken = fence?.token;+ globalLockFenceKey = fence?.key;+ globalLockLegacyFenceKey = fence?.legacyKey;+ try {+ return await operation();+ } finally {+ globalLockToken = undefined;+ globalLockFenceKey = undefined;+ globalLockLegacyFenceKey = undefined;+ }+ },+ );});const result = globalQueue.then(locked, locked);globalQueue = result.then(@@ -582,7 +702,12 @@ export function createMCPCatalogStore(deps: CatalogStoreDeps): MCPCatalogStore {}const toolsTtl = Time.TWELVE_HOURS;const written = await deps.keyvRedisClient!.eval(MIGRATE_USER_TOOLS_IF_OWNER_SCRIPT, {- keys: [fence.key, rawKey(legacyFenceKey), rawKey(generationKey), rawKey(toolsKey)],+ keys: ownershipKeys(+ fence,+ rawKey(legacyFenceKey),+ rawKey(generationKey),+ rawKey(toolsKey),+ ),arguments: [fence.token,publicationGeneration,@@ -680,7 +805,7 @@ export function createMCPCatalogStore(deps: CatalogStoreDeps): MCPCatalogStore {throw new Error('Tool publication generation creation requires lock ownership');}const created = await deps.keyvRedisClient!.eval(CREATE_GENERATION_IF_OWNER_SCRIPT, {- keys: [fence.key, rawKey(key)],+ keys: ownershipKeys(fence, rawKey(key)),arguments: [fence.token,generation,