Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active
TypeScript aianthropicartifactsawsazurechatgptchatgpt-cloneclaudeclonedeepseekgeminigooglegpt-5librechatmcpo1openairesponses-apivisionwebui
👑 fix: Elect a Single Expired-File Sweeper (#15909)
Danny Avila7e05c4848a70
diff --git a/api/server/experimental.js b/api/server/experimental.jsindex b77d5c33fd..61f1842ff3 100644--- a/api/server/experimental.js+++ b/api/server/experimental.js@@ -53,6 +53,8 @@ const {startCodeEnvironmentLifecycleReconciler,waitForKeyvRedisClient,createCodeApiUploadRegistry,+ cacheConfig,+ createClusteredFileSweep,} = require('@librechat/api');const { connectDb, indexSync } = require('~/db');const initializeOAuthReconnectManager = require('./services/initializeOAuthReconnectManager');@@ -406,14 +408,7 @@ if (cluster.isMaster) {};// Tear down stream resources before shared caches and telemetry exporters shut down.registerShutdownTask('generation job manager', destroyGenerationJobManager, { priority: 100 });- /**- * The master may assign the sweep worker before or after this worker has- * loaded app config. These flags join the IPC assignment with config- * availability and ensure the background sweep starts only once.- */- let shouldStartExpiredFileSweep = false;- let expiredFileSweepOptions = null;- let expiredFileSweepStarted = false;+ const expiredFileSweep = createClusteredFileSweep(cacheConfig.USE_REDIS, startExpiredFileSweep);const SCHEDULE_ENGINE_OPTIONAL_METHODS = new Set(['GET', 'HEAD', 'OPTIONS', 'DELETE']);const rejectScheduleWritesUntilReady = (req, res, next) => {@@ -426,15 +421,6 @@ if (cluster.isMaster) {});};- const startExpiredFileSweepOnce = () => {- if (!shouldStartExpiredFileSweep || expiredFileSweepStarted || !expiredFileSweepOptions) {- return;- }-- expiredFileSweepStarted = true;- startExpiredFileSweep(expiredFileSweepOptions);- };-/** Handle inter-process messages from master */process.on('message', (msg) => {if (msg != null && msg.type === 'cluster-shutdown' && Number.isFinite(msg.deadlineAt)) {@@ -452,12 +438,10 @@ if (cluster.isMaster) {});process.on('message', (msg) => {if (msg.type === 'file-retention-sweep-worker') {- shouldStartExpiredFileSweep = true;logger.info(wrapLogMessage(`Worker ${process.pid} is assigned file-retention sweep`));- startExpiredFileSweepOnce();+ expiredFileSweep.assign();}});-const startServer = async () => {logger.info(`Worker ${process.pid} initializing...`);@@ -529,8 +513,7 @@ if (cluster.isMaster) {await loadToolApprovalHooks(toolApproval?.enabled ? toolApproval.hooks : undefined, {basePath: path.resolve(__dirname, '../..'),});- expiredFileSweepOptions = { appConfig, loadAppConfig: getAppConfig };- startExpiredFileSweepOnce();+ expiredFileSweep.configure({ appConfig, loadAppConfig: getAppConfig });await runAsSystem(async () => {await performStartupChecks(appConfig);await updateInterfacePerms({ appConfig, getRoleByName, updateAccessPermissions });diff --git a/api/server/services/Files/process.js b/api/server/services/Files/process.jsindex 3e216c1aad..0a5705e60c 100644--- a/api/server/services/Files/process.js+++ b/api/server/services/Files/process.js@@ -48,6 +48,7 @@ const {createCodeApiRateLimitBudget,getCodeApiUploadOptions,withCodeApiUploadRecovery,+ isLeader,} = require('@librechat/api');const {convertImage,@@ -371,6 +372,7 @@ function startExpiredFileSweep(options = {}) {return startExpiredFileSweepWithDeps(options, {sweepExpiredFiles,runAsSystem,+ isLeader,logger,});}diff --git a/api/server/services/Files/process.spec.js b/api/server/services/Files/process.spec.jsindex 8ac28b207b..fbf1aca9b4 100644--- a/api/server/services/Files/process.spec.js+++ b/api/server/services/Files/process.spec.js@@ -168,6 +168,7 @@ jest.mock('@librechat/api', () => {}),sweepExpiredFiles: jest.fn().mockResolvedValue({ scanned: 0, deleted: 0, failed: 0 }),startExpiredFileSweep: jest.fn().mockReturnValue('sweep-interval'),+ isLeader: jest.fn().mockResolvedValue(true),};});@@ -2750,6 +2751,7 @@ describe('startExpiredFileSweep', () => {expect.objectContaining({sweepExpiredFiles: expect.any(Function),runAsSystem: expect.any(Function),+ isLeader: expect.any(Function),logger: expect.objectContaining({error: expect.any(Function),info: expect.any(Function),diff --git a/packages/api/src/cluster/LeaderElection.spec.ts b/packages/api/src/cluster/LeaderElection.spec.tsnew file mode 100644index 0000000000..6a9c1ec3ce--- /dev/null+++ b/packages/api/src/cluster/LeaderElection.spec.ts@@ -0,0 +1,61 @@+jest.mock('~/cache/cacheConfig', () => ({ cacheConfig: { USE_REDIS: true } }));+jest.mock('~/cache/redisClients', () => ({+ ioredisClient: { get: jest.fn(), set: jest.fn(), eval: jest.fn() },+}));+jest.mock('~/cache/redisTelemetry', () => ({+ instrumentIORedisClient: (client: unknown) => client,+ RedisUseCases: { LEADER_ELECTION: 'leader' },+}));++import { ioredisClient } from '~/cache/redisClients';+import { LeaderElection } from './LeaderElection';++describe('concurrent leader ownership checks', () => {+ const election = new LeaderElection();+ const redis = ioredisClient as jest.Mocked<NonNullable<typeof ioredisClient>>;++ beforeEach(() => {+ jest.useFakeTimers();+ jest.clearAllMocks();+ jest.spyOn(Math, 'random').mockReturnValue(0);+ });++ afterEach(() => {+ election.clearRefreshTimer();+ jest.restoreAllMocks();+ jest.useRealTimers();+ });++ it('reports leadership to both startup callers after one wins SET NX', async () => {+ redis.get+ .mockResolvedValueOnce(null)+ .mockResolvedValueOnce(null)+ .mockResolvedValue(election.UUID);+ redis.set.mockResolvedValueOnce('OK').mockResolvedValueOnce(null);+ const result = Promise.all([election.isLeader(), election.isLeader()]);+ await jest.advanceTimersByTimeAsync(0);+ expect(await result).toEqual([true, true]);+ expect(redis.set).toHaveBeenCalledTimes(2);+ expect(jest.getTimerCount()).toBe(1);+ });++ it.each(['another-process', null])(+ 'does not claim another or missing lease: %s',+ async (owner) => {+ redis.get.mockResolvedValueOnce(null).mockResolvedValue(owner);+ redis.set.mockResolvedValue(null);+ const result = election.isLeader();+ await jest.advanceTimersByTimeAsync(0);+ expect(await result).toBe(false);+ expect(jest.getTimerCount()).toBe(0);+ },+ );++ it('does not revive ownership without a renewal timer', async () => {+ redis.get.mockResolvedValueOnce(null).mockResolvedValue(election.UUID);+ redis.set.mockResolvedValue(null);+ const result = election.isLeader();+ await jest.advanceTimersByTimeAsync(0);+ expect(await result).toBe(false);+ });+});diff --git a/packages/api/src/cluster/LeaderElection.ts b/packages/api/src/cluster/LeaderElection.tsindex 9a9a3be716..f4ed7c9539 100644--- a/packages/api/src/cluster/LeaderElection.ts+++ b/packages/api/src/cluster/LeaderElection.ts@@ -136,7 +136,9 @@ export class LeaderElection {'NX',);- if (result !== 'OK') return false;+ if (result !== 'OK') {+ return (await LeaderElection.getLeaderUUID()) === this.UUID && this.refreshTimer != null;+ }this.clearRefreshTimer();this.refreshTimer = setInterval(async () => {diff --git a/packages/api/src/cluster/__tests__/LeaderElection.cache_integration.spec.ts b/packages/api/src/cluster/__tests__/LeaderElection.cache_integration.spec.tsindex 85ace5774b..3886629577 100644--- a/packages/api/src/cluster/__tests__/LeaderElection.cache_integration.spec.ts+++ b/packages/api/src/cluster/__tests__/LeaderElection.cache_integration.spec.ts@@ -145,16 +145,16 @@ describe('LeaderElection with Redis', () => {});describe('Test Case 3: Stress testing', () => {- it('should ensure only one instance becomes leader even when multiple instances call electSelf() at once', async () => {- // Create 10 instances+ it('reports ownership to concurrent callers of the same singleton', async () => {+ // The constructor returns the singleton, not ten independent replicas.instances = Array.from({ length: 10 }, () => new LeaderElection());// Call electSelf on all instances in parallelconst results = await Promise.all(instances.map((instance) => instance['electSelf']()));- // Verify only one returned true+ // All callers observe the same process owning the lease.const successCount = results.filter((success) => success).length;- expect(successCount).toBe(1);+ expect(successCount).toBe(10);// Find the winning instanceconst winnerInstance = instances.find((_, index) => results[index]);diff --git a/packages/api/src/files/sweep.spec.ts b/packages/api/src/files/sweep.spec.tsindex 49609bd2cc..1e5c0e5042 100644--- a/packages/api/src/files/sweep.spec.ts+++ b/packages/api/src/files/sweep.spec.ts@@ -3,6 +3,7 @@ import type { AppConfig } from '@librechat/data-schemas';import {sweepExpiredFiles,startExpiredFileSweep,+ createClusteredFileSweep,getExpiredFileRetryDelay,getFileRetentionMaxAttempts,getFileRetentionSweepInterval,@@ -308,6 +309,7 @@ describe('expired file sweep helpers', () => {{sweepExpiredFiles: jest.fn(),runAsSystem: jest.fn((fn) => fn()),+ isLeader: jest.fn().mockResolvedValue(true),logger,},);@@ -317,4 +319,107 @@ describe('expired file sweep helpers', () => {'[sweepExpiredFiles] Disabled by FILE_RETENTION_SWEEP_INTERVAL_MS=0',);});++ it.each([true, false])('joins local assignment and config in either order: %s', (assignFirst) => {+ const start = jest.fn().mockReturnValue(null);+ const coordinator = createClusteredFileSweep(false, start);+ const options = { appConfig: {} as AppConfig };+ if (assignFirst) coordinator.assign();+ else coordinator.configure(options);+ expect(start).not.toHaveBeenCalled();+ if (assignFirst) coordinator.configure(options);+ else coordinator.assign();+ coordinator.assign();+ coordinator.configure(options);+ expect(start).toHaveBeenCalledTimes(1);+ expect(start).toHaveBeenCalledWith(options);+ });++ it('registers distributed candidates without primary assignment', () => {+ const start = jest.fn().mockReturnValue(null);+ const coordinator = createClusteredFileSweep(true, start);+ expect(start).not.toHaveBeenCalled();+ coordinator.configure({ appConfig: {} as AppConfig });+ coordinator.assign();+ expect(start).toHaveBeenCalledTimes(1);+ });++ it('does not overlap a slow sweep with the next interval', async () => {+ jest.useFakeTimers();+ process.env.FILE_RETENTION_SWEEP_INTERVAL_MS = '1000';+ let finish!: () => void;+ const sweep = jest.fn().mockImplementation(+ () =>+ new Promise<void>((resolve) => {+ finish = resolve;+ }),+ );+ const isLeader = jest.fn().mockResolvedValue(true);+ const interval = startExpiredFileSweep(+ {},+ {+ sweepExpiredFiles: sweep,+ isLeader,+ runAsSystem: (fn) => fn(),+ logger,+ },+ );+ await jest.advanceTimersByTimeAsync(3000);+ expect(sweep).toHaveBeenCalledTimes(1);+ expect(isLeader).toHaveBeenCalledTimes(1);+ finish();+ await jest.advanceTimersByTimeAsync(1000);+ expect(sweep).toHaveBeenCalledTimes(2);+ finish();+ clearInterval(interval ?? undefined);+ });++ it('fails closed on leadership errors and retries on the next interval', async () => {+ jest.useFakeTimers();+ process.env.FILE_RETENTION_SWEEP_INTERVAL_MS = '1000';+ const sweep = jest.fn().mockResolvedValue({ scanned: 0, deleted: 0, failed: 0 });+ const isLeader = jest+ .fn()+ .mockRejectedValueOnce(new Error('redis down'))+ .mockResolvedValue(true);+ const interval = startExpiredFileSweep(+ {},+ {+ sweepExpiredFiles: sweep,+ isLeader,+ runAsSystem: (fn) => fn(),+ logger,+ },+ );+ await jest.advanceTimersByTimeAsync(0);+ expect(sweep).not.toHaveBeenCalled();+ await jest.advanceTimersByTimeAsync(1000);+ expect(sweep).toHaveBeenCalledTimes(1);+ clearInterval(interval ?? undefined);+ });++ it('runs the sweep only on the elected leader and permits follower takeover', async () => {+ jest.useFakeTimers();+ process.env.FILE_RETENTION_SWEEP_INTERVAL_MS = '1000';+ const sweep = jest.fn().mockResolvedValue({ scanned: 0, deleted: 0, failed: 0 });+ const runAsSystem = jest.fn((fn) => fn());+ const isLeader = jest.fn().mockResolvedValueOnce(false).mockResolvedValueOnce(true);++ const interval = startExpiredFileSweep(+ { appConfig: {} as AppConfig },+ { sweepExpiredFiles: sweep, runAsSystem, isLeader, logger },+ );+ await jest.advanceTimersByTimeAsync(0);++ expect(isLeader).toHaveBeenCalledTimes(1);+ expect(runAsSystem).not.toHaveBeenCalled();+ expect(sweep).not.toHaveBeenCalled();++ await jest.advanceTimersByTimeAsync(1000);++ expect(isLeader).toHaveBeenCalledTimes(2);+ expect(runAsSystem).toHaveBeenCalledTimes(1);+ expect(sweep).toHaveBeenCalledTimes(1);+ clearInterval(interval ?? undefined);+ });});diff --git a/packages/api/src/files/sweep.ts b/packages/api/src/files/sweep.tsindex e20e5afdd2..7b736e9d23 100644--- a/packages/api/src/files/sweep.ts+++ b/packages/api/src/files/sweep.ts@@ -63,6 +63,7 @@ type SweepDependencies = {type StartSweepDependencies = {sweepExpiredFiles: (options?: ExpiredFileSweepOptions) => Promise<ExpiredFileSweepResult>;runAsSystem: <T>(fn: () => Promise<T>) => Promise<T>;+ isLeader: () => Promise<boolean>;logger: SweepLogger;};@@ -368,9 +369,34 @@ export async function sweepExpiredFiles(return { scanned: files.length, deleted, failed };}+/** Joins configuration readiness with distributed or primary-assigned ownership. */+export function createClusteredFileSweep(+ distributed: boolean,+ start: (options: ExpiredFileSweepOptions) => NodeJS.Timeout | null,+): { configure: (options: ExpiredFileSweepOptions) => void; assign: () => void } {+ let eligible = distributed;+ let started = false;+ let options: ExpiredFileSweepOptions | undefined;+ const maybeStart = () => {+ if (!eligible || started || options == null) return;+ started = true;+ start(options);+ };+ return {+ configure: (value) => {+ options = value;+ maybeStart();+ },+ assign: () => {+ eligible = true;+ maybeStart();+ },+ };+}+export function startExpiredFileSweep(options: ExpiredFileSweepOptions | undefined = {},- { sweepExpiredFiles, runAsSystem, logger }: StartSweepDependencies,+ { sweepExpiredFiles, runAsSystem, isLeader, logger }: StartSweepDependencies,): NodeJS.Timeout | null {const intervalMs = getFileRetentionSweepInterval();if (intervalMs === 0) {@@ -386,6 +412,9 @@ export function startExpiredFileSweep(isSweeping = true;try {+ if (!(await isLeader())) {+ return;+ }await runAsSystem(() => sweepExpiredFiles(options));} catch (error) {logger.error('[sweepExpiredFiles] Background sweep failed:', error);diff --git a/packages/api/src/index.ts b/packages/api/src/index.tsindex aefdd8e179..a1bb4dd789 100644--- a/packages/api/src/index.ts+++ b/packages/api/src/index.ts@@ -106,6 +106,8 @@ export * from './traces';export * from './insights';/* Cache */export * from './cache';+/* Cluster */+export * from './cluster';/* Shared Links */export * from './shared-links/access';export * from './shared-links/service';