Skip to content

Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active

👑 fix: Elect a Single Expired-File Sweeper (#15909)

Danny Avila7e05c4848a70
diff --git a/api/server/experimental.js b/api/server/experimental.js
index b77d5c33fd..61f1842ff3 100644
--- a/api/server/experimental.js
+++ b/api/server/experimental.js
@@ -53,6 +53,8 @@ const {
startCodeEnvironmentLifecycleReconciler,
waitForKeyvRedisClient,
createCodeApiUploadRegistry,
+ cacheConfig,
+ createClusteredFileSweep,
} = require('@librechat/api');
const { connectDb, indexSync } = require('~/db');
const initializeOAuthReconnectManager = require('./services/initializeOAuthReconnectManager');
@@ -406,14 +408,7 @@ if (cluster.isMaster) {
};
// Tear down stream resources before shared caches and telemetry exporters shut down.
registerShutdownTask('generation job manager', destroyGenerationJobManager, { priority: 100 });
- /**
- * The master may assign the sweep worker before or after this worker has
- * loaded app config. These flags join the IPC assignment with config
- * availability and ensure the background sweep starts only once.
- */
- let shouldStartExpiredFileSweep = false;
- let expiredFileSweepOptions = null;
- let expiredFileSweepStarted = false;
+ const expiredFileSweep = createClusteredFileSweep(cacheConfig.USE_REDIS, startExpiredFileSweep);
const SCHEDULE_ENGINE_OPTIONAL_METHODS = new Set(['GET', 'HEAD', 'OPTIONS', 'DELETE']);
const rejectScheduleWritesUntilReady = (req, res, next) => {
@@ -426,15 +421,6 @@ if (cluster.isMaster) {
});
};
- const startExpiredFileSweepOnce = () => {
- if (!shouldStartExpiredFileSweep || expiredFileSweepStarted || !expiredFileSweepOptions) {
- return;
- }
-
- expiredFileSweepStarted = true;
- startExpiredFileSweep(expiredFileSweepOptions);
- };
-
/** Handle inter-process messages from master */
process.on('message', (msg) => {
if (msg != null && msg.type === 'cluster-shutdown' && Number.isFinite(msg.deadlineAt)) {
@@ -452,12 +438,10 @@ if (cluster.isMaster) {
});
process.on('message', (msg) => {
if (msg.type === 'file-retention-sweep-worker') {
- shouldStartExpiredFileSweep = true;
logger.info(wrapLogMessage(`Worker ${process.pid} is assigned file-retention sweep`));
- startExpiredFileSweepOnce();
+ expiredFileSweep.assign();
}
});
-
const startServer = async () => {
logger.info(`Worker ${process.pid} initializing...`);
@@ -529,8 +513,7 @@ if (cluster.isMaster) {
await loadToolApprovalHooks(toolApproval?.enabled ? toolApproval.hooks : undefined, {
basePath: path.resolve(__dirname, '../..'),
});
- expiredFileSweepOptions = { appConfig, loadAppConfig: getAppConfig };
- startExpiredFileSweepOnce();
+ expiredFileSweep.configure({ appConfig, loadAppConfig: getAppConfig });
await runAsSystem(async () => {
await performStartupChecks(appConfig);
await updateInterfacePerms({ appConfig, getRoleByName, updateAccessPermissions });
diff --git a/api/server/services/Files/process.js b/api/server/services/Files/process.js
index 3e216c1aad..0a5705e60c 100644
--- a/api/server/services/Files/process.js
+++ b/api/server/services/Files/process.js
@@ -48,6 +48,7 @@ const {
createCodeApiRateLimitBudget,
getCodeApiUploadOptions,
withCodeApiUploadRecovery,
+ isLeader,
} = require('@librechat/api');
const {
convertImage,
@@ -371,6 +372,7 @@ function startExpiredFileSweep(options = {}) {
return startExpiredFileSweepWithDeps(options, {
sweepExpiredFiles,
runAsSystem,
+ isLeader,
logger,
});
}
diff --git a/api/server/services/Files/process.spec.js b/api/server/services/Files/process.spec.js
index 8ac28b207b..fbf1aca9b4 100644
--- a/api/server/services/Files/process.spec.js
+++ b/api/server/services/Files/process.spec.js
@@ -168,6 +168,7 @@ jest.mock('@librechat/api', () => {
}),
sweepExpiredFiles: jest.fn().mockResolvedValue({ scanned: 0, deleted: 0, failed: 0 }),
startExpiredFileSweep: jest.fn().mockReturnValue('sweep-interval'),
+ isLeader: jest.fn().mockResolvedValue(true),
};
});
@@ -2750,6 +2751,7 @@ describe('startExpiredFileSweep', () => {
expect.objectContaining({
sweepExpiredFiles: expect.any(Function),
runAsSystem: expect.any(Function),
+ isLeader: expect.any(Function),
logger: expect.objectContaining({
error: expect.any(Function),
info: expect.any(Function),
diff --git a/packages/api/src/cluster/LeaderElection.spec.ts b/packages/api/src/cluster/LeaderElection.spec.ts
new file mode 100644
index 0000000000..6a9c1ec3ce
--- /dev/null
+++ b/packages/api/src/cluster/LeaderElection.spec.ts
@@ -0,0 +1,61 @@
+jest.mock('~/cache/cacheConfig', () => ({ cacheConfig: { USE_REDIS: true } }));
+jest.mock('~/cache/redisClients', () => ({
+ ioredisClient: { get: jest.fn(), set: jest.fn(), eval: jest.fn() },
+}));
+jest.mock('~/cache/redisTelemetry', () => ({
+ instrumentIORedisClient: (client: unknown) => client,
+ RedisUseCases: { LEADER_ELECTION: 'leader' },
+}));
+
+import { ioredisClient } from '~/cache/redisClients';
+import { LeaderElection } from './LeaderElection';
+
+describe('concurrent leader ownership checks', () => {
+ const election = new LeaderElection();
+ const redis = ioredisClient as jest.Mocked<NonNullable<typeof ioredisClient>>;
+
+ beforeEach(() => {
+ jest.useFakeTimers();
+ jest.clearAllMocks();
+ jest.spyOn(Math, 'random').mockReturnValue(0);
+ });
+
+ afterEach(() => {
+ election.clearRefreshTimer();
+ jest.restoreAllMocks();
+ jest.useRealTimers();
+ });
+
+ it('reports leadership to both startup callers after one wins SET NX', async () => {
+ redis.get
+ .mockResolvedValueOnce(null)
+ .mockResolvedValueOnce(null)
+ .mockResolvedValue(election.UUID);
+ redis.set.mockResolvedValueOnce('OK').mockResolvedValueOnce(null);
+ const result = Promise.all([election.isLeader(), election.isLeader()]);
+ await jest.advanceTimersByTimeAsync(0);
+ expect(await result).toEqual([true, true]);
+ expect(redis.set).toHaveBeenCalledTimes(2);
+ expect(jest.getTimerCount()).toBe(1);
+ });
+
+ it.each(['another-process', null])(
+ 'does not claim another or missing lease: %s',
+ async (owner) => {
+ redis.get.mockResolvedValueOnce(null).mockResolvedValue(owner);
+ redis.set.mockResolvedValue(null);
+ const result = election.isLeader();
+ await jest.advanceTimersByTimeAsync(0);
+ expect(await result).toBe(false);
+ expect(jest.getTimerCount()).toBe(0);
+ },
+ );
+
+ it('does not revive ownership without a renewal timer', async () => {
+ redis.get.mockResolvedValueOnce(null).mockResolvedValue(election.UUID);
+ redis.set.mockResolvedValue(null);
+ const result = election.isLeader();
+ await jest.advanceTimersByTimeAsync(0);
+ expect(await result).toBe(false);
+ });
+});
diff --git a/packages/api/src/cluster/LeaderElection.ts b/packages/api/src/cluster/LeaderElection.ts
index 9a9a3be716..f4ed7c9539 100644
--- a/packages/api/src/cluster/LeaderElection.ts
+++ b/packages/api/src/cluster/LeaderElection.ts
@@ -136,7 +136,9 @@ export class LeaderElection {
'NX',
);
- if (result !== 'OK') return false;
+ if (result !== 'OK') {
+ return (await LeaderElection.getLeaderUUID()) === this.UUID && this.refreshTimer != null;
+ }
this.clearRefreshTimer();
this.refreshTimer = setInterval(async () => {
diff --git a/packages/api/src/cluster/__tests__/LeaderElection.cache_integration.spec.ts b/packages/api/src/cluster/__tests__/LeaderElection.cache_integration.spec.ts
index 85ace5774b..3886629577 100644
--- a/packages/api/src/cluster/__tests__/LeaderElection.cache_integration.spec.ts
+++ b/packages/api/src/cluster/__tests__/LeaderElection.cache_integration.spec.ts
@@ -145,16 +145,16 @@ describe('LeaderElection with Redis', () => {
});
describe('Test Case 3: Stress testing', () => {
- it('should ensure only one instance becomes leader even when multiple instances call electSelf() at once', async () => {
- // Create 10 instances
+ it('reports ownership to concurrent callers of the same singleton', async () => {
+ // The constructor returns the singleton, not ten independent replicas.
instances = Array.from({ length: 10 }, () => new LeaderElection());
// Call electSelf on all instances in parallel
const results = await Promise.all(instances.map((instance) => instance['electSelf']()));
- // Verify only one returned true
+ // All callers observe the same process owning the lease.
const successCount = results.filter((success) => success).length;
- expect(successCount).toBe(1);
+ expect(successCount).toBe(10);
// Find the winning instance
const winnerInstance = instances.find((_, index) => results[index]);
diff --git a/packages/api/src/files/sweep.spec.ts b/packages/api/src/files/sweep.spec.ts
index 49609bd2cc..1e5c0e5042 100644
--- a/packages/api/src/files/sweep.spec.ts
+++ b/packages/api/src/files/sweep.spec.ts
@@ -3,6 +3,7 @@ import type { AppConfig } from '@librechat/data-schemas';
import {
sweepExpiredFiles,
startExpiredFileSweep,
+ createClusteredFileSweep,
getExpiredFileRetryDelay,
getFileRetentionMaxAttempts,
getFileRetentionSweepInterval,
@@ -308,6 +309,7 @@ describe('expired file sweep helpers', () => {
{
sweepExpiredFiles: jest.fn(),
runAsSystem: jest.fn((fn) => fn()),
+ isLeader: jest.fn().mockResolvedValue(true),
logger,
},
);
@@ -317,4 +319,107 @@ describe('expired file sweep helpers', () => {
'[sweepExpiredFiles] Disabled by FILE_RETENTION_SWEEP_INTERVAL_MS=0',
);
});
+
+ it.each([true, false])('joins local assignment and config in either order: %s', (assignFirst) => {
+ const start = jest.fn().mockReturnValue(null);
+ const coordinator = createClusteredFileSweep(false, start);
+ const options = { appConfig: {} as AppConfig };
+ if (assignFirst) coordinator.assign();
+ else coordinator.configure(options);
+ expect(start).not.toHaveBeenCalled();
+ if (assignFirst) coordinator.configure(options);
+ else coordinator.assign();
+ coordinator.assign();
+ coordinator.configure(options);
+ expect(start).toHaveBeenCalledTimes(1);
+ expect(start).toHaveBeenCalledWith(options);
+ });
+
+ it('registers distributed candidates without primary assignment', () => {
+ const start = jest.fn().mockReturnValue(null);
+ const coordinator = createClusteredFileSweep(true, start);
+ expect(start).not.toHaveBeenCalled();
+ coordinator.configure({ appConfig: {} as AppConfig });
+ coordinator.assign();
+ expect(start).toHaveBeenCalledTimes(1);
+ });
+
+ it('does not overlap a slow sweep with the next interval', async () => {
+ jest.useFakeTimers();
+ process.env.FILE_RETENTION_SWEEP_INTERVAL_MS = '1000';
+ let finish!: () => void;
+ const sweep = jest.fn().mockImplementation(
+ () =>
+ new Promise<void>((resolve) => {
+ finish = resolve;
+ }),
+ );
+ const isLeader = jest.fn().mockResolvedValue(true);
+ const interval = startExpiredFileSweep(
+ {},
+ {
+ sweepExpiredFiles: sweep,
+ isLeader,
+ runAsSystem: (fn) => fn(),
+ logger,
+ },
+ );
+ await jest.advanceTimersByTimeAsync(3000);
+ expect(sweep).toHaveBeenCalledTimes(1);
+ expect(isLeader).toHaveBeenCalledTimes(1);
+ finish();
+ await jest.advanceTimersByTimeAsync(1000);
+ expect(sweep).toHaveBeenCalledTimes(2);
+ finish();
+ clearInterval(interval ?? undefined);
+ });
+
+ it('fails closed on leadership errors and retries on the next interval', async () => {
+ jest.useFakeTimers();
+ process.env.FILE_RETENTION_SWEEP_INTERVAL_MS = '1000';
+ const sweep = jest.fn().mockResolvedValue({ scanned: 0, deleted: 0, failed: 0 });
+ const isLeader = jest
+ .fn()
+ .mockRejectedValueOnce(new Error('redis down'))
+ .mockResolvedValue(true);
+ const interval = startExpiredFileSweep(
+ {},
+ {
+ sweepExpiredFiles: sweep,
+ isLeader,
+ runAsSystem: (fn) => fn(),
+ logger,
+ },
+ );
+ await jest.advanceTimersByTimeAsync(0);
+ expect(sweep).not.toHaveBeenCalled();
+ await jest.advanceTimersByTimeAsync(1000);
+ expect(sweep).toHaveBeenCalledTimes(1);
+ clearInterval(interval ?? undefined);
+ });
+
+ it('runs the sweep only on the elected leader and permits follower takeover', async () => {
+ jest.useFakeTimers();
+ process.env.FILE_RETENTION_SWEEP_INTERVAL_MS = '1000';
+ const sweep = jest.fn().mockResolvedValue({ scanned: 0, deleted: 0, failed: 0 });
+ const runAsSystem = jest.fn((fn) => fn());
+ const isLeader = jest.fn().mockResolvedValueOnce(false).mockResolvedValueOnce(true);
+
+ const interval = startExpiredFileSweep(
+ { appConfig: {} as AppConfig },
+ { sweepExpiredFiles: sweep, runAsSystem, isLeader, logger },
+ );
+ await jest.advanceTimersByTimeAsync(0);
+
+ expect(isLeader).toHaveBeenCalledTimes(1);
+ expect(runAsSystem).not.toHaveBeenCalled();
+ expect(sweep).not.toHaveBeenCalled();
+
+ await jest.advanceTimersByTimeAsync(1000);
+
+ expect(isLeader).toHaveBeenCalledTimes(2);
+ expect(runAsSystem).toHaveBeenCalledTimes(1);
+ expect(sweep).toHaveBeenCalledTimes(1);
+ clearInterval(interval ?? undefined);
+ });
});
diff --git a/packages/api/src/files/sweep.ts b/packages/api/src/files/sweep.ts
index e20e5afdd2..7b736e9d23 100644
--- a/packages/api/src/files/sweep.ts
+++ b/packages/api/src/files/sweep.ts
@@ -63,6 +63,7 @@ type SweepDependencies = {
type StartSweepDependencies = {
sweepExpiredFiles: (options?: ExpiredFileSweepOptions) => Promise<ExpiredFileSweepResult>;
runAsSystem: <T>(fn: () => Promise<T>) => Promise<T>;
+ isLeader: () => Promise<boolean>;
logger: SweepLogger;
};
@@ -368,9 +369,34 @@ export async function sweepExpiredFiles(
return { scanned: files.length, deleted, failed };
}
+/** Joins configuration readiness with distributed or primary-assigned ownership. */
+export function createClusteredFileSweep(
+ distributed: boolean,
+ start: (options: ExpiredFileSweepOptions) => NodeJS.Timeout | null,
+): { configure: (options: ExpiredFileSweepOptions) => void; assign: () => void } {
+ let eligible = distributed;
+ let started = false;
+ let options: ExpiredFileSweepOptions | undefined;
+ const maybeStart = () => {
+ if (!eligible || started || options == null) return;
+ started = true;
+ start(options);
+ };
+ return {
+ configure: (value) => {
+ options = value;
+ maybeStart();
+ },
+ assign: () => {
+ eligible = true;
+ maybeStart();
+ },
+ };
+}
+
export function startExpiredFileSweep(
options: ExpiredFileSweepOptions | undefined = {},
- { sweepExpiredFiles, runAsSystem, logger }: StartSweepDependencies,
+ { sweepExpiredFiles, runAsSystem, isLeader, logger }: StartSweepDependencies,
): NodeJS.Timeout | null {
const intervalMs = getFileRetentionSweepInterval();
if (intervalMs === 0) {
@@ -386,6 +412,9 @@ export function startExpiredFileSweep(
isSweeping = true;
try {
+ if (!(await isLeader())) {
+ return;
+ }
await runAsSystem(() => sweepExpiredFiles(options));
} catch (error) {
logger.error('[sweepExpiredFiles] Background sweep failed:', error);
diff --git a/packages/api/src/index.ts b/packages/api/src/index.ts
index aefdd8e179..a1bb4dd789 100644
--- a/packages/api/src/index.ts
+++ b/packages/api/src/index.ts
@@ -106,6 +106,8 @@ export * from './traces';
export * from './insights';
/* Cache */
export * from './cache';
+/* Cluster */
+export * from './cluster';
/* Shared Links */
export * from './shared-links/access';
export * from './shared-links/service';