Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(redis): disconnect per-queue clients on shutdown so the worker process exits (#17097)

pilsoo0169d3489e52
diff --git a/packages/shared/src/server/redis/redis.ts b/packages/shared/src/server/redis/redis.ts
index 230078dc33..b95c687cc0 100644
--- a/packages/shared/src/server/redis/redis.ts
+++ b/packages/shared/src/server/redis/redis.ts
@@ -235,6 +235,43 @@ const createRedisSentinelInstance = (
return instance;
};
+/**
+ * Every connection handed out by createNewRedisInstance, so that a shutdown can
+ * release all of them. Each queue owns a dedicated client, and those clients
+ * retry forever; left connected they keep the event loop alive and the process
+ * never exits.
+ */
+const activeRedisInstances = new Set<Redis | Cluster>();
+
+const trackRedisInstance = <T extends Redis | Cluster | null>(
+ instance: T,
+): T => {
+ if (instance) {
+ activeRedisInstances.add(instance);
+ instance.once("end", () => activeRedisInstances.delete(instance));
+ }
+ return instance;
+};
+
+/**
+ * Disconnect every client created by createNewRedisInstance, including the
+ * shared `redis` client, which is created through the same path. Returns how
+ * many were closed, for shutdown logging.
+ */
+export const disconnectAllRedisInstances = (): number => {
+ let closed = 0;
+ for (const instance of activeRedisInstances) {
+ try {
+ instance.disconnect();
+ closed++;
+ } catch (error) {
+ logRedisError("Failed to disconnect Redis instance on shutdown", error);
+ }
+ }
+ activeRedisInstances.clear();
+ return closed;
+};
+
export const createNewRedisInstance = (
additionalOptions: Partial<RedisOptions> = {},
): Redis | Cluster | null => {
@@ -249,11 +286,11 @@ export const createNewRedisInstance = (
}
if (env.REDIS_CLUSTER_ENABLED === "true") {
- return createRedisClusterInstance(additionalOptions);
+ return trackRedisInstance(createRedisClusterInstance(additionalOptions));
}
if (env.REDIS_SENTINEL_ENABLED === "true") {
- return createRedisSentinelInstance(additionalOptions);
+ return trackRedisInstance(createRedisSentinelInstance(additionalOptions));
}
const tlsOptions = buildTlsOptions();
@@ -280,7 +317,7 @@ export const createNewRedisInstance = (
logRedisError("Redis error", error);
});
- return instance;
+ return trackRedisInstance(instance);
};
/**
diff --git a/worker/src/__tests__/bullmqVersionCheckOptions.test.ts b/worker/src/__tests__/bullmqVersionCheckOptions.test.ts
index 3badce0bf6..3dd401787a 100644
--- a/worker/src/__tests__/bullmqVersionCheckOptions.test.ts
+++ b/worker/src/__tests__/bullmqVersionCheckOptions.test.ts
@@ -47,6 +47,7 @@ const mockIoredis = () => {
public status = "ready";
public isCluster = false;
public on = vi.fn();
+ public once = vi.fn();
public duplicate = vi.fn(() => new MockRedis(this.options));
constructor(...args: unknown[]) {
diff --git a/worker/src/utils/shutdown.ts b/worker/src/utils/shutdown.ts
index 812fe1ce59..cad437ebd2 100644
--- a/worker/src/utils/shutdown.ts
+++ b/worker/src/utils/shutdown.ts
@@ -1,5 +1,5 @@
import { ClickHouseClientManager, logger } from "@langfuse/shared/src/server";
-import { redis } from "@langfuse/shared/src/server";
+import { disconnectAllRedisInstances } from "@langfuse/shared/src/server";
import { ClickhouseWriter } from "../services/ClickhouseWriter";
import { setSigtermReceived } from "../features/health";
@@ -112,8 +112,13 @@ const runDrainAndClose = async () => {
await ClickhouseWriter.getInstance().shutdown();
logger.info("Clickhouse writer has been shut down.");
- redis?.disconnect();
- logger.info("Redis connection has been closed.");
+ // Closes the shared client and every per-queue client in one pass. Each
+ // queue holds its own client; without this they stay connected, retry
+ // forever, and keep the event loop alive so the process never exits.
+ const closedRedisConnections = disconnectAllRedisInstances();
+ logger.info(
+ `Redis connections have been closed (${closedRedisConnections} clients).`,
+ );
await prisma.$disconnect();
logger.info("Prisma connection has been closed.");