🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(redis): disconnect per-queue clients on shutdown so the worker process exits (#17097)
pilsoo0169d3489e52
diff --git a/packages/shared/src/server/redis/redis.ts b/packages/shared/src/server/redis/redis.tsindex 230078dc33..b95c687cc0 100644--- a/packages/shared/src/server/redis/redis.ts+++ b/packages/shared/src/server/redis/redis.ts@@ -235,6 +235,43 @@ const createRedisSentinelInstance = (return instance;};+/**+ * Every connection handed out by createNewRedisInstance, so that a shutdown can+ * release all of them. Each queue owns a dedicated client, and those clients+ * retry forever; left connected they keep the event loop alive and the process+ * never exits.+ */+const activeRedisInstances = new Set<Redis | Cluster>();++const trackRedisInstance = <T extends Redis | Cluster | null>(+ instance: T,+): T => {+ if (instance) {+ activeRedisInstances.add(instance);+ instance.once("end", () => activeRedisInstances.delete(instance));+ }+ return instance;+};++/**+ * Disconnect every client created by createNewRedisInstance, including the+ * shared `redis` client, which is created through the same path. Returns how+ * many were closed, for shutdown logging.+ */+export const disconnectAllRedisInstances = (): number => {+ let closed = 0;+ for (const instance of activeRedisInstances) {+ try {+ instance.disconnect();+ closed++;+ } catch (error) {+ logRedisError("Failed to disconnect Redis instance on shutdown", error);+ }+ }+ activeRedisInstances.clear();+ return closed;+};+export const createNewRedisInstance = (additionalOptions: Partial<RedisOptions> = {},): Redis | Cluster | null => {@@ -249,11 +286,11 @@ export const createNewRedisInstance = (}if (env.REDIS_CLUSTER_ENABLED === "true") {- return createRedisClusterInstance(additionalOptions);+ return trackRedisInstance(createRedisClusterInstance(additionalOptions));}if (env.REDIS_SENTINEL_ENABLED === "true") {- return createRedisSentinelInstance(additionalOptions);+ return trackRedisInstance(createRedisSentinelInstance(additionalOptions));}const tlsOptions = buildTlsOptions();@@ -280,7 +317,7 @@ export const createNewRedisInstance = (logRedisError("Redis error", error);});- return instance;+ return trackRedisInstance(instance);};/**diff --git a/worker/src/__tests__/bullmqVersionCheckOptions.test.ts b/worker/src/__tests__/bullmqVersionCheckOptions.test.tsindex 3badce0bf6..3dd401787a 100644--- a/worker/src/__tests__/bullmqVersionCheckOptions.test.ts+++ b/worker/src/__tests__/bullmqVersionCheckOptions.test.ts@@ -47,6 +47,7 @@ const mockIoredis = () => {public status = "ready";public isCluster = false;public on = vi.fn();+ public once = vi.fn();public duplicate = vi.fn(() => new MockRedis(this.options));constructor(...args: unknown[]) {diff --git a/worker/src/utils/shutdown.ts b/worker/src/utils/shutdown.tsindex 812fe1ce59..cad437ebd2 100644--- a/worker/src/utils/shutdown.ts+++ b/worker/src/utils/shutdown.ts@@ -1,5 +1,5 @@import { ClickHouseClientManager, logger } from "@langfuse/shared/src/server";-import { redis } from "@langfuse/shared/src/server";+import { disconnectAllRedisInstances } from "@langfuse/shared/src/server";import { ClickhouseWriter } from "../services/ClickhouseWriter";import { setSigtermReceived } from "../features/health";@@ -112,8 +112,13 @@ const runDrainAndClose = async () => {await ClickhouseWriter.getInstance().shutdown();logger.info("Clickhouse writer has been shut down.");- redis?.disconnect();- logger.info("Redis connection has been closed.");+ // Closes the shared client and every per-queue client in one pass. Each+ // queue holds its own client; without this they stay connected, retry+ // forever, and keep the event loop alive so the process never exits.+ const closedRedisConnections = disconnectAllRedisInstances();+ logger.info(+ `Redis connections have been closed (${closedRedisConnections} clients).`,+ );await prisma.$disconnect();logger.info("Prisma connection has been closed.");