Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(redis): avoid CROSSSLOT on v4 legacy API usage multi-key ops (#16180)

Max Deichmanncd3f1772b3d9
diff --git a/packages/shared/src/server/redis/redis.test.ts b/packages/shared/src/server/redis/redis.test.ts
index 3fd3062a57..3deff78cca 100644
--- a/packages/shared/src/server/redis/redis.test.ts
+++ b/packages/shared/src/server/redis/redis.test.ts
@@ -9,7 +9,8 @@ vi.mock("../../env", () => ({
},
}));
-import { scanKeys } from "./redis";
+import { env } from "../../env";
+import { safeMultiGet, scanKeys } from "./redis";
type ScanCall = [string, "MATCH", string, "COUNT", number];
@@ -80,3 +81,39 @@ describe("scanKeys", () => {
);
});
});
+
+describe("safeMultiGet", () => {
+ it("uses mget when cluster mode is disabled", async () => {
+ env.REDIS_CLUSTER_ENABLED = "false";
+ const mget = vi.fn(async () => ["a", null, "c"]);
+ const get = vi.fn();
+ const client = { mget, get } as unknown as Redis;
+
+ await expect(safeMultiGet(client, ["k1", "k2", "k3"])).resolves.toEqual([
+ "a",
+ null,
+ "c",
+ ]);
+ expect(mget).toHaveBeenCalledWith(["k1", "k2", "k3"]);
+ expect(get).not.toHaveBeenCalled();
+ });
+
+ it("uses per-key get when cluster mode is enabled", async () => {
+ env.REDIS_CLUSTER_ENABLED = "true";
+ const mget = vi.fn();
+ const get = vi.fn(async (key: string) => `value:${key}`);
+ const client = { mget, get } as unknown as Redis;
+
+ await expect(safeMultiGet(client, ["k1", "k2"])).resolves.toEqual([
+ "value:k1",
+ "value:k2",
+ ]);
+ expect(get).toHaveBeenCalledTimes(2);
+ expect(mget).not.toHaveBeenCalled();
+ });
+
+ it("returns an empty array for empty input", async () => {
+ const client = { mget: vi.fn(), get: vi.fn() } as unknown as Redis;
+ await expect(safeMultiGet(client, [])).resolves.toEqual([]);
+ });
+});
diff --git a/packages/shared/src/server/redis/redis.ts b/packages/shared/src/server/redis/redis.ts
index e6af9fdef4..36ceb183a2 100644
--- a/packages/shared/src/server/redis/redis.ts
+++ b/packages/shared/src/server/redis/redis.ts
@@ -335,6 +335,23 @@ export const safeMultiDel = async (
}
};
+/**
+ * Execute multiple Redis GET operations safely in cluster mode.
+ * MGET requires all keys to hash to the same slot; fall back to per-key GET.
+ */
+export const safeMultiGet = async (
+ redis: Redis | Cluster | null,
+ keys: string[],
+): Promise<(string | null)[]> => {
+ if (!redis || keys.length === 0) return [];
+
+ if (env.REDIS_CLUSTER_ENABLED === "true") {
+ return Promise.all(keys.map(async (key: string) => redis.get(key)));
+ }
+
+ return redis.mget(keys);
+};
+
const scanKeysForNode = async (
client: Redis,
pattern: string,
diff --git a/web/src/features/v4/server/v4TransitionCache.ts b/web/src/features/v4/server/v4TransitionCache.ts
index 4ee56d056f..24fc51ab2a 100644
--- a/web/src/features/v4/server/v4TransitionCache.ts
+++ b/web/src/features/v4/server/v4TransitionCache.ts
@@ -2,6 +2,7 @@ import { z } from "zod/v4";
import {
logger,
redis,
+ safeMultiGet,
v4ExperimentPostUsageBlobSchema,
v4ExperimentPostUsageProjectKey,
v4LegacyApiUsageBlobSchema,
@@ -100,7 +101,8 @@ const readBlobs = async <T>(
return keys.map(() => null);
}
try {
- const rawValues = await redis!.mget(keys);
+ // Cluster-safe: project keys hash to different slots.
+ const rawValues = await safeMultiGet(redis, keys);
return rawValues.map((rawValue) => {
if (!rawValue) return null;
try {
diff --git a/worker/src/features/v4/__tests__/handleV4LegacyApiUsageJob.test.ts b/worker/src/features/v4/__tests__/handleV4LegacyApiUsageJob.test.ts
index 881ed0de00..1e43a1ce88 100644
--- a/worker/src/features/v4/__tests__/handleV4LegacyApiUsageJob.test.ts
+++ b/worker/src/features/v4/__tests__/handleV4LegacyApiUsageJob.test.ts
@@ -49,7 +49,9 @@ const redisMock = vi.hoisted(() => ({
}
return 0;
}),
- del: vi.fn(async (...keys: string[]) => {
+ del: vi.fn(async (...args: Array<string | string[]>) => {
+ // Mirror ioredis: del(...keys) and del(keys[]) are both valid.
+ const keys = args.flat();
let deleted = 0;
for (const key of keys) {
if (redisState.store.delete(key)) deleted += 1;
diff --git a/worker/src/features/v4/handleV4LegacyApiUsageJob.ts b/worker/src/features/v4/handleV4LegacyApiUsageJob.ts
index a5c0b8a14b..8dbcdbcc6b 100644
--- a/worker/src/features/v4/handleV4LegacyApiUsageJob.ts
+++ b/worker/src/features/v4/handleV4LegacyApiUsageJob.ts
@@ -5,6 +5,8 @@ import {
logger,
queryClickhouse,
redis,
+ safeMultiDel,
+ safeMultiGet,
systemTableRef,
v4ExperimentPostUsageProjectKey,
v4LegacyApiHourBucketKey,
@@ -318,14 +320,17 @@ const setexInChunks = async (
const mgetInChunks = async (keys: string[]): Promise<(string | null)[]> => {
const values: (string | null)[] = [];
for (let index = 0; index < keys.length; index += CHUNK_SIZE) {
- values.push(...(await redis!.mget(keys.slice(index, index + CHUNK_SIZE))));
+ // Cluster-safe: MGET/multi-key DEL fail with CROSSSLOT across hour/project keys.
+ values.push(
+ ...(await safeMultiGet(redis, keys.slice(index, index + CHUNK_SIZE))),
+ );
}
return values;
};
const delInChunks = async (keys: string[]): Promise<void> => {
for (let index = 0; index < keys.length; index += CHUNK_SIZE) {
- await redis!.del(...keys.slice(index, index + CHUNK_SIZE));
+ await safeMultiDel(redis, keys.slice(index, index + CHUNK_SIZE));
}
};