Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(worker): fail closed on unknown BullMQ attempt budget (#16280)

Niklas Semmler0c792f11675f
diff --git a/packages/shared/src/server/redis/analyticsQueueAttempts.test.ts b/packages/shared/src/server/redis/analyticsQueueAttempts.test.ts
new file mode 100644
index 0000000000..a4b6d02f20
--- /dev/null
+++ b/packages/shared/src/server/redis/analyticsQueueAttempts.test.ts
@@ -0,0 +1,36 @@
+import { describe, it, expect, vi } from "vitest";
+
+const constructed: { name: string; opts: any }[] = [];
+
+vi.mock("bullmq", () => ({
+ Queue: class {
+ opts: any;
+ on = vi.fn();
+ constructor(name: string, opts: any) {
+ this.opts = opts;
+ constructed.push({ name, opts });
+ }
+ },
+}));
+
+vi.mock("./redis", () => ({
+ createBullMQQueueOptionsWithRedis: vi.fn(() => ({
+ connection: {},
+ prefix: "test",
+ })),
+}));
+
+import { MixpanelIntegrationProcessingQueue } from "./mixpanelIntegrationProcessingQueue";
+import { BlobStorageIntegrationProcessingQueue } from "./blobStorageIntegrationProcessingQueue";
+
+describe("analytics/blob storage processing queue attempts budget", () => {
+ it("Mixpanel processing queue has a fixed 5-attempt budget", () => {
+ const queue = MixpanelIntegrationProcessingQueue.getInstance();
+ expect((queue as any)?.opts.defaultJobOptions?.attempts).toBe(5);
+ });
+
+ it("Blob storage processing queue has a fixed 5-attempt budget", () => {
+ const queue = BlobStorageIntegrationProcessingQueue.getInstance();
+ expect((queue as any)?.opts.defaultJobOptions?.attempts).toBe(5);
+ });
+});
diff --git a/worker/src/features/blobstorage/handleBlobStorageIntegrationProjectJob.ts b/worker/src/features/blobstorage/handleBlobStorageIntegrationProjectJob.ts
index 215d5a8de8..d93e66d032 100644
--- a/worker/src/features/blobstorage/handleBlobStorageIntegrationProjectJob.ts
+++ b/worker/src/features/blobstorage/handleBlobStorageIntegrationProjectJob.ts
@@ -52,6 +52,7 @@ import {
type CustomerFaultReason,
} from "./isCustomerFaultError";
import { isRecordNotFoundError } from "../integrations/prismaErrors";
+import { isFinalBullmqAttempt } from "../integrations/bullmqAttempts";
import { ByteCounter, TimedByteCounter } from "./byteCounters";
import { WORKER_HOST_ID } from "../../utils/hostId";
import {
@@ -1521,10 +1522,8 @@ export const handleBlobStorageIntegrationProjectJob = async (
// A deterministic customer-config/credential fault can't succeed until the
// customer fixes it. Once BullMQ exhausts its retries, disable the
- // integration so it stops re-scheduling and spamming. attemptsMade is
- // 0-based, so the final attempt is attempts - 1.
- const isFinalAttempt =
- (job.attemptsMade ?? 0) >= (job.opts?.attempts ?? 1) - 1;
+ // integration so it stops re-scheduling and spamming.
+ const isFinalAttempt = isFinalBullmqAttempt(job, error);
// Defined => disable; also tags the disable log + metric.
const customerFaultReason = classifyCustomerFault(error);
diff --git a/worker/src/features/integrations/bullmqAttempts.test.ts b/worker/src/features/integrations/bullmqAttempts.test.ts
new file mode 100644
index 0000000000..6a71219369
--- /dev/null
+++ b/worker/src/features/integrations/bullmqAttempts.test.ts
@@ -0,0 +1,66 @@
+import { describe, it, expect } from "vitest";
+import { UnrecoverableError } from "../../errors/UnrecoverableError";
+import { isFinalBullmqAttempt } from "./bullmqAttempts";
+
+describe("isFinalBullmqAttempt", () => {
+ it("is false before the last attempt of a known budget", () => {
+ expect(
+ isFinalBullmqAttempt({ attemptsMade: 0, opts: { attempts: 5 } }),
+ ).toBe(false);
+ expect(
+ isFinalBullmqAttempt({ attemptsMade: 3, opts: { attempts: 5 } }),
+ ).toBe(false);
+ });
+
+ it("is true on the last attempt of a known budget (attemptsMade is 0-based)", () => {
+ expect(
+ isFinalBullmqAttempt({ attemptsMade: 4, opts: { attempts: 5 } }),
+ ).toBe(true);
+ });
+
+ it("is true past the budget, defensively", () => {
+ expect(
+ isFinalBullmqAttempt({ attemptsMade: 10, opts: { attempts: 5 } }),
+ ).toBe(true);
+ });
+
+ it("fails closed (never final) when opts is entirely absent", () => {
+ expect(isFinalBullmqAttempt({ attemptsMade: 0 })).toBe(false);
+ expect(isFinalBullmqAttempt({ attemptsMade: 0, opts: {} })).toBe(false);
+ });
+
+ it("fails closed (never final) when opts.attempts is undefined", () => {
+ expect(
+ isFinalBullmqAttempt({
+ attemptsMade: 0,
+ opts: { attempts: undefined },
+ }),
+ ).toBe(false);
+ });
+
+ it("treats an UnrecoverableError as an exhausted budget at attemptsMade 0", () => {
+ const error = new UnrecoverableError("blocked by SSRF protection");
+ expect(
+ isFinalBullmqAttempt({ attemptsMade: 0, opts: { attempts: 5 } }, error),
+ ).toBe(true);
+ });
+
+ it("duck-types UnrecoverableError by .name, not instanceof", () => {
+ const lookalike = { name: "UnrecoverableError", message: "blocked" };
+ expect(
+ isFinalBullmqAttempt(
+ { attemptsMade: 0, opts: { attempts: 5 } },
+ lookalike,
+ ),
+ ).toBe(true);
+ });
+
+ it("does not treat an ordinary Error as terminal on its own", () => {
+ expect(
+ isFinalBullmqAttempt(
+ { attemptsMade: 0, opts: { attempts: 5 } },
+ new Error("transient"),
+ ),
+ ).toBe(false);
+ });
+});
diff --git a/worker/src/features/integrations/bullmqAttempts.ts b/worker/src/features/integrations/bullmqAttempts.ts
new file mode 100644
index 0000000000..afb4278e6e
--- /dev/null
+++ b/worker/src/features/integrations/bullmqAttempts.ts
@@ -0,0 +1,32 @@
+import { logger } from "@langfuse/shared/src/server";
+
+interface BullmqAttemptState {
+ attemptsMade?: number;
+ opts?: { attempts?: number };
+}
+
+export function isFinalBullmqAttempt(
+ job: BullmqAttemptState,
+ error?: unknown,
+): boolean {
+ // Duck-type on `.name`: instanceof can miss a real UnrecoverableError
+ // across a module boundary and let the loop keep retrying it.
+ if (
+ error &&
+ typeof error === "object" &&
+ (error as { name?: unknown }).name === "UnrecoverableError"
+ ) {
+ return true;
+ }
+
+ const attempts = job.opts?.attempts;
+ if (typeof attempts !== "number") {
+ logger.warn(
+ "[BULLMQ ATTEMPTS] job.opts.attempts is missing; treating as not the final attempt (fail closed)",
+ );
+ return false;
+ }
+
+ const attemptsMade = job.attemptsMade ?? 0;
+ return attemptsMade >= attempts - 1;
+}