🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(worker): fail closed on unknown BullMQ attempt budget (#16280)
Niklas Semmler0c792f11675f
diff --git a/packages/shared/src/server/redis/analyticsQueueAttempts.test.ts b/packages/shared/src/server/redis/analyticsQueueAttempts.test.tsnew file mode 100644index 0000000000..a4b6d02f20--- /dev/null+++ b/packages/shared/src/server/redis/analyticsQueueAttempts.test.ts@@ -0,0 +1,36 @@+import { describe, it, expect, vi } from "vitest";++const constructed: { name: string; opts: any }[] = [];++vi.mock("bullmq", () => ({+ Queue: class {+ opts: any;+ on = vi.fn();+ constructor(name: string, opts: any) {+ this.opts = opts;+ constructed.push({ name, opts });+ }+ },+}));++vi.mock("./redis", () => ({+ createBullMQQueueOptionsWithRedis: vi.fn(() => ({+ connection: {},+ prefix: "test",+ })),+}));++import { MixpanelIntegrationProcessingQueue } from "./mixpanelIntegrationProcessingQueue";+import { BlobStorageIntegrationProcessingQueue } from "./blobStorageIntegrationProcessingQueue";++describe("analytics/blob storage processing queue attempts budget", () => {+ it("Mixpanel processing queue has a fixed 5-attempt budget", () => {+ const queue = MixpanelIntegrationProcessingQueue.getInstance();+ expect((queue as any)?.opts.defaultJobOptions?.attempts).toBe(5);+ });++ it("Blob storage processing queue has a fixed 5-attempt budget", () => {+ const queue = BlobStorageIntegrationProcessingQueue.getInstance();+ expect((queue as any)?.opts.defaultJobOptions?.attempts).toBe(5);+ });+});diff --git a/worker/src/features/blobstorage/handleBlobStorageIntegrationProjectJob.ts b/worker/src/features/blobstorage/handleBlobStorageIntegrationProjectJob.tsindex 215d5a8de8..d93e66d032 100644--- a/worker/src/features/blobstorage/handleBlobStorageIntegrationProjectJob.ts+++ b/worker/src/features/blobstorage/handleBlobStorageIntegrationProjectJob.ts@@ -52,6 +52,7 @@ import {type CustomerFaultReason,} from "./isCustomerFaultError";import { isRecordNotFoundError } from "../integrations/prismaErrors";+import { isFinalBullmqAttempt } from "../integrations/bullmqAttempts";import { ByteCounter, TimedByteCounter } from "./byteCounters";import { WORKER_HOST_ID } from "../../utils/hostId";import {@@ -1521,10 +1522,8 @@ export const handleBlobStorageIntegrationProjectJob = async (// A deterministic customer-config/credential fault can't succeed until the// customer fixes it. Once BullMQ exhausts its retries, disable the- // integration so it stops re-scheduling and spamming. attemptsMade is- // 0-based, so the final attempt is attempts - 1.- const isFinalAttempt =- (job.attemptsMade ?? 0) >= (job.opts?.attempts ?? 1) - 1;+ // integration so it stops re-scheduling and spamming.+ const isFinalAttempt = isFinalBullmqAttempt(job, error);// Defined => disable; also tags the disable log + metric.const customerFaultReason = classifyCustomerFault(error);diff --git a/worker/src/features/integrations/bullmqAttempts.test.ts b/worker/src/features/integrations/bullmqAttempts.test.tsnew file mode 100644index 0000000000..6a71219369--- /dev/null+++ b/worker/src/features/integrations/bullmqAttempts.test.ts@@ -0,0 +1,66 @@+import { describe, it, expect } from "vitest";+import { UnrecoverableError } from "../../errors/UnrecoverableError";+import { isFinalBullmqAttempt } from "./bullmqAttempts";++describe("isFinalBullmqAttempt", () => {+ it("is false before the last attempt of a known budget", () => {+ expect(+ isFinalBullmqAttempt({ attemptsMade: 0, opts: { attempts: 5 } }),+ ).toBe(false);+ expect(+ isFinalBullmqAttempt({ attemptsMade: 3, opts: { attempts: 5 } }),+ ).toBe(false);+ });++ it("is true on the last attempt of a known budget (attemptsMade is 0-based)", () => {+ expect(+ isFinalBullmqAttempt({ attemptsMade: 4, opts: { attempts: 5 } }),+ ).toBe(true);+ });++ it("is true past the budget, defensively", () => {+ expect(+ isFinalBullmqAttempt({ attemptsMade: 10, opts: { attempts: 5 } }),+ ).toBe(true);+ });++ it("fails closed (never final) when opts is entirely absent", () => {+ expect(isFinalBullmqAttempt({ attemptsMade: 0 })).toBe(false);+ expect(isFinalBullmqAttempt({ attemptsMade: 0, opts: {} })).toBe(false);+ });++ it("fails closed (never final) when opts.attempts is undefined", () => {+ expect(+ isFinalBullmqAttempt({+ attemptsMade: 0,+ opts: { attempts: undefined },+ }),+ ).toBe(false);+ });++ it("treats an UnrecoverableError as an exhausted budget at attemptsMade 0", () => {+ const error = new UnrecoverableError("blocked by SSRF protection");+ expect(+ isFinalBullmqAttempt({ attemptsMade: 0, opts: { attempts: 5 } }, error),+ ).toBe(true);+ });++ it("duck-types UnrecoverableError by .name, not instanceof", () => {+ const lookalike = { name: "UnrecoverableError", message: "blocked" };+ expect(+ isFinalBullmqAttempt(+ { attemptsMade: 0, opts: { attempts: 5 } },+ lookalike,+ ),+ ).toBe(true);+ });++ it("does not treat an ordinary Error as terminal on its own", () => {+ expect(+ isFinalBullmqAttempt(+ { attemptsMade: 0, opts: { attempts: 5 } },+ new Error("transient"),+ ),+ ).toBe(false);+ });+});diff --git a/worker/src/features/integrations/bullmqAttempts.ts b/worker/src/features/integrations/bullmqAttempts.tsnew file mode 100644index 0000000000..afb4278e6e--- /dev/null+++ b/worker/src/features/integrations/bullmqAttempts.ts@@ -0,0 +1,32 @@+import { logger } from "@langfuse/shared/src/server";++interface BullmqAttemptState {+ attemptsMade?: number;+ opts?: { attempts?: number };+}++export function isFinalBullmqAttempt(+ job: BullmqAttemptState,+ error?: unknown,+): boolean {+ // Duck-type on `.name`: instanceof can miss a real UnrecoverableError+ // across a module boundary and let the loop keep retrying it.+ if (+ error &&+ typeof error === "object" &&+ (error as { name?: unknown }).name === "UnrecoverableError"+ ) {+ return true;+ }++ const attempts = job.opts?.attempts;+ if (typeof attempts !== "number") {+ logger.warn(+ "[BULLMQ ATTEMPTS] job.opts.attempts is missing; treating as not the final attempt (fail closed)",+ );+ return false;+ }++ const attemptsMade = job.attemptsMade ?? 0;+ return attemptsMade >= attempts - 1;+}