Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

chore(worker): count terminal queue job failures separately (#18372)

Niklas Semmler23518d407d4e
diff --git a/worker/src/__tests__/workerManagerMetrics.test.ts b/worker/src/__tests__/workerManagerMetrics.test.ts
index 8b1935d296..ce67bfbb34 100644
--- a/worker/src/__tests__/workerManagerMetrics.test.ts
+++ b/worker/src/__tests__/workerManagerMetrics.test.ts
@@ -148,6 +148,32 @@ describe("WorkerManager queue metrics", () => {
]);
});
+ it("counts failed_terminal only once BullMQ moved the job to the failed set", () => {
+ WorkerManager.register("project-delete" as never, async () => undefined);
+
+ // BullMQ sets finishedOn only when it stops retrying.
+ mocks.handlers.get("failed")?.(
+ { id: "job-id", name: "job", attemptsMade: 1, opts: { attempts: 5 } },
+ new Error("retrying"),
+ );
+ mocks.handlers.get("failed")?.(
+ {
+ id: "job-id",
+ name: "job",
+ attemptsMade: 5,
+ opts: { attempts: 5 },
+ finishedOn: Date.now(),
+ },
+ new Error("exhausted"),
+ );
+
+ expect(mocks.recordIncrement.mock.calls).toEqual([
+ ["langfuse.queue.project_delete.rate", 1, { type: "failed" }],
+ ["langfuse.queue.project_delete.rate", 1, { type: "failed" }],
+ ["langfuse.queue.project_delete.rate", 1, { type: "failed_terminal" }],
+ ]);
+ });
+
it("logs retryable S3 SlowDown as a warning until retries are exhausted", () => {
WorkerManager.register("score-delete" as never, async () => undefined);
const slowDown = new Error("Failed to download file from S3", {
diff --git a/worker/src/queues/workerManager.ts b/worker/src/queues/workerManager.ts
index b67a60ceb4..b3d76bae48 100644
--- a/worker/src/queues/workerManager.ts
+++ b/worker/src/queues/workerManager.ts
@@ -183,6 +183,14 @@ export class WorkerManager {
type: "failed",
...shardTag,
});
+ // BullMQ sets finishedOn only when it moves the job to the failed set
+ // instead of scheduling a retry.
+ if (job?.finishedOn) {
+ recordIncrement(baseMetric + ".rate", 1, {
+ type: "failed_terminal",
+ ...shardTag,
+ });
+ }
});
worker.on("error", (failedReason: Error) => {
logger.error(