🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
chore(worker): count terminal queue job failures separately (#18372)
Niklas Semmler23518d407d4e
diff --git a/worker/src/__tests__/workerManagerMetrics.test.ts b/worker/src/__tests__/workerManagerMetrics.test.tsindex 8b1935d296..ce67bfbb34 100644--- a/worker/src/__tests__/workerManagerMetrics.test.ts+++ b/worker/src/__tests__/workerManagerMetrics.test.ts@@ -148,6 +148,32 @@ describe("WorkerManager queue metrics", () => {]);});+ it("counts failed_terminal only once BullMQ moved the job to the failed set", () => {+ WorkerManager.register("project-delete" as never, async () => undefined);++ // BullMQ sets finishedOn only when it stops retrying.+ mocks.handlers.get("failed")?.(+ { id: "job-id", name: "job", attemptsMade: 1, opts: { attempts: 5 } },+ new Error("retrying"),+ );+ mocks.handlers.get("failed")?.(+ {+ id: "job-id",+ name: "job",+ attemptsMade: 5,+ opts: { attempts: 5 },+ finishedOn: Date.now(),+ },+ new Error("exhausted"),+ );++ expect(mocks.recordIncrement.mock.calls).toEqual([+ ["langfuse.queue.project_delete.rate", 1, { type: "failed" }],+ ["langfuse.queue.project_delete.rate", 1, { type: "failed" }],+ ["langfuse.queue.project_delete.rate", 1, { type: "failed_terminal" }],+ ]);+ });+it("logs retryable S3 SlowDown as a warning until retries are exhausted", () => {WorkerManager.register("score-delete" as never, async () => undefined);const slowDown = new Error("Failed to download file from S3", {diff --git a/worker/src/queues/workerManager.ts b/worker/src/queues/workerManager.tsindex b67a60ceb4..b3d76bae48 100644--- a/worker/src/queues/workerManager.ts+++ b/worker/src/queues/workerManager.ts@@ -183,6 +183,14 @@ export class WorkerManager {type: "failed",...shardTag,});+ // BullMQ sets finishedOn only when it moves the job to the failed set+ // instead of scheduling a retry.+ if (job?.finishedOn) {+ recordIncrement(baseMetric + ".rate", 1, {+ type: "failed_terminal",+ ...shardTag,+ });+ }});worker.on("error", (failedReason: Error) => {logger.error(