Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(evals): preserve LLM judge BullMQ retries (#16510)

Tobias Wochingerb4fe77f667d7
diff --git a/worker/src/queues/__tests__/llmAsJudgeExecutionQueueProcessor.test.ts b/worker/src/queues/__tests__/llmAsJudgeExecutionQueueProcessor.test.ts
index 8f639d6bca..35efd76fad 100644
--- a/worker/src/queues/__tests__/llmAsJudgeExecutionQueueProcessor.test.ts
+++ b/worker/src/queues/__tests__/llmAsJudgeExecutionQueueProcessor.test.ts
@@ -139,7 +139,13 @@ describe("llmAsJudgeExecutionQueueProcessor", () => {
}>,
) => Promise<unknown>;
- const createMockJob = (overrides: Record<string, unknown> = {}): Job<any> => {
+ const createMockJob = (
+ overrides: {
+ data?: Record<string, unknown>;
+ attemptsMade?: number;
+ opts?: { attempts?: number };
+ } = {},
+ ): Job<any> => {
return {
data: {
id: "queue-job-123",
@@ -151,8 +157,10 @@ describe("llmAsJudgeExecutionQueueProcessor", () => {
observationS3Path,
},
retryBaggage: { attempt: 0 },
- ...overrides,
+ ...overrides.data,
},
+ attemptsMade: overrides.attemptsMade ?? 0,
+ opts: overrides.opts ?? { attempts: 10 },
} as Job<any>;
};
@@ -399,7 +407,7 @@ describe("llmAsJudgeExecutionQueueProcessor", () => {
});
describe("unexpected errors (retryable by BullMQ)", () => {
- it("should set ERROR status with generic message for unexpected errors", async () => {
+ it("should rethrow unexpected errors without persisting ERROR while BullMQ retries remain", async () => {
const unexpectedError = new Error("Database connection failed");
(processObservationEval as Mock).mockRejectedValue(unexpectedError);
@@ -410,17 +418,7 @@ describe("llmAsJudgeExecutionQueueProcessor", () => {
"Database connection failed",
);
- expect(prisma.jobExecution.update).toHaveBeenCalledWith({
- where: {
- id: jobExecutionId,
- projectId,
- },
- data: expect.objectContaining({
- status: JobExecutionStatus.ERROR,
- error: "An internal error occurred",
- executionTraceId: "test-trace-id",
- }),
- });
+ expect(prisma.jobExecution.update).not.toHaveBeenCalled();
});
it("should call traceException for unexpected errors", async () => {
@@ -448,6 +446,30 @@ describe("llmAsJudgeExecutionQueueProcessor", () => {
"Network timeout",
);
});
+
+ it("should persist ERROR on the final BullMQ attempt", async () => {
+ const unexpectedError = new Error("Database connection failed");
+ (processObservationEval as Mock).mockRejectedValue(unexpectedError);
+
+ await expect(
+ llmAsJudgeExecutionQueueProcessor(
+ createMockJob({ attemptsMade: 9, opts: { attempts: 10 } }),
+ ),
+ ).rejects.toThrow(unexpectedError);
+
+ expect(prisma.jobExecution.update).toHaveBeenCalledWith({
+ where: {
+ id: jobExecutionId,
+ projectId,
+ },
+ data: {
+ status: JobExecutionStatus.ERROR,
+ endTime: expect.any(Date),
+ error: "An internal error occurred",
+ executionTraceId: "test-trace-id",
+ },
+ });
+ });
});
describe("execution trace ID", () => {
@@ -473,7 +495,7 @@ describe("llmAsJudgeExecutionQueueProcessor", () => {
(processObservationEval as Mock).mockResolvedValue(undefined);
const job = createMockJob({
- retryBaggage: { attempt: 3 },
+ data: { retryBaggage: { attempt: 3 } },
});
await llmAsJudgeExecutionQueueProcessor(job as Job<any>);
diff --git a/worker/src/queues/evalQueue.ts b/worker/src/queues/evalQueue.ts
index e3433e6fbf..9cde4faf9c 100644
--- a/worker/src/queues/evalQueue.ts
+++ b/worker/src/queues/evalQueue.ts
@@ -335,23 +335,31 @@ export const llmAsJudgeExecutionQueueProcessorBuilder =
}
}
- await prisma.jobExecution.update({
- where: {
- id: job.data.payload.jobExecutionId,
- projectId: job.data.payload.projectId,
- },
- data: {
- status: JobExecutionStatus.ERROR,
- endTime: new Date(),
- error:
- llmError || isUnrecoverableError(e)
+ const isTerminalError = Boolean(llmError) || isUnrecoverableError(e);
+ const totalAttempts = job.opts.attempts ?? 1;
+ const isFinalAttempt = job.attemptsMade + 1 >= totalAttempts;
+
+ // Only persist the terminal ERROR state when there will be no more
+ // retries; otherwise observationEvalProcessor would short-circuit the
+ // retry attempts because it skips jobs already in ERROR status.
+ if (isTerminalError || isFinalAttempt) {
+ await prisma.jobExecution.update({
+ where: {
+ id: job.data.payload.jobExecutionId,
+ projectId: job.data.payload.projectId,
+ },
+ data: {
+ status: JobExecutionStatus.ERROR,
+ endTime: new Date(),
+ error: isTerminalError
? (llmError?.message ?? (e as Error).message)
: "An internal error occurred",
- executionTraceId,
- },
- });
+ executionTraceId,
+ },
+ });
+ }
- if (llmError || isUnrecoverableError(e)) return;
+ if (isTerminalError) return;
traceException(e);
logger.error(