🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(evals): preserve LLM judge BullMQ retries (#16510)
Tobias Wochingerb4fe77f667d7
diff --git a/worker/src/queues/__tests__/llmAsJudgeExecutionQueueProcessor.test.ts b/worker/src/queues/__tests__/llmAsJudgeExecutionQueueProcessor.test.tsindex 8f639d6bca..35efd76fad 100644--- a/worker/src/queues/__tests__/llmAsJudgeExecutionQueueProcessor.test.ts+++ b/worker/src/queues/__tests__/llmAsJudgeExecutionQueueProcessor.test.ts@@ -139,7 +139,13 @@ describe("llmAsJudgeExecutionQueueProcessor", () => {}>,) => Promise<unknown>;- const createMockJob = (overrides: Record<string, unknown> = {}): Job<any> => {+ const createMockJob = (+ overrides: {+ data?: Record<string, unknown>;+ attemptsMade?: number;+ opts?: { attempts?: number };+ } = {},+ ): Job<any> => {return {data: {id: "queue-job-123",@@ -151,8 +157,10 @@ describe("llmAsJudgeExecutionQueueProcessor", () => {observationS3Path,},retryBaggage: { attempt: 0 },- ...overrides,+ ...overrides.data,},+ attemptsMade: overrides.attemptsMade ?? 0,+ opts: overrides.opts ?? { attempts: 10 },} as Job<any>;};@@ -399,7 +407,7 @@ describe("llmAsJudgeExecutionQueueProcessor", () => {});describe("unexpected errors (retryable by BullMQ)", () => {- it("should set ERROR status with generic message for unexpected errors", async () => {+ it("should rethrow unexpected errors without persisting ERROR while BullMQ retries remain", async () => {const unexpectedError = new Error("Database connection failed");(processObservationEval as Mock).mockRejectedValue(unexpectedError);@@ -410,17 +418,7 @@ describe("llmAsJudgeExecutionQueueProcessor", () => {"Database connection failed",);- expect(prisma.jobExecution.update).toHaveBeenCalledWith({- where: {- id: jobExecutionId,- projectId,- },- data: expect.objectContaining({- status: JobExecutionStatus.ERROR,- error: "An internal error occurred",- executionTraceId: "test-trace-id",- }),- });+ expect(prisma.jobExecution.update).not.toHaveBeenCalled();});it("should call traceException for unexpected errors", async () => {@@ -448,6 +446,30 @@ describe("llmAsJudgeExecutionQueueProcessor", () => {"Network timeout",);});++ it("should persist ERROR on the final BullMQ attempt", async () => {+ const unexpectedError = new Error("Database connection failed");+ (processObservationEval as Mock).mockRejectedValue(unexpectedError);++ await expect(+ llmAsJudgeExecutionQueueProcessor(+ createMockJob({ attemptsMade: 9, opts: { attempts: 10 } }),+ ),+ ).rejects.toThrow(unexpectedError);++ expect(prisma.jobExecution.update).toHaveBeenCalledWith({+ where: {+ id: jobExecutionId,+ projectId,+ },+ data: {+ status: JobExecutionStatus.ERROR,+ endTime: expect.any(Date),+ error: "An internal error occurred",+ executionTraceId: "test-trace-id",+ },+ });+ });});describe("execution trace ID", () => {@@ -473,7 +495,7 @@ describe("llmAsJudgeExecutionQueueProcessor", () => {(processObservationEval as Mock).mockResolvedValue(undefined);const job = createMockJob({- retryBaggage: { attempt: 3 },+ data: { retryBaggage: { attempt: 3 } },});await llmAsJudgeExecutionQueueProcessor(job as Job<any>);diff --git a/worker/src/queues/evalQueue.ts b/worker/src/queues/evalQueue.tsindex e3433e6fbf..9cde4faf9c 100644--- a/worker/src/queues/evalQueue.ts+++ b/worker/src/queues/evalQueue.ts@@ -335,23 +335,31 @@ export const llmAsJudgeExecutionQueueProcessorBuilder =}}- await prisma.jobExecution.update({- where: {- id: job.data.payload.jobExecutionId,- projectId: job.data.payload.projectId,- },- data: {- status: JobExecutionStatus.ERROR,- endTime: new Date(),- error:- llmError || isUnrecoverableError(e)+ const isTerminalError = Boolean(llmError) || isUnrecoverableError(e);+ const totalAttempts = job.opts.attempts ?? 1;+ const isFinalAttempt = job.attemptsMade + 1 >= totalAttempts;++ // Only persist the terminal ERROR state when there will be no more+ // retries; otherwise observationEvalProcessor would short-circuit the+ // retry attempts because it skips jobs already in ERROR status.+ if (isTerminalError || isFinalAttempt) {+ await prisma.jobExecution.update({+ where: {+ id: job.data.payload.jobExecutionId,+ projectId: job.data.payload.projectId,+ },+ data: {+ status: JobExecutionStatus.ERROR,+ endTime: new Date(),+ error: isTerminalError? (llmError?.message ?? (e as Error).message): "An internal error occurred",- executionTraceId,- },- });+ executionTraceId,+ },+ });+ }- if (llmError || isUnrecoverableError(e)) return;+ if (isTerminalError) return;traceException(e);logger.error(