🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(evals): retry missing evaluator media (#18311)
Tobias Wochingerb727c4c522a9
diff --git a/packages/shared/src/server/llm/errors.ts b/packages/shared/src/server/llm/errors.tsindex cad2bf4045..1d341b2544 100644--- a/packages/shared/src/server/llm/errors.ts+++ b/packages/shared/src/server/llm/errors.ts@@ -7,7 +7,8 @@ const LLM_VALIDATION_ERROR_MARKER = Symbol.for(export type LLMValidationErrorCode =| "invalid-connection"| "invalid-request"- | "endpoint-unreachable";+ | "endpoint-unreachable"+ | "media-not-found";/*** A deterministic validation failure owned by Langfuse, before or around thediff --git a/packages/shared/src/server/llm/mediaMessages.test.ts b/packages/shared/src/server/llm/mediaMessages.test.tsindex 642bcd2112..5ef9858ec1 100644--- a/packages/shared/src/server/llm/mediaMessages.test.ts+++ b/packages/shared/src/server/llm/mediaMessages.test.ts@@ -496,6 +496,7 @@ describe("compileLangfuseMediaMessages", () => {}),).rejects.toMatchObject({name: "LLMValidationError",+ code: "media-not-found",message: "Media asset image-1 was not found in this project",});expect(resolver).toHaveBeenCalledWith({diff --git a/packages/shared/src/server/llm/mediaMessages.ts b/packages/shared/src/server/llm/mediaMessages.tsindex efd042de9b..4bb05919cd 100644--- a/packages/shared/src/server/llm/mediaMessages.ts+++ b/packages/shared/src/server/llm/mediaMessages.ts@@ -262,7 +262,7 @@ export async function compileLangfuseMediaMessages(params: {});if (!resolved) {throw new LLMValidationError({- code: "invalid-request",+ code: "media-not-found",message: `Media asset ${match.id} was not found in this project`,});}diff --git a/worker/src/queues/__tests__/llmAsJudgeExecutionQueueProcessor.test.ts b/worker/src/queues/__tests__/llmAsJudgeExecutionQueueProcessor.test.tsindex 154ea6a84c..d70836c353 100644--- a/worker/src/queues/__tests__/llmAsJudgeExecutionQueueProcessor.test.ts+++ b/worker/src/queues/__tests__/llmAsJudgeExecutionQueueProcessor.test.ts@@ -393,6 +393,49 @@ describe("llmAsJudgeExecutionQueueProcessor", () => {});describe("LLM completion errors (non-retryable)", () => {+ it("retries media-not-found validation errors until the final attempt", async () => {+ const mediaNotFoundError = new Error(+ "Media asset image-1 was not found in this project",+ );+ (processObservationEval as Mock).mockRejectedValue(mediaNotFoundError);+ vi.mocked(classifyEvaluatorLlmError).mockReturnValue({+ kind: "validation",+ message: mediaNotFoundError.message,+ statusCode: 400,+ isRetryable: false,+ error: mediaNotFoundError,+ validationError: {+ code: "media-not-found",+ },+ blockReason: null,+ } as ReturnType<typeof classifyEvaluatorLlmError>);++ const job = createMockJob();++ await expect(llmAsJudgeExecutionQueueProcessor(job)).rejects.toThrow(+ mediaNotFoundError,+ );+ expect(prisma.jobExecution.update).not.toHaveBeenCalled();++ vi.mocked(traceException).mockClear();+ await expect(+ llmAsJudgeExecutionQueueProcessor(+ createMockJob({ attemptsMade: 9, opts: { attempts: 10 } }),+ ),+ ).resolves.toBeUndefined();+ expect(prisma.jobExecution.update).toHaveBeenCalledWith({+ where: {+ id: jobExecutionId,+ projectId,+ },+ data: expect.objectContaining({+ status: JobExecutionStatus.ERROR,+ error: mediaNotFoundError.message,+ }),+ });+ expect(traceException).not.toHaveBeenCalled();+ });+it("should set ERROR status for non-retryable LLM errors", async () => {const llmError = new Error("Invalid API key");(processObservationEval as Mock).mockRejectedValue(llmError);diff --git a/worker/src/queues/evalQueue.ts b/worker/src/queues/evalQueue.tsindex 24e6097ac7..8ad27f4e25 100644--- a/worker/src/queues/evalQueue.ts+++ b/worker/src/queues/evalQueue.ts@@ -323,6 +323,11 @@ export const llmAsJudgeExecutionQueueProcessorBuilder =return true;} catch (e) {const llmError = classifyEvaluatorLlmError(e);+ // Media may still be uploading when evaluation starts after ingestion.+ // Retry only this evaluator job instead of making the error globally retryable.+ const isMediaNotFoundError =+ llmError?.kind === "validation" &&+ llmError.validationError?.code === "media-not-found";const executionTraceId = createW3CTraceId(job.data.payload.jobExecutionId,);@@ -355,7 +360,9 @@ export const llmAsJudgeExecutionQueueProcessorBuilder =}}- const isTerminalError = Boolean(llmError) || isUnrecoverableError(e);+ const isUnrecoverable = isUnrecoverableError(e);+ const isTerminalError =+ (Boolean(llmError) && !isMediaNotFoundError) || isUnrecoverable;const totalAttempts = job.opts.attempts ?? 1;const isFinalAttempt = job.attemptsMade + 1 >= totalAttempts;@@ -371,9 +378,10 @@ export const llmAsJudgeExecutionQueueProcessorBuilder =data: {status: JobExecutionStatus.ERROR,endTime: new Date(),- error: isTerminalError- ? (llmError?.message ?? (e as Error).message)- : "An internal error occurred",+ error:+ llmError || isUnrecoverable+ ? (llmError?.message ?? (e as Error).message)+ : "An internal error occurred",executionTraceId,},});@@ -383,7 +391,7 @@ export const llmAsJudgeExecutionQueueProcessorBuilder =);}- if (isTerminalError) return;+ if (isTerminalError || (isMediaNotFoundError && isFinalAttempt)) return;traceException(e);logger.error(