Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(evals): retry missing evaluator media (#18311)

Tobias Wochingerb727c4c522a9
diff --git a/packages/shared/src/server/llm/errors.ts b/packages/shared/src/server/llm/errors.ts
index cad2bf4045..1d341b2544 100644
--- a/packages/shared/src/server/llm/errors.ts
+++ b/packages/shared/src/server/llm/errors.ts
@@ -7,7 +7,8 @@ const LLM_VALIDATION_ERROR_MARKER = Symbol.for(
export type LLMValidationErrorCode =
| "invalid-connection"
| "invalid-request"
- | "endpoint-unreachable";
+ | "endpoint-unreachable"
+ | "media-not-found";
/**
* A deterministic validation failure owned by Langfuse, before or around the
diff --git a/packages/shared/src/server/llm/mediaMessages.test.ts b/packages/shared/src/server/llm/mediaMessages.test.ts
index 642bcd2112..5ef9858ec1 100644
--- a/packages/shared/src/server/llm/mediaMessages.test.ts
+++ b/packages/shared/src/server/llm/mediaMessages.test.ts
@@ -496,6 +496,7 @@ describe("compileLangfuseMediaMessages", () => {
}),
).rejects.toMatchObject({
name: "LLMValidationError",
+ code: "media-not-found",
message: "Media asset image-1 was not found in this project",
});
expect(resolver).toHaveBeenCalledWith({
diff --git a/packages/shared/src/server/llm/mediaMessages.ts b/packages/shared/src/server/llm/mediaMessages.ts
index efd042de9b..4bb05919cd 100644
--- a/packages/shared/src/server/llm/mediaMessages.ts
+++ b/packages/shared/src/server/llm/mediaMessages.ts
@@ -262,7 +262,7 @@ export async function compileLangfuseMediaMessages(params: {
});
if (!resolved) {
throw new LLMValidationError({
- code: "invalid-request",
+ code: "media-not-found",
message: `Media asset ${match.id} was not found in this project`,
});
}
diff --git a/worker/src/queues/__tests__/llmAsJudgeExecutionQueueProcessor.test.ts b/worker/src/queues/__tests__/llmAsJudgeExecutionQueueProcessor.test.ts
index 154ea6a84c..d70836c353 100644
--- a/worker/src/queues/__tests__/llmAsJudgeExecutionQueueProcessor.test.ts
+++ b/worker/src/queues/__tests__/llmAsJudgeExecutionQueueProcessor.test.ts
@@ -393,6 +393,49 @@ describe("llmAsJudgeExecutionQueueProcessor", () => {
});
describe("LLM completion errors (non-retryable)", () => {
+ it("retries media-not-found validation errors until the final attempt", async () => {
+ const mediaNotFoundError = new Error(
+ "Media asset image-1 was not found in this project",
+ );
+ (processObservationEval as Mock).mockRejectedValue(mediaNotFoundError);
+ vi.mocked(classifyEvaluatorLlmError).mockReturnValue({
+ kind: "validation",
+ message: mediaNotFoundError.message,
+ statusCode: 400,
+ isRetryable: false,
+ error: mediaNotFoundError,
+ validationError: {
+ code: "media-not-found",
+ },
+ blockReason: null,
+ } as ReturnType<typeof classifyEvaluatorLlmError>);
+
+ const job = createMockJob();
+
+ await expect(llmAsJudgeExecutionQueueProcessor(job)).rejects.toThrow(
+ mediaNotFoundError,
+ );
+ expect(prisma.jobExecution.update).not.toHaveBeenCalled();
+
+ vi.mocked(traceException).mockClear();
+ await expect(
+ llmAsJudgeExecutionQueueProcessor(
+ createMockJob({ attemptsMade: 9, opts: { attempts: 10 } }),
+ ),
+ ).resolves.toBeUndefined();
+ expect(prisma.jobExecution.update).toHaveBeenCalledWith({
+ where: {
+ id: jobExecutionId,
+ projectId,
+ },
+ data: expect.objectContaining({
+ status: JobExecutionStatus.ERROR,
+ error: mediaNotFoundError.message,
+ }),
+ });
+ expect(traceException).not.toHaveBeenCalled();
+ });
+
it("should set ERROR status for non-retryable LLM errors", async () => {
const llmError = new Error("Invalid API key");
(processObservationEval as Mock).mockRejectedValue(llmError);
diff --git a/worker/src/queues/evalQueue.ts b/worker/src/queues/evalQueue.ts
index 24e6097ac7..8ad27f4e25 100644
--- a/worker/src/queues/evalQueue.ts
+++ b/worker/src/queues/evalQueue.ts
@@ -323,6 +323,11 @@ export const llmAsJudgeExecutionQueueProcessorBuilder =
return true;
} catch (e) {
const llmError = classifyEvaluatorLlmError(e);
+ // Media may still be uploading when evaluation starts after ingestion.
+ // Retry only this evaluator job instead of making the error globally retryable.
+ const isMediaNotFoundError =
+ llmError?.kind === "validation" &&
+ llmError.validationError?.code === "media-not-found";
const executionTraceId = createW3CTraceId(
job.data.payload.jobExecutionId,
);
@@ -355,7 +360,9 @@ export const llmAsJudgeExecutionQueueProcessorBuilder =
}
}
- const isTerminalError = Boolean(llmError) || isUnrecoverableError(e);
+ const isUnrecoverable = isUnrecoverableError(e);
+ const isTerminalError =
+ (Boolean(llmError) && !isMediaNotFoundError) || isUnrecoverable;
const totalAttempts = job.opts.attempts ?? 1;
const isFinalAttempt = job.attemptsMade + 1 >= totalAttempts;
@@ -371,9 +378,10 @@ export const llmAsJudgeExecutionQueueProcessorBuilder =
data: {
status: JobExecutionStatus.ERROR,
endTime: new Date(),
- error: isTerminalError
- ? (llmError?.message ?? (e as Error).message)
- : "An internal error occurred",
+ error:
+ llmError || isUnrecoverable
+ ? (llmError?.message ?? (e as Error).message)
+ : "An internal error occurred",
executionTraceId,
},
});
@@ -383,7 +391,7 @@ export const llmAsJudgeExecutionQueueProcessorBuilder =
);
}
- if (isTerminalError) return;
+ if (isTerminalError || (isMediaNotFoundError && isFinalAttempt)) return;
traceException(e);
logger.error(