Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(evals): return precondition errors for model validation failures (#18208)

Tobias Wochinger950f7ddc6f00
diff --git a/web/src/__tests__/server/unit/evaluator-v2-validation.servertest.ts b/web/src/__tests__/server/unit/evaluator-v2-validation.servertest.ts
index e04f8cb149..0be71daa85 100644
--- a/web/src/__tests__/server/unit/evaluator-v2-validation.servertest.ts
+++ b/web/src/__tests__/server/unit/evaluator-v2-validation.servertest.ts
@@ -1,12 +1,23 @@
import { EvalTemplateType } from "@langfuse/shared";
+import type * as SharedServer from "@langfuse/shared/src/server";
import { beforeEach, describe, expect, it, vi } from "vitest";
const mocks = vi.hoisted(() => ({
getEvaluatorDefinitionPreflightError: vi.fn(),
+ getLLMErrorInfo: vi.fn(),
isCodeEvalEnabled: vi.fn(),
isCodeEvalSourceCodeLanguageSupported: vi.fn(),
}));
+vi.mock("@langfuse/shared/src/server", async (importOriginal) => {
+ const actual = await importOriginal<typeof SharedServer>();
+ return {
+ ...actual,
+ getLLMErrorInfo: (error: unknown) =>
+ mocks.getLLMErrorInfo(error) ?? actual.getLLMErrorInfo(error),
+ };
+});
+
vi.mock("@/src/features/evals/server/evaluator-preflight", () => ({
getEvaluatorDefinitionPreflightError:
mocks.getEvaluatorDefinitionPreflightError,
@@ -19,6 +30,11 @@ vi.mock("@/src/features/evals/server/isCodeEvalEnabled", () => ({
}));
import { assertEvaluatorConfigurationValid } from "@/src/features/evals/v2/server/evaluators/evaluatorValidation";
+import {
+ EvaluatorConfigurationError,
+ EvaluatorModelConfigurationError,
+} from "@/src/features/evals/v2/server/evaluators/evaluatorErrors";
+import { toStructuredPublicApiError } from "@/src/features/public-api/server/structuredPublicApiErrorContract";
import {
CreateEvaluatorSchema,
ListEvaluatorsSchema,
@@ -139,8 +155,98 @@ describe("evaluator configuration validation", () => {
expect(mocks.getEvaluatorDefinitionPreflightError).toHaveBeenCalledOnce();
});
- it("propagates transient model preflight failures", async () => {
- const transientError = new Error("Provider rate limit");
+ it("returns a precondition error without blocking the evaluator when model validation times out", async () => {
+ mocks.getEvaluatorDefinitionPreflightError.mockRejectedValue(
+ new DOMException(
+ "The operation was aborted due to timeout",
+ "TimeoutError",
+ ),
+ );
+
+ const error = await assertEvaluatorConfigurationValid({
+ projectId: "project-id",
+ name: "LLM evaluator",
+ definition: {
+ type: EvalTemplateType.LLM_AS_JUDGE,
+ promptMessages: [{ role: "user", content: "Judge {{output}}" }],
+ provider: null,
+ model: null,
+ modelParams: null,
+ vars: ["output"],
+ variableMapping: null,
+ outputDefinition: {
+ dataType: "NUMERIC",
+ score: { description: "Quality" },
+ reasoning: { description: "Reasoning" },
+ },
+ },
+ }).catch((error: unknown) => error);
+
+ expect(error).toBeInstanceOf(EvaluatorConfigurationError);
+ expect(error).not.toBeInstanceOf(EvaluatorModelConfigurationError);
+ expect(toStructuredPublicApiError(error)).toMatchObject({
+ httpCode: 412,
+ code: "invalid_request",
+ message:
+ "The model did not respond within 95 seconds during evaluator validation. The evaluator was not saved. Retry or check your LLM connection and model settings.",
+ });
+ });
+
+ it.each([
+ {
+ kind: "abort",
+ error: new DOMException("The operation was aborted", "AbortError"),
+ errorInfo: undefined,
+ },
+ {
+ kind: "provider rate limit",
+ error: new Error("Rate limit reached"),
+ errorInfo: { kind: "provider", statusCode: 429, isRetryable: true },
+ },
+ {
+ kind: "retryable provider server error",
+ error: new Error("Service unavailable"),
+ errorInfo: { kind: "provider", statusCode: 503, isRetryable: true },
+ },
+ ])(
+ "returns a precondition error without blocking the evaluator for $kind",
+ async ({ error: preflightError, errorInfo }) => {
+ mocks.getEvaluatorDefinitionPreflightError.mockRejectedValue(
+ preflightError,
+ );
+ mocks.getLLMErrorInfo.mockReturnValue(errorInfo);
+
+ const error = await assertEvaluatorConfigurationValid({
+ projectId: "project-id",
+ name: "LLM evaluator",
+ definition: {
+ type: EvalTemplateType.LLM_AS_JUDGE,
+ promptMessages: [{ role: "user", content: "Judge {{output}}" }],
+ provider: null,
+ model: null,
+ modelParams: null,
+ vars: ["output"],
+ variableMapping: null,
+ outputDefinition: {
+ dataType: "NUMERIC",
+ score: { description: "Quality" },
+ reasoning: { description: "Reasoning" },
+ },
+ },
+ }).catch((error: unknown) => error);
+
+ expect(error).toBeInstanceOf(EvaluatorConfigurationError);
+ expect(error).not.toBeInstanceOf(EvaluatorModelConfigurationError);
+ expect(toStructuredPublicApiError(error)).toMatchObject({
+ httpCode: 412,
+ code: "invalid_request",
+ message: expect.stringContaining("The evaluator was not saved. Retry"),
+ });
+ },
+ );
+
+ it("keeps unknown preflight failures as internal errors without exposing their messages", async () => {
+ const transientError = new Error("sensitive internal detail");
mocks.getEvaluatorDefinitionPreflightError.mockRejectedValue(
transientError,
);
@@ -165,6 +271,38 @@ describe("evaluator configuration validation", () => {
},
}),
).rejects.toBe(transientError);
+ expect(toStructuredPublicApiError(transientError)).toMatchObject({
+ httpCode: 500,
+ code: "internal_error",
+ message: "Internal Server Error",
+ });
+ });
+
+ it("preserves permanent model-configuration errors for blocked evaluator persistence", async () => {
+ mocks.getEvaluatorDefinitionPreflightError.mockResolvedValue(
+ "The configured model is unavailable",
+ );
+
+ await expect(
+ assertEvaluatorConfigurationValid({
+ projectId: "project-id",
+ name: "LLM evaluator",
+ definition: {
+ type: EvalTemplateType.LLM_AS_JUDGE,
+ promptMessages: [{ role: "user", content: "Judge {{output}}" }],
+ provider: null,
+ model: null,
+ modelParams: null,
+ vars: ["output"],
+ variableMapping: null,
+ outputDefinition: {
+ dataType: "NUMERIC",
+ score: { description: "Quality" },
+ reasoning: { description: "Reasoning" },
+ },
+ },
+ }),
+ ).rejects.toBeInstanceOf(EvaluatorModelConfigurationError);
});
// The schema is the only boundary that can see a caller-supplied mapping:
diff --git a/web/src/features/evals/v2/server/evaluators/evaluatorValidation.ts b/web/src/features/evals/v2/server/evaluators/evaluatorValidation.ts
index 0f9a8ff33e..36bb4cd2ab 100644
--- a/web/src/features/evals/v2/server/evaluators/evaluatorValidation.ts
+++ b/web/src/features/evals/v2/server/evaluators/evaluatorValidation.ts
@@ -6,6 +6,8 @@ import {
} from "@langfuse/shared";
import {
DefaultEvalModelService,
+ getClientInitiatedNonStreamingLlmTimeoutMs,
+ getLLMErrorInfo,
isDecisionModelAdapter,
} from "@langfuse/shared/src/server";
import { getEvaluatorDefinitionPreflightError } from "@/src/features/evals/server/evaluator-preflight";
@@ -167,21 +169,42 @@ export async function assertEvaluatorConfigurationValid(params: {
}
}
- const error = await getEvaluatorDefinitionPreflightError(
- {
- projectId: params.projectId,
- template: {
- name: params.name,
- type: params.definition.type,
- provider: params.definition.provider,
- model: params.definition.model,
- modelParams: params.definition.modelParams,
- outputDefinition: params.definition.outputDefinition,
+ try {
+ const error = await getEvaluatorDefinitionPreflightError(
+ {
+ projectId: params.projectId,
+ template: {
+ name: params.name,
+ type: params.definition.type,
+ provider: params.definition.provider,
+ model: params.definition.model,
+ modelParams: params.definition.modelParams,
+ outputDefinition: params.definition.outputDefinition,
+ },
},
- },
- { throwOnOperationalError: true },
- );
- if (error) throw new EvaluatorModelConfigurationError(error);
+ { throwOnOperationalError: true },
+ );
+ if (error) throw new EvaluatorModelConfigurationError(error);
+ } catch (error) {
+ const llmError = getLLMErrorInfo(error);
+ if (llmError?.kind === "timeout") {
+ const timeoutSeconds =
+ getClientInitiatedNonStreamingLlmTimeoutMs() / 1000;
+ throw new EvaluatorConfigurationError(
+ `The model did not respond within ${timeoutSeconds} seconds during evaluator validation. The evaluator was not saved. Retry or check your LLM connection and model settings.`,
+ );
+ }
+ if (llmError && (llmError.isRetryable || llmError.kind === "abort")) {
+ const message =
+ llmError.kind === "abort"
+ ? "The model request was aborted during evaluator validation."
+ : "The LLM provider could not complete the model request during evaluator validation.";
+ throw new EvaluatorConfigurationError(
+ `${message} The evaluator was not saved. Retry or check your LLM connection and model settings.`,
+ );
+ }
+ throw error;
+ }
}
export async function getDecisionModelConfigurationError(params: {