🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(evals): return precondition errors for model validation failures (#18208)
Tobias Wochinger950f7ddc6f00
diff --git a/web/src/__tests__/server/unit/evaluator-v2-validation.servertest.ts b/web/src/__tests__/server/unit/evaluator-v2-validation.servertest.tsindex e04f8cb149..0be71daa85 100644--- a/web/src/__tests__/server/unit/evaluator-v2-validation.servertest.ts+++ b/web/src/__tests__/server/unit/evaluator-v2-validation.servertest.ts@@ -1,12 +1,23 @@import { EvalTemplateType } from "@langfuse/shared";+import type * as SharedServer from "@langfuse/shared/src/server";import { beforeEach, describe, expect, it, vi } from "vitest";const mocks = vi.hoisted(() => ({getEvaluatorDefinitionPreflightError: vi.fn(),+ getLLMErrorInfo: vi.fn(),isCodeEvalEnabled: vi.fn(),isCodeEvalSourceCodeLanguageSupported: vi.fn(),}));+vi.mock("@langfuse/shared/src/server", async (importOriginal) => {+ const actual = await importOriginal<typeof SharedServer>();+ return {+ ...actual,+ getLLMErrorInfo: (error: unknown) =>+ mocks.getLLMErrorInfo(error) ?? actual.getLLMErrorInfo(error),+ };+});+vi.mock("@/src/features/evals/server/evaluator-preflight", () => ({getEvaluatorDefinitionPreflightError:mocks.getEvaluatorDefinitionPreflightError,@@ -19,6 +30,11 @@ vi.mock("@/src/features/evals/server/isCodeEvalEnabled", () => ({}));import { assertEvaluatorConfigurationValid } from "@/src/features/evals/v2/server/evaluators/evaluatorValidation";+import {+ EvaluatorConfigurationError,+ EvaluatorModelConfigurationError,+} from "@/src/features/evals/v2/server/evaluators/evaluatorErrors";+import { toStructuredPublicApiError } from "@/src/features/public-api/server/structuredPublicApiErrorContract";import {CreateEvaluatorSchema,ListEvaluatorsSchema,@@ -139,8 +155,98 @@ describe("evaluator configuration validation", () => {expect(mocks.getEvaluatorDefinitionPreflightError).toHaveBeenCalledOnce();});- it("propagates transient model preflight failures", async () => {- const transientError = new Error("Provider rate limit");+ it("returns a precondition error without blocking the evaluator when model validation times out", async () => {+ mocks.getEvaluatorDefinitionPreflightError.mockRejectedValue(+ new DOMException(+ "The operation was aborted due to timeout",+ "TimeoutError",+ ),+ );++ const error = await assertEvaluatorConfigurationValid({+ projectId: "project-id",+ name: "LLM evaluator",+ definition: {+ type: EvalTemplateType.LLM_AS_JUDGE,+ promptMessages: [{ role: "user", content: "Judge {{output}}" }],+ provider: null,+ model: null,+ modelParams: null,+ vars: ["output"],+ variableMapping: null,+ outputDefinition: {+ dataType: "NUMERIC",+ score: { description: "Quality" },+ reasoning: { description: "Reasoning" },+ },+ },+ }).catch((error: unknown) => error);++ expect(error).toBeInstanceOf(EvaluatorConfigurationError);+ expect(error).not.toBeInstanceOf(EvaluatorModelConfigurationError);+ expect(toStructuredPublicApiError(error)).toMatchObject({+ httpCode: 412,+ code: "invalid_request",+ message:+ "The model did not respond within 95 seconds during evaluator validation. The evaluator was not saved. Retry or check your LLM connection and model settings.",+ });+ });++ it.each([+ {+ kind: "abort",+ error: new DOMException("The operation was aborted", "AbortError"),+ errorInfo: undefined,+ },+ {+ kind: "provider rate limit",+ error: new Error("Rate limit reached"),+ errorInfo: { kind: "provider", statusCode: 429, isRetryable: true },+ },+ {+ kind: "retryable provider server error",+ error: new Error("Service unavailable"),+ errorInfo: { kind: "provider", statusCode: 503, isRetryable: true },+ },+ ])(+ "returns a precondition error without blocking the evaluator for $kind",+ async ({ error: preflightError, errorInfo }) => {+ mocks.getEvaluatorDefinitionPreflightError.mockRejectedValue(+ preflightError,+ );+ mocks.getLLMErrorInfo.mockReturnValue(errorInfo);++ const error = await assertEvaluatorConfigurationValid({+ projectId: "project-id",+ name: "LLM evaluator",+ definition: {+ type: EvalTemplateType.LLM_AS_JUDGE,+ promptMessages: [{ role: "user", content: "Judge {{output}}" }],+ provider: null,+ model: null,+ modelParams: null,+ vars: ["output"],+ variableMapping: null,+ outputDefinition: {+ dataType: "NUMERIC",+ score: { description: "Quality" },+ reasoning: { description: "Reasoning" },+ },+ },+ }).catch((error: unknown) => error);++ expect(error).toBeInstanceOf(EvaluatorConfigurationError);+ expect(error).not.toBeInstanceOf(EvaluatorModelConfigurationError);+ expect(toStructuredPublicApiError(error)).toMatchObject({+ httpCode: 412,+ code: "invalid_request",+ message: expect.stringContaining("The evaluator was not saved. Retry"),+ });+ },+ );++ it("keeps unknown preflight failures as internal errors without exposing their messages", async () => {+ const transientError = new Error("sensitive internal detail");mocks.getEvaluatorDefinitionPreflightError.mockRejectedValue(transientError,);@@ -165,6 +271,38 @@ describe("evaluator configuration validation", () => {},}),).rejects.toBe(transientError);+ expect(toStructuredPublicApiError(transientError)).toMatchObject({+ httpCode: 500,+ code: "internal_error",+ message: "Internal Server Error",+ });+ });++ it("preserves permanent model-configuration errors for blocked evaluator persistence", async () => {+ mocks.getEvaluatorDefinitionPreflightError.mockResolvedValue(+ "The configured model is unavailable",+ );++ await expect(+ assertEvaluatorConfigurationValid({+ projectId: "project-id",+ name: "LLM evaluator",+ definition: {+ type: EvalTemplateType.LLM_AS_JUDGE,+ promptMessages: [{ role: "user", content: "Judge {{output}}" }],+ provider: null,+ model: null,+ modelParams: null,+ vars: ["output"],+ variableMapping: null,+ outputDefinition: {+ dataType: "NUMERIC",+ score: { description: "Quality" },+ reasoning: { description: "Reasoning" },+ },+ },+ }),+ ).rejects.toBeInstanceOf(EvaluatorModelConfigurationError);});// The schema is the only boundary that can see a caller-supplied mapping:diff --git a/web/src/features/evals/v2/server/evaluators/evaluatorValidation.ts b/web/src/features/evals/v2/server/evaluators/evaluatorValidation.tsindex 0f9a8ff33e..36bb4cd2ab 100644--- a/web/src/features/evals/v2/server/evaluators/evaluatorValidation.ts+++ b/web/src/features/evals/v2/server/evaluators/evaluatorValidation.ts@@ -6,6 +6,8 @@ import {} from "@langfuse/shared";import {DefaultEvalModelService,+ getClientInitiatedNonStreamingLlmTimeoutMs,+ getLLMErrorInfo,isDecisionModelAdapter,} from "@langfuse/shared/src/server";import { getEvaluatorDefinitionPreflightError } from "@/src/features/evals/server/evaluator-preflight";@@ -167,21 +169,42 @@ export async function assertEvaluatorConfigurationValid(params: {}}- const error = await getEvaluatorDefinitionPreflightError(- {- projectId: params.projectId,- template: {- name: params.name,- type: params.definition.type,- provider: params.definition.provider,- model: params.definition.model,- modelParams: params.definition.modelParams,- outputDefinition: params.definition.outputDefinition,+ try {+ const error = await getEvaluatorDefinitionPreflightError(+ {+ projectId: params.projectId,+ template: {+ name: params.name,+ type: params.definition.type,+ provider: params.definition.provider,+ model: params.definition.model,+ modelParams: params.definition.modelParams,+ outputDefinition: params.definition.outputDefinition,+ },},- },- { throwOnOperationalError: true },- );- if (error) throw new EvaluatorModelConfigurationError(error);+ { throwOnOperationalError: true },+ );+ if (error) throw new EvaluatorModelConfigurationError(error);+ } catch (error) {+ const llmError = getLLMErrorInfo(error);+ if (llmError?.kind === "timeout") {+ const timeoutSeconds =+ getClientInitiatedNonStreamingLlmTimeoutMs() / 1000;+ throw new EvaluatorConfigurationError(+ `The model did not respond within ${timeoutSeconds} seconds during evaluator validation. The evaluator was not saved. Retry or check your LLM connection and model settings.`,+ );+ }+ if (llmError && (llmError.isRetryable || llmError.kind === "abort")) {+ const message =+ llmError.kind === "abort"+ ? "The model request was aborted during evaluator validation."+ : "The LLM provider could not complete the model request during evaluator validation.";+ throw new EvaluatorConfigurationError(+ `${message} The evaluator was not saved. Retry or check your LLM connection and model settings.`,+ );+ }+ throw error;+ }}export async function getDecisionModelConfigurationError(params: {