🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(evals): validate evaluator models before saving (#18145)
Tobias Wochingerc0e82da9333e
diff --git a/packages/shared/package.json b/packages/shared/package.jsonindex 4123b499e5..8a58809e01 100644--- a/packages/shared/package.json+++ b/packages/shared/package.json@@ -162,7 +162,7 @@"dependencies": {"@ag-ui/client": "^0.0.52","@ag-ui/core": "^0.0.52",- "@ai-sdk/amazon-bedrock": "^5.0.25",+ "@ai-sdk/amazon-bedrock": "^5.0.97","@ai-sdk/anthropic": "^4.0.5","@ai-sdk/azure": "^4.0.5","@ai-sdk/google": "^4.0.6",diff --git a/packages/shared/src/server/llm/ai-sdk/requestShape.test.ts b/packages/shared/src/server/llm/ai-sdk/requestShape.test.tsindex 162cf240bf..5aa0c4fa76 100644--- a/packages/shared/src/server/llm/ai-sdk/requestShape.test.ts+++ b/packages/shared/src/server/llm/ai-sdk/requestShape.test.ts@@ -875,6 +875,34 @@ describe("AI SDK request shapes", () => {});});+ it("Bedrock: models rejecting forced tools use JSON instructions", async () => {+ const schema = {+ type: "object",+ properties: { answer: { type: "string" } },+ required: ["answer"],+ additionalProperties: false,+ } as const;+ const { result, request } = await runBedrockCompletion({+ model: "us.anthropic.claude-fable-5-1",+ output: createLLMOutput(schema),+ response: {+ ...BEDROCK_RESPONSE,+ output: {+ message: {+ role: "assistant",+ content: [{ text: '{"answer":"ok"}' }],+ },+ },+ },+ });++ expect(result.output).toEqual({ answer: "ok" });+ expect(request.body.toolConfig).toBeUndefined();+ expect(JSON.stringify(request.body)).toContain(+ "You MUST answer with only a JSON object",+ );+ });+it("Bedrock: tenant credentials suppress server-level env auth fallbacks", async () => {// A self-hosted operator may set these for their own purposes; tenant// connections must never authenticate with them. Unsuppressed, thediff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex 1d06717720..3bd5f845cd 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -282,11 +282,11 @@ importers:specifier: ^0.0.52version: 0.0.52'@ai-sdk/amazon-bedrock':- specifier: ^5.0.25- version: 5.0.25([email protected])+ specifier: ^5.0.97+ version: 5.0.97([email protected])'@ai-sdk/anthropic':specifier: ^4.0.5- version: 4.0.16([email protected])+ version: 4.0.65([email protected])'@ai-sdk/azure':specifier: ^4.0.5version: 4.0.5([email protected])@@ -298,7 +298,7 @@ importers:version: 5.0.8([email protected])([email protected])'@ai-sdk/openai':specifier: ^4.0.0- version: 4.0.16([email protected])+ version: 4.0.78([email protected])'@ai-sdk/openai-compatible':specifier: ^3.0.3version: 3.0.3([email protected])@@ -1334,8 +1334,8 @@ packages:peerDependencies:zod: 4.3.6- '@ai-sdk/[email protected]':- resolution: {integrity: sha512-cWDjIbD6CoFicwGjgwsU6sWMS9gpBBIQlMDgV5YfnPpmQVsQ3G+wBEwRyxCNxFVtxKxkMz8Bt6Uo6oTbfDCCNQ==}+ '@ai-sdk/[email protected]':+ resolution: {integrity: sha512-me8WEbCimP2f1yKuGvS0MSDiaqptdBeOs66dedJheNSWRCgIyqaaXL9k3lFKb0D6fMIn98NzpHB4tbHJFAzSXA==}engines: {node: '>=22'}peerDependencies:zod: 4.3.6@@ -1352,14 +1352,14 @@ packages:peerDependencies:zod: 4.3.6- '@ai-sdk/[email protected]':- resolution: {integrity: sha512-vyH4D6Auih5H2xvVzzh2ep5pbdWiaV7JDC+jHUE7zZJ5Kyv0TteLav4DrOgHzRuyv8ptfUSqFF6Y8//f/Ec0fQ==}+ '@ai-sdk/[email protected]':+ resolution: {integrity: sha512-JyJYKWGYz8vfoQ1pCm48a9zgX3Gvb2/6iGHb+8pEelpqnOiUq3aDHVq+nYMf9VepqcjCuH0JSOwBrKiPg/PD9w==}engines: {node: '>=22'}peerDependencies:zod: 4.3.6- '@ai-sdk/[email protected]':- resolution: {integrity: sha512-JyJYKWGYz8vfoQ1pCm48a9zgX3Gvb2/6iGHb+8pEelpqnOiUq3aDHVq+nYMf9VepqcjCuH0JSOwBrKiPg/PD9w==}+ '@ai-sdk/[email protected]':+ resolution: {integrity: sha512-3fmxxn5xWGOPU1wGrUCI30YFW5r6h2WrY+sz3rokkVlBHrDr889hPYP1yNoSmOVvtAwCFWlm79P2lRngIPOObg==}engines: {node: '>=22'}peerDependencies:zod: 4.3.6@@ -1430,14 +1430,14 @@ packages:peerDependencies:zod: 4.3.6- '@ai-sdk/[email protected]':- resolution: {integrity: sha512-Yh+PsXaf9NbN7oA3oKwOuyjTiHMPD75phf3SqGbDNNKQ3Yj3oTntp/WhO3nCHIPA6gr5/1lyridQokmOpPf9oQ==}+ '@ai-sdk/[email protected]':+ resolution: {integrity: sha512-a9cYQNAKAbkMTt6bHAvDuLv3klwtJT9m0A6x7s8EZlf1ebcxadg/1CONaYh1GA80ZRA9UQHJUS2wHoOphkQQkg==}engines: {node: '>=22'}peerDependencies:zod: 4.3.6- '@ai-sdk/[email protected]':- resolution: {integrity: sha512-a9cYQNAKAbkMTt6bHAvDuLv3klwtJT9m0A6x7s8EZlf1ebcxadg/1CONaYh1GA80ZRA9UQHJUS2wHoOphkQQkg==}+ '@ai-sdk/[email protected]':+ resolution: {integrity: sha512-S9WLSZUqebRjZbwYM8uorb4fsjo4Lhz/PGA3rWl01x88WeF/1PdHtNckcxqNZy5xMRzCHPVv9iA3F+ckV74zfA==}engines: {node: '>=22'}peerDependencies:zod: 4.3.6@@ -1466,12 +1466,6 @@ packages:peerDependencies:zod: 4.3.6- '@ai-sdk/[email protected]':- resolution: {integrity: sha512-7/96wE+ZsKB35iS9ASyllrE4Ym/EolXEB7AkuJ5FI++fmS85BVTAs77890C+1Z2jwHfBKjBQSBmsliOsAh0iFQ==}- engines: {node: '>=22'}- peerDependencies:- zod: 4.3.6-'@ai-sdk/[email protected]':resolution: {integrity: sha512-El0JOXXGcHFe6+owJ1UV0VgB9XtdivP8vcZni+rUIt6lt+cBHCnUdddaA6LE2avJGJ5AD0uXSY+uAvRL2tSvgw==}engines: {node: '>=22'}@@ -1484,6 +1478,12 @@ packages:peerDependencies:zod: 4.3.6+ '@ai-sdk/[email protected]':+ resolution: {integrity: sha512-T+/H8DCvqJoCqLhltVatS7ffA779Iyk/ZZrgNXkrgv51E0n2Cd6eIdCmtwhCGIKqtbTtwCvTmQdAMv5Z8rOGHQ==}+ engines: {node: '>=22'}+ peerDependencies:+ zod: 4.3.6+'@ai-sdk/[email protected]':resolution: {integrity: sha512-qZMxYJ0qqX/RfnuIaab+zp8UAeJn/ygXXAffR5I4N0n1IrvA6qBsjc8hXLmBiMV2zoXlifkacF7sEFnYnjBcqg==}engines: {node: '>=18'}@@ -1520,8 +1520,8 @@ packages:resolution: {integrity: sha512-VYMBxIQdcHqbIf1j+YZlI9Ati6LZ4wJe0GGd4z4a5H/KxTggjeOiyaVYTnfF7LHZK5jMQ+rofmzz4QPqf++NUw==}engines: {node: '>=22'}- '@ai-sdk/[email protected]':- resolution: {integrity: sha512-e0CpNWJUY7OxAFAnCZkw+ri9QOHWwTs1tXP42782KFGCU07qt8NiXCrCVowyCB5dP2r5/Uls+g2oPd8kOJn9dw==}+ '@ai-sdk/[email protected]':+ resolution: {integrity: sha512-+GZJIgz1jk86pwEbb3f1BD2bdoSKyWE4Jg4YUc7NMnMozbWemSKYZcw2F4nMaO5qwsL5A8RmioAKW81YktRpIQ==}engines: {node: '>=22'}'@ai-sdk/[email protected]':@@ -12765,12 +12765,12 @@ snapshots:aws4fetch: 1.0.20zod: 4.3.6- '@ai-sdk/[email protected]([email protected])':+ '@ai-sdk/[email protected]([email protected])':dependencies:- '@ai-sdk/anthropic': 4.0.16([email protected])- '@ai-sdk/openai': 4.0.16([email protected])- '@ai-sdk/provider': 4.0.3- '@ai-sdk/provider-utils': 5.0.11([email protected])+ '@ai-sdk/anthropic': 4.0.65([email protected])+ '@ai-sdk/openai': 4.0.78([email protected])+ '@ai-sdk/provider': 4.0.18+ '@ai-sdk/provider-utils': 5.0.49([email protected])'@smithy/eventstream-codec': 4.4.14'@smithy/util-utf8': 4.4.14aws4fetch: 1.0.20@@ -12788,18 +12788,18 @@ snapshots:'@ai-sdk/provider-utils': 4.0.33([email protected])zod: 4.3.6- '@ai-sdk/[email protected]([email protected])':- dependencies:- '@ai-sdk/provider': 4.0.3- '@ai-sdk/provider-utils': 5.0.11([email protected])- zod: 4.3.6-'@ai-sdk/[email protected]([email protected])':dependencies:'@ai-sdk/provider': 4.0.1'@ai-sdk/provider-utils': 5.0.3([email protected])zod: 4.3.6+ '@ai-sdk/[email protected]([email protected])':+ dependencies:+ '@ai-sdk/provider': 4.0.18+ '@ai-sdk/provider-utils': 5.0.49([email protected])+ zod: 4.3.6+'@ai-sdk/[email protected]([email protected])':dependencies:'@ai-sdk/deepseek': 3.0.3([email protected])@@ -12882,18 +12882,18 @@ snapshots:'@ai-sdk/provider-utils': 4.0.46([email protected])zod: 4.3.6- '@ai-sdk/[email protected]([email protected])':- dependencies:- '@ai-sdk/provider': 4.0.3- '@ai-sdk/provider-utils': 5.0.11([email protected])- zod: 4.3.6-'@ai-sdk/[email protected]([email protected])':dependencies:'@ai-sdk/provider': 4.0.1'@ai-sdk/provider-utils': 5.0.3([email protected])zod: 4.3.6+ '@ai-sdk/[email protected]([email protected])':+ dependencies:+ '@ai-sdk/provider': 4.0.18+ '@ai-sdk/provider-utils': 5.0.49([email protected])+ zod: 4.3.6+'@ai-sdk/[email protected]([email protected])':dependencies:'@ai-sdk/provider': 1.1.3@@ -12923,25 +12923,26 @@ snapshots:undici: 6.28.1zod: 4.3.6- '@ai-sdk/[email protected]([email protected])':+ '@ai-sdk/[email protected]([email protected])':dependencies:- '@ai-sdk/provider': 4.0.3+ '@ai-sdk/provider': 4.0.1'@standard-schema/spec': 1.1.0'@workflow/serde': 4.1.0eventsource-parser: 3.1.0zod: 4.3.6- '@ai-sdk/[email protected]([email protected])':+ '@ai-sdk/[email protected]([email protected])':dependencies:- '@ai-sdk/provider': 4.0.1+ '@ai-sdk/provider': 4.0.17'@standard-schema/spec': 1.1.0'@workflow/serde': 4.1.0eventsource-parser: 3.1.0+ undici: 7.29.1zod: 4.3.6- '@ai-sdk/[email protected]([email protected])':+ '@ai-sdk/[email protected]([email protected])':dependencies:- '@ai-sdk/provider': 4.0.17+ '@ai-sdk/provider': 4.0.18'@standard-schema/spec': 1.1.0'@workflow/serde': 4.1.0eventsource-parser: 3.1.0@@ -12984,7 +12985,7 @@ snapshots:dependencies:json-schema: 0.4.0- '@ai-sdk/[email protected]':+ '@ai-sdk/[email protected]':dependencies:json-schema: 0.4.0diff --git a/web/src/__tests__/server/unit/evaluator-preflight.servertest.ts b/web/src/__tests__/server/unit/evaluator-preflight.servertest.tsindex 76b86f1358..13321f6bde 100644--- a/web/src/__tests__/server/unit/evaluator-preflight.servertest.ts+++ b/web/src/__tests__/server/unit/evaluator-preflight.servertest.ts@@ -227,24 +227,124 @@ describe("evaluator preflight", () => {);});- it("does not expose unknown model call errors", async () => {+ it("rethrows retryable provider failures", async () => {+ const providerError = new Error("429 Rate limit reached");+ mockTestModelCall.mockRejectedValue(providerError);+ mockGetLLMErrorInfo.mockReturnValue({+ kind: "provider",+ message: "429 Rate limit reached",+ statusCode: 429,+ isRetryable: true,+ error: providerError,+ });++ await expect(+ getEvaluatorDefinitionPreflightError(+ {+ projectId: "project_test",+ template: {+ name: "Answer correctness",+ outputDefinition: numericOutputDefinition,+ },+ },+ {+ throwOnOperationalError: true,+ },+ ),+ ).rejects.toBe(providerError);+ });++ it("returns retryable provider failures for activation callers", async () => {+ const providerError = new Error("429 Rate limit reached");+ mockTestModelCall.mockRejectedValue(providerError);+ mockGetLLMErrorInfo.mockReturnValue({+ kind: "provider",+ message: "429 Rate limit reached",+ statusCode: 429,+ isRetryable: true,+ error: providerError,+ });++ await expect(+ getEvaluatorDefinitionPreflightError({+ projectId: "project_test",+ template: {+ name: "Answer correctness",+ outputDefinition: numericOutputDefinition,+ },+ }),+ ).resolves.toBe(+ `Model configuration not valid for evaluator "Answer correctness". 429 Rate limit reached`,+ );+ });++ it.each(["timeout", "abort"] as const)(+ "rethrows %s failures",+ async (kind) => {+ const operationalError = new Error(`${kind} during model call`);+ mockTestModelCall.mockRejectedValue(operationalError);+ mockGetLLMErrorInfo.mockReturnValue({+ kind,+ message: operationalError.message,+ isRetryable: false,+ error: operationalError,+ });++ await expect(+ getEvaluatorDefinitionPreflightError(+ {+ projectId: "project_test",+ template: {+ name: "Answer correctness",+ outputDefinition: numericOutputDefinition,+ },+ },+ {+ throwOnOperationalError: true,+ },+ ),+ ).rejects.toBe(operationalError);+ },+ );++ it("rethrows unknown model call errors without exposing them", async () => {+ const unknownError = new Error("sensitive internal model call detail");+ mockTestModelCall.mockRejectedValue(unknownError);+ mockGetLLMErrorInfo.mockReturnValue(null);++ await expect(+ getEvaluatorDefinitionPreflightError(+ {+ projectId: "project_test",+ template: {+ name: "Answer correctness",+ outputDefinition: numericOutputDefinition,+ },+ },+ {+ throwOnOperationalError: true,+ },+ ),+ ).rejects.toBe(unknownError);+ });++ it("hides unknown model call errors from activation callers", async () => {mockTestModelCall.mockRejectedValue(new Error("sensitive internal model call detail"),);mockGetLLMErrorInfo.mockReturnValue(null);- const result = await getEvaluatorDefinitionPreflightError({- projectId: "project_test",- template: {- name: "Answer correctness",- outputDefinition: numericOutputDefinition,- },- });-- expect(result).toBe(+ await expect(+ getEvaluatorDefinitionPreflightError({+ projectId: "project_test",+ template: {+ name: "Answer correctness",+ outputDefinition: numericOutputDefinition,+ },+ }),+ ).resolves.toBe(`Model configuration not valid for evaluator "Answer correctness". An internal error occurred`,);- expect(result).not.toContain("sensitive internal model call detail");});});});diff --git a/web/src/__tests__/server/unit/evaluator-v2-validation.servertest.ts b/web/src/__tests__/server/unit/evaluator-v2-validation.servertest.tsindex 0b4f081329..e04f8cb149 100644--- a/web/src/__tests__/server/unit/evaluator-v2-validation.servertest.ts+++ b/web/src/__tests__/server/unit/evaluator-v2-validation.servertest.ts@@ -2,14 +2,14 @@ import { EvalTemplateType } from "@langfuse/shared";import { beforeEach, describe, expect, it, vi } from "vitest";const mocks = vi.hoisted(() => ({- getEvaluatorDefinitionConfigurationError: vi.fn(),+ getEvaluatorDefinitionPreflightError: vi.fn(),isCodeEvalEnabled: vi.fn(),isCodeEvalSourceCodeLanguageSupported: vi.fn(),}));vi.mock("@/src/features/evals/server/evaluator-preflight", () => ({- getEvaluatorDefinitionConfigurationError:- mocks.getEvaluatorDefinitionConfigurationError,+ getEvaluatorDefinitionPreflightError:+ mocks.getEvaluatorDefinitionPreflightError,}));vi.mock("@/src/features/evals/server/isCodeEvalEnabled", () => ({@@ -29,7 +29,7 @@ describe("evaluator configuration validation", () => {vi.resetAllMocks();mocks.isCodeEvalEnabled.mockReturnValue(true);mocks.isCodeEvalSourceCodeLanguageSupported.mockReturnValue(true);- mocks.getEvaluatorDefinitionConfigurationError.mockResolvedValue(null);+ mocks.getEvaluatorDefinitionPreflightError.mockResolvedValue(null);});it("accepts evaluator names longer than 200 characters", () => {@@ -90,7 +90,7 @@ describe("evaluator configuration validation", () => {).toBe(true);});- it("validates deployment capabilities without executing the evaluator", async () => {+ it("validates deployment capabilities and the selected evaluator model", async () => {const codeDefinition = {type: EvalTemplateType.CODE,sourceCode: "return 1;",@@ -136,9 +136,35 @@ describe("evaluator configuration validation", () => {}),).resolves.toBeUndefined();- expect(- mocks.getEvaluatorDefinitionConfigurationError,- ).toHaveBeenCalledOnce();+ expect(mocks.getEvaluatorDefinitionPreflightError).toHaveBeenCalledOnce();+ });++ it("propagates transient model preflight failures", async () => {+ const transientError = new Error("Provider rate limit");+ mocks.getEvaluatorDefinitionPreflightError.mockRejectedValue(+ transientError,+ );++ await expect(+ assertEvaluatorConfigurationValid({+ projectId: "project-id",+ name: "LLM evaluator",+ definition: {+ type: EvalTemplateType.LLM_AS_JUDGE,+ promptMessages: [{ role: "user", content: "Judge {{output}}" }],+ provider: null,+ model: null,+ modelParams: null,+ vars: ["output"],+ variableMapping: null,+ outputDefinition: {+ dataType: "NUMERIC",+ score: { description: "Quality" },+ reasoning: { description: "Reasoning" },+ },+ },+ }),+ ).rejects.toBe(transientError);});// The schema is the only boundary that can see a caller-supplied mapping:diff --git a/web/src/features/evals/server/evaluator-preflight.ts b/web/src/features/evals/server/evaluator-preflight.tsindex 6df33f7699..fa37c77dd0 100644--- a/web/src/features/evals/server/evaluator-preflight.ts+++ b/web/src/features/evals/server/evaluator-preflight.ts@@ -70,10 +70,15 @@ export async function getEvaluatorDefinitionConfigurationError(params: {return prepared.valid ? null : prepared.error;}-export async function getEvaluatorDefinitionPreflightError(params: {- projectId: string;- template: EvaluatorPreflightDefinition;-}): Promise<string | null> {+export async function getEvaluatorDefinitionPreflightError(+ params: {+ projectId: string;+ template: EvaluatorPreflightDefinition;+ },+ options?: {+ throwOnOperationalError?: boolean;+ },+): Promise<string | null> {if (params.template.type === EvalTemplateType.CODE) return null;const prepared = await prepareEvaluatorDefinition(params);@@ -101,6 +106,14 @@ export async function getEvaluatorDefinitionPreflightError(params: {});} catch (err) {const llmError = getLLMErrorInfo(err);+ const isOperationalError =+ !llmError ||+ llmError.isRetryable ||+ llmError.kind === "timeout" ||+ llmError.kind === "abort";+ if (isOperationalError && options?.throwOnOperationalError) {+ throw err;+ }// A provider 404 also covers typos, missing model access, and bad base// URLs — not just retired models, so don't claim "retired" as fact.if (llmError?.statusCode === 404) {diff --git a/web/src/features/evals/v2/server/evaluators/evaluatorValidation.ts b/web/src/features/evals/v2/server/evaluators/evaluatorValidation.tsindex 3b072f8157..0f9a8ff33e 100644--- a/web/src/features/evals/v2/server/evaluators/evaluatorValidation.ts+++ b/web/src/features/evals/v2/server/evaluators/evaluatorValidation.ts@@ -8,7 +8,7 @@ import {DefaultEvalModelService,isDecisionModelAdapter,} from "@langfuse/shared/src/server";-import { getEvaluatorDefinitionConfigurationError } from "@/src/features/evals/server/evaluator-preflight";+import { getEvaluatorDefinitionPreflightError } from "@/src/features/evals/server/evaluator-preflight";import { getPromptMessagesValidationError } from "@/src/features/evals/v2/fns/promptMessages/hasInvalidSystemPromptMessage";import {isCodeEvalEnabled,@@ -167,17 +167,20 @@ export async function assertEvaluatorConfigurationValid(params: {}}- const error = await getEvaluatorDefinitionConfigurationError({- projectId: params.projectId,- template: {- name: params.name,- type: params.definition.type,- provider: params.definition.provider,- model: params.definition.model,- modelParams: params.definition.modelParams,- outputDefinition: params.definition.outputDefinition,+ const error = await getEvaluatorDefinitionPreflightError(+ {+ projectId: params.projectId,+ template: {+ name: params.name,+ type: params.definition.type,+ provider: params.definition.provider,+ model: params.definition.model,+ modelParams: params.definition.modelParams,+ outputDefinition: params.definition.outputDefinition,+ },},- });+ { throwOnOperationalError: true },+ );if (error) throw new EvaluatorModelConfigurationError(error);}