Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(evals): validate evaluator models before saving (#18145)

Tobias Wochingerc0e82da9333e
diff --git a/packages/shared/package.json b/packages/shared/package.json
index 4123b499e5..8a58809e01 100644
--- a/packages/shared/package.json
+++ b/packages/shared/package.json
@@ -162,7 +162,7 @@
"dependencies": {
"@ag-ui/client": "^0.0.52",
"@ag-ui/core": "^0.0.52",
- "@ai-sdk/amazon-bedrock": "^5.0.25",
+ "@ai-sdk/amazon-bedrock": "^5.0.97",
"@ai-sdk/anthropic": "^4.0.5",
"@ai-sdk/azure": "^4.0.5",
"@ai-sdk/google": "^4.0.6",
diff --git a/packages/shared/src/server/llm/ai-sdk/requestShape.test.ts b/packages/shared/src/server/llm/ai-sdk/requestShape.test.ts
index 162cf240bf..5aa0c4fa76 100644
--- a/packages/shared/src/server/llm/ai-sdk/requestShape.test.ts
+++ b/packages/shared/src/server/llm/ai-sdk/requestShape.test.ts
@@ -875,6 +875,34 @@ describe("AI SDK request shapes", () => {
});
});
+ it("Bedrock: models rejecting forced tools use JSON instructions", async () => {
+ const schema = {
+ type: "object",
+ properties: { answer: { type: "string" } },
+ required: ["answer"],
+ additionalProperties: false,
+ } as const;
+ const { result, request } = await runBedrockCompletion({
+ model: "us.anthropic.claude-fable-5-1",
+ output: createLLMOutput(schema),
+ response: {
+ ...BEDROCK_RESPONSE,
+ output: {
+ message: {
+ role: "assistant",
+ content: [{ text: '{"answer":"ok"}' }],
+ },
+ },
+ },
+ });
+
+ expect(result.output).toEqual({ answer: "ok" });
+ expect(request.body.toolConfig).toBeUndefined();
+ expect(JSON.stringify(request.body)).toContain(
+ "You MUST answer with only a JSON object",
+ );
+ });
+
it("Bedrock: tenant credentials suppress server-level env auth fallbacks", async () => {
// A self-hosted operator may set these for their own purposes; tenant
// connections must never authenticate with them. Unsuppressed, the
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 1d06717720..3bd5f845cd 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -282,11 +282,11 @@ importers:
specifier: ^0.0.52
version: 0.0.52
'@ai-sdk/amazon-bedrock':
- specifier: ^5.0.25
- version: 5.0.25([email protected])
+ specifier: ^5.0.97
+ version: 5.0.97([email protected])
'@ai-sdk/anthropic':
specifier: ^4.0.5
- version: 4.0.16([email protected])
+ version: 4.0.65([email protected])
'@ai-sdk/azure':
specifier: ^4.0.5
version: 4.0.5([email protected])
@@ -298,7 +298,7 @@ importers:
'@ai-sdk/openai':
specifier: ^4.0.0
- version: 4.0.16([email protected])
+ version: 4.0.78([email protected])
'@ai-sdk/openai-compatible':
specifier: ^3.0.3
version: 3.0.3([email protected])
@@ -1334,8 +1334,8 @@ packages:
peerDependencies:
zod: 4.3.6
- '@ai-sdk/[email protected]':
- resolution: {integrity: sha512-cWDjIbD6CoFicwGjgwsU6sWMS9gpBBIQlMDgV5YfnPpmQVsQ3G+wBEwRyxCNxFVtxKxkMz8Bt6Uo6oTbfDCCNQ==}
+ '@ai-sdk/[email protected]':
+ resolution: {integrity: sha512-me8WEbCimP2f1yKuGvS0MSDiaqptdBeOs66dedJheNSWRCgIyqaaXL9k3lFKb0D6fMIn98NzpHB4tbHJFAzSXA==}
engines: {node: '>=22'}
peerDependencies:
zod: 4.3.6
@@ -1352,14 +1352,14 @@ packages:
peerDependencies:
zod: 4.3.6
- '@ai-sdk/[email protected]':
- resolution: {integrity: sha512-vyH4D6Auih5H2xvVzzh2ep5pbdWiaV7JDC+jHUE7zZJ5Kyv0TteLav4DrOgHzRuyv8ptfUSqFF6Y8//f/Ec0fQ==}
+ '@ai-sdk/[email protected]':
+ resolution: {integrity: sha512-JyJYKWGYz8vfoQ1pCm48a9zgX3Gvb2/6iGHb+8pEelpqnOiUq3aDHVq+nYMf9VepqcjCuH0JSOwBrKiPg/PD9w==}
engines: {node: '>=22'}
peerDependencies:
zod: 4.3.6
- '@ai-sdk/[email protected]':
- resolution: {integrity: sha512-JyJYKWGYz8vfoQ1pCm48a9zgX3Gvb2/6iGHb+8pEelpqnOiUq3aDHVq+nYMf9VepqcjCuH0JSOwBrKiPg/PD9w==}
+ '@ai-sdk/[email protected]':
+ resolution: {integrity: sha512-3fmxxn5xWGOPU1wGrUCI30YFW5r6h2WrY+sz3rokkVlBHrDr889hPYP1yNoSmOVvtAwCFWlm79P2lRngIPOObg==}
engines: {node: '>=22'}
peerDependencies:
zod: 4.3.6
@@ -1430,14 +1430,14 @@ packages:
peerDependencies:
zod: 4.3.6
- '@ai-sdk/[email protected]':
- resolution: {integrity: sha512-Yh+PsXaf9NbN7oA3oKwOuyjTiHMPD75phf3SqGbDNNKQ3Yj3oTntp/WhO3nCHIPA6gr5/1lyridQokmOpPf9oQ==}
+ '@ai-sdk/[email protected]':
+ resolution: {integrity: sha512-a9cYQNAKAbkMTt6bHAvDuLv3klwtJT9m0A6x7s8EZlf1ebcxadg/1CONaYh1GA80ZRA9UQHJUS2wHoOphkQQkg==}
engines: {node: '>=22'}
peerDependencies:
zod: 4.3.6
- '@ai-sdk/[email protected]':
- resolution: {integrity: sha512-a9cYQNAKAbkMTt6bHAvDuLv3klwtJT9m0A6x7s8EZlf1ebcxadg/1CONaYh1GA80ZRA9UQHJUS2wHoOphkQQkg==}
+ '@ai-sdk/[email protected]':
+ resolution: {integrity: sha512-S9WLSZUqebRjZbwYM8uorb4fsjo4Lhz/PGA3rWl01x88WeF/1PdHtNckcxqNZy5xMRzCHPVv9iA3F+ckV74zfA==}
engines: {node: '>=22'}
peerDependencies:
zod: 4.3.6
@@ -1466,12 +1466,6 @@ packages:
peerDependencies:
zod: 4.3.6
- '@ai-sdk/[email protected]':
- resolution: {integrity: sha512-7/96wE+ZsKB35iS9ASyllrE4Ym/EolXEB7AkuJ5FI++fmS85BVTAs77890C+1Z2jwHfBKjBQSBmsliOsAh0iFQ==}
- engines: {node: '>=22'}
- peerDependencies:
- zod: 4.3.6
-
'@ai-sdk/[email protected]':
resolution: {integrity: sha512-El0JOXXGcHFe6+owJ1UV0VgB9XtdivP8vcZni+rUIt6lt+cBHCnUdddaA6LE2avJGJ5AD0uXSY+uAvRL2tSvgw==}
engines: {node: '>=22'}
@@ -1484,6 +1478,12 @@ packages:
peerDependencies:
zod: 4.3.6
+ '@ai-sdk/[email protected]':
+ resolution: {integrity: sha512-T+/H8DCvqJoCqLhltVatS7ffA779Iyk/ZZrgNXkrgv51E0n2Cd6eIdCmtwhCGIKqtbTtwCvTmQdAMv5Z8rOGHQ==}
+ engines: {node: '>=22'}
+ peerDependencies:
+ zod: 4.3.6
+
'@ai-sdk/[email protected]':
resolution: {integrity: sha512-qZMxYJ0qqX/RfnuIaab+zp8UAeJn/ygXXAffR5I4N0n1IrvA6qBsjc8hXLmBiMV2zoXlifkacF7sEFnYnjBcqg==}
engines: {node: '>=18'}
@@ -1520,8 +1520,8 @@ packages:
resolution: {integrity: sha512-VYMBxIQdcHqbIf1j+YZlI9Ati6LZ4wJe0GGd4z4a5H/KxTggjeOiyaVYTnfF7LHZK5jMQ+rofmzz4QPqf++NUw==}
engines: {node: '>=22'}
- '@ai-sdk/[email protected]':
- resolution: {integrity: sha512-e0CpNWJUY7OxAFAnCZkw+ri9QOHWwTs1tXP42782KFGCU07qt8NiXCrCVowyCB5dP2r5/Uls+g2oPd8kOJn9dw==}
+ '@ai-sdk/[email protected]':
+ resolution: {integrity: sha512-+GZJIgz1jk86pwEbb3f1BD2bdoSKyWE4Jg4YUc7NMnMozbWemSKYZcw2F4nMaO5qwsL5A8RmioAKW81YktRpIQ==}
engines: {node: '>=22'}
'@ai-sdk/[email protected]':
@@ -12765,12 +12765,12 @@ snapshots:
aws4fetch: 1.0.20
zod: 4.3.6
dependencies:
- '@ai-sdk/anthropic': 4.0.16([email protected])
- '@ai-sdk/openai': 4.0.16([email protected])
- '@ai-sdk/provider': 4.0.3
- '@ai-sdk/provider-utils': 5.0.11([email protected])
+ '@ai-sdk/anthropic': 4.0.65([email protected])
+ '@ai-sdk/openai': 4.0.78([email protected])
+ '@ai-sdk/provider': 4.0.18
+ '@ai-sdk/provider-utils': 5.0.49([email protected])
'@smithy/eventstream-codec': 4.4.14
'@smithy/util-utf8': 4.4.14
aws4fetch: 1.0.20
@@ -12788,18 +12788,18 @@ snapshots:
'@ai-sdk/provider-utils': 4.0.33([email protected])
zod: 4.3.6
- dependencies:
- '@ai-sdk/provider': 4.0.3
- '@ai-sdk/provider-utils': 5.0.11([email protected])
- zod: 4.3.6
-
dependencies:
'@ai-sdk/provider': 4.0.1
'@ai-sdk/provider-utils': 5.0.3([email protected])
zod: 4.3.6
+ dependencies:
+ '@ai-sdk/provider': 4.0.18
+ '@ai-sdk/provider-utils': 5.0.49([email protected])
+ zod: 4.3.6
+
dependencies:
'@ai-sdk/deepseek': 3.0.3([email protected])
@@ -12882,18 +12882,18 @@ snapshots:
'@ai-sdk/provider-utils': 4.0.46([email protected])
zod: 4.3.6
- dependencies:
- '@ai-sdk/provider': 4.0.3
- '@ai-sdk/provider-utils': 5.0.11([email protected])
- zod: 4.3.6
-
dependencies:
'@ai-sdk/provider': 4.0.1
'@ai-sdk/provider-utils': 5.0.3([email protected])
zod: 4.3.6
+ dependencies:
+ '@ai-sdk/provider': 4.0.18
+ '@ai-sdk/provider-utils': 5.0.49([email protected])
+ zod: 4.3.6
+
dependencies:
'@ai-sdk/provider': 1.1.3
@@ -12923,25 +12923,26 @@ snapshots:
undici: 6.28.1
zod: 4.3.6
dependencies:
- '@ai-sdk/provider': 4.0.3
+ '@ai-sdk/provider': 4.0.1
'@standard-schema/spec': 1.1.0
'@workflow/serde': 4.1.0
eventsource-parser: 3.1.0
zod: 4.3.6
dependencies:
- '@ai-sdk/provider': 4.0.1
+ '@ai-sdk/provider': 4.0.17
'@standard-schema/spec': 1.1.0
'@workflow/serde': 4.1.0
eventsource-parser: 3.1.0
+ undici: 7.29.1
zod: 4.3.6
dependencies:
- '@ai-sdk/provider': 4.0.17
+ '@ai-sdk/provider': 4.0.18
'@standard-schema/spec': 1.1.0
'@workflow/serde': 4.1.0
eventsource-parser: 3.1.0
@@ -12984,7 +12985,7 @@ snapshots:
dependencies:
json-schema: 0.4.0
- '@ai-sdk/[email protected]':
+ '@ai-sdk/[email protected]':
dependencies:
json-schema: 0.4.0
diff --git a/web/src/__tests__/server/unit/evaluator-preflight.servertest.ts b/web/src/__tests__/server/unit/evaluator-preflight.servertest.ts
index 76b86f1358..13321f6bde 100644
--- a/web/src/__tests__/server/unit/evaluator-preflight.servertest.ts
+++ b/web/src/__tests__/server/unit/evaluator-preflight.servertest.ts
@@ -227,24 +227,124 @@ describe("evaluator preflight", () => {
);
});
- it("does not expose unknown model call errors", async () => {
+ it("rethrows retryable provider failures", async () => {
+ const providerError = new Error("429 Rate limit reached");
+ mockTestModelCall.mockRejectedValue(providerError);
+ mockGetLLMErrorInfo.mockReturnValue({
+ kind: "provider",
+ message: "429 Rate limit reached",
+ statusCode: 429,
+ isRetryable: true,
+ error: providerError,
+ });
+
+ await expect(
+ getEvaluatorDefinitionPreflightError(
+ {
+ projectId: "project_test",
+ template: {
+ name: "Answer correctness",
+ outputDefinition: numericOutputDefinition,
+ },
+ },
+ {
+ throwOnOperationalError: true,
+ },
+ ),
+ ).rejects.toBe(providerError);
+ });
+
+ it("returns retryable provider failures for activation callers", async () => {
+ const providerError = new Error("429 Rate limit reached");
+ mockTestModelCall.mockRejectedValue(providerError);
+ mockGetLLMErrorInfo.mockReturnValue({
+ kind: "provider",
+ message: "429 Rate limit reached",
+ statusCode: 429,
+ isRetryable: true,
+ error: providerError,
+ });
+
+ await expect(
+ getEvaluatorDefinitionPreflightError({
+ projectId: "project_test",
+ template: {
+ name: "Answer correctness",
+ outputDefinition: numericOutputDefinition,
+ },
+ }),
+ ).resolves.toBe(
+ `Model configuration not valid for evaluator "Answer correctness". 429 Rate limit reached`,
+ );
+ });
+
+ it.each(["timeout", "abort"] as const)(
+ "rethrows %s failures",
+ async (kind) => {
+ const operationalError = new Error(`${kind} during model call`);
+ mockTestModelCall.mockRejectedValue(operationalError);
+ mockGetLLMErrorInfo.mockReturnValue({
+ kind,
+ message: operationalError.message,
+ isRetryable: false,
+ error: operationalError,
+ });
+
+ await expect(
+ getEvaluatorDefinitionPreflightError(
+ {
+ projectId: "project_test",
+ template: {
+ name: "Answer correctness",
+ outputDefinition: numericOutputDefinition,
+ },
+ },
+ {
+ throwOnOperationalError: true,
+ },
+ ),
+ ).rejects.toBe(operationalError);
+ },
+ );
+
+ it("rethrows unknown model call errors without exposing them", async () => {
+ const unknownError = new Error("sensitive internal model call detail");
+ mockTestModelCall.mockRejectedValue(unknownError);
+ mockGetLLMErrorInfo.mockReturnValue(null);
+
+ await expect(
+ getEvaluatorDefinitionPreflightError(
+ {
+ projectId: "project_test",
+ template: {
+ name: "Answer correctness",
+ outputDefinition: numericOutputDefinition,
+ },
+ },
+ {
+ throwOnOperationalError: true,
+ },
+ ),
+ ).rejects.toBe(unknownError);
+ });
+
+ it("hides unknown model call errors from activation callers", async () => {
mockTestModelCall.mockRejectedValue(
new Error("sensitive internal model call detail"),
);
mockGetLLMErrorInfo.mockReturnValue(null);
- const result = await getEvaluatorDefinitionPreflightError({
- projectId: "project_test",
- template: {
- name: "Answer correctness",
- outputDefinition: numericOutputDefinition,
- },
- });
-
- expect(result).toBe(
+ await expect(
+ getEvaluatorDefinitionPreflightError({
+ projectId: "project_test",
+ template: {
+ name: "Answer correctness",
+ outputDefinition: numericOutputDefinition,
+ },
+ }),
+ ).resolves.toBe(
`Model configuration not valid for evaluator "Answer correctness". An internal error occurred`,
);
- expect(result).not.toContain("sensitive internal model call detail");
});
});
});
diff --git a/web/src/__tests__/server/unit/evaluator-v2-validation.servertest.ts b/web/src/__tests__/server/unit/evaluator-v2-validation.servertest.ts
index 0b4f081329..e04f8cb149 100644
--- a/web/src/__tests__/server/unit/evaluator-v2-validation.servertest.ts
+++ b/web/src/__tests__/server/unit/evaluator-v2-validation.servertest.ts
@@ -2,14 +2,14 @@ import { EvalTemplateType } from "@langfuse/shared";
import { beforeEach, describe, expect, it, vi } from "vitest";
const mocks = vi.hoisted(() => ({
- getEvaluatorDefinitionConfigurationError: vi.fn(),
+ getEvaluatorDefinitionPreflightError: vi.fn(),
isCodeEvalEnabled: vi.fn(),
isCodeEvalSourceCodeLanguageSupported: vi.fn(),
}));
vi.mock("@/src/features/evals/server/evaluator-preflight", () => ({
- getEvaluatorDefinitionConfigurationError:
- mocks.getEvaluatorDefinitionConfigurationError,
+ getEvaluatorDefinitionPreflightError:
+ mocks.getEvaluatorDefinitionPreflightError,
}));
vi.mock("@/src/features/evals/server/isCodeEvalEnabled", () => ({
@@ -29,7 +29,7 @@ describe("evaluator configuration validation", () => {
vi.resetAllMocks();
mocks.isCodeEvalEnabled.mockReturnValue(true);
mocks.isCodeEvalSourceCodeLanguageSupported.mockReturnValue(true);
- mocks.getEvaluatorDefinitionConfigurationError.mockResolvedValue(null);
+ mocks.getEvaluatorDefinitionPreflightError.mockResolvedValue(null);
});
it("accepts evaluator names longer than 200 characters", () => {
@@ -90,7 +90,7 @@ describe("evaluator configuration validation", () => {
).toBe(true);
});
- it("validates deployment capabilities without executing the evaluator", async () => {
+ it("validates deployment capabilities and the selected evaluator model", async () => {
const codeDefinition = {
type: EvalTemplateType.CODE,
sourceCode: "return 1;",
@@ -136,9 +136,35 @@ describe("evaluator configuration validation", () => {
}),
).resolves.toBeUndefined();
- expect(
- mocks.getEvaluatorDefinitionConfigurationError,
- ).toHaveBeenCalledOnce();
+ expect(mocks.getEvaluatorDefinitionPreflightError).toHaveBeenCalledOnce();
+ });
+
+ it("propagates transient model preflight failures", async () => {
+ const transientError = new Error("Provider rate limit");
+ mocks.getEvaluatorDefinitionPreflightError.mockRejectedValue(
+ transientError,
+ );
+
+ await expect(
+ assertEvaluatorConfigurationValid({
+ projectId: "project-id",
+ name: "LLM evaluator",
+ definition: {
+ type: EvalTemplateType.LLM_AS_JUDGE,
+ promptMessages: [{ role: "user", content: "Judge {{output}}" }],
+ provider: null,
+ model: null,
+ modelParams: null,
+ vars: ["output"],
+ variableMapping: null,
+ outputDefinition: {
+ dataType: "NUMERIC",
+ score: { description: "Quality" },
+ reasoning: { description: "Reasoning" },
+ },
+ },
+ }),
+ ).rejects.toBe(transientError);
});
// The schema is the only boundary that can see a caller-supplied mapping:
diff --git a/web/src/features/evals/server/evaluator-preflight.ts b/web/src/features/evals/server/evaluator-preflight.ts
index 6df33f7699..fa37c77dd0 100644
--- a/web/src/features/evals/server/evaluator-preflight.ts
+++ b/web/src/features/evals/server/evaluator-preflight.ts
@@ -70,10 +70,15 @@ export async function getEvaluatorDefinitionConfigurationError(params: {
return prepared.valid ? null : prepared.error;
}
-export async function getEvaluatorDefinitionPreflightError(params: {
- projectId: string;
- template: EvaluatorPreflightDefinition;
-}): Promise<string | null> {
+export async function getEvaluatorDefinitionPreflightError(
+ params: {
+ projectId: string;
+ template: EvaluatorPreflightDefinition;
+ },
+ options?: {
+ throwOnOperationalError?: boolean;
+ },
+): Promise<string | null> {
if (params.template.type === EvalTemplateType.CODE) return null;
const prepared = await prepareEvaluatorDefinition(params);
@@ -101,6 +106,14 @@ export async function getEvaluatorDefinitionPreflightError(params: {
});
} catch (err) {
const llmError = getLLMErrorInfo(err);
+ const isOperationalError =
+ !llmError ||
+ llmError.isRetryable ||
+ llmError.kind === "timeout" ||
+ llmError.kind === "abort";
+ if (isOperationalError && options?.throwOnOperationalError) {
+ throw err;
+ }
// A provider 404 also covers typos, missing model access, and bad base
// URLs — not just retired models, so don't claim "retired" as fact.
if (llmError?.statusCode === 404) {
diff --git a/web/src/features/evals/v2/server/evaluators/evaluatorValidation.ts b/web/src/features/evals/v2/server/evaluators/evaluatorValidation.ts
index 3b072f8157..0f9a8ff33e 100644
--- a/web/src/features/evals/v2/server/evaluators/evaluatorValidation.ts
+++ b/web/src/features/evals/v2/server/evaluators/evaluatorValidation.ts
@@ -8,7 +8,7 @@ import {
DefaultEvalModelService,
isDecisionModelAdapter,
} from "@langfuse/shared/src/server";
-import { getEvaluatorDefinitionConfigurationError } from "@/src/features/evals/server/evaluator-preflight";
+import { getEvaluatorDefinitionPreflightError } from "@/src/features/evals/server/evaluator-preflight";
import { getPromptMessagesValidationError } from "@/src/features/evals/v2/fns/promptMessages/hasInvalidSystemPromptMessage";
import {
isCodeEvalEnabled,
@@ -167,17 +167,20 @@ export async function assertEvaluatorConfigurationValid(params: {
}
}
- const error = await getEvaluatorDefinitionConfigurationError({
- projectId: params.projectId,
- template: {
- name: params.name,
- type: params.definition.type,
- provider: params.definition.provider,
- model: params.definition.model,
- modelParams: params.definition.modelParams,
- outputDefinition: params.definition.outputDefinition,
+ const error = await getEvaluatorDefinitionPreflightError(
+ {
+ projectId: params.projectId,
+ template: {
+ name: params.name,
+ type: params.definition.type,
+ provider: params.definition.provider,
+ model: params.definition.model,
+ modelParams: params.definition.modelParams,
+ outputDefinition: params.definition.outputDefinition,
+ },
},
- });
+ { throwOnOperationalError: true },
+ );
if (error) throw new EvaluatorModelConfigurationError(error);
}