🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
vi.mock("@langfuse/shared/src/server", async () => ({
...(await vi.importActual("@langfuse/shared/src/server")),
DefaultEvalModelService: {
fetchValidModelConfig: vi.fn(),
},
logger: {
debug: vi.fn(),
},
getLLMErrorInfo: vi.fn(),
testModelCall: vi.fn(),
}));
import {
createNumericEvalOutputDefinition,
LLMAdapter,
} from "@langfuse/shared";
import {
DefaultEvalModelService,
getLLMErrorInfo,
testModelCall,
} from "@langfuse/shared/src/server";
import {
getEvaluatorDefinitionConfigurationError,
getEvaluatorDefinitionPreflightError,
} from "@/src/features/evals/server/evaluator-preflight";
const numericOutputDefinition = createNumericEvalOutputDefinition({
reasoningDescription: "Why the score was assigned",
scoreDescription: "A score between 0 and 1",
});
// The real config carries a full parsed LLM API key record plus the adapter;
// the preflight only reads the connection fields stubbed below, so the
// fixture stays minimal behind a cast.
type ValidModelConfig = Extract<
Awaited<ReturnType<typeof DefaultEvalModelService.fetchValidModelConfig>>,
{ valid: true }
>["config"];
describe("evaluator preflight", () => {
const mockFetchValidModelConfig = vi.mocked(
DefaultEvalModelService.fetchValidModelConfig,
);
const mockGetLLMErrorInfo = vi.mocked(getLLMErrorInfo);
const mockTestModelCall = vi.mocked(testModelCall);
const originalSkipFlag =
process.env.LANGFUSE_SKIP_EVALUATOR_MODEL_CALL_VALIDATION;
beforeEach(() => {
vi.resetAllMocks();
delete process.env.LANGFUSE_SKIP_EVALUATOR_MODEL_CALL_VALIDATION;
mockFetchValidModelConfig.mockResolvedValue({
valid: true,
config: {
provider: "openai-test",
model: "gpt-4.1-mini",
modelParams: undefined,
apiKey: {
secretKey: "encrypted",
extraHeaders: null,
baseURL: "http://127.0.0.1:4011/v1",
config: null,
adapter: LLMAdapter.OpenAI,
},
} as unknown as ValidModelConfig,
});
});
afterAll(() => {
if (originalSkipFlag === undefined) {
delete process.env.LANGFUSE_SKIP_EVALUATOR_MODEL_CALL_VALIDATION;
return;
}
process.env.LANGFUSE_SKIP_EVALUATOR_MODEL_CALL_VALIDATION =
originalSkipFlag;
});
it("points to the LLM connection settings when no valid model is configured", async () => {
mockFetchValidModelConfig.mockResolvedValue({
valid: false,
error: "No default model or custom model configured for project p1",
});
const result = await getEvaluatorDefinitionPreflightError({
projectId: "p1",
template: {
name: "Answer correctness",
outputDefinition: numericOutputDefinition,
},
});
expect(result).toBe(
`No valid LLM model found for evaluator "Answer correctness". No default model or custom model configured for project p1. Configure an LLM connection for this project under Settings → LLM Connections (/project/p1/settings/llm-connections) before creating llm_as_judge evaluators.`,
);
});
it("skips the live provider call when the explicit test flag is enabled", async () => {
process.env.LANGFUSE_SKIP_EVALUATOR_MODEL_CALL_VALIDATION = "true";
const result = await getEvaluatorDefinitionPreflightError({
projectId: "project_test",
template: {
name: "Answer correctness",
outputDefinition: numericOutputDefinition,
},
});
expect(result).toBeNull();
expect(mockTestModelCall).not.toHaveBeenCalled();
});
it("validates evaluator configuration without calling the provider", async () => {
const result = await getEvaluatorDefinitionConfigurationError({
projectId: "project_test",
template: {
name: "Answer correctness",
outputDefinition: numericOutputDefinition,
},
});
expect(result).toBeNull();
expect(mockFetchValidModelConfig).toHaveBeenCalledOnce();
expect(mockTestModelCall).not.toHaveBeenCalled();
});
it("preserves actionable output definition validation errors", async () => {
const result = await getEvaluatorDefinitionPreflightError({
projectId: "project_test",
template: {
name: "Answer correctness",
outputDefinition: {
version: 2,
dataType: "CATEGORICAL",
reasoning: { description: "Why the score was assigned" },
score: {
description: "The matching category",
categories: [],
shouldAllowMultipleMatches: false,
},
},
},
});
expect(result).toContain("Add at least 2 categories");
expect(mockTestModelCall).not.toHaveBeenCalled();
expect(mockGetLLMErrorInfo).not.toHaveBeenCalled();
});
describe("live provider calls", () => {
// The preflight skips the provider call in test-like environments;
// stub these so the mocked testModelCall is actually reached.
beforeEach(() => {
vi.stubEnv("NODE_ENV", "production");
vi.stubEnv("DATABASE_URL", "postgresql://localhost:5432/postgres");
});
afterEach(() => {
vi.unstubAllEnvs();
});
it("limits the non-streaming model call to 95 seconds", async () => {
const result = await getEvaluatorDefinitionPreflightError({
projectId: "project_test",
template: {
name: "Answer correctness",
outputDefinition: numericOutputDefinition,
},
});
expect(result).toBeNull();
expect(mockTestModelCall).toHaveBeenCalledWith(
expect.objectContaining({ timeout: 95_000 }),
);
});
it("returns a clean model-not-found message when the provider responds with 404", async () => {
mockTestModelCall.mockRejectedValue(
new Error(
'404 {"type":"error","error":{"type":"not_found_error","message":"model: claude-3-sonnet-20240229"}}',
),
);
mockGetLLMErrorInfo.mockReturnValue({
kind: "provider",
message:
'404 {"type":"error","error":{"type":"not_found_error","message":"model: claude-3-sonnet-20240229"}}',
statusCode: 404,
isRetryable: false,
error: new Error("provider error"),
});
const result = await getEvaluatorDefinitionPreflightError({
projectId: "project_test",
template: {
name: "Answer correctness",
outputDefinition: numericOutputDefinition,
},
});
expect(result).toBe(
`Model configuration not valid for evaluator "Answer correctness". The provider could not find model 'gpt-4.1-mini' — it may be retired, misspelled, or not available to your API key. Update the evaluator's model or the project's default evaluation model.`,
);
});
it("keeps the provider error message for non-404 failures", async () => {
mockTestModelCall.mockRejectedValue(
new Error("401 Incorrect API key provided"),
);
mockGetLLMErrorInfo.mockReturnValue({
kind: "provider",
message: "401 Incorrect API key provided",
statusCode: 401,
isRetryable: false,
error: new Error("provider error"),
});
const result = await getEvaluatorDefinitionPreflightError({
projectId: "project_test",
template: {
name: "Answer correctness",
outputDefinition: numericOutputDefinition,
},
});
expect(result).toBe(
`Model configuration not valid for evaluator "Answer correctness". 401 Incorrect API key provided`,
);
});
it("rethrows retryable provider failures", async () => {
const providerError = new Error("429 Rate limit reached");
mockTestModelCall.mockRejectedValue(providerError);
mockGetLLMErrorInfo.mockReturnValue({
kind: "provider",
message: "429 Rate limit reached",
statusCode: 429,
isRetryable: true,
error: providerError,
});
await expect(
getEvaluatorDefinitionPreflightError(
{
projectId: "project_test",
template: {
name: "Answer correctness",
outputDefinition: numericOutputDefinition,
},
},
{
throwOnOperationalError: true,
},
),
).rejects.toBe(providerError);
});
it("returns retryable provider failures for activation callers", async () => {
const providerError = new Error("429 Rate limit reached");
mockTestModelCall.mockRejectedValue(providerError);
mockGetLLMErrorInfo.mockReturnValue({
kind: "provider",
message: "429 Rate limit reached",
statusCode: 429,
isRetryable: true,
error: providerError,
});
await expect(
getEvaluatorDefinitionPreflightError({
projectId: "project_test",
template: {
name: "Answer correctness",
outputDefinition: numericOutputDefinition,
},
}),
).resolves.toBe(
`Model configuration not valid for evaluator "Answer correctness". 429 Rate limit reached`,
);
});
it.each(["timeout", "abort"] as const)(
"rethrows %s failures",
async (kind) => {
const operationalError = new Error(`${kind} during model call`);
mockTestModelCall.mockRejectedValue(operationalError);
mockGetLLMErrorInfo.mockReturnValue({
kind,
message: operationalError.message,
isRetryable: false,
error: operationalError,
});
await expect(
getEvaluatorDefinitionPreflightError(
{
projectId: "project_test",
template: {
name: "Answer correctness",
outputDefinition: numericOutputDefinition,
},
},
{
throwOnOperationalError: true,
},
),
).rejects.toBe(operationalError);
},
);
it("rethrows unknown model call errors without exposing them", async () => {
const unknownError = new Error("sensitive internal model call detail");
mockTestModelCall.mockRejectedValue(unknownError);
mockGetLLMErrorInfo.mockReturnValue(null);
await expect(
getEvaluatorDefinitionPreflightError(
{
projectId: "project_test",
template: {
name: "Answer correctness",
outputDefinition: numericOutputDefinition,
},
},
{
throwOnOperationalError: true,
},
),
).rejects.toBe(unknownError);
});
it("hides unknown model call errors from activation callers", async () => {
mockTestModelCall.mockRejectedValue(
new Error("sensitive internal model call detail"),
);
mockGetLLMErrorInfo.mockReturnValue(null);
await expect(
getEvaluatorDefinitionPreflightError({
projectId: "project_test",
template: {
name: "Answer correctness",
outputDefinition: numericOutputDefinition,
},
}),
).resolves.toBe(
`Model configuration not valid for evaluator "Answer correctness". An internal error occurred`,
);
});
});
});