🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
import {
compilePersistedEvalOutputDefinition,
EvalTemplateType,
PersistedEvalOutputDefinitionSchema,
} from "@langfuse/shared";
import {
DefaultEvalModelService,
getClientInitiatedNonStreamingLlmTimeoutMs,
getLLMErrorInfo,
testModelCall,
} from "@langfuse/shared/src/server";
export type EvaluatorPreflightDefinition = {
name: string;
type?: EvalTemplateType | null;
provider?: string | null;
model?: string | null;
modelParams?: unknown;
outputDefinition: unknown;
};
async function prepareEvaluatorDefinition(params: {
projectId: string;
template: EvaluatorPreflightDefinition;
}) {
const modelConfig = await DefaultEvalModelService.fetchValidModelConfig(
params.projectId,
params.template.provider ?? undefined,
params.template.model ?? undefined,
params.template.modelParams,
);
if (!modelConfig.valid) {
return {
valid: false as const,
error: `No valid LLM model found for evaluator "${params.template.name}". ${modelConfig.error}. Configure an LLM connection for this project under Settings → LLM Connections (/project/${params.projectId}/settings/llm-connections) before creating llm_as_judge evaluators.`,
};
}
try {
const parsedOutputDefinition = PersistedEvalOutputDefinitionSchema.parse(
params.template.outputDefinition,
);
return {
valid: true as const,
modelConfig: modelConfig.config,
outputResultSchema: compilePersistedEvalOutputDefinition(
parsedOutputDefinition,
).outputResultSchema,
};
} catch (err) {
const message =
err instanceof Error
? err.message
: "Invalid evaluator output definition";
return {
valid: false as const,
error: `Model configuration not valid for evaluator "${params.template.name}". ${message}`,
};
}
}
export async function getEvaluatorDefinitionConfigurationError(params: {
projectId: string;
template: EvaluatorPreflightDefinition;
}): Promise<string | null> {
if (params.template.type === EvalTemplateType.CODE) return null;
const prepared = await prepareEvaluatorDefinition(params);
return prepared.valid ? null : prepared.error;
}
export async function getEvaluatorDefinitionPreflightError(
params: {
projectId: string;
template: EvaluatorPreflightDefinition;
},
options?: {
throwOnOperationalError?: boolean;
},
): Promise<string | null> {
if (params.template.type === EvalTemplateType.CODE) return null;
const prepared = await prepareEvaluatorDefinition(params);
if (!prepared.valid) return prepared.error;
// Some test environments run a built app against seeded local data. In
// those cases we still want to validate model selection and schema
// compilation without depending on live provider credentials.
if (
process.env.LANGFUSE_SKIP_EVALUATOR_MODEL_CALL_VALIDATION === "true" ||
process.env.NODE_ENV === "test" ||
process.env.DATABASE_URL?.includes("langfuse_test")
) {
return null;
}
try {
await testModelCall({
provider: prepared.modelConfig.provider,
model: prepared.modelConfig.model,
apiKey: prepared.modelConfig.apiKey,
modelConfig: prepared.modelConfig.modelParams,
structuredOutputSchema: prepared.outputResultSchema,
timeout: getClientInitiatedNonStreamingLlmTimeoutMs(),
});
} catch (err) {
const llmError = getLLMErrorInfo(err);
const isOperationalError =
!llmError ||
llmError.isRetryable ||
llmError.kind === "timeout" ||
llmError.kind === "abort";
if (isOperationalError && options?.throwOnOperationalError) {
throw err;
}
// A provider 404 also covers typos, missing model access, and bad base
// URLs — not just retired models, so don't claim "retired" as fact.
if (llmError?.statusCode === 404) {
return `Model configuration not valid for evaluator "${params.template.name}". The provider could not find model '${prepared.modelConfig.model}' — it may be retired, misspelled, or not available to your API key. Update the evaluator's model or the project's default evaluation model.`;
}
const message = llmError?.message ?? "An internal error occurred";
return `Model configuration not valid for evaluator "${params.template.name}". ${message}`;
}
return null;
}