Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

4.2 KB
Raw Blame
import {
  compilePersistedEvalOutputDefinition,
  EvalTemplateType,
  PersistedEvalOutputDefinitionSchema,
} from "@langfuse/shared";
import {
  DefaultEvalModelService,
  getClientInitiatedNonStreamingLlmTimeoutMs,
  getLLMErrorInfo,
  testModelCall,
} from "@langfuse/shared/src/server";

export type EvaluatorPreflightDefinition = {
  name: string;
  type?: EvalTemplateType | null;
  provider?: string | null;
  model?: string | null;
  modelParams?: unknown;
  outputDefinition: unknown;
};

async function prepareEvaluatorDefinition(params: {
  projectId: string;
  template: EvaluatorPreflightDefinition;
}) {
  const modelConfig = await DefaultEvalModelService.fetchValidModelConfig(
    params.projectId,
    params.template.provider ?? undefined,
    params.template.model ?? undefined,
    params.template.modelParams,
  );

  if (!modelConfig.valid) {
    return {
      valid: false as const,
      error: `No valid LLM model found for evaluator "${params.template.name}". ${modelConfig.error}. Configure an LLM connection for this project under Settings → LLM Connections (/project/${params.projectId}/settings/llm-connections) before creating llm_as_judge evaluators.`,
    };
  }

  try {
    const parsedOutputDefinition = PersistedEvalOutputDefinitionSchema.parse(
      params.template.outputDefinition,
    );
    return {
      valid: true as const,
      modelConfig: modelConfig.config,
      outputResultSchema: compilePersistedEvalOutputDefinition(
        parsedOutputDefinition,
      ).outputResultSchema,
    };
  } catch (err) {
    const message =
      err instanceof Error
        ? err.message
        : "Invalid evaluator output definition";
    return {
      valid: false as const,
      error: `Model configuration not valid for evaluator "${params.template.name}". ${message}`,
    };
  }
}

export async function getEvaluatorDefinitionConfigurationError(params: {
  projectId: string;
  template: EvaluatorPreflightDefinition;
}): Promise<string | null> {
  if (params.template.type === EvalTemplateType.CODE) return null;

  const prepared = await prepareEvaluatorDefinition(params);
  return prepared.valid ? null : prepared.error;
}

export async function getEvaluatorDefinitionPreflightError(
  params: {
    projectId: string;
    template: EvaluatorPreflightDefinition;
  },
  options?: {
    throwOnOperationalError?: boolean;
  },
): Promise<string | null> {
  if (params.template.type === EvalTemplateType.CODE) return null;

  const prepared = await prepareEvaluatorDefinition(params);
  if (!prepared.valid) return prepared.error;

  // Some test environments run a built app against seeded local data. In
  // those cases we still want to validate model selection and schema
  // compilation without depending on live provider credentials.
  if (
    process.env.LANGFUSE_SKIP_EVALUATOR_MODEL_CALL_VALIDATION === "true" ||
    process.env.NODE_ENV === "test" ||
    process.env.DATABASE_URL?.includes("langfuse_test")
  ) {
    return null;
  }

  try {
    await testModelCall({
      provider: prepared.modelConfig.provider,
      model: prepared.modelConfig.model,
      apiKey: prepared.modelConfig.apiKey,
      modelConfig: prepared.modelConfig.modelParams,
      structuredOutputSchema: prepared.outputResultSchema,
      timeout: getClientInitiatedNonStreamingLlmTimeoutMs(),
    });
  } catch (err) {
    const llmError = getLLMErrorInfo(err);
    const isOperationalError =
      !llmError ||
      llmError.isRetryable ||
      llmError.kind === "timeout" ||
      llmError.kind === "abort";
    if (isOperationalError && options?.throwOnOperationalError) {
      throw err;
    }
    // A provider 404 also covers typos, missing model access, and bad base
    // URLs — not just retired models, so don't claim "retired" as fact.
    if (llmError?.statusCode === 404) {
      return `Model configuration not valid for evaluator "${params.template.name}". The provider could not find model '${prepared.modelConfig.model}' — it may be retired, misspelled, or not available to your API key. Update the evaluator's model or the project's default evaluation model.`;
    }
    const message = llmError?.message ?? "An internal error occurred";
    return `Model configuration not valid for evaluator "${params.template.name}". ${message}`;
  }

  return null;
}