🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org
Python agentai-societiesartificial-intelligencecommunicative-aicooperative-aideep-learninglarge-language-modelsmulti-agent-systemsnatural-language-processing
feat: Support customized clients in model backends (#3317)
Wendong-Fanac475e4eca2d
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yamlindex 9509f01ce..38fa0cf6c 100644--- a/.pre-commit-config.yaml+++ b/.pre-commit-config.yaml@@ -4,7 +4,7 @@ repos:hooks:- id: ruffargs: [--fix, --exit-non-zero-on-fix, --show-fixes]- exclude: ^(docs/cookbooks/|examples/usecases/) # Ignore files under docs/cookbooks and examples/usecases+ exclude: ^(docs/cookbooks/|examples/usecases/|examples/custom_client_usage\.py) # Ignore files under docs/cookbooks and examples/usecases- id: ruff-formatexclude: ^(docs/cookbooks/|examples/usecases/) # Ignore files under docs/cookbooks and examples/usecasesdiff --git a/camel/models/azure_openai_model.py b/camel/models/azure_openai_model.pyindex b7cef4218..93f7acc3b 100644--- a/camel/models/azure_openai_model.py+++ b/camel/models/azure_openai_model.py@@ -88,8 +88,19 @@ class AzureOpenAIModel(BaseModelBackend):(default: :obj:`None`)max_retries (int, optional): Maximum number of retries for API calls.(default: :obj:`3`)+ client (Optional[Any], optional): A custom synchronous AzureOpenAI+ client instance. If provided, this client will be used instead of+ creating a new one. Useful for RL frameworks like AReaL or rLLM+ that provide Azure OpenAI-compatible clients. The client should+ implement the AzureOpenAI client interface with+ `.chat.completions.create()` and `.beta.chat.completions.parse()`+ methods. (default: :obj:`None`)+ async_client (Optional[Any], optional): A custom asynchronous+ AzureOpenAI client instance. If provided, this client will be+ used instead of creating a new one. The client should implement+ the AsyncAzureOpenAI client interface. (default: :obj:`None`)**kwargs (Any): Additional arguments to pass to the client- initialization.+ initialization. Ignored if custom clients are provided.References:https://learn.microsoft.com/en-us/azure/ai-services/openai/@@ -108,6 +119,8 @@ class AzureOpenAIModel(BaseModelBackend):azure_ad_token_provider: Optional["AzureADTokenProvider"] = None,azure_ad_token: Optional[str] = None,max_retries: int = 3,+ client: Optional[Any] = None,+ async_client: Optional[Any] = None,**kwargs: Any,) -> None:if model_config_dict is None:@@ -138,56 +151,72 @@ class AzureOpenAIModel(BaseModelBackend):"or `AZURE_DEPLOYMENT_NAME` environment variable.")- if is_langfuse_available():- from langfuse.openai import AsyncAzureOpenAI as LangfuseAsyncOpenAI- from langfuse.openai import AzureOpenAI as LangfuseOpenAI-- self._client = LangfuseOpenAI(- azure_endpoint=str(self._url),- azure_deployment=self._azure_deployment_name,- api_version=self.api_version,- api_key=self._api_key,- azure_ad_token=self._azure_ad_token,- azure_ad_token_provider=self.azure_ad_token_provider,- timeout=self._timeout,- max_retries=max_retries,- **kwargs,- )- self._async_client = LangfuseAsyncOpenAI(- azure_endpoint=str(self._url),- azure_deployment=self._azure_deployment_name,- api_version=self.api_version,- api_key=self._api_key,- azure_ad_token=self._azure_ad_token,- azure_ad_token_provider=self.azure_ad_token_provider,- timeout=self._timeout,- max_retries=max_retries,- **kwargs,- )+ # Use custom clients if provided, otherwise create new ones+ if client is not None:+ # Use the provided custom sync client+ self._client = clientelse:- self._client = AzureOpenAI(- azure_endpoint=str(self._url),- azure_deployment=self._azure_deployment_name,- api_version=self.api_version,- api_key=self._api_key,- azure_ad_token=self._azure_ad_token,- azure_ad_token_provider=self.azure_ad_token_provider,- timeout=self._timeout,- max_retries=max_retries,- **kwargs,- )+ # Create default sync client+ if is_langfuse_available():+ from langfuse.openai import AzureOpenAI as LangfuseOpenAI++ self._client = LangfuseOpenAI(+ azure_endpoint=str(self._url),+ azure_deployment=self._azure_deployment_name,+ api_version=self.api_version,+ api_key=self._api_key,+ azure_ad_token=self._azure_ad_token,+ azure_ad_token_provider=self.azure_ad_token_provider,+ timeout=self._timeout,+ max_retries=max_retries,+ **kwargs,+ )+ else:+ self._client = AzureOpenAI(+ azure_endpoint=str(self._url),+ azure_deployment=self._azure_deployment_name,+ api_version=self.api_version,+ api_key=self._api_key,+ azure_ad_token=self._azure_ad_token,+ azure_ad_token_provider=self.azure_ad_token_provider,+ timeout=self._timeout,+ max_retries=max_retries,+ **kwargs,+ )- self._async_client = AsyncAzureOpenAI(- azure_endpoint=str(self._url),- azure_deployment=self._azure_deployment_name,- api_version=self.api_version,- api_key=self._api_key,- azure_ad_token=self._azure_ad_token,- azure_ad_token_provider=self.azure_ad_token_provider,- timeout=self._timeout,- max_retries=max_retries,- **kwargs,- )+ if async_client is not None:+ # Use the provided custom async client+ self._async_client = async_client+ else:+ # Create default async client+ if is_langfuse_available():+ from langfuse.openai import (+ AsyncAzureOpenAI as LangfuseAsyncOpenAI,+ )++ self._async_client = LangfuseAsyncOpenAI(+ azure_endpoint=str(self._url),+ azure_deployment=self._azure_deployment_name,+ api_version=self.api_version,+ api_key=self._api_key,+ azure_ad_token=self._azure_ad_token,+ azure_ad_token_provider=self.azure_ad_token_provider,+ timeout=self._timeout,+ max_retries=max_retries,+ **kwargs,+ )+ else:+ self._async_client = AsyncAzureOpenAI(+ azure_endpoint=str(self._url),+ azure_deployment=self._azure_deployment_name,+ api_version=self.api_version,+ api_key=self._api_key,+ azure_ad_token=self._azure_ad_token,+ azure_ad_token_provider=self.azure_ad_token_provider,+ timeout=self._timeout,+ max_retries=max_retries,+ **kwargs,+ )@propertydef token_counter(self) -> BaseTokenCounter:diff --git a/camel/models/model_factory.py b/camel/models/model_factory.pyindex 6155ca82e..1669fce90 100644--- a/camel/models/model_factory.py+++ b/camel/models/model_factory.py@@ -13,7 +13,7 @@# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========import jsonimport os-from typing import ClassVar, Dict, Optional, Type, Union+from typing import Any, ClassVar, Dict, Optional, Type, Unionfrom camel.models.aiml_model import AIMLModelfrom camel.models.amd_model import AMDModel@@ -119,6 +119,8 @@ class ModelFactory:url: Optional[str] = None,timeout: Optional[float] = None,max_retries: int = 3,+ client: Optional[Any] = None,+ async_client: Optional[Any] = None,**kwargs,) -> BaseModelBackend:r"""Creates an instance of `BaseModelBackend` of the specified type.@@ -145,6 +147,14 @@ class ModelFactory:for API calls. (default: :obj:`None`)max_retries (int, optional): Maximum number of retriesfor API calls. (default: :obj:`3`)+ client (Optional[Any], optional): A custom synchronous client+ instance. Supported by models that use OpenAI-compatible APIs+ . The client should implement the appropriate client interface+ for the platform. (default: :obj:`None`)+ async_client (Optional[Any], optional): A custom asynchronous+ client instance. Supported by models that use OpenAI-compatible+ APIs. The client should implement the appropriate async client+ interface for the platform. (default: :obj:`None`)**kwargs: Additional model-specific parameters that will be passedto the model constructor. For example, Azure OpenAI models mayrequire `api_version`, `azure_deployment_name`,@@ -190,6 +200,12 @@ class ModelFactory:if model_class is None:raise ValueError(f"Unknown model platform `{model_platform}`")+ # Pass client and async_client via kwargs if provided+ if client is not None:+ kwargs['client'] = client+ if async_client is not None:+ kwargs['async_client'] = async_client+return model_class(model_type=model_type,model_config_dict=model_config_dict,diff --git a/camel/models/openai_compatible_model.py b/camel/models/openai_compatible_model.pyindex 14f7d8271..5c2f5413c 100644--- a/camel/models/openai_compatible_model.py+++ b/camel/models/openai_compatible_model.py@@ -78,9 +78,21 @@ class OpenAICompatibleModel(BaseModelBackend):(default: :obj:`None`)max_retries (int, optional): Maximum number of retries for API calls.(default: :obj:`3`)+ client (Optional[Any], optional): A custom synchronous+ OpenAI-compatible client instance. If provided, this client will+ be used instead of creating a new one. Useful for RL frameworks+ like AReaL or rLLM that provide OpenAI-compatible clients (e.g.,+ ArealOpenAI). The client should implement the OpenAI client+ interface with `.chat.completions.create()` and `.beta.chat.+ completions.parse()` methods. (default: :obj:`None`)+ async_client (Optional[Any], optional): A custom asynchronous+ OpenAI-compatible client instance. If provided, this client will+ be used instead of creating a new one. The client should implement+ the AsyncOpenAI client interface. (default: :obj:`None`)**kwargs (Any): Additional arguments to pass to theOpenAI client initialization. These can include parameters like'organization', 'default_headers', 'http_client', etc.+ Ignored if custom clients are provided."""def __init__(@@ -92,6 +104,8 @@ class OpenAICompatibleModel(BaseModelBackend):token_counter: Optional[BaseTokenCounter] = None,timeout: Optional[float] = None,max_retries: int = 3,+ client: Optional[Any] = None,+ async_client: Optional[Any] = None,**kwargs: Any,) -> None:api_key = api_key or os.environ.get("OPENAI_COMPATIBILITY_API_KEY")@@ -107,39 +121,55 @@ class OpenAICompatibleModel(BaseModelBackend):timeout,max_retries,)- if is_langfuse_available():- from langfuse.openai import AsyncOpenAI as LangfuseAsyncOpenAI- from langfuse.openai import OpenAI as LangfuseOpenAI-- self._client = LangfuseOpenAI(- timeout=self._timeout,- max_retries=max_retries,- base_url=self._url,- api_key=self._api_key,- **kwargs,- )- self._async_client = LangfuseAsyncOpenAI(- timeout=self._timeout,- max_retries=max_retries,- base_url=self._url,- api_key=self._api_key,- **kwargs,- )++ # Use custom clients if provided, otherwise create new ones+ if client is not None:+ # Use the provided custom sync client+ self._client = clientelse:- self._client = OpenAI(- timeout=self._timeout,- max_retries=max_retries,- base_url=self._url,- api_key=self._api_key,- **kwargs,- )- self._async_client = AsyncOpenAI(- timeout=self._timeout,- max_retries=max_retries,- base_url=self._url,- api_key=self._api_key,- **kwargs,- )+ # Create default sync client+ if is_langfuse_available():+ from langfuse.openai import OpenAI as LangfuseOpenAI++ self._client = LangfuseOpenAI(+ timeout=self._timeout,+ max_retries=max_retries,+ base_url=self._url,+ api_key=self._api_key,+ **kwargs,+ )+ else:+ self._client = OpenAI(+ timeout=self._timeout,+ max_retries=max_retries,+ base_url=self._url,+ api_key=self._api_key,+ **kwargs,+ )++ if async_client is not None:+ # Use the provided custom async client+ self._async_client = async_client+ else:+ # Create default async client+ if is_langfuse_available():+ from langfuse.openai import AsyncOpenAI as LangfuseAsyncOpenAI++ self._async_client = LangfuseAsyncOpenAI(+ timeout=self._timeout,+ max_retries=max_retries,+ base_url=self._url,+ api_key=self._api_key,+ **kwargs,+ )+ else:+ self._async_client = AsyncOpenAI(+ timeout=self._timeout,+ max_retries=max_retries,+ base_url=self._url,+ api_key=self._api_key,+ **kwargs,+ )@observe()def _run(diff --git a/camel/models/openai_model.py b/camel/models/openai_model.pyindex 216c4acb6..4c8751787 100644--- a/camel/models/openai_model.py+++ b/camel/models/openai_model.py@@ -90,9 +90,21 @@ class OpenAIModel(BaseModelBackend):(default: :obj:`None`)max_retries (int, optional): Maximum number of retries for API calls.(default: :obj:`3`)+ client (Optional[Any], optional): A custom synchronous OpenAI client+ instance. If provided, this client will be used instead of+ creating a new one. Useful for RL frameworks like AReaL or rLLM+ that provide OpenAI-compatible clients. The client should+ implement the OpenAI client interface with+ `.chat.completions.create()` and `.beta.chat.completions.parse()`+ methods. (default: :obj:`None`)+ async_client (Optional[Any], optional): A custom asynchronous OpenAI+ client instance. If provided, this client will be used instead of+ creating a new one. The client should implement the AsyncOpenAI+ client interface. (default: :obj:`None`)**kwargs (Any): Additional arguments to pass to theOpenAI client initialization. These can include parameters like'organization', 'default_headers', 'http_client', etc.+ Ignored if custom clients are provided."""@api_keys_required(@@ -109,6 +121,8 @@ class OpenAIModel(BaseModelBackend):token_counter: Optional[BaseTokenCounter] = None,timeout: Optional[float] = None,max_retries: int = 3,+ client: Optional[Any] = None,+ async_client: Optional[Any] = None,**kwargs: Any,) -> None:if model_config_dict is None:@@ -124,42 +138,54 @@ class OpenAIModel(BaseModelBackend):model_type, model_config_dict, api_key, url, token_counter, timeout)- if is_langfuse_available():- from langfuse.openai import AsyncOpenAI as LangfuseAsyncOpenAI- from langfuse.openai import OpenAI as LangfuseOpenAI-- # Create Langfuse client with base parameters and additional- # arguments- self._client = LangfuseOpenAI(- timeout=self._timeout,- max_retries=self._max_retries,- base_url=self._url,- api_key=self._api_key,- **kwargs,- )- self._async_client = LangfuseAsyncOpenAI(- timeout=self._timeout,- max_retries=self._max_retries,- base_url=self._url,- api_key=self._api_key,- **kwargs,- )+ # Use custom clients if provided, otherwise create new ones+ if client is not None:+ # Use the provided custom sync client+ self._client = clientelse:- # Create client with base parameters and additional arguments- self._client = OpenAI(- timeout=self._timeout,- max_retries=self._max_retries,- base_url=self._url,- api_key=self._api_key,- **kwargs,- )- self._async_client = AsyncOpenAI(- timeout=self._timeout,- max_retries=self._max_retries,- base_url=self._url,- api_key=self._api_key,- **kwargs,- )+ # Create default sync client+ if is_langfuse_available():+ from langfuse.openai import OpenAI as LangfuseOpenAI++ self._client = LangfuseOpenAI(+ timeout=self._timeout,+ max_retries=self._max_retries,+ base_url=self._url,+ api_key=self._api_key,+ **kwargs,+ )+ else:+ self._client = OpenAI(+ timeout=self._timeout,+ max_retries=self._max_retries,+ base_url=self._url,+ api_key=self._api_key,+ **kwargs,+ )++ if async_client is not None:+ # Use the provided custom async client+ self._async_client = async_client+ else:+ # Create default async client+ if is_langfuse_available():+ from langfuse.openai import AsyncOpenAI as LangfuseAsyncOpenAI++ self._async_client = LangfuseAsyncOpenAI(+ timeout=self._timeout,+ max_retries=self._max_retries,+ base_url=self._url,+ api_key=self._api_key,+ **kwargs,+ )+ else:+ self._async_client = AsyncOpenAI(+ timeout=self._timeout,+ max_retries=self._max_retries,+ base_url=self._url,+ api_key=self._api_key,+ **kwargs,+ )def _sanitize_config(self, config_dict: Dict[str, Any]) -> Dict[str, Any]:r"""Sanitize the model configuration for O1 models."""diff --git a/camel/models/samba_model.py b/camel/models/samba_model.pyindex 77af30668..8da5c4770 100644--- a/camel/models/samba_model.py+++ b/camel/models/samba_model.py@@ -88,8 +88,16 @@ class SambaModel(BaseModelBackend):(default: :obj:`None`)max_retries (int, optional): Maximum number of retries for API calls.(default: :obj:`3`)+ client (Optional[Any], optional): A custom synchronous+ OpenAI-compatible client instance. If provided, this client will+ be used instead of creating a new one. Only applicable when using+ SambaNova Cloud API. (default: :obj:`None`)+ async_client (Optional[Any], optional): A custom asynchronous+ OpenAI-compatible client instance. If provided, this client will+ be used instead of creating a new one. Only applicable when using+ SambaNova Cloud API. (default: :obj:`None`)**kwargs (Any): Additional arguments to pass to the client- initialization.+ initialization. Ignored if custom clients are provided."""@api_keys_required(@@ -106,6 +114,8 @@ class SambaModel(BaseModelBackend):token_counter: Optional[BaseTokenCounter] = None,timeout: Optional[float] = None,max_retries: int = 3,+ client: Optional[Any] = None,+ async_client: Optional[Any] = None,**kwargs: Any,) -> None:if model_config_dict is None:@@ -126,21 +136,30 @@ class SambaModel(BaseModelBackend):max_retries,)+ # Only create clients for Cloud API modeif self._url == "https://api.sambanova.ai/v1":- self._client = OpenAI(- timeout=self._timeout,- max_retries=self._max_retries,- base_url=self._url,- api_key=self._api_key,- **kwargs,- )- self._async_client = AsyncOpenAI(- timeout=self._timeout,- max_retries=self._max_retries,- base_url=self._url,- api_key=self._api_key,- **kwargs,- )+ # Use custom clients if provided, otherwise create new ones+ if client is not None:+ self._client = client+ else:+ self._client = OpenAI(+ timeout=self._timeout,+ max_retries=self._max_retries,+ base_url=self._url,+ api_key=self._api_key,+ **kwargs,+ )++ if async_client is not None:+ self._async_client = async_client+ else:+ self._async_client = AsyncOpenAI(+ timeout=self._timeout,+ max_retries=self._max_retries,+ base_url=self._url,+ api_key=self._api_key,+ **kwargs,+ )@propertydef token_counter(self) -> BaseTokenCounter:diff --git a/camel/models/sglang_model.py b/camel/models/sglang_model.pyindex 6f8e63f00..b29053613 100644--- a/camel/models/sglang_model.py+++ b/camel/models/sglang_model.py@@ -72,8 +72,16 @@ class SGLangModel(BaseModelBackend):(default: :obj:`None`)max_retries (int, optional): Maximum number of retries for API calls.(default: :obj:`3`)+ client (Optional[Any], optional): A custom synchronous+ OpenAI-compatible client instance. If provided, this client will+ be used instead of creating a new one. Note: When using custom+ clients with SGLang, server auto-start features will be disabled.+ (default: :obj:`None`)+ async_client (Optional[Any], optional): A custom asynchronous+ OpenAI-compatible client instance. If provided, this client will+ be used instead of creating a new one. (default: :obj:`None`)**kwargs (Any): Additional arguments to pass to the client- initialization.+ initialization. Ignored if custom clients are provided.Reference: https://sgl-project.github.io/backend/openai_api_completions.html@@ -88,6 +96,8 @@ class SGLangModel(BaseModelBackend):token_counter: Optional[BaseTokenCounter] = None,timeout: Optional[float] = None,max_retries: int = 3,+ client: Optional[Any] = None,+ async_client: Optional[Any] = None,**kwargs: Any,) -> None:if model_config_dict is None:@@ -111,9 +121,10 @@ class SGLangModel(BaseModelBackend):max_retries,)- self._client = None-- if self._url:+ # Use custom clients if provided, otherwise create new ones+ if client is not None:+ self._client = client+ elif self._url:# Initialize the client if an existing URL is providedself._client = OpenAI(timeout=self._timeout,@@ -122,6 +133,12 @@ class SGLangModel(BaseModelBackend):base_url=self._url,**kwargs,)+ else:+ self._client = None++ if async_client is not None:+ self._async_client = async_client+ elif self._url:self._async_client = AsyncOpenAI(timeout=self._timeout,max_retries=self._max_retries,@@ -129,6 +146,8 @@ class SGLangModel(BaseModelBackend):base_url=self._url,**kwargs,)+ else:+ self._async_client = Nonedef _start_server(self) -> None:try:@@ -159,13 +178,24 @@ class SGLangModel(BaseModelBackend):)self._inactivity_thread.start()self.last_run_time = time.time()- # Initialize the client after the server starts- self._client = OpenAI(- timeout=self._timeout,- max_retries=self._max_retries,- api_key="Set-but-ignored", # required but ignored- base_url=self._url,- )+ # Initialize client after server starts if not already set+ if self._client is None:+ self._client = OpenAI(+ timeout=self._timeout,+ max_retries=self._max_retries,+ api_key="Set-but-ignored", # required but ignored+ base_url=self._url,+ )+ if (+ not hasattr(self, '_async_client')+ or self._async_client is None+ ):+ self._async_client = AsyncOpenAI(+ timeout=self._timeout,+ max_retries=self._max_retries,+ api_key="Set-but-ignored", # required but ignored+ base_url=self._url,+ )except Exception as e:raise RuntimeError(f"Failed to start SGLang server: {e}") from ediff --git a/examples/custom_client_usage.py b/examples/custom_client_usage.pynew file mode 100644index 000000000..4ae896e07--- /dev/null+++ b/examples/custom_client_usage.py@@ -0,0 +1,52 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========++"""+Custom Client Integration Example++Demonstrates using custom OpenAI-compatible clients (e.g., from RL frameworks+like AReaL or rLLM) with CAMEL agents.+"""++import asyncio++from areal.experimental.openai import ArealOpenAI++from camel.agents import ChatAgent+from camel.models import OpenAICompatibleModel+++async def async_example():+ # Create model with async client+ areal_client = ArealOpenAI(engine=engine, tokenizer=tokenizer)+ model = OpenAICompatibleModel(+ model_type="model-name",+ async_client=areal_client,+ )++ # Create agent+ agent = ChatAgent(+ system_message="You are a helpful async assistant.",+ model=model,+ )++ user_msg = "Hello, async world!"++ # Use async step+ response = await agent.astep(user_msg)+ print(response.msgs[0].content)+++# Run async example+asyncio.run(async_example())