š« CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org
Python agentai-societiesartificial-intelligencecommunicative-aicooperative-aideep-learninglarge-language-modelsmulti-agent-systemsnatural-language-processing
feat: Support vLLM as a model backend (#734)
Zecheng Zhangbc5430125989
diff --git a/README.md b/README.mdindex e5eda0a71..086bee998 100644--- a/README.md+++ b/README.md@@ -236,6 +236,41 @@ Please note that the environment variable is session-specific. If you open a newprint(assistant_response.msg.content)```+## Use Open-Source Models as Backends (ex. using vLLM to set Phi-3 locally)+- [Install vLLM](https://docs.vllm.ai/en/latest/getting_started/installation.html)+- After setting up vLLM, start an OpenAI compatible server for example by+ ```bash+ python -m vllm.entrypoints.openai.api_server --model microsoft/Phi-3-mini-4k-instruct --api-key vllm --dtype bfloat16+ ```+- Create and run following script (more details please refer to this [example](https://github.com/camel-ai/camel/blob/master/examples/models/vllm_model_example.py))+ ```python+ from camel.agents import ChatAgent+ from camel.messages import BaseMessage+ from camel.models import ModelFactory+ from camel.types import ModelPlatformType++ vllm_model = ModelFactory.create(+ model_platform=ModelPlatformType.VLLM,+ model_type="microsoft/Phi-3-mini-4k-instruct",+ url="http://localhost:8000/v1",+ model_config_dict={"temperature": 0.0},+ api_key="vllm",+ )++ assistant_sys_msg = BaseMessage.make_assistant_message(+ role_name="Assistant",+ content="You are a helpful assistant.",+ )+ agent = ChatAgent(assistant_sys_msg, model=vllm_model, token_limit=4096)++ user_msg = BaseMessage.make_user_message(+ role_name="User",+ content="Say hi to CAMEL AI",+ )+ assistant_response = agent.step(user_msg)+ print(assistant_response.msg.content)+ ```+## Data (Hosted on Hugging Face)| Dataset | Chat format | Instruction format | Chat format (translated) ||----------------|-----------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------|diff --git a/camel/configs/__init__.py b/camel/configs/__init__.pyindex dac32a3e6..30fdec55f 100644--- a/camel/configs/__init__.py+++ b/camel/configs/__init__.py@@ -24,6 +24,7 @@ from .openai_config import (ChatGPTConfig,OpenSourceConfig,)+from .vllm_config import VLLM_API_PARAMS, VLLMConfigfrom .zhipuai_config import ZHIPUAI_API_PARAMS, ZhipuAIConfig__all__ = [@@ -41,4 +42,6 @@ __all__ = ['ZHIPUAI_API_PARAMS','GeminiConfig','Gemini_API_PARAMS',+ 'VLLMConfig',+ 'VLLM_API_PARAMS',]diff --git a/camel/configs/vllm_config.py b/camel/configs/vllm_config.pynew file mode 100644index 000000000..2e0df9598--- /dev/null+++ b/camel/configs/vllm_config.py@@ -0,0 +1,103 @@+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========+# Licensed under the Apache License, Version 2.0 (the āLicenseā);+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an āAS ISā BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========+from __future__ import annotations++from dataclasses import asdict, dataclass, field+from typing import Sequence++from openai._types import NOT_GIVEN, NotGiven++from camel.configs.base_config import BaseConfig+++# flake8: noqa: E501+@dataclass(frozen=True)+class VLLMConfig(BaseConfig):+ r"""Defines the parameters for generating chat completions using the+ OpenAI API.++ Reference: https://docs.vllm.ai/en/latest/serving/openai_compatible_server.html++ Args:+ temperature (float, optional): Sampling temperature to use, between+ :obj:`0` and :obj:`2`. Higher values make the output more random,+ while lower values make it more focused and deterministic.+ (default: :obj:`0.2`)+ top_p (float, optional): An alternative to sampling with temperature,+ called nucleus sampling, where the model considers the results of+ the tokens with top_p probability mass. So :obj:`0.1` means only+ the tokens comprising the top 10% probability mass are considered.+ (default: :obj:`1.0`)+ n (int, optional): How many chat completion choices to generate for+ each input message. (default: :obj:`1`)+ response_format (object, optional): An object specifying the format+ that the model must output. Compatible with GPT-4 Turbo and all+ GPT-3.5 Turbo models newer than gpt-3.5-turbo-1106. Setting to+ {"type": "json_object"} enables JSON mode, which guarantees the+ message the model generates is valid JSON. Important: when using+ JSON mode, you must also instruct the model to produce JSON+ yourself via a system or user message. Without this, the model+ may generate an unending stream of whitespace until the generation+ reaches the token limit, resulting in a long-running and seemingly+ "stuck" request. Also note that the message content may be+ partially cut off if finish_reason="length", which indicates the+ generation exceeded max_tokens or the conversation exceeded the+ max context length.+ stream (bool, optional): If True, partial message deltas will be sent+ as data-only server-sent events as they become available.+ (default: :obj:`False`)+ stop (str or list, optional): Up to :obj:`4` sequences where the API+ will stop generating further tokens. (default: :obj:`None`)+ max_tokens (int, optional): The maximum number of tokens to generate+ in the chat completion. The total length of input tokens and+ generated tokens is limited by the model's context length.+ (default: :obj:`None`)+ presence_penalty (float, optional): Number between :obj:`-2.0` and+ :obj:`2.0`. Positive values penalize new tokens based on whether+ they appear in the text so far, increasing the model's likelihood+ to talk about new topics. See more information about frequency and+ presence penalties. (default: :obj:`0.0`)+ frequency_penalty (float, optional): Number between :obj:`-2.0` and+ :obj:`2.0`. Positive values penalize new tokens based on their+ existing frequency in the text so far, decreasing the model's+ likelihood to repeat the same line verbatim. See more information+ about frequency and presence penalties. (default: :obj:`0.0`)+ logit_bias (dict, optional): Modify the likelihood of specified tokens+ appearing in the completion. Accepts a json object that maps tokens+ (specified by their token ID in the tokenizer) to an associated+ bias value from :obj:`-100` to :obj:`100`. Mathematically, the bias+ is added to the logits generated by the model prior to sampling.+ The exact effect will vary per model, but values between:obj:` -1`+ and :obj:`1` should decrease or increase likelihood of selection;+ values like :obj:`-100` or :obj:`100` should result in a ban or+ exclusive selection of the relevant token. (default: :obj:`{}`)+ user (str, optional): A unique identifier representing your end-user,+ which can help OpenAI to monitor and detect abuse.+ (default: :obj:`""`)+ """++ temperature: float = 0.2 # openai default: 1.0+ top_p: float = 1.0+ n: int = 1+ stream: bool = False+ stop: str | Sequence[str] | NotGiven = NOT_GIVEN+ max_tokens: int | NotGiven = NOT_GIVEN+ presence_penalty: float = 0.0+ response_format: dict | NotGiven = NOT_GIVEN+ frequency_penalty: float = 0.0+ logit_bias: dict = field(default_factory=dict)+ user: str = ""+++VLLM_API_PARAMS = {param for param in asdict(VLLMConfig()).keys()}diff --git a/camel/models/__init__.py b/camel/models/__init__.pyindex be1be4fc0..42fca4415 100644--- a/camel/models/__init__.py+++ b/camel/models/__init__.py@@ -22,6 +22,7 @@ from .open_source_model import OpenSourceModelfrom .openai_audio_models import OpenAIAudioModelsfrom .openai_model import OpenAIModelfrom .stub_model import StubModel+from .vllm_model import VLLMModelfrom .zhipuai_model import ZhipuAIModel__all__ = [@@ -36,5 +37,6 @@ __all__ = ['OpenAIAudioModels','NemotronModel','OllamaModel',+ 'VLLMModel','GeminiModel',]diff --git a/camel/models/model_factory.py b/camel/models/model_factory.pyindex da1cfd234..78a5ec8e5 100644--- a/camel/models/model_factory.py+++ b/camel/models/model_factory.py@@ -21,6 +21,7 @@ from camel.models.ollama_model import OllamaModelfrom camel.models.open_source_model import OpenSourceModelfrom camel.models.openai_model import OpenAIModelfrom camel.models.stub_model import StubModel+from camel.models.vllm_model import VLLMModelfrom camel.models.zhipuai_model import ZhipuAIModelfrom camel.types import ModelPlatformType, ModelType@@ -83,6 +84,9 @@ class ModelFactory:if model_platform.is_ollama:model_class = OllamaModelreturn model_class(model_type, model_config_dict, url)+ elif model_platform.is_vllm:+ model_class = VLLMModel+ return model_class(model_type, model_config_dict, url, api_key)elif model_platform.is_litellm:model_class = LiteLLMModelelse:diff --git a/camel/models/vllm_model.py b/camel/models/vllm_model.pynew file mode 100644index 000000000..61eac05b9--- /dev/null+++ b/camel/models/vllm_model.py@@ -0,0 +1,135 @@+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========+# Licensed under the Apache License, Version 2.0 (the āLicenseā);+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an āAS ISā BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========+from typing import Any, Dict, List, Optional, Union++from openai import OpenAI, Stream++from camel.configs import VLLM_API_PARAMS+from camel.messages import OpenAIMessage+from camel.types import ChatCompletion, ChatCompletionChunk, ModelType+from camel.utils import BaseTokenCounter, OpenAITokenCounter+++# flake8: noqa: E501+class VLLMModel:+ r"""vLLM service interface."""++ def __init__(+ self,+ model_type: str,+ model_config_dict: Dict[str, Any],+ url: Optional[str] = None,+ api_key: Optional[str] = None,+ ) -> None:+ r"""Constructor for vLLM backend with OpenAI compatibility.++ # Reference: https://docs.vllm.ai/en/latest/serving/openai_compatible_server.html++ Args:+ model_type (str): Model for which a backend is created.+ model_config_dict (Dict[str, Any]): A dictionary that will+ be fed into openai.ChatCompletion.create().+ url (Optional[str]): The url to the model service. (default:+ :obj:`None`)+ api_key (Optional[str]): The API key for authenticating with the+ model service.+ """+ self.model_type = model_type+ self.model_config_dict = model_config_dict+ # Use OpenAI cilent as interface call vLLM+ self._client = OpenAI(+ timeout=60,+ max_retries=3,+ base_url=url,+ api_key=api_key,+ )+ self._token_counter: Optional[BaseTokenCounter] = None+ self.check_model_config()++ @property+ def token_counter(self) -> BaseTokenCounter:+ r"""Initialize the token counter for the model backend.++ Returns:+ BaseTokenCounter: The token counter following the model's+ tokenization style.+ """+ # NOTE: Use OpenAITokenCounter temporarily+ if not self._token_counter:+ self._token_counter = OpenAITokenCounter(ModelType.GPT_3_5_TURBO)+ return self._token_counter++ def check_model_config(self):+ r"""Check whether the model configuration contains any+ unexpected arguments to vLLM API.++ Raises:+ ValueError: If the model configuration dictionary contains any+ unexpected arguments to OpenAI API.+ """+ for param in self.model_config_dict:+ if param not in VLLM_API_PARAMS:+ raise ValueError(+ f"Unexpected argument `{param}` is "+ "input into vLLM model backend."+ )++ def run(+ self,+ messages: List[OpenAIMessage],+ ) -> Union[ChatCompletion, Stream[ChatCompletionChunk]]:+ r"""Runs inference of OpenAI chat completion.++ Args:+ messages (List[OpenAIMessage]): Message list with the chat history+ in OpenAI API format.++ Returns:+ Union[ChatCompletion, Stream[ChatCompletionChunk]]:+ `ChatCompletion` in the non-stream mode, or+ `Stream[ChatCompletionChunk]` in the stream mode.+ """++ response = self._client.chat.completions.create(+ messages=messages,+ model=self.model_type,+ **self.model_config_dict,+ )+ return response++ @property+ def token_limit(self) -> int:+ """Returns the maximum token limit for the given model.++ Returns:+ int: The maximum token limit for the given model.+ """+ max_tokens = self.model_config_dict.get("max_tokens")+ if isinstance(max_tokens, int):+ return max_tokens+ print(+ "Must set `max_tokens` as an integer in `model_config_dict` when"+ " setting up the model. Using 4096 as default value."+ )+ return 4096++ @property+ def stream(self) -> bool:+ r"""Returns whether the model is in stream mode, which sends partial+ results each time.++ Returns:+ bool: Whether the model is in stream mode.+ """+ return self.model_config_dict.get('stream', False)diff --git a/camel/types/enums.py b/camel/types/enums.pyindex e13d3163d..8998eab1b 100644--- a/camel/types/enums.py+++ b/camel/types/enums.py@@ -344,6 +344,7 @@ class ModelPlatformType(Enum):ZHIPU = "zhipuai"DEFAULT = "default"GEMINI = "gemini"+ VLLM = "vllm"@propertydef is_openai(self) -> bool:@@ -365,6 +366,11 @@ class ModelPlatformType(Enum):r"""Returns whether this platform is ollama."""return self is ModelPlatformType.OLLAMA+ @property+ def is_vllm(self) -> bool:+ r"""Returns whether this platform is vllm."""+ return self is ModelPlatformType.VLLM+@propertydef is_litellm(self) -> bool:r"""Returns whether this platform is litellm."""diff --git a/examples/models/vllm_model_example.py b/examples/models/vllm_model_example.pynew file mode 100644index 000000000..62d1b44ae--- /dev/null+++ b/examples/models/vllm_model_example.py@@ -0,0 +1,45 @@+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========+# Licensed under the Apache License, Version 2.0 (the āLicenseā);+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an āAS ISā BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========++from camel.agents import ChatAgent+from camel.messages import BaseMessage+from camel.models import ModelFactory+from camel.types import ModelPlatformType++vllm_model = ModelFactory.create(+ model_platform=ModelPlatformType.VLLM,+ model_type="microsoft/Phi-3-mini-4k-instruct",+ url="http://localhost:8000/v1",+ model_config_dict={"temperature": 0.0},+ api_key="vllm",+)++assistant_sys_msg = BaseMessage.make_assistant_message(+ role_name="Assistant",+ content="You are a helpful assistant.",+)+agent = ChatAgent(assistant_sys_msg, model=vllm_model, token_limit=4096)++user_msg = BaseMessage.make_user_message(+ role_name="User",+ content="Say hi to CAMEL AI",+)+assistant_response = agent.step(user_msg)+print(assistant_response.msg.content)++"""+===============================================================================+Hello! I'm Phi, an AI developed by Microsoft. How can I help you today?+===============================================================================+"""diff --git a/test/models/test_ollama_model.py b/test/models/test_ollama_model.pyindex 1754447c9..c0ed5f630 100644--- a/test/models/test_ollama_model.py+++ b/test/models/test_ollama_model.py@@ -32,14 +32,14 @@ from camel.utils import OpenAITokenCounterModelType.GPT_4O,],)-def test_ollama_model(model_type):+def test_ollama_model(model_type: ModelType):model_config_dict = OllamaConfig().__dict__- model = OllamaModel(model_type, model_config_dict)- assert model.model_type == model_type+ model = OllamaModel(model_type.value, model_config_dict)+ assert model.model_type == model_type.valueassert model.model_config_dict == model_config_dictassert isinstance(model.token_counter, OpenAITokenCounter)- assert isinstance(model.model_type.value_for_tiktoken, str)- assert isinstance(model.model_type.token_limit, int)+ assert isinstance(model.model_type, str)+ assert isinstance(model.token_limit, int)@pytest.mark.model_backenddiff --git a/test/models/test_vllm_model.py b/test/models/test_vllm_model.pynew file mode 100644index 000000000..a6fab1c55--- /dev/null+++ b/test/models/test_vllm_model.py@@ -0,0 +1,59 @@+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========+# Licensed under the Apache License, Version 2.0 (the āLicenseā);+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an āAS ISā BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========+import re++import pytest++from camel.configs import VLLMConfig+from camel.models import VLLMModel+from camel.types import ModelType+from camel.utils import OpenAITokenCounter++[email protected]_backend+ "model_type",+ [+ ModelType.GPT_3_5_TURBO,+ ModelType.GPT_4,+ ModelType.GPT_4_32K,+ ModelType.GPT_4_TURBO,+ ModelType.GPT_4O,+ ],+)+def test_vllm_model(model_type: ModelType):+ model_config_dict = VLLMConfig().__dict__+ model = VLLMModel(model_type.value, model_config_dict, api_key="vllm")+ assert model.model_type == model_type.value+ assert model.model_config_dict == model_config_dict+ assert isinstance(model.token_counter, OpenAITokenCounter)+ assert isinstance(model.model_type, str)+ assert isinstance(model.token_limit, int)++[email protected]_backend+def test_vllm_model_unexpected_argument():+ model_type = ModelType.GPT_4+ model_config_dict = {"model_path": "vicuna-7b-v1.5"}++ with pytest.raises(+ ValueError,+ match=re.escape(+ (+ "Unexpected argument `model_path` is "+ "input into vLLM model backend."+ )+ ),+ ):+ _ = VLLMModel(model_type, model_config_dict, api_key="vllm")