🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org
Python agentai-societiesartificial-intelligencecommunicative-aicooperative-aideep-learninglarge-language-modelsmulti-agent-systemsnatural-language-processing
feat: integrate sglang in camel (#1255)
Neil Johnson2063923d5cfa
diff --git a/camel/configs/__init__.py b/camel/configs/__init__.pyindex ff4cc8c28..fcd9b7a9c 100644--- a/camel/configs/__init__.py+++ b/camel/configs/__init__.py@@ -30,6 +30,7 @@ from .samba_config import (SambaCloudAPIConfig,SambaVerseAPIConfig,)+from .sglang_config import SGLANG_API_PARAMS, SGLangConfigfrom .togetherai_config import TOGETHERAI_API_PARAMS, TogetherAIConfigfrom .vllm_config import VLLM_API_PARAMS, VLLMConfigfrom .yi_config import YI_API_PARAMS, YiConfig@@ -55,6 +56,8 @@ __all__ = ['Gemini_API_PARAMS','VLLMConfig','VLLM_API_PARAMS',+ 'SGLangConfig',+ 'SGLANG_API_PARAMS','MistralConfig','MISTRAL_API_PARAMS','RekaConfig',diff --git a/camel/configs/sglang_config.py b/camel/configs/sglang_config.pynew file mode 100644index 000000000..821698157--- /dev/null+++ b/camel/configs/sglang_config.py@@ -0,0 +1,71 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+from __future__ import annotations++from typing import Sequence, Union++from camel.configs.base_config import BaseConfig+from camel.types import NOT_GIVEN, NotGiven+++class SGLangConfig(BaseConfig):+ r"""Defines the parameters for generating chat completions using the+ OpenAI API.++ Reference: https://sgl-project.github.io/references/sampling_params.html++ Args:+ stop (str or list, optional): Up to :obj:`4` sequences where the API+ will stop generating further tokens. (default: :obj:`None`)+ temperature (float, optional): Sampling temperature to use, between+ :obj:`0` and :obj:`2`. Higher values make the output more random,+ while lower values make it more focused and deterministic.+ (default: :obj:`1.0`)+ top_p (float, optional): An alternative to sampling with temperature,+ called nucleus sampling, where the model considers the results of+ the tokens with top_p probability mass. So :obj:`0.1` means only+ the tokens comprising the top 10% probability mass are considered.+ (default: :obj:`1.0`)+ n (int, optional): How many chat completion choices to generate for+ each input message. (default: :obj:`1`)+ frequency_penalty (float, optional): Number between :obj:`-2.0` and+ :obj:`2.0`. Positive values penalize new tokens based on their+ existing frequency in the text so far, decreasing the model's+ likelihood to repeat the same line verbatim. See more information+ about frequency and presence penalties. (default: :obj:`0.0`)+ presence_penalty (float, optional): Number between :obj:`-2.0` and+ :obj:`2.0`. Positive values penalize new tokens based on whether+ they appear in the text so far, increasing the model's likelihood+ to talk about new topics. See more information about frequency and+ presence penalties. (default: :obj:`0.0`)+ stream (bool, optional): Whether to stream the generated output in+ chunks. If set to `True`, the response will be streamed as it is+ generated. (default: :obj:`False`)+ max_tokens (int, optional): The maximum number of tokens to generate+ in the chat completion. The total length of input tokens and+ generated tokens is limited by the model's context length.+ (default: :obj:`None`)+ """++ stop: Union[str, Sequence[str], NotGiven] = NOT_GIVEN+ temperature: float = 1.0+ top_p: float = 1.0+ n: int = 1+ frequency_penalty: float = 0.0+ presence_penalty: float = 0.0+ stream: bool = False+ max_tokens: Union[int, NotGiven] = NOT_GIVEN+++SGLANG_API_PARAMS = {param for param in SGLangConfig.model_fields.keys()}diff --git a/camel/models/__init__.py b/camel/models/__init__.pyindex 15dad9fd2..94fc012bf 100644--- a/camel/models/__init__.py+++ b/camel/models/__init__.py@@ -31,6 +31,7 @@ from .openai_model import OpenAIModelfrom .qwen_model import QwenModelfrom .reka_model import RekaModelfrom .samba_model import SambaModel+from .sglang_model import SGLangModelfrom .stub_model import StubModelfrom .togetherai_model import TogetherAIModelfrom .vllm_model import VLLMModel@@ -55,6 +56,7 @@ __all__ = ['NvidiaModel','OllamaModel','VLLMModel',+ 'SGLangModel','GeminiModel','OpenAICompatibleModel','RekaModel',diff --git a/camel/models/model_factory.py b/camel/models/model_factory.pyindex f9250b54a..309c3dce6 100644--- a/camel/models/model_factory.py+++ b/camel/models/model_factory.py@@ -29,6 +29,7 @@ from camel.models.openai_model import OpenAIModelfrom camel.models.qwen_model import QwenModelfrom camel.models.reka_model import RekaModelfrom camel.models.samba_model import SambaModel+from camel.models.sglang_model import SGLangModelfrom camel.models.stub_model import StubModelfrom camel.models.togetherai_model import TogetherAIModelfrom camel.models.vllm_model import VLLMModel@@ -86,6 +87,8 @@ class ModelFactory:model_class = OllamaModelelif model_platform.is_vllm:model_class = VLLMModel+ elif model_platform.is_sglang:+ model_class = SGLangModelelif model_platform.is_openai_compatible_model:model_class = OpenAICompatibleModelelif model_platform.is_samba:diff --git a/camel/models/sglang_model.py b/camel/models/sglang_model.pynew file mode 100644index 000000000..67c846e86--- /dev/null+++ b/camel/models/sglang_model.py@@ -0,0 +1,225 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+import logging+import threading+import time+from typing import Any, Dict, List, Optional, Union++from openai import OpenAI, Stream++from camel.configs import SGLANG_API_PARAMS, SGLangConfig+from camel.messages import OpenAIMessage+from camel.models import BaseModelBackend+from camel.types import (+ ChatCompletion,+ ChatCompletionChunk,+ ModelType,+)+from camel.utils import BaseTokenCounter, OpenAITokenCounter+++class SGLangModel(BaseModelBackend):+ r"""SGLang service interface.++ Args:+ model_type (Union[ModelType, str]): Model for which a backend is+ created.+ model_config_dict (Optional[Dict[str, Any]], optional): A dictionary+ that will be fed into:obj:`openai.ChatCompletion.create()`. If+ :obj:`None`, :obj:`SGLangConfig().as_dict()` will be used.+ (default: :obj:`None`)+ api_key (Optional[str], optional): The API key for authenticating with+ the model service. SGLang doesn't need API key, it would be ignored+ if set. (default: :obj:`None`)+ url (Optional[str], optional): The url to the model service. If not+ provided, :obj:`"http://127.0.0.1:30000/v1"` will be used.+ (default: :obj:`None`)+ token_counter (Optional[BaseTokenCounter], optional): Token counter to+ use for the model. If not provided, :obj:`OpenAITokenCounter(+ ModelType.GPT_4O_MINI)` will be used.+ (default: :obj:`None`)++ Reference: https://sgl-project.github.io/backend/openai_api_completions.html+ """++ def __init__(+ self,+ model_type: Union[ModelType, str],+ model_config_dict: Optional[Dict[str, Any]] = None,+ api_key: Optional[str] = None,+ url: Optional[str] = None,+ token_counter: Optional[BaseTokenCounter] = None,+ ) -> None:+ if model_config_dict is None:+ model_config_dict = SGLangConfig().as_dict()++ self.server_process = None+ self.last_run_time: Optional[float] = (+ None # Will be set when the server starts+ )+ self._lock = threading.Lock()+ self._inactivity_thread: Optional[threading.Thread] = None++ super().__init__(+ model_type, model_config_dict, api_key, url, token_counter+ )++ self._client = None++ if self._url:+ # Initialize the client if an existing URL is provided+ self._client = OpenAI(+ timeout=60,+ max_retries=3,+ api_key="Set-but-ignored", # required but ignored+ base_url=self._url,+ )++ def _start_server(self) -> None:+ from sglang.utils import ( # type: ignore[import-untyped]+ execute_shell_command,+ wait_for_server,+ )++ try:+ if not self._url:+ cmd = (+ f"python -m sglang.launch_server "+ f"--model-path {self.model_type} "+ f"--port 30000 "+ f"--host 0.0.0.0"+ )++ server_process = execute_shell_command(cmd)+ wait_for_server("http://localhost:30000")+ self._url = "http://127.0.0.1:30000/v1"+ self.server_process = server_process+ # Start the inactivity monitor in a background thread+ self._inactivity_thread = threading.Thread(+ target=self._monitor_inactivity, daemon=True+ )+ self._inactivity_thread.start()+ self.last_run_time = time.time()+ # Initialize the client after the server starts+ self._client = OpenAI(+ timeout=60,+ max_retries=3,+ api_key="Set-but-ignored", # required but ignored+ base_url=self._url,+ )+ except Exception as e:+ raise RuntimeError(f"Failed to start SGLang server: {e}") from e++ def _ensure_server_running(self) -> None:+ r"""Ensures that the server is running. If not, starts the server."""+ with self._lock:+ if self.server_process is None:+ self._start_server()++ def _monitor_inactivity(self):+ r"""Monitor whether the server process has been inactive for over 10+ minutes.+ """+ from sglang.utils import terminate_process++ while True:+ # Check every 10 seconds+ time.sleep(10)+ # Over 10 minutes+ with self._lock:+ # Over 10 minutes+ if self.last_run_time and (+ time.time() - self.last_run_time > 600+ ):+ if self.server_process:+ terminate_process(self.server_process)+ self.server_process = None+ self._client = None # Invalidate the client+ logging.info(+ "Server process terminated due to inactivity."+ )+ break++ @property+ def token_counter(self) -> BaseTokenCounter:+ r"""Initialize the token counter for the model backend.++ Returns:+ BaseTokenCounter: The token counter following the model's+ tokenization style.+ """+ if not self._token_counter:+ self._token_counter = OpenAITokenCounter(ModelType.GPT_4O_MINI)+ return self._token_counter++ def check_model_config(self):+ r"""Check whether the model configuration contains any+ unexpected arguments to SGLang API.++ Raises:+ ValueError: If the model configuration dictionary contains any+ unexpected arguments to OpenAI API.+ """+ for param in self.model_config_dict:+ if param not in SGLANG_API_PARAMS:+ raise ValueError(+ f"Unexpected argument `{param}` is "+ "input into SGLang model backend."+ )++ def run(+ self,+ messages: List[OpenAIMessage],+ ) -> Union[ChatCompletion, Stream[ChatCompletionChunk]]:+ r"""Runs inference of OpenAI chat completion.++ Args:+ messages (List[OpenAIMessage]): Message list with the chat history+ in OpenAI API format.++ Returns:+ Union[ChatCompletion, Stream[ChatCompletionChunk]]:+ `ChatCompletion` in the non-stream mode, or+ `Stream[ChatCompletionChunk]` in the stream mode.+ """++ # Ensure server is running+ self._ensure_server_running()++ with self._lock:+ # Update last run time+ self.last_run_time = time.time()++ if self._client is None:+ raise RuntimeError(+ "Client is not initialized. Ensure the server is running."+ )++ response = self._client.chat.completions.create(+ messages=messages,+ model=self.model_type,+ **self.model_config_dict,+ )++ return response++ @property+ def stream(self) -> bool:+ r"""Returns whether the model is in stream mode, which sends partial+ results each time.++ Returns:+ bool: Whether the model is in stream mode.+ """+ return self.model_config_dict.get('stream', False)diff --git a/camel/types/enums.py b/camel/types/enums.pyindex 00aa0b85f..353bc7aac 100644--- a/camel/types/enums.py+++ b/camel/types/enums.py@@ -621,6 +621,7 @@ class ModelPlatformType(Enum):QWEN = "tongyi-qianwen"NVIDIA = "nvidia"DEEPSEEK = "deepseek"+ SGLANG = "sglang"@propertydef is_openai(self) -> bool:@@ -652,6 +653,11 @@ class ModelPlatformType(Enum):r"""Returns whether this platform is vllm."""return self is ModelPlatformType.VLLM+ @property+ def is_sglang(self) -> bool:+ r"""Returns whether this platform is sglang."""+ return self is ModelPlatformType.SGLANG+@propertydef is_together(self) -> bool:r"""Returns whether this platform is together."""diff --git a/examples/models/sglang_model_example.py b/examples/models/sglang_model_example.pynew file mode 100644index 000000000..8a50a7469--- /dev/null+++ b/examples/models/sglang_model_example.py@@ -0,0 +1,56 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+++from dotenv import load_dotenv++from camel.agents import ChatAgent+from camel.models import ModelFactory+from camel.types import ModelPlatformType++r"""+Before using sglang to run LLM model offline,+you need to install flashinfer which cannot+be installed by poetry.+Consider your machine's configuration and+install flashinfer in a appropriate version.+For more details, please refer to:+https://sgl-project.github.io/start/install.html+https://docs.flashinfer.ai/installation.html++Please load HF_token in your environment variable.+export HF_TOKEN=""+"""+load_dotenv()+sglang_model = ModelFactory.create(+ model_platform=ModelPlatformType.SGLANG,+ model_type="meta-llama/Llama-3.2-1B",+ model_config_dict={"temperature": 0.0},+ api_key="sglang",+)++assistant_sys_msg = "You are a helpful assistant."++agent = ChatAgent(assistant_sys_msg, model=sglang_model, token_limit=4096)++user_msg = "Say hi to CAMEL AI"++assistant_response = agent.step(user_msg)+print(assistant_response.msg.content)++"""+===============================================================================+CAMEL AI ReferentialAction+===============================================================================+"""diff --git a/poetry.lock b/poetry.lockindex aeecbd071..b1c131784 100644--- a/poetry.lock+++ b/poetry.lock@@ -1,4 +1,4 @@-# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand.+# This file is automatically @generated by Poetry 1.8.4 and should not be changed by hand.[[package]]name = "accelerate"@@ -1839,13 +1839,13 @@ files = [[[package]]name = "firecrawl-py"-version = "1.6.3"+version = "1.6.4"description = "Python SDK for Firecrawl API"optional = truepython-versions = ">=3.8"files = [- {file = "firecrawl_py-1.6.3-py3-none-any.whl", hash = "sha256:c561c6b3cb4dd6c499c1a5298b1cf7b38ae23f875ee897ae1141cff440e699c9"},- {file = "firecrawl_py-1.6.3.tar.gz", hash = "sha256:cb7d82a9a551b84c07f236368e2c8c274c27978ccb10ac91757e97fb6b581bd6"},+ {file = "firecrawl_py-1.6.4-py3-none-any.whl", hash = "sha256:3b8be937f7f91d68c636d40188fe5de26808b0c9e6578a9ad10ef81595f2cd03"},+ {file = "firecrawl_py-1.6.4.tar.gz", hash = "sha256:35422a518b37c4194d5e238776128d66e92c1e1c0832a10bebee3057dc65e59a"},][package.dependencies]@@ -3918,7 +3918,6 @@ python-versions = ">=3.7"files = [{file = "milvus_lite-2.4.10-py3-none-macosx_10_9_x86_64.whl", hash = "sha256:fc4246d3ed7d1910847afce0c9ba18212e93a6e9b8406048436940578dfad5cb"},{file = "milvus_lite-2.4.10-py3-none-macosx_11_0_arm64.whl", hash = "sha256:74a8e07c5e3b057df17fbb46913388e84df1dc403a200f4e423799a58184c800"},- {file = "milvus_lite-2.4.10-py3-none-manylinux2014_aarch64.whl", hash = "sha256:240c7386b747bad696ecb5bd1f58d491e86b9d4b92dccee3315ed7256256eddc"},{file = "milvus_lite-2.4.10-py3-none-manylinux2014_x86_64.whl", hash = "sha256:211d2e334a043f9282bdd9755f76b9b2d93b23bffa7af240919ffce6a8dfe325"},]@@ -4813,13 +4812,13 @@ sympy = "*"[[package]]name = "openai"-version = "1.57.2"+version = "1.57.3"description = "The official Python library for the openai API"optional = falsepython-versions = ">=3.8"files = [- {file = "openai-1.57.2-py3-none-any.whl", hash = "sha256:f7326283c156fdee875746e7e54d36959fb198eadc683952ee05e3302fbd638d"},- {file = "openai-1.57.2.tar.gz", hash = "sha256:5f49fd0f38e9f2131cda7deb45dafdd1aee4f52a637e190ce0ecf40147ce8cee"},+ {file = "openai-1.57.3-py3-none-any.whl", hash = "sha256:c4034a5676eb252ef2e0ed1f46d040ca3bdde24bb61b432f50bb0b38d0cf9ecf"},+ {file = "openai-1.57.3.tar.gz", hash = "sha256:2c98ca6532b30d8bc5029974d2fcbd793b650009c2b014f47ffd4f9fdfc1f9eb"},][package.dependencies]@@ -5024,31 +5023,43 @@ python-versions = ">=3.9"files = [{file = "pandas-2.2.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:1948ddde24197a0f7add2bdc4ca83bf2b1ef84a1bc8ccffd95eda17fd836ecb5"},{file = "pandas-2.2.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:381175499d3802cde0eabbaf6324cce0c4f5d52ca6f8c377c29ad442f50f6348"},+ {file = "pandas-2.2.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d9c45366def9a3dd85a6454c0e7908f2b3b8e9c138f5dc38fed7ce720d8453ed"},{file = "pandas-2.2.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:86976a1c5b25ae3f8ccae3a5306e443569ee3c3faf444dfd0f41cda24667ad57"},+ {file = "pandas-2.2.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b8661b0238a69d7aafe156b7fa86c44b881387509653fdf857bebc5e4008ad42"},{file = "pandas-2.2.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:37e0aced3e8f539eccf2e099f65cdb9c8aa85109b0be6e93e2baff94264bdc6f"},{file = "pandas-2.2.3-cp310-cp310-win_amd64.whl", hash = "sha256:56534ce0746a58afaf7942ba4863e0ef81c9c50d3f0ae93e9497d6a41a057645"},{file = "pandas-2.2.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:66108071e1b935240e74525006034333f98bcdb87ea116de573a6a0dccb6c039"},{file = "pandas-2.2.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7c2875855b0ff77b2a64a0365e24455d9990730d6431b9e0ee18ad8acee13dbd"},+ {file = "pandas-2.2.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cd8d0c3be0515c12fed0bdbae072551c8b54b7192c7b1fda0ba56059a0179698"},{file = "pandas-2.2.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c124333816c3a9b03fbeef3a9f230ba9a737e9e5bb4060aa2107a86cc0a497fc"},+ {file = "pandas-2.2.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:63cc132e40a2e084cf01adf0775b15ac515ba905d7dcca47e9a251819c575ef3"},{file = "pandas-2.2.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:29401dbfa9ad77319367d36940cd8a0b3a11aba16063e39632d98b0e931ddf32"},{file = "pandas-2.2.3-cp311-cp311-win_amd64.whl", hash = "sha256:3fc6873a41186404dad67245896a6e440baacc92f5b716ccd1bc9ed2995ab2c5"},{file = "pandas-2.2.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:b1d432e8d08679a40e2a6d8b2f9770a5c21793a6f9f47fdd52c5ce1948a5a8a9"},{file = "pandas-2.2.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a5a1595fe639f5988ba6a8e5bc9649af3baf26df3998a0abe56c02609392e0a4"},+ {file = "pandas-2.2.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5de54125a92bb4d1c051c0659e6fcb75256bf799a732a87184e5ea503965bce3"},{file = "pandas-2.2.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fffb8ae78d8af97f849404f21411c95062db1496aeb3e56f146f0355c9989319"},+ {file = "pandas-2.2.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6dfcb5ee8d4d50c06a51c2fffa6cff6272098ad6540aed1a76d15fb9318194d8"},{file = "pandas-2.2.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:062309c1b9ea12a50e8ce661145c6aab431b1e99530d3cd60640e255778bd43a"},{file = "pandas-2.2.3-cp312-cp312-win_amd64.whl", hash = "sha256:59ef3764d0fe818125a5097d2ae867ca3fa64df032331b7e0917cf5d7bf66b13"},{file = "pandas-2.2.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f00d1345d84d8c86a63e476bb4955e46458b304b9575dcf71102b5c705320015"},{file = "pandas-2.2.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:3508d914817e153ad359d7e069d752cdd736a247c322d932eb89e6bc84217f28"},+ {file = "pandas-2.2.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:22a9d949bfc9a502d320aa04e5d02feab689d61da4e7764b62c30b991c42c5f0"},{file = "pandas-2.2.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f3a255b2c19987fbbe62a9dfd6cff7ff2aa9ccab3fc75218fd4b7530f01efa24"},+ {file = "pandas-2.2.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:800250ecdadb6d9c78eae4990da62743b857b470883fa27f652db8bdde7f6659"},{file = "pandas-2.2.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6374c452ff3ec675a8f46fd9ab25c4ad0ba590b71cf0656f8b6daa5202bca3fb"},{file = "pandas-2.2.3-cp313-cp313-win_amd64.whl", hash = "sha256:61c5ad4043f791b61dd4752191d9f07f0ae412515d59ba8f005832a532f8736d"},{file = "pandas-2.2.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:3b71f27954685ee685317063bf13c7709a7ba74fc996b84fc6821c59b0f06468"},{file = "pandas-2.2.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:38cf8125c40dae9d5acc10fa66af8ea6fdf760b2714ee482ca691fc66e6fcb18"},+ {file = "pandas-2.2.3-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ba96630bc17c875161df3818780af30e43be9b166ce51c9a18c1feae342906c2"},{file = "pandas-2.2.3-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1db71525a1538b30142094edb9adc10be3f3e176748cd7acc2240c2f2e5aa3a4"},+ {file = "pandas-2.2.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:15c0e1e02e93116177d29ff83e8b1619c93ddc9c49083f237d4312337a61165d"},{file = "pandas-2.2.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:ad5b65698ab28ed8d7f18790a0dc58005c7629f227be9ecc1072aa74c0c1d43a"},{file = "pandas-2.2.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:bc6b93f9b966093cb0fd62ff1a7e4c09e6d546ad7c1de191767baffc57628f39"},{file = "pandas-2.2.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:5dbca4c1acd72e8eeef4753eeca07de9b1db4f398669d5994086f788a5d7cc30"},+ {file = "pandas-2.2.3-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8cd6d7cc958a3910f934ea8dbdf17b2364827bb4dafc38ce6eef6bb3d65ff09c"},{file = "pandas-2.2.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:99df71520d25fade9db7c1076ac94eb994f4d2673ef2aa2e86ee039b6746d20c"},+ {file = "pandas-2.2.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:31d0ced62d4ea3e231a9f228366919a5ea0b07440d9d4dac345376fd8e1477ea"},{file = "pandas-2.2.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:7eee9e7cea6adf3e3d24e304ac6b8300646e2a5d1cd3a3c2abed9101b0846761"},{file = "pandas-2.2.3-cp39-cp39-win_amd64.whl", hash = "sha256:4850ba03528b6dd51d6c5d273c46f183f39a9baf3f0143e566b89450965b105e"},{file = "pandas-2.2.3.tar.gz", hash = "sha256:4f18ba62b61d7e192368b84517265a99b4d7ee8912f8708660fb4a366cc82667"},@@ -5902,7 +5913,6 @@ description = "Pure-Python implementation of ASN.1 types and DER/BER/CER codecsoptional = truepython-versions = ">=3.8"files = [- {file = "pyasn1-0.6.1-py3-none-any.whl", hash = "sha256:0d632f46f2ba09143da3a8afe9e33fb6f92fa2320ab7e886e2d0f7672af84629"},{file = "pyasn1-0.6.1.tar.gz", hash = "sha256:6f580d2bdd84365380830acf45550f2511469f673cb4a5ae3857a3170128b034"},]@@ -5913,7 +5923,6 @@ description = "A collection of ASN.1-based protocols modules"optional = truepython-versions = ">=3.8"files = [- {file = "pyasn1_modules-0.4.1-py3-none-any.whl", hash = "sha256:49bfa96b45a292b711e986f222502c1c9a5e1f4e568fc30e2574a6c7d07838fd"},{file = "pyasn1_modules-0.4.1.tar.gz", hash = "sha256:c28e2dbf9c06ad61c71a075c7e0f9fd0f1b0bb2d2ad4377f240d33ac2ab60a7c"},]@@ -7824,6 +7833,42 @@ enabler = ["pytest-enabler (>=2.2)"]test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "jaraco.test (>=5.5)", "packaging (>=24.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-home (>=0.5)", "pytest-perf", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel (>=0.44.0)"]type = ["importlib_metadata (>=7.0.2)", "jaraco.develop (>=7.21)", "mypy (>=1.12,<1.14)", "pytest-mypy"]+[[package]]+name = "sglang"+version = "0.4.0.post1"+description = "SGLang is yet another fast serving framework for large language models and vision language models."+optional = true+python-versions = ">=3.8"+files = [+ {file = "sglang-0.4.0.post1-py3-none-any.whl", hash = "sha256:5989da07be334b69aac12efba454a20b1e6eb72803d80bf064f71d5e62145acf"},+ {file = "sglang-0.4.0.post1.tar.gz", hash = "sha256:a82aec4ce1ef1988bc57561b7700a9d7b0f5465ed343bb43b10496ee13f95c50"},+]++[package.dependencies]+IPython = "*"+numpy = "*"+requests = "*"+tqdm = "*"++[package.extras]+all = ["sglang[anthropic]", "sglang[litellm]", "sglang[openai]", "sglang[srt]"]+all-hip = ["sglang[anthropic]", "sglang[litellm]", "sglang[openai]", "sglang[srt-hip]"]+all-hpu = ["sglang[anthropic]", "sglang[litellm]", "sglang[openai]", "sglang[srt-hpu]"]+all-xpu = ["sglang[anthropic]", "sglang[litellm]", "sglang[openai]", "sglang[srt-xpu]"]+anthropic = ["anthropic (>=0.20.0)"]+dev = ["sglang[all]", "sglang[test]"]+dev-hip = ["sglang[all-hip]", "sglang[test]"]+dev-hpu = ["sglang[all-hpu]", "sglang[test]"]+dev-xpu = ["sglang[all-xpu]", "sglang[test]"]+litellm = ["litellm (>=1.0.0)"]+openai = ["openai (>=1.0)", "tiktoken"]+runtime-common = ["aiohttp", "decord", "fastapi", "hf_transfer", "huggingface_hub", "interegular", "modelscope", "orjson", "outlines (>=0.0.44,<0.1.0)", "packaging", "pillow", "prometheus-client (>=0.20.0)", "psutil", "pydantic", "python-multipart", "pyzmq (>=25.1.2)", "torchao", "uvicorn", "uvloop", "xgrammar (>=0.1.4)"]+srt = ["cuda-python", "flashinfer (>=0.1.6)", "sglang[runtime-common]", "torch", "vllm (>=0.6.3.post1,<=0.6.4.post1)"]+srt-hip = ["sglang[runtime-common]", "torch", "vllm (==0.6.3.dev13)"]+srt-hpu = ["sglang[runtime-common]"]+srt-xpu = ["sglang[runtime-common]"]+test = ["accelerate", "jsonlines", "matplotlib", "pandas", "peft", "sentence_transformers"]+[[package]]name = "sgmllib3k"version = "1.0.0"@@ -9899,12 +9944,12 @@ test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools",type = ["pytest-mypy"][extras]-all = ["PyMuPDF", "accelerate", "agentops", "anthropic", "apify_client", "arxiv", "arxiv2text", "asknews", "azure-storage-blob", "beautifulsoup4", "botocore", "cohere", "cohere", "datacommons", "datacommons_pandas", "datasets", "diffusers", "discord.py", "docker", "docx2txt", "duckduckgo-search", "ffmpeg-python", "firecrawl-py", "google-cloud-storage", "google-generativeai", "googlemaps", "imageio", "ipykernel", "jupyter_client", "litellm", "mistralai", "nebula3-python", "neo4j", "newspaper3k", "nltk", "notion-client", "openapi-spec-validator", "opencv-python", "pdfplumber", "pillow", "prance", "praw", "pyTelegramBotAPI", "pydub", "pygithub", "pymilvus", "pyowm", "pyyaml", "qdrant-client", "rank-bm25", "redis", "reka-api", "requests_oauthlib", "scholarly", "sentence-transformers", "sentencepiece", "slack-bolt", "slack-sdk", "soundfile", "tavily-python", "textblob", "torch", "torch", "transformers", "unstructured", "wikipedia", "wolframalpha", "yt-dlp"]+all = ["PyMuPDF", "accelerate", "agentops", "anthropic", "apify_client", "arxiv", "arxiv2text", "asknews", "azure-storage-blob", "beautifulsoup4", "botocore", "cohere", "cohere", "datacommons", "datacommons_pandas", "datasets", "diffusers", "discord.py", "docker", "docx2txt", "duckduckgo-search", "ffmpeg-python", "firecrawl-py", "google-cloud-storage", "google-generativeai", "googlemaps", "imageio", "ipykernel", "jupyter_client", "litellm", "mistralai", "nebula3-python", "neo4j", "newspaper3k", "nltk", "notion-client", "openapi-spec-validator", "opencv-python", "pdfplumber", "pillow", "prance", "praw", "pyTelegramBotAPI", "pydub", "pygithub", "pymilvus", "pyowm", "pyyaml", "qdrant-client", "rank-bm25", "redis", "reka-api", "requests_oauthlib", "scholarly", "sentence-transformers", "sentencepiece", "sglang", "slack-bolt", "slack-sdk", "soundfile", "tavily-python", "textblob", "torch", "torch", "transformers", "unstructured", "wikipedia", "wolframalpha", "yt-dlp"]encoders = ["sentence-transformers"]graph-storages = ["nebula3-python", "neo4j"]huggingface-agent = ["accelerate", "datasets", "diffusers", "opencv-python", "sentencepiece", "soundfile", "torch", "torch", "transformers"]kv-stroages = ["redis"]-model-platforms = ["anthropic", "cohere", "google-generativeai", "litellm", "mistralai", "reka-api"]+model-platforms = ["anthropic", "cohere", "google-generativeai", "litellm", "mistralai", "reka-api", "sglang"]object-storages = ["azure-storage-blob", "botocore", "google-cloud-storage"]rag = ["cohere", "nebula3-python", "neo4j", "pymilvus", "qdrant-client", "rank-bm25", "sentence-transformers", "unstructured"]retrievers = ["cohere", "rank-bm25"]@@ -9917,4 +9962,4 @@ vector-databases = ["pymilvus", "qdrant-client"][metadata]lock-version = "2.0"python-versions = ">=3.10,<3.13"-content-hash = "66d081b88fb592beaee2ce10058723920f82106a987f90490f6d75b746e43085"+content-hash = "f131fd25b9baded1d7b3695ef0b732e49e149f138c2c79443af87e72e99c6972"diff --git a/pyproject.toml b/pyproject.tomlindex aa1235443..1791ba228 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -49,7 +49,7 @@ mistralai = { version = "^1.1.0", optional = true }reka-api = { version = "^3.0.8", optional = true }anthropic = { version = "^0.29.0", optional = true }cohere = { version = "^5.11.0", optional = true }-+sglang = { version = "^0.4.0", optional = true }# huggingface-agenttransformers = { version = "^4", optional = true }@@ -162,6 +162,7 @@ model-platforms = ["reka-api","anthropic","cohere",+ 'sglang',]huggingface-agent = [@@ -323,6 +324,7 @@ all = ["google-generativeai","anthropic","reka-api",+ "sglang",# kv-storages"redis",# object-storages@@ -472,6 +474,7 @@ module = ["mistralai","cohere","reka-api",+ "sglang","agentops","botocore.*","asknews",diff --git a/test/models/test_sglang_model.py b/test/models/test_sglang_model.pynew file mode 100644index 000000000..07c87d182--- /dev/null+++ b/test/models/test_sglang_model.py@@ -0,0 +1,57 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+import re++import pytest++from camel.configs import SGLangConfig+from camel.models import SGLangModel+from camel.types import ModelType, UnifiedModelType+from camel.utils import OpenAITokenCounter++[email protected]_backend+ "model_type",+ [+ ModelType.GPT_4,+ ModelType.GPT_4_TURBO,+ ModelType.GPT_4O,+ ModelType.GPT_4O_MINI,+ ],+)+def test_sglang_model(model_type: ModelType):+ model = SGLangModel(model_type, api_key="sglang")+ assert model.model_type == model_type+ assert model.model_config_dict == SGLangConfig().as_dict()+ assert isinstance(model.token_counter, OpenAITokenCounter)+ assert isinstance(model.model_type, UnifiedModelType)+ assert isinstance(model.token_limit, int)++[email protected]_backend+def test_sglang_model_unexpected_argument():+ model_type = ModelType.GPT_4+ model_config_dict = {"model_path": "vicuna-7b-v1.5"}++ with pytest.raises(+ ValueError,+ match=re.escape(+ (+ "Unexpected argument `model_path` is "+ "input into SGLang model backend."+ )+ ),+ ):+ _ = SGLangModel(model_type, model_config_dict, api_key="sglang")