Skip to content

armgit / camel

Public Imported from GitHub
0

🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org

feat: integrate sglang in camel (#1255)

Neil Johnson2063923d5cfa
diff --git a/camel/configs/__init__.py b/camel/configs/__init__.py
index ff4cc8c28..fcd9b7a9c 100644
--- a/camel/configs/__init__.py
+++ b/camel/configs/__init__.py
@@ -30,6 +30,7 @@ from .samba_config import (
SambaCloudAPIConfig,
SambaVerseAPIConfig,
)
+from .sglang_config import SGLANG_API_PARAMS, SGLangConfig
from .togetherai_config import TOGETHERAI_API_PARAMS, TogetherAIConfig
from .vllm_config import VLLM_API_PARAMS, VLLMConfig
from .yi_config import YI_API_PARAMS, YiConfig
@@ -55,6 +56,8 @@ __all__ = [
'Gemini_API_PARAMS',
'VLLMConfig',
'VLLM_API_PARAMS',
+ 'SGLangConfig',
+ 'SGLANG_API_PARAMS',
'MistralConfig',
'MISTRAL_API_PARAMS',
'RekaConfig',
diff --git a/camel/configs/sglang_config.py b/camel/configs/sglang_config.py
new file mode 100644
index 000000000..821698157
--- /dev/null
+++ b/camel/configs/sglang_config.py
@@ -0,0 +1,71 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+from __future__ import annotations
+
+from typing import Sequence, Union
+
+from camel.configs.base_config import BaseConfig
+from camel.types import NOT_GIVEN, NotGiven
+
+
+class SGLangConfig(BaseConfig):
+ r"""Defines the parameters for generating chat completions using the
+ OpenAI API.
+
+ Reference: https://sgl-project.github.io/references/sampling_params.html
+
+ Args:
+ stop (str or list, optional): Up to :obj:`4` sequences where the API
+ will stop generating further tokens. (default: :obj:`None`)
+ temperature (float, optional): Sampling temperature to use, between
+ :obj:`0` and :obj:`2`. Higher values make the output more random,
+ while lower values make it more focused and deterministic.
+ (default: :obj:`1.0`)
+ top_p (float, optional): An alternative to sampling with temperature,
+ called nucleus sampling, where the model considers the results of
+ the tokens with top_p probability mass. So :obj:`0.1` means only
+ the tokens comprising the top 10% probability mass are considered.
+ (default: :obj:`1.0`)
+ n (int, optional): How many chat completion choices to generate for
+ each input message. (default: :obj:`1`)
+ frequency_penalty (float, optional): Number between :obj:`-2.0` and
+ :obj:`2.0`. Positive values penalize new tokens based on their
+ existing frequency in the text so far, decreasing the model's
+ likelihood to repeat the same line verbatim. See more information
+ about frequency and presence penalties. (default: :obj:`0.0`)
+ presence_penalty (float, optional): Number between :obj:`-2.0` and
+ :obj:`2.0`. Positive values penalize new tokens based on whether
+ they appear in the text so far, increasing the model's likelihood
+ to talk about new topics. See more information about frequency and
+ presence penalties. (default: :obj:`0.0`)
+ stream (bool, optional): Whether to stream the generated output in
+ chunks. If set to `True`, the response will be streamed as it is
+ generated. (default: :obj:`False`)
+ max_tokens (int, optional): The maximum number of tokens to generate
+ in the chat completion. The total length of input tokens and
+ generated tokens is limited by the model's context length.
+ (default: :obj:`None`)
+ """
+
+ stop: Union[str, Sequence[str], NotGiven] = NOT_GIVEN
+ temperature: float = 1.0
+ top_p: float = 1.0
+ n: int = 1
+ frequency_penalty: float = 0.0
+ presence_penalty: float = 0.0
+ stream: bool = False
+ max_tokens: Union[int, NotGiven] = NOT_GIVEN
+
+
+SGLANG_API_PARAMS = {param for param in SGLangConfig.model_fields.keys()}
diff --git a/camel/models/__init__.py b/camel/models/__init__.py
index 15dad9fd2..94fc012bf 100644
--- a/camel/models/__init__.py
+++ b/camel/models/__init__.py
@@ -31,6 +31,7 @@ from .openai_model import OpenAIModel
from .qwen_model import QwenModel
from .reka_model import RekaModel
from .samba_model import SambaModel
+from .sglang_model import SGLangModel
from .stub_model import StubModel
from .togetherai_model import TogetherAIModel
from .vllm_model import VLLMModel
@@ -55,6 +56,7 @@ __all__ = [
'NvidiaModel',
'OllamaModel',
'VLLMModel',
+ 'SGLangModel',
'GeminiModel',
'OpenAICompatibleModel',
'RekaModel',
diff --git a/camel/models/model_factory.py b/camel/models/model_factory.py
index f9250b54a..309c3dce6 100644
--- a/camel/models/model_factory.py
+++ b/camel/models/model_factory.py
@@ -29,6 +29,7 @@ from camel.models.openai_model import OpenAIModel
from camel.models.qwen_model import QwenModel
from camel.models.reka_model import RekaModel
from camel.models.samba_model import SambaModel
+from camel.models.sglang_model import SGLangModel
from camel.models.stub_model import StubModel
from camel.models.togetherai_model import TogetherAIModel
from camel.models.vllm_model import VLLMModel
@@ -86,6 +87,8 @@ class ModelFactory:
model_class = OllamaModel
elif model_platform.is_vllm:
model_class = VLLMModel
+ elif model_platform.is_sglang:
+ model_class = SGLangModel
elif model_platform.is_openai_compatible_model:
model_class = OpenAICompatibleModel
elif model_platform.is_samba:
diff --git a/camel/models/sglang_model.py b/camel/models/sglang_model.py
new file mode 100644
index 000000000..67c846e86
--- /dev/null
+++ b/camel/models/sglang_model.py
@@ -0,0 +1,225 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+import logging
+import threading
+import time
+from typing import Any, Dict, List, Optional, Union
+
+from openai import OpenAI, Stream
+
+from camel.configs import SGLANG_API_PARAMS, SGLangConfig
+from camel.messages import OpenAIMessage
+from camel.models import BaseModelBackend
+from camel.types import (
+ ChatCompletion,
+ ChatCompletionChunk,
+ ModelType,
+)
+from camel.utils import BaseTokenCounter, OpenAITokenCounter
+
+
+class SGLangModel(BaseModelBackend):
+ r"""SGLang service interface.
+
+ Args:
+ model_type (Union[ModelType, str]): Model for which a backend is
+ created.
+ model_config_dict (Optional[Dict[str, Any]], optional): A dictionary
+ that will be fed into:obj:`openai.ChatCompletion.create()`. If
+ :obj:`None`, :obj:`SGLangConfig().as_dict()` will be used.
+ (default: :obj:`None`)
+ api_key (Optional[str], optional): The API key for authenticating with
+ the model service. SGLang doesn't need API key, it would be ignored
+ if set. (default: :obj:`None`)
+ url (Optional[str], optional): The url to the model service. If not
+ provided, :obj:`"http://127.0.0.1:30000/v1"` will be used.
+ (default: :obj:`None`)
+ token_counter (Optional[BaseTokenCounter], optional): Token counter to
+ use for the model. If not provided, :obj:`OpenAITokenCounter(
+ ModelType.GPT_4O_MINI)` will be used.
+ (default: :obj:`None`)
+
+ Reference: https://sgl-project.github.io/backend/openai_api_completions.html
+ """
+
+ def __init__(
+ self,
+ model_type: Union[ModelType, str],
+ model_config_dict: Optional[Dict[str, Any]] = None,
+ api_key: Optional[str] = None,
+ url: Optional[str] = None,
+ token_counter: Optional[BaseTokenCounter] = None,
+ ) -> None:
+ if model_config_dict is None:
+ model_config_dict = SGLangConfig().as_dict()
+
+ self.server_process = None
+ self.last_run_time: Optional[float] = (
+ None # Will be set when the server starts
+ )
+ self._lock = threading.Lock()
+ self._inactivity_thread: Optional[threading.Thread] = None
+
+ super().__init__(
+ model_type, model_config_dict, api_key, url, token_counter
+ )
+
+ self._client = None
+
+ if self._url:
+ # Initialize the client if an existing URL is provided
+ self._client = OpenAI(
+ timeout=60,
+ max_retries=3,
+ api_key="Set-but-ignored", # required but ignored
+ base_url=self._url,
+ )
+
+ def _start_server(self) -> None:
+ from sglang.utils import ( # type: ignore[import-untyped]
+ execute_shell_command,
+ wait_for_server,
+ )
+
+ try:
+ if not self._url:
+ cmd = (
+ f"python -m sglang.launch_server "
+ f"--model-path {self.model_type} "
+ f"--port 30000 "
+ f"--host 0.0.0.0"
+ )
+
+ server_process = execute_shell_command(cmd)
+ wait_for_server("http://localhost:30000")
+ self._url = "http://127.0.0.1:30000/v1"
+ self.server_process = server_process
+ # Start the inactivity monitor in a background thread
+ self._inactivity_thread = threading.Thread(
+ target=self._monitor_inactivity, daemon=True
+ )
+ self._inactivity_thread.start()
+ self.last_run_time = time.time()
+ # Initialize the client after the server starts
+ self._client = OpenAI(
+ timeout=60,
+ max_retries=3,
+ api_key="Set-but-ignored", # required but ignored
+ base_url=self._url,
+ )
+ except Exception as e:
+ raise RuntimeError(f"Failed to start SGLang server: {e}") from e
+
+ def _ensure_server_running(self) -> None:
+ r"""Ensures that the server is running. If not, starts the server."""
+ with self._lock:
+ if self.server_process is None:
+ self._start_server()
+
+ def _monitor_inactivity(self):
+ r"""Monitor whether the server process has been inactive for over 10
+ minutes.
+ """
+ from sglang.utils import terminate_process
+
+ while True:
+ # Check every 10 seconds
+ time.sleep(10)
+ # Over 10 minutes
+ with self._lock:
+ # Over 10 minutes
+ if self.last_run_time and (
+ time.time() - self.last_run_time > 600
+ ):
+ if self.server_process:
+ terminate_process(self.server_process)
+ self.server_process = None
+ self._client = None # Invalidate the client
+ logging.info(
+ "Server process terminated due to inactivity."
+ )
+ break
+
+ @property
+ def token_counter(self) -> BaseTokenCounter:
+ r"""Initialize the token counter for the model backend.
+
+ Returns:
+ BaseTokenCounter: The token counter following the model's
+ tokenization style.
+ """
+ if not self._token_counter:
+ self._token_counter = OpenAITokenCounter(ModelType.GPT_4O_MINI)
+ return self._token_counter
+
+ def check_model_config(self):
+ r"""Check whether the model configuration contains any
+ unexpected arguments to SGLang API.
+
+ Raises:
+ ValueError: If the model configuration dictionary contains any
+ unexpected arguments to OpenAI API.
+ """
+ for param in self.model_config_dict:
+ if param not in SGLANG_API_PARAMS:
+ raise ValueError(
+ f"Unexpected argument `{param}` is "
+ "input into SGLang model backend."
+ )
+
+ def run(
+ self,
+ messages: List[OpenAIMessage],
+ ) -> Union[ChatCompletion, Stream[ChatCompletionChunk]]:
+ r"""Runs inference of OpenAI chat completion.
+
+ Args:
+ messages (List[OpenAIMessage]): Message list with the chat history
+ in OpenAI API format.
+
+ Returns:
+ Union[ChatCompletion, Stream[ChatCompletionChunk]]:
+ `ChatCompletion` in the non-stream mode, or
+ `Stream[ChatCompletionChunk]` in the stream mode.
+ """
+
+ # Ensure server is running
+ self._ensure_server_running()
+
+ with self._lock:
+ # Update last run time
+ self.last_run_time = time.time()
+
+ if self._client is None:
+ raise RuntimeError(
+ "Client is not initialized. Ensure the server is running."
+ )
+
+ response = self._client.chat.completions.create(
+ messages=messages,
+ model=self.model_type,
+ **self.model_config_dict,
+ )
+
+ return response
+
+ @property
+ def stream(self) -> bool:
+ r"""Returns whether the model is in stream mode, which sends partial
+ results each time.
+
+ Returns:
+ bool: Whether the model is in stream mode.
+ """
+ return self.model_config_dict.get('stream', False)
diff --git a/camel/types/enums.py b/camel/types/enums.py
index 00aa0b85f..353bc7aac 100644
--- a/camel/types/enums.py
+++ b/camel/types/enums.py
@@ -621,6 +621,7 @@ class ModelPlatformType(Enum):
QWEN = "tongyi-qianwen"
NVIDIA = "nvidia"
DEEPSEEK = "deepseek"
+ SGLANG = "sglang"
@property
def is_openai(self) -> bool:
@@ -652,6 +653,11 @@ class ModelPlatformType(Enum):
r"""Returns whether this platform is vllm."""
return self is ModelPlatformType.VLLM
+ @property
+ def is_sglang(self) -> bool:
+ r"""Returns whether this platform is sglang."""
+ return self is ModelPlatformType.SGLANG
+
@property
def is_together(self) -> bool:
r"""Returns whether this platform is together."""
diff --git a/examples/models/sglang_model_example.py b/examples/models/sglang_model_example.py
new file mode 100644
index 000000000..8a50a7469
--- /dev/null
+++ b/examples/models/sglang_model_example.py
@@ -0,0 +1,56 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+
+
+from dotenv import load_dotenv
+
+from camel.agents import ChatAgent
+from camel.models import ModelFactory
+from camel.types import ModelPlatformType
+
+r"""
+Before using sglang to run LLM model offline,
+you need to install flashinfer which cannot
+be installed by poetry.
+Consider your machine's configuration and
+install flashinfer in a appropriate version.
+For more details, please refer to:
+https://sgl-project.github.io/start/install.html
+https://docs.flashinfer.ai/installation.html
+
+Please load HF_token in your environment variable.
+export HF_TOKEN=""
+"""
+load_dotenv()
+sglang_model = ModelFactory.create(
+ model_platform=ModelPlatformType.SGLANG,
+ model_type="meta-llama/Llama-3.2-1B",
+ model_config_dict={"temperature": 0.0},
+ api_key="sglang",
+)
+
+assistant_sys_msg = "You are a helpful assistant."
+
+agent = ChatAgent(assistant_sys_msg, model=sglang_model, token_limit=4096)
+
+user_msg = "Say hi to CAMEL AI"
+
+assistant_response = agent.step(user_msg)
+print(assistant_response.msg.content)
+
+"""
+===============================================================================
+CAMEL AI ReferentialAction
+===============================================================================
+"""
diff --git a/poetry.lock b/poetry.lock
index aeecbd071..b1c131784 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -1,4 +1,4 @@
-# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand.
+# This file is automatically @generated by Poetry 1.8.4 and should not be changed by hand.
[[package]]
name = "accelerate"
@@ -1839,13 +1839,13 @@ files = [
[[package]]
name = "firecrawl-py"
-version = "1.6.3"
+version = "1.6.4"
description = "Python SDK for Firecrawl API"
optional = true
python-versions = ">=3.8"
files = [
- {file = "firecrawl_py-1.6.3-py3-none-any.whl", hash = "sha256:c561c6b3cb4dd6c499c1a5298b1cf7b38ae23f875ee897ae1141cff440e699c9"},
- {file = "firecrawl_py-1.6.3.tar.gz", hash = "sha256:cb7d82a9a551b84c07f236368e2c8c274c27978ccb10ac91757e97fb6b581bd6"},
+ {file = "firecrawl_py-1.6.4-py3-none-any.whl", hash = "sha256:3b8be937f7f91d68c636d40188fe5de26808b0c9e6578a9ad10ef81595f2cd03"},
+ {file = "firecrawl_py-1.6.4.tar.gz", hash = "sha256:35422a518b37c4194d5e238776128d66e92c1e1c0832a10bebee3057dc65e59a"},
]
[package.dependencies]
@@ -3918,7 +3918,6 @@ python-versions = ">=3.7"
files = [
{file = "milvus_lite-2.4.10-py3-none-macosx_10_9_x86_64.whl", hash = "sha256:fc4246d3ed7d1910847afce0c9ba18212e93a6e9b8406048436940578dfad5cb"},
{file = "milvus_lite-2.4.10-py3-none-macosx_11_0_arm64.whl", hash = "sha256:74a8e07c5e3b057df17fbb46913388e84df1dc403a200f4e423799a58184c800"},
- {file = "milvus_lite-2.4.10-py3-none-manylinux2014_aarch64.whl", hash = "sha256:240c7386b747bad696ecb5bd1f58d491e86b9d4b92dccee3315ed7256256eddc"},
{file = "milvus_lite-2.4.10-py3-none-manylinux2014_x86_64.whl", hash = "sha256:211d2e334a043f9282bdd9755f76b9b2d93b23bffa7af240919ffce6a8dfe325"},
]
@@ -4813,13 +4812,13 @@ sympy = "*"
[[package]]
name = "openai"
-version = "1.57.2"
+version = "1.57.3"
description = "The official Python library for the openai API"
optional = false
python-versions = ">=3.8"
files = [
- {file = "openai-1.57.2-py3-none-any.whl", hash = "sha256:f7326283c156fdee875746e7e54d36959fb198eadc683952ee05e3302fbd638d"},
- {file = "openai-1.57.2.tar.gz", hash = "sha256:5f49fd0f38e9f2131cda7deb45dafdd1aee4f52a637e190ce0ecf40147ce8cee"},
+ {file = "openai-1.57.3-py3-none-any.whl", hash = "sha256:c4034a5676eb252ef2e0ed1f46d040ca3bdde24bb61b432f50bb0b38d0cf9ecf"},
+ {file = "openai-1.57.3.tar.gz", hash = "sha256:2c98ca6532b30d8bc5029974d2fcbd793b650009c2b014f47ffd4f9fdfc1f9eb"},
]
[package.dependencies]
@@ -5024,31 +5023,43 @@ python-versions = ">=3.9"
files = [
{file = "pandas-2.2.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:1948ddde24197a0f7add2bdc4ca83bf2b1ef84a1bc8ccffd95eda17fd836ecb5"},
{file = "pandas-2.2.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:381175499d3802cde0eabbaf6324cce0c4f5d52ca6f8c377c29ad442f50f6348"},
+ {file = "pandas-2.2.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d9c45366def9a3dd85a6454c0e7908f2b3b8e9c138f5dc38fed7ce720d8453ed"},
{file = "pandas-2.2.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:86976a1c5b25ae3f8ccae3a5306e443569ee3c3faf444dfd0f41cda24667ad57"},
+ {file = "pandas-2.2.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b8661b0238a69d7aafe156b7fa86c44b881387509653fdf857bebc5e4008ad42"},
{file = "pandas-2.2.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:37e0aced3e8f539eccf2e099f65cdb9c8aa85109b0be6e93e2baff94264bdc6f"},
{file = "pandas-2.2.3-cp310-cp310-win_amd64.whl", hash = "sha256:56534ce0746a58afaf7942ba4863e0ef81c9c50d3f0ae93e9497d6a41a057645"},
{file = "pandas-2.2.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:66108071e1b935240e74525006034333f98bcdb87ea116de573a6a0dccb6c039"},
{file = "pandas-2.2.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7c2875855b0ff77b2a64a0365e24455d9990730d6431b9e0ee18ad8acee13dbd"},
+ {file = "pandas-2.2.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cd8d0c3be0515c12fed0bdbae072551c8b54b7192c7b1fda0ba56059a0179698"},
{file = "pandas-2.2.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c124333816c3a9b03fbeef3a9f230ba9a737e9e5bb4060aa2107a86cc0a497fc"},
+ {file = "pandas-2.2.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:63cc132e40a2e084cf01adf0775b15ac515ba905d7dcca47e9a251819c575ef3"},
{file = "pandas-2.2.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:29401dbfa9ad77319367d36940cd8a0b3a11aba16063e39632d98b0e931ddf32"},
{file = "pandas-2.2.3-cp311-cp311-win_amd64.whl", hash = "sha256:3fc6873a41186404dad67245896a6e440baacc92f5b716ccd1bc9ed2995ab2c5"},
{file = "pandas-2.2.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:b1d432e8d08679a40e2a6d8b2f9770a5c21793a6f9f47fdd52c5ce1948a5a8a9"},
{file = "pandas-2.2.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a5a1595fe639f5988ba6a8e5bc9649af3baf26df3998a0abe56c02609392e0a4"},
+ {file = "pandas-2.2.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5de54125a92bb4d1c051c0659e6fcb75256bf799a732a87184e5ea503965bce3"},
{file = "pandas-2.2.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fffb8ae78d8af97f849404f21411c95062db1496aeb3e56f146f0355c9989319"},
+ {file = "pandas-2.2.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6dfcb5ee8d4d50c06a51c2fffa6cff6272098ad6540aed1a76d15fb9318194d8"},
{file = "pandas-2.2.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:062309c1b9ea12a50e8ce661145c6aab431b1e99530d3cd60640e255778bd43a"},
{file = "pandas-2.2.3-cp312-cp312-win_amd64.whl", hash = "sha256:59ef3764d0fe818125a5097d2ae867ca3fa64df032331b7e0917cf5d7bf66b13"},
{file = "pandas-2.2.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f00d1345d84d8c86a63e476bb4955e46458b304b9575dcf71102b5c705320015"},
{file = "pandas-2.2.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:3508d914817e153ad359d7e069d752cdd736a247c322d932eb89e6bc84217f28"},
+ {file = "pandas-2.2.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:22a9d949bfc9a502d320aa04e5d02feab689d61da4e7764b62c30b991c42c5f0"},
{file = "pandas-2.2.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f3a255b2c19987fbbe62a9dfd6cff7ff2aa9ccab3fc75218fd4b7530f01efa24"},
+ {file = "pandas-2.2.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:800250ecdadb6d9c78eae4990da62743b857b470883fa27f652db8bdde7f6659"},
{file = "pandas-2.2.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6374c452ff3ec675a8f46fd9ab25c4ad0ba590b71cf0656f8b6daa5202bca3fb"},
{file = "pandas-2.2.3-cp313-cp313-win_amd64.whl", hash = "sha256:61c5ad4043f791b61dd4752191d9f07f0ae412515d59ba8f005832a532f8736d"},
{file = "pandas-2.2.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:3b71f27954685ee685317063bf13c7709a7ba74fc996b84fc6821c59b0f06468"},
{file = "pandas-2.2.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:38cf8125c40dae9d5acc10fa66af8ea6fdf760b2714ee482ca691fc66e6fcb18"},
+ {file = "pandas-2.2.3-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ba96630bc17c875161df3818780af30e43be9b166ce51c9a18c1feae342906c2"},
{file = "pandas-2.2.3-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1db71525a1538b30142094edb9adc10be3f3e176748cd7acc2240c2f2e5aa3a4"},
+ {file = "pandas-2.2.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:15c0e1e02e93116177d29ff83e8b1619c93ddc9c49083f237d4312337a61165d"},
{file = "pandas-2.2.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:ad5b65698ab28ed8d7f18790a0dc58005c7629f227be9ecc1072aa74c0c1d43a"},
{file = "pandas-2.2.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:bc6b93f9b966093cb0fd62ff1a7e4c09e6d546ad7c1de191767baffc57628f39"},
{file = "pandas-2.2.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:5dbca4c1acd72e8eeef4753eeca07de9b1db4f398669d5994086f788a5d7cc30"},
+ {file = "pandas-2.2.3-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8cd6d7cc958a3910f934ea8dbdf17b2364827bb4dafc38ce6eef6bb3d65ff09c"},
{file = "pandas-2.2.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:99df71520d25fade9db7c1076ac94eb994f4d2673ef2aa2e86ee039b6746d20c"},
+ {file = "pandas-2.2.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:31d0ced62d4ea3e231a9f228366919a5ea0b07440d9d4dac345376fd8e1477ea"},
{file = "pandas-2.2.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:7eee9e7cea6adf3e3d24e304ac6b8300646e2a5d1cd3a3c2abed9101b0846761"},
{file = "pandas-2.2.3-cp39-cp39-win_amd64.whl", hash = "sha256:4850ba03528b6dd51d6c5d273c46f183f39a9baf3f0143e566b89450965b105e"},
{file = "pandas-2.2.3.tar.gz", hash = "sha256:4f18ba62b61d7e192368b84517265a99b4d7ee8912f8708660fb4a366cc82667"},
@@ -5902,7 +5913,6 @@ description = "Pure-Python implementation of ASN.1 types and DER/BER/CER codecs
optional = true
python-versions = ">=3.8"
files = [
- {file = "pyasn1-0.6.1-py3-none-any.whl", hash = "sha256:0d632f46f2ba09143da3a8afe9e33fb6f92fa2320ab7e886e2d0f7672af84629"},
{file = "pyasn1-0.6.1.tar.gz", hash = "sha256:6f580d2bdd84365380830acf45550f2511469f673cb4a5ae3857a3170128b034"},
]
@@ -5913,7 +5923,6 @@ description = "A collection of ASN.1-based protocols modules"
optional = true
python-versions = ">=3.8"
files = [
- {file = "pyasn1_modules-0.4.1-py3-none-any.whl", hash = "sha256:49bfa96b45a292b711e986f222502c1c9a5e1f4e568fc30e2574a6c7d07838fd"},
{file = "pyasn1_modules-0.4.1.tar.gz", hash = "sha256:c28e2dbf9c06ad61c71a075c7e0f9fd0f1b0bb2d2ad4377f240d33ac2ab60a7c"},
]
@@ -7824,6 +7833,42 @@ enabler = ["pytest-enabler (>=2.2)"]
test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "jaraco.test (>=5.5)", "packaging (>=24.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-home (>=0.5)", "pytest-perf", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel (>=0.44.0)"]
type = ["importlib_metadata (>=7.0.2)", "jaraco.develop (>=7.21)", "mypy (>=1.12,<1.14)", "pytest-mypy"]
+[[package]]
+name = "sglang"
+version = "0.4.0.post1"
+description = "SGLang is yet another fast serving framework for large language models and vision language models."
+optional = true
+python-versions = ">=3.8"
+files = [
+ {file = "sglang-0.4.0.post1-py3-none-any.whl", hash = "sha256:5989da07be334b69aac12efba454a20b1e6eb72803d80bf064f71d5e62145acf"},
+ {file = "sglang-0.4.0.post1.tar.gz", hash = "sha256:a82aec4ce1ef1988bc57561b7700a9d7b0f5465ed343bb43b10496ee13f95c50"},
+]
+
+[package.dependencies]
+IPython = "*"
+numpy = "*"
+requests = "*"
+tqdm = "*"
+
+[package.extras]
+all = ["sglang[anthropic]", "sglang[litellm]", "sglang[openai]", "sglang[srt]"]
+all-hip = ["sglang[anthropic]", "sglang[litellm]", "sglang[openai]", "sglang[srt-hip]"]
+all-hpu = ["sglang[anthropic]", "sglang[litellm]", "sglang[openai]", "sglang[srt-hpu]"]
+all-xpu = ["sglang[anthropic]", "sglang[litellm]", "sglang[openai]", "sglang[srt-xpu]"]
+anthropic = ["anthropic (>=0.20.0)"]
+dev = ["sglang[all]", "sglang[test]"]
+dev-hip = ["sglang[all-hip]", "sglang[test]"]
+dev-hpu = ["sglang[all-hpu]", "sglang[test]"]
+dev-xpu = ["sglang[all-xpu]", "sglang[test]"]
+litellm = ["litellm (>=1.0.0)"]
+openai = ["openai (>=1.0)", "tiktoken"]
+runtime-common = ["aiohttp", "decord", "fastapi", "hf_transfer", "huggingface_hub", "interegular", "modelscope", "orjson", "outlines (>=0.0.44,<0.1.0)", "packaging", "pillow", "prometheus-client (>=0.20.0)", "psutil", "pydantic", "python-multipart", "pyzmq (>=25.1.2)", "torchao", "uvicorn", "uvloop", "xgrammar (>=0.1.4)"]
+srt = ["cuda-python", "flashinfer (>=0.1.6)", "sglang[runtime-common]", "torch", "vllm (>=0.6.3.post1,<=0.6.4.post1)"]
+srt-hip = ["sglang[runtime-common]", "torch", "vllm (==0.6.3.dev13)"]
+srt-hpu = ["sglang[runtime-common]"]
+srt-xpu = ["sglang[runtime-common]"]
+test = ["accelerate", "jsonlines", "matplotlib", "pandas", "peft", "sentence_transformers"]
+
[[package]]
name = "sgmllib3k"
version = "1.0.0"
@@ -9899,12 +9944,12 @@ test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools",
type = ["pytest-mypy"]
[extras]
-all = ["PyMuPDF", "accelerate", "agentops", "anthropic", "apify_client", "arxiv", "arxiv2text", "asknews", "azure-storage-blob", "beautifulsoup4", "botocore", "cohere", "cohere", "datacommons", "datacommons_pandas", "datasets", "diffusers", "discord.py", "docker", "docx2txt", "duckduckgo-search", "ffmpeg-python", "firecrawl-py", "google-cloud-storage", "google-generativeai", "googlemaps", "imageio", "ipykernel", "jupyter_client", "litellm", "mistralai", "nebula3-python", "neo4j", "newspaper3k", "nltk", "notion-client", "openapi-spec-validator", "opencv-python", "pdfplumber", "pillow", "prance", "praw", "pyTelegramBotAPI", "pydub", "pygithub", "pymilvus", "pyowm", "pyyaml", "qdrant-client", "rank-bm25", "redis", "reka-api", "requests_oauthlib", "scholarly", "sentence-transformers", "sentencepiece", "slack-bolt", "slack-sdk", "soundfile", "tavily-python", "textblob", "torch", "torch", "transformers", "unstructured", "wikipedia", "wolframalpha", "yt-dlp"]
+all = ["PyMuPDF", "accelerate", "agentops", "anthropic", "apify_client", "arxiv", "arxiv2text", "asknews", "azure-storage-blob", "beautifulsoup4", "botocore", "cohere", "cohere", "datacommons", "datacommons_pandas", "datasets", "diffusers", "discord.py", "docker", "docx2txt", "duckduckgo-search", "ffmpeg-python", "firecrawl-py", "google-cloud-storage", "google-generativeai", "googlemaps", "imageio", "ipykernel", "jupyter_client", "litellm", "mistralai", "nebula3-python", "neo4j", "newspaper3k", "nltk", "notion-client", "openapi-spec-validator", "opencv-python", "pdfplumber", "pillow", "prance", "praw", "pyTelegramBotAPI", "pydub", "pygithub", "pymilvus", "pyowm", "pyyaml", "qdrant-client", "rank-bm25", "redis", "reka-api", "requests_oauthlib", "scholarly", "sentence-transformers", "sentencepiece", "sglang", "slack-bolt", "slack-sdk", "soundfile", "tavily-python", "textblob", "torch", "torch", "transformers", "unstructured", "wikipedia", "wolframalpha", "yt-dlp"]
encoders = ["sentence-transformers"]
graph-storages = ["nebula3-python", "neo4j"]
huggingface-agent = ["accelerate", "datasets", "diffusers", "opencv-python", "sentencepiece", "soundfile", "torch", "torch", "transformers"]
kv-stroages = ["redis"]
-model-platforms = ["anthropic", "cohere", "google-generativeai", "litellm", "mistralai", "reka-api"]
+model-platforms = ["anthropic", "cohere", "google-generativeai", "litellm", "mistralai", "reka-api", "sglang"]
object-storages = ["azure-storage-blob", "botocore", "google-cloud-storage"]
rag = ["cohere", "nebula3-python", "neo4j", "pymilvus", "qdrant-client", "rank-bm25", "sentence-transformers", "unstructured"]
retrievers = ["cohere", "rank-bm25"]
@@ -9917,4 +9962,4 @@ vector-databases = ["pymilvus", "qdrant-client"]
[metadata]
lock-version = "2.0"
python-versions = ">=3.10,<3.13"
-content-hash = "66d081b88fb592beaee2ce10058723920f82106a987f90490f6d75b746e43085"
+content-hash = "f131fd25b9baded1d7b3695ef0b732e49e149f138c2c79443af87e72e99c6972"
diff --git a/pyproject.toml b/pyproject.toml
index aa1235443..1791ba228 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -49,7 +49,7 @@ mistralai = { version = "^1.1.0", optional = true }
reka-api = { version = "^3.0.8", optional = true }
anthropic = { version = "^0.29.0", optional = true }
cohere = { version = "^5.11.0", optional = true }
-
+sglang = { version = "^0.4.0", optional = true }
# huggingface-agent
transformers = { version = "^4", optional = true }
@@ -162,6 +162,7 @@ model-platforms = [
"reka-api",
"anthropic",
"cohere",
+ 'sglang',
]
huggingface-agent = [
@@ -323,6 +324,7 @@ all = [
"google-generativeai",
"anthropic",
"reka-api",
+ "sglang",
# kv-storages
"redis",
# object-storages
@@ -472,6 +474,7 @@ module = [
"mistralai",
"cohere",
"reka-api",
+ "sglang",
"agentops",
"botocore.*",
"asknews",
diff --git a/test/models/test_sglang_model.py b/test/models/test_sglang_model.py
new file mode 100644
index 000000000..07c87d182
--- /dev/null
+++ b/test/models/test_sglang_model.py
@@ -0,0 +1,57 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+import re
+
+import pytest
+
+from camel.configs import SGLangConfig
+from camel.models import SGLangModel
+from camel.types import ModelType, UnifiedModelType
+from camel.utils import OpenAITokenCounter
+
+
+ "model_type",
+ [
+ ModelType.GPT_4,
+ ModelType.GPT_4_TURBO,
+ ModelType.GPT_4O,
+ ModelType.GPT_4O_MINI,
+ ],
+)
+def test_sglang_model(model_type: ModelType):
+ model = SGLangModel(model_type, api_key="sglang")
+ assert model.model_type == model_type
+ assert model.model_config_dict == SGLangConfig().as_dict()
+ assert isinstance(model.token_counter, OpenAITokenCounter)
+ assert isinstance(model.model_type, UnifiedModelType)
+ assert isinstance(model.token_limit, int)
+
+
+def test_sglang_model_unexpected_argument():
+ model_type = ModelType.GPT_4
+ model_config_dict = {"model_path": "vicuna-7b-v1.5"}
+
+ with pytest.raises(
+ ValueError,
+ match=re.escape(
+ (
+ "Unexpected argument `model_path` is "
+ "input into SGLang model backend."
+ )
+ ),
+ ):
+ _ = SGLangModel(model_type, model_config_dict, api_key="sglang")