Skip to content

armgit / camel

Public Imported from GitHub
0

🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org

feat: add function call in SGLang (#1666)

Neil Johnson1dc816d379a2
diff --git a/camel/configs/sglang_config.py b/camel/configs/sglang_config.py
index ce19d56fd..fe5251935 100644
--- a/camel/configs/sglang_config.py
+++ b/camel/configs/sglang_config.py
@@ -13,7 +13,7 @@
# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
from __future__ import annotations
-from typing import Sequence, Union
+from typing import Any, Dict, List, Optional, Sequence, Union
from camel.configs.base_config import BaseConfig
from camel.types import NOT_GIVEN, NotGiven
@@ -56,10 +56,11 @@ class SGLangConfig(BaseConfig):
in the chat completion. The total length of input tokens and
generated tokens is limited by the model's context length.
(default: :obj:`None`)
- tools (list[FunctionTool], optional): A list of tools the model may
- call. Currently, only functions are supported as a tool. Use this
- to provide a list of functions the model may generate JSON inputs
- for. A max of 128 functions are supported.
+ tools (list[Dict[str, Any]], optional): A list of tool definitions
+ that the model can dynamically invoke. Each tool should be
+ defined as a dictionary following OpenAI's function calling
+ specification format. For more details, refer to the OpenAI
+ documentation.
"""
stop: Union[str, Sequence[str], NotGiven] = NOT_GIVEN
@@ -70,6 +71,7 @@ class SGLangConfig(BaseConfig):
presence_penalty: float = 0.0
stream: bool = False
max_tokens: Union[int, NotGiven] = NOT_GIVEN
+ tools: Optional[Union[List[Dict[str, Any]]]] = None
SGLANG_API_PARAMS = {param for param in SGLangConfig.model_fields.keys()}
diff --git a/camel/models/sglang_model.py b/camel/models/sglang_model.py
index 727b50173..314bf9524 100644
--- a/camel/models/sglang_model.py
+++ b/camel/models/sglang_model.py
@@ -97,9 +97,16 @@ class SGLangModel(BaseModelBackend):
def _start_server(self) -> None:
try:
if not self._url:
+ tool_call_flag = self.model_config_dict.get("tools")
+ tool_call_arg = (
+ f"--tool-call-parser {self._api_key} "
+ if tool_call_flag
+ else ""
+ )
cmd = (
f"python -m sglang.launch_server "
f"--model-path {self.model_type} "
+ f"{tool_call_arg}"
f"--port 30000 "
f"--host 0.0.0.0"
)
@@ -265,6 +272,19 @@ class SGLangModel(BaseModelBackend):
"""
return self.model_config_dict.get('stream', False)
+ def __del__(self):
+ r"""Properly clean up resources when the model is destroyed."""
+ self.cleanup()
+
+ def cleanup(self):
+ r"""Terminate the server process and clean up resources."""
+ with self._lock:
+ if self.server_process:
+ _terminate_process(self.server_process)
+ self.server_process = None
+ self._client = None
+ logging.info("Server process terminated during cleanup.")
+
# Below are helper functions from sglang.utils
def _terminate_process(process):
@@ -326,21 +346,25 @@ def _execute_shell_command(command: str) -> subprocess.Popen:
return subprocess.Popen(parts, text=True, stderr=subprocess.STDOUT)
-def _wait_for_server(base_url: str, timeout: Optional[int] = None) -> None:
+def _wait_for_server(base_url: str, timeout: Optional[int] = 30) -> None:
r"""Wait for the server to be ready by polling the /v1/models endpoint.
Args:
base_url: The base URL of the server
- timeout: Maximum time to wait in seconds. None means wait forever.
+ timeout: Maximum time to wait in seconds. Default is 30 seconds.
"""
import requests
+ # Set a default value if timeout is None
+ actual_timeout = 30 if timeout is None else timeout
+
start_time = time.time()
while True:
try:
response = requests.get(
f"{base_url}/v1/models",
headers={"Authorization": "Bearer None"},
+ timeout=5, # Add a timeout for the request itself
)
if response.status_code == 200:
time.sleep(5)
@@ -356,9 +380,15 @@ def _wait_for_server(base_url: str, timeout: Optional[int] = None) -> None:
)
break
- if timeout and time.time() - start_time > timeout:
+ if time.time() - start_time > actual_timeout:
+ raise TimeoutError(
+ f"Server did not become ready within "
+ f"{actual_timeout} seconds"
+ )
+ except (requests.exceptions.RequestException, TimeoutError) as e:
+ if time.time() - start_time > actual_timeout:
raise TimeoutError(
- "Server did not become ready within timeout period"
+ f"Server did not become ready within "
+ f"{actual_timeout} seconds: {e}"
)
- except requests.exceptions.RequestException:
time.sleep(1)
diff --git a/examples/models/sglang_model_example.py b/examples/models/sglang_model_example.py
index c01c5e42f..5e2b59821 100644
--- a/examples/models/sglang_model_example.py
+++ b/examples/models/sglang_model_example.py
@@ -51,6 +51,84 @@ print(assistant_response.msg.content)
"""
===============================================================================
-CAMEL AI ReferentialAction
+Hello CAMEL AI. How can I assist you today?
+===============================================================================
+"""
+
+weather_tool = [
+ {
+ "type": "function",
+ "function": {
+ "name": "get_current_weather",
+ "description": "Get the current weather in a given location",
+ "parameters": {
+ "type": "object",
+ "properties": {
+ "city": {
+ "type": "string",
+ "description": "The city to find the weather for,\n"
+ "e.g. 'San Francisco'",
+ },
+ "state": {
+ "type": "string",
+ "description": "The two-letter abbreviation for,\n"
+ "the state (e.g., 'CA'), e.g. CA for California",
+ },
+ "unit": {
+ "type": "string",
+ "description": "Temperature unit (celsius/fahrenheit)",
+ "enum": ["celsius", "fahrenheit"],
+ },
+ },
+ "required": ["city", "state", "unit"],
+ },
+ },
+ }
+]
+
+
+r"""
+Note that api_key defines the parser used to interpret responses.
+Currently supported parsers include:
+llama3: Llama 3.1 / 3.2 (e.g. meta-llama/Llama-3.1-8B-Instruct,
+ meta-llama/Llama-3.2-1B-Instruct).
+mistral: Mistral (e.g. mistralai/Mistral-7B-Instruct-v0.3,
+ mistralai/Mistral-Nemo-Instruct-2407,
+ mistralai/ Mistral-Nemo-Instruct-2407, mistralai/Mistral-7B-v0.3).
+qwen25: Qwen 2.5 (e.g. Qwen/Qwen2.5-1.5B-Instruct, Qwen/Qwen2.5-7B-Instruct).
+"""
+sglang_model_with_tool = ModelFactory.create(
+ model_platform=ModelPlatformType.SGLANG,
+ model_type="meta-llama/Llama-3.2-1B-Instruct",
+ model_config_dict={"temperature": 0.0, "tools": weather_tool},
+ api_key="llama3",
+)
+
+assistant_sys_msg = (
+ "You are a helpful assistant.\n"
+ "Use the get_current_weather tool when asked about weather."
+)
+agent_with_tool = ChatAgent(
+ assistant_sys_msg,
+ model=sglang_model_with_tool,
+ token_limit=4096,
+ external_tools=weather_tool,
+)
+user_msg = "What's the weather in Boston today?"
+
+assistant_response = agent_with_tool.step(user_msg)
+external_tool_call = assistant_response.info.get('external_tool_call_request')
+if external_tool_call:
+ print(f"Detected external tool call: {external_tool_call.tool_name}")
+ print(f"Arguments: {external_tool_call.args}")
+ print(f"Tool Call ID: {external_tool_call.tool_call_id}")
+else:
+ print("No external tool call detected")
+
+"""
+===============================================================================
+Detected external tool call: get_current_weather
+Arguments: {'city': 'Boston', 'state': 'MA', 'unit': 'celsius'}
+Tool Call ID: 0
===============================================================================
"""
diff --git a/test/models/test_sglang_model.py b/test/models/test_sglang_model.py
index 07c87d182..abd07c39d 100644
--- a/test/models/test_sglang_model.py
+++ b/test/models/test_sglang_model.py
@@ -12,6 +12,7 @@
# limitations under the License.
# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
import re
+from unittest.mock import MagicMock, patch
import pytest
@@ -21,6 +22,47 @@ from camel.types import ModelType, UnifiedModelType
from camel.utils import OpenAITokenCounter
+def sglang_model_cleanup():
+ r"""Fixture to ensure SGLang model is cleaned up after each test."""
+ models = []
+
+ # Mock the server-related functions to avoid actual server startup
+ with (
+ patch(
+ 'camel.models.sglang_model._execute_shell_command'
+ ) as mock_execute,
+ patch('camel.models.sglang_model._wait_for_server') as mock_wait,
+ patch('camel.models.sglang_model.OpenAI') as mock_client,
+ patch('camel.models.sglang_model.AsyncOpenAI') as mock_async_client,
+ ):
+ # Configure mocks
+ mock_execute.return_value = MagicMock()
+ mock_wait.return_value = None
+ mock_client.return_value = MagicMock()
+ mock_async_client.return_value = MagicMock()
+
+ def _create_model(
+ model_type, model_config_dict=None, api_key="sglang"
+ ):
+ model = SGLangModel(model_type, model_config_dict, api_key=api_key)
+ # Set up the model to use our mocks
+ model._url = "http://mock-server:30000/v1"
+ model._client = mock_client.return_value
+ model._async_client = mock_async_client.return_value
+ models.append(model)
+ return model
+
+ yield _create_model
+
+ # Clean up all models after test
+ for model in models:
+ try:
+ model.cleanup()
+ except Exception as e:
+ print(f"Error during cleanup: {e}")
+
+
@pytest.mark.model_backend
@pytest.mark.parametrize(
"model_type",
@@ -31,8 +73,8 @@ from camel.utils import OpenAITokenCounter
ModelType.GPT_4O_MINI,
],
)
-def test_sglang_model(model_type: ModelType):
- model = SGLangModel(model_type, api_key="sglang")
+def test_sglang_model(model_type: ModelType, sglang_model_cleanup):
+ model = sglang_model_cleanup(model_type)
assert model.model_type == model_type
assert model.model_config_dict == SGLangConfig().as_dict()
assert isinstance(model.token_counter, OpenAITokenCounter)
@@ -41,7 +83,7 @@ def test_sglang_model(model_type: ModelType):
@pytest.mark.model_backend
-def test_sglang_model_unexpected_argument():
+def test_sglang_model_unexpected_argument(sglang_model_cleanup):
model_type = ModelType.GPT_4
model_config_dict = {"model_path": "vicuna-7b-v1.5"}
@@ -54,4 +96,75 @@ def test_sglang_model_unexpected_argument():
)
),
):
- _ = SGLangModel(model_type, model_config_dict, api_key="sglang")
+ _ = sglang_model_cleanup(model_type, model_config_dict)
+
+
+def test_sglang_function_call(sglang_model_cleanup):
+ test_tool = {
+ "type": "function",
+ "function": {
+ "name": "test_tool",
+ "description": "Test function",
+ "parameters": {"type": "object", "properties": {}},
+ },
+ }
+
+ model = sglang_model_cleanup(
+ ModelType.GPT_4,
+ model_config_dict={"tools": [test_tool]},
+ )
+
+ # Create a mock response object
+ from camel.types import (
+ ChatCompletion,
+ ChatCompletionMessage,
+ ChatCompletionMessageToolCall,
+ Choice,
+ CompletionUsage,
+ )
+
+ # create mock response
+ mock_response = ChatCompletion(
+ id="mock_id",
+ object="chat.completion",
+ created=1234567890,
+ model="meta-llama/Meta-Llama-3.1-8B-Instruct",
+ choices=[
+ Choice(
+ index=0,
+ message=ChatCompletionMessage(
+ role="assistant",
+ content=None,
+ tool_calls=[
+ ChatCompletionMessageToolCall(
+ id="0",
+ type="function",
+ function={"name": "test_tool", "arguments": "{}"},
+ )
+ ],
+ ),
+ finish_reason="tool_calls",
+ )
+ ],
+ usage=CompletionUsage(
+ prompt_tokens=10,
+ completion_tokens=20,
+ total_tokens=30,
+ ),
+ )
+
+ # Patch the run method to return our mock response
+ with patch.object(model, '_run', return_value=mock_response):
+ messages = [
+ {
+ "role": "user",
+ "content": "Use test_tool and respond with result",
+ }
+ ]
+
+ response = model.run(messages=messages)
+
+ assert len(response.choices[0].message.tool_calls) > 0
+ tool_call = response.choices[0].message.tool_calls[0]
+ assert tool_call.function.name == "test_tool"