🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org
Python agentai-societiesartificial-intelligencecommunicative-aicooperative-aideep-learninglarge-language-modelsmulti-agent-systemsnatural-language-processing
feat: add function call in SGLang (#1666)
Neil Johnson1dc816d379a2
diff --git a/camel/configs/sglang_config.py b/camel/configs/sglang_config.pyindex ce19d56fd..fe5251935 100644--- a/camel/configs/sglang_config.py+++ b/camel/configs/sglang_config.py@@ -13,7 +13,7 @@# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========from __future__ import annotations-from typing import Sequence, Union+from typing import Any, Dict, List, Optional, Sequence, Unionfrom camel.configs.base_config import BaseConfigfrom camel.types import NOT_GIVEN, NotGiven@@ -56,10 +56,11 @@ class SGLangConfig(BaseConfig):in the chat completion. The total length of input tokens andgenerated tokens is limited by the model's context length.(default: :obj:`None`)- tools (list[FunctionTool], optional): A list of tools the model may- call. Currently, only functions are supported as a tool. Use this- to provide a list of functions the model may generate JSON inputs- for. A max of 128 functions are supported.+ tools (list[Dict[str, Any]], optional): A list of tool definitions+ that the model can dynamically invoke. Each tool should be+ defined as a dictionary following OpenAI's function calling+ specification format. For more details, refer to the OpenAI+ documentation."""stop: Union[str, Sequence[str], NotGiven] = NOT_GIVEN@@ -70,6 +71,7 @@ class SGLangConfig(BaseConfig):presence_penalty: float = 0.0stream: bool = Falsemax_tokens: Union[int, NotGiven] = NOT_GIVEN+ tools: Optional[Union[List[Dict[str, Any]]]] = NoneSGLANG_API_PARAMS = {param for param in SGLangConfig.model_fields.keys()}diff --git a/camel/models/sglang_model.py b/camel/models/sglang_model.pyindex 727b50173..314bf9524 100644--- a/camel/models/sglang_model.py+++ b/camel/models/sglang_model.py@@ -97,9 +97,16 @@ class SGLangModel(BaseModelBackend):def _start_server(self) -> None:try:if not self._url:+ tool_call_flag = self.model_config_dict.get("tools")+ tool_call_arg = (+ f"--tool-call-parser {self._api_key} "+ if tool_call_flag+ else ""+ )cmd = (f"python -m sglang.launch_server "f"--model-path {self.model_type} "+ f"{tool_call_arg}"f"--port 30000 "f"--host 0.0.0.0")@@ -265,6 +272,19 @@ class SGLangModel(BaseModelBackend):"""return self.model_config_dict.get('stream', False)+ def __del__(self):+ r"""Properly clean up resources when the model is destroyed."""+ self.cleanup()++ def cleanup(self):+ r"""Terminate the server process and clean up resources."""+ with self._lock:+ if self.server_process:+ _terminate_process(self.server_process)+ self.server_process = None+ self._client = None+ logging.info("Server process terminated during cleanup.")+# Below are helper functions from sglang.utilsdef _terminate_process(process):@@ -326,21 +346,25 @@ def _execute_shell_command(command: str) -> subprocess.Popen:return subprocess.Popen(parts, text=True, stderr=subprocess.STDOUT)-def _wait_for_server(base_url: str, timeout: Optional[int] = None) -> None:+def _wait_for_server(base_url: str, timeout: Optional[int] = 30) -> None:r"""Wait for the server to be ready by polling the /v1/models endpoint.Args:base_url: The base URL of the server- timeout: Maximum time to wait in seconds. None means wait forever.+ timeout: Maximum time to wait in seconds. Default is 30 seconds."""import requests+ # Set a default value if timeout is None+ actual_timeout = 30 if timeout is None else timeout+start_time = time.time()while True:try:response = requests.get(f"{base_url}/v1/models",headers={"Authorization": "Bearer None"},+ timeout=5, # Add a timeout for the request itself)if response.status_code == 200:time.sleep(5)@@ -356,9 +380,15 @@ def _wait_for_server(base_url: str, timeout: Optional[int] = None) -> None:)break- if timeout and time.time() - start_time > timeout:+ if time.time() - start_time > actual_timeout:+ raise TimeoutError(+ f"Server did not become ready within "+ f"{actual_timeout} seconds"+ )+ except (requests.exceptions.RequestException, TimeoutError) as e:+ if time.time() - start_time > actual_timeout:raise TimeoutError(- "Server did not become ready within timeout period"+ f"Server did not become ready within "+ f"{actual_timeout} seconds: {e}")- except requests.exceptions.RequestException:time.sleep(1)diff --git a/examples/models/sglang_model_example.py b/examples/models/sglang_model_example.pyindex c01c5e42f..5e2b59821 100644--- a/examples/models/sglang_model_example.py+++ b/examples/models/sglang_model_example.py@@ -51,6 +51,84 @@ print(assistant_response.msg.content)"""===============================================================================-CAMEL AI ReferentialAction+Hello CAMEL AI. How can I assist you today?+===============================================================================+"""++weather_tool = [+ {+ "type": "function",+ "function": {+ "name": "get_current_weather",+ "description": "Get the current weather in a given location",+ "parameters": {+ "type": "object",+ "properties": {+ "city": {+ "type": "string",+ "description": "The city to find the weather for,\n"+ "e.g. 'San Francisco'",+ },+ "state": {+ "type": "string",+ "description": "The two-letter abbreviation for,\n"+ "the state (e.g., 'CA'), e.g. CA for California",+ },+ "unit": {+ "type": "string",+ "description": "Temperature unit (celsius/fahrenheit)",+ "enum": ["celsius", "fahrenheit"],+ },+ },+ "required": ["city", "state", "unit"],+ },+ },+ }+]+++r"""+Note that api_key defines the parser used to interpret responses.+Currently supported parsers include:+llama3: Llama 3.1 / 3.2 (e.g. meta-llama/Llama-3.1-8B-Instruct,+ meta-llama/Llama-3.2-1B-Instruct).+mistral: Mistral (e.g. mistralai/Mistral-7B-Instruct-v0.3,+ mistralai/Mistral-Nemo-Instruct-2407,+ mistralai/ Mistral-Nemo-Instruct-2407, mistralai/Mistral-7B-v0.3).+qwen25: Qwen 2.5 (e.g. Qwen/Qwen2.5-1.5B-Instruct, Qwen/Qwen2.5-7B-Instruct).+"""+sglang_model_with_tool = ModelFactory.create(+ model_platform=ModelPlatformType.SGLANG,+ model_type="meta-llama/Llama-3.2-1B-Instruct",+ model_config_dict={"temperature": 0.0, "tools": weather_tool},+ api_key="llama3",+)++assistant_sys_msg = (+ "You are a helpful assistant.\n"+ "Use the get_current_weather tool when asked about weather."+)+agent_with_tool = ChatAgent(+ assistant_sys_msg,+ model=sglang_model_with_tool,+ token_limit=4096,+ external_tools=weather_tool,+)+user_msg = "What's the weather in Boston today?"++assistant_response = agent_with_tool.step(user_msg)+external_tool_call = assistant_response.info.get('external_tool_call_request')+if external_tool_call:+ print(f"Detected external tool call: {external_tool_call.tool_name}")+ print(f"Arguments: {external_tool_call.args}")+ print(f"Tool Call ID: {external_tool_call.tool_call_id}")+else:+ print("No external tool call detected")++"""+===============================================================================+Detected external tool call: get_current_weather+Arguments: {'city': 'Boston', 'state': 'MA', 'unit': 'celsius'}+Tool Call ID: 0==============================================================================="""diff --git a/test/models/test_sglang_model.py b/test/models/test_sglang_model.pyindex 07c87d182..abd07c39d 100644--- a/test/models/test_sglang_model.py+++ b/test/models/test_sglang_model.py@@ -12,6 +12,7 @@# limitations under the License.# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========import re+from unittest.mock import MagicMock, patchimport pytest@@ -21,6 +22,47 @@ from camel.types import ModelType, UnifiedModelTypefrom camel.utils import OpenAITokenCounter+def sglang_model_cleanup():+ r"""Fixture to ensure SGLang model is cleaned up after each test."""+ models = []++ # Mock the server-related functions to avoid actual server startup+ with (+ patch(+ 'camel.models.sglang_model._execute_shell_command'+ ) as mock_execute,+ patch('camel.models.sglang_model._wait_for_server') as mock_wait,+ patch('camel.models.sglang_model.OpenAI') as mock_client,+ patch('camel.models.sglang_model.AsyncOpenAI') as mock_async_client,+ ):+ # Configure mocks+ mock_execute.return_value = MagicMock()+ mock_wait.return_value = None+ mock_client.return_value = MagicMock()+ mock_async_client.return_value = MagicMock()++ def _create_model(+ model_type, model_config_dict=None, api_key="sglang"+ ):+ model = SGLangModel(model_type, model_config_dict, api_key=api_key)+ # Set up the model to use our mocks+ model._url = "http://mock-server:30000/v1"+ model._client = mock_client.return_value+ model._async_client = mock_async_client.return_value+ models.append(model)+ return model++ yield _create_model++ # Clean up all models after test+ for model in models:+ try:+ model.cleanup()+ except Exception as e:+ print(f"Error during cleanup: {e}")++@pytest.mark.model_backend@pytest.mark.parametrize("model_type",@@ -31,8 +73,8 @@ from camel.utils import OpenAITokenCounterModelType.GPT_4O_MINI,],)-def test_sglang_model(model_type: ModelType):- model = SGLangModel(model_type, api_key="sglang")+def test_sglang_model(model_type: ModelType, sglang_model_cleanup):+ model = sglang_model_cleanup(model_type)assert model.model_type == model_typeassert model.model_config_dict == SGLangConfig().as_dict()assert isinstance(model.token_counter, OpenAITokenCounter)@@ -41,7 +83,7 @@ def test_sglang_model(model_type: ModelType):@pytest.mark.model_backend-def test_sglang_model_unexpected_argument():+def test_sglang_model_unexpected_argument(sglang_model_cleanup):model_type = ModelType.GPT_4model_config_dict = {"model_path": "vicuna-7b-v1.5"}@@ -54,4 +96,75 @@ def test_sglang_model_unexpected_argument():)),):- _ = SGLangModel(model_type, model_config_dict, api_key="sglang")+ _ = sglang_model_cleanup(model_type, model_config_dict)++[email protected]_backend+def test_sglang_function_call(sglang_model_cleanup):+ test_tool = {+ "type": "function",+ "function": {+ "name": "test_tool",+ "description": "Test function",+ "parameters": {"type": "object", "properties": {}},+ },+ }++ model = sglang_model_cleanup(+ ModelType.GPT_4,+ model_config_dict={"tools": [test_tool]},+ )++ # Create a mock response object+ from camel.types import (+ ChatCompletion,+ ChatCompletionMessage,+ ChatCompletionMessageToolCall,+ Choice,+ CompletionUsage,+ )++ # create mock response+ mock_response = ChatCompletion(+ id="mock_id",+ object="chat.completion",+ created=1234567890,+ model="meta-llama/Meta-Llama-3.1-8B-Instruct",+ choices=[+ Choice(+ index=0,+ message=ChatCompletionMessage(+ role="assistant",+ content=None,+ tool_calls=[+ ChatCompletionMessageToolCall(+ id="0",+ type="function",+ function={"name": "test_tool", "arguments": "{}"},+ )+ ],+ ),+ finish_reason="tool_calls",+ )+ ],+ usage=CompletionUsage(+ prompt_tokens=10,+ completion_tokens=20,+ total_tokens=30,+ ),+ )++ # Patch the run method to return our mock response+ with patch.object(model, '_run', return_value=mock_response):+ messages = [+ {+ "role": "user",+ "content": "Use test_tool and respond with result",+ }+ ]++ response = model.run(messages=messages)++ assert len(response.choices[0].message.tool_calls) > 0+ tool_call = response.choices[0].message.tool_calls[0]+ assert tool_call.function.name == "test_tool"