Skip to content

armgit / camel

Public Imported from GitHub
0

🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org

Add test for covering method `run` of Open-source models (#297)

Wenxuan Licccffabfa38f
diff --git a/camel/models/open_source_model.py b/camel/models/open_source_model.py
index cea4f0c99..a9ebf71ab 100644
--- a/camel/models/open_source_model.py
+++ b/camel/models/open_source_model.py
@@ -89,7 +89,7 @@ class OpenSourceModel(BaseModelBackend):
self,
messages: List[Dict],
) -> Dict[str, Any]:
- r"""Run inference of OpenAI chat completion.
+ r"""Run inference of OpenAI-API-style chat completion.
Args:
messages (List[Dict]): Message list with the chat history
diff --git a/camel/utils/__init__.py b/camel/utils/__init__.py
index 2df33571f..c833d676d 100644
--- a/camel/utils/__init__.py
+++ b/camel/utils/__init__.py
@@ -20,6 +20,7 @@ from .functions import (
download_tasks,
parse_doc,
get_task_list,
+ check_server_running,
)
from .token_counting import (
get_model_encoding,
@@ -39,6 +40,7 @@ __all__ = [
'parse_doc',
'get_task_list',
'get_model_encoding',
+ 'check_server_running',
'BaseTokenCounter',
'OpenAITokenCounter',
'OpenSourceTokenCounter',
diff --git a/camel/utils/functions.py b/camel/utils/functions.py
index 2b778ed2e..a4a1cd048 100644
--- a/camel/utils/functions.py
+++ b/camel/utils/functions.py
@@ -14,6 +14,7 @@
import inspect
import os
import re
+import socket
import time
import zipfile
from functools import wraps
@@ -28,6 +29,7 @@ from typing import (
TypeVar,
cast,
)
+from urllib.parse import urlparse
import requests
@@ -224,3 +226,25 @@ def get_task_list(task_response: str) -> List[str]:
if task_name.strip() and task_id.isnumeric():
new_tasks_list.append(task_name)
return new_tasks_list
+
+
+def check_server_running(server_url: str) -> bool:
+ r"""Check whether the port refered by the URL to the server
+ is open.
+
+ Args:
+ server_url (str): The URL to the server running LLM inference
+ service.
+
+ Returns:
+ bool: Whether the port is open for packets (server is running).
+ """
+ parsed_url = urlparse(server_url)
+ url_tuple = (parsed_url.hostname, parsed_url.port)
+
+ sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ result = sock.connect_ex(url_tuple)
+ sock.close()
+
+ # if the port is open, the result should be 0.
+ return result == 0
diff --git a/test/models/test_open_source_model.py b/test/models/test_open_source_model.py
index 7a804a231..aad7d9059 100644
--- a/test/models/test_open_source_model.py
+++ b/test/models/test_open_source_model.py
@@ -22,12 +22,13 @@ from camel.configs import (
)
from camel.models import OpenSourceModel
from camel.typing import ModelType
-from camel.utils import OpenSourceTokenCounter
+from camel.utils import OpenSourceTokenCounter, check_server_running
MODEL_PATH_MAP = {
ModelType.VICUNA: "lmsys/vicuna-7b-v1.5",
ModelType.VICUNA_16K: "lmsys/vicuna-7b-v1.5-16k",
}
+DEFAULT_SERVER_URL = "http://localhost:8000/v1"
@pytest.mark.model_backend
@@ -40,7 +41,7 @@ def test_open_source_model(model_type):
model_name = model_path.split('/')[-1]
model_config = OpenSourceConfig(
model_path=model_path,
- server_url="http://localhost:8000/v1",
+ server_url=DEFAULT_SERVER_URL,
)
model_config_dict = model_config.__dict__
model = OpenSourceModel(model_type, model_config_dict)
@@ -55,13 +56,32 @@ def test_open_source_model(model_type):
assert isinstance(model.model_type.token_limit, int)
[email protected]("model_type", [ModelType.VICUNA])
[email protected](not check_server_running(DEFAULT_SERVER_URL),
+ reason="No server running LLM inference is provided.")
+def test_open_source_model_run(model_type):
+ model_path = MODEL_PATH_MAP[model_type]
+ model_config = OpenSourceConfig(
+ model_path=model_path,
+ server_url=DEFAULT_SERVER_URL,
+ )
+ model_config_dict = model_config.__dict__
+ model = OpenSourceModel(model_type, model_config_dict)
+
+ messages = [{"role": "user", "content": "Tell me a joke."}]
+ response = model.run(messages)
+
+ assert isinstance(response, dict)
+
+
@pytest.mark.model_backend
def test_open_source_model_close_source_model_type():
model_type = ModelType.GPT_3_5_TURBO
model_path = MODEL_PATH_MAP[ModelType.VICUNA]
model_config = OpenSourceConfig(
model_path=model_path,
- server_url="http://localhost:8000/v1",
+ server_url=DEFAULT_SERVER_URL,
)
model_config_dict = model_config.__dict__
@@ -91,7 +111,7 @@ def test_open_source_model_unexpected_argument():
model_path = MODEL_PATH_MAP[ModelType.VICUNA]
model_config = OpenSourceConfig(
model_path=model_path,
- server_url="http://localhost:8000/v1",
+ server_url=DEFAULT_SERVER_URL,
api_params=FunctionCallingConfig(),
)
model_config_dict = model_config.__dict__
@@ -109,7 +129,7 @@ def test_open_source_model_invalid_model_path():
model_path = "vicuna-7b-v1.5"
model_config = OpenSourceConfig(
model_path=model_path,
- server_url="http://localhost:8000/v1",
+ server_url=DEFAULT_SERVER_URL,
)
model_config_dict = model_config.__dict__
@@ -127,7 +147,7 @@ def test_open_source_model_unmatched_model_path():
model_path = MODEL_PATH_MAP[ModelType.VICUNA]
model_config = OpenSourceConfig(
model_path=model_path,
- server_url="http://localhost:8000/v1",
+ server_url=DEFAULT_SERVER_URL,
)
model_config_dict = model_config.__dict__
@@ -143,7 +163,7 @@ def test_open_source_model_missing_model_path():
model_type = ModelType.VICUNA
model_config = OpenSourceConfig(
model_path=None,
- server_url="http://localhost:8000/v1",
+ server_url=DEFAULT_SERVER_URL,
)
model_config_dict = model_config.__dict__