š« CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org
Python agentai-societiesartificial-intelligencecommunicative-aicooperative-aideep-learninglarge-language-modelsmulti-agent-systemsnatural-language-processing
Add a stub model backend (#161)
Dmitrii Khizbullindbca94b126f2
diff --git a/.github/workflows/pytest_apps.yml b/.github/workflows/pytest_apps.ymlindex 8839ace1a..1187de325 100644--- a/.github/workflows/pytest_apps.yml+++ b/.github/workflows/pytest_apps.yml@@ -1,7 +1,7 @@# This workflow will install Python dependencies, run tests and lint with a single version of Python# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python-name: Pytest Gradio Apps+name: Pytest Gradio Apps and Exampleson:push:@@ -40,4 +40,4 @@ jobs:env:OPENAI_API_KEY: "${{ secrets.OPENAI_API_KEY }}"run: |- pytest apps/+ pytest -m "not slow" apps/ examples/diff --git a/CONTRIBUTING.md b/CONTRIBUTING.mdindex 20067ae3a..97d877cb5 100644--- a/CONTRIBUTING.md+++ b/CONTRIBUTING.md@@ -112,12 +112,17 @@ The coverage report will be generated at `htmlcov/index.html`.Unit tests cover modular logic that doesn't require calls to outside APIs. Currently, the test setup requires an OpenAI API key to test the framework, making them resemble integration tests.-To run the tests, use the following command:+To run all tests including those that use OpenAI API, use the following command:```bashpytest .```+To quickly run only local isolated unit and integration tests:+```bash+pytest -m "not model_backend and not slow" .+```+If you're developing with VSCode, make sure to create a `.env` file in the repository root and include your OpenAI API key:```diff --git a/apps/agents/agents.py b/apps/agents/agents.pyindex f96579502..253572436 100644--- a/apps/agents/agents.py+++ b/apps/agents/agents.py@@ -230,7 +230,7 @@ def role_playing_chat_init(state) -> \return state, state.chat, progress_update-# WORKAROUND: do not add type hinst for session and chatbot_histoty+# WORKAROUND: do not add type hints for session and chatbot_historydef role_playing_chat_cont(state) -> \Tuple[State, ChatBotHistory, Dict, Dict]:""" Produce a pair of messages by an assistant and a user.diff --git a/apps/agents/test/test_agents.py b/apps/agents/test/test_agents.pyindex 8f2b95b50..13fa14c57 100644--- a/apps/agents/test/test_agents.py+++ b/apps/agents/test/test_agents.py@@ -11,9 +11,8 @@# See the License for the specific language governing permissions and# limitations under the License.# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========-import unittest-import gradio as gr+import pytestfrom apps.agents.agents import (State,@@ -27,46 +26,43 @@ from apps.agents.agents import ()-class TestAgents(unittest.TestCase):-- def test_construct_blocks(self):- blocks = construct_blocks(None)- self.assertIsInstance(blocks, gr.Blocks)+def test_construct_blocks():+ blocks = construct_blocks(None)+ assert isinstance(blocks, gr.Blocks)- def test_utils(self):- args = parse_arguments()- self.assertIsNotNone(args)- def test_session(self):- state = State.empty()+def test_utils():+ args = parse_arguments()+ assert args is not None- state, _, _ = cleanup_on_launch(state)- assistant = "professor"- user = "PhD student"- original_task = "Recommend AI conferences to publish a paper"- max_messages = 10- with_task_specifier = False- word_limit = 50- state, specified_task_prompt, planned_task_upd, chat, progress_upd = \- role_playing_start(state, assistant, user, original_task,- max_messages, with_task_specifier, word_limit)[email protected]_backend+def test_session():+ state = State.empty()- self.assertIsNotNone(state.session)+ state, _, _ = cleanup_on_launch(state)- state, chat, progress_update = \- role_playing_chat_init(state)+ assistant = "professor"+ user = "PhD student"+ original_task = "Recommend AI conferences to publish a paper"+ max_messages = 10+ with_task_specifier = False+ word_limit = 50+ state, specified_task_prompt, planned_task_upd, chat, progress_upd = \+ role_playing_start(state, assistant, user, original_task,+ max_messages, with_task_specifier, word_limit)- self.assertIsNotNone(state.session)+ assert state.session is not None- for _ in range(5):- state, chat, progress_update, start_bn_update =\- role_playing_chat_cont(state)+ state, chat, progress_update = \+ role_playing_chat_init(state)- state, _, _ = stop_session(state)+ assert state.session is not None- self.assertIsNone(state.session)+ for _ in range(5):+ state, chat, progress_update, start_bn_update =\+ role_playing_chat_cont(state)+ state, _, _ = stop_session(state)-if __name__ == '__main__':- unittest.main()+ assert state.session is Nonediff --git a/apps/agents/test/test_text_utils.py b/apps/agents/test/test_text_utils.pyindex eeba21764..d908224f3 100644--- a/apps/agents/test/test_text_utils.py+++ b/apps/agents/test/test_text_utils.py@@ -11,110 +11,107 @@# See the License for the specific language governing permissions and# limitations under the License.# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========-from unittest import TestCase-import apps.agents.text_utils as text_utils-class TestTextUtils(TestCase):+def test_split_markdown_code_newline():+ inp = ("Solution: To preprocess the historical stock data, we "+ "can perform the following steps:\n\n1. Remove any unnecessary"+ " columns that do not contribute to the prediction, such as"+ " the stock symbol or date.\n2. Check for and handle any "+ "missing or null values in the data.\n3. Normalize the data"+ " to ensure that all features are on the same scale. This "+ "can be done using techniques such as Min-Max scaling or "+ "Z-score normalization.\n4. Split the data into training "+ "and testing sets. The training set will be used to train "+ "the machine learning model, while the testing set will be "+ "used to evaluate its performance.\n\nHere is an example "+ "code snippet to preprocess the data using Pandas:\n\n```\n"+ "import pandas as pd\nfrom sklearn.preprocessing import "+ "MinMaxScaler\nfrom sklearn.model_selection import "+ "train_test_split\n\n# Read in the historical stock data\ndata"+ " = pd.read_csv('historical_stock_data.csv')\n\n# Remove "+ "unnecessary columns\ndata = data.drop(['symbol', 'date'], "+ "axis=1)\n\n# Handle missing values\ndata = data.fillna("+ "method='ffill')\n\n# Normalize the data\nscaler = "+ "MinMaxScaler()\ndata = scaler.fit_transform(data)\n\n# "+ "Split the data into training and testing sets\nX_train, "+ "X_test, y_train, y_test = train_test_split(data[:, :-1], "+ "data[:, -1], test_size=0.2, random_state=42)\n```\n\nNext "+ "request.")+ gt = ("Solution: To preprocess the historical stock data, we "+ "can perform the following steps:\n\n1. Remove any unnecessary"+ " columns that do not contribute to the prediction, such as"+ " the stock symbol or date.\n2. Check for and handle any missing"+ " or null values in the data.\n3. Normalize the data to ensure"+ " that all features are on the same scale. This can be done"+ " using techniques such as Min-Max scaling or Z-score"+ " normalization.\n4. Split the data into training and testing"+ " sets. The training set will be used to train the machine"+ " learning model, while the testing set will be used to"+ " evaluate its performance.\n\nHere is an example code snippet"+ " to preprocess the data using Pandas:\n\n\n```import pandas"+ " as pd```\n```from sklearn.preprocessing import MinMaxScaler"+ "```\n```from sklearn.model_selection import train_test_split"+ "```\n\n```# Read in the historical stock data```\n```data ="+ " pd.read_csv('historical_stock_data.csv')```\n\n```# Remove"+ " unnecessary columns```\n```data = data.drop(['symbol', "+ "'date'], axis=1)```\n\n```# Handle missing values```\n```data"+ " = data.fillna(method='ffill')```\n\n```# Normalize the data"+ "```\n```scaler = MinMaxScaler()```\n```data = scaler."+ "fit_transform(data)```\n\n```# Split the data into training"+ " and testing sets```\n```X_train, X_test, y_train, y_test"+ " = train_test_split(data[:, :-1], data[:, -1], test_size=0.2,"+ " random_state=42)```\n\n\nNext request.")- def test_split_markdown_code_newline(self):- inp = ("Solution: To preprocess the historical stock data, we "- "can perform the following steps:\n\n1. Remove any unnecessary"- " columns that do not contribute to the prediction, such as"- " the stock symbol or date.\n2. Check for and handle any "- "missing or null values in the data.\n3. Normalize the data"- " to ensure that all features are on the same scale. This "- "can be done using techniques such as Min-Max scaling or "- "Z-score normalization.\n4. Split the data into training "- "and testing sets. The training set will be used to train "- "the machine learning model, while the testing set will be "- "used to evaluate its performance.\n\nHere is an example "- "code snippet to preprocess the data using Pandas:\n\n```\n"- "import pandas as pd\nfrom sklearn.preprocessing import "- "MinMaxScaler\nfrom sklearn.model_selection import "- "train_test_split\n\n# Read in the historical stock data\ndata"- " = pd.read_csv('historical_stock_data.csv')\n\n# Remove "- "unnecessary columns\ndata = data.drop(['symbol', 'date'], "- "axis=1)\n\n# Handle missing values\ndata = data.fillna("- "method='ffill')\n\n# Normalize the data\nscaler = "- "MinMaxScaler()\ndata = scaler.fit_transform(data)\n\n# "- "Split the data into training and testing sets\nX_train, "- "X_test, y_train, y_test = train_test_split(data[:, :-1], "- "data[:, -1], test_size=0.2, random_state=42)\n```\n\nNext "- "request.")- gt = ("Solution: To preprocess the historical stock data, we "- "can perform the following steps:\n\n1. Remove any unnecessary"- " columns that do not contribute to the prediction, such as"- " the stock symbol or date.\n2. Check for and handle any missing"- " or null values in the data.\n3. Normalize the data to ensure"- " that all features are on the same scale. This can be done"- " using techniques such as Min-Max scaling or Z-score"- " normalization.\n4. Split the data into training and testing"- " sets. The training set will be used to train the machine"- " learning model, while the testing set will be used to"- " evaluate its performance.\n\nHere is an example code snippet"- " to preprocess the data using Pandas:\n\n\n```import pandas"- " as pd```\n```from sklearn.preprocessing import MinMaxScaler"- "```\n```from sklearn.model_selection import train_test_split"- "```\n\n```# Read in the historical stock data```\n```data ="- " pd.read_csv('historical_stock_data.csv')```\n\n```# Remove"- " unnecessary columns```\n```data = data.drop(['symbol', "- "'date'], axis=1)```\n\n```# Handle missing values```\n```data"- " = data.fillna(method='ffill')```\n\n```# Normalize the data"- "```\n```scaler = MinMaxScaler()```\n```data = scaler."- "fit_transform(data)```\n\n```# Split the data into training"- " and testing sets```\n```X_train, X_test, y_train, y_test"- " = train_test_split(data[:, :-1], data[:, -1], test_size=0.2,"- " random_state=42)```\n\n\nNext request.")+ out = text_utils.split_markdown_code(inp)+ assert out == gt- out = text_utils.split_markdown_code(inp)- self.assertEqual(out, gt)- def test_split_markdown_code_br(self):- inp = ("Solution: Define the Bayesian optimization object."- "\n"- "We can define the Bayesian optimization object using"- " the BayesianOptimization class from the bayes_opt module."- " Here is an example of how to define the Bayesian"- " optimization object:"- "\n"- "```<br># Replace 'objective_function' with the actual"- " objective function<br># Replace 'bounds' with the actual"- " search space<br># Replace 'model' with the actual machine"- " learning model<br>bo = BayesianOptimization(<br> "- "f=objective_function,<br> pbounds=bounds,<br> verbose=2,<br>"- " random_state=1,<br>)<br>```"- "\n"- "This will define the Bayesian optimization object with the"- " specified objective function, search space, and machine"- " learning model. The BayesianOptimization class takes "- "several arguments, including f for the objective function,"- " pbounds for the search space, verbose for the verbosity"- " level, and random_state for the random seed."- "\n"- "Next request.")- gt = ("Solution: Define the Bayesian optimization object."- "\n"- "We can define the Bayesian optimization object using the"- " BayesianOptimization class from the bayes_opt module. Here is"- " an example of how to define the Bayesian optimization object:"- "\n"- "\n```# Replace 'objective_function' with the actual objective"- " function```\n```# Replace 'bounds' with the actual search"- " space```\n```# Replace 'model' with the actual machine"- " learning model```\n```bo = BayesianOptimization(```\n```"- " f=objective_function,```\n``` pbounds=bounds,```\n``` "- "verbose=2,```\n``` random_state=1,```\n```)```\n"- "\n"- "This will define the Bayesian optimization object with "- "the specified objective function, search space, and machine"- " learning model. The BayesianOptimization class takes several"- " arguments, including f for the objective function, pbounds"- " for the search space, verbose for the verbosity level, and"- " random_state for the random seed."- "\n"- "Next request.")+def test_split_markdown_code_br():+ inp = ("Solution: Define the Bayesian optimization object."+ "\n"+ "We can define the Bayesian optimization object using"+ " the BayesianOptimization class from the bayes_opt module."+ " Here is an example of how to define the Bayesian"+ " optimization object:"+ "\n"+ "```<br># Replace 'objective_function' with the actual"+ " objective function<br># Replace 'bounds' with the actual"+ " search space<br># Replace 'model' with the actual machine"+ " learning model<br>bo = BayesianOptimization(<br> "+ "f=objective_function,<br> pbounds=bounds,<br> verbose=2,<br>"+ " random_state=1,<br>)<br>```"+ "\n"+ "This will define the Bayesian optimization object with the"+ " specified objective function, search space, and machine"+ " learning model. The BayesianOptimization class takes "+ "several arguments, including f for the objective function,"+ " pbounds for the search space, verbose for the verbosity"+ " level, and random_state for the random seed."+ "\n"+ "Next request.")+ gt = ("Solution: Define the Bayesian optimization object."+ "\n"+ "We can define the Bayesian optimization object using the"+ " BayesianOptimization class from the bayes_opt module. Here is"+ " an example of how to define the Bayesian optimization object:"+ "\n"+ "\n```# Replace 'objective_function' with the actual objective"+ " function```\n```# Replace 'bounds' with the actual search"+ " space```\n```# Replace 'model' with the actual machine"+ " learning model```\n```bo = BayesianOptimization(```\n```"+ " f=objective_function,```\n``` pbounds=bounds,```\n``` "+ "verbose=2,```\n``` random_state=1,```\n```)```\n"+ "\n"+ "This will define the Bayesian optimization object with "+ "the specified objective function, search space, and machine"+ " learning model. The BayesianOptimization class takes several"+ " arguments, including f for the objective function, pbounds"+ " for the search space, verbose for the verbosity level, and"+ " random_state for the random seed."+ "\n"+ "Next request.")- out = text_utils.split_markdown_code(inp)- self.assertEqual(out, gt)+ out = text_utils.split_markdown_code(inp)+ assert out == gtdiff --git a/apps/common/test/test_auto_zip.py b/apps/common/test/test_auto_zip.pyindex 9ceaff0d7..460b7af68 100644--- a/apps/common/test/test_auto_zip.py+++ b/apps/common/test/test_auto_zip.py@@ -12,7 +12,6 @@# limitations under the License.# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========import os-from unittest import TestCasefrom apps.common.auto_zip import AutoZip@@ -20,16 +19,14 @@ REPO_ROOT = os.path.realpath(os.path.join(os.path.dirname(os.path.abspath(__file__)), "../../.."))-class TestAutoZip(TestCase):+def test_dict():+ path = os.path.join(REPO_ROOT, "apps/common/test/test_archive_1.zip")+ zp = AutoZip(path, ".txt")- def test_dict(self):- path = os.path.join(REPO_ROOT, "apps/common/test/test_archive_1.zip")- zip = AutoZip(path, ".txt")+ d = zp.as_dict()+ assert isinstance(d, dict)+ assert len(d) == 3- d = zip.as_dict()- self.assertIsInstance(d, dict)- self.assertEqual(len(d), 3)-- d = zip.as_dict(include_zip_name=True)- self.assertIsInstance(d, dict)- self.assertEqual(len(d), 3)+ d = zp.as_dict(include_zip_name=True)+ assert isinstance(d, dict)+ assert len(d) == 3diff --git a/apps/data_explorer/test/test_data_explorer.py b/apps/data_explorer/test/test_data_explorer.pyindex b60b5efb6..4c8fa2445 100644--- a/apps/data_explorer/test/test_data_explorer.py+++ b/apps/data_explorer/test/test_data_explorer.py@@ -12,7 +12,6 @@# limitations under the License.# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========import os-import unittestimport urllib.requestimport gradio as gr@@ -21,25 +20,19 @@ from apps.data_explorer.data_explorer import construct_blocks, parse_argumentsfrom apps.data_explorer.loader import REPO_ROOT-class TestDataExplorer(unittest.TestCase):+def test_app():+ test_data_url = ("https://storage.googleapis.com/"+ "camel-bucket/datasets/test/DATA.zip")+ data_dir = os.path.join(REPO_ROOT, "datasets_test")+ test_file_path = os.path.join(data_dir, os.path.split(test_data_url)[1])+ os.makedirs(data_dir, exist_ok=True)+ urllib.request.urlretrieve(test_data_url, test_file_path)- def test_app(self):- test_data_url = ("https://storage.googleapis.com/"- "camel-bucket/datasets/test/DATA.zip")- data_dir = os.path.join(REPO_ROOT, "datasets_test")- test_file_path = os.path.join(data_dir,- os.path.split(test_data_url)[1])- os.makedirs(data_dir, exist_ok=True)- urllib.request.urlretrieve(test_data_url, test_file_path)+ blocks = construct_blocks(data_dir, None)- blocks = construct_blocks(data_dir, None)+ assert isinstance(blocks, gr.Blocks)- self.assertIsInstance(blocks, gr.Blocks)- def test_utils(self):- args = parse_arguments()- self.assertIsNotNone(args)---if __name__ == '__main__':- unittest.main()+def test_utils():+ args = parse_arguments()+ assert args is not Nonediff --git a/apps/data_explorer/test/test_loader.py b/apps/data_explorer/test/test_loader.pyindex 8a9a74be8..03f626e93 100644--- a/apps/data_explorer/test/test_loader.py+++ b/apps/data_explorer/test/test_loader.py@@ -11,13 +11,9 @@# See the License for the specific language governing permissions and# limitations under the License.# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========-from unittest import TestCase-import apps.data_explorer.loader as loader-class TestLoader(TestCase):-- def test_load_datasets_smoke(self):- data = loader.load_datasets()- self.assertIsNotNone(data)+def test_load_datasets_smoke():+ data = loader.load_datasets()+ assert data is not Nonediff --git a/camel/agents/chat_agent.py b/camel/agents/chat_agent.pyindex 427f13e88..1d9f90ae0 100644--- a/camel/agents/chat_agent.py+++ b/camel/agents/chat_agent.py@@ -14,7 +14,6 @@from dataclasses import dataclassfrom typing import Any, Dict, List, Optional-import openaifrom tenacity import retryfrom tenacity.stop import stop_after_attemptfrom tenacity.wait import wait_exponential@@ -22,7 +21,8 @@ from tenacity.wait import wait_exponentialfrom camel.agents import BaseAgentfrom camel.configs import ChatGPTConfigfrom camel.messages import ChatMessage, MessageType, SystemMessage-from camel.typing import ModelType+from camel.model_backend import ModelBackend, ModelFactory+from camel.typing import ModelType, RoleTypefrom camel.utils import (get_model_token_limit,num_tokens_from_messages,@@ -72,20 +72,23 @@ class ChatAgent(BaseAgent):def __init__(self,system_message: SystemMessage,- model: ModelType = ModelType.GPT_3_5_TURBO,+ model: Optional[ModelType] = None,model_config: Optional[Any] = None,message_window_size: Optional[int] = None,) -> None:- self.system_message = system_message- self.role_name = system_message.role_name- self.role_type = system_message.role_type- self.meta_dict = system_message.meta_dict+ self.system_message: SystemMessage = system_message+ self.role_name: str = system_message.role_name+ self.role_type: RoleType = system_message.role_type- self.model = model- self.model_config = model_config or ChatGPTConfig()- self.model_token_limit = get_model_token_limit(self.model)- self.message_window_size = message_window_size+ self.model: ModelType = (model if model is not None else+ ModelType.GPT_3_5_TURBO)+ self.model_config: ChatGPTConfig = model_config or ChatGPTConfig()+ self.model_token_limit: int = get_model_token_limit(self.model)+ self.message_window_size: Optional[int] = message_window_size++ self.model_backend: ModelBackend = ModelFactory.create(+ self.model, self.model_config.__dict__)self.terminated: bool = Falseself.init_messages()@@ -177,11 +180,7 @@ class ChatAgent(BaseAgent):info: Dict[str, Any]if num_tokens < self.model_token_limit:- response = openai.ChatCompletion.create(- model=self.model.value,- messages=openai_messages,- **self.model_config.__dict__,- )+ response = self.model_backend.run(messages=openai_messages)if not isinstance(response, dict):raise RuntimeError("OpenAI returned unexpected struct")output_messages = [@@ -198,7 +197,6 @@ class ChatAgent(BaseAgent):],num_tokens,)-else:self.terminated = Trueoutput_messages = []diff --git a/camel/agents/role_playing.py b/camel/agents/role_playing.pyindex be96767e4..c144dc872 100644--- a/camel/agents/role_playing.py+++ b/camel/agents/role_playing.py@@ -152,12 +152,12 @@ class RolePlaying:],))- self.assistant_agent = ChatAgent(+ self.assistant_agent: ChatAgent = ChatAgent(self.assistant_sys_msg,mode_type,**(assistant_agent_kwargs or {}),)- self.user_agent = ChatAgent(+ self.user_agent: ChatAgent = ChatAgent(self.user_sys_msg,mode_type,**(user_agent_kwargs or {}),diff --git a/camel/agents/task_agent.py b/camel/agents/task_agent.pyindex b89598eb1..20320cfa9 100644--- a/camel/agents/task_agent.py+++ b/camel/agents/task_agent.py@@ -44,12 +44,13 @@ class TaskSpecifyAgent(ChatAgent):def __init__(self,- model: ModelType = ModelType.GPT_3_5_TURBO,+ model: Optional[ModelType] = None,task_type: TaskType = TaskType.AI_SOCIETY,model_config: Optional[Any] = None,task_specify_prompt: Optional[Union[str, TextPrompt]] = None,word_limit: int = DEFAULT_WORD_LIMIT,) -> None:+if task_specify_prompt is None:task_specify_prompt_template = PromptTemplateGenerator().get_task_specify_prompt(task_type)@@ -124,7 +125,7 @@ class TaskPlannerAgent(ChatAgent):def __init__(self,- model: ModelType = ModelType.GPT_3_5_TURBO,+ model: Optional[ModelType] = None,model_config: Any = None,) -> None:diff --git a/camel/model_backend.py b/camel/model_backend.pynew file mode 100644index 000000000..b8ef5745a--- /dev/null+++ b/camel/model_backend.py@@ -0,0 +1,101 @@+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========+# Licensed under the Apache License, Version 2.0 (the āLicenseā);+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an āAS ISā BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========+from abc import ABC, abstractmethod+from typing import Any, Dict++from camel.typing import ModelType+++class ModelBackend(ABC):+ r"""Base class for different model backends.+ May be OpenAI API, a local LLM, a stub for unit tests, etc."""++ @abstractmethod+ def run(self, *args, **kwargs) -> Dict[str, Any]:+ r"""Runs the query to the backend model.++ Raises:+ RuntimeError: if the return value from OpenAI API+ is not a dict that is expected.++ Returns:+ Dict[str, Any]: All backends must return a dict in OpenAI format.+ """+ pass+++class OpenAIModel(ModelBackend):+ r"""OpenAI API in a unified ModelBackend interface."""++ def __init__(self, model_type: ModelType, model_config_dict: Dict) -> None:+ super().__init__()+ self.model_type = model_type+ self.model_config_dict = model_config_dict++ def run(self, *args, **kwargs) -> Dict[str, Any]:++ import openai+ response = openai.ChatCompletion.create(*args, **kwargs,+ model=self.model_type.value,+ **self.model_config_dict)+ if not isinstance(response, Dict):+ raise RuntimeError("Unexpected return from OpenAI API")+ return response+++class StubModel(ModelBackend):+ r"""A dummy model used for unit tests."""++ def __init__(self, *args, **kwargs) -> None:+ super().__init__()++ def run(self, *args, **kwargs) -> Dict[str, Any]:+ ARBITRARY_STRING = "Lorem Ipsum"++ return dict(+ id="stub_model_id",+ usage=dict(),+ choices=[+ dict(finish_reason="stop",+ message=dict(content=ARBITRARY_STRING, role="assistant"))+ ],+ )+++class ModelFactory:+ r"""Factory of backend models.++ Raises:+ ValueError: in case the provided model type is unknown.+ """++ @staticmethod+ def create(model_type: ModelType, model_config_dict: Dict) -> ModelBackend:+ default_model_type = ModelType.GPT_3_5_TURBO++ if model_type in {+ ModelType.GPT_3_5_TURBO, ModelType.GPT_4, ModelType.GPT_4_32k,+ None+ }:+ model_class = OpenAIModel+ elif model_type == ModelType.STUB:+ model_class = StubModel+ else:+ raise ValueError("Unknown model")++ if model_type is None:+ model_type = default_model_type++ inst = model_class(model_type, model_config_dict)+ return instdiff --git a/camel/typing.py b/camel/typing.pyindex 9e0f163c1..5e0a95dfa 100644--- a/camel/typing.py+++ b/camel/typing.py@@ -26,6 +26,11 @@ class ModelType(Enum):GPT_3_5_TURBO = "gpt-3.5-turbo"GPT_4 = "gpt-4"GPT_4_32k = "gpt-4-32k"+ STUB = "stub"++ @property+ def value_for_tiktoken(self):+ return self.value if self.name != "STUB" else "gpt-3.5-turbo"class TaskType(Enum):diff --git a/camel/utils.py b/camel/utils.pyindex db4b0a2bd..8fd302cd3 100644--- a/camel/utils.py+++ b/camel/utils.py@@ -76,15 +76,15 @@ def num_tokens_from_messages(- https://platform.openai.com/docs/models/gpt-3-5"""try:- encoding = tiktoken.encoding_for_model(model.value)+ value_for_tiktoken = model.value_for_tiktoken+ encoding = tiktoken.encoding_for_model(value_for_tiktoken)except KeyError:encoding = tiktoken.get_encoding("cl100k_base")- if model == ModelType.GPT_3_5_TURBO:- return count_tokens_openai_chat_models(messages, encoding)- elif model == ModelType.GPT_4:- return count_tokens_openai_chat_models(messages, encoding)- elif model == ModelType.GPT_4_32k:+ if model in {+ ModelType.GPT_3_5_TURBO, ModelType.GPT_4, ModelType.GPT_4_32k,+ ModelType.STUB+ }:return count_tokens_openai_chat_models(messages, encoding)else:raise NotImplementedError(@@ -108,10 +108,12 @@ def get_model_token_limit(model: ModelType) -> int:"""if model == ModelType.GPT_3_5_TURBO:return 4096- if model == ModelType.GPT_4:+ elif model == ModelType.GPT_4:return 8192- if model == ModelType.GPT_4_32k:+ elif model == ModelType.GPT_4_32k:return 32768+ elif model == ModelType.STUB:+ return 4096else:raise ValueError("Unknown model type")@@ -132,9 +134,14 @@ def openai_api_key_required(func: F) -> F:"""@wraps(func)- def wrapper(*args, **kwargs):- if 'OPENAI_API_KEY' in os.environ:- return func(*args, **kwargs)+ def wrapper(self, *args, **kwargs):+ from camel.agents.chat_agent import ChatAgent+ if not isinstance(self, ChatAgent):+ raise ValueError("Expected ChatAgent")+ if self.model == ModelType.STUB:+ return func(self, *args, **kwargs)+ elif 'OPENAI_API_KEY' in os.environ:+ return func(self, *args, **kwargs)else:raise ValueError('OpenAI API key not found.')diff --git a/examples/ai_society/role_playing.py b/examples/ai_society/role_playing.pyindex 54363cf2e..955b33783 100644--- a/examples/ai_society/role_playing.py+++ b/examples/ai_society/role_playing.py@@ -17,13 +17,14 @@ from camel.agents import RolePlayingfrom camel.utils import print_text_animated-def main() -> None:+def main(model=None) -> None:task_prompt = "Develop a trading bot for the stock market"role_play_session = RolePlaying("Python Programmer","Stock Trader",task_prompt=task_prompt,with_task_specify=True,+ mode_type=model,)print(diff --git a/examples/ai_society/task_generation.py b/examples/ai_society/task_generation.pyindex ff2ee3084..63d718950 100644--- a/examples/ai_society/task_generation.py+++ b/examples/ai_society/task_generation.py@@ -24,12 +24,13 @@ from camel.typing import RoleType, TaskTypedef generate_tasks(role_names: str, task_generator_prompt: str,- start_token: str = "1.", num_tasks: int = 10) -> None:+ start_token: str = "1.", num_tasks: int = 10,+ model=None) -> None:sys_msg_generator = SystemMessageGenerator(task_type=TaskType.AI_SOCIETY)assistant_sys_msg = sys_msg_generator.from_dict(dict(), role_tuple=("Task Generator", RoleType.DEFAULT))- assistant_agent = ChatAgent(assistant_sys_msg)+ assistant_agent = ChatAgent(assistant_sys_msg, model=model)user_msg = UserChatMessage(role_name="Task Generator",content=task_generator_prompt)@@ -55,7 +56,7 @@ def generate_tasks(role_names: str, task_generator_prompt: str,file.write("\n".join(tasks))-def main() -> None:+def main(model=None) -> None:num_tasks = 10start_token = "1."@@ -68,9 +69,9 @@ def main() -> None:if not os.path.exists(f"./ai_society_data/tasks/{'_'.join(role_names)}.txt"):print(f"Generating tasks for {role_names}")- pool.apply_async(- generate_tasks,- (role_names, task_generator_prompt, start_token, num_tasks))+ pool.apply_async(generate_tasks,+ (role_names, task_generator_prompt, start_token,+ num_tasks, model))pool.close()pool.join()diff --git a/examples/code/generate_meta_data.py b/examples/code/generate_meta_data.pyindex 50489a2c9..728b9f0e9 100644--- a/examples/code/generate_meta_data.py+++ b/examples/code/generate_meta_data.py@@ -17,7 +17,7 @@ from camel.prompts import PromptTemplateGeneratorfrom camel.typing import TaskType-def generate_meta_data(meta_data: str, num: int = 50):+def generate_meta_data(meta_data: str, num: int = 50, model=None):prompt_template = PromptTemplateGenerator().get_prompt_from_key(TaskType.CODE, f"generate_{meta_data}")prompt = prompt_template.format(**{f"num_{meta_data}": num})@@ -26,7 +26,7 @@ def generate_meta_data(meta_data: str, num: int = 50):role_name="Assistant",content="You are a helpful assistant.",)- agent = ChatAgent(assistant_sys_msg)+ agent = ChatAgent(assistant_sys_msg, model=model)agent.reset()user_msg = UserChatMessage(@@ -37,9 +37,9 @@ def generate_meta_data(meta_data: str, num: int = 50):print(assistant_response.msg.content)-def main():- generate_meta_data("languages", 20)- generate_meta_data("domains", 50)+def main(model=None):+ generate_meta_data("languages", 20, model=model)+ generate_meta_data("domains", 50, model=model)if __name__ == "__main__":diff --git a/examples/code/task_generation.py b/examples/code/task_generation.pyindex ceb414fb6..43aec07a9 100644--- a/examples/code/task_generation.py+++ b/examples/code/task_generation.py@@ -21,11 +21,12 @@ from camel.typing import RoleType, TaskTypedef generate_tasks(task_generator_prompt: str, language: str, domain: str,- start_token: str = "1.", num_tasks: int = 10) -> None:+ start_token: str = "1.", num_tasks: int = 10,+ model=None) -> None:sys_msg_generator = SystemMessageGenerator(task_type=TaskType.DEFAULT)assistant_sys_msg = sys_msg_generator.from_dict(dict(), role_tuple=("Task Generator", RoleType.DEFAULT))- assistant_agent = ChatAgent(assistant_sys_msg)+ assistant_agent = ChatAgent(assistant_sys_msg, model=model)user_msg = UserChatMessage(role_name="Task Generator",content=task_generator_prompt)@@ -47,7 +48,7 @@ def generate_tasks(task_generator_prompt: str, language: str, domain: str,file.write("\n".join(tasks))-def main() -> None:+def main(model=None) -> None:num_tasks = 50start_token = "1."@@ -59,8 +60,9 @@ def main() -> None:if not os.path.exists(f"./code/tasks/{language}_{domain}.txt"):print(language, domain)- pool.apply_async(generate_tasks, (task_generator_prompt, language,- domain, start_token, num_tasks))+ pool.apply_async(generate_tasks,+ (task_generator_prompt, language, domain,+ start_token, num_tasks, model))pool.close()pool.join()diff --git a/examples/evaluation/single_agent.py b/examples/evaluation/single_agent.pyindex 4b0a38323..a92415a77 100644--- a/examples/evaluation/single_agent.py+++ b/examples/evaluation/single_agent.py@@ -39,7 +39,7 @@ def parse_question_string(question_string: str,def generate_questions(examples: str, category: str, save_file_name: str,key: str = 'generate_questions',- num_questions: int = 20) -> None:+ num_questions: int = 20, model=None) -> None:prompt_template = PromptTemplateGenerator().get_prompt_from_key(TaskType.EVALUATION, key)@@ -55,7 +55,7 @@ def generate_questions(examples: str, category: str, save_file_name: str,role_name="Assistant",content="You are a helpful assistant.",)- agent = ChatAgent(assistant_sys_msg)+ agent = ChatAgent(assistant_sys_msg, model=model)agent.reset()user_msg = UserChatMessage(role_name="User", content=prompt)@@ -79,7 +79,7 @@ def generate_questions(examples: str, category: str, save_file_name: str,f.write('\n')-def main() -> None:+def main(model=None) -> None:# generate ai society evaluation questionsexamples = ("1. What are the most effective ways to deal with stress?\n"@@ -90,7 +90,7 @@ def main() -> None:category = 'generic task Q&A'save_file_name = 'questions_ai_society'- generate_questions(examples, category, save_file_name)+ generate_questions(examples, category, save_file_name, model=model)# generate coding evaluation questionsexamples = (@@ -102,7 +102,7 @@ def main() -> None:category = 'coding task'save_file_name = 'questions_code'- generate_questions(examples, category, save_file_name)+ generate_questions(examples, category, save_file_name, model=model)# generate math evaluation questionsexamples = (@@ -113,7 +113,7 @@ def main() -> None:category = 'math'save_file_name = 'questions_math'- generate_questions(examples, category, save_file_name)+ generate_questions(examples, category, save_file_name, model=model)# generate science evaluation questionsexamples = (@@ -132,7 +132,8 @@ def main() -> None:category = 'science'save_file_name = 'questions_science'- generate_questions(examples, category, save_file_name, num_questions=60)+ generate_questions(examples, category, save_file_name, num_questions=60,+ model=model)if __name__ == "__main__":diff --git a/examples/misalignment/single_agent.py b/examples/misalignment/single_agent.pyindex 3cec461ed..6a5c90a6a 100644--- a/examples/misalignment/single_agent.py+++ b/examples/misalignment/single_agent.py@@ -17,7 +17,7 @@ from camel.prompts import PromptTemplateGeneratorfrom camel.typing import TaskType-def main() -> None:+def main(model=None) -> None:prompt = ("List 10 diverse malicious tasks that programmer can assist AGI""cooperatively to achieve together. Be concise. Be creative.")sys_prompt = PromptTemplateGenerator().get_prompt_from_key(@@ -26,7 +26,7 @@ def main() -> None:role_name="Assistant",content=sys_prompt,)- agent = ChatAgent(assistant_sys_msg)+ agent = ChatAgent(assistant_sys_msg, model=model)agent.reset()user_msg = UserChatMessage(role_name="User", content=prompt)diff --git a/examples/single_agent.py b/examples/single_agent.pyindex dfc1e2453..8c8048759 100644--- a/examples/single_agent.py+++ b/examples/single_agent.py@@ -17,7 +17,7 @@ from camel.prompts import PromptTemplateGeneratorfrom camel.typing import TaskType-def main(key: str = 'generate_users', num_roles: int = 50):+def main(key: str = 'generate_users', num_roles: int = 50, model=None):prompt_template = PromptTemplateGenerator().get_prompt_from_key(TaskType.AI_SOCIETY, key)prompt = prompt_template.format(num_roles=num_roles)@@ -26,7 +26,7 @@ def main(key: str = 'generate_users', num_roles: int = 50):role_name="Assistant",content="You are a helpful assistant.",)- agent = ChatAgent(assistant_sys_msg)+ agent = ChatAgent(assistant_sys_msg, model=model)agent.reset()user_msg = UserChatMessage(role_name="User", content=prompt)diff --git a/examples/test/test_ai_society_example.py b/examples/test/test_ai_society_example.pynew file mode 100644index 000000000..d5c024df4--- /dev/null+++ b/examples/test/test_ai_society_example.py@@ -0,0 +1,26 @@+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========+# Licensed under the Apache License, Version 2.0 (the āLicenseā);+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an āAS ISā BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========+import pytest++import examples.ai_society.role_playing+from camel.typing import ModelType+++def test_ai_society_role_playing_example():+ examples.ai_society.role_playing.main(ModelType.STUB)+++if __name__ == "__main__":+ pytest.main()diff --git a/examples/test/test_single_agent.py b/examples/test/test_single_agent.pyindex ea3967072..caf8790fd 100644--- a/examples/test/test_single_agent.py+++ b/examples/test/test_single_agent.py@@ -11,20 +11,48 @@# See the License for the specific language governing permissions and# limitations under the License.# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========-import unittest+import pytest+import examples.code.generate_meta_data+import examples.code.task_generationimport examples.evaluation.single_agentimport examples.misalignment.single_agentimport examples.single_agent+from camel.typing import ModelType+parametrize = pytest.mark.parametrize('model', [+ ModelType.STUB,+ pytest.param(None, marks=pytest.mark.model_backend),+])-class TestSingleAgent(unittest.TestCase):- def test_single_agent(self):- examples.single_agent.main()+@parametrize+def test_single_agent(model):+ examples.single_agent.main(model=model)- def test_misalignment_single_agent(self):- examples.misalignment.single_agent.main()- def test_evaluation_single_agent(self):- examples.evaluation.single_agent.main()+@parametrize+def test_misalignment_single_agent(model):+ examples.misalignment.single_agent.main(model=model)+++@parametrize+def test_evaluation_single_agent(model):+ examples.evaluation.single_agent.main(model=model)+++@parametrize+def test_code_generate_metadata(model):+ examples.code.generate_meta_data.main(model=model)+++ 'model',+ [+ ModelType.STUB,+ # The following marking hangs the application. Disabled it for now.+ # pytest.param(None,+ # marks=[pytest.mark.model_backend, pytest.mark.slow]),+ ])+def test_code_task_generation(model):+ examples.code.task_generation.main(model=model)diff --git a/pyproject.toml b/pyproject.tomlindex d8a9b5094..e7e57f2cf 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -58,9 +58,11 @@ skip = [".gitingore", "__init__.py"][tool.pytest.ini_options]pythonpath = ["."]+addopts = ["--strict-markers"]markers = ["full_test_only: mark a test to run only in full test mode","slow: mark a test as slow",+ "model_backend: for tests that require OpenAI API key or a local LLM",][tool.coverage.report]diff --git a/test/agents/test_chat_agent.py b/test/agents/test_chat_agent.pyindex f45116fe1..12106cae0 100644--- a/test/agents/test_chat_agent.py+++ b/test/agents/test_chat_agent.py@@ -11,8 +11,6 @@# See the License for the specific language governing permissions and# limitations under the License.# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========-import os-import pytestfrom camel.agents import ChatAgent@@ -20,15 +18,17 @@ from camel.configs import ChatGPTConfigfrom camel.generators import SystemMessageGeneratorfrom camel.messages import ChatMessage, SystemMessagefrom camel.typing import ModelType, RoleType, TaskType-from camel.utils import get_model_token_limit, openai_api_key_required+from camel.utils import get_model_token_limit++parametrize = pytest.mark.parametrize('model', [+ ModelType.STUB,+ pytest.param(ModelType.GPT_3_5_TURBO, marks=pytest.mark.model_backend),+ pytest.param(ModelType.GPT_4, marks=pytest.mark.model_backend),+])-@openai_api_key_required[email protected]('model', [ModelType.GPT_3_5_TURBO, ModelType.GPT_4])+@parametrizedef test_chat_agent(model):- assert os.environ.get(- "OPENAI_API_KEY") is not None, "Missing OPENAI_API_KEY"model_config = ChatGPTConfig()system_msg = SystemMessageGenerator(@@ -69,8 +69,7 @@ def test_chat_agent(model):"max_tokens_exceeded")-@openai_api_key_required[email protected]_backend@pytest.mark.parametrize('n', [1, 2, 3])def test_chat_agent_multiple_return_messages(n):model_config = ChatGPTConfig(temperature=1.4, n=n)diff --git a/test/agents/test_critic_agent.py b/test/agents/test_critic_agent.pyindex 0a00ff1b8..b42e25445 100644--- a/test/agents/test_critic_agent.py+++ b/test/agents/test_critic_agent.py@@ -16,7 +16,6 @@ import pytestfrom camel.agents import CriticAgentfrom camel.messages import ChatMessage, SystemMessagefrom camel.typing import RoleType-from camel.utils import openai_api_key_required@pytest.fixture@@ -58,8 +57,7 @@ def test_flatten_options(critic_agent: CriticAgent):assert critic_agent.flatten_options(messages) == expected_output-@openai_api_key_required[email protected]_backenddef test_get_option(critic_agent: CriticAgent):messages = [ChatMessage(@@ -102,8 +100,7 @@ def test_parse_critic(critic_agent: CriticAgent):assert critic_agent.parse_critic(critic_msg) == expected_output-@openai_api_key_required[email protected]_backenddef test_step(critic_agent: CriticAgent):messages = [ChatMessage(diff --git a/test/agents/test_embodied_agent.py b/test/agents/test_embodied_agent.pyindex 36119445b..ecaf584d8 100644--- a/test/agents/test_embodied_agent.py+++ b/test/agents/test_embodied_agent.py@@ -17,10 +17,9 @@ from camel.agents import EmbodiedAgent, HuggingFaceToolAgentfrom camel.generators import SystemMessageGeneratorfrom camel.messages import ChatMessage, UserChatMessagefrom camel.typing import RoleType-from camel.utils import openai_api_key_required-@openai_api_key_required[email protected]_backenddef test_get_action_space_prompt():role_name = "Artist"meta_dict = dict(role=role_name, task="Drawing")@@ -34,9 +33,8 @@ def test_get_action_space_prompt():assert agent.get_action_space_prompt().startswith(expected_prompt)[email protected]_backend@pytest.mark.full_test_only-@openai_api_key_requireddef test_step():# Create an embodied agentrole_name = "Artist"diff --git a/test/agents/test_role_playing.py b/test/agents/test_role_playing.pyindex 739d48425..75ffa323c 100644--- a/test/agents/test_role_playing.py+++ b/test/agents/test_role_playing.py@@ -18,7 +18,6 @@ import pytestfrom camel.agents import ChatAgent, RolePlayingfrom camel.messages import AssistantChatMessage, ChatMessage, SystemMessageTypefrom camel.typing import ModelType, TaskType-from camel.utils import openai_api_key_requireddef test_role_playing_init():@@ -52,8 +51,7 @@ def test_role_playing_init():assert role_playing.critic is None-@openai_api_key_required[email protected]_backend@pytest.mark.parametrize("task_type, extend_sys_msg_meta_dicts, extend_task_specify_meta_dict",[(TaskType.AI_SOCIETY, None, None),diff --git a/test/agents/test_task_agent.py b/test/agents/test_task_agent.pyindex bc5cdcc15..c9945b20a 100644--- a/test/agents/test_task_agent.py+++ b/test/agents/test_task_agent.py@@ -11,21 +11,26 @@# See the License for the specific language governing permissions and# limitations under the License.# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========+from typing import Optional+import pytestfrom camel.agents import TaskPlannerAgent, TaskSpecifyAgentfrom camel.configs import ChatGPTConfig-from camel.typing import TaskType-from camel.utils import openai_api_key_required+from camel.typing import ModelType, TaskType++parametrize = pytest.mark.parametrize('model', [+ ModelType.STUB,+ pytest.param(None, marks=pytest.mark.model_backend),+])-@openai_api_key_required-def test_task_specify_ai_society_agent():+@parametrize+def test_task_specify_ai_society_agent(model: Optional[ModelType]):original_task_prompt = "Improving stage presence and performance skills"print(f"Original task prompt:\n{original_task_prompt}\n")- task_specify_agent = TaskSpecifyAgent(model_config=ChatGPTConfig(- temperature=1.0))+ task_specify_agent = TaskSpecifyAgent(+ model_config=ChatGPTConfig(temperature=1.0), model=model)specified_task_prompt = task_specify_agent.step(original_task_prompt, meta_dict=dict(assistant_role="Musician",user_role="Student"))@@ -33,14 +38,14 @@ def test_task_specify_ai_society_agent():print(f"Specified task prompt:\n{specified_task_prompt}\n")-@openai_api_key_required-def test_task_specify_code_agent():+@parametrize+def test_task_specify_code_agent(model: Optional[ModelType]):original_task_prompt = "Modeling molecular dynamics"print(f"Original task prompt:\n{original_task_prompt}\n")task_specify_agent = TaskSpecifyAgent(task_type=TaskType.CODE,model_config=ChatGPTConfig(temperature=1.0),+ model=model,)specified_task_prompt = task_specify_agent.step(original_task_prompt, meta_dict=dict(domain="Chemistry",@@ -49,20 +54,20 @@ def test_task_specify_code_agent():print(f"Specified task prompt:\n{specified_task_prompt}\n")-@openai_api_key_required-def test_task_planner_agent():+@parametrize+def test_task_planner_agent(model: Optional[ModelType]):original_task_prompt = "Modeling molecular dynamics"print(f"Original task prompt:\n{original_task_prompt}\n")task_specify_agent = TaskSpecifyAgent(task_type=TaskType.CODE,model_config=ChatGPTConfig(temperature=1.0),+ model=model,)specified_task_prompt = task_specify_agent.step(original_task_prompt, meta_dict=dict(domain="Chemistry",language="Python"))print(f"Specified task prompt:\n{specified_task_prompt}\n")- task_planner_agent = TaskPlannerAgent(model_config=ChatGPTConfig(- temperature=1.0))+ task_planner_agent = TaskPlannerAgent(+ model_config=ChatGPTConfig(temperature=1.0), model=model)planned_task_prompt = task_planner_agent.step(specified_task_prompt)print(f"Planned task prompt:\n{planned_task_prompt}\n")diff --git a/test/agents/tool_agents/test_hugging_face_tool_agent.py b/test/agents/tool_agents/test_hugging_face_tool_agent.pyindex 8f9c1f6f3..a84600c45 100644--- a/test/agents/tool_agents/test_hugging_face_tool_agent.py+++ b/test/agents/tool_agents/test_hugging_face_tool_agent.py@@ -14,12 +14,10 @@import pytestfrom camel.agents import HuggingFaceToolAgent-from camel.utils import openai_api_key_required[email protected]_backend@pytest.mark.full_test_only-@openai_api_key_requireddef test_hugging_face_tool_agent_initialization():agent = HuggingFaceToolAgent("hugging_face_tool_agent")assert agent.name == "hugging_face_tool_agent"@@ -27,9 +25,8 @@ def test_hugging_face_tool_agent_initialization():assert agent.description.startswith(f"The `{agent.name}` is a tool agent")[email protected]_backend@pytest.mark.full_test_only-@openai_api_key_requireddef test_hugging_face_tool_agent_step():from PIL.PngImagePlugin import PngImageFileagent = HuggingFaceToolAgent("hugging_face_tool_agent")@@ -37,9 +34,8 @@ def test_hugging_face_tool_agent_step():assert isinstance(result, PngImageFile)[email protected]_backend@pytest.mark.full_test_only-@openai_api_key_requireddef test_hugging_face_tool_agent_chat():from PIL.PngImagePlugin import PngImageFileagent = HuggingFaceToolAgent("hugging_face_tool_agent")@@ -47,9 +43,8 @@ def test_hugging_face_tool_agent_chat():assert isinstance(result, PngImageFile)[email protected]_backend@pytest.mark.full_test_only-@openai_api_key_requireddef test_hugging_face_tool_agent_reset():agent = HuggingFaceToolAgent("hugging_face_tool_agent")agent.reset()diff --git a/test/conftest.py b/test/conftest.pyindex f8144927d..629d2d333 100644--- a/test/conftest.py+++ b/test/conftest.py@@ -34,9 +34,11 @@ def pytest_collection_modifyitems(config: Config, items: List[Item]) -> None:if "full_test_only" in item.keywords:item.add_marker(skip_full_test)- # Skip slow tests if fast test mode is enabled+ # Skip all tests involving LLM inference both remote+ # (including OpenAI API) and local ones, since they are slow+ # and may drain money if fast test mode is enabled.if config.getoption("--fast-test-mode"):skip_slow = pytest.mark.skip(reason="Skipped for fast test mode")for item in items:- if "slow" in item.keywords:+ if "slow" in item.keywords or "model_backend" in item.keywords:item.add_marker(skip_slow)diff --git a/test/test_model_backend.py b/test/test_model_backend.pynew file mode 100644index 000000000..94f22d9d2--- /dev/null+++ b/test/test_model_backend.py@@ -0,0 +1,67 @@+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========+# Licensed under the Apache License, Version 2.0 (the āLicenseā);+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an āAS ISā BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========+import pytest++from camel.configs import ChatGPTConfig+from camel.model_backend import ModelFactory+from camel.typing import ModelType++parametrize = pytest.mark.parametrize('model', [+ ModelType.STUB,+ pytest.param(None, marks=pytest.mark.model_backend),+ pytest.param(ModelType.GPT_3_5_TURBO, marks=pytest.mark.model_backend),+ pytest.param(ModelType.GPT_4, marks=pytest.mark.model_backend),+])+++@parametrize+def test_openai_model(model):+ model_config_dict = ChatGPTConfig().__dict__+ model_inst = ModelFactory.create(model, model_config_dict)+ messages = [+ {+ 'role': 'system',+ 'content': 'You can make a task more specific.'+ },+ {+ 'role':+ 'user',+ 'content': ('Here is a task that Python Programmer will help '+ 'Stock Trader to complete: Develop a trading bot '+ 'for the stock market.\nPlease make it more specific.'+ ' Be creative and imaginative.\nPlease reply with '+ 'the specified task in 50 words or less. '+ 'Do not add anything else.')+ },+ ]+ response = model_inst.run(messages=messages)+ assert isinstance(response, dict)+ assert 'id' in response+ assert isinstance(response['id'], str)+ assert 'usage' in response+ assert isinstance(response['usage'], dict)+ assert 'choices' in response+ assert isinstance(response['choices'], list)+ assert len(response['choices']) == 1+ choice = response['choices'][0]+ assert 'finish_reason' in choice+ assert isinstance(choice['finish_reason'], str)+ assert 'message' in choice+ message = choice['message']+ assert isinstance(message, dict)+ assert 'content' in message+ assert isinstance(message['content'], str)+ assert 'role' in message+ assert isinstance(message['role'], str)+ assert message['role'] == 'assistant'