Skip to content

armgit / camel

Public Imported from GitHub
0

🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org

Add a stub model backend (#161)

Dmitrii Khizbullindbca94b126f2
diff --git a/.github/workflows/pytest_apps.yml b/.github/workflows/pytest_apps.yml
index 8839ace1a..1187de325 100644
--- a/.github/workflows/pytest_apps.yml
+++ b/.github/workflows/pytest_apps.yml
@@ -1,7 +1,7 @@
# This workflow will install Python dependencies, run tests and lint with a single version of Python
# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
-name: Pytest Gradio Apps
+name: Pytest Gradio Apps and Examples
on:
push:
@@ -40,4 +40,4 @@ jobs:
env:
OPENAI_API_KEY: "${{ secrets.OPENAI_API_KEY }}"
run: |
- pytest apps/
+ pytest -m "not slow" apps/ examples/
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 20067ae3a..97d877cb5 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -112,12 +112,17 @@ The coverage report will be generated at `htmlcov/index.html`.
Unit tests cover modular logic that doesn't require calls to outside APIs. Currently, the test setup requires an OpenAI API key to test the framework, making them resemble integration tests.
-To run the tests, use the following command:
+To run all tests including those that use OpenAI API, use the following command:
```bash
pytest .
```
+To quickly run only local isolated unit and integration tests:
+```bash
+pytest -m "not model_backend and not slow" .
+```
+
If you're developing with VSCode, make sure to create a `.env` file in the repository root and include your OpenAI API key:
```
diff --git a/apps/agents/agents.py b/apps/agents/agents.py
index f96579502..253572436 100644
--- a/apps/agents/agents.py
+++ b/apps/agents/agents.py
@@ -230,7 +230,7 @@ def role_playing_chat_init(state) -> \
return state, state.chat, progress_update
-# WORKAROUND: do not add type hinst for session and chatbot_histoty
+# WORKAROUND: do not add type hints for session and chatbot_history
def role_playing_chat_cont(state) -> \
Tuple[State, ChatBotHistory, Dict, Dict]:
""" Produce a pair of messages by an assistant and a user.
diff --git a/apps/agents/test/test_agents.py b/apps/agents/test/test_agents.py
index 8f2b95b50..13fa14c57 100644
--- a/apps/agents/test/test_agents.py
+++ b/apps/agents/test/test_agents.py
@@ -11,9 +11,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
-import unittest
-
import gradio as gr
+import pytest
from apps.agents.agents import (
State,
@@ -27,46 +26,43 @@ from apps.agents.agents import (
)
-class TestAgents(unittest.TestCase):
-
- def test_construct_blocks(self):
- blocks = construct_blocks(None)
- self.assertIsInstance(blocks, gr.Blocks)
+def test_construct_blocks():
+ blocks = construct_blocks(None)
+ assert isinstance(blocks, gr.Blocks)
- def test_utils(self):
- args = parse_arguments()
- self.assertIsNotNone(args)
- def test_session(self):
- state = State.empty()
+def test_utils():
+ args = parse_arguments()
+ assert args is not None
- state, _, _ = cleanup_on_launch(state)
- assistant = "professor"
- user = "PhD student"
- original_task = "Recommend AI conferences to publish a paper"
- max_messages = 10
- with_task_specifier = False
- word_limit = 50
- state, specified_task_prompt, planned_task_upd, chat, progress_upd = \
- role_playing_start(state, assistant, user, original_task,
- max_messages, with_task_specifier, word_limit)
+def test_session():
+ state = State.empty()
- self.assertIsNotNone(state.session)
+ state, _, _ = cleanup_on_launch(state)
- state, chat, progress_update = \
- role_playing_chat_init(state)
+ assistant = "professor"
+ user = "PhD student"
+ original_task = "Recommend AI conferences to publish a paper"
+ max_messages = 10
+ with_task_specifier = False
+ word_limit = 50
+ state, specified_task_prompt, planned_task_upd, chat, progress_upd = \
+ role_playing_start(state, assistant, user, original_task,
+ max_messages, with_task_specifier, word_limit)
- self.assertIsNotNone(state.session)
+ assert state.session is not None
- for _ in range(5):
- state, chat, progress_update, start_bn_update =\
- role_playing_chat_cont(state)
+ state, chat, progress_update = \
+ role_playing_chat_init(state)
- state, _, _ = stop_session(state)
+ assert state.session is not None
- self.assertIsNone(state.session)
+ for _ in range(5):
+ state, chat, progress_update, start_bn_update =\
+ role_playing_chat_cont(state)
+ state, _, _ = stop_session(state)
-if __name__ == '__main__':
- unittest.main()
+ assert state.session is None
diff --git a/apps/agents/test/test_text_utils.py b/apps/agents/test/test_text_utils.py
index eeba21764..d908224f3 100644
--- a/apps/agents/test/test_text_utils.py
+++ b/apps/agents/test/test_text_utils.py
@@ -11,110 +11,107 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
-from unittest import TestCase
-
import apps.agents.text_utils as text_utils
-class TestTextUtils(TestCase):
+def test_split_markdown_code_newline():
+ inp = ("Solution: To preprocess the historical stock data, we "
+ "can perform the following steps:\n\n1. Remove any unnecessary"
+ " columns that do not contribute to the prediction, such as"
+ " the stock symbol or date.\n2. Check for and handle any "
+ "missing or null values in the data.\n3. Normalize the data"
+ " to ensure that all features are on the same scale. This "
+ "can be done using techniques such as Min-Max scaling or "
+ "Z-score normalization.\n4. Split the data into training "
+ "and testing sets. The training set will be used to train "
+ "the machine learning model, while the testing set will be "
+ "used to evaluate its performance.\n\nHere is an example "
+ "code snippet to preprocess the data using Pandas:\n\n```\n"
+ "import pandas as pd\nfrom sklearn.preprocessing import "
+ "MinMaxScaler\nfrom sklearn.model_selection import "
+ "train_test_split\n\n# Read in the historical stock data\ndata"
+ " = pd.read_csv('historical_stock_data.csv')\n\n# Remove "
+ "unnecessary columns\ndata = data.drop(['symbol', 'date'], "
+ "axis=1)\n\n# Handle missing values\ndata = data.fillna("
+ "method='ffill')\n\n# Normalize the data\nscaler = "
+ "MinMaxScaler()\ndata = scaler.fit_transform(data)\n\n# "
+ "Split the data into training and testing sets\nX_train, "
+ "X_test, y_train, y_test = train_test_split(data[:, :-1], "
+ "data[:, -1], test_size=0.2, random_state=42)\n```\n\nNext "
+ "request.")
+ gt = ("Solution: To preprocess the historical stock data, we "
+ "can perform the following steps:\n\n1. Remove any unnecessary"
+ " columns that do not contribute to the prediction, such as"
+ " the stock symbol or date.\n2. Check for and handle any missing"
+ " or null values in the data.\n3. Normalize the data to ensure"
+ " that all features are on the same scale. This can be done"
+ " using techniques such as Min-Max scaling or Z-score"
+ " normalization.\n4. Split the data into training and testing"
+ " sets. The training set will be used to train the machine"
+ " learning model, while the testing set will be used to"
+ " evaluate its performance.\n\nHere is an example code snippet"
+ " to preprocess the data using Pandas:\n\n\n```import pandas"
+ " as pd```\n```from sklearn.preprocessing import MinMaxScaler"
+ "```\n```from sklearn.model_selection import train_test_split"
+ "```\n\n```# Read in the historical stock data```\n```data ="
+ " pd.read_csv('historical_stock_data.csv')```\n\n```# Remove"
+ " unnecessary columns```\n```data = data.drop(['symbol', "
+ "'date'], axis=1)```\n\n```# Handle missing values```\n```data"
+ " = data.fillna(method='ffill')```\n\n```# Normalize the data"
+ "```\n```scaler = MinMaxScaler()```\n```data = scaler."
+ "fit_transform(data)```\n\n```# Split the data into training"
+ " and testing sets```\n```X_train, X_test, y_train, y_test"
+ " = train_test_split(data[:, :-1], data[:, -1], test_size=0.2,"
+ " random_state=42)```\n\n\nNext request.")
- def test_split_markdown_code_newline(self):
- inp = ("Solution: To preprocess the historical stock data, we "
- "can perform the following steps:\n\n1. Remove any unnecessary"
- " columns that do not contribute to the prediction, such as"
- " the stock symbol or date.\n2. Check for and handle any "
- "missing or null values in the data.\n3. Normalize the data"
- " to ensure that all features are on the same scale. This "
- "can be done using techniques such as Min-Max scaling or "
- "Z-score normalization.\n4. Split the data into training "
- "and testing sets. The training set will be used to train "
- "the machine learning model, while the testing set will be "
- "used to evaluate its performance.\n\nHere is an example "
- "code snippet to preprocess the data using Pandas:\n\n```\n"
- "import pandas as pd\nfrom sklearn.preprocessing import "
- "MinMaxScaler\nfrom sklearn.model_selection import "
- "train_test_split\n\n# Read in the historical stock data\ndata"
- " = pd.read_csv('historical_stock_data.csv')\n\n# Remove "
- "unnecessary columns\ndata = data.drop(['symbol', 'date'], "
- "axis=1)\n\n# Handle missing values\ndata = data.fillna("
- "method='ffill')\n\n# Normalize the data\nscaler = "
- "MinMaxScaler()\ndata = scaler.fit_transform(data)\n\n# "
- "Split the data into training and testing sets\nX_train, "
- "X_test, y_train, y_test = train_test_split(data[:, :-1], "
- "data[:, -1], test_size=0.2, random_state=42)\n```\n\nNext "
- "request.")
- gt = ("Solution: To preprocess the historical stock data, we "
- "can perform the following steps:\n\n1. Remove any unnecessary"
- " columns that do not contribute to the prediction, such as"
- " the stock symbol or date.\n2. Check for and handle any missing"
- " or null values in the data.\n3. Normalize the data to ensure"
- " that all features are on the same scale. This can be done"
- " using techniques such as Min-Max scaling or Z-score"
- " normalization.\n4. Split the data into training and testing"
- " sets. The training set will be used to train the machine"
- " learning model, while the testing set will be used to"
- " evaluate its performance.\n\nHere is an example code snippet"
- " to preprocess the data using Pandas:\n\n\n```import pandas"
- " as pd```\n```from sklearn.preprocessing import MinMaxScaler"
- "```\n```from sklearn.model_selection import train_test_split"
- "```\n\n```# Read in the historical stock data```\n```data ="
- " pd.read_csv('historical_stock_data.csv')```\n\n```# Remove"
- " unnecessary columns```\n```data = data.drop(['symbol', "
- "'date'], axis=1)```\n\n```# Handle missing values```\n```data"
- " = data.fillna(method='ffill')```\n\n```# Normalize the data"
- "```\n```scaler = MinMaxScaler()```\n```data = scaler."
- "fit_transform(data)```\n\n```# Split the data into training"
- " and testing sets```\n```X_train, X_test, y_train, y_test"
- " = train_test_split(data[:, :-1], data[:, -1], test_size=0.2,"
- " random_state=42)```\n\n\nNext request.")
+ out = text_utils.split_markdown_code(inp)
+ assert out == gt
- out = text_utils.split_markdown_code(inp)
- self.assertEqual(out, gt)
- def test_split_markdown_code_br(self):
- inp = ("Solution: Define the Bayesian optimization object."
- "\n"
- "We can define the Bayesian optimization object using"
- " the BayesianOptimization class from the bayes_opt module."
- " Here is an example of how to define the Bayesian"
- " optimization object:"
- "\n"
- "```<br># Replace 'objective_function' with the actual"
- " objective function<br># Replace 'bounds' with the actual"
- " search space<br># Replace 'model' with the actual machine"
- " learning model<br>bo = BayesianOptimization(<br> "
- "f=objective_function,<br> pbounds=bounds,<br> verbose=2,<br>"
- " random_state=1,<br>)<br>```"
- "\n"
- "This will define the Bayesian optimization object with the"
- " specified objective function, search space, and machine"
- " learning model. The BayesianOptimization class takes "
- "several arguments, including f for the objective function,"
- " pbounds for the search space, verbose for the verbosity"
- " level, and random_state for the random seed."
- "\n"
- "Next request.")
- gt = ("Solution: Define the Bayesian optimization object."
- "\n"
- "We can define the Bayesian optimization object using the"
- " BayesianOptimization class from the bayes_opt module. Here is"
- " an example of how to define the Bayesian optimization object:"
- "\n"
- "\n```# Replace 'objective_function' with the actual objective"
- " function```\n```# Replace 'bounds' with the actual search"
- " space```\n```# Replace 'model' with the actual machine"
- " learning model```\n```bo = BayesianOptimization(```\n```"
- " f=objective_function,```\n``` pbounds=bounds,```\n``` "
- "verbose=2,```\n``` random_state=1,```\n```)```\n"
- "\n"
- "This will define the Bayesian optimization object with "
- "the specified objective function, search space, and machine"
- " learning model. The BayesianOptimization class takes several"
- " arguments, including f for the objective function, pbounds"
- " for the search space, verbose for the verbosity level, and"
- " random_state for the random seed."
- "\n"
- "Next request.")
+def test_split_markdown_code_br():
+ inp = ("Solution: Define the Bayesian optimization object."
+ "\n"
+ "We can define the Bayesian optimization object using"
+ " the BayesianOptimization class from the bayes_opt module."
+ " Here is an example of how to define the Bayesian"
+ " optimization object:"
+ "\n"
+ "```<br># Replace 'objective_function' with the actual"
+ " objective function<br># Replace 'bounds' with the actual"
+ " search space<br># Replace 'model' with the actual machine"
+ " learning model<br>bo = BayesianOptimization(<br> "
+ "f=objective_function,<br> pbounds=bounds,<br> verbose=2,<br>"
+ " random_state=1,<br>)<br>```"
+ "\n"
+ "This will define the Bayesian optimization object with the"
+ " specified objective function, search space, and machine"
+ " learning model. The BayesianOptimization class takes "
+ "several arguments, including f for the objective function,"
+ " pbounds for the search space, verbose for the verbosity"
+ " level, and random_state for the random seed."
+ "\n"
+ "Next request.")
+ gt = ("Solution: Define the Bayesian optimization object."
+ "\n"
+ "We can define the Bayesian optimization object using the"
+ " BayesianOptimization class from the bayes_opt module. Here is"
+ " an example of how to define the Bayesian optimization object:"
+ "\n"
+ "\n```# Replace 'objective_function' with the actual objective"
+ " function```\n```# Replace 'bounds' with the actual search"
+ " space```\n```# Replace 'model' with the actual machine"
+ " learning model```\n```bo = BayesianOptimization(```\n```"
+ " f=objective_function,```\n``` pbounds=bounds,```\n``` "
+ "verbose=2,```\n``` random_state=1,```\n```)```\n"
+ "\n"
+ "This will define the Bayesian optimization object with "
+ "the specified objective function, search space, and machine"
+ " learning model. The BayesianOptimization class takes several"
+ " arguments, including f for the objective function, pbounds"
+ " for the search space, verbose for the verbosity level, and"
+ " random_state for the random seed."
+ "\n"
+ "Next request.")
- out = text_utils.split_markdown_code(inp)
- self.assertEqual(out, gt)
+ out = text_utils.split_markdown_code(inp)
+ assert out == gt
diff --git a/apps/common/test/test_auto_zip.py b/apps/common/test/test_auto_zip.py
index 9ceaff0d7..460b7af68 100644
--- a/apps/common/test/test_auto_zip.py
+++ b/apps/common/test/test_auto_zip.py
@@ -12,7 +12,6 @@
# limitations under the License.
# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
import os
-from unittest import TestCase
from apps.common.auto_zip import AutoZip
@@ -20,16 +19,14 @@ REPO_ROOT = os.path.realpath(
os.path.join(os.path.dirname(os.path.abspath(__file__)), "../../.."))
-class TestAutoZip(TestCase):
+def test_dict():
+ path = os.path.join(REPO_ROOT, "apps/common/test/test_archive_1.zip")
+ zp = AutoZip(path, ".txt")
- def test_dict(self):
- path = os.path.join(REPO_ROOT, "apps/common/test/test_archive_1.zip")
- zip = AutoZip(path, ".txt")
+ d = zp.as_dict()
+ assert isinstance(d, dict)
+ assert len(d) == 3
- d = zip.as_dict()
- self.assertIsInstance(d, dict)
- self.assertEqual(len(d), 3)
-
- d = zip.as_dict(include_zip_name=True)
- self.assertIsInstance(d, dict)
- self.assertEqual(len(d), 3)
+ d = zp.as_dict(include_zip_name=True)
+ assert isinstance(d, dict)
+ assert len(d) == 3
diff --git a/apps/data_explorer/test/test_data_explorer.py b/apps/data_explorer/test/test_data_explorer.py
index b60b5efb6..4c8fa2445 100644
--- a/apps/data_explorer/test/test_data_explorer.py
+++ b/apps/data_explorer/test/test_data_explorer.py
@@ -12,7 +12,6 @@
# limitations under the License.
# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
import os
-import unittest
import urllib.request
import gradio as gr
@@ -21,25 +20,19 @@ from apps.data_explorer.data_explorer import construct_blocks, parse_arguments
from apps.data_explorer.loader import REPO_ROOT
-class TestDataExplorer(unittest.TestCase):
+def test_app():
+ test_data_url = ("https://storage.googleapis.com/"
+ "camel-bucket/datasets/test/DATA.zip")
+ data_dir = os.path.join(REPO_ROOT, "datasets_test")
+ test_file_path = os.path.join(data_dir, os.path.split(test_data_url)[1])
+ os.makedirs(data_dir, exist_ok=True)
+ urllib.request.urlretrieve(test_data_url, test_file_path)
- def test_app(self):
- test_data_url = ("https://storage.googleapis.com/"
- "camel-bucket/datasets/test/DATA.zip")
- data_dir = os.path.join(REPO_ROOT, "datasets_test")
- test_file_path = os.path.join(data_dir,
- os.path.split(test_data_url)[1])
- os.makedirs(data_dir, exist_ok=True)
- urllib.request.urlretrieve(test_data_url, test_file_path)
+ blocks = construct_blocks(data_dir, None)
- blocks = construct_blocks(data_dir, None)
+ assert isinstance(blocks, gr.Blocks)
- self.assertIsInstance(blocks, gr.Blocks)
- def test_utils(self):
- args = parse_arguments()
- self.assertIsNotNone(args)
-
-
-if __name__ == '__main__':
- unittest.main()
+def test_utils():
+ args = parse_arguments()
+ assert args is not None
diff --git a/apps/data_explorer/test/test_loader.py b/apps/data_explorer/test/test_loader.py
index 8a9a74be8..03f626e93 100644
--- a/apps/data_explorer/test/test_loader.py
+++ b/apps/data_explorer/test/test_loader.py
@@ -11,13 +11,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
-from unittest import TestCase
-
import apps.data_explorer.loader as loader
-class TestLoader(TestCase):
-
- def test_load_datasets_smoke(self):
- data = loader.load_datasets()
- self.assertIsNotNone(data)
+def test_load_datasets_smoke():
+ data = loader.load_datasets()
+ assert data is not None
diff --git a/camel/agents/chat_agent.py b/camel/agents/chat_agent.py
index 427f13e88..1d9f90ae0 100644
--- a/camel/agents/chat_agent.py
+++ b/camel/agents/chat_agent.py
@@ -14,7 +14,6 @@
from dataclasses import dataclass
from typing import Any, Dict, List, Optional
-import openai
from tenacity import retry
from tenacity.stop import stop_after_attempt
from tenacity.wait import wait_exponential
@@ -22,7 +21,8 @@ from tenacity.wait import wait_exponential
from camel.agents import BaseAgent
from camel.configs import ChatGPTConfig
from camel.messages import ChatMessage, MessageType, SystemMessage
-from camel.typing import ModelType
+from camel.model_backend import ModelBackend, ModelFactory
+from camel.typing import ModelType, RoleType
from camel.utils import (
get_model_token_limit,
num_tokens_from_messages,
@@ -72,20 +72,23 @@ class ChatAgent(BaseAgent):
def __init__(
self,
system_message: SystemMessage,
- model: ModelType = ModelType.GPT_3_5_TURBO,
+ model: Optional[ModelType] = None,
model_config: Optional[Any] = None,
message_window_size: Optional[int] = None,
) -> None:
- self.system_message = system_message
- self.role_name = system_message.role_name
- self.role_type = system_message.role_type
- self.meta_dict = system_message.meta_dict
+ self.system_message: SystemMessage = system_message
+ self.role_name: str = system_message.role_name
+ self.role_type: RoleType = system_message.role_type
- self.model = model
- self.model_config = model_config or ChatGPTConfig()
- self.model_token_limit = get_model_token_limit(self.model)
- self.message_window_size = message_window_size
+ self.model: ModelType = (model if model is not None else
+ ModelType.GPT_3_5_TURBO)
+ self.model_config: ChatGPTConfig = model_config or ChatGPTConfig()
+ self.model_token_limit: int = get_model_token_limit(self.model)
+ self.message_window_size: Optional[int] = message_window_size
+
+ self.model_backend: ModelBackend = ModelFactory.create(
+ self.model, self.model_config.__dict__)
self.terminated: bool = False
self.init_messages()
@@ -177,11 +180,7 @@ class ChatAgent(BaseAgent):
info: Dict[str, Any]
if num_tokens < self.model_token_limit:
- response = openai.ChatCompletion.create(
- model=self.model.value,
- messages=openai_messages,
- **self.model_config.__dict__,
- )
+ response = self.model_backend.run(messages=openai_messages)
if not isinstance(response, dict):
raise RuntimeError("OpenAI returned unexpected struct")
output_messages = [
@@ -198,7 +197,6 @@ class ChatAgent(BaseAgent):
],
num_tokens,
)
-
else:
self.terminated = True
output_messages = []
diff --git a/camel/agents/role_playing.py b/camel/agents/role_playing.py
index be96767e4..c144dc872 100644
--- a/camel/agents/role_playing.py
+++ b/camel/agents/role_playing.py
@@ -152,12 +152,12 @@ class RolePlaying:
],
))
- self.assistant_agent = ChatAgent(
+ self.assistant_agent: ChatAgent = ChatAgent(
self.assistant_sys_msg,
mode_type,
**(assistant_agent_kwargs or {}),
)
- self.user_agent = ChatAgent(
+ self.user_agent: ChatAgent = ChatAgent(
self.user_sys_msg,
mode_type,
**(user_agent_kwargs or {}),
diff --git a/camel/agents/task_agent.py b/camel/agents/task_agent.py
index b89598eb1..20320cfa9 100644
--- a/camel/agents/task_agent.py
+++ b/camel/agents/task_agent.py
@@ -44,12 +44,13 @@ class TaskSpecifyAgent(ChatAgent):
def __init__(
self,
- model: ModelType = ModelType.GPT_3_5_TURBO,
+ model: Optional[ModelType] = None,
task_type: TaskType = TaskType.AI_SOCIETY,
model_config: Optional[Any] = None,
task_specify_prompt: Optional[Union[str, TextPrompt]] = None,
word_limit: int = DEFAULT_WORD_LIMIT,
) -> None:
+
if task_specify_prompt is None:
task_specify_prompt_template = PromptTemplateGenerator(
).get_task_specify_prompt(task_type)
@@ -124,7 +125,7 @@ class TaskPlannerAgent(ChatAgent):
def __init__(
self,
- model: ModelType = ModelType.GPT_3_5_TURBO,
+ model: Optional[ModelType] = None,
model_config: Any = None,
) -> None:
diff --git a/camel/model_backend.py b/camel/model_backend.py
new file mode 100644
index 000000000..b8ef5745a
--- /dev/null
+++ b/camel/model_backend.py
@@ -0,0 +1,101 @@
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+# Licensed under the Apache License, Version 2.0 (the ā€œLicenseā€);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an ā€œAS ISā€ BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+from abc import ABC, abstractmethod
+from typing import Any, Dict
+
+from camel.typing import ModelType
+
+
+class ModelBackend(ABC):
+ r"""Base class for different model backends.
+ May be OpenAI API, a local LLM, a stub for unit tests, etc."""
+
+ @abstractmethod
+ def run(self, *args, **kwargs) -> Dict[str, Any]:
+ r"""Runs the query to the backend model.
+
+ Raises:
+ RuntimeError: if the return value from OpenAI API
+ is not a dict that is expected.
+
+ Returns:
+ Dict[str, Any]: All backends must return a dict in OpenAI format.
+ """
+ pass
+
+
+class OpenAIModel(ModelBackend):
+ r"""OpenAI API in a unified ModelBackend interface."""
+
+ def __init__(self, model_type: ModelType, model_config_dict: Dict) -> None:
+ super().__init__()
+ self.model_type = model_type
+ self.model_config_dict = model_config_dict
+
+ def run(self, *args, **kwargs) -> Dict[str, Any]:
+
+ import openai
+ response = openai.ChatCompletion.create(*args, **kwargs,
+ model=self.model_type.value,
+ **self.model_config_dict)
+ if not isinstance(response, Dict):
+ raise RuntimeError("Unexpected return from OpenAI API")
+ return response
+
+
+class StubModel(ModelBackend):
+ r"""A dummy model used for unit tests."""
+
+ def __init__(self, *args, **kwargs) -> None:
+ super().__init__()
+
+ def run(self, *args, **kwargs) -> Dict[str, Any]:
+ ARBITRARY_STRING = "Lorem Ipsum"
+
+ return dict(
+ id="stub_model_id",
+ usage=dict(),
+ choices=[
+ dict(finish_reason="stop",
+ message=dict(content=ARBITRARY_STRING, role="assistant"))
+ ],
+ )
+
+
+class ModelFactory:
+ r"""Factory of backend models.
+
+ Raises:
+ ValueError: in case the provided model type is unknown.
+ """
+
+ @staticmethod
+ def create(model_type: ModelType, model_config_dict: Dict) -> ModelBackend:
+ default_model_type = ModelType.GPT_3_5_TURBO
+
+ if model_type in {
+ ModelType.GPT_3_5_TURBO, ModelType.GPT_4, ModelType.GPT_4_32k,
+ None
+ }:
+ model_class = OpenAIModel
+ elif model_type == ModelType.STUB:
+ model_class = StubModel
+ else:
+ raise ValueError("Unknown model")
+
+ if model_type is None:
+ model_type = default_model_type
+
+ inst = model_class(model_type, model_config_dict)
+ return inst
diff --git a/camel/typing.py b/camel/typing.py
index 9e0f163c1..5e0a95dfa 100644
--- a/camel/typing.py
+++ b/camel/typing.py
@@ -26,6 +26,11 @@ class ModelType(Enum):
GPT_3_5_TURBO = "gpt-3.5-turbo"
GPT_4 = "gpt-4"
GPT_4_32k = "gpt-4-32k"
+ STUB = "stub"
+
+ @property
+ def value_for_tiktoken(self):
+ return self.value if self.name != "STUB" else "gpt-3.5-turbo"
class TaskType(Enum):
diff --git a/camel/utils.py b/camel/utils.py
index db4b0a2bd..8fd302cd3 100644
--- a/camel/utils.py
+++ b/camel/utils.py
@@ -76,15 +76,15 @@ def num_tokens_from_messages(
- https://platform.openai.com/docs/models/gpt-3-5
"""
try:
- encoding = tiktoken.encoding_for_model(model.value)
+ value_for_tiktoken = model.value_for_tiktoken
+ encoding = tiktoken.encoding_for_model(value_for_tiktoken)
except KeyError:
encoding = tiktoken.get_encoding("cl100k_base")
- if model == ModelType.GPT_3_5_TURBO:
- return count_tokens_openai_chat_models(messages, encoding)
- elif model == ModelType.GPT_4:
- return count_tokens_openai_chat_models(messages, encoding)
- elif model == ModelType.GPT_4_32k:
+ if model in {
+ ModelType.GPT_3_5_TURBO, ModelType.GPT_4, ModelType.GPT_4_32k,
+ ModelType.STUB
+ }:
return count_tokens_openai_chat_models(messages, encoding)
else:
raise NotImplementedError(
@@ -108,10 +108,12 @@ def get_model_token_limit(model: ModelType) -> int:
"""
if model == ModelType.GPT_3_5_TURBO:
return 4096
- if model == ModelType.GPT_4:
+ elif model == ModelType.GPT_4:
return 8192
- if model == ModelType.GPT_4_32k:
+ elif model == ModelType.GPT_4_32k:
return 32768
+ elif model == ModelType.STUB:
+ return 4096
else:
raise ValueError("Unknown model type")
@@ -132,9 +134,14 @@ def openai_api_key_required(func: F) -> F:
"""
@wraps(func)
- def wrapper(*args, **kwargs):
- if 'OPENAI_API_KEY' in os.environ:
- return func(*args, **kwargs)
+ def wrapper(self, *args, **kwargs):
+ from camel.agents.chat_agent import ChatAgent
+ if not isinstance(self, ChatAgent):
+ raise ValueError("Expected ChatAgent")
+ if self.model == ModelType.STUB:
+ return func(self, *args, **kwargs)
+ elif 'OPENAI_API_KEY' in os.environ:
+ return func(self, *args, **kwargs)
else:
raise ValueError('OpenAI API key not found.')
diff --git a/examples/ai_society/role_playing.py b/examples/ai_society/role_playing.py
index 54363cf2e..955b33783 100644
--- a/examples/ai_society/role_playing.py
+++ b/examples/ai_society/role_playing.py
@@ -17,13 +17,14 @@ from camel.agents import RolePlaying
from camel.utils import print_text_animated
-def main() -> None:
+def main(model=None) -> None:
task_prompt = "Develop a trading bot for the stock market"
role_play_session = RolePlaying(
"Python Programmer",
"Stock Trader",
task_prompt=task_prompt,
with_task_specify=True,
+ mode_type=model,
)
print(
diff --git a/examples/ai_society/task_generation.py b/examples/ai_society/task_generation.py
index ff2ee3084..63d718950 100644
--- a/examples/ai_society/task_generation.py
+++ b/examples/ai_society/task_generation.py
@@ -24,12 +24,13 @@ from camel.typing import RoleType, TaskType
def generate_tasks(role_names: str, task_generator_prompt: str,
- start_token: str = "1.", num_tasks: int = 10) -> None:
+ start_token: str = "1.", num_tasks: int = 10,
+ model=None) -> None:
sys_msg_generator = SystemMessageGenerator(task_type=TaskType.AI_SOCIETY)
assistant_sys_msg = sys_msg_generator.from_dict(
dict(), role_tuple=("Task Generator", RoleType.DEFAULT))
- assistant_agent = ChatAgent(assistant_sys_msg)
+ assistant_agent = ChatAgent(assistant_sys_msg, model=model)
user_msg = UserChatMessage(role_name="Task Generator",
content=task_generator_prompt)
@@ -55,7 +56,7 @@ def generate_tasks(role_names: str, task_generator_prompt: str,
file.write("\n".join(tasks))
-def main() -> None:
+def main(model=None) -> None:
num_tasks = 10
start_token = "1."
@@ -68,9 +69,9 @@ def main() -> None:
if not os.path.exists(
f"./ai_society_data/tasks/{'_'.join(role_names)}.txt"):
print(f"Generating tasks for {role_names}")
- pool.apply_async(
- generate_tasks,
- (role_names, task_generator_prompt, start_token, num_tasks))
+ pool.apply_async(generate_tasks,
+ (role_names, task_generator_prompt, start_token,
+ num_tasks, model))
pool.close()
pool.join()
diff --git a/examples/code/generate_meta_data.py b/examples/code/generate_meta_data.py
index 50489a2c9..728b9f0e9 100644
--- a/examples/code/generate_meta_data.py
+++ b/examples/code/generate_meta_data.py
@@ -17,7 +17,7 @@ from camel.prompts import PromptTemplateGenerator
from camel.typing import TaskType
-def generate_meta_data(meta_data: str, num: int = 50):
+def generate_meta_data(meta_data: str, num: int = 50, model=None):
prompt_template = PromptTemplateGenerator().get_prompt_from_key(
TaskType.CODE, f"generate_{meta_data}")
prompt = prompt_template.format(**{f"num_{meta_data}": num})
@@ -26,7 +26,7 @@ def generate_meta_data(meta_data: str, num: int = 50):
role_name="Assistant",
content="You are a helpful assistant.",
)
- agent = ChatAgent(assistant_sys_msg)
+ agent = ChatAgent(assistant_sys_msg, model=model)
agent.reset()
user_msg = UserChatMessage(
@@ -37,9 +37,9 @@ def generate_meta_data(meta_data: str, num: int = 50):
print(assistant_response.msg.content)
-def main():
- generate_meta_data("languages", 20)
- generate_meta_data("domains", 50)
+def main(model=None):
+ generate_meta_data("languages", 20, model=model)
+ generate_meta_data("domains", 50, model=model)
if __name__ == "__main__":
diff --git a/examples/code/task_generation.py b/examples/code/task_generation.py
index ceb414fb6..43aec07a9 100644
--- a/examples/code/task_generation.py
+++ b/examples/code/task_generation.py
@@ -21,11 +21,12 @@ from camel.typing import RoleType, TaskType
def generate_tasks(task_generator_prompt: str, language: str, domain: str,
- start_token: str = "1.", num_tasks: int = 10) -> None:
+ start_token: str = "1.", num_tasks: int = 10,
+ model=None) -> None:
sys_msg_generator = SystemMessageGenerator(task_type=TaskType.DEFAULT)
assistant_sys_msg = sys_msg_generator.from_dict(
dict(), role_tuple=("Task Generator", RoleType.DEFAULT))
- assistant_agent = ChatAgent(assistant_sys_msg)
+ assistant_agent = ChatAgent(assistant_sys_msg, model=model)
user_msg = UserChatMessage(role_name="Task Generator",
content=task_generator_prompt)
@@ -47,7 +48,7 @@ def generate_tasks(task_generator_prompt: str, language: str, domain: str,
file.write("\n".join(tasks))
-def main() -> None:
+def main(model=None) -> None:
num_tasks = 50
start_token = "1."
@@ -59,8 +60,9 @@ def main() -> None:
if not os.path.exists(f"./code/tasks/{language}_{domain}.txt"):
print(language, domain)
- pool.apply_async(generate_tasks, (task_generator_prompt, language,
- domain, start_token, num_tasks))
+ pool.apply_async(generate_tasks,
+ (task_generator_prompt, language, domain,
+ start_token, num_tasks, model))
pool.close()
pool.join()
diff --git a/examples/evaluation/single_agent.py b/examples/evaluation/single_agent.py
index 4b0a38323..a92415a77 100644
--- a/examples/evaluation/single_agent.py
+++ b/examples/evaluation/single_agent.py
@@ -39,7 +39,7 @@ def parse_question_string(question_string: str,
def generate_questions(examples: str, category: str, save_file_name: str,
key: str = 'generate_questions',
- num_questions: int = 20) -> None:
+ num_questions: int = 20, model=None) -> None:
prompt_template = PromptTemplateGenerator().get_prompt_from_key(
TaskType.EVALUATION, key)
@@ -55,7 +55,7 @@ def generate_questions(examples: str, category: str, save_file_name: str,
role_name="Assistant",
content="You are a helpful assistant.",
)
- agent = ChatAgent(assistant_sys_msg)
+ agent = ChatAgent(assistant_sys_msg, model=model)
agent.reset()
user_msg = UserChatMessage(role_name="User", content=prompt)
@@ -79,7 +79,7 @@ def generate_questions(examples: str, category: str, save_file_name: str,
f.write('\n')
-def main() -> None:
+def main(model=None) -> None:
# generate ai society evaluation questions
examples = ("1. What are the most effective ways to deal with stress?\n"
@@ -90,7 +90,7 @@ def main() -> None:
category = 'generic task Q&A'
save_file_name = 'questions_ai_society'
- generate_questions(examples, category, save_file_name)
+ generate_questions(examples, category, save_file_name, model=model)
# generate coding evaluation questions
examples = (
@@ -102,7 +102,7 @@ def main() -> None:
category = 'coding task'
save_file_name = 'questions_code'
- generate_questions(examples, category, save_file_name)
+ generate_questions(examples, category, save_file_name, model=model)
# generate math evaluation questions
examples = (
@@ -113,7 +113,7 @@ def main() -> None:
category = 'math'
save_file_name = 'questions_math'
- generate_questions(examples, category, save_file_name)
+ generate_questions(examples, category, save_file_name, model=model)
# generate science evaluation questions
examples = (
@@ -132,7 +132,8 @@ def main() -> None:
category = 'science'
save_file_name = 'questions_science'
- generate_questions(examples, category, save_file_name, num_questions=60)
+ generate_questions(examples, category, save_file_name, num_questions=60,
+ model=model)
if __name__ == "__main__":
diff --git a/examples/misalignment/single_agent.py b/examples/misalignment/single_agent.py
index 3cec461ed..6a5c90a6a 100644
--- a/examples/misalignment/single_agent.py
+++ b/examples/misalignment/single_agent.py
@@ -17,7 +17,7 @@ from camel.prompts import PromptTemplateGenerator
from camel.typing import TaskType
-def main() -> None:
+def main(model=None) -> None:
prompt = ("List 10 diverse malicious tasks that programmer can assist AGI"
"cooperatively to achieve together. Be concise. Be creative.")
sys_prompt = PromptTemplateGenerator().get_prompt_from_key(
@@ -26,7 +26,7 @@ def main() -> None:
role_name="Assistant",
content=sys_prompt,
)
- agent = ChatAgent(assistant_sys_msg)
+ agent = ChatAgent(assistant_sys_msg, model=model)
agent.reset()
user_msg = UserChatMessage(role_name="User", content=prompt)
diff --git a/examples/single_agent.py b/examples/single_agent.py
index dfc1e2453..8c8048759 100644
--- a/examples/single_agent.py
+++ b/examples/single_agent.py
@@ -17,7 +17,7 @@ from camel.prompts import PromptTemplateGenerator
from camel.typing import TaskType
-def main(key: str = 'generate_users', num_roles: int = 50):
+def main(key: str = 'generate_users', num_roles: int = 50, model=None):
prompt_template = PromptTemplateGenerator().get_prompt_from_key(
TaskType.AI_SOCIETY, key)
prompt = prompt_template.format(num_roles=num_roles)
@@ -26,7 +26,7 @@ def main(key: str = 'generate_users', num_roles: int = 50):
role_name="Assistant",
content="You are a helpful assistant.",
)
- agent = ChatAgent(assistant_sys_msg)
+ agent = ChatAgent(assistant_sys_msg, model=model)
agent.reset()
user_msg = UserChatMessage(role_name="User", content=prompt)
diff --git a/examples/test/test_ai_society_example.py b/examples/test/test_ai_society_example.py
new file mode 100644
index 000000000..d5c024df4
--- /dev/null
+++ b/examples/test/test_ai_society_example.py
@@ -0,0 +1,26 @@
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+# Licensed under the Apache License, Version 2.0 (the ā€œLicenseā€);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an ā€œAS ISā€ BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+import pytest
+
+import examples.ai_society.role_playing
+from camel.typing import ModelType
+
+
+def test_ai_society_role_playing_example():
+ examples.ai_society.role_playing.main(ModelType.STUB)
+
+
+if __name__ == "__main__":
+ pytest.main()
diff --git a/examples/test/test_single_agent.py b/examples/test/test_single_agent.py
index ea3967072..caf8790fd 100644
--- a/examples/test/test_single_agent.py
+++ b/examples/test/test_single_agent.py
@@ -11,20 +11,48 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
-import unittest
+import pytest
+import examples.code.generate_meta_data
+import examples.code.task_generation
import examples.evaluation.single_agent
import examples.misalignment.single_agent
import examples.single_agent
+from camel.typing import ModelType
+parametrize = pytest.mark.parametrize('model', [
+ ModelType.STUB,
+ pytest.param(None, marks=pytest.mark.model_backend),
+])
-class TestSingleAgent(unittest.TestCase):
- def test_single_agent(self):
- examples.single_agent.main()
+@parametrize
+def test_single_agent(model):
+ examples.single_agent.main(model=model)
- def test_misalignment_single_agent(self):
- examples.misalignment.single_agent.main()
- def test_evaluation_single_agent(self):
- examples.evaluation.single_agent.main()
+@parametrize
+def test_misalignment_single_agent(model):
+ examples.misalignment.single_agent.main(model=model)
+
+
+@parametrize
+def test_evaluation_single_agent(model):
+ examples.evaluation.single_agent.main(model=model)
+
+
+@parametrize
+def test_code_generate_metadata(model):
+ examples.code.generate_meta_data.main(model=model)
+
+
+ 'model',
+ [
+ ModelType.STUB,
+ # The following marking hangs the application. Disabled it for now.
+ # pytest.param(None,
+ # marks=[pytest.mark.model_backend, pytest.mark.slow]),
+ ])
+def test_code_task_generation(model):
+ examples.code.task_generation.main(model=model)
diff --git a/pyproject.toml b/pyproject.toml
index d8a9b5094..e7e57f2cf 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -58,9 +58,11 @@ skip = [".gitingore", "__init__.py"]
[tool.pytest.ini_options]
pythonpath = ["."]
+addopts = ["--strict-markers"]
markers = [
"full_test_only: mark a test to run only in full test mode",
"slow: mark a test as slow",
+ "model_backend: for tests that require OpenAI API key or a local LLM",
]
[tool.coverage.report]
diff --git a/test/agents/test_chat_agent.py b/test/agents/test_chat_agent.py
index f45116fe1..12106cae0 100644
--- a/test/agents/test_chat_agent.py
+++ b/test/agents/test_chat_agent.py
@@ -11,8 +11,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
-import os
-
import pytest
from camel.agents import ChatAgent
@@ -20,15 +18,17 @@ from camel.configs import ChatGPTConfig
from camel.generators import SystemMessageGenerator
from camel.messages import ChatMessage, SystemMessage
from camel.typing import ModelType, RoleType, TaskType
-from camel.utils import get_model_token_limit, openai_api_key_required
+from camel.utils import get_model_token_limit
+
+parametrize = pytest.mark.parametrize('model', [
+ ModelType.STUB,
+ pytest.param(ModelType.GPT_3_5_TURBO, marks=pytest.mark.model_backend),
+ pytest.param(ModelType.GPT_4, marks=pytest.mark.model_backend),
+])
-@openai_api_key_required
[email protected]('model', [ModelType.GPT_3_5_TURBO, ModelType.GPT_4])
+@parametrize
def test_chat_agent(model):
- assert os.environ.get(
- "OPENAI_API_KEY") is not None, "Missing OPENAI_API_KEY"
model_config = ChatGPTConfig()
system_msg = SystemMessageGenerator(
@@ -69,8 +69,7 @@ def test_chat_agent(model):
"max_tokens_exceeded")
-@openai_api_key_required
@pytest.mark.parametrize('n', [1, 2, 3])
def test_chat_agent_multiple_return_messages(n):
model_config = ChatGPTConfig(temperature=1.4, n=n)
diff --git a/test/agents/test_critic_agent.py b/test/agents/test_critic_agent.py
index 0a00ff1b8..b42e25445 100644
--- a/test/agents/test_critic_agent.py
+++ b/test/agents/test_critic_agent.py
@@ -16,7 +16,6 @@ import pytest
from camel.agents import CriticAgent
from camel.messages import ChatMessage, SystemMessage
from camel.typing import RoleType
-from camel.utils import openai_api_key_required
@pytest.fixture
@@ -58,8 +57,7 @@ def test_flatten_options(critic_agent: CriticAgent):
assert critic_agent.flatten_options(messages) == expected_output
-@openai_api_key_required
def test_get_option(critic_agent: CriticAgent):
messages = [
ChatMessage(
@@ -102,8 +100,7 @@ def test_parse_critic(critic_agent: CriticAgent):
assert critic_agent.parse_critic(critic_msg) == expected_output
-@openai_api_key_required
def test_step(critic_agent: CriticAgent):
messages = [
ChatMessage(
diff --git a/test/agents/test_embodied_agent.py b/test/agents/test_embodied_agent.py
index 36119445b..ecaf584d8 100644
--- a/test/agents/test_embodied_agent.py
+++ b/test/agents/test_embodied_agent.py
@@ -17,10 +17,9 @@ from camel.agents import EmbodiedAgent, HuggingFaceToolAgent
from camel.generators import SystemMessageGenerator
from camel.messages import ChatMessage, UserChatMessage
from camel.typing import RoleType
-from camel.utils import openai_api_key_required
-@openai_api_key_required
def test_get_action_space_prompt():
role_name = "Artist"
meta_dict = dict(role=role_name, task="Drawing")
@@ -34,9 +33,8 @@ def test_get_action_space_prompt():
assert agent.get_action_space_prompt().startswith(expected_prompt)
@pytest.mark.full_test_only
-@openai_api_key_required
def test_step():
# Create an embodied agent
role_name = "Artist"
diff --git a/test/agents/test_role_playing.py b/test/agents/test_role_playing.py
index 739d48425..75ffa323c 100644
--- a/test/agents/test_role_playing.py
+++ b/test/agents/test_role_playing.py
@@ -18,7 +18,6 @@ import pytest
from camel.agents import ChatAgent, RolePlaying
from camel.messages import AssistantChatMessage, ChatMessage, SystemMessageType
from camel.typing import ModelType, TaskType
-from camel.utils import openai_api_key_required
def test_role_playing_init():
@@ -52,8 +51,7 @@ def test_role_playing_init():
assert role_playing.critic is None
-@openai_api_key_required
@pytest.mark.parametrize(
"task_type, extend_sys_msg_meta_dicts, extend_task_specify_meta_dict",
[(TaskType.AI_SOCIETY, None, None),
diff --git a/test/agents/test_task_agent.py b/test/agents/test_task_agent.py
index bc5cdcc15..c9945b20a 100644
--- a/test/agents/test_task_agent.py
+++ b/test/agents/test_task_agent.py
@@ -11,21 +11,26 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+from typing import Optional
+
import pytest
from camel.agents import TaskPlannerAgent, TaskSpecifyAgent
from camel.configs import ChatGPTConfig
-from camel.typing import TaskType
-from camel.utils import openai_api_key_required
+from camel.typing import ModelType, TaskType
+
+parametrize = pytest.mark.parametrize('model', [
+ ModelType.STUB,
+ pytest.param(None, marks=pytest.mark.model_backend),
+])
-@openai_api_key_required
-def test_task_specify_ai_society_agent():
+@parametrize
+def test_task_specify_ai_society_agent(model: Optional[ModelType]):
original_task_prompt = "Improving stage presence and performance skills"
print(f"Original task prompt:\n{original_task_prompt}\n")
- task_specify_agent = TaskSpecifyAgent(model_config=ChatGPTConfig(
- temperature=1.0))
+ task_specify_agent = TaskSpecifyAgent(
+ model_config=ChatGPTConfig(temperature=1.0), model=model)
specified_task_prompt = task_specify_agent.step(
original_task_prompt, meta_dict=dict(assistant_role="Musician",
user_role="Student"))
@@ -33,14 +38,14 @@ def test_task_specify_ai_society_agent():
print(f"Specified task prompt:\n{specified_task_prompt}\n")
-@openai_api_key_required
-def test_task_specify_code_agent():
+@parametrize
+def test_task_specify_code_agent(model: Optional[ModelType]):
original_task_prompt = "Modeling molecular dynamics"
print(f"Original task prompt:\n{original_task_prompt}\n")
task_specify_agent = TaskSpecifyAgent(
task_type=TaskType.CODE,
model_config=ChatGPTConfig(temperature=1.0),
+ model=model,
)
specified_task_prompt = task_specify_agent.step(
original_task_prompt, meta_dict=dict(domain="Chemistry",
@@ -49,20 +54,20 @@ def test_task_specify_code_agent():
print(f"Specified task prompt:\n{specified_task_prompt}\n")
-@openai_api_key_required
-def test_task_planner_agent():
+@parametrize
+def test_task_planner_agent(model: Optional[ModelType]):
original_task_prompt = "Modeling molecular dynamics"
print(f"Original task prompt:\n{original_task_prompt}\n")
task_specify_agent = TaskSpecifyAgent(
task_type=TaskType.CODE,
model_config=ChatGPTConfig(temperature=1.0),
+ model=model,
)
specified_task_prompt = task_specify_agent.step(
original_task_prompt, meta_dict=dict(domain="Chemistry",
language="Python"))
print(f"Specified task prompt:\n{specified_task_prompt}\n")
- task_planner_agent = TaskPlannerAgent(model_config=ChatGPTConfig(
- temperature=1.0))
+ task_planner_agent = TaskPlannerAgent(
+ model_config=ChatGPTConfig(temperature=1.0), model=model)
planned_task_prompt = task_planner_agent.step(specified_task_prompt)
print(f"Planned task prompt:\n{planned_task_prompt}\n")
diff --git a/test/agents/tool_agents/test_hugging_face_tool_agent.py b/test/agents/tool_agents/test_hugging_face_tool_agent.py
index 8f9c1f6f3..a84600c45 100644
--- a/test/agents/tool_agents/test_hugging_face_tool_agent.py
+++ b/test/agents/tool_agents/test_hugging_face_tool_agent.py
@@ -14,12 +14,10 @@
import pytest
from camel.agents import HuggingFaceToolAgent
-from camel.utils import openai_api_key_required
@pytest.mark.full_test_only
-@openai_api_key_required
def test_hugging_face_tool_agent_initialization():
agent = HuggingFaceToolAgent("hugging_face_tool_agent")
assert agent.name == "hugging_face_tool_agent"
@@ -27,9 +25,8 @@ def test_hugging_face_tool_agent_initialization():
assert agent.description.startswith(f"The `{agent.name}` is a tool agent")
@pytest.mark.full_test_only
-@openai_api_key_required
def test_hugging_face_tool_agent_step():
from PIL.PngImagePlugin import PngImageFile
agent = HuggingFaceToolAgent("hugging_face_tool_agent")
@@ -37,9 +34,8 @@ def test_hugging_face_tool_agent_step():
assert isinstance(result, PngImageFile)
@pytest.mark.full_test_only
-@openai_api_key_required
def test_hugging_face_tool_agent_chat():
from PIL.PngImagePlugin import PngImageFile
agent = HuggingFaceToolAgent("hugging_face_tool_agent")
@@ -47,9 +43,8 @@ def test_hugging_face_tool_agent_chat():
assert isinstance(result, PngImageFile)
@pytest.mark.full_test_only
-@openai_api_key_required
def test_hugging_face_tool_agent_reset():
agent = HuggingFaceToolAgent("hugging_face_tool_agent")
agent.reset()
diff --git a/test/conftest.py b/test/conftest.py
index f8144927d..629d2d333 100644
--- a/test/conftest.py
+++ b/test/conftest.py
@@ -34,9 +34,11 @@ def pytest_collection_modifyitems(config: Config, items: List[Item]) -> None:
if "full_test_only" in item.keywords:
item.add_marker(skip_full_test)
- # Skip slow tests if fast test mode is enabled
+ # Skip all tests involving LLM inference both remote
+ # (including OpenAI API) and local ones, since they are slow
+ # and may drain money if fast test mode is enabled.
if config.getoption("--fast-test-mode"):
skip_slow = pytest.mark.skip(reason="Skipped for fast test mode")
for item in items:
- if "slow" in item.keywords:
+ if "slow" in item.keywords or "model_backend" in item.keywords:
item.add_marker(skip_slow)
diff --git a/test/test_model_backend.py b/test/test_model_backend.py
new file mode 100644
index 000000000..94f22d9d2
--- /dev/null
+++ b/test/test_model_backend.py
@@ -0,0 +1,67 @@
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+# Licensed under the Apache License, Version 2.0 (the ā€œLicenseā€);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an ā€œAS ISā€ BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+import pytest
+
+from camel.configs import ChatGPTConfig
+from camel.model_backend import ModelFactory
+from camel.typing import ModelType
+
+parametrize = pytest.mark.parametrize('model', [
+ ModelType.STUB,
+ pytest.param(None, marks=pytest.mark.model_backend),
+ pytest.param(ModelType.GPT_3_5_TURBO, marks=pytest.mark.model_backend),
+ pytest.param(ModelType.GPT_4, marks=pytest.mark.model_backend),
+])
+
+
+@parametrize
+def test_openai_model(model):
+ model_config_dict = ChatGPTConfig().__dict__
+ model_inst = ModelFactory.create(model, model_config_dict)
+ messages = [
+ {
+ 'role': 'system',
+ 'content': 'You can make a task more specific.'
+ },
+ {
+ 'role':
+ 'user',
+ 'content': ('Here is a task that Python Programmer will help '
+ 'Stock Trader to complete: Develop a trading bot '
+ 'for the stock market.\nPlease make it more specific.'
+ ' Be creative and imaginative.\nPlease reply with '
+ 'the specified task in 50 words or less. '
+ 'Do not add anything else.')
+ },
+ ]
+ response = model_inst.run(messages=messages)
+ assert isinstance(response, dict)
+ assert 'id' in response
+ assert isinstance(response['id'], str)
+ assert 'usage' in response
+ assert isinstance(response['usage'], dict)
+ assert 'choices' in response
+ assert isinstance(response['choices'], list)
+ assert len(response['choices']) == 1
+ choice = response['choices'][0]
+ assert 'finish_reason' in choice
+ assert isinstance(choice['finish_reason'], str)
+ assert 'message' in choice
+ message = choice['message']
+ assert isinstance(message, dict)
+ assert 'content' in message
+ assert isinstance(message['content'], str)
+ assert 'role' in message
+ assert isinstance(message['role'], str)
+ assert message['role'] == 'assistant'