Skip to content

armgit / camel

Public Imported from GitHub
0

🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org

chore: Add mock LLM for test purposes; Enable multiple `step()` calls in test units to enhance test robustness (#1128)

Yihan Lin689094538d12
diff --git a/test/agents/test_babyagi_playing.py b/test/agents/test_babyagi_playing.py
index 64084dd59..fcacb4fed 100644
--- a/test/agents/test_babyagi_playing.py
+++ b/test/agents/test_babyagi_playing.py
@@ -11,19 +11,52 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+from copy import deepcopy
+from unittest.mock import MagicMock
+
import pytest
+from openai.types.chat.chat_completion import Choice
+from openai.types.chat.chat_completion_message import ChatCompletionMessage
+from openai.types.completion_usage import CompletionUsage
from camel.agents import ChatAgent, TaskCreationAgent, TaskPrioritizationAgent
from camel.messages import BaseMessage
from camel.models import ModelFactory
from camel.societies import BabyAGI
from camel.types import (
+ ChatCompletion,
ModelPlatformType,
ModelType,
RoleType,
TaskType,
)
+model_backend_rsp_base = ChatCompletion(
+ id="mock_response_id",
+ choices=[
+ Choice(
+ finish_reason="stop",
+ index=0,
+ logprobs=None,
+ message=ChatCompletionMessage(
+ content="Mock task specifier response",
+ role="assistant",
+ function_call=None,
+ tool_calls=None,
+ ),
+ )
+ ],
+ created=123456789,
+ model="gpt-4o-2024-05-13",
+ object="chat.completion",
+ usage=CompletionUsage(
+ completion_tokens=32,
+ prompt_tokens=15,
+ total_tokens=47,
+ ),
+)
+
+
parametrize = pytest.mark.parametrize(
'model',
[
@@ -66,7 +99,7 @@ def test_babyagi_playing_init(model):
@parametrize
-def test_babyagi_playing_step(model):
+def test_babyagi_playing_step(model, step_call_count=3):
task_prompt = "Develop a trading bot for the stock market"
babyagi_playing = BabyAGI(
@@ -78,18 +111,118 @@ def test_babyagi_playing_step(model):
message_window_size=5,
)
+ # Mock model_backend responses
+ # Initial task list when first calling babyagi_playing.step()
+ task_creation_agent_model_rsp0 = deepcopy(model_backend_rsp_base)
+ task_creation_agent_model_rsp0.choices[0].message.content = "1. Task 0"
+ task_prioritization_agent_model_rsp0 = deepcopy(model_backend_rsp_base)
+ task_prioritization_agent_model_rsp0.choices[
+ 0
+ ].message.content = "1. Task 0"
+
+ # Solve the highest priority (left most) task: Task 0
+ assistant_agent_model_rsp1 = deepcopy(model_backend_rsp_base)
+ assistant_agent_model_rsp1.choices[
+ 0
+ ].message.content = "Solution for Task 0"
+
+ # Task 0 is solved, remove from TaskPrioritizationAgent model response
+ task_creation_agent_model_rsp1 = deepcopy(model_backend_rsp_base)
+ task_creation_agent_model_rsp1.choices[
+ 0
+ ].message.content = "1. \nTask 1\n\n2. Task 2\n\n3. Task 3"
+ task_prioritization_agent_model_rsp1 = deepcopy(model_backend_rsp_base)
+ task_prioritization_agent_model_rsp1.choices[
+ 0
+ ].message.content = "1. Task 2 \nTask 1 \n3. Task 3 "
+
+ # Solve the highest priority (left most) task: Task 2
+ assistant_agent_model_rsp2 = deepcopy(model_backend_rsp_base)
+ assistant_agent_model_rsp2.choices[
+ 0
+ ].message.content = "Solution for Task 2"
+
+ # Task 2 is solved, remove from TaskPrioritizationAgent model response
+ task_creation_agent_model_rsp2 = deepcopy(model_backend_rsp_base)
+ task_creation_agent_model_rsp2.choices[
+ 0
+ ].message.content = "1. Task 4\n\n2. Task 5"
+ task_prioritization_agent_model_rsp2 = deepcopy(model_backend_rsp_base)
+ task_prioritization_agent_model_rsp2.choices[
+ 0
+ ].message.content = "1. Task 1 \nTask 3 \n3. Task 4 \n4. Task 5"
+
+ # Solve the highest priority (left most) task: Task 1
+ assistant_agent_model_rsp3 = deepcopy(model_backend_rsp_base)
+ assistant_agent_model_rsp3.choices[
+ 0
+ ].message.content = "Solution for Task 1"
+
+ # Task 1 is solved, remove from TaskPrioritizationAgent model response
+ task_creation_agent_model_rsp3 = deepcopy(model_backend_rsp_base)
+ task_creation_agent_model_rsp3.choices[
+ 0
+ ].message.content = "1. Task 6\n\n2. Task 7\n\n3. Task 8"
+ task_prioritization_agent_model_rsp3 = deepcopy(model_backend_rsp_base)
+ task_prioritization_agent_model_rsp3.choices[0].message.content = (
+ "1. Task 6 \nTask 3 \n3. Task 4 \n"
+ "4. Task 7 \n5. Task 5 \n6. Task 8"
+ )
+
+ babyagi_playing.task_creation_agent.model_backend.run = MagicMock(
+ side_effect=[
+ task_creation_agent_model_rsp0,
+ task_creation_agent_model_rsp1,
+ task_creation_agent_model_rsp2,
+ task_creation_agent_model_rsp3,
+ ]
+ )
+ babyagi_playing.task_prioritization_agent.model_backend.run = MagicMock(
+ side_effect=[
+ task_prioritization_agent_model_rsp0,
+ task_prioritization_agent_model_rsp1,
+ task_prioritization_agent_model_rsp2,
+ task_prioritization_agent_model_rsp3,
+ ]
+ )
+ babyagi_playing.assistant_agent.model_backend.run = MagicMock(
+ side_effect=[
+ assistant_agent_model_rsp1,
+ assistant_agent_model_rsp2,
+ assistant_agent_model_rsp3,
+ ]
+ )
+
print(f"AI Assistant sys message:\n{babyagi_playing.assistant_sys_msg}\n")
print(f"Original task prompt:\n{task_prompt}\n")
print(f"Specified task prompt:\n{babyagi_playing.specified_task_prompt}\n")
- assistant_response = babyagi_playing.step()
-
- assert isinstance(assistant_response.msgs, list)
- assert len(assistant_response.msgs) == 1
- assert isinstance(assistant_response.msgs[0], BaseMessage)
- assert isinstance(assistant_response.terminated, bool)
- assert assistant_response.terminated is False
- assert isinstance(assistant_response.info, dict)
-
- assert len(babyagi_playing.subtasks) > 0
- assert len(babyagi_playing.solved_subtasks) == 1
+ for i in range(step_call_count):
+ # Call assistant for multiple times to make test units more robust
+ assistant_response = babyagi_playing.step()
+
+ assert isinstance(
+ assistant_response.msgs, list
+ ), f"Error in calling round {i+1}"
+ assert (
+ len(assistant_response.msgs) == 1
+ ), f"Error in calling round {i+1}"
+ assert isinstance(
+ assistant_response.msgs[0], BaseMessage
+ ), f"Error in calling round {i+1}"
+ assert isinstance(
+ assistant_response.terminated, bool
+ ), f"Error in calling round {i+1}"
+ assert (
+ assistant_response.terminated is False
+ ), f"Error in calling round {i+1}"
+ assert isinstance(
+ assistant_response.info, dict
+ ), f"Error in calling round {i+1}"
+
+ assert (
+ len(babyagi_playing.subtasks) > 0
+ ), f"Error in calling round {i+1}"
+ assert (
+ len(babyagi_playing.solved_subtasks) == i + 1
+ ), f"Error in calling round {i+1}"
diff --git a/test/agents/test_chat_agent.py b/test/agents/test_chat_agent.py
index cad238abe..870a75100 100644
--- a/test/agents/test_chat_agent.py
+++ b/test/agents/test_chat_agent.py
@@ -13,13 +13,18 @@
# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
import asyncio
import json
+from copy import deepcopy
from io import BytesIO
from typing import List
-from unittest.mock import Mock
+from unittest.mock import MagicMock
import pytest
from openai.types.chat.chat_completion import Choice
from openai.types.chat.chat_completion_message import ChatCompletionMessage
+from openai.types.chat.chat_completion_message_tool_call import (
+ ChatCompletionMessageToolCall,
+ Function,
+)
from openai.types.completion_usage import CompletionUsage
from PIL import Image
from pydantic import BaseModel, Field
@@ -48,6 +53,31 @@ from camel.types import (
)
from camel.utils.async_func import sync_funcs_to_async
+model_backend_rsp_base = ChatCompletion(
+ id="mock_response_id",
+ choices=[
+ Choice(
+ finish_reason="stop",
+ index=0,
+ logprobs=None,
+ message=ChatCompletionMessage(
+ content="This is a mock response content.",
+ role="assistant",
+ function_call=None,
+ tool_calls=None,
+ ),
+ )
+ ],
+ created=123456789,
+ model="gpt-4o-2024-05-13",
+ object="chat.completion",
+ usage=CompletionUsage(
+ completion_tokens=32,
+ prompt_tokens=15,
+ total_tokens=47,
+ ),
+)
+
parametrize = pytest.mark.parametrize(
'model',
[
@@ -61,7 +91,7 @@ parametrize = pytest.mark.parametrize(
@parametrize
-def test_chat_agent(model):
+def test_chat_agent(model, step_call_count=3):
model = model
system_msg = SystemMessageGenerator(
task_type=TaskType.AI_SOCIETY
@@ -82,6 +112,9 @@ def test_chat_agent(model):
for assistant in [assistant_with_sys_msg, assistant_without_sys_msg]:
assistant.reset()
+ assistant.model_backend.run = MagicMock(
+ return_value=model_backend_rsp_base
+ )
user_msg_bm = BaseMessage(
role_name="Patient",
@@ -93,14 +126,17 @@ def test_chat_agent(model):
user_msg_str = "Hello!"
for assistant in [assistant_with_sys_msg, assistant_without_sys_msg]:
- for user_msg in [user_msg_bm, user_msg_str]:
- response = assistant.step(user_msg)
- assert isinstance(response.msgs, list)
- assert len(response.msgs) > 0
- assert isinstance(response.terminated, bool)
- assert response.terminated is False
- assert isinstance(response.info, dict)
- assert response.info['id'] is not None
+ for i in range(step_call_count):
+ for user_msg in [user_msg_bm, user_msg_str]:
+ response = assistant.step(user_msg)
+ assert isinstance(response.msgs, list), f"Error in round {i+1}"
+ assert len(response.msgs) > 0, f"Error in round {i+1}"
+ assert isinstance(
+ response.terminated, bool
+ ), f"Error in round {i+1}"
+ assert response.terminated is False, f"Error in round {i+1}"
+ assert isinstance(response.info, dict), f"Error in round {i+1}"
+ assert response.info['id'] is not None, f"Error in round {i+1}"
@pytest.mark.model_backend
@@ -147,7 +183,7 @@ def test_chat_agent_stored_messages():
@pytest.mark.model_backend
-def test_chat_agent_step_with_structure_response():
+def test_chat_agent_step_with_structure_response(step_call_count=3):
system_msg = BaseMessage(
role_name="assistant",
role_type=RoleType.ASSISTANT,
@@ -167,26 +203,43 @@ def test_chat_agent_step_with_structure_response():
content="Tell a jokes.",
)
- response = assistant.step(user_msg, response_format=JokeResponse)
- response_content_json = json.loads(response.msg.content)
- joke_response_keys = set(
- JokeResponse.model_json_schema()["properties"].keys()
+ model_backend_rsp_tool = deepcopy(model_backend_rsp_base)
+
+ model_backend_rsp_tool.choices[0].message.content = '{ \
+ "joke":"What do you call fake spaghetti? An impasta!", \
+ "funny_level":"6" \
+ }'
+ model_backend_rsp_tool.choices[0].message.tool_calls = []
+ model_backend_rsp_tool.choices[0].message.parsed = JokeResponse(
+ joke="What do you call fake spaghetti? An impasta!", funny_level='6'
+ )
+ assistant.model_backend.run = MagicMock(
+ return_value=model_backend_rsp_tool
)
- response_content_keys = set(response_content_json.keys())
+ for i in range(step_call_count):
+ response = assistant.step(user_msg, response_format=JokeResponse)
+ response_content_json = json.loads(response.msg.content)
+ joke_response_keys = set(
+ JokeResponse.model_json_schema()["properties"].keys()
+ )
- assert joke_response_keys.issubset(
- response_content_keys
- ), f"Missing keys: {joke_response_keys - response_content_keys}"
+ response_content_keys = set(response_content_json.keys())
- for key in joke_response_keys:
- assert (
- key in response_content_json
- ), f"Key {key} not found in response content"
+ assert joke_response_keys.issubset(response_content_keys), (
+ f"Error in calling round {i+1}: "
+ f"Missing keys: {joke_response_keys - response_content_keys}"
+ )
+
+ for key in joke_response_keys:
+ assert key in response_content_json, (
+ f"Error in calling round {i+1}: "
+ f"Key {key} not found in response content"
+ )
@pytest.mark.model_backend
-def test_chat_agent_step_with_external_tools():
+def test_chat_agent_step_with_external_tools(step_call_count=3):
internal_tools = [FunctionTool(SearchToolkit().search_duckduckgo)]
external_tools = MathToolkit().get_tools()
tool_list = internal_tools + external_tools
@@ -201,6 +254,91 @@ def test_chat_agent_step_with_external_tools():
model_type=ModelType.GPT_4O_MINI,
model_config_dict=model_config_dict,
)
+ model_backend_external1 = ChatCompletion(
+ id='mock_id_123456',
+ choices=[
+ Choice(
+ finish_reason='tool_calls',
+ index=0,
+ logprobs=None,
+ message=ChatCompletionMessage(
+ content=None,
+ refusal=None,
+ role='assistant',
+ audio=None,
+ function_call=None,
+ tool_calls=[
+ ChatCompletionMessageToolCall(
+ id='call_mock_123456',
+ function=Function(
+ arguments='{ \
+ "query": "Portal game release year" \
+ }',
+ name='search_duckduckgo',
+ ),
+ type='function',
+ ),
+ ChatCompletionMessageToolCall(
+ id='call_mock_123457',
+ function=Function(
+ arguments='{ \
+ "query": "United States founding year" \
+ }',
+ name='search_duckduckgo',
+ ),
+ type='function',
+ ),
+ ],
+ ),
+ )
+ ],
+ created=1730745899,
+ model='gpt-4o-mini-2024-07-18',
+ object='chat.completion',
+ service_tier=None,
+ usage=CompletionUsage(
+ completion_tokens=56, prompt_tokens=292, total_tokens=348
+ ),
+ )
+
+ model_backend_external2 = ChatCompletion(
+ id='chatcmpl-mock_id_123456',
+ choices=[
+ Choice(
+ finish_reason='tool_calls',
+ index=0,
+ logprobs=None,
+ message=ChatCompletionMessage(
+ content=None,
+ refusal=None,
+ role='assistant',
+ audio=None,
+ function_call=None,
+ tool_calls=[
+ ChatCompletionMessageToolCall(
+ id='call_mock_123456',
+ function=Function(
+ arguments='{"a":1776,"b":2007}', name='sub'
+ ),
+ type='function',
+ )
+ ],
+ ),
+ )
+ ],
+ created=1730745902,
+ model='gpt-4o-mini-2024-07-18',
+ object='chat.completion',
+ service_tier=None,
+ usage=CompletionUsage(
+ completion_tokens=19, prompt_tokens=991, total_tokens=1010
+ ),
+ )
+
+ model.run = MagicMock(
+ side_effect=[model_backend_external1, model_backend_external2]
+ * step_call_count
+ )
# Set external_tools
external_tool_agent = ChatAgent(
@@ -219,11 +357,14 @@ def test_chat_agent_step_with_external_tools():
"from the year that United States was founded?",
)
- response = external_tool_agent.step(usr_msg)
- assert not response.msg.content
+ for i in range(step_call_count):
+ response = external_tool_agent.step(usr_msg)
+ assert not response.msg.content
- external_tool_request = response.info["external_tool_request"]
- assert external_tool_request.function.name == "sub"
+ external_tool_request = response.info["external_tool_request"]
+ assert (
+ external_tool_request.function.name == "sub"
+ ), f"Error in calling round {i+1}"
@pytest.mark.model_backend
@@ -260,7 +401,7 @@ def test_chat_agent_messages_window():
@pytest.mark.model_backend
-def test_chat_agent_step_exceed_token_number():
+def test_chat_agent_step_exceed_token_number(step_call_count=3):
system_msg = BaseMessage(
role_name="assistant",
role_type=RoleType.ASSISTANT,
@@ -271,6 +412,9 @@ def test_chat_agent_step_exceed_token_number():
system_message=system_msg,
token_limit=1,
)
+ assistant.model_backend.run = MagicMock(
+ return_value=model_backend_rsp_base
+ )
user_msg = BaseMessage(
role_name="User",
@@ -279,20 +423,61 @@ def test_chat_agent_step_exceed_token_number():
content="Tell me a joke.",
)
- response = assistant.step(user_msg)
- assert len(response.msgs) == 0
- assert response.terminated
+ for i in range(step_call_count):
+ response = assistant.step(user_msg)
+ assert len(response.msgs) == 0, f"Error in calling round {i+1}"
+ assert response.terminated, f"Error in calling round {i+1}"
@pytest.mark.model_backend
@pytest.mark.parametrize('n', [1, 2, 3])
-def test_chat_agent_multiple_return_messages(n):
+def test_chat_agent_multiple_return_messages(n, step_call_count=3):
model_config = ChatGPTConfig(temperature=1.4, n=n)
model = ModelFactory.create(
model_platform=ModelPlatformType.OPENAI,
model_type=ModelType.GPT_4O_MINI,
model_config_dict=model_config.as_dict(),
)
+ model_backend_rsp_tool = ChatCompletion(
+ id="mock_response_id",
+ choices=[
+ Choice(
+ finish_reason="stop",
+ index=0,
+ logprobs=None,
+ message=ChatCompletionMessage(
+ content="What do you call fake spaghetti? An impasta!",
+ role="assistant",
+ function_call=None,
+ tool_calls=[
+ ChatCompletionMessageToolCall(
+ id="call_mock123456",
+ function=Function(
+ arguments='{ \
+ "joke":"What do you call fake spaghetti?" \
+ " An impasta!", \
+ "funny_level":"6" \
+ }',
+ name="return_json_response",
+ ),
+ type="function",
+ )
+ ],
+ ),
+ )
+ ]
+ * n,
+ created=123456789,
+ model="gpt-4o-2024-05-13",
+ object="chat.completion",
+ usage=CompletionUsage(
+ completion_tokens=32,
+ prompt_tokens=15,
+ total_tokens=47,
+ ),
+ )
+ model.run = MagicMock(return_value=model_backend_rsp_tool)
+
system_msg = BaseMessage(
"Assistant",
RoleType.ASSISTANT,
@@ -316,21 +501,49 @@ def test_chat_agent_multiple_return_messages(n):
user_msg
)
- assert assistant_with_sys_msg_response.msgs is not None
- assert len(assistant_with_sys_msg_response.msgs) == n
- assert assistant_without_sys_msg_response.msgs is not None
- assert len(assistant_without_sys_msg_response.msgs) == n
+ for i in range(step_call_count):
+ assert (
+ assistant_with_sys_msg_response.msgs is not None
+ ), f"Error in calling round {i+1}"
+ assert (
+ len(assistant_with_sys_msg_response.msgs) == n
+ ), f"Error in calling round {i+1}"
+ assert (
+ assistant_without_sys_msg_response.msgs is not None
+ ), f"Error in calling round {i+1}"
+ assert (
+ len(assistant_without_sys_msg_response.msgs) == n
+ ), f"Error in calling round {i+1}"
@pytest.mark.model_backend
@pytest.mark.parametrize('n', [2])
-def test_chat_agent_multiple_return_message_error(n):
+def test_chat_agent_multiple_return_message_error(n, step_call_count=3):
model_config = ChatGPTConfig(temperature=1.4, n=n)
model = ModelFactory.create(
model_platform=ModelPlatformType.OPENAI,
model_type=ModelType.GPT_4O_MINI,
model_config_dict=model_config.as_dict(),
)
+ model_backend_multi_messages = deepcopy(model_backend_rsp_base)
+ model_backend_multi_messages.choices.append(
+ Choice(
+ finish_reason='stop',
+ index=1,
+ logprobs=None,
+ message=ChatCompletionMessage(
+ content='Why did the scarecrow win an award? '
+ 'Because he was outstanding in his field!',
+ refusal=None,
+ role='assistant',
+ audio=None,
+ function_call=None,
+ tool_calls=None,
+ ),
+ )
+ )
+ model.run = MagicMock(return_value=model_backend_multi_messages)
+
system_msg = BaseMessage(
"Assistant",
RoleType.ASSISTANT,
@@ -347,19 +560,21 @@ def test_chat_agent_multiple_return_message_error(n):
meta_dict=dict(),
content="Tell me a joke.",
)
- assistant_response = assistant.step(user_msg)
+ for _ in range(step_call_count):
+ assistant_response = assistant.step(user_msg)
- with pytest.raises(
- RuntimeError,
- match=(
- "Property msg is only available " "for a single message in msgs."
- ),
- ):
- _ = assistant_response.msg
+ with pytest.raises(
+ RuntimeError,
+ match=(
+ "Property msg is only available "
+ "for a single message in msgs."
+ ),
+ ):
+ _ = assistant_response.msg
@pytest.mark.model_backend
-def test_chat_agent_stream_output():
+def test_chat_agent_stream_output(step_call_count=3):
system_msg = BaseMessage(
"Assistant",
RoleType.ASSISTANT,
@@ -379,20 +594,27 @@ def test_chat_agent_stream_output():
model_type=ModelType.GPT_4O_MINI,
model_config_dict=stream_model_config.as_dict(),
)
+ model.run = MagicMock(return_value=model_backend_rsp_base)
stream_assistant = ChatAgent(system_msg, model=model)
stream_assistant.reset()
- stream_assistant_response = stream_assistant.step(user_msg)
+ for i in range(step_call_count):
+ stream_assistant_response = stream_assistant.step(user_msg)
- for msg in stream_assistant_response.msgs:
- assert len(msg.content) > 0
+ for msg in stream_assistant_response.msgs:
+ assert len(msg.content) > 0, f"Error in calling round {i+1}"
- stream_usage = stream_assistant_response.info["usage"]
- assert stream_usage["completion_tokens"] > 0
- assert stream_usage["prompt_tokens"] > 0
- assert (
- stream_usage["total_tokens"]
- == stream_usage["completion_tokens"] + stream_usage["prompt_tokens"]
- )
+ stream_usage = stream_assistant_response.info["usage"]
+ assert (
+ stream_usage["completion_tokens"] > 0
+ ), f"Error in calling round {i+1}"
+ assert (
+ stream_usage["prompt_tokens"] > 0
+ ), f"Error in calling round {i+1}"
+ assert (
+ stream_usage["total_tokens"]
+ == stream_usage["completion_tokens"]
+ + stream_usage["prompt_tokens"]
+ ), f"Error in calling round {i+1}"
@pytest.mark.model_backend
@@ -491,7 +713,7 @@ def test_function_enabled():
@pytest.mark.model_backend
-def test_tool_calling_sync():
+def test_tool_calling_sync(step_call_count=3):
system_message = BaseMessage(
role_name="assistant",
role_type=RoleType.ASSISTANT,
@@ -518,23 +740,145 @@ def test_tool_calling_sync():
meta_dict=dict(),
content="Calculate the result of: 2*8-10.",
)
- agent_response = agent.step(user_msg)
+ model_backend_rsp_tool = ChatCompletion(
+ id='mock_id_123456',
+ choices=[
+ Choice(
+ finish_reason='tool_calls',
+ index=0,
+ logprobs=None,
+ message=ChatCompletionMessage(
+ content=None,
+ refusal=None,
+ role='assistant',
+ audio=None,
+ function_call=None,
+ tool_calls=[
+ ChatCompletionMessageToolCall(
+ id='call_mock_123456',
+ function=Function(
+ arguments='{ \
+ "a": 2, \
+ "b": 8, \
+ "decimal_places": 0 \
+ }',
+ name='multiply',
+ ),
+ type='function',
+ ),
+ ChatCompletionMessageToolCall(
+ id='call_mock_123457',
+ function=Function(
+ arguments='{"a": 0, "b": 10}', name='sub'
+ ),
+ type='function',
+ ),
+ ],
+ ),
+ )
+ ],
+ created=1730752528,
+ model='gpt-4o-mini-2024-07-18',
+ object='chat.completion',
+ service_tier=None,
+ usage=CompletionUsage(
+ completion_tokens=50, prompt_tokens=152, total_tokens=202
+ ),
+ )
+ model_backend_rsp_tool1 = ChatCompletion(
+ id='mock_id_123456',
+ choices=[
+ Choice(
+ finish_reason='tool_calls',
+ index=0,
+ logprobs=None,
+ message=ChatCompletionMessage(
+ content=None,
+ refusal=None,
+ role='assistant',
+ audio=None,
+ function_call=None,
+ tool_calls=[
+ ChatCompletionMessageToolCall(
+ id='call_kIJby7Y6As6gAbWoxDqxD6HG',
+ function=Function(
+ arguments='{"a":16,"b":10}', name='sub'
+ ),
+ type='function',
+ )
+ ],
+ ),
+ )
+ ],
+ created=1730752528,
+ model='gpt-4o-mini-2024-07-18',
+ object='chat.completion',
+ service_tier=None,
+ usage=CompletionUsage(
+ completion_tokens=50, prompt_tokens=152, total_tokens=202
+ ),
+ )
+ model_backend_rsp_tool2 = ChatCompletion(
+ id='mock_id_123456',
+ choices=[
+ Choice(
+ finish_reason='tool_calls',
+ index=0,
+ logprobs=None,
+ message=ChatCompletionMessage(
+ content='The result of the calculation \
+ (2 times 8 - 10) is (6).',
+ refusal=None,
+ role='assistant',
+ audio=None,
+ function_call=None,
+ tool_calls=None,
+ ),
+ )
+ ],
+ created=1730752528,
+ model='gpt-4o-mini-2024-07-18',
+ object='chat.completion',
+ service_tier=None,
+ usage=CompletionUsage(
+ completion_tokens=50, prompt_tokens=152, total_tokens=202
+ ),
+ )
- tool_calls: List[FunctionCallingRecord] = [
- call for call in agent_response.info['tool_calls']
- ]
+ model.run = MagicMock(
+ side_effect=[
+ model_backend_rsp_tool,
+ model_backend_rsp_tool1,
+ model_backend_rsp_tool2,
+ ]
+ * step_call_count
+ )
- assert len(tool_calls) > 0
- assert str(tool_calls[0]).startswith("Function Execution")
+ for i in range(step_call_count):
+ agent_response = agent.step(user_msg)
- assert tool_calls[0].func_name == "multiply"
- assert tool_calls[0].args == {'a': 2, 'b': 8, 'decimal_places': 0}
- assert tool_calls[0].result == 16
+ tool_calls: List[FunctionCallingRecord] = [
+ call for call in agent_response.info['tool_calls']
+ ]
+
+ assert len(tool_calls) > 0, f"Error in calling round {i+1}"
+ assert str(tool_calls[0]).startswith(
+ "Function Execution"
+ ), f"Error in calling round {i+1}"
+ assert (
+ tool_calls[0].func_name == "multiply"
+ ), f"Error in calling round {i+1}"
+ assert tool_calls[0].args == {
+ "a": 2,
+ "b": 8,
+ 'decimal_places': 0,
+ }, f"Error in calling round {i+1}"
+ assert tool_calls[0].result == 16, f"Error in calling round {i+1}"
@pytest.mark.model_backend
@pytest.mark.asyncio
-async def test_tool_calling_math_async():
+async def test_tool_calling_math_async(step_call_count=3):
system_message = BaseMessage(
role_name="assistant",
role_type=RoleType.ASSISTANT,
@@ -562,21 +906,98 @@ async def test_tool_calling_math_async():
meta_dict=dict(),
content="Calculate the result of: 2*8",
)
- agent_response = await agent.step_async(user_msg)
- tool_calls = agent_response.info['tool_calls']
+ model_backend_rsp_tool = ChatCompletion(
+ id='mock_id_123456',
+ choices=[
+ Choice(
+ finish_reason='tool_calls',
+ index=0,
+ logprobs=None,
+ message=ChatCompletionMessage(
+ content=None,
+ refusal=None,
+ role='assistant',
+ audio=None,
+ function_call=None,
+ tool_calls=[
+ ChatCompletionMessageToolCall(
+ id='call_mock_123456',
+ function=Function(
+ arguments='{ \
+ "a": 2, \
+ "b": 8, \
+ "decimal_places": 0 \
+ }',
+ name='multiply',
+ ),
+ type='function',
+ )
+ ],
+ ),
+ )
+ ],
+ created=1730752528,
+ model='gpt-4o-mini-2024-07-18',
+ object='chat.completion',
+ service_tier=None,
+ usage=CompletionUsage(
+ completion_tokens=50, prompt_tokens=152, total_tokens=202
+ ),
+ )
+ model_backend_rsp_tool1 = ChatCompletion(
+ id='mock_id_123456',
+ choices=[
+ Choice(
+ finish_reason='tool_calls',
+ index=0,
+ logprobs=None,
+ message=ChatCompletionMessage(
+ content='The result of ( 2 times 8 ) is ( 16 ).',
+ refusal=None,
+ role='assistant',
+ audio=None,
+ function_call=None,
+ tool_calls=None,
+ ),
+ )
+ ],
+ created=1730752528,
+ model='gpt-4o-mini-2024-07-18',
+ object='chat.completion',
+ service_tier=None,
+ usage=CompletionUsage(
+ completion_tokens=50, prompt_tokens=152, total_tokens=202
+ ),
+ )
+
+ model.run = MagicMock(
+ side_effect=[
+ model_backend_rsp_tool,
+ model_backend_rsp_tool1,
+ ]
+ * step_call_count
+ )
+
+ for i in range(step_call_count):
+ agent_response = await agent.step_async(user_msg)
- assert tool_calls
- assert str(tool_calls[0]).startswith("Function Execution")
+ tool_calls = agent_response.info['tool_calls']
- assert tool_calls[0].func_name == "multiply"
- assert tool_calls[0].args == {'a': 2, 'b': 8, 'decimal_places': 0}
- assert tool_calls[0].result == 16
+ assert (
+ tool_calls[0].func_name == "multiply"
+ ), f"Error in calling round {i+1}"
+ assert tool_calls[0].args == {
+ "a": 2,
+ "b": 8,
+ 'decimal_places': 0,
+ }, f"Error in calling round {i+1}"
+ assert tool_calls[0].result == 16, f"Error in calling round {i+1}"
@pytest.mark.model_backend
@pytest.mark.asyncio
-async def test_tool_calling_async():
+async def test_tool_calling_async(step_call_count=3):
system_message = BaseMessage(
role_name="assistant",
role_type=RoleType.ASSISTANT,
@@ -600,6 +1021,35 @@ async def test_tool_calling_async():
model_platform=ModelPlatformType.OPENAI,
model_type=ModelType.GPT_4O_MINI,
)
+ model_backend_rsp_tool_async = deepcopy(model_backend_rsp_base)
+
+ # Mock tool calling
+ def mock_run_tool_calling_async(*args, **kwargs):
+ # Reset tool_calls at the beginning of each new round of step() call
+ if model.run.call_count % 2 == 1:
+ model_backend_rsp_tool_async.choices[0].message.tool_calls = [
+ ChatCompletionMessageToolCall(
+ id='call_mock_123456',
+ function=Function(
+ arguments='{"second":1}', name='async_sleep'
+ ),
+ type='function',
+ )
+ ]
+
+ # mock the completion of the tool call
+ elif model_backend_rsp_tool_async.choices[0].message.tool_calls:
+ model_backend_rsp_tool_async.choices[0].message.tool_calls.pop(0)
+
+ if (
+ len(model_backend_rsp_tool_async.choices[0].message.tool_calls)
+ == 0
+ ):
+ model_backend_rsp_tool_async.choices[0].message.tool_calls = None
+
+ return model_backend_rsp_tool_async
+
+ model.run = MagicMock(side_effect=mock_run_tool_calling_async)
agent = ChatAgent(
system_message=system_message,
@@ -616,19 +1066,27 @@ async def test_tool_calling_async():
content="Call the async sleep which is specified in function list with"
" 1 second.",
)
- agent_response = await agent.step_async(user_msg)
- tool_calls = agent_response.info['tool_calls']
+ for i in range(step_call_count):
+ agent_response = await agent.step_async(user_msg)
+
+ tool_calls = agent_response.info['tool_calls']
- assert tool_calls
- assert str(tool_calls[0]).startswith("Function Execution")
+ assert tool_calls, f"Error in calling round {i+1}"
+ assert str(tool_calls[0]).startswith(
+ "Function Execution"
+ ), f"Error in calling round {i+1}"
- assert tool_calls[0].func_name == "async_sleep"
- assert tool_calls[0].args == {'second': 1}
- assert tool_calls[0].result == 1
+ assert (
+ tool_calls[0].func_name == "async_sleep"
+ ), f"Error in calling round {i+1}"
+ assert tool_calls[0].args == {
+ 'second': 1
+ }, f"Error in calling round {i+1}"
+ assert tool_calls[0].result == 1, f"Error in calling round {i+1}"
-def test_response_words_termination():
+def test_response_words_termination(step_call_count=3):
system_message = BaseMessage(
role_name="assistant",
role_type=RoleType.ASSISTANT,
@@ -646,14 +1104,21 @@ def test_response_words_termination():
meta_dict=dict(),
content="Just say 'goodbye' once.",
)
- agent_response = agent.step(user_msg)
+ model_backend_rsp = deepcopy(model_backend_rsp_base)
+ model_backend_rsp.choices[0].message.content = "Goodbye."
+ agent.model_backend.run = MagicMock(return_value=model_backend_rsp)
+
+ for i in range(step_call_count):
+ agent_response = agent.step(user_msg)
- assert agent.terminated
- assert agent_response.terminated
- assert "goodbye" in agent_response.info['termination_reasons'][0]
+ assert agent.terminated, f"Error in calling round {i+1}"
+ assert agent_response.terminated, f"Error in calling round {i+1}"
+ assert (
+ "goodbye" in agent_response.info['termination_reasons'][0]
+ ), f"Error in calling round {i+1}"
-def test_chat_agent_vision():
+def test_chat_agent_vision(step_call_count=3):
system_message = BaseMessage(
role_name="assistant",
role_type=RoleType.ASSISTANT,
@@ -688,30 +1153,34 @@ def test_chat_agent_vision():
image_detail="low",
)
# Mock the OpenAI model return value:
- agent.model_backend.run = Mock()
- agent.model_backend.run.return_value = ChatCompletion(
- id="mock_vision_id",
- choices=[
- Choice(
- finish_reason='stop',
- index=0,
- logprobs=None,
- message=ChatCompletionMessage(
- content='Yes.',
- role='assistant',
- function_call=None,
- tool_calls=None,
- ),
- )
- ],
- created=123456,
- model='gpt-4-turbo-2024-04-09',
- object='chat.completion',
- system_fingerprint='fp_5d12056990',
- usage=CompletionUsage(
- completion_tokens=2, prompt_tokens=113, total_tokens=115
- ),
+ agent.model_backend.run = MagicMock(
+ return_value=ChatCompletion(
+ id="mock_vision_id",
+ choices=[
+ Choice(
+ finish_reason='stop',
+ index=0,
+ logprobs=None,
+ message=ChatCompletionMessage(
+ content='Yes.',
+ role='assistant',
+ function_call=None,
+ tool_calls=None,
+ ),
+ )
+ ],
+ created=123456,
+ model='gpt-4-turbo-2024-04-09',
+ object='chat.completion',
+ system_fingerprint='fp_5d12056990',
+ usage=CompletionUsage(
+ completion_tokens=2, prompt_tokens=113, total_tokens=115
+ ),
+ )
)
- agent_response = agent.step(user_msg)
- assert agent_response.msgs[0].content == "Yes."
+ for i in range(step_call_count):
+ agent_response = agent.step(user_msg)
+ assert (
+ agent_response.msgs[0].content == "Yes."
+ ), f"Error in calling round {i+1}"
diff --git a/test/agents/test_embodied_agent.py b/test/agents/test_embodied_agent.py
index a2406660b..c9f0dfa62 100644
--- a/test/agents/test_embodied_agent.py
+++ b/test/agents/test_embodied_agent.py
@@ -12,14 +12,43 @@
# limitations under the License.
# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
import binascii
+from unittest.mock import MagicMock
import pytest
import requests
+from openai.types.chat.chat_completion import Choice
+from openai.types.chat.chat_completion_message import ChatCompletionMessage
+from openai.types.completion_usage import CompletionUsage
from camel.agents import EmbodiedAgent, HuggingFaceToolAgent
from camel.generators import SystemMessageGenerator
from camel.messages import BaseMessage
-from camel.types import RoleType
+from camel.types import ChatCompletion, RoleType
+
+model_backend_rsp_base = ChatCompletion(
+ id="mock_response_id",
+ choices=[
+ Choice(
+ finish_reason="stop",
+ index=0,
+ logprobs=None,
+ message=ChatCompletionMessage(
+ content="This is a mock response content.",
+ role="assistant",
+ function_call=None,
+ tool_calls=None,
+ ),
+ )
+ ],
+ created=123456789,
+ model="gpt-4o-2024-05-13",
+ object="chat.completion",
+ usage=CompletionUsage(
+ completion_tokens=32,
+ prompt_tokens=15,
+ total_tokens=47,
+ ),
+)
@pytest.mark.skip(reason="Wait huggingface to update openaiv1")
@@ -32,15 +61,15 @@ def test_get_action_space_prompt():
role_tuple=(f"{role_name}'s Embodiment", RoleType.EMBODIMENT),
)
agent = EmbodiedAgent(
- sys_msg, tool_agents=[HuggingFaceToolAgent('hugging_face_tool_agent')]
+ sys_msg, tool_agents=[HuggingFaceToolAgent("hugging_face_tool_agent")]
)
- assert 'hugging_face_tool_agent' in agent.get_tool_agent_names()
+ assert "hugging_face_tool_agent" in agent.get_tool_agent_names()
@pytest.mark.skip(reason="Wait huggingface to update openaiv1")
@pytest.mark.model_backend
@pytest.mark.very_slow
-def test_step():
+def test_step(step_call_count=3):
# Create an embodied agent
role_name = "Artist"
meta_dict = dict(role=role_name, task="Drawing")
@@ -49,18 +78,23 @@ def test_step():
role_tuple=(f"{role_name}'s Embodiment", RoleType.EMBODIMENT),
)
embodied_agent = EmbodiedAgent(sys_msg, verbose=True)
+ embodied_agent.model_backend.run = MagicMock(
+ return_value=model_backend_rsp_base
+ )
+
user_msg = BaseMessage.make_user_message(
role_name=role_name,
content="Draw all the Camelidae species.",
)
- try:
- response = embodied_agent.step(user_msg)
- except (binascii.Error, requests.exceptions.ConnectionError) as ex:
- print(
- "Warning: caught an exception, ignoring it since "
- f"it is a known issue of Huggingface ({ex!s})"
- )
- return
- assert isinstance(response.msg, BaseMessage)
- assert not response.terminated
- assert isinstance(response.info, dict)
+ for i in range(step_call_count):
+ try:
+ response = embodied_agent.step(user_msg)
+ except (binascii.Error, requests.exceptions.ConnectionError) as ex:
+ print(
+ "Warning: caught an exception, ignoring it since "
+ f"it is a known issue of Huggingface ({ex!s})"
+ )
+ return
+ assert isinstance(response.msg, BaseMessage), f"Error in round {i+1}"
+ assert not response.terminated, f"Error in round {i+1}"
+ assert isinstance(response.info, dict), f"Error in round {i+1}"
diff --git a/test/agents/test_role_playing.py b/test/agents/test_role_playing.py
index 03ee6730b..5a12808b9 100644
--- a/test/agents/test_role_playing.py
+++ b/test/agents/test_role_playing.py
@@ -11,7 +11,12 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+from unittest.mock import MagicMock
+
import pytest
+from openai.types.chat.chat_completion import Choice
+from openai.types.chat.chat_completion_message import ChatCompletionMessage
+from openai.types.completion_usage import CompletionUsage
from camel.agents import ChatAgent, CriticAgent
from camel.human import Human
@@ -20,6 +25,7 @@ from camel.models import ModelFactory
from camel.societies import RolePlaying
from camel.toolkits import MathToolkit
from camel.types import (
+ ChatCompletion,
ModelPlatformType,
ModelType,
RoleType,
@@ -31,11 +37,39 @@ model = ModelFactory.create(
model_type=ModelType.GPT_4O,
)
+model_backend_rsp = ChatCompletion(
+ id="mock_response_id",
+ choices=[
+ Choice(
+ finish_reason="stop",
+ index=0,
+ logprobs=None,
+ message=ChatCompletionMessage(
+ content="This is a mock response content.",
+ role="assistant",
+ function_call=None,
+ tool_calls=None,
+ ),
+ )
+ ],
+ created=123456789,
+ model="gpt-4o-2024-05-13",
+ object="chat.completion",
+ usage=CompletionUsage(
+ completion_tokens=32,
+ prompt_tokens=15,
+ total_tokens=47,
+ ),
+)
+
@pytest.mark.parametrize("model", [None, model])
@pytest.mark.parametrize("critic_role_name", ["human", "critic agent"])
@pytest.mark.parametrize("with_critic_in_the_loop", [True, False])
def test_role_playing_init(model, critic_role_name, with_critic_in_the_loop):
+ if model is not None:
+ model.run = MagicMock(return_value=model_backend_rsp)
+
role_playing = RolePlaying(
assistant_role_name="assistant",
assistant_agent_kwargs=dict(model=model),
@@ -108,8 +142,14 @@ def test_role_playing_init(model, critic_role_name, with_critic_in_the_loop):
],
)
def test_role_playing_step(
- task_type, extend_sys_msg_meta_dicts, extend_task_specify_meta_dict
+ task_type,
+ extend_sys_msg_meta_dicts,
+ extend_task_specify_meta_dict,
+ step_call_count=3,
):
+ if model is not None:
+ model.run = MagicMock(return_value=model_backend_rsp)
+
role_playing = RolePlaying(
assistant_role_name="AI Assistant",
assistant_agent_kwargs=dict(model=model),
@@ -127,24 +167,38 @@ def test_role_playing_step(
print(role_playing.assistant_agent.system_message)
print(role_playing.user_agent.system_message)
- assistant_response, user_response = role_playing.step(init_assistant_msg)
+ for i in range(step_call_count):
+ assistant_response, user_response = role_playing.step(
+ init_assistant_msg
+ )
- for response in (assistant_response, user_response):
- assert isinstance(response.msgs, list)
- assert len(response.msgs) == 1
- assert isinstance(response.msgs[0], BaseMessage)
- assert isinstance(response.terminated, bool)
- assert response.terminated is False
- assert isinstance(response.info, dict)
+ for response in (assistant_response, user_response):
+ assert isinstance(
+ response.msgs, list
+ ), f"Error in round {i+1}: response.msgs is not a list"
+ assert (
+ len(response.msgs) == 1
+ ), f"Error in round {i+1}: len(response.msgs) is not 1"
+ assert isinstance(
+ response.msgs[0], BaseMessage
+ ), f"Error in round {i+1}: response.msgs[0] is not a BaseMessage"
+ assert isinstance(
+ response.terminated, bool
+ ), f"Error in round {i+1}: response.terminated is not a bool"
+ assert (
+ response.terminated is False
+ ), f"Error in round {i+1}: response.terminated is not False"
+ assert isinstance(
+ response.info, dict
+ ), f"Error in round {i+1}: response.info is not a dict"
@pytest.mark.model_backend
-def test_role_playing_with_function():
+def test_role_playing_with_function(step_call_count=3):
+ if model is not None:
+ model.run = MagicMock(return_value=model_backend_rsp)
+
tools = MathToolkit().get_tools()
- model = ModelFactory.create(
- model_platform=ModelPlatformType.OPENAI,
- model_type=ModelType.GPT_4O_MINI,
- )
role_playing = RolePlaying(
assistant_role_name="AI Assistant",
@@ -160,17 +214,32 @@ def test_role_playing_with_function():
)
input_msg = role_playing.init_chat()
- assistant_response, user_response = role_playing.step(input_msg)
- for response in (assistant_response, user_response):
- assert isinstance(response.msgs, list)
- assert len(response.msgs) == 1
- assert isinstance(response.msgs[0], BaseMessage)
- assert isinstance(response.terminated, bool)
- assert response.terminated is False
- assert isinstance(response.info, dict)
+ for i in range(step_call_count):
+ assistant_response, user_response = role_playing.step(input_msg)
+ for response in (assistant_response, user_response):
+ assert isinstance(
+ response.msgs, list
+ ), f"Error in round {i+1}: response.msgs is not a list"
+ assert (
+ len(response.msgs) == 1
+ ), f"Error in round {i+1}: len(response.msgs) is not 1"
+ assert isinstance(
+ response.msgs[0], BaseMessage
+ ), f"Error in round {i+1}: response.msgs[0] is not a BaseMessage"
+ assert isinstance(
+ response.terminated, bool
+ ), f"Error in round {i+1}: response.terminated is not a bool"
+ assert (
+ response.terminated is False
+ ), f"Error in round {i+1}: response.terminated is not False"
+ assert isinstance(
+ response.info, dict
+ ), f"Error in round {i+1}: response.info is not a dict"
-def test_role_playing_role_sequence(model=None):
+def test_role_playing_role_sequence(
+ model=None,
+):
task_prompt = "Develop a trading bot for the stock market"
role_playing = RolePlaying(
assistant_role_name="Python Programmer",
@@ -181,6 +250,13 @@ def test_role_playing_role_sequence(model=None):
with_task_specify=True,
task_specify_agent_kwargs=dict(model=model),
)
+ role_playing.user_agent.model_backend.run = MagicMock(
+ return_value=model_backend_rsp
+ )
+ role_playing.assistant_agent.model_backend.run = MagicMock(
+ return_value=model_backend_rsp
+ )
+
assistant_role_sequence = []
user_role_sequence = []
@@ -195,16 +271,16 @@ def test_role_playing_role_sequence(model=None):
assistant_role_sequence.append(record["role"])
assert user_role_sequence == [
- 'system',
- 'user',
- 'assistant',
- 'user',
- 'assistant',
+ "system",
+ "user",
+ "assistant",
+ "user",
+ "assistant",
]
assert assistant_role_sequence == [
- 'system',
- 'user',
- 'assistant',
- 'user',
- 'assistant',
+ "system",
+ "user",
+ "assistant",
+ "user",
+ "assistant",
]
diff --git a/test/agents/tool_agents/test_hugging_face_tool_agent.py b/test/agents/tool_agents/test_hugging_face_tool_agent.py
index 230c8e6d2..7d4d23631 100644
--- a/test/agents/tool_agents/test_hugging_face_tool_agent.py
+++ b/test/agents/tool_agents/test_hugging_face_tool_agent.py
@@ -30,19 +30,22 @@ def test_hugging_face_tool_agent_initialization():
@pytest.mark.skip(reason="Wait huggingface to update openaiv1")
@pytest.mark.model_backend
@pytest.mark.very_slow
-def test_hugging_face_tool_agent_step():
+def test_hugging_face_tool_agent_step(step_call_count=3):
from PIL.PngImagePlugin import PngImageFile
agent = HuggingFaceToolAgent("hugging_face_tool_agent")
- try:
- result = agent.step("Generate an image of a boat in the water")
- except (binascii.Error, requests.exceptions.ConnectionError) as ex:
- print(
- "Warning: caught an exception, ignoring it since "
- f"it is a known issue of Huggingface ({ex!s})"
- )
- return
- assert isinstance(result, PngImageFile)
+ for i in range(step_call_count):
+ try:
+ result = agent.step("Generate an image of a boat in the water")
+ except (binascii.Error, requests.exceptions.ConnectionError) as ex:
+ print(
+ "Warning: caught an exception, ignoring it since "
+ f"it is a known issue of Huggingface ({ex!s})"
+ )
+ return
+ assert isinstance(
+ result, PngImageFile
+ ), f"Error in calling round {i+1}"
@pytest.mark.skip(reason="Wait huggingface to update openaiv1")