🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org
Python agentai-societiesartificial-intelligencecommunicative-aicooperative-aideep-learninglarge-language-modelsmulti-agent-systemsnatural-language-processing
chore: Add mock LLM for test purposes; Enable multiple `step()` calls in test units to enhance test robustness (#1128)
Yihan Lin689094538d12
diff --git a/test/agents/test_babyagi_playing.py b/test/agents/test_babyagi_playing.pyindex 64084dd59..fcacb4fed 100644--- a/test/agents/test_babyagi_playing.py+++ b/test/agents/test_babyagi_playing.py@@ -11,19 +11,52 @@# See the License for the specific language governing permissions and# limitations under the License.# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+from copy import deepcopy+from unittest.mock import MagicMock+import pytest+from openai.types.chat.chat_completion import Choice+from openai.types.chat.chat_completion_message import ChatCompletionMessage+from openai.types.completion_usage import CompletionUsagefrom camel.agents import ChatAgent, TaskCreationAgent, TaskPrioritizationAgentfrom camel.messages import BaseMessagefrom camel.models import ModelFactoryfrom camel.societies import BabyAGIfrom camel.types import (+ ChatCompletion,ModelPlatformType,ModelType,RoleType,TaskType,)+model_backend_rsp_base = ChatCompletion(+ id="mock_response_id",+ choices=[+ Choice(+ finish_reason="stop",+ index=0,+ logprobs=None,+ message=ChatCompletionMessage(+ content="Mock task specifier response",+ role="assistant",+ function_call=None,+ tool_calls=None,+ ),+ )+ ],+ created=123456789,+ model="gpt-4o-2024-05-13",+ object="chat.completion",+ usage=CompletionUsage(+ completion_tokens=32,+ prompt_tokens=15,+ total_tokens=47,+ ),+)++parametrize = pytest.mark.parametrize('model',[@@ -66,7 +99,7 @@ def test_babyagi_playing_init(model):@parametrize-def test_babyagi_playing_step(model):+def test_babyagi_playing_step(model, step_call_count=3):task_prompt = "Develop a trading bot for the stock market"babyagi_playing = BabyAGI(@@ -78,18 +111,118 @@ def test_babyagi_playing_step(model):message_window_size=5,)+ # Mock model_backend responses+ # Initial task list when first calling babyagi_playing.step()+ task_creation_agent_model_rsp0 = deepcopy(model_backend_rsp_base)+ task_creation_agent_model_rsp0.choices[0].message.content = "1. Task 0"+ task_prioritization_agent_model_rsp0 = deepcopy(model_backend_rsp_base)+ task_prioritization_agent_model_rsp0.choices[+ 0+ ].message.content = "1. Task 0"++ # Solve the highest priority (left most) task: Task 0+ assistant_agent_model_rsp1 = deepcopy(model_backend_rsp_base)+ assistant_agent_model_rsp1.choices[+ 0+ ].message.content = "Solution for Task 0"++ # Task 0 is solved, remove from TaskPrioritizationAgent model response+ task_creation_agent_model_rsp1 = deepcopy(model_backend_rsp_base)+ task_creation_agent_model_rsp1.choices[+ 0+ ].message.content = "1. \nTask 1\n\n2. Task 2\n\n3. Task 3"+ task_prioritization_agent_model_rsp1 = deepcopy(model_backend_rsp_base)+ task_prioritization_agent_model_rsp1.choices[+ 0+ ].message.content = "1. Task 2 \nTask 1 \n3. Task 3 "++ # Solve the highest priority (left most) task: Task 2+ assistant_agent_model_rsp2 = deepcopy(model_backend_rsp_base)+ assistant_agent_model_rsp2.choices[+ 0+ ].message.content = "Solution for Task 2"++ # Task 2 is solved, remove from TaskPrioritizationAgent model response+ task_creation_agent_model_rsp2 = deepcopy(model_backend_rsp_base)+ task_creation_agent_model_rsp2.choices[+ 0+ ].message.content = "1. Task 4\n\n2. Task 5"+ task_prioritization_agent_model_rsp2 = deepcopy(model_backend_rsp_base)+ task_prioritization_agent_model_rsp2.choices[+ 0+ ].message.content = "1. Task 1 \nTask 3 \n3. Task 4 \n4. Task 5"++ # Solve the highest priority (left most) task: Task 1+ assistant_agent_model_rsp3 = deepcopy(model_backend_rsp_base)+ assistant_agent_model_rsp3.choices[+ 0+ ].message.content = "Solution for Task 1"++ # Task 1 is solved, remove from TaskPrioritizationAgent model response+ task_creation_agent_model_rsp3 = deepcopy(model_backend_rsp_base)+ task_creation_agent_model_rsp3.choices[+ 0+ ].message.content = "1. Task 6\n\n2. Task 7\n\n3. Task 8"+ task_prioritization_agent_model_rsp3 = deepcopy(model_backend_rsp_base)+ task_prioritization_agent_model_rsp3.choices[0].message.content = (+ "1. Task 6 \nTask 3 \n3. Task 4 \n"+ "4. Task 7 \n5. Task 5 \n6. Task 8"+ )++ babyagi_playing.task_creation_agent.model_backend.run = MagicMock(+ side_effect=[+ task_creation_agent_model_rsp0,+ task_creation_agent_model_rsp1,+ task_creation_agent_model_rsp2,+ task_creation_agent_model_rsp3,+ ]+ )+ babyagi_playing.task_prioritization_agent.model_backend.run = MagicMock(+ side_effect=[+ task_prioritization_agent_model_rsp0,+ task_prioritization_agent_model_rsp1,+ task_prioritization_agent_model_rsp2,+ task_prioritization_agent_model_rsp3,+ ]+ )+ babyagi_playing.assistant_agent.model_backend.run = MagicMock(+ side_effect=[+ assistant_agent_model_rsp1,+ assistant_agent_model_rsp2,+ assistant_agent_model_rsp3,+ ]+ )+print(f"AI Assistant sys message:\n{babyagi_playing.assistant_sys_msg}\n")print(f"Original task prompt:\n{task_prompt}\n")print(f"Specified task prompt:\n{babyagi_playing.specified_task_prompt}\n")- assistant_response = babyagi_playing.step()-- assert isinstance(assistant_response.msgs, list)- assert len(assistant_response.msgs) == 1- assert isinstance(assistant_response.msgs[0], BaseMessage)- assert isinstance(assistant_response.terminated, bool)- assert assistant_response.terminated is False- assert isinstance(assistant_response.info, dict)-- assert len(babyagi_playing.subtasks) > 0- assert len(babyagi_playing.solved_subtasks) == 1+ for i in range(step_call_count):+ # Call assistant for multiple times to make test units more robust+ assistant_response = babyagi_playing.step()++ assert isinstance(+ assistant_response.msgs, list+ ), f"Error in calling round {i+1}"+ assert (+ len(assistant_response.msgs) == 1+ ), f"Error in calling round {i+1}"+ assert isinstance(+ assistant_response.msgs[0], BaseMessage+ ), f"Error in calling round {i+1}"+ assert isinstance(+ assistant_response.terminated, bool+ ), f"Error in calling round {i+1}"+ assert (+ assistant_response.terminated is False+ ), f"Error in calling round {i+1}"+ assert isinstance(+ assistant_response.info, dict+ ), f"Error in calling round {i+1}"++ assert (+ len(babyagi_playing.subtasks) > 0+ ), f"Error in calling round {i+1}"+ assert (+ len(babyagi_playing.solved_subtasks) == i + 1+ ), f"Error in calling round {i+1}"diff --git a/test/agents/test_chat_agent.py b/test/agents/test_chat_agent.pyindex cad238abe..870a75100 100644--- a/test/agents/test_chat_agent.py+++ b/test/agents/test_chat_agent.py@@ -13,13 +13,18 @@# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========import asyncioimport json+from copy import deepcopyfrom io import BytesIOfrom typing import List-from unittest.mock import Mock+from unittest.mock import MagicMockimport pytestfrom openai.types.chat.chat_completion import Choicefrom openai.types.chat.chat_completion_message import ChatCompletionMessage+from openai.types.chat.chat_completion_message_tool_call import (+ ChatCompletionMessageToolCall,+ Function,+)from openai.types.completion_usage import CompletionUsagefrom PIL import Imagefrom pydantic import BaseModel, Field@@ -48,6 +53,31 @@ from camel.types import ()from camel.utils.async_func import sync_funcs_to_async+model_backend_rsp_base = ChatCompletion(+ id="mock_response_id",+ choices=[+ Choice(+ finish_reason="stop",+ index=0,+ logprobs=None,+ message=ChatCompletionMessage(+ content="This is a mock response content.",+ role="assistant",+ function_call=None,+ tool_calls=None,+ ),+ )+ ],+ created=123456789,+ model="gpt-4o-2024-05-13",+ object="chat.completion",+ usage=CompletionUsage(+ completion_tokens=32,+ prompt_tokens=15,+ total_tokens=47,+ ),+)+parametrize = pytest.mark.parametrize('model',[@@ -61,7 +91,7 @@ parametrize = pytest.mark.parametrize(@parametrize-def test_chat_agent(model):+def test_chat_agent(model, step_call_count=3):model = modelsystem_msg = SystemMessageGenerator(task_type=TaskType.AI_SOCIETY@@ -82,6 +112,9 @@ def test_chat_agent(model):for assistant in [assistant_with_sys_msg, assistant_without_sys_msg]:assistant.reset()+ assistant.model_backend.run = MagicMock(+ return_value=model_backend_rsp_base+ )user_msg_bm = BaseMessage(role_name="Patient",@@ -93,14 +126,17 @@ def test_chat_agent(model):user_msg_str = "Hello!"for assistant in [assistant_with_sys_msg, assistant_without_sys_msg]:- for user_msg in [user_msg_bm, user_msg_str]:- response = assistant.step(user_msg)- assert isinstance(response.msgs, list)- assert len(response.msgs) > 0- assert isinstance(response.terminated, bool)- assert response.terminated is False- assert isinstance(response.info, dict)- assert response.info['id'] is not None+ for i in range(step_call_count):+ for user_msg in [user_msg_bm, user_msg_str]:+ response = assistant.step(user_msg)+ assert isinstance(response.msgs, list), f"Error in round {i+1}"+ assert len(response.msgs) > 0, f"Error in round {i+1}"+ assert isinstance(+ response.terminated, bool+ ), f"Error in round {i+1}"+ assert response.terminated is False, f"Error in round {i+1}"+ assert isinstance(response.info, dict), f"Error in round {i+1}"+ assert response.info['id'] is not None, f"Error in round {i+1}"@pytest.mark.model_backend@@ -147,7 +183,7 @@ def test_chat_agent_stored_messages():@pytest.mark.model_backend-def test_chat_agent_step_with_structure_response():+def test_chat_agent_step_with_structure_response(step_call_count=3):system_msg = BaseMessage(role_name="assistant",role_type=RoleType.ASSISTANT,@@ -167,26 +203,43 @@ def test_chat_agent_step_with_structure_response():content="Tell a jokes.",)- response = assistant.step(user_msg, response_format=JokeResponse)- response_content_json = json.loads(response.msg.content)- joke_response_keys = set(- JokeResponse.model_json_schema()["properties"].keys()+ model_backend_rsp_tool = deepcopy(model_backend_rsp_base)++ model_backend_rsp_tool.choices[0].message.content = '{ \+ "joke":"What do you call fake spaghetti? An impasta!", \+ "funny_level":"6" \+ }'+ model_backend_rsp_tool.choices[0].message.tool_calls = []+ model_backend_rsp_tool.choices[0].message.parsed = JokeResponse(+ joke="What do you call fake spaghetti? An impasta!", funny_level='6'+ )+ assistant.model_backend.run = MagicMock(+ return_value=model_backend_rsp_tool)- response_content_keys = set(response_content_json.keys())+ for i in range(step_call_count):+ response = assistant.step(user_msg, response_format=JokeResponse)+ response_content_json = json.loads(response.msg.content)+ joke_response_keys = set(+ JokeResponse.model_json_schema()["properties"].keys()+ )- assert joke_response_keys.issubset(- response_content_keys- ), f"Missing keys: {joke_response_keys - response_content_keys}"+ response_content_keys = set(response_content_json.keys())- for key in joke_response_keys:- assert (- key in response_content_json- ), f"Key {key} not found in response content"+ assert joke_response_keys.issubset(response_content_keys), (+ f"Error in calling round {i+1}: "+ f"Missing keys: {joke_response_keys - response_content_keys}"+ )++ for key in joke_response_keys:+ assert key in response_content_json, (+ f"Error in calling round {i+1}: "+ f"Key {key} not found in response content"+ )@pytest.mark.model_backend-def test_chat_agent_step_with_external_tools():+def test_chat_agent_step_with_external_tools(step_call_count=3):internal_tools = [FunctionTool(SearchToolkit().search_duckduckgo)]external_tools = MathToolkit().get_tools()tool_list = internal_tools + external_tools@@ -201,6 +254,91 @@ def test_chat_agent_step_with_external_tools():model_type=ModelType.GPT_4O_MINI,model_config_dict=model_config_dict,)+ model_backend_external1 = ChatCompletion(+ id='mock_id_123456',+ choices=[+ Choice(+ finish_reason='tool_calls',+ index=0,+ logprobs=None,+ message=ChatCompletionMessage(+ content=None,+ refusal=None,+ role='assistant',+ audio=None,+ function_call=None,+ tool_calls=[+ ChatCompletionMessageToolCall(+ id='call_mock_123456',+ function=Function(+ arguments='{ \+ "query": "Portal game release year" \+ }',+ name='search_duckduckgo',+ ),+ type='function',+ ),+ ChatCompletionMessageToolCall(+ id='call_mock_123457',+ function=Function(+ arguments='{ \+ "query": "United States founding year" \+ }',+ name='search_duckduckgo',+ ),+ type='function',+ ),+ ],+ ),+ )+ ],+ created=1730745899,+ model='gpt-4o-mini-2024-07-18',+ object='chat.completion',+ service_tier=None,+ usage=CompletionUsage(+ completion_tokens=56, prompt_tokens=292, total_tokens=348+ ),+ )++ model_backend_external2 = ChatCompletion(+ id='chatcmpl-mock_id_123456',+ choices=[+ Choice(+ finish_reason='tool_calls',+ index=0,+ logprobs=None,+ message=ChatCompletionMessage(+ content=None,+ refusal=None,+ role='assistant',+ audio=None,+ function_call=None,+ tool_calls=[+ ChatCompletionMessageToolCall(+ id='call_mock_123456',+ function=Function(+ arguments='{"a":1776,"b":2007}', name='sub'+ ),+ type='function',+ )+ ],+ ),+ )+ ],+ created=1730745902,+ model='gpt-4o-mini-2024-07-18',+ object='chat.completion',+ service_tier=None,+ usage=CompletionUsage(+ completion_tokens=19, prompt_tokens=991, total_tokens=1010+ ),+ )++ model.run = MagicMock(+ side_effect=[model_backend_external1, model_backend_external2]+ * step_call_count+ )# Set external_toolsexternal_tool_agent = ChatAgent(@@ -219,11 +357,14 @@ def test_chat_agent_step_with_external_tools():"from the year that United States was founded?",)- response = external_tool_agent.step(usr_msg)- assert not response.msg.content+ for i in range(step_call_count):+ response = external_tool_agent.step(usr_msg)+ assert not response.msg.content- external_tool_request = response.info["external_tool_request"]- assert external_tool_request.function.name == "sub"+ external_tool_request = response.info["external_tool_request"]+ assert (+ external_tool_request.function.name == "sub"+ ), f"Error in calling round {i+1}"@pytest.mark.model_backend@@ -260,7 +401,7 @@ def test_chat_agent_messages_window():@pytest.mark.model_backend-def test_chat_agent_step_exceed_token_number():+def test_chat_agent_step_exceed_token_number(step_call_count=3):system_msg = BaseMessage(role_name="assistant",role_type=RoleType.ASSISTANT,@@ -271,6 +412,9 @@ def test_chat_agent_step_exceed_token_number():system_message=system_msg,token_limit=1,)+ assistant.model_backend.run = MagicMock(+ return_value=model_backend_rsp_base+ )user_msg = BaseMessage(role_name="User",@@ -279,20 +423,61 @@ def test_chat_agent_step_exceed_token_number():content="Tell me a joke.",)- response = assistant.step(user_msg)- assert len(response.msgs) == 0- assert response.terminated+ for i in range(step_call_count):+ response = assistant.step(user_msg)+ assert len(response.msgs) == 0, f"Error in calling round {i+1}"+ assert response.terminated, f"Error in calling round {i+1}"@pytest.mark.model_backend@pytest.mark.parametrize('n', [1, 2, 3])-def test_chat_agent_multiple_return_messages(n):+def test_chat_agent_multiple_return_messages(n, step_call_count=3):model_config = ChatGPTConfig(temperature=1.4, n=n)model = ModelFactory.create(model_platform=ModelPlatformType.OPENAI,model_type=ModelType.GPT_4O_MINI,model_config_dict=model_config.as_dict(),)+ model_backend_rsp_tool = ChatCompletion(+ id="mock_response_id",+ choices=[+ Choice(+ finish_reason="stop",+ index=0,+ logprobs=None,+ message=ChatCompletionMessage(+ content="What do you call fake spaghetti? An impasta!",+ role="assistant",+ function_call=None,+ tool_calls=[+ ChatCompletionMessageToolCall(+ id="call_mock123456",+ function=Function(+ arguments='{ \+ "joke":"What do you call fake spaghetti?" \+ " An impasta!", \+ "funny_level":"6" \+ }',+ name="return_json_response",+ ),+ type="function",+ )+ ],+ ),+ )+ ]+ * n,+ created=123456789,+ model="gpt-4o-2024-05-13",+ object="chat.completion",+ usage=CompletionUsage(+ completion_tokens=32,+ prompt_tokens=15,+ total_tokens=47,+ ),+ )+ model.run = MagicMock(return_value=model_backend_rsp_tool)+system_msg = BaseMessage("Assistant",RoleType.ASSISTANT,@@ -316,21 +501,49 @@ def test_chat_agent_multiple_return_messages(n):user_msg)- assert assistant_with_sys_msg_response.msgs is not None- assert len(assistant_with_sys_msg_response.msgs) == n- assert assistant_without_sys_msg_response.msgs is not None- assert len(assistant_without_sys_msg_response.msgs) == n+ for i in range(step_call_count):+ assert (+ assistant_with_sys_msg_response.msgs is not None+ ), f"Error in calling round {i+1}"+ assert (+ len(assistant_with_sys_msg_response.msgs) == n+ ), f"Error in calling round {i+1}"+ assert (+ assistant_without_sys_msg_response.msgs is not None+ ), f"Error in calling round {i+1}"+ assert (+ len(assistant_without_sys_msg_response.msgs) == n+ ), f"Error in calling round {i+1}"@pytest.mark.model_backend@pytest.mark.parametrize('n', [2])-def test_chat_agent_multiple_return_message_error(n):+def test_chat_agent_multiple_return_message_error(n, step_call_count=3):model_config = ChatGPTConfig(temperature=1.4, n=n)model = ModelFactory.create(model_platform=ModelPlatformType.OPENAI,model_type=ModelType.GPT_4O_MINI,model_config_dict=model_config.as_dict(),)+ model_backend_multi_messages = deepcopy(model_backend_rsp_base)+ model_backend_multi_messages.choices.append(+ Choice(+ finish_reason='stop',+ index=1,+ logprobs=None,+ message=ChatCompletionMessage(+ content='Why did the scarecrow win an award? '+ 'Because he was outstanding in his field!',+ refusal=None,+ role='assistant',+ audio=None,+ function_call=None,+ tool_calls=None,+ ),+ )+ )+ model.run = MagicMock(return_value=model_backend_multi_messages)+system_msg = BaseMessage("Assistant",RoleType.ASSISTANT,@@ -347,19 +560,21 @@ def test_chat_agent_multiple_return_message_error(n):meta_dict=dict(),content="Tell me a joke.",)- assistant_response = assistant.step(user_msg)+ for _ in range(step_call_count):+ assistant_response = assistant.step(user_msg)- with pytest.raises(- RuntimeError,- match=(- "Property msg is only available " "for a single message in msgs."- ),- ):- _ = assistant_response.msg+ with pytest.raises(+ RuntimeError,+ match=(+ "Property msg is only available "+ "for a single message in msgs."+ ),+ ):+ _ = assistant_response.msg@pytest.mark.model_backend-def test_chat_agent_stream_output():+def test_chat_agent_stream_output(step_call_count=3):system_msg = BaseMessage("Assistant",RoleType.ASSISTANT,@@ -379,20 +594,27 @@ def test_chat_agent_stream_output():model_type=ModelType.GPT_4O_MINI,model_config_dict=stream_model_config.as_dict(),)+ model.run = MagicMock(return_value=model_backend_rsp_base)stream_assistant = ChatAgent(system_msg, model=model)stream_assistant.reset()- stream_assistant_response = stream_assistant.step(user_msg)+ for i in range(step_call_count):+ stream_assistant_response = stream_assistant.step(user_msg)- for msg in stream_assistant_response.msgs:- assert len(msg.content) > 0+ for msg in stream_assistant_response.msgs:+ assert len(msg.content) > 0, f"Error in calling round {i+1}"- stream_usage = stream_assistant_response.info["usage"]- assert stream_usage["completion_tokens"] > 0- assert stream_usage["prompt_tokens"] > 0- assert (- stream_usage["total_tokens"]- == stream_usage["completion_tokens"] + stream_usage["prompt_tokens"]- )+ stream_usage = stream_assistant_response.info["usage"]+ assert (+ stream_usage["completion_tokens"] > 0+ ), f"Error in calling round {i+1}"+ assert (+ stream_usage["prompt_tokens"] > 0+ ), f"Error in calling round {i+1}"+ assert (+ stream_usage["total_tokens"]+ == stream_usage["completion_tokens"]+ + stream_usage["prompt_tokens"]+ ), f"Error in calling round {i+1}"@pytest.mark.model_backend@@ -491,7 +713,7 @@ def test_function_enabled():@pytest.mark.model_backend-def test_tool_calling_sync():+def test_tool_calling_sync(step_call_count=3):system_message = BaseMessage(role_name="assistant",role_type=RoleType.ASSISTANT,@@ -518,23 +740,145 @@ def test_tool_calling_sync():meta_dict=dict(),content="Calculate the result of: 2*8-10.",)- agent_response = agent.step(user_msg)+ model_backend_rsp_tool = ChatCompletion(+ id='mock_id_123456',+ choices=[+ Choice(+ finish_reason='tool_calls',+ index=0,+ logprobs=None,+ message=ChatCompletionMessage(+ content=None,+ refusal=None,+ role='assistant',+ audio=None,+ function_call=None,+ tool_calls=[+ ChatCompletionMessageToolCall(+ id='call_mock_123456',+ function=Function(+ arguments='{ \+ "a": 2, \+ "b": 8, \+ "decimal_places": 0 \+ }',+ name='multiply',+ ),+ type='function',+ ),+ ChatCompletionMessageToolCall(+ id='call_mock_123457',+ function=Function(+ arguments='{"a": 0, "b": 10}', name='sub'+ ),+ type='function',+ ),+ ],+ ),+ )+ ],+ created=1730752528,+ model='gpt-4o-mini-2024-07-18',+ object='chat.completion',+ service_tier=None,+ usage=CompletionUsage(+ completion_tokens=50, prompt_tokens=152, total_tokens=202+ ),+ )+ model_backend_rsp_tool1 = ChatCompletion(+ id='mock_id_123456',+ choices=[+ Choice(+ finish_reason='tool_calls',+ index=0,+ logprobs=None,+ message=ChatCompletionMessage(+ content=None,+ refusal=None,+ role='assistant',+ audio=None,+ function_call=None,+ tool_calls=[+ ChatCompletionMessageToolCall(+ id='call_kIJby7Y6As6gAbWoxDqxD6HG',+ function=Function(+ arguments='{"a":16,"b":10}', name='sub'+ ),+ type='function',+ )+ ],+ ),+ )+ ],+ created=1730752528,+ model='gpt-4o-mini-2024-07-18',+ object='chat.completion',+ service_tier=None,+ usage=CompletionUsage(+ completion_tokens=50, prompt_tokens=152, total_tokens=202+ ),+ )+ model_backend_rsp_tool2 = ChatCompletion(+ id='mock_id_123456',+ choices=[+ Choice(+ finish_reason='tool_calls',+ index=0,+ logprobs=None,+ message=ChatCompletionMessage(+ content='The result of the calculation \+ (2 times 8 - 10) is (6).',+ refusal=None,+ role='assistant',+ audio=None,+ function_call=None,+ tool_calls=None,+ ),+ )+ ],+ created=1730752528,+ model='gpt-4o-mini-2024-07-18',+ object='chat.completion',+ service_tier=None,+ usage=CompletionUsage(+ completion_tokens=50, prompt_tokens=152, total_tokens=202+ ),+ )- tool_calls: List[FunctionCallingRecord] = [- call for call in agent_response.info['tool_calls']- ]+ model.run = MagicMock(+ side_effect=[+ model_backend_rsp_tool,+ model_backend_rsp_tool1,+ model_backend_rsp_tool2,+ ]+ * step_call_count+ )- assert len(tool_calls) > 0- assert str(tool_calls[0]).startswith("Function Execution")+ for i in range(step_call_count):+ agent_response = agent.step(user_msg)- assert tool_calls[0].func_name == "multiply"- assert tool_calls[0].args == {'a': 2, 'b': 8, 'decimal_places': 0}- assert tool_calls[0].result == 16+ tool_calls: List[FunctionCallingRecord] = [+ call for call in agent_response.info['tool_calls']+ ]++ assert len(tool_calls) > 0, f"Error in calling round {i+1}"+ assert str(tool_calls[0]).startswith(+ "Function Execution"+ ), f"Error in calling round {i+1}"+ assert (+ tool_calls[0].func_name == "multiply"+ ), f"Error in calling round {i+1}"+ assert tool_calls[0].args == {+ "a": 2,+ "b": 8,+ 'decimal_places': 0,+ }, f"Error in calling round {i+1}"+ assert tool_calls[0].result == 16, f"Error in calling round {i+1}"@pytest.mark.model_backend@pytest.mark.asyncio-async def test_tool_calling_math_async():+async def test_tool_calling_math_async(step_call_count=3):system_message = BaseMessage(role_name="assistant",role_type=RoleType.ASSISTANT,@@ -562,21 +906,98 @@ async def test_tool_calling_math_async():meta_dict=dict(),content="Calculate the result of: 2*8",)- agent_response = await agent.step_async(user_msg)- tool_calls = agent_response.info['tool_calls']+ model_backend_rsp_tool = ChatCompletion(+ id='mock_id_123456',+ choices=[+ Choice(+ finish_reason='tool_calls',+ index=0,+ logprobs=None,+ message=ChatCompletionMessage(+ content=None,+ refusal=None,+ role='assistant',+ audio=None,+ function_call=None,+ tool_calls=[+ ChatCompletionMessageToolCall(+ id='call_mock_123456',+ function=Function(+ arguments='{ \+ "a": 2, \+ "b": 8, \+ "decimal_places": 0 \+ }',+ name='multiply',+ ),+ type='function',+ )+ ],+ ),+ )+ ],+ created=1730752528,+ model='gpt-4o-mini-2024-07-18',+ object='chat.completion',+ service_tier=None,+ usage=CompletionUsage(+ completion_tokens=50, prompt_tokens=152, total_tokens=202+ ),+ )+ model_backend_rsp_tool1 = ChatCompletion(+ id='mock_id_123456',+ choices=[+ Choice(+ finish_reason='tool_calls',+ index=0,+ logprobs=None,+ message=ChatCompletionMessage(+ content='The result of ( 2 times 8 ) is ( 16 ).',+ refusal=None,+ role='assistant',+ audio=None,+ function_call=None,+ tool_calls=None,+ ),+ )+ ],+ created=1730752528,+ model='gpt-4o-mini-2024-07-18',+ object='chat.completion',+ service_tier=None,+ usage=CompletionUsage(+ completion_tokens=50, prompt_tokens=152, total_tokens=202+ ),+ )++ model.run = MagicMock(+ side_effect=[+ model_backend_rsp_tool,+ model_backend_rsp_tool1,+ ]+ * step_call_count+ )++ for i in range(step_call_count):+ agent_response = await agent.step_async(user_msg)- assert tool_calls- assert str(tool_calls[0]).startswith("Function Execution")+ tool_calls = agent_response.info['tool_calls']- assert tool_calls[0].func_name == "multiply"- assert tool_calls[0].args == {'a': 2, 'b': 8, 'decimal_places': 0}- assert tool_calls[0].result == 16+ assert (+ tool_calls[0].func_name == "multiply"+ ), f"Error in calling round {i+1}"+ assert tool_calls[0].args == {+ "a": 2,+ "b": 8,+ 'decimal_places': 0,+ }, f"Error in calling round {i+1}"+ assert tool_calls[0].result == 16, f"Error in calling round {i+1}"@pytest.mark.model_backend@pytest.mark.asyncio-async def test_tool_calling_async():+async def test_tool_calling_async(step_call_count=3):system_message = BaseMessage(role_name="assistant",role_type=RoleType.ASSISTANT,@@ -600,6 +1021,35 @@ async def test_tool_calling_async():model_platform=ModelPlatformType.OPENAI,model_type=ModelType.GPT_4O_MINI,)+ model_backend_rsp_tool_async = deepcopy(model_backend_rsp_base)++ # Mock tool calling+ def mock_run_tool_calling_async(*args, **kwargs):+ # Reset tool_calls at the beginning of each new round of step() call+ if model.run.call_count % 2 == 1:+ model_backend_rsp_tool_async.choices[0].message.tool_calls = [+ ChatCompletionMessageToolCall(+ id='call_mock_123456',+ function=Function(+ arguments='{"second":1}', name='async_sleep'+ ),+ type='function',+ )+ ]++ # mock the completion of the tool call+ elif model_backend_rsp_tool_async.choices[0].message.tool_calls:+ model_backend_rsp_tool_async.choices[0].message.tool_calls.pop(0)++ if (+ len(model_backend_rsp_tool_async.choices[0].message.tool_calls)+ == 0+ ):+ model_backend_rsp_tool_async.choices[0].message.tool_calls = None++ return model_backend_rsp_tool_async++ model.run = MagicMock(side_effect=mock_run_tool_calling_async)agent = ChatAgent(system_message=system_message,@@ -616,19 +1066,27 @@ async def test_tool_calling_async():content="Call the async sleep which is specified in function list with"" 1 second.",)- agent_response = await agent.step_async(user_msg)- tool_calls = agent_response.info['tool_calls']+ for i in range(step_call_count):+ agent_response = await agent.step_async(user_msg)++ tool_calls = agent_response.info['tool_calls']- assert tool_calls- assert str(tool_calls[0]).startswith("Function Execution")+ assert tool_calls, f"Error in calling round {i+1}"+ assert str(tool_calls[0]).startswith(+ "Function Execution"+ ), f"Error in calling round {i+1}"- assert tool_calls[0].func_name == "async_sleep"- assert tool_calls[0].args == {'second': 1}- assert tool_calls[0].result == 1+ assert (+ tool_calls[0].func_name == "async_sleep"+ ), f"Error in calling round {i+1}"+ assert tool_calls[0].args == {+ 'second': 1+ }, f"Error in calling round {i+1}"+ assert tool_calls[0].result == 1, f"Error in calling round {i+1}"-def test_response_words_termination():+def test_response_words_termination(step_call_count=3):system_message = BaseMessage(role_name="assistant",role_type=RoleType.ASSISTANT,@@ -646,14 +1104,21 @@ def test_response_words_termination():meta_dict=dict(),content="Just say 'goodbye' once.",)- agent_response = agent.step(user_msg)+ model_backend_rsp = deepcopy(model_backend_rsp_base)+ model_backend_rsp.choices[0].message.content = "Goodbye."+ agent.model_backend.run = MagicMock(return_value=model_backend_rsp)++ for i in range(step_call_count):+ agent_response = agent.step(user_msg)- assert agent.terminated- assert agent_response.terminated- assert "goodbye" in agent_response.info['termination_reasons'][0]+ assert agent.terminated, f"Error in calling round {i+1}"+ assert agent_response.terminated, f"Error in calling round {i+1}"+ assert (+ "goodbye" in agent_response.info['termination_reasons'][0]+ ), f"Error in calling round {i+1}"-def test_chat_agent_vision():+def test_chat_agent_vision(step_call_count=3):system_message = BaseMessage(role_name="assistant",role_type=RoleType.ASSISTANT,@@ -688,30 +1153,34 @@ def test_chat_agent_vision():image_detail="low",)# Mock the OpenAI model return value:- agent.model_backend.run = Mock()- agent.model_backend.run.return_value = ChatCompletion(- id="mock_vision_id",- choices=[- Choice(- finish_reason='stop',- index=0,- logprobs=None,- message=ChatCompletionMessage(- content='Yes.',- role='assistant',- function_call=None,- tool_calls=None,- ),- )- ],- created=123456,- model='gpt-4-turbo-2024-04-09',- object='chat.completion',- system_fingerprint='fp_5d12056990',- usage=CompletionUsage(- completion_tokens=2, prompt_tokens=113, total_tokens=115- ),+ agent.model_backend.run = MagicMock(+ return_value=ChatCompletion(+ id="mock_vision_id",+ choices=[+ Choice(+ finish_reason='stop',+ index=0,+ logprobs=None,+ message=ChatCompletionMessage(+ content='Yes.',+ role='assistant',+ function_call=None,+ tool_calls=None,+ ),+ )+ ],+ created=123456,+ model='gpt-4-turbo-2024-04-09',+ object='chat.completion',+ system_fingerprint='fp_5d12056990',+ usage=CompletionUsage(+ completion_tokens=2, prompt_tokens=113, total_tokens=115+ ),+ ))- agent_response = agent.step(user_msg)- assert agent_response.msgs[0].content == "Yes."+ for i in range(step_call_count):+ agent_response = agent.step(user_msg)+ assert (+ agent_response.msgs[0].content == "Yes."+ ), f"Error in calling round {i+1}"diff --git a/test/agents/test_embodied_agent.py b/test/agents/test_embodied_agent.pyindex a2406660b..c9f0dfa62 100644--- a/test/agents/test_embodied_agent.py+++ b/test/agents/test_embodied_agent.py@@ -12,14 +12,43 @@# limitations under the License.# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========import binascii+from unittest.mock import MagicMockimport pytestimport requests+from openai.types.chat.chat_completion import Choice+from openai.types.chat.chat_completion_message import ChatCompletionMessage+from openai.types.completion_usage import CompletionUsagefrom camel.agents import EmbodiedAgent, HuggingFaceToolAgentfrom camel.generators import SystemMessageGeneratorfrom camel.messages import BaseMessage-from camel.types import RoleType+from camel.types import ChatCompletion, RoleType++model_backend_rsp_base = ChatCompletion(+ id="mock_response_id",+ choices=[+ Choice(+ finish_reason="stop",+ index=0,+ logprobs=None,+ message=ChatCompletionMessage(+ content="This is a mock response content.",+ role="assistant",+ function_call=None,+ tool_calls=None,+ ),+ )+ ],+ created=123456789,+ model="gpt-4o-2024-05-13",+ object="chat.completion",+ usage=CompletionUsage(+ completion_tokens=32,+ prompt_tokens=15,+ total_tokens=47,+ ),+)@pytest.mark.skip(reason="Wait huggingface to update openaiv1")@@ -32,15 +61,15 @@ def test_get_action_space_prompt():role_tuple=(f"{role_name}'s Embodiment", RoleType.EMBODIMENT),)agent = EmbodiedAgent(- sys_msg, tool_agents=[HuggingFaceToolAgent('hugging_face_tool_agent')]+ sys_msg, tool_agents=[HuggingFaceToolAgent("hugging_face_tool_agent")])- assert 'hugging_face_tool_agent' in agent.get_tool_agent_names()+ assert "hugging_face_tool_agent" in agent.get_tool_agent_names()@pytest.mark.skip(reason="Wait huggingface to update openaiv1")@pytest.mark.model_backend@pytest.mark.very_slow-def test_step():+def test_step(step_call_count=3):# Create an embodied agentrole_name = "Artist"meta_dict = dict(role=role_name, task="Drawing")@@ -49,18 +78,23 @@ def test_step():role_tuple=(f"{role_name}'s Embodiment", RoleType.EMBODIMENT),)embodied_agent = EmbodiedAgent(sys_msg, verbose=True)+ embodied_agent.model_backend.run = MagicMock(+ return_value=model_backend_rsp_base+ )+user_msg = BaseMessage.make_user_message(role_name=role_name,content="Draw all the Camelidae species.",)- try:- response = embodied_agent.step(user_msg)- except (binascii.Error, requests.exceptions.ConnectionError) as ex:- print(- "Warning: caught an exception, ignoring it since "- f"it is a known issue of Huggingface ({ex!s})"- )- return- assert isinstance(response.msg, BaseMessage)- assert not response.terminated- assert isinstance(response.info, dict)+ for i in range(step_call_count):+ try:+ response = embodied_agent.step(user_msg)+ except (binascii.Error, requests.exceptions.ConnectionError) as ex:+ print(+ "Warning: caught an exception, ignoring it since "+ f"it is a known issue of Huggingface ({ex!s})"+ )+ return+ assert isinstance(response.msg, BaseMessage), f"Error in round {i+1}"+ assert not response.terminated, f"Error in round {i+1}"+ assert isinstance(response.info, dict), f"Error in round {i+1}"diff --git a/test/agents/test_role_playing.py b/test/agents/test_role_playing.pyindex 03ee6730b..5a12808b9 100644--- a/test/agents/test_role_playing.py+++ b/test/agents/test_role_playing.py@@ -11,7 +11,12 @@# See the License for the specific language governing permissions and# limitations under the License.# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+from unittest.mock import MagicMock+import pytest+from openai.types.chat.chat_completion import Choice+from openai.types.chat.chat_completion_message import ChatCompletionMessage+from openai.types.completion_usage import CompletionUsagefrom camel.agents import ChatAgent, CriticAgentfrom camel.human import Human@@ -20,6 +25,7 @@ from camel.models import ModelFactoryfrom camel.societies import RolePlayingfrom camel.toolkits import MathToolkitfrom camel.types import (+ ChatCompletion,ModelPlatformType,ModelType,RoleType,@@ -31,11 +37,39 @@ model = ModelFactory.create(model_type=ModelType.GPT_4O,)+model_backend_rsp = ChatCompletion(+ id="mock_response_id",+ choices=[+ Choice(+ finish_reason="stop",+ index=0,+ logprobs=None,+ message=ChatCompletionMessage(+ content="This is a mock response content.",+ role="assistant",+ function_call=None,+ tool_calls=None,+ ),+ )+ ],+ created=123456789,+ model="gpt-4o-2024-05-13",+ object="chat.completion",+ usage=CompletionUsage(+ completion_tokens=32,+ prompt_tokens=15,+ total_tokens=47,+ ),+)+@pytest.mark.parametrize("model", [None, model])@pytest.mark.parametrize("critic_role_name", ["human", "critic agent"])@pytest.mark.parametrize("with_critic_in_the_loop", [True, False])def test_role_playing_init(model, critic_role_name, with_critic_in_the_loop):+ if model is not None:+ model.run = MagicMock(return_value=model_backend_rsp)+role_playing = RolePlaying(assistant_role_name="assistant",assistant_agent_kwargs=dict(model=model),@@ -108,8 +142,14 @@ def test_role_playing_init(model, critic_role_name, with_critic_in_the_loop):],)def test_role_playing_step(- task_type, extend_sys_msg_meta_dicts, extend_task_specify_meta_dict+ task_type,+ extend_sys_msg_meta_dicts,+ extend_task_specify_meta_dict,+ step_call_count=3,):+ if model is not None:+ model.run = MagicMock(return_value=model_backend_rsp)+role_playing = RolePlaying(assistant_role_name="AI Assistant",assistant_agent_kwargs=dict(model=model),@@ -127,24 +167,38 @@ def test_role_playing_step(print(role_playing.assistant_agent.system_message)print(role_playing.user_agent.system_message)- assistant_response, user_response = role_playing.step(init_assistant_msg)+ for i in range(step_call_count):+ assistant_response, user_response = role_playing.step(+ init_assistant_msg+ )- for response in (assistant_response, user_response):- assert isinstance(response.msgs, list)- assert len(response.msgs) == 1- assert isinstance(response.msgs[0], BaseMessage)- assert isinstance(response.terminated, bool)- assert response.terminated is False- assert isinstance(response.info, dict)+ for response in (assistant_response, user_response):+ assert isinstance(+ response.msgs, list+ ), f"Error in round {i+1}: response.msgs is not a list"+ assert (+ len(response.msgs) == 1+ ), f"Error in round {i+1}: len(response.msgs) is not 1"+ assert isinstance(+ response.msgs[0], BaseMessage+ ), f"Error in round {i+1}: response.msgs[0] is not a BaseMessage"+ assert isinstance(+ response.terminated, bool+ ), f"Error in round {i+1}: response.terminated is not a bool"+ assert (+ response.terminated is False+ ), f"Error in round {i+1}: response.terminated is not False"+ assert isinstance(+ response.info, dict+ ), f"Error in round {i+1}: response.info is not a dict"@pytest.mark.model_backend-def test_role_playing_with_function():+def test_role_playing_with_function(step_call_count=3):+ if model is not None:+ model.run = MagicMock(return_value=model_backend_rsp)+tools = MathToolkit().get_tools()- model = ModelFactory.create(- model_platform=ModelPlatformType.OPENAI,- model_type=ModelType.GPT_4O_MINI,- )role_playing = RolePlaying(assistant_role_name="AI Assistant",@@ -160,17 +214,32 @@ def test_role_playing_with_function():)input_msg = role_playing.init_chat()- assistant_response, user_response = role_playing.step(input_msg)- for response in (assistant_response, user_response):- assert isinstance(response.msgs, list)- assert len(response.msgs) == 1- assert isinstance(response.msgs[0], BaseMessage)- assert isinstance(response.terminated, bool)- assert response.terminated is False- assert isinstance(response.info, dict)+ for i in range(step_call_count):+ assistant_response, user_response = role_playing.step(input_msg)+ for response in (assistant_response, user_response):+ assert isinstance(+ response.msgs, list+ ), f"Error in round {i+1}: response.msgs is not a list"+ assert (+ len(response.msgs) == 1+ ), f"Error in round {i+1}: len(response.msgs) is not 1"+ assert isinstance(+ response.msgs[0], BaseMessage+ ), f"Error in round {i+1}: response.msgs[0] is not a BaseMessage"+ assert isinstance(+ response.terminated, bool+ ), f"Error in round {i+1}: response.terminated is not a bool"+ assert (+ response.terminated is False+ ), f"Error in round {i+1}: response.terminated is not False"+ assert isinstance(+ response.info, dict+ ), f"Error in round {i+1}: response.info is not a dict"-def test_role_playing_role_sequence(model=None):+def test_role_playing_role_sequence(+ model=None,+):task_prompt = "Develop a trading bot for the stock market"role_playing = RolePlaying(assistant_role_name="Python Programmer",@@ -181,6 +250,13 @@ def test_role_playing_role_sequence(model=None):with_task_specify=True,task_specify_agent_kwargs=dict(model=model),)+ role_playing.user_agent.model_backend.run = MagicMock(+ return_value=model_backend_rsp+ )+ role_playing.assistant_agent.model_backend.run = MagicMock(+ return_value=model_backend_rsp+ )+assistant_role_sequence = []user_role_sequence = []@@ -195,16 +271,16 @@ def test_role_playing_role_sequence(model=None):assistant_role_sequence.append(record["role"])assert user_role_sequence == [- 'system',- 'user',- 'assistant',- 'user',- 'assistant',+ "system",+ "user",+ "assistant",+ "user",+ "assistant",]assert assistant_role_sequence == [- 'system',- 'user',- 'assistant',- 'user',- 'assistant',+ "system",+ "user",+ "assistant",+ "user",+ "assistant",]diff --git a/test/agents/tool_agents/test_hugging_face_tool_agent.py b/test/agents/tool_agents/test_hugging_face_tool_agent.pyindex 230c8e6d2..7d4d23631 100644--- a/test/agents/tool_agents/test_hugging_face_tool_agent.py+++ b/test/agents/tool_agents/test_hugging_face_tool_agent.py@@ -30,19 +30,22 @@ def test_hugging_face_tool_agent_initialization():@pytest.mark.skip(reason="Wait huggingface to update openaiv1")@pytest.mark.model_backend@pytest.mark.very_slow-def test_hugging_face_tool_agent_step():+def test_hugging_face_tool_agent_step(step_call_count=3):from PIL.PngImagePlugin import PngImageFileagent = HuggingFaceToolAgent("hugging_face_tool_agent")- try:- result = agent.step("Generate an image of a boat in the water")- except (binascii.Error, requests.exceptions.ConnectionError) as ex:- print(- "Warning: caught an exception, ignoring it since "- f"it is a known issue of Huggingface ({ex!s})"- )- return- assert isinstance(result, PngImageFile)+ for i in range(step_call_count):+ try:+ result = agent.step("Generate an image of a boat in the water")+ except (binascii.Error, requests.exceptions.ConnectionError) as ex:+ print(+ "Warning: caught an exception, ignoring it since "+ f"it is a known issue of Huggingface ({ex!s})"+ )+ return+ assert isinstance(+ result, PngImageFile+ ), f"Error in calling round {i+1}"@pytest.mark.skip(reason="Wait huggingface to update openaiv1")