🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org
Python agentai-societiesartificial-intelligencecommunicative-aicooperative-aideep-learninglarge-language-modelsmulti-agent-systemsnatural-language-processing
fix: image handling in ChatAgent (#2899)
Wendong-Fan39bb285bfbd2
diff --git a/camel/agents/chat_agent.py b/camel/agents/chat_agent.pyindex cd01a855b..ecd0a78cf 100644--- a/camel/agents/chat_agent.py+++ b/camel/agents/chat_agent.py@@ -74,7 +74,7 @@ from camel.models import (from camel.prompts import TextPromptfrom camel.responses import ChatAgentResponsefrom camel.storages import JsonStorage-from camel.toolkits import FunctionTool+from camel.toolkits import FunctionTool, RegisteredAgentToolkitfrom camel.types import (ChatCompletion,ChatCompletionChunk,@@ -89,7 +89,6 @@ from camel.utils import (model_from_json_schema,)from camel.utils.commons import dependencies_required-from camel.utils.tool_result import ToolResultif TYPE_CHECKING:from camel.terminators import ResponseTerminator@@ -348,6 +347,13 @@ class ChatAgent(BaseAgent):tools (Optional[List[Union[FunctionTool, Callable]]], optional): Listof available :obj:`FunctionTool` or :obj:`Callable`. (default::obj:`None`)+ toolkits_to_register_agent (Optional[List[RegisteredAgentToolkit]],+ optional): List of toolkit instances that inherit from+ :obj:`RegisteredAgentToolkit`. The agent will register itself with+ these toolkits, allowing them to access the agent instance. Note:+ This does NOT add the toolkit's tools to the agent. To use tools+ from these toolkits, pass them explicitly via the `tools`+ parameter. (default: :obj:`None`)external_tools (Optional[List[Union[FunctionTool, Callable,Dict[str, Any]]]], optional): List of external tools(:obj:`FunctionTool` or :obj:`Callable` or :obj:`Dict[str, Any]`)@@ -405,6 +411,9 @@ class ChatAgent(BaseAgent):token_limit: Optional[int] = None,output_language: Optional[str] = None,tools: Optional[List[Union[FunctionTool, Callable]]] = None,+ toolkits_to_register_agent: Optional[+ List[RegisteredAgentToolkit]+ ] = None,external_tools: Optional[List[Union[FunctionTool, Callable, Dict[str, Any]]]] = None,@@ -479,6 +488,12 @@ class ChatAgent(BaseAgent):]}+ # Register agent with toolkits that have RegisteredAgentToolkit mixin+ if toolkits_to_register_agent:+ for toolkit in toolkits_to_register_agent:+ if isinstance(toolkit, RegisteredAgentToolkit):+ toolkit.register_agent(self)+self._external_tool_schemas = {tool_schema["function"]["name"]: tool_schemafor tool_schema in [@@ -494,9 +509,6 @@ class ChatAgent(BaseAgent):self.tool_execution_timeout = tool_execution_timeoutself.mask_tool_output = mask_tool_outputself._secure_result_store: Dict[str, Any] = {}- self._pending_images: List[str] = []- self._image_retry_count: Dict[str, int] = {}- # Store images to attach to next user messageself.pause_event = pause_eventself.prune_tool_calls_from_memory = prune_tool_calls_from_memory@@ -504,8 +516,6 @@ class ChatAgent(BaseAgent):r"""Resets the :obj:`ChatAgent` to its initial state."""self.terminated = Falseself.init_messages()- self._pending_images = []- self._image_retry_count = {}for terminator in self.response_terminators:terminator.reset()@@ -1367,16 +1377,6 @@ class ChatAgent(BaseAgent):role_name="User", content=input_message)- # Attach any pending images from previous tool calls- image_list = self._process_pending_images()- if image_list:- # Create new message with images attached- input_message = BaseMessage.make_user_message(- role_name="User",- content=input_message.content,- image_list=image_list,- )-# Add user input to memoryself.update_memory(input_message, OpenAIBackendRole.USER)@@ -1571,16 +1571,6 @@ class ChatAgent(BaseAgent):role_name="User", content=input_message)- # Attach any pending images from previous tool calls- image_list = self._process_pending_images()- if image_list:- # Create new message with images attached- input_message = BaseMessage.make_user_message(- role_name="User",- content=input_message.content,- image_list=image_list,- )-self.update_memory(input_message, OpenAIBackendRole.USER)tool_call_records: List[ToolCallingRecord] = []@@ -1634,7 +1624,6 @@ class ChatAgent(BaseAgent):if tool_call_requests := response.tool_call_requests:# Process all tool calls- new_images_from_tools = []for tool_call_request in tool_call_requests:if (tool_call_request.tool_name@@ -1654,72 +1643,10 @@ class ChatAgent(BaseAgent):)tool_call_records.append(tool_call_record)- # Check if this tool call produced images- if (- hasattr(tool_call_record, 'images')- and tool_call_record.images- ):- new_images_from_tools.extend(- tool_call_record.images- )-# If we found an external tool call, break the loopif external_tool_call_requests:break- # If tools produced images- # send them to the model as a user message- if new_images_from_tools:- # Convert base64 images to PIL Images- image_list = []- for img_data in new_images_from_tools:- try:- import base64- import io-- from PIL import Image-- # Extract base64 data from data URL format- if img_data.startswith("data:image"):- # Format:- # "data:image/png;base64,iVBORw0KGgo..."- base64_data = img_data.split(',', 1)[1]- else:- # Raw base64 data- base64_data = img_data-- # Decode and create PIL Image- image_bytes = base64.b64decode(base64_data)- pil_image = Image.open(io.BytesIO(image_bytes))- # Convert to ensure proper- # Image.Image type for compatibility- pil_image_tool_result: Image.Image = (- pil_image.convert('RGB')- )- image_list.append(pil_image_tool_result)-- except Exception as e:- logger.warning(- f"Failed to convert "- f"base64 image to PIL for immediate use: {e}"- )- continue-- # If we have valid images- # create a user message with images- if image_list:- # Create a user message with images- # to provide visual context immediately- image_message = BaseMessage.make_user_message(- role_name="User",- content="[Visual content from tool execution - please analyze and continue]", # noqa: E501- image_list=image_list,- )-- self.update_memory(- image_message, OpenAIBackendRole.USER- )-if (self.max_iteration is not Noneand iteration_count >= self.max_iteration@@ -1810,69 +1737,6 @@ class ChatAgent(BaseAgent):info=info,)- def _process_pending_images(self) -> List:- r"""Process pending images with retry logic and return PIL Image list.-- Returns:- List: List of successfully converted PIL Images.- """- if not self._pending_images:- return []-- image_list = []- successfully_processed = []- failed_images = []-- for img_data in self._pending_images:- # Track retry count- retry_count = self._image_retry_count.get(img_data, 0)-- # Remove images that have failed too many times (max 3 attempts)- if retry_count >= 3:- failed_images.append(img_data)- logger.warning(- f"Removing image after {retry_count} failed attempts"- )- continue-- try:- import base64- import io-- from PIL import Image-- # Extract base64 data from data URL format- if img_data.startswith("data:image"):- # Format: "data:image/png;base64,iVBORw0KGgo..."- base64_data = img_data.split(',', 1)[1]- else:- # Raw base64 data- base64_data = img_data-- # Decode and create PIL Image- image_bytes = base64.b64decode(base64_data)- pil_image = Image.open(io.BytesIO(image_bytes))- pil_image_converted: Image.Image = pil_image.convert('RGB')- image_list.append(pil_image_converted)- successfully_processed.append(img_data)-- except Exception as e:- # Increment retry count for failed conversion- self._image_retry_count[img_data] = retry_count + 1- logger.warning(- f"Failed to convert base64 image to PIL "- f"(attempt {retry_count + 1}/3): {e}"- )- continue-- # Clean up processed and failed images- for img in successfully_processed + failed_images:- self._pending_images.remove(img)- # Clean up retry count for processed/removed images- self._image_retry_count.pop(img, None)-- return image_list-def _record_final_output(self, output_messages: List[BaseMessage]) -> None:r"""Log final messages or warnings about multiple responses."""if len(output_messages) == 1:@@ -1883,61 +1747,6 @@ class ChatAgent(BaseAgent):"selected message manually using `record_message()`.")- def _is_vision_error(self, exc: Exception) -> bool:- r"""Check if the exception is likely related to vision/image is not- supported by the model."""- # TODO: more robust vision error detection- error_msg = str(exc).lower()- vision_keywords = [- 'vision',- 'image',- 'multimodal',- 'unsupported',- 'invalid content type',- 'image_url',- 'visual',- ]- return any(keyword in error_msg for keyword in vision_keywords)-- def _has_images(self, messages: List[OpenAIMessage]) -> bool:- r"""Check if any message contains images."""- for msg in messages:- content = msg.get('content')- if isinstance(content, list):- for item in content:- if (- isinstance(item, dict)- and item.get('type') == 'image_url'- ):- return True- return False-- def _strip_images_from_messages(- self, messages: List[OpenAIMessage]- ) -> List[OpenAIMessage]:- r"""Remove images from messages, keeping only text content."""- stripped_messages = []- for msg in messages:- content = msg.get('content')- if isinstance(content, list):- # Extract only text content from multimodal messages- text_content = ""- for item in content:- if isinstance(item, dict) and item.get('type') == 'text':- text_content += item.get('text', '')-- # Create new message with only text content- new_msg = msg.copy()- new_msg['content'] = (- text_content- or "[Image content removed - model doesn't support vision]"- )- stripped_messages.append(new_msg)- else:- # Regular text message, keep as is- stripped_messages.append(msg)- return stripped_messages-@observe()def _get_model_response(self,@@ -1971,35 +1780,13 @@ class ChatAgent(BaseAgent):openai_messages, response_format, tool_schemas or None)except Exception as exc:- # Try again without images if the error might be vision-related- if self._is_vision_error(exc) and self._has_images(- openai_messages- ):- logger.warning(- "Model appears to not support vision."- "Retrying without images."- )- try:- stripped_messages = self._strip_images_from_messages(- openai_messages- )- response = self.model_backend.run(- stripped_messages,- response_format,- tool_schemas or None,- )- except Exception:- pass # Fall through to original error handling-- if not response:- logger.error(- f"An error occurred while running model "- f"iteration {current_iteration}, "- f"{self.model_backend.model_type}, "- f"index: {self.model_backend.current_model_index}",- exc_info=exc,- )- error_info = str(exc)+ logger.error(+ f"An error occurred while running model "+ f"{self.model_backend.model_type}, "+ f"index: {self.model_backend.current_model_index}",+ exc_info=exc,+ )+ error_info = str(exc)if not response and self.model_backend.num_models > 1:raise ModelProcessingError(@@ -2060,33 +1847,13 @@ class ChatAgent(BaseAgent):openai_messages, response_format, tool_schemas or None)except Exception as exc:- # Try again without images if the error might be vision-related- if self._is_vision_error(exc) and self._has_images(- openai_messages- ):- logger.warning(- "Model appears to not support vision. Retrying without images." # noqa: E501- )- try:- stripped_messages = self._strip_images_from_messages(- openai_messages- )- response = await self.model_backend.arun(- stripped_messages,- response_format,- tool_schemas or None,- )- except Exception:- pass # Fall through to original error handling-- if not response:- logger.error(- f"An error occurred while running model "- f"{self.model_backend.model_type}, "- f"index: {self.model_backend.current_model_index}",- exc_info=exc,- )- error_info = str(exc)+ logger.error(+ f"An error occurred while running model "+ f"{self.model_backend.model_type}, "+ f"index: {self.model_backend.current_model_index}",+ exc_info=exc,+ )+ error_info = str(exc)if not response and self.model_backend.num_models > 1:raise ModelProcessingError(@@ -2457,26 +2224,9 @@ class ChatAgent(BaseAgent):mask_flag = Falselogger.warning(f"{error_msg} with result: {result}")- # Check if result is a ToolResult with images- images_to_attach = None- if isinstance(result, ToolResult):- images_to_attach = result.images- result = str(result) # Use string representation for storage-- tool_record = self._record_tool_calling(+ return self._record_tool_calling(func_name, args, result, tool_call_id, mask_output=mask_flag)- logger.info(f"Tool calling record:\n{tool_record}")-- # Store images for later attachment to next user message- if images_to_attach:- tool_record.images = images_to_attach- # Add images with duplicate prevention- for img in images_to_attach:- if img not in self._pending_images:- self._pending_images.append(img)-- return tool_recordasync def _aexecute_tool(self,@@ -2517,26 +2267,7 @@ class ChatAgent(BaseAgent):error_msg = f"Error executing async tool '{func_name}': {e!s}"result = f"Tool execution failed: {error_msg}"logging.warning(error_msg)-- # Check if result is a ToolResult with images- images_to_attach = None- if isinstance(result, ToolResult):- images_to_attach = result.images- result = str(result) # Use string representation for storage-- tool_record = self._record_tool_calling(- func_name, args, result, tool_call_id- )-- # Store images for later attachment to next user message- if images_to_attach:- tool_record.images = images_to_attach- # Add images with duplicate prevention- for img in images_to_attach:- if img not in self._pending_images:- self._pending_images.append(img)-- return tool_record+ return self._record_tool_calling(func_name, args, result, tool_call_id)def _record_tool_calling(self,diff --git a/camel/societies/workforce/single_agent_worker.py b/camel/societies/workforce/single_agent_worker.pyindex 8782d03ed..ebddac934 100644--- a/camel/societies/workforce/single_agent_worker.py+++ b/camel/societies/workforce/single_agent_worker.py@@ -410,7 +410,7 @@ class SingleAgentWorker(Worker):f"{getattr(worker_agent, 'agent_id', worker_agent.role_name)} "f"(from pool/clone of "f"{getattr(self.worker, 'agent_id', self.worker.role_name)}) "- f"to process task {task.content}",+ f"to process task: {task.content}","response_content": response_content,"tool_calls": final_response.info.get("tool_calls")if isinstance(response, AsyncStreamingChatAgentResponse)diff --git a/camel/toolkits/__init__.py b/camel/toolkits/__init__.pyindex b1d01ab94..953037f3d 100644--- a/camel/toolkits/__init__.py+++ b/camel/toolkits/__init__.py@@ -31,7 +31,7 @@ from .meshy_toolkit import MeshyToolkitfrom .openbb_toolkit import OpenBBToolkitfrom .bohrium_toolkit import BohriumToolkit-from .base import BaseToolkit+from .base import BaseToolkit, RegisteredAgentToolkitfrom .google_maps_toolkit import GoogleMapsToolkitfrom .code_execution import CodeExecutionToolkitfrom .github_toolkit import GithubToolkit@@ -162,4 +162,5 @@ __all__ = ['AgentCommunicationToolkit','WebDeployToolkit','ScreenshotToolkit',+ 'RegisteredAgentToolkit',]diff --git a/camel/toolkits/base.py b/camel/toolkits/base.pyindex 19420f9e5..343302b90 100644--- a/camel/toolkits/base.py+++ b/camel/toolkits/base.py@@ -12,11 +12,18 @@# limitations under the License.# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========-from typing import List, Literal, Optional+from typing import TYPE_CHECKING, List, Literal, Optional+from camel.logger import get_loggerfrom camel.toolkits import FunctionToolfrom camel.utils import AgentOpsMeta, with_timeout+if TYPE_CHECKING:+ from camel.agents import ChatAgent+++logger = get_logger(__name__)+class BaseToolkit(metaclass=AgentOpsMeta):r"""Base class for toolkits.@@ -63,3 +70,52 @@ class BaseToolkit(metaclass=AgentOpsMeta):the MCP server in."""self.mcp.run(mode)+++class RegisteredAgentToolkit:+ r"""Mixin class for toolkits that need to register a ChatAgent.++ This mixin provides a standard interface for toolkits that require+ a reference to a ChatAgent instance. The ChatAgent will check if a+ toolkit has this mixin and automatically register itself.+ """++ def __init__(self):+ self._agent: Optional["ChatAgent"] = None++ @property+ def agent(self) -> Optional["ChatAgent"]:+ r"""Get the registered ChatAgent instance.++ Returns:+ Optional[ChatAgent]: The registered agent, or None if not+ registered.++ Note:+ If None is returned, it means the toolkit has not been registered+ with a ChatAgent yet. Make sure to pass this toolkit to a ChatAgent+ via the toolkits parameter during initialization.+ """+ if self._agent is None:+ logger.warning(+ f"{self.__class__.__name__} does not have a "+ f"registered ChatAgent. "+ f"Please ensure this toolkit is passed to a ChatAgent via the "+ f"'toolkits_to_register_agent' parameter during ChatAgent "+ f"initialization if you want to use the tools that require a "+ f"registered agent."+ )+ return self._agent++ def register_agent(self, agent: "ChatAgent") -> None:+ r"""Register a ChatAgent with this toolkit.++ This method allows registering an agent after initialization. The+ ChatAgent will automatically call this method if the toolkit to+ register inherits from RegisteredAgentToolkit.++ Args:+ agent (ChatAgent): The ChatAgent instance to register.+ """+ self._agent = agent+ logger.info(f"Agent registered with {self.__class__.__name__}")diff --git a/camel/toolkits/screenshot_toolkit.py b/camel/toolkits/screenshot_toolkit.pyindex b45f4fe31..fdca9c565 100644--- a/camel/toolkits/screenshot_toolkit.py+++ b/camel/toolkits/screenshot_toolkit.py@@ -12,22 +12,22 @@# limitations under the License.# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========-import base64-import ioimport os-import timefrom pathlib import Pathfrom typing import List, Optional+from PIL import Image+from camel.logger import get_logger+from camel.messages import BaseMessagefrom camel.toolkits import BaseToolkit, FunctionTool+from camel.toolkits.base import RegisteredAgentToolkitfrom camel.utils import dependencies_required-from camel.utils.tool_result import ToolResultlogger = get_logger(__name__)-class ScreenshotToolkit(BaseToolkit):+class ScreenshotToolkit(BaseToolkit, RegisteredAgentToolkit):r"""A toolkit for taking screenshots."""@dependencies_required('PIL')@@ -50,6 +50,7 @@ class ScreenshotToolkit(BaseToolkit):from PIL import ImageGrabsuper().__init__(timeout=timeout)+ RegisteredAgentToolkit.__init__(self)camel_workdir = os.environ.get("CAMEL_WORKDIR")if working_directory:@@ -63,22 +64,96 @@ class ScreenshotToolkit(BaseToolkit):self.screenshots_dir = path / "screenshots"self.screenshots_dir.mkdir(parents=True, exist_ok=True)- def take_screenshot(+ def read_image(+ self,+ image_path: str,+ instruction: str = "",+ ) -> str:+ r"""Analyzes an image from a local file path.++ This function enables you to "see" and interpret an image from a+ file. It's useful for tasks where you need to understand visual+ information, such as reading a screenshot of a webpage or a diagram.++ Args:+ image_path (str): The local file path to the image.+ For example: 'screenshots/login_page.png'.+ instruction (str, optional): Specific instructions for what to look+ for or what to do with the image. For example: "What is the+ main headline on this page?" or "Find the 'Submit' button.".++ Returns:+ str: The response after analyzing the image, which could be a+ description, an answer, or a confirmation of an action.+ """+ if self.agent is None:+ logger.error(+ "Cannot record screenshot in memory: No agent registered. "+ "Please pass this toolkit to ChatAgent via "+ "toolkits_to_register_agent parameter."+ )+ return (+ "Error: No agent registered. Please pass this toolkit to "+ "ChatAgent via toolkits_to_register_agent parameter."+ )++ try:+ # Check if file exists before trying to open+ if not os.path.exists(image_path):+ error_msg = f"Screenshot file not found: {image_path}"+ logger.error(error_msg)+ return f"Error: {error_msg}"++ # Load the image from the path+ img = Image.open(image_path)++ # Create a message with the screenshot image+ message = BaseMessage.make_user_message(+ role_name="User",+ content=instruction,+ image_list=[img],+ )++ # Record the message in agent's memory+ response = self.agent.step(message)+ return response.msgs[0].content++ except Exception as e:+ logger.error(f"Error reading screenshot: {e}")+ return f"Error reading screenshot: {e}"++ def take_screenshot_and_read_image(self,+ filename: str,save_to_file: bool = True,- ) -> ToolResult:- r"""Take a screenshot of the entire screen and return it as a- base64-encoded image.+ read_image: bool = True,+ instruction: Optional[str] = None,+ ) -> str:+ r"""Captures a screenshot of the entire screen.++ This function can save the screenshot to a file and optionally analyze+ it. It's useful for capturing the current state of the UI for+ documentation, analysis, or to guide subsequent actions.Args:- save_to_file (bool): Whether to save the screenshot to a file.+ filename (str): The name for the screenshot file (e.g.,+ "homepage.png"). The file is saved in a `screenshots`+ subdirectory within the working directory. Must end with+ `.png`. (default: :obj:`None`)+ save_to_file (bool, optional): If `True`, saves the screenshot to+ a file. (default: :obj:`True`)+ read_image (bool, optional): If `True`, the agent will analyze+ the screenshot. `save_to_file` must also be `True`.(default: :obj:`True`)+ instruction (Optional[str], optional): A specific question or+ command for the agent regarding the screenshot, used only if+ `read_image` is `True`. For example: "Confirm that the+ user is logged in.".Returns:- ToolResult: An object containing:- - text (str): A description of the screenshot.- - images (List[str]): A list containing one base64-encoded- PNG image data URL.+ str: A confirmation message indicating success or failure,+ including the file path if saved, and the agent's response+ if `read_image` is `True`."""try:# Take screenshot of entire screen@@ -90,32 +165,37 @@ class ScreenshotToolkit(BaseToolkit):# Create directory if it doesn't existos.makedirs(self.screenshots_dir, exist_ok=True)- # Generate filename with timestamp- timestamp = int(time.time())- filename = f"screenshot_{timestamp}.png"- file_path = os.path.join(self.screenshots_dir, filename)+ # Create unique filename if file already exists+ base_path = os.path.join(self.screenshots_dir, filename)+ file_path = base_path+ counter = 1+ while os.path.exists(file_path):+ name, ext = os.path.splitext(filename)+ unique_filename = f"{name}_{counter}{ext}"+ file_path = os.path.join(+ self.screenshots_dir, unique_filename+ )+ counter += 1+screenshot.save(file_path)logger.info(f"Screenshot saved to {file_path}")- # Convert to base64- img_buffer = io.BytesIO()- screenshot.save(img_buffer, format="PNG")- img_buffer.seek(0)- img_base64 = base64.b64encode(img_buffer.getvalue()).decode(- 'utf-8'- )- img_data_url = f"data:image/png;base64,{img_base64}"-# Create result textresult_text = "Screenshot captured successfully"if file_path:result_text += f" and saved to {file_path}"- return ToolResult(text=result_text, images=[img_data_url])+ # Record in agent memory if requested+ if read_image and file_path is not None:+ inst = instruction if instruction is not None else ""+ response = self.read_image(file_path, inst)+ result_text += f". Agent response: {response}"++ return result_textexcept Exception as e:logger.error(f"Error taking screenshot: {e}")- return ToolResult(text=f"Error taking screenshot: {e}", images=[])+ return f"Error taking screenshot: {e}"def get_tools(self) -> List[FunctionTool]:r"""Returns a list of FunctionTool objects for screenshot operations.@@ -124,5 +204,6 @@ class ScreenshotToolkit(BaseToolkit):List[FunctionTool]: List of screenshot functions."""return [- FunctionTool(self.take_screenshot),+ FunctionTool(self.take_screenshot_and_read_image),+ FunctionTool(self.read_image),]diff --git a/camel/toolkits/terminal_toolkit.py b/camel/toolkits/terminal_toolkit.pyindex f6e6274c2..6f1a48559 100644--- a/camel/toolkits/terminal_toolkit.py+++ b/camel/toolkits/terminal_toolkit.py@@ -319,7 +319,7 @@ class TerminalToolkit(BaseToolkit):)# Install essential packages using uv- essential_packages = ["pip", "setuptools", "wheel"]+ essential_packages = ["pip", "setuptools", "wheel", "pyautogui"]subprocess.run(["uv",@@ -371,7 +371,7 @@ class TerminalToolkit(BaseToolkit):pip_path = os.path.join(self.initial_env_path, "bin", "pip")# Upgrade pip and install essential packages- essential_packages = ["pip", "setuptools", "wheel"]+ essential_packages = ["pip", "setuptools", "wheel", "pyautogui"]subprocess.run([pip_path, "install", "--upgrade", *essential_packages],check=True,diff --git a/examples/toolkits/screenshot_toolkit_example.py b/examples/toolkits/screenshot_toolkit_example.pynew file mode 100644index 000000000..07024abb6--- /dev/null+++ b/examples/toolkits/screenshot_toolkit_example.py@@ -0,0 +1,54 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+from pathlib import Path++from camel.agents import ChatAgent+from camel.models import ModelFactory+from camel.toolkits import ScreenshotToolkit+from camel.types import ModelPlatformType, ModelType++# Initialize the model+model = ModelFactory.create(+ model_platform=ModelPlatformType.DEFAULT,+ model_type=ModelType.DEFAULT,+)++# Set up working directory+working_dir = Path("screenshot_example_workspace")+working_dir.mkdir(exist_ok=True)++# Create the ScreenshotToolkit+screenshot_toolkit = ScreenshotToolkit(+ working_directory=str(working_dir),+ timeout=30.0,+)++agent = ChatAgent(+ system_message=(+ "You are a helpful assistant that can take and analyze "+ "screenshots. When analyzing screenshots, describe what you see "+ "in detail, including any text, UI elements, applications, or "+ "content visible on the screen."+ ),+ model=model,+ tools=screenshot_toolkit.get_tools(),+ toolkits_to_register_agent=[screenshot_toolkit],+)++response = agent.step(+ "Take a screenshot called 'screen_analysis.png' and analyze what "+ "you see on the screen. Tell me about any applications, windows, "+ "or content visible."+)+print(f"Agent response: {response.msgs[0].content}")diff --git a/examples/workforce/eigent.py b/examples/workforce/eigent.pyindex e3cdadb7e..07648ef57 100644--- a/examples/workforce/eigent.py+++ b/examples/workforce/eigent.py@@ -40,6 +40,7 @@ from camel.toolkits import (OpenAIImageToolkit,PPTXToolkit,RedditToolkit,+ ScreenshotToolkit,SearchToolkit,SlackToolkit,TerminalToolkit,@@ -119,112 +120,118 @@ def developer_agent_factory(*TerminalToolkit(safe_mode=True, clone_current_env=False).get_tools(),*NoteTakingToolkit(working_directory=WORKING_DIRECTORY).get_tools(),*WebDeployToolkit().get_tools(),+ *ScreenshotToolkit(working_directory=WORKING_DIRECTORY).get_tools(),]system_message = f"""<intro>-You are a skilled coding assistant. You can write and execute code by using-the available terminal tools.+You are a master-level coding assistant, equipped with a powerful and+unrestricted terminal. Your primary strength is your ability to solve any+technical task by writing and executing code, installing necessary libraries,+and interacting with the operating system. Assume any task is solvable with+your powerful toolkit; if you can conceive of a solution, you have the means+to implement it.You are working in a team with team members. Your team members are:-- Search Agent: Can search the web, extract webpage content, simulate browser+- Search Agent: Can search the web, extract webpage content, simulate browseractions, and provide relevant information to solve the given task.-- Document Agent: A document processing assistant for creating, modifying, and+- Document Agent: A document processing assistant for creating, modifying, andmanaging various document formats, including presentations.-- Multi-Modal Agent: A multi-modal processing assistant for analyzing, and+- Multi-Modal Agent: A multi-modal processing assistant for analyzing, andgenerating media content like audio and images.-You are now working in system {platform.system()} with architecture-{platform.machine()} at working directory `{WORKING_DIRECTORY}`. All your+You are now working in system {platform.system()} with architecture+{platform.machine()} at working directory `{WORKING_DIRECTORY}`. All yourwork related to local operations should be done in that directory.-The current time is {datetime.datetime.now()}.+The current date is {datetime.date.today()}.</intro>-+<mandatory_instructions>-- You MUST use the `send_message_to_user` tool to inform the user of every-decision and action you take. Your message must include a short title and+- You MUST use the `send_message_to_user` tool to inform the user of every+decision and action you take. Your message must include a short title anda one-sentence description. This is a mandatory part of your workflow.- You MUST use the `read_note` tool to read the notes from other agents.-- When you complete your task, your final response must be a comprehensive-summary of your work and the outcome, presented in a clear, detailed, and-easy-to-read format. Avoid using markdown tables for presenting data; use+- When you complete your task, your final response must be a comprehensive+summary of your work and the outcome, presented in a clear, detailed, and+easy-to-read format. Avoid using markdown tables for presenting data; useplain text formatting instead.<mandatory_instructions><capabilities>-Your capabilities include:-- Writing code to solve tasks. To execute the code, you MUST first save-it to a file in the workspace (e.g., `script.py`), and then run it using-the terminal tool (e.g., `python script.py`).-- Running terminal commands to install packages (e.g., with `pip3` or-`uv`), process files, or test functionality. All files you create should-be in the designated workspace.-- You can use `uv` or `pip3` to install packages, for example, `uv pip3-install requests` or `pip3 install requests`.-- Verifying your solutions through immediate execution and testing in the-terminal.-- Utilizing any Python libraries (e.g., requests, BeautifulSoup, pandas,-etc.) needed for efficient solutions. You can install missing packages-using `pip3` or `uv` in the terminal.-- Deploying web applications and content. You can:- - Serve a single HTML file or a string of HTML content on a local web- server.- - Deploy an entire folder as a static website—for example, a built- React application.- - Manage your deployments by stopping servers on specific ports.- - List all currently running web servers to keep track of your work.-- Asking for human input via the console if you are stuck or need-clarification.-- Write and check notes from other agents to keep track of your work.+Your capabilities are extensive and powerful:+- **Unrestricted Code Execution**: You can write and execute code to solve+any task. You MUST first save your code to a file (e.g., `script.py`)+and then run it using the terminal (e.g., `python script.py`).+- **Full Terminal Control**: You have complete control over the terminal. You+can install any packages (e.g., with `pip3`, `uv`, or `apt-get`), run any+command-line tools, and manage files and processes. If a tool is missing,+you are empowered to install it. There are no limitations on what you can do+via the terminal.+- **Screen Observation**: You can take screenshots of the user's screen to+understand the state of GUIs or applications, enabling you to perform tasks+that require visual context.+- **Solution Verification**: You can verify your solutions through immediate+execution and testing in the terminal.+- **Library Utilization**: You can utilize any Python library (e.g.,+requests, BeautifulSoup, pandas) needed for efficient solutions. If a+package is missing, install it.+- **Web Deployment**: You can deploy web applications and content. You can+serve files, deploy static sites, and manage your deployments.+- **Human Collaboration**: You can ask for human input via the console if you+are stuck or need clarification.+- **Note Management**: You can write and read notes to coordinate with other+agents and keep track of your work.</capabilities><terminal_tips>-The terminal tools are session-based. You must manage one or more terminal+The terminal tools are session-based. You must manage one or more terminalsessions to perform your tasks. A session is identified by a unique `id`.-- Avoid commands requiring confirmation; actively use -y or -f flags for+- The terminal is your most powerful tool. Almost any problem can be solved by+using it creatively. Be bold and resourceful.+- Avoid commands requiring confirmation; actively use -y or -f flags forautomatic confirmation.- For commands with excessive output, redirect stdout to a file.- Chain multiple commands with `&&` to minimize interruptions.- Use the pipe operator `|` to pass command outputs and simplify operations.- Use the `ls -F` command to check the execution permissions of files.-- If you need to install any new packages, use `pip3 install` or `apt-get+- If you need to install any new packages, use `pip3 install` or `apt-getinstall` after checking the OS type.-- Execute Commands: Use `shell_exec(id="...", command="...")` to run+- Execute Commands: Use `shell_exec(id="...", command="...")` to runa command. If the `id` is new, a new session is created.Example: `shell_exec(id="session_1", command="ls -l")`-- Manage Long-Running Tasks: For commands that take time, run them- in one step, and then use `shell_wait(id="...")` to wait for- completion. This prevents blocking and allows you to perform other+- Manage Long-Running Tasks: For commands that take time, run them+ in one step, and then use `shell_wait(id="...")` to wait for+ completion. This prevents blocking and allows you to perform othertasks in parallel.-- View Output: Use `shell_view(id="...")` to see the full command+- View Output: Use `shell_view(id="...")` to see the full commandhistory and output of a session.-- Run Tasks in Parallel: Use different session IDs to run multiple+- Run Tasks in Parallel: Use different session IDs to run multiplecommands concurrently.- `shell_exec(id="install", command="pip3 install numpy")`- `shell_exec(id="test", command="python my_script.py")`-- Interact with Processes: For commands that require input, you can+- Interact with Processes: For commands that require input, you canuse:- - `shell_exec(id="...", command="...", interactive=True)` for+ - `shell_exec(id="...", command="...", interactive=True)` forreal-time interactive sessions.- - `shell_write_to_process(id="...", content="...")` to send input to a+ - `shell_write_to_process(id="...", content="...")` to send input to anon-interactive running process.-- Stop a Process: If a process needs to be terminated, use+- Stop a Process: If a process needs to be terminated, use`shell_kill_process(id="...")`.</terminal_tips><collaboration_and_assistance>-- If you get stuck, encounter an issue you cannot solve (like a CAPTCHA),+- If you get stuck, encounter an issue you cannot solve (like a CAPTCHA),or need clarification, use the `ask_human_via_console` tool.-- Document your progress and findings in notes so other agents can build+- Document your progress and findings in notes so other agents can buildupon your work.</collaboration_and_assistance>"""@@ -292,61 +299,61 @@ def search_agent_factory(system_message = f"""<intro>-You are a helpful assistant that can search the web,-extract webpage content, simulate browser actions, and provide relevant+You are a helpful assistant that can search the web,+extract webpage content, simulate browser actions, and provide relevantinformation to solve the given task.You are working in a team with team members. Your team members are:-- Developer Agent: A skilled coding assistant that can write and execute code,+- Developer Agent: A skilled coding assistant that can write and execute code,run terminal commands, and verify solutions to complete tasks.-- Document Agent: A document processing assistant for creating, modifying, and+- Document Agent: A document processing assistant for creating, modifying, andmanaging various document formats, including presentations.-- Multi-Modal Agent: A multi-modal processing assistant for analyzing, and+- Multi-Modal Agent: A multi-modal processing assistant for analyzing, andgenerating media content like audio and images.-You are now working in system {platform.system()} with architecture-{platform.machine()} at working directory `{WORKING_DIRECTORY}`. All your+You are now working in system {platform.system()} with architecture+{platform.machine()} at working directory `{WORKING_DIRECTORY}`. All yourwork related to local operations should be done in that directory.-The current time is {datetime.datetime.now()}.+The current date is {datetime.date.today()}.</intro><mandatory_instructions>-- You MUST use the `send_message_to_user` tool to inform the user of every- decision and action you take. Your message must include a short title and+- You MUST use the `send_message_to_user` tool to inform the user of every+ decision and action you take. Your message must include a short title anda one-sentence description. This is a mandatory part of your workflow.-- You MUST use the note-taking tools to record your findings. This is a- critical part of your role. Your notes are the primary source of+- You MUST use the note-taking tools to record your findings. This is a+ critical part of your role. Your notes are the primary source ofinformation for your teammates. To avoid information loss, you must notsummarize your findings. Instead, record all information in detail.For every piece of information you gather, you must:- 1. **Extract ALL relevant details**: Quote all important sentences,- statistics, or data points. Your goal is to capture the information+ 1. **Extract ALL relevant details**: Quote all important sentences,+ statistics, or data points. Your goal is to capture the informationas completely as possible.- 2. **Cite your source**: Include the exact URL where you found the+ 2. **Cite your source**: Include the exact URL where you found theinformation.- 3. **Explain relevance**: Briefly state why this information is+ 3. **Explain relevance**: Briefly state why this information isuseful for the overall task.- Your notes should be a detailed and complete record of the information- you have discovered. High-quality, detailed notes are essential for the+ Your notes should be a detailed and complete record of the information+ you have discovered. High-quality, detailed notes are essential for theteam's success.- You MUST only use URLs from trusted sources. A trusted source is a URLthat is either:- 1. Returned by a search tool (like `search_google`, `search_bing`,+ 1. Returned by a search tool (like `search_google`, `search_bing`,or `search_exa`).2. Found on a webpage you have visited.-- You are strictly forbidden from inventing, guessing, or constructing URLs+- You are strictly forbidden from inventing, guessing, or constructing URLsyourself. Fabricating URLs will be considered a critical error.- You MUST NOT answer from your own knowledge. All informationMUST be sourced from the web using the available tools. If you don't knowsomething, find it out using your tools.-- When you complete your task, your final response must be a comprehensive- summary of your findings, presented in a clear, detailed, and- easy-to-read format. Avoid using markdown tables for presenting data;+- When you complete your task, your final response must be a comprehensive+ summary of your findings, presented in a clear, detailed, and+ easy-to-read format. Avoid using markdown tables for presenting data;use plain text formatting instead.<mandatory_instructions>@@ -368,7 +375,7 @@ Your capabilities include:- **Navigation and Exploration**: Use `visit_page` to open a URL.`visit_page` provides a snapshot of currently visible interactiveelements, not the full page text. To see more content on long- pages, Navigate with `click`, `back`, and `forward`. Manage multiple+ pages, Navigate with `click`, `back`, and `forward`. Manage multiplepages with `switch_tab`.- **Analysis**: Use `get_som_screenshot` to understand the page layoutand identify interactive elements. Since this is a heavy operation,@@ -378,12 +385,12 @@ Your capabilities include:- Alternative Search: If you are unable to get sufficientinformation through browser-based exploration and scraping, use`search_exa`. This tool is best used for getting quick summaries or- finding specific answers when visiting web page is could not find the+ finding specific answers when visiting web page is could not find theinformation.- In your response, you should mention the URLs you have visited and processed.-- When encountering verification challenges (like login, CAPTCHAs or+- When encountering verification challenges (like login, CAPTCHAs orrobot checks), you MUST request help using the human toolkit.</web_search_workflow>"""@@ -422,48 +429,51 @@ def document_agent_factory(system_message = f"""<intro>-You are a Document Processing Assistant specialized+You are a Document Processing Assistant specializedin creating, modifying, and managing various document formats.You are working in a team with team members. Your team members are:-- Developer Agent: A skilled coding assistant that can write and execute code,+- Developer Agent: A skilled coding assistant that can write and execute code,run terminal commands, and verify solutions to complete tasks.-- Search Agent: Can search the web, extract webpage content, simulate browser+- Search Agent: Can search the web, extract webpage content, simulate browseractions, and provide relevant information to solve the given task.-- Multi-Modal Agent: A multi-modal processing assistant for analyzing, and+- Multi-Modal Agent: A multi-modal processing assistant for analyzing, andgenerating media content like audio and images.-You are now working in system {platform.system()} with architecture-{platform.machine()} at working directory `{WORKING_DIRECTORY}`. All your+You are now working in system {platform.system()} with architecture+{platform.machine()} at working directory `{WORKING_DIRECTORY}`. All yourwork related to local operations should be done in that directory.-The current time is {datetime.datetime.now()}.+The current date is {datetime.date.today()}.</intro><mandatory_instructions>-- You MUST use the `send_message_to_user` tool to inform the user of every- decision and action you take. Your message must include a short title and+- You MUST use the `send_message_to_user` tool to inform the user of every+ decision and action you take. Your message must include a short title anda one-sentence description. This is a mandatory part of your workflow.-- Before creating any document, you MUST use the `read_note` tool to gather+- Before creating any document, you MUST use the `read_note` tool to gatherall information collected by other team members.- You MUST use the available tools to create or modify documents (e.g.,`write_to_file`, `create_presentation`). Your primary output should bea file, not just content within your response.-- If the document has many data, you MUST use the terminal tool to+- If there's no specified format for the document/report/paper, you should use+ the `write_to_file` tool to create a HTML file.++- If the document has many data, you MUST use the terminal tool togenerate charts and graphs and add them to the document.-- When you complete your task, your final response must be a summary of- your work and the path to the final document, presented in a clear,- detailed, and easy-to-read format. Avoid using markdown tables for+- When you complete your task, your final response must be a summary of+ your work and the path to the final document, presented in a clear,+ detailed, and easy-to-read format. Avoid using markdown tables forpresenting data; use plain text formatting instead.<mandatory_instructions><capabilities>Your capabilities include:- Document Reading:- - Read and understand the content of various file formats including+ - Read and understand the content of various file formats including- PDF (.pdf)- Microsoft Office: Word (.doc, .docx), Excel (.xls, .xlsx),PowerPoint (.ppt, .pptx)@@ -475,16 +485,16 @@ Your capabilities include:- ZIP archives (.zip) using the `read_files` tool.- Document Creation & Editing:- - Create and write to various file formats including Markdown (.md),+ - Create and write to various file formats including Markdown (.md),Word documents (.docx), PDFs, CSV files, JSON, YAML, and HTML- - Apply formatting options including custom encoding, font styles, and+ - Apply formatting options including custom encoding, font styles, andlayout settings- Modify existing files with automatic backup functionality- - Support for mathematical expressions in PDF documents through LaTeX+ - Support for mathematical expressions in PDF documents through LaTeXrendering- PowerPoint Presentation Creation:- - Create professional PowerPoint presentations with title slides and+ - Create professional PowerPoint presentations with title slides andcontent slides- Format text with bold and italic styling- Create bullet point lists with proper hierarchical structure@@ -493,7 +503,7 @@ Your capabilities include:- Support for custom templates and slide layouts- Excel Spreadsheet Management:- - Extract and analyze content from Excel files (.xlsx, .xls, .csv)+ - Extract and analyze content from Excel files (.xlsx, .xls, .csv)with detailed cell information and markdown formatting- Create new Excel workbooks from scratch with multiple sheets- Perform comprehensive spreadsheet operations including:@@ -506,13 +516,13 @@ Your capabilities include:- Support for both programmatic data entry and manual cell updates- Terminal and File System:- - You have access to a full suite of terminal tools to interact with+ - You have access to a full suite of terminal tools to interact withthe file system within your working directory (`{WORKING_DIRECTORY}`).- - You can execute shell commands (`shell_exec`), list files, and manage+ - You can execute shell commands (`shell_exec`), list files, and manageyour workspace as needed to support your document creation tasks.- - You can also use the terminal to create data visualizations such as- charts and graphs. For example, you can write a Python script that uses- libraries like `plotly` or `matplotlib` to create a chart and save it+ - You can also use the terminal to create data visualizations such as+ charts and graphs. For example, you can write a Python script that uses+ libraries like `plotly` or `matplotlib` to create a chart and save itas an image file.- Human Interaction:@@ -528,15 +538,15 @@ When working with documents, you should:- Ask clarifying questions when user requirements are ambiguous- Recommend best practices for document organization and presentation- For Excel files, always provide clear data structure and organization-- When creating spreadsheets, consider data relationships and use+- When creating spreadsheets, consider data relationships and useappropriate sheet naming conventions- To include data visualizations, write and execute Python scripts usingthe terminal. Use libraries like `plotly` to generate charts andgraphs, and save them as image files that can be embedded in documents.</document_creation_workflow>-Your goal is to help users efficiently create, modify, and manage their-documents with professional quality and appropriate formatting across all+Your goal is to help users efficiently create, modify, and manage their+documents with professional quality and appropriate formatting across allsupported formats including advanced spreadsheet functionality."""@@ -573,34 +583,34 @@ def multi_modal_agent_factory(model: BaseModelBackend, task_id: str):system_message = f"""<intro>-You are a Multi-Modal Processing Assistant-specialized in analyzing and generating various types of media content.+You are a Multi-Modal Processing Assistant+specialized in analyzing and generating various types of media content.You are working in a team with team members. Your team members are:-- Developer Agent: A skilled coding assistant that can write and execute code,+- Developer Agent: A skilled coding assistant that can write and execute code,run terminal commands, and verify solutions to complete tasks.-- Search Agent: Can search the web, extract webpage content, simulate browser+- Search Agent: Can search the web, extract webpage content, simulate browseractions, and provide relevant information to solve the given task.-- Document Agent: A document processing assistant for creating, modifying, and+- Document Agent: A document processing assistant for creating, modifying, andmanaging various document formats, including presentations.-You are now working in system {platform.system()} with architecture-{platform.machine()} at working directory `{WORKING_DIRECTORY}`. All your+You are now working in system {platform.system()} with architecture+{platform.machine()} at working directory `{WORKING_DIRECTORY}`. All yourwork related to local operations should be done in that directory.-The current time is {datetime.datetime.now()}.+The current date is {datetime.date.today()}.</intro><mandatory_instructions>-- You MUST use the `send_message_to_user` tool to inform the user of every- decision and action you take. Your message must include a short title and+- You MUST use the `send_message_to_user` tool to inform the user of every+ decision and action you take. Your message must include a short title anda one-sentence description. This is a mandatory part of your workflow.-- You MUST use the `read_note` tool to to gather all information collected+- You MUST use the `read_note` tool to to gather all information collectedby other team members and write down your findings in the notes.-- When you complete your task, your final response must be a comprehensive- summary of your analysis or the generated media, presented in a clear,- detailed, and easy-to-read format. Avoid using markdown tables for+- When you complete your task, your final response must be a comprehensive+ summary of your analysis or the generated media, presented in a clear,+ detailed, and easy-to-read format. Avoid using markdown tables forpresenting data; use plain text formatting instead.<mandatory_instructions>@@ -639,7 +649,7 @@ When working with multi-modal content, you should:- Ask clarifying questions when user requirements are ambiguous</multi_modal_processing_workflow>-Your goal is to help users effectively process, understand, and create+Your goal is to help users effectively process, understand, and createmulti-modal content across audio and visual domains."""@@ -659,13 +669,13 @@ def social_medium_agent_factory(model: BaseModelBackend, task_id: str):BaseMessage.make_assistant_message(role_name="Social Medium Agent",content=f"""-You are a Social Media Management Assistant with comprehensive capabilities-across multiple platforms. You MUST use the `send_message_to_user` tool to-inform the user of every decision and action you take. Your message must-include a short title and a one-sentence description. This is a mandatory-part of your workflow. When you complete your task, your final response must-be a comprehensive summary of your actions, presented in a clear, detailed,-and easy-to-read format. Avoid using markdown tables for presenting data;+You are a Social Media Management Assistant with comprehensive capabilities+across multiple platforms. You MUST use the `send_message_to_user` tool to+inform the user of every decision and action you take. Your message must+include a short title and a one-sentence description. This is a mandatory+part of your workflow. When you complete your task, your final response must+be a comprehensive summary of your actions, presented in a clear, detailed,+and easy-to-read format. Avoid using markdown tables for presenting data;use plain text formatting instead.You are now working in `{WORKING_DIRECTORY}`. All your work@@ -674,7 +684,7 @@ related to local operations should be done in that directory.Your integrated toolkits enable you to:1. WhatsApp Business Management (WhatsAppToolkit):- - Send text and template messages to customers via the WhatsApp Business+ - Send text and template messages to customers via the WhatsApp BusinessAPI.- Retrieve business profile information.@@ -707,9 +717,9 @@ Your integrated toolkits enable you to:- Ask questions to users and send messages via console.8. Agent Communication:- - Communicate with other agents using messaging tools when collaboration- is needed. Use `list_available_agents` to see available team members and- `send_message` to coordinate with them, especially when you need content+ - Communicate with other agents using messaging tools when collaboration+ is needed. Use `list_available_agents` to see available team members and+ `send_message` to coordinate with them, especially when you need contentfrom document agents or research from search agents.9. File System Access:@@ -719,7 +729,7 @@ Your integrated toolkits enable you to:When assisting users, always:- Identify which platform's functionality is needed for the task.-- Check if required API credentials are available before attempting+- Check if required API credentials are available before attemptingoperations.- Provide clear explanations of what actions you're taking.- Handle rate limits and API restrictions appropriately.@@ -757,7 +767,7 @@ async def main():# Create a single model backend for all agentsmodel_backend = ModelFactory.create(model_platform=ModelPlatformType.OPENAI,- model_type=ModelType.GPT_4_1_MINI,+ model_type=ModelType.GPT_4_1,model_config_dict={"stream": False,},@@ -775,19 +785,23 @@ async def main():# Create custom agents for the workforcecoordinator_agent = ChatAgent(- f"You are a helpful coordinator. You MUST use the "- f"`send_message_to_user` tool to inform the user of every "- f"decision and action you take. Your message must include a short "- f"title and a one-sentence description. This is a mandatory part "- f"of your workflow. You are now working in "- f"`{WORKING_DIRECTORY}`. "- "All your work related to local operations should be done in that "- "directory. "- "### Note-Taking: You have access to comprehensive note-taking tools "- "for coordinating team activities. Use create_note, append_note, "- "read_note, and list_note to track task progress, team coordination, "- "and important decisions. Create notes for task assignments, "- "progress updates, and coordination activities.",+ system_message=(+ f""""+You are a helpful coordinator.+- You are now working in system {platform.system()} with architecture+{platform.machine()} at working directory `{WORKING_DIRECTORY}`. All your+work related to local operations should be done in that directory.+The current date is {datetime.date.today()}.++- You MUST use the `send_message_to_user` tool to inform the user of every+ decision and action you take. Your message must include a short title and+ a one-sentence description. This is a mandatory part of your workflow.++- If a task assigned to another agent fails, you should re-assign it to the+`Developer_Agent`. The `Developer_Agent` is a powerful agent with terminal+access and can resolve a wide range of issues.+ """+ ),model=model_backend_reason,tools=[send_message_to_user,@@ -797,13 +811,18 @@ async def main():],)task_agent = ChatAgent(- f"You are a helpful task planner. You MUST use the "- f"`send_message_to_user` tool to inform the user of every decision "- f"and action you take. Your message must include a short title and "- f"a one-sentence description. This is a mandatory part of your "- f"workflow. You are now working in `{WORKING_DIRECTORY}`. "- "All your work related to local operations should be done in that "- "directory. ",+ f"""++You are a helpful task planner.+- You are now working in system {platform.system()} with architecture+{platform.machine()} at working directory `{WORKING_DIRECTORY}`. All your+work related to local operations should be done in that directory.+The current date is {datetime.date.today()}.++- You MUST use the `send_message_to_user` tool to inform the user of every+ decision and action you take. Your message must include a short title and+ a one-sentence description. This is a mandatory part of your workflow.+ """,model=model_backend_reason,tools=[send_message_to_user,@@ -922,14 +941,11 @@ async def main():human_task = Task(content=("""-Analyze the UK healthcare industry to support the planning of my next company.-Provide a comprehensive market overview, including current trends, growth-projections, and relevant regulations. Identify the top 5-10 major competitors-in the space, including their names, website URLs, estimated market size or-share, core services or products, key strengths, and notable weaknesses. Also-highlight any significant opportunities, gaps, or underserved segments within-the market. Present all findings in a well-structured, professional html-report.+help me use terminal to open slack, find Regina Bai and read the latest+message she sent to me, think then help me reply to her message+you can use pyaugui to control the mouse and keyboard to send the message.+for each action, you should do a screenshot before your action to check the+current state of the screen"""),id='0',diff --git a/test/storages/vector_storages/test_surreal.py b/test/storages/vector_storages/test_surreal.pyindex 8bc5bad72..a6ffdc936 100644--- a/test/storages/vector_storages/test_surreal.py+++ b/test/storages/vector_storages/test_surreal.py@@ -124,15 +124,18 @@ def test_delete_ids(storage):storage_instance, mock_db = storage# Mock RecordID class- with patch("surrealdb.data.types.record_id.RecordID") as mock_record_id:- mock_record_instance = MagicMock()- mock_record_id.return_value = mock_record_instance+ mock_record_id = MagicMock()+ mock_record_instance = MagicMock()+ mock_record_id.return_value = mock_record_instance- storage_instance.delete(ids=["id1", "id2"])+ # Set up the mock in sys.modules+ sys.modules['surrealdb.data.types.record_id'].RecordID = mock_record_id- assert mock_record_id.call_count == 2- assert mock_db.delete.call_count == 2+ storage_instance.delete(ids=["id1", "id2"])- # Verify RecordID was called with correct parameters- mock_record_id.assert_any_call(storage_instance.table, "id1")- mock_record_id.assert_any_call(storage_instance.table, "id2")+ assert mock_record_id.call_count == 2+ assert mock_db.delete.call_count == 2++ # Verify RecordID was called with correct parameters+ mock_record_id.assert_any_call(storage_instance.table, "id1")+ mock_record_id.assert_any_call(storage_instance.table, "id2")