Skip to content

armgit / camel

Public Imported from GitHub
0

🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org

feat: add screenshot toolkit, update notetaking toolkit (#2892)

Wendong-Fan90390342b424
diff --git a/camel/toolkits/__init__.py b/camel/toolkits/__init__.py
index 65ba1f74f..0d99354f1 100644
--- a/camel/toolkits/__init__.py
+++ b/camel/toolkits/__init__.py
@@ -85,6 +85,7 @@ from .craw4ai_toolkit import Crawl4AIToolkit
from .markitdown_toolkit import MarkItDownToolkit
from .note_taking_toolkit import NoteTakingToolkit
from .message_agent_toolkit import AgentCommunicationToolkit
+from .screenshot_toolkit import ScreenshotToolkit
__all__ = [
'BaseToolkit',
@@ -158,4 +159,5 @@ __all__ = [
'MarkItDownToolkit',
'NoteTakingToolkit',
'AgentCommunicationToolkit',
+ 'ScreenshotToolkit',
]
diff --git a/camel/toolkits/markitdown_toolkit.py b/camel/toolkits/markitdown_toolkit.py
index 6f6557a05..d841c821b 100644
--- a/camel/toolkits/markitdown_toolkit.py
+++ b/camel/toolkits/markitdown_toolkit.py
@@ -33,7 +33,7 @@ class MarkItDownToolkit(BaseToolkit):
):
super().__init__(timeout=timeout)
- def load_files(self, file_paths: List[str]) -> Dict[str, str]:
+ def read_files(self, file_paths: List[str]) -> Dict[str, str]:
r"""Scrapes content from a list of files and converts it to Markdown.
This function takes a list of local file paths, attempts to convert
@@ -74,5 +74,5 @@ class MarkItDownToolkit(BaseToolkit):
representing the functions in the toolkit.
"""
return [
- FunctionTool(self.load_files),
+ FunctionTool(self.read_files),
]
diff --git a/camel/toolkits/note_taking_toolkit.py b/camel/toolkits/note_taking_toolkit.py
index 2722ee2ba..ec2643f14 100644
--- a/camel/toolkits/note_taking_toolkit.py
+++ b/camel/toolkits/note_taking_toolkit.py
@@ -11,7 +11,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+import fcntl
import os
+import time
from pathlib import Path
from typing import List, Optional
@@ -20,10 +22,11 @@ from camel.toolkits.function_tool import FunctionTool
class NoteTakingToolkit(BaseToolkit):
- r"""A toolkit for taking notes in a Markdown file.
+ r"""A toolkit for managing and interacting with markdown note files.
- This toolkit allows an agent to create, append to, and update a specific
- Markdown file for note-taking purposes.
+ This toolkit provides tools for creating, reading, appending to, and
+ listing notes. All notes are stored as `.md` files in a dedicated working
+ directory and are tracked in a registry.
"""
def __init__(
@@ -34,12 +37,11 @@ class NoteTakingToolkit(BaseToolkit):
r"""Initialize the NoteTakingToolkit.
Args:
- working_directory (str, optional): The path to the note file.
- If not provided, it will be determined by the
- `CAMEL_WORKDIR` environment variable (if set), saving
- the note as `notes.md` in that directory. If the
+ working_directory (str, optional): The directory path where notes
+ will be stored. If not provided, it will be determined by the
+ `CAMEL_WORKDIR` environment variable (if set). If the
environment variable is not set, it defaults to
- `camel_working_dir/notes.md`.
+ `camel_working_dir`.
timeout (Optional[float]): The timeout for the toolkit.
"""
super().__init__(timeout=timeout)
@@ -47,42 +49,218 @@ class NoteTakingToolkit(BaseToolkit):
if working_directory:
path = Path(working_directory)
elif camel_workdir:
- path = Path(camel_workdir) / "notes.md"
+ path = Path(camel_workdir)
else:
- path = Path("camel_working_dir") / "notes.md"
+ path = Path("camel_working_dir")
self.working_directory = path
- self.working_directory.parent.mkdir(parents=True, exist_ok=True)
+ self.working_directory.mkdir(parents=True, exist_ok=True)
+ self.registry_file = self.working_directory / ".note_register"
+ self._load_registry()
- def append_note(self, content: str) -> str:
- r"""Appends a note to the note file.
+ def append_note(self, note_name: str, content: str) -> str:
+ r"""Appends content to a note.
+
+ If the note does not exist, it will be created with the given content.
+ If the note already exists, the new content will be added to the end of
+ the note.
Args:
- content (str): The content of the note to be appended.
+ note_name (str): The name of the note (without the .md extension).
+ content (str): The content to append to the note.
Returns:
- str: A message indicating the result of the operation.
+ str: A message confirming that the content was appended or the note
+ was created.
"""
try:
- with self.working_directory.open("a", encoding="utf-8") as f:
+ # Reload registry to get latest state
+ self._load_registry()
+ note_path = self.working_directory / f"{note_name}.md"
+ if note_name not in self.registry or not note_path.exists():
+ self.create_note(note_name, content)
+ return f"Note '{note_name}' created with content added."
+
+ with note_path.open("a", encoding="utf-8") as f:
f.write(content + "\n")
- return (
- f"Note successfully appended to in {self.working_directory}."
- )
+ return f"Content successfully appended to '{note_name}.md'."
except Exception as e:
return f"Error appending note: {e}"
- def read_note(self) -> str:
- r"""Reads the content of the note file.
+ def _load_registry(self) -> None:
+ r"""Load the note registry from file with file locking."""
+ max_retries = 5
+ retry_delay = 0.1
+
+ for attempt in range(max_retries):
+ try:
+ if self.registry_file.exists():
+ with open(self.registry_file, 'r') as f:
+ fcntl.flock(f.fileno(), fcntl.LOCK_SH)
+ try:
+ content = f.read().strip()
+ self.registry = (
+ content.split('\n') if content else []
+ )
+ finally:
+ fcntl.flock(f.fileno(), fcntl.LOCK_UN)
+ return
+ else:
+ self.registry = []
+ return
+ except IOError:
+ if attempt < max_retries - 1:
+ time.sleep(retry_delay * (attempt + 1))
+ else:
+ self.registry = []
+ except Exception:
+ self.registry = []
+ return
+
+ def _save_registry(self) -> None:
+ r"""Save the note registry to file with file locking."""
+ max_retries = 5
+ retry_delay = 0.1
+
+ for attempt in range(max_retries):
+ try:
+ with open(self.registry_file, 'w') as f:
+ fcntl.flock(f.fileno(), fcntl.LOCK_EX)
+ try:
+ f.write('\n'.join(self.registry))
+ f.flush()
+ os.fsync(f.fileno())
+ finally:
+ fcntl.flock(f.fileno(), fcntl.LOCK_UN)
+ return
+ except IOError:
+ if attempt < max_retries - 1:
+ time.sleep(retry_delay * (attempt + 1))
+ else:
+ raise
+ except Exception:
+ pass
+
+ def _register_note(self, note_name: str) -> None:
+ r"""Register a new note in the registry with thread-safe operations."""
+ # Reload registry to get latest state
+ self._load_registry()
+ if note_name not in self.registry:
+ self.registry.append(note_name)
+ self._save_registry()
+
+ def create_note(self, note_name: str, content: str = "") -> str:
+ r"""Creates a new note with a unique name.
+
+ This function will create a new file for your note.
+ You must provide a `note_name` that does not already exist. If you want
+ to add content to an existing note, use the `append_note` function
+ instead.
+
+ Args:
+ note_name (str): The name for your new note (without the .md
+ extension). This name must be unique.
+ content (str, optional): The initial content to write in the note.
+ If not provided, an empty note will be created. Defaults to "".
Returns:
- str: The content of the note file, or an error message if the
- file cannot be read.
+ str: A message confirming the creation of the note or an error if
+ the note name is not valid or already exists.
"""
try:
- if not self.working_directory.exists():
- return "Note file does not exist yet."
- return self.working_directory.read_text(encoding="utf-8")
+ note_path = self.working_directory / f"{note_name}.md"
+
+ if note_path.exists():
+ return f"Error: Note '{note_name}.md' already exists."
+
+ note_path.write_text(content, encoding="utf-8")
+ self._register_note(note_name)
+
+ return f"Note '{note_name}.md' successfully created."
+ except Exception as e:
+ return f"Error creating note: {e}"
+
+ def list_note(self) -> str:
+ r"""Lists all the notes you have created.
+
+ This function will show you a list of all your notes, along with their
+ sizes in bytes. This is useful for seeing what notes you have available
+ to read or append to.
+
+ Returns:
+ str: A string containing a list of available notes and their sizes,
+ or a message indicating that no notes have been created yet.
+ """
+ try:
+ # Reload registry to get latest state
+ self._load_registry()
+ if not self.registry:
+ return "No notes have been created yet."
+
+ notes_info = []
+ for note_name in self.registry:
+ note_path = self.working_directory / f"{note_name}.md"
+ if note_path.exists():
+ size = note_path.stat().st_size
+ notes_info.append(f"- {note_name}.md ({size} bytes)")
+ else:
+ notes_info.append(f"- {note_name}.md (file missing)")
+
+ return "Available notes:\n" + "\n".join(notes_info)
+ except Exception as e:
+ return f"Error listing notes: {e}"
+
+ def read_note(self, note_name: Optional[str] = "all_notes") -> str:
+ r"""Reads the content of a specific note or all notes.
+
+ You can use this function in two ways:
+ 1. **Read a specific note:** Provide the `note_name` (without the .md
+ extension) to get the content of that single note.
+ 2. **Read all notes:** Use `note_name="all_notes"` (default), and this
+ function will return the content of all your notes, concatenated
+ together.
+
+ Args:
+ note_name (str, optional): The name of the note you want to read.
+ Defaults to "all_notes" which reads all notes.
+
+ Returns:
+ str: The content of the specified note(s), or an error message if
+ a note cannot be read.
+ """
+ try:
+ # Reload registry to get latest state
+ self._load_registry()
+ if note_name and note_name != "all_notes":
+ if note_name not in self.registry:
+ return (
+ f"Error: Note '{note_name}' is not registered "
+ f"or was not created by this toolkit."
+ )
+ note_path = self.working_directory / f"{note_name}.md"
+ if not note_path.exists():
+ return f"Note file '{note_path.name}' does not exist."
+ return note_path.read_text(encoding="utf-8")
+ else:
+ if not self.registry:
+ return "No notes have been created yet."
+
+ all_notes = []
+ for registered_note in self.registry:
+ note_path = (
+ self.working_directory / f"{registered_note}.md"
+ )
+ if note_path.exists():
+ content = note_path.read_text(encoding="utf-8")
+ all_notes.append(
+ f"=== {registered_note}.md ===\n{content}"
+ )
+ else:
+ all_notes.append(
+ f"=== {registered_note}.md ===\n[File not found]"
+ )
+
+ return "\n\n".join(all_notes)
except Exception as e:
return f"Error reading note: {e}"
@@ -96,4 +274,6 @@ class NoteTakingToolkit(BaseToolkit):
return [
FunctionTool(self.append_note),
FunctionTool(self.read_note),
+ FunctionTool(self.create_note),
+ FunctionTool(self.list_note),
]
diff --git a/camel/toolkits/openai_image_toolkit.py b/camel/toolkits/openai_image_toolkit.py
index 0bedab520..32db1fc73 100644
--- a/camel/toolkits/openai_image_toolkit.py
+++ b/camel/toolkits/openai_image_toolkit.py
@@ -69,7 +69,7 @@ class OpenAIImageToolkit(BaseToolkit):
Literal["transparent", "opaque", "auto"]
] = "auto",
style: Optional[Literal["vivid", "natural"]] = None,
- image_save_path: Optional[str] = "image_save",
+ working_directory: Optional[str] = "image_save",
):
r"""Initializes a new instance of the OpenAIImageToolkit class.
@@ -100,7 +100,7 @@ class OpenAIImageToolkit(BaseToolkit):
The background of the image.(default: :obj:`"auto"`)
style (Optional[Literal["vivid", "natural"]]): The style of the
image.(default: :obj:`None`)
- image_save_path (Optional[str]): The path to save the generated
+ working_directory (Optional[str]): The path to save the generated
image.(default: :obj:`"image_save"`)
"""
super().__init__(timeout=timeout)
@@ -114,7 +114,7 @@ class OpenAIImageToolkit(BaseToolkit):
self.n = n
self.background = background
self.style = style
- self.image_save_path: str = image_save_path or "image_save"
+ self.working_directory: str = working_directory or "image_save"
def base64_to_image(self, base64_string: str) -> Optional[Image.Image]:
r"""Converts a base64 encoded string into a PIL Image object.
@@ -213,7 +213,7 @@ class OpenAIImageToolkit(BaseToolkit):
# Save the image from base64
image_bytes = base64.b64decode(image_b64)
- os.makedirs(self.image_save_path, exist_ok=True)
+ os.makedirs(self.working_directory, exist_ok=True)
# Add index to filename when multiple images
if len(response.data) > 1:
@@ -221,7 +221,7 @@ class OpenAIImageToolkit(BaseToolkit):
else:
filename = f"{image_name}_{uuid.uuid4().hex}.png"
- image_path = os.path.join(self.image_save_path, filename)
+ image_path = os.path.join(self.working_directory, filename)
with open(image_path, "wb") as f:
f.write(image_bytes)
diff --git a/camel/toolkits/screenshot_toolkit.py b/camel/toolkits/screenshot_toolkit.py
new file mode 100644
index 000000000..b45f4fe31
--- /dev/null
+++ b/camel/toolkits/screenshot_toolkit.py
@@ -0,0 +1,128 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+
+import base64
+import io
+import os
+import time
+from pathlib import Path
+from typing import List, Optional
+
+from camel.logger import get_logger
+from camel.toolkits import BaseToolkit, FunctionTool
+from camel.utils import dependencies_required
+from camel.utils.tool_result import ToolResult
+
+logger = get_logger(__name__)
+
+
+class ScreenshotToolkit(BaseToolkit):
+ r"""A toolkit for taking screenshots."""
+
+ @dependencies_required('PIL')
+ def __init__(
+ self,
+ working_directory: Optional[str] = None,
+ timeout: Optional[float] = None,
+ ):
+ r"""Initializes the ScreenshotToolkit.
+
+ Args:
+ working_directory (str, optional): The directory path where notes
+ will be stored. If not provided, it will be determined by the
+ `CAMEL_WORKDIR` environment variable (if set). If the
+ environment variable is not set, it defaults to
+ `camel_working_dir`.
+ timeout (Optional[float]): Timeout for API requests in seconds.
+ (default: :obj:`None`)
+ """
+ from PIL import ImageGrab
+
+ super().__init__(timeout=timeout)
+
+ camel_workdir = os.environ.get("CAMEL_WORKDIR")
+ if working_directory:
+ path = Path(working_directory)
+ elif camel_workdir:
+ path = Path(camel_workdir)
+ else:
+ path = Path("camel_working_dir")
+
+ self.ImageGrab = ImageGrab
+ self.screenshots_dir = path / "screenshots"
+ self.screenshots_dir.mkdir(parents=True, exist_ok=True)
+
+ def take_screenshot(
+ self,
+ save_to_file: bool = True,
+ ) -> ToolResult:
+ r"""Take a screenshot of the entire screen and return it as a
+ base64-encoded image.
+
+ Args:
+ save_to_file (bool): Whether to save the screenshot to a file.
+ (default: :obj:`True`)
+
+ Returns:
+ ToolResult: An object containing:
+ - text (str): A description of the screenshot.
+ - images (List[str]): A list containing one base64-encoded
+ PNG image data URL.
+ """
+ try:
+ # Take screenshot of entire screen
+ screenshot = self.ImageGrab.grab()
+
+ # Save to file if requested
+ file_path = None
+ if save_to_file:
+ # Create directory if it doesn't exist
+ os.makedirs(self.screenshots_dir, exist_ok=True)
+
+ # Generate filename with timestamp
+ timestamp = int(time.time())
+ filename = f"screenshot_{timestamp}.png"
+ file_path = os.path.join(self.screenshots_dir, filename)
+ screenshot.save(file_path)
+ logger.info(f"Screenshot saved to {file_path}")
+
+ # Convert to base64
+ img_buffer = io.BytesIO()
+ screenshot.save(img_buffer, format="PNG")
+ img_buffer.seek(0)
+ img_base64 = base64.b64encode(img_buffer.getvalue()).decode(
+ 'utf-8'
+ )
+ img_data_url = f"data:image/png;base64,{img_base64}"
+
+ # Create result text
+ result_text = "Screenshot captured successfully"
+ if file_path:
+ result_text += f" and saved to {file_path}"
+
+ return ToolResult(text=result_text, images=[img_data_url])
+
+ except Exception as e:
+ logger.error(f"Error taking screenshot: {e}")
+ return ToolResult(text=f"Error taking screenshot: {e}", images=[])
+
+ def get_tools(self) -> List[FunctionTool]:
+ r"""Returns a list of FunctionTool objects for screenshot operations.
+
+ Returns:
+ List[FunctionTool]: List of screenshot functions.
+ """
+ return [
+ FunctionTool(self.take_screenshot),
+ ]
diff --git a/docs/mintlify/reference/camel.toolkits.openai_image_toolkit.mdx b/docs/mintlify/reference/camel.toolkits.openai_image_toolkit.mdx
index 07efbcfd3..ca8efd4f8 100644
--- a/docs/mintlify/reference/camel.toolkits.openai_image_toolkit.mdx
+++ b/docs/mintlify/reference/camel.toolkits.openai_image_toolkit.mdx
@@ -28,7 +28,7 @@ def __init__(
n: Optional[int] = 1,
background: Optional[Literal['transparent', 'opaque', 'auto']] = 'auto',
style: Optional[Literal['vivid', 'natural']] = None,
- image_save_path: Optional[str] = 'image_save'
+ working_directory: Optional[str] = 'image_save'
):
```
@@ -44,7 +44,7 @@ Initializes a new instance of the OpenAIImageToolkit class.
- **n** (Optional[int]): The number of images to generate. (default: :obj:`1`)
- **background** (`Optional[Literal["transparent", "opaque", "auto"]]`): The background of the image.(default: :obj:`"auto"`)
- **style** (`Optional[Literal["vivid", "natural"]]`): The style of the image.(default: :obj:`None`)
-- **image_save_path** (Optional[str]): The path to save the generated image.(default: :obj:`"image_save"`)
+- **working_directory** (Optional[str]): The path to save the generated image.(default: :obj:`"image_save"`)
<a id="camel.toolkits.openai_image_toolkit.OpenAIImageToolkit.base64_to_image"></a>
diff --git a/docs/reference/camel.toolkits.markitdown_toolkit.md b/docs/reference/camel.toolkits.markitdown_toolkit.md
index dd1ab916e..617cc3b2f 100644
--- a/docs/reference/camel.toolkits.markitdown_toolkit.md
+++ b/docs/reference/camel.toolkits.markitdown_toolkit.md
@@ -18,12 +18,12 @@ A class representing a toolkit for MarkItDown.
def __init__(self, timeout: Optional[float] = None):
```
-<a id="camel.toolkits.markitdown_toolkit.MarkItDownToolkit.load_files"></a>
+<a id="camel.toolkits.markitdown_toolkit.MarkItDownToolkit.read_files"></a>
-### load_files
+### read_files
```python
-def load_files(self, file_paths: List[str]):
+def read_files(self, file_paths: List[str]):
```
Scrapes content from a list of files and converts it to Markdown.
diff --git a/docs/reference/camel.toolkits.openai_image_toolkit.md b/docs/reference/camel.toolkits.openai_image_toolkit.md
index 07efbcfd3..ca8efd4f8 100644
--- a/docs/reference/camel.toolkits.openai_image_toolkit.md
+++ b/docs/reference/camel.toolkits.openai_image_toolkit.md
@@ -28,7 +28,7 @@ def __init__(
n: Optional[int] = 1,
background: Optional[Literal['transparent', 'opaque', 'auto']] = 'auto',
style: Optional[Literal['vivid', 'natural']] = None,
- image_save_path: Optional[str] = 'image_save'
+ working_directory: Optional[str] = 'image_save'
):
```
@@ -44,7 +44,7 @@ Initializes a new instance of the OpenAIImageToolkit class.
- **n** (Optional[int]): The number of images to generate. (default: :obj:`1`)
- **background** (`Optional[Literal["transparent", "opaque", "auto"]]`): The background of the image.(default: :obj:`"auto"`)
- **style** (`Optional[Literal["vivid", "natural"]]`): The style of the image.(default: :obj:`None`)
-- **image_save_path** (Optional[str]): The path to save the generated image.(default: :obj:`"image_save"`)
+- **working_directory** (Optional[str]): The path to save the generated image.(default: :obj:`"image_save"`)
<a id="camel.toolkits.openai_image_toolkit.OpenAIImageToolkit.base64_to_image"></a>
diff --git a/examples/workforce/eigent.py b/examples/workforce/eigent.py
index 78be1322a..69d341fa9 100644
--- a/examples/workforce/eigent.py
+++ b/examples/workforce/eigent.py
@@ -13,8 +13,11 @@
# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
import asyncio
+import datetime
import os
+import platform
import uuid
+from typing import Optional
from camel.agents.chat_agent import ChatAgent
from camel.logger import get_logger
@@ -25,7 +28,6 @@ from camel.tasks.task import Task
from camel.toolkits import (
AgentCommunicationToolkit,
AudioAnalysisToolkit,
- Crawl4AIToolkit,
ExcelToolkit,
FileWriteToolkit,
# GoogleDriveMCPToolkit,
@@ -54,9 +56,14 @@ logger = get_logger(__name__)
WORKING_DIRECTORY = os.environ.get("CAMEL_WORKDIR") or "working_dir/"
-def send_message_to_user(message: str) -> str:
+def send_message_to_user(
+ message_title: str,
+ message_description: str,
+ message_attachment: Optional[str] = None,
+) -> str:
r"""Use this tool to send a tidy message to the user, including a
- short title and a one-sentence description.
+ short title and a one-sentence description and an optional attachment like
+ file path or url.
This one-way tool keeps the user informed about your progress,
decisions, or actions. It does not require a response.
@@ -73,15 +80,25 @@ def send_message_to_user(message: str) -> str:
- Give a status update during a long-running task.
Args:
- message (str): The tidy and informative message for the user,
- which should contain a title and a description.
+ message_title (str): The title of the message.
+ message_description (str): The short description.
+ message_attachment (Optional[str]): The attachment of the message.
Returns:
str: Confirmation that the message was successfully sent.
"""
- print(f"\nAgent Message:\n{message}")
- logger.info(f"\nAgent Message:\n{message}")
- return f"Message successfully sent to user: '{message}'"
+ print(
+ f"\nAgent Message:\n{message_title} "
+ f"{message_description} {message_attachment}"
+ )
+ logger.info(
+ f"\nAgent Message:\n{message_title} "
+ f"{message_description} {message_attachment}"
+ )
+ return (
+ f"Message successfully sent to user: '{message_title} "
+ f"{message_description} {message_attachment}'"
+ )
def developer_agent_factory(
@@ -92,81 +109,108 @@ def developer_agent_factory(
tools = [
send_message_to_user,
HumanToolkit().ask_human_via_console,
- *TerminalToolkit(clone_current_env=True).get_tools(),
+ *TerminalToolkit(safe_mode=True, clone_current_env=False).get_tools(),
+ *NoteTakingToolkit(working_directory=WORKING_DIRECTORY).get_tools(),
]
- system_message = f"""You are a skilled coding assistant. You can write and
- execute code by using the available terminal tools. You MUST use the
- `send_message_to_user` tool to inform the user of every decision and
- action you take. Your message must include a short title and a
- one-sentence description. This is a mandatory part of your workflow.
-
- You are now working in `{WORKING_DIRECTORY}`. All your work
- related to local operations should be done in that directory.
-
- Your capabilities include:
- - Writing code to solve tasks. To execute the code, you MUST first save
- it to a file in the workspace (e.g., `script.py`), and then run it using
- the terminal tool (e.g., `python script.py`).
- - Running terminal commands to install packages (e.g., with `pip` or
- `uv`), process files, or test functionality. All files you create should
- be in the designated workspace.
- - You can use `uv` or `pip` to install packages, for example, `uv pip
- install requests` or `pip install requests`.
- - Verifying your solutions through immediate execution and testing in the
- terminal.
- - Utilizing any Python libraries (e.g., requests, BeautifulSoup, pandas,
- etc.) needed for efficient solutions. You can install missing packages
- using `pip` or `uv` in the terminal.
- - Implementing complete, production-ready code rather than theoretical
- examples.
- - Demonstrating results with proper error handling and practical
- implementation.
- - Asking for human input via the console if you are stuck or need
- clarification.
- - Communicating with other agents using messaging tools. You can use
- `list_available_agents` to see available team members and `send_message`
- to coordinate with them for complex tasks requiring collaboration.
-
- ### Terminal Tool Workflow:
- The terminal tools are session-based. You must manage one or more terminal
- sessions to perform your tasks. A session is identified by a unique `id`.
-
- 1. **Execute Commands**: Use `shell_exec(id="...", command="...")` to run
- a command. If the `id` is new, a new session is created.
- Example: `shell_exec(id="session_1", command="ls -l")`
-
- 2. **Manage Long-Running Tasks**: For commands that take time, run them
- in one step, and then use `shell_wait(id="...")` to wait for
- completion. This prevents blocking and allows you to perform other
- tasks in parallel.
-
- 3. **View Output**: Use `shell_view(id="...")` to see the full command
- history and output of a session.
-
- 4. **Run Tasks in Parallel**: Use different session IDs to run multiple
- commands concurrently.
- - `shell_exec(id="install", command="pip install numpy")`
- - `shell_exec(id="test", command="python my_script.py")`
-
- 5. **Interact with Processes**: For commands that require input, you can
- use:
- - `shell_exec(id="...", command="...", interactive=True)` for
- real-time interactive sessions.
- - `shell_write_to_process(id="...", content="...")` to send input to a
- non-interactive running process.
-
- 6. **Stop a Process**: If a process needs to be terminated, use
- `shell_kill_process(id="...")`.
-
- ### Collaboration and Assistance:
- - If you get stuck, encounter an issue you cannot solve (like a CAPTCHA),
- or need clarification, use the `ask_human_via_console` tool.
- - For complex tasks, you can collaborate with other agents. Use
- `list_available_agents` to see your team members and `send_message` to
- communicate with them.
- Remember to manage your terminal sessions. You can create new sessions
- and run commands in them.
+ system_message = f"""
+<intro>
+You are a skilled coding assistant. You can write and execute code by using
+the available terminal tools.
+
+You are working in a team with team members. Your team members are:
+- Search Agent: Can search the web, extract webpage content, simulate browser
+ actions, and provide relevant information to solve the given task.
+- Document Agent: A document processing assistant for creating, modifying, and
+ managing various document formats, including presentations.
+- Multi-Modal Agent: A multi-modal processing assistant for analyzing, and
+ generating media content like audio and images.
+
+You are now working in system {platform.system()} with architecture
+{platform.machine()} at working directory `{WORKING_DIRECTORY}`. All your
+work related to local operations should be done in that directory.
+The current time is {datetime.datetime.now()}.
+</intro>
+
+<mandatory_instructions>
+- You MUST use the `send_message_to_user` tool to inform the user of every
+decision and action you take. Your message must include a short title and
+a one-sentence description. This is a mandatory part of your workflow.
+
+- You MUST use the `read_note` tool to read the notes from other agents.
+<mandatory_instructions>
+
+<capabilities>
+Your capabilities include:
+- Writing code to solve tasks. To execute the code, you MUST first save
+it to a file in the workspace (e.g., `script.py`), and then run it using
+the terminal tool (e.g., `python script.py`).
+- Running terminal commands to install packages (e.g., with `pip3` or
+`uv`), process files, or test functionality. All files you create should
+be in the designated workspace.
+- You can use `uv` or `pip3` to install packages, for example, `uv pip3
+install requests` or `pip3 install requests`.
+- Verifying your solutions through immediate execution and testing in the
+terminal.
+- Utilizing any Python libraries (e.g., requests, BeautifulSoup, pandas,
+etc.) needed for efficient solutions. You can install missing packages
+using `pip3` or `uv` in the terminal.
+- Implementing complete, production-ready code rather than theoretical
+examples.
+- Demonstrating results with proper error handling and practical
+implementation.
+- Asking for human input via the console if you are stuck or need
+clarification.
+- Write and check notes from other agents to keep track of your work.
+</capabilities>
+
+<terminal_tips>
+The terminal tools are session-based. You must manage one or more terminal
+sessions to perform your tasks. A session is identified by a unique `id`.
+
+- Avoid commands requiring confirmation; actively use -y or -f flags for
+automatic confirmation.
+- For commands with excessive output, redirect stdout to a file.
+- Chain multiple commands with `&&` to minimize interruptions.
+- Use the pipe operator `|` to pass command outputs and simplify operations.
+- Use the `ls -F` command to check the execution permissions of files.
+- If you need to install any new packages, use `pip3 install` or `apt-get
+install` after checking the OS type.
+
+- Execute Commands: Use `shell_exec(id="...", command="...")` to run
+ a command. If the `id` is new, a new session is created.
+ Example: `shell_exec(id="session_1", command="ls -l")`
+
+- Manage Long-Running Tasks: For commands that take time, run them
+ in one step, and then use `shell_wait(id="...")` to wait for
+ completion. This prevents blocking and allows you to perform other
+ tasks in parallel.
+
+- View Output: Use `shell_view(id="...")` to see the full command
+ history and output of a session.
+
+- Run Tasks in Parallel: Use different session IDs to run multiple
+ commands concurrently.
+ - `shell_exec(id="install", command="pip3 install numpy")`
+ - `shell_exec(id="test", command="python my_script.py")`
+
+- Interact with Processes: For commands that require input, you can
+use:
+ - `shell_exec(id="...", command="...", interactive=True)` for
+ real-time interactive sessions.
+ - `shell_write_to_process(id="...", content="...")` to send input to a
+ non-interactive running process.
+
+- Stop a Process: If a process needs to be terminated, use
+ `shell_kill_process(id="...")`.
+</terminal_tips>
+
+<collaboration_and_assistance>
+- If you get stuck, encounter an issue you cannot solve (like a CAPTCHA),
+ or need clarification, use the `ask_human_via_console` tool.
+- Document your progress and findings in notes so other agents can build
+ upon your work.
+</collaboration_and_assistance>
"""
return ChatAgent(
@@ -224,82 +268,89 @@ def search_agent_factory(
TerminalToolkit().shell_exec,
send_message_to_user,
HumanToolkit().ask_human_via_console,
- NoteTakingToolkit().append_note,
- *Crawl4AIToolkit().get_tools(),
+ *NoteTakingToolkit(working_directory=WORKING_DIRECTORY).get_tools(),
SearchToolkit().search_exa,
SearchToolkit().search_google,
SearchToolkit().search_bing,
+ *TerminalToolkit(safe_mode=True, clone_current_env=False).get_tools(),
]
- system_message = f"""You are a helpful assistant that can search the web,
- extract webpage content, simulate browser actions, and provide relevant
- information to solve the given task.
+ system_message = f"""
+<intro>
+You are a helpful assistant that can search the web,
+extract webpage content, simulate browser actions, and provide relevant
+information to solve the given task.
+
+You are working in a team with team members. Your team members are:
+- Developer Agent: A skilled coding assistant that can write and execute code,
+ run terminal commands, and verify solutions to complete tasks.
+- Document Agent: A document processing assistant for creating, modifying, and
+ managing various document formats, including presentations.
+- Multi-Modal Agent: A multi-modal processing assistant for analyzing, and
+ generating media content like audio and images.
+
+You are now working in system {platform.system()} with architecture
+{platform.machine()} at working directory `{WORKING_DIRECTORY}`. All your
+work related to local operations should be done in that directory.
+The current time is {datetime.datetime.now()}.
+</intro>
+
+
+<mandatory_instructions>
+- You MUST use the `send_message_to_user` tool to inform the user of every
+ decision and action you take. Your message must include a short title and
+ a one-sentence description. This is a mandatory part of your workflow.
+
+- You MUST use the note-taking tools to record your findings. Ensure notes
+ are detailed, well-organized, and include source URLs. Your notes are
+ crucial for other team members so make sure to write down all the
+ information you find.
- **CRITICAL**: You MUST NOT answer from your own knowledge. All information
+- You MUST only use URLs from trusted sources (e.g., search engine results
+ or links on from search engine like brave search, google search, bing
+ search, etc.). NEVER invent or guess URLs.
+
+- You MUST NOT answer from your own knowledge. All information
MUST be sourced from the web using the available tools. If you don't know
something, find it out using your tools.
-
- You are now working in `{WORKING_DIRECTORY}`. All your work
- related to local operations should be done in that directory.
-
- ### Mandatory Instructions
- 1. **Inform the User**: You MUST use the `send_message_to_user` tool to
- inform the user of every decision and action you take. Your message
- must include a short title and a one-sentence description.
- 2. **Take Detailed Notes**: You MUST use the `append_note` tool to
- record your findings. Ensure notes are detailed, well-organized,
- and include source URLs. Do not overwrite notes unless summarizing;
- append new information. Your notes are crucial for the Document
- Agent.
-
- ### Web Search Workflow
- 1. **Initial Search**: Start with a search engine like `search_google` or
- `search_bing` to get a list of relevant URLs for your research if
- available, the URLs here will be used for `visit_page`.
- 2. **Browser-Based Exploration**: Use the rich browser related toolset to
- investigate websites.
- - **Navigation and Exploration**: Use `visit_page` to open a URL.
- `visit_page` provides a snapshot of currently visible interactive
- elements, not the full page text. To see more content on long
- pages, you MUST use the `scroll(direction='down')` tool. Repeat
- scrolling to ensure you have covered the entire page. Navigate
- with `click`, `back`, and `forward`. Manage multiple pages with
- `switch_tab`.
- - **Analysis**: Use `get_som_screenshot` to understand the page layout
- and identify interactive elements. Since this is a heavy operation,
- only use it when visual analysis is necessary.
- - **Interaction**: Use `type` to fill out forms and `enter` to submit
- or confirm search.
- 3. **Detailed Content Extraction**: Prioritize using the scraping tools
- from `Crawl4AIToolkit` for in-depth information gathering from a
- webpage.
- 4. **Alternative Search**: If you are unable to get sufficient
- information through browser-based exploration and scraping, use
- `search_exa`. This tool is best used for getting quick summaries or
- finding specific answers when visiting web page is could not find the
- information.
-
- ### Guidelines and Best Practices
- - **URL Integrity**: You MUST only use URLs from trusted sources (e.g.,
- search engine results or links on visited pages). NEVER invent or
- guess URLs.
- - **Thoroughness**: If a search query is complex, break it down. If a
- snippet is unhelpful but the URL seems authoritative, visit the page.
- Check subpages for more information.
- - **Local File Operations**: You can use `shell_exec` to perform
- terminal commands within your working directory, such as listing files
- (`ls`) or checking file content (`cat`).
- - **Persistence**: If one method fails, try another. Combine search,
- scraper, and browser tools for comprehensive information gathering.
- - **Collaboration**: Communicate with other agents using `send_message`
- when you need help. Use `list_available_agents` to see who is
- available.
- - **Clarity**: In your response, you should mention the URLs you have
- visited and processed.
-
- ### Handling Obstacles
- - When encountering verification challenges (like login, CAPTCHAs or
+<mandatory_instructions>
+
+<capabilities>
+Your capabilities include:
+- Search and get information from the web using the search tools.
+- Use the rich browser related toolset to investigate websites.
+- Use the terminal tools to perform local operations.
+- Use the note-taking tools to record your findings.
+- Use the human toolkit to ask for help when you are stuck.
+</capabilities>
+
+<web_search_workflow>
+- Initial Search: Start with a search engine like `search_google` or
+ `search_bing` to get a list of relevant URLs for your research if
+ available, the URLs here will be used for `visit_page`.
+- Browser-Based Exploration: Use the rich browser related toolset to
+ investigate websites.
+ - **Navigation and Exploration**: Use `visit_page` to open a URL.
+ `visit_page` provides a snapshot of currently visible interactive
+ elements, not the full page text. To see more content on long
+ pages, Navigate with `click`, `back`, and `forward`. Manage multiple
+ pages with `switch_tab`.
+ - **Analysis**: Use `get_som_screenshot` to understand the page layout
+ and identify interactive elements. Since this is a heavy operation,
+ only use it when visual analysis is necessary.
+ - **Interaction**: Use `type` to fill out forms and `enter` to submit
+ or confirm search.
+- Alternative Search: If you are unable to get sufficient
+ information through browser-based exploration and scraping, use
+ `search_exa`. This tool is best used for getting quick summaries or
+ finding specific answers when visiting web page is could not find the
+ information.
+
+- In your response, you should mention the URLs you have visited and processed.
+
+- When encountering verification challenges (like login, CAPTCHAs or
robot checks), you MUST request help using the human toolkit.
+</web_search_workflow>
"""
return ChatAgent(
@@ -320,94 +371,121 @@ def document_agent_factory(
r"""Factory for creating a document agent, based on user-provided code
structure."""
tools = [
- *FileWriteToolkit().get_tools(),
- *PPTXToolkit().get_tools(),
+ *FileWriteToolkit(working_directory=WORKING_DIRECTORY).get_tools(),
+ *PPTXToolkit(working_directory=WORKING_DIRECTORY).get_tools(),
# *google_drive_mcp_toolkit.get_tools(),
# *RetrievalToolkit().get_tools(),
send_message_to_user,
HumanToolkit().ask_human_via_console,
*MarkItDownToolkit().get_tools(),
*ExcelToolkit().get_tools(),
- NoteTakingToolkit().read_note,
+ *NoteTakingToolkit(working_directory=WORKING_DIRECTORY).get_tools(),
SearchToolkit().search_exa,
- *TerminalToolkit().get_tools(),
+ *TerminalToolkit(safe_mode=True, clone_current_env=False).get_tools(),
]
- system_message = f"""You are a Document Processing Assistant specialized
- in creating, modifying, and managing various document formats. You MUST
- use the `send_message_to_user` tool to inform the user of every decision
- and action you take. Your message must include a short title and a
- one-sentence description. This is a mandatory part of your workflow.
-
- You are now working in `{WORKING_DIRECTORY}`. All your work
- related to local operations should be done in that directory.
-
- Your capabilities include:
-
- 1. Information Gathering:
- - Before creating any document, you MUST use the `read_note` tool to
- get all the information gathered by the Search Agent.
- - The notes contain all the raw data, findings, and sources you need
- to complete your work.
- - You can communicate with other agents using messaging tools when you
- need additional information. Use `list_available_agents` to see
- available team members and `send_message` to request specific data or
- clarifications.
-
- 2. Document Creation & Editing:
- - Create and write to various file formats including Markdown (.md),
- Word documents (.docx), PDFs, CSV files, JSON, YAML, and HTML
- - Apply formatting options including custom encoding, font styles, and
- layout settings
- - Modify existing files with automatic backup functionality
- - Support for mathematical expressions in PDF documents through LaTeX
- rendering
-
- 3. PowerPoint Presentation Creation:
- - Create professional PowerPoint presentations with title slides and
- content slides
- - Format text with bold and italic styling
- - Create bullet point lists with proper hierarchical structure
- - Support for step-by-step process slides with visual indicators
- - Create tables with headers and rows of data
- - Support for custom templates and slide layouts
-
- 4. Excel Spreadsheet Management:
- - Extract and analyze content from Excel files (.xlsx, .xls, .csv)
- with detailed cell information and markdown formatting
- - Create new Excel workbooks from scratch with multiple sheets
- - Perform comprehensive spreadsheet operations including:
- * Sheet creation, deletion, and data clearing
- * Cell-level operations (read, write, find specific values)
- * Row and column manipulation (add, update, delete)
- * Range operations for bulk data processing
- * Data export to CSV format for compatibility
- - Handle complex data structures with proper formatting and validation
- - Support for both programmatic data entry and manual cell updates
-
- 5. Human Interaction:
- - Ask questions to users and receive their responses
- - Send informative messages to users without requiring responses
-
- 6. Terminal and File System:
- - You have access to a full suite of terminal tools to interact with
- the file system within your working directory (`{WORKING_DIRECTORY}`).
- - You can execute shell commands (`shell_exec`), list files, and manage
- your workspace as needed to support your document creation tasks.
-
- When working with documents, you should:
- - Suggest appropriate file formats based on content requirements
- - Maintain proper formatting and structure in all created documents
- - Provide clear feedback about document creation and modification processes
- - Ask clarifying questions when user requirements are ambiguous
- - Recommend best practices for document organization and presentation
- - For Excel files, always provide clear data structure and organization
- - When creating spreadsheets, consider data relationships and use
- appropriate sheet naming conventions
-
- Your goal is to help users efficiently create, modify, and manage their
- documents with professional quality and appropriate formatting across all
- supported formats including advanced spreadsheet functionality."""
+ system_message = f"""
+<intro>
+You are a Document Processing Assistant specialized
+in creating, modifying, and managing various document formats. You MUST
+use the `send_message_to_user` tool to inform the user of every decision
+and action you take. Your message must include a short title and a
+one-sentence description. This is a mandatory part of your workflow.
+
+You are working in a team with team members. Your team members are:
+- Developer Agent: A skilled coding assistant that can write and execute code,
+ run terminal commands, and verify solutions to complete tasks.
+- Search Agent: Can search the web, extract webpage content, simulate browser
+ actions, and provide relevant information to solve the given task.
+- Multi-Modal Agent: A multi-modal processing assistant for analyzing, and
+ generating media content like audio and images.
+
+You are now working in system {platform.system()} with architecture
+{platform.machine()} at working directory `{WORKING_DIRECTORY}`. All your
+work related to local operations should be done in that directory.
+The current time is {datetime.datetime.now()}.
+</intro>
+
+<mandatory_instructions>
+- You MUST use the `send_message_to_user` tool to inform the user of every
+ decision and action you take. Your message must include a short title and
+ a one-sentence description. This is a mandatory part of your workflow.
+
+- Before creating any document, you MUST use the `read_note` tool to to gather
+all information collected by other team members.
+<mandatory_instructions>
+
+<capabilities>
+Your capabilities include:
+- Document Reading:
+ - Read and understand the content of various file formats including
+ - PDF (.pdf)
+ - Microsoft Office: Word (.doc, .docx), Excel (.xls, .xlsx),
+ PowerPoint (.ppt, .pptx)
+ - EPUB (.epub)
+ - HTML (.html, .htm)
+ - Images (.jpg, .jpeg, .png) for OCR
+ - Audio (.mp3, .wav) for transcription
+ - Text-based formats (.csv, .json, .xml, .txt)
+ - ZIP archives (.zip) using the `read_files` tool.
+- Document Creation & Editing:
+ - Create and write to various file formats including Markdown (.md),
+ Word documents (.docx), PDFs, CSV files, JSON, YAML, and HTML
+ - Apply formatting options including custom encoding, font styles, and
+ layout settings
+ - Modify existing files with automatic backup functionality
+ - Support for mathematical expressions in PDF documents through LaTeX
+ rendering
+
+- PowerPoint Presentation Creation:
+ - Create professional PowerPoint presentations with title slides and
+ content slides
+ - Format text with bold and italic styling
+ - Create bullet point lists with proper hierarchical structure
+ - Support for step-by-step process slides with visual indicators
+ - Create tables with headers and rows of data
+ - Support for custom templates and slide layouts
+
+- Excel Spreadsheet Management:
+ - Extract and analyze content from Excel files (.xlsx, .xls, .csv)
+ with detailed cell information and markdown formatting
+ - Create new Excel workbooks from scratch with multiple sheets
+ - Perform comprehensive spreadsheet operations including:
+ * Sheet creation, deletion, and data clearing
+ * Cell-level operations (read, write, find specific values)
+ * Row and column manipulation (add, update, delete)
+ * Range operations for bulk data processing
+ * Data export to CSV format for compatibility
+ - Handle complex data structures with proper formatting and validation
+ - Support for both programmatic data entry and manual cell updates
+
+- Human Interaction:
+ - Ask questions to users and receive their responses
+ - Send informative messages to users without requiring responses
+
+- Terminal and File System:
+ - You have access to a full suite of terminal tools to interact with
+ the file system within your working directory (`{WORKING_DIRECTORY}`).
+ - You can execute shell commands (`shell_exec`), list files, and manage
+ your workspace as needed to support your document creation tasks.
+</capabilities>
+
+<document_creation_workflow>
+When working with documents, you should:
+- Suggest appropriate file formats based on content requirements
+- Maintain proper formatting and structure in all created documents
+- Provide clear feedback about document creation and modification processes
+- Ask clarifying questions when user requirements are ambiguous
+- Recommend best practices for document organization and presentation
+- For Excel files, always provide clear data structure and organization
+- When creating spreadsheets, consider data relationships and use
+appropriate sheet naming conventions
+</document_creation_workflow>
+
+Your goal is to help users efficiently create, modify, and manage their
+documents with professional quality and appropriate formatting across all
+supported formats including advanced spreadsheet functionality.
+ """
return ChatAgent(
system_message=BaseMessage.make_assistant_message(
@@ -431,67 +509,81 @@ def multi_modal_agent_factory(model: BaseModelBackend, task_id: str):
response_format="b64_json",
size="1024x1024",
quality="standard",
+ working_directory=WORKING_DIRECTORY,
).get_tools(),
send_message_to_user,
HumanToolkit().ask_human_via_console,
SearchToolkit().search_exa,
*TerminalToolkit().get_tools(),
+ *NoteTakingToolkit(working_directory=WORKING_DIRECTORY).get_tools(),
]
- system_message = f"""You are a Multi-Modal Processing Assistant
- specialized in analyzing and generating various types of media content.
- You MUST use the `send_message_to_user` tool to inform the user of every
+ system_message = f"""
+<intro>
+You are a Multi-Modal Processing Assistant
+specialized in analyzing and generating various types of media content.
+
+You are working in a team with team members. Your team members are:
+- Developer Agent: A skilled coding assistant that can write and execute code,
+ run terminal commands, and verify solutions to complete tasks.
+- Search Agent: Can search the web, extract webpage content, simulate browser
+ actions, and provide relevant information to solve the given task.
+- Document Agent: A document processing assistant for creating, modifying, and
+ managing various document formats, including presentations.
+
+You are now working in system {platform.system()} with architecture
+{platform.machine()} at working directory `{WORKING_DIRECTORY}`. All your
+work related to local operations should be done in that directory.
+The current time is {datetime.datetime.now()}.
+</intro>
+
+<mandatory_instructions>
+- You MUST use the `send_message_to_user` tool to inform the user of every
decision and action you take. Your message must include a short title and
a one-sentence description. This is a mandatory part of your workflow.
- You are now working in `{WORKING_DIRECTORY}`. All your work
- related to local operations should be done in that directory.
-
- Your capabilities include:
-
- 1. Video & Audio Analysis:
- - Download videos from URLs for analysis.
- - Transcribe speech from audio files to text with high accuracy
- - Answer specific questions about audio content
- - Process audio from both local files and URLs
- - Handle various audio formats including MP3, WAV, and OGG
-
- 2. Image Analysis & Understanding:
- - Generate detailed descriptions of image content
- - Answer specific questions about images
- - Identify objects, text, people, and scenes in images
- - Process images from both local files and URLs
-
- 3. Image Generation:
- - Create high-quality images based on detailed text prompts using DALL-E
- - Generate images in 1024x1792 resolution
- - Save generated images to specified directories
-
- 4. Human Interaction:
- - Ask questions to users and receive their responses
- - Send informative messages to users without requiring responses
-
- 5. Agent Communication:
- - Communicate with other agents using messaging tools when
- collaboration is needed. Use `list_available_agents` to see available
- team members and `send_message` to coordinate with them, especially
- when you need to share analysis results or request additional
- processing capabilities.
-
- 6. File Management:
- - You can use terminal tools to manage files within your working
- directory (`{WORKING_DIRECTORY}`). This is useful for listing
- downloaded media or organizing generated images.
-
- When working with multi-modal content, you should:
- - Provide detailed and accurate descriptions of media content
- - Extract relevant information based on user queries
- - Generate appropriate media when requested
- - Explain your analysis process and reasoning
- - Ask clarifying questions when user requirements are ambiguous
-
- Your goal is to help users effectively process, understand, and create
- multi-modal content across audio and visual domains."""
+- You MUST use the `read_note` tool to to gather all information collected
+ by other team members and write down your findings in the notes.
+<mandatory_instructions>
+
+<capabilities>
+Your capabilities include:
+- Video & Audio Analysis:
+ - Download videos from URLs for analysis.
+ - Transcribe speech from audio files to text with high accuracy
+ - Answer specific questions about audio content
+ - Process audio from both local files and URLs
+ - Handle various audio formats including MP3, WAV, and OGG
+
+- Image Analysis & Understanding:
+ - Generate detailed descriptions of image content
+ - Answer specific questions about images
+ - Identify objects, text, people, and scenes in images
+ - Process images from both local files and URLs
+
+- Image Generation:
+ - Create high-quality images based on detailed text prompts using DALL-E
+ - Generate images in 1024x1792 resolution
+ - Save generated images to specified directories
+
+- Human Interaction:
+ - Ask questions to users and receive their responses
+ - Send informative messages to users without requiring responses
+
+</capabilities>
+
+<multi_modal_processing_workflow>
+When working with multi-modal content, you should:
+- Provide detailed and accurate descriptions of media content
+- Extract relevant information based on user queries
+- Generate appropriate media when requested
+- Explain your analysis process and reasoning
+- Ask clarifying questions when user requirements are ambiguous
+</multi_modal_processing_workflow>
+
+Your goal is to help users effectively process, understand, and create
+multi-modal content across audio and visual domains.
+"""
return ChatAgent(
system_message=BaseMessage.make_assistant_message(
@@ -515,7 +607,6 @@ inform the user of every decision and action you take. Your message must
include a short title and a one-sentence description. This is a mandatory
part of your workflow.
-
You are now working in `{WORKING_DIRECTORY}`. All your work
related to local operations should be done in that directory.
@@ -585,6 +676,9 @@ operations.
HumanToolkit().ask_human_via_console,
SearchToolkit().search_exa,
*TerminalToolkit().get_tools(),
+ *NoteTakingToolkit(
+ working_directory=WORKING_DIRECTORY
+ ).get_tools(),
],
)
@@ -627,10 +721,18 @@ async def main():
f"of your workflow. You are now working in "
f"`{WORKING_DIRECTORY}`. "
"All your work related to local operations should be done in that "
- "directory.",
+ "directory. "
+ "### Note-Taking: You have access to comprehensive note-taking tools "
+ "for coordinating team activities. Use create_note, append_note, "
+ "read_note, and list_note to track task progress, team coordination, "
+ "and important decisions. Create notes for task assignments, "
+ "progress updates, and coordination activities.",
model=model_backend_reason,
tools=[
send_message_to_user,
+ *NoteTakingToolkit(
+ working_directory=WORKING_DIRECTORY
+ ).get_tools(),
],
)
task_agent = ChatAgent(
@@ -640,10 +742,13 @@ async def main():
f"a one-sentence description. This is a mandatory part of your "
f"workflow. You are now working in `{WORKING_DIRECTORY}`. "
"All your work related to local operations should be done in that "
- "directory.",
+ "directory. ",
model=model_backend_reason,
tools=[
send_message_to_user,
+ *NoteTakingToolkit(
+ working_directory=WORKING_DIRECTORY
+ ).get_tools(),
],
)
new_worker_agent = ChatAgent(
@@ -657,11 +762,18 @@ async def main():
"directory. You can also communicate with other agents "
"using messaging tools - use `list_available_agents` to see "
"available team members and `send_message` to coordinate work "
- "and ask for help when needed.",
+ "and ask for help when needed. "
+ "### Note-Taking: You have access to comprehensive note-taking tools "
+ "for documenting work progress and collaborating with team members. "
+ "Use create_note, append_note, read_note, and list_note to track "
+ "your work, share findings, and access information from other agents. "
+ "Create notes for work progress, discoveries, and collaboration "
+ "points.",
model=model_backend,
tools=[
send_message_to_user,
HumanToolkit().ask_human_via_console,
+ *NoteTakingToolkit().get_tools(),
],
)
@@ -743,9 +855,14 @@ async def main():
human_task = Task(
content=(
"""
- go to amazon and find a popular product,
- check the comments and reviews,
- and then write a report about the product.
+Analyze the UK healthcare industry to support the planning of my next company.
+Provide a comprehensive market overview, including current trends, growth
+projections, and relevant regulations. Identify the top 5-10 major competitors
+in the space, including their names, website URLs, estimated market size or
+share, core services or products, key strengths, and notable weaknesses. Also
+highlight any significant opportunities, gaps, or underserved segments within
+the market. Present all findings in a well-structured, professional html
+report.
"""
),
id='0',
diff --git a/test/toolkits/test_note_taking_toolkit.py b/test/toolkits/test_note_taking_toolkit.py
index 3eb68742a..69a19c898 100644
--- a/test/toolkits/test_note_taking_toolkit.py
+++ b/test/toolkits/test_note_taking_toolkit.py
@@ -11,6 +11,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+import shutil
from pathlib import Path
import pytest
@@ -20,41 +21,129 @@ from camel.toolkits import NoteTakingToolkit
@pytest.fixture
def note_taking_toolkit():
- # Create a toolkit with a temporary note file
- return NoteTakingToolkit(working_directory="test_notes.md")
+ # Create a toolkit with a temporary directory
+ test_dir = Path("test_notes_dir")
+ if test_dir.exists():
+ shutil.rmtree(test_dir)
+ toolkit = NoteTakingToolkit(working_directory=str(test_dir))
+ yield toolkit
+ # Cleanup after tests
+ if test_dir.exists():
+ shutil.rmtree(test_dir)
+
+
+def test_create_note(note_taking_toolkit):
+ # Test creating a new note
+ result = note_taking_toolkit.create_note("test_note", "Initial content")
+ assert "successfully created" in result
+
+ # Test creating a note that already exists
+ result = note_taking_toolkit.create_note("test_note", "Another content")
+ assert "already exists" in result
+
+ # Test creating a note with empty content
+ result = note_taking_toolkit.create_note("empty_note")
+ assert "successfully created" in result
def test_append_note(note_taking_toolkit):
- # Test appending a note
- note_file = Path("test_notes.md")
- if note_file.exists():
- note_file.unlink()
+ # Test appending to a non-existent note (should create it)
+ result = note_taking_toolkit.append_note("new_note", "First line")
+ assert "created with content added" in result
+
+ # Test appending to an existing note
+ result = note_taking_toolkit.append_note("new_note", "Second line")
+ assert "successfully appended" in result
- result = note_taking_toolkit.append_note("First note.")
- assert "appended" in result
- assert note_file.read_text() == "First note.\n"
+ # Read and verify content
+ content = note_taking_toolkit.read_note("new_note")
+ assert "First line" in content
+ assert "Second line" in content
- result = note_taking_toolkit.append_note("Second note.")
- assert "appended" in result
- assert note_file.read_text() == "First note.\nSecond note.\n"
- # Clean up the test file
- if note_file.exists():
- note_file.unlink()
+def test_list_note(note_taking_toolkit):
+ # Test listing when no notes exist
+ result = note_taking_toolkit.list_note()
+ assert "No notes have been created yet" in result
+
+ # Create some notes
+ note_taking_toolkit.create_note("note1", "Content 1")
+ note_taking_toolkit.create_note("note2", "Content 2")
+
+ # Test listing notes
+ result = note_taking_toolkit.list_note()
+ assert "Available notes:" in result
+ assert "note1.md" in result
+ assert "note2.md" in result
+ assert "bytes" in result
def test_read_note(note_taking_toolkit):
- # Test reading a note
- note_file = Path("test_notes.md")
- if note_file.exists():
- note_file.unlink()
+ # Test reading non-existent registry
+ result = note_taking_toolkit.read_note()
+ assert "No notes have been created yet" in result
+
+ # Create notes
+ note_taking_toolkit.create_note("note1", "Content of note 1")
+ note_taking_toolkit.create_note("note2", "Content of note 2")
+
+ # Test reading a specific note
+ result = note_taking_toolkit.read_note("note1")
+ assert result == "Content of note 1"
+
+ # Test reading a non-registered note
+ result = note_taking_toolkit.read_note("non_existent")
+ assert "not registered" in result
+
+ # Test reading all notes
+ result = note_taking_toolkit.read_note()
+ assert "=== note1.md ===" in result
+ assert "Content of note 1" in result
+ assert "=== note2.md ===" in result
+ assert "Content of note 2" in result
+
+
+def test_registry_persistence(note_taking_toolkit):
+ # Create a note
+ note_taking_toolkit.create_note("persistent_note", "Test content")
+
+ # Create a new toolkit instance with the same directory
+ toolkit2 = NoteTakingToolkit(
+ working_directory=str(note_taking_toolkit.working_directory)
+ )
+
+ # Verify the note is still in the registry
+ result = toolkit2.read_note("persistent_note")
+ assert result == "Test content"
+
+ # Verify listing shows the note
+ result = toolkit2.list_note()
+ assert "persistent_note.md" in result
+
+
+def test_registry_security(note_taking_toolkit):
+ # Manually create a file that's not in the registry
+ test_dir = note_taking_toolkit.working_directory
+ unauthorized_file = test_dir / "unauthorized.md"
+ unauthorized_file.write_text("Unauthorized content")
+
+ # Try to read the unauthorized file
+ result = note_taking_toolkit.read_note("unauthorized")
+ assert "not registered" in result
+
+ # Verify it doesn't appear in list
+ result = note_taking_toolkit.list_note()
+ assert "unauthorized.md" not in result
- # Test reading non-existent file
- assert "does not exist" in note_taking_toolkit.read_note()
+ # Clean up
+ unauthorized_file.unlink()
- note_taking_toolkit.append_note("Hello, world!")
- assert note_taking_toolkit.read_note() == "Hello, world!\n"
- # Clean up the test file
- if note_file.exists():
- note_file.unlink()
+def test_get_tools(note_taking_toolkit):
+ tools = note_taking_toolkit.get_tools()
+ assert len(tools) == 4
+ tool_names = [tool.func.__name__ for tool in tools]
+ assert "append_note" in tool_names
+ assert "read_note" in tool_names
+ assert "create_note" in tool_names
+ assert "list_note" in tool_names