Skip to content

Building AI agents, atomically

initial commit

Kenny Vaneetvelde32fbe2291604
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..0730d60
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,137 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+site-packages/
+
+# Virtual environment
+venv/
+ENV/
+env/
+venv.bak/
+venv/
+
+# PyInstaller
+# Usually these files are written by a python script from a template
+# before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+# Virtual environment
+venv/
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.env.*
+.venv
+.venv.*
+env/
+env.bak/
+venv/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+chromadb_persist/
+chroma_data/
+chromadb_tmp/
\ No newline at end of file
diff --git a/atomic_agents/agents/basic_chat_agent.py b/atomic_agents/agents/basic_chat_agent.py
new file mode 100644
index 0000000..04242d7
--- /dev/null
+++ b/atomic_agents/agents/basic_chat_agent.py
@@ -0,0 +1,107 @@
+import os
+from groq import Groq
+import instructor
+import openai
+from pydantic import BaseModel, Field
+from datetime import datetime
+from atomic_agents.lib.utils.logger import logger
+from atomic_agents.lib.chat_memory import ChatMemory
+from atomic_agents.lib.vectordb.chroma.manager import ChromaDBDocumentManager
+from atomic_agents.tools.vectordb_query_tool import VectorDBQueryTool
+
+class BasicChatAgentInputSchema(BaseModel):
+ chat_input: str = Field(..., description="The input text for the chat agent.")
+
+class BasicChatAgentResponse(BaseModel):
+ response: str = Field(..., description='The response from the chat agent.')
+
+class BasicChatAgent:
+ def __init__(self, client, model: str = "gpt-3.5-turbo", agent_background: str = "You are a helpful AI assistant.", initial_message: str = None, initial_memory: list = [], vector_db = None, include_context_in_system_prompt: bool = False, input_schema = None, output_schema = None):
+ self.input_schema = input_schema or BasicChatAgentInputSchema
+ self.output_schema = output_schema or BasicChatAgentResponse
+
+ self.client = client
+ self.model = model
+ self.agent_background = agent_background
+ self.memory = ChatMemory()
+ self.include_context_in_system_prompt = include_context_in_system_prompt
+
+ self.vector_db = vector_db
+
+ if initial_memory:
+ self.memory.load_from_dict_list(initial_memory)
+
+ if initial_message:
+ self.memory.add_message("assistant", initial_message)
+
+ def get_system_prompt(self) -> str:
+ current_date = datetime.now().isoformat()
+ system_prompt = (
+ f"{self.agent_background}\n"
+ "\n"
+ f"The current date, in ISO format, is {current_date}.\n"
+ )
+
+ if self.vector_db and self.include_context_in_system_prompt:
+ system_prompt += (
+ f"The AI has access to the following context to answer questions. If certain context is irrelevant to the question, the AI will ignore it.\n"
+ f"Context:\n"
+ )
+
+ for i, document in enumerate(self.vector_db.run(self.vector_db.input_schema(queries=[self.memory.get_history()[-1].content])).results):
+ system_prompt += f"Document {i}. {document.metadata['url']}\n"
+ system_prompt += f"====================\n"
+ system_prompt += f"{document.content}\n"
+ system_prompt += f"====================\n\n"
+ return system_prompt
+
+ def get_response(self, response_model=None) -> BaseModel:
+ if response_model is None:
+ response_model = self.output_schema
+
+ messages = [{"role": "system", "content": self.get_system_prompt()}] + self.memory.get_history()
+ response = self.client.chat.completions.create(
+ model=self.model,
+ messages=messages,
+ response_model=response_model
+ )
+ return response
+
+
+ def run(self, user_input: str) -> str:
+ self.memory.add_message("user", user_input)
+ response = self.get_response(response_model=self.output_schema)
+ self.memory.add_message("assistant", response.model_dump_json())
+ return response
+
+
+if __name__ == "__main__":
+ from rich.console import Console
+ console = Console()
+
+ openai_client = openai.OpenAI()
+ client = instructor.from_openai(openai_client)
+ model = "gpt-3.5-turbo"
+ groq_client = instructor.from_groq(Groq(api_key=os.getenv("GROQ_API_KEY")))
+ groq_model = os.getenv("GROQ_CHAT_MODEL")
+ initial_message = "Hello! How can I assist you today?"
+
+ db = VectorDBQueryTool(db_path='chromadb_persist', collection_name='brainblendai.com', max_results=3)
+ agent = BasicChatAgent(
+ client=groq_client,
+ model=groq_model,
+ initial_message=initial_message,
+ agent_background="You are an AI assistant that, whatever the user input, will provide an answer in Alexandrine verse.",
+ vector_db=db,
+ include_context_in_system_prompt=False
+ )
+ console.print(f'Agent: {initial_message}')
+
+ while True:
+ user_input = input("You: ")
+ if user_input.lower() in ['exit', 'quit']:
+ print("Exiting chat...")
+ break
+
+ response = agent.run(user_input)
+ console.print(f"Agent: {response.response}")
diff --git a/atomic_agents/agents/orchestrator_agent.py b/atomic_agents/agents/orchestrator_agent.py
new file mode 100644
index 0000000..379c1e2
--- /dev/null
+++ b/atomic_agents/agents/orchestrator_agent.py
@@ -0,0 +1,193 @@
+from typing import List, Optional, Union
+import instructor
+import openai
+from pydantic import BaseModel, Field, create_model
+from datetime import datetime
+from atomic_agents.agents.tool_interface_agent import ToolInterfaceAgent
+from atomic_agents.lib.utils.format_tool_message import format_tool_message
+from atomic_agents.lib.utils.logger import logger
+from atomic_agents.lib.chat_memory import ChatMemory
+from atomic_agents.tools.calculator_tool import CalculatorTool
+from atomic_agents.tools.searx import SearxNGSearchTool
+from atomic_agents.tools.user_input_tool import UserInputTool
+from atomic_agents.tools.web_scraping_tool import WebScrapingTool
+
+class OrchestratorAgentInputSchema(BaseModel):
+ chat_input: str = Field(..., description="The input text for the chat agent.")
+
+class MarkdownChatResponse(BaseModel):
+ markdown: str = Field(..., description='The response from the chat agent to the user, in markdown format. ONLY use this when there are no more questions for the user and the task is complete.')
+
+class TreeNode(BaseModel):
+ """
+ A node in a binary yes/no tree. The node can be the root node or a child node. A tree is at least 3 layers deep and should be a complete model with questions and results.
+ This is used to determine which tool to use based on the input and the available tools.
+ Tool names should be mentioned explicitly.
+ """
+
+ is_leaf_node: bool = Field(..., description="Whether the node is a leaf node.")
+ result: Optional[str] = Field(None, description="The result of the tree if the node is a leaf node.", examples=["Use the calculator tool to perform the calculation.", "Use the SearXNG search tool to google for the latest gaming news", "Use the web scraping tool to scrape the website for the latest news."])
+ question: Optional[str] = Field(None, description="The question to ask at this node.", examples=["Can the task be resolved with a web search, or does it require scraping?", "Does the task require a calculation, or is it a search for information?"])
+ if_yes: Optional['TreeNode'] = Field(None, description="The node to follow if the answer to the question is 'yes'.")
+ if_no: Optional['TreeNode'] = Field(None, description="The node to follow if the answer to the question is 'no'.")
+
+
+class PlanningAgentResponse(BaseModel):
+ """
+ Response from the agent that details the next steps to take in the conversation based on the current state, available tools, and the agent's thought process.
+ Tool names should be mentioned explicitly.
+ """
+
+ observation: str = Field(..., description="Observation/summary of the current state of the conversation.")
+ analysis_tree: TreeNode = Field(..., description="Detailed analysis tree of the task and available tools to help determine the plan.")
+
+ plan: List[str] = Field(..., description="The step-by-step plan to solve the task, based on the analysis tree. Must have at least 3 steps in the list.")
+
+class OrchestratorAgentResponse(BaseModel):
+ internal_thought: str = Field(..., description="Internal thought process involved in formulating the response or tool call.")
+
+ response: Union[None] = Field(..., description='Response to the user or tool call from the orchestrator agent.')
+
+ class Config:
+ title = "OrchestratorAgentResponse"
+ description = "Response from the orchestrator agent, which can either be a final answer response or a tool interface agent input."
+ json_schema_extra = {
+ "title": title,
+ "description": description
+ }
+
+
+class OrchestratorAgent:
+ def __init__(
+ self,
+ client,
+ model: str = "gpt-3.5-turbo",
+ agent_background: str = "This is a conversation with a helpful AI assistant.",
+ initial_message: str = None,
+ initial_memory: list = [],
+ available_tools: list = []
+ ):
+ self.client = client
+ self.model = model
+ self.agent_background = agent_background
+ self.memory = ChatMemory()
+ self.available_tools = available_tools
+
+ if initial_memory:
+ self.memory.load_from_dict_list(initial_memory)
+
+ if initial_message:
+ self.memory.add_message("assistant", initial_message)
+
+ new_input_schema_union_list = []
+ for tool in available_tools:
+ new_input_schema_union_list.append(tool.input_schema)
+ new_input_schema_union_list.append(MarkdownChatResponse)
+
+ global OrchestratorAgentResponse
+ OrchestratorAgentResponse = create_model(
+ "OrchestratorAgentResponse",
+ __base__=OrchestratorAgentResponse,
+ response=(Union[tuple(new_input_schema_union_list)], Field(description='Response from the orchestrator agent, which can be a final answer response or a tool interface agent input.'))
+ )
+ logger.verbose(f"Created dynamic OrchestratorAgentResponse model with new input schema union list: {new_input_schema_union_list}")
+
+
+
+ def get_system_prompt(self) -> str:
+ current_date = datetime.now().isoformat()
+ system_prompt = (
+ f"{self.agent_background}\n"
+ "\n"
+ "This AI, when given a task, will always first analyze the task and determine which tool to use through a series of logical examinations and a decision tree.\n"
+ "The AI will then generate a step-by-step plan to solve the task using the chosen tools and intermittently re-evaluate the plan based on the task and the tools.\n"
+ f"The current date, in ISO format, is {current_date}."
+ )
+
+ if len(self.available_tools) > 0:
+ system_prompt += "\n\nAvailable tools:"
+
+ for tool in self.available_tools:
+ system_prompt += f"\n- {tool.tool_name}: {tool.tool_description}"
+
+ return system_prompt
+
+ def get_response(self, response_model):
+ messages = [{"role": "system", "content": self.get_system_prompt()}] + self.memory.get_history()
+ response = self.client.chat.completions.create(
+ model=self.model,
+ messages=messages,
+ response_model=response_model
+ )
+ return response
+
+ def run(self, user_input: str) -> str:
+ self.memory.add_message("user", f"Analyze and execute the plan for the following task: \"{user_input}\"")
+ self.memory.add_message("assistant", self.get_response(PlanningAgentResponse).model_dump_json())
+ response = self.get_response(response_model=OrchestratorAgentResponse)
+ return self.handle_response(response)
+
+ def handle_response(self, response):
+ logger.verbose(f"Handling response: {response}")
+ if isinstance(response.response, MarkdownChatResponse):
+ self.memory.add_message("assistant", response.model_dump_json())
+ else:
+ tool_agent = None
+ for agent in self.available_tools:
+ if isinstance(response.response, agent.input_schema):
+ tool_agent = agent
+ break
+
+ if tool_agent:
+ tool_message = format_tool_message(response.response)
+ logger.verbose(f"Formatted tool message: {tool_message}")
+ tool_output = tool_agent.run(response.response)
+ logger.verbose(f"Tool output: {tool_output}")
+ self.memory.add_message("assistant", "", tool_message=tool_message)
+ self.memory.add_message("tool", tool_output.model_dump_json(), tool_id=tool_message['id'])
+ self.memory.add_message("assistant", self.get_response(PlanningAgentResponse).model_dump_json())
+ response = self.get_response(OrchestratorAgentResponse)
+ logger.verbose(f"ToolInterfaceAgent response: {response}")
+
+ if not isinstance(response.response, MarkdownChatResponse):
+ return self.handle_response(response)
+
+ else:
+ self.memory.add_message("assistant", "I'm sorry, I couldn't understand your request.")
+ return response
+
+if __name__ == "__main__":
+ from rich.console import Console
+ console = Console()
+
+ # openai_client = openai.OpenAI(base_url="http://localhost:1234/v1")
+ openai_client = openai.OpenAI()
+
+ client = instructor.from_openai(openai_client)
+
+ # Instantiate tool instances directly
+ web_search_tool = SearxNGSearchTool(max_results=20)
+ web_scraping_tool = WebScrapingTool()
+ calculator_tool = CalculatorTool()
+ user_input_tool = UserInputTool()
+
+ # Instantiate agents with tool instances
+ web_search_agent = ToolInterfaceAgent(web_search_tool, client=client, model="gpt-3.5-turbo")
+ web_scraping_agent = ToolInterfaceAgent(web_scraping_tool, client=client, model="gpt-3.5-turbo")
+ calculator_agent = ToolInterfaceAgent(calculator_tool, client=client, model="gpt-3.5-turbo")
+ user_input_agent = ToolInterfaceAgent(user_input_tool, client=client, model="gpt-3.5-turbo")
+
+ initial_message = "Hello! How can I assist you today?"
+
+ agent = OrchestratorAgent(client=client, model="gpt-3.5-turbo", available_tools=[web_scraping_tool, web_search_agent, calculator_agent, user_input_tool], initial_message=initial_message)
+
+ console.print(f'Agent: {initial_message}')
+
+ while True:
+ user_input = input("You: ")
+ if user_input.lower() in ['exit', 'quit']:
+ print("Exiting chat...")
+ break
+
+ response = agent.run(user_input)
+ console.print(f"Agent: {response.response.markdown}")
\ No newline at end of file
diff --git a/atomic_agents/agents/tool_interface_agent.py b/atomic_agents/agents/tool_interface_agent.py
new file mode 100644
index 0000000..f726091
--- /dev/null
+++ b/atomic_agents/agents/tool_interface_agent.py
@@ -0,0 +1,151 @@
+from datetime import datetime
+import os
+from typing import Optional, Type
+import instructor
+import openai
+from pydantic import BaseModel, Field, create_model
+from rich.console import Console
+from atomic_agents.lib.chat_memory import ChatMemory
+from atomic_agents.lib.utils.format_tool_message import format_tool_message
+from atomic_agents.tools.base import BaseTool
+from atomic_agents.tools.searx import SearxNGSearchTool
+from atomic_agents.tools.calculator_tool import CalculatorTool
+from atomic_agents.lib.utils.logger import logger
+
+class NextStepsResponse(BaseModel):
+ observation: str = Field(..., description='Observation of the current state of the conversation.')
+ thought: str = Field(..., description='Thought process of the agent.')
+ next_action: str = Field(..., description='Description of the next action the assistant should take.')
+ conclusion: str = Field(..., description='Conclusion of the current state of the conversation.')
+
+ class Config:
+ title = "NextStepsResponse"
+ description = "Response from the agent that details the next steps to take in the conversation based on the current state and the agent's thought process."
+ json_schema_extra = {
+ "title": title,
+ "description": description
+ }
+
+class ToolInterfaceAgentOutput(BaseModel):
+ internal_thought: NextStepsResponse = Field(..., description="Internal thought process involved in formulating the response or tool call.")
+ processed_output: str = Field(..., description="Processed markdown-enabled output of the tool to answer the original query or instruction.")
+
+ class Config:
+ title = "ToolInterfaceAgentOutput"
+ description = "Processed output of the ToolInterfaceAgent. The first property will always be the 'internal_thought' property, which is the internal thought process of the agent."
+ json_schema_extra = {
+ "title": title,
+ "description": description
+ }
+
+class ToolInterfaceAgent:
+ def __init__(
+ self,
+ tool_instance: BaseTool,
+ client, openai_api_key: Optional[str] = None,
+ openai_base_url: Optional[str] = None,
+ model: str = "gpt-3.5-turbo",
+ process_output: bool = True,
+ agent_background: Optional[str] = "This AI agent is designed to interact with tools to perform various tasks. The agent gets human readable input, converts it to a command, and returns the output in a human readable format."
+ ):
+ self.tool_instance = tool_instance
+
+ self.tool_name = f"{self.tool_instance.tool_name}"
+ self.tool_description = f"{self.tool_instance.tool_description}"
+
+ self.client = client
+ self.openai_api_key = openai_api_key or os.getenv("OPENAI_API_KEY")
+ self.openai_base_url = openai_base_url or os.getenv("OPENAI_BASE_URL")
+ self.model = model
+ self.memory = ChatMemory()
+ self.process_output = process_output
+ self.agent_background = agent_background
+
+ self.console = Console()
+
+ self.input_schema = create_model(
+ self.tool_name,
+ tool_input=(str, Field(..., description=f"{self.tool_name} tool input. Presented as a single question or instruction", alias=f'tool_input_{self.tool_name}')),
+ __config__=type('Config', (), {
+ 'title': self.tool_name,
+ 'description': self.tool_description,
+ 'json_schema_extra': {
+ "title": self.tool_name,
+ "description": self.tool_description
+ }
+ })
+ )
+
+ if self.process_output:
+ self.output_schema = ToolInterfaceAgentOutput
+ else:
+ self.output_schema = self.tool_instance.output_schema
+
+ def get_system_prompt(self) -> str:
+ current_date = datetime.now().isoformat()
+ system_prompt = (
+ f"{self.agent_background}\n"
+ "\n"
+ f"The tool being used is {self.tool_name}: {self.tool_description}.\n"
+ f"\n"
+ f"The current date, in ISO format, is {current_date}.\n"
+ )
+ return system_prompt
+
+ def get_response(self, response_schema: Type[BaseModel] = ToolInterfaceAgentOutput) -> Type[BaseModel]:
+ messages = [{"role": "system", "content": self.get_system_prompt()}] + self.memory.get_history()
+ result = self.client.chat.completions.create(
+ model=self.model,
+ response_model=response_schema,
+ messages=messages,
+ )
+ logger.verbose(f"get_response result: {result}")
+ return result
+
+ def run(self, params: Type[BaseModel]) -> Type[BaseModel]:
+ text = params.tool_input
+ self.memory.add_message("user", text)
+ logger.verbose(f"User input added to memory: {text}")
+ input_params = self.get_response(self.tool_instance.input_schema)
+ logger.verbose(f"Input parameters received: {input_params}")
+ tool_message = format_tool_message(input_params)
+ logger.verbose(f"Formatted tool message: {tool_message}")
+ output = self.tool_instance.run(input_params)
+ logger.verbose(f"Tool output: {output}")
+
+ if self.process_output:
+ self.memory.add_message("assistant", "", tool_message=tool_message)
+ self.memory.add_message("tool", output.model_dump_json(), tool_id=tool_message['id'])
+ response = self.get_response(ToolInterfaceAgentOutput)
+ logger.info(f"ToolInterfaceAgent response: {response}")
+ self.memory.add_message("assistant", response.model_dump_json())
+ return response
+ else:
+ return output
+
+
+if __name__ == "__main__":
+ # Initialize the OpenAI client
+ openai_client = instructor.from_openai(
+ openai.OpenAI(
+ api_key=os.getenv("OPENAI_API_KEY"),
+ base_url=os.getenv("OPENAI_BASE_URL")
+ )
+ )
+
+ # Instantiate tool instances directly
+ web_search_tool = SearxNGSearchTool(max_results=20)
+ calculator_tool = CalculatorTool()
+
+ # Instantiate agents with tool instances and the client
+ web_search_agent = ToolInterfaceAgent(web_search_tool, openai_client, model="gpt-3.5-turbo")
+ calculator_agent = ToolInterfaceAgent(calculator_tool, openai_client, model="gpt-3.5-turbo")
+
+ console = Console()
+
+ # Example usage with SearxNGSearchTool
+ web_search_output = web_search_agent.run(web_search_agent.input_schema(tool_input_SearxNGSearchTool="Search for the best restaurants in New York City"))
+ console.print(web_search_output) # Type will be ToolInterfaceAgentOutput
+ console.print("\n====================\n")
+ calculator_output = calculator_agent.run(calculator_agent.input_schema(tool_input_CalculatorTool="Calculate the square root of 144"))
+ console.print(calculator_output) # Type will be ToolInterfaceAgentOutput
\ No newline at end of file
diff --git a/atomic_agents/lib/chat_memory.py b/atomic_agents/lib/chat_memory.py
new file mode 100644
index 0000000..516d8f9
--- /dev/null
+++ b/atomic_agents/lib/chat_memory.py
@@ -0,0 +1,80 @@
+import json
+from typing import List, Dict, Union, Optional
+from pydantic import BaseModel
+
+class Message(BaseModel):
+ """
+ Represents a message in the chat history.
+
+ Attributes:
+ role (str): The role of the message sender (e.g., 'user', 'system', 'tool').
+ content (Union[str, Dict]): The content of the message.
+ tool_calls (Optional[List[Dict]]): Optional list of tool call messages.
+ tool_call_id (Optional[str]): Optional unique identifier for the tool call.
+ """
+ role: str
+ content: Union[str, Dict]
+ tool_calls: Optional[List[Dict]] = None
+ tool_call_id: Optional[str] = None
+
+class ChatMemory:
+ """
+ Manages the chat history for an AI agent.
+
+ Attributes:
+ history (List[Message]): A list of messages representing the chat history.
+ """
+
+ def __init__(self):
+ """
+ Initializes the ChatMemory with an empty history.
+ """
+ self.history: List[Message] = []
+
+ def add_message(self, role: str, content: Union[str, Dict], tool_message: Optional[Dict] = None, tool_id: Optional[str] = None) -> None:
+ """
+ Adds a message to the chat history.
+
+ Args:
+ role (str): The role of the message sender.
+ content (Union[str, Dict]): The content of the message.
+ tool_message (Optional[Dict]): Optional tool message to be included.
+ tool_id (Optional[str]): Optional unique identifier for the tool call.
+ """
+ message = Message(role=role, content=json.dumps(content))
+ if tool_message:
+ if tool_id is None:
+ tool_id = tool_message["id"]
+
+ message.tool_calls = [tool_message]
+ message.tool_call_id = tool_id
+ elif role == "tool":
+ message.tool_call_id = tool_id
+ self.history.append(message)
+
+ def get_history(self) -> List[Message]:
+ """
+ Retrieves the chat history.
+
+ Returns:
+ List[Message]: The list of messages in the chat history.
+ """
+ return self.history
+
+ def to_dict_list(self) -> List[Dict]:
+ """
+ Converts the chat history to a list of dictionaries.
+
+ Returns:
+ List[Dict]: The list of messages as dictionaries.
+ """
+ return [message.model_dump() for message in self.history]
+
+ def load_from_dict_list(self, dict_list: List[Dict]) -> None:
+ """
+ Loads the chat history from a list of dictionaries.
+
+ Args:
+ dict_list (List[Dict]): The list of messages as dictionaries.
+ """
+ self.history = [Message(**message_dict) for message_dict in dict_list]
\ No newline at end of file
diff --git a/atomic_agents/lib/diff_file_contents.py b/atomic_agents/lib/diff_file_contents.py
new file mode 100644
index 0000000..adcd58f
--- /dev/null
+++ b/atomic_agents/lib/diff_file_contents.py
@@ -0,0 +1,69 @@
+import os
+from collections import defaultdict
+
+def diff_file_contents(file_contents_list):
+ """ Find and remove lines that are common in three or more files from the given list of file contents. """
+ line_occurrences = defaultdict(list)
+ for i, lines in enumerate(file_contents_list):
+ for line in set(lines): # Use set to avoid duplicate lines in the same file
+ line_occurrences[line].append(i)
+
+ # Filter lines that appear in three or more files
+ common_lines = {line: files for line, files in line_occurrences.items() if len(files) >= 3}
+
+ # Remove common lines from file contents
+ for line, files in common_lines.items():
+ for file_index in files:
+ file_contents_list[file_index] = [l for l in file_contents_list[file_index] if l != line]
+
+ return file_contents_list
+
+if __name__ == "__main__":
+ # Define mock file contents
+ file1_contents = [
+ "This is a common line",
+ "File 1 specific line",
+ "Another common line",
+ "File 1 unique line"
+ ]
+
+ file2_contents = [
+ "This is a common line",
+ "File 2 specific line",
+ "Another common line",
+ "File 2 unique line"
+ ]
+
+ file3_contents = [
+ "This is a common line",
+ "File 3 specific line",
+ "Another common line",
+ "File 3 unique line"
+ ]
+
+ file4_contents = [
+ "File 4 specific line",
+ "File 4 unique line"
+ ]
+
+ # Create a list of file contents
+ file_contents_list = [file1_contents, file2_contents, file3_contents, file4_contents]
+
+ # Call the diff_file_contents function
+ result = diff_file_contents(file_contents_list)
+
+ # Print the original file contents
+ print("Original file contents:")
+ for i, contents in enumerate(file_contents_list):
+ print(f"File {i+1}:")
+ for line in contents:
+ print(line)
+ print()
+
+ # Print the modified file contents
+ print("Modified file contents:")
+ for i, contents in enumerate(result):
+ print(f"File {i+1}:")
+ for line in contents:
+ print(line)
+ print()
diff --git a/atomic_agents/lib/models/web_document.py b/atomic_agents/lib/models/web_document.py
new file mode 100644
index 0000000..254e527
--- /dev/null
+++ b/atomic_agents/lib/models/web_document.py
@@ -0,0 +1,17 @@
+from pydantic import BaseModel, Field
+
+class WebDocumentMetadata(BaseModel):
+ url: str
+ title: str = Field(default='')
+ description: str = Field(default='')
+ keywords: str = Field(default='')
+ author: str = Field(default='')
+
+class VectorDBDocumentMetadata(WebDocumentMetadata):
+ document_part: int = Field(default=1)
+ document_total_parts: int = Field(default=1)
+
+class Document(BaseModel):
+ content: str
+ metadata: WebDocumentMetadata | VectorDBDocumentMetadata = Field(default_factory=WebDocumentMetadata)
+
diff --git a/atomic_agents/lib/scraping/link_scraper.py b/atomic_agents/lib/scraping/link_scraper.py
new file mode 100644
index 0000000..28eec4d
--- /dev/null
+++ b/atomic_agents/lib/scraping/link_scraper.py
@@ -0,0 +1,80 @@
+import asyncio
+import logging
+from urllib.parse import urlparse, urljoin, urlunparse
+from collections import deque
+from typing import Set, List
+
+import aiohttp
+from bs4 import BeautifulSoup
+
+logging.basicConfig(level=logging.INFO)
+
+class InternalLinkScraper:
+ @staticmethod
+ def _normalize_url(url: str) -> str:
+ parsed_url = urlparse(url)
+ scheme = "https" if parsed_url.scheme == "http" else parsed_url.scheme
+ path = parsed_url.path.rstrip("/")
+ return urlunparse(parsed_url._replace(scheme=scheme, path=path))
+
+ @staticmethod
+ async def _get_internal_links(session: aiohttp.ClientSession, url: str, visited: Set[str], queue: deque, continue_on_error: bool = True):
+ try:
+ logging.info(f"Processing {url}")
+ async with session.get(url) as response:
+ response.raise_for_status()
+ text = await response.text()
+ soup = BeautifulSoup(text, 'html.parser')
+ links = soup.find_all('a', href=True)
+
+ for link in links:
+ href = link['href']
+ if href and not href.startswith("#"):
+ href = InternalLinkScraper._normalize_url(urljoin(url, href))
+ parsed_href = urlparse(href)
+ parsed_base = urlparse(url)
+
+ if parsed_href.netloc == parsed_base.netloc and href not in visited:
+ queue.append(href)
+
+ except Exception as e:
+ logging.error(f"Failed to process {url}: {str(e)}")
+ if not continue_on_error:
+ raise e
+
+ @staticmethod
+ async def scrape_internal_links(start_url: str, continue_on_error: bool = True, max_workers: int = 10) -> List[str]:
+ visited = set()
+ queue = deque([InternalLinkScraper._normalize_url(start_url)])
+
+ async with aiohttp.ClientSession() as session:
+ tasks = []
+ while queue or tasks:
+ while queue and len(tasks) < max_workers:
+ url = queue.popleft()
+ if url not in visited:
+ task = asyncio.create_task(InternalLinkScraper._get_internal_links(session, url, visited, queue, continue_on_error))
+ tasks.append(task)
+ visited.add(url)
+
+ if tasks:
+ done, pending = await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED)
+ tasks = list(pending)
+ for task in done:
+ try:
+ await task
+ except Exception as e:
+ logging.error(f"Error processing task: {str(e)}")
+
+ return list(visited)
+
+if __name__ == "__main__":
+ from time import time
+ start_time = time()
+ url = "https://brainblendai.com"
+ internal_links = asyncio.run(InternalLinkScraper.scrape_internal_links(url))
+ scrape_time = time() - start_time
+ print('SCRAPING STATS')
+ print('============')
+ print(f"Scraped {len(internal_links)} internal links from {url} in {scrape_time:.2f} seconds")
+ print(internal_links)
\ No newline at end of file
diff --git a/atomic_agents/lib/scraping/pdf_to_markdown.py b/atomic_agents/lib/scraping/pdf_to_markdown.py
new file mode 100644
index 0000000..0da73c1
--- /dev/null
+++ b/atomic_agents/lib/scraping/pdf_to_markdown.py
@@ -0,0 +1,54 @@
+import os
+import requests
+from pydantic import BaseModel
+from PyPDF2 import PdfReader
+from io import BytesIO, StringIO
+import markdownify
+
+from atomic_agents.lib.models.web_document import Document, WebDocumentMetadata
+
+class PdfToMarkdownConverter:
+ @staticmethod
+ def _fetch_pdf_content(url):
+ response = requests.get(url)
+ response.raise_for_status()
+ return BytesIO(response.content)
+
+ @staticmethod
+ def _read_pdf(file):
+ pdf = PdfReader(file)
+ content = []
+ for page_num in range(len(pdf.pages)):
+ page = pdf.pages[page_num]
+ content.append(page.extract_text())
+ return "\n".join(content)
+
+ @staticmethod
+ def _convert_to_markdown(text):
+ return markdownify.markdownify(text, heading_style="ATX")
+
+ @staticmethod
+ def convert(url=None, file_path=None):
+ if url:
+ pdf_content = PdfToMarkdownConverter._fetch_pdf_content(url)
+ elif file_path:
+ with open(file_path, 'rb') as f:
+ pdf_content = BytesIO(f.read())
+ else:
+ raise ValueError("Either url or file_path must be provided")
+
+ text_content = PdfToMarkdownConverter._read_pdf(pdf_content)
+ markdown_content = PdfToMarkdownConverter._convert_to_markdown(text_content)
+ metadata = WebDocumentMetadata(url=url or file_path, title="", description="", keywords="", author="")
+
+ return Document(content=markdown_content, metadata=metadata)
+
+if __name__ == '__main__':
+ from rich.console import Console
+ from rich.markdown import Markdown
+ console = Console()
+
+ url = "https://web.pdx.edu/~arhodes/ai5.pdf"
+ document = PdfToMarkdownConverter.convert(url=url)
+
+ console.print(Markdown(document.content))
\ No newline at end of file
diff --git a/atomic_agents/lib/scraping/url_to_markdown.py b/atomic_agents/lib/scraping/url_to_markdown.py
new file mode 100644
index 0000000..a14d884
--- /dev/null
+++ b/atomic_agents/lib/scraping/url_to_markdown.py
@@ -0,0 +1,69 @@
+import os
+from bs4 import BeautifulSoup
+import requests
+from markdownify import MarkdownConverter
+from pydantic import BaseModel, Field
+
+from atomic_agents.lib.models.web_document import Document, WebDocumentMetadata
+
+
+class UrlToMarkdownConverter:
+ METADATA_ATTRS = {
+ 'url': '',
+ 'title': ('title', None),
+ 'description': ('meta', {'name': 'description'}),
+ 'keywords': ('meta', {'name': 'keywords'}),
+ 'author': ('meta', {'name': 'author'})
+ }
+
+ @staticmethod
+ def _fetch_url_content(url):
+ response = requests.get(url)
+ response.raise_for_status()
+ return response.text
+
+ @staticmethod
+ def _parse_html(html):
+ return BeautifulSoup(html, 'html.parser')
+
+ @staticmethod
+ def _extract_metadata(soup, url):
+ metadata_dict = {}
+ for attr, selector in UrlToMarkdownConverter.METADATA_ATTRS.items():
+ if selector:
+ element = soup.find(*selector)
+ metadata_dict[attr] = element.get('content', '').strip() if element else ''
+ else:
+ metadata_dict[attr] = url
+ return WebDocumentMetadata(**metadata_dict)
+
+ @staticmethod
+ def _convert_to_markdown(html):
+ class CustomMarkdownConverter(MarkdownConverter):
+ def convert_img(self, el, text, convert_as_inline):
+ return ''
+ def convert_p(self, el, text, convert_as_inline):
+ return text + '\n\n'
+
+ for script in html.find_all('script'):
+ script.decompose()
+
+ for style in html.find_all('style'):
+ style.decompose()
+
+ converter = CustomMarkdownConverter()
+ return converter.convert(str(html)).strip()
+
+ @staticmethod
+ def convert(url):
+ html = UrlToMarkdownConverter._fetch_url_content(url)
+ soup = UrlToMarkdownConverter._parse_html(html)
+ metadata = UrlToMarkdownConverter._extract_metadata(soup, url)
+ markdown_content = UrlToMarkdownConverter._convert_to_markdown(soup)
+
+ return Document(content=markdown_content, metadata=metadata)
+
+if __name__ == '__main__':
+ url = "https://brainblendai.com"
+ document = UrlToMarkdownConverter.convert(url)
+ print(document)
diff --git a/atomic_agents/lib/utils/doc_splitter.py b/atomic_agents/lib/utils/doc_splitter.py
new file mode 100644
index 0000000..54a87ed
--- /dev/null
+++ b/atomic_agents/lib/utils/doc_splitter.py
@@ -0,0 +1,27 @@
+from typing import List
+from atomic_agents.lib.models.web_document import Document, VectorDBDocumentMetadata
+
+def split_document(document: Document, max_length=1000) -> List[Document]:
+ if len(document.content) <= max_length:
+ return [Document(content=document.content, metadata=VectorDBDocumentMetadata(**document.metadata.model_dump(), document_part=1, document_total_parts=1))]
+
+ parts = []
+ current_part = ''
+ for line in document.content.splitlines(keepends=True):
+ if len(current_part) + len(line) <= max_length:
+ current_part += line
+ else:
+ parts.append(current_part)
+ current_part = line
+
+ if current_part:
+ parts.append(current_part)
+
+ total_parts = len(parts)
+ return [
+ Document(
+ content=part.strip(),
+ metadata=VectorDBDocumentMetadata(**document.metadata.model_dump(), document_part=i+1, document_total_parts=total_parts)
+ )
+ for i, part in enumerate(parts)
+ ]
\ No newline at end of file
diff --git a/atomic_agents/lib/utils/document_to_string.py b/atomic_agents/lib/utils/document_to_string.py
new file mode 100644
index 0000000..a09b2eb
--- /dev/null
+++ b/atomic_agents/lib/utils/document_to_string.py
@@ -0,0 +1,46 @@
+from atomic_agents.lib.models.web_document import Document, VectorDBDocumentMetadata
+
+def document_to_string(document: Document) -> str:
+ output = []
+
+ # Determine the title or URL for the header
+ header_title = document.metadata.title if document.metadata.title else document.metadata.url
+ part_info = ""
+
+ if isinstance(document.metadata, VectorDBDocumentMetadata):
+ part_info = f" (Part {document.metadata.document_part} of {document.metadata.document_total_parts})"
+
+ # Add the header with title and part information
+ output.append(f"--- [{header_title}]{part_info} ---")
+
+ # Add metadata fields except 'title' if it's already in the header
+ for field_name, field_value in document.metadata.model_dump().items():
+ if field_value and field_name != "title":
+ output.append(f"{field_name.replace('_', ' ').title()}: {field_value}")
+
+ # Add the content of the document
+ output.append("Document Content:")
+ output.append(document.content)
+
+ # Join all parts with newlines and return
+ return "\n".join(output) + "\n"
+
+if __name__ == '__main__':
+ # Example usage
+ metadata = VectorDBDocumentMetadata(
+ url="http://example.com",
+ title="Example Document",
+ description="An example document stored in VectorDB.",
+ keywords="example, sample, document",
+ author="John Doe",
+ document_part=1,
+ document_total_parts=3
+ )
+
+ doc = Document(
+ content="This is the content of the document.",
+ metadata=metadata
+ )
+
+ document_str = document_to_string(doc)
+ print(document_str)
diff --git a/atomic_agents/lib/utils/format_tool_message.py b/atomic_agents/lib/utils/format_tool_message.py
new file mode 100644
index 0000000..3e20656
--- /dev/null
+++ b/atomic_agents/lib/utils/format_tool_message.py
@@ -0,0 +1,31 @@
+import json
+from typing import Dict, Type
+import uuid
+
+from pydantic import BaseModel
+
+
+def format_tool_message(tool_call: Type[BaseModel], tool_id: str = None) -> Dict:
+ """
+ Formats a message for a tool call.
+
+ Args:
+ tool_name (str): The type of the tool.
+ tool_calls (Dict): A dictionary containing the tool calls.
+ tool_id (str, optional): The unique identifier for the tool call. If not provided, a random UUID will be generated.
+
+ Returns:
+ Dict: A formatted message dictionary for the tool call.
+ """
+ if tool_id is None:
+ tool_id = str(uuid.uuid4())
+
+ return {
+ "id": tool_id,
+ "type": "function",
+ "function": {
+ "name": tool_call.model_config["title"],
+ "arguments": tool_call.model_dump_json()
+ }
+ }
+
\ No newline at end of file
diff --git a/atomic_agents/lib/utils/logger.py b/atomic_agents/lib/utils/logger.py
new file mode 100644
index 0000000..02b91cc
--- /dev/null
+++ b/atomic_agents/lib/utils/logger.py
@@ -0,0 +1,18 @@
+import logging
+from rich.logging import RichHandler
+
+# Step 1: Define a new logging level
+VERBOSE = 25
+logging.addLevelName(VERBOSE, "VERBOSE")
+
+# Step 2: Add a method to the logging module
+def verbose(self, message, *args, **kwargs):
+ if self.isEnabledFor(VERBOSE):
+ kwargs['stacklevel'] = kwargs.get('stacklevel', 1) + 1
+ self._log(VERBOSE, message, args, **kwargs)
+
+logging.Logger.verbose = verbose
+
+# Configure logging
+logging.basicConfig(level=VERBOSE, handlers=[RichHandler(level=VERBOSE)])
+logger = logging.getLogger("rich")
\ No newline at end of file
diff --git a/atomic_agents/lib/vectordb/chroma/ingest.py b/atomic_agents/lib/vectordb/chroma/ingest.py
new file mode 100644
index 0000000..f3b4057
--- /dev/null
+++ b/atomic_agents/lib/vectordb/chroma/ingest.py
@@ -0,0 +1,48 @@
+import asyncio
+import os
+from typing import List
+from pydantic import HttpUrl
+from atomic_agents.lib.diff_file_contents import diff_file_contents
+from atomic_agents.lib.scraping.link_scraper import InternalLinkScraper
+from atomic_agents.lib.scraping.url_to_markdown import UrlToMarkdownConverter
+from atomic_agents.lib.utils.doc_splitter import split_document
+from atomic_agents.lib.utils.document_to_string import document_to_string
+from atomic_agents.lib.models.web_document import Document, VectorDBDocumentMetadata
+from atomic_agents.lib.vectordb.chroma.manager import ChromaDBDocumentManager
+
+def scrape_and_clean_urls(urls: List[HttpUrl], db_path='chromadb_tmp', collection_name = None, do_diff=True) -> List[Document]:
+ try:
+ web_documents = []
+ for url in urls:
+ try:
+ document = UrlToMarkdownConverter.convert(str(url))
+ web_documents.append(document)
+ print(f"Scraped URL: {url}")
+ except Exception as e:
+ print(f"Error converting URL to markdown: {e}")
+ pass
+
+ if do_diff:
+ cleaned_contents = diff_file_contents([doc.content.split('\n') for doc in web_documents])
+ else:
+ cleaned_contents = [doc.content.split('\n') for doc in web_documents]
+ cleaned_documents = []
+ for content, metadata in zip(cleaned_contents, [doc.metadata for doc in web_documents]):
+ cleaned_documents.append(Document(content='\n'.join(content).strip(), metadata=metadata))
+
+ # Ensure the URL is parsed correctly to get the host
+ db = ChromaDBDocumentManager(db_path, collection_name=collection_name or urls[0].host)
+ db.process_documents(cleaned_documents)
+
+ return cleaned_documents
+ except Exception as e:
+ print(f"Error scraping and cleaning URLs: {e}")
+ return []
+
+if __name__ == "__main__":
+ # Example usage
+ url = "https://brainblendai.com"
+ internal_links = asyncio.run(InternalLinkScraper.scrape_internal_links(url))
+ documents = scrape_and_clean_urls(internal_links, collection_name='brainblendai')
+ for doc in documents:
+ print(document_to_string(doc))
\ No newline at end of file
diff --git a/atomic_agents/lib/vectordb/chroma/manager.py b/atomic_agents/lib/vectordb/chroma/manager.py
new file mode 100644
index 0000000..59e907c
--- /dev/null
+++ b/atomic_agents/lib/vectordb/chroma/manager.py
@@ -0,0 +1,103 @@
+import os
+from typing import List
+import chromadb
+import openai
+from atomic_agents.lib.utils.doc_splitter import split_document
+from atomic_agents.lib.utils.document_to_string import document_to_string
+from atomic_agents.lib.models.web_document import Document, VectorDBDocumentMetadata
+
+class ChromaDBDocumentManager:
+ def __init__(self, db_path, collection_name):
+ self.db = chromadb.PersistentClient(path=db_path)
+ self.collection_name = collection_name
+
+ def process_documents(self, documents: List[Document], max_length=1000):
+ collection = self.db.get_or_create_collection(self.collection_name)
+ for document in documents:
+ document_parts = split_document(document, max_length)
+ for part in document_parts:
+ embedding = self._create_embedding(part.content)
+ collection.upsert(
+ ids=[f"{document.metadata.url}_{part.metadata.document_part}_{part.metadata.document_total_parts}"],
+ embeddings=embedding,
+ metadatas=[part.metadata.model_dump()],
+ documents=[part.content]
+ )
+
+ def create_collection(self):
+ self.db.create_collection(self.collection_name)
+
+ def delete_collection(self):
+ self.db.delete_collection(self.collection_name)
+
+ def query_documents(self, queries: List[str], n_results: int = 3) -> List[Document]:
+ collection = self.db.get_collection(self.collection_name)
+ if not collection:
+ raise ValueError(f"Collection '{self.collection_name}' not found.")
+
+ embeddings = self._create_embedding(queries)
+
+ results = collection.query(
+ query_embeddings=embeddings,
+ n_results=n_results,
+ )
+
+ flattened_result_documents = [doc for sublist in results['documents'] for doc in sublist]
+ flattened_result_metadatas = [metadata for sublist in results['metadatas'] for metadata in sublist]
+ flattened_result_distances = [distance for sublist in results['distances'] for distance in sublist]
+
+ # Sort all lists by distance
+ results = sorted(zip(flattened_result_documents, flattened_result_metadatas, flattened_result_distances), key=lambda x: x[2])
+
+ # Deduplicate results
+ deduplicated_results = []
+ seen_content = set()
+ for content, metadata, distance in results:
+ if content not in seen_content:
+ deduplicated_results.append((content, metadata))
+ seen_content.add(content)
+
+ results = {
+ 'documents': [content for content, metadata in deduplicated_results],
+ 'metadatas': [metadata for content, metadata in deduplicated_results]
+ }
+
+ # Construct Document objects from the results
+ documents = []
+ for content, metadata in list(zip(results['documents'], results['metadatas']))[:n_results]:
+ document = Document(content=content, metadata=VectorDBDocumentMetadata(**metadata))
+ documents.append(document)
+
+ return documents
+
+ def _create_embedding(self, content):
+ embedding_client = os.getenv('EMBEDDING_CLIENT')
+
+ if embedding_client == 'openai':
+ openai_api_key = os.getenv('OPENAI_API_KEY')
+ embedding_client = openai.OpenAI(api_key=openai_api_key)
+ response = embedding_client.embeddings.create(
+ input=content,
+ model="text-embedding-3-small"
+ )
+ elif embedding_client == 'local':
+ local_host = os.getenv('LOCAL_CLIENT_HOST')
+ embedding_client = openai.OpenAI(base_url=local_host, api_key='N/A')
+ response = embedding_client.embeddings.create(
+ input=content,
+ model="N/A"
+ )
+ else:
+ raise ValueError("Unsupported embedding client. Choose 'openai' or 'local'.")
+
+ return [response.data[i].embedding for i in range(len(response.data))]
+
+ @staticmethod
+ def document_to_string(doc: Document) -> str:
+ return document_to_string(doc)
+
+if __name__ == '__main__':
+ db = ChromaDBDocumentManager(db_path='chromadb_persist', collection_name='brainblendai.com')
+ documents = db.query_documents(["What is the best AI assistant?", "What does describeit do?" ,"What is the best AI assistant?"], n_results=3)
+ for doc in documents:
+ print(doc.metadata.url)
\ No newline at end of file
diff --git a/atomic_agents/tools/README.md b/atomic_agents/tools/README.md
new file mode 100644
index 0000000..a5ce5eb
--- /dev/null
+++ b/atomic_agents/tools/README.md
@@ -0,0 +1,124 @@
+# Creating New Tools
+
+This guide will help you create new tools based on the structure and patterns observed in `calculator_tool.py` and `searx.py`.
+
+## Step-by-Step Guide
+
+### 1. Import Necessary Modules
+
+Start by importing the necessary modules. Typically, you will need `os`, `pydantic`, and any other modules specific to your tool's functionality.
+
+```python
+import os
+from pydantic import BaseModel, Field
+# Add other necessary imports here
+```
+
+### 2. Define Input Schema
+
+Create a class for the input schema using `pydantic.BaseModel`. Define the fields and their descriptions. Also, include a nested `Config` class with `title`, `description`, and `json_schema_extra`.
+
+```python
+class YourToolInputSchema(BaseModel):
+ # Define your input fields here
+ example_field: str = Field(..., description="Description of the example field.")
+
+ class Config:
+ title = "YourTool"
+ description = "Tool for performing specific tasks."
+ json_schema_extra = {
+ "title": title,
+ "description": description
+ }
+```
+
+### 3. Define Output Schema
+
+Create a class for the output schema using `pydantic.BaseModel`. Define the fields and their descriptions.
+
+```python
+class YourToolOutputSchema(BaseModel):
+ # Define your output fields here
+ result: str = Field(..., description="Result of the operation.")
+```
+
+### 4. Implement the Tool Logic
+
+Create a class for your tool. Define `input_schema` and `output_schema` attributes. Implement the `run` method to process the input and produce the output.
+
+```python
+class YourTool:
+ input_schema = YourToolInputSchema
+ output_schema = YourToolOutputSchema
+
+ def run(self, params: YourToolInputSchema) -> YourToolOutputSchema:
+ try:
+ # Implement your tool logic here
+ result = "Your result here"
+ return YourToolOutputSchema(result=result)
+ except Exception as e:
+ raise ValueError(f"Error processing input: {e}")
+```
+
+### 5. Example Usage
+
+Include an example usage section to demonstrate how to initialize and use your tool.
+
+```python
+if __name__ == "__main__":
+ # Initialize your tool
+ your_tool = YourTool()
+
+ # Create input parameters
+ input_params = YourToolInputSchema(
+ example_field="Example input"
+ )
+
+ # Run the tool and get the output
+ output = your_tool.run(input_params)
+
+ # Print the result
+ print(output)
+```
+
+## Example
+
+Here is a complete example of a simple tool that echoes the input string.
+
+```python
+import os
+from pydantic import BaseModel, Field
+
+class EchoToolInputSchema(BaseModel):
+ message: str = Field(..., description="Message to echo.")
+
+ class Config:
+ title = "EchoTool"
+ description = "Tool for echoing messages."
+ json_schema_extra = {
+ "title": title,
+ "description": description
+ }
+
+class EchoToolOutputSchema(BaseModel):
+ echoed_message: str = Field(..., description="Echoed message.")
+
+class EchoTool:
+ input_schema = EchoToolInputSchema
+ output_schema = EchoToolOutputSchema
+
+ def run(self, params: EchoToolInputSchema) -> EchoToolOutputSchema:
+ try:
+ echoed_message = params.message
+ return EchoToolOutputSchema(echoed_message=echoed_message)
+ except Exception as e:
+ raise ValueError(f"Error echoing message: {e}")
+
+if __name__ == "__main__":
+ echo_tool = EchoTool()
+ input_params = EchoToolInputSchema(message="Hello, World!")
+ output = echo_tool.run(input_params)
+ print(output)
+```
+
+By following this guide, you can create new tools with a consistent structure and easily integrate them into your projects.
\ No newline at end of file
diff --git a/atomic_agents/tools/base.py b/atomic_agents/tools/base.py
new file mode 100644
index 0000000..92ee767
--- /dev/null
+++ b/atomic_agents/tools/base.py
@@ -0,0 +1,13 @@
+from typing import Type, Optional
+from pydantic import BaseModel
+
+class BaseTool:
+ input_schema: Type[BaseModel]
+ output_schema: Type[BaseModel]
+
+ def __init__(self, tool_description_override: Optional[str] = None):
+ self.tool_name = self.input_schema.Config.title
+ self.tool_description = tool_description_override or self.input_schema.Config.description
+
+ def run(self, params: BaseModel) -> BaseModel:
+ raise NotImplementedError("Subclasses should implement this method")
\ No newline at end of file
diff --git a/atomic_agents/tools/calculator_tool.py b/atomic_agents/tools/calculator_tool.py
new file mode 100644
index 0000000..3c77c8e
--- /dev/null
+++ b/atomic_agents/tools/calculator_tool.py
@@ -0,0 +1,72 @@
+import os
+from typing import Optional
+import openai
+from pydantic import BaseModel, Field
+from sympy import sympify
+import instructor
+from openai import OpenAI
+
+from atomic_agents.tools.base import BaseTool
+
+################
+# INPUT SCHEMA #
+################
+class CalculatorToolSchema(BaseModel):
+ expression: str = Field(..., description="Mathematical expression to evaluate. For example, '2 + 2'.")
+
+ class Config:
+ title = "CalculatorTool"
+ description = "Tool for performing calculations. Supports basic arithmetic operations like addition, subtraction, multiplication, and division, but also more complex operations like exponentiation and trigonometric functions. Use this tool to evaluate mathematical expressions."
+ json_schema_extra = {
+ "title": title,
+ "description": description
+ }
+
+####################
+# OUTPUT SCHEMA(S) #
+####################
+class CalculatorToolOutputSchema(BaseModel):
+ result: str = Field(..., description="Result of the calculation.")
+
+################
+# TOOL LOGIC #
+################
+class CalculatorTool(BaseTool):
+ input_schema = CalculatorToolSchema
+ output_schema = CalculatorToolOutputSchema
+
+ def __init__(self, tool_description_override: Optional[str] = None):
+ super().__init__(tool_description_override)
+
+ def run(self, params: CalculatorToolSchema) -> CalculatorToolOutputSchema:
+ try:
+ # Explicitly convert the string form of the expression
+ parsed_expr = sympify(str(params.expression))
+ # Evaluate the expression numerically
+ result = parsed_expr.evalf()
+ return CalculatorToolOutputSchema(result=str(result))
+ except Exception as e:
+ raise ValueError(f"Error evaluating expression: {e}")
+
+#################
+# EXAMPLE USAGE #
+#################
+if __name__ == "__main__":
+
+ # Initialize the client outside
+ client = instructor.from_openai(
+ openai.OpenAI(
+ api_key=os.getenv("OPENAI_API_KEY"),
+ base_url=os.getenv("OPENAI_BASE_URL")
+ )
+ )
+
+ # Extract structured data from natural language
+ result = client.chat.completions.create(
+ model="gpt-3.5-turbo",
+ response_model=CalculatorTool.input_schema,
+ messages=[{"role": "user", "content": "Calculate 2 + 2"}],
+ )
+
+ # Print the result
+ print(CalculatorTool().run(result))
\ No newline at end of file
diff --git a/atomic_agents/tools/searx.py b/atomic_agents/tools/searx.py
new file mode 100644
index 0000000..6fe3b0b
--- /dev/null
+++ b/atomic_agents/tools/searx.py
@@ -0,0 +1,128 @@
+import os
+import requests
+from pydantic import BaseModel, Field
+from typing import List, Literal, Optional
+from rich.console import Console
+import openai
+import instructor
+
+from atomic_agents.tools.base import BaseTool
+
+################
+# INPUT SCHEMA #
+################
+class SearxNGSearchToolSchema(BaseModel):
+ queries: List[str] = Field(..., description="List of search queries.")
+ category: Optional[Literal["general", "news", "social_media"]] = Field(None, description="Category of the search queries.")
+
+ class Config:
+ title = "SearxNGSearchTool"
+ description = "Tool for searching for information, news, references, and other content on the SearxNG. Returns a list of search results with a short description or content snippet and URLs for further exploration."
+ json_schema_extra = {
+ "title": title,
+ "description": description
+ }
+
+####################
+# OUTPUT SCHEMA(S) #
+####################
+class SearxNGSearchResultSchema(BaseModel):
+ url: str
+ title: str
+ content: Optional[str] = None
+ category: Optional[str] = None
+
+class SearxNGSearchToolOutputSchema(BaseModel):
+ results: List[SearxNGSearchResultSchema]
+
+##############
+# TOOL LOGIC #
+##############
+class SearxNGSearchTool(BaseTool):
+ input_schema = SearxNGSearchToolSchema
+ output_schema = SearxNGSearchToolOutputSchema
+
+ def __init__(self, tool_description_override: Optional[str] = None, max_results: int = 10):
+ super().__init__(tool_description_override)
+ self.max_results = max_results
+
+ def run(self, params: SearxNGSearchToolSchema, max_results: Optional[int] = None) -> SearxNGSearchToolOutputSchema:
+ SEARXNG_BASE_URL = os.getenv('SEARXNG_BASE_URL')
+ if not SEARXNG_BASE_URL:
+ raise ValueError("SEARXNG_BASE_URL environment variable not set")
+
+ all_results = []
+
+ for query in params.queries:
+ # Prepare the query parameters
+ query_params = {
+ 'q': query,
+ 'safesearch': '0',
+ 'format': 'json',
+ 'language': 'en',
+ 'engines': 'bing,duckduckgo,google,startpage,yandex',
+ }
+
+ # Add category to query parameters if it is set
+ if params.category:
+ query_params['categories'] = params.category
+
+ # Make the GET request
+ response = requests.get(f"{SEARXNG_BASE_URL}/search", params=query_params)
+
+ # Check if the request was successful
+ if response.status_code != 200:
+ raise Exception(f"Failed to fetch search results for query '{query}': {response.status_code} {response.reason}")
+
+ results = response.json().get('results', [])
+ all_results.extend(results)
+
+ # Sort the combined results by score in descending order
+ sorted_results = sorted(all_results, key=lambda x: x.get('score', 0), reverse=True)
+
+ # Remove duplicates while preserving order
+ seen_urls = set()
+ unique_results = []
+ for result in sorted_results:
+ if 'content' not in result or 'title' not in result or 'url' not in result:
+ continue
+ if result['url'] not in seen_urls:
+ unique_results.append(result)
+ seen_urls.add(result['url'])
+
+ # Filter results to include only those with the correct category if it is set
+ if params.category:
+ filtered_results = [result for result in unique_results if result.get('category') == params.category]
+ else:
+ filtered_results = unique_results
+
+ filtered_results = filtered_results[:max_results or self.max_results]
+
+ return SearxNGSearchToolOutputSchema(results=filtered_results)
+
+
+#################
+# EXAMPLE USAGE #
+#################
+if __name__ == "__main__":
+ rich_console = Console()
+
+ # Initialize the client outside
+ client = instructor.from_openai(
+ openai.OpenAI(
+ api_key=os.getenv("OPENAI_API_KEY"),
+ base_url=os.getenv("OPENAI_BASE_URL")
+ )
+ )
+
+ # Extract structured data from natural language
+ result = client.chat.completions.create(
+ model="gpt-3.5-turbo",
+ response_model=SearxNGSearchTool.input_schema,
+ messages=[{"role": "user", "content": "Search for the latest PC gaming news of May 2024"}],
+ )
+
+ # Print the result
+ output = SearxNGSearchTool(tool_description_override="derp", max_results=15).run(result)
+ for i, result in enumerate(output.results):
+ rich_console.print(f"{i}. Title: {result.title}, URL: {result.url}")
\ No newline at end of file
diff --git a/atomic_agents/tools/user_input_tool.py b/atomic_agents/tools/user_input_tool.py
new file mode 100644
index 0000000..6d63c63
--- /dev/null
+++ b/atomic_agents/tools/user_input_tool.py
@@ -0,0 +1,60 @@
+import os
+from pydantic import BaseModel, Field
+from typing import Optional
+from rich.console import Console
+import openai
+import instructor
+
+from atomic_agents.tools.base import BaseTool
+
+################
+# INPUT SCHEMA #
+################
+class UserInputToolSchema(BaseModel):
+ question: str = Field(..., description="The question to ask the user.")
+ context: Optional[str] = Field(None, description="Additional context or information to provide to the user.")
+
+ class Config:
+ title = "UserInputTool"
+ description = "Tool for asking clarifying questions to the user. Use this tool when the AI needs more information or clarification from the user."
+ json_schema_extra = {
+ "title": title,
+ "description": description
+ }
+
+####################
+# OUTPUT SCHEMA(S) #
+####################
+class UserInputToolOutputSchema(BaseModel):
+ answer: str = Field(..., description="The user's answer to the question.")
+
+##############
+# TOOL LOGIC #
+##############
+class UserInputTool(BaseTool):
+ input_schema = UserInputToolSchema
+ output_schema = UserInputToolOutputSchema
+
+ def __init__(self, tool_description_override: Optional[str] = None):
+ super().__init__(tool_description_override)
+
+ def run(self, params: UserInputToolSchema) -> UserInputToolOutputSchema:
+ print(f"Question: {params.question}")
+ if params.context:
+ print(f"Context: {params.context}")
+ answer = input("Your answer: ")
+ return UserInputToolOutputSchema(answer=answer)
+
+#################
+# EXAMPLE USAGE #
+#################
+if __name__ == "__main__":
+ rich_console = Console()
+
+ question = "What is your favorite color?"
+
+ user_input_tool = UserInputTool()
+ user_input_params = UserInputToolSchema(question=question)
+ user_input_output = user_input_tool.run(user_input_params)
+
+ rich_console.print(f"User's answer: {user_input_output.answer}")
\ No newline at end of file
diff --git a/atomic_agents/tools/vectordb_query_tool.py b/atomic_agents/tools/vectordb_query_tool.py
new file mode 100644
index 0000000..8c38622
--- /dev/null
+++ b/atomic_agents/tools/vectordb_query_tool.py
@@ -0,0 +1,82 @@
+import os
+from pydantic import BaseModel, Field
+from typing import List, Optional
+from rich.console import Console
+import openai
+import instructor
+
+from atomic_agents.tools.base import BaseTool
+from atomic_agents.lib.vectordb.chroma.manager import ChromaDBDocumentManager
+from atomic_agents.lib.models.web_document import Document
+
+################
+# INPUT SCHEMA #
+################
+class VectorDBQueryToolSchema(BaseModel):
+ queries: List[str] = Field(..., description="List of 3 queries to query the vector database.")
+
+ class Config:
+ title = "VectorDBQueryTool"
+ description = "Tool for querying the Chroma VectorDB."
+ json_schema_extra = {
+ "title": title,
+ "description": description
+ }
+
+####################
+# OUTPUT SCHEMA(S) #
+####################
+class VectorDBQueryResultSchema(BaseModel):
+ content: str
+ metadata: Optional[dict] = None
+
+class VectorDBQueryToolOutputSchema(BaseModel):
+ results: List[VectorDBQueryResultSchema]
+
+##############
+# TOOL LOGIC #
+##############
+class VectorDBQueryTool(BaseTool):
+ input_schema = VectorDBQueryToolSchema
+ output_schema = VectorDBQueryToolOutputSchema
+
+ def __init__(self, db_path: str, collection_name: str, tool_description_override: Optional[str] = None, max_results: int = 10):
+ super().__init__(tool_description_override)
+ self.db_path = db_path
+ self.collection_name = collection_name
+ self.max_results = max_results
+ self.manager = ChromaDBDocumentManager(db_path, collection_name)
+
+ def run(self, params: VectorDBQueryToolSchema) -> VectorDBQueryToolOutputSchema:
+ documents = self.manager.query_documents(params.queries, self.max_results)
+ results = [VectorDBQueryResultSchema(content=doc.content, metadata=doc.metadata.model_dump()) for doc in documents]
+ return VectorDBQueryToolOutputSchema(results=results)
+
+#################
+# EXAMPLE USAGE #
+#################
+if __name__ == "__main__":
+ rich_console = Console()
+
+ # Initialize the client outside
+ client = instructor.from_openai(
+ openai.OpenAI(
+ api_key=os.getenv("OPENAI_API_KEY"),
+ base_url=os.getenv("OPENAI_BASE_URL")
+ )
+ )
+
+ # Extract structured data from natural language
+ result = client.chat.completions.create(
+ model="gpt-3.5-turbo",
+ response_model=VectorDBQueryTool.input_schema,
+ messages=[
+ {"role": "system", "content": "This AI generates 3 queries that can be used to gather context to help with reacting to the user input."},
+ {"role": "user", "content": "What does the describeit tool do?"}
+ ],
+ )
+
+ # Print the result
+ output = VectorDBQueryTool(db_path="chromadb_persist", collection_name="brainblendai.com", max_results=3).run(result)
+ for i, result in enumerate(output.results):
+ rich_console.print(f"{i}. Content: {result.content}, Metadata: {result.metadata}")
\ No newline at end of file
diff --git a/atomic_agents/tools/web_scraping_tool.py b/atomic_agents/tools/web_scraping_tool.py
new file mode 100644
index 0000000..d7eba9b
--- /dev/null
+++ b/atomic_agents/tools/web_scraping_tool.py
@@ -0,0 +1,84 @@
+import os
+from pydantic import BaseModel, Field
+from typing import Optional
+from rich.console import Console
+import openai
+import instructor
+import requests
+
+from atomic_agents.tools.base import BaseTool
+from atomic_agents.lib.scraping.url_to_markdown import UrlToMarkdownConverter
+from atomic_agents.lib.scraping.pdf_to_markdown import PdfToMarkdownConverter
+from atomic_agents.lib.models.web_document import Document
+
+################
+# INPUT SCHEMA #
+################
+class WebScrapingToolSchema(BaseModel):
+ url: str = Field(..., description="URL of the web page to scrape.")
+
+ class Config:
+ title = "WebScrapingTool"
+ description = "Tool for scraping web pages and converting content to markdown in order to extract information or summarize the content."
+ json_schema_extra = {
+ "title": title,
+ "description": description
+ }
+
+####################
+# OUTPUT SCHEMA(S) #
+####################
+class WebScrapingResultSchema(BaseModel):
+ content: str
+ metadata: Optional[dict] = None
+
+class WebScrapingToolOutputSchema(BaseModel):
+ result: WebScrapingResultSchema
+
+##############
+# TOOL LOGIC #
+##############
+class WebScrapingTool(BaseTool):
+ input_schema = WebScrapingToolSchema
+ output_schema = WebScrapingToolOutputSchema
+
+ def __init__(self, tool_description_override: Optional[str] = None):
+ super().__init__(tool_description_override)
+
+ def run(self, params: WebScrapingToolSchema) -> WebScrapingToolOutputSchema:
+ url = params.url
+ response = requests.head(url)
+ content_type = response.headers.get('Content-Type', '')
+
+ if 'application/pdf' in content_type:
+ document = PdfToMarkdownConverter.convert(url=url)
+ else:
+ document = UrlToMarkdownConverter.convert(url=url)
+
+ result = WebScrapingResultSchema(content=document.content, metadata=document.metadata.model_dump())
+ return WebScrapingToolOutputSchema(result=result)
+
+#################
+# EXAMPLE USAGE #
+#################
+if __name__ == "__main__":
+ rich_console = Console()
+
+ # Initialize the client outside
+ client = instructor.from_openai(
+ openai.OpenAI(
+ api_key=os.getenv("OPENAI_API_KEY"),
+ base_url=os.getenv("OPENAI_BASE_URL")
+ )
+ )
+
+ # Extract structured data from natural language
+ result = client.chat.completions.create(
+ model="gpt-3.5-turbo",
+ response_model=WebScrapingTool.input_schema,
+ messages=[{"role": "user", "content": "Scrape the content of https://example.com"}],
+ )
+
+ # Print the result
+ output = WebScrapingTool().run(result)
+ rich_console.print(f"Content: {output.result.content}, Metadata: {output.result.metadata}")
\ No newline at end of file
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 0000000..341a952
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1,183 @@
+aiohttp==3.9.5
+aiosignal==1.3.1
+altair==5.3.0
+annotated-types==0.6.0
+anyio==4.3.0
+asgiref==3.8.1
+astunparse==1.6.3
+attrs==23.2.0
+Babel==2.15.0
+backoff==2.2.1
+bcrypt==4.1.3
+beautifulsoup4==4.12.3
+blinker==1.8.2
+boilerpy3==1.0.7
+build==1.2.1
+cachetools==5.3.3
+certifi==2024.2.2
+chardet==5.2.0
+charset-normalizer==3.3.2
+chroma-hnswlib==0.7.3
+chromadb==0.5.0
+click==8.1.7
+colorama==0.4.6
+coloredlogs==15.0.1
+colour==0.1.5
+contourpy==1.2.1
+courlan==1.1.0
+cssselect==1.2.0
+cycler==0.12.1
+dateparser==1.2.0
+Deprecated==1.2.14
+distro==1.9.0
+dnspython==2.6.1
+docstring_parser==0.16
+dtreeviz==2.2.2
+email_validator==2.1.1
+fastapi==0.111.0
+fastapi-cli==0.0.4
+feedfinder2==0.0.4
+feedparser==6.0.11
+filelock==3.14.0
+flatbuffers==24.3.25
+fonttools==4.52.4
+frozenlist==1.4.1
+fsspec==2024.5.0
+gitdb==4.0.11
+GitPython==3.1.41
+google-auth==2.29.0
+googleapis-common-protos==1.63.0
+goose3==3.1.19
+graphviz==0.20.3
+greenlet==3.0.3
+groq==0.8.0
+grpcio==1.64.0
+h11==0.14.0
+html2text==2024.2.26
+htmldate==1.8.1
+httpcore==1.0.5
+httptools==0.6.1
+httpx==0.27.0
+huggingface-hub==0.23.1
+humanfriendly==10.0
+idna==3.7
+importlib-metadata==7.0.0
+importlib_resources==6.4.0
+iniconfig==2.0.0
+instructor==1.2.6
+jieba3k==0.35.1
+Jinja2==3.1.4
+joblib==1.4.2
+jsonschema==4.22.0
+jsonschema-specifications==2023.12.1
+jusText==3.0.1
+kiwisolver==1.4.5
+kubernetes==29.0.0
+langdetect==1.0.9
+lmql==0.7.3
+lxml==5.1.1
+lxml_html_clean==0.1.1
+markdown-it-py==3.0.0
+markdownify==0.12.1
+MarkupSafe==2.1.5
+matplotlib==3.9.0
+mdurl==0.1.2
+mmh3==4.1.0
+monotonic==1.6
+mpmath==1.3.0
+multidict==6.0.5
+newspaper3k==0.2.8
+nltk==3.8.1
+numpy==1.26.4
+oauthlib==3.2.2
+onnxruntime==1.18.0
+openai==1.30.1
+opentelemetry-api==1.24.0
+opentelemetry-exporter-otlp-proto-common==1.24.0
+opentelemetry-exporter-otlp-proto-grpc==1.24.0
+opentelemetry-instrumentation==0.45b0
+opentelemetry-instrumentation-asgi==0.45b0
+opentelemetry-instrumentation-fastapi==0.45b0
+opentelemetry-proto==1.24.0
+opentelemetry-sdk==1.24.0
+opentelemetry-semantic-conventions==0.45b0
+opentelemetry-util-http==0.45b0
+orjson==3.10.3
+overrides==7.7.0
+packaging==24.0
+pandas==2.2.2
+pillow==10.3.0
+playwright==1.44.0
+playwright-stealth==1.0.6
+pluggy==1.5.0
+posthog==3.5.0
+protobuf==4.25.3
+pyahocorasick==2.1.0
+pyarrow==16.1.0
+pyasn1==0.6.0
+pyasn1_modules==0.4.0
+pydantic==2.7.1
+pydantic-2-mermaid==0.7.3
+pydantic_core==2.18.2
+pydeck==0.9.1
+pyee==11.1.0
+Pygments==2.18.0
+pyparsing==3.1.2
+PyPDF2==3.0.1
+PyPika==0.48.9
+pyproject_hooks==1.1.0
+pyreadline3==3.4.1
+pytest==8.2.1
+python-dateutil==2.9.0.post0
+python-dotenv==1.0.1
+python-multipart==0.0.9
+pytz==2024.1
+PyYAML==6.0.1
+readability-lxml==0.8.1
+referencing==0.35.1
+regex==2024.5.15
+requests==2.31.0
+requests-file==2.0.0
+requests-oauthlib==2.0.0
+rich==13.7.1
+rpds-py==0.18.1
+rsa==4.9
+scikit-learn==1.5.0
+scipy==1.13.1
+sgmllib3k==1.0.0
+shellingham==1.5.4
+six==1.16.0
+smmap==5.0.1
+snakeviz==2.2.0
+sniffio==1.3.1
+soupsieve==2.5
+starlette==0.37.2
+streamlit==1.35.0
+sympy==1.12
+tenacity==8.3.0
+termcolor==2.4.0
+threadpoolctl==3.5.0
+tiktoken==0.7.0
+tinysegmenter==0.3
+tld==0.13
+tldextract==5.1.2
+tokenizers==0.19.1
+toml==0.10.2
+toolz==0.12.1
+tornado==6.4
+tqdm==4.66.4
+trafilatura==1.9.0
+typer==0.12.3
+typing_extensions==4.11.0
+tzdata==2024.1
+tzlocal==5.2
+ujson==5.10.0
+urllib3==2.2.1
+uvicorn==0.29.0
+watchdog==4.0.1
+watchfiles==0.21.0
+websocket-client==1.8.0
+websockets==12.0
+wrapt==1.16.0
+yarl==1.9.4
+zipp==3.18.2