Building AI agents, atomically
The Big Lint
KennyVaneetvelde85dab967e920
diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.ymlindex 80aab27..8ee71b5 100644--- a/.github/workflows/python-package.yml+++ b/.github/workflows/python-package.yml@@ -27,15 +27,18 @@ jobs:- name: Install dependenciesrun: |python -m pip install --upgrade pip- python -m pip install flake8 pytest+ python -m pip install flake8 pytest blackif [ -f requirements.txt ]; then pip install -r requirements.txt; fipip install -e .+ - name: Check Black formatting+ run: |+ black --check --diff atomic_agents- name: Lint with flake8run: |# stop the build if there are Python syntax errors or undefined namesflake8 atomic_agents --count --select=E9,F63,F7,F82 --show-source --statistics# exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide- # flake8 atomic_agents --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics+ flake8 atomic_agents --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics- name: Test with pytestrun: |- pytest --import-mode=importlib+ pytest --import-mode=importlib\ No newline at end of filediff --git a/atomic_agents/agents/base_chat_agent.py b/atomic_agents/agents/base_chat_agent.pyindex c32160c..06ffab6 100644--- a/atomic_agents/agents/base_chat_agent.py+++ b/atomic_agents/agents/base_chat_agent.py@@ -1,64 +1,73 @@-from typing import List, Optional, Type+from typing import Optional, Type+import instructor-import openaifrom pydantic import BaseModel, Fieldfrom rich.json import JSONfrom atomic_agents.lib.components.chat_memory import ChatMemoryfrom atomic_agents.lib.components.system_prompt_generator import SystemPromptContextProviderBase, SystemPromptGenerator+class BaseAgentIO(BaseModel):"""Base class for input and output schemas for chat agents."""+def __str__(self):return self.model_dump_json()-+def __rich__(self):json_str = self.model_dump_json()return JSON(json_str)+class BaseChatAgentInputSchema(BaseAgentIO):chat_message: str = Field(...,- description='The chat message exchanged between the user and the chat agent. '- 'This represents the message sent by the user.'+ description="The chat message sent by the user to the assistant.",)class Config:- title = 'BaseChatAgentInputSchema'- description = 'This schema represents the user input message exchanged between the user and the chat agent.'+ title = "BaseChatAgentInputSchema"+ description = "This schema represents the user input message exchanged between the user and the chat agent."json_schema_extra = {"title": title,"description": description,}+class BaseChatAgentOutputSchema(BaseAgentIO):chat_message: str = Field(...,- description='The chat message exchanged between the user and the chat agent. '- 'This contains the markdown-enabled response generated by the chat agent.'+ description=(+ "The chat message exchanged between the user and the chat agent. "+ "This contains the markdown-enabled response generated by the chat agent."+ ),)class Config:- title = 'BaseChatAgentOutputSchema'- description = 'This schema represents the response message exchanged between the user and the chat agent.'+ title = "BaseChatAgentOutputSchema"+ description = "This schema represents the response message exchanged between the user and the chat agent."json_schema_extra = {"title": title,"description": description,}+class BaseChatAgentConfig(BaseModel):- client: instructor.client.Instructor = Field(..., description='Client for interacting with the language model.')- model: str = Field("gpt-3.5-turbo", description='The model to use for generating responses.')- memory: Optional[ChatMemory] = Field(None, description='Memory component for storing chat history.')- system_prompt_generator: Optional[SystemPromptGenerator] = Field(None, description='Component for generating system prompts.')- input_schema: Optional[Type[BaseModel]] = Field(None, description='The schema for the input data.')- output_schema: Optional[Type[BaseModel]] = Field(None, description='The schema for the output data.')-+ client: instructor.client.Instructor = Field(..., description="Client for interacting with the language model.")+ model: str = Field("gpt-3.5-turbo", description="The model to use for generating responses.")+ memory: Optional[ChatMemory] = Field(None, description="Memory component for storing chat history.")+ system_prompt_generator: Optional[SystemPromptGenerator] = Field(+ None, description="Component for generating system prompts."+ )+ input_schema: Optional[Type[BaseModel]] = Field(None, description="The schema for the input data.")+ output_schema: Optional[Type[BaseModel]] = Field(None, description="The schema for the output data.")+class Config:arbitrary_types_allowed = True+class BaseChatAgent:"""Base class for chat agents.@@ -75,6 +84,7 @@ class BaseChatAgent:system_prompt_generator (SystemPromptGenerator): Component for generating system prompts.initial_memory (ChatMemory): Initial state of the memory."""+input_schema = BaseChatAgentInputSchemaoutput_schema = BaseChatAgentOutputSchema@@ -105,7 +115,8 @@ class BaseChatAgent:Obtains a response from the language model.Args:- response_model (Type[BaseModel], optional): The schema for the response data. If not set, self.output_schema is used.+ response_model (Type[BaseModel], optional):+ The schema for the response data. If not set, self.output_schema is used.Returns:Type[BaseModel]: The response from the language model.@@ -113,12 +124,13 @@ class BaseChatAgent:if response_model is None:response_model = self.output_schema- messages = [{'role': 'system', 'content': self.system_prompt_generator.generate_prompt()}] + self.memory.get_history()- response = self.client.chat.completions.create(- model=self.model,- messages=messages,- response_model=response_model- )+ messages = [+ {+ "role": "system",+ "content": self.system_prompt_generator.generate_prompt(),+ }+ ] + self.memory.get_history()+ response = self.client.chat.completions.create(model=self.model, messages=messages, response_model=response_model)return responsedef run(self, user_input: Optional[Type[BaseAgentIO]] = None) -> Type[BaseAgentIO]:@@ -155,7 +167,7 @@ class BaseChatAgent:user_input (str): The input text from the user."""self.current_user_input = user_input- self.memory.add_message('user', str(user_input))+ self.memory.add_message("user", str(user_input))def _pre_run(self):"""@@ -170,7 +182,7 @@ class BaseChatAgent:Args:response (Type[BaseModel]): The response from the chat agent."""- self.memory.add_message('assistant', str(response))+ self.memory.add_message("assistant", str(response))def get_context_provider(self, provider_name: str) -> Type[SystemPromptContextProviderBase]:"""@@ -209,4 +221,4 @@ class BaseChatAgent:if provider_name in self.system_prompt_generator.system_prompt_info.context_providers:del self.system_prompt_generator.system_prompt_info.context_providers[provider_name]else:- raise KeyError(f"Context provider '{provider_name}' not found.")\ No newline at end of file+ raise KeyError(f"Context provider '{provider_name}' not found.")diff --git a/atomic_agents/agents/tool_interface_agent.py b/atomic_agents/agents/tool_interface_agent.pyindex 4bbe86b..b8aefc2 100644--- a/atomic_agents/agents/tool_interface_agent.py+++ b/atomic_agents/agents/tool_interface_agent.py@@ -1,14 +1,18 @@import json+from pydantic import Field, create_model+from atomic_agents.agents.base_chat_agent import BaseAgentIO, BaseChatAgent, BaseChatAgentConfigfrom atomic_agents.lib.components.system_prompt_generator import SystemPromptGenerator, SystemPromptInfofrom atomic_agents.lib.tools.base import BaseToolfrom atomic_agents.lib.utils.format_tool_message import format_tool_message+class ToolInterfaceAgentConfig(BaseChatAgentConfig):tool_instance: BaseToolreturn_raw_output: bool = False+class ToolInputModel(BaseAgentIO):tool_input: str = Field(..., description="Tool input. Presented as a single question or instruction")@@ -17,9 +21,10 @@ class ToolInputModel(BaseAgentIO):description = "Default tool description"json_schema_extra = {"title": "Default Tool",- "description": "Default tool description"+ "description": "Default tool description",}+class ToolInterfaceAgent(BaseChatAgent):"""A specialized chat agent designed to interact with a specific tool.@@ -40,47 +45,62 @@ class ToolInterfaceAgent(BaseChatAgent):config (ToolInterfaceAgentConfig): Configuration for the tool interface agent."""super().__init__(config=config)-+self.tool_instance = config.tool_instanceself.return_raw_output = config.return_raw_output-+# Create a new model with the updated schemaself.input_schema = create_model(self.tool_instance.tool_name,- tool_input=(str, Field(..., description=f"{self.tool_instance.tool_name} tool input. Presented as a single question or instruction", alias=f'tool_input_{self.tool_instance.tool_name}')),- __base__=ToolInputModel+ tool_input=(+ str,+ Field(+ ...,+ description=f"{self.tool_instance.tool_name} tool input. Presented as a single question or instruction",+ alias=f"tool_input_{self.tool_instance.tool_name}",+ ),+ ),+ __base__=ToolInputModel,)-+# Manually set the configuration attributes- self.input_schema.model_config['title'] = self.tool_instance.tool_name- self.input_schema.model_config['description'] = self.tool_instance.tool_description- self.input_schema.model_config['json_schema_extra'] = {- 'title': self.tool_instance.tool_name,- 'description': self.tool_instance.tool_description+ self.input_schema.model_config["title"] = self.tool_instance.tool_name+ self.input_schema.model_config["description"] = self.tool_instance.tool_description+ self.input_schema.model_config["json_schema_extra"] = {+ "title": self.tool_instance.tool_name,+ "description": self.tool_instance.tool_description,}-+if self.return_raw_output:self.output_schema = self.tool_instance.output_schema-++ output_instructions = [+ "Make sure the tool call will maximize the utility of the tool in the context of the user input.",+ ]++ if self.return_raw_output:+ output_instructions.append(+ "Process the output of the tool into a human readable format " "and/or use it to respond to the user input."+ )+ else:+ output_instructions.append("Return the raw output of the tool.")+self.system_prompt_generator = SystemPromptGenerator(system_prompt_info=SystemPromptInfo(background=[f"This AI agent is designed to interact with the {self.tool_instance.tool_name} tool.",- f"Tool description: {self.tool_instance.tool_description}"+ f"Tool description: {self.tool_instance.tool_description}",],steps=["Get the user input.","Convert the input to the proper parameters to call the tool.","Call the tool with the parameters.",- "Respond to the user"+ "Respond to the user",],- output_instructions=[- "Make sure the tool call will maximize the utility of the tool in the context of the user input.",- "Process the output of the tool into a human readable format and/or use it to respond to the user input." if self.return_raw_output else "Return the raw output of the tool."- ]+ output_instructions=output_instructions,))-+def _get_and_handle_response(self):"""Handles obtaining and processing the response from the tool.@@ -88,20 +108,23 @@ class ToolInterfaceAgent(BaseChatAgent):This method gets the response from the tool, formats the tool input, adds it to memory,runs the tool, and processes the tool output. If `return_raw_output` is True, it returnsthe raw tool output; otherwise, it processes the output and returns the response.-+Returns:BaseModel: The processed response or raw tool output."""tool_input = self.get_response(response_model=self.tool_instance.input_schema)formatted_tool_input = format_tool_message(tool_input)-- self.memory.add_message('assistant', 'TOOL CALL: ' + json.dumps(formatted_tool_input))++ self.memory.add_message("assistant", "TOOL CALL: " + json.dumps(formatted_tool_input))tool_output = self.tool_instance.run(tool_input)- self.memory.add_message('assistant', 'TOOL RESPONSE: ' + tool_output.model_dump_json())-+ self.memory.add_message("assistant", "TOOL RESPONSE: " + tool_output.model_dump_json())+if self.return_raw_output:return tool_output-- self.memory.add_message('assistant', 'I will now formulate a response for the user based on the tool output.')- response = self.get_response(response_model=self.output_schema)++ self.memory.add_message(+ "assistant",+ "I will now formulate a response for the user based on the tool output.",+ )+ response = self.get_response(response_model=self.output_schema)return responsediff --git a/atomic_agents/lib/components/chat_memory.py b/atomic_agents/lib/components/chat_memory.pyindex d8bb47c..91fcf7e 100644--- a/atomic_agents/lib/components/chat_memory.py+++ b/atomic_agents/lib/components/chat_memory.py@@ -1,6 +1,8 @@-from typing import List, Dict, Union, Optional+from typing import Dict, List, Optional, Union+from pydantic import BaseModel, ValidationError+class Message(BaseModel):"""Represents a message in the chat history.@@ -11,11 +13,13 @@ class Message(BaseModel):tool_calls (Optional[List[Dict]]): Optional list of tool call messages.tool_call_id (Optional[str]): Optional unique identifier for the tool call."""+role: strcontent: Union[str, Dict]tool_calls: Optional[List[Dict]] = Nonetool_call_id: Optional[str] = None+class ChatMemory:"""Manages the chat history for an AI agent.@@ -35,7 +39,13 @@ class ChatMemory:self.history: List[Message] = []self.max_messages = max_messages- def add_message(self, role: str, content: Union[str, Dict], tool_message: Optional[Dict] = None, tool_id: Optional[str] = None) -> None:+ def add_message(+ self,+ role: str,+ content: Union[str, Dict],+ tool_message: Optional[Dict] = None,+ tool_id: Optional[str] = None,+ ) -> None:"""Adds a message to the chat history and manages overflow.@@ -49,12 +59,12 @@ class ChatMemory:if tool_message:if tool_id is None:tool_id = tool_message["id"]-+message.tool_calls = [tool_message]message.tool_call_id = tool_idelif role == "tool":message.tool_call_id = tool_id-+self.history.append(message)self._manage_overflow()@@ -105,8 +115,8 @@ class ChatMemory:# Optionally, manage overflow after loading all messagesself._manage_overflow()-- def copy(self) -> 'ChatMemory':++ def copy(self) -> "ChatMemory":"""Creates a copy of the chat memory.@@ -115,7 +125,7 @@ class ChatMemory:"""new_memory = ChatMemory(max_messages=self.max_messages)new_memory.load(self.dump())-+return new_memorydef get_message_count(self) -> int:@@ -125,4 +135,4 @@ class ChatMemory:Returns:int: The number of messages."""- return len(self.history)\ No newline at end of file+ return len(self.history)diff --git a/atomic_agents/lib/components/system_prompt_generator.py b/atomic_agents/lib/components/system_prompt_generator.pyindex dce1167..0f69f63 100644--- a/atomic_agents/lib/components/system_prompt_generator.py+++ b/atomic_agents/lib/components/system_prompt_generator.py@@ -1,6 +1,7 @@-from typing import List, Dict, Optional-from dataclasses import dataclass, fieldfrom abc import ABC, abstractmethod+from dataclasses import dataclass, field+from typing import Dict, List, Optional+class SystemPromptContextProviderBase(ABC):def __init__(self, title: str):@@ -8,53 +9,55 @@ class SystemPromptContextProviderBase(ABC):@abstractmethoddef get_info(self) -> str:- return-+ pass+def __repr__(self) -> str:return self.get_info()+@dataclassclass SystemPromptInfo:background: List[str]- steps: Optional[List[str]] = field(default_factory=list)- output_instructions: Optional[List[str]] = field(default_factory=list)- context_providers: Optional[Dict[str, SystemPromptContextProviderBase]] = field(default_factory=dict)+ steps: List[str] = field(default_factory=list)+ output_instructions: List[str] = field(default_factory=list)+ context_providers: Dict[str, SystemPromptContextProviderBase] = field(default_factory=dict)+class SystemPromptGenerator:- def __init__(self, system_prompt_info: SystemPromptInfo = None):+ def __init__(self, system_prompt_info: Optional[SystemPromptInfo] = None):self.system_prompt_info = system_prompt_info or SystemPromptInfo(- background=['This is a conversation with a helpful and friendly AI assistant.'],- steps=[],- output_instructions=[],- context_providers={}+ background=["This is a conversation with a helpful and friendly AI assistant."])-- self.system_prompt_info.output_instructions.extend([- 'Always respond using the proper JSON schema.',- 'Always use the available additional information and context to enhance the response.'- ])- def generate_prompt(self) -> str:- system_prompt = ''+ self.system_prompt_info.output_instructions.extend(+ [+ "Always respond using the proper JSON schema.",+ "Always use the available additional information and context to enhance the response.",+ ]+ )- if self.system_prompt_info.background:- system_prompt += '# IDENTITY and PURPOSE\n'- system_prompt += '- ' + '\n- '.join(self.system_prompt_info.background) + '\n\n'+ def generate_prompt(self) -> str:+ sections = [+ ("IDENTITY and PURPOSE", self.system_prompt_info.background),+ ("INTERNAL ASSISTANT STEPS", self.system_prompt_info.steps),+ ("OUTPUT INSTRUCTIONS", self.system_prompt_info.output_instructions),+ ]- if self.system_prompt_info.steps:- system_prompt += '# INTERNAL ASSISTANT STEPS\n'- system_prompt += '- ' + '\n- '.join(self.system_prompt_info.steps) + '\n\n'+ prompt_parts = []- if self.system_prompt_info.output_instructions:- system_prompt += '# OUTPUT INSTRUCTIONS\n'- system_prompt += '- ' + '\n- '.join(self.system_prompt_info.output_instructions) + '\n\n'+ for title, content in sections:+ if content:+ prompt_parts.append(f"# {title}")+ prompt_parts.extend(f"- {item}" for item in content)+ prompt_parts.append("")if self.system_prompt_info.context_providers:- system_prompt += '# EXTRA INFORMATION AND CONTEXT\n'+ prompt_parts.append("# EXTRA INFORMATION AND CONTEXT")for provider in self.system_prompt_info.context_providers.values():info = provider.get_info()if info:- system_prompt += f'## {provider.title}\n'- system_prompt += f'{info}\n\n'+ prompt_parts.append(f"## {provider.title}")+ prompt_parts.append(info)+ prompt_parts.append("")- return system_prompt\ No newline at end of file+ return "\n".join(prompt_parts).strip()diff --git a/atomic_agents/lib/models/web_document.py b/atomic_agents/lib/models/web_document.pyindex 4fdd606..a4c6730 100644--- a/atomic_agents/lib/models/web_document.py+++ b/atomic_agents/lib/models/web_document.py@@ -2,14 +2,15 @@ from pydantic import BaseModel, Fieldfrom atomic_agents.agents.base_chat_agent import BaseAgentIO+class WebDocumentMetadata(BaseModel):url: str- title: str = Field(default='')- description: str = Field(default='')- keywords: str = Field(default='')- author: str = Field(default='')-+ title: str = Field(default="")+ description: str = Field(default="")+ keywords: str = Field(default="")+ author: str = Field(default="")++class WebDocument(BaseAgentIO):content: strmetadata: WebDocumentMetadata = Field(default_factory=WebDocumentMetadata)-diff --git a/atomic_agents/lib/tools/base.py b/atomic_agents/lib/tools/base.pyindex 734ff2e..019357c 100644--- a/atomic_agents/lib/tools/base.py+++ b/atomic_agents/lib/tools/base.py@@ -1,13 +1,15 @@-from typing import Type, Optional+from typing import Optional, Type+from pydantic import BaseModel-from dataclasses import dataclassfrom atomic_agents.agents.base_chat_agent import BaseAgentIO+class BaseToolConfig(BaseModel):title: Optional[str] = Nonedescription: Optional[str] = None+class BaseTool:"""Base class for all tools in the Atomic Agents framework.@@ -16,7 +18,8 @@ class BaseTool:input_schema (Type[BaseAgentIO]): The schema for the input data.output_schema (Type[BaseAgentIO]): The schema for the output data.tool_name (str): The name of the tool, derived from the input schema's title.- tool_description (str): The description of the tool, derived from the input schema's description or overridden by the user.+ tool_description (str):+ The description of the tool, derived from the input schema's description or overridden by the user."""input_schema: Type[BaseAgentIO]@@ -45,4 +48,4 @@ class BaseTool:Raises:NotImplementedError: If the method is not implemented by a subclass."""- raise NotImplementedError("Subclasses should implement this method")\ No newline at end of file+ raise NotImplementedError("Subclasses should implement this method")diff --git a/atomic_agents/lib/tools/calculator_tool.py b/atomic_agents/lib/tools/calculator_tool.pyindex 8495078..9843b06 100644--- a/atomic_agents/lib/tools/calculator_tool.py+++ b/atomic_agents/lib/tools/calculator_tool.py@@ -1,6 +1,6 @@from pydantic import Field-from sympy import sympifyfrom rich.console import Console+from sympy import sympifyfrom atomic_agents.agents.base_chat_agent import BaseAgentIOfrom atomic_agents.lib.tools.base import BaseTool, BaseToolConfig@@ -8,20 +8,27 @@ from atomic_agents.lib.tools.base import BaseTool, BaseToolConfig################# INPUT SCHEMA #################++class CalculatorToolSchema(BaseAgentIO):expression: str = Field(..., description="Mathematical expression to evaluate. For example, '2 + 2'.")class Config:title = "CalculatorTool"- description = "Tool for performing calculations. Supports basic arithmetic operations like addition, subtraction, multiplication, and division, but also more complex operations like exponentiation and trigonometric functions. Use this tool to evaluate mathematical expressions."- json_schema_extra = {- "title": title,- "description": description- }-+ description = (+ "Tool for performing calculations. Supports basic arithmetic operations "+ "like addition, subtraction, multiplication, and division, but also more "+ "complex operations like exponentiation and trigonometric functions. "+ "Use this tool to evaluate mathematical expressions."+ )+ json_schema_extra = {"title": title, "description": description}++##################### OUTPUT SCHEMA(S) #####################++class CalculatorToolOutputSchema(BaseAgentIO):result: str = Field(..., description="Result of the calculation.")@@ -32,6 +39,7 @@ class CalculatorToolOutputSchema(BaseAgentIO):class CalculatorToolConfig(BaseToolConfig):pass+class CalculatorTool(BaseTool):"""Tool for performing calculations based on the provided mathematical expression.@@ -40,13 +48,14 @@ class CalculatorTool(BaseTool):input_schema (CalculatorToolSchema): The schema for the input data.output_schema (CalculatorToolOutputSchema): The schema for the output data."""+input_schema = CalculatorToolSchemaoutput_schema = CalculatorToolOutputSchema-+def __init__(self, config: CalculatorToolConfig = CalculatorToolConfig()):"""Initializes the CalculatorTool.-+Args:config (CalculatorToolConfig): Configuration for the tool."""@@ -73,8 +82,8 @@ class CalculatorTool(BaseTool):return CalculatorToolOutputSchema(result=str(result))except Exception as e:raise ValueError(f"Error evaluating expression: {e}")--++################## EXAMPLE USAGE ##################diff --git a/atomic_agents/lib/tools/content_scraping_tool.py b/atomic_agents/lib/tools/content_scraping_tool.pyindex 651b462..df87e8d 100644--- a/atomic_agents/lib/tools/content_scraping_tool.py+++ b/atomic_agents/lib/tools/content_scraping_tool.py@@ -1,34 +1,40 @@import os-from pydantic import BaseModel, Fieldfrom typing import Optional-from rich.console import Console-from rich.markdown import Markdown-import openai+import instructor+import openaiimport requests+from pydantic import Field+from rich.console import Console+from rich.markdown import Markdownfrom atomic_agents.agents.base_chat_agent import BaseAgentIOfrom atomic_agents.lib.tools.base import BaseTool, BaseToolConfig-from atomic_agents.lib.utils.scraping.url_to_markdown import UrlToMarkdownConverterfrom atomic_agents.lib.utils.scraping.pdf_to_markdown import PdfToMarkdownConverter+from atomic_agents.lib.utils.scraping.url_to_markdown import UrlToMarkdownConverter################# INPUT SCHEMA #################++class ContentScrapingToolSchema(BaseAgentIO):url: str = Field(..., description="URL of the web page or PDF to scrape.")class Config:title = "ContentScrapingTool"- description = "Tool for scraping web pages or PDFs and converting content to markdown in order to extract information or summarize the content."- json_schema_extra = {- "title": title,- "description": description- }+ description = (+ "Tool for scraping web pages or PDFs and converting content to markdown "+ "in order to extract information or summarize the content."+ )+ json_schema_extra = {"title": title, "description": description}+##################### OUTPUT SCHEMA(S) #####################++class ContentScrapingToolOutputSchema(BaseAgentIO):content: strmetadata: Optional[dict] = None@@ -49,6 +55,7 @@ class ContentScrapingTool(BaseTool):input_schema (ContentScrapingToolSchema): The schema for the input data.output_schema (ContentScrapingToolOutputSchema): The schema for the output data."""+input_schema = ContentScrapingToolSchemaoutput_schema = ContentScrapingToolOutputSchema@@ -73,9 +80,9 @@ class ContentScrapingTool(BaseTool):"""url = params.urlresponse = requests.head(url)- content_type = response.headers.get('Content-Type', '')+ content_type = response.headers.get("Content-Type", "")- if 'application/pdf' in content_type:+ if "application/pdf" in content_type:document = PdfToMarkdownConverter.convert(url=url)else:document = UrlToMarkdownConverter.convert(url=url)@@ -89,12 +96,7 @@ class ContentScrapingTool(BaseTool):if __name__ == "__main__":rich_console = Console()- client = instructor.from_openai(- openai.OpenAI(- api_key=os.getenv("OPENAI_API_KEY"),- base_url=os.getenv("OPENAI_BASE_URL")- )- )+ client = instructor.from_openai(openai.OpenAI(api_key=os.getenv("OPENAI_API_KEY"), base_url=os.getenv("OPENAI_BASE_URL")))################## TEST WEB PAGE #@@ -102,7 +104,12 @@ if __name__ == "__main__":result = client.chat.completions.create(model="gpt-3.5-turbo",response_model=ContentScrapingTool.input_schema,- messages=[{"role": "user", "content": "Scrape the content of https://brainblendai.com"}],+ messages=[+ {+ "role": "user",+ "content": "Scrape the content of https://brainblendai.com",+ }+ ],)output = ContentScrapingTool().run(result)@@ -114,8 +121,13 @@ if __name__ == "__main__":result = client.chat.completions.create(model="gpt-3.5-turbo",response_model=ContentScrapingTool.input_schema,- messages=[{"role": "user", "content": "Scrape the content of https://pdfobject.com/pdf/sample.pdf"}],+ messages=[+ {+ "role": "user",+ "content": "Scrape the content of https://pdfobject.com/pdf/sample.pdf",+ }+ ],)output = ContentScrapingTool().run(result)- rich_console.print(Markdown(output.content))\ No newline at end of file+ rich_console.print(Markdown(output.content))diff --git a/atomic_agents/lib/tools/search/searx_tool.py b/atomic_agents/lib/tools/search/searx_tool.pyindex a04760a..657393c 100644--- a/atomic_agents/lib/tools/search/searx_tool.py+++ b/atomic_agents/lib/tools/search/searx_tool.py@@ -1,7 +1,8 @@import os-import requests-from pydantic import BaseModel, Fieldfrom typing import List, Literal, Optional++import requests+from pydantic import Fieldfrom rich.console import Consolefrom atomic_agents.agents.base_chat_agent import BaseAgentIO@@ -10,38 +11,49 @@ from atomic_agents.lib.tools.base import BaseTool, BaseToolConfig################# INPUT SCHEMA #################++class SearxNGSearchToolSchema(BaseAgentIO):queries: List[str] = Field(..., description="List of search queries.")- category: Optional[Literal["general", "news", "social_media"]] = Field("general", description="Category of the search queries.")+ category: Optional[Literal["general", "news", "social_media"]] = Field(+ "general", description="Category of the search queries."+ )class Config:title = "SearxNGSearchTool"- description = "Tool for searching for information, news, references, and other content on the SearxNG. Returns a list of search results with a short description or content snippet and URLs for further exploration."- json_schema_extra = {- "title": title,- "description": description- }+ description = (+ "Tool for searching for information, news, references, and other content on the SearxNG. "+ "Returns a list of search results with a short description or content snippet and URLs for further exploration."+ )+ json_schema_extra = {"title": title, "description": description}+##################### OUTPUT SCHEMA(S) #####################++class SearxNGSearchResultSchema(BaseAgentIO):url: strtitle: strcontent: Optional[str] = None-+class SearxNGSearchToolOutputSchema(BaseAgentIO):results: List[SearxNGSearchResultSchema]category: Optional[str] = None+############### TOOL LOGIC ###############++class SearxNGSearchToolConfig(BaseToolConfig):base_url: str = ""max_results: int = 10+class SearxNGSearchTool(BaseTool):"""Tool for performing searches on SearxNG based on the provided queries and category.@@ -52,6 +64,7 @@ class SearxNGSearchTool(BaseTool):max_results (int): The maximum number of search results to return.base_url (str): The base URL for the SearxNG instance to use."""+input_schema = SearxNGSearchToolSchemaoutput_schema = SearxNGSearchToolOutputSchema@@ -60,7 +73,8 @@ class SearxNGSearchTool(BaseTool):Initializes the SearxNGSearchTool.Args:- config (SearxNGSearchToolConfig): Configuration for the tool, including base URL, max results, and optional title and description overrides.+ config (SearxNGSearchToolConfig):+ Configuration for the tool, including base URL, max results, and optional title and description overrides."""super().__init__(config)self.base_url = config.base_url@@ -86,63 +100,71 @@ class SearxNGSearchTool(BaseTool):for query in params.queries:# Prepare the query parametersquery_params = {- 'q': query,- 'safesearch': '0',- 'format': 'json',- 'language': 'en',- 'engines': 'bing,duckduckgo,google,startpage,yandex',+ "q": query,+ "safesearch": "0",+ "format": "json",+ "language": "en",+ "engines": "bing,duckduckgo,google,startpage,yandex",}# Add category to query parameters if it is setif params.category:- query_params['categories'] = params.category+ query_params["categories"] = params.category# Make the GET requestresponse = requests.get(f"{self.base_url}/search", params=query_params)# Check if the request was successfulif response.status_code != 200:- raise Exception(f"Failed to fetch search results for query '{query}': {response.status_code} {response.reason}")+ raise Exception(+ f"Failed to fetch search results for query '{query}': {response.status_code} {response.reason}"+ )- results = response.json().get('results', [])+ results = response.json().get("results", [])all_results.extend(results)# Sort the combined results by score in descending order- sorted_results = sorted(all_results, key=lambda x: x.get('score', 0), reverse=True)+ sorted_results = sorted(all_results, key=lambda x: x.get("score", 0), reverse=True)# Remove duplicates while preserving orderseen_urls = set()unique_results = []for result in sorted_results:- if 'content' not in result or 'title' not in result or 'url' not in result:+ if "content" not in result or "title" not in result or "url" not in result:continue- if result['url'] not in seen_urls:+ if result["url"] not in seen_urls:unique_results.append(result)- if 'metadata' in result:- result['title'] = result['title'] + ' - (Published ' + result['metadata'] + ')'- if 'publishedDate' in result and result['publishedDate']:- result['title'] = result['title'] + ' - (Published ' + result['publishedDate'] + ')'- seen_urls.add(result['url'])+ if "metadata" in result:+ result["title"] = f"{result['title']} - (Published {result['metadata']})"+ if "publishedDate" in result and result["publishedDate"]:+ result["title"] = f"{result['title']} - (Published {result['publishedDate']})"+ seen_urls.add(result["url"])# Filter results to include only those with the correct category if it is setif params.category:- filtered_results = [result for result in unique_results if result.get('category') == params.category]+ filtered_results = [result for result in unique_results if result.get("category") == params.category]else:filtered_results = unique_results- filtered_results = filtered_results[:max_results or self.max_results]+ filtered_results = filtered_results[: max_results or self.max_results]return SearxNGSearchToolOutputSchema(results=filtered_results, category=params.category)+################## EXAMPLE USAGE ##################if __name__ == "__main__":rich_console = Console()- search_tool_instance = SearxNGSearchTool(config=SearxNGSearchToolConfig(base_url=os.getenv('SEARXNG_BASE_URL'), max_results=5))-- search_input = SearxNGSearchTool.input_schema(queries=["Python programming", "Machine learning", "Artificial intelligence"], category="news")+ search_tool_instance = SearxNGSearchTool(+ config=SearxNGSearchToolConfig(base_url=os.getenv("SEARXNG_BASE_URL"), max_results=5)+ )++ search_input = SearxNGSearchTool.input_schema(+ queries=["Python programming", "Machine learning", "Artificial intelligence"],+ category="news",+ )output = search_tool_instance.run(search_input)- rich_console.print(output)\ No newline at end of file+ rich_console.print(output)diff --git a/atomic_agents/lib/tools/search/serper_tool.py b/atomic_agents/lib/tools/search/serper_tool.pyindex bba050a..eb3c97c 100644--- a/atomic_agents/lib/tools/search/serper_tool.py+++ b/atomic_agents/lib/tools/search/serper_tool.py@@ -1,48 +1,57 @@-import os-import requestsimport json-from pydantic import BaseModel, Field+import osfrom typing import List, Optional++import requests+from pydantic import Fieldfrom rich.console import Console-import openai-import instructor-from atomic_agents.lib.tools.base import BaseTool, BaseToolConfigfrom atomic_agents.agents.base_chat_agent import BaseAgentIO+from atomic_agents.lib.tools.base import BaseTool, BaseToolConfig################# INPUT SCHEMA #################++class SerperSearchToolSchema(BaseAgentIO):queries: List[str] = Field(..., description="List of search queries.")class Config:title = "SerperSearchTool"- description = "Tool for searching for information, news, references, and other content using the Serper API. Returns a list of search results with a short description or content snippet and URLs for further exploration."- json_schema_extra = {- "title": title,- "description": description- }+ description = (+ "Tool for searching for information, news, references, and other content using the Serper API. "+ "Returns a list of search results with a short description or content snippet and URLs for further exploration."+ )+ json_schema_extra = {"title": title, "description": description}+##################### OUTPUT SCHEMA(S) #####################++class SerperSearchResultSchema(BaseAgentIO):url: strtitle: strcontent: Optional[str] = Noneposition: Optional[int] = None+class SerperSearchToolOutputSchema(BaseAgentIO):results: List[SerperSearchResultSchema]+############### TOOL LOGIC ###############++class SerperSearchToolConfig(BaseToolConfig):api_key: str = ""max_results: int = 10+class SerperSearchTool(BaseTool):"""Tool for performing searches using the Serper API based on the provided queries.@@ -53,6 +62,7 @@ class SerperSearchTool(BaseTool):api_key (str): The API key for the Serper API.max_results (int): The maximum number of search results to return."""+input_schema = SerperSearchToolSchemaoutput_schema = SerperSearchToolOutputSchema@@ -61,7 +71,8 @@ class SerperSearchTool(BaseTool):Initializes the SerperSearchTool.Args:- config (SerperSearchToolConfig): Configuration for the tool, including API key, max results, and optional title and description overrides.+ config (SerperSearchToolConfig):+ Configuration for the tool, including API key, max results, and optional title and description overrides."""super().__init__(config)self.api_key = config.api_key@@ -89,56 +100,56 @@ class SerperSearchTool(BaseTool):payload = json.dumps({"q": query})# Make the POST request- headers = {- 'X-API-KEY': self.api_key,- 'Content-Type': 'application/json'- }+ headers = {"X-API-KEY": self.api_key, "Content-Type": "application/json"}response = requests.post("https://google.serper.dev/search", headers=headers, data=payload)# Check if the request was successfulif response.status_code != 200:- raise Exception(f"Failed to fetch search results for query '{query}': {response.status_code} {response.reason}")+ raise Exception(+ f"Failed to fetch search results for query '{query}': {response.status_code} {response.reason}"+ )- results = response.json().get('organic', [])+ results = response.json().get("organic", [])all_results.extend(results)# Sort the combined results by position in ascending order- sorted_results = sorted(all_results, key=lambda x: x.get('position', float('inf')))+ sorted_results = sorted(all_results, key=lambda x: x.get("position", float("inf")))# Remove duplicates while preserving orderseen_urls = set()unique_results = []for result in sorted_results:- if 'snippet' not in result or 'title' not in result or 'link' not in result:+ if "snippet" not in result or "title" not in result or "link" not in result:continue- if result['link'] not in seen_urls:+ if result["link"] not in seen_urls:unique_results.append(result)- seen_urls.add(result['link'])+ seen_urls.add(result["link"])- filtered_results = unique_results[:max_results or self.max_results]+ filtered_results = unique_results[: max_results or self.max_results]# Convert to output schema formatoutput_results = [SerperSearchResultSchema(- url=result['link'],- title=result['title'],- content=result.get('snippet'),- position=result.get('position')+ url=result["link"],+ title=result["title"],+ content=result.get("snippet"),+ position=result.get("position"),)for result in filtered_results]return SerperSearchToolOutputSchema(results=output_results)+################## EXAMPLE USAGE ##################if __name__ == "__main__":rich_console = Console()search_tool_instance = SerperSearchTool(config=SerperSearchToolConfig(api_key=os.getenv("SERPER_API_KEY"), max_results=5))-+search_input = SerperSearchTool.input_schema(queries=["Python programming", "Machine learning", "Quantum computing"])output = search_tool_instance.run(search_input)-- rich_console.print(output)\ No newline at end of file++ rich_console.print(output)diff --git a/atomic_agents/lib/tools/yelp_restaurant_finder_tool.py b/atomic_agents/lib/tools/yelp_restaurant_finder_tool.pyindex faf5d4b..d3a7a22 100644--- a/atomic_agents/lib/tools/yelp_restaurant_finder_tool.py+++ b/atomic_agents/lib/tools/yelp_restaurant_finder_tool.py@@ -1,10 +1,10 @@import os-import requests-from pydantic import BaseModel, Field+from enum import Enumfrom typing import List, Optional++import requests+from pydantic import Fieldfrom rich.console import Console-from rich.table import Table-from enum import Enumfrom atomic_agents.agents.base_chat_agent import BaseAgentIOfrom atomic_agents.lib.tools.base import BaseTool, BaseToolConfig@@ -12,6 +12,8 @@ from atomic_agents.lib.tools.base import BaseTool, BaseToolConfig################# INPUT SCHEMA #################++class YelpCategory(Enum):ITALIAN = "italian"MEXICAN = "mexican"@@ -33,32 +35,46 @@ class YelpCategory(Enum):VEGAN = "vegan"VEGETARIAN = "vegetarian"+class PriceRange(Enum):ONE = "1"TWO = "2"THREE = "3"FOUR = "4"+class YelpSearchToolSchema(BaseAgentIO):location: str = Field(..., description="Location to search for food.")term: Optional[str] = Field(None, description="Search term (e.g., 'pizza', 'sushi').")categories: Optional[List[YelpCategory]] = Field(None, description="Categories to filter by (e.g., 'italian, mexican').")- price: Optional[List[PriceRange]] = Field(None, description="Price range to filter by (e.g., '1', '2', '3', '4'). Can be multiple. 1 is cheap. 2 and 3 are mid-range. 4 is the most high-end.")+ price: Optional[List[PriceRange]] = Field(+ None,+ description=(+ "Price range to filter by (e.g., '1', '2', '3', '4'). "+ "Can be multiple. 1 is cheap. 2 and 3 are mid-range. 4 is the most high-end."+ ),+ )open_now: Optional[bool] = Field(False, description="Filter for businesses that are open now.")- sort_by: Optional[str] = Field("best_match", description="Sort by criteria (e.g., 'best_match', 'rating', 'review_count', 'distance').")+ sort_by: Optional[str] = Field(+ "best_match",+ description="Sort by criteria (e.g., 'best_match', 'rating', 'review_count', 'distance').",+ )limit: Optional[int] = Field(10, description="Number of results to return.")class Config:title = "YelpSearchTool"- description = "Tool for searching for food using the Yelp API. Returns a list of businesses with details such as name, rating, and address."- json_schema_extra = {- "title": title,- "description": description- }+ description = (+ "Tool for searching for food using the Yelp API. "+ "Returns a list of businesses with details such as name, rating, and address."+ )+ json_schema_extra = {"title": title, "description": description}+##################### OUTPUT SCHEMA(S) #####################++class YelpSearchResultSchema(BaseAgentIO):name: strurl: str@@ -68,24 +84,29 @@ class YelpSearchResultSchema(BaseAgentIO):phone: Optional[str] = Nonecategories: List[str]+class YelpSearchToolOutputSchema(BaseAgentIO):results: List[YelpSearchResultSchema]-+class Config:title = "YelpSearchToolOutput"- description = "Output schema for the YelpSearchTool, containing a list of search results with details such as name, rating, and address."- json_schema_extra = {- "title": title,- "description": description- }+ description = (+ "Output schema for the YelpSearchTool, containing a list of search results with details"+ "such as name, rating, and address."+ )+ json_schema_extra = {"title": title, "description": description}+############### TOOL LOGIC ###############++class YelpSearchToolConfig(BaseToolConfig):api_key: str = ""max_results: int = 10+class YelpSearchTool(BaseTool):"""Tool for performing searches using the Yelp API based on the provided queries.@@ -96,6 +117,7 @@ class YelpSearchTool(BaseTool):api_key (str): The API key for the Yelp API.max_results (int): The maximum number of search results to return."""+input_schema = YelpSearchToolSchemaoutput_schema = YelpSearchToolOutputSchema@@ -104,7 +126,8 @@ class YelpSearchTool(BaseTool):Initializes the YelpSearchTool.Args:- config (YelpSearchToolConfig): Configuration for the tool, including API key, max results, and optional title and description overrides.+ config (YelpSearchToolConfig):+ Configuration for the tool, including API key, max results, and optional title and description overrides."""super().__init__(config)self.api_key = config.api_key@@ -132,56 +155,59 @@ class YelpSearchTool(BaseTool):query_params = {"location": params.location,"term": params.term,- "categories": ",".join([category.value for category in params.categories]) if params.categories else None,- "price": ",".join([price.value for price in params.price]) if params.price else None,+ "categories": (",".join([category.value for category in params.categories]) if params.categories else None),+ "price": (",".join([price.value for price in params.price]) if params.price else None),"sort_by": params.sort_by,"limit": self.max_results,- "open_now": params.open_now+ "open_now": params.open_now,}# Make the GET request- headers = {- 'Authorization': f'Bearer {self.api_key}'- }- response = requests.get("https://api.yelp.com/v3/businesses/search", headers=headers, params=query_params)+ headers = {"Authorization": f"Bearer {self.api_key}"}+ response = requests.get(+ "https://api.yelp.com/v3/businesses/search",+ headers=headers,+ params=query_params,+ )# Check if the request was successfulif response.status_code != 200:raise Exception(f"Failed to fetch search results: {response.status_code} {response.reason}")- results = response.json().get('businesses', [])+ results = response.json().get("businesses", [])# Convert to output schema formatoutput_results = [YelpSearchResultSchema(- name=result['name'],- url=result['url'].split('?')[0],- rating=result['rating'],- review_count=result['review_count'],- address=", ".join(result['location']['display_address']),- phone=result.get('display_phone'),- categories=[category['title'] for category in result['categories']]+ name=result["name"],+ url=result["url"].split("?")[0],+ rating=result["rating"],+ review_count=result["review_count"],+ address=", ".join(result["location"]["display_address"]),+ phone=result.get("display_phone"),+ categories=[category["title"] for category in result["categories"]],)for result in results]return YelpSearchToolOutputSchema(results=output_results)+################## EXAMPLE USAGE ##################if __name__ == "__main__":rich_console = Console()- search_tool_instance = YelpSearchTool(config=YelpSearchToolConfig(api_key=os.getenv('YELP_API_KEY'), max_results=10))-+ search_tool_instance = YelpSearchTool(config=YelpSearchToolConfig(api_key=os.getenv("YELP_API_KEY"), max_results=10))+search_input = YelpSearchTool.input_schema(location="Diepenbeek, Belgium",term="",categories=[YelpCategory.PIZZA],price=[PriceRange.ONE, PriceRange.TWO, PriceRange.THREE, PriceRange.FOUR],- sort_by="best_match"+ sort_by="best_match",)output = search_tool_instance.run(search_input)-- rich_console.print(output)\ No newline at end of file++ rich_console.print(output)diff --git a/atomic_agents/lib/tools/yt_transcript_scraper.py b/atomic_agents/lib/tools/yt_transcript_scraper.pyindex 46d8c06..efe9bfd 100644--- a/atomic_agents/lib/tools/yt_transcript_scraper.py+++ b/atomic_agents/lib/tools/yt_transcript_scraper.py@@ -1,50 +1,60 @@import os-from pydantic import BaseModel, Fieldfrom typing import List, Optional++from googleapiclient.discovery import build+from pydantic import Fieldfrom rich.console import Console-from youtube_transcript_api import YouTubeTranscriptApi, NoTranscriptFound, TranscriptsDisabled+from youtube_transcript_api import NoTranscriptFound, TranscriptsDisabled, YouTubeTranscriptApi+from atomic_agents.agents.base_chat_agent import BaseAgentIOfrom atomic_agents.lib.tools.base import BaseTool, BaseToolConfig-from googleapiclient.discovery import build################# INPUT SCHEMA #################++class YouTubeTranscriptToolSchema(BaseAgentIO):video_url: str = Field(..., description="URL of the YouTube video to fetch the transcript for.")language: Optional[str] = Field(None, description="Language code for the transcript (e.g., 'en' for English).")class Config:title = "YouTubeTranscriptTool"- description = "Tool for fetching the transcript of a YouTube video using the YouTube Transcript API. Returns the transcript with text, start time, and duration."- json_schema_extra = {- "title": title,- "description": description- }+ description = (+ "Tool for fetching the transcript of a YouTube video using the YouTube Transcript API. "+ "Returns the transcript with text, start time, and duration."+ )+ json_schema_extra = {"title": title, "description": description}+##################### OUTPUT SCHEMA(S) #####################++class YouTubeTranscriptToolOutputSchema(BaseAgentIO):transcript: strduration: floatcomments: List[str]metadata: dict-+class Config:title = "YouTubeTranscriptToolOutput"- description = "Output schema for the YouTubeTranscriptTool. Contains the transcript text, duration, comments, and metadata."- json_schema_extra = {- "title": title,- "description": description- }-+ description = (+ "Output schema for the YouTubeTranscriptTool. " "Contains the transcript text, duration, comments, and metadata."+ )+ json_schema_extra = {"title": title, "description": description}+############### TOOL LOGIC ###############class YouTubeTranscriptToolConfig(BaseToolConfig):- api_key: str = Field(description="YouTube API key for fetching video metadata.", default=os.getenv('YOUTUBE_API_KEY'))+ api_key: str = Field(+ description="YouTube API key for fetching video metadata.",+ default=os.getenv("YOUTUBE_API_KEY"),+ )+class YouTubeTranscriptTool(BaseTool):"""@@ -54,6 +64,7 @@ class YouTubeTranscriptTool(BaseTool):input_schema (YouTubeTranscriptToolSchema): The schema for the input data.output_schema (YouTubeTranscriptToolOutputSchema): The schema for the output data."""+input_schema = YouTubeTranscriptToolSchemaoutput_schema = YouTubeTranscriptToolOutputSchema@@ -89,8 +100,8 @@ class YouTubeTranscriptTool(BaseTool):except (NoTranscriptFound, TranscriptsDisabled) as e:raise Exception(f"Failed to fetch transcript for video '{video_id}': {str(e)}")- transcript_text = " ".join([transcript['text'] for transcript in transcripts])- total_duration = sum([transcript['duration'] for transcript in transcripts])+ transcript_text = " ".join([transcript["text"] for transcript in transcripts])+ total_duration = sum([transcript["duration"] for transcript in transcripts])metadata = self.fetch_video_metadata(video_id)@@ -98,7 +109,7 @@ class YouTubeTranscriptTool(BaseTool):transcript=transcript_text,duration=total_duration,comments=[],- metadata=metadata+ metadata=metadata,)@staticmethod@@ -112,7 +123,7 @@ class YouTubeTranscriptTool(BaseTool):Returns:str: The extracted video ID."""- return url.split('v=')[-1].split('&')[0]+ return url.split("v=")[-1].split("&")[0]def fetch_video_metadata(self, video_id: str) -> dict:"""@@ -124,31 +135,32 @@ class YouTubeTranscriptTool(BaseTool):Returns:dict: The metadata of the video."""- youtube = build('youtube', 'v3', developerKey=self.api_key)+ youtube = build("youtube", "v3", developerKey=self.api_key)request = youtube.videos().list(part="snippet", id=video_id)response = request.execute()- if not response['items']:+ if not response["items"]:raise Exception(f"No metadata found for video '{video_id}'")- video_info = response['items'][0]['snippet']+ video_info = response["items"][0]["snippet"]metadata = {"id": video_id,- "title": video_info['title'],- "channel": video_info['channelTitle'],- "published_at": video_info['publishedAt']+ "title": video_info["title"],+ "channel": video_info["channelTitle"],+ "published_at": video_info["publishedAt"],}return metadata+################## EXAMPLE USAGE ##################if __name__ == "__main__":rich_console = Console()- api_key = os.getenv('YOUTUBE_API_KEY')+ api_key = os.getenv("YOUTUBE_API_KEY")search_tool_instance = YouTubeTranscriptTool(config=YouTubeTranscriptToolConfig(api_key=api_key))-+search_input = YouTubeTranscriptTool.input_schema(video_url="https://www.youtube.com/watch?v=t1e8gqXLbsU", language="en")output = search_tool_instance.run(search_input)- rich_console.print(output)\ No newline at end of file+ rich_console.print(output)diff --git a/atomic_agents/lib/utils/format_tool_message.py b/atomic_agents/lib/utils/format_tool_message.pyindex 3e20656..d78196a 100644--- a/atomic_agents/lib/utils/format_tool_message.py+++ b/atomic_agents/lib/utils/format_tool_message.py@@ -1,11 +1,10 @@-import json-from typing import Dict, Typeimport uuid+from typing import Dict, Optional, Typefrom pydantic import BaseModel-def format_tool_message(tool_call: Type[BaseModel], tool_id: str = None) -> Dict:+def format_tool_message(tool_call: Type[BaseModel], tool_id: Optional[str] = None) -> Dict:"""Formats a message for a tool call.@@ -19,13 +18,12 @@ def format_tool_message(tool_call: Type[BaseModel], tool_id: str = None) -> Dict"""if tool_id is None:tool_id = str(uuid.uuid4())-+return {"id": tool_id,"type": "function","function": {"name": tool_call.model_config["title"],- "arguments": tool_call.model_dump_json()- }+ "arguments": tool_call.model_dump_json(),+ },}-\ No newline at end of filediff --git a/atomic_agents/lib/utils/scraping/pdf_to_markdown.py b/atomic_agents/lib/utils/scraping/pdf_to_markdown.pyindex b48709b..109f57e 100644--- a/atomic_agents/lib/utils/scraping/pdf_to_markdown.py+++ b/atomic_agents/lib/utils/scraping/pdf_to_markdown.py@@ -1,11 +1,13 @@import re-import requests-from PyPDF2 import PdfReaderfrom io import BytesIO+import markdownify+import requests+from PyPDF2 import PdfReaderfrom atomic_agents.lib.models.web_document import WebDocument, WebDocumentMetadata+class PdfToMarkdownConverter:@staticmethoddef _fetch_pdf_content(url):@@ -26,7 +28,7 @@ class PdfToMarkdownConverter:def _convert_to_markdown(text):markdown_text = markdownify.markdownify(text, heading_style="ATX")# Replace more than two consecutive newlines with exactly two newlines- markdown_text = re.sub(r'\n{3,}', '\n\n', markdown_text)+ markdown_text = re.sub(r"\n{3,}", "\n\n", markdown_text)return markdown_text@staticmethod@@ -34,7 +36,7 @@ class PdfToMarkdownConverter:if url:pdf_content = PdfToMarkdownConverter._fetch_pdf_content(url)elif file_path:- with open(file_path, 'rb') as f:+ with open(file_path, "rb") as f:pdf_content = BytesIO(f.read())else:raise ValueError("Either url or file_path must be provided")@@ -45,12 +47,14 @@ class PdfToMarkdownConverter:return WebDocument(content=markdown_content, metadata=metadata)-if __name__ == '__main__':++if __name__ == "__main__":from rich.console import Consolefrom rich.markdown import Markdown+console = Console()-+url = "https://pdfobject.com/pdf/sample.pdf"document = PdfToMarkdownConverter.convert(url=url)-- console.print(Markdown(document.content))\ No newline at end of file++ console.print(Markdown(document.content))diff --git a/atomic_agents/lib/utils/scraping/url_to_markdown.py b/atomic_agents/lib/utils/scraping/url_to_markdown.pyindex 7b4467a..020842a 100644--- a/atomic_agents/lib/utils/scraping/url_to_markdown.py+++ b/atomic_agents/lib/utils/scraping/url_to_markdown.py@@ -1,18 +1,19 @@-import osimport re-from bs4 import BeautifulSoup+import requests+from bs4 import BeautifulSoupfrom markdownify import MarkdownConverter+from atomic_agents.lib.models.web_document import WebDocument, WebDocumentMetadataclass UrlToMarkdownConverter:METADATA_ATTRS = {- 'url': '',- 'title': ('title', None),- 'description': ('meta', {'name': 'description'}),- 'keywords': ('meta', {'name': 'keywords'}),- 'author': ('meta', {'name': 'author'})+ "url": "",+ "title": ("title", None),+ "description": ("meta", {"name": "description"}),+ "keywords": ("meta", {"name": "keywords"}),+ "author": ("meta", {"name": "author"}),}@staticmethod@@ -23,7 +24,7 @@ class UrlToMarkdownConverter:@staticmethoddef _parse_html(html):- return BeautifulSoup(html, 'html.parser')+ return BeautifulSoup(html, "html.parser")@staticmethoddef _extract_metadata(soup, url):@@ -31,7 +32,7 @@ class UrlToMarkdownConverter:for attr, selector in UrlToMarkdownConverter.METADATA_ATTRS.items():if selector:element = soup.find(*selector)- metadata_dict[attr] = element.get('content', '').strip() if element else ''+ metadata_dict[attr] = element.get("content", "").strip() if element else ""else:metadata_dict[attr] = urlreturn WebDocumentMetadata(**metadata_dict)@@ -40,20 +41,21 @@ class UrlToMarkdownConverter:def _convert_to_markdown(html):class CustomMarkdownConverter(MarkdownConverter):def convert_img(self, el, text, convert_as_inline):- return ''+ return ""+def convert_p(self, el, text, convert_as_inline):- return text + '\n\n'+ return text + "\n\n"- for script in html.find_all('script'):+ for script in html.find_all("script"):script.decompose()-- for style in html.find_all('style'):++ for style in html.find_all("style"):style.decompose()converter = CustomMarkdownConverter()markdown_text = converter.convert(str(html))# Replace more than two consecutive newlines with exactly two newlines- markdown_text = re.sub(r'\n{3,}', '\n\n', markdown_text)+ markdown_text = re.sub(r"\n{3,}", "\n\n", markdown_text)return markdown_text@staticmethod@@ -65,7 +67,8 @@ class UrlToMarkdownConverter:return WebDocument(content=markdown_content, metadata=metadata)-if __name__ == '__main__':++if __name__ == "__main__":url = "https://brainblendai.com"document = UrlToMarkdownConverter.convert(url)print(document)diff --git a/pyproject.toml b/pyproject.tomlindex 3f2b354..77ac867 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -28,3 +28,6 @@ repository = "https://github.com/KennyVaneetvelde/atomic_agents"[tool.setuptools.packages.find]where = ["."]include = ["atomic_agents"]++[tool.black]+line-length = 127\ No newline at end of filediff --git a/requirements.txt b/requirements.txtindex 24dce6c..a73ec8b 100644--- a/requirements.txt+++ b/requirements.txt@@ -5,6 +5,7 @@ anyio==4.4.0asgiref==3.8.1attrs==23.2.0beautifulsoup4==4.12.3+black==24.4.2cachetools==5.3.3certifi==2024.6.2charset-normalizer==3.3.2