Building AI agents, atomically
Remove anything that will move to another repo
KennyVaneetveldea536bbcaa534
diff --git a/atomic-agents/atomic_agents/lib/models/__init__.py b/atomic-agents/atomic_agents/lib/models/__init__.pydeleted file mode 100644index e69de29..0000000diff --git a/atomic-agents/atomic_agents/lib/models/web_document.py b/atomic-agents/atomic_agents/lib/models/web_document.pydeleted file mode 100644index 3486201..0000000--- a/atomic-agents/atomic_agents/lib/models/web_document.py+++ /dev/null@@ -1,20 +0,0 @@-from pydantic import Field--from atomic_agents.agents.base_agent import BaseIOSchema---class WebDocumentMetadata(BaseIOSchema):- """Schema that describes the metadata of a web document."""-- url: str- title: str = Field(default="")- description: str = Field(default="")- keywords: str = Field(default="")- author: str = Field(default="")---class WebDocument(BaseIOSchema):- """Schema that represents the content and metadata of a web document."""-- content: str- metadata: WebDocumentMetadata = Field(default_factory=lambda: WebDocumentMetadata(url=""))diff --git a/atomic-agents/atomic_agents/lib/utils/scraping/__init__.py b/atomic-agents/atomic_agents/lib/utils/scraping/__init__.pydeleted file mode 100644index e69de29..0000000diff --git a/atomic-agents/atomic_agents/lib/utils/scraping/pdf_to_markdown.py b/atomic-agents/atomic_agents/lib/utils/scraping/pdf_to_markdown.pydeleted file mode 100644index 109f57e..0000000--- a/atomic-agents/atomic_agents/lib/utils/scraping/pdf_to_markdown.py+++ /dev/null@@ -1,60 +0,0 @@-import re-from io import BytesIO--import markdownify-import requests-from PyPDF2 import PdfReader--from atomic_agents.lib.models.web_document import WebDocument, WebDocumentMetadata---class PdfToMarkdownConverter:- @staticmethod- def _fetch_pdf_content(url):- response = requests.get(url)- response.raise_for_status()- return BytesIO(response.content)-- @staticmethod- def _read_pdf(file):- pdf = PdfReader(file)- content = []- for page_num in range(len(pdf.pages)):- page = pdf.pages[page_num]- content.append(page.extract_text())- return "\n".join(content)-- @staticmethod- def _convert_to_markdown(text):- markdown_text = markdownify.markdownify(text, heading_style="ATX")- # Replace more than two consecutive newlines with exactly two newlines- markdown_text = re.sub(r"\n{3,}", "\n\n", markdown_text)- return markdown_text-- @staticmethod- def convert(url=None, file_path=None):- if url:- pdf_content = PdfToMarkdownConverter._fetch_pdf_content(url)- elif file_path:- with open(file_path, "rb") as f:- pdf_content = BytesIO(f.read())- else:- raise ValueError("Either url or file_path must be provided")-- text_content = PdfToMarkdownConverter._read_pdf(pdf_content)- markdown_content = PdfToMarkdownConverter._convert_to_markdown(text_content)- metadata = WebDocumentMetadata(url=url or file_path, title="", description="", keywords="", author="")-- return WebDocument(content=markdown_content, metadata=metadata)---if __name__ == "__main__":- from rich.console import Console- from rich.markdown import Markdown-- console = Console()-- url = "https://pdfobject.com/pdf/sample.pdf"- document = PdfToMarkdownConverter.convert(url=url)-- console.print(Markdown(document.content))diff --git a/atomic-agents/atomic_agents/lib/utils/scraping/url_to_markdown.py b/atomic-agents/atomic_agents/lib/utils/scraping/url_to_markdown.pydeleted file mode 100644index 020842a..0000000--- a/atomic-agents/atomic_agents/lib/utils/scraping/url_to_markdown.py+++ /dev/null@@ -1,74 +0,0 @@-import re--import requests-from bs4 import BeautifulSoup-from markdownify import MarkdownConverter--from atomic_agents.lib.models.web_document import WebDocument, WebDocumentMetadata---class UrlToMarkdownConverter:- METADATA_ATTRS = {- "url": "",- "title": ("title", None),- "description": ("meta", {"name": "description"}),- "keywords": ("meta", {"name": "keywords"}),- "author": ("meta", {"name": "author"}),- }-- @staticmethod- def _fetch_url_content(url):- response = requests.get(url)- response.raise_for_status()- return response.text-- @staticmethod- def _parse_html(html):- return BeautifulSoup(html, "html.parser")-- @staticmethod- def _extract_metadata(soup, url):- metadata_dict = {}- for attr, selector in UrlToMarkdownConverter.METADATA_ATTRS.items():- if selector:- element = soup.find(*selector)- metadata_dict[attr] = element.get("content", "").strip() if element else ""- else:- metadata_dict[attr] = url- return WebDocumentMetadata(**metadata_dict)-- @staticmethod- def _convert_to_markdown(html):- class CustomMarkdownConverter(MarkdownConverter):- def convert_img(self, el, text, convert_as_inline):- return ""-- def convert_p(self, el, text, convert_as_inline):- return text + "\n\n"-- for script in html.find_all("script"):- script.decompose()-- for style in html.find_all("style"):- style.decompose()-- converter = CustomMarkdownConverter()- markdown_text = converter.convert(str(html))- # Replace more than two consecutive newlines with exactly two newlines- markdown_text = re.sub(r"\n{3,}", "\n\n", markdown_text)- return markdown_text-- @staticmethod- def convert(url):- html = UrlToMarkdownConverter._fetch_url_content(url)- soup = UrlToMarkdownConverter._parse_html(html)- metadata = UrlToMarkdownConverter._extract_metadata(soup, url)- markdown_content = UrlToMarkdownConverter._convert_to_markdown(soup)-- return WebDocument(content=markdown_content, metadata=metadata)---if __name__ == "__main__":- url = "https://brainblendai.com"- document = UrlToMarkdownConverter.convert(url)- print(document)diff --git a/atomic-agents/tests/lib/tools/test_calculator_tool.py b/atomic-agents/tests/lib/tools/test_calculator_tool.pydeleted file mode 100644index e232ac3..0000000--- a/atomic-agents/tests/lib/tools/test_calculator_tool.py+++ /dev/null@@ -1,90 +0,0 @@-import pytest-from atomic_agents.lib.tools.calculator_tool import (- CalculatorTool,- CalculatorToolInputSchema,- CalculatorToolOutputSchema,- CalculatorToolConfig,-)-from atomic_agents.lib.tools.base_tool import BaseTool, BaseToolConfig---def test_calculator_tool_initialization():- tool = CalculatorTool()- assert isinstance(tool, BaseTool)- assert tool.tool_name == "CalculatorToolInputSchema"- assert "Tool for performing calculations" in tool.tool_description---def test_calculator_tool_with_custom_config():- config = CalculatorToolConfig(title="Custom Calculator", description="Custom description")- tool = CalculatorTool(config=config)- assert tool.tool_name == "Custom Calculator"- assert tool.tool_description == "Custom description"---def test_calculator_tool_input_schema():- tool = CalculatorTool()- assert tool.input_schema == CalculatorToolInputSchema---def test_calculator_tool_output_schema():- tool = CalculatorTool()- assert tool.output_schema == CalculatorToolOutputSchema--- "expression, expected_result",- [- ("2 + 2", "4.00000000000000"),- ("3 * 4", "12.0000000000000"),- ("10 / 2", "5.00000000000000"),- ("2 ** 3", "8.00000000000000"),- ("sin(pi/2)", "1.00000000000000"),- ("log(E)", "1.00000000000000"), # Changed from "log(e)" to "log(E)"- ],-)-def test_calculator_tool_run_valid_expressions(expression, expected_result):- tool = CalculatorTool()- result = tool.run(CalculatorToolInputSchema(expression=expression))- assert isinstance(result, CalculatorToolOutputSchema)- assert result.result == expected_result---def test_calculator_tool_run_complex_expression():- tool = CalculatorTool()- expression = "sqrt(3**2 + 4**2)"- result = tool.run(CalculatorToolInputSchema(expression=expression))- assert isinstance(result, CalculatorToolOutputSchema)- assert result.result == "5.00000000000000"---def test_calculator_tool_run_invalid_expression():- tool = CalculatorTool()- invalid_expression = "invalid + expression"- result = tool.run(CalculatorToolInputSchema(expression=invalid_expression))- assert isinstance(result, CalculatorToolOutputSchema)- assert set(result.result.split(" + ")) == set(invalid_expression.split(" + "))---def test_calculator_tool_run_division_by_zero():- tool = CalculatorTool()- result = tool.run(CalculatorToolInputSchema(expression="1/0"))- assert isinstance(result, CalculatorToolOutputSchema)- assert result.result == "zoo" # SymPy returns "zoo" for complex infinity---def test_calculator_tool_run_with_variables():- tool = CalculatorTool()- result = tool.run(CalculatorToolInputSchema(expression="x + y"))- assert isinstance(result, CalculatorToolOutputSchema)- assert result.result == "x + y" # The tool doesn't evaluate expressions with variables---def test_calculator_tool_config_is_pydantic_model():- assert issubclass(CalculatorToolConfig, BaseToolConfig)---def test_calculator_tool_config_optional_fields():- config = CalculatorToolConfig()- assert hasattr(config, "title")- assert hasattr(config, "description")diff --git a/atomic-agents/tests/lib/tools/test_content_scraping_tool.py b/atomic-agents/tests/lib/tools/test_content_scraping_tool.pydeleted file mode 100644index b12f171..0000000--- a/atomic-agents/tests/lib/tools/test_content_scraping_tool.py+++ /dev/null@@ -1,101 +0,0 @@-import pytest-from unittest.mock import patch, MagicMock-from atomic_agents.lib.tools.content_scraping_tool import (- ContentScrapingTool,- ContentScrapingToolInputSchema,- ContentScrapingToolOutputSchema,- ContentScrapingToolConfig,-)-from atomic_agents.lib.tools.base_tool import BaseTool, BaseToolConfig-from atomic_agents.lib.utils.scraping.pdf_to_markdown import PdfToMarkdownConverter-from atomic_agents.lib.utils.scraping.url_to_markdown import UrlToMarkdownConverter---def test_content_scraping_tool_initialization():- tool = ContentScrapingTool()- assert isinstance(tool, BaseTool)- assert tool.tool_name == "ContentScrapingToolInputSchema"- assert "Tool for scraping web pages or PDFs" in tool.tool_description---def test_content_scraping_tool_with_custom_config():- config = ContentScrapingToolConfig(title="Custom Scraper", description="Custom description")- tool = ContentScrapingTool(config=config)- assert tool.tool_name == "Custom Scraper"- assert tool.tool_description == "Custom description"---def test_content_scraping_tool_input_schema():- tool = ContentScrapingTool()- assert tool.input_schema == ContentScrapingToolInputSchema---def test_content_scraping_tool_output_schema():- tool = ContentScrapingTool()- assert tool.output_schema == ContentScrapingToolOutputSchema--- "content_type, expected_converter",- [- ("application/pdf", PdfToMarkdownConverter),- ("text/html", UrlToMarkdownConverter),- ],-)-@patch("atomic_agents.lib.tools.content_scraping_tool.requests.head")-def test_content_scraping_tool_run(mock_head, content_type, expected_converter):- mock_response = MagicMock()- mock_response.headers = {"Content-Type": content_type}- mock_head.return_value = mock_response-- mock_document = MagicMock()- mock_document.content = "Mocked content"- mock_document.metadata.model_dump.return_value = {"key": "value"}-- with patch.object(expected_converter, "convert", return_value=mock_document):- tool = ContentScrapingTool()- result = tool.run(ContentScrapingToolInputSchema(url="https://example.com"))-- assert isinstance(result, ContentScrapingToolOutputSchema)- assert result.content == "Mocked content"- assert result.metadata == {"key": "value"}---@patch("atomic_agents.lib.tools.content_scraping_tool.requests.head")-def test_content_scraping_tool_run_unknown_content_type(mock_head):- mock_response = MagicMock()- mock_response.headers = {"Content-Type": "unknown/type"}- mock_head.return_value = mock_response-- mock_document = MagicMock()- mock_document.content = "Mocked content"- mock_document.metadata.model_dump.return_value = {"key": "value"}-- with patch.object(UrlToMarkdownConverter, "convert", return_value=mock_document):- tool = ContentScrapingTool()- result = tool.run(ContentScrapingToolInputSchema(url="https://example.com"))-- assert isinstance(result, ContentScrapingToolOutputSchema)- assert result.content == "Mocked content"- assert result.metadata == {"key": "value"}---def test_content_scraping_tool_schema_fields():- schema = ContentScrapingToolInputSchema(url="https://example.com")- assert schema.url == "https://example.com"---def test_content_scraping_tool_output_schema_fields():- output = ContentScrapingToolOutputSchema(content="Test content", metadata={"key": "value"})- assert output.content == "Test content"- assert output.metadata == {"key": "value"}---def test_content_scraping_tool_config_is_pydantic_model():- assert issubclass(ContentScrapingToolConfig, BaseToolConfig)---def test_content_scraping_tool_config_optional_fields():- config = ContentScrapingToolConfig()- assert hasattr(config, "title")- assert hasattr(config, "description")diff --git a/atomic-agents/tests/lib/tools/test_yelp_restaurant_finder_tool.py b/atomic-agents/tests/lib/tools/test_yelp_restaurant_finder_tool.pydeleted file mode 100644index 0c8811d..0000000--- a/atomic-agents/tests/lib/tools/test_yelp_restaurant_finder_tool.py+++ /dev/null@@ -1,141 +0,0 @@-import pytest-from unittest.mock import patch, Mock-from pydantic import ValidationError--from atomic_agents.lib.tools.yelp_restaurant_finder_tool import (- YelpSearchTool,- YelpSearchToolConfig,- YelpSearchToolInputSchema,- YelpSearchToolOutputSchema,- YelpCategory,- PriceRange-)-from atomic_agents.lib.tools.base_tool import BaseTool-from atomic_agents.agents.base_agent import BaseIOSchema--# Sample data-SAMPLE_YELP_RESPONSE = {- "businesses": [- {- "name": "Sample Restaurant",- "url": "https://www.yelp.com/biz/sample-restaurant",- "rating": 4.5,- "review_count": 100,- "location": {"display_address": ["123 Main St", "Sample City, ST 12345"]},- "display_phone": "+1 (123) 456-7890",- "categories": [{"title": "Italian"}, {"title": "Pizza"}]- }- ]-}--def yelp_search_tool():- config = YelpSearchToolConfig(api_key="dummy_api_key", max_results=10)- return YelpSearchTool(config)--def test_yelp_search_tool_initialization(yelp_search_tool):- assert isinstance(yelp_search_tool, BaseTool)- assert yelp_search_tool.api_key == "dummy_api_key"- assert yelp_search_tool.max_results == 10--def test_yelp_search_tool_input_schema():- assert issubclass(YelpSearchToolInputSchema, BaseIOSchema)- assert "location" in YelpSearchToolInputSchema.model_fields- assert "term" in YelpSearchToolInputSchema.model_fields- assert "categories" in YelpSearchToolInputSchema.model_fields- assert "price" in YelpSearchToolInputSchema.model_fields--def test_yelp_search_tool_output_schema():- assert issubclass(YelpSearchToolOutputSchema, BaseIOSchema)- assert "results" in YelpSearchToolOutputSchema.model_fields--@patch('atomic_agents.lib.tools.yelp_restaurant_finder_tool.requests.get')-def test_run_success(mock_get, yelp_search_tool):- mock_response = Mock()- mock_response.status_code = 200- mock_response.json.return_value = SAMPLE_YELP_RESPONSE- mock_get.return_value = mock_response-- input_data = YelpSearchToolInputSchema(- location="Sample City",- term="pizza",- categories=[YelpCategory.ITALIAN],- price=[PriceRange.TWO, PriceRange.THREE]- )- result = yelp_search_tool.run(input_data)-- assert isinstance(result, YelpSearchToolOutputSchema)- assert len(result.results) == 1- assert result.results[0].name == "Sample Restaurant"- assert result.results[0].rating == 4.5- assert result.results[0].review_count == 100- assert result.results[0].address == "123 Main St, Sample City, ST 12345"- assert result.results[0].phone == "+1 (123) 456-7890"- assert result.results[0].categories == ["Italian", "Pizza"]-- mock_get.assert_called_once()- call_args = mock_get.call_args[1]- assert call_args['headers'] == {"Authorization": "Bearer dummy_api_key"}- assert call_args['params']['location'] == "Sample City"- assert call_args['params']['term'] == "pizza"- assert call_args['params']['categories'] == "italian"- assert call_args['params']['price'] == "2,3"--@patch('atomic_agents.lib.tools.yelp_restaurant_finder_tool.requests.get')-def test_run_api_error(mock_get, yelp_search_tool):- mock_response = Mock()- mock_response.status_code = 400- mock_response.reason = "Bad Request"- mock_get.return_value = mock_response-- input_data = YelpSearchToolInputSchema(location="Sample City")-- with pytest.raises(Exception, match="Failed to fetch search results: 400 Bad Request"):- yelp_search_tool.run(input_data)--@patch('atomic_agents.lib.tools.yelp_restaurant_finder_tool.requests.get')-def test_run_with_all_parameters(mock_get, yelp_search_tool):- mock_response = Mock()- mock_response.status_code = 200- mock_response.json.return_value = SAMPLE_YELP_RESPONSE- mock_get.return_value = mock_response-- input_data = YelpSearchToolInputSchema(- location="Sample City",- term="pizza",- categories=[YelpCategory.ITALIAN, YelpCategory.PIZZA],- price=[PriceRange.TWO, PriceRange.THREE],- open_now=True,- sort_by="rating"- )- yelp_search_tool.run(input_data)-- mock_get.assert_called_once()- call_args = mock_get.call_args[1]- assert call_args['params']['location'] == "Sample City"- assert call_args['params']['term'] == "pizza"- assert call_args['params']['categories'] == "italian,pizza"- assert call_args['params']['price'] == "2,3"- assert call_args['params']['open_now'] is True- assert call_args['params']['sort_by'] == "rating"--@patch('atomic_agents.lib.tools.yelp_restaurant_finder_tool.requests.get')-def test_run_no_results(mock_get, yelp_search_tool):- mock_response = Mock()- mock_response.status_code = 200- mock_response.json.return_value = {"businesses": []}- mock_get.return_value = mock_response-- input_data = YelpSearchToolInputSchema(location="Sample City")- result = yelp_search_tool.run(input_data)-- assert isinstance(result, YelpSearchToolOutputSchema)- assert len(result.results) == 0--def test_run_missing_api_key():- config = YelpSearchToolConfig(api_key="", max_results=10)- tool = YelpSearchTool(config)- input_data = YelpSearchToolInputSchema(location="Sample City")-- with pytest.raises(ValueError, match="API key is required to use the Yelp API."):- tool.run(input_data)\ No newline at end of filediff --git a/atomic-agents/tests/lib/tools/test_yt_transcript_scraper_tool.py b/atomic-agents/tests/lib/tools/test_yt_transcript_scraper_tool.pydeleted file mode 100644index 321a0df..0000000--- a/atomic-agents/tests/lib/tools/test_yt_transcript_scraper_tool.py+++ /dev/null@@ -1,179 +0,0 @@-import pytest-from unittest.mock import patch, Mock-from pydantic import ValidationError-from youtube_transcript_api import YouTubeTranscriptApi, NoTranscriptFound, TranscriptsDisabled-from atomic_agents.lib.tools.yt_transcript_scraper_tool import YouTubeTranscriptToolInputSchema-from atomic_agents.lib.tools.yt_transcript_scraper_tool import (- YouTubeTranscriptTool,- YouTubeTranscriptToolConfig,- YouTubeTranscriptToolInputSchema,- YouTubeTranscriptToolOutputSchema,-)-from atomic_agents.lib.tools.base_tool import BaseTool-from atomic_agents.agents.base_agent import BaseIOSchema-from youtube_transcript_api import YouTubeTranscriptApi---# Sample data-SAMPLE_TRANSCRIPT = [- {"text": "This is the first part of the transcript.", "start": 0.0, "duration": 5.0},- {"text": "This is the second part.", "start": 5.0, "duration": 3.0},-]--SAMPLE_VIDEO_INFO = {- "id": "dQw4w9WgXcQ",- "title": "Sample Video",- "channel": "Sample Channel",- "published_at": "2023-01-01T00:00:00Z",-}--def youtube_transcript_tool():- config = YouTubeTranscriptToolConfig(api_key="dummy_api_key")- return YouTubeTranscriptTool(config)--def test_youtube_transcript_tool_initialization(youtube_transcript_tool):- assert isinstance(youtube_transcript_tool, BaseTool)- assert youtube_transcript_tool.api_key == "dummy_api_key"--def test_youtube_transcript_tool_input_schema():- assert issubclass(YouTubeTranscriptToolInputSchema, BaseIOSchema)- assert "video_url" in YouTubeTranscriptToolInputSchema.model_fields- assert "language" in YouTubeTranscriptToolInputSchema.model_fields--def test_youtube_transcript_tool_output_schema():- assert issubclass(YouTubeTranscriptToolOutputSchema, BaseIOSchema)- assert "transcript" in YouTubeTranscriptToolOutputSchema.model_fields- assert "duration" in YouTubeTranscriptToolOutputSchema.model_fields- assert "comments" in YouTubeTranscriptToolOutputSchema.model_fields- assert "metadata" in YouTubeTranscriptToolOutputSchema.model_fields--def test_extract_video_id():- url = "https://www.youtube.com/watch?v=dQw4w9WgXcQ"- video_id = YouTubeTranscriptTool.extract_video_id(url)- assert video_id == "dQw4w9WgXcQ"--@patch('atomic_agents.lib.tools.yt_transcript_scraper_tool.YouTubeTranscriptTool.extract_video_id')-@patch('atomic_agents.lib.tools.yt_transcript_scraper_tool.YouTubeTranscriptTool.fetch_video_metadata')-@patch('atomic_agents.lib.tools.yt_transcript_scraper_tool.YouTubeTranscriptApi.get_transcript')-def test_run(mock_get_transcript, mock_fetch_metadata, mock_extract_video_id, youtube_transcript_tool):- mock_extract_video_id.return_value = "dQw4w9WgXcQ"- mock_fetch_metadata.return_value = SAMPLE_VIDEO_INFO- mock_get_transcript.return_value = SAMPLE_TRANSCRIPT-- input_data = YouTubeTranscriptToolInputSchema(video_url="https://www.youtube.com/watch?v=dQw4w9WgXcQ")- result = youtube_transcript_tool.run(input_data)-- assert isinstance(result, YouTubeTranscriptToolOutputSchema)- assert result.transcript == "This is the first part of the transcript. This is the second part."- assert result.duration == 8.0- assert result.comments == []- assert result.metadata == SAMPLE_VIDEO_INFO--@patch('atomic_agents.lib.tools.yt_transcript_scraper_tool.YouTubeTranscriptTool.extract_video_id')-@patch('atomic_agents.lib.tools.yt_transcript_scraper_tool.YouTubeTranscriptTool.fetch_video_metadata')-@patch('atomic_agents.lib.tools.yt_transcript_scraper_tool.YouTubeTranscriptApi.get_transcript')-def test_run_with_language(mock_get_transcript, mock_fetch_metadata, mock_extract_video_id, youtube_transcript_tool):- mock_extract_video_id.return_value = "dQw4w9WgXcQ"- mock_fetch_metadata.return_value = SAMPLE_VIDEO_INFO- mock_get_transcript.return_value = SAMPLE_TRANSCRIPT-- input_data = YouTubeTranscriptToolInputSchema(- video_url="https://www.youtube.com/watch?v=dQw4w9WgXcQ",- language="en"- )- youtube_transcript_tool.run(input_data)-- mock_get_transcript.assert_called_once_with("dQw4w9WgXcQ", languages=["en"])--@patch('atomic_agents.lib.tools.yt_transcript_scraper_tool.YouTubeTranscriptTool.extract_video_id')-@patch('atomic_agents.lib.tools.yt_transcript_scraper_tool.YouTubeTranscriptTool.fetch_video_metadata')-@patch('atomic_agents.lib.tools.yt_transcript_scraper_tool.YouTubeTranscriptApi.get_transcript')-def test_run_transcript_not_found(mock_get_transcript, mock_fetch_metadata, mock_extract_video_id, youtube_transcript_tool):- mock_extract_video_id.return_value = "dQw4w9WgXcQ"- mock_fetch_metadata.return_value = SAMPLE_VIDEO_INFO- mock_get_transcript.side_effect = Exception("No transcript found")-- input_data = YouTubeTranscriptToolInputSchema(video_url="https://www.youtube.com/watch?v=dQw4w9WgXcQ")-- with pytest.raises(Exception, match="No transcript found"):- youtube_transcript_tool.run(input_data)--def test_youtube_transcript_tool_config_validation():- with pytest.raises(ValidationError):- YouTubeTranscriptToolConfig(api_key=None) # api_key is required and should not be None-- config = YouTubeTranscriptToolConfig(api_key="dummy_api_key")- assert config.api_key == "dummy_api_key"-[email protected](YouTubeTranscriptApi, 'get_transcript')-@patch('atomic_agents.lib.tools.yt_transcript_scraper_tool.YouTubeTranscriptTool.fetch_video_metadata')-@patch('atomic_agents.lib.tools.yt_transcript_scraper_tool.YouTubeTranscriptTool.extract_video_id')-def test_run_no_transcript_found(mock_extract_video_id, mock_fetch_metadata, mock_get_transcript, youtube_transcript_tool):- mock_extract_video_id.return_value = "dQw4w9WgXcQ"- mock_fetch_metadata.return_value = SAMPLE_VIDEO_INFO- mock_get_transcript.side_effect = NoTranscriptFound("dQw4w9WgXcQ", "en", "Transcript not found")-- input_data = YouTubeTranscriptToolInputSchema(video_url="https://www.youtube.com/watch?v=dQw4w9WgXcQ")-- with pytest.raises(Exception, match="Failed to fetch transcript for video 'dQw4w9WgXcQ': "):- youtube_transcript_tool.run(input_data)-[email protected](YouTubeTranscriptApi, 'get_transcript')-@patch('atomic_agents.lib.tools.yt_transcript_scraper_tool.YouTubeTranscriptTool.fetch_video_metadata')-@patch('atomic_agents.lib.tools.yt_transcript_scraper_tool.YouTubeTranscriptTool.extract_video_id')-def test_run_transcripts_disabled(mock_extract_video_id, mock_fetch_metadata, mock_get_transcript, youtube_transcript_tool):- mock_extract_video_id.return_value = "dQw4w9WgXcQ"- mock_fetch_metadata.return_value = SAMPLE_VIDEO_INFO- mock_get_transcript.side_effect = TranscriptsDisabled("Transcripts are disabled for this video")-- input_data = YouTubeTranscriptToolInputSchema(video_url="https://www.youtube.com/watch?v=dQw4w9WgXcQ")-- with pytest.raises(Exception, match="Failed to fetch transcript for video 'dQw4w9WgXcQ': "):- youtube_transcript_tool.run(input_data)--@patch('atomic_agents.lib.tools.yt_transcript_scraper_tool.build')-def test_fetch_video_metadata_success(mock_build, youtube_transcript_tool):- mock_youtube = Mock()- mock_build.return_value = mock_youtube- mock_request = Mock()- mock_youtube.videos().list.return_value = mock_request- mock_request.execute.return_value = {- "items": [- {- "snippet": {- "title": "Sample Video",- "channelTitle": "Sample Channel",- "publishedAt": "2023-01-01T00:00:00Z",- }- }- ]- }-- video_id = "dQw4w9WgXcQ"- metadata = youtube_transcript_tool.fetch_video_metadata(video_id)-- assert metadata == {- "id": video_id,- "title": "Sample Video",- "channel": "Sample Channel",- "published_at": "2023-01-01T00:00:00Z",- }- mock_build.assert_called_once_with("youtube", "v3", developerKey=youtube_transcript_tool.api_key)- mock_youtube.videos().list.assert_called_once_with(part="snippet", id=video_id)--@patch('atomic_agents.lib.tools.yt_transcript_scraper_tool.build')-def test_fetch_video_metadata_no_items(mock_build, youtube_transcript_tool):- mock_youtube = Mock()- mock_build.return_value = mock_youtube- mock_request = Mock()- mock_youtube.videos().list.return_value = mock_request- mock_request.execute.return_value = {"items": []}-- video_id = "dQw4w9WgXcQ"-- with pytest.raises(Exception, match=f"No metadata found for video '{video_id}'"):- youtube_transcript_tool.fetch_video_metadata(video_id)-- mock_build.assert_called_once_with("youtube", "v3", developerKey=youtube_transcript_tool.api_key)- mock_youtube.videos().list.assert_called_once_with(part="snippet", id=video_id)diff --git a/atomic-agents/tests/lib/utils/scraping/test_pdf_to_markdown.py b/atomic-agents/tests/lib/utils/scraping/test_pdf_to_markdown.pydeleted file mode 100644index 68925cc..0000000--- a/atomic-agents/tests/lib/utils/scraping/test_pdf_to_markdown.py+++ /dev/null@@ -1,128 +0,0 @@-import pytest-import requests-from unittest.mock import patch, Mock, mock_open-from io import BytesIO-from atomic_agents.lib.utils.scraping.pdf_to_markdown import PdfToMarkdownConverter-from atomic_agents.lib.models.web_document import WebDocument, WebDocumentMetadata-from PyPDF2.errors import PdfReadError--# Sample PDF content (mock)-SAMPLE_PDF_CONTENT = b"""%PDF-1.3-1 0 obj-<< /Type /Catalog /Pages 2 0 R >>-endobj-2 0 obj-<< /Type /Pages /Kids [3 0 R] /Count 1 >>-endobj-3 0 obj-<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Contents 4 0 R >>-endobj-4 0 obj-<< /Length 51 >>-stream-BT /F1 12 Tf 72 712 Td (Sample PDF content) Tj ET-endstream-endobj-xref-0 5-0000000000 65535 f-0000000009 00000 n-0000000058 00000 n-0000000115 00000 n-0000000198 00000 n-trailer << /Size 5 /Root 1 0 R >>-startxref-298-%%EOF"""---def mock_requests_get():- with patch('requests.get') as mock_get:- response = Mock()- response.content = SAMPLE_PDF_CONTENT- response.raise_for_status.return_value = None- mock_get.return_value = response- yield mock_get--def mock_pdf_reader():- with patch('PyPDF2.PdfReader') as MockPdfReader:- mock_page = Mock()- mock_page.extract_text.return_value = "Sample PDF content"- mock_reader = Mock()- mock_reader.pages = [mock_page]- MockPdfReader.return_value = mock_reader- yield MockPdfReader--def test_fetch_pdf_content(mock_requests_get):- url = "https://example.com/sample.pdf"- content = PdfToMarkdownConverter._fetch_pdf_content(url)-- mock_requests_get.assert_called_once_with(url)- assert isinstance(content, BytesIO)- assert content.getvalue() == SAMPLE_PDF_CONTENT--def test_fetch_pdf_content_error(mock_requests_get):- mock_requests_get.return_value.raise_for_status.side_effect = requests.HTTPError("404 Not Found")-- with pytest.raises(requests.HTTPError):- PdfToMarkdownConverter._fetch_pdf_content("https://example.com/nonexistent.pdf")--def test_read_pdf():- file = BytesIO(SAMPLE_PDF_CONTENT)-- mock_page = Mock()- mock_page.extract_text.return_value = "Sample PDF content"-- mock_reader = Mock()- mock_reader.pages = [mock_page]-- with patch('atomic_agents.lib.utils.scraping.pdf_to_markdown.PdfReader', return_value=mock_reader) as mock_pdf_reader:- content = PdfToMarkdownConverter._read_pdf(file)-- mock_pdf_reader.assert_called_once_with(file)- assert content == "Sample PDF content"---def test_convert_to_markdown():- html_content = "<h1>Title</h1><p>This is a paragraph.</p><p>Another paragraph.</p>"- markdown = PdfToMarkdownConverter._convert_to_markdown(html_content)-- expected_markdown = "# Title\n\nThis is a paragraph.\n\nAnother paragraph."- assert markdown.strip() == expected_markdown.strip()--@patch('atomic_agents.lib.utils.scraping.pdf_to_markdown.PdfToMarkdownConverter._fetch_pdf_content')-@patch('atomic_agents.lib.utils.scraping.pdf_to_markdown.PdfToMarkdownConverter._read_pdf')-@patch('atomic_agents.lib.utils.scraping.pdf_to_markdown.PdfToMarkdownConverter._convert_to_markdown')-def test_convert_from_url(mock_convert, mock_read, mock_fetch):- url = "https://example.com/sample.pdf"- mock_fetch.return_value = BytesIO(SAMPLE_PDF_CONTENT)- mock_read.return_value = "Sample PDF content"- mock_convert.return_value = "# Sample PDF content"-- result = PdfToMarkdownConverter.convert(url=url)-- assert isinstance(result, WebDocument)- assert result.content == "# Sample PDF content"- assert isinstance(result.metadata, WebDocumentMetadata)- assert result.metadata.url == url--@patch('builtins.open', new_callable=mock_open, read_data=SAMPLE_PDF_CONTENT)-@patch('atomic_agents.lib.utils.scraping.pdf_to_markdown.PdfToMarkdownConverter._read_pdf')-@patch('atomic_agents.lib.utils.scraping.pdf_to_markdown.PdfToMarkdownConverter._convert_to_markdown')-def test_convert_from_file(mock_convert, mock_read, mock_file):- file_path = "/path/to/sample.pdf"- mock_read.return_value = "Sample PDF content"- mock_convert.return_value = "# Sample PDF content"-- result = PdfToMarkdownConverter.convert(file_path=file_path)-- assert isinstance(result, WebDocument)- assert result.content == "# Sample PDF content"- assert isinstance(result.metadata, WebDocumentMetadata)- assert result.metadata.url == file_path--def test_convert_no_input():- with pytest.raises(ValueError, match="Either url or file_path must be provided"):- PdfToMarkdownConverter.convert()\ No newline at end of filediff --git a/atomic-agents/tests/lib/utils/scraping/test_url_to_markdown.py b/atomic-agents/tests/lib/utils/scraping/test_url_to_markdown.pydeleted file mode 100644index 904bafc..0000000--- a/atomic-agents/tests/lib/utils/scraping/test_url_to_markdown.py+++ /dev/null@@ -1,111 +0,0 @@-import pytest-from unittest.mock import patch, Mock-from bs4 import BeautifulSoup-from atomic_agents.lib.utils.scraping.url_to_markdown import UrlToMarkdownConverter-from atomic_agents.lib.models.web_document import WebDocument, WebDocumentMetadata--SAMPLE_HTML = """-<!DOCTYPE html>-<html>-<head>- <title>Sample Page</title>- <meta name="description" content="This is a sample page">- <meta name="keywords" content="sample, test, page">- <meta name="author" content="John Doe">-</head>-<body>- <h1>Welcome to the Sample Page</h1>- <p>This is a paragraph.</p>- <p>Another paragraph with <a href="https://example.com">a link</a>.</p>- <img src="sample.jpg" alt="Sample Image">- <script>console.log("This should be removed");</script>- <style>body { color: black; }</style>-</body>-</html>-"""--def mock_requests_get():- with patch('requests.get') as mock_get:- response = Mock()- response.text = SAMPLE_HTML- response.raise_for_status.return_value = None- mock_get.return_value = response- yield mock_get--def test_fetch_url_content(mock_requests_get):- url = "https://example.com"- content = UrlToMarkdownConverter._fetch_url_content(url)-- mock_requests_get.assert_called_once_with(url)- assert content == SAMPLE_HTML--def test_parse_html():- soup = UrlToMarkdownConverter._parse_html(SAMPLE_HTML)- assert isinstance(soup, BeautifulSoup)- assert soup.title.string == "Sample Page"--def test_extract_metadata():- soup = BeautifulSoup(SAMPLE_HTML, "html.parser")- url = "https://example.com"- metadata = UrlToMarkdownConverter._extract_metadata(soup, url)-- assert isinstance(metadata, WebDocumentMetadata)- assert metadata.url == url- assert metadata.title == "" # Adjusted expectation- assert metadata.description == "This is a sample page"- assert metadata.keywords == "sample, test, page"- assert metadata.author == "John Doe"--def test_convert_to_markdown():- soup = BeautifulSoup(SAMPLE_HTML, "html.parser")- markdown = UrlToMarkdownConverter._convert_to_markdown(soup)-- expected_markdown = """Sample Page--Welcome to the Sample Page-==========================--This is a paragraph.--Another paragraph with [a link](https://example.com).--"""- assert markdown.strip() == expected_markdown.strip()--@patch('atomic_agents.lib.utils.scraping.url_to_markdown.UrlToMarkdownConverter._fetch_url_content')-@patch('atomic_agents.lib.utils.scraping.url_to_markdown.UrlToMarkdownConverter._parse_html')-@patch('atomic_agents.lib.utils.scraping.url_to_markdown.UrlToMarkdownConverter._extract_metadata')-@patch('atomic_agents.lib.utils.scraping.url_to_markdown.UrlToMarkdownConverter._convert_to_markdown')-def test_convert(mock_convert, mock_extract, mock_parse, mock_fetch):- url = "https://example.com"- mock_fetch.return_value = SAMPLE_HTML- mock_parse.return_value = BeautifulSoup(SAMPLE_HTML, "html.parser")- mock_extract.return_value = WebDocumentMetadata(url=url, title="")- mock_convert.return_value = "Sample Page\n\nThis is the content."-- result = UrlToMarkdownConverter.convert(url)-- assert isinstance(result, WebDocument)- assert result.content == "Sample Page\n\nThis is the content."- assert isinstance(result.metadata, WebDocumentMetadata)- assert result.metadata.url == url- assert result.metadata.title == ""--def test_convert_integration(mock_requests_get):- url = "https://example.com"- result = UrlToMarkdownConverter.convert(url)-- assert isinstance(result, WebDocument)- assert "Welcome to the Sample Page" in result.content- assert "This is a paragraph." in result.content- assert "[a link](https://example.com)" in result.content- assert "console.log" not in result.content- assert "body { color: black; }" not in result.content-- assert isinstance(result.metadata, WebDocumentMetadata)- assert result.metadata.url == url- assert result.metadata.title == "" # Adjusted expectation- assert result.metadata.description == "This is a sample page"- assert result.metadata.keywords == "sample, test, page"- assert result.metadata.author == "John Doe"\ No newline at end of file