Skip to content

Building AI agents, atomically

Remove anything that will move to another repo

KennyVaneetveldea536bbcaa534
diff --git a/atomic-agents/atomic_agents/lib/models/__init__.py b/atomic-agents/atomic_agents/lib/models/__init__.py
deleted file mode 100644
index e69de29..0000000
diff --git a/atomic-agents/atomic_agents/lib/models/web_document.py b/atomic-agents/atomic_agents/lib/models/web_document.py
deleted file mode 100644
index 3486201..0000000
--- a/atomic-agents/atomic_agents/lib/models/web_document.py
+++ /dev/null
@@ -1,20 +0,0 @@
-from pydantic import Field
-
-from atomic_agents.agents.base_agent import BaseIOSchema
-
-
-class WebDocumentMetadata(BaseIOSchema):
- """Schema that describes the metadata of a web document."""
-
- url: str
- title: str = Field(default="")
- description: str = Field(default="")
- keywords: str = Field(default="")
- author: str = Field(default="")
-
-
-class WebDocument(BaseIOSchema):
- """Schema that represents the content and metadata of a web document."""
-
- content: str
- metadata: WebDocumentMetadata = Field(default_factory=lambda: WebDocumentMetadata(url=""))
diff --git a/atomic-agents/atomic_agents/lib/utils/scraping/__init__.py b/atomic-agents/atomic_agents/lib/utils/scraping/__init__.py
deleted file mode 100644
index e69de29..0000000
diff --git a/atomic-agents/atomic_agents/lib/utils/scraping/pdf_to_markdown.py b/atomic-agents/atomic_agents/lib/utils/scraping/pdf_to_markdown.py
deleted file mode 100644
index 109f57e..0000000
--- a/atomic-agents/atomic_agents/lib/utils/scraping/pdf_to_markdown.py
+++ /dev/null
@@ -1,60 +0,0 @@
-import re
-from io import BytesIO
-
-import markdownify
-import requests
-from PyPDF2 import PdfReader
-
-from atomic_agents.lib.models.web_document import WebDocument, WebDocumentMetadata
-
-
-class PdfToMarkdownConverter:
- @staticmethod
- def _fetch_pdf_content(url):
- response = requests.get(url)
- response.raise_for_status()
- return BytesIO(response.content)
-
- @staticmethod
- def _read_pdf(file):
- pdf = PdfReader(file)
- content = []
- for page_num in range(len(pdf.pages)):
- page = pdf.pages[page_num]
- content.append(page.extract_text())
- return "\n".join(content)
-
- @staticmethod
- def _convert_to_markdown(text):
- markdown_text = markdownify.markdownify(text, heading_style="ATX")
- # Replace more than two consecutive newlines with exactly two newlines
- markdown_text = re.sub(r"\n{3,}", "\n\n", markdown_text)
- return markdown_text
-
- @staticmethod
- def convert(url=None, file_path=None):
- if url:
- pdf_content = PdfToMarkdownConverter._fetch_pdf_content(url)
- elif file_path:
- with open(file_path, "rb") as f:
- pdf_content = BytesIO(f.read())
- else:
- raise ValueError("Either url or file_path must be provided")
-
- text_content = PdfToMarkdownConverter._read_pdf(pdf_content)
- markdown_content = PdfToMarkdownConverter._convert_to_markdown(text_content)
- metadata = WebDocumentMetadata(url=url or file_path, title="", description="", keywords="", author="")
-
- return WebDocument(content=markdown_content, metadata=metadata)
-
-
-if __name__ == "__main__":
- from rich.console import Console
- from rich.markdown import Markdown
-
- console = Console()
-
- url = "https://pdfobject.com/pdf/sample.pdf"
- document = PdfToMarkdownConverter.convert(url=url)
-
- console.print(Markdown(document.content))
diff --git a/atomic-agents/atomic_agents/lib/utils/scraping/url_to_markdown.py b/atomic-agents/atomic_agents/lib/utils/scraping/url_to_markdown.py
deleted file mode 100644
index 020842a..0000000
--- a/atomic-agents/atomic_agents/lib/utils/scraping/url_to_markdown.py
+++ /dev/null
@@ -1,74 +0,0 @@
-import re
-
-import requests
-from bs4 import BeautifulSoup
-from markdownify import MarkdownConverter
-
-from atomic_agents.lib.models.web_document import WebDocument, WebDocumentMetadata
-
-
-class UrlToMarkdownConverter:
- METADATA_ATTRS = {
- "url": "",
- "title": ("title", None),
- "description": ("meta", {"name": "description"}),
- "keywords": ("meta", {"name": "keywords"}),
- "author": ("meta", {"name": "author"}),
- }
-
- @staticmethod
- def _fetch_url_content(url):
- response = requests.get(url)
- response.raise_for_status()
- return response.text
-
- @staticmethod
- def _parse_html(html):
- return BeautifulSoup(html, "html.parser")
-
- @staticmethod
- def _extract_metadata(soup, url):
- metadata_dict = {}
- for attr, selector in UrlToMarkdownConverter.METADATA_ATTRS.items():
- if selector:
- element = soup.find(*selector)
- metadata_dict[attr] = element.get("content", "").strip() if element else ""
- else:
- metadata_dict[attr] = url
- return WebDocumentMetadata(**metadata_dict)
-
- @staticmethod
- def _convert_to_markdown(html):
- class CustomMarkdownConverter(MarkdownConverter):
- def convert_img(self, el, text, convert_as_inline):
- return ""
-
- def convert_p(self, el, text, convert_as_inline):
- return text + "\n\n"
-
- for script in html.find_all("script"):
- script.decompose()
-
- for style in html.find_all("style"):
- style.decompose()
-
- converter = CustomMarkdownConverter()
- markdown_text = converter.convert(str(html))
- # Replace more than two consecutive newlines with exactly two newlines
- markdown_text = re.sub(r"\n{3,}", "\n\n", markdown_text)
- return markdown_text
-
- @staticmethod
- def convert(url):
- html = UrlToMarkdownConverter._fetch_url_content(url)
- soup = UrlToMarkdownConverter._parse_html(html)
- metadata = UrlToMarkdownConverter._extract_metadata(soup, url)
- markdown_content = UrlToMarkdownConverter._convert_to_markdown(soup)
-
- return WebDocument(content=markdown_content, metadata=metadata)
-
-
-if __name__ == "__main__":
- url = "https://brainblendai.com"
- document = UrlToMarkdownConverter.convert(url)
- print(document)
diff --git a/atomic-agents/tests/lib/tools/test_calculator_tool.py b/atomic-agents/tests/lib/tools/test_calculator_tool.py
deleted file mode 100644
index e232ac3..0000000
--- a/atomic-agents/tests/lib/tools/test_calculator_tool.py
+++ /dev/null
@@ -1,90 +0,0 @@
-import pytest
-from atomic_agents.lib.tools.calculator_tool import (
- CalculatorTool,
- CalculatorToolInputSchema,
- CalculatorToolOutputSchema,
- CalculatorToolConfig,
-)
-from atomic_agents.lib.tools.base_tool import BaseTool, BaseToolConfig
-
-
-def test_calculator_tool_initialization():
- tool = CalculatorTool()
- assert isinstance(tool, BaseTool)
- assert tool.tool_name == "CalculatorToolInputSchema"
- assert "Tool for performing calculations" in tool.tool_description
-
-
-def test_calculator_tool_with_custom_config():
- config = CalculatorToolConfig(title="Custom Calculator", description="Custom description")
- tool = CalculatorTool(config=config)
- assert tool.tool_name == "Custom Calculator"
- assert tool.tool_description == "Custom description"
-
-
-def test_calculator_tool_input_schema():
- tool = CalculatorTool()
- assert tool.input_schema == CalculatorToolInputSchema
-
-
-def test_calculator_tool_output_schema():
- tool = CalculatorTool()
- assert tool.output_schema == CalculatorToolOutputSchema
-
-
- "expression, expected_result",
- [
- ("2 + 2", "4.00000000000000"),
- ("3 * 4", "12.0000000000000"),
- ("10 / 2", "5.00000000000000"),
- ("2 ** 3", "8.00000000000000"),
- ("sin(pi/2)", "1.00000000000000"),
- ("log(E)", "1.00000000000000"), # Changed from "log(e)" to "log(E)"
- ],
-)
-def test_calculator_tool_run_valid_expressions(expression, expected_result):
- tool = CalculatorTool()
- result = tool.run(CalculatorToolInputSchema(expression=expression))
- assert isinstance(result, CalculatorToolOutputSchema)
- assert result.result == expected_result
-
-
-def test_calculator_tool_run_complex_expression():
- tool = CalculatorTool()
- expression = "sqrt(3**2 + 4**2)"
- result = tool.run(CalculatorToolInputSchema(expression=expression))
- assert isinstance(result, CalculatorToolOutputSchema)
- assert result.result == "5.00000000000000"
-
-
-def test_calculator_tool_run_invalid_expression():
- tool = CalculatorTool()
- invalid_expression = "invalid + expression"
- result = tool.run(CalculatorToolInputSchema(expression=invalid_expression))
- assert isinstance(result, CalculatorToolOutputSchema)
- assert set(result.result.split(" + ")) == set(invalid_expression.split(" + "))
-
-
-def test_calculator_tool_run_division_by_zero():
- tool = CalculatorTool()
- result = tool.run(CalculatorToolInputSchema(expression="1/0"))
- assert isinstance(result, CalculatorToolOutputSchema)
- assert result.result == "zoo" # SymPy returns "zoo" for complex infinity
-
-
-def test_calculator_tool_run_with_variables():
- tool = CalculatorTool()
- result = tool.run(CalculatorToolInputSchema(expression="x + y"))
- assert isinstance(result, CalculatorToolOutputSchema)
- assert result.result == "x + y" # The tool doesn't evaluate expressions with variables
-
-
-def test_calculator_tool_config_is_pydantic_model():
- assert issubclass(CalculatorToolConfig, BaseToolConfig)
-
-
-def test_calculator_tool_config_optional_fields():
- config = CalculatorToolConfig()
- assert hasattr(config, "title")
- assert hasattr(config, "description")
diff --git a/atomic-agents/tests/lib/tools/test_content_scraping_tool.py b/atomic-agents/tests/lib/tools/test_content_scraping_tool.py
deleted file mode 100644
index b12f171..0000000
--- a/atomic-agents/tests/lib/tools/test_content_scraping_tool.py
+++ /dev/null
@@ -1,101 +0,0 @@
-import pytest
-from unittest.mock import patch, MagicMock
-from atomic_agents.lib.tools.content_scraping_tool import (
- ContentScrapingTool,
- ContentScrapingToolInputSchema,
- ContentScrapingToolOutputSchema,
- ContentScrapingToolConfig,
-)
-from atomic_agents.lib.tools.base_tool import BaseTool, BaseToolConfig
-from atomic_agents.lib.utils.scraping.pdf_to_markdown import PdfToMarkdownConverter
-from atomic_agents.lib.utils.scraping.url_to_markdown import UrlToMarkdownConverter
-
-
-def test_content_scraping_tool_initialization():
- tool = ContentScrapingTool()
- assert isinstance(tool, BaseTool)
- assert tool.tool_name == "ContentScrapingToolInputSchema"
- assert "Tool for scraping web pages or PDFs" in tool.tool_description
-
-
-def test_content_scraping_tool_with_custom_config():
- config = ContentScrapingToolConfig(title="Custom Scraper", description="Custom description")
- tool = ContentScrapingTool(config=config)
- assert tool.tool_name == "Custom Scraper"
- assert tool.tool_description == "Custom description"
-
-
-def test_content_scraping_tool_input_schema():
- tool = ContentScrapingTool()
- assert tool.input_schema == ContentScrapingToolInputSchema
-
-
-def test_content_scraping_tool_output_schema():
- tool = ContentScrapingTool()
- assert tool.output_schema == ContentScrapingToolOutputSchema
-
-
- "content_type, expected_converter",
- [
- ("application/pdf", PdfToMarkdownConverter),
- ("text/html", UrlToMarkdownConverter),
- ],
-)
-@patch("atomic_agents.lib.tools.content_scraping_tool.requests.head")
-def test_content_scraping_tool_run(mock_head, content_type, expected_converter):
- mock_response = MagicMock()
- mock_response.headers = {"Content-Type": content_type}
- mock_head.return_value = mock_response
-
- mock_document = MagicMock()
- mock_document.content = "Mocked content"
- mock_document.metadata.model_dump.return_value = {"key": "value"}
-
- with patch.object(expected_converter, "convert", return_value=mock_document):
- tool = ContentScrapingTool()
- result = tool.run(ContentScrapingToolInputSchema(url="https://example.com"))
-
- assert isinstance(result, ContentScrapingToolOutputSchema)
- assert result.content == "Mocked content"
- assert result.metadata == {"key": "value"}
-
-
-@patch("atomic_agents.lib.tools.content_scraping_tool.requests.head")
-def test_content_scraping_tool_run_unknown_content_type(mock_head):
- mock_response = MagicMock()
- mock_response.headers = {"Content-Type": "unknown/type"}
- mock_head.return_value = mock_response
-
- mock_document = MagicMock()
- mock_document.content = "Mocked content"
- mock_document.metadata.model_dump.return_value = {"key": "value"}
-
- with patch.object(UrlToMarkdownConverter, "convert", return_value=mock_document):
- tool = ContentScrapingTool()
- result = tool.run(ContentScrapingToolInputSchema(url="https://example.com"))
-
- assert isinstance(result, ContentScrapingToolOutputSchema)
- assert result.content == "Mocked content"
- assert result.metadata == {"key": "value"}
-
-
-def test_content_scraping_tool_schema_fields():
- schema = ContentScrapingToolInputSchema(url="https://example.com")
- assert schema.url == "https://example.com"
-
-
-def test_content_scraping_tool_output_schema_fields():
- output = ContentScrapingToolOutputSchema(content="Test content", metadata={"key": "value"})
- assert output.content == "Test content"
- assert output.metadata == {"key": "value"}
-
-
-def test_content_scraping_tool_config_is_pydantic_model():
- assert issubclass(ContentScrapingToolConfig, BaseToolConfig)
-
-
-def test_content_scraping_tool_config_optional_fields():
- config = ContentScrapingToolConfig()
- assert hasattr(config, "title")
- assert hasattr(config, "description")
diff --git a/atomic-agents/tests/lib/tools/test_yelp_restaurant_finder_tool.py b/atomic-agents/tests/lib/tools/test_yelp_restaurant_finder_tool.py
deleted file mode 100644
index 0c8811d..0000000
--- a/atomic-agents/tests/lib/tools/test_yelp_restaurant_finder_tool.py
+++ /dev/null
@@ -1,141 +0,0 @@
-import pytest
-from unittest.mock import patch, Mock
-from pydantic import ValidationError
-
-from atomic_agents.lib.tools.yelp_restaurant_finder_tool import (
- YelpSearchTool,
- YelpSearchToolConfig,
- YelpSearchToolInputSchema,
- YelpSearchToolOutputSchema,
- YelpCategory,
- PriceRange
-)
-from atomic_agents.lib.tools.base_tool import BaseTool
-from atomic_agents.agents.base_agent import BaseIOSchema
-
-# Sample data
-SAMPLE_YELP_RESPONSE = {
- "businesses": [
- {
- "name": "Sample Restaurant",
- "url": "https://www.yelp.com/biz/sample-restaurant",
- "rating": 4.5,
- "review_count": 100,
- "location": {"display_address": ["123 Main St", "Sample City, ST 12345"]},
- "display_phone": "+1 (123) 456-7890",
- "categories": [{"title": "Italian"}, {"title": "Pizza"}]
- }
- ]
-}
-
-def yelp_search_tool():
- config = YelpSearchToolConfig(api_key="dummy_api_key", max_results=10)
- return YelpSearchTool(config)
-
-def test_yelp_search_tool_initialization(yelp_search_tool):
- assert isinstance(yelp_search_tool, BaseTool)
- assert yelp_search_tool.api_key == "dummy_api_key"
- assert yelp_search_tool.max_results == 10
-
-def test_yelp_search_tool_input_schema():
- assert issubclass(YelpSearchToolInputSchema, BaseIOSchema)
- assert "location" in YelpSearchToolInputSchema.model_fields
- assert "term" in YelpSearchToolInputSchema.model_fields
- assert "categories" in YelpSearchToolInputSchema.model_fields
- assert "price" in YelpSearchToolInputSchema.model_fields
-
-def test_yelp_search_tool_output_schema():
- assert issubclass(YelpSearchToolOutputSchema, BaseIOSchema)
- assert "results" in YelpSearchToolOutputSchema.model_fields
-
-@patch('atomic_agents.lib.tools.yelp_restaurant_finder_tool.requests.get')
-def test_run_success(mock_get, yelp_search_tool):
- mock_response = Mock()
- mock_response.status_code = 200
- mock_response.json.return_value = SAMPLE_YELP_RESPONSE
- mock_get.return_value = mock_response
-
- input_data = YelpSearchToolInputSchema(
- location="Sample City",
- term="pizza",
- categories=[YelpCategory.ITALIAN],
- price=[PriceRange.TWO, PriceRange.THREE]
- )
- result = yelp_search_tool.run(input_data)
-
- assert isinstance(result, YelpSearchToolOutputSchema)
- assert len(result.results) == 1
- assert result.results[0].name == "Sample Restaurant"
- assert result.results[0].rating == 4.5
- assert result.results[0].review_count == 100
- assert result.results[0].address == "123 Main St, Sample City, ST 12345"
- assert result.results[0].phone == "+1 (123) 456-7890"
- assert result.results[0].categories == ["Italian", "Pizza"]
-
- mock_get.assert_called_once()
- call_args = mock_get.call_args[1]
- assert call_args['headers'] == {"Authorization": "Bearer dummy_api_key"}
- assert call_args['params']['location'] == "Sample City"
- assert call_args['params']['term'] == "pizza"
- assert call_args['params']['categories'] == "italian"
- assert call_args['params']['price'] == "2,3"
-
-@patch('atomic_agents.lib.tools.yelp_restaurant_finder_tool.requests.get')
-def test_run_api_error(mock_get, yelp_search_tool):
- mock_response = Mock()
- mock_response.status_code = 400
- mock_response.reason = "Bad Request"
- mock_get.return_value = mock_response
-
- input_data = YelpSearchToolInputSchema(location="Sample City")
-
- with pytest.raises(Exception, match="Failed to fetch search results: 400 Bad Request"):
- yelp_search_tool.run(input_data)
-
-@patch('atomic_agents.lib.tools.yelp_restaurant_finder_tool.requests.get')
-def test_run_with_all_parameters(mock_get, yelp_search_tool):
- mock_response = Mock()
- mock_response.status_code = 200
- mock_response.json.return_value = SAMPLE_YELP_RESPONSE
- mock_get.return_value = mock_response
-
- input_data = YelpSearchToolInputSchema(
- location="Sample City",
- term="pizza",
- categories=[YelpCategory.ITALIAN, YelpCategory.PIZZA],
- price=[PriceRange.TWO, PriceRange.THREE],
- open_now=True,
- sort_by="rating"
- )
- yelp_search_tool.run(input_data)
-
- mock_get.assert_called_once()
- call_args = mock_get.call_args[1]
- assert call_args['params']['location'] == "Sample City"
- assert call_args['params']['term'] == "pizza"
- assert call_args['params']['categories'] == "italian,pizza"
- assert call_args['params']['price'] == "2,3"
- assert call_args['params']['open_now'] is True
- assert call_args['params']['sort_by'] == "rating"
-
-@patch('atomic_agents.lib.tools.yelp_restaurant_finder_tool.requests.get')
-def test_run_no_results(mock_get, yelp_search_tool):
- mock_response = Mock()
- mock_response.status_code = 200
- mock_response.json.return_value = {"businesses": []}
- mock_get.return_value = mock_response
-
- input_data = YelpSearchToolInputSchema(location="Sample City")
- result = yelp_search_tool.run(input_data)
-
- assert isinstance(result, YelpSearchToolOutputSchema)
- assert len(result.results) == 0
-
-def test_run_missing_api_key():
- config = YelpSearchToolConfig(api_key="", max_results=10)
- tool = YelpSearchTool(config)
- input_data = YelpSearchToolInputSchema(location="Sample City")
-
- with pytest.raises(ValueError, match="API key is required to use the Yelp API."):
- tool.run(input_data)
\ No newline at end of file
diff --git a/atomic-agents/tests/lib/tools/test_yt_transcript_scraper_tool.py b/atomic-agents/tests/lib/tools/test_yt_transcript_scraper_tool.py
deleted file mode 100644
index 321a0df..0000000
--- a/atomic-agents/tests/lib/tools/test_yt_transcript_scraper_tool.py
+++ /dev/null
@@ -1,179 +0,0 @@
-import pytest
-from unittest.mock import patch, Mock
-from pydantic import ValidationError
-from youtube_transcript_api import YouTubeTranscriptApi, NoTranscriptFound, TranscriptsDisabled
-from atomic_agents.lib.tools.yt_transcript_scraper_tool import YouTubeTranscriptToolInputSchema
-from atomic_agents.lib.tools.yt_transcript_scraper_tool import (
- YouTubeTranscriptTool,
- YouTubeTranscriptToolConfig,
- YouTubeTranscriptToolInputSchema,
- YouTubeTranscriptToolOutputSchema,
-)
-from atomic_agents.lib.tools.base_tool import BaseTool
-from atomic_agents.agents.base_agent import BaseIOSchema
-from youtube_transcript_api import YouTubeTranscriptApi
-
-
-# Sample data
-SAMPLE_TRANSCRIPT = [
- {"text": "This is the first part of the transcript.", "start": 0.0, "duration": 5.0},
- {"text": "This is the second part.", "start": 5.0, "duration": 3.0},
-]
-
-SAMPLE_VIDEO_INFO = {
- "id": "dQw4w9WgXcQ",
- "title": "Sample Video",
- "channel": "Sample Channel",
- "published_at": "2023-01-01T00:00:00Z",
-}
-
-def youtube_transcript_tool():
- config = YouTubeTranscriptToolConfig(api_key="dummy_api_key")
- return YouTubeTranscriptTool(config)
-
-def test_youtube_transcript_tool_initialization(youtube_transcript_tool):
- assert isinstance(youtube_transcript_tool, BaseTool)
- assert youtube_transcript_tool.api_key == "dummy_api_key"
-
-def test_youtube_transcript_tool_input_schema():
- assert issubclass(YouTubeTranscriptToolInputSchema, BaseIOSchema)
- assert "video_url" in YouTubeTranscriptToolInputSchema.model_fields
- assert "language" in YouTubeTranscriptToolInputSchema.model_fields
-
-def test_youtube_transcript_tool_output_schema():
- assert issubclass(YouTubeTranscriptToolOutputSchema, BaseIOSchema)
- assert "transcript" in YouTubeTranscriptToolOutputSchema.model_fields
- assert "duration" in YouTubeTranscriptToolOutputSchema.model_fields
- assert "comments" in YouTubeTranscriptToolOutputSchema.model_fields
- assert "metadata" in YouTubeTranscriptToolOutputSchema.model_fields
-
-def test_extract_video_id():
- url = "https://www.youtube.com/watch?v=dQw4w9WgXcQ"
- video_id = YouTubeTranscriptTool.extract_video_id(url)
- assert video_id == "dQw4w9WgXcQ"
-
-@patch('atomic_agents.lib.tools.yt_transcript_scraper_tool.YouTubeTranscriptTool.extract_video_id')
-@patch('atomic_agents.lib.tools.yt_transcript_scraper_tool.YouTubeTranscriptTool.fetch_video_metadata')
-@patch('atomic_agents.lib.tools.yt_transcript_scraper_tool.YouTubeTranscriptApi.get_transcript')
-def test_run(mock_get_transcript, mock_fetch_metadata, mock_extract_video_id, youtube_transcript_tool):
- mock_extract_video_id.return_value = "dQw4w9WgXcQ"
- mock_fetch_metadata.return_value = SAMPLE_VIDEO_INFO
- mock_get_transcript.return_value = SAMPLE_TRANSCRIPT
-
- input_data = YouTubeTranscriptToolInputSchema(video_url="https://www.youtube.com/watch?v=dQw4w9WgXcQ")
- result = youtube_transcript_tool.run(input_data)
-
- assert isinstance(result, YouTubeTranscriptToolOutputSchema)
- assert result.transcript == "This is the first part of the transcript. This is the second part."
- assert result.duration == 8.0
- assert result.comments == []
- assert result.metadata == SAMPLE_VIDEO_INFO
-
-@patch('atomic_agents.lib.tools.yt_transcript_scraper_tool.YouTubeTranscriptTool.extract_video_id')
-@patch('atomic_agents.lib.tools.yt_transcript_scraper_tool.YouTubeTranscriptTool.fetch_video_metadata')
-@patch('atomic_agents.lib.tools.yt_transcript_scraper_tool.YouTubeTranscriptApi.get_transcript')
-def test_run_with_language(mock_get_transcript, mock_fetch_metadata, mock_extract_video_id, youtube_transcript_tool):
- mock_extract_video_id.return_value = "dQw4w9WgXcQ"
- mock_fetch_metadata.return_value = SAMPLE_VIDEO_INFO
- mock_get_transcript.return_value = SAMPLE_TRANSCRIPT
-
- input_data = YouTubeTranscriptToolInputSchema(
- video_url="https://www.youtube.com/watch?v=dQw4w9WgXcQ",
- language="en"
- )
- youtube_transcript_tool.run(input_data)
-
- mock_get_transcript.assert_called_once_with("dQw4w9WgXcQ", languages=["en"])
-
-@patch('atomic_agents.lib.tools.yt_transcript_scraper_tool.YouTubeTranscriptTool.extract_video_id')
-@patch('atomic_agents.lib.tools.yt_transcript_scraper_tool.YouTubeTranscriptTool.fetch_video_metadata')
-@patch('atomic_agents.lib.tools.yt_transcript_scraper_tool.YouTubeTranscriptApi.get_transcript')
-def test_run_transcript_not_found(mock_get_transcript, mock_fetch_metadata, mock_extract_video_id, youtube_transcript_tool):
- mock_extract_video_id.return_value = "dQw4w9WgXcQ"
- mock_fetch_metadata.return_value = SAMPLE_VIDEO_INFO
- mock_get_transcript.side_effect = Exception("No transcript found")
-
- input_data = YouTubeTranscriptToolInputSchema(video_url="https://www.youtube.com/watch?v=dQw4w9WgXcQ")
-
- with pytest.raises(Exception, match="No transcript found"):
- youtube_transcript_tool.run(input_data)
-
-def test_youtube_transcript_tool_config_validation():
- with pytest.raises(ValidationError):
- YouTubeTranscriptToolConfig(api_key=None) # api_key is required and should not be None
-
- config = YouTubeTranscriptToolConfig(api_key="dummy_api_key")
- assert config.api_key == "dummy_api_key"
-
[email protected](YouTubeTranscriptApi, 'get_transcript')
-@patch('atomic_agents.lib.tools.yt_transcript_scraper_tool.YouTubeTranscriptTool.fetch_video_metadata')
-@patch('atomic_agents.lib.tools.yt_transcript_scraper_tool.YouTubeTranscriptTool.extract_video_id')
-def test_run_no_transcript_found(mock_extract_video_id, mock_fetch_metadata, mock_get_transcript, youtube_transcript_tool):
- mock_extract_video_id.return_value = "dQw4w9WgXcQ"
- mock_fetch_metadata.return_value = SAMPLE_VIDEO_INFO
- mock_get_transcript.side_effect = NoTranscriptFound("dQw4w9WgXcQ", "en", "Transcript not found")
-
- input_data = YouTubeTranscriptToolInputSchema(video_url="https://www.youtube.com/watch?v=dQw4w9WgXcQ")
-
- with pytest.raises(Exception, match="Failed to fetch transcript for video 'dQw4w9WgXcQ': "):
- youtube_transcript_tool.run(input_data)
-
[email protected](YouTubeTranscriptApi, 'get_transcript')
-@patch('atomic_agents.lib.tools.yt_transcript_scraper_tool.YouTubeTranscriptTool.fetch_video_metadata')
-@patch('atomic_agents.lib.tools.yt_transcript_scraper_tool.YouTubeTranscriptTool.extract_video_id')
-def test_run_transcripts_disabled(mock_extract_video_id, mock_fetch_metadata, mock_get_transcript, youtube_transcript_tool):
- mock_extract_video_id.return_value = "dQw4w9WgXcQ"
- mock_fetch_metadata.return_value = SAMPLE_VIDEO_INFO
- mock_get_transcript.side_effect = TranscriptsDisabled("Transcripts are disabled for this video")
-
- input_data = YouTubeTranscriptToolInputSchema(video_url="https://www.youtube.com/watch?v=dQw4w9WgXcQ")
-
- with pytest.raises(Exception, match="Failed to fetch transcript for video 'dQw4w9WgXcQ': "):
- youtube_transcript_tool.run(input_data)
-
-@patch('atomic_agents.lib.tools.yt_transcript_scraper_tool.build')
-def test_fetch_video_metadata_success(mock_build, youtube_transcript_tool):
- mock_youtube = Mock()
- mock_build.return_value = mock_youtube
- mock_request = Mock()
- mock_youtube.videos().list.return_value = mock_request
- mock_request.execute.return_value = {
- "items": [
- {
- "snippet": {
- "title": "Sample Video",
- "channelTitle": "Sample Channel",
- "publishedAt": "2023-01-01T00:00:00Z",
- }
- }
- ]
- }
-
- video_id = "dQw4w9WgXcQ"
- metadata = youtube_transcript_tool.fetch_video_metadata(video_id)
-
- assert metadata == {
- "id": video_id,
- "title": "Sample Video",
- "channel": "Sample Channel",
- "published_at": "2023-01-01T00:00:00Z",
- }
- mock_build.assert_called_once_with("youtube", "v3", developerKey=youtube_transcript_tool.api_key)
- mock_youtube.videos().list.assert_called_once_with(part="snippet", id=video_id)
-
-@patch('atomic_agents.lib.tools.yt_transcript_scraper_tool.build')
-def test_fetch_video_metadata_no_items(mock_build, youtube_transcript_tool):
- mock_youtube = Mock()
- mock_build.return_value = mock_youtube
- mock_request = Mock()
- mock_youtube.videos().list.return_value = mock_request
- mock_request.execute.return_value = {"items": []}
-
- video_id = "dQw4w9WgXcQ"
-
- with pytest.raises(Exception, match=f"No metadata found for video '{video_id}'"):
- youtube_transcript_tool.fetch_video_metadata(video_id)
-
- mock_build.assert_called_once_with("youtube", "v3", developerKey=youtube_transcript_tool.api_key)
- mock_youtube.videos().list.assert_called_once_with(part="snippet", id=video_id)
diff --git a/atomic-agents/tests/lib/utils/scraping/test_pdf_to_markdown.py b/atomic-agents/tests/lib/utils/scraping/test_pdf_to_markdown.py
deleted file mode 100644
index 68925cc..0000000
--- a/atomic-agents/tests/lib/utils/scraping/test_pdf_to_markdown.py
+++ /dev/null
@@ -1,128 +0,0 @@
-import pytest
-import requests
-from unittest.mock import patch, Mock, mock_open
-from io import BytesIO
-from atomic_agents.lib.utils.scraping.pdf_to_markdown import PdfToMarkdownConverter
-from atomic_agents.lib.models.web_document import WebDocument, WebDocumentMetadata
-from PyPDF2.errors import PdfReadError
-
-# Sample PDF content (mock)
-SAMPLE_PDF_CONTENT = b"""%PDF-1.3
-1 0 obj
-<< /Type /Catalog /Pages 2 0 R >>
-endobj
-2 0 obj
-<< /Type /Pages /Kids [3 0 R] /Count 1 >>
-endobj
-3 0 obj
-<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Contents 4 0 R >>
-endobj
-4 0 obj
-<< /Length 51 >>
-stream
-BT /F1 12 Tf 72 712 Td (Sample PDF content) Tj ET
-endstream
-endobj
-xref
-0 5
-0000000000 65535 f
-0000000009 00000 n
-0000000058 00000 n
-0000000115 00000 n
-0000000198 00000 n
-trailer << /Size 5 /Root 1 0 R >>
-startxref
-298
-%%EOF"""
-
-
-def mock_requests_get():
- with patch('requests.get') as mock_get:
- response = Mock()
- response.content = SAMPLE_PDF_CONTENT
- response.raise_for_status.return_value = None
- mock_get.return_value = response
- yield mock_get
-
-def mock_pdf_reader():
- with patch('PyPDF2.PdfReader') as MockPdfReader:
- mock_page = Mock()
- mock_page.extract_text.return_value = "Sample PDF content"
- mock_reader = Mock()
- mock_reader.pages = [mock_page]
- MockPdfReader.return_value = mock_reader
- yield MockPdfReader
-
-def test_fetch_pdf_content(mock_requests_get):
- url = "https://example.com/sample.pdf"
- content = PdfToMarkdownConverter._fetch_pdf_content(url)
-
- mock_requests_get.assert_called_once_with(url)
- assert isinstance(content, BytesIO)
- assert content.getvalue() == SAMPLE_PDF_CONTENT
-
-def test_fetch_pdf_content_error(mock_requests_get):
- mock_requests_get.return_value.raise_for_status.side_effect = requests.HTTPError("404 Not Found")
-
- with pytest.raises(requests.HTTPError):
- PdfToMarkdownConverter._fetch_pdf_content("https://example.com/nonexistent.pdf")
-
-def test_read_pdf():
- file = BytesIO(SAMPLE_PDF_CONTENT)
-
- mock_page = Mock()
- mock_page.extract_text.return_value = "Sample PDF content"
-
- mock_reader = Mock()
- mock_reader.pages = [mock_page]
-
- with patch('atomic_agents.lib.utils.scraping.pdf_to_markdown.PdfReader', return_value=mock_reader) as mock_pdf_reader:
- content = PdfToMarkdownConverter._read_pdf(file)
-
- mock_pdf_reader.assert_called_once_with(file)
- assert content == "Sample PDF content"
-
-
-def test_convert_to_markdown():
- html_content = "<h1>Title</h1><p>This is a paragraph.</p><p>Another paragraph.</p>"
- markdown = PdfToMarkdownConverter._convert_to_markdown(html_content)
-
- expected_markdown = "# Title\n\nThis is a paragraph.\n\nAnother paragraph."
- assert markdown.strip() == expected_markdown.strip()
-
-@patch('atomic_agents.lib.utils.scraping.pdf_to_markdown.PdfToMarkdownConverter._fetch_pdf_content')
-@patch('atomic_agents.lib.utils.scraping.pdf_to_markdown.PdfToMarkdownConverter._read_pdf')
-@patch('atomic_agents.lib.utils.scraping.pdf_to_markdown.PdfToMarkdownConverter._convert_to_markdown')
-def test_convert_from_url(mock_convert, mock_read, mock_fetch):
- url = "https://example.com/sample.pdf"
- mock_fetch.return_value = BytesIO(SAMPLE_PDF_CONTENT)
- mock_read.return_value = "Sample PDF content"
- mock_convert.return_value = "# Sample PDF content"
-
- result = PdfToMarkdownConverter.convert(url=url)
-
- assert isinstance(result, WebDocument)
- assert result.content == "# Sample PDF content"
- assert isinstance(result.metadata, WebDocumentMetadata)
- assert result.metadata.url == url
-
-@patch('builtins.open', new_callable=mock_open, read_data=SAMPLE_PDF_CONTENT)
-@patch('atomic_agents.lib.utils.scraping.pdf_to_markdown.PdfToMarkdownConverter._read_pdf')
-@patch('atomic_agents.lib.utils.scraping.pdf_to_markdown.PdfToMarkdownConverter._convert_to_markdown')
-def test_convert_from_file(mock_convert, mock_read, mock_file):
- file_path = "/path/to/sample.pdf"
- mock_read.return_value = "Sample PDF content"
- mock_convert.return_value = "# Sample PDF content"
-
- result = PdfToMarkdownConverter.convert(file_path=file_path)
-
- assert isinstance(result, WebDocument)
- assert result.content == "# Sample PDF content"
- assert isinstance(result.metadata, WebDocumentMetadata)
- assert result.metadata.url == file_path
-
-def test_convert_no_input():
- with pytest.raises(ValueError, match="Either url or file_path must be provided"):
- PdfToMarkdownConverter.convert()
\ No newline at end of file
diff --git a/atomic-agents/tests/lib/utils/scraping/test_url_to_markdown.py b/atomic-agents/tests/lib/utils/scraping/test_url_to_markdown.py
deleted file mode 100644
index 904bafc..0000000
--- a/atomic-agents/tests/lib/utils/scraping/test_url_to_markdown.py
+++ /dev/null
@@ -1,111 +0,0 @@
-import pytest
-from unittest.mock import patch, Mock
-from bs4 import BeautifulSoup
-from atomic_agents.lib.utils.scraping.url_to_markdown import UrlToMarkdownConverter
-from atomic_agents.lib.models.web_document import WebDocument, WebDocumentMetadata
-
-SAMPLE_HTML = """
-<!DOCTYPE html>
-<html>
-<head>
- <title>Sample Page</title>
- <meta name="description" content="This is a sample page">
- <meta name="keywords" content="sample, test, page">
- <meta name="author" content="John Doe">
-</head>
-<body>
- <h1>Welcome to the Sample Page</h1>
- <p>This is a paragraph.</p>
- <p>Another paragraph with <a href="https://example.com">a link</a>.</p>
- <img src="sample.jpg" alt="Sample Image">
- <script>console.log("This should be removed");</script>
- <style>body { color: black; }</style>
-</body>
-</html>
-"""
-
-def mock_requests_get():
- with patch('requests.get') as mock_get:
- response = Mock()
- response.text = SAMPLE_HTML
- response.raise_for_status.return_value = None
- mock_get.return_value = response
- yield mock_get
-
-def test_fetch_url_content(mock_requests_get):
- url = "https://example.com"
- content = UrlToMarkdownConverter._fetch_url_content(url)
-
- mock_requests_get.assert_called_once_with(url)
- assert content == SAMPLE_HTML
-
-def test_parse_html():
- soup = UrlToMarkdownConverter._parse_html(SAMPLE_HTML)
- assert isinstance(soup, BeautifulSoup)
- assert soup.title.string == "Sample Page"
-
-def test_extract_metadata():
- soup = BeautifulSoup(SAMPLE_HTML, "html.parser")
- url = "https://example.com"
- metadata = UrlToMarkdownConverter._extract_metadata(soup, url)
-
- assert isinstance(metadata, WebDocumentMetadata)
- assert metadata.url == url
- assert metadata.title == "" # Adjusted expectation
- assert metadata.description == "This is a sample page"
- assert metadata.keywords == "sample, test, page"
- assert metadata.author == "John Doe"
-
-def test_convert_to_markdown():
- soup = BeautifulSoup(SAMPLE_HTML, "html.parser")
- markdown = UrlToMarkdownConverter._convert_to_markdown(soup)
-
- expected_markdown = """Sample Page
-
-Welcome to the Sample Page
-==========================
-
-This is a paragraph.
-
-Another paragraph with [a link](https://example.com).
-
-"""
- assert markdown.strip() == expected_markdown.strip()
-
-@patch('atomic_agents.lib.utils.scraping.url_to_markdown.UrlToMarkdownConverter._fetch_url_content')
-@patch('atomic_agents.lib.utils.scraping.url_to_markdown.UrlToMarkdownConverter._parse_html')
-@patch('atomic_agents.lib.utils.scraping.url_to_markdown.UrlToMarkdownConverter._extract_metadata')
-@patch('atomic_agents.lib.utils.scraping.url_to_markdown.UrlToMarkdownConverter._convert_to_markdown')
-def test_convert(mock_convert, mock_extract, mock_parse, mock_fetch):
- url = "https://example.com"
- mock_fetch.return_value = SAMPLE_HTML
- mock_parse.return_value = BeautifulSoup(SAMPLE_HTML, "html.parser")
- mock_extract.return_value = WebDocumentMetadata(url=url, title="")
- mock_convert.return_value = "Sample Page\n\nThis is the content."
-
- result = UrlToMarkdownConverter.convert(url)
-
- assert isinstance(result, WebDocument)
- assert result.content == "Sample Page\n\nThis is the content."
- assert isinstance(result.metadata, WebDocumentMetadata)
- assert result.metadata.url == url
- assert result.metadata.title == ""
-
-def test_convert_integration(mock_requests_get):
- url = "https://example.com"
- result = UrlToMarkdownConverter.convert(url)
-
- assert isinstance(result, WebDocument)
- assert "Welcome to the Sample Page" in result.content
- assert "This is a paragraph." in result.content
- assert "[a link](https://example.com)" in result.content
- assert "console.log" not in result.content
- assert "body { color: black; }" not in result.content
-
- assert isinstance(result.metadata, WebDocumentMetadata)
- assert result.metadata.url == url
- assert result.metadata.title == "" # Adjusted expectation
- assert result.metadata.description == "This is a sample page"
- assert result.metadata.keywords == "sample, test, page"
- assert result.metadata.author == "John Doe"
\ No newline at end of file