Building AI agents, atomically
refactor: Remove unused scraper.py file
KennyVaneetvelde8371057dd28f
diff --git a/examples/deep_research_multi_agent/scraper.py b/examples/deep_research_multi_agent/scraper.pydeleted file mode 100644index f80d8ef..0000000--- a/examples/deep_research_multi_agent/scraper.py+++ /dev/null@@ -1,78 +0,0 @@-import os-from bs4 import BeautifulSoup-import requests-from markdownify import MarkdownConverter-from pydantic import BaseModel, Field--class WebDocumentMetadata(BaseModel):- url: str- title: str = Field(default='')- description: str = Field(default='')- keywords: str = Field(default='')- author: str = Field(default='')--class VectorDBDocumentMetadata(WebDocumentMetadata):- document_part: int = Field(default=1)- document_total_parts: int = Field(default=1)--class Document(BaseModel):- content: str- metadata: WebDocumentMetadata | VectorDBDocumentMetadata = Field(default_factory=WebDocumentMetadata)----class UrlToMarkdownConverter:- METADATA_ATTRS = {- 'url': '',- 'title': ('title', None),- 'description': ('meta', {'name': 'description'}),- 'keywords': ('meta', {'name': 'keywords'}),- 'author': ('meta', {'name': 'author'})- }-- @staticmethod- def _fetch_url_content(url):- response = requests.get(url)- response.raise_for_status()- return response.text-- @staticmethod- def _parse_html(html):- return BeautifulSoup(html, 'html.parser')-- @staticmethod- def _extract_metadata(soup, url):- metadata_dict = {}- for attr, selector in UrlToMarkdownConverter.METADATA_ATTRS.items():- if selector:- element = soup.find(*selector)- metadata_dict[attr] = element.get('content', '').strip() if element else ''- else:- metadata_dict[attr] = url- return WebDocumentMetadata(**metadata_dict)-- @staticmethod- def _convert_to_markdown(html):- class CustomMarkdownConverter(MarkdownConverter):- def convert_img(self, el, text, convert_as_inline):- return ''- def convert_p(self, el, text, convert_as_inline):- return text + '\n\n'-- for script in html.find_all('script'):- script.decompose()-- for style in html.find_all('style'):- style.decompose()-- converter = CustomMarkdownConverter()- return converter.convert(str(html)).strip()-- @staticmethod- def convert(url):- html = UrlToMarkdownConverter._fetch_url_content(url)- soup = UrlToMarkdownConverter._parse_html(html)- metadata = UrlToMarkdownConverter._extract_metadata(soup, url)- markdown_content = UrlToMarkdownConverter._convert_to_markdown(soup)-- return Document(content=markdown_content, metadata=metadata)\ No newline at end of file