🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org
Python agentai-societiesartificial-intelligencecommunicative-aicooperative-aideep-learninglarge-language-modelsmulti-agent-systemsnatural-language-processing
feat: integrate crawl4ai (#2030)
Abhay Kashyapbbd63fbd3a4c
diff --git a/camel/loaders/__init__.py b/camel/loaders/__init__.pyindex 7dc4e0d53..83963c33a 100644--- a/camel/loaders/__init__.py+++ b/camel/loaders/__init__.py@@ -15,6 +15,7 @@from .apify_reader import Apifyfrom .base_io import File, create_file, create_file_from_raw_bytesfrom .chunkr_reader import ChunkrReader+from .crawl4ai_reader import Crawl4AIfrom .firecrawl_reader import Firecrawlfrom .jina_url_reader import JinaURLReaderfrom .mineru_extractor import MinerU@@ -32,4 +33,5 @@ __all__ = ['ChunkrReader','PandasReader','MinerU',+ 'Crawl4AI',]diff --git a/camel/loaders/crawl4ai_reader.py b/camel/loaders/crawl4ai_reader.pynew file mode 100644index 000000000..0f3be56ee--- /dev/null+++ b/camel/loaders/crawl4ai_reader.py@@ -0,0 +1,243 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========++import asyncio+import logging+from typing import Any, Dict, List, Optional++from dotenv import load_dotenv+from pydantic import BaseModel, ValidationError++from camel.utils import api_keys_required++logger = logging.getLogger(__name__)+load_dotenv()+++class Crawl4AI:+ r"""Class for converting websites into LLM-ready data.++ This class uses asynchronous crawling with CSS selectors or LLM-based+ extraction to convert entire websites into structured data.++ Args:+ None: No parameters are required for initialization.++ References:+ https://docs.crawl4ai.com/+ """++ def __init__(self) -> None:+ from crawl4ai import AsyncWebCrawler++ self.crawler = AsyncWebCrawler++ async def _run_crawler(self, url: str, **kwargs) -> Any:+ r"""Run the asynchronous web crawler on a given URL.++ Args:+ url (str): URL to crawl or scrape.+ **kwargs: Additional keyword arguments for crawler configuration.++ Returns:+ Any: The result from the crawler.++ Raises:+ RuntimeError: If crawler execution fails.+ """++ try:+ async with self.crawler() as c:+ return await c.arun(url, **kwargs)+ except Exception as e:+ logger.error("Crawler run failed: %s", e)+ raise RuntimeError(f"Crawler run failed: {e}") from e++ async def crawl(+ self,+ start_url: str,+ max_depth: int = 1,+ extraction_strategy=None,+ **kwargs,+ ) -> List[Dict[str, Any]]:+ r"""Crawl a URL and its subpages using breadth-first search.++ Args:+ start_url (str): URL to start crawling from.+ max_depth (int, optional): Maximum depth of links to follow+ (default: 1).+ extraction_strategy (ExtractionStrategy, optional): Strategy+ for data extraction.+ **kwargs: Additional arguments for crawler configuration.++ Returns:+ List[Dict[str, Any]]: List of crawled page results.++ Raises:+ RuntimeError: If an error occurs during crawling.+ """++ all_results: List[Dict[str, Any]] = []+ visited_urls: set[str] = set()+ queue: asyncio.Queue[tuple[str, int]] = asyncio.Queue()++ await queue.put((start_url, 1))+ visited_urls.add(start_url)++ while not queue.empty():+ url, depth = await queue.get()+ try:+ result = await self._run_crawler(+ url, extraction_strategy=extraction_strategy, **kwargs+ )+ all_results.append(+ {+ "url": url,+ "raw_result": result,+ "markdown": result.markdown,+ "cleaned_html": result.cleaned_html,+ "links": result.links,+ }+ )++ if depth < max_depth and result.links:+ for _, links in result.links.items():+ for link in links:+ if link['href'] not in visited_urls:+ visited_urls.add(link['href'])+ await queue.put((link['href'], depth + 1))++ except Exception as e:+ logger.error("Error crawling %s: %s", url, e)+ raise RuntimeError(f"Error crawling {url}: {e}") from e++ queue.task_done()++ await queue.join()++ return all_results++ async def scrape(+ self,+ url: str,+ extraction_strategy=None,+ **kwargs,+ ) -> Dict[str, Any]:+ r"""Scrape a single URL using CSS or LLM-based extraction.++ Args:+ url (str): URL to scrape.+ extraction_strategy (ExtractionStrategy, optional): Extraction+ strategy to use.+ **kwargs: Additional arguments for crawler configuration.++ Returns:+ Dict[str, Any]: Dictionary containing scraped data such as markdown+ and HTML content.++ Raises:+ RuntimeError: If scraping fails.+ """++ result = await self._run_crawler(+ url, extraction_strategy=extraction_strategy, **kwargs+ )+ return {+ "url": url,+ "raw_result": result,+ "markdown": result.markdown,+ "cleaned_html": result.cleaned_html,+ "links": result.links,+ }++ @api_keys_required(+ [+ # ("api_key", "ANTHROPIC_API_KEY"),+ # ("api_key", "DEEPSEEK_API_KEY"),+ # ("api_key", "GEMINI_API_KEY"),+ # ("api_key", "GROQ_API_KEY"),+ ("api_key", "OPENAI_API_KEY"),+ ]+ )+ async def structured_scrape(+ self,+ url: str,+ response_format: BaseModel,+ api_key: Optional[str] = None,+ llm_provider: str = 'ollama/llama3',+ **kwargs,+ ) -> Any:+ r"""Extract structured data from a URL using an LLM.++ Args:+ url (str): URL to scrape.+ response_format (BaseModel): Model defining the expected output+ schema.+ api_key (str, optional): API key for the LLM provider+ (default: "no-token").+ llm_provider (str, optional): Identifier for the LLM provider+ (default: 'ollama/llama3').+ **kwargs: Additional arguments for crawler configuration.++ Returns:+ Any: Crawl result containing the extracted data+ structured according to the schema.++ Raises:+ ValidationError: If extracted data does not match the schema.+ RuntimeError: If extraction fails.+ """++ from crawl4ai.extraction_strategy import (+ LLMExtractionStrategy,+ )++ extraction_strategy = LLMExtractionStrategy(+ provider=llm_provider,+ api_token=api_key,+ schema=response_format.model_json_schema(),+ extraction_type="schema",+ instruction="Extract the data according to the schema.",+ )++ try:+ return await self._run_crawler(+ url, extraction_strategy=extraction_strategy, **kwargs+ )+ except ValidationError as e:+ raise ValidationError(+ f"Extracted data does not match schema: {e}"+ ) from e+ except Exception as e:+ raise RuntimeError(e) from e++ async def map_site(self, url: str, **kwargs) -> List[str]:+ r"""Map a website by extracting all accessible URLs.++ Args:+ url (str): Starting URL to map.+ **kwargs: Additional configuration arguments.++ Returns:+ List[str]: List of URLs discovered on the website.++ Raises:+ RuntimeError: If mapping fails.+ """++ try:+ result = await self.crawl(url, **kwargs)+ return [page["url"] for page in result]+ except Exception as e:+ raise RuntimeError(f"Failed to map url: {e}") from ediff --git a/examples/loaders/crawl4ai_example.py b/examples/loaders/crawl4ai_example.pynew file mode 100644index 000000000..590c70aa8--- /dev/null+++ b/examples/loaders/crawl4ai_example.py@@ -0,0 +1,105 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========++import asyncio++from pydantic import BaseModel, Field++from camel.loaders import Crawl4AI++# Instantiate the Crawl4AI crawler+crawler = Crawl4AI()++URL = "https://toscrape.com/"++# --- Crawling (BFS) ---+crawl_results = asyncio.run(crawler.crawl(URL, max_depth=1))+print(crawl_results[0]["markdown"])+'''+++# Web Scraping Sandbox++## Books++A [fictional bookstore](http://books.toscrape.com) that desperately wants to+be scraped. It's a safe place for beginners learning web scraping and for+developers validating their scraping technologies as well.+Available at: [books.toscrape.com](http://books.toscrape.com)++[](http://books.toscrape.com)++Details+---+Amount of items | 1000+Pagination | ✔+Items per page | max 20+Requires JavaScript | ✘++## Quotes++[A website](http://quotes.toscrape.com/) that lists quotes from famous people.+It has many endpoints showing the quotes in many different ways, each of them+including new scraping challenges for you, as described below.++[](http://quotes.toscrape.com)++Endpoints+---+[Default](http://quotes.toscrape.com/)| Microdata and pagination+[Scroll](http://quotes.toscrape.com/scroll) | infinite scrolling pagination+[JavaScript](http://quotes.toscrape.com/js) | JavaScript generated content+[Delayed](http://quotes.toscrape.com/js-delayed) | Same as JavaScript but with+ a delay (?delay=10000)+[Tableful](http://quotes.toscrape.com/tableful) | a table based messed-up+ layout+[Login](http://quotes.toscrape.com/login) | login with CSRF token+ (any user/passwd works)+[ViewState](http://quotes.toscrape.com/search.aspx) | an AJAX based filter+ form with ViewStates+[Random](http://quotes.toscrape.com/random) | a single random quote+'''+++# --- Scraping ---+scrape_result = asyncio.run(crawler.scrape(URL))+print(scrape_result)+'''+{url: 'https://toscrape.com/',+'raw_result': CrawlResult(url='https://toscrape.com/', markdown=...,+ cleaned_html=..., links=...),+'markdown': "\n\n# Web Scraping Sandbox\n\n## Books...",+'cleaned_html': '\n<div>\n<div>\n<div>\n<img class="logo" height="108"+ src="img/zyte.png" width="200"/>\n+ <h1>Web Scraping Sandbox</h1>\n...'}+'''+++# --- Structured Scraping ---+class MovieResponse(BaseModel):+ title: str = Field(..., description="The name of the website.")+ year: int = Field(..., description="The amount of items on the bookstore.")+ rating: float = Field(+ ..., description="The names of the endpoints of quotes."+ )+++structured_scrape_result = asyncio.run(+ crawler.structured_scrape(+ URL,+ response_format=MovieResponse,+ llm_provider="openai/gpt-4o",+ )+)+print(structured_scrape_result.extracted_content)diff --git a/pyproject.toml b/pyproject.tomlindex bbcbe2a8d..706db4134 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -30,6 +30,7 @@ dependencies = ["pydantic>=2.10.6","httpx>=0.28.0,<1.0.0dev","psutil>=5.9.8,<6",+ "crawl4ai>=0.3.745",][project.optional-dependencies]diff --git a/test/loaders/test_crawl4ai.py b/test/loaders/test_crawl4ai.pynew file mode 100644index 000000000..e6eba4a4a--- /dev/null+++ b/test/loaders/test_crawl4ai.py@@ -0,0 +1,175 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========++from typing import Optional+from unittest.mock import AsyncMock, patch++import pytest+from pydantic import BaseModel++from camel.loaders import Crawl4AI+++# Dummy CrawlResult to simulate the results from AsyncWebCrawler.arun+class DummyCrawlResult:+ def __init__(+ self, markdown: str, cleaned_html: str, links: Optional[dict] = None+ ):+ self.markdown = markdown+ self.cleaned_html = cleaned_html+ self.links = links or {}+++URL = "https://example.com"+++# ------------------------+# Test Cases for Crawl4AI+# ------------------------+++def test_init():+ crawler = Crawl4AI()+ # Verify that the crawler attribute is set to AsyncWebCrawler+ assert crawler.crawler is not None+++async def test_crawl_success():+ dummy_result = DummyCrawlResult(+ markdown="Test markdown", cleaned_html="<html></html>", links={}+ )++ with patch.object(+ Crawl4AI, "_run_crawler", new_callable=AsyncMock+ ) as mock_run_crawler:+ mock_run_crawler.return_value = dummy_result+ crawler = Crawl4AI()+ results = await crawler.crawl(URL, max_depth=1)+ assert isinstance(results, list)+ page = results[0]+ assert page["url"] == URL+ assert page["markdown"] == "Test markdown"+ assert page["cleaned_html"] == "<html></html>"+ assert page["links"] == {}+++async def test_crawl_failure_in_run():+ with patch.object(+ Crawl4AI, "_run_crawler", new_callable=AsyncMock+ ) as mock_run_crawler:+ mock_run_crawler.side_effect = Exception(+ f"Error crawling {URL}: Crawl error"+ )+ crawler = Crawl4AI()+ with pytest.raises(+ RuntimeError, match=f"Error crawling {URL}: Crawl error"+ ):+ await crawler.crawl(URL)+++async def test_scrape_success():+ dummy_result = DummyCrawlResult(+ markdown="Scraped markdown",+ cleaned_html="<html>scraped</html>",+ links={},+ )++ with patch.object(+ Crawl4AI, "_run_crawler", new_callable=AsyncMock+ ) as mock_run_crawler:+ mock_run_crawler.return_value = dummy_result+ crawler = Crawl4AI()+ result = await crawler.scrape(URL)+ assert result["url"] == URL+ assert result["markdown"] == "Scraped markdown"+ assert result["cleaned_html"] == "<html>scraped</html>"+ assert result["links"] == {}+++async def test_structured_scrape_success():+ dummy_result = DummyCrawlResult(+ markdown="Scraped markdown",+ cleaned_html="<html>scraped</html>",+ links={},+ )++ class DummySchema(BaseModel):+ field: str++ with patch.object(+ Crawl4AI, "_run_crawler", new_callable=AsyncMock+ ) as mock_run_crawler:+ mock_run_crawler.return_value = dummy_result+ crawler = Crawl4AI()+ result = await crawler.structured_scrape(+ URL, DummySchema, llm_provider="openai"+ )+ assert result.markdown == "Scraped markdown"+ assert result.cleaned_html == "<html>scraped</html>"+ assert result.links == {}+++async def test_structured_scrape_failure():+ # Simulate a failure in the structured scrape method.+ with patch.object(+ Crawl4AI, "_run_crawler", new_callable=AsyncMock+ ) as mock_run_crawler:+ mock_run_crawler.side_effect = RuntimeError("Structured scrape failed")+ crawler = Crawl4AI()++ class DummySchema(BaseModel):+ field: str++ with pytest.raises(RuntimeError, match="Structured scrape failed"):+ await crawler.structured_scrape(URL, DummySchema)+++async def test_map_site_success():+ # Simulate the crawl method to return a list with a dummy page+ dummy_page = {+ "url": "https://example.com/page1",+ "markdown": "md",+ "cleaned_html": "html",+ }++ # Instead of patching AsyncWebCrawler here, patch the crawl method itself+ async def dummy_crawl(url: str, **kwargs):+ return [dummy_page]++ crawler = Crawl4AI()+ crawler.crawl = dummy_crawl # Override crawl method for testing++ urls = await crawler.map_site(URL)+ assert urls == ["https://example.com/page1"]+++async def test_map_site_failure():+ # Simulate crawl raising an exception in map_site+ async def dummy_crawl_fail(url: str, **kwargs):+ raise Exception("Mapping error")++ crawler = Crawl4AI()+ crawler.crawl = (+ dummy_crawl_fail # Override crawl method to simulate failure+ )++ with pytest.raises(RuntimeError, match="Failed to map url"):+ await crawler.map_site(URL)diff --git a/uv.lock b/uv.lockindex c0fc6daf6..8f8eb6ed4 100644--- a/uv.lock+++ b/uv.lock@@ -64,11 +64,11 @@ wheels = [[[package]]name = "aiofiles"-version = "23.2.1"+version = "24.1.0"source = { registry = "https://pypi.org/simple" }-sdist = { url = "https://files.pythonhosted.org/packages/af/41/cfed10bc64d774f497a86e5ede9248e1d062db675504b41c320954d99641/aiofiles-23.2.1.tar.gz", hash = "sha256:84ec2218d8419404abcb9f0c02df3f34c6e0a68ed41072acfb1cef5cbc29051a", size = 32072 }+sdist = { url = "https://files.pythonhosted.org/packages/0b/03/a88171e277e8caa88a4c77808c20ebb04ba74cc4681bf1e9416c862de237/aiofiles-24.1.0.tar.gz", hash = "sha256:22a075c9e5a3810f0c2e48f3008c94d68c65d763b9b03857924c99e57355166c", size = 30247 }wheels = [- { url = "https://files.pythonhosted.org/packages/c5/19/5af6804c4cc0fed83f47bff6e413a98a36618e7d40185cd36e69737f3b0e/aiofiles-23.2.1-py3-none-any.whl", hash = "sha256:19297512c647d4b27a2cf7c34caa7e405c0d60b5560618a29a9fe027b18b0107", size = 15727 },+ { url = "https://files.pythonhosted.org/packages/a5/45/30bb92d442636f570cb5651bc661f52b610e2eec3f891a5dc3a4c3667db0/aiofiles-24.1.0-py3-none-any.whl", hash = "sha256:b4ec55f4195e3eb5d7abd1bf7e061763e864dd4954231fb8539a0ef8bb8260e5", size = 15896 },][[package]]@@ -547,6 +547,7 @@ version = "0.2.43"source = { editable = "." }dependencies = [{ name = "colorama" },+ { name = "crawl4ai" },{ name = "docstring-parser" },{ name = "httpx" },{ name = "jsonschema" },@@ -893,6 +894,7 @@ requires-dist = [{ name = "cohere", marker = "extra == 'model-platforms'", specifier = ">=5.11.0,<6" },{ name = "cohere", marker = "extra == 'rag'", specifier = ">=5.11.0,<6" },{ name = "colorama", specifier = ">=0.4.6,<0.5" },+ { name = "crawl4ai", specifier = ">=0.3.745" },{ name = "dappier", marker = "extra == 'all'", specifier = ">=0.3.3,<0.4" },{ name = "dappier", marker = "extra == 'web-tools'", specifier = ">=0.3.3,<0.4" },{ name = "datacommons", marker = "extra == 'all'", specifier = ">=1.4.3,<2" },@@ -1437,6 +1439,33 @@ toml = [{ name = "tomli", marker = "python_full_version <= '3.11'" },]+[[package]]+name = "crawl4ai"+version = "0.3.745"+source = { registry = "https://pypi.org/simple" }+dependencies = [+ { name = "aiofiles" },+ { name = "aiosqlite" },+ { name = "beautifulsoup4" },+ { name = "colorama" },+ { name = "html2text" },+ { name = "litellm" },+ { name = "lxml" },+ { name = "numpy" },+ { name = "pillow" },+ { name = "playwright" },+ { name = "python-dotenv" },+ { name = "rank-bm25" },+ { name = "requests" },+ { name = "snowballstemmer" },+ { name = "tf-playwright-stealth" },+ { name = "xxhash" },+]+sdist = { url = "https://files.pythonhosted.org/packages/02/5a/919e64ff2977d7aa1b2cda4d45f16ff8996cd2c2dc1f55936fb6cd214222/crawl4ai-0.3.745.tar.gz", hash = "sha256:990396d57e10ae7ccabf35c34a317dbd8c59a3ceca475eac75320a8808334438", size = 117277 }+wheels = [+ { url = "https://files.pythonhosted.org/packages/ed/7e/ebe351a457140330b20b6d8289b8f243b21de6e6bce505cd15b230a83bcb/Crawl4AI-0.3.745-py3-none-any.whl", hash = "sha256:763e6aba80959e60e1fe70cb9d954a4cf257eb230af30f51fcd99ff641a7a88d", size = 121997 },+]+[[package]]name = "cryptography"version = "44.0.2"@@ -1915,6 +1944,14 @@ wheels = [{ url = "https://files.pythonhosted.org/packages/7b/8f/c4d9bafc34ad7ad5d8dc16dd1347ee0e507a52c3adb6bfa8887e1c6a26ba/executing-2.2.0-py2.py3-none-any.whl", hash = "sha256:11387150cad388d62750327a53d3339fad4888b39a6fe233c3afbb54ecffd3aa", size = 26702 },]+[[package]]+name = "fake-http-header"+version = "0.3.5"+source = { registry = "https://pypi.org/simple" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/e3/0b/2849c87d9f13766e29c0a2f4d31681aa72e035016b251ab19d99bde7b592/fake_http_header-0.3.5-py3-none-any.whl", hash = "sha256:cd05f4bebf1b7e38b5f5c03d7fb820c0c17e87d9614fbee0afa39c32c7a2ad3c", size = 14938 },+]+[[package]]name = "fake-useragent"version = "2.1.0"@@ -2342,56 +2379,37 @@ sdist = { url = "https://files.pythonhosted.org/packages/fe/26/bca4d737a9acea25e[[package]]name = "gradio"-version = "3.50.2"+version = "3.18.0"source = { registry = "https://pypi.org/simple" }dependencies = [{ name = "aiofiles" },+ { name = "aiohttp" },{ name = "altair" },{ name = "fastapi" },{ name = "ffmpy" },- { name = "gradio-client" },+ { name = "fsspec" },{ name = "httpx" },- { name = "huggingface-hub" },- { name = "importlib-resources" },{ name = "jinja2" },+ { name = "markdown-it-py", extra = ["linkify", "plugins"] },{ name = "markupsafe" },{ name = "matplotlib" },{ name = "numpy" },{ name = "orjson" },- { name = "packaging" },{ name = "pandas" },{ name = "pillow" },+ { name = "pycryptodome" },{ name = "pydantic" },{ name = "pydub" },{ name = "python-multipart" },{ name = "pyyaml" },{ name = "requests" },- { name = "semantic-version" },{ name = "typing-extensions" },{ name = "uvicorn" },{ name = "websockets" },]-sdist = { url = "https://files.pythonhosted.org/packages/e4/8e/d83c07975d823b8e6e807c29116cbd34e46d5d980d886645ab729d3ff784/gradio-3.50.2.tar.gz", hash = "sha256:c6c81320566ba3e5688a1a58201d0729565a97b828b2bf6895e54f7bf25c01de", size = 20763672 }-wheels = [- { url = "https://files.pythonhosted.org/packages/bd/ea/ca6506e4da9b5338da3bfdd6115dc1c90ffd58c1ec50ca2792b84a7b4bdb/gradio-3.50.2-py3-none-any.whl", hash = "sha256:dffc070955225dbc3af0cf4ce93a169267485bb6972ee76ff7a0f31be77346a0", size = 20298198 },-]--[[package]]-name = "gradio-client"-version = "0.6.1"-source = { registry = "https://pypi.org/simple" }-dependencies = [- { name = "fsspec" },- { name = "httpx" },- { name = "huggingface-hub" },- { name = "packaging" },- { name = "requests" },- { name = "typing-extensions" },- { name = "websockets" },-]-sdist = { url = "https://files.pythonhosted.org/packages/e2/49/eb84403432f2527239ef7b3d9d0835998101a69f6e93c00d8fd59cb58896/gradio_client-0.6.1.tar.gz", hash = "sha256:2f36a8467622f748539c84d142f1a71df6b83666d017ee4fb6ef8a2696f70f66", size = 296938 }+sdist = { url = "https://files.pythonhosted.org/packages/3c/6d/24cb3d245c5e5306ccc39b717399bd084ba424b156f2b07fd83f8dea0ab0/gradio-3.18.0.tar.gz", hash = "sha256:f66d19c651c740da6cfa2b411b0e19942579532e0ffc5c41f71a2adbf0bc5c30", size = 15008984 }wheels = [- { url = "https://files.pythonhosted.org/packages/7d/04/e1654ee28fb2686514ca8ae31af5e489403964d48764788f9a168e069c0f/gradio_client-0.6.1-py3-none-any.whl", hash = "sha256:ee530fb3437f24943e51da70335909cc9b4f1e3e1ab1d219071b8ce9c1969777", size = 299220 },+ { url = "https://files.pythonhosted.org/packages/ef/15/06837222516d7d455191fbcc19ee827cb60f51bfe78918d1d938f2788e97/gradio-3.18.0-py3-none-any.whl", hash = "sha256:de608f310584d1b16c9554015352adfa4804e6319d20a6b66e271377d2bbb31d", size = 14192044 },][[package]]@@ -2707,15 +2725,6 @@ wheels = [{ url = "https://files.pythonhosted.org/packages/79/9d/0fb148dc4d6fa4a7dd1d8378168d9b4cd8d4560a6fbf6f0121c5fc34eb68/importlib_metadata-8.6.1-py3-none-any.whl", hash = "sha256:02a89390c1e15fdfdc0d7c6b25cb3e62650d0494005c97d6f148bf5b9787525e", size = 26971 },]-[[package]]-name = "importlib-resources"-version = "6.5.2"-source = { registry = "https://pypi.org/simple" }-sdist = { url = "https://files.pythonhosted.org/packages/cf/8c/f834fbf984f691b4f7ff60f50b514cc3de5cc08abfc3295564dd89c5e2e7/importlib_resources-6.5.2.tar.gz", hash = "sha256:185f87adef5bcc288449d98fb4fba07cea78bc036455dd44c5fc4a2fe78fed2c", size = 44693 }-wheels = [- { url = "https://files.pythonhosted.org/packages/a4/ed/1f1afb2e9e7f38a545d628f864d562a5ae64fe6f7a10e28ffb9b185b4e89/importlib_resources-6.5.2-py3-none-any.whl", hash = "sha256:789cfdc3ed28c78b67a06acb8126751ced69a3d5f79c095a98298cd8a760ccec", size = 37461 },-]-[[package]]name = "iniconfig"version = "2.0.0"@@ -3151,6 +3160,18 @@ wheels = [{ url = "https://files.pythonhosted.org/packages/31/8b/94dc8d58704ab87b39faed6f2fc0090b9d90e2e2aa2bbec35c79f3d2a054/lazy_object_proxy-1.10.0-pp310.pp311.pp312.pp38.pp39-none-any.whl", hash = "sha256:80fa48bd89c8f2f456fc0765c11c23bf5af827febacd2f523ca5bc1893fcc09d", size = 16405 },]+[[package]]+name = "linkify-it-py"+version = "2.0.3"+source = { registry = "https://pypi.org/simple" }+dependencies = [+ { name = "uc-micro-py" },+]+sdist = { url = "https://files.pythonhosted.org/packages/2a/ae/bb56c6828e4797ba5a4821eec7c43b8bf40f69cda4d4f5f8c8a2810ec96a/linkify-it-py-2.0.3.tar.gz", hash = "sha256:68cda27e162e9215c17d786649d1da0021a451bdc436ef9e0fa0ba5234b9b048", size = 27946 }+wheels = [+ { url = "https://files.pythonhosted.org/packages/04/1e/b832de447dee8b582cac175871d2f6c3d5077cc56d5575cadba1fd1cccfa/linkify_it_py-2.0.3-py3-none-any.whl", hash = "sha256:6bcbc417b0ac14323382aef5c5192c0075bf8a9d6b41820a2b66371eac6b6d79", size = 19820 },+]+[[package]]name = "linkup-sdk"version = "0.2.3"@@ -3263,6 +3284,14 @@ wheels = [{ url = "https://files.pythonhosted.org/packages/42/d7/1ec15b46af6af88f19b8e5ffea08fa375d433c998b8a7639e76935c14f1f/markdown_it_py-3.0.0-py3-none-any.whl", hash = "sha256:355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1", size = 87528 },]+[package.optional-dependencies]+linkify = [+ { name = "linkify-it-py" },+]+plugins = [+ { name = "mdit-py-plugins" },+]+[[package]]name = "markdownify"version = "1.1.0"@@ -4892,6 +4921,29 @@ wheels = [{ url = "https://files.pythonhosted.org/packages/13/a3/a812df4e2dd5696d1f351d58b8fe16a405b234ad2886a0dab9183fb78109/pycparser-2.22-py3-none-any.whl", hash = "sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc", size = 117552 },]+[[package]]+name = "pycryptodome"+version = "3.22.0"+source = { registry = "https://pypi.org/simple" }+sdist = { url = "https://files.pythonhosted.org/packages/44/e6/099310419df5ada522ff34ffc2f1a48a11b37fc6a76f51a6854c182dbd3e/pycryptodome-3.22.0.tar.gz", hash = "sha256:fd7ab568b3ad7b77c908d7c3f7e167ec5a8f035c64ff74f10d47a4edd043d723", size = 4917300 }+wheels = [+ { url = "https://files.pythonhosted.org/packages/1f/65/a05831c3e4bcd1bf6c2a034e399f74b3d6f30bb4e37e36b9c310c09dc8c0/pycryptodome-3.22.0-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:009e1c80eea42401a5bd5983c4bab8d516aef22e014a4705622e24e6d9d703c6", size = 2490637 },+ { url = "https://files.pythonhosted.org/packages/5c/76/ff3c2e7a60d17c080c4c6120ebaf60f38717cd387e77f84da4dcf7f64ff0/pycryptodome-3.22.0-cp37-abi3-macosx_10_9_x86_64.whl", hash = "sha256:3b76fa80daeff9519d7e9f6d9e40708f2fce36b9295a847f00624a08293f4f00", size = 1635372 },+ { url = "https://files.pythonhosted.org/packages/cc/7f/cc5d6da0dbc36acd978d80a72b228e33aadaec9c4f91c93221166d8bdc05/pycryptodome-3.22.0-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a31fa5914b255ab62aac9265654292ce0404f6b66540a065f538466474baedbc", size = 2177456 },+ { url = "https://files.pythonhosted.org/packages/92/65/35f5063e68790602d892ad36e35ac723147232a9084d1999630045c34593/pycryptodome-3.22.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a0092fd476701eeeb04df5cc509d8b739fa381583cda6a46ff0a60639b7cd70d", size = 2263744 },+ { url = "https://files.pythonhosted.org/packages/cc/67/46acdd35b1081c3dbc72dc466b1b95b80d2f64cad3520f994a9b6c5c7d00/pycryptodome-3.22.0-cp37-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:18d5b0ddc7cf69231736d778bd3ae2b3efb681ae33b64b0c92fb4626bb48bb89", size = 2303356 },+ { url = "https://files.pythonhosted.org/packages/3d/f9/a4f8a83384626098e3f55664519bec113002b9ef751887086ae63a53135a/pycryptodome-3.22.0-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:f6cf6aa36fcf463e622d2165a5ad9963b2762bebae2f632d719dfb8544903cf5", size = 2176714 },+ { url = "https://files.pythonhosted.org/packages/88/65/e5f8c3a885f70a6e05c84844cd5542120576f4369158946e8cfc623a464d/pycryptodome-3.22.0-cp37-abi3-musllinux_1_2_i686.whl", hash = "sha256:aec7b40a7ea5af7c40f8837adf20a137d5e11a6eb202cde7e588a48fb2d871a8", size = 2337329 },+ { url = "https://files.pythonhosted.org/packages/b8/2a/25e0be2b509c28375c7f75c7e8d8d060773f2cce4856a1654276e3202339/pycryptodome-3.22.0-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:d21c1eda2f42211f18a25db4eaf8056c94a8563cd39da3683f89fe0d881fb772", size = 2262255 },+ { url = "https://files.pythonhosted.org/packages/41/58/60917bc4bbd91712e53ce04daf237a74a0ad731383a01288130672994328/pycryptodome-3.22.0-cp37-abi3-win32.whl", hash = "sha256:f02baa9f5e35934c6e8dcec91fcde96612bdefef6e442813b8ea34e82c84bbfb", size = 1763403 },+ { url = "https://files.pythonhosted.org/packages/55/f4/244c621afcf7867e23f63cfd7a9630f14cfe946c9be7e566af6c3915bcde/pycryptodome-3.22.0-cp37-abi3-win_amd64.whl", hash = "sha256:d086aed307e96d40c23c42418cbbca22ecc0ab4a8a0e24f87932eeab26c08627", size = 1794568 },+ { url = "https://files.pythonhosted.org/packages/37/c3/e3423e72669ca09f141aae493e1feaa8b8475859898b04f57078280a61c4/pycryptodome-3.22.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:b4bdce34af16c1dcc7f8c66185684be15f5818afd2a82b75a4ce6b55f9783e13", size = 1618698 },+ { url = "https://files.pythonhosted.org/packages/f9/b7/35eec0b3919cafea362dcb68bb0654d9cb3cde6da6b7a9d8480ce0bf203a/pycryptodome-3.22.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2988ffcd5137dc2d27eb51cd18c0f0f68e5b009d5fec56fbccb638f90934f333", size = 1666957 },+ { url = "https://files.pythonhosted.org/packages/b0/1f/f49bccdd8d61f1da4278eb0d6aee7f988f1a6ec4056b0c2dc51eda45ae27/pycryptodome-3.22.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e653519dedcd1532788547f00eeb6108cc7ce9efdf5cc9996abce0d53f95d5a9", size = 1659242 },+ { url = "https://files.pythonhosted.org/packages/95/43/a01dcf1ed39c9a9e9c9d3f9d98040deeceedaa7cdf043e175251f2e13f44/pycryptodome-3.22.0-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f5810bc7494e4ac12a4afef5a32218129e7d3890ce3f2b5ec520cc69eb1102ad", size = 1697246 },+ { url = "https://files.pythonhosted.org/packages/3b/49/195842931f9ee6f14cd63ef85e06b93073463ed59601fb283ba9b813cd53/pycryptodome-3.22.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:e7514a1aebee8e85802d154fdb261381f1cb9b7c5a54594545145b8ec3056ae6", size = 1797436 },+]+[[package]]name = "pydantic"version = "2.11.2"@@ -6036,15 +6088,6 @@ wheels = [{ url = "https://files.pythonhosted.org/packages/2f/a6/fc66ea71ec0769f72abdf15cb9ec9269517abe68a160839383ddff7478f1/selenium-4.29.0-py3-none-any.whl", hash = "sha256:ce5d26f1ddc1111641113653af33694c13947dd36c2df09cdd33f554351d372e", size = 9536642 },]-[[package]]-name = "semantic-version"-version = "2.10.0"-source = { registry = "https://pypi.org/simple" }-sdist = { url = "https://files.pythonhosted.org/packages/7d/31/f2289ce78b9b473d582568c234e104d2a342fd658cc288a7553d83bb8595/semantic_version-2.10.0.tar.gz", hash = "sha256:bdabb6d336998cbb378d4b9db3a4b56a1e3235701dc05ea2690d9a997ed5041c", size = 52289 }-wheels = [- { url = "https://files.pythonhosted.org/packages/6a/23/8146aad7d88f4fcb3a6218f41a60f6c2d4e3a72de72da1825dc7c8f7877c/semantic_version-2.10.0-py2.py3-none-any.whl", hash = "sha256:de78a3b8e0feda74cabc54aab2da702113e33ac9d9eb9d2389bcf1f58b7d9177", size = 15552 },-]-[[package]]name = "sentence-transformers"version = "3.4.1"@@ -6553,6 +6596,19 @@ wheels = [{ url = "https://files.pythonhosted.org/packages/44/a4/595b66c1342a830f4ff476307c4367541cce3e361b9ee96d22a5eeeddd1d/textblob-0.17.1-py2.py3-none-any.whl", hash = "sha256:15546d7f309e96a3f542bee42751c8e5ce4d519d3d274ee79df2318141f0b788", size = 636763 },]+[[package]]+name = "tf-playwright-stealth"+version = "1.1.2"+source = { registry = "https://pypi.org/simple" }+dependencies = [+ { name = "fake-http-header" },+ { name = "playwright" },+]+sdist = { url = "https://files.pythonhosted.org/packages/eb/46/d73c62c4d84a06bac77e1f515560a08dee212b630afec9162c38f29c1d68/tf_playwright_stealth-1.1.2.tar.gz", hash = "sha256:d9f78890940c1d1de5b73c366f68930a206bd62d7a06aba4be32fc222ba058b4", size = 23361 }+wheels = [+ { url = "https://files.pythonhosted.org/packages/3a/2b/10101d8db05e5b1a1fcb197bbde9ee87c6066108f546356771bc6d84b1cc/tf_playwright_stealth-1.1.2-py3-none-any.whl", hash = "sha256:050bb98d221909de40ee5e75ec7c3d351320eab3b6ad6d8df608090efc16a0c5", size = 33208 },+]+[[package]]name = "threadpoolctl"version = "3.6.0"@@ -7001,18 +7057,6 @@ wheels = [{ url = "https://files.pythonhosted.org/packages/65/f3/107a22063bf27bdccf2024833d3445f4eea42b2e598abfbd46f6a63b6cb0/typing_inspect-0.9.0-py3-none-any.whl", hash = "sha256:9ee6fc59062311ef8547596ab6b955e1b8aa46242d854bfc78f4f6b0eff35f9f", size = 8827 },]-[[package]]-name = "typing-inspection"-version = "0.4.0"-source = { registry = "https://pypi.org/simple" }-dependencies = [- { name = "typing-extensions" },-]-sdist = { url = "https://files.pythonhosted.org/packages/82/5c/e6082df02e215b846b4b8c0b887a64d7d08ffaba30605502639d44c06b82/typing_inspection-0.4.0.tar.gz", hash = "sha256:9765c87de36671694a67904bf2c96e395be9c6439bb6c87b5142569dcdd65122", size = 76222 }-wheels = [- { url = "https://files.pythonhosted.org/packages/31/08/aa4fdfb71f7de5176385bd9e90852eaf6b5d622735020ad600f2bab54385/typing_inspection-0.4.0-py3-none-any.whl", hash = "sha256:50e72559fcd2a6367a19f7a7e610e6afcb9fac940c650290eed893d61386832f", size = 14125 },-]-[[package]]name = "ujson"version = "5.10.0"