Skip to content

armgit / camel

Public Imported from GitHub
0

🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org

feat: integrate crawl4ai (#2030)

Abhay Kashyapbbd63fbd3a4c
diff --git a/camel/loaders/__init__.py b/camel/loaders/__init__.py
index 7dc4e0d53..83963c33a 100644
--- a/camel/loaders/__init__.py
+++ b/camel/loaders/__init__.py
@@ -15,6 +15,7 @@
from .apify_reader import Apify
from .base_io import File, create_file, create_file_from_raw_bytes
from .chunkr_reader import ChunkrReader
+from .crawl4ai_reader import Crawl4AI
from .firecrawl_reader import Firecrawl
from .jina_url_reader import JinaURLReader
from .mineru_extractor import MinerU
@@ -32,4 +33,5 @@ __all__ = [
'ChunkrReader',
'PandasReader',
'MinerU',
+ 'Crawl4AI',
]
diff --git a/camel/loaders/crawl4ai_reader.py b/camel/loaders/crawl4ai_reader.py
new file mode 100644
index 000000000..0f3be56ee
--- /dev/null
+++ b/camel/loaders/crawl4ai_reader.py
@@ -0,0 +1,243 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+
+import asyncio
+import logging
+from typing import Any, Dict, List, Optional
+
+from dotenv import load_dotenv
+from pydantic import BaseModel, ValidationError
+
+from camel.utils import api_keys_required
+
+logger = logging.getLogger(__name__)
+load_dotenv()
+
+
+class Crawl4AI:
+ r"""Class for converting websites into LLM-ready data.
+
+ This class uses asynchronous crawling with CSS selectors or LLM-based
+ extraction to convert entire websites into structured data.
+
+ Args:
+ None: No parameters are required for initialization.
+
+ References:
+ https://docs.crawl4ai.com/
+ """
+
+ def __init__(self) -> None:
+ from crawl4ai import AsyncWebCrawler
+
+ self.crawler = AsyncWebCrawler
+
+ async def _run_crawler(self, url: str, **kwargs) -> Any:
+ r"""Run the asynchronous web crawler on a given URL.
+
+ Args:
+ url (str): URL to crawl or scrape.
+ **kwargs: Additional keyword arguments for crawler configuration.
+
+ Returns:
+ Any: The result from the crawler.
+
+ Raises:
+ RuntimeError: If crawler execution fails.
+ """
+
+ try:
+ async with self.crawler() as c:
+ return await c.arun(url, **kwargs)
+ except Exception as e:
+ logger.error("Crawler run failed: %s", e)
+ raise RuntimeError(f"Crawler run failed: {e}") from e
+
+ async def crawl(
+ self,
+ start_url: str,
+ max_depth: int = 1,
+ extraction_strategy=None,
+ **kwargs,
+ ) -> List[Dict[str, Any]]:
+ r"""Crawl a URL and its subpages using breadth-first search.
+
+ Args:
+ start_url (str): URL to start crawling from.
+ max_depth (int, optional): Maximum depth of links to follow
+ (default: 1).
+ extraction_strategy (ExtractionStrategy, optional): Strategy
+ for data extraction.
+ **kwargs: Additional arguments for crawler configuration.
+
+ Returns:
+ List[Dict[str, Any]]: List of crawled page results.
+
+ Raises:
+ RuntimeError: If an error occurs during crawling.
+ """
+
+ all_results: List[Dict[str, Any]] = []
+ visited_urls: set[str] = set()
+ queue: asyncio.Queue[tuple[str, int]] = asyncio.Queue()
+
+ await queue.put((start_url, 1))
+ visited_urls.add(start_url)
+
+ while not queue.empty():
+ url, depth = await queue.get()
+ try:
+ result = await self._run_crawler(
+ url, extraction_strategy=extraction_strategy, **kwargs
+ )
+ all_results.append(
+ {
+ "url": url,
+ "raw_result": result,
+ "markdown": result.markdown,
+ "cleaned_html": result.cleaned_html,
+ "links": result.links,
+ }
+ )
+
+ if depth < max_depth and result.links:
+ for _, links in result.links.items():
+ for link in links:
+ if link['href'] not in visited_urls:
+ visited_urls.add(link['href'])
+ await queue.put((link['href'], depth + 1))
+
+ except Exception as e:
+ logger.error("Error crawling %s: %s", url, e)
+ raise RuntimeError(f"Error crawling {url}: {e}") from e
+
+ queue.task_done()
+
+ await queue.join()
+
+ return all_results
+
+ async def scrape(
+ self,
+ url: str,
+ extraction_strategy=None,
+ **kwargs,
+ ) -> Dict[str, Any]:
+ r"""Scrape a single URL using CSS or LLM-based extraction.
+
+ Args:
+ url (str): URL to scrape.
+ extraction_strategy (ExtractionStrategy, optional): Extraction
+ strategy to use.
+ **kwargs: Additional arguments for crawler configuration.
+
+ Returns:
+ Dict[str, Any]: Dictionary containing scraped data such as markdown
+ and HTML content.
+
+ Raises:
+ RuntimeError: If scraping fails.
+ """
+
+ result = await self._run_crawler(
+ url, extraction_strategy=extraction_strategy, **kwargs
+ )
+ return {
+ "url": url,
+ "raw_result": result,
+ "markdown": result.markdown,
+ "cleaned_html": result.cleaned_html,
+ "links": result.links,
+ }
+
+ @api_keys_required(
+ [
+ # ("api_key", "ANTHROPIC_API_KEY"),
+ # ("api_key", "DEEPSEEK_API_KEY"),
+ # ("api_key", "GEMINI_API_KEY"),
+ # ("api_key", "GROQ_API_KEY"),
+ ("api_key", "OPENAI_API_KEY"),
+ ]
+ )
+ async def structured_scrape(
+ self,
+ url: str,
+ response_format: BaseModel,
+ api_key: Optional[str] = None,
+ llm_provider: str = 'ollama/llama3',
+ **kwargs,
+ ) -> Any:
+ r"""Extract structured data from a URL using an LLM.
+
+ Args:
+ url (str): URL to scrape.
+ response_format (BaseModel): Model defining the expected output
+ schema.
+ api_key (str, optional): API key for the LLM provider
+ (default: "no-token").
+ llm_provider (str, optional): Identifier for the LLM provider
+ (default: 'ollama/llama3').
+ **kwargs: Additional arguments for crawler configuration.
+
+ Returns:
+ Any: Crawl result containing the extracted data
+ structured according to the schema.
+
+ Raises:
+ ValidationError: If extracted data does not match the schema.
+ RuntimeError: If extraction fails.
+ """
+
+ from crawl4ai.extraction_strategy import (
+ LLMExtractionStrategy,
+ )
+
+ extraction_strategy = LLMExtractionStrategy(
+ provider=llm_provider,
+ api_token=api_key,
+ schema=response_format.model_json_schema(),
+ extraction_type="schema",
+ instruction="Extract the data according to the schema.",
+ )
+
+ try:
+ return await self._run_crawler(
+ url, extraction_strategy=extraction_strategy, **kwargs
+ )
+ except ValidationError as e:
+ raise ValidationError(
+ f"Extracted data does not match schema: {e}"
+ ) from e
+ except Exception as e:
+ raise RuntimeError(e) from e
+
+ async def map_site(self, url: str, **kwargs) -> List[str]:
+ r"""Map a website by extracting all accessible URLs.
+
+ Args:
+ url (str): Starting URL to map.
+ **kwargs: Additional configuration arguments.
+
+ Returns:
+ List[str]: List of URLs discovered on the website.
+
+ Raises:
+ RuntimeError: If mapping fails.
+ """
+
+ try:
+ result = await self.crawl(url, **kwargs)
+ return [page["url"] for page in result]
+ except Exception as e:
+ raise RuntimeError(f"Failed to map url: {e}") from e
diff --git a/examples/loaders/crawl4ai_example.py b/examples/loaders/crawl4ai_example.py
new file mode 100644
index 000000000..590c70aa8
--- /dev/null
+++ b/examples/loaders/crawl4ai_example.py
@@ -0,0 +1,105 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+
+import asyncio
+
+from pydantic import BaseModel, Field
+
+from camel.loaders import Crawl4AI
+
+# Instantiate the Crawl4AI crawler
+crawler = Crawl4AI()
+
+URL = "https://toscrape.com/"
+
+# --- Crawling (BFS) ---
+crawl_results = asyncio.run(crawler.crawl(URL, max_depth=1))
+print(crawl_results[0]["markdown"])
+'''
+![](img/zyte.png)
+
+# Web Scraping Sandbox
+
+## Books
+
+A [fictional bookstore](http://books.toscrape.com) that desperately wants to
+be scraped. It's a safe place for beginners learning web scraping and for
+developers validating their scraping technologies as well.
+Available at: [books.toscrape.com](http://books.toscrape.com)
+
+[![](./img/books.png)](http://books.toscrape.com)
+
+Details
+---
+Amount of items | 1000
+Pagination | ✔
+Items per page | max 20
+Requires JavaScript | ✘
+
+## Quotes
+
+[A website](http://quotes.toscrape.com/) that lists quotes from famous people.
+It has many endpoints showing the quotes in many different ways, each of them
+including new scraping challenges for you, as described below.
+
+[![](./img/quotes.png)](http://quotes.toscrape.com)
+
+Endpoints
+---
+[Default](http://quotes.toscrape.com/)| Microdata and pagination
+[Scroll](http://quotes.toscrape.com/scroll) | infinite scrolling pagination
+[JavaScript](http://quotes.toscrape.com/js) | JavaScript generated content
+[Delayed](http://quotes.toscrape.com/js-delayed) | Same as JavaScript but with
+ a delay (?delay=10000)
+[Tableful](http://quotes.toscrape.com/tableful) | a table based messed-up
+ layout
+[Login](http://quotes.toscrape.com/login) | login with CSRF token
+ (any user/passwd works)
+[ViewState](http://quotes.toscrape.com/search.aspx) | an AJAX based filter
+ form with ViewStates
+[Random](http://quotes.toscrape.com/random) | a single random quote
+'''
+
+
+# --- Scraping ---
+scrape_result = asyncio.run(crawler.scrape(URL))
+print(scrape_result)
+'''
+{url: 'https://toscrape.com/',
+'raw_result': CrawlResult(url='https://toscrape.com/', markdown=...,
+ cleaned_html=..., links=...),
+'markdown': "![](img/zyte.png)\n\n# Web Scraping Sandbox\n\n## Books...",
+'cleaned_html': '\n<div>\n<div>\n<div>\n<img class="logo" height="108"
+ src="img/zyte.png" width="200"/>\n
+ <h1>Web Scraping Sandbox</h1>\n...'}
+'''
+
+
+# --- Structured Scraping ---
+class MovieResponse(BaseModel):
+ title: str = Field(..., description="The name of the website.")
+ year: int = Field(..., description="The amount of items on the bookstore.")
+ rating: float = Field(
+ ..., description="The names of the endpoints of quotes."
+ )
+
+
+structured_scrape_result = asyncio.run(
+ crawler.structured_scrape(
+ URL,
+ response_format=MovieResponse,
+ llm_provider="openai/gpt-4o",
+ )
+)
+print(structured_scrape_result.extracted_content)
diff --git a/pyproject.toml b/pyproject.toml
index bbcbe2a8d..706db4134 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -30,6 +30,7 @@ dependencies = [
"pydantic>=2.10.6",
"httpx>=0.28.0,<1.0.0dev",
"psutil>=5.9.8,<6",
+ "crawl4ai>=0.3.745",
]
[project.optional-dependencies]
diff --git a/test/loaders/test_crawl4ai.py b/test/loaders/test_crawl4ai.py
new file mode 100644
index 000000000..e6eba4a4a
--- /dev/null
+++ b/test/loaders/test_crawl4ai.py
@@ -0,0 +1,175 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+
+from typing import Optional
+from unittest.mock import AsyncMock, patch
+
+import pytest
+from pydantic import BaseModel
+
+from camel.loaders import Crawl4AI
+
+
+# Dummy CrawlResult to simulate the results from AsyncWebCrawler.arun
+class DummyCrawlResult:
+ def __init__(
+ self, markdown: str, cleaned_html: str, links: Optional[dict] = None
+ ):
+ self.markdown = markdown
+ self.cleaned_html = cleaned_html
+ self.links = links or {}
+
+
+URL = "https://example.com"
+
+
+# ------------------------
+# Test Cases for Crawl4AI
+# ------------------------
+
+
+def test_init():
+ crawler = Crawl4AI()
+ # Verify that the crawler attribute is set to AsyncWebCrawler
+ assert crawler.crawler is not None
+
+
+async def test_crawl_success():
+ dummy_result = DummyCrawlResult(
+ markdown="Test markdown", cleaned_html="<html></html>", links={}
+ )
+
+ with patch.object(
+ Crawl4AI, "_run_crawler", new_callable=AsyncMock
+ ) as mock_run_crawler:
+ mock_run_crawler.return_value = dummy_result
+ crawler = Crawl4AI()
+ results = await crawler.crawl(URL, max_depth=1)
+ assert isinstance(results, list)
+ page = results[0]
+ assert page["url"] == URL
+ assert page["markdown"] == "Test markdown"
+ assert page["cleaned_html"] == "<html></html>"
+ assert page["links"] == {}
+
+
+async def test_crawl_failure_in_run():
+ with patch.object(
+ Crawl4AI, "_run_crawler", new_callable=AsyncMock
+ ) as mock_run_crawler:
+ mock_run_crawler.side_effect = Exception(
+ f"Error crawling {URL}: Crawl error"
+ )
+ crawler = Crawl4AI()
+ with pytest.raises(
+ RuntimeError, match=f"Error crawling {URL}: Crawl error"
+ ):
+ await crawler.crawl(URL)
+
+
+async def test_scrape_success():
+ dummy_result = DummyCrawlResult(
+ markdown="Scraped markdown",
+ cleaned_html="<html>scraped</html>",
+ links={},
+ )
+
+ with patch.object(
+ Crawl4AI, "_run_crawler", new_callable=AsyncMock
+ ) as mock_run_crawler:
+ mock_run_crawler.return_value = dummy_result
+ crawler = Crawl4AI()
+ result = await crawler.scrape(URL)
+ assert result["url"] == URL
+ assert result["markdown"] == "Scraped markdown"
+ assert result["cleaned_html"] == "<html>scraped</html>"
+ assert result["links"] == {}
+
+
+async def test_structured_scrape_success():
+ dummy_result = DummyCrawlResult(
+ markdown="Scraped markdown",
+ cleaned_html="<html>scraped</html>",
+ links={},
+ )
+
+ class DummySchema(BaseModel):
+ field: str
+
+ with patch.object(
+ Crawl4AI, "_run_crawler", new_callable=AsyncMock
+ ) as mock_run_crawler:
+ mock_run_crawler.return_value = dummy_result
+ crawler = Crawl4AI()
+ result = await crawler.structured_scrape(
+ URL, DummySchema, llm_provider="openai"
+ )
+ assert result.markdown == "Scraped markdown"
+ assert result.cleaned_html == "<html>scraped</html>"
+ assert result.links == {}
+
+
+async def test_structured_scrape_failure():
+ # Simulate a failure in the structured scrape method.
+ with patch.object(
+ Crawl4AI, "_run_crawler", new_callable=AsyncMock
+ ) as mock_run_crawler:
+ mock_run_crawler.side_effect = RuntimeError("Structured scrape failed")
+ crawler = Crawl4AI()
+
+ class DummySchema(BaseModel):
+ field: str
+
+ with pytest.raises(RuntimeError, match="Structured scrape failed"):
+ await crawler.structured_scrape(URL, DummySchema)
+
+
+async def test_map_site_success():
+ # Simulate the crawl method to return a list with a dummy page
+ dummy_page = {
+ "url": "https://example.com/page1",
+ "markdown": "md",
+ "cleaned_html": "html",
+ }
+
+ # Instead of patching AsyncWebCrawler here, patch the crawl method itself
+ async def dummy_crawl(url: str, **kwargs):
+ return [dummy_page]
+
+ crawler = Crawl4AI()
+ crawler.crawl = dummy_crawl # Override crawl method for testing
+
+ urls = await crawler.map_site(URL)
+ assert urls == ["https://example.com/page1"]
+
+
+async def test_map_site_failure():
+ # Simulate crawl raising an exception in map_site
+ async def dummy_crawl_fail(url: str, **kwargs):
+ raise Exception("Mapping error")
+
+ crawler = Crawl4AI()
+ crawler.crawl = (
+ dummy_crawl_fail # Override crawl method to simulate failure
+ )
+
+ with pytest.raises(RuntimeError, match="Failed to map url"):
+ await crawler.map_site(URL)
diff --git a/uv.lock b/uv.lock
index c0fc6daf6..8f8eb6ed4 100644
--- a/uv.lock
+++ b/uv.lock
@@ -64,11 +64,11 @@ wheels = [
[[package]]
name = "aiofiles"
-version = "23.2.1"
+version = "24.1.0"
source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/af/41/cfed10bc64d774f497a86e5ede9248e1d062db675504b41c320954d99641/aiofiles-23.2.1.tar.gz", hash = "sha256:84ec2218d8419404abcb9f0c02df3f34c6e0a68ed41072acfb1cef5cbc29051a", size = 32072 }
+sdist = { url = "https://files.pythonhosted.org/packages/0b/03/a88171e277e8caa88a4c77808c20ebb04ba74cc4681bf1e9416c862de237/aiofiles-24.1.0.tar.gz", hash = "sha256:22a075c9e5a3810f0c2e48f3008c94d68c65d763b9b03857924c99e57355166c", size = 30247 }
wheels = [
- { url = "https://files.pythonhosted.org/packages/c5/19/5af6804c4cc0fed83f47bff6e413a98a36618e7d40185cd36e69737f3b0e/aiofiles-23.2.1-py3-none-any.whl", hash = "sha256:19297512c647d4b27a2cf7c34caa7e405c0d60b5560618a29a9fe027b18b0107", size = 15727 },
+ { url = "https://files.pythonhosted.org/packages/a5/45/30bb92d442636f570cb5651bc661f52b610e2eec3f891a5dc3a4c3667db0/aiofiles-24.1.0-py3-none-any.whl", hash = "sha256:b4ec55f4195e3eb5d7abd1bf7e061763e864dd4954231fb8539a0ef8bb8260e5", size = 15896 },
]
[[package]]
@@ -547,6 +547,7 @@ version = "0.2.43"
source = { editable = "." }
dependencies = [
{ name = "colorama" },
+ { name = "crawl4ai" },
{ name = "docstring-parser" },
{ name = "httpx" },
{ name = "jsonschema" },
@@ -893,6 +894,7 @@ requires-dist = [
{ name = "cohere", marker = "extra == 'model-platforms'", specifier = ">=5.11.0,<6" },
{ name = "cohere", marker = "extra == 'rag'", specifier = ">=5.11.0,<6" },
{ name = "colorama", specifier = ">=0.4.6,<0.5" },
+ { name = "crawl4ai", specifier = ">=0.3.745" },
{ name = "dappier", marker = "extra == 'all'", specifier = ">=0.3.3,<0.4" },
{ name = "dappier", marker = "extra == 'web-tools'", specifier = ">=0.3.3,<0.4" },
{ name = "datacommons", marker = "extra == 'all'", specifier = ">=1.4.3,<2" },
@@ -1437,6 +1439,33 @@ toml = [
{ name = "tomli", marker = "python_full_version <= '3.11'" },
]
+[[package]]
+name = "crawl4ai"
+version = "0.3.745"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "aiofiles" },
+ { name = "aiosqlite" },
+ { name = "beautifulsoup4" },
+ { name = "colorama" },
+ { name = "html2text" },
+ { name = "litellm" },
+ { name = "lxml" },
+ { name = "numpy" },
+ { name = "pillow" },
+ { name = "playwright" },
+ { name = "python-dotenv" },
+ { name = "rank-bm25" },
+ { name = "requests" },
+ { name = "snowballstemmer" },
+ { name = "tf-playwright-stealth" },
+ { name = "xxhash" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/02/5a/919e64ff2977d7aa1b2cda4d45f16ff8996cd2c2dc1f55936fb6cd214222/crawl4ai-0.3.745.tar.gz", hash = "sha256:990396d57e10ae7ccabf35c34a317dbd8c59a3ceca475eac75320a8808334438", size = 117277 }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/ed/7e/ebe351a457140330b20b6d8289b8f243b21de6e6bce505cd15b230a83bcb/Crawl4AI-0.3.745-py3-none-any.whl", hash = "sha256:763e6aba80959e60e1fe70cb9d954a4cf257eb230af30f51fcd99ff641a7a88d", size = 121997 },
+]
+
[[package]]
name = "cryptography"
version = "44.0.2"
@@ -1915,6 +1944,14 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/7b/8f/c4d9bafc34ad7ad5d8dc16dd1347ee0e507a52c3adb6bfa8887e1c6a26ba/executing-2.2.0-py2.py3-none-any.whl", hash = "sha256:11387150cad388d62750327a53d3339fad4888b39a6fe233c3afbb54ecffd3aa", size = 26702 },
]
+[[package]]
+name = "fake-http-header"
+version = "0.3.5"
+source = { registry = "https://pypi.org/simple" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/e3/0b/2849c87d9f13766e29c0a2f4d31681aa72e035016b251ab19d99bde7b592/fake_http_header-0.3.5-py3-none-any.whl", hash = "sha256:cd05f4bebf1b7e38b5f5c03d7fb820c0c17e87d9614fbee0afa39c32c7a2ad3c", size = 14938 },
+]
+
[[package]]
name = "fake-useragent"
version = "2.1.0"
@@ -2342,56 +2379,37 @@ sdist = { url = "https://files.pythonhosted.org/packages/fe/26/bca4d737a9acea25e
[[package]]
name = "gradio"
-version = "3.50.2"
+version = "3.18.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "aiofiles" },
+ { name = "aiohttp" },
{ name = "altair" },
{ name = "fastapi" },
{ name = "ffmpy" },
- { name = "gradio-client" },
+ { name = "fsspec" },
{ name = "httpx" },
- { name = "huggingface-hub" },
- { name = "importlib-resources" },
{ name = "jinja2" },
+ { name = "markdown-it-py", extra = ["linkify", "plugins"] },
{ name = "markupsafe" },
{ name = "matplotlib" },
{ name = "numpy" },
{ name = "orjson" },
- { name = "packaging" },
{ name = "pandas" },
{ name = "pillow" },
+ { name = "pycryptodome" },
{ name = "pydantic" },
{ name = "pydub" },
{ name = "python-multipart" },
{ name = "pyyaml" },
{ name = "requests" },
- { name = "semantic-version" },
{ name = "typing-extensions" },
{ name = "uvicorn" },
{ name = "websockets" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/e4/8e/d83c07975d823b8e6e807c29116cbd34e46d5d980d886645ab729d3ff784/gradio-3.50.2.tar.gz", hash = "sha256:c6c81320566ba3e5688a1a58201d0729565a97b828b2bf6895e54f7bf25c01de", size = 20763672 }
-wheels = [
- { url = "https://files.pythonhosted.org/packages/bd/ea/ca6506e4da9b5338da3bfdd6115dc1c90ffd58c1ec50ca2792b84a7b4bdb/gradio-3.50.2-py3-none-any.whl", hash = "sha256:dffc070955225dbc3af0cf4ce93a169267485bb6972ee76ff7a0f31be77346a0", size = 20298198 },
-]
-
-[[package]]
-name = "gradio-client"
-version = "0.6.1"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
- { name = "fsspec" },
- { name = "httpx" },
- { name = "huggingface-hub" },
- { name = "packaging" },
- { name = "requests" },
- { name = "typing-extensions" },
- { name = "websockets" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/e2/49/eb84403432f2527239ef7b3d9d0835998101a69f6e93c00d8fd59cb58896/gradio_client-0.6.1.tar.gz", hash = "sha256:2f36a8467622f748539c84d142f1a71df6b83666d017ee4fb6ef8a2696f70f66", size = 296938 }
+sdist = { url = "https://files.pythonhosted.org/packages/3c/6d/24cb3d245c5e5306ccc39b717399bd084ba424b156f2b07fd83f8dea0ab0/gradio-3.18.0.tar.gz", hash = "sha256:f66d19c651c740da6cfa2b411b0e19942579532e0ffc5c41f71a2adbf0bc5c30", size = 15008984 }
wheels = [
- { url = "https://files.pythonhosted.org/packages/7d/04/e1654ee28fb2686514ca8ae31af5e489403964d48764788f9a168e069c0f/gradio_client-0.6.1-py3-none-any.whl", hash = "sha256:ee530fb3437f24943e51da70335909cc9b4f1e3e1ab1d219071b8ce9c1969777", size = 299220 },
+ { url = "https://files.pythonhosted.org/packages/ef/15/06837222516d7d455191fbcc19ee827cb60f51bfe78918d1d938f2788e97/gradio-3.18.0-py3-none-any.whl", hash = "sha256:de608f310584d1b16c9554015352adfa4804e6319d20a6b66e271377d2bbb31d", size = 14192044 },
]
[[package]]
@@ -2707,15 +2725,6 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/79/9d/0fb148dc4d6fa4a7dd1d8378168d9b4cd8d4560a6fbf6f0121c5fc34eb68/importlib_metadata-8.6.1-py3-none-any.whl", hash = "sha256:02a89390c1e15fdfdc0d7c6b25cb3e62650d0494005c97d6f148bf5b9787525e", size = 26971 },
]
-[[package]]
-name = "importlib-resources"
-version = "6.5.2"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/cf/8c/f834fbf984f691b4f7ff60f50b514cc3de5cc08abfc3295564dd89c5e2e7/importlib_resources-6.5.2.tar.gz", hash = "sha256:185f87adef5bcc288449d98fb4fba07cea78bc036455dd44c5fc4a2fe78fed2c", size = 44693 }
-wheels = [
- { url = "https://files.pythonhosted.org/packages/a4/ed/1f1afb2e9e7f38a545d628f864d562a5ae64fe6f7a10e28ffb9b185b4e89/importlib_resources-6.5.2-py3-none-any.whl", hash = "sha256:789cfdc3ed28c78b67a06acb8126751ced69a3d5f79c095a98298cd8a760ccec", size = 37461 },
-]
-
[[package]]
name = "iniconfig"
version = "2.0.0"
@@ -3151,6 +3160,18 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/31/8b/94dc8d58704ab87b39faed6f2fc0090b9d90e2e2aa2bbec35c79f3d2a054/lazy_object_proxy-1.10.0-pp310.pp311.pp312.pp38.pp39-none-any.whl", hash = "sha256:80fa48bd89c8f2f456fc0765c11c23bf5af827febacd2f523ca5bc1893fcc09d", size = 16405 },
]
+[[package]]
+name = "linkify-it-py"
+version = "2.0.3"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "uc-micro-py" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/2a/ae/bb56c6828e4797ba5a4821eec7c43b8bf40f69cda4d4f5f8c8a2810ec96a/linkify-it-py-2.0.3.tar.gz", hash = "sha256:68cda27e162e9215c17d786649d1da0021a451bdc436ef9e0fa0ba5234b9b048", size = 27946 }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/04/1e/b832de447dee8b582cac175871d2f6c3d5077cc56d5575cadba1fd1cccfa/linkify_it_py-2.0.3-py3-none-any.whl", hash = "sha256:6bcbc417b0ac14323382aef5c5192c0075bf8a9d6b41820a2b66371eac6b6d79", size = 19820 },
+]
+
[[package]]
name = "linkup-sdk"
version = "0.2.3"
@@ -3263,6 +3284,14 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/42/d7/1ec15b46af6af88f19b8e5ffea08fa375d433c998b8a7639e76935c14f1f/markdown_it_py-3.0.0-py3-none-any.whl", hash = "sha256:355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1", size = 87528 },
]
+[package.optional-dependencies]
+linkify = [
+ { name = "linkify-it-py" },
+]
+plugins = [
+ { name = "mdit-py-plugins" },
+]
+
[[package]]
name = "markdownify"
version = "1.1.0"
@@ -4892,6 +4921,29 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/13/a3/a812df4e2dd5696d1f351d58b8fe16a405b234ad2886a0dab9183fb78109/pycparser-2.22-py3-none-any.whl", hash = "sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc", size = 117552 },
]
+[[package]]
+name = "pycryptodome"
+version = "3.22.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/44/e6/099310419df5ada522ff34ffc2f1a48a11b37fc6a76f51a6854c182dbd3e/pycryptodome-3.22.0.tar.gz", hash = "sha256:fd7ab568b3ad7b77c908d7c3f7e167ec5a8f035c64ff74f10d47a4edd043d723", size = 4917300 }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/1f/65/a05831c3e4bcd1bf6c2a034e399f74b3d6f30bb4e37e36b9c310c09dc8c0/pycryptodome-3.22.0-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:009e1c80eea42401a5bd5983c4bab8d516aef22e014a4705622e24e6d9d703c6", size = 2490637 },
+ { url = "https://files.pythonhosted.org/packages/5c/76/ff3c2e7a60d17c080c4c6120ebaf60f38717cd387e77f84da4dcf7f64ff0/pycryptodome-3.22.0-cp37-abi3-macosx_10_9_x86_64.whl", hash = "sha256:3b76fa80daeff9519d7e9f6d9e40708f2fce36b9295a847f00624a08293f4f00", size = 1635372 },
+ { url = "https://files.pythonhosted.org/packages/cc/7f/cc5d6da0dbc36acd978d80a72b228e33aadaec9c4f91c93221166d8bdc05/pycryptodome-3.22.0-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a31fa5914b255ab62aac9265654292ce0404f6b66540a065f538466474baedbc", size = 2177456 },
+ { url = "https://files.pythonhosted.org/packages/92/65/35f5063e68790602d892ad36e35ac723147232a9084d1999630045c34593/pycryptodome-3.22.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a0092fd476701eeeb04df5cc509d8b739fa381583cda6a46ff0a60639b7cd70d", size = 2263744 },
+ { url = "https://files.pythonhosted.org/packages/cc/67/46acdd35b1081c3dbc72dc466b1b95b80d2f64cad3520f994a9b6c5c7d00/pycryptodome-3.22.0-cp37-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:18d5b0ddc7cf69231736d778bd3ae2b3efb681ae33b64b0c92fb4626bb48bb89", size = 2303356 },
+ { url = "https://files.pythonhosted.org/packages/3d/f9/a4f8a83384626098e3f55664519bec113002b9ef751887086ae63a53135a/pycryptodome-3.22.0-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:f6cf6aa36fcf463e622d2165a5ad9963b2762bebae2f632d719dfb8544903cf5", size = 2176714 },
+ { url = "https://files.pythonhosted.org/packages/88/65/e5f8c3a885f70a6e05c84844cd5542120576f4369158946e8cfc623a464d/pycryptodome-3.22.0-cp37-abi3-musllinux_1_2_i686.whl", hash = "sha256:aec7b40a7ea5af7c40f8837adf20a137d5e11a6eb202cde7e588a48fb2d871a8", size = 2337329 },
+ { url = "https://files.pythonhosted.org/packages/b8/2a/25e0be2b509c28375c7f75c7e8d8d060773f2cce4856a1654276e3202339/pycryptodome-3.22.0-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:d21c1eda2f42211f18a25db4eaf8056c94a8563cd39da3683f89fe0d881fb772", size = 2262255 },
+ { url = "https://files.pythonhosted.org/packages/41/58/60917bc4bbd91712e53ce04daf237a74a0ad731383a01288130672994328/pycryptodome-3.22.0-cp37-abi3-win32.whl", hash = "sha256:f02baa9f5e35934c6e8dcec91fcde96612bdefef6e442813b8ea34e82c84bbfb", size = 1763403 },
+ { url = "https://files.pythonhosted.org/packages/55/f4/244c621afcf7867e23f63cfd7a9630f14cfe946c9be7e566af6c3915bcde/pycryptodome-3.22.0-cp37-abi3-win_amd64.whl", hash = "sha256:d086aed307e96d40c23c42418cbbca22ecc0ab4a8a0e24f87932eeab26c08627", size = 1794568 },
+ { url = "https://files.pythonhosted.org/packages/37/c3/e3423e72669ca09f141aae493e1feaa8b8475859898b04f57078280a61c4/pycryptodome-3.22.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:b4bdce34af16c1dcc7f8c66185684be15f5818afd2a82b75a4ce6b55f9783e13", size = 1618698 },
+ { url = "https://files.pythonhosted.org/packages/f9/b7/35eec0b3919cafea362dcb68bb0654d9cb3cde6da6b7a9d8480ce0bf203a/pycryptodome-3.22.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2988ffcd5137dc2d27eb51cd18c0f0f68e5b009d5fec56fbccb638f90934f333", size = 1666957 },
+ { url = "https://files.pythonhosted.org/packages/b0/1f/f49bccdd8d61f1da4278eb0d6aee7f988f1a6ec4056b0c2dc51eda45ae27/pycryptodome-3.22.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e653519dedcd1532788547f00eeb6108cc7ce9efdf5cc9996abce0d53f95d5a9", size = 1659242 },
+ { url = "https://files.pythonhosted.org/packages/95/43/a01dcf1ed39c9a9e9c9d3f9d98040deeceedaa7cdf043e175251f2e13f44/pycryptodome-3.22.0-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f5810bc7494e4ac12a4afef5a32218129e7d3890ce3f2b5ec520cc69eb1102ad", size = 1697246 },
+ { url = "https://files.pythonhosted.org/packages/3b/49/195842931f9ee6f14cd63ef85e06b93073463ed59601fb283ba9b813cd53/pycryptodome-3.22.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:e7514a1aebee8e85802d154fdb261381f1cb9b7c5a54594545145b8ec3056ae6", size = 1797436 },
+]
+
[[package]]
name = "pydantic"
version = "2.11.2"
@@ -6036,15 +6088,6 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/2f/a6/fc66ea71ec0769f72abdf15cb9ec9269517abe68a160839383ddff7478f1/selenium-4.29.0-py3-none-any.whl", hash = "sha256:ce5d26f1ddc1111641113653af33694c13947dd36c2df09cdd33f554351d372e", size = 9536642 },
]
-[[package]]
-name = "semantic-version"
-version = "2.10.0"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/7d/31/f2289ce78b9b473d582568c234e104d2a342fd658cc288a7553d83bb8595/semantic_version-2.10.0.tar.gz", hash = "sha256:bdabb6d336998cbb378d4b9db3a4b56a1e3235701dc05ea2690d9a997ed5041c", size = 52289 }
-wheels = [
- { url = "https://files.pythonhosted.org/packages/6a/23/8146aad7d88f4fcb3a6218f41a60f6c2d4e3a72de72da1825dc7c8f7877c/semantic_version-2.10.0-py2.py3-none-any.whl", hash = "sha256:de78a3b8e0feda74cabc54aab2da702113e33ac9d9eb9d2389bcf1f58b7d9177", size = 15552 },
-]
-
[[package]]
name = "sentence-transformers"
version = "3.4.1"
@@ -6553,6 +6596,19 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/44/a4/595b66c1342a830f4ff476307c4367541cce3e361b9ee96d22a5eeeddd1d/textblob-0.17.1-py2.py3-none-any.whl", hash = "sha256:15546d7f309e96a3f542bee42751c8e5ce4d519d3d274ee79df2318141f0b788", size = 636763 },
]
+[[package]]
+name = "tf-playwright-stealth"
+version = "1.1.2"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "fake-http-header" },
+ { name = "playwright" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/eb/46/d73c62c4d84a06bac77e1f515560a08dee212b630afec9162c38f29c1d68/tf_playwright_stealth-1.1.2.tar.gz", hash = "sha256:d9f78890940c1d1de5b73c366f68930a206bd62d7a06aba4be32fc222ba058b4", size = 23361 }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/3a/2b/10101d8db05e5b1a1fcb197bbde9ee87c6066108f546356771bc6d84b1cc/tf_playwright_stealth-1.1.2-py3-none-any.whl", hash = "sha256:050bb98d221909de40ee5e75ec7c3d351320eab3b6ad6d8df608090efc16a0c5", size = 33208 },
+]
+
[[package]]
name = "threadpoolctl"
version = "3.6.0"
@@ -7001,18 +7057,6 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/65/f3/107a22063bf27bdccf2024833d3445f4eea42b2e598abfbd46f6a63b6cb0/typing_inspect-0.9.0-py3-none-any.whl", hash = "sha256:9ee6fc59062311ef8547596ab6b955e1b8aa46242d854bfc78f4f6b0eff35f9f", size = 8827 },
]
-[[package]]
-name = "typing-inspection"
-version = "0.4.0"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
- { name = "typing-extensions" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/82/5c/e6082df02e215b846b4b8c0b887a64d7d08ffaba30605502639d44c06b82/typing_inspection-0.4.0.tar.gz", hash = "sha256:9765c87de36671694a67904bf2c96e395be9c6439bb6c87b5142569dcdd65122", size = 76222 }
-wheels = [
- { url = "https://files.pythonhosted.org/packages/31/08/aa4fdfb71f7de5176385bd9e90852eaf6b5d622735020ad600f2bab54385/typing_inspection-0.4.0-py3-none-any.whl", hash = "sha256:50e72559fcd2a6367a19f7a7e610e6afcb9fac940c650290eed893d61386832f", size = 14125 },
-]
-
[[package]]
name = "ujson"
version = "5.10.0"