🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org
Python agentai-societiesartificial-intelligencecommunicative-aicooperative-aideep-learninglarge-language-modelsmulti-agent-systemsnatural-language-processing
enhance: add scrapegraph-sdk integration PR2206 (#2259)
Wendong-Fan1a2f3c4bdea1
diff --git a/camel/loaders/__init__.py b/camel/loaders/__init__.pyindex 83963c33a..23cad7f98 100644--- a/camel/loaders/__init__.py+++ b/camel/loaders/__init__.py@@ -20,6 +20,7 @@ from .firecrawl_reader import Firecrawlfrom .jina_url_reader import JinaURLReaderfrom .mineru_extractor import MinerUfrom .pandas_reader import PandasReader+from .scrapegraph_reader import ScrapeGraphAIfrom .unstructured_io import UnstructuredIO__all__ = [@@ -34,4 +35,5 @@ __all__ = ['PandasReader','MinerU','Crawl4AI',+ 'ScrapeGraphAI',]diff --git a/camel/loaders/scrapegraph_reader.py b/camel/loaders/scrapegraph_reader.pyindex d787f9815..4cd19f191 100644--- a/camel/loaders/scrapegraph_reader.py+++ b/camel/loaders/scrapegraph_reader.py@@ -1,4 +1,4 @@-# ========= Copyright 2023-2025@ CAMEL-AI.org. All Rights Reserved. =========+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========# Licensed under the Apache License, Version 2.0 (the "License");# you may not use this file except in compliance with the License.# You may obtain a copy of the License at@@ -10,20 +10,19 @@# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.# See the License for the specific language governing permissions and# limitations under the License.-# ========= Copyright 2023-2025 @ CAMEL-AI.org. All Rights Reserved. =========+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========import osfrom typing import Any, Dict, Optional-from pydantic import BaseModel-class ScrapeGraphAI:- r"""ScrapeGraphAI allows you to perform AI-powered web scraping and searching.+ r"""ScrapeGraphAI allows you to perform AI-powered web scraping and+ searching.Args:- api_key (Optional[str]): API key for authenticating with the ScrapeGraphAI- API.+ api_key (Optional[str]): API key for authenticating with the+ ScrapeGraphAI API.References:https://scrapegraph.ai/@@ -50,7 +49,8 @@ class ScrapeGraphAI:user_prompt (str): The search query or instructions.Returns:- Dict[str, Any]: The search results including answer and reference URLs.+ Dict[str, Any]: The search results including answer and reference+ URLs.Raises:RuntimeError: If the search process fails.@@ -72,8 +72,8 @@ class ScrapeGraphAI:Args:website_url (str): The URL to scrape.user_prompt (str): Instructions for what data to extract.- website_html (Optional[str]): Optional HTML content to use instead of- fetching from the URL.+ website_html (Optional[str]): Optional HTML content to use instead+ of fetching from the URL.Returns:Dict[str, Any]: The scraped data including request ID and result.diff --git a/examples/loaders/scrapegraph_example.py b/examples/loaders/scrapegraph_example.pyindex 4991e662a..ccd441631 100644--- a/examples/loaders/scrapegraph_example.py+++ b/examples/loaders/scrapegraph_example.py@@ -1,4 +1,4 @@-# ========= Copyright 2023-2025@ CAMEL-AI.org. All Rights Reserved. =========+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========# Licensed under the Apache License, Version 2.0 (the "License");# you may not use this file except in compliance with the License.# You may obtain a copy of the License at@@ -10,10 +10,11 @@# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.# See the License for the specific language governing permissions and# limitations under the License.-# ========= Copyright 2023-2025 @ CAMEL-AI.org. All Rights Reserved. =========+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. ========="""-Example demonstrating how to use the ScrapeGraphAI reader for web scraping and searching.+Example demonstrating how to use the ScrapeGraphAI reader for web scraping and+searching.This example shows:1. How to initialize the ScrapeGraphAI reader@@ -23,27 +24,27 @@ This example shows:"""import os-from typing import Dict, Any+from typing import Any, Dictfrom camel.loaders.scrapegraph_reader import ScrapeGraphAIdef search_example(api_key: str) -> Dict[str, Any]:- """Example of performing an AI-powered web search."""+ r"""Example of performing an AI-powered web search."""# Initialize the ScrapeGraphAI readerscraper = ScrapeGraphAI(api_key=api_key)-+try:# Perform a searchsearch_query = "What are the latest developments in AI?"result = scraper.search(user_prompt=search_query)-+print("\nSearch Results:")print(f"Answer: {result.get('answer', 'No answer found')}")print("References:")for url in result.get('references', []):print(f"- {url}")-+return resultfinally:# Always close the connection@@ -51,10 +52,10 @@ def search_example(api_key: str) -> Dict[str, Any]:def scrape_example(api_key: str) -> Dict[str, Any]:- """Example of scraping a website with specific instructions."""+ r"""Example of scraping a website with specific instructions."""# Initialize the ScrapeGraphAI readerscraper = ScrapeGraphAI(api_key=api_key)-+try:# Scrape a website with specific instructionswebsite_url = "https://example.com"@@ -64,17 +65,16 @@ def scrape_example(api_key: str) -> Dict[str, Any]:2. All paragraph texts3. Any links to other pages"""-+result = scraper.scrape(- website_url=website_url,- user_prompt=instructions+ website_url=website_url, user_prompt=instructions)-+print("\nScraping Results:")print(f"Request ID: {result.get('request_id', 'No ID')}")print("Extracted Data:")print(result.get('result', {}))-+return resultfinally:# Always close the connection@@ -84,17 +84,17 @@ def scrape_example(api_key: str) -> Dict[str, Any]:def main():# Get API key from environment variable or use a placeholderapi_key = os.environ.get("SCRAPEGRAPH_API_KEY", "your_api_key_here")-+if api_key == "your_api_key_here":print("Please set your SCRAPEGRAPH_API_KEY environment variable")return-+print("Running search example...")search_example(api_key)-+print("\nRunning scrape example...")scrape_example(api_key)if __name__ == "__main__":- main()\ No newline at end of file+ main()diff --git a/pyproject.toml b/pyproject.tomlindex 5ca8078ec..1d900312b 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -240,6 +240,7 @@ owl = ["pandas>=1.5.3,<2","rouge>=1.0.1,<2","crawl4ai>=0.3.745",+ "scrapegraph-py>=1.12.0,<2",]all = ["numpy~=1.26",@@ -352,6 +353,7 @@ all = ["crawl4ai>=0.3.745","pyautogui>=0.9.54,<0.10","pyobvector>=0.1.18",+ "scrapegraph-py>=1.12.0,<2",][project.urls]@@ -538,6 +540,7 @@ module = ["crawl4ai.*","pyautogui","pyobvector.*",+ "scrapegraph_py.*",]ignore_missing_imports = truediff --git a/test/loaders/test_scrapegraph_reader.py b/test/loaders/test_scrapegraph_reader.pyindex 193585de8..88c89d299 100644--- a/test/loaders/test_scrapegraph_reader.py+++ b/test/loaders/test_scrapegraph_reader.py@@ -1,4 +1,4 @@-# ========= Copyright 2023-2025@ CAMEL-AI.org. All Rights Reserved. =========+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========# Licensed under the Apache License, Version 2.0 (the "License");# you may not use this file except in compliance with the License.# You may obtain a copy of the License at@@ -10,7 +10,7 @@# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.# See the License for the specific language governing permissions and# limitations under the License.-# ========= Copyright 2023-2025 @ CAMEL-AI.org. All Rights Reserved. =========+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========import osfrom unittest.mock import MagicMock, patch@@ -22,21 +22,23 @@ from camel.loaders.scrapegraph_reader import ScrapeGraphAI@pytest.fixturedef scrapegraph_ai():- with patch("camel.loaders.scrapegraph_reader.Client") as mock_client:+ with patch("scrapegraph_py.Client") as mock_client:mock_client_instance = MagicMock()mock_client.return_value = mock_client_instanceyield ScrapeGraphAI(api_key="test_api_key")def test_init_with_api_key():- with patch("camel.loaders.scrapegraph_reader.Client") as mock_client:+ with patch("scrapegraph_py.Client") as mock_client:ScrapeGraphAI(api_key="test_api_key")mock_client.assert_called_once_with(api_key="test_api_key")def test_init_with_env_var():- with patch("camel.loaders.scrapegraph_reader.Client") as mock_client, \- patch.dict(os.environ, {"SCRAPEGRAPH_API_KEY": "env_api_key"}):+ with (+ patch("scrapegraph_py.Client") as mock_client,+ patch.dict(os.environ, {"SCRAPEGRAPH_API_KEY": "env_api_key"}),+ ):ScrapeGraphAI()mock_client.assert_called_once_with(api_key="env_api_key")@@ -47,13 +49,19 @@ def test_search_success(scrapegraph_ai):result = scrapegraph_ai.search("test query")assert result == mock_response- scrapegraph_ai.client.searchscraper.assert_called_once_with(user_prompt="test query")+ scrapegraph_ai.client.searchscraper.assert_called_once_with(+ user_prompt="test query"+ )def test_search_failure(scrapegraph_ai):- scrapegraph_ai.client.searchscraper.side_effect = Exception("Search failed")+ scrapegraph_ai.client.searchscraper.side_effect = Exception(+ "Search failed"+ )- with pytest.raises(RuntimeError, match="Failed to perform search: Search failed"):+ with pytest.raises(+ RuntimeError, match="Failed to perform search: Search failed"+ ):scrapegraph_ai.search("test query")@@ -63,13 +71,13 @@ def test_scrape_success(scrapegraph_ai):result = scrapegraph_ai.scrape(website_url="https://example.com",- user_prompt="Extract title and description"+ user_prompt="Extract title and description",)assert result == mock_responsescrapegraph_ai.client.smartscraper.assert_called_once_with(website_url="https://example.com",user_prompt="Extract title and description",- website_html=None+ website_html=None,)@@ -80,26 +88,28 @@ def test_scrape_with_html(scrapegraph_ai):result = scrapegraph_ai.scrape(website_url="https://example.com",user_prompt="Extract title and description",- website_html="<html>test</html>"+ website_html="<html>test</html>",)assert result == mock_responsescrapegraph_ai.client.smartscraper.assert_called_once_with(website_url="https://example.com",user_prompt="Extract title and description",- website_html="<html>test</html>"+ website_html="<html>test</html>",)def test_scrape_failure(scrapegraph_ai):scrapegraph_ai.client.smartscraper.side_effect = Exception("Scrape failed")- with pytest.raises(RuntimeError, match="Failed to perform scrape: Scrape failed"):+ with pytest.raises(+ RuntimeError, match="Failed to perform scrape: Scrape failed"+ ):scrapegraph_ai.scrape(website_url="https://example.com",- user_prompt="Extract title and description"+ user_prompt="Extract title and description",)def test_close(scrapegraph_ai):scrapegraph_ai.close()- scrapegraph_ai.client.close.assert_called_once()\ No newline at end of file+ scrapegraph_ai.client.close.assert_called_once()diff --git a/uv.lock b/uv.lockindex 13ba459d7..f07c5182a 100644--- a/uv.lock+++ b/uv.lock@@ -639,6 +639,7 @@ all = [{ name = "rouge" },{ name = "scenedetect" },{ name = "scholarly", extra = ["tor"] },+ { name = "scrapegraph-py" },{ name = "sentence-transformers" },{ name = "sentencepiece" },{ name = "slack-bolt" },@@ -799,6 +800,7 @@ owl = [{ name = "requests-oauthlib" },{ name = "rouge" },{ name = "scenedetect" },+ { name = "scrapegraph-py" },{ name = "sentencepiece" },{ name = "soundfile" },{ name = "tabulate" },@@ -867,6 +869,7 @@ web-tools = [{ name = "playwright" },{ name = "pyowm" },{ name = "requests-oauthlib" },+ { name = "scrapegraph-py" },{ name = "sympy" },{ name = "tavily-python" },{ name = "wikipedia" },@@ -1100,6 +1103,9 @@ requires-dist = [{ name = "scenedetect", marker = "extra == 'owl'", specifier = ">=0.6.5.2" },{ name = "scholarly", extras = ["tor"], marker = "extra == 'all'", specifier = "==1.7.11" },{ name = "scholarly", extras = ["tor"], marker = "extra == 'research-tools'", specifier = "==1.7.11" },+ { name = "scrapegraph-py", marker = "extra == 'all'", specifier = ">=1.12.0,<2" },+ { name = "scrapegraph-py", marker = "extra == 'owl'", specifier = ">=1.12.0,<2" },+ { name = "scrapegraph-py", marker = "extra == 'web-tools'", specifier = ">=1.12.0,<2" },{ name = "sentence-transformers", marker = "extra == 'all'", specifier = ">=3.0.1,<4" },{ name = "sentence-transformers", marker = "extra == 'rag'", specifier = ">=3.0.1,<4" },{ name = "sentencepiece", marker = "extra == 'all'", specifier = ">=0.2,<0.3" },@@ -6284,6 +6290,22 @@ wheels = [{ url = "https://files.pythonhosted.org/packages/f5/6f/e6e5aff77ea2a48dd96808bb51d7450875af154ee7cbe72188afb0b37929/scipy-1.15.2-cp312-cp312-win_amd64.whl", hash = "sha256:e7c68b6a43259ba0aab737237876e5c2c549a031ddb7abc28c7b47f22e202ded", size = 40942317 },]+[[package]]+name = "scrapegraph-py"+version = "1.12.0"+source = { registry = "https://pypi.org/simple" }+dependencies = [+ { name = "aiohttp" },+ { name = "beautifulsoup4" },+ { name = "pydantic" },+ { name = "python-dotenv" },+ { name = "requests" },+]+sdist = { url = "https://files.pythonhosted.org/packages/42/36/10546d18157cd2efb2de09098fac8dd6f689b98842a4cc71bb7fc29ba4b9/scrapegraph_py-1.12.0.tar.gz", hash = "sha256:82d27e8ea325975f768f80d4edf403b6294518dae6a1e3ae63e27b8934a5dacb", size = 113290 }+wheels = [+ { url = "https://files.pythonhosted.org/packages/11/a8/8610143e9ebad9596e402260f63cbb6168f99719f07e13847b1df5a28f4d/scrapegraph_py-1.12.0-py3-none-any.whl", hash = "sha256:fd74d091529d3f8f5ba057950333e15a48ac5c0be7e2a56a8f2bad04cebdac30", size = 15458 },+]+[[package]]name = "selenium"version = "4.31.0"