Skip to content

armgit / camel

Public Imported from GitHub
0

🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org

enhance: add scrapegraph-sdk integration PR2206 (#2259)

Wendong-Fan1a2f3c4bdea1
diff --git a/camel/loaders/__init__.py b/camel/loaders/__init__.py
index 83963c33a..23cad7f98 100644
--- a/camel/loaders/__init__.py
+++ b/camel/loaders/__init__.py
@@ -20,6 +20,7 @@ from .firecrawl_reader import Firecrawl
from .jina_url_reader import JinaURLReader
from .mineru_extractor import MinerU
from .pandas_reader import PandasReader
+from .scrapegraph_reader import ScrapeGraphAI
from .unstructured_io import UnstructuredIO
__all__ = [
@@ -34,4 +35,5 @@ __all__ = [
'PandasReader',
'MinerU',
'Crawl4AI',
+ 'ScrapeGraphAI',
]
diff --git a/camel/loaders/scrapegraph_reader.py b/camel/loaders/scrapegraph_reader.py
index d787f9815..4cd19f191 100644
--- a/camel/loaders/scrapegraph_reader.py
+++ b/camel/loaders/scrapegraph_reader.py
@@ -1,4 +1,4 @@
-# ========= Copyright 2023-2025@ CAMEL-AI.org. All Rights Reserved. =========
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
@@ -10,20 +10,19 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ========= Copyright 2023-2025 @ CAMEL-AI.org. All Rights Reserved. =========
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
import os
from typing import Any, Dict, Optional
-from pydantic import BaseModel
-
class ScrapeGraphAI:
- r"""ScrapeGraphAI allows you to perform AI-powered web scraping and searching.
+ r"""ScrapeGraphAI allows you to perform AI-powered web scraping and
+ searching.
Args:
- api_key (Optional[str]): API key for authenticating with the ScrapeGraphAI
- API.
+ api_key (Optional[str]): API key for authenticating with the
+ ScrapeGraphAI API.
References:
https://scrapegraph.ai/
@@ -50,7 +49,8 @@ class ScrapeGraphAI:
user_prompt (str): The search query or instructions.
Returns:
- Dict[str, Any]: The search results including answer and reference URLs.
+ Dict[str, Any]: The search results including answer and reference
+ URLs.
Raises:
RuntimeError: If the search process fails.
@@ -72,8 +72,8 @@ class ScrapeGraphAI:
Args:
website_url (str): The URL to scrape.
user_prompt (str): Instructions for what data to extract.
- website_html (Optional[str]): Optional HTML content to use instead of
- fetching from the URL.
+ website_html (Optional[str]): Optional HTML content to use instead
+ of fetching from the URL.
Returns:
Dict[str, Any]: The scraped data including request ID and result.
diff --git a/examples/loaders/scrapegraph_example.py b/examples/loaders/scrapegraph_example.py
index 4991e662a..ccd441631 100644
--- a/examples/loaders/scrapegraph_example.py
+++ b/examples/loaders/scrapegraph_example.py
@@ -1,4 +1,4 @@
-# ========= Copyright 2023-2025@ CAMEL-AI.org. All Rights Reserved. =========
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
@@ -10,10 +10,11 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ========= Copyright 2023-2025 @ CAMEL-AI.org. All Rights Reserved. =========
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
"""
-Example demonstrating how to use the ScrapeGraphAI reader for web scraping and searching.
+Example demonstrating how to use the ScrapeGraphAI reader for web scraping and
+searching.
This example shows:
1. How to initialize the ScrapeGraphAI reader
@@ -23,27 +24,27 @@ This example shows:
"""
import os
-from typing import Dict, Any
+from typing import Any, Dict
from camel.loaders.scrapegraph_reader import ScrapeGraphAI
def search_example(api_key: str) -> Dict[str, Any]:
- """Example of performing an AI-powered web search."""
+ r"""Example of performing an AI-powered web search."""
# Initialize the ScrapeGraphAI reader
scraper = ScrapeGraphAI(api_key=api_key)
-
+
try:
# Perform a search
search_query = "What are the latest developments in AI?"
result = scraper.search(user_prompt=search_query)
-
+
print("\nSearch Results:")
print(f"Answer: {result.get('answer', 'No answer found')}")
print("References:")
for url in result.get('references', []):
print(f"- {url}")
-
+
return result
finally:
# Always close the connection
@@ -51,10 +52,10 @@ def search_example(api_key: str) -> Dict[str, Any]:
def scrape_example(api_key: str) -> Dict[str, Any]:
- """Example of scraping a website with specific instructions."""
+ r"""Example of scraping a website with specific instructions."""
# Initialize the ScrapeGraphAI reader
scraper = ScrapeGraphAI(api_key=api_key)
-
+
try:
# Scrape a website with specific instructions
website_url = "https://example.com"
@@ -64,17 +65,16 @@ def scrape_example(api_key: str) -> Dict[str, Any]:
2. All paragraph texts
3. Any links to other pages
"""
-
+
result = scraper.scrape(
- website_url=website_url,
- user_prompt=instructions
+ website_url=website_url, user_prompt=instructions
)
-
+
print("\nScraping Results:")
print(f"Request ID: {result.get('request_id', 'No ID')}")
print("Extracted Data:")
print(result.get('result', {}))
-
+
return result
finally:
# Always close the connection
@@ -84,17 +84,17 @@ def scrape_example(api_key: str) -> Dict[str, Any]:
def main():
# Get API key from environment variable or use a placeholder
api_key = os.environ.get("SCRAPEGRAPH_API_KEY", "your_api_key_here")
-
+
if api_key == "your_api_key_here":
print("Please set your SCRAPEGRAPH_API_KEY environment variable")
return
-
+
print("Running search example...")
search_example(api_key)
-
+
print("\nRunning scrape example...")
scrape_example(api_key)
if __name__ == "__main__":
- main()
\ No newline at end of file
+ main()
diff --git a/pyproject.toml b/pyproject.toml
index 5ca8078ec..1d900312b 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -240,6 +240,7 @@ owl = [
"pandas>=1.5.3,<2",
"rouge>=1.0.1,<2",
"crawl4ai>=0.3.745",
+ "scrapegraph-py>=1.12.0,<2",
]
all = [
"numpy~=1.26",
@@ -352,6 +353,7 @@ all = [
"crawl4ai>=0.3.745",
"pyautogui>=0.9.54,<0.10",
"pyobvector>=0.1.18",
+ "scrapegraph-py>=1.12.0,<2",
]
[project.urls]
@@ -538,6 +540,7 @@ module = [
"crawl4ai.*",
"pyautogui",
"pyobvector.*",
+ "scrapegraph_py.*",
]
ignore_missing_imports = true
diff --git a/test/loaders/test_scrapegraph_reader.py b/test/loaders/test_scrapegraph_reader.py
index 193585de8..88c89d299 100644
--- a/test/loaders/test_scrapegraph_reader.py
+++ b/test/loaders/test_scrapegraph_reader.py
@@ -1,4 +1,4 @@
-# ========= Copyright 2023-2025@ CAMEL-AI.org. All Rights Reserved. =========
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
@@ -10,7 +10,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ========= Copyright 2023-2025 @ CAMEL-AI.org. All Rights Reserved. =========
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
import os
from unittest.mock import MagicMock, patch
@@ -22,21 +22,23 @@ from camel.loaders.scrapegraph_reader import ScrapeGraphAI
@pytest.fixture
def scrapegraph_ai():
- with patch("camel.loaders.scrapegraph_reader.Client") as mock_client:
+ with patch("scrapegraph_py.Client") as mock_client:
mock_client_instance = MagicMock()
mock_client.return_value = mock_client_instance
yield ScrapeGraphAI(api_key="test_api_key")
def test_init_with_api_key():
- with patch("camel.loaders.scrapegraph_reader.Client") as mock_client:
+ with patch("scrapegraph_py.Client") as mock_client:
ScrapeGraphAI(api_key="test_api_key")
mock_client.assert_called_once_with(api_key="test_api_key")
def test_init_with_env_var():
- with patch("camel.loaders.scrapegraph_reader.Client") as mock_client, \
- patch.dict(os.environ, {"SCRAPEGRAPH_API_KEY": "env_api_key"}):
+ with (
+ patch("scrapegraph_py.Client") as mock_client,
+ patch.dict(os.environ, {"SCRAPEGRAPH_API_KEY": "env_api_key"}),
+ ):
ScrapeGraphAI()
mock_client.assert_called_once_with(api_key="env_api_key")
@@ -47,13 +49,19 @@ def test_search_success(scrapegraph_ai):
result = scrapegraph_ai.search("test query")
assert result == mock_response
- scrapegraph_ai.client.searchscraper.assert_called_once_with(user_prompt="test query")
+ scrapegraph_ai.client.searchscraper.assert_called_once_with(
+ user_prompt="test query"
+ )
def test_search_failure(scrapegraph_ai):
- scrapegraph_ai.client.searchscraper.side_effect = Exception("Search failed")
+ scrapegraph_ai.client.searchscraper.side_effect = Exception(
+ "Search failed"
+ )
- with pytest.raises(RuntimeError, match="Failed to perform search: Search failed"):
+ with pytest.raises(
+ RuntimeError, match="Failed to perform search: Search failed"
+ ):
scrapegraph_ai.search("test query")
@@ -63,13 +71,13 @@ def test_scrape_success(scrapegraph_ai):
result = scrapegraph_ai.scrape(
website_url="https://example.com",
- user_prompt="Extract title and description"
+ user_prompt="Extract title and description",
)
assert result == mock_response
scrapegraph_ai.client.smartscraper.assert_called_once_with(
website_url="https://example.com",
user_prompt="Extract title and description",
- website_html=None
+ website_html=None,
)
@@ -80,26 +88,28 @@ def test_scrape_with_html(scrapegraph_ai):
result = scrapegraph_ai.scrape(
website_url="https://example.com",
user_prompt="Extract title and description",
- website_html="<html>test</html>"
+ website_html="<html>test</html>",
)
assert result == mock_response
scrapegraph_ai.client.smartscraper.assert_called_once_with(
website_url="https://example.com",
user_prompt="Extract title and description",
- website_html="<html>test</html>"
+ website_html="<html>test</html>",
)
def test_scrape_failure(scrapegraph_ai):
scrapegraph_ai.client.smartscraper.side_effect = Exception("Scrape failed")
- with pytest.raises(RuntimeError, match="Failed to perform scrape: Scrape failed"):
+ with pytest.raises(
+ RuntimeError, match="Failed to perform scrape: Scrape failed"
+ ):
scrapegraph_ai.scrape(
website_url="https://example.com",
- user_prompt="Extract title and description"
+ user_prompt="Extract title and description",
)
def test_close(scrapegraph_ai):
scrapegraph_ai.close()
- scrapegraph_ai.client.close.assert_called_once()
\ No newline at end of file
+ scrapegraph_ai.client.close.assert_called_once()
diff --git a/uv.lock b/uv.lock
index 13ba459d7..f07c5182a 100644
--- a/uv.lock
+++ b/uv.lock
@@ -639,6 +639,7 @@ all = [
{ name = "rouge" },
{ name = "scenedetect" },
{ name = "scholarly", extra = ["tor"] },
+ { name = "scrapegraph-py" },
{ name = "sentence-transformers" },
{ name = "sentencepiece" },
{ name = "slack-bolt" },
@@ -799,6 +800,7 @@ owl = [
{ name = "requests-oauthlib" },
{ name = "rouge" },
{ name = "scenedetect" },
+ { name = "scrapegraph-py" },
{ name = "sentencepiece" },
{ name = "soundfile" },
{ name = "tabulate" },
@@ -867,6 +869,7 @@ web-tools = [
{ name = "playwright" },
{ name = "pyowm" },
{ name = "requests-oauthlib" },
+ { name = "scrapegraph-py" },
{ name = "sympy" },
{ name = "tavily-python" },
{ name = "wikipedia" },
@@ -1100,6 +1103,9 @@ requires-dist = [
{ name = "scenedetect", marker = "extra == 'owl'", specifier = ">=0.6.5.2" },
{ name = "scholarly", extras = ["tor"], marker = "extra == 'all'", specifier = "==1.7.11" },
{ name = "scholarly", extras = ["tor"], marker = "extra == 'research-tools'", specifier = "==1.7.11" },
+ { name = "scrapegraph-py", marker = "extra == 'all'", specifier = ">=1.12.0,<2" },
+ { name = "scrapegraph-py", marker = "extra == 'owl'", specifier = ">=1.12.0,<2" },
+ { name = "scrapegraph-py", marker = "extra == 'web-tools'", specifier = ">=1.12.0,<2" },
{ name = "sentence-transformers", marker = "extra == 'all'", specifier = ">=3.0.1,<4" },
{ name = "sentence-transformers", marker = "extra == 'rag'", specifier = ">=3.0.1,<4" },
{ name = "sentencepiece", marker = "extra == 'all'", specifier = ">=0.2,<0.3" },
@@ -6284,6 +6290,22 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/f5/6f/e6e5aff77ea2a48dd96808bb51d7450875af154ee7cbe72188afb0b37929/scipy-1.15.2-cp312-cp312-win_amd64.whl", hash = "sha256:e7c68b6a43259ba0aab737237876e5c2c549a031ddb7abc28c7b47f22e202ded", size = 40942317 },
]
+[[package]]
+name = "scrapegraph-py"
+version = "1.12.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "aiohttp" },
+ { name = "beautifulsoup4" },
+ { name = "pydantic" },
+ { name = "python-dotenv" },
+ { name = "requests" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/42/36/10546d18157cd2efb2de09098fac8dd6f689b98842a4cc71bb7fc29ba4b9/scrapegraph_py-1.12.0.tar.gz", hash = "sha256:82d27e8ea325975f768f80d4edf403b6294518dae6a1e3ae63e27b8934a5dacb", size = 113290 }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/11/a8/8610143e9ebad9596e402260f63cbb6168f99719f07e13847b1df5a28f4d/scrapegraph_py-1.12.0-py3-none-any.whl", hash = "sha256:fd74d091529d3f8f5ba057950333e15a48ac5c0be7e2a56a8f2bad04cebdac30", size = 15458 },
+]
+
[[package]]
name = "selenium"
version = "4.31.0"