Skip to content

armgit / camel

Public Imported from GitHub
0

🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org

feat: Integrate apify (#1123)

Asher-hss2c144f2a685c
diff --git a/camel/loaders/__init__.py b/camel/loaders/__init__.py
index 6e66dff25..73926bd91 100644
--- a/camel/loaders/__init__.py
+++ b/camel/loaders/__init__.py
@@ -12,6 +12,7 @@
# limitations under the License.
# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+from .apify_reader import Apify
from .base_io import File
from .chunkr_reader import ChunkrReader
from .firecrawl_reader import Firecrawl
@@ -23,5 +24,6 @@ __all__ = [
'UnstructuredIO',
'JinaURLReader',
'Firecrawl',
+ 'Apify',
'ChunkrReader',
]
diff --git a/camel/loaders/apify_reader.py b/camel/loaders/apify_reader.py
new file mode 100644
index 000000000..58cbde806
--- /dev/null
+++ b/camel/loaders/apify_reader.py
@@ -0,0 +1,223 @@
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+# Licensed under the Apache License, Version 2.0 (the ā€œLicenseā€);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an ā€œAS ISā€ BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+import os
+from typing import TYPE_CHECKING, List, Optional
+
+if TYPE_CHECKING:
+ from apify_client.clients import DatasetClient
+
+from camel.utils import api_keys_required
+
+
+class Apify:
+ r"""Apify is a platform that allows you to automate any web workflow.
+
+ Args:
+ api_key (Optional[str]): API key for authenticating with the Apify API.
+ """
+
+ @api_keys_required("APIFY_API_KEY")
+ def __init__(
+ self,
+ api_key: Optional[str] = None,
+ ) -> None:
+ from apify_client import ApifyClient
+
+ self._api_key = api_key or os.environ.get("APIFY_API_KEY")
+ self.client = ApifyClient(token=self._api_key)
+
+ def run_actor(
+ self,
+ actor_id: str,
+ run_input: Optional[dict] = None,
+ content_type: Optional[str] = None,
+ build: Optional[str] = None,
+ max_items: Optional[int] = None,
+ memory_mbytes: Optional[int] = None,
+ timeout_secs: Optional[int] = None,
+ webhooks: Optional[list] = None,
+ wait_secs: Optional[int] = None,
+ ) -> Optional[dict]:
+ r"""Run an actor on the Apify platform.
+
+ Args:
+ actor_id (str): The ID of the actor to run.
+ run_input (Optional[dict]): The input data for the actor. Defaults
+ to `None`.
+ content_type (str, optional): The content type of the input.
+ build (str, optional): Specifies the Actor build to run. It can be
+ either a build tag or build number. By default, the run uses
+ the build specified in the default run configuration for the
+ Actor (typically latest).
+ max_items (int, optional): Maximum number of results that will be
+ returned by this run. If the Actor is charged per result, you
+ will not be charged for more results than the given limit.
+ memory_mbytes (int, optional): Memory limit for the run, in
+ megabytes. By default, the run uses a memory limit specified in
+ the default run configuration for the Actor.
+ timeout_secs (int, optional): Optional timeout for the run, in
+ seconds. By default, the run uses timeout specified in the
+ default run configuration for the Actor.
+ webhooks (list, optional): Optional webhooks
+ (https://docs.apify.com/webhooks) associated with the Actor
+ run, which can be used to receive a notification, e.g. when the
+ Actor finished or failed. If you already have a webhook set up
+ for the Actor, you do not have to add it again here.
+ wait_secs (int, optional): The maximum number of seconds the server
+ waits for finish. If not provided, waits indefinitely.
+
+ Returns:
+ Optional[dict]: The output data from the actor if successful.
+ # please use the 'defaultDatasetId' to get the dataset
+
+ Raises:
+ RuntimeError: If the actor fails to run.
+ """
+ try:
+ return self.client.actor(actor_id).call(
+ run_input=run_input,
+ content_type=content_type,
+ build=build,
+ max_items=max_items,
+ memory_mbytes=memory_mbytes,
+ timeout_secs=timeout_secs,
+ webhooks=webhooks,
+ wait_secs=wait_secs,
+ )
+ except Exception as e:
+ raise RuntimeError(f"Failed to run actor {actor_id}: {e}") from e
+
+ def get_dataset_client(
+ self,
+ dataset_id: str,
+ ) -> "DatasetClient":
+ r"""Get a dataset client from the Apify platform.
+
+ Args:
+ dataset_id (str): The ID of the dataset to get the client for.
+
+ Returns:
+ DatasetClient: The dataset client.
+
+ Raises:
+ RuntimeError: If the dataset client fails to be retrieved.
+ """
+ try:
+ return self.client.dataset(dataset_id)
+ except Exception as e:
+ raise RuntimeError(
+ f"Failed to get dataset {dataset_id}: {e}"
+ ) from e
+
+ def get_dataset(
+ self,
+ dataset_id: str,
+ ) -> Optional[dict]:
+ r"""Get a dataset from the Apify platform.
+
+ Args:
+ dataset_id (str): The ID of the dataset to get.
+
+ Returns:
+ dict: The dataset.
+
+ Raises:
+ RuntimeError: If the dataset fails to be retrieved.
+ """
+ try:
+ return self.get_dataset_client(dataset_id).get()
+ except Exception as e:
+ raise RuntimeError(
+ f"Failed to get dataset {dataset_id}: {e}"
+ ) from e
+
+ def update_dataset(
+ self,
+ dataset_id: str,
+ name: str,
+ ) -> dict:
+ r"""Update a dataset on the Apify platform.
+
+ Args:
+ dataset_id (str): The ID of the dataset to update.
+ name (str): The new name for the dataset.
+
+ Returns:
+ dict: The updated dataset.
+
+ Raises:
+ RuntimeError: If the dataset fails to be updated.
+ """
+ try:
+ return self.get_dataset_client(dataset_id).update(name=name)
+ except Exception as e:
+ raise RuntimeError(
+ f"Failed to update dataset {dataset_id}: {e}"
+ ) from e
+
+ def get_dataset_items(
+ self,
+ dataset_id: str,
+ ) -> List:
+ r"""Get items from a dataset on the Apify platform.
+
+ Args:
+ dataset_id (str): The ID of the dataset to get items from.
+
+ Returns:
+ list: The items in the dataset.
+
+ Raises:
+ RuntimeError: If the items fail to be retrieved.
+ """
+ try:
+ items = self.get_dataset_client(dataset_id).list_items().items
+ return items
+ except Exception as e:
+ raise RuntimeError(
+ f"Failed to get dataset items {dataset_id}: {e}"
+ ) from e
+
+ def get_datasets(
+ self,
+ unnamed: Optional[bool] = None,
+ limit: Optional[int] = None,
+ offset: Optional[int] = None,
+ desc: Optional[bool] = None,
+ ) -> List[dict]:
+ r"""Get all named datasets from the Apify platform.
+
+ Args:
+ unnamed (bool, optional): Whether to include unnamed key-value
+ stores in the list
+ limit (int, optional): How many key-value stores to retrieve
+ offset (int, optional): What key-value store to include as first
+ when retrieving the list
+ desc (bool, optional): Whether to sort the key-value stores in
+ descending order based on their modification date
+
+ Returns:
+ List[dict]: The datasets.
+
+ Raises:
+ RuntimeError: If the datasets fail to be retrieved.
+ """
+ try:
+ return (
+ self.client.datasets()
+ .list(unnamed=unnamed, limit=limit, offset=offset, desc=desc)
+ .items
+ )
+ except Exception as e:
+ raise RuntimeError(f"Failed to get datasets: {e}") from e
diff --git a/examples/loaders/apify_example.py b/examples/loaders/apify_example.py
new file mode 100644
index 000000000..834c495ee
--- /dev/null
+++ b/examples/loaders/apify_example.py
@@ -0,0 +1,82 @@
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+# Licensed under the Apache License, Version 2.0 (the ā€œLicenseā€);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an ā€œAS ISā€ BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+
+from camel.loaders import Apify
+
+apify = Apify()
+
+run_input = {
+ "startUrls": [{"url": "https://www.camel-ai.org/"}],
+ "maxCrawlDepth": 0,
+ "maxCrawlPages": 1,
+}
+actor_result = apify.run_actor(
+ actor_id="apify/website-content-crawler", run_input=run_input
+)
+
+print(actor_result["status"])
+'''
+===============================================================================
+SUCCEEDED
+===============================================================================
+'''
+
+print(actor_result["defaultDatasetId"])
+'''
+===============================================================================
+HoKb31FJzHm0ni51k
+===============================================================================
+'''
+
+dataset_result = apify.get_dataset_items(
+ dataset_id=actor_result["defaultDatasetId"]
+)
+
+print(dataset_result)
+'''
+===============================================================================
+[{'url': 'https://www.camel-ai.org/', 'crawl': {'loadedUrl': 'https://www.camel
+-ai.org/', 'loadedTime': '2024-10-27T04:51:16.651Z', 'referrerUrl': 'https://ww
+w.camel-ai.org/', 'depth': 0, 'httpStatusCode': 200}, 'metadata': {'canonicalUr
+l': 'https://www.camel-ai.org/', 'title': 'CAMEL-AI', 'description': 'CAMEL-AI.
+org is the 1st LLM multi-agent framework and an open-source community dedicated
+ to finding the scaling law of agents.', 'author': None, 'keywords': None, 'lan
+ guageCode': 'en', 'openGraph': [{'property': 'og:title', 'content': 'CAMEL-AI'
+ }, {'property': 'og:description', 'content': 'CAMEL-AI.org is the 1st LLM mult
+ i-agent framework and an open-source community dedicated to finding the scalin
+ g law of agents.'}, {'property': 'twitter:title', 'content': 'CAMEL-AI'}, {'pr
+ operty': 'twitter:description', 'content': 'CAMEL-AI.org is the 1st LLM multi-
+ agent framework and an open-source community dedicated to finding the scaling
+ law of agents.'}, {'property': 'og:type', 'content': 'website'}], 'jsonLd': No
+ ne, 'headers': {'date': 'Sun, 27 Oct 2024 04:50:18 GMT', 'content-type': 'text
+ /html', 'cf-ray': '8d901082dae7efbe-PDX', 'cf-cache-status': 'HIT', 'age': '10
+ 81', 'content-encoding': 'gzip', 'last-modified': 'Sat, 26 Oct 2024 11:51:32 G
+ MT', 'strict-transport-security': 'max-age=31536000', 'surrogate-control': 'ma
+ x-age=432000', 'surrogate-key': 'www.camel-ai.org 6659a154491a54a40551bc78 pag
+ eId:6686a2bcb7ece5fb40457491 668181a0a818ade34e653b24 6659a155491a54a40551bd7e
+ ', 'x-lambda-id': 'd6c4424b-ac67-4c54-b52a-cb2a22ca09f0', 'vary': 'Accept-Enco
+ ding', 'set-cookie': '__cf_bm=oX5EmZ2SNJDOBQXI8dL_reCYlCpp1FMzu40qCNxiopU-1730
+ 004618-1.0.1.1-3teEeqUoemzHWAeGCtlPJVqdmAbiFkyu3JxopKfQFFndSCi_Z56RR.UDjLGZiq.
+ L_4LvAZYmNKxQ.k6VRhbA7g; path=/; expires=Sun, 27-Oct-24 05:20:18 GMT; domain=.
+ cdn.webflow.com; HttpOnly; Secure; SameSite=None\n_cfuvid=om_8lj9jNMIh.HEIxEAq
+ gszhEWaKlyS2kdXKwqGedSM-1730004618924-0.0.1.1-604800000; path=/; domain=.cdn.w
+ ebflow.com; HttpOnly; Secure; SameSite=None', 'alt-svc': 'h3=":443"; ma=86400'
+ , 'x-cluster-name': 'us-west-2-prod-hosting-red', 'x-firefox-spdy': 'h2'}}, 's
+ creenshotUrl': None, 'text': 'Build Multi-Agent Systems for _\nFEATURES & Inte
+ grations\nSeamless integrations with\npopular platforms \nScroll to explore ou
+ r features & integrations.', 'markdown': '# Build Multi-Agent Systems for \\_
+ \n\nFEATURES & Integrations\n\n## Seamless integrations with \npopular platfo
+ rms\n\nScroll to explore our features & integrations.'}]
+===============================================================================
+'''
diff --git a/poetry.lock b/poetry.lock
index cdeb1cf23..a6fb8a1e8 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -1,4 +1,4 @@
-# This file is automatically @generated by Poetry 1.8.4 and should not be changed by hand.
+# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand.
[[package]]
name = "accelerate"
@@ -321,6 +321,36 @@ doc = ["Sphinx (>=7.4,<8.0)", "packaging", "sphinx-autodoc-typehints (>=1.2.0)",
test = ["anyio[trio]", "coverage[toml] (>=7)", "exceptiongroup (>=1.2.0)", "hypothesis (>=4.0)", "psutil (>=5.9)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "truststore (>=0.9.1)", "uvloop (>=0.21.0b1)"]
trio = ["trio (>=0.26.1)"]
+[[package]]
+name = "apify-client"
+version = "1.8.1"
+description = "Apify API client for Python"
+optional = true
+python-versions = "<4.0,>=3.9"
+files = [
+ {file = "apify_client-1.8.1-py3-none-any.whl", hash = "sha256:cfa6df3816c436204e37457fba28981a0ef6a7602cde372463f0f078eee64747"},
+ {file = "apify_client-1.8.1.tar.gz", hash = "sha256:2be1be7879570655bddeebf126833efe94cabb95b3755592845e92c20c70c674"},
+]
+
+[package.dependencies]
+apify-shared = ">=1.1.2"
+httpx = ">=0.25.0"
+more_itertools = ">=10.0.0"
+
+[[package]]
+name = "apify-shared"
+version = "1.1.2"
+description = "Tools and constants shared across Apify projects."
+optional = true
+python-versions = ">=3.8"
+files = [
+ {file = "apify_shared-1.1.2-py3-none-any.whl", hash = "sha256:a422b4cb3ea536257f0c0dd3778b3c22ec6647708d2fc06c97fe0c9fe183c3c9"},
+ {file = "apify_shared-1.1.2.tar.gz", hash = "sha256:3f98f526635c4c3f92c1496916513a1c4013bb47158d7e0f914df8987147366c"},
+]
+
+[package.extras]
+dev = ["build (>=1.0.3,<1.1.0)", "filelock (>=3.12.4,<3.13.0)", "mypy (>=1.7.1,<1.8.0)", "pre-commit (>=3.4.0,<3.5.0)", "pydoc-markdown (>=4.8.2,<4.9.0)", "pytest (>=7.4.2,<7.5.0)", "pytest-asyncio (>=0.21.0,<0.22.0)", "pytest-cov (>=4.1.0,<4.2.0)", "pytest-only (>=2.0.0,<2.1.0)", "pytest-timeout (>=2.2.0,<2.3.0)", "pytest-xdist (>=3.3.1,<3.4.0)", "respx (>=0.20.1,<0.21.0)", "ruff (>=0.1.13,<0.2.0)", "setuptools (>=68.0.0)", "twine (>=5.1.1,<5.2.0)"]
+
[[package]]
name = "appnope"
version = "0.1.4"
@@ -650,13 +680,13 @@ css = ["tinycss2 (>=1.1.0,<1.3)"]
[[package]]
name = "botocore"
-version = "1.35.48"
+version = "1.35.49"
description = "Low-level, data-driven core of boto 3."
optional = true
python-versions = ">=3.8"
files = [
- {file = "botocore-1.35.48-py3-none-any.whl", hash = "sha256:34fa25fd717208b05745e60f271a39636108fa87a3512fbca18e7e6f787a3239"},
- {file = "botocore-1.35.48.tar.gz", hash = "sha256:3e766cc251053c9ef98542fdf225381ed58531769c3811a6282bd7247f7e2bdf"},
+ {file = "botocore-1.35.49-py3-none-any.whl", hash = "sha256:aed4d3643afd702920792b68fbe712a8c3847993820d1048cd238a6469354da1"},
+ {file = "botocore-1.35.49.tar.gz", hash = "sha256:07d0c1325fdbfa49a4a054413dbdeab0a6030449b2aa66099241af2dac48afd8"},
]
[package.dependencies]
@@ -1593,13 +1623,13 @@ tests = ["dj-database-url", "dj-email-url", "django-cache-url", "pytest"]
[[package]]
name = "et-xmlfile"
-version = "1.1.0"
+version = "2.0.0"
description = "An implementation of lxml.xmlfile for the standard library"
optional = true
-python-versions = ">=3.6"
+python-versions = ">=3.8"
files = [
- {file = "et_xmlfile-1.1.0-py3-none-any.whl", hash = "sha256:a2ba85d1d6a74ef63837eed693bcb89c3f752169b0e3e7ae5b16ca5e1b3deada"},
- {file = "et_xmlfile-1.1.0.tar.gz", hash = "sha256:8eb9e2bc2f8c97e37a2dc85a09ecdcdec9d8a396530a6d5a33b30b9a92da0c5c"},
+ {file = "et_xmlfile-2.0.0-py3-none-any.whl", hash = "sha256:7a91720bc756843502c3b7504c77b8fe44217c85c537d85037f0f536151b2caa"},
+ {file = "et_xmlfile-2.0.0.tar.gz", hash = "sha256:dab3f4764309081ce75662649be815c4c9081e88f0837825f90fd28317d4da54"},
]
[[package]]
@@ -3448,13 +3478,13 @@ files = [
[[package]]
name = "litellm"
-version = "1.50.4"
+version = "1.51.0"
description = "Library to easily interface with LLM API providers"
optional = true
python-versions = "!=2.7.*,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,!=3.7.*,>=3.8"
files = [
- {file = "litellm-1.50.4-py3-none-any.whl", hash = "sha256:cc6992275e24a0bbb4a3b377e6842d45a8510fc85d7f255930a64bb872980a36"},
- {file = "litellm-1.50.4.tar.gz", hash = "sha256:a7e68ef614f631b58969c2c7a5154a565ba5974558d437c8cd6c8623654880ea"},
+ {file = "litellm-1.51.0-py3-none-any.whl", hash = "sha256:0b2c20d116834166c8440e5698d7d927dbcc78fcaa08ce0c5cbea2d0de55ec6c"},
+ {file = "litellm-1.51.0.tar.gz", hash = "sha256:8bf648677ee145a8fe5054a2e3f3a34895b9ab65a6015e4b94efca7ef406f466"},
]
[package.dependencies]
@@ -3871,6 +3901,7 @@ python-versions = ">=3.7"
files = [
{file = "milvus_lite-2.4.10-py3-none-macosx_10_9_x86_64.whl", hash = "sha256:fc4246d3ed7d1910847afce0c9ba18212e93a6e9b8406048436940578dfad5cb"},
{file = "milvus_lite-2.4.10-py3-none-macosx_11_0_arm64.whl", hash = "sha256:74a8e07c5e3b057df17fbb46913388e84df1dc403a200f4e423799a58184c800"},
+ {file = "milvus_lite-2.4.10-py3-none-manylinux2014_aarch64.whl", hash = "sha256:240c7386b747bad696ecb5bd1f58d491e86b9d4b92dccee3315ed7256256eddc"},
{file = "milvus_lite-2.4.10-py3-none-manylinux2014_x86_64.whl", hash = "sha256:211d2e334a043f9282bdd9755f76b9b2d93b23bffa7af240919ffce6a8dfe325"},
]
@@ -5496,17 +5527,17 @@ ssv = ["swagger-spec-validator (>=2.4,<3.0)"]
[[package]]
name = "praw"
-version = "7.8.0"
+version = "7.8.1"
description = "Python Reddit API Wrapper."
optional = true
python-versions = "~=3.8"
files = [
- {file = "praw-7.8.0-py3-none-any.whl", hash = "sha256:66bb2d9dc4f9a35910bf8b4339067c8646ab03270b1c38d77b0487bef8c5c3e2"},
- {file = "praw-7.8.0.tar.gz", hash = "sha256:910a1478d79ce3892bf35b2c52c90f575d6618dfce248e7bf90eed354c19589c"},
+ {file = "praw-7.8.1-py3-none-any.whl", hash = "sha256:15917a81a06e20ff0aaaf1358481f4588449fa2421233040cb25e5c8202a3e2f"},
+ {file = "praw-7.8.1.tar.gz", hash = "sha256:3c5767909f71e48853eb6335fef7b50a43cbe3da728cdfb16d3be92904b0a4d8"},
]
[package.dependencies]
-prawcore = ">=2.1,<3"
+prawcore = ">=2.4,<3"
update_checker = ">=0.18"
websocket-client = ">=0.54.0"
@@ -5844,6 +5875,7 @@ description = "Pure-Python implementation of ASN.1 types and DER/BER/CER codecs
optional = true
python-versions = ">=3.8"
files = [
+ {file = "pyasn1-0.6.1-py3-none-any.whl", hash = "sha256:0d632f46f2ba09143da3a8afe9e33fb6f92fa2320ab7e886e2d0f7672af84629"},
{file = "pyasn1-0.6.1.tar.gz", hash = "sha256:6f580d2bdd84365380830acf45550f2511469f673cb4a5ae3857a3170128b034"},
]
@@ -5854,6 +5886,7 @@ description = "A collection of ASN.1-based protocols modules"
optional = true
python-versions = ">=3.8"
files = [
+ {file = "pyasn1_modules-0.4.1-py3-none-any.whl", hash = "sha256:49bfa96b45a292b711e986f222502c1c9a5e1f4e568fc30e2574a6c7d07838fd"},
{file = "pyasn1_modules-0.4.1.tar.gz", hash = "sha256:c28e2dbf9c06ad61c71a075c7e0f9fd0f1b0bb2d2ad4377f240d33ac2ab60a7c"},
]
@@ -8585,6 +8618,10 @@ description = "image and video datasets and models for torch deep learning"
optional = true
python-versions = ">=3.8"
files = [
+ {file = "torchvision-0.20.0-1-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:e084f50ecbdbe7a9cc2fc51ea0367ae35fde46e84a964bf4046cb1c7feb7e3e6"},
+ {file = "torchvision-0.20.0-1-cp311-cp311-manylinux2014_aarch64.whl", hash = "sha256:55d7f43ef912ebc4da4bba73a0bbf387d38a6be9cd521679c0f4056f9564b698"},
+ {file = "torchvision-0.20.0-1-cp312-cp312-manylinux2014_aarch64.whl", hash = "sha256:f8d0213489acfb138369f2455a6893880c194a8195e381c19f872b277f2654c3"},
+ {file = "torchvision-0.20.0-1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:8d6cea8ab0bf72ecb71b07cd0fe836eacf5a5fa98f6629d2261212e90977b963"},
{file = "torchvision-0.20.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f164d545965186ffd66014e34a966706d12c84198302dd46748cae45984609a4"},
{file = "torchvision-0.20.0-cp310-cp310-manylinux1_x86_64.whl", hash = "sha256:9c18208575d60b96e7d53a09c453781afea4a81487c9ebc501dfc2bc88daa308"},
{file = "torchvision-0.20.0-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:09080359be90314fc4fdd64b11a4d231c1999018f19d58bf7764f5e15f8e9fb3"},
@@ -8597,10 +8634,6 @@ files = [
{file = "torchvision-0.20.0-cp312-cp312-manylinux1_x86_64.whl", hash = "sha256:c8f3bc399d9c3e4ba05d74ca6dd5e63fed08ad5c5b302a946c8fcaa56216220f"},
{file = "torchvision-0.20.0-cp312-cp312-manylinux2014_aarch64.whl", hash = "sha256:a78c99ebe1a62857b68e97ff9417b92f299f2ee61f009491a114ddad050c493d"},
{file = "torchvision-0.20.0-cp312-cp312-win_amd64.whl", hash = "sha256:bb0da0950d2034a0412c251a3a9117ff9612157f45177d37ba1b20b472c0864b"},
- {file = "torchvision-0.20.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:eac0839db36eb812362f06359a33e8c7101f161c1393e2971b8402aefefa5b58"},
- {file = "torchvision-0.20.0-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:bbb6c3287e15ab710139d63912b6f1bca6a34a6912cdb5c98fdb850b260ec20b"},
- {file = "torchvision-0.20.0-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:cd20f972d594a7900b57724e168ee8636e3cb255f54f6a9d8409e3e218b93721"},
- {file = "torchvision-0.20.0-cp38-cp38-win_amd64.whl", hash = "sha256:1dd590a48d0a68a3be598d0d0a41e72872dd317418753aa0c97335d50558ba8a"},
{file = "torchvision-0.20.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:6a70c81ea5068dd7b1e340ebeabb65364576d8b9819454cfdf812290cf03e45a"},
{file = "torchvision-0.20.0-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:95d8c817681a4c2156f66ef83cafc4c5c4b97e4694956d54d7dc554804ee510d"},
{file = "torchvision-0.20.0-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:1ab53244701eab897e5c65026ba178c0abbc5bd08629c3d20f737d618e9e5a37"},
@@ -9841,7 +9874,7 @@ test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools",
type = ["pytest-mypy"]
[extras]
-all = ["PyMuPDF", "accelerate", "agentops", "anthropic", "arxiv", "arxiv2text", "asknews", "azure-storage-blob", "beautifulsoup4", "botocore", "cohere", "datasets", "diffusers", "discord.py", "docker", "docx2txt", "duckduckgo-search", "firecrawl-py", "google-cloud-storage", "google-generativeai", "googlemaps", "imageio", "ipykernel", "jupyter_client", "litellm", "mistralai", "nebula3-python", "neo4j", "newspaper3k", "nltk", "notion-client", "openapi-spec-validator", "opencv-python", "pdfplumber", "pillow", "prance", "praw", "pyTelegramBotAPI", "pydub", "pygithub", "pymilvus", "pyowm", "qdrant-client", "rank-bm25", "redis", "reka-api", "requests_oauthlib", "scholarly", "sentence-transformers", "sentencepiece", "slack-bolt", "slack-sdk", "soundfile", "textblob", "torch", "torch", "transformers", "unstructured", "wikipedia", "wolframalpha"]
+all = ["PyMuPDF", "accelerate", "agentops", "anthropic", "apify_client", "arxiv", "arxiv2text", "asknews", "azure-storage-blob", "beautifulsoup4", "botocore", "cohere", "datasets", "diffusers", "discord.py", "docker", "docx2txt", "duckduckgo-search", "firecrawl-py", "google-cloud-storage", "google-generativeai", "googlemaps", "imageio", "ipykernel", "jupyter_client", "litellm", "mistralai", "nebula3-python", "neo4j", "newspaper3k", "nltk", "notion-client", "openapi-spec-validator", "opencv-python", "pdfplumber", "pillow", "prance", "praw", "pyTelegramBotAPI", "pydub", "pygithub", "pymilvus", "pyowm", "qdrant-client", "rank-bm25", "redis", "reka-api", "requests_oauthlib", "scholarly", "sentence-transformers", "sentencepiece", "slack-bolt", "slack-sdk", "soundfile", "textblob", "torch", "torch", "transformers", "unstructured", "wikipedia", "wolframalpha"]
encoders = ["sentence-transformers"]
graph-storages = ["nebula3-python", "neo4j"]
huggingface-agent = ["accelerate", "datasets", "diffusers", "opencv-python", "sentencepiece", "soundfile", "torch", "torch", "transformers"]
@@ -9852,10 +9885,10 @@ rag = ["cohere", "nebula3-python", "neo4j", "pymilvus", "qdrant-client", "rank-b
retrievers = ["cohere", "rank-bm25"]
search-tools = ["duckduckgo-search", "wikipedia", "wolframalpha"]
test = ["mock", "pytest", "pytest-asyncio"]
-tools = ["PyMuPDF", "agentops", "arxiv", "arxiv2text", "asknews", "beautifulsoup4", "discord.py", "docker", "docx2txt", "duckduckgo-search", "firecrawl-py", "googlemaps", "imageio", "ipykernel", "jupyter_client", "newspaper3k", "nltk", "notion-client", "openapi-spec-validator", "pdfplumber", "pillow", "prance", "praw", "pyTelegramBotAPI", "pydub", "pygithub", "pyowm", "requests_oauthlib", "scholarly", "slack-bolt", "slack-sdk", "textblob", "unstructured", "wikipedia", "wolframalpha"]
+tools = ["PyMuPDF", "agentops", "apify_client", "arxiv", "arxiv2text", "asknews", "beautifulsoup4", "discord.py", "docker", "docx2txt", "duckduckgo-search", "firecrawl-py", "googlemaps", "imageio", "ipykernel", "jupyter_client", "newspaper3k", "nltk", "notion-client", "openapi-spec-validator", "pdfplumber", "pillow", "prance", "praw", "pyTelegramBotAPI", "pydub", "pygithub", "pyowm", "requests_oauthlib", "scholarly", "slack-bolt", "slack-sdk", "textblob", "unstructured", "wikipedia", "wolframalpha"]
vector-databases = ["pymilvus", "qdrant-client"]
[metadata]
lock-version = "2.0"
python-versions = ">=3.10,<3.13"
-content-hash = "b4cec55ebd8227386e1658ef0d0c204cd171c20b6907e4d5973fb180b210e55e"
+content-hash = "4e31fcb53ef5ee0000d3182536cda5930d54f6287c240889b149cab73800a6de"
diff --git a/pyproject.toml b/pyproject.toml
index 4f5ac3f13..fd55c47a1 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -100,6 +100,7 @@ arxiv2text = { version = "^0.1.14", optional = true }
jupyter_client = { version = "^8.6.2", optional = true }
ipykernel = { version = "^6.0.0", optional = true }
pdfplumber = { version = "^0.11.0", optional = true }
+apify_client = { version = "^1.8.1", optional = true }
notion-client = { version = "^2.2.1", optional = true }
@@ -204,6 +205,7 @@ tools = [
"textblob",
"scholarly",
"pdfplumber",
+ "apify_client",
"notion-client",
]
@@ -302,6 +304,7 @@ all = [
"azure-storage-blob",
"google-cloud-storage",
"botocore",
+ "apify_client",
]
[tool.poetry.group.dev]
diff --git a/test/loaders/test_apify.py b/test/loaders/test_apify.py
new file mode 100644
index 000000000..280117adf
--- /dev/null
+++ b/test/loaders/test_apify.py
@@ -0,0 +1,126 @@
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+# Licensed under the Apache License, Version 2.0 (the ā€œLicenseā€);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an ā€œAS ISā€ BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+import os
+from unittest.mock import Mock, patch
+
+import pytest
+
+from camel.loaders.apify_reader import Apify
+
+
+def apify_client_mock():
+ with patch('apify_client.ApifyClient', autospec=True) as mock:
+ yield mock
+
+
[email protected](autouse=True)
+def set_env_vars():
+ with patch.dict(os.environ, {"APIFY_API_KEY": "test_api_key"}):
+ yield
+
+
+def test_init(apify_client_mock):
+ api_key = "test_api_key"
+ apify = Apify(api_key=api_key)
+ assert apify._api_key == api_key
+ apify_client_mock.assert_called_once_with(token=api_key)
+
+
+def test_run_actor():
+ api_key = "test_api_key"
+ apify = Apify(api_key=api_key)
+ actor_id = "test_actor_id"
+ run_input = {"key": "value"}
+ mock_actor = Mock()
+ mock_actor.call.return_value = {"result": "success"}
+ apify.client.actor = Mock(return_value=mock_actor)
+ result = apify.run_actor(actor_id, run_input=run_input)
+ apify.client.actor.assert_called_once_with(actor_id)
+ apify.client.actor.return_value.call.assert_called_once_with(
+ run_input=run_input,
+ content_type=None,
+ build=None,
+ max_items=None,
+ memory_mbytes=None,
+ timeout_secs=None,
+ webhooks=None,
+ wait_secs=None,
+ )
+ assert result == {"result": "success"}
+
+
+def test_get_dataset_client():
+ api_key = "test_api_key"
+ apify = Apify(api_key=api_key)
+ dataset_id = "test_dataset_id"
+ mock_dataset = Mock()
+ apify.client.dataset = Mock(return_value=mock_dataset)
+ result = apify.get_dataset_client(dataset_id)
+ apify.client.dataset.assert_called_once_with(dataset_id)
+ assert result == mock_dataset
+
+
+def test_get_dataset():
+ api_key = "test_api_key"
+ apify = Apify(api_key=api_key)
+ dataset_id = "test_dataset_id"
+ mock_client = Mock()
+ mock_client.get.return_value = {"data": "dataset"}
+ apify.get_dataset_client = lambda x: mock_client
+
+ result = apify.get_dataset(dataset_id)
+ assert result == {"data": "dataset"}
+
+
+def test_update_dataset():
+ api_key = "test_api_key"
+ apify = Apify(api_key=api_key)
+ dataset_id = "test_dataset_id"
+ name = "new_name"
+ mock_client = Mock()
+ mock_client.update.return_value = {"name": name}
+ apify.get_dataset_client = lambda x: mock_client
+
+ result = apify.update_dataset(dataset_id, name)
+ assert result == {"name": name}
+
+
+def test_get_dataset_items():
+ api_key = "test_api_key"
+ apify = Apify(api_key=api_key)
+ dataset_id = "test_dataset_id"
+ mock_client = Mock()
+ mock_client.list_items.return_value.items = ["item1", "item2"]
+ apify.get_dataset_client = lambda x: mock_client
+
+ result = apify.get_dataset_items(dataset_id)
+ assert result == ["item1", "item2"]
+
+
+def test_get_datasets():
+ api_key = "test_api_key"
+ apify = Apify(api_key=api_key)
+ mock_client = Mock()
+ mock_client.datasets.return_value.list.return_value.items = [
+ {"name": "dataset1"},
+ {"name": "dataset2"},
+ ]
+ apify.client = mock_client
+
+ result = apify.get_datasets()
+ apify.client.datasets.return_value.list.assert_called_once_with(
+ unnamed=None, limit=None, offset=None, desc=None
+ )
+ assert result == [{"name": "dataset1"}, {"name": "dataset2"}]