š« CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org
Python agentai-societiesartificial-intelligencecommunicative-aicooperative-aideep-learninglarge-language-modelsmulti-agent-systemsnatural-language-processing
feat: Integrate apify (#1123)
Asher-hss2c144f2a685c
diff --git a/camel/loaders/__init__.py b/camel/loaders/__init__.pyindex 6e66dff25..73926bd91 100644--- a/camel/loaders/__init__.py+++ b/camel/loaders/__init__.py@@ -12,6 +12,7 @@# limitations under the License.# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========+from .apify_reader import Apifyfrom .base_io import Filefrom .chunkr_reader import ChunkrReaderfrom .firecrawl_reader import Firecrawl@@ -23,5 +24,6 @@ __all__ = ['UnstructuredIO','JinaURLReader','Firecrawl',+ 'Apify','ChunkrReader',]diff --git a/camel/loaders/apify_reader.py b/camel/loaders/apify_reader.pynew file mode 100644index 000000000..58cbde806--- /dev/null+++ b/camel/loaders/apify_reader.py@@ -0,0 +1,223 @@+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========+# Licensed under the Apache License, Version 2.0 (the āLicenseā);+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an āAS ISā BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========+import os+from typing import TYPE_CHECKING, List, Optional++if TYPE_CHECKING:+ from apify_client.clients import DatasetClient++from camel.utils import api_keys_required+++class Apify:+ r"""Apify is a platform that allows you to automate any web workflow.++ Args:+ api_key (Optional[str]): API key for authenticating with the Apify API.+ """++ @api_keys_required("APIFY_API_KEY")+ def __init__(+ self,+ api_key: Optional[str] = None,+ ) -> None:+ from apify_client import ApifyClient++ self._api_key = api_key or os.environ.get("APIFY_API_KEY")+ self.client = ApifyClient(token=self._api_key)++ def run_actor(+ self,+ actor_id: str,+ run_input: Optional[dict] = None,+ content_type: Optional[str] = None,+ build: Optional[str] = None,+ max_items: Optional[int] = None,+ memory_mbytes: Optional[int] = None,+ timeout_secs: Optional[int] = None,+ webhooks: Optional[list] = None,+ wait_secs: Optional[int] = None,+ ) -> Optional[dict]:+ r"""Run an actor on the Apify platform.++ Args:+ actor_id (str): The ID of the actor to run.+ run_input (Optional[dict]): The input data for the actor. Defaults+ to `None`.+ content_type (str, optional): The content type of the input.+ build (str, optional): Specifies the Actor build to run. It can be+ either a build tag or build number. By default, the run uses+ the build specified in the default run configuration for the+ Actor (typically latest).+ max_items (int, optional): Maximum number of results that will be+ returned by this run. If the Actor is charged per result, you+ will not be charged for more results than the given limit.+ memory_mbytes (int, optional): Memory limit for the run, in+ megabytes. By default, the run uses a memory limit specified in+ the default run configuration for the Actor.+ timeout_secs (int, optional): Optional timeout for the run, in+ seconds. By default, the run uses timeout specified in the+ default run configuration for the Actor.+ webhooks (list, optional): Optional webhooks+ (https://docs.apify.com/webhooks) associated with the Actor+ run, which can be used to receive a notification, e.g. when the+ Actor finished or failed. If you already have a webhook set up+ for the Actor, you do not have to add it again here.+ wait_secs (int, optional): The maximum number of seconds the server+ waits for finish. If not provided, waits indefinitely.++ Returns:+ Optional[dict]: The output data from the actor if successful.+ # please use the 'defaultDatasetId' to get the dataset++ Raises:+ RuntimeError: If the actor fails to run.+ """+ try:+ return self.client.actor(actor_id).call(+ run_input=run_input,+ content_type=content_type,+ build=build,+ max_items=max_items,+ memory_mbytes=memory_mbytes,+ timeout_secs=timeout_secs,+ webhooks=webhooks,+ wait_secs=wait_secs,+ )+ except Exception as e:+ raise RuntimeError(f"Failed to run actor {actor_id}: {e}") from e++ def get_dataset_client(+ self,+ dataset_id: str,+ ) -> "DatasetClient":+ r"""Get a dataset client from the Apify platform.++ Args:+ dataset_id (str): The ID of the dataset to get the client for.++ Returns:+ DatasetClient: The dataset client.++ Raises:+ RuntimeError: If the dataset client fails to be retrieved.+ """+ try:+ return self.client.dataset(dataset_id)+ except Exception as e:+ raise RuntimeError(+ f"Failed to get dataset {dataset_id}: {e}"+ ) from e++ def get_dataset(+ self,+ dataset_id: str,+ ) -> Optional[dict]:+ r"""Get a dataset from the Apify platform.++ Args:+ dataset_id (str): The ID of the dataset to get.++ Returns:+ dict: The dataset.++ Raises:+ RuntimeError: If the dataset fails to be retrieved.+ """+ try:+ return self.get_dataset_client(dataset_id).get()+ except Exception as e:+ raise RuntimeError(+ f"Failed to get dataset {dataset_id}: {e}"+ ) from e++ def update_dataset(+ self,+ dataset_id: str,+ name: str,+ ) -> dict:+ r"""Update a dataset on the Apify platform.++ Args:+ dataset_id (str): The ID of the dataset to update.+ name (str): The new name for the dataset.++ Returns:+ dict: The updated dataset.++ Raises:+ RuntimeError: If the dataset fails to be updated.+ """+ try:+ return self.get_dataset_client(dataset_id).update(name=name)+ except Exception as e:+ raise RuntimeError(+ f"Failed to update dataset {dataset_id}: {e}"+ ) from e++ def get_dataset_items(+ self,+ dataset_id: str,+ ) -> List:+ r"""Get items from a dataset on the Apify platform.++ Args:+ dataset_id (str): The ID of the dataset to get items from.++ Returns:+ list: The items in the dataset.++ Raises:+ RuntimeError: If the items fail to be retrieved.+ """+ try:+ items = self.get_dataset_client(dataset_id).list_items().items+ return items+ except Exception as e:+ raise RuntimeError(+ f"Failed to get dataset items {dataset_id}: {e}"+ ) from e++ def get_datasets(+ self,+ unnamed: Optional[bool] = None,+ limit: Optional[int] = None,+ offset: Optional[int] = None,+ desc: Optional[bool] = None,+ ) -> List[dict]:+ r"""Get all named datasets from the Apify platform.++ Args:+ unnamed (bool, optional): Whether to include unnamed key-value+ stores in the list+ limit (int, optional): How many key-value stores to retrieve+ offset (int, optional): What key-value store to include as first+ when retrieving the list+ desc (bool, optional): Whether to sort the key-value stores in+ descending order based on their modification date++ Returns:+ List[dict]: The datasets.++ Raises:+ RuntimeError: If the datasets fail to be retrieved.+ """+ try:+ return (+ self.client.datasets()+ .list(unnamed=unnamed, limit=limit, offset=offset, desc=desc)+ .items+ )+ except Exception as e:+ raise RuntimeError(f"Failed to get datasets: {e}") from ediff --git a/examples/loaders/apify_example.py b/examples/loaders/apify_example.pynew file mode 100644index 000000000..834c495ee--- /dev/null+++ b/examples/loaders/apify_example.py@@ -0,0 +1,82 @@+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========+# Licensed under the Apache License, Version 2.0 (the āLicenseā);+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an āAS ISā BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========++from camel.loaders import Apify++apify = Apify()++run_input = {+ "startUrls": [{"url": "https://www.camel-ai.org/"}],+ "maxCrawlDepth": 0,+ "maxCrawlPages": 1,+}+actor_result = apify.run_actor(+ actor_id="apify/website-content-crawler", run_input=run_input+)++print(actor_result["status"])+'''+===============================================================================+SUCCEEDED+===============================================================================+'''++print(actor_result["defaultDatasetId"])+'''+===============================================================================+HoKb31FJzHm0ni51k+===============================================================================+'''++dataset_result = apify.get_dataset_items(+ dataset_id=actor_result["defaultDatasetId"]+)++print(dataset_result)+'''+===============================================================================+[{'url': 'https://www.camel-ai.org/', 'crawl': {'loadedUrl': 'https://www.camel+-ai.org/', 'loadedTime': '2024-10-27T04:51:16.651Z', 'referrerUrl': 'https://ww+w.camel-ai.org/', 'depth': 0, 'httpStatusCode': 200}, 'metadata': {'canonicalUr+l': 'https://www.camel-ai.org/', 'title': 'CAMEL-AI', 'description': 'CAMEL-AI.+org is the 1st LLM multi-agent framework and an open-source community dedicated+ to finding the scaling law of agents.', 'author': None, 'keywords': None, 'lan+ guageCode': 'en', 'openGraph': [{'property': 'og:title', 'content': 'CAMEL-AI'+ }, {'property': 'og:description', 'content': 'CAMEL-AI.org is the 1st LLM mult+ i-agent framework and an open-source community dedicated to finding the scalin+ g law of agents.'}, {'property': 'twitter:title', 'content': 'CAMEL-AI'}, {'pr+ operty': 'twitter:description', 'content': 'CAMEL-AI.org is the 1st LLM multi-+ agent framework and an open-source community dedicated to finding the scaling+ law of agents.'}, {'property': 'og:type', 'content': 'website'}], 'jsonLd': No+ ne, 'headers': {'date': 'Sun, 27 Oct 2024 04:50:18 GMT', 'content-type': 'text+ /html', 'cf-ray': '8d901082dae7efbe-PDX', 'cf-cache-status': 'HIT', 'age': '10+ 81', 'content-encoding': 'gzip', 'last-modified': 'Sat, 26 Oct 2024 11:51:32 G+ MT', 'strict-transport-security': 'max-age=31536000', 'surrogate-control': 'ma+ x-age=432000', 'surrogate-key': 'www.camel-ai.org 6659a154491a54a40551bc78 pag+ eId:6686a2bcb7ece5fb40457491 668181a0a818ade34e653b24 6659a155491a54a40551bd7e+ ', 'x-lambda-id': 'd6c4424b-ac67-4c54-b52a-cb2a22ca09f0', 'vary': 'Accept-Enco+ ding', 'set-cookie': '__cf_bm=oX5EmZ2SNJDOBQXI8dL_reCYlCpp1FMzu40qCNxiopU-1730+ 004618-1.0.1.1-3teEeqUoemzHWAeGCtlPJVqdmAbiFkyu3JxopKfQFFndSCi_Z56RR.UDjLGZiq.+ L_4LvAZYmNKxQ.k6VRhbA7g; path=/; expires=Sun, 27-Oct-24 05:20:18 GMT; domain=.+ cdn.webflow.com; HttpOnly; Secure; SameSite=None\n_cfuvid=om_8lj9jNMIh.HEIxEAq+ gszhEWaKlyS2kdXKwqGedSM-1730004618924-0.0.1.1-604800000; path=/; domain=.cdn.w+ ebflow.com; HttpOnly; Secure; SameSite=None', 'alt-svc': 'h3=":443"; ma=86400'+ , 'x-cluster-name': 'us-west-2-prod-hosting-red', 'x-firefox-spdy': 'h2'}}, 's+ creenshotUrl': None, 'text': 'Build Multi-Agent Systems for _\nFEATURES & Inte+ grations\nSeamless integrations with\npopular platforms \nScroll to explore ou+ r features & integrations.', 'markdown': '# Build Multi-Agent Systems for \\_+ \n\nFEATURES & Integrations\n\n## Seamless integrations with \npopular platfo+ rms\n\nScroll to explore our features & integrations.'}]+===============================================================================+'''diff --git a/poetry.lock b/poetry.lockindex cdeb1cf23..a6fb8a1e8 100644--- a/poetry.lock+++ b/poetry.lock@@ -1,4 +1,4 @@-# This file is automatically @generated by Poetry 1.8.4 and should not be changed by hand.+# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand.[[package]]name = "accelerate"@@ -321,6 +321,36 @@ doc = ["Sphinx (>=7.4,<8.0)", "packaging", "sphinx-autodoc-typehints (>=1.2.0)",test = ["anyio[trio]", "coverage[toml] (>=7)", "exceptiongroup (>=1.2.0)", "hypothesis (>=4.0)", "psutil (>=5.9)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "truststore (>=0.9.1)", "uvloop (>=0.21.0b1)"]trio = ["trio (>=0.26.1)"]+[[package]]+name = "apify-client"+version = "1.8.1"+description = "Apify API client for Python"+optional = true+python-versions = "<4.0,>=3.9"+files = [+ {file = "apify_client-1.8.1-py3-none-any.whl", hash = "sha256:cfa6df3816c436204e37457fba28981a0ef6a7602cde372463f0f078eee64747"},+ {file = "apify_client-1.8.1.tar.gz", hash = "sha256:2be1be7879570655bddeebf126833efe94cabb95b3755592845e92c20c70c674"},+]++[package.dependencies]+apify-shared = ">=1.1.2"+httpx = ">=0.25.0"+more_itertools = ">=10.0.0"++[[package]]+name = "apify-shared"+version = "1.1.2"+description = "Tools and constants shared across Apify projects."+optional = true+python-versions = ">=3.8"+files = [+ {file = "apify_shared-1.1.2-py3-none-any.whl", hash = "sha256:a422b4cb3ea536257f0c0dd3778b3c22ec6647708d2fc06c97fe0c9fe183c3c9"},+ {file = "apify_shared-1.1.2.tar.gz", hash = "sha256:3f98f526635c4c3f92c1496916513a1c4013bb47158d7e0f914df8987147366c"},+]++[package.extras]+dev = ["build (>=1.0.3,<1.1.0)", "filelock (>=3.12.4,<3.13.0)", "mypy (>=1.7.1,<1.8.0)", "pre-commit (>=3.4.0,<3.5.0)", "pydoc-markdown (>=4.8.2,<4.9.0)", "pytest (>=7.4.2,<7.5.0)", "pytest-asyncio (>=0.21.0,<0.22.0)", "pytest-cov (>=4.1.0,<4.2.0)", "pytest-only (>=2.0.0,<2.1.0)", "pytest-timeout (>=2.2.0,<2.3.0)", "pytest-xdist (>=3.3.1,<3.4.0)", "respx (>=0.20.1,<0.21.0)", "ruff (>=0.1.13,<0.2.0)", "setuptools (>=68.0.0)", "twine (>=5.1.1,<5.2.0)"]+[[package]]name = "appnope"version = "0.1.4"@@ -650,13 +680,13 @@ css = ["tinycss2 (>=1.1.0,<1.3)"][[package]]name = "botocore"-version = "1.35.48"+version = "1.35.49"description = "Low-level, data-driven core of boto 3."optional = truepython-versions = ">=3.8"files = [- {file = "botocore-1.35.48-py3-none-any.whl", hash = "sha256:34fa25fd717208b05745e60f271a39636108fa87a3512fbca18e7e6f787a3239"},- {file = "botocore-1.35.48.tar.gz", hash = "sha256:3e766cc251053c9ef98542fdf225381ed58531769c3811a6282bd7247f7e2bdf"},+ {file = "botocore-1.35.49-py3-none-any.whl", hash = "sha256:aed4d3643afd702920792b68fbe712a8c3847993820d1048cd238a6469354da1"},+ {file = "botocore-1.35.49.tar.gz", hash = "sha256:07d0c1325fdbfa49a4a054413dbdeab0a6030449b2aa66099241af2dac48afd8"},][package.dependencies]@@ -1593,13 +1623,13 @@ tests = ["dj-database-url", "dj-email-url", "django-cache-url", "pytest"][[package]]name = "et-xmlfile"-version = "1.1.0"+version = "2.0.0"description = "An implementation of lxml.xmlfile for the standard library"optional = true-python-versions = ">=3.6"+python-versions = ">=3.8"files = [- {file = "et_xmlfile-1.1.0-py3-none-any.whl", hash = "sha256:a2ba85d1d6a74ef63837eed693bcb89c3f752169b0e3e7ae5b16ca5e1b3deada"},- {file = "et_xmlfile-1.1.0.tar.gz", hash = "sha256:8eb9e2bc2f8c97e37a2dc85a09ecdcdec9d8a396530a6d5a33b30b9a92da0c5c"},+ {file = "et_xmlfile-2.0.0-py3-none-any.whl", hash = "sha256:7a91720bc756843502c3b7504c77b8fe44217c85c537d85037f0f536151b2caa"},+ {file = "et_xmlfile-2.0.0.tar.gz", hash = "sha256:dab3f4764309081ce75662649be815c4c9081e88f0837825f90fd28317d4da54"},][[package]]@@ -3448,13 +3478,13 @@ files = [[[package]]name = "litellm"-version = "1.50.4"+version = "1.51.0"description = "Library to easily interface with LLM API providers"optional = truepython-versions = "!=2.7.*,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,!=3.7.*,>=3.8"files = [- {file = "litellm-1.50.4-py3-none-any.whl", hash = "sha256:cc6992275e24a0bbb4a3b377e6842d45a8510fc85d7f255930a64bb872980a36"},- {file = "litellm-1.50.4.tar.gz", hash = "sha256:a7e68ef614f631b58969c2c7a5154a565ba5974558d437c8cd6c8623654880ea"},+ {file = "litellm-1.51.0-py3-none-any.whl", hash = "sha256:0b2c20d116834166c8440e5698d7d927dbcc78fcaa08ce0c5cbea2d0de55ec6c"},+ {file = "litellm-1.51.0.tar.gz", hash = "sha256:8bf648677ee145a8fe5054a2e3f3a34895b9ab65a6015e4b94efca7ef406f466"},][package.dependencies]@@ -3871,6 +3901,7 @@ python-versions = ">=3.7"files = [{file = "milvus_lite-2.4.10-py3-none-macosx_10_9_x86_64.whl", hash = "sha256:fc4246d3ed7d1910847afce0c9ba18212e93a6e9b8406048436940578dfad5cb"},{file = "milvus_lite-2.4.10-py3-none-macosx_11_0_arm64.whl", hash = "sha256:74a8e07c5e3b057df17fbb46913388e84df1dc403a200f4e423799a58184c800"},+ {file = "milvus_lite-2.4.10-py3-none-manylinux2014_aarch64.whl", hash = "sha256:240c7386b747bad696ecb5bd1f58d491e86b9d4b92dccee3315ed7256256eddc"},{file = "milvus_lite-2.4.10-py3-none-manylinux2014_x86_64.whl", hash = "sha256:211d2e334a043f9282bdd9755f76b9b2d93b23bffa7af240919ffce6a8dfe325"},]@@ -5496,17 +5527,17 @@ ssv = ["swagger-spec-validator (>=2.4,<3.0)"][[package]]name = "praw"-version = "7.8.0"+version = "7.8.1"description = "Python Reddit API Wrapper."optional = truepython-versions = "~=3.8"files = [- {file = "praw-7.8.0-py3-none-any.whl", hash = "sha256:66bb2d9dc4f9a35910bf8b4339067c8646ab03270b1c38d77b0487bef8c5c3e2"},- {file = "praw-7.8.0.tar.gz", hash = "sha256:910a1478d79ce3892bf35b2c52c90f575d6618dfce248e7bf90eed354c19589c"},+ {file = "praw-7.8.1-py3-none-any.whl", hash = "sha256:15917a81a06e20ff0aaaf1358481f4588449fa2421233040cb25e5c8202a3e2f"},+ {file = "praw-7.8.1.tar.gz", hash = "sha256:3c5767909f71e48853eb6335fef7b50a43cbe3da728cdfb16d3be92904b0a4d8"},][package.dependencies]-prawcore = ">=2.1,<3"+prawcore = ">=2.4,<3"update_checker = ">=0.18"websocket-client = ">=0.54.0"@@ -5844,6 +5875,7 @@ description = "Pure-Python implementation of ASN.1 types and DER/BER/CER codecsoptional = truepython-versions = ">=3.8"files = [+ {file = "pyasn1-0.6.1-py3-none-any.whl", hash = "sha256:0d632f46f2ba09143da3a8afe9e33fb6f92fa2320ab7e886e2d0f7672af84629"},{file = "pyasn1-0.6.1.tar.gz", hash = "sha256:6f580d2bdd84365380830acf45550f2511469f673cb4a5ae3857a3170128b034"},]@@ -5854,6 +5886,7 @@ description = "A collection of ASN.1-based protocols modules"optional = truepython-versions = ">=3.8"files = [+ {file = "pyasn1_modules-0.4.1-py3-none-any.whl", hash = "sha256:49bfa96b45a292b711e986f222502c1c9a5e1f4e568fc30e2574a6c7d07838fd"},{file = "pyasn1_modules-0.4.1.tar.gz", hash = "sha256:c28e2dbf9c06ad61c71a075c7e0f9fd0f1b0bb2d2ad4377f240d33ac2ab60a7c"},]@@ -8585,6 +8618,10 @@ description = "image and video datasets and models for torch deep learning"optional = truepython-versions = ">=3.8"files = [+ {file = "torchvision-0.20.0-1-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:e084f50ecbdbe7a9cc2fc51ea0367ae35fde46e84a964bf4046cb1c7feb7e3e6"},+ {file = "torchvision-0.20.0-1-cp311-cp311-manylinux2014_aarch64.whl", hash = "sha256:55d7f43ef912ebc4da4bba73a0bbf387d38a6be9cd521679c0f4056f9564b698"},+ {file = "torchvision-0.20.0-1-cp312-cp312-manylinux2014_aarch64.whl", hash = "sha256:f8d0213489acfb138369f2455a6893880c194a8195e381c19f872b277f2654c3"},+ {file = "torchvision-0.20.0-1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:8d6cea8ab0bf72ecb71b07cd0fe836eacf5a5fa98f6629d2261212e90977b963"},{file = "torchvision-0.20.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f164d545965186ffd66014e34a966706d12c84198302dd46748cae45984609a4"},{file = "torchvision-0.20.0-cp310-cp310-manylinux1_x86_64.whl", hash = "sha256:9c18208575d60b96e7d53a09c453781afea4a81487c9ebc501dfc2bc88daa308"},{file = "torchvision-0.20.0-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:09080359be90314fc4fdd64b11a4d231c1999018f19d58bf7764f5e15f8e9fb3"},@@ -8597,10 +8634,6 @@ files = [{file = "torchvision-0.20.0-cp312-cp312-manylinux1_x86_64.whl", hash = "sha256:c8f3bc399d9c3e4ba05d74ca6dd5e63fed08ad5c5b302a946c8fcaa56216220f"},{file = "torchvision-0.20.0-cp312-cp312-manylinux2014_aarch64.whl", hash = "sha256:a78c99ebe1a62857b68e97ff9417b92f299f2ee61f009491a114ddad050c493d"},{file = "torchvision-0.20.0-cp312-cp312-win_amd64.whl", hash = "sha256:bb0da0950d2034a0412c251a3a9117ff9612157f45177d37ba1b20b472c0864b"},- {file = "torchvision-0.20.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:eac0839db36eb812362f06359a33e8c7101f161c1393e2971b8402aefefa5b58"},- {file = "torchvision-0.20.0-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:bbb6c3287e15ab710139d63912b6f1bca6a34a6912cdb5c98fdb850b260ec20b"},- {file = "torchvision-0.20.0-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:cd20f972d594a7900b57724e168ee8636e3cb255f54f6a9d8409e3e218b93721"},- {file = "torchvision-0.20.0-cp38-cp38-win_amd64.whl", hash = "sha256:1dd590a48d0a68a3be598d0d0a41e72872dd317418753aa0c97335d50558ba8a"},{file = "torchvision-0.20.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:6a70c81ea5068dd7b1e340ebeabb65364576d8b9819454cfdf812290cf03e45a"},{file = "torchvision-0.20.0-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:95d8c817681a4c2156f66ef83cafc4c5c4b97e4694956d54d7dc554804ee510d"},{file = "torchvision-0.20.0-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:1ab53244701eab897e5c65026ba178c0abbc5bd08629c3d20f737d618e9e5a37"},@@ -9841,7 +9874,7 @@ test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools",type = ["pytest-mypy"][extras]-all = ["PyMuPDF", "accelerate", "agentops", "anthropic", "arxiv", "arxiv2text", "asknews", "azure-storage-blob", "beautifulsoup4", "botocore", "cohere", "datasets", "diffusers", "discord.py", "docker", "docx2txt", "duckduckgo-search", "firecrawl-py", "google-cloud-storage", "google-generativeai", "googlemaps", "imageio", "ipykernel", "jupyter_client", "litellm", "mistralai", "nebula3-python", "neo4j", "newspaper3k", "nltk", "notion-client", "openapi-spec-validator", "opencv-python", "pdfplumber", "pillow", "prance", "praw", "pyTelegramBotAPI", "pydub", "pygithub", "pymilvus", "pyowm", "qdrant-client", "rank-bm25", "redis", "reka-api", "requests_oauthlib", "scholarly", "sentence-transformers", "sentencepiece", "slack-bolt", "slack-sdk", "soundfile", "textblob", "torch", "torch", "transformers", "unstructured", "wikipedia", "wolframalpha"]+all = ["PyMuPDF", "accelerate", "agentops", "anthropic", "apify_client", "arxiv", "arxiv2text", "asknews", "azure-storage-blob", "beautifulsoup4", "botocore", "cohere", "datasets", "diffusers", "discord.py", "docker", "docx2txt", "duckduckgo-search", "firecrawl-py", "google-cloud-storage", "google-generativeai", "googlemaps", "imageio", "ipykernel", "jupyter_client", "litellm", "mistralai", "nebula3-python", "neo4j", "newspaper3k", "nltk", "notion-client", "openapi-spec-validator", "opencv-python", "pdfplumber", "pillow", "prance", "praw", "pyTelegramBotAPI", "pydub", "pygithub", "pymilvus", "pyowm", "qdrant-client", "rank-bm25", "redis", "reka-api", "requests_oauthlib", "scholarly", "sentence-transformers", "sentencepiece", "slack-bolt", "slack-sdk", "soundfile", "textblob", "torch", "torch", "transformers", "unstructured", "wikipedia", "wolframalpha"]encoders = ["sentence-transformers"]graph-storages = ["nebula3-python", "neo4j"]huggingface-agent = ["accelerate", "datasets", "diffusers", "opencv-python", "sentencepiece", "soundfile", "torch", "torch", "transformers"]@@ -9852,10 +9885,10 @@ rag = ["cohere", "nebula3-python", "neo4j", "pymilvus", "qdrant-client", "rank-bretrievers = ["cohere", "rank-bm25"]search-tools = ["duckduckgo-search", "wikipedia", "wolframalpha"]test = ["mock", "pytest", "pytest-asyncio"]-tools = ["PyMuPDF", "agentops", "arxiv", "arxiv2text", "asknews", "beautifulsoup4", "discord.py", "docker", "docx2txt", "duckduckgo-search", "firecrawl-py", "googlemaps", "imageio", "ipykernel", "jupyter_client", "newspaper3k", "nltk", "notion-client", "openapi-spec-validator", "pdfplumber", "pillow", "prance", "praw", "pyTelegramBotAPI", "pydub", "pygithub", "pyowm", "requests_oauthlib", "scholarly", "slack-bolt", "slack-sdk", "textblob", "unstructured", "wikipedia", "wolframalpha"]+tools = ["PyMuPDF", "agentops", "apify_client", "arxiv", "arxiv2text", "asknews", "beautifulsoup4", "discord.py", "docker", "docx2txt", "duckduckgo-search", "firecrawl-py", "googlemaps", "imageio", "ipykernel", "jupyter_client", "newspaper3k", "nltk", "notion-client", "openapi-spec-validator", "pdfplumber", "pillow", "prance", "praw", "pyTelegramBotAPI", "pydub", "pygithub", "pyowm", "requests_oauthlib", "scholarly", "slack-bolt", "slack-sdk", "textblob", "unstructured", "wikipedia", "wolframalpha"]vector-databases = ["pymilvus", "qdrant-client"][metadata]lock-version = "2.0"python-versions = ">=3.10,<3.13"-content-hash = "b4cec55ebd8227386e1658ef0d0c204cd171c20b6907e4d5973fb180b210e55e"+content-hash = "4e31fcb53ef5ee0000d3182536cda5930d54f6287c240889b149cab73800a6de"diff --git a/pyproject.toml b/pyproject.tomlindex 4f5ac3f13..fd55c47a1 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -100,6 +100,7 @@ arxiv2text = { version = "^0.1.14", optional = true }jupyter_client = { version = "^8.6.2", optional = true }ipykernel = { version = "^6.0.0", optional = true }pdfplumber = { version = "^0.11.0", optional = true }+apify_client = { version = "^1.8.1", optional = true }notion-client = { version = "^2.2.1", optional = true }@@ -204,6 +205,7 @@ tools = ["textblob","scholarly","pdfplumber",+ "apify_client","notion-client",]@@ -302,6 +304,7 @@ all = ["azure-storage-blob","google-cloud-storage","botocore",+ "apify_client",][tool.poetry.group.dev]diff --git a/test/loaders/test_apify.py b/test/loaders/test_apify.pynew file mode 100644index 000000000..280117adf--- /dev/null+++ b/test/loaders/test_apify.py@@ -0,0 +1,126 @@+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========+# Licensed under the Apache License, Version 2.0 (the āLicenseā);+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an āAS ISā BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========+import os+from unittest.mock import Mock, patch++import pytest++from camel.loaders.apify_reader import Apify+++def apify_client_mock():+ with patch('apify_client.ApifyClient', autospec=True) as mock:+ yield mock++[email protected](autouse=True)+def set_env_vars():+ with patch.dict(os.environ, {"APIFY_API_KEY": "test_api_key"}):+ yield+++def test_init(apify_client_mock):+ api_key = "test_api_key"+ apify = Apify(api_key=api_key)+ assert apify._api_key == api_key+ apify_client_mock.assert_called_once_with(token=api_key)+++def test_run_actor():+ api_key = "test_api_key"+ apify = Apify(api_key=api_key)+ actor_id = "test_actor_id"+ run_input = {"key": "value"}+ mock_actor = Mock()+ mock_actor.call.return_value = {"result": "success"}+ apify.client.actor = Mock(return_value=mock_actor)+ result = apify.run_actor(actor_id, run_input=run_input)+ apify.client.actor.assert_called_once_with(actor_id)+ apify.client.actor.return_value.call.assert_called_once_with(+ run_input=run_input,+ content_type=None,+ build=None,+ max_items=None,+ memory_mbytes=None,+ timeout_secs=None,+ webhooks=None,+ wait_secs=None,+ )+ assert result == {"result": "success"}+++def test_get_dataset_client():+ api_key = "test_api_key"+ apify = Apify(api_key=api_key)+ dataset_id = "test_dataset_id"+ mock_dataset = Mock()+ apify.client.dataset = Mock(return_value=mock_dataset)+ result = apify.get_dataset_client(dataset_id)+ apify.client.dataset.assert_called_once_with(dataset_id)+ assert result == mock_dataset+++def test_get_dataset():+ api_key = "test_api_key"+ apify = Apify(api_key=api_key)+ dataset_id = "test_dataset_id"+ mock_client = Mock()+ mock_client.get.return_value = {"data": "dataset"}+ apify.get_dataset_client = lambda x: mock_client++ result = apify.get_dataset(dataset_id)+ assert result == {"data": "dataset"}+++def test_update_dataset():+ api_key = "test_api_key"+ apify = Apify(api_key=api_key)+ dataset_id = "test_dataset_id"+ name = "new_name"+ mock_client = Mock()+ mock_client.update.return_value = {"name": name}+ apify.get_dataset_client = lambda x: mock_client++ result = apify.update_dataset(dataset_id, name)+ assert result == {"name": name}+++def test_get_dataset_items():+ api_key = "test_api_key"+ apify = Apify(api_key=api_key)+ dataset_id = "test_dataset_id"+ mock_client = Mock()+ mock_client.list_items.return_value.items = ["item1", "item2"]+ apify.get_dataset_client = lambda x: mock_client++ result = apify.get_dataset_items(dataset_id)+ assert result == ["item1", "item2"]+++def test_get_datasets():+ api_key = "test_api_key"+ apify = Apify(api_key=api_key)+ mock_client = Mock()+ mock_client.datasets.return_value.list.return_value.items = [+ {"name": "dataset1"},+ {"name": "dataset2"},+ ]+ apify.client = mock_client++ result = apify.get_datasets()+ apify.client.datasets.return_value.list.assert_called_once_with(+ unnamed=None, limit=None, offset=None, desc=None+ )+ assert result == [{"name": "dataset1"}, {"name": "dataset2"}]